The following changes since commit 0ab4537f08e09b13788db67efd760592fb7db769:

Merge remote-tracking branch 'remotes/stefanberger/tags/pull-tpm-2018-03-07-1' into staging (2018-03-08 12:56:39 +0000)

git://github.com/stefanha/qemu.git tags/block-pull-request

for you to fetch changes up to 4486e89c219c0d1b9bd8dfa0b1dd5b0d51ff2268:

vl: introduce vm_shutdown() (2018-03-08 17:38:51 +0000)

----------------------------------------------------------------

Deepa Srinivasan (1):

block: Fix qemu crash when using scsi-block

Fam Zheng (1):

README: Fix typo 'git-publish'

Sergio Lopez (1):

virtio-blk: dataplane: Don't batch notifications if EVENT_IDX is

present

Stefan Hajnoczi (4):

block: add aio_wait_bh_oneshot()

virtio-blk: fix race between .ioeventfd_stop() and vq handler

virtio-scsi: fix race between .ioeventfd_stop() and vq handler

vl: introduce vm_shutdown()

include/block/aio-wait.h | 13 +++++++++++

include/sysemu/iothread.h | 1 -

include/sysemu/sysemu.h | 1 +

block/block-backend.c | 51 ++++++++++++++++++++---------------------

cpus.c | 16 ++++++++++---

hw/block/dataplane/virtio-blk.c | 39 +++++++++++++++++++++++--------

hw/scsi/virtio-scsi-dataplane.c | 9 ++++----

iothread.c | 31 -------------------------

util/aio-wait.c | 31 +++++++++++++++++++++++++

vl.c | 13 +++--------

README | 2 +-

11 files changed, 122 insertions(+), 85 deletions(-)

2.14.3

If the main loop thread invokes .ioeventfd_stop() just as the vq handler

function begins in the IOThread then the handler may lose the race for

the AioContext lock. By the time the vq handler is able to acquire the

AioContext lock the ioeventfd has already been removed and the handler

isn't supposed to run anymore!

Use the new aio_wait_bh_oneshot() function to perform ioeventfd removal

from within the IOThread. This way no races with the vq handler are

possible.

Reviewed-by: Fam Zheng <famz@redhat.com>

Acked-by: Paolo Bonzini <pbonzini@redhat.com>

Message-id: 20180307144205.20619-4-stefanha@redhat.com

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

hw/scsi/virtio-scsi-dataplane.c | 9 +++++----

1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/hw/scsi/virtio-scsi-dataplane.c b/hw/scsi/virtio-scsi-dataplane.c

--- a/hw/scsi/virtio-scsi-dataplane.c

+++ b/hw/scsi/virtio-scsi-dataplane.c

@@ -XXX,XX +XXX,XX @@ static int virtio_scsi_vring_init(VirtIOSCSI *s, VirtQueue *vq, int n,

return 0;

-/* assumes s->ctx held */

-static void virtio_scsi_clear_aio(VirtIOSCSI *s)

+/* Context: BH in IOThread */

+static void virtio_scsi_dataplane_stop_bh(void *opaque)

+ VirtIOSCSI *s = opaque;

VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);

int i;

@@ -XXX,XX +XXX,XX @@ int virtio_scsi_dataplane_start(VirtIODevice *vdev)

return 0;

fail_vrings:

- virtio_scsi_clear_aio(s);

+ aio_wait_bh_oneshot(s->ctx, virtio_scsi_dataplane_stop_bh, s);

aio_context_release(s->ctx);

for (i = 0; i < vs->conf.num_queues + 2; i++) {

virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false);

@@ -XXX,XX +XXX,XX @@ void virtio_scsi_dataplane_stop(VirtIODevice *vdev)

s->dataplane_stopping = true;

aio_context_acquire(s->ctx);

- virtio_scsi_clear_aio(s);

+ aio_wait_bh_oneshot(s->ctx, virtio_scsi_dataplane_stop_bh, s);

aio_context_release(s->ctx);

blk_drain_all(); /* ensure there are no in-flight requests */

2.14.3

Sometimes it's necessary for the main loop thread to run a BH in an

IOThread and wait for its completion. This primitive is useful during

startup/shutdown to synchronize and avoid race conditions.

Reviewed-by: Fam Zheng <famz@redhat.com>

Acked-by: Paolo Bonzini <pbonzini@redhat.com>

Message-id: 20180307144205.20619-2-stefanha@redhat.com

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

include/block/aio-wait.h | 13 +++++++++++++

util/aio-wait.c | 31 +++++++++++++++++++++++++++++++

2 files changed, 44 insertions(+)

diff --git a/include/block/aio-wait.h b/include/block/aio-wait.h

--- a/include/block/aio-wait.h

+++ b/include/block/aio-wait.h

@@ -XXX,XX +XXX,XX @@ typedef struct {

*/

void aio_wait_kick(AioWait *wait);

+/**

+ * aio_wait_bh_oneshot:

+ * @ctx: the aio context

+ * @cb: the BH callback function

+ * @opaque: user data for the BH callback function

+ *

+ * Run a BH in @ctx and wait for it to complete.

+ *

+ * Must be called from the main loop thread with @ctx acquired exactly once.

+ * Note that main loop event processing may occur.

+ */

+void aio_wait_bh_oneshot(AioContext *ctx, QEMUBHFunc *cb, void *opaque);

+

#endif /* QEMU_AIO_WAIT */

diff --git a/util/aio-wait.c b/util/aio-wait.c

index XXXXXXX..XXXXXXX 100644

--- a/util/aio-wait.c

+++ b/util/aio-wait.c

@@ -XXX,XX +XXX,XX @@ void aio_wait_kick(AioWait *wait)

aio_bh_schedule_oneshot(qemu_get_aio_context(), dummy_bh_cb, NULL);

2.14.3

Commit 00d09fdbbae5f7864ce754913efc84c12fdf9f1a ("vl: pause vcpus before

stopping iothreads") and commit dce8921b2baaf95974af8176406881872067adfa

("iothread: Stop threads before main() quits") tried to work around the

fact that emulation was still active during termination by stopping

iothreads. They suffer from race conditions:

1. virtio_scsi_handle_cmd_vq() racing with iothread_stop_all() hits the

virtio_scsi_ctx_check() assertion failure because the BDS AioContext

has been modified by iothread_stop_all().

2. Guest vq kick racing with main loop termination leaves a readable

ioeventfd that is handled by the next aio_poll() when external

clients are enabled again, resulting in unwanted emulation activity.

This patch obsoletes those commits by fully disabling emulation activity

when vcpus are stopped.

Use the new vm_shutdown() function instead of pause_all_vcpus() so that

vm change state handlers are invoked too. Virtio devices will now stop

their ioeventfds, preventing further emulation activity after vm_stop().

Note that vm_stop(RUN_STATE_SHUTDOWN) cannot be used because it emits a

QMP STOP event that may affect existing clients.

It is no longer necessary to call replay_disable_events() directly since

vm_shutdown() does so already.

Drop iothread_stop_all() since it is no longer used.

Cc: Fam Zheng <famz@redhat.com>

Cc: Kevin Wolf <kwolf@redhat.com>

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

Reviewed-by: Fam Zheng <famz@redhat.com>

Acked-by: Paolo Bonzini <pbonzini@redhat.com>

Message-id: 20180307144205.20619-5-stefanha@redhat.com

include/sysemu/iothread.h | 1 -

include/sysemu/sysemu.h | 1 +

cpus.c | 16 +++++++++++++---

iothread.c | 31 -------------------------------

vl.c | 13 +++----------

5 files changed, 17 insertions(+), 45 deletions(-)

diff --git a/include/sysemu/iothread.h b/include/sysemu/iothread.h

--- a/include/sysemu/iothread.h

+++ b/include/sysemu/iothread.h

@@ -XXX,XX +XXX,XX @@ typedef struct {

char *iothread_get_id(IOThread *iothread);

IOThread *iothread_by_id(const char *id);

AioContext *iothread_get_aio_context(IOThread *iothread);

-void iothread_stop_all(void);

GMainContext *iothread_get_g_main_context(IOThread *iothread);

/*

diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h

index XXXXXXX..XXXXXXX 100644

--- a/include/sysemu/sysemu.h

+++ b/include/sysemu/sysemu.h

@@ -XXX,XX +XXX,XX @@ void vm_start(void);

int vm_prepare_start(void);

int vm_stop(RunState state);

int vm_stop_force_state(RunState state);

+int vm_shutdown(void);

typedef enum WakeupReason {

/* Always keep QEMU_WAKEUP_REASON_NONE = 0 */

diff --git a/cpus.c b/cpus.c

index XXXXXXX..XXXXXXX 100644

--- a/cpus.c

+++ b/cpus.c

@@ -XXX,XX +XXX,XX @@ void cpu_synchronize_all_pre_loadvm(void)

-static int do_vm_stop(RunState state)

+static int do_vm_stop(RunState state, bool send_stop)

int ret = 0;

@@ -XXX,XX +XXX,XX @@ static int do_vm_stop(RunState state)

pause_all_vcpus();

runstate_set(state);

vm_state_notify(0, state);

- qapi_event_send_stop(&error_abort);

+ if (send_stop) {

+ qapi_event_send_stop(&error_abort);

bdrv_drain_all();

@@ -XXX,XX +XXX,XX @@ static int do_vm_stop(RunState state)

return ret;

}

+/* Special vm_stop() variant for terminating the process. Historically clients

+ * did not expect a QMP STOP event and so we need to retain compatibility.

+ */

+int vm_shutdown(void)

+{

+ return do_vm_stop(RUN_STATE_SHUTDOWN, false);

+}

+

static bool cpu_can_run(CPUState *cpu)

{

if (cpu->stop) {

@@ -XXX,XX +XXX,XX @@ int vm_stop(RunState state)

return 0;

2.14.3

If the main loop thread invokes .ioeventfd_stop() just as the vq handler

function begins in the IOThread then the handler may lose the race for

the AioContext lock. By the time the vq handler is able to acquire the

AioContext lock the ioeventfd has already been removed and the handler

isn't supposed to run anymore!

Use the new aio_wait_bh_oneshot() function to perform ioeventfd removal

from within the IOThread. This way no races with the vq handler are

possible.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

Reviewed-by: Fam Zheng <famz@redhat.com>

Acked-by: Paolo Bonzini <pbonzini@redhat.com>

Message-id: 20180307144205.20619-3-stefanha@redhat.com

hw/block/dataplane/virtio-blk.c | 24 +++++++++++++++++-------

1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c

--- a/hw/block/dataplane/virtio-blk.c

+++ b/hw/block/dataplane/virtio-blk.c

@@ -XXX,XX +XXX,XX @@ int virtio_blk_data_plane_start(VirtIODevice *vdev)

return -ENOSYS;

+/* Stop notifications for new requests from guest.

+ *

+ * Context: BH in IOThread

+ */

+static void virtio_blk_data_plane_stop_bh(void *opaque)

+{

+ VirtIOBlockDataPlane *s = opaque;

+ unsigned i;

+ for (i = 0; i < s->conf->num_queues; i++) {

+ VirtQueue *vq = virtio_get_queue(s->vdev, i);

+ virtio_queue_aio_set_host_notifier_handler(vq, s->ctx, NULL);

+}

+

/* Context: QEMU global mutex held */

void virtio_blk_data_plane_stop(VirtIODevice *vdev)

{

@@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_stop(VirtIODevice *vdev)

trace_virtio_blk_data_plane_stop(s);

aio_context_acquire(s->ctx);

-

- /* Stop notifications for new requests from guest */

- for (i = 0; i < nvqs; i++) {

- VirtQueue *vq = virtio_get_queue(s->vdev, i);

-

- virtio_queue_aio_set_host_notifier_handler(vq, s->ctx, NULL);

- }

+ aio_wait_bh_oneshot(s->ctx, virtio_blk_data_plane_stop_bh, s);

/* Drain and switch bs back to the QEMU main loop */

blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context());

2.14.3

From: Sergio Lopez <slp@redhat.com>

Commit 5b2ffbe4d99843fd8305c573a100047a8c962327 ("virtio-blk: dataplane:

notify guest as a batch") deferred guest notification to a BH in order

batch notifications, with purpose of avoiding flooding the guest with

interruptions.

This optimization came with a cost. The average latency perceived in the

guest is increased by a few microseconds, but also when multiple IO

operations finish at the same time, the guest won't be notified until

all completions from each operation has been run. On the contrary,

virtio-scsi issues the notification at the end of each completion.

On the other hand, nowadays we have the EVENT_IDX feature that allows a

better coordination between QEMU and the Guest OS to avoid sending

unnecessary interruptions.

With this change, virtio-blk/dataplane only batches notifications if the

EVENT_IDX feature is not present.

Some numbers obtained with fio (ioengine=sync, iodepth=1, direct=1):

- Test specs:

* fio-3.4 (ioengine=sync, iodepth=1, direct=1)

* qemu master

* virtio-blk with a dedicated iothread (default poll-max-ns)

* backend: null_blk nr_devices=1 irqmode=2 completion_nsec=280000

* 8 vCPUs pinned to isolated physical cores

* Emulator and iothread also pinned to separate isolated cores

* variance between runs < 1%

- Not patched

* numjobs=1: lat_avg=327.32 irqs=29998

* numjobs=4: lat_avg=337.89 irqs=29073

* numjobs=8: lat_avg=342.98 irqs=28643

- Patched:

* numjobs=1: lat_avg=323.92 irqs=30262

* numjobs=4: lat_avg=332.65 irqs=29520

* numjobs=8: lat_avg=335.54 irqs=29323

Signed-off-by: Sergio Lopez <slp@redhat.com>

Message-id: 20180307114459.26636-1-slp@redhat.com

hw/block/dataplane/virtio-blk.c | 15 +++++++++++++--

1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c

--- a/hw/block/dataplane/virtio-blk.c

+++ b/hw/block/dataplane/virtio-blk.c

@@ -XXX,XX +XXX,XX @@ struct VirtIOBlockDataPlane {

VirtIODevice *vdev;

QEMUBH *bh; /* bh for guest notification */

unsigned long *batch_notify_vqs;

+ bool batch_notifications;

/* Note that these EventNotifiers are assigned by value. This is

* fine as long as you do not call event_notifier_cleanup on them

@@ -XXX,XX +XXX,XX @@ struct VirtIOBlockDataPlane {

/* Raise an interrupt to signal guest, if necessary */

void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq)

- set_bit(virtio_get_queue_index(vq), s->batch_notify_vqs);

- qemu_bh_schedule(s->bh);

+ if (s->batch_notifications) {

+ set_bit(virtio_get_queue_index(vq), s->batch_notify_vqs);

+ qemu_bh_schedule(s->bh);

+ } else {

+ virtio_notify_irqfd(s->vdev, vq);

+ }

}

static void notify_guest_bh(void *opaque)

@@ -XXX,XX +XXX,XX @@ int virtio_blk_data_plane_start(VirtIODevice *vdev)

s->starting = true;

+ if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {

+ s->batch_notifications = true;

+ } else {

+ s->batch_notifications = false;

+ }

+

/* Set up guest notifier (irq) */

r = k->set_guest_notifiers(qbus->parent, nvqs, true);