The following changes since commit 5dae13cd71f0755a1395b5a4cde635b8a6ee3f58:

Merge remote-tracking branch 'remotes/rth/tags/pull-or-20170214' into staging (2017-02-14 09:55:48 +0000)

are available in the git repository at:

git://github.com/stefanha/qemu.git tags/block-pull-request

for you to fetch changes up to decc18f33adecb1316437a47fff0cf0a7665906a:

coroutine-lock: make CoRwlock thread-safe and fair (2017-02-16 17:17:34 +0000)

----------------------------------------------------------------

Paolo Bonzini (24):

block: move AioContext, QEMUTimer, main-loop to libqemuutil

aio: introduce aio_co_schedule and aio_co_wake

block-backend: allow blk_prw from coroutine context

test-thread-pool: use generic AioContext infrastructure

io: add methods to set I/O handlers on AioContext

io: make qio_channel_yield aware of AioContexts

nbd: convert to use qio_channel_yield

coroutine-lock: reschedule coroutine on the AioContext it was running

on

blkdebug: reschedule coroutine on the AioContext it is running on

qed: introduce qed_aio_start_io and qed_aio_next_io_cb

aio: push aio_context_acquire/release down to dispatching

block: explicitly acquire aiocontext in timers that need it

block: explicitly acquire aiocontext in callbacks that need it

block: explicitly acquire aiocontext in bottom halves that need it

block: explicitly acquire aiocontext in aio callbacks that need it

aio-posix: partially inline aio_dispatch into aio_poll

async: remove unnecessary inc/dec pairs

block: document fields protected by AioContext lock

coroutine-lock: make CoMutex thread-safe

coroutine-lock: add limited spinning to CoMutex

test-aio-multithread: add performance comparison with thread-based

mutexes

coroutine-lock: place CoMutex before CoQueue in header

coroutine-lock: add mutex argument to CoQueue APIs

coroutine-lock: make CoRwlock thread-safe and fair

Makefile.objs | 4 -

stubs/Makefile.objs | 1 +

tests/Makefile.include | 19 +-

util/Makefile.objs | 6 +-

block/nbd-client.h | 2 +-

block/qed.h | 3 +

include/block/aio.h | 38 ++-

include/block/block_int.h | 64 +++--

include/io/channel.h | 72 +++++-

include/qemu/coroutine.h | 84 ++++---

include/qemu/coroutine_int.h | 11 +-

include/sysemu/block-backend.h | 14 +-

tests/iothread.h | 25 ++

block/backup.c | 2 +-

block/blkdebug.c | 9 +-

block/blkreplay.c | 2 +-

block/block-backend.c | 13 +-

block/curl.c | 44 +++-

block/gluster.c | 9 +-

block/io.c | 42 +---

block/iscsi.c | 15 +-

block/linux-aio.c | 10 +-

block/mirror.c | 12 +-

block/nbd-client.c | 119 +++++----

block/nfs.c | 9 +-

block/qcow2-cluster.c | 4 +-

block/qed-cluster.c | 2 +

block/qed-table.c | 12 +-

block/qed.c | 58 +++--

block/sheepdog.c | 31 +--

block/ssh.c | 29 +--

block/throttle-groups.c | 4 +-

block/win32-aio.c | 9 +-

dma-helpers.c | 2 +

hw/9pfs/9p.c | 2 +-

hw/block/virtio-blk.c | 19 +-

hw/scsi/scsi-bus.c | 2 +

hw/scsi/scsi-disk.c | 15 ++

hw/scsi/scsi-generic.c | 20 +-

hw/scsi/virtio-scsi.c | 6 +

io/channel-command.c | 13 +

io/channel-file.c | 11 +

io/channel-socket.c | 16 +-

io/channel-tls.c | 12 +

io/channel-watch.c | 6 +

io/channel.c | 97 ++++++--

nbd/client.c | 2 +-

nbd/common.c | 9 +-

nbd/server.c | 94 +++-----

stubs/linux-aio.c | 32 +++

stubs/set-fd-handler.c | 11 -

tests/iothread.c | 91 +++++++

tests/test-aio-multithread.c | 463 ++++++++++++++++++++++++++++++++++++

tests/test-thread-pool.c | 12 +-

aio-posix.c => util/aio-posix.c | 62 ++---

aio-win32.c => util/aio-win32.c | 30 +--

util/aiocb.c | 55 +++++

async.c => util/async.c | 84 ++++++-

iohandler.c => util/iohandler.c | 0

main-loop.c => util/main-loop.c | 0

util/qemu-coroutine-lock.c | 254 ++++++++++++++++++--

util/qemu-coroutine-sleep.c | 2 +-

util/qemu-coroutine.c | 8 +

qemu-timer.c => util/qemu-timer.c | 0

thread-pool.c => util/thread-pool.c | 8 +-

trace-events | 11 -

util/trace-events | 17 +-

67 files changed, 1711 insertions(+), 533 deletions(-)

create mode 100644 tests/iothread.h

create mode 100644 stubs/linux-aio.c

create mode 100644 tests/iothread.c

create mode 100644 tests/test-aio-multithread.c

rename aio-posix.c => util/aio-posix.c (94%)

rename aio-win32.c => util/aio-win32.c (95%)

create mode 100644 util/aiocb.c

rename async.c => util/async.c (82%)

rename iohandler.c => util/iohandler.c (100%)

rename main-loop.c => util/main-loop.c (100%)

rename qemu-timer.c => util/qemu-timer.c (100%)

rename thread-pool.c => util/thread-pool.c (97%)

2.9.3

From: Paolo Bonzini <pbonzini@redhat.com>

This uses the lock-free mutex described in the paper '"Blocking without

Locking", or LFTHREADS: A lock-free thread library' by Gidenstam and

Papatriantafilou. The same technique is used in OSv, and in fact

the code is essentially a conversion to C of OSv's code.

[Added missing coroutine_fn in tests/test-aio-multithread.c.

--Stefan]

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

Reviewed-by: Fam Zheng <famz@redhat.com>

Message-id: 20170213181244.16297-2-pbonzini@redhat.com

include/qemu/coroutine.h | 17 ++++-

tests/test-aio-multithread.c | 86 ++++++++++++++++++++++++

util/qemu-coroutine-lock.c | 155 ++++++++++++++++++++++++++++++++++++++++---

util/trace-events | 1 +

4 files changed, 246 insertions(+), 13 deletions(-)

diff --git a/include/qemu/coroutine.h b/include/qemu/coroutine.h

--- a/include/qemu/coroutine.h

+++ b/include/qemu/coroutine.h

@@ -XXX,XX +XXX,XX @@ bool qemu_co_queue_empty(CoQueue *queue);

/**

* Provides a mutex that can be used to synchronise coroutines

*/

+struct CoWaitRecord;

typedef struct CoMutex {

- bool locked;

+ /* Count of pending lockers; 0 for a free mutex, 1 for an

+ * uncontended mutex.

+ */

+ unsigned locked;

+ /* A queue of waiters. Elements are added atomically in front of

+ * from_push. to_pop is only populated, and popped from, by whoever

+ * is in charge of the next wakeup. This can be an unlocker or,

+ * through the handoff protocol, a locker that is about to go to sleep.

+ */

+ QSLIST_HEAD(, CoWaitRecord) from_push, to_pop;

+

+ unsigned handoff, sequence;

+

Coroutine *holder;

- CoQueue queue;

} CoMutex;

/**

diff --git a/tests/test-aio-multithread.c b/tests/test-aio-multithread.c

index XXXXXXX..XXXXXXX 100644

--- a/tests/test-aio-multithread.c

+++ b/tests/test-aio-multithread.c

@@ -XXX,XX +XXX,XX @@ static void test_multi_co_schedule_10(void)

test_multi_co_schedule(10);

}

+/* CoMutex thread-safety. */

+

+static uint32_t atomic_counter;

+static uint32_t running;

+static uint32_t counter;

+static CoMutex comutex;

+

+static void coroutine_fn test_multi_co_mutex_entry(void *opaque)

+{

+ while (!atomic_mb_read(&now_stopping)) {

+ qemu_co_mutex_lock(&comutex);

+ counter++;

+ qemu_co_mutex_unlock(&comutex);

+

+ /* Increase atomic_counter *after* releasing the mutex. Otherwise

+ * there is a chance (it happens about 1 in 3 runs) that the iothread

+ * exits before the coroutine is woken up, causing a spurious

+ * assertion failure.

+ */

+ atomic_inc(&atomic_counter);

+ }

+ atomic_dec(&running);

+static void test_multi_co_mutex(int threads, int seconds)

+ int i;

+ qemu_co_mutex_init(&comutex);

+ counter = 0;

+ atomic_counter = 0;

+ now_stopping = false;

+ create_aio_contexts();

+ assert(threads <= NUM_CONTEXTS);

+ running = threads;

+ for (i = 0; i < threads; i++) {

+ Coroutine *co1 = qemu_coroutine_create(test_multi_co_mutex_entry, NULL);

+ aio_co_schedule(ctx[i], co1);

+ g_usleep(seconds * 1000000);

+ atomic_mb_set(&now_stopping, true);

+ while (running > 0) {

+ g_usleep(100000);

+ join_aio_contexts();

+ g_test_message("%d iterations/second\n", counter / seconds);

+ g_assert_cmpint(counter, ==, atomic_counter);

+}

+/* Testing with NUM_CONTEXTS threads focuses on the queue. The mutex however

+ * is too contended (and the threads spend too much time in aio_poll)

+ * to actually stress the handoff protocol.

+ */

+static void test_multi_co_mutex_1(void)

+{

+ test_multi_co_mutex(NUM_CONTEXTS, 1);

+}

+static void test_multi_co_mutex_10(void)

+{

+ test_multi_co_mutex(NUM_CONTEXTS, 10);

+}

+/* Testing with fewer threads stresses the handoff protocol too. Still, the

+ * case where the locker _can_ pick up a handoff is very rare, happening

+ * about 10 times in 1 million, so increase the runtime a bit compared to

+ * other "quick" testcases that only run for 1 second.

+ */

+static void test_multi_co_mutex_2_3(void)

+{

+ test_multi_co_mutex(2, 3);

+}

+

+static void test_multi_co_mutex_2_30(void)

+{

+ test_multi_co_mutex(2, 30);

+}

+

/* End of tests. */

int main(int argc, char **argv)

@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)

g_test_add_func("/aio/multi/lifecycle", test_lifecycle);

if (g_test_quick()) {

g_test_add_func("/aio/multi/schedule", test_multi_co_schedule_1);

+ g_test_add_func("/aio/multi/mutex/contended", test_multi_co_mutex_1);

+ g_test_add_func("/aio/multi/mutex/handoff", test_multi_co_mutex_2_3);

} else {

g_test_add_func("/aio/multi/schedule", test_multi_co_schedule_10);

+ g_test_add_func("/aio/multi/mutex/contended", test_multi_co_mutex_10);

+ g_test_add_func("/aio/multi/mutex/handoff", test_multi_co_mutex_2_30);

}

return g_test_run();

}

diff --git a/util/qemu-coroutine-lock.c b/util/qemu-coroutine-lock.c

index XXXXXXX..XXXXXXX 100644

--- a/util/qemu-coroutine-lock.c

+++ b/util/qemu-coroutine-lock.c

@@ -XXX,XX +XXX,XX @@

* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN

* THE SOFTWARE.

+ *

+ * The lock-free mutex implementation is based on OSv

+ * (core/lfmutex.cc, include/lockfree/mutex.hh).

+ * Copyright (C) 2013 Cloudius Systems, Ltd.

*/

#include "qemu/osdep.h"

@@ -XXX,XX +XXX,XX @@ bool qemu_co_queue_empty(CoQueue *queue)

return QSIMPLEQ_FIRST(&queue->entries) == NULL;

}

+/* The wait records are handled with a multiple-producer, single-consumer

+ * lock-free queue. There cannot be two concurrent pop_waiter() calls

+ * because pop_waiter() can only be called while mutex->handoff is zero.

+ * This can happen in three cases:

+ * - in qemu_co_mutex_unlock, before the hand-off protocol has started.

+ * In this case, qemu_co_mutex_lock will see mutex->handoff == 0 and

+ * not take part in the handoff.

+ * - in qemu_co_mutex_lock, if it steals the hand-off responsibility from

+ * qemu_co_mutex_unlock. In this case, qemu_co_mutex_unlock will fail

+ * the cmpxchg (it will see either 0 or the next sequence value) and

+ * exit. The next hand-off cannot begin until qemu_co_mutex_lock has

+ * woken up someone.

+ * - in qemu_co_mutex_unlock, if it takes the hand-off token itself.

+ * In this case another iteration starts with mutex->handoff == 0;

+ * a concurrent qemu_co_mutex_lock will fail the cmpxchg, and

+ * qemu_co_mutex_unlock will go back to case (1).

+ *

+ * The following functions manage this queue.

+ */

+typedef struct CoWaitRecord {

+ Coroutine *co;

+ QSLIST_ENTRY(CoWaitRecord) next;

+} CoWaitRecord;

+

+static void push_waiter(CoMutex *mutex, CoWaitRecord *w)

+{

+ w->co = qemu_coroutine_self();

+ QSLIST_INSERT_HEAD_ATOMIC(&mutex->from_push, w, next);

+}

+

+static void move_waiters(CoMutex *mutex)

+{

+ QSLIST_HEAD(, CoWaitRecord) reversed;

+ QSLIST_MOVE_ATOMIC(&reversed, &mutex->from_push);

+ while (!QSLIST_EMPTY(&reversed)) {

+ CoWaitRecord *w = QSLIST_FIRST(&reversed);

+ QSLIST_REMOVE_HEAD(&reversed, next);

+ QSLIST_INSERT_HEAD(&mutex->to_pop, w, next);

+static CoWaitRecord *pop_waiter(CoMutex *mutex)

+ CoWaitRecord *w;

+

+ if (QSLIST_EMPTY(&mutex->to_pop)) {

+ move_waiters(mutex);

+ if (QSLIST_EMPTY(&mutex->to_pop)) {

+ return NULL;

+ }

+ }

+ w = QSLIST_FIRST(&mutex->to_pop);

+ QSLIST_REMOVE_HEAD(&mutex->to_pop, next);

+ return w;

+static bool has_waiters(CoMutex *mutex)

+{

+ return QSLIST_EMPTY(&mutex->to_pop) || QSLIST_EMPTY(&mutex->from_push);

+}

+

void qemu_co_mutex_init(CoMutex *mutex)

memset(mutex, 0, sizeof(*mutex));

- qemu_co_queue_init(&mutex->queue);

-void coroutine_fn qemu_co_mutex_lock(CoMutex *mutex)

+static void coroutine_fn qemu_co_mutex_lock_slowpath(CoMutex *mutex)

{

Coroutine *self = qemu_coroutine_self();

+ CoWaitRecord w;

+ unsigned old_handoff;

trace_qemu_co_mutex_lock_entry(mutex, self);

+ w.co = self;

+ push_waiter(mutex, &w);

- while (mutex->locked) {

- qemu_co_queue_wait(&mutex->queue);

+ /* This is the "Responsibility Hand-Off" protocol; a lock() picks from

+ * a concurrent unlock() the responsibility of waking somebody up.

+ */

+ old_handoff = atomic_mb_read(&mutex->handoff);

+ if (old_handoff &&

+ has_waiters(mutex) &&

+ atomic_cmpxchg(&mutex->handoff, old_handoff, 0) == old_handoff) {

+ /* There can be no concurrent pops, because there can be only

+ * one active handoff at a time.

+ */

+ CoWaitRecord *to_wake = pop_waiter(mutex);

+ Coroutine *co = to_wake->co;

+ if (co == self) {

+ /* We got the lock ourselves! */

+ assert(to_wake == &w);

+ return;

+ }

+

+ aio_co_wake(co);

}

- mutex->locked = true;

- mutex->holder = self;

- self->locks_held++;

-

+ qemu_coroutine_yield();

trace_qemu_co_mutex_lock_return(mutex, self);

}

+void coroutine_fn qemu_co_mutex_lock(CoMutex *mutex)

+{

+ Coroutine *self = qemu_coroutine_self();

+

+ if (atomic_fetch_inc(&mutex->locked) == 0) {

+ /* Uncontended. */

+ trace_qemu_co_mutex_lock_uncontended(mutex, self);

+ } else {

+ qemu_co_mutex_lock_slowpath(mutex);

+ }

+ mutex->holder = self;

+ self->locks_held++;

+}

+

void coroutine_fn qemu_co_mutex_unlock(CoMutex *mutex)

{

Coroutine *self = qemu_coroutine_self();

trace_qemu_co_mutex_unlock_entry(mutex, self);

- assert(mutex->locked == true);

+ assert(mutex->locked);

assert(mutex->holder == self);

assert(qemu_in_coroutine());

- mutex->locked = false;

mutex->holder = NULL;

self->locks_held--;

- qemu_co_queue_next(&mutex->queue);

+ if (atomic_fetch_dec(&mutex->locked) == 1) {

+ /* No waiting qemu_co_mutex_lock(). Pfew, that was easy! */

+ return;

+ }

+

+ for (;;) {

+ CoWaitRecord *to_wake = pop_waiter(mutex);

+ unsigned our_handoff;

+

+ if (to_wake) {

+ Coroutine *co = to_wake->co;

+ aio_co_wake(co);

+ break;

+ }

+

+ /* Some concurrent lock() is in progress (we know this because

+ * mutex->locked was >1) but it hasn't yet put itself on the wait

+ * queue. Pick a sequence number for the handoff protocol (not 0).

+ */

+ if (++mutex->sequence == 0) {

+ mutex->sequence = 1;

+ }

+

+ our_handoff = mutex->sequence;

+ atomic_mb_set(&mutex->handoff, our_handoff);

+ if (!has_waiters(mutex)) {

+ /* The concurrent lock has not added itself yet, so it

+ * will be able to pick our handoff.

+ */

+ break;

+ }

+

+ /* Try to do the handoff protocol ourselves; if somebody else has

+ * already taken it, however, we're done and they're responsible.

+ */

+ if (atomic_cmpxchg(&mutex->handoff, our_handoff, 0) != our_handoff) {

+ break;

+ }

+ }

trace_qemu_co_mutex_unlock_return(mutex, self);

}

diff --git a/util/trace-events b/util/trace-events

index XXXXXXX..XXXXXXX 100644

--- a/util/trace-events

+++ b/util/trace-events

@@ -XXX,XX +XXX,XX @@ qemu_coroutine_terminate(void *co) "self %p"

# util/qemu-coroutine-lock.c

qemu_co_queue_run_restart(void *co) "co %p"

+qemu_co_mutex_lock_uncontended(void *mutex, void *self) "mutex %p self %p"

qemu_co_mutex_lock_entry(void *mutex, void *self) "mutex %p self %p"

qemu_co_mutex_lock_return(void *mutex, void *self) "mutex %p self %p"

qemu_co_mutex_unlock_entry(void *mutex, void *self) "mutex %p self %p"

2.9.3

From: Paolo Bonzini <pbonzini@redhat.com>

Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

Reviewed-by: Fam Zheng <famz@redhat.com>

Reviewed-by: Daniel P. Berrange <berrange@redhat.com>

Message-id: 20170213135235.12274-15-pbonzini@redhat.com

block/archipelago.c | 3 +++

block/blkreplay.c | 2 +-

block/block-backend.c | 6 ++++++

block/curl.c | 26 ++++++++++++++++++--------

block/gluster.c | 9 +--------

block/io.c | 6 +++++-

block/iscsi.c | 6 +++++-

block/linux-aio.c | 15 +++++++++------

block/nfs.c | 3 ++-

block/null.c | 4 ++++

block/qed.c | 3 +++

block/rbd.c | 4 ++++

dma-helpers.c | 2 ++

hw/block/virtio-blk.c | 2 ++

hw/scsi/scsi-bus.c | 2 ++

util/async.c | 4 ++--

util/thread-pool.c | 2 ++

17 files changed, 71 insertions(+), 28 deletions(-)

diff --git a/block/archipelago.c b/block/archipelago.c

--- a/block/archipelago.c

+++ b/block/archipelago.c

@@ -XXX,XX +XXX,XX @@ static void qemu_archipelago_complete_aio(void *opaque)

{

AIORequestData *reqdata = (AIORequestData *) opaque;

ArchipelagoAIOCB *aio_cb = (ArchipelagoAIOCB *) reqdata->aio_cb;

+ AioContext *ctx = bdrv_get_aio_context(aio_cb->common.bs);

+ aio_context_acquire(ctx);

aio_cb->common.cb(aio_cb->common.opaque, aio_cb->ret);

+ aio_context_release(ctx);

aio_cb->status = 0;

qemu_aio_unref(aio_cb);

diff --git a/block/blkreplay.c b/block/blkreplay.c

index XXXXXXX..XXXXXXX 100755

--- a/block/blkreplay.c

+++ b/block/blkreplay.c

@@ -XXX,XX +XXX,XX @@ static int64_t blkreplay_getlength(BlockDriverState *bs)

static void blkreplay_bh_cb(void *opaque)

{

Request *req = opaque;

- qemu_coroutine_enter(req->co);

+ aio_co_wake(req->co);

qemu_bh_delete(req->bh);

g_free(req);

}

diff --git a/block/block-backend.c b/block/block-backend.c

index XXXXXXX..XXXXXXX 100644

--- a/block/block-backend.c

+++ b/block/block-backend.c

@@ -XXX,XX +XXX,XX @@ int blk_make_zero(BlockBackend *blk, BdrvRequestFlags flags)

static void error_callback_bh(void *opaque)

{

struct BlockBackendAIOCB *acb = opaque;

+ AioContext *ctx = bdrv_get_aio_context(acb->common.bs);

bdrv_dec_in_flight(acb->common.bs);

+ aio_context_acquire(ctx);

acb->common.cb(acb->common.opaque, acb->ret);

+ aio_context_release(ctx);

qemu_aio_unref(acb);

}

@@ -XXX,XX +XXX,XX @@ static void blk_aio_complete(BlkAioEmAIOCB *acb)

static void blk_aio_complete_bh(void *opaque)

{

BlkAioEmAIOCB *acb = opaque;

+ AioContext *ctx = bdrv_get_aio_context(acb->common.bs);

assert(acb->has_returned);

+ aio_context_acquire(ctx);

blk_aio_complete(acb);

+ aio_context_release(ctx);

}

static BlockAIOCB *blk_aio_prwv(BlockBackend *blk, int64_t offset, int bytes,

diff --git a/block/curl.c b/block/curl.c

index XXXXXXX..XXXXXXX 100644

--- a/block/curl.c

+++ b/block/curl.c

@@ -XXX,XX +XXX,XX @@ static void curl_readv_bh_cb(void *p)

{

CURLState *state;

int running;

+ int ret = -EINPROGRESS;

CURLAIOCB *acb = p;

- BDRVCURLState *s = acb->common.bs->opaque;

+ BlockDriverState *bs = acb->common.bs;

+ BDRVCURLState *s = bs->opaque;

+ AioContext *ctx = bdrv_get_aio_context(bs);

size_t start = acb->sector_num * BDRV_SECTOR_SIZE;

size_t end;

+ aio_context_acquire(ctx);

+

// In case we have the requested data already (e.g. read-ahead),

// we can just call the callback and be done.

switch (curl_find_buf(s, start, acb->nb_sectors * BDRV_SECTOR_SIZE, acb)) {

@@ -XXX,XX +XXX,XX @@ static void curl_readv_bh_cb(void *p)

qemu_aio_unref(acb);

// fall through

case FIND_RET_WAIT:

- return;

+ goto out;

default:

break;

}

@@ -XXX,XX +XXX,XX @@ static void curl_readv_bh_cb(void *p)

// No cache found, so let's start a new request

state = curl_init_state(acb->common.bs, s);

if (!state) {

- acb->common.cb(acb->common.opaque, -EIO);

- qemu_aio_unref(acb);

- return;

+ ret = -EIO;

+ goto out;

}

acb->start = 0;

@@ -XXX,XX +XXX,XX @@ static void curl_readv_bh_cb(void *p)

state->orig_buf = g_try_malloc(state->buf_len);

if (state->buf_len && state->orig_buf == NULL) {

curl_clean_state(state);

- acb->common.cb(acb->common.opaque, -ENOMEM);

- qemu_aio_unref(acb);

- return;

+ ret = -ENOMEM;

+ goto out;

}

state->acb[0] = acb;

@@ -XXX,XX +XXX,XX @@ static void curl_readv_bh_cb(void *p)

/* Tell curl it needs to kick things off */

curl_multi_socket_action(s->multi, CURL_SOCKET_TIMEOUT, 0, &running);

+

+out:

+ if (ret != -EINPROGRESS) {

+ acb->common.cb(acb->common.opaque, ret);

+ qemu_aio_unref(acb);

+ }

+ aio_context_release(ctx);

}

static BlockAIOCB *curl_aio_readv(BlockDriverState *bs,

diff --git a/block/gluster.c b/block/gluster.c

index XXXXXXX..XXXXXXX 100644

--- a/block/gluster.c

+++ b/block/gluster.c

@@ -XXX,XX +XXX,XX @@ static struct glfs *qemu_gluster_init(BlockdevOptionsGluster *gconf,

return qemu_gluster_glfs_init(gconf, errp);

}

-static void qemu_gluster_complete_aio(void *opaque)

-{

- GlusterAIOCB *acb = (GlusterAIOCB *)opaque;

-

- qemu_coroutine_enter(acb->coroutine);

-}

-

/*

* AIO callback routine called from GlusterFS thread.

*/

@@ -XXX,XX +XXX,XX @@ static void gluster_finish_aiocb(struct glfs_fd *fd, ssize_t ret, void *arg)

acb->ret = -EIO; /* Partial read/write - fail it */

}

- aio_bh_schedule_oneshot(acb->aio_context, qemu_gluster_complete_aio, acb);

+ aio_co_schedule(acb->aio_context, acb->coroutine);

}

static void qemu_gluster_parse_flags(int bdrv_flags, int *open_flags)

@@ -XXX,XX +XXX,XX @@ static void bdrv_co_drain_bh_cb(void *opaque)

bdrv_dec_in_flight(bs);

bdrv_drained_begin(bs);

data->done = true;

- qemu_coroutine_enter(co);

+ aio_co_wake(co);

static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs)

@@ -XXX,XX +XXX,XX @@ static void bdrv_co_complete(BlockAIOCBCoroutine *acb)

static void bdrv_co_em_bh(void *opaque)

BlockAIOCBCoroutine *acb = opaque;

+ BlockDriverState *bs = acb->common.bs;

+ AioContext *ctx = bdrv_get_aio_context(bs);

assert(!acb->need_bh);

+ aio_context_acquire(ctx);

bdrv_co_complete(acb);

+ aio_context_release(ctx);

}

static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb)

diff --git a/block/iscsi.c b/block/iscsi.c

index XXXXXXX..XXXXXXX 100644

--- a/block/iscsi.c

+++ b/block/iscsi.c

@@ -XXX,XX +XXX,XX @@ static void

iscsi_bh_cb(void *p)

{

IscsiAIOCB *acb = p;

+ AioContext *ctx = bdrv_get_aio_context(acb->common.bs);

qemu_bh_delete(acb->bh);

g_free(acb->buf);

acb->buf = NULL;

+ aio_context_acquire(ctx);

acb->common.cb(acb->common.opaque, acb->status);

+ aio_context_release(ctx);

if (acb->task != NULL) {

scsi_free_scsi_task(acb->task);

@@ -XXX,XX +XXX,XX @@ iscsi_schedule_bh(IscsiAIOCB *acb)

static void iscsi_co_generic_bh_cb(void *opaque)

{

struct IscsiTask *iTask = opaque;

iTask->complete = 1;

- qemu_coroutine_enter(iTask->co);

+ aio_co_wake(iTask->co);

}

static void iscsi_retry_timer_expired(void *opaque)

diff --git a/block/linux-aio.c b/block/linux-aio.c

index XXXXXXX..XXXXXXX 100644

--- a/block/linux-aio.c

+++ b/block/linux-aio.c

@@ -XXX,XX +XXX,XX @@ struct LinuxAioState {

io_context_t ctx;

EventNotifier e;

- /* io queue for submit at batch */

+ /* io queue for submit at batch. Protected by AioContext lock. */

LaioQueue io_q;

- /* I/O completion processing */

+ /* I/O completion processing. Only runs in I/O thread. */

QEMUBH *completion_bh;

int event_idx;

int event_max;

@@ -XXX,XX +XXX,XX @@ static inline ssize_t io_event_ret(struct io_event *ev)

*/

static void qemu_laio_process_completion(struct qemu_laiocb *laiocb)

{

+ LinuxAioState *s = laiocb->ctx;

int ret;

ret = laiocb->ret;

@@ -XXX,XX +XXX,XX @@ static void qemu_laio_process_completion(struct qemu_laiocb *laiocb)

2.9.3

From: Paolo Bonzini <pbonzini@redhat.com>

aio_co_wake provides the infrastructure to start a coroutine on a "home"

AioContext. It will be used by CoMutex and CoQueue, so that coroutines

don't jump from one context to another when they go to sleep on a

mutex or waitqueue. However, it can also be used as a more efficient

alternative to one-shot bottom halves, and saves the effort of tracking

which AioContext a coroutine is running on.

aio_co_schedule is the part of aio_co_wake that starts a coroutine

on a remove AioContext, but it is also useful to implement e.g.

bdrv_set_aio_context callbacks.

The implementation of aio_co_schedule is based on a lock-free

multiple-producer, single-consumer queue. The multiple producers use

cmpxchg to add to a LIFO stack. The consumer (a per-AioContext bottom

half) grabs all items added so far, inverts the list to make it FIFO,

and goes through it one item at a time until it's empty. The data

structure was inspired by OSv, which uses it in the very code we'll

"port" to QEMU for the thread-safe CoMutex.

Most of the new code is really tests.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

Reviewed-by: Fam Zheng <famz@redhat.com>

Message-id: 20170213135235.12274-3-pbonzini@redhat.com

tests/Makefile.include | 8 +-

include/block/aio.h | 32 +++++++

include/qemu/coroutine_int.h | 11 ++-

tests/iothread.h | 25 +++++

tests/iothread.c | 91 ++++++++++++++++++

tests/test-aio-multithread.c | 213 +++++++++++++++++++++++++++++++++++++++++++

util/async.c | 65 +++++++++++++

util/qemu-coroutine.c | 8 ++

util/trace-events | 4 +

9 files changed, 453 insertions(+), 4 deletions(-)

create mode 100644 tests/iothread.h

create mode 100644 tests/iothread.c

create mode 100644 tests/test-aio-multithread.c

diff --git a/tests/Makefile.include b/tests/Makefile.include

--- a/tests/Makefile.include

+++ b/tests/Makefile.include

@@ -XXX,XX +XXX,XX @@ check-unit-y += tests/test-aio$(EXESUF)

gcov-files-test-aio-y = util/async.c util/qemu-timer.o

gcov-files-test-aio-$(CONFIG_WIN32) += util/aio-win32.c

gcov-files-test-aio-$(CONFIG_POSIX) += util/aio-posix.c

+check-unit-y += tests/test-aio-multithread$(EXESUF)

+gcov-files-test-aio-multithread-y = $(gcov-files-test-aio-y)

+gcov-files-test-aio-multithread-y += util/qemu-coroutine.c tests/iothread.c

check-unit-y += tests/test-throttle$(EXESUF)

-gcov-files-test-aio-$(CONFIG_WIN32) = aio-win32.c

-gcov-files-test-aio-$(CONFIG_POSIX) = aio-posix.c

check-unit-y += tests/test-thread-pool$(EXESUF)

gcov-files-test-thread-pool-y = thread-pool.c

gcov-files-test-hbitmap-y = util/hbitmap.c

@@ -XXX,XX +XXX,XX @@ test-qapi-obj-y = tests/test-qapi-visit.o tests/test-qapi-types.o \

    $(test-qom-obj-y)

test-crypto-obj-y = $(crypto-obj-y) $(test-qom-obj-y)

test-io-obj-y = $(io-obj-y) $(test-crypto-obj-y)

-test-block-obj-y = $(block-obj-y) $(test-io-obj-y)

+test-block-obj-y = $(block-obj-y) $(test-io-obj-y) tests/iothread.o

tests/check-qint$(EXESUF): tests/check-qint.o $(test-util-obj-y)

tests/check-qstring$(EXESUF): tests/check-qstring.o $(test-util-obj-y)

@@ -XXX,XX +XXX,XX @@ tests/check-qom-proplist$(EXESUF): tests/check-qom-proplist.o $(test-qom-obj-y)

tests/test-char$(EXESUF): tests/test-char.o $(test-util-obj-y) $(qtest-obj-y) $(test-io-obj-y) $(chardev-obj-y)

tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(test-block-obj-y)

tests/test-aio$(EXESUF): tests/test-aio.o $(test-block-obj-y)

+tests/test-aio-multithread$(EXESUF): tests/test-aio-multithread.o $(test-block-obj-y)

tests/test-throttle$(EXESUF): tests/test-throttle.o $(test-block-obj-y)

tests/test-blockjob$(EXESUF): tests/test-blockjob.o $(test-block-obj-y) $(test-util-obj-y)

tests/test-blockjob-txn$(EXESUF): tests/test-blockjob-txn.o $(test-block-obj-y) $(test-util-obj-y)

diff --git a/include/block/aio.h b/include/block/aio.h

index XXXXXXX..XXXXXXX 100644

--- a/include/block/aio.h

+++ b/include/block/aio.h

@@ -XXX,XX +XXX,XX @@ typedef void QEMUBHFunc(void *opaque);

typedef bool AioPollFn(void *opaque);

typedef void IOHandler(void *opaque);

+struct Coroutine;

struct ThreadPool;

struct LinuxAioState;

@@ -XXX,XX +XXX,XX @@ struct AioContext {

bool notified;

EventNotifier notifier;

+ QSLIST_HEAD(, Coroutine) scheduled_coroutines;

+ QEMUBH *co_schedule_bh;

+

/* Thread pool for performing work and receiving completion callbacks.

* Has its own locking.

*/

@@ -XXX,XX +XXX,XX @@ static inline bool aio_node_check(AioContext *ctx, bool is_external)

/**

+ * aio_co_schedule:

+ * @ctx: the aio context

+ * @co: the coroutine

+ *

+ * Start a coroutine on a remote AioContext.

+ *

+ * The coroutine must not be entered by anyone else while aio_co_schedule()

+ * is active. In addition the coroutine must have yielded unless ctx

+ * is the context in which the coroutine is running (i.e. the value of

+ * qemu_get_current_aio_context() from the coroutine itself).

+ */

+void aio_co_schedule(AioContext *ctx, struct Coroutine *co);

+

+/**

+ * aio_co_wake:

+ * @co: the coroutine

+ *

+ * Restart a coroutine on the AioContext where it was running last, thus

+ * preventing coroutines from jumping from one context to another when they

+ * go to sleep.

+ *

+ * aio_co_wake may be executed either in coroutine or non-coroutine

+ * context. The coroutine must not be entered by anyone else while

+ * aio_co_wake() is active.

+ */

+void aio_co_wake(struct Coroutine *co);

+

+/**

* Return the AioContext whose event loop runs in the current thread.

*

* If called from an IOThread this will be the IOThread's AioContext. If

diff --git a/include/qemu/coroutine_int.h b/include/qemu/coroutine_int.h

index XXXXXXX..XXXXXXX 100644

--- a/include/qemu/coroutine_int.h

+++ b/include/qemu/coroutine_int.h

@@ -XXX,XX +XXX,XX @@ struct Coroutine {

CoroutineEntry *entry;

void *entry_arg;

Coroutine *caller;

+

+ /* Only used when the coroutine has terminated. */

QSLIST_ENTRY(Coroutine) pool_next;

+

size_t locks_held;

- /* Coroutines that should be woken up when we yield or terminate */

+ /* Coroutines that should be woken up when we yield or terminate.

+ * Only used when the coroutine is running.

+ */

QSIMPLEQ_HEAD(, Coroutine) co_queue_wakeup;

+

+ /* Only used when the coroutine has yielded. */

+ AioContext *ctx;

QSIMPLEQ_ENTRY(Coroutine) co_queue_next;

+ QSLIST_ENTRY(Coroutine) co_scheduled_next;

};

Coroutine *qemu_coroutine_new(void);