Series comparison

-[Qemu-devel] [PULL 00/13] Block layer patches
+[Qemu-devel] [PULL v3 00/35] Block layer patches
-The following changes since commit 0274f45bdef73283f2c213610f11d4e5dcba43b6:
+The following changes since commit 281f327487c9c9b1599f93c589a408bbf4a651b8:
-  Merge remote-tracking branch 'remotes/vivier2/tags/linux-user-for-4.1-pull-request' into staging (2019-07-19 09:44:43 +0100)
+  Merge remote-tracking branch 'remotes/vivier/tags/m68k-for-2.12-pull-request' into staging (2017-12-22 00:11:36 +0000)
-are available in the Git repository at:
+are available in the git repository at:
   git://repo.or.cz/qemu/kevin.git tags/for-upstream
-for you to fetch changes up to 49278ec065da3fbf90f7effcde3b39ac606b2e9e:
+for you to fetch changes up to 1a63a907507fbbcfaee3f622907ec244b7eabda8:
-  iotests: Test quitting with job on throttled node (2019-07-19 15:17:55 +0200)
+  block: Keep nodes drained between reopen_queue/multiple (2017-12-22 15:05:32 +0100)
 ----------------------------------------------------------------
-Block layer patches:
+Block layer patches
 - block: Fix forbidden use of polling in drained_end
 - block: Don't wait for I/O throttling while exiting QEMU
 - iotests: Use read-zeroes for the null driver to be Valgrind-friendly
 ----------------------------------------------------------------
-Andrey Shinkevich (1):
+Doug Gale (1):
-      iotests: Set read-zeroes on in null block driver for Valgrind
+      nvme: Add tracing
-Max Reitz (12):
+Edgar Kaziakhmedov (1):
-      block: Introduce BdrvChild.parent_quiesce_counter
+      qcow2: get rid of qcow2_backing_read1 routine
       tests: Add job commit by drained_end test
       block: Add @drained_end_counter
       block: Make bdrv_parent_drained_[^_]*() static
       tests: Lock AioContexts in test-block-iothread
       block: Do not poll in bdrv_do_drained_end()
       tests: Extend commit by drained_end test
       block: Loop unsafely in bdrv*drained_end()
       iotests: Add @has_quit to vm.shutdown()
       iotests: Test commit with a filter on the chain
       vl: Drain before (block) job cancel when quitting
       iotests: Test quitting with job on throttled node
- include/block/block.h         |  42 ++++++++----
+Fam Zheng (2):
- include/block/block_int.h     |  15 ++++-
+      block: Open backing image in force share mode for size probe
- block.c                       |  52 ++++++++++-----
+      block: Remove unused bdrv_requests_pending
  block/block-backend.c         |   6 +-
  block/io.c                    | 134 +++++++++++++++++++++++++++----------
  blockjob.c                    |   2 +-
  tests/test-bdrv-drain.c       | 147 ++++++++++++++++++++++++++++++++++++++++
  tests/test-block-iothread.c   |  40 +++++++----
  vl.c                          |  11 +++
  python/qemu/machine.py        |   5 +-
  tests/qemu-iotests/040        |  40 ++++++++++-
  tests/qemu-iotests/040.out    |   4 +-
  tests/qemu-iotests/051        |  10 +--
  tests/qemu-iotests/051.pc.out |  10 +--
  tests/qemu-iotests/093        |   9 +--
  tests/qemu-iotests/136        |   1 +
  tests/qemu-iotests/186        |  20 +++---
  tests/qemu-iotests/186.out    | 152 +++++++++++++++++++++---------------------
  tests/qemu-iotests/218        |  55 ++++++++++++++-
  tests/qemu-iotests/218.out    |   4 ++
  tests/qemu-iotests/227        |   4 +-
  tests/qemu-iotests/227.out    |   4 +-
  tests/qemu-iotests/238        |   2 +-
  tests/qemu-iotests/240        |   8 +--
  tests/qemu-iotests/255        |   2 +-
 files changed, 576 insertions(+), 203 deletions(-)
+John Snow (1):
+      iotests: fix 197 for vpc
+Kevin Wolf (27):
+      block: Formats don't need CONSISTENT_READ with NO_IO
+      block: Make bdrv_drain_invoke() recursive
+      block: Call .drain_begin only once in bdrv_drain_all_begin()
+      test-bdrv-drain: Test BlockDriver callbacks for drain
+      block: bdrv_drain_recurse(): Remove unused begin parameter
+      block: Don't wait for requests in bdrv_drain*_end()
+      block: Unify order in drain functions
+      block: Don't acquire AioContext in hmp_qemu_io()
+      block: Document that x-blockdev-change breaks quorum children list
+      block: Assert drain_all is only called from main AioContext
+      block: Make bdrv_drain() driver callbacks non-recursive
+      test-bdrv-drain: Test callback for bdrv_drain
+      test-bdrv-drain: Test bs->quiesce_counter
+      blockjob: Pause job on draining any job BDS
+      test-bdrv-drain: Test drain vs. block jobs
+      block: Don't block_job_pause_all() in bdrv_drain_all()
+      block: Nested drain_end must still call callbacks
+      test-bdrv-drain: Test nested drain sections
+      block: Don't notify parents in drain call chain
+      block: Add bdrv_subtree_drained_begin/end()
+      test-bdrv-drain: Tests for bdrv_subtree_drain
+      test-bdrv-drain: Test behaviour in coroutine context
+      test-bdrv-drain: Recursive draining with multiple parents
+      block: Allow graph changes in subtree drained section
+      test-bdrv-drain: Test graph changes in drained section
+      commit: Simplify reopen of base
+      block: Keep nodes drained between reopen_queue/multiple
+Thomas Huth (3):
+      block: Remove the obsolete -drive boot=on|off parameter
+      block: Remove the deprecated -hdachs option
+      block: Mention -drive cyls/heads/secs/trans/serial/addr in deprecation chapter
+ qapi/block-core.json             |   4 +
+ block/qcow2.h                    |   3 -
+ include/block/block.h            |  15 +-
+ include/block/block_int.h        |   6 +-
+ block.c                          |  75 ++++-
+ block/commit.c                   |   8 +-
+ block/io.c                       | 164 +++++++---
+ block/qcow2.c                    |  51 +--
+ block/replication.c              |   6 +
+ blockdev.c                       |  11 -
+ blockjob.c                       |  22 +-
+ hmp.c                            |   6 -
+ hw/block/nvme.c                  | 349 +++++++++++++++++----
+ qemu-io-cmds.c                   |   3 +
+ tests/test-bdrv-drain.c          | 651 +++++++++++++++++++++++++++++++++++++++
+ vl.c                             |  86 +-----
+ hw/block/trace-events            |  93 ++++++
+ qemu-doc.texi                    |  29 +-
+ qemu-options.hx                  |  19 +-
+ tests/Makefile.include           |   2 +
+ tests/qemu-iotests/197           |   4 +
+ tests/qemu-iotests/common.filter |   3 +-
+files changed, 1294 insertions(+), 316 deletions(-)
+ create mode 100644 tests/test-bdrv-drain.c

-New patch
+[Qemu-devel] [PULL v3 01/35] block: Formats don't need CONSISTENT_READ with NO_IO
+Commit 1f4ad7d fixed 'qemu-img info' for raw images that are currently
+in use as a mirror target. It is not enough for image formats, though,
+as these still unconditionally request BLK_PERM_CONSISTENT_READ.
+As this permission is geared towards whether the guest-visible data is
+consistent, and has no impact on whether the metadata is sane, and
+'qemu-img info' does not read guest-visible data (except for the raw
+format), it makes sense to not require BLK_PERM_CONSISTENT_READ if there
+is not going to be any guest I/O performed, regardless of image format.
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+---
+ block.c | 6 +++++-
+file changed, 5 insertions(+), 1 deletion(-)
+diff --git a/block.c b/block.c
+index XXXXXXX..XXXXXXX 100644
+--- a/block.c
++++ b/block.c
+@@ -XXX,XX +XXX,XX @@ void bdrv_format_default_perms(BlockDriverState *bs, BdrvChild *c,
+     assert(role == &child_backing || role == &child_file);
+     if (!backing) {
++        int flags = bdrv_reopen_get_flags(reopen_queue, bs);
++
+         /* Apart from the modifications below, the same permissions are
+          * forwarded and left alone as for filters */
+         bdrv_filter_default_perms(bs, c, role, reopen_queue, perm, shared,
+@@ -XXX,XX +XXX,XX @@ void bdrv_format_default_perms(BlockDriverState *bs, BdrvChild *c,
+         /* bs->file always needs to be consistent because of the metadata. We
+          * can never allow other users to resize or write to it. */
+-        perm |= BLK_PERM_CONSISTENT_READ;
++        if (!(flags & BDRV_O_NO_IO)) {
++            perm |= BLK_PERM_CONSISTENT_READ;
++        }
+         shared &= ~(BLK_PERM_WRITE | BLK_PERM_RESIZE);
+     } else {
+         /* We want consistent read from backing files if the parent needs it.
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 02/35] iotests: fix 197 for vpc
+From: John Snow <jsnow@redhat.com>
+VPC has some difficulty creating geometries of particular size.
+However, we can indeed force it to use a literal one, so let's
+do that for the sake of test 197, which is testing some specific
+offsets.
+Signed-off-by: John Snow <jsnow@redhat.com>
+Reviewed-by: Eric Blake <eblake@redhat.com>
+Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+Reviewed-by: Lukáš Doktor <ldoktor@redhat.com>
+---
+ tests/qemu-iotests/197           | 4 ++++
+ tests/qemu-iotests/common.filter | 3 ++-
+files changed, 6 insertions(+), 1 deletion(-)
+diff --git a/tests/qemu-iotests/197 b/tests/qemu-iotests/197
+index XXXXXXX..XXXXXXX 100755
+--- a/tests/qemu-iotests/197
++++ b/tests/qemu-iotests/197
+@@ -XXX,XX +XXX,XX @@ echo '=== Copy-on-read ==='
+ echo
+ # Prep the images
++# VPC rounds image sizes to a specific geometry, force a specific size.
++if [ "$IMGFMT" = "vpc" ]; then
++    IMGOPTS=$(_optstr_add "$IMGOPTS" "force_size")
++fi
+ _make_test_img 4G
+ $QEMU_IO -c "write -P 55 3G 1k" "$TEST_IMG" | _filter_qemu_io
+ IMGPROTO=file IMGFMT=qcow2 IMGOPTS= TEST_IMG_FILE="$TEST_WRAP" \
+diff --git a/tests/qemu-iotests/common.filter b/tests/qemu-iotests/common.filter
+index XXXXXXX..XXXXXXX 100644
+--- a/tests/qemu-iotests/common.filter
++++ b/tests/qemu-iotests/common.filter
+@@ -XXX,XX +XXX,XX @@ _filter_img_create()
+         -e "s# log_size=[0-9]\\+##g" \
+         -e "s# refcount_bits=[0-9]\\+##g" \
+         -e "s# key-secret=[a-zA-Z0-9]\\+##g" \
+-        -e "s# iter-time=[0-9]\\+##g"
++        -e "s# iter-time=[0-9]\\+##g" \
++        -e "s# force_size=\\(on\\|off\\)##g"
+ }
+ _filter_img_info()
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 03/35] block: Make bdrv_drain_invoke() recursive
+This change separates bdrv_drain_invoke(), which calls the BlockDriver
+drain callbacks, from bdrv_drain_recurse(). Instead, the function
+performs its own recursion now.
+One reason for this is that bdrv_drain_recurse() can be called multiple
+times by bdrv_drain_all_begin(), but the callbacks may only be called
+once. The separation is necessary to fix this bug.
+The other reason is that we intend to go to a model where we call all
+driver callbacks first, and only then start polling. This is not fully
+achieved yet with this patch, as bdrv_drain_invoke() contains a
+BDRV_POLL_WHILE() loop for the block driver callbacks, which can still
+call callbacks for any unrelated event. It's a step in this direction
+anyway.
+Cc: qemu-stable@nongnu.org
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
+---
+ block/io.c | 14 +++++++++++---
+file changed, 11 insertions(+), 3 deletions(-)
+diff --git a/block/io.c b/block/io.c
+index XXXXXXX..XXXXXXX 100644
+--- a/block/io.c
++++ b/block/io.c
+@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
+     bdrv_wakeup(bs);
+ }
++/* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */
+ static void bdrv_drain_invoke(BlockDriverState *bs, bool begin)
+ {
++    BdrvChild *child, *tmp;
+     BdrvCoDrainData data = { .bs = bs, .done = false, .begin = begin};
+     if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) ||
+@@ -XXX,XX +XXX,XX @@ static void bdrv_drain_invoke(BlockDriverState *bs, bool begin)
+     data.co = qemu_coroutine_create(bdrv_drain_invoke_entry, &data);
+     bdrv_coroutine_enter(bs, data.co);
+     BDRV_POLL_WHILE(bs, !data.done);
++
++    QLIST_FOREACH_SAFE(child, &bs->children, next, tmp) {
++        bdrv_drain_invoke(child->bs, begin);
++    }
+ }
+ static bool bdrv_drain_recurse(BlockDriverState *bs, bool begin)
+@@ -XXX,XX +XXX,XX @@ static bool bdrv_drain_recurse(BlockDriverState *bs, bool begin)
+     BdrvChild *child, *tmp;
+     bool waited;
+-    /* Ensure any pending metadata writes are submitted to bs->file.  */
+-    bdrv_drain_invoke(bs, begin);
+-
+     /* Wait for drained requests to finish */
+     waited = BDRV_POLL_WHILE(bs, atomic_read(&bs->in_flight) > 0);
+@@ -XXX,XX +XXX,XX @@ void bdrv_drained_begin(BlockDriverState *bs)
+         bdrv_parent_drained_begin(bs);
+     }
++    bdrv_drain_invoke(bs, true);
+     bdrv_drain_recurse(bs, true);
+ }
+@@ -XXX,XX +XXX,XX @@ void bdrv_drained_end(BlockDriverState *bs)
+     }
+     bdrv_parent_drained_end(bs);
++    bdrv_drain_invoke(bs, false);
+     bdrv_drain_recurse(bs, false);
+     aio_enable_external(bdrv_get_aio_context(bs));
+ }
+@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
+             aio_context_acquire(aio_context);
+             for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+                 if (aio_context == bdrv_get_aio_context(bs)) {
++                    /* FIXME Calling this multiple times is wrong */
++                    bdrv_drain_invoke(bs, true);
+                     waited |= bdrv_drain_recurse(bs, true);
+                 }
+             }
+@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_end(void)
+         aio_context_acquire(aio_context);
+         aio_enable_external(aio_context);
+         bdrv_parent_drained_end(bs);
++        bdrv_drain_invoke(bs, false);
+         bdrv_drain_recurse(bs, false);
+         aio_context_release(aio_context);
+     }
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 04/35] block: Call .drain_begin only once in bdrv_drain_all_begin()
+bdrv_drain_all_begin() used to call the .bdrv_co_drain_begin() driver
+callback inside its polling loop. This means that how many times it got
+called for each node depended on long it had to poll the event loop.
+This is obviously not right and results in nodes that stay drained even
+after bdrv_drain_all_end(), which calls .bdrv_co_drain_begin() once per
+node.
+Fix bdrv_drain_all_begin() to call the callback only once, too.
+Cc: qemu-stable@nongnu.org
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
+---
+ block/io.c | 3 +--
+file changed, 1 insertion(+), 2 deletions(-)
+diff --git a/block/io.c b/block/io.c
+index XXXXXXX..XXXXXXX 100644
+--- a/block/io.c
++++ b/block/io.c
+@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
+         aio_context_acquire(aio_context);
+         bdrv_parent_drained_begin(bs);
+         aio_disable_external(aio_context);
++        bdrv_drain_invoke(bs, true);
+         aio_context_release(aio_context);
+         if (!g_slist_find(aio_ctxs, aio_context)) {
+@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
+             aio_context_acquire(aio_context);
+             for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+                 if (aio_context == bdrv_get_aio_context(bs)) {
+-                    /* FIXME Calling this multiple times is wrong */
+-                    bdrv_drain_invoke(bs, true);
+                     waited |= bdrv_drain_recurse(bs, true);
+                 }
+             }
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 05/35] test-bdrv-drain: Test BlockDriver callbacks for drain
+This adds a test case that the BlockDriver callbacks for drain are
+called in bdrv_drained_all_begin/end(), and that both of them are called
+exactly once.
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
+Reviewed-by: Eric Blake <eblake@redhat.com>
+---
+ tests/test-bdrv-drain.c | 137 ++++++++++++++++++++++++++++++++++++++++++++++++
+ tests/Makefile.include  |   2 +
+files changed, 139 insertions(+)
+ create mode 100644 tests/test-bdrv-drain.c
+diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
+new file mode 100644
+index XXXXXXX..XXXXXXX
+--- /dev/null
++++ b/tests/test-bdrv-drain.c
+@@ -XXX,XX +XXX,XX @@
++/*
++ * Block node draining tests
++ *
++ * Copyright (c) 2017 Kevin Wolf <kwolf@redhat.com>
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a copy
++ * of this software and associated documentation files (the "Software"), to deal
++ * in the Software without restriction, including without limitation the rights
++ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
++ * copies of the Software, and to permit persons to whom the Software is
++ * furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice shall be included in
++ * all copies or substantial portions of the Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
++ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
++ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
++ * THE SOFTWARE.
++ */
++
++#include "qemu/osdep.h"
++#include "block/block.h"
++#include "sysemu/block-backend.h"
++#include "qapi/error.h"
++
++typedef struct BDRVTestState {
++    int drain_count;
++} BDRVTestState;
++
++static void coroutine_fn bdrv_test_co_drain_begin(BlockDriverState *bs)
++{
++    BDRVTestState *s = bs->opaque;
++    s->drain_count++;
++}
++
++static void coroutine_fn bdrv_test_co_drain_end(BlockDriverState *bs)
++{
++    BDRVTestState *s = bs->opaque;
++    s->drain_count--;
++}
++
++static void bdrv_test_close(BlockDriverState *bs)
++{
++    BDRVTestState *s = bs->opaque;
++    g_assert_cmpint(s->drain_count, >, 0);
++}
++
++static int coroutine_fn bdrv_test_co_preadv(BlockDriverState *bs,
++                                            uint64_t offset, uint64_t bytes,
++                                            QEMUIOVector *qiov, int flags)
++{
++    /* We want this request to stay until the polling loop in drain waits for
++     * it to complete. We need to sleep a while as bdrv_drain_invoke() comes
++     * first and polls its result, too, but it shouldn't accidentally complete
++     * this request yet. */
++    qemu_co_sleep_ns(QEMU_CLOCK_REALTIME, 100000);
++
++    return 0;
++}
++
++static BlockDriver bdrv_test = {
++    .format_name            = "test",
++    .instance_size          = sizeof(BDRVTestState),
++
++    .bdrv_close             = bdrv_test_close,
++    .bdrv_co_preadv         = bdrv_test_co_preadv,
++
++    .bdrv_co_drain_begin    = bdrv_test_co_drain_begin,
++    .bdrv_co_drain_end      = bdrv_test_co_drain_end,
++};
++
++static void aio_ret_cb(void *opaque, int ret)
++{
++    int *aio_ret = opaque;
++    *aio_ret = ret;
++}
++
++static void test_drv_cb_drain_all(void)
++{
++    BlockBackend *blk;
++    BlockDriverState *bs;
++    BDRVTestState *s;
++    BlockAIOCB *acb;
++    int aio_ret;
++
++    QEMUIOVector qiov;
++    struct iovec iov = {
++        .iov_base = NULL,
++        .iov_len = 0,
++    };
++    qemu_iovec_init_external(&qiov, &iov, 1);
++
++    blk = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
++    bs = bdrv_new_open_driver(&bdrv_test, "test-node", BDRV_O_RDWR,
++                              &error_abort);
++    s = bs->opaque;
++    blk_insert_bs(blk, bs, &error_abort);
++
++    /* Simple bdrv_drain_all_begin/end pair, check that CBs are called */
++    g_assert_cmpint(s->drain_count, ==, 0);
++    bdrv_drain_all_begin();
++    g_assert_cmpint(s->drain_count, ==, 1);
++    bdrv_drain_all_end();
++    g_assert_cmpint(s->drain_count, ==, 0);
++
++    /* Now do the same while a request is pending */
++    aio_ret = -EINPROGRESS;
++    acb = blk_aio_preadv(blk, 0, &qiov, 0, aio_ret_cb, &aio_ret);
++    g_assert(acb != NULL);
++    g_assert_cmpint(aio_ret, ==, -EINPROGRESS);
++
++    g_assert_cmpint(s->drain_count, ==, 0);
++    bdrv_drain_all_begin();
++    g_assert_cmpint(aio_ret, ==, 0);
++    g_assert_cmpint(s->drain_count, ==, 1);
++    bdrv_drain_all_end();
++    g_assert_cmpint(s->drain_count, ==, 0);
++
++    bdrv_unref(bs);
++    blk_unref(blk);
++}
++
++int main(int argc, char **argv)
++{
++    bdrv_init();
++    qemu_init_main_loop(&error_abort);
++
++    g_test_init(&argc, &argv, NULL);
++
++    g_test_add_func("/bdrv-drain/driver-cb/drain_all", test_drv_cb_drain_all);
++
++    return g_test_run();
++}
+diff --git a/tests/Makefile.include b/tests/Makefile.include
+index XXXXXXX..XXXXXXX 100644
+--- a/tests/Makefile.include
++++ b/tests/Makefile.include
+@@ -XXX,XX +XXX,XX @@ gcov-files-test-thread-pool-y = thread-pool.c
+ gcov-files-test-hbitmap-y = util/hbitmap.c
+ check-unit-y += tests/test-hbitmap$(EXESUF)
+ gcov-files-test-hbitmap-y = blockjob.c
++check-unit-y += tests/test-bdrv-drain$(EXESUF)
+ check-unit-y += tests/test-blockjob$(EXESUF)
+ check-unit-y += tests/test-blockjob-txn$(EXESUF)
+ check-unit-y += tests/test-x86-cpuid$(EXESUF)
+@@ -XXX,XX +XXX,XX @@ tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(test-block-obj-y)
+ tests/test-aio$(EXESUF): tests/test-aio.o $(test-block-obj-y)
+ tests/test-aio-multithread$(EXESUF): tests/test-aio-multithread.o $(test-block-obj-y)
+ tests/test-throttle$(EXESUF): tests/test-throttle.o $(test-block-obj-y)
++tests/test-bdrv-drain$(EXESUF): tests/test-bdrv-drain.o $(test-block-obj-y) $(test-util-obj-y)
+ tests/test-blockjob$(EXESUF): tests/test-blockjob.o $(test-block-obj-y) $(test-util-obj-y)
+ tests/test-blockjob-txn$(EXESUF): tests/test-blockjob-txn.o $(test-block-obj-y) $(test-util-obj-y)
+ tests/test-thread-pool$(EXESUF): tests/test-thread-pool.o $(test-block-obj-y)
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 06/35] block: bdrv_drain_recurse(): Remove unused begin parameter
+Now that the bdrv_drain_invoke() calls are pulled up to the callers of
+bdrv_drain_recurse(), the 'begin' parameter isn't needed any more.
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
+---
+ block/io.c | 12 ++++++------
+file changed, 6 insertions(+), 6 deletions(-)
+diff --git a/block/io.c b/block/io.c
+index XXXXXXX..XXXXXXX 100644
+--- a/block/io.c
++++ b/block/io.c
+@@ -XXX,XX +XXX,XX @@ static void bdrv_drain_invoke(BlockDriverState *bs, bool begin)
+     }
+ }
+-static bool bdrv_drain_recurse(BlockDriverState *bs, bool begin)
++static bool bdrv_drain_recurse(BlockDriverState *bs)
+ {
+     BdrvChild *child, *tmp;
+     bool waited;
+@@ -XXX,XX +XXX,XX @@ static bool bdrv_drain_recurse(BlockDriverState *bs, bool begin)
+              */
+             bdrv_ref(bs);
+         }
+-        waited |= bdrv_drain_recurse(bs, begin);
++        waited |= bdrv_drain_recurse(bs);
+         if (in_main_loop) {
+             bdrv_unref(bs);
+         }
+@@ -XXX,XX +XXX,XX @@ void bdrv_drained_begin(BlockDriverState *bs)
+     }
+     bdrv_drain_invoke(bs, true);
+-    bdrv_drain_recurse(bs, true);
++    bdrv_drain_recurse(bs);
+ }
+ void bdrv_drained_end(BlockDriverState *bs)
+@@ -XXX,XX +XXX,XX @@ void bdrv_drained_end(BlockDriverState *bs)
+     bdrv_parent_drained_end(bs);
+     bdrv_drain_invoke(bs, false);
+-    bdrv_drain_recurse(bs, false);
++    bdrv_drain_recurse(bs);
+     aio_enable_external(bdrv_get_aio_context(bs));
+ }
+@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
+             aio_context_acquire(aio_context);
+             for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+                 if (aio_context == bdrv_get_aio_context(bs)) {
+-                    waited |= bdrv_drain_recurse(bs, true);
++                    waited |= bdrv_drain_recurse(bs);
+                 }
+             }
+             aio_context_release(aio_context);
+@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_end(void)
+         aio_enable_external(aio_context);
+         bdrv_parent_drained_end(bs);
+         bdrv_drain_invoke(bs, false);
+-        bdrv_drain_recurse(bs, false);
++        bdrv_drain_recurse(bs);
+         aio_context_release(aio_context);
+     }
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 07/35] block: Don't wait for requests in bdrv_drain*_end()
+The device is drained, so there is no point in waiting for requests at
+the end of the drained section. Remove the bdrv_drain_recurse() calls
+there.
+The bdrv_drain_recurse() calls were introduced in commit 481cad48e5e
+in order to call the .bdrv_co_drain_end() driver callback. This is now
+done by a separate bdrv_drain_invoke() call.
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
+Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
+---
+ block/io.c | 2 --
+file changed, 2 deletions(-)
+diff --git a/block/io.c b/block/io.c
+index XXXXXXX..XXXXXXX 100644
+--- a/block/io.c
++++ b/block/io.c
+@@ -XXX,XX +XXX,XX @@ void bdrv_drained_end(BlockDriverState *bs)
+     bdrv_parent_drained_end(bs);
+     bdrv_drain_invoke(bs, false);
+-    bdrv_drain_recurse(bs);
+     aio_enable_external(bdrv_get_aio_context(bs));
+ }
+@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_end(void)
+         aio_enable_external(aio_context);
+         bdrv_parent_drained_end(bs);
+         bdrv_drain_invoke(bs, false);
+-        bdrv_drain_recurse(bs);
+         aio_context_release(aio_context);
+     }
+--
+.13.6

-[Qemu-devel] [PULL 04/13] block: Add @drained_end_counter
+[Qemu-devel] [PULL v3 08/35] block: Unify order in drain functions
-From: Max Reitz <mreitz@redhat.com>
+Drain requests are propagated to child nodes, parent nodes and directly
 to the AioContext. The order in which this happened was different
 between all combinations of drain/drain_all and begin/end.
-Callers can now pass a pointer to an integer that bdrv_drain_invoke()
+The correct order is to keep children only drained when their parents
-(and its recursive callees) will increment for every
+are also drained. This means that at the start of a drained section, the
-bdrv_drain_invoke_entry() operation they schedule.
+AioContext needs to be drained first, the parents second and only then
-bdrv_drain_invoke_entry() in turn will decrement it once it has invoked
+the children. The correct order for the end of a drained section is the
-BlockDriver.bdrv_co_drain_end().
+opposite.
-We use atomic operations to access the pointee, because the
+This patch changes the three other functions to follow the example of
-bdrv_do_drained_end() caller may wish to end drained sections for
+bdrv_drained_begin(), which is the only one that got it right.
 multiple nodes in different AioContexts (bdrv_drain_all_end() does, for
 example).
-This is the first step to moving the polling for BdrvCoDrainData.done to
-become true out of bdrv_drain_invoke() and into the root drained_end
-function.
-Signed-off-by: Max Reitz <mreitz@redhat.com>
 Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
 ---
- block/io.c | 58 +++++++++++++++++++++++++++++++++++++-----------------
+ block/io.c | 12 ++++++++----
-file changed, 40 insertions(+), 18 deletions(-)
+file changed, 8 insertions(+), 4 deletions(-)
 diff --git a/block/io.c b/block/io.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/io.c
 +++ b/block/io.c
-@@ -XXX,XX +XXX,XX @@ typedef struct {
+@@ -XXX,XX +XXX,XX @@ void bdrv_drained_begin(BlockDriverState *bs)
      bool poll;
      BdrvChild *parent;
      bool ignore_bds_parents;
 +    int *drained_end_counter;
  } BdrvCoDrainData;
  static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
      atomic_mb_set(&data->done, true);
      bdrv_dec_in_flight(bs);
 -    if (data->begin) {
 +    if (data->drained_end_counter) {
 +        atomic_dec(data->drained_end_counter);
 +    }
 +
 +    if (data->begin || data->drained_end_counter) {
          g_free(data);
      }
  }
  /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */
 -static void bdrv_drain_invoke(BlockDriverState *bs, bool begin)
 +static void bdrv_drain_invoke(BlockDriverState *bs, bool begin,
 +                              int *drained_end_counter)
  {
      BdrvCoDrainData *data;
@@ -XXX,XX +XXX,XX @@ static void bdrv_drain_invoke(BlockDriverState *bs, bool begin)
      *data = (BdrvCoDrainData) {
          .bs = bs,
          .done = false,
 -        .begin = begin
 +        .begin = begin,
 +        .drained_end_counter = drained_end_counter,
      };
 +    if (!begin && drained_end_counter) {
 +        atomic_inc(drained_end_counter);
 +    }
 +
      /* Make sure the driver callback completes during the polling phase for
       * drain_begin. */
      bdrv_inc_in_flight(bs);
      data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data);
      aio_co_schedule(bdrv_get_aio_context(bs), data->co);
 -    if (!begin) {
 +    /*
 +     * TODO: Drop this and make callers pass @drained_end_counter and poll
 +     * themselves
 +     */
 +    if (!begin && !drained_end_counter) {
          BDRV_POLL_WHILE(bs, !data->done);
          g_free(data);
      }
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
                                    BdrvChild *parent, bool ignore_bds_parents,
                                    bool poll);
  static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
 -                                BdrvChild *parent, bool ignore_bds_parents);
 +                                BdrvChild *parent, bool ignore_bds_parents,
 +                                int *drained_end_counter);
  static void bdrv_co_drain_bh_cb(void *opaque)
  {
@@ -XXX,XX +XXX,XX @@ static void bdrv_co_drain_bh_cb(void *opaque)
                                    data->ignore_bds_parents, data->poll);
          } else {
              bdrv_do_drained_end(bs, data->recursive, data->parent,
 -                                data->ignore_bds_parents);
 +                                data->ignore_bds_parents,
 +                                data->drained_end_counter);
          }
          if (ctx == co_ctx) {
              aio_context_release(ctx);
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
                                                  bool begin, bool recursive,
                                                  BdrvChild *parent,
                                                  bool ignore_bds_parents,
 -                                                bool poll)
 +                                                bool poll,
 +                                                int *drained_end_counter)
  {
      BdrvCoDrainData data;
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
          .parent = parent,
          .ignore_bds_parents = ignore_bds_parents,
          .poll = poll,
 +        .drained_end_counter = drained_end_counter,
      };
 +
      if (bs) {
          bdrv_inc_in_flight(bs);
      }
@@ -XXX,XX +XXX,XX @@ void bdrv_do_drained_begin_quiesce(BlockDriverState *bs,
      }
      bdrv_parent_drained_begin(bs, parent, ignore_bds_parents);
 -    bdrv_drain_invoke(bs, true);
 +    bdrv_drain_invoke(bs, true, NULL);
  }
  static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
      if (qemu_in_coroutine()) {
          bdrv_co_yield_to_drain(bs, true, recursive, parent, ignore_bds_parents,
 -                               poll);
 +                               poll, NULL);
          return;
      }
-@@ -XXX,XX +XXX,XX @@ void bdrv_subtree_drained_begin(BlockDriverState *bs)
++    /* Stop things in parent-to-child order */
- }
+     if (atomic_fetch_inc(&bs->quiesce_counter) == 0) {
+         aio_disable_external(bdrv_get_aio_context(bs));
- static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
+         bdrv_parent_drained_begin(bs);
--                                BdrvChild *parent, bool ignore_bds_parents)
+@@ -XXX,XX +XXX,XX @@ void bdrv_drained_end(BlockDriverState *bs)
 +                                BdrvChild *parent, bool ignore_bds_parents,
 +                                int *drained_end_counter)
  {
      BdrvChild *child, *next;
      int old_quiesce_counter;
      if (qemu_in_coroutine()) {
          bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents,
 -                               false);
 +                               false, drained_end_counter);
          return;
      }
-     assert(bs->quiesce_counter > 0);
+-    bdrv_parent_drained_end(bs);
-     /* Re-enable things in child-to-parent order */
++    /* Re-enable things in child-to-parent order */
--    bdrv_drain_invoke(bs, false);
+     bdrv_drain_invoke(bs, false);
-+    bdrv_drain_invoke(bs, false, drained_end_counter);
++    bdrv_parent_drained_end(bs);
-     bdrv_parent_drained_end(bs, parent, ignore_bds_parents);
+     aio_enable_external(bdrv_get_aio_context(bs));
      old_quiesce_counter = atomic_fetch_dec(&bs->quiesce_counter);
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
          assert(!ignore_bds_parents);
          bs->recursive_quiesce_counter--;
          QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
 -            bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents);
 +            bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents,
 +                                drained_end_counter);
          }
      }
  }
- void bdrv_drained_end(BlockDriverState *bs)
- {
--    bdrv_do_drained_end(bs, false, NULL, false);
-+    bdrv_do_drained_end(bs, false, NULL, false, NULL);
- }
- void bdrv_subtree_drained_end(BlockDriverState *bs)
- {
--    bdrv_do_drained_end(bs, true, NULL, false);
-+    bdrv_do_drained_end(bs, true, NULL, false, NULL);
- }
- void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
-@@ -XXX,XX +XXX,XX @@ void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent)
-     int i;
-     for (i = 0; i < old_parent->recursive_quiesce_counter; i++) {
--        bdrv_do_drained_end(child->bs, true, child, false);
-+        bdrv_do_drained_end(child->bs, true, child, false, NULL);
-     }
- }
 @@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
-     BlockDriverState *bs = NULL;
+     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+         AioContext *aio_context = bdrv_get_aio_context(bs);
-     if (qemu_in_coroutine()) {
--        bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true);
++        /* Stop things in parent-to-child order */
-+        bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true, NULL);
+         aio_context_acquire(aio_context);
-         return;
+-        bdrv_parent_drained_begin(bs);
-     }
+         aio_disable_external(aio_context);
 +        bdrv_parent_drained_begin(bs);
          bdrv_drain_invoke(bs, true);
          aio_context_release(aio_context);
 @@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_end(void)
+     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
          AioContext *aio_context = bdrv_get_aio_context(bs);
++        /* Re-enable things in child-to-parent order */
          aio_context_acquire(aio_context);
--        bdrv_do_drained_end(bs, false, NULL, true);
+-        aio_enable_external(aio_context);
-+        bdrv_do_drained_end(bs, false, NULL, true, NULL);
+-        bdrv_parent_drained_end(bs);
          bdrv_drain_invoke(bs, false);
 +        bdrv_parent_drained_end(bs);
 +        aio_enable_external(aio_context);
          aio_context_release(aio_context);
      }
 --
-.20.1
+.13.6

-New patch
+[Qemu-devel] [PULL v3 09/35] block: Don't acquire AioContext in hmp_qemu_io()
+Commit 15afd94a047 added code to acquire and release the AioContext in
+qemuio_command(). This means that the lock is taken twice now in the
+call path from hmp_qemu_io(). This causes BDRV_POLL_WHILE() to hang for
+any requests issued to nodes in a non-mainloop AioContext.
+Dropping the first locking from hmp_qemu_io() fixes the problem.
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
+---
+ hmp.c | 6 ------
+file changed, 6 deletions(-)
+diff --git a/hmp.c b/hmp.c
+index XXXXXXX..XXXXXXX 100644
+--- a/hmp.c
++++ b/hmp.c
+@@ -XXX,XX +XXX,XX @@ void hmp_qemu_io(Monitor *mon, const QDict *qdict)
+ {
+     BlockBackend *blk;
+     BlockBackend *local_blk = NULL;
+-    AioContext *aio_context;
+     const char* device = qdict_get_str(qdict, "device");
+     const char* command = qdict_get_str(qdict, "command");
+     Error *err = NULL;
+@@ -XXX,XX +XXX,XX @@ void hmp_qemu_io(Monitor *mon, const QDict *qdict)
+         }
+     }
+-    aio_context = blk_get_aio_context(blk);
+-    aio_context_acquire(aio_context);
+-
+     /*
+      * Notably absent: Proper permission management. This is sad, but it seems
+      * almost impossible to achieve without changing the semantics and thereby
+@@ -XXX,XX +XXX,XX @@ void hmp_qemu_io(Monitor *mon, const QDict *qdict)
+      */
+     qemuio_command(blk, command);
+-    aio_context_release(aio_context);
+-
+ fail:
+     blk_unref(local_blk);
+     hmp_handle_error(mon, &err);
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 10/35] qcow2: get rid of qcow2_backing_read1 routine
+From: Edgar Kaziakhmedov <edgar.kaziakhmedov@virtuozzo.com>
+Since bdrv_co_preadv does all neccessary checks including
+reading after the end of the backing file, avoid duplication
+of verification before bdrv_co_preadv call.
+Signed-off-by: Edgar Kaziakhmedov <edgar.kaziakhmedov@virtuozzo.com>
+Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
+Reviewed-by: Eric Blake <eblake@redhat.com>
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+---
+ block/qcow2.h |  3 ---
+ block/qcow2.c | 51 ++++++++-------------------------------------------
+files changed, 8 insertions(+), 46 deletions(-)
+diff --git a/block/qcow2.h b/block/qcow2.h
+index XXXXXXX..XXXXXXX 100644
+--- a/block/qcow2.h
++++ b/block/qcow2.h
+@@ -XXX,XX +XXX,XX @@ uint32_t offset_to_reftable_index(BDRVQcow2State *s, uint64_t offset)
+ }
+ /* qcow2.c functions */
+-int qcow2_backing_read1(BlockDriverState *bs, QEMUIOVector *qiov,
+-                  int64_t sector_num, int nb_sectors);
+-
+ int64_t qcow2_refcount_metadata_size(int64_t clusters, size_t cluster_size,
+                                      int refcount_order, bool generous_increase,
+                                      uint64_t *refblock_count);
+diff --git a/block/qcow2.c b/block/qcow2.c
+index XXXXXXX..XXXXXXX 100644
+--- a/block/qcow2.c
++++ b/block/qcow2.c
+@@ -XXX,XX +XXX,XX @@ static int64_t coroutine_fn qcow2_co_get_block_status(BlockDriverState *bs,
+     return status;
+ }
+-/* handle reading after the end of the backing file */
+-int qcow2_backing_read1(BlockDriverState *bs, QEMUIOVector *qiov,
+-                        int64_t offset, int bytes)
+-{
+-    uint64_t bs_size = bs->total_sectors * BDRV_SECTOR_SIZE;
+-    int n1;
+-
+-    if ((offset + bytes) <= bs_size) {
+-        return bytes;
+-    }
+-
+-    if (offset >= bs_size) {
+-        n1 = 0;
+-    } else {
+-        n1 = bs_size - offset;
+-    }
+-
+-    qemu_iovec_memset(qiov, n1, 0, bytes - n1);
+-
+-    return n1;
+-}
+-
+ static coroutine_fn int qcow2_co_preadv(BlockDriverState *bs, uint64_t offset,
+                                         uint64_t bytes, QEMUIOVector *qiov,
+                                         int flags)
+ {
+     BDRVQcow2State *s = bs->opaque;
+-    int offset_in_cluster, n1;
++    int offset_in_cluster;
+     int ret;
+     unsigned int cur_bytes; /* number of bytes in current iteration */
+     uint64_t cluster_offset = 0;
+@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_preadv(BlockDriverState *bs, uint64_t offset,
+         case QCOW2_CLUSTER_UNALLOCATED:
+             if (bs->backing) {
+-                /* read from the base image */
+-                n1 = qcow2_backing_read1(bs->backing->bs, &hd_qiov,
+-                                         offset, cur_bytes);
+-                if (n1 > 0) {
+-                    QEMUIOVector local_qiov;
+-
+-                    qemu_iovec_init(&local_qiov, hd_qiov.niov);
+-                    qemu_iovec_concat(&local_qiov, &hd_qiov, 0, n1);
+-
+-                    BLKDBG_EVENT(bs->file, BLKDBG_READ_BACKING_AIO);
+-                    qemu_co_mutex_unlock(&s->lock);
+-                    ret = bdrv_co_preadv(bs->backing, offset, n1,
+-                                         &local_qiov, 0);
+-                    qemu_co_mutex_lock(&s->lock);
+-
+-                    qemu_iovec_destroy(&local_qiov);
+-
+-                    if (ret < 0) {
+-                        goto fail;
+-                    }
++                BLKDBG_EVENT(bs->file, BLKDBG_READ_BACKING_AIO);
++                qemu_co_mutex_unlock(&s->lock);
++                ret = bdrv_co_preadv(bs->backing, offset, cur_bytes,
++                                     &hd_qiov, 0);
++                qemu_co_mutex_lock(&s->lock);
++                if (ret < 0) {
++                    goto fail;
+                 }
+             } else {
+                 /* Note: in this case, no need to wait */
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 11/35] block: Document that x-blockdev-change breaks quorum children list
+Removing a quorum child node with x-blockdev-change results in a quorum
+driver state that cannot be recreated with create options because it
+would require a list with gaps. This causes trouble in at least
+.bdrv_refresh_filename().
+Document this problem so that we won't accidentally mark the command
+stable without having addressed it.
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+Reviewed-by: Alberto Garcia <berto@igalia.com>
+---
+ qapi/block-core.json | 4 ++++
+file changed, 4 insertions(+)
+diff --git a/qapi/block-core.json b/qapi/block-core.json
+index XXXXXXX..XXXXXXX 100644
+--- a/qapi/block-core.json
++++ b/qapi/block-core.json
+@@ -XXX,XX +XXX,XX @@
+ # does not support all kinds of operations, all kinds of children, nor
+ # all block drivers.
+ #
++# FIXME Removing children from a quorum node means introducing gaps in the
++# child indices. This cannot be represented in the 'children' list of
++# BlockdevOptionsQuorum, as returned by .bdrv_refresh_filename().
++#
+ # Warning: The data in a new quorum child MUST be consistent with that of
+ # the rest of the array.
+ #
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 12/35] nvme: Add tracing
+From: Doug Gale <doug16k@gmail.com>
+Add trace output for commands, errors, and undefined behavior.
+Add guest error log output for undefined behavior.
+Report invalid undefined accesses to MMIO.
+Annotate unlikely error checks with unlikely.
+Signed-off-by: Doug Gale <doug16k@gmail.com>
+Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
+Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+---
+ hw/block/nvme.c       | 349 ++++++++++++++++++++++++++++++++++++++++++--------
+ hw/block/trace-events |  93 ++++++++++++++
+files changed, 390 insertions(+), 52 deletions(-)
+diff --git a/hw/block/nvme.c b/hw/block/nvme.c
+index XXXXXXX..XXXXXXX 100644
+--- a/hw/block/nvme.c
++++ b/hw/block/nvme.c
+@@ -XXX,XX +XXX,XX @@
+ #include "qapi/visitor.h"
+ #include "sysemu/block-backend.h"
++#include "qemu/log.h"
++#include "trace.h"
+ #include "nvme.h"
++#define NVME_GUEST_ERR(trace, fmt, ...) \
++    do { \
++        (trace_##trace)(__VA_ARGS__); \
++        qemu_log_mask(LOG_GUEST_ERROR, #trace \
++            " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
++    } while (0)
++
+ static void nvme_process_sq(void *opaque);
+ static void nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size)
+@@ -XXX,XX +XXX,XX @@ static void nvme_isr_notify(NvmeCtrl *n, NvmeCQueue *cq)
+ {
+     if (cq->irq_enabled) {
+         if (msix_enabled(&(n->parent_obj))) {
++            trace_nvme_irq_msix(cq->vector);
+             msix_notify(&(n->parent_obj), cq->vector);
+         } else {
++            trace_nvme_irq_pin();
+             pci_irq_pulse(&n->parent_obj);
+         }
++    } else {
++        trace_nvme_irq_masked();
+     }
+ }
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1,
+     trans_len = MIN(len, trans_len);
+     int num_prps = (len >> n->page_bits) + 1;
+-    if (!prp1) {
++    if (unlikely(!prp1)) {
++        trace_nvme_err_invalid_prp();
+         return NVME_INVALID_FIELD | NVME_DNR;
+     } else if (n->cmbsz && prp1 >= n->ctrl_mem.addr &&
+                prp1 < n->ctrl_mem.addr + int128_get64(n->ctrl_mem.size)) {
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1,
+     }
+     len -= trans_len;
+     if (len) {
+-        if (!prp2) {
++        if (unlikely(!prp2)) {
++            trace_nvme_err_invalid_prp2_missing();
+             goto unmap;
+         }
+         if (len > n->page_size) {
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1,
+                 uint64_t prp_ent = le64_to_cpu(prp_list[i]);
+                 if (i == n->max_prp_ents - 1 && len > n->page_size) {
+-                    if (!prp_ent || prp_ent & (n->page_size - 1)) {
++                    if (unlikely(!prp_ent || prp_ent & (n->page_size - 1))) {
++                        trace_nvme_err_invalid_prplist_ent(prp_ent);
+                         goto unmap;
+                     }
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1,
+                     prp_ent = le64_to_cpu(prp_list[i]);
+                 }
+-                if (!prp_ent || prp_ent & (n->page_size - 1)) {
++                if (unlikely(!prp_ent || prp_ent & (n->page_size - 1))) {
++                    trace_nvme_err_invalid_prplist_ent(prp_ent);
+                     goto unmap;
+                 }
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1,
+                 i++;
+             }
+         } else {
+-            if (prp2 & (n->page_size - 1)) {
++            if (unlikely(prp2 & (n->page_size - 1))) {
++                trace_nvme_err_invalid_prp2_align(prp2);
+                 goto unmap;
+             }
+             if (qsg->nsg) {
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_dma_read_prp(NvmeCtrl *n, uint8_t *ptr, uint32_t len,
+     QEMUIOVector iov;
+     uint16_t status = NVME_SUCCESS;
++    trace_nvme_dma_read(prp1, prp2);
++
+     if (nvme_map_prp(&qsg, &iov, prp1, prp2, len, n)) {
+         return NVME_INVALID_FIELD | NVME_DNR;
+     }
+     if (qsg.nsg > 0) {
+-        if (dma_buf_read(ptr, len, &qsg)) {
++        if (unlikely(dma_buf_read(ptr, len, &qsg))) {
++            trace_nvme_err_invalid_dma();
+             status = NVME_INVALID_FIELD | NVME_DNR;
+         }
+         qemu_sglist_destroy(&qsg);
+     } else {
+-        if (qemu_iovec_to_buf(&iov, 0, ptr, len) != len) {
++        if (unlikely(qemu_iovec_to_buf(&iov, 0, ptr, len) != len)) {
++            trace_nvme_err_invalid_dma();
+             status = NVME_INVALID_FIELD | NVME_DNR;
+         }
+         qemu_iovec_destroy(&iov);
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_write_zeros(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
+     uint64_t aio_slba = slba << (data_shift - BDRV_SECTOR_BITS);
+     uint32_t aio_nlb = nlb << (data_shift - BDRV_SECTOR_BITS);
+-    if (slba + nlb > ns->id_ns.nsze) {
++    if (unlikely(slba + nlb > ns->id_ns.nsze)) {
++        trace_nvme_err_invalid_lba_range(slba, nlb, ns->id_ns.nsze);
+         return NVME_LBA_RANGE | NVME_DNR;
+     }
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_rw(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
+     int is_write = rw->opcode == NVME_CMD_WRITE ? 1 : 0;
+     enum BlockAcctType acct = is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ;
+-    if ((slba + nlb) > ns->id_ns.nsze) {
++    trace_nvme_rw(is_write ? "write" : "read", nlb, data_size, slba);
++
++    if (unlikely((slba + nlb) > ns->id_ns.nsze)) {
+         block_acct_invalid(blk_get_stats(n->conf.blk), acct);
++        trace_nvme_err_invalid_lba_range(slba, nlb, ns->id_ns.nsze);
+         return NVME_LBA_RANGE | NVME_DNR;
+     }
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+     NvmeNamespace *ns;
+     uint32_t nsid = le32_to_cpu(cmd->nsid);
+-    if (nsid == 0 || nsid > n->num_namespaces) {
++    if (unlikely(nsid == 0 || nsid > n->num_namespaces)) {
++        trace_nvme_err_invalid_ns(nsid, n->num_namespaces);
+         return NVME_INVALID_NSID | NVME_DNR;
+     }
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+     case NVME_CMD_READ:
+         return nvme_rw(n, ns, cmd, req);
+     default:
++        trace_nvme_err_invalid_opc(cmd->opcode);
+         return NVME_INVALID_OPCODE | NVME_DNR;
+     }
+ }
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeCmd *cmd)
+     NvmeCQueue *cq;
+     uint16_t qid = le16_to_cpu(c->qid);
+-    if (!qid || nvme_check_sqid(n, qid)) {
++    if (unlikely(!qid || nvme_check_sqid(n, qid))) {
++        trace_nvme_err_invalid_del_sq(qid);
+         return NVME_INVALID_QID | NVME_DNR;
+     }
++    trace_nvme_del_sq(qid);
++
+     sq = n->sq[qid];
+     while (!QTAILQ_EMPTY(&sq->out_req_list)) {
+         req = QTAILQ_FIRST(&sq->out_req_list);
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeCmd *cmd)
+     uint16_t qflags = le16_to_cpu(c->sq_flags);
+     uint64_t prp1 = le64_to_cpu(c->prp1);
+-    if (!cqid || nvme_check_cqid(n, cqid)) {
++    trace_nvme_create_sq(prp1, sqid, cqid, qsize, qflags);
++
++    if (unlikely(!cqid || nvme_check_cqid(n, cqid))) {
++        trace_nvme_err_invalid_create_sq_cqid(cqid);
+         return NVME_INVALID_CQID | NVME_DNR;
+     }
+-    if (!sqid || !nvme_check_sqid(n, sqid)) {
++    if (unlikely(!sqid || !nvme_check_sqid(n, sqid))) {
++        trace_nvme_err_invalid_create_sq_sqid(sqid);
+         return NVME_INVALID_QID | NVME_DNR;
+     }
+-    if (!qsize || qsize > NVME_CAP_MQES(n->bar.cap)) {
++    if (unlikely(!qsize || qsize > NVME_CAP_MQES(n->bar.cap))) {
++        trace_nvme_err_invalid_create_sq_size(qsize);
+         return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
+     }
+-    if (!prp1 || prp1 & (n->page_size - 1)) {
++    if (unlikely(!prp1 || prp1 & (n->page_size - 1))) {
++        trace_nvme_err_invalid_create_sq_addr(prp1);
+         return NVME_INVALID_FIELD | NVME_DNR;
+     }
+-    if (!(NVME_SQ_FLAGS_PC(qflags))) {
++    if (unlikely(!(NVME_SQ_FLAGS_PC(qflags)))) {
++        trace_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags));
+         return NVME_INVALID_FIELD | NVME_DNR;
+     }
+     sq = g_malloc0(sizeof(*sq));
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeCmd *cmd)
+     NvmeCQueue *cq;
+     uint16_t qid = le16_to_cpu(c->qid);
+-    if (!qid || nvme_check_cqid(n, qid)) {
++    if (unlikely(!qid || nvme_check_cqid(n, qid))) {
++        trace_nvme_err_invalid_del_cq_cqid(qid);
+         return NVME_INVALID_CQID | NVME_DNR;
+     }
+     cq = n->cq[qid];
+-    if (!QTAILQ_EMPTY(&cq->sq_list)) {
++    if (unlikely(!QTAILQ_EMPTY(&cq->sq_list))) {
++        trace_nvme_err_invalid_del_cq_notempty(qid);
+         return NVME_INVALID_QUEUE_DEL;
+     }
++    trace_nvme_del_cq(qid);
+     nvme_free_cq(cq, n);
+     return NVME_SUCCESS;
+ }
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeCmd *cmd)
+     uint16_t qflags = le16_to_cpu(c->cq_flags);
+     uint64_t prp1 = le64_to_cpu(c->prp1);
+-    if (!cqid || !nvme_check_cqid(n, cqid)) {
++    trace_nvme_create_cq(prp1, cqid, vector, qsize, qflags,
++                         NVME_CQ_FLAGS_IEN(qflags) != 0);
++
++    if (unlikely(!cqid || !nvme_check_cqid(n, cqid))) {
++        trace_nvme_err_invalid_create_cq_cqid(cqid);
+         return NVME_INVALID_CQID | NVME_DNR;
+     }
+-    if (!qsize || qsize > NVME_CAP_MQES(n->bar.cap)) {
++    if (unlikely(!qsize || qsize > NVME_CAP_MQES(n->bar.cap))) {
++        trace_nvme_err_invalid_create_cq_size(qsize);
+         return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
+     }
+-    if (!prp1) {
++    if (unlikely(!prp1)) {
++        trace_nvme_err_invalid_create_cq_addr(prp1);
+         return NVME_INVALID_FIELD | NVME_DNR;
+     }
+-    if (vector > n->num_queues) {
++    if (unlikely(vector > n->num_queues)) {
++        trace_nvme_err_invalid_create_cq_vector(vector);
+         return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
+     }
+-    if (!(NVME_CQ_FLAGS_PC(qflags))) {
++    if (unlikely(!(NVME_CQ_FLAGS_PC(qflags)))) {
++        trace_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags));
+         return NVME_INVALID_FIELD | NVME_DNR;
+     }
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeIdentify *c)
+     uint64_t prp1 = le64_to_cpu(c->prp1);
+     uint64_t prp2 = le64_to_cpu(c->prp2);
++    trace_nvme_identify_ctrl();
++
+     return nvme_dma_read_prp(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl),
+         prp1, prp2);
+ }
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeIdentify *c)
+     uint64_t prp1 = le64_to_cpu(c->prp1);
+     uint64_t prp2 = le64_to_cpu(c->prp2);
+-    if (nsid == 0 || nsid > n->num_namespaces) {
++    trace_nvme_identify_ns(nsid);
++
++    if (unlikely(nsid == 0 || nsid > n->num_namespaces)) {
++        trace_nvme_err_invalid_ns(nsid, n->num_namespaces);
+         return NVME_INVALID_NSID | NVME_DNR;
+     }
+     ns = &n->namespaces[nsid - 1];
++
+     return nvme_dma_read_prp(n, (uint8_t *)&ns->id_ns, sizeof(ns->id_ns),
+         prp1, prp2);
+ }
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeIdentify *c)
+     uint16_t ret;
+     int i, j = 0;
++    trace_nvme_identify_nslist(min_nsid);
++
+     list = g_malloc0(data_len);
+     for (i = 0; i < n->num_namespaces; i++) {
+         if (i < min_nsid) {
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_identify(NvmeCtrl *n, NvmeCmd *cmd)
+     case 0x02:
+         return nvme_identify_nslist(n, c);
+     default:
++        trace_nvme_err_invalid_identify_cns(le32_to_cpu(c->cns));
+         return NVME_INVALID_FIELD | NVME_DNR;
+     }
+ }
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+     switch (dw10) {
+     case NVME_VOLATILE_WRITE_CACHE:
+         result = blk_enable_write_cache(n->conf.blk);
++        trace_nvme_getfeat_vwcache(result ? "enabled" : "disabled");
+         break;
+     case NVME_NUMBER_OF_QUEUES:
+         result = cpu_to_le32((n->num_queues - 2) | ((n->num_queues - 2) << 16));
++        trace_nvme_getfeat_numq(result);
+         break;
+     default:
++        trace_nvme_err_invalid_getfeat(dw10);
+         return NVME_INVALID_FIELD | NVME_DNR;
+     }
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+         blk_set_enable_write_cache(n->conf.blk, dw11 & 1);
+         break;
+     case NVME_NUMBER_OF_QUEUES:
++        trace_nvme_setfeat_numq((dw11 & 0xFFFF) + 1,
++                                ((dw11 >> 16) & 0xFFFF) + 1,
++                                n->num_queues - 1, n->num_queues - 1);
+         req->cqe.result =
+             cpu_to_le32((n->num_queues - 2) | ((n->num_queues - 2) << 16));
+         break;
+     default:
++        trace_nvme_err_invalid_setfeat(dw10);
+         return NVME_INVALID_FIELD | NVME_DNR;
+     }
+     return NVME_SUCCESS;
+@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
+     case NVME_ADM_CMD_GET_FEATURES:
+         return nvme_get_feature(n, cmd, req);
+     default:
++        trace_nvme_err_invalid_admin_opc(cmd->opcode);
+         return NVME_INVALID_OPCODE | NVME_DNR;
+     }
+ }
+@@ -XXX,XX +XXX,XX @@ static int nvme_start_ctrl(NvmeCtrl *n)
+     uint32_t page_bits = NVME_CC_MPS(n->bar.cc) + 12;
+     uint32_t page_size = 1 << page_bits;
+-    if (n->cq[0] || n->sq[0] || !n->bar.asq || !n->bar.acq ||
+-            n->bar.asq & (page_size - 1) || n->bar.acq & (page_size - 1) ||
+-            NVME_CC_MPS(n->bar.cc) < NVME_CAP_MPSMIN(n->bar.cap) ||
+-            NVME_CC_MPS(n->bar.cc) > NVME_CAP_MPSMAX(n->bar.cap) ||
+-            NVME_CC_IOCQES(n->bar.cc) < NVME_CTRL_CQES_MIN(n->id_ctrl.cqes) ||
+-            NVME_CC_IOCQES(n->bar.cc) > NVME_CTRL_CQES_MAX(n->id_ctrl.cqes) ||
+-            NVME_CC_IOSQES(n->bar.cc) < NVME_CTRL_SQES_MIN(n->id_ctrl.sqes) ||
+-            NVME_CC_IOSQES(n->bar.cc) > NVME_CTRL_SQES_MAX(n->id_ctrl.sqes) ||
+-            !NVME_AQA_ASQS(n->bar.aqa) || !NVME_AQA_ACQS(n->bar.aqa)) {
++    if (unlikely(n->cq[0])) {
++        trace_nvme_err_startfail_cq();
++        return -1;
++    }
++    if (unlikely(n->sq[0])) {
++        trace_nvme_err_startfail_sq();
++        return -1;
++    }
++    if (unlikely(!n->bar.asq)) {
++        trace_nvme_err_startfail_nbarasq();
++        return -1;
++    }
++    if (unlikely(!n->bar.acq)) {
++        trace_nvme_err_startfail_nbaracq();
++        return -1;
++    }
++    if (unlikely(n->bar.asq & (page_size - 1))) {
++        trace_nvme_err_startfail_asq_misaligned(n->bar.asq);
++        return -1;
++    }
++    if (unlikely(n->bar.acq & (page_size - 1))) {
++        trace_nvme_err_startfail_acq_misaligned(n->bar.acq);
++        return -1;
++    }
++    if (unlikely(NVME_CC_MPS(n->bar.cc) <
++                 NVME_CAP_MPSMIN(n->bar.cap))) {
++        trace_nvme_err_startfail_page_too_small(
++                    NVME_CC_MPS(n->bar.cc),
++                    NVME_CAP_MPSMIN(n->bar.cap));
++        return -1;
++    }
++    if (unlikely(NVME_CC_MPS(n->bar.cc) >
++                 NVME_CAP_MPSMAX(n->bar.cap))) {
++        trace_nvme_err_startfail_page_too_large(
++                    NVME_CC_MPS(n->bar.cc),
++                    NVME_CAP_MPSMAX(n->bar.cap));
++        return -1;
++    }
++    if (unlikely(NVME_CC_IOCQES(n->bar.cc) <
++                 NVME_CTRL_CQES_MIN(n->id_ctrl.cqes))) {
++        trace_nvme_err_startfail_cqent_too_small(
++                    NVME_CC_IOCQES(n->bar.cc),
++                    NVME_CTRL_CQES_MIN(n->bar.cap));
++        return -1;
++    }
++    if (unlikely(NVME_CC_IOCQES(n->bar.cc) >
++                 NVME_CTRL_CQES_MAX(n->id_ctrl.cqes))) {
++        trace_nvme_err_startfail_cqent_too_large(
++                    NVME_CC_IOCQES(n->bar.cc),
++                    NVME_CTRL_CQES_MAX(n->bar.cap));
++        return -1;
++    }
++    if (unlikely(NVME_CC_IOSQES(n->bar.cc) <
++                 NVME_CTRL_SQES_MIN(n->id_ctrl.sqes))) {
++        trace_nvme_err_startfail_sqent_too_small(
++                    NVME_CC_IOSQES(n->bar.cc),
++                    NVME_CTRL_SQES_MIN(n->bar.cap));
++        return -1;
++    }
++    if (unlikely(NVME_CC_IOSQES(n->bar.cc) >
++                 NVME_CTRL_SQES_MAX(n->id_ctrl.sqes))) {
++        trace_nvme_err_startfail_sqent_too_large(
++                    NVME_CC_IOSQES(n->bar.cc),
++                    NVME_CTRL_SQES_MAX(n->bar.cap));
++        return -1;
++    }
++    if (unlikely(!NVME_AQA_ASQS(n->bar.aqa))) {
++        trace_nvme_err_startfail_asqent_sz_zero();
++        return -1;
++    }
++    if (unlikely(!NVME_AQA_ACQS(n->bar.aqa))) {
++        trace_nvme_err_startfail_acqent_sz_zero();
+         return -1;
+     }
+@@ -XXX,XX +XXX,XX @@ static int nvme_start_ctrl(NvmeCtrl *n)
+ static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
+     unsigned size)
+ {
++    if (unlikely(offset & (sizeof(uint32_t) - 1))) {
++        NVME_GUEST_ERR(nvme_ub_mmiowr_misaligned32,
++                       "MMIO write not 32-bit aligned,"
++                       " offset=0x%"PRIx64"", offset);
++        /* should be ignored, fall through for now */
++    }
++
++    if (unlikely(size < sizeof(uint32_t))) {
++        NVME_GUEST_ERR(nvme_ub_mmiowr_toosmall,
++                       "MMIO write smaller than 32-bits,"
++                       " offset=0x%"PRIx64", size=%u",
++                       offset, size);
++        /* should be ignored, fall through for now */
++    }
++
+     switch (offset) {
+-    case 0xc:
++    case 0xc:   /* INTMS */
++        if (unlikely(msix_enabled(&(n->parent_obj)))) {
++            NVME_GUEST_ERR(nvme_ub_mmiowr_intmask_with_msix,
++                           "undefined access to interrupt mask set"
++                           " when MSI-X is enabled");
++            /* should be ignored, fall through for now */
++        }
+         n->bar.intms |= data & 0xffffffff;
+         n->bar.intmc = n->bar.intms;
++        trace_nvme_mmio_intm_set(data & 0xffffffff,
++                                 n->bar.intmc);
+         break;
+-    case 0x10:
++    case 0x10:  /* INTMC */
++        if (unlikely(msix_enabled(&(n->parent_obj)))) {
++            NVME_GUEST_ERR(nvme_ub_mmiowr_intmask_with_msix,
++                           "undefined access to interrupt mask clr"
++                           " when MSI-X is enabled");
++            /* should be ignored, fall through for now */
++        }
+         n->bar.intms &= ~(data & 0xffffffff);
+         n->bar.intmc = n->bar.intms;
++        trace_nvme_mmio_intm_clr(data & 0xffffffff,
++                                 n->bar.intmc);
+         break;
+-    case 0x14:
++    case 0x14:  /* CC */
++        trace_nvme_mmio_cfg(data & 0xffffffff);
+         /* Windows first sends data, then sends enable bit */
+         if (!NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc) &&
+             !NVME_CC_SHN(data) && !NVME_CC_SHN(n->bar.cc))
+@@ -XXX,XX +XXX,XX @@ static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
+         if (NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc)) {
+             n->bar.cc = data;
+-            if (nvme_start_ctrl(n)) {
++            if (unlikely(nvme_start_ctrl(n))) {
++                trace_nvme_err_startfail();
+                 n->bar.csts = NVME_CSTS_FAILED;
+             } else {
++                trace_nvme_mmio_start_success();
+                 n->bar.csts = NVME_CSTS_READY;
+             }
+         } else if (!NVME_CC_EN(data) && NVME_CC_EN(n->bar.cc)) {
++            trace_nvme_mmio_stopped();
+             nvme_clear_ctrl(n);
+             n->bar.csts &= ~NVME_CSTS_READY;
+         }
+         if (NVME_CC_SHN(data) && !(NVME_CC_SHN(n->bar.cc))) {
+-                nvme_clear_ctrl(n);
+-                n->bar.cc = data;
+-                n->bar.csts |= NVME_CSTS_SHST_COMPLETE;
++            trace_nvme_mmio_shutdown_set();
++            nvme_clear_ctrl(n);
++            n->bar.cc = data;
++            n->bar.csts |= NVME_CSTS_SHST_COMPLETE;
+         } else if (!NVME_CC_SHN(data) && NVME_CC_SHN(n->bar.cc)) {
+-                n->bar.csts &= ~NVME_CSTS_SHST_COMPLETE;
+-                n->bar.cc = data;
++            trace_nvme_mmio_shutdown_cleared();
++            n->bar.csts &= ~NVME_CSTS_SHST_COMPLETE;
++            n->bar.cc = data;
++        }
++        break;
++    case 0x1C:  /* CSTS */
++        if (data & (1 << 4)) {
++            NVME_GUEST_ERR(nvme_ub_mmiowr_ssreset_w1c_unsupported,
++                           "attempted to W1C CSTS.NSSRO"
++                           " but CAP.NSSRS is zero (not supported)");
++        } else if (data != 0) {
++            NVME_GUEST_ERR(nvme_ub_mmiowr_ro_csts,
++                           "attempted to set a read only bit"
++                           " of controller status");
++        }
++        break;
++    case 0x20:  /* NSSR */
++        if (data == 0x4E564D65) {
++            trace_nvme_ub_mmiowr_ssreset_unsupported();
++        } else {
++            /* The spec says that writes of other values have no effect */
++            return;
+         }
+         break;
+-    case 0x24:
++    case 0x24:  /* AQA */
+         n->bar.aqa = data & 0xffffffff;
++        trace_nvme_mmio_aqattr(data & 0xffffffff);
+         break;
+-    case 0x28:
++    case 0x28:  /* ASQ */
+         n->bar.asq = data;
++        trace_nvme_mmio_asqaddr(data);
+         break;
+-    case 0x2c:
++    case 0x2c:  /* ASQ hi */
+         n->bar.asq |= data << 32;
++        trace_nvme_mmio_asqaddr_hi(data, n->bar.asq);
+         break;
+-    case 0x30:
++    case 0x30:  /* ACQ */
++        trace_nvme_mmio_acqaddr(data);
+         n->bar.acq = data;
+         break;
+-    case 0x34:
++    case 0x34:  /* ACQ hi */
+         n->bar.acq |= data << 32;
++        trace_nvme_mmio_acqaddr_hi(data, n->bar.acq);
+         break;
++    case 0x38:  /* CMBLOC */
++        NVME_GUEST_ERR(nvme_ub_mmiowr_cmbloc_reserved,
++                       "invalid write to reserved CMBLOC"
++                       " when CMBSZ is zero, ignored");
++        return;
++    case 0x3C:  /* CMBSZ */
++        NVME_GUEST_ERR(nvme_ub_mmiowr_cmbsz_readonly,
++                       "invalid write to read only CMBSZ, ignored");
++        return;
+     default:
++        NVME_GUEST_ERR(nvme_ub_mmiowr_invalid,
++                       "invalid MMIO write,"
++                       " offset=0x%"PRIx64", data=%"PRIx64"",
++                       offset, data);
+         break;
+     }
+ }
+@@ -XXX,XX +XXX,XX @@ static uint64_t nvme_mmio_read(void *opaque, hwaddr addr, unsigned size)
+     uint8_t *ptr = (uint8_t *)&n->bar;
+     uint64_t val = 0;
++    if (unlikely(addr & (sizeof(uint32_t) - 1))) {
++        NVME_GUEST_ERR(nvme_ub_mmiord_misaligned32,
++                       "MMIO read not 32-bit aligned,"
++                       " offset=0x%"PRIx64"", addr);
++        /* should RAZ, fall through for now */
++    } else if (unlikely(size < sizeof(uint32_t))) {
++        NVME_GUEST_ERR(nvme_ub_mmiord_toosmall,
++                       "MMIO read smaller than 32-bits,"
++                       " offset=0x%"PRIx64"", addr);
++        /* should RAZ, fall through for now */
++    }
++
+     if (addr < sizeof(n->bar)) {
+         memcpy(&val, ptr + addr, size);
++    } else {
++        NVME_GUEST_ERR(nvme_ub_mmiord_invalid_ofs,
++                       "MMIO read beyond last register,"
++                       " offset=0x%"PRIx64", returning 0", addr);
+     }
++
+     return val;
+ }
+@@ -XXX,XX +XXX,XX @@ static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
+ {
+     uint32_t qid;
+-    if (addr & ((1 << 2) - 1)) {
++    if (unlikely(addr & ((1 << 2) - 1))) {
++        NVME_GUEST_ERR(nvme_ub_db_wr_misaligned,
++                       "doorbell write not 32-bit aligned,"
++                       " offset=0x%"PRIx64", ignoring", addr);
+         return;
+     }
+     if (((addr - 0x1000) >> 2) & 1) {
++        /* Completion queue doorbell write */
++
+         uint16_t new_head = val & 0xffff;
+         int start_sqs;
+         NvmeCQueue *cq;
+         qid = (addr - (0x1000 + (1 << 2))) >> 3;
+-        if (nvme_check_cqid(n, qid)) {
++        if (unlikely(nvme_check_cqid(n, qid))) {
++            NVME_GUEST_ERR(nvme_ub_db_wr_invalid_cq,
++                           "completion queue doorbell write"
++                           " for nonexistent queue,"
++                           " sqid=%"PRIu32", ignoring", qid);
+             return;
+         }
+         cq = n->cq[qid];
+-        if (new_head >= cq->size) {
++        if (unlikely(new_head >= cq->size)) {
++            NVME_GUEST_ERR(nvme_ub_db_wr_invalid_cqhead,
++                           "completion queue doorbell write value"
++                           " beyond queue size, sqid=%"PRIu32","
++                           " new_head=%"PRIu16", ignoring",
++                           qid, new_head);
+             return;
+         }
+@@ -XXX,XX +XXX,XX @@ static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
+             nvme_isr_notify(n, cq);
+         }
+     } else {
++        /* Submission queue doorbell write */
++
+         uint16_t new_tail = val & 0xffff;
+         NvmeSQueue *sq;
+         qid = (addr - 0x1000) >> 3;
+-        if (nvme_check_sqid(n, qid)) {
++        if (unlikely(nvme_check_sqid(n, qid))) {
++            NVME_GUEST_ERR(nvme_ub_db_wr_invalid_sq,
++                           "submission queue doorbell write"
++                           " for nonexistent queue,"
++                           " sqid=%"PRIu32", ignoring", qid);
+             return;
+         }
+         sq = n->sq[qid];
+-        if (new_tail >= sq->size) {
++        if (unlikely(new_tail >= sq->size)) {
++            NVME_GUEST_ERR(nvme_ub_db_wr_invalid_sqtail,
++                           "submission queue doorbell write value"
++                           " beyond queue size, sqid=%"PRIu32","
++                           " new_tail=%"PRIu16", ignoring",
++                           qid, new_tail);
+             return;
+         }
+diff --git a/hw/block/trace-events b/hw/block/trace-events
+index XXXXXXX..XXXXXXX 100644
+--- a/hw/block/trace-events
++++ b/hw/block/trace-events
+@@ -XXX,XX +XXX,XX @@ virtio_blk_submit_multireq(void *vdev, void *mrb, int start, int num_reqs, uint6
+ hd_geometry_lchs_guess(void *blk, int cyls, int heads, int secs) "blk %p LCHS %d %d %d"
+ hd_geometry_guess(void *blk, uint32_t cyls, uint32_t heads, uint32_t secs, int trans) "blk %p CHS %u %u %u trans %d"
++# hw/block/nvme.c
++# nvme traces for successful events
++nvme_irq_msix(uint32_t vector) "raising MSI-X IRQ vector %u"
++nvme_irq_pin(void) "pulsing IRQ pin"
++nvme_irq_masked(void) "IRQ is masked"
++nvme_dma_read(uint64_t prp1, uint64_t prp2) "DMA read, prp1=0x%"PRIx64" prp2=0x%"PRIx64""
++nvme_rw(char const *verb, uint32_t blk_count, uint64_t byte_count, uint64_t lba) "%s %"PRIu32" blocks (%"PRIu64" bytes) from LBA %"PRIu64""
++nvme_create_sq(uint64_t addr, uint16_t sqid, uint16_t cqid, uint16_t qsize, uint16_t qflags) "create submission queue, addr=0x%"PRIx64", sqid=%"PRIu16", cqid=%"PRIu16", qsize=%"PRIu16", qflags=%"PRIu16""
++nvme_create_cq(uint64_t addr, uint16_t cqid, uint16_t vector, uint16_t size, uint16_t qflags, int ien) "create completion queue, addr=0x%"PRIx64", cqid=%"PRIu16", vector=%"PRIu16", qsize=%"PRIu16", qflags=%"PRIu16", ien=%d"
++nvme_del_sq(uint16_t qid) "deleting submission queue sqid=%"PRIu16""
++nvme_del_cq(uint16_t cqid) "deleted completion queue, sqid=%"PRIu16""
++nvme_identify_ctrl(void) "identify controller"
++nvme_identify_ns(uint16_t ns) "identify namespace, nsid=%"PRIu16""
++nvme_identify_nslist(uint16_t ns) "identify namespace list, nsid=%"PRIu16""
++nvme_getfeat_vwcache(char const* result) "get feature volatile write cache, result=%s"
++nvme_getfeat_numq(int result) "get feature number of queues, result=%d"
++nvme_setfeat_numq(int reqcq, int reqsq, int gotcq, int gotsq) "requested cq_count=%d sq_count=%d, responding with cq_count=%d sq_count=%d"
++nvme_mmio_intm_set(uint64_t data, uint64_t new_mask) "wrote MMIO, interrupt mask set, data=0x%"PRIx64", new_mask=0x%"PRIx64""
++nvme_mmio_intm_clr(uint64_t data, uint64_t new_mask) "wrote MMIO, interrupt mask clr, data=0x%"PRIx64", new_mask=0x%"PRIx64""
++nvme_mmio_cfg(uint64_t data) "wrote MMIO, config controller config=0x%"PRIx64""
++nvme_mmio_aqattr(uint64_t data) "wrote MMIO, admin queue attributes=0x%"PRIx64""
++nvme_mmio_asqaddr(uint64_t data) "wrote MMIO, admin submission queue address=0x%"PRIx64""
++nvme_mmio_acqaddr(uint64_t data) "wrote MMIO, admin completion queue address=0x%"PRIx64""
++nvme_mmio_asqaddr_hi(uint64_t data, uint64_t new_addr) "wrote MMIO, admin submission queue high half=0x%"PRIx64", new_address=0x%"PRIx64""
++nvme_mmio_acqaddr_hi(uint64_t data, uint64_t new_addr) "wrote MMIO, admin completion queue high half=0x%"PRIx64", new_address=0x%"PRIx64""
++nvme_mmio_start_success(void) "setting controller enable bit succeeded"
++nvme_mmio_stopped(void) "cleared controller enable bit"
++nvme_mmio_shutdown_set(void) "shutdown bit set"
++nvme_mmio_shutdown_cleared(void) "shutdown bit cleared"
++
++# nvme traces for error conditions
++nvme_err_invalid_dma(void) "PRP/SGL is too small for transfer size"
++nvme_err_invalid_prplist_ent(uint64_t prplist) "PRP list entry is null or not page aligned: 0x%"PRIx64""
++nvme_err_invalid_prp2_align(uint64_t prp2) "PRP2 is not page aligned: 0x%"PRIx64""
++nvme_err_invalid_prp2_missing(void) "PRP2 is null and more data to be transferred"
++nvme_err_invalid_field(void) "invalid field"
++nvme_err_invalid_prp(void) "invalid PRP"
++nvme_err_invalid_sgl(void) "invalid SGL"
++nvme_err_invalid_ns(uint32_t ns, uint32_t limit) "invalid namespace %u not within 1-%u"
++nvme_err_invalid_opc(uint8_t opc) "invalid opcode 0x%"PRIx8""
++nvme_err_invalid_admin_opc(uint8_t opc) "invalid admin opcode 0x%"PRIx8""
++nvme_err_invalid_lba_range(uint64_t start, uint64_t len, uint64_t limit) "Invalid LBA start=%"PRIu64" len=%"PRIu64" limit=%"PRIu64""
++nvme_err_invalid_del_sq(uint16_t qid) "invalid submission queue deletion, sid=%"PRIu16""
++nvme_err_invalid_create_sq_cqid(uint16_t cqid) "failed creating submission queue, invalid cqid=%"PRIu16""
++nvme_err_invalid_create_sq_sqid(uint16_t sqid) "failed creating submission queue, invalid sqid=%"PRIu16""
++nvme_err_invalid_create_sq_size(uint16_t qsize) "failed creating submission queue, invalid qsize=%"PRIu16""
++nvme_err_invalid_create_sq_addr(uint64_t addr) "failed creating submission queue, addr=0x%"PRIx64""
++nvme_err_invalid_create_sq_qflags(uint16_t qflags) "failed creating submission queue, qflags=%"PRIu16""
++nvme_err_invalid_del_cq_cqid(uint16_t cqid) "failed deleting completion queue, cqid=%"PRIu16""
++nvme_err_invalid_del_cq_notempty(uint16_t cqid) "failed deleting completion queue, it is not empty, cqid=%"PRIu16""
++nvme_err_invalid_create_cq_cqid(uint16_t cqid) "failed creating completion queue, cqid=%"PRIu16""
++nvme_err_invalid_create_cq_size(uint16_t size) "failed creating completion queue, size=%"PRIu16""
++nvme_err_invalid_create_cq_addr(uint64_t addr) "failed creating completion queue, addr=0x%"PRIx64""
++nvme_err_invalid_create_cq_vector(uint16_t vector) "failed creating completion queue, vector=%"PRIu16""
++nvme_err_invalid_create_cq_qflags(uint16_t qflags) "failed creating completion queue, qflags=%"PRIu16""
++nvme_err_invalid_identify_cns(uint16_t cns) "identify, invalid cns=0x%"PRIx16""
++nvme_err_invalid_getfeat(int dw10) "invalid get features, dw10=0x%"PRIx32""
++nvme_err_invalid_setfeat(uint32_t dw10) "invalid set features, dw10=0x%"PRIx32""
++nvme_err_startfail_cq(void) "nvme_start_ctrl failed because there are non-admin completion queues"
++nvme_err_startfail_sq(void) "nvme_start_ctrl failed because there are non-admin submission queues"
++nvme_err_startfail_nbarasq(void) "nvme_start_ctrl failed because the admin submission queue address is null"
++nvme_err_startfail_nbaracq(void) "nvme_start_ctrl failed because the admin completion queue address is null"
++nvme_err_startfail_asq_misaligned(uint64_t addr) "nvme_start_ctrl failed because the admin submission queue address is misaligned: 0x%"PRIx64""
++nvme_err_startfail_acq_misaligned(uint64_t addr) "nvme_start_ctrl failed because the admin completion queue address is misaligned: 0x%"PRIx64""
++nvme_err_startfail_page_too_small(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the page size is too small: log2size=%u, min=%u"
++nvme_err_startfail_page_too_large(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the page size is too large: log2size=%u, max=%u"
++nvme_err_startfail_cqent_too_small(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the completion queue entry size is too small: log2size=%u, min=%u"
++nvme_err_startfail_cqent_too_large(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the completion queue entry size is too large: log2size=%u, max=%u"
++nvme_err_startfail_sqent_too_small(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the submission queue entry size is too small: log2size=%u, min=%u"
++nvme_err_startfail_sqent_too_large(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the submission queue entry size is too large: log2size=%u, max=%u"
++nvme_err_startfail_asqent_sz_zero(void) "nvme_start_ctrl failed because the admin submission queue size is zero"
++nvme_err_startfail_acqent_sz_zero(void) "nvme_start_ctrl failed because the admin completion queue size is zero"
++nvme_err_startfail(void) "setting controller enable bit failed"
++
++# Traces for undefined behavior
++nvme_ub_mmiowr_misaligned32(uint64_t offset) "MMIO write not 32-bit aligned, offset=0x%"PRIx64""
++nvme_ub_mmiowr_toosmall(uint64_t offset, unsigned size) "MMIO write smaller than 32 bits, offset=0x%"PRIx64", size=%u"
++nvme_ub_mmiowr_intmask_with_msix(void) "undefined access to interrupt mask set when MSI-X is enabled"
++nvme_ub_mmiowr_ro_csts(void) "attempted to set a read only bit of controller status"
++nvme_ub_mmiowr_ssreset_w1c_unsupported(void) "attempted to W1C CSTS.NSSRO but CAP.NSSRS is zero (not supported)"
++nvme_ub_mmiowr_ssreset_unsupported(void) "attempted NVM subsystem reset but CAP.NSSRS is zero (not supported)"
++nvme_ub_mmiowr_cmbloc_reserved(void) "invalid write to reserved CMBLOC when CMBSZ is zero, ignored"
++nvme_ub_mmiowr_cmbsz_readonly(void) "invalid write to read only CMBSZ, ignored"
++nvme_ub_mmiowr_invalid(uint64_t offset, uint64_t data) "invalid MMIO write, offset=0x%"PRIx64", data=0x%"PRIx64""
++nvme_ub_mmiord_misaligned32(uint64_t offset) "MMIO read not 32-bit aligned, offset=0x%"PRIx64""
++nvme_ub_mmiord_toosmall(uint64_t offset) "MMIO read smaller than 32-bits, offset=0x%"PRIx64""
++nvme_ub_mmiord_invalid_ofs(uint64_t offset) "MMIO read beyond last register, offset=0x%"PRIx64", returning 0"
++nvme_ub_db_wr_misaligned(uint64_t offset) "doorbell write not 32-bit aligned, offset=0x%"PRIx64", ignoring"
++nvme_ub_db_wr_invalid_cq(uint32_t qid) "completion queue doorbell write for nonexistent queue, cqid=%"PRIu32", ignoring"
++nvme_ub_db_wr_invalid_cqhead(uint32_t qid, uint16_t new_head) "completion queue doorbell write value beyond queue size, cqid=%"PRIu32", new_head=%"PRIu16", ignoring"
++nvme_ub_db_wr_invalid_sq(uint32_t qid) "submission queue doorbell write for nonexistent queue, sqid=%"PRIu32", ignoring"
++nvme_ub_db_wr_invalid_sqtail(uint32_t qid, uint16_t new_tail) "submission queue doorbell write value beyond queue size, sqid=%"PRIu32", new_head=%"PRIu16", ignoring"
++
+ # hw/block/xen_disk.c
+ xen_disk_alloc(char *name) "%s"
+ xen_disk_init(char *name) "%s"
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 13/35] block: Open backing image in force share mode for size probe
+From: Fam Zheng <famz@redhat.com>
+Management tools create overlays of running guests with qemu-img:
+  $ qemu-img create -b /image/in/use.qcow2 -f qcow2 /overlay/image.qcow2
+but this doesn't work anymore due to image locking:
+    qemu-img: /overlay/image.qcow2: Failed to get shared "write" lock
+    Is another process using the image?
+    Could not open backing image to determine size.
+Use the force share option to allow this use case again.
+Cc: qemu-stable@nongnu.org
+Signed-off-by: Fam Zheng <famz@redhat.com>
+Reviewed-by: Eric Blake <eblake@redhat.com>
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+---
+ block.c | 3 ++-
+file changed, 2 insertions(+), 1 deletion(-)
+diff --git a/block.c b/block.c
+index XXXXXXX..XXXXXXX 100644
+--- a/block.c
++++ b/block.c
+@@ -XXX,XX +XXX,XX @@ void bdrv_img_create(const char *filename, const char *fmt,
+         back_flags = flags;
+         back_flags &= ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
++        backing_options = qdict_new();
+         if (backing_fmt) {
+-            backing_options = qdict_new();
+             qdict_put_str(backing_options, "driver", backing_fmt);
+         }
++        qdict_put_bool(backing_options, BDRV_OPT_FORCE_SHARE, true);
+         bs = bdrv_open(full_backing, NULL, backing_options, back_flags,
+                        &local_err);
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 14/35] block: Remove the obsolete -drive boot=on|off parameter
+From: Thomas Huth <thuth@redhat.com>
+It's not working anymore since QEMU v1.3.0 - time to remove it now.
+Signed-off-by: Thomas Huth <thuth@redhat.com>
+Reviewed-by: John Snow <jsnow@redhat.com>
+Reviewed-by: Markus Armbruster <armbru@redhat.com>
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+---
+ blockdev.c    | 11 -----------
+ qemu-doc.texi |  6 ------
+files changed, 17 deletions(-)
+diff --git a/blockdev.c b/blockdev.c
+index XXXXXXX..XXXXXXX 100644
+--- a/blockdev.c
++++ b/blockdev.c
+@@ -XXX,XX +XXX,XX @@ QemuOptsList qemu_legacy_drive_opts = {
+             .type = QEMU_OPT_STRING,
+             .help = "chs translation (auto, lba, none)",
+         },{
+-            .name = "boot",
+-            .type = QEMU_OPT_BOOL,
+-            .help = "(deprecated, ignored)",
+-        },{
+             .name = "addr",
+             .type = QEMU_OPT_STRING,
+             .help = "pci address (virtio only)",
+@@ -XXX,XX +XXX,XX @@ DriveInfo *drive_new(QemuOpts *all_opts, BlockInterfaceType block_default_type)
+         goto fail;
+     }
+-    /* Deprecated option boot=[on|off] */
+-    if (qemu_opt_get(legacy_opts, "boot") != NULL) {
+-        fprintf(stderr, "qemu-kvm: boot=on|off is deprecated and will be "
+-                "ignored. Future versions will reject this parameter. Please "
+-                "update your scripts.\n");
+-    }
+-
+     /* Other deprecated options */
+     if (!qtest_enabled()) {
+         for (i = 0; i < ARRAY_SIZE(deprecated); i++) {
+diff --git a/qemu-doc.texi b/qemu-doc.texi
+index XXXXXXX..XXXXXXX 100644
+--- a/qemu-doc.texi
++++ b/qemu-doc.texi
+@@ -XXX,XX +XXX,XX @@ deprecated.
+ @section System emulator command line arguments
+-@subsection -drive boot=on|off (since 1.3.0)
+-
+-The ``boot=on|off'' option to the ``-drive'' argument is
+-ignored. Applications should use the ``bootindex=N'' parameter
+-to set an absolute ordering between devices instead.
+-
+ @subsection -tdf (since 1.3.0)
+ The ``-tdf'' argument is ignored. The behaviour implemented
+--
+.13.6

-[Qemu-devel] [PULL 12/13] vl: Drain before (block) job cancel when quitting
+[Qemu-devel] [PULL v3 15/35] block: Remove the deprecated -hdachs option
-From: Max Reitz <mreitz@redhat.com>
+From: Thomas Huth <thuth@redhat.com>
-If the main loop cancels all block jobs while the block layer is not
+It's been marked as deprecated since QEMU v2.10.0, and so far nobody
-drained, this cancelling may not happen instantaneously.  We can start a
+complained that we should keep it, so let's remove this legacy option
-drained section before vm_shutdown(), which entails another
+now to simplify the code quite a bit.
-bdrv_drain_all(); this nested bdrv_drain_all() will thus be a no-op,
-basically.
+Signed-off-by: Thomas Huth <thuth@redhat.com>
+Reviewed-by: John Snow <jsnow@redhat.com>
-We do not have to end the drained section, because we actually do not
+Reviewed-by: Markus Armbruster <armbru@redhat.com>
 want any requests to happen from this point on.
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 Signed-off-by: Kevin Wolf <kwolf@redhat.com>
 ---
- vl.c | 11 +++++++++++
+ vl.c            | 86 ++-------------------------------------------------------
-file changed, 11 insertions(+)
+ qemu-doc.texi   |  8 ------
  qemu-options.hx | 19 ++-----------
 files changed, 4 insertions(+), 109 deletions(-)
 diff --git a/vl.c b/vl.c
 index XXXXXXX..XXXXXXX 100644
 --- a/vl.c
 +++ b/vl.c
 @@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
-      */
+     const char *boot_order = NULL;
-     migration_shutdown();
+     const char *boot_once = NULL;
+     DisplayState *ds;
-+    /*
+-    int cyls, heads, secs, translation;
-+     * We must cancel all block jobs while the block layer is drained,
+     QemuOpts *opts, *machine_opts;
-+     * or cancelling will be affected by throttling and thus may block
+-    QemuOpts *hda_opts = NULL, *icount_opts = NULL, *accel_opts = NULL;
-+     * for an extended period of time.
++    QemuOpts *icount_opts = NULL, *accel_opts = NULL;
-+     * vm_shutdown() will bdrv_drain_all(), so we may as well include
+     QemuOptsList *olist;
-+     * it in the drained section.
+     int optind;
-+     * We do not need to end this section, because we do not want any
+     const char *optarg;
-+     * requests happening from here on anyway.
+@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
-+     */
-+    bdrv_drain_all_begin();
+     cpu_model = NULL;
-+
+     snapshot = 0;
-     /* No more vcpu or device emulation activity beyond this point */
+-    cyls = heads = secs = 0;
-     vm_shutdown();
+-    translation = BIOS_ATA_TRANSLATION_AUTO;
      nb_nics = 0;
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
          if (optind >= argc)
              break;
          if (argv[optind][0] != '-') {
 -            hda_opts = drive_add(IF_DEFAULT, 0, argv[optind++], HD_OPTS);
 +            drive_add(IF_DEFAULT, 0, argv[optind++], HD_OPTS);
          } else {
              const QEMUOption *popt;
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
                  cpu_model = optarg;
                  break;
              case QEMU_OPTION_hda:
 -                {
 -                    char buf[256];
 -                    if (cyls == 0)
 -                        snprintf(buf, sizeof(buf), "%s", HD_OPTS);
 -                    else
 -                        snprintf(buf, sizeof(buf),
 -                                 "%s,cyls=%d,heads=%d,secs=%d%s",
 -                                 HD_OPTS , cyls, heads, secs,
 -                                 translation == BIOS_ATA_TRANSLATION_LBA ?
 -                                 ",trans=lba" :
 -                                 translation == BIOS_ATA_TRANSLATION_NONE ?
 -                                 ",trans=none" : "");
 -                    drive_add(IF_DEFAULT, 0, optarg, buf);
 -                    break;
 -                }
              case QEMU_OPTION_hdb:
              case QEMU_OPTION_hdc:
              case QEMU_OPTION_hdd:
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
              case QEMU_OPTION_snapshot:
                  snapshot = 1;
                  break;
 -            case QEMU_OPTION_hdachs:
 -                {
 -                    const char *p;
 -                    p = optarg;
 -                    cyls = strtol(p, (char **)&p, 0);
 -                    if (cyls < 1 || cyls > 16383)
 -                        goto chs_fail;
 -                    if (*p != ',')
 -                        goto chs_fail;
 -                    p++;
 -                    heads = strtol(p, (char **)&p, 0);
 -                    if (heads < 1 || heads > 16)
 -                        goto chs_fail;
 -                    if (*p != ',')
 -                        goto chs_fail;
 -                    p++;
 -                    secs = strtol(p, (char **)&p, 0);
 -                    if (secs < 1 || secs > 63)
 -                        goto chs_fail;
 -                    if (*p == ',') {
 -                        p++;
 -                        if (!strcmp(p, "large")) {
 -                            translation = BIOS_ATA_TRANSLATION_LARGE;
 -                        } else if (!strcmp(p, "rechs")) {
 -                            translation = BIOS_ATA_TRANSLATION_RECHS;
 -                        } else if (!strcmp(p, "none")) {
 -                            translation = BIOS_ATA_TRANSLATION_NONE;
 -                        } else if (!strcmp(p, "lba")) {
 -                            translation = BIOS_ATA_TRANSLATION_LBA;
 -                        } else if (!strcmp(p, "auto")) {
 -                            translation = BIOS_ATA_TRANSLATION_AUTO;
 -                        } else {
 -                            goto chs_fail;
 -                        }
 -                    } else if (*p != '\0') {
 -                    chs_fail:
 -                        error_report("invalid physical CHS format");
 -                        exit(1);
 -                    }
 -                    if (hda_opts != NULL) {
 -                        qemu_opt_set_number(hda_opts, "cyls", cyls,
 -                                            &error_abort);
 -                        qemu_opt_set_number(hda_opts, "heads", heads,
 -                                            &error_abort);
 -                        qemu_opt_set_number(hda_opts, "secs", secs,
 -                                            &error_abort);
 -                        if (translation == BIOS_ATA_TRANSLATION_LARGE) {
 -                            qemu_opt_set(hda_opts, "trans", "large",
 -                                         &error_abort);
 -                        } else if (translation == BIOS_ATA_TRANSLATION_RECHS) {
 -                            qemu_opt_set(hda_opts, "trans", "rechs",
 -                                         &error_abort);
 -                        } else if (translation == BIOS_ATA_TRANSLATION_LBA) {
 -                            qemu_opt_set(hda_opts, "trans", "lba",
 -                                         &error_abort);
 -                        } else if (translation == BIOS_ATA_TRANSLATION_NONE) {
 -                            qemu_opt_set(hda_opts, "trans", "none",
 -                                         &error_abort);
 -                        }
 -                    }
 -                }
 -                error_report("'-hdachs' is deprecated, please use '-device"
 -                             " ide-hd,cyls=c,heads=h,secs=s,...' instead");
 -                break;
              case QEMU_OPTION_numa:
                  opts = qemu_opts_parse_noisily(qemu_find_opts("numa"),
                                                 optarg, true);
 diff --git a/qemu-doc.texi b/qemu-doc.texi
 index XXXXXXX..XXXXXXX 100644
 --- a/qemu-doc.texi
 +++ b/qemu-doc.texi
@@ -XXX,XX +XXX,XX @@ The ``--net dump'' argument is now replaced with the
  ``-object filter-dump'' argument which works in combination
  with the modern ``-netdev`` backends instead.
 -@subsection -hdachs (since 2.10.0)
 -
 -The ``-hdachs'' argument is now a synonym for setting
 -the ``cyls'', ``heads'', ``secs'', and ``trans'' properties
 -on the ``ide-hd'' device using the ``-device'' argument.
 -The new syntax allows different settings to be provided
 -per disk.
 -
  @subsection -usbdevice (since 2.10.0)
  The ``-usbdevice DEV'' argument is now a synonym for setting
 diff --git a/qemu-options.hx b/qemu-options.hx
 index XXXXXXX..XXXXXXX 100644
 --- a/qemu-options.hx
 +++ b/qemu-options.hx
@@ -XXX,XX +XXX,XX @@ of available connectors of a given interface type.
  @item media=@var{media}
  This option defines the type of the media: disk or cdrom.
  @item cyls=@var{c},heads=@var{h},secs=@var{s}[,trans=@var{t}]
 -These options have the same definition as they have in @option{-hdachs}.
 -These parameters are deprecated, use the corresponding parameters
 +Force disk physical geometry and the optional BIOS translation (trans=none or
 +lba). These parameters are deprecated, use the corresponding parameters
  of @code{-device} instead.
  @item snapshot=@var{snapshot}
  @var{snapshot} is "on" or "off" and controls snapshot mode for the given drive
@@ -XXX,XX +XXX,XX @@ the raw disk image you use is not written back. You can however force
  the write back by pressing @key{C-a s} (@pxref{disk_images}).
  ETEXI
 -DEF("hdachs", HAS_ARG, QEMU_OPTION_hdachs, \
 -    "-hdachs c,h,s[,t]\n" \
 -    "                force hard disk 0 physical geometry and the optional BIOS\n" \
 -    "                translation (t=none or lba) (usually QEMU can guess them)\n",
 -    QEMU_ARCH_ALL)
 -STEXI
 -@item -hdachs @var{c},@var{h},@var{s},[,@var{t}]
 -@findex -hdachs
 -Force hard disk 0 physical geometry (1 <= @var{c} <= 16383, 1 <=
 -@var{h} <= 16, 1 <= @var{s} <= 63) and optionally force the BIOS
 -translation mode (@var{t}=none, lba or auto). Usually QEMU can guess
 -all those parameters. This option is deprecated, please use
 -@code{-device ide-hd,cyls=c,heads=h,secs=s,...} instead.
 -ETEXI
 -
  DEF("fsdev", HAS_ARG, QEMU_OPTION_fsdev,
      "-fsdev fsdriver,id=id[,path=path,][security_model={mapped-xattr|mapped-file|passthrough|none}]\n"
      " [,writeout=immediate][,readonly][,socket=socket|sock_fd=sock_fd][,fmode=fmode][,dmode=dmode]\n"
 --
-.20.1
+.13.6

-New patch
+[Qemu-devel] [PULL v3 16/35] block: Mention -drive cyls/heads/secs/trans/serial/addr in deprecation chapter
+From: Thomas Huth <thuth@redhat.com>
+Looks like we forgot to announce the deprecation of these options in
+the corresponding chapter of the qemu-doc text, so let's do that now.
+Signed-off-by: Thomas Huth <thuth@redhat.com>
+Reviewed-by: John Snow <jsnow@redhat.com>
+Reviewed-by: Markus Armbruster <armbru@redhat.com>
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+---
+ qemu-doc.texi | 15 +++++++++++++++
+file changed, 15 insertions(+)
+diff --git a/qemu-doc.texi b/qemu-doc.texi
+index XXXXXXX..XXXXXXX 100644
+--- a/qemu-doc.texi
++++ b/qemu-doc.texi
+@@ -XXX,XX +XXX,XX @@ longer be directly supported in QEMU.
+ The ``-drive if=scsi'' argument is replaced by the the
+ ``-device BUS-TYPE'' argument combined with ``-drive if=none''.
++@subsection -drive cyls=...,heads=...,secs=...,trans=... (since 2.10.0)
++
++The drive geometry arguments are replaced by the the geometry arguments
++that can be specified with the ``-device'' parameter.
++
++@subsection -drive serial=... (since 2.10.0)
++
++The drive serial argument is replaced by the the serial argument
++that can be specified with the ``-device'' parameter.
++
++@subsection -drive addr=... (since 2.10.0)
++
++The drive addr argument is replaced by the the addr argument
++that can be specified with the ``-device'' parameter.
++
+ @subsection -net dump (since 2.10.0)
+ The ``--net dump'' argument is now replaced with the
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 17/35] block: Remove unused bdrv_requests_pending
+From: Fam Zheng <famz@redhat.com>
+Signed-off-by: Fam Zheng <famz@redhat.com>
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+---
+ include/block/block_int.h |  1 -
+ block/io.c                | 18 ------------------
+files changed, 19 deletions(-)
+diff --git a/include/block/block_int.h b/include/block/block_int.h
+index XXXXXXX..XXXXXXX 100644
+--- a/include/block/block_int.h
++++ b/include/block/block_int.h
+@@ -XXX,XX +XXX,XX @@ bool blk_dev_is_tray_open(BlockBackend *blk);
+ bool blk_dev_is_medium_locked(BlockBackend *blk);
+ void bdrv_set_dirty(BlockDriverState *bs, int64_t offset, int64_t bytes);
+-bool bdrv_requests_pending(BlockDriverState *bs);
+ void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap **out);
+ void bdrv_undo_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap *in);
+diff --git a/block/io.c b/block/io.c
+index XXXXXXX..XXXXXXX 100644
+--- a/block/io.c
++++ b/block/io.c
+@@ -XXX,XX +XXX,XX @@ void bdrv_disable_copy_on_read(BlockDriverState *bs)
+     assert(old >= 1);
+ }
+-/* Check if any requests are in-flight (including throttled requests) */
+-bool bdrv_requests_pending(BlockDriverState *bs)
+-{
+-    BdrvChild *child;
+-
+-    if (atomic_read(&bs->in_flight)) {
+-        return true;
+-    }
+-
+-    QLIST_FOREACH(child, &bs->children, next) {
+-        if (bdrv_requests_pending(child->bs)) {
+-            return true;
+-        }
+-    }
+-
+-    return false;
+-}
+-
+ typedef struct {
+     Coroutine *co;
+     BlockDriverState *bs;
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 18/35] block: Assert drain_all is only called from main AioContext
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+Reviewed-by: Fam Zheng <famz@redhat.com>
+---
+ block/io.c | 6 ++++++
+file changed, 6 insertions(+)
+diff --git a/block/io.c b/block/io.c
+index XXXXXXX..XXXXXXX 100644
+--- a/block/io.c
++++ b/block/io.c
+@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
+     BdrvNextIterator it;
+     GSList *aio_ctxs = NULL, *ctx;
++    /* BDRV_POLL_WHILE() for a node can only be called from its own I/O thread
++     * or the main loop AioContext. We potentially use BDRV_POLL_WHILE() on
++     * nodes in several different AioContexts, so make sure we're in the main
++     * context. */
++    assert(qemu_get_current_aio_context() == qemu_get_aio_context());
++
+     block_job_pause_all();
+     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 19/35] block: Make bdrv_drain() driver callbacks non-recursive
+bdrv_drained_begin() doesn't increase bs->quiesce_counter recursively
+and also doesn't notify other parent nodes of children, which both means
+that the child nodes are not actually drained, and bdrv_drained_begin()
+is providing useful functionality only on a single node.
+To keep things consistent, we also shouldn't call the block driver
+callbacks recursively.
+A proper recursive drain version that provides an actually working
+drained section for child nodes will be introduced later.
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+Reviewed-by: Fam Zheng <famz@redhat.com>
+---
+ block/io.c | 16 +++++++++-------
+file changed, 9 insertions(+), 7 deletions(-)
+diff --git a/block/io.c b/block/io.c
+index XXXXXXX..XXXXXXX 100644
+--- a/block/io.c
++++ b/block/io.c
+@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
+ }
+ /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */
+-static void bdrv_drain_invoke(BlockDriverState *bs, bool begin)
++static void bdrv_drain_invoke(BlockDriverState *bs, bool begin, bool recursive)
+ {
+     BdrvChild *child, *tmp;
+     BdrvCoDrainData data = { .bs = bs, .done = false, .begin = begin};
+@@ -XXX,XX +XXX,XX @@ static void bdrv_drain_invoke(BlockDriverState *bs, bool begin)
+     bdrv_coroutine_enter(bs, data.co);
+     BDRV_POLL_WHILE(bs, !data.done);
+-    QLIST_FOREACH_SAFE(child, &bs->children, next, tmp) {
+-        bdrv_drain_invoke(child->bs, begin);
++    if (recursive) {
++        QLIST_FOREACH_SAFE(child, &bs->children, next, tmp) {
++            bdrv_drain_invoke(child->bs, begin, true);
++        }
+     }
+ }
+@@ -XXX,XX +XXX,XX @@ void bdrv_drained_begin(BlockDriverState *bs)
+         bdrv_parent_drained_begin(bs);
+     }
+-    bdrv_drain_invoke(bs, true);
++    bdrv_drain_invoke(bs, true, false);
+     bdrv_drain_recurse(bs);
+ }
+@@ -XXX,XX +XXX,XX @@ void bdrv_drained_end(BlockDriverState *bs)
+     }
+     /* Re-enable things in child-to-parent order */
+-    bdrv_drain_invoke(bs, false);
++    bdrv_drain_invoke(bs, false, false);
+     bdrv_parent_drained_end(bs);
+     aio_enable_external(bdrv_get_aio_context(bs));
+ }
+@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
+         aio_context_acquire(aio_context);
+         aio_disable_external(aio_context);
+         bdrv_parent_drained_begin(bs);
+-        bdrv_drain_invoke(bs, true);
++        bdrv_drain_invoke(bs, true, true);
+         aio_context_release(aio_context);
+         if (!g_slist_find(aio_ctxs, aio_context)) {
+@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_end(void)
+         /* Re-enable things in child-to-parent order */
+         aio_context_acquire(aio_context);
+-        bdrv_drain_invoke(bs, false);
++        bdrv_drain_invoke(bs, false, true);
+         bdrv_parent_drained_end(bs);
+         aio_enable_external(aio_context);
+         aio_context_release(aio_context);
+--
+.13.6

-[Qemu-devel] [PULL 06/13] tests: Lock AioContexts in test-block-iothread
+[Qemu-devel] [PULL v3 20/35] test-bdrv-drain: Test callback for bdrv_drain
-From: Max Reitz <mreitz@redhat.com>
+The existing test is for bdrv_drain_all_begin/end() only. Generalise the
 test case so that it can be run for the other variants as well. At the
 moment this is only bdrv_drain_begin/end(), but in a while, we'll add
 another one.
-When changing a node's AioContext, the caller must acquire the old
+Also, add a backing file to the test node to test whether the operations
-AioContext (unless it currently runs in that old context).  Therefore,
+work recursively.
 unless the node currently is in the main context, we always have to
 acquire the old context around calls that may change a node's
 AioContext.
-Signed-off-by: Max Reitz <mreitz@redhat.com>
 Signed-off-by: Kevin Wolf <kwolf@redhat.com>
 ---
- tests/test-block-iothread.c | 40 ++++++++++++++++++++++++-------------
+ tests/test-bdrv-drain.c | 69 ++++++++++++++++++++++++++++++++++++++++++++-----
-file changed, 26 insertions(+), 14 deletions(-)
+file changed, 62 insertions(+), 7 deletions(-)
-diff --git a/tests/test-block-iothread.c b/tests/test-block-iothread.c
+diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
 index XXXXXXX..XXXXXXX 100644
---- a/tests/test-block-iothread.c
+--- a/tests/test-bdrv-drain.c
-+++ b/tests/test-block-iothread.c
++++ b/tests/test-bdrv-drain.c
-@@ -XXX,XX +XXX,XX @@ static void test_sync_op(const void *opaque)
+@@ -XXX,XX +XXX,XX @@ static BlockDriver bdrv_test = {
-     if (t->blkfn) {
-         t->blkfn(blk);
+     .bdrv_co_drain_begin    = bdrv_test_co_drain_begin,
-     }
+     .bdrv_co_drain_end      = bdrv_test_co_drain_end,
--    aio_context_release(ctx);
++
-     blk_set_aio_context(blk, qemu_get_aio_context(), &error_abort);
++    .bdrv_child_perm        = bdrv_format_default_perms,
-+    aio_context_release(ctx);
+ };
-     bdrv_unref(bs);
+ static void aio_ret_cb(void *opaque, int ret)
-     blk_unref(blk);
+@@ -XXX,XX +XXX,XX @@ static void aio_ret_cb(void *opaque, int ret)
-@@ -XXX,XX +XXX,XX @@ static void test_propagate_basic(void)
+     *aio_ret = ret;
  }
 -static void test_drv_cb_drain_all(void)
 +enum drain_type {
 +    BDRV_DRAIN_ALL,
 +    BDRV_DRAIN,
 +};
 +
 +static void do_drain_begin(enum drain_type drain_type, BlockDriverState *bs)
 +{
 +    switch (drain_type) {
 +    case BDRV_DRAIN_ALL:        bdrv_drain_all_begin(); break;
 +    case BDRV_DRAIN:            bdrv_drained_begin(bs); break;
 +    default:                    g_assert_not_reached();
 +    }
 +}
 +
 +static void do_drain_end(enum drain_type drain_type, BlockDriverState *bs)
 +{
 +    switch (drain_type) {
 +    case BDRV_DRAIN_ALL:        bdrv_drain_all_end(); break;
 +    case BDRV_DRAIN:            bdrv_drained_end(bs); break;
 +    default:                    g_assert_not_reached();
 +    }
 +}
 +
 +static void test_drv_cb_common(enum drain_type drain_type, bool recursive)
  {
-     IOThread *iothread = iothread_new();
-     AioContext *ctx = iothread_get_aio_context(iothread);
-+    AioContext *main_ctx;
      BlockBackend *blk;
-     BlockDriverState *bs_a, *bs_b, *bs_verify;
+-    BlockDriverState *bs;
-     QDict *options;
+-    BDRVTestState *s;
-@@ -XXX,XX +XXX,XX @@ static void test_propagate_basic(void)
++    BlockDriverState *bs, *backing;
-     g_assert(bdrv_get_aio_context(bs_b) == ctx);
++    BDRVTestState *s, *backing_s;
+     BlockAIOCB *acb;
-     /* Switch the AioContext back */
+     int aio_ret;
--    ctx = qemu_get_aio_context();
--    blk_set_aio_context(blk, ctx, &error_abort);
+@@ -XXX,XX +XXX,XX @@ static void test_drv_cb_drain_all(void)
--    g_assert(blk_get_aio_context(blk) == ctx);
+     s = bs->opaque;
--    g_assert(bdrv_get_aio_context(bs_a) == ctx);
+     blk_insert_bs(blk, bs, &error_abort);
--    g_assert(bdrv_get_aio_context(bs_verify) == ctx);
--    g_assert(bdrv_get_aio_context(bs_b) == ctx);
++    backing = bdrv_new_open_driver(&bdrv_test, "backing", 0, &error_abort);
-+    main_ctx = qemu_get_aio_context();
++    backing_s = backing->opaque;
-+    aio_context_acquire(ctx);
++    bdrv_set_backing_hd(bs, backing, &error_abort);
-+    blk_set_aio_context(blk, main_ctx, &error_abort);
++
-+    aio_context_release(ctx);
+     /* Simple bdrv_drain_all_begin/end pair, check that CBs are called */
-+    g_assert(blk_get_aio_context(blk) == main_ctx);
+     g_assert_cmpint(s->drain_count, ==, 0);
-+    g_assert(bdrv_get_aio_context(bs_a) == main_ctx);
+-    bdrv_drain_all_begin();
-+    g_assert(bdrv_get_aio_context(bs_verify) == main_ctx);
++    g_assert_cmpint(backing_s->drain_count, ==, 0);
-+    g_assert(bdrv_get_aio_context(bs_b) == main_ctx);
++
++    do_drain_begin(drain_type, bs);
-     bdrv_unref(bs_verify);
++
-     bdrv_unref(bs_b);
+     g_assert_cmpint(s->drain_count, ==, 1);
-@@ -XXX,XX +XXX,XX @@ static void test_propagate_diamond(void)
+-    bdrv_drain_all_end();
- {
++    g_assert_cmpint(backing_s->drain_count, ==, !!recursive);
-     IOThread *iothread = iothread_new();
++
-     AioContext *ctx = iothread_get_aio_context(iothread);
++    do_drain_end(drain_type, bs);
-+    AioContext *main_ctx;
++
-     BlockBackend *blk;
+     g_assert_cmpint(s->drain_count, ==, 0);
-     BlockDriverState *bs_a, *bs_b, *bs_c, *bs_verify;
++    g_assert_cmpint(backing_s->drain_count, ==, 0);
-     QDict *options;
-@@ -XXX,XX +XXX,XX @@ static void test_propagate_diamond(void)
+     /* Now do the same while a request is pending */
-     g_assert(bdrv_get_aio_context(bs_c) == ctx);
+     aio_ret = -EINPROGRESS;
+@@ -XXX,XX +XXX,XX @@ static void test_drv_cb_drain_all(void)
-     /* Switch the AioContext back */
+     g_assert_cmpint(aio_ret, ==, -EINPROGRESS);
--    ctx = qemu_get_aio_context();
--    blk_set_aio_context(blk, ctx, &error_abort);
+     g_assert_cmpint(s->drain_count, ==, 0);
--    g_assert(blk_get_aio_context(blk) == ctx);
+-    bdrv_drain_all_begin();
--    g_assert(bdrv_get_aio_context(bs_verify) == ctx);
++    g_assert_cmpint(backing_s->drain_count, ==, 0);
--    g_assert(bdrv_get_aio_context(bs_a) == ctx);
++
--    g_assert(bdrv_get_aio_context(bs_b) == ctx);
++    do_drain_begin(drain_type, bs);
--    g_assert(bdrv_get_aio_context(bs_c) == ctx);
++
-+    main_ctx = qemu_get_aio_context();
+     g_assert_cmpint(aio_ret, ==, 0);
-+    aio_context_acquire(ctx);
+     g_assert_cmpint(s->drain_count, ==, 1);
-+    blk_set_aio_context(blk, main_ctx, &error_abort);
+-    bdrv_drain_all_end();
-+    aio_context_release(ctx);
++    g_assert_cmpint(backing_s->drain_count, ==, !!recursive);
-+    g_assert(blk_get_aio_context(blk) == main_ctx);
++
-+    g_assert(bdrv_get_aio_context(bs_verify) == main_ctx);
++    do_drain_end(drain_type, bs);
-+    g_assert(bdrv_get_aio_context(bs_a) == main_ctx);
++
-+    g_assert(bdrv_get_aio_context(bs_b) == main_ctx);
+     g_assert_cmpint(s->drain_count, ==, 0);
-+    g_assert(bdrv_get_aio_context(bs_c) == main_ctx);
++    g_assert_cmpint(backing_s->drain_count, ==, 0);
-     blk_unref(blk);
++    bdrv_unref(backing);
      bdrv_unref(bs_verify);
@@ -XXX,XX +XXX,XX @@ static void test_attach_second_node(void)
      g_assert(bdrv_get_aio_context(bs) == ctx);
      g_assert(bdrv_get_aio_context(filter) == ctx);
 +    aio_context_acquire(ctx);
      blk_set_aio_context(blk, main_ctx, &error_abort);
 +    aio_context_release(ctx);
      g_assert(blk_get_aio_context(blk) == main_ctx);
      g_assert(bdrv_get_aio_context(bs) == main_ctx);
      g_assert(bdrv_get_aio_context(filter) == main_ctx);
@@ -XXX,XX +XXX,XX @@ static void test_attach_preserve_blk_ctx(void)
      g_assert(bdrv_get_aio_context(bs) == ctx);
      /* Remove the node again */
 +    aio_context_acquire(ctx);
      blk_remove_bs(blk);
 +    aio_context_release(ctx);
      g_assert(blk_get_aio_context(blk) == ctx);
      g_assert(bdrv_get_aio_context(bs) == qemu_get_aio_context());
@@ -XXX,XX +XXX,XX @@ static void test_attach_preserve_blk_ctx(void)
      g_assert(blk_get_aio_context(blk) == ctx);
      g_assert(bdrv_get_aio_context(bs) == ctx);
 +    aio_context_acquire(ctx);
      blk_set_aio_context(blk, qemu_get_aio_context(), &error_abort);
 +    aio_context_release(ctx);
      bdrv_unref(bs);
      blk_unref(blk);
  }
++static void test_drv_cb_drain_all(void)
++{
++    test_drv_cb_common(BDRV_DRAIN_ALL, true);
++}
++
++static void test_drv_cb_drain(void)
++{
++    test_drv_cb_common(BDRV_DRAIN, false);
++}
++
+ int main(int argc, char **argv)
+ {
+     bdrv_init();
+@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
+     g_test_init(&argc, &argv, NULL);
+     g_test_add_func("/bdrv-drain/driver-cb/drain_all", test_drv_cb_drain_all);
++    g_test_add_func("/bdrv-drain/driver-cb/drain", test_drv_cb_drain);
+     return g_test_run();
+ }
 --
-.20.1
+.13.6

-New patch
+[Qemu-devel] [PULL v3 21/35] test-bdrv-drain: Test bs->quiesce_counter
+This is currently only working correctly for bdrv_drain(), not for
+bdrv_drain_all(). Leave a comment for the drain_all case, we'll address
+it later.
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+---
+ tests/test-bdrv-drain.c | 45 +++++++++++++++++++++++++++++++++++++++++++++
+file changed, 45 insertions(+)
+diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
+index XXXXXXX..XXXXXXX 100644
+--- a/tests/test-bdrv-drain.c
++++ b/tests/test-bdrv-drain.c
+@@ -XXX,XX +XXX,XX @@ static void test_drv_cb_drain(void)
+     test_drv_cb_common(BDRV_DRAIN, false);
+ }
++static void test_quiesce_common(enum drain_type drain_type, bool recursive)
++{
++    BlockBackend *blk;
++    BlockDriverState *bs, *backing;
++
++    blk = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
++    bs = bdrv_new_open_driver(&bdrv_test, "test-node", BDRV_O_RDWR,
++                              &error_abort);
++    blk_insert_bs(blk, bs, &error_abort);
++
++    backing = bdrv_new_open_driver(&bdrv_test, "backing", 0, &error_abort);
++    bdrv_set_backing_hd(bs, backing, &error_abort);
++
++    g_assert_cmpint(bs->quiesce_counter, ==, 0);
++    g_assert_cmpint(backing->quiesce_counter, ==, 0);
++
++    do_drain_begin(drain_type, bs);
++
++    g_assert_cmpint(bs->quiesce_counter, ==, 1);
++    g_assert_cmpint(backing->quiesce_counter, ==, !!recursive);
++
++    do_drain_end(drain_type, bs);
++
++    g_assert_cmpint(bs->quiesce_counter, ==, 0);
++    g_assert_cmpint(backing->quiesce_counter, ==, 0);
++
++    bdrv_unref(backing);
++    bdrv_unref(bs);
++    blk_unref(blk);
++}
++
++static void test_quiesce_drain_all(void)
++{
++    // XXX drain_all doesn't quiesce
++    //test_quiesce_common(BDRV_DRAIN_ALL, true);
++}
++
++static void test_quiesce_drain(void)
++{
++    test_quiesce_common(BDRV_DRAIN, false);
++}
++
+ int main(int argc, char **argv)
+ {
+     bdrv_init();
+@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
+     g_test_add_func("/bdrv-drain/driver-cb/drain_all", test_drv_cb_drain_all);
+     g_test_add_func("/bdrv-drain/driver-cb/drain", test_drv_cb_drain);
++    g_test_add_func("/bdrv-drain/quiesce/drain_all", test_quiesce_drain_all);
++    g_test_add_func("/bdrv-drain/quiesce/drain", test_quiesce_drain);
++
+     return g_test_run();
+ }
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 22/35] blockjob: Pause job on draining any job BDS
+Block jobs already paused themselves when their main BlockBackend
+entered a drained section. This is not good enough: We also want to
+pause a block job and may not submit new requests if, for example, the
+mirror target node should be drained.
+This implements .drained_begin/end callbacks in child_job in order to
+consider all block nodes related to the job, and removes the
+BlockBackend callbacks which are unnecessary now because the root of the
+job main BlockBackend is always referenced with a child_job, too.
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+---
+ blockjob.c | 22 +++++++++-------------
+file changed, 9 insertions(+), 13 deletions(-)
+diff --git a/blockjob.c b/blockjob.c
+index XXXXXXX..XXXXXXX 100644
+--- a/blockjob.c
++++ b/blockjob.c
+@@ -XXX,XX +XXX,XX @@ static char *child_job_get_parent_desc(BdrvChild *c)
+                            job->id);
+ }
+-static const BdrvChildRole child_job = {
+-    .get_parent_desc    = child_job_get_parent_desc,
+-    .stay_at_node       = true,
+-};
+-
+-static void block_job_drained_begin(void *opaque)
++static void child_job_drained_begin(BdrvChild *c)
+ {
+-    BlockJob *job = opaque;
++    BlockJob *job = c->opaque;
+     block_job_pause(job);
+ }
+-static void block_job_drained_end(void *opaque)
++static void child_job_drained_end(BdrvChild *c)
+ {
+-    BlockJob *job = opaque;
++    BlockJob *job = c->opaque;
+     block_job_resume(job);
+ }
+-static const BlockDevOps block_job_dev_ops = {
+-    .drained_begin = block_job_drained_begin,
+-    .drained_end = block_job_drained_end,
++static const BdrvChildRole child_job = {
++    .get_parent_desc    = child_job_get_parent_desc,
++    .drained_begin      = child_job_drained_begin,
++    .drained_end        = child_job_drained_end,
++    .stay_at_node       = true,
+ };
+ void block_job_remove_all_bdrv(BlockJob *job)
+@@ -XXX,XX +XXX,XX @@ void *block_job_create(const char *job_id, const BlockJobDriver *driver,
+     block_job_add_bdrv(job, "main node", bs, 0, BLK_PERM_ALL, &error_abort);
+     bs->job = job;
+-    blk_set_dev_ops(blk, &block_job_dev_ops, job);
+     bdrv_op_unblock(bs, BLOCK_OP_TYPE_DATAPLANE, job->blocker);
+     QLIST_INSERT_HEAD(&block_jobs, job, job_list);
+--
+.13.6

-[Qemu-devel] [PULL 03/13] tests: Add job commit by drained_end test
+[Qemu-devel] [PULL v3 23/35] test-bdrv-drain: Test drain vs. block jobs
-From: Max Reitz <mreitz@redhat.com>
+Block jobs must be paused if any of the involved nodes are drained.
-Signed-off-by: Max Reitz <mreitz@redhat.com>
 Signed-off-by: Kevin Wolf <kwolf@redhat.com>
 ---
- tests/test-bdrv-drain.c | 119 ++++++++++++++++++++++++++++++++++++++++
+ tests/test-bdrv-drain.c | 121 ++++++++++++++++++++++++++++++++++++++++++++++++
-file changed, 119 insertions(+)
+file changed, 121 insertions(+)
 diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/test-bdrv-drain.c
 +++ b/tests/test-bdrv-drain.c
-@@ -XXX,XX +XXX,XX @@ static void test_set_aio_context(void)
+@@ -XXX,XX +XXX,XX @@
-     iothread_join(b);
  #include "qemu/osdep.h"
  #include "block/block.h"
 +#include "block/blockjob_int.h"
  #include "sysemu/block-backend.h"
  #include "qapi/error.h"
@@ -XXX,XX +XXX,XX @@ static void test_quiesce_drain(void)
      test_quiesce_common(BDRV_DRAIN, false);
  }
 +
-+typedef struct TestDropBackingBlockJob {
++typedef struct TestBlockJob {
 +    BlockJob common;
 +    bool should_complete;
-+    bool *did_complete;
++} TestBlockJob;
 +} TestDropBackingBlockJob;
 +
-+static int coroutine_fn test_drop_backing_job_run(Job *job, Error **errp)
++static void test_job_completed(BlockJob *job, void *opaque)
 +{
-+    TestDropBackingBlockJob *s =
++    block_job_completed(job, 0);
-+        container_of(job, TestDropBackingBlockJob, common.job);
++}
 +
 +static void coroutine_fn test_job_start(void *opaque)
 +{
 +    TestBlockJob *s = opaque;
 +
 +    while (!s->should_complete) {
-+        job_sleep_ns(job, 0);
++        block_job_sleep_ns(&s->common, 100000);
 +    }
 +
-+    return 0;
++    block_job_defer_to_main_loop(&s->common, test_job_completed, NULL);
 +}
 +
-+static void test_drop_backing_job_commit(Job *job)
++static void test_job_complete(BlockJob *job, Error **errp)
 +{
-+    TestDropBackingBlockJob *s =
++    TestBlockJob *s = container_of(job, TestBlockJob, common);
-+        container_of(job, TestDropBackingBlockJob, common.job);
++    s->should_complete = true;
 +
 +    bdrv_set_backing_hd(blk_bs(s->common.blk), NULL, &error_abort);
 +
 +    *s->did_complete = true;
 +}
 +
-+static const BlockJobDriver test_drop_backing_job_driver = {
++BlockJobDriver test_job_driver = {
-+    .job_driver = {
++    .instance_size  = sizeof(TestBlockJob),
-+        .instance_size  = sizeof(TestDropBackingBlockJob),
++    .start          = test_job_start,
-+        .free           = block_job_free,
++    .complete       = test_job_complete,
 +        .user_resume    = block_job_user_resume,
 +        .drain          = block_job_drain,
 +        .run            = test_drop_backing_job_run,
 +        .commit         = test_drop_backing_job_commit,
 +    }
 +};
 +
-+/**
++static void test_blockjob_common(enum drain_type drain_type)
 + * Creates a child node with three parent nodes on it, and then runs a
 + * block job on the final one, parent-node-2.
 + *
 + * (TODO: parent-node-0 currently serves no purpose, but will as of a
 + * follow-up patch.)
 + *
 + * The job is then asked to complete before a section where the child
 + * is drained.
 + *
 + * Ending this section will undrain the child's parents, first
 + * parent-node-2, then parent-node-1, then parent-node-0 -- the parent
 + * list is in reverse order of how they were added.  Ending the drain
 + * on parent-node-2 will resume the job, thus completing it and
 + * scheduling job_exit().
 + *
 + * Ending the drain on parent-node-1 will poll the AioContext, which
 + * lets job_exit() and thus test_drop_backing_job_commit() run.  That
 + * function removes the child as parent-node-2's backing file.
 + *
 + * In old (and buggy) implementations, there are two problems with
 + * that:
 + * (A) bdrv_drain_invoke() polls for every node that leaves the
 + *     drained section.  This means that job_exit() is scheduled
 + *     before the child has left the drained section.  Its
 + *     quiesce_counter is therefore still 1 when it is removed from
 + *     parent-node-2.
 + *
 + * (B) bdrv_replace_child_noperm() calls drained_end() on the old
 + *     child's parents as many times as the child is quiesced.  This
 + *     means it will call drained_end() on parent-node-2 once.
 + *     Because parent-node-2 is no longer quiesced at this point, this
 + *     will fail.
 + *
 + * bdrv_replace_child_noperm() therefore must call drained_end() on
 + * the parent only if it really is still drained because the child is
 + * drained.
 + */
 +static void test_blockjob_commit_by_drained_end(void)
 +{
-+    BlockDriverState *bs_child, *bs_parents[3];
++    BlockBackend *blk_src, *blk_target;
-+    TestDropBackingBlockJob *job;
++    BlockDriverState *src, *target;
-+    bool job_has_completed = false;
++    BlockJob *job;
-+    int i;
++    int ret;
 +
-+    bs_child = bdrv_new_open_driver(&bdrv_test, "child-node", BDRV_O_RDWR,
++    src = bdrv_new_open_driver(&bdrv_test, "source", BDRV_O_RDWR,
-+                                    &error_abort);
++                               &error_abort);
 +    blk_src = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
 +    blk_insert_bs(blk_src, src, &error_abort);
 +
-+    for (i = 0; i < 3; i++) {
++    target = bdrv_new_open_driver(&bdrv_test, "target", BDRV_O_RDWR,
-+        char name[32];
++                                  &error_abort);
-+        snprintf(name, sizeof(name), "parent-node-%i", i);
++    blk_target = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
-+        bs_parents[i] = bdrv_new_open_driver(&bdrv_test, name, BDRV_O_RDWR,
++    blk_insert_bs(blk_target, target, &error_abort);
-+                                             &error_abort);
++
-+        bdrv_set_backing_hd(bs_parents[i], bs_child, &error_abort);
++    job = block_job_create("job0", &test_job_driver, src, 0, BLK_PERM_ALL, 0,
 +                           0, NULL, NULL, &error_abort);
 +    block_job_add_bdrv(job, "target", target, 0, BLK_PERM_ALL, &error_abort);
 +    block_job_start(job);
 +
 +    g_assert_cmpint(job->pause_count, ==, 0);
 +    g_assert_false(job->paused);
 +    g_assert_false(job->busy); /* We're in block_job_sleep_ns() */
 +
 +    do_drain_begin(drain_type, src);
 +
 +    if (drain_type == BDRV_DRAIN_ALL) {
 +        /* bdrv_drain_all() drains both src and target, and involves an
 +         * additional block_job_pause_all() */
 +        g_assert_cmpint(job->pause_count, ==, 3);
 +    } else {
 +        g_assert_cmpint(job->pause_count, ==, 1);
 +    }
++    /* XXX We don't wait until the job is actually paused. Is this okay? */
++    /* g_assert_true(job->paused); */
++    g_assert_false(job->busy); /* The job is paused */
 +
-+    job = block_job_create("job", &test_drop_backing_job_driver, NULL,
++    do_drain_end(drain_type, src);
 +                           bs_parents[2], 0, BLK_PERM_ALL, 0, 0, NULL, NULL,
 +                           &error_abort);
 +
-+    job->did_complete = &job_has_completed;
++    g_assert_cmpint(job->pause_count, ==, 0);
 +    g_assert_false(job->paused);
 +    g_assert_false(job->busy); /* We're in block_job_sleep_ns() */
 +
-+    job_start(&job->common.job);
++    do_drain_begin(drain_type, target);
 +
-+    job->should_complete = true;
++    if (drain_type == BDRV_DRAIN_ALL) {
-+    bdrv_drained_begin(bs_child);
++        /* bdrv_drain_all() drains both src and target, and involves an
-+    g_assert(!job_has_completed);
++         * additional block_job_pause_all() */
-+    bdrv_drained_end(bs_child);
++        g_assert_cmpint(job->pause_count, ==, 3);
-+    g_assert(job_has_completed);
++    } else {
 +        g_assert_cmpint(job->pause_count, ==, 1);
 +    }
 +    /* XXX We don't wait until the job is actually paused. Is this okay? */
 +    /* g_assert_true(job->paused); */
 +    g_assert_false(job->busy); /* The job is paused */
 +
-+    bdrv_unref(bs_parents[0]);
++    do_drain_end(drain_type, target);
-+    bdrv_unref(bs_parents[1]);
++
-+    bdrv_unref(bs_parents[2]);
++    g_assert_cmpint(job->pause_count, ==, 0);
-+    bdrv_unref(bs_child);
++    g_assert_false(job->paused);
 +    g_assert_false(job->busy); /* We're in block_job_sleep_ns() */
 +
 +    ret = block_job_complete_sync(job, &error_abort);
 +    g_assert_cmpint(ret, ==, 0);
 +
 +    blk_unref(blk_src);
 +    blk_unref(blk_target);
 +    bdrv_unref(src);
 +    bdrv_unref(target);
 +}
 +
 +static void test_blockjob_drain_all(void)
 +{
 +    test_blockjob_common(BDRV_DRAIN_ALL);
 +}
 +
 +static void test_blockjob_drain(void)
 +{
 +    test_blockjob_common(BDRV_DRAIN);
 +}
 +
  int main(int argc, char **argv)
  {
-     int ret;
+     bdrv_init();
 @@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
+     g_test_add_func("/bdrv-drain/quiesce/drain_all", test_quiesce_drain_all);
-     g_test_add_func("/bdrv-drain/set_aio_context", test_set_aio_context);
+     g_test_add_func("/bdrv-drain/quiesce/drain", test_quiesce_drain);
-+    g_test_add_func("/bdrv-drain/blockjob/commit_by_drained_end",
++    g_test_add_func("/bdrv-drain/blockjob/drain_all", test_blockjob_drain_all);
-+                    test_blockjob_commit_by_drained_end);
++    g_test_add_func("/bdrv-drain/blockjob/drain", test_blockjob_drain);
 +
-     ret = g_test_run();
+     return g_test_run();
-     qemu_event_destroy(&done_event);
+ }
      return ret;
 --
-.20.1
+.13.6

-New patch
+[Qemu-devel] [PULL v3 24/35] block: Don't block_job_pause_all() in bdrv_drain_all()
+Block jobs are already paused using the BdrvChildRole drain callbacks,
+so we don't need an additional block_job_pause_all() call.
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+---
+ block/io.c              |  4 ----
+ tests/test-bdrv-drain.c | 10 ++++------
+files changed, 4 insertions(+), 10 deletions(-)
+diff --git a/block/io.c b/block/io.c
+index XXXXXXX..XXXXXXX 100644
+--- a/block/io.c
++++ b/block/io.c
+@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
+      * context. */
+     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
+-    block_job_pause_all();
+-
+     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+         AioContext *aio_context = bdrv_get_aio_context(bs);
+@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_end(void)
+         aio_enable_external(aio_context);
+         aio_context_release(aio_context);
+     }
+-
+-    block_job_resume_all();
+ }
+ void bdrv_drain_all(void)
+diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
+index XXXXXXX..XXXXXXX 100644
+--- a/tests/test-bdrv-drain.c
++++ b/tests/test-bdrv-drain.c
+@@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type)
+     do_drain_begin(drain_type, src);
+     if (drain_type == BDRV_DRAIN_ALL) {
+-        /* bdrv_drain_all() drains both src and target, and involves an
+-         * additional block_job_pause_all() */
+-        g_assert_cmpint(job->pause_count, ==, 3);
++        /* bdrv_drain_all() drains both src and target */
++        g_assert_cmpint(job->pause_count, ==, 2);
+     } else {
+         g_assert_cmpint(job->pause_count, ==, 1);
+     }
+@@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type)
+     do_drain_begin(drain_type, target);
+     if (drain_type == BDRV_DRAIN_ALL) {
+-        /* bdrv_drain_all() drains both src and target, and involves an
+-         * additional block_job_pause_all() */
+-        g_assert_cmpint(job->pause_count, ==, 3);
++        /* bdrv_drain_all() drains both src and target */
++        g_assert_cmpint(job->pause_count, ==, 2);
+     } else {
+         g_assert_cmpint(job->pause_count, ==, 1);
+     }
+--
+.13.6

-[Qemu-devel] [PULL 09/13] block: Loop unsafely in bdrv*drained_end()
+[Qemu-devel] [PULL v3 25/35] block: Nested drain_end must still call callbacks
-From: Max Reitz <mreitz@redhat.com>
+bdrv_do_drained_begin() restricts the call of parent callbacks and
 aio_disable_external() to the outermost drain section, but the block
 driver callbacks are always called. bdrv_do_drained_end() must match
 this behaviour, otherwise nodes stay drained even if begin/end calls
 were balanced.
-The graph must not change in these loops (or a QLIST_FOREACH_SAFE would
-not even be enough).  We now ensure this by only polling once in the
-root bdrv_drained_end() call, so we can drop the _SAFE suffix.  Doing so
-makes it clear that the graph must not change.
-Signed-off-by: Max Reitz <mreitz@redhat.com>
 Signed-off-by: Kevin Wolf <kwolf@redhat.com>
 ---
- block/io.c | 8 ++++----
+ block/io.c | 12 +++++++-----
-file changed, 4 insertions(+), 4 deletions(-)
+file changed, 7 insertions(+), 5 deletions(-)
 diff --git a/block/io.c b/block/io.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/io.c
 +++ b/block/io.c
-@@ -XXX,XX +XXX,XX @@ static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
+@@ -XXX,XX +XXX,XX @@ void bdrv_drained_begin(BlockDriverState *bs)
-                                     bool ignore_bds_parents,
-                                     int *drained_end_counter)
+ void bdrv_drained_end(BlockDriverState *bs)
  {
--    BdrvChild *c, *next;
++    int old_quiesce_counter;
-+    BdrvChild *c;
++
+     if (qemu_in_coroutine()) {
--    QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
+         bdrv_co_yield_to_drain(bs, false);
-+    QLIST_FOREACH(c, &bs->parents, next_parent) {
+         return;
-         if (c == ignore || (ignore_bds_parents && c->role->parent_is_bds)) {
+     }
-             continue;
+     assert(bs->quiesce_counter > 0);
-         }
+-    if (atomic_fetch_dec(&bs->quiesce_counter) > 1) {
-@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
+-        return;
-                                 BdrvChild *parent, bool ignore_bds_parents,
+-    }
-                                 int *drained_end_counter)
++    old_quiesce_counter = atomic_fetch_dec(&bs->quiesce_counter);
- {
--    BdrvChild *child, *next;
+     /* Re-enable things in child-to-parent order */
-+    BdrvChild *child;
+     bdrv_drain_invoke(bs, false, false);
-     int old_quiesce_counter;
+-    bdrv_parent_drained_end(bs);
+-    aio_enable_external(bdrv_get_aio_context(bs));
-     assert(drained_end_counter != NULL);
++    if (old_quiesce_counter == 1) {
-@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
++        bdrv_parent_drained_end(bs);
-     if (recursive) {
++        aio_enable_external(bdrv_get_aio_context(bs));
-         assert(!ignore_bds_parents);
++    }
-         bs->recursive_quiesce_counter--;
+ }
--        QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
-+        QLIST_FOREACH(child, &bs->children, next) {
+ /*
              bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents,
                                  drained_end_counter);
          }
 --
-.20.1
+.13.6

-[Qemu-devel] [PULL 13/13] iotests: Test quitting with job on throttled node
+[Qemu-devel] [PULL v3 26/35] test-bdrv-drain: Test nested drain sections
-From: Max Reitz <mreitz@redhat.com>
-When qemu quits, all throttling should be ignored.  That means, if there
-is a mirror job running from a throttled node, it should be cancelled
-immediately and qemu close without blocking.
-Signed-off-by: Max Reitz <mreitz@redhat.com>
 Signed-off-by: Kevin Wolf <kwolf@redhat.com>
 ---
- tests/qemu-iotests/218     | 55 ++++++++++++++++++++++++++++++++++++--
+ tests/test-bdrv-drain.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++
- tests/qemu-iotests/218.out |  4 +++
+file changed, 57 insertions(+)
 files changed, 57 insertions(+), 2 deletions(-)
-diff --git a/tests/qemu-iotests/218 b/tests/qemu-iotests/218
+diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
-index XXXXXXX..XXXXXXX 100755
+index XXXXXXX..XXXXXXX 100644
---- a/tests/qemu-iotests/218
+--- a/tests/test-bdrv-drain.c
-+++ b/tests/qemu-iotests/218
++++ b/tests/test-bdrv-drain.c
-@@ -XXX,XX +XXX,XX @@
+@@ -XXX,XX +XXX,XX @@ static void aio_ret_cb(void *opaque, int ret)
- # Creator/Owner: Max Reitz <mreitz@redhat.com>
+ enum drain_type {
+     BDRV_DRAIN_ALL,
- import iotests
+     BDRV_DRAIN,
--from iotests import log
++    DRAIN_TYPE_MAX,
-+from iotests import log, qemu_img, qemu_io_silent
+ };
--iotests.verify_platform(['linux'])
+ static void do_drain_begin(enum drain_type drain_type, BlockDriverState *bs)
-+iotests.verify_image_format(supported_fmts=['qcow2', 'raw'])
+@@ -XXX,XX +XXX,XX @@ static void test_quiesce_drain(void)
+     test_quiesce_common(BDRV_DRAIN, false);
+ }
- # Launches the VM, adds two null-co nodes (source and target), and
-@@ -XXX,XX +XXX,XX @@ with iotests.VM() as vm:
++static void test_nested(void)
++{
-     log(vm.event_wait('BLOCK_JOB_CANCELLED'),
++    BlockBackend *blk;
-         filters=[iotests.filter_qmp_event])
++    BlockDriverState *bs, *backing;
 +    BDRVTestState *s, *backing_s;
 +    enum drain_type outer, inner;
 +
-+log('')
++    blk = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
-+log('=== Cancel mirror job from throttled node by quitting ===')
++    bs = bdrv_new_open_driver(&bdrv_test, "test-node", BDRV_O_RDWR,
-+log('')
++                              &error_abort);
 +    s = bs->opaque;
 +    blk_insert_bs(blk, bs, &error_abort);
 +
-+with iotests.VM() as vm, \
++    backing = bdrv_new_open_driver(&bdrv_test, "backing", 0, &error_abort);
-+     iotests.FilePath('src.img') as src_img_path:
++    backing_s = backing->opaque;
 +    bdrv_set_backing_hd(bs, backing, &error_abort);
 +
-+    assert qemu_img('create', '-f', iotests.imgfmt, src_img_path, '64M') == 0
++    for (outer = 0; outer < DRAIN_TYPE_MAX; outer++) {
-+    assert qemu_io_silent('-f', iotests.imgfmt, src_img_path,
++        for (inner = 0; inner < DRAIN_TYPE_MAX; inner++) {
-+                          '-c', 'write -P 42 0M 64M') == 0
++            /* XXX bdrv_drain_all() doesn't increase the quiesce_counter */
 +            int bs_quiesce      = (outer != BDRV_DRAIN_ALL) +
 +                                  (inner != BDRV_DRAIN_ALL);
 +            int backing_quiesce = 0;
 +            int backing_cb_cnt  = (outer != BDRV_DRAIN) +
 +                                  (inner != BDRV_DRAIN);
 +
-+    vm.launch()
++            g_assert_cmpint(bs->quiesce_counter, ==, 0);
 +            g_assert_cmpint(backing->quiesce_counter, ==, 0);
 +            g_assert_cmpint(s->drain_count, ==, 0);
 +            g_assert_cmpint(backing_s->drain_count, ==, 0);
 +
-+    ret = vm.qmp('object-add', qom_type='throttle-group', id='tg',
++            do_drain_begin(outer, bs);
-+                 props={'x-bps-read': 4096})
++            do_drain_begin(inner, bs);
 +    assert ret['return'] == {}
 +
-+    ret = vm.qmp('blockdev-add',
++            g_assert_cmpint(bs->quiesce_counter, ==, bs_quiesce);
-+                 node_name='source',
++            g_assert_cmpint(backing->quiesce_counter, ==, backing_quiesce);
-+                 driver=iotests.imgfmt,
++            g_assert_cmpint(s->drain_count, ==, 2);
-+                 file={
++            g_assert_cmpint(backing_s->drain_count, ==, backing_cb_cnt);
 +                     'driver': 'file',
 +                     'filename': src_img_path
 +                 })
 +    assert ret['return'] == {}
 +
-+    ret = vm.qmp('blockdev-add',
++            do_drain_end(inner, bs);
-+                 node_name='throttled-source',
++            do_drain_end(outer, bs);
 +                 driver='throttle',
 +                 throttle_group='tg',
 +                 file='source')
 +    assert ret['return'] == {}
 +
-+    ret = vm.qmp('blockdev-add',
++            g_assert_cmpint(bs->quiesce_counter, ==, 0);
-+                 node_name='target',
++            g_assert_cmpint(backing->quiesce_counter, ==, 0);
-+                 driver='null-co',
++            g_assert_cmpint(s->drain_count, ==, 0);
-+                 size=(64 * 1048576))
++            g_assert_cmpint(backing_s->drain_count, ==, 0);
-+    assert ret['return'] == {}
++        }
 +    }
 +
-+    ret = vm.qmp('blockdev-mirror',
++    bdrv_unref(backing);
-+                 job_id='mirror',
++    bdrv_unref(bs);
-+                 device='throttled-source',
++    blk_unref(blk);
-+                 target='target',
++}
 +                 sync='full')
 +    assert ret['return'] == {}
 +
-+    log(vm.qmp('quit'))
  typedef struct TestBlockJob {
      BlockJob common;
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
      g_test_add_func("/bdrv-drain/quiesce/drain_all", test_quiesce_drain_all);
      g_test_add_func("/bdrv-drain/quiesce/drain", test_quiesce_drain);
 +    g_test_add_func("/bdrv-drain/nested", test_nested);
 +
-+    with iotests.Timeout(5, 'Timeout waiting for VM to quit'):
+     g_test_add_func("/bdrv-drain/blockjob/drain_all", test_blockjob_drain_all);
-+        vm.shutdown(has_quit=True)
+     g_test_add_func("/bdrv-drain/blockjob/drain", test_blockjob_drain);
-diff --git a/tests/qemu-iotests/218.out b/tests/qemu-iotests/218.out
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/218.out
 +++ b/tests/qemu-iotests/218.out
@@ -XXX,XX +XXX,XX @@ Cancelling job
  Cancelling job
  {"return": {}}
  {"data": {"device": "mirror", "len": 1048576, "offset": 1048576, "speed": 0, "type": "mirror"}, "event": "BLOCK_JOB_CANCELLED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
 +
 +=== Cancel mirror job from throttled node by quitting ===
 +
 +{"return": {}}
 --
-.20.1
+.13.6

-[Qemu-devel] [PULL 05/13] block: Make bdrv_parent_drained_[^_]*() static
+[Qemu-devel] [PULL v3 27/35] block: Don't notify parents in drain call chain
-From: Max Reitz <mreitz@redhat.com>
+This is in preparation for subtree drains, i.e. drained sections that
+affect not only a single node, but recursively all child nodes, too.
-These functions are not used outside of block/io.c, there is no reason
-why they should be globally available.
+Calling the parent callbacks for drain is pointless when we just came
+from that parent node recursively and leads to multiple increases of
-Signed-off-by: Max Reitz <mreitz@redhat.com>
+bs->quiesce_counter in a single drain call. Don't do it.
 In order for this to work correctly, the parent callback must be called
 for every bdrv_drain_begin/end() call, not only for the outermost one:
 If we have a node N with two parents A and B, recursive draining of A
 should cause the quiesce_counter of B to increase because its child N is
 drained independently of B. If now B is recursively drained, too, A must
 increase its quiesce_counter because N is drained independently of A
 only now, even if N is going from quiesce_counter 1 to 2.
 Signed-off-by: Kevin Wolf <kwolf@redhat.com>
 ---
- include/block/block.h | 18 ------------------
+ include/block/block.h |  4 ++--
- block/io.c            |  8 ++++----
+ block.c               | 13 +++++++++----
-files changed, 4 insertions(+), 22 deletions(-)
+ block/io.c            | 47 ++++++++++++++++++++++++++++++++++-------------
 files changed, 45 insertions(+), 19 deletions(-)
 diff --git a/include/block/block.h b/include/block/block.h
 index XXXXXXX..XXXXXXX 100644
 --- a/include/block/block.h
 +++ b/include/block/block.h
-@@ -XXX,XX +XXX,XX @@ int bdrv_probe_geometry(BlockDriverState *bs, HDGeometry *geo);
+@@ -XXX,XX +XXX,XX @@ void bdrv_io_unplug(BlockDriverState *bs);
- void bdrv_io_plug(BlockDriverState *bs);
+  * Begin a quiesced section of all users of @bs. This is part of
- void bdrv_io_unplug(BlockDriverState *bs);
+  * bdrv_drained_begin.
+  */
--/**
+-void bdrv_parent_drained_begin(BlockDriverState *bs);
-- * bdrv_parent_drained_begin:
++void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore);
-- *
 - * Begin a quiesced section of all users of @bs. This is part of
 - * bdrv_drained_begin.
 - */
 -void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
 -                               bool ignore_bds_parents);
 -
  /**
-  * bdrv_parent_drained_begin_single:
+  * bdrv_parent_drained_end:
-  *
+@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_begin(BlockDriverState *bs);
-@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll);
+  * End a quiesced section of all users of @bs. This is part of
   * bdrv_drained_end.
   */
- void bdrv_parent_drained_end_single(BdrvChild *c);
+-void bdrv_parent_drained_end(BlockDriverState *bs);
++void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore);
--/**
 - * bdrv_parent_drained_end:
 - *
 - * End a quiesced section of all users of @bs. This is part of
 - * bdrv_drained_end.
 - */
 -void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
 -                             bool ignore_bds_parents);
 -
  /**
-  * bdrv_drain_poll:
+  * bdrv_drained_begin:
-  *
+diff --git a/block.c b/block.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block.c
 +++ b/block.c
@@ -XXX,XX +XXX,XX @@ static void bdrv_replace_child_noperm(BdrvChild *child,
                                        BlockDriverState *new_bs)
  {
      BlockDriverState *old_bs = child->bs;
 +    int i;
      if (old_bs && new_bs) {
          assert(bdrv_get_aio_context(old_bs) == bdrv_get_aio_context(new_bs));
      }
      if (old_bs) {
          if (old_bs->quiesce_counter && child->role->drained_end) {
 -            child->role->drained_end(child);
 +            for (i = 0; i < old_bs->quiesce_counter; i++) {
 +                child->role->drained_end(child);
 +            }
          }
          if (child->role->detach) {
              child->role->detach(child);
@@ -XXX,XX +XXX,XX @@ static void bdrv_replace_child_noperm(BdrvChild *child,
      if (new_bs) {
          QLIST_INSERT_HEAD(&new_bs->parents, child, next_parent);
          if (new_bs->quiesce_counter && child->role->drained_begin) {
 -            child->role->drained_begin(child);
 +            for (i = 0; i < new_bs->quiesce_counter; i++) {
 +                child->role->drained_begin(child);
 +            }
          }
          if (child->role->attach) {
@@ -XXX,XX +XXX,XX @@ void bdrv_set_aio_context(BlockDriverState *bs, AioContext *new_context)
      AioContext *ctx = bdrv_get_aio_context(bs);
      aio_disable_external(ctx);
 -    bdrv_parent_drained_begin(bs);
 +    bdrv_parent_drained_begin(bs, NULL);
      bdrv_drain(bs); /* ensure there are no in-flight requests */
      while (aio_poll(ctx, false)) {
@@ -XXX,XX +XXX,XX @@ void bdrv_set_aio_context(BlockDriverState *bs, AioContext *new_context)
       */
      aio_context_acquire(new_context);
      bdrv_attach_aio_context(bs, new_context);
 -    bdrv_parent_drained_end(bs);
 +    bdrv_parent_drained_end(bs, NULL);
      aio_enable_external(ctx);
      aio_context_release(new_context);
  }
 diff --git a/block/io.c b/block/io.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/io.c
 +++ b/block/io.c
-@@ -XXX,XX +XXX,XX @@ static void bdrv_parent_cb_resize(BlockDriverState *bs);
+@@ -XXX,XX +XXX,XX @@
  static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
      int64_t offset, int bytes, BdrvRequestFlags flags);
--void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
+-void bdrv_parent_drained_begin(BlockDriverState *bs)
--                               bool ignore_bds_parents)
++void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore)
 +static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
 +                                      bool ignore_bds_parents)
  {
      BdrvChild *c, *next;
-@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_end_single(BdrvChild *c)
+     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
-     }
++        if (c == ignore) {
- }
++            continue;
++        }
--void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
+         if (c->role->drained_begin) {
--                             bool ignore_bds_parents)
+             c->role->drained_begin(c);
-+static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
+         }
-+                                    bool ignore_bds_parents)
+     }
  }
 -void bdrv_parent_drained_end(BlockDriverState *bs)
 +void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore)
  {
      BdrvChild *c, *next;
+     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
++        if (c == ignore) {
++            continue;
++        }
+         if (c->role->drained_end) {
+             c->role->drained_end(c);
+         }
+@@ -XXX,XX +XXX,XX @@ typedef struct {
+     BlockDriverState *bs;
+     bool done;
+     bool begin;
++    BdrvChild *parent;
+ } BdrvCoDrainData;
+ static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
+@@ -XXX,XX +XXX,XX @@ static bool bdrv_drain_recurse(BlockDriverState *bs)
+     return waited;
+ }
++static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent);
++static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent);
++
+ static void bdrv_co_drain_bh_cb(void *opaque)
+ {
+     BdrvCoDrainData *data = opaque;
+@@ -XXX,XX +XXX,XX @@ static void bdrv_co_drain_bh_cb(void *opaque)
+     bdrv_dec_in_flight(bs);
+     if (data->begin) {
+-        bdrv_drained_begin(bs);
++        bdrv_do_drained_begin(bs, data->parent);
+     } else {
+-        bdrv_drained_end(bs);
++        bdrv_do_drained_end(bs, data->parent);
+     }
+     data->done = true;
+@@ -XXX,XX +XXX,XX @@ static void bdrv_co_drain_bh_cb(void *opaque)
+ }
+ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
+-                                                bool begin)
++                                                bool begin, BdrvChild *parent)
+ {
+     BdrvCoDrainData data;
+@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
+         .bs = bs,
+         .done = false,
+         .begin = begin,
++        .parent = parent,
+     };
+     bdrv_inc_in_flight(bs);
+     aio_bh_schedule_oneshot(bdrv_get_aio_context(bs),
+@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
+     assert(data.done);
+ }
+-void bdrv_drained_begin(BlockDriverState *bs)
++static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent)
+ {
+     if (qemu_in_coroutine()) {
+-        bdrv_co_yield_to_drain(bs, true);
++        bdrv_co_yield_to_drain(bs, true, parent);
+         return;
+     }
+     /* Stop things in parent-to-child order */
+     if (atomic_fetch_inc(&bs->quiesce_counter) == 0) {
+         aio_disable_external(bdrv_get_aio_context(bs));
+-        bdrv_parent_drained_begin(bs);
+     }
++    bdrv_parent_drained_begin(bs, parent);
+     bdrv_drain_invoke(bs, true, false);
+     bdrv_drain_recurse(bs);
+ }
+-void bdrv_drained_end(BlockDriverState *bs)
++void bdrv_drained_begin(BlockDriverState *bs)
++{
++    bdrv_do_drained_begin(bs, NULL);
++}
++
++static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent)
+ {
+     int old_quiesce_counter;
+     if (qemu_in_coroutine()) {
+-        bdrv_co_yield_to_drain(bs, false);
++        bdrv_co_yield_to_drain(bs, false, parent);
+         return;
+     }
+     assert(bs->quiesce_counter > 0);
+@@ -XXX,XX +XXX,XX @@ void bdrv_drained_end(BlockDriverState *bs)
+     /* Re-enable things in child-to-parent order */
+     bdrv_drain_invoke(bs, false, false);
++    bdrv_parent_drained_end(bs, parent);
+     if (old_quiesce_counter == 1) {
+-        bdrv_parent_drained_end(bs);
+         aio_enable_external(bdrv_get_aio_context(bs));
+     }
+ }
++void bdrv_drained_end(BlockDriverState *bs)
++{
++    bdrv_do_drained_end(bs, NULL);
++}
++
+ /*
+  * Wait for pending requests to complete on a single BlockDriverState subtree,
+  * and suspend block driver's internal I/O until next request arrives.
+@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
+         /* Stop things in parent-to-child order */
+         aio_context_acquire(aio_context);
+         aio_disable_external(aio_context);
+-        bdrv_parent_drained_begin(bs);
++        bdrv_parent_drained_begin(bs, NULL);
+         bdrv_drain_invoke(bs, true, true);
+         aio_context_release(aio_context);
+@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_end(void)
+         /* Re-enable things in child-to-parent order */
+         aio_context_acquire(aio_context);
+         bdrv_drain_invoke(bs, false, true);
+-        bdrv_parent_drained_end(bs);
++        bdrv_parent_drained_end(bs, NULL);
+         aio_enable_external(aio_context);
+         aio_context_release(aio_context);
+     }
 --
-.20.1
+.13.6

-[Qemu-devel] [PULL 07/13] block: Do not poll in bdrv_do_drained_end()
+[Qemu-devel] [PULL v3 28/35] block: Add bdrv_subtree_drained_begin/end()
-From: Max Reitz <mreitz@redhat.com>
+bdrv_drained_begin() waits for the completion of requests in the whole
 subtree, but it only actually keeps its immediate bs parameter quiesced
 until bdrv_drained_end().
-We should never poll anywhere in bdrv_do_drained_end() (including its
+Add a version that keeps the whole subtree drained. As of this commit,
-recursive callees like bdrv_drain_invoke()), because it does not cope
+graph changes cannot be allowed during a subtree drained section, but
-well with graph changes.  In fact, it has been written based on the
+this will be fixed soon.
 postulation that no graph changes will happen in it.
-Instead, the callers that want to poll must poll, i.e. all currently
-globally available wrappers: bdrv_drained_end(),
-bdrv_subtree_drained_end(), bdrv_unapply_subtree_drain(), and
-bdrv_drain_all_end().  Graph changes there do not matter.
-They can poll simply by passing a pointer to a drained_end_counter and
-wait until it reaches 0.
-This patch also adds a non-polling global wrapper for
-bdrv_do_drained_end() that takes a drained_end_counter pointer.  We need
-such a variant because now no function called anywhere from
-bdrv_do_drained_end() must poll.  This includes
-BdrvChildRole.drained_end(), which already must not poll according to
-its interface documentation, but bdrv_child_cb_drained_end() just
-violates that by invoking bdrv_drained_end() (which does poll).
-Therefore, BdrvChildRole.drained_end() must take a *drained_end_counter
-parameter, which bdrv_child_cb_drained_end() can pass on to the new
-bdrv_drained_end_no_poll() function.
-Note that we now have a pattern of all drained_end-related functions
-either polling or receiving a *drained_end_counter to let the caller
-poll based on that.
-A problem with a single poll loop is that when the drained section in
-bdrv_set_aio_context_ignore() ends, some nodes in the subgraph may be in
-the old contexts, while others are in the new context already.  To let
-the collective poll in bdrv_drained_end() work correctly, we must not
-hold a lock to the old context, so that the old context can make
-progress in case it is different from the current context.
-(In the process, remove the comment saying that the current context is
-always the old context, because it is wrong.)
-In all other places, all nodes in a subtree must be in the same context,
-so we can just poll that.  The exception of course is
-bdrv_drain_all_end(), but that always runs in the main context, so we
-can just poll NULL (like bdrv_drain_all_begin() does).
-Signed-off-by: Max Reitz <mreitz@redhat.com>
 Signed-off-by: Kevin Wolf <kwolf@redhat.com>
 ---
- include/block/block.h     | 25 ++++++++++++
+ include/block/block.h | 13 +++++++++++++
- include/block/block_int.h |  6 ++-
+ block/io.c            | 54 ++++++++++++++++++++++++++++++++++++++++-----------
- block.c                   | 37 ++++++++++++++----
+files changed, 56 insertions(+), 11 deletions(-)
  block/block-backend.c     |  6 +--
  block/io.c                | 80 ++++++++++++++++++++++++++++-----------
  blockjob.c                |  2 +-
 files changed, 120 insertions(+), 36 deletions(-)
 diff --git a/include/block/block.h b/include/block/block.h
 index XXXXXXX..XXXXXXX 100644
 --- a/include/block/block.h
 +++ b/include/block/block.h
-@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll);
+@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore);
-  * bdrv_parent_drained_end_single:
+ void bdrv_drained_begin(BlockDriverState *bs);
-  *
-  * End a quiesced section for the parent of @c.
+ /**
 + * Like bdrv_drained_begin, but recursively begins a quiesced section for
 + * exclusive access to all child nodes as well.
 + *
-+ * This polls @bs's AioContext until all scheduled sub-drained_ends
++ * Graph changes are not allowed during a subtree drain section.
-+ * have settled, which may result in graph changes.
++ */
-  */
++void bdrv_subtree_drained_begin(BlockDriverState *bs);
- void bdrv_parent_drained_end_single(BdrvChild *c);
++
++/**
@@ -XXX,XX +XXX,XX @@ void bdrv_subtree_drained_begin(BlockDriverState *bs);
   * bdrv_drained_end:
   *
   * End a quiescent section started by bdrv_drained_begin().
-+ *
-+ * This polls @bs's AioContext until all scheduled sub-drained_ends
-+ * have settled.  On one hand, that may result in graph changes.  On
-+ * the other, this requires that all involved nodes (@bs and all of
-+ * its parents) are in the same AioContext, and that the caller has
-+ * acquired it.
-+ * If there are any nodes that are in different contexts from @bs,
-+ * these contexts must not be acquired.
   */
  void bdrv_drained_end(BlockDriverState *bs);
 +/**
-+ * bdrv_drained_end_no_poll:
++ * End a quiescent section started by bdrv_subtree_drained_begin().
 + *
 + * Same as bdrv_drained_end(), but do not poll for the subgraph to
 + * actually become unquiesced.  Therefore, no graph changes will occur
 + * with this function.
 + *
 + * *drained_end_counter is incremented for every background operation
 + * that is scheduled, and will be decremented for every operation once
 + * it settles.  The caller must poll until it reaches 0.  The counter
 + * should be accessed using atomic operations only.
 + */
-+void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter);
++void bdrv_subtree_drained_end(BlockDriverState *bs);
 +
- /**
+ void bdrv_add_child(BlockDriverState *parent, BlockDriverState *child,
-  * End a quiescent section started by bdrv_subtree_drained_begin().
+                     Error **errp);
-  */
+ void bdrv_del_child(BlockDriverState *parent, BdrvChild *child, Error **errp);
 diff --git a/include/block/block_int.h b/include/block/block_int.h
 index XXXXXXX..XXXXXXX 100644
 --- a/include/block/block_int.h
 +++ b/include/block/block_int.h
@@ -XXX,XX +XXX,XX @@ struct BdrvChildRole {
       * These functions must not change the graph (and therefore also must not
       * call aio_poll(), which could change the graph indirectly).
       *
 +     * If drained_end() schedules background operations, it must atomically
 +     * increment *drained_end_counter for each such operation and atomically
 +     * decrement it once the operation has settled.
 +     *
       * Note that this can be nested. If drained_begin() was called twice, new
       * I/O is allowed only after drained_end() was called twice, too.
       */
      void (*drained_begin)(BdrvChild *child);
 -    void (*drained_end)(BdrvChild *child);
 +    void (*drained_end)(BdrvChild *child, int *drained_end_counter);
      /*
       * Returns whether the parent has pending requests for the child. This
 diff --git a/block.c b/block.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block.c
 +++ b/block.c
@@ -XXX,XX +XXX,XX @@ static bool bdrv_child_cb_drained_poll(BdrvChild *child)
      return bdrv_drain_poll(bs, false, NULL, false);
  }
 -static void bdrv_child_cb_drained_end(BdrvChild *child)
 +static void bdrv_child_cb_drained_end(BdrvChild *child,
 +                                      int *drained_end_counter)
  {
      BlockDriverState *bs = child->opaque;
 -    bdrv_drained_end(bs);
 +    bdrv_drained_end_no_poll(bs, drained_end_counter);
  }
  static void bdrv_child_cb_attach(BdrvChild *child)
@@ -XXX,XX +XXX,XX @@ static void bdrv_attach_aio_context(BlockDriverState *bs,
  void bdrv_set_aio_context_ignore(BlockDriverState *bs,
                                   AioContext *new_context, GSList **ignore)
  {
 +    AioContext *old_context = bdrv_get_aio_context(bs);
 +    AioContext *current_context = qemu_get_current_aio_context();
      BdrvChild *child;
 -    if (bdrv_get_aio_context(bs) == new_context) {
 +    if (old_context == new_context) {
          return;
      }
@@ -XXX,XX +XXX,XX @@ void bdrv_set_aio_context_ignore(BlockDriverState *bs,
      bdrv_detach_aio_context(bs);
 -    /* This function executes in the old AioContext so acquire the new one in
 -     * case it runs in a different thread.
 -     */
 -    aio_context_acquire(new_context);
 +    /* Acquire the new context, if necessary */
 +    if (current_context != new_context) {
 +        aio_context_acquire(new_context);
 +    }
 +
      bdrv_attach_aio_context(bs, new_context);
 +
 +    /*
 +     * If this function was recursively called from
 +     * bdrv_set_aio_context_ignore(), there may be nodes in the
 +     * subtree that have not yet been moved to the new AioContext.
 +     * Release the old one so bdrv_drained_end() can poll them.
 +     */
 +    if (current_context != old_context) {
 +        aio_context_release(old_context);
 +    }
 +
      bdrv_drained_end(bs);
 -    aio_context_release(new_context);
 +
 +    if (current_context != old_context) {
 +        aio_context_acquire(old_context);
 +    }
 +    if (current_context != new_context) {
 +        aio_context_release(new_context);
 +    }
  }
  static bool bdrv_parent_can_set_aio_context(BdrvChild *c, AioContext *ctx,
 diff --git a/block/block-backend.c b/block/block-backend.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/block-backend.c
 +++ b/block/block-backend.c
@@ -XXX,XX +XXX,XX @@ static void blk_root_inherit_options(int *child_flags, QDict *child_options,
  }
  static void blk_root_drained_begin(BdrvChild *child);
  static bool blk_root_drained_poll(BdrvChild *child);
 -static void blk_root_drained_end(BdrvChild *child);
 +static void blk_root_drained_end(BdrvChild *child, int *drained_end_counter);
  static void blk_root_change_media(BdrvChild *child, bool load);
  static void blk_root_resize(BdrvChild *child);
@@ -XXX,XX +XXX,XX @@ int blk_pread_unthrottled(BlockBackend *blk, int64_t offset, uint8_t *buf,
      blk_root_drained_begin(blk->root);
      ret = blk_pread(blk, offset, buf, count);
 -    blk_root_drained_end(blk->root);
 +    blk_root_drained_end(blk->root, NULL);
      return ret;
  }
@@ -XXX,XX +XXX,XX @@ static bool blk_root_drained_poll(BdrvChild *child)
      return !!blk->in_flight;
  }
 -static void blk_root_drained_end(BdrvChild *child)
 +static void blk_root_drained_end(BdrvChild *child, int *drained_end_counter)
  {
      BlockBackend *blk = child->opaque;
      assert(blk->quiesce_counter);
 diff --git a/block/io.c b/block/io.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/io.c
 +++ b/block/io.c
-@@ -XXX,XX +XXX,XX @@ static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
+@@ -XXX,XX +XXX,XX @@ typedef struct {
      BlockDriverState *bs;
      bool done;
      bool begin;
 +    bool recursive;
      BdrvChild *parent;
  } BdrvCoDrainData;
@@ -XXX,XX +XXX,XX @@ static bool bdrv_drain_recurse(BlockDriverState *bs)
      return waited;
  }
 -static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent);
 -static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent);
 +static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
 +                                  BdrvChild *parent);
 +static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
 +                                BdrvChild *parent);
  static void bdrv_co_drain_bh_cb(void *opaque)
  {
@@ -XXX,XX +XXX,XX @@ static void bdrv_co_drain_bh_cb(void *opaque)
      bdrv_dec_in_flight(bs);
      if (data->begin) {
 -        bdrv_do_drained_begin(bs, data->parent);
 +        bdrv_do_drained_begin(bs, data->recursive, data->parent);
      } else {
 -        bdrv_do_drained_end(bs, data->parent);
 +        bdrv_do_drained_end(bs, data->recursive, data->parent);
      }
+     data->done = true;
+@@ -XXX,XX +XXX,XX @@ static void bdrv_co_drain_bh_cb(void *opaque)
  }
--void bdrv_parent_drained_end_single(BdrvChild *c)
+ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
-+static void bdrv_parent_drained_end_single_no_poll(BdrvChild *c,
+-                                                bool begin, BdrvChild *parent)
-+                                                   int *drained_end_counter)
++                                                bool begin, bool recursive,
 +                                                BdrvChild *parent)
  {
-     assert(c->parent_quiesce_counter > 0);
+     BdrvCoDrainData data;
-     c->parent_quiesce_counter--;
-     if (c->role->drained_end) {
+@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
--        c->role->drained_end(c);
+         .bs = bs,
-+        c->role->drained_end(c, drained_end_counter);
+         .done = false,
          .begin = begin,
 +        .recursive = recursive,
          .parent = parent,
      };
      bdrv_inc_in_flight(bs);
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
      assert(data.done);
  }
 -static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent)
 +static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
 +                                  BdrvChild *parent)
  {
 +    BdrvChild *child, *next;
 +
      if (qemu_in_coroutine()) {
 -        bdrv_co_yield_to_drain(bs, true, parent);
 +        bdrv_co_yield_to_drain(bs, true, recursive, parent);
          return;
      }
+@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent)
+     bdrv_parent_drained_begin(bs, parent);
+     bdrv_drain_invoke(bs, true, false);
+     bdrv_drain_recurse(bs);
++
++    if (recursive) {
++        QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
++            bdrv_do_drained_begin(child->bs, true, child);
++        }
++    }
  }
-+void bdrv_parent_drained_end_single(BdrvChild *c)
+ void bdrv_drained_begin(BlockDriverState *bs)
-+{
+ {
-+    int drained_end_counter = 0;
+-    bdrv_do_drained_begin(bs, NULL);
-+    bdrv_parent_drained_end_single_no_poll(c, &drained_end_counter);
++    bdrv_do_drained_begin(bs, false, NULL);
 +    BDRV_POLL_WHILE(c->bs, atomic_read(&drained_end_counter) > 0);
 +}
 +
- static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
++void bdrv_subtree_drained_begin(BlockDriverState *bs)
--                                    bool ignore_bds_parents)
++{
-+                                    bool ignore_bds_parents,
++    bdrv_do_drained_begin(bs, true, NULL);
-+                                    int *drained_end_counter)
+ }
 -static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent)
 +static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
 +                                BdrvChild *parent)
  {
-     BdrvChild *c, *next;
++    BdrvChild *child, *next;
+     int old_quiesce_counter;
-@@ -XXX,XX +XXX,XX @@ static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
-         if (c == ignore || (ignore_bds_parents && c->role->parent_is_bds)) {
+     if (qemu_in_coroutine()) {
-             continue;
+-        bdrv_co_yield_to_drain(bs, false, parent);
-         }
++        bdrv_co_yield_to_drain(bs, false, recursive, parent);
--        bdrv_parent_drained_end_single(c);
+         return;
 +        bdrv_parent_drained_end_single_no_poll(c, drained_end_counter);
      }
+     assert(bs->quiesce_counter > 0);
+@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent)
+     if (old_quiesce_counter == 1) {
+         aio_enable_external(bdrv_get_aio_context(bs));
+     }
++
++    if (recursive) {
++        QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
++            bdrv_do_drained_end(child->bs, true, child);
++        }
++    }
  }
-@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
-     atomic_mb_set(&data->done, true);
-     bdrv_dec_in_flight(bs);
--    if (data->drained_end_counter) {
-+    if (!data->begin) {
-         atomic_dec(data->drained_end_counter);
-     }
--    if (data->begin || data->drained_end_counter) {
--        g_free(data);
--    }
-+    g_free(data);
- }
- /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */
-@@ -XXX,XX +XXX,XX @@ static void bdrv_drain_invoke(BlockDriverState *bs, bool begin,
-         .drained_end_counter = drained_end_counter,
-     };
--    if (!begin && drained_end_counter) {
-+    if (!begin) {
-         atomic_inc(drained_end_counter);
-     }
-@@ -XXX,XX +XXX,XX @@ static void bdrv_drain_invoke(BlockDriverState *bs, bool begin,
-     bdrv_inc_in_flight(bs);
-     data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data);
-     aio_co_schedule(bdrv_get_aio_context(bs), data->co);
--
--    /*
--     * TODO: Drop this and make callers pass @drained_end_counter and poll
--     * themselves
--     */
--    if (!begin && !drained_end_counter) {
--        BDRV_POLL_WHILE(bs, !data->done);
--        g_free(data);
--    }
- }
- /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */
-@@ -XXX,XX +XXX,XX @@ static void bdrv_co_drain_bh_cb(void *opaque)
-         }
-         bdrv_dec_in_flight(bs);
-         if (data->begin) {
-+            assert(!data->drained_end_counter);
-             bdrv_do_drained_begin(bs, data->recursive, data->parent,
-                                   data->ignore_bds_parents, data->poll);
-         } else {
-+            assert(!data->poll);
-             bdrv_do_drained_end(bs, data->recursive, data->parent,
-                                 data->ignore_bds_parents,
-                                 data->drained_end_counter);
-@@ -XXX,XX +XXX,XX @@ void bdrv_subtree_drained_begin(BlockDriverState *bs)
-     bdrv_do_drained_begin(bs, true, NULL, false, true);
- }
-+/**
-+ * This function does not poll, nor must any of its recursively called
-+ * functions.  The *drained_end_counter pointee will be incremented
-+ * once for every background operation scheduled, and decremented once
-+ * the operation settles.  Therefore, the pointer must remain valid
-+ * until the pointee reaches 0.  That implies that whoever sets up the
-+ * pointee has to poll until it is 0.
-+ *
-+ * We use atomic operations to access *drained_end_counter, because
-+ * (1) when called from bdrv_set_aio_context_ignore(), the subgraph of
-+ *     @bs may contain nodes in different AioContexts,
-+ * (2) bdrv_drain_all_end() uses the same counter for all nodes,
-+ *     regardless of which AioContext they are in.
-+ */
- static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
-                                 BdrvChild *parent, bool ignore_bds_parents,
-                                 int *drained_end_counter)
-@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
-     BdrvChild *child, *next;
-     int old_quiesce_counter;
-+    assert(drained_end_counter != NULL);
-+
-     if (qemu_in_coroutine()) {
-         bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents,
-                                false, drained_end_counter);
-@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
-     /* Re-enable things in child-to-parent order */
-     bdrv_drain_invoke(bs, false, drained_end_counter);
--    bdrv_parent_drained_end(bs, parent, ignore_bds_parents);
-+    bdrv_parent_drained_end(bs, parent, ignore_bds_parents,
-+                            drained_end_counter);
-     old_quiesce_counter = atomic_fetch_dec(&bs->quiesce_counter);
-     if (old_quiesce_counter == 1) {
-@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
  void bdrv_drained_end(BlockDriverState *bs)
  {
--    bdrv_do_drained_end(bs, false, NULL, false, NULL);
+-    bdrv_do_drained_end(bs, NULL);
-+    int drained_end_counter = 0;
++    bdrv_do_drained_end(bs, false, NULL);
 +    bdrv_do_drained_end(bs, false, NULL, false, &drained_end_counter);
 +    BDRV_POLL_WHILE(bs, atomic_read(&drained_end_counter) > 0);
 +}
 +
-+void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter)
++void bdrv_subtree_drained_end(BlockDriverState *bs)
 +{
-+    bdrv_do_drained_end(bs, false, NULL, false, drained_end_counter);
++    bdrv_do_drained_end(bs, true, NULL);
  }
- void bdrv_subtree_drained_end(BlockDriverState *bs)
- {
--    bdrv_do_drained_end(bs, true, NULL, false, NULL);
-+    int drained_end_counter = 0;
-+    bdrv_do_drained_end(bs, true, NULL, false, &drained_end_counter);
-+    BDRV_POLL_WHILE(bs, atomic_read(&drained_end_counter) > 0);
- }
- void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
-@@ -XXX,XX +XXX,XX @@ void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
- void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent)
- {
-+    int drained_end_counter = 0;
-     int i;
-     for (i = 0; i < old_parent->recursive_quiesce_counter; i++) {
--        bdrv_do_drained_end(child->bs, true, child, false, NULL);
-+        bdrv_do_drained_end(child->bs, true, child, false,
-+                            &drained_end_counter);
-     }
-+
-+    BDRV_POLL_WHILE(child->bs, atomic_read(&drained_end_counter) > 0);
- }
  /*
-@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
- void bdrv_drain_all_end(void)
- {
-     BlockDriverState *bs = NULL;
-+    int drained_end_counter = 0;
-     while ((bs = bdrv_next_all_states(bs))) {
-         AioContext *aio_context = bdrv_get_aio_context(bs);
-         aio_context_acquire(aio_context);
--        bdrv_do_drained_end(bs, false, NULL, true, NULL);
-+        bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter);
-         aio_context_release(aio_context);
-     }
-+    assert(qemu_get_current_aio_context() == qemu_get_aio_context());
-+    AIO_WAIT_WHILE(NULL, atomic_read(&drained_end_counter) > 0);
-+
-     assert(bdrv_drain_all_count > 0);
-     bdrv_drain_all_count--;
- }
-diff --git a/blockjob.c b/blockjob.c
-index XXXXXXX..XXXXXXX 100644
---- a/blockjob.c
-+++ b/blockjob.c
-@@ -XXX,XX +XXX,XX @@ static bool child_job_drained_poll(BdrvChild *c)
-     }
- }
--static void child_job_drained_end(BdrvChild *c)
-+static void child_job_drained_end(BdrvChild *c, int *drained_end_counter)
- {
-     BlockJob *job = c->opaque;
-     job_resume(&job->job);
 --
-.20.1
+.13.6

-[Qemu-devel] [PULL 11/13] iotests: Test commit with a filter on the chain
+[Qemu-devel] [PULL v3 29/35] test-bdrv-drain: Tests for bdrv_subtree_drain
-From: Max Reitz <mreitz@redhat.com>
+Add a subtree drain version to the existing test cases.
-Before the previous patches, the first case resulted in a failed
-assertion (which is noted as qemu receiving a SIGABRT in the test
-output), and the second usually triggered a segmentation fault.
-Signed-off-by: Max Reitz <mreitz@redhat.com>
 Signed-off-by: Kevin Wolf <kwolf@redhat.com>
 ---
- tests/qemu-iotests/040     | 40 +++++++++++++++++++++++++++++++++++++-
+ tests/test-bdrv-drain.c | 27 ++++++++++++++++++++++++++-
- tests/qemu-iotests/040.out |  4 ++--
+file changed, 26 insertions(+), 1 deletion(-)
 files changed, 41 insertions(+), 3 deletions(-)
-diff --git a/tests/qemu-iotests/040 b/tests/qemu-iotests/040
+diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
-index XXXXXXX..XXXXXXX 100755
+index XXXXXXX..XXXXXXX 100644
---- a/tests/qemu-iotests/040
+--- a/tests/test-bdrv-drain.c
-+++ b/tests/qemu-iotests/040
++++ b/tests/test-bdrv-drain.c
-@@ -XXX,XX +XXX,XX @@ class TestSingleDrive(ImageCommitTestCase):
+@@ -XXX,XX +XXX,XX @@ static void aio_ret_cb(void *opaque, int ret)
+ enum drain_type {
-         self.vm.add_device("scsi-hd,id=scsi0,drive=drive0")
+     BDRV_DRAIN_ALL,
-         self.vm.launch()
+     BDRV_DRAIN,
-+        self.has_quit = False
++    BDRV_SUBTREE_DRAIN,
+     DRAIN_TYPE_MAX,
-     def tearDown(self):
+ };
--        self.vm.shutdown()
-+        self.vm.shutdown(has_quit=self.has_quit)
+@@ -XXX,XX +XXX,XX @@ static void do_drain_begin(enum drain_type drain_type, BlockDriverState *bs)
-         os.remove(test_img)
+     switch (drain_type) {
-         os.remove(mid_img)
+     case BDRV_DRAIN_ALL:        bdrv_drain_all_begin(); break;
-         os.remove(backing_img)
+     case BDRV_DRAIN:            bdrv_drained_begin(bs); break;
-@@ -XXX,XX +XXX,XX @@ class TestSingleDrive(ImageCommitTestCase):
++    case BDRV_SUBTREE_DRAIN:    bdrv_subtree_drained_begin(bs); break;
-         self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xab 0 524288', backing_img).find("verification failed"))
+     default:                    g_assert_not_reached();
-         self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xef 524288 524288', backing_img).find("verification failed"))
+     }
+ }
-+    def test_commit_with_filter_and_quit(self):
+@@ -XXX,XX +XXX,XX @@ static void do_drain_end(enum drain_type drain_type, BlockDriverState *bs)
-+        result = self.vm.qmp('object-add', qom_type='throttle-group', id='tg')
+     switch (drain_type) {
-+        self.assert_qmp(result, 'return', {})
+     case BDRV_DRAIN_ALL:        bdrv_drain_all_end(); break;
      case BDRV_DRAIN:            bdrv_drained_end(bs); break;
 +    case BDRV_SUBTREE_DRAIN:    bdrv_subtree_drained_end(bs); break;
      default:                    g_assert_not_reached();
      }
  }
@@ -XXX,XX +XXX,XX @@ static void test_drv_cb_drain(void)
      test_drv_cb_common(BDRV_DRAIN, false);
  }
 +static void test_drv_cb_drain_subtree(void)
 +{
 +    test_drv_cb_common(BDRV_SUBTREE_DRAIN, true);
 +}
 +
-+        # Add a filter outside of the backing chain
+ static void test_quiesce_common(enum drain_type drain_type, bool recursive)
-+        result = self.vm.qmp('blockdev-add', driver='throttle', node_name='filter', throttle_group='tg', file='mid')
+ {
-+        self.assert_qmp(result, 'return', {})
+     BlockBackend *blk;
@@ -XXX,XX +XXX,XX @@ static void test_quiesce_drain(void)
      test_quiesce_common(BDRV_DRAIN, false);
  }
 +static void test_quiesce_drain_subtree(void)
 +{
 +    test_quiesce_common(BDRV_SUBTREE_DRAIN, true);
 +}
 +
-+        result = self.vm.qmp('block-commit', device='drive0')
+ static void test_nested(void)
-+        self.assert_qmp(result, 'return', {})
+ {
      BlockBackend *blk;
@@ -XXX,XX +XXX,XX @@ static void test_nested(void)
              /* XXX bdrv_drain_all() doesn't increase the quiesce_counter */
              int bs_quiesce      = (outer != BDRV_DRAIN_ALL) +
                                    (inner != BDRV_DRAIN_ALL);
 -            int backing_quiesce = 0;
 +            int backing_quiesce = (outer == BDRV_SUBTREE_DRAIN) +
 +                                  (inner == BDRV_SUBTREE_DRAIN);
              int backing_cb_cnt  = (outer != BDRV_DRAIN) +
                                    (inner != BDRV_DRAIN);
@@ -XXX,XX +XXX,XX @@ static void test_blockjob_drain(void)
      test_blockjob_common(BDRV_DRAIN);
  }
 +static void test_blockjob_drain_subtree(void)
 +{
 +    test_blockjob_common(BDRV_SUBTREE_DRAIN);
 +}
 +
-+        # Quit immediately, thus forcing a simultaneous cancel of the
+ int main(int argc, char **argv)
-+        # block job and a bdrv_drain_all()
+ {
-+        result = self.vm.qmp('quit')
+     bdrv_init();
-+        self.assert_qmp(result, 'return', {})
+@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
-+
-+        self.has_quit = True
+     g_test_add_func("/bdrv-drain/driver-cb/drain_all", test_drv_cb_drain_all);
-+
+     g_test_add_func("/bdrv-drain/driver-cb/drain", test_drv_cb_drain);
-+    # Same as above, but this time we add the filter after starting the job
++    g_test_add_func("/bdrv-drain/driver-cb/drain_subtree",
-+    def test_commit_plus_filter_and_quit(self):
++                    test_drv_cb_drain_subtree);
-+        result = self.vm.qmp('object-add', qom_type='throttle-group', id='tg')
-+        self.assert_qmp(result, 'return', {})
+     g_test_add_func("/bdrv-drain/quiesce/drain_all", test_quiesce_drain_all);
-+
+     g_test_add_func("/bdrv-drain/quiesce/drain", test_quiesce_drain);
-+        result = self.vm.qmp('block-commit', device='drive0')
++    g_test_add_func("/bdrv-drain/quiesce/drain_subtree",
-+        self.assert_qmp(result, 'return', {})
++                    test_quiesce_drain_subtree);
-+
-+        # Add a filter outside of the backing chain
+     g_test_add_func("/bdrv-drain/nested", test_nested);
-+        result = self.vm.qmp('blockdev-add', driver='throttle', node_name='filter', throttle_group='tg', file='mid')
-+        self.assert_qmp(result, 'return', {})
+     g_test_add_func("/bdrv-drain/blockjob/drain_all", test_blockjob_drain_all);
-+
+     g_test_add_func("/bdrv-drain/blockjob/drain", test_blockjob_drain);
-+        # Quit immediately, thus forcing a simultaneous cancel of the
++    g_test_add_func("/bdrv-drain/blockjob/drain_subtree",
-+        # block job and a bdrv_drain_all()
++                    test_blockjob_drain_subtree);
-+        result = self.vm.qmp('quit')
-+        self.assert_qmp(result, 'return', {})
+     return g_test_run();
-+
+ }
 +        self.has_quit = True
 +
      def test_device_not_found(self):
          result = self.vm.qmp('block-commit', device='nonexistent', top='%s' % mid_img)
          self.assert_qmp(result, 'error/class', 'DeviceNotFound')
 diff --git a/tests/qemu-iotests/040.out b/tests/qemu-iotests/040.out
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/040.out
 +++ b/tests/qemu-iotests/040.out
@@ -XXX,XX +XXX,XX @@
 -...........................................
 +...............................................
  ----------------------------------------------------------------------
 -Ran 43 tests
 +Ran 47 tests
  OK
 --
-.20.1
+.13.6

-[Qemu-devel] [PULL 10/13] iotests: Add @has_quit to vm.shutdown()
+[Qemu-devel] [PULL v3 30/35] test-bdrv-drain: Test behaviour in coroutine context
-From: Max Reitz <mreitz@redhat.com>
+If bdrv_do_drained_begin/end() are called in coroutine context, they
 first use a BH to get out of the coroutine context. Call some existing
 tests again from a coroutine to cover this code path.
-If a test has issued a quit command already (which may be useful to do
-explicitly because the test wants to show its effects),
-QEMUMachine.shutdown() should not do so again.  Otherwise, the VM may
-well return an ECONNRESET which will lead QEMUMachine.shutdown() to
-killing it, which then turns into a "qemu received signal 9" line.
-Signed-off-by: Max Reitz <mreitz@redhat.com>
 Signed-off-by: Kevin Wolf <kwolf@redhat.com>
 ---
- python/qemu/machine.py | 5 +++--
+ tests/test-bdrv-drain.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++
- tests/qemu-iotests/255 | 2 +-
+file changed, 59 insertions(+)
 files changed, 4 insertions(+), 3 deletions(-)
-diff --git a/python/qemu/machine.py b/python/qemu/machine.py
+diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
 index XXXXXXX..XXXXXXX 100644
---- a/python/qemu/machine.py
+--- a/tests/test-bdrv-drain.c
-+++ b/python/qemu/machine.py
++++ b/tests/test-bdrv-drain.c
-@@ -XXX,XX +XXX,XX @@ class QEMUMachine(object):
+@@ -XXX,XX +XXX,XX @@ static void aio_ret_cb(void *opaque, int ret)
-         self._load_io_log()
+     *aio_ret = ret;
-         self._post_shutdown()
+ }
--    def shutdown(self):
++typedef struct CallInCoroutineData {
-+    def shutdown(self, has_quit=False):
++    void (*entry)(void);
-         """
++    bool done;
-         Terminate the VM and clean up
++} CallInCoroutineData;
-         """
++
-         if self.is_running():
++static coroutine_fn void call_in_coroutine_entry(void *opaque)
-             try:
++{
--                self._qmp.cmd('quit')
++    CallInCoroutineData *data = opaque;
-+                if not has_quit:
++
-+                    self._qmp.cmd('quit')
++    data->entry();
-                 self._qmp.close()
++    data->done = true;
-             except:
++}
-                 self._popen.kill()
++
-diff --git a/tests/qemu-iotests/255 b/tests/qemu-iotests/255
++static void call_in_coroutine(void (*entry)(void))
-index XXXXXXX..XXXXXXX 100755
++{
---- a/tests/qemu-iotests/255
++    Coroutine *co;
-+++ b/tests/qemu-iotests/255
++    CallInCoroutineData data = {
-@@ -XXX,XX +XXX,XX @@ with iotests.FilePath('src.qcow2') as src_path, \
++        .entry  = entry,
-     vm.qmp_log('block-job-cancel', device='job0')
++        .done   = false,
-     vm.qmp_log('quit')
++    };
++
--    vm.shutdown()
++    co = qemu_coroutine_create(call_in_coroutine_entry, &data);
-+    vm.shutdown(has_quit=True)
++    qemu_coroutine_enter(co);
 +    while (!data.done) {
 +        aio_poll(qemu_get_aio_context(), true);
 +    }
 +}
 +
  enum drain_type {
      BDRV_DRAIN_ALL,
      BDRV_DRAIN,
@@ -XXX,XX +XXX,XX @@ static void test_drv_cb_drain_subtree(void)
      test_drv_cb_common(BDRV_SUBTREE_DRAIN, true);
  }
 +static void test_drv_cb_co_drain(void)
 +{
 +    call_in_coroutine(test_drv_cb_drain);
 +}
 +
 +static void test_drv_cb_co_drain_subtree(void)
 +{
 +    call_in_coroutine(test_drv_cb_drain_subtree);
 +}
 +
  static void test_quiesce_common(enum drain_type drain_type, bool recursive)
  {
      BlockBackend *blk;
@@ -XXX,XX +XXX,XX @@ static void test_quiesce_drain_subtree(void)
      test_quiesce_common(BDRV_SUBTREE_DRAIN, true);
  }
 +static void test_quiesce_co_drain(void)
 +{
 +    call_in_coroutine(test_quiesce_drain);
 +}
 +
 +static void test_quiesce_co_drain_subtree(void)
 +{
 +    call_in_coroutine(test_quiesce_drain_subtree);
 +}
 +
  static void test_nested(void)
  {
      BlockBackend *blk;
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
      g_test_add_func("/bdrv-drain/driver-cb/drain_subtree",
                      test_drv_cb_drain_subtree);
 +    // XXX bdrv_drain_all() doesn't work in coroutine context
 +    g_test_add_func("/bdrv-drain/driver-cb/co/drain", test_drv_cb_co_drain);
 +    g_test_add_func("/bdrv-drain/driver-cb/co/drain_subtree",
 +                    test_drv_cb_co_drain_subtree);
 +
 +
      g_test_add_func("/bdrv-drain/quiesce/drain_all", test_quiesce_drain_all);
      g_test_add_func("/bdrv-drain/quiesce/drain", test_quiesce_drain);
      g_test_add_func("/bdrv-drain/quiesce/drain_subtree",
                      test_quiesce_drain_subtree);
 +    // XXX bdrv_drain_all() doesn't work in coroutine context
 +    g_test_add_func("/bdrv-drain/quiesce/co/drain", test_quiesce_co_drain);
 +    g_test_add_func("/bdrv-drain/quiesce/co/drain_subtree",
 +                    test_quiesce_co_drain_subtree);
 +
      g_test_add_func("/bdrv-drain/nested", test_nested);
      g_test_add_func("/bdrv-drain/blockjob/drain_all", test_blockjob_drain_all);
 --
-.20.1
+.13.6

-[Qemu-devel] [PULL 01/13] iotests: Set read-zeroes on in null block driver for Valgrind
+[Qemu-devel] [PULL v3 31/35] test-bdrv-drain: Recursive draining with multiple parents
-From: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
+Test that drain sections are correctly propagated through the graph.
-The Valgrind tool reports about the uninitialised buffer 'buf'
-instantiated on the stack of the function guess_disk_lchs().
-Pass 'read-zeroes=on' to the null block driver to make it deterministic.
-The output of the tests 051, 186 and 227 now includes the parameter
-'read-zeroes'. So, the benchmark output files are being changed too.
-Suggested-by: Kevin Wolf <kwolf@redhat.com>
-Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
 Signed-off-by: Kevin Wolf <kwolf@redhat.com>
 ---
- tests/qemu-iotests/051        |  10 +--
+ tests/test-bdrv-drain.c | 74 +++++++++++++++++++++++++++++++++++++++++++++++++
- tests/qemu-iotests/051.pc.out |  10 +--
+file changed, 74 insertions(+)
  tests/qemu-iotests/093        |   9 +-
  tests/qemu-iotests/136        |   1 +
  tests/qemu-iotests/186        |  20 ++---
  tests/qemu-iotests/186.out    | 152 +++++++++++++++++-----------------
  tests/qemu-iotests/227        |   4 +-
  tests/qemu-iotests/227.out    |   4 +-
  tests/qemu-iotests/238        |   2 +-
  tests/qemu-iotests/240        |   8 +-
 files changed, 111 insertions(+), 109 deletions(-)
-diff --git a/tests/qemu-iotests/051 b/tests/qemu-iotests/051
+diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/051
 +++ b/tests/qemu-iotests/051
@@ -XXX,XX +XXX,XX @@ echo
  # Cannot use the test image because cache=none might not work on the host FS
  # Use cdrom so that we won't get errors about missing media
 -run_qemu -drive driver=null-co,cache=none
 -run_qemu -drive driver=null-co,cache=directsync
 -run_qemu -drive driver=null-co,cache=writeback
 -run_qemu -drive driver=null-co,cache=writethrough
 -run_qemu -drive driver=null-co,cache=unsafe
 +run_qemu -drive driver=null-co,read-zeroes=on,cache=none
 +run_qemu -drive driver=null-co,read-zeroes=on,cache=directsync
 +run_qemu -drive driver=null-co,read-zeroes=on,cache=writeback
 +run_qemu -drive driver=null-co,read-zeroes=on,cache=writethrough
 +run_qemu -drive driver=null-co,read-zeroes=on,cache=unsafe
  run_qemu -drive driver=null-co,cache=invalid_value
  # Can't test direct=on here because O_DIRECT might not be supported on this FS
 diff --git a/tests/qemu-iotests/051.pc.out b/tests/qemu-iotests/051.pc.out
 index XXXXXXX..XXXXXXX 100644
---- a/tests/qemu-iotests/051.pc.out
+--- a/tests/test-bdrv-drain.c
-+++ b/tests/qemu-iotests/051.pc.out
++++ b/tests/test-bdrv-drain.c
-@@ -XXX,XX +XXX,XX @@ QEMU X.Y.Z monitor - type 'help' for more information
+@@ -XXX,XX +XXX,XX @@ static void test_nested(void)
+     blk_unref(blk);
  === Cache modes ===
 -Testing: -drive driver=null-co,cache=none
 +Testing: -drive driver=null-co,read-zeroes=on,cache=none
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) quit
 -Testing: -drive driver=null-co,cache=directsync
 +Testing: -drive driver=null-co,read-zeroes=on,cache=directsync
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) quit
 -Testing: -drive driver=null-co,cache=writeback
 +Testing: -drive driver=null-co,read-zeroes=on,cache=writeback
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) quit
 -Testing: -drive driver=null-co,cache=writethrough
 +Testing: -drive driver=null-co,read-zeroes=on,cache=writethrough
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) quit
 -Testing: -drive driver=null-co,cache=unsafe
 +Testing: -drive driver=null-co,read-zeroes=on,cache=unsafe
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) quit
 diff --git a/tests/qemu-iotests/093 b/tests/qemu-iotests/093
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/093
 +++ b/tests/qemu-iotests/093
@@ -XXX,XX +XXX,XX @@ class ThrottleTestCase(iotests.QMPTestCase):
      def setUp(self):
          self.vm = iotests.VM()
          for i in range(0, self.max_drives):
 -            self.vm.add_drive(self.test_img)
 +            self.vm.add_drive(self.test_img, "file.read-zeroes=on")
          self.vm.launch()
      def tearDown(self):
@@ -XXX,XX +XXX,XX @@ class ThrottleTestGroupNames(iotests.QMPTestCase):
      def setUp(self):
          self.vm = iotests.VM()
          for i in range(0, self.max_drives):
 -            self.vm.add_drive(self.test_img, "throttling.iops-total=100")
 +            self.vm.add_drive(self.test_img,
 +                              "throttling.iops-total=100,file.read-zeroes=on")
          self.vm.launch()
      def tearDown(self):
@@ -XXX,XX +XXX,XX @@ class ThrottleTestRemovableMedia(iotests.QMPTestCase):
      def test_removable_media(self):
          # Add a couple of dummy nodes named cd0 and cd1
          result = self.vm.qmp("blockdev-add", driver="null-aio",
 -                             node_name="cd0")
 +                             read_zeroes=True, node_name="cd0")
          self.assert_qmp(result, 'return', {})
          result = self.vm.qmp("blockdev-add", driver="null-aio",
 -                             node_name="cd1")
 +                             read_zeroes=True, node_name="cd1")
          self.assert_qmp(result, 'return', {})
          # Attach a CD drive with cd0 inserted
 diff --git a/tests/qemu-iotests/136 b/tests/qemu-iotests/136
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/136
 +++ b/tests/qemu-iotests/136
@@ -XXX,XX +XXX,XX @@ sector = "%d"
                            (self.account_invalid and "on" or "off"))
          drive_args.append("stats-account-failed=%s" %
                            (self.account_failed and "on" or "off"))
 +        drive_args.append("file.image.read-zeroes=on")
          self.create_blkdebug_file()
          self.vm = iotests.VM().add_drive('blkdebug:%s:%s' %
                                           (blkdebug_file, self.test_img),
 diff --git a/tests/qemu-iotests/186 b/tests/qemu-iotests/186
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/186
 +++ b/tests/qemu-iotests/186
@@ -XXX,XX +XXX,XX @@ echo "=== -blockdev/-device=<node-name> ==="
  echo
  for dev in $fixed $removable; do
 -    check_info_block -blockdev driver=null-co,node-name=null -device $dev,drive=null
 -    check_info_block -blockdev driver=null-co,node-name=null -device $dev,drive=null,id=qdev_id
 +    check_info_block -blockdev driver=null-co,read-zeroes=on,node-name=null -device $dev,drive=null
 +    check_info_block -blockdev driver=null-co,read-zeroes=on,node-name=null -device $dev,drive=null,id=qdev_id
  done
  echo
@@ -XXX,XX +XXX,XX @@ echo
  # This creates two BlockBackends that will show up in 'info block'!
  # A monitor-owned one from -drive, and anonymous one from -device
  for dev in $fixed $removable; do
 -    check_info_block -drive if=none,driver=null-co,node-name=null -device $dev,drive=null,id=qdev_id
 +    check_info_block -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device $dev,drive=null,id=qdev_id
  done
  echo
@@ -XXX,XX +XXX,XX @@ echo "=== -drive if=none/-device=<bb-name> (with medium) ==="
  echo
  for dev in $fixed $removable; do
 -    check_info_block -drive if=none,driver=null-co,node-name=null -device $dev,drive=none0
 -    check_info_block -drive if=none,driver=null-co,node-name=null -device $dev,drive=none0,id=qdev_id
 +    check_info_block -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device $dev,drive=none0
 +    check_info_block -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device $dev,drive=none0,id=qdev_id
  done
  echo
@@ -XXX,XX +XXX,XX @@ echo "=== -drive if=... ==="
  echo
  check_info_block -drive if=floppy
 -check_info_block -drive if=floppy,driver=null-co
 +check_info_block -drive if=floppy,driver=null-co,read-zeroes=on
 -check_info_block -drive if=ide,driver=null-co
 +check_info_block -drive if=ide,driver=null-co,read-zeroes=on
  check_info_block -drive if=ide,media=cdrom
 -check_info_block -drive if=ide,driver=null-co,media=cdrom
 +check_info_block -drive if=ide,driver=null-co,read-zeroes=on,media=cdrom
 -check_info_block -drive if=virtio,driver=null-co
 +check_info_block -drive if=virtio,driver=null-co,read-zeroes=on
 -check_info_block -drive if=pflash,driver=null-co,size=1M
 +check_info_block -drive if=pflash,driver=null-co,read-zeroes=on,size=1M
  # success, all done
  echo "*** done"
 diff --git a/tests/qemu-iotests/186.out b/tests/qemu-iotests/186.out
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/186.out
 +++ b/tests/qemu-iotests/186.out
@@ -XXX,XX +XXX,XX @@ qdev_id: [not inserted]
  === -blockdev/-device=<node-name> ===
 -Testing: -blockdev driver=null-co,node-name=null -device ide-hd,drive=null
 +Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device ide-hd,drive=null
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
      Attached to:      PATH
      Cache mode:       writeback
  (qemu) quit
 -Testing: -blockdev driver=null-co,node-name=null -device ide-hd,drive=null,id=qdev_id
 +Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device ide-hd,drive=null,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
      Attached to:      qdev_id
      Cache mode:       writeback
  (qemu) quit
 -Testing: -blockdev driver=null-co,node-name=null -device scsi-hd,drive=null
 +Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device scsi-hd,drive=null
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
      Attached to:      PATH
      Cache mode:       writeback
  (qemu) quit
 -Testing: -blockdev driver=null-co,node-name=null -device scsi-hd,drive=null,id=qdev_id
 +Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device scsi-hd,drive=null,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
      Attached to:      qdev_id
      Cache mode:       writeback
  (qemu) quit
 -Testing: -blockdev driver=null-co,node-name=null -device virtio-blk-pci,drive=null
 +Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device virtio-blk-pci,drive=null
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
      Attached to:      PATH
      Cache mode:       writeback
  (qemu) quit
 -Testing: -blockdev driver=null-co,node-name=null -device virtio-blk-pci,drive=null,id=qdev_id
 +Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device virtio-blk-pci,drive=null,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
      Attached to:      PATH
      Cache mode:       writeback
  (qemu) quit
 -Testing: -blockdev driver=null-co,node-name=null -device floppy,drive=null
 +Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device floppy,drive=null
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
      Attached to:      PATH
      Removable device: not locked, tray closed
      Cache mode:       writeback
  (qemu) quit
 -Testing: -blockdev driver=null-co,node-name=null -device floppy,drive=null,id=qdev_id
 +Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device floppy,drive=null,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
      Attached to:      qdev_id
      Removable device: not locked, tray closed
      Cache mode:       writeback
  (qemu) quit
 -Testing: -blockdev driver=null-co,node-name=null -device ide-cd,drive=null
 +Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device ide-cd,drive=null
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
      Attached to:      PATH
      Removable device: not locked, tray closed
      Cache mode:       writeback
  (qemu) quit
 -Testing: -blockdev driver=null-co,node-name=null -device ide-cd,drive=null,id=qdev_id
 +Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device ide-cd,drive=null,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
      Attached to:      qdev_id
      Removable device: not locked, tray closed
      Cache mode:       writeback
  (qemu) quit
 -Testing: -blockdev driver=null-co,node-name=null -device scsi-cd,drive=null
 +Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device scsi-cd,drive=null
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
      Attached to:      PATH
      Removable device: not locked, tray closed
      Cache mode:       writeback
  (qemu) quit
 -Testing: -blockdev driver=null-co,node-name=null -device scsi-cd,drive=null,id=qdev_id
 +Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device scsi-cd,drive=null,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
      Attached to:      qdev_id
      Removable device: not locked, tray closed
      Cache mode:       writeback
@@ -XXX,XX +XXX,XX @@ null: null-co:// (null-co)
  === -drive if=none/-device=<node-name> ===
 -Testing: -drive if=none,driver=null-co,node-name=null -device ide-hd,drive=null,id=qdev_id
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device ide-hd,drive=null,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Removable device: not locked, tray closed
      Cache mode:       writeback
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      qdev_id
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device scsi-hd,drive=null,id=qdev_id
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device scsi-hd,drive=null,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Removable device: not locked, tray closed
      Cache mode:       writeback
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      qdev_id
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device virtio-blk-pci,drive=null,id=qdev_id
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device virtio-blk-pci,drive=null,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Removable device: not locked, tray closed
      Cache mode:       writeback
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      PATH
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device floppy,drive=null,id=qdev_id
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device floppy,drive=null,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Removable device: not locked, tray closed
      Cache mode:       writeback
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      qdev_id
      Removable device: not locked, tray closed
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device ide-cd,drive=null,id=qdev_id
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device ide-cd,drive=null,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Removable device: not locked, tray closed
      Cache mode:       writeback
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      qdev_id
      Removable device: not locked, tray closed
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device scsi-cd,drive=null,id=qdev_id
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device scsi-cd,drive=null,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Removable device: not locked, tray closed
      Cache mode:       writeback
 -null: null-co:// (null-co)
 +null: json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      qdev_id
      Removable device: not locked, tray closed
      Cache mode:       writeback
@@ -XXX,XX +XXX,XX @@ null: null-co:// (null-co)
  === -drive if=none/-device=<bb-name> (with medium) ===
 -Testing: -drive if=none,driver=null-co,node-name=null -device ide-hd,drive=none0
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device ide-hd,drive=none0
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      PATH
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device ide-hd,drive=none0,id=qdev_id
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device ide-hd,drive=none0,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      qdev_id
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device scsi-hd,drive=none0
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device scsi-hd,drive=none0
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      PATH
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device scsi-hd,drive=none0,id=qdev_id
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device scsi-hd,drive=none0,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      qdev_id
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device virtio-blk-pci,drive=none0
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device virtio-blk-pci,drive=none0
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      PATH
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device virtio-blk-pci,drive=none0,id=qdev_id
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device virtio-blk-pci,drive=none0,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      PATH
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device floppy,drive=none0
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device floppy,drive=none0
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      PATH
      Removable device: not locked, tray closed
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device floppy,drive=none0,id=qdev_id
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device floppy,drive=none0,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      qdev_id
      Removable device: not locked, tray closed
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device ide-cd,drive=none0
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device ide-cd,drive=none0
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      PATH
      Removable device: not locked, tray closed
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device ide-cd,drive=none0,id=qdev_id
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device ide-cd,drive=none0,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      qdev_id
      Removable device: not locked, tray closed
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device scsi-cd,drive=none0
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device scsi-cd,drive=none0
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      PATH
      Removable device: not locked, tray closed
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=none,driver=null-co,node-name=null -device scsi-cd,drive=none0,id=qdev_id
 +Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device scsi-cd,drive=none0,id=qdev_id
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -none0 (null): null-co:// (null-co)
 +none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      qdev_id
      Removable device: not locked, tray closed
      Cache mode:       writeback
@@ -XXX,XX +XXX,XX @@ floppy0: [not inserted]
      Removable device: not locked, tray closed
  (qemu) quit
 -Testing: -drive if=floppy,driver=null-co
 +Testing: -drive if=floppy,driver=null-co,read-zeroes=on
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -floppy0 (NODE_NAME): null-co:// (null-co)
 +floppy0 (NODE_NAME): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      PATH
      Removable device: not locked, tray closed
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=ide,driver=null-co
 +Testing: -drive if=ide,driver=null-co,read-zeroes=on
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -ide0-hd0 (NODE_NAME): null-co:// (null-co)
 +ide0-hd0 (NODE_NAME): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      PATH
      Cache mode:       writeback
  (qemu) quit
@@ -XXX,XX +XXX,XX @@ ide0-cd0: [not inserted]
      Removable device: not locked, tray closed
  (qemu) quit
 -Testing: -drive if=ide,driver=null-co,media=cdrom
 +Testing: -drive if=ide,driver=null-co,read-zeroes=on,media=cdrom
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -ide0-cd0 (NODE_NAME): null-co:// (null-co, read-only)
 +ide0-cd0 (NODE_NAME): json:{"read-zeroes": "on", "driver": "null-co"} (null-co, read-only)
      Attached to:      PATH
      Removable device: not locked, tray closed
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=virtio,driver=null-co
 +Testing: -drive if=virtio,driver=null-co,read-zeroes=on
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -virtio0 (NODE_NAME): null-co:// (null-co)
 +virtio0 (NODE_NAME): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
      Attached to:      PATH
      Cache mode:       writeback
  (qemu) quit
 -Testing: -drive if=pflash,driver=null-co,size=1M
 +Testing: -drive if=pflash,driver=null-co,read-zeroes=on,size=1M
  QEMU X.Y.Z monitor - type 'help' for more information
  (qemu) info block
 -pflash0 (NODE_NAME): json:{"driver": "null-co", "size": "1M"} (null-co)
 +pflash0 (NODE_NAME): json:{"read-zeroes": "on", "driver": "null-co", "size": "1M"} (null-co)
      Attached to:      PATH
      Cache mode:       writeback
  (qemu) quit
 diff --git a/tests/qemu-iotests/227 b/tests/qemu-iotests/227
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/227
 +++ b/tests/qemu-iotests/227
@@ -XXX,XX +XXX,XX @@ echo
  echo '=== blockstats with -drive if=virtio ==='
  echo
 -run_qemu -drive driver=null-co,if=virtio <<EOF
 +run_qemu -drive driver=null-co,read-zeroes=on,if=virtio <<EOF
  { "execute": "qmp_capabilities" }
  { "execute": "query-blockstats"}
  { "execute": "quit" }
@@ -XXX,XX +XXX,XX @@ echo
  echo '=== blockstats with -blockdev and -device ==='
  echo
 -run_qemu -blockdev driver=null-co,node-name=null -device virtio-blk,drive=null,id=virtio0 <<EOF
 +run_qemu -blockdev driver=null-co,read-zeroes=on,node-name=null -device virtio-blk,drive=null,id=virtio0 <<EOF
  { "execute": "qmp_capabilities" }
  { "execute": "query-blockstats"}
  { "execute": "quit" }
 diff --git a/tests/qemu-iotests/227.out b/tests/qemu-iotests/227.out
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/227.out
 +++ b/tests/qemu-iotests/227.out
@@ -XXX,XX +XXX,XX @@ QA output created by 227
  === blockstats with -drive if=virtio ===
 -Testing: -drive driver=null-co,if=virtio
 +Testing: -drive driver=null-co,read-zeroes=on,if=virtio
  {
      QMP_VERSION
  }
-@@ -XXX,XX +XXX,XX @@ Testing: -blockdev driver=null-co,node-name=null
++static void test_multiparent(void)
- === blockstats with -blockdev and -device ===
++{
++    BlockBackend *blk_a, *blk_b;
--Testing: -blockdev driver=null-co,node-name=null -device virtio-blk,drive=null,id=virtio0
++    BlockDriverState *bs_a, *bs_b, *backing;
-+Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device virtio-blk,drive=null,id=virtio0
++    BDRVTestState *a_s, *b_s, *backing_s;
- {
++
-     QMP_VERSION
++    blk_a = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
- }
++    bs_a = bdrv_new_open_driver(&bdrv_test, "test-node-a", BDRV_O_RDWR,
-diff --git a/tests/qemu-iotests/238 b/tests/qemu-iotests/238
++                                &error_abort);
-index XXXXXXX..XXXXXXX 100755
++    a_s = bs_a->opaque;
---- a/tests/qemu-iotests/238
++    blk_insert_bs(blk_a, bs_a, &error_abort);
-+++ b/tests/qemu-iotests/238
++
-@@ -XXX,XX +XXX,XX @@ else:
++    blk_b = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
- vm = iotests.VM()
++    bs_b = bdrv_new_open_driver(&bdrv_test, "test-node-b", BDRV_O_RDWR,
- vm.launch()
++                                &error_abort);
++    b_s = bs_b->opaque;
--log(vm.qmp('blockdev-add', node_name='hd0', driver='null-co'))
++    blk_insert_bs(blk_b, bs_b, &error_abort);
-+log(vm.qmp('blockdev-add', node_name='hd0', driver='null-co', read_zeroes=True))
++
- log(vm.qmp('object-add', qom_type='iothread', id='iothread0'))
++    backing = bdrv_new_open_driver(&bdrv_test, "backing", 0, &error_abort);
- log(vm.qmp('device_add', id='scsi0', driver=virtio_scsi_device, iothread='iothread0'))
++    backing_s = backing->opaque;
- log(vm.qmp('device_add', id='scsi-hd0', driver='scsi-hd', drive='hd0'))
++    bdrv_set_backing_hd(bs_a, backing, &error_abort);
-diff --git a/tests/qemu-iotests/240 b/tests/qemu-iotests/240
++    bdrv_set_backing_hd(bs_b, backing, &error_abort);
-index XXXXXXX..XXXXXXX 100755
++
---- a/tests/qemu-iotests/240
++    g_assert_cmpint(bs_a->quiesce_counter, ==, 0);
-+++ b/tests/qemu-iotests/240
++    g_assert_cmpint(bs_b->quiesce_counter, ==, 0);
-@@ -XXX,XX +XXX,XX @@ echo
++    g_assert_cmpint(backing->quiesce_counter, ==, 0);
++    g_assert_cmpint(a_s->drain_count, ==, 0);
- run_qemu <<EOF
++    g_assert_cmpint(b_s->drain_count, ==, 0);
- { "execute": "qmp_capabilities" }
++    g_assert_cmpint(backing_s->drain_count, ==, 0);
--{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0"}}
++
-+{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "read-zeroes": true, "node-name": "hd0"}}
++    do_drain_begin(BDRV_SUBTREE_DRAIN, bs_a);
- { "execute": "object-add", "arguments": {"qom-type": "iothread", "id": "iothread0"}}
++
- { "execute": "device_add", "arguments": {"id": "scsi0", "driver": "${virtio_scsi}", "iothread": "iothread0"}}
++    g_assert_cmpint(bs_a->quiesce_counter, ==, 1);
- { "execute": "device_add", "arguments": {"id": "scsi-hd0", "driver": "scsi-hd", "drive": "hd0"}}
++    g_assert_cmpint(bs_b->quiesce_counter, ==, 1);
-@@ -XXX,XX +XXX,XX @@ echo
++    g_assert_cmpint(backing->quiesce_counter, ==, 1);
++    g_assert_cmpint(a_s->drain_count, ==, 1);
- run_qemu <<EOF
++    g_assert_cmpint(b_s->drain_count, ==, 1);
- { "execute": "qmp_capabilities" }
++    g_assert_cmpint(backing_s->drain_count, ==, 1);
--{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0", "read-only": true}}
++
-+{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "read-zeroes": true, "node-name": "hd0", "read-only": true}}
++    do_drain_begin(BDRV_SUBTREE_DRAIN, bs_b);
- { "execute": "object-add", "arguments": {"qom-type": "iothread", "id": "iothread0"}}
++
- { "execute": "device_add", "arguments": {"id": "scsi0", "driver": "${virtio_scsi}", "iothread": "iothread0"}}
++    g_assert_cmpint(bs_a->quiesce_counter, ==, 2);
- { "execute": "device_add", "arguments": {"id": "scsi-hd0", "driver": "scsi-hd", "drive": "hd0"}}
++    g_assert_cmpint(bs_b->quiesce_counter, ==, 2);
-@@ -XXX,XX +XXX,XX @@ echo
++    g_assert_cmpint(backing->quiesce_counter, ==, 2);
++    g_assert_cmpint(a_s->drain_count, ==, 2);
- run_qemu <<EOF
++    g_assert_cmpint(b_s->drain_count, ==, 2);
- { "execute": "qmp_capabilities" }
++    g_assert_cmpint(backing_s->drain_count, ==, 2);
--{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0", "read-only": true}}
++
-+{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "read-zeroes": true, "node-name": "hd0", "read-only": true}}
++    do_drain_end(BDRV_SUBTREE_DRAIN, bs_b);
- { "execute": "object-add", "arguments": {"qom-type": "iothread", "id": "iothread0"}}
++
- { "execute": "object-add", "arguments": {"qom-type": "iothread", "id": "iothread1"}}
++    g_assert_cmpint(bs_a->quiesce_counter, ==, 1);
- { "execute": "device_add", "arguments": {"id": "scsi0", "driver": "${virtio_scsi}", "iothread": "iothread0"}}
++    g_assert_cmpint(bs_b->quiesce_counter, ==, 1);
-@@ -XXX,XX +XXX,XX @@ echo
++    g_assert_cmpint(backing->quiesce_counter, ==, 1);
++    g_assert_cmpint(a_s->drain_count, ==, 1);
- run_qemu <<EOF
++    g_assert_cmpint(b_s->drain_count, ==, 1);
- { "execute": "qmp_capabilities" }
++    g_assert_cmpint(backing_s->drain_count, ==, 1);
--{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0", "read-only": true}}
++
-+{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "read-zeroes": true, "node-name": "hd0", "read-only": true}}
++    do_drain_end(BDRV_SUBTREE_DRAIN, bs_a);
- { "execute": "nbd-server-start", "arguments": {"addr":{"type":"unix","data":{"path":"$TEST_DIR/nbd"}}}}
++
- { "execute": "nbd-server-add", "arguments": {"device":"hd0"}}
++    g_assert_cmpint(bs_a->quiesce_counter, ==, 0);
- { "execute": "object-add", "arguments": {"qom-type": "iothread", "id": "iothread0"}}
++    g_assert_cmpint(bs_b->quiesce_counter, ==, 0);
 +    g_assert_cmpint(backing->quiesce_counter, ==, 0);
 +    g_assert_cmpint(a_s->drain_count, ==, 0);
 +    g_assert_cmpint(b_s->drain_count, ==, 0);
 +    g_assert_cmpint(backing_s->drain_count, ==, 0);
 +
 +    bdrv_unref(backing);
 +    bdrv_unref(bs_a);
 +    bdrv_unref(bs_b);
 +    blk_unref(blk_a);
 +    blk_unref(blk_b);
 +}
 +
  typedef struct TestBlockJob {
      BlockJob common;
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
                      test_quiesce_co_drain_subtree);
      g_test_add_func("/bdrv-drain/nested", test_nested);
 +    g_test_add_func("/bdrv-drain/multiparent", test_multiparent);
      g_test_add_func("/bdrv-drain/blockjob/drain_all", test_blockjob_drain_all);
      g_test_add_func("/bdrv-drain/blockjob/drain", test_blockjob_drain);
 --
-.20.1
+.13.6

-[Qemu-devel] [PULL 02/13] block: Introduce BdrvChild.parent_quiesce_counter
+[Qemu-devel] [PULL v3 32/35] block: Allow graph changes in subtree drained section
-From: Max Reitz <mreitz@redhat.com>
+We need to remember how many of the drain sections in which a node is
+were recursive (i.e. subtree drain rather than node drain), so that they
-Commit 5cb2737e925042e6c7cd3fb0b01313950b03cddf laid out why
+can be correctly applied when children are added or removed during the
-bdrv_do_drained_end() must decrement the quiesce_counter after
+drained section.
-bdrv_drain_invoke().  It did not give a very good reason why it has to
-happen after bdrv_parent_drained_end(), instead only claiming symmetry
+With this change, it is safe to modify the graph even inside a
-to bdrv_do_drained_begin().
+bdrv_subtree_drained_begin/end() section.
 It turns out that delaying it for so long is wrong.
 Situation: We have an active commit job (i.e. a mirror job) from top to
 base for the following graph:
                   filter
                     |
                   [file]
                     |
                     v
 top --[backing]--> base
 Now the VM is closed, which results in the job being cancelled and a
 bdrv_drain_all() happening pretty much simultaneously.
 Beginning the drain means the job is paused once whenever one of its
 nodes is quiesced.  This is reversed when the drain ends.
 With how the code currently is, after base's drain ends (which means
 that it will have unpaused the job once), its quiesce_counter remains at
 while it goes to undrain its parents (bdrv_parent_drained_end()).  For
 some reason or another, undraining filter causes the job to be kicked
 and enter mirror_exit_common(), where it proceeds to invoke
 block_job_remove_all_bdrv().
 Now base will be detached from the job.  Because its quiesce_counter is
 still 1, it will unpause the job once more.  So in total, undraining
 base will unpause the job twice.  Eventually, this will lead to the
 job's pause_count going negative -- well, it would, were there not an
 assertion against this, which crashes qemu.
 The general problem is that if in bdrv_parent_drained_end() we undrain
 parent A, and then undrain parent B, which then leads to A detaching the
 child, bdrv_replace_child_noperm() will undrain A as if we had not done
 so yet; that is, one time too many.
 It follows that we cannot decrement the quiesce_counter after invoking
 bdrv_parent_drained_end().
 Unfortunately, decrementing it before bdrv_parent_drained_end() would be
 wrong, too.  Imagine the above situation in reverse: Undraining A leads
 to B detaching the child.  If we had already decremented the
 quiesce_counter by that point, bdrv_replace_child_noperm() would undrain
 B one time too little; because it expects bdrv_parent_drained_end() to
 issue this undrain.  But bdrv_parent_drained_end() won't do that,
 because B is no longer a parent.
 Therefore, we have to do something else.  This patch opts for
 introducing a second quiesce_counter that counts how many times a
 child's parent has been quiesced (though c->role->drained_*).  With
 that, bdrv_replace_child_noperm() just has to undrain the parent exactly
 that many times when removing a child, and it will always be right.
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 Signed-off-by: Kevin Wolf <kwolf@redhat.com>
 ---
- include/block/block.h     |  7 +++++++
+ include/block/block.h     |  2 --
- include/block/block_int.h |  9 +++++++++
+ include/block/block_int.h |  5 +++++
- block.c                   | 15 +++++----------
+ block.c                   | 32 +++++++++++++++++++++++++++++---
- block/io.c                | 14 +++++++++++---
+ block/io.c                | 28 ++++++++++++++++++++++++----
-files changed, 32 insertions(+), 13 deletions(-)
+files changed, 58 insertions(+), 9 deletions(-)
 diff --git a/include/block/block.h b/include/block/block.h
 index XXXXXXX..XXXXXXX 100644
 --- a/include/block/block.h
 +++ b/include/block/block.h
-@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
+@@ -XXX,XX +XXX,XX @@ void bdrv_drained_begin(BlockDriverState *bs);
  /**
   * Like bdrv_drained_begin, but recursively begins a quiesced section for
   * exclusive access to all child nodes as well.
 - *
 - * Graph changes are not allowed during a subtree drain section.
   */
- void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll);
+ void bdrv_subtree_drained_begin(BlockDriverState *bs);
 +/**
 + * bdrv_parent_drained_end_single:
 + *
 + * End a quiesced section for the parent of @c.
 + */
 +void bdrv_parent_drained_end_single(BdrvChild *c);
 +
  /**
   * bdrv_parent_drained_end:
   *
 diff --git a/include/block/block_int.h b/include/block/block_int.h
 index XXXXXXX..XXXXXXX 100644
 --- a/include/block/block_int.h
 +++ b/include/block/block_int.h
-@@ -XXX,XX +XXX,XX @@ struct BdrvChild {
+@@ -XXX,XX +XXX,XX @@ struct BlockDriverState {
-      */
-     bool frozen;
+     /* Accessed with atomic ops.  */
+     int quiesce_counter;
-+    /*
++    int recursive_quiesce_counter;
-+     * How many times the parent of this child has been drained
++
-+     * (through role->drained_*).
+     unsigned int write_gen;               /* Current data generation */
-+     * Usually, this is equal to bs->quiesce_counter (potentially
-+     * reduced by bdrv_drain_all_count).  It may differ while the
+     /* Protected by reqs_lock.  */
-+     * child is entering or leaving a drained section.
+@@ -XXX,XX +XXX,XX @@ int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
-+     */
+     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
-+    int parent_quiesce_counter;
+     BdrvRequestFlags flags);
-+
-     QLIST_ENTRY(BdrvChild) next;
++void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent);
-     QLIST_ENTRY(BdrvChild) next_parent;
++void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent);
- };
++
  int get_tmp_filename(char *filename, int size);
  BlockDriver *bdrv_probe_all(const uint8_t *buf, int buf_size,
                              const char *filename);
 diff --git a/block.c b/block.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block.c
 +++ b/block.c
+@@ -XXX,XX +XXX,XX @@ static void bdrv_child_cb_drained_end(BdrvChild *child)
+     bdrv_drained_end(bs);
+ }
++static void bdrv_child_cb_attach(BdrvChild *child)
++{
++    BlockDriverState *bs = child->opaque;
++    bdrv_apply_subtree_drain(child, bs);
++}
++
++static void bdrv_child_cb_detach(BdrvChild *child)
++{
++    BlockDriverState *bs = child->opaque;
++    bdrv_unapply_subtree_drain(child, bs);
++}
++
+ static int bdrv_child_cb_inactivate(BdrvChild *child)
+ {
+     BlockDriverState *bs = child->opaque;
+@@ -XXX,XX +XXX,XX @@ const BdrvChildRole child_file = {
+     .inherit_options = bdrv_inherited_options,
+     .drained_begin   = bdrv_child_cb_drained_begin,
+     .drained_end     = bdrv_child_cb_drained_end,
++    .attach          = bdrv_child_cb_attach,
++    .detach          = bdrv_child_cb_detach,
+     .inactivate      = bdrv_child_cb_inactivate,
+ };
+@@ -XXX,XX +XXX,XX @@ const BdrvChildRole child_format = {
+     .inherit_options = bdrv_inherited_fmt_options,
+     .drained_begin   = bdrv_child_cb_drained_begin,
+     .drained_end     = bdrv_child_cb_drained_end,
++    .attach          = bdrv_child_cb_attach,
++    .detach          = bdrv_child_cb_detach,
+     .inactivate      = bdrv_child_cb_inactivate,
+ };
+@@ -XXX,XX +XXX,XX @@ static void bdrv_backing_attach(BdrvChild *c)
+                     parent->backing_blocker);
+     bdrv_op_unblock(backing_hd, BLOCK_OP_TYPE_BACKUP_TARGET,
+                     parent->backing_blocker);
++
++    bdrv_child_cb_attach(c);
+ }
+ static void bdrv_backing_detach(BdrvChild *c)
+@@ -XXX,XX +XXX,XX @@ static void bdrv_backing_detach(BdrvChild *c)
+     bdrv_op_unblock_all(c->bs, parent->backing_blocker);
+     error_free(parent->backing_blocker);
+     parent->backing_blocker = NULL;
++
++    bdrv_child_cb_detach(c);
+ }
+ /*
 @@ -XXX,XX +XXX,XX @@ static void bdrv_replace_child_noperm(BdrvChild *child,
-         if (child->role->detach) {
+         assert(bdrv_get_aio_context(old_bs) == bdrv_get_aio_context(new_bs));
-             child->role->detach(child);
+     }
-         }
+     if (old_bs) {
--        if (old_bs->quiesce_counter && child->role->drained_end) {
++        /* Detach first so that the recursive drain sections coming from @child
--            int num = old_bs->quiesce_counter;
++         * are already gone and we only end the drain sections that came from
--            if (child->role->parent_is_bds) {
++         * elsewhere. */
--                num -= bdrv_drain_all_count;
++        if (child->role->detach) {
--            }
++            child->role->detach(child);
--            assert(num >= 0);
++        }
--            for (i = 0; i < num; i++) {
+         if (old_bs->quiesce_counter && child->role->drained_end) {
--                child->role->drained_end(child);
+             for (i = 0; i < old_bs->quiesce_counter; i++) {
--            }
+                 child->role->drained_end(child);
-+        while (child->parent_quiesce_counter) {
+             }
-+            bdrv_parent_drained_end_single(child);
+         }
-         }
+-        if (child->role->detach) {
 -            child->role->detach(child);
 -        }
          QLIST_REMOVE(child, next_parent);
-+    } else {
-+        assert(child->parent_quiesce_counter == 0);
      }
-     child->bs = new_bs;
+@@ -XXX,XX +XXX,XX @@ static void bdrv_replace_child_noperm(BdrvChild *child,
+             }
-     if (new_bs) {
+         }
-         QLIST_INSERT_HEAD(&new_bs->parents, child, next_parent);
--        if (new_bs->quiesce_counter && child->role->drained_begin) {
++        /* Attach only after starting new drained sections, so that recursive
-+        if (new_bs->quiesce_counter) {
++         * drain sections coming from @child don't get an extra .drained_begin
-             int num = new_bs->quiesce_counter;
++         * callback. */
-             if (child->role->parent_is_bds) {
+         if (child->role->attach) {
-                 num -= bdrv_drain_all_count;
+             child->role->attach(child);
          }
 diff --git a/block/io.c b/block/io.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/io.c
 +++ b/block/io.c
-@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
+@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
      assert(data.done);
  }
 -static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
 -                                  BdrvChild *parent)
 +void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
 +                           BdrvChild *parent)
  {
      BdrvChild *child, *next;
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
      bdrv_drain_recurse(bs);
      if (recursive) {
 +        bs->recursive_quiesce_counter++;
          QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
              bdrv_do_drained_begin(child->bs, true, child);
          }
@@ -XXX,XX +XXX,XX @@ void bdrv_subtree_drained_begin(BlockDriverState *bs)
      bdrv_do_drained_begin(bs, true, NULL);
  }
 -static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
 -                                BdrvChild *parent)
 +void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
 +                         BdrvChild *parent)
  {
      BdrvChild *child, *next;
      int old_quiesce_counter;
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
      }
- }
+     if (recursive) {
-+void bdrv_parent_drained_end_single(BdrvChild *c)
++        bs->recursive_quiesce_counter--;
-+{
+         QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
-+    assert(c->parent_quiesce_counter > 0);
+             bdrv_do_drained_end(child->bs, true, child);
-+    c->parent_quiesce_counter--;
+         }
-+    if (c->role->drained_end) {
+@@ -XXX,XX +XXX,XX @@ void bdrv_subtree_drained_end(BlockDriverState *bs)
-+        c->role->drained_end(c);
+     bdrv_do_drained_end(bs, true, NULL);
  }
 +void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
 +{
 +    int i;
 +
 +    for (i = 0; i < new_parent->recursive_quiesce_counter; i++) {
 +        bdrv_do_drained_begin(child->bs, true, child);
 +    }
 +}
 +
- void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
++void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent)
-                              bool ignore_bds_parents)
++{
- {
++    int i;
-@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
++
-         if (c == ignore || (ignore_bds_parents && c->role->parent_is_bds)) {
++    for (i = 0; i < old_parent->recursive_quiesce_counter; i++) {
-             continue;
++        bdrv_do_drained_end(child->bs, true, child);
-         }
++    }
--        if (c->role->drained_end) {
++}
--            c->role->drained_end(c);
++
--        }
+ /*
-+        bdrv_parent_drained_end_single(c);
+  * Wait for pending requests to complete on a single BlockDriverState subtree,
-     }
+  * and suspend block driver's internal I/O until next request arrives.
  }
@@ -XXX,XX +XXX,XX @@ static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore,
  void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll)
  {
 +    c->parent_quiesce_counter++;
      if (c->role->drained_begin) {
          c->role->drained_begin(c);
      }
 --
-.20.1
+.13.6

-[Qemu-devel] [PULL 08/13] tests: Extend commit by drained_end test
+[Qemu-devel] [PULL v3 33/35] test-bdrv-drain: Test graph changes in drained section
-From: Max Reitz <mreitz@redhat.com>
-Signed-off-by: Max Reitz <mreitz@redhat.com>
 Signed-off-by: Kevin Wolf <kwolf@redhat.com>
 ---
- tests/test-bdrv-drain.c | 36 ++++++++++++++++++++++++++++++++----
+ tests/test-bdrv-drain.c | 80 +++++++++++++++++++++++++++++++++++++++++++++++++
-file changed, 32 insertions(+), 4 deletions(-)
+file changed, 80 insertions(+)
 diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/test-bdrv-drain.c
 +++ b/tests/test-bdrv-drain.c
-@@ -XXX,XX +XXX,XX @@ typedef struct TestDropBackingBlockJob {
+@@ -XXX,XX +XXX,XX @@ static void test_multiparent(void)
      blk_unref(blk_b);
  }
 +static void test_graph_change(void)
 +{
 +    BlockBackend *blk_a, *blk_b;
 +    BlockDriverState *bs_a, *bs_b, *backing;
 +    BDRVTestState *a_s, *b_s, *backing_s;
 +
 +    blk_a = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
 +    bs_a = bdrv_new_open_driver(&bdrv_test, "test-node-a", BDRV_O_RDWR,
 +                                &error_abort);
 +    a_s = bs_a->opaque;
 +    blk_insert_bs(blk_a, bs_a, &error_abort);
 +
 +    blk_b = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
 +    bs_b = bdrv_new_open_driver(&bdrv_test, "test-node-b", BDRV_O_RDWR,
 +                                &error_abort);
 +    b_s = bs_b->opaque;
 +    blk_insert_bs(blk_b, bs_b, &error_abort);
 +
 +    backing = bdrv_new_open_driver(&bdrv_test, "backing", 0, &error_abort);
 +    backing_s = backing->opaque;
 +    bdrv_set_backing_hd(bs_a, backing, &error_abort);
 +
 +    g_assert_cmpint(bs_a->quiesce_counter, ==, 0);
 +    g_assert_cmpint(bs_b->quiesce_counter, ==, 0);
 +    g_assert_cmpint(backing->quiesce_counter, ==, 0);
 +    g_assert_cmpint(a_s->drain_count, ==, 0);
 +    g_assert_cmpint(b_s->drain_count, ==, 0);
 +    g_assert_cmpint(backing_s->drain_count, ==, 0);
 +
 +    do_drain_begin(BDRV_SUBTREE_DRAIN, bs_a);
 +    do_drain_begin(BDRV_SUBTREE_DRAIN, bs_a);
 +    do_drain_begin(BDRV_SUBTREE_DRAIN, bs_a);
 +    do_drain_begin(BDRV_SUBTREE_DRAIN, bs_b);
 +    do_drain_begin(BDRV_SUBTREE_DRAIN, bs_b);
 +
 +    bdrv_set_backing_hd(bs_b, backing, &error_abort);
 +    g_assert_cmpint(bs_a->quiesce_counter, ==, 5);
 +    g_assert_cmpint(bs_b->quiesce_counter, ==, 5);
 +    g_assert_cmpint(backing->quiesce_counter, ==, 5);
 +    g_assert_cmpint(a_s->drain_count, ==, 5);
 +    g_assert_cmpint(b_s->drain_count, ==, 5);
 +    g_assert_cmpint(backing_s->drain_count, ==, 5);
 +
 +    bdrv_set_backing_hd(bs_b, NULL, &error_abort);
 +    g_assert_cmpint(bs_a->quiesce_counter, ==, 3);
 +    g_assert_cmpint(bs_b->quiesce_counter, ==, 2);
 +    g_assert_cmpint(backing->quiesce_counter, ==, 3);
 +    g_assert_cmpint(a_s->drain_count, ==, 3);
 +    g_assert_cmpint(b_s->drain_count, ==, 2);
 +    g_assert_cmpint(backing_s->drain_count, ==, 3);
 +
 +    bdrv_set_backing_hd(bs_b, backing, &error_abort);
 +    g_assert_cmpint(bs_a->quiesce_counter, ==, 5);
 +    g_assert_cmpint(bs_b->quiesce_counter, ==, 5);
 +    g_assert_cmpint(backing->quiesce_counter, ==, 5);
 +    g_assert_cmpint(a_s->drain_count, ==, 5);
 +    g_assert_cmpint(b_s->drain_count, ==, 5);
 +    g_assert_cmpint(backing_s->drain_count, ==, 5);
 +
 +    do_drain_end(BDRV_SUBTREE_DRAIN, bs_b);
 +    do_drain_end(BDRV_SUBTREE_DRAIN, bs_b);
 +    do_drain_end(BDRV_SUBTREE_DRAIN, bs_a);
 +    do_drain_end(BDRV_SUBTREE_DRAIN, bs_a);
 +    do_drain_end(BDRV_SUBTREE_DRAIN, bs_a);
 +
 +    g_assert_cmpint(bs_a->quiesce_counter, ==, 0);
 +    g_assert_cmpint(bs_b->quiesce_counter, ==, 0);
 +    g_assert_cmpint(backing->quiesce_counter, ==, 0);
 +    g_assert_cmpint(a_s->drain_count, ==, 0);
 +    g_assert_cmpint(b_s->drain_count, ==, 0);
 +    g_assert_cmpint(backing_s->drain_count, ==, 0);
 +
 +    bdrv_unref(backing);
 +    bdrv_unref(bs_a);
 +    bdrv_unref(bs_b);
 +    blk_unref(blk_a);
 +    blk_unref(blk_b);
 +}
 +
  typedef struct TestBlockJob {
      BlockJob common;
-     bool should_complete;
+@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
-     bool *did_complete;
-+    BlockDriverState *detach_also;
+     g_test_add_func("/bdrv-drain/nested", test_nested);
- } TestDropBackingBlockJob;
+     g_test_add_func("/bdrv-drain/multiparent", test_multiparent);
++    g_test_add_func("/bdrv-drain/graph-change", test_graph_change);
- static int coroutine_fn test_drop_backing_job_run(Job *job, Error **errp)
-@@ -XXX,XX +XXX,XX @@ static void test_drop_backing_job_commit(Job *job)
+     g_test_add_func("/bdrv-drain/blockjob/drain_all", test_blockjob_drain_all);
-         container_of(job, TestDropBackingBlockJob, common.job);
+     g_test_add_func("/bdrv-drain/blockjob/drain", test_blockjob_drain);
      bdrv_set_backing_hd(blk_bs(s->common.blk), NULL, &error_abort);
 +    bdrv_set_backing_hd(s->detach_also, NULL, &error_abort);
      *s->did_complete = true;
  }
@@ -XXX,XX +XXX,XX @@ static const BlockJobDriver test_drop_backing_job_driver = {
   * Creates a child node with three parent nodes on it, and then runs a
   * block job on the final one, parent-node-2.
   *
 - * (TODO: parent-node-0 currently serves no purpose, but will as of a
 - * follow-up patch.)
 - *
   * The job is then asked to complete before a section where the child
   * is drained.
   *
@@ -XXX,XX +XXX,XX @@ static const BlockJobDriver test_drop_backing_job_driver = {
   *
   * Ending the drain on parent-node-1 will poll the AioContext, which
   * lets job_exit() and thus test_drop_backing_job_commit() run.  That
 - * function removes the child as parent-node-2's backing file.
 + * function first removes the child as parent-node-2's backing file.
   *
   * In old (and buggy) implementations, there are two problems with
   * that:
@@ -XXX,XX +XXX,XX @@ static const BlockJobDriver test_drop_backing_job_driver = {
   * bdrv_replace_child_noperm() therefore must call drained_end() on
   * the parent only if it really is still drained because the child is
   * drained.
 + *
 + * If removing child from parent-node-2 was successful (as it should
 + * be), test_drop_backing_job_commit() will then also remove the child
 + * from parent-node-0.
 + *
 + * With an old version of our drain infrastructure ((A) above), that
 + * resulted in the following flow:
 + *
 + * 1. child attempts to leave its drained section.  The call recurses
 + *    to its parents.
 + *
 + * 2. parent-node-2 leaves the drained section.  Polling in
 + *    bdrv_drain_invoke() will schedule job_exit().
 + *
 + * 3. parent-node-1 leaves the drained section.  Polling in
 + *    bdrv_drain_invoke() will run job_exit(), thus disconnecting
 + *    parent-node-0 from the child node.
 + *
 + * 4. bdrv_parent_drained_end() uses a QLIST_FOREACH_SAFE() loop to
 + *    iterate over the parents.  Thus, it now accesses the BdrvChild
 + *    object that used to connect parent-node-0 and the child node.
 + *    However, that object no longer exists, so it accesses a dangling
 + *    pointer.
 + *
 + * The solution is to only poll once when running a bdrv_drained_end()
 + * operation, specifically at the end when all drained_end()
 + * operations for all involved nodes have been scheduled.
 + * Note that this also solves (A) above, thus hiding (B).
   */
  static void test_blockjob_commit_by_drained_end(void)
  {
@@ -XXX,XX +XXX,XX @@ static void test_blockjob_commit_by_drained_end(void)
                             bs_parents[2], 0, BLK_PERM_ALL, 0, 0, NULL, NULL,
                             &error_abort);
 +    job->detach_also = bs_parents[0];
      job->did_complete = &job_has_completed;
      job_start(&job->common.job);
 --
-.20.1
+.13.6

-New patch
+[Qemu-devel] [PULL v3 34/35] commit: Simplify reopen of base
+Since commit bde70715, base is the only node that is reopened in
+commit_start(). This means that the code, which still involves an
+explicit BlockReopenQueue, can now be simplified by using bdrv_reopen().
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+Reviewed-by: Fam Zheng <famz@redhat.com>
+---
+ block/commit.c | 8 +-------
+file changed, 1 insertion(+), 7 deletions(-)
+diff --git a/block/commit.c b/block/commit.c
+index XXXXXXX..XXXXXXX 100644
+--- a/block/commit.c
++++ b/block/commit.c
+@@ -XXX,XX +XXX,XX @@ void commit_start(const char *job_id, BlockDriverState *bs,
+                   const char *filter_node_name, Error **errp)
+ {
+     CommitBlockJob *s;
+-    BlockReopenQueue *reopen_queue = NULL;
+     int orig_base_flags;
+     BlockDriverState *iter;
+     BlockDriverState *commit_top_bs = NULL;
+@@ -XXX,XX +XXX,XX @@ void commit_start(const char *job_id, BlockDriverState *bs,
+     /* convert base to r/w, if necessary */
+     orig_base_flags = bdrv_get_flags(base);
+     if (!(orig_base_flags & BDRV_O_RDWR)) {
+-        reopen_queue = bdrv_reopen_queue(reopen_queue, base, NULL,
+-                                         orig_base_flags | BDRV_O_RDWR);
+-    }
+-
+-    if (reopen_queue) {
+-        bdrv_reopen_multiple(bdrv_get_aio_context(bs), reopen_queue, &local_err);
++        bdrv_reopen(base, orig_base_flags | BDRV_O_RDWR, &local_err);
+         if (local_err != NULL) {
+             error_propagate(errp, local_err);
+             goto fail;
+--
+.13.6

-New patch
+[Qemu-devel] [PULL v3 35/35] block: Keep nodes drained between reopen_queue/multiple
+The bdrv_reopen*() implementation doesn't like it if the graph is
+changed between queuing nodes for reopen and actually reopening them
+(one of the reasons is that queuing can be recursive).
+So instead of draining the device only in bdrv_reopen_multiple(),
+require that callers already drained all affected nodes, and assert this
+in bdrv_reopen_queue().
+Signed-off-by: Kevin Wolf <kwolf@redhat.com>
+Reviewed-by: Fam Zheng <famz@redhat.com>
+---
+ block.c             | 23 ++++++++++++++++-------
+ block/replication.c |  6 ++++++
+ qemu-io-cmds.c      |  3 +++
+files changed, 25 insertions(+), 7 deletions(-)
+diff --git a/block.c b/block.c
+index XXXXXXX..XXXXXXX 100644
+--- a/block.c
++++ b/block.c
+@@ -XXX,XX +XXX,XX @@ BlockDriverState *bdrv_open(const char *filename, const char *reference,
+  * returns a pointer to bs_queue, which is either the newly allocated
+  * bs_queue, or the existing bs_queue being used.
+  *
++ * bs must be drained between bdrv_reopen_queue() and bdrv_reopen_multiple().
+  */
+ static BlockReopenQueue *bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
+                                                  BlockDriverState *bs,
+@@ -XXX,XX +XXX,XX @@ static BlockReopenQueue *bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
+     BdrvChild *child;
+     QDict *old_options, *explicit_options;
++    /* Make sure that the caller remembered to use a drained section. This is
++     * important to avoid graph changes between the recursive queuing here and
++     * bdrv_reopen_multiple(). */
++    assert(bs->quiesce_counter > 0);
++
+     if (bs_queue == NULL) {
+         bs_queue = g_new0(BlockReopenQueue, 1);
+         QSIMPLEQ_INIT(bs_queue);
+@@ -XXX,XX +XXX,XX @@ BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
+  * If all devices prepare successfully, then the changes are committed
+  * to all devices.
+  *
++ * All affected nodes must be drained between bdrv_reopen_queue() and
++ * bdrv_reopen_multiple().
+  */
+ int bdrv_reopen_multiple(AioContext *ctx, BlockReopenQueue *bs_queue, Error **errp)
+ {
+@@ -XXX,XX +XXX,XX @@ int bdrv_reopen_multiple(AioContext *ctx, BlockReopenQueue *bs_queue, Error **er
+     assert(bs_queue != NULL);
+-    aio_context_release(ctx);
+-    bdrv_drain_all_begin();
+-    aio_context_acquire(ctx);
+-
+     QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
++        assert(bs_entry->state.bs->quiesce_counter > 0);
+         if (bdrv_reopen_prepare(&bs_entry->state, bs_queue, &local_err)) {
+             error_propagate(errp, local_err);
+             goto cleanup;
+@@ -XXX,XX +XXX,XX @@ cleanup:
+     }
+     g_free(bs_queue);
+-    bdrv_drain_all_end();
+-
+     return ret;
+ }
+@@ -XXX,XX +XXX,XX @@ int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp)
+ {
+     int ret = -1;
+     Error *local_err = NULL;
+-    BlockReopenQueue *queue = bdrv_reopen_queue(NULL, bs, NULL, bdrv_flags);
++    BlockReopenQueue *queue;
++    bdrv_subtree_drained_begin(bs);
++
++    queue = bdrv_reopen_queue(NULL, bs, NULL, bdrv_flags);
+     ret = bdrv_reopen_multiple(bdrv_get_aio_context(bs), queue, &local_err);
+     if (local_err != NULL) {
+         error_propagate(errp, local_err);
+     }
++
++    bdrv_subtree_drained_end(bs);
++
+     return ret;
+ }
+diff --git a/block/replication.c b/block/replication.c
+index XXXXXXX..XXXXXXX 100644
+--- a/block/replication.c
++++ b/block/replication.c
+@@ -XXX,XX +XXX,XX @@ static void reopen_backing_file(BlockDriverState *bs, bool writable,
+         new_secondary_flags = s->orig_secondary_flags;
+     }
++    bdrv_subtree_drained_begin(s->hidden_disk->bs);
++    bdrv_subtree_drained_begin(s->secondary_disk->bs);
++
+     if (orig_hidden_flags != new_hidden_flags) {
+         reopen_queue = bdrv_reopen_queue(reopen_queue, s->hidden_disk->bs, NULL,
+                                          new_hidden_flags);
+@@ -XXX,XX +XXX,XX @@ static void reopen_backing_file(BlockDriverState *bs, bool writable,
+                              reopen_queue, &local_err);
+         error_propagate(errp, local_err);
+     }
++
++    bdrv_subtree_drained_end(s->hidden_disk->bs);
++    bdrv_subtree_drained_end(s->secondary_disk->bs);
+ }
+ static void backup_job_cleanup(BlockDriverState *bs)
+diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c
+index XXXXXXX..XXXXXXX 100644
+--- a/qemu-io-cmds.c
++++ b/qemu-io-cmds.c
+@@ -XXX,XX +XXX,XX @@ static int reopen_f(BlockBackend *blk, int argc, char **argv)
+     opts = qopts ? qemu_opts_to_qdict(qopts, NULL) : NULL;
+     qemu_opts_reset(&reopen_opts);
++    bdrv_subtree_drained_begin(bs);
+     brq = bdrv_reopen_queue(NULL, bs, opts, flags);
+     bdrv_reopen_multiple(bdrv_get_aio_context(bs), brq, &local_err);
++    bdrv_subtree_drained_end(bs);
++
+     if (local_err) {
+         error_report_err(local_err);
+     } else {
+--
+.13.6

The following changes since commit 0274f45bdef73283f2c213610f11d4e5dcba43b6:

Merge remote-tracking branch 'remotes/vivier2/tags/linux-user-for-4.1-pull-request' into staging (2019-07-19 09:44:43 +0100)

are available in the Git repository at:

git://repo.or.cz/qemu/kevin.git tags/for-upstream

for you to fetch changes up to 49278ec065da3fbf90f7effcde3b39ac606b2e9e:

iotests: Test quitting with job on throttled node (2019-07-19 15:17:55 +0200)

----------------------------------------------------------------
Block layer patches:

- block: Fix forbidden use of polling in drained_end
- block: Don't wait for I/O throttling while exiting QEMU
- iotests: Use read-zeroes for the null driver to be Valgrind-friendly

----------------------------------------------------------------
Andrey Shinkevich (1):
      iotests: Set read-zeroes on in null block driver for Valgrind

Max Reitz (12):
      block: Introduce BdrvChild.parent_quiesce_counter
      tests: Add job commit by drained_end test
      block: Add @drained_end_counter
      block: Make bdrv_parent_drained_[^_]*() static
      tests: Lock AioContexts in test-block-iothread
      block: Do not poll in bdrv_do_drained_end()
      tests: Extend commit by drained_end test
      block: Loop unsafely in bdrv*drained_end()
      iotests: Add @has_quit to vm.shutdown()
      iotests: Test commit with a filter on the chain
      vl: Drain before (block) job cancel when quitting
      iotests: Test quitting with job on throttled node

From: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>

The Valgrind tool reports about the uninitialised buffer 'buf'
instantiated on the stack of the function guess_disk_lchs().
Pass 'read-zeroes=on' to the null block driver to make it deterministic.
The output of the tests 051, 186 and 227 now includes the parameter
'read-zeroes'. So, the benchmark output files are being changed too.

Suggested-by: Kevin Wolf <kwolf@redhat.com>
Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/qemu-iotests/051        |  10 +--
 tests/qemu-iotests/051.pc.out |  10 +--
 tests/qemu-iotests/093        |   9 +-
 tests/qemu-iotests/136        |   1 +
 tests/qemu-iotests/186        |  20 ++---
 tests/qemu-iotests/186.out    | 152 +++++++++++++++++-----------------
 tests/qemu-iotests/227        |   4 +-
 tests/qemu-iotests/227.out    |   4 +-
 tests/qemu-iotests/238        |   2 +-
 tests/qemu-iotests/240        |   8 +-
 10 files changed, 111 insertions(+), 109 deletions(-)

diff --git a/tests/qemu-iotests/051 b/tests/qemu-iotests/051
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/051
+++ b/tests/qemu-iotests/051
@@ -XXX,XX +XXX,XX @@ echo
 # Cannot use the test image because cache=none might not work on the host FS
 # Use cdrom so that we won't get errors about missing media
 
-run_qemu -drive driver=null-co,cache=none
-run_qemu -drive driver=null-co,cache=directsync
-run_qemu -drive driver=null-co,cache=writeback
-run_qemu -drive driver=null-co,cache=writethrough
-run_qemu -drive driver=null-co,cache=unsafe
+run_qemu -drive driver=null-co,read-zeroes=on,cache=none
+run_qemu -drive driver=null-co,read-zeroes=on,cache=directsync
+run_qemu -drive driver=null-co,read-zeroes=on,cache=writeback
+run_qemu -drive driver=null-co,read-zeroes=on,cache=writethrough
+run_qemu -drive driver=null-co,read-zeroes=on,cache=unsafe
 run_qemu -drive driver=null-co,cache=invalid_value
 
 # Can't test direct=on here because O_DIRECT might not be supported on this FS
diff --git a/tests/qemu-iotests/051.pc.out b/tests/qemu-iotests/051.pc.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/051.pc.out
+++ b/tests/qemu-iotests/051.pc.out
@@ -XXX,XX +XXX,XX @@ QEMU X.Y.Z monitor - type 'help' for more information
 
 === Cache modes ===
 
-Testing: -drive driver=null-co,cache=none
+Testing: -drive driver=null-co,read-zeroes=on,cache=none
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) quit
 
-Testing: -drive driver=null-co,cache=directsync
+Testing: -drive driver=null-co,read-zeroes=on,cache=directsync
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) quit
 
-Testing: -drive driver=null-co,cache=writeback
+Testing: -drive driver=null-co,read-zeroes=on,cache=writeback
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) quit
 
-Testing: -drive driver=null-co,cache=writethrough
+Testing: -drive driver=null-co,read-zeroes=on,cache=writethrough
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) quit
 
-Testing: -drive driver=null-co,cache=unsafe
+Testing: -drive driver=null-co,read-zeroes=on,cache=unsafe
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) quit
 
diff --git a/tests/qemu-iotests/093 b/tests/qemu-iotests/093
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/093
+++ b/tests/qemu-iotests/093
@@ -XXX,XX +XXX,XX @@ class ThrottleTestCase(iotests.QMPTestCase):
     def setUp(self):
         self.vm = iotests.VM()
         for i in range(0, self.max_drives):
-            self.vm.add_drive(self.test_img)
+            self.vm.add_drive(self.test_img, "file.read-zeroes=on")
         self.vm.launch()
 
     def tearDown(self):
@@ -XXX,XX +XXX,XX @@ class ThrottleTestGroupNames(iotests.QMPTestCase):
     def setUp(self):
         self.vm = iotests.VM()
         for i in range(0, self.max_drives):
-            self.vm.add_drive(self.test_img, "throttling.iops-total=100")
+            self.vm.add_drive(self.test_img,
+                              "throttling.iops-total=100,file.read-zeroes=on")
         self.vm.launch()
 
     def tearDown(self):
@@ -XXX,XX +XXX,XX @@ class ThrottleTestRemovableMedia(iotests.QMPTestCase):
     def test_removable_media(self):
         # Add a couple of dummy nodes named cd0 and cd1
         result = self.vm.qmp("blockdev-add", driver="null-aio",
-                             node_name="cd0")
+                             read_zeroes=True, node_name="cd0")
         self.assert_qmp(result, 'return', {})
         result = self.vm.qmp("blockdev-add", driver="null-aio",
-                             node_name="cd1")
+                             read_zeroes=True, node_name="cd1")
         self.assert_qmp(result, 'return', {})
 
         # Attach a CD drive with cd0 inserted
diff --git a/tests/qemu-iotests/136 b/tests/qemu-iotests/136
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/136
+++ b/tests/qemu-iotests/136
@@ -XXX,XX +XXX,XX @@ sector = "%d"
                           (self.account_invalid and "on" or "off"))
         drive_args.append("stats-account-failed=%s" %
                           (self.account_failed and "on" or "off"))
+        drive_args.append("file.image.read-zeroes=on")
         self.create_blkdebug_file()
         self.vm = iotests.VM().add_drive('blkdebug:%s:%s' %
                                          (blkdebug_file, self.test_img),
diff --git a/tests/qemu-iotests/186 b/tests/qemu-iotests/186
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/186
+++ b/tests/qemu-iotests/186
@@ -XXX,XX +XXX,XX @@ echo "=== -blockdev/-device=<node-name> ==="
 echo
 
 for dev in $fixed $removable; do
-    check_info_block -blockdev driver=null-co,node-name=null -device $dev,drive=null
-    check_info_block -blockdev driver=null-co,node-name=null -device $dev,drive=null,id=qdev_id
+    check_info_block -blockdev driver=null-co,read-zeroes=on,node-name=null -device $dev,drive=null
+    check_info_block -blockdev driver=null-co,read-zeroes=on,node-name=null -device $dev,drive=null,id=qdev_id
 done
 
 echo
@@ -XXX,XX +XXX,XX @@ echo
 # This creates two BlockBackends that will show up in 'info block'!
 # A monitor-owned one from -drive, and anonymous one from -device
 for dev in $fixed $removable; do
-    check_info_block -drive if=none,driver=null-co,node-name=null -device $dev,drive=null,id=qdev_id
+    check_info_block -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device $dev,drive=null,id=qdev_id
 done
 
 echo
@@ -XXX,XX +XXX,XX @@ echo "=== -drive if=none/-device=<bb-name> (with medium) ==="
 echo
 
 for dev in $fixed $removable; do
-    check_info_block -drive if=none,driver=null-co,node-name=null -device $dev,drive=none0
-    check_info_block -drive if=none,driver=null-co,node-name=null -device $dev,drive=none0,id=qdev_id
+    check_info_block -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device $dev,drive=none0
+    check_info_block -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device $dev,drive=none0,id=qdev_id
 done
 
 echo
@@ -XXX,XX +XXX,XX @@ echo "=== -drive if=... ==="
 echo
 
 check_info_block -drive if=floppy
-check_info_block -drive if=floppy,driver=null-co
+check_info_block -drive if=floppy,driver=null-co,read-zeroes=on
 
-check_info_block -drive if=ide,driver=null-co
+check_info_block -drive if=ide,driver=null-co,read-zeroes=on
 check_info_block -drive if=ide,media=cdrom
-check_info_block -drive if=ide,driver=null-co,media=cdrom
+check_info_block -drive if=ide,driver=null-co,read-zeroes=on,media=cdrom
 
-check_info_block -drive if=virtio,driver=null-co
+check_info_block -drive if=virtio,driver=null-co,read-zeroes=on
 
-check_info_block -drive if=pflash,driver=null-co,size=1M
+check_info_block -drive if=pflash,driver=null-co,read-zeroes=on,size=1M
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/186.out b/tests/qemu-iotests/186.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/186.out
+++ b/tests/qemu-iotests/186.out
@@ -XXX,XX +XXX,XX @@ qdev_id: [not inserted]
 
 === -blockdev/-device=<node-name> ===
 
-Testing: -blockdev driver=null-co,node-name=null -device ide-hd,drive=null
+Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device ide-hd,drive=null
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-null: null-co:// (null-co)
+null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
     Attached to:      PATH
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -blockdev driver=null-co,node-name=null -device ide-hd,drive=null,id=qdev_id
+Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device ide-hd,drive=null,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-null: null-co:// (null-co)
+null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
     Attached to:      qdev_id
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -blockdev driver=null-co,node-name=null -device scsi-hd,drive=null
+Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device scsi-hd,drive=null
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-null: null-co:// (null-co)
+null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
     Attached to:      PATH
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -blockdev driver=null-co,node-name=null -device scsi-hd,drive=null,id=qdev_id
+Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device scsi-hd,drive=null,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-null: null-co:// (null-co)
+null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
     Attached to:      qdev_id
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -blockdev driver=null-co,node-name=null -device virtio-blk-pci,drive=null
+Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device virtio-blk-pci,drive=null
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-null: null-co:// (null-co)
+null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
     Attached to:      PATH
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -blockdev driver=null-co,node-name=null -device virtio-blk-pci,drive=null,id=qdev_id
+Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device virtio-blk-pci,drive=null,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-null: null-co:// (null-co)
+null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
     Attached to:      PATH
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -blockdev driver=null-co,node-name=null -device floppy,drive=null
+Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device floppy,drive=null
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-null: null-co:// (null-co)
+null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
     Attached to:      PATH
     Removable device: not locked, tray closed
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -blockdev driver=null-co,node-name=null -device floppy,drive=null,id=qdev_id
+Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device floppy,drive=null,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-null: null-co:// (null-co)
+null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
     Attached to:      qdev_id
     Removable device: not locked, tray closed
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -blockdev driver=null-co,node-name=null -device ide-cd,drive=null
+Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device ide-cd,drive=null
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-null: null-co:// (null-co)
+null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
     Attached to:      PATH
     Removable device: not locked, tray closed
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -blockdev driver=null-co,node-name=null -device ide-cd,drive=null,id=qdev_id
+Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device ide-cd,drive=null,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-null: null-co:// (null-co)
+null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
     Attached to:      qdev_id
     Removable device: not locked, tray closed
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -blockdev driver=null-co,node-name=null -device scsi-cd,drive=null
+Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device scsi-cd,drive=null
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-null: null-co:// (null-co)
+null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
     Attached to:      PATH
     Removable device: not locked, tray closed
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -blockdev driver=null-co,node-name=null -device scsi-cd,drive=null,id=qdev_id
+Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device scsi-cd,drive=null,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-null: null-co:// (null-co)
+null: json:{"read-zeroes": true, "driver": "null-co"} (null-co)
     Attached to:      qdev_id
     Removable device: not locked, tray closed
     Cache mode:       writeback
@@ -XXX,XX +XXX,XX @@ null: null-co:// (null-co)
 
 === -drive if=none/-device=<node-name> ===
 
-Testing: -drive if=none,driver=null-co,node-name=null -device ide-hd,drive=null,id=qdev_id
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device ide-hd,drive=null,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Removable device: not locked, tray closed
     Cache mode:       writeback
 
-null: null-co:// (null-co)
+null: json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      qdev_id
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device scsi-hd,drive=null,id=qdev_id
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device scsi-hd,drive=null,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Removable device: not locked, tray closed
     Cache mode:       writeback
 
-null: null-co:// (null-co)
+null: json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      qdev_id
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device virtio-blk-pci,drive=null,id=qdev_id
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device virtio-blk-pci,drive=null,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Removable device: not locked, tray closed
     Cache mode:       writeback
 
-null: null-co:// (null-co)
+null: json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      PATH
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device floppy,drive=null,id=qdev_id
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device floppy,drive=null,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Removable device: not locked, tray closed
     Cache mode:       writeback
 
-null: null-co:// (null-co)
+null: json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      qdev_id
     Removable device: not locked, tray closed
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device ide-cd,drive=null,id=qdev_id
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device ide-cd,drive=null,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Removable device: not locked, tray closed
     Cache mode:       writeback
 
-null: null-co:// (null-co)
+null: json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      qdev_id
     Removable device: not locked, tray closed
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device scsi-cd,drive=null,id=qdev_id
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device scsi-cd,drive=null,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Removable device: not locked, tray closed
     Cache mode:       writeback
 
-null: null-co:// (null-co)
+null: json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      qdev_id
     Removable device: not locked, tray closed
     Cache mode:       writeback
@@ -XXX,XX +XXX,XX @@ null: null-co:// (null-co)
 
 === -drive if=none/-device=<bb-name> (with medium) ===
 
-Testing: -drive if=none,driver=null-co,node-name=null -device ide-hd,drive=none0
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device ide-hd,drive=none0
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      PATH
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device ide-hd,drive=none0,id=qdev_id
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device ide-hd,drive=none0,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      qdev_id
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device scsi-hd,drive=none0
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device scsi-hd,drive=none0
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      PATH
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device scsi-hd,drive=none0,id=qdev_id
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device scsi-hd,drive=none0,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      qdev_id
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device virtio-blk-pci,drive=none0
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device virtio-blk-pci,drive=none0
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      PATH
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device virtio-blk-pci,drive=none0,id=qdev_id
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device virtio-blk-pci,drive=none0,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      PATH
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device floppy,drive=none0
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device floppy,drive=none0
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      PATH
     Removable device: not locked, tray closed
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device floppy,drive=none0,id=qdev_id
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device floppy,drive=none0,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      qdev_id
     Removable device: not locked, tray closed
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device ide-cd,drive=none0
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device ide-cd,drive=none0
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      PATH
     Removable device: not locked, tray closed
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device ide-cd,drive=none0,id=qdev_id
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device ide-cd,drive=none0,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      qdev_id
     Removable device: not locked, tray closed
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device scsi-cd,drive=none0
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device scsi-cd,drive=none0
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      PATH
     Removable device: not locked, tray closed
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=none,driver=null-co,node-name=null -device scsi-cd,drive=none0,id=qdev_id
+Testing: -drive if=none,driver=null-co,read-zeroes=on,node-name=null -device scsi-cd,drive=none0,id=qdev_id
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-none0 (null): null-co:// (null-co)
+none0 (null): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      qdev_id
     Removable device: not locked, tray closed
     Cache mode:       writeback
@@ -XXX,XX +XXX,XX @@ floppy0: [not inserted]
     Removable device: not locked, tray closed
 (qemu) quit
 
-Testing: -drive if=floppy,driver=null-co
+Testing: -drive if=floppy,driver=null-co,read-zeroes=on
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-floppy0 (NODE_NAME): null-co:// (null-co)
+floppy0 (NODE_NAME): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      PATH
     Removable device: not locked, tray closed
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=ide,driver=null-co
+Testing: -drive if=ide,driver=null-co,read-zeroes=on
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-ide0-hd0 (NODE_NAME): null-co:// (null-co)
+ide0-hd0 (NODE_NAME): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      PATH
     Cache mode:       writeback
 (qemu) quit
@@ -XXX,XX +XXX,XX @@ ide0-cd0: [not inserted]
     Removable device: not locked, tray closed
 (qemu) quit
 
-Testing: -drive if=ide,driver=null-co,media=cdrom
+Testing: -drive if=ide,driver=null-co,read-zeroes=on,media=cdrom
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-ide0-cd0 (NODE_NAME): null-co:// (null-co, read-only)
+ide0-cd0 (NODE_NAME): json:{"read-zeroes": "on", "driver": "null-co"} (null-co, read-only)
     Attached to:      PATH
     Removable device: not locked, tray closed
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=virtio,driver=null-co
+Testing: -drive if=virtio,driver=null-co,read-zeroes=on
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-virtio0 (NODE_NAME): null-co:// (null-co)
+virtio0 (NODE_NAME): json:{"read-zeroes": "on", "driver": "null-co"} (null-co)
     Attached to:      PATH
     Cache mode:       writeback
 (qemu) quit
 
-Testing: -drive if=pflash,driver=null-co,size=1M
+Testing: -drive if=pflash,driver=null-co,read-zeroes=on,size=1M
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) info block
-pflash0 (NODE_NAME): json:{"driver": "null-co", "size": "1M"} (null-co)
+pflash0 (NODE_NAME): json:{"read-zeroes": "on", "driver": "null-co", "size": "1M"} (null-co)
     Attached to:      PATH
     Cache mode:       writeback
 (qemu) quit
diff --git a/tests/qemu-iotests/227 b/tests/qemu-iotests/227
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/227
+++ b/tests/qemu-iotests/227
@@ -XXX,XX +XXX,XX @@ echo
 echo '=== blockstats with -drive if=virtio ==='
 echo
 
-run_qemu -drive driver=null-co,if=virtio <<EOF
+run_qemu -drive driver=null-co,read-zeroes=on,if=virtio <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "query-blockstats"}
 { "execute": "quit" }
@@ -XXX,XX +XXX,XX @@ echo
 echo '=== blockstats with -blockdev and -device ==='
 echo
 
-run_qemu -blockdev driver=null-co,node-name=null -device virtio-blk,drive=null,id=virtio0 <<EOF
+run_qemu -blockdev driver=null-co,read-zeroes=on,node-name=null -device virtio-blk,drive=null,id=virtio0 <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "query-blockstats"}
 { "execute": "quit" }
diff --git a/tests/qemu-iotests/227.out b/tests/qemu-iotests/227.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/227.out
+++ b/tests/qemu-iotests/227.out
@@ -XXX,XX +XXX,XX @@ QA output created by 227
 
 === blockstats with -drive if=virtio ===
 
-Testing: -drive driver=null-co,if=virtio
+Testing: -drive driver=null-co,read-zeroes=on,if=virtio
 {
     QMP_VERSION
 }
@@ -XXX,XX +XXX,XX @@ Testing: -blockdev driver=null-co,node-name=null
 
 === blockstats with -blockdev and -device ===
 
-Testing: -blockdev driver=null-co,node-name=null -device virtio-blk,drive=null,id=virtio0
+Testing: -blockdev driver=null-co,read-zeroes=on,node-name=null -device virtio-blk,drive=null,id=virtio0
 {
     QMP_VERSION
 }
diff --git a/tests/qemu-iotests/238 b/tests/qemu-iotests/238
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/238
+++ b/tests/qemu-iotests/238
@@ -XXX,XX +XXX,XX @@ else:
 vm = iotests.VM()
 vm.launch()
 
-log(vm.qmp('blockdev-add', node_name='hd0', driver='null-co'))
+log(vm.qmp('blockdev-add', node_name='hd0', driver='null-co', read_zeroes=True))
 log(vm.qmp('object-add', qom_type='iothread', id='iothread0'))
 log(vm.qmp('device_add', id='scsi0', driver=virtio_scsi_device, iothread='iothread0'))
 log(vm.qmp('device_add', id='scsi-hd0', driver='scsi-hd', drive='hd0'))
diff --git a/tests/qemu-iotests/240 b/tests/qemu-iotests/240
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/240
+++ b/tests/qemu-iotests/240
@@ -XXX,XX +XXX,XX @@ echo
 
 run_qemu <<EOF
 { "execute": "qmp_capabilities" }
-{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0"}}
+{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "read-zeroes": true, "node-name": "hd0"}}
 { "execute": "object-add", "arguments": {"qom-type": "iothread", "id": "iothread0"}}
 { "execute": "device_add", "arguments": {"id": "scsi0", "driver": "${virtio_scsi}", "iothread": "iothread0"}}
 { "execute": "device_add", "arguments": {"id": "scsi-hd0", "driver": "scsi-hd", "drive": "hd0"}}
@@ -XXX,XX +XXX,XX @@ echo
 
 run_qemu <<EOF
 { "execute": "qmp_capabilities" }
-{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0", "read-only": true}}
+{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "read-zeroes": true, "node-name": "hd0", "read-only": true}}
 { "execute": "object-add", "arguments": {"qom-type": "iothread", "id": "iothread0"}}
 { "execute": "device_add", "arguments": {"id": "scsi0", "driver": "${virtio_scsi}", "iothread": "iothread0"}}
 { "execute": "device_add", "arguments": {"id": "scsi-hd0", "driver": "scsi-hd", "drive": "hd0"}}
@@ -XXX,XX +XXX,XX @@ echo
 
 run_qemu <<EOF
 { "execute": "qmp_capabilities" }
-{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0", "read-only": true}}
+{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "read-zeroes": true, "node-name": "hd0", "read-only": true}}
 { "execute": "object-add", "arguments": {"qom-type": "iothread", "id": "iothread0"}}
 { "execute": "object-add", "arguments": {"qom-type": "iothread", "id": "iothread1"}}
 { "execute": "device_add", "arguments": {"id": "scsi0", "driver": "${virtio_scsi}", "iothread": "iothread0"}}
@@ -XXX,XX +XXX,XX @@ echo
 
 run_qemu <<EOF
 { "execute": "qmp_capabilities" }
-{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0", "read-only": true}}
+{ "execute": "blockdev-add", "arguments": {"driver": "null-co", "read-zeroes": true, "node-name": "hd0", "read-only": true}}
 { "execute": "nbd-server-start", "arguments": {"addr":{"type":"unix","data":{"path":"$TEST_DIR/nbd"}}}}
 { "execute": "nbd-server-add", "arguments": {"device":"hd0"}}
 { "execute": "object-add", "arguments": {"qom-type": "iothread", "id": "iothread0"}}
-- 
2.20.1

From: Max Reitz <mreitz@redhat.com>

Commit 5cb2737e925042e6c7cd3fb0b01313950b03cddf laid out why
bdrv_do_drained_end() must decrement the quiesce_counter after
bdrv_drain_invoke().  It did not give a very good reason why it has to
happen after bdrv_parent_drained_end(), instead only claiming symmetry
to bdrv_do_drained_begin().

It turns out that delaying it for so long is wrong.

Situation: We have an active commit job (i.e. a mirror job) from top to
base for the following graph:

filter
                    |
                  [file]
                    |
                    v
top --[backing]--> base

Now the VM is closed, which results in the job being cancelled and a
bdrv_drain_all() happening pretty much simultaneously.

Beginning the drain means the job is paused once whenever one of its
nodes is quiesced.  This is reversed when the drain ends.

With how the code currently is, after base's drain ends (which means
that it will have unpaused the job once), its quiesce_counter remains at
1 while it goes to undrain its parents (bdrv_parent_drained_end()).  For
some reason or another, undraining filter causes the job to be kicked
and enter mirror_exit_common(), where it proceeds to invoke
block_job_remove_all_bdrv().

Now base will be detached from the job.  Because its quiesce_counter is
still 1, it will unpause the job once more.  So in total, undraining
base will unpause the job twice.  Eventually, this will lead to the
job's pause_count going negative -- well, it would, were there not an
assertion against this, which crashes qemu.

The general problem is that if in bdrv_parent_drained_end() we undrain
parent A, and then undrain parent B, which then leads to A detaching the
child, bdrv_replace_child_noperm() will undrain A as if we had not done
so yet; that is, one time too many.

It follows that we cannot decrement the quiesce_counter after invoking
bdrv_parent_drained_end().

Unfortunately, decrementing it before bdrv_parent_drained_end() would be
wrong, too.  Imagine the above situation in reverse: Undraining A leads
to B detaching the child.  If we had already decremented the
quiesce_counter by that point, bdrv_replace_child_noperm() would undrain
B one time too little; because it expects bdrv_parent_drained_end() to
issue this undrain.  But bdrv_parent_drained_end() won't do that,
because B is no longer a parent.

Therefore, we have to do something else.  This patch opts for
introducing a second quiesce_counter that counts how many times a
child's parent has been quiesced (though c->role->drained_*).  With
that, bdrv_replace_child_noperm() just has to undrain the parent exactly
that many times when removing a child, and it will always be right.

Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 include/block/block.h     |  7 +++++++
 include/block/block_int.h |  9 +++++++++
 block.c                   | 15 +++++----------
 block/io.c                | 14 +++++++++++---
 4 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/include/block/block.h b/include/block/block.h
index XXXXXXX..XXXXXXX 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
  */
 void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll);
 
+/**
+ * bdrv_parent_drained_end_single:
+ *
+ * End a quiesced section for the parent of @c.
+ */
+void bdrv_parent_drained_end_single(BdrvChild *c);
+
 /**
  * bdrv_parent_drained_end:
  *
diff --git a/include/block/block_int.h b/include/block/block_int.h
index XXXXXXX..XXXXXXX 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -XXX,XX +XXX,XX @@ struct BdrvChild {
      */
     bool frozen;
 
+    /*
+     * How many times the parent of this child has been drained
+     * (through role->drained_*).
+     * Usually, this is equal to bs->quiesce_counter (potentially
+     * reduced by bdrv_drain_all_count).  It may differ while the
+     * child is entering or leaving a drained section.
+     */
+    int parent_quiesce_counter;
+
     QLIST_ENTRY(BdrvChild) next;
     QLIST_ENTRY(BdrvChild) next_parent;
 };
diff --git a/block.c b/block.c
index XXXXXXX..XXXXXXX 100644
--- a/block.c
+++ b/block.c
@@ -XXX,XX +XXX,XX @@ static void bdrv_replace_child_noperm(BdrvChild *child,
         if (child->role->detach) {
             child->role->detach(child);
         }
-        if (old_bs->quiesce_counter && child->role->drained_end) {
-            int num = old_bs->quiesce_counter;
-            if (child->role->parent_is_bds) {
-                num -= bdrv_drain_all_count;
-            }
-            assert(num >= 0);
-            for (i = 0; i < num; i++) {
-                child->role->drained_end(child);
-            }
+        while (child->parent_quiesce_counter) {
+            bdrv_parent_drained_end_single(child);
         }
         QLIST_REMOVE(child, next_parent);
+    } else {
+        assert(child->parent_quiesce_counter == 0);
     }
 
     child->bs = new_bs;
 
     if (new_bs) {
         QLIST_INSERT_HEAD(&new_bs->parents, child, next_parent);
-        if (new_bs->quiesce_counter && child->role->drained_begin) {
+        if (new_bs->quiesce_counter) {
             int num = new_bs->quiesce_counter;
             if (child->role->parent_is_bds) {
                 num -= bdrv_drain_all_count;
diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
     }
 }
 
+void bdrv_parent_drained_end_single(BdrvChild *c)
+{
+    assert(c->parent_quiesce_counter > 0);
+    c->parent_quiesce_counter--;
+    if (c->role->drained_end) {
+        c->role->drained_end(c);
+    }
+}
+
 void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
                              bool ignore_bds_parents)
 {
@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
         if (c == ignore || (ignore_bds_parents && c->role->parent_is_bds)) {
             continue;
         }
-        if (c->role->drained_end) {
-            c->role->drained_end(c);
-        }
+        bdrv_parent_drained_end_single(c);
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore,
 
 void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll)
 {
+    c->parent_quiesce_counter++;
     if (c->role->drained_begin) {
         c->role->drained_begin(c);
     }
-- 
2.20.1

From: Max Reitz <mreitz@redhat.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/test-bdrv-drain.c | 119 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 119 insertions(+)

diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
index XXXXXXX..XXXXXXX 100644
--- a/tests/test-bdrv-drain.c
+++ b/tests/test-bdrv-drain.c
@@ -XXX,XX +XXX,XX @@ static void test_set_aio_context(void)
     iothread_join(b);
 }
 
+
+typedef struct TestDropBackingBlockJob {
+    BlockJob common;
+    bool should_complete;
+    bool *did_complete;
+} TestDropBackingBlockJob;
+
+static int coroutine_fn test_drop_backing_job_run(Job *job, Error **errp)
+{
+    TestDropBackingBlockJob *s =
+        container_of(job, TestDropBackingBlockJob, common.job);
+
+    while (!s->should_complete) {
+        job_sleep_ns(job, 0);
+    }
+
+    return 0;
+}
+
+static void test_drop_backing_job_commit(Job *job)
+{
+    TestDropBackingBlockJob *s =
+        container_of(job, TestDropBackingBlockJob, common.job);
+
+    bdrv_set_backing_hd(blk_bs(s->common.blk), NULL, &error_abort);
+
+    *s->did_complete = true;
+}
+
+static const BlockJobDriver test_drop_backing_job_driver = {
+    .job_driver = {
+        .instance_size  = sizeof(TestDropBackingBlockJob),
+        .free           = block_job_free,
+        .user_resume    = block_job_user_resume,
+        .drain          = block_job_drain,
+        .run            = test_drop_backing_job_run,
+        .commit         = test_drop_backing_job_commit,
+    }
+};
+
+/**
+ * Creates a child node with three parent nodes on it, and then runs a
+ * block job on the final one, parent-node-2.
+ *
+ * (TODO: parent-node-0 currently serves no purpose, but will as of a
+ * follow-up patch.)
+ *
+ * The job is then asked to complete before a section where the child
+ * is drained.
+ *
+ * Ending this section will undrain the child's parents, first
+ * parent-node-2, then parent-node-1, then parent-node-0 -- the parent
+ * list is in reverse order of how they were added.  Ending the drain
+ * on parent-node-2 will resume the job, thus completing it and
+ * scheduling job_exit().
+ *
+ * Ending the drain on parent-node-1 will poll the AioContext, which
+ * lets job_exit() and thus test_drop_backing_job_commit() run.  That
+ * function removes the child as parent-node-2's backing file.
+ *
+ * In old (and buggy) implementations, there are two problems with
+ * that:
+ * (A) bdrv_drain_invoke() polls for every node that leaves the
+ *     drained section.  This means that job_exit() is scheduled
+ *     before the child has left the drained section.  Its
+ *     quiesce_counter is therefore still 1 when it is removed from
+ *     parent-node-2.
+ *
+ * (B) bdrv_replace_child_noperm() calls drained_end() on the old
+ *     child's parents as many times as the child is quiesced.  This
+ *     means it will call drained_end() on parent-node-2 once.
+ *     Because parent-node-2 is no longer quiesced at this point, this
+ *     will fail.
+ *
+ * bdrv_replace_child_noperm() therefore must call drained_end() on
+ * the parent only if it really is still drained because the child is
+ * drained.
+ */
+static void test_blockjob_commit_by_drained_end(void)
+{
+    BlockDriverState *bs_child, *bs_parents[3];
+    TestDropBackingBlockJob *job;
+    bool job_has_completed = false;
+    int i;
+
+    bs_child = bdrv_new_open_driver(&bdrv_test, "child-node", BDRV_O_RDWR,
+                                    &error_abort);
+
+    for (i = 0; i < 3; i++) {
+        char name[32];
+        snprintf(name, sizeof(name), "parent-node-%i", i);
+        bs_parents[i] = bdrv_new_open_driver(&bdrv_test, name, BDRV_O_RDWR,
+                                             &error_abort);
+        bdrv_set_backing_hd(bs_parents[i], bs_child, &error_abort);
+    }
+
+    job = block_job_create("job", &test_drop_backing_job_driver, NULL,
+                           bs_parents[2], 0, BLK_PERM_ALL, 0, 0, NULL, NULL,
+                           &error_abort);
+
+    job->did_complete = &job_has_completed;
+
+    job_start(&job->common.job);
+
+    job->should_complete = true;
+    bdrv_drained_begin(bs_child);
+    g_assert(!job_has_completed);
+    bdrv_drained_end(bs_child);
+    g_assert(job_has_completed);
+
+    bdrv_unref(bs_parents[0]);
+    bdrv_unref(bs_parents[1]);
+    bdrv_unref(bs_parents[2]);
+    bdrv_unref(bs_child);
+}
+
 int main(int argc, char **argv)
 {
     int ret;
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
 
     g_test_add_func("/bdrv-drain/set_aio_context", test_set_aio_context);
 
+    g_test_add_func("/bdrv-drain/blockjob/commit_by_drained_end",
+                    test_blockjob_commit_by_drained_end);
+
     ret = g_test_run();
     qemu_event_destroy(&done_event);
     return ret;
-- 
2.20.1

From: Max Reitz <mreitz@redhat.com>

Callers can now pass a pointer to an integer that bdrv_drain_invoke()
(and its recursive callees) will increment for every
bdrv_drain_invoke_entry() operation they schedule.
bdrv_drain_invoke_entry() in turn will decrement it once it has invoked
BlockDriver.bdrv_co_drain_end().

We use atomic operations to access the pointee, because the
bdrv_do_drained_end() caller may wish to end drained sections for
multiple nodes in different AioContexts (bdrv_drain_all_end() does, for
example).

This is the first step to moving the polling for BdrvCoDrainData.done to
become true out of bdrv_drain_invoke() and into the root drained_end
function.

Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 block/io.c | 58 +++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 40 insertions(+), 18 deletions(-)

diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ typedef struct {
     bool poll;
     BdrvChild *parent;
     bool ignore_bds_parents;
+    int *drained_end_counter;
 } BdrvCoDrainData;
 
 static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
     atomic_mb_set(&data->done, true);
     bdrv_dec_in_flight(bs);
 
-    if (data->begin) {
+    if (data->drained_end_counter) {
+        atomic_dec(data->drained_end_counter);
+    }
+
+    if (data->begin || data->drained_end_counter) {
         g_free(data);
     }
 }
 
 /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */
-static void bdrv_drain_invoke(BlockDriverState *bs, bool begin)
+static void bdrv_drain_invoke(BlockDriverState *bs, bool begin,
+                              int *drained_end_counter)
 {
     BdrvCoDrainData *data;
 
@@ -XXX,XX +XXX,XX @@ static void bdrv_drain_invoke(BlockDriverState *bs, bool begin)
     *data = (BdrvCoDrainData) {
         .bs = bs,
         .done = false,
-        .begin = begin
+        .begin = begin,
+        .drained_end_counter = drained_end_counter,
     };
 
+    if (!begin && drained_end_counter) {
+        atomic_inc(drained_end_counter);
+    }
+
     /* Make sure the driver callback completes during the polling phase for
      * drain_begin. */
     bdrv_inc_in_flight(bs);
     data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data);
     aio_co_schedule(bdrv_get_aio_context(bs), data->co);
 
-    if (!begin) {
+    /*
+     * TODO: Drop this and make callers pass @drained_end_counter and poll
+     * themselves
+     */
+    if (!begin && !drained_end_counter) {
         BDRV_POLL_WHILE(bs, !data->done);
         g_free(data);
     }
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
                                   BdrvChild *parent, bool ignore_bds_parents,
                                   bool poll);
 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
-                                BdrvChild *parent, bool ignore_bds_parents);
+                                BdrvChild *parent, bool ignore_bds_parents,
+                                int *drained_end_counter);
 
 static void bdrv_co_drain_bh_cb(void *opaque)
 {
@@ -XXX,XX +XXX,XX @@ static void bdrv_co_drain_bh_cb(void *opaque)
                                   data->ignore_bds_parents, data->poll);
         } else {
             bdrv_do_drained_end(bs, data->recursive, data->parent,
-                                data->ignore_bds_parents);
+                                data->ignore_bds_parents,
+                                data->drained_end_counter);
         }
         if (ctx == co_ctx) {
             aio_context_release(ctx);
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
                                                 bool begin, bool recursive,
                                                 BdrvChild *parent,
                                                 bool ignore_bds_parents,
-                                                bool poll)
+                                                bool poll,
+                                                int *drained_end_counter)
 {
     BdrvCoDrainData data;
 
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
         .parent = parent,
         .ignore_bds_parents = ignore_bds_parents,
         .poll = poll,
+        .drained_end_counter = drained_end_counter,
     };
+
     if (bs) {
         bdrv_inc_in_flight(bs);
     }
@@ -XXX,XX +XXX,XX @@ void bdrv_do_drained_begin_quiesce(BlockDriverState *bs,
     }
 
     bdrv_parent_drained_begin(bs, parent, ignore_bds_parents);
-    bdrv_drain_invoke(bs, true);
+    bdrv_drain_invoke(bs, true, NULL);
 }
 
 static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
 
     if (qemu_in_coroutine()) {
         bdrv_co_yield_to_drain(bs, true, recursive, parent, ignore_bds_parents,
-                               poll);
+                               poll, NULL);
         return;
     }
 
@@ -XXX,XX +XXX,XX @@ void bdrv_subtree_drained_begin(BlockDriverState *bs)
 }
 
 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
-                                BdrvChild *parent, bool ignore_bds_parents)
+                                BdrvChild *parent, bool ignore_bds_parents,
+                                int *drained_end_counter)
 {
     BdrvChild *child, *next;
     int old_quiesce_counter;
 
     if (qemu_in_coroutine()) {
         bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents,
-                               false);
+                               false, drained_end_counter);
         return;
     }
     assert(bs->quiesce_counter > 0);
 
     /* Re-enable things in child-to-parent order */
-    bdrv_drain_invoke(bs, false);
+    bdrv_drain_invoke(bs, false, drained_end_counter);
     bdrv_parent_drained_end(bs, parent, ignore_bds_parents);
 
     old_quiesce_counter = atomic_fetch_dec(&bs->quiesce_counter);
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
         assert(!ignore_bds_parents);
         bs->recursive_quiesce_counter--;
         QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
-            bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents);
+            bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents,
+                                drained_end_counter);
         }
     }
 }
 
 void bdrv_drained_end(BlockDriverState *bs)
 {
-    bdrv_do_drained_end(bs, false, NULL, false);
+    bdrv_do_drained_end(bs, false, NULL, false, NULL);
 }
 
 void bdrv_subtree_drained_end(BlockDriverState *bs)
 {
-    bdrv_do_drained_end(bs, true, NULL, false);
+    bdrv_do_drained_end(bs, true, NULL, false, NULL);
 }
 
 void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
@@ -XXX,XX +XXX,XX @@ void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent)
     int i;
 
     for (i = 0; i < old_parent->recursive_quiesce_counter; i++) {
-        bdrv_do_drained_end(child->bs, true, child, false);
+        bdrv_do_drained_end(child->bs, true, child, false, NULL);
     }
 }
 
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
     BlockDriverState *bs = NULL;
 
     if (qemu_in_coroutine()) {
-        bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true);
+        bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true, NULL);
         return;
     }
 
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_end(void)
         AioContext *aio_context = bdrv_get_aio_context(bs);
 
         aio_context_acquire(aio_context);
-        bdrv_do_drained_end(bs, false, NULL, true);
+        bdrv_do_drained_end(bs, false, NULL, true, NULL);
         aio_context_release(aio_context);
     }
 
-- 
2.20.1

From: Max Reitz <mreitz@redhat.com>

These functions are not used outside of block/io.c, there is no reason
why they should be globally available.

Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 include/block/block.h | 18 ------------------
 block/io.c            |  8 ++++----
 2 files changed, 4 insertions(+), 22 deletions(-)

diff --git a/include/block/block.h b/include/block/block.h
index XXXXXXX..XXXXXXX 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -XXX,XX +XXX,XX @@ int bdrv_probe_geometry(BlockDriverState *bs, HDGeometry *geo);
 void bdrv_io_plug(BlockDriverState *bs);
 void bdrv_io_unplug(BlockDriverState *bs);
 
-/**
- * bdrv_parent_drained_begin:
- *
- * Begin a quiesced section of all users of @bs. This is part of
- * bdrv_drained_begin.
- */
-void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
-                               bool ignore_bds_parents);
-
 /**
  * bdrv_parent_drained_begin_single:
  *
@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll);
  */
 void bdrv_parent_drained_end_single(BdrvChild *c);
 
-/**
- * bdrv_parent_drained_end:
- *
- * End a quiesced section of all users of @bs. This is part of
- * bdrv_drained_end.
- */
-void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
-                             bool ignore_bds_parents);
-
 /**
  * bdrv_drain_poll:
  *
diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ static void bdrv_parent_cb_resize(BlockDriverState *bs);
 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
     int64_t offset, int bytes, BdrvRequestFlags flags);
 
-void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
-                               bool ignore_bds_parents)
+static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
+                                      bool ignore_bds_parents)
 {
     BdrvChild *c, *next;
 
@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_end_single(BdrvChild *c)
     }
 }
 
-void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
-                             bool ignore_bds_parents)
+static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
+                                    bool ignore_bds_parents)
 {
     BdrvChild *c, *next;
 
-- 
2.20.1

From: Max Reitz <mreitz@redhat.com>

When changing a node's AioContext, the caller must acquire the old
AioContext (unless it currently runs in that old context).  Therefore,
unless the node currently is in the main context, we always have to
acquire the old context around calls that may change a node's
AioContext.

Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/test-block-iothread.c | 40 ++++++++++++++++++++++++-------------
 1 file changed, 26 insertions(+), 14 deletions(-)

diff --git a/tests/test-block-iothread.c b/tests/test-block-iothread.c
index XXXXXXX..XXXXXXX 100644
--- a/tests/test-block-iothread.c
+++ b/tests/test-block-iothread.c
@@ -XXX,XX +XXX,XX @@ static void test_sync_op(const void *opaque)
     if (t->blkfn) {
         t->blkfn(blk);
     }
-    aio_context_release(ctx);
     blk_set_aio_context(blk, qemu_get_aio_context(), &error_abort);
+    aio_context_release(ctx);
 
     bdrv_unref(bs);
     blk_unref(blk);
@@ -XXX,XX +XXX,XX @@ static void test_propagate_basic(void)
 {
     IOThread *iothread = iothread_new();
     AioContext *ctx = iothread_get_aio_context(iothread);
+    AioContext *main_ctx;
     BlockBackend *blk;
     BlockDriverState *bs_a, *bs_b, *bs_verify;
     QDict *options;
@@ -XXX,XX +XXX,XX @@ static void test_propagate_basic(void)
     g_assert(bdrv_get_aio_context(bs_b) == ctx);
 
     /* Switch the AioContext back */
-    ctx = qemu_get_aio_context();
-    blk_set_aio_context(blk, ctx, &error_abort);
-    g_assert(blk_get_aio_context(blk) == ctx);
-    g_assert(bdrv_get_aio_context(bs_a) == ctx);
-    g_assert(bdrv_get_aio_context(bs_verify) == ctx);
-    g_assert(bdrv_get_aio_context(bs_b) == ctx);
+    main_ctx = qemu_get_aio_context();
+    aio_context_acquire(ctx);
+    blk_set_aio_context(blk, main_ctx, &error_abort);
+    aio_context_release(ctx);
+    g_assert(blk_get_aio_context(blk) == main_ctx);
+    g_assert(bdrv_get_aio_context(bs_a) == main_ctx);
+    g_assert(bdrv_get_aio_context(bs_verify) == main_ctx);
+    g_assert(bdrv_get_aio_context(bs_b) == main_ctx);
 
     bdrv_unref(bs_verify);
     bdrv_unref(bs_b);
@@ -XXX,XX +XXX,XX @@ static void test_propagate_diamond(void)
 {
     IOThread *iothread = iothread_new();
     AioContext *ctx = iothread_get_aio_context(iothread);
+    AioContext *main_ctx;
     BlockBackend *blk;
     BlockDriverState *bs_a, *bs_b, *bs_c, *bs_verify;
     QDict *options;
@@ -XXX,XX +XXX,XX @@ static void test_propagate_diamond(void)
     g_assert(bdrv_get_aio_context(bs_c) == ctx);
 
     /* Switch the AioContext back */
-    ctx = qemu_get_aio_context();
-    blk_set_aio_context(blk, ctx, &error_abort);
-    g_assert(blk_get_aio_context(blk) == ctx);
-    g_assert(bdrv_get_aio_context(bs_verify) == ctx);
-    g_assert(bdrv_get_aio_context(bs_a) == ctx);
-    g_assert(bdrv_get_aio_context(bs_b) == ctx);
-    g_assert(bdrv_get_aio_context(bs_c) == ctx);
+    main_ctx = qemu_get_aio_context();
+    aio_context_acquire(ctx);
+    blk_set_aio_context(blk, main_ctx, &error_abort);
+    aio_context_release(ctx);
+    g_assert(blk_get_aio_context(blk) == main_ctx);
+    g_assert(bdrv_get_aio_context(bs_verify) == main_ctx);
+    g_assert(bdrv_get_aio_context(bs_a) == main_ctx);
+    g_assert(bdrv_get_aio_context(bs_b) == main_ctx);
+    g_assert(bdrv_get_aio_context(bs_c) == main_ctx);
 
     blk_unref(blk);
     bdrv_unref(bs_verify);
@@ -XXX,XX +XXX,XX @@ static void test_attach_second_node(void)
     g_assert(bdrv_get_aio_context(bs) == ctx);
     g_assert(bdrv_get_aio_context(filter) == ctx);
 
+    aio_context_acquire(ctx);
     blk_set_aio_context(blk, main_ctx, &error_abort);
+    aio_context_release(ctx);
     g_assert(blk_get_aio_context(blk) == main_ctx);
     g_assert(bdrv_get_aio_context(bs) == main_ctx);
     g_assert(bdrv_get_aio_context(filter) == main_ctx);
@@ -XXX,XX +XXX,XX @@ static void test_attach_preserve_blk_ctx(void)
     g_assert(bdrv_get_aio_context(bs) == ctx);
 
     /* Remove the node again */
+    aio_context_acquire(ctx);
     blk_remove_bs(blk);
+    aio_context_release(ctx);
     g_assert(blk_get_aio_context(blk) == ctx);
     g_assert(bdrv_get_aio_context(bs) == qemu_get_aio_context());
 
@@ -XXX,XX +XXX,XX @@ static void test_attach_preserve_blk_ctx(void)
     g_assert(blk_get_aio_context(blk) == ctx);
     g_assert(bdrv_get_aio_context(bs) == ctx);
 
+    aio_context_acquire(ctx);
     blk_set_aio_context(blk, qemu_get_aio_context(), &error_abort);
+    aio_context_release(ctx);
     bdrv_unref(bs);
     blk_unref(blk);
 }
-- 
2.20.1

From: Max Reitz <mreitz@redhat.com>

We should never poll anywhere in bdrv_do_drained_end() (including its
recursive callees like bdrv_drain_invoke()), because it does not cope
well with graph changes.  In fact, it has been written based on the
postulation that no graph changes will happen in it.

Instead, the callers that want to poll must poll, i.e. all currently
globally available wrappers: bdrv_drained_end(),
bdrv_subtree_drained_end(), bdrv_unapply_subtree_drain(), and
bdrv_drain_all_end().  Graph changes there do not matter.

They can poll simply by passing a pointer to a drained_end_counter and
wait until it reaches 0.

This patch also adds a non-polling global wrapper for
bdrv_do_drained_end() that takes a drained_end_counter pointer.  We need
such a variant because now no function called anywhere from
bdrv_do_drained_end() must poll.  This includes
BdrvChildRole.drained_end(), which already must not poll according to
its interface documentation, but bdrv_child_cb_drained_end() just
violates that by invoking bdrv_drained_end() (which does poll).
Therefore, BdrvChildRole.drained_end() must take a *drained_end_counter
parameter, which bdrv_child_cb_drained_end() can pass on to the new
bdrv_drained_end_no_poll() function.

Note that we now have a pattern of all drained_end-related functions
either polling or receiving a *drained_end_counter to let the caller
poll based on that.

A problem with a single poll loop is that when the drained section in
bdrv_set_aio_context_ignore() ends, some nodes in the subgraph may be in
the old contexts, while others are in the new context already.  To let
the collective poll in bdrv_drained_end() work correctly, we must not
hold a lock to the old context, so that the old context can make
progress in case it is different from the current context.

(In the process, remove the comment saying that the current context is
always the old context, because it is wrong.)

In all other places, all nodes in a subtree must be in the same context,
so we can just poll that.  The exception of course is
bdrv_drain_all_end(), but that always runs in the main context, so we
can just poll NULL (like bdrv_drain_all_begin() does).

Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 include/block/block.h     | 25 ++++++++++++
 include/block/block_int.h |  6 ++-
 block.c                   | 37 ++++++++++++++----
 block/block-backend.c     |  6 +--
 block/io.c                | 80 ++++++++++++++++++++++++++++-----------
 blockjob.c                |  2 +-
 6 files changed, 120 insertions(+), 36 deletions(-)

diff --git a/include/block/block.h b/include/block/block.h
index XXXXXXX..XXXXXXX 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll);
  * bdrv_parent_drained_end_single:
  *
  * End a quiesced section for the parent of @c.
+ *
+ * This polls @bs's AioContext until all scheduled sub-drained_ends
+ * have settled, which may result in graph changes.
  */
 void bdrv_parent_drained_end_single(BdrvChild *c);
 
@@ -XXX,XX +XXX,XX @@ void bdrv_subtree_drained_begin(BlockDriverState *bs);
  * bdrv_drained_end:
  *
  * End a quiescent section started by bdrv_drained_begin().
+ *
+ * This polls @bs's AioContext until all scheduled sub-drained_ends
+ * have settled.  On one hand, that may result in graph changes.  On
+ * the other, this requires that all involved nodes (@bs and all of
+ * its parents) are in the same AioContext, and that the caller has
+ * acquired it.
+ * If there are any nodes that are in different contexts from @bs,
+ * these contexts must not be acquired.
  */
 void bdrv_drained_end(BlockDriverState *bs);
 
+/**
+ * bdrv_drained_end_no_poll:
+ *
+ * Same as bdrv_drained_end(), but do not poll for the subgraph to
+ * actually become unquiesced.  Therefore, no graph changes will occur
+ * with this function.
+ *
+ * *drained_end_counter is incremented for every background operation
+ * that is scheduled, and will be decremented for every operation once
+ * it settles.  The caller must poll until it reaches 0.  The counter
+ * should be accessed using atomic operations only.
+ */
+void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter);
+
 /**
  * End a quiescent section started by bdrv_subtree_drained_begin().
  */
diff --git a/include/block/block_int.h b/include/block/block_int.h
index XXXXXXX..XXXXXXX 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -XXX,XX +XXX,XX @@ struct BdrvChildRole {
      * These functions must not change the graph (and therefore also must not
      * call aio_poll(), which could change the graph indirectly).
      *
+     * If drained_end() schedules background operations, it must atomically
+     * increment *drained_end_counter for each such operation and atomically
+     * decrement it once the operation has settled.
+     *
      * Note that this can be nested. If drained_begin() was called twice, new
      * I/O is allowed only after drained_end() was called twice, too.
      */
     void (*drained_begin)(BdrvChild *child);
-    void (*drained_end)(BdrvChild *child);
+    void (*drained_end)(BdrvChild *child, int *drained_end_counter);
 
     /*
      * Returns whether the parent has pending requests for the child. This
diff --git a/block.c b/block.c
index XXXXXXX..XXXXXXX 100644
--- a/block.c
+++ b/block.c
@@ -XXX,XX +XXX,XX @@ static bool bdrv_child_cb_drained_poll(BdrvChild *child)
     return bdrv_drain_poll(bs, false, NULL, false);
 }
 
-static void bdrv_child_cb_drained_end(BdrvChild *child)
+static void bdrv_child_cb_drained_end(BdrvChild *child,
+                                      int *drained_end_counter)
 {
     BlockDriverState *bs = child->opaque;
-    bdrv_drained_end(bs);
+    bdrv_drained_end_no_poll(bs, drained_end_counter);
 }
 
 static void bdrv_child_cb_attach(BdrvChild *child)
@@ -XXX,XX +XXX,XX @@ static void bdrv_attach_aio_context(BlockDriverState *bs,
 void bdrv_set_aio_context_ignore(BlockDriverState *bs,
                                  AioContext *new_context, GSList **ignore)
 {
+    AioContext *old_context = bdrv_get_aio_context(bs);
+    AioContext *current_context = qemu_get_current_aio_context();
     BdrvChild *child;
 
-    if (bdrv_get_aio_context(bs) == new_context) {
+    if (old_context == new_context) {
         return;
     }
 
@@ -XXX,XX +XXX,XX @@ void bdrv_set_aio_context_ignore(BlockDriverState *bs,
 
     bdrv_detach_aio_context(bs);
 
-    /* This function executes in the old AioContext so acquire the new one in
-     * case it runs in a different thread.
-     */
-    aio_context_acquire(new_context);
+    /* Acquire the new context, if necessary */
+    if (current_context != new_context) {
+        aio_context_acquire(new_context);
+    }
+
     bdrv_attach_aio_context(bs, new_context);
+
+    /*
+     * If this function was recursively called from
+     * bdrv_set_aio_context_ignore(), there may be nodes in the
+     * subtree that have not yet been moved to the new AioContext.
+     * Release the old one so bdrv_drained_end() can poll them.
+     */
+    if (current_context != old_context) {
+        aio_context_release(old_context);
+    }
+
     bdrv_drained_end(bs);
-    aio_context_release(new_context);
+
+    if (current_context != old_context) {
+        aio_context_acquire(old_context);
+    }
+    if (current_context != new_context) {
+        aio_context_release(new_context);
+    }
 }
 
 static bool bdrv_parent_can_set_aio_context(BdrvChild *c, AioContext *ctx,
diff --git a/block/block-backend.c b/block/block-backend.c
index XXXXXXX..XXXXXXX 100644
--- a/block/block-backend.c
+++ b/block/block-backend.c
@@ -XXX,XX +XXX,XX @@ static void blk_root_inherit_options(int *child_flags, QDict *child_options,
 }
 static void blk_root_drained_begin(BdrvChild *child);
 static bool blk_root_drained_poll(BdrvChild *child);
-static void blk_root_drained_end(BdrvChild *child);
+static void blk_root_drained_end(BdrvChild *child, int *drained_end_counter);
 
 static void blk_root_change_media(BdrvChild *child, bool load);
 static void blk_root_resize(BdrvChild *child);
@@ -XXX,XX +XXX,XX @@ int blk_pread_unthrottled(BlockBackend *blk, int64_t offset, uint8_t *buf,
 
     blk_root_drained_begin(blk->root);
     ret = blk_pread(blk, offset, buf, count);
-    blk_root_drained_end(blk->root);
+    blk_root_drained_end(blk->root, NULL);
     return ret;
 }
 
@@ -XXX,XX +XXX,XX @@ static bool blk_root_drained_poll(BdrvChild *child)
     return !!blk->in_flight;
 }
 
-static void blk_root_drained_end(BdrvChild *child)
+static void blk_root_drained_end(BdrvChild *child, int *drained_end_counter)
 {
     BlockBackend *blk = child->opaque;
     assert(blk->quiesce_counter);
diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
     }
 }
 
-void bdrv_parent_drained_end_single(BdrvChild *c)
+static void bdrv_parent_drained_end_single_no_poll(BdrvChild *c,
+                                                   int *drained_end_counter)
 {
     assert(c->parent_quiesce_counter > 0);
     c->parent_quiesce_counter--;
     if (c->role->drained_end) {
-        c->role->drained_end(c);
+        c->role->drained_end(c, drained_end_counter);
     }
 }
 
+void bdrv_parent_drained_end_single(BdrvChild *c)
+{
+    int drained_end_counter = 0;
+    bdrv_parent_drained_end_single_no_poll(c, &drained_end_counter);
+    BDRV_POLL_WHILE(c->bs, atomic_read(&drained_end_counter) > 0);
+}
+
 static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
-                                    bool ignore_bds_parents)
+                                    bool ignore_bds_parents,
+                                    int *drained_end_counter)
 {
     BdrvChild *c, *next;
 
@@ -XXX,XX +XXX,XX @@ static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
         if (c == ignore || (ignore_bds_parents && c->role->parent_is_bds)) {
             continue;
         }
-        bdrv_parent_drained_end_single(c);
+        bdrv_parent_drained_end_single_no_poll(c, drained_end_counter);
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
     atomic_mb_set(&data->done, true);
     bdrv_dec_in_flight(bs);
 
-    if (data->drained_end_counter) {
+    if (!data->begin) {
         atomic_dec(data->drained_end_counter);
     }
 
-    if (data->begin || data->drained_end_counter) {
-        g_free(data);
-    }
+    g_free(data);
 }
 
 /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */
@@ -XXX,XX +XXX,XX @@ static void bdrv_drain_invoke(BlockDriverState *bs, bool begin,
         .drained_end_counter = drained_end_counter,
     };
 
-    if (!begin && drained_end_counter) {
+    if (!begin) {
         atomic_inc(drained_end_counter);
     }
 
@@ -XXX,XX +XXX,XX @@ static void bdrv_drain_invoke(BlockDriverState *bs, bool begin,
     bdrv_inc_in_flight(bs);
     data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data);
     aio_co_schedule(bdrv_get_aio_context(bs), data->co);
-
-    /*
-     * TODO: Drop this and make callers pass @drained_end_counter and poll
-     * themselves
-     */
-    if (!begin && !drained_end_counter) {
-        BDRV_POLL_WHILE(bs, !data->done);
-        g_free(data);
-    }
 }
 
 /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */
@@ -XXX,XX +XXX,XX @@ static void bdrv_co_drain_bh_cb(void *opaque)
         }
         bdrv_dec_in_flight(bs);
         if (data->begin) {
+            assert(!data->drained_end_counter);
             bdrv_do_drained_begin(bs, data->recursive, data->parent,
                                   data->ignore_bds_parents, data->poll);
         } else {
+            assert(!data->poll);
             bdrv_do_drained_end(bs, data->recursive, data->parent,
                                 data->ignore_bds_parents,
                                 data->drained_end_counter);
@@ -XXX,XX +XXX,XX @@ void bdrv_subtree_drained_begin(BlockDriverState *bs)
     bdrv_do_drained_begin(bs, true, NULL, false, true);
 }
 
+/**
+ * This function does not poll, nor must any of its recursively called
+ * functions.  The *drained_end_counter pointee will be incremented
+ * once for every background operation scheduled, and decremented once
+ * the operation settles.  Therefore, the pointer must remain valid
+ * until the pointee reaches 0.  That implies that whoever sets up the
+ * pointee has to poll until it is 0.
+ *
+ * We use atomic operations to access *drained_end_counter, because
+ * (1) when called from bdrv_set_aio_context_ignore(), the subgraph of
+ *     @bs may contain nodes in different AioContexts,
+ * (2) bdrv_drain_all_end() uses the same counter for all nodes,
+ *     regardless of which AioContext they are in.
+ */
 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
                                 BdrvChild *parent, bool ignore_bds_parents,
                                 int *drained_end_counter)
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
     BdrvChild *child, *next;
     int old_quiesce_counter;
 
+    assert(drained_end_counter != NULL);
+
     if (qemu_in_coroutine()) {
         bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents,
                                false, drained_end_counter);
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
 
     /* Re-enable things in child-to-parent order */
     bdrv_drain_invoke(bs, false, drained_end_counter);
-    bdrv_parent_drained_end(bs, parent, ignore_bds_parents);
+    bdrv_parent_drained_end(bs, parent, ignore_bds_parents,
+                            drained_end_counter);
 
     old_quiesce_counter = atomic_fetch_dec(&bs->quiesce_counter);
     if (old_quiesce_counter == 1) {
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
 
 void bdrv_drained_end(BlockDriverState *bs)
 {
-    bdrv_do_drained_end(bs, false, NULL, false, NULL);
+    int drained_end_counter = 0;
+    bdrv_do_drained_end(bs, false, NULL, false, &drained_end_counter);
+    BDRV_POLL_WHILE(bs, atomic_read(&drained_end_counter) > 0);
+}
+
+void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter)
+{
+    bdrv_do_drained_end(bs, false, NULL, false, drained_end_counter);
 }
 
 void bdrv_subtree_drained_end(BlockDriverState *bs)
 {
-    bdrv_do_drained_end(bs, true, NULL, false, NULL);
+    int drained_end_counter = 0;
+    bdrv_do_drained_end(bs, true, NULL, false, &drained_end_counter);
+    BDRV_POLL_WHILE(bs, atomic_read(&drained_end_counter) > 0);
 }
 
 void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
@@ -XXX,XX +XXX,XX @@ void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
 
 void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent)
 {
+    int drained_end_counter = 0;
     int i;
 
     for (i = 0; i < old_parent->recursive_quiesce_counter; i++) {
-        bdrv_do_drained_end(child->bs, true, child, false, NULL);
+        bdrv_do_drained_end(child->bs, true, child, false,
+                            &drained_end_counter);
     }
+
+    BDRV_POLL_WHILE(child->bs, atomic_read(&drained_end_counter) > 0);
 }
 
 /*
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
 void bdrv_drain_all_end(void)
 {
     BlockDriverState *bs = NULL;
+    int drained_end_counter = 0;
 
     while ((bs = bdrv_next_all_states(bs))) {
         AioContext *aio_context = bdrv_get_aio_context(bs);
 
         aio_context_acquire(aio_context);
-        bdrv_do_drained_end(bs, false, NULL, true, NULL);
+        bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter);
         aio_context_release(aio_context);
     }
 
+    assert(qemu_get_current_aio_context() == qemu_get_aio_context());
+    AIO_WAIT_WHILE(NULL, atomic_read(&drained_end_counter) > 0);
+
     assert(bdrv_drain_all_count > 0);
     bdrv_drain_all_count--;
 }
diff --git a/blockjob.c b/blockjob.c
index XXXXXXX..XXXXXXX 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -XXX,XX +XXX,XX @@ static bool child_job_drained_poll(BdrvChild *c)
     }
 }
 
-static void child_job_drained_end(BdrvChild *c)
+static void child_job_drained_end(BdrvChild *c, int *drained_end_counter)
 {
     BlockJob *job = c->opaque;
     job_resume(&job->job);
-- 
2.20.1

From: Max Reitz <mreitz@redhat.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/test-bdrv-drain.c | 36 ++++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
index XXXXXXX..XXXXXXX 100644
--- a/tests/test-bdrv-drain.c
+++ b/tests/test-bdrv-drain.c
@@ -XXX,XX +XXX,XX @@ typedef struct TestDropBackingBlockJob {
     BlockJob common;
     bool should_complete;
     bool *did_complete;
+    BlockDriverState *detach_also;
 } TestDropBackingBlockJob;
 
 static int coroutine_fn test_drop_backing_job_run(Job *job, Error **errp)
@@ -XXX,XX +XXX,XX @@ static void test_drop_backing_job_commit(Job *job)
         container_of(job, TestDropBackingBlockJob, common.job);
 
     bdrv_set_backing_hd(blk_bs(s->common.blk), NULL, &error_abort);
+    bdrv_set_backing_hd(s->detach_also, NULL, &error_abort);
 
     *s->did_complete = true;
 }
@@ -XXX,XX +XXX,XX @@ static const BlockJobDriver test_drop_backing_job_driver = {
  * Creates a child node with three parent nodes on it, and then runs a
  * block job on the final one, parent-node-2.
  *
- * (TODO: parent-node-0 currently serves no purpose, but will as of a
- * follow-up patch.)
- *
  * The job is then asked to complete before a section where the child
  * is drained.
  *
@@ -XXX,XX +XXX,XX @@ static const BlockJobDriver test_drop_backing_job_driver = {
  *
  * Ending the drain on parent-node-1 will poll the AioContext, which
  * lets job_exit() and thus test_drop_backing_job_commit() run.  That
- * function removes the child as parent-node-2's backing file.
+ * function first removes the child as parent-node-2's backing file.
  *
  * In old (and buggy) implementations, there are two problems with
  * that:
@@ -XXX,XX +XXX,XX @@ static const BlockJobDriver test_drop_backing_job_driver = {
  * bdrv_replace_child_noperm() therefore must call drained_end() on
  * the parent only if it really is still drained because the child is
  * drained.
+ *
+ * If removing child from parent-node-2 was successful (as it should
+ * be), test_drop_backing_job_commit() will then also remove the child
+ * from parent-node-0.
+ *
+ * With an old version of our drain infrastructure ((A) above), that
+ * resulted in the following flow:
+ *
+ * 1. child attempts to leave its drained section.  The call recurses
+ *    to its parents.
+ *
+ * 2. parent-node-2 leaves the drained section.  Polling in
+ *    bdrv_drain_invoke() will schedule job_exit().
+ *
+ * 3. parent-node-1 leaves the drained section.  Polling in
+ *    bdrv_drain_invoke() will run job_exit(), thus disconnecting
+ *    parent-node-0 from the child node.
+ *
+ * 4. bdrv_parent_drained_end() uses a QLIST_FOREACH_SAFE() loop to
+ *    iterate over the parents.  Thus, it now accesses the BdrvChild
+ *    object that used to connect parent-node-0 and the child node.
+ *    However, that object no longer exists, so it accesses a dangling
+ *    pointer.
+ *
+ * The solution is to only poll once when running a bdrv_drained_end()
+ * operation, specifically at the end when all drained_end()
+ * operations for all involved nodes have been scheduled.
+ * Note that this also solves (A) above, thus hiding (B).
  */
 static void test_blockjob_commit_by_drained_end(void)
 {
@@ -XXX,XX +XXX,XX @@ static void test_blockjob_commit_by_drained_end(void)
                            bs_parents[2], 0, BLK_PERM_ALL, 0, 0, NULL, NULL,
                            &error_abort);
 
+    job->detach_also = bs_parents[0];
     job->did_complete = &job_has_completed;
 
     job_start(&job->common.job);
-- 
2.20.1

From: Max Reitz <mreitz@redhat.com>

The graph must not change in these loops (or a QLIST_FOREACH_SAFE would
not even be enough).  We now ensure this by only polling once in the
root bdrv_drained_end() call, so we can drop the _SAFE suffix.  Doing so
makes it clear that the graph must not change.

Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 block/io.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
                                     bool ignore_bds_parents,
                                     int *drained_end_counter)
 {
-    BdrvChild *c, *next;
+    BdrvChild *c;
 
-    QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
+    QLIST_FOREACH(c, &bs->parents, next_parent) {
         if (c == ignore || (ignore_bds_parents && c->role->parent_is_bds)) {
             continue;
         }
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
                                 BdrvChild *parent, bool ignore_bds_parents,
                                 int *drained_end_counter)
 {
-    BdrvChild *child, *next;
+    BdrvChild *child;
     int old_quiesce_counter;
 
     assert(drained_end_counter != NULL);
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
     if (recursive) {
         assert(!ignore_bds_parents);
         bs->recursive_quiesce_counter--;
-        QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
+        QLIST_FOREACH(child, &bs->children, next) {
             bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents,
                                 drained_end_counter);
         }
-- 
2.20.1

From: Max Reitz <mreitz@redhat.com>

If a test has issued a quit command already (which may be useful to do
explicitly because the test wants to show its effects),
QEMUMachine.shutdown() should not do so again.  Otherwise, the VM may
well return an ECONNRESET which will lead QEMUMachine.shutdown() to
killing it, which then turns into a "qemu received signal 9" line.

Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 python/qemu/machine.py | 5 +++--
 tests/qemu-iotests/255 | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/python/qemu/machine.py b/python/qemu/machine.py
index XXXXXXX..XXXXXXX 100644
--- a/python/qemu/machine.py
+++ b/python/qemu/machine.py
@@ -XXX,XX +XXX,XX @@ class QEMUMachine(object):
         self._load_io_log()
         self._post_shutdown()
 
-    def shutdown(self):
+    def shutdown(self, has_quit=False):
         """
         Terminate the VM and clean up
         """
         if self.is_running():
             try:
-                self._qmp.cmd('quit')
+                if not has_quit:
+                    self._qmp.cmd('quit')
                 self._qmp.close()
             except:
                 self._popen.kill()
diff --git a/tests/qemu-iotests/255 b/tests/qemu-iotests/255
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/255
+++ b/tests/qemu-iotests/255
@@ -XXX,XX +XXX,XX @@ with iotests.FilePath('src.qcow2') as src_path, \
     vm.qmp_log('block-job-cancel', device='job0')
     vm.qmp_log('quit')
 
-    vm.shutdown()
+    vm.shutdown(has_quit=True)
-- 
2.20.1

From: Max Reitz <mreitz@redhat.com>

Before the previous patches, the first case resulted in a failed
assertion (which is noted as qemu receiving a SIGABRT in the test
output), and the second usually triggered a segmentation fault.

Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/qemu-iotests/040     | 40 +++++++++++++++++++++++++++++++++++++-
 tests/qemu-iotests/040.out |  4 ++--
 2 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/tests/qemu-iotests/040 b/tests/qemu-iotests/040
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/040
+++ b/tests/qemu-iotests/040
@@ -XXX,XX +XXX,XX @@ class TestSingleDrive(ImageCommitTestCase):
 
         self.vm.add_device("scsi-hd,id=scsi0,drive=drive0")
         self.vm.launch()
+        self.has_quit = False
 
     def tearDown(self):
-        self.vm.shutdown()
+        self.vm.shutdown(has_quit=self.has_quit)
         os.remove(test_img)
         os.remove(mid_img)
         os.remove(backing_img)
@@ -XXX,XX +XXX,XX @@ class TestSingleDrive(ImageCommitTestCase):
         self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xab 0 524288', backing_img).find("verification failed"))
         self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xef 524288 524288', backing_img).find("verification failed"))
 
+    def test_commit_with_filter_and_quit(self):
+        result = self.vm.qmp('object-add', qom_type='throttle-group', id='tg')
+        self.assert_qmp(result, 'return', {})
+
+        # Add a filter outside of the backing chain
+        result = self.vm.qmp('blockdev-add', driver='throttle', node_name='filter', throttle_group='tg', file='mid')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('block-commit', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        # Quit immediately, thus forcing a simultaneous cancel of the
+        # block job and a bdrv_drain_all()
+        result = self.vm.qmp('quit')
+        self.assert_qmp(result, 'return', {})
+
+        self.has_quit = True
+
+    # Same as above, but this time we add the filter after starting the job
+    def test_commit_plus_filter_and_quit(self):
+        result = self.vm.qmp('object-add', qom_type='throttle-group', id='tg')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('block-commit', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        # Add a filter outside of the backing chain
+        result = self.vm.qmp('blockdev-add', driver='throttle', node_name='filter', throttle_group='tg', file='mid')
+        self.assert_qmp(result, 'return', {})
+
+        # Quit immediately, thus forcing a simultaneous cancel of the
+        # block job and a bdrv_drain_all()
+        result = self.vm.qmp('quit')
+        self.assert_qmp(result, 'return', {})
+
+        self.has_quit = True
+
     def test_device_not_found(self):
         result = self.vm.qmp('block-commit', device='nonexistent', top='%s' % mid_img)
         self.assert_qmp(result, 'error/class', 'DeviceNotFound')
diff --git a/tests/qemu-iotests/040.out b/tests/qemu-iotests/040.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/040.out
+++ b/tests/qemu-iotests/040.out
@@ -XXX,XX +XXX,XX @@
-...........................................
+...............................................
 ----------------------------------------------------------------------
-Ran 43 tests
+Ran 47 tests
 
 OK
-- 
2.20.1

From: Max Reitz <mreitz@redhat.com>

If the main loop cancels all block jobs while the block layer is not
drained, this cancelling may not happen instantaneously.  We can start a
drained section before vm_shutdown(), which entails another
bdrv_drain_all(); this nested bdrv_drain_all() will thus be a no-op,
basically.

We do not have to end the drained section, because we actually do not
want any requests to happen from this point on.

Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 vl.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/vl.c b/vl.c
index XXXXXXX..XXXXXXX 100644
--- a/vl.c
+++ b/vl.c
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
      */
     migration_shutdown();
 
+    /*
+     * We must cancel all block jobs while the block layer is drained,
+     * or cancelling will be affected by throttling and thus may block
+     * for an extended period of time.
+     * vm_shutdown() will bdrv_drain_all(), so we may as well include
+     * it in the drained section.
+     * We do not need to end this section, because we do not want any
+     * requests happening from here on anyway.
+     */
+    bdrv_drain_all_begin();
+
     /* No more vcpu or device emulation activity beyond this point */
     vm_shutdown();
 
-- 
2.20.1

From: Max Reitz <mreitz@redhat.com>

When qemu quits, all throttling should be ignored.  That means, if there
is a mirror job running from a throttled node, it should be cancelled
immediately and qemu close without blocking.

Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/qemu-iotests/218     | 55 ++++++++++++++++++++++++++++++++++++--
 tests/qemu-iotests/218.out |  4 +++
 2 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/tests/qemu-iotests/218 b/tests/qemu-iotests/218
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/218
+++ b/tests/qemu-iotests/218
@@ -XXX,XX +XXX,XX @@
 # Creator/Owner: Max Reitz <mreitz@redhat.com>
 
 import iotests
-from iotests import log
+from iotests import log, qemu_img, qemu_io_silent
 
-iotests.verify_platform(['linux'])
+iotests.verify_image_format(supported_fmts=['qcow2', 'raw'])
 
 
 # Launches the VM, adds two null-co nodes (source and target), and
@@ -XXX,XX +XXX,XX @@ with iotests.VM() as vm:
 
     log(vm.event_wait('BLOCK_JOB_CANCELLED'),
         filters=[iotests.filter_qmp_event])
+
+log('')
+log('=== Cancel mirror job from throttled node by quitting ===')
+log('')
+
+with iotests.VM() as vm, \
+     iotests.FilePath('src.img') as src_img_path:
+
+    assert qemu_img('create', '-f', iotests.imgfmt, src_img_path, '64M') == 0
+    assert qemu_io_silent('-f', iotests.imgfmt, src_img_path,
+                          '-c', 'write -P 42 0M 64M') == 0
+
+    vm.launch()
+
+    ret = vm.qmp('object-add', qom_type='throttle-group', id='tg',
+                 props={'x-bps-read': 4096})
+    assert ret['return'] == {}
+
+    ret = vm.qmp('blockdev-add',
+                 node_name='source',
+                 driver=iotests.imgfmt,
+                 file={
+                     'driver': 'file',
+                     'filename': src_img_path
+                 })
+    assert ret['return'] == {}
+
+    ret = vm.qmp('blockdev-add',
+                 node_name='throttled-source',
+                 driver='throttle',
+                 throttle_group='tg',
+                 file='source')
+    assert ret['return'] == {}
+
+    ret = vm.qmp('blockdev-add',
+                 node_name='target',
+                 driver='null-co',
+                 size=(64 * 1048576))
+    assert ret['return'] == {}
+
+    ret = vm.qmp('blockdev-mirror',
+                 job_id='mirror',
+                 device='throttled-source',
+                 target='target',
+                 sync='full')
+    assert ret['return'] == {}
+
+    log(vm.qmp('quit'))
+
+    with iotests.Timeout(5, 'Timeout waiting for VM to quit'):
+        vm.shutdown(has_quit=True)
diff --git a/tests/qemu-iotests/218.out b/tests/qemu-iotests/218.out
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/218.out
+++ b/tests/qemu-iotests/218.out
@@ -XXX,XX +XXX,XX @@ Cancelling job
 Cancelling job
 {"return": {}}
 {"data": {"device": "mirror", "len": 1048576, "offset": 1048576, "speed": 0, "type": "mirror"}, "event": "BLOCK_JOB_CANCELLED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
+
+=== Cancel mirror job from throttled node by quitting ===
+
+{"return": {}}
-- 
2.20.1

The following changes since commit 281f327487c9c9b1599f93c589a408bbf4a651b8:

Merge remote-tracking branch 'remotes/vivier/tags/m68k-for-2.12-pull-request' into staging (2017-12-22 00:11:36 +0000)

are available in the git repository at:

git://repo.or.cz/qemu/kevin.git tags/for-upstream

for you to fetch changes up to 1a63a907507fbbcfaee3f622907ec244b7eabda8:

block: Keep nodes drained between reopen_queue/multiple (2017-12-22 15:05:32 +0100)

----------------------------------------------------------------
Block layer patches

----------------------------------------------------------------
Doug Gale (1):
      nvme: Add tracing

Edgar Kaziakhmedov (1):
      qcow2: get rid of qcow2_backing_read1 routine

Fam Zheng (2):
      block: Open backing image in force share mode for size probe
      block: Remove unused bdrv_requests_pending

John Snow (1):
      iotests: fix 197 for vpc

Kevin Wolf (27):
      block: Formats don't need CONSISTENT_READ with NO_IO
      block: Make bdrv_drain_invoke() recursive
      block: Call .drain_begin only once in bdrv_drain_all_begin()
      test-bdrv-drain: Test BlockDriver callbacks for drain
      block: bdrv_drain_recurse(): Remove unused begin parameter
      block: Don't wait for requests in bdrv_drain*_end()
      block: Unify order in drain functions
      block: Don't acquire AioContext in hmp_qemu_io()
      block: Document that x-blockdev-change breaks quorum children list
      block: Assert drain_all is only called from main AioContext
      block: Make bdrv_drain() driver callbacks non-recursive
      test-bdrv-drain: Test callback for bdrv_drain
      test-bdrv-drain: Test bs->quiesce_counter
      blockjob: Pause job on draining any job BDS
      test-bdrv-drain: Test drain vs. block jobs
      block: Don't block_job_pause_all() in bdrv_drain_all()
      block: Nested drain_end must still call callbacks
      test-bdrv-drain: Test nested drain sections
      block: Don't notify parents in drain call chain
      block: Add bdrv_subtree_drained_begin/end()
      test-bdrv-drain: Tests for bdrv_subtree_drain
      test-bdrv-drain: Test behaviour in coroutine context
      test-bdrv-drain: Recursive draining with multiple parents
      block: Allow graph changes in subtree drained section
      test-bdrv-drain: Test graph changes in drained section
      commit: Simplify reopen of base
      block: Keep nodes drained between reopen_queue/multiple

Thomas Huth (3):
      block: Remove the obsolete -drive boot=on|off parameter
      block: Remove the deprecated -hdachs option
      block: Mention -drive cyls/heads/secs/trans/serial/addr in deprecation chapter

Commit 1f4ad7d fixed 'qemu-img info' for raw images that are currently
in use as a mirror target. It is not enough for image formats, though,
as these still unconditionally request BLK_PERM_CONSISTENT_READ.

As this permission is geared towards whether the guest-visible data is
consistent, and has no impact on whether the metadata is sane, and
'qemu-img info' does not read guest-visible data (except for the raw
format), it makes sense to not require BLK_PERM_CONSISTENT_READ if there
is not going to be any guest I/O performed, regardless of image format.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 block.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/block.c b/block.c
index XXXXXXX..XXXXXXX 100644
--- a/block.c
+++ b/block.c
@@ -XXX,XX +XXX,XX @@ void bdrv_format_default_perms(BlockDriverState *bs, BdrvChild *c,
     assert(role == &child_backing || role == &child_file);
 
     if (!backing) {
+        int flags = bdrv_reopen_get_flags(reopen_queue, bs);
+
         /* Apart from the modifications below, the same permissions are
          * forwarded and left alone as for filters */
         bdrv_filter_default_perms(bs, c, role, reopen_queue, perm, shared,
@@ -XXX,XX +XXX,XX @@ void bdrv_format_default_perms(BlockDriverState *bs, BdrvChild *c,
 
         /* bs->file always needs to be consistent because of the metadata. We
          * can never allow other users to resize or write to it. */
-        perm |= BLK_PERM_CONSISTENT_READ;
+        if (!(flags & BDRV_O_NO_IO)) {
+            perm |= BLK_PERM_CONSISTENT_READ;
+        }
         shared &= ~(BLK_PERM_WRITE | BLK_PERM_RESIZE);
     } else {
         /* We want consistent read from backing files if the parent needs it.
-- 
2.13.6

From: John Snow <jsnow@redhat.com>

VPC has some difficulty creating geometries of particular size.
However, we can indeed force it to use a literal one, so let's
do that for the sake of test 197, which is testing some specific
offsets.

Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Lukáš Doktor <ldoktor@redhat.com>
---
 tests/qemu-iotests/197           | 4 ++++
 tests/qemu-iotests/common.filter | 3 ++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/qemu-iotests/197 b/tests/qemu-iotests/197
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/197
+++ b/tests/qemu-iotests/197
@@ -XXX,XX +XXX,XX @@ echo '=== Copy-on-read ==='
 echo
 
 # Prep the images
+# VPC rounds image sizes to a specific geometry, force a specific size.
+if [ "$IMGFMT" = "vpc" ]; then
+    IMGOPTS=$(_optstr_add "$IMGOPTS" "force_size")
+fi
 _make_test_img 4G
 $QEMU_IO -c "write -P 55 3G 1k" "$TEST_IMG" | _filter_qemu_io
 IMGPROTO=file IMGFMT=qcow2 IMGOPTS= TEST_IMG_FILE="$TEST_WRAP" \
diff --git a/tests/qemu-iotests/common.filter b/tests/qemu-iotests/common.filter
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/common.filter
+++ b/tests/qemu-iotests/common.filter
@@ -XXX,XX +XXX,XX @@ _filter_img_create()
         -e "s# log_size=[0-9]\\+##g" \
         -e "s# refcount_bits=[0-9]\\+##g" \
         -e "s# key-secret=[a-zA-Z0-9]\\+##g" \
-        -e "s# iter-time=[0-9]\\+##g"
+        -e "s# iter-time=[0-9]\\+##g" \
+        -e "s# force_size=\$on\\|off\$##g"
 }
 
 _filter_img_info()
-- 
2.13.6

This change separates bdrv_drain_invoke(), which calls the BlockDriver
drain callbacks, from bdrv_drain_recurse(). Instead, the function
performs its own recursion now.

One reason for this is that bdrv_drain_recurse() can be called multiple
times by bdrv_drain_all_begin(), but the callbacks may only be called
once. The separation is necessary to fix this bug.

The other reason is that we intend to go to a model where we call all
driver callbacks first, and only then start polling. This is not fully
achieved yet with this patch, as bdrv_drain_invoke() contains a
BDRV_POLL_WHILE() loop for the block driver callbacks, which can still
call callbacks for any unrelated event. It's a step in this direction
anyway.

Cc: qemu-stable@nongnu.org
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 block/io.c | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
     bdrv_wakeup(bs);
 }
 
+/* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */
 static void bdrv_drain_invoke(BlockDriverState *bs, bool begin)
 {
+    BdrvChild *child, *tmp;
     BdrvCoDrainData data = { .bs = bs, .done = false, .begin = begin};
 
     if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) ||
@@ -XXX,XX +XXX,XX @@ static void bdrv_drain_invoke(BlockDriverState *bs, bool begin)
     data.co = qemu_coroutine_create(bdrv_drain_invoke_entry, &data);
     bdrv_coroutine_enter(bs, data.co);
     BDRV_POLL_WHILE(bs, !data.done);
+
+    QLIST_FOREACH_SAFE(child, &bs->children, next, tmp) {
+        bdrv_drain_invoke(child->bs, begin);
+    }
 }
 
 static bool bdrv_drain_recurse(BlockDriverState *bs, bool begin)
@@ -XXX,XX +XXX,XX @@ static bool bdrv_drain_recurse(BlockDriverState *bs, bool begin)
     BdrvChild *child, *tmp;
     bool waited;
 
-    /* Ensure any pending metadata writes are submitted to bs->file.  */
-    bdrv_drain_invoke(bs, begin);
-
     /* Wait for drained requests to finish */
     waited = BDRV_POLL_WHILE(bs, atomic_read(&bs->in_flight) > 0);
 
@@ -XXX,XX +XXX,XX @@ void bdrv_drained_begin(BlockDriverState *bs)
         bdrv_parent_drained_begin(bs);
     }
 
+    bdrv_drain_invoke(bs, true);
     bdrv_drain_recurse(bs, true);
 }
 
@@ -XXX,XX +XXX,XX @@ void bdrv_drained_end(BlockDriverState *bs)
     }
 
     bdrv_parent_drained_end(bs);
+    bdrv_drain_invoke(bs, false);
     bdrv_drain_recurse(bs, false);
     aio_enable_external(bdrv_get_aio_context(bs));
 }
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
             aio_context_acquire(aio_context);
             for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
                 if (aio_context == bdrv_get_aio_context(bs)) {
+                    /* FIXME Calling this multiple times is wrong */
+                    bdrv_drain_invoke(bs, true);
                     waited |= bdrv_drain_recurse(bs, true);
                 }
             }
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_end(void)
         aio_context_acquire(aio_context);
         aio_enable_external(aio_context);
         bdrv_parent_drained_end(bs);
+        bdrv_drain_invoke(bs, false);
         bdrv_drain_recurse(bs, false);
         aio_context_release(aio_context);
     }
-- 
2.13.6

bdrv_drain_all_begin() used to call the .bdrv_co_drain_begin() driver
callback inside its polling loop. This means that how many times it got
called for each node depended on long it had to poll the event loop.

This is obviously not right and results in nodes that stay drained even
after bdrv_drain_all_end(), which calls .bdrv_co_drain_begin() once per
node.

Fix bdrv_drain_all_begin() to call the callback only once, too.

Cc: qemu-stable@nongnu.org
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 block/io.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
         aio_context_acquire(aio_context);
         bdrv_parent_drained_begin(bs);
         aio_disable_external(aio_context);
+        bdrv_drain_invoke(bs, true);
         aio_context_release(aio_context);
 
         if (!g_slist_find(aio_ctxs, aio_context)) {
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
             aio_context_acquire(aio_context);
             for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
                 if (aio_context == bdrv_get_aio_context(bs)) {
-                    /* FIXME Calling this multiple times is wrong */
-                    bdrv_drain_invoke(bs, true);
                     waited |= bdrv_drain_recurse(bs, true);
                 }
             }
-- 
2.13.6

This adds a test case that the BlockDriver callbacks for drain are
called in bdrv_drained_all_begin/end(), and that both of them are called
exactly once.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
---
 tests/test-bdrv-drain.c | 137 ++++++++++++++++++++++++++++++++++++++++++++++++
 tests/Makefile.include  |   2 +
 2 files changed, 139 insertions(+)
 create mode 100644 tests/test-bdrv-drain.c

diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/tests/test-bdrv-drain.c
@@ -XXX,XX +XXX,XX @@
+/*
+ * Block node draining tests
+ *
+ * Copyright (c) 2017 Kevin Wolf <kwolf@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "block/block.h"
+#include "sysemu/block-backend.h"
+#include "qapi/error.h"
+
+typedef struct BDRVTestState {
+    int drain_count;
+} BDRVTestState;
+
+static void coroutine_fn bdrv_test_co_drain_begin(BlockDriverState *bs)
+{
+    BDRVTestState *s = bs->opaque;
+    s->drain_count++;
+}
+
+static void coroutine_fn bdrv_test_co_drain_end(BlockDriverState *bs)
+{
+    BDRVTestState *s = bs->opaque;
+    s->drain_count--;
+}
+
+static void bdrv_test_close(BlockDriverState *bs)
+{
+    BDRVTestState *s = bs->opaque;
+    g_assert_cmpint(s->drain_count, >, 0);
+}
+
+static int coroutine_fn bdrv_test_co_preadv(BlockDriverState *bs,
+                                            uint64_t offset, uint64_t bytes,
+                                            QEMUIOVector *qiov, int flags)
+{
+    /* We want this request to stay until the polling loop in drain waits for
+     * it to complete. We need to sleep a while as bdrv_drain_invoke() comes
+     * first and polls its result, too, but it shouldn't accidentally complete
+     * this request yet. */
+    qemu_co_sleep_ns(QEMU_CLOCK_REALTIME, 100000);
+
+    return 0;
+}
+
+static BlockDriver bdrv_test = {
+    .format_name            = "test",
+    .instance_size          = sizeof(BDRVTestState),
+
+    .bdrv_close             = bdrv_test_close,
+    .bdrv_co_preadv         = bdrv_test_co_preadv,
+
+    .bdrv_co_drain_begin    = bdrv_test_co_drain_begin,
+    .bdrv_co_drain_end      = bdrv_test_co_drain_end,
+};
+
+static void aio_ret_cb(void *opaque, int ret)
+{
+    int *aio_ret = opaque;
+    *aio_ret = ret;
+}
+
+static void test_drv_cb_drain_all(void)
+{
+    BlockBackend *blk;
+    BlockDriverState *bs;
+    BDRVTestState *s;
+    BlockAIOCB *acb;
+    int aio_ret;
+
+    QEMUIOVector qiov;
+    struct iovec iov = {
+        .iov_base = NULL,
+        .iov_len = 0,
+    };
+    qemu_iovec_init_external(&qiov, &iov, 1);
+
+    blk = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
+    bs = bdrv_new_open_driver(&bdrv_test, "test-node", BDRV_O_RDWR,
+                              &error_abort);
+    s = bs->opaque;
+    blk_insert_bs(blk, bs, &error_abort);
+
+    /* Simple bdrv_drain_all_begin/end pair, check that CBs are called */
+    g_assert_cmpint(s->drain_count, ==, 0);
+    bdrv_drain_all_begin();
+    g_assert_cmpint(s->drain_count, ==, 1);
+    bdrv_drain_all_end();
+    g_assert_cmpint(s->drain_count, ==, 0);
+
+    /* Now do the same while a request is pending */
+    aio_ret = -EINPROGRESS;
+    acb = blk_aio_preadv(blk, 0, &qiov, 0, aio_ret_cb, &aio_ret);
+    g_assert(acb != NULL);
+    g_assert_cmpint(aio_ret, ==, -EINPROGRESS);
+
+    g_assert_cmpint(s->drain_count, ==, 0);
+    bdrv_drain_all_begin();
+    g_assert_cmpint(aio_ret, ==, 0);
+    g_assert_cmpint(s->drain_count, ==, 1);
+    bdrv_drain_all_end();
+    g_assert_cmpint(s->drain_count, ==, 0);
+
+    bdrv_unref(bs);
+    blk_unref(blk);
+}
+
+int main(int argc, char **argv)
+{
+    bdrv_init();
+    qemu_init_main_loop(&error_abort);
+
+    g_test_init(&argc, &argv, NULL);
+
+    g_test_add_func("/bdrv-drain/driver-cb/drain_all", test_drv_cb_drain_all);
+
+    return g_test_run();
+}
diff --git a/tests/Makefile.include b/tests/Makefile.include
index XXXXXXX..XXXXXXX 100644
--- a/tests/Makefile.include
+++ b/tests/Makefile.include
@@ -XXX,XX +XXX,XX @@ gcov-files-test-thread-pool-y = thread-pool.c
 gcov-files-test-hbitmap-y = util/hbitmap.c
 check-unit-y += tests/test-hbitmap$(EXESUF)
 gcov-files-test-hbitmap-y = blockjob.c
+check-unit-y += tests/test-bdrv-drain$(EXESUF)
 check-unit-y += tests/test-blockjob$(EXESUF)
 check-unit-y += tests/test-blockjob-txn$(EXESUF)
 check-unit-y += tests/test-x86-cpuid$(EXESUF)
@@ -XXX,XX +XXX,XX @@ tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(test-block-obj-y)
 tests/test-aio$(EXESUF): tests/test-aio.o $(test-block-obj-y)
 tests/test-aio-multithread$(EXESUF): tests/test-aio-multithread.o $(test-block-obj-y)
 tests/test-throttle$(EXESUF): tests/test-throttle.o $(test-block-obj-y)
+tests/test-bdrv-drain$(EXESUF): tests/test-bdrv-drain.o $(test-block-obj-y) $(test-util-obj-y)
 tests/test-blockjob$(EXESUF): tests/test-blockjob.o $(test-block-obj-y) $(test-util-obj-y)
 tests/test-blockjob-txn$(EXESUF): tests/test-blockjob-txn.o $(test-block-obj-y) $(test-util-obj-y)
 tests/test-thread-pool$(EXESUF): tests/test-thread-pool.o $(test-block-obj-y)
-- 
2.13.6

Now that the bdrv_drain_invoke() calls are pulled up to the callers of
bdrv_drain_recurse(), the 'begin' parameter isn't needed any more.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 block/io.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ static void bdrv_drain_invoke(BlockDriverState *bs, bool begin)
     }
 }
 
-static bool bdrv_drain_recurse(BlockDriverState *bs, bool begin)
+static bool bdrv_drain_recurse(BlockDriverState *bs)
 {
     BdrvChild *child, *tmp;
     bool waited;
@@ -XXX,XX +XXX,XX @@ static bool bdrv_drain_recurse(BlockDriverState *bs, bool begin)
              */
             bdrv_ref(bs);
         }
-        waited |= bdrv_drain_recurse(bs, begin);
+        waited |= bdrv_drain_recurse(bs);
         if (in_main_loop) {
             bdrv_unref(bs);
         }
@@ -XXX,XX +XXX,XX @@ void bdrv_drained_begin(BlockDriverState *bs)
     }
 
     bdrv_drain_invoke(bs, true);
-    bdrv_drain_recurse(bs, true);
+    bdrv_drain_recurse(bs);
 }
 
 void bdrv_drained_end(BlockDriverState *bs)
@@ -XXX,XX +XXX,XX @@ void bdrv_drained_end(BlockDriverState *bs)
 
     bdrv_parent_drained_end(bs);
     bdrv_drain_invoke(bs, false);
-    bdrv_drain_recurse(bs, false);
+    bdrv_drain_recurse(bs);
     aio_enable_external(bdrv_get_aio_context(bs));
 }
 
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
             aio_context_acquire(aio_context);
             for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
                 if (aio_context == bdrv_get_aio_context(bs)) {
-                    waited |= bdrv_drain_recurse(bs, true);
+                    waited |= bdrv_drain_recurse(bs);
                 }
             }
             aio_context_release(aio_context);
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_end(void)
         aio_enable_external(aio_context);
         bdrv_parent_drained_end(bs);
         bdrv_drain_invoke(bs, false);
-        bdrv_drain_recurse(bs, false);
+        bdrv_drain_recurse(bs);
         aio_context_release(aio_context);
     }
 
-- 
2.13.6

The device is drained, so there is no point in waiting for requests at
the end of the drained section. Remove the bdrv_drain_recurse() calls
there.

The bdrv_drain_recurse() calls were introduced in commit 481cad48e5e
in order to call the .bdrv_co_drain_end() driver callback. This is now
done by a separate bdrv_drain_invoke() call.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 block/io.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ void bdrv_drained_end(BlockDriverState *bs)
 
     bdrv_parent_drained_end(bs);
     bdrv_drain_invoke(bs, false);
-    bdrv_drain_recurse(bs);
     aio_enable_external(bdrv_get_aio_context(bs));
 }
 
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_end(void)
         aio_enable_external(aio_context);
         bdrv_parent_drained_end(bs);
         bdrv_drain_invoke(bs, false);
-        bdrv_drain_recurse(bs);
         aio_context_release(aio_context);
     }
 
-- 
2.13.6

Drain requests are propagated to child nodes, parent nodes and directly
to the AioContext. The order in which this happened was different
between all combinations of drain/drain_all and begin/end.

The correct order is to keep children only drained when their parents
are also drained. This means that at the start of a drained section, the
AioContext needs to be drained first, the parents second and only then
the children. The correct order for the end of a drained section is the
opposite.

This patch changes the three other functions to follow the example of
bdrv_drained_begin(), which is the only one that got it right.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 block/io.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ void bdrv_drained_begin(BlockDriverState *bs)
         return;
     }
 
+    /* Stop things in parent-to-child order */
     if (atomic_fetch_inc(&bs->quiesce_counter) == 0) {
         aio_disable_external(bdrv_get_aio_context(bs));
         bdrv_parent_drained_begin(bs);
@@ -XXX,XX +XXX,XX @@ void bdrv_drained_end(BlockDriverState *bs)
         return;
     }
 
-    bdrv_parent_drained_end(bs);
+    /* Re-enable things in child-to-parent order */
     bdrv_drain_invoke(bs, false);
+    bdrv_parent_drained_end(bs);
     aio_enable_external(bdrv_get_aio_context(bs));
 }
 
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
         AioContext *aio_context = bdrv_get_aio_context(bs);
 
+        /* Stop things in parent-to-child order */
         aio_context_acquire(aio_context);
-        bdrv_parent_drained_begin(bs);
         aio_disable_external(aio_context);
+        bdrv_parent_drained_begin(bs);
         bdrv_drain_invoke(bs, true);
         aio_context_release(aio_context);
 
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_end(void)
     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
         AioContext *aio_context = bdrv_get_aio_context(bs);
 
+        /* Re-enable things in child-to-parent order */
         aio_context_acquire(aio_context);
-        aio_enable_external(aio_context);
-        bdrv_parent_drained_end(bs);
         bdrv_drain_invoke(bs, false);
+        bdrv_parent_drained_end(bs);
+        aio_enable_external(aio_context);
         aio_context_release(aio_context);
     }
 
-- 
2.13.6

Commit 15afd94a047 added code to acquire and release the AioContext in
qemuio_command(). This means that the lock is taken twice now in the
call path from hmp_qemu_io(). This causes BDRV_POLL_WHILE() to hang for
any requests issued to nodes in a non-mainloop AioContext.

Dropping the first locking from hmp_qemu_io() fixes the problem.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 hmp.c | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/hmp.c b/hmp.c
index XXXXXXX..XXXXXXX 100644
--- a/hmp.c
+++ b/hmp.c
@@ -XXX,XX +XXX,XX @@ void hmp_qemu_io(Monitor *mon, const QDict *qdict)
 {
     BlockBackend *blk;
     BlockBackend *local_blk = NULL;
-    AioContext *aio_context;
     const char* device = qdict_get_str(qdict, "device");
     const char* command = qdict_get_str(qdict, "command");
     Error *err = NULL;
@@ -XXX,XX +XXX,XX @@ void hmp_qemu_io(Monitor *mon, const QDict *qdict)
         }
     }
 
-    aio_context = blk_get_aio_context(blk);
-    aio_context_acquire(aio_context);
-
     /*
      * Notably absent: Proper permission management. This is sad, but it seems
      * almost impossible to achieve without changing the semantics and thereby
@@ -XXX,XX +XXX,XX @@ void hmp_qemu_io(Monitor *mon, const QDict *qdict)
      */
     qemuio_command(blk, command);
 
-    aio_context_release(aio_context);
-
 fail:
     blk_unref(local_blk);
     hmp_handle_error(mon, &err);
-- 
2.13.6

From: Edgar Kaziakhmedov <edgar.kaziakhmedov@virtuozzo.com>

Since bdrv_co_preadv does all neccessary checks including
reading after the end of the backing file, avoid duplication
of verification before bdrv_co_preadv call.

Signed-off-by: Edgar Kaziakhmedov <edgar.kaziakhmedov@virtuozzo.com>
Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 block/qcow2.h |  3 ---
 block/qcow2.c | 51 ++++++++-------------------------------------------
 2 files changed, 8 insertions(+), 46 deletions(-)

diff --git a/block/qcow2.h b/block/qcow2.h
index XXXXXXX..XXXXXXX 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -XXX,XX +XXX,XX @@ uint32_t offset_to_reftable_index(BDRVQcow2State *s, uint64_t offset)
 }
 
 /* qcow2.c functions */
-int qcow2_backing_read1(BlockDriverState *bs, QEMUIOVector *qiov,
-                  int64_t sector_num, int nb_sectors);
-
 int64_t qcow2_refcount_metadata_size(int64_t clusters, size_t cluster_size,
                                      int refcount_order, bool generous_increase,
                                      uint64_t *refblock_count);
diff --git a/block/qcow2.c b/block/qcow2.c
index XXXXXXX..XXXXXXX 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -XXX,XX +XXX,XX @@ static int64_t coroutine_fn qcow2_co_get_block_status(BlockDriverState *bs,
     return status;
 }
 
-/* handle reading after the end of the backing file */
-int qcow2_backing_read1(BlockDriverState *bs, QEMUIOVector *qiov,
-                        int64_t offset, int bytes)
-{
-    uint64_t bs_size = bs->total_sectors * BDRV_SECTOR_SIZE;
-    int n1;
-
-    if ((offset + bytes) <= bs_size) {
-        return bytes;
-    }
-
-    if (offset >= bs_size) {
-        n1 = 0;
-    } else {
-        n1 = bs_size - offset;
-    }
-
-    qemu_iovec_memset(qiov, n1, 0, bytes - n1);
-
-    return n1;
-}
-
 static coroutine_fn int qcow2_co_preadv(BlockDriverState *bs, uint64_t offset,
                                         uint64_t bytes, QEMUIOVector *qiov,
                                         int flags)
 {
     BDRVQcow2State *s = bs->opaque;
-    int offset_in_cluster, n1;
+    int offset_in_cluster;
     int ret;
     unsigned int cur_bytes; /* number of bytes in current iteration */
     uint64_t cluster_offset = 0;
@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_preadv(BlockDriverState *bs, uint64_t offset,
         case QCOW2_CLUSTER_UNALLOCATED:
 
             if (bs->backing) {
-                /* read from the base image */
-                n1 = qcow2_backing_read1(bs->backing->bs, &hd_qiov,
-                                         offset, cur_bytes);
-                if (n1 > 0) {
-                    QEMUIOVector local_qiov;
-
-                    qemu_iovec_init(&local_qiov, hd_qiov.niov);
-                    qemu_iovec_concat(&local_qiov, &hd_qiov, 0, n1);
-
-                    BLKDBG_EVENT(bs->file, BLKDBG_READ_BACKING_AIO);
-                    qemu_co_mutex_unlock(&s->lock);
-                    ret = bdrv_co_preadv(bs->backing, offset, n1,
-                                         &local_qiov, 0);
-                    qemu_co_mutex_lock(&s->lock);
-
-                    qemu_iovec_destroy(&local_qiov);
-
-                    if (ret < 0) {
-                        goto fail;
-                    }
+                BLKDBG_EVENT(bs->file, BLKDBG_READ_BACKING_AIO);
+                qemu_co_mutex_unlock(&s->lock);
+                ret = bdrv_co_preadv(bs->backing, offset, cur_bytes,
+                                     &hd_qiov, 0);
+                qemu_co_mutex_lock(&s->lock);
+                if (ret < 0) {
+                    goto fail;
                 }
             } else {
                 /* Note: in this case, no need to wait */
-- 
2.13.6

Removing a quorum child node with x-blockdev-change results in a quorum
driver state that cannot be recreated with create options because it
would require a list with gaps. This causes trouble in at least
.bdrv_refresh_filename().

Document this problem so that we won't accidentally mark the command
stable without having addressed it.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Alberto Garcia <berto@igalia.com>
---
 qapi/block-core.json | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index XXXXXXX..XXXXXXX 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -XXX,XX +XXX,XX @@
 # does not support all kinds of operations, all kinds of children, nor
 # all block drivers.
 #
+# FIXME Removing children from a quorum node means introducing gaps in the
+# child indices. This cannot be represented in the 'children' list of
+# BlockdevOptionsQuorum, as returned by .bdrv_refresh_filename().
+#
 # Warning: The data in a new quorum child MUST be consistent with that of
 # the rest of the array.
 #
-- 
2.13.6

From: Doug Gale <doug16k@gmail.com>

Add trace output for commands, errors, and undefined behavior.
Add guest error log output for undefined behavior.
Report invalid undefined accesses to MMIO.
Annotate unlikely error checks with unlikely.

Signed-off-by: Doug Gale <doug16k@gmail.com>
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 hw/block/nvme.c       | 349 ++++++++++++++++++++++++++++++++++++++++++--------
 hw/block/trace-events |  93 ++++++++++++++
 2 files changed, 390 insertions(+), 52 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -XXX,XX +XXX,XX @@
 #include "qapi/visitor.h"
 #include "sysemu/block-backend.h"
 
+#include "qemu/log.h"
+#include "trace.h"
 #include "nvme.h"
 
+#define NVME_GUEST_ERR(trace, fmt, ...) \
+    do { \
+        (trace_##trace)(__VA_ARGS__); \
+        qemu_log_mask(LOG_GUEST_ERROR, #trace \
+            " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
+    } while (0)
+
 static void nvme_process_sq(void *opaque);
 
 static void nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size)
@@ -XXX,XX +XXX,XX @@ static void nvme_isr_notify(NvmeCtrl *n, NvmeCQueue *cq)
 {
     if (cq->irq_enabled) {
         if (msix_enabled(&(n->parent_obj))) {
+            trace_nvme_irq_msix(cq->vector);
             msix_notify(&(n->parent_obj), cq->vector);
         } else {
+            trace_nvme_irq_pin();
             pci_irq_pulse(&n->parent_obj);
         }
+    } else {
+        trace_nvme_irq_masked();
     }
 }
 
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1,
     trans_len = MIN(len, trans_len);
     int num_prps = (len >> n->page_bits) + 1;
 
-    if (!prp1) {
+    if (unlikely(!prp1)) {
+        trace_nvme_err_invalid_prp();
         return NVME_INVALID_FIELD | NVME_DNR;
     } else if (n->cmbsz && prp1 >= n->ctrl_mem.addr &&
                prp1 < n->ctrl_mem.addr + int128_get64(n->ctrl_mem.size)) {
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1,
     }
     len -= trans_len;
     if (len) {
-        if (!prp2) {
+        if (unlikely(!prp2)) {
+            trace_nvme_err_invalid_prp2_missing();
             goto unmap;
         }
         if (len > n->page_size) {
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1,
                 uint64_t prp_ent = le64_to_cpu(prp_list[i]);
 
                 if (i == n->max_prp_ents - 1 && len > n->page_size) {
-                    if (!prp_ent || prp_ent & (n->page_size - 1)) {
+                    if (unlikely(!prp_ent || prp_ent & (n->page_size - 1))) {
+                        trace_nvme_err_invalid_prplist_ent(prp_ent);
                         goto unmap;
                     }
 
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1,
                     prp_ent = le64_to_cpu(prp_list[i]);
                 }
 
-                if (!prp_ent || prp_ent & (n->page_size - 1)) {
+                if (unlikely(!prp_ent || prp_ent & (n->page_size - 1))) {
+                    trace_nvme_err_invalid_prplist_ent(prp_ent);
                     goto unmap;
                 }
 
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1,
                 i++;
             }
         } else {
-            if (prp2 & (n->page_size - 1)) {
+            if (unlikely(prp2 & (n->page_size - 1))) {
+                trace_nvme_err_invalid_prp2_align(prp2);
                 goto unmap;
             }
             if (qsg->nsg) {
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_dma_read_prp(NvmeCtrl *n, uint8_t *ptr, uint32_t len,
     QEMUIOVector iov;
     uint16_t status = NVME_SUCCESS;
 
+    trace_nvme_dma_read(prp1, prp2);
+
     if (nvme_map_prp(&qsg, &iov, prp1, prp2, len, n)) {
         return NVME_INVALID_FIELD | NVME_DNR;
     }
     if (qsg.nsg > 0) {
-        if (dma_buf_read(ptr, len, &qsg)) {
+        if (unlikely(dma_buf_read(ptr, len, &qsg))) {
+            trace_nvme_err_invalid_dma();
             status = NVME_INVALID_FIELD | NVME_DNR;
         }
         qemu_sglist_destroy(&qsg);
     } else {
-        if (qemu_iovec_to_buf(&iov, 0, ptr, len) != len) {
+        if (unlikely(qemu_iovec_to_buf(&iov, 0, ptr, len) != len)) {
+            trace_nvme_err_invalid_dma();
             status = NVME_INVALID_FIELD | NVME_DNR;
         }
         qemu_iovec_destroy(&iov);
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_write_zeros(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
     uint64_t aio_slba = slba << (data_shift - BDRV_SECTOR_BITS);
     uint32_t aio_nlb = nlb << (data_shift - BDRV_SECTOR_BITS);
 
-    if (slba + nlb > ns->id_ns.nsze) {
+    if (unlikely(slba + nlb > ns->id_ns.nsze)) {
+        trace_nvme_err_invalid_lba_range(slba, nlb, ns->id_ns.nsze);
         return NVME_LBA_RANGE | NVME_DNR;
     }
 
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_rw(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
     int is_write = rw->opcode == NVME_CMD_WRITE ? 1 : 0;
     enum BlockAcctType acct = is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ;
 
-    if ((slba + nlb) > ns->id_ns.nsze) {
+    trace_nvme_rw(is_write ? "write" : "read", nlb, data_size, slba);
+
+    if (unlikely((slba + nlb) > ns->id_ns.nsze)) {
         block_acct_invalid(blk_get_stats(n->conf.blk), acct);
+        trace_nvme_err_invalid_lba_range(slba, nlb, ns->id_ns.nsze);
         return NVME_LBA_RANGE | NVME_DNR;
     }
 
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
     NvmeNamespace *ns;
     uint32_t nsid = le32_to_cpu(cmd->nsid);
 
-    if (nsid == 0 || nsid > n->num_namespaces) {
+    if (unlikely(nsid == 0 || nsid > n->num_namespaces)) {
+        trace_nvme_err_invalid_ns(nsid, n->num_namespaces);
         return NVME_INVALID_NSID | NVME_DNR;
     }
 
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
     case NVME_CMD_READ:
         return nvme_rw(n, ns, cmd, req);
     default:
+        trace_nvme_err_invalid_opc(cmd->opcode);
         return NVME_INVALID_OPCODE | NVME_DNR;
     }
 }
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeCmd *cmd)
     NvmeCQueue *cq;
     uint16_t qid = le16_to_cpu(c->qid);
 
-    if (!qid || nvme_check_sqid(n, qid)) {
+    if (unlikely(!qid || nvme_check_sqid(n, qid))) {
+        trace_nvme_err_invalid_del_sq(qid);
         return NVME_INVALID_QID | NVME_DNR;
     }
 
+    trace_nvme_del_sq(qid);
+
     sq = n->sq[qid];
     while (!QTAILQ_EMPTY(&sq->out_req_list)) {
         req = QTAILQ_FIRST(&sq->out_req_list);
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeCmd *cmd)
     uint16_t qflags = le16_to_cpu(c->sq_flags);
     uint64_t prp1 = le64_to_cpu(c->prp1);
 
-    if (!cqid || nvme_check_cqid(n, cqid)) {
+    trace_nvme_create_sq(prp1, sqid, cqid, qsize, qflags);
+
+    if (unlikely(!cqid || nvme_check_cqid(n, cqid))) {
+        trace_nvme_err_invalid_create_sq_cqid(cqid);
         return NVME_INVALID_CQID | NVME_DNR;
     }
-    if (!sqid || !nvme_check_sqid(n, sqid)) {
+    if (unlikely(!sqid || !nvme_check_sqid(n, sqid))) {
+        trace_nvme_err_invalid_create_sq_sqid(sqid);
         return NVME_INVALID_QID | NVME_DNR;
     }
-    if (!qsize || qsize > NVME_CAP_MQES(n->bar.cap)) {
+    if (unlikely(!qsize || qsize > NVME_CAP_MQES(n->bar.cap))) {
+        trace_nvme_err_invalid_create_sq_size(qsize);
         return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
     }
-    if (!prp1 || prp1 & (n->page_size - 1)) {
+    if (unlikely(!prp1 || prp1 & (n->page_size - 1))) {
+        trace_nvme_err_invalid_create_sq_addr(prp1);
         return NVME_INVALID_FIELD | NVME_DNR;
     }
-    if (!(NVME_SQ_FLAGS_PC(qflags))) {
+    if (unlikely(!(NVME_SQ_FLAGS_PC(qflags)))) {
+        trace_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags));
         return NVME_INVALID_FIELD | NVME_DNR;
     }
     sq = g_malloc0(sizeof(*sq));
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeCmd *cmd)
     NvmeCQueue *cq;
     uint16_t qid = le16_to_cpu(c->qid);
 
-    if (!qid || nvme_check_cqid(n, qid)) {
+    if (unlikely(!qid || nvme_check_cqid(n, qid))) {
+        trace_nvme_err_invalid_del_cq_cqid(qid);
         return NVME_INVALID_CQID | NVME_DNR;
     }
 
     cq = n->cq[qid];
-    if (!QTAILQ_EMPTY(&cq->sq_list)) {
+    if (unlikely(!QTAILQ_EMPTY(&cq->sq_list))) {
+        trace_nvme_err_invalid_del_cq_notempty(qid);
         return NVME_INVALID_QUEUE_DEL;
     }
+    trace_nvme_del_cq(qid);
     nvme_free_cq(cq, n);
     return NVME_SUCCESS;
 }
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeCmd *cmd)
     uint16_t qflags = le16_to_cpu(c->cq_flags);
     uint64_t prp1 = le64_to_cpu(c->prp1);
 
-    if (!cqid || !nvme_check_cqid(n, cqid)) {
+    trace_nvme_create_cq(prp1, cqid, vector, qsize, qflags,
+                         NVME_CQ_FLAGS_IEN(qflags) != 0);
+
+    if (unlikely(!cqid || !nvme_check_cqid(n, cqid))) {
+        trace_nvme_err_invalid_create_cq_cqid(cqid);
         return NVME_INVALID_CQID | NVME_DNR;
     }
-    if (!qsize || qsize > NVME_CAP_MQES(n->bar.cap)) {
+    if (unlikely(!qsize || qsize > NVME_CAP_MQES(n->bar.cap))) {
+        trace_nvme_err_invalid_create_cq_size(qsize);
         return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
     }
-    if (!prp1) {
+    if (unlikely(!prp1)) {
+        trace_nvme_err_invalid_create_cq_addr(prp1);
         return NVME_INVALID_FIELD | NVME_DNR;
     }
-    if (vector > n->num_queues) {
+    if (unlikely(vector > n->num_queues)) {
+        trace_nvme_err_invalid_create_cq_vector(vector);
         return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
     }
-    if (!(NVME_CQ_FLAGS_PC(qflags))) {
+    if (unlikely(!(NVME_CQ_FLAGS_PC(qflags)))) {
+        trace_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags));
         return NVME_INVALID_FIELD | NVME_DNR;
     }
 
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeIdentify *c)
     uint64_t prp1 = le64_to_cpu(c->prp1);
     uint64_t prp2 = le64_to_cpu(c->prp2);
 
+    trace_nvme_identify_ctrl();
+
     return nvme_dma_read_prp(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl),
         prp1, prp2);
 }
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeIdentify *c)
     uint64_t prp1 = le64_to_cpu(c->prp1);
     uint64_t prp2 = le64_to_cpu(c->prp2);
 
-    if (nsid == 0 || nsid > n->num_namespaces) {
+    trace_nvme_identify_ns(nsid);
+
+    if (unlikely(nsid == 0 || nsid > n->num_namespaces)) {
+        trace_nvme_err_invalid_ns(nsid, n->num_namespaces);
         return NVME_INVALID_NSID | NVME_DNR;
     }
 
     ns = &n->namespaces[nsid - 1];
+
     return nvme_dma_read_prp(n, (uint8_t *)&ns->id_ns, sizeof(ns->id_ns),
         prp1, prp2);
 }
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeIdentify *c)
     uint16_t ret;
     int i, j = 0;
 
+    trace_nvme_identify_nslist(min_nsid);
+
     list = g_malloc0(data_len);
     for (i = 0; i < n->num_namespaces; i++) {
         if (i < min_nsid) {
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_identify(NvmeCtrl *n, NvmeCmd *cmd)
     case 0x02:
         return nvme_identify_nslist(n, c);
     default:
+        trace_nvme_err_invalid_identify_cns(le32_to_cpu(c->cns));
         return NVME_INVALID_FIELD | NVME_DNR;
     }
 }
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
     switch (dw10) {
     case NVME_VOLATILE_WRITE_CACHE:
         result = blk_enable_write_cache(n->conf.blk);
+        trace_nvme_getfeat_vwcache(result ? "enabled" : "disabled");
         break;
     case NVME_NUMBER_OF_QUEUES:
         result = cpu_to_le32((n->num_queues - 2) | ((n->num_queues - 2) << 16));
+        trace_nvme_getfeat_numq(result);
         break;
     default:
+        trace_nvme_err_invalid_getfeat(dw10);
         return NVME_INVALID_FIELD | NVME_DNR;
     }
 
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
         blk_set_enable_write_cache(n->conf.blk, dw11 & 1);
         break;
     case NVME_NUMBER_OF_QUEUES:
+        trace_nvme_setfeat_numq((dw11 & 0xFFFF) + 1,
+                                ((dw11 >> 16) & 0xFFFF) + 1,
+                                n->num_queues - 1, n->num_queues - 1);
         req->cqe.result =
             cpu_to_le32((n->num_queues - 2) | ((n->num_queues - 2) << 16));
         break;
     default:
+        trace_nvme_err_invalid_setfeat(dw10);
         return NVME_INVALID_FIELD | NVME_DNR;
     }
     return NVME_SUCCESS;
@@ -XXX,XX +XXX,XX @@ static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
     case NVME_ADM_CMD_GET_FEATURES:
         return nvme_get_feature(n, cmd, req);
     default:
+        trace_nvme_err_invalid_admin_opc(cmd->opcode);
         return NVME_INVALID_OPCODE | NVME_DNR;
     }
 }
@@ -XXX,XX +XXX,XX @@ static int nvme_start_ctrl(NvmeCtrl *n)
     uint32_t page_bits = NVME_CC_MPS(n->bar.cc) + 12;
     uint32_t page_size = 1 << page_bits;
 
-    if (n->cq[0] || n->sq[0] || !n->bar.asq || !n->bar.acq ||
-            n->bar.asq & (page_size - 1) || n->bar.acq & (page_size - 1) ||
-            NVME_CC_MPS(n->bar.cc) < NVME_CAP_MPSMIN(n->bar.cap) ||
-            NVME_CC_MPS(n->bar.cc) > NVME_CAP_MPSMAX(n->bar.cap) ||
-            NVME_CC_IOCQES(n->bar.cc) < NVME_CTRL_CQES_MIN(n->id_ctrl.cqes) ||
-            NVME_CC_IOCQES(n->bar.cc) > NVME_CTRL_CQES_MAX(n->id_ctrl.cqes) ||
-            NVME_CC_IOSQES(n->bar.cc) < NVME_CTRL_SQES_MIN(n->id_ctrl.sqes) ||
-            NVME_CC_IOSQES(n->bar.cc) > NVME_CTRL_SQES_MAX(n->id_ctrl.sqes) ||
-            !NVME_AQA_ASQS(n->bar.aqa) || !NVME_AQA_ACQS(n->bar.aqa)) {
+    if (unlikely(n->cq[0])) {
+        trace_nvme_err_startfail_cq();
+        return -1;
+    }
+    if (unlikely(n->sq[0])) {
+        trace_nvme_err_startfail_sq();
+        return -1;
+    }
+    if (unlikely(!n->bar.asq)) {
+        trace_nvme_err_startfail_nbarasq();
+        return -1;
+    }
+    if (unlikely(!n->bar.acq)) {
+        trace_nvme_err_startfail_nbaracq();
+        return -1;
+    }
+    if (unlikely(n->bar.asq & (page_size - 1))) {
+        trace_nvme_err_startfail_asq_misaligned(n->bar.asq);
+        return -1;
+    }
+    if (unlikely(n->bar.acq & (page_size - 1))) {
+        trace_nvme_err_startfail_acq_misaligned(n->bar.acq);
+        return -1;
+    }
+    if (unlikely(NVME_CC_MPS(n->bar.cc) <
+                 NVME_CAP_MPSMIN(n->bar.cap))) {
+        trace_nvme_err_startfail_page_too_small(
+                    NVME_CC_MPS(n->bar.cc),
+                    NVME_CAP_MPSMIN(n->bar.cap));
+        return -1;
+    }
+    if (unlikely(NVME_CC_MPS(n->bar.cc) >
+                 NVME_CAP_MPSMAX(n->bar.cap))) {
+        trace_nvme_err_startfail_page_too_large(
+                    NVME_CC_MPS(n->bar.cc),
+                    NVME_CAP_MPSMAX(n->bar.cap));
+        return -1;
+    }
+    if (unlikely(NVME_CC_IOCQES(n->bar.cc) <
+                 NVME_CTRL_CQES_MIN(n->id_ctrl.cqes))) {
+        trace_nvme_err_startfail_cqent_too_small(
+                    NVME_CC_IOCQES(n->bar.cc),
+                    NVME_CTRL_CQES_MIN(n->bar.cap));
+        return -1;
+    }
+    if (unlikely(NVME_CC_IOCQES(n->bar.cc) >
+                 NVME_CTRL_CQES_MAX(n->id_ctrl.cqes))) {
+        trace_nvme_err_startfail_cqent_too_large(
+                    NVME_CC_IOCQES(n->bar.cc),
+                    NVME_CTRL_CQES_MAX(n->bar.cap));
+        return -1;
+    }
+    if (unlikely(NVME_CC_IOSQES(n->bar.cc) <
+                 NVME_CTRL_SQES_MIN(n->id_ctrl.sqes))) {
+        trace_nvme_err_startfail_sqent_too_small(
+                    NVME_CC_IOSQES(n->bar.cc),
+                    NVME_CTRL_SQES_MIN(n->bar.cap));
+        return -1;
+    }
+    if (unlikely(NVME_CC_IOSQES(n->bar.cc) >
+                 NVME_CTRL_SQES_MAX(n->id_ctrl.sqes))) {
+        trace_nvme_err_startfail_sqent_too_large(
+                    NVME_CC_IOSQES(n->bar.cc),
+                    NVME_CTRL_SQES_MAX(n->bar.cap));
+        return -1;
+    }
+    if (unlikely(!NVME_AQA_ASQS(n->bar.aqa))) {
+        trace_nvme_err_startfail_asqent_sz_zero();
+        return -1;
+    }
+    if (unlikely(!NVME_AQA_ACQS(n->bar.aqa))) {
+        trace_nvme_err_startfail_acqent_sz_zero();
         return -1;
     }
 
@@ -XXX,XX +XXX,XX @@ static int nvme_start_ctrl(NvmeCtrl *n)
 static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
     unsigned size)
 {
+    if (unlikely(offset & (sizeof(uint32_t) - 1))) {
+        NVME_GUEST_ERR(nvme_ub_mmiowr_misaligned32,
+                       "MMIO write not 32-bit aligned,"
+                       " offset=0x%"PRIx64"", offset);
+        /* should be ignored, fall through for now */
+    }
+
+    if (unlikely(size < sizeof(uint32_t))) {
+        NVME_GUEST_ERR(nvme_ub_mmiowr_toosmall,
+                       "MMIO write smaller than 32-bits,"
+                       " offset=0x%"PRIx64", size=%u",
+                       offset, size);
+        /* should be ignored, fall through for now */
+    }
+
     switch (offset) {
-    case 0xc:
+    case 0xc:   /* INTMS */
+        if (unlikely(msix_enabled(&(n->parent_obj)))) {
+            NVME_GUEST_ERR(nvme_ub_mmiowr_intmask_with_msix,
+                           "undefined access to interrupt mask set"
+                           " when MSI-X is enabled");
+            /* should be ignored, fall through for now */
+        }
         n->bar.intms |= data & 0xffffffff;
         n->bar.intmc = n->bar.intms;
+        trace_nvme_mmio_intm_set(data & 0xffffffff,
+                                 n->bar.intmc);
         break;
-    case 0x10:
+    case 0x10:  /* INTMC */
+        if (unlikely(msix_enabled(&(n->parent_obj)))) {
+            NVME_GUEST_ERR(nvme_ub_mmiowr_intmask_with_msix,
+                           "undefined access to interrupt mask clr"
+                           " when MSI-X is enabled");
+            /* should be ignored, fall through for now */
+        }
         n->bar.intms &= ~(data & 0xffffffff);
         n->bar.intmc = n->bar.intms;
+        trace_nvme_mmio_intm_clr(data & 0xffffffff,
+                                 n->bar.intmc);
         break;
-    case 0x14:
+    case 0x14:  /* CC */
+        trace_nvme_mmio_cfg(data & 0xffffffff);
         /* Windows first sends data, then sends enable bit */
         if (!NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc) &&
             !NVME_CC_SHN(data) && !NVME_CC_SHN(n->bar.cc))
@@ -XXX,XX +XXX,XX @@ static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
 
         if (NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc)) {
             n->bar.cc = data;
-            if (nvme_start_ctrl(n)) {
+            if (unlikely(nvme_start_ctrl(n))) {
+                trace_nvme_err_startfail();
                 n->bar.csts = NVME_CSTS_FAILED;
             } else {
+                trace_nvme_mmio_start_success();
                 n->bar.csts = NVME_CSTS_READY;
             }
         } else if (!NVME_CC_EN(data) && NVME_CC_EN(n->bar.cc)) {
+            trace_nvme_mmio_stopped();
             nvme_clear_ctrl(n);
             n->bar.csts &= ~NVME_CSTS_READY;
         }
         if (NVME_CC_SHN(data) && !(NVME_CC_SHN(n->bar.cc))) {
-                nvme_clear_ctrl(n);
-                n->bar.cc = data;
-                n->bar.csts |= NVME_CSTS_SHST_COMPLETE;
+            trace_nvme_mmio_shutdown_set();
+            nvme_clear_ctrl(n);
+            n->bar.cc = data;
+            n->bar.csts |= NVME_CSTS_SHST_COMPLETE;
         } else if (!NVME_CC_SHN(data) && NVME_CC_SHN(n->bar.cc)) {
-                n->bar.csts &= ~NVME_CSTS_SHST_COMPLETE;
-                n->bar.cc = data;
+            trace_nvme_mmio_shutdown_cleared();
+            n->bar.csts &= ~NVME_CSTS_SHST_COMPLETE;
+            n->bar.cc = data;
+        }
+        break;
+    case 0x1C:  /* CSTS */
+        if (data & (1 << 4)) {
+            NVME_GUEST_ERR(nvme_ub_mmiowr_ssreset_w1c_unsupported,
+                           "attempted to W1C CSTS.NSSRO"
+                           " but CAP.NSSRS is zero (not supported)");
+        } else if (data != 0) {
+            NVME_GUEST_ERR(nvme_ub_mmiowr_ro_csts,
+                           "attempted to set a read only bit"
+                           " of controller status");
+        }
+        break;
+    case 0x20:  /* NSSR */
+        if (data == 0x4E564D65) {
+            trace_nvme_ub_mmiowr_ssreset_unsupported();
+        } else {
+            /* The spec says that writes of other values have no effect */
+            return;
         }
         break;
-    case 0x24:
+    case 0x24:  /* AQA */
         n->bar.aqa = data & 0xffffffff;
+        trace_nvme_mmio_aqattr(data & 0xffffffff);
         break;
-    case 0x28:
+    case 0x28:  /* ASQ */
         n->bar.asq = data;
+        trace_nvme_mmio_asqaddr(data);
         break;
-    case 0x2c:
+    case 0x2c:  /* ASQ hi */
         n->bar.asq |= data << 32;
+        trace_nvme_mmio_asqaddr_hi(data, n->bar.asq);
         break;
-    case 0x30:
+    case 0x30:  /* ACQ */
+        trace_nvme_mmio_acqaddr(data);
         n->bar.acq = data;
         break;
-    case 0x34:
+    case 0x34:  /* ACQ hi */
         n->bar.acq |= data << 32;
+        trace_nvme_mmio_acqaddr_hi(data, n->bar.acq);
         break;
+    case 0x38:  /* CMBLOC */
+        NVME_GUEST_ERR(nvme_ub_mmiowr_cmbloc_reserved,
+                       "invalid write to reserved CMBLOC"
+                       " when CMBSZ is zero, ignored");
+        return;
+    case 0x3C:  /* CMBSZ */
+        NVME_GUEST_ERR(nvme_ub_mmiowr_cmbsz_readonly,
+                       "invalid write to read only CMBSZ, ignored");
+        return;
     default:
+        NVME_GUEST_ERR(nvme_ub_mmiowr_invalid,
+                       "invalid MMIO write,"
+                       " offset=0x%"PRIx64", data=%"PRIx64"",
+                       offset, data);
         break;
     }
 }
@@ -XXX,XX +XXX,XX @@ static uint64_t nvme_mmio_read(void *opaque, hwaddr addr, unsigned size)
     uint8_t *ptr = (uint8_t *)&n->bar;
     uint64_t val = 0;
 
+    if (unlikely(addr & (sizeof(uint32_t) - 1))) {
+        NVME_GUEST_ERR(nvme_ub_mmiord_misaligned32,
+                       "MMIO read not 32-bit aligned,"
+                       " offset=0x%"PRIx64"", addr);
+        /* should RAZ, fall through for now */
+    } else if (unlikely(size < sizeof(uint32_t))) {
+        NVME_GUEST_ERR(nvme_ub_mmiord_toosmall,
+                       "MMIO read smaller than 32-bits,"
+                       " offset=0x%"PRIx64"", addr);
+        /* should RAZ, fall through for now */
+    }
+
     if (addr < sizeof(n->bar)) {
         memcpy(&val, ptr + addr, size);
+    } else {
+        NVME_GUEST_ERR(nvme_ub_mmiord_invalid_ofs,
+                       "MMIO read beyond last register,"
+                       " offset=0x%"PRIx64", returning 0", addr);
     }
+
     return val;
 }
 
@@ -XXX,XX +XXX,XX @@ static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
 {
     uint32_t qid;
 
-    if (addr & ((1 << 2) - 1)) {
+    if (unlikely(addr & ((1 << 2) - 1))) {
+        NVME_GUEST_ERR(nvme_ub_db_wr_misaligned,
+                       "doorbell write not 32-bit aligned,"
+                       " offset=0x%"PRIx64", ignoring", addr);
         return;
     }
 
     if (((addr - 0x1000) >> 2) & 1) {
+        /* Completion queue doorbell write */
+
         uint16_t new_head = val & 0xffff;
         int start_sqs;
         NvmeCQueue *cq;
 
         qid = (addr - (0x1000 + (1 << 2))) >> 3;
-        if (nvme_check_cqid(n, qid)) {
+        if (unlikely(nvme_check_cqid(n, qid))) {
+            NVME_GUEST_ERR(nvme_ub_db_wr_invalid_cq,
+                           "completion queue doorbell write"
+                           " for nonexistent queue,"
+                           " sqid=%"PRIu32", ignoring", qid);
             return;
         }
 
         cq = n->cq[qid];
-        if (new_head >= cq->size) {
+        if (unlikely(new_head >= cq->size)) {
+            NVME_GUEST_ERR(nvme_ub_db_wr_invalid_cqhead,
+                           "completion queue doorbell write value"
+                           " beyond queue size, sqid=%"PRIu32","
+                           " new_head=%"PRIu16", ignoring",
+                           qid, new_head);
             return;
         }
 
@@ -XXX,XX +XXX,XX @@ static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
             nvme_isr_notify(n, cq);
         }
     } else {
+        /* Submission queue doorbell write */
+
         uint16_t new_tail = val & 0xffff;
         NvmeSQueue *sq;
 
         qid = (addr - 0x1000) >> 3;
-        if (nvme_check_sqid(n, qid)) {
+        if (unlikely(nvme_check_sqid(n, qid))) {
+            NVME_GUEST_ERR(nvme_ub_db_wr_invalid_sq,
+                           "submission queue doorbell write"
+                           " for nonexistent queue,"
+                           " sqid=%"PRIu32", ignoring", qid);
             return;
         }
 
         sq = n->sq[qid];
-        if (new_tail >= sq->size) {
+        if (unlikely(new_tail >= sq->size)) {
+            NVME_GUEST_ERR(nvme_ub_db_wr_invalid_sqtail,
+                           "submission queue doorbell write value"
+                           " beyond queue size, sqid=%"PRIu32","
+                           " new_tail=%"PRIu16", ignoring",
+                           qid, new_tail);
             return;
         }
 
diff --git a/hw/block/trace-events b/hw/block/trace-events
index XXXXXXX..XXXXXXX 100644
--- a/hw/block/trace-events
+++ b/hw/block/trace-events
@@ -XXX,XX +XXX,XX @@ virtio_blk_submit_multireq(void *vdev, void *mrb, int start, int num_reqs, uint6
 hd_geometry_lchs_guess(void *blk, int cyls, int heads, int secs) "blk %p LCHS %d %d %d"
 hd_geometry_guess(void *blk, uint32_t cyls, uint32_t heads, uint32_t secs, int trans) "blk %p CHS %u %u %u trans %d"
 
+# hw/block/nvme.c
+# nvme traces for successful events
+nvme_irq_msix(uint32_t vector) "raising MSI-X IRQ vector %u"
+nvme_irq_pin(void) "pulsing IRQ pin"
+nvme_irq_masked(void) "IRQ is masked"
+nvme_dma_read(uint64_t prp1, uint64_t prp2) "DMA read, prp1=0x%"PRIx64" prp2=0x%"PRIx64""
+nvme_rw(char const *verb, uint32_t blk_count, uint64_t byte_count, uint64_t lba) "%s %"PRIu32" blocks (%"PRIu64" bytes) from LBA %"PRIu64""
+nvme_create_sq(uint64_t addr, uint16_t sqid, uint16_t cqid, uint16_t qsize, uint16_t qflags) "create submission queue, addr=0x%"PRIx64", sqid=%"PRIu16", cqid=%"PRIu16", qsize=%"PRIu16", qflags=%"PRIu16""
+nvme_create_cq(uint64_t addr, uint16_t cqid, uint16_t vector, uint16_t size, uint16_t qflags, int ien) "create completion queue, addr=0x%"PRIx64", cqid=%"PRIu16", vector=%"PRIu16", qsize=%"PRIu16", qflags=%"PRIu16", ien=%d"
+nvme_del_sq(uint16_t qid) "deleting submission queue sqid=%"PRIu16""
+nvme_del_cq(uint16_t cqid) "deleted completion queue, sqid=%"PRIu16""
+nvme_identify_ctrl(void) "identify controller"
+nvme_identify_ns(uint16_t ns) "identify namespace, nsid=%"PRIu16""
+nvme_identify_nslist(uint16_t ns) "identify namespace list, nsid=%"PRIu16""
+nvme_getfeat_vwcache(char const* result) "get feature volatile write cache, result=%s"
+nvme_getfeat_numq(int result) "get feature number of queues, result=%d"
+nvme_setfeat_numq(int reqcq, int reqsq, int gotcq, int gotsq) "requested cq_count=%d sq_count=%d, responding with cq_count=%d sq_count=%d"
+nvme_mmio_intm_set(uint64_t data, uint64_t new_mask) "wrote MMIO, interrupt mask set, data=0x%"PRIx64", new_mask=0x%"PRIx64""
+nvme_mmio_intm_clr(uint64_t data, uint64_t new_mask) "wrote MMIO, interrupt mask clr, data=0x%"PRIx64", new_mask=0x%"PRIx64""
+nvme_mmio_cfg(uint64_t data) "wrote MMIO, config controller config=0x%"PRIx64""
+nvme_mmio_aqattr(uint64_t data) "wrote MMIO, admin queue attributes=0x%"PRIx64""
+nvme_mmio_asqaddr(uint64_t data) "wrote MMIO, admin submission queue address=0x%"PRIx64""
+nvme_mmio_acqaddr(uint64_t data) "wrote MMIO, admin completion queue address=0x%"PRIx64""
+nvme_mmio_asqaddr_hi(uint64_t data, uint64_t new_addr) "wrote MMIO, admin submission queue high half=0x%"PRIx64", new_address=0x%"PRIx64""
+nvme_mmio_acqaddr_hi(uint64_t data, uint64_t new_addr) "wrote MMIO, admin completion queue high half=0x%"PRIx64", new_address=0x%"PRIx64""
+nvme_mmio_start_success(void) "setting controller enable bit succeeded"
+nvme_mmio_stopped(void) "cleared controller enable bit"
+nvme_mmio_shutdown_set(void) "shutdown bit set"
+nvme_mmio_shutdown_cleared(void) "shutdown bit cleared"
+
+# nvme traces for error conditions
+nvme_err_invalid_dma(void) "PRP/SGL is too small for transfer size"
+nvme_err_invalid_prplist_ent(uint64_t prplist) "PRP list entry is null or not page aligned: 0x%"PRIx64""
+nvme_err_invalid_prp2_align(uint64_t prp2) "PRP2 is not page aligned: 0x%"PRIx64""
+nvme_err_invalid_prp2_missing(void) "PRP2 is null and more data to be transferred"
+nvme_err_invalid_field(void) "invalid field"
+nvme_err_invalid_prp(void) "invalid PRP"
+nvme_err_invalid_sgl(void) "invalid SGL"
+nvme_err_invalid_ns(uint32_t ns, uint32_t limit) "invalid namespace %u not within 1-%u"
+nvme_err_invalid_opc(uint8_t opc) "invalid opcode 0x%"PRIx8""
+nvme_err_invalid_admin_opc(uint8_t opc) "invalid admin opcode 0x%"PRIx8""
+nvme_err_invalid_lba_range(uint64_t start, uint64_t len, uint64_t limit) "Invalid LBA start=%"PRIu64" len=%"PRIu64" limit=%"PRIu64""
+nvme_err_invalid_del_sq(uint16_t qid) "invalid submission queue deletion, sid=%"PRIu16""
+nvme_err_invalid_create_sq_cqid(uint16_t cqid) "failed creating submission queue, invalid cqid=%"PRIu16""
+nvme_err_invalid_create_sq_sqid(uint16_t sqid) "failed creating submission queue, invalid sqid=%"PRIu16""
+nvme_err_invalid_create_sq_size(uint16_t qsize) "failed creating submission queue, invalid qsize=%"PRIu16""
+nvme_err_invalid_create_sq_addr(uint64_t addr) "failed creating submission queue, addr=0x%"PRIx64""
+nvme_err_invalid_create_sq_qflags(uint16_t qflags) "failed creating submission queue, qflags=%"PRIu16""
+nvme_err_invalid_del_cq_cqid(uint16_t cqid) "failed deleting completion queue, cqid=%"PRIu16""
+nvme_err_invalid_del_cq_notempty(uint16_t cqid) "failed deleting completion queue, it is not empty, cqid=%"PRIu16""
+nvme_err_invalid_create_cq_cqid(uint16_t cqid) "failed creating completion queue, cqid=%"PRIu16""
+nvme_err_invalid_create_cq_size(uint16_t size) "failed creating completion queue, size=%"PRIu16""
+nvme_err_invalid_create_cq_addr(uint64_t addr) "failed creating completion queue, addr=0x%"PRIx64""
+nvme_err_invalid_create_cq_vector(uint16_t vector) "failed creating completion queue, vector=%"PRIu16""
+nvme_err_invalid_create_cq_qflags(uint16_t qflags) "failed creating completion queue, qflags=%"PRIu16""
+nvme_err_invalid_identify_cns(uint16_t cns) "identify, invalid cns=0x%"PRIx16""
+nvme_err_invalid_getfeat(int dw10) "invalid get features, dw10=0x%"PRIx32""
+nvme_err_invalid_setfeat(uint32_t dw10) "invalid set features, dw10=0x%"PRIx32""
+nvme_err_startfail_cq(void) "nvme_start_ctrl failed because there are non-admin completion queues"
+nvme_err_startfail_sq(void) "nvme_start_ctrl failed because there are non-admin submission queues"
+nvme_err_startfail_nbarasq(void) "nvme_start_ctrl failed because the admin submission queue address is null"
+nvme_err_startfail_nbaracq(void) "nvme_start_ctrl failed because the admin completion queue address is null"
+nvme_err_startfail_asq_misaligned(uint64_t addr) "nvme_start_ctrl failed because the admin submission queue address is misaligned: 0x%"PRIx64""
+nvme_err_startfail_acq_misaligned(uint64_t addr) "nvme_start_ctrl failed because the admin completion queue address is misaligned: 0x%"PRIx64""
+nvme_err_startfail_page_too_small(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the page size is too small: log2size=%u, min=%u"
+nvme_err_startfail_page_too_large(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the page size is too large: log2size=%u, max=%u"
+nvme_err_startfail_cqent_too_small(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the completion queue entry size is too small: log2size=%u, min=%u"
+nvme_err_startfail_cqent_too_large(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the completion queue entry size is too large: log2size=%u, max=%u"
+nvme_err_startfail_sqent_too_small(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the submission queue entry size is too small: log2size=%u, min=%u"
+nvme_err_startfail_sqent_too_large(uint8_t log2ps, uint8_t maxlog2ps) "nvme_start_ctrl failed because the submission queue entry size is too large: log2size=%u, max=%u"
+nvme_err_startfail_asqent_sz_zero(void) "nvme_start_ctrl failed because the admin submission queue size is zero"
+nvme_err_startfail_acqent_sz_zero(void) "nvme_start_ctrl failed because the admin completion queue size is zero"
+nvme_err_startfail(void) "setting controller enable bit failed"
+
+# Traces for undefined behavior
+nvme_ub_mmiowr_misaligned32(uint64_t offset) "MMIO write not 32-bit aligned, offset=0x%"PRIx64""
+nvme_ub_mmiowr_toosmall(uint64_t offset, unsigned size) "MMIO write smaller than 32 bits, offset=0x%"PRIx64", size=%u"
+nvme_ub_mmiowr_intmask_with_msix(void) "undefined access to interrupt mask set when MSI-X is enabled"
+nvme_ub_mmiowr_ro_csts(void) "attempted to set a read only bit of controller status"
+nvme_ub_mmiowr_ssreset_w1c_unsupported(void) "attempted to W1C CSTS.NSSRO but CAP.NSSRS is zero (not supported)"
+nvme_ub_mmiowr_ssreset_unsupported(void) "attempted NVM subsystem reset but CAP.NSSRS is zero (not supported)"
+nvme_ub_mmiowr_cmbloc_reserved(void) "invalid write to reserved CMBLOC when CMBSZ is zero, ignored"
+nvme_ub_mmiowr_cmbsz_readonly(void) "invalid write to read only CMBSZ, ignored"
+nvme_ub_mmiowr_invalid(uint64_t offset, uint64_t data) "invalid MMIO write, offset=0x%"PRIx64", data=0x%"PRIx64""
+nvme_ub_mmiord_misaligned32(uint64_t offset) "MMIO read not 32-bit aligned, offset=0x%"PRIx64""
+nvme_ub_mmiord_toosmall(uint64_t offset) "MMIO read smaller than 32-bits, offset=0x%"PRIx64""
+nvme_ub_mmiord_invalid_ofs(uint64_t offset) "MMIO read beyond last register, offset=0x%"PRIx64", returning 0"
+nvme_ub_db_wr_misaligned(uint64_t offset) "doorbell write not 32-bit aligned, offset=0x%"PRIx64", ignoring"
+nvme_ub_db_wr_invalid_cq(uint32_t qid) "completion queue doorbell write for nonexistent queue, cqid=%"PRIu32", ignoring"
+nvme_ub_db_wr_invalid_cqhead(uint32_t qid, uint16_t new_head) "completion queue doorbell write value beyond queue size, cqid=%"PRIu32", new_head=%"PRIu16", ignoring"
+nvme_ub_db_wr_invalid_sq(uint32_t qid) "submission queue doorbell write for nonexistent queue, sqid=%"PRIu32", ignoring"
+nvme_ub_db_wr_invalid_sqtail(uint32_t qid, uint16_t new_tail) "submission queue doorbell write value beyond queue size, sqid=%"PRIu32", new_head=%"PRIu16", ignoring"
+
 # hw/block/xen_disk.c
 xen_disk_alloc(char *name) "%s"
 xen_disk_init(char *name) "%s"
-- 
2.13.6

From: Fam Zheng <famz@redhat.com>

Management tools create overlays of running guests with qemu-img:

$ qemu-img create -b /image/in/use.qcow2 -f qcow2 /overlay/image.qcow2

but this doesn't work anymore due to image locking:

qemu-img: /overlay/image.qcow2: Failed to get shared "write" lock
    Is another process using the image?
    Could not open backing image to determine size.
Use the force share option to allow this use case again.

Cc: qemu-stable@nongnu.org
Signed-off-by: Fam Zheng <famz@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 block.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/block.c b/block.c
index XXXXXXX..XXXXXXX 100644
--- a/block.c
+++ b/block.c
@@ -XXX,XX +XXX,XX @@ void bdrv_img_create(const char *filename, const char *fmt,
         back_flags = flags;
         back_flags &= ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
 
+        backing_options = qdict_new();
         if (backing_fmt) {
-            backing_options = qdict_new();
             qdict_put_str(backing_options, "driver", backing_fmt);
         }
+        qdict_put_bool(backing_options, BDRV_OPT_FORCE_SHARE, true);
 
         bs = bdrv_open(full_backing, NULL, backing_options, back_flags,
                        &local_err);
-- 
2.13.6

From: Thomas Huth <thuth@redhat.com>

It's not working anymore since QEMU v1.3.0 - time to remove it now.

Signed-off-by: Thomas Huth <thuth@redhat.com>
Reviewed-by: John Snow <jsnow@redhat.com>
Reviewed-by: Markus Armbruster <armbru@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 blockdev.c    | 11 -----------
 qemu-doc.texi |  6 ------
 2 files changed, 17 deletions(-)

diff --git a/blockdev.c b/blockdev.c
index XXXXXXX..XXXXXXX 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -XXX,XX +XXX,XX @@ QemuOptsList qemu_legacy_drive_opts = {
             .type = QEMU_OPT_STRING,
             .help = "chs translation (auto, lba, none)",
         },{
-            .name = "boot",
-            .type = QEMU_OPT_BOOL,
-            .help = "(deprecated, ignored)",
-        },{
             .name = "addr",
             .type = QEMU_OPT_STRING,
             .help = "pci address (virtio only)",
@@ -XXX,XX +XXX,XX @@ DriveInfo *drive_new(QemuOpts *all_opts, BlockInterfaceType block_default_type)
         goto fail;
     }
 
-    /* Deprecated option boot=[on|off] */
-    if (qemu_opt_get(legacy_opts, "boot") != NULL) {
-        fprintf(stderr, "qemu-kvm: boot=on|off is deprecated and will be "
-                "ignored. Future versions will reject this parameter. Please "
-                "update your scripts.\n");
-    }
-
     /* Other deprecated options */
     if (!qtest_enabled()) {
         for (i = 0; i < ARRAY_SIZE(deprecated); i++) {
diff --git a/qemu-doc.texi b/qemu-doc.texi
index XXXXXXX..XXXXXXX 100644
--- a/qemu-doc.texi
+++ b/qemu-doc.texi
@@ -XXX,XX +XXX,XX @@ deprecated.
 
 @section System emulator command line arguments
 
-@subsection -drive boot=on|off (since 1.3.0)
-
-The ``boot=on|off'' option to the ``-drive'' argument is
-ignored. Applications should use the ``bootindex=N'' parameter
-to set an absolute ordering between devices instead.
-
 @subsection -tdf (since 1.3.0)
 
 The ``-tdf'' argument is ignored. The behaviour implemented
-- 
2.13.6

From: Thomas Huth <thuth@redhat.com>

It's been marked as deprecated since QEMU v2.10.0, and so far nobody
complained that we should keep it, so let's remove this legacy option
now to simplify the code quite a bit.

Signed-off-by: Thomas Huth <thuth@redhat.com>
Reviewed-by: John Snow <jsnow@redhat.com>
Reviewed-by: Markus Armbruster <armbru@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 vl.c            | 86 ++-------------------------------------------------------
 qemu-doc.texi   |  8 ------
 qemu-options.hx | 19 ++-----------
 3 files changed, 4 insertions(+), 109 deletions(-)

diff --git a/vl.c b/vl.c
index XXXXXXX..XXXXXXX 100644
--- a/vl.c
+++ b/vl.c
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
     const char *boot_order = NULL;
     const char *boot_once = NULL;
     DisplayState *ds;
-    int cyls, heads, secs, translation;
     QemuOpts *opts, *machine_opts;
-    QemuOpts *hda_opts = NULL, *icount_opts = NULL, *accel_opts = NULL;
+    QemuOpts *icount_opts = NULL, *accel_opts = NULL;
     QemuOptsList *olist;
     int optind;
     const char *optarg;
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
 
     cpu_model = NULL;
     snapshot = 0;
-    cyls = heads = secs = 0;
-    translation = BIOS_ATA_TRANSLATION_AUTO;
 
     nb_nics = 0;
 
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
         if (optind >= argc)
             break;
         if (argv[optind][0] != '-') {
-            hda_opts = drive_add(IF_DEFAULT, 0, argv[optind++], HD_OPTS);
+            drive_add(IF_DEFAULT, 0, argv[optind++], HD_OPTS);
         } else {
             const QEMUOption *popt;
 
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
                 cpu_model = optarg;
                 break;
             case QEMU_OPTION_hda:
-                {
-                    char buf[256];
-                    if (cyls == 0)
-                        snprintf(buf, sizeof(buf), "%s", HD_OPTS);
-                    else
-                        snprintf(buf, sizeof(buf),
-                                 "%s,cyls=%d,heads=%d,secs=%d%s",
-                                 HD_OPTS , cyls, heads, secs,
-                                 translation == BIOS_ATA_TRANSLATION_LBA ?
-                                 ",trans=lba" :
-                                 translation == BIOS_ATA_TRANSLATION_NONE ?
-                                 ",trans=none" : "");
-                    drive_add(IF_DEFAULT, 0, optarg, buf);
-                    break;
-                }
             case QEMU_OPTION_hdb:
             case QEMU_OPTION_hdc:
             case QEMU_OPTION_hdd:
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv, char **envp)
             case QEMU_OPTION_snapshot:
                 snapshot = 1;
                 break;
-            case QEMU_OPTION_hdachs:
-                {
-                    const char *p;
-                    p = optarg;
-                    cyls = strtol(p, (char **)&p, 0);
-                    if (cyls < 1 || cyls > 16383)
-                        goto chs_fail;
-                    if (*p != ',')
-                        goto chs_fail;
-                    p++;
-                    heads = strtol(p, (char **)&p, 0);
-                    if (heads < 1 || heads > 16)
-                        goto chs_fail;
-                    if (*p != ',')
-                        goto chs_fail;
-                    p++;
-                    secs = strtol(p, (char **)&p, 0);
-                    if (secs < 1 || secs > 63)
-                        goto chs_fail;
-                    if (*p == ',') {
-                        p++;
-                        if (!strcmp(p, "large")) {
-                            translation = BIOS_ATA_TRANSLATION_LARGE;
-                        } else if (!strcmp(p, "rechs")) {
-                            translation = BIOS_ATA_TRANSLATION_RECHS;
-                        } else if (!strcmp(p, "none")) {
-                            translation = BIOS_ATA_TRANSLATION_NONE;
-                        } else if (!strcmp(p, "lba")) {
-                            translation = BIOS_ATA_TRANSLATION_LBA;
-                        } else if (!strcmp(p, "auto")) {
-                            translation = BIOS_ATA_TRANSLATION_AUTO;
-                        } else {
-                            goto chs_fail;
-                        }
-                    } else if (*p != '\0') {
-                    chs_fail:
-                        error_report("invalid physical CHS format");
-                        exit(1);
-                    }
-                    if (hda_opts != NULL) {
-                        qemu_opt_set_number(hda_opts, "cyls", cyls,
-                                            &error_abort);
-                        qemu_opt_set_number(hda_opts, "heads", heads,
-                                            &error_abort);
-                        qemu_opt_set_number(hda_opts, "secs", secs,
-                                            &error_abort);
-                        if (translation == BIOS_ATA_TRANSLATION_LARGE) {
-                            qemu_opt_set(hda_opts, "trans", "large",
-                                         &error_abort);
-                        } else if (translation == BIOS_ATA_TRANSLATION_RECHS) {
-                            qemu_opt_set(hda_opts, "trans", "rechs",
-                                         &error_abort);
-                        } else if (translation == BIOS_ATA_TRANSLATION_LBA) {
-                            qemu_opt_set(hda_opts, "trans", "lba",
-                                         &error_abort);
-                        } else if (translation == BIOS_ATA_TRANSLATION_NONE) {
-                            qemu_opt_set(hda_opts, "trans", "none",
-                                         &error_abort);
-                        }
-                    }
-                }
-                error_report("'-hdachs' is deprecated, please use '-device"
-                             " ide-hd,cyls=c,heads=h,secs=s,...' instead");
-                break;
             case QEMU_OPTION_numa:
                 opts = qemu_opts_parse_noisily(qemu_find_opts("numa"),
                                                optarg, true);
diff --git a/qemu-doc.texi b/qemu-doc.texi
index XXXXXXX..XXXXXXX 100644
--- a/qemu-doc.texi
+++ b/qemu-doc.texi
@@ -XXX,XX +XXX,XX @@ The ``--net dump'' argument is now replaced with the
 ``-object filter-dump'' argument which works in combination
 with the modern ``-netdev`` backends instead.
 
-@subsection -hdachs (since 2.10.0)
-
-The ``-hdachs'' argument is now a synonym for setting
-the ``cyls'', ``heads'', ``secs'', and ``trans'' properties
-on the ``ide-hd'' device using the ``-device'' argument.
-The new syntax allows different settings to be provided
-per disk.
-
 @subsection -usbdevice (since 2.10.0)
 
 The ``-usbdevice DEV'' argument is now a synonym for setting
diff --git a/qemu-options.hx b/qemu-options.hx
index XXXXXXX..XXXXXXX 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -XXX,XX +XXX,XX @@ of available connectors of a given interface type.
 @item media=@var{media}
 This option defines the type of the media: disk or cdrom.
 @item cyls=@var{c},heads=@var{h},secs=@var{s}[,trans=@var{t}]
-These options have the same definition as they have in @option{-hdachs}.
-These parameters are deprecated, use the corresponding parameters
+Force disk physical geometry and the optional BIOS translation (trans=none or
+lba). These parameters are deprecated, use the corresponding parameters
 of @code{-device} instead.
 @item snapshot=@var{snapshot}
 @var{snapshot} is "on" or "off" and controls snapshot mode for the given drive
@@ -XXX,XX +XXX,XX @@ the raw disk image you use is not written back. You can however force
 the write back by pressing @key{C-a s} (@pxref{disk_images}).
 ETEXI
 
-DEF("hdachs", HAS_ARG, QEMU_OPTION_hdachs, \
-    "-hdachs c,h,s[,t]\n" \
-    "                force hard disk 0 physical geometry and the optional BIOS\n" \
-    "                translation (t=none or lba) (usually QEMU can guess them)\n",
-    QEMU_ARCH_ALL)
-STEXI
-@item -hdachs @var{c},@var{h},@var{s},[,@var{t}]
-@findex -hdachs
-Force hard disk 0 physical geometry (1 <= @var{c} <= 16383, 1 <=
-@var{h} <= 16, 1 <= @var{s} <= 63) and optionally force the BIOS
-translation mode (@var{t}=none, lba or auto). Usually QEMU can guess
-all those parameters. This option is deprecated, please use
-@code{-device ide-hd,cyls=c,heads=h,secs=s,...} instead.
-ETEXI
-
 DEF("fsdev", HAS_ARG, QEMU_OPTION_fsdev,
     "-fsdev fsdriver,id=id[,path=path,][security_model={mapped-xattr|mapped-file|passthrough|none}]\n"
     " [,writeout=immediate][,readonly][,socket=socket|sock_fd=sock_fd][,fmode=fmode][,dmode=dmode]\n"
-- 
2.13.6

From: Thomas Huth <thuth@redhat.com>

Looks like we forgot to announce the deprecation of these options in
the corresponding chapter of the qemu-doc text, so let's do that now.

diff --git a/qemu-doc.texi b/qemu-doc.texi
index XXXXXXX..XXXXXXX 100644
--- a/qemu-doc.texi
+++ b/qemu-doc.texi
@@ -XXX,XX +XXX,XX @@ longer be directly supported in QEMU.
 The ``-drive if=scsi'' argument is replaced by the the
 ``-device BUS-TYPE'' argument combined with ``-drive if=none''.
 
+@subsection -drive cyls=...,heads=...,secs=...,trans=... (since 2.10.0)
+
+The drive geometry arguments are replaced by the the geometry arguments
+that can be specified with the ``-device'' parameter.
+
+@subsection -drive serial=... (since 2.10.0)
+
+The drive serial argument is replaced by the the serial argument
+that can be specified with the ``-device'' parameter.
+
+@subsection -drive addr=... (since 2.10.0)
+
+The drive addr argument is replaced by the the addr argument
+that can be specified with the ``-device'' parameter.
+
 @subsection -net dump (since 2.10.0)
 
 The ``--net dump'' argument is now replaced with the
-- 
2.13.6

From: Fam Zheng <famz@redhat.com>

Signed-off-by: Fam Zheng <famz@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 include/block/block_int.h |  1 -
 block/io.c                | 18 ------------------
 2 files changed, 19 deletions(-)

diff --git a/include/block/block_int.h b/include/block/block_int.h
index XXXXXXX..XXXXXXX 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -XXX,XX +XXX,XX @@ bool blk_dev_is_tray_open(BlockBackend *blk);
 bool blk_dev_is_medium_locked(BlockBackend *blk);
 
 void bdrv_set_dirty(BlockDriverState *bs, int64_t offset, int64_t bytes);
-bool bdrv_requests_pending(BlockDriverState *bs);
 
 void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap **out);
 void bdrv_undo_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap *in);
diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ void bdrv_disable_copy_on_read(BlockDriverState *bs)
     assert(old >= 1);
 }
 
-/* Check if any requests are in-flight (including throttled requests) */
-bool bdrv_requests_pending(BlockDriverState *bs)
-{
-    BdrvChild *child;
-
-    if (atomic_read(&bs->in_flight)) {
-        return true;
-    }
-
-    QLIST_FOREACH(child, &bs->children, next) {
-        if (bdrv_requests_pending(child->bs)) {
-            return true;
-        }
-    }
-
-    return false;
-}
-
 typedef struct {
     Coroutine *co;
     BlockDriverState *bs;
-- 
2.13.6

bdrv_drained_begin() doesn't increase bs->quiesce_counter recursively
and also doesn't notify other parent nodes of children, which both means
that the child nodes are not actually drained, and bdrv_drained_begin()
is providing useful functionality only on a single node.

To keep things consistent, we also shouldn't call the block driver
callbacks recursively.

A proper recursive drain version that provides an actually working
drained section for child nodes will be introduced later.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Fam Zheng <famz@redhat.com>
---
 block/io.c | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
 }
 
 /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */
-static void bdrv_drain_invoke(BlockDriverState *bs, bool begin)
+static void bdrv_drain_invoke(BlockDriverState *bs, bool begin, bool recursive)
 {
     BdrvChild *child, *tmp;
     BdrvCoDrainData data = { .bs = bs, .done = false, .begin = begin};
@@ -XXX,XX +XXX,XX @@ static void bdrv_drain_invoke(BlockDriverState *bs, bool begin)
     bdrv_coroutine_enter(bs, data.co);
     BDRV_POLL_WHILE(bs, !data.done);
 
-    QLIST_FOREACH_SAFE(child, &bs->children, next, tmp) {
-        bdrv_drain_invoke(child->bs, begin);
+    if (recursive) {
+        QLIST_FOREACH_SAFE(child, &bs->children, next, tmp) {
+            bdrv_drain_invoke(child->bs, begin, true);
+        }
     }
 }
 
@@ -XXX,XX +XXX,XX @@ void bdrv_drained_begin(BlockDriverState *bs)
         bdrv_parent_drained_begin(bs);
     }
 
-    bdrv_drain_invoke(bs, true);
+    bdrv_drain_invoke(bs, true, false);
     bdrv_drain_recurse(bs);
 }
 
@@ -XXX,XX +XXX,XX @@ void bdrv_drained_end(BlockDriverState *bs)
     }
 
     /* Re-enable things in child-to-parent order */
-    bdrv_drain_invoke(bs, false);
+    bdrv_drain_invoke(bs, false, false);
     bdrv_parent_drained_end(bs);
     aio_enable_external(bdrv_get_aio_context(bs));
 }
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
         aio_context_acquire(aio_context);
         aio_disable_external(aio_context);
         bdrv_parent_drained_begin(bs);
-        bdrv_drain_invoke(bs, true);
+        bdrv_drain_invoke(bs, true, true);
         aio_context_release(aio_context);
 
         if (!g_slist_find(aio_ctxs, aio_context)) {
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_end(void)
 
         /* Re-enable things in child-to-parent order */
         aio_context_acquire(aio_context);
-        bdrv_drain_invoke(bs, false);
+        bdrv_drain_invoke(bs, false, true);
         bdrv_parent_drained_end(bs);
         aio_enable_external(aio_context);
         aio_context_release(aio_context);
-- 
2.13.6

The existing test is for bdrv_drain_all_begin/end() only. Generalise the
test case so that it can be run for the other variants as well. At the
moment this is only bdrv_drain_begin/end(), but in a while, we'll add
another one.

Also, add a backing file to the test node to test whether the operations
work recursively.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/test-bdrv-drain.c | 69 ++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 62 insertions(+), 7 deletions(-)

diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
index XXXXXXX..XXXXXXX 100644
--- a/tests/test-bdrv-drain.c
+++ b/tests/test-bdrv-drain.c
@@ -XXX,XX +XXX,XX @@ static BlockDriver bdrv_test = {
 
     .bdrv_co_drain_begin    = bdrv_test_co_drain_begin,
     .bdrv_co_drain_end      = bdrv_test_co_drain_end,
+
+    .bdrv_child_perm        = bdrv_format_default_perms,
 };
 
 static void aio_ret_cb(void *opaque, int ret)
@@ -XXX,XX +XXX,XX @@ static void aio_ret_cb(void *opaque, int ret)
     *aio_ret = ret;
 }
 
-static void test_drv_cb_drain_all(void)
+enum drain_type {
+    BDRV_DRAIN_ALL,
+    BDRV_DRAIN,
+};
+
+static void do_drain_begin(enum drain_type drain_type, BlockDriverState *bs)
+{
+    switch (drain_type) {
+    case BDRV_DRAIN_ALL:        bdrv_drain_all_begin(); break;
+    case BDRV_DRAIN:            bdrv_drained_begin(bs); break;
+    default:                    g_assert_not_reached();
+    }
+}
+
+static void do_drain_end(enum drain_type drain_type, BlockDriverState *bs)
+{
+    switch (drain_type) {
+    case BDRV_DRAIN_ALL:        bdrv_drain_all_end(); break;
+    case BDRV_DRAIN:            bdrv_drained_end(bs); break;
+    default:                    g_assert_not_reached();
+    }
+}
+
+static void test_drv_cb_common(enum drain_type drain_type, bool recursive)
 {
     BlockBackend *blk;
-    BlockDriverState *bs;
-    BDRVTestState *s;
+    BlockDriverState *bs, *backing;
+    BDRVTestState *s, *backing_s;
     BlockAIOCB *acb;
     int aio_ret;
 
@@ -XXX,XX +XXX,XX @@ static void test_drv_cb_drain_all(void)
     s = bs->opaque;
     blk_insert_bs(blk, bs, &error_abort);
 
+    backing = bdrv_new_open_driver(&bdrv_test, "backing", 0, &error_abort);
+    backing_s = backing->opaque;
+    bdrv_set_backing_hd(bs, backing, &error_abort);
+
     /* Simple bdrv_drain_all_begin/end pair, check that CBs are called */
     g_assert_cmpint(s->drain_count, ==, 0);
-    bdrv_drain_all_begin();
+    g_assert_cmpint(backing_s->drain_count, ==, 0);
+
+    do_drain_begin(drain_type, bs);
+
     g_assert_cmpint(s->drain_count, ==, 1);
-    bdrv_drain_all_end();
+    g_assert_cmpint(backing_s->drain_count, ==, !!recursive);
+
+    do_drain_end(drain_type, bs);
+
     g_assert_cmpint(s->drain_count, ==, 0);
+    g_assert_cmpint(backing_s->drain_count, ==, 0);
 
     /* Now do the same while a request is pending */
     aio_ret = -EINPROGRESS;
@@ -XXX,XX +XXX,XX @@ static void test_drv_cb_drain_all(void)
     g_assert_cmpint(aio_ret, ==, -EINPROGRESS);
 
     g_assert_cmpint(s->drain_count, ==, 0);
-    bdrv_drain_all_begin();
+    g_assert_cmpint(backing_s->drain_count, ==, 0);
+
+    do_drain_begin(drain_type, bs);
+
     g_assert_cmpint(aio_ret, ==, 0);
     g_assert_cmpint(s->drain_count, ==, 1);
-    bdrv_drain_all_end();
+    g_assert_cmpint(backing_s->drain_count, ==, !!recursive);
+
+    do_drain_end(drain_type, bs);
+
     g_assert_cmpint(s->drain_count, ==, 0);
+    g_assert_cmpint(backing_s->drain_count, ==, 0);
 
+    bdrv_unref(backing);
     bdrv_unref(bs);
     blk_unref(blk);
 }
 
+static void test_drv_cb_drain_all(void)
+{
+    test_drv_cb_common(BDRV_DRAIN_ALL, true);
+}
+
+static void test_drv_cb_drain(void)
+{
+    test_drv_cb_common(BDRV_DRAIN, false);
+}
+
 int main(int argc, char **argv)
 {
     bdrv_init();
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
     g_test_init(&argc, &argv, NULL);
 
     g_test_add_func("/bdrv-drain/driver-cb/drain_all", test_drv_cb_drain_all);
+    g_test_add_func("/bdrv-drain/driver-cb/drain", test_drv_cb_drain);
 
     return g_test_run();
 }
-- 
2.13.6

This is currently only working correctly for bdrv_drain(), not for
bdrv_drain_all(). Leave a comment for the drain_all case, we'll address
it later.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/test-bdrv-drain.c | 45 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
index XXXXXXX..XXXXXXX 100644
--- a/tests/test-bdrv-drain.c
+++ b/tests/test-bdrv-drain.c
@@ -XXX,XX +XXX,XX @@ static void test_drv_cb_drain(void)
     test_drv_cb_common(BDRV_DRAIN, false);
 }
 
+static void test_quiesce_common(enum drain_type drain_type, bool recursive)
+{
+    BlockBackend *blk;
+    BlockDriverState *bs, *backing;
+
+    blk = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
+    bs = bdrv_new_open_driver(&bdrv_test, "test-node", BDRV_O_RDWR,
+                              &error_abort);
+    blk_insert_bs(blk, bs, &error_abort);
+
+    backing = bdrv_new_open_driver(&bdrv_test, "backing", 0, &error_abort);
+    bdrv_set_backing_hd(bs, backing, &error_abort);
+
+    g_assert_cmpint(bs->quiesce_counter, ==, 0);
+    g_assert_cmpint(backing->quiesce_counter, ==, 0);
+
+    do_drain_begin(drain_type, bs);
+
+    g_assert_cmpint(bs->quiesce_counter, ==, 1);
+    g_assert_cmpint(backing->quiesce_counter, ==, !!recursive);
+
+    do_drain_end(drain_type, bs);
+
+    g_assert_cmpint(bs->quiesce_counter, ==, 0);
+    g_assert_cmpint(backing->quiesce_counter, ==, 0);
+
+    bdrv_unref(backing);
+    bdrv_unref(bs);
+    blk_unref(blk);
+}
+
+static void test_quiesce_drain_all(void)
+{
+    // XXX drain_all doesn't quiesce
+    //test_quiesce_common(BDRV_DRAIN_ALL, true);
+}
+
+static void test_quiesce_drain(void)
+{
+    test_quiesce_common(BDRV_DRAIN, false);
+}
+
 int main(int argc, char **argv)
 {
     bdrv_init();
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
     g_test_add_func("/bdrv-drain/driver-cb/drain_all", test_drv_cb_drain_all);
     g_test_add_func("/bdrv-drain/driver-cb/drain", test_drv_cb_drain);
 
+    g_test_add_func("/bdrv-drain/quiesce/drain_all", test_quiesce_drain_all);
+    g_test_add_func("/bdrv-drain/quiesce/drain", test_quiesce_drain);
+
     return g_test_run();
 }
-- 
2.13.6

Block jobs already paused themselves when their main BlockBackend
entered a drained section. This is not good enough: We also want to
pause a block job and may not submit new requests if, for example, the
mirror target node should be drained.

This implements .drained_begin/end callbacks in child_job in order to
consider all block nodes related to the job, and removes the
BlockBackend callbacks which are unnecessary now because the root of the
job main BlockBackend is always referenced with a child_job, too.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 blockjob.c | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/blockjob.c b/blockjob.c
index XXXXXXX..XXXXXXX 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -XXX,XX +XXX,XX @@ static char *child_job_get_parent_desc(BdrvChild *c)
                            job->id);
 }
 
-static const BdrvChildRole child_job = {
-    .get_parent_desc    = child_job_get_parent_desc,
-    .stay_at_node       = true,
-};
-
-static void block_job_drained_begin(void *opaque)
+static void child_job_drained_begin(BdrvChild *c)
 {
-    BlockJob *job = opaque;
+    BlockJob *job = c->opaque;
     block_job_pause(job);
 }
 
-static void block_job_drained_end(void *opaque)
+static void child_job_drained_end(BdrvChild *c)
 {
-    BlockJob *job = opaque;
+    BlockJob *job = c->opaque;
     block_job_resume(job);
 }
 
-static const BlockDevOps block_job_dev_ops = {
-    .drained_begin = block_job_drained_begin,
-    .drained_end = block_job_drained_end,
+static const BdrvChildRole child_job = {
+    .get_parent_desc    = child_job_get_parent_desc,
+    .drained_begin      = child_job_drained_begin,
+    .drained_end        = child_job_drained_end,
+    .stay_at_node       = true,
 };
 
 void block_job_remove_all_bdrv(BlockJob *job)
@@ -XXX,XX +XXX,XX @@ void *block_job_create(const char *job_id, const BlockJobDriver *driver,
     block_job_add_bdrv(job, "main node", bs, 0, BLK_PERM_ALL, &error_abort);
     bs->job = job;
 
-    blk_set_dev_ops(blk, &block_job_dev_ops, job);
     bdrv_op_unblock(bs, BLOCK_OP_TYPE_DATAPLANE, job->blocker);
 
     QLIST_INSERT_HEAD(&block_jobs, job, job_list);
-- 
2.13.6

Block jobs must be paused if any of the involved nodes are drained.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/test-bdrv-drain.c | 121 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 121 insertions(+)

diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
index XXXXXXX..XXXXXXX 100644
--- a/tests/test-bdrv-drain.c
+++ b/tests/test-bdrv-drain.c
@@ -XXX,XX +XXX,XX @@
 
 #include "qemu/osdep.h"
 #include "block/block.h"
+#include "block/blockjob_int.h"
 #include "sysemu/block-backend.h"
 #include "qapi/error.h"
 
@@ -XXX,XX +XXX,XX @@ static void test_quiesce_drain(void)
     test_quiesce_common(BDRV_DRAIN, false);
 }
 
+
+typedef struct TestBlockJob {
+    BlockJob common;
+    bool should_complete;
+} TestBlockJob;
+
+static void test_job_completed(BlockJob *job, void *opaque)
+{
+    block_job_completed(job, 0);
+}
+
+static void coroutine_fn test_job_start(void *opaque)
+{
+    TestBlockJob *s = opaque;
+
+    while (!s->should_complete) {
+        block_job_sleep_ns(&s->common, 100000);
+    }
+
+    block_job_defer_to_main_loop(&s->common, test_job_completed, NULL);
+}
+
+static void test_job_complete(BlockJob *job, Error **errp)
+{
+    TestBlockJob *s = container_of(job, TestBlockJob, common);
+    s->should_complete = true;
+}
+
+BlockJobDriver test_job_driver = {
+    .instance_size  = sizeof(TestBlockJob),
+    .start          = test_job_start,
+    .complete       = test_job_complete,
+};
+
+static void test_blockjob_common(enum drain_type drain_type)
+{
+    BlockBackend *blk_src, *blk_target;
+    BlockDriverState *src, *target;
+    BlockJob *job;
+    int ret;
+
+    src = bdrv_new_open_driver(&bdrv_test, "source", BDRV_O_RDWR,
+                               &error_abort);
+    blk_src = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
+    blk_insert_bs(blk_src, src, &error_abort);
+
+    target = bdrv_new_open_driver(&bdrv_test, "target", BDRV_O_RDWR,
+                                  &error_abort);
+    blk_target = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
+    blk_insert_bs(blk_target, target, &error_abort);
+
+    job = block_job_create("job0", &test_job_driver, src, 0, BLK_PERM_ALL, 0,
+                           0, NULL, NULL, &error_abort);
+    block_job_add_bdrv(job, "target", target, 0, BLK_PERM_ALL, &error_abort);
+    block_job_start(job);
+
+    g_assert_cmpint(job->pause_count, ==, 0);
+    g_assert_false(job->paused);
+    g_assert_false(job->busy); /* We're in block_job_sleep_ns() */
+
+    do_drain_begin(drain_type, src);
+
+    if (drain_type == BDRV_DRAIN_ALL) {
+        /* bdrv_drain_all() drains both src and target, and involves an
+         * additional block_job_pause_all() */
+        g_assert_cmpint(job->pause_count, ==, 3);
+    } else {
+        g_assert_cmpint(job->pause_count, ==, 1);
+    }
+    /* XXX We don't wait until the job is actually paused. Is this okay? */
+    /* g_assert_true(job->paused); */
+    g_assert_false(job->busy); /* The job is paused */
+
+    do_drain_end(drain_type, src);
+
+    g_assert_cmpint(job->pause_count, ==, 0);
+    g_assert_false(job->paused);
+    g_assert_false(job->busy); /* We're in block_job_sleep_ns() */
+
+    do_drain_begin(drain_type, target);
+
+    if (drain_type == BDRV_DRAIN_ALL) {
+        /* bdrv_drain_all() drains both src and target, and involves an
+         * additional block_job_pause_all() */
+        g_assert_cmpint(job->pause_count, ==, 3);
+    } else {
+        g_assert_cmpint(job->pause_count, ==, 1);
+    }
+    /* XXX We don't wait until the job is actually paused. Is this okay? */
+    /* g_assert_true(job->paused); */
+    g_assert_false(job->busy); /* The job is paused */
+
+    do_drain_end(drain_type, target);
+
+    g_assert_cmpint(job->pause_count, ==, 0);
+    g_assert_false(job->paused);
+    g_assert_false(job->busy); /* We're in block_job_sleep_ns() */
+
+    ret = block_job_complete_sync(job, &error_abort);
+    g_assert_cmpint(ret, ==, 0);
+
+    blk_unref(blk_src);
+    blk_unref(blk_target);
+    bdrv_unref(src);
+    bdrv_unref(target);
+}
+
+static void test_blockjob_drain_all(void)
+{
+    test_blockjob_common(BDRV_DRAIN_ALL);
+}
+
+static void test_blockjob_drain(void)
+{
+    test_blockjob_common(BDRV_DRAIN);
+}
+
 int main(int argc, char **argv)
 {
     bdrv_init();
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
     g_test_add_func("/bdrv-drain/quiesce/drain_all", test_quiesce_drain_all);
     g_test_add_func("/bdrv-drain/quiesce/drain", test_quiesce_drain);
 
+    g_test_add_func("/bdrv-drain/blockjob/drain_all", test_blockjob_drain_all);
+    g_test_add_func("/bdrv-drain/blockjob/drain", test_blockjob_drain);
+
     return g_test_run();
 }
-- 
2.13.6

Block jobs are already paused using the BdrvChildRole drain callbacks,
so we don't need an additional block_job_pause_all() call.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 block/io.c              |  4 ----
 tests/test-bdrv-drain.c | 10 ++++------
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
      * context. */
     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
 
-    block_job_pause_all();
-
     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
         AioContext *aio_context = bdrv_get_aio_context(bs);
 
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_end(void)
         aio_enable_external(aio_context);
         aio_context_release(aio_context);
     }
-
-    block_job_resume_all();
 }
 
 void bdrv_drain_all(void)
diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
index XXXXXXX..XXXXXXX 100644
--- a/tests/test-bdrv-drain.c
+++ b/tests/test-bdrv-drain.c
@@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type)
     do_drain_begin(drain_type, src);
 
     if (drain_type == BDRV_DRAIN_ALL) {
-        /* bdrv_drain_all() drains both src and target, and involves an
-         * additional block_job_pause_all() */
-        g_assert_cmpint(job->pause_count, ==, 3);
+        /* bdrv_drain_all() drains both src and target */
+        g_assert_cmpint(job->pause_count, ==, 2);
     } else {
         g_assert_cmpint(job->pause_count, ==, 1);
     }
@@ -XXX,XX +XXX,XX @@ static void test_blockjob_common(enum drain_type drain_type)
     do_drain_begin(drain_type, target);
 
     if (drain_type == BDRV_DRAIN_ALL) {
-        /* bdrv_drain_all() drains both src and target, and involves an
-         * additional block_job_pause_all() */
-        g_assert_cmpint(job->pause_count, ==, 3);
+        /* bdrv_drain_all() drains both src and target */
+        g_assert_cmpint(job->pause_count, ==, 2);
     } else {
         g_assert_cmpint(job->pause_count, ==, 1);
     }
-- 
2.13.6

bdrv_do_drained_begin() restricts the call of parent callbacks and
aio_disable_external() to the outermost drain section, but the block
driver callbacks are always called. bdrv_do_drained_end() must match
this behaviour, otherwise nodes stay drained even if begin/end calls
were balanced.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 block/io.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ void bdrv_drained_begin(BlockDriverState *bs)
 
 void bdrv_drained_end(BlockDriverState *bs)
 {
+    int old_quiesce_counter;
+
     if (qemu_in_coroutine()) {
         bdrv_co_yield_to_drain(bs, false);
         return;
     }
     assert(bs->quiesce_counter > 0);
-    if (atomic_fetch_dec(&bs->quiesce_counter) > 1) {
-        return;
-    }
+    old_quiesce_counter = atomic_fetch_dec(&bs->quiesce_counter);
 
     /* Re-enable things in child-to-parent order */
     bdrv_drain_invoke(bs, false, false);
-    bdrv_parent_drained_end(bs);
-    aio_enable_external(bdrv_get_aio_context(bs));
+    if (old_quiesce_counter == 1) {
+        bdrv_parent_drained_end(bs);
+        aio_enable_external(bdrv_get_aio_context(bs));
+    }
 }
 
 /*
-- 
2.13.6

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/test-bdrv-drain.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
index XXXXXXX..XXXXXXX 100644
--- a/tests/test-bdrv-drain.c
+++ b/tests/test-bdrv-drain.c
@@ -XXX,XX +XXX,XX @@ static void aio_ret_cb(void *opaque, int ret)
 enum drain_type {
     BDRV_DRAIN_ALL,
     BDRV_DRAIN,
+    DRAIN_TYPE_MAX,
 };
 
 static void do_drain_begin(enum drain_type drain_type, BlockDriverState *bs)
@@ -XXX,XX +XXX,XX @@ static void test_quiesce_drain(void)
     test_quiesce_common(BDRV_DRAIN, false);
 }
 
+static void test_nested(void)
+{
+    BlockBackend *blk;
+    BlockDriverState *bs, *backing;
+    BDRVTestState *s, *backing_s;
+    enum drain_type outer, inner;
+
+    blk = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
+    bs = bdrv_new_open_driver(&bdrv_test, "test-node", BDRV_O_RDWR,
+                              &error_abort);
+    s = bs->opaque;
+    blk_insert_bs(blk, bs, &error_abort);
+
+    backing = bdrv_new_open_driver(&bdrv_test, "backing", 0, &error_abort);
+    backing_s = backing->opaque;
+    bdrv_set_backing_hd(bs, backing, &error_abort);
+
+    for (outer = 0; outer < DRAIN_TYPE_MAX; outer++) {
+        for (inner = 0; inner < DRAIN_TYPE_MAX; inner++) {
+            /* XXX bdrv_drain_all() doesn't increase the quiesce_counter */
+            int bs_quiesce      = (outer != BDRV_DRAIN_ALL) +
+                                  (inner != BDRV_DRAIN_ALL);
+            int backing_quiesce = 0;
+            int backing_cb_cnt  = (outer != BDRV_DRAIN) +
+                                  (inner != BDRV_DRAIN);
+
+            g_assert_cmpint(bs->quiesce_counter, ==, 0);
+            g_assert_cmpint(backing->quiesce_counter, ==, 0);
+            g_assert_cmpint(s->drain_count, ==, 0);
+            g_assert_cmpint(backing_s->drain_count, ==, 0);
+
+            do_drain_begin(outer, bs);
+            do_drain_begin(inner, bs);
+
+            g_assert_cmpint(bs->quiesce_counter, ==, bs_quiesce);
+            g_assert_cmpint(backing->quiesce_counter, ==, backing_quiesce);
+            g_assert_cmpint(s->drain_count, ==, 2);
+            g_assert_cmpint(backing_s->drain_count, ==, backing_cb_cnt);
+
+            do_drain_end(inner, bs);
+            do_drain_end(outer, bs);
+
+            g_assert_cmpint(bs->quiesce_counter, ==, 0);
+            g_assert_cmpint(backing->quiesce_counter, ==, 0);
+            g_assert_cmpint(s->drain_count, ==, 0);
+            g_assert_cmpint(backing_s->drain_count, ==, 0);
+        }
+    }
+
+    bdrv_unref(backing);
+    bdrv_unref(bs);
+    blk_unref(blk);
+}
+
 
 typedef struct TestBlockJob {
     BlockJob common;
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
     g_test_add_func("/bdrv-drain/quiesce/drain_all", test_quiesce_drain_all);
     g_test_add_func("/bdrv-drain/quiesce/drain", test_quiesce_drain);
 
+    g_test_add_func("/bdrv-drain/nested", test_nested);
+
     g_test_add_func("/bdrv-drain/blockjob/drain_all", test_blockjob_drain_all);
     g_test_add_func("/bdrv-drain/blockjob/drain", test_blockjob_drain);
 
-- 
2.13.6

This is in preparation for subtree drains, i.e. drained sections that
affect not only a single node, but recursively all child nodes, too.

Calling the parent callbacks for drain is pointless when we just came
from that parent node recursively and leads to multiple increases of
bs->quiesce_counter in a single drain call. Don't do it.

In order for this to work correctly, the parent callback must be called
for every bdrv_drain_begin/end() call, not only for the outermost one:

If we have a node N with two parents A and B, recursive draining of A
should cause the quiesce_counter of B to increase because its child N is
drained independently of B. If now B is recursively drained, too, A must
increase its quiesce_counter because N is drained independently of A
only now, even if N is going from quiesce_counter 1 to 2.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 include/block/block.h |  4 ++--
 block.c               | 13 +++++++++----
 block/io.c            | 47 ++++++++++++++++++++++++++++++++++-------------
 3 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/include/block/block.h b/include/block/block.h
index XXXXXXX..XXXXXXX 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -XXX,XX +XXX,XX @@ void bdrv_io_unplug(BlockDriverState *bs);
  * Begin a quiesced section of all users of @bs. This is part of
  * bdrv_drained_begin.
  */
-void bdrv_parent_drained_begin(BlockDriverState *bs);
+void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore);
 
 /**
  * bdrv_parent_drained_end:
@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_begin(BlockDriverState *bs);
  * End a quiesced section of all users of @bs. This is part of
  * bdrv_drained_end.
  */
-void bdrv_parent_drained_end(BlockDriverState *bs);
+void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore);
 
 /**
  * bdrv_drained_begin:
diff --git a/block.c b/block.c
index XXXXXXX..XXXXXXX 100644
--- a/block.c
+++ b/block.c
@@ -XXX,XX +XXX,XX @@ static void bdrv_replace_child_noperm(BdrvChild *child,
                                       BlockDriverState *new_bs)
 {
     BlockDriverState *old_bs = child->bs;
+    int i;
 
     if (old_bs && new_bs) {
         assert(bdrv_get_aio_context(old_bs) == bdrv_get_aio_context(new_bs));
     }
     if (old_bs) {
         if (old_bs->quiesce_counter && child->role->drained_end) {
-            child->role->drained_end(child);
+            for (i = 0; i < old_bs->quiesce_counter; i++) {
+                child->role->drained_end(child);
+            }
         }
         if (child->role->detach) {
             child->role->detach(child);
@@ -XXX,XX +XXX,XX @@ static void bdrv_replace_child_noperm(BdrvChild *child,
     if (new_bs) {
         QLIST_INSERT_HEAD(&new_bs->parents, child, next_parent);
         if (new_bs->quiesce_counter && child->role->drained_begin) {
-            child->role->drained_begin(child);
+            for (i = 0; i < new_bs->quiesce_counter; i++) {
+                child->role->drained_begin(child);
+            }
         }
 
         if (child->role->attach) {
@@ -XXX,XX +XXX,XX @@ void bdrv_set_aio_context(BlockDriverState *bs, AioContext *new_context)
     AioContext *ctx = bdrv_get_aio_context(bs);
 
     aio_disable_external(ctx);
-    bdrv_parent_drained_begin(bs);
+    bdrv_parent_drained_begin(bs, NULL);
     bdrv_drain(bs); /* ensure there are no in-flight requests */
 
     while (aio_poll(ctx, false)) {
@@ -XXX,XX +XXX,XX @@ void bdrv_set_aio_context(BlockDriverState *bs, AioContext *new_context)
      */
     aio_context_acquire(new_context);
     bdrv_attach_aio_context(bs, new_context);
-    bdrv_parent_drained_end(bs);
+    bdrv_parent_drained_end(bs, NULL);
     aio_enable_external(ctx);
     aio_context_release(new_context);
 }
diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@
 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
     int64_t offset, int bytes, BdrvRequestFlags flags);
 
-void bdrv_parent_drained_begin(BlockDriverState *bs)
+void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore)
 {
     BdrvChild *c, *next;
 
     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
+        if (c == ignore) {
+            continue;
+        }
         if (c->role->drained_begin) {
             c->role->drained_begin(c);
         }
     }
 }
 
-void bdrv_parent_drained_end(BlockDriverState *bs)
+void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore)
 {
     BdrvChild *c, *next;
 
     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
+        if (c == ignore) {
+            continue;
+        }
         if (c->role->drained_end) {
             c->role->drained_end(c);
         }
@@ -XXX,XX +XXX,XX @@ typedef struct {
     BlockDriverState *bs;
     bool done;
     bool begin;
+    BdrvChild *parent;
 } BdrvCoDrainData;
 
 static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
@@ -XXX,XX +XXX,XX @@ static bool bdrv_drain_recurse(BlockDriverState *bs)
     return waited;
 }
 
+static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent);
+static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent);
+
 static void bdrv_co_drain_bh_cb(void *opaque)
 {
     BdrvCoDrainData *data = opaque;
@@ -XXX,XX +XXX,XX @@ static void bdrv_co_drain_bh_cb(void *opaque)
 
     bdrv_dec_in_flight(bs);
     if (data->begin) {
-        bdrv_drained_begin(bs);
+        bdrv_do_drained_begin(bs, data->parent);
     } else {
-        bdrv_drained_end(bs);
+        bdrv_do_drained_end(bs, data->parent);
     }
 
     data->done = true;
@@ -XXX,XX +XXX,XX @@ static void bdrv_co_drain_bh_cb(void *opaque)
 }
 
 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
-                                                bool begin)
+                                                bool begin, BdrvChild *parent)
 {
     BdrvCoDrainData data;
 
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
         .bs = bs,
         .done = false,
         .begin = begin,
+        .parent = parent,
     };
     bdrv_inc_in_flight(bs);
     aio_bh_schedule_oneshot(bdrv_get_aio_context(bs),
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
     assert(data.done);
 }
 
-void bdrv_drained_begin(BlockDriverState *bs)
+static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent)
 {
     if (qemu_in_coroutine()) {
-        bdrv_co_yield_to_drain(bs, true);
+        bdrv_co_yield_to_drain(bs, true, parent);
         return;
     }
 
     /* Stop things in parent-to-child order */
     if (atomic_fetch_inc(&bs->quiesce_counter) == 0) {
         aio_disable_external(bdrv_get_aio_context(bs));
-        bdrv_parent_drained_begin(bs);
     }
 
+    bdrv_parent_drained_begin(bs, parent);
     bdrv_drain_invoke(bs, true, false);
     bdrv_drain_recurse(bs);
 }
 
-void bdrv_drained_end(BlockDriverState *bs)
+void bdrv_drained_begin(BlockDriverState *bs)
+{
+    bdrv_do_drained_begin(bs, NULL);
+}
+
+static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent)
 {
     int old_quiesce_counter;
 
     if (qemu_in_coroutine()) {
-        bdrv_co_yield_to_drain(bs, false);
+        bdrv_co_yield_to_drain(bs, false, parent);
         return;
     }
     assert(bs->quiesce_counter > 0);
@@ -XXX,XX +XXX,XX @@ void bdrv_drained_end(BlockDriverState *bs)
 
     /* Re-enable things in child-to-parent order */
     bdrv_drain_invoke(bs, false, false);
+    bdrv_parent_drained_end(bs, parent);
     if (old_quiesce_counter == 1) {
-        bdrv_parent_drained_end(bs);
         aio_enable_external(bdrv_get_aio_context(bs));
     }
 }
 
+void bdrv_drained_end(BlockDriverState *bs)
+{
+    bdrv_do_drained_end(bs, NULL);
+}
+
 /*
  * Wait for pending requests to complete on a single BlockDriverState subtree,
  * and suspend block driver's internal I/O until next request arrives.
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_begin(void)
         /* Stop things in parent-to-child order */
         aio_context_acquire(aio_context);
         aio_disable_external(aio_context);
-        bdrv_parent_drained_begin(bs);
+        bdrv_parent_drained_begin(bs, NULL);
         bdrv_drain_invoke(bs, true, true);
         aio_context_release(aio_context);
 
@@ -XXX,XX +XXX,XX @@ void bdrv_drain_all_end(void)
         /* Re-enable things in child-to-parent order */
         aio_context_acquire(aio_context);
         bdrv_drain_invoke(bs, false, true);
-        bdrv_parent_drained_end(bs);
+        bdrv_parent_drained_end(bs, NULL);
         aio_enable_external(aio_context);
         aio_context_release(aio_context);
     }
-- 
2.13.6

bdrv_drained_begin() waits for the completion of requests in the whole
subtree, but it only actually keeps its immediate bs parameter quiesced
until bdrv_drained_end().

Add a version that keeps the whole subtree drained. As of this commit,
graph changes cannot be allowed during a subtree drained section, but
this will be fixed soon.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 include/block/block.h | 13 +++++++++++++
 block/io.c            | 54 ++++++++++++++++++++++++++++++++++++++++-----------
 2 files changed, 56 insertions(+), 11 deletions(-)

diff --git a/include/block/block.h b/include/block/block.h
index XXXXXXX..XXXXXXX 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -XXX,XX +XXX,XX @@ void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore);
 void bdrv_drained_begin(BlockDriverState *bs);
 
 /**
+ * Like bdrv_drained_begin, but recursively begins a quiesced section for
+ * exclusive access to all child nodes as well.
+ *
+ * Graph changes are not allowed during a subtree drain section.
+ */
+void bdrv_subtree_drained_begin(BlockDriverState *bs);
+
+/**
  * bdrv_drained_end:
  *
  * End a quiescent section started by bdrv_drained_begin().
  */
 void bdrv_drained_end(BlockDriverState *bs);
 
+/**
+ * End a quiescent section started by bdrv_subtree_drained_begin().
+ */
+void bdrv_subtree_drained_end(BlockDriverState *bs);
+
 void bdrv_add_child(BlockDriverState *parent, BlockDriverState *child,
                     Error **errp);
 void bdrv_del_child(BlockDriverState *parent, BdrvChild *child, Error **errp);
diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ typedef struct {
     BlockDriverState *bs;
     bool done;
     bool begin;
+    bool recursive;
     BdrvChild *parent;
 } BdrvCoDrainData;
 
@@ -XXX,XX +XXX,XX @@ static bool bdrv_drain_recurse(BlockDriverState *bs)
     return waited;
 }
 
-static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent);
-static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent);
+static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
+                                  BdrvChild *parent);
+static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
+                                BdrvChild *parent);
 
 static void bdrv_co_drain_bh_cb(void *opaque)
 {
@@ -XXX,XX +XXX,XX @@ static void bdrv_co_drain_bh_cb(void *opaque)
 
     bdrv_dec_in_flight(bs);
     if (data->begin) {
-        bdrv_do_drained_begin(bs, data->parent);
+        bdrv_do_drained_begin(bs, data->recursive, data->parent);
     } else {
-        bdrv_do_drained_end(bs, data->parent);
+        bdrv_do_drained_end(bs, data->recursive, data->parent);
     }
 
     data->done = true;
@@ -XXX,XX +XXX,XX @@ static void bdrv_co_drain_bh_cb(void *opaque)
 }
 
 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
-                                                bool begin, BdrvChild *parent)
+                                                bool begin, bool recursive,
+                                                BdrvChild *parent)
 {
     BdrvCoDrainData data;
 
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
         .bs = bs,
         .done = false,
         .begin = begin,
+        .recursive = recursive,
         .parent = parent,
     };
     bdrv_inc_in_flight(bs);
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
     assert(data.done);
 }
 
-static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent)
+static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
+                                  BdrvChild *parent)
 {
+    BdrvChild *child, *next;
+
     if (qemu_in_coroutine()) {
-        bdrv_co_yield_to_drain(bs, true, parent);
+        bdrv_co_yield_to_drain(bs, true, recursive, parent);
         return;
     }
 
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent)
     bdrv_parent_drained_begin(bs, parent);
     bdrv_drain_invoke(bs, true, false);
     bdrv_drain_recurse(bs);
+
+    if (recursive) {
+        QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
+            bdrv_do_drained_begin(child->bs, true, child);
+        }
+    }
 }
 
 void bdrv_drained_begin(BlockDriverState *bs)
 {
-    bdrv_do_drained_begin(bs, NULL);
+    bdrv_do_drained_begin(bs, false, NULL);
+}
+
+void bdrv_subtree_drained_begin(BlockDriverState *bs)
+{
+    bdrv_do_drained_begin(bs, true, NULL);
 }
 
-static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent)
+static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
+                                BdrvChild *parent)
 {
+    BdrvChild *child, *next;
     int old_quiesce_counter;
 
     if (qemu_in_coroutine()) {
-        bdrv_co_yield_to_drain(bs, false, parent);
+        bdrv_co_yield_to_drain(bs, false, recursive, parent);
         return;
     }
     assert(bs->quiesce_counter > 0);
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent)
     if (old_quiesce_counter == 1) {
         aio_enable_external(bdrv_get_aio_context(bs));
     }
+
+    if (recursive) {
+        QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
+            bdrv_do_drained_end(child->bs, true, child);
+        }
+    }
 }
 
 void bdrv_drained_end(BlockDriverState *bs)
 {
-    bdrv_do_drained_end(bs, NULL);
+    bdrv_do_drained_end(bs, false, NULL);
+}
+
+void bdrv_subtree_drained_end(BlockDriverState *bs)
+{
+    bdrv_do_drained_end(bs, true, NULL);
 }
 
 /*
-- 
2.13.6

Add a subtree drain version to the existing test cases.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/test-bdrv-drain.c | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
index XXXXXXX..XXXXXXX 100644
--- a/tests/test-bdrv-drain.c
+++ b/tests/test-bdrv-drain.c
@@ -XXX,XX +XXX,XX @@ static void aio_ret_cb(void *opaque, int ret)
 enum drain_type {
     BDRV_DRAIN_ALL,
     BDRV_DRAIN,
+    BDRV_SUBTREE_DRAIN,
     DRAIN_TYPE_MAX,
 };
 
@@ -XXX,XX +XXX,XX @@ static void do_drain_begin(enum drain_type drain_type, BlockDriverState *bs)
     switch (drain_type) {
     case BDRV_DRAIN_ALL:        bdrv_drain_all_begin(); break;
     case BDRV_DRAIN:            bdrv_drained_begin(bs); break;
+    case BDRV_SUBTREE_DRAIN:    bdrv_subtree_drained_begin(bs); break;
     default:                    g_assert_not_reached();
     }
 }
@@ -XXX,XX +XXX,XX @@ static void do_drain_end(enum drain_type drain_type, BlockDriverState *bs)
     switch (drain_type) {
     case BDRV_DRAIN_ALL:        bdrv_drain_all_end(); break;
     case BDRV_DRAIN:            bdrv_drained_end(bs); break;
+    case BDRV_SUBTREE_DRAIN:    bdrv_subtree_drained_end(bs); break;
     default:                    g_assert_not_reached();
     }
 }
@@ -XXX,XX +XXX,XX @@ static void test_drv_cb_drain(void)
     test_drv_cb_common(BDRV_DRAIN, false);
 }
 
+static void test_drv_cb_drain_subtree(void)
+{
+    test_drv_cb_common(BDRV_SUBTREE_DRAIN, true);
+}
+
 static void test_quiesce_common(enum drain_type drain_type, bool recursive)
 {
     BlockBackend *blk;
@@ -XXX,XX +XXX,XX @@ static void test_quiesce_drain(void)
     test_quiesce_common(BDRV_DRAIN, false);
 }
 
+static void test_quiesce_drain_subtree(void)
+{
+    test_quiesce_common(BDRV_SUBTREE_DRAIN, true);
+}
+
 static void test_nested(void)
 {
     BlockBackend *blk;
@@ -XXX,XX +XXX,XX @@ static void test_nested(void)
             /* XXX bdrv_drain_all() doesn't increase the quiesce_counter */
             int bs_quiesce      = (outer != BDRV_DRAIN_ALL) +
                                   (inner != BDRV_DRAIN_ALL);
-            int backing_quiesce = 0;
+            int backing_quiesce = (outer == BDRV_SUBTREE_DRAIN) +
+                                  (inner == BDRV_SUBTREE_DRAIN);
             int backing_cb_cnt  = (outer != BDRV_DRAIN) +
                                   (inner != BDRV_DRAIN);
 
@@ -XXX,XX +XXX,XX @@ static void test_blockjob_drain(void)
     test_blockjob_common(BDRV_DRAIN);
 }
 
+static void test_blockjob_drain_subtree(void)
+{
+    test_blockjob_common(BDRV_SUBTREE_DRAIN);
+}
+
 int main(int argc, char **argv)
 {
     bdrv_init();
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
 
     g_test_add_func("/bdrv-drain/driver-cb/drain_all", test_drv_cb_drain_all);
     g_test_add_func("/bdrv-drain/driver-cb/drain", test_drv_cb_drain);
+    g_test_add_func("/bdrv-drain/driver-cb/drain_subtree",
+                    test_drv_cb_drain_subtree);
 
     g_test_add_func("/bdrv-drain/quiesce/drain_all", test_quiesce_drain_all);
     g_test_add_func("/bdrv-drain/quiesce/drain", test_quiesce_drain);
+    g_test_add_func("/bdrv-drain/quiesce/drain_subtree",
+                    test_quiesce_drain_subtree);
 
     g_test_add_func("/bdrv-drain/nested", test_nested);
 
     g_test_add_func("/bdrv-drain/blockjob/drain_all", test_blockjob_drain_all);
     g_test_add_func("/bdrv-drain/blockjob/drain", test_blockjob_drain);
+    g_test_add_func("/bdrv-drain/blockjob/drain_subtree",
+                    test_blockjob_drain_subtree);
 
     return g_test_run();
 }
-- 
2.13.6

If bdrv_do_drained_begin/end() are called in coroutine context, they
first use a BH to get out of the coroutine context. Call some existing
tests again from a coroutine to cover this code path.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/test-bdrv-drain.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
index XXXXXXX..XXXXXXX 100644
--- a/tests/test-bdrv-drain.c
+++ b/tests/test-bdrv-drain.c
@@ -XXX,XX +XXX,XX @@ static void aio_ret_cb(void *opaque, int ret)
     *aio_ret = ret;
 }
 
+typedef struct CallInCoroutineData {
+    void (*entry)(void);
+    bool done;
+} CallInCoroutineData;
+
+static coroutine_fn void call_in_coroutine_entry(void *opaque)
+{
+    CallInCoroutineData *data = opaque;
+
+    data->entry();
+    data->done = true;
+}
+
+static void call_in_coroutine(void (*entry)(void))
+{
+    Coroutine *co;
+    CallInCoroutineData data = {
+        .entry  = entry,
+        .done   = false,
+    };
+
+    co = qemu_coroutine_create(call_in_coroutine_entry, &data);
+    qemu_coroutine_enter(co);
+    while (!data.done) {
+        aio_poll(qemu_get_aio_context(), true);
+    }
+}
+
 enum drain_type {
     BDRV_DRAIN_ALL,
     BDRV_DRAIN,
@@ -XXX,XX +XXX,XX @@ static void test_drv_cb_drain_subtree(void)
     test_drv_cb_common(BDRV_SUBTREE_DRAIN, true);
 }
 
+static void test_drv_cb_co_drain(void)
+{
+    call_in_coroutine(test_drv_cb_drain);
+}
+
+static void test_drv_cb_co_drain_subtree(void)
+{
+    call_in_coroutine(test_drv_cb_drain_subtree);
+}
+
 static void test_quiesce_common(enum drain_type drain_type, bool recursive)
 {
     BlockBackend *blk;
@@ -XXX,XX +XXX,XX @@ static void test_quiesce_drain_subtree(void)
     test_quiesce_common(BDRV_SUBTREE_DRAIN, true);
 }
 
+static void test_quiesce_co_drain(void)
+{
+    call_in_coroutine(test_quiesce_drain);
+}
+
+static void test_quiesce_co_drain_subtree(void)
+{
+    call_in_coroutine(test_quiesce_drain_subtree);
+}
+
 static void test_nested(void)
 {
     BlockBackend *blk;
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
     g_test_add_func("/bdrv-drain/driver-cb/drain_subtree",
                     test_drv_cb_drain_subtree);
 
+    // XXX bdrv_drain_all() doesn't work in coroutine context
+    g_test_add_func("/bdrv-drain/driver-cb/co/drain", test_drv_cb_co_drain);
+    g_test_add_func("/bdrv-drain/driver-cb/co/drain_subtree",
+                    test_drv_cb_co_drain_subtree);
+
+
     g_test_add_func("/bdrv-drain/quiesce/drain_all", test_quiesce_drain_all);
     g_test_add_func("/bdrv-drain/quiesce/drain", test_quiesce_drain);
     g_test_add_func("/bdrv-drain/quiesce/drain_subtree",
                     test_quiesce_drain_subtree);
 
+    // XXX bdrv_drain_all() doesn't work in coroutine context
+    g_test_add_func("/bdrv-drain/quiesce/co/drain", test_quiesce_co_drain);
+    g_test_add_func("/bdrv-drain/quiesce/co/drain_subtree",
+                    test_quiesce_co_drain_subtree);
+
     g_test_add_func("/bdrv-drain/nested", test_nested);
 
     g_test_add_func("/bdrv-drain/blockjob/drain_all", test_blockjob_drain_all);
-- 
2.13.6

Test that drain sections are correctly propagated through the graph.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/test-bdrv-drain.c | 74 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)

diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
index XXXXXXX..XXXXXXX 100644
--- a/tests/test-bdrv-drain.c
+++ b/tests/test-bdrv-drain.c
@@ -XXX,XX +XXX,XX @@ static void test_nested(void)
     blk_unref(blk);
 }
 
+static void test_multiparent(void)
+{
+    BlockBackend *blk_a, *blk_b;
+    BlockDriverState *bs_a, *bs_b, *backing;
+    BDRVTestState *a_s, *b_s, *backing_s;
+
+    blk_a = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
+    bs_a = bdrv_new_open_driver(&bdrv_test, "test-node-a", BDRV_O_RDWR,
+                                &error_abort);
+    a_s = bs_a->opaque;
+    blk_insert_bs(blk_a, bs_a, &error_abort);
+
+    blk_b = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
+    bs_b = bdrv_new_open_driver(&bdrv_test, "test-node-b", BDRV_O_RDWR,
+                                &error_abort);
+    b_s = bs_b->opaque;
+    blk_insert_bs(blk_b, bs_b, &error_abort);
+
+    backing = bdrv_new_open_driver(&bdrv_test, "backing", 0, &error_abort);
+    backing_s = backing->opaque;
+    bdrv_set_backing_hd(bs_a, backing, &error_abort);
+    bdrv_set_backing_hd(bs_b, backing, &error_abort);
+
+    g_assert_cmpint(bs_a->quiesce_counter, ==, 0);
+    g_assert_cmpint(bs_b->quiesce_counter, ==, 0);
+    g_assert_cmpint(backing->quiesce_counter, ==, 0);
+    g_assert_cmpint(a_s->drain_count, ==, 0);
+    g_assert_cmpint(b_s->drain_count, ==, 0);
+    g_assert_cmpint(backing_s->drain_count, ==, 0);
+
+    do_drain_begin(BDRV_SUBTREE_DRAIN, bs_a);
+
+    g_assert_cmpint(bs_a->quiesce_counter, ==, 1);
+    g_assert_cmpint(bs_b->quiesce_counter, ==, 1);
+    g_assert_cmpint(backing->quiesce_counter, ==, 1);
+    g_assert_cmpint(a_s->drain_count, ==, 1);
+    g_assert_cmpint(b_s->drain_count, ==, 1);
+    g_assert_cmpint(backing_s->drain_count, ==, 1);
+
+    do_drain_begin(BDRV_SUBTREE_DRAIN, bs_b);
+
+    g_assert_cmpint(bs_a->quiesce_counter, ==, 2);
+    g_assert_cmpint(bs_b->quiesce_counter, ==, 2);
+    g_assert_cmpint(backing->quiesce_counter, ==, 2);
+    g_assert_cmpint(a_s->drain_count, ==, 2);
+    g_assert_cmpint(b_s->drain_count, ==, 2);
+    g_assert_cmpint(backing_s->drain_count, ==, 2);
+
+    do_drain_end(BDRV_SUBTREE_DRAIN, bs_b);
+
+    g_assert_cmpint(bs_a->quiesce_counter, ==, 1);
+    g_assert_cmpint(bs_b->quiesce_counter, ==, 1);
+    g_assert_cmpint(backing->quiesce_counter, ==, 1);
+    g_assert_cmpint(a_s->drain_count, ==, 1);
+    g_assert_cmpint(b_s->drain_count, ==, 1);
+    g_assert_cmpint(backing_s->drain_count, ==, 1);
+
+    do_drain_end(BDRV_SUBTREE_DRAIN, bs_a);
+
+    g_assert_cmpint(bs_a->quiesce_counter, ==, 0);
+    g_assert_cmpint(bs_b->quiesce_counter, ==, 0);
+    g_assert_cmpint(backing->quiesce_counter, ==, 0);
+    g_assert_cmpint(a_s->drain_count, ==, 0);
+    g_assert_cmpint(b_s->drain_count, ==, 0);
+    g_assert_cmpint(backing_s->drain_count, ==, 0);
+
+    bdrv_unref(backing);
+    bdrv_unref(bs_a);
+    bdrv_unref(bs_b);
+    blk_unref(blk_a);
+    blk_unref(blk_b);
+}
+
 
 typedef struct TestBlockJob {
     BlockJob common;
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
                     test_quiesce_co_drain_subtree);
 
     g_test_add_func("/bdrv-drain/nested", test_nested);
+    g_test_add_func("/bdrv-drain/multiparent", test_multiparent);
 
     g_test_add_func("/bdrv-drain/blockjob/drain_all", test_blockjob_drain_all);
     g_test_add_func("/bdrv-drain/blockjob/drain", test_blockjob_drain);
-- 
2.13.6

We need to remember how many of the drain sections in which a node is
were recursive (i.e. subtree drain rather than node drain), so that they
can be correctly applied when children are added or removed during the
drained section.

With this change, it is safe to modify the graph even inside a
bdrv_subtree_drained_begin/end() section.

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 include/block/block.h     |  2 --
 include/block/block_int.h |  5 +++++
 block.c                   | 32 +++++++++++++++++++++++++++++---
 block/io.c                | 28 ++++++++++++++++++++++++----
 4 files changed, 58 insertions(+), 9 deletions(-)

diff --git a/include/block/block.h b/include/block/block.h
index XXXXXXX..XXXXXXX 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -XXX,XX +XXX,XX @@ void bdrv_drained_begin(BlockDriverState *bs);
 /**
  * Like bdrv_drained_begin, but recursively begins a quiesced section for
  * exclusive access to all child nodes as well.
- *
- * Graph changes are not allowed during a subtree drain section.
  */
 void bdrv_subtree_drained_begin(BlockDriverState *bs);
 
diff --git a/include/block/block_int.h b/include/block/block_int.h
index XXXXXXX..XXXXXXX 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -XXX,XX +XXX,XX @@ struct BlockDriverState {
 
     /* Accessed with atomic ops.  */
     int quiesce_counter;
+    int recursive_quiesce_counter;
+
     unsigned int write_gen;               /* Current data generation */
 
     /* Protected by reqs_lock.  */
@@ -XXX,XX +XXX,XX @@ int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
     BdrvRequestFlags flags);
 
+void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent);
+void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent);
+
 int get_tmp_filename(char *filename, int size);
 BlockDriver *bdrv_probe_all(const uint8_t *buf, int buf_size,
                             const char *filename);
diff --git a/block.c b/block.c
index XXXXXXX..XXXXXXX 100644
--- a/block.c
+++ b/block.c
@@ -XXX,XX +XXX,XX @@ static void bdrv_child_cb_drained_end(BdrvChild *child)
     bdrv_drained_end(bs);
 }
 
+static void bdrv_child_cb_attach(BdrvChild *child)
+{
+    BlockDriverState *bs = child->opaque;
+    bdrv_apply_subtree_drain(child, bs);
+}
+
+static void bdrv_child_cb_detach(BdrvChild *child)
+{
+    BlockDriverState *bs = child->opaque;
+    bdrv_unapply_subtree_drain(child, bs);
+}
+
 static int bdrv_child_cb_inactivate(BdrvChild *child)
 {
     BlockDriverState *bs = child->opaque;
@@ -XXX,XX +XXX,XX @@ const BdrvChildRole child_file = {
     .inherit_options = bdrv_inherited_options,
     .drained_begin   = bdrv_child_cb_drained_begin,
     .drained_end     = bdrv_child_cb_drained_end,
+    .attach          = bdrv_child_cb_attach,
+    .detach          = bdrv_child_cb_detach,
     .inactivate      = bdrv_child_cb_inactivate,
 };
 
@@ -XXX,XX +XXX,XX @@ const BdrvChildRole child_format = {
     .inherit_options = bdrv_inherited_fmt_options,
     .drained_begin   = bdrv_child_cb_drained_begin,
     .drained_end     = bdrv_child_cb_drained_end,
+    .attach          = bdrv_child_cb_attach,
+    .detach          = bdrv_child_cb_detach,
     .inactivate      = bdrv_child_cb_inactivate,
 };
 
@@ -XXX,XX +XXX,XX @@ static void bdrv_backing_attach(BdrvChild *c)
                     parent->backing_blocker);
     bdrv_op_unblock(backing_hd, BLOCK_OP_TYPE_BACKUP_TARGET,
                     parent->backing_blocker);
+
+    bdrv_child_cb_attach(c);
 }
 
 static void bdrv_backing_detach(BdrvChild *c)
@@ -XXX,XX +XXX,XX @@ static void bdrv_backing_detach(BdrvChild *c)
     bdrv_op_unblock_all(c->bs, parent->backing_blocker);
     error_free(parent->backing_blocker);
     parent->backing_blocker = NULL;
+
+    bdrv_child_cb_detach(c);
 }
 
 /*
@@ -XXX,XX +XXX,XX @@ static void bdrv_replace_child_noperm(BdrvChild *child,
         assert(bdrv_get_aio_context(old_bs) == bdrv_get_aio_context(new_bs));
     }
     if (old_bs) {
+        /* Detach first so that the recursive drain sections coming from @child
+         * are already gone and we only end the drain sections that came from
+         * elsewhere. */
+        if (child->role->detach) {
+            child->role->detach(child);
+        }
         if (old_bs->quiesce_counter && child->role->drained_end) {
             for (i = 0; i < old_bs->quiesce_counter; i++) {
                 child->role->drained_end(child);
             }
         }
-        if (child->role->detach) {
-            child->role->detach(child);
-        }
         QLIST_REMOVE(child, next_parent);
     }
 
@@ -XXX,XX +XXX,XX @@ static void bdrv_replace_child_noperm(BdrvChild *child,
             }
         }
 
+        /* Attach only after starting new drained sections, so that recursive
+         * drain sections coming from @child don't get an extra .drained_begin
+         * callback. */
         if (child->role->attach) {
             child->role->attach(child);
         }
diff --git a/block/io.c b/block/io.c
index XXXXXXX..XXXXXXX 100644
--- a/block/io.c
+++ b/block/io.c
@@ -XXX,XX +XXX,XX @@ static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
     assert(data.done);
 }
 
-static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
-                                  BdrvChild *parent)
+void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
+                           BdrvChild *parent)
 {
     BdrvChild *child, *next;
 
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
     bdrv_drain_recurse(bs);
 
     if (recursive) {
+        bs->recursive_quiesce_counter++;
         QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
             bdrv_do_drained_begin(child->bs, true, child);
         }
@@ -XXX,XX +XXX,XX @@ void bdrv_subtree_drained_begin(BlockDriverState *bs)
     bdrv_do_drained_begin(bs, true, NULL);
 }
 
-static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
-                                BdrvChild *parent)
+void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
+                         BdrvChild *parent)
 {
     BdrvChild *child, *next;
     int old_quiesce_counter;
@@ -XXX,XX +XXX,XX @@ static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
     }
 
     if (recursive) {
+        bs->recursive_quiesce_counter--;
         QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
             bdrv_do_drained_end(child->bs, true, child);
         }
@@ -XXX,XX +XXX,XX @@ void bdrv_subtree_drained_end(BlockDriverState *bs)
     bdrv_do_drained_end(bs, true, NULL);
 }
 
+void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
+{
+    int i;
+
+    for (i = 0; i < new_parent->recursive_quiesce_counter; i++) {
+        bdrv_do_drained_begin(child->bs, true, child);
+    }
+}
+
+void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent)
+{
+    int i;
+
+    for (i = 0; i < old_parent->recursive_quiesce_counter; i++) {
+        bdrv_do_drained_end(child->bs, true, child);
+    }
+}
+
 /*
  * Wait for pending requests to complete on a single BlockDriverState subtree,
  * and suspend block driver's internal I/O until next request arrives.
-- 
2.13.6

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
---
 tests/test-bdrv-drain.c | 80 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)

diff --git a/tests/test-bdrv-drain.c b/tests/test-bdrv-drain.c
index XXXXXXX..XXXXXXX 100644
--- a/tests/test-bdrv-drain.c
+++ b/tests/test-bdrv-drain.c
@@ -XXX,XX +XXX,XX @@ static void test_multiparent(void)
     blk_unref(blk_b);
 }
 
+static void test_graph_change(void)
+{
+    BlockBackend *blk_a, *blk_b;
+    BlockDriverState *bs_a, *bs_b, *backing;
+    BDRVTestState *a_s, *b_s, *backing_s;
+
+    blk_a = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
+    bs_a = bdrv_new_open_driver(&bdrv_test, "test-node-a", BDRV_O_RDWR,
+                                &error_abort);
+    a_s = bs_a->opaque;
+    blk_insert_bs(blk_a, bs_a, &error_abort);
+
+    blk_b = blk_new(BLK_PERM_ALL, BLK_PERM_ALL);
+    bs_b = bdrv_new_open_driver(&bdrv_test, "test-node-b", BDRV_O_RDWR,
+                                &error_abort);
+    b_s = bs_b->opaque;
+    blk_insert_bs(blk_b, bs_b, &error_abort);
+
+    backing = bdrv_new_open_driver(&bdrv_test, "backing", 0, &error_abort);
+    backing_s = backing->opaque;
+    bdrv_set_backing_hd(bs_a, backing, &error_abort);
+
+    g_assert_cmpint(bs_a->quiesce_counter, ==, 0);
+    g_assert_cmpint(bs_b->quiesce_counter, ==, 0);
+    g_assert_cmpint(backing->quiesce_counter, ==, 0);
+    g_assert_cmpint(a_s->drain_count, ==, 0);
+    g_assert_cmpint(b_s->drain_count, ==, 0);
+    g_assert_cmpint(backing_s->drain_count, ==, 0);
+
+    do_drain_begin(BDRV_SUBTREE_DRAIN, bs_a);
+    do_drain_begin(BDRV_SUBTREE_DRAIN, bs_a);
+    do_drain_begin(BDRV_SUBTREE_DRAIN, bs_a);
+    do_drain_begin(BDRV_SUBTREE_DRAIN, bs_b);
+    do_drain_begin(BDRV_SUBTREE_DRAIN, bs_b);
+
+    bdrv_set_backing_hd(bs_b, backing, &error_abort);
+    g_assert_cmpint(bs_a->quiesce_counter, ==, 5);
+    g_assert_cmpint(bs_b->quiesce_counter, ==, 5);
+    g_assert_cmpint(backing->quiesce_counter, ==, 5);
+    g_assert_cmpint(a_s->drain_count, ==, 5);
+    g_assert_cmpint(b_s->drain_count, ==, 5);
+    g_assert_cmpint(backing_s->drain_count, ==, 5);
+
+    bdrv_set_backing_hd(bs_b, NULL, &error_abort);
+    g_assert_cmpint(bs_a->quiesce_counter, ==, 3);
+    g_assert_cmpint(bs_b->quiesce_counter, ==, 2);
+    g_assert_cmpint(backing->quiesce_counter, ==, 3);
+    g_assert_cmpint(a_s->drain_count, ==, 3);
+    g_assert_cmpint(b_s->drain_count, ==, 2);
+    g_assert_cmpint(backing_s->drain_count, ==, 3);
+
+    bdrv_set_backing_hd(bs_b, backing, &error_abort);
+    g_assert_cmpint(bs_a->quiesce_counter, ==, 5);
+    g_assert_cmpint(bs_b->quiesce_counter, ==, 5);
+    g_assert_cmpint(backing->quiesce_counter, ==, 5);
+    g_assert_cmpint(a_s->drain_count, ==, 5);
+    g_assert_cmpint(b_s->drain_count, ==, 5);
+    g_assert_cmpint(backing_s->drain_count, ==, 5);
+
+    do_drain_end(BDRV_SUBTREE_DRAIN, bs_b);
+    do_drain_end(BDRV_SUBTREE_DRAIN, bs_b);
+    do_drain_end(BDRV_SUBTREE_DRAIN, bs_a);
+    do_drain_end(BDRV_SUBTREE_DRAIN, bs_a);
+    do_drain_end(BDRV_SUBTREE_DRAIN, bs_a);
+
+    g_assert_cmpint(bs_a->quiesce_counter, ==, 0);
+    g_assert_cmpint(bs_b->quiesce_counter, ==, 0);
+    g_assert_cmpint(backing->quiesce_counter, ==, 0);
+    g_assert_cmpint(a_s->drain_count, ==, 0);
+    g_assert_cmpint(b_s->drain_count, ==, 0);
+    g_assert_cmpint(backing_s->drain_count, ==, 0);
+
+    bdrv_unref(backing);
+    bdrv_unref(bs_a);
+    bdrv_unref(bs_b);
+    blk_unref(blk_a);
+    blk_unref(blk_b);
+}
+
 
 typedef struct TestBlockJob {
     BlockJob common;
@@ -XXX,XX +XXX,XX @@ int main(int argc, char **argv)
 
     g_test_add_func("/bdrv-drain/nested", test_nested);
     g_test_add_func("/bdrv-drain/multiparent", test_multiparent);
+    g_test_add_func("/bdrv-drain/graph-change", test_graph_change);
 
     g_test_add_func("/bdrv-drain/blockjob/drain_all", test_blockjob_drain_all);
     g_test_add_func("/bdrv-drain/blockjob/drain", test_blockjob_drain);
-- 
2.13.6

Since commit bde70715, base is the only node that is reopened in
commit_start(). This means that the code, which still involves an
explicit BlockReopenQueue, can now be simplified by using bdrv_reopen().

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Fam Zheng <famz@redhat.com>
---
 block/commit.c | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/block/commit.c b/block/commit.c
index XXXXXXX..XXXXXXX 100644
--- a/block/commit.c
+++ b/block/commit.c
@@ -XXX,XX +XXX,XX @@ void commit_start(const char *job_id, BlockDriverState *bs,
                   const char *filter_node_name, Error **errp)
 {
     CommitBlockJob *s;
-    BlockReopenQueue *reopen_queue = NULL;
     int orig_base_flags;
     BlockDriverState *iter;
     BlockDriverState *commit_top_bs = NULL;
@@ -XXX,XX +XXX,XX @@ void commit_start(const char *job_id, BlockDriverState *bs,
     /* convert base to r/w, if necessary */
     orig_base_flags = bdrv_get_flags(base);
     if (!(orig_base_flags & BDRV_O_RDWR)) {
-        reopen_queue = bdrv_reopen_queue(reopen_queue, base, NULL,
-                                         orig_base_flags | BDRV_O_RDWR);
-    }
-
-    if (reopen_queue) {
-        bdrv_reopen_multiple(bdrv_get_aio_context(bs), reopen_queue, &local_err);
+        bdrv_reopen(base, orig_base_flags | BDRV_O_RDWR, &local_err);
         if (local_err != NULL) {
             error_propagate(errp, local_err);
             goto fail;
-- 
2.13.6

The bdrv_reopen*() implementation doesn't like it if the graph is
changed between queuing nodes for reopen and actually reopening them
(one of the reasons is that queuing can be recursive).

So instead of draining the device only in bdrv_reopen_multiple(),
require that callers already drained all affected nodes, and assert this
in bdrv_reopen_queue().

Signed-off-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Fam Zheng <famz@redhat.com>
---
 block.c             | 23 ++++++++++++++++-------
 block/replication.c |  6 ++++++
 qemu-io-cmds.c      |  3 +++
 3 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/block.c b/block.c
index XXXXXXX..XXXXXXX 100644
--- a/block.c
+++ b/block.c
@@ -XXX,XX +XXX,XX @@ BlockDriverState *bdrv_open(const char *filename, const char *reference,
  * returns a pointer to bs_queue, which is either the newly allocated
  * bs_queue, or the existing bs_queue being used.
  *
+ * bs must be drained between bdrv_reopen_queue() and bdrv_reopen_multiple().
  */
 static BlockReopenQueue *bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
                                                  BlockDriverState *bs,
@@ -XXX,XX +XXX,XX @@ static BlockReopenQueue *bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
     BdrvChild *child;
     QDict *old_options, *explicit_options;
 
+    /* Make sure that the caller remembered to use a drained section. This is
+     * important to avoid graph changes between the recursive queuing here and
+     * bdrv_reopen_multiple(). */
+    assert(bs->quiesce_counter > 0);
+
     if (bs_queue == NULL) {
         bs_queue = g_new0(BlockReopenQueue, 1);
         QSIMPLEQ_INIT(bs_queue);
@@ -XXX,XX +XXX,XX @@ BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
  * If all devices prepare successfully, then the changes are committed
  * to all devices.
  *
+ * All affected nodes must be drained between bdrv_reopen_queue() and
+ * bdrv_reopen_multiple().
  */
 int bdrv_reopen_multiple(AioContext *ctx, BlockReopenQueue *bs_queue, Error **errp)
 {
@@ -XXX,XX +XXX,XX @@ int bdrv_reopen_multiple(AioContext *ctx, BlockReopenQueue *bs_queue, Error **er
 
     assert(bs_queue != NULL);
 
-    aio_context_release(ctx);
-    bdrv_drain_all_begin();
-    aio_context_acquire(ctx);
-
     QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+        assert(bs_entry->state.bs->quiesce_counter > 0);
         if (bdrv_reopen_prepare(&bs_entry->state, bs_queue, &local_err)) {
             error_propagate(errp, local_err);
             goto cleanup;
@@ -XXX,XX +XXX,XX @@ cleanup:
     }
     g_free(bs_queue);
 
-    bdrv_drain_all_end();
-
     return ret;
 }
 
@@ -XXX,XX +XXX,XX @@ int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp)
 {
     int ret = -1;
     Error *local_err = NULL;
-    BlockReopenQueue *queue = bdrv_reopen_queue(NULL, bs, NULL, bdrv_flags);
+    BlockReopenQueue *queue;
 
+    bdrv_subtree_drained_begin(bs);
+
+    queue = bdrv_reopen_queue(NULL, bs, NULL, bdrv_flags);
     ret = bdrv_reopen_multiple(bdrv_get_aio_context(bs), queue, &local_err);
     if (local_err != NULL) {
         error_propagate(errp, local_err);
     }
+
+    bdrv_subtree_drained_end(bs);
+
     return ret;
 }
 
diff --git a/block/replication.c b/block/replication.c
index XXXXXXX..XXXXXXX 100644
--- a/block/replication.c
+++ b/block/replication.c
@@ -XXX,XX +XXX,XX @@ static void reopen_backing_file(BlockDriverState *bs, bool writable,
         new_secondary_flags = s->orig_secondary_flags;
     }
 
+    bdrv_subtree_drained_begin(s->hidden_disk->bs);
+    bdrv_subtree_drained_begin(s->secondary_disk->bs);
+
     if (orig_hidden_flags != new_hidden_flags) {
         reopen_queue = bdrv_reopen_queue(reopen_queue, s->hidden_disk->bs, NULL,
                                          new_hidden_flags);
@@ -XXX,XX +XXX,XX @@ static void reopen_backing_file(BlockDriverState *bs, bool writable,
                              reopen_queue, &local_err);
         error_propagate(errp, local_err);
     }
+
+    bdrv_subtree_drained_end(s->hidden_disk->bs);
+    bdrv_subtree_drained_end(s->secondary_disk->bs);
 }
 
 static void backup_job_cleanup(BlockDriverState *bs)
diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c
index XXXXXXX..XXXXXXX 100644
--- a/qemu-io-cmds.c
+++ b/qemu-io-cmds.c
@@ -XXX,XX +XXX,XX @@ static int reopen_f(BlockBackend *blk, int argc, char **argv)
     opts = qopts ? qemu_opts_to_qdict(qopts, NULL) : NULL;
     qemu_opts_reset(&reopen_opts);
 
+    bdrv_subtree_drained_begin(bs);
     brq = bdrv_reopen_queue(NULL, bs, opts, flags);
     bdrv_reopen_multiple(bdrv_get_aio_context(bs), brq, &local_err);
+    bdrv_subtree_drained_end(bs);
+
     if (local_err) {
         error_report_err(local_err);
     } else {
-- 
2.13.6