Series comparison

-[Qemu-devel] [PULL 00/23] Block patches
+[PULL 00/19] Block patches
-The following changes since commit 411ad78115ebeb3411cf4b7622784b93dfabe259:
+The following changes since commit ba29883206d92a29ad5a466e679ccfc2ee6132ef:
-  Merge remote-tracking branch 'remotes/stefanberger/tags/pull-tpm-2017-12-15-1' into staging (2017-12-17 15:27:41 +0000)
+  Merge remote-tracking branch 'remotes/borntraeger/tags/s390x-20200310' into staging (2020-03-10 16:50:28 +0000)
 are available in the Git repository at:
-  git://github.com/stefanha/qemu.git tags/block-pull-request
+  https://github.com/XanClic/qemu.git tags/pull-block-2020-03-11
-for you to fetch changes up to 585426c518958aa768564596091474be786aae51:
+for you to fetch changes up to 397f4e9d83e9c0000905f0a988ba1aeda162571c:
-  qemu-iotests: add 203 savevm with IOThreads test (2017-12-18 13:12:53 +0000)
+  block/block-copy: hide structure definitions (2020-03-11 12:42:30 +0100)
 ----------------------------------------------------------------
+Block patches for the 5.0 softfreeze:
+- qemu-img measure for LUKS
+- Improve block-copy's performance by reducing inter-request
+  dependencies
+- Make curl's detection of accept-ranges more robust
+- Memleak fixes
+- iotest fix
 ----------------------------------------------------------------
+David Edmondson (2):
+  block/curl: HTTP header fields allow whitespace around values
+  block/curl: HTTP header field names are case insensitive
-Mao Zhongyi (4):
+Eric Blake (1):
-  hw/block/nvme: Convert to realize
+  iotests: Fix nonportable use of od --endian
   hw/block: Fix the return type
   hw/block: Use errp directly rather than local_err
   dev-storage: Fix the unusual function name
-Mark Kanda (2):
+Pan Nengyuan (2):
-  virtio-blk: make queue size configurable
+  block/qcow2: do free crypto_opts in qcow2_close()
-  virtio-blk: reject configs with logical block size > physical block
+  qemu-img: free memory before re-assign
     size
-Paolo Bonzini (1):
+Stefan Hajnoczi (4):
-  block: avoid recursive AioContext acquire in bdrv_inactivate_all()
+  luks: extract qcrypto_block_calculate_payload_offset()
   luks: implement .bdrv_measure()
   qemu-img: allow qemu-img measure --object without a filename
   iotests: add 288 luks qemu-img measure test
-Stefan Hajnoczi (16):
+Vladimir Sementsov-Ogievskiy (10):
-  coroutine: simplify co_aio_sleep_ns() prototype
+  block/qcow2-threads: fix qcow2_decompress
-  qdev: drop unused #include "sysemu/iothread.h"
+  job: refactor progress to separate object
-  blockdev: hold AioContext for bdrv_unref() in
+  block/block-copy: fix progress calculation
-    external_snapshot_clean()
+  block/block-copy: specialcase first copy_range request
-  block: don't keep AioContext acquired after
+  block/block-copy: use block_status
-    external_snapshot_prepare()
+  block/block-copy: factor out find_conflicting_inflight_req
-  block: don't keep AioContext acquired after drive_backup_prepare()
+  block/block-copy: refactor interfaces to use bytes instead of end
-  block: don't keep AioContext acquired after blockdev_backup_prepare()
+  block/block-copy: rename start to offset in interfaces
-  block: don't keep AioContext acquired after
+  block/block-copy: reduce intersecting request lock
-    internal_snapshot_prepare()
+  block/block-copy: hide structure definitions
   block: drop unused BlockDirtyBitmapState->aio_context field
   iothread: add iothread_by_id() API
   blockdev: add x-blockdev-set-iothread testing command
   qemu-iotests: add 202 external snapshots IOThread test
   docs: mark nested AioContext locking as a legacy API
   blockdev: add x-blockdev-set-iothread force boolean
   iotests: add VM.add_object()
   iothread: fix iothread_stop() race condition
   qemu-iotests: add 203 savevm with IOThreads test
- docs/devel/multiple-iothreads.txt |   7 +-
+ block/backup-top.c               |   6 +-
- qapi/block-core.json              |  40 ++++++
+ block/backup.c                   |  38 ++-
- hw/block/dataplane/virtio-blk.h   |   2 +-
+ block/block-copy.c               | 405 ++++++++++++++++++++++++-------
- include/hw/block/block.h          |   4 +-
+ block/crypto.c                   |  62 +++++
- include/hw/virtio/virtio-blk.h    |   1 +
+ block/curl.c                     |  32 ++-
- include/qemu/coroutine.h          |   6 +-
+ block/qcow2-threads.c            |  12 +-
- include/sysemu/iothread.h         |   4 +-
+ block/qcow2.c                    |  75 ++----
- block.c                           |  14 ++-
+ block/trace-events               |   1 +
- block/null.c                      |   3 +-
+ blockjob.c                       |  16 +-
- block/sheepdog.c                  |   3 +-
+ crypto/block.c                   |  36 +++
- blockdev.c                        | 259 +++++++++++++++++++++++++++-----------
+ include/block/block-copy.h       |  65 +----
- hw/block/block.c                  |  15 ++-
+ include/crypto/block.h           |  22 ++
- hw/block/dataplane/virtio-blk.c   |  12 +-
+ include/qemu/job.h               |  11 +-
- hw/block/fdc.c                    |  17 +--
+ include/qemu/progress_meter.h    |  58 +++++
- hw/block/nvme.c                   |  23 ++--
+ job-qmp.c                        |   4 +-
- hw/block/virtio-blk.c             |  35 ++++--
+ job.c                            |   6 +-
- hw/core/qdev-properties-system.c  |   1 -
+ qemu-img.c                       |  14 +-
- hw/ide/qdev.c                     |  12 +-
+ tests/qemu-iotests/178           |   2 +-
- hw/scsi/scsi-disk.c               |  13 +-
+ tests/qemu-iotests/178.out.qcow2 |   8 +-
- hw/usb/dev-storage.c              |  29 ++---
+ tests/qemu-iotests/178.out.raw   |   8 +-
- iothread.c                        |  27 +++-
+ tests/qemu-iotests/288           |  93 +++++++
- util/qemu-coroutine-sleep.c       |   4 +-
+ tests/qemu-iotests/288.out       |  30 +++
- tests/qemu-iotests/202            |  95 ++++++++++++++
+ tests/qemu-iotests/common.rc     |  22 +-
- tests/qemu-iotests/202.out        |  11 ++
+ tests/qemu-iotests/group         |   1 +
- tests/qemu-iotests/203            |  59 +++++++++
+files changed, 749 insertions(+), 278 deletions(-)
- tests/qemu-iotests/203.out        |   6 +
+ create mode 100644 include/qemu/progress_meter.h
- tests/qemu-iotests/group          |   2 +
+ create mode 100755 tests/qemu-iotests/288
- tests/qemu-iotests/iotests.py     |   5 +
+ create mode 100644 tests/qemu-iotests/288.out
 files changed, 532 insertions(+), 177 deletions(-)
  create mode 100755 tests/qemu-iotests/202
  create mode 100644 tests/qemu-iotests/202.out
  create mode 100755 tests/qemu-iotests/203
  create mode 100644 tests/qemu-iotests/203.out
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 01/23] coroutine: simplify co_aio_sleep_ns() prototype
+Deleted patch
-The AioContext pointer argument to co_aio_sleep_ns() is only used for
-the sleep timer.  It does not affect where the caller coroutine is
-resumed.
-Due to changes to coroutine and AIO APIs it is now possible to drop the
-AioContext pointer argument.  This is safe to do since no caller has
-specific requirements for which AioContext the timer must run in.
-This patch drops the AioContext pointer argument and renames the
-function to simplify the API.
-Reported-by: Paolo Bonzini <pbonzini@redhat.com>
-Reported-by: Eric Blake <eblake@redhat.com>
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
-Reviewed-by: Eric Blake <eblake@redhat.com>
-Message-id: 20171109102652.6360-1-stefanha@redhat.com
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
----
- include/qemu/coroutine.h    | 6 +-----
- block/null.c                | 3 +--
- block/sheepdog.c            | 3 +--
- util/qemu-coroutine-sleep.c | 4 ++--
-files changed, 5 insertions(+), 11 deletions(-)
-diff --git a/include/qemu/coroutine.h b/include/qemu/coroutine.h
-index XXXXXXX..XXXXXXX 100644
---- a/include/qemu/coroutine.h
-+++ b/include/qemu/coroutine.h
-@@ -XXX,XX +XXX,XX @@ void qemu_co_rwlock_unlock(CoRwlock *lock);
- /**
-  * Yield the coroutine for a given duration
-- *
-- * Behaves similarly to co_sleep_ns(), but the sleeping coroutine will be
-- * resumed when using aio_poll().
-  */
--void coroutine_fn co_aio_sleep_ns(AioContext *ctx, QEMUClockType type,
--                                  int64_t ns);
-+void coroutine_fn qemu_co_sleep_ns(QEMUClockType type, int64_t ns);
- /**
-  * Yield until a file descriptor becomes readable
-diff --git a/block/null.c b/block/null.c
-index XXXXXXX..XXXXXXX 100644
---- a/block/null.c
-+++ b/block/null.c
-@@ -XXX,XX +XXX,XX @@ static coroutine_fn int null_co_common(BlockDriverState *bs)
-     BDRVNullState *s = bs->opaque;
-     if (s->latency_ns) {
--        co_aio_sleep_ns(bdrv_get_aio_context(bs), QEMU_CLOCK_REALTIME,
--                        s->latency_ns);
-+        qemu_co_sleep_ns(QEMU_CLOCK_REALTIME, s->latency_ns);
-     }
-     return 0;
- }
-diff --git a/block/sheepdog.c b/block/sheepdog.c
-index XXXXXXX..XXXXXXX 100644
---- a/block/sheepdog.c
-+++ b/block/sheepdog.c
-@@ -XXX,XX +XXX,XX @@ static coroutine_fn void reconnect_to_sdog(void *opaque)
-         if (s->fd < 0) {
-             DPRINTF("Wait for connection to be established\n");
-             error_report_err(local_err);
--            co_aio_sleep_ns(bdrv_get_aio_context(s->bs), QEMU_CLOCK_REALTIME,
--                            1000000000ULL);
-+            qemu_co_sleep_ns(QEMU_CLOCK_REALTIME, 1000000000ULL);
-         }
-     };
-diff --git a/util/qemu-coroutine-sleep.c b/util/qemu-coroutine-sleep.c
-index XXXXXXX..XXXXXXX 100644
---- a/util/qemu-coroutine-sleep.c
-+++ b/util/qemu-coroutine-sleep.c
-@@ -XXX,XX +XXX,XX @@ static void co_sleep_cb(void *opaque)
-     aio_co_wake(sleep_cb->co);
- }
--void coroutine_fn co_aio_sleep_ns(AioContext *ctx, QEMUClockType type,
--                                  int64_t ns)
-+void coroutine_fn qemu_co_sleep_ns(QEMUClockType type, int64_t ns)
- {
-+    AioContext *ctx = qemu_get_current_aio_context();
-     CoSleepCB sleep_cb = {
-         .co = qemu_coroutine_self(),
-     };
---
-.14.3

-[Qemu-devel] [PULL 12/23] block: drop unused BlockDirtyBitmapState->aio_context field
+[PULL 01/19] luks: extract qcrypto_block_calculate_payload_offset()
-The dirty bitmap actions in qmp_transaction have not used AioContext
+From: Stefan Hajnoczi <stefanha@redhat.com>
-since the dirty bitmap locking discipline was introduced in commit
-2119882c7eb7e2c612b24fc0c8d86f5887d6f1c3 ("block: introduce
+The qcow2 .bdrv_measure() code calculates the crypto payload offset.
-dirty_bitmap_mutex").  Remove the unused field.
+This logic really belongs in crypto/block.c where it can be reused by
 other image formats.
 The "luks" block driver will need this same logic in order to implement
 .bdrv_measure(), so extract the qcrypto_block_calculate_payload_offset()
 function now.
 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
-Reviewed-by: Kevin Wolf <kwolf@redhat.com>
+Reviewed-by: Max Reitz <mreitz@redhat.com>
-Reviewed-by: Eric Blake <eblake@redhat.com>
+Message-Id: <20200221112522.1497712-2-stefanha@redhat.com>
-Message-id: 20171206144550.22295-7-stefanha@redhat.com
+Signed-off-by: Max Reitz <mreitz@redhat.com>
 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
 ---
- blockdev.c | 13 -------------
+ block/qcow2.c          | 74 +++++++++++-------------------------------
-file changed, 13 deletions(-)
+ crypto/block.c         | 36 ++++++++++++++++++++
+ include/crypto/block.h | 22 +++++++++++++
-diff --git a/blockdev.c b/blockdev.c
+files changed, 77 insertions(+), 55 deletions(-)
 diff --git a/block/qcow2.c b/block/qcow2.c
 index XXXXXXX..XXXXXXX 100644
---- a/blockdev.c
+--- a/block/qcow2.c
-+++ b/blockdev.c
++++ b/block/qcow2.c
-@@ -XXX,XX +XXX,XX @@ typedef struct BlockDirtyBitmapState {
+@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_flush_to_os(BlockDriverState *bs)
-     BlkActionState common;
+     return ret;
      BdrvDirtyBitmap *bitmap;
      BlockDriverState *bs;
 -    AioContext *aio_context;
      HBitmap *backup;
      bool prepared;
  } BlockDirtyBitmapState;
@@ -XXX,XX +XXX,XX @@ static void block_dirty_bitmap_clear_prepare(BlkActionState *common,
      }
      bdrv_clear_dirty_bitmap(state->bitmap, &state->backup);
 -    /* AioContext is released in .clean() */
  }
- static void block_dirty_bitmap_clear_abort(BlkActionState *common)
+-static ssize_t qcow2_measure_crypto_hdr_init_func(QCryptoBlock *block,
-@@ -XXX,XX +XXX,XX @@ static void block_dirty_bitmap_clear_commit(BlkActionState *common)
+-        size_t headerlen, void *opaque, Error **errp)
-     hbitmap_free(state->backup);
+-{
 -    size_t *headerlenp = opaque;
 -
 -    /* Stash away the payload size */
 -    *headerlenp = headerlen;
 -    return 0;
 -}
 -
 -static ssize_t qcow2_measure_crypto_hdr_write_func(QCryptoBlock *block,
 -        size_t offset, const uint8_t *buf, size_t buflen,
 -        void *opaque, Error **errp)
 -{
 -    /* Discard the bytes, we're not actually writing to an image */
 -    return buflen;
 -}
 -
 -/* Determine the number of bytes for the LUKS payload */
 -static bool qcow2_measure_luks_headerlen(QemuOpts *opts, size_t *len,
 -                                         Error **errp)
 -{
 -    QDict *opts_qdict;
 -    QDict *cryptoopts_qdict;
 -    QCryptoBlockCreateOptions *cryptoopts;
 -    QCryptoBlock *crypto;
 -
 -    /* Extract "encrypt." options into a qdict */
 -    opts_qdict = qemu_opts_to_qdict(opts, NULL);
 -    qdict_extract_subqdict(opts_qdict, &cryptoopts_qdict, "encrypt.");
 -    qobject_unref(opts_qdict);
 -
 -    /* Build QCryptoBlockCreateOptions object from qdict */
 -    qdict_put_str(cryptoopts_qdict, "format", "luks");
 -    cryptoopts = block_crypto_create_opts_init(cryptoopts_qdict, errp);
 -    qobject_unref(cryptoopts_qdict);
 -    if (!cryptoopts) {
 -        return false;
 -    }
 -
 -    /* Fake LUKS creation in order to determine the payload size */
 -    crypto = qcrypto_block_create(cryptoopts, "encrypt.",
 -                                  qcow2_measure_crypto_hdr_init_func,
 -                                  qcow2_measure_crypto_hdr_write_func,
 -                                  len, errp);
 -    qapi_free_QCryptoBlockCreateOptions(cryptoopts);
 -    if (!crypto) {
 -        return false;
 -    }
 -
 -    qcrypto_block_free(crypto);
 -    return true;
 -}
 -
  static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs,
                                         Error **errp)
  {
@@ -XXX,XX +XXX,XX @@ static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs,
      g_free(optstr);
      if (has_luks) {
 +        g_autoptr(QCryptoBlockCreateOptions) create_opts = NULL;
 +        QDict *opts_qdict;
 +        QDict *cryptoopts;
          size_t headerlen;
 -        if (!qcow2_measure_luks_headerlen(opts, &headerlen, &local_err)) {
 +        opts_qdict = qemu_opts_to_qdict(opts, NULL);
 +        qdict_extract_subqdict(opts_qdict, &cryptoopts, "encrypt.");
 +        qobject_unref(opts_qdict);
 +
 +        qdict_put_str(cryptoopts, "format", "luks");
 +
 +        create_opts = block_crypto_create_opts_init(cryptoopts, errp);
 +        qobject_unref(cryptoopts);
 +        if (!create_opts) {
 +            goto err;
 +        }
 +
 +        if (!qcrypto_block_calculate_payload_offset(create_opts,
 +                                                    "encrypt.",
 +                                                    &headerlen,
 +                                                    &local_err)) {
              goto err;
          }
 diff --git a/crypto/block.c b/crypto/block.c
 index XXXXXXX..XXXXXXX 100644
 --- a/crypto/block.c
 +++ b/crypto/block.c
@@ -XXX,XX +XXX,XX @@ QCryptoBlock *qcrypto_block_create(QCryptoBlockCreateOptions *options,
  }
--static void block_dirty_bitmap_clear_clean(BlkActionState *common)
--{
++static ssize_t qcrypto_block_headerlen_hdr_init_func(QCryptoBlock *block,
--    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
++        size_t headerlen, void *opaque, Error **errp)
--                                             common, common);
++{
--
++    size_t *headerlenp = opaque;
--    if (state->aio_context) {
++
--        aio_context_release(state->aio_context);
++    /* Stash away the payload size */
--    }
++    *headerlenp = headerlen;
--}
++    return 0;
--
++}
- static void abort_prepare(BlkActionState *common, Error **errp)
++
 +
 +static ssize_t qcrypto_block_headerlen_hdr_write_func(QCryptoBlock *block,
 +        size_t offset, const uint8_t *buf, size_t buflen,
 +        void *opaque, Error **errp)
 +{
 +    /* Discard the bytes, we're not actually writing to an image */
 +    return buflen;
 +}
 +
 +
 +bool
 +qcrypto_block_calculate_payload_offset(QCryptoBlockCreateOptions *create_opts,
 +                                       const char *optprefix,
 +                                       size_t *len,
 +                                       Error **errp)
 +{
 +    /* Fake LUKS creation in order to determine the payload size */
 +    g_autoptr(QCryptoBlock) crypto =
 +        qcrypto_block_create(create_opts, optprefix,
 +                             qcrypto_block_headerlen_hdr_init_func,
 +                             qcrypto_block_headerlen_hdr_write_func,
 +                             len, errp);
 +    return crypto != NULL;
 +}
 +
 +
  QCryptoBlockInfo *qcrypto_block_get_info(QCryptoBlock *block,
                                           Error **errp)
  {
-     error_setg(errp, "Transaction aborted using Abort action");
+diff --git a/include/crypto/block.h b/include/crypto/block.h
-@@ -XXX,XX +XXX,XX @@ static const BlkActionOps actions[] = {
+index XXXXXXX..XXXXXXX 100644
-         .prepare = block_dirty_bitmap_clear_prepare,
+--- a/include/crypto/block.h
-         .commit = block_dirty_bitmap_clear_commit,
++++ b/include/crypto/block.h
-         .abort = block_dirty_bitmap_clear_abort,
+@@ -XXX,XX +XXX,XX @@ QCryptoBlock *qcrypto_block_create(QCryptoBlockCreateOptions *options,
--        .clean = block_dirty_bitmap_clear_clean,
+                                    Error **errp);
-     }
- };
++/**
 + * qcrypto_block_calculate_payload_offset:
 + * @create_opts: the encryption options
 + * @optprefix: name prefix for options
 + * @len: output for number of header bytes before payload
 + * @errp: pointer to a NULL-initialized error object
 + *
 + * Calculate the number of header bytes before the payload in an encrypted
 + * storage volume.  The header is an area before the payload that is reserved
 + * for encryption metadata.
 + *
 + * Returns: true on success, false on error
 + */
 +bool
 +qcrypto_block_calculate_payload_offset(QCryptoBlockCreateOptions *create_opts,
 +                                       const char *optprefix,
 +                                       size_t *len,
 +                                       Error **errp);
 +
 +
  /**
   * qcrypto_block_get_info:
   * @block: the block encryption object
@@ -XXX,XX +XXX,XX @@ uint64_t qcrypto_block_get_sector_size(QCryptoBlock *block);
  void qcrypto_block_free(QCryptoBlock *block);
  G_DEFINE_AUTOPTR_CLEANUP_FUNC(QCryptoBlock, qcrypto_block_free)
 +G_DEFINE_AUTOPTR_CLEANUP_FUNC(QCryptoBlockCreateOptions,
 +                              qapi_free_QCryptoBlockCreateOptions)
  #endif /* QCRYPTO_BLOCK_H */
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 20/23] blockdev: add x-blockdev-set-iothread force boolean
+[PULL 02/19] luks: implement .bdrv_measure()
-When a node is already associated with a BlockBackend the
+From: Stefan Hajnoczi <stefanha@redhat.com>
 x-blockdev-set-iothread command refuses to set the IOThread.  This is to
 prevent accidentally changing the IOThread when the nodes are in use.
-When the nodes are created with -drive they automatically get a
+Add qemu-img measure support in the "luks" block driver.
 BlockBackend.  In that case we know nothing is using them yet and it's
 safe to set the IOThread.  Add a force boolean to override the check.
 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
-Reviewed-by: Eric Blake <eblake@redhat.com>
+Reviewed-by: Max Reitz <mreitz@redhat.com>
-Message-id: 20171207201320.19284-4-stefanha@redhat.com
+Message-Id: <20200221112522.1497712-3-stefanha@redhat.com>
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- qapi/block-core.json |  6 +++++-
+ block/crypto.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++
- blockdev.c           | 11 ++++++-----
+file changed, 62 insertions(+)
 files changed, 11 insertions(+), 6 deletions(-)
-diff --git a/qapi/block-core.json b/qapi/block-core.json
+diff --git a/block/crypto.c b/block/crypto.c
 index XXXXXXX..XXXXXXX 100644
---- a/qapi/block-core.json
+--- a/block/crypto.c
-+++ b/qapi/block-core.json
++++ b/block/crypto.c
-@@ -XXX,XX +XXX,XX @@
+@@ -XXX,XX +XXX,XX @@ static int64_t block_crypto_getlength(BlockDriverState *bs)
  #
  # @iothread: the name of the IOThread object or null for the main loop
  #
 +# @force: true if the node and its children should be moved when a BlockBackend
 +#         is already attached
 +#
  # Note: this command is experimental and intended for test cases that need
  # control over IOThreads only.
  #
@@ -XXX,XX +XXX,XX @@
  ##
  { 'command': 'x-blockdev-set-iothread',
    'data' : { 'node-name': 'str',
 -             'iothread': 'StrOrNull' } }
 +             'iothread': 'StrOrNull',
 +             '*force': 'bool' } }
 diff --git a/blockdev.c b/blockdev.c
 index XXXXXXX..XXXXXXX 100644
 --- a/blockdev.c
 +++ b/blockdev.c
@@ -XXX,XX +XXX,XX @@ BlockJobInfoList *qmp_query_block_jobs(Error **errp)
  }
- void qmp_x_blockdev_set_iothread(const char *node_name, StrOrNull *iothread,
--                                 Error **errp)
++static BlockMeasureInfo *block_crypto_measure(QemuOpts *opts,
-+                                 bool has_force, bool force, Error **errp)
++                                              BlockDriverState *in_bs,
- {
++                                              Error **errp)
-     AioContext *old_context;
++{
-     AioContext *new_context;
++    g_autoptr(QCryptoBlockCreateOptions) create_opts = NULL;
-@@ -XXX,XX +XXX,XX @@ void qmp_x_blockdev_set_iothread(const char *node_name, StrOrNull *iothread,
++    Error *local_err = NULL;
-         return;
++    BlockMeasureInfo *info;
-     }
++    uint64_t size;
++    size_t luks_payload_size;
--    /* If we want to allow more extreme test scenarios this guard could be
++    QDict *cryptoopts;
--     * removed.  For now it protects against accidents. */
++
--    if (bdrv_has_blk(bs)) {
++    /*
--        error_setg(errp, "Node %s is in use", node_name);
++     * Preallocation mode doesn't affect size requirements but we must consume
-+    /* Protects against accidents. */
++     * the option.
-+    if (!(has_force && force) && bdrv_has_blk(bs)) {
++     */
-+        error_setg(errp, "Node %s is associated with a BlockBackend and could "
++    g_free(qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC));
-+                         "be in use (use force=true to override this check)",
++
-+                         node_name);
++    size = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);
-         return;
++
-     }
++    if (in_bs) {
 +        int64_t ssize = bdrv_getlength(in_bs);
 +
 +        if (ssize < 0) {
 +            error_setg_errno(&local_err, -ssize,
 +                             "Unable to get image virtual_size");
 +            goto err;
 +        }
 +
 +        size = ssize;
 +    }
 +
 +    cryptoopts = qemu_opts_to_qdict_filtered(opts, NULL,
 +            &block_crypto_create_opts_luks, true);
 +    qdict_put_str(cryptoopts, "format", "luks");
 +    create_opts = block_crypto_create_opts_init(cryptoopts, &local_err);
 +    qobject_unref(cryptoopts);
 +    if (!create_opts) {
 +        goto err;
 +    }
 +
 +    if (!qcrypto_block_calculate_payload_offset(create_opts, NULL,
 +                                                &luks_payload_size,
 +                                                &local_err)) {
 +        goto err;
 +    }
 +
 +    /*
 +     * Unallocated blocks are still encrypted so allocation status makes no
 +     * difference to the file size.
 +     */
 +    info = g_new(BlockMeasureInfo, 1);
 +    info->fully_allocated = luks_payload_size + size;
 +    info->required = luks_payload_size + size;
 +    return info;
 +
 +err:
 +    error_propagate(errp, local_err);
 +    return NULL;
 +}
 +
 +
  static int block_crypto_probe_luks(const uint8_t *buf,
                                     int buf_size,
                                     const char *filename) {
@@ -XXX,XX +XXX,XX @@ static BlockDriver bdrv_crypto_luks = {
      .bdrv_co_preadv     = block_crypto_co_preadv,
      .bdrv_co_pwritev    = block_crypto_co_pwritev,
      .bdrv_getlength     = block_crypto_getlength,
 +    .bdrv_measure       = block_crypto_measure,
      .bdrv_get_info      = block_crypto_get_info_luks,
      .bdrv_get_specific_info = block_crypto_get_specific_info_luks,
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 21/23] iotests: add VM.add_object()
+[PULL 03/19] qemu-img: allow qemu-img measure --object without a filename
-The VM.add_object() method can be used to add IOThreads or memory
+From: Stefan Hajnoczi <stefanha@redhat.com>
-backend objects.
 In most qemu-img sub-commands the --object option only makes sense when
 there is a filename.  qemu-img measure is an exception because objects
 may be referenced from the image creation options instead of an existing
 image file.  Allow --object without a filename.
 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
-Reviewed-by: Eric Blake <eblake@redhat.com>
+Reviewed-by: Max Reitz <mreitz@redhat.com>
-Message-id: 20171207201320.19284-5-stefanha@redhat.com
+Message-Id: <20200221112522.1497712-4-stefanha@redhat.com>
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- tests/qemu-iotests/iotests.py | 5 +++++
+ qemu-img.c                       | 6 ++----
-file changed, 5 insertions(+)
+ tests/qemu-iotests/178           | 2 +-
  tests/qemu-iotests/178.out.qcow2 | 8 ++++----
  tests/qemu-iotests/178.out.raw   | 8 ++++----
 files changed, 11 insertions(+), 13 deletions(-)
-diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
+diff --git a/qemu-img.c b/qemu-img.c
 index XXXXXXX..XXXXXXX 100644
---- a/tests/qemu-iotests/iotests.py
+--- a/qemu-img.c
-+++ b/tests/qemu-iotests/iotests.py
++++ b/qemu-img.c
-@@ -XXX,XX +XXX,XX @@ class VM(qtest.QEMUQtestMachine):
+@@ -XXX,XX +XXX,XX @@ static int img_measure(int argc, char **argv)
-                                  socket_scm_helper=socket_scm_helper)
+         filename = argv[optind];
-         self._num_drives = 0
+     }
-+    def add_object(self, opts):
+-    if (!filename &&
-+        self._args.append('-object')
+-        (object_opts || image_opts || fmt || snapshot_name || sn_opts)) {
-+        self._args.append(opts)
+-        error_report("--object, --image-opts, -f, and -l "
-+        return self
+-                     "require a filename argument.");
-+
++    if (!filename && (image_opts || fmt || snapshot_name || sn_opts)) {
-     def add_device(self, opts):
++        error_report("--image-opts, -f, and -l require a filename argument.");
-         self._args.append('-device')
+         goto out;
-         self._args.append(opts)
+     }
      if (filename && img_size != UINT64_MAX) {
 diff --git a/tests/qemu-iotests/178 b/tests/qemu-iotests/178
 index XXXXXXX..XXXXXXX 100755
 --- a/tests/qemu-iotests/178
 +++ b/tests/qemu-iotests/178
@@ -XXX,XX +XXX,XX @@ _make_test_img 1G
  $QEMU_IMG measure # missing arguments
  $QEMU_IMG measure --size 2G "$TEST_IMG" # only one allowed
  $QEMU_IMG measure "$TEST_IMG" a # only one filename allowed
 -$QEMU_IMG measure --object secret,id=sec0,data=MTIzNDU2,format=base64 # missing filename
 +$QEMU_IMG measure --object secret,id=sec0,data=MTIzNDU2,format=base64 # size or filename needed
  $QEMU_IMG measure --image-opts # missing filename
  $QEMU_IMG measure -f qcow2 # missing filename
  $QEMU_IMG measure -l snap1 # missing filename
 diff --git a/tests/qemu-iotests/178.out.qcow2 b/tests/qemu-iotests/178.out.qcow2
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/178.out.qcow2
 +++ b/tests/qemu-iotests/178.out.qcow2
@@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
  qemu-img: Either --size N or one filename must be specified.
  qemu-img: --size N cannot be used together with a filename.
  qemu-img: At most one filename argument is allowed.
 -qemu-img: --object, --image-opts, -f, and -l require a filename argument.
 -qemu-img: --object, --image-opts, -f, and -l require a filename argument.
 -qemu-img: --object, --image-opts, -f, and -l require a filename argument.
 -qemu-img: --object, --image-opts, -f, and -l require a filename argument.
 +qemu-img: Either --size N or one filename must be specified.
 +qemu-img: --image-opts, -f, and -l require a filename argument.
 +qemu-img: --image-opts, -f, and -l require a filename argument.
 +qemu-img: --image-opts, -f, and -l require a filename argument.
  qemu-img: Invalid option list: ,
  qemu-img: Invalid parameter 'snapshot.foo'
  qemu-img: Failed in parsing snapshot param 'snapshot.foo'
 diff --git a/tests/qemu-iotests/178.out.raw b/tests/qemu-iotests/178.out.raw
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/178.out.raw
 +++ b/tests/qemu-iotests/178.out.raw
@@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
  qemu-img: Either --size N or one filename must be specified.
  qemu-img: --size N cannot be used together with a filename.
  qemu-img: At most one filename argument is allowed.
 -qemu-img: --object, --image-opts, -f, and -l require a filename argument.
 -qemu-img: --object, --image-opts, -f, and -l require a filename argument.
 -qemu-img: --object, --image-opts, -f, and -l require a filename argument.
 -qemu-img: --object, --image-opts, -f, and -l require a filename argument.
 +qemu-img: Either --size N or one filename must be specified.
 +qemu-img: --image-opts, -f, and -l require a filename argument.
 +qemu-img: --image-opts, -f, and -l require a filename argument.
 +qemu-img: --image-opts, -f, and -l require a filename argument.
  qemu-img: Invalid option list: ,
  qemu-img: Invalid parameter 'snapshot.foo'
  qemu-img: Failed in parsing snapshot param 'snapshot.foo'
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 23/23] qemu-iotests: add 203 savevm with IOThreads test
+[PULL 04/19] iotests: add 288 luks qemu-img measure test
-This test case will prevent future regressions with savevm and
+From: Stefan Hajnoczi <stefanha@redhat.com>
-IOThreads.
 This test exercises the block/crypto.c "luks" block driver
 .bdrv_measure() code.
 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
-Reviewed-by: Eric Blake <eblake@redhat.com>
+Reviewed-by: Max Reitz <mreitz@redhat.com>
-Message-id: 20171207201320.19284-7-stefanha@redhat.com
+Message-Id: <20200221112522.1497712-5-stefanha@redhat.com>
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+[mreitz: Renamed test from 282 to 288]
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- tests/qemu-iotests/203     | 59 ++++++++++++++++++++++++++++++++++++++++++++++
+ tests/qemu-iotests/288     | 93 ++++++++++++++++++++++++++++++++++++++
- tests/qemu-iotests/203.out |  6 +++++
+ tests/qemu-iotests/288.out | 30 ++++++++++++
  tests/qemu-iotests/group   |  1 +
-files changed, 66 insertions(+)
+files changed, 124 insertions(+)
- create mode 100755 tests/qemu-iotests/203
+ create mode 100755 tests/qemu-iotests/288
- create mode 100644 tests/qemu-iotests/203.out
+ create mode 100644 tests/qemu-iotests/288.out
-diff --git a/tests/qemu-iotests/203 b/tests/qemu-iotests/203
+diff --git a/tests/qemu-iotests/288 b/tests/qemu-iotests/288
 new file mode 100755
 index XXXXXXX..XXXXXXX
 --- /dev/null
-+++ b/tests/qemu-iotests/203
++++ b/tests/qemu-iotests/288
 @@ -XXX,XX +XXX,XX @@
-+#!/usr/bin/env python
++#!/usr/bin/env bash
 +#
-+# Copyright (C) 2017 Red Hat, Inc.
++# qemu-img measure tests for LUKS images
 +#
 +# Copyright (C) 2020 Red Hat, Inc.
 +#
 +# This program is free software; you can redistribute it and/or modify
 +# it under the terms of the GNU General Public License as published by
 +# the Free Software Foundation; either version 2 of the License, or
 +# (at your option) any later version.
 ...
 +# GNU General Public License for more details.
 +#
 +# You should have received a copy of the GNU General Public License
 +# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 +#
-+# Creator/Owner: Stefan Hajnoczi <stefanha@redhat.com>
-+#
-+# Check that QMP 'migrate' with multiple drives on a single IOThread completes
-+# successfully.  This particular command triggered a hang in the source QEMU
-+# process due to recursive AioContext locking in bdrv_invalidate_all() and
-+# BDRV_POLL_WHILE().
 +
-+import iotests
++# creator
 +owner=stefanha@redhat.com
 +
-+iotests.verify_image_format(supported_fmts=['qcow2'])
++seq=`basename $0`
-+iotests.verify_platform(['linux'])
++echo "QA output created by $seq"
 +
-+with iotests.FilePath('disk0.img') as disk0_img_path, \
++status=1    # failure is the default!
 +     iotests.FilePath('disk1.img') as disk1_img_path, \
 +     iotests.VM() as vm:
 +
-+    img_size = '10M'
++_cleanup()
-+    iotests.qemu_img_pipe('create', '-f', iotests.imgfmt, disk0_img_path, img_size)
++{
-+    iotests.qemu_img_pipe('create', '-f', iotests.imgfmt, disk1_img_path, img_size)
++    _cleanup_test_img
 +    rm -f "$TEST_IMG.converted"
 +}
 +trap "_cleanup; exit \$status" 0 1 2 3 15
 +
-+    iotests.log('Launching VM...')
++# get standard environment, filters and checks
-+    (vm.add_object('iothread,id=iothread0')
++. ./common.rc
-+       .add_drive(disk0_img_path, 'node-name=drive0-node', interface='none')
++. ./common.filter
-+       .add_drive(disk1_img_path, 'node-name=drive1-node', interface='none')
++. ./common.pattern
 +       .launch())
 +
-+    iotests.log('Setting IOThreads...')
++_supported_fmt luks
-+    iotests.log(vm.qmp('x-blockdev-set-iothread',
++_supported_proto file
-+                       node_name='drive0-node', iothread='iothread0',
++_supported_os Linux
 +                       force=True))
 +    iotests.log(vm.qmp('x-blockdev-set-iothread',
 +                       node_name='drive1-node', iothread='iothread0',
 +                       force=True))
 +
-+    iotests.log('Starting migration...')
++SECRET=secret,id=sec0,data=passphrase
-+    iotests.log(vm.qmp('migrate', uri='exec:cat >/dev/null'))
++
-+    while True:
++echo "== measure 1G image file =="
-+        vm.get_qmp_event(wait=60.0)
++echo
-+        result = vm.qmp('query-migrate')
++
-+        status = result.get('return', {}).get('status', None)
++$QEMU_IMG measure --object "$SECRET" \
-+        if status == 'completed':
++              -O "$IMGFMT" \
-+            break
++          -o key-secret=sec0,iter-time=10 \
-diff --git a/tests/qemu-iotests/203.out b/tests/qemu-iotests/203.out
++          --size 1G
 +
 +echo
 +echo "== create 1G image file (size should be no greater than measured) =="
 +echo
 +
 +_make_test_img 1G
 +stat -c "image file size in bytes: %s" "$TEST_IMG_FILE"
 +
 +echo
 +echo "== modified 1G image file (size should be no greater than measured) =="
 +echo
 +
 +$QEMU_IO --object "$SECRET" --image-opts "$TEST_IMG" -c "write -P 0x51 0x10000 0x400" | _filter_qemu_io | _filter_testdir
 +stat -c "image file size in bytes: %s" "$TEST_IMG_FILE"
 +
 +echo
 +echo "== measure preallocation=falloc 1G image file =="
 +echo
 +
 +$QEMU_IMG measure --object "$SECRET" \
 +              -O "$IMGFMT" \
 +          -o key-secret=sec0,iter-time=10,preallocation=falloc \
 +          --size 1G
 +
 +echo
 +echo "== measure with input image file =="
 +echo
 +
 +IMGFMT=raw IMGKEYSECRET= IMGOPTS= _make_test_img 1G | _filter_imgfmt
 +QEMU_IO_OPTIONS= IMGOPTSSYNTAX= $QEMU_IO -f raw -c "write -P 0x51 0x10000 0x400" "$TEST_IMG_FILE" | _filter_qemu_io | _filter_testdir
 +$QEMU_IMG measure --object "$SECRET" \
 +              -O "$IMGFMT" \
 +          -o key-secret=sec0,iter-time=10 \
 +          -f raw \
 +          "$TEST_IMG_FILE"
 +
 +# success, all done
 +echo "*** done"
 +rm -f $seq.full
 +status=0
 diff --git a/tests/qemu-iotests/288.out b/tests/qemu-iotests/288.out
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
-+++ b/tests/qemu-iotests/203.out
++++ b/tests/qemu-iotests/288.out
 @@ -XXX,XX +XXX,XX @@
-+Launching VM...
++QA output created by 288
-+Setting IOThreads...
++== measure 1G image file ==
-+{u'return': {}}
++
-+{u'return': {}}
++required size: 1075810304
-+Starting migration...
++fully allocated size: 1075810304
-+{u'return': {}}
++
 +== create 1G image file (size should be no greater than measured) ==
 +
 +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 +image file size in bytes: 1075810304
 +
 +== modified 1G image file (size should be no greater than measured) ==
 +
 +wrote 1024/1024 bytes at offset 65536
 +1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 +image file size in bytes: 1075810304
 +
 +== measure preallocation=falloc 1G image file ==
 +
 +required size: 1075810304
 +fully allocated size: 1075810304
 +
 +== measure with input image file ==
 +
 +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 +wrote 1024/1024 bytes at offset 65536
 +1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 +required size: 1075810304
 +fully allocated size: 1075810304
 +*** done
 diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
 index XXXXXXX..XXXXXXX 100644
 --- a/tests/qemu-iotests/group
 +++ b/tests/qemu-iotests/group
 @@ -XXX,XX +XXX,XX @@
-rw auto
+auto quick
-rw auto
+rw
-rw auto quick
+rw quick
-+203 rw auto
++288 quick
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 18/23] block: avoid recursive AioContext acquire in bdrv_inactivate_all()
+[PULL 05/19] block/curl: HTTP header fields allow whitespace around values
-From: Paolo Bonzini <pbonzini@redhat.com>
+From: David Edmondson <david.edmondson@oracle.com>
-BDRV_POLL_WHILE() does not support recursive AioContext locking.  It
+RFC 7230 section 3.2 indicates that whitespace is permitted between
-only releases the AioContext lock once regardless of how many times the
+the field name and field value and after the field value.
 caller has acquired it.  This results in a hang since the IOThread does
 not make progress while the AioContext is still locked.
-The following steps trigger the hang:
+Signed-off-by: David Edmondson <david.edmondson@oracle.com>
 Message-Id: <20200224101310.101169-2-david.edmondson@oracle.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
  block/curl.c | 31 +++++++++++++++++++++++++++----
 file changed, 27 insertions(+), 4 deletions(-)
-  $ qemu-system-x86_64 -M accel=kvm -m 1G -cpu host \
+diff --git a/block/curl.c b/block/curl.c
                        -object iothread,id=iothread0 \
                        -device virtio-scsi-pci,iothread=iothread0 \
                        -drive if=none,id=drive0,file=test.img,format=raw \
                        -device scsi-hd,drive=drive0 \
                        -drive if=none,id=drive1,file=test.img,format=raw \
                        -device scsi-hd,drive=drive1
   $ qemu-system-x86_64 ...same options... \
                        -incoming tcp::1234
   (qemu) migrate tcp:127.0.0.1:1234
   ...hang...
 Tested-by: Stefan Hajnoczi <stefanha@redhat.com>
 Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
 Reviewed-by: Eric Blake <eblake@redhat.com>
 Message-id: 20171207201320.19284-2-stefanha@redhat.com
 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
 ---
  block.c | 14 +++++++++++---
 file changed, 11 insertions(+), 3 deletions(-)
 diff --git a/block.c b/block.c
 index XXXXXXX..XXXXXXX 100644
---- a/block.c
+--- a/block/curl.c
-+++ b/block.c
++++ b/block/curl.c
-@@ -XXX,XX +XXX,XX @@ int bdrv_inactivate_all(void)
+@@ -XXX,XX +XXX,XX @@ static size_t curl_header_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
-     BdrvNextIterator it;
+ {
-     int ret = 0;
+     BDRVCURLState *s = opaque;
-     int pass;
+     size_t realsize = size * nmemb;
-+    GSList *aio_ctxs = NULL, *ctx;
+-    const char *accept_line = "Accept-Ranges: bytes";
++    const char *header = (char *)ptr;
-     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
++    const char *end = header + realsize;
--        aio_context_acquire(bdrv_get_aio_context(bs));
++    const char *accept_ranges = "Accept-Ranges:";
-+        AioContext *aio_context = bdrv_get_aio_context(bs);
++    const char *bytes = "bytes";
 -    if (realsize >= strlen(accept_line)
 -        && strncmp((char *)ptr, accept_line, strlen(accept_line)) == 0) {
 -        s->accept_range = true;
 +    if (realsize >= strlen(accept_ranges)
 +        && strncmp(header, accept_ranges, strlen(accept_ranges)) == 0) {
 +
-+        if (!g_slist_find(aio_ctxs, aio_context)) {
++        char *p = strchr(header, ':') + 1;
-+            aio_ctxs = g_slist_prepend(aio_ctxs, aio_context);
++
-+            aio_context_acquire(aio_context);
++        /* Skip whitespace between the header name and value. */
 +        while (p < end && *p && g_ascii_isspace(*p)) {
 +            p++;
 +        }
 +
 +        if (end - p >= strlen(bytes)
 +            && strncmp(p, bytes, strlen(bytes)) == 0) {
 +
 +            /* Check that there is nothing but whitespace after the value. */
 +            p += strlen(bytes);
 +            while (p < end && *p && g_ascii_isspace(*p)) {
 +                p++;
 +            }
 +
 +            if (p == end || !*p) {
 +                s->accept_range = true;
 +            }
 +        }
      }
-     /* We do two passes of inactivation. The first pass calls to drivers'
+     return realsize;
@@ -XXX,XX +XXX,XX @@ int bdrv_inactivate_all(void)
      }
  out:
 -    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
 -        aio_context_release(bdrv_get_aio_context(bs));
 +    for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) {
 +        AioContext *aio_context = ctx->data;
 +        aio_context_release(aio_context);
      }
 +    g_slist_free(aio_ctxs);
      return ret;
  }
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 17/23] virtio-blk: reject configs with logical block size > physical block size
+[PULL 06/19] block/curl: HTTP header field names are case insensitive
-From: Mark Kanda <mark.kanda@oracle.com>
+From: David Edmondson <david.edmondson@oracle.com>
-virtio-blk logical block size should never be larger than physical block
+RFC 7230 section 3.2 indicates that HTTP header field names are case
-size because it doesn't make sense to have such configurations. QEMU doesn't
+insensitive.
 have a way to effectively express this condition; the best it can do is
 report the physical block exponent as 0 - indicating the logical block size
 equals the physical block size.
-This is identical to commit 3da023b5827543ee4c022986ea2ad9d1274410b2
+Signed-off-by: David Edmondson <david.edmondson@oracle.com>
-but applied to virtio-blk (instead of virtio-scsi).
+Message-Id: <20200224101310.101169-3-david.edmondson@oracle.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
  block/curl.c | 5 +++--
 file changed, 3 insertions(+), 2 deletions(-)
-Signed-off-by: Mark Kanda <mark.kanda@oracle.com>
+diff --git a/block/curl.c b/block/curl.c
 Reviewed-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
 Reviewed-by: Ameya More <ameya.more@oracle.com>
 Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com>
 Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
 Message-id: 773169891f9f2deb4cb7c4ef2655580dbe24c1d1.1513005190.git.mark.kanda@oracle.com
 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
 ---
  hw/block/virtio-blk.c | 7 +++++++
 file changed, 7 insertions(+)
 diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
 index XXXXXXX..XXXXXXX 100644
---- a/hw/block/virtio-blk.c
+--- a/block/curl.c
-+++ b/hw/block/virtio-blk.c
++++ b/block/curl.c
-@@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
+@@ -XXX,XX +XXX,XX @@ static size_t curl_header_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
+     size_t realsize = size * nmemb;
-     blkconf_blocksizes(&conf->conf);
+     const char *header = (char *)ptr;
+     const char *end = header + realsize;
-+    if (conf->conf.logical_block_size >
+-    const char *accept_ranges = "Accept-Ranges:";
-+        conf->conf.physical_block_size) {
++    const char *accept_ranges = "accept-ranges:";
-+        error_setg(errp,
+     const char *bytes = "bytes";
-+                   "logical_block_size > physical_block_size not supported");
-+        return;
+     if (realsize >= strlen(accept_ranges)
-+    }
+-        && strncmp(header, accept_ranges, strlen(accept_ranges)) == 0) {
-+
++        && g_ascii_strncasecmp(header, accept_ranges,
-     virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK,
++                               strlen(accept_ranges)) == 0) {
-                 sizeof(struct virtio_blk_config));
          char *p = strchr(header, ':') + 1;
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 11/23] block: don't keep AioContext acquired after internal_snapshot_prepare()
+[PULL 07/19] iotests: Fix nonportable use of od --endian
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+From: Eric Blake <eblake@redhat.com>
-Reviewed-by: Kevin Wolf <kwolf@redhat.com>
-Reviewed-by: Eric Blake <eblake@redhat.com>
+Tests 261 and 272 fail on RHEL 7 with coreutils 8.22, since od
-Message-id: 20171206144550.22295-6-stefanha@redhat.com
+--endian was not added until coreutils 8.23.  Fix this by manually
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+constructing the final value one byte at a time.
 Fixes: fc8ba423
 Reported-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
 Signed-off-by: Eric Blake <eblake@redhat.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200226125424.481840-1-eblake@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- blockdev.c | 47 +++++++++++++++++++++++++++++++----------------
+ tests/qemu-iotests/common.rc | 22 +++++++++++++++++-----
-file changed, 31 insertions(+), 16 deletions(-)
+file changed, 17 insertions(+), 5 deletions(-)
-diff --git a/blockdev.c b/blockdev.c
+diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc
 index XXXXXXX..XXXXXXX 100644
---- a/blockdev.c
+--- a/tests/qemu-iotests/common.rc
-+++ b/blockdev.c
++++ b/tests/qemu-iotests/common.rc
-@@ -XXX,XX +XXX,XX @@ struct BlkActionState {
+@@ -XXX,XX +XXX,XX @@ poke_file()
- typedef struct InternalSnapshotState {
+ # peek_file_le 'test.img' 512 2 => 65534
-     BlkActionState common;
+ peek_file_le()
-     BlockDriverState *bs;
+ {
--    AioContext *aio_context;
+-    # Wrap in echo $() to strip spaces
-     QEMUSnapshotInfo sn;
+-    echo $(od -j"$2" -N"$3" --endian=little -An -vtu"$3" "$1")
-     bool created;
++    local val=0 shift=0 byte
  } InternalSnapshotState;
@@ -XXX,XX +XXX,XX @@ static void internal_snapshot_prepare(BlkActionState *common,
      qemu_timeval tv;
      BlockdevSnapshotInternal *internal;
      InternalSnapshotState *state;
 +    AioContext *aio_context;
      int ret1;
      g_assert(common->action->type ==
@@ -XXX,XX +XXX,XX @@ static void internal_snapshot_prepare(BlkActionState *common,
          return;
      }
 -    /* AioContext is released in .clean() */
 -    state->aio_context = bdrv_get_aio_context(bs);
 -    aio_context_acquire(state->aio_context);
 +    aio_context = bdrv_get_aio_context(bs);
 +    aio_context_acquire(aio_context);
      state->bs = bs;
 +
-+    /* Paired with .clean() */
++    # coreutils' od --endian is not portable, so manually assemble bytes.
-     bdrv_drained_begin(bs);
++    for byte in $(od -j"$2" -N"$3" -An -v -tu1 "$1"); do
++        val=$(( val | (byte << shift) ))
-     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT, errp)) {
++        shift=$((shift + 8))
--        return;
++    done
-+        goto out;
++    printf %llu $val
-     }
+ }
-     if (bdrv_is_read_only(bs)) {
+ # peek_file_be 'test.img' 512 2 => 65279
-         error_setg(errp, "Device '%s' is read only", device);
+ peek_file_be()
--        return;
+ {
-+        goto out;
+-    # Wrap in echo $() to strip spaces
-     }
+-    echo $(od -j"$2" -N"$3" --endian=big -An -vtu"$3" "$1")
++    local val=0 byte
      if (!bdrv_can_snapshot(bs)) {
          error_setg(errp, "Block format '%s' used by device '%s' "
                     "does not support internal snapshots",
                     bs->drv->format_name, device);
 -        return;
 +        goto out;
      }
      if (!strlen(name)) {
          error_setg(errp, "Name is empty");
 -        return;
 +        goto out;
      }
      /* check whether a snapshot with name exist */
@@ -XXX,XX +XXX,XX @@ static void internal_snapshot_prepare(BlkActionState *common,
                                              &local_err);
      if (local_err) {
          error_propagate(errp, local_err);
 -        return;
 +        goto out;
      } else if (ret) {
          error_setg(errp,
                     "Snapshot with name '%s' already exists on device '%s'",
                     name, device);
 -        return;
 +        goto out;
      }
      /* 3. take the snapshot */
@@ -XXX,XX +XXX,XX @@ static void internal_snapshot_prepare(BlkActionState *common,
          error_setg_errno(errp, -ret1,
                           "Failed to create snapshot '%s' on device '%s'",
                           name, device);
 -        return;
 +        goto out;
      }
      /* 4. succeed, mark a snapshot is created */
      state->created = true;
 +
-+out:
++    # coreutils' od --endian is not portable, so manually assemble bytes.
-+    aio_context_release(aio_context);
++    for byte in $(od -j"$2" -N"$3" -An -v -tu1 "$1"); do
 +        val=$(( (val << 8) | byte ))
 +    done
 +    printf %llu $val
  }
- static void internal_snapshot_abort(BlkActionState *common)
+-# peek_file_raw 'test.img' 512 2 => '\xff\xfe'
-@@ -XXX,XX +XXX,XX @@ static void internal_snapshot_abort(BlkActionState *common)
++# peek_file_raw 'test.img' 512 2 => '\xff\xfe'. Do not use if the raw data
-                              DO_UPCAST(InternalSnapshotState, common, common);
++# is likely to contain \0 or trailing \n.
-     BlockDriverState *bs = state->bs;
+ peek_file_raw()
      QEMUSnapshotInfo *sn = &state->sn;
 +    AioContext *aio_context;
      Error *local_error = NULL;
      if (!state->created) {
          return;
      }
 +    aio_context = bdrv_get_aio_context(state->bs);
 +    aio_context_acquire(aio_context);
 +
      if (bdrv_snapshot_delete(bs, sn->id_str, sn->name, &local_error) < 0) {
          error_reportf_err(local_error,
                            "Failed to delete snapshot with id '%s' and "
@@ -XXX,XX +XXX,XX @@ static void internal_snapshot_abort(BlkActionState *common)
                            sn->id_str, sn->name,
                            bdrv_get_device_name(bs));
      }
 +
 +    aio_context_release(aio_context);
  }
  static void internal_snapshot_clean(BlkActionState *common)
  {
-     InternalSnapshotState *state = DO_UPCAST(InternalSnapshotState,
+     dd if="$1" bs=1 skip="$2" count="$3" status=none
                                               common, common);
 +    AioContext *aio_context;
 -    if (state->aio_context) {
 -        if (state->bs) {
 -            bdrv_drained_end(state->bs);
 -        }
 -        aio_context_release(state->aio_context);
 +    if (!state->bs) {
 +        return;
      }
 +
 +    aio_context = bdrv_get_aio_context(state->bs);
 +    aio_context_acquire(aio_context);
 +
 +    bdrv_drained_end(state->bs);
 +
 +    aio_context_release(aio_context);
  }
  /* external snapshot private data */
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 19/23] docs: mark nested AioContext locking as a legacy API
+[PULL 08/19] block/qcow2: do free crypto_opts in qcow2_close()
-See the patch for why nested AioContext locking is no longer allowed.
+From: Pan Nengyuan <pannengyuan@huawei.com>
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+'crypto_opts' forgot to free in qcow2_close(), this patch fix the bellow leak stack:
-Reviewed-by: Eric Blake <eblake@redhat.com>
-Message-id: 20171207201320.19284-3-stefanha@redhat.com
+Direct leak of 24 byte(s) in 1 object(s) allocated from:
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+    #0 0x7f0edd81f970 in __interceptor_calloc (/lib64/libasan.so.5+0xef970)
     #1 0x7f0edc6d149d in g_malloc0 (/lib64/libglib-2.0.so.0+0x5249d)
     #2 0x55d7eaede63d in qobject_input_start_struct /mnt/sdb/qemu-new/qemu_test/qemu/qapi/qobject-input-visitor.c:295
     #3 0x55d7eaed78b8 in visit_start_struct /mnt/sdb/qemu-new/qemu_test/qemu/qapi/qapi-visit-core.c:49
     #4 0x55d7eaf5140b in visit_type_QCryptoBlockOpenOptions qapi/qapi-visit-crypto.c:290
     #5 0x55d7eae43af3 in block_crypto_open_opts_init /mnt/sdb/qemu-new/qemu_test/qemu/block/crypto.c:163
     #6 0x55d7eacd2924 in qcow2_update_options_prepare /mnt/sdb/qemu-new/qemu_test/qemu/block/qcow2.c:1148
     #7 0x55d7eacd33f7 in qcow2_update_options /mnt/sdb/qemu-new/qemu_test/qemu/block/qcow2.c:1232
     #8 0x55d7eacd9680 in qcow2_do_open /mnt/sdb/qemu-new/qemu_test/qemu/block/qcow2.c:1512
     #9 0x55d7eacdc55e in qcow2_open_entry /mnt/sdb/qemu-new/qemu_test/qemu/block/qcow2.c:1792
     #10 0x55d7eacdc8fe in qcow2_open /mnt/sdb/qemu-new/qemu_test/qemu/block/qcow2.c:1819
     #11 0x55d7eac3742d in bdrv_open_driver /mnt/sdb/qemu-new/qemu_test/qemu/block.c:1317
     #12 0x55d7eac3e990 in bdrv_open_common /mnt/sdb/qemu-new/qemu_test/qemu/block.c:1575
     #13 0x55d7eac4442c in bdrv_open_inherit /mnt/sdb/qemu-new/qemu_test/qemu/block.c:3126
     #14 0x55d7eac45c3f in bdrv_open /mnt/sdb/qemu-new/qemu_test/qemu/block.c:3219
     #15 0x55d7ead8e8a4 in blk_new_open /mnt/sdb/qemu-new/qemu_test/qemu/block/block-backend.c:397
     #16 0x55d7eacde74c in qcow2_co_create /mnt/sdb/qemu-new/qemu_test/qemu/block/qcow2.c:3534
     #17 0x55d7eacdfa6d in qcow2_co_create_opts /mnt/sdb/qemu-new/qemu_test/qemu/block/qcow2.c:3668
     #18 0x55d7eac1c678 in bdrv_create_co_entry /mnt/sdb/qemu-new/qemu_test/qemu/block.c:485
     #19 0x55d7eb0024d2 in coroutine_trampoline /mnt/sdb/qemu-new/qemu_test/qemu/util/coroutine-ucontext.c:115
 Reported-by: Euler Robot <euler.robot@huawei.com>
 Signed-off-by: Pan Nengyuan <pannengyuan@huawei.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200227012950.12256-2-pannengyuan@huawei.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- docs/devel/multiple-iothreads.txt | 7 ++++---
+ block/qcow2.c | 1 +
-file changed, 4 insertions(+), 3 deletions(-)
+file changed, 1 insertion(+)
-diff --git a/docs/devel/multiple-iothreads.txt b/docs/devel/multiple-iothreads.txt
+diff --git a/block/qcow2.c b/block/qcow2.c
 index XXXXXXX..XXXXXXX 100644
---- a/docs/devel/multiple-iothreads.txt
+--- a/block/qcow2.c
-+++ b/docs/devel/multiple-iothreads.txt
++++ b/block/qcow2.c
-@@ -XXX,XX +XXX,XX @@
+@@ -XXX,XX +XXX,XX @@ static void qcow2_close(BlockDriverState *bs)
--Copyright (c) 2014 Red Hat Inc.
-+Copyright (c) 2014-2017 Red Hat Inc.
+     qcrypto_block_free(s->crypto);
+     s->crypto = NULL;
- This work is licensed under the terms of the GNU GPL, version 2 or later.  See
++    qapi_free_QCryptoBlockOpenOptions(s->crypto_opts);
- the COPYING file in the top-level directory.
-@@ -XXX,XX +XXX,XX @@ aio_context_acquire()/aio_context_release() for mutual exclusion.  Once the
+     g_free(s->unknown_header_fields);
- context is acquired no other thread can access it or run event loop iterations
+     cleanup_unknown_header_ext(bs);
  in this AioContext.
 -aio_context_acquire()/aio_context_release() calls may be nested.  This
 -means you can call them if you're not sure whether #2 applies.
 +Legacy code sometimes nests aio_context_acquire()/aio_context_release() calls.
 +Do not use nesting anymore, it is incompatible with the BDRV_POLL_WHILE() macro
 +used in the block layer and can lead to hangs.
  There is currently no lock ordering rule if a thread needs to acquire multiple
  AioContexts simultaneously.  Therefore, it is only safe for code holding the
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 16/23] virtio-blk: make queue size configurable
+[PULL 09/19] qemu-img: free memory before re-assign
-From: Mark Kanda <mark.kanda@oracle.com>
+From: Pan Nengyuan <pannengyuan@huawei.com>
-Depending on the configuration, it can be beneficial to adjust the virtio-blk
+collect_image_check() is called twice in img_check(), the filename/format will be alloced without free the original memory.
-queue size to something other than the current default of 128. Add a new
+It is not a big deal since the process will exit anyway, but seems like a clean code and it will remove the warning spotted by asan.
 property to make the queue size configurable.
-Signed-off-by: Mark Kanda <mark.kanda@oracle.com>
+Reported-by: Euler Robot <euler.robot@huawei.com>
-Reviewed-by: Karl Heubaum <karl.heubaum@oracle.com>
+Signed-off-by: Pan Nengyuan <pannengyuan@huawei.com>
-Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com>
+Message-Id: <20200227012950.12256-3-pannengyuan@huawei.com>
-Reviewed-by: Ameya More <ameya.more@oracle.com>
+Signed-off-by: Max Reitz <mreitz@redhat.com>
 Message-id: 52e6d742811f10dbd16e996e86cf375b9577c187.1513005190.git.mark.kanda@oracle.com
 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
 ---
- include/hw/virtio/virtio-blk.h |  1 +
+ qemu-img.c | 2 ++
- hw/block/virtio-blk.c          | 10 +++++++++-
+file changed, 2 insertions(+)
 files changed, 10 insertions(+), 1 deletion(-)
-diff --git a/include/hw/virtio/virtio-blk.h b/include/hw/virtio/virtio-blk.h
+diff --git a/qemu-img.c b/qemu-img.c
 index XXXXXXX..XXXXXXX 100644
---- a/include/hw/virtio/virtio-blk.h
+--- a/qemu-img.c
-+++ b/include/hw/virtio/virtio-blk.h
++++ b/qemu-img.c
-@@ -XXX,XX +XXX,XX @@ struct VirtIOBlkConf
+@@ -XXX,XX +XXX,XX @@ static int img_check(int argc, char **argv)
-     uint32_t config_wce;
+                     check->corruptions_fixed);
-     uint32_t request_merging;
+         }
-     uint16_t num_queues;
-+    uint16_t queue_size;
++        qapi_free_ImageCheck(check);
- };
++        check = g_new0(ImageCheck, 1);
+         ret = collect_image_check(bs, check, filename, fmt, 0);
- struct VirtIOBlockDataPlane;
-diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
+         check->leaks_fixed          = leaks_fixed;
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/block/virtio-blk.c
 +++ b/hw/block/virtio-blk.c
@@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
          error_setg(errp, "num-queues property must be larger than 0");
          return;
      }
 +    if (!is_power_of_2(conf->queue_size) ||
 +        conf->queue_size > VIRTQUEUE_MAX_SIZE) {
 +        error_setg(errp, "invalid queue-size property (%" PRIu16 "), "
 +                   "must be a power of 2 (max %d)",
 +                   conf->queue_size, VIRTQUEUE_MAX_SIZE);
 +        return;
 +    }
      blkconf_serial(&conf->conf, &conf->serial);
      if (!blkconf_apply_backend_options(&conf->conf,
@@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
      s->sector_mask = (s->conf.conf.logical_block_size / BDRV_SECTOR_SIZE) - 1;
      for (i = 0; i < conf->num_queues; i++) {
 -        virtio_add_queue(vdev, 128, virtio_blk_handle_output);
 +        virtio_add_queue(vdev, conf->queue_size, virtio_blk_handle_output);
      }
      if (!virtio_blk_data_plane_create(vdev, conf, &s->dataplane, errp)) {
          virtio_cleanup(vdev);
@@ -XXX,XX +XXX,XX @@ static Property virtio_blk_properties[] = {
      DEFINE_PROP_BIT("request-merging", VirtIOBlock, conf.request_merging, 0,
                      true),
      DEFINE_PROP_UINT16("num-queues", VirtIOBlock, conf.num_queues, 1),
 +    DEFINE_PROP_UINT16("queue-size", VirtIOBlock, conf.queue_size, 128),
      DEFINE_PROP_LINK("iothread", VirtIOBlock, conf.iothread, TYPE_IOTHREAD,
                       IOThread *),
      DEFINE_PROP_END_OF_LIST(),
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 02/23] hw/block/nvme: Convert to realize
+[PULL 10/19] block/qcow2-threads: fix qcow2_decompress
-From: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>
+From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-Convert nvme_init() to realize and rename it to nvme_realize().
+On success path we return what inflate() returns instead of 0. And it
 most probably works for Z_STREAM_END as it is positive, but is
 definitely broken for Z_BUF_ERROR.
-Cc: John Snow <jsnow@redhat.com>
+While being here, switch to errno return code, to be closer to
-Cc: Keith Busch <keith.busch@intel.com>
+qcow2_compress API (and usual expectations).
 Cc: Kevin Wolf <kwolf@redhat.com>
 Cc: Max Reitz <mreitz@redhat.com>
 Cc: Markus Armbruster <armbru@redhat.com>
-Signed-off-by: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>
+Revert condition in if to be more positive. Drop dead initialization of
-Message-id: 2882e72d795e04cbe2120f569d551aef2467ac60.1511317952.git.maozy.fnst@cn.fujitsu.com
+ret.
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
 Cc: qemu-stable@nongnu.org # v4.0
 Fixes: 341926ab83e2b
 Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
 Message-Id: <20200302150930.16218-1-vsementsov@virtuozzo.com>
 Reviewed-by: Alberto Garcia <berto@igalia.com>
 Reviewed-by: Ján Tomko <jtomko@redhat.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- hw/block/nvme.c | 18 ++++++++++--------
+ block/qcow2-threads.c | 12 +++++++-----
-file changed, 10 insertions(+), 8 deletions(-)
+file changed, 7 insertions(+), 5 deletions(-)
-diff --git a/hw/block/nvme.c b/hw/block/nvme.c
+diff --git a/block/qcow2-threads.c b/block/qcow2-threads.c
 index XXXXXXX..XXXXXXX 100644
---- a/hw/block/nvme.c
+--- a/block/qcow2-threads.c
-+++ b/hw/block/nvme.c
++++ b/block/qcow2-threads.c
-@@ -XXX,XX +XXX,XX @@ static const MemoryRegionOps nvme_cmb_ops = {
+@@ -XXX,XX +XXX,XX @@ static ssize_t qcow2_compress(void *dest, size_t dest_size,
-     },
+  * @src - source buffer, @src_size bytes
- };
+  *
+  * Returns: 0 on success
--static int nvme_init(PCIDevice *pci_dev)
+- *          -1 on fail
-+static void nvme_realize(PCIDevice *pci_dev, Error **errp)
++ *          -EIO on fail
   */
  static ssize_t qcow2_decompress(void *dest, size_t dest_size,
                                  const void *src, size_t src_size)
  {
-     NvmeCtrl *n = NVME(pci_dev);
+-    int ret = 0;
-     NvmeIdCtrl *id = &n->id_ctrl;
++    int ret;
-@@ -XXX,XX +XXX,XX @@ static int nvme_init(PCIDevice *pci_dev)
+     z_stream strm;
-     Error *local_err = NULL;
+     memset(&strm, 0, sizeof(strm));
-     if (!n->conf.blk) {
+@@ -XXX,XX +XXX,XX @@ static ssize_t qcow2_decompress(void *dest, size_t dest_size,
      ret = inflateInit2(&strm, -12);
      if (ret != Z_OK) {
 -        return -1;
-+        error_setg(errp, "drive property not set");
++        return -EIO;
 +        return;
      }
-     bs_size = blk_getlength(n->conf.blk);
+     ret = inflate(&strm, Z_FINISH);
-     if (bs_size < 0) {
+-    if ((ret != Z_STREAM_END && ret != Z_BUF_ERROR) || strm.avail_out != 0) {
--        return -1;
++    if ((ret == Z_STREAM_END || ret == Z_BUF_ERROR) && strm.avail_out == 0) {
-+        error_setg(errp, "could not get backing file size");
+         /*
-+        return;
+          * We approve Z_BUF_ERROR because we need @dest buffer to be filled, but
           * @src buffer may be processed partly (because in qcow2 we know size of
           * compressed data with precision of one sector)
           */
 -        ret = -1;
 +        ret = 0;
 +    } else {
 +        ret = -EIO;
      }
-     blkconf_serial(&n->conf, &n->serial);
+     inflateEnd(&strm);
      if (!n->serial) {
 -        return -1;
 +        error_setg(errp, "serial property not set");
 +        return;
      }
      blkconf_blocksizes(&n->conf);
      blkconf_apply_backend_options(&n->conf, blk_is_read_only(n->conf.blk),
                                    false, &local_err);
      if (local_err) {
 -        error_report_err(local_err);
 -        return -1;
 +        error_propagate(errp, local_err);
 +        return;
      }
      pci_conf = pci_dev->config;
@@ -XXX,XX +XXX,XX @@ static int nvme_init(PCIDevice *pci_dev)
              cpu_to_le64(n->ns_size >>
                  id_ns->lbaf[NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas)].ds);
      }
 -    return 0;
  }
  static void nvme_exit(PCIDevice *pci_dev)
@@ -XXX,XX +XXX,XX @@ static void nvme_class_init(ObjectClass *oc, void *data)
      DeviceClass *dc = DEVICE_CLASS(oc);
      PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
 -    pc->init = nvme_init;
 +    pc->realize = nvme_realize;
      pc->exit = nvme_exit;
      pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
      pc->vendor_id = PCI_VENDOR_ID_INTEL;
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 15/23] qemu-iotests: add 202 external snapshots IOThread test
+[PULL 11/19] job: refactor progress to separate object
-QMP 'transaction' blockdev-snapshot-sync with multiple disks in an
+From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-IOThread is an untested code path.  Several bugs have been found in
-connection with this command.  This patch adds a test case to prevent
+We need it in separate to pass to the block-copy object in the next
-future regressions.
+commit.
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+Cc: qemu-stable@nongnu.org
-Reviewed-by: Kevin Wolf <kwolf@redhat.com>
+Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-Reviewed-by: Eric Blake <eblake@redhat.com>
+Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
-Message-id: 20171206144550.22295-10-stefanha@redhat.com
+Reviewed-by: Max Reitz <mreitz@redhat.com>
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+Message-Id: <20200311103004.7649-2-vsementsov@virtuozzo.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- tests/qemu-iotests/202     | 95 ++++++++++++++++++++++++++++++++++++++++++++++
+ blockjob.c                    | 16 +++++-----
- tests/qemu-iotests/202.out | 11 ++++++
+ include/qemu/job.h            | 11 ++-----
- tests/qemu-iotests/group   |  1 +
+ include/qemu/progress_meter.h | 58 +++++++++++++++++++++++++++++++++++
-files changed, 107 insertions(+)
+ job-qmp.c                     |  4 +--
- create mode 100755 tests/qemu-iotests/202
+ job.c                         |  6 ++--
- create mode 100644 tests/qemu-iotests/202.out
+ qemu-img.c                    |  6 ++--
+files changed, 76 insertions(+), 25 deletions(-)
-diff --git a/tests/qemu-iotests/202 b/tests/qemu-iotests/202
+ create mode 100644 include/qemu/progress_meter.h
-new file mode 100755
-index XXXXXXX..XXXXXXX
+diff --git a/blockjob.c b/blockjob.c
---- /dev/null
+index XXXXXXX..XXXXXXX 100644
-+++ b/tests/qemu-iotests/202
+--- a/blockjob.c
 +++ b/blockjob.c
@@ -XXX,XX +XXX,XX @@ BlockJobInfo *block_job_query(BlockJob *job, Error **errp)
      info->device    = g_strdup(job->job.id);
      info->busy      = atomic_read(&job->job.busy);
      info->paused    = job->job.pause_count > 0;
 -    info->offset    = job->job.progress_current;
 -    info->len       = job->job.progress_total;
 +    info->offset    = job->job.progress.current;
 +    info->len       = job->job.progress.total;
      info->speed     = job->speed;
      info->io_status = job->iostatus;
      info->ready     = job_is_ready(&job->job),
@@ -XXX,XX +XXX,XX @@ static void block_job_event_cancelled(Notifier *n, void *opaque)
      qapi_event_send_block_job_cancelled(job_type(&job->job),
                                          job->job.id,
 -                                        job->job.progress_total,
 -                                        job->job.progress_current,
 +                                        job->job.progress.total,
 +                                        job->job.progress.current,
                                          job->speed);
  }
@@ -XXX,XX +XXX,XX @@ static void block_job_event_completed(Notifier *n, void *opaque)
      qapi_event_send_block_job_completed(job_type(&job->job),
                                          job->job.id,
 -                                        job->job.progress_total,
 -                                        job->job.progress_current,
 +                                        job->job.progress.total,
 +                                        job->job.progress.current,
                                          job->speed,
                                          !!msg,
                                          msg);
@@ -XXX,XX +XXX,XX @@ static void block_job_event_ready(Notifier *n, void *opaque)
      qapi_event_send_block_job_ready(job_type(&job->job),
                                      job->job.id,
 -                                    job->job.progress_total,
 -                                    job->job.progress_current,
 +                                    job->job.progress.total,
 +                                    job->job.progress.current,
                                      job->speed);
  }
 diff --git a/include/qemu/job.h b/include/qemu/job.h
 index XXXXXXX..XXXXXXX 100644
 --- a/include/qemu/job.h
 +++ b/include/qemu/job.h
 @@ -XXX,XX +XXX,XX @@
-+#!/usr/bin/env python
-+#
+ #include "qapi/qapi-types-job.h"
-+# Copyright (C) 2017 Red Hat, Inc.
+ #include "qemu/queue.h"
-+#
++#include "qemu/progress_meter.h"
-+# This program is free software; you can redistribute it and/or modify
+ #include "qemu/coroutine.h"
-+# it under the terms of the GNU General Public License as published by
+ #include "block/aio.h"
-+# the Free Software Foundation; either version 2 of the License, or
-+# (at your option) any later version.
+@@ -XXX,XX +XXX,XX @@ typedef struct Job {
-+#
+     /** True if this job should automatically dismiss itself */
-+# This program is distributed in the hope that it will be useful,
+     bool auto_dismiss;
-+# but WITHOUT ANY WARRANTY; without even the implied warranty of
-+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-    /**
-+# GNU General Public License for more details.
+-     * Current progress. The unit is arbitrary as long as the ratio between
-+#
+-     * progress_current and progress_total represents the estimated percentage
-+# You should have received a copy of the GNU General Public License
+-     * of work already done.
-+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-     */
-+#
+-    int64_t progress_current;
-+# Creator/Owner: Stefan Hajnoczi <stefanha@redhat.com>
+-
-+#
+-    /** Estimated progress_current value at the completion of the job */
-+# Check that QMP 'transaction' blockdev-snapshot-sync with multiple drives on a
+-    int64_t progress_total;
-+# single IOThread completes successfully.  This particular command triggered a
++    ProgressMeter progress;
-+# hang due to recursive AioContext locking and BDRV_POLL_WHILE().  Protect
-+# against regressions.
+     /**
-+
+      * Return code from @run and/or @prepare callback(s).
-+import iotests
+diff --git a/include/qemu/progress_meter.h b/include/qemu/progress_meter.h
 +
 +iotests.verify_image_format(supported_fmts=['qcow2'])
 +iotests.verify_platform(['linux'])
 +
 +with iotests.FilePath('disk0.img') as disk0_img_path, \
 +     iotests.FilePath('disk1.img') as disk1_img_path, \
 +     iotests.FilePath('disk0-snap.img') as disk0_snap_img_path, \
 +     iotests.FilePath('disk1-snap.img') as disk1_snap_img_path, \
 +     iotests.VM() as vm:
 +
 +    img_size = '10M'
 +    iotests.qemu_img_pipe('create', '-f', iotests.imgfmt, disk0_img_path, img_size)
 +    iotests.qemu_img_pipe('create', '-f', iotests.imgfmt, disk1_img_path, img_size)
 +
 +    iotests.log('Launching VM...')
 +    vm.launch()
 +
 +    iotests.log('Adding IOThread...')
 +    iotests.log(vm.qmp('object-add',
 +                       qom_type='iothread',
 +                       id='iothread0'))
 +
 +    iotests.log('Adding blockdevs...')
 +    iotests.log(vm.qmp('blockdev-add',
 +                       driver=iotests.imgfmt,
 +                       node_name='disk0',
 +                       file={
 +                           'driver': 'file',
 +                           'filename': disk0_img_path,
 +                       }))
 +    iotests.log(vm.qmp('blockdev-add',
 +                       driver=iotests.imgfmt,
 +                       node_name='disk1',
 +                       file={
 +                           'driver': 'file',
 +                           'filename': disk1_img_path,
 +                       }))
 +
 +    iotests.log('Setting iothread...')
 +    iotests.log(vm.qmp('x-blockdev-set-iothread',
 +                       node_name='disk0',
 +                       iothread='iothread0'))
 +    iotests.log(vm.qmp('x-blockdev-set-iothread',
 +                       node_name='disk1',
 +                       iothread='iothread0'))
 +
 +    iotests.log('Creating external snapshots...')
 +    iotests.log(vm.qmp(
 +                  'transaction',
 +                  actions=[
 +                      {
 +                          'data': {
 +                              'node-name': 'disk0',
 +                              'snapshot-file': disk0_snap_img_path,
 +                              'snapshot-node-name': 'disk0-snap',
 +                              'mode': 'absolute-paths',
 +                              'format': iotests.imgfmt,
 +                          },
 +                          'type': 'blockdev-snapshot-sync'
 +                      }, {
 +                          'data': {
 +                              'node-name': 'disk1',
 +                              'snapshot-file': disk1_snap_img_path,
 +                              'snapshot-node-name': 'disk1-snap',
 +                              'mode': 'absolute-paths',
 +                              'format': iotests.imgfmt
 +                          },
 +                          'type': 'blockdev-snapshot-sync'
 +                      }
 +                  ]))
 diff --git a/tests/qemu-iotests/202.out b/tests/qemu-iotests/202.out
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
-+++ b/tests/qemu-iotests/202.out
++++ b/include/qemu/progress_meter.h
 @@ -XXX,XX +XXX,XX @@
-+Launching VM...
++/*
-+Adding IOThread...
++ * Helper functionality for some process progress tracking.
-+{u'return': {}}
++ *
-+Adding blockdevs...
++ * Copyright (c) 2011 IBM Corp.
-+{u'return': {}}
++ * Copyright (c) 2012, 2018 Red Hat, Inc.
-+{u'return': {}}
++ * Copyright (c) 2020 Virtuozzo International GmbH
-+Setting iothread...
++ *
-+{u'return': {}}
++ * Permission is hereby granted, free of charge, to any person obtaining a copy
-+{u'return': {}}
++ * of this software and associated documentation files (the "Software"), to deal
-+Creating external snapshots...
++ * in the Software without restriction, including without limitation the rights
-+{u'return': {}}
++ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
++ * copies of the Software, and to permit persons to whom the Software is
-index XXXXXXX..XXXXXXX 100644
++ * furnished to do so, subject to the following conditions:
---- a/tests/qemu-iotests/group
++ *
-+++ b/tests/qemu-iotests/group
++ * The above copyright notice and this permission notice shall be included in
-@@ -XXX,XX +XXX,XX @@
++ * all copies or substantial portions of the Software.
-rw auto quick
++ *
-rw auto
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-rw auto
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-+202 rw auto quick
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 + * THE SOFTWARE.
 + */
 +
 +#ifndef QEMU_PROGRESS_METER_H
 +#define QEMU_PROGRESS_METER_H
 +
 +typedef struct ProgressMeter {
 +    /**
 +     * Current progress. The unit is arbitrary as long as the ratio between
 +     * current and total represents the estimated percentage
 +     * of work already done.
 +     */
 +    uint64_t current;
 +
 +    /** Estimated current value at the completion of the process */
 +    uint64_t total;
 +} ProgressMeter;
 +
 +static inline void progress_work_done(ProgressMeter *pm, uint64_t done)
 +{
 +    pm->current += done;
 +}
 +
 +static inline void progress_set_remaining(ProgressMeter *pm, uint64_t remaining)
 +{
 +    pm->total = pm->current + remaining;
 +}
 +
 +static inline void progress_increase_remaining(ProgressMeter *pm,
 +                                               uint64_t delta)
 +{
 +    pm->total += delta;
 +}
 +
 +#endif /* QEMU_PROGRESS_METER_H */
 diff --git a/job-qmp.c b/job-qmp.c
 index XXXXXXX..XXXXXXX 100644
 --- a/job-qmp.c
 +++ b/job-qmp.c
@@ -XXX,XX +XXX,XX @@ static JobInfo *job_query_single(Job *job, Error **errp)
          .id                 = g_strdup(job->id),
          .type               = job_type(job),
          .status             = job->status,
 -        .current_progress   = job->progress_current,
 -        .total_progress     = job->progress_total,
 +        .current_progress   = job->progress.current,
 +        .total_progress     = job->progress.total,
          .has_error          = !!job->err,
          .error              = job->err ? \
                                g_strdup(error_get_pretty(job->err)) : NULL,
 diff --git a/job.c b/job.c
 index XXXXXXX..XXXXXXX 100644
 --- a/job.c
 +++ b/job.c
@@ -XXX,XX +XXX,XX @@ void job_unref(Job *job)
  void job_progress_update(Job *job, uint64_t done)
  {
 -    job->progress_current += done;
 +    progress_work_done(&job->progress, done);
  }
  void job_progress_set_remaining(Job *job, uint64_t remaining)
  {
 -    job->progress_total = job->progress_current + remaining;
 +    progress_set_remaining(&job->progress, remaining);
  }
  void job_progress_increase_remaining(Job *job, uint64_t delta)
  {
 -    job->progress_total += delta;
 +    progress_increase_remaining(&job->progress, delta);
  }
  void job_event_cancelled(Job *job)
 diff --git a/qemu-img.c b/qemu-img.c
 index XXXXXXX..XXXXXXX 100644
 --- a/qemu-img.c
 +++ b/qemu-img.c
@@ -XXX,XX +XXX,XX @@ static void run_block_job(BlockJob *job, Error **errp)
      do {
          float progress = 0.0f;
          aio_poll(aio_context, true);
 -        if (job->job.progress_total) {
 -            progress = (float)job->job.progress_current /
 -                       job->job.progress_total * 100.f;
 +        if (job->job.progress.total) {
 +            progress = (float)job->job.progress.current /
 +                       job->job.progress.total * 100.f;
          }
          qemu_progress_print(progress, 0);
      } while (!job_is_ready(&job->job) && !job_is_completed(&job->job));
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 05/23] dev-storage: Fix the unusual function name
+[PULL 12/19] block/block-copy: fix progress calculation
-From: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>
+From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-The function name of usb_msd_{realize,unrealize}_*,
+Assume we have two regions, A and B, and region B is in-flight now,
-usb_msd_class_initfn_* are unusual. Rename it to
+region A is not yet touched, but it is unallocated and should be
-usb_msd_*_{realize,unrealize}, usb_msd_class_*_initfn.
+skipped.
-Cc: Gerd Hoffmann <kraxel@redhat.com>
+Correspondingly, as progress we have
-Signed-off-by: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>
+  total = A + B
-Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
+  current = 0
-Message-id: 11e6003433abce35f3f4970e1acc71ee92dbcf51.1511317952.git.maozy.fnst@cn.fujitsu.com
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+If we reset unallocated region A and call progress_reset_callback,
 it will calculate 0 bytes dirty in the bitmap and call
 job_progress_set_remaining, which will set
    total = current + 0 = 0 + 0 = 0
 So, B bytes are actually removed from total accounting. When job
 finishes we'll have
    total = 0
    current = B
 , which doesn't sound good.
 This is because we didn't considered in-flight bytes, actually when
 calculating remaining, we should have set (in_flight + dirty_bytes)
 as remaining, not only dirty_bytes.
 To fix it, let's refactor progress calculation, moving it to block-copy
 itself instead of fixing callback. And, of course, track in_flight
 bytes count.
 We still have to keep one callback, to maintain backup job bytes_read
 calculation, but it will go on soon, when we turn the whole backup
 process into one block_copy call.
 Cc: qemu-stable@nongnu.org
 Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
 Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
 Message-Id: <20200311103004.7649-3-vsementsov@virtuozzo.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- hw/usb/dev-storage.c | 20 ++++++++++----------
+ block/backup.c             | 13 ++-----------
-file changed, 10 insertions(+), 10 deletions(-)
+ block/block-copy.c         | 16 ++++++++++++----
  include/block/block-copy.h | 15 +++++----------
 files changed, 19 insertions(+), 25 deletions(-)
-diff --git a/hw/usb/dev-storage.c b/hw/usb/dev-storage.c
+diff --git a/block/backup.c b/block/backup.c
 index XXXXXXX..XXXXXXX 100644
---- a/hw/usb/dev-storage.c
+--- a/block/backup.c
-+++ b/hw/usb/dev-storage.c
++++ b/block/backup.c
-@@ -XXX,XX +XXX,XX @@ static void usb_msd_unrealize_storage(USBDevice *dev, Error **errp)
+@@ -XXX,XX +XXX,XX @@ static void backup_progress_bytes_callback(int64_t bytes, void *opaque)
-     object_unref(OBJECT(&s->bus));
+     BackupBlockJob *s = opaque;
      s->bytes_read += bytes;
 -    job_progress_update(&s->common.job, bytes);
 -}
 -
 -static void backup_progress_reset_callback(void *opaque)
 -{
 -    BackupBlockJob *s = opaque;
 -    uint64_t estimate = bdrv_get_dirty_count(s->bcs->copy_bitmap);
 -
 -    job_progress_set_remaining(&s->common.job, estimate);
  }
--static void usb_msd_realize_storage(USBDevice *dev, Error **errp)
+ static int coroutine_fn backup_do_cow(BackupBlockJob *job,
-+static void usb_msd_storage_realize(USBDevice *dev, Error **errp)
+@@ -XXX,XX +XXX,XX @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
      job->cluster_size = cluster_size;
      job->len = len;
 -    block_copy_set_callbacks(bcs, backup_progress_bytes_callback,
 -                             backup_progress_reset_callback, job);
 +    block_copy_set_progress_callback(bcs, backup_progress_bytes_callback, job);
 +    block_copy_set_progress_meter(bcs, &job->common.job.progress);
      /* Required permissions are already taken by backup-top target */
      block_job_add_bdrv(&job->common, "target", target, 0, BLK_PERM_ALL,
 diff --git a/block/block-copy.c b/block/block-copy.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/block-copy.c
 +++ b/block/block-copy.c
@@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
      return s;
  }
 -void block_copy_set_callbacks(
 +void block_copy_set_progress_callback(
          BlockCopyState *s,
          ProgressBytesCallbackFunc progress_bytes_callback,
 -        ProgressResetCallbackFunc progress_reset_callback,
          void *progress_opaque)
  {
-     MSDState *s = USB_STORAGE_DEV(dev);
+     s->progress_bytes_callback = progress_bytes_callback;
-     BlockBackend *blk = s->conf.blk;
+-    s->progress_reset_callback = progress_reset_callback;
-@@ -XXX,XX +XXX,XX @@ static void usb_msd_realize_storage(USBDevice *dev, Error **errp)
+     s->progress_opaque = progress_opaque;
      s->scsi_dev = scsi_dev;
  }
--static void usb_msd_unrealize_bot(USBDevice *dev, Error **errp)
++void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm)
-+static void usb_msd_bot_unrealize(USBDevice *dev, Error **errp)
++{
- {
++    s->progress = pm;
-     MSDState *s = USB_STORAGE_DEV(dev);
++}
++
-     object_unref(OBJECT(&s->bus));
+ /*
- }
+  * block_copy_do_copy
+  *
--static void usb_msd_realize_bot(USBDevice *dev, Error **errp)
+@@ -XXX,XX +XXX,XX @@ int64_t block_copy_reset_unallocated(BlockCopyState *s,
-+static void usb_msd_bot_realize(USBDevice *dev, Error **errp)
- {
+     if (!ret) {
-     MSDState *s = USB_STORAGE_DEV(dev);
+         bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
-     DeviceState *d = DEVICE(dev);
+-        s->progress_reset_callback(s->progress_opaque);
-@@ -XXX,XX +XXX,XX @@ static void usb_msd_class_initfn_common(ObjectClass *klass, void *data)
++        progress_set_remaining(s->progress,
-     dc->vmsd = &vmstate_usb_msd;
++                               bdrv_get_dirty_count(s->copy_bitmap) +
- }
++                               s->in_flight_bytes);
+     }
--static void usb_msd_class_initfn_storage(ObjectClass *klass, void *data)
-+static void usb_msd_class_storage_initfn(ObjectClass *klass, void *data)
+     *count = bytes;
- {
+@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
-     DeviceClass *dc = DEVICE_CLASS(klass);
+         trace_block_copy_process(s, start);
-     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
+         bdrv_reset_dirty_bitmap(s->copy_bitmap, start, chunk_end - start);
--    uc->realize = usb_msd_realize_storage;
++        s->in_flight_bytes += chunk_end - start;
-+    uc->realize = usb_msd_storage_realize;
-     uc->unrealize = usb_msd_unrealize_storage;
+         co_get_from_shres(s->mem, chunk_end - start);
-     dc->props = msd_properties;
+         ret = block_copy_do_copy(s, start, chunk_end, error_is_read);
- }
+         co_put_to_shres(s->mem, chunk_end - start);
-@@ -XXX,XX +XXX,XX @@ static void usb_msd_instance_init(Object *obj)
++        s->in_flight_bytes -= chunk_end - start;
-     object_property_set_int(obj, -1, "bootindex", NULL);
+         if (ret < 0) {
- }
+             bdrv_set_dirty_bitmap(s->copy_bitmap, start, chunk_end - start);
+             break;
--static void usb_msd_class_initfn_bot(ObjectClass *klass, void *data)
+         }
-+static void usb_msd_class_bot_initfn(ObjectClass *klass, void *data)
- {
++        progress_work_done(s->progress, chunk_end - start);
-     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
+         s->progress_bytes_callback(chunk_end - start, s->progress_opaque);
+         start = chunk_end;
--    uc->realize = usb_msd_realize_bot;
+         ret = 0;
--    uc->unrealize = usb_msd_unrealize_bot;
+diff --git a/include/block/block-copy.h b/include/block/block-copy.h
-+    uc->realize = usb_msd_bot_realize;
+index XXXXXXX..XXXXXXX 100644
-+    uc->unrealize = usb_msd_bot_unrealize;
+--- a/include/block/block-copy.h
-     uc->attached_settable = true;
++++ b/include/block/block-copy.h
- }
+@@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyInFlightReq {
+ } BlockCopyInFlightReq;
- static const TypeInfo msd_info = {
-     .name          = "usb-storage",
+ typedef void (*ProgressBytesCallbackFunc)(int64_t bytes, void *opaque);
-     .parent        = TYPE_USB_STORAGE,
+-typedef void (*ProgressResetCallbackFunc)(void *opaque);
--    .class_init    = usb_msd_class_initfn_storage,
+ typedef struct BlockCopyState {
-+    .class_init    = usb_msd_class_storage_initfn,
+     /*
-     .instance_init = usb_msd_instance_init,
+      * BdrvChild objects are not owned or managed by block-copy. They are
- };
+@@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyState {
+     BdrvChild *source;
- static const TypeInfo bot_info = {
+     BdrvChild *target;
-     .name          = "usb-bot",
+     BdrvDirtyBitmap *copy_bitmap;
-     .parent        = TYPE_USB_STORAGE,
++    int64_t in_flight_bytes;
--    .class_init    = usb_msd_class_initfn_bot,
+     int64_t cluster_size;
-+    .class_init    = usb_msd_class_bot_initfn,
+     bool use_copy_range;
- };
+     int64_t copy_size;
+@@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyState {
- static void usb_msd_register_types(void)
+      */
      bool skip_unallocated;
 +    ProgressMeter *progress;
      /* progress_bytes_callback: called when some copying progress is done. */
      ProgressBytesCallbackFunc progress_bytes_callback;
 -
 -    /*
 -     * progress_reset_callback: called when some bytes reset from copy_bitmap
 -     * (see @skip_unallocated above). The callee is assumed to recalculate how
 -     * many bytes remain based on the dirty bit count of copy_bitmap.
 -     */
 -    ProgressResetCallbackFunc progress_reset_callback;
      void *progress_opaque;
      SharedResource *mem;
@@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
                                       BdrvRequestFlags write_flags,
                                       Error **errp);
 -void block_copy_set_callbacks(
 +void block_copy_set_progress_callback(
          BlockCopyState *s,
          ProgressBytesCallbackFunc progress_bytes_callback,
 -        ProgressResetCallbackFunc progress_reset_callback,
          void *progress_opaque);
 +void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm);
 +
  void block_copy_state_free(BlockCopyState *s);
  int64_t block_copy_reset_unallocated(BlockCopyState *s,
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 08/23] block: don't keep AioContext acquired after external_snapshot_prepare()
+[PULL 13/19] block/block-copy: specialcase first copy_range request
-It is not necessary to hold AioContext across transactions anymore since
+From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
 bdrv_drained_begin/end() is used to keep the nodes quiesced.  In fact,
 using the AioContext lock for this purpose was always buggy.
-This patch reduces the scope of AioContext locked regions.  This is not
+In block_copy_do_copy we fallback to read+write if copy_range failed.
-just a cleanup but also fixes hangs that occur in BDRV_POLL_WHILE()
+In this case copy_size is larger than defined for buffered IO, and
-because it is unware of recursive locking and does not release the
+there is corresponding commit. Still, backup copies data cluster by
-AioContext the necessary number of times to allow progress to be made.
+cluster, and most of requests are limited to one cluster anyway, so the
 only source of this one bad-limited request is copy-before-write
 operation.
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+Further patch will move backup to use block_copy directly, than for
-Reviewed-by: Kevin Wolf <kwolf@redhat.com>
+cases where copy_range is not supported, first request will be
-Reviewed-by: Eric Blake <eblake@redhat.com>
+oversized in each backup. It's not good, let's change it now.
-Message-id: 20171206144550.22295-3-stefanha@redhat.com
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+Fix is simple: just limit first copy_range request like buffer-based
 request. If it succeed, set larger copy_range limit.
 Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
 Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200311103004.7649-4-vsementsov@virtuozzo.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- blockdev.c | 71 ++++++++++++++++++++++++++++++++++++++++++--------------------
+ block/block-copy.c | 41 +++++++++++++++++++++++++++++++----------
-file changed, 48 insertions(+), 23 deletions(-)
+file changed, 31 insertions(+), 10 deletions(-)
-diff --git a/blockdev.c b/blockdev.c
+diff --git a/block/block-copy.c b/block/block-copy.c
 index XXXXXXX..XXXXXXX 100644
---- a/blockdev.c
+--- a/block/block-copy.c
-+++ b/blockdev.c
++++ b/block/block-copy.c
-@@ -XXX,XX +XXX,XX @@ typedef struct ExternalSnapshotState {
+@@ -XXX,XX +XXX,XX @@ void block_copy_state_free(BlockCopyState *s)
-     BlkActionState common;
+     g_free(s);
-     BlockDriverState *old_bs;
+ }
-     BlockDriverState *new_bs;
--    AioContext *aio_context;
++static uint32_t block_copy_max_transfer(BdrvChild *source, BdrvChild *target)
-     bool overlay_appended;
++{
- } ExternalSnapshotState;
++    return MIN_NON_ZERO(INT_MAX,
++                        MIN_NON_ZERO(source->bs->bl.max_transfer,
-@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,
++                                     target->bs->bl.max_transfer));
-     ExternalSnapshotState *state =
++}
-                              DO_UPCAST(ExternalSnapshotState, common, common);
++
-     TransactionAction *action = common->action;
+ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
-+    AioContext *aio_context;
+                                      int64_t cluster_size,
+                                      BdrvRequestFlags write_flags, Error **errp)
-     /* 'blockdev-snapshot' and 'blockdev-snapshot-sync' have similar
+ {
-      * purpose but a different set of parameters */
+     BlockCopyState *s;
-@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,
+     BdrvDirtyBitmap *copy_bitmap;
-         return;
+-    uint32_t max_transfer =
 -            MIN_NON_ZERO(INT_MAX,
 -                         MIN_NON_ZERO(source->bs->bl.max_transfer,
 -                                      target->bs->bl.max_transfer));
      copy_bitmap = bdrv_create_dirty_bitmap(source->bs, cluster_size, NULL,
                                             errp);
@@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
          .mem = shres_create(BLOCK_COPY_MAX_MEM),
      };
 -    if (max_transfer < cluster_size) {
 +    if (block_copy_max_transfer(source, target) < cluster_size) {
          /*
           * copy_range does not respect max_transfer. We don't want to bother
           * with requests smaller than block-copy cluster size, so fallback to
@@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
          s->copy_size = cluster_size;
      } else {
          /*
 -         * copy_range does not respect max_transfer (it's a TODO), so we factor
 -         * that in here.
 +         * We enable copy-range, but keep small copy_size, until first
 +         * successful copy_range (look at block_copy_do_copy).
           */
          s->use_copy_range = true;
 -        s->copy_size = MIN(MAX(cluster_size, BLOCK_COPY_MAX_COPY_RANGE),
 -                           QEMU_ALIGN_DOWN(max_transfer, cluster_size));
 +        s->copy_size = MAX(s->cluster_size, BLOCK_COPY_MAX_BUFFER);
      }
--    /* Acquire AioContext now so any threads operating on old_bs stop */
+     QLIST_INIT(&s->inflight_reqs);
--    state->aio_context = bdrv_get_aio_context(state->old_bs);
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
--    aio_context_acquire(state->aio_context);
+             s->copy_size = MAX(s->cluster_size, BLOCK_COPY_MAX_BUFFER);
-+    aio_context = bdrv_get_aio_context(state->old_bs);
+             /* Fallback to read+write with allocated buffer */
-+    aio_context_acquire(aio_context);
+         } else {
-+
++            if (s->use_copy_range) {
-+    /* Paired with .clean() */
++                /*
-     bdrv_drained_begin(state->old_bs);
++                 * Successful copy-range. Now increase copy_size.  copy_range
++                 * does not respect max_transfer (it's a TODO), so we factor
-     if (!bdrv_is_inserted(state->old_bs)) {
++                 * that in here.
-         error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
++                 *
--        return;
++                 * Note: we double-check s->use_copy_range for the case when
-+        goto out;
++                 * parallel block-copy request unsets it during previous
-     }
++                 * bdrv_co_copy_range call.
++                 */
-     if (bdrv_op_is_blocked(state->old_bs,
++                s->copy_size =
-                            BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT, errp)) {
++                        MIN(MAX(s->cluster_size, BLOCK_COPY_MAX_COPY_RANGE),
--        return;
++                            QEMU_ALIGN_DOWN(block_copy_max_transfer(s->source,
-+        goto out;
++                                                                    s->target),
-     }
++                                            s->cluster_size));
++            }
-     if (!bdrv_is_read_only(state->old_bs)) {
+             goto out;
          if (bdrv_flush(state->old_bs)) {
              error_setg(errp, QERR_IO_ERROR);
 -            return;
 +            goto out;
          }
      }
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
-     if (!bdrv_is_first_non_filter(state->old_bs)) {
+     /*
-         error_setg(errp, QERR_FEATURE_DISABLED, "snapshot");
+      * In case of failed copy_range request above, we may proceed with buffered
--        return;
+      * request larger than BLOCK_COPY_MAX_BUFFER. Still, further requests will
-+        goto out;
+-     * be properly limited, so don't care too much.
-     }
++     * be properly limited, so don't care too much. Moreover the most likely
++     * case (copy_range is unsupported for the configuration, so the very first
-     if (action->type == TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC) {
++     * copy_range request fails) is handled by setting large copy_size only
-@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,
++     * after first successful copy_range.
+      */
-         if (node_name && !snapshot_node_name) {
-             error_setg(errp, "New snapshot node name missing");
+     bounce_buffer = qemu_blockalign(s->source->bs, nbytes);
 -            return;
 +            goto out;
          }
          if (snapshot_node_name &&
              bdrv_lookup_bs(snapshot_node_name, snapshot_node_name, NULL)) {
              error_setg(errp, "New snapshot node name already in use");
 -            return;
 +            goto out;
          }
          flags = state->old_bs->open_flags;
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,
              int64_t size = bdrv_getlength(state->old_bs);
              if (size < 0) {
                  error_setg_errno(errp, -size, "bdrv_getlength failed");
 -                return;
 +                goto out;
              }
              bdrv_img_create(new_image_file, format,
                              state->old_bs->filename,
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,
                              NULL, size, flags, false, &local_err);
              if (local_err) {
                  error_propagate(errp, local_err);
 -                return;
 +                goto out;
              }
          }
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,
                                errp);
      /* We will manually add the backing_hd field to the bs later */
      if (!state->new_bs) {
 -        return;
 +        goto out;
      }
      if (bdrv_has_blk(state->new_bs)) {
          error_setg(errp, "The snapshot is already in use");
 -        return;
 +        goto out;
      }
      if (bdrv_op_is_blocked(state->new_bs, BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT,
                             errp)) {
 -        return;
 +        goto out;
      }
      if (state->new_bs->backing != NULL) {
          error_setg(errp, "The snapshot already has a backing image");
 -        return;
 +        goto out;
      }
      if (!state->new_bs->drv->supports_backing) {
          error_setg(errp, "The snapshot does not support backing images");
 -        return;
 +        goto out;
      }
 -    bdrv_set_aio_context(state->new_bs, state->aio_context);
 +    bdrv_set_aio_context(state->new_bs, aio_context);
      /* This removes our old bs and adds the new bs. This is an operation that
       * can fail, so we need to do it in .prepare; undoing it for abort is
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,
      bdrv_append(state->new_bs, state->old_bs, &local_err);
      if (local_err) {
          error_propagate(errp, local_err);
 -        return;
 +        goto out;
      }
      state->overlay_appended = true;
 +
 +out:
 +    aio_context_release(aio_context);
  }
  static void external_snapshot_commit(BlkActionState *common)
  {
      ExternalSnapshotState *state =
                               DO_UPCAST(ExternalSnapshotState, common, common);
 +    AioContext *aio_context;
 +
 +    aio_context = bdrv_get_aio_context(state->old_bs);
 +    aio_context_acquire(aio_context);
      /* We don't need (or want) to use the transactional
       * bdrv_reopen_multiple() across all the entries at once, because we
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_commit(BlkActionState *common)
          bdrv_reopen(state->old_bs, state->old_bs->open_flags & ~BDRV_O_RDWR,
                      NULL);
      }
 +
 +    aio_context_release(aio_context);
  }
  static void external_snapshot_abort(BlkActionState *common)
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_abort(BlkActionState *common)
                               DO_UPCAST(ExternalSnapshotState, common, common);
      if (state->new_bs) {
          if (state->overlay_appended) {
 +            AioContext *aio_context;
 +
 +            aio_context = bdrv_get_aio_context(state->old_bs);
 +            aio_context_acquire(aio_context);
 +
              bdrv_ref(state->old_bs);   /* we can't let bdrv_set_backind_hd()
                                            close state->old_bs; we need it */
              bdrv_set_backing_hd(state->new_bs, NULL, &error_abort);
              bdrv_replace_node(state->new_bs, state->old_bs, &error_abort);
              bdrv_unref(state->old_bs); /* bdrv_replace_node() ref'ed old_bs */
 +
 +            aio_context_release(aio_context);
          }
      }
  }
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_clean(BlkActionState *common)
  {
      ExternalSnapshotState *state =
                               DO_UPCAST(ExternalSnapshotState, common, common);
 -    if (state->aio_context) {
 -        bdrv_drained_end(state->old_bs);
 -        bdrv_unref(state->new_bs);
 -        aio_context_release(state->aio_context);
 +    AioContext *aio_context;
 +
 +    if (!state->old_bs) {
 +        return;
      }
 +
 +    aio_context = bdrv_get_aio_context(state->old_bs);
 +    aio_context_acquire(aio_context);
 +
 +    bdrv_drained_end(state->old_bs);
 +    bdrv_unref(state->new_bs);
 +
 +    aio_context_release(aio_context);
  }
  typedef struct DriveBackupState {
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 14/23] blockdev: add x-blockdev-set-iothread testing command
+[PULL 14/19] block/block-copy: use block_status
-Currently there is no easy way for iotests to ensure that a BDS is bound
+From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
 to a particular IOThread.  Normally the virtio-blk device calls
 blk_set_aio_context() when dataplane is enabled during guest driver
 initialization.  This never happens in iotests since -machine
 accel=qtest means there is no guest activity (including device driver
 initialization).
-This patch adds a QMP command to explicitly assign IOThreads in test
+Use bdrv_block_status_above to chose effective chunk size and to handle
-cases.  See qapi/block-core.json for a description of the command.
+zeroes effectively.
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+This substitutes checking for just being allocated or not, and drops
-Reviewed-by: Kevin Wolf <kwolf@redhat.com>
+old code path for it. Assistance by backup job is dropped too, as
-Reviewed-by: Eric Blake <eblake@redhat.com>
+caching block-status information is more difficult than just caching
-Message-id: 20171206144550.22295-9-stefanha@redhat.com
+is-allocated information in our dirty bitmap, and backup job is not
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+good place for this caching anyway.
 Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
 Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200311103004.7649-5-vsementsov@virtuozzo.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- qapi/block-core.json | 36 ++++++++++++++++++++++++++++++++++++
+ block/block-copy.c | 73 +++++++++++++++++++++++++++++++++++++---------
- blockdev.c           | 41 +++++++++++++++++++++++++++++++++++++++++
+ block/trace-events |  1 +
-files changed, 77 insertions(+)
+files changed, 61 insertions(+), 13 deletions(-)
-diff --git a/qapi/block-core.json b/qapi/block-core.json
+diff --git a/block/block-copy.c b/block/block-copy.c
 index XXXXXXX..XXXXXXX 100644
---- a/qapi/block-core.json
+--- a/block/block-copy.c
-+++ b/qapi/block-core.json
++++ b/block/block-copy.c
-@@ -XXX,XX +XXX,XX @@
+@@ -XXX,XX +XXX,XX @@ void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm)
-   'data' : { 'parent': 'str',
+  */
-              '*child': 'str',
+ static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
-              '*node': 'str' } }
+                                            int64_t start, int64_t end,
-+
+-                                           bool *error_is_read)
-+##
++                                           bool zeroes, bool *error_is_read)
-+# @x-blockdev-set-iothread:
+ {
-+#
+     int ret;
-+# Move @node and its children into the @iothread.  If @iothread is null then
+     int nbytes = MIN(end, s->len) - start;
-+# move @node and its children into the main loop.
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
-+#
+     assert(QEMU_IS_ALIGNED(end, s->cluster_size));
-+# The node must not be attached to a BlockBackend.
+     assert(end < s->len || end == QEMU_ALIGN_UP(s->len, s->cluster_size));
-+#
-+# @node-name: the name of the block driver node
++    if (zeroes) {
-+#
++        ret = bdrv_co_pwrite_zeroes(s->target, start, nbytes, s->write_flags &
-+# @iothread: the name of the IOThread object or null for the main loop
++                                    ~BDRV_REQ_WRITE_COMPRESSED);
-+#
++        if (ret < 0) {
-+# Note: this command is experimental and intended for test cases that need
++            trace_block_copy_write_zeroes_fail(s, start, ret);
-+# control over IOThreads only.
++            if (error_is_read) {
-+#
++                *error_is_read = false;
-+# Since: 2.12
++            }
-+#
++        }
-+# Example:
++        return ret;
 +#
 +# 1. Move a node into an IOThread
 +# -> { "execute": "x-blockdev-set-iothread",
 +#      "arguments": { "node-name": "disk1",
 +#                     "iothread": "iothread0" } }
 +# <- { "return": {} }
 +#
 +# 2. Move a node into the main loop
 +# -> { "execute": "x-blockdev-set-iothread",
 +#      "arguments": { "node-name": "disk1",
 +#                     "iothread": null } }
 +# <- { "return": {} }
 +#
 +##
 +{ 'command': 'x-blockdev-set-iothread',
 +  'data' : { 'node-name': 'str',
 +             'iothread': 'StrOrNull' } }
 diff --git a/blockdev.c b/blockdev.c
 index XXXXXXX..XXXXXXX 100644
 --- a/blockdev.c
 +++ b/blockdev.c
@@ -XXX,XX +XXX,XX @@
  #include "qapi/qmp/qerror.h"
  #include "qapi/qobject-output-visitor.h"
  #include "sysemu/sysemu.h"
 +#include "sysemu/iothread.h"
  #include "block/block_int.h"
  #include "qmp-commands.h"
  #include "block/trace.h"
@@ -XXX,XX +XXX,XX @@ BlockJobInfoList *qmp_query_block_jobs(Error **errp)
      return head;
  }
 +void qmp_x_blockdev_set_iothread(const char *node_name, StrOrNull *iothread,
 +                                 Error **errp)
 +{
 +    AioContext *old_context;
 +    AioContext *new_context;
 +    BlockDriverState *bs;
 +
 +    bs = bdrv_find_node(node_name);
 +    if (!bs) {
 +        error_setg(errp, "Cannot find node %s", node_name);
 +        return;
 +    }
 +
-+    /* If we want to allow more extreme test scenarios this guard could be
+     if (s->use_copy_range) {
-+     * removed.  For now it protects against accidents. */
+         ret = bdrv_co_copy_range(s->source, start, s->target, start, nbytes,
-+    if (bdrv_has_blk(bs)) {
+, s->write_flags);
-+        error_setg(errp, "Node %s is in use", node_name);
+@@ -XXX,XX +XXX,XX @@ out:
-+        return;
+     return ret;
  }
 +static int block_copy_block_status(BlockCopyState *s, int64_t offset,
 +                                   int64_t bytes, int64_t *pnum)
 +{
 +    int64_t num;
 +    BlockDriverState *base;
 +    int ret;
 +
 +    if (s->skip_unallocated && s->source->bs->backing) {
 +        base = s->source->bs->backing->bs;
 +    } else {
 +        base = NULL;
 +    }
 +
-+    if (iothread->type == QTYPE_QSTRING) {
++    ret = bdrv_block_status_above(s->source->bs, base, offset, bytes, &num,
-+        IOThread *obj = iothread_by_id(iothread->u.s);
++                                  NULL, NULL);
-+        if (!obj) {
++    if (ret < 0 || num < s->cluster_size) {
-+            error_setg(errp, "Cannot find iothread %s", iothread->u.s);
++        /*
-+            return;
++         * On error or if failed to obtain large enough chunk just fallback to
-+        }
++         * copy one cluster.
-+
++         */
-+        new_context = iothread_get_aio_context(obj);
++        num = s->cluster_size;
 +        ret = BDRV_BLOCK_ALLOCATED | BDRV_BLOCK_DATA;
 +    } else if (offset + num == s->len) {
 +        num = QEMU_ALIGN_UP(num, s->cluster_size);
 +    } else {
-+        new_context = qemu_get_aio_context();
++        num = QEMU_ALIGN_DOWN(num, s->cluster_size);
 +    }
 +
-+    old_context = bdrv_get_aio_context(bs);
++    *pnum = num;
-+    aio_context_acquire(old_context);
++    return ret;
 +
 +    bdrv_set_aio_context(bs, new_context);
 +
 +    aio_context_release(old_context);
 +}
 +
- QemuOptsList qemu_common_drive_opts = {
+ /*
-     .name = "drive",
+  * Check if the cluster starting at offset is allocated or not.
-     .head = QTAILQ_HEAD_INITIALIZER(qemu_common_drive_opts.head),
+  * return via pnum the number of contiguous clusters sharing this allocation.
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
  {
      int ret = 0;
      int64_t end = bytes + start; /* bytes */
 -    int64_t status_bytes;
      BlockCopyInFlightReq req;
      /*
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
      block_copy_inflight_req_begin(s, &req, start, end);
      while (start < end) {
 -        int64_t next_zero, chunk_end;
 +        int64_t next_zero, chunk_end, status_bytes;
          if (!bdrv_dirty_bitmap_get(s->copy_bitmap, start)) {
              trace_block_copy_skip(s, start);
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
              chunk_end = next_zero;
          }
 -        if (s->skip_unallocated) {
 -            ret = block_copy_reset_unallocated(s, start, &status_bytes);
 -            if (ret == 0) {
 -                trace_block_copy_skip_range(s, start, status_bytes);
 -                start += status_bytes;
 -                continue;
 -            }
 -            /* Clamp to known allocated region */
 -            chunk_end = MIN(chunk_end, start + status_bytes);
 +        ret = block_copy_block_status(s, start, chunk_end - start,
 +                                      &status_bytes);
 +        if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
 +            bdrv_reset_dirty_bitmap(s->copy_bitmap, start, status_bytes);
 +            progress_set_remaining(s->progress,
 +                                   bdrv_get_dirty_count(s->copy_bitmap) +
 +                                   s->in_flight_bytes);
 +            trace_block_copy_skip_range(s, start, status_bytes);
 +            start += status_bytes;
 +            continue;
          }
 +        chunk_end = MIN(chunk_end, start + status_bytes);
 +
          trace_block_copy_process(s, start);
          bdrv_reset_dirty_bitmap(s->copy_bitmap, start, chunk_end - start);
          s->in_flight_bytes += chunk_end - start;
          co_get_from_shres(s->mem, chunk_end - start);
 -        ret = block_copy_do_copy(s, start, chunk_end, error_is_read);
 +        ret = block_copy_do_copy(s, start, chunk_end, ret & BDRV_BLOCK_ZERO,
 +                                 error_is_read);
          co_put_to_shres(s->mem, chunk_end - start);
          s->in_flight_bytes -= chunk_end - start;
          if (ret < 0) {
 diff --git a/block/trace-events b/block/trace-events
 index XXXXXXX..XXXXXXX 100644
 --- a/block/trace-events
 +++ b/block/trace-events
@@ -XXX,XX +XXX,XX @@ block_copy_process(void *bcs, int64_t start) "bcs %p start %"PRId64
  block_copy_copy_range_fail(void *bcs, int64_t start, int ret) "bcs %p start %"PRId64" ret %d"
  block_copy_read_fail(void *bcs, int64_t start, int ret) "bcs %p start %"PRId64" ret %d"
  block_copy_write_fail(void *bcs, int64_t start, int ret) "bcs %p start %"PRId64" ret %d"
 +block_copy_write_zeroes_fail(void *bcs, int64_t start, int ret) "bcs %p start %"PRId64" ret %d"
  # ../blockdev.c
  qmp_block_job_cancel(void *job) "job %p"
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 22/23] iothread: fix iothread_stop() race condition
+[PULL 15/19] block/block-copy: factor out find_conflicting_inflight_req
-There is a small chance that iothread_stop() hangs as follows:
+From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-  Thread 3 (Thread 0x7f63eba5f700 (LWP 16105)):
+Split find_conflicting_inflight_req to be used separately.
   #0  0x00007f64012c09b6 in ppoll () at /lib64/libc.so.6
   #1  0x000055959992eac9 in ppoll (__ss=0x0, __timeout=0x0, __nfds=<optimized out>, __fds=<optimized out>) at /usr/include/bits/poll2.h:77
   #2  0x000055959992eac9 in qemu_poll_ns (fds=<optimized out>, nfds=<optimized out>, timeout=<optimized out>) at util/qemu-timer.c:322
   #3  0x0000559599930711 in aio_poll (ctx=0x55959bdb83c0, blocking=blocking@entry=true) at util/aio-posix.c:629
   #4  0x00005595996806fe in iothread_run (opaque=0x55959bd78400) at iothread.c:59
   #5  0x00007f640159f609 in start_thread () at /lib64/libpthread.so.0
   #6  0x00007f64012cce6f in clone () at /lib64/libc.so.6
-  Thread 1 (Thread 0x7f640b45b280 (LWP 16103)):
+Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-  #0  0x00007f64015a0b6d in pthread_join () at /lib64/libpthread.so.0
+Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
-  #1  0x00005595999332ef in qemu_thread_join (thread=<optimized out>) at util/qemu-thread-posix.c:547
+Reviewed-by: Max Reitz <mreitz@redhat.com>
-  #2  0x00005595996808ae in iothread_stop (iothread=<optimized out>) at iothread.c:91
+Message-Id: <20200311103004.7649-6-vsementsov@virtuozzo.com>
-  #3  0x000055959968094d in iothread_stop_iter (object=<optimized out>, opaque=<optimized out>) at iothread.c:102
+Signed-off-by: Max Reitz <mreitz@redhat.com>
-  #4  0x0000559599857d97 in do_object_child_foreach (obj=obj@entry=0x55959bdb8100, fn=fn@entry=0x559599680930 <iothread_stop_iter>, opaque=opaque@entry=0x0, recurse=recurse@entry=false) at qom/object.c:852
+---
-  #5  0x0000559599859477 in object_child_foreach (obj=obj@entry=0x55959bdb8100, fn=fn@entry=0x559599680930 <iothread_stop_iter>, opaque=opaque@entry=0x0) at qom/object.c:867
+ block/block-copy.c | 31 +++++++++++++++++++------------
-  #6  0x0000559599680a6e in iothread_stop_all () at iothread.c:341
+file changed, 19 insertions(+), 12 deletions(-)
   #7  0x000055959955b1d5 in main (argc=<optimized out>, argv=<optimized out>, envp=<optimized out>) at vl.c:4913
-The relevant code from iothread_run() is:
+diff --git a/block/block-copy.c b/block/block-copy.c
   while (!atomic_read(&iothread->stopping)) {
       aio_poll(iothread->ctx, true);
 and iothread_stop():
   iothread->stopping = true;
   aio_notify(iothread->ctx);
   ...
   qemu_thread_join(&iothread->thread);
 The following scenario can occur:
 . IOThread:
   while (!atomic_read(&iothread->stopping)) -> stopping=false
 . Main loop:
   iothread->stopping = true;
   aio_notify(iothread->ctx);
 . IOThread:
   aio_poll(iothread->ctx, true); -> hang
 The bug is explained by the AioContext->notify_me doc comments:
   "If this field is 0, everything (file descriptors, bottom halves,
   timers) will be re-evaluated before the next blocking poll(), thus the
   event_notifier_set call can be skipped."
 The problem is that "everything" does not include checking
 iothread->stopping.  This means iothread_run() will block in aio_poll()
 if aio_notify() was called just before aio_poll().
 This patch fixes the hang by replacing aio_notify() with
 aio_bh_schedule_oneshot().  This makes aio_poll() or g_main_loop_run()
 to return.
 Implementing this properly required a new bool running flag.  The new
 flag prevents races that are tricky if we try to use iothread->stopping.
 Now iothread->stopping is purely for iothread_stop() and
 iothread->running is purely for the iothread_run() thread.
 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
 Reviewed-by: Eric Blake <eblake@redhat.com>
 Message-id: 20171207201320.19284-6-stefanha@redhat.com
 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
 ---
  include/sysemu/iothread.h |  3 ++-
  iothread.c                | 20 +++++++++++++++-----
 files changed, 17 insertions(+), 6 deletions(-)
 diff --git a/include/sysemu/iothread.h b/include/sysemu/iothread.h
 index XXXXXXX..XXXXXXX 100644
---- a/include/sysemu/iothread.h
+--- a/block/block-copy.c
-+++ b/include/sysemu/iothread.h
++++ b/block/block-copy.c
-@@ -XXX,XX +XXX,XX @@ typedef struct {
+@@ -XXX,XX +XXX,XX @@
-     GOnce once;
+ #define BLOCK_COPY_MAX_BUFFER (1 * MiB)
-     QemuMutex init_done_lock;
+ #define BLOCK_COPY_MAX_MEM (128 * MiB)
-     QemuCond init_done_cond;    /* is thread initialization done? */
--    bool stopping;
++static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
-+    bool stopping;              /* has iothread_stop() been called? */
++                                                           int64_t start,
-+    bool running;               /* should iothread_run() continue? */
++                                                           int64_t end)
      int thread_id;
      /* AioContext poll parameters */
 diff --git a/iothread.c b/iothread.c
 index XXXXXXX..XXXXXXX 100644
 --- a/iothread.c
 +++ b/iothread.c
@@ -XXX,XX +XXX,XX @@ static void *iothread_run(void *opaque)
      qemu_cond_signal(&iothread->init_done_cond);
      qemu_mutex_unlock(&iothread->init_done_lock);
 -    while (!atomic_read(&iothread->stopping)) {
 +    while (iothread->running) {
          aio_poll(iothread->ctx, true);
          if (atomic_read(&iothread->worker_context)) {
@@ -XXX,XX +XXX,XX @@ static void *iothread_run(void *opaque)
      return NULL;
  }
 +/* Runs in iothread_run() thread */
 +static void iothread_stop_bh(void *opaque)
 +{
-+    IOThread *iothread = opaque;
++    BlockCopyInFlightReq *req;
 +
-+    iothread->running = false; /* stop iothread_run() */
++    QLIST_FOREACH(req, &s->inflight_reqs, list) {
 +        if (end > req->start_byte && start < req->end_byte) {
 +            return req;
 +        }
 +    }
 +
-+    if (iothread->main_loop) {
++    return NULL;
 +        g_main_loop_quit(iothread->main_loop);
 +    }
 +}
 +
- void iothread_stop(IOThread *iothread)
+ static void coroutine_fn block_copy_wait_inflight_reqs(BlockCopyState *s,
                                                         int64_t start,
                                                         int64_t end)
  {
-     if (!iothread->ctx || iothread->stopping) {
+     BlockCopyInFlightReq *req;
-         return;
+-    bool waited;
-     }
+-
-     iothread->stopping = true;
+-    do {
--    aio_notify(iothread->ctx);
+-        waited = false;
--    if (atomic_read(&iothread->main_loop)) {
+-        QLIST_FOREACH(req, &s->inflight_reqs, list) {
--        g_main_loop_quit(iothread->main_loop);
+-            if (end > req->start_byte && start < req->end_byte) {
--    }
+-                qemu_co_queue_wait(&req->wait_queue, NULL);
-+    aio_bh_schedule_oneshot(iothread->ctx, iothread_stop_bh, iothread);
+-                waited = true;
-     qemu_thread_join(&iothread->thread);
+-                break;
 -            }
 -        }
 -    } while (waited);
 +
 +    while ((req = find_conflicting_inflight_req(s, start, end))) {
 +        qemu_co_queue_wait(&req->wait_queue, NULL);
 +    }
  }
-@@ -XXX,XX +XXX,XX @@ static void iothread_complete(UserCreatable *obj, Error **errp)
+ static void block_copy_inflight_req_begin(BlockCopyState *s,
      char *name, *thread_name;
      iothread->stopping = false;
 +    iothread->running = true;
      iothread->thread_id = -1;
      iothread->ctx = aio_context_new(&local_error);
      if (!iothread->ctx) {
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 07/23] blockdev: hold AioContext for bdrv_unref() in external_snapshot_clean()
+[PULL 16/19] block/block-copy: refactor interfaces to use bytes instead of end
-bdrv_unref() requires the AioContext lock because bdrv_flush() uses
+From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-BDRV_POLL_WHILE(), which assumes the AioContext is currently held.  If
-BDRV_POLL_WHILE() runs without AioContext held the
+We have a lot of "chunk_end - start" invocations, let's switch to
-pthread_mutex_unlock() call in aio_context_release() fails.
+bytes/cur_bytes scheme instead.
-This patch moves bdrv_unref() into the AioContext locked region to solve
+While being here, improve check on block_copy_do_copy parameters to not
-the following pthread_mutex_unlock() failure:
+overflow when calculating nbytes and use int64_t for bytes in
+block_copy for consistency.
-  #0  0x00007f566181969b in raise () at /lib64/libc.so.6
-  #1  0x00007f566181b3b1 in abort () at /lib64/libc.so.6
+Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-  #2  0x00005592cd590458 in error_exit (err=<optimized out>, msg=msg@entry=0x5592cdaf6d60 <__func__.23977> "qemu_mutex_unlock") at util/qemu-thread-posix.c:36
+Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
-  #3  0x00005592cd96e738 in qemu_mutex_unlock (mutex=mutex@entry=0x5592ce9505e0) at util/qemu-thread-posix.c:96
+Reviewed-by: Max Reitz <mreitz@redhat.com>
-  #4  0x00005592cd969b69 in aio_context_release (ctx=ctx@entry=0x5592ce950580) at util/async.c:507
+Message-Id: <20200311103004.7649-7-vsementsov@virtuozzo.com>
-  #5  0x00005592cd8ead78 in bdrv_flush (bs=bs@entry=0x5592cfa87210) at block/io.c:2478
+Signed-off-by: Max Reitz <mreitz@redhat.com>
   #6  0x00005592cd89df30 in bdrv_close (bs=0x5592cfa87210) at block.c:3207
   #7  0x00005592cd89df30 in bdrv_delete (bs=0x5592cfa87210) at block.c:3395
   #8  0x00005592cd89df30 in bdrv_unref (bs=0x5592cfa87210) at block.c:4418
   #9  0x00005592cd6b7f86 in qmp_transaction (dev_list=<optimized out>, has_props=<optimized out>, props=<optimized out>, errp=errp@entry=0x7ffe4a1fc9d8) at blockdev.c:2308
 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
 Reviewed-by: Kevin Wolf <kwolf@redhat.com>
 Reviewed-by: Eric Blake <eblake@redhat.com>
 Message-id: 20171206144550.22295-2-stefanha@redhat.com
 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
 ---
- blockdev.c | 2 +-
+ block/block-copy.c         | 78 ++++++++++++++++++++------------------
-file changed, 1 insertion(+), 1 deletion(-)
+ include/block/block-copy.h |  6 +--
+files changed, 44 insertions(+), 40 deletions(-)
-diff --git a/blockdev.c b/blockdev.c
 diff --git a/block/block-copy.c b/block/block-copy.c
 index XXXXXXX..XXXXXXX 100644
---- a/blockdev.c
+--- a/block/block-copy.c
-+++ b/blockdev.c
++++ b/block/block-copy.c
-@@ -XXX,XX +XXX,XX @@ static void external_snapshot_clean(BlkActionState *common)
+@@ -XXX,XX +XXX,XX @@
-                              DO_UPCAST(ExternalSnapshotState, common, common);
-     if (state->aio_context) {
+ static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
-         bdrv_drained_end(state->old_bs);
+                                                            int64_t start,
--        aio_context_release(state->aio_context);
+-                                                           int64_t end)
-         bdrv_unref(state->new_bs);
++                                                           int64_t bytes)
-+        aio_context_release(state->aio_context);
+ {
      BlockCopyInFlightReq *req;
      QLIST_FOREACH(req, &s->inflight_reqs, list) {
 -        if (end > req->start_byte && start < req->end_byte) {
 +        if (start + bytes > req->start && start < req->start + req->bytes) {
              return req;
          }
      }
@@ -XXX,XX +XXX,XX @@ static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
  static void coroutine_fn block_copy_wait_inflight_reqs(BlockCopyState *s,
                                                         int64_t start,
 -                                                       int64_t end)
 +                                                       int64_t bytes)
  {
      BlockCopyInFlightReq *req;
 -    while ((req = find_conflicting_inflight_req(s, start, end))) {
 +    while ((req = find_conflicting_inflight_req(s, start, bytes))) {
          qemu_co_queue_wait(&req->wait_queue, NULL);
      }
  }
+ static void block_copy_inflight_req_begin(BlockCopyState *s,
+                                           BlockCopyInFlightReq *req,
+-                                          int64_t start, int64_t end)
++                                          int64_t start, int64_t bytes)
+ {
+-    req->start_byte = start;
+-    req->end_byte = end;
++    req->start = start;
++    req->bytes = bytes;
+     qemu_co_queue_init(&req->wait_queue);
+     QLIST_INSERT_HEAD(&s->inflight_reqs, req, list);
+ }
+@@ -XXX,XX +XXX,XX @@ void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm)
+ /*
+  * block_copy_do_copy
+  *
+- * Do copy of cluser-aligned chunk. @end is allowed to exceed s->len only to
+- * cover last cluster when s->len is not aligned to clusters.
++ * Do copy of cluster-aligned chunk. Requested region is allowed to exceed
++ * s->len only to cover last cluster when s->len is not aligned to clusters.
+  *
+  * No sync here: nor bitmap neighter intersecting requests handling, only copy.
+  *
+  * Returns 0 on success.
+  */
+ static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
+-                                           int64_t start, int64_t end,
++                                           int64_t start, int64_t bytes,
+                                            bool zeroes, bool *error_is_read)
+ {
+     int ret;
+-    int nbytes = MIN(end, s->len) - start;
++    int64_t nbytes = MIN(start + bytes, s->len) - start;
+     void *bounce_buffer = NULL;
++    assert(start >= 0 && bytes > 0 && INT64_MAX - start >= bytes);
+     assert(QEMU_IS_ALIGNED(start, s->cluster_size));
+-    assert(QEMU_IS_ALIGNED(end, s->cluster_size));
+-    assert(end < s->len || end == QEMU_ALIGN_UP(s->len, s->cluster_size));
++    assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
++    assert(start < s->len);
++    assert(start + bytes <= s->len ||
++           start + bytes == QEMU_ALIGN_UP(s->len, s->cluster_size));
++    assert(nbytes < INT_MAX);
+     if (zeroes) {
+         ret = bdrv_co_pwrite_zeroes(s->target, start, nbytes, s->write_flags &
+@@ -XXX,XX +XXX,XX @@ int64_t block_copy_reset_unallocated(BlockCopyState *s,
+ }
+ int coroutine_fn block_copy(BlockCopyState *s,
+-                            int64_t start, uint64_t bytes,
++                            int64_t start, int64_t bytes,
+                             bool *error_is_read)
+ {
+     int ret = 0;
+-    int64_t end = bytes + start; /* bytes */
+     BlockCopyInFlightReq req;
+     /*
+@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
+            bdrv_get_aio_context(s->target->bs));
+     assert(QEMU_IS_ALIGNED(start, s->cluster_size));
+-    assert(QEMU_IS_ALIGNED(end, s->cluster_size));
++    assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
+     block_copy_wait_inflight_reqs(s, start, bytes);
+-    block_copy_inflight_req_begin(s, &req, start, end);
++    block_copy_inflight_req_begin(s, &req, start, bytes);
+-    while (start < end) {
+-        int64_t next_zero, chunk_end, status_bytes;
++    while (bytes) {
++        int64_t next_zero, cur_bytes, status_bytes;
+         if (!bdrv_dirty_bitmap_get(s->copy_bitmap, start)) {
+             trace_block_copy_skip(s, start);
+             start += s->cluster_size;
++            bytes -= s->cluster_size;
+             continue; /* already copied */
+         }
+-        chunk_end = MIN(end, start + s->copy_size);
++        cur_bytes = MIN(bytes, s->copy_size);
+         next_zero = bdrv_dirty_bitmap_next_zero(s->copy_bitmap, start,
+-                                                chunk_end - start);
++                                                cur_bytes);
+         if (next_zero >= 0) {
+             assert(next_zero > start); /* start is dirty */
+-            assert(next_zero < chunk_end); /* no need to do MIN() */
+-            chunk_end = next_zero;
++            assert(next_zero < start + cur_bytes); /* no need to do MIN() */
++            cur_bytes = next_zero - start;
+         }
+-        ret = block_copy_block_status(s, start, chunk_end - start,
+-                                      &status_bytes);
++        ret = block_copy_block_status(s, start, cur_bytes, &status_bytes);
+         if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
+             bdrv_reset_dirty_bitmap(s->copy_bitmap, start, status_bytes);
+             progress_set_remaining(s->progress,
+@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
+                                    s->in_flight_bytes);
+             trace_block_copy_skip_range(s, start, status_bytes);
+             start += status_bytes;
++            bytes -= status_bytes;
+             continue;
+         }
+-        chunk_end = MIN(chunk_end, start + status_bytes);
++        cur_bytes = MIN(cur_bytes, status_bytes);
+         trace_block_copy_process(s, start);
+-        bdrv_reset_dirty_bitmap(s->copy_bitmap, start, chunk_end - start);
+-        s->in_flight_bytes += chunk_end - start;
++        bdrv_reset_dirty_bitmap(s->copy_bitmap, start, cur_bytes);
++        s->in_flight_bytes += cur_bytes;
+-        co_get_from_shres(s->mem, chunk_end - start);
+-        ret = block_copy_do_copy(s, start, chunk_end, ret & BDRV_BLOCK_ZERO,
++        co_get_from_shres(s->mem, cur_bytes);
++        ret = block_copy_do_copy(s, start, cur_bytes, ret & BDRV_BLOCK_ZERO,
+                                  error_is_read);
+-        co_put_to_shres(s->mem, chunk_end - start);
+-        s->in_flight_bytes -= chunk_end - start;
++        co_put_to_shres(s->mem, cur_bytes);
++        s->in_flight_bytes -= cur_bytes;
+         if (ret < 0) {
+-            bdrv_set_dirty_bitmap(s->copy_bitmap, start, chunk_end - start);
++            bdrv_set_dirty_bitmap(s->copy_bitmap, start, cur_bytes);
+             break;
+         }
+-        progress_work_done(s->progress, chunk_end - start);
+-        s->progress_bytes_callback(chunk_end - start, s->progress_opaque);
+-        start = chunk_end;
+-        ret = 0;
++        progress_work_done(s->progress, cur_bytes);
++        s->progress_bytes_callback(cur_bytes, s->progress_opaque);
++        start += cur_bytes;
++        bytes -= cur_bytes;
+     }
+     block_copy_inflight_req_end(&req);
+diff --git a/include/block/block-copy.h b/include/block/block-copy.h
+index XXXXXXX..XXXXXXX 100644
+--- a/include/block/block-copy.h
++++ b/include/block/block-copy.h
+@@ -XXX,XX +XXX,XX @@
+ #include "qemu/co-shared-resource.h"
+ typedef struct BlockCopyInFlightReq {
+-    int64_t start_byte;
+-    int64_t end_byte;
++    int64_t start;
++    int64_t bytes;
+     QLIST_ENTRY(BlockCopyInFlightReq) list;
+     CoQueue wait_queue; /* coroutines blocked on this request */
+ } BlockCopyInFlightReq;
+@@ -XXX,XX +XXX,XX @@ void block_copy_state_free(BlockCopyState *s);
+ int64_t block_copy_reset_unallocated(BlockCopyState *s,
+                                      int64_t offset, int64_t *count);
+-int coroutine_fn block_copy(BlockCopyState *s, int64_t start, uint64_t bytes,
++int coroutine_fn block_copy(BlockCopyState *s, int64_t start, int64_t bytes,
+                             bool *error_is_read);
+ #endif /* BLOCK_COPY_H */
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 10/23] block: don't keep AioContext acquired after blockdev_backup_prepare()
+[PULL 17/19] block/block-copy: rename start to offset in interfaces
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-Reviewed-by: Kevin Wolf <kwolf@redhat.com>
-Reviewed-by: Eric Blake <eblake@redhat.com>
+offset/bytes pair is more usual naming in block layer, let's use it.
-Message-id: 20171206144550.22295-5-stefanha@redhat.com
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
 Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200311103004.7649-8-vsementsov@virtuozzo.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- blockdev.c | 44 ++++++++++++++++++++++++++++++++++----------
+ block/block-copy.c         | 82 +++++++++++++++++++-------------------
-file changed, 34 insertions(+), 10 deletions(-)
+ include/block/block-copy.h |  4 +-
+files changed, 43 insertions(+), 43 deletions(-)
-diff --git a/blockdev.c b/blockdev.c
 diff --git a/block/block-copy.c b/block/block-copy.c
 index XXXXXXX..XXXXXXX 100644
---- a/blockdev.c
+--- a/block/block-copy.c
-+++ b/blockdev.c
++++ b/block/block-copy.c
-@@ -XXX,XX +XXX,XX @@ typedef struct BlockdevBackupState {
+@@ -XXX,XX +XXX,XX @@
-     BlkActionState common;
+ #define BLOCK_COPY_MAX_MEM (128 * MiB)
-     BlockDriverState *bs;
-     BlockJob *job;
+ static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
--    AioContext *aio_context;
+-                                                           int64_t start,
- } BlockdevBackupState;
++                                                           int64_t offset,
+                                                            int64_t bytes)
- static BlockJob *do_blockdev_backup(BlockdevBackup *backup, BlockJobTxn *txn,
+ {
-@@ -XXX,XX +XXX,XX @@ static void blockdev_backup_prepare(BlkActionState *common, Error **errp)
+     BlockCopyInFlightReq *req;
-     BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
-     BlockdevBackup *backup;
+     QLIST_FOREACH(req, &s->inflight_reqs, list) {
-     BlockDriverState *bs, *target;
+-        if (start + bytes > req->start && start < req->start + req->bytes) {
-+    AioContext *aio_context;
++        if (offset + bytes > req->offset && offset < req->offset + req->bytes) {
-     Error *local_err = NULL;
+             return req;
+         }
-     assert(common->action->type == TRANSACTION_ACTION_KIND_BLOCKDEV_BACKUP);
+     }
-@@ -XXX,XX +XXX,XX @@ static void blockdev_backup_prepare(BlkActionState *common, Error **errp)
+@@ -XXX,XX +XXX,XX @@ static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
          return;
      }
 -    /* AioContext is released in .clean() */
 -    state->aio_context = bdrv_get_aio_context(bs);
 -    if (state->aio_context != bdrv_get_aio_context(target)) {
 -        state->aio_context = NULL;
 +    aio_context = bdrv_get_aio_context(bs);
 +    if (aio_context != bdrv_get_aio_context(target)) {
          error_setg(errp, "Backup between two IO threads is not implemented");
          return;
      }
 -    aio_context_acquire(state->aio_context);
 +    aio_context_acquire(aio_context);
      state->bs = bs;
 +
 +    /* Paired with .clean() */
      bdrv_drained_begin(state->bs);
      state->job = do_blockdev_backup(backup, common->block_job_txn, &local_err);
      if (local_err) {
          error_propagate(errp, local_err);
 -        return;
 +        goto out;
      }
 +
 +out:
 +    aio_context_release(aio_context);
  }
- static void blockdev_backup_commit(BlkActionState *common)
+ static void coroutine_fn block_copy_wait_inflight_reqs(BlockCopyState *s,
- {
+-                                                       int64_t start,
-     BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
++                                                       int64_t offset,
-+    AioContext *aio_context;
+                                                        int64_t bytes)
-+
+ {
-+    aio_context = bdrv_get_aio_context(state->bs);
+     BlockCopyInFlightReq *req;
-+    aio_context_acquire(aio_context);
-+
+-    while ((req = find_conflicting_inflight_req(s, start, bytes))) {
-     assert(state->job);
++    while ((req = find_conflicting_inflight_req(s, offset, bytes))) {
-     block_job_start(state->job);
+         qemu_co_queue_wait(&req->wait_queue, NULL);
-+
+     }
 +    aio_context_release(aio_context);
  }
- static void blockdev_backup_abort(BlkActionState *common)
+ static void block_copy_inflight_req_begin(BlockCopyState *s,
-@@ -XXX,XX +XXX,XX @@ static void blockdev_backup_abort(BlkActionState *common)
+                                           BlockCopyInFlightReq *req,
-     BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
+-                                          int64_t start, int64_t bytes)
++                                          int64_t offset, int64_t bytes)
-     if (state->job) {
+ {
-+        AioContext *aio_context;
+-    req->start = start;
-+
++    req->offset = offset;
-+        aio_context = bdrv_get_aio_context(state->bs);
+     req->bytes = bytes;
-+        aio_context_acquire(aio_context);
+     qemu_co_queue_init(&req->wait_queue);
-+
+     QLIST_INSERT_HEAD(&s->inflight_reqs, req, list);
-         block_job_cancel_sync(state->job);
+@@ -XXX,XX +XXX,XX @@ void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm)
-+
+  * Returns 0 on success.
-+        aio_context_release(aio_context);
+  */
-     }
+ static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
 -                                           int64_t start, int64_t bytes,
 +                                           int64_t offset, int64_t bytes,
                                             bool zeroes, bool *error_is_read)
  {
      int ret;
 -    int64_t nbytes = MIN(start + bytes, s->len) - start;
 +    int64_t nbytes = MIN(offset + bytes, s->len) - offset;
      void *bounce_buffer = NULL;
 -    assert(start >= 0 && bytes > 0 && INT64_MAX - start >= bytes);
 -    assert(QEMU_IS_ALIGNED(start, s->cluster_size));
 +    assert(offset >= 0 && bytes > 0 && INT64_MAX - offset >= bytes);
 +    assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
      assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
 -    assert(start < s->len);
 -    assert(start + bytes <= s->len ||
 -           start + bytes == QEMU_ALIGN_UP(s->len, s->cluster_size));
 +    assert(offset < s->len);
 +    assert(offset + bytes <= s->len ||
 +           offset + bytes == QEMU_ALIGN_UP(s->len, s->cluster_size));
      assert(nbytes < INT_MAX);
      if (zeroes) {
 -        ret = bdrv_co_pwrite_zeroes(s->target, start, nbytes, s->write_flags &
 +        ret = bdrv_co_pwrite_zeroes(s->target, offset, nbytes, s->write_flags &
                                      ~BDRV_REQ_WRITE_COMPRESSED);
          if (ret < 0) {
 -            trace_block_copy_write_zeroes_fail(s, start, ret);
 +            trace_block_copy_write_zeroes_fail(s, offset, ret);
              if (error_is_read) {
                  *error_is_read = false;
              }
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
      }
      if (s->use_copy_range) {
 -        ret = bdrv_co_copy_range(s->source, start, s->target, start, nbytes,
 +        ret = bdrv_co_copy_range(s->source, offset, s->target, offset, nbytes,
 , s->write_flags);
          if (ret < 0) {
 -            trace_block_copy_copy_range_fail(s, start, ret);
 +            trace_block_copy_copy_range_fail(s, offset, ret);
              s->use_copy_range = false;
              s->copy_size = MAX(s->cluster_size, BLOCK_COPY_MAX_BUFFER);
              /* Fallback to read+write with allocated buffer */
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
      bounce_buffer = qemu_blockalign(s->source->bs, nbytes);
 -    ret = bdrv_co_pread(s->source, start, nbytes, bounce_buffer, 0);
 +    ret = bdrv_co_pread(s->source, offset, nbytes, bounce_buffer, 0);
      if (ret < 0) {
 -        trace_block_copy_read_fail(s, start, ret);
 +        trace_block_copy_read_fail(s, offset, ret);
          if (error_is_read) {
              *error_is_read = true;
          }
          goto out;
      }
 -    ret = bdrv_co_pwrite(s->target, start, nbytes, bounce_buffer,
 +    ret = bdrv_co_pwrite(s->target, offset, nbytes, bounce_buffer,
                           s->write_flags);
      if (ret < 0) {
 -        trace_block_copy_write_fail(s, start, ret);
 +        trace_block_copy_write_fail(s, offset, ret);
          if (error_is_read) {
              *error_is_read = false;
          }
@@ -XXX,XX +XXX,XX @@ int64_t block_copy_reset_unallocated(BlockCopyState *s,
  }
- static void blockdev_backup_clean(BlkActionState *common)
+ int coroutine_fn block_copy(BlockCopyState *s,
- {
+-                            int64_t start, int64_t bytes,
-     BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
++                            int64_t offset, int64_t bytes,
-+    AioContext *aio_context;
+                             bool *error_is_read)
+ {
--    if (state->aio_context) {
+     int ret = 0;
--        bdrv_drained_end(state->bs);
+@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
--        aio_context_release(state->aio_context);
+     assert(bdrv_get_aio_context(s->source->bs) ==
-+    if (!state->bs) {
+            bdrv_get_aio_context(s->target->bs));
-+        return;
-     }
+-    assert(QEMU_IS_ALIGNED(start, s->cluster_size));
-+
++    assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
-+    aio_context = bdrv_get_aio_context(state->bs);
+     assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
-+    aio_context_acquire(aio_context);
-+
+-    block_copy_wait_inflight_reqs(s, start, bytes);
-+    bdrv_drained_end(state->bs);
+-    block_copy_inflight_req_begin(s, &req, start, bytes);
-+
++    block_copy_wait_inflight_reqs(s, offset, bytes);
-+    aio_context_release(aio_context);
++    block_copy_inflight_req_begin(s, &req, offset, bytes);
- }
+     while (bytes) {
- typedef struct BlockDirtyBitmapState {
+         int64_t next_zero, cur_bytes, status_bytes;
 -        if (!bdrv_dirty_bitmap_get(s->copy_bitmap, start)) {
 -            trace_block_copy_skip(s, start);
 -            start += s->cluster_size;
 +        if (!bdrv_dirty_bitmap_get(s->copy_bitmap, offset)) {
 +            trace_block_copy_skip(s, offset);
 +            offset += s->cluster_size;
              bytes -= s->cluster_size;
              continue; /* already copied */
          }
          cur_bytes = MIN(bytes, s->copy_size);
 -        next_zero = bdrv_dirty_bitmap_next_zero(s->copy_bitmap, start,
 +        next_zero = bdrv_dirty_bitmap_next_zero(s->copy_bitmap, offset,
                                                  cur_bytes);
          if (next_zero >= 0) {
 -            assert(next_zero > start); /* start is dirty */
 -            assert(next_zero < start + cur_bytes); /* no need to do MIN() */
 -            cur_bytes = next_zero - start;
 +            assert(next_zero > offset); /* offset is dirty */
 +            assert(next_zero < offset + cur_bytes); /* no need to do MIN() */
 +            cur_bytes = next_zero - offset;
          }
 -        ret = block_copy_block_status(s, start, cur_bytes, &status_bytes);
 +        ret = block_copy_block_status(s, offset, cur_bytes, &status_bytes);
          if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
 -            bdrv_reset_dirty_bitmap(s->copy_bitmap, start, status_bytes);
 +            bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, status_bytes);
              progress_set_remaining(s->progress,
                                     bdrv_get_dirty_count(s->copy_bitmap) +
                                     s->in_flight_bytes);
 -            trace_block_copy_skip_range(s, start, status_bytes);
 -            start += status_bytes;
 +            trace_block_copy_skip_range(s, offset, status_bytes);
 +            offset += status_bytes;
              bytes -= status_bytes;
              continue;
          }
          cur_bytes = MIN(cur_bytes, status_bytes);
 -        trace_block_copy_process(s, start);
 +        trace_block_copy_process(s, offset);
 -        bdrv_reset_dirty_bitmap(s->copy_bitmap, start, cur_bytes);
 +        bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, cur_bytes);
          s->in_flight_bytes += cur_bytes;
          co_get_from_shres(s->mem, cur_bytes);
 -        ret = block_copy_do_copy(s, start, cur_bytes, ret & BDRV_BLOCK_ZERO,
 +        ret = block_copy_do_copy(s, offset, cur_bytes, ret & BDRV_BLOCK_ZERO,
                                   error_is_read);
          co_put_to_shres(s->mem, cur_bytes);
          s->in_flight_bytes -= cur_bytes;
          if (ret < 0) {
 -            bdrv_set_dirty_bitmap(s->copy_bitmap, start, cur_bytes);
 +            bdrv_set_dirty_bitmap(s->copy_bitmap, offset, cur_bytes);
              break;
          }
          progress_work_done(s->progress, cur_bytes);
          s->progress_bytes_callback(cur_bytes, s->progress_opaque);
 -        start += cur_bytes;
 +        offset += cur_bytes;
          bytes -= cur_bytes;
      }
 diff --git a/include/block/block-copy.h b/include/block/block-copy.h
 index XXXXXXX..XXXXXXX 100644
 --- a/include/block/block-copy.h
 +++ b/include/block/block-copy.h
@@ -XXX,XX +XXX,XX @@
  #include "qemu/co-shared-resource.h"
  typedef struct BlockCopyInFlightReq {
 -    int64_t start;
 +    int64_t offset;
      int64_t bytes;
      QLIST_ENTRY(BlockCopyInFlightReq) list;
      CoQueue wait_queue; /* coroutines blocked on this request */
@@ -XXX,XX +XXX,XX @@ void block_copy_state_free(BlockCopyState *s);
  int64_t block_copy_reset_unallocated(BlockCopyState *s,
                                       int64_t offset, int64_t *count);
 -int coroutine_fn block_copy(BlockCopyState *s, int64_t start, int64_t bytes,
 +int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes,
                              bool *error_is_read);
  #endif /* BLOCK_COPY_H */
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 03/23] hw/block: Fix the return type
+[PULL 18/19] block/block-copy: reduce intersecting request lock
-From: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>
+From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-When the function no success value to transmit, it usually make the
+Currently, block_copy operation lock the whole requested region. But
-function return void. It has turned out not to be a success, because
+there is no reason to lock clusters, which are already copied, it will
-it means that the extra local_err variable and error_propagate() will
+disturb other parallel block_copy requests for no reason.
-be needed. It leads to cumbersome code, therefore, transmit success/
-failure in the return value is worth.
+Let's instead do the following:
-So fix the return type of blkconf_apply_backend_options(),
+Lock only sub-region, which we are going to operate on. Then, after
-blkconf_geometry() and virtio_blk_data_plane_create() to avoid it.
+copying all dirty sub-regions, we should wait for intersecting
+requests block-copy, if they failed, we should retry these new dirty
-Cc: John Snow <jsnow@redhat.com>
+clusters.
-Cc: Kevin Wolf <kwolf@redhat.com>
-Cc: Max Reitz <mreitz@redhat.com>
+Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-Cc: Stefan Hajnoczi <stefanha@redhat.com>
+Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
+Message-Id: <20200311103004.7649-9-vsementsov@virtuozzo.com>
-Signed-off-by: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>
+Signed-off-by: Max Reitz <mreitz@redhat.com>
 Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
 Message-id: ac0edc1fc70c4457e5cec94405eb7d1f89f9c2c1.1511317952.git.maozy.fnst@cn.fujitsu.com
 Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
 ---
- hw/block/dataplane/virtio-blk.h |  2 +-
+ block/block-copy.c | 129 ++++++++++++++++++++++++++++++++++++---------
- include/hw/block/block.h        |  4 ++--
+file changed, 105 insertions(+), 24 deletions(-)
- hw/block/block.c                | 15 +++++++++------
- hw/block/dataplane/virtio-blk.c | 12 +++++++-----
+diff --git a/block/block-copy.c b/block/block-copy.c
 files changed, 19 insertions(+), 14 deletions(-)
 diff --git a/hw/block/dataplane/virtio-blk.h b/hw/block/dataplane/virtio-blk.h
 index XXXXXXX..XXXXXXX 100644
---- a/hw/block/dataplane/virtio-blk.h
+--- a/block/block-copy.c
-+++ b/hw/block/dataplane/virtio-blk.h
++++ b/block/block-copy.c
-@@ -XXX,XX +XXX,XX @@
+@@ -XXX,XX +XXX,XX @@ static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
+     return NULL;
- typedef struct VirtIOBlockDataPlane VirtIOBlockDataPlane;
+ }
--void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
+-static void coroutine_fn block_copy_wait_inflight_reqs(BlockCopyState *s,
-+bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
+-                                                       int64_t offset,
-                                   VirtIOBlockDataPlane **dataplane,
+-                                                       int64_t bytes)
-                                   Error **errp);
++/*
- void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s);
++ * If there are no intersecting requests return false. Otherwise, wait for the
-diff --git a/include/hw/block/block.h b/include/hw/block/block.h
++ * first found intersecting request to finish and return true.
-index XXXXXXX..XXXXXXX 100644
++ */
---- a/include/hw/block/block.h
++static bool coroutine_fn block_copy_wait_one(BlockCopyState *s, int64_t offset,
-+++ b/include/hw/block/block.h
++                                             int64_t bytes)
-@@ -XXX,XX +XXX,XX @@ static inline unsigned int get_physical_block_exp(BlockConf *conf)
+ {
- /* Configuration helpers */
+-    BlockCopyInFlightReq *req;
++    BlockCopyInFlightReq *req = find_conflicting_inflight_req(s, offset, bytes);
- void blkconf_serial(BlockConf *conf, char **serial);
--void blkconf_geometry(BlockConf *conf, int *trans,
+-    while ((req = find_conflicting_inflight_req(s, offset, bytes))) {
-+bool blkconf_geometry(BlockConf *conf, int *trans,
+-        qemu_co_queue_wait(&req->wait_queue, NULL);
-                       unsigned cyls_max, unsigned heads_max, unsigned secs_max,
++    if (!req) {
                        Error **errp);
  void blkconf_blocksizes(BlockConf *conf);
 -void blkconf_apply_backend_options(BlockConf *conf, bool readonly,
 +bool blkconf_apply_backend_options(BlockConf *conf, bool readonly,
                                     bool resizable, Error **errp);
  /* Hard disk geometry */
 diff --git a/hw/block/block.c b/hw/block/block.c
 index XXXXXXX..XXXXXXX 100644
 --- a/hw/block/block.c
 +++ b/hw/block/block.c
@@ -XXX,XX +XXX,XX @@ void blkconf_blocksizes(BlockConf *conf)
      }
  }
 -void blkconf_apply_backend_options(BlockConf *conf, bool readonly,
 +bool blkconf_apply_backend_options(BlockConf *conf, bool readonly,
                                     bool resizable, Error **errp)
  {
      BlockBackend *blk = conf->blk;
@@ -XXX,XX +XXX,XX @@ void blkconf_apply_backend_options(BlockConf *conf, bool readonly,
      ret = blk_set_perm(blk, perm, shared_perm, errp);
      if (ret < 0) {
 -        return;
 +        return false;
      }
++
-     switch (conf->wce) {
++    qemu_co_queue_wait(&req->wait_queue, NULL);
@@ -XXX,XX +XXX,XX @@ void blkconf_apply_backend_options(BlockConf *conf, bool readonly,
      blk_set_enable_write_cache(blk, wce);
      blk_set_on_error(blk, rerror, werror);
 +
 +    return true;
  }
--void blkconf_geometry(BlockConf *conf, int *ptrans,
++/* Called only on full-dirty region */
-+bool blkconf_geometry(BlockConf *conf, int *ptrans,
+ static void block_copy_inflight_req_begin(BlockCopyState *s,
-                       unsigned cyls_max, unsigned heads_max, unsigned secs_max,
+                                           BlockCopyInFlightReq *req,
-                       Error **errp)
+                                           int64_t offset, int64_t bytes)
  {
-@@ -XXX,XX +XXX,XX @@ void blkconf_geometry(BlockConf *conf, int *ptrans,
++    assert(!find_conflicting_inflight_req(s, offset, bytes));
-     if (conf->cyls || conf->heads || conf->secs) {
++
-         if (conf->cyls < 1 || conf->cyls > cyls_max) {
++    bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
-             error_setg(errp, "cyls must be between 1 and %u", cyls_max);
++    s->in_flight_bytes += bytes;
--            return;
++
-+            return false;
+     req->offset = offset;
-         }
+     req->bytes = bytes;
-         if (conf->heads < 1 || conf->heads > heads_max) {
+     qemu_co_queue_init(&req->wait_queue);
-             error_setg(errp, "heads must be between 1 and %u", heads_max);
+     QLIST_INSERT_HEAD(&s->inflight_reqs, req, list);
--            return;
+ }
-+            return false;
-         }
+-static void coroutine_fn block_copy_inflight_req_end(BlockCopyInFlightReq *req)
-         if (conf->secs < 1 || conf->secs > secs_max) {
++/*
-             error_setg(errp, "secs must be between 1 and %u", secs_max);
++ * block_copy_inflight_req_shrink
--            return;
++ *
-+            return false;
++ * Drop the tail of the request to be handled later. Set dirty bits back and
-         }
++ * wake up all requests waiting for us (may be some of them are not intersecting
 + * with shrunk request)
 + */
 +static void coroutine_fn block_copy_inflight_req_shrink(BlockCopyState *s,
 +        BlockCopyInFlightReq *req, int64_t new_bytes)
  {
 +    if (new_bytes == req->bytes) {
 +        return;
 +    }
 +
 +    assert(new_bytes > 0 && new_bytes < req->bytes);
 +
 +    s->in_flight_bytes -= req->bytes - new_bytes;
 +    bdrv_set_dirty_bitmap(s->copy_bitmap,
 +                          req->offset + new_bytes, req->bytes - new_bytes);
 +
 +    req->bytes = new_bytes;
 +    qemu_co_queue_restart_all(&req->wait_queue);
 +}
 +
 +static void coroutine_fn block_copy_inflight_req_end(BlockCopyState *s,
 +                                                     BlockCopyInFlightReq *req,
 +                                                     int ret)
 +{
 +    s->in_flight_bytes -= req->bytes;
 +    if (ret < 0) {
 +        bdrv_set_dirty_bitmap(s->copy_bitmap, req->offset, req->bytes);
 +    }
      QLIST_REMOVE(req, list);
      qemu_co_queue_restart_all(&req->wait_queue);
  }
@@ -XXX,XX +XXX,XX @@ int64_t block_copy_reset_unallocated(BlockCopyState *s,
      return ret;
  }
 -int coroutine_fn block_copy(BlockCopyState *s,
 -                            int64_t offset, int64_t bytes,
 -                            bool *error_is_read)
 +/*
 + * block_copy_dirty_clusters
 + *
 + * Copy dirty clusters in @offset/@bytes range.
 + * Returns 1 if dirty clusters found and successfully copied, 0 if no dirty
 + * clusters found and -errno on failure.
 + */
 +static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
 +                                                  int64_t offset, int64_t bytes,
 +                                                  bool *error_is_read)
  {
      int ret = 0;
 -    BlockCopyInFlightReq req;
 +    bool found_dirty = false;
      /*
       * block_copy() user is responsible for keeping source and target in same
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
      assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
      assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
 -    block_copy_wait_inflight_reqs(s, offset, bytes);
 -    block_copy_inflight_req_begin(s, &req, offset, bytes);
 -
      while (bytes) {
 +        BlockCopyInFlightReq req;
          int64_t next_zero, cur_bytes, status_bytes;
          if (!bdrv_dirty_bitmap_get(s->copy_bitmap, offset)) {
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
              continue; /* already copied */
          }
 +        found_dirty = true;
 +
          cur_bytes = MIN(bytes, s->copy_size);
          next_zero = bdrv_dirty_bitmap_next_zero(s->copy_bitmap, offset,
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
              assert(next_zero < offset + cur_bytes); /* no need to do MIN() */
              cur_bytes = next_zero - offset;
          }
 +        block_copy_inflight_req_begin(s, &req, offset, cur_bytes);
          ret = block_copy_block_status(s, offset, cur_bytes, &status_bytes);
 +        assert(ret >= 0); /* never fail */
 +        cur_bytes = MIN(cur_bytes, status_bytes);
 +        block_copy_inflight_req_shrink(s, &req, cur_bytes);
          if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
 -            bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, status_bytes);
 +            block_copy_inflight_req_end(s, &req, 0);
              progress_set_remaining(s->progress,
                                     bdrv_get_dirty_count(s->copy_bitmap) +
                                     s->in_flight_bytes);
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
              continue;
          }
 -        cur_bytes = MIN(cur_bytes, status_bytes);
 -
          trace_block_copy_process(s, offset);
 -        bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, cur_bytes);
 -        s->in_flight_bytes += cur_bytes;
 -
          co_get_from_shres(s->mem, cur_bytes);
          ret = block_copy_do_copy(s, offset, cur_bytes, ret & BDRV_BLOCK_ZERO,
                                   error_is_read);
          co_put_to_shres(s->mem, cur_bytes);
 -        s->in_flight_bytes -= cur_bytes;
 +        block_copy_inflight_req_end(s, &req, ret);
          if (ret < 0) {
 -            bdrv_set_dirty_bitmap(s->copy_bitmap, offset, cur_bytes);
 -            break;
 +            return ret;
          }
          progress_work_done(s->progress, cur_bytes);
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
          bytes -= cur_bytes;
      }
-+    return true;
- }
+-    block_copy_inflight_req_end(&req);
-diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c
++    return found_dirty;
-index XXXXXXX..XXXXXXX 100644
++}
---- a/hw/block/dataplane/virtio-blk.c
++
-+++ b/hw/block/dataplane/virtio-blk.c
++/*
-@@ -XXX,XX +XXX,XX @@ static void notify_guest_bh(void *opaque)
++ * block_copy
- }
++ *
++ * Copy requested region, accordingly to dirty bitmap.
- /* Context: QEMU global mutex held */
++ * Collaborate with parallel block_copy requests: if they succeed it will help
--void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
++ * us. If they fail, we will retry not-copied regions. So, if we return error,
-+bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
++ * it means that some I/O operation failed in context of _this_ block_copy call,
-                                   VirtIOBlockDataPlane **dataplane,
++ * not some parallel operation.
-                                   Error **errp)
++ */
- {
++int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes,
-@@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
++                            bool *error_is_read)
-             error_setg(errp,
++{
-                        "device is incompatible with iothread "
++    int ret;
-                        "(transport does not support notifiers)");
++
--            return;
++    do {
-+            return false;
++        ret = block_copy_dirty_clusters(s, offset, bytes, error_is_read);
-         }
++
-         if (!virtio_device_ioeventfd_enabled(vdev)) {
++        if (ret == 0) {
-             error_setg(errp, "ioeventfd is required for iothread");
++            ret = block_copy_wait_one(s, offset, bytes);
--            return;
++        }
-+            return false;
++
-         }
++        /*
++         * We retry in two cases:
-         /* If dataplane is (re-)enabled while the guest is running there could
++         * 1. Some progress done
-@@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
++         *    Something was copied, which means that there were yield points
-          */
++         *    and some new dirty bits may have appeared (due to failed parallel
-         if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) {
++         *    block-copy requests).
-             error_prepend(errp, "cannot start virtio-blk dataplane: ");
++         * 2. We have waited for some intersecting block-copy request
--            return;
++         *    It may have failed and produced new dirty bits.
-+            return false;
++         */
-         }
++    } while (ret > 0);
-     }
-     /* Don't try if transport does not support notifiers. */
+     return ret;
-     if (!virtio_device_ioeventfd_enabled(vdev)) {
+ }
 -        return;
 +        return false;
      }
      s = g_new0(VirtIOBlockDataPlane, 1);
@@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
      s->batch_notify_vqs = bitmap_new(conf->num_queues);
      *dataplane = s;
 +
 +    return true;
  }
  /* Context: QEMU global mutex held */
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 04/23] hw/block: Use errp directly rather than local_err
+Deleted patch
-From: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>
-Cc: John Snow <jsnow@redhat.com>
-Cc: Kevin Wolf <kwolf@redhat.com>
-Cc: Max Reitz <mreitz@redhat.com>
-Cc: Keith Busch <keith.busch@intel.com>
-Cc: Stefan Hajnoczi <stefanha@redhat.com>
-Cc: "Michael S. Tsirkin" <mst@redhat.com>
-Cc: Paolo Bonzini <pbonzini@redhat.com>
-Cc: Gerd Hoffmann <kraxel@redhat.com>
-Cc: Markus Armbruster <armbru@redhat.com>
-Signed-off-by: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>
-Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
-Message-id: e77848d3735ba590f23ffbf8094379c646c33d79.1511317952.git.maozy.fnst@cn.fujitsu.com
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
----
- hw/block/fdc.c        | 17 ++++++-----------
- hw/block/nvme.c       |  7 ++-----
- hw/block/virtio-blk.c | 18 ++++++------------
- hw/ide/qdev.c         | 12 ++++--------
- hw/scsi/scsi-disk.c   | 13 ++++---------
- hw/usb/dev-storage.c  |  9 +++------
-files changed, 25 insertions(+), 51 deletions(-)
-diff --git a/hw/block/fdc.c b/hw/block/fdc.c
-index XXXXXXX..XXXXXXX 100644
---- a/hw/block/fdc.c
-+++ b/hw/block/fdc.c
-@@ -XXX,XX +XXX,XX @@ static void fd_revalidate(FDrive *drv)
- static void fd_change_cb(void *opaque, bool load, Error **errp)
- {
-     FDrive *drive = opaque;
--    Error *local_err = NULL;
-     if (!load) {
-         blk_set_perm(drive->blk, 0, BLK_PERM_ALL, &error_abort);
-     } else {
--        blkconf_apply_backend_options(drive->conf,
--                                      blk_is_read_only(drive->blk), false,
--                                      &local_err);
--        if (local_err) {
--            error_propagate(errp, local_err);
-+        if (!blkconf_apply_backend_options(drive->conf,
-+                                           blk_is_read_only(drive->blk), false,
-+                                           errp)) {
-             return;
-         }
-     }
-@@ -XXX,XX +XXX,XX @@ static void floppy_drive_realize(DeviceState *qdev, Error **errp)
-     FloppyDrive *dev = FLOPPY_DRIVE(qdev);
-     FloppyBus *bus = FLOPPY_BUS(qdev->parent_bus);
-     FDrive *drive;
--    Error *local_err = NULL;
-     int ret;
-     if (dev->unit == -1) {
-@@ -XXX,XX +XXX,XX @@ static void floppy_drive_realize(DeviceState *qdev, Error **errp)
-     dev->conf.rerror = BLOCKDEV_ON_ERROR_AUTO;
-     dev->conf.werror = BLOCKDEV_ON_ERROR_AUTO;
--    blkconf_apply_backend_options(&dev->conf, blk_is_read_only(dev->conf.blk),
--                                  false, &local_err);
--    if (local_err) {
--        error_propagate(errp, local_err);
-+    if (!blkconf_apply_backend_options(&dev->conf,
-+                                       blk_is_read_only(dev->conf.blk),
-+                                       false, errp)) {
-         return;
-     }
-diff --git a/hw/block/nvme.c b/hw/block/nvme.c
-index XXXXXXX..XXXXXXX 100644
---- a/hw/block/nvme.c
-+++ b/hw/block/nvme.c
-@@ -XXX,XX +XXX,XX @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)
-     int i;
-     int64_t bs_size;
-     uint8_t *pci_conf;
--    Error *local_err = NULL;
-     if (!n->conf.blk) {
-         error_setg(errp, "drive property not set");
-@@ -XXX,XX +XXX,XX @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)
-         return;
-     }
-     blkconf_blocksizes(&n->conf);
--    blkconf_apply_backend_options(&n->conf, blk_is_read_only(n->conf.blk),
--                                  false, &local_err);
--    if (local_err) {
--        error_propagate(errp, local_err);
-+    if (!blkconf_apply_backend_options(&n->conf, blk_is_read_only(n->conf.blk),
-+                                       false, errp)) {
-         return;
-     }
-diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
-index XXXXXXX..XXXXXXX 100644
---- a/hw/block/virtio-blk.c
-+++ b/hw/block/virtio-blk.c
-@@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
-     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
-     VirtIOBlock *s = VIRTIO_BLK(dev);
-     VirtIOBlkConf *conf = &s->conf;
--    Error *err = NULL;
-     unsigned i;
-     if (!conf->conf.blk) {
-@@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
-     }
-     blkconf_serial(&conf->conf, &conf->serial);
--    blkconf_apply_backend_options(&conf->conf,
--                                  blk_is_read_only(conf->conf.blk), true,
--                                  &err);
--    if (err) {
--        error_propagate(errp, err);
-+    if (!blkconf_apply_backend_options(&conf->conf,
-+                                       blk_is_read_only(conf->conf.blk), true,
-+                                       errp)) {
-         return;
-     }
-     s->original_wce = blk_enable_write_cache(conf->conf.blk);
--    blkconf_geometry(&conf->conf, NULL, 65535, 255, 255, &err);
--    if (err) {
--        error_propagate(errp, err);
-+    if (!blkconf_geometry(&conf->conf, NULL, 65535, 255, 255, errp)) {
-         return;
-     }
-+
-     blkconf_blocksizes(&conf->conf);
-     virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK,
-@@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
-     for (i = 0; i < conf->num_queues; i++) {
-         virtio_add_queue(vdev, 128, virtio_blk_handle_output);
-     }
--    virtio_blk_data_plane_create(vdev, conf, &s->dataplane, &err);
--    if (err != NULL) {
--        error_propagate(errp, err);
-+    if (!virtio_blk_data_plane_create(vdev, conf, &s->dataplane, errp)) {
-         virtio_cleanup(vdev);
-         return;
-     }
-diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c
-index XXXXXXX..XXXXXXX 100644
---- a/hw/ide/qdev.c
-+++ b/hw/ide/qdev.c
-@@ -XXX,XX +XXX,XX @@ static void ide_dev_initfn(IDEDevice *dev, IDEDriveKind kind, Error **errp)
- {
-     IDEBus *bus = DO_UPCAST(IDEBus, qbus, dev->qdev.parent_bus);
-     IDEState *s = bus->ifs + dev->unit;
--    Error *err = NULL;
-     int ret;
-     if (!dev->conf.blk) {
-@@ -XXX,XX +XXX,XX @@ static void ide_dev_initfn(IDEDevice *dev, IDEDriveKind kind, Error **errp)
-     blkconf_serial(&dev->conf, &dev->serial);
-     if (kind != IDE_CD) {
--        blkconf_geometry(&dev->conf, &dev->chs_trans, 65535, 16, 255, &err);
--        if (err) {
--            error_propagate(errp, err);
-+        if (!blkconf_geometry(&dev->conf, &dev->chs_trans, 65535, 16, 255,
-+                              errp)) {
-             return;
-         }
-     }
--    blkconf_apply_backend_options(&dev->conf, kind == IDE_CD, kind != IDE_CD,
--                                  &err);
--    if (err) {
--        error_propagate(errp, err);
-+    if (!blkconf_apply_backend_options(&dev->conf, kind == IDE_CD,
-+                                       kind != IDE_CD, errp)) {
-         return;
-     }
-diff --git a/hw/scsi/scsi-disk.c b/hw/scsi/scsi-disk.c
-index XXXXXXX..XXXXXXX 100644
---- a/hw/scsi/scsi-disk.c
-+++ b/hw/scsi/scsi-disk.c
-@@ -XXX,XX +XXX,XX @@ static void scsi_disk_unit_attention_reported(SCSIDevice *dev)
- static void scsi_realize(SCSIDevice *dev, Error **errp)
- {
-     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, dev);
--    Error *err = NULL;
-     if (!s->qdev.conf.blk) {
-         error_setg(errp, "drive property not set");
-@@ -XXX,XX +XXX,XX @@ static void scsi_realize(SCSIDevice *dev, Error **errp)
-     }
-     if (dev->type == TYPE_DISK) {
--        blkconf_geometry(&dev->conf, NULL, 65535, 255, 255, &err);
--        if (err) {
--            error_propagate(errp, err);
-+        if (!blkconf_geometry(&dev->conf, NULL, 65535, 255, 255, errp)) {
-             return;
-         }
-     }
--    blkconf_apply_backend_options(&dev->conf,
--                                  blk_is_read_only(s->qdev.conf.blk),
--                                  dev->type == TYPE_DISK, &err);
--    if (err) {
--        error_propagate(errp, err);
-+    if (!blkconf_apply_backend_options(&dev->conf,
-+                                       blk_is_read_only(s->qdev.conf.blk),
-+                                       dev->type == TYPE_DISK, errp)) {
-         return;
-     }
-diff --git a/hw/usb/dev-storage.c b/hw/usb/dev-storage.c
-index XXXXXXX..XXXXXXX 100644
---- a/hw/usb/dev-storage.c
-+++ b/hw/usb/dev-storage.c
-@@ -XXX,XX +XXX,XX @@ static void usb_msd_realize_storage(USBDevice *dev, Error **errp)
-     MSDState *s = USB_STORAGE_DEV(dev);
-     BlockBackend *blk = s->conf.blk;
-     SCSIDevice *scsi_dev;
--    Error *err = NULL;
-     if (!blk) {
-         error_setg(errp, "drive property not set");
-@@ -XXX,XX +XXX,XX @@ static void usb_msd_realize_storage(USBDevice *dev, Error **errp)
-     blkconf_serial(&s->conf, &dev->serial);
-     blkconf_blocksizes(&s->conf);
--    blkconf_apply_backend_options(&s->conf, blk_is_read_only(blk), true, &err);
--    if (err) {
--        error_propagate(errp, err);
-+    if (!blkconf_apply_backend_options(&s->conf, blk_is_read_only(blk), true,
-+                                       errp)) {
-         return;
-     }
-@@ -XXX,XX +XXX,XX @@ static void usb_msd_realize_storage(USBDevice *dev, Error **errp)
-                  &usb_msd_scsi_info_storage, NULL);
-     scsi_dev = scsi_bus_legacy_add_drive(&s->bus, blk, 0, !!s->removable,
-                                          s->conf.bootindex, dev->serial,
--                                         &err);
-+                                         errp);
-     blk_unref(blk);
-     if (!scsi_dev) {
--        error_propagate(errp, err);
-         return;
-     }
-     usb_msd_handle_reset(dev);
---
-.14.3

-[Qemu-devel] [PULL 06/23] qdev: drop unused #include "sysemu/iothread.h"
+Deleted patch
-Commit 1351d1ec89eabebc9fdff20451a62c413d7accc1 ("qdev: drop iothread
-property type") forgot to remove this include.
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
-Message-id: 20171205133954.31006-1-stefanha@redhat.com
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
----
- hw/core/qdev-properties-system.c | 1 -
-file changed, 1 deletion(-)
-diff --git a/hw/core/qdev-properties-system.c b/hw/core/qdev-properties-system.c
-index XXXXXXX..XXXXXXX 100644
---- a/hw/core/qdev-properties-system.c
-+++ b/hw/core/qdev-properties-system.c
-@@ -XXX,XX +XXX,XX @@
- #include "qapi/visitor.h"
- #include "chardev/char-fe.h"
- #include "sysemu/tpm_backend.h"
--#include "sysemu/iothread.h"
- static void get_pointer(Object *obj, Visitor *v, Property *prop,
-                         char *(*print)(void *ptr),
---
-.14.3

-[Qemu-devel] [PULL 09/23] block: don't keep AioContext acquired after drive_backup_prepare()
+[PULL 19/19] block/block-copy: hide structure definitions
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
+From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
-Reviewed-by: Kevin Wolf <kwolf@redhat.com>
-Reviewed-by: Eric Blake <eblake@redhat.com>
+Hide structure definitions and add explicit API instead, to keep an
-Message-id: 20171206144550.22295-4-stefanha@redhat.com
+eye on the scope of the shared fields.
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
 Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
 Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
 Reviewed-by: Max Reitz <mreitz@redhat.com>
 Message-Id: <20200311103004.7649-10-vsementsov@virtuozzo.com>
 Signed-off-by: Max Reitz <mreitz@redhat.com>
 ---
- blockdev.c | 42 ++++++++++++++++++++++++++++++++++--------
+ block/backup-top.c         |  6 ++--
-file changed, 34 insertions(+), 8 deletions(-)
+ block/backup.c             | 25 ++++++++--------
+ block/block-copy.c         | 59 ++++++++++++++++++++++++++++++++++++++
-diff --git a/blockdev.c b/blockdev.c
+ include/block/block-copy.h | 52 +++------------------------------
-index XXXXXXX..XXXXXXX 100644
+files changed, 80 insertions(+), 62 deletions(-)
---- a/blockdev.c
-+++ b/blockdev.c
+diff --git a/block/backup-top.c b/block/backup-top.c
-@@ -XXX,XX +XXX,XX @@ static void external_snapshot_clean(BlkActionState *common)
+index XXXXXXX..XXXXXXX 100644
- typedef struct DriveBackupState {
+--- a/block/backup-top.c
-     BlkActionState common;
++++ b/block/backup-top.c
-     BlockDriverState *bs;
+@@ -XXX,XX +XXX,XX @@ typedef struct BDRVBackupTopState {
--    AioContext *aio_context;
+     BlockCopyState *bcs;
-     BlockJob *job;
+     BdrvChild *target;
- } DriveBackupState;
+     bool active;
++    int64_t cluster_size;
-@@ -XXX,XX +XXX,XX @@ static void drive_backup_prepare(BlkActionState *common, Error **errp)
+ } BDRVBackupTopState;
-     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
-     BlockDriverState *bs;
+ static coroutine_fn int backup_top_co_preadv(
-     DriveBackup *backup;
+@@ -XXX,XX +XXX,XX @@ static coroutine_fn int backup_top_cbw(BlockDriverState *bs, uint64_t offset,
-+    AioContext *aio_context;
+         return 0;
-     Error *local_err = NULL;
+     }
-     assert(common->action->type == TRANSACTION_ACTION_KIND_DRIVE_BACKUP);
+-    off = QEMU_ALIGN_DOWN(offset, s->bcs->cluster_size);
-@@ -XXX,XX +XXX,XX @@ static void drive_backup_prepare(BlkActionState *common, Error **errp)
+-    end = QEMU_ALIGN_UP(offset + bytes, s->bcs->cluster_size);
 +    off = QEMU_ALIGN_DOWN(offset, s->cluster_size);
 +    end = QEMU_ALIGN_UP(offset + bytes, s->cluster_size);
      return block_copy(s->bcs, off, end - off, NULL);
  }
@@ -XXX,XX +XXX,XX @@ BlockDriverState *bdrv_backup_top_append(BlockDriverState *source,
          goto fail;
      }
 +    state->cluster_size = cluster_size;
      state->bcs = block_copy_state_new(top->backing, state->target,
                                        cluster_size, write_flags, &local_err);
      if (local_err) {
 diff --git a/block/backup.c b/block/backup.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/backup.c
 +++ b/block/backup.c
@@ -XXX,XX +XXX,XX @@ static void backup_cleanup_sync_bitmap(BackupBlockJob *job, int ret)
      if (ret < 0 && job->bitmap_mode == BITMAP_SYNC_MODE_ALWAYS) {
          /* If we failed and synced, merge in the bits we didn't copy: */
 -        bdrv_dirty_bitmap_merge_internal(bm, job->bcs->copy_bitmap,
 +        bdrv_dirty_bitmap_merge_internal(bm, block_copy_dirty_bitmap(job->bcs),
                                           NULL, true);
      }
  }
@@ -XXX,XX +XXX,XX @@ void backup_do_checkpoint(BlockJob *job, Error **errp)
          return;
      }
--    /* AioContext is released in .clean() */
+-    bdrv_set_dirty_bitmap(backup_job->bcs->copy_bitmap, 0, backup_job->len);
--    state->aio_context = bdrv_get_aio_context(bs);
++    bdrv_set_dirty_bitmap(block_copy_dirty_bitmap(backup_job->bcs), 0,
--    aio_context_acquire(state->aio_context);
++                          backup_job->len);
-+    aio_context = bdrv_get_aio_context(bs);
+ }
-+    aio_context_acquire(aio_context);
-+
+ static BlockErrorAction backup_error_action(BackupBlockJob *job,
-+    /* Paired with .clean() */
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_loop(BackupBlockJob *job)
-     bdrv_drained_begin(bs);
+     BdrvDirtyBitmapIter *bdbi;
-+
+     int ret = 0;
-     state->bs = bs;
+-    bdbi = bdrv_dirty_iter_new(job->bcs->copy_bitmap);
-     state->job = do_drive_backup(backup, common->block_job_txn, &local_err);
++    bdbi = bdrv_dirty_iter_new(block_copy_dirty_bitmap(job->bcs));
-     if (local_err) {
+     while ((offset = bdrv_dirty_iter_next(bdbi)) != -1) {
-         error_propagate(errp, local_err);
+         do {
--        return;
+             if (yield_and_check(job)) {
-+        goto out;
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_loop(BackupBlockJob *job)
-     }
+     return ret;
-+
+ }
-+out:
-+    aio_context_release(aio_context);
+-static void backup_init_copy_bitmap(BackupBlockJob *job)
- }
++static void backup_init_bcs_bitmap(BackupBlockJob *job)
  static void drive_backup_commit(BlkActionState *common)
  {
-     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
+     bool ret;
-+    AioContext *aio_context;
+     uint64_t estimate;
-+
++    BdrvDirtyBitmap *bcs_bitmap = block_copy_dirty_bitmap(job->bcs);
-+    aio_context = bdrv_get_aio_context(state->bs);
-+    aio_context_acquire(aio_context);
+     if (job->sync_mode == MIRROR_SYNC_MODE_BITMAP) {
-+
+-        ret = bdrv_dirty_bitmap_merge_internal(job->bcs->copy_bitmap,
-     assert(state->job);
+-                                               job->sync_bitmap,
-     block_job_start(state->job);
++        ret = bdrv_dirty_bitmap_merge_internal(bcs_bitmap, job->sync_bitmap,
-+
+                                                NULL, true);
-+    aio_context_release(aio_context);
+         assert(ret);
- }
+     } else {
+@@ -XXX,XX +XXX,XX @@ static void backup_init_copy_bitmap(BackupBlockJob *job)
- static void drive_backup_abort(BlkActionState *common)
+              * We can't hog the coroutine to initialize this thoroughly.
-@@ -XXX,XX +XXX,XX @@ static void drive_backup_abort(BlkActionState *common)
+              * Set a flag and resume work when we are able to yield safely.
-     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
+              */
+-            job->bcs->skip_unallocated = true;
-     if (state->job) {
++            block_copy_set_skip_unallocated(job->bcs, true);
-+        AioContext *aio_context;
+         }
-+
+-        bdrv_set_dirty_bitmap(job->bcs->copy_bitmap, 0, job->len);
-+        aio_context = bdrv_get_aio_context(state->bs);
++        bdrv_set_dirty_bitmap(bcs_bitmap, 0, job->len);
-+        aio_context_acquire(aio_context);
+     }
-+
-         block_job_cancel_sync(state->job);
+-    estimate = bdrv_get_dirty_count(job->bcs->copy_bitmap);
-+
++    estimate = bdrv_get_dirty_count(bcs_bitmap);
-+        aio_context_release(aio_context);
+     job_progress_set_remaining(&job->common.job, estimate);
-     }
+ }
- }
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_run(Job *job, Error **errp)
- static void drive_backup_clean(BlkActionState *common)
+     BackupBlockJob *s = container_of(job, BackupBlockJob, common.job);
- {
+     int ret = 0;
-     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
-+    AioContext *aio_context;
+-    backup_init_copy_bitmap(s);
++    backup_init_bcs_bitmap(s);
--    if (state->aio_context) {
--        bdrv_drained_end(state->bs);
+     if (s->sync_mode == MIRROR_SYNC_MODE_TOP) {
--        aio_context_release(state->aio_context);
+         int64_t offset = 0;
-+    if (!state->bs) {
+@@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_run(Job *job, Error **errp)
-+        return;
-     }
+             offset += count;
-+
+         }
-+    aio_context = bdrv_get_aio_context(state->bs);
+-        s->bcs->skip_unallocated = false;
-+    aio_context_acquire(aio_context);
++        block_copy_set_skip_unallocated(s->bcs, false);
-+
+     }
-+    bdrv_drained_end(state->bs);
-+
+     if (s->sync_mode == MIRROR_SYNC_MODE_NONE) {
-+    aio_context_release(aio_context);
+         /*
- }
+-         * All bits are set in copy_bitmap to allow any cluster to be copied.
++         * All bits are set in bcs bitmap to allow any cluster to be copied.
- typedef struct BlockdevBackupState {
+          * This does not actually require them to be copied.
           */
          while (!job_is_cancelled(job)) {
 diff --git a/block/block-copy.c b/block/block-copy.c
 index XXXXXXX..XXXXXXX 100644
 --- a/block/block-copy.c
 +++ b/block/block-copy.c
@@ -XXX,XX +XXX,XX @@
  #define BLOCK_COPY_MAX_BUFFER (1 * MiB)
  #define BLOCK_COPY_MAX_MEM (128 * MiB)
 +typedef struct BlockCopyInFlightReq {
 +    int64_t offset;
 +    int64_t bytes;
 +    QLIST_ENTRY(BlockCopyInFlightReq) list;
 +    CoQueue wait_queue; /* coroutines blocked on this request */
 +} BlockCopyInFlightReq;
 +
 +typedef struct BlockCopyState {
 +    /*
 +     * BdrvChild objects are not owned or managed by block-copy. They are
 +     * provided by block-copy user and user is responsible for appropriate
 +     * permissions on these children.
 +     */
 +    BdrvChild *source;
 +    BdrvChild *target;
 +    BdrvDirtyBitmap *copy_bitmap;
 +    int64_t in_flight_bytes;
 +    int64_t cluster_size;
 +    bool use_copy_range;
 +    int64_t copy_size;
 +    uint64_t len;
 +    QLIST_HEAD(, BlockCopyInFlightReq) inflight_reqs;
 +
 +    BdrvRequestFlags write_flags;
 +
 +    /*
 +     * skip_unallocated:
 +     *
 +     * Used by sync=top jobs, which first scan the source node for unallocated
 +     * areas and clear them in the copy_bitmap.  During this process, the bitmap
 +     * is thus not fully initialized: It may still have bits set for areas that
 +     * are unallocated and should actually not be copied.
 +     *
 +     * This is indicated by skip_unallocated.
 +     *
 +     * In this case, block_copy() will query the source’s allocation status,
 +     * skip unallocated regions, clear them in the copy_bitmap, and invoke
 +     * block_copy_reset_unallocated() every time it does.
 +     */
 +    bool skip_unallocated;
 +
 +    ProgressMeter *progress;
 +    /* progress_bytes_callback: called when some copying progress is done. */
 +    ProgressBytesCallbackFunc progress_bytes_callback;
 +    void *progress_opaque;
 +
 +    SharedResource *mem;
 +} BlockCopyState;
 +
  static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
                                                             int64_t offset,
                                                             int64_t bytes)
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes,
      return ret;
  }
 +
 +BdrvDirtyBitmap *block_copy_dirty_bitmap(BlockCopyState *s)
 +{
 +    return s->copy_bitmap;
 +}
 +
 +void block_copy_set_skip_unallocated(BlockCopyState *s, bool skip)
 +{
 +    s->skip_unallocated = skip;
 +}
 diff --git a/include/block/block-copy.h b/include/block/block-copy.h
 index XXXXXXX..XXXXXXX 100644
 --- a/include/block/block-copy.h
 +++ b/include/block/block-copy.h
@@ -XXX,XX +XXX,XX @@
  #include "block/block.h"
  #include "qemu/co-shared-resource.h"
 -typedef struct BlockCopyInFlightReq {
 -    int64_t offset;
 -    int64_t bytes;
 -    QLIST_ENTRY(BlockCopyInFlightReq) list;
 -    CoQueue wait_queue; /* coroutines blocked on this request */
 -} BlockCopyInFlightReq;
 -
  typedef void (*ProgressBytesCallbackFunc)(int64_t bytes, void *opaque);
 -typedef struct BlockCopyState {
 -    /*
 -     * BdrvChild objects are not owned or managed by block-copy. They are
 -     * provided by block-copy user and user is responsible for appropriate
 -     * permissions on these children.
 -     */
 -    BdrvChild *source;
 -    BdrvChild *target;
 -    BdrvDirtyBitmap *copy_bitmap;
 -    int64_t in_flight_bytes;
 -    int64_t cluster_size;
 -    bool use_copy_range;
 -    int64_t copy_size;
 -    uint64_t len;
 -    QLIST_HEAD(, BlockCopyInFlightReq) inflight_reqs;
 -
 -    BdrvRequestFlags write_flags;
 -
 -    /*
 -     * skip_unallocated:
 -     *
 -     * Used by sync=top jobs, which first scan the source node for unallocated
 -     * areas and clear them in the copy_bitmap.  During this process, the bitmap
 -     * is thus not fully initialized: It may still have bits set for areas that
 -     * are unallocated and should actually not be copied.
 -     *
 -     * This is indicated by skip_unallocated.
 -     *
 -     * In this case, block_copy() will query the source’s allocation status,
 -     * skip unallocated regions, clear them in the copy_bitmap, and invoke
 -     * block_copy_reset_unallocated() every time it does.
 -     */
 -    bool skip_unallocated;
 -
 -    ProgressMeter *progress;
 -    /* progress_bytes_callback: called when some copying progress is done. */
 -    ProgressBytesCallbackFunc progress_bytes_callback;
 -    void *progress_opaque;
 -
 -    SharedResource *mem;
 -} BlockCopyState;
 +typedef struct BlockCopyState BlockCopyState;
  BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
                                       int64_t cluster_size,
@@ -XXX,XX +XXX,XX @@ int64_t block_copy_reset_unallocated(BlockCopyState *s,
  int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes,
                              bool *error_is_read);
 +BdrvDirtyBitmap *block_copy_dirty_bitmap(BlockCopyState *s);
 +void block_copy_set_skip_unallocated(BlockCopyState *s, bool skip);
 +
  #endif /* BLOCK_COPY_H */
 --
-.14.3
+.24.1

-[Qemu-devel] [PULL 13/23] iothread: add iothread_by_id() API
+Deleted patch
-Encapsulate IOThread QOM object lookup so that callers don't need to
-know how and where IOThread objects live.
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
-Reviewed-by: Kevin Wolf <kwolf@redhat.com>
-Reviewed-by: Eric Blake <eblake@redhat.com>
-Message-id: 20171206144550.22295-8-stefanha@redhat.com
-Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
----
- include/sysemu/iothread.h | 1 +
- iothread.c                | 7 +++++++
-files changed, 8 insertions(+)
-diff --git a/include/sysemu/iothread.h b/include/sysemu/iothread.h
-index XXXXXXX..XXXXXXX 100644
---- a/include/sysemu/iothread.h
-+++ b/include/sysemu/iothread.h
-@@ -XXX,XX +XXX,XX @@ typedef struct {
-    OBJECT_CHECK(IOThread, obj, TYPE_IOTHREAD)
- char *iothread_get_id(IOThread *iothread);
-+IOThread *iothread_by_id(const char *id);
- AioContext *iothread_get_aio_context(IOThread *iothread);
- void iothread_stop_all(void);
- GMainContext *iothread_get_g_main_context(IOThread *iothread);
-diff --git a/iothread.c b/iothread.c
-index XXXXXXX..XXXXXXX 100644
---- a/iothread.c
-+++ b/iothread.c
-@@ -XXX,XX +XXX,XX @@ void iothread_destroy(IOThread *iothread)
- {
-     object_unparent(OBJECT(iothread));
- }
-+
-+/* Lookup IOThread by its id.  Only finds user-created objects, not internal
-+ * iothread_create() objects. */
-+IOThread *iothread_by_id(const char *id)
-+{
-+    return IOTHREAD(object_resolve_path_type(id, TYPE_IOTHREAD, NULL));
-+}
---
-.14.3

The following changes since commit 411ad78115ebeb3411cf4b7622784b93dfabe259:

Merge remote-tracking branch 'remotes/stefanberger/tags/pull-tpm-2017-12-15-1' into staging (2017-12-17 15:27:41 +0000)

are available in the Git repository at:

git://github.com/stefanha/qemu.git tags/block-pull-request

for you to fetch changes up to 585426c518958aa768564596091474be786aae51:

qemu-iotests: add 203 savevm with IOThreads test (2017-12-18 13:12:53 +0000)

----------------------------------------------------------------

Mao Zhongyi (4):
  hw/block/nvme: Convert to realize
  hw/block: Fix the return type
  hw/block: Use errp directly rather than local_err
  dev-storage: Fix the unusual function name

Mark Kanda (2):
  virtio-blk: make queue size configurable
  virtio-blk: reject configs with logical block size > physical block
    size

Paolo Bonzini (1):
  block: avoid recursive AioContext acquire in bdrv_inactivate_all()

Stefan Hajnoczi (16):
  coroutine: simplify co_aio_sleep_ns() prototype
  qdev: drop unused #include "sysemu/iothread.h"
  blockdev: hold AioContext for bdrv_unref() in
    external_snapshot_clean()
  block: don't keep AioContext acquired after
    external_snapshot_prepare()
  block: don't keep AioContext acquired after drive_backup_prepare()
  block: don't keep AioContext acquired after blockdev_backup_prepare()
  block: don't keep AioContext acquired after
    internal_snapshot_prepare()
  block: drop unused BlockDirtyBitmapState->aio_context field
  iothread: add iothread_by_id() API
  blockdev: add x-blockdev-set-iothread testing command
  qemu-iotests: add 202 external snapshots IOThread test
  docs: mark nested AioContext locking as a legacy API
  blockdev: add x-blockdev-set-iothread force boolean
  iotests: add VM.add_object()
  iothread: fix iothread_stop() race condition
  qemu-iotests: add 203 savevm with IOThreads test

-- 
2.14.3

The AioContext pointer argument to co_aio_sleep_ns() is only used for
the sleep timer.  It does not affect where the caller coroutine is
resumed.

Due to changes to coroutine and AIO APIs it is now possible to drop the
AioContext pointer argument.  This is safe to do since no caller has
specific requirements for which AioContext the timer must run in.

This patch drops the AioContext pointer argument and renames the
function to simplify the API.

Reported-by: Paolo Bonzini <pbonzini@redhat.com>
Reported-by: Eric Blake <eblake@redhat.com>
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-id: 20171109102652.6360-1-stefanha@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 include/qemu/coroutine.h    | 6 +-----
 block/null.c                | 3 +--
 block/sheepdog.c            | 3 +--
 util/qemu-coroutine-sleep.c | 4 ++--
 4 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/include/qemu/coroutine.h b/include/qemu/coroutine.h
index XXXXXXX..XXXXXXX 100644
--- a/include/qemu/coroutine.h
+++ b/include/qemu/coroutine.h
@@ -XXX,XX +XXX,XX @@ void qemu_co_rwlock_unlock(CoRwlock *lock);
 
 /**
  * Yield the coroutine for a given duration
- *
- * Behaves similarly to co_sleep_ns(), but the sleeping coroutine will be
- * resumed when using aio_poll().
  */
-void coroutine_fn co_aio_sleep_ns(AioContext *ctx, QEMUClockType type,
-                                  int64_t ns);
+void coroutine_fn qemu_co_sleep_ns(QEMUClockType type, int64_t ns);
 
 /**
  * Yield until a file descriptor becomes readable
diff --git a/block/null.c b/block/null.c
index XXXXXXX..XXXXXXX 100644
--- a/block/null.c
+++ b/block/null.c
@@ -XXX,XX +XXX,XX @@ static coroutine_fn int null_co_common(BlockDriverState *bs)
     BDRVNullState *s = bs->opaque;
 
     if (s->latency_ns) {
-        co_aio_sleep_ns(bdrv_get_aio_context(bs), QEMU_CLOCK_REALTIME,
-                        s->latency_ns);
+        qemu_co_sleep_ns(QEMU_CLOCK_REALTIME, s->latency_ns);
     }
     return 0;
 }
diff --git a/block/sheepdog.c b/block/sheepdog.c
index XXXXXXX..XXXXXXX 100644
--- a/block/sheepdog.c
+++ b/block/sheepdog.c
@@ -XXX,XX +XXX,XX @@ static coroutine_fn void reconnect_to_sdog(void *opaque)
         if (s->fd < 0) {
             DPRINTF("Wait for connection to be established\n");
             error_report_err(local_err);
-            co_aio_sleep_ns(bdrv_get_aio_context(s->bs), QEMU_CLOCK_REALTIME,
-                            1000000000ULL);
+            qemu_co_sleep_ns(QEMU_CLOCK_REALTIME, 1000000000ULL);
         }
     };
 
diff --git a/util/qemu-coroutine-sleep.c b/util/qemu-coroutine-sleep.c
index XXXXXXX..XXXXXXX 100644
--- a/util/qemu-coroutine-sleep.c
+++ b/util/qemu-coroutine-sleep.c
@@ -XXX,XX +XXX,XX @@ static void co_sleep_cb(void *opaque)
     aio_co_wake(sleep_cb->co);
 }
 
-void coroutine_fn co_aio_sleep_ns(AioContext *ctx, QEMUClockType type,
-                                  int64_t ns)
+void coroutine_fn qemu_co_sleep_ns(QEMUClockType type, int64_t ns)
 {
+    AioContext *ctx = qemu_get_current_aio_context();
     CoSleepCB sleep_cb = {
         .co = qemu_coroutine_self(),
     };
-- 
2.14.3

From: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>

Convert nvme_init() to realize and rename it to nvme_realize().

Cc: John Snow <jsnow@redhat.com>
Cc: Keith Busch <keith.busch@intel.com>
Cc: Kevin Wolf <kwolf@redhat.com>
Cc: Max Reitz <mreitz@redhat.com>
Cc: Markus Armbruster <armbru@redhat.com>

Signed-off-by: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>
Message-id: 2882e72d795e04cbe2120f569d551aef2467ac60.1511317952.git.maozy.fnst@cn.fujitsu.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 hw/block/nvme.c | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -XXX,XX +XXX,XX @@ static const MemoryRegionOps nvme_cmb_ops = {
     },
 };
 
-static int nvme_init(PCIDevice *pci_dev)
+static void nvme_realize(PCIDevice *pci_dev, Error **errp)
 {
     NvmeCtrl *n = NVME(pci_dev);
     NvmeIdCtrl *id = &n->id_ctrl;
@@ -XXX,XX +XXX,XX @@ static int nvme_init(PCIDevice *pci_dev)
     Error *local_err = NULL;
 
     if (!n->conf.blk) {
-        return -1;
+        error_setg(errp, "drive property not set");
+        return;
     }
 
     bs_size = blk_getlength(n->conf.blk);
     if (bs_size < 0) {
-        return -1;
+        error_setg(errp, "could not get backing file size");
+        return;
     }
 
     blkconf_serial(&n->conf, &n->serial);
     if (!n->serial) {
-        return -1;
+        error_setg(errp, "serial property not set");
+        return;
     }
     blkconf_blocksizes(&n->conf);
     blkconf_apply_backend_options(&n->conf, blk_is_read_only(n->conf.blk),
                                   false, &local_err);
     if (local_err) {
-        error_report_err(local_err);
-        return -1;
+        error_propagate(errp, local_err);
+        return;
     }
 
     pci_conf = pci_dev->config;
@@ -XXX,XX +XXX,XX @@ static int nvme_init(PCIDevice *pci_dev)
             cpu_to_le64(n->ns_size >>
                 id_ns->lbaf[NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas)].ds);
     }
-    return 0;
 }
 
 static void nvme_exit(PCIDevice *pci_dev)
@@ -XXX,XX +XXX,XX @@ static void nvme_class_init(ObjectClass *oc, void *data)
     DeviceClass *dc = DEVICE_CLASS(oc);
     PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
 
-    pc->init = nvme_init;
+    pc->realize = nvme_realize;
     pc->exit = nvme_exit;
     pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
     pc->vendor_id = PCI_VENDOR_ID_INTEL;
-- 
2.14.3

From: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>

When the function no success value to transmit, it usually make the
function return void. It has turned out not to be a success, because
it means that the extra local_err variable and error_propagate() will
be needed. It leads to cumbersome code, therefore, transmit success/
failure in the return value is worth.

So fix the return type of blkconf_apply_backend_options(),
blkconf_geometry() and virtio_blk_data_plane_create() to avoid it.

Cc: John Snow <jsnow@redhat.com>
Cc: Kevin Wolf <kwolf@redhat.com>
Cc: Max Reitz <mreitz@redhat.com>
Cc: Stefan Hajnoczi <stefanha@redhat.com>

Signed-off-by: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
Message-id: ac0edc1fc70c4457e5cec94405eb7d1f89f9c2c1.1511317952.git.maozy.fnst@cn.fujitsu.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 hw/block/dataplane/virtio-blk.h |  2 +-
 include/hw/block/block.h        |  4 ++--
 hw/block/block.c                | 15 +++++++++------
 hw/block/dataplane/virtio-blk.c | 12 +++++++-----
 4 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/hw/block/dataplane/virtio-blk.h b/hw/block/dataplane/virtio-blk.h
index XXXXXXX..XXXXXXX 100644
--- a/hw/block/dataplane/virtio-blk.h
+++ b/hw/block/dataplane/virtio-blk.h
@@ -XXX,XX +XXX,XX @@
 
 typedef struct VirtIOBlockDataPlane VirtIOBlockDataPlane;
 
-void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
+bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
                                   VirtIOBlockDataPlane **dataplane,
                                   Error **errp);
 void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s);
diff --git a/include/hw/block/block.h b/include/hw/block/block.h
index XXXXXXX..XXXXXXX 100644
--- a/include/hw/block/block.h
+++ b/include/hw/block/block.h
@@ -XXX,XX +XXX,XX @@ static inline unsigned int get_physical_block_exp(BlockConf *conf)
 /* Configuration helpers */
 
 void blkconf_serial(BlockConf *conf, char **serial);
-void blkconf_geometry(BlockConf *conf, int *trans,
+bool blkconf_geometry(BlockConf *conf, int *trans,
                       unsigned cyls_max, unsigned heads_max, unsigned secs_max,
                       Error **errp);
 void blkconf_blocksizes(BlockConf *conf);
-void blkconf_apply_backend_options(BlockConf *conf, bool readonly,
+bool blkconf_apply_backend_options(BlockConf *conf, bool readonly,
                                    bool resizable, Error **errp);
 
 /* Hard disk geometry */
diff --git a/hw/block/block.c b/hw/block/block.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/block/block.c
+++ b/hw/block/block.c
@@ -XXX,XX +XXX,XX @@ void blkconf_blocksizes(BlockConf *conf)
     }
 }
 
-void blkconf_apply_backend_options(BlockConf *conf, bool readonly,
+bool blkconf_apply_backend_options(BlockConf *conf, bool readonly,
                                    bool resizable, Error **errp)
 {
     BlockBackend *blk = conf->blk;
@@ -XXX,XX +XXX,XX @@ void blkconf_apply_backend_options(BlockConf *conf, bool readonly,
 
     ret = blk_set_perm(blk, perm, shared_perm, errp);
     if (ret < 0) {
-        return;
+        return false;
     }
 
     switch (conf->wce) {
@@ -XXX,XX +XXX,XX @@ void blkconf_apply_backend_options(BlockConf *conf, bool readonly,
 
     blk_set_enable_write_cache(blk, wce);
     blk_set_on_error(blk, rerror, werror);
+
+    return true;
 }
 
-void blkconf_geometry(BlockConf *conf, int *ptrans,
+bool blkconf_geometry(BlockConf *conf, int *ptrans,
                       unsigned cyls_max, unsigned heads_max, unsigned secs_max,
                       Error **errp)
 {
@@ -XXX,XX +XXX,XX @@ void blkconf_geometry(BlockConf *conf, int *ptrans,
     if (conf->cyls || conf->heads || conf->secs) {
         if (conf->cyls < 1 || conf->cyls > cyls_max) {
             error_setg(errp, "cyls must be between 1 and %u", cyls_max);
-            return;
+            return false;
         }
         if (conf->heads < 1 || conf->heads > heads_max) {
             error_setg(errp, "heads must be between 1 and %u", heads_max);
-            return;
+            return false;
         }
         if (conf->secs < 1 || conf->secs > secs_max) {
             error_setg(errp, "secs must be between 1 and %u", secs_max);
-            return;
+            return false;
         }
     }
+    return true;
 }
diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/block/dataplane/virtio-blk.c
+++ b/hw/block/dataplane/virtio-blk.c
@@ -XXX,XX +XXX,XX @@ static void notify_guest_bh(void *opaque)
 }
 
 /* Context: QEMU global mutex held */
-void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
+bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
                                   VirtIOBlockDataPlane **dataplane,
                                   Error **errp)
 {
@@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
             error_setg(errp,
                        "device is incompatible with iothread "
                        "(transport does not support notifiers)");
-            return;
+            return false;
         }
         if (!virtio_device_ioeventfd_enabled(vdev)) {
             error_setg(errp, "ioeventfd is required for iothread");
-            return;
+            return false;
         }
 
         /* If dataplane is (re-)enabled while the guest is running there could
@@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
          */
         if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) {
             error_prepend(errp, "cannot start virtio-blk dataplane: ");
-            return;
+            return false;
         }
     }
     /* Don't try if transport does not support notifiers. */
     if (!virtio_device_ioeventfd_enabled(vdev)) {
-        return;
+        return false;
     }
 
     s = g_new0(VirtIOBlockDataPlane, 1);
@@ -XXX,XX +XXX,XX @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
     s->batch_notify_vqs = bitmap_new(conf->num_queues);
 
     *dataplane = s;
+
+    return true;
 }
 
 /* Context: QEMU global mutex held */
-- 
2.14.3

From: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>

Cc: John Snow <jsnow@redhat.com>
Cc: Kevin Wolf <kwolf@redhat.com>
Cc: Max Reitz <mreitz@redhat.com>
Cc: Keith Busch <keith.busch@intel.com>
Cc: Stefan Hajnoczi <stefanha@redhat.com>
Cc: "Michael S. Tsirkin" <mst@redhat.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Gerd Hoffmann <kraxel@redhat.com>
Cc: Markus Armbruster <armbru@redhat.com>

Signed-off-by: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
Message-id: e77848d3735ba590f23ffbf8094379c646c33d79.1511317952.git.maozy.fnst@cn.fujitsu.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 hw/block/fdc.c        | 17 ++++++-----------
 hw/block/nvme.c       |  7 ++-----
 hw/block/virtio-blk.c | 18 ++++++------------
 hw/ide/qdev.c         | 12 ++++--------
 hw/scsi/scsi-disk.c   | 13 ++++---------
 hw/usb/dev-storage.c  |  9 +++------
 6 files changed, 25 insertions(+), 51 deletions(-)

diff --git a/hw/block/fdc.c b/hw/block/fdc.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/block/fdc.c
+++ b/hw/block/fdc.c
@@ -XXX,XX +XXX,XX @@ static void fd_revalidate(FDrive *drv)
 static void fd_change_cb(void *opaque, bool load, Error **errp)
 {
     FDrive *drive = opaque;
-    Error *local_err = NULL;
 
     if (!load) {
         blk_set_perm(drive->blk, 0, BLK_PERM_ALL, &error_abort);
     } else {
-        blkconf_apply_backend_options(drive->conf,
-                                      blk_is_read_only(drive->blk), false,
-                                      &local_err);
-        if (local_err) {
-            error_propagate(errp, local_err);
+        if (!blkconf_apply_backend_options(drive->conf,
+                                           blk_is_read_only(drive->blk), false,
+                                           errp)) {
             return;
         }
     }
@@ -XXX,XX +XXX,XX @@ static void floppy_drive_realize(DeviceState *qdev, Error **errp)
     FloppyDrive *dev = FLOPPY_DRIVE(qdev);
     FloppyBus *bus = FLOPPY_BUS(qdev->parent_bus);
     FDrive *drive;
-    Error *local_err = NULL;
     int ret;
 
     if (dev->unit == -1) {
@@ -XXX,XX +XXX,XX @@ static void floppy_drive_realize(DeviceState *qdev, Error **errp)
     dev->conf.rerror = BLOCKDEV_ON_ERROR_AUTO;
     dev->conf.werror = BLOCKDEV_ON_ERROR_AUTO;
 
-    blkconf_apply_backend_options(&dev->conf, blk_is_read_only(dev->conf.blk),
-                                  false, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
+    if (!blkconf_apply_backend_options(&dev->conf,
+                                       blk_is_read_only(dev->conf.blk),
+                                       false, errp)) {
         return;
     }
 
diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -XXX,XX +XXX,XX @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)
     int i;
     int64_t bs_size;
     uint8_t *pci_conf;
-    Error *local_err = NULL;
 
     if (!n->conf.blk) {
         error_setg(errp, "drive property not set");
@@ -XXX,XX +XXX,XX @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)
         return;
     }
     blkconf_blocksizes(&n->conf);
-    blkconf_apply_backend_options(&n->conf, blk_is_read_only(n->conf.blk),
-                                  false, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
+    if (!blkconf_apply_backend_options(&n->conf, blk_is_read_only(n->conf.blk),
+                                       false, errp)) {
         return;
     }
 
diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/block/virtio-blk.c
+++ b/hw/block/virtio-blk.c
@@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VirtIOBlock *s = VIRTIO_BLK(dev);
     VirtIOBlkConf *conf = &s->conf;
-    Error *err = NULL;
     unsigned i;
 
     if (!conf->conf.blk) {
@@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
     }
 
     blkconf_serial(&conf->conf, &conf->serial);
-    blkconf_apply_backend_options(&conf->conf,
-                                  blk_is_read_only(conf->conf.blk), true,
-                                  &err);
-    if (err) {
-        error_propagate(errp, err);
+    if (!blkconf_apply_backend_options(&conf->conf,
+                                       blk_is_read_only(conf->conf.blk), true,
+                                       errp)) {
         return;
     }
     s->original_wce = blk_enable_write_cache(conf->conf.blk);
-    blkconf_geometry(&conf->conf, NULL, 65535, 255, 255, &err);
-    if (err) {
-        error_propagate(errp, err);
+    if (!blkconf_geometry(&conf->conf, NULL, 65535, 255, 255, errp)) {
         return;
     }
+
     blkconf_blocksizes(&conf->conf);
 
     virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK,
@@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
     for (i = 0; i < conf->num_queues; i++) {
         virtio_add_queue(vdev, 128, virtio_blk_handle_output);
     }
-    virtio_blk_data_plane_create(vdev, conf, &s->dataplane, &err);
-    if (err != NULL) {
-        error_propagate(errp, err);
+    if (!virtio_blk_data_plane_create(vdev, conf, &s->dataplane, errp)) {
         virtio_cleanup(vdev);
         return;
     }
diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/ide/qdev.c
+++ b/hw/ide/qdev.c
@@ -XXX,XX +XXX,XX @@ static void ide_dev_initfn(IDEDevice *dev, IDEDriveKind kind, Error **errp)
 {
     IDEBus *bus = DO_UPCAST(IDEBus, qbus, dev->qdev.parent_bus);
     IDEState *s = bus->ifs + dev->unit;
-    Error *err = NULL;
     int ret;
 
     if (!dev->conf.blk) {
@@ -XXX,XX +XXX,XX @@ static void ide_dev_initfn(IDEDevice *dev, IDEDriveKind kind, Error **errp)
 
     blkconf_serial(&dev->conf, &dev->serial);
     if (kind != IDE_CD) {
-        blkconf_geometry(&dev->conf, &dev->chs_trans, 65535, 16, 255, &err);
-        if (err) {
-            error_propagate(errp, err);
+        if (!blkconf_geometry(&dev->conf, &dev->chs_trans, 65535, 16, 255,
+                              errp)) {
             return;
         }
     }
-    blkconf_apply_backend_options(&dev->conf, kind == IDE_CD, kind != IDE_CD,
-                                  &err);
-    if (err) {
-        error_propagate(errp, err);
+    if (!blkconf_apply_backend_options(&dev->conf, kind == IDE_CD,
+                                       kind != IDE_CD, errp)) {
         return;
     }
 
diff --git a/hw/scsi/scsi-disk.c b/hw/scsi/scsi-disk.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/scsi/scsi-disk.c
+++ b/hw/scsi/scsi-disk.c
@@ -XXX,XX +XXX,XX @@ static void scsi_disk_unit_attention_reported(SCSIDevice *dev)
 static void scsi_realize(SCSIDevice *dev, Error **errp)
 {
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, dev);
-    Error *err = NULL;
 
     if (!s->qdev.conf.blk) {
         error_setg(errp, "drive property not set");
@@ -XXX,XX +XXX,XX @@ static void scsi_realize(SCSIDevice *dev, Error **errp)
     }
 
     if (dev->type == TYPE_DISK) {
-        blkconf_geometry(&dev->conf, NULL, 65535, 255, 255, &err);
-        if (err) {
-            error_propagate(errp, err);
+        if (!blkconf_geometry(&dev->conf, NULL, 65535, 255, 255, errp)) {
             return;
         }
     }
-    blkconf_apply_backend_options(&dev->conf,
-                                  blk_is_read_only(s->qdev.conf.blk),
-                                  dev->type == TYPE_DISK, &err);
-    if (err) {
-        error_propagate(errp, err);
+    if (!blkconf_apply_backend_options(&dev->conf,
+                                       blk_is_read_only(s->qdev.conf.blk),
+                                       dev->type == TYPE_DISK, errp)) {
         return;
     }
 
diff --git a/hw/usb/dev-storage.c b/hw/usb/dev-storage.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/usb/dev-storage.c
+++ b/hw/usb/dev-storage.c
@@ -XXX,XX +XXX,XX @@ static void usb_msd_realize_storage(USBDevice *dev, Error **errp)
     MSDState *s = USB_STORAGE_DEV(dev);
     BlockBackend *blk = s->conf.blk;
     SCSIDevice *scsi_dev;
-    Error *err = NULL;
 
     if (!blk) {
         error_setg(errp, "drive property not set");
@@ -XXX,XX +XXX,XX @@ static void usb_msd_realize_storage(USBDevice *dev, Error **errp)
 
     blkconf_serial(&s->conf, &dev->serial);
     blkconf_blocksizes(&s->conf);
-    blkconf_apply_backend_options(&s->conf, blk_is_read_only(blk), true, &err);
-    if (err) {
-        error_propagate(errp, err);
+    if (!blkconf_apply_backend_options(&s->conf, blk_is_read_only(blk), true,
+                                       errp)) {
         return;
     }
 
@@ -XXX,XX +XXX,XX @@ static void usb_msd_realize_storage(USBDevice *dev, Error **errp)
                  &usb_msd_scsi_info_storage, NULL);
     scsi_dev = scsi_bus_legacy_add_drive(&s->bus, blk, 0, !!s->removable,
                                          s->conf.bootindex, dev->serial,
-                                         &err);
+                                         errp);
     blk_unref(blk);
     if (!scsi_dev) {
-        error_propagate(errp, err);
         return;
     }
     usb_msd_handle_reset(dev);
-- 
2.14.3

From: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>

The function name of usb_msd_{realize,unrealize}_*,
usb_msd_class_initfn_* are unusual. Rename it to
usb_msd_*_{realize,unrealize}, usb_msd_class_*_initfn.

Cc: Gerd Hoffmann <kraxel@redhat.com>

Signed-off-by: Mao Zhongyi <maozy.fnst@cn.fujitsu.com>
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Message-id: 11e6003433abce35f3f4970e1acc71ee92dbcf51.1511317952.git.maozy.fnst@cn.fujitsu.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 hw/usb/dev-storage.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/hw/usb/dev-storage.c b/hw/usb/dev-storage.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/usb/dev-storage.c
+++ b/hw/usb/dev-storage.c
@@ -XXX,XX +XXX,XX @@ static void usb_msd_unrealize_storage(USBDevice *dev, Error **errp)
     object_unref(OBJECT(&s->bus));
 }
 
-static void usb_msd_realize_storage(USBDevice *dev, Error **errp)
+static void usb_msd_storage_realize(USBDevice *dev, Error **errp)
 {
     MSDState *s = USB_STORAGE_DEV(dev);
     BlockBackend *blk = s->conf.blk;
@@ -XXX,XX +XXX,XX @@ static void usb_msd_realize_storage(USBDevice *dev, Error **errp)
     s->scsi_dev = scsi_dev;
 }
 
-static void usb_msd_unrealize_bot(USBDevice *dev, Error **errp)
+static void usb_msd_bot_unrealize(USBDevice *dev, Error **errp)
 {
     MSDState *s = USB_STORAGE_DEV(dev);
 
     object_unref(OBJECT(&s->bus));
 }
 
-static void usb_msd_realize_bot(USBDevice *dev, Error **errp)
+static void usb_msd_bot_realize(USBDevice *dev, Error **errp)
 {
     MSDState *s = USB_STORAGE_DEV(dev);
     DeviceState *d = DEVICE(dev);
@@ -XXX,XX +XXX,XX @@ static void usb_msd_class_initfn_common(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_usb_msd;
 }
 
-static void usb_msd_class_initfn_storage(ObjectClass *klass, void *data)
+static void usb_msd_class_storage_initfn(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    uc->realize = usb_msd_realize_storage;
+    uc->realize = usb_msd_storage_realize;
     uc->unrealize = usb_msd_unrealize_storage;
     dc->props = msd_properties;
 }
@@ -XXX,XX +XXX,XX @@ static void usb_msd_instance_init(Object *obj)
     object_property_set_int(obj, -1, "bootindex", NULL);
 }
 
-static void usb_msd_class_initfn_bot(ObjectClass *klass, void *data)
+static void usb_msd_class_bot_initfn(ObjectClass *klass, void *data)
 {
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    uc->realize = usb_msd_realize_bot;
-    uc->unrealize = usb_msd_unrealize_bot;
+    uc->realize = usb_msd_bot_realize;
+    uc->unrealize = usb_msd_bot_unrealize;
     uc->attached_settable = true;
 }
 
 static const TypeInfo msd_info = {
     .name          = "usb-storage",
     .parent        = TYPE_USB_STORAGE,
-    .class_init    = usb_msd_class_initfn_storage,
+    .class_init    = usb_msd_class_storage_initfn,
     .instance_init = usb_msd_instance_init,
 };
 
 static const TypeInfo bot_info = {
     .name          = "usb-bot",
     .parent        = TYPE_USB_STORAGE,
-    .class_init    = usb_msd_class_initfn_bot,
+    .class_init    = usb_msd_class_bot_initfn,
 };
 
 static void usb_msd_register_types(void)
-- 
2.14.3

bdrv_unref() requires the AioContext lock because bdrv_flush() uses
BDRV_POLL_WHILE(), which assumes the AioContext is currently held.  If
BDRV_POLL_WHILE() runs without AioContext held the
pthread_mutex_unlock() call in aio_context_release() fails.

This patch moves bdrv_unref() into the AioContext locked region to solve
the following pthread_mutex_unlock() failure:

#0  0x00007f566181969b in raise () at /lib64/libc.so.6
  #1  0x00007f566181b3b1 in abort () at /lib64/libc.so.6
  #2  0x00005592cd590458 in error_exit (err=<optimized out>, msg=msg@entry=0x5592cdaf6d60 <__func__.23977> "qemu_mutex_unlock") at util/qemu-thread-posix.c:36
  #3  0x00005592cd96e738 in qemu_mutex_unlock (mutex=mutex@entry=0x5592ce9505e0) at util/qemu-thread-posix.c:96
  #4  0x00005592cd969b69 in aio_context_release (ctx=ctx@entry=0x5592ce950580) at util/async.c:507
  #5  0x00005592cd8ead78 in bdrv_flush (bs=bs@entry=0x5592cfa87210) at block/io.c:2478
  #6  0x00005592cd89df30 in bdrv_close (bs=0x5592cfa87210) at block.c:3207
  #7  0x00005592cd89df30 in bdrv_delete (bs=0x5592cfa87210) at block.c:3395
  #8  0x00005592cd89df30 in bdrv_unref (bs=0x5592cfa87210) at block.c:4418
  #9  0x00005592cd6b7f86 in qmp_transaction (dev_list=<optimized out>, has_props=<optimized out>, props=<optimized out>, errp=errp@entry=0x7ffe4a1fc9d8) at blockdev.c:2308

diff --git a/blockdev.c b/blockdev.c
index XXXXXXX..XXXXXXX 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_clean(BlkActionState *common)
                              DO_UPCAST(ExternalSnapshotState, common, common);
     if (state->aio_context) {
         bdrv_drained_end(state->old_bs);
-        aio_context_release(state->aio_context);
         bdrv_unref(state->new_bs);
+        aio_context_release(state->aio_context);
     }
 }
 
-- 
2.14.3

It is not necessary to hold AioContext across transactions anymore since
bdrv_drained_begin/end() is used to keep the nodes quiesced.  In fact,
using the AioContext lock for this purpose was always buggy.

This patch reduces the scope of AioContext locked regions.  This is not
just a cleanup but also fixes hangs that occur in BDRV_POLL_WHILE()
because it is unware of recursive locking and does not release the
AioContext the necessary number of times to allow progress to be made.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-id: 20171206144550.22295-3-stefanha@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 blockdev.c | 71 ++++++++++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 48 insertions(+), 23 deletions(-)

diff --git a/blockdev.c b/blockdev.c
index XXXXXXX..XXXXXXX 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -XXX,XX +XXX,XX @@ typedef struct ExternalSnapshotState {
     BlkActionState common;
     BlockDriverState *old_bs;
     BlockDriverState *new_bs;
-    AioContext *aio_context;
     bool overlay_appended;
 } ExternalSnapshotState;
 
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,
     ExternalSnapshotState *state =
                              DO_UPCAST(ExternalSnapshotState, common, common);
     TransactionAction *action = common->action;
+    AioContext *aio_context;
 
     /* 'blockdev-snapshot' and 'blockdev-snapshot-sync' have similar
      * purpose but a different set of parameters */
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,
         return;
     }
 
-    /* Acquire AioContext now so any threads operating on old_bs stop */
-    state->aio_context = bdrv_get_aio_context(state->old_bs);
-    aio_context_acquire(state->aio_context);
+    aio_context = bdrv_get_aio_context(state->old_bs);
+    aio_context_acquire(aio_context);
+
+    /* Paired with .clean() */
     bdrv_drained_begin(state->old_bs);
 
     if (!bdrv_is_inserted(state->old_bs)) {
         error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
-        return;
+        goto out;
     }
 
     if (bdrv_op_is_blocked(state->old_bs,
                            BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT, errp)) {
-        return;
+        goto out;
     }
 
     if (!bdrv_is_read_only(state->old_bs)) {
         if (bdrv_flush(state->old_bs)) {
             error_setg(errp, QERR_IO_ERROR);
-            return;
+            goto out;
         }
     }
 
     if (!bdrv_is_first_non_filter(state->old_bs)) {
         error_setg(errp, QERR_FEATURE_DISABLED, "snapshot");
-        return;
+        goto out;
     }
 
     if (action->type == TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC) {
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,
 
         if (node_name && !snapshot_node_name) {
             error_setg(errp, "New snapshot node name missing");
-            return;
+            goto out;
         }
 
         if (snapshot_node_name &&
             bdrv_lookup_bs(snapshot_node_name, snapshot_node_name, NULL)) {
             error_setg(errp, "New snapshot node name already in use");
-            return;
+            goto out;
         }
 
         flags = state->old_bs->open_flags;
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,
             int64_t size = bdrv_getlength(state->old_bs);
             if (size < 0) {
                 error_setg_errno(errp, -size, "bdrv_getlength failed");
-                return;
+                goto out;
             }
             bdrv_img_create(new_image_file, format,
                             state->old_bs->filename,
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,
                             NULL, size, flags, false, &local_err);
             if (local_err) {
                 error_propagate(errp, local_err);
-                return;
+                goto out;
             }
         }
 
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,
                               errp);
     /* We will manually add the backing_hd field to the bs later */
     if (!state->new_bs) {
-        return;
+        goto out;
     }
 
     if (bdrv_has_blk(state->new_bs)) {
         error_setg(errp, "The snapshot is already in use");
-        return;
+        goto out;
     }
 
     if (bdrv_op_is_blocked(state->new_bs, BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT,
                            errp)) {
-        return;
+        goto out;
     }
 
     if (state->new_bs->backing != NULL) {
         error_setg(errp, "The snapshot already has a backing image");
-        return;
+        goto out;
     }
 
     if (!state->new_bs->drv->supports_backing) {
         error_setg(errp, "The snapshot does not support backing images");
-        return;
+        goto out;
     }
 
-    bdrv_set_aio_context(state->new_bs, state->aio_context);
+    bdrv_set_aio_context(state->new_bs, aio_context);
 
     /* This removes our old bs and adds the new bs. This is an operation that
      * can fail, so we need to do it in .prepare; undoing it for abort is
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,
     bdrv_append(state->new_bs, state->old_bs, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
-        return;
+        goto out;
     }
     state->overlay_appended = true;
+
+out:
+    aio_context_release(aio_context);
 }
 
 static void external_snapshot_commit(BlkActionState *common)
 {
     ExternalSnapshotState *state =
                              DO_UPCAST(ExternalSnapshotState, common, common);
+    AioContext *aio_context;
+
+    aio_context = bdrv_get_aio_context(state->old_bs);
+    aio_context_acquire(aio_context);
 
     /* We don't need (or want) to use the transactional
      * bdrv_reopen_multiple() across all the entries at once, because we
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_commit(BlkActionState *common)
         bdrv_reopen(state->old_bs, state->old_bs->open_flags & ~BDRV_O_RDWR,
                     NULL);
     }
+
+    aio_context_release(aio_context);
 }
 
 static void external_snapshot_abort(BlkActionState *common)
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_abort(BlkActionState *common)
                              DO_UPCAST(ExternalSnapshotState, common, common);
     if (state->new_bs) {
         if (state->overlay_appended) {
+            AioContext *aio_context;
+
+            aio_context = bdrv_get_aio_context(state->old_bs);
+            aio_context_acquire(aio_context);
+
             bdrv_ref(state->old_bs);   /* we can't let bdrv_set_backind_hd()
                                           close state->old_bs; we need it */
             bdrv_set_backing_hd(state->new_bs, NULL, &error_abort);
             bdrv_replace_node(state->new_bs, state->old_bs, &error_abort);
             bdrv_unref(state->old_bs); /* bdrv_replace_node() ref'ed old_bs */
+
+            aio_context_release(aio_context);
         }
     }
 }
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_clean(BlkActionState *common)
 {
     ExternalSnapshotState *state =
                              DO_UPCAST(ExternalSnapshotState, common, common);
-    if (state->aio_context) {
-        bdrv_drained_end(state->old_bs);
-        bdrv_unref(state->new_bs);
-        aio_context_release(state->aio_context);
+    AioContext *aio_context;
+
+    if (!state->old_bs) {
+        return;
     }
+
+    aio_context = bdrv_get_aio_context(state->old_bs);
+    aio_context_acquire(aio_context);
+
+    bdrv_drained_end(state->old_bs);
+    bdrv_unref(state->new_bs);
+
+    aio_context_release(aio_context);
 }
 
 typedef struct DriveBackupState {
-- 
2.14.3

diff --git a/blockdev.c b/blockdev.c
index XXXXXXX..XXXXXXX 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -XXX,XX +XXX,XX @@ static void external_snapshot_clean(BlkActionState *common)
 typedef struct DriveBackupState {
     BlkActionState common;
     BlockDriverState *bs;
-    AioContext *aio_context;
     BlockJob *job;
 } DriveBackupState;
 
@@ -XXX,XX +XXX,XX @@ static void drive_backup_prepare(BlkActionState *common, Error **errp)
     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
     BlockDriverState *bs;
     DriveBackup *backup;
+    AioContext *aio_context;
     Error *local_err = NULL;
 
     assert(common->action->type == TRANSACTION_ACTION_KIND_DRIVE_BACKUP);
@@ -XXX,XX +XXX,XX @@ static void drive_backup_prepare(BlkActionState *common, Error **errp)
         return;
     }
 
-    /* AioContext is released in .clean() */
-    state->aio_context = bdrv_get_aio_context(bs);
-    aio_context_acquire(state->aio_context);
+    aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(aio_context);
+
+    /* Paired with .clean() */
     bdrv_drained_begin(bs);
+
     state->bs = bs;
 
     state->job = do_drive_backup(backup, common->block_job_txn, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
-        return;
+        goto out;
     }
+
+out:
+    aio_context_release(aio_context);
 }
 
 static void drive_backup_commit(BlkActionState *common)
 {
     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
+    AioContext *aio_context;
+
+    aio_context = bdrv_get_aio_context(state->bs);
+    aio_context_acquire(aio_context);
+
     assert(state->job);
     block_job_start(state->job);
+
+    aio_context_release(aio_context);
 }
 
 static void drive_backup_abort(BlkActionState *common)
@@ -XXX,XX +XXX,XX @@ static void drive_backup_abort(BlkActionState *common)
     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
 
     if (state->job) {
+        AioContext *aio_context;
+
+        aio_context = bdrv_get_aio_context(state->bs);
+        aio_context_acquire(aio_context);
+
         block_job_cancel_sync(state->job);
+
+        aio_context_release(aio_context);
     }
 }
 
 static void drive_backup_clean(BlkActionState *common)
 {
     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
+    AioContext *aio_context;
 
-    if (state->aio_context) {
-        bdrv_drained_end(state->bs);
-        aio_context_release(state->aio_context);
+    if (!state->bs) {
+        return;
     }
+
+    aio_context = bdrv_get_aio_context(state->bs);
+    aio_context_acquire(aio_context);
+
+    bdrv_drained_end(state->bs);
+
+    aio_context_release(aio_context);
 }
 
 typedef struct BlockdevBackupState {
-- 
2.14.3

diff --git a/blockdev.c b/blockdev.c
index XXXXXXX..XXXXXXX 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -XXX,XX +XXX,XX @@ typedef struct BlockdevBackupState {
     BlkActionState common;
     BlockDriverState *bs;
     BlockJob *job;
-    AioContext *aio_context;
 } BlockdevBackupState;
 
 static BlockJob *do_blockdev_backup(BlockdevBackup *backup, BlockJobTxn *txn,
@@ -XXX,XX +XXX,XX @@ static void blockdev_backup_prepare(BlkActionState *common, Error **errp)
     BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
     BlockdevBackup *backup;
     BlockDriverState *bs, *target;
+    AioContext *aio_context;
     Error *local_err = NULL;
 
     assert(common->action->type == TRANSACTION_ACTION_KIND_BLOCKDEV_BACKUP);
@@ -XXX,XX +XXX,XX @@ static void blockdev_backup_prepare(BlkActionState *common, Error **errp)
         return;
     }
 
-    /* AioContext is released in .clean() */
-    state->aio_context = bdrv_get_aio_context(bs);
-    if (state->aio_context != bdrv_get_aio_context(target)) {
-        state->aio_context = NULL;
+    aio_context = bdrv_get_aio_context(bs);
+    if (aio_context != bdrv_get_aio_context(target)) {
         error_setg(errp, "Backup between two IO threads is not implemented");
         return;
     }
-    aio_context_acquire(state->aio_context);
+    aio_context_acquire(aio_context);
     state->bs = bs;
+
+    /* Paired with .clean() */
     bdrv_drained_begin(state->bs);
 
     state->job = do_blockdev_backup(backup, common->block_job_txn, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
-        return;
+        goto out;
     }
+
+out:
+    aio_context_release(aio_context);
 }
 
 static void blockdev_backup_commit(BlkActionState *common)
 {
     BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
+    AioContext *aio_context;
+
+    aio_context = bdrv_get_aio_context(state->bs);
+    aio_context_acquire(aio_context);
+
     assert(state->job);
     block_job_start(state->job);
+
+    aio_context_release(aio_context);
 }
 
 static void blockdev_backup_abort(BlkActionState *common)
@@ -XXX,XX +XXX,XX @@ static void blockdev_backup_abort(BlkActionState *common)
     BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
 
     if (state->job) {
+        AioContext *aio_context;
+
+        aio_context = bdrv_get_aio_context(state->bs);
+        aio_context_acquire(aio_context);
+
         block_job_cancel_sync(state->job);
+
+        aio_context_release(aio_context);
     }
 }
 
 static void blockdev_backup_clean(BlkActionState *common)
 {
     BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
+    AioContext *aio_context;
 
-    if (state->aio_context) {
-        bdrv_drained_end(state->bs);
-        aio_context_release(state->aio_context);
+    if (!state->bs) {
+        return;
     }
+
+    aio_context = bdrv_get_aio_context(state->bs);
+    aio_context_acquire(aio_context);
+
+    bdrv_drained_end(state->bs);
+
+    aio_context_release(aio_context);
 }
 
 typedef struct BlockDirtyBitmapState {
-- 
2.14.3

diff --git a/blockdev.c b/blockdev.c
index XXXXXXX..XXXXXXX 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -XXX,XX +XXX,XX @@ struct BlkActionState {
 typedef struct InternalSnapshotState {
     BlkActionState common;
     BlockDriverState *bs;
-    AioContext *aio_context;
     QEMUSnapshotInfo sn;
     bool created;
 } InternalSnapshotState;
@@ -XXX,XX +XXX,XX @@ static void internal_snapshot_prepare(BlkActionState *common,
     qemu_timeval tv;
     BlockdevSnapshotInternal *internal;
     InternalSnapshotState *state;
+    AioContext *aio_context;
     int ret1;
 
     g_assert(common->action->type ==
@@ -XXX,XX +XXX,XX @@ static void internal_snapshot_prepare(BlkActionState *common,
         return;
     }
 
-    /* AioContext is released in .clean() */
-    state->aio_context = bdrv_get_aio_context(bs);
-    aio_context_acquire(state->aio_context);
+    aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(aio_context);
 
     state->bs = bs;
+
+    /* Paired with .clean() */
     bdrv_drained_begin(bs);
 
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT, errp)) {
-        return;
+        goto out;
     }
 
     if (bdrv_is_read_only(bs)) {
         error_setg(errp, "Device '%s' is read only", device);
-        return;
+        goto out;
     }
 
     if (!bdrv_can_snapshot(bs)) {
         error_setg(errp, "Block format '%s' used by device '%s' "
                    "does not support internal snapshots",
                    bs->drv->format_name, device);
-        return;
+        goto out;
     }
 
     if (!strlen(name)) {
         error_setg(errp, "Name is empty");
-        return;
+        goto out;
     }
 
     /* check whether a snapshot with name exist */
@@ -XXX,XX +XXX,XX @@ static void internal_snapshot_prepare(BlkActionState *common,
                                             &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
-        return;
+        goto out;
     } else if (ret) {
         error_setg(errp,
                    "Snapshot with name '%s' already exists on device '%s'",
                    name, device);
-        return;
+        goto out;
     }
 
     /* 3. take the snapshot */
@@ -XXX,XX +XXX,XX @@ static void internal_snapshot_prepare(BlkActionState *common,
         error_setg_errno(errp, -ret1,
                          "Failed to create snapshot '%s' on device '%s'",
                          name, device);
-        return;
+        goto out;
     }
 
     /* 4. succeed, mark a snapshot is created */
     state->created = true;
+
+out:
+    aio_context_release(aio_context);
 }
 
 static void internal_snapshot_abort(BlkActionState *common)
@@ -XXX,XX +XXX,XX @@ static void internal_snapshot_abort(BlkActionState *common)
                              DO_UPCAST(InternalSnapshotState, common, common);
     BlockDriverState *bs = state->bs;
     QEMUSnapshotInfo *sn = &state->sn;
+    AioContext *aio_context;
     Error *local_error = NULL;
 
     if (!state->created) {
         return;
     }
 
+    aio_context = bdrv_get_aio_context(state->bs);
+    aio_context_acquire(aio_context);
+
     if (bdrv_snapshot_delete(bs, sn->id_str, sn->name, &local_error) < 0) {
         error_reportf_err(local_error,
                           "Failed to delete snapshot with id '%s' and "
@@ -XXX,XX +XXX,XX @@ static void internal_snapshot_abort(BlkActionState *common)
                           sn->id_str, sn->name,
                           bdrv_get_device_name(bs));
     }
+
+    aio_context_release(aio_context);
 }
 
 static void internal_snapshot_clean(BlkActionState *common)
 {
     InternalSnapshotState *state = DO_UPCAST(InternalSnapshotState,
                                              common, common);
+    AioContext *aio_context;
 
-    if (state->aio_context) {
-        if (state->bs) {
-            bdrv_drained_end(state->bs);
-        }
-        aio_context_release(state->aio_context);
+    if (!state->bs) {
+        return;
     }
+
+    aio_context = bdrv_get_aio_context(state->bs);
+    aio_context_acquire(aio_context);
+
+    bdrv_drained_end(state->bs);
+
+    aio_context_release(aio_context);
 }
 
 /* external snapshot private data */
-- 
2.14.3

The dirty bitmap actions in qmp_transaction have not used AioContext
since the dirty bitmap locking discipline was introduced in commit
2119882c7eb7e2c612b24fc0c8d86f5887d6f1c3 ("block: introduce
dirty_bitmap_mutex").  Remove the unused field.

diff --git a/blockdev.c b/blockdev.c
index XXXXXXX..XXXXXXX 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -XXX,XX +XXX,XX @@ typedef struct BlockDirtyBitmapState {
     BlkActionState common;
     BdrvDirtyBitmap *bitmap;
     BlockDriverState *bs;
-    AioContext *aio_context;
     HBitmap *backup;
     bool prepared;
 } BlockDirtyBitmapState;
@@ -XXX,XX +XXX,XX @@ static void block_dirty_bitmap_clear_prepare(BlkActionState *common,
     }
 
     bdrv_clear_dirty_bitmap(state->bitmap, &state->backup);
-    /* AioContext is released in .clean() */
 }
 
 static void block_dirty_bitmap_clear_abort(BlkActionState *common)
@@ -XXX,XX +XXX,XX @@ static void block_dirty_bitmap_clear_commit(BlkActionState *common)
     hbitmap_free(state->backup);
 }
 
-static void block_dirty_bitmap_clear_clean(BlkActionState *common)
-{
-    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
-                                             common, common);
-
-    if (state->aio_context) {
-        aio_context_release(state->aio_context);
-    }
-}
-
 static void abort_prepare(BlkActionState *common, Error **errp)
 {
     error_setg(errp, "Transaction aborted using Abort action");
@@ -XXX,XX +XXX,XX @@ static const BlkActionOps actions[] = {
         .prepare = block_dirty_bitmap_clear_prepare,
         .commit = block_dirty_bitmap_clear_commit,
         .abort = block_dirty_bitmap_clear_abort,
-        .clean = block_dirty_bitmap_clear_clean,
     }
 };
 
-- 
2.14.3

Encapsulate IOThread QOM object lookup so that callers don't need to
know how and where IOThread objects live.

diff --git a/include/sysemu/iothread.h b/include/sysemu/iothread.h
index XXXXXXX..XXXXXXX 100644
--- a/include/sysemu/iothread.h
+++ b/include/sysemu/iothread.h
@@ -XXX,XX +XXX,XX @@ typedef struct {
    OBJECT_CHECK(IOThread, obj, TYPE_IOTHREAD)
 
 char *iothread_get_id(IOThread *iothread);
+IOThread *iothread_by_id(const char *id);
 AioContext *iothread_get_aio_context(IOThread *iothread);
 void iothread_stop_all(void);
 GMainContext *iothread_get_g_main_context(IOThread *iothread);
diff --git a/iothread.c b/iothread.c
index XXXXXXX..XXXXXXX 100644
--- a/iothread.c
+++ b/iothread.c
@@ -XXX,XX +XXX,XX @@ void iothread_destroy(IOThread *iothread)
 {
     object_unparent(OBJECT(iothread));
 }
+
+/* Lookup IOThread by its id.  Only finds user-created objects, not internal
+ * iothread_create() objects. */
+IOThread *iothread_by_id(const char *id)
+{
+    return IOTHREAD(object_resolve_path_type(id, TYPE_IOTHREAD, NULL));
+}
-- 
2.14.3

Currently there is no easy way for iotests to ensure that a BDS is bound
to a particular IOThread.  Normally the virtio-blk device calls
blk_set_aio_context() when dataplane is enabled during guest driver
initialization.  This never happens in iotests since -machine
accel=qtest means there is no guest activity (including device driver
initialization).

This patch adds a QMP command to explicitly assign IOThreads in test
cases.  See qapi/block-core.json for a description of the command.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-id: 20171206144550.22295-9-stefanha@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 qapi/block-core.json | 36 ++++++++++++++++++++++++++++++++++++
 blockdev.c           | 41 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index XXXXXXX..XXXXXXX 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -XXX,XX +XXX,XX @@
   'data' : { 'parent': 'str',
              '*child': 'str',
              '*node': 'str' } }
+
+##
+# @x-blockdev-set-iothread:
+#
+# Move @node and its children into the @iothread.  If @iothread is null then
+# move @node and its children into the main loop.
+#
+# The node must not be attached to a BlockBackend.
+#
+# @node-name: the name of the block driver node
+#
+# @iothread: the name of the IOThread object or null for the main loop
+#
+# Note: this command is experimental and intended for test cases that need
+# control over IOThreads only.
+#
+# Since: 2.12
+#
+# Example:
+#
+# 1. Move a node into an IOThread
+# -> { "execute": "x-blockdev-set-iothread",
+#      "arguments": { "node-name": "disk1",
+#                     "iothread": "iothread0" } }
+# <- { "return": {} }
+#
+# 2. Move a node into the main loop
+# -> { "execute": "x-blockdev-set-iothread",
+#      "arguments": { "node-name": "disk1",
+#                     "iothread": null } }
+# <- { "return": {} }
+#
+##
+{ 'command': 'x-blockdev-set-iothread',
+  'data' : { 'node-name': 'str',
+             'iothread': 'StrOrNull' } }
diff --git a/blockdev.c b/blockdev.c
index XXXXXXX..XXXXXXX 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -XXX,XX +XXX,XX @@
 #include "qapi/qmp/qerror.h"
 #include "qapi/qobject-output-visitor.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/iothread.h"
 #include "block/block_int.h"
 #include "qmp-commands.h"
 #include "block/trace.h"
@@ -XXX,XX +XXX,XX @@ BlockJobInfoList *qmp_query_block_jobs(Error **errp)
     return head;
 }
 
+void qmp_x_blockdev_set_iothread(const char *node_name, StrOrNull *iothread,
+                                 Error **errp)
+{
+    AioContext *old_context;
+    AioContext *new_context;
+    BlockDriverState *bs;
+
+    bs = bdrv_find_node(node_name);
+    if (!bs) {
+        error_setg(errp, "Cannot find node %s", node_name);
+        return;
+    }
+
+    /* If we want to allow more extreme test scenarios this guard could be
+     * removed.  For now it protects against accidents. */
+    if (bdrv_has_blk(bs)) {
+        error_setg(errp, "Node %s is in use", node_name);
+        return;
+    }
+
+    if (iothread->type == QTYPE_QSTRING) {
+        IOThread *obj = iothread_by_id(iothread->u.s);
+        if (!obj) {
+            error_setg(errp, "Cannot find iothread %s", iothread->u.s);
+            return;
+        }
+
+        new_context = iothread_get_aio_context(obj);
+    } else {
+        new_context = qemu_get_aio_context();
+    }
+
+    old_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(old_context);
+
+    bdrv_set_aio_context(bs, new_context);
+
+    aio_context_release(old_context);
+}
+
 QemuOptsList qemu_common_drive_opts = {
     .name = "drive",
     .head = QTAILQ_HEAD_INITIALIZER(qemu_common_drive_opts.head),
-- 
2.14.3

QMP 'transaction' blockdev-snapshot-sync with multiple disks in an
IOThread is an untested code path.  Several bugs have been found in
connection with this command.  This patch adds a test case to prevent
future regressions.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-id: 20171206144550.22295-10-stefanha@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 tests/qemu-iotests/202     | 95 ++++++++++++++++++++++++++++++++++++++++++++++
 tests/qemu-iotests/202.out | 11 ++++++
 tests/qemu-iotests/group   |  1 +
 3 files changed, 107 insertions(+)
 create mode 100755 tests/qemu-iotests/202
 create mode 100644 tests/qemu-iotests/202.out

diff --git a/tests/qemu-iotests/202 b/tests/qemu-iotests/202
new file mode 100755
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/tests/qemu-iotests/202
@@ -XXX,XX +XXX,XX @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2017 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Creator/Owner: Stefan Hajnoczi <stefanha@redhat.com>
+#
+# Check that QMP 'transaction' blockdev-snapshot-sync with multiple drives on a
+# single IOThread completes successfully.  This particular command triggered a
+# hang due to recursive AioContext locking and BDRV_POLL_WHILE().  Protect
+# against regressions.
+
+import iotests
+
+iotests.verify_image_format(supported_fmts=['qcow2'])
+iotests.verify_platform(['linux'])
+
+with iotests.FilePath('disk0.img') as disk0_img_path, \
+     iotests.FilePath('disk1.img') as disk1_img_path, \
+     iotests.FilePath('disk0-snap.img') as disk0_snap_img_path, \
+     iotests.FilePath('disk1-snap.img') as disk1_snap_img_path, \
+     iotests.VM() as vm:
+
+    img_size = '10M'
+    iotests.qemu_img_pipe('create', '-f', iotests.imgfmt, disk0_img_path, img_size)
+    iotests.qemu_img_pipe('create', '-f', iotests.imgfmt, disk1_img_path, img_size)
+
+    iotests.log('Launching VM...')
+    vm.launch()
+
+    iotests.log('Adding IOThread...')
+    iotests.log(vm.qmp('object-add',
+                       qom_type='iothread',
+                       id='iothread0'))
+
+    iotests.log('Adding blockdevs...')
+    iotests.log(vm.qmp('blockdev-add',
+                       driver=iotests.imgfmt,
+                       node_name='disk0',
+                       file={
+                           'driver': 'file',
+                           'filename': disk0_img_path,
+                       }))
+    iotests.log(vm.qmp('blockdev-add',
+                       driver=iotests.imgfmt,
+                       node_name='disk1',
+                       file={
+                           'driver': 'file',
+                           'filename': disk1_img_path,
+                       }))
+
+    iotests.log('Setting iothread...')
+    iotests.log(vm.qmp('x-blockdev-set-iothread',
+                       node_name='disk0',
+                       iothread='iothread0'))
+    iotests.log(vm.qmp('x-blockdev-set-iothread',
+                       node_name='disk1',
+                       iothread='iothread0'))
+
+    iotests.log('Creating external snapshots...')
+    iotests.log(vm.qmp(
+                  'transaction',
+                  actions=[
+                      {
+                          'data': {
+                              'node-name': 'disk0',
+                              'snapshot-file': disk0_snap_img_path,
+                              'snapshot-node-name': 'disk0-snap',
+                              'mode': 'absolute-paths',
+                              'format': iotests.imgfmt,
+                          },
+                          'type': 'blockdev-snapshot-sync'
+                      }, {
+                          'data': {
+                              'node-name': 'disk1',
+                              'snapshot-file': disk1_snap_img_path,
+                              'snapshot-node-name': 'disk1-snap',
+                              'mode': 'absolute-paths',
+                              'format': iotests.imgfmt
+                          },
+                          'type': 'blockdev-snapshot-sync'
+                      }
+                  ]))
diff --git a/tests/qemu-iotests/202.out b/tests/qemu-iotests/202.out
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/tests/qemu-iotests/202.out
@@ -XXX,XX +XXX,XX @@
+Launching VM...
+Adding IOThread...
+{u'return': {}}
+Adding blockdevs...
+{u'return': {}}
+{u'return': {}}
+Setting iothread...
+{u'return': {}}
+{u'return': {}}
+Creating external snapshots...
+{u'return': {}}
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -XXX,XX +XXX,XX @@
 197 rw auto quick
 198 rw auto
 200 rw auto
+202 rw auto quick
-- 
2.14.3

From: Mark Kanda <mark.kanda@oracle.com>

Depending on the configuration, it can be beneficial to adjust the virtio-blk
queue size to something other than the current default of 128. Add a new
property to make the queue size configurable.

Signed-off-by: Mark Kanda <mark.kanda@oracle.com>
Reviewed-by: Karl Heubaum <karl.heubaum@oracle.com>
Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com>
Reviewed-by: Ameya More <ameya.more@oracle.com>
Message-id: 52e6d742811f10dbd16e996e86cf375b9577c187.1513005190.git.mark.kanda@oracle.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 include/hw/virtio/virtio-blk.h |  1 +
 hw/block/virtio-blk.c          | 10 +++++++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/include/hw/virtio/virtio-blk.h b/include/hw/virtio/virtio-blk.h
index XXXXXXX..XXXXXXX 100644
--- a/include/hw/virtio/virtio-blk.h
+++ b/include/hw/virtio/virtio-blk.h
@@ -XXX,XX +XXX,XX @@ struct VirtIOBlkConf
     uint32_t config_wce;
     uint32_t request_merging;
     uint16_t num_queues;
+    uint16_t queue_size;
 };
 
 struct VirtIOBlockDataPlane;
diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/block/virtio-blk.c
+++ b/hw/block/virtio-blk.c
@@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
         error_setg(errp, "num-queues property must be larger than 0");
         return;
     }
+    if (!is_power_of_2(conf->queue_size) ||
+        conf->queue_size > VIRTQUEUE_MAX_SIZE) {
+        error_setg(errp, "invalid queue-size property (%" PRIu16 "), "
+                   "must be a power of 2 (max %d)",
+                   conf->queue_size, VIRTQUEUE_MAX_SIZE);
+        return;
+    }
 
     blkconf_serial(&conf->conf, &conf->serial);
     if (!blkconf_apply_backend_options(&conf->conf,
@@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
     s->sector_mask = (s->conf.conf.logical_block_size / BDRV_SECTOR_SIZE) - 1;
 
     for (i = 0; i < conf->num_queues; i++) {
-        virtio_add_queue(vdev, 128, virtio_blk_handle_output);
+        virtio_add_queue(vdev, conf->queue_size, virtio_blk_handle_output);
     }
     if (!virtio_blk_data_plane_create(vdev, conf, &s->dataplane, errp)) {
         virtio_cleanup(vdev);
@@ -XXX,XX +XXX,XX @@ static Property virtio_blk_properties[] = {
     DEFINE_PROP_BIT("request-merging", VirtIOBlock, conf.request_merging, 0,
                     true),
     DEFINE_PROP_UINT16("num-queues", VirtIOBlock, conf.num_queues, 1),
+    DEFINE_PROP_UINT16("queue-size", VirtIOBlock, conf.queue_size, 128),
     DEFINE_PROP_LINK("iothread", VirtIOBlock, conf.iothread, TYPE_IOTHREAD,
                      IOThread *),
     DEFINE_PROP_END_OF_LIST(),
-- 
2.14.3

From: Mark Kanda <mark.kanda@oracle.com>

virtio-blk logical block size should never be larger than physical block
size because it doesn't make sense to have such configurations. QEMU doesn't
have a way to effectively express this condition; the best it can do is
report the physical block exponent as 0 - indicating the logical block size
equals the physical block size.

This is identical to commit 3da023b5827543ee4c022986ea2ad9d1274410b2
but applied to virtio-blk (instead of virtio-scsi).

Signed-off-by: Mark Kanda <mark.kanda@oracle.com>
Reviewed-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Reviewed-by: Ameya More <ameya.more@oracle.com>
Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
Message-id: 773169891f9f2deb4cb7c4ef2655580dbe24c1d1.1513005190.git.mark.kanda@oracle.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 hw/block/virtio-blk.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
index XXXXXXX..XXXXXXX 100644
--- a/hw/block/virtio-blk.c
+++ b/hw/block/virtio-blk.c
@@ -XXX,XX +XXX,XX @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
 
     blkconf_blocksizes(&conf->conf);
 
+    if (conf->conf.logical_block_size >
+        conf->conf.physical_block_size) {
+        error_setg(errp,
+                   "logical_block_size > physical_block_size not supported");
+        return;
+    }
+
     virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK,
                 sizeof(struct virtio_blk_config));
 
-- 
2.14.3

From: Paolo Bonzini <pbonzini@redhat.com>

BDRV_POLL_WHILE() does not support recursive AioContext locking.  It
only releases the AioContext lock once regardless of how many times the
caller has acquired it.  This results in a hang since the IOThread does
not make progress while the AioContext is still locked.

The following steps trigger the hang:

$ qemu-system-x86_64 -M accel=kvm -m 1G -cpu host \
                       -object iothread,id=iothread0 \
                       -device virtio-scsi-pci,iothread=iothread0 \
                       -drive if=none,id=drive0,file=test.img,format=raw \
                       -device scsi-hd,drive=drive0 \
                       -drive if=none,id=drive1,file=test.img,format=raw \
                       -device scsi-hd,drive=drive1
  $ qemu-system-x86_64 ...same options... \
                       -incoming tcp::1234
  (qemu) migrate tcp:127.0.0.1:1234
  ...hang...

Tested-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-id: 20171207201320.19284-2-stefanha@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 block.c | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/block.c b/block.c
index XXXXXXX..XXXXXXX 100644
--- a/block.c
+++ b/block.c
@@ -XXX,XX +XXX,XX @@ int bdrv_inactivate_all(void)
     BdrvNextIterator it;
     int ret = 0;
     int pass;
+    GSList *aio_ctxs = NULL, *ctx;
 
     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
-        aio_context_acquire(bdrv_get_aio_context(bs));
+        AioContext *aio_context = bdrv_get_aio_context(bs);
+
+        if (!g_slist_find(aio_ctxs, aio_context)) {
+            aio_ctxs = g_slist_prepend(aio_ctxs, aio_context);
+            aio_context_acquire(aio_context);
+        }
     }
 
     /* We do two passes of inactivation. The first pass calls to drivers'
@@ -XXX,XX +XXX,XX @@ int bdrv_inactivate_all(void)
     }
 
 out:
-    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
-        aio_context_release(bdrv_get_aio_context(bs));
+    for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) {
+        AioContext *aio_context = ctx->data;
+        aio_context_release(aio_context);
     }
+    g_slist_free(aio_ctxs);
 
     return ret;
 }
-- 
2.14.3

See the patch for why nested AioContext locking is no longer allowed.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-id: 20171207201320.19284-3-stefanha@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 docs/devel/multiple-iothreads.txt | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/devel/multiple-iothreads.txt b/docs/devel/multiple-iothreads.txt
index XXXXXXX..XXXXXXX 100644
--- a/docs/devel/multiple-iothreads.txt
+++ b/docs/devel/multiple-iothreads.txt
@@ -XXX,XX +XXX,XX @@
-Copyright (c) 2014 Red Hat Inc.
+Copyright (c) 2014-2017 Red Hat Inc.
 
 This work is licensed under the terms of the GNU GPL, version 2 or later.  See
 the COPYING file in the top-level directory.
@@ -XXX,XX +XXX,XX @@ aio_context_acquire()/aio_context_release() for mutual exclusion.  Once the
 context is acquired no other thread can access it or run event loop iterations
 in this AioContext.
 
-aio_context_acquire()/aio_context_release() calls may be nested.  This
-means you can call them if you're not sure whether #2 applies.
+Legacy code sometimes nests aio_context_acquire()/aio_context_release() calls.
+Do not use nesting anymore, it is incompatible with the BDRV_POLL_WHILE() macro
+used in the block layer and can lead to hangs.
 
 There is currently no lock ordering rule if a thread needs to acquire multiple
 AioContexts simultaneously.  Therefore, it is only safe for code holding the
-- 
2.14.3

When a node is already associated with a BlockBackend the
x-blockdev-set-iothread command refuses to set the IOThread.  This is to
prevent accidentally changing the IOThread when the nodes are in use.

When the nodes are created with -drive they automatically get a
BlockBackend.  In that case we know nothing is using them yet and it's
safe to set the IOThread.  Add a force boolean to override the check.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-id: 20171207201320.19284-4-stefanha@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 qapi/block-core.json |  6 +++++-
 blockdev.c           | 11 ++++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index XXXXXXX..XXXXXXX 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -XXX,XX +XXX,XX @@
 #
 # @iothread: the name of the IOThread object or null for the main loop
 #
+# @force: true if the node and its children should be moved when a BlockBackend
+#         is already attached
+#
 # Note: this command is experimental and intended for test cases that need
 # control over IOThreads only.
 #
@@ -XXX,XX +XXX,XX @@
 ##
 { 'command': 'x-blockdev-set-iothread',
   'data' : { 'node-name': 'str',
-             'iothread': 'StrOrNull' } }
+             'iothread': 'StrOrNull',
+             '*force': 'bool' } }
diff --git a/blockdev.c b/blockdev.c
index XXXXXXX..XXXXXXX 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -XXX,XX +XXX,XX @@ BlockJobInfoList *qmp_query_block_jobs(Error **errp)
 }
 
 void qmp_x_blockdev_set_iothread(const char *node_name, StrOrNull *iothread,
-                                 Error **errp)
+                                 bool has_force, bool force, Error **errp)
 {
     AioContext *old_context;
     AioContext *new_context;
@@ -XXX,XX +XXX,XX @@ void qmp_x_blockdev_set_iothread(const char *node_name, StrOrNull *iothread,
         return;
     }
 
-    /* If we want to allow more extreme test scenarios this guard could be
-     * removed.  For now it protects against accidents. */
-    if (bdrv_has_blk(bs)) {
-        error_setg(errp, "Node %s is in use", node_name);
+    /* Protects against accidents. */
+    if (!(has_force && force) && bdrv_has_blk(bs)) {
+        error_setg(errp, "Node %s is associated with a BlockBackend and could "
+                         "be in use (use force=true to override this check)",
+                         node_name);
         return;
     }
 
-- 
2.14.3

There is a small chance that iothread_stop() hangs as follows:

Thread 3 (Thread 0x7f63eba5f700 (LWP 16105)):
  #0  0x00007f64012c09b6 in ppoll () at /lib64/libc.so.6
  #1  0x000055959992eac9 in ppoll (__ss=0x0, __timeout=0x0, __nfds=<optimized out>, __fds=<optimized out>) at /usr/include/bits/poll2.h:77
  #2  0x000055959992eac9 in qemu_poll_ns (fds=<optimized out>, nfds=<optimized out>, timeout=<optimized out>) at util/qemu-timer.c:322
  #3  0x0000559599930711 in aio_poll (ctx=0x55959bdb83c0, blocking=blocking@entry=true) at util/aio-posix.c:629
  #4  0x00005595996806fe in iothread_run (opaque=0x55959bd78400) at iothread.c:59
  #5  0x00007f640159f609 in start_thread () at /lib64/libpthread.so.0
  #6  0x00007f64012cce6f in clone () at /lib64/libc.so.6

Thread 1 (Thread 0x7f640b45b280 (LWP 16103)):
  #0  0x00007f64015a0b6d in pthread_join () at /lib64/libpthread.so.0
  #1  0x00005595999332ef in qemu_thread_join (thread=<optimized out>) at util/qemu-thread-posix.c:547
  #2  0x00005595996808ae in iothread_stop (iothread=<optimized out>) at iothread.c:91
  #3  0x000055959968094d in iothread_stop_iter (object=<optimized out>, opaque=<optimized out>) at iothread.c:102
  #4  0x0000559599857d97 in do_object_child_foreach (obj=obj@entry=0x55959bdb8100, fn=fn@entry=0x559599680930 <iothread_stop_iter>, opaque=opaque@entry=0x0, recurse=recurse@entry=false) at qom/object.c:852
  #5  0x0000559599859477 in object_child_foreach (obj=obj@entry=0x55959bdb8100, fn=fn@entry=0x559599680930 <iothread_stop_iter>, opaque=opaque@entry=0x0) at qom/object.c:867
  #6  0x0000559599680a6e in iothread_stop_all () at iothread.c:341
  #7  0x000055959955b1d5 in main (argc=<optimized out>, argv=<optimized out>, envp=<optimized out>) at vl.c:4913

The relevant code from iothread_run() is:

while (!atomic_read(&iothread->stopping)) {
      aio_poll(iothread->ctx, true);

and iothread_stop():

iothread->stopping = true;
  aio_notify(iothread->ctx);
  ...
  qemu_thread_join(&iothread->thread);

The following scenario can occur:

1. IOThread:
  while (!atomic_read(&iothread->stopping)) -> stopping=false

2. Main loop:
  iothread->stopping = true;
  aio_notify(iothread->ctx);

3. IOThread:
  aio_poll(iothread->ctx, true); -> hang

The bug is explained by the AioContext->notify_me doc comments:

"If this field is 0, everything (file descriptors, bottom halves,
  timers) will be re-evaluated before the next blocking poll(), thus the
  event_notifier_set call can be skipped."

The problem is that "everything" does not include checking
iothread->stopping.  This means iothread_run() will block in aio_poll()
if aio_notify() was called just before aio_poll().

This patch fixes the hang by replacing aio_notify() with
aio_bh_schedule_oneshot().  This makes aio_poll() or g_main_loop_run()
to return.

Implementing this properly required a new bool running flag.  The new
flag prevents races that are tricky if we try to use iothread->stopping.
Now iothread->stopping is purely for iothread_stop() and
iothread->running is purely for the iothread_run() thread.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-id: 20171207201320.19284-6-stefanha@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 include/sysemu/iothread.h |  3 ++-
 iothread.c                | 20 +++++++++++++++-----
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/include/sysemu/iothread.h b/include/sysemu/iothread.h
index XXXXXXX..XXXXXXX 100644
--- a/include/sysemu/iothread.h
+++ b/include/sysemu/iothread.h
@@ -XXX,XX +XXX,XX @@ typedef struct {
     GOnce once;
     QemuMutex init_done_lock;
     QemuCond init_done_cond;    /* is thread initialization done? */
-    bool stopping;
+    bool stopping;              /* has iothread_stop() been called? */
+    bool running;               /* should iothread_run() continue? */
     int thread_id;
 
     /* AioContext poll parameters */
diff --git a/iothread.c b/iothread.c
index XXXXXXX..XXXXXXX 100644
--- a/iothread.c
+++ b/iothread.c
@@ -XXX,XX +XXX,XX @@ static void *iothread_run(void *opaque)
     qemu_cond_signal(&iothread->init_done_cond);
     qemu_mutex_unlock(&iothread->init_done_lock);
 
-    while (!atomic_read(&iothread->stopping)) {
+    while (iothread->running) {
         aio_poll(iothread->ctx, true);
 
         if (atomic_read(&iothread->worker_context)) {
@@ -XXX,XX +XXX,XX @@ static void *iothread_run(void *opaque)
     return NULL;
 }
 
+/* Runs in iothread_run() thread */
+static void iothread_stop_bh(void *opaque)
+{
+    IOThread *iothread = opaque;
+
+    iothread->running = false; /* stop iothread_run() */
+
+    if (iothread->main_loop) {
+        g_main_loop_quit(iothread->main_loop);
+    }
+}
+
 void iothread_stop(IOThread *iothread)
 {
     if (!iothread->ctx || iothread->stopping) {
         return;
     }
     iothread->stopping = true;
-    aio_notify(iothread->ctx);
-    if (atomic_read(&iothread->main_loop)) {
-        g_main_loop_quit(iothread->main_loop);
-    }
+    aio_bh_schedule_oneshot(iothread->ctx, iothread_stop_bh, iothread);
     qemu_thread_join(&iothread->thread);
 }
 
@@ -XXX,XX +XXX,XX @@ static void iothread_complete(UserCreatable *obj, Error **errp)
     char *name, *thread_name;
 
     iothread->stopping = false;
+    iothread->running = true;
     iothread->thread_id = -1;
     iothread->ctx = aio_context_new(&local_error);
     if (!iothread->ctx) {
-- 
2.14.3

This test case will prevent future regressions with savevm and
IOThreads.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Message-id: 20171207201320.19284-7-stefanha@redhat.com
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 tests/qemu-iotests/203     | 59 ++++++++++++++++++++++++++++++++++++++++++++++
 tests/qemu-iotests/203.out |  6 +++++
 tests/qemu-iotests/group   |  1 +
 3 files changed, 66 insertions(+)
 create mode 100755 tests/qemu-iotests/203
 create mode 100644 tests/qemu-iotests/203.out

diff --git a/tests/qemu-iotests/203 b/tests/qemu-iotests/203
new file mode 100755
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/tests/qemu-iotests/203
@@ -XXX,XX +XXX,XX @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2017 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Creator/Owner: Stefan Hajnoczi <stefanha@redhat.com>
+#
+# Check that QMP 'migrate' with multiple drives on a single IOThread completes
+# successfully.  This particular command triggered a hang in the source QEMU
+# process due to recursive AioContext locking in bdrv_invalidate_all() and
+# BDRV_POLL_WHILE().
+
+import iotests
+
+iotests.verify_image_format(supported_fmts=['qcow2'])
+iotests.verify_platform(['linux'])
+
+with iotests.FilePath('disk0.img') as disk0_img_path, \
+     iotests.FilePath('disk1.img') as disk1_img_path, \
+     iotests.VM() as vm:
+
+    img_size = '10M'
+    iotests.qemu_img_pipe('create', '-f', iotests.imgfmt, disk0_img_path, img_size)
+    iotests.qemu_img_pipe('create', '-f', iotests.imgfmt, disk1_img_path, img_size)
+
+    iotests.log('Launching VM...')
+    (vm.add_object('iothread,id=iothread0')
+       .add_drive(disk0_img_path, 'node-name=drive0-node', interface='none')
+       .add_drive(disk1_img_path, 'node-name=drive1-node', interface='none')
+       .launch())
+
+    iotests.log('Setting IOThreads...')
+    iotests.log(vm.qmp('x-blockdev-set-iothread',
+                       node_name='drive0-node', iothread='iothread0',
+                       force=True))
+    iotests.log(vm.qmp('x-blockdev-set-iothread',
+                       node_name='drive1-node', iothread='iothread0',
+                       force=True))
+
+    iotests.log('Starting migration...')
+    iotests.log(vm.qmp('migrate', uri='exec:cat >/dev/null'))
+    while True:
+        vm.get_qmp_event(wait=60.0)
+        result = vm.qmp('query-migrate')
+        status = result.get('return', {}).get('status', None)
+        if status == 'completed':
+            break
diff --git a/tests/qemu-iotests/203.out b/tests/qemu-iotests/203.out
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/tests/qemu-iotests/203.out
@@ -XXX,XX +XXX,XX @@
+Launching VM...
+Setting IOThreads...
+{u'return': {}}
+{u'return': {}}
+Starting migration...
+{u'return': {}}
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -XXX,XX +XXX,XX @@
 198 rw auto
 200 rw auto
 202 rw auto quick
+203 rw auto
-- 
2.14.3

The following changes since commit ba29883206d92a29ad5a466e679ccfc2ee6132ef:

Merge remote-tracking branch 'remotes/borntraeger/tags/s390x-20200310' into staging (2020-03-10 16:50:28 +0000)

are available in the Git repository at:

https://github.com/XanClic/qemu.git tags/pull-block-2020-03-11

for you to fetch changes up to 397f4e9d83e9c0000905f0a988ba1aeda162571c:

block/block-copy: hide structure definitions (2020-03-11 12:42:30 +0100)

----------------------------------------------------------------
Block patches for the 5.0 softfreeze:
- qemu-img measure for LUKS
- Improve block-copy's performance by reducing inter-request
  dependencies
- Make curl's detection of accept-ranges more robust
- Memleak fixes
- iotest fix

----------------------------------------------------------------
David Edmondson (2):
  block/curl: HTTP header fields allow whitespace around values
  block/curl: HTTP header field names are case insensitive

Eric Blake (1):
  iotests: Fix nonportable use of od --endian

Pan Nengyuan (2):
  block/qcow2: do free crypto_opts in qcow2_close()
  qemu-img: free memory before re-assign

Stefan Hajnoczi (4):
  luks: extract qcrypto_block_calculate_payload_offset()
  luks: implement .bdrv_measure()
  qemu-img: allow qemu-img measure --object without a filename
  iotests: add 288 luks qemu-img measure test

Vladimir Sementsov-Ogievskiy (10):
  block/qcow2-threads: fix qcow2_decompress
  job: refactor progress to separate object
  block/block-copy: fix progress calculation
  block/block-copy: specialcase first copy_range request
  block/block-copy: use block_status
  block/block-copy: factor out find_conflicting_inflight_req
  block/block-copy: refactor interfaces to use bytes instead of end
  block/block-copy: rename start to offset in interfaces
  block/block-copy: reduce intersecting request lock
  block/block-copy: hide structure definitions

-- 
2.24.1

From: Stefan Hajnoczi <stefanha@redhat.com>

The qcow2 .bdrv_measure() code calculates the crypto payload offset.
This logic really belongs in crypto/block.c where it can be reused by
other image formats.

The "luks" block driver will need this same logic in order to implement
.bdrv_measure(), so extract the qcrypto_block_calculate_payload_offset()
function now.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200221112522.1497712-2-stefanha@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/qcow2.c          | 74 +++++++++++-------------------------------
 crypto/block.c         | 36 ++++++++++++++++++++
 include/crypto/block.h | 22 +++++++++++++
 3 files changed, 77 insertions(+), 55 deletions(-)

diff --git a/block/qcow2.c b/block/qcow2.c
index XXXXXXX..XXXXXXX 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_flush_to_os(BlockDriverState *bs)
     return ret;
 }
 
-static ssize_t qcow2_measure_crypto_hdr_init_func(QCryptoBlock *block,
-        size_t headerlen, void *opaque, Error **errp)
-{
-    size_t *headerlenp = opaque;
-
-    /* Stash away the payload size */
-    *headerlenp = headerlen;
-    return 0;
-}
-
-static ssize_t qcow2_measure_crypto_hdr_write_func(QCryptoBlock *block,
-        size_t offset, const uint8_t *buf, size_t buflen,
-        void *opaque, Error **errp)
-{
-    /* Discard the bytes, we're not actually writing to an image */
-    return buflen;
-}
-
-/* Determine the number of bytes for the LUKS payload */
-static bool qcow2_measure_luks_headerlen(QemuOpts *opts, size_t *len,
-                                         Error **errp)
-{
-    QDict *opts_qdict;
-    QDict *cryptoopts_qdict;
-    QCryptoBlockCreateOptions *cryptoopts;
-    QCryptoBlock *crypto;
-
-    /* Extract "encrypt." options into a qdict */
-    opts_qdict = qemu_opts_to_qdict(opts, NULL);
-    qdict_extract_subqdict(opts_qdict, &cryptoopts_qdict, "encrypt.");
-    qobject_unref(opts_qdict);
-
-    /* Build QCryptoBlockCreateOptions object from qdict */
-    qdict_put_str(cryptoopts_qdict, "format", "luks");
-    cryptoopts = block_crypto_create_opts_init(cryptoopts_qdict, errp);
-    qobject_unref(cryptoopts_qdict);
-    if (!cryptoopts) {
-        return false;
-    }
-
-    /* Fake LUKS creation in order to determine the payload size */
-    crypto = qcrypto_block_create(cryptoopts, "encrypt.",
-                                  qcow2_measure_crypto_hdr_init_func,
-                                  qcow2_measure_crypto_hdr_write_func,
-                                  len, errp);
-    qapi_free_QCryptoBlockCreateOptions(cryptoopts);
-    if (!crypto) {
-        return false;
-    }
-
-    qcrypto_block_free(crypto);
-    return true;
-}
-
 static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs,
                                        Error **errp)
 {
@@ -XXX,XX +XXX,XX @@ static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs,
     g_free(optstr);
 
     if (has_luks) {
+        g_autoptr(QCryptoBlockCreateOptions) create_opts = NULL;
+        QDict *opts_qdict;
+        QDict *cryptoopts;
         size_t headerlen;
 
-        if (!qcow2_measure_luks_headerlen(opts, &headerlen, &local_err)) {
+        opts_qdict = qemu_opts_to_qdict(opts, NULL);
+        qdict_extract_subqdict(opts_qdict, &cryptoopts, "encrypt.");
+        qobject_unref(opts_qdict);
+
+        qdict_put_str(cryptoopts, "format", "luks");
+
+        create_opts = block_crypto_create_opts_init(cryptoopts, errp);
+        qobject_unref(cryptoopts);
+        if (!create_opts) {
+            goto err;
+        }
+
+        if (!qcrypto_block_calculate_payload_offset(create_opts,
+                                                    "encrypt.",
+                                                    &headerlen,
+                                                    &local_err)) {
             goto err;
         }
 
diff --git a/crypto/block.c b/crypto/block.c
index XXXXXXX..XXXXXXX 100644
--- a/crypto/block.c
+++ b/crypto/block.c
@@ -XXX,XX +XXX,XX @@ QCryptoBlock *qcrypto_block_create(QCryptoBlockCreateOptions *options,
 }
 
 
+static ssize_t qcrypto_block_headerlen_hdr_init_func(QCryptoBlock *block,
+        size_t headerlen, void *opaque, Error **errp)
+{
+    size_t *headerlenp = opaque;
+
+    /* Stash away the payload size */
+    *headerlenp = headerlen;
+    return 0;
+}
+
+
+static ssize_t qcrypto_block_headerlen_hdr_write_func(QCryptoBlock *block,
+        size_t offset, const uint8_t *buf, size_t buflen,
+        void *opaque, Error **errp)
+{
+    /* Discard the bytes, we're not actually writing to an image */
+    return buflen;
+}
+
+
+bool
+qcrypto_block_calculate_payload_offset(QCryptoBlockCreateOptions *create_opts,
+                                       const char *optprefix,
+                                       size_t *len,
+                                       Error **errp)
+{
+    /* Fake LUKS creation in order to determine the payload size */
+    g_autoptr(QCryptoBlock) crypto =
+        qcrypto_block_create(create_opts, optprefix,
+                             qcrypto_block_headerlen_hdr_init_func,
+                             qcrypto_block_headerlen_hdr_write_func,
+                             len, errp);
+    return crypto != NULL;
+}
+
+
 QCryptoBlockInfo *qcrypto_block_get_info(QCryptoBlock *block,
                                          Error **errp)
 {
diff --git a/include/crypto/block.h b/include/crypto/block.h
index XXXXXXX..XXXXXXX 100644
--- a/include/crypto/block.h
+++ b/include/crypto/block.h
@@ -XXX,XX +XXX,XX @@ QCryptoBlock *qcrypto_block_create(QCryptoBlockCreateOptions *options,
                                    Error **errp);
 
 
+/**
+ * qcrypto_block_calculate_payload_offset:
+ * @create_opts: the encryption options
+ * @optprefix: name prefix for options
+ * @len: output for number of header bytes before payload
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Calculate the number of header bytes before the payload in an encrypted
+ * storage volume.  The header is an area before the payload that is reserved
+ * for encryption metadata.
+ *
+ * Returns: true on success, false on error
+ */
+bool
+qcrypto_block_calculate_payload_offset(QCryptoBlockCreateOptions *create_opts,
+                                       const char *optprefix,
+                                       size_t *len,
+                                       Error **errp);
+
+
 /**
  * qcrypto_block_get_info:
  * @block: the block encryption object
@@ -XXX,XX +XXX,XX @@ uint64_t qcrypto_block_get_sector_size(QCryptoBlock *block);
 void qcrypto_block_free(QCryptoBlock *block);
 
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(QCryptoBlock, qcrypto_block_free)
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(QCryptoBlockCreateOptions,
+                              qapi_free_QCryptoBlockCreateOptions)
 
 #endif /* QCRYPTO_BLOCK_H */
-- 
2.24.1

From: Stefan Hajnoczi <stefanha@redhat.com>

Add qemu-img measure support in the "luks" block driver.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200221112522.1497712-3-stefanha@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/crypto.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/block/crypto.c b/block/crypto.c
index XXXXXXX..XXXXXXX 100644
--- a/block/crypto.c
+++ b/block/crypto.c
@@ -XXX,XX +XXX,XX @@ static int64_t block_crypto_getlength(BlockDriverState *bs)
 }
 
 
+static BlockMeasureInfo *block_crypto_measure(QemuOpts *opts,
+                                              BlockDriverState *in_bs,
+                                              Error **errp)
+{
+    g_autoptr(QCryptoBlockCreateOptions) create_opts = NULL;
+    Error *local_err = NULL;
+    BlockMeasureInfo *info;
+    uint64_t size;
+    size_t luks_payload_size;
+    QDict *cryptoopts;
+
+    /*
+     * Preallocation mode doesn't affect size requirements but we must consume
+     * the option.
+     */
+    g_free(qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC));
+
+    size = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);
+
+    if (in_bs) {
+        int64_t ssize = bdrv_getlength(in_bs);
+
+        if (ssize < 0) {
+            error_setg_errno(&local_err, -ssize,
+                             "Unable to get image virtual_size");
+            goto err;
+        }
+
+        size = ssize;
+    }
+
+    cryptoopts = qemu_opts_to_qdict_filtered(opts, NULL,
+            &block_crypto_create_opts_luks, true);
+    qdict_put_str(cryptoopts, "format", "luks");
+    create_opts = block_crypto_create_opts_init(cryptoopts, &local_err);
+    qobject_unref(cryptoopts);
+    if (!create_opts) {
+        goto err;
+    }
+
+    if (!qcrypto_block_calculate_payload_offset(create_opts, NULL,
+                                                &luks_payload_size,
+                                                &local_err)) {
+        goto err;
+    }
+
+    /*
+     * Unallocated blocks are still encrypted so allocation status makes no
+     * difference to the file size.
+     */
+    info = g_new(BlockMeasureInfo, 1);
+    info->fully_allocated = luks_payload_size + size;
+    info->required = luks_payload_size + size;
+    return info;
+
+err:
+    error_propagate(errp, local_err);
+    return NULL;
+}
+
+
 static int block_crypto_probe_luks(const uint8_t *buf,
                                    int buf_size,
                                    const char *filename) {
@@ -XXX,XX +XXX,XX @@ static BlockDriver bdrv_crypto_luks = {
     .bdrv_co_preadv     = block_crypto_co_preadv,
     .bdrv_co_pwritev    = block_crypto_co_pwritev,
     .bdrv_getlength     = block_crypto_getlength,
+    .bdrv_measure       = block_crypto_measure,
     .bdrv_get_info      = block_crypto_get_info_luks,
     .bdrv_get_specific_info = block_crypto_get_specific_info_luks,
 
-- 
2.24.1

From: Stefan Hajnoczi <stefanha@redhat.com>

In most qemu-img sub-commands the --object option only makes sense when
there is a filename.  qemu-img measure is an exception because objects
may be referenced from the image creation options instead of an existing
image file.  Allow --object without a filename.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200221112522.1497712-4-stefanha@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 qemu-img.c                       | 6 ++----
 tests/qemu-iotests/178           | 2 +-
 tests/qemu-iotests/178.out.qcow2 | 8 ++++----
 tests/qemu-iotests/178.out.raw   | 8 ++++----
 4 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/qemu-img.c b/qemu-img.c
index XXXXXXX..XXXXXXX 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -XXX,XX +XXX,XX @@ static int img_measure(int argc, char **argv)
         filename = argv[optind];
     }
 
-    if (!filename &&
-        (object_opts || image_opts || fmt || snapshot_name || sn_opts)) {
-        error_report("--object, --image-opts, -f, and -l "
-                     "require a filename argument.");
+    if (!filename && (image_opts || fmt || snapshot_name || sn_opts)) {
+        error_report("--image-opts, -f, and -l require a filename argument.");
         goto out;
     }
     if (filename && img_size != UINT64_MAX) {
diff --git a/tests/qemu-iotests/178 b/tests/qemu-iotests/178
index XXXXXXX..XXXXXXX 100755
--- a/tests/qemu-iotests/178
+++ b/tests/qemu-iotests/178
@@ -XXX,XX +XXX,XX @@ _make_test_img 1G
 $QEMU_IMG measure # missing arguments
 $QEMU_IMG measure --size 2G "$TEST_IMG" # only one allowed
 $QEMU_IMG measure "$TEST_IMG" a # only one filename allowed
-$QEMU_IMG measure --object secret,id=sec0,data=MTIzNDU2,format=base64 # missing filename
+$QEMU_IMG measure --object secret,id=sec0,data=MTIzNDU2,format=base64 # size or filename needed
 $QEMU_IMG measure --image-opts # missing filename
 $QEMU_IMG measure -f qcow2 # missing filename
 $QEMU_IMG measure -l snap1 # missing filename
diff --git a/tests/qemu-iotests/178.out.qcow2 b/tests/qemu-iotests/178.out.qcow2
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/178.out.qcow2
+++ b/tests/qemu-iotests/178.out.qcow2
@@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 qemu-img: Either --size N or one filename must be specified.
 qemu-img: --size N cannot be used together with a filename.
 qemu-img: At most one filename argument is allowed.
-qemu-img: --object, --image-opts, -f, and -l require a filename argument.
-qemu-img: --object, --image-opts, -f, and -l require a filename argument.
-qemu-img: --object, --image-opts, -f, and -l require a filename argument.
-qemu-img: --object, --image-opts, -f, and -l require a filename argument.
+qemu-img: Either --size N or one filename must be specified.
+qemu-img: --image-opts, -f, and -l require a filename argument.
+qemu-img: --image-opts, -f, and -l require a filename argument.
+qemu-img: --image-opts, -f, and -l require a filename argument.
 qemu-img: Invalid option list: ,
 qemu-img: Invalid parameter 'snapshot.foo'
 qemu-img: Failed in parsing snapshot param 'snapshot.foo'
diff --git a/tests/qemu-iotests/178.out.raw b/tests/qemu-iotests/178.out.raw
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/178.out.raw
+++ b/tests/qemu-iotests/178.out.raw
@@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 qemu-img: Either --size N or one filename must be specified.
 qemu-img: --size N cannot be used together with a filename.
 qemu-img: At most one filename argument is allowed.
-qemu-img: --object, --image-opts, -f, and -l require a filename argument.
-qemu-img: --object, --image-opts, -f, and -l require a filename argument.
-qemu-img: --object, --image-opts, -f, and -l require a filename argument.
-qemu-img: --object, --image-opts, -f, and -l require a filename argument.
+qemu-img: Either --size N or one filename must be specified.
+qemu-img: --image-opts, -f, and -l require a filename argument.
+qemu-img: --image-opts, -f, and -l require a filename argument.
+qemu-img: --image-opts, -f, and -l require a filename argument.
 qemu-img: Invalid option list: ,
 qemu-img: Invalid parameter 'snapshot.foo'
 qemu-img: Failed in parsing snapshot param 'snapshot.foo'
-- 
2.24.1

From: Stefan Hajnoczi <stefanha@redhat.com>

This test exercises the block/crypto.c "luks" block driver
.bdrv_measure() code.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200221112522.1497712-5-stefanha@redhat.com>
[mreitz: Renamed test from 282 to 288]
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/288     | 93 ++++++++++++++++++++++++++++++++++++++
 tests/qemu-iotests/288.out | 30 ++++++++++++
 tests/qemu-iotests/group   |  1 +
 3 files changed, 124 insertions(+)
 create mode 100755 tests/qemu-iotests/288
 create mode 100644 tests/qemu-iotests/288.out

diff --git a/tests/qemu-iotests/288 b/tests/qemu-iotests/288
new file mode 100755
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/tests/qemu-iotests/288
@@ -XXX,XX +XXX,XX @@
+#!/usr/bin/env bash
+#
+# qemu-img measure tests for LUKS images
+#
+# Copyright (C) 2020 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=stefanha@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+status=1    # failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+    rm -f "$TEST_IMG.converted"
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.pattern
+
+_supported_fmt luks
+_supported_proto file
+_supported_os Linux
+
+SECRET=secret,id=sec0,data=passphrase
+
+echo "== measure 1G image file =="
+echo
+
+$QEMU_IMG measure --object "$SECRET" \
+	          -O "$IMGFMT" \
+		  -o key-secret=sec0,iter-time=10 \
+		  --size 1G
+
+echo
+echo "== create 1G image file (size should be no greater than measured) =="
+echo
+
+_make_test_img 1G
+stat -c "image file size in bytes: %s" "$TEST_IMG_FILE"
+
+echo
+echo "== modified 1G image file (size should be no greater than measured) =="
+echo
+
+$QEMU_IO --object "$SECRET" --image-opts "$TEST_IMG" -c "write -P 0x51 0x10000 0x400" | _filter_qemu_io | _filter_testdir
+stat -c "image file size in bytes: %s" "$TEST_IMG_FILE"
+
+echo
+echo "== measure preallocation=falloc 1G image file =="
+echo
+
+$QEMU_IMG measure --object "$SECRET" \
+	          -O "$IMGFMT" \
+		  -o key-secret=sec0,iter-time=10,preallocation=falloc \
+		  --size 1G
+
+echo
+echo "== measure with input image file =="
+echo
+
+IMGFMT=raw IMGKEYSECRET= IMGOPTS= _make_test_img 1G | _filter_imgfmt
+QEMU_IO_OPTIONS= IMGOPTSSYNTAX= $QEMU_IO -f raw -c "write -P 0x51 0x10000 0x400" "$TEST_IMG_FILE" | _filter_qemu_io | _filter_testdir
+$QEMU_IMG measure --object "$SECRET" \
+	          -O "$IMGFMT" \
+		  -o key-secret=sec0,iter-time=10 \
+		  -f raw \
+		  "$TEST_IMG_FILE"
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/288.out b/tests/qemu-iotests/288.out
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/tests/qemu-iotests/288.out
@@ -XXX,XX +XXX,XX @@
+QA output created by 288
+== measure 1G image file ==
+
+required size: 1075810304
+fully allocated size: 1075810304
+
+== create 1G image file (size should be no greater than measured) ==
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
+image file size in bytes: 1075810304
+
+== modified 1G image file (size should be no greater than measured) ==
+
+wrote 1024/1024 bytes at offset 65536
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+image file size in bytes: 1075810304
+
+== measure preallocation=falloc 1G image file ==
+
+required size: 1075810304
+fully allocated size: 1075810304
+
+== measure with input image file ==
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
+wrote 1024/1024 bytes at offset 65536
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+required size: 1075810304
+fully allocated size: 1075810304
+*** done
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -XXX,XX +XXX,XX @@
 283 auto quick
 284 rw
 286 rw quick
+288 quick
-- 
2.24.1

From: David Edmondson <david.edmondson@oracle.com>

RFC 7230 section 3.2 indicates that whitespace is permitted between
the field name and field value and after the field value.

Signed-off-by: David Edmondson <david.edmondson@oracle.com>
Message-Id: <20200224101310.101169-2-david.edmondson@oracle.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/curl.c | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/block/curl.c b/block/curl.c
index XXXXXXX..XXXXXXX 100644
--- a/block/curl.c
+++ b/block/curl.c
@@ -XXX,XX +XXX,XX @@ static size_t curl_header_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
 {
     BDRVCURLState *s = opaque;
     size_t realsize = size * nmemb;
-    const char *accept_line = "Accept-Ranges: bytes";
+    const char *header = (char *)ptr;
+    const char *end = header + realsize;
+    const char *accept_ranges = "Accept-Ranges:";
+    const char *bytes = "bytes";
 
-    if (realsize >= strlen(accept_line)
-        && strncmp((char *)ptr, accept_line, strlen(accept_line)) == 0) {
-        s->accept_range = true;
+    if (realsize >= strlen(accept_ranges)
+        && strncmp(header, accept_ranges, strlen(accept_ranges)) == 0) {
+
+        char *p = strchr(header, ':') + 1;
+
+        /* Skip whitespace between the header name and value. */
+        while (p < end && *p && g_ascii_isspace(*p)) {
+            p++;
+        }
+
+        if (end - p >= strlen(bytes)
+            && strncmp(p, bytes, strlen(bytes)) == 0) {
+
+            /* Check that there is nothing but whitespace after the value. */
+            p += strlen(bytes);
+            while (p < end && *p && g_ascii_isspace(*p)) {
+                p++;
+            }
+
+            if (p == end || !*p) {
+                s->accept_range = true;
+            }
+        }
     }
 
     return realsize;
-- 
2.24.1

From: David Edmondson <david.edmondson@oracle.com>

RFC 7230 section 3.2 indicates that HTTP header field names are case
insensitive.

Signed-off-by: David Edmondson <david.edmondson@oracle.com>
Message-Id: <20200224101310.101169-3-david.edmondson@oracle.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/curl.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/block/curl.c b/block/curl.c
index XXXXXXX..XXXXXXX 100644
--- a/block/curl.c
+++ b/block/curl.c
@@ -XXX,XX +XXX,XX @@ static size_t curl_header_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
     size_t realsize = size * nmemb;
     const char *header = (char *)ptr;
     const char *end = header + realsize;
-    const char *accept_ranges = "Accept-Ranges:";
+    const char *accept_ranges = "accept-ranges:";
     const char *bytes = "bytes";
 
     if (realsize >= strlen(accept_ranges)
-        && strncmp(header, accept_ranges, strlen(accept_ranges)) == 0) {
+        && g_ascii_strncasecmp(header, accept_ranges,
+                               strlen(accept_ranges)) == 0) {
 
         char *p = strchr(header, ':') + 1;
 
-- 
2.24.1

From: Eric Blake <eblake@redhat.com>

Tests 261 and 272 fail on RHEL 7 with coreutils 8.22, since od
--endian was not added until coreutils 8.23.  Fix this by manually
constructing the final value one byte at a time.

Fixes: fc8ba423
Reported-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
Signed-off-by: Eric Blake <eblake@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200226125424.481840-1-eblake@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 tests/qemu-iotests/common.rc | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc
index XXXXXXX..XXXXXXX 100644
--- a/tests/qemu-iotests/common.rc
+++ b/tests/qemu-iotests/common.rc
@@ -XXX,XX +XXX,XX @@ poke_file()
 # peek_file_le 'test.img' 512 2 => 65534
 peek_file_le()
 {
-    # Wrap in echo $() to strip spaces
-    echo $(od -j"$2" -N"$3" --endian=little -An -vtu"$3" "$1")
+    local val=0 shift=0 byte
+
+    # coreutils' od --endian is not portable, so manually assemble bytes.
+    for byte in $(od -j"$2" -N"$3" -An -v -tu1 "$1"); do
+        val=$(( val | (byte << shift) ))
+        shift=$((shift + 8))
+    done
+    printf %llu $val
 }
 
 # peek_file_be 'test.img' 512 2 => 65279
 peek_file_be()
 {
-    # Wrap in echo $() to strip spaces
-    echo $(od -j"$2" -N"$3" --endian=big -An -vtu"$3" "$1")
+    local val=0 byte
+
+    # coreutils' od --endian is not portable, so manually assemble bytes.
+    for byte in $(od -j"$2" -N"$3" -An -v -tu1 "$1"); do
+        val=$(( (val << 8) | byte ))
+    done
+    printf %llu $val
 }
 
-# peek_file_raw 'test.img' 512 2 => '\xff\xfe'
+# peek_file_raw 'test.img' 512 2 => '\xff\xfe'. Do not use if the raw data
+# is likely to contain \0 or trailing \n.
 peek_file_raw()
 {
     dd if="$1" bs=1 skip="$2" count="$3" status=none
-- 
2.24.1

From: Pan Nengyuan <pannengyuan@huawei.com>

'crypto_opts' forgot to free in qcow2_close(), this patch fix the bellow leak stack:

Direct leak of 24 byte(s) in 1 object(s) allocated from:
    #0 0x7f0edd81f970 in __interceptor_calloc (/lib64/libasan.so.5+0xef970)
    #1 0x7f0edc6d149d in g_malloc0 (/lib64/libglib-2.0.so.0+0x5249d)
    #2 0x55d7eaede63d in qobject_input_start_struct /mnt/sdb/qemu-new/qemu_test/qemu/qapi/qobject-input-visitor.c:295
    #3 0x55d7eaed78b8 in visit_start_struct /mnt/sdb/qemu-new/qemu_test/qemu/qapi/qapi-visit-core.c:49
    #4 0x55d7eaf5140b in visit_type_QCryptoBlockOpenOptions qapi/qapi-visit-crypto.c:290
    #5 0x55d7eae43af3 in block_crypto_open_opts_init /mnt/sdb/qemu-new/qemu_test/qemu/block/crypto.c:163
    #6 0x55d7eacd2924 in qcow2_update_options_prepare /mnt/sdb/qemu-new/qemu_test/qemu/block/qcow2.c:1148
    #7 0x55d7eacd33f7 in qcow2_update_options /mnt/sdb/qemu-new/qemu_test/qemu/block/qcow2.c:1232
    #8 0x55d7eacd9680 in qcow2_do_open /mnt/sdb/qemu-new/qemu_test/qemu/block/qcow2.c:1512
    #9 0x55d7eacdc55e in qcow2_open_entry /mnt/sdb/qemu-new/qemu_test/qemu/block/qcow2.c:1792
    #10 0x55d7eacdc8fe in qcow2_open /mnt/sdb/qemu-new/qemu_test/qemu/block/qcow2.c:1819
    #11 0x55d7eac3742d in bdrv_open_driver /mnt/sdb/qemu-new/qemu_test/qemu/block.c:1317
    #12 0x55d7eac3e990 in bdrv_open_common /mnt/sdb/qemu-new/qemu_test/qemu/block.c:1575
    #13 0x55d7eac4442c in bdrv_open_inherit /mnt/sdb/qemu-new/qemu_test/qemu/block.c:3126
    #14 0x55d7eac45c3f in bdrv_open /mnt/sdb/qemu-new/qemu_test/qemu/block.c:3219
    #15 0x55d7ead8e8a4 in blk_new_open /mnt/sdb/qemu-new/qemu_test/qemu/block/block-backend.c:397
    #16 0x55d7eacde74c in qcow2_co_create /mnt/sdb/qemu-new/qemu_test/qemu/block/qcow2.c:3534
    #17 0x55d7eacdfa6d in qcow2_co_create_opts /mnt/sdb/qemu-new/qemu_test/qemu/block/qcow2.c:3668
    #18 0x55d7eac1c678 in bdrv_create_co_entry /mnt/sdb/qemu-new/qemu_test/qemu/block.c:485
    #19 0x55d7eb0024d2 in coroutine_trampoline /mnt/sdb/qemu-new/qemu_test/qemu/util/coroutine-ucontext.c:115

Reported-by: Euler Robot <euler.robot@huawei.com>
Signed-off-by: Pan Nengyuan <pannengyuan@huawei.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200227012950.12256-2-pannengyuan@huawei.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/qcow2.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/block/qcow2.c b/block/qcow2.c
index XXXXXXX..XXXXXXX 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -XXX,XX +XXX,XX @@ static void qcow2_close(BlockDriverState *bs)
 
     qcrypto_block_free(s->crypto);
     s->crypto = NULL;
+    qapi_free_QCryptoBlockOpenOptions(s->crypto_opts);
 
     g_free(s->unknown_header_fields);
     cleanup_unknown_header_ext(bs);
-- 
2.24.1

From: Pan Nengyuan <pannengyuan@huawei.com>

collect_image_check() is called twice in img_check(), the filename/format will be alloced without free the original memory.
It is not a big deal since the process will exit anyway, but seems like a clean code and it will remove the warning spotted by asan.

Reported-by: Euler Robot <euler.robot@huawei.com>
Signed-off-by: Pan Nengyuan <pannengyuan@huawei.com>
Message-Id: <20200227012950.12256-3-pannengyuan@huawei.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 qemu-img.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/qemu-img.c b/qemu-img.c
index XXXXXXX..XXXXXXX 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -XXX,XX +XXX,XX @@ static int img_check(int argc, char **argv)
                     check->corruptions_fixed);
         }
 
+        qapi_free_ImageCheck(check);
+        check = g_new0(ImageCheck, 1);
         ret = collect_image_check(bs, check, filename, fmt, 0);
 
         check->leaks_fixed          = leaks_fixed;
-- 
2.24.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

On success path we return what inflate() returns instead of 0. And it
most probably works for Z_STREAM_END as it is positive, but is
definitely broken for Z_BUF_ERROR.

While being here, switch to errno return code, to be closer to
qcow2_compress API (and usual expectations).

Revert condition in if to be more positive. Drop dead initialization of
ret.

Cc: qemu-stable@nongnu.org # v4.0
Fixes: 341926ab83e2b
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Message-Id: <20200302150930.16218-1-vsementsov@virtuozzo.com>
Reviewed-by: Alberto Garcia <berto@igalia.com>
Reviewed-by: Ján Tomko <jtomko@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/qcow2-threads.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/block/qcow2-threads.c b/block/qcow2-threads.c
index XXXXXXX..XXXXXXX 100644
--- a/block/qcow2-threads.c
+++ b/block/qcow2-threads.c
@@ -XXX,XX +XXX,XX @@ static ssize_t qcow2_compress(void *dest, size_t dest_size,
  * @src - source buffer, @src_size bytes
  *
  * Returns: 0 on success
- *          -1 on fail
+ *          -EIO on fail
  */
 static ssize_t qcow2_decompress(void *dest, size_t dest_size,
                                 const void *src, size_t src_size)
 {
-    int ret = 0;
+    int ret;
     z_stream strm;
 
     memset(&strm, 0, sizeof(strm));
@@ -XXX,XX +XXX,XX @@ static ssize_t qcow2_decompress(void *dest, size_t dest_size,
 
     ret = inflateInit2(&strm, -12);
     if (ret != Z_OK) {
-        return -1;
+        return -EIO;
     }
 
     ret = inflate(&strm, Z_FINISH);
-    if ((ret != Z_STREAM_END && ret != Z_BUF_ERROR) || strm.avail_out != 0) {
+    if ((ret == Z_STREAM_END || ret == Z_BUF_ERROR) && strm.avail_out == 0) {
         /*
          * We approve Z_BUF_ERROR because we need @dest buffer to be filled, but
          * @src buffer may be processed partly (because in qcow2 we know size of
          * compressed data with precision of one sector)
          */
-        ret = -1;
+        ret = 0;
+    } else {
+        ret = -EIO;
     }
 
     inflateEnd(&strm);
-- 
2.24.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

We need it in separate to pass to the block-copy object in the next
commit.

Cc: qemu-stable@nongnu.org
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200311103004.7649-2-vsementsov@virtuozzo.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 blockjob.c                    | 16 +++++-----
 include/qemu/job.h            | 11 ++-----
 include/qemu/progress_meter.h | 58 +++++++++++++++++++++++++++++++++++
 job-qmp.c                     |  4 +--
 job.c                         |  6 ++--
 qemu-img.c                    |  6 ++--
 6 files changed, 76 insertions(+), 25 deletions(-)
 create mode 100644 include/qemu/progress_meter.h

diff --git a/blockjob.c b/blockjob.c
index XXXXXXX..XXXXXXX 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -XXX,XX +XXX,XX @@ BlockJobInfo *block_job_query(BlockJob *job, Error **errp)
     info->device    = g_strdup(job->job.id);
     info->busy      = atomic_read(&job->job.busy);
     info->paused    = job->job.pause_count > 0;
-    info->offset    = job->job.progress_current;
-    info->len       = job->job.progress_total;
+    info->offset    = job->job.progress.current;
+    info->len       = job->job.progress.total;
     info->speed     = job->speed;
     info->io_status = job->iostatus;
     info->ready     = job_is_ready(&job->job),
@@ -XXX,XX +XXX,XX @@ static void block_job_event_cancelled(Notifier *n, void *opaque)
 
     qapi_event_send_block_job_cancelled(job_type(&job->job),
                                         job->job.id,
-                                        job->job.progress_total,
-                                        job->job.progress_current,
+                                        job->job.progress.total,
+                                        job->job.progress.current,
                                         job->speed);
 }
 
@@ -XXX,XX +XXX,XX @@ static void block_job_event_completed(Notifier *n, void *opaque)
 
     qapi_event_send_block_job_completed(job_type(&job->job),
                                         job->job.id,
-                                        job->job.progress_total,
-                                        job->job.progress_current,
+                                        job->job.progress.total,
+                                        job->job.progress.current,
                                         job->speed,
                                         !!msg,
                                         msg);
@@ -XXX,XX +XXX,XX @@ static void block_job_event_ready(Notifier *n, void *opaque)
 
     qapi_event_send_block_job_ready(job_type(&job->job),
                                     job->job.id,
-                                    job->job.progress_total,
-                                    job->job.progress_current,
+                                    job->job.progress.total,
+                                    job->job.progress.current,
                                     job->speed);
 }
 
diff --git a/include/qemu/job.h b/include/qemu/job.h
index XXXXXXX..XXXXXXX 100644
--- a/include/qemu/job.h
+++ b/include/qemu/job.h
@@ -XXX,XX +XXX,XX @@
 
 #include "qapi/qapi-types-job.h"
 #include "qemu/queue.h"
+#include "qemu/progress_meter.h"
 #include "qemu/coroutine.h"
 #include "block/aio.h"
 
@@ -XXX,XX +XXX,XX @@ typedef struct Job {
     /** True if this job should automatically dismiss itself */
     bool auto_dismiss;
 
-    /**
-     * Current progress. The unit is arbitrary as long as the ratio between
-     * progress_current and progress_total represents the estimated percentage
-     * of work already done.
-     */
-    int64_t progress_current;
-
-    /** Estimated progress_current value at the completion of the job */
-    int64_t progress_total;
+    ProgressMeter progress;
 
     /**
      * Return code from @run and/or @prepare callback(s).
diff --git a/include/qemu/progress_meter.h b/include/qemu/progress_meter.h
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/include/qemu/progress_meter.h
@@ -XXX,XX +XXX,XX @@
+/*
+ * Helper functionality for some process progress tracking.
+ *
+ * Copyright (c) 2011 IBM Corp.
+ * Copyright (c) 2012, 2018 Red Hat, Inc.
+ * Copyright (c) 2020 Virtuozzo International GmbH
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef QEMU_PROGRESS_METER_H
+#define QEMU_PROGRESS_METER_H
+
+typedef struct ProgressMeter {
+    /**
+     * Current progress. The unit is arbitrary as long as the ratio between
+     * current and total represents the estimated percentage
+     * of work already done.
+     */
+    uint64_t current;
+
+    /** Estimated current value at the completion of the process */
+    uint64_t total;
+} ProgressMeter;
+
+static inline void progress_work_done(ProgressMeter *pm, uint64_t done)
+{
+    pm->current += done;
+}
+
+static inline void progress_set_remaining(ProgressMeter *pm, uint64_t remaining)
+{
+    pm->total = pm->current + remaining;
+}
+
+static inline void progress_increase_remaining(ProgressMeter *pm,
+                                               uint64_t delta)
+{
+    pm->total += delta;
+}
+
+#endif /* QEMU_PROGRESS_METER_H */
diff --git a/job-qmp.c b/job-qmp.c
index XXXXXXX..XXXXXXX 100644
--- a/job-qmp.c
+++ b/job-qmp.c
@@ -XXX,XX +XXX,XX @@ static JobInfo *job_query_single(Job *job, Error **errp)
         .id                 = g_strdup(job->id),
         .type               = job_type(job),
         .status             = job->status,
-        .current_progress   = job->progress_current,
-        .total_progress     = job->progress_total,
+        .current_progress   = job->progress.current,
+        .total_progress     = job->progress.total,
         .has_error          = !!job->err,
         .error              = job->err ? \
                               g_strdup(error_get_pretty(job->err)) : NULL,
diff --git a/job.c b/job.c
index XXXXXXX..XXXXXXX 100644
--- a/job.c
+++ b/job.c
@@ -XXX,XX +XXX,XX @@ void job_unref(Job *job)
 
 void job_progress_update(Job *job, uint64_t done)
 {
-    job->progress_current += done;
+    progress_work_done(&job->progress, done);
 }
 
 void job_progress_set_remaining(Job *job, uint64_t remaining)
 {
-    job->progress_total = job->progress_current + remaining;
+    progress_set_remaining(&job->progress, remaining);
 }
 
 void job_progress_increase_remaining(Job *job, uint64_t delta)
 {
-    job->progress_total += delta;
+    progress_increase_remaining(&job->progress, delta);
 }
 
 void job_event_cancelled(Job *job)
diff --git a/qemu-img.c b/qemu-img.c
index XXXXXXX..XXXXXXX 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -XXX,XX +XXX,XX @@ static void run_block_job(BlockJob *job, Error **errp)
     do {
         float progress = 0.0f;
         aio_poll(aio_context, true);
-        if (job->job.progress_total) {
-            progress = (float)job->job.progress_current /
-                       job->job.progress_total * 100.f;
+        if (job->job.progress.total) {
+            progress = (float)job->job.progress.current /
+                       job->job.progress.total * 100.f;
         }
         qemu_progress_print(progress, 0);
     } while (!job_is_ready(&job->job) && !job_is_completed(&job->job));
-- 
2.24.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Assume we have two regions, A and B, and region B is in-flight now,
region A is not yet touched, but it is unallocated and should be
skipped.

Correspondingly, as progress we have

total = A + B
  current = 0

If we reset unallocated region A and call progress_reset_callback,
it will calculate 0 bytes dirty in the bitmap and call
job_progress_set_remaining, which will set

total = current + 0 = 0 + 0 = 0

So, B bytes are actually removed from total accounting. When job
finishes we'll have

total = 0
   current = B

, which doesn't sound good.

This is because we didn't considered in-flight bytes, actually when
calculating remaining, we should have set (in_flight + dirty_bytes)
as remaining, not only dirty_bytes.

To fix it, let's refactor progress calculation, moving it to block-copy
itself instead of fixing callback. And, of course, track in_flight
bytes count.

We still have to keep one callback, to maintain backup job bytes_read
calculation, but it will go on soon, when we turn the whole backup
process into one block_copy call.

Cc: qemu-stable@nongnu.org
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
Message-Id: <20200311103004.7649-3-vsementsov@virtuozzo.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/backup.c             | 13 ++-----------
 block/block-copy.c         | 16 ++++++++++++----
 include/block/block-copy.h | 15 +++++----------
 3 files changed, 19 insertions(+), 25 deletions(-)

diff --git a/block/backup.c b/block/backup.c
index XXXXXXX..XXXXXXX 100644
--- a/block/backup.c
+++ b/block/backup.c
@@ -XXX,XX +XXX,XX @@ static void backup_progress_bytes_callback(int64_t bytes, void *opaque)
     BackupBlockJob *s = opaque;
 
     s->bytes_read += bytes;
-    job_progress_update(&s->common.job, bytes);
-}
-
-static void backup_progress_reset_callback(void *opaque)
-{
-    BackupBlockJob *s = opaque;
-    uint64_t estimate = bdrv_get_dirty_count(s->bcs->copy_bitmap);
-
-    job_progress_set_remaining(&s->common.job, estimate);
 }
 
 static int coroutine_fn backup_do_cow(BackupBlockJob *job,
@@ -XXX,XX +XXX,XX @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs,
     job->cluster_size = cluster_size;
     job->len = len;
 
-    block_copy_set_callbacks(bcs, backup_progress_bytes_callback,
-                             backup_progress_reset_callback, job);
+    block_copy_set_progress_callback(bcs, backup_progress_bytes_callback, job);
+    block_copy_set_progress_meter(bcs, &job->common.job.progress);
 
     /* Required permissions are already taken by backup-top target */
     block_job_add_bdrv(&job->common, "target", target, 0, BLK_PERM_ALL,
diff --git a/block/block-copy.c b/block/block-copy.c
index XXXXXXX..XXXXXXX 100644
--- a/block/block-copy.c
+++ b/block/block-copy.c
@@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
     return s;
 }
 
-void block_copy_set_callbacks(
+void block_copy_set_progress_callback(
         BlockCopyState *s,
         ProgressBytesCallbackFunc progress_bytes_callback,
-        ProgressResetCallbackFunc progress_reset_callback,
         void *progress_opaque)
 {
     s->progress_bytes_callback = progress_bytes_callback;
-    s->progress_reset_callback = progress_reset_callback;
     s->progress_opaque = progress_opaque;
 }
 
+void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm)
+{
+    s->progress = pm;
+}
+
 /*
  * block_copy_do_copy
  *
@@ -XXX,XX +XXX,XX @@ int64_t block_copy_reset_unallocated(BlockCopyState *s,
 
     if (!ret) {
         bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
-        s->progress_reset_callback(s->progress_opaque);
+        progress_set_remaining(s->progress,
+                               bdrv_get_dirty_count(s->copy_bitmap) +
+                               s->in_flight_bytes);
     }
 
     *count = bytes;
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
         trace_block_copy_process(s, start);
 
         bdrv_reset_dirty_bitmap(s->copy_bitmap, start, chunk_end - start);
+        s->in_flight_bytes += chunk_end - start;
 
         co_get_from_shres(s->mem, chunk_end - start);
         ret = block_copy_do_copy(s, start, chunk_end, error_is_read);
         co_put_to_shres(s->mem, chunk_end - start);
+        s->in_flight_bytes -= chunk_end - start;
         if (ret < 0) {
             bdrv_set_dirty_bitmap(s->copy_bitmap, start, chunk_end - start);
             break;
         }
 
+        progress_work_done(s->progress, chunk_end - start);
         s->progress_bytes_callback(chunk_end - start, s->progress_opaque);
         start = chunk_end;
         ret = 0;
diff --git a/include/block/block-copy.h b/include/block/block-copy.h
index XXXXXXX..XXXXXXX 100644
--- a/include/block/block-copy.h
+++ b/include/block/block-copy.h
@@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyInFlightReq {
 } BlockCopyInFlightReq;
 
 typedef void (*ProgressBytesCallbackFunc)(int64_t bytes, void *opaque);
-typedef void (*ProgressResetCallbackFunc)(void *opaque);
 typedef struct BlockCopyState {
     /*
      * BdrvChild objects are not owned or managed by block-copy. They are
@@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyState {
     BdrvChild *source;
     BdrvChild *target;
     BdrvDirtyBitmap *copy_bitmap;
+    int64_t in_flight_bytes;
     int64_t cluster_size;
     bool use_copy_range;
     int64_t copy_size;
@@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyState {
      */
     bool skip_unallocated;
 
+    ProgressMeter *progress;
     /* progress_bytes_callback: called when some copying progress is done. */
     ProgressBytesCallbackFunc progress_bytes_callback;
-
-    /*
-     * progress_reset_callback: called when some bytes reset from copy_bitmap
-     * (see @skip_unallocated above). The callee is assumed to recalculate how
-     * many bytes remain based on the dirty bit count of copy_bitmap.
-     */
-    ProgressResetCallbackFunc progress_reset_callback;
     void *progress_opaque;
 
     SharedResource *mem;
@@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
                                      BdrvRequestFlags write_flags,
                                      Error **errp);
 
-void block_copy_set_callbacks(
+void block_copy_set_progress_callback(
         BlockCopyState *s,
         ProgressBytesCallbackFunc progress_bytes_callback,
-        ProgressResetCallbackFunc progress_reset_callback,
         void *progress_opaque);
 
+void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm);
+
 void block_copy_state_free(BlockCopyState *s);
 
 int64_t block_copy_reset_unallocated(BlockCopyState *s,
-- 
2.24.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

In block_copy_do_copy we fallback to read+write if copy_range failed.
In this case copy_size is larger than defined for buffered IO, and
there is corresponding commit. Still, backup copies data cluster by
cluster, and most of requests are limited to one cluster anyway, so the
only source of this one bad-limited request is copy-before-write
operation.

Further patch will move backup to use block_copy directly, than for
cases where copy_range is not supported, first request will be
oversized in each backup. It's not good, let's change it now.

Fix is simple: just limit first copy_range request like buffer-based
request. If it succeed, set larger copy_range limit.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200311103004.7649-4-vsementsov@virtuozzo.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/block-copy.c | 41 +++++++++++++++++++++++++++++++----------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/block/block-copy.c b/block/block-copy.c
index XXXXXXX..XXXXXXX 100644
--- a/block/block-copy.c
+++ b/block/block-copy.c
@@ -XXX,XX +XXX,XX @@ void block_copy_state_free(BlockCopyState *s)
     g_free(s);
 }
 
+static uint32_t block_copy_max_transfer(BdrvChild *source, BdrvChild *target)
+{
+    return MIN_NON_ZERO(INT_MAX,
+                        MIN_NON_ZERO(source->bs->bl.max_transfer,
+                                     target->bs->bl.max_transfer));
+}
+
 BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
                                      int64_t cluster_size,
                                      BdrvRequestFlags write_flags, Error **errp)
 {
     BlockCopyState *s;
     BdrvDirtyBitmap *copy_bitmap;
-    uint32_t max_transfer =
-            MIN_NON_ZERO(INT_MAX,
-                         MIN_NON_ZERO(source->bs->bl.max_transfer,
-                                      target->bs->bl.max_transfer));
 
     copy_bitmap = bdrv_create_dirty_bitmap(source->bs, cluster_size, NULL,
                                            errp);
@@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
         .mem = shres_create(BLOCK_COPY_MAX_MEM),
     };
 
-    if (max_transfer < cluster_size) {
+    if (block_copy_max_transfer(source, target) < cluster_size) {
         /*
          * copy_range does not respect max_transfer. We don't want to bother
          * with requests smaller than block-copy cluster size, so fallback to
@@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
         s->copy_size = cluster_size;
     } else {
         /*
-         * copy_range does not respect max_transfer (it's a TODO), so we factor
-         * that in here.
+         * We enable copy-range, but keep small copy_size, until first
+         * successful copy_range (look at block_copy_do_copy).
          */
         s->use_copy_range = true;
-        s->copy_size = MIN(MAX(cluster_size, BLOCK_COPY_MAX_COPY_RANGE),
-                           QEMU_ALIGN_DOWN(max_transfer, cluster_size));
+        s->copy_size = MAX(s->cluster_size, BLOCK_COPY_MAX_BUFFER);
     }
 
     QLIST_INIT(&s->inflight_reqs);
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
             s->copy_size = MAX(s->cluster_size, BLOCK_COPY_MAX_BUFFER);
             /* Fallback to read+write with allocated buffer */
         } else {
+            if (s->use_copy_range) {
+                /*
+                 * Successful copy-range. Now increase copy_size.  copy_range
+                 * does not respect max_transfer (it's a TODO), so we factor
+                 * that in here.
+                 *
+                 * Note: we double-check s->use_copy_range for the case when
+                 * parallel block-copy request unsets it during previous
+                 * bdrv_co_copy_range call.
+                 */
+                s->copy_size =
+                        MIN(MAX(s->cluster_size, BLOCK_COPY_MAX_COPY_RANGE),
+                            QEMU_ALIGN_DOWN(block_copy_max_transfer(s->source,
+                                                                    s->target),
+                                            s->cluster_size));
+            }
             goto out;
         }
     }
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
     /*
      * In case of failed copy_range request above, we may proceed with buffered
      * request larger than BLOCK_COPY_MAX_BUFFER. Still, further requests will
-     * be properly limited, so don't care too much.
+     * be properly limited, so don't care too much. Moreover the most likely
+     * case (copy_range is unsupported for the configuration, so the very first
+     * copy_range request fails) is handled by setting large copy_size only
+     * after first successful copy_range.
      */
 
     bounce_buffer = qemu_blockalign(s->source->bs, nbytes);
-- 
2.24.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Use bdrv_block_status_above to chose effective chunk size and to handle
zeroes effectively.

This substitutes checking for just being allocated or not, and drops
old code path for it. Assistance by backup job is dropped too, as
caching block-status information is more difficult than just caching
is-allocated information in our dirty bitmap, and backup job is not
good place for this caching anyway.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200311103004.7649-5-vsementsov@virtuozzo.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/block-copy.c | 73 +++++++++++++++++++++++++++++++++++++---------
 block/trace-events |  1 +
 2 files changed, 61 insertions(+), 13 deletions(-)

diff --git a/block/block-copy.c b/block/block-copy.c
index XXXXXXX..XXXXXXX 100644
--- a/block/block-copy.c
+++ b/block/block-copy.c
@@ -XXX,XX +XXX,XX @@ void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm)
  */
 static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
                                            int64_t start, int64_t end,
-                                           bool *error_is_read)
+                                           bool zeroes, bool *error_is_read)
 {
     int ret;
     int nbytes = MIN(end, s->len) - start;
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
     assert(QEMU_IS_ALIGNED(end, s->cluster_size));
     assert(end < s->len || end == QEMU_ALIGN_UP(s->len, s->cluster_size));
 
+    if (zeroes) {
+        ret = bdrv_co_pwrite_zeroes(s->target, start, nbytes, s->write_flags &
+                                    ~BDRV_REQ_WRITE_COMPRESSED);
+        if (ret < 0) {
+            trace_block_copy_write_zeroes_fail(s, start, ret);
+            if (error_is_read) {
+                *error_is_read = false;
+            }
+        }
+        return ret;
+    }
+
     if (s->use_copy_range) {
         ret = bdrv_co_copy_range(s->source, start, s->target, start, nbytes,
                                  0, s->write_flags);
@@ -XXX,XX +XXX,XX @@ out:
     return ret;
 }
 
+static int block_copy_block_status(BlockCopyState *s, int64_t offset,
+                                   int64_t bytes, int64_t *pnum)
+{
+    int64_t num;
+    BlockDriverState *base;
+    int ret;
+
+    if (s->skip_unallocated && s->source->bs->backing) {
+        base = s->source->bs->backing->bs;
+    } else {
+        base = NULL;
+    }
+
+    ret = bdrv_block_status_above(s->source->bs, base, offset, bytes, &num,
+                                  NULL, NULL);
+    if (ret < 0 || num < s->cluster_size) {
+        /*
+         * On error or if failed to obtain large enough chunk just fallback to
+         * copy one cluster.
+         */
+        num = s->cluster_size;
+        ret = BDRV_BLOCK_ALLOCATED | BDRV_BLOCK_DATA;
+    } else if (offset + num == s->len) {
+        num = QEMU_ALIGN_UP(num, s->cluster_size);
+    } else {
+        num = QEMU_ALIGN_DOWN(num, s->cluster_size);
+    }
+
+    *pnum = num;
+    return ret;
+}
+
 /*
  * Check if the cluster starting at offset is allocated or not.
  * return via pnum the number of contiguous clusters sharing this allocation.
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
 {
     int ret = 0;
     int64_t end = bytes + start; /* bytes */
-    int64_t status_bytes;
     BlockCopyInFlightReq req;
 
     /*
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
     block_copy_inflight_req_begin(s, &req, start, end);
 
     while (start < end) {
-        int64_t next_zero, chunk_end;
+        int64_t next_zero, chunk_end, status_bytes;
 
         if (!bdrv_dirty_bitmap_get(s->copy_bitmap, start)) {
             trace_block_copy_skip(s, start);
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
             chunk_end = next_zero;
         }
 
-        if (s->skip_unallocated) {
-            ret = block_copy_reset_unallocated(s, start, &status_bytes);
-            if (ret == 0) {
-                trace_block_copy_skip_range(s, start, status_bytes);
-                start += status_bytes;
-                continue;
-            }
-            /* Clamp to known allocated region */
-            chunk_end = MIN(chunk_end, start + status_bytes);
+        ret = block_copy_block_status(s, start, chunk_end - start,
+                                      &status_bytes);
+        if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
+            bdrv_reset_dirty_bitmap(s->copy_bitmap, start, status_bytes);
+            progress_set_remaining(s->progress,
+                                   bdrv_get_dirty_count(s->copy_bitmap) +
+                                   s->in_flight_bytes);
+            trace_block_copy_skip_range(s, start, status_bytes);
+            start += status_bytes;
+            continue;
         }
 
+        chunk_end = MIN(chunk_end, start + status_bytes);
+
         trace_block_copy_process(s, start);
 
         bdrv_reset_dirty_bitmap(s->copy_bitmap, start, chunk_end - start);
         s->in_flight_bytes += chunk_end - start;
 
         co_get_from_shres(s->mem, chunk_end - start);
-        ret = block_copy_do_copy(s, start, chunk_end, error_is_read);
+        ret = block_copy_do_copy(s, start, chunk_end, ret & BDRV_BLOCK_ZERO,
+                                 error_is_read);
         co_put_to_shres(s->mem, chunk_end - start);
         s->in_flight_bytes -= chunk_end - start;
         if (ret < 0) {
diff --git a/block/trace-events b/block/trace-events
index XXXXXXX..XXXXXXX 100644
--- a/block/trace-events
+++ b/block/trace-events
@@ -XXX,XX +XXX,XX @@ block_copy_process(void *bcs, int64_t start) "bcs %p start %"PRId64
 block_copy_copy_range_fail(void *bcs, int64_t start, int ret) "bcs %p start %"PRId64" ret %d"
 block_copy_read_fail(void *bcs, int64_t start, int ret) "bcs %p start %"PRId64" ret %d"
 block_copy_write_fail(void *bcs, int64_t start, int ret) "bcs %p start %"PRId64" ret %d"
+block_copy_write_zeroes_fail(void *bcs, int64_t start, int ret) "bcs %p start %"PRId64" ret %d"
 
 # ../blockdev.c
 qmp_block_job_cancel(void *job) "job %p"
-- 
2.24.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Split find_conflicting_inflight_req to be used separately.

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

We have a lot of "chunk_end - start" invocations, let's switch to
bytes/cur_bytes scheme instead.

While being here, improve check on block_copy_do_copy parameters to not
overflow when calculating nbytes and use int64_t for bytes in
block_copy for consistency.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200311103004.7649-7-vsementsov@virtuozzo.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/block-copy.c         | 78 ++++++++++++++++++++------------------
 include/block/block-copy.h |  6 +--
 2 files changed, 44 insertions(+), 40 deletions(-)

diff --git a/block/block-copy.c b/block/block-copy.c
index XXXXXXX..XXXXXXX 100644
--- a/block/block-copy.c
+++ b/block/block-copy.c
@@ -XXX,XX +XXX,XX @@
 
 static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
                                                            int64_t start,
-                                                           int64_t end)
+                                                           int64_t bytes)
 {
     BlockCopyInFlightReq *req;
 
     QLIST_FOREACH(req, &s->inflight_reqs, list) {
-        if (end > req->start_byte && start < req->end_byte) {
+        if (start + bytes > req->start && start < req->start + req->bytes) {
             return req;
         }
     }
@@ -XXX,XX +XXX,XX @@ static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
 
 static void coroutine_fn block_copy_wait_inflight_reqs(BlockCopyState *s,
                                                        int64_t start,
-                                                       int64_t end)
+                                                       int64_t bytes)
 {
     BlockCopyInFlightReq *req;
 
-    while ((req = find_conflicting_inflight_req(s, start, end))) {
+    while ((req = find_conflicting_inflight_req(s, start, bytes))) {
         qemu_co_queue_wait(&req->wait_queue, NULL);
     }
 }
 
 static void block_copy_inflight_req_begin(BlockCopyState *s,
                                           BlockCopyInFlightReq *req,
-                                          int64_t start, int64_t end)
+                                          int64_t start, int64_t bytes)
 {
-    req->start_byte = start;
-    req->end_byte = end;
+    req->start = start;
+    req->bytes = bytes;
     qemu_co_queue_init(&req->wait_queue);
     QLIST_INSERT_HEAD(&s->inflight_reqs, req, list);
 }
@@ -XXX,XX +XXX,XX @@ void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm)
 /*
  * block_copy_do_copy
  *
- * Do copy of cluser-aligned chunk. @end is allowed to exceed s->len only to
- * cover last cluster when s->len is not aligned to clusters.
+ * Do copy of cluster-aligned chunk. Requested region is allowed to exceed
+ * s->len only to cover last cluster when s->len is not aligned to clusters.
  *
  * No sync here: nor bitmap neighter intersecting requests handling, only copy.
  *
  * Returns 0 on success.
  */
 static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
-                                           int64_t start, int64_t end,
+                                           int64_t start, int64_t bytes,
                                            bool zeroes, bool *error_is_read)
 {
     int ret;
-    int nbytes = MIN(end, s->len) - start;
+    int64_t nbytes = MIN(start + bytes, s->len) - start;
     void *bounce_buffer = NULL;
 
+    assert(start >= 0 && bytes > 0 && INT64_MAX - start >= bytes);
     assert(QEMU_IS_ALIGNED(start, s->cluster_size));
-    assert(QEMU_IS_ALIGNED(end, s->cluster_size));
-    assert(end < s->len || end == QEMU_ALIGN_UP(s->len, s->cluster_size));
+    assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
+    assert(start < s->len);
+    assert(start + bytes <= s->len ||
+           start + bytes == QEMU_ALIGN_UP(s->len, s->cluster_size));
+    assert(nbytes < INT_MAX);
 
     if (zeroes) {
         ret = bdrv_co_pwrite_zeroes(s->target, start, nbytes, s->write_flags &
@@ -XXX,XX +XXX,XX @@ int64_t block_copy_reset_unallocated(BlockCopyState *s,
 }
 
 int coroutine_fn block_copy(BlockCopyState *s,
-                            int64_t start, uint64_t bytes,
+                            int64_t start, int64_t bytes,
                             bool *error_is_read)
 {
     int ret = 0;
-    int64_t end = bytes + start; /* bytes */
     BlockCopyInFlightReq req;
 
     /*
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
            bdrv_get_aio_context(s->target->bs));
 
     assert(QEMU_IS_ALIGNED(start, s->cluster_size));
-    assert(QEMU_IS_ALIGNED(end, s->cluster_size));
+    assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
 
     block_copy_wait_inflight_reqs(s, start, bytes);
-    block_copy_inflight_req_begin(s, &req, start, end);
+    block_copy_inflight_req_begin(s, &req, start, bytes);
 
-    while (start < end) {
-        int64_t next_zero, chunk_end, status_bytes;
+    while (bytes) {
+        int64_t next_zero, cur_bytes, status_bytes;
 
         if (!bdrv_dirty_bitmap_get(s->copy_bitmap, start)) {
             trace_block_copy_skip(s, start);
             start += s->cluster_size;
+            bytes -= s->cluster_size;
             continue; /* already copied */
         }
 
-        chunk_end = MIN(end, start + s->copy_size);
+        cur_bytes = MIN(bytes, s->copy_size);
 
         next_zero = bdrv_dirty_bitmap_next_zero(s->copy_bitmap, start,
-                                                chunk_end - start);
+                                                cur_bytes);
         if (next_zero >= 0) {
             assert(next_zero > start); /* start is dirty */
-            assert(next_zero < chunk_end); /* no need to do MIN() */
-            chunk_end = next_zero;
+            assert(next_zero < start + cur_bytes); /* no need to do MIN() */
+            cur_bytes = next_zero - start;
         }
 
-        ret = block_copy_block_status(s, start, chunk_end - start,
-                                      &status_bytes);
+        ret = block_copy_block_status(s, start, cur_bytes, &status_bytes);
         if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
             bdrv_reset_dirty_bitmap(s->copy_bitmap, start, status_bytes);
             progress_set_remaining(s->progress,
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
                                    s->in_flight_bytes);
             trace_block_copy_skip_range(s, start, status_bytes);
             start += status_bytes;
+            bytes -= status_bytes;
             continue;
         }
 
-        chunk_end = MIN(chunk_end, start + status_bytes);
+        cur_bytes = MIN(cur_bytes, status_bytes);
 
         trace_block_copy_process(s, start);
 
-        bdrv_reset_dirty_bitmap(s->copy_bitmap, start, chunk_end - start);
-        s->in_flight_bytes += chunk_end - start;
+        bdrv_reset_dirty_bitmap(s->copy_bitmap, start, cur_bytes);
+        s->in_flight_bytes += cur_bytes;
 
-        co_get_from_shres(s->mem, chunk_end - start);
-        ret = block_copy_do_copy(s, start, chunk_end, ret & BDRV_BLOCK_ZERO,
+        co_get_from_shres(s->mem, cur_bytes);
+        ret = block_copy_do_copy(s, start, cur_bytes, ret & BDRV_BLOCK_ZERO,
                                  error_is_read);
-        co_put_to_shres(s->mem, chunk_end - start);
-        s->in_flight_bytes -= chunk_end - start;
+        co_put_to_shres(s->mem, cur_bytes);
+        s->in_flight_bytes -= cur_bytes;
         if (ret < 0) {
-            bdrv_set_dirty_bitmap(s->copy_bitmap, start, chunk_end - start);
+            bdrv_set_dirty_bitmap(s->copy_bitmap, start, cur_bytes);
             break;
         }
 
-        progress_work_done(s->progress, chunk_end - start);
-        s->progress_bytes_callback(chunk_end - start, s->progress_opaque);
-        start = chunk_end;
-        ret = 0;
+        progress_work_done(s->progress, cur_bytes);
+        s->progress_bytes_callback(cur_bytes, s->progress_opaque);
+        start += cur_bytes;
+        bytes -= cur_bytes;
     }
 
     block_copy_inflight_req_end(&req);
diff --git a/include/block/block-copy.h b/include/block/block-copy.h
index XXXXXXX..XXXXXXX 100644
--- a/include/block/block-copy.h
+++ b/include/block/block-copy.h
@@ -XXX,XX +XXX,XX @@
 #include "qemu/co-shared-resource.h"
 
 typedef struct BlockCopyInFlightReq {
-    int64_t start_byte;
-    int64_t end_byte;
+    int64_t start;
+    int64_t bytes;
     QLIST_ENTRY(BlockCopyInFlightReq) list;
     CoQueue wait_queue; /* coroutines blocked on this request */
 } BlockCopyInFlightReq;
@@ -XXX,XX +XXX,XX @@ void block_copy_state_free(BlockCopyState *s);
 int64_t block_copy_reset_unallocated(BlockCopyState *s,
                                      int64_t offset, int64_t *count);
 
-int coroutine_fn block_copy(BlockCopyState *s, int64_t start, uint64_t bytes,
+int coroutine_fn block_copy(BlockCopyState *s, int64_t start, int64_t bytes,
                             bool *error_is_read);
 
 #endif /* BLOCK_COPY_H */
-- 
2.24.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

offset/bytes pair is more usual naming in block layer, let's use it.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200311103004.7649-8-vsementsov@virtuozzo.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/block-copy.c         | 82 +++++++++++++++++++-------------------
 include/block/block-copy.h |  4 +-
 2 files changed, 43 insertions(+), 43 deletions(-)

diff --git a/block/block-copy.c b/block/block-copy.c
index XXXXXXX..XXXXXXX 100644
--- a/block/block-copy.c
+++ b/block/block-copy.c
@@ -XXX,XX +XXX,XX @@
 #define BLOCK_COPY_MAX_MEM (128 * MiB)
 
 static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
-                                                           int64_t start,
+                                                           int64_t offset,
                                                            int64_t bytes)
 {
     BlockCopyInFlightReq *req;
 
     QLIST_FOREACH(req, &s->inflight_reqs, list) {
-        if (start + bytes > req->start && start < req->start + req->bytes) {
+        if (offset + bytes > req->offset && offset < req->offset + req->bytes) {
             return req;
         }
     }
@@ -XXX,XX +XXX,XX @@ static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
 }
 
 static void coroutine_fn block_copy_wait_inflight_reqs(BlockCopyState *s,
-                                                       int64_t start,
+                                                       int64_t offset,
                                                        int64_t bytes)
 {
     BlockCopyInFlightReq *req;
 
-    while ((req = find_conflicting_inflight_req(s, start, bytes))) {
+    while ((req = find_conflicting_inflight_req(s, offset, bytes))) {
         qemu_co_queue_wait(&req->wait_queue, NULL);
     }
 }
 
 static void block_copy_inflight_req_begin(BlockCopyState *s,
                                           BlockCopyInFlightReq *req,
-                                          int64_t start, int64_t bytes)
+                                          int64_t offset, int64_t bytes)
 {
-    req->start = start;
+    req->offset = offset;
     req->bytes = bytes;
     qemu_co_queue_init(&req->wait_queue);
     QLIST_INSERT_HEAD(&s->inflight_reqs, req, list);
@@ -XXX,XX +XXX,XX @@ void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm)
  * Returns 0 on success.
  */
 static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
-                                           int64_t start, int64_t bytes,
+                                           int64_t offset, int64_t bytes,
                                            bool zeroes, bool *error_is_read)
 {
     int ret;
-    int64_t nbytes = MIN(start + bytes, s->len) - start;
+    int64_t nbytes = MIN(offset + bytes, s->len) - offset;
     void *bounce_buffer = NULL;
 
-    assert(start >= 0 && bytes > 0 && INT64_MAX - start >= bytes);
-    assert(QEMU_IS_ALIGNED(start, s->cluster_size));
+    assert(offset >= 0 && bytes > 0 && INT64_MAX - offset >= bytes);
+    assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
     assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
-    assert(start < s->len);
-    assert(start + bytes <= s->len ||
-           start + bytes == QEMU_ALIGN_UP(s->len, s->cluster_size));
+    assert(offset < s->len);
+    assert(offset + bytes <= s->len ||
+           offset + bytes == QEMU_ALIGN_UP(s->len, s->cluster_size));
     assert(nbytes < INT_MAX);
 
     if (zeroes) {
-        ret = bdrv_co_pwrite_zeroes(s->target, start, nbytes, s->write_flags &
+        ret = bdrv_co_pwrite_zeroes(s->target, offset, nbytes, s->write_flags &
                                     ~BDRV_REQ_WRITE_COMPRESSED);
         if (ret < 0) {
-            trace_block_copy_write_zeroes_fail(s, start, ret);
+            trace_block_copy_write_zeroes_fail(s, offset, ret);
             if (error_is_read) {
                 *error_is_read = false;
             }
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
     }
 
     if (s->use_copy_range) {
-        ret = bdrv_co_copy_range(s->source, start, s->target, start, nbytes,
+        ret = bdrv_co_copy_range(s->source, offset, s->target, offset, nbytes,
                                  0, s->write_flags);
         if (ret < 0) {
-            trace_block_copy_copy_range_fail(s, start, ret);
+            trace_block_copy_copy_range_fail(s, offset, ret);
             s->use_copy_range = false;
             s->copy_size = MAX(s->cluster_size, BLOCK_COPY_MAX_BUFFER);
             /* Fallback to read+write with allocated buffer */
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_do_copy(BlockCopyState *s,
 
     bounce_buffer = qemu_blockalign(s->source->bs, nbytes);
 
-    ret = bdrv_co_pread(s->source, start, nbytes, bounce_buffer, 0);
+    ret = bdrv_co_pread(s->source, offset, nbytes, bounce_buffer, 0);
     if (ret < 0) {
-        trace_block_copy_read_fail(s, start, ret);
+        trace_block_copy_read_fail(s, offset, ret);
         if (error_is_read) {
             *error_is_read = true;
         }
         goto out;
     }
 
-    ret = bdrv_co_pwrite(s->target, start, nbytes, bounce_buffer,
+    ret = bdrv_co_pwrite(s->target, offset, nbytes, bounce_buffer,
                          s->write_flags);
     if (ret < 0) {
-        trace_block_copy_write_fail(s, start, ret);
+        trace_block_copy_write_fail(s, offset, ret);
         if (error_is_read) {
             *error_is_read = false;
         }
@@ -XXX,XX +XXX,XX @@ int64_t block_copy_reset_unallocated(BlockCopyState *s,
 }
 
 int coroutine_fn block_copy(BlockCopyState *s,
-                            int64_t start, int64_t bytes,
+                            int64_t offset, int64_t bytes,
                             bool *error_is_read)
 {
     int ret = 0;
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
     assert(bdrv_get_aio_context(s->source->bs) ==
            bdrv_get_aio_context(s->target->bs));
 
-    assert(QEMU_IS_ALIGNED(start, s->cluster_size));
+    assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
     assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
 
-    block_copy_wait_inflight_reqs(s, start, bytes);
-    block_copy_inflight_req_begin(s, &req, start, bytes);
+    block_copy_wait_inflight_reqs(s, offset, bytes);
+    block_copy_inflight_req_begin(s, &req, offset, bytes);
 
     while (bytes) {
         int64_t next_zero, cur_bytes, status_bytes;
 
-        if (!bdrv_dirty_bitmap_get(s->copy_bitmap, start)) {
-            trace_block_copy_skip(s, start);
-            start += s->cluster_size;
+        if (!bdrv_dirty_bitmap_get(s->copy_bitmap, offset)) {
+            trace_block_copy_skip(s, offset);
+            offset += s->cluster_size;
             bytes -= s->cluster_size;
             continue; /* already copied */
         }
 
         cur_bytes = MIN(bytes, s->copy_size);
 
-        next_zero = bdrv_dirty_bitmap_next_zero(s->copy_bitmap, start,
+        next_zero = bdrv_dirty_bitmap_next_zero(s->copy_bitmap, offset,
                                                 cur_bytes);
         if (next_zero >= 0) {
-            assert(next_zero > start); /* start is dirty */
-            assert(next_zero < start + cur_bytes); /* no need to do MIN() */
-            cur_bytes = next_zero - start;
+            assert(next_zero > offset); /* offset is dirty */
+            assert(next_zero < offset + cur_bytes); /* no need to do MIN() */
+            cur_bytes = next_zero - offset;
         }
 
-        ret = block_copy_block_status(s, start, cur_bytes, &status_bytes);
+        ret = block_copy_block_status(s, offset, cur_bytes, &status_bytes);
         if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
-            bdrv_reset_dirty_bitmap(s->copy_bitmap, start, status_bytes);
+            bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, status_bytes);
             progress_set_remaining(s->progress,
                                    bdrv_get_dirty_count(s->copy_bitmap) +
                                    s->in_flight_bytes);
-            trace_block_copy_skip_range(s, start, status_bytes);
-            start += status_bytes;
+            trace_block_copy_skip_range(s, offset, status_bytes);
+            offset += status_bytes;
             bytes -= status_bytes;
             continue;
         }
 
         cur_bytes = MIN(cur_bytes, status_bytes);
 
-        trace_block_copy_process(s, start);
+        trace_block_copy_process(s, offset);
 
-        bdrv_reset_dirty_bitmap(s->copy_bitmap, start, cur_bytes);
+        bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, cur_bytes);
         s->in_flight_bytes += cur_bytes;
 
         co_get_from_shres(s->mem, cur_bytes);
-        ret = block_copy_do_copy(s, start, cur_bytes, ret & BDRV_BLOCK_ZERO,
+        ret = block_copy_do_copy(s, offset, cur_bytes, ret & BDRV_BLOCK_ZERO,
                                  error_is_read);
         co_put_to_shres(s->mem, cur_bytes);
         s->in_flight_bytes -= cur_bytes;
         if (ret < 0) {
-            bdrv_set_dirty_bitmap(s->copy_bitmap, start, cur_bytes);
+            bdrv_set_dirty_bitmap(s->copy_bitmap, offset, cur_bytes);
             break;
         }
 
         progress_work_done(s->progress, cur_bytes);
         s->progress_bytes_callback(cur_bytes, s->progress_opaque);
-        start += cur_bytes;
+        offset += cur_bytes;
         bytes -= cur_bytes;
     }
 
diff --git a/include/block/block-copy.h b/include/block/block-copy.h
index XXXXXXX..XXXXXXX 100644
--- a/include/block/block-copy.h
+++ b/include/block/block-copy.h
@@ -XXX,XX +XXX,XX @@
 #include "qemu/co-shared-resource.h"
 
 typedef struct BlockCopyInFlightReq {
-    int64_t start;
+    int64_t offset;
     int64_t bytes;
     QLIST_ENTRY(BlockCopyInFlightReq) list;
     CoQueue wait_queue; /* coroutines blocked on this request */
@@ -XXX,XX +XXX,XX @@ void block_copy_state_free(BlockCopyState *s);
 int64_t block_copy_reset_unallocated(BlockCopyState *s,
                                      int64_t offset, int64_t *count);
 
-int coroutine_fn block_copy(BlockCopyState *s, int64_t start, int64_t bytes,
+int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes,
                             bool *error_is_read);
 
 #endif /* BLOCK_COPY_H */
-- 
2.24.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Currently, block_copy operation lock the whole requested region. But
there is no reason to lock clusters, which are already copied, it will
disturb other parallel block_copy requests for no reason.

Let's instead do the following:

Lock only sub-region, which we are going to operate on. Then, after
copying all dirty sub-regions, we should wait for intersecting
requests block-copy, if they failed, we should retry these new dirty
clusters.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
Message-Id: <20200311103004.7649-9-vsementsov@virtuozzo.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/block-copy.c | 129 ++++++++++++++++++++++++++++++++++++---------
 1 file changed, 105 insertions(+), 24 deletions(-)

diff --git a/block/block-copy.c b/block/block-copy.c
index XXXXXXX..XXXXXXX 100644
--- a/block/block-copy.c
+++ b/block/block-copy.c
@@ -XXX,XX +XXX,XX @@ static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
     return NULL;
 }
 
-static void coroutine_fn block_copy_wait_inflight_reqs(BlockCopyState *s,
-                                                       int64_t offset,
-                                                       int64_t bytes)
+/*
+ * If there are no intersecting requests return false. Otherwise, wait for the
+ * first found intersecting request to finish and return true.
+ */
+static bool coroutine_fn block_copy_wait_one(BlockCopyState *s, int64_t offset,
+                                             int64_t bytes)
 {
-    BlockCopyInFlightReq *req;
+    BlockCopyInFlightReq *req = find_conflicting_inflight_req(s, offset, bytes);
 
-    while ((req = find_conflicting_inflight_req(s, offset, bytes))) {
-        qemu_co_queue_wait(&req->wait_queue, NULL);
+    if (!req) {
+        return false;
     }
+
+    qemu_co_queue_wait(&req->wait_queue, NULL);
+
+    return true;
 }
 
+/* Called only on full-dirty region */
 static void block_copy_inflight_req_begin(BlockCopyState *s,
                                           BlockCopyInFlightReq *req,
                                           int64_t offset, int64_t bytes)
 {
+    assert(!find_conflicting_inflight_req(s, offset, bytes));
+
+    bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);
+    s->in_flight_bytes += bytes;
+
     req->offset = offset;
     req->bytes = bytes;
     qemu_co_queue_init(&req->wait_queue);
     QLIST_INSERT_HEAD(&s->inflight_reqs, req, list);
 }
 
-static void coroutine_fn block_copy_inflight_req_end(BlockCopyInFlightReq *req)
+/*
+ * block_copy_inflight_req_shrink
+ *
+ * Drop the tail of the request to be handled later. Set dirty bits back and
+ * wake up all requests waiting for us (may be some of them are not intersecting
+ * with shrunk request)
+ */
+static void coroutine_fn block_copy_inflight_req_shrink(BlockCopyState *s,
+        BlockCopyInFlightReq *req, int64_t new_bytes)
 {
+    if (new_bytes == req->bytes) {
+        return;
+    }
+
+    assert(new_bytes > 0 && new_bytes < req->bytes);
+
+    s->in_flight_bytes -= req->bytes - new_bytes;
+    bdrv_set_dirty_bitmap(s->copy_bitmap,
+                          req->offset + new_bytes, req->bytes - new_bytes);
+
+    req->bytes = new_bytes;
+    qemu_co_queue_restart_all(&req->wait_queue);
+}
+
+static void coroutine_fn block_copy_inflight_req_end(BlockCopyState *s,
+                                                     BlockCopyInFlightReq *req,
+                                                     int ret)
+{
+    s->in_flight_bytes -= req->bytes;
+    if (ret < 0) {
+        bdrv_set_dirty_bitmap(s->copy_bitmap, req->offset, req->bytes);
+    }
     QLIST_REMOVE(req, list);
     qemu_co_queue_restart_all(&req->wait_queue);
 }
@@ -XXX,XX +XXX,XX @@ int64_t block_copy_reset_unallocated(BlockCopyState *s,
     return ret;
 }
 
-int coroutine_fn block_copy(BlockCopyState *s,
-                            int64_t offset, int64_t bytes,
-                            bool *error_is_read)
+/*
+ * block_copy_dirty_clusters
+ *
+ * Copy dirty clusters in @offset/@bytes range.
+ * Returns 1 if dirty clusters found and successfully copied, 0 if no dirty
+ * clusters found and -errno on failure.
+ */
+static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,
+                                                  int64_t offset, int64_t bytes,
+                                                  bool *error_is_read)
 {
     int ret = 0;
-    BlockCopyInFlightReq req;
+    bool found_dirty = false;
 
     /*
      * block_copy() user is responsible for keeping source and target in same
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
     assert(QEMU_IS_ALIGNED(offset, s->cluster_size));
     assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));
 
-    block_copy_wait_inflight_reqs(s, offset, bytes);
-    block_copy_inflight_req_begin(s, &req, offset, bytes);
-
     while (bytes) {
+        BlockCopyInFlightReq req;
         int64_t next_zero, cur_bytes, status_bytes;
 
         if (!bdrv_dirty_bitmap_get(s->copy_bitmap, offset)) {
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
             continue; /* already copied */
         }
 
+        found_dirty = true;
+
         cur_bytes = MIN(bytes, s->copy_size);
 
         next_zero = bdrv_dirty_bitmap_next_zero(s->copy_bitmap, offset,
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
             assert(next_zero < offset + cur_bytes); /* no need to do MIN() */
             cur_bytes = next_zero - offset;
         }
+        block_copy_inflight_req_begin(s, &req, offset, cur_bytes);
 
         ret = block_copy_block_status(s, offset, cur_bytes, &status_bytes);
+        assert(ret >= 0); /* never fail */
+        cur_bytes = MIN(cur_bytes, status_bytes);
+        block_copy_inflight_req_shrink(s, &req, cur_bytes);
         if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {
-            bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, status_bytes);
+            block_copy_inflight_req_end(s, &req, 0);
             progress_set_remaining(s->progress,
                                    bdrv_get_dirty_count(s->copy_bitmap) +
                                    s->in_flight_bytes);
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
             continue;
         }
 
-        cur_bytes = MIN(cur_bytes, status_bytes);
-
         trace_block_copy_process(s, offset);
 
-        bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, cur_bytes);
-        s->in_flight_bytes += cur_bytes;
-
         co_get_from_shres(s->mem, cur_bytes);
         ret = block_copy_do_copy(s, offset, cur_bytes, ret & BDRV_BLOCK_ZERO,
                                  error_is_read);
         co_put_to_shres(s->mem, cur_bytes);
-        s->in_flight_bytes -= cur_bytes;
+        block_copy_inflight_req_end(s, &req, ret);
         if (ret < 0) {
-            bdrv_set_dirty_bitmap(s->copy_bitmap, offset, cur_bytes);
-            break;
+            return ret;
         }
 
         progress_work_done(s->progress, cur_bytes);
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,
         bytes -= cur_bytes;
     }
 
-    block_copy_inflight_req_end(&req);
+    return found_dirty;
+}
+
+/*
+ * block_copy
+ *
+ * Copy requested region, accordingly to dirty bitmap.
+ * Collaborate with parallel block_copy requests: if they succeed it will help
+ * us. If they fail, we will retry not-copied regions. So, if we return error,
+ * it means that some I/O operation failed in context of _this_ block_copy call,
+ * not some parallel operation.
+ */
+int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes,
+                            bool *error_is_read)
+{
+    int ret;
+
+    do {
+        ret = block_copy_dirty_clusters(s, offset, bytes, error_is_read);
+
+        if (ret == 0) {
+            ret = block_copy_wait_one(s, offset, bytes);
+        }
+
+        /*
+         * We retry in two cases:
+         * 1. Some progress done
+         *    Something was copied, which means that there were yield points
+         *    and some new dirty bits may have appeared (due to failed parallel
+         *    block-copy requests).
+         * 2. We have waited for some intersecting block-copy request
+         *    It may have failed and produced new dirty bits.
+         */
+    } while (ret > 0);
 
     return ret;
 }
-- 
2.24.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Hide structure definitions and add explicit API instead, to keep an
eye on the scope of the shared fields.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200311103004.7649-10-vsementsov@virtuozzo.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
 block/backup-top.c         |  6 ++--
 block/backup.c             | 25 ++++++++--------
 block/block-copy.c         | 59 ++++++++++++++++++++++++++++++++++++++
 include/block/block-copy.h | 52 +++------------------------------
 4 files changed, 80 insertions(+), 62 deletions(-)

diff --git a/block/backup-top.c b/block/backup-top.c
index XXXXXXX..XXXXXXX 100644
--- a/block/backup-top.c
+++ b/block/backup-top.c
@@ -XXX,XX +XXX,XX @@ typedef struct BDRVBackupTopState {
     BlockCopyState *bcs;
     BdrvChild *target;
     bool active;
+    int64_t cluster_size;
 } BDRVBackupTopState;
 
 static coroutine_fn int backup_top_co_preadv(
@@ -XXX,XX +XXX,XX @@ static coroutine_fn int backup_top_cbw(BlockDriverState *bs, uint64_t offset,
         return 0;
     }
 
-    off = QEMU_ALIGN_DOWN(offset, s->bcs->cluster_size);
-    end = QEMU_ALIGN_UP(offset + bytes, s->bcs->cluster_size);
+    off = QEMU_ALIGN_DOWN(offset, s->cluster_size);
+    end = QEMU_ALIGN_UP(offset + bytes, s->cluster_size);
 
     return block_copy(s->bcs, off, end - off, NULL);
 }
@@ -XXX,XX +XXX,XX @@ BlockDriverState *bdrv_backup_top_append(BlockDriverState *source,
         goto fail;
     }
 
+    state->cluster_size = cluster_size;
     state->bcs = block_copy_state_new(top->backing, state->target,
                                       cluster_size, write_flags, &local_err);
     if (local_err) {
diff --git a/block/backup.c b/block/backup.c
index XXXXXXX..XXXXXXX 100644
--- a/block/backup.c
+++ b/block/backup.c
@@ -XXX,XX +XXX,XX @@ static void backup_cleanup_sync_bitmap(BackupBlockJob *job, int ret)
 
     if (ret < 0 && job->bitmap_mode == BITMAP_SYNC_MODE_ALWAYS) {
         /* If we failed and synced, merge in the bits we didn't copy: */
-        bdrv_dirty_bitmap_merge_internal(bm, job->bcs->copy_bitmap,
+        bdrv_dirty_bitmap_merge_internal(bm, block_copy_dirty_bitmap(job->bcs),
                                          NULL, true);
     }
 }
@@ -XXX,XX +XXX,XX @@ void backup_do_checkpoint(BlockJob *job, Error **errp)
         return;
     }
 
-    bdrv_set_dirty_bitmap(backup_job->bcs->copy_bitmap, 0, backup_job->len);
+    bdrv_set_dirty_bitmap(block_copy_dirty_bitmap(backup_job->bcs), 0,
+                          backup_job->len);
 }
 
 static BlockErrorAction backup_error_action(BackupBlockJob *job,
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_loop(BackupBlockJob *job)
     BdrvDirtyBitmapIter *bdbi;
     int ret = 0;
 
-    bdbi = bdrv_dirty_iter_new(job->bcs->copy_bitmap);
+    bdbi = bdrv_dirty_iter_new(block_copy_dirty_bitmap(job->bcs));
     while ((offset = bdrv_dirty_iter_next(bdbi)) != -1) {
         do {
             if (yield_and_check(job)) {
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_loop(BackupBlockJob *job)
     return ret;
 }
 
-static void backup_init_copy_bitmap(BackupBlockJob *job)
+static void backup_init_bcs_bitmap(BackupBlockJob *job)
 {
     bool ret;
     uint64_t estimate;
+    BdrvDirtyBitmap *bcs_bitmap = block_copy_dirty_bitmap(job->bcs);
 
     if (job->sync_mode == MIRROR_SYNC_MODE_BITMAP) {
-        ret = bdrv_dirty_bitmap_merge_internal(job->bcs->copy_bitmap,
-                                               job->sync_bitmap,
+        ret = bdrv_dirty_bitmap_merge_internal(bcs_bitmap, job->sync_bitmap,
                                                NULL, true);
         assert(ret);
     } else {
@@ -XXX,XX +XXX,XX @@ static void backup_init_copy_bitmap(BackupBlockJob *job)
              * We can't hog the coroutine to initialize this thoroughly.
              * Set a flag and resume work when we are able to yield safely.
              */
-            job->bcs->skip_unallocated = true;
+            block_copy_set_skip_unallocated(job->bcs, true);
         }
-        bdrv_set_dirty_bitmap(job->bcs->copy_bitmap, 0, job->len);
+        bdrv_set_dirty_bitmap(bcs_bitmap, 0, job->len);
     }
 
-    estimate = bdrv_get_dirty_count(job->bcs->copy_bitmap);
+    estimate = bdrv_get_dirty_count(bcs_bitmap);
     job_progress_set_remaining(&job->common.job, estimate);
 }
 
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_run(Job *job, Error **errp)
     BackupBlockJob *s = container_of(job, BackupBlockJob, common.job);
     int ret = 0;
 
-    backup_init_copy_bitmap(s);
+    backup_init_bcs_bitmap(s);
 
     if (s->sync_mode == MIRROR_SYNC_MODE_TOP) {
         int64_t offset = 0;
@@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_run(Job *job, Error **errp)
 
             offset += count;
         }
-        s->bcs->skip_unallocated = false;
+        block_copy_set_skip_unallocated(s->bcs, false);
     }
 
     if (s->sync_mode == MIRROR_SYNC_MODE_NONE) {
         /*
-         * All bits are set in copy_bitmap to allow any cluster to be copied.
+         * All bits are set in bcs bitmap to allow any cluster to be copied.
          * This does not actually require them to be copied.
          */
         while (!job_is_cancelled(job)) {
diff --git a/block/block-copy.c b/block/block-copy.c
index XXXXXXX..XXXXXXX 100644
--- a/block/block-copy.c
+++ b/block/block-copy.c
@@ -XXX,XX +XXX,XX @@
 #define BLOCK_COPY_MAX_BUFFER (1 * MiB)
 #define BLOCK_COPY_MAX_MEM (128 * MiB)
 
+typedef struct BlockCopyInFlightReq {
+    int64_t offset;
+    int64_t bytes;
+    QLIST_ENTRY(BlockCopyInFlightReq) list;
+    CoQueue wait_queue; /* coroutines blocked on this request */
+} BlockCopyInFlightReq;
+
+typedef struct BlockCopyState {
+    /*
+     * BdrvChild objects are not owned or managed by block-copy. They are
+     * provided by block-copy user and user is responsible for appropriate
+     * permissions on these children.
+     */
+    BdrvChild *source;
+    BdrvChild *target;
+    BdrvDirtyBitmap *copy_bitmap;
+    int64_t in_flight_bytes;
+    int64_t cluster_size;
+    bool use_copy_range;
+    int64_t copy_size;
+    uint64_t len;
+    QLIST_HEAD(, BlockCopyInFlightReq) inflight_reqs;
+
+    BdrvRequestFlags write_flags;
+
+    /*
+     * skip_unallocated:
+     *
+     * Used by sync=top jobs, which first scan the source node for unallocated
+     * areas and clear them in the copy_bitmap.  During this process, the bitmap
+     * is thus not fully initialized: It may still have bits set for areas that
+     * are unallocated and should actually not be copied.
+     *
+     * This is indicated by skip_unallocated.
+     *
+     * In this case, block_copy() will query the source’s allocation status,
+     * skip unallocated regions, clear them in the copy_bitmap, and invoke
+     * block_copy_reset_unallocated() every time it does.
+     */
+    bool skip_unallocated;
+
+    ProgressMeter *progress;
+    /* progress_bytes_callback: called when some copying progress is done. */
+    ProgressBytesCallbackFunc progress_bytes_callback;
+    void *progress_opaque;
+
+    SharedResource *mem;
+} BlockCopyState;
+
 static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,
                                                            int64_t offset,
                                                            int64_t bytes)
@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes,
 
     return ret;
 }
+
+BdrvDirtyBitmap *block_copy_dirty_bitmap(BlockCopyState *s)
+{
+    return s->copy_bitmap;
+}
+
+void block_copy_set_skip_unallocated(BlockCopyState *s, bool skip)
+{
+    s->skip_unallocated = skip;
+}
diff --git a/include/block/block-copy.h b/include/block/block-copy.h
index XXXXXXX..XXXXXXX 100644
--- a/include/block/block-copy.h
+++ b/include/block/block-copy.h
@@ -XXX,XX +XXX,XX @@
 #include "block/block.h"
 #include "qemu/co-shared-resource.h"
 
-typedef struct BlockCopyInFlightReq {
-    int64_t offset;
-    int64_t bytes;
-    QLIST_ENTRY(BlockCopyInFlightReq) list;
-    CoQueue wait_queue; /* coroutines blocked on this request */
-} BlockCopyInFlightReq;
-
 typedef void (*ProgressBytesCallbackFunc)(int64_t bytes, void *opaque);
-typedef struct BlockCopyState {
-    /*
-     * BdrvChild objects are not owned or managed by block-copy. They are
-     * provided by block-copy user and user is responsible for appropriate
-     * permissions on these children.
-     */
-    BdrvChild *source;
-    BdrvChild *target;
-    BdrvDirtyBitmap *copy_bitmap;
-    int64_t in_flight_bytes;
-    int64_t cluster_size;
-    bool use_copy_range;
-    int64_t copy_size;
-    uint64_t len;
-    QLIST_HEAD(, BlockCopyInFlightReq) inflight_reqs;
-
-    BdrvRequestFlags write_flags;
-
-    /*
-     * skip_unallocated:
-     *
-     * Used by sync=top jobs, which first scan the source node for unallocated
-     * areas and clear them in the copy_bitmap.  During this process, the bitmap
-     * is thus not fully initialized: It may still have bits set for areas that
-     * are unallocated and should actually not be copied.
-     *
-     * This is indicated by skip_unallocated.
-     *
-     * In this case, block_copy() will query the source’s allocation status,
-     * skip unallocated regions, clear them in the copy_bitmap, and invoke
-     * block_copy_reset_unallocated() every time it does.
-     */
-    bool skip_unallocated;
-
-    ProgressMeter *progress;
-    /* progress_bytes_callback: called when some copying progress is done. */
-    ProgressBytesCallbackFunc progress_bytes_callback;
-    void *progress_opaque;
-
-    SharedResource *mem;
-} BlockCopyState;
+typedef struct BlockCopyState BlockCopyState;
 
 BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,
                                      int64_t cluster_size,
@@ -XXX,XX +XXX,XX @@ int64_t block_copy_reset_unallocated(BlockCopyState *s,
 int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes,
                             bool *error_is_read);
 
+BdrvDirtyBitmap *block_copy_dirty_bitmap(BlockCopyState *s);
+void block_copy_set_skip_unallocated(BlockCopyState *s, bool skip);
+
 #endif /* BLOCK_COPY_H */
-- 
2.24.1