The following changes since commit ba29883206d92a29ad5a466e679ccfc2ee6132ef:

Merge remote-tracking branch 'remotes/borntraeger/tags/s390x-20200310' into staging (2020-03-10 16:50:28 +0000)

https://github.com/XanClic/qemu.git tags/pull-block-2020-03-11

for you to fetch changes up to 397f4e9d83e9c0000905f0a988ba1aeda162571c:

block/block-copy: hide structure definitions (2020-03-11 12:42:30 +0100)

Block patches for the 5.0 softfreeze:

- qemu-img measure for LUKS

- Improve block-copy's performance by reducing inter-request

dependencies

- Make curl's detection of accept-ranges more robust

- Memleak fixes

- iotest fix

Eric Blake (1):

iotests: Fix nonportable use of od --endian

Pan Nengyuan (2):

block/qcow2: do free crypto_opts in qcow2_close()

qemu-img: free memory before re-assign

Stefan Hajnoczi (4):

luks: extract qcrypto_block_calculate_payload_offset()

luks: implement .bdrv_measure()

qemu-img: allow qemu-img measure --object without a filename

iotests: add 288 luks qemu-img measure test

Vladimir Sementsov-Ogievskiy (10):

block/qcow2-threads: fix qcow2_decompress

job: refactor progress to separate object

block/block-copy: fix progress calculation

block/block-copy: specialcase first copy_range request

block/block-copy: use block_status

block/block-copy: factor out find_conflicting_inflight_req

block/block-copy: refactor interfaces to use bytes instead of end

block/block-copy: rename start to offset in interfaces

block/block-copy: reduce intersecting request lock

block/block-copy: hide structure definitions

block/backup-top.c | 6 +-

block/backup.c | 38 ++-

block/block-copy.c | 405 ++++++++++++++++++++++++-------

block/crypto.c | 62 +++++

block/curl.c | 32 ++-

block/qcow2-threads.c | 12 +-

block/qcow2.c | 75 ++----

block/trace-events | 1 +

blockjob.c | 16 +-

crypto/block.c | 36 +++

include/block/block-copy.h | 65 +----

include/crypto/block.h | 22 ++

include/qemu/job.h | 11 +-

include/qemu/progress_meter.h | 58 +++++

job-qmp.c | 4 +-

job.c | 6 +-

qemu-img.c | 14 +-

tests/qemu-iotests/178 | 2 +-

tests/qemu-iotests/178.out.qcow2 | 8 +-

tests/qemu-iotests/178.out.raw | 8 +-

tests/qemu-iotests/288 | 93 +++++++

tests/qemu-iotests/288.out | 30 +++

tests/qemu-iotests/common.rc | 22 +-

tests/qemu-iotests/group | 1 +

24 files changed, 749 insertions(+), 278 deletions(-)

create mode 100644 include/qemu/progress_meter.h

create mode 100755 tests/qemu-iotests/288

create mode 100644 tests/qemu-iotests/288.out

2.24.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

In block_copy_do_copy we fallback to read+write if copy_range failed.

In this case copy_size is larger than defined for buffered IO, and

there is corresponding commit. Still, backup copies data cluster by

cluster, and most of requests are limited to one cluster anyway, so the

only source of this one bad-limited request is copy-before-write

operation.

Further patch will move backup to use block_copy directly, than for

cases where copy_range is not supported, first request will be

oversized in each backup. It's not good, let's change it now.

Fix is simple: just limit first copy_range request like buffer-based

request. If it succeed, set larger copy_range limit.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>

Reviewed-by: Max Reitz <mreitz@redhat.com>

Message-Id: <20200311103004.7649-4-vsementsov@virtuozzo.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

block/block-copy.c | 41 +++++++++++++++++++++++++++++++----------

1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/block/block-copy.c b/block/block-copy.c

--- a/block/block-copy.c

+++ b/block/block-copy.c

@@ -XXX,XX +XXX,XX @@ void block_copy_state_free(BlockCopyState *s)

g_free(s);

+static uint32_t block_copy_max_transfer(BdrvChild *source, BdrvChild *target)

+ return MIN_NON_ZERO(INT_MAX,

+ MIN_NON_ZERO(source->bs->bl.max_transfer,

+ target->bs->bl.max_transfer));

BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,

int64_t cluster_size,

BdrvRequestFlags write_flags, Error **errp)

{

BlockCopyState *s;

BdrvDirtyBitmap *copy_bitmap;

- uint32_t max_transfer =

- MIN_NON_ZERO(INT_MAX,

- MIN_NON_ZERO(source->bs->bl.max_transfer,

- target->bs->bl.max_transfer));

copy_bitmap = bdrv_create_dirty_bitmap(source->bs, cluster_size, NULL,

errp);

@@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,

.mem = shres_create(BLOCK_COPY_MAX_MEM),

};

- if (max_transfer < cluster_size) {

+ if (block_copy_max_transfer(source, target) < cluster_size) {

/*

* copy_range does not respect max_transfer. We don't want to bother

* with requests smaller than block-copy cluster size, so fallback to

@@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target,

s->copy_size = cluster_size;

} else {

/*

- * copy_range does not respect max_transfer (it's a TODO), so we factor

- * that in here.

+ * We enable copy-range, but keep small copy_size, until first

+ * successful copy_range (look at block_copy_do_copy).

*/

s->use_copy_range = true;

- s->copy_size = MIN(MAX(cluster_size, BLOCK_COPY_MAX_COPY_RANGE),

- QEMU_ALIGN_DOWN(max_transfer, cluster_size));

+ s->copy_size = MAX(s->cluster_size, BLOCK_COPY_MAX_BUFFER);

}

QLIST_INIT(&s->inflight_reqs);

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_do_copy(BlockCopyState *s,

s->copy_size = MAX(s->cluster_size, BLOCK_COPY_MAX_BUFFER);

/* Fallback to read+write with allocated buffer */

} else {

+ if (s->use_copy_range) {

+ /*

+ * Successful copy-range. Now increase copy_size. copy_range

+ * does not respect max_transfer (it's a TODO), so we factor

+ * that in here.

+ *

+ * Note: we double-check s->use_copy_range for the case when

+ * parallel block-copy request unsets it during previous

+ * bdrv_co_copy_range call.

+ */

+ s->copy_size =

+ MIN(MAX(s->cluster_size, BLOCK_COPY_MAX_COPY_RANGE),

+ QEMU_ALIGN_DOWN(block_copy_max_transfer(s->source,

+ s->target),

+ s->cluster_size));

+ }

goto out;

}

}

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_do_copy(BlockCopyState *s,

/*

* In case of failed copy_range request above, we may proceed with buffered

* request larger than BLOCK_COPY_MAX_BUFFER. Still, further requests will

- * be properly limited, so don't care too much.

+ * be properly limited, so don't care too much. Moreover the most likely

+ * case (copy_range is unsupported for the configuration, so the very first

+ * copy_range request fails) is handled by setting large copy_size only

+ * after first successful copy_range.

*/

bounce_buffer = qemu_blockalign(s->source->bs, nbytes);

2.24.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Hide structure definitions and add explicit API instead, to keep an

eye on the scope of the shared fields.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>

Reviewed-by: Max Reitz <mreitz@redhat.com>

Message-Id: <20200311103004.7649-10-vsementsov@virtuozzo.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

block/backup-top.c | 6 ++--

block/backup.c | 25 ++++++++--------

block/block-copy.c | 59 ++++++++++++++++++++++++++++++++++++++

include/block/block-copy.h | 52 +++------------------------------

4 files changed, 80 insertions(+), 62 deletions(-)

diff --git a/block/backup-top.c b/block/backup-top.c

--- a/block/backup-top.c

+++ b/block/backup-top.c

@@ -XXX,XX +XXX,XX @@ typedef struct BDRVBackupTopState {

BlockCopyState *bcs;

BdrvChild *target;

bool active;

+ int64_t cluster_size;

} BDRVBackupTopState;

static coroutine_fn int backup_top_co_preadv(

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int backup_top_cbw(BlockDriverState *bs, uint64_t offset,

return 0;

- off = QEMU_ALIGN_DOWN(offset, s->bcs->cluster_size);

- end = QEMU_ALIGN_UP(offset + bytes, s->bcs->cluster_size);

+ off = QEMU_ALIGN_DOWN(offset, s->cluster_size);

+ end = QEMU_ALIGN_UP(offset + bytes, s->cluster_size);

return block_copy(s->bcs, off, end - off, NULL);

@@ -XXX,XX +XXX,XX @@ BlockDriverState *bdrv_backup_top_append(BlockDriverState *source,

goto fail;

+ state->cluster_size = cluster_size;

state->bcs = block_copy_state_new(top->backing, state->target,

cluster_size, write_flags, &local_err);

if (local_err) {

diff --git a/block/backup.c b/block/backup.c

index XXXXXXX..XXXXXXX 100644

--- a/block/backup.c

+++ b/block/backup.c

@@ -XXX,XX +XXX,XX @@ static void backup_cleanup_sync_bitmap(BackupBlockJob *job, int ret)

if (ret < 0 && job->bitmap_mode == BITMAP_SYNC_MODE_ALWAYS) {

/* If we failed and synced, merge in the bits we didn't copy: */

- bdrv_dirty_bitmap_merge_internal(bm, job->bcs->copy_bitmap,

+ bdrv_dirty_bitmap_merge_internal(bm, block_copy_dirty_bitmap(job->bcs),

NULL, true);

}

@@ -XXX,XX +XXX,XX @@ void backup_do_checkpoint(BlockJob *job, Error **errp)

- bdrv_set_dirty_bitmap(backup_job->bcs->copy_bitmap, 0, backup_job->len);

+ bdrv_set_dirty_bitmap(block_copy_dirty_bitmap(backup_job->bcs), 0,

+ backup_job->len);

2.24.1

From: Pan Nengyuan <pannengyuan@huawei.com>

collect_image_check() is called twice in img_check(), the filename/format will be alloced without free the original memory.

It is not a big deal since the process will exit anyway, but seems like a clean code and it will remove the warning spotted by asan.

Reported-by: Euler Robot <euler.robot@huawei.com>

Signed-off-by: Pan Nengyuan <pannengyuan@huawei.com>

Message-Id: <20200227012950.12256-3-pannengyuan@huawei.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

qemu-img.c | 2 ++

1 file changed, 2 insertions(+)

diff --git a/qemu-img.c b/qemu-img.c

--- a/qemu-img.c

+++ b/qemu-img.c

@@ -XXX,XX +XXX,XX @@ static int img_check(int argc, char **argv)

check->corruptions_fixed);

+ qapi_free_ImageCheck(check);

+ check = g_new0(ImageCheck, 1);

ret = collect_image_check(bs, check, filename, fmt, 0);

check->leaks_fixed = leaks_fixed;

2.24.1