The following changes since commit ba29883206d92a29ad5a466e679ccfc2ee6132ef:

Merge remote-tracking branch 'remotes/borntraeger/tags/s390x-20200310' into staging (2020-03-10 16:50:28 +0000)

https://github.com/XanClic/qemu.git tags/pull-block-2020-03-11

for you to fetch changes up to 397f4e9d83e9c0000905f0a988ba1aeda162571c:

block/block-copy: hide structure definitions (2020-03-11 12:42:30 +0100)

Block patches for the 5.0 softfreeze:

- qemu-img measure for LUKS

- Improve block-copy's performance by reducing inter-request

dependencies

- Make curl's detection of accept-ranges more robust

- Memleak fixes

- iotest fix

David Edmondson (2):

block/curl: HTTP header fields allow whitespace around values

block/curl: HTTP header field names are case insensitive

Eric Blake (1):

iotests: Fix nonportable use of od --endian

Pan Nengyuan (2):

block/qcow2: do free crypto_opts in qcow2_close()

qemu-img: free memory before re-assign

Stefan Hajnoczi (4):

luks: extract qcrypto_block_calculate_payload_offset()

luks: implement .bdrv_measure()

qemu-img: allow qemu-img measure --object without a filename

iotests: add 288 luks qemu-img measure test

Vladimir Sementsov-Ogievskiy (10):

block/qcow2-threads: fix qcow2_decompress

job: refactor progress to separate object

block/block-copy: fix progress calculation

block/block-copy: specialcase first copy_range request

block/block-copy: use block_status

block/block-copy: factor out find_conflicting_inflight_req

block/block-copy: refactor interfaces to use bytes instead of end

block/block-copy: rename start to offset in interfaces

block/block-copy: reduce intersecting request lock

block/block-copy: hide structure definitions

block/backup-top.c | 6 +-

block/backup.c | 38 ++-

block/block-copy.c | 405 ++++++++++++++++++++++++-------

block/crypto.c | 62 +++++

block/curl.c | 32 ++-

block/qcow2-threads.c | 12 +-

block/qcow2.c | 75 ++----

block/trace-events | 1 +

blockjob.c | 16 +-

crypto/block.c | 36 +++

include/block/block-copy.h | 65 +----

include/crypto/block.h | 22 ++

include/qemu/job.h | 11 +-

include/qemu/progress_meter.h | 58 +++++

job-qmp.c | 4 +-

job.c | 6 +-

qemu-img.c | 14 +-

tests/qemu-iotests/178 | 2 +-

tests/qemu-iotests/178.out.qcow2 | 8 +-

tests/qemu-iotests/178.out.raw | 8 +-

tests/qemu-iotests/288 | 93 +++++++

tests/qemu-iotests/288.out | 30 +++

tests/qemu-iotests/common.rc | 22 +-

tests/qemu-iotests/group | 1 +

24 files changed, 749 insertions(+), 278 deletions(-)

create mode 100644 include/qemu/progress_meter.h

create mode 100755 tests/qemu-iotests/288

create mode 100644 tests/qemu-iotests/288.out

2.24.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Currently, block_copy operation lock the whole requested region. But

there is no reason to lock clusters, which are already copied, it will

disturb other parallel block_copy requests for no reason.

Let's instead do the following:

Lock only sub-region, which we are going to operate on. Then, after

copying all dirty sub-regions, we should wait for intersecting

requests block-copy, if they failed, we should retry these new dirty

clusters.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>

Message-Id: <20200311103004.7649-9-vsementsov@virtuozzo.com>

block/block-copy.c | 129 ++++++++++++++++++++++++++++++++++++---------

1 file changed, 105 insertions(+), 24 deletions(-)

diff --git a/block/block-copy.c b/block/block-copy.c

--- a/block/block-copy.c

+++ b/block/block-copy.c

@@ -XXX,XX +XXX,XX @@ static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,

return NULL;

}

-static void coroutine_fn block_copy_wait_inflight_reqs(BlockCopyState *s,

- int64_t offset,

- int64_t bytes)

+/*

+ * If there are no intersecting requests return false. Otherwise, wait for the

+ * first found intersecting request to finish and return true.

+ */

+static bool coroutine_fn block_copy_wait_one(BlockCopyState *s, int64_t offset,

+ int64_t bytes)

{

- BlockCopyInFlightReq *req;

+ BlockCopyInFlightReq *req = find_conflicting_inflight_req(s, offset, bytes);

- while ((req = find_conflicting_inflight_req(s, offset, bytes))) {

- qemu_co_queue_wait(&req->wait_queue, NULL);

+ if (!req) {

+ return false;

}

+

+ qemu_co_queue_wait(&req->wait_queue, NULL);

+

+ return true;

}

+/* Called only on full-dirty region */

static void block_copy_inflight_req_begin(BlockCopyState *s,

BlockCopyInFlightReq *req,

int64_t offset, int64_t bytes)

{

+ assert(!find_conflicting_inflight_req(s, offset, bytes));

+

+ bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes);

+ s->in_flight_bytes += bytes;

+

req->offset = offset;

req->bytes = bytes;

qemu_co_queue_init(&req->wait_queue);

QLIST_INSERT_HEAD(&s->inflight_reqs, req, list);

}

-static void coroutine_fn block_copy_inflight_req_end(BlockCopyInFlightReq *req)

+/*

+ * block_copy_inflight_req_shrink

+ *

+ * Drop the tail of the request to be handled later. Set dirty bits back and

+ * wake up all requests waiting for us (may be some of them are not intersecting

+ * with shrunk request)

+ */

+static void coroutine_fn block_copy_inflight_req_shrink(BlockCopyState *s,

+ BlockCopyInFlightReq *req, int64_t new_bytes)

{

+ if (new_bytes == req->bytes) {

+ return;

+ }

+

+ assert(new_bytes > 0 && new_bytes < req->bytes);

+

+ s->in_flight_bytes -= req->bytes - new_bytes;

+ bdrv_set_dirty_bitmap(s->copy_bitmap,

+ req->offset + new_bytes, req->bytes - new_bytes);

+

+ req->bytes = new_bytes;

+ qemu_co_queue_restart_all(&req->wait_queue);

+}

+

+static void coroutine_fn block_copy_inflight_req_end(BlockCopyState *s,

+ BlockCopyInFlightReq *req,

+ int ret)

+{

+ s->in_flight_bytes -= req->bytes;

+ if (ret < 0) {

+ bdrv_set_dirty_bitmap(s->copy_bitmap, req->offset, req->bytes);

+ }

QLIST_REMOVE(req, list);

qemu_co_queue_restart_all(&req->wait_queue);

}

@@ -XXX,XX +XXX,XX @@ int64_t block_copy_reset_unallocated(BlockCopyState *s,

return ret;

}

-int coroutine_fn block_copy(BlockCopyState *s,

- int64_t offset, int64_t bytes,

- bool *error_is_read)

+/*

+ * block_copy_dirty_clusters

+ *

+ * Copy dirty clusters in @offset/@bytes range.

+ * Returns 1 if dirty clusters found and successfully copied, 0 if no dirty

+ * clusters found and -errno on failure.

+ */

+static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s,

+ int64_t offset, int64_t bytes,

+ bool *error_is_read)

{

int ret = 0;

- BlockCopyInFlightReq req;

+ bool found_dirty = false;

/*

* block_copy() user is responsible for keeping source and target in same

@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,

assert(QEMU_IS_ALIGNED(offset, s->cluster_size));

assert(QEMU_IS_ALIGNED(bytes, s->cluster_size));

- block_copy_wait_inflight_reqs(s, offset, bytes);

- block_copy_inflight_req_begin(s, &req, offset, bytes);

-

while (bytes) {

+ BlockCopyInFlightReq req;

int64_t next_zero, cur_bytes, status_bytes;

if (!bdrv_dirty_bitmap_get(s->copy_bitmap, offset)) {

@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,

continue; /* already copied */

}

+ found_dirty = true;

+

cur_bytes = MIN(bytes, s->copy_size);

next_zero = bdrv_dirty_bitmap_next_zero(s->copy_bitmap, offset,

@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,

assert(next_zero < offset + cur_bytes); /* no need to do MIN() */

cur_bytes = next_zero - offset;

}

+ block_copy_inflight_req_begin(s, &req, offset, cur_bytes);

ret = block_copy_block_status(s, offset, cur_bytes, &status_bytes);

+ assert(ret >= 0); /* never fail */

+ cur_bytes = MIN(cur_bytes, status_bytes);

+ block_copy_inflight_req_shrink(s, &req, cur_bytes);

if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) {

- bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, status_bytes);

+ block_copy_inflight_req_end(s, &req, 0);

progress_set_remaining(s->progress,

bdrv_get_dirty_count(s->copy_bitmap) +

s->in_flight_bytes);

@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,

continue;

}

- cur_bytes = MIN(cur_bytes, status_bytes);

-

trace_block_copy_process(s, offset);

- bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, cur_bytes);

- s->in_flight_bytes += cur_bytes;

-

co_get_from_shres(s->mem, cur_bytes);

ret = block_copy_do_copy(s, offset, cur_bytes, ret & BDRV_BLOCK_ZERO,

error_is_read);

co_put_to_shres(s->mem, cur_bytes);

- s->in_flight_bytes -= cur_bytes;

+ block_copy_inflight_req_end(s, &req, ret);

if (ret < 0) {

- bdrv_set_dirty_bitmap(s->copy_bitmap, offset, cur_bytes);

- break;

+ return ret;

}

progress_work_done(s->progress, cur_bytes);

@@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s,

bytes -= cur_bytes;

}

- block_copy_inflight_req_end(&req);

+ return found_dirty;

+}

+

+/*

+ * block_copy

+ *

+ * Copy requested region, accordingly to dirty bitmap.

+ * Collaborate with parallel block_copy requests: if they succeed it will help

+ * us. If they fail, we will retry not-copied regions. So, if we return error,

+ * it means that some I/O operation failed in context of _this_ block_copy call,

+ * not some parallel operation.

+ */

+int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes,

+ bool *error_is_read)

+{

+ int ret;

+

+ do {

+ ret = block_copy_dirty_clusters(s, offset, bytes, error_is_read);

+

+ if (ret == 0) {

+ ret = block_copy_wait_one(s, offset, bytes);

+ }

+

+ /*

+ * We retry in two cases:

+ * 1. Some progress done

+ * Something was copied, which means that there were yield points

+ * and some new dirty bits may have appeared (due to failed parallel

+ * block-copy requests).

+ * 2. We have waited for some intersecting block-copy request

+ * It may have failed and produced new dirty bits.

+ */

+ } while (ret > 0);

return ret;

}

2.24.1

Hide structure definitions and add explicit API instead, to keep an

eye on the scope of the shared fields.

Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>

Reviewed-by: Max Reitz <mreitz@redhat.com>

Message-Id: <20200311103004.7649-10-vsementsov@virtuozzo.com>

block/backup-top.c | 6 ++--

block/backup.c | 25 ++++++++--------

block/block-copy.c | 59 ++++++++++++++++++++++++++++++++++++++

include/block/block-copy.h | 52 +++------------------------------

4 files changed, 80 insertions(+), 62 deletions(-)

diff --git a/block/backup-top.c b/block/backup-top.c

--- a/block/backup-top.c

+++ b/block/backup-top.c

@@ -XXX,XX +XXX,XX @@ typedef struct BDRVBackupTopState {

BlockCopyState *bcs;

BdrvChild *target;

bool active;

+ int64_t cluster_size;

} BDRVBackupTopState;

static coroutine_fn int backup_top_co_preadv(

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int backup_top_cbw(BlockDriverState *bs, uint64_t offset,

return 0;

}

- off = QEMU_ALIGN_DOWN(offset, s->bcs->cluster_size);

- end = QEMU_ALIGN_UP(offset + bytes, s->bcs->cluster_size);

+ off = QEMU_ALIGN_DOWN(offset, s->cluster_size);

+ end = QEMU_ALIGN_UP(offset + bytes, s->cluster_size);

return block_copy(s->bcs, off, end - off, NULL);

}

@@ -XXX,XX +XXX,XX @@ BlockDriverState *bdrv_backup_top_append(BlockDriverState *source,

goto fail;

}

+ state->cluster_size = cluster_size;

state->bcs = block_copy_state_new(top->backing, state->target,

cluster_size, write_flags, &local_err);

if (local_err) {

diff --git a/block/backup.c b/block/backup.c

index XXXXXXX..XXXXXXX 100644

--- a/block/backup.c

+++ b/block/backup.c

@@ -XXX,XX +XXX,XX @@ static void backup_cleanup_sync_bitmap(BackupBlockJob *job, int ret)

if (ret < 0 && job->bitmap_mode == BITMAP_SYNC_MODE_ALWAYS) {

/* If we failed and synced, merge in the bits we didn't copy: */

- bdrv_dirty_bitmap_merge_internal(bm, job->bcs->copy_bitmap,

+ bdrv_dirty_bitmap_merge_internal(bm, block_copy_dirty_bitmap(job->bcs),

NULL, true);

}

}

@@ -XXX,XX +XXX,XX @@ void backup_do_checkpoint(BlockJob *job, Error **errp)

2.24.1

From: Stefan Hajnoczi <stefanha@redhat.com>

This test exercises the block/crypto.c "luks" block driver

.bdrv_measure() code.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

Message-Id: <20200221112522.1497712-5-stefanha@redhat.com>

[mreitz: Renamed test from 282 to 288]

tests/qemu-iotests/288 | 93 ++++++++++++++++++++++++++++++++++++++

tests/qemu-iotests/288.out | 30 ++++++++++++

tests/qemu-iotests/group | 1 +

3 files changed, 124 insertions(+)

create mode 100755 tests/qemu-iotests/288

create mode 100644 tests/qemu-iotests/288.out

diff --git a/tests/qemu-iotests/288 b/tests/qemu-iotests/288

new file mode 100755

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/tests/qemu-iotests/288

@@ -XXX,XX +XXX,XX @@

+#!/usr/bin/env bash

+#

+# qemu-img measure tests for LUKS images

+#

+# Copyright (C) 2020 Red Hat, Inc.

+#

+# This program is free software; you can redistribute it and/or modify

+# it under the terms of the GNU General Public License as published by

+# the Free Software Foundation; either version 2 of the License, or

+# (at your option) any later version.

+#

+# This program is distributed in the hope that it will be useful,

+# but WITHOUT ANY WARRANTY; without even the implied warranty of

+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

+# GNU General Public License for more details.

+#

+# You should have received a copy of the GNU General Public License

+# along with this program. If not, see <http://www.gnu.org/licenses/>.

+#

+

+# creator

+owner=stefanha@redhat.com

+

+seq=`basename $0`

+echo "QA output created by $seq"

+

+status=1 # failure is the default!

+

+_cleanup()

+{

+ _cleanup_test_img

+ rm -f "$TEST_IMG.converted"

+}

+trap "_cleanup; exit \$status" 0 1 2 3 15

+

+# get standard environment, filters and checks

+. ./common.rc

+. ./common.filter

+. ./common.pattern

+

+_supported_fmt luks

+_supported_proto file

+_supported_os Linux

+

+SECRET=secret,id=sec0,data=passphrase

+

+echo "== measure 1G image file =="

+

+$QEMU_IMG measure --object "$SECRET" \

+     -O "$IMGFMT" \

+         -o key-secret=sec0,iter-time=10 \

+         --size 1G

+echo "== create 1G image file (size should be no greater than measured) =="

+echo

+_make_test_img 1G

+stat -c "image file size in bytes: %s" "$TEST_IMG_FILE"

+echo

+echo "== modified 1G image file (size should be no greater than measured) =="

+echo

+$QEMU_IO --object "$SECRET" --image-opts "$TEST_IMG" -c "write -P 0x51 0x10000 0x400" | _filter_qemu_io | _filter_testdir

+stat -c "image file size in bytes: %s" "$TEST_IMG_FILE"

+

+echo

+echo "== measure preallocation=falloc 1G image file =="

+echo

+

+$QEMU_IMG measure --object "$SECRET" \

+     -O "$IMGFMT" \

+         -o key-secret=sec0,iter-time=10,preallocation=falloc \

+         --size 1G

+

+echo

+echo "== measure with input image file =="

+echo

+

+IMGFMT=raw IMGKEYSECRET= IMGOPTS= _make_test_img 1G | _filter_imgfmt

+QEMU_IO_OPTIONS= IMGOPTSSYNTAX= $QEMU_IO -f raw -c "write -P 0x51 0x10000 0x400" "$TEST_IMG_FILE" | _filter_qemu_io | _filter_testdir

+$QEMU_IMG measure --object "$SECRET" \

+     -O "$IMGFMT" \

+         -o key-secret=sec0,iter-time=10 \

+         -f raw \

+         "$TEST_IMG_FILE"

+

+# success, all done

+echo "*** done"

+rm -f $seq.full

+status=0

diff --git a/tests/qemu-iotests/288.out b/tests/qemu-iotests/288.out

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/tests/qemu-iotests/288.out

@@ -XXX,XX +XXX,XX @@

+QA output created by 288

+== measure 1G image file ==

+

+required size: 1075810304

+fully allocated size: 1075810304

+

+== create 1G image file (size should be no greater than measured) ==

+

+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824

+image file size in bytes: 1075810304

+

+== modified 1G image file (size should be no greater than measured) ==

+

+wrote 1024/1024 bytes at offset 65536

+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)

+image file size in bytes: 1075810304

+

+== measure preallocation=falloc 1G image file ==

+

+required size: 1075810304

+fully allocated size: 1075810304

+

+== measure with input image file ==

+

+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824

+wrote 1024/1024 bytes at offset 65536

+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)

+required size: 1075810304

+fully allocated size: 1075810304

+*** done

diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/group

+++ b/tests/qemu-iotests/group

@@ -XXX,XX +XXX,XX @@

283 auto quick

284 rw

286 rw quick

+288 quick

2.24.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

offset/bytes pair is more usual naming in block layer, let's use it.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>

block/block-copy.c | 82 +++++++++++++++++++-------------------

include/block/block-copy.h | 4 +-

2 files changed, 43 insertions(+), 43 deletions(-)

diff --git a/block/block-copy.c b/block/block-copy.c

--- a/block/block-copy.c

+++ b/block/block-copy.c

@@ -XXX,XX +XXX,XX @@

#define BLOCK_COPY_MAX_MEM (128 * MiB)

static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,

- int64_t start,

+ int64_t offset,

int64_t bytes)

{

BlockCopyInFlightReq *req;

QLIST_FOREACH(req, &s->inflight_reqs, list) {

- if (start + bytes > req->start && start < req->start + req->bytes) {

+ if (offset + bytes > req->offset && offset < req->offset + req->bytes) {

return req;

}

@@ -XXX,XX +XXX,XX @@ static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s,

}

static void coroutine_fn block_copy_wait_inflight_reqs(BlockCopyState *s,

- int64_t start,

+ int64_t offset,

int64_t bytes)

{

BlockCopyInFlightReq *req;

- while ((req = find_conflicting_inflight_req(s, start, bytes))) {

+ while ((req = find_conflicting_inflight_req(s, offset, bytes))) {

qemu_co_queue_wait(&req->wait_queue, NULL);

2.24.1

From: David Edmondson <david.edmondson@oracle.com>

RFC 7230 section 3.2 indicates that whitespace is permitted between

the field name and field value and after the field value.

Signed-off-by: David Edmondson <david.edmondson@oracle.com>

Message-Id: <20200224101310.101169-2-david.edmondson@oracle.com>

block/curl.c | 31 +++++++++++++++++++++++++++----

1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/block/curl.c b/block/curl.c

--- a/block/curl.c

+++ b/block/curl.c

@@ -XXX,XX +XXX,XX @@ static size_t curl_header_cb(void *ptr, size_t size, size_t nmemb, void *opaque)

{

BDRVCURLState *s = opaque;

size_t realsize = size * nmemb;

- const char *accept_line = "Accept-Ranges: bytes";

+ const char *header = (char *)ptr;

+ const char *end = header + realsize;

+ const char *accept_ranges = "Accept-Ranges:";

+ const char *bytes = "bytes";

- if (realsize >= strlen(accept_line)

- && strncmp((char *)ptr, accept_line, strlen(accept_line)) == 0) {

- s->accept_range = true;

+ if (realsize >= strlen(accept_ranges)

+ && strncmp(header, accept_ranges, strlen(accept_ranges)) == 0) {

+

+ char *p = strchr(header, ':') + 1;

+

+ /* Skip whitespace between the header name and value. */

+ while (p < end && *p && g_ascii_isspace(*p)) {

+ p++;

+ }

+

+ if (end - p >= strlen(bytes)

+ && strncmp(p, bytes, strlen(bytes)) == 0) {

+

+ /* Check that there is nothing but whitespace after the value. */

+ p += strlen(bytes);

+ while (p < end && *p && g_ascii_isspace(*p)) {

+ p++;

+ }

+

+ if (p == end || !*p) {

+ s->accept_range = true;

+ }

+ }

return realsize;

2.24.1

From: Stefan Hajnoczi <stefanha@redhat.com>

The qcow2 .bdrv_measure() code calculates the crypto payload offset.

This logic really belongs in crypto/block.c where it can be reused by

other image formats.

The "luks" block driver will need this same logic in order to implement

.bdrv_measure(), so extract the qcrypto_block_calculate_payload_offset()

function now.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

Reviewed-by: Max Reitz <mreitz@redhat.com>

Message-Id: <20200221112522.1497712-2-stefanha@redhat.com>