The following changes since commit ba29883206d92a29ad5a466e679ccfc2ee6132ef:

Merge remote-tracking branch 'remotes/borntraeger/tags/s390x-20200310' into staging (2020-03-10 16:50:28 +0000)

are available in the Git repository at:

https://github.com/XanClic/qemu.git tags/pull-block-2020-03-11

for you to fetch changes up to 397f4e9d83e9c0000905f0a988ba1aeda162571c:

block/block-copy: hide structure definitions (2020-03-11 12:42:30 +0100)

Eric Blake (1):

iotests: Fix nonportable use of od --endian

Pan Nengyuan (2):

block/qcow2: do free crypto_opts in qcow2_close()

qemu-img: free memory before re-assign

luks: extract qcrypto_block_calculate_payload_offset()

luks: implement .bdrv_measure()

qemu-img: allow qemu-img measure --object without a filename

iotests: add 288 luks qemu-img measure test

Vladimir Sementsov-Ogievskiy (10):

block/qcow2-threads: fix qcow2_decompress

job: refactor progress to separate object

block/block-copy: fix progress calculation

block/block-copy: specialcase first copy_range request

block/block-copy: use block_status

block/block-copy: factor out find_conflicting_inflight_req

block/block-copy: refactor interfaces to use bytes instead of end

block/block-copy: rename start to offset in interfaces

block/block-copy: reduce intersecting request lock

block/block-copy: hide structure definitions

block/backup-top.c | 6 +-

block/backup.c | 38 ++-

block/block-copy.c | 405 ++++++++++++++++++++++++-------

block/crypto.c | 62 +++++

block/curl.c | 32 ++-

block/qcow2-threads.c | 12 +-

block/qcow2.c | 75 ++----

block/trace-events | 1 +

blockjob.c | 16 +-

crypto/block.c | 36 +++

include/block/block-copy.h | 65 +----

include/crypto/block.h | 22 ++

include/qemu/job.h | 11 +-

include/qemu/progress_meter.h | 58 +++++

job-qmp.c | 4 +-

job.c | 6 +-

qemu-img.c | 14 +-

tests/qemu-iotests/178 | 2 +-

tests/qemu-iotests/178.out.qcow2 | 8 +-

tests/qemu-iotests/178.out.raw | 8 +-

tests/qemu-iotests/288 | 93 +++++++

tests/qemu-iotests/288.out | 30 +++

tests/qemu-iotests/common.rc | 22 +-

tests/qemu-iotests/group | 1 +

24 files changed, 749 insertions(+), 278 deletions(-)

create mode 100644 include/qemu/progress_meter.h

create mode 100755 tests/qemu-iotests/288

create mode 100644 tests/qemu-iotests/288.out

2.24.1

From: Stefan Hajnoczi <stefanha@redhat.com>

The qcow2 .bdrv_measure() code calculates the crypto payload offset.

This logic really belongs in crypto/block.c where it can be reused by

other image formats.

The "luks" block driver will need this same logic in order to implement

.bdrv_measure(), so extract the qcrypto_block_calculate_payload_offset()

function now.

Reviewed-by: Max Reitz <mreitz@redhat.com>

Message-Id: <20200221112522.1497712-2-stefanha@redhat.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

block/qcow2.c | 74 +++++++++++-------------------------------

crypto/block.c | 36 ++++++++++++++++++++

include/crypto/block.h | 22 +++++++++++++

3 files changed, 77 insertions(+), 55 deletions(-)

@@ -XXX,XX +XXX,XX @@ static coroutine_fn int qcow2_co_flush_to_os(BlockDriverState *bs)

return ret;

}

-static ssize_t qcow2_measure_crypto_hdr_init_func(QCryptoBlock *block,

- size_t headerlen, void *opaque, Error **errp)

-{

- size_t *headerlenp = opaque;

-

- /* Stash away the payload size */

- *headerlenp = headerlen;

- return 0;

-}

-

-static ssize_t qcow2_measure_crypto_hdr_write_func(QCryptoBlock *block,

- size_t offset, const uint8_t *buf, size_t buflen,

- void *opaque, Error **errp)

-{

- /* Discard the bytes, we're not actually writing to an image */

- return buflen;

-}

-

-/* Determine the number of bytes for the LUKS payload */

-static bool qcow2_measure_luks_headerlen(QemuOpts *opts, size_t *len,

- Error **errp)

-{

- QDict *opts_qdict;

- QDict *cryptoopts_qdict;

- QCryptoBlockCreateOptions *cryptoopts;

- QCryptoBlock *crypto;

-

- /* Extract "encrypt." options into a qdict */

- opts_qdict = qemu_opts_to_qdict(opts, NULL);

- qdict_extract_subqdict(opts_qdict, &cryptoopts_qdict, "encrypt.");

- qobject_unref(opts_qdict);

-

- /* Build QCryptoBlockCreateOptions object from qdict */

- qdict_put_str(cryptoopts_qdict, "format", "luks");

- cryptoopts = block_crypto_create_opts_init(cryptoopts_qdict, errp);

- qobject_unref(cryptoopts_qdict);

- if (!cryptoopts) {

- return false;

- /* Fake LUKS creation in order to determine the payload size */

- crypto = qcrypto_block_create(cryptoopts, "encrypt.",

- qcow2_measure_crypto_hdr_init_func,

- qcow2_measure_crypto_hdr_write_func,

- len, errp);

- qapi_free_QCryptoBlockCreateOptions(cryptoopts);

- if (!crypto) {

- return false;

- }

-

- qcrypto_block_free(crypto);

- return true;

-}

-

static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs,

Error **errp)

{

@@ -XXX,XX +XXX,XX @@ static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs,

g_free(optstr);

if (has_luks) {

+ g_autoptr(QCryptoBlockCreateOptions) create_opts = NULL;

+ QDict *opts_qdict;

+ QDict *cryptoopts;

size_t headerlen;

- if (!qcow2_measure_luks_headerlen(opts, &headerlen, &local_err)) {

+ opts_qdict = qemu_opts_to_qdict(opts, NULL);

+ qdict_extract_subqdict(opts_qdict, &cryptoopts, "encrypt.");

+ qobject_unref(opts_qdict);

+

+ qdict_put_str(cryptoopts, "format", "luks");

+

+ create_opts = block_crypto_create_opts_init(cryptoopts, errp);

+ qobject_unref(cryptoopts);

+ if (!create_opts) {

+ goto err;

+ }

+

+ if (!qcrypto_block_calculate_payload_offset(create_opts,

+ "encrypt.",

+ &headerlen,

+ &local_err)) {

goto err;

}

diff --git a/crypto/block.c b/crypto/block.c

index XXXXXXX..XXXXXXX 100644

--- a/crypto/block.c

+++ b/crypto/block.c

@@ -XXX,XX +XXX,XX @@ QCryptoBlock *qcrypto_block_create(QCryptoBlockCreateOptions *options,

}

+static ssize_t qcrypto_block_headerlen_hdr_init_func(QCryptoBlock *block,

+ size_t headerlen, void *opaque, Error **errp)

+{

+ size_t *headerlenp = opaque;

+

+ /* Stash away the payload size */

+ *headerlenp = headerlen;

+ return 0;

+}

+

+

+static ssize_t qcrypto_block_headerlen_hdr_write_func(QCryptoBlock *block,

+ size_t offset, const uint8_t *buf, size_t buflen,

+ void *opaque, Error **errp)

+{

+ /* Discard the bytes, we're not actually writing to an image */

+ return buflen;

+}

+

+

+bool

+qcrypto_block_calculate_payload_offset(QCryptoBlockCreateOptions *create_opts,

+ const char *optprefix,

+ size_t *len,

+ Error **errp)

+{

+ /* Fake LUKS creation in order to determine the payload size */

+ g_autoptr(QCryptoBlock) crypto =

+ qcrypto_block_create(create_opts, optprefix,

+ qcrypto_block_headerlen_hdr_init_func,

+ qcrypto_block_headerlen_hdr_write_func,

+ len, errp);

+ return crypto != NULL;

+}

+

+

QCryptoBlockInfo *qcrypto_block_get_info(QCryptoBlock *block,

Error **errp)

{

diff --git a/include/crypto/block.h b/include/crypto/block.h

index XXXXXXX..XXXXXXX 100644

--- a/include/crypto/block.h

+++ b/include/crypto/block.h

@@ -XXX,XX +XXX,XX @@ QCryptoBlock *qcrypto_block_create(QCryptoBlockCreateOptions *options,

Error **errp);

+/**

+ * qcrypto_block_calculate_payload_offset:

+ * @create_opts: the encryption options

+ * @optprefix: name prefix for options

+ * @len: output for number of header bytes before payload

+ * @errp: pointer to a NULL-initialized error object

+ *

+ * Calculate the number of header bytes before the payload in an encrypted

+ * storage volume. The header is an area before the payload that is reserved

+ * for encryption metadata.

+ *

+ * Returns: true on success, false on error

+ */

+bool

+qcrypto_block_calculate_payload_offset(QCryptoBlockCreateOptions *create_opts,

+ const char *optprefix,

+ size_t *len,

+ Error **errp);

+

+

/**

* qcrypto_block_get_info:

* @block: the block encryption object

@@ -XXX,XX +XXX,XX @@ uint64_t qcrypto_block_get_sector_size(QCryptoBlock *block);

void qcrypto_block_free(QCryptoBlock *block);

G_DEFINE_AUTOPTR_CLEANUP_FUNC(QCryptoBlock, qcrypto_block_free)

+G_DEFINE_AUTOPTR_CLEANUP_FUNC(QCryptoBlockCreateOptions,

+ qapi_free_QCryptoBlockCreateOptions)

#endif /* QCRYPTO_BLOCK_H */

2.24.1

From: Stefan Hajnoczi <stefanha@redhat.com>

Add qemu-img measure support in the "luks" block driver.

Reviewed-by: Max Reitz <mreitz@redhat.com>

Message-Id: <20200221112522.1497712-3-stefanha@redhat.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

block/crypto.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++

1 file changed, 62 insertions(+)

diff --git a/block/crypto.c b/block/crypto.c

--- a/block/crypto.c

+++ b/block/crypto.c

@@ -XXX,XX +XXX,XX @@ static int64_t block_crypto_getlength(BlockDriverState *bs)

}

+static BlockMeasureInfo *block_crypto_measure(QemuOpts *opts,

+ BlockDriverState *in_bs,

+ Error **errp)

+{

+ g_autoptr(QCryptoBlockCreateOptions) create_opts = NULL;

+ Error *local_err = NULL;

+ BlockMeasureInfo *info;

+ uint64_t size;

+ size_t luks_payload_size;

+ QDict *cryptoopts;

+

+ /*

+ * Preallocation mode doesn't affect size requirements but we must consume

+ * the option.

+ */

+ g_free(qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC));

+

+ size = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0);

+

+ if (in_bs) {

+ int64_t ssize = bdrv_getlength(in_bs);

+

+ if (ssize < 0) {

+ error_setg_errno(&local_err, -ssize,

+ "Unable to get image virtual_size");

+ goto err;

+ }

+

+ size = ssize;

+ }

+

+ cryptoopts = qemu_opts_to_qdict_filtered(opts, NULL,

+ &block_crypto_create_opts_luks, true);

+ qdict_put_str(cryptoopts, "format", "luks");

+ create_opts = block_crypto_create_opts_init(cryptoopts, &local_err);

+ qobject_unref(cryptoopts);

+ if (!create_opts) {

+ goto err;

+ }

+

+ if (!qcrypto_block_calculate_payload_offset(create_opts, NULL,

+ &luks_payload_size,

+ &local_err)) {

+ goto err;

+ }

+

+ /*

+ * Unallocated blocks are still encrypted so allocation status makes no

+ * difference to the file size.

+ */

+ info = g_new(BlockMeasureInfo, 1);

+ info->fully_allocated = luks_payload_size + size;

+ info->required = luks_payload_size + size;

+ return info;

+

+err:

+ error_propagate(errp, local_err);

+ return NULL;

+}

+

+

static int block_crypto_probe_luks(const uint8_t *buf,

int buf_size,

const char *filename) {

@@ -XXX,XX +XXX,XX @@ static BlockDriver bdrv_crypto_luks = {

.bdrv_co_preadv = block_crypto_co_preadv,

.bdrv_co_pwritev = block_crypto_co_pwritev,

.bdrv_getlength = block_crypto_getlength,

+ .bdrv_measure = block_crypto_measure,

.bdrv_get_info = block_crypto_get_info_luks,

.bdrv_get_specific_info = block_crypto_get_specific_info_luks,

2.24.1

From: Stefan Hajnoczi <stefanha@redhat.com>

In most qemu-img sub-commands the --object option only makes sense when

there is a filename. qemu-img measure is an exception because objects

may be referenced from the image creation options instead of an existing

image file. Allow --object without a filename.

Reviewed-by: Max Reitz <mreitz@redhat.com>

Message-Id: <20200221112522.1497712-4-stefanha@redhat.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

qemu-img.c | 6 ++----

tests/qemu-iotests/178 | 2 +-

tests/qemu-iotests/178.out.qcow2 | 8 ++++----

tests/qemu-iotests/178.out.raw | 8 ++++----

4 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/qemu-img.c b/qemu-img.c

index XXXXXXX..XXXXXXX 100644

--- a/qemu-img.c

+++ b/qemu-img.c

@@ -XXX,XX +XXX,XX @@ static int img_measure(int argc, char **argv)

filename = argv[optind];

}

- if (!filename &&

- (object_opts || image_opts || fmt || snapshot_name || sn_opts)) {

- error_report("--object, --image-opts, -f, and -l "

- "require a filename argument.");

+ if (!filename && (image_opts || fmt || snapshot_name || sn_opts)) {

+ error_report("--image-opts, -f, and -l require a filename argument.");

goto out;

}

if (filename && img_size != UINT64_MAX) {

diff --git a/tests/qemu-iotests/178 b/tests/qemu-iotests/178

--- a/tests/qemu-iotests/178

+++ b/tests/qemu-iotests/178

@@ -XXX,XX +XXX,XX @@ _make_test_img 1G

$QEMU_IMG measure # missing arguments

$QEMU_IMG measure --size 2G "$TEST_IMG" # only one allowed

$QEMU_IMG measure "$TEST_IMG" a # only one filename allowed

-$QEMU_IMG measure --object secret,id=sec0,data=MTIzNDU2,format=base64 # missing filename

+$QEMU_IMG measure --object secret,id=sec0,data=MTIzNDU2,format=base64 # size or filename needed

$QEMU_IMG measure --image-opts # missing filename

$QEMU_IMG measure -f qcow2 # missing filename

$QEMU_IMG measure -l snap1 # missing filename

diff --git a/tests/qemu-iotests/178.out.qcow2 b/tests/qemu-iotests/178.out.qcow2

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/178.out.qcow2

+++ b/tests/qemu-iotests/178.out.qcow2

@@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824

qemu-img: Either --size N or one filename must be specified.

qemu-img: --size N cannot be used together with a filename.

qemu-img: At most one filename argument is allowed.

-qemu-img: --object, --image-opts, -f, and -l require a filename argument.

-qemu-img: --object, --image-opts, -f, and -l require a filename argument.

-qemu-img: --object, --image-opts, -f, and -l require a filename argument.

-qemu-img: --object, --image-opts, -f, and -l require a filename argument.

+qemu-img: Either --size N or one filename must be specified.

+qemu-img: --image-opts, -f, and -l require a filename argument.

+qemu-img: --image-opts, -f, and -l require a filename argument.

+qemu-img: --image-opts, -f, and -l require a filename argument.

qemu-img: Invalid option list: ,

qemu-img: Invalid parameter 'snapshot.foo'

qemu-img: Failed in parsing snapshot param 'snapshot.foo'

diff --git a/tests/qemu-iotests/178.out.raw b/tests/qemu-iotests/178.out.raw

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/178.out.raw

+++ b/tests/qemu-iotests/178.out.raw

@@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824

qemu-img: Either --size N or one filename must be specified.

qemu-img: --size N cannot be used together with a filename.

qemu-img: At most one filename argument is allowed.

-qemu-img: --object, --image-opts, -f, and -l require a filename argument.

-qemu-img: --object, --image-opts, -f, and -l require a filename argument.

-qemu-img: --object, --image-opts, -f, and -l require a filename argument.

-qemu-img: --object, --image-opts, -f, and -l require a filename argument.

+qemu-img: Either --size N or one filename must be specified.

+qemu-img: --image-opts, -f, and -l require a filename argument.

+qemu-img: --image-opts, -f, and -l require a filename argument.

+qemu-img: --image-opts, -f, and -l require a filename argument.

qemu-img: Invalid option list: ,

qemu-img: Invalid parameter 'snapshot.foo'

qemu-img: Failed in parsing snapshot param 'snapshot.foo'

2.24.1

From: Stefan Hajnoczi <stefanha@redhat.com>

This test exercises the block/crypto.c "luks" block driver

.bdrv_measure() code.

Reviewed-by: Max Reitz <mreitz@redhat.com>

Message-Id: <20200221112522.1497712-5-stefanha@redhat.com>

[mreitz: Renamed test from 282 to 288]

Signed-off-by: Max Reitz <mreitz@redhat.com>

tests/qemu-iotests/288 | 93 ++++++++++++++++++++++++++++++++++++++

tests/qemu-iotests/288.out | 30 ++++++++++++

tests/qemu-iotests/group | 1 +

3 files changed, 124 insertions(+)

create mode 100755 tests/qemu-iotests/288

create mode 100644 tests/qemu-iotests/288.out

diff --git a/tests/qemu-iotests/288 b/tests/qemu-iotests/288

new file mode 100755

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/tests/qemu-iotests/288

@@ -XXX,XX +XXX,XX @@

+#!/usr/bin/env bash

+#

+# qemu-img measure tests for LUKS images

+#

+# Copyright (C) 2020 Red Hat, Inc.

+#

+# This program is free software; you can redistribute it and/or modify

+# it under the terms of the GNU General Public License as published by

+# the Free Software Foundation; either version 2 of the License, or

+# (at your option) any later version.

+#

+# This program is distributed in the hope that it will be useful,

+# but WITHOUT ANY WARRANTY; without even the implied warranty of

+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

+# GNU General Public License for more details.

+#

+# You should have received a copy of the GNU General Public License

+# along with this program. If not, see <http://www.gnu.org/licenses/>.

+#

+

+# creator

+owner=stefanha@redhat.com

+

+seq=`basename $0`

+echo "QA output created by $seq"

+

+status=1 # failure is the default!

+

+_cleanup()

+{

+ _cleanup_test_img

+ rm -f "$TEST_IMG.converted"

+}

+trap "_cleanup; exit \$status" 0 1 2 3 15

+

+# get standard environment, filters and checks

+. ./common.rc

+. ./common.filter

+. ./common.pattern

+

+_supported_fmt luks

+_supported_proto file

+_supported_os Linux

+

+SECRET=secret,id=sec0,data=passphrase

+

+echo "== measure 1G image file =="

+echo

+

+$QEMU_IMG measure --object "$SECRET" \

+     -O "$IMGFMT" \

+         -o key-secret=sec0,iter-time=10 \

+         --size 1G

+

+echo

+echo "== create 1G image file (size should be no greater than measured) =="

+echo

+

+_make_test_img 1G

+stat -c "image file size in bytes: %s" "$TEST_IMG_FILE"

+

+echo

+echo "== modified 1G image file (size should be no greater than measured) =="

+echo

+

+$QEMU_IO --object "$SECRET" --image-opts "$TEST_IMG" -c "write -P 0x51 0x10000 0x400" | _filter_qemu_io | _filter_testdir

+stat -c "image file size in bytes: %s" "$TEST_IMG_FILE"

+

+echo

+echo "== measure preallocation=falloc 1G image file =="

+echo

+

+$QEMU_IMG measure --object "$SECRET" \

+     -O "$IMGFMT" \

+         -o key-secret=sec0,iter-time=10,preallocation=falloc \

+         --size 1G

+

+echo

+echo "== measure with input image file =="

+echo

+

+IMGFMT=raw IMGKEYSECRET= IMGOPTS= _make_test_img 1G | _filter_imgfmt

+QEMU_IO_OPTIONS= IMGOPTSSYNTAX= $QEMU_IO -f raw -c "write -P 0x51 0x10000 0x400" "$TEST_IMG_FILE" | _filter_qemu_io | _filter_testdir

+$QEMU_IMG measure --object "$SECRET" \

+     -O "$IMGFMT" \

+         -o key-secret=sec0,iter-time=10 \

+         -f raw \

+         "$TEST_IMG_FILE"

+

+# success, all done

+echo "*** done"

+rm -f $seq.full

+status=0

diff --git a/tests/qemu-iotests/288.out b/tests/qemu-iotests/288.out

+++ b/tests/qemu-iotests/288.out

+QA output created by 288

+== measure 1G image file ==

+required size: 1075810304

+fully allocated size: 1075810304

+== create 1G image file (size should be no greater than measured) ==

+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824

+image file size in bytes: 1075810304

+== modified 1G image file (size should be no greater than measured) ==

+wrote 1024/1024 bytes at offset 65536

+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)

+image file size in bytes: 1075810304

+== measure preallocation=falloc 1G image file ==

+required size: 1075810304

+fully allocated size: 1075810304

+== measure with input image file ==

+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824

+wrote 1024/1024 bytes at offset 65536

+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)

+required size: 1075810304

+fully allocated size: 1075810304

+*** done

diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/group

+++ b/tests/qemu-iotests/group

283 auto quick

284 rw

286 rw quick

+288 quick