The following changes since commit ba29883206d92a29ad5a466e679ccfc2ee6132ef:

Merge remote-tracking branch 'remotes/borntraeger/tags/s390x-20200310' into staging (2020-03-10 16:50:28 +0000)

are available in the Git repository at:

for you to fetch changes up to 8bb3b023f2055054ee119cb45b42d2b14be7fc8a:

qemu-iotests: adding LUKS cleanup for non-UTF8 secret error (2020-03-11 15:54:38 +0100)

- Relax restrictions for blockdev-snapshot (allows libvirt to do live

storage migration with blockdev-mirror)

- luks: Delete created files when block_crypto_co_create_opts_luks fails

- Fix memleaks in qmp_object_add

Daniel Henrique Barboza (4):

block: introducing 'bdrv_co_delete_file' interface

block.c: adding bdrv_co_delete_file

crypto.c: cleanup created file when block_crypto_co_create_opts_luks fails

qemu-iotests: adding LUKS cleanup for non-UTF8 secret error

Kevin Wolf (6):

block: Make bdrv_get_cumulative_perm() public

block: Relax restrictions for blockdev-snapshot

iotests: Fix run_job() with use_log=False

iotests: Test mirror with temporarily disabled target backing file

block: Fix cross-AioContext blockdev-snapshot

iotests: Add iothread cases to 155

Pan Nengyuan (1):

qom-qmp-cmds: fix two memleaks in qmp_object_add

Peter Krempa (1):

qapi: Add '@allow-write-only-overlay' feature for 'blockdev-snapshot'

Philippe Mathieu-Daudé (1):

tests/qemu-iotests: Fix socket_scm_helper build path

qapi/block-core.json | 9 ++++-

include/block/block.h | 1 +

include/block/block_int.h | 7 ++++

block.c | 33 ++++++++++++++--

block/crypto.c | 18 +++++++++

block/file-posix.c | 23 +++++++++++

blockdev.c | 30 ++++-----------

qom/qom-qmp-cmds.c | 16 +++-----

tests/qemu-iotests/iotests.py | 5 ++-

tests/Makefile.include | 1 +

tests/qemu-iotests/085.out | 4 +-

tests/qemu-iotests/155 | 88 ++++++++++++++++++++++++++++++++++++-------

tests/qemu-iotests/155.out | 4 +-

tests/qemu-iotests/282 | 67 ++++++++++++++++++++++++++++++++

tests/qemu-iotests/282.out | 11 ++++++

tests/qemu-iotests/group | 1 +

tests/qtest/Makefile.include | 1 -

17 files changed, 262 insertions(+), 57 deletions(-)

create mode 100755 tests/qemu-iotests/282

create mode 100644 tests/qemu-iotests/282.out

blockdev-snapshot returned an error if the overlay was already in use,

which it defined as having any BlockBackend parent. This is in fact both

too strict (some parents can tolerate the change of visible data caused

by attaching a backing file) and too loose (some non-BlockBackend

parents may not be happy with it).

One important use case that is prevented by the too strict check is live

storage migration with blockdev-mirror. Here, the target node is

usually opened without a backing file so that the active layer is

mirrored while its backing chain can be copied in the background.

The backing chain should be attached to the mirror target node when

finalising the job, just before switching the users of the source node

to the new copy (at which point the mirror job still has a reference to

the node). drive-mirror did this automatically, but with blockdev-mirror

this is the job of the QMP client, so it needs a way to do this.

blockdev-snapshot is the obvious way, so this patch makes it work in

this scenario. The new condition is that no parent uses CONSISTENT_READ

permissions. This will ensure that the operation will still be blocked

when the node is attached to the guest device, so blockdev-snapshot

remains safe.

(For the sake of completeness, x-blockdev-reopen can be used to achieve

the same, however it is a big hammer, performs the graph change

completely unchecked and is still experimental. So even with the option

of using x-blockdev-reopen, there are reasons why blockdev-snapshot

should be able to perform this operation.)

Signed-off-by: Kevin Wolf <kwolf@redhat.com>

Message-Id: <20200310113831.27293-3-kwolf@redhat.com>

Reviewed-by: Peter Krempa <pkrempa@redhat.com>

Tested-by: Peter Krempa <pkrempa@redhat.com>

blockdev.c | 14 ++++++++------

tests/qemu-iotests/085.out | 4 ++--

2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/blockdev.c b/blockdev.c

--- a/blockdev.c

+++ b/blockdev.c

@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,

TransactionAction *action = common->action;

AioContext *aio_context;

AioContext *old_context;

+ uint64_t perm, shared;

int ret;

/* 'blockdev-snapshot' and 'blockdev-snapshot-sync' have similar

@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,

goto out;

}

- if (bdrv_has_blk(state->new_bs)) {

+ /*

+ * Allow attaching a backing file to an overlay that's already in use only

+ * if the parents don't assume that they are already seeing a valid image.

+ * (Specifically, allow it as a mirror target, which is write-only access.)

+ */

+ bdrv_get_cumulative_perm(state->new_bs, &perm, &shared);

+ if (perm & BLK_PERM_CONSISTENT_READ) {

error_setg(errp, "The overlay is already in use");

goto out;

}

- if (bdrv_op_is_blocked(state->new_bs, BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT,

- errp)) {

- goto out;

- }

-

if (state->new_bs->backing != NULL) {

error_setg(errp, "The overlay already has a backing image");

goto out;

diff --git a/tests/qemu-iotests/085.out b/tests/qemu-iotests/085.out

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/085.out

+++ b/tests/qemu-iotests/085.out

@@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/12-snapshot-v0.IMGFMT', fmt=IMGFMT size=134217728 backing_f

=== Invalid command - cannot create a snapshot using a file BDS ===

{ 'execute': 'blockdev-snapshot', 'arguments': { 'node':'virtio0', 'overlay':'file_12' } }

-{"error": {"class": "GenericError", "desc": "The overlay does not support backing images"}}

+{"error": {"class": "GenericError", "desc": "The overlay is already in use"}}

=== Invalid command - snapshot node used as active layer ===

@@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/12-snapshot-v0.IMGFMT', fmt=IMGFMT size=134217728 backing_f

=== Invalid command - snapshot node used as backing hd ===

{ 'execute': 'blockdev-snapshot', 'arguments': { 'node': 'virtio0', 'overlay':'snap_11' } }

-{"error": {"class": "GenericError", "desc": "Node 'snap_11' is busy: node is used as backing hd of 'snap_12'"}}

+{"error": {"class": "GenericError", "desc": "The overlay is already in use"}}

=== Invalid command - snapshot node has a backing image ===

2.20.1

From: Peter Krempa <pkrempa@redhat.com>

Anounce that 'blockdev-snapshot' command's permissions allow changing

of the backing file if the 'consistent_read' permission is not required.

This is useful for libvirt to allow late opening of the backing chain

during a blockdev-mirror.

Signed-off-by: Peter Krempa <pkrempa@redhat.com>

Signed-off-by: Kevin Wolf <kwolf@redhat.com>

Message-Id: <20200310113831.27293-8-kwolf@redhat.com>

qapi/block-core.json | 9 ++++++++-

1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json

index XXXXXXX..XXXXXXX 100644

--- a/qapi/block-core.json

+++ b/qapi/block-core.json

@@ -XXX,XX +XXX,XX @@

#

# For the arguments, see the documentation of BlockdevSnapshot.

#

+# Features:

+# @allow-write-only-overlay: If present, the check whether this operation is safe

+# was relaxed so that it can be used to change

+# backing file of a destination of a blockdev-mirror.

+# (since 5.0)

+#

# Since: 2.5

#

# Example:

@@ -XXX,XX +XXX,XX @@

#

##

{ 'command': 'blockdev-snapshot',

- 'data': 'BlockdevSnapshot' }

+ 'data': 'BlockdevSnapshot',

+ 'features': [ 'allow-write-only-overlay' ] }

##

# @change-backing-file:

2.20.1

From: Philippe Mathieu-Daudé <philmd@redhat.com>

The socket_scm_helper path got corrupted during the mechanical

refactor moving the qtests files into their own sub-directory.

Fixes: 1e8a1fae7 ("test: Move qtests to a separate directory")

Signed-off-by: Philippe Mathieu-Daudé <philmd@redhat.com>

Message-Id: <20200306165751.18986-1-philmd@redhat.com>

Reviewed-by: Laurent Vivier <laurent@vivier.eu>

tests/Makefile.include | 1 +

tests/qtest/Makefile.include | 1 -

2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/Makefile.include b/tests/Makefile.include

index XXXXXXX..XXXXXXX 100644

--- a/tests/Makefile.include

+++ b/tests/Makefile.include

@@ -XXX,XX +XXX,XX @@ include $(SRC_PATH)/tests/qtest/Makefile.include

tests/test-qga$(EXESUF): qemu-ga$(EXESUF)

tests/test-qga$(EXESUF): tests/test-qga.o $(qtest-obj-y)

tests/vhost-user-bridge$(EXESUF): tests/vhost-user-bridge.o $(test-util-obj-y) libvhost-user.a

+tests/qemu-iotests/socket_scm_helper$(EXESUF): tests/qemu-iotests/socket_scm_helper.o

SPEED = quick

diff --git a/tests/qtest/Makefile.include b/tests/qtest/Makefile.include

index XXXXXXX..XXXXXXX 100644

--- a/tests/qtest/Makefile.include

+++ b/tests/qtest/Makefile.include

@@ -XXX,XX +XXX,XX @@ tests/qtest/usb-hcd-ehci-test$(EXESUF): tests/qtest/usb-hcd-ehci-test.o $(libqos

tests/qtest/usb-hcd-xhci-test$(EXESUF): tests/qtest/usb-hcd-xhci-test.o $(libqos-usb-obj-y)

tests/qtest/cpu-plug-test$(EXESUF): tests/qtest/cpu-plug-test.o

tests/qtest/migration-test$(EXESUF): tests/qtest/migration-test.o tests/qtest/migration-helpers.o

-tests/qtest/qemu-iotests/qtest/socket_scm_helper$(EXESUF): tests/qtest/qemu-iotests/qtest/socket_scm_helper.o

tests/qtest/test-netfilter$(EXESUF): tests/qtest/test-netfilter.o $(qtest-obj-y)

tests/qtest/test-filter-mirror$(EXESUF): tests/qtest/test-filter-mirror.o $(qtest-obj-y)

tests/qtest/test-filter-redirector$(EXESUF): tests/qtest/test-filter-redirector.o $(qtest-obj-y)

2.20.1

From: Daniel Henrique Barboza <danielhb413@gmail.com>

Adding to Block Drivers the capability of being able to clean up

its created files can be useful in certain situations. For the

LUKS driver, for instance, a failure in one of its authentication

steps can leave files in the host that weren't there before.

This patch adds the 'bdrv_co_delete_file' interface to block

drivers and add it to the 'file' driver in file-posix.c. The

implementation is given by 'raw_co_delete_file'.

Suggested-by: Daniel P. Berrangé <berrange@redhat.com>

Signed-off-by: Daniel Henrique Barboza <danielhb413@gmail.com>

Message-Id: <20200130213907.2830642-2-danielhb413@gmail.com>

include/block/block_int.h | 4 ++++

block/file-posix.c | 23 +++++++++++++++++++++++

2 files changed, 27 insertions(+)

diff --git a/include/block/block_int.h b/include/block/block_int.h

--- a/include/block/block_int.h

+++ b/include/block/block_int.h

@@ -XXX,XX +XXX,XX @@ struct BlockDriver {

*/

int coroutine_fn (*bdrv_co_flush)(BlockDriverState *bs);

+ /* Delete a created file. */

+ int coroutine_fn (*bdrv_co_delete_file)(BlockDriverState *bs,

+ Error **errp);

+

/*

* Flushes all data that was already written to the OS all the way down to

* the disk (for example file-posix.c calls fsync()).

diff --git a/block/file-posix.c b/block/file-posix.c

index XXXXXXX..XXXXXXX 100644

--- a/block/file-posix.c

+++ b/block/file-posix.c

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn raw_co_create_opts(const char *filename, QemuOpts *opts,

return raw_co_create(&options, errp);

+static int coroutine_fn raw_co_delete_file(BlockDriverState *bs,

+ Error **errp)

+{

+ struct stat st;

+ int ret;

+ if (!(stat(bs->filename, &st) == 0) || !S_ISREG(st.st_mode)) {

+ error_setg_errno(errp, ENOENT, "%s is not a regular file",

+ bs->filename);

+ return -ENOENT;

+ }

+

+ ret = unlink(bs->filename);

+ if (ret < 0) {

+ ret = -errno;

+ error_setg_errno(errp, -ret, "Error when deleting file %s",

+ bs->filename);

+ }

+

+ return ret;

+}

+

/*

* Find allocation range in @bs around offset @start.

* May change underlying file descriptor's file offset.

@@ -XXX,XX +XXX,XX @@ BlockDriver bdrv_file = {

.bdrv_co_block_status = raw_co_block_status,

.bdrv_co_invalidate_cache = raw_co_invalidate_cache,

.bdrv_co_pwrite_zeroes = raw_co_pwrite_zeroes,

+ .bdrv_co_delete_file = raw_co_delete_file,

.bdrv_co_preadv = raw_co_preadv,

.bdrv_co_pwritev = raw_co_pwritev,

2.20.1

From: Daniel Henrique Barboza <danielhb413@gmail.com>

Using the new 'bdrv_co_delete_file' interface, a pure co_routine function

'bdrv_co_delete_file' inside block.c can can be used in a way similar of

the existing bdrv_create_file to to clean up a created file.

We're creating a pure co_routine because the only caller of

'bdrv_co_delete_file' will be already in co_routine context, thus there

is no need to add all the machinery to check for qemu_in_coroutine() and

create a separated co_routine to do the job.

Suggested-by: Daniel P. Berrangé <berrange@redhat.com>

Signed-off-by: Daniel Henrique Barboza <danielhb413@gmail.com>

Message-Id: <20200130213907.2830642-3-danielhb413@gmail.com>

include/block/block.h | 1 +

block.c | 26 ++++++++++++++++++++++++++

2 files changed, 27 insertions(+)

@@ -XXX,XX +XXX,XX @@ int bdrv_create_file(const char *filename, QemuOpts *opts, Error **errp)

}

+int coroutine_fn bdrv_co_delete_file(BlockDriverState *bs, Error **errp)

+{

+ Error *local_err = NULL;

+ int ret;

+

+ assert(bs != NULL);

+

+ if (!bs->drv) {

+ error_setg(errp, "Block node '%s' is not opened", bs->filename);

+ return -ENOMEDIUM;

+ }

+

+ if (!bs->drv->bdrv_co_delete_file) {

+ error_setg(errp, "Driver '%s' does not support image deletion",

+ bs->drv->format_name);

+ return -ENOTSUP;

+ }

+

+ ret = bs->drv->bdrv_co_delete_file(bs, &local_err);

+ if (ret < 0) {

+ error_propagate(errp, local_err);

+ }

+

+ return ret;

+}

+

/**

* Try to get @bs's logical and physical block size.

* On success, store them in @bsz struct and return 0.

2.20.1

From: Daniel Henrique Barboza <danielhb413@gmail.com>

When using a non-UTF8 secret to create a volume using qemu-img, the

following error happens:

$ qemu-img create -f luks --object secret,id=vol_1_encrypt0,file=vol_resize_pool.vol_1.secret.qzVQrI -o key-secret=vol_1_encrypt0 /var/tmp/pool_target/vol_1 10240K

Formatting '/var/tmp/pool_target/vol_1', fmt=luks size=10485760 key-secret=vol_1_encrypt0

qemu-img: /var/tmp/pool_target/vol_1: Data from secret vol_1_encrypt0 is not valid UTF-8

However, the created file '/var/tmp/pool_target/vol_1' is left behind in the

file system after the failure. This behavior can be observed when creating

the volume using Libvirt, via 'virsh vol-create', and then getting "volume

target path already exist" errors when trying to re-create the volume.

The volume file is created inside block_crypto_co_create_opts_luks(), in

block/crypto.c. If the bdrv_create_file() call is successful but any

succeeding step fails*, the existing 'fail' label does not take into

account the created file, leaving it behind.

This patch changes block_crypto_co_create_opts_luks() to delete

'filename' in case of failure. A failure in this point means that

the volume is now truncated/corrupted, so even if 'filename' was an

existing volume before calling qemu-img, it is now unusable. Deleting

the file it is not much worse than leaving it in the filesystem in

this scenario, and we don't have to deal with checking the file

pre-existence in the code.

* in our case, block_crypto_co_create_generic calls qcrypto_block_create,

which calls qcrypto_block_luks_create, and this function fails when

calling qcrypto_secret_lookup_as_utf8.

Reported-by: Srikanth Aithal <bssrikanth@in.ibm.com>

Suggested-by: Kevin Wolf <kwolf@redhat.com>

Signed-off-by: Daniel Henrique Barboza <danielhb413@gmail.com>

Message-Id: <20200130213907.2830642-4-danielhb413@gmail.com>

block/crypto.c | 18 ++++++++++++++++++

1 file changed, 18 insertions(+)

diff --git a/block/crypto.c b/block/crypto.c

index XXXXXXX..XXXXXXX 100644

--- a/block/crypto.c

+++ b/block/crypto.c

@@ -XXX,XX +XXX,XX @@

#include "qapi/error.h"

#include "qemu/module.h"

#include "qemu/option.h"

+#include "qemu/cutils.h"

#include "crypto.h"

typedef struct BlockCrypto BlockCrypto;

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_crypto_co_create_opts_luks(const char *filename,

ret = 0;

fail:

+ /*

+ * If an error occurred, delete 'filename'. Even if the file existed

+ * beforehand, it has been truncated and corrupted in the process.

+ */

+ if (ret && bs) {

+ Error *local_delete_err = NULL;

+ int r_del = bdrv_co_delete_file(bs, &local_delete_err);

+ /*

+ * ENOTSUP will happen if the block driver doesn't support

+ * the 'bdrv_co_delete_file' interface. This is a predictable

+ * scenario and shouldn't be reported back to the user.

+ */

+ if ((r_del < 0) && (r_del != -ENOTSUP)) {

+ error_report_err(local_delete_err);

+ }

+ }

+

bdrv_unref(bs);

qapi_free_QCryptoBlockCreateOptions(create_opts);

qobject_unref(cryptoopts);

2.20.1

From: Daniel Henrique Barboza <danielhb413@gmail.com>

tests/qemu-iotests/282 | 67 ++++++++++++++++++++++++++++++++++++++

tests/qemu-iotests/282.out | 11 +++++++

tests/qemu-iotests/group | 1 +

3 files changed, 79 insertions(+)

create mode 100755 tests/qemu-iotests/282

create mode 100644 tests/qemu-iotests/282.out

diff --git a/tests/qemu-iotests/282 b/tests/qemu-iotests/282

new file mode 100755

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/tests/qemu-iotests/282

@@ -XXX,XX +XXX,XX @@

+#!/usr/bin/env bash

+#

+# Test qemu-img file cleanup for LUKS when using a non-UTF8 secret

+#

+# Copyright (C) 2020, IBM Corporation.

+#

+# This program is free software; you can redistribute it and/or modify

+# it under the terms of the GNU General Public License as published by

+# the Free Software Foundation; either version 2 of the License, or

+# (at your option) any later version.

+#

+# This program is distributed in the hope that it will be useful,

+# but WITHOUT ANY WARRANTY; without even the implied warranty of

+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

+# GNU General Public License for more details.

+#

+# You should have received a copy of the GNU General Public License

+# along with this program. If not, see <http://www.gnu.org/licenses/>.

+#

+

+seq=`basename $0`

+echo "QA output created by $seq"

+

+status=1    # failure is the default!

+TEST_IMAGE_FILE='vol.img'

+

+_cleanup()

+{

+ _cleanup_test_img

+ rm non_utf8_secret

+ rm -f $TEST_IMAGE_FILE

+}

+trap "_cleanup; exit \$status" 0 1 2 3 15

+

+# get standard environment, filters and checks

+. ./common.rc

+. ./common.filter

+

+_supported_fmt luks

+_supported_proto generic

+_unsupported_proto vxhs

+

+echo "== Create non-UTF8 secret =="

+echo -n -e '\x3a\x3c\x3b\xff' > non_utf8_secret

+SECRET="secret,id=sec0,file=non_utf8_secret"

+

+echo "== Throws an error because of invalid UTF-8 secret =="

+$QEMU_IMG create -f $IMGFMT --object $SECRET -o "key-secret=sec0" $TEST_IMAGE_FILE 4M

+

+echo "== Image file should not exist after the error =="

+if test -f "$TEST_IMAGE_FILE"; then

+ exit 1

+fi

+

+echo "== Create a stub image file and run qemu-img again =="

+touch $TEST_IMAGE_FILE

+$QEMU_IMG create -f $IMGFMT --object $SECRET -o "key-secret=sec0" $TEST_IMAGE_FILE 4M

+

+echo "== Pre-existing image file should also be deleted after the error =="

+if test -f "$TEST_IMAGE_FILE"; then

+ exit 1

+fi

+

+# success, all done

+echo "*** done"

+rm -f $seq.full

+status=0

diff --git a/tests/qemu-iotests/282.out b/tests/qemu-iotests/282.out

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/tests/qemu-iotests/282.out

@@ -XXX,XX +XXX,XX @@

+QA output created by 282

+== Create non-UTF8 secret ==

+== Throws an error because of invalid UTF-8 secret ==

+qemu-img: vol.img: Data from secret sec0 is not valid UTF-8

+Formatting 'vol.img', fmt=luks size=4194304 key-secret=sec0

+== Image file should not exist after the error ==

+== Create a stub image file and run qemu-img again ==

+qemu-img: vol.img: Data from secret sec0 is not valid UTF-8

+Formatting 'vol.img', fmt=luks size=4194304 key-secret=sec0

+== Pre-existing image file should also be deleted after the error ==

+ *** done

diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group

--- a/tests/qemu-iotests/group

+++ b/tests/qemu-iotests/group

@@ -XXX,XX +XXX,XX @@

279 rw backing quick

280 rw migration quick

281 rw quick

+282 rw img quick

283 auto quick

284 rw

286 rw quick