The following changes since commit 5ec2eca83dc478ddf24077e02a8b34dd26cd3ff9:

Merge remote-tracking branch 'remotes/awilliam/tags/vfio-updates-20190613.0' into staging (2019-06-14 09:33:55 +0100)

https://github.com/XanClic/qemu.git tags/pull-block-2019-06-14

for you to fetch changes up to 21c1ce592a144188dfe59b9e156a97da412a59a2:

iotests: Test qemu-img convert -C --salvage (2019-06-14 15:09:42 +0200)

Block patches:

- Allow blockdev-backup from nodes that are not in qemu's main AIO

context to newly added nodes

- Add salvaging mode to qemu-img convert

- Minor fixes to tests, documentation, and for less Valgrind annoyance

John Snow (6):

blockdev-backup: don't check aio_context too early

iotests.py: do not use infinite waits

QEMUMachine: add events_wait method

iotests.py: rewrite run_job to be pickier

iotests: add iotest 256 for testing blockdev-backup across iothread

contexts

event_match: always match on None value

Max Reitz (12):

iotests: Filter 175's allocation information

iotests: Fix intermittent failure in 219

qemu-img: Fix options leakage in img_rebase()

qapi/block-core: Overlays are not snapshots

blockdev: Overlays are not snapshots

qemu-img: Move quiet into ImgConvertState

qemu-img: Add salvaging mode to convert

blkdebug: Add @iotype error option

blkdebug: Add "none" event

blkdebug: Inject errors on .bdrv_co_block_status()

iotests: Test qemu-img convert --salvage

iotests: Test qemu-img convert -C --salvage

Vladimir Sementsov-Ogievskiy (1):

iotests: restrict 254 to support only qcow2

qapi/block-core.json | 53 ++++++++---

block/blkdebug.c | 60 ++++++++++--

blockdev.c | 14 +--

hw/block/fdc.c | 1 +

qemu-img.c | 106 +++++++++++++++------

python/qemu/__init__.py | 67 ++++++++++----

qemu-img-cmds.hx | 4 +-

qemu-img.texi | 4 +

tests/qemu-iotests/082 | 1 +

tests/qemu-iotests/082.out | 3 +

tests/qemu-iotests/085.out | 10 +-

tests/qemu-iotests/175 | 26 +++++-

tests/qemu-iotests/175.out | 8 +-

tests/qemu-iotests/219 | 13 ++-

tests/qemu-iotests/251 | 170 ++++++++++++++++++++++++++++++++++

tests/qemu-iotests/251.out | 43 +++++++++

tests/qemu-iotests/254 | 2 +

tests/qemu-iotests/256 | 122 ++++++++++++++++++++++++

tests/qemu-iotests/256.out | 119 ++++++++++++++++++++++++

tests/qemu-iotests/group | 2 +

tests/qemu-iotests/iotests.py | 60 +++++++-----

21 files changed, 772 insertions(+), 116 deletions(-)

create mode 100755 tests/qemu-iotests/251

create mode 100644 tests/qemu-iotests/251.out

create mode 100755 tests/qemu-iotests/256

create mode 100644 tests/qemu-iotests/256.out

2.21.0

There are error messages which refer to an overlay node as the snapshot.

That is wrong, those are two different things.

Signed-off-by: Max Reitz <mreitz@redhat.com>

Reviewed-by: Eric Blake <eblake@redhat.com>

Message-id: 20190603202236.1342-3-mreitz@redhat.com

Reviewed-by: John Snow <jsnow@redhat.com>

Reviewed-by: Alberto Garcia <berto@igalia.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

blockdev.c | 10 +++++-----

tests/qemu-iotests/085.out | 10 +++++-----

2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/blockdev.c b/blockdev.c

--- a/blockdev.c

+++ b/blockdev.c

@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,

s->has_snapshot_node_name ? s->snapshot_node_name : NULL;

if (node_name && !snapshot_node_name) {

- error_setg(errp, "New snapshot node name missing");

+ error_setg(errp, "New overlay node name missing");

goto out;

}

if (snapshot_node_name &&

bdrv_lookup_bs(snapshot_node_name, snapshot_node_name, NULL)) {

- error_setg(errp, "New snapshot node name already in use");

+ error_setg(errp, "New overlay node name already in use");

goto out;

}

@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,

if (bdrv_has_blk(state->new_bs)) {

- error_setg(errp, "The snapshot is already in use");

+ error_setg(errp, "The overlay is already in use");

goto out;

@@ -XXX,XX +XXX,XX @@ static void external_snapshot_prepare(BlkActionState *common,

if (state->new_bs->backing != NULL) {

- error_setg(errp, "The snapshot already has a backing image");

+ error_setg(errp, "The overlay already has a backing image");

goto out;

if (!state->new_bs->drv->supports_backing) {

- error_setg(errp, "The snapshot does not support backing images");

+ error_setg(errp, "The overlay does not support backing images");

goto out;

diff --git a/tests/qemu-iotests/085.out b/tests/qemu-iotests/085.out

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/085.out

+++ b/tests/qemu-iotests/085.out

@@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/

=== Invalid command - cannot create a snapshot using a file BDS ===

-{"error": {"class": "GenericError", "desc": "The snapshot does not support backing images"}}

+{"error": {"class": "GenericError", "desc": "The overlay does not support backing images"}}

=== Invalid command - snapshot node used as active layer ===

-{"error": {"class": "GenericError", "desc": "The snapshot is already in use"}}

-{"error": {"class": "GenericError", "desc": "The snapshot is already in use"}}

-{"error": {"class": "GenericError", "desc": "The snapshot is already in use"}}

+{"error": {"class": "GenericError", "desc": "The overlay is already in use"}}

+{"error": {"class": "GenericError", "desc": "The overlay is already in use"}}

+{"error": {"class": "GenericError", "desc": "The overlay is already in use"}}

=== Invalid command - snapshot node used as backing hd ===

@@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/

Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728

Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base

{"return": {}}

-{"error": {"class": "GenericError", "desc": "The snapshot already has a backing image"}}

+{"error": {"class": "GenericError", "desc": "The overlay already has a backing image"}}

=== Invalid command - The node does not exist ===

2.21.0

This adds a salvaging mode (--salvage) to qemu-img convert which ignores

read errors and treats the respective areas as containing only zeroes.

This can be used for instance to at least partially recover the data

from terminally corrupted qcow2 images.

Signed-off-by: Max Reitz <mreitz@redhat.com>

Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Message-id: 20190507203508.18026-3-mreitz@redhat.com

Signed-off-by: Max Reitz <mreitz@redhat.com>

qemu-img.c | 90 +++++++++++++++++++++++++++++++++++++-----------

qemu-img-cmds.hx | 4 +--

qemu-img.texi | 4 +++

3 files changed, 75 insertions(+), 23 deletions(-)

diff --git a/qemu-img.c b/qemu-img.c

--- a/qemu-img.c

+++ b/qemu-img.c

@@ -XXX,XX +XXX,XX @@ enum {

OPTION_SIZE = 264,

OPTION_PREALLOCATION = 265,

OPTION_SHRINK = 266,

+ OPTION_SALVAGE = 267,

typedef enum OutputFormat {

@@ -XXX,XX +XXX,XX @@ typedef struct ImgConvertState {

int64_t target_backing_sectors; /* negative if unknown */

bool wr_in_order;

bool copy_range;

+ bool salvage;

bool quiet;

int min_sparse;

int alignment;

@@ -XXX,XX +XXX,XX @@ static int convert_iteration_sectors(ImgConvertState *s, int64_t sector_num)

}

if (s->sector_next_status <= sector_num) {

- int64_t count = n * BDRV_SECTOR_SIZE;

+ uint64_t offset = (sector_num - src_cur_offset) * BDRV_SECTOR_SIZE;

+ int64_t count;

- if (s->target_has_backing) {

+ do {

+ count = n * BDRV_SECTOR_SIZE;

+

+ if (s->target_has_backing) {

+ ret = bdrv_block_status(blk_bs(s->src[src_cur]), offset,

+ count, &count, NULL, NULL);

+ } else {

+ ret = bdrv_block_status_above(blk_bs(s->src[src_cur]), NULL,

+ offset, count, &count, NULL,

+ NULL);

+ }

+

+ if (ret < 0) {

+ if (s->salvage) {

+ if (n == 1) {

+ if (!s->quiet) {

+ warn_report("error while reading block status at "

+ "offset %" PRIu64 ": %s", offset,

+ strerror(-ret));

+ }

+ /* Just try to read the data, then */

+ ret = BDRV_BLOCK_DATA;

+ count = BDRV_SECTOR_SIZE;

+ } else {

+ /* Retry on a shorter range */

+ n = DIV_ROUND_UP(n, 4);

+ }

+ } else {

+ error_report("error while reading block status at offset "

+ "%" PRIu64 ": %s", offset, strerror(-ret));

+ return ret;

+ }

+ }

+ } while (ret < 0);

- ret = bdrv_block_status(blk_bs(s->src[src_cur]),

- (sector_num - src_cur_offset) *

- BDRV_SECTOR_SIZE,

- count, &count, NULL, NULL);

- } else {

- ret = bdrv_block_status_above(blk_bs(s->src[src_cur]), NULL,

- (sector_num - src_cur_offset) *

- BDRV_SECTOR_SIZE,

- count, &count, NULL, NULL);

- }

- if (ret < 0) {

- error_report("error while reading block status of sector %" PRId64

- ": %s", sector_num, strerror(-ret));

- return ret;

- }

n = DIV_ROUND_UP(count, BDRV_SECTOR_SIZE);

if (ret & BDRV_BLOCK_ZERO) {

@@ -XXX,XX +XXX,XX @@ static int convert_iteration_sectors(ImgConvertState *s, int64_t sector_num)

static int coroutine_fn convert_co_read(ImgConvertState *s, int64_t sector_num,

int nb_sectors, uint8_t *buf)

{

+ uint64_t single_read_until = 0;

int n, ret;

assert(nb_sectors <= s->buf_sectors);

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn convert_co_read(ImgConvertState *s, int64_t sector_num,

BlockBackend *blk;

int src_cur;

int64_t bs_sectors, src_cur_offset;

+ uint64_t offset;

/* In the case of compression with multiple source files, we can get a

* nb_sectors that spreads into the next part. So we must be able to

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn convert_co_read(ImgConvertState *s, int64_t sector_num,

blk = s->src[src_cur];

bs_sectors = s->src_sectors[src_cur];

+ offset = (sector_num - src_cur_offset) << BDRV_SECTOR_BITS;

+

n = MIN(nb_sectors, bs_sectors - (sector_num - src_cur_offset));

+ if (single_read_until > offset) {

+ n = 1;

+ }

- ret = blk_co_pread(

- blk, (sector_num - src_cur_offset) << BDRV_SECTOR_BITS,

- n << BDRV_SECTOR_BITS, buf, 0);

+ ret = blk_co_pread(blk, offset, n << BDRV_SECTOR_BITS, buf, 0);

if (ret < 0) {

- return ret;

+ if (s->salvage) {

+ if (n > 1) {

+ single_read_until = offset + (n << BDRV_SECTOR_BITS);

+ continue;

+ } else {

+ if (!s->quiet) {

+ warn_report("error while reading offset %" PRIu64

+ ": %s", offset, strerror(-ret));

+ }

+ memset(buf, 0, BDRV_SECTOR_SIZE);

+ }

+ } else {

+ return ret;

+ }

}

sector_num += n;

@@ -XXX,XX +XXX,XX @@ static int img_convert(int argc, char **argv)

{"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},

{"force-share", no_argument, 0, 'U'},

{"target-image-opts", no_argument, 0, OPTION_TARGET_IMAGE_OPTS},

+ {"salvage", no_argument, 0, OPTION_SALVAGE},

{0, 0, 0, 0}

};

c = getopt_long(argc, argv, ":hf:O:B:Cco:l:S:pt:T:qnm:WU",

@@ -XXX,XX +XXX,XX @@ static int img_convert(int argc, char **argv)

case OPTION_IMAGE_OPTS:

image_opts = true;

break;

+ case OPTION_SALVAGE:

+ s.salvage = true;

+ break;

case OPTION_TARGET_IMAGE_OPTS:

tgt_image_opts = true;

break;

@@ -XXX,XX +XXX,XX @@ static int img_convert(int argc, char **argv)

goto fail_getopt;

}

+ if (s.copy_range && s.salvage) {

+ error_report("Cannot use copy offloading in salvaging mode");

+ goto fail_getopt;

+ }

+

if (tgt_image_opts && !skip_create) {

error_report("--target-image-opts requires use of -n flag");

goto fail_getopt;

diff --git a/qemu-img-cmds.hx b/qemu-img-cmds.hx

index XXXXXXX..XXXXXXX 100644

--- a/qemu-img-cmds.hx

+++ b/qemu-img-cmds.hx

@@ -XXX,XX +XXX,XX @@ STEXI

ETEXI

DEF("convert", img_convert,

- "convert [--object objectdef] [--image-opts] [--target-image-opts] [-U] [-C] [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-B backing_file] [-o options] [-l snapshot_param] [-S sparse_size] [-m num_coroutines] [-W] filename [filename2 [...]] output_filename")

+ "convert [--object objectdef] [--image-opts] [--target-image-opts] [-U] [-C] [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-B backing_file] [-o options] [-l snapshot_param] [-S sparse_size] [-m num_coroutines] [-W] [--salvage] filename [filename2 [...]] output_filename")

STEXI

-@item convert [--object @var{objectdef}] [--image-opts] [--target-image-opts] [-U] [-C] [-c] [-p] [-q] [-n] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-O @var{output_fmt}] [-B @var{backing_file}] [-o @var{options}] [-l @var{snapshot_param}] [-S @var{sparse_size}] [-m @var{num_coroutines}] [-W] @var{filename} [@var{filename2} [...]] @var{output_filename}

+@item convert [--object @var{objectdef}] [--image-opts] [--target-image-opts] [-U] [-C] [-c] [-p] [-q] [-n] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-O @var{output_fmt}] [-B @var{backing_file}] [-o @var{options}] [-l @var{snapshot_param}] [-S @var{sparse_size}] [-m @var{num_coroutines}] [-W] [--salvage] @var{filename} [@var{filename2} [...]] @var{output_filename}

ETEXI

DEF("create", img_create,

diff --git a/qemu-img.texi b/qemu-img.texi

index XXXXXXX..XXXXXXX 100644

--- a/qemu-img.texi

+++ b/qemu-img.texi

@@ -XXX,XX +XXX,XX @@ improve performance if the data is remote, such as with NFS or iSCSI backends,

but will not automatically sparsify zero sectors, and may result in a fully

allocated target image depending on the host support for getting allocation

information.

+@item --salvage

+Try to ignore I/O errors when reading. Unless in quiet mode (@code{-q}), errors

+will still be printed. Areas that cannot be read from the source will be

+treated as containing only zeroes.

@end table

Parameters to dd subcommand:

2.21.0

From: John Snow <jsnow@redhat.com>

Instead of event_wait which looks for a single event, add an events_wait

which can look for any number of events simultaneously. However, it

will still only return one at a time, whichever happens first.

Signed-off-by: John Snow <jsnow@redhat.com>

Message-id: 20190523170643.20794-4-jsnow@redhat.com

Reviewed-by: Max Reitz <mreitz@redhat.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

python/qemu/__init__.py | 69 +++++++++++++++++++++++++++++------------

1 file changed, 49 insertions(+), 20 deletions(-)

diff --git a/python/qemu/__init__.py b/python/qemu/__init__.py

--- a/python/qemu/__init__.py

+++ b/python/qemu/__init__.py

@@ -XXX,XX +XXX,XX @@ class QEMUMachine(object):

self._qmp.clear_events()

return events

- def event_wait(self, name, timeout=60.0, match=None):

+ @staticmethod

+ def event_match(event, match=None):

"""

- Wait for specified timeout on named event in QMP; optionally filter

- results by match.

+ Check if an event matches optional match criteria.

- The 'match' is checked to be a recursive subset of the 'event'; skips

- branch processing on match's value None

- {"foo": {"bar": 1}} matches {"foo": None}

- {"foo": {"bar": 1}} does not matches {"foo": {"baz": None}}

+ The match criteria takes the form of a matching subdict. The event is

+ checked to be a superset of the subdict, recursively, with matching

+ values whenever those values are not None.

+ Examples, with the subdict queries on the left:

+ - None matches any object.

+ - {"foo": None} matches {"foo": {"bar": 1}}

+ - {"foo": {"baz": None}} does not match {"foo": {"bar": 1}}

+ - {"foo": {"baz": 2}} matches {"foo": {"bar": 1, "baz": 2}}

"""

- def event_match(event, match=None):

- if match is None:

- return True

+ if match is None:

+ return True

- for key in match:

- if key in event:

- if isinstance(event[key], dict):

- if not event_match(event[key], match[key]):

- return False

- elif event[key] != match[key]:

+ for key in match:

+ if key in event:

+ if isinstance(event[key], dict):

+ if not QEMUMachine.event_match(event[key], match[key]):

return False

- else:

+ elif event[key] != match[key]:

return False

+ else:

+ return False

+ return True

- return True

+ def event_wait(self, name, timeout=60.0, match=None):

+ """

+ event_wait waits for and returns a named event from QMP with a timeout.

+

+ name: The event to wait for.

+ timeout: QEMUMonitorProtocol.pull_event timeout parameter.

+ match: Optional match criteria. See event_match for details.

+ """

+ return self.events_wait([(name, match)], timeout)

+

+ def events_wait(self, events, timeout=60.0):

+ """

+ events_wait waits for and returns a named event from QMP with a timeout.

+

+ events: a sequence of (name, match_criteria) tuples.

+ The match criteria are optional and may be None.

+ See event_match for details.

+ timeout: QEMUMonitorProtocol.pull_event timeout parameter.

+ """

+ def _match(event):

+ for name, match in events:

+ if (event['event'] == name and

+ self.event_match(event, match)):

+ return True

+ return False

# Search cached events

for event in self._events:

- if (event['event'] == name) and event_match(event, match):

+ if _match(event):

self._events.remove(event)

return event

# Poll for new events

while True:

event = self._qmp.pull_event(wait=timeout)

- if (event['event'] == name) and event_match(event, match):

+ if _match(event):

return event

self._events.append(event)

2.21.0