The following changes since commit 527266f324def9f7f392fe3b0dd940cb8dc699d9:

Merge remote-tracking branch 'remotes/armbru/tags/pull-pflash-2019-03-26' into staging (2019-03-26 09:57:07 +0000)

for you to fetch changes up to c6e3f520c802c5cb2de80576aba7f9f1fe985d8b:

qemu-io: Add write -n for BDRV_REQ_NO_FALLBACK (2019-03-26 11:37:51 +0100)

- Fix slow pre-zeroing in qemu-img convert

- Test case for block job pausing on I/O errors

Kevin Wolf (6):

block: Remove error messages in bdrv_make_zero()

block: Add BDRV_REQ_NO_FALLBACK

block: Advertise BDRV_REQ_NO_FALLBACK in filter drivers

file-posix: Support BDRV_REQ_NO_FALLBACK for zero writes

qemu-img: Use BDRV_REQ_NO_FALLBACK for pre-zeroing

qemu-io: Add write -n for BDRV_REQ_NO_FALLBACK

Vladimir Sementsov-Ogievskiy (1):

iotests: add 248: test resume mirror after auto pause on ENOSPC

include/block/block.h | 7 ++++-

include/block/raw-aio.h | 1 +

block/blkdebug.c | 2 +-

block/copy-on-read.c | 7 ++---

block/file-posix.c | 24 ++++++++++------

block/io.c | 16 +++++++----

block/mirror.c | 3 +-

block/raw-format.c | 2 +-

qemu-img.c | 2 +-

qemu-io-cmds.c | 13 +++++++--

tests/qemu-iotests/248 | 71 ++++++++++++++++++++++++++++++++++++++++++++++

tests/qemu-iotests/248.out | 8 ++++++

tests/qemu-iotests/group | 1 +

13 files changed, 133 insertions(+), 24 deletions(-)

create mode 100755 tests/qemu-iotests/248

create mode 100644 tests/qemu-iotests/248.out

This makes the new BDRV_REQ_NO_FALLBACK flag available in the qemu-io

write command.

qemu-io-cmds.c | 13 +++++++++++--

1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c

--- a/qemu-io-cmds.c

+++ b/qemu-io-cmds.c

@@ -XXX,XX +XXX,XX @@ static void write_help(void)

" -b, -- write to the VM state rather than the virtual disk\n"

" -c, -- write compressed data with blk_write_compressed\n"

" -f, -- use Force Unit Access semantics\n"

+" -n, -- with -z, don't allow slow fallback\n"

" -p, -- ignored for backwards compatibility\n"

" -P, -- use different pattern to fill file\n"

" -C, -- report statistics in a machine parsable format\n"

@@ -XXX,XX +XXX,XX @@ static const cmdinfo_t write_cmd = {

.perm = BLK_PERM_WRITE,

.argmin = 2,

.argmax = -1,

- .args = "[-bcCfquz] [-P pattern] off len",

+ .args = "[-bcCfnquz] [-P pattern] off len",

.oneline = "writes a number of bytes at a specified offset",

.help = write_help,

};

@@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv)

int64_t total = 0;

int pattern = 0xcd;

- while ((c = getopt(argc, argv, "bcCfpP:quz")) != -1) {

+ while ((c = getopt(argc, argv, "bcCfnpP:quz")) != -1) {

switch (c) {

case 'b':

bflag = true;

@@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv)

case 'f':

flags |= BDRV_REQ_FUA;

break;

+ case 'n':

+ flags |= BDRV_REQ_NO_FALLBACK;

+ break;

case 'p':

/* Ignored for backwards compatibility */

break;

@@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv)

return -EINVAL;

+ if ((flags & BDRV_REQ_NO_FALLBACK) && !zflag) {

+ printf("-n requires -z to be specified\n");

+ return -EINVAL;

+ }

+

if ((flags & BDRV_REQ_MAY_UNMAP) && !zflag) {

printf("-u requires -z to be specified\n");

return -EINVAL;

2.20.1

If qemu-img convert sees that the target image isn't zero-initialised

yet, it tries to do an efficient zero write for the whole image first

to save the overhead of repeated explicit zero writes during the

conversion. Obviously, this provides only an advantage if the

pre-zeroing is actually efficient. Otherwise, we can end up writing

zeroes slowly while zeroing out the whole image, and then overwrite the

same blocks again with real data, potentially doubling the written data.

Pass BDRV_REQ_NO_FALLBACK to blk_make_zero() to avoid this case. If we

can't efficiently zero out, we'll instead write explicit zeroes only if

there is no data to be written to a block.

qemu-img.c | 2 +-

diff --git a/qemu-img.c b/qemu-img.c

index XXXXXXX..XXXXXXX 100644

--- a/qemu-img.c

+++ b/qemu-img.c

@@ -XXX,XX +XXX,XX @@ static int convert_do_copy(ImgConvertState *s)

if (!s->has_zero_init && !s->target_has_backing &&

bdrv_can_write_zeroes_with_unmap(blk_bs(s->target)))

{

- ret = blk_make_zero(s->target, BDRV_REQ_MAY_UNMAP);

+ ret = blk_make_zero(s->target, BDRV_REQ_MAY_UNMAP | BDRV_REQ_NO_FALLBACK);

if (ret == 0) {

s->has_zero_init = true;

}

2.20.1

We know that the kernel implements a slow fallback code path for

BLKZEROOUT, so if BDRV_REQ_NO_FALLBACK is given, we shouldn't call it.

The other operations we call in the context of .bdrv_co_pwrite_zeroes

should usually be quick, so no modification should be needed for them.

If we ever notice that there are additional problematic cases, we can

still make these conditional as well.

include/block/raw-aio.h | 1 +

block/file-posix.c | 24 ++++++++++++++++--------

2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/include/block/raw-aio.h b/include/block/raw-aio.h

--- a/include/block/raw-aio.h

+++ b/include/block/raw-aio.h

@@ -XXX,XX +XXX,XX @@

/* AIO flags */

#define QEMU_AIO_MISALIGNED 0x1000

#define QEMU_AIO_BLKDEV 0x2000

+#define QEMU_AIO_NO_FALLBACK 0x4000

/* linux-aio.c - Linux native implementation */

diff --git a/block/file-posix.c b/block/file-posix.c

index XXXXXXX..XXXXXXX 100644

--- a/block/file-posix.c

+++ b/block/file-posix.c

@@ -XXX,XX +XXX,XX @@ static int raw_open_common(BlockDriverState *bs, QDict *options,

#endif

- bs->supported_zero_flags = BDRV_REQ_MAY_UNMAP;

+ bs->supported_zero_flags = BDRV_REQ_MAY_UNMAP | BDRV_REQ_NO_FALLBACK;

ret = 0;

fail:

if (filename && (bdrv_flags & BDRV_O_TEMPORARY)) {

@@ -XXX,XX +XXX,XX @@ static ssize_t handle_aiocb_write_zeroes_block(RawPosixAIOData *aiocb)

}

#ifdef BLKZEROOUT

- do {

- uint64_t range[2] = { aiocb->aio_offset, aiocb->aio_nbytes };

- if (ioctl(aiocb->aio_fildes, BLKZEROOUT, range) == 0) {

- return 0;

- }

- } while (errno == EINTR);

+ /* The BLKZEROOUT implementation in the kernel doesn't set

+ * BLKDEV_ZERO_NOFALLBACK, so we can't call this if we have to avoid slow

+ * fallbacks. */

+ if (!(aiocb->aio_type & QEMU_AIO_NO_FALLBACK)) {

+ do {

+ uint64_t range[2] = { aiocb->aio_offset, aiocb->aio_nbytes };

+ if (ioctl(aiocb->aio_fildes, BLKZEROOUT, range) == 0) {

+ return 0;

+ }

+ } while (errno == EINTR);

- ret = translate_err(-errno);

+ ret = translate_err(-errno);

+ }

#endif

if (ret == -ENOTSUP) {

@@ -XXX,XX +XXX,XX @@ raw_do_pwrite_zeroes(BlockDriverState *bs, int64_t offset, int bytes,

if (blkdev) {

acb.aio_type |= QEMU_AIO_BLKDEV;

}

+ if (flags & BDRV_REQ_NO_FALLBACK) {

+ acb.aio_type |= QEMU_AIO_NO_FALLBACK;

+ }

if (flags & BDRV_REQ_MAY_UNMAP) {

acb.aio_type |= QEMU_AIO_DISCARD;

2.20.1

For qemu-img convert, we want an operation that zeroes out the whole

image if this can be done efficiently, but that returns an error

otherwise so we don't write explicit zeroes and immediately overwrite

them with the real data, potentially doubling the amount of data to be

written.

include/block/block.h | 7 ++++++-

block/io.c | 12 +++++++++++-

2 files changed, 17 insertions(+), 2 deletions(-)

@@ -XXX,XX +XXX,XX @@ typedef enum {

*/

BDRV_REQ_SERIALISING = 0x80,

+ /* Execute the request only if the operation can be offloaded or otherwise

+ * be executed efficiently, but return an error instead of using a slow

+ * fallback. */

+ BDRV_REQ_NO_FALLBACK = 0x100,

/* Mask of valid flags */

- BDRV_REQ_MASK = 0xff,

+ BDRV_REQ_MASK = 0x1ff,

} BdrvRequestFlags;

typedef struct BlockSizes {

diff --git a/block/io.c b/block/io.c

index XXXXXXX..XXXXXXX 100644

--- a/block/io.c

+++ b/block/io.c

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,

unsigned int nb_sectors;

assert(!(flags & ~BDRV_REQ_MASK));

+ assert(!(flags & BDRV_REQ_NO_FALLBACK));

if (!drv) {

return -ENOMEDIUM;

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,

int ret;

assert(!(flags & ~BDRV_REQ_MASK));

+ assert(!(flags & BDRV_REQ_NO_FALLBACK));

if (!drv) {

return -ENOMEDIUM;

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,

return -ENOMEDIUM;

}

+ if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) {

+ return -ENOTSUP;

assert(alignment % bs->bl.request_alignment == 0);

head = offset % alignment;

tail = (offset + bytes) % alignment;

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,

assert(!bs->supported_zero_flags);

}

- if (ret == -ENOTSUP) {

+ if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) {

/* Fall back to bounce buffer if write zeroes is unsupported */

BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;

@@ -XXX,XX +XXX,XX @@ static int coroutine_fn bdrv_co_copy_range_internal(

BdrvTrackedRequest req;

int ret;

+ /* TODO We can support BDRV_REQ_NO_FALLBACK here */

+ assert(!(read_flags & BDRV_REQ_NO_FALLBACK));

+ assert(!(write_flags & BDRV_REQ_NO_FALLBACK));

if (!dst || !dst->bs) {

return -ENOMEDIUM;

2.20.1

Filter drivers that support .bdrv_co_pwrite_zeroes can safely advertise

BDRV_REQ_NO_FALLBACK because they just forward the request flags to

their child node.

block/blkdebug.c | 2 +-

block/copy-on-read.c | 7 +++----

block/mirror.c | 3 ++-

block/raw-format.c | 2 +-

4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/block/blkdebug.c b/block/blkdebug.c

--- a/block/blkdebug.c

+++ b/block/blkdebug.c

@@ -XXX,XX +XXX,XX @@ static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags,

bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED |

(BDRV_REQ_FUA & bs->file->bs->supported_write_flags);

bs->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED |

- ((BDRV_REQ_FUA | BDRV_REQ_MAY_UNMAP) &

+ ((BDRV_REQ_FUA | BDRV_REQ_MAY_UNMAP | BDRV_REQ_NO_FALLBACK) &

bs->file->bs->supported_zero_flags);

ret = -EINVAL;

diff --git a/block/copy-on-read.c b/block/copy-on-read.c

index XXXXXXX..XXXXXXX 100644

--- a/block/copy-on-read.c

+++ b/block/copy-on-read.c

@@ -XXX,XX +XXX,XX @@ static int cor_open(BlockDriverState *bs, QDict *options, int flags,

bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED |

- (BDRV_REQ_FUA &

- bs->file->bs->supported_write_flags);

+ (BDRV_REQ_FUA & bs->file->bs->supported_write_flags);

bs->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED |

- ((BDRV_REQ_FUA | BDRV_REQ_MAY_UNMAP) &

- bs->file->bs->supported_zero_flags);

+ ((BDRV_REQ_FUA | BDRV_REQ_MAY_UNMAP | BDRV_REQ_NO_FALLBACK) &

+ bs->file->bs->supported_zero_flags);

return 0;

}

diff --git a/block/mirror.c b/block/mirror.c

index XXXXXXX..XXXXXXX 100644

--- a/block/mirror.c

+++ b/block/mirror.c

@@ -XXX,XX +XXX,XX @@ static void mirror_start_job(const char *job_id, BlockDriverState *bs,

}

mirror_top_bs->total_sectors = bs->total_sectors;

mirror_top_bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED;

- mirror_top_bs->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED;

+ mirror_top_bs->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED |

+ BDRV_REQ_NO_FALLBACK;

bs_opaque = g_new0(MirrorBDSOpaque, 1);

mirror_top_bs->opaque = bs_opaque;

bdrv_set_aio_context(mirror_top_bs, bdrv_get_aio_context(bs));

diff --git a/block/raw-format.c b/block/raw-format.c

index XXXXXXX..XXXXXXX 100644

--- a/block/raw-format.c

+++ b/block/raw-format.c

@@ -XXX,XX +XXX,XX @@ static int raw_open(BlockDriverState *bs, QDict *options, int flags,

bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED |

(BDRV_REQ_FUA & bs->file->bs->supported_write_flags);

bs->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED |

- ((BDRV_REQ_FUA | BDRV_REQ_MAY_UNMAP) &

+ ((BDRV_REQ_FUA | BDRV_REQ_MAY_UNMAP | BDRV_REQ_NO_FALLBACK) &

bs->file->bs->supported_zero_flags);

if (bs->probed && !bdrv_is_read_only(bs)) {

2.20.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Test that mirror job actually resume on resume command after being

automatically paused on ENOSPC error.

It's a follow-up test for 8d9648cbf3e

"blockjob: fix user pause in block_job_error_action"

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Tested-by: John Snow <jsnow@redhat.com>

Reviewed-by: John Snow <jsnow@redhat.com>

tests/qemu-iotests/248 | 71 ++++++++++++++++++++++++++++++++++++++

tests/qemu-iotests/248.out | 8 +++++

tests/qemu-iotests/group | 1 +

3 files changed, 80 insertions(+)

create mode 100755 tests/qemu-iotests/248

create mode 100644 tests/qemu-iotests/248.out

diff --git a/tests/qemu-iotests/248 b/tests/qemu-iotests/248

new file mode 100755

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/tests/qemu-iotests/248

@@ -XXX,XX +XXX,XX @@

+#!/usr/bin/env python

+#

+# Test resume mirror after auto pause on ENOSPC

+#

+# Copyright (c) 2019 Virtuozzo International GmbH. All rights reserved.

+#

+# This program is free software; you can redistribute it and/or modify

+# it under the terms of the GNU General Public License as published by

+# the Free Software Foundation; either version 2 of the License, or

+# (at your option) any later version.

+#

+# This program is distributed in the hope that it will be useful,

+# but WITHOUT ANY WARRANTY; without even the implied warranty of

+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

+# GNU General Public License for more details.

+#

+# You should have received a copy of the GNU General Public License

+# along with this program. If not, see <http://www.gnu.org/licenses/>.

+#

+import iotests

+from iotests import qemu_img_create, qemu_io, file_path, filter_qmp_testfiles

+

+iotests.verify_image_format(supported_fmts=['qcow2'])

+

+source, target = file_path('source', 'target')

+size = 5 * 1024 * 1024

+limit = 2 * 1024 * 1024

+

+qemu_img_create('-f', iotests.imgfmt, source, str(size))

+qemu_img_create('-f', iotests.imgfmt, target, str(size))

+qemu_io('-c', 'write 0 {}'.format(size), source)

+

+# raw format don't like empty files

+qemu_io('-c', 'write 0 {}'.format(size), target)

+

+vm = iotests.VM().add_drive(source)

+vm.launch()

+

+blockdev_opts = {

+ 'driver': iotests.imgfmt,

+ 'node-name': 'target',

+ 'file': {

+ 'driver': 'raw',

+ 'size': limit,

+ 'file': {

+ 'driver': 'file',

+ 'filename': target

+ }

+ }

+}

+vm.qmp_log('blockdev-add', filters=[filter_qmp_testfiles], **blockdev_opts)

+

+vm.qmp_log('blockdev-mirror', device='drive0', sync='full', target='target',

+ on_target_error='enospc')

+

+vm.event_wait('JOB_STATUS_CHANGE', timeout=3.0,

+ match={'data': {'status': 'paused'}})

+

+# drop other cached events, to not interfere with further wait for 'running'

+vm.get_qmp_events()

+

+del blockdev_opts['file']['size']

+vm.qmp_log('x-blockdev-reopen', filters=[filter_qmp_testfiles],

+ **blockdev_opts)

+

+vm.qmp_log('block-job-resume', device='drive0')

+vm.event_wait('JOB_STATUS_CHANGE', timeout=1.0,

+ match={'data': {'status': 'running'}})

+

+vm.shutdown()

diff --git a/tests/qemu-iotests/248.out b/tests/qemu-iotests/248.out

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/tests/qemu-iotests/248.out

@@ -XXX,XX +XXX,XX @@

+{"execute": "blockdev-add", "arguments": {"driver": "qcow2", "file": {"driver": "raw", "file": {"driver": "file", "filename": "TEST_DIR/PID-target"}, "size": 2097152}, "node-name": "target"}}

+{"return": {}}

+{"execute": "blockdev-mirror", "arguments": {"device": "drive0", "on-target-error": "enospc", "sync": "full", "target": "target"}}

+{"return": {}}

+{"execute": "x-blockdev-reopen", "arguments": {"driver": "qcow2", "file": {"driver": "raw", "file": {"driver": "file", "filename": "TEST_DIR/PID-target"}}, "node-name": "target"}}

+{"return": {}}

+{"execute": "block-job-resume", "arguments": {"device": "drive0"}}

+{"return": {}}

diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/group

+++ b/tests/qemu-iotests/group

@@ -XXX,XX +XXX,XX @@

245 rw auto

246 rw auto quick

247 rw auto quick

+248 rw auto quick

2.20.1

There is only a single caller of bdrv_make_zero(), which is qemu-img

convert. If the function fails, we just fall back to a different method

of zeroing out blocks on the target image. There is no good reason to

print error messages on stderr when the higher level operation will

actually succeed.

Acked-by: Eric Blake <eblake@redhat.com>

block/io.c | 4 ----

1 file changed, 4 deletions(-)

diff --git a/block/io.c b/block/io.c

--- a/block/io.c

+++ b/block/io.c

@@ -XXX,XX +XXX,XX @@ int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)

}

ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL);

if (ret < 0) {

- error_report("error getting block status at offset %" PRId64 ": %s",

- offset, strerror(-ret));

return ret;

}

if (ret & BDRV_BLOCK_ZERO) {

@@ -XXX,XX +XXX,XX @@ int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)

}

ret = bdrv_pwrite_zeroes(child, offset, bytes, flags);

if (ret < 0) {

- error_report("error writing zeroes at offset %" PRId64 ": %s",

- offset, strerror(-ret));

return ret;

}

offset += bytes;

2.20.1