The following changes since commit 23919ddfd56135cad3cb468a8f54d5a595f024f4: Merge remote-tracking branch 'remotes/aperard/tags/pull-xen-20190827' into staging (2019-08-27 15:52:36 +0100) are available in the Git repository at: https://github.com/XanClic/qemu.git tags/pull-block-2019-08-27 for you to fetch changes up to bb043c056cffcc2f3ce88bfdaf2e76e455c09e2c: iotests: Unify cache mode quoting (2019-08-27 19:48:44 +0200) ---------------------------------------------------------------- Block patches: - qemu-io now accepts a file to read a write pattern from - Ensure that raw files have their first block allocated so we can probe the O_DIRECT alignment if necessary - Various fixes ---------------------------------------------------------------- Denis Plotnikov (1): qemu-io: add pattern file for write command Max Reitz (7): iotests: Fix _filter_img_create() vmdk: Use bdrv_dirname() for relative extent paths iotests: Keep testing broken relative extent paths vmdk: Reject invalid compressed writes iotests: Disable broken streamOptimized tests iotests: Disable 110 for vmdk.twoGbMaxExtentSparse iotests: Disable 126 for flat vmdk subformats Nir Soffer (3): block: posix: Always allocate the first block iotests: Test allocate_first_block() with O_DIRECT iotests: Unify cache mode quoting Stefan Hajnoczi (1): file-posix: fix request_alignment typo Thomas Huth (2): iotests: Check for enabled drivers before testing them tests/check-block: Skip iotests when sanitizers are enabled Vladimir Sementsov-Ogievskiy (1): block: fix permission update in bdrv_replace_node block.c | 5 +- block/file-posix.c | 53 +++++++++- block/vmdk.c | 64 ++++++++---- qemu-io-cmds.c | 99 +++++++++++++++++-- tests/check-block.sh | 5 + tests/qemu-iotests/002 | 1 + tests/qemu-iotests/003 | 1 + tests/qemu-iotests/005 | 3 +- tests/qemu-iotests/009 | 1 + tests/qemu-iotests/010 | 1 + tests/qemu-iotests/011 | 1 + tests/qemu-iotests/017 | 3 +- tests/qemu-iotests/018 | 3 +- tests/qemu-iotests/019 | 3 +- tests/qemu-iotests/020 | 3 +- tests/qemu-iotests/026 | 4 +- tests/qemu-iotests/027 | 1 + tests/qemu-iotests/032 | 1 + tests/qemu-iotests/033 | 1 + tests/qemu-iotests/034 | 3 +- tests/qemu-iotests/037 | 3 +- tests/qemu-iotests/039 | 4 +- tests/qemu-iotests/052 | 2 +- tests/qemu-iotests/059 | 34 ++++++- tests/qemu-iotests/059.out | 26 +++-- tests/qemu-iotests/063 | 3 +- tests/qemu-iotests/071 | 1 + tests/qemu-iotests/072 | 1 + tests/qemu-iotests/081 | 4 +- tests/qemu-iotests/091 | 4 +- tests/qemu-iotests/099 | 1 + tests/qemu-iotests/105 | 3 +- tests/qemu-iotests/110 | 3 +- tests/qemu-iotests/120 | 1 + tests/qemu-iotests/126 | 2 + tests/qemu-iotests/{150.out => 150.out.qcow2} | 0 tests/qemu-iotests/150.out.raw | 12 +++ tests/qemu-iotests/162 | 4 +- tests/qemu-iotests/175 | 47 +++++++-- tests/qemu-iotests/175.out | 16 ++- tests/qemu-iotests/178.out.qcow2 | 4 +- tests/qemu-iotests/184 | 1 + tests/qemu-iotests/186 | 1 + tests/qemu-iotests/197 | 1 + tests/qemu-iotests/215 | 1 + tests/qemu-iotests/221.out | 12 ++- tests/qemu-iotests/251 | 1 + tests/qemu-iotests/253.out | 12 ++- tests/qemu-iotests/common.filter | 4 +- tests/qemu-iotests/common.rc | 14 +++ 50 files changed, 391 insertions(+), 87 deletions(-) rename tests/qemu-iotests/{150.out => 150.out.qcow2} (100%) create mode 100644 tests/qemu-iotests/150.out.raw -- 2.21.0 From: Denis Plotnikov <dplotnikov@virtuozzo.com> The patch allows to provide a pattern file for write command. There was no similar ability before. Signed-off-by: Denis Plotnikov <dplotnikov@virtuozzo.com> Message-id: 20190820164616.4072-1-dplotnikov@virtuozzo.com Reviewed-by: Eric Blake <eblake@redhat.com> [mreitz: Keep optstring in alphabetical order] Signed-off-by: Max Reitz <mreitz@redhat.com> --- qemu-io-cmds.c | 99 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 93 insertions(+), 6 deletions(-) diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c index XXXXXXX..XXXXXXX 100644 --- a/qemu-io-cmds.c +++ b/qemu-io-cmds.c @@ -XXX,XX +XXX,XX @@ static void qemu_io_free(void *p) qemu_vfree(p); } +/* + * qemu_io_alloc_from_file() + * + * Allocates the buffer and populates it with the content of the given file + * up to @len bytes. If the file length is less than @len, then the buffer + * is populated with the file content cyclically. + * + * @blk - the block backend where the buffer content is going to be written to + * @len - the buffer length + * @file_name - the file to read the content from + * + * Returns: the buffer pointer on success + * NULL on error + */ +static void *qemu_io_alloc_from_file(BlockBackend *blk, size_t len, + const char *file_name) +{ + char *buf, *buf_origin; + FILE *f = fopen(file_name, "r"); + int pattern_len; + + if (!f) { + perror(file_name); + return NULL; + } + + if (qemuio_misalign) { + len += MISALIGN_OFFSET; + } + + buf_origin = buf = blk_blockalign(blk, len); + + if (qemuio_misalign) { + buf_origin += MISALIGN_OFFSET; + buf += MISALIGN_OFFSET; + len -= MISALIGN_OFFSET; + } + + pattern_len = fread(buf_origin, 1, len, f); + + if (ferror(f)) { + perror(file_name); + goto error; + } + + if (pattern_len == 0) { + fprintf(stderr, "%s: file is empty\n", file_name); + goto error; + } + + fclose(f); + + if (len > pattern_len) { + len -= pattern_len; + buf += pattern_len; + + while (len > 0) { + size_t len_to_copy = MIN(pattern_len, len); + + memcpy(buf, buf_origin, len_to_copy); + + len -= len_to_copy; + buf += len_to_copy; + } + } + + return buf_origin; + +error: + qemu_io_free(buf_origin); + return NULL; +} + static void dump_buffer(const void *buffer, int64_t offset, int64_t len) { uint64_t i; @@ -XXX,XX +XXX,XX @@ static void write_help(void) " -n, -- with -z, don't allow slow fallback\n" " -p, -- ignored for backwards compatibility\n" " -P, -- use different pattern to fill file\n" +" -s, -- use a pattern file to fill the write buffer\n" " -C, -- report statistics in a machine parsable format\n" " -q, -- quiet mode, do not show I/O statistics\n" " -u, -- with -z, allow unmapping\n" @@ -XXX,XX +XXX,XX @@ static const cmdinfo_t write_cmd = { .perm = BLK_PERM_WRITE, .argmin = 2, .argmax = -1, - .args = "[-bcCfnquz] [-P pattern] off len", + .args = "[-bcCfnquz] [-P pattern | -s source_file] off len", .oneline = "writes a number of bytes at a specified offset", .help = write_help, }; @@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv) { struct timespec t1, t2; bool Cflag = false, qflag = false, bflag = false; - bool Pflag = false, zflag = false, cflag = false; + bool Pflag = false, zflag = false, cflag = false, sflag = false; int flags = 0; int c, cnt, ret; char *buf = NULL; @@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv) /* Some compilers get confused and warn if this is not initialized. */ int64_t total = 0; int pattern = 0xcd; + const char *file_name = NULL; - while ((c = getopt(argc, argv, "bcCfnpP:quz")) != -1) { + while ((c = getopt(argc, argv, "bcCfnpP:qs:uz")) != -1) { switch (c) { case 'b': bflag = true; @@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv) case 'q': qflag = true; break; + case 's': + sflag = true; + file_name = optarg; + break; case 'u': flags |= BDRV_REQ_MAY_UNMAP; break; @@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv) return -EINVAL; } - if (zflag && Pflag) { - printf("-z and -P cannot be specified at the same time\n"); + if (zflag + Pflag + sflag > 1) { + printf("Only one of -z, -P, and -s " + "can be specified at the same time\n"); return -EINVAL; } @@ -XXX,XX +XXX,XX @@ static int write_f(BlockBackend *blk, int argc, char **argv) } if (!zflag) { - buf = qemu_io_alloc(blk, count, pattern); + if (sflag) { + buf = qemu_io_alloc_from_file(blk, count, file_name); + if (!buf) { + return -EINVAL; + } + } else { + buf = qemu_io_alloc(blk, count, pattern); + } } clock_gettime(CLOCK_MONOTONIC, &t1); -- 2.21.0 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> It's wrong to OR shared permissions. It may lead to crash on further permission updates. Also, no needs to consider previously calculated permissions, as at this point we already bind all new parents and bdrv_get_cumulative_perm result is enough. So fix the bug by just set permissions by bdrv_get_cumulative_perm result. Bug was introduced in long ago 234ac1a9025, in 2.9. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Message-id: 20190824100740.61635-1-vsementsov@virtuozzo.com Signed-off-by: Max Reitz <mreitz@redhat.com> --- block.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/block.c b/block.c index XXXXXXX..XXXXXXX 100644 --- a/block.c +++ b/block.c @@ -XXX,XX +XXX,XX @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to, { BdrvChild *c, *next; GSList *list = NULL, *p; - uint64_t old_perm, old_shared; uint64_t perm = 0, shared = BLK_PERM_ALL; int ret; @@ -XXX,XX +XXX,XX @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to, bdrv_unref(from); } - bdrv_get_cumulative_perm(to, &old_perm, &old_shared); - bdrv_set_perm(to, old_perm | perm, old_shared | shared); + bdrv_get_cumulative_perm(to, &perm, &shared); + bdrv_set_perm(to, perm, shared); out: g_slist_free(list); -- 2.21.0 From: Nir Soffer <nirsof@gmail.com> When creating an image with preallocation "off" or "falloc", the first block of the image is typically not allocated. When using Gluster storage backed by XFS filesystem, reading this block using direct I/O succeeds regardless of request length, fooling alignment detection. In this case we fallback to a safe value (4096) instead of the optimal value (512), which may lead to unneeded data copying when aligning requests. Allocating the first block avoids the fallback. Since we allocate the first block even with preallocation=off, we no longer create images with zero disk size: $ ./qemu-img create -f raw test.raw 1g Formatting 'test.raw', fmt=raw size=1073741824 $ ls -lhs test.raw 4.0K -rw-r--r--. 1 nsoffer nsoffer 1.0G Aug 16 23:48 test.raw And converting the image requires additional cluster: $ ./qemu-img measure -f raw -O qcow2 test.raw required size: 458752 fully allocated size: 1074135040 When using format like vmdk with multiple files per image, we allocate one block per file: $ ./qemu-img create -f vmdk -o subformat=twoGbMaxExtentFlat test.vmdk 4g Formatting 'test.vmdk', fmt=vmdk size=4294967296 compat6=off hwversion=undefined subformat=twoGbMaxExtentFlat $ ls -lhs test*.vmdk 4.0K -rw-r--r--. 1 nsoffer nsoffer 2.0G Aug 27 03:23 test-f001.vmdk 4.0K -rw-r--r--. 1 nsoffer nsoffer 2.0G Aug 27 03:23 test-f002.vmdk 4.0K -rw-r--r--. 1 nsoffer nsoffer 353 Aug 27 03:23 test.vmdk I did quick performance test for copying disks with qemu-img convert to new raw target image to Gluster storage with sector size of 512 bytes: for i in $(seq 10); do rm -f dst.raw sleep 10 time ./qemu-img convert -f raw -O raw -t none -T none src.raw dst.raw done Here is a table comparing the total time spent: Type Before(s) After(s) Diff(%) --------------------------------------- real 530.028 469.123 -11.4 user 17.204 10.768 -37.4 sys 17.881 7.011 -60.7 We can see very clear improvement in CPU usage. Signed-off-by: Nir Soffer <nsoffer@redhat.com> Message-id: 20190827010528.8818-2-nsoffer@redhat.com Reviewed-by: Max Reitz <mreitz@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/file-posix.c | 51 +++++++++++++++++++ tests/qemu-iotests/059.out | 2 +- tests/qemu-iotests/{150.out => 150.out.qcow2} | 0 tests/qemu-iotests/150.out.raw | 12 +++++ tests/qemu-iotests/175 | 19 ++++--- tests/qemu-iotests/175.out | 8 +-- tests/qemu-iotests/178.out.qcow2 | 4 +- tests/qemu-iotests/221.out | 12 +++-- tests/qemu-iotests/253.out | 12 +++-- 9 files changed, 99 insertions(+), 21 deletions(-) rename tests/qemu-iotests/{150.out => 150.out.qcow2} (100%) create mode 100644 tests/qemu-iotests/150.out.raw diff --git a/block/file-posix.c b/block/file-posix.c index XXXXXXX..XXXXXXX 100644 --- a/block/file-posix.c +++ b/block/file-posix.c @@ -XXX,XX +XXX,XX @@ static int handle_aiocb_discard(void *opaque) return ret; } +/* + * Help alignment probing by allocating the first block. + * + * When reading with direct I/O from unallocated area on Gluster backed by XFS, + * reading succeeds regardless of request length. In this case we fallback to + * safe alignment which is not optimal. Allocating the first block avoids this + * fallback. + * + * fd may be opened with O_DIRECT, but we don't know the buffer alignment or + * request alignment, so we use safe values. + * + * Returns: 0 on success, -errno on failure. Since this is an optimization, + * caller may ignore failures. + */ +static int allocate_first_block(int fd, size_t max_size) +{ + size_t write_size = (max_size < MAX_BLOCKSIZE) + ? BDRV_SECTOR_SIZE + : MAX_BLOCKSIZE; + size_t max_align = MAX(MAX_BLOCKSIZE, getpagesize()); + void *buf; + ssize_t n; + int ret; + + buf = qemu_memalign(max_align, write_size); + memset(buf, 0, write_size); + + do { + n = pwrite(fd, buf, write_size, 0); + } while (n == -1 && errno == EINTR); + + ret = (n == -1) ? -errno : 0; + + qemu_vfree(buf); + return ret; +} + static int handle_aiocb_truncate(void *opaque) { RawPosixAIOData *aiocb = opaque; @@ -XXX,XX +XXX,XX @@ static int handle_aiocb_truncate(void *opaque) /* posix_fallocate() doesn't set errno. */ error_setg_errno(errp, -result, "Could not preallocate new data"); + } else if (current_length == 0) { + /* + * posix_fallocate() uses fallocate() if the filesystem + * supports it, or fallback to manually writing zeroes. If + * fallocate() was used, unaligned reads from the fallocated + * area in raw_probe_alignment() will succeed, hence we need to + * allocate the first block. + * + * Optimize future alignment probing; ignore failures. + */ + allocate_first_block(fd, offset); } } else { result = 0; @@ -XXX,XX +XXX,XX @@ static int handle_aiocb_truncate(void *opaque) if (ftruncate(fd, offset) != 0) { result = -errno; error_setg_errno(errp, -result, "Could not resize file"); + } else if (current_length == 0 && offset > current_length) { + /* Optimize future alignment probing; ignore failures. */ + allocate_first_block(fd, offset); } return result; default: diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/059.out +++ b/tests/qemu-iotests/059.out @@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000 subformat=twoGbMax image: TEST_DIR/t.vmdk file format: vmdk virtual size: 0.977 TiB (1073741824000 bytes) -disk size: 16 KiB +disk size: 1.97 MiB Format specific information: cid: XXXXXXXX parent cid: XXXXXXXX diff --git a/tests/qemu-iotests/150.out b/tests/qemu-iotests/150.out.qcow2 similarity index 100% rename from tests/qemu-iotests/150.out rename to tests/qemu-iotests/150.out.qcow2 diff --git a/tests/qemu-iotests/150.out.raw b/tests/qemu-iotests/150.out.raw new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/qemu-iotests/150.out.raw @@ -XXX,XX +XXX,XX @@ +QA output created by 150 + +=== Mapping sparse conversion === + +Offset Length File +0 0x1000 TEST_DIR/t.IMGFMT + +=== Mapping non-sparse conversion === + +Offset Length File +0 0x100000 TEST_DIR/t.IMGFMT +*** done diff --git a/tests/qemu-iotests/175 b/tests/qemu-iotests/175 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/175 +++ b/tests/qemu-iotests/175 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 # the file size. This function hides the resulting difference in the # stat -c '%b' output. # Parameter 1: Number of blocks an empty file occupies -# Parameter 2: Image size in bytes +# Parameter 2: Minimal number of blocks in an image +# Parameter 3: Image size in bytes _filter_blocks() { extra_blocks=$1 - img_size=$2 + min_blocks=$2 + img_size=$3 - sed -e "s/blocks=$extra_blocks\\(\$\\|[^0-9]\\)/nothing allocated/" \ - -e "s/blocks=$((extra_blocks + img_size / 512))\\(\$\\|[^0-9]\\)/everything allocated/" + sed -e "s/blocks=$min_blocks\\(\$\\|[^0-9]\\)/min allocation/" \ + -e "s/blocks=$((extra_blocks + img_size / 512))\\(\$\\|[^0-9]\\)/max allocation/" } # get standard environment, filters and checks @@ -XXX,XX +XXX,XX @@ size=$((1 * 1024 * 1024)) touch "$TEST_DIR/empty" extra_blocks=$(stat -c '%b' "$TEST_DIR/empty") +# We always write the first byte; check how many blocks this filesystem +# allocates to match empty image alloation. +printf "\0" > "$TEST_DIR/empty" +min_blocks=$(stat -c '%b' "$TEST_DIR/empty") + echo echo "== creating image with default preallocation ==" _make_test_img $size | _filter_imgfmt -stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $size +stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size for mode in off full falloc; do echo echo "== creating image with preallocation $mode ==" IMGOPTS=preallocation=$mode _make_test_img $size | _filter_imgfmt - stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $size + stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size done # success, all done diff --git a/tests/qemu-iotests/175.out b/tests/qemu-iotests/175.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/175.out +++ b/tests/qemu-iotests/175.out @@ -XXX,XX +XXX,XX @@ QA output created by 175 == creating image with default preallocation == Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 -size=1048576, nothing allocated +size=1048576, min allocation == creating image with preallocation off == Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=off -size=1048576, nothing allocated +size=1048576, min allocation == creating image with preallocation full == Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=full -size=1048576, everything allocated +size=1048576, max allocation == creating image with preallocation falloc == Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=falloc -size=1048576, everything allocated +size=1048576, max allocation *** done diff --git a/tests/qemu-iotests/178.out.qcow2 b/tests/qemu-iotests/178.out.qcow2 index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/178.out.qcow2 +++ b/tests/qemu-iotests/178.out.qcow2 @@ -XXX,XX +XXX,XX @@ converted image file size in bytes: 196608 == raw input image with data (human) == Formatting 'TEST_DIR/t.qcow2', fmt=IMGFMT size=1073741824 -required size: 393216 +required size: 458752 fully allocated size: 1074135040 wrote 512/512 bytes at offset 512 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) @@ -XXX,XX +XXX,XX @@ converted image file size in bytes: 196608 Formatting 'TEST_DIR/t.qcow2', fmt=IMGFMT size=1073741824 { - "required": 393216, + "required": 458752, "fully-allocated": 1074135040 } wrote 512/512 bytes at offset 512 diff --git a/tests/qemu-iotests/221.out b/tests/qemu-iotests/221.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/221.out +++ b/tests/qemu-iotests/221.out @@ -XXX,XX +XXX,XX @@ QA output created by 221 === Check mapping of unaligned raw image === Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65537 -[{ "start": 0, "length": 66048, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] -[{ "start": 0, "length": 66048, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, +{ "start": 4096, "length": 61952, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, +{ "start": 4096, "length": 61952, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] wrote 1/1 bytes at offset 65536 1 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 65536, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, +{ "start": 4096, "length": 61440, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, { "start": 65536, "length": 1, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, { "start": 65537, "length": 511, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] -[{ "start": 0, "length": 65536, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, +{ "start": 4096, "length": 61440, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, { "start": 65536, "length": 1, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, { "start": 65537, "length": 511, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] *** done diff --git a/tests/qemu-iotests/253.out b/tests/qemu-iotests/253.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/253.out +++ b/tests/qemu-iotests/253.out @@ -XXX,XX +XXX,XX @@ QA output created by 253 === Check mapping of unaligned raw image === Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048575 -[{ "start": 0, "length": 1048576, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] -[{ "start": 0, "length": 1048576, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, +{ "start": 4096, "length": 1044480, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, +{ "start": 4096, "length": 1044480, "depth": 0, "zero": true, "data": false, "offset": OFFSET}] wrote 65535/65535 bytes at offset 983040 63.999 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -[{ "start": 0, "length": 983040, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, +{ "start": 4096, "length": 978944, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, { "start": 983040, "length": 65536, "depth": 0, "zero": false, "data": true, "offset": OFFSET}] -[{ "start": 0, "length": 983040, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, +[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, +{ "start": 4096, "length": 978944, "depth": 0, "zero": true, "data": false, "offset": OFFSET}, { "start": 983040, "length": 65536, "depth": 0, "zero": false, "data": true, "offset": OFFSET}] *** done -- 2.21.0 From: Nir Soffer <nirsof@gmail.com> Using block_resize we can test allocate_first_block() with file descriptor opened with O_DIRECT, ensuring that it works for any size larger than 4096 bytes. Testing smaller sizes is tricky as the result depends on the filesystem used for testing. For example on NFS any size will work since O_DIRECT does not require any alignment. Signed-off-by: Nir Soffer <nsoffer@redhat.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-id: 20190827010528.8818-3-nsoffer@redhat.com Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/175 | 28 ++++++++++++++++++++++++++++ tests/qemu-iotests/175.out | 8 ++++++++ 2 files changed, 36 insertions(+) diff --git a/tests/qemu-iotests/175 b/tests/qemu-iotests/175 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/175 +++ b/tests/qemu-iotests/175 @@ -XXX,XX +XXX,XX @@ _filter_blocks() -e "s/blocks=$((extra_blocks + img_size / 512))\\(\$\\|[^0-9]\\)/max allocation/" } +# Resize image using block_resize. +# Parameter 1: image path +# Parameter 2: new size +_block_resize() +{ + local path=$1 + local size=$2 + + $QEMU -qmp stdio -nographic -nodefaults \ + -blockdev file,node-name=file,filename=$path,cache.direct=on \ + <<EOF +{'execute': 'qmp_capabilities'} +{'execute': 'block_resize', 'arguments': {'node-name': 'file', 'size': $size}} +{'execute': 'quit'} +EOF +} + # get standard environment, filters and checks . ./common.rc . ./common.filter @@ -XXX,XX +XXX,XX @@ _supported_fmt raw _supported_proto file _supported_os Linux +_default_cache_mode none +_supported_cache_modes none directsync + size=$((1 * 1024 * 1024)) touch "$TEST_DIR/empty" @@ -XXX,XX +XXX,XX @@ for mode in off full falloc; do stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size done +for new_size in 4096 1048576; do + echo + echo "== resize empty image with block_resize ==" + _make_test_img 0 | _filter_imgfmt + _block_resize $TEST_IMG $new_size >/dev/null + stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $new_size +done + # success, all done echo "*** done" rm -f $seq.full diff --git a/tests/qemu-iotests/175.out b/tests/qemu-iotests/175.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/175.out +++ b/tests/qemu-iotests/175.out @@ -XXX,XX +XXX,XX @@ size=1048576, max allocation == creating image with preallocation falloc == Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=falloc size=1048576, max allocation + +== resize empty image with block_resize == +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0 +size=4096, min allocation + +== resize empty image with block_resize == +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0 +size=1048576, min allocation *** done -- 2.21.0 fe646693acc changed qemu-img create's output so that it no longer prints single quotes around parameter values. The subformat and adapter_type filters in _filter_img_create() have never been adapted to that change. Fixes: fe646693acc13ac48b98435d14149ab04dc597bc Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: John Snow <jsnow@redhat.com> Message-id: 20190815153638.4600-2-mreitz@redhat.com Reviewed-by: John Snow <jsnow@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/059.out | 16 ++++++++-------- tests/qemu-iotests/common.filter | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/059.out +++ b/tests/qemu-iotests/059.out @@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 qemu-io: can't open device TEST_DIR/t.vmdk: L1 size too big === Testing monolithicFlat creation and opening === -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 image: TEST_DIR/t.IMGFMT file format: IMGFMT virtual size: 2 GiB (2147483648 bytes) === Testing monolithicFlat with zeroed_grain === qemu-img: TEST_DIR/t.IMGFMT: Flat image can't enable zeroed grain -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 === Testing big twoGbMaxExtentFlat === -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000 subformat=twoGbMaxExtentFlat +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000 image: TEST_DIR/t.vmdk file format: vmdk virtual size: 0.977 TiB (1073741824000 bytes) @@ -XXX,XX +XXX,XX @@ Format specific information: qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Invalid extent line: RW 12582912 VMFS "dummy.IMGFMT" 1 === Testing truncated sparse === -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400 subformat=monolithicSparse +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400 qemu-img: Could not open 'TEST_DIR/t.IMGFMT': File truncated, expecting at least 13172736 bytes === Converting to streamOptimized from image with small cluster size=== @@ -XXX,XX +XXX,XX @@ wrote 512/512 bytes at offset 10240 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) === Testing monolithicFlat with internally generated JSON file name === -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 subformat=monolithicFlat +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 qemu-io: can't open: Cannot use relative extent paths with VMDK descriptor file 'json:{"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "read_aio"}' === Testing version 3 === @@ -XXX,XX +XXX,XX @@ read 512/512 bytes at offset 64931328 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) === Testing 4TB monolithicFlat creation and IO === -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4398046511104 subformat=monolithicFlat +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4398046511104 image: TEST_DIR/t.IMGFMT file format: IMGFMT virtual size: 4 TiB (4398046511104 bytes) @@ -XXX,XX +XXX,XX @@ read 1024/1024 bytes at offset 966367641600 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) === Testing qemu-img map on extents === -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 subformat=monolithicSparse +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 wrote 1024/1024 bytes at offset 65024 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) wrote 1024/1024 bytes at offset 2147483136 @@ -XXX,XX +XXX,XX @@ Offset Length Mapped to File 0 0x20000 0x3f0000 TEST_DIR/t.vmdk 0x7fff0000 0x20000 0x410000 TEST_DIR/t.vmdk 0x140000000 0x10000 0x430000 TEST_DIR/t.vmdk -Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 subformat=twoGbMaxExtentSparse +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 wrote 1024/1024 bytes at offset 65024 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) wrote 1024/1024 bytes at offset 2147483136 diff --git a/tests/qemu-iotests/common.filter b/tests/qemu-iotests/common.filter index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/common.filter +++ b/tests/qemu-iotests/common.filter @@ -XXX,XX +XXX,XX @@ _filter_img_create() -e "s# compat6=\\(on\\|off\\)##g" \ -e "s# static=\\(on\\|off\\)##g" \ -e "s# zeroed_grain=\\(on\\|off\\)##g" \ - -e "s# subformat='[^']*'##g" \ - -e "s# adapter_type='[^']*'##g" \ + -e "s# subformat=[^ ]*##g" \ + -e "s# adapter_type=[^ ]*##g" \ -e "s# hwversion=[^ ]*##g" \ -e "s# lazy_refcounts=\\(on\\|off\\)##g" \ -e "s# block_size=[0-9]\\+##g" \ -- 2.21.0 This makes iotest 033 pass with e.g. subformat=monolithicFlat. It also turns a former error in 059 into success. Signed-off-by: Max Reitz <mreitz@redhat.com> Message-id: 20190815153638.4600-3-mreitz@redhat.com Reviewed-by: John Snow <jsnow@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/vmdk.c | 54 ++++++++++++++++++++++++-------------- tests/qemu-iotests/059 | 7 +++-- tests/qemu-iotests/059.out | 4 ++- 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/block/vmdk.c b/block/vmdk.c index XXXXXXX..XXXXXXX 100644 --- a/block/vmdk.c +++ b/block/vmdk.c @@ -XXX,XX +XXX,XX @@ static const char *next_line(const char *s) } static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, - const char *desc_file_path, QDict *options, - Error **errp) + QDict *options, Error **errp) { int ret; int matches; @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, const char *p, *np; int64_t sectors = 0; int64_t flat_offset; + char *desc_file_dir = NULL; char *extent_path; BdrvChild *extent_file; BDRVVmdkState *s = bs->opaque; @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, continue; } - if (!path_is_absolute(fname) && !path_has_protocol(fname) && - !desc_file_path[0]) - { - bdrv_refresh_filename(bs->file->bs); - error_setg(errp, "Cannot use relative extent paths with VMDK " - "descriptor file '%s'", bs->file->bs->filename); - return -EINVAL; - } + if (path_is_absolute(fname)) { + extent_path = g_strdup(fname); + } else { + if (!desc_file_dir) { + desc_file_dir = bdrv_dirname(bs->file->bs, errp); + if (!desc_file_dir) { + bdrv_refresh_filename(bs->file->bs); + error_prepend(errp, "Cannot use relative paths with VMDK " + "descriptor file '%s': ", + bs->file->bs->filename); + ret = -EINVAL; + goto out; + } + } - extent_path = path_combine(desc_file_path, fname); + extent_path = g_strconcat(desc_file_dir, fname, NULL); + } ret = snprintf(extent_opt_prefix, 32, "extents.%d", s->num_extents); assert(ret < 32); @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, g_free(extent_path); if (local_err) { error_propagate(errp, local_err); - return -EINVAL; + ret = -EINVAL; + goto out; } /* save to extents array */ @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, 0, 0, 0, 0, 0, &extent, errp); if (ret < 0) { bdrv_unref_child(bs, extent_file); - return ret; + goto out; } extent->flat_start_offset = flat_offset << 9; } else if (!strcmp(type, "SPARSE") || !strcmp(type, "VMFSSPARSE")) { @@ -XXX,XX +XXX,XX @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs, g_free(buf); if (ret) { bdrv_unref_child(bs, extent_file); - return ret; + goto out; } extent = &s->extents[s->num_extents - 1]; } else if (!strcmp(type, "SESPARSE")) { ret = vmdk_open_se_sparse(bs, extent_file, bs->open_flags, errp); if (ret) { bdrv_unref_child(bs, extent_file); - return ret; + goto out; } extent = &s->extents[s->num_extents - 1]; } else { error_setg(errp, "Unsupported extent type '%s'", type); bdrv_unref_child(bs, extent_file); - return -ENOTSUP; + ret = -ENOTSUP; + goto out; } extent->type = g_strdup(type); } - return 0; + + ret = 0; + goto out; invalid: np = next_line(p); @@ -XXX,XX +XXX,XX @@ invalid: np--; } error_setg(errp, "Invalid extent line: %.*s", (int)(np - p), p); - return -EINVAL; + ret = -EINVAL; + +out: + g_free(desc_file_dir); + return ret; } static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf, @@ -XXX,XX +XXX,XX @@ static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf, } s->create_type = g_strdup(ct); s->desc_offset = 0; - ret = vmdk_parse_extents(buf, bs, bs->file->bs->exact_filename, options, - errp); + ret = vmdk_parse_extents(buf, bs, options, errp); exit: return ret; } diff --git a/tests/qemu-iotests/059 b/tests/qemu-iotests/059 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/059 +++ b/tests/qemu-iotests/059 @@ -XXX,XX +XXX,XX @@ $QEMU_IMG convert -f qcow2 -O vmdk -o subformat=streamOptimized "$TEST_IMG.qcow2 echo echo "=== Testing monolithicFlat with internally generated JSON file name ===" +# Should work, because bdrv_dirname() works fine with blkdebug IMGOPTS="subformat=monolithicFlat" _make_test_img 64M -$QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" 2>&1 \ - | _filter_testdir | _filter_imgfmt +$QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" \ + -c info \ + 2>&1 \ + | _filter_testdir | _filter_imgfmt | _filter_img_info _cleanup_test_img echo diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/059.out +++ b/tests/qemu-iotests/059.out @@ -XXX,XX +XXX,XX @@ wrote 512/512 bytes at offset 10240 === Testing monolithicFlat with internally generated JSON file name === Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 -qemu-io: can't open: Cannot use relative extent paths with VMDK descriptor file 'json:{"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "read_aio"}' +format name: IMGFMT +cluster size: 0 bytes +vm state offset: 0 bytes === Testing version 3 === image: TEST_DIR/iotest-version3.IMGFMT -- 2.21.0 We had a test for a case where relative extent paths did not work, but unfortunately we just fixed the underlying problem, so it works now. This patch adds a new test case that still fails. Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: John Snow <jsnow@redhat.com> Message-id: 20190815153638.4600-4-mreitz@redhat.com Reviewed-by: John Snow <jsnow@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/059 | 27 +++++++++++++++++++++++++++ tests/qemu-iotests/059.out | 4 ++++ 2 files changed, 31 insertions(+) diff --git a/tests/qemu-iotests/059 b/tests/qemu-iotests/059 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/059 +++ b/tests/qemu-iotests/059 @@ -XXX,XX +XXX,XX @@ $QEMU_IMG convert -f qcow2 -O vmdk -o subformat=streamOptimized "$TEST_IMG.qcow2 echo echo "=== Testing monolithicFlat with internally generated JSON file name ===" + +echo '--- blkdebug ---' # Should work, because bdrv_dirname() works fine with blkdebug IMGOPTS="subformat=monolithicFlat" _make_test_img 64M $QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" \ @@ -XXX,XX +XXX,XX @@ $QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TE | _filter_testdir | _filter_imgfmt | _filter_img_info _cleanup_test_img +echo '--- quorum ---' +# Should not work, because bdrv_dirname() does not work with quorum +IMGOPTS="subformat=monolithicFlat" _make_test_img 64M +cp "$TEST_IMG" "$TEST_IMG.orig" + +filename="json:{ + \"driver\": \"$IMGFMT\", + \"file\": { + \"driver\": \"quorum\", + \"children\": [ { + \"driver\": \"file\", + \"filename\": \"$TEST_IMG\" + }, { + \"driver\": \"file\", + \"filename\": \"$TEST_IMG.orig\" + } ], + \"vote-threshold\": 1 + } }" + +filename=$(echo "$filename" | tr '\n' ' ' | sed -e 's/\s\+/ /g') +$QEMU_IMG info "$filename" 2>&1 \ + | sed -e "s/'json:[^']*'/\$QUORUM_FILE/g" \ + | _filter_testdir | _filter_imgfmt | _filter_img_info + + echo echo "=== Testing version 3 ===" _use_sample_img iotest-version3.vmdk.bz2 diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/059.out +++ b/tests/qemu-iotests/059.out @@ -XXX,XX +XXX,XX @@ wrote 512/512 bytes at offset 10240 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) === Testing monolithicFlat with internally generated JSON file name === +--- blkdebug --- Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 format name: IMGFMT cluster size: 0 bytes vm state offset: 0 bytes +--- quorum --- +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 +qemu-img: Could not open $QUORUM_FILE: Cannot use relative paths with VMDK descriptor file $QUORUM_FILE: Cannot generate a base directory for quorum nodes === Testing version 3 === image: TEST_DIR/iotest-version3.IMGFMT -- 2.21.0 Compressed writes generally have to write full clusters, not just in theory but also in practice when it comes to vmdk's streamOptimized subformat. It currently is just silently broken for writes with non-zero in-cluster offsets: $ qemu-img create -f vmdk -o subformat=streamOptimized foo.vmdk 1M $ qemu-io -c 'write 4k 4k' -c 'read 4k 4k' foo.vmdk wrote 4096/4096 bytes at offset 4096 4 KiB, 1 ops; 00.01 sec (443.724 KiB/sec and 110.9309 ops/sec) read failed: Invalid argument (The technical reason is that vmdk_write_extent() just writes the incomplete compressed data actually to offset 4k. When reading the data, vmdk_read_extent() looks at offset 0 and finds the compressed data size to be 0, because that is what it reads from there. This yields an error.) For incomplete writes with zero in-cluster offsets, the error path when reading the rest of the cluster is a bit different, but the result is the same: $ qemu-img create -f vmdk -o subformat=streamOptimized foo.vmdk 1M $ qemu-io -c 'write 0k 4k' -c 'read 4k 4k' foo.vmdk wrote 4096/4096 bytes at offset 0 4 KiB, 1 ops; 00.01 sec (362.641 KiB/sec and 90.6603 ops/sec) read failed: Invalid argument (Here, vmdk_read_extent() finds the data and then sees that the uncompressed data is short.) It is better to reject invalid writes than to make the user believe they might have succeeded and then fail when trying to read it back. Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: John Snow <jsnow@redhat.com> Message-id: 20190815153638.4600-5-mreitz@redhat.com Reviewed-by: John Snow <jsnow@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/vmdk.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/block/vmdk.c b/block/vmdk.c index XXXXXXX..XXXXXXX 100644 --- a/block/vmdk.c +++ b/block/vmdk.c @@ -XXX,XX +XXX,XX @@ static int vmdk_write_extent(VmdkExtent *extent, int64_t cluster_offset, if (extent->compressed) { void *compressed_data; + /* Only whole clusters */ + if (offset_in_cluster || + n_bytes > (extent->cluster_sectors * SECTOR_SIZE) || + (n_bytes < (extent->cluster_sectors * SECTOR_SIZE) && + offset + n_bytes != extent->end_sector * SECTOR_SIZE)) + { + ret = -EINVAL; + goto out; + } + if (!extent->has_marker) { ret = -EINVAL; goto out; -- 2.21.0 streamOptimized does not support writes that do not span exactly one cluster. Furthermore, it cannot rewrite already allocated clusters. As such, many iotests do not work with it. Disable them. Signed-off-by: Max Reitz <mreitz@redhat.com> Message-id: 20190815153638.4600-6-mreitz@redhat.com Reviewed-by: John Snow <jsnow@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/002 | 1 + tests/qemu-iotests/003 | 1 + tests/qemu-iotests/005 | 3 ++- tests/qemu-iotests/009 | 1 + tests/qemu-iotests/010 | 1 + tests/qemu-iotests/011 | 1 + tests/qemu-iotests/017 | 3 ++- tests/qemu-iotests/018 | 3 ++- tests/qemu-iotests/019 | 3 ++- tests/qemu-iotests/020 | 3 ++- tests/qemu-iotests/027 | 1 + tests/qemu-iotests/032 | 1 + tests/qemu-iotests/033 | 1 + tests/qemu-iotests/034 | 3 ++- tests/qemu-iotests/037 | 3 ++- tests/qemu-iotests/063 | 3 ++- tests/qemu-iotests/072 | 1 + tests/qemu-iotests/105 | 3 ++- tests/qemu-iotests/197 | 1 + tests/qemu-iotests/215 | 1 + tests/qemu-iotests/251 | 1 + 21 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/qemu-iotests/002 b/tests/qemu-iotests/002 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/002 +++ b/tests/qemu-iotests/002 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt generic _supported_proto generic +_unsupported_imgopts "subformat=streamOptimized" size=128M diff --git a/tests/qemu-iotests/003 b/tests/qemu-iotests/003 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/003 +++ b/tests/qemu-iotests/003 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt generic _supported_proto generic +_unsupported_imgopts "subformat=streamOptimized" size=128M offset=67M diff --git a/tests/qemu-iotests/005 b/tests/qemu-iotests/005 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/005 +++ b/tests/qemu-iotests/005 @@ -XXX,XX +XXX,XX @@ _supported_fmt generic _supported_proto generic _supported_os Linux _unsupported_imgopts "subformat=twoGbMaxExtentFlat" \ - "subformat=twoGbMaxExtentSparse" + "subformat=twoGbMaxExtentSparse" \ + "subformat=streamOptimized" # vpc is limited to 127GB, so we can't test it here if [ "$IMGFMT" = "vpc" ]; then diff --git a/tests/qemu-iotests/009 b/tests/qemu-iotests/009 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/009 +++ b/tests/qemu-iotests/009 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt generic _supported_proto generic +_unsupported_imgopts "subformat=streamOptimized" size=6G diff --git a/tests/qemu-iotests/010 b/tests/qemu-iotests/010 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/010 +++ b/tests/qemu-iotests/010 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt generic _supported_proto generic +_unsupported_imgopts "subformat=streamOptimized" size=6G diff --git a/tests/qemu-iotests/011 b/tests/qemu-iotests/011 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/011 +++ b/tests/qemu-iotests/011 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt generic _supported_proto generic +_unsupported_imgopts "subformat=streamOptimized" size=6G diff --git a/tests/qemu-iotests/017 b/tests/qemu-iotests/017 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/017 +++ b/tests/qemu-iotests/017 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt qcow qcow2 vmdk qed _supported_proto generic _unsupported_proto vxhs -_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" +_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \ + "subformat=streamOptimized" TEST_OFFSETS="0 4294967296" diff --git a/tests/qemu-iotests/018 b/tests/qemu-iotests/018 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/018 +++ b/tests/qemu-iotests/018 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt qcow qcow2 vmdk qed _supported_proto file _supported_os Linux -_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" +_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \ + "streamOptimized" TEST_OFFSETS="0 4294967296" diff --git a/tests/qemu-iotests/019 b/tests/qemu-iotests/019 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/019 +++ b/tests/qemu-iotests/019 @@ -XXX,XX +XXX,XX @@ _supported_proto file _supported_os Linux _unsupported_imgopts "subformat=monolithicFlat" \ "subformat=twoGbMaxExtentFlat" \ - "subformat=twoGbMaxExtentSparse" + "subformat=twoGbMaxExtentSparse" \ + "subformat=streamOptimized" TEST_OFFSETS="0 4294967296" CLUSTER_SIZE=65536 diff --git a/tests/qemu-iotests/020 b/tests/qemu-iotests/020 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/020 +++ b/tests/qemu-iotests/020 @@ -XXX,XX +XXX,XX @@ _supported_fmt qcow qcow2 vmdk qed _supported_proto file _unsupported_imgopts "subformat=monolithicFlat" \ "subformat=twoGbMaxExtentFlat" \ - "subformat=twoGbMaxExtentSparse" + "subformat=twoGbMaxExtentSparse" \ + "subformat=streamOptimized" TEST_OFFSETS="0 4294967296" diff --git a/tests/qemu-iotests/027 b/tests/qemu-iotests/027 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/027 +++ b/tests/qemu-iotests/027 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt vmdk qcow qcow2 qed _supported_proto generic +_unsupported_imgopts "subformat=streamOptimized" size=128M diff --git a/tests/qemu-iotests/032 b/tests/qemu-iotests/032 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/032 +++ b/tests/qemu-iotests/032 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 # This works for any image format (though unlikely to segfault for raw) _supported_fmt generic _supported_proto generic +_unsupported_imgopts "subformat=streamOptimized" echo echo === Prepare image === diff --git a/tests/qemu-iotests/033 b/tests/qemu-iotests/033 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/033 +++ b/tests/qemu-iotests/033 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt generic _supported_proto generic +_unsupported_imgopts "subformat=streamOptimized" size=128M diff --git a/tests/qemu-iotests/034 b/tests/qemu-iotests/034 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/034 +++ b/tests/qemu-iotests/034 @@ -XXX,XX +XXX,XX @@ _supported_proto file _supported_os Linux _unsupported_imgopts "subformat=monolithicFlat" \ "subformat=twoGbMaxExtentFlat" \ - "subformat=twoGbMaxExtentSparse" + "subformat=twoGbMaxExtentSparse" \ + "subformat=streamOptimized" CLUSTER_SIZE=4k size=128M diff --git a/tests/qemu-iotests/037 b/tests/qemu-iotests/037 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/037 +++ b/tests/qemu-iotests/037 @@ -XXX,XX +XXX,XX @@ _supported_fmt qcow qcow2 vmdk qed _supported_proto file _unsupported_imgopts "subformat=monolithicFlat" \ "subformat=twoGbMaxExtentFlat" \ - "subformat=twoGbMaxExtentSparse" + "subformat=twoGbMaxExtentSparse" \ + "subformat=streamOptimized" CLUSTER_SIZE=4k size=128M diff --git a/tests/qemu-iotests/063 b/tests/qemu-iotests/063 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/063 +++ b/tests/qemu-iotests/063 @@ -XXX,XX +XXX,XX @@ _supported_fmt qcow qcow2 vmdk qed raw _supported_proto file _unsupported_imgopts "subformat=monolithicFlat" \ "subformat=twoGbMaxExtentFlat" \ - "subformat=twoGbMaxExtentSparse" + "subformat=twoGbMaxExtentSparse" \ + "subformat=streamOptimized" _make_test_img 4M diff --git a/tests/qemu-iotests/072 b/tests/qemu-iotests/072 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/072 +++ b/tests/qemu-iotests/072 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt vpc vmdk vhdx vdi qed qcow2 qcow _supported_proto file +_unsupported_imgopts "subformat=streamOptimized" IMG_SIZE=64M diff --git a/tests/qemu-iotests/105 b/tests/qemu-iotests/105 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/105 +++ b/tests/qemu-iotests/105 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt qcow2 vmdk vhdx qed _supported_proto generic _unsupported_imgopts "subformat=twoGbMaxExtentFlat" \ - "subformat=twoGbMaxExtentSparse" + "subformat=twoGbMaxExtentSparse" \ + "subformat=streamOptimized" echo echo "creating large image" diff --git a/tests/qemu-iotests/197 b/tests/qemu-iotests/197 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/197 +++ b/tests/qemu-iotests/197 @@ -XXX,XX +XXX,XX @@ _supported_fmt generic _supported_proto generic # LUKS support may be possible, but it complicates things. _unsupported_fmt luks +_unsupported_imgopts "subformat=streamOptimized" echo echo '=== Copy-on-read ===' diff --git a/tests/qemu-iotests/215 b/tests/qemu-iotests/215 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/215 +++ b/tests/qemu-iotests/215 @@ -XXX,XX +XXX,XX @@ _supported_fmt generic _supported_proto generic # LUKS support may be possible, but it complicates things. _unsupported_fmt luks +_unsupported_imgopts "subformat=streamOptimized" echo echo '=== Copy-on-read ===' diff --git a/tests/qemu-iotests/251 b/tests/qemu-iotests/251 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/251 +++ b/tests/qemu-iotests/251 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt generic _supported_proto file _supported_os Linux +_unsupported_imgopts "subformat=streamOptimized" if [ "$IMGOPTSSYNTAX" = "true" ]; then # We use json:{} filenames here, so we cannot work with additional options. -- 2.21.0 The error message for the test case where we have a quorum node for which no directory name can be generated is different: For twoGbMaxExtentSparse, it complains that it cannot open the extent file. For other (sub)formats, it just notes that it cannot determine the backing file path. Both are fine, but just disable twoGbMaxExtentSparse for simplicity's sake. Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: John Snow <jsnow@redhat.com> Message-id: 20190815153638.4600-7-mreitz@redhat.com Reviewed-by: John Snow <jsnow@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/110 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/qemu-iotests/110 b/tests/qemu-iotests/110 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/110 +++ b/tests/qemu-iotests/110 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 # Any format supporting backing files _supported_fmt qed qcow qcow2 vmdk _supported_proto file -_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" +_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \ + "subformat=twoGbMaxExtentSparse" TEST_IMG_REL=$(basename "$TEST_IMG") -- 2.21.0 iotest 126 requires backing file support, which flat vmdks cannot offer. Skip this test for such subformats. Signed-off-by: Max Reitz <mreitz@redhat.com> Message-id: 20190815153638.4600-8-mreitz@redhat.com Reviewed-by: John Snow <jsnow@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/126 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/qemu-iotests/126 b/tests/qemu-iotests/126 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/126 +++ b/tests/qemu-iotests/126 @@ -XXX,XX +XXX,XX @@ status=1 # failure is the default! # Needs backing file support _supported_fmt qcow qcow2 qed vmdk +_unsupported_imgopts "subformat=monolithicFlat" \ + "subformat=twoGbMaxExtentFlat" # This is the default protocol (and we want to test the difference between # colons which separate a protocol prefix from the rest and colons which are # just part of the filename, so we cannot test protocols which require a prefix) -- 2.21.0 From: Stefan Hajnoczi <stefanha@redhat.com> Fixes: a6b257a08e3d72219f03e461a52152672fec0612 ("file-posix: Handle undetectable alignment") Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com> Message-id: 20190827101328.4062-1-stefanha@redhat.com Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/file-posix.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/file-posix.c b/block/file-posix.c index XXXXXXX..XXXXXXX 100644 --- a/block/file-posix.c +++ b/block/file-posix.c @@ -XXX,XX +XXX,XX @@ static void raw_probe_alignment(BlockDriverState *bs, int fd, Error **errp) for (i = 0; i < ARRAY_SIZE(alignments); i++) { align = alignments[i]; if (raw_is_io_aligned(fd, buf + align, max_align)) { - /* Fallback to request_aligment. */ + /* Fallback to request_alignment. */ s->buf_align = (align != 1) ? align : bs->bl.request_alignment; break; } -- 2.21.0 From: Thomas Huth <thuth@redhat.com> It is possible to enable only a subset of the block drivers with the "--block-drv-rw-whitelist" option of the "configure" script. All other drivers are marked as unusable (or only included as read-only with the "--block-drv-ro-whitelist" option). If an iotest is now using such a disabled block driver, it is failing - which is bad, since at least the tests in the "auto" group should be able to deal with this situation. Thus let's introduce a "_require_drivers" function that can be used by the shell tests to check for the availability of certain drivers first, and marks the test as "not run" if one of the drivers is missing. This patch mainly targets the test in the "auto" group which should never fail in such a case, but also improves some of the other tests along the way. Note that we also assume that the "qcow2" and "file" drivers are always available - otherwise it does not make sense to run "make check-block" at all (which only tests with qcow2 by default). Signed-off-by: Thomas Huth <thuth@redhat.com> Message-id: 20190823133552.11680-1-thuth@redhat.com Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/071 | 1 + tests/qemu-iotests/081 | 4 +--- tests/qemu-iotests/099 | 1 + tests/qemu-iotests/120 | 1 + tests/qemu-iotests/162 | 4 +--- tests/qemu-iotests/184 | 1 + tests/qemu-iotests/186 | 1 + tests/qemu-iotests/common.rc | 14 ++++++++++++++ 8 files changed, 21 insertions(+), 6 deletions(-) diff --git a/tests/qemu-iotests/071 b/tests/qemu-iotests/071 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/071 +++ b/tests/qemu-iotests/071 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt qcow2 _supported_proto file +_require_drivers blkdebug blkverify do_run_qemu() { diff --git a/tests/qemu-iotests/081 b/tests/qemu-iotests/081 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/081 +++ b/tests/qemu-iotests/081 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt raw _supported_proto file _supported_os Linux +_require_drivers quorum do_run_qemu() { @@ -XXX,XX +XXX,XX @@ run_qemu() | _filter_qemu_io | _filter_generated_node_ids } -test_quorum=$($QEMU_IMG --help|grep quorum) -[ "$test_quorum" = "" ] && _supported_fmt quorum - quorum="driver=raw,file.driver=quorum,file.vote-threshold=2" quorum="$quorum,file.children.0.file.filename=$TEST_DIR/1.raw" quorum="$quorum,file.children.1.file.filename=$TEST_DIR/2.raw" diff --git a/tests/qemu-iotests/099 b/tests/qemu-iotests/099 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/099 +++ b/tests/qemu-iotests/099 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt qcow qcow2 qed vdi vhdx vmdk vpc _supported_proto file _supported_os Linux +_require_drivers blkdebug blkverify _unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \ "subformat=twoGbMaxExtentSparse" diff --git a/tests/qemu-iotests/120 b/tests/qemu-iotests/120 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/120 +++ b/tests/qemu-iotests/120 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt generic _supported_proto file _unsupported_fmt luks +_require_drivers raw _make_test_img 64M diff --git a/tests/qemu-iotests/162 b/tests/qemu-iotests/162 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/162 +++ b/tests/qemu-iotests/162 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 . ./common.filter _supported_fmt generic - -test_ssh=$($QEMU_IMG --help | grep '^Supported formats:.* ssh\( \|$\)') -[ "$test_ssh" = "" ] && _notrun "ssh support required" +_require_drivers ssh echo echo '=== NBD ===' diff --git a/tests/qemu-iotests/184 b/tests/qemu-iotests/184 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/184 +++ b/tests/qemu-iotests/184 @@ -XXX,XX +XXX,XX @@ trap "exit \$status" 0 1 2 3 15 . ./common.filter _supported_os Linux +_require_drivers throttle do_run_qemu() { diff --git a/tests/qemu-iotests/186 b/tests/qemu-iotests/186 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/186 +++ b/tests/qemu-iotests/186 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt qcow2 _supported_proto file +_require_drivers null-co if [ "$QEMU_DEFAULT_MACHINE" != "pc" ]; then _notrun "Requires a PC machine" diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/common.rc +++ b/tests/qemu-iotests/common.rc @@ -XXX,XX +XXX,XX @@ _require_command() [ -x "$c" ] || _notrun "$1 utility required, skipped this test" } +# Check that a set of drivers has been whitelisted in the QEMU binary +# +_require_drivers() +{ + available=$($QEMU -drive format=help | \ + sed -e '/Supported formats:/!d' -e 's/Supported formats://') + for driver + do + if ! echo "$available" | grep -q " $driver\( \|$\)"; then + _notrun "$driver not available" + fi + done +} + # make sure this script returns success true -- 2.21.0 From: Thomas Huth <thuth@redhat.com> The sanitizers (especially the address sanitizer from Clang) are sometimes printing out warnings or false positives - this spoils the output of the iotests, causing some of the tests to fail. Thus let's skip the automatic iotests during "make check" when the user configured QEMU with --enable-sanitizers. Signed-off-by: Thomas Huth <thuth@redhat.com> Message-id: 20190823084203.29734-1-thuth@redhat.com Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/check-block.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/check-block.sh b/tests/check-block.sh index XXXXXXX..XXXXXXX 100755 --- a/tests/check-block.sh +++ b/tests/check-block.sh @@ -XXX,XX +XXX,XX @@ if grep -q "TARGET_GPROF=y" *-softmmu/config-target.mak 2>/dev/null ; then exit 0 fi +if grep -q "CFLAGS.*-fsanitize" config-host.mak 2>/dev/null ; then + echo "Sanitizers are enabled ==> Not running the qemu-iotests." + exit 0 +fi + if [ -z "$(find . -name 'qemu-system-*' -print)" ]; then echo "No qemu-system binary available ==> Not running the qemu-iotests." exit 0 -- 2.21.0 From: Nir Soffer <nirsof@gmail.com> Quoting cache mode is not needed, and most tests use unquoted values. Unify all test to use the same style. Message-id: 20190827173432.7656-1-nsoffer@redhat.com Signed-off-by: Nir Soffer <nsoffer@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/026 | 4 ++-- tests/qemu-iotests/039 | 4 ++-- tests/qemu-iotests/052 | 2 +- tests/qemu-iotests/091 | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/qemu-iotests/026 b/tests/qemu-iotests/026 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/026 +++ b/tests/qemu-iotests/026 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 # Currently only qcow2 supports rebasing _supported_fmt qcow2 _supported_proto file -_default_cache_mode "writethrough" -_supported_cache_modes "writethrough" "none" +_default_cache_mode writethrough +_supported_cache_modes writethrough none # The refcount table tests expect a certain minimum width for refcount entries # (so that the refcount table actually needs to grow); that minimum is 16 bits, # being the default refcount entry width. diff --git a/tests/qemu-iotests/039 b/tests/qemu-iotests/039 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/039 +++ b/tests/qemu-iotests/039 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt qcow2 _supported_proto file _supported_os Linux -_default_cache_mode "writethrough" -_supported_cache_modes "writethrough" +_default_cache_mode writethrough +_supported_cache_modes writethrough size=128M diff --git a/tests/qemu-iotests/052 b/tests/qemu-iotests/052 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/052 +++ b/tests/qemu-iotests/052 @@ -XXX,XX +XXX,XX @@ _supported_fmt generic _supported_proto file # Don't do O_DIRECT on tmpfs -_supported_cache_modes "writeback" "writethrough" "unsafe" +_supported_cache_modes writeback writethrough unsafe size=128M _make_test_img $size diff --git a/tests/qemu-iotests/091 b/tests/qemu-iotests/091 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/091 +++ b/tests/qemu-iotests/091 @@ -XXX,XX +XXX,XX @@ trap "_cleanup; exit \$status" 0 1 2 3 15 _supported_fmt qcow2 _supported_proto file _supported_os Linux -_default_cache_mode "none" -_supported_cache_modes "writethrough" "none" "writeback" +_default_cache_mode none +_supported_cache_modes writethrough none writeback size=1G -- 2.21.0 The following changes since commit 31ee895047bdcf7387e3570cbd2a473c6f744b08: Merge remote-tracking branch 'remotes/jasowang/tags/net-pull-request' into staging (2021-01-25 15:56:13 +0000) are available in the Git repository at: https://github.com/XanClic/qemu.git tags/pull-block-2021-01-26 for you to fetch changes up to bb24cdc5efee580e81f71c5ff0fd980f2cc179d0: iotests/178: Pass value to invalid option (2021-01-26 14:36:37 +0100) ---------------------------------------------------------------- Block patches: - Make backup block jobs use asynchronous requests with the block-copy module - Use COR filter node for stream block jobs - Make coroutine-sigaltstack’s qemu_coroutine_new() function thread-safe - Report error string when file locking fails with an unexpected errno - iotest fixes, additions, and some refactoring ---------------------------------------------------------------- Alberto Garcia (1): iotests: Add test for the regression fixed in c8bf9a9169 Andrey Shinkevich (10): copy-on-read: support preadv/pwritev_part functions block: add API function to insert a node copy-on-read: add filter drop function qapi: add filter-node-name to block-stream qapi: copy-on-read filter: add 'bottom' option iotests: add #310 to test bottom node in COR driver block: include supported_read_flags into BDS structure copy-on-read: skip non-guest reads if no copy needed stream: rework backing-file changing block: apply COR-filter to block-stream jobs David Edmondson (1): block: report errno when flock fcntl fails Max Reitz (14): iotests.py: Assume a couple of variables as given iotests/297: Rewrite in Python and extend reach iotests: Move try_remove to iotests.py iotests/129: Remove test images in tearDown() iotests/129: Do not check @busy iotests/129: Use throttle node iotests/129: Actually test a commit job iotests/129: Limit mirror job's buffer size iotests/129: Clean up pylint and mypy complaints iotests/300: Clean up pylint and mypy complaints coroutine-sigaltstack: Add SIGUSR2 mutex iotests/129: Limit backup's max-chunk/max-workers iotests/118: Drop 'change' test iotests/178: Pass value to invalid option Vladimir Sementsov-Ogievskiy (27): iotests: fix _check_o_direct qapi: block-stream: add "bottom" argument iotests: 30: prepare to COR filter insertion by stream job block/stream: add s->target_bs qapi: backup: add perf.use-copy-range parameter block/block-copy: More explicit call_state block/block-copy: implement block_copy_async block/block-copy: add max_chunk and max_workers parameters block/block-copy: add list of all call-states block/block-copy: add ratelimit to block-copy block/block-copy: add block_copy_cancel blockjob: add set_speed to BlockJobDriver job: call job_enter from job_pause qapi: backup: add max-chunk and max-workers to x-perf struct iotests: 56: prepare for backup over block-copy iotests: 185: prepare for backup over block-copy iotests: 219: prepare for backup over block-copy iotests: 257: prepare for backup over block-copy block/block-copy: make progress_bytes_callback optional block/backup: drop extra gotos from backup_run() backup: move to block-copy qapi: backup: disable copy_range by default block/block-copy: drop unused block_copy_set_progress_callback() block/block-copy: drop unused argument of block_copy() simplebench/bench_block_job: use correct shebang line with python3 simplebench: bench_block_job: add cmd_options argument simplebench: add bench-backup.py qapi/block-core.json | 66 +++++- block/backup-top.h | 1 + block/copy-on-read.h | 32 +++ include/block/block-copy.h | 61 ++++- include/block/block.h | 10 +- include/block/block_int.h | 15 +- include/block/blockjob_int.h | 2 + block.c | 25 ++ block/backup-top.c | 6 +- block/backup.c | 233 ++++++++++++------- block/block-copy.c | 227 +++++++++++++++--- block/copy-on-read.c | 184 ++++++++++++++- block/file-posix.c | 38 ++- block/io.c | 10 +- block/monitor/block-hmp-cmds.c | 7 +- block/replication.c | 2 + block/stream.c | 185 +++++++++------ blockdev.c | 83 +++++-- blockjob.c | 6 + job.c | 3 + util/coroutine-sigaltstack.c | 9 + scripts/simplebench/bench-backup.py | 167 ++++++++++++++ scripts/simplebench/bench-example.py | 2 +- scripts/simplebench/bench_block_job.py | 13 +- tests/qemu-iotests/030 | 12 +- tests/qemu-iotests/056 | 9 +- tests/qemu-iotests/109.out | 24 ++ tests/qemu-iotests/118 | 20 +- tests/qemu-iotests/118.out | 4 +- tests/qemu-iotests/124 | 8 +- tests/qemu-iotests/129 | 79 ++++--- tests/qemu-iotests/141.out | 2 +- tests/qemu-iotests/178 | 2 +- tests/qemu-iotests/178.out.qcow2 | 2 +- tests/qemu-iotests/178.out.raw | 2 +- tests/qemu-iotests/185 | 3 +- tests/qemu-iotests/185.out | 3 +- tests/qemu-iotests/219 | 13 +- tests/qemu-iotests/245 | 20 +- tests/qemu-iotests/257 | 1 + tests/qemu-iotests/257.out | 306 ++++++++++++------------- tests/qemu-iotests/297 | 112 +++++++-- tests/qemu-iotests/297.out | 5 +- tests/qemu-iotests/300 | 19 +- tests/qemu-iotests/310 | 117 ++++++++++ tests/qemu-iotests/310.out | 15 ++ tests/qemu-iotests/313 | 104 +++++++++ tests/qemu-iotests/313.out | 29 +++ tests/qemu-iotests/common.rc | 7 +- tests/qemu-iotests/group | 2 + tests/qemu-iotests/iotests.py | 37 +-- 51 files changed, 1797 insertions(+), 547 deletions(-) create mode 100644 block/copy-on-read.h create mode 100755 scripts/simplebench/bench-backup.py create mode 100755 tests/qemu-iotests/310 create mode 100644 tests/qemu-iotests/310.out create mode 100755 tests/qemu-iotests/313 create mode 100644 tests/qemu-iotests/313.out -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Unfortunately commit "iotests: handle tmpfs" breaks running iotests with -nbd -nocache, as _check_o_direct tries to create $TEST_IMG.test_o_direct, but in case of nbd TEST_IMG is something like nbd+unix:///... , and test fails with message qemu-img: nbd+unix:///?socket[...]test_o_direct: Protocol driver 'nbd' does not support image creation, and opening the image failed: Failed to connect to '/tmp/tmp.[...]/nbd/test_o_direct': No such file or directory Use TEST_DIR instead. Fixes: cfdca2b9f9d4ca26bb2b2dfe8de3149092e39170 Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Message-Id: <20201218182012.47607-1-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/common.rc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/common.rc +++ b/tests/qemu-iotests/common.rc @@ -XXX,XX +XXX,XX @@ _supported_cache_modes() # Check whether the filesystem supports O_DIRECT _check_o_direct() { - $QEMU_IMG create -f raw "$TEST_IMG".test_o_direct 1M > /dev/null - out=$($QEMU_IO -f raw -t none -c quit "$TEST_IMG".test_o_direct 2>&1) - rm -f "$TEST_IMG".test_o_direct + testfile="$TEST_DIR"/_check_o_direct + $QEMU_IMG create -f raw "$testfile" 1M > /dev/null + out=$($QEMU_IO -f raw -t none -c quit "$testfile" 2>&1) + rm -f "$testfile" [[ "$out" != *"O_DIRECT"* ]] } -- 2.29.2 From: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Add support for the recently introduced functions bdrv_co_preadv_part() and bdrv_co_pwritev_part() to the COR-filter driver. Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Message-Id: <20201216061703.70908-2-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/copy-on-read.c | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/block/copy-on-read.c b/block/copy-on-read.c index XXXXXXX..XXXXXXX 100644 --- a/block/copy-on-read.c +++ b/block/copy-on-read.c @@ -XXX,XX +XXX,XX @@ static int64_t cor_getlength(BlockDriverState *bs) } -static int coroutine_fn cor_co_preadv(BlockDriverState *bs, - uint64_t offset, uint64_t bytes, - QEMUIOVector *qiov, int flags) +static int coroutine_fn cor_co_preadv_part(BlockDriverState *bs, + uint64_t offset, uint64_t bytes, + QEMUIOVector *qiov, + size_t qiov_offset, + int flags) { - return bdrv_co_preadv(bs->file, offset, bytes, qiov, - flags | BDRV_REQ_COPY_ON_READ); + return bdrv_co_preadv_part(bs->file, offset, bytes, qiov, qiov_offset, + flags | BDRV_REQ_COPY_ON_READ); } -static int coroutine_fn cor_co_pwritev(BlockDriverState *bs, - uint64_t offset, uint64_t bytes, - QEMUIOVector *qiov, int flags) +static int coroutine_fn cor_co_pwritev_part(BlockDriverState *bs, + uint64_t offset, + uint64_t bytes, + QEMUIOVector *qiov, + size_t qiov_offset, int flags) { - - return bdrv_co_pwritev(bs->file, offset, bytes, qiov, flags); + return bdrv_co_pwritev_part(bs->file, offset, bytes, qiov, qiov_offset, + flags); } @@ -XXX,XX +XXX,XX @@ static BlockDriver bdrv_copy_on_read = { .bdrv_getlength = cor_getlength, - .bdrv_co_preadv = cor_co_preadv, - .bdrv_co_pwritev = cor_co_pwritev, + .bdrv_co_preadv_part = cor_co_preadv_part, + .bdrv_co_pwritev_part = cor_co_pwritev_part, .bdrv_co_pwrite_zeroes = cor_co_pwrite_zeroes, .bdrv_co_pdiscard = cor_co_pdiscard, .bdrv_co_pwritev_compressed = cor_co_pwritev_compressed, -- 2.29.2 From: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Provide API for insertion a node to backing chain. Suggested-by: Max Reitz <mreitz@redhat.com> Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20201216061703.70908-3-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- include/block/block.h | 2 ++ block.c | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/include/block/block.h b/include/block/block.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block.h +++ b/include/block/block.h @@ -XXX,XX +XXX,XX @@ void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top, Error **errp); void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to, Error **errp); +BlockDriverState *bdrv_insert_node(BlockDriverState *bs, QDict *node_options, + int flags, Error **errp); int bdrv_parse_aio(const char *mode, int *flags); int bdrv_parse_cache_mode(const char *mode, int *flags, bool *writethrough); diff --git a/block.c b/block.c index XXXXXXX..XXXXXXX 100644 --- a/block.c +++ b/block.c @@ -XXX,XX +XXX,XX @@ static void bdrv_delete(BlockDriverState *bs) g_free(bs); } +BlockDriverState *bdrv_insert_node(BlockDriverState *bs, QDict *node_options, + int flags, Error **errp) +{ + BlockDriverState *new_node_bs; + Error *local_err = NULL; + + new_node_bs = bdrv_open(NULL, NULL, node_options, flags, errp); + if (new_node_bs == NULL) { + error_prepend(errp, "Could not create node: "); + return NULL; + } + + bdrv_drained_begin(bs); + bdrv_replace_node(bs, new_node_bs, &local_err); + bdrv_drained_end(bs); + + if (local_err) { + bdrv_unref(new_node_bs); + error_propagate(errp, local_err); + return NULL; + } + + return new_node_bs; +} + /* * Run consistency checks on an image * -- 2.29.2 From: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Provide API for the COR-filter removal. Also, drop the filter child permissions for an inactive state when the filter node is being removed. To insert the filter, the block generic layer function bdrv_insert_node() can be used. The new function bdrv_cor_filter_drop() may be considered as an intermediate solution before the QEMU permission update system has overhauled. Then we are able to implement the API function bdrv_remove_node() on the block generic layer. Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20201216061703.70908-4-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/copy-on-read.h | 32 +++++++++++++++++++++++++ block/copy-on-read.c | 56 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 block/copy-on-read.h diff --git a/block/copy-on-read.h b/block/copy-on-read.h new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/block/copy-on-read.h @@ -XXX,XX +XXX,XX @@ +/* + * Copy-on-read filter block driver + * + * The filter driver performs Copy-On-Read (COR) operations + * + * Copyright (c) 2018-2020 Virtuozzo International GmbH. + * + * Author: + * Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef BLOCK_COPY_ON_READ +#define BLOCK_COPY_ON_READ + +#include "block/block_int.h" + +void bdrv_cor_filter_drop(BlockDriverState *cor_filter_bs); + +#endif /* BLOCK_COPY_ON_READ */ diff --git a/block/copy-on-read.c b/block/copy-on-read.c index XXXXXXX..XXXXXXX 100644 --- a/block/copy-on-read.c +++ b/block/copy-on-read.c @@ -XXX,XX +XXX,XX @@ #include "qemu/osdep.h" #include "block/block_int.h" #include "qemu/module.h" +#include "qapi/error.h" +#include "block/copy-on-read.h" + + +typedef struct BDRVStateCOR { + bool active; +} BDRVStateCOR; static int cor_open(BlockDriverState *bs, QDict *options, int flags, Error **errp) { + BDRVStateCOR *state = bs->opaque; + bs->file = bdrv_open_child(NULL, options, "file", bs, &child_of_bds, BDRV_CHILD_FILTERED | BDRV_CHILD_PRIMARY, false, errp); @@ -XXX,XX +XXX,XX @@ static int cor_open(BlockDriverState *bs, QDict *options, int flags, ((BDRV_REQ_FUA | BDRV_REQ_MAY_UNMAP | BDRV_REQ_NO_FALLBACK) & bs->file->bs->supported_zero_flags); + state->active = true; + + /* + * We don't need to call bdrv_child_refresh_perms() now as the permissions + * will be updated later when the filter node gets its parent. + */ + return 0; } @@ -XXX,XX +XXX,XX @@ static void cor_child_perm(BlockDriverState *bs, BdrvChild *c, uint64_t perm, uint64_t shared, uint64_t *nperm, uint64_t *nshared) { + BDRVStateCOR *s = bs->opaque; + + if (!s->active) { + /* + * While the filter is being removed + */ + *nperm = 0; + *nshared = BLK_PERM_ALL; + return; + } + *nperm = perm & PERM_PASSTHROUGH; *nshared = (shared & PERM_PASSTHROUGH) | PERM_UNCHANGED; @@ -XXX,XX +XXX,XX @@ static void cor_lock_medium(BlockDriverState *bs, bool locked) static BlockDriver bdrv_copy_on_read = { .format_name = "copy-on-read", + .instance_size = sizeof(BDRVStateCOR), .bdrv_open = cor_open, .bdrv_child_perm = cor_child_perm, @@ -XXX,XX +XXX,XX @@ static BlockDriver bdrv_copy_on_read = { .is_filter = true, }; + +void bdrv_cor_filter_drop(BlockDriverState *cor_filter_bs) +{ + BdrvChild *child; + BlockDriverState *bs; + BDRVStateCOR *s = cor_filter_bs->opaque; + + child = bdrv_filter_child(cor_filter_bs); + if (!child) { + return; + } + bs = child->bs; + + /* Retain the BDS until we complete the graph change. */ + bdrv_ref(bs); + /* Hold a guest back from writing while permissions are being reset. */ + bdrv_drained_begin(bs); + /* Drop permissions before the graph change. */ + s->active = false; + bdrv_child_refresh_perms(cor_filter_bs, child, &error_abort); + bdrv_replace_node(cor_filter_bs, bs, &error_abort); + + bdrv_drained_end(bs); + bdrv_unref(bs); + bdrv_unref(cor_filter_bs); +} + + static void bdrv_copy_on_read_init(void) { bdrv_register(&bdrv_copy_on_read); -- 2.29.2 From: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Provide the possibility to pass the 'filter-node-name' parameter to the block-stream job as it is done for the commit block job. Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> [vsementsov: comment indentation, s/Since: 5.2/Since: 6.0/] Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20201216061703.70908-5-vsementsov@virtuozzo.com> [mreitz: s/commit/stream/] Signed-off-by: Max Reitz <mreitz@redhat.com> --- qapi/block-core.json | 6 ++++++ include/block/block_int.h | 7 ++++++- block/monitor/block-hmp-cmds.c | 4 ++-- block/stream.c | 4 +++- blockdev.c | 4 +++- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/qapi/block-core.json b/qapi/block-core.json index XXXXXXX..XXXXXXX 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -XXX,XX +XXX,XX @@ # 'stop' and 'enospc' can only be used if the block device # supports io-status (see BlockInfo). Since 1.3. # +# @filter-node-name: the node name that should be assigned to the +# filter driver that the stream job inserts into the graph +# above @device. If this option is not given, a node name is +# autogenerated. (Since: 6.0) +# # @auto-finalize: When false, this job will wait in a PENDING state after it has # finished its work, waiting for @block-job-finalize before # making any block graph changes. @@ -XXX,XX +XXX,XX @@ 'data': { '*job-id': 'str', 'device': 'str', '*base': 'str', '*base-node': 'str', '*backing-file': 'str', '*speed': 'int', '*on-error': 'BlockdevOnError', + '*filter-node-name': 'str', '*auto-finalize': 'bool', '*auto-dismiss': 'bool' } } ## diff --git a/include/block/block_int.h b/include/block/block_int.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -XXX,XX +XXX,XX @@ int is_windows_drive(const char *filename); * See @BlockJobCreateFlags * @speed: The maximum speed, in bytes per second, or 0 for unlimited. * @on_error: The action to take upon error. + * @filter_node_name: The node name that should be assigned to the filter + * driver that the stream job inserts into the graph above + * @bs. NULL means that a node name should be autogenerated. * @errp: Error object. * * Start a streaming operation on @bs. Clusters that are unallocated @@ -XXX,XX +XXX,XX @@ int is_windows_drive(const char *filename); void stream_start(const char *job_id, BlockDriverState *bs, BlockDriverState *base, const char *backing_file_str, int creation_flags, int64_t speed, - BlockdevOnError on_error, Error **errp); + BlockdevOnError on_error, + const char *filter_node_name, + Error **errp); /** * commit_start: diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c index XXXXXXX..XXXXXXX 100644 --- a/block/monitor/block-hmp-cmds.c +++ b/block/monitor/block-hmp-cmds.c @@ -XXX,XX +XXX,XX @@ void hmp_block_stream(Monitor *mon, const QDict *qdict) qmp_block_stream(true, device, device, base != NULL, base, false, NULL, false, NULL, qdict_haskey(qdict, "speed"), speed, true, - BLOCKDEV_ON_ERROR_REPORT, false, false, false, false, - &error); + BLOCKDEV_ON_ERROR_REPORT, false, NULL, false, false, false, + false, &error); hmp_handle_error(mon, error); } diff --git a/block/stream.c b/block/stream.c index XXXXXXX..XXXXXXX 100644 --- a/block/stream.c +++ b/block/stream.c @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver stream_job_driver = { void stream_start(const char *job_id, BlockDriverState *bs, BlockDriverState *base, const char *backing_file_str, int creation_flags, int64_t speed, - BlockdevOnError on_error, Error **errp) + BlockdevOnError on_error, + const char *filter_node_name, + Error **errp) { StreamBlockJob *s; BlockDriverState *iter; diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, bool has_backing_file, const char *backing_file, bool has_speed, int64_t speed, bool has_on_error, BlockdevOnError on_error, + bool has_filter_node_name, const char *filter_node_name, bool has_auto_finalize, bool auto_finalize, bool has_auto_dismiss, bool auto_dismiss, Error **errp) @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, } stream_start(has_job_id ? job_id : NULL, bs, base_bs, base_name, - job_flags, has_speed ? speed : 0, on_error, &local_err); + job_flags, has_speed ? speed : 0, on_error, + filter_node_name, &local_err); if (local_err) { error_propagate(errp, local_err); goto out; -- 2.29.2 From: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Add an option to limit copy-on-read operations to specified sub-chain of backing-chain, to make copy-on-read filter useful for block-stream job. Suggested-by: Max Reitz <mreitz@redhat.com> Suggested-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> [vsementsov: change subject, modified to freeze the chain, do some fixes] Message-Id: <20201216061703.70908-6-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- qapi/block-core.json | 20 ++++++++- block/copy-on-read.c | 98 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 115 insertions(+), 3 deletions(-) diff --git a/qapi/block-core.json b/qapi/block-core.json index XXXXXXX..XXXXXXX 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -XXX,XX +XXX,XX @@ 'data': { 'throttle-group': 'str', 'file' : 'BlockdevRef' } } + +## +# @BlockdevOptionsCor: +# +# Driver specific block device options for the copy-on-read driver. +# +# @bottom: The name of a non-filter node (allocation-bearing layer) that +# limits the COR operations in the backing chain (inclusive), so +# that no data below this node will be copied by this filter. +# If option is absent, the limit is not applied, so that data +# from all backing layers may be copied. +# +# Since: 6.0 +## +{ 'struct': 'BlockdevOptionsCor', + 'base': 'BlockdevOptionsGenericFormat', + 'data': { '*bottom': 'str' } } + ## # @BlockdevOptions: # @@ -XXX,XX +XXX,XX @@ 'bochs': 'BlockdevOptionsGenericFormat', 'cloop': 'BlockdevOptionsGenericFormat', 'compress': 'BlockdevOptionsGenericFormat', - 'copy-on-read':'BlockdevOptionsGenericFormat', + 'copy-on-read':'BlockdevOptionsCor', 'dmg': 'BlockdevOptionsGenericFormat', 'file': 'BlockdevOptionsFile', 'ftp': 'BlockdevOptionsCurlFtp', diff --git a/block/copy-on-read.c b/block/copy-on-read.c index XXXXXXX..XXXXXXX 100644 --- a/block/copy-on-read.c +++ b/block/copy-on-read.c @@ -XXX,XX +XXX,XX @@ #include "block/block_int.h" #include "qemu/module.h" #include "qapi/error.h" +#include "qapi/qmp/qdict.h" #include "block/copy-on-read.h" typedef struct BDRVStateCOR { bool active; + BlockDriverState *bottom_bs; + bool chain_frozen; } BDRVStateCOR; static int cor_open(BlockDriverState *bs, QDict *options, int flags, Error **errp) { + BlockDriverState *bottom_bs = NULL; BDRVStateCOR *state = bs->opaque; + /* Find a bottom node name, if any */ + const char *bottom_node = qdict_get_try_str(options, "bottom"); bs->file = bdrv_open_child(NULL, options, "file", bs, &child_of_bds, BDRV_CHILD_FILTERED | BDRV_CHILD_PRIMARY, @@ -XXX,XX +XXX,XX @@ static int cor_open(BlockDriverState *bs, QDict *options, int flags, ((BDRV_REQ_FUA | BDRV_REQ_MAY_UNMAP | BDRV_REQ_NO_FALLBACK) & bs->file->bs->supported_zero_flags); + if (bottom_node) { + bottom_bs = bdrv_find_node(bottom_node); + if (!bottom_bs) { + error_setg(errp, "Bottom node '%s' not found", bottom_node); + qdict_del(options, "bottom"); + return -EINVAL; + } + qdict_del(options, "bottom"); + + if (!bottom_bs->drv) { + error_setg(errp, "Bottom node '%s' not opened", bottom_node); + return -EINVAL; + } + + if (bottom_bs->drv->is_filter) { + error_setg(errp, "Bottom node '%s' is a filter", bottom_node); + return -EINVAL; + } + + if (bdrv_freeze_backing_chain(bs, bottom_bs, errp) < 0) { + return -EINVAL; + } + state->chain_frozen = true; + + /* + * We do freeze the chain, so it shouldn't be removed. Still, storing a + * pointer worth bdrv_ref(). + */ + bdrv_ref(bottom_bs); + } state->active = true; + state->bottom_bs = bottom_bs; /* * We don't need to call bdrv_child_refresh_perms() now as the permissions @@ -XXX,XX +XXX,XX @@ static int coroutine_fn cor_co_preadv_part(BlockDriverState *bs, size_t qiov_offset, int flags) { - return bdrv_co_preadv_part(bs->file, offset, bytes, qiov, qiov_offset, - flags | BDRV_REQ_COPY_ON_READ); + int64_t n; + int local_flags; + int ret; + BDRVStateCOR *state = bs->opaque; + + if (!state->bottom_bs) { + return bdrv_co_preadv_part(bs->file, offset, bytes, qiov, qiov_offset, + flags | BDRV_REQ_COPY_ON_READ); + } + + while (bytes) { + local_flags = flags; + + /* In case of failure, try to copy-on-read anyway */ + ret = bdrv_is_allocated(bs->file->bs, offset, bytes, &n); + if (ret <= 0) { + ret = bdrv_is_allocated_above(bdrv_backing_chain_next(bs->file->bs), + state->bottom_bs, true, offset, + n, &n); + if (ret > 0 || ret < 0) { + local_flags |= BDRV_REQ_COPY_ON_READ; + } + /* Finish earlier if the end of a backing file has been reached */ + if (n == 0) { + break; + } + } + + ret = bdrv_co_preadv_part(bs->file, offset, n, qiov, qiov_offset, + local_flags); + if (ret < 0) { + return ret; + } + + offset += n; + qiov_offset += n; + bytes -= n; + } + + return 0; } @@ -XXX,XX +XXX,XX @@ static void cor_lock_medium(BlockDriverState *bs, bool locked) } +static void cor_close(BlockDriverState *bs) +{ + BDRVStateCOR *s = bs->opaque; + + if (s->chain_frozen) { + s->chain_frozen = false; + bdrv_unfreeze_backing_chain(bs, s->bottom_bs); + } + + bdrv_unref(s->bottom_bs); +} + + static BlockDriver bdrv_copy_on_read = { .format_name = "copy-on-read", .instance_size = sizeof(BDRVStateCOR), .bdrv_open = cor_open, + .bdrv_close = cor_close, .bdrv_child_perm = cor_child_perm, .bdrv_getlength = cor_getlength, @@ -XXX,XX +XXX,XX @@ void bdrv_cor_filter_drop(BlockDriverState *cor_filter_bs) bdrv_drained_begin(bs); /* Drop permissions before the graph change. */ s->active = false; + /* unfreeze, as otherwise bdrv_replace_node() will fail */ + if (s->chain_frozen) { + s->chain_frozen = false; + bdrv_unfreeze_backing_chain(cor_filter_bs, s->bottom_bs); + } bdrv_child_refresh_perms(cor_filter_bs, child, &error_abort); bdrv_replace_node(cor_filter_bs, bs, &error_abort); -- 2.29.2 From: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> The test case #310 is similar to #216 by Max Reitz. The difference is that the test #310 involves a bottom node to the COR filter driver. Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> [vsementsov: detach backing to test reads from top, limit to qcow2] Message-Id: <20201216061703.70908-7-vsementsov@virtuozzo.com> [mreitz: Add "# group:" line] Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/310 | 117 +++++++++++++++++++++++++++++++++++++ tests/qemu-iotests/310.out | 15 +++++ tests/qemu-iotests/group | 1 + 3 files changed, 133 insertions(+) create mode 100755 tests/qemu-iotests/310 create mode 100644 tests/qemu-iotests/310.out diff --git a/tests/qemu-iotests/310 b/tests/qemu-iotests/310 new file mode 100755 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/qemu-iotests/310 @@ -XXX,XX +XXX,XX @@ +#!/usr/bin/env python3 +# group: rw quick +# +# Copy-on-read tests using a COR filter with a bottom node +# +# Copyright (C) 2018 Red Hat, Inc. +# Copyright (c) 2020 Virtuozzo International GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +import iotests +from iotests import log, qemu_img, qemu_io_silent + +# Need backing file support +iotests.script_initialize(supported_fmts=['qcow2'], + supported_platforms=['linux']) + +log('') +log('=== Copy-on-read across nodes ===') +log('') + +# This test is similar to the 216 one by Max Reitz <mreitz@redhat.com> +# The difference is that this test case involves a bottom node to the +# COR filter driver. + +with iotests.FilePath('base.img') as base_img_path, \ + iotests.FilePath('mid.img') as mid_img_path, \ + iotests.FilePath('top.img') as top_img_path, \ + iotests.VM() as vm: + + log('--- Setting up images ---') + log('') + + assert qemu_img('create', '-f', iotests.imgfmt, base_img_path, '64M') == 0 + assert qemu_io_silent(base_img_path, '-c', 'write -P 1 0M 1M') == 0 + assert qemu_io_silent(base_img_path, '-c', 'write -P 1 3M 1M') == 0 + assert qemu_img('create', '-f', iotests.imgfmt, '-b', base_img_path, + '-F', iotests.imgfmt, mid_img_path) == 0 + assert qemu_io_silent(mid_img_path, '-c', 'write -P 3 2M 1M') == 0 + assert qemu_io_silent(mid_img_path, '-c', 'write -P 3 4M 1M') == 0 + assert qemu_img('create', '-f', iotests.imgfmt, '-b', mid_img_path, + '-F', iotests.imgfmt, top_img_path) == 0 + assert qemu_io_silent(top_img_path, '-c', 'write -P 2 1M 1M') == 0 + +# 0 1 2 3 4 +# top 2 +# mid 3 3 +# base 1 1 + + log('Done') + + log('') + log('--- Doing COR ---') + log('') + + vm.launch() + + log(vm.qmp('blockdev-add', + node_name='node0', + driver='copy-on-read', + bottom='node2', + file={ + 'driver': iotests.imgfmt, + 'file': { + 'driver': 'file', + 'filename': top_img_path + }, + 'backing': { + 'node-name': 'node2', + 'driver': iotests.imgfmt, + 'file': { + 'driver': 'file', + 'filename': mid_img_path + }, + 'backing': { + 'driver': iotests.imgfmt, + 'file': { + 'driver': 'file', + 'filename': base_img_path + } + }, + } + })) + + # Trigger COR + log(vm.qmp('human-monitor-command', + command_line='qemu-io node0 "read 0 5M"')) + + vm.shutdown() + + log('') + log('--- Checking COR result ---') + log('') + + # Detach backing to check that we can read the data from the top level now + assert qemu_img('rebase', '-u', '-b', '', '-f', iotests.imgfmt, + top_img_path) == 0 + + assert qemu_io_silent(top_img_path, '-c', 'read -P 0 0 1M') == 0 + assert qemu_io_silent(top_img_path, '-c', 'read -P 2 1M 1M') == 0 + assert qemu_io_silent(top_img_path, '-c', 'read -P 3 2M 1M') == 0 + assert qemu_io_silent(top_img_path, '-c', 'read -P 0 3M 1M') == 0 + assert qemu_io_silent(top_img_path, '-c', 'read -P 3 4M 1M') == 0 + + log('Done') diff --git a/tests/qemu-iotests/310.out b/tests/qemu-iotests/310.out new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/qemu-iotests/310.out @@ -XXX,XX +XXX,XX @@ + +=== Copy-on-read across nodes === + +--- Setting up images --- + +Done + +--- Doing COR --- + +{"return": {}} +{"return": ""} + +--- Checking COR result --- + +Done diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/group +++ b/tests/qemu-iotests/group @@ -XXX,XX +XXX,XX @@ 307 rw quick export 308 rw 309 rw auto quick +310 rw quick 312 rw quick -- 2.29.2 From: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Add the new member supported_read_flags to the BlockDriverState structure. It will control the flags set for copy-on-read operations. Make the block generic layer evaluate supported read flags before they go to a block driver. Suggested-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> [vsementsov: use assert instead of abort] Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20201216061703.70908-8-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- include/block/block_int.h | 4 ++++ block/io.c | 10 ++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/include/block/block_int.h b/include/block/block_int.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -XXX,XX +XXX,XX @@ struct BlockDriverState { /* I/O Limits */ BlockLimits bl; + /* + * Flags honored during pread + */ + unsigned int supported_read_flags; /* Flags honored during pwrite (so far: BDRV_REQ_FUA, * BDRV_REQ_WRITE_UNCHANGED). * If a driver does not support BDRV_REQ_WRITE_UNCHANGED, those diff --git a/block/io.c b/block/io.c index XXXXXXX..XXXXXXX 100644 --- a/block/io.c +++ b/block/io.c @@ -XXX,XX +XXX,XX @@ static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child, if (flags & BDRV_REQ_COPY_ON_READ) { int64_t pnum; + /* The flag BDRV_REQ_COPY_ON_READ has reached its addressee */ + flags &= ~BDRV_REQ_COPY_ON_READ; + ret = bdrv_is_allocated(bs, offset, bytes, &pnum); if (ret < 0) { goto out; @@ -XXX,XX +XXX,XX @@ static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child, goto out; } + assert(!(flags & ~bs->supported_read_flags)); + max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); if (bytes <= max_bytes && bytes <= max_transfer) { - ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, 0); + ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, flags); goto out; } @@ -XXX,XX +XXX,XX @@ static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child, ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining, num, qiov, - qiov_offset + bytes - bytes_remaining, 0); + qiov_offset + bytes - bytes_remaining, + flags); max_bytes -= num; } else { num = bytes_remaining; -- 2.29.2 From: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> If the flag BDRV_REQ_PREFETCH was set, skip idling read/write operations in COR-driver. It can be taken into account for the COR-algorithms optimization. That check is being made during the block stream job by the moment. Add the BDRV_REQ_PREFETCH flag to the supported_read_flags of the COR-filter. block: Modify the comment for the flag BDRV_REQ_PREFETCH as we are going to use it alone and pass it to the COR-filter driver for further processing. Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20201216061703.70908-9-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- include/block/block.h | 8 +++++--- block/copy-on-read.c | 14 ++++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/include/block/block.h b/include/block/block.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block.h +++ b/include/block/block.h @@ -XXX,XX +XXX,XX @@ typedef enum { BDRV_REQ_NO_FALLBACK = 0x100, /* - * BDRV_REQ_PREFETCH may be used only together with BDRV_REQ_COPY_ON_READ - * on read request and means that caller doesn't really need data to be - * written to qiov parameter which may be NULL. + * BDRV_REQ_PREFETCH makes sense only in the context of copy-on-read + * (i.e., together with the BDRV_REQ_COPY_ON_READ flag or when a COR + * filter is involved), in which case it signals that the COR operation + * need not read the data into memory (qiov) but only ensure they are + * copied to the top layer (i.e., that COR operation is done). */ BDRV_REQ_PREFETCH = 0x200, diff --git a/block/copy-on-read.c b/block/copy-on-read.c index XXXXXXX..XXXXXXX 100644 --- a/block/copy-on-read.c +++ b/block/copy-on-read.c @@ -XXX,XX +XXX,XX @@ static int cor_open(BlockDriverState *bs, QDict *options, int flags, return -EINVAL; } + bs->supported_read_flags = BDRV_REQ_PREFETCH; + bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED | (BDRV_REQ_FUA & bs->file->bs->supported_write_flags); @@ -XXX,XX +XXX,XX @@ static int coroutine_fn cor_co_preadv_part(BlockDriverState *bs, } } - ret = bdrv_co_preadv_part(bs->file, offset, n, qiov, qiov_offset, - local_flags); - if (ret < 0) { - return ret; + /* Skip if neither read nor write are needed */ + if ((local_flags & (BDRV_REQ_PREFETCH | BDRV_REQ_COPY_ON_READ)) != + BDRV_REQ_PREFETCH) { + ret = bdrv_co_preadv_part(bs->file, offset, n, qiov, qiov_offset, + local_flags); + if (ret < 0) { + return ret; + } } offset += n; -- 2.29.2 From: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Stream in stream_prepare calls bdrv_change_backing_file() to change backing-file in the metadata of bs. It may use either backing-file parameter given by user or just take filename of base on job start. Backing file format is determined by base on job finish. There are some problems with this design, we solve only two by this patch: 1. Consider scenario with backing-file unset. Current concept of stream supports changing of the base during the job (we don't freeze link to the base). So, we should not save base filename at job start, - let's determine name of the base on job finish. 2. Using direct base to determine filename and format is not very good: base node may be a filter, so its filename may be JSON, and format_name is not good for storing into qcow2 metadata as backing file format. - let's use unfiltered_base Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> [vsementsov: change commit subject, change logic in stream_prepare] Message-Id: <20201216061703.70908-10-vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/stream.c | 9 +++++---- blockdev.c | 8 +------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/block/stream.c b/block/stream.c index XXXXXXX..XXXXXXX 100644 --- a/block/stream.c +++ b/block/stream.c @@ -XXX,XX +XXX,XX @@ static int stream_prepare(Job *job) BlockDriverState *bs = blk_bs(bjob->blk); BlockDriverState *unfiltered_bs = bdrv_skip_filters(bs); BlockDriverState *base = bdrv_filter_or_cow_bs(s->above_base); + BlockDriverState *unfiltered_base = bdrv_skip_filters(base); Error *local_err = NULL; int ret = 0; @@ -XXX,XX +XXX,XX @@ static int stream_prepare(Job *job) if (bdrv_cow_child(unfiltered_bs)) { const char *base_id = NULL, *base_fmt = NULL; - if (base) { - base_id = s->backing_file_str; - if (base->drv) { - base_fmt = base->drv->format_name; + if (unfiltered_base) { + base_id = s->backing_file_str ?: unfiltered_base->filename; + if (unfiltered_base->drv) { + base_fmt = unfiltered_base->drv->format_name; } } bdrv_set_backing_hd(unfiltered_bs, base, &local_err); diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, BlockDriverState *base_bs = NULL; AioContext *aio_context; Error *local_err = NULL; - const char *base_name = NULL; int job_flags = JOB_DEFAULT; if (!has_on_error) { @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, goto out; } assert(bdrv_get_aio_context(base_bs) == aio_context); - base_name = base; } if (has_base_node) { @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, } assert(bdrv_get_aio_context(base_bs) == aio_context); bdrv_refresh_filename(base_bs); - base_name = base_bs->filename; } /* Check for op blockers in the whole chain between bs and base */ @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, goto out; } - /* backing_file string overrides base bs filename */ - base_name = has_backing_file ? backing_file : base_name; - if (has_auto_finalize && !auto_finalize) { job_flags |= JOB_MANUAL_FINALIZE; } @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, job_flags |= JOB_MANUAL_DISMISS; } - stream_start(has_job_id ? job_id : NULL, bs, base_bs, base_name, + stream_start(has_job_id ? job_id : NULL, bs, base_bs, backing_file, job_flags, has_speed ? speed : 0, on_error, filter_node_name, &local_err); if (local_err) { -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> The code already don't freeze base node and we try to make it prepared for the situation when base node is changed during the operation. In other words, block-stream doesn't own base node. Let's introduce a new interface which should replace the current one, which will in better relations with the code. Specifying bottom node instead of base, and requiring it to be non-filter gives us the following benefits: - drop difference between above_base and base_overlay, which will be renamed to just bottom, when old interface dropped - clean way to work with parallel streams/commits on the same backing chain, which otherwise become a problem when we introduce a filter for stream job - cleaner interface. Nobody will surprised the fact that base node may disappear during block-stream, when there is no word about "base" in the interface. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Message-Id: <20201216061703.70908-11-vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- qapi/block-core.json | 12 ++++--- include/block/block_int.h | 1 + block/monitor/block-hmp-cmds.c | 3 +- block/stream.c | 50 +++++++++++++++++++--------- blockdev.c | 59 ++++++++++++++++++++++++++++------ 5 files changed, 94 insertions(+), 31 deletions(-) diff --git a/qapi/block-core.json b/qapi/block-core.json index XXXXXXX..XXXXXXX 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -XXX,XX +XXX,XX @@ # @device: the device or node name of the top image # # @base: the common backing file name. -# It cannot be set if @base-node is also set. +# It cannot be set if @base-node or @bottom is also set. # # @base-node: the node name of the backing file. -# It cannot be set if @base is also set. (Since 2.8) +# It cannot be set if @base or @bottom is also set. (Since 2.8) +# +# @bottom: the last node in the chain that should be streamed into +# top. It cannot be set if @base or @base-node is also set. +# It cannot be filter node. (Since 6.0) # # @backing-file: The backing file string to write into the top # image. This filename is not validated. @@ -XXX,XX +XXX,XX @@ ## { 'command': 'block-stream', 'data': { '*job-id': 'str', 'device': 'str', '*base': 'str', - '*base-node': 'str', '*backing-file': 'str', '*speed': 'int', - '*on-error': 'BlockdevOnError', + '*base-node': 'str', '*backing-file': 'str', '*bottom': 'str', + '*speed': 'int', '*on-error': 'BlockdevOnError', '*filter-node-name': 'str', '*auto-finalize': 'bool', '*auto-dismiss': 'bool' } } diff --git a/include/block/block_int.h b/include/block/block_int.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -XXX,XX +XXX,XX @@ int is_windows_drive(const char *filename); */ void stream_start(const char *job_id, BlockDriverState *bs, BlockDriverState *base, const char *backing_file_str, + BlockDriverState *bottom, int creation_flags, int64_t speed, BlockdevOnError on_error, const char *filter_node_name, diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c index XXXXXXX..XXXXXXX 100644 --- a/block/monitor/block-hmp-cmds.c +++ b/block/monitor/block-hmp-cmds.c @@ -XXX,XX +XXX,XX @@ void hmp_block_stream(Monitor *mon, const QDict *qdict) int64_t speed = qdict_get_try_int(qdict, "speed", 0); qmp_block_stream(true, device, device, base != NULL, base, false, NULL, - false, NULL, qdict_haskey(qdict, "speed"), speed, true, + false, NULL, false, NULL, + qdict_haskey(qdict, "speed"), speed, true, BLOCKDEV_ON_ERROR_REPORT, false, NULL, false, false, false, false, &error); diff --git a/block/stream.c b/block/stream.c index XXXXXXX..XXXXXXX 100644 --- a/block/stream.c +++ b/block/stream.c @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver stream_job_driver = { void stream_start(const char *job_id, BlockDriverState *bs, BlockDriverState *base, const char *backing_file_str, + BlockDriverState *bottom, int creation_flags, int64_t speed, BlockdevOnError on_error, const char *filter_node_name, @@ -XXX,XX +XXX,XX @@ void stream_start(const char *job_id, BlockDriverState *bs, BlockDriverState *iter; bool bs_read_only; int basic_flags = BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED; - BlockDriverState *base_overlay = bdrv_find_overlay(bs, base); + BlockDriverState *base_overlay; BlockDriverState *above_base; - if (!base_overlay) { - error_setg(errp, "'%s' is not in the backing chain of '%s'", - base->node_name, bs->node_name); - return; - } + assert(!(base && bottom)); + assert(!(backing_file_str && bottom)); + + if (bottom) { + /* + * New simple interface. The code is written in terms of old interface + * with @base parameter (still, it doesn't freeze link to base, so in + * this mean old code is correct for new interface). So, for now, just + * emulate base_overlay and above_base. Still, when old interface + * finally removed, we should refactor code to use only "bottom", but + * not "*base*" things. + */ + assert(!bottom->drv->is_filter); + base_overlay = above_base = bottom; + } else { + base_overlay = bdrv_find_overlay(bs, base); + if (!base_overlay) { + error_setg(errp, "'%s' is not in the backing chain of '%s'", + base->node_name, bs->node_name); + return; + } - /* - * Find the node directly above @base. @base_overlay is a COW overlay, so - * it must have a bdrv_cow_child(), but it is the immediate overlay of - * @base, so between the two there can only be filters. - */ - above_base = base_overlay; - if (bdrv_cow_bs(above_base) != base) { - above_base = bdrv_cow_bs(above_base); - while (bdrv_filter_bs(above_base) != base) { - above_base = bdrv_filter_bs(above_base); + /* + * Find the node directly above @base. @base_overlay is a COW overlay, + * so it must have a bdrv_cow_child(), but it is the immediate overlay + * of @base, so between the two there can only be filters. + */ + above_base = base_overlay; + if (bdrv_cow_bs(above_base) != base) { + above_base = bdrv_cow_bs(above_base); + while (bdrv_filter_bs(above_base) != base) { + above_base = bdrv_filter_bs(above_base); + } } } diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, bool has_base, const char *base, bool has_base_node, const char *base_node, bool has_backing_file, const char *backing_file, + bool has_bottom, const char *bottom, bool has_speed, int64_t speed, bool has_on_error, BlockdevOnError on_error, bool has_filter_node_name, const char *filter_node_name, @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, bool has_auto_dismiss, bool auto_dismiss, Error **errp) { - BlockDriverState *bs, *iter; + BlockDriverState *bs, *iter, *iter_end; BlockDriverState *base_bs = NULL; + BlockDriverState *bottom_bs = NULL; AioContext *aio_context; Error *local_err = NULL; int job_flags = JOB_DEFAULT; + if (has_base && has_base_node) { + error_setg(errp, "'base' and 'base-node' cannot be specified " + "at the same time"); + return; + } + + if (has_base && has_bottom) { + error_setg(errp, "'base' and 'bottom' cannot be specified " + "at the same time"); + return; + } + + if (has_bottom && has_base_node) { + error_setg(errp, "'bottom' and 'base-node' cannot be specified " + "at the same time"); + return; + } + if (!has_on_error) { on_error = BLOCKDEV_ON_ERROR_REPORT; } @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, aio_context = bdrv_get_aio_context(bs); aio_context_acquire(aio_context); - if (has_base && has_base_node) { - error_setg(errp, "'base' and 'base-node' cannot be specified " - "at the same time"); - goto out; - } - if (has_base) { base_bs = bdrv_find_backing_image(bs, base); if (base_bs == NULL) { @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, bdrv_refresh_filename(base_bs); } - /* Check for op blockers in the whole chain between bs and base */ - for (iter = bs; iter && iter != base_bs; + if (has_bottom) { + bottom_bs = bdrv_lookup_bs(NULL, bottom, errp); + if (!bottom_bs) { + goto out; + } + if (!bottom_bs->drv) { + error_setg(errp, "Node '%s' is not open", bottom); + goto out; + } + if (bottom_bs->drv->is_filter) { + error_setg(errp, "Node '%s' is a filter, use a non-filter node " + "as 'bottom'", bottom); + goto out; + } + if (!bdrv_chain_contains(bs, bottom_bs)) { + error_setg(errp, "Node '%s' is not in a chain starting from '%s'", + bottom, device); + goto out; + } + assert(bdrv_get_aio_context(bottom_bs) == aio_context); + } + + /* + * Check for op blockers in the whole chain between bs and base (or bottom) + */ + iter_end = has_bottom ? bdrv_filter_or_cow_bs(bottom_bs) : base_bs; + for (iter = bs; iter && iter != iter_end; iter = bdrv_filter_or_cow_bs(iter)) { if (bdrv_op_is_blocked(iter, BLOCK_OP_TYPE_STREAM, errp)) { @@ -XXX,XX +XXX,XX @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device, } stream_start(has_job_id ? job_id : NULL, bs, base_bs, backing_file, - job_flags, has_speed ? speed : 0, on_error, + bottom_bs, job_flags, has_speed ? speed : 0, on_error, filter_node_name, &local_err); if (local_err) { error_propagate(errp, local_err); -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> test_stream_parallel run parallel stream jobs, intersecting so that top of one is base of another. It's OK now, but it would be a problem if insert the filter, as one job will want to use another job's filter as above_base node. Correct thing to do is move to new interface: "bottom" argument instead of base. This guarantees that jobs don't intersect by their actions. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20201216061703.70908-12-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/030 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/qemu-iotests/030 b/tests/qemu-iotests/030 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/030 +++ b/tests/qemu-iotests/030 @@ -XXX,XX +XXX,XX @@ class TestParallelOps(iotests.QMPTestCase): node_name = 'node%d' % i job_id = 'stream-%s' % node_name pending_jobs.append(job_id) - result = self.vm.qmp('block-stream', device=node_name, job_id=job_id, base=self.imgs[i-2], speed=1024) + result = self.vm.qmp('block-stream', device=node_name, + job_id=job_id, bottom=f'node{i-1}', + speed=1024) self.assert_qmp(result, 'return', {}) for job in pending_jobs: -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Add a direct link to target bs for convenience and to simplify following commit which will insert COR filter above target bs. This is a part of original commit written by Andrey. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20201216061703.70908-13-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/stream.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/block/stream.c b/block/stream.c index XXXXXXX..XXXXXXX 100644 --- a/block/stream.c +++ b/block/stream.c @@ -XXX,XX +XXX,XX @@ typedef struct StreamBlockJob { BlockJob common; BlockDriverState *base_overlay; /* COW overlay (stream from this) */ BlockDriverState *above_base; /* Node directly above the base */ + BlockDriverState *target_bs; BlockdevOnError on_error; char *backing_file_str; bool bs_read_only; @@ -XXX,XX +XXX,XX @@ static void stream_abort(Job *job) StreamBlockJob *s = container_of(job, StreamBlockJob, common.job); if (s->chain_frozen) { - BlockJob *bjob = &s->common; - bdrv_unfreeze_backing_chain(blk_bs(bjob->blk), s->above_base); + bdrv_unfreeze_backing_chain(s->target_bs, s->above_base); } } static int stream_prepare(Job *job) { StreamBlockJob *s = container_of(job, StreamBlockJob, common.job); - BlockJob *bjob = &s->common; - BlockDriverState *bs = blk_bs(bjob->blk); - BlockDriverState *unfiltered_bs = bdrv_skip_filters(bs); + BlockDriverState *unfiltered_bs = bdrv_skip_filters(s->target_bs); BlockDriverState *base = bdrv_filter_or_cow_bs(s->above_base); BlockDriverState *unfiltered_base = bdrv_skip_filters(base); Error *local_err = NULL; int ret = 0; - bdrv_unfreeze_backing_chain(bs, s->above_base); + bdrv_unfreeze_backing_chain(s->target_bs, s->above_base); s->chain_frozen = false; if (bdrv_cow_child(unfiltered_bs)) { @@ -XXX,XX +XXX,XX @@ static void stream_clean(Job *job) { StreamBlockJob *s = container_of(job, StreamBlockJob, common.job); BlockJob *bjob = &s->common; - BlockDriverState *bs = blk_bs(bjob->blk); /* Reopen the image back in read-only mode if necessary */ if (s->bs_read_only) { /* Give up write permissions before making it read-only */ blk_set_perm(bjob->blk, 0, BLK_PERM_ALL, &error_abort); - bdrv_reopen_set_read_only(bs, true, NULL); + bdrv_reopen_set_read_only(s->target_bs, true, NULL); } g_free(s->backing_file_str); @@ -XXX,XX +XXX,XX @@ static int coroutine_fn stream_run(Job *job, Error **errp) { StreamBlockJob *s = container_of(job, StreamBlockJob, common.job); BlockBackend *blk = s->common.blk; - BlockDriverState *bs = blk_bs(blk); - BlockDriverState *unfiltered_bs = bdrv_skip_filters(bs); + BlockDriverState *unfiltered_bs = bdrv_skip_filters(s->target_bs); bool enable_cor = !bdrv_cow_child(s->base_overlay); int64_t len; int64_t offset = 0; @@ -XXX,XX +XXX,XX @@ static int coroutine_fn stream_run(Job *job, Error **errp) return 0; } - len = bdrv_getlength(bs); + len = bdrv_getlength(s->target_bs); if (len < 0) { return len; } @@ -XXX,XX +XXX,XX @@ static int coroutine_fn stream_run(Job *job, Error **errp) * account. */ if (enable_cor) { - bdrv_enable_copy_on_read(bs); + bdrv_enable_copy_on_read(s->target_bs); } for ( ; offset < len; offset += n) { @@ -XXX,XX +XXX,XX @@ static int coroutine_fn stream_run(Job *job, Error **errp) } if (enable_cor) { - bdrv_disable_copy_on_read(bs); + bdrv_disable_copy_on_read(s->target_bs); } /* Do not remove the backing file if an error was there but ignored. */ @@ -XXX,XX +XXX,XX @@ void stream_start(const char *job_id, BlockDriverState *bs, s->base_overlay = base_overlay; s->above_base = above_base; s->backing_file_str = g_strdup(backing_file_str); + s->target_bs = bs; s->bs_read_only = bs_read_only; s->chain_frozen = true; -- 2.29.2 From: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> This patch completes the series with the COR-filter applied to block-stream operations. Adding the filter makes it possible in future implement discarding copied regions in backing files during the block-stream job, to reduce the disk overuse (we need control on permissions). Also, the filter now is smart enough to do copy-on-read with specified base, so we have benefit on guest reads even when doing block-stream of the part of the backing chain. Several iotests are slightly modified due to filter insertion. Signed-off-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Message-Id: <20201216061703.70908-14-vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/stream.c | 105 ++++++++++++++++++++++--------------- tests/qemu-iotests/030 | 8 +-- tests/qemu-iotests/141.out | 2 +- tests/qemu-iotests/245 | 20 ++++--- 4 files changed, 80 insertions(+), 55 deletions(-) diff --git a/block/stream.c b/block/stream.c index XXXXXXX..XXXXXXX 100644 --- a/block/stream.c +++ b/block/stream.c @@ -XXX,XX +XXX,XX @@ #include "block/blockjob_int.h" #include "qapi/error.h" #include "qapi/qmp/qerror.h" +#include "qapi/qmp/qdict.h" #include "qemu/ratelimit.h" #include "sysemu/block-backend.h" +#include "block/copy-on-read.h" enum { /* @@ -XXX,XX +XXX,XX @@ typedef struct StreamBlockJob { BlockJob common; BlockDriverState *base_overlay; /* COW overlay (stream from this) */ BlockDriverState *above_base; /* Node directly above the base */ + BlockDriverState *cor_filter_bs; BlockDriverState *target_bs; BlockdevOnError on_error; char *backing_file_str; bool bs_read_only; - bool chain_frozen; } StreamBlockJob; static int coroutine_fn stream_populate(BlockBackend *blk, @@ -XXX,XX +XXX,XX @@ static int coroutine_fn stream_populate(BlockBackend *blk, { assert(bytes < SIZE_MAX); - return blk_co_preadv(blk, offset, bytes, NULL, - BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH); -} - -static void stream_abort(Job *job) -{ - StreamBlockJob *s = container_of(job, StreamBlockJob, common.job); - - if (s->chain_frozen) { - bdrv_unfreeze_backing_chain(s->target_bs, s->above_base); - } + return blk_co_preadv(blk, offset, bytes, NULL, BDRV_REQ_PREFETCH); } static int stream_prepare(Job *job) @@ -XXX,XX +XXX,XX @@ static int stream_prepare(Job *job) Error *local_err = NULL; int ret = 0; - bdrv_unfreeze_backing_chain(s->target_bs, s->above_base); - s->chain_frozen = false; + /* We should drop filter at this point, as filter hold the backing chain */ + bdrv_cor_filter_drop(s->cor_filter_bs); + s->cor_filter_bs = NULL; if (bdrv_cow_child(unfiltered_bs)) { const char *base_id = NULL, *base_fmt = NULL; @@ -XXX,XX +XXX,XX @@ static void stream_clean(Job *job) StreamBlockJob *s = container_of(job, StreamBlockJob, common.job); BlockJob *bjob = &s->common; + if (s->cor_filter_bs) { + bdrv_cor_filter_drop(s->cor_filter_bs); + s->cor_filter_bs = NULL; + } + /* Reopen the image back in read-only mode if necessary */ if (s->bs_read_only) { /* Give up write permissions before making it read-only */ @@ -XXX,XX +XXX,XX @@ static int coroutine_fn stream_run(Job *job, Error **errp) StreamBlockJob *s = container_of(job, StreamBlockJob, common.job); BlockBackend *blk = s->common.blk; BlockDriverState *unfiltered_bs = bdrv_skip_filters(s->target_bs); - bool enable_cor = !bdrv_cow_child(s->base_overlay); int64_t len; int64_t offset = 0; uint64_t delay_ns = 0; @@ -XXX,XX +XXX,XX @@ static int coroutine_fn stream_run(Job *job, Error **errp) } job_progress_set_remaining(&s->common.job, len); - /* Turn on copy-on-read for the whole block device so that guest read - * requests help us make progress. Only do this when copying the entire - * backing chain since the copy-on-read operation does not take base into - * account. - */ - if (enable_cor) { - bdrv_enable_copy_on_read(s->target_bs); - } - for ( ; offset < len; offset += n) { bool copy; int ret; @@ -XXX,XX +XXX,XX @@ static int coroutine_fn stream_run(Job *job, Error **errp) } } - if (enable_cor) { - bdrv_disable_copy_on_read(s->target_bs); - } - /* Do not remove the backing file if an error was there but ignored. */ return error; } @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver stream_job_driver = { .free = block_job_free, .run = stream_run, .prepare = stream_prepare, - .abort = stream_abort, .clean = stream_clean, .user_resume = block_job_user_resume, }, @@ -XXX,XX +XXX,XX @@ void stream_start(const char *job_id, BlockDriverState *bs, bool bs_read_only; int basic_flags = BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED; BlockDriverState *base_overlay; + BlockDriverState *cor_filter_bs = NULL; BlockDriverState *above_base; + QDict *opts; assert(!(base && bottom)); assert(!(backing_file_str && bottom)); @@ -XXX,XX +XXX,XX @@ void stream_start(const char *job_id, BlockDriverState *bs, } } - if (bdrv_freeze_backing_chain(bs, above_base, errp) < 0) { - return; - } - /* Make sure that the image is opened in read-write mode */ bs_read_only = bdrv_is_read_only(bs); if (bs_read_only) { - if (bdrv_reopen_set_read_only(bs, false, errp) != 0) { - bs_read_only = false; - goto fail; + int ret; + /* Hold the chain during reopen */ + if (bdrv_freeze_backing_chain(bs, above_base, errp) < 0) { + return; + } + + ret = bdrv_reopen_set_read_only(bs, false, errp); + + /* failure, or cor-filter will hold the chain */ + bdrv_unfreeze_backing_chain(bs, above_base); + + if (ret < 0) { + return; } } - /* Prevent concurrent jobs trying to modify the graph structure here, we - * already have our own plans. Also don't allow resize as the image size is - * queried only at the job start and then cached. */ - s = block_job_create(job_id, &stream_job_driver, NULL, bs, - basic_flags | BLK_PERM_GRAPH_MOD, + opts = qdict_new(); + + qdict_put_str(opts, "driver", "copy-on-read"); + qdict_put_str(opts, "file", bdrv_get_node_name(bs)); + /* Pass the base_overlay node name as 'bottom' to COR driver */ + qdict_put_str(opts, "bottom", base_overlay->node_name); + if (filter_node_name) { + qdict_put_str(opts, "node-name", filter_node_name); + } + + cor_filter_bs = bdrv_insert_node(bs, opts, BDRV_O_RDWR, errp); + if (!cor_filter_bs) { + goto fail; + } + + if (!filter_node_name) { + cor_filter_bs->implicit = true; + } + + s = block_job_create(job_id, &stream_job_driver, NULL, cor_filter_bs, + BLK_PERM_CONSISTENT_READ, basic_flags | BLK_PERM_WRITE, speed, creation_flags, NULL, NULL, errp); if (!s) { goto fail; } + /* + * Prevent concurrent jobs trying to modify the graph structure here, we + * already have our own plans. Also don't allow resize as the image size is + * queried only at the job start and then cached. + */ + if (block_job_add_bdrv(&s->common, "active node", bs, 0, + basic_flags | BLK_PERM_WRITE, &error_abort)) { + goto fail; + } + /* Block all intermediate nodes between bs and base, because they will * disappear from the chain after this operation. The streaming job reads * every block only once, assuming that it doesn't change, so forbid writes @@ -XXX,XX +XXX,XX @@ void stream_start(const char *job_id, BlockDriverState *bs, s->base_overlay = base_overlay; s->above_base = above_base; s->backing_file_str = g_strdup(backing_file_str); + s->cor_filter_bs = cor_filter_bs; s->target_bs = bs; s->bs_read_only = bs_read_only; - s->chain_frozen = true; s->on_error = on_error; trace_stream_start(bs, base, s); @@ -XXX,XX +XXX,XX @@ void stream_start(const char *job_id, BlockDriverState *bs, return; fail: + if (cor_filter_bs) { + bdrv_cor_filter_drop(cor_filter_bs); + } if (bs_read_only) { bdrv_reopen_set_read_only(bs, true, NULL); } - bdrv_unfreeze_backing_chain(bs, above_base); } diff --git a/tests/qemu-iotests/030 b/tests/qemu-iotests/030 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/030 +++ b/tests/qemu-iotests/030 @@ -XXX,XX +XXX,XX @@ class TestParallelOps(iotests.QMPTestCase): self.assert_no_active_block_jobs() # Set a speed limit to make sure that this job blocks the rest - result = self.vm.qmp('block-stream', device='node4', job_id='stream-node4', base=self.imgs[1], speed=1024*1024) + result = self.vm.qmp('block-stream', device='node4', + job_id='stream-node4', base=self.imgs[1], + filter_node_name='stream-filter', speed=1024*1024) self.assert_qmp(result, 'return', {}) result = self.vm.qmp('block-stream', device='node5', job_id='stream-node5', base=self.imgs[2]) self.assert_qmp(result, 'error/desc', - "Node 'node4' is busy: block device is in use by block job: stream") + "Node 'stream-filter' is busy: block device is in use by block job: stream") result = self.vm.qmp('block-stream', device='node3', job_id='stream-node3', base=self.imgs[2]) self.assert_qmp(result, 'error/desc', @@ -XXX,XX +XXX,XX @@ class TestParallelOps(iotests.QMPTestCase): # block-commit should also fail if it touches nodes used by the stream job result = self.vm.qmp('block-commit', device='drive0', base=self.imgs[4], job_id='commit-node4') self.assert_qmp(result, 'error/desc', - "Node 'node4' is busy: block device is in use by block job: stream") + "Node 'stream-filter' is busy: block device is in use by block job: stream") result = self.vm.qmp('block-commit', device='drive0', base=self.imgs[1], top=self.imgs[3], job_id='commit-node1') self.assert_qmp(result, 'error/desc', diff --git a/tests/qemu-iotests/141.out b/tests/qemu-iotests/141.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/141.out +++ b/tests/qemu-iotests/141.out @@ -XXX,XX +XXX,XX @@ wrote 1048576/1048576 bytes at offset 0 {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job0"}} {'execute': 'blockdev-del', 'arguments': {'node-name': 'drv0'}} -{"error": {"class": "GenericError", "desc": "Node drv0 is in use"}} +{"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: stream"}} {'execute': 'block-job-cancel', 'arguments': {'device': 'job0'}} {"return": {}} diff --git a/tests/qemu-iotests/245 b/tests/qemu-iotests/245 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/245 +++ b/tests/qemu-iotests/245 @@ -XXX,XX +XXX,XX @@ class TestBlockdevReopen(iotests.QMPTestCase): # hd1 <- hd0 result = self.vm.qmp('block-stream', conv_keys = True, job_id = 'stream0', - device = 'hd1', auto_finalize = False) + device = 'hd1', filter_node_name='cor', + auto_finalize = False) self.assert_qmp(result, 'return', {}) - # We can't reopen with the original options because that would - # make hd1 read-only and block-stream requires it to be read-write - # (Which error message appears depends on whether the stream job is - # already done with copying at this point.) + # We can't reopen with the original options because there is a filter + # inserted by stream job above hd1. self.reopen(opts, {}, - ["Can't set node 'hd1' to r/o with copy-on-read enabled", - "Cannot make block node read-only, there is a writer on it"]) + "Cannot change the option 'backing.backing.file.node-name'") + + # We can't reopen hd1 to read-only, as block-stream requires it to be + # read-write + self.reopen(opts['backing'], {'read-only': True}, + "Cannot make block node read-only, there is a writer on it") # We can't remove hd2 while the stream job is ongoing opts['backing']['backing'] = None - self.reopen(opts, {'backing.read-only': False}, "Cannot change 'backing' link from 'hd1' to 'hd2'") + self.reopen(opts['backing'], {'read-only': False}, + "Cannot change 'backing' link from 'hd1' to 'hd2'") # We can detach hd1 from hd0 because it doesn't affect the stream job opts['backing'] = None -- 2.29.2 There are a couple of environment variables that we fetch with os.environ.get() without supplying a default. Clearly they are required and expected to be set by the ./check script (as evidenced by execute_setup_common(), which checks for test_dir and qemu_default_machine to be set, and aborts if they are not). Using .get() this way has the disadvantage of returning an Optional[str] type, which mypy will complain about when tests just assume these values to be str. Use [] instead, which raises a KeyError for environment variables that are not set. When this exception is raised, catch it and move the abort code from execute_setup_common() there. Drop the 'assert iotests.sock_dir is not None' from iotest 300, because that sort of thing is precisely what this patch wants to prevent. Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Willian Rampazzo <willianr@redhat.com> Message-Id: <20210118105720.14824-2-mreitz@redhat.com> --- tests/qemu-iotests/300 | 1 - tests/qemu-iotests/iotests.py | 26 +++++++++++++------------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/qemu-iotests/300 b/tests/qemu-iotests/300 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/300 +++ b/tests/qemu-iotests/300 @@ -XXX,XX +XXX,XX @@ import qemu BlockBitmapMapping = List[Dict[str, Union[str, List[Dict[str, str]]]]] -assert iotests.sock_dir is not None mig_sock = os.path.join(iotests.sock_dir, 'mig_sock') diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -XXX,XX +XXX,XX @@ qemu_opts = os.environ.get('QEMU_OPTIONS', '').strip().split(' ') imgfmt = os.environ.get('IMGFMT', 'raw') imgproto = os.environ.get('IMGPROTO', 'file') -test_dir = os.environ.get('TEST_DIR') -sock_dir = os.environ.get('SOCK_DIR') output_dir = os.environ.get('OUTPUT_DIR', '.') -cachemode = os.environ.get('CACHEMODE') -aiomode = os.environ.get('AIOMODE') -qemu_default_machine = os.environ.get('QEMU_DEFAULT_MACHINE') + +try: + test_dir = os.environ['TEST_DIR'] + sock_dir = os.environ['SOCK_DIR'] + cachemode = os.environ['CACHEMODE'] + aiomode = os.environ['AIOMODE'] + qemu_default_machine = os.environ['QEMU_DEFAULT_MACHINE'] +except KeyError: + # We are using these variables as proxies to indicate that we're + # not being run via "check". There may be other things set up by + # "check" that individual test cases rely on. + sys.stderr.write('Please run this test via the "check" script\n') + sys.exit(os.EX_USAGE) socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper') @@ -XXX,XX +XXX,XX @@ def execute_setup_common(supported_fmts: Sequence[str] = (), """ # Note: Python 3.6 and pylint do not like 'Collection' so use 'Sequence'. - # We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to - # indicate that we're not being run via "check". There may be - # other things set up by "check" that individual test cases rely - # on. - if test_dir is None or qemu_default_machine is None: - sys.stderr.write('Please run this test via the "check" script\n') - sys.exit(os.EX_USAGE) - debug = '-d' in sys.argv if debug: sys.argv.remove('-d') -- 2.29.2 Instead of checking iotests.py only, check all Python files in the qemu-iotests/ directory. Of course, most of them do not pass, so there is an extensive skip list for now. (The only files that do pass are 209, 254, 283, and iotests.py.) (Alternatively, we could have the opposite, i.e. an explicit list of files that we do want to check, but I think it is better to check files by default.) Unless started in debug mode (./check -d), the output has no information on which files are tested, so we will not have a problem e.g. with backports, where some files may be missing when compared to upstream. Besides the technical rewrite, some more things are changed: - For the pylint invocation, PYTHONPATH is adjusted. This mirrors setting MYPYPATH for mypy. - Also, MYPYPATH is now derived from PYTHONPATH, so that we include paths set by the environment. Maybe at some point we want to let the check script add '../../python/' to PYTHONPATH so that iotests.py does not need to do that. - Passing --notes=FIXME,XXX to pylint suppresses warnings for TODO comments. TODO is fine, we do not need 297 to complain about such comments. - The "Success" line from mypy's output is suppressed, because (A) it does not add useful information, and (B) it would leak information about the files having been tested to the reference output, which we decidedly do not want. Suggested-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210118105720.14824-3-mreitz@redhat.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> --- tests/qemu-iotests/297 | 112 +++++++++++++++++++++++++++++-------- tests/qemu-iotests/297.out | 5 +- 2 files changed, 92 insertions(+), 25 deletions(-) diff --git a/tests/qemu-iotests/297 b/tests/qemu-iotests/297 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/297 +++ b/tests/qemu-iotests/297 @@ -XXX,XX +XXX,XX @@ -#!/usr/bin/env bash +#!/usr/bin/env python3 # group: meta # # Copyright (C) 2020 Red Hat, Inc. @@ -XXX,XX +XXX,XX @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -seq=$(basename $0) -echo "QA output created by $seq" +import os +import re +import shutil +import subprocess +import sys -status=1 # failure is the default! +import iotests -# get standard environment -. ./common.rc -if ! type -p "pylint-3" > /dev/null; then - _notrun "pylint-3 not found" -fi -if ! type -p "mypy" > /dev/null; then - _notrun "mypy not found" -fi +# TODO: Empty this list! +SKIP_FILES = ( + '030', '040', '041', '044', '045', '055', '056', '057', '065', '093', + '096', '118', '124', '129', '132', '136', '139', '147', '148', '149', + '151', '152', '155', '163', '165', '169', '194', '196', '199', '202', + '203', '205', '206', '207', '208', '210', '211', '212', '213', '216', + '218', '219', '222', '224', '228', '234', '235', '236', '237', '238', + '240', '242', '245', '246', '248', '255', '256', '257', '258', '260', + '262', '264', '266', '274', '277', '280', '281', '295', '296', '298', + '299', '300', '302', '303', '304', '307', + 'nbd-fault-injector.py', 'qcow2.py', 'qcow2_format.py', 'qed.py' +) -pylint-3 --score=n iotests.py -MYPYPATH=../../python/ mypy --warn-unused-configs --disallow-subclassing-any \ - --disallow-any-generics --disallow-incomplete-defs \ - --disallow-untyped-decorators --no-implicit-optional \ - --warn-redundant-casts --warn-unused-ignores \ - --no-implicit-reexport iotests.py +def is_python_file(filename): + if not os.path.isfile(filename): + return False -# success, all done -echo "*** done" -rm -f $seq.full -status=0 + if filename.endswith('.py'): + return True + + with open(filename) as f: + try: + first_line = f.readline() + return re.match('^#!.*python', first_line) is not None + except UnicodeDecodeError: # Ignore binary files + return False + + +def run_linters(): + files = [filename for filename in (set(os.listdir('.')) - set(SKIP_FILES)) + if is_python_file(filename)] + + iotests.logger.debug('Files to be checked:') + iotests.logger.debug(', '.join(sorted(files))) + + print('=== pylint ===') + sys.stdout.flush() + + # Todo notes are fine, but fixme's or xxx's should probably just be + # fixed (in tests, at least) + env = os.environ.copy() + qemu_module_path = os.path.join(os.path.dirname(__file__), + '..', '..', 'python') + try: + env['PYTHONPATH'] += os.pathsep + qemu_module_path + except KeyError: + env['PYTHONPATH'] = qemu_module_path + subprocess.run(('pylint-3', '--score=n', '--notes=FIXME,XXX', *files), + env=env, check=False) + + print('=== mypy ===') + sys.stdout.flush() + + # We have to call mypy separately for each file. Otherwise, it + # will interpret all given files as belonging together (i.e., they + # may not both define the same classes, etc.; most notably, they + # must not both define the __main__ module). + env['MYPYPATH'] = env['PYTHONPATH'] + for filename in files: + p = subprocess.run(('mypy', + '--warn-unused-configs', + '--disallow-subclassing-any', + '--disallow-any-generics', + '--disallow-incomplete-defs', + '--disallow-untyped-decorators', + '--no-implicit-optional', + '--warn-redundant-casts', + '--warn-unused-ignores', + '--no-implicit-reexport', + filename), + env=env, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + + if p.returncode != 0: + print(p.stdout) + + +for linter in ('pylint-3', 'mypy'): + if shutil.which(linter) is None: + iotests.notrun(f'{linter} not found') + +iotests.script_main(run_linters) diff --git a/tests/qemu-iotests/297.out b/tests/qemu-iotests/297.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/297.out +++ b/tests/qemu-iotests/297.out @@ -XXX,XX +XXX,XX @@ -QA output created by 297 -Success: no issues found in 1 source file -*** done +=== pylint === +=== mypy === -- 2.29.2 Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Willian Rampazzo <willianr@redhat.com> Message-Id: <20210118105720.14824-4-mreitz@redhat.com> --- tests/qemu-iotests/124 | 8 +------- tests/qemu-iotests/iotests.py | 11 +++++++---- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/tests/qemu-iotests/124 b/tests/qemu-iotests/124 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/124 +++ b/tests/qemu-iotests/124 @@ -XXX,XX +XXX,XX @@ import os import iotests +from iotests import try_remove def io_write_patterns(img, patterns): @@ -XXX,XX +XXX,XX @@ def io_write_patterns(img, patterns): iotests.qemu_io('-c', 'write -P%s %s %s' % pattern, img) -def try_remove(img): - try: - os.remove(img) - except OSError: - pass - - def transaction_action(action, **kwargs): return { 'type': action, diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -XXX,XX +XXX,XX @@ class FilePath: return False +def try_remove(img): + try: + os.remove(img) + except OSError: + pass + def file_path_remover(): for path in reversed(file_path_remover.paths): - try: - os.remove(path) - except OSError: - pass + try_remove(path) def file_path(*names, base_dir=test_dir): -- 2.29.2 Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Eric Blake <eblake@redhat.com> Reviewed-by: Willian Rampazzo <willianr@redhat.com> Message-Id: <20210118105720.14824-5-mreitz@redhat.com> --- tests/qemu-iotests/129 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/qemu-iotests/129 b/tests/qemu-iotests/129 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/129 +++ b/tests/qemu-iotests/129 @@ -XXX,XX +XXX,XX @@ class TestStopWithBlockJob(iotests.QMPTestCase): result = self.vm.qmp("block_set_io_throttle", conv_keys=False, **params) self.vm.shutdown() + for img in (self.test_img, self.target_img, self.base_img): + iotests.try_remove(img) def do_test_stop(self, cmd, **args): """Test 'stop' while block job is running on a throttled drive. -- 2.29.2 @busy is false when the job is paused, which happens all the time because that is how jobs yield (e.g. for mirror at least since commit 565ac01f8d3). Back when 129 was added (2015), perhaps there was no better way of checking whether the job was still actually running. Now we have the @status field (as of 58b295ba52c, i.e. 2018), which can give us exactly that information. Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Willian Rampazzo <willianr@redhat.com> Message-Id: <20210118105720.14824-6-mreitz@redhat.com> --- tests/qemu-iotests/129 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/qemu-iotests/129 b/tests/qemu-iotests/129 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/129 +++ b/tests/qemu-iotests/129 @@ -XXX,XX +XXX,XX @@ class TestStopWithBlockJob(iotests.QMPTestCase): result = self.vm.qmp("stop") self.assert_qmp(result, 'return', {}) result = self.vm.qmp("query-block-jobs") - self.assert_qmp(result, 'return[0]/busy', True) + self.assert_qmp(result, 'return[0]/status', 'running') self.assert_qmp(result, 'return[0]/ready', False) def test_drive_mirror(self): -- 2.29.2 Throttling on the BB has not affected block jobs in a while, so it is possible that one of the jobs in 129 finishes before the VM is stopped. We can fix that by running the job from a throttle node. Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Willian Rampazzo <willianr@redhat.com> Message-Id: <20210118105720.14824-7-mreitz@redhat.com> --- tests/qemu-iotests/129 | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/tests/qemu-iotests/129 b/tests/qemu-iotests/129 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/129 +++ b/tests/qemu-iotests/129 @@ -XXX,XX +XXX,XX @@ class TestStopWithBlockJob(iotests.QMPTestCase): iotests.qemu_img('create', '-f', iotests.imgfmt, self.test_img, "-b", self.base_img, '-F', iotests.imgfmt) iotests.qemu_io('-f', iotests.imgfmt, '-c', 'write -P0x5d 1M 128M', self.test_img) - self.vm = iotests.VM().add_drive(self.test_img) + self.vm = iotests.VM() + self.vm.add_object('throttle-group,id=tg0,x-bps-total=1024') + + source_drive = 'driver=throttle,' \ + 'throttle-group=tg0,' \ + f'file.driver={iotests.imgfmt},' \ + f'file.file.filename={self.test_img}' + + self.vm.add_drive(None, source_drive) self.vm.launch() def tearDown(self): - params = {"device": "drive0", - "bps": 0, - "bps_rd": 0, - "bps_wr": 0, - "iops": 0, - "iops_rd": 0, - "iops_wr": 0, - } - result = self.vm.qmp("block_set_io_throttle", conv_keys=False, - **params) self.vm.shutdown() for img in (self.test_img, self.target_img, self.base_img): iotests.try_remove(img) @@ -XXX,XX +XXX,XX @@ class TestStopWithBlockJob(iotests.QMPTestCase): def do_test_stop(self, cmd, **args): """Test 'stop' while block job is running on a throttled drive. The 'stop' command shouldn't drain the job""" - params = {"device": "drive0", - "bps": 1024, - "bps_rd": 0, - "bps_wr": 0, - "iops": 0, - "iops_rd": 0, - "iops_wr": 0, - } - result = self.vm.qmp("block_set_io_throttle", conv_keys=False, - **params) - self.assert_qmp(result, 'return', {}) result = self.vm.qmp(cmd, **args) self.assert_qmp(result, 'return', {}) + result = self.vm.qmp("stop") self.assert_qmp(result, 'return', {}) result = self.vm.qmp("query-block-jobs") + self.assert_qmp(result, 'return[0]/status', 'running') self.assert_qmp(result, 'return[0]/ready', False) def test_drive_mirror(self): self.do_test_stop("drive-mirror", device="drive0", - target=self.target_img, + target=self.target_img, format=iotests.imgfmt, sync="full") def test_drive_backup(self): self.do_test_stop("drive-backup", device="drive0", - target=self.target_img, + target=self.target_img, format=iotests.imgfmt, sync="full") def test_block_commit(self): -- 2.29.2 Before this patch, test_block_commit() performs an active commit, which under the hood is a mirror job. If we want to test various different block jobs, we should perhaps run an actual commit job instead. Doing so requires adding an overlay above the source node before the commit is done (and then specifying the source node as the top node for the commit job). Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Willian Rampazzo <willianr@redhat.com> Message-Id: <20210118105720.14824-8-mreitz@redhat.com> --- tests/qemu-iotests/129 | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/qemu-iotests/129 b/tests/qemu-iotests/129 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/129 +++ b/tests/qemu-iotests/129 @@ -XXX,XX +XXX,XX @@ class TestStopWithBlockJob(iotests.QMPTestCase): test_img = os.path.join(iotests.test_dir, 'test.img') target_img = os.path.join(iotests.test_dir, 'target.img') base_img = os.path.join(iotests.test_dir, 'base.img') + overlay_img = os.path.join(iotests.test_dir, 'overlay.img') def setUp(self): iotests.qemu_img('create', '-f', iotests.imgfmt, self.base_img, "1G") @@ -XXX,XX +XXX,XX @@ class TestStopWithBlockJob(iotests.QMPTestCase): self.vm.add_object('throttle-group,id=tg0,x-bps-total=1024') source_drive = 'driver=throttle,' \ + 'node-name=source,' \ 'throttle-group=tg0,' \ f'file.driver={iotests.imgfmt},' \ f'file.file.filename={self.test_img}' @@ -XXX,XX +XXX,XX @@ class TestStopWithBlockJob(iotests.QMPTestCase): def tearDown(self): self.vm.shutdown() - for img in (self.test_img, self.target_img, self.base_img): + for img in (self.test_img, self.target_img, self.base_img, + self.overlay_img): iotests.try_remove(img) def do_test_stop(self, cmd, **args): @@ -XXX,XX +XXX,XX @@ class TestStopWithBlockJob(iotests.QMPTestCase): sync="full") def test_block_commit(self): - self.do_test_stop("block-commit", device="drive0") + # Add overlay above the source node so that we actually use a + # commit job instead of a mirror job + + iotests.qemu_img('create', '-f', iotests.imgfmt, self.overlay_img, + '1G') + + result = self.vm.qmp('blockdev-add', **{ + 'node-name': 'overlay', + 'driver': iotests.imgfmt, + 'file': { + 'driver': 'file', + 'filename': self.overlay_img + } + }) + self.assert_qmp(result, 'return', {}) + + result = self.vm.qmp('blockdev-snapshot', + node='source', overlay='overlay') + self.assert_qmp(result, 'return', {}) + + self.do_test_stop('block-commit', device='drive0', top_node='source') if __name__ == '__main__': iotests.main(supported_fmts=["qcow2"], -- 2.29.2 Issuing 'stop' on the VM drains all nodes. If the mirror job has many large requests in flight, this may lead to significant I/O that looks a bit like 'stop' would make the job try to complete (which is what 129 should verify not to happen). We can limit the I/O in flight by limiting the buffer size, so mirror will make very little progress during the 'stop' drain. (We do not need to do anything about commit, which has a buffer size of 512 kB by default; or backup, which goes cluster by cluster. Once we have asynchronous requests for backup, that will change, but then we can fine-tune the backup job to only perform a single request on a very small chunk, too.) Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Eric Blake <eblake@redhat.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Willian Rampazzo <willianr@redhat.com> Message-Id: <20210118105720.14824-9-mreitz@redhat.com> --- tests/qemu-iotests/129 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/qemu-iotests/129 b/tests/qemu-iotests/129 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/129 +++ b/tests/qemu-iotests/129 @@ -XXX,XX +XXX,XX @@ class TestStopWithBlockJob(iotests.QMPTestCase): def test_drive_mirror(self): self.do_test_stop("drive-mirror", device="drive0", target=self.target_img, format=iotests.imgfmt, - sync="full") + sync="full", buf_size=65536) def test_drive_backup(self): self.do_test_stop("drive-backup", device="drive0", -- 2.29.2 And consequentially drop it from 297's skip list. Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Willian Rampazzo <willianr@redhat.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Message-Id: <20210118105720.14824-10-mreitz@redhat.com> --- tests/qemu-iotests/129 | 4 ++-- tests/qemu-iotests/297 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/qemu-iotests/129 b/tests/qemu-iotests/129 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/129 +++ b/tests/qemu-iotests/129 @@ -XXX,XX +XXX,XX @@ import os import iotests -import time class TestStopWithBlockJob(iotests.QMPTestCase): test_img = os.path.join(iotests.test_dir, 'test.img') @@ -XXX,XX +XXX,XX @@ class TestStopWithBlockJob(iotests.QMPTestCase): iotests.qemu_img('create', '-f', iotests.imgfmt, self.base_img, "1G") iotests.qemu_img('create', '-f', iotests.imgfmt, self.test_img, "-b", self.base_img, '-F', iotests.imgfmt) - iotests.qemu_io('-f', iotests.imgfmt, '-c', 'write -P0x5d 1M 128M', self.test_img) + iotests.qemu_io('-f', iotests.imgfmt, '-c', 'write -P0x5d 1M 128M', + self.test_img) self.vm = iotests.VM() self.vm.add_object('throttle-group,id=tg0,x-bps-total=1024') diff --git a/tests/qemu-iotests/297 b/tests/qemu-iotests/297 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/297 +++ b/tests/qemu-iotests/297 @@ -XXX,XX +XXX,XX @@ import iotests # TODO: Empty this list! SKIP_FILES = ( '030', '040', '041', '044', '045', '055', '056', '057', '065', '093', - '096', '118', '124', '129', '132', '136', '139', '147', '148', '149', + '096', '118', '124', '132', '136', '139', '147', '148', '149', '151', '152', '155', '163', '165', '169', '194', '196', '199', '202', '203', '205', '206', '207', '208', '210', '211', '212', '213', '216', '218', '219', '222', '224', '228', '234', '235', '236', '237', '238', -- 2.29.2 And consequentially drop it from 297's skip list. Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Willian Rampazzo <willianr@redhat.com> Message-Id: <20210118105720.14824-11-mreitz@redhat.com> --- tests/qemu-iotests/297 | 2 +- tests/qemu-iotests/300 | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/qemu-iotests/297 b/tests/qemu-iotests/297 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/297 +++ b/tests/qemu-iotests/297 @@ -XXX,XX +XXX,XX @@ SKIP_FILES = ( '218', '219', '222', '224', '228', '234', '235', '236', '237', '238', '240', '242', '245', '246', '248', '255', '256', '257', '258', '260', '262', '264', '266', '274', '277', '280', '281', '295', '296', '298', - '299', '300', '302', '303', '304', '307', + '299', '302', '303', '304', '307', 'nbd-fault-injector.py', 'qcow2.py', 'qcow2_format.py', 'qed.py' ) diff --git a/tests/qemu-iotests/300 b/tests/qemu-iotests/300 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/300 +++ b/tests/qemu-iotests/300 @@ -XXX,XX +XXX,XX @@ import os import random import re from typing import Dict, List, Optional, Union + import iotests + +# Import qemu after iotests.py has amended sys.path +# pylint: disable=wrong-import-order import qemu BlockBitmapMapping = List[Dict[str, Union[str, List[Dict[str, str]]]]] @@ -XXX,XX +XXX,XX @@ class TestDirtyBitmapMigration(iotests.QMPTestCase): If @msg is None, check that there has not been any error. """ self.vm_b.shutdown() + + log = self.vm_b.get_log() + assert log is not None # Loaded after shutdown + if msg is None: - self.assertNotIn('qemu-system-', self.vm_b.get_log()) + self.assertNotIn('qemu-system-', log) else: - self.assertIn(msg, self.vm_b.get_log()) + self.assertIn(msg, log) @staticmethod def mapping(node_name: str, node_alias: str, @@ -XXX,XX +XXX,XX @@ class TestBlockBitmapMappingErrors(TestDirtyBitmapMigration): # Check for the error in the source's log self.vm_a.shutdown() + + log = self.vm_a.get_log() + assert log is not None # Loaded after shutdown + self.assertIn(f"Cannot migrate bitmap '{name}' on node " f"'{self.src_node_name}': Name is longer than 255 bytes", - self.vm_a.get_log()) + log) # Expect abnormal shutdown of the destination VM because of # the failed migration -- 2.29.2 Disposition (action) for any given signal is global for the process. When two threads run coroutine-sigaltstack's qemu_coroutine_new() concurrently, they may interfere with each other: One of them may revert the SIGUSR2 handler to SIG_DFL, between the other thread (a) setting up coroutine_trampoline() as the handler and (b) raising SIGUSR2. That SIGUSR2 will then terminate the QEMU process abnormally. We have to ensure that only one thread at a time can modify the process-global SIGUSR2 handler. To do so, wrap the whole section where that is done in a mutex. Alternatively, we could for example have the SIGUSR2 handler always be coroutine_trampoline(), so there would be no need to invoke sigaction() in qemu_coroutine_new(). Laszlo has posted a patch to do so here: https://lists.nongnu.org/archive/html/qemu-devel/2021-01/msg05962.html However, given that coroutine-sigaltstack is more of a fallback implementation for platforms that do not support ucontext, that change may be a bit too invasive to be comfortable with it. The mutex proposed here may negatively impact performance, but the change is much simpler. Signed-off-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210125120305.19520-1-mreitz@redhat.com> Reviewed-by: Laszlo Ersek <lersek@redhat.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> --- util/coroutine-sigaltstack.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/util/coroutine-sigaltstack.c b/util/coroutine-sigaltstack.c index XXXXXXX..XXXXXXX 100644 --- a/util/coroutine-sigaltstack.c +++ b/util/coroutine-sigaltstack.c @@ -XXX,XX +XXX,XX @@ Coroutine *qemu_coroutine_new(void) sigset_t sigs; sigset_t osigs; sigjmp_buf old_env; + static pthread_mutex_t sigusr2_mutex = PTHREAD_MUTEX_INITIALIZER; /* The way to manipulate stack is with the sigaltstack function. We * prepare a stack, with it delivering a signal to ourselves and then @@ -XXX,XX +XXX,XX @@ Coroutine *qemu_coroutine_new(void) sa.sa_handler = coroutine_trampoline; sigfillset(&sa.sa_mask); sa.sa_flags = SA_ONSTACK; + + /* + * sigaction() is a process-global operation. We must not run + * this code in multiple threads at once. + */ + pthread_mutex_lock(&sigusr2_mutex); if (sigaction(SIGUSR2, &sa, &osa) != 0) { abort(); } @@ -XXX,XX +XXX,XX @@ Coroutine *qemu_coroutine_new(void) * Restore the old SIGUSR2 signal handler and mask */ sigaction(SIGUSR2, &osa, NULL); + pthread_mutex_unlock(&sigusr2_mutex); + pthread_sigmask(SIG_SETMASK, &osigs, NULL); /* -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Experiments show, that copy_range is not always making things faster. So, to make experimentation simpler, let's add a parameter. Some more perf parameters will be added soon, so here is a new struct. For now, add new backup qmp parameter with x- prefix for the following reasons: - We are going to add more performance parameters, some will be related to the whole block-copy process, some only to background copying in backup (ignored for copy-before-write operations). - On the other hand, we are going to use block-copy interface in other block jobs, which will need performance options as well.. And it should be the same structure or at least somehow related. So, there are too much unclean things about how the interface and now we need the new options mostly for testing. Let's keep them experimental for a while. In do_backup_common() new x-perf parameter handled in a way to make further options addition simpler. We add use-copy-range with default=true, and we'll change the default in further patch, after moving backup to use block-copy. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-2-vsementsov@virtuozzo.com> [mreitz: s/5\.2/6.0/] Signed-off-by: Max Reitz <mreitz@redhat.com> --- qapi/block-core.json | 17 ++++++++++++++++- block/backup-top.h | 1 + include/block/block-copy.h | 2 +- include/block/block_int.h | 3 +++ block/backup-top.c | 4 +++- block/backup.c | 6 +++++- block/block-copy.c | 4 ++-- block/replication.c | 2 ++ blockdev.c | 8 ++++++++ 9 files changed, 41 insertions(+), 6 deletions(-) diff --git a/qapi/block-core.json b/qapi/block-core.json index XXXXXXX..XXXXXXX 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -XXX,XX +XXX,XX @@ { 'struct': 'BlockdevSnapshot', 'data': { 'node': 'str', 'overlay': 'str' } } +## +# @BackupPerf: +# +# Optional parameters for backup. These parameters don't affect +# functionality, but may significantly affect performance. +# +# @use-copy-range: Use copy offloading. Default true. +# +# Since: 6.0 +## +{ 'struct': 'BackupPerf', + 'data': { '*use-copy-range': 'bool' }} + ## # @BackupCommon: # @@ -XXX,XX +XXX,XX @@ # above node specified by @drive. If this option is not given, # a node name is autogenerated. (Since: 4.2) # +# @x-perf: Performance options. (Since 6.0) +# # Note: @on-source-error and @on-target-error only affect background # I/O. If an error occurs during a guest write request, the device's # rerror/werror actions will be used. @@ -XXX,XX +XXX,XX @@ '*on-source-error': 'BlockdevOnError', '*on-target-error': 'BlockdevOnError', '*auto-finalize': 'bool', '*auto-dismiss': 'bool', - '*filter-node-name': 'str' } } + '*filter-node-name': 'str', '*x-perf': 'BackupPerf' } } ## # @DriveBackup: diff --git a/block/backup-top.h b/block/backup-top.h index XXXXXXX..XXXXXXX 100644 --- a/block/backup-top.h +++ b/block/backup-top.h @@ -XXX,XX +XXX,XX @@ BlockDriverState *bdrv_backup_top_append(BlockDriverState *source, BlockDriverState *target, const char *filter_node_name, uint64_t cluster_size, + BackupPerf *perf, BdrvRequestFlags write_flags, BlockCopyState **bcs, Error **errp); diff --git a/include/block/block-copy.h b/include/block/block-copy.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block-copy.h +++ b/include/block/block-copy.h @@ -XXX,XX +XXX,XX @@ typedef void (*ProgressBytesCallbackFunc)(int64_t bytes, void *opaque); typedef struct BlockCopyState BlockCopyState; BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target, - int64_t cluster_size, + int64_t cluster_size, bool use_copy_range, BdrvRequestFlags write_flags, Error **errp); diff --git a/include/block/block_int.h b/include/block/block_int.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -XXX,XX +XXX,XX @@ void mirror_start(const char *job_id, BlockDriverState *bs, * @sync_mode: What parts of the disk image should be copied to the destination. * @sync_bitmap: The dirty bitmap if sync_mode is 'bitmap' or 'incremental' * @bitmap_mode: The bitmap synchronization policy to use. + * @perf: Performance options. All actual fields assumed to be present, + * all ".has_*" fields are ignored. * @on_source_error: The action to take upon error reading from the source. * @on_target_error: The action to take upon error writing to the target. * @creation_flags: Flags that control the behavior of the Job lifetime. @@ -XXX,XX +XXX,XX @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs, BitmapSyncMode bitmap_mode, bool compress, const char *filter_node_name, + BackupPerf *perf, BlockdevOnError on_source_error, BlockdevOnError on_target_error, int creation_flags, diff --git a/block/backup-top.c b/block/backup-top.c index XXXXXXX..XXXXXXX 100644 --- a/block/backup-top.c +++ b/block/backup-top.c @@ -XXX,XX +XXX,XX @@ BlockDriverState *bdrv_backup_top_append(BlockDriverState *source, BlockDriverState *target, const char *filter_node_name, uint64_t cluster_size, + BackupPerf *perf, BdrvRequestFlags write_flags, BlockCopyState **bcs, Error **errp) @@ -XXX,XX +XXX,XX @@ BlockDriverState *bdrv_backup_top_append(BlockDriverState *source, state->cluster_size = cluster_size; state->bcs = block_copy_state_new(top->backing, state->target, - cluster_size, write_flags, &local_err); + cluster_size, perf->use_copy_range, + write_flags, &local_err); if (local_err) { error_prepend(&local_err, "Cannot create block-copy-state: "); goto fail; diff --git a/block/backup.c b/block/backup.c index XXXXXXX..XXXXXXX 100644 --- a/block/backup.c +++ b/block/backup.c @@ -XXX,XX +XXX,XX @@ typedef struct BackupBlockJob { uint64_t len; uint64_t bytes_read; int64_t cluster_size; + BackupPerf perf; BlockCopyState *bcs; } BackupBlockJob; @@ -XXX,XX +XXX,XX @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs, BitmapSyncMode bitmap_mode, bool compress, const char *filter_node_name, + BackupPerf *perf, BlockdevOnError on_source_error, BlockdevOnError on_target_error, int creation_flags, @@ -XXX,XX +XXX,XX @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs, (compress ? BDRV_REQ_WRITE_COMPRESSED : 0), backup_top = bdrv_backup_top_append(bs, target, filter_node_name, - cluster_size, write_flags, &bcs, errp); + cluster_size, perf, + write_flags, &bcs, errp); if (!backup_top) { goto error; } @@ -XXX,XX +XXX,XX @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs, job->bcs = bcs; job->cluster_size = cluster_size; job->len = len; + job->perf = *perf; block_copy_set_progress_callback(bcs, backup_progress_bytes_callback, job); block_copy_set_progress_meter(bcs, &job->common.job.progress); diff --git a/block/block-copy.c b/block/block-copy.c index XXXXXXX..XXXXXXX 100644 --- a/block/block-copy.c +++ b/block/block-copy.c @@ -XXX,XX +XXX,XX @@ static uint32_t block_copy_max_transfer(BdrvChild *source, BdrvChild *target) } BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target, - int64_t cluster_size, + int64_t cluster_size, bool use_copy_range, BdrvRequestFlags write_flags, Error **errp) { BlockCopyState *s; @@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target, * We enable copy-range, but keep small copy_size, until first * successful copy_range (look at block_copy_do_copy). */ - s->use_copy_range = true; + s->use_copy_range = use_copy_range; s->copy_size = MAX(s->cluster_size, BLOCK_COPY_MAX_BUFFER); } diff --git a/block/replication.c b/block/replication.c index XXXXXXX..XXXXXXX 100644 --- a/block/replication.c +++ b/block/replication.c @@ -XXX,XX +XXX,XX @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, int64_t active_length, hidden_length, disk_length; AioContext *aio_context; Error *local_err = NULL; + BackupPerf perf = { .use_copy_range = true }; aio_context = bdrv_get_aio_context(bs); aio_context_acquire(aio_context); @@ -XXX,XX +XXX,XX @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, s->backup_job = backup_job_create( NULL, s->secondary_disk->bs, s->hidden_disk->bs, 0, MIRROR_SYNC_MODE_NONE, NULL, 0, false, NULL, + &perf, BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT, JOB_INTERNAL, backup_job_completed, bs, NULL, &local_err); diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ static BlockJob *do_backup_common(BackupCommon *backup, { BlockJob *job = NULL; BdrvDirtyBitmap *bmap = NULL; + BackupPerf perf = { .use_copy_range = true }; int job_flags = JOB_DEFAULT; if (!backup->has_speed) { @@ -XXX,XX +XXX,XX @@ static BlockJob *do_backup_common(BackupCommon *backup, backup->compress = false; } + if (backup->x_perf) { + if (backup->x_perf->has_use_copy_range) { + perf.use_copy_range = backup->x_perf->use_copy_range; + } + } + if ((backup->sync == MIRROR_SYNC_MODE_BITMAP) || (backup->sync == MIRROR_SYNC_MODE_INCREMENTAL)) { /* done before desugaring 'incremental' to print the right message */ @@ -XXX,XX +XXX,XX @@ static BlockJob *do_backup_common(BackupCommon *backup, backup->sync, bmap, backup->bitmap_mode, backup->compress, backup->filter_node_name, + &perf, backup->on_source_error, backup->on_target_error, job_flags, NULL, NULL, txn, errp); -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Refactor common path to use BlockCopyCallState pointer as parameter, to prepare it for use in asynchronous block-copy (at least, we'll need to run block-copy in a coroutine, passing the whole parameters as one pointer). Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-3-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/block-copy.c | 51 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/block/block-copy.c b/block/block-copy.c index XXXXXXX..XXXXXXX 100644 --- a/block/block-copy.c +++ b/block/block-copy.c @@ -XXX,XX +XXX,XX @@ static coroutine_fn int block_copy_task_entry(AioTask *task); typedef struct BlockCopyCallState { + /* IN parameters */ + BlockCopyState *s; + int64_t offset; + int64_t bytes; + + /* State */ bool failed; + + /* OUT parameters */ bool error_is_read; } BlockCopyCallState; @@ -XXX,XX +XXX,XX @@ int64_t block_copy_reset_unallocated(BlockCopyState *s, * Returns 1 if dirty clusters found and successfully copied, 0 if no dirty * clusters found and -errno on failure. */ -static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s, - int64_t offset, int64_t bytes, - bool *error_is_read) +static int coroutine_fn +block_copy_dirty_clusters(BlockCopyCallState *call_state) { + BlockCopyState *s = call_state->s; + int64_t offset = call_state->offset; + int64_t bytes = call_state->bytes; + int ret = 0; bool found_dirty = false; int64_t end = offset + bytes; AioTaskPool *aio = NULL; - BlockCopyCallState call_state = {false, false}; /* * block_copy() user is responsible for keeping source and target in same @@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s, BlockCopyTask *task; int64_t status_bytes; - task = block_copy_task_create(s, &call_state, offset, bytes); + task = block_copy_task_create(s, call_state, offset, bytes); if (!task) { /* No more dirty bits in the bitmap */ trace_block_copy_skip_range(s, offset, bytes); @@ -XXX,XX +XXX,XX @@ out: aio_task_pool_free(aio); } - if (error_is_read && ret < 0) { - *error_is_read = call_state.error_is_read; - } return ret < 0 ? ret : found_dirty; } /* - * block_copy + * block_copy_common * * Copy requested region, accordingly to dirty bitmap. * Collaborate with parallel block_copy requests: if they succeed it will help @@ -XXX,XX +XXX,XX @@ out: * it means that some I/O operation failed in context of _this_ block_copy call, * not some parallel operation. */ -int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes, - bool *error_is_read) +static int coroutine_fn block_copy_common(BlockCopyCallState *call_state) { int ret; do { - ret = block_copy_dirty_clusters(s, offset, bytes, error_is_read); + ret = block_copy_dirty_clusters(call_state); if (ret == 0) { - ret = block_copy_wait_one(s, offset, bytes); + ret = block_copy_wait_one(call_state->s, call_state->offset, + call_state->bytes); } /* @@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes, return ret; } +int coroutine_fn block_copy(BlockCopyState *s, int64_t start, int64_t bytes, + bool *error_is_read) +{ + BlockCopyCallState call_state = { + .s = s, + .offset = start, + .bytes = bytes, + }; + + int ret = block_copy_common(&call_state); + + if (error_is_read && ret < 0) { + *error_is_read = call_state.error_is_read; + } + + return ret; +} + BdrvDirtyBitmap *block_copy_dirty_bitmap(BlockCopyState *s) { return s->copy_bitmap; -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> We'll need async block-copy invocation to use in backup directly. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-4-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- include/block/block-copy.h | 29 ++++++++++++++ block/block-copy.c | 81 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/include/block/block-copy.h b/include/block/block-copy.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block-copy.h +++ b/include/block/block-copy.h @@ -XXX,XX +XXX,XX @@ #include "qemu/co-shared-resource.h" typedef void (*ProgressBytesCallbackFunc)(int64_t bytes, void *opaque); +typedef void (*BlockCopyAsyncCallbackFunc)(void *opaque); typedef struct BlockCopyState BlockCopyState; +typedef struct BlockCopyCallState BlockCopyCallState; BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target, int64_t cluster_size, bool use_copy_range, @@ -XXX,XX +XXX,XX @@ int64_t block_copy_reset_unallocated(BlockCopyState *s, int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes, bool *error_is_read); +/* + * Run block-copy in a coroutine, create corresponding BlockCopyCallState + * object and return pointer to it. Never returns NULL. + * + * Caller is responsible to call block_copy_call_free() to free + * BlockCopyCallState object. + */ +BlockCopyCallState *block_copy_async(BlockCopyState *s, + int64_t offset, int64_t bytes, + BlockCopyAsyncCallbackFunc cb, + void *cb_opaque); + +/* + * Free finished BlockCopyCallState. Trying to free running + * block-copy will crash. + */ +void block_copy_call_free(BlockCopyCallState *call_state); + +/* + * Note, that block-copy call is marked finished prior to calling + * the callback. + */ +bool block_copy_call_finished(BlockCopyCallState *call_state); +bool block_copy_call_succeeded(BlockCopyCallState *call_state); +bool block_copy_call_failed(BlockCopyCallState *call_state); +int block_copy_call_status(BlockCopyCallState *call_state, bool *error_is_read); + BdrvDirtyBitmap *block_copy_dirty_bitmap(BlockCopyState *s); void block_copy_set_skip_unallocated(BlockCopyState *s, bool skip); diff --git a/block/block-copy.c b/block/block-copy.c index XXXXXXX..XXXXXXX 100644 --- a/block/block-copy.c +++ b/block/block-copy.c @@ -XXX,XX +XXX,XX @@ static coroutine_fn int block_copy_task_entry(AioTask *task); typedef struct BlockCopyCallState { - /* IN parameters */ + /* IN parameters. Initialized in block_copy_async() and never changed. */ BlockCopyState *s; int64_t offset; int64_t bytes; + BlockCopyAsyncCallbackFunc cb; + void *cb_opaque; + + /* Coroutine where async block-copy is running */ + Coroutine *co; /* State */ - bool failed; + int ret; + bool finished; /* OUT parameters */ bool error_is_read; @@ -XXX,XX +XXX,XX @@ static coroutine_fn int block_copy_task_entry(AioTask *task) ret = block_copy_do_copy(t->s, t->offset, t->bytes, t->zeroes, &error_is_read); - if (ret < 0 && !t->call_state->failed) { - t->call_state->failed = true; + if (ret < 0 && !t->call_state->ret) { + t->call_state->ret = ret; t->call_state->error_is_read = error_is_read; } else { progress_work_done(t->s->progress, t->bytes); @@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_common(BlockCopyCallState *call_state) */ } while (ret > 0); + call_state->finished = true; + + if (call_state->cb) { + call_state->cb(call_state->cb_opaque); + } + return ret; } @@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s, int64_t start, int64_t bytes, return ret; } +static void coroutine_fn block_copy_async_co_entry(void *opaque) +{ + block_copy_common(opaque); +} + +BlockCopyCallState *block_copy_async(BlockCopyState *s, + int64_t offset, int64_t bytes, + BlockCopyAsyncCallbackFunc cb, + void *cb_opaque) +{ + BlockCopyCallState *call_state = g_new(BlockCopyCallState, 1); + + *call_state = (BlockCopyCallState) { + .s = s, + .offset = offset, + .bytes = bytes, + .cb = cb, + .cb_opaque = cb_opaque, + + .co = qemu_coroutine_create(block_copy_async_co_entry, call_state), + }; + + qemu_coroutine_enter(call_state->co); + + return call_state; +} + +void block_copy_call_free(BlockCopyCallState *call_state) +{ + if (!call_state) { + return; + } + + assert(call_state->finished); + g_free(call_state); +} + +bool block_copy_call_finished(BlockCopyCallState *call_state) +{ + return call_state->finished; +} + +bool block_copy_call_succeeded(BlockCopyCallState *call_state) +{ + return call_state->finished && call_state->ret == 0; +} + +bool block_copy_call_failed(BlockCopyCallState *call_state) +{ + return call_state->finished && call_state->ret < 0; +} + +int block_copy_call_status(BlockCopyCallState *call_state, bool *error_is_read) +{ + assert(call_state->finished); + if (error_is_read) { + *error_is_read = call_state->error_is_read; + } + return call_state->ret; +} + BdrvDirtyBitmap *block_copy_dirty_bitmap(BlockCopyState *s) { return s->copy_bitmap; -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> They will be used for backup. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-5-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- include/block/block-copy.h | 6 ++++++ block/block-copy.c | 11 +++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/include/block/block-copy.h b/include/block/block-copy.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block-copy.h +++ b/include/block/block-copy.h @@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes, * * Caller is responsible to call block_copy_call_free() to free * BlockCopyCallState object. + * + * @max_workers means maximum of parallel coroutines to execute sub-requests, + * must be > 0. + * + * @max_chunk means maximum length for one IO operation. Zero means unlimited. */ BlockCopyCallState *block_copy_async(BlockCopyState *s, int64_t offset, int64_t bytes, + int max_workers, int64_t max_chunk, BlockCopyAsyncCallbackFunc cb, void *cb_opaque); diff --git a/block/block-copy.c b/block/block-copy.c index XXXXXXX..XXXXXXX 100644 --- a/block/block-copy.c +++ b/block/block-copy.c @@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyCallState { BlockCopyState *s; int64_t offset; int64_t bytes; + int max_workers; + int64_t max_chunk; BlockCopyAsyncCallbackFunc cb; void *cb_opaque; @@ -XXX,XX +XXX,XX @@ static BlockCopyTask *block_copy_task_create(BlockCopyState *s, int64_t offset, int64_t bytes) { BlockCopyTask *task; + int64_t max_chunk = MIN_NON_ZERO(s->copy_size, call_state->max_chunk); if (!bdrv_dirty_bitmap_next_dirty_area(s->copy_bitmap, offset, offset + bytes, - s->copy_size, &offset, &bytes)) + max_chunk, &offset, &bytes)) { return NULL; } @@ -XXX,XX +XXX,XX @@ block_copy_dirty_clusters(BlockCopyCallState *call_state) bytes = end - offset; if (!aio && bytes) { - aio = aio_task_pool_new(BLOCK_COPY_MAX_WORKERS); + aio = aio_task_pool_new(call_state->max_workers); } ret = block_copy_task_run(aio, task); @@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s, int64_t start, int64_t bytes, .s = s, .offset = start, .bytes = bytes, + .max_workers = BLOCK_COPY_MAX_WORKERS, }; int ret = block_copy_common(&call_state); @@ -XXX,XX +XXX,XX @@ static void coroutine_fn block_copy_async_co_entry(void *opaque) BlockCopyCallState *block_copy_async(BlockCopyState *s, int64_t offset, int64_t bytes, + int max_workers, int64_t max_chunk, BlockCopyAsyncCallbackFunc cb, void *cb_opaque) { @@ -XXX,XX +XXX,XX @@ BlockCopyCallState *block_copy_async(BlockCopyState *s, .s = s, .offset = offset, .bytes = bytes, + .max_workers = max_workers, + .max_chunk = max_chunk, .cb = cb, .cb_opaque = cb_opaque, -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> It simplifies debugging. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-6-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/block-copy.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/block/block-copy.c b/block/block-copy.c index XXXXXXX..XXXXXXX 100644 --- a/block/block-copy.c +++ b/block/block-copy.c @@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyCallState { /* Coroutine where async block-copy is running */ Coroutine *co; + /* To reference all call states from BlockCopyState */ + QLIST_ENTRY(BlockCopyCallState) list; + /* State */ int ret; bool finished; @@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyState { bool use_copy_range; int64_t copy_size; uint64_t len; - QLIST_HEAD(, BlockCopyTask) tasks; + QLIST_HEAD(, BlockCopyTask) tasks; /* All tasks from all block-copy calls */ + QLIST_HEAD(, BlockCopyCallState) calls; BdrvRequestFlags write_flags; @@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target, } QLIST_INIT(&s->tasks); + QLIST_INIT(&s->calls); return s; } @@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_common(BlockCopyCallState *call_state) { int ret; + QLIST_INSERT_HEAD(&call_state->s->calls, call_state, list); + do { ret = block_copy_dirty_clusters(call_state); @@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_common(BlockCopyCallState *call_state) call_state->cb(call_state->cb_opaque); } + QLIST_REMOVE(call_state, list); + return ret; } -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> We are going to directly use one async block-copy operation for backup job, so we need rate limiter. We want to maintain current backup behavior: only background copying is limited and copy-before-write operations only participate in limit calculation. Therefore we need one rate limiter for block-copy state and boolean flag for block-copy call state for actual limitation. Note, that we can't just calculate each chunk in limiter after successful copying: it will not save us from starting a lot of async sub-requests which will exceed limit too much. Instead let's use the following scheme on sub-request creation: 1. If at the moment limit is not exceeded, create the request and account it immediately. 2. If at the moment limit is already exceeded, drop create sub-request and handle limit instead (by sleep). With this approach we'll never exceed the limit more than by one sub-request (which pretty much matches current backup behavior). Note also, that if there is in-flight block-copy async call, block_copy_kick() should be used after set-speed to apply new setup faster. For that block_copy_kick() published in this patch. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-7-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- include/block/block-copy.h | 5 ++++- block/backup-top.c | 2 +- block/backup.c | 2 +- block/block-copy.c | 46 +++++++++++++++++++++++++++++++++++++- 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/include/block/block-copy.h b/include/block/block-copy.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block-copy.h +++ b/include/block/block-copy.h @@ -XXX,XX +XXX,XX @@ int64_t block_copy_reset_unallocated(BlockCopyState *s, int64_t offset, int64_t *count); int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes, - bool *error_is_read); + bool ignore_ratelimit, bool *error_is_read); /* * Run block-copy in a coroutine, create corresponding BlockCopyCallState @@ -XXX,XX +XXX,XX @@ bool block_copy_call_succeeded(BlockCopyCallState *call_state); bool block_copy_call_failed(BlockCopyCallState *call_state); int block_copy_call_status(BlockCopyCallState *call_state, bool *error_is_read); +void block_copy_set_speed(BlockCopyState *s, uint64_t speed); +void block_copy_kick(BlockCopyCallState *call_state); + BdrvDirtyBitmap *block_copy_dirty_bitmap(BlockCopyState *s); void block_copy_set_skip_unallocated(BlockCopyState *s, bool skip); diff --git a/block/backup-top.c b/block/backup-top.c index XXXXXXX..XXXXXXX 100644 --- a/block/backup-top.c +++ b/block/backup-top.c @@ -XXX,XX +XXX,XX @@ static coroutine_fn int backup_top_cbw(BlockDriverState *bs, uint64_t offset, off = QEMU_ALIGN_DOWN(offset, s->cluster_size); end = QEMU_ALIGN_UP(offset + bytes, s->cluster_size); - return block_copy(s->bcs, off, end - off, NULL); + return block_copy(s->bcs, off, end - off, true, NULL); } static int coroutine_fn backup_top_co_pdiscard(BlockDriverState *bs, diff --git a/block/backup.c b/block/backup.c index XXXXXXX..XXXXXXX 100644 --- a/block/backup.c +++ b/block/backup.c @@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_do_cow(BackupBlockJob *job, trace_backup_do_cow_enter(job, start, offset, bytes); - ret = block_copy(job->bcs, start, end - start, error_is_read); + ret = block_copy(job->bcs, start, end - start, true, error_is_read); trace_backup_do_cow_return(job, offset, bytes, ret); diff --git a/block/block-copy.c b/block/block-copy.c index XXXXXXX..XXXXXXX 100644 --- a/block/block-copy.c +++ b/block/block-copy.c @@ -XXX,XX +XXX,XX @@ #define BLOCK_COPY_MAX_BUFFER (1 * MiB) #define BLOCK_COPY_MAX_MEM (128 * MiB) #define BLOCK_COPY_MAX_WORKERS 64 +#define BLOCK_COPY_SLICE_TIME 100000000ULL /* ns */ static coroutine_fn int block_copy_task_entry(AioTask *task); @@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyCallState { int64_t bytes; int max_workers; int64_t max_chunk; + bool ignore_ratelimit; BlockCopyAsyncCallbackFunc cb; void *cb_opaque; @@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyCallState { /* State */ int ret; bool finished; + QemuCoSleepState *sleep_state; /* OUT parameters */ bool error_is_read; @@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyState { void *progress_opaque; SharedResource *mem; + + uint64_t speed; + RateLimit rate_limit; } BlockCopyState; static BlockCopyTask *find_conflicting_task(BlockCopyState *s, @@ -XXX,XX +XXX,XX @@ block_copy_dirty_clusters(BlockCopyCallState *call_state) } task->zeroes = ret & BDRV_BLOCK_ZERO; + if (s->speed) { + if (!call_state->ignore_ratelimit) { + uint64_t ns = ratelimit_calculate_delay(&s->rate_limit, 0); + if (ns > 0) { + block_copy_task_end(task, -EAGAIN); + g_free(task); + qemu_co_sleep_ns_wakeable(QEMU_CLOCK_REALTIME, ns, + &call_state->sleep_state); + continue; + } + } + + ratelimit_calculate_delay(&s->rate_limit, task->bytes); + } + trace_block_copy_process(s, task->offset); co_get_from_shres(s->mem, task->bytes); @@ -XXX,XX +XXX,XX @@ out: return ret < 0 ? ret : found_dirty; } +void block_copy_kick(BlockCopyCallState *call_state) +{ + if (call_state->sleep_state) { + qemu_co_sleep_wake(call_state->sleep_state); + } +} + /* * block_copy_common * @@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_common(BlockCopyCallState *call_state) } int coroutine_fn block_copy(BlockCopyState *s, int64_t start, int64_t bytes, - bool *error_is_read) + bool ignore_ratelimit, bool *error_is_read) { BlockCopyCallState call_state = { .s = s, .offset = start, .bytes = bytes, + .ignore_ratelimit = ignore_ratelimit, .max_workers = BLOCK_COPY_MAX_WORKERS, }; @@ -XXX,XX +XXX,XX @@ void block_copy_set_skip_unallocated(BlockCopyState *s, bool skip) { s->skip_unallocated = skip; } + +void block_copy_set_speed(BlockCopyState *s, uint64_t speed) +{ + s->speed = speed; + if (speed > 0) { + ratelimit_set_speed(&s->rate_limit, speed, BLOCK_COPY_SLICE_TIME); + } + + /* + * Note: it's good to kick all call states from here, but it should be done + * only from a coroutine, to not crash if s->calls list changed while + * entering one call. So for now, the only user of this function kicks its + * only one call_state by hand. + */ +} -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Add function to cancel running async block-copy call. It will be used in backup. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-8-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- include/block/block-copy.h | 13 +++++++++++++ block/block-copy.c | 24 +++++++++++++++++++----- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/include/block/block-copy.h b/include/block/block-copy.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block-copy.h +++ b/include/block/block-copy.h @@ -XXX,XX +XXX,XX @@ void block_copy_call_free(BlockCopyCallState *call_state); bool block_copy_call_finished(BlockCopyCallState *call_state); bool block_copy_call_succeeded(BlockCopyCallState *call_state); bool block_copy_call_failed(BlockCopyCallState *call_state); +bool block_copy_call_cancelled(BlockCopyCallState *call_state); int block_copy_call_status(BlockCopyCallState *call_state, bool *error_is_read); void block_copy_set_speed(BlockCopyState *s, uint64_t speed); void block_copy_kick(BlockCopyCallState *call_state); +/* + * Cancel running block-copy call. + * + * Cancel leaves block-copy state valid: dirty bits are correct and you may use + * cancel + <run block_copy with same parameters> to emulate pause/resume. + * + * Note also, that the cancel is async: it only marks block-copy call to be + * cancelled. So, the call may be cancelled (block_copy_call_cancelled() reports + * true) but not yet finished (block_copy_call_finished() reports false). + */ +void block_copy_call_cancel(BlockCopyCallState *call_state); + BdrvDirtyBitmap *block_copy_dirty_bitmap(BlockCopyState *s); void block_copy_set_skip_unallocated(BlockCopyState *s, bool skip); diff --git a/block/block-copy.c b/block/block-copy.c index XXXXXXX..XXXXXXX 100644 --- a/block/block-copy.c +++ b/block/block-copy.c @@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyCallState { int ret; bool finished; QemuCoSleepState *sleep_state; + bool cancelled; /* OUT parameters */ bool error_is_read; @@ -XXX,XX +XXX,XX @@ block_copy_dirty_clusters(BlockCopyCallState *call_state) assert(QEMU_IS_ALIGNED(offset, s->cluster_size)); assert(QEMU_IS_ALIGNED(bytes, s->cluster_size)); - while (bytes && aio_task_pool_status(aio) == 0) { + while (bytes && aio_task_pool_status(aio) == 0 && !call_state->cancelled) { BlockCopyTask *task; int64_t status_bytes; @@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_common(BlockCopyCallState *call_state) do { ret = block_copy_dirty_clusters(call_state); - if (ret == 0) { + if (ret == 0 && !call_state->cancelled) { ret = block_copy_wait_one(call_state->s, call_state->offset, call_state->bytes); } @@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_common(BlockCopyCallState *call_state) * 2. We have waited for some intersecting block-copy request * It may have failed and produced new dirty bits. */ - } while (ret > 0); + } while (ret > 0 && !call_state->cancelled); call_state->finished = true; @@ -XXX,XX +XXX,XX @@ bool block_copy_call_finished(BlockCopyCallState *call_state) bool block_copy_call_succeeded(BlockCopyCallState *call_state) { - return call_state->finished && call_state->ret == 0; + return call_state->finished && !call_state->cancelled && + call_state->ret == 0; } bool block_copy_call_failed(BlockCopyCallState *call_state) { - return call_state->finished && call_state->ret < 0; + return call_state->finished && !call_state->cancelled && + call_state->ret < 0; +} + +bool block_copy_call_cancelled(BlockCopyCallState *call_state) +{ + return call_state->cancelled; } int block_copy_call_status(BlockCopyCallState *call_state, bool *error_is_read) @@ -XXX,XX +XXX,XX @@ int block_copy_call_status(BlockCopyCallState *call_state, bool *error_is_read) return call_state->ret; } +void block_copy_call_cancel(BlockCopyCallState *call_state) +{ + call_state->cancelled = true; + block_copy_kick(call_state); +} + BdrvDirtyBitmap *block_copy_dirty_bitmap(BlockCopyState *s) { return s->copy_bitmap; -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> We are going to use async block-copy call in backup, so we'll need to passthrough setting backup speed to block-copy call. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-9-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- include/block/blockjob_int.h | 2 ++ blockjob.c | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/include/block/blockjob_int.h b/include/block/blockjob_int.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/blockjob_int.h +++ b/include/block/blockjob_int.h @@ -XXX,XX +XXX,XX @@ struct BlockJobDriver { * besides job->blk to the new AioContext. */ void (*attached_aio_context)(BlockJob *job, AioContext *new_context); + + void (*set_speed)(BlockJob *job, int64_t speed); }; /** diff --git a/blockjob.c b/blockjob.c index XXXXXXX..XXXXXXX 100644 --- a/blockjob.c +++ b/blockjob.c @@ -XXX,XX +XXX,XX @@ static bool job_timer_pending(Job *job) void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp) { + const BlockJobDriver *drv = block_job_driver(job); int64_t old_speed = job->speed; if (job_apply_verb(&job->job, JOB_VERB_SET_SPEED, errp)) { @@ -XXX,XX +XXX,XX @@ void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp) ratelimit_set_speed(&job->limit, speed, BLOCK_JOB_SLICE_TIME); job->speed = speed; + + if (drv->set_speed) { + drv->set_speed(job, speed); + } + if (speed && speed <= old_speed) { return; } -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> If main job coroutine called job_yield (while some background process is in progress), we should give it a chance to call job_pause_point(). It will be used in backup, when moved on async block-copy. Note, that job_user_pause is not enough: we want to handle child_job_drained_begin() as well, which call job_pause(). Still, if job is already in job_do_yield() in job_pause_point() we should not enter it. iotest 109 output is modified: on stop we do bdrv_drain_all() which now triggers job pause immediately (and pause after ready is standby). Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Message-Id: <20210116214705.822267-10-vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- job.c | 3 +++ tests/qemu-iotests/109.out | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/job.c b/job.c index XXXXXXX..XXXXXXX 100644 --- a/job.c +++ b/job.c @@ -XXX,XX +XXX,XX @@ static bool job_timer_not_pending(Job *job) void job_pause(Job *job) { job->pause_count++; + if (!job->paused) { + job_enter(job); + } } void job_resume(Job *job) diff --git a/tests/qemu-iotests/109.out b/tests/qemu-iotests/109.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/109.out +++ b/tests/qemu-iotests/109.out @@ -XXX,XX +XXX,XX @@ read 512/512 bytes at offset 0 {"execute":"quit"} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "standby", "id": "src"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 1024, "offset": 1024, "speed": 0, "type": "mirror"}} @@ -XXX,XX +XXX,XX @@ read 512/512 bytes at offset 0 {"execute":"quit"} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "standby", "id": "src"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 197120, "offset": 197120, "speed": 0, "type": "mirror"}} @@ -XXX,XX +XXX,XX @@ read 512/512 bytes at offset 0 {"execute":"quit"} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "standby", "id": "src"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 327680, "offset": 327680, "speed": 0, "type": "mirror"}} @@ -XXX,XX +XXX,XX @@ read 512/512 bytes at offset 0 {"execute":"quit"} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "standby", "id": "src"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 1024, "offset": 1024, "speed": 0, "type": "mirror"}} @@ -XXX,XX +XXX,XX @@ read 512/512 bytes at offset 0 {"execute":"quit"} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "standby", "id": "src"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 65536, "offset": 65536, "speed": 0, "type": "mirror"}} @@ -XXX,XX +XXX,XX @@ read 512/512 bytes at offset 0 {"execute":"quit"} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "standby", "id": "src"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 2560, "offset": 2560, "speed": 0, "type": "mirror"}} @@ -XXX,XX +XXX,XX @@ read 512/512 bytes at offset 0 {"execute":"quit"} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "standby", "id": "src"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 2560, "offset": 2560, "speed": 0, "type": "mirror"}} @@ -XXX,XX +XXX,XX @@ read 512/512 bytes at offset 0 {"execute":"quit"} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "standby", "id": "src"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 31457280, "offset": 31457280, "speed": 0, "type": "mirror"}} @@ -XXX,XX +XXX,XX @@ read 512/512 bytes at offset 0 {"execute":"quit"} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "standby", "id": "src"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 327680, "offset": 327680, "speed": 0, "type": "mirror"}} @@ -XXX,XX +XXX,XX @@ read 512/512 bytes at offset 0 {"execute":"quit"} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "standby", "id": "src"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 2048, "offset": 2048, "speed": 0, "type": "mirror"}} @@ -XXX,XX +XXX,XX @@ WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed {"execute":"quit"} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "standby", "id": "src"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 512, "offset": 512, "speed": 0, "type": "mirror"}} @@ -XXX,XX +XXX,XX @@ Images are identical. {"execute":"quit"} {"return": {}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "standby", "id": "src"}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "src"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 512, "offset": 512, "speed": 0, "type": "mirror"}} -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Add new parameters to configure future backup features. The patch doesn't introduce aio backup requests (so we actually have only one worker) neither requests larger than one cluster. Still, formally we satisfy these maximums anyway, so add the parameters now, to facilitate further patch which will really change backup job behavior. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-11-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- qapi/block-core.json | 13 ++++++++++++- block/backup.c | 28 +++++++++++++++++++++++----- block/replication.c | 2 +- blockdev.c | 8 +++++++- 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/qapi/block-core.json b/qapi/block-core.json index XXXXXXX..XXXXXXX 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -XXX,XX +XXX,XX @@ # # @use-copy-range: Use copy offloading. Default true. # +# @max-workers: Maximum number of parallel requests for the sustained background +# copying process. Doesn't influence copy-before-write operations. +# Default 64. +# +# @max-chunk: Maximum request length for the sustained background copying +# process. Doesn't influence copy-before-write operations. +# 0 means unlimited. If max-chunk is non-zero then it should not be +# less than job cluster size which is calculated as maximum of +# target image cluster size and 64k. Default 0. +# # Since: 6.0 ## { 'struct': 'BackupPerf', - 'data': { '*use-copy-range': 'bool' }} + 'data': { '*use-copy-range': 'bool', + '*max-workers': 'int', '*max-chunk': 'int64' } } ## # @BackupCommon: diff --git a/block/backup.c b/block/backup.c index XXXXXXX..XXXXXXX 100644 --- a/block/backup.c +++ b/block/backup.c @@ -XXX,XX +XXX,XX @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs, return NULL; } + cluster_size = backup_calculate_cluster_size(target, errp); + if (cluster_size < 0) { + goto error; + } + + if (perf->max_workers < 1) { + error_setg(errp, "max-workers must be greater than zero"); + return NULL; + } + + if (perf->max_chunk < 0) { + error_setg(errp, "max-chunk must be zero (which means no limit) or " + "positive"); + return NULL; + } + + if (perf->max_chunk && perf->max_chunk < cluster_size) { + error_setg(errp, "Required max-chunk (%" PRIi64 ") is less than backup " + "cluster size (%" PRIi64 ")", perf->max_chunk, cluster_size); + return NULL; + } + + if (sync_bitmap) { /* If we need to write to this bitmap, check that we can: */ if (bitmap_mode != BITMAP_SYNC_MODE_NEVER && @@ -XXX,XX +XXX,XX @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs, goto error; } - cluster_size = backup_calculate_cluster_size(target, errp); - if (cluster_size < 0) { - goto error; - } - /* * If source is in backing chain of target assume that target is going to be * used for "image fleecing", i.e. it should represent a kind of snapshot of diff --git a/block/replication.c b/block/replication.c index XXXXXXX..XXXXXXX 100644 --- a/block/replication.c +++ b/block/replication.c @@ -XXX,XX +XXX,XX @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, int64_t active_length, hidden_length, disk_length; AioContext *aio_context; Error *local_err = NULL; - BackupPerf perf = { .use_copy_range = true }; + BackupPerf perf = { .use_copy_range = true, .max_workers = 1 }; aio_context = bdrv_get_aio_context(bs); aio_context_acquire(aio_context); diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ static BlockJob *do_backup_common(BackupCommon *backup, { BlockJob *job = NULL; BdrvDirtyBitmap *bmap = NULL; - BackupPerf perf = { .use_copy_range = true }; + BackupPerf perf = { .use_copy_range = true, .max_workers = 64 }; int job_flags = JOB_DEFAULT; if (!backup->has_speed) { @@ -XXX,XX +XXX,XX @@ static BlockJob *do_backup_common(BackupCommon *backup, if (backup->x_perf->has_use_copy_range) { perf.use_copy_range = backup->x_perf->use_copy_range; } + if (backup->x_perf->has_max_workers) { + perf.max_workers = backup->x_perf->max_workers; + } + if (backup->x_perf->has_max_chunk) { + perf.max_chunk = backup->x_perf->max_chunk; + } } if ((backup->sync == MIRROR_SYNC_MODE_BITMAP) || -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> After introducing parallel async copy requests instead of plain cluster-by-cluster copying loop, we'll have to wait for paused status, as we need to wait for several parallel request. So, let's gently wait instead of just asserting that job already paused. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-12-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/056 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/qemu-iotests/056 b/tests/qemu-iotests/056 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/056 +++ b/tests/qemu-iotests/056 @@ -XXX,XX +XXX,XX @@ class BackupTest(iotests.QMPTestCase): event = self.vm.event_wait(name="BLOCK_JOB_ERROR", match={'data': {'device': 'drive0'}}) self.assertNotEqual(event, None) - # OK, job should be wedged - res = self.vm.qmp('query-block-jobs') + # OK, job should pause, but it can't do it immediately, as it can't + # cancel other parallel requests (which didn't fail) + with iotests.Timeout(60, "Timeout waiting for backup actually paused"): + while True: + res = self.vm.qmp('query-block-jobs') + if res['return'][0]['status'] == 'paused': + break self.assert_qmp(res, 'return[0]/status', 'paused') res = self.vm.qmp('block-job-dismiss', id='drive0') self.assert_qmp(res, 'error/desc', -- 2.29.2 Right now, this does not change anything, because backup ignores max-chunk and max-workers. However, as soon as backup is switched over to block-copy for the background copying process, we will need it to keep 129 passing. Signed-off-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210120102043.28346-1-mreitz@redhat.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> --- tests/qemu-iotests/129 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/qemu-iotests/129 b/tests/qemu-iotests/129 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/129 +++ b/tests/qemu-iotests/129 @@ -XXX,XX +XXX,XX @@ class TestStopWithBlockJob(iotests.QMPTestCase): sync="full", buf_size=65536) def test_drive_backup(self): + # Limit max-chunk and max-workers so that block-copy will not + # launch so many workers working on so much data each that + # stop's bdrv_drain_all() would finish the job self.do_test_stop("drive-backup", device="drive0", target=self.target_img, format=iotests.imgfmt, - sync="full") + sync="full", + x_perf={ 'max-chunk': 65536, + 'max-workers': 8 }) def test_block_commit(self): # Add overlay above the source node so that we actually use a -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> The further change of moving backup to be a one block-copy call will make copying chunk-size and cluster-size two separate things. So, even with 64k cluster sized qcow2 image, default chunk would be 1M. 185 test however assumes, that with speed limited to 64K, one iteration would result in offset=64K. It will change, as first iteration would result in offset=1M independently of speed. So, let's explicitly specify, what test wants: set max-chunk to 64K, so that one iteration is 64K. Note, that we don't need to limit max-workers, as block-copy rate limiter will handle the situation and wouldn't start new workers when speed limit is obviously reached. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-13-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/185 | 3 ++- tests/qemu-iotests/185.out | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/qemu-iotests/185 b/tests/qemu-iotests/185 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/185 +++ b/tests/qemu-iotests/185 @@ -XXX,XX +XXX,XX @@ _send_qemu_cmd $h \ 'target': '$TEST_IMG.copy', 'format': '$IMGFMT', 'sync': 'full', - 'speed': 65536 } }" \ + 'speed': 65536, + 'x-perf': {'max-chunk': 65536} } }" \ "return" # If we don't sleep here 'quit' command races with disk I/O diff --git a/tests/qemu-iotests/185.out b/tests/qemu-iotests/185.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/185.out +++ b/tests/qemu-iotests/185.out @@ -XXX,XX +XXX,XX @@ Formatting 'TEST_DIR/t.qcow2.copy', fmt=qcow2 cluster_size=65536 extended_l2=off 'target': 'TEST_DIR/t.IMGFMT.copy', 'format': 'IMGFMT', 'sync': 'full', - 'speed': 65536 } } + 'speed': 65536, + 'x-perf': { 'max-chunk': 65536 } } } Formatting 'TEST_DIR/t.qcow2.copy', fmt=qcow2 cluster_size=65536 extended_l2=off compression_type=zlib size=67108864 lazy_refcounts=off refcount_bits=16 {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "disk"}} {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "disk"}} -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> The further change of moving backup to be a one block-copy call will make copying chunk-size and cluster-size two separate things. So, even with 64k cluster sized qcow2 image, default chunk would be 1M. Test 219 depends on specified chunk-size. Update it for explicit chunk-size for backup as for mirror. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-14-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/219 | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/qemu-iotests/219 b/tests/qemu-iotests/219 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/219 +++ b/tests/qemu-iotests/219 @@ -XXX,XX +XXX,XX @@ with iotests.FilePath('disk.img') as disk_path, \ # but related to this also automatic state transitions like job # completion), but still get pause points often enough to avoid making this # test very slow, it's important to have the right ratio between speed and - # buf_size. + # copy-chunk-size. # - # For backup, buf_size is hard-coded to the source image cluster size (64k), - # so we'll pick the same for mirror. The slice time, i.e. the granularity - # of the rate limiting is 100ms. With a speed of 256k per second, we can - # get four pause points per second. This gives us 250ms per iteration, - # which should be enough to stay deterministic. + # Chose 64k copy-chunk-size both for mirror (by buf_size) and backup (by + # x-max-chunk). The slice time, i.e. the granularity of the rate limiting + # is 100ms. With a speed of 256k per second, we can get four pause points + # per second. This gives us 250ms per iteration, which should be enough to + # stay deterministic. test_job_lifecycle(vm, 'drive-mirror', has_ready=True, job_args={ 'device': 'drive0-node', @@ -XXX,XX +XXX,XX @@ with iotests.FilePath('disk.img') as disk_path, \ 'target': copy_path, 'sync': 'full', 'speed': 262144, + 'x-perf': {'max-chunk': 65536}, 'auto-finalize': auto_finalize, 'auto-dismiss': auto_dismiss, }) -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Iotest 257 dumps a lot of in-progress information of backup job, such as offset and bitmap dirtiness. Further commit will move backup to be one block-copy call, which will introduce async parallel requests instead of plain cluster-by-cluster copying. To keep things deterministic, allow only one worker (only one copy request at a time) for this test. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-15-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/257 | 1 + tests/qemu-iotests/257.out | 306 ++++++++++++++++++------------------- 2 files changed, 154 insertions(+), 153 deletions(-) diff --git a/tests/qemu-iotests/257 b/tests/qemu-iotests/257 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/257 +++ b/tests/qemu-iotests/257 @@ -XXX,XX +XXX,XX @@ def blockdev_backup(vm, device, target, sync, **kwargs): target=target, sync=sync, filter_node_name='backup-top', + x_perf={'max-workers': 1}, **kwargs) return result diff --git a/tests/qemu-iotests/257.out b/tests/qemu-iotests/257.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/257.out +++ b/tests/qemu-iotests/257.out @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} --- Write #2 --- @@ -XXX,XX +XXX,XX @@ expecting 15 dirty sectors; have 15. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 15 dirty sectors; have 15. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"action": "report", "device": "backup_1", "operation": "read"}, "event": "BLOCK_JOB_ERROR", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} {"data": {"device": "backup_1", "error": "Input/output error", "len": 393216, "offset": 65536, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 14 dirty sectors; have 14. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 14 dirty sectors; have 14. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} --- Write #2 --- @@ -XXX,XX +XXX,XX @@ expecting 15 dirty sectors; have 15. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 15 dirty sectors; have 15. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} --- Write #2 --- @@ -XXX,XX +XXX,XX @@ expecting 15 dirty sectors; have 15. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 15 dirty sectors; have 15. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"action": "report", "device": "backup_1", "operation": "read"}, "event": "BLOCK_JOB_ERROR", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} {"data": {"device": "backup_1", "error": "Input/output error", "len": 393216, "offset": 65536, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 14 dirty sectors; have 14. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 14 dirty sectors; have 14. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} --- Write #2 --- @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} --- Write #2 --- @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"action": "report", "device": "backup_1", "operation": "read"}, "event": "BLOCK_JOB_ERROR", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} {"data": {"device": "backup_1", "error": "Input/output error", "len": 393216, "offset": 65536, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 13 dirty sectors; have 13. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 13 dirty sectors; have 13. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "bitmap", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} --- Write #2 --- @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "full", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "full", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} --- Write #2 --- @@ -XXX,XX +XXX,XX @@ expecting 15 dirty sectors; have 15. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 15 dirty sectors; have 15. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "full", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "full", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"action": "report", "device": "backup_1", "operation": "read"}, "event": "BLOCK_JOB_ERROR", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} {"data": {"device": "backup_1", "error": "Input/output error", "len": 67108864, "offset": 983040, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 14 dirty sectors; have 14. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 14 dirty sectors; have 14. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "full", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "full", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} --- Write #2 --- @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "full", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "full", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} --- Write #2 --- @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "full", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "full", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"action": "report", "device": "backup_1", "operation": "read"}, "event": "BLOCK_JOB_ERROR", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} {"data": {"device": "backup_1", "error": "Input/output error", "len": 67108864, "offset": 983040, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 1014 dirty sectors; have 1014. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 1014 dirty sectors; have 1014. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "full", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "full", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} --- Write #2 --- @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "top", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "top", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} --- Write #2 --- @@ -XXX,XX +XXX,XX @@ expecting 15 dirty sectors; have 15. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 15 dirty sectors; have 15. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "top", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "top", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"action": "report", "device": "backup_1", "operation": "read"}, "event": "BLOCK_JOB_ERROR", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} {"data": {"device": "backup_1", "error": "Input/output error", "len": 458752, "offset": 65536, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 14 dirty sectors; have 14. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 14 dirty sectors; have 14. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "top", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "top", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} --- Write #2 --- @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "top", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "top", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} --- Write #2 --- @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "top", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "top", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"action": "report", "device": "backup_1", "operation": "read"}, "event": "BLOCK_JOB_ERROR", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} {"data": {"device": "backup_1", "error": "Input/output error", "len": 458752, "offset": 65536, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 14 dirty sectors; have 14. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 14 dirty sectors; have 14. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ write -P0x76 0x3ff0000 0x10000 {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_0", "sync": "full", "target": "ref_target_0", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_0", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_1", "sync": "full", "target": "ref_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_1", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 6 dirty sectors; have 6. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "top", "target": "backup_target_1"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_1", "sync": "top", "target": "backup_target_1", "x-perf": {"max-workers": 1}}} {"return": {}} --- Write #2 --- @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "ref_backup_2", "sync": "full", "target": "ref_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"data": {"device": "ref_backup_2", "len": 67108864, "offset": 67108864, "speed": 0, "type": "backup"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} @@ -XXX,XX +XXX,XX @@ expecting 12 dirty sectors; have 12. OK! {"execute": "job-dismiss", "arguments": {"id": "bdc-fmt-job"}} {"return": {}} {} -{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2"}} +{"execute": "blockdev-backup", "arguments": {"auto-finalize": false, "bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "backup_2", "sync": "bitmap", "target": "backup_target_2", "x-perf": {"max-workers": 1}}} {"return": {}} {"execute": "job-finalize", "arguments": {"id": "backup_2"}} {"return": {}} @@ -XXX,XX +XXX,XX @@ qemu_img compare "TEST_DIR/PID-img" "TEST_DIR/PID-fbackup2" ==> Identical, OK! -- Sync mode incremental tests -- -{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "must provide a valid bitmap name for 'incremental' sync mode"}} -{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "must provide a valid bitmap name for 'incremental' sync mode"}} -{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "must provide a valid bitmap name for 'incremental' sync mode"}} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "must provide a valid bitmap name for 'incremental' sync mode"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap sync mode must be 'on-success' when using sync mode 'incremental'"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap sync mode must be 'on-success' when using sync mode 'incremental'"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap sync mode must be 'on-success' when using sync mode 'incremental'"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "incremental", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap sync mode must be 'on-success' when using sync mode 'incremental'"}} -- Sync mode bitmap tests -- -{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "must provide a valid bitmap name for 'bitmap' sync mode"}} -{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "must provide a valid bitmap name for 'bitmap' sync mode"}} -{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "must provide a valid bitmap name for 'bitmap' sync mode"}} -{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "must provide a valid bitmap name for 'bitmap' sync mode"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "bitmap", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap sync mode must be given when providing a bitmap"}} -- Sync mode full tests -- -{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Cannot specify bitmap sync mode without a bitmap"}} -{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Cannot specify bitmap sync mode without a bitmap"}} -{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Cannot specify bitmap sync mode without a bitmap"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap sync mode 'never' has no meaningful effect when combined with sync mode 'full'"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "full", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap sync mode must be given when providing a bitmap"}} -- Sync mode top tests -- -{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Cannot specify bitmap sync mode without a bitmap"}} -{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Cannot specify bitmap sync mode without a bitmap"}} -{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Cannot specify bitmap sync mode without a bitmap"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap sync mode 'never' has no meaningful effect when combined with sync mode 'top'"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "top", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap sync mode must be given when providing a bitmap"}} -- Sync mode none tests -- -{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Cannot specify bitmap sync mode without a bitmap"}} -{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Cannot specify bitmap sync mode without a bitmap"}} -{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Cannot specify bitmap sync mode without a bitmap"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap404", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap 'bitmap404' could not be found"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "bitmap-mode": "on-success", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "sync mode 'none' does not produce meaningful bitmap outputs"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "bitmap-mode": "always", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "sync mode 'none' does not produce meaningful bitmap outputs"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "bitmap-mode": "never", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "sync mode 'none' does not produce meaningful bitmap outputs"}} -{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target"}} +{"execute": "blockdev-backup", "arguments": {"bitmap": "bitmap0", "device": "drive0", "filter-node-name": "backup-top", "job-id": "api_job", "sync": "none", "target": "backup_target", "x-perf": {"max-workers": 1}}} {"error": {"class": "GenericError", "desc": "Bitmap sync mode must be given when providing a bitmap"}} -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> We are going to stop use of this callback in the following commit. Still the callback handling code will be dropped in a separate commit. So, for now let's make it optional. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-16-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/block-copy.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/block/block-copy.c b/block/block-copy.c index XXXXXXX..XXXXXXX 100644 --- a/block/block-copy.c +++ b/block/block-copy.c @@ -XXX,XX +XXX,XX @@ static coroutine_fn int block_copy_task_entry(AioTask *task) t->call_state->error_is_read = error_is_read; } else { progress_work_done(t->s->progress, t->bytes); - t->s->progress_bytes_callback(t->bytes, t->s->progress_opaque); + if (t->s->progress_bytes_callback) { + t->s->progress_bytes_callback(t->bytes, t->s->progress_opaque); + } } co_put_to_shres(t->s->mem, t->bytes); block_copy_task_end(t, ret); -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-17-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/backup.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/block/backup.c b/block/backup.c index XXXXXXX..XXXXXXX 100644 --- a/block/backup.c +++ b/block/backup.c @@ -XXX,XX +XXX,XX @@ static void backup_init_bcs_bitmap(BackupBlockJob *job) static int coroutine_fn backup_run(Job *job, Error **errp) { BackupBlockJob *s = container_of(job, BackupBlockJob, common.job); - int ret = 0; + int ret; backup_init_bcs_bitmap(s); @@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_run(Job *job, Error **errp) for (offset = 0; offset < s->len; ) { if (yield_and_check(s)) { - ret = -ECANCELED; - goto out; + return -ECANCELED; } ret = block_copy_reset_unallocated(s->bcs, offset, &count); if (ret < 0) { - goto out; + return ret; } offset += count; @@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_run(Job *job, Error **errp) job_yield(job); } } else { - ret = backup_loop(s); + return backup_loop(s); } - out: - return ret; + return 0; } static const BlockJobDriver backup_job_driver = { -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> This brings async request handling and block-status driven chunk sizes to backup out of the box, which improves backup performance. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-18-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/backup.c | 187 +++++++++++++++++++++++++++++++------------------ 1 file changed, 120 insertions(+), 67 deletions(-) diff --git a/block/backup.c b/block/backup.c index XXXXXXX..XXXXXXX 100644 --- a/block/backup.c +++ b/block/backup.c @@ -XXX,XX +XXX,XX @@ #include "block/block-copy.h" #include "qapi/error.h" #include "qapi/qmp/qerror.h" -#include "qemu/ratelimit.h" #include "qemu/cutils.h" #include "sysemu/block-backend.h" #include "qemu/bitmap.h" @@ -XXX,XX +XXX,XX @@ typedef struct BackupBlockJob { BlockdevOnError on_source_error; BlockdevOnError on_target_error; uint64_t len; - uint64_t bytes_read; int64_t cluster_size; BackupPerf perf; BlockCopyState *bcs; + + bool wait; + BlockCopyCallState *bg_bcs_call; } BackupBlockJob; static const BlockJobDriver backup_job_driver; -static void backup_progress_bytes_callback(int64_t bytes, void *opaque) -{ - BackupBlockJob *s = opaque; - - s->bytes_read += bytes; -} - -static int coroutine_fn backup_do_cow(BackupBlockJob *job, - int64_t offset, uint64_t bytes, - bool *error_is_read) -{ - int ret = 0; - int64_t start, end; /* bytes */ - - start = QEMU_ALIGN_DOWN(offset, job->cluster_size); - end = QEMU_ALIGN_UP(bytes + offset, job->cluster_size); - - trace_backup_do_cow_enter(job, start, offset, bytes); - - ret = block_copy(job->bcs, start, end - start, true, error_is_read); - - trace_backup_do_cow_return(job, offset, bytes, ret); - - return ret; -} - static void backup_cleanup_sync_bitmap(BackupBlockJob *job, int ret) { BdrvDirtyBitmap *bm; @@ -XXX,XX +XXX,XX @@ static BlockErrorAction backup_error_action(BackupBlockJob *job, } } -static bool coroutine_fn yield_and_check(BackupBlockJob *job) +static void coroutine_fn backup_block_copy_callback(void *opaque) { - uint64_t delay_ns; - - if (job_is_cancelled(&job->common.job)) { - return true; - } - - /* - * We need to yield even for delay_ns = 0 so that bdrv_drain_all() can - * return. Without a yield, the VM would not reboot. - */ - delay_ns = block_job_ratelimit_get_delay(&job->common, job->bytes_read); - job->bytes_read = 0; - job_sleep_ns(&job->common.job, delay_ns); + BackupBlockJob *s = opaque; - if (job_is_cancelled(&job->common.job)) { - return true; + if (s->wait) { + s->wait = false; + aio_co_wake(s->common.job.co); + } else { + job_enter(&s->common.job); } - - return false; } static int coroutine_fn backup_loop(BackupBlockJob *job) { - bool error_is_read; - int64_t offset; - BdrvDirtyBitmapIter *bdbi; + BlockCopyCallState *s = NULL; int ret = 0; + bool error_is_read; + BlockErrorAction act; + + while (true) { /* retry loop */ + job->bg_bcs_call = s = block_copy_async(job->bcs, 0, + QEMU_ALIGN_UP(job->len, job->cluster_size), + job->perf.max_workers, job->perf.max_chunk, + backup_block_copy_callback, job); + + while (!block_copy_call_finished(s) && + !job_is_cancelled(&job->common.job)) + { + job_yield(&job->common.job); + } - bdbi = bdrv_dirty_iter_new(block_copy_dirty_bitmap(job->bcs)); - while ((offset = bdrv_dirty_iter_next(bdbi)) != -1) { - do { - if (yield_and_check(job)) { - goto out; - } - ret = backup_do_cow(job, offset, job->cluster_size, &error_is_read); - if (ret < 0 && backup_error_action(job, error_is_read, -ret) == - BLOCK_ERROR_ACTION_REPORT) - { - goto out; - } - } while (ret < 0); + if (!block_copy_call_finished(s)) { + assert(job_is_cancelled(&job->common.job)); + /* + * Note that we can't use job_yield() here, as it doesn't work for + * cancelled job. + */ + block_copy_call_cancel(s); + job->wait = true; + qemu_coroutine_yield(); + assert(block_copy_call_finished(s)); + ret = 0; + goto out; + } + + if (job_is_cancelled(&job->common.job) || + block_copy_call_succeeded(s)) + { + ret = 0; + goto out; + } + + if (block_copy_call_cancelled(s)) { + /* + * Job is not cancelled but only block-copy call. This is possible + * after job pause. Now the pause is finished, start new block-copy + * iteration. + */ + block_copy_call_free(s); + continue; + } + + /* The only remaining case is failed block-copy call. */ + assert(block_copy_call_failed(s)); + + ret = block_copy_call_status(s, &error_is_read); + act = backup_error_action(job, error_is_read, -ret); + switch (act) { + case BLOCK_ERROR_ACTION_REPORT: + goto out; + case BLOCK_ERROR_ACTION_STOP: + /* + * Go to pause prior to starting new block-copy call on the next + * iteration. + */ + job_pause_point(&job->common.job); + break; + case BLOCK_ERROR_ACTION_IGNORE: + /* Proceed to new block-copy call to retry. */ + break; + default: + abort(); + } + + block_copy_call_free(s); } - out: - bdrv_dirty_iter_free(bdbi); +out: + block_copy_call_free(s); + job->bg_bcs_call = NULL; return ret; } @@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_run(Job *job, Error **errp) int64_t count; for (offset = 0; offset < s->len; ) { - if (yield_and_check(s)) { + if (job_is_cancelled(job)) { + return -ECANCELED; + } + + job_pause_point(job); + + if (job_is_cancelled(job)) { return -ECANCELED; } @@ -XXX,XX +XXX,XX @@ static int coroutine_fn backup_run(Job *job, Error **errp) return 0; } +static void coroutine_fn backup_pause(Job *job) +{ + BackupBlockJob *s = container_of(job, BackupBlockJob, common.job); + + if (s->bg_bcs_call && !block_copy_call_finished(s->bg_bcs_call)) { + block_copy_call_cancel(s->bg_bcs_call); + s->wait = true; + qemu_coroutine_yield(); + } +} + +static void coroutine_fn backup_set_speed(BlockJob *job, int64_t speed) +{ + BackupBlockJob *s = container_of(job, BackupBlockJob, common); + + /* + * block_job_set_speed() is called first from block_job_create(), when we + * don't yet have s->bcs. + */ + if (s->bcs) { + block_copy_set_speed(s->bcs, speed); + if (s->bg_bcs_call) { + block_copy_kick(s->bg_bcs_call); + } + } +} + static const BlockJobDriver backup_job_driver = { .job_driver = { .instance_size = sizeof(BackupBlockJob), @@ -XXX,XX +XXX,XX @@ static const BlockJobDriver backup_job_driver = { .commit = backup_commit, .abort = backup_abort, .clean = backup_clean, - } + .pause = backup_pause, + }, + .set_speed = backup_set_speed, }; static int64_t backup_calculate_cluster_size(BlockDriverState *target, @@ -XXX,XX +XXX,XX @@ BlockJob *backup_job_create(const char *job_id, BlockDriverState *bs, job->len = len; job->perf = *perf; - block_copy_set_progress_callback(bcs, backup_progress_bytes_callback, job); block_copy_set_progress_meter(bcs, &job->common.job.progress); + block_copy_set_speed(bcs, speed); /* Required permissions are already taken by backup-top target */ block_job_add_bdrv(&job->common, "target", target, 0, BLK_PERM_ALL, -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Further commit will add a benchmark (scripts/simplebench/bench-backup.py), which will show that backup works better with async parallel requests (previous commit) and disabled copy_range. So, let's disable copy_range by default. Note: the option was added several commits ago with default to true, to follow old behavior (the feature was enabled unconditionally), and only now we are going to change the default behavior. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-19-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- qapi/block-core.json | 2 +- blockdev.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qapi/block-core.json b/qapi/block-core.json index XXXXXXX..XXXXXXX 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -XXX,XX +XXX,XX @@ # Optional parameters for backup. These parameters don't affect # functionality, but may significantly affect performance. # -# @use-copy-range: Use copy offloading. Default true. +# @use-copy-range: Use copy offloading. Default false. # # @max-workers: Maximum number of parallel requests for the sustained background # copying process. Doesn't influence copy-before-write operations. diff --git a/blockdev.c b/blockdev.c index XXXXXXX..XXXXXXX 100644 --- a/blockdev.c +++ b/blockdev.c @@ -XXX,XX +XXX,XX @@ static BlockJob *do_backup_common(BackupCommon *backup, { BlockJob *job = NULL; BdrvDirtyBitmap *bmap = NULL; - BackupPerf perf = { .use_copy_range = true, .max_workers = 64 }; + BackupPerf perf = { .max_workers = 64 }; int job_flags = JOB_DEFAULT; if (!backup->has_speed) { -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Drop unused code. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-20-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- include/block/block-copy.h | 6 ------ block/block-copy.c | 15 --------------- 2 files changed, 21 deletions(-) diff --git a/include/block/block-copy.h b/include/block/block-copy.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block-copy.h +++ b/include/block/block-copy.h @@ -XXX,XX +XXX,XX @@ #include "block/block.h" #include "qemu/co-shared-resource.h" -typedef void (*ProgressBytesCallbackFunc)(int64_t bytes, void *opaque); typedef void (*BlockCopyAsyncCallbackFunc)(void *opaque); typedef struct BlockCopyState BlockCopyState; typedef struct BlockCopyCallState BlockCopyCallState; @@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target, BdrvRequestFlags write_flags, Error **errp); -void block_copy_set_progress_callback( - BlockCopyState *s, - ProgressBytesCallbackFunc progress_bytes_callback, - void *progress_opaque); - void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm); void block_copy_state_free(BlockCopyState *s); diff --git a/block/block-copy.c b/block/block-copy.c index XXXXXXX..XXXXXXX 100644 --- a/block/block-copy.c +++ b/block/block-copy.c @@ -XXX,XX +XXX,XX @@ typedef struct BlockCopyState { bool skip_unallocated; ProgressMeter *progress; - /* progress_bytes_callback: called when some copying progress is done. */ - ProgressBytesCallbackFunc progress_bytes_callback; - void *progress_opaque; SharedResource *mem; @@ -XXX,XX +XXX,XX @@ BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target, return s; } -void block_copy_set_progress_callback( - BlockCopyState *s, - ProgressBytesCallbackFunc progress_bytes_callback, - void *progress_opaque) -{ - s->progress_bytes_callback = progress_bytes_callback; - s->progress_opaque = progress_opaque; -} - void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm) { s->progress = pm; @@ -XXX,XX +XXX,XX @@ static coroutine_fn int block_copy_task_entry(AioTask *task) t->call_state->error_is_read = error_is_read; } else { progress_work_done(t->s->progress, t->bytes); - if (t->s->progress_bytes_callback) { - t->s->progress_bytes_callback(t->bytes, t->s->progress_opaque); - } } co_put_to_shres(t->s->mem, t->bytes); block_copy_task_end(t, ret); -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-21-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- include/block/block-copy.h | 2 +- block/backup-top.c | 2 +- block/block-copy.c | 10 ++-------- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/include/block/block-copy.h b/include/block/block-copy.h index XXXXXXX..XXXXXXX 100644 --- a/include/block/block-copy.h +++ b/include/block/block-copy.h @@ -XXX,XX +XXX,XX @@ int64_t block_copy_reset_unallocated(BlockCopyState *s, int64_t offset, int64_t *count); int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes, - bool ignore_ratelimit, bool *error_is_read); + bool ignore_ratelimit); /* * Run block-copy in a coroutine, create corresponding BlockCopyCallState diff --git a/block/backup-top.c b/block/backup-top.c index XXXXXXX..XXXXXXX 100644 --- a/block/backup-top.c +++ b/block/backup-top.c @@ -XXX,XX +XXX,XX @@ static coroutine_fn int backup_top_cbw(BlockDriverState *bs, uint64_t offset, off = QEMU_ALIGN_DOWN(offset, s->cluster_size); end = QEMU_ALIGN_UP(offset + bytes, s->cluster_size); - return block_copy(s->bcs, off, end - off, true, NULL); + return block_copy(s->bcs, off, end - off, true); } static int coroutine_fn backup_top_co_pdiscard(BlockDriverState *bs, diff --git a/block/block-copy.c b/block/block-copy.c index XXXXXXX..XXXXXXX 100644 --- a/block/block-copy.c +++ b/block/block-copy.c @@ -XXX,XX +XXX,XX @@ static int coroutine_fn block_copy_common(BlockCopyCallState *call_state) } int coroutine_fn block_copy(BlockCopyState *s, int64_t start, int64_t bytes, - bool ignore_ratelimit, bool *error_is_read) + bool ignore_ratelimit) { BlockCopyCallState call_state = { .s = s, @@ -XXX,XX +XXX,XX @@ int coroutine_fn block_copy(BlockCopyState *s, int64_t start, int64_t bytes, .max_workers = BLOCK_COPY_MAX_WORKERS, }; - int ret = block_copy_common(&call_state); - - if (error_is_read && ret < 0) { - *error_is_read = call_state.error_is_read; - } - - return ret; + return block_copy_common(&call_state); } static void coroutine_fn block_copy_async_co_entry(void *opaque) -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-22-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- scripts/simplebench/bench_block_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/simplebench/bench_block_job.py b/scripts/simplebench/bench_block_job.py index XXXXXXX..XXXXXXX 100755 --- a/scripts/simplebench/bench_block_job.py +++ b/scripts/simplebench/bench_block_job.py @@ -XXX,XX +XXX,XX @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Benchmark block jobs # -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Add argument to allow additional block-job options. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Reviewed-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210116214705.822267-23-vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- scripts/simplebench/bench-example.py | 2 +- scripts/simplebench/bench_block_job.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/simplebench/bench-example.py b/scripts/simplebench/bench-example.py index XXXXXXX..XXXXXXX 100644 --- a/scripts/simplebench/bench-example.py +++ b/scripts/simplebench/bench-example.py @@ -XXX,XX +XXX,XX @@ from bench_block_job import bench_block_copy, drv_file, drv_nbd def bench_func(env, case): """ Handle one "cell" of benchmarking table. """ - return bench_block_copy(env['qemu_binary'], env['cmd'], + return bench_block_copy(env['qemu_binary'], env['cmd'], {} case['source'], case['target']) diff --git a/scripts/simplebench/bench_block_job.py b/scripts/simplebench/bench_block_job.py index XXXXXXX..XXXXXXX 100755 --- a/scripts/simplebench/bench_block_job.py +++ b/scripts/simplebench/bench_block_job.py @@ -XXX,XX +XXX,XX @@ def bench_block_job(cmd, cmd_args, qemu_args): # Bench backup or mirror -def bench_block_copy(qemu_binary, cmd, source, target): +def bench_block_copy(qemu_binary, cmd, cmd_options, source, target): """Helper to run bench_block_job() for mirror or backup""" assert cmd in ('blockdev-backup', 'blockdev-mirror') source['node-name'] = 'source' target['node-name'] = 'target' - return bench_block_job(cmd, - {'job-id': 'job0', 'device': 'source', - 'target': 'target', 'sync': 'full'}, + cmd_options['job-id'] = 'job0' + cmd_options['device'] = 'source' + cmd_options['target'] = 'target' + cmd_options['sync'] = 'full' + + return bench_block_job(cmd, cmd_options, [qemu_binary, '-blockdev', json.dumps(source), '-blockdev', json.dumps(target)]) -- 2.29.2 From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Add script to benchmark new backup architecture. Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Message-Id: <20210116214705.822267-24-vsementsov@virtuozzo.com> [mreitz: s/not unsupported/not supported/] Signed-off-by: Max Reitz <mreitz@redhat.com> --- scripts/simplebench/bench-backup.py | 167 ++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100755 scripts/simplebench/bench-backup.py diff --git a/scripts/simplebench/bench-backup.py b/scripts/simplebench/bench-backup.py new file mode 100755 index XXXXXXX..XXXXXXX --- /dev/null +++ b/scripts/simplebench/bench-backup.py @@ -XXX,XX +XXX,XX @@ +#!/usr/bin/env python3 +# +# Bench backup block-job +# +# Copyright (c) 2020 Virtuozzo International GmbH. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +import argparse +import json + +import simplebench +from results_to_text import results_to_text +from bench_block_job import bench_block_copy, drv_file, drv_nbd + + +def bench_func(env, case): + """ Handle one "cell" of benchmarking table. """ + cmd_options = env['cmd-options'] if 'cmd-options' in env else {} + return bench_block_copy(env['qemu-binary'], env['cmd'], + cmd_options, + case['source'], case['target']) + + +def bench(args): + test_cases = [] + + sources = {} + targets = {} + for d in args.dir: + label, path = d.split(':') # paths with colon not supported + sources[label] = drv_file(path + '/test-source') + targets[label] = drv_file(path + '/test-target') + + if args.nbd: + nbd = args.nbd.split(':') + host = nbd[0] + port = '10809' if len(nbd) == 1 else nbd[1] + drv = drv_nbd(host, port) + sources['nbd'] = drv + targets['nbd'] = drv + + for t in args.test: + src, dst = t.split(':') + + test_cases.append({ + 'id': t, + 'source': sources[src], + 'target': targets[dst] + }) + + binaries = [] # list of (<label>, <path>, [<options>]) + for i, q in enumerate(args.env): + name_path = q.split(':') + if len(name_path) == 1: + label = f'q{i}' + path_opts = name_path[0].split(',') + else: + assert len(name_path) == 2 # paths with colon not supported + label = name_path[0] + path_opts = name_path[1].split(',') + + binaries.append((label, path_opts[0], path_opts[1:])) + + test_envs = [] + + bin_paths = {} + for i, q in enumerate(args.env): + opts = q.split(',') + label_path = opts[0] + opts = opts[1:] + + if ':' in label_path: + # path with colon inside is not supported + label, path = label_path.split(':') + bin_paths[label] = path + elif label_path in bin_paths: + label = label_path + path = bin_paths[label] + else: + path = label_path + label = f'q{i}' + bin_paths[label] = path + + x_perf = {} + is_mirror = False + for opt in opts: + if opt == 'mirror': + is_mirror = True + elif opt == 'copy-range=on': + x_perf['use-copy-range'] = True + elif opt == 'copy-range=off': + x_perf['use-copy-range'] = False + elif opt.startswith('max-workers='): + x_perf['max-workers'] = int(opt.split('=')[1]) + + if is_mirror: + assert not x_perf + test_envs.append({ + 'id': f'mirror({label})', + 'cmd': 'blockdev-mirror', + 'qemu-binary': path + }) + else: + test_envs.append({ + 'id': f'backup({label})\n' + '\n'.join(opts), + 'cmd': 'blockdev-backup', + 'cmd-options': {'x-perf': x_perf} if x_perf else {}, + 'qemu-binary': path + }) + + result = simplebench.bench(bench_func, test_envs, test_cases, count=3) + with open('results.json', 'w') as f: + json.dump(result, f, indent=4) + print(results_to_text(result)) + + +class ExtendAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + items = getattr(namespace, self.dest) or [] + items.extend(values) + setattr(namespace, self.dest, items) + + +if __name__ == '__main__': + p = argparse.ArgumentParser('Backup benchmark', epilog=''' +ENV format + + (LABEL:PATH|LABEL|PATH)[,max-workers=N][,use-copy-range=(on|off)][,mirror] + + LABEL short name for the binary + PATH path to the binary + max-workers set x-perf.max-workers of backup job + use-copy-range set x-perf.use-copy-range of backup job + mirror use mirror job instead of backup''', + formatter_class=argparse.RawTextHelpFormatter) + p.add_argument('--env', nargs='+', help='''\ +Qemu binaries with labels and options, see below +"ENV format" section''', + action=ExtendAction) + p.add_argument('--dir', nargs='+', help='''\ +Directories, each containing "test-source" and/or +"test-target" files, raw images to used in +benchmarking. File path with label, like +label:/path/to/directory''', + action=ExtendAction) + p.add_argument('--nbd', help='''\ +host:port for remote NBD image, (or just host, for +default port 10809). Use it in tests, label is "nbd" +(but you cannot create test nbd:nbd).''') + p.add_argument('--test', nargs='+', help='''\ +Tests, in form source-dir-label:target-dir-label''', + action=ExtendAction) + + bench(p.parse_args()) -- 2.29.2 From: David Edmondson <david.edmondson@oracle.com> When a call to fcntl(2) for the purpose of adding file locks fails with an error other than EAGAIN or EACCES, report the error returned by fcntl. EAGAIN or EACCES are elided as they are considered to be common failures, indicating that a conflicting lock is held by another process. No errors are elided when removing file locks. Signed-off-by: David Edmondson <david.edmondson@oracle.com> Message-Id: <20210113164447.2545785-1-david.edmondson@oracle.com> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> Signed-off-by: Max Reitz <mreitz@redhat.com> --- block/file-posix.c | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/block/file-posix.c b/block/file-posix.c index XXXXXXX..XXXXXXX 100644 --- a/block/file-posix.c +++ b/block/file-posix.c @@ -XXX,XX +XXX,XX @@ typedef struct RawPosixAIOData { static int cdrom_reopen(BlockDriverState *bs); #endif +/* + * Elide EAGAIN and EACCES details when failing to lock, as this + * indicates that the specified file region is already locked by + * another process, which is considered a common scenario. + */ +#define raw_lock_error_setg_errno(errp, err, fmt, ...) \ + do { \ + if ((err) == EAGAIN || (err) == EACCES) { \ + error_setg((errp), (fmt), ## __VA_ARGS__); \ + } else { \ + error_setg_errno((errp), (err), (fmt), ## __VA_ARGS__); \ + } \ + } while (0) + #if defined(__NetBSD__) static int raw_normalize_devicepath(const char **filename, Error **errp) { @@ -XXX,XX +XXX,XX @@ static int raw_apply_lock_bytes(BDRVRawState *s, int fd, if ((perm_lock_bits & bit) && !(locked_perm & bit)) { ret = qemu_lock_fd(fd, off, 1, false); if (ret) { - error_setg(errp, "Failed to lock byte %d", off); + raw_lock_error_setg_errno(errp, -ret, "Failed to lock byte %d", + off); return ret; } else if (s) { s->locked_perm |= bit; @@ -XXX,XX +XXX,XX @@ static int raw_apply_lock_bytes(BDRVRawState *s, int fd, } else if (unlock && (locked_perm & bit) && !(perm_lock_bits & bit)) { ret = qemu_unlock_fd(fd, off, 1); if (ret) { - error_setg(errp, "Failed to unlock byte %d", off); + error_setg_errno(errp, -ret, "Failed to unlock byte %d", off); return ret; } else if (s) { s->locked_perm &= ~bit; @@ -XXX,XX +XXX,XX @@ static int raw_apply_lock_bytes(BDRVRawState *s, int fd, if ((shared_perm_lock_bits & bit) && !(locked_shared_perm & bit)) { ret = qemu_lock_fd(fd, off, 1, false); if (ret) { - error_setg(errp, "Failed to lock byte %d", off); + raw_lock_error_setg_errno(errp, -ret, "Failed to lock byte %d", + off); return ret; } else if (s) { s->locked_shared_perm |= bit; @@ -XXX,XX +XXX,XX @@ static int raw_apply_lock_bytes(BDRVRawState *s, int fd, !(shared_perm_lock_bits & bit)) { ret = qemu_unlock_fd(fd, off, 1); if (ret) { - error_setg(errp, "Failed to unlock byte %d", off); + error_setg_errno(errp, -ret, "Failed to unlock byte %d", off); return ret; } else if (s) { s->locked_shared_perm &= ~bit; @@ -XXX,XX +XXX,XX @@ static int raw_check_lock_bytes(int fd, uint64_t perm, uint64_t shared_perm, ret = qemu_lock_fd_test(fd, off, 1, true); if (ret) { char *perm_name = bdrv_perm_names(p); - error_setg(errp, - "Failed to get \"%s\" lock", - perm_name); + + raw_lock_error_setg_errno(errp, -ret, + "Failed to get \"%s\" lock", + perm_name); g_free(perm_name); return ret; } @@ -XXX,XX +XXX,XX @@ static int raw_check_lock_bytes(int fd, uint64_t perm, uint64_t shared_perm, ret = qemu_lock_fd_test(fd, off, 1, true); if (ret) { char *perm_name = bdrv_perm_names(p); - error_setg(errp, - "Failed to get shared \"%s\" lock", - perm_name); + + raw_lock_error_setg_errno(errp, -ret, + "Failed to get shared \"%s\" lock", + perm_name); g_free(perm_name); return ret; } -- 2.29.2 From: Alberto Garcia <berto@igalia.com> Signed-off-by: Alberto Garcia <berto@igalia.com> Suggested-by: Maxim Levitsky <mlevitsk@redhat.com> Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com> Message-Id: <20210112170540.2912-1-berto@igalia.com> [mreitz: Add "# group:" line] Signed-off-by: Max Reitz <mreitz@redhat.com> --- tests/qemu-iotests/313 | 104 +++++++++++++++++++++++++++++++++++++ tests/qemu-iotests/313.out | 29 +++++++++++ tests/qemu-iotests/group | 1 + 3 files changed, 134 insertions(+) create mode 100755 tests/qemu-iotests/313 create mode 100644 tests/qemu-iotests/313.out diff --git a/tests/qemu-iotests/313 b/tests/qemu-iotests/313 new file mode 100755 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/qemu-iotests/313 @@ -XXX,XX +XXX,XX @@ +#!/usr/bin/env bash +# group: rw auto quick +# +# Test for the regression fixed in commit c8bf9a9169 +# +# Copyright (C) 2020 Igalia, S.L. +# Author: Alberto Garcia <berto@igalia.com> +# Based on a test case by Maxim Levitsky <mlevitsk@redhat.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +# creator +owner=berto@igalia.com + +seq=`basename $0` +echo "QA output created by $seq" + +status=1 # failure is the default! + +_cleanup() +{ + _cleanup_test_img +} +trap "_cleanup; exit \$status" 0 1 2 3 15 + +# get standard environment, filters and checks +. ./common.rc +. ./common.filter + +_supported_fmt qcow2 +_supported_proto file +_supported_os Linux +_unsupported_imgopts cluster_size refcount_bits extended_l2 compat=0.10 data_file + +# The cluster size must be at least the granularity of the mirror job (4KB) +# Note that larger cluster sizes will produce very large images (several GBs) +cluster_size=4096 +refcount_bits=64 # Make it equal to the L2 entry size for convenience +options="cluster_size=${cluster_size},refcount_bits=${refcount_bits}" + +# Number of refcount entries per refcount blocks +ref_entries=$(( ${cluster_size} * 8 / ${refcount_bits} )) + +# Number of data clusters needed to fill a refcount block +# Equals ${ref_entries} minus two (one L2 table and one refcount block) +data_clusters_per_refblock=$(( ${ref_entries} - 2 )) + +# Number of entries in the refcount cache +ref_blocks=4 + +# Write enough data clusters to fill the refcount cache and allocate +# one more refcount block. +# Subtract 3 clusters from the total: qcow2 header, refcount table, L1 table +total_data_clusters=$(( ${data_clusters_per_refblock} * ${ref_blocks} + 1 - 3 )) + +# Total size to write in bytes +total_size=$(( ${total_data_clusters} * ${cluster_size} )) + +echo +echo '### Create the image' +echo +TEST_IMG_FILE=$TEST_IMG.base _make_test_img -o $options $total_size | _filter_img_create_size + +echo +echo '### Write data to allocate more refcount blocks than the cache can hold' +echo +$QEMU_IO -c "write -P 1 0 $total_size" $TEST_IMG.base | _filter_qemu_io + +echo +echo '### Create an overlay' +echo +_make_test_img -F $IMGFMT -b $TEST_IMG.base -o $options | _filter_img_create_size + +echo +echo '### Fill the overlay with zeroes' +echo +$QEMU_IO -c "write -z 0 $total_size" $TEST_IMG | _filter_qemu_io + +echo +echo '### Commit changes to the base image' +echo +$QEMU_IMG commit $TEST_IMG + +echo +echo '### Check the base image' +echo +$QEMU_IMG check $TEST_IMG.base + +# success, all done +echo "*** done" +rm -f $seq.full +status=0 diff --git a/tests/qemu-iotests/313.out b/tests/qemu-iotests/313.out new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/qemu-iotests/313.out @@ -XXX,XX +XXX,XX @@ +QA output created by 313 + +### Create the image + +Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=SIZE + +### Write data to allocate more refcount blocks than the cache can hold + +wrote 8347648/8347648 bytes at offset 0 +7.961 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +### Create an overlay + +Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=SIZE backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT + +### Fill the overlay with zeroes + +wrote 8347648/8347648 bytes at offset 0 +7.961 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) + +### Commit changes to the base image + +Image committed. + +### Check the base image + +No errors were found on the image. +Image end offset: 8396800 +*** done diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/group +++ b/tests/qemu-iotests/group @@ -XXX,XX +XXX,XX @@ 309 rw auto quick 310 rw quick 312 rw quick +313 rw auto quick -- 2.29.2 Commit 0afec75734331 removed the 'change' QMP command, so we can no longer test it in 118. Fixes: 0afec75734331a0b52fa3aa4235220eda8c7846f ('qmp: remove deprecated "change" command') Signed-off-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210126104833.57026-1-mreitz@redhat.com> Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com> --- tests/qemu-iotests/118 | 20 +------------------- tests/qemu-iotests/118.out | 4 ++-- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/tests/qemu-iotests/118 b/tests/qemu-iotests/118 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/118 +++ b/tests/qemu-iotests/118 @@ -XXX,XX +XXX,XX @@ #!/usr/bin/env python3 # group: rw # -# Test case for the QMP 'change' command and all other associated -# commands +# Test case for media change monitor commands # # Copyright (C) 2015 Red Hat, Inc. # @@ -XXX,XX +XXX,XX @@ class ChangeBaseClass(iotests.QMPTestCase): class GeneralChangeTestsBaseClass(ChangeBaseClass): - def test_change(self): - # 'change' requires a drive name, so skip the test for blockdev - if not self.use_drive: - return - - result = self.vm.qmp('change', device='drive0', target=new_img, - arg=iotests.imgfmt) - self.assert_qmp(result, 'return', {}) - - self.wait_for_open() - self.wait_for_close() - - result = self.vm.qmp('query-block') - if self.has_real_tray: - self.assert_qmp(result, 'return[0]/tray_open', False) - self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img) - def test_blockdev_change_medium(self): result = self.vm.qmp('blockdev-change-medium', id=self.device_name, filename=new_img, diff --git a/tests/qemu-iotests/118.out b/tests/qemu-iotests/118.out index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/118.out +++ b/tests/qemu-iotests/118.out @@ -XXX,XX +XXX,XX @@ -....................................................................................................................................................................... +........................................................................................................................................................... ---------------------------------------------------------------------- -Ran 167 tests +Ran 155 tests OK -- 2.29.2 ccd3b3b8112 has deprecated short-hand boolean options (i.e., options with values). All options without values are interpreted as boolean options, so this includes the invalid option "snapshot.foo" used in iotest 178. So after ccd3b3b8112, 178 fails with: +qemu-img: warning: short-form boolean option 'snapshot.foo' deprecated +Please use snapshot.foo=on instead Suppress that deprecation warning by passing some value to it (it does not matter which, because the option is invalid anyway). Fixes: ccd3b3b8112b670fdccf8a392b8419b173ffccb4 ("qemu-option: warn for short-form boolean options") Signed-off-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210126123834.115915-1-mreitz@redhat.com> --- tests/qemu-iotests/178 | 2 +- tests/qemu-iotests/178.out.qcow2 | 2 +- tests/qemu-iotests/178.out.raw | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/qemu-iotests/178 b/tests/qemu-iotests/178 index XXXXXXX..XXXXXXX 100755 --- a/tests/qemu-iotests/178 +++ b/tests/qemu-iotests/178 @@ -XXX,XX +XXX,XX @@ $QEMU_IMG measure --image-opts # missing filename $QEMU_IMG measure -f qcow2 # missing filename $QEMU_IMG measure -l snap1 # missing filename $QEMU_IMG measure -o , # invalid option list -$QEMU_IMG measure -l snapshot.foo # invalid snapshot option +$QEMU_IMG measure -l snapshot.foo=bar # invalid snapshot option $QEMU_IMG measure --output foo # invalid output format $QEMU_IMG measure --size -1 # invalid image size $QEMU_IMG measure -O foo "$TEST_IMG" # unknown image file format diff --git a/tests/qemu-iotests/178.out.qcow2 b/tests/qemu-iotests/178.out.qcow2 index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/178.out.qcow2 +++ b/tests/qemu-iotests/178.out.qcow2 @@ -XXX,XX +XXX,XX @@ qemu-img: --image-opts, -f, and -l require a filename argument. qemu-img: --image-opts, -f, and -l require a filename argument. qemu-img: Invalid option list: , qemu-img: Invalid parameter 'snapshot.foo' -qemu-img: Failed in parsing snapshot param 'snapshot.foo' +qemu-img: Failed in parsing snapshot param 'snapshot.foo=bar' qemu-img: --output must be used with human or json as argument. qemu-img: Invalid image size specified. Must be between 0 and 9223372036854775807. qemu-img: Unknown file format 'foo' diff --git a/tests/qemu-iotests/178.out.raw b/tests/qemu-iotests/178.out.raw index XXXXXXX..XXXXXXX 100644 --- a/tests/qemu-iotests/178.out.raw +++ b/tests/qemu-iotests/178.out.raw @@ -XXX,XX +XXX,XX @@ qemu-img: --image-opts, -f, and -l require a filename argument. qemu-img: --image-opts, -f, and -l require a filename argument. qemu-img: Invalid option list: , qemu-img: Invalid parameter 'snapshot.foo' -qemu-img: Failed in parsing snapshot param 'snapshot.foo' +qemu-img: Failed in parsing snapshot param 'snapshot.foo=bar' qemu-img: --output must be used with human or json as argument. qemu-img: Invalid image size specified. Must be between 0 and 9223372036854775807. qemu-img: Unknown file format 'foo' -- 2.29.2