Prove that blockdev-mirror can now result in sparse raw destination
files, regardless of whether the source is raw or qcow2. By making
this a separate test, it was possible to test effects of individual
patches for the various pieces that all have to work together for a
sparse mirror to be successful.
Note that ./check -file produces different job lengths than ./check
-qcow2 (the test uses a filter to normalize); that's because when
deciding how much of the image to be mirrored, the code looks at how
much of the source image was allocated (for qcow2, this is only the
written clusters; for raw, it is the entire file). But the important
part is that the destination file ends up smaller than 3M, rather than
the 20M it used to be before this patch series.
Signed-off-by: Eric Blake <eblake@redhat.com>
Message-ID: <20250509204341.3553601-28-eblake@redhat.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
---
tests/qemu-iotests/tests/mirror-sparse | 125 +++++++
tests/qemu-iotests/tests/mirror-sparse.out | 365 +++++++++++++++++++++
2 files changed, 490 insertions(+)
create mode 100755 tests/qemu-iotests/tests/mirror-sparse
create mode 100644 tests/qemu-iotests/tests/mirror-sparse.out
diff --git a/tests/qemu-iotests/tests/mirror-sparse b/tests/qemu-iotests/tests/mirror-sparse
new file mode 100755
index 00000000000..8c52a4e2448
--- /dev/null
+++ b/tests/qemu-iotests/tests/mirror-sparse
@@ -0,0 +1,125 @@
+#!/usr/bin/env bash
+# group: rw auto quick
+#
+# Test blockdev-mirror with raw sparse destination
+#
+# Copyright (C) 2025 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+status=1 # failure is the default!
+
+_cleanup()
+{
+ _cleanup_test_img
+ _cleanup_qemu
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+cd ..
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt qcow2 raw # Format of the source. dst is always raw file
+_supported_proto file
+_supported_os Linux
+
+echo
+echo "=== Initial image setup ==="
+echo
+
+TEST_IMG="$TEST_IMG.base" _make_test_img 20M
+$QEMU_IO -c 'w 8M 2M' -f $IMGFMT "$TEST_IMG.base" | _filter_qemu_io
+
+_launch_qemu \
+ -blockdev '{"driver":"file", "cache":{"direct":true, "no-flush":false},
+ "filename":"'"$TEST_IMG.base"'", "node-name":"src-file"}' \
+ -blockdev '{"driver":"'$IMGFMT'", "node-name":"src", "file":"src-file"}'
+h1=$QEMU_HANDLE
+_send_qemu_cmd $h1 '{"execute": "qmp_capabilities"}' 'return'
+
+# Check several combinations; most should result in a sparse destination;
+# the destination should only be fully allocated if pre-allocated
+# and not punching holes due to detect-zeroes
+# do_test creation discard zeroes result
+do_test() {
+ creation=$1
+ discard=$2
+ zeroes=$3
+ expected=$4
+
+echo
+echo "=== Testing creation=$creation discard=$discard zeroes=$zeroes ==="
+echo
+
+rm -f $TEST_IMG
+if test $creation = external; then
+ truncate --size=20M $TEST_IMG
+else
+ _send_qemu_cmd $h1 '{"execute": "blockdev-create", "arguments":
+ {"options": {"driver":"file", "filename":"'$TEST_IMG'",
+ "size":'$((20*1024*1024))', "preallocation":"'$creation'"},
+ "job-id":"job1"}}' 'concluded'
+ _send_qemu_cmd $h1 '{"execute": "job-dismiss", "arguments":
+ {"id": "job1"}}' 'return'
+fi
+_send_qemu_cmd $h1 '{"execute": "blockdev-add", "arguments":
+ {"node-name": "dst", "driver":"file",
+ "filename":"'$TEST_IMG'", "aio":"threads",
+ "auto-read-only":true, "discard":"'$discard'",
+ "detect-zeroes":"'$zeroes'"}}' 'return'
+_send_qemu_cmd $h1 '{"execute":"blockdev-mirror", "arguments":
+ {"sync":"full", "device":"src", "target":"dst",
+ "job-id":"job2"}}' 'return'
+_timed_wait_for $h1 '"ready"'
+_send_qemu_cmd $h1 '{"execute": "job-complete", "arguments":
+ {"id":"job2"}}' 'return' \
+ | _filter_block_job_offset | _filter_block_job_len
+_send_qemu_cmd $h1 '{"execute": "blockdev-del", "arguments":
+ {"node-name": "dst"}}' 'return' \
+ | _filter_block_job_offset | _filter_block_job_len
+$QEMU_IMG compare -U -f $IMGFMT -F raw $TEST_IMG.base $TEST_IMG
+result=$(disk_usage $TEST_IMG)
+if test $result -lt $((3*1024*1024)); then
+ actual=sparse
+elif test $result = $((20*1024*1024)); then
+ actual=full
+else
+ actual=unknown
+fi
+echo "Destination is $actual; expected $expected"
+}
+
+do_test external ignore off sparse
+do_test external unmap off sparse
+do_test external unmap unmap sparse
+do_test off ignore off sparse
+do_test off unmap off sparse
+do_test off unmap unmap sparse
+do_test full ignore off full
+do_test full unmap off sparse
+do_test full unmap unmap sparse
+
+_send_qemu_cmd $h1 '{"execute":"quit"}' ''
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/tests/mirror-sparse.out b/tests/qemu-iotests/tests/mirror-sparse.out
new file mode 100644
index 00000000000..2103b891c3f
--- /dev/null
+++ b/tests/qemu-iotests/tests/mirror-sparse.out
@@ -0,0 +1,365 @@
+QA output created by mirror-sparse
+
+=== Initial image setup ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=20971520
+wrote 2097152/2097152 bytes at offset 8388608
+2 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"execute": "qmp_capabilities"}
+{"return": {}}
+
+=== Testing creation=external discard=ignore zeroes=off ===
+
+{"execute": "blockdev-add", "arguments":
+ {"node-name": "dst", "driver":"file",
+ "filename":"TEST_DIR/t.IMGFMT", "aio":"threads",
+ "auto-read-only":true, "discard":"ignore",
+ "detect-zeroes":"off"}}
+{"return": {}}
+{"execute":"blockdev-mirror", "arguments":
+ {"sync":"full", "device":"src", "target":"dst",
+ "job-id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job2"}}
+{"return": {}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "job2"}}
+{"execute": "job-complete", "arguments":
+ {"id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"return": {}}
+{"execute": "blockdev-del", "arguments":
+ {"node-name": "dst"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job2"}}
+{"return": {}}
+Images are identical.
+Destination is sparse; expected sparse
+
+=== Testing creation=external discard=unmap zeroes=off ===
+
+{"execute": "blockdev-add", "arguments":
+ {"node-name": "dst", "driver":"file",
+ "filename":"TEST_DIR/t.IMGFMT", "aio":"threads",
+ "auto-read-only":true, "discard":"unmap",
+ "detect-zeroes":"off"}}
+{"return": {}}
+{"execute":"blockdev-mirror", "arguments":
+ {"sync":"full", "device":"src", "target":"dst",
+ "job-id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job2"}}
+{"return": {}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "job2"}}
+{"execute": "job-complete", "arguments":
+ {"id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"return": {}}
+{"execute": "blockdev-del", "arguments":
+ {"node-name": "dst"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job2"}}
+{"return": {}}
+Images are identical.
+Destination is sparse; expected sparse
+
+=== Testing creation=external discard=unmap zeroes=unmap ===
+
+{"execute": "blockdev-add", "arguments":
+ {"node-name": "dst", "driver":"file",
+ "filename":"TEST_DIR/t.IMGFMT", "aio":"threads",
+ "auto-read-only":true, "discard":"unmap",
+ "detect-zeroes":"unmap"}}
+{"return": {}}
+{"execute":"blockdev-mirror", "arguments":
+ {"sync":"full", "device":"src", "target":"dst",
+ "job-id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job2"}}
+{"return": {}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "job2"}}
+{"execute": "job-complete", "arguments":
+ {"id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"return": {}}
+{"execute": "blockdev-del", "arguments":
+ {"node-name": "dst"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job2"}}
+{"return": {}}
+Images are identical.
+Destination is sparse; expected sparse
+
+=== Testing creation=off discard=ignore zeroes=off ===
+
+{"execute": "blockdev-create", "arguments":
+ {"options": {"driver":"file", "filename":"TEST_DIR/t.IMGFMT",
+ "size":20971520, "preallocation":"off"},
+ "job-id":"job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job1"}}
+{"return": {}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "job1"}}
+{"execute": "job-dismiss", "arguments":
+ {"id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job1"}}
+{"return": {}}
+{"execute": "blockdev-add", "arguments":
+ {"node-name": "dst", "driver":"file",
+ "filename":"TEST_DIR/t.IMGFMT", "aio":"threads",
+ "auto-read-only":true, "discard":"ignore",
+ "detect-zeroes":"off"}}
+{"return": {}}
+{"execute":"blockdev-mirror", "arguments":
+ {"sync":"full", "device":"src", "target":"dst",
+ "job-id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job2"}}
+{"return": {}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "job2"}}
+{"execute": "job-complete", "arguments":
+ {"id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"return": {}}
+{"execute": "blockdev-del", "arguments":
+ {"node-name": "dst"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job2"}}
+{"return": {}}
+Images are identical.
+Destination is sparse; expected sparse
+
+=== Testing creation=off discard=unmap zeroes=off ===
+
+{"execute": "blockdev-create", "arguments":
+ {"options": {"driver":"file", "filename":"TEST_DIR/t.IMGFMT",
+ "size":20971520, "preallocation":"off"},
+ "job-id":"job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job1"}}
+{"return": {}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "job1"}}
+{"execute": "job-dismiss", "arguments":
+ {"id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job1"}}
+{"return": {}}
+{"execute": "blockdev-add", "arguments":
+ {"node-name": "dst", "driver":"file",
+ "filename":"TEST_DIR/t.IMGFMT", "aio":"threads",
+ "auto-read-only":true, "discard":"unmap",
+ "detect-zeroes":"off"}}
+{"return": {}}
+{"execute":"blockdev-mirror", "arguments":
+ {"sync":"full", "device":"src", "target":"dst",
+ "job-id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job2"}}
+{"return": {}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "job2"}}
+{"execute": "job-complete", "arguments":
+ {"id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"return": {}}
+{"execute": "blockdev-del", "arguments":
+ {"node-name": "dst"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job2"}}
+{"return": {}}
+Images are identical.
+Destination is sparse; expected sparse
+
+=== Testing creation=off discard=unmap zeroes=unmap ===
+
+{"execute": "blockdev-create", "arguments":
+ {"options": {"driver":"file", "filename":"TEST_DIR/t.IMGFMT",
+ "size":20971520, "preallocation":"off"},
+ "job-id":"job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job1"}}
+{"return": {}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "job1"}}
+{"execute": "job-dismiss", "arguments":
+ {"id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job1"}}
+{"return": {}}
+{"execute": "blockdev-add", "arguments":
+ {"node-name": "dst", "driver":"file",
+ "filename":"TEST_DIR/t.IMGFMT", "aio":"threads",
+ "auto-read-only":true, "discard":"unmap",
+ "detect-zeroes":"unmap"}}
+{"return": {}}
+{"execute":"blockdev-mirror", "arguments":
+ {"sync":"full", "device":"src", "target":"dst",
+ "job-id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job2"}}
+{"return": {}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "job2"}}
+{"execute": "job-complete", "arguments":
+ {"id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"return": {}}
+{"execute": "blockdev-del", "arguments":
+ {"node-name": "dst"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job2"}}
+{"return": {}}
+Images are identical.
+Destination is sparse; expected sparse
+
+=== Testing creation=full discard=ignore zeroes=off ===
+
+{"execute": "blockdev-create", "arguments":
+ {"options": {"driver":"file", "filename":"TEST_DIR/t.IMGFMT",
+ "size":20971520, "preallocation":"full"},
+ "job-id":"job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job1"}}
+{"return": {}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "job1"}}
+{"execute": "job-dismiss", "arguments":
+ {"id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job1"}}
+{"return": {}}
+{"execute": "blockdev-add", "arguments":
+ {"node-name": "dst", "driver":"file",
+ "filename":"TEST_DIR/t.IMGFMT", "aio":"threads",
+ "auto-read-only":true, "discard":"ignore",
+ "detect-zeroes":"off"}}
+{"return": {}}
+{"execute":"blockdev-mirror", "arguments":
+ {"sync":"full", "device":"src", "target":"dst",
+ "job-id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job2"}}
+{"return": {}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "job2"}}
+{"execute": "job-complete", "arguments":
+ {"id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"return": {}}
+{"execute": "blockdev-del", "arguments":
+ {"node-name": "dst"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job2"}}
+{"return": {}}
+Images are identical.
+Destination is full; expected full
+
+=== Testing creation=full discard=unmap zeroes=off ===
+
+{"execute": "blockdev-create", "arguments":
+ {"options": {"driver":"file", "filename":"TEST_DIR/t.IMGFMT",
+ "size":20971520, "preallocation":"full"},
+ "job-id":"job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job1"}}
+{"return": {}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "job1"}}
+{"execute": "job-dismiss", "arguments":
+ {"id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job1"}}
+{"return": {}}
+{"execute": "blockdev-add", "arguments":
+ {"node-name": "dst", "driver":"file",
+ "filename":"TEST_DIR/t.IMGFMT", "aio":"threads",
+ "auto-read-only":true, "discard":"unmap",
+ "detect-zeroes":"off"}}
+{"return": {}}
+{"execute":"blockdev-mirror", "arguments":
+ {"sync":"full", "device":"src", "target":"dst",
+ "job-id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job2"}}
+{"return": {}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "job2"}}
+{"execute": "job-complete", "arguments":
+ {"id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"return": {}}
+{"execute": "blockdev-del", "arguments":
+ {"node-name": "dst"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job2"}}
+{"return": {}}
+Images are identical.
+Destination is sparse; expected sparse
+
+=== Testing creation=full discard=unmap zeroes=unmap ===
+
+{"execute": "blockdev-create", "arguments":
+ {"options": {"driver":"file", "filename":"TEST_DIR/t.IMGFMT",
+ "size":20971520, "preallocation":"full"},
+ "job-id":"job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job1"}}
+{"return": {}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "job1"}}
+{"execute": "job-dismiss", "arguments":
+ {"id": "job1"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job1"}}
+{"return": {}}
+{"execute": "blockdev-add", "arguments":
+ {"node-name": "dst", "driver":"file",
+ "filename":"TEST_DIR/t.IMGFMT", "aio":"threads",
+ "auto-read-only":true, "discard":"unmap",
+ "detect-zeroes":"unmap"}}
+{"return": {}}
+{"execute":"blockdev-mirror", "arguments":
+ {"sync":"full", "device":"src", "target":"dst",
+ "job-id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job2"}}
+{"return": {}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "ready", "id": "job2"}}
+{"execute": "job-complete", "arguments":
+ {"id":"job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"return": {}}
+{"execute": "blockdev-del", "arguments":
+ {"node-name": "dst"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "waiting", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "pending", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "job2", "len": LEN, "offset": OFFSET, "speed": 0, "type": "mirror"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "concluded", "id": "job2"}}
+{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job2"}}
+{"return": {}}
+Images are identical.
+Destination is sparse; expected sparse
+{"execute":"quit"}
+*** done
--
2.49.0Eric Blake <eblake@redhat.com> writes:
> Prove that blockdev-mirror can now result in sparse raw destination
> files, regardless of whether the source is raw or qcow2. By making
> this a separate test, it was possible to test effects of individual
> patches for the various pieces that all have to work together for a
> sparse mirror to be successful.
>
> Note that ./check -file produces different job lengths than ./check
> -qcow2 (the test uses a filter to normalize); that's because when
> deciding how much of the image to be mirrored, the code looks at how
> much of the source image was allocated (for qcow2, this is only the
> written clusters; for raw, it is the entire file). But the important
> part is that the destination file ends up smaller than 3M, rather than
> the 20M it used to be before this patch series.
>
> Signed-off-by: Eric Blake <eblake@redhat.com>
> Message-ID: <20250509204341.3553601-28-eblake@redhat.com>
> Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
Fails for me:
TAP version 13
# QEMU -- "/work/armbru/qemu/bld/qemu-system-x86_64" -nodefaults -display none -accel qtest
# QEMU_IMG -- "/work/armbru/qemu/bld/qemu-img"
# QEMU_IO -- "/work/armbru/qemu/bld/qemu-io" --cache writeback --aio threads -f qcow2
# QEMU_NBD -- "/work/armbru/qemu/bld/qemu-nbd"
# IMGFMT -- qcow2
# IMGPROTO -- file
# PLATFORM -- Linux/x86_64 dusky 6.12.7-200.fc41.x86_64
# TEST_DIR -- /work/armbru/qemu/bld-x86/scratch
# SOCK_DIR -- /tmp/qemu-iotests-nqettsyq
# GDB_OPTIONS --
# VALGRIND_QEMU --
# PRINT_QEMU_OUTPUT --
#
1..1
# running qcow2 mirror-sparse
not ok qcow2 mirror-sparse
--- /work/armbru/qemu/tests/qemu-iotests/tests/mirror-sparse.out
+++ /work/armbru/qemu/bld-x86/scratch/qcow2-file-mirror-sparse/mirror-sparse.out.bad
@@ -140,7 +140,7 @@
{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job2"}}
{"return": {}}
Images are identical.
-Destination is sparse; expected sparse
+Destination is unknown; expected sparse
=== Testing creation=off discard=unmap zeroes=off ===
@@ -184,7 +184,7 @@
{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job2"}}
{"return": {}}
Images are identical.
-Destination is sparse; expected sparse
+Destination is unknown; expected sparse
=== Testing creation=off discard=unmap zeroes=unmap ===
On Wed, May 28, 2025 at 01:39:25PM +0200, Markus Armbruster wrote:
> Eric Blake <eblake@redhat.com> writes:
>
> > Prove that blockdev-mirror can now result in sparse raw destination
> > files, regardless of whether the source is raw or qcow2. By making
> > this a separate test, it was possible to test effects of individual
> > patches for the various pieces that all have to work together for a
> > sparse mirror to be successful.
> >
> Fails for me:
>
> TAP version 13
> # QEMU -- "/work/armbru/qemu/bld/qemu-system-x86_64" -nodefaults -display none -accel qtest
> # QEMU_IMG -- "/work/armbru/qemu/bld/qemu-img"
> # QEMU_IO -- "/work/armbru/qemu/bld/qemu-io" --cache writeback --aio threads -f qcow2
> # QEMU_NBD -- "/work/armbru/qemu/bld/qemu-nbd"
> # IMGFMT -- qcow2
> # IMGPROTO -- file
> # PLATFORM -- Linux/x86_64 dusky 6.12.7-200.fc41.x86_64
> # TEST_DIR -- /work/armbru/qemu/bld-x86/scratch
Which filesystem is TEST_DIR on?
> # SOCK_DIR -- /tmp/qemu-iotests-nqettsyq
> # GDB_OPTIONS --
> # VALGRIND_QEMU --
> # PRINT_QEMU_OUTPUT --
> #
> 1..1
> # running qcow2 mirror-sparse
> not ok qcow2 mirror-sparse
> --- /work/armbru/qemu/tests/qemu-iotests/tests/mirror-sparse.out
> +++ /work/armbru/qemu/bld-x86/scratch/qcow2-file-mirror-sparse/mirror-sparse.out.bad
> @@ -140,7 +140,7 @@
> {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job2"}}
> {"return": {}}
> Images are identical.
> -Destination is sparse; expected sparse
> +Destination is unknown; expected sparse
> > Looks like the same failure Fiona reported; does this fix it?
> >
> > https://lists.gnu.org/archive/html/qemu-devel/2025-05/msg05567.html
>
> It does not.
Since my patch for Fiona is not working for you, I will tweak it
slightly to output the actual du output (in addition to "unknown") for
the cases where the size is neither -lt 3M or -gt 19M. The test is
dealing with a 20M image containing 2M of data, so du reporting
something in between 3 and 19 is unexpected. Could it be the result
of du reporting smaller numbers on a compressed filesystem? But even
then, my followup series was trying to filter out any filesystem where
-o preallocation=full of 5M results in a du report of < 4M on the
grounds that it would catch compression artifacts as rendering the
test unreliable.
--
Eric Blake, Principal Software Engineer
Red Hat, Inc.
Virtualization: qemu.org | libguestfs.org
Eric Blake <eblake@redhat.com> writes:
> On Wed, May 28, 2025 at 01:39:25PM +0200, Markus Armbruster wrote:
>> Eric Blake <eblake@redhat.com> writes:
>>
>> > Prove that blockdev-mirror can now result in sparse raw destination
>> > files, regardless of whether the source is raw or qcow2. By making
>> > this a separate test, it was possible to test effects of individual
>> > patches for the various pieces that all have to work together for a
>> > sparse mirror to be successful.
>> >
>
>> Fails for me:
>>
>> TAP version 13
>> # QEMU -- "/work/armbru/qemu/bld/qemu-system-x86_64" -nodefaults -display none -accel qtest
>> # QEMU_IMG -- "/work/armbru/qemu/bld/qemu-img"
>> # QEMU_IO -- "/work/armbru/qemu/bld/qemu-io" --cache writeback --aio threads -f qcow2
>> # QEMU_NBD -- "/work/armbru/qemu/bld/qemu-nbd"
>> # IMGFMT -- qcow2
>> # IMGPROTO -- file
>> # PLATFORM -- Linux/x86_64 dusky 6.12.7-200.fc41.x86_64
>> # TEST_DIR -- /work/armbru/qemu/bld-x86/scratch
>
> Which filesystem is TEST_DIR on?
$ findmnt --target /work/armbru/qemu/bld-x86/scratch
TARGET SOURCE FSTYPE OPTIONS
/work /dev/mapper/luks-898519e7-f64e-4a69-8cf9-8532063d8bb9
xfs rw,relatime,seclabel,at
[...]
On Wed, May 28, 2025 at 06:23:58PM +0200, Markus Armbruster wrote:
> >> # TEST_DIR -- /work/armbru/qemu/bld-x86/scratch
> >
> > Which filesystem is TEST_DIR on?
>
> $ findmnt --target /work/armbru/qemu/bld-x86/scratch
> TARGET SOURCE FSTYPE OPTIONS
> /work /dev/mapper/luks-898519e7-f64e-4a69-8cf9-8532063d8bb9
> xfs rw,relatime,seclabel,at
Thanks; I installed a temporary xfs system (a bit easier to do on
Fedora than zfs), with my tweak to show actual du output, and see:
-Destination is sparse; expected sparse
+Destination is unexpected size (3145728); expected sparse
which is exactly at the 3M cutoff. In fact:
$ findmnt --target /mnt/sysroot/
TARGET SOURCE FSTYPE OPTIONS
/mnt/sysroot
/dev/mapper/my--vg-temp
xfs rw,relatime,seclabel,attr2,inode64,logbufs=8,lo
$ qemu-img create -f raw /mnt/sysroot/dir/src.img 20M
Formatting '/mnt/sysroot/dir/src.img', fmt=raw size=20971520
$ qemu-io -c 'w 8M 2M' -f raw /mnt/sysroot/dir/src.img
wrote 2097152/2097152 bytes at offset 8388608
2 MiB, 1 ops; 00.00 sec (704.781 MiB/sec and 352.3905 ops/sec)
$ qemu-img map --output=json -f raw '/mnt/sysroot/dir/src.img'
[{ "start": 0, "length": 4096, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": 0},
{ "start": 4096, "length": 8384512, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": 4096},
{ "start": 8388608, "length": 2097152, "depth": 0, "present": true, "zero": false, "data": true, "compressed": false, "offset": 8388608},
{ "start": 10485760, "length": 10485760, "depth": 0, "present": true, "zero": true, "data": false, "compressed": false, "offset": 10485760}]
$ du --block-size=1 /mnt/sysroot/dir/src.img
3145728 /mnt/sysroot/dir/src.img
So, even though SEEK_HOLE was able to show 4k data at the head + 2M
data at the 8M offset (matching how the file was built), du reports
3702 * 1k sectors or exactly 3M of disk usage.
On the bright side, it's an easy fix: either s/-lt/-le/ or s/3/4/ in
mirror-sparse for the bound of what forms a sparse file (we can easily
explain how even though less than 3M is reported as a hole, disk usage
can plausibly round its answers up and list every megabyte of content
that contains at least some data, while still being an "accurate"
measure of how sparse the file is).
--
Eric Blake, Principal Software Engineer
Red Hat, Inc.
Virtualization: qemu.org | libguestfs.org
On Wed, May 28, 2025 at 01:39:25PM +0200, Markus Armbruster wrote:
> Eric Blake <eblake@redhat.com> writes:
>
> > Prove that blockdev-mirror can now result in sparse raw destination
> > files, regardless of whether the source is raw or qcow2. By making
> > this a separate test, it was possible to test effects of individual
> > patches for the various pieces that all have to work together for a
> > sparse mirror to be successful.
> >
> > Note that ./check -file produces different job lengths than ./check
> > -qcow2 (the test uses a filter to normalize); that's because when
> > deciding how much of the image to be mirrored, the code looks at how
> > much of the source image was allocated (for qcow2, this is only the
> > written clusters; for raw, it is the entire file). But the important
> > part is that the destination file ends up smaller than 3M, rather than
> > the 20M it used to be before this patch series.
> >
> > Signed-off-by: Eric Blake <eblake@redhat.com>
> > Message-ID: <20250509204341.3553601-28-eblake@redhat.com>
> > Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
>
> Fails for me:
>
> @@ -184,7 +184,7 @@
> {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job2"}}
> {"return": {}}
> Images are identical.
> -Destination is sparse; expected sparse
> +Destination is unknown; expected sparse
Looks like the same failure Fiona reported; does this fix it?
https://lists.gnu.org/archive/html/qemu-devel/2025-05/msg05567.html
--
Eric Blake, Principal Software Engineer
Red Hat, Inc.
Virtualization: qemu.org | libguestfs.org
Eric Blake <eblake@redhat.com> writes:
> On Wed, May 28, 2025 at 01:39:25PM +0200, Markus Armbruster wrote:
>> Eric Blake <eblake@redhat.com> writes:
>>
>> > Prove that blockdev-mirror can now result in sparse raw destination
>> > files, regardless of whether the source is raw or qcow2. By making
>> > this a separate test, it was possible to test effects of individual
>> > patches for the various pieces that all have to work together for a
>> > sparse mirror to be successful.
>> >
>> > Note that ./check -file produces different job lengths than ./check
>> > -qcow2 (the test uses a filter to normalize); that's because when
>> > deciding how much of the image to be mirrored, the code looks at how
>> > much of the source image was allocated (for qcow2, this is only the
>> > written clusters; for raw, it is the entire file). But the important
>> > part is that the destination file ends up smaller than 3M, rather than
>> > the 20M it used to be before this patch series.
>> >
>> > Signed-off-by: Eric Blake <eblake@redhat.com>
>> > Message-ID: <20250509204341.3553601-28-eblake@redhat.com>
>> > Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
>>
>> Fails for me:
>>
>
>> @@ -184,7 +184,7 @@
>> {"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "null", "id": "job2"}}
>> {"return": {}}
>> Images are identical.
>> -Destination is sparse; expected sparse
>> +Destination is unknown; expected sparse
>
> Looks like the same failure Fiona reported; does this fix it?
>
> https://lists.gnu.org/archive/html/qemu-devel/2025-05/msg05567.html
It does not.
Hi, Am 15.05.25 um 04:28 schrieb Eric Blake: > +do_test full ignore off full for me, this one fails irregularly (more than 50% of the time) with: -Destination is full; expected full +Destination is unknown; expected full My file system is ext4 inside a virtual machine. > [I] febner@dev8 ~/repos/qemu/build/tests/qemu-iotests (master) [1]> findmnt --target . > TARGET SOURCE FSTYPE OPTIONS > /home/febner/repos /dev/sdb ext4 rw,relatime The virtual disk is an LVM image. When the test fails, I consistently get 20975616 bytes rather than 20971520 bytes as the result of disk_usage(), i.e. it has one 4 KiB block more. ls -l will still report 20971520 bytes as the length, qemu-img compare succeeds. After the image creation, i.e. job1, the allocation is still as expected: > +20971520 /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw > +-rw-r--r-- 1 febner febner 20971520 May 21 11:18 /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw Only after mirroring, i.e. job2, it will be larger: > +20975616 /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw > +-rw-r--r-- 1 febner febner 20971520 May 21 11:18 /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw Some more information: > [I] febner@dev8 ~/repos/qemu/tests/qemu-iotests/tests (master)> sudo filefrag /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw -e > Filesystem type is: ef53 > File size of /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw is 20971520 (5120 blocks of 4096 bytes) > ext: logical_offset: physical_offset: length: expected: flags: > 0: 0.. 15: 36233216.. 36233231: 16: > 1: 16.. 2047: 36233232.. 36235263: 2032: unwritten > 2: 2048.. 2559: 36239360.. 36239871: 512: 36235264: > 3: 2560.. 4095: 36239872.. 36241407: 1536: unwritten > 4: 4096.. 5119: 33792000.. 33793023: 1024: 36241408: last,unwritten,eof > /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw: 3 extents found Note that one of the offsets is not 4KiB-aligned: > 36239360%4096 > 2048 I suppose that is the reason? > [I] febner@dev8 ~/repos/qemu/tests/qemu-iotests/tests (master)> stat /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw > File: /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw > Size: 20971520 Blocks: 40968 IO Block: 4096 regular file > Device: 8,16 Inode: 4866541 Links: 1 > Access: (0644/-rw-r--r--) Uid: ( 1000/ febner) Gid: ( 1000/ febner) > Access: 2025-05-21 11:18:37.622692254 +0200 > Modify: 2025-05-21 11:18:37.540686997 +0200 > Change: 2025-05-21 11:18:37.540686997 +0200 > Birth: 2025-05-21 11:18:37.463682059 +0200 I also tried it on my host, where the filesystem is ZFS using compression, and there, 'du' will already report a lower value after creating the image, because of compression. And even without compression it seems that preallocation=full on ZFS is somehow async :/ > [I] febner@enia ~/qemu/build/tests/qemu-iotests (master)> qemu-img create my.raw 20M -f raw -o preallocation=full > Formatting 'my.raw', fmt=raw size=20971520 preallocation=full > [I] febner@enia ~/qemu/build/tests/qemu-iotests (master)> du --block-size=1 my.raw > 512 my.raw > [I] febner@enia ~/qemu/build/tests/qemu-iotests (master)> du --block-size=1 my.raw > 20980224 my.raw Best Regards, Fiona
On Wed, May 21, 2025 at 11:54:03AM +0200, Fiona Ebner wrote: > Hi, > > Am 15.05.25 um 04:28 schrieb Eric Blake: > > +do_test full ignore off full > > for me, this one fails irregularly (more than 50% of the time) with: > -Destination is full; expected full > +Destination is unknown; expected full > > My file system is ext4 inside a virtual machine. > > [I] febner@dev8 ~/repos/qemu/build/tests/qemu-iotests (master) [1]> findmnt --target . > > TARGET SOURCE FSTYPE OPTIONS > > /home/febner/repos /dev/sdb ext4 rw,relatime > > The virtual disk is an LVM image. When the test fails, I consistently > get 20975616 bytes rather than 20971520 bytes as the result of > disk_usage(), i.e. it has one 4 KiB block more. ls -l will still report > 20971520 bytes as the length, qemu-img compare succeeds. Odd, but should be something I can cater to by treating any value >= 20M (rather than exactly 20M) as fully allocated (the test still does the qemu-img compare to ensure that the actual length compares correctly, even if the allocated length was slightly overallocated due to filesystem magic). > > After the image creation, i.e. job1, the allocation is still as expected: > > +20971520 /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw > > +-rw-r--r-- 1 febner febner 20971520 May 21 11:18 /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw > > Only after mirroring, i.e. job2, it will be larger: > > +20975616 /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw > > +-rw-r--r-- 1 febner febner 20971520 May 21 11:18 /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw Weird, but filesystems have been known to do weird things. So the test should be updated to tolerate it. > > Some more information: > > > [I] febner@dev8 ~/repos/qemu/tests/qemu-iotests/tests (master)> sudo filefrag /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw -e > > Filesystem type is: ef53 > > File size of /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw is 20971520 (5120 blocks of 4096 bytes) > > ext: logical_offset: physical_offset: length: expected: flags: > > 0: 0.. 15: 36233216.. 36233231: 16: > > 1: 16.. 2047: 36233232.. 36235263: 2032: unwritten > > 2: 2048.. 2559: 36239360.. 36239871: 512: 36235264: > > 3: 2560.. 4095: 36239872.. 36241407: 1536: unwritten > > 4: 4096.. 5119: 33792000.. 33793023: 1024: 36241408: last,unwritten,eof > > /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw: 3 extents found > > Note that one of the offsets is not 4KiB-aligned: > > 36239360%4096 > > 2048 > > I suppose that is the reason? > > > [I] febner@dev8 ~/repos/qemu/tests/qemu-iotests/tests (master)> stat /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw > > File: /home/febner/repos/qemu/build/tests/qemu-iotests/scratch/raw-file-mirror-sparse/t.raw > > Size: 20971520 Blocks: 40968 IO Block: 4096 regular file > > Device: 8,16 Inode: 4866541 Links: 1 > > Access: (0644/-rw-r--r--) Uid: ( 1000/ febner) Gid: ( 1000/ febner) > > Access: 2025-05-21 11:18:37.622692254 +0200 > > Modify: 2025-05-21 11:18:37.540686997 +0200 > > Change: 2025-05-21 11:18:37.540686997 +0200 > > Birth: 2025-05-21 11:18:37.463682059 +0200 > > I also tried it on my host, where the filesystem is ZFS using > compression, and there, 'du' will already report a lower value after > creating the image, because of compression. And even without compression > it seems that preallocation=full on ZFS is somehow async :/ > > > [I] febner@enia ~/qemu/build/tests/qemu-iotests (master)> qemu-img create my.raw 20M -f raw -o preallocation=full > > Formatting 'my.raw', fmt=raw size=20971520 preallocation=full > > [I] febner@enia ~/qemu/build/tests/qemu-iotests (master)> du --block-size=1 my.raw > > 512 my.raw > > [I] febner@enia ~/qemu/build/tests/qemu-iotests (master)> du --block-size=1 my.raw > > 20980224 my.raw That one may be a bit harder to work around, but I'll give it a shot while I'm patching the first one. -- Eric Blake, Principal Software Engineer Red Hat, Inc. Virtualization: qemu.org | libguestfs.org
Am 21.05.25 um 17:32 schrieb Eric Blake: > On Wed, May 21, 2025 at 11:54:03AM +0200, Fiona Ebner wrote: >> I also tried it on my host, where the filesystem is ZFS using >> compression, and there, 'du' will already report a lower value after >> creating the image, because of compression. And even without compression >> it seems that preallocation=full on ZFS is somehow async :/ >> >>> [I] febner@enia ~/qemu/build/tests/qemu-iotests (master)> qemu-img create my.raw 20M -f raw -o preallocation=full >>> Formatting 'my.raw', fmt=raw size=20971520 preallocation=full >>> [I] febner@enia ~/qemu/build/tests/qemu-iotests (master)> du --block-size=1 my.raw >>> 512 my.raw >>> [I] febner@enia ~/qemu/build/tests/qemu-iotests (master)> du --block-size=1 my.raw >>> 20980224 my.raw > > That one may be a bit harder to work around, but I'll give it a shot > while I'm patching the first one. FWIW, doing a 'sync' before querying with 'du' will avoid the issue for the allocation. But even then, after mirroring, the file will be sparse again. Best Regards, Fiona
© 2016 - 2025 Red Hat, Inc.