The following changes since commit 96662996eda78c48aadddd4e76d8615c7eb72d80:

Merge remote-tracking branch 'remotes/dgilbert/tags/pull-migration-20210513a' into staging (2021-05-14 12:03:47 +0100)

https://github.com/XanClic/qemu.git tags/pull-block-2021-05-14

for you to fetch changes up to c61ebf362d0abf288ce266845519d5a550a1d89f:

write-threshold: deal with includes (2021-05-14 16:14:10 +0200)

- drop block/io write notifiers

- qemu-iotests enhancements to make debugging easier

- rbd parsing fix

- HMP qemu-io fix (for iothreads)

- mirror job cancel relaxation (do not cancel in-flight requests when a

READY mirror job is canceled with force=false)

- document qcow2's data_file and data_file_raw features

- fix iotest 297 for pylint 2.8

- block/copy-on-read refactoring

Connor Kuehl (3):

iotests/231: Update expected deprecation message

block/rbd: Add an escape-aware strchr helper

Document qemu-img options data_file and data_file_raw

Emanuele Giuseppe Esposito (1):

qemu-iotests: fix pylint 2.8 consider-using-with error

Paolo Bonzini (5):

qemu-iotests: do not buffer the test output

qemu-iotests: allow passing unittest.main arguments to the test

scripts

qemu-iotests: move command line and environment handling from

TestRunner to TestEnv

qemu-iotests: let "check" spawn an arbitrary test command

qemu-iotests: fix case of SOCK_DIR already in the environment

Vladimir Sementsov-Ogievskiy (10):

monitor: hmp_qemu_io: acquire aio contex, fix crash

mirror: stop cancelling in-flight requests on non-force cancel in

READY

block/copy-on-read: use bdrv_drop_filter() and drop s->active

block/write-threshold: don't use write notifiers

block: drop write notifiers

test-write-threshold: rewrite test_threshold_(not_)trigger tests

block/write-threshold: drop extra APIs

test-write-threshold: drop extra tests

test-write-threshold: drop extra TestStruct structure

write-threshold: deal with includes

docs/tools/qemu-img.rst | 31 +++++++

include/block/block_int.h | 15 +---

include/block/write-threshold.h | 27 ++----

include/qemu/job.h | 2 +-

block.c | 1 -

block/backup.c | 2 +-

block/copy-on-read.c | 33 +------

block/io.c | 11 +--

block/mirror.c | 6 +-

block/monitor/block-hmp-cmds.c | 31 ++++---

block/rbd.c | 32 ++++---

block/write-threshold.c | 91 ++++---------------

job.c | 2 +-

qemu-io-cmds.c | 8 +-

qemu-io.c | 17 +++-

tests/unit/test-write-threshold.c | 90 ++-----------------

tests/qemu-iotests/231 | 4 +

tests/qemu-iotests/231.out | 7 +-

tests/qemu-iotests/240.out | 8 +-

tests/qemu-iotests/245.out | 8 +-

tests/qemu-iotests/264 | 2 +-

tests/qemu-iotests/295.out | 6 +-

tests/qemu-iotests/296.out | 8 +-

tests/qemu-iotests/check | 19 +++-

tests/qemu-iotests/iotests.py | 145 +++++++++++++++++-------------

tests/qemu-iotests/pylintrc | 3 +

tests/qemu-iotests/testenv.py | 22 ++++-

tests/qemu-iotests/testrunner.py | 37 +++-----

28 files changed, 289 insertions(+), 379 deletions(-)

2.31.1

From: Connor Kuehl <ckuehl@redhat.com>

Sometimes the parser needs to further split a token it has collected

from the token input stream. Right now, it does a cursory check to see

if the relevant characters appear in the token to determine if it should

break it down further.

However, qemu_rbd_next_tok() will escape characters as it removes tokens

from the token stream and plain strchr() won't. This can make the

initial strchr() check slightly misleading since it implies

qemu_rbd_next_tok() will find the token and split on it, except the

reality is that qemu_rbd_next_tok() will pass over it if it is escaped.

Use a custom strchr to avoid mixing escaped and unescaped string

operations. Furthermore, this code is identical to how

qemu_rbd_next_tok() seeks its next token, so incorporate this custom

strchr into the body of that function to reduce duplication.

Reported-by: Han Han <hhan@redhat.com>

Fixes: https://bugzilla.redhat.com/1873913

Signed-off-by: Connor Kuehl <ckuehl@redhat.com>

Message-Id: <20210421212343.85524-3-ckuehl@redhat.com>

Reviewed-by: Stefano Garzarella <sgarzare@redhat.com>

block/rbd.c | 32 +++++++++++++++++++++-----------

tests/qemu-iotests/231 | 4 ++++

tests/qemu-iotests/231.out | 3 +++

3 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/block/rbd.c b/block/rbd.c

index XXXXXXX..XXXXXXX 100644

--- a/block/rbd.c

+++ b/block/rbd.c

@@ -XXX,XX +XXX,XX @@ static int qemu_rbd_connect(rados_t *cluster, rados_ioctx_t *io_ctx,

const char *keypairs, const char *secretid,

Error **errp);

+static char *qemu_rbd_strchr(char *src, char delim)

+ char *p;

+ for (p = src; *p; ++p) {

+ if (*p == delim) {

+ return p;

+ }

+ if (*p == '\\' && p[1] != '\0') {

+ ++p;

+ }

+ }

+

+ return NULL;

static char *qemu_rbd_next_tok(char *src, char delim, char **p)

{

char *end;

*p = NULL;

- for (end = src; *end; ++end) {

- if (*end == delim) {

- break;

- }

- if (*end == '\\' && end[1] != '\0') {

- end++;

- }

- }

- if (*end == delim) {

+ end = qemu_rbd_strchr(src, delim);

+ if (end) {

*p = end + 1;

*end = '\0';

}

@@ -XXX,XX +XXX,XX @@ static void qemu_rbd_parse_filename(const char *filename, QDict *options,

qemu_rbd_unescape(found_str);

qdict_put_str(options, "pool", found_str);

- if (strchr(p, '@')) {

+ if (qemu_rbd_strchr(p, '@')) {

image_name = qemu_rbd_next_tok(p, '@', &p);

found_str = qemu_rbd_next_tok(p, ':', &p);

@@ -XXX,XX +XXX,XX @@ static void qemu_rbd_parse_filename(const char *filename, QDict *options,

image_name = qemu_rbd_next_tok(p, ':', &p);

}

/* Check for namespace in the image_name */

- if (strchr(image_name, '/')) {

+ if (qemu_rbd_strchr(image_name, '/')) {

found_str = qemu_rbd_next_tok(image_name, '/', &image_name);

qemu_rbd_unescape(found_str);

qdict_put_str(options, "namespace", found_str);

diff --git a/tests/qemu-iotests/231 b/tests/qemu-iotests/231

index XXXXXXX..XXXXXXX 100755

--- a/tests/qemu-iotests/231

+++ b/tests/qemu-iotests/231

@@ -XXX,XX +XXX,XX @@ _filter_conf()

$QEMU_IMG info "json:{'file.driver':'rbd','file.filename':'rbd:rbd/bogus:conf=${BOGUS_CONF}'}" 2>&1 | _filter_conf

$QEMU_IMG info "json:{'file.driver':'rbd','file.pool':'rbd','file.image':'bogus','file.conf':'${BOGUS_CONF}'}" 2>&1 | _filter_conf

+# Regression test: the qemu-img invocation is expected to fail, but it should

+# not seg fault the parser.

+$QEMU_IMG create "rbd:rbd/aa\/bb:conf=${BOGUS_CONF}" 1M 2>&1 | _filter_conf

diff --git a/tests/qemu-iotests/231.out b/tests/qemu-iotests/231.out

--- a/tests/qemu-iotests/231.out

+++ b/tests/qemu-iotests/231.out

@@ -XXX,XX +XXX,XX @@ unable to get monitor info from DNS SRV with service name: ceph-mon

qemu-img: Could not open 'json:{'file.driver':'rbd','file.filename':'rbd:rbd/bogus:conf=BOGUS_CONF'}': error connecting: No such file or directory

unable to get monitor info from DNS SRV with service name: ceph-mon

qemu-img: Could not open 'json:{'file.driver':'rbd','file.pool':'rbd','file.image':'bogus','file.conf':'BOGUS_CONF'}': error connecting: No such file or directory

+Formatting 'rbd:rbd/aa\/bb:conf=BOGUS_CONF', fmt=raw size=1048576

+unable to get monitor info from DNS SRV with service name: ceph-mon

+qemu-img: rbd:rbd/aa\/bb:conf=BOGUS_CONF: error connecting: No such file or directory

*** done

2.31.1

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Max reported the following bug:

$ ./qemu-img create -f raw src.img 1G

$ ./qemu-img create -f raw dst.img 1G

$ (echo '

{"execute":"qmp_capabilities"}

{"execute":"blockdev-mirror",

"arguments":{"job-id":"mirror",

"device":"source",

"target":"target",

"sync":"full",

"filter-node-name":"mirror-top"}}

'; sleep 3; echo '

{"execute":"human-monitor-command",

"arguments":{"command-line":

"qemu-io mirror-top \"write 0 1G\""}}') \

| x86_64-softmmu/qemu-system-x86_64 \

-qmp stdio \

-blockdev file,node-name=source,filename=src.img \

-blockdev file,node-name=target,filename=dst.img \

-object iothread,id=iothr0 \

-device virtio-blk,drive=source,iothread=iothr0

crashes:

0 raise () at /usr/lib/libc.so.6

1 abort () at /usr/lib/libc.so.6

2 error_exit

(err=<optimized out>,

msg=msg@entry=0x55fbb1634790 <__func__.27> "qemu_mutex_unlock_impl")

at ../util/qemu-thread-posix.c:37

3 qemu_mutex_unlock_impl

(mutex=mutex@entry=0x55fbb25ab6e0,

file=file@entry=0x55fbb1636957 "../util/async.c",

line=line@entry=650)

at ../util/qemu-thread-posix.c:109

4 aio_context_release (ctx=ctx@entry=0x55fbb25ab680) at ../util/async.c:650

5 bdrv_do_drained_begin

(bs=bs@entry=0x55fbb3a87000, recursive=recursive@entry=false,

parent=parent@entry=0x0,

ignore_bds_parents=ignore_bds_parents@entry=false,

poll=poll@entry=true) at ../block/io.c:441

6 bdrv_do_drained_begin

(poll=true, ignore_bds_parents=false, parent=0x0, recursive=false,

bs=0x55fbb3a87000) at ../block/io.c:448

7 blk_drain (blk=0x55fbb26c5a00) at ../block/block-backend.c:1718

8 blk_unref (blk=0x55fbb26c5a00) at ../block/block-backend.c:498

9 blk_unref (blk=0x55fbb26c5a00) at ../block/block-backend.c:491

10 hmp_qemu_io (mon=0x7fffaf3fc7d0, qdict=<optimized out>)

at ../block/monitor/block-hmp-cmds.c:628

man pthread_mutex_unlock

...

EPERM The mutex type is PTHREAD_MUTEX_ERRORCHECK or

PTHREAD_MUTEX_RECURSIVE, or the mutex is a robust mutex, and the

current thread does not own the mutex.

So, thread doesn't own the mutex. And we have iothread here.

Next, note that AIO_WAIT_WHILE() documents that ctx must be acquired

exactly once by caller. But where is it acquired in the call stack?

Seems nowhere.

qemuio_command do acquire aio context.. But we need context acquired

around blk_unref() as well and actually around blk_insert_bs() too.

Let's refactor qemuio_command so that it doesn't acquire aio context

but callers do that instead. This way we can cleanly acquire aio

context in hmp_qemu_io() around all three calls.

Reported-by: Max Reitz <mreitz@redhat.com>

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Message-Id: <20210423134233.51495-1-vsementsov@virtuozzo.com>

[mreitz: Fixed comment]

block/monitor/block-hmp-cmds.c | 31 +++++++++++++++++++++----------

qemu-io-cmds.c | 8 ++++----

qemu-io.c | 17 +++++++++++++++--

3 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c

index XXXXXXX..XXXXXXX 100644

--- a/block/monitor/block-hmp-cmds.c

+++ b/block/monitor/block-hmp-cmds.c

@@ -XXX,XX +XXX,XX @@ void hmp_eject(Monitor *mon, const QDict *qdict)

void hmp_qemu_io(Monitor *mon, const QDict *qdict)

{

- BlockBackend *blk;

+ BlockBackend *blk = NULL;

+ BlockDriverState *bs = NULL;

BlockBackend *local_blk = NULL;

+ AioContext *ctx = NULL;

bool qdev = qdict_get_try_bool(qdict, "qdev", false);

const char *device = qdict_get_str(qdict, "device");

const char *command = qdict_get_str(qdict, "command");

@@ -XXX,XX +XXX,XX @@ void hmp_qemu_io(Monitor *mon, const QDict *qdict)

} else {

blk = blk_by_name(device);

if (!blk) {

- BlockDriverState *bs = bdrv_lookup_bs(NULL, device, &err);

- if (bs) {

- blk = local_blk = blk_new(bdrv_get_aio_context(bs),

- 0, BLK_PERM_ALL);

- ret = blk_insert_bs(blk, bs, &err);

- if (ret < 0) {

- goto fail;

- }

- } else {

+ bs = bdrv_lookup_bs(NULL, device, &err);

+ if (!bs) {

goto fail;

}

}

}

+ ctx = blk ? blk_get_aio_context(blk) : bdrv_get_aio_context(bs);

+ aio_context_acquire(ctx);

+

+ if (bs) {

+ blk = local_blk = blk_new(bdrv_get_aio_context(bs), 0, BLK_PERM_ALL);

+ ret = blk_insert_bs(blk, bs, &err);

+ if (ret < 0) {

+ goto fail;

+ }

+ }

+

/*

* Notably absent: Proper permission management. This is sad, but it seems

* almost impossible to achieve without changing the semantics and thereby

@@ -XXX,XX +XXX,XX @@ void hmp_qemu_io(Monitor *mon, const QDict *qdict)

fail:

blk_unref(local_blk);

+

+ if (ctx) {

+ aio_context_release(ctx);

+ }

+

hmp_handle_error(mon, err);

}

@@ -XXX,XX +XXX,XX @@ static const cmdinfo_t help_cmd = {

.oneline = "help for one or all commands",

};

+/*

+ * Called with aio context of blk acquired. Or with qemu_get_aio_context()

+ * context acquired if blk is NULL.

+ */

int qemuio_command(BlockBackend *blk, const char *cmd)

{

- AioContext *ctx;

char *input;

const cmdinfo_t *ct;

char **v;

@@ -XXX,XX +XXX,XX @@ int qemuio_command(BlockBackend *blk, const char *cmd)

if (c) {

ct = find_command(v[0]);

if (ct) {

- ctx = blk ? blk_get_aio_context(blk) : qemu_get_aio_context();

- aio_context_acquire(ctx);

ret = command(blk, ct, c, v);

- aio_context_release(ctx);

} else {

fprintf(stderr, "command \"%s\" not found\n", v[0]);

ret = -EINVAL;

diff --git a/qemu-io.c b/qemu-io.c

index XXXXXXX..XXXXXXX 100644

--- a/qemu-io.c

+++ b/qemu-io.c

@@ -XXX,XX +XXX,XX @@ static void prep_fetchline(void *opaque)

*fetchable= 1;

+static int do_qemuio_command(const char *cmd)

+ int ret;

+ AioContext *ctx =

+ qemuio_blk ? blk_get_aio_context(qemuio_blk) : qemu_get_aio_context();

+

+ aio_context_acquire(ctx);

+ ret = qemuio_command(qemuio_blk, cmd);

+ aio_context_release(ctx);

+

+ return ret;

static int command_loop(void)

int i, fetchable = 0, prompted = 0;

@@ -XXX,XX +XXX,XX @@ static int command_loop(void)

char *input;

for (i = 0; !quit_qemu_io && i < ncmdline; i++) {

- ret = qemuio_command(qemuio_blk, cmdline[i]);

+ ret = do_qemuio_command(cmdline[i]);

if (ret < 0) {

last_error = ret;

}

@@ -XXX,XX +XXX,XX @@ static int command_loop(void)

if (input == NULL) {

}

- ret = qemuio_command(qemuio_blk, input);

+ ret = do_qemuio_command(input);

g_free(input);

if (ret < 0) {

2.31.1

From: Paolo Bonzini <pbonzini@redhat.com>

Instead of buffering the test output into a StringIO, patch it on

the fly by wrapping sys.stdout's write method. This can be

done unconditionally, even if using -d, which makes execute_unittest

a bit simpler.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Tested-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>

Message-Id: <20210323181928.311862-2-pbonzini@redhat.com>

Message-Id: <20210503110110.476887-2-pbonzini@redhat.com>

Signed-off-by: Max Reitz <mreitz@redhat.com>

---

tests/qemu-iotests/240.out | 8 ++--

tests/qemu-iotests/245.out | 8 ++--

tests/qemu-iotests/295.out | 6 +--

tests/qemu-iotests/296.out | 8 ++--

tests/qemu-iotests/iotests.py | 70 ++++++++++++++++++++---------------

5 files changed, 56 insertions(+), 44 deletions(-)

diff --git a/tests/qemu-iotests/240.out b/tests/qemu-iotests/240.out

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/240.out

+++ b/tests/qemu-iotests/240.out

@@ -XXX,XX +XXX,XX @@

{"return": {}}

{"execute": "blockdev-del", "arguments": {"node-name": "hd0"}}

{"return": {}}

-==Attach two SCSI disks using the same block device and the same iothread==

+.==Attach two SCSI disks using the same block device and the same iothread==

{"execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0", "read-only": true, "read-zeroes": true}}

{"return": {}}

{"execute": "object-add", "arguments": {"id": "iothread0", "qom-type": "iothread"}}

@@ -XXX,XX +XXX,XX @@

{"return": {}}

{"execute": "blockdev-del", "arguments": {"node-name": "hd0"}}

{"return": {}}

-==Attach two SCSI disks using the same block device but different iothreads==

+.==Attach two SCSI disks using the same block device but different iothreads==

{"execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0", "read-only": true, "read-zeroes": true}}

{"return": {}}

{"execute": "object-add", "arguments": {"id": "iothread0", "qom-type": "iothread"}}

@@ -XXX,XX +XXX,XX @@

{"return": {}}

{"execute": "blockdev-del", "arguments": {"node-name": "hd0"}}

{"return": {}}

-==Attach a SCSI disks using the same block device as a NBD server==

+.==Attach a SCSI disks using the same block device as a NBD server==

{"execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0", "read-only": true, "read-zeroes": true}}

{"return": {}}

{"execute": "nbd-server-start", "arguments": {"addr": {"data": {"path": "SOCK_DIR/PID-nbd.sock"}, "type": "unix"}}}

@@ -XXX,XX +XXX,XX @@

{"return": {}}

{"execute": "device_add", "arguments": {"drive": "hd0", "driver": "scsi-hd", "id": "scsi-hd0"}}

{"return": {}}

-....

+.

----------------------------------------------------------------------

Ran 4 tests

diff --git a/tests/qemu-iotests/245.out b/tests/qemu-iotests/245.out

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/245.out

+++ b/tests/qemu-iotests/245.out

@@ -XXX,XX +XXX,XX @@

-{"execute": "job-finalize", "arguments": {"id": "commit0"}}

+..{"execute": "job-finalize", "arguments": {"id": "commit0"}}

{"return": {}}

{"data": {"id": "commit0", "type": "commit"}, "event": "BLOCK_JOB_PENDING", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}

{"data": {"device": "commit0", "len": 3145728, "offset": 3145728, "speed": 0, "type": "commit"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}

-{"execute": "job-finalize", "arguments": {"id": "stream0"}}

+...{"execute": "job-finalize", "arguments": {"id": "stream0"}}

{"return": {}}

{"data": {"id": "stream0", "type": "stream"}, "event": "BLOCK_JOB_PENDING", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}

{"data": {"device": "stream0", "len": 3145728, "offset": 3145728, "speed": 0, "type": "stream"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}

-{"execute": "job-finalize", "arguments": {"id": "stream0"}}

+.{"execute": "job-finalize", "arguments": {"id": "stream0"}}

{"return": {}}

{"data": {"id": "stream0", "type": "stream"}, "event": "BLOCK_JOB_PENDING", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}

{"data": {"device": "stream0", "len": 3145728, "offset": 3145728, "speed": 0, "type": "stream"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}

-.....................

+...............

----------------------------------------------------------------------

Ran 21 tests

diff --git a/tests/qemu-iotests/295.out b/tests/qemu-iotests/295.out

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/295.out

+++ b/tests/qemu-iotests/295.out

@@ -XXX,XX +XXX,XX @@

{"return": {}}

{"execute": "job-dismiss", "arguments": {"id": "job_erase_key"}}

{"return": {}}

-{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}

+.{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}

{"return": {}}

{"execute": "job-dismiss", "arguments": {"id": "job_erase_key"}}

{"return": {}}

@@ -XXX,XX +XXX,XX @@ Job failed: Invalid password, cannot unlock any keyslot

{"return": {}}

{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}

{"return": {}}

-{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}

+.{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}

{"return": {}}

{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}

{"return": {}}

@@ -XXX,XX +XXX,XX @@ Job failed: All the active keyslots match the (old) password that was given and

{"return": {}}

{"execute": "job-dismiss", "arguments": {"id": "job_erase_key"}}

{"return": {}}

-...

+.

----------------------------------------------------------------------

Ran 3 tests

diff --git a/tests/qemu-iotests/296.out b/tests/qemu-iotests/296.out

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/296.out

+++ b/tests/qemu-iotests/296.out

@@ -XXX,XX +XXX,XX @@ Job failed: Failed to get shared "consistent read" lock

qemu-img: Failed to get shared "consistent read" lock

Is another process using the image [TEST_DIR/test.img]?

-Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10

+.Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10

Job failed: Block node is read-only

{"execute": "job-dismiss", "arguments": {"id": "job0"}}

@@ -XXX,XX +XXX,XX @@ Job failed: Failed to get shared "consistent read" lock

{"return": {}}

{"execute": "job-dismiss", "arguments": {"id": "job0"}}

{"return": {}}

-Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10

+.Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10

{"return": {}}

{"error": {"class": "GenericError", "desc": "Failed to get \"write\" lock"}}

-Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10

+.Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10

{"return": {}}

{"return": {}}

-....

+.

----------------------------------------------------------------------

Ran 4 tests

diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/iotests.py

+++ b/tests/qemu-iotests/iotests.py

@@ -XXX,XX +XXX,XX @@

import bz2

from collections import OrderedDict

import faulthandler

-import io

import json

import logging

import os

@@ -XXX,XX +XXX,XX @@

import sys

import time

from typing import (Any, Callable, Dict, Iterable,

- List, Optional, Sequence, Tuple, TypeVar)

+ List, Optional, Sequence, TextIO, Tuple, Type, TypeVar)

import unittest

from contextlib import contextmanager

@@ -XXX,XX +XXX,XX @@ def func_wrapper(*args, **kwargs):

return func(*args, **kwargs)

return func_wrapper

+# We need to filter out the time taken from the output so that

+# qemu-iotest can reliably diff the results against master output,

+# and hide skipped tests from the reference output.

+

+class ReproducibleTestResult(unittest.TextTestResult):

+ def addSkip(self, test, reason):

+ # Same as TextTestResult, but print dot instead of "s"

+ unittest.TestResult.addSkip(self, test, reason)

+ if self.showAll:

+ self.stream.writeln("skipped {0!r}".format(reason))

+ elif self.dots:

+ self.stream.write(".")

+ self.stream.flush()

+

+class ReproducibleStreamWrapper:

+ def __init__(self, stream: TextIO):

+ self.stream = stream

+

+ def __getattr__(self, attr):

+ if attr in ('stream', '__getstate__'):

+ raise AttributeError(attr)

+ return getattr(self.stream, attr)

+

+ def write(self, arg=None):

+ arg = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', arg)

+ arg = re.sub(r' \(skipped=\d+\)', r'', arg)

+ self.stream.write(arg)

+

+class ReproducibleTestRunner(unittest.TextTestRunner):

+ def __init__(self, stream: Optional[TextIO] = None,

+ resultclass: Type[unittest.TestResult] = ReproducibleTestResult,

+ **kwargs: Any) -> None:

+ rstream = ReproducibleStreamWrapper(stream or sys.stdout)

+ super().__init__(stream=rstream, # type: ignore

+ descriptions=True,

+ resultclass=resultclass,

+ **kwargs)

+

def execute_unittest(debug=False):

"""Executes unittests within the calling module."""

verbosity = 2 if debug else 1

-

- if debug:

- output = sys.stdout

- else:

- # We need to filter out the time taken from the output so that

- # qemu-iotest can reliably diff the results against master output.

- output = io.StringIO()

-

- runner = unittest.TextTestRunner(stream=output, descriptions=True,

- verbosity=verbosity)

- try:

- # unittest.main() will use sys.exit(); so expect a SystemExit

- # exception

- unittest.main(testRunner=runner)

- finally:

- # We need to filter out the time taken from the output so that

- # qemu-iotest can reliably diff the results against master output.

- if not debug:

- out = output.getvalue()

- out = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', out)

-

- # Hide skipped tests from the reference output

- out = re.sub(r'OK \(skipped=\d+\)', 'OK', out)

- out_first_line, out_rest = out.split('\n', 1)

- out = out_first_line.replace('s', '.') + '\n' + out_rest

-

- sys.stderr.write(out)

+ runner = ReproducibleTestRunner(verbosity=verbosity)

+ unittest.main(testRunner=runner)

def execute_setup_common(supported_fmts: Sequence[str] = (),

supported_platforms: Sequence[str] = (),

--

2.31.1

From: Paolo Bonzini <pbonzini@redhat.com>

In the next patch, "check" will learn how to execute a test script without

going through TestRunner. To enable this, keep only the text output

and subprocess handling in the TestRunner; move into TestEnv the logic

to prepare for running a subprocess.

Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

Tested-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>

Message-Id: <20210323181928.311862-4-pbonzini@redhat.com>

Message-Id: <20210503110110.476887-4-pbonzini@redhat.com>

tests/qemu-iotests/testenv.py | 17 ++++++++++++++++-

tests/qemu-iotests/testrunner.py | 14 +-------------

2 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/testenv.py

+++ b/tests/qemu-iotests/testenv.py

@@ -XXX,XX +XXX,XX @@

import random

import subprocess

import glob

-from typing import Dict, Any, Optional, ContextManager

+from typing import List, Dict, Any, Optional, ContextManager

def isxfile(path: str) -> bool:

@@ -XXX,XX +XXX,XX @@ class TestEnv(ContextManager['TestEnv']):

'CACHEMODE_IS_DEFAULT', 'IMGFMT_GENERIC', 'IMGOPTSSYNTAX',

'IMGKEYSECRET', 'QEMU_DEFAULT_MACHINE', 'MALLOC_PERTURB_']

+ def prepare_subprocess(self, args: List[str]) -> Dict[str, str]:

+ if self.debug:

+ args.append('-d')

+ with open(args[0], encoding="utf-8") as f:

+ try:

+ if f.readline().rstrip() == '#!/usr/bin/env python3':

+ args.insert(0, self.python)

+ except UnicodeDecodeError: # binary test? for future.

+ pass

+

+ os_env = os.environ.copy()

+ os_env.update(self.get_env())

+ return os_env

+

def get_env(self) -> Dict[str, str]:

env = {}

for v in self.env_variables:

diff --git a/tests/qemu-iotests/testrunner.py b/tests/qemu-iotests/testrunner.py

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/testrunner.py

+++ b/tests/qemu-iotests/testrunner.py

@@ -XXX,XX +XXX,XX @@ class TestRunner(ContextManager['TestRunner']):

def __init__(self, env: TestEnv, makecheck: bool = False,

color: str = 'auto') -> None:

self.env = env

- self.test_run_env = self.env.get_env()

self.makecheck = makecheck

self.last_elapsed = LastElapsedTime('.last-elapsed-cache', env)

@@ -XXX,XX +XXX,XX @@ def do_run_test(self, test: str) -> TestResult:

silent_unlink(p)

args = [str(f_test.resolve())]

- if self.env.debug:

- args.append('-d')

-

- with f_test.open(encoding="utf-8") as f:

- try:

- if f.readline().rstrip() == '#!/usr/bin/env python3':

- args.insert(0, self.env.python)

- except UnicodeDecodeError: # binary test? for future.

- pass

-

- env = os.environ.copy()

- env.update(self.test_run_env)

+ env = self.env.prepare_subprocess(args)

t0 = time.time()

with f_bad.open('w', encoding="utf-8") as f:

2.31.1

From: Paolo Bonzini <pbonzini@redhat.com>

Right now there is no easy way for "check" to print a reproducer command.

Because such a reproducer command line would be huge, we can instead teach

check to start a command of our choice. This can be for example a Python

unit test with arguments to only run a specific subtest.

Move the trailing empty line to print_env(), since it always looks better

and one caller was not adding it.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Tested-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>

Message-Id: <20210323181928.311862-5-pbonzini@redhat.com>

Message-Id: <20210503110110.476887-5-pbonzini@redhat.com>

tests/qemu-iotests/check | 19 ++++++++++++++++++-

tests/qemu-iotests/testenv.py | 3 ++-

tests/qemu-iotests/testrunner.py | 1 -

3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/tests/qemu-iotests/check b/tests/qemu-iotests/check

--- a/tests/qemu-iotests/check

+++ b/tests/qemu-iotests/check

@@ -XXX,XX +XXX,XX @@

import os

import sys

import argparse

+import shutil

+from pathlib import Path

+

from findtests import TestFinder

from testenv import TestEnv

from testrunner import TestRunner

@@ -XXX,XX +XXX,XX @@ def make_argparser() -> argparse.ArgumentParser:

'rerun failed ./check command, starting from the '

'middle of the process.')

g_sel.add_argument('tests', metavar='TEST_FILES', nargs='*',

- help='tests to run')

+ help='tests to run, or "--" followed by a command')

return p

@@ -XXX,XX +XXX,XX @@ if __name__ == '__main__':

imgopts=args.imgopts, misalign=args.misalign,

debug=args.debug, valgrind=args.valgrind)

+ if len(sys.argv) > 1 and sys.argv[-len(args.tests)-1] == '--':

+ if not args.tests:

+ sys.exit("missing command after '--'")

+ cmd = args.tests

+ env.print_env()

+ exec_pathstr = shutil.which(cmd[0])

+ if exec_pathstr is None:

+ sys.exit('command not found: ' + cmd[0])

+ exec_path = Path(exec_pathstr).resolve()

+ cmd[0] = str(exec_path)

+ full_env = env.prepare_subprocess(cmd)

+ os.chdir(exec_path.parent)

+ os.execve(cmd[0], cmd, full_env)

+

testfinder = TestFinder(test_dir=env.source_iotests)

groups = args.groups.split(',') if args.groups else None

diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/testenv.py

+++ b/tests/qemu-iotests/testenv.py

@@ -XXX,XX +XXX,XX @@ def print_env(self) -> None:

PLATFORM -- {platform}

TEST_DIR -- {TEST_DIR}

SOCK_DIR -- {SOCK_DIR}

-SOCKET_SCM_HELPER -- {SOCKET_SCM_HELPER}"""

+SOCKET_SCM_HELPER -- {SOCKET_SCM_HELPER}

+"""

args = collections.defaultdict(str, self.get_env())

diff --git a/tests/qemu-iotests/testrunner.py b/tests/qemu-iotests/testrunner.py

index XXXXXXX..XXXXXXX 100644

--- a/tests/qemu-iotests/testrunner.py

+++ b/tests/qemu-iotests/testrunner.py

@@ -XXX,XX +XXX,XX @@ def run_tests(self, tests: List[str]) -> bool:

if not self.makecheck:

self.env.print_env()

- print()

test_field_width = max(len(os.path.basename(t)) for t in tests) + 2