[PULL 0/3] Block patches
[PULL for-8.1 0/1] Block patches
1
The following changes since commit 8c5f94cd4182753959c8be8de415120dc879d8f0:
1
The following changes since commit ed8ad9728a9c0eec34db9dff61dfa2f1dd625637:
2
2
3
Merge tag 'pull-loong-20211221-2' of https://gitlab.com/rth7680/qemu into staging (2021-12-21 13:30:35 -0800)
3
Merge tag 'pull-tpm-2023-07-14-1' of https://github.com/stefanberger/qemu-tpm into staging (2023-07-15 14:54:04 +0100)
4
4
5
are available in the Git repository at:
5
are available in the Git repository at:
6
6
7
https://gitlab.com/hreitz/qemu.git tags/pull-block-2021-12-22
7
https://gitlab.com/stefanha/qemu.git tags/block-pull-request
8
8
9
for you to fetch changes up to 722f87df2545b308aec49b459b028f0802b4fd9e:
9
for you to fetch changes up to 66547f416a61e0cb711dc76821890242432ba193:
10
10
11
iotests: check: multiprocessing support (2021-12-22 16:29:48 +0100)
11
block/nvme: invoke blk_io_plug_call() outside q->lock (2023-07-17 09:17:41 -0400)
12
12
13
----------------------------------------------------------------
13
----------------------------------------------------------------
14
Block patches:
14
Pull request
15
- Added support to the iotests for running tests in several parallel
15
16
jobs (using the new -j parameter)
16
Fix the hang in the nvme:// block driver during startup.
17
17
18
----------------------------------------------------------------
18
----------------------------------------------------------------
19
Vladimir Sementsov-Ogievskiy (3):
20
iotests/testrunner.py: add doc string for run_test()
21
iotests/testrunner.py: move updating last_elapsed to run_tests
22
iotests: check: multiprocessing support
23
19
24
tests/qemu-iotests/check | 4 +-
20
Stefan Hajnoczi (1):
25
tests/qemu-iotests/testrunner.py | 86 ++++++++++++++++++++++++++++----
21
block/nvme: invoke blk_io_plug_call() outside q->lock
26
2 files changed, 80 insertions(+), 10 deletions(-)
22
23
block/nvme.c | 3 ++-
24
1 file changed, 2 insertions(+), 1 deletion(-)
27
25
28
--
26
--
29
2.33.1
27
2.40.1
30
31
diff view generated by jsdifflib
[PULL 1/3] iotests/testrunner.py: add doc string for run_test()
Deleted patch
1
From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
2
1
3
We are going to modify these methods and will add more documentation in
4
further commit. As a preparation add basic documentation.
5
6
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
7
Message-Id: <20211203122223.2780098-2-vsementsov@virtuozzo.com>
8
Reviewed-by: John Snow <jsnow@redhat.com>
9
Tested-by: John Snow <jsnow@redhat.com>
10
Signed-off-by: Hanna Reitz <hreitz@redhat.com>
11
---
12
tests/qemu-iotests/testrunner.py | 13 +++++++++++++
13
1 file changed, 13 insertions(+)
14
15
diff --git a/tests/qemu-iotests/testrunner.py b/tests/qemu-iotests/testrunner.py
16
index XXXXXXX..XXXXXXX 100644
17
--- a/tests/qemu-iotests/testrunner.py
18
+++ b/tests/qemu-iotests/testrunner.py
19
@@ -XXX,XX +XXX,XX @@ def find_reference(self, test: str) -> str:
20
return f'{test}.out'
21
22
def do_run_test(self, test: str) -> TestResult:
23
+ """
24
+ Run one test
25
+
26
+ :param test: test file path
27
+ """
28
+
29
f_test = Path(test)
30
f_bad = Path(f_test.name + '.out.bad')
31
f_notrun = Path(f_test.name + '.notrun')
32
@@ -XXX,XX +XXX,XX @@ def do_run_test(self, test: str) -> TestResult:
33
34
def run_test(self, test: str,
35
test_field_width: Optional[int] = None) -> TestResult:
36
+ """
37
+ Run one test and print short status
38
+
39
+ :param test: test file path
40
+ :param test_field_width: width for first field of status format
41
+ """
42
+
43
last_el = self.last_elapsed.get(test)
44
start = datetime.datetime.now().strftime('%H:%M:%S')
45
46
--
47
2.33.1
48
49
diff view generated by jsdifflib
[PULL 2/3] iotests/testrunner.py: move updating last_elapsed to run_tests
Deleted patch
1
From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
2
1
3
We are going to use do_run_test() in multiprocessing environment, where
4
we'll not be able to change original runner object.
5
6
Happily, the only thing we change is that last_elapsed and it's simple
7
to do it in run_tests() instead. All other accesses to self in
8
do_runt_test() and in run_test() are read-only.
9
10
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
11
Message-Id: <20211203122223.2780098-3-vsementsov@virtuozzo.com>
12
Reviewed-by: John Snow <jsnow@redhat.com>
13
Tested-by: John Snow <jsnow@redhat.com>
14
Signed-off-by: Hanna Reitz <hreitz@redhat.com>
15
---
16
tests/qemu-iotests/testrunner.py | 4 +++-
17
1 file changed, 3 insertions(+), 1 deletion(-)
18
19
diff --git a/tests/qemu-iotests/testrunner.py b/tests/qemu-iotests/testrunner.py
20
index XXXXXXX..XXXXXXX 100644
21
--- a/tests/qemu-iotests/testrunner.py
22
+++ b/tests/qemu-iotests/testrunner.py
23
@@ -XXX,XX +XXX,XX @@ def do_run_test(self, test: str) -> TestResult:
24
diff=diff, casenotrun=casenotrun)
25
else:
26
f_bad.unlink()
27
- self.last_elapsed.update(test, elapsed)
28
return TestResult(status='pass', elapsed=elapsed,
29
casenotrun=casenotrun)
30
31
@@ -XXX,XX +XXX,XX @@ def run_tests(self, tests: List[str]) -> bool:
32
print('\n'.join(res.diff))
33
elif res.status == 'not run':
34
notrun.append(name)
35
+ elif res.status == 'pass':
36
+ assert res.elapsed is not None
37
+ self.last_elapsed.update(t, res.elapsed)
38
39
sys.stdout.flush()
40
if res.interrupted:
41
--
42
2.33.1
43
44
diff view generated by jsdifflib
[PULL 3/3] iotests: check: multiprocessing support
[PULL for-8.1 1/1] block/nvme: invoke blk_io_plug_call() outside q->lock
1
From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
1
blk_io_plug_call() is invoked outside a blk_io_plug()/blk_io_unplug()
2
section while opening the NVMe drive from:
2
3
3
Add -j <JOBS> parameter, to run tests in several jobs simultaneously.
4
nvme_file_open() ->
4
For realization - simply utilize multiprocessing.Pool class.
5
nvme_init() ->
6
nvme_identify() ->
7
nvme_admin_cmd_sync() ->
8
nvme_submit_command() ->
9
blk_io_plug_call()
5
10
6
Notes:
11
blk_io_plug_call() immediately invokes the given callback when the
12
current thread is not plugged, as is the case during nvme_file_open().
7
13
8
1. Of course, tests can't run simultaneously in same TEST_DIR. So,
14
Unfortunately, nvme_submit_command() calls blk_io_plug_call() with
9
use subdirectories TEST_DIR/testname/ and SOCK_DIR/testname/
15
q->lock still held:
10
instead of simply TEST_DIR and SOCK_DIR
11
16
12
2. multiprocessing.Pool.starmap function doesn't support passing
17
...
13
context managers, so we can't simply pass "self". Happily, we need
18
q->sq.tail = (q->sq.tail + 1) % NVME_QUEUE_SIZE;
14
self only for read-only access, and it just works if it is defined
19
q->need_kick++;
15
in global space. So, add a temporary link TestRunner.shared_self
20
blk_io_plug_call(nvme_unplug_fn, q);
16
during run_tests().
21
qemu_mutex_unlock(&q->lock);
22
^^^^^^^^^^^^^^^^^^^^^^^^^^^
17
23
18
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
24
nvme_unplug_fn() deadlocks trying to acquire q->lock because the lock is
19
Message-Id: <20211203122223.2780098-4-vsementsov@virtuozzo.com>
25
already acquired by the same thread. The symptom is that QEMU hangs
20
Reviewed-by: John Snow <jsnow@redhat.com>
26
during startup while opening the NVMe drive.
21
Tested-by: John Snow <jsnow@redhat.com>
27
22
Signed-off-by: Hanna Reitz <hreitz@redhat.com>
28
Fix this by moving the blk_io_plug_call() outside q->lock. This is safe
29
because no other thread runs code related to this queue and
30
blk_io_plug_call()'s internal state is immune to thread safety issues
31
since it is thread-local.
32
33
Reported-by: Lukáš Doktor <ldoktor@redhat.com>
34
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
35
Tested-by: Lukas Doktor <ldoktor@redhat.com>
36
Message-id: 20230712191628.252806-1-stefanha@redhat.com
37
Fixes: f2e590002bd6 ("block/nvme: convert to blk_io_plug_call() API")
38
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
23
---
39
---
24
tests/qemu-iotests/check | 4 +-
40
block/nvme.c | 3 ++-
25
tests/qemu-iotests/testrunner.py | 69 ++++++++++++++++++++++++++++----
41
1 file changed, 2 insertions(+), 1 deletion(-)
26
2 files changed, 64 insertions(+), 9 deletions(-)
27
42
28
diff --git a/tests/qemu-iotests/check b/tests/qemu-iotests/check
43
diff --git a/block/nvme.c b/block/nvme.c
29
index XXXXXXX..XXXXXXX 100755
30
--- a/tests/qemu-iotests/check
31
+++ b/tests/qemu-iotests/check
32
@@ -XXX,XX +XXX,XX @@ def make_argparser() -> argparse.ArgumentParser:
33
help='show me, do not run tests')
34
p.add_argument('-makecheck', action='store_true',
35
help='pretty print output for make check')
36
+ p.add_argument('-j', dest='jobs', type=int, default=1,
37
+ help='run tests in multiple parallel jobs')
38
39
p.add_argument('-d', dest='debug', action='store_true', help='debug')
40
p.add_argument('-p', dest='print', action='store_true',
41
@@ -XXX,XX +XXX,XX @@ if __name__ == '__main__':
42
with TestRunner(env, makecheck=args.makecheck,
43
color=args.color) as tr:
44
paths = [os.path.join(env.source_iotests, t) for t in tests]
45
- ok = tr.run_tests(paths)
46
+ ok = tr.run_tests(paths, args.jobs)
47
if not ok:
48
sys.exit(1)
49
diff --git a/tests/qemu-iotests/testrunner.py b/tests/qemu-iotests/testrunner.py
50
index XXXXXXX..XXXXXXX 100644
44
index XXXXXXX..XXXXXXX 100644
51
--- a/tests/qemu-iotests/testrunner.py
45
--- a/block/nvme.c
52
+++ b/tests/qemu-iotests/testrunner.py
46
+++ b/block/nvme.c
53
@@ -XXX,XX +XXX,XX @@
47
@@ -XXX,XX +XXX,XX @@ static void nvme_submit_command(NVMeQueuePair *q, NVMeRequest *req,
54
import json
48
q->sq.tail * NVME_SQ_ENTRY_BYTES, cmd, sizeof(*cmd));
55
import termios
49
q->sq.tail = (q->sq.tail + 1) % NVME_QUEUE_SIZE;
56
import sys
50
q->need_kick++;
57
+from multiprocessing import Pool
51
+ qemu_mutex_unlock(&q->lock);
58
from contextlib import contextmanager
59
from typing import List, Optional, Iterator, Any, Sequence, Dict, \
60
ContextManager
61
@@ -XXX,XX +XXX,XX @@ def __init__(self, status: str, description: str = '',
62
63
64
class TestRunner(ContextManager['TestRunner']):
65
+ shared_self = None
66
+
52
+
67
+ @staticmethod
53
blk_io_plug_call(nvme_unplug_fn, q);
68
+ def proc_run_test(test: str, test_field_width: int) -> TestResult:
54
- qemu_mutex_unlock(&q->lock);
69
+ # We are in a subprocess, we can't change the runner object!
55
}
70
+ runner = TestRunner.shared_self
56
71
+ assert runner is not None
57
static void nvme_admin_cmd_sync_cb(void *opaque, int ret)
72
+ return runner.run_test(test, test_field_width, mp=True)
73
+
74
+ def run_tests_pool(self, tests: List[str],
75
+ test_field_width: int, jobs: int) -> List[TestResult]:
76
+
77
+ # passing self directly to Pool.starmap() just doesn't work, because
78
+ # it's a context manager.
79
+ assert TestRunner.shared_self is None
80
+ TestRunner.shared_self = self
81
+
82
+ with Pool(jobs) as p:
83
+ results = p.starmap(self.proc_run_test,
84
+ zip(tests, [test_field_width] * len(tests)))
85
+
86
+ TestRunner.shared_self = None
87
+
88
+ return results
89
+
90
def __init__(self, env: TestEnv, makecheck: bool = False,
91
color: str = 'auto') -> None:
92
self.env = env
93
@@ -XXX,XX +XXX,XX @@ def find_reference(self, test: str) -> str:
94
95
return f'{test}.out'
96
97
- def do_run_test(self, test: str) -> TestResult:
98
+ def do_run_test(self, test: str, mp: bool) -> TestResult:
99
"""
100
Run one test
101
102
:param test: test file path
103
+ :param mp: if true, we are in a multiprocessing environment, use
104
+ personal subdirectories for test run
105
+
106
+ Note: this method may be called from subprocess, so it does not
107
+ change ``self`` object in any way!
108
"""
109
110
f_test = Path(test)
111
@@ -XXX,XX +XXX,XX @@ def do_run_test(self, test: str) -> TestResult:
112
113
args = [str(f_test.resolve())]
114
env = self.env.prepare_subprocess(args)
115
+ if mp:
116
+ # Split test directories, so that tests running in parallel don't
117
+ # break each other.
118
+ for d in ['TEST_DIR', 'SOCK_DIR']:
119
+ env[d] = os.path.join(env[d], f_test.name)
120
+ Path(env[d]).mkdir(parents=True, exist_ok=True)
121
122
t0 = time.time()
123
with f_bad.open('w', encoding="utf-8") as f:
124
@@ -XXX,XX +XXX,XX @@ def do_run_test(self, test: str) -> TestResult:
125
casenotrun=casenotrun)
126
127
def run_test(self, test: str,
128
- test_field_width: Optional[int] = None) -> TestResult:
129
+ test_field_width: Optional[int] = None,
130
+ mp: bool = False) -> TestResult:
131
"""
132
Run one test and print short status
133
134
:param test: test file path
135
:param test_field_width: width for first field of status format
136
+ :param mp: if true, we are in a multiprocessing environment, don't try
137
+ to rewrite things in stdout
138
+
139
+ Note: this method may be called from subprocess, so it does not
140
+ change ``self`` object in any way!
141
"""
142
143
last_el = self.last_elapsed.get(test)
144
start = datetime.datetime.now().strftime('%H:%M:%S')
145
146
if not self.makecheck:
147
- self.test_print_one_line(test=test, starttime=start,
148
- lasttime=last_el, end='\r',
149
+ self.test_print_one_line(test=test,
150
+ status = 'started' if mp else '...',
151
+ starttime=start,
152
+ lasttime=last_el,
153
+ end = '\n' if mp else '\r',
154
test_field_width=test_field_width)
155
156
- res = self.do_run_test(test)
157
+ res = self.do_run_test(test, mp)
158
159
end = datetime.datetime.now().strftime('%H:%M:%S')
160
self.test_print_one_line(test=test, status=res.status,
161
@@ -XXX,XX +XXX,XX @@ def run_test(self, test: str,
162
163
return res
164
165
- def run_tests(self, tests: List[str]) -> bool:
166
+ def run_tests(self, tests: List[str], jobs: int = 1) -> bool:
167
n_run = 0
168
failed = []
169
notrun = []
170
@@ -XXX,XX +XXX,XX @@ def run_tests(self, tests: List[str]) -> bool:
171
172
test_field_width = max(len(os.path.basename(t)) for t in tests) + 2
173
174
- for t in tests:
175
+ if jobs > 1:
176
+ results = self.run_tests_pool(tests, test_field_width, jobs)
177
+
178
+ for i, t in enumerate(tests):
179
name = os.path.basename(t)
180
- res = self.run_test(t, test_field_width=test_field_width)
181
+
182
+ if jobs > 1:
183
+ res = results[i]
184
+ else:
185
+ res = self.run_test(t, test_field_width)
186
187
assert res.status in ('pass', 'fail', 'not run')
188
189
--
58
--
190
2.33.1
59
2.40.1
191
60
192
61
diff view generated by jsdifflib

Patchew 2.1-devel-b67e4c2

© 2016 - 2025 Red Hat, Inc.