Add a CLI argument that takes fnmatch(3)-style patterns as value and can
be specified many times. Only tests that match the pattern will be
executed. This argument is passed to unittest.main which takes the same
argument.
Acked-by: Thomas Huth <thuth@redhat.com>
Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>
---
tests/functional/qemu_test/testcase.py | 23 +++++++++++++++++++++--
1 file changed, 21 insertions(+), 2 deletions(-)
diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py
index ab564f873c303bcc28c3bf7bec8c8c4569fae91c..b045d82caa79d9d161fb868b0b0748ad7de453d9 100644
--- a/tests/functional/qemu_test/testcase.py
+++ b/tests/functional/qemu_test/testcase.py
@@ -16,6 +16,7 @@
import os
from pathlib import Path
import pycotap
+import itertools
import shutil
from subprocess import run
import sys
@@ -37,6 +38,7 @@ class QemuBaseTest(unittest.TestCase):
debug: bool = False
keep_scratch: bool = "QEMU_TEST_KEEP_SCRATCH" in os.environ
list_tests: bool = False
+ test_name_patterns: list[str] = []
"""
Class method that initializes class attributes from given command-line
@@ -67,10 +69,19 @@ def parse_args():
action="store_true",
help="List all tests that would be executed and exit.",
)
+ parser.add_argument(
+ "-k",
+ dest="test_name_patterns",
+ action="append",
+ type=str,
+ help="Only run tests which match the given substring. "
+ "This argument is passed to unittest.main verbatim.",
+ )
args = parser.parse_args()
QemuBaseTest.debug = args.debug
QemuBaseTest.keep_scratch |= args.keep_scratch
QemuBaseTest.list_tests = args.list_tests
+ QemuBaseTest.test_name_patterns = args.test_name_patterns
return
'''
@@ -313,8 +324,16 @@ def main():
tr = pycotap.TAPTestRunner(message_log = pycotap.LogMode.LogToError,
test_output_log = pycotap.LogMode.LogToError)
- res = unittest.main(module = None, testRunner = tr, exit = False,
- argv=["__dummy__", path])
+ argv = ["__dummy__", path] + (
+ list(
+ itertools.chain.from_iterable(
+ ["-k", x] for x in QemuBaseTest.test_name_patterns
+ )
+ )
+ if QemuBaseTest.test_name_patterns
+ else []
+ )
+ res = unittest.main(module=None, testRunner=tr, exit=False, argv=argv)
for (test, message) in res.result.errors + res.result.failures:
if hasattr(test, "log_filename"):
--
2.47.2On Fri, Jul 25, 2025 at 12:41:25PM +0300, Manos Pitsidianakis wrote:
> Add a CLI argument that takes fnmatch(3)-style patterns as value and can
> be specified many times. Only tests that match the pattern will be
> executed. This argument is passed to unittest.main which takes the same
> argument.
>
> Acked-by: Thomas Huth <thuth@redhat.com>
> Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>
> ---
> tests/functional/qemu_test/testcase.py | 23 +++++++++++++++++++++--
> 1 file changed, 21 insertions(+), 2 deletions(-)
>
> diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py
> index ab564f873c303bcc28c3bf7bec8c8c4569fae91c..b045d82caa79d9d161fb868b0b0748ad7de453d9 100644
> --- a/tests/functional/qemu_test/testcase.py
> +++ b/tests/functional/qemu_test/testcase.py
> @@ -16,6 +16,7 @@
> import os
> from pathlib import Path
> import pycotap
> +import itertools
> import shutil
> from subprocess import run
> import sys
> @@ -37,6 +38,7 @@ class QemuBaseTest(unittest.TestCase):
> debug: bool = False
> keep_scratch: bool = "QEMU_TEST_KEEP_SCRATCH" in os.environ
> list_tests: bool = False
> + test_name_patterns: list[str] = []
>
> """
> Class method that initializes class attributes from given command-line
> @@ -67,10 +69,19 @@ def parse_args():
> action="store_true",
> help="List all tests that would be executed and exit.",
> )
> + parser.add_argument(
> + "-k",
> + dest="test_name_patterns",
> + action="append",
> + type=str,
> + help="Only run tests which match the given substring. "
> + "This argument is passed to unittest.main verbatim.",
> + )
> args = parser.parse_args()
> QemuBaseTest.debug = args.debug
> QemuBaseTest.keep_scratch |= args.keep_scratch
> QemuBaseTest.list_tests = args.list_tests
> + QemuBaseTest.test_name_patterns = args.test_name_patterns
> return
>
> '''
> @@ -313,8 +324,16 @@ def main():
>
> tr = pycotap.TAPTestRunner(message_log = pycotap.LogMode.LogToError,
> test_output_log = pycotap.LogMode.LogToError)
> - res = unittest.main(module = None, testRunner = tr, exit = False,
> - argv=["__dummy__", path])
> + argv = ["__dummy__", path] + (
> + list(
> + itertools.chain.from_iterable(
> + ["-k", x] for x in QemuBaseTest.test_name_patterns
> + )
> + )
> + if QemuBaseTest.test_name_patterns
> + else []
> + )
> + res = unittest.main(module=None, testRunner=tr, exit=False, argv=argv)
unittest.main() supports a whole bunch of CLI args beyond '-k', but none
of them are accessible as we're not forwarding the sys.argv that we have
received. eg we're missing
$ git diff
diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py
index 2a78e735f1..5caf7b13fe 100644
--- a/tests/functional/qemu_test/testcase.py
+++ b/tests/functional/qemu_test/testcase.py
@@ -249,7 +249,7 @@ def main():
tr = pycotap.TAPTestRunner(message_log = pycotap.LogMode.LogToError,
test_output_log = pycotap.LogMode.LogToError)
res = unittest.main(module = None, testRunner = tr, exit = False,
- argv=["__dummy__", path])
+ argv=[sys.argv[0], path] + sys.argv[1:])
for (test, message) in res.result.errors + res.result.failures:
if hasattr(test, "log_filename"):
which would unlock
$ QEMU_TEST_QEMU_BINARY=./build/qemu-system-x86_64 PYTHONPATH=`pwd`/python ./tests/functional/test_version.py -h
usage: test_version.py [-h] [-v] [-q] [--locals] [--durations N] [-f] [-c] [-b] [-k TESTNAMEPATTERNS] [tests ...]
positional arguments:
tests a list of any number of test modules, classes and test methods.
options:
-h, --help show this help message and exit
-v, --verbose Verbose output
-q, --quiet Quiet output
--locals Show local variables in tracebacks
--durations N Show the N slowest test cases (N=0 for all)
-f, --failfast Stop on first fail or error
-c, --catch Catch Ctrl-C and display results so far
-b, --buffer Buffer stdout and stderr during tests
-k TESTNAMEPATTERNS Only run tests which match the given substring
One of the goals with the new functional test system was that we stop trying
to (re-)invent a custom test runner harness, as was the case with Avocado,
in favour of relying on the pre-existing python infrastructure to the
greatest extent possible.
Seeing this, and all the other CLI arg handling added in this series, makes
me fairly uncomfortable, as it is effectively inventing a custom test runner
once again which is exactly what we wanted to get away from.
At the same time, there are some pieces in this series that do things that
unittest.main() can't do on its own.
With regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
On Fri, Jul 25, 2025 at 02:25:46PM +0100, Daniel P. Berrangé wrote: > On Fri, Jul 25, 2025 at 12:41:25PM +0300, Manos Pitsidianakis wrote: > > Add a CLI argument that takes fnmatch(3)-style patterns as value and can > > be specified many times. Only tests that match the pattern will be > > executed. This argument is passed to unittest.main which takes the same > > argument. > > > > Acked-by: Thomas Huth <thuth@redhat.com> > > Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org> > > --- > > tests/functional/qemu_test/testcase.py | 23 +++++++++++++++++++++-- > > 1 file changed, 21 insertions(+), 2 deletions(-) > One of the goals with the new functional test system was that we stop trying > to (re-)invent a custom test runner harness, as was the case with Avocado, > in favour of relying on the pre-existing python infrastructure to the > greatest extent possible. > > Seeing this, and all the other CLI arg handling added in this series, makes > me fairly uncomfortable, as it is effectively inventing a custom test runner > once again which is exactly what we wanted to get away from. > > At the same time, there are some pieces in this series that do things that > unittest.main() can't do on its own. So considering the broader picture, we already have a load of tunables on the test execution that we control exclusively via envirnoment variables, as that gives us independance of the test runner, which owns sys.argv processing. So in terms of this series, IMHO, we should just add support for QEMU_TEST_DEBUG=1 and QEMU_TEST_KEEP_SCRATCH=1 as two new tunables, and not touch sys.argv at all. With that, the only thing we're missing is a way to enumerate the test cases, but IMHO that's the job of the test runner and thus out of scope for QEMU to solve. With regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
On 25/07/2025 16.48, Daniel P. Berrangé wrote: > On Fri, Jul 25, 2025 at 02:25:46PM +0100, Daniel P. Berrangé wrote: >> On Fri, Jul 25, 2025 at 12:41:25PM +0300, Manos Pitsidianakis wrote: >>> Add a CLI argument that takes fnmatch(3)-style patterns as value and can >>> be specified many times. Only tests that match the pattern will be >>> executed. This argument is passed to unittest.main which takes the same >>> argument. >>> >>> Acked-by: Thomas Huth <thuth@redhat.com> >>> Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org> >>> --- >>> tests/functional/qemu_test/testcase.py | 23 +++++++++++++++++++++-- >>> 1 file changed, 21 insertions(+), 2 deletions(-) > > > >> One of the goals with the new functional test system was that we stop trying >> to (re-)invent a custom test runner harness, as was the case with Avocado, >> in favour of relying on the pre-existing python infrastructure to the >> greatest extent possible. >> >> Seeing this, and all the other CLI arg handling added in this series, makes >> me fairly uncomfortable, as it is effectively inventing a custom test runner >> once again which is exactly what we wanted to get away from. >> >> At the same time, there are some pieces in this series that do things that >> unittest.main() can't do on its own. > > So considering the broader picture, we already have a load of tunables > on the test execution that we control exclusively via envirnoment > variables, as that gives us independance of the test runner, which > owns sys.argv processing. > > So in terms of this series, IMHO, we should just add support for > QEMU_TEST_DEBUG=1 and QEMU_TEST_KEEP_SCRATCH=1 as two new tunables, > and not touch sys.argv at all. I basically agree, but I wonder if we could maybe also have both? First parse the sys.argv for our parameters, then pass the unknown ones to unittest.main() ? > With that, the only thing we're missing is a way to enumerate the > test cases, but IMHO that's the job of the test runner and thus > out of scope for QEMU to solve. With the patch applied that passes the argv to unittest.main(), there also seems to be a "discover" mode: ~/devel/qemu/tests/functional/test_vnc.py -h | grep usage: usage: test_vnc.py [-h] [-v] [-q] [--locals] [--durations N] [-f] [-c] [-b] usage: test_vnc.py discover [-h] [-v] [-q] [--locals] [--durations N] [-f] ... however, when I try to run it, it does not work as expected (I get a "ModuleNotFoundError: No module named 'discover'" error). Thomas
On Sat, Jul 26, 2025 at 08:54:51AM +0200, Thomas Huth wrote: > On 25/07/2025 16.48, Daniel P. Berrangé wrote: > > On Fri, Jul 25, 2025 at 02:25:46PM +0100, Daniel P. Berrangé wrote: > > > On Fri, Jul 25, 2025 at 12:41:25PM +0300, Manos Pitsidianakis wrote: > > > > Add a CLI argument that takes fnmatch(3)-style patterns as value and can > > > > be specified many times. Only tests that match the pattern will be > > > > executed. This argument is passed to unittest.main which takes the same > > > > argument. > > > > > > > > Acked-by: Thomas Huth <thuth@redhat.com> > > > > Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org> > > > > --- > > > > tests/functional/qemu_test/testcase.py | 23 +++++++++++++++++++++-- > > > > 1 file changed, 21 insertions(+), 2 deletions(-) > > > > > > > > > One of the goals with the new functional test system was that we stop trying > > > to (re-)invent a custom test runner harness, as was the case with Avocado, > > > in favour of relying on the pre-existing python infrastructure to the > > > greatest extent possible. > > > > > > Seeing this, and all the other CLI arg handling added in this series, makes > > > me fairly uncomfortable, as it is effectively inventing a custom test runner > > > once again which is exactly what we wanted to get away from. > > > > > > At the same time, there are some pieces in this series that do things that > > > unittest.main() can't do on its own. > > > > So considering the broader picture, we already have a load of tunables > > on the test execution that we control exclusively via envirnoment > > variables, as that gives us independance of the test runner, which > > owns sys.argv processing. > > > > So in terms of this series, IMHO, we should just add support for > > QEMU_TEST_DEBUG=1 and QEMU_TEST_KEEP_SCRATCH=1 as two new tunables, > > and not touch sys.argv at all. > > I basically agree, but I wonder if we could maybe also have both? First > parse the sys.argv for our parameters, then pass the unknown ones to > unittest.main() ? Any argv that we process are subject to clashing with that defined by the test harness and it is desirable to avoid making assumptions about what test harness args exist. To avoid that they would all have to be long options named with a fixed prefix --qemu-XXXX. Also that sould be something applied to all env vars, not just the couple of params in this series. > > With that, the only thing we're missing is a way to enumerate the > > test cases, but IMHO that's the job of the test runner and thus > > out of scope for QEMU to solve. > > With the patch applied that passes the argv to unittest.main(), there also > seems to be a "discover" mode: > > ~/devel/qemu/tests/functional/test_vnc.py -h | grep usage: > usage: test_vnc.py [-h] [-v] [-q] [--locals] [--durations N] [-f] [-c] [-b] > usage: test_vnc.py discover [-h] [-v] [-q] [--locals] [--durations N] [-f] > > ... however, when I try to run it, it does not work as expected (I get a > "ModuleNotFoundError: No module named 'discover'" error). Hmm, not sure that the 'discover' option makes sense when we're asking to run a single test file. With regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Manos Pitsidianakis <manos.pitsidianakis@linaro.org> writes: > Add a CLI argument that takes fnmatch(3)-style patterns as value and can > be specified many times. Only tests that match the pattern will be > executed. This argument is passed to unittest.main which takes the same > argument. > > Acked-by: Thomas Huth <thuth@redhat.com> > Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org> Tested-by: Alex Bennée <alex.bennee@linaro.org> -- Alex Bennée Virtualisation Tech Lead @ Linaro
© 2016 - 2025 Red Hat, Inc.