docs/devel/testing/functional.rst | 2 ++ tests/functional/qemu_test/testcase.py | 51 ++++++++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 3 deletions(-)
Add argument parsing to functional tests to improve developer experience
when running individual tests. All logs are printed to stdout
interspersed with TAP output.
./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help
usage: test_aarch64_virt [-h] [-d]
QEMU Functional test
options:
-h, --help show this help message and exit
-d, --debug Also print test and console logs on stdout. This will
make the TAP output invalid and is meant for debugging
only.
Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>
---
docs/devel/testing/functional.rst | 2 ++
tests/functional/qemu_test/testcase.py | 51 ++++++++++++++++++++++++++++++++--
2 files changed, 50 insertions(+), 3 deletions(-)
diff --git a/docs/devel/testing/functional.rst b/docs/devel/testing/functional.rst
index 9e56dd1b1189216b9b4aede00174c15203f38b41..9d08abe2848277d635befb0296f578cfaa4bd66d 100644
--- a/docs/devel/testing/functional.rst
+++ b/docs/devel/testing/functional.rst
@@ -63,6 +63,8 @@ directory should be your build folder. For example::
$ export QEMU_TEST_QEMU_BINARY=$PWD/qemu-system-x86_64
$ pyvenv/bin/python3 ../tests/functional/test_file.py
+By default, functional tests redirect informational logs and console output to
+log files. Specify the ``--debug`` flag to also print those to standard output.
The test framework will automatically purge any scratch files created during
the tests. If needing to debug a failed test, it is possible to keep these
files around on disk by setting ```QEMU_TEST_KEEP_SCRATCH=1``` as an env
diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py
index 2082c6fce43b0544d4e4258cd4155f555ed30cd4..fad7a946c6677e9ef5c42b8f77187ba836c11aeb 100644
--- a/tests/functional/qemu_test/testcase.py
+++ b/tests/functional/qemu_test/testcase.py
@@ -11,6 +11,7 @@
# This work is licensed under the terms of the GNU GPL, version 2 or
# later. See the COPYING file in the top-level directory.
+import argparse
import logging
import os
from pathlib import Path
@@ -31,6 +32,20 @@
from .uncompress import uncompress
+def parse_args(test_name: str) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ prog=test_name, description="QEMU Functional test"
+ )
+ parser.add_argument(
+ "-d",
+ "--debug",
+ action="store_true",
+ help="Also print test and console logs on stdout. This will make the"
+ " TAP output invalid and is meant for debugging only.",
+ )
+ return parser.parse_args()
+
+
class QemuBaseTest(unittest.TestCase):
'''
@@ -196,6 +211,9 @@ def assets_available(self):
return True
def setUp(self):
+ path = os.path.basename(sys.argv[0])[:-3]
+ args = parse_args(path)
+ self.debug_output = args.debug
self.qemu_bin = os.getenv('QEMU_TEST_QEMU_BINARY')
self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set')
self.arch = self.qemu_bin.split('-')[-1]
@@ -221,6 +239,16 @@ def setUp(self):
self.machinelog.setLevel(logging.DEBUG)
self.machinelog.addHandler(self._log_fh)
+ if self.debug_output:
+ handler = logging.StreamHandler(sys.stdout)
+ handler.setLevel(logging.DEBUG)
+ formatter = logging.Formatter(
+ "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+ )
+ handler.setFormatter(formatter)
+ self.log.addHandler(handler)
+ self.machinelog.addHandler(handler)
+
if not self.assets_available():
self.skipTest('One or more assets is not available')
@@ -230,11 +258,16 @@ def tearDown(self):
if self.socketdir is not None:
shutil.rmtree(self.socketdir.name)
self.socketdir = None
- self.machinelog.removeHandler(self._log_fh)
- self.log.removeHandler(self._log_fh)
+ for handler in [self._log_fh, logging.StreamHandler(sys.stdout)]:
+ self.machinelog.removeHandler(handler)
+ self.log.removeHandler(handler)
def main():
path = os.path.basename(sys.argv[0])[:-3]
+ # If argparse receives --help or an unknown argument, it will raise a
+ # SystemExit which will get caught by the test runner. Parse the
+ # arguments here too to handle that case.
+ _ = parse_args(path)
cache = os.environ.get("QEMU_TEST_PRECACHE", None)
if cache is not None:
@@ -292,6 +325,14 @@ def setUp(self):
fileFormatter = logging.Formatter('%(asctime)s: %(message)s')
self._console_log_fh.setFormatter(fileFormatter)
console_log.addHandler(self._console_log_fh)
+ if self.debug_output:
+ handler = logging.StreamHandler(sys.stdout)
+ handler.setLevel(logging.DEBUG)
+ formatter = logging.Formatter(
+ "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+ )
+ handler.setFormatter(formatter)
+ console_log.addHandler(handler)
def set_machine(self, machinename):
# TODO: We should use QMP to get the list of available machines
@@ -398,5 +439,9 @@ def set_vm_arg(self, arg, value):
def tearDown(self):
for vm in self._vms.values():
vm.shutdown()
- logging.getLogger('console').removeHandler(self._console_log_fh)
+ for handler in [
+ self._console_log_fh,
+ logging.StreamHandler(sys.stdout),
+ ]:
+ logging.getLogger("console").removeHandler(handler)
super().tearDown()
---
base-commit: c079d3a31e45093286c65f8ca5350beb3a4404a9
change-id: 20250716-functional_tests_debug_arg-aa0a5f6b9375
--
γαῖα πυρί μιχθήτω
On Wed, Jul 16, 2025 at 09:08:00AM +0300, Manos Pitsidianakis wrote:
> Add argument parsing to functional tests to improve developer experience
> when running individual tests. All logs are printed to stdout
> interspersed with TAP output.
>
> ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help
> usage: test_aarch64_virt [-h] [-d]
>
> QEMU Functional test
>
> options:
> -h, --help show this help message and exit
> -d, --debug Also print test and console logs on stdout. This will
> make the TAP output invalid and is meant for debugging
> only.
>
> Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>
> ---
> docs/devel/testing/functional.rst | 2 ++
> tests/functional/qemu_test/testcase.py | 51 ++++++++++++++++++++++++++++++++--
> 2 files changed, 50 insertions(+), 3 deletions(-)
>
> diff --git a/docs/devel/testing/functional.rst b/docs/devel/testing/functional.rst
> index 9e56dd1b1189216b9b4aede00174c15203f38b41..9d08abe2848277d635befb0296f578cfaa4bd66d 100644
> --- a/docs/devel/testing/functional.rst
> +++ b/docs/devel/testing/functional.rst
> @@ -63,6 +63,8 @@ directory should be your build folder. For example::
> $ export QEMU_TEST_QEMU_BINARY=$PWD/qemu-system-x86_64
> $ pyvenv/bin/python3 ../tests/functional/test_file.py
>
> +By default, functional tests redirect informational logs and console output to
> +log files. Specify the ``--debug`` flag to also print those to standard output.
> The test framework will automatically purge any scratch files created during
> the tests. If needing to debug a failed test, it is possible to keep these
> files around on disk by setting ```QEMU_TEST_KEEP_SCRATCH=1``` as an env
> diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py
> index 2082c6fce43b0544d4e4258cd4155f555ed30cd4..fad7a946c6677e9ef5c42b8f77187ba836c11aeb 100644
> --- a/tests/functional/qemu_test/testcase.py
> +++ b/tests/functional/qemu_test/testcase.py
> @@ -11,6 +11,7 @@
> # This work is licensed under the terms of the GNU GPL, version 2 or
> # later. See the COPYING file in the top-level directory.
>
> +import argparse
> import logging
> import os
> from pathlib import Path
> @@ -31,6 +32,20 @@
> from .uncompress import uncompress
>
>
> +def parse_args(test_name: str) -> argparse.Namespace:
> + parser = argparse.ArgumentParser(
> + prog=test_name, description="QEMU Functional test"
> + )
> + parser.add_argument(
> + "-d",
> + "--debug",
> + action="store_true",
> + help="Also print test and console logs on stdout. This will make the"
> + " TAP output invalid and is meant for debugging only.",
> + )
> + return parser.parse_args()
> +
> +
> class QemuBaseTest(unittest.TestCase):
>
> '''
> @@ -196,6 +211,9 @@ def assets_available(self):
> return True
>
> def setUp(self):
> + path = os.path.basename(sys.argv[0])[:-3]
> + args = parse_args(path)
IMHO this is not code that belongs in setUp. Indeed, I don't think
it belongs in this file at all, better have a helper for parsing
args in 'utils', and expose a global 'debug_enabled' flag from utils
that we can reference elsewhere.
> + self.debug_output = args.debug
> self.qemu_bin = os.getenv('QEMU_TEST_QEMU_BINARY')
> self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set')
> self.arch = self.qemu_bin.split('-')[-1]
> @@ -221,6 +239,16 @@ def setUp(self):
> self.machinelog.setLevel(logging.DEBUG)
> self.machinelog.addHandler(self._log_fh)
>
> + if self.debug_output:
> + handler = logging.StreamHandler(sys.stdout)
> + handler.setLevel(logging.DEBUG)
> + formatter = logging.Formatter(
> + "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
> + )
> + handler.setFormatter(formatter)
> + self.log.addHandler(handler)
> + self.machinelog.addHandler(handler)
There was already a lot of effectively duplicated code between
this and the console_log stuff below. This new addition makes
that duplication even more substantial, such that I think it
all needs to be spun out into a helper method.
> +
> if not self.assets_available():
> self.skipTest('One or more assets is not available')
>
> @@ -230,11 +258,16 @@ def tearDown(self):
> if self.socketdir is not None:
> shutil.rmtree(self.socketdir.name)
> self.socketdir = None
> - self.machinelog.removeHandler(self._log_fh)
> - self.log.removeHandler(self._log_fh)
> + for handler in [self._log_fh, logging.StreamHandler(sys.stdout)]:
We should have stashed the original handler when created,
rather than re-creating StreamHandler at time of removal.
I'm kinda of surprised it even works to re-create it.
> + self.machinelog.removeHandler(handler)
> + self.log.removeHandler(handler)
>
> def main():
> path = os.path.basename(sys.argv[0])[:-3]
> + # If argparse receives --help or an unknown argument, it will raise a
> + # SystemExit which will get caught by the test runner. Parse the
> + # arguments here too to handle that case.
> + _ = parse_args(path)
>
> cache = os.environ.get("QEMU_TEST_PRECACHE", None)
> if cache is not None:
> @@ -292,6 +325,14 @@ def setUp(self):
> fileFormatter = logging.Formatter('%(asctime)s: %(message)s')
> self._console_log_fh.setFormatter(fileFormatter)
> console_log.addHandler(self._console_log_fh)
> + if self.debug_output:
> + handler = logging.StreamHandler(sys.stdout)
> + handler.setLevel(logging.DEBUG)
> + formatter = logging.Formatter(
> + "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
> + )
> + handler.setFormatter(formatter)
> + console_log.addHandler(handler)
>
> def set_machine(self, machinename):
> # TODO: We should use QMP to get the list of available machines
> @@ -398,5 +439,9 @@ def set_vm_arg(self, arg, value):
> def tearDown(self):
> for vm in self._vms.values():
> vm.shutdown()
> - logging.getLogger('console').removeHandler(self._console_log_fh)
> + for handler in [
> + self._console_log_fh,
> + logging.StreamHandler(sys.stdout),
> + ]:
> + logging.getLogger("console").removeHandler(handler)
> super().tearDown()
>
> ---
> base-commit: c079d3a31e45093286c65f8ca5350beb3a4404a9
> change-id: 20250716-functional_tests_debug_arg-aa0a5f6b9375
>
> --
> γαῖα πυρί μιχθήτω
>
With regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
On Thu, Jul 17, 2025 at 12:22 PM Daniel P. Berrangé <berrange@redhat.com> wrote:
>
> On Wed, Jul 16, 2025 at 09:08:00AM +0300, Manos Pitsidianakis wrote:
> > Add argument parsing to functional tests to improve developer experience
> > when running individual tests. All logs are printed to stdout
> > interspersed with TAP output.
> >
> > ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help
> > usage: test_aarch64_virt [-h] [-d]
> >
> > QEMU Functional test
> >
> > options:
> > -h, --help show this help message and exit
> > -d, --debug Also print test and console logs on stdout. This will
> > make the TAP output invalid and is meant for debugging
> > only.
> >
> > Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>
> > ---
> > docs/devel/testing/functional.rst | 2 ++
> > tests/functional/qemu_test/testcase.py | 51 ++++++++++++++++++++++++++++++++--
> > 2 files changed, 50 insertions(+), 3 deletions(-)
> >
> > diff --git a/docs/devel/testing/functional.rst b/docs/devel/testing/functional.rst
> > index 9e56dd1b1189216b9b4aede00174c15203f38b41..9d08abe2848277d635befb0296f578cfaa4bd66d 100644
> > --- a/docs/devel/testing/functional.rst
> > +++ b/docs/devel/testing/functional.rst
> > @@ -63,6 +63,8 @@ directory should be your build folder. For example::
> > $ export QEMU_TEST_QEMU_BINARY=$PWD/qemu-system-x86_64
> > $ pyvenv/bin/python3 ../tests/functional/test_file.py
> >
> > +By default, functional tests redirect informational logs and console output to
> > +log files. Specify the ``--debug`` flag to also print those to standard output.
> > The test framework will automatically purge any scratch files created during
> > the tests. If needing to debug a failed test, it is possible to keep these
> > files around on disk by setting ```QEMU_TEST_KEEP_SCRATCH=1``` as an env
> > diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py
> > index 2082c6fce43b0544d4e4258cd4155f555ed30cd4..fad7a946c6677e9ef5c42b8f77187ba836c11aeb 100644
> > --- a/tests/functional/qemu_test/testcase.py
> > +++ b/tests/functional/qemu_test/testcase.py
> > @@ -11,6 +11,7 @@
> > # This work is licensed under the terms of the GNU GPL, version 2 or
> > # later. See the COPYING file in the top-level directory.
> >
> > +import argparse
> > import logging
> > import os
> > from pathlib import Path
> > @@ -31,6 +32,20 @@
> > from .uncompress import uncompress
> >
> >
> > +def parse_args(test_name: str) -> argparse.Namespace:
> > + parser = argparse.ArgumentParser(
> > + prog=test_name, description="QEMU Functional test"
> > + )
> > + parser.add_argument(
> > + "-d",
> > + "--debug",
> > + action="store_true",
> > + help="Also print test and console logs on stdout. This will make the"
> > + " TAP output invalid and is meant for debugging only.",
> > + )
> > + return parser.parse_args()
> > +
> > +
> > class QemuBaseTest(unittest.TestCase):
> >
> > '''
> > @@ -196,6 +211,9 @@ def assets_available(self):
> > return True
> >
> > def setUp(self):
> > + path = os.path.basename(sys.argv[0])[:-3]
> > + args = parse_args(path)
>
> IMHO this is not code that belongs in setUp. Indeed, I don't think
> it belongs in this file at all, better have a helper for parsing
> args in 'utils', and expose a global 'debug_enabled' flag from utils
> that we can reference elsewhere.
setUp is where the logs are setup, do you mean logs should be split
into another function/file altogether? Maybe out of scope for this
patch, I can another one that does it before adding the --debug
option. What do you think?
>
> > + self.debug_output = args.debug
> > self.qemu_bin = os.getenv('QEMU_TEST_QEMU_BINARY')
> > self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set')
> > self.arch = self.qemu_bin.split('-')[-1]
> > @@ -221,6 +239,16 @@ def setUp(self):
> > self.machinelog.setLevel(logging.DEBUG)
> > self.machinelog.addHandler(self._log_fh)
> >
> > + if self.debug_output:
> > + handler = logging.StreamHandler(sys.stdout)
> > + handler.setLevel(logging.DEBUG)
> > + formatter = logging.Formatter(
> > + "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
> > + )
> > + handler.setFormatter(formatter)
> > + self.log.addHandler(handler)
> > + self.machinelog.addHandler(handler)
>
> There was already a lot of effectively duplicated code between
> this and the console_log stuff below. This new addition makes
> that duplication even more substantial, such that I think it
> all needs to be spun out into a helper method.
Ditto
>
> > +
> > if not self.assets_available():
> > self.skipTest('One or more assets is not available')
> >
> > @@ -230,11 +258,16 @@ def tearDown(self):
> > if self.socketdir is not None:
> > shutil.rmtree(self.socketdir.name)
> > self.socketdir = None
> > - self.machinelog.removeHandler(self._log_fh)
> > - self.log.removeHandler(self._log_fh)
> > + for handler in [self._log_fh, logging.StreamHandler(sys.stdout)]:
>
> We should have stashed the original handler when created,
> rather than re-creating StreamHandler at time of removal.
> I'm kinda of surprised it even works to re-create it.
It's the same file descriptor, after all. I'd be surprised if it didn't work.
>
> > + self.machinelog.removeHandler(handler)
> > + self.log.removeHandler(handler)
> >
> > def main():
> > path = os.path.basename(sys.argv[0])[:-3]
> > + # If argparse receives --help or an unknown argument, it will raise a
> > + # SystemExit which will get caught by the test runner. Parse the
> > + # arguments here too to handle that case.
> > + _ = parse_args(path)
> >
> > cache = os.environ.get("QEMU_TEST_PRECACHE", None)
> > if cache is not None:
> > @@ -292,6 +325,14 @@ def setUp(self):
> > fileFormatter = logging.Formatter('%(asctime)s: %(message)s')
> > self._console_log_fh.setFormatter(fileFormatter)
> > console_log.addHandler(self._console_log_fh)
> > + if self.debug_output:
> > + handler = logging.StreamHandler(sys.stdout)
> > + handler.setLevel(logging.DEBUG)
> > + formatter = logging.Formatter(
> > + "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
> > + )
> > + handler.setFormatter(formatter)
> > + console_log.addHandler(handler)
> >
> > def set_machine(self, machinename):
> > # TODO: We should use QMP to get the list of available machines
> > @@ -398,5 +439,9 @@ def set_vm_arg(self, arg, value):
> > def tearDown(self):
> > for vm in self._vms.values():
> > vm.shutdown()
> > - logging.getLogger('console').removeHandler(self._console_log_fh)
> > + for handler in [
> > + self._console_log_fh,
> > + logging.StreamHandler(sys.stdout),
> > + ]:
> > + logging.getLogger("console").removeHandler(handler)
> > super().tearDown()
> >
> > ---
> > base-commit: c079d3a31e45093286c65f8ca5350beb3a4404a9
> > change-id: 20250716-functional_tests_debug_arg-aa0a5f6b9375
> >
> > --
> > γαῖα πυρί μιχθήτω
> >
>
> With regards,
> Daniel
> --
> |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
> |: https://libvirt.org -o- https://fstop138.berrange.com :|
> |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
>
--
Manos Pitsidianakis
Emulation and Virtualization Engineer at Linaro Ltd
On Thu, Jul 17, 2025 at 12:27:08PM +0300, Manos Pitsidianakis wrote:
> On Thu, Jul 17, 2025 at 12:22 PM Daniel P. Berrangé <berrange@redhat.com> wrote:
> >
> > On Wed, Jul 16, 2025 at 09:08:00AM +0300, Manos Pitsidianakis wrote:
> > > Add argument parsing to functional tests to improve developer experience
> > > when running individual tests. All logs are printed to stdout
> > > interspersed with TAP output.
> > >
> > > ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help
> > > usage: test_aarch64_virt [-h] [-d]
> > >
> > > QEMU Functional test
> > >
> > > options:
> > > -h, --help show this help message and exit
> > > -d, --debug Also print test and console logs on stdout. This will
> > > make the TAP output invalid and is meant for debugging
> > > only.
> > >
> > > Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>
> > > ---
> > > docs/devel/testing/functional.rst | 2 ++
> > > tests/functional/qemu_test/testcase.py | 51 ++++++++++++++++++++++++++++++++--
> > > 2 files changed, 50 insertions(+), 3 deletions(-)
> > >
> > > diff --git a/docs/devel/testing/functional.rst b/docs/devel/testing/functional.rst
> > > index 9e56dd1b1189216b9b4aede00174c15203f38b41..9d08abe2848277d635befb0296f578cfaa4bd66d 100644
> > > --- a/docs/devel/testing/functional.rst
> > > +++ b/docs/devel/testing/functional.rst
> > > @@ -63,6 +63,8 @@ directory should be your build folder. For example::
> > > $ export QEMU_TEST_QEMU_BINARY=$PWD/qemu-system-x86_64
> > > $ pyvenv/bin/python3 ../tests/functional/test_file.py
> > >
> > > +By default, functional tests redirect informational logs and console output to
> > > +log files. Specify the ``--debug`` flag to also print those to standard output.
> > > The test framework will automatically purge any scratch files created during
> > > the tests. If needing to debug a failed test, it is possible to keep these
> > > files around on disk by setting ```QEMU_TEST_KEEP_SCRATCH=1``` as an env
> > > diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py
> > > index 2082c6fce43b0544d4e4258cd4155f555ed30cd4..fad7a946c6677e9ef5c42b8f77187ba836c11aeb 100644
> > > --- a/tests/functional/qemu_test/testcase.py
> > > +++ b/tests/functional/qemu_test/testcase.py
> > > @@ -11,6 +11,7 @@
> > > # This work is licensed under the terms of the GNU GPL, version 2 or
> > > # later. See the COPYING file in the top-level directory.
> > >
> > > +import argparse
> > > import logging
> > > import os
> > > from pathlib import Path
> > > @@ -31,6 +32,20 @@
> > > from .uncompress import uncompress
> > >
> > >
> > > +def parse_args(test_name: str) -> argparse.Namespace:
> > > + parser = argparse.ArgumentParser(
> > > + prog=test_name, description="QEMU Functional test"
> > > + )
> > > + parser.add_argument(
> > > + "-d",
> > > + "--debug",
> > > + action="store_true",
> > > + help="Also print test and console logs on stdout. This will make the"
> > > + " TAP output invalid and is meant for debugging only.",
> > > + )
> > > + return parser.parse_args()
> > > +
> > > +
> > > class QemuBaseTest(unittest.TestCase):
> > >
> > > '''
> > > @@ -196,6 +211,9 @@ def assets_available(self):
> > > return True
> > >
> > > def setUp(self):
> > > + path = os.path.basename(sys.argv[0])[:-3]
> > > + args = parse_args(path)
> >
> > IMHO this is not code that belongs in setUp. Indeed, I don't think
> > it belongs in this file at all, better have a helper for parsing
> > args in 'utils', and expose a global 'debug_enabled' flag from utils
> > that we can reference elsewhere.
>
> setUp is where the logs are setup, do you mean logs should be split
> into another function/file altogether? Maybe out of scope for this
> patch, I can another one that does it before adding the --debug
> option. What do you think?
I'm saying 'parse_args' should be in util.py
> > > + self.debug_output = args.debug
> > > self.qemu_bin = os.getenv('QEMU_TEST_QEMU_BINARY')
> > > self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set')
> > > self.arch = self.qemu_bin.split('-')[-1]
> > > @@ -221,6 +239,16 @@ def setUp(self):
> > > self.machinelog.setLevel(logging.DEBUG)
> > > self.machinelog.addHandler(self._log_fh)
> > >
> > > + if self.debug_output:
> > > + handler = logging.StreamHandler(sys.stdout)
> > > + handler.setLevel(logging.DEBUG)
> > > + formatter = logging.Formatter(
> > > + "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
> > > + )
> > > + handler.setFormatter(formatter)
> > > + self.log.addHandler(handler)
> > > + self.machinelog.addHandler(handler)
> >
> > There was already a lot of effectively duplicated code between
> > this and the console_log stuff below. This new addition makes
> > that duplication even more substantial, such that I think it
> > all needs to be spun out into a helper method.
>
> Ditto
This should be a method we can call like:
handlers = create_loggers("base.log", "qemu-test",
"%(asctime)s - %(levelname)s - %(message)s")
which is able to return multiple handlers. Initially it would just
return the current "self.log_fh" handler, and then this patch would
extend it to return a second handler for the console stream when
'--debug' is set.
> > > @@ -230,11 +258,16 @@ def tearDown(self):
> > > if self.socketdir is not None:
> > > shutil.rmtree(self.socketdir.name)
> > > self.socketdir = None
> > > - self.machinelog.removeHandler(self._log_fh)
> > > - self.log.removeHandler(self._log_fh)
> > > + for handler in [self._log_fh, logging.StreamHandler(sys.stdout)]:
> >
> > We should have stashed the original handler when created,
> > rather than re-creating StreamHandler at time of removal.
> > I'm kinda of surprised it even works to re-create it.
>
> It's the same file descriptor, after all. I'd be surprised if it didn't work.
log.removeHandler has no visibility of the file descriptiors.
It will just compare the python handler object instances, which
I very much doubt will match with this code
With regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
On Thu, Jul 17, 2025 at 12:39 PM Daniel P. Berrangé <berrange@redhat.com> wrote:
>
> On Thu, Jul 17, 2025 at 12:27:08PM +0300, Manos Pitsidianakis wrote:
> > On Thu, Jul 17, 2025 at 12:22 PM Daniel P. Berrangé <berrange@redhat.com> wrote:
> > >
> > > On Wed, Jul 16, 2025 at 09:08:00AM +0300, Manos Pitsidianakis wrote:
> > > > Add argument parsing to functional tests to improve developer experience
> > > > when running individual tests. All logs are printed to stdout
> > > > interspersed with TAP output.
> > > >
> > > > ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help
> > > > usage: test_aarch64_virt [-h] [-d]
> > > >
> > > > QEMU Functional test
> > > >
> > > > options:
> > > > -h, --help show this help message and exit
> > > > -d, --debug Also print test and console logs on stdout. This will
> > > > make the TAP output invalid and is meant for debugging
> > > > only.
> > > >
> > > > Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>
> > > > ---
> > > > docs/devel/testing/functional.rst | 2 ++
> > > > tests/functional/qemu_test/testcase.py | 51 ++++++++++++++++++++++++++++++++--
> > > > 2 files changed, 50 insertions(+), 3 deletions(-)
> > > >
> > > > diff --git a/docs/devel/testing/functional.rst b/docs/devel/testing/functional.rst
> > > > index 9e56dd1b1189216b9b4aede00174c15203f38b41..9d08abe2848277d635befb0296f578cfaa4bd66d 100644
> > > > --- a/docs/devel/testing/functional.rst
> > > > +++ b/docs/devel/testing/functional.rst
> > > > @@ -63,6 +63,8 @@ directory should be your build folder. For example::
> > > > $ export QEMU_TEST_QEMU_BINARY=$PWD/qemu-system-x86_64
> > > > $ pyvenv/bin/python3 ../tests/functional/test_file.py
> > > >
> > > > +By default, functional tests redirect informational logs and console output to
> > > > +log files. Specify the ``--debug`` flag to also print those to standard output.
> > > > The test framework will automatically purge any scratch files created during
> > > > the tests. If needing to debug a failed test, it is possible to keep these
> > > > files around on disk by setting ```QEMU_TEST_KEEP_SCRATCH=1``` as an env
> > > > diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py
> > > > index 2082c6fce43b0544d4e4258cd4155f555ed30cd4..fad7a946c6677e9ef5c42b8f77187ba836c11aeb 100644
> > > > --- a/tests/functional/qemu_test/testcase.py
> > > > +++ b/tests/functional/qemu_test/testcase.py
> > > > @@ -11,6 +11,7 @@
> > > > # This work is licensed under the terms of the GNU GPL, version 2 or
> > > > # later. See the COPYING file in the top-level directory.
> > > >
> > > > +import argparse
> > > > import logging
> > > > import os
> > > > from pathlib import Path
> > > > @@ -31,6 +32,20 @@
> > > > from .uncompress import uncompress
> > > >
> > > >
> > > > +def parse_args(test_name: str) -> argparse.Namespace:
> > > > + parser = argparse.ArgumentParser(
> > > > + prog=test_name, description="QEMU Functional test"
> > > > + )
> > > > + parser.add_argument(
> > > > + "-d",
> > > > + "--debug",
> > > > + action="store_true",
> > > > + help="Also print test and console logs on stdout. This will make the"
> > > > + " TAP output invalid and is meant for debugging only.",
> > > > + )
> > > > + return parser.parse_args()
> > > > +
> > > > +
> > > > class QemuBaseTest(unittest.TestCase):
> > > >
> > > > '''
> > > > @@ -196,6 +211,9 @@ def assets_available(self):
> > > > return True
> > > >
> > > > def setUp(self):
> > > > + path = os.path.basename(sys.argv[0])[:-3]
> > > > + args = parse_args(path)
> > >
> > > IMHO this is not code that belongs in setUp. Indeed, I don't think
> > > it belongs in this file at all, better have a helper for parsing
> > > args in 'utils', and expose a global 'debug_enabled' flag from utils
> > > that we can reference elsewhere.
> >
> > setUp is where the logs are setup, do you mean logs should be split
> > into another function/file altogether? Maybe out of scope for this
> > patch, I can another one that does it before adding the --debug
> > option. What do you think?
>
> I'm saying 'parse_args' should be in util.py
>
> > > > + self.debug_output = args.debug
> > > > self.qemu_bin = os.getenv('QEMU_TEST_QEMU_BINARY')
> > > > self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set')
> > > > self.arch = self.qemu_bin.split('-')[-1]
> > > > @@ -221,6 +239,16 @@ def setUp(self):
> > > > self.machinelog.setLevel(logging.DEBUG)
> > > > self.machinelog.addHandler(self._log_fh)
> > > >
> > > > + if self.debug_output:
> > > > + handler = logging.StreamHandler(sys.stdout)
> > > > + handler.setLevel(logging.DEBUG)
> > > > + formatter = logging.Formatter(
> > > > + "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
> > > > + )
> > > > + handler.setFormatter(formatter)
> > > > + self.log.addHandler(handler)
> > > > + self.machinelog.addHandler(handler)
> > >
> > > There was already a lot of effectively duplicated code between
> > > this and the console_log stuff below. This new addition makes
> > > that duplication even more substantial, such that I think it
> > > all needs to be spun out into a helper method.
> >
> > Ditto
>
> This should be a method we can call like:
>
> handlers = create_loggers("base.log", "qemu-test",
> "%(asctime)s - %(levelname)s - %(message)s")
>
> which is able to return multiple handlers. Initially it would just
> return the current "self.log_fh" handler, and then this patch would
> extend it to return a second handler for the console stream when
> '--debug' is set.
>
>
> > > > @@ -230,11 +258,16 @@ def tearDown(self):
> > > > if self.socketdir is not None:
> > > > shutil.rmtree(self.socketdir.name)
> > > > self.socketdir = None
> > > > - self.machinelog.removeHandler(self._log_fh)
> > > > - self.log.removeHandler(self._log_fh)
> > > > + for handler in [self._log_fh, logging.StreamHandler(sys.stdout)]:
> > >
> > > We should have stashed the original handler when created,
> > > rather than re-creating StreamHandler at time of removal.
> > > I'm kinda of surprised it even works to re-create it.
> >
> > It's the same file descriptor, after all. I'd be surprised if it didn't work.
>
> log.removeHandler has no visibility of the file descriptiors.
>
> It will just compare the python handler object instances, which
> I very much doubt will match with this code
>
Hm probably, yeah. I will send a v2 that stores the handler.
Thanks,
--
Manos Pitsidianakis
Emulation and Virtualization Engineer at Linaro Ltd
Manos Pitsidianakis <manos.pitsidianakis@linaro.org> writes:
> Add argument parsing to functional tests to improve developer experience
> when running individual tests. All logs are printed to stdout
> interspersed with TAP output.
>
> ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help
> usage: test_aarch64_virt [-h] [-d]
Am I holding it wrong?
➜ ./pyvenv/bin/python ../../tests/functional/test_aarch64_virt.py --help
Traceback (most recent call last):
File "/home/alex/lsrc/qemu.git/builds/all/../../tests/functional/test_aarch64_virt.py", line 16, in <module>
from qemu_test import QemuSystemTest, Asset, exec_command_and_wait_for_pattern
File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/__init__.py", line 14, in <module>
from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest
File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/testcase.py", line 26, in <module>
from qemu.machine import QEMUMachine
ModuleNotFoundError: No module named 'qemu'
I thought the point of the venv is we had all the modules we need
automatically available to the PYTHONPATH?
>
> QEMU Functional test
>
> options:
> -h, --help show this help message and exit
> -d, --debug Also print test and console logs on stdout. This will
> make the TAP output invalid and is meant for debugging
> only.
>
> Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>
> ---
> docs/devel/testing/functional.rst | 2 ++
> tests/functional/qemu_test/testcase.py | 51 ++++++++++++++++++++++++++++++++--
> 2 files changed, 50 insertions(+), 3 deletions(-)
>
> diff --git a/docs/devel/testing/functional.rst b/docs/devel/testing/functional.rst
> index 9e56dd1b1189216b9b4aede00174c15203f38b41..9d08abe2848277d635befb0296f578cfaa4bd66d 100644
> --- a/docs/devel/testing/functional.rst
> +++ b/docs/devel/testing/functional.rst
> @@ -63,6 +63,8 @@ directory should be your build folder. For example::
> $ export QEMU_TEST_QEMU_BINARY=$PWD/qemu-system-x86_64
> $ pyvenv/bin/python3 ../tests/functional/test_file.py
>
> +By default, functional tests redirect informational logs and console output to
> +log files. Specify the ``--debug`` flag to also print those to standard output.
> The test framework will automatically purge any scratch files created during
> the tests. If needing to debug a failed test, it is possible to keep these
> files around on disk by setting ```QEMU_TEST_KEEP_SCRATCH=1``` as an env
> diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py
> index 2082c6fce43b0544d4e4258cd4155f555ed30cd4..fad7a946c6677e9ef5c42b8f77187ba836c11aeb 100644
> --- a/tests/functional/qemu_test/testcase.py
> +++ b/tests/functional/qemu_test/testcase.py
> @@ -11,6 +11,7 @@
> # This work is licensed under the terms of the GNU GPL, version 2 or
> # later. See the COPYING file in the top-level directory.
>
> +import argparse
> import logging
> import os
> from pathlib import Path
> @@ -31,6 +32,20 @@
> from .uncompress import uncompress
>
>
> +def parse_args(test_name: str) -> argparse.Namespace:
> + parser = argparse.ArgumentParser(
> + prog=test_name, description="QEMU Functional test"
> + )
> + parser.add_argument(
> + "-d",
> + "--debug",
> + action="store_true",
> + help="Also print test and console logs on stdout. This will make the"
> + " TAP output invalid and is meant for debugging only.",
> + )
> + return parser.parse_args()
> +
> +
> class QemuBaseTest(unittest.TestCase):
>
> '''
> @@ -196,6 +211,9 @@ def assets_available(self):
> return True
>
> def setUp(self):
> + path = os.path.basename(sys.argv[0])[:-3]
> + args = parse_args(path)
> + self.debug_output = args.debug
> self.qemu_bin = os.getenv('QEMU_TEST_QEMU_BINARY')
> self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set')
> self.arch = self.qemu_bin.split('-')[-1]
> @@ -221,6 +239,16 @@ def setUp(self):
> self.machinelog.setLevel(logging.DEBUG)
> self.machinelog.addHandler(self._log_fh)
>
> + if self.debug_output:
> + handler = logging.StreamHandler(sys.stdout)
> + handler.setLevel(logging.DEBUG)
> + formatter = logging.Formatter(
> + "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
> + )
> + handler.setFormatter(formatter)
> + self.log.addHandler(handler)
> + self.machinelog.addHandler(handler)
> +
> if not self.assets_available():
> self.skipTest('One or more assets is not available')
>
> @@ -230,11 +258,16 @@ def tearDown(self):
> if self.socketdir is not None:
> shutil.rmtree(self.socketdir.name)
> self.socketdir = None
> - self.machinelog.removeHandler(self._log_fh)
> - self.log.removeHandler(self._log_fh)
> + for handler in [self._log_fh, logging.StreamHandler(sys.stdout)]:
> + self.machinelog.removeHandler(handler)
> + self.log.removeHandler(handler)
>
> def main():
> path = os.path.basename(sys.argv[0])[:-3]
> + # If argparse receives --help or an unknown argument, it will raise a
> + # SystemExit which will get caught by the test runner. Parse the
> + # arguments here too to handle that case.
> + _ = parse_args(path)
>
> cache = os.environ.get("QEMU_TEST_PRECACHE", None)
> if cache is not None:
> @@ -292,6 +325,14 @@ def setUp(self):
> fileFormatter = logging.Formatter('%(asctime)s: %(message)s')
> self._console_log_fh.setFormatter(fileFormatter)
> console_log.addHandler(self._console_log_fh)
> + if self.debug_output:
> + handler = logging.StreamHandler(sys.stdout)
> + handler.setLevel(logging.DEBUG)
> + formatter = logging.Formatter(
> + "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
> + )
> + handler.setFormatter(formatter)
> + console_log.addHandler(handler)
>
> def set_machine(self, machinename):
> # TODO: We should use QMP to get the list of available machines
> @@ -398,5 +439,9 @@ def set_vm_arg(self, arg, value):
> def tearDown(self):
> for vm in self._vms.values():
> vm.shutdown()
> - logging.getLogger('console').removeHandler(self._console_log_fh)
> + for handler in [
> + self._console_log_fh,
> + logging.StreamHandler(sys.stdout),
> + ]:
> + logging.getLogger("console").removeHandler(handler)
> super().tearDown()
>
> ---
> base-commit: c079d3a31e45093286c65f8ca5350beb3a4404a9
> change-id: 20250716-functional_tests_debug_arg-aa0a5f6b9375
--
Alex Bennée
Virtualisation Tech Lead @ Linaro
On Thu, Jul 17, 2025 at 4:44 AM Alex Bennée <alex.bennee@linaro.org> wrote:
>
> Manos Pitsidianakis <manos.pitsidianakis@linaro.org> writes:
>
> > Add argument parsing to functional tests to improve developer experience
> > when running individual tests. All logs are printed to stdout
> > interspersed with TAP output.
> >
> > ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help
> > usage: test_aarch64_virt [-h] [-d]
>
> Am I holding it wrong?
>
> ➜ ./pyvenv/bin/python ../../tests/functional/test_aarch64_virt.py --help
> Traceback (most recent call last):
> File "/home/alex/lsrc/qemu.git/builds/all/../../tests/functional/test_aarch64_virt.py", line 16, in <module>
> from qemu_test import QemuSystemTest, Asset, exec_command_and_wait_for_pattern
> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/__init__.py", line 14, in <module>
> from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest
> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/testcase.py", line 26, in <module>
> from qemu.machine import QEMUMachine
> ModuleNotFoundError: No module named 'qemu'
>
> I thought the point of the venv is we had all the modules we need
> automatically available to the PYTHONPATH?
As Thomas points out, "qemu" is special since it's already in the
tree. There has been some dragging-of-feet by yours-truly because
installing the "qemu" module by default when running configure
introduces a considerable startup lag time, and the module is not
actually needed for the simple configuration and building of QEMU -
only testing.
It's something I want to fix, but must admit to being a bit stumped as
to how I will bridge that gap long term. Currently, all of the modules
we need are in the tree with no dependencies, so it can be fixed with
a simple PYTHONPATH hack. However, if I actually remove the QMP
library from the tree like I have wanted to, then we need pip to do a
real install and process dependencies, and this creates some
complications and extra startup lag.
Naively, I think adding a "just in time installation of testing
dependencies" when you go to run a testing command from "make XXXX"
might be sufficient for us, possibly in conjunction with a configure
flag that lets you pre-load testing dependencies.
>
>
> >
> > QEMU Functional test
> >
> > options:
> > -h, --help show this help message and exit
> > -d, --debug Also print test and console logs on stdout. This will
> > make the TAP output invalid and is meant for debugging
> > only.
> >
> > Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>
> > ---
> > docs/devel/testing/functional.rst | 2 ++
> > tests/functional/qemu_test/testcase.py | 51 ++++++++++++++++++++++++++++++++--
> > 2 files changed, 50 insertions(+), 3 deletions(-)
> >
> > diff --git a/docs/devel/testing/functional.rst b/docs/devel/testing/functional.rst
> > index 9e56dd1b1189216b9b4aede00174c15203f38b41..9d08abe2848277d635befb0296f578cfaa4bd66d 100644
> > --- a/docs/devel/testing/functional.rst
> > +++ b/docs/devel/testing/functional.rst
> > @@ -63,6 +63,8 @@ directory should be your build folder. For example::
> > $ export QEMU_TEST_QEMU_BINARY=$PWD/qemu-system-x86_64
> > $ pyvenv/bin/python3 ../tests/functional/test_file.py
> >
> > +By default, functional tests redirect informational logs and console output to
> > +log files. Specify the ``--debug`` flag to also print those to standard output.
> > The test framework will automatically purge any scratch files created during
> > the tests. If needing to debug a failed test, it is possible to keep these
> > files around on disk by setting ```QEMU_TEST_KEEP_SCRATCH=1``` as an env
> > diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py
> > index 2082c6fce43b0544d4e4258cd4155f555ed30cd4..fad7a946c6677e9ef5c42b8f77187ba836c11aeb 100644
> > --- a/tests/functional/qemu_test/testcase.py
> > +++ b/tests/functional/qemu_test/testcase.py
> > @@ -11,6 +11,7 @@
> > # This work is licensed under the terms of the GNU GPL, version 2 or
> > # later. See the COPYING file in the top-level directory.
> >
> > +import argparse
> > import logging
> > import os
> > from pathlib import Path
> > @@ -31,6 +32,20 @@
> > from .uncompress import uncompress
> >
> >
> > +def parse_args(test_name: str) -> argparse.Namespace:
> > + parser = argparse.ArgumentParser(
> > + prog=test_name, description="QEMU Functional test"
> > + )
> > + parser.add_argument(
> > + "-d",
> > + "--debug",
> > + action="store_true",
> > + help="Also print test and console logs on stdout. This will make the"
> > + " TAP output invalid and is meant for debugging only.",
> > + )
> > + return parser.parse_args()
> > +
> > +
> > class QemuBaseTest(unittest.TestCase):
> >
> > '''
> > @@ -196,6 +211,9 @@ def assets_available(self):
> > return True
> >
> > def setUp(self):
> > + path = os.path.basename(sys.argv[0])[:-3]
> > + args = parse_args(path)
> > + self.debug_output = args.debug
> > self.qemu_bin = os.getenv('QEMU_TEST_QEMU_BINARY')
> > self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set')
> > self.arch = self.qemu_bin.split('-')[-1]
> > @@ -221,6 +239,16 @@ def setUp(self):
> > self.machinelog.setLevel(logging.DEBUG)
> > self.machinelog.addHandler(self._log_fh)
> >
> > + if self.debug_output:
> > + handler = logging.StreamHandler(sys.stdout)
> > + handler.setLevel(logging.DEBUG)
> > + formatter = logging.Formatter(
> > + "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
> > + )
> > + handler.setFormatter(formatter)
> > + self.log.addHandler(handler)
> > + self.machinelog.addHandler(handler)
> > +
> > if not self.assets_available():
> > self.skipTest('One or more assets is not available')
> >
> > @@ -230,11 +258,16 @@ def tearDown(self):
> > if self.socketdir is not None:
> > shutil.rmtree(self.socketdir.name)
> > self.socketdir = None
> > - self.machinelog.removeHandler(self._log_fh)
> > - self.log.removeHandler(self._log_fh)
> > + for handler in [self._log_fh, logging.StreamHandler(sys.stdout)]:
> > + self.machinelog.removeHandler(handler)
> > + self.log.removeHandler(handler)
> >
> > def main():
> > path = os.path.basename(sys.argv[0])[:-3]
> > + # If argparse receives --help or an unknown argument, it will raise a
> > + # SystemExit which will get caught by the test runner. Parse the
> > + # arguments here too to handle that case.
> > + _ = parse_args(path)
> >
> > cache = os.environ.get("QEMU_TEST_PRECACHE", None)
> > if cache is not None:
> > @@ -292,6 +325,14 @@ def setUp(self):
> > fileFormatter = logging.Formatter('%(asctime)s: %(message)s')
> > self._console_log_fh.setFormatter(fileFormatter)
> > console_log.addHandler(self._console_log_fh)
> > + if self.debug_output:
> > + handler = logging.StreamHandler(sys.stdout)
> > + handler.setLevel(logging.DEBUG)
> > + formatter = logging.Formatter(
> > + "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
> > + )
> > + handler.setFormatter(formatter)
> > + console_log.addHandler(handler)
> >
> > def set_machine(self, machinename):
> > # TODO: We should use QMP to get the list of available machines
> > @@ -398,5 +439,9 @@ def set_vm_arg(self, arg, value):
> > def tearDown(self):
> > for vm in self._vms.values():
> > vm.shutdown()
> > - logging.getLogger('console').removeHandler(self._console_log_fh)
> > + for handler in [
> > + self._console_log_fh,
> > + logging.StreamHandler(sys.stdout),
> > + ]:
> > + logging.getLogger("console").removeHandler(handler)
> > super().tearDown()
> >
> > ---
> > base-commit: c079d3a31e45093286c65f8ca5350beb3a4404a9
> > change-id: 20250716-functional_tests_debug_arg-aa0a5f6b9375
>
> --
> Alex Bennée
> Virtualisation Tech Lead @ Linaro
>
On 21/07/2025 22.38, John Snow wrote: > On Thu, Jul 17, 2025 at 4:44 AM Alex Bennée <alex.bennee@linaro.org> wrote: ... >> Am I holding it wrong? >> >> ➜ ./pyvenv/bin/python ../../tests/functional/test_aarch64_virt.py --help >> Traceback (most recent call last): >> File "/home/alex/lsrc/qemu.git/builds/all/../../tests/functional/test_aarch64_virt.py", line 16, in <module> >> from qemu_test import QemuSystemTest, Asset, exec_command_and_wait_for_pattern >> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/__init__.py", line 14, in <module> >> from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest >> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/testcase.py", line 26, in <module> >> from qemu.machine import QEMUMachine >> ModuleNotFoundError: No module named 'qemu' >> >> I thought the point of the venv is we had all the modules we need >> automatically available to the PYTHONPATH? > > As Thomas points out, "qemu" is special since it's already in the > tree. There has been some dragging-of-feet by yours-truly because > installing the "qemu" module by default when running configure > introduces a considerable startup lag time, and the module is not > actually needed for the simple configuration and building of QEMU - > only testing. > > It's something I want to fix, but must admit to being a bit stumped as > to how I will bridge that gap long term. Currently, all of the modules > we need are in the tree with no dependencies, so it can be fixed with > a simple PYTHONPATH hack. However, if I actually remove the QMP > library from the tree like I have wanted to, then we need pip to do a > real install and process dependencies, and this creates some > complications and extra startup lag. Wouldn't it be possible to add the module as a wheel in python/wheels/ ? That's maybe the easiest solution, isn't it? > Naively, I think adding a "just in time installation of testing > dependencies" when you go to run a testing command from "make XXXX" > might be sufficient for us, possibly in conjunction with a configure > flag that lets you pre-load testing dependencies. We could likely re-use "make check-venv" for the functional tests ... it's already installed in that case. However, you then still have to remember to call it first before you can run a test directly, without the Makefile wrappers. Thomas
On Thu, Jul 24, 2025 at 3:47 PM Thomas Huth <thuth@redhat.com> wrote: > > On 21/07/2025 22.38, John Snow wrote: > > On Thu, Jul 17, 2025 at 4:44 AM Alex Bennée <alex.bennee@linaro.org> wrote: > ... > >> Am I holding it wrong? > >> > >> ➜ ./pyvenv/bin/python ../../tests/functional/test_aarch64_virt.py --help > >> Traceback (most recent call last): > >> File "/home/alex/lsrc/qemu.git/builds/all/../../tests/functional/test_aarch64_virt.py", line 16, in <module> > >> from qemu_test import QemuSystemTest, Asset, exec_command_and_wait_for_pattern > >> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/__init__.py", line 14, in <module> > >> from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest > >> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/testcase.py", line 26, in <module> > >> from qemu.machine import QEMUMachine > >> ModuleNotFoundError: No module named 'qemu' > >> > >> I thought the point of the venv is we had all the modules we need > >> automatically available to the PYTHONPATH? > > > > As Thomas points out, "qemu" is special since it's already in the > > tree. There has been some dragging-of-feet by yours-truly because > > installing the "qemu" module by default when running configure > > introduces a considerable startup lag time, and the module is not > > actually needed for the simple configuration and building of QEMU - > > only testing. > > > > It's something I want to fix, but must admit to being a bit stumped as > > to how I will bridge that gap long term. Currently, all of the modules > > we need are in the tree with no dependencies, so it can be fixed with > > a simple PYTHONPATH hack. However, if I actually remove the QMP > > library from the tree like I have wanted to, then we need pip to do a > > real install and process dependencies, and this creates some > > complications and extra startup lag. > > Wouldn't it be possible to add the module as a wheel in python/wheels/ ? > That's maybe the easiest solution, isn't it? The qemu.qmp wheel, yes, because it's purepython. The part I am worried about is that by removing qemu.qmp from the tree, you cannot just use the PYTHONPATH hack anymore to use the remaining in-tree goodies, namely the machine module used extensively throughout testing, because they will now rely on an external dependency. Normally, you'd just install the in-tree stuff like any other package and pip would take care of the qemu.qmp dependency. As handled through mkvenv, it would search our vendored wheels directory in the process and use that wheel, it's not a big deal. The tricky part in my head is that this "pip install" of the in-tree goodies takes a few seconds because pip is quite slow - it technically has to "build" the package before it installs it, so it adds some lag time. Since qemu.qmp isn't actually needed to configure or build QEMU, I thought it'd be inappropriate to add it to those dependencies - and inappropriate to enforce a 3-4 second hit on every configure call whether or not you intend to run tests afterwards. I'm sure it's solvable, I just tried once a while back and hit a wall, got distracted, and haven't revisited it yet... Maybe a good rubber-ducking session would help. > > > Naively, I think adding a "just in time installation of testing > > dependencies" when you go to run a testing command from "make XXXX" > > might be sufficient for us, possibly in conjunction with a configure > > flag that lets you pre-load testing dependencies. > > We could likely re-use "make check-venv" for the functional tests ... it's > already installed in that case. However, you then still have to remember to > call it first before you can run a test directly, without the Makefile wrappers. > > Thomas >
On 07/08/2025 23.46, John Snow wrote: > On Thu, Jul 24, 2025 at 3:47 PM Thomas Huth <thuth@redhat.com> wrote: >> >> On 21/07/2025 22.38, John Snow wrote: >>> On Thu, Jul 17, 2025 at 4:44 AM Alex Bennée <alex.bennee@linaro.org> wrote: >> ... >>>> Am I holding it wrong? >>>> >>>> ➜ ./pyvenv/bin/python ../../tests/functional/test_aarch64_virt.py --help >>>> Traceback (most recent call last): >>>> File "/home/alex/lsrc/qemu.git/builds/all/../../tests/functional/test_aarch64_virt.py", line 16, in <module> >>>> from qemu_test import QemuSystemTest, Asset, exec_command_and_wait_for_pattern >>>> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/__init__.py", line 14, in <module> >>>> from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest >>>> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/testcase.py", line 26, in <module> >>>> from qemu.machine import QEMUMachine >>>> ModuleNotFoundError: No module named 'qemu' >>>> >>>> I thought the point of the venv is we had all the modules we need >>>> automatically available to the PYTHONPATH? >>> >>> As Thomas points out, "qemu" is special since it's already in the >>> tree. There has been some dragging-of-feet by yours-truly because >>> installing the "qemu" module by default when running configure >>> introduces a considerable startup lag time, and the module is not >>> actually needed for the simple configuration and building of QEMU - >>> only testing. >>> >>> It's something I want to fix, but must admit to being a bit stumped as >>> to how I will bridge that gap long term. Currently, all of the modules >>> we need are in the tree with no dependencies, so it can be fixed with >>> a simple PYTHONPATH hack. However, if I actually remove the QMP >>> library from the tree like I have wanted to, then we need pip to do a >>> real install and process dependencies, and this creates some >>> complications and extra startup lag. >> >> Wouldn't it be possible to add the module as a wheel in python/wheels/ ? >> That's maybe the easiest solution, isn't it? > > The qemu.qmp wheel, yes, because it's purepython. The part I am > worried about is that by removing qemu.qmp from the tree, you cannot > just use the PYTHONPATH hack anymore to use the remaining in-tree > goodies, namely the machine module used extensively throughout > testing, because they will now rely on an external dependency. Would it maybe be possible to add the machine part to the qemu.qmp module, too? (Gerd also ask me at KVM forum about this, so there is certainly interest for extending the qemu.qmp module with the machine part) > I'm sure it's solvable, I just tried once a while back and hit a wall, > got distracted, and haven't revisited it yet... Maybe a good > rubber-ducking session would help. For the (removed) avocado tests, they had a dependency on the "check-venv" target in QEMU, so its python dependencies were only added when you ran "make check-avocado". We could maybe do the same for "check-functional", too? Thomas
On Tue, Sep 9, 2025 at 6:37 AM Thomas Huth <thuth@redhat.com> wrote: > > On 07/08/2025 23.46, John Snow wrote: > > On Thu, Jul 24, 2025 at 3:47 PM Thomas Huth <thuth@redhat.com> wrote: > >> > >> On 21/07/2025 22.38, John Snow wrote: > >>> On Thu, Jul 17, 2025 at 4:44 AM Alex Bennée <alex.bennee@linaro.org> wrote: > >> ... > >>>> Am I holding it wrong? > >>>> > >>>> ➜ ./pyvenv/bin/python ../../tests/functional/test_aarch64_virt.py --help > >>>> Traceback (most recent call last): > >>>> File "/home/alex/lsrc/qemu.git/builds/all/../../tests/functional/test_aarch64_virt.py", line 16, in <module> > >>>> from qemu_test import QemuSystemTest, Asset, exec_command_and_wait_for_pattern > >>>> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/__init__.py", line 14, in <module> > >>>> from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest > >>>> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/testcase.py", line 26, in <module> > >>>> from qemu.machine import QEMUMachine > >>>> ModuleNotFoundError: No module named 'qemu' > >>>> > >>>> I thought the point of the venv is we had all the modules we need > >>>> automatically available to the PYTHONPATH? > >>> > >>> As Thomas points out, "qemu" is special since it's already in the > >>> tree. There has been some dragging-of-feet by yours-truly because > >>> installing the "qemu" module by default when running configure > >>> introduces a considerable startup lag time, and the module is not > >>> actually needed for the simple configuration and building of QEMU - > >>> only testing. > >>> > >>> It's something I want to fix, but must admit to being a bit stumped as > >>> to how I will bridge that gap long term. Currently, all of the modules > >>> we need are in the tree with no dependencies, so it can be fixed with > >>> a simple PYTHONPATH hack. However, if I actually remove the QMP > >>> library from the tree like I have wanted to, then we need pip to do a > >>> real install and process dependencies, and this creates some > >>> complications and extra startup lag. > >> > >> Wouldn't it be possible to add the module as a wheel in python/wheels/ ? > >> That's maybe the easiest solution, isn't it? > > > > The qemu.qmp wheel, yes, because it's purepython. The part I am > > worried about is that by removing qemu.qmp from the tree, you cannot > > just use the PYTHONPATH hack anymore to use the remaining in-tree > > goodies, namely the machine module used extensively throughout > > testing, because they will now rely on an external dependency. > > Would it maybe be possible to add the machine part to the qemu.qmp module, > too? (Gerd also ask me at KVM forum about this, so there is certainly > interest for extending the qemu.qmp module with the machine part) I could, can you explain a little why we want it? It would involve cleaning up the interfaces a good deal and while I don't think it is hard, it will be a lot of clerical work and shuffling for a little while. I have some drafts somewhere that add an asyncio VM appliance to replace the existing one in the same manner as for what I did for async QMP to help solve the complicated logging stack we have for VM console scripts. Having a solid justification and guinea pigs^H^H^H^H^H^H^H^H^H^H^H^H beta users would push the project over the line for me. > > > I'm sure it's solvable, I just tried once a while back and hit a wall, > > got distracted, and haven't revisited it yet... Maybe a good > > rubber-ducking session would help. > > For the (removed) avocado tests, they had a dependency on the "check-venv" > target in QEMU, so its python dependencies were only added when you ran > "make check-avocado". We could maybe do the same for "check-functional", too? Something like that, yeah. I'm going to send a patchset soon that does the dumbest possible thing and we can chat in review for that series.
On 15/09/2025 21.54, John Snow wrote: > On Tue, Sep 9, 2025 at 6:37 AM Thomas Huth <thuth@redhat.com> wrote: >> >> On 07/08/2025 23.46, John Snow wrote: >>> On Thu, Jul 24, 2025 at 3:47 PM Thomas Huth <thuth@redhat.com> wrote: ... >>>> Wouldn't it be possible to add the module as a wheel in python/wheels/ ? >>>> That's maybe the easiest solution, isn't it? >>> >>> The qemu.qmp wheel, yes, because it's purepython. The part I am >>> worried about is that by removing qemu.qmp from the tree, you cannot >>> just use the PYTHONPATH hack anymore to use the remaining in-tree >>> goodies, namely the machine module used extensively throughout >>> testing, because they will now rely on an external dependency. >> >> Would it maybe be possible to add the machine part to the qemu.qmp module, >> too? (Gerd also ask me at KVM forum about this, so there is certainly >> interest for extending the qemu.qmp module with the machine part) > > I could, can you explain a little why we want it? I hope that Gerd can chime in, but basically he wanted to test another piece of software in a similar way we do it in QEMU with the functional framework. That means some function that manage the life-cycle of a QEMU VM would be handy. > It would involve > cleaning up the interfaces a good deal and while I don't think it is > hard, it will be a lot of clerical work and shuffling for a little > while. I have some drafts somewhere that add an asyncio VM appliance > to replace the existing one in the same manner as for what I did for > async QMP to help solve the complicated logging stack we have for VM > console scripts. > > Having a solid justification and guinea pigs^H^H^H^H^H^H^H^H^H^H^H^H > beta users would push the project over the line for me. I'm sure Gerd is happy to help if he can find some spare time, and I'm of course happy to help with regards to the QEMU functional testing framework - let me know when you've got something ready and we want to have a try to move the functional tests to that new VM management code. Thomas
On Thu, Jul 17, 2025 at 11:42 AM Alex Bennée <alex.bennee@linaro.org> wrote:
>
> Manos Pitsidianakis <manos.pitsidianakis@linaro.org> writes:
>
> > Add argument parsing to functional tests to improve developer experience
> > when running individual tests. All logs are printed to stdout
> > interspersed with TAP output.
> >
> > ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help
> > usage: test_aarch64_virt [-h] [-d]
>
> Am I holding it wrong?
>
> ➜ ./pyvenv/bin/python ../../tests/functional/test_aarch64_virt.py --help
> Traceback (most recent call last):
> File "/home/alex/lsrc/qemu.git/builds/all/../../tests/functional/test_aarch64_virt.py", line 16, in <module>
> from qemu_test import QemuSystemTest, Asset, exec_command_and_wait_for_pattern
> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/__init__.py", line 14, in <module>
> from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest
> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/testcase.py", line 26, in <module>
> from qemu.machine import QEMUMachine
> ModuleNotFoundError: No module named 'qemu'
>
> I thought the point of the venv is we had all the modules we need
> automatically available to the PYTHONPATH?
Is PYTHONPATH exported? Check that you've done the instructions
detailed here: https://www.qemu.org/docs/master/devel/testing/functional.html#running-tests
>
> >
> > QEMU Functional test
> >
> > options:
> > -h, --help show this help message and exit
> > -d, --debug Also print test and console logs on stdout. This will
> > make the TAP output invalid and is meant for debugging
> > only.
> >
> > Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>
> > ---
> > docs/devel/testing/functional.rst | 2 ++
> > tests/functional/qemu_test/testcase.py | 51 ++++++++++++++++++++++++++++++++--
> > 2 files changed, 50 insertions(+), 3 deletions(-)
> >
> > diff --git a/docs/devel/testing/functional.rst b/docs/devel/testing/functional.rst
> > index 9e56dd1b1189216b9b4aede00174c15203f38b41..9d08abe2848277d635befb0296f578cfaa4bd66d 100644
> > --- a/docs/devel/testing/functional.rst
> > +++ b/docs/devel/testing/functional.rst
> > @@ -63,6 +63,8 @@ directory should be your build folder. For example::
> > $ export QEMU_TEST_QEMU_BINARY=$PWD/qemu-system-x86_64
> > $ pyvenv/bin/python3 ../tests/functional/test_file.py
> >
> > +By default, functional tests redirect informational logs and console output to
> > +log files. Specify the ``--debug`` flag to also print those to standard output.
> > The test framework will automatically purge any scratch files created during
> > the tests. If needing to debug a failed test, it is possible to keep these
> > files around on disk by setting ```QEMU_TEST_KEEP_SCRATCH=1``` as an env
> > diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py
> > index 2082c6fce43b0544d4e4258cd4155f555ed30cd4..fad7a946c6677e9ef5c42b8f77187ba836c11aeb 100644
> > --- a/tests/functional/qemu_test/testcase.py
> > +++ b/tests/functional/qemu_test/testcase.py
> > @@ -11,6 +11,7 @@
> > # This work is licensed under the terms of the GNU GPL, version 2 or
> > # later. See the COPYING file in the top-level directory.
> >
> > +import argparse
> > import logging
> > import os
> > from pathlib import Path
> > @@ -31,6 +32,20 @@
> > from .uncompress import uncompress
> >
> >
> > +def parse_args(test_name: str) -> argparse.Namespace:
> > + parser = argparse.ArgumentParser(
> > + prog=test_name, description="QEMU Functional test"
> > + )
> > + parser.add_argument(
> > + "-d",
> > + "--debug",
> > + action="store_true",
> > + help="Also print test and console logs on stdout. This will make the"
> > + " TAP output invalid and is meant for debugging only.",
> > + )
> > + return parser.parse_args()
> > +
> > +
> > class QemuBaseTest(unittest.TestCase):
> >
> > '''
> > @@ -196,6 +211,9 @@ def assets_available(self):
> > return True
> >
> > def setUp(self):
> > + path = os.path.basename(sys.argv[0])[:-3]
> > + args = parse_args(path)
> > + self.debug_output = args.debug
> > self.qemu_bin = os.getenv('QEMU_TEST_QEMU_BINARY')
> > self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set')
> > self.arch = self.qemu_bin.split('-')[-1]
> > @@ -221,6 +239,16 @@ def setUp(self):
> > self.machinelog.setLevel(logging.DEBUG)
> > self.machinelog.addHandler(self._log_fh)
> >
> > + if self.debug_output:
> > + handler = logging.StreamHandler(sys.stdout)
> > + handler.setLevel(logging.DEBUG)
> > + formatter = logging.Formatter(
> > + "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
> > + )
> > + handler.setFormatter(formatter)
> > + self.log.addHandler(handler)
> > + self.machinelog.addHandler(handler)
> > +
> > if not self.assets_available():
> > self.skipTest('One or more assets is not available')
> >
> > @@ -230,11 +258,16 @@ def tearDown(self):
> > if self.socketdir is not None:
> > shutil.rmtree(self.socketdir.name)
> > self.socketdir = None
> > - self.machinelog.removeHandler(self._log_fh)
> > - self.log.removeHandler(self._log_fh)
> > + for handler in [self._log_fh, logging.StreamHandler(sys.stdout)]:
> > + self.machinelog.removeHandler(handler)
> > + self.log.removeHandler(handler)
> >
> > def main():
> > path = os.path.basename(sys.argv[0])[:-3]
> > + # If argparse receives --help or an unknown argument, it will raise a
> > + # SystemExit which will get caught by the test runner. Parse the
> > + # arguments here too to handle that case.
> > + _ = parse_args(path)
> >
> > cache = os.environ.get("QEMU_TEST_PRECACHE", None)
> > if cache is not None:
> > @@ -292,6 +325,14 @@ def setUp(self):
> > fileFormatter = logging.Formatter('%(asctime)s: %(message)s')
> > self._console_log_fh.setFormatter(fileFormatter)
> > console_log.addHandler(self._console_log_fh)
> > + if self.debug_output:
> > + handler = logging.StreamHandler(sys.stdout)
> > + handler.setLevel(logging.DEBUG)
> > + formatter = logging.Formatter(
> > + "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
> > + )
> > + handler.setFormatter(formatter)
> > + console_log.addHandler(handler)
> >
> > def set_machine(self, machinename):
> > # TODO: We should use QMP to get the list of available machines
> > @@ -398,5 +439,9 @@ def set_vm_arg(self, arg, value):
> > def tearDown(self):
> > for vm in self._vms.values():
> > vm.shutdown()
> > - logging.getLogger('console').removeHandler(self._console_log_fh)
> > + for handler in [
> > + self._console_log_fh,
> > + logging.StreamHandler(sys.stdout),
> > + ]:
> > + logging.getLogger("console").removeHandler(handler)
> > super().tearDown()
> >
> > ---
> > base-commit: c079d3a31e45093286c65f8ca5350beb3a4404a9
> > change-id: 20250716-functional_tests_debug_arg-aa0a5f6b9375
>
> --
> Alex Bennée
> Virtualisation Tech Lead @ Linaro
Manos Pitsidianakis <manos.pitsidianakis@linaro.org> writes: > On Thu, Jul 17, 2025 at 11:42 AM Alex Bennée <alex.bennee@linaro.org> wrote: >> >> Manos Pitsidianakis <manos.pitsidianakis@linaro.org> writes: >> >> > Add argument parsing to functional tests to improve developer experience >> > when running individual tests. All logs are printed to stdout >> > interspersed with TAP output. >> > >> > ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help >> > usage: test_aarch64_virt [-h] [-d] >> >> Am I holding it wrong? >> >> ➜ ./pyvenv/bin/python ../../tests/functional/test_aarch64_virt.py --help >> Traceback (most recent call last): >> File "/home/alex/lsrc/qemu.git/builds/all/../../tests/functional/test_aarch64_virt.py", line 16, in <module> >> from qemu_test import QemuSystemTest, Asset, exec_command_and_wait_for_pattern >> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/__init__.py", line 14, in <module> >> from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest >> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/testcase.py", line 26, in <module> >> from qemu.machine import QEMUMachine >> ModuleNotFoundError: No module named 'qemu' >> >> I thought the point of the venv is we had all the modules we need >> automatically available to the PYTHONPATH? > > Is PYTHONPATH exported? Check that you've done the instructions > detailed here: > https://www.qemu.org/docs/master/devel/testing/functional.html#running-tests We should probably include enough in the commit message so a blind copy and paste works, or at least indicates you need more setup to run the test: set -x PYTHONPATH ../../python:../../tests/functional/ env QEMU_TEST_QEMU_BINARY=./qemu-system-aarch64 ./pyvenv/bin/python ../../tests/functional/test_aarch64_virt.py --debug >> >> > >> > QEMU Functional test >> > >> > options: >> > -h, --help show this help message and exit >> > -d, --debug Also print test and console logs on stdout. This will >> > make the TAP output invalid and is meant for debugging >> > only. >> > >> > Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org> >> > --- >> > docs/devel/testing/functional.rst | 2 ++ >> > tests/functional/qemu_test/testcase.py | 51 ++++++++++++++++++++++++++++++++-- >> > 2 files changed, 50 insertions(+), 3 deletions(-) >> > >> > diff --git a/docs/devel/testing/functional.rst b/docs/devel/testing/functional.rst >> > index 9e56dd1b1189216b9b4aede00174c15203f38b41..9d08abe2848277d635befb0296f578cfaa4bd66d 100644 >> > --- a/docs/devel/testing/functional.rst >> > +++ b/docs/devel/testing/functional.rst >> > @@ -63,6 +63,8 @@ directory should be your build folder. For example:: >> > $ export QEMU_TEST_QEMU_BINARY=$PWD/qemu-system-x86_64 >> > $ pyvenv/bin/python3 ../tests/functional/test_file.py >> > >> > +By default, functional tests redirect informational logs and console output to >> > +log files. Specify the ``--debug`` flag to also print those to standard output. >> > The test framework will automatically purge any scratch files created during >> > the tests. If needing to debug a failed test, it is possible to keep these >> > files around on disk by setting ```QEMU_TEST_KEEP_SCRATCH=1``` as an env >> > diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py >> > index 2082c6fce43b0544d4e4258cd4155f555ed30cd4..fad7a946c6677e9ef5c42b8f77187ba836c11aeb 100644 >> > --- a/tests/functional/qemu_test/testcase.py >> > +++ b/tests/functional/qemu_test/testcase.py >> > @@ -11,6 +11,7 @@ >> > # This work is licensed under the terms of the GNU GPL, version 2 or >> > # later. See the COPYING file in the top-level directory. >> > >> > +import argparse >> > import logging >> > import os >> > from pathlib import Path >> > @@ -31,6 +32,20 @@ >> > from .uncompress import uncompress >> > >> > >> > +def parse_args(test_name: str) -> argparse.Namespace: >> > + parser = argparse.ArgumentParser( >> > + prog=test_name, description="QEMU Functional test" >> > + ) >> > + parser.add_argument( >> > + "-d", >> > + "--debug", >> > + action="store_true", >> > + help="Also print test and console logs on stdout. This will make the" >> > + " TAP output invalid and is meant for debugging only.", >> > + ) >> > + return parser.parse_args() >> > + >> > + I'm definitely onboard for improving the ergonomics of calling the tests directly. Others to consider: - triggering the behaviour of QEMU_TEST_KEEP_SCRATCH - dumping the command line without the test harness QMP/serial pipes connected <snip> -- Alex Bennée Virtualisation Tech Lead @ Linaro
On Thu, Jul 17, 2025 at 1:04 PM Alex Bennée <alex.bennee@linaro.org> wrote: > > Manos Pitsidianakis <manos.pitsidianakis@linaro.org> writes: > > > On Thu, Jul 17, 2025 at 11:42 AM Alex Bennée <alex.bennee@linaro.org> wrote: > >> > >> Manos Pitsidianakis <manos.pitsidianakis@linaro.org> writes: > >> > >> > Add argument parsing to functional tests to improve developer experience > >> > when running individual tests. All logs are printed to stdout > >> > interspersed with TAP output. > >> > > >> > ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help > >> > usage: test_aarch64_virt [-h] [-d] > >> > >> Am I holding it wrong? > >> > >> ➜ ./pyvenv/bin/python ../../tests/functional/test_aarch64_virt.py --help > >> Traceback (most recent call last): > >> File "/home/alex/lsrc/qemu.git/builds/all/../../tests/functional/test_aarch64_virt.py", line 16, in <module> > >> from qemu_test import QemuSystemTest, Asset, exec_command_and_wait_for_pattern > >> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/__init__.py", line 14, in <module> > >> from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest > >> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/testcase.py", line 26, in <module> > >> from qemu.machine import QEMUMachine > >> ModuleNotFoundError: No module named 'qemu' > >> > >> I thought the point of the venv is we had all the modules we need > >> automatically available to the PYTHONPATH? > > > > Is PYTHONPATH exported? Check that you've done the instructions > > detailed here: > > https://www.qemu.org/docs/master/devel/testing/functional.html#running-tests > > We should probably include enough in the commit message so a blind copy > and paste works, or at least indicates you need more setup to run the > test: > > set -x PYTHONPATH ../../python:../../tests/functional/ > env QEMU_TEST_QEMU_BINARY=./qemu-system-aarch64 ./pyvenv/bin/python ../../tests/functional/test_aarch64_virt.py --debug > > >> > >> > > >> > QEMU Functional test > >> > > >> > options: > >> > -h, --help show this help message and exit > >> > -d, --debug Also print test and console logs on stdout. This will > >> > make the TAP output invalid and is meant for debugging > >> > only. > >> > > >> > Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org> > >> > --- > >> > docs/devel/testing/functional.rst | 2 ++ > >> > tests/functional/qemu_test/testcase.py | 51 ++++++++++++++++++++++++++++++++-- > >> > 2 files changed, 50 insertions(+), 3 deletions(-) > >> > > >> > diff --git a/docs/devel/testing/functional.rst b/docs/devel/testing/functional.rst > >> > index 9e56dd1b1189216b9b4aede00174c15203f38b41..9d08abe2848277d635befb0296f578cfaa4bd66d 100644 > >> > --- a/docs/devel/testing/functional.rst > >> > +++ b/docs/devel/testing/functional.rst > >> > @@ -63,6 +63,8 @@ directory should be your build folder. For example:: > >> > $ export QEMU_TEST_QEMU_BINARY=$PWD/qemu-system-x86_64 > >> > $ pyvenv/bin/python3 ../tests/functional/test_file.py > >> > > >> > +By default, functional tests redirect informational logs and console output to > >> > +log files. Specify the ``--debug`` flag to also print those to standard output. > >> > The test framework will automatically purge any scratch files created during > >> > the tests. If needing to debug a failed test, it is possible to keep these > >> > files around on disk by setting ```QEMU_TEST_KEEP_SCRATCH=1``` as an env > >> > diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py > >> > index 2082c6fce43b0544d4e4258cd4155f555ed30cd4..fad7a946c6677e9ef5c42b8f77187ba836c11aeb 100644 > >> > --- a/tests/functional/qemu_test/testcase.py > >> > +++ b/tests/functional/qemu_test/testcase.py > >> > @@ -11,6 +11,7 @@ > >> > # This work is licensed under the terms of the GNU GPL, version 2 or > >> > # later. See the COPYING file in the top-level directory. > >> > > >> > +import argparse > >> > import logging > >> > import os > >> > from pathlib import Path > >> > @@ -31,6 +32,20 @@ > >> > from .uncompress import uncompress > >> > > >> > > >> > +def parse_args(test_name: str) -> argparse.Namespace: > >> > + parser = argparse.ArgumentParser( > >> > + prog=test_name, description="QEMU Functional test" > >> > + ) > >> > + parser.add_argument( > >> > + "-d", > >> > + "--debug", > >> > + action="store_true", > >> > + help="Also print test and console logs on stdout. This will make the" > >> > + " TAP output invalid and is meant for debugging only.", > >> > + ) > >> > + return parser.parse_args() > >> > + > >> > + > > I'm definitely onboard for improving the ergonomics of calling the tests > directly. Others to consider: > > - triggering the behaviour of QEMU_TEST_KEEP_SCRATCH > - dumping the command line without the test harness QMP/serial pipes connected Definitely useful, I've needed those behaviors in the past! I will send more patches. I will also see if it's easy to make it possible to list the tests that would run in a specific file, and if possible to run only specified tests. Thanks, -- Manos Pitsidianakis Emulation and Virtualization Engineer at Linaro Ltd
On 17/07/2025 10.42, Alex Bennée wrote: > Manos Pitsidianakis <manos.pitsidianakis@linaro.org> writes: > >> Add argument parsing to functional tests to improve developer experience >> when running individual tests. All logs are printed to stdout >> interspersed with TAP output. >> >> ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help >> usage: test_aarch64_virt [-h] [-d] > > Am I holding it wrong? > > ➜ ./pyvenv/bin/python ../../tests/functional/test_aarch64_virt.py --help > Traceback (most recent call last): > File "/home/alex/lsrc/qemu.git/builds/all/../../tests/functional/test_aarch64_virt.py", line 16, in <module> > from qemu_test import QemuSystemTest, Asset, exec_command_and_wait_for_pattern > File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/__init__.py", line 14, in <module> > from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest > File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/testcase.py", line 26, in <module> > from qemu.machine import QEMUMachine > ModuleNotFoundError: No module named 'qemu' > > I thought the point of the venv is we had all the modules we need > automatically available to the PYTHONPATH? The "qemu" module is special since we ship it in our repository. See the "PYTHONPATH" description in docs/devel/testing/functional.rst. Thomas
Thomas Huth <thuth@redhat.com> writes: > On 17/07/2025 10.42, Alex Bennée wrote: >> Manos Pitsidianakis <manos.pitsidianakis@linaro.org> writes: >> >>> Add argument parsing to functional tests to improve developer experience >>> when running individual tests. All logs are printed to stdout >>> interspersed with TAP output. >>> >>> ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help >>> usage: test_aarch64_virt [-h] [-d] >> Am I holding it wrong? >> ➜ ./pyvenv/bin/python >> ../../tests/functional/test_aarch64_virt.py --help >> Traceback (most recent call last): >> File "/home/alex/lsrc/qemu.git/builds/all/../../tests/functional/test_aarch64_virt.py", line 16, in <module> >> from qemu_test import QemuSystemTest, Asset, exec_command_and_wait_for_pattern >> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/__init__.py", line 14, in <module> >> from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest >> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/testcase.py", line 26, in <module> >> from qemu.machine import QEMUMachine >> ModuleNotFoundError: No module named 'qemu' >> I thought the point of the venv is we had all the modules we need >> automatically available to the PYTHONPATH? > > The "qemu" module is special since we ship it in our repository. See > the "PYTHONPATH" description in docs/devel/testing/functional.rst. Is there anyway to setup the venv so it automatically does that? -- Alex Bennée Virtualisation Tech Lead @ Linaro
On 17/07/2025 12.36, Alex Bennée wrote: > Thomas Huth <thuth@redhat.com> writes: > >> On 17/07/2025 10.42, Alex Bennée wrote: >>> Manos Pitsidianakis <manos.pitsidianakis@linaro.org> writes: >>> >>>> Add argument parsing to functional tests to improve developer experience >>>> when running individual tests. All logs are printed to stdout >>>> interspersed with TAP output. >>>> >>>> ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help >>>> usage: test_aarch64_virt [-h] [-d] >>> Am I holding it wrong? >>> ➜ ./pyvenv/bin/python >>> ../../tests/functional/test_aarch64_virt.py --help >>> Traceback (most recent call last): >>> File "/home/alex/lsrc/qemu.git/builds/all/../../tests/functional/test_aarch64_virt.py", line 16, in <module> >>> from qemu_test import QemuSystemTest, Asset, exec_command_and_wait_for_pattern >>> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/__init__.py", line 14, in <module> >>> from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest >>> File "/home/alex/lsrc/qemu.git/tests/functional/qemu_test/testcase.py", line 26, in <module> >>> from qemu.machine import QEMUMachine >>> ModuleNotFoundError: No module named 'qemu' >>> I thought the point of the venv is we had all the modules we need >>> automatically available to the PYTHONPATH? >> >> The "qemu" module is special since we ship it in our repository. See >> the "PYTHONPATH" description in docs/devel/testing/functional.rst. > > Is there anyway to setup the venv so it automatically does that? I guess so. I just noticed that we already do it when running "make check-venv" ... so I assume this could also be done right from the start instead? Paolo, do you remember whether there was a special reason for not doing it? Thomas
On 7/15/25 11:08 PM, Manos Pitsidianakis wrote: > Add argument parsing to functional tests to improve developer experience > when running individual tests. All logs are printed to stdout > interspersed with TAP output. > > ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help > usage: test_aarch64_virt [-h] [-d] > > QEMU Functional test > > options: > -h, --help show this help message and exit > -d, --debug Also print test and console logs on stdout. This will > make the TAP output invalid and is meant for debugging > only. > > Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org> > --- > docs/devel/testing/functional.rst | 2 ++ > tests/functional/qemu_test/testcase.py | 51 ++++++++++++++++++++++++++++++++-- > 2 files changed, 50 insertions(+), 3 deletions(-) Thanks, that's a great idea, and more convenient than the poor man's tail -f I was using for that. Tested-by: Pierrick Bouvier <pierrick.bouvier@linaro.org> Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org>
© 2016 - 2025 Red Hat, Inc.