Add argument parsing to functional tests to improve developer experience
when running individual tests. All logs are printed to stdout
interspersed with TAP output.

Example usage, assuming current build directory with qemu source code in
the parent directory (see docs/devel/testing/functional.rst for details):

  $ export PYTHONPATH=../python:../tests/functional
  $ export QEMU_TEST_QEMU_BINARY="$(pwd)/qemu-system-aarch64"
  $ ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help
  usage: test_aarch64_virt [-h] [-d]

  QEMU Functional test

  options:
    -h, --help   show this help message and exit
    -d, --debug  Also print test and console logs on stdout. This will
                 make the TAP output invalid and is meant for debugging
                 only.

Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>
---
 docs/devel/testing/functional.rst      |  2 ++
 tests/functional/qemu_test/testcase.py | 54 +++++++++++++++++++++++++++++++---
 2 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/docs/devel/testing/functional.rst b/docs/devel/testing/functional.rst
index 3728bab6c0c4b0cbacd00744bdb5c1462c71d7a8..2725633e09104db3912ec6167bbda652f40aa969 100644
--- a/docs/devel/testing/functional.rst
+++ b/docs/devel/testing/functional.rst
@@ -63,6 +63,8 @@ directory should be your build folder. For example::
   $ export QEMU_TEST_QEMU_BINARY=$PWD/qemu-system-x86_64
   $ pyvenv/bin/python3 ../tests/functional/test_file.py
 
+By default, functional tests redirect informational logs and console output to
+log files. Specify the ``--debug`` flag to also print those to standard output.
 The test framework will automatically purge any scratch files created during
 the tests. If needing to debug a failed test, it is possible to keep these
 files around on disk by setting ``QEMU_TEST_KEEP_SCRATCH=1`` as an env
diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py
index 2a78e735f1604f21efd18e38ee0d586496d6b38c..eedca7f1ad29c9e654cf56535acf9639d679f5c4 100644
--- a/tests/functional/qemu_test/testcase.py
+++ b/tests/functional/qemu_test/testcase.py
@@ -11,6 +11,7 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later.  See the COPYING file in the top-level directory.
 
+import argparse
 import logging
 import os
 from pathlib import Path
@@ -33,6 +34,28 @@
 
 
 class QemuBaseTest(unittest.TestCase):
+    debug: bool = False
+
+    """
+    Class method that initializes class attributes from given command-line
+    arguments.
+    """
+    @staticmethod
+    def parse_args():
+        test_name = os.path.basename(sys.argv[0])[:-3]
+        parser = argparse.ArgumentParser(
+            prog=test_name, description="QEMU Functional test"
+        )
+        parser.add_argument(
+            "-d",
+            "--debug",
+            action="store_true",
+            help="Also print test and console logs on stdout. This will make "
+            "the TAP output invalid and is meant for debugging only.",
+        )
+        args = parser.parse_args()
+        QemuBaseTest.debug = args.debug
+        return
 
     '''
     @params compressed: filename, Asset, or file-like object to uncompress
@@ -197,6 +220,14 @@ def assets_available(self):
         return True
 
     def setUp(self):
+        self.stdout_handler = None
+        if QemuBaseTest.debug:
+            self.stdout_handler = logging.StreamHandler(sys.stdout)
+            self.stdout_handler.setLevel(logging.DEBUG)
+            formatter = logging.Formatter(
+                "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+            )
+            self.stdout_handler.setFormatter(formatter)
         self.qemu_bin = os.getenv('QEMU_TEST_QEMU_BINARY')
         self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set')
         self.arch = self.qemu_bin.split('-')[-1]
@@ -216,12 +247,17 @@ def setUp(self):
             '%(asctime)s - %(levelname)s: %(message)s')
         self._log_fh.setFormatter(fileFormatter)
         self.log.addHandler(self._log_fh)
+        if self.stdout_handler:
+            self.log.addHandler(self.stdout_handler)
 
         # Capture QEMUMachine logging
         self.machinelog = logging.getLogger('qemu.machine')
         self.machinelog.setLevel(logging.DEBUG)
         self.machinelog.addHandler(self._log_fh)
 
+        if self.stdout_handler:
+            self.machinelog.addHandler(self.stdout_handler)
+
         if not self.assets_available():
             self.skipTest('One or more assets is not available')
 
@@ -231,15 +267,19 @@ def tearDown(self):
         if self.socketdir is not None:
             shutil.rmtree(self.socketdir.name)
             self.socketdir = None
-        self.machinelog.removeHandler(self._log_fh)
-        self.log.removeHandler(self._log_fh)
-        self._log_fh.close()
+        for handler in [self._log_fh, self.stdout_handler]:
+            if handler is None:
+                continue
+            self.machinelog.removeHandler(handler)
+            self.log.removeHandler(handler)
+            handler.close()
 
     def main():
         warnings.simplefilter("default")
         os.environ["PYTHONWARNINGS"] = "default"
 
         path = os.path.basename(sys.argv[0])[:-3]
+        QemuBaseTest.parse_args()
 
         cache = os.environ.get("QEMU_TEST_PRECACHE", None)
         if cache is not None:
@@ -297,6 +337,8 @@ def setUp(self):
         fileFormatter = logging.Formatter('%(asctime)s: %(message)s')
         self._console_log_fh.setFormatter(fileFormatter)
         console_log.addHandler(self._console_log_fh)
+        if self.stdout_handler:
+            console_log.addHandler(self.stdout_handler)
 
     def set_machine(self, machinename):
         # TODO: We should use QMP to get the list of available machines
@@ -403,6 +445,10 @@ def set_vm_arg(self, arg, value):
     def tearDown(self):
         for vm in self._vms.values():
             vm.shutdown()
-        logging.getLogger('console').removeHandler(self._console_log_fh)
+        console_log = logging.getLogger("console")
+        console_log.removeHandler(self._console_log_fh)
         self._console_log_fh.close()
+        if self.stdout_handler:
+            console_log.removeHandler(self.stdout_handler)
+            self.stdout_handler.close()
         super().tearDown()

-- 
2.47.2

Manos Pitsidianakis <manos.pitsidianakis@linaro.org> writes:

> Add argument parsing to functional tests to improve developer experience
> when running individual tests. All logs are printed to stdout
> interspersed with TAP output.
>
> Example usage, assuming current build directory with qemu source code in
> the parent directory (see docs/devel/testing/functional.rst for details):
>
>   $ export PYTHONPATH=../python:../tests/functional
>   $ export QEMU_TEST_QEMU_BINARY="$(pwd)/qemu-system-aarch64"
>   $ ./pyvenv/bin/python3 ../tests/functional/test_aarch64_virt.py --help
>   usage: test_aarch64_virt [-h] [-d]
>
>   QEMU Functional test
>
>   options:
>     -h, --help   show this help message and exit
>     -d, --debug  Also print test and console logs on stdout. This will
>                  make the TAP output invalid and is meant for debugging
>                  only.
>
> Signed-off-by: Manos Pitsidianakis <manos.pitsidianakis@linaro.org>



Tested-by: Alex Bennée <alex.bennee@linaro.org>

<snip>

-- 
Alex Bennée
Virtualisation Tech Lead @ Linaro