This series is an attempt to bring a little more guaranteed order to asset and scratch file handling in the functional tests. The main highlights are: * Add custom @skipXXXXX decorators for common scenarios present in QEMU tests * Add helpers for creating file paths for various well known types of data, or well known locations, to avoid adhoc path manipulation * Add helpers to simplify uncompressing and extracting archives, from files downloaded as assets The series overall has a neutral diffstat, but if you look at just test files, as opposed to the shared infra, you'll see a significant reduction of lines of code in the tests, and I believe its easier to read them with less boilerplate. Daniel P. Berrangé (22): tests/functional: increase timeouts for arm sx1 test tests/functional: remove unused system imports tests/functional: remove duplicated 'qemu_test' import statements tests/functional: remove pointless with statement tests/functional: remove duplicated 'which' function impl tests/functional: introduce some helpful decorators tests/functional: switch to new test skip decorators tests/functional: add helpers for building file paths tests/functional: switch over to using self.log_file(...) tests/functional: switch over to using self.build_file(...) tests/functional: switch over to using self.data_file(...) tests/functional: switch over to using self.scratch_file() tests/functional: switch over to using self.socket_dir(...) tests/functional: remove redundant 'rmtree' call tests/functional: add common zip_extract helper tests/functional: add common deb_extract helper tests/functional: generalize archive_extract tests/functional: add 'archive_extract' to QemuBaseTest tests/functional: convert tests to new archive_extract helper tests/functional: generalize uncompress tests/functional: add 'uncompress' to QemuBaseTest tests/functional: convert tests to new uncompress helper tests/functional/qemu_test/__init__.py | 5 +- tests/functional/qemu_test/cmd.py | 10 ++ tests/functional/qemu_test/decorators.py | 105 ++++++++++++ tests/functional/qemu_test/linuxkernel.py | 27 +--- tests/functional/qemu_test/testcase.py | 149 +++++++++++++++++- tests/functional/qemu_test/tuxruntest.py | 11 +- tests/functional/qemu_test/utils.py | 66 +++++++- tests/functional/test_aarch64_aspeed.py | 21 +-- tests/functional/test_aarch64_raspi3.py | 9 +- tests/functional/test_aarch64_raspi4.py | 25 ++- tests/functional/test_aarch64_sbsaref.py | 16 +- .../functional/test_aarch64_sbsaref_alpine.py | 5 +- .../test_aarch64_sbsaref_freebsd.py | 5 +- tests/functional/test_aarch64_virt.py | 12 +- tests/functional/test_acpi_bits.py | 121 +++++--------- tests/functional/test_alpha_clipper.py | 6 +- tests/functional/test_arm_aspeed.py | 52 +++--- tests/functional/test_arm_bflt.py | 16 +- tests/functional/test_arm_bpim2u.py | 44 +++--- tests/functional/test_arm_canona1100.py | 12 +- tests/functional/test_arm_collie.py | 1 - tests/functional/test_arm_integratorcp.py | 31 ++-- tests/functional/test_arm_orangepi.py | 53 +++---- tests/functional/test_arm_raspi2.py | 25 ++- tests/functional/test_arm_sx1.py | 7 +- tests/functional/test_arm_vexpress.py | 9 +- tests/functional/test_linux_initrd.py | 7 +- tests/functional/test_loongarch64_virt.py | 6 +- tests/functional/test_m68k_mcf5208evb.py | 9 +- tests/functional/test_m68k_nextcube.py | 19 +-- tests/functional/test_m68k_q800.py | 5 +- .../functional/test_microblaze_s3adsp1800.py | 13 +- .../test_microblazeel_s3adsp1800.py | 14 +- tests/functional/test_mips64el_fuloong2e.py | 13 +- tests/functional/test_mips64el_loongson3v.py | 10 +- tests/functional/test_mips64el_malta.py | 49 +++--- tests/functional/test_mips_malta.py | 19 +-- tests/functional/test_mipsel_malta.py | 28 ++-- tests/functional/test_multiprocess.py | 4 +- tests/functional/test_netdev_ethtool.py | 3 +- tests/functional/test_or1k_sim.py | 8 +- tests/functional/test_ppc64_e500.py | 6 +- tests/functional/test_ppc64_hv.py | 49 ++---- tests/functional/test_ppc64_powernv.py | 3 +- tests/functional/test_ppc64_pseries.py | 3 +- tests/functional/test_ppc_405.py | 6 +- tests/functional/test_ppc_40p.py | 9 +- tests/functional/test_ppc_74xx.py | 3 +- tests/functional/test_ppc_amiga.py | 20 ++- tests/functional/test_ppc_bamboo.py | 20 +-- tests/functional/test_ppc_mac.py | 7 +- tests/functional/test_ppc_mpc8544ds.py | 10 +- tests/functional/test_ppc_virtex_ml507.py | 12 +- tests/functional/test_riscv_opensbi.py | 4 +- tests/functional/test_rx_gdbsim.py | 17 +- tests/functional/test_s390x_ccw_virtio.py | 12 +- tests/functional/test_s390x_topology.py | 15 +- tests/functional/test_sh4_r2d.py | 14 +- tests/functional/test_sh4_tuxrun.py | 4 - tests/functional/test_sh4eb_r2d.py | 16 +- tests/functional/test_sparc64_sun4u.py | 12 +- tests/functional/test_sparc_sun4m.py | 6 +- tests/functional/test_virtio_gpu.py | 19 +-- tests/functional/test_virtio_version.py | 2 - tests/functional/test_xtensa_lx60.py | 7 +- 65 files changed, 677 insertions(+), 649 deletions(-) create mode 100644 tests/functional/qemu_test/decorators.py -- 2.46.0 When under high load the test VM does not complete running in the default 30 second timeout. Double it to give more headroom. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/test_arm_sx1.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/test_arm_sx1.py b/tests/functional/test_arm_sx1.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_sx1.py +++ b/tests/functional/test_arm_sx1.py @@ -XXX,XX +XXX,XX @@ def test_arm_sx1_initrd(self): self.vm.add_args('-no-reboot') self.launch_kernel(zimage_path, initrd=initrd_path) - self.vm.wait() + self.vm.wait(timeout=60) def test_arm_sx1_sd(self): self.set_machine('sx1') @@ -XXX,XX +XXX,XX @@ def test_arm_sx1_sd(self): self.vm.add_args('-snapshot') self.vm.add_args('-drive', f'format=raw,if=sd,file={sd_fs_path}') self.launch_kernel(zimage_path) - self.vm.wait() + self.vm.wait(timeout=60) def test_arm_sx1_flash(self): self.set_machine('sx1') @@ -XXX,XX +XXX,XX @@ def test_arm_sx1_flash(self): self.vm.add_args('-snapshot') self.vm.add_args('-drive', f'format=raw,if=pflash,file={flash_path}') self.launch_kernel(zimage_path) - self.vm.wait() + self.vm.wait(timeout=60) if __name__ == '__main__': LinuxKernelTest.main() -- 2.46.0 Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/test_aarch64_sbsaref.py | 1 - tests/functional/test_acpi_bits.py | 1 - tests/functional/test_m68k_mcf5208evb.py | 2 -- tests/functional/test_microblaze_s3adsp1800.py | 1 - tests/functional/test_mips64el_loongson3v.py | 1 - tests/functional/test_or1k_sim.py | 2 -- tests/functional/test_s390x_topology.py | 1 - tests/functional/test_sh4_tuxrun.py | 4 ---- tests/functional/test_sh4eb_r2d.py | 1 - tests/functional/test_virtio_version.py | 2 -- 10 files changed, 16 deletions(-) diff --git a/tests/functional/test_aarch64_sbsaref.py b/tests/functional/test_aarch64_sbsaref.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_sbsaref.py +++ b/tests/functional/test_aarch64_sbsaref.py @@ -XXX,XX +XXX,XX @@ from qemu_test import wait_for_console_pattern from qemu_test import interrupt_interactive_console_until_pattern from qemu_test.utils import lzma_uncompress -from unittest import skipUnless def fetch_firmware(test): """ diff --git a/tests/functional/test_acpi_bits.py b/tests/functional/test_acpi_bits.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_acpi_bits.py +++ b/tests/functional/test_acpi_bits.py @@ -XXX,XX +XXX,XX @@ import subprocess import tarfile import tempfile -import time import zipfile from pathlib import Path diff --git a/tests/functional/test_m68k_mcf5208evb.py b/tests/functional/test_m68k_mcf5208evb.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_m68k_mcf5208evb.py +++ b/tests/functional/test_m68k_mcf5208evb.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - from qemu_test import LinuxKernelTest, Asset from qemu_test.utils import archive_extract diff --git a/tests/functional/test_microblaze_s3adsp1800.py b/tests/functional/test_microblaze_s3adsp1800.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_microblaze_s3adsp1800.py +++ b/tests/functional/test_microblaze_s3adsp1800.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import time from qemu_test import exec_command, exec_command_and_wait_for_pattern from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern diff --git a/tests/functional/test_mips64el_loongson3v.py b/tests/functional/test_mips64el_loongson3v.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_loongson3v.py +++ b/tests/functional/test_mips64el_loongson3v.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later import os -import time from unittest import skipUnless from qemu_test import QemuSystemTest, Asset diff --git a/tests/functional/test_or1k_sim.py b/tests/functional/test_or1k_sim.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_or1k_sim.py +++ b/tests/functional/test_or1k_sim.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - from qemu_test import LinuxKernelTest, Asset from qemu_test.utils import archive_extract diff --git a/tests/functional/test_s390x_topology.py b/tests/functional/test_s390x_topology.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_s390x_topology.py +++ b/tests/functional/test_s390x_topology.py @@ -XXX,XX +XXX,XX @@ # later. See the COPYING file in the top-level directory. import os -import time from qemu_test import QemuSystemTest, Asset from qemu_test import exec_command diff --git a/tests/functional/test_sh4_tuxrun.py b/tests/functional/test_sh4_tuxrun.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sh4_tuxrun.py +++ b/tests/functional/test_sh4_tuxrun.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os -import time - -from unittest import skipUnless from qemu_test import Asset, exec_command_and_wait_for_pattern from qemu_test.tuxruntest import TuxRunBaselineTest diff --git a/tests/functional/test_sh4eb_r2d.py b/tests/functional/test_sh4eb_r2d.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sh4eb_r2d.py +++ b/tests/functional/test_sh4eb_r2d.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern from qemu_test.utils import archive_extract -from unittest import skipUnless class R2dEBTest(LinuxKernelTest): diff --git a/tests/functional/test_virtio_version.py b/tests/functional/test_virtio_version.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_virtio_version.py +++ b/tests/functional/test_virtio_version.py @@ -XXX,XX +XXX,XX @@ # # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import sys -import os from qemu.machine import QEMUMachine from qemu_test import QemuSystemTest -- 2.46.0 Group all imports that originate in the 'qemu_test' package into one statement. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/tuxruntest.py | 9 +++++---- tests/functional/test_aarch64_raspi4.py | 4 ++-- tests/functional/test_aarch64_sbsaref.py | 5 ++--- tests/functional/test_aarch64_sbsaref_alpine.py | 5 ++--- tests/functional/test_aarch64_sbsaref_freebsd.py | 5 ++--- tests/functional/test_aarch64_virt.py | 6 ++---- tests/functional/test_arm_aspeed.py | 7 +++---- tests/functional/test_arm_bflt.py | 3 +-- tests/functional/test_arm_bpim2u.py | 8 ++++---- tests/functional/test_arm_canona1100.py | 3 +-- tests/functional/test_arm_integratorcp.py | 3 +-- tests/functional/test_arm_orangepi.py | 10 +++++----- tests/functional/test_arm_raspi2.py | 4 ++-- tests/functional/test_loongarch64_virt.py | 6 +++--- tests/functional/test_microblaze_s3adsp1800.py | 5 ++--- tests/functional/test_microblazeel_s3adsp1800.py | 5 ++--- tests/functional/test_mips64el_fuloong2e.py | 7 ++++--- tests/functional/test_mips64el_loongson3v.py | 3 +-- tests/functional/test_mips64el_malta.py | 4 ++-- tests/functional/test_mipsel_malta.py | 6 +++--- tests/functional/test_multiprocess.py | 4 ++-- tests/functional/test_netdev_ethtool.py | 3 +-- tests/functional/test_ppc64_hv.py | 4 ++-- tests/functional/test_ppc64_powernv.py | 3 +-- tests/functional/test_ppc64_pseries.py | 3 +-- tests/functional/test_ppc_405.py | 6 +++--- tests/functional/test_ppc_40p.py | 3 +-- tests/functional/test_ppc_74xx.py | 3 +-- tests/functional/test_ppc_amiga.py | 3 +-- tests/functional/test_ppc_bamboo.py | 6 +++--- tests/functional/test_ppc_mpc8544ds.py | 3 +-- tests/functional/test_ppc_virtex_ml507.py | 3 +-- tests/functional/test_riscv_opensbi.py | 4 ++-- tests/functional/test_rx_gdbsim.py | 6 +++--- tests/functional/test_s390x_ccw_virtio.py | 6 +++--- tests/functional/test_s390x_topology.py | 7 +++---- tests/functional/test_sh4eb_r2d.py | 4 ++-- tests/functional/test_sparc64_sun4u.py | 3 +-- tests/functional/test_virtio_gpu.py | 10 ++++------ 39 files changed, 85 insertions(+), 107 deletions(-) diff --git a/tests/functional/qemu_test/tuxruntest.py b/tests/functional/qemu_test/tuxruntest.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/tuxruntest.py +++ b/tests/functional/qemu_test/tuxruntest.py @@ -XXX,XX +XXX,XX @@ import stat import time -from qemu_test import QemuSystemTest -from qemu_test import exec_command, exec_command_and_wait_for_pattern -from qemu_test import wait_for_console_pattern -from qemu_test import has_cmd, run_cmd, get_qemu_img +from qemu_test import (QemuSystemTest, exec_command, + exec_command_and_wait_for_pattern, + wait_for_console_pattern, + has_cmd, run_cmd, get_qemu_img) + class TuxRunBaselineTest(QemuSystemTest): diff --git a/tests/functional/test_aarch64_raspi4.py b/tests/functional/test_aarch64_raspi4.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_raspi4.py +++ b/tests/functional/test_aarch64_raspi4.py @@ -XXX,XX +XXX,XX @@ import os -from qemu_test import LinuxKernelTest, Asset -from qemu_test import exec_command_and_wait_for_pattern +from qemu_test import (LinuxKernelTest, Asset, + exec_command_and_wait_for_pattern) from qemu_test.utils import gzip_uncompress diff --git a/tests/functional/test_aarch64_sbsaref.py b/tests/functional/test_aarch64_sbsaref.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_sbsaref.py +++ b/tests/functional/test_aarch64_sbsaref.py @@ -XXX,XX +XXX,XX @@ import os -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern -from qemu_test import interrupt_interactive_console_until_pattern +from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, + interrupt_interactive_console_until_pattern) from qemu_test.utils import lzma_uncompress def fetch_firmware(test): diff --git a/tests/functional/test_aarch64_sbsaref_alpine.py b/tests/functional/test_aarch64_sbsaref_alpine.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_sbsaref_alpine.py +++ b/tests/functional/test_aarch64_sbsaref_alpine.py @@ -XXX,XX +XXX,XX @@ import os -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern -from qemu_test import interrupt_interactive_console_until_pattern +from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, + interrupt_interactive_console_until_pattern) from unittest import skipUnless from test_aarch64_sbsaref import fetch_firmware diff --git a/tests/functional/test_aarch64_sbsaref_freebsd.py b/tests/functional/test_aarch64_sbsaref_freebsd.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_sbsaref_freebsd.py +++ b/tests/functional/test_aarch64_sbsaref_freebsd.py @@ -XXX,XX +XXX,XX @@ import os -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern -from qemu_test import interrupt_interactive_console_until_pattern +from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, + interrupt_interactive_console_until_pattern) from unittest import skipUnless from test_aarch64_sbsaref import fetch_firmware diff --git a/tests/functional/test_aarch64_virt.py b/tests/functional/test_aarch64_virt.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_virt.py +++ b/tests/functional/test_aarch64_virt.py @@ -XXX,XX +XXX,XX @@ import os import logging -from qemu_test import BUILD_DIR -from qemu_test import QemuSystemTest, Asset -from qemu_test import exec_command, wait_for_console_pattern -from qemu_test import get_qemu_img, run_cmd +from qemu_test import (BUILD_DIR, QemuSystemTest, Asset, exec_command, + wait_for_console_pattern, get_qemu_img, run_cmd) class Aarch64VirtMachine(QemuSystemTest): diff --git a/tests/functional/test_arm_aspeed.py b/tests/functional/test_arm_aspeed.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_aspeed.py +++ b/tests/functional/test_arm_aspeed.py @@ -XXX,XX +XXX,XX @@ import subprocess import tempfile -from qemu_test import LinuxKernelTest, Asset -from qemu_test import exec_command_and_wait_for_pattern -from qemu_test import interrupt_interactive_console_until_pattern -from qemu_test import has_cmd +from qemu_test import (LinuxKernelTest, Asset, + exec_command_and_wait_for_pattern, + interrupt_interactive_console_until_pattern, has_cmd) from qemu_test.utils import archive_extract from zipfile import ZipFile from unittest import skipUnless diff --git a/tests/functional/test_arm_bflt.py b/tests/functional/test_arm_bflt.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bflt.py +++ b/tests/functional/test_arm_bflt.py @@ -XXX,XX +XXX,XX @@ import os import bz2 -from qemu_test import QemuUserTest, Asset -from qemu_test import has_cmd +from qemu_test import QemuUserTest, Asset, has_cmd from qemu_test.utils import cpio_extract from unittest import skipUnless diff --git a/tests/functional/test_arm_bpim2u.py b/tests/functional/test_arm_bpim2u.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bpim2u.py +++ b/tests/functional/test_arm_bpim2u.py @@ -XXX,XX +XXX,XX @@ import os -from qemu_test import LinuxKernelTest, exec_command_and_wait_for_pattern -from qemu_test import Asset, interrupt_interactive_console_until_pattern -from qemu_test.utils import archive_extract, gzip_uncompress, lzma_uncompress -from qemu_test.utils import image_pow2ceil_expand +from qemu_test import (LinuxKernelTest, exec_command_and_wait_for_pattern, + Asset, interrupt_interactive_console_until_pattern) +from qemu_test.utils import (archive_extract, gzip_uncompress, lzma_uncompress, + image_pow2ceil_expand) from unittest import skipUnless class BananaPiMachine(LinuxKernelTest): diff --git a/tests/functional/test_arm_canona1100.py b/tests/functional/test_arm_canona1100.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_canona1100.py +++ b/tests/functional/test_arm_canona1100.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern from qemu_test.utils import archive_extract class CanonA1100Machine(QemuSystemTest): diff --git a/tests/functional/test_arm_integratorcp.py b/tests/functional/test_arm_integratorcp.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_integratorcp.py +++ b/tests/functional/test_arm_integratorcp.py @@ -XXX,XX +XXX,XX @@ import os import logging -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern from unittest import skipUnless diff --git a/tests/functional/test_arm_orangepi.py b/tests/functional/test_arm_orangepi.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_orangepi.py +++ b/tests/functional/test_arm_orangepi.py @@ -XXX,XX +XXX,XX @@ import os import shutil -from qemu_test import LinuxKernelTest, exec_command_and_wait_for_pattern -from qemu_test import Asset, interrupt_interactive_console_until_pattern -from qemu_test import wait_for_console_pattern -from qemu_test.utils import archive_extract, gzip_uncompress, lzma_uncompress -from qemu_test.utils import image_pow2ceil_expand +from qemu_test import (LinuxKernelTest, exec_command_and_wait_for_pattern, + Asset, interrupt_interactive_console_until_pattern, + wait_for_console_pattern) +from qemu_test.utils import (archive_extract, gzip_uncompress, lzma_uncompress, + image_pow2ceil_expand) from unittest import skipUnless class BananaPiMachine(LinuxKernelTest): diff --git a/tests/functional/test_arm_raspi2.py b/tests/functional/test_arm_raspi2.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_raspi2.py +++ b/tests/functional/test_arm_raspi2.py @@ -XXX,XX +XXX,XX @@ import os -from qemu_test import LinuxKernelTest, Asset -from qemu_test import exec_command_and_wait_for_pattern +from qemu_test import (LinuxKernelTest, Asset, + exec_command_and_wait_for_pattern) from qemu_test.utils import gzip_uncompress diff --git a/tests/functional/test_loongarch64_virt.py b/tests/functional/test_loongarch64_virt.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_loongarch64_virt.py +++ b/tests/functional/test_loongarch64_virt.py @@ -XXX,XX +XXX,XX @@ # Copyright (c) 2023 Loongson Technology Corporation Limited # -from qemu_test import QemuSystemTest, Asset -from qemu_test import exec_command_and_wait_for_pattern -from qemu_test import wait_for_console_pattern +from qemu_test import (QemuSystemTest, Asset, + exec_command_and_wait_for_pattern, + wait_for_console_pattern) class LoongArchMachine(QemuSystemTest): KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 ' diff --git a/tests/functional/test_microblaze_s3adsp1800.py b/tests/functional/test_microblaze_s3adsp1800.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_microblaze_s3adsp1800.py +++ b/tests/functional/test_microblaze_s3adsp1800.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from qemu_test import exec_command, exec_command_and_wait_for_pattern -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import (exec_command, exec_command_and_wait_for_pattern, + QemuSystemTest, Asset, wait_for_console_pattern) from qemu_test.utils import archive_extract class MicroblazeMachine(QemuSystemTest): diff --git a/tests/functional/test_microblazeel_s3adsp1800.py b/tests/functional/test_microblazeel_s3adsp1800.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_microblazeel_s3adsp1800.py +++ b/tests/functional/test_microblazeel_s3adsp1800.py @@ -XXX,XX +XXX,XX @@ # later. See the COPYING file in the top-level directory. import time -from qemu_test import exec_command, exec_command_and_wait_for_pattern -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import (exec_command, exec_command_and_wait_for_pattern, + QemuSystemTest, Asset, wait_for_console_pattern) from qemu_test.utils import archive_extract class MicroblazeelMachine(QemuSystemTest): diff --git a/tests/functional/test_mips64el_fuloong2e.py b/tests/functional/test_mips64el_fuloong2e.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_fuloong2e.py +++ b/tests/functional/test_mips64el_fuloong2e.py @@ -XXX,XX +XXX,XX @@ import os import subprocess -from qemu_test import LinuxKernelTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import (LinuxKernelTest, Asset, wait_for_console_pattern, + skipUntrustedTest) from unittest import skipUnless + class MipsFuloong2e(LinuxKernelTest): timeout = 60 @@ -XXX,XX +XXX,XX @@ def test_linux_kernel_3_16(self): console_pattern = 'Kernel command line: %s' % kernel_command_line self.wait_for_console_pattern(console_pattern) - @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code') + @skipUntrustedTest() @skipUnless(os.getenv('RESCUE_YL_PATH'), 'RESCUE_YL_PATH not available') def test_linux_kernel_2_6_27_isa_serial(self): # Recovery system for the Yeeloong laptop diff --git a/tests/functional/test_mips64el_loongson3v.py b/tests/functional/test_mips64el_loongson3v.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_loongson3v.py +++ b/tests/functional/test_mips64el_loongson3v.py @@ -XXX,XX +XXX,XX @@ import os from unittest import skipUnless -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern class MipsLoongson3v(QemuSystemTest): timeout = 60 diff --git a/tests/functional/test_mips64el_malta.py b/tests/functional/test_mips64el_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_malta.py +++ b/tests/functional/test_mips64el_malta.py @@ -XXX,XX +XXX,XX @@ import os import logging -from qemu_test import LinuxKernelTest, Asset -from qemu_test import exec_command_and_wait_for_pattern +from qemu_test import (LinuxKernelTest, Asset, + exec_command_and_wait_for_pattern) from qemu_test.utils import gzip_uncompress from unittest import skipUnless diff --git a/tests/functional/test_mipsel_malta.py b/tests/functional/test_mipsel_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mipsel_malta.py +++ b/tests/functional/test_mipsel_malta.py @@ -XXX,XX +XXX,XX @@ import os -from qemu_test import QemuSystemTest, LinuxKernelTest, Asset -from qemu_test import interrupt_interactive_console_until_pattern -from qemu_test import wait_for_console_pattern +from qemu_test import (QemuSystemTest, LinuxKernelTest, Asset, + interrupt_interactive_console_until_pattern, + wait_for_console_pattern) from qemu_test.utils import lzma_uncompress from zipfile import ZipFile diff --git a/tests/functional/test_multiprocess.py b/tests/functional/test_multiprocess.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_multiprocess.py +++ b/tests/functional/test_multiprocess.py @@ -XXX,XX +XXX,XX @@ import os import socket -from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern -from qemu_test import exec_command, exec_command_and_wait_for_pattern +from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, + exec_command, exec_command_and_wait_for_pattern) class Multiprocess(QemuSystemTest): diff --git a/tests/functional/test_netdev_ethtool.py b/tests/functional/test_netdev_ethtool.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_netdev_ethtool.py +++ b/tests/functional/test_netdev_ethtool.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from unittest import skip -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern class NetDevEthtool(QemuSystemTest): diff --git a/tests/functional/test_ppc64_hv.py b/tests/functional/test_ppc64_hv.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc64_hv.py +++ b/tests/functional/test_ppc64_hv.py @@ -XXX,XX +XXX,XX @@ # later. See the COPYING file in the top-level directory. from unittest import skipIf, skipUnless -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern, exec_command +from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, + exec_command) import os import time import subprocess diff --git a/tests/functional/test_ppc64_powernv.py b/tests/functional/test_ppc64_powernv.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc64_powernv.py +++ b/tests/functional/test_ppc64_powernv.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from qemu_test import LinuxKernelTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import LinuxKernelTest, Asset, wait_for_console_pattern class powernvMachine(LinuxKernelTest): diff --git a/tests/functional/test_ppc64_pseries.py b/tests/functional/test_ppc64_pseries.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc64_pseries.py +++ b/tests/functional/test_ppc64_pseries.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern class pseriesMachine(QemuSystemTest): diff --git a/tests/functional/test_ppc_405.py b/tests/functional/test_ppc_405.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_405.py +++ b/tests/functional/test_ppc_405.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern -from qemu_test import exec_command_and_wait_for_pattern +from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, + exec_command_and_wait_for_pattern) + class Ppc405Machine(QemuSystemTest): diff --git a/tests/functional/test_ppc_40p.py b/tests/functional/test_ppc_40p.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_40p.py +++ b/tests/functional/test_ppc_40p.py @@ -XXX,XX +XXX,XX @@ import os from unittest import skipUnless -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern class IbmPrep40pMachine(QemuSystemTest): diff --git a/tests/functional/test_ppc_74xx.py b/tests/functional/test_ppc_74xx.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_74xx.py +++ b/tests/functional/test_ppc_74xx.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from qemu_test import QemuSystemTest -from qemu_test import wait_for_console_pattern +from qemu_test import QemuSystemTest, wait_for_console_pattern class ppc74xxCpu(QemuSystemTest): diff --git a/tests/functional/test_ppc_amiga.py b/tests/functional/test_ppc_amiga.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_amiga.py +++ b/tests/functional/test_ppc_amiga.py @@ -XXX,XX +XXX,XX @@ import subprocess -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern, run_cmd +from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern, run_cmd from zipfile import ZipFile class AmigaOneMachine(QemuSystemTest): diff --git a/tests/functional/test_ppc_bamboo.py b/tests/functional/test_ppc_bamboo.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_bamboo.py +++ b/tests/functional/test_ppc_bamboo.py @@ -XXX,XX +XXX,XX @@ # later. See the COPYING file in the top-level directory. from qemu_test.utils import archive_extract -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern -from qemu_test import exec_command_and_wait_for_pattern +from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, + exec_command_and_wait_for_pattern) + class BambooMachine(QemuSystemTest): diff --git a/tests/functional/test_ppc_mpc8544ds.py b/tests/functional/test_ppc_mpc8544ds.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_mpc8544ds.py +++ b/tests/functional/test_ppc_mpc8544ds.py @@ -XXX,XX +XXX,XX @@ # later. See the COPYING file in the top-level directory. from qemu_test.utils import archive_extract -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern class Mpc8544dsMachine(QemuSystemTest): diff --git a/tests/functional/test_ppc_virtex_ml507.py b/tests/functional/test_ppc_virtex_ml507.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_virtex_ml507.py +++ b/tests/functional/test_ppc_virtex_ml507.py @@ -XXX,XX +XXX,XX @@ # later. See the COPYING file in the top-level directory. from qemu_test.utils import archive_extract -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern class VirtexMl507Machine(QemuSystemTest): diff --git a/tests/functional/test_riscv_opensbi.py b/tests/functional/test_riscv_opensbi.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_riscv_opensbi.py +++ b/tests/functional/test_riscv_opensbi.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from qemu_test import QemuSystemTest -from qemu_test import wait_for_console_pattern +from qemu_test import QemuSystemTest, wait_for_console_pattern + class RiscvOpenSBI(QemuSystemTest): diff --git a/tests/functional/test_rx_gdbsim.py b/tests/functional/test_rx_gdbsim.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_rx_gdbsim.py +++ b/tests/functional/test_rx_gdbsim.py @@ -XXX,XX +XXX,XX @@ import os from unittest import skipUnless -from qemu_test import QemuSystemTest, Asset -from qemu_test import exec_command_and_wait_for_pattern -from qemu_test import wait_for_console_pattern +from qemu_test import (QemuSystemTest, Asset, + exec_command_and_wait_for_pattern, + wait_for_console_pattern) from qemu_test.utils import gzip_uncompress diff --git a/tests/functional/test_s390x_ccw_virtio.py b/tests/functional/test_s390x_ccw_virtio.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_s390x_ccw_virtio.py +++ b/tests/functional/test_s390x_ccw_virtio.py @@ -XXX,XX +XXX,XX @@ import os import tempfile -from qemu_test import QemuSystemTest, Asset -from qemu_test import exec_command_and_wait_for_pattern -from qemu_test import wait_for_console_pattern +from qemu_test import (QemuSystemTest, Asset, + exec_command_and_wait_for_pattern, + wait_for_console_pattern) from qemu_test.utils import lzma_uncompress class S390CCWVirtioMachine(QemuSystemTest): diff --git a/tests/functional/test_s390x_topology.py b/tests/functional/test_s390x_topology.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_s390x_topology.py +++ b/tests/functional/test_s390x_topology.py @@ -XXX,XX +XXX,XX @@ import os -from qemu_test import QemuSystemTest, Asset -from qemu_test import exec_command -from qemu_test import exec_command_and_wait_for_pattern -from qemu_test import wait_for_console_pattern +from qemu_test import (QemuSystemTest, Asset, exec_command, + exec_command_and_wait_for_pattern, + wait_for_console_pattern) from qemu_test.utils import lzma_uncompress diff --git a/tests/functional/test_sh4eb_r2d.py b/tests/functional/test_sh4eb_r2d.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sh4eb_r2d.py +++ b/tests/functional/test_sh4eb_r2d.py @@ -XXX,XX +XXX,XX @@ import os import shutil -from qemu_test import LinuxKernelTest, Asset -from qemu_test import exec_command_and_wait_for_pattern +from qemu_test import (LinuxKernelTest, Asset, + exec_command_and_wait_for_pattern) from qemu_test.utils import archive_extract class R2dEBTest(LinuxKernelTest): diff --git a/tests/functional/test_sparc64_sun4u.py b/tests/functional/test_sparc64_sun4u.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sparc64_sun4u.py +++ b/tests/functional/test_sparc64_sun4u.py @@ -XXX,XX +XXX,XX @@ import os -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern from qemu_test.utils import archive_extract class Sun4uMachine(QemuSystemTest): diff --git a/tests/functional/test_virtio_gpu.py b/tests/functional/test_virtio_gpu.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_virtio_gpu.py +++ b/tests/functional/test_virtio_gpu.py @@ -XXX,XX +XXX,XX @@ # later. See the COPYING file in the top-level directory. -from qemu_test import BUILD_DIR -from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern -from qemu_test import exec_command_and_wait_for_pattern -from qemu_test import is_readable_executable_file - +from qemu_test import (BUILD_DIR, QemuSystemTest, Asset, + wait_for_console_pattern, + exec_command_and_wait_for_pattern, + is_readable_executable_file) from qemu.utils import kvm_available import os -- 2.46.0 The xorriso command directly writes to 'filename', so the surrounding 'with' statement is pointless. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/test_ppc64_hv.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/functional/test_ppc64_hv.py b/tests/functional/test_ppc64_hv.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc64_hv.py +++ b/tests/functional/test_ppc64_hv.py @@ -XXX,XX +XXX,XX @@ def extract_from_iso(self, iso, path): cwd = os.getcwd() os.chdir(self.workdir) - with open(filename, "w") as outfile: - cmd = "xorriso -osirrox on -indev %s -cpx %s %s" % (iso, path, filename) - subprocess.run(cmd.split(), - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + cmd = "xorriso -osirrox on -indev %s -cpx %s %s" % (iso, path, filename) + subprocess.run(cmd.split(), + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) os.chmod(filename, 0o600) os.chdir(cwd) -- 2.46.0 Put the 'which' function into shared code. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/__init__.py | 2 +- tests/functional/qemu_test/cmd.py | 10 ++++++++++ tests/functional/test_acpi_bits.py | 13 +------------ tests/functional/test_ppc64_hv.py | 13 +------------ 4 files changed, 13 insertions(+), 25 deletions(-) diff --git a/tests/functional/qemu_test/__init__.py b/tests/functional/qemu_test/__init__.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/__init__.py +++ b/tests/functional/qemu_test/__init__.py @@ -XXX,XX +XXX,XX @@ from .config import BUILD_DIR from .cmd import has_cmd, has_cmds, run_cmd, is_readable_executable_file, \ interrupt_interactive_console_until_pattern, wait_for_console_pattern, \ - exec_command, exec_command_and_wait_for_pattern, get_qemu_img + exec_command, exec_command_and_wait_for_pattern, get_qemu_img, which from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest from .linuxkernel import LinuxKernelTest diff --git a/tests/functional/qemu_test/cmd.py b/tests/functional/qemu_test/cmd.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/cmd.py +++ b/tests/functional/qemu_test/cmd.py @@ -XXX,XX +XXX,XX @@ from .config import BUILD_DIR +def which(tool): + """ looks up the full path for @tool, returns None if not found + or if @tool does not have executable permissions. + """ + paths=os.getenv('PATH') + for p in paths.split(os.path.pathsep): + p = os.path.join(p, tool) + if os.path.exists(p) and os.access(p, os.X_OK): + return p + return None def has_cmd(name, args=None): """ diff --git a/tests/functional/test_acpi_bits.py b/tests/functional/test_acpi_bits.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_acpi_bits.py +++ b/tests/functional/test_acpi_bits.py @@ -XXX,XX +XXX,XX @@ ) from qemu.machine import QEMUMachine from unittest import skipIf -from qemu_test import QemuBaseTest, Asset +from qemu_test import QemuBaseTest, Asset, which deps = ["xorriso", "mformat"] # dependent tools needed in the test setup/box. supported_platforms = ['x86_64'] # supported test platforms. @@ -XXX,XX +XXX,XX @@ # default timeout of 120 secs is sometimes not enough for bits test. BITS_TIMEOUT = 200 -def which(tool): - """ looks up the full path for @tool, returns None if not found - or if @tool does not have executable permissions. - """ - paths=os.getenv('PATH') - for p in paths.split(os.path.pathsep): - p = os.path.join(p, tool) - if os.path.exists(p) and os.access(p, os.X_OK): - return p - return None - def missing_deps(): """ returns True if any of the test dependent tools are absent. """ diff --git a/tests/functional/test_ppc64_hv.py b/tests/functional/test_ppc64_hv.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc64_hv.py +++ b/tests/functional/test_ppc64_hv.py @@ -XXX,XX +XXX,XX @@ from unittest import skipIf, skipUnless from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, - exec_command) + exec_command, which) import os import time import subprocess @@ -XXX,XX +XXX,XX @@ deps = ["xorriso"] # dependent tools needed in the test setup/box. -def which(tool): - """ looks up the full path for @tool, returns None if not found - or if @tool does not have executable permissions. - """ - paths=os.getenv('PATH') - for p in paths.split(os.path.pathsep): - p = os.path.join(p, tool) - if os.path.exists(p) and os.access(p, os.X_OK): - return p - return None - def missing_deps(): """ returns True if any of the test dependent tools are absent. """ -- 2.46.0 Reduce repeated boilerplate with some helper decorators: @skipIfNotPlatform("x86_64", "aarch64") => Skip unless the build host platform matches @skipIfMissingCommands("mkisofs", "losetup") => Skips unless all listed commands are found in $PATH @skipIfMissingImports("numpy", "cv2") => Skips unless all listed modules can be imported @skipFlakyTest("https://gitlab.com/qemu-project/qemu/-/issues/NNN") => Skips unless env var requests flaky tests with the reason documented in the referenced gitlab bug @skipBigData => Skips unless env var permits tests creating big data files @skipUntrustedTest => Skips unless env var permits tests which are potentially dangerous to the host Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/__init__.py | 3 + tests/functional/qemu_test/decorators.py | 105 +++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 tests/functional/qemu_test/decorators.py diff --git a/tests/functional/qemu_test/__init__.py b/tests/functional/qemu_test/__init__.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/__init__.py +++ b/tests/functional/qemu_test/__init__.py @@ -XXX,XX +XXX,XX @@ exec_command, exec_command_and_wait_for_pattern, get_qemu_img, which from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest from .linuxkernel import LinuxKernelTest +from .decorators import skipIfMissingCommands, skipIfNotMachine, \ + skipFlakyTest, skipUntrustedTest, skipBigDataTest, \ + skipIfMissingImports diff --git a/tests/functional/qemu_test/decorators.py b/tests/functional/qemu_test/decorators.py new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/functional/qemu_test/decorators.py @@ -XXX,XX +XXX,XX @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Decorators useful in functional tests + +import os +import platform +from unittest import skipUnless + +from .cmd import which + +''' +Decorator to skip execution of a test if the list +of command binaries is not available in $PATH. +Example: + + @skipIfMissingCommands("mkisofs", "losetup") +''' +def skipIfMissingCommands(*args): + def has_cmds(cmdlist): + for cmd in cmdlist: + if not which(cmd): + return False + return True + + return skipUnless(lambda: has_cmds(args), + 'required commands(s) "%s" not installed' % + ", ".join(args)) + +''' +Decorator to skip execution of a test if the current +host machine does not match one of the permitted +machines. +Example + + @skipIfNotMachine("x86_64", "aarch64") +''' +def skipIfNotMachine(*args): + return skipUnless(lambda: platform.machine() in args, + 'not running on required machine(s) "%s"' % + ", ".join(args)) + +''' +Decorator to skip execution of flaky tests, unless +the $QEMU_TEST_FLAKY_TESTS env var is set. A bug URL +must be provided that documents the observed failure +behaviour, so it can be tracked & re-evaluated in future. + +Historical tests may be providing "None" as the bug_url +but this should not be done for new test. + +Example: + + @skipFlakyTest("https://gitlab.com/qemu-project/qemu/-/issues/NNN") +''' +def skipFlakyTest(bug_url): + if bug_url is None: + bug_url = "FIXME: reproduce flaky test and file bug report or remove" + return skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), + f'Test is unstable: {bug_url}') + +''' +Decorator to skip execution of tests which are likely +to execute untrusted commands on the host, or commands +which process untrusted code, unles the +$QEMU_TEST_ALLOW_UNTRUSTED_CODE env var is set. +Example: + + @skipUntrustedTest() +''' +def skipUntrustedTest(): + return skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), + 'Test runs untrusted code / processes untrusted data') + +''' +Decorator to skip execution of tests which need large +data storage on the host, unless the +$QEMU_TEST_ALLOW_LARGE_STORAGE env var is set + +Example: + + @skipBigDataTest() +''' +def skipBigDataTest(): + return skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), + 'Test required large host storage space') + +''' +Decorator to skip execution of a test if the list +of python imports is not available. +Example: + + @skipIfMissingImports("numpy", "cv2") +''' +def skipIfMissingImports(*args): + def has_imports(importlist): + for impname in importlist: + try: + import impname + except ImportError: + return False + return True + + return skipUnless(lambda: has_imports(args), + 'required imports(s) "%s" not installed' % + ", ".join(args)) -- 2.46.0 This ensures consistency of behaviour across all the tests, and requires that we provide gitlab bug links when marking a test to be skipped due to unreliability. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/test_acpi_bits.py | 24 +++------------- tests/functional/test_arm_aspeed.py | 6 ++-- tests/functional/test_arm_bflt.py | 7 +++-- tests/functional/test_arm_bpim2u.py | 7 +++-- tests/functional/test_arm_integratorcp.py | 27 ++++++------------ tests/functional/test_arm_orangepi.py | 8 +++--- tests/functional/test_linux_initrd.py | 7 ++--- tests/functional/test_m68k_nextcube.py | 14 +++------ tests/functional/test_mips64el_loongson3v.py | 8 +++--- tests/functional/test_mips64el_malta.py | 30 +++++++------------- tests/functional/test_ppc64_hv.py | 17 ++--------- tests/functional/test_ppc_40p.py | 8 ++---- tests/functional/test_rx_gdbsim.py | 6 ++-- tests/functional/test_sh4_r2d.py | 8 ++---- 14 files changed, 62 insertions(+), 115 deletions(-) diff --git a/tests/functional/test_acpi_bits.py b/tests/functional/test_acpi_bits.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_acpi_bits.py +++ b/tests/functional/test_acpi_bits.py @@ -XXX,XX +XXX,XX @@ Sequence, ) from qemu.machine import QEMUMachine -from unittest import skipIf -from qemu_test import QemuBaseTest, Asset, which +from qemu_test import (QemuBaseTest, Asset, skipIfMissingCommands, + skipIfNotMachine) -deps = ["xorriso", "mformat"] # dependent tools needed in the test setup/box. -supported_platforms = ['x86_64'] # supported test platforms. # default timeout of 120 secs is sometimes not enough for bits test. BITS_TIMEOUT = 200 -def missing_deps(): - """ returns True if any of the test dependent tools are absent. - """ - for dep in deps: - if which(dep) is None: - return True - return False - -def supported_platform(): - """ checks if the test is running on a supported platform. - """ - return platform.machine() in supported_platforms - class QEMUBitsMachine(QEMUMachine): # pylint: disable=too-few-public-methods """ A QEMU VM, with isa-debugcon enabled and bits iso passed @@ -XXX,XX +XXX,XX @@ def base_args(self): """return the base argument to QEMU binary""" return self._base_args -@skipIf(not supported_platform() or missing_deps(), - 'unsupported platform or dependencies (%s) not installed' \ - % ','.join(deps)) +@skipIfMissingCommands("xorriso", "mformat") +@skipIfNotMachine("x86_64") class AcpiBitsTest(QemuBaseTest): #pylint: disable=too-many-instance-attributes """ ACPI and SMBIOS tests using biosbits. diff --git a/tests/functional/test_arm_aspeed.py b/tests/functional/test_arm_aspeed.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_aspeed.py +++ b/tests/functional/test_arm_aspeed.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (LinuxKernelTest, Asset, exec_command_and_wait_for_pattern, - interrupt_interactive_console_until_pattern, has_cmd) + interrupt_interactive_console_until_pattern, + skipIfMissingCommands, +) from qemu_test.utils import archive_extract from zipfile import ZipFile from unittest import skipUnless @@ -XXX,XX +XXX,XX @@ def test_arm_ast2600_evb_buildroot(self): 'images/ast2600-evb/buildroot-2023.02-tpm/flash.img'), 'a46009ae8a5403a0826d607215e731a8c68d27c14c41e55331706b8f9c7bd997') - @skipUnless(*has_cmd('swtpm')) + @skipIfMissingCommands('swtpm') def test_arm_ast2600_evb_buildroot_tpm(self): self.set_machine('ast2600-evb') diff --git a/tests/functional/test_arm_bflt.py b/tests/functional/test_arm_bflt.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bflt.py +++ b/tests/functional/test_arm_bflt.py @@ -XXX,XX +XXX,XX @@ import bz2 from qemu_test import QemuUserTest, Asset, has_cmd +from qemu_test import (QemuUserTest, Asset, skipIfMissingCommands, + skipUntrustedTest) from qemu_test.utils import cpio_extract -from unittest import skipUnless class LoadBFLT(QemuUserTest): @@ -XXX,XX +XXX,XX @@ class LoadBFLT(QemuUserTest): ('https://elinux.org/images/5/51/Stm32_mini_rootfs.cpio.bz2'), 'eefb788e4980c9e8d6c9d60ce7d15d4da6bf4fbc6a80f487673824600d5ba9cc') - @skipUnless(*has_cmd('cpio')) - @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code') + @skipIfMissingCommands('cpio') + @skipUntrustedTest() def test_stm32(self): # See https://elinux.org/STM32#User_Space rootfs_path_bz2 = self.ASSET_ROOTFS.fetch() diff --git a/tests/functional/test_arm_bpim2u.py b/tests/functional/test_arm_bpim2u.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bpim2u.py +++ b/tests/functional/test_arm_bpim2u.py @@ -XXX,XX +XXX,XX @@ import os from qemu_test import (LinuxKernelTest, exec_command_and_wait_for_pattern, - Asset, interrupt_interactive_console_until_pattern) + Asset, interrupt_interactive_console_until_pattern, + skipBigDataTest) from qemu_test.utils import (archive_extract, gzip_uncompress, lzma_uncompress, image_pow2ceil_expand) -from unittest import skipUnless + class BananaPiMachine(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_gmac(self): os.remove(dtb_path) os.remove(rootfs_path) - @skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), 'storage limited') + @skipBigDataTest() def test_arm_bpim2u_openwrt_22_03_3(self): self.set_machine('bpim2u') # This test download a 8.9 MiB compressed image and expand it diff --git a/tests/functional/test_arm_integratorcp.py b/tests/functional/test_arm_integratorcp.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_integratorcp.py +++ b/tests/functional/test_arm_integratorcp.py @@ -XXX,XX +XXX,XX @@ import os import logging -from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern -from unittest import skipUnless - - -NUMPY_AVAILABLE = True -try: - import numpy as np -except ImportError: - NUMPY_AVAILABLE = False - -CV2_AVAILABLE = True -try: - import cv2 -except ImportError: - CV2_AVAILABLE = False +from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, + skipIfMissingImports, skipUntrustedTest) class IntegratorMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def boot_integratorcp(self): '-append', 'printk.time=0 console=ttyAMA0') self.vm.launch() - @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code') + @skipUntrustedTest() def test_integratorcp_console(self): """ Boots the Linux kernel and checks that the console is operational @@ -XXX,XX +XXX,XX @@ def test_integratorcp_console(self): self.boot_integratorcp() wait_for_console_pattern(self, 'Log in as root') - @skipUnless(NUMPY_AVAILABLE, 'Python NumPy not installed') - @skipUnless(CV2_AVAILABLE, 'Python OpenCV not installed') - @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code') + @skipIfMissingImports("numpy", "cv2") + @skipUntrustedTest() def test_framebuffer_tux_logo(self): """ Boot Linux and verify the Tux logo is displayed on the framebuffer. """ + import numpy as np + import cv2 + screendump_path = os.path.join(self.workdir, "screendump.pbm") tuxlogo_path = self.ASSET_TUXLOGO.fetch() diff --git a/tests/functional/test_arm_orangepi.py b/tests/functional/test_arm_orangepi.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_orangepi.py +++ b/tests/functional/test_arm_orangepi.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (LinuxKernelTest, exec_command_and_wait_for_pattern, Asset, interrupt_interactive_console_until_pattern, - wait_for_console_pattern) + wait_for_console_pattern, skipBigDataTest) from qemu_test.utils import (archive_extract, gzip_uncompress, lzma_uncompress, image_pow2ceil_expand) -from unittest import skipUnless + class BananaPiMachine(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_sd(self): os.remove(dtb_path) os.remove(rootfs_path) - @skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), 'storage limited') + @skipBigDataTest() def test_arm_orangepi_armbian(self): self.set_machine('orangepi-pc') # This test download a 275 MiB compressed image and expand it @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_armbian(self): 'to <orangepipc>') self.wait_for_console_pattern('Starting Load Kernel Modules...') - @skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), 'storage limited') + @skipBigDataTest() def test_arm_orangepi_uboot_netbsd9(self): self.set_machine('orangepi-pc') # This test download a 304MB compressed image and expand it to 2GB diff --git a/tests/functional/test_linux_initrd.py b/tests/functional/test_linux_initrd.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_linux_initrd.py +++ b/tests/functional/test_linux_initrd.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import os import logging import tempfile -from qemu_test import QemuSystemTest, Asset -from unittest import skipUnless +from qemu_test import QemuSystemTest, Asset, skipFlakyTest class LinuxInitrd(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def test_with_2gib_file_should_exit_error_msg_with_linux_v3_6(self): max_size + 1) self.assertRegex(self.vm.get_log(), expected_msg) - @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab') + # XXX file tracking bug + @skipFlakyTest(bug_url=None) def test_with_2gib_file_should_work_with_linux_v4_16(self): """ QEMU has supported up to 4 GiB initrd for recent kernel diff --git a/tests/functional/test_m68k_nextcube.py b/tests/functional/test_m68k_nextcube.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_m68k_nextcube.py +++ b/tests/functional/test_m68k_nextcube.py @@ -XXX,XX +XXX,XX @@ import os import time -from qemu_test import QemuSystemTest, Asset -from unittest import skipUnless - +from qemu_test import QemuSystemTest, Asset, skipIfMissingImports from qemu_test.tesseract import tesseract_available, tesseract_ocr - -PIL_AVAILABLE = True -try: - from PIL import Image -except ImportError: - PIL_AVAILABLE = False +from unittest import skipUnless class NextCubeMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def check_bootrom_framebuffer(self, screenshot_path): self.vm.cmd('human-monitor-command', command_line='screendump %s' % screenshot_path) - @skipUnless(PIL_AVAILABLE, 'Python PIL not installed') + @skipIfMissingImports("PIL") def test_bootrom_framebuffer_size(self): self.set_machine('next-cube') screenshot_path = os.path.join(self.workdir, "dump.ppm") self.check_bootrom_framebuffer(screenshot_path) + from PIL import Image width, height = Image.open(screenshot_path).size self.assertEqual(width, 1120) self.assertEqual(height, 832) diff --git a/tests/functional/test_mips64el_loongson3v.py b/tests/functional/test_mips64el_loongson3v.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_loongson3v.py +++ b/tests/functional/test_mips64el_loongson3v.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - from unittest import skipUnless -from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern +from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, + skipUntrustedTest) + class MipsLoongson3v(QemuSystemTest): timeout = 60 @@ -XXX,XX +XXX,XX @@ class MipsLoongson3v(QemuSystemTest): 'releases/download/20210112/pmon-3avirt.bin'), 'fcdf6bb2cb7885a4a62f31fcb0d5e368bac7b6cea28f40c6dfa678af22fea20a') - @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code') + @skipUntrustedTest() def test_pmon_serial_console(self): self.set_machine('loongson3-virt') diff --git a/tests/functional/test_mips64el_malta.py b/tests/functional/test_mips64el_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_malta.py +++ b/tests/functional/test_mips64el_malta.py @@ -XXX,XX +XXX,XX @@ import logging from qemu_test import (LinuxKernelTest, Asset, - exec_command_and_wait_for_pattern) + exec_command_and_wait_for_pattern, + skipIfMissingImports, skipFlakyTest, skipUntrustedTest) from qemu_test.utils import gzip_uncompress -from unittest import skipUnless - -NUMPY_AVAILABLE = True -try: - import numpy as np -except ImportError: - NUMPY_AVAILABLE = False - -CV2_AVAILABLE = True -try: - import cv2 -except ImportError: - CV2_AVAILABLE = False class MaltaMachineConsole(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_mips64el_malta(self): 'rootfs.mipsel64r1.cpio.gz'), '75ba10cd35fb44e32948eeb26974f061b703c81c4ba2fab1ebcacf1d1bec3b61') - @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code') + @skipUntrustedTest() def test_mips64el_malta_5KEc_cpio(self): kernel_path = self.ASSET_KERNEL_3_19_3.fetch() initrd_path_gz = self.ASSET_CPIO_R1.fetch() @@ -XXX,XX +XXX,XX @@ def test_mips64el_malta_5KEc_cpio(self): self.vm.wait() -@skipUnless(NUMPY_AVAILABLE, 'Python NumPy not installed') -@skipUnless(CV2_AVAILABLE, 'Python OpenCV not installed') +@skipIfMissingImports('numpy', 'cv2') class MaltaMachineFramebuffer(LinuxKernelTest): timeout = 30 @@ -XXX,XX +XXX,XX @@ def do_test_i6400_framebuffer_logo(self, cpu_cores_count): """ Boot Linux kernel and check Tux logo is displayed on the framebuffer. """ + + import numpy as np + import cv2 + screendump_path = os.path.join(self.workdir, 'screendump.pbm') kernel_path_gz = self.ASSET_KERNEL_4_7_0.fetch() @@ -XXX,XX +XXX,XX @@ def do_test_i6400_framebuffer_logo(self, cpu_cores_count): def test_mips_malta_i6400_framebuffer_logo_1core(self): self.do_test_i6400_framebuffer_logo(1) - @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab') + # XXX file tracking bug + @skipFlakyTest(bug_url=None) def test_mips_malta_i6400_framebuffer_logo_7cores(self): self.do_test_i6400_framebuffer_logo(7) - @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab') + @skipFlakyTest(bug_url=None) def test_mips_malta_i6400_framebuffer_logo_8cores(self): self.do_test_i6400_framebuffer_logo(8) diff --git a/tests/functional/test_ppc64_hv.py b/tests/functional/test_ppc64_hv.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc64_hv.py +++ b/tests/functional/test_ppc64_hv.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from unittest import skipIf, skipUnless from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, - exec_command, which) + exec_command, skipIfMissingCommands, skipBigDataTest) import os import time import subprocess from datetime import datetime -deps = ["xorriso"] # dependent tools needed in the test setup/box. - -def missing_deps(): - """ returns True if any of the test dependent tools are absent. - """ - for dep in deps: - if which(dep) is None: - return True - return False - # Alpine is a light weight distro that supports QEMU. These tests boot # that on the machine then run a QEMU guest inside it in KVM mode, # that runs the same Alpine distro image. @@ -XXX,XX +XXX,XX @@ def missing_deps(): # large download, but it may be more polite to create qcow2 image with # QEMU already installed and use that. # XXX: The order of these tests seems to matter, see git blame. -@skipIf(missing_deps(), 'dependencies (%s) not installed' % ','.join(deps)) -@skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), 'storage limited') +@skipIfMissingCommands("xorriso") +@skipBigDataTest() class HypervisorTest(QemuSystemTest): timeout = 1000 diff --git a/tests/functional/test_ppc_40p.py b/tests/functional/test_ppc_40p.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_40p.py +++ b/tests/functional/test_ppc_40p.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import os - -from unittest import skipUnless -from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern +from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, + skipUntrustedTest) class IbmPrep40pMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ class IbmPrep40pMachine(QemuSystemTest): # All rights reserved. # U.S. Government Users Restricted Rights - Use, duplication or disclosure # restricted by GSA ADP Schedule Contract with IBM Corp. - @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code') + @skipUntrustedTest() def test_factory_firmware_and_netbsd(self): self.set_machine('40p') self.require_accelerator("tcg") diff --git a/tests/functional/test_rx_gdbsim.py b/tests/functional/test_rx_gdbsim.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_rx_gdbsim.py +++ b/tests/functional/test_rx_gdbsim.py @@ -XXX,XX +XXX,XX @@ import os -from unittest import skipUnless from qemu_test import (QemuSystemTest, Asset, exec_command_and_wait_for_pattern, - wait_for_console_pattern) + wait_for_console_pattern, skipFlakyTest) from qemu_test.utils import gzip_uncompress @@ -XXX,XX +XXX,XX @@ def test_uboot(self): # FIXME limit baudrate on chardev, else we type too fast #exec_command_and_wait_for_pattern(self, 'version', gcc_version) - @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab') + # XXX file tracking bug + @skipFlakyTest(bug_url=None) def test_linux_sash(self): """ Boots a Linux kernel and checks that the console is operational. diff --git a/tests/functional/test_sh4_r2d.py b/tests/functional/test_sh4_r2d.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sh4_r2d.py +++ b/tests/functional/test_sh4_r2d.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - -from qemu_test import LinuxKernelTest, Asset +from qemu_test import LinuxKernelTest, Asset, skipFlakyTest from qemu_test.utils import archive_extract -from unittest import skipUnless class R2dTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class R2dTest(LinuxKernelTest): # This test has a 6-10% failure rate on various hosts that look # like issues with a buggy kernel. - @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable') + # XXX file tracking bug + @skipFlakyTest(bug_url=None) def test_r2d(self): self.set_machine('r2d') file_path = self.ASSET_DAY09.fetch() -- 2.46.0 Add helper methods that construct paths for * log files - to be preserved at the end of a test * scratch files - to be purged at the end of a test * build files - anything relative to the build root * data files - anything relative to the functional test source root * socket files - a short temporary dir to avoid UNIX socket limits These are to be used instead of direct access to the self.workdir, or self.logdir variables, or any other place where paths are built manually. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/testcase.py | 86 ++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -XXX,XX +XXX,XX @@ import logging import os +from pathlib import Path import pycotap import shutil import subprocess import sys +import tempfile import unittest import uuid @@ -XXX,XX +XXX,XX @@ class QemuBaseTest(unittest.TestCase): log = None logdir = None + def socket_dir(self): + if self.socketdir is None: + self.socketdir = tempfile.TemporaryDirectory( + prefix="qemu_func_test_sock_") + return self.socketdir + + ''' + @params args list of zero or more subdirectories or file + + Construct a path for accessing a data file located + relative to the source directory that is the root for + functional tests. + + @args may be an empty list to reference the root dir + itself, may be a single element to reference a file in + the root directory, or may be multiple elements to + reference a file nested below. The path components + will be joined using the platform appropriate path + separator. + + Returns: a qualified path + ''' + def data_file(self, *args): + return str(Path(Path(__file__).parent.parent, *args)) + + ''' + @params args list of zero or more subdirectories or file + + Construct a path for accessing a data file located + relative to the build directory root. + + @args may be an empty list to reference the build dir + itself, may be a single element to reference a file in + the build directory, or may be multiple elements to + reference a file nested below. The path components + will be joined using the platform appropriate path + separator. + + Returns: a qualified path + ''' + def build_file(self, *args): + return str(Path(BUILD_DIR, *args)) + + ''' + @params args list of zero or more subdirectories or file + + Construct a path for accessing/creating a scratch file + located relative to a temporary directory dedicated to + this test case. The directory and its contents will be + purged upon completion of the test. + + @args may be an empty list to reference the scratch dir + itself, may be a single element to reference a file in + the scratch directory, or may be multiple elements to + reference a file nested below. The path components + will be joined using the platform appropriate path + separator. + + Returns: a qualified path + ''' + def scratch_file(self, *args): + return str(Path(self.workdir, *args)) + + ''' + @params args list of zero or more subdirectories or file + + Construct a path for accessing/creating a log file + located relative to a temporary directory dedicated to + this test case. The directory and its log files will be + preserved upon completion of the test. + + @args may be an empty list to reference the log dir + itself, may be a single element to reference a file in + the log directory, or may be multiple elements to + reference a file nested below. The path components + will be joined using the platform appropriate path + separator. + + Returns: a pathlib.Path object + ''' + def log_file(self, *args): + return str(Path(self.logdir, *args)) + def setUp(self, bin_prefix): self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set') self.arch = self.qemu_bin.split('-')[-1] + self.socketdir = None self.outputdir = os.path.join(BUILD_DIR, 'tests', 'functional', self.arch, self.id()) -- 2.46.0 This removes direct access of the 'self.logdir' variable. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/testcase.py | 9 ++++----- tests/functional/test_virtio_gpu.py | 4 +--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -XXX,XX +XXX,XX @@ def scratch_file(self, *args): Returns: a pathlib.Path object ''' def log_file(self, *args): - return str(Path(self.logdir, *args)) + return str(Path(self.outputdir, *args)) def setUp(self, bin_prefix): self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set') @@ -XXX,XX +XXX,XX @@ def setUp(self, bin_prefix): self.workdir = os.path.join(self.outputdir, 'scratch') os.makedirs(self.workdir, exist_ok=True) - self.logdir = self.outputdir - self.log_filename = os.path.join(self.logdir, 'base.log') + self.log_filename = self.log_file('base.log') self.log = logging.getLogger('qemu-test') self.log.setLevel(logging.DEBUG) self._log_fh = logging.FileHandler(self.log_filename, mode='w') @@ -XXX,XX +XXX,XX @@ def setUp(self): console_log = logging.getLogger('console') console_log.setLevel(logging.DEBUG) - self.console_log_name = os.path.join(self.logdir, 'console.log') + self.console_log_name = self.log_file('console.log') self._console_log_fh = logging.FileHandler(self.console_log_name, mode='w') self._console_log_fh.setLevel(logging.DEBUG) @@ -XXX,XX +XXX,XX @@ def _new_vm(self, name, *args): vm = QEMUMachine(self.qemu_bin, name=name, base_temp_dir=self.workdir, - log_dir=self.logdir) + log_dir=self.log_file()) self.log.debug('QEMUMachine "%s" created', name) self.log.debug('QEMUMachine "%s" temp_dir: %s', name, vm.temp_dir) diff --git a/tests/functional/test_virtio_gpu.py b/tests/functional/test_virtio_gpu.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_virtio_gpu.py +++ b/tests/functional/test_virtio_gpu.py @@ -XXX,XX +XXX,XX @@ def test_vhost_user_vga_virgl(self): os.set_inheritable(qemu_sock.fileno(), True) os.set_inheritable(vug_sock.fileno(), True) - self._vug_log_path = os.path.join( - self.logdir, "vhost-user-gpu.log" - ) + self._vug_log_path = self.log_file("vhost-user-gpu.log") self._vug_log_file = open(self._vug_log_path, "wb") self.log.info('Complete vhost-user-gpu.log file can be ' 'found at %s', self._vug_log_path) -- 2.46.0 This removes direct access of the 'BUILD_DIR' variable. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/testcase.py | 4 ++-- tests/functional/test_aarch64_virt.py | 6 +++--- tests/functional/test_virtio_gpu.py | 9 ++++----- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -XXX,XX +XXX,XX @@ def setUp(self, bin_prefix): self.arch = self.qemu_bin.split('-')[-1] self.socketdir = None - self.outputdir = os.path.join(BUILD_DIR, 'tests', 'functional', - self.arch, self.id()) + self.outputdir = self.build_file('tests', 'functional', + self.arch, self.id()) self.workdir = os.path.join(self.outputdir, 'scratch') os.makedirs(self.workdir, exist_ok=True) diff --git a/tests/functional/test_aarch64_virt.py b/tests/functional/test_aarch64_virt.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_virt.py +++ b/tests/functional/test_aarch64_virt.py @@ -XXX,XX +XXX,XX @@ import os import logging -from qemu_test import (BUILD_DIR, QemuSystemTest, Asset, exec_command, +from qemu_test import (QemuSystemTest, Asset, exec_command, wait_for_console_pattern, get_qemu_img, run_cmd) @@ -XXX,XX +XXX,XX @@ def test_alpine_virt_tcg_gic_max(self): "mte=on," "gic-version=max,iommu=smmuv3") self.vm.add_args("-smp", "2", "-m", "1024") - self.vm.add_args('-bios', os.path.join(BUILD_DIR, 'pc-bios', - 'edk2-aarch64-code.fd')) + self.vm.add_args('-bios', self.build_file('pc-bios', + 'edk2-aarch64-code.fd')) self.vm.add_args("-drive", f"file={iso_path},media=cdrom,format=raw") self.vm.add_args('-device', 'virtio-rng-pci,rng=rng0') self.vm.add_args('-object', 'rng-random,id=rng0,filename=/dev/urandom') diff --git a/tests/functional/test_virtio_gpu.py b/tests/functional/test_virtio_gpu.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_virtio_gpu.py +++ b/tests/functional/test_virtio_gpu.py @@ -XXX,XX +XXX,XX @@ # later. See the COPYING file in the top-level directory. -from qemu_test import (BUILD_DIR, QemuSystemTest, Asset, - wait_for_console_pattern, +from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, exec_command_and_wait_for_pattern, is_readable_executable_file) from qemu.utils import kvm_available @@ -XXX,XX +XXX,XX @@ import subprocess -def pick_default_vug_bin(): +def pick_default_vug_bin(test): relative_path = "./contrib/vhost-user-gpu/vhost-user-gpu" if is_readable_executable_file(relative_path): return relative_path - bld_dir_path = os.path.join(BUILD_DIR, relative_path) + bld_dir_path = test.build_file(relative_path) if is_readable_executable_file(bld_dir_path): return bld_dir_path @@ -XXX,XX +XXX,XX @@ def test_vhost_user_vga_virgl(self): # FIXME: should check presence of vhost-user-gpu, virgl, memfd etc self.require_accelerator('kvm') - vug = pick_default_vug_bin() + vug = pick_default_vug_bin(self) if not vug: self.skipTest("Could not find vhost-user-gpu") -- 2.46.0 This removes direct path manipulation to figure out the source dir Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/test_acpi_bits.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/tests/functional/test_acpi_bits.py b/tests/functional/test_acpi_bits.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_acpi_bits.py +++ b/tests/functional/test_acpi_bits.py @@ -XXX,XX +XXX,XX @@ import tempfile import zipfile -from pathlib import Path from typing import ( List, Optional, @@ -XXX,XX +XXX,XX @@ class AcpiBitsTest(QemuBaseTest): #pylint: disable=too-many-instance-attributes def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._vm = None - self._baseDir = None self._debugcon_addr = '0x403' self._debugcon_log = 'debugcon-log.txt' @@ -XXX,XX +XXX,XX @@ def _print_log(self, log): def copy_bits_config(self): """ copies the bios bits config file into bits. """ - config_file = 'bits-cfg.txt' - bits_config_dir = os.path.join(self._baseDir, 'acpi-bits', - 'bits-config') + bits_config_file = self.data_file('acpi-bits', + 'bits-config', + 'bits-cfg.txt') target_config_dir = os.path.join(self.workdir, 'bits-%d' %self.BITS_INTERNAL_VER, 'boot') - self.assertTrue(os.path.exists(bits_config_dir)) + self.assertTrue(os.path.exists(bits_config_file)) self.assertTrue(os.path.exists(target_config_dir)) - self.assertTrue(os.access(os.path.join(bits_config_dir, - config_file), os.R_OK)) - shutil.copy2(os.path.join(bits_config_dir, config_file), - target_config_dir) + shutil.copy2(bits_config_file, target_config_dir) self.logger.info('copied config file %s to %s', - config_file, target_config_dir) + bits_config_file, target_config_dir) def copy_test_scripts(self): """copies the python test scripts into bits. """ - bits_test_dir = os.path.join(self._baseDir, 'acpi-bits', - 'bits-tests') + bits_test_dir = self.data_file('acpi-bits', 'bits-tests') target_test_dir = os.path.join(self.workdir, 'bits-%d' %self.BITS_INTERNAL_VER, 'boot', 'python') @@ -XXX,XX +XXX,XX @@ def setUp(self): # pylint: disable=arguments-differ super().setUp('qemu-system-') self.logger = self.log - self._baseDir = Path(__file__).parent - prebuiltDir = os.path.join(self.workdir, 'prebuilt') if not os.path.isdir(prebuiltDir): os.mkdir(prebuiltDir, mode=0o775) -- 2.46.0 Replace any instances of os.path.join(self.workdir, ".../...") self.workdir + "/.../..." with self.scratch_file("...", "...") which is more compact and portable Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/linuxkernel.py | 7 ++- tests/functional/qemu_test/tuxruntest.py | 2 +- tests/functional/test_aarch64_aspeed.py | 17 ++++--- tests/functional/test_aarch64_raspi3.py | 3 +- tests/functional/test_aarch64_raspi4.py | 4 +- tests/functional/test_aarch64_sbsaref.py | 7 ++- tests/functional/test_aarch64_virt.py | 2 +- tests/functional/test_acpi_bits.py | 48 +++++++++---------- tests/functional/test_alpha_clipper.py | 4 +- tests/functional/test_arm_aspeed.py | 14 +++--- tests/functional/test_arm_bflt.py | 3 +- tests/functional/test_arm_bpim2u.py | 6 +-- tests/functional/test_arm_canona1100.py | 3 +- tests/functional/test_arm_integratorcp.py | 3 +- tests/functional/test_arm_orangepi.py | 8 ++-- tests/functional/test_arm_raspi2.py | 4 +- tests/functional/test_arm_vexpress.py | 5 +- tests/functional/test_m68k_mcf5208evb.py | 3 +- tests/functional/test_m68k_nextcube.py | 5 +- .../functional/test_microblaze_s3adsp1800.py | 3 +- .../test_microblazeel_s3adsp1800.py | 5 +- tests/functional/test_mips64el_malta.py | 6 +-- tests/functional/test_mips_malta.py | 4 +- tests/functional/test_mipsel_malta.py | 4 +- tests/functional/test_or1k_sim.py | 2 +- tests/functional/test_ppc64_e500.py | 2 +- tests/functional/test_ppc64_hv.py | 12 +---- tests/functional/test_ppc_amiga.py | 12 +++-- tests/functional/test_ppc_bamboo.py | 10 ++-- tests/functional/test_ppc_mac.py | 2 +- tests/functional/test_ppc_mpc8544ds.py | 2 +- tests/functional/test_ppc_virtex_ml507.py | 5 +- tests/functional/test_rx_gdbsim.py | 4 +- tests/functional/test_s390x_ccw_virtio.py | 2 +- tests/functional/test_s390x_topology.py | 4 +- tests/functional/test_sh4_r2d.py | 3 +- tests/functional/test_sh4eb_r2d.py | 7 ++- tests/functional/test_sparc64_sun4u.py | 5 +- tests/functional/test_sparc_sun4m.py | 2 +- tests/functional/test_xtensa_lx60.py | 3 +- 40 files changed, 116 insertions(+), 131 deletions(-) diff --git a/tests/functional/qemu_test/linuxkernel.py b/tests/functional/qemu_test/linuxkernel.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/linuxkernel.py +++ b/tests/functional/qemu_test/linuxkernel.py @@ -XXX,XX +XXX,XX @@ def extract_from_deb(self, deb_path, path): os.chdir(cwd) # Return complete path to extracted file. Because callers to # extract_from_deb() specify 'path' with a leading slash, it is - # necessary to use os.path.relpath() as otherwise os.path.join() - # interprets it as an absolute path and drops the self.workdir part. - return os.path.normpath(os.path.join(self.workdir, - os.path.relpath(path, '/'))) + # necessary to use 'relative_to()' to turn it into a relative + # path for joining to the scratch dir + return os.path.normpath(self.scratch_file(os.path.relpath(path, '/'))) diff --git a/tests/functional/qemu_test/tuxruntest.py b/tests/functional/qemu_test/tuxruntest.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/tuxruntest.py +++ b/tests/functional/qemu_test/tuxruntest.py @@ -XXX,XX +XXX,XX @@ def fetch_tuxrun_assets(self, kernel_asset, rootfs_asset, dtb_asset=None): kernel_image = kernel_asset.fetch() disk_image_zst = rootfs_asset.fetch() - disk_image = self.workdir + "/rootfs.ext4" + disk_image = self.scratch_file("rootfs.ext4") run_cmd([self.zstd, "-f", "-d", disk_image_zst, "-o", disk_image]) diff --git a/tests/functional/test_aarch64_aspeed.py b/tests/functional/test_aarch64_aspeed.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/test_aarch64_aspeed.py +++ b/tests/functional/test_aarch64_aspeed.py @@ -XXX,XX +XXX,XX @@ def test_aarch64_ast2700_evb_sdk_v09_02(self): archive_extract(image_path, self.workdir) num_cpu = 4 - image_dir = self.workdir + '/ast2700-default/' - uboot_size = os.path.getsize(image_dir + 'u-boot-nodtb.bin') + uboot_size = os.path.getsize(self.scratch_file('ast2700-default', + 'u-boot-nodtb.bin')) uboot_dtb_load_addr = hex(0x400000000 + uboot_size) load_images_list = [ { 'addr': '0x400000000', - 'file': image_dir + 'u-boot-nodtb.bin' + 'file': self.scratch_file('ast2700-default', + 'u-boot-nodtb.bin') }, { 'addr': str(uboot_dtb_load_addr), - 'file': image_dir + 'u-boot.dtb' + 'file': self.scratch_file('ast2700-default', 'u-boot.dtb') }, { 'addr': '0x430000000', - 'file': image_dir + 'bl31.bin' + 'file': self.scratch_file('ast2700-default', 'bl31.bin') }, { 'addr': '0x430080000', - 'file': image_dir + 'optee/tee-raw.bin' + 'file': self.scratch_file('ast2700-default', 'optee', + 'tee-raw.bin') } ] @@ -XXX,XX +XXX,XX @@ def test_aarch64_ast2700_evb_sdk_v09_02(self): self.vm.add_args('-smp', str(num_cpu)) self.vm.add_args('-device', 'tmp105,bus=aspeed.i2c.bus.1,address=0x4d,id=tmp-test') - self.do_test_aarch64_aspeed_sdk_start(image_dir + 'image-bmc') + self.do_test_aarch64_aspeed_sdk_start( + self.scratch_file('ast2700-default', 'image-bmc')) wait_for_console_pattern(self, 'ast2700-default login:') diff --git a/tests/functional/test_aarch64_raspi3.py b/tests/functional/test_aarch64_raspi3.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_raspi3.py +++ b/tests/functional/test_aarch64_raspi3.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os from zipfile import ZipFile from qemu_test import LinuxKernelTest, Asset @@ -XXX,XX +XXX,XX @@ def test_aarch64_raspi3_atf(self): with ZipFile(zip_path, 'r') as zf: zf.extract(efi_name, path=self.workdir) - efi_fd = os.path.join(self.workdir, efi_name) + efi_fd = self.scratch_file(efi_name) self.set_machine('raspi3b') self.vm.set_console(console_index=1) diff --git a/tests/functional/test_aarch64_raspi4.py b/tests/functional/test_aarch64_raspi4.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_raspi4.py +++ b/tests/functional/test_aarch64_raspi4.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - from qemu_test import (LinuxKernelTest, Asset, exec_command_and_wait_for_pattern) from qemu_test.utils import gzip_uncompress @@ -XXX,XX +XXX,XX @@ def test_arm_raspi4_initrd(self): kernel_path = self.extract_from_deb(deb_path, '/boot/kernel8.img') dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2711-rpi-4-b.dtb') initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = os.path.join(self.workdir, 'rootfs.cpio') + initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) self.set_machine('raspi4b') diff --git a/tests/functional/test_aarch64_sbsaref.py b/tests/functional/test_aarch64_sbsaref.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_sbsaref.py +++ b/tests/functional/test_aarch64_sbsaref.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, interrupt_interactive_console_until_pattern) from qemu_test.utils import lzma_uncompress + def fetch_firmware(test): """ Flash volumes generated using: @@ -XXX,XX +XXX,XX @@ def fetch_firmware(test): # Secure BootRom (TF-A code) fs0_xz_path = Aarch64SbsarefMachine.ASSET_FLASH0.fetch() - fs0_path = os.path.join(test.workdir, "SBSA_FLASH0.fd") + fs0_path = test.scratch_file("SBSA_FLASH0.fd") lzma_uncompress(fs0_xz_path, fs0_path) # Non-secure rom (UEFI and EFI variables) fs1_xz_path = Aarch64SbsarefMachine.ASSET_FLASH1.fetch() - fs1_path = os.path.join(test.workdir, "SBSA_FLASH1.fd") + fs1_path = test.scratch_file("SBSA_FLASH1.fd") lzma_uncompress(fs1_xz_path, fs1_path) for path in [fs0_path, fs1_path]: diff --git a/tests/functional/test_aarch64_virt.py b/tests/functional/test_aarch64_virt.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_virt.py +++ b/tests/functional/test_aarch64_virt.py @@ -XXX,XX +XXX,XX @@ def common_aarch64_virt(self, machine): # Also add a scratch block device logger.info('creating scratch qcow2 image') - image_path = os.path.join(self.workdir, 'scratch.qcow2') + image_path = self.scratch_file('scratch.qcow2') qemu_img = get_qemu_img(self) run_cmd([qemu_img, 'create', '-f', 'qcow2', image_path, '8M']) diff --git a/tests/functional/test_acpi_bits.py b/tests/functional/test_acpi_bits.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_acpi_bits.py +++ b/tests/functional/test_acpi_bits.py @@ -XXX,XX +XXX,XX @@ def copy_bits_config(self): bits_config_file = self.data_file('acpi-bits', 'bits-config', 'bits-cfg.txt') - target_config_dir = os.path.join(self.workdir, - 'bits-%d' %self.BITS_INTERNAL_VER, - 'boot') + target_config_dir = self.scratch_file('bits-%d' % self.BITS_INTERNAL_VER, + 'boot') self.assertTrue(os.path.exists(bits_config_file)) self.assertTrue(os.path.exists(target_config_dir)) shutil.copy2(bits_config_file, target_config_dir) @@ -XXX,XX +XXX,XX @@ def copy_test_scripts(self): """copies the python test scripts into bits. """ bits_test_dir = self.data_file('acpi-bits', 'bits-tests') - target_test_dir = os.path.join(self.workdir, - 'bits-%d' %self.BITS_INTERNAL_VER, - 'boot', 'python') + target_test_dir = self.scratch_file('bits-%d' % self.BITS_INTERNAL_VER, + 'boot', 'python') self.assertTrue(os.path.exists(bits_test_dir)) self.assertTrue(os.path.exists(target_test_dir)) @@ -XXX,XX +XXX,XX @@ def fix_mkrescue(self, mkrescue): the directory where we have extracted our pre-built bits grub tarball. """ - grub_x86_64_mods = os.path.join(self.workdir, 'grub-inst-x86_64-efi') - grub_i386_mods = os.path.join(self.workdir, 'grub-inst') + grub_x86_64_mods = self.scratch_file('grub-inst-x86_64-efi') + grub_i386_mods = self.scratch_file('grub-inst') self.assertTrue(os.path.exists(grub_x86_64_mods)) self.assertTrue(os.path.exists(grub_i386_mods)) @@ -XXX,XX +XXX,XX @@ def generate_bits_iso(self): """ Uses grub-mkrescue to generate a fresh bits iso with the python test scripts """ - bits_dir = os.path.join(self.workdir, - 'bits-%d' %self.BITS_INTERNAL_VER) - iso_file = os.path.join(self.workdir, - 'bits-%d.iso' %self.BITS_INTERNAL_VER) - mkrescue_script = os.path.join(self.workdir, - 'grub-inst-x86_64-efi', 'bin', - 'grub-mkrescue') + bits_dir = self.scratch_file('bits-%d' % self.BITS_INTERNAL_VER) + iso_file = self.scratch_file('bits-%d.iso' % self.BITS_INTERNAL_VER) + mkrescue_script = self.scratch_file('grub-inst-x86_64-efi', + 'bin', + 'grub-mkrescue') self.assertTrue(os.access(mkrescue_script, os.R_OK | os.W_OK | os.X_OK)) @@ -XXX,XX +XXX,XX @@ def setUp(self): # pylint: disable=arguments-differ super().setUp('qemu-system-') self.logger = self.log - prebuiltDir = os.path.join(self.workdir, 'prebuilt') + prebuiltDir = self.scratch_file('prebuilt') if not os.path.isdir(prebuiltDir): os.mkdir(prebuiltDir, mode=0o775) - bits_zip_file = os.path.join(prebuiltDir, 'bits-%d-%s.zip' - %(self.BITS_INTERNAL_VER, - self.BITS_COMMIT_HASH)) - grub_tar_file = os.path.join(prebuiltDir, - 'bits-%d-%s-grub.tar.gz' - %(self.BITS_INTERNAL_VER, - self.BITS_COMMIT_HASH)) + bits_zip_file = self.scratch_file('prebuilt', + 'bits-%d-%s.zip' + %(self.BITS_INTERNAL_VER, + self.BITS_COMMIT_HASH)) + grub_tar_file = self.scratch_file('prebuilt', + 'bits-%d-%s-grub.tar.gz' + %(self.BITS_INTERNAL_VER, + self.BITS_COMMIT_HASH)) bitsLocalArtLoc = self.ASSET_BITS.fetch() self.logger.info("downloaded bits artifacts to %s", bitsLocalArtLoc) @@ -XXX,XX +XXX,XX @@ def parse_log(self): """parse the log generated by running bits tests and check for failures. """ - debugconf = os.path.join(self.workdir, self._debugcon_log) + debugconf = self.scratch_file(self._debugcon_log) log = "" with open(debugconf, 'r', encoding='utf-8') as filehandle: log = filehandle.read() @@ -XXX,XX +XXX,XX @@ def tearDown(self): def test_acpi_smbios_bits(self): """The main test case implementation.""" - iso_file = os.path.join(self.workdir, - 'bits-%d.iso' %self.BITS_INTERNAL_VER) + iso_file = self.scratch_file('bits-%d.iso' % self.BITS_INTERNAL_VER) self.assertTrue(os.access(iso_file, os.R_OK)) diff --git a/tests/functional/test_alpha_clipper.py b/tests/functional/test_alpha_clipper.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_alpha_clipper.py +++ b/tests/functional/test_alpha_clipper.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - from qemu_test import LinuxKernelTest, Asset from qemu_test.utils import gzip_uncompress @@ -XXX,XX +XXX,XX @@ def test_alpha_clipper(self): self.set_machine('clipper') kernel_path = self.ASSET_KERNEL.fetch() - uncompressed_kernel = os.path.join(self.workdir, 'vmlinux') + uncompressed_kernel = self.scratch_file('vmlinux') gzip_uncompress(kernel_path, uncompressed_kernel) self.vm.set_console() diff --git a/tests/functional/test_arm_aspeed.py b/tests/functional/test_arm_aspeed.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_aspeed.py +++ b/tests/functional/test_arm_aspeed.py @@ -XXX,XX +XXX,XX @@ def test_ast1030_zephyros_1_04(self): kernel_name = "ast1030-evb-demo/zephyr.elf" with ZipFile(zip_file, 'r') as zf: zf.extract(kernel_name, path=self.workdir) - kernel_file = os.path.join(self.workdir, kernel_name) + kernel_file = self.scratch_file(kernel_name) self.vm.set_console() self.vm.add_args('-kernel', kernel_file, '-nographic') @@ -XXX,XX +XXX,XX @@ def test_ast1030_zephyros_1_07(self): kernel_name = "ast1030-evb-demo/zephyr.bin" with ZipFile(zip_file, 'r') as zf: zf.extract(kernel_name, path=self.workdir) - kernel_file = os.path.join(self.workdir, kernel_name) + kernel_file = self.scratch_file(kernel_name) self.vm.set_console() self.vm.add_args('-kernel', kernel_file, '-nographic') @@ -XXX,XX +XXX,XX @@ def test_arm_ast2600_evb_buildroot_tpm(self): image_path = self.ASSET_BR2_202302_AST2600_TPM_FLASH.fetch() - tpmstate_dir = tempfile.TemporaryDirectory(prefix="qemu_") + tpmstate_dir = self.scratch_file('swtpmstate') + os.mkdir(tpmstate_dir) + socket_dir = tempfile.TemporaryDirectory(prefix="qemu_") socket = os.path.join(tpmstate_dir.name, 'swtpm-socket') # We must put the TPM state dir in /tmp/, not the build dir, # because some distros use AppArmor to lock down swtpm and # restrict the set of locations it can access files in. subprocess.run(['swtpm', 'socket', '-d', '--tpm2', - '--tpmstate', f'dir={tpmstate_dir.name}', + '--tpmstate', f'dir={tpmstate_dir}', '--ctrl', f'type=unixio,path={socket}']) self.vm.add_args('-chardev', f'socket,id=chrtpm,path={socket}') @@ -XXX,XX +XXX,XX @@ def test_arm_ast2500_evb_sdk(self): archive_extract(image_path, self.workdir) self.do_test_arm_aspeed_sdk_start( - self.workdir + '/ast2500-default/image-bmc') + self.scratch_file('ast2500-default', 'image-bmc')) self.wait_for_console_pattern('ast2500-default login:') @@ -XXX,XX +XXX,XX @@ def test_arm_ast2600_evb_sdk(self): self.vm.add_args('-device', 'ds1338,bus=aspeed.i2c.bus.5,address=0x32'); self.do_test_arm_aspeed_sdk_start( - self.workdir + '/ast2600-a2/image-bmc') + self.scratch_file('ast2600-a2', 'image-bmc')) self.wait_for_console_pattern('ast2600-a2 login:') diff --git a/tests/functional/test_arm_bflt.py b/tests/functional/test_arm_bflt.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bflt.py +++ b/tests/functional/test_arm_bflt.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os import bz2 from qemu_test import QemuUserTest, Asset, has_cmd @@ -XXX,XX +XXX,XX @@ class LoadBFLT(QemuUserTest): def test_stm32(self): # See https://elinux.org/STM32#User_Space rootfs_path_bz2 = self.ASSET_ROOTFS.fetch() - busybox_path = os.path.join(self.workdir, "bin/busybox") + busybox_path = self.scratch_file("bin", "busybox") with bz2.open(rootfs_path_bz2, 'rb') as cpio_handle: cpio_extract(cpio_handle, self.workdir) diff --git a/tests/functional/test_arm_bpim2u.py b/tests/functional/test_arm_bpim2u.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bpim2u.py +++ b/tests/functional/test_arm_bpim2u.py @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_initrd(self): 'sun8i-r40-bananapi-m2-ultra.dtb') dtb_path = self.extract_from_deb(deb_path, dtb_path) initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = os.path.join(self.workdir, 'rootfs.cpio') + initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_gmac(self): 'sun8i-r40-bananapi-m2-ultra.dtb') dtb_path = self.extract_from_deb(deb_path, dtb_path) rootfs_path_xz = self.ASSET_ROOTFS.fetch() - rootfs_path = os.path.join(self.workdir, 'rootfs.cpio') + rootfs_path = self.scratch_file('rootfs.cpio') lzma_uncompress(rootfs_path_xz, rootfs_path) image_pow2ceil_expand(rootfs_path) @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_openwrt_22_03_3(self): # This test download a 8.9 MiB compressed image and expand it # to 127 MiB. image_path_gz = self.ASSET_SD_IMAGE.fetch() - image_path = os.path.join(self.workdir, 'sdcard.img') + image_path = self.scratch_file('sdcard.img') gzip_uncompress(image_path_gz, image_path) image_pow2ceil_expand(image_path) diff --git a/tests/functional/test_arm_canona1100.py b/tests/functional/test_arm_canona1100.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_canona1100.py +++ b/tests/functional/test_arm_canona1100.py @@ -XXX,XX +XXX,XX @@ def test_arm_canona1100(self): member="day18/barebox.canon-a1100.bin") self.vm.set_console() self.vm.add_args('-bios', - self.workdir + '/day18/barebox.canon-a1100.bin') + self.scratch_file('day18', + 'barebox.canon-a1100.bin')) self.vm.launch() wait_for_console_pattern(self, 'running /env/bin/init') diff --git a/tests/functional/test_arm_integratorcp.py b/tests/functional/test_arm_integratorcp.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_integratorcp.py +++ b/tests/functional/test_arm_integratorcp.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os import logging from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, @@ -XXX,XX +XXX,XX @@ def test_framebuffer_tux_logo(self): import numpy as np import cv2 - screendump_path = os.path.join(self.workdir, "screendump.pbm") + screendump_path = self.scratch_file("screendump.pbm") tuxlogo_path = self.ASSET_TUXLOGO.fetch() self.boot_integratorcp() diff --git a/tests/functional/test_arm_orangepi.py b/tests/functional/test_arm_orangepi.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_orangepi.py +++ b/tests/functional/test_arm_orangepi.py @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_initrd(self): dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun8i-h3-orangepi-pc.dtb' dtb_path = self.extract_from_deb(deb_path, dtb_path) initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = os.path.join(self.workdir, 'rootfs.cpio') + initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_sd(self): dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun8i-h3-orangepi-pc.dtb' dtb_path = self.extract_from_deb(deb_path, dtb_path) rootfs_path_xz = self.ASSET_ROOTFS.fetch() - rootfs_path = os.path.join(self.workdir, 'rootfs.cpio') + rootfs_path = self.scratch_file('rootfs.cpio') lzma_uncompress(rootfs_path_xz, rootfs_path) image_pow2ceil_expand(rootfs_path) @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_armbian(self): # to 1036 MiB, but the underlying filesystem is 1552 MiB... # As we expand it to 2 GiB we are safe. image_path_xz = self.ASSET_ARMBIAN.fetch() - image_path = os.path.join(self.workdir, 'armbian.img') + image_path = self.scratch_file('armbian.img') lzma_uncompress(image_path_xz, image_path) image_pow2ceil_expand(image_path) @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_uboot_netbsd9(self): uboot_path = '/usr/lib/u-boot/orangepi_plus/u-boot-sunxi-with-spl.bin' uboot_path = self.extract_from_deb(deb_path, uboot_path) image_path_gz = self.ASSET_NETBSD.fetch() - image_path = os.path.join(self.workdir, 'armv7.img') + image_path = self.scratch_file('armv7.img') gzip_uncompress(image_path_gz, image_path) image_pow2ceil_expand(image_path) image_drive_args = 'if=sd,format=raw,snapshot=on,file=' + image_path diff --git a/tests/functional/test_arm_raspi2.py b/tests/functional/test_arm_raspi2.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_raspi2.py +++ b/tests/functional/test_arm_raspi2.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - from qemu_test import (LinuxKernelTest, Asset, exec_command_and_wait_for_pattern) from qemu_test.utils import gzip_uncompress @@ -XXX,XX +XXX,XX @@ def test_arm_raspi2_initrd(self): kernel_path = self.extract_from_deb(deb_path, '/boot/kernel7.img') dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2709-rpi-2-b.dtb') initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = os.path.join(self.workdir, 'rootfs.cpio') + initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) self.set_machine('raspi2b') diff --git a/tests/functional/test_arm_vexpress.py b/tests/functional/test_arm_vexpress.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_vexpress.py +++ b/tests/functional/test_arm_vexpress.py @@ -XXX,XX +XXX,XX @@ def test_arm_vexpressa9(self): self.set_machine('vexpress-a9') file_path = self.ASSET_DAY16.fetch() archive_extract(file_path, self.workdir) - self.launch_kernel(self.workdir + '/day16/winter.zImage', - dtb=self.workdir + '/day16/vexpress-v2p-ca9.dtb', + self.launch_kernel(self.scratch_file('day16', 'winter.zImage'), + dtb=self.scratch_file('day16', + 'vexpress-v2p-ca9.dtb'), wait_for='QEMU advent calendar') if __name__ == '__main__': diff --git a/tests/functional/test_m68k_mcf5208evb.py b/tests/functional/test_m68k_mcf5208evb.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_m68k_mcf5208evb.py +++ b/tests/functional/test_m68k_mcf5208evb.py @@ -XXX,XX +XXX,XX @@ def test_m68k_mcf5208evb(self): file_path = self.ASSET_DAY07.fetch() archive_extract(file_path, self.workdir) self.vm.set_console() - self.vm.add_args('-kernel', self.workdir + '/day07/sanity-clause.elf') + self.vm.add_args('-kernel', + self.scratch_file('day07', 'sanity-clause.elf')) self.vm.launch() self.wait_for_console_pattern('QEMU advent calendar') diff --git a/tests/functional/test_m68k_nextcube.py b/tests/functional/test_m68k_nextcube.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_m68k_nextcube.py +++ b/tests/functional/test_m68k_nextcube.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import os import time from qemu_test import QemuSystemTest, Asset, skipIfMissingImports @@ -XXX,XX +XXX,XX @@ def check_bootrom_framebuffer(self, screenshot_path): @skipIfMissingImports("PIL") def test_bootrom_framebuffer_size(self): self.set_machine('next-cube') - screenshot_path = os.path.join(self.workdir, "dump.ppm") + screenshot_path = self.scratch_file("dump.ppm") self.check_bootrom_framebuffer(screenshot_path) from PIL import Image @@ -XXX,XX +XXX,XX @@ def test_bootrom_framebuffer_size(self): @skipUnless(tesseract_available(4), 'tesseract OCR tool not available') def test_bootrom_framebuffer_ocr_with_tesseract(self): self.set_machine('next-cube') - screenshot_path = os.path.join(self.workdir, "dump.ppm") + screenshot_path = self.scratch_file("dump.ppm") self.check_bootrom_framebuffer(screenshot_path) lines = tesseract_ocr(screenshot_path) text = '\n'.join(lines) diff --git a/tests/functional/test_microblaze_s3adsp1800.py b/tests/functional/test_microblaze_s3adsp1800.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_microblaze_s3adsp1800.py +++ b/tests/functional/test_microblaze_s3adsp1800.py @@ -XXX,XX +XXX,XX @@ def test_microblaze_s3adsp1800(self): file_path = self.ASSET_IMAGE.fetch() archive_extract(file_path, self.workdir) self.vm.set_console() - self.vm.add_args('-kernel', self.workdir + '/day17/ballerina.bin') + self.vm.add_args('-kernel', + self.scratch_file('day17', 'ballerina.bin')) self.vm.launch() wait_for_console_pattern(self, 'This architecture does not have ' 'kernel memory protection') diff --git a/tests/functional/test_microblazeel_s3adsp1800.py b/tests/functional/test_microblazeel_s3adsp1800.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_microblazeel_s3adsp1800.py +++ b/tests/functional/test_microblazeel_s3adsp1800.py @@ -XXX,XX +XXX,XX @@ def test_microblazeel_s3adsp1800(self): file_path = self.ASSET_IMAGE.fetch() archive_extract(file_path, self.workdir) self.vm.set_console() - self.vm.add_args('-kernel', self.workdir + '/day13/xmaton.bin') - self.vm.add_args('-nic', 'user,tftp=' + self.workdir + '/day13/') + self.vm.add_args('-kernel', self.scratch_file('day13', 'xmaton.bin')) + tftproot = self.scratch_file('day13') + self.vm.add_args('-nic', f'user,tftp={tftproot}') self.vm.launch() wait_for_console_pattern(self, 'QEMU Advent Calendar 2023') time.sleep(0.1) diff --git a/tests/functional/test_mips64el_malta.py b/tests/functional/test_mips64el_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_malta.py +++ b/tests/functional/test_mips64el_malta.py @@ -XXX,XX +XXX,XX @@ def test_mips64el_malta(self): def test_mips64el_malta_5KEc_cpio(self): kernel_path = self.ASSET_KERNEL_3_19_3.fetch() initrd_path_gz = self.ASSET_CPIO_R1.fetch() - initrd_path = os.path.join(self.workdir, 'rootfs.cpio') + initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) self.set_machine('malta') @@ -XXX,XX +XXX,XX @@ def do_test_i6400_framebuffer_logo(self, cpu_cores_count): import numpy as np import cv2 - screendump_path = os.path.join(self.workdir, 'screendump.pbm') + screendump_path = self.scratch_file('screendump.pbm') kernel_path_gz = self.ASSET_KERNEL_4_7_0.fetch() - kernel_path = self.workdir + "/vmlinux" + kernel_path = self.scratch_file("vmlinux") gzip_uncompress(kernel_path_gz, kernel_path) tuxlogo_path = self.ASSET_TUXLOGO.fetch() diff --git a/tests/functional/test_mips_malta.py b/tests/functional/test_mips_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips_malta.py +++ b/tests/functional/test_mips_malta.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern from qemu_test.utils import gzip_uncompress @@ -XXX,XX +XXX,XX @@ def test_mips_malta_cpio(self): kernel_path = self.extract_from_deb(deb_path, '/boot/vmlinux-4.5.0-2-4kc-malta') initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = os.path.join(self.workdir, 'rootfs.cpio') + initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) self.set_machine('malta') diff --git a/tests/functional/test_mipsel_malta.py b/tests/functional/test_mipsel_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mipsel_malta.py +++ b/tests/functional/test_mipsel_malta.py @@ -XXX,XX +XXX,XX @@ class MaltaMachineConsole(LinuxKernelTest): 'ce21ff4b07a981ecb8a39db2876616f5a2473eb2ab459c6f67465b9914b0c6b6') def do_test_mips_malta32el_nanomips(self, kernel_path_xz): - kernel_path = os.path.join(self.workdir, 'kernel') + kernel_path = self.scratch_file('kernel') lzma_uncompress(kernel_path_xz, kernel_path) self.set_machine('malta') @@ -XXX,XX +XXX,XX @@ def test_mipsel_malta_yamon(self): zip_path = self.ASSET_YAMON_ROM.fetch() with ZipFile(zip_path, 'r') as zf: zf.extract(yamon_bin, path=self.workdir) - yamon_path = os.path.join(self.workdir, yamon_bin) + yamon_path = self.scratch_file(yamon_bin) self.set_machine('malta') self.vm.set_console() diff --git a/tests/functional/test_or1k_sim.py b/tests/functional/test_or1k_sim.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_or1k_sim.py +++ b/tests/functional/test_or1k_sim.py @@ -XXX,XX +XXX,XX @@ def test_or1k_sim(self): file_path = self.ASSET_DAY20.fetch() archive_extract(file_path, self.workdir) self.vm.set_console() - self.vm.add_args('-kernel', self.workdir + '/day20/vmlinux') + self.vm.add_args('-kernel', self.scratch_file('day20', 'vmlinux')) self.vm.launch() self.wait_for_console_pattern('QEMU advent calendar') diff --git a/tests/functional/test_ppc64_e500.py b/tests/functional/test_ppc64_e500.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc64_e500.py +++ b/tests/functional/test_ppc64_e500.py @@ -XXX,XX +XXX,XX @@ def test_ppc64_e500(self): self.cpu = 'e5500' file_path = self.ASSET_DAY19.fetch() archive_extract(file_path, self.workdir) - self.launch_kernel(self.workdir + '/day19/uImage', + self.launch_kernel(self.scratch_file('day19', 'uImage'), wait_for='QEMU advent calendar') if __name__ == '__main__': diff --git a/tests/functional/test_ppc64_hv.py b/tests/functional/test_ppc64_hv.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc64_hv.py +++ b/tests/functional/test_ppc64_hv.py @@ -XXX,XX +XXX,XX @@ def extract_from_iso(self, iso, path): :param path: path within the iso file of the file to be extracted :returns: path of the extracted file """ - filename = os.path.basename(path) - - cwd = os.getcwd() - os.chdir(self.workdir) + filename = self.scratch_file(os.path.basename(path)) cmd = "xorriso -osirrox on -indev %s -cpx %s %s" % (iso, path, filename) subprocess.run(cmd.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) os.chmod(filename, 0o600) - os.chdir(cwd) - # Return complete path to extracted file. Because callers to - # extract_from_iso() specify 'path' with a leading slash, it is - # necessary to use os.path.relpath() as otherwise os.path.join() - # interprets it as an absolute path and drops the self.workdir part. - return os.path.normpath(os.path.join(self.workdir, filename)) + return filename def setUp(self): super().setUp() diff --git a/tests/functional/test_ppc_amiga.py b/tests/functional/test_ppc_amiga.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_amiga.py +++ b/tests/functional/test_ppc_amiga.py @@ -XXX,XX +XXX,XX @@ def test_ppc_amigaone(self): zip_file = self.ASSET_IMAGE.fetch() with ZipFile(zip_file, 'r') as zf: zf.extractall(path=self.workdir) - bios_fh = open(self.workdir + "/u-boot-amigaone.bin", "wb") - subprocess.run(['tail', '-c', '524288', - self.workdir + "/floppy_edition/updater.image"], - stdout=bios_fh) + bios = self.scratch_file("u-boot-amigaone.bin") + with open(bios, "wb") as bios_fh: + subprocess.run(['tail', '-c', '524288', + self.scratch_file("floppy_edition", + "updater.image")], + stdout=bios_fh) self.vm.set_console() - self.vm.add_args('-bios', self.workdir + '/u-boot-amigaone.bin') + self.vm.add_args('-bios', bios) self.vm.launch() wait_for_console_pattern(self, 'FLASH:') diff --git a/tests/functional/test_ppc_bamboo.py b/tests/functional/test_ppc_bamboo.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_bamboo.py +++ b/tests/functional/test_ppc_bamboo.py @@ -XXX,XX +XXX,XX @@ def test_ppc_bamboo(self): file_path = self.ASSET_IMAGE.fetch() archive_extract(file_path, self.workdir) self.vm.set_console() - self.vm.add_args('-kernel', self.workdir + - '/system-image-powerpc-440fp/linux', - '-initrd', self.workdir + - '/system-image-powerpc-440fp/rootfs.cpio.gz', + self.vm.add_args('-kernel', + self.scratch_file('system-image-powerpc-440fp', + 'linux'), + '-initrd', + self.scratch_file('system-image-powerpc-440fp', + 'rootfs.cpio.gz'), '-nic', 'user,model=rtl8139,restrict=on') self.vm.launch() wait_for_console_pattern(self, 'Type exit when done') diff --git a/tests/functional/test_ppc_mac.py b/tests/functional/test_ppc_mac.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_mac.py +++ b/tests/functional/test_ppc_mac.py @@ -XXX,XX +XXX,XX @@ def do_day15_test(self): file_path = self.ASSET_DAY15.fetch() archive_extract(file_path, self.workdir) self.vm.add_args('-M', 'graphics=off') - self.launch_kernel(self.workdir + '/day15/invaders.elf', + self.launch_kernel(self.scratch_file('day15', 'invaders.elf'), wait_for='QEMU advent calendar') def test_ppc_g3beige(self): diff --git a/tests/functional/test_ppc_mpc8544ds.py b/tests/functional/test_ppc_mpc8544ds.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_mpc8544ds.py +++ b/tests/functional/test_ppc_mpc8544ds.py @@ -XXX,XX +XXX,XX @@ def test_ppc_mpc8544ds(self): file_path = self.ASSET_IMAGE.fetch() archive_extract(file_path, self.workdir, member='creek/creek.bin') self.vm.set_console() - self.vm.add_args('-kernel', self.workdir + '/creek/creek.bin') + self.vm.add_args('-kernel', self.scratch_file('creek', 'creek.bin')) self.vm.launch() wait_for_console_pattern(self, 'QEMU advent calendar 2020', self.panic_message) diff --git a/tests/functional/test_ppc_virtex_ml507.py b/tests/functional/test_ppc_virtex_ml507.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_virtex_ml507.py +++ b/tests/functional/test_ppc_virtex_ml507.py @@ -XXX,XX +XXX,XX @@ def test_ppc_virtex_ml507(self): file_path = self.ASSET_IMAGE.fetch() archive_extract(file_path, self.workdir) self.vm.set_console() - self.vm.add_args('-kernel', self.workdir + '/hippo/hippo.linux', - '-dtb', self.workdir + '/hippo/virtex440-ml507.dtb', + self.vm.add_args('-kernel', self.scratch_file('hippo', 'hippo.linux'), + '-dtb', self.scratch_file('hippo', + 'virtex440-ml507.dtb'), '-m', '512') self.vm.launch() wait_for_console_pattern(self, 'QEMU advent calendar 2020', diff --git a/tests/functional/test_rx_gdbsim.py b/tests/functional/test_rx_gdbsim.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_rx_gdbsim.py +++ b/tests/functional/test_rx_gdbsim.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import os - from qemu_test import (QemuSystemTest, Asset, exec_command_and_wait_for_pattern, wait_for_console_pattern, skipFlakyTest) @@ -XXX,XX +XXX,XX @@ def test_uboot(self): self.set_machine('gdbsim-r5f562n8') uboot_path_gz = self.ASSET_UBOOT.fetch() - uboot_path = os.path.join(self.workdir, 'u-boot.bin') + uboot_path = self.scratch_file('u-boot.bin') gzip_uncompress(uboot_path_gz, uboot_path) self.vm.set_console() diff --git a/tests/functional/test_s390x_ccw_virtio.py b/tests/functional/test_s390x_ccw_virtio.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_s390x_ccw_virtio.py +++ b/tests/functional/test_s390x_ccw_virtio.py @@ -XXX,XX +XXX,XX @@ def test_s390x_fedora(self): kernel_path = self.ASSET_F31_KERNEL.fetch() initrd_path_xz = self.ASSET_F31_INITRD.fetch() - initrd_path = os.path.join(self.workdir, 'initrd-raw.img') + initrd_path = self.scratch_file('initrd-raw.img') lzma_uncompress(initrd_path_xz, initrd_path) self.vm.set_console() diff --git a/tests/functional/test_s390x_topology.py b/tests/functional/test_s390x_topology.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_s390x_topology.py +++ b/tests/functional/test_s390x_topology.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import os - from qemu_test import (QemuSystemTest, Asset, exec_command, exec_command_and_wait_for_pattern, wait_for_console_pattern) @@ -XXX,XX +XXX,XX @@ def kernel_init(self): self.require_accelerator("kvm") kernel_path = self.ASSET_F35_KERNEL.fetch() initrd_path_xz = self.ASSET_F35_INITRD.fetch() - initrd_path = os.path.join(self.workdir, 'initrd-raw.img') + initrd_path = self.scratch_file('initrd-raw.img') lzma_uncompress(initrd_path_xz, initrd_path) self.vm.set_console() diff --git a/tests/functional/test_sh4_r2d.py b/tests/functional/test_sh4_r2d.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sh4_r2d.py +++ b/tests/functional/test_sh4_r2d.py @@ -XXX,XX +XXX,XX @@ def test_r2d(self): file_path = self.ASSET_DAY09.fetch() archive_extract(file_path, self.workdir) self.vm.add_args('-append', 'console=ttySC1') - self.launch_kernel(self.workdir + '/day09/zImage', console_index=1, + self.launch_kernel(self.scratch_file('day09', 'zImage'), + console_index=1, wait_for='QEMU advent calendar') if __name__ == '__main__': diff --git a/tests/functional/test_sh4eb_r2d.py b/tests/functional/test_sh4eb_r2d.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sh4eb_r2d.py +++ b/tests/functional/test_sh4eb_r2d.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os import shutil from qemu_test import (LinuxKernelTest, Asset, @@ -XXX,XX +XXX,XX @@ def test_sh4eb_r2d(self): file_path = self.ASSET_TGZ.fetch() archive_extract(file_path, self.workdir) self.vm.add_args('-append', 'console=ttySC1 noiotrap') - self.launch_kernel(os.path.join(self.workdir, 'sh4eb/linux-kernel'), - initrd=os.path.join(self.workdir, 'sh4eb/initramfs.cpio.gz'), + self.launch_kernel(self.scratch_file('sh4eb', 'linux-kernel'), + initrd=self.scratch_file('sh4eb', 'initramfs.cpio.gz'), console_index=1, wait_for='Type exit when done') exec_command_and_wait_for_pattern(self, 'exit', 'Restarting system') - shutil.rmtree(os.path.join(self.workdir, 'sh4eb')) + shutil.rmtree(self.scratch_file('sh4eb')) if __name__ == '__main__': LinuxKernelTest.main() diff --git a/tests/functional/test_sparc64_sun4u.py b/tests/functional/test_sparc64_sun4u.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sparc64_sun4u.py +++ b/tests/functional/test_sparc64_sun4u.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import os - from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern from qemu_test.utils import archive_extract + class Sun4uMachine(QemuSystemTest): """Boots the Linux kernel and checks that the console is operational""" @@ -XXX,XX +XXX,XX @@ def test_sparc64_sun4u(self): kernel_name = 'day23/vmlinux' archive_extract(file_path, self.workdir, kernel_name) self.vm.set_console() - self.vm.add_args('-kernel', os.path.join(self.workdir, kernel_name), + self.vm.add_args('-kernel', self.scratch_file(kernel_name), '-append', 'printk.time=0') self.vm.launch() wait_for_console_pattern(self, 'Starting logging: OK') diff --git a/tests/functional/test_sparc_sun4m.py b/tests/functional/test_sparc_sun4m.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sparc_sun4m.py +++ b/tests/functional/test_sparc_sun4m.py @@ -XXX,XX +XXX,XX @@ def test_sparc_ss20(self): self.set_machine('SS-20') file_path = self.ASSET_DAY11.fetch() archive_extract(file_path, self.workdir) - self.launch_kernel(self.workdir + '/day11/zImage.elf', + self.launch_kernel(self.scratch_file('day11', 'zImage.elf'), wait_for='QEMU advent calendar') if __name__ == '__main__': diff --git a/tests/functional/test_xtensa_lx60.py b/tests/functional/test_xtensa_lx60.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_xtensa_lx60.py +++ b/tests/functional/test_xtensa_lx60.py @@ -XXX,XX +XXX,XX @@ def test_xtensa_lx60(self): self.cpu = 'dc233c' file_path = self.ASSET_DAY02.fetch() archive_extract(file_path, self.workdir) - self.launch_kernel(self.workdir + '/day02/santas-sleigh-ride.elf', + self.launch_kernel(self.scratch_file('day02', + 'santas-sleigh-ride.elf'), wait_for='QEMU advent calendar') if __name__ == '__main__': -- 2.46.0 This removes direct creation of temporary dirs Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/test_arm_aspeed.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/functional/test_arm_aspeed.py b/tests/functional/test_arm_aspeed.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_aspeed.py +++ b/tests/functional/test_arm_aspeed.py @@ -XXX,XX +XXX,XX @@ import os import time import subprocess -import tempfile from qemu_test import (LinuxKernelTest, Asset, exec_command_and_wait_for_pattern, @@ -XXX,XX +XXX,XX @@ def test_arm_ast2600_evb_buildroot_tpm(self): tpmstate_dir = self.scratch_file('swtpmstate') os.mkdir(tpmstate_dir) - socket_dir = tempfile.TemporaryDirectory(prefix="qemu_") - socket = os.path.join(tpmstate_dir.name, 'swtpm-socket') + socket_dir = self.socket_dir() + socket = os.path.join(socket_dir.name, 'swtpm-socket') # We must put the TPM state dir in /tmp/, not the build dir, # because some distros use AppArmor to lock down swtpm and -- 2.46.0 Everything in the scratch directory is automatically purged. Calling 'rmtree' again breaks the ability to optionally preserve the scratch directory contents. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/test_sh4eb_r2d.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/functional/test_sh4eb_r2d.py b/tests/functional/test_sh4eb_r2d.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sh4eb_r2d.py +++ b/tests/functional/test_sh4eb_r2d.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import shutil - from qemu_test import (LinuxKernelTest, Asset, exec_command_and_wait_for_pattern) from qemu_test.utils import archive_extract @@ -XXX,XX +XXX,XX @@ def test_sh4eb_r2d(self): initrd=self.scratch_file('sh4eb', 'initramfs.cpio.gz'), console_index=1, wait_for='Type exit when done') exec_command_and_wait_for_pattern(self, 'exit', 'Restarting system') - shutil.rmtree(self.scratch_file('sh4eb')) if __name__ == '__main__': LinuxKernelTest.main() -- 2.46.0 This mirrors the existing archive_extract and cpio_extract helpers Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/functional/qemu_test/utils.py b/tests/functional/qemu_test/utils.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/utils.py +++ b/tests/functional/qemu_test/utils.py @@ -XXX,XX +XXX,XX @@ def archive_extract(archive, dest_dir, member=None): else: tf.extractall(path=dest_dir) +def zip_extract(archive, dest_dir, member=None): + with zipfile.ZipFile(archive, 'r') as zf: + if member: + zf.extract(member=member, path=dest_dir) + else: + zf.extractall(path=dest_dir) + def gzip_uncompress(gz_path, output_path): if os.path.exists(output_path): return -- 2.46.0 This mirrors the existing archive_extract, cpio_extract and zip_extract helpers Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/linuxkernel.py | 15 +++++---------- tests/functional/qemu_test/utils.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/tests/functional/qemu_test/linuxkernel.py b/tests/functional/qemu_test/linuxkernel.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/linuxkernel.py +++ b/tests/functional/qemu_test/linuxkernel.py @@ -XXX,XX +XXX,XX @@ import os from .testcase import QemuSystemTest -from .cmd import run_cmd, wait_for_console_pattern -from .utils import archive_extract +from .cmd import wait_for_console_pattern +from .utils import deb_extract class LinuxKernelTest(QemuSystemTest): KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 ' @@ -XXX,XX +XXX,XX @@ def extract_from_deb(self, deb_path, path): :param path: path within the deb archive of the file to be extracted :returns: path of the extracted file """ - cwd = os.getcwd() - os.chdir(self.workdir) - (stdout, stderr, ret) = run_cmd(['ar', 't', deb_path]) - file_path = stdout.split()[2] - run_cmd(['ar', 'x', deb_path, file_path]) - archive_extract(file_path, self.workdir) - os.chdir(cwd) + relpath = os.path.relpath(path, '/') + deb_extract(deb_path, self.workdir, member="." + path) # Return complete path to extracted file. Because callers to # extract_from_deb() specify 'path' with a leading slash, it is # necessary to use 'relative_to()' to turn it into a relative # path for joining to the scratch dir - return os.path.normpath(self.scratch_file(os.path.relpath(path, '/'))) + return os.path.normpath(self.scratch_file(relpath)) diff --git a/tests/functional/qemu_test/utils.py b/tests/functional/qemu_test/utils.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/utils.py +++ b/tests/functional/qemu_test/utils.py @@ -XXX,XX +XXX,XX @@ import subprocess import tarfile +from .cmd import run_cmd + """ Round up to next power of 2 """ @@ -XXX,XX +XXX,XX @@ def zip_extract(archive, dest_dir, member=None): else: zf.extractall(path=dest_dir) +def deb_extract(archive, dest_dir, member=None): + cwd = os.getcwd() + os.chdir(dest_dir) + try: + (stdout, stderr, ret) = run_cmd(['ar', 't', archive]) + file_path = stdout.split()[2] + run_cmd(['ar', 'x', archive, file_path]) + archive_extract(file_path, dest_dir, member) + finally: + os.chdir(cwd) + def gzip_uncompress(gz_path, output_path): if os.path.exists(output_path): return -- 2.46.0 There are many types of archives that the tests deal with, and 'archive_extract' suggests it can cope with any, rather than only tar files. Rename the existing method to 'tar_extract' and add a new method that can dynamically extract any zip, tar or cpio file based on file extension. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/utils.py | 31 +++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/functional/qemu_test/utils.py b/tests/functional/qemu_test/utils.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/utils.py +++ b/tests/functional/qemu_test/utils.py @@ -XXX,XX +XXX,XX @@ import shutil import subprocess import tarfile +import zipfile from .cmd import run_cmd @@ -XXX,XX +XXX,XX @@ def image_pow2ceil_expand(path): with open(path, 'ab+') as fd: fd.truncate(size_aligned) -def archive_extract(archive, dest_dir, member=None): +def archive_extract(archive, dest_dir, format=None, member=None): + if format == "tar": + tar_extract(archive, dest_dir, member) + elif format == "zip": + zip_extract(archive, dest_dir, member) + elif format == "cpio": + if member is not None: + raise Exception("Unable to filter cpio extraction") + cpio_extract(archive, dest_dir) + elif format == "deb": + deb_extract(archive, dest_dir, "./" + member) + else: + raise Exception(f"Unknown archive format {format}") + +def guess_archive_format(path): + if ".tar." in path or path.endswith("tgz"): + return "tar" + elif path.endswith(".zip"): + return "zip" + elif path.endswith(".cpio"): + return "cpio" + elif path.endswith(".deb") or path.endswith(".udeb"): + return "deb" + else: + raise Exception(f"Unknown archive format for {path}") + +def tar_extract(archive, dest_dir, member=None): with tarfile.open(archive) as tf: if hasattr(tarfile, 'data_filter'): tf.extraction_filter = getattr(tarfile, 'data_filter', @@ -XXX,XX +XXX,XX @@ def deb_extract(archive, dest_dir, member=None): (stdout, stderr, ret) = run_cmd(['ar', 't', archive]) file_path = stdout.split()[2] run_cmd(['ar', 'x', archive, file_path]) - archive_extract(file_path, dest_dir, member) + archive_extract(file_path, dest_dir, format="tar", member=member) finally: os.chdir(cwd) -- 2.46.0 This helper wrappers utils.archive_extract, forcing the use of the scratch directory, to ensure any extracted files are cleaned at test termination. If a specific member is requested, then the path to the extracted file is also returned. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/testcase.py | 36 ++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -XXX,XX +XXX,XX @@ from .asset import Asset from .cmd import run_cmd from .config import BUILD_DIR +from .utils import (archive_extract as utils_archive_extract, + guess_archive_format) class QemuBaseTest(unittest.TestCase): @@ -XXX,XX +XXX,XX @@ class QemuBaseTest(unittest.TestCase): log = None logdir = None + ''' + @params archive: filename, Asset, or file-like object to extract + @params sub_dir: optional sub-directory to extract into + @params member: optional member file to limit extraction to + + Extracts @archive into the scratch directory, or a + directory beneath named by @sub_dir. All files are + extracted unless @member specifies a limit. + + If @member is non-None, returns the fully qualified + path to @member + ''' + def archive_extract(self, archive, format=None, sub_dir=None, member=None): + if type(archive) == Asset: + if format is None: + format = guess_archive_format(archive.url) + archive = archive.fetch() + elif format is None: + format = guess_archive_format(archive) + + if member is not None: + if os.path.isabs(member): + member = os.path.relpath(member, '/') + + if sub_dir is None: + utils_archive_extract(archive, self.scratch_file(), format, member) + else: + utils_archive_extract(archive, self.scratch_file(sub_dir), + format, member) + + if member is not None: + return self.scratch_file(member) + return None + def socket_dir(self): if self.socketdir is None: self.socketdir = tempfile.TemporaryDirectory( -- 2.46.0 Replace use of utils.archive_extract and extract_from_deb with the new archive_extract helper. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/linuxkernel.py | 19 +------------- tests/functional/test_aarch64_aspeed.py | 4 +-- tests/functional/test_aarch64_raspi3.py | 8 +----- tests/functional/test_aarch64_raspi4.py | 14 +++++----- tests/functional/test_acpi_bits.py | 15 +++-------- tests/functional/test_arm_aspeed.py | 26 +++++-------------- tests/functional/test_arm_bflt.py | 3 +-- tests/functional/test_arm_bpim2u.py | 22 +++++++--------- tests/functional/test_arm_canona1100.py | 10 +++---- tests/functional/test_arm_collie.py | 1 - tests/functional/test_arm_orangepi.py | 26 ++++++++----------- tests/functional/test_arm_raspi2.py | 14 +++++----- tests/functional/test_arm_sx1.py | 1 - tests/functional/test_arm_vexpress.py | 4 +-- tests/functional/test_m68k_mcf5208evb.py | 4 +-- tests/functional/test_m68k_q800.py | 5 ++-- .../functional/test_microblaze_s3adsp1800.py | 4 +-- .../test_microblazeel_s3adsp1800.py | 4 +-- tests/functional/test_mips64el_fuloong2e.py | 6 ++--- tests/functional/test_mips64el_malta.py | 6 ++--- tests/functional/test_mips_malta.py | 12 ++++----- tests/functional/test_mipsel_malta.py | 5 +--- tests/functional/test_or1k_sim.py | 4 +-- tests/functional/test_ppc64_e500.py | 4 +-- tests/functional/test_ppc_amiga.py | 5 +--- tests/functional/test_ppc_bamboo.py | 4 +-- tests/functional/test_ppc_mac.py | 5 +--- tests/functional/test_ppc_mpc8544ds.py | 7 +++-- tests/functional/test_ppc_virtex_ml507.py | 4 +-- tests/functional/test_sh4_r2d.py | 5 ++-- tests/functional/test_sh4eb_r2d.py | 5 ++-- tests/functional/test_sparc64_sun4u.py | 8 +++--- tests/functional/test_sparc_sun4m.py | 4 +-- tests/functional/test_xtensa_lx60.py | 4 +-- 34 files changed, 90 insertions(+), 182 deletions(-) diff --git a/tests/functional/qemu_test/linuxkernel.py b/tests/functional/qemu_test/linuxkernel.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/linuxkernel.py +++ b/tests/functional/qemu_test/linuxkernel.py @@ -XXX,XX +XXX,XX @@ from .testcase import QemuSystemTest from .cmd import wait_for_console_pattern -from .utils import deb_extract + class LinuxKernelTest(QemuSystemTest): KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 ' @@ -XXX,XX +XXX,XX @@ def launch_kernel(self, kernel, initrd=None, dtb=None, console_index=0, self.vm.launch() if wait_for: self.wait_for_console_pattern(wait_for) - - def extract_from_deb(self, deb_path, path): - """ - Extracts a file from a deb package into the test workdir - - :param deb_path: path to the deb archive - :param path: path within the deb archive of the file to be extracted - :returns: path of the extracted file - """ - relpath = os.path.relpath(path, '/') - deb_extract(deb_path, self.workdir, member="." + path) - # Return complete path to extracted file. Because callers to - # extract_from_deb() specify 'path' with a leading slash, it is - # necessary to use 'relative_to()' to turn it into a relative - # path for joining to the scratch dir - return os.path.normpath(self.scratch_file(relpath)) - diff --git a/tests/functional/test_aarch64_aspeed.py b/tests/functional/test_aarch64_aspeed.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/test_aarch64_aspeed.py +++ b/tests/functional/test_aarch64_aspeed.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern from qemu_test import exec_command_and_wait_for_pattern -from qemu_test.utils import archive_extract class AST2x00MachineSDK(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def do_test_aarch64_aspeed_sdk_start(self, image): def test_aarch64_ast2700_evb_sdk_v09_02(self): self.set_machine('ast2700-evb') - image_path = self.ASSET_SDK_V902_AST2700.fetch() - archive_extract(image_path, self.workdir) + self.archive_extract(self.ASSET_SDK_V902_AST2700) num_cpu = 4 uboot_size = os.path.getsize(self.scratch_file('ast2700-default', diff --git a/tests/functional/test_aarch64_raspi3.py b/tests/functional/test_aarch64_raspi3.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_raspi3.py +++ b/tests/functional/test_aarch64_raspi3.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -from zipfile import ZipFile - from qemu_test import LinuxKernelTest, Asset @@ -XXX,XX +XXX,XX @@ class Aarch64Raspi3Machine(LinuxKernelTest): def test_aarch64_raspi3_atf(self): efi_name = 'RPI_EFI.fd' - zip_path = self.ASSET_RPI3_UEFI.fetch() - - with ZipFile(zip_path, 'r') as zf: - zf.extract(efi_name, path=self.workdir) - efi_fd = self.scratch_file(efi_name) + efi_fd = self.archive_extract(self.ASSET_RPI3_UEFI, member=efi_name) self.set_machine('raspi3b') self.vm.set_console(console_index=1) diff --git a/tests/functional/test_aarch64_raspi4.py b/tests/functional/test_aarch64_raspi4.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_raspi4.py +++ b/tests/functional/test_aarch64_raspi4.py @@ -XXX,XX +XXX,XX @@ class Aarch64Raspi4Machine(LinuxKernelTest): '7c0b16d1853772f6f4c3ca63e789b3b9ff4936efac9c8a01fb0c98c05c7a7648') def test_arm_raspi4(self): - deb_path = self.ASSET_KERNEL_20190215.fetch() - kernel_path = self.extract_from_deb(deb_path, '/boot/kernel8.img') - dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2711-rpi-4-b.dtb') + kernel_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='/boot/kernel8.img') + dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='/boot/bcm2711-rpi-4-b.dtb') self.set_machine('raspi4b') self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_raspi4(self): def test_arm_raspi4_initrd(self): - deb_path = self.ASSET_KERNEL_20190215.fetch() - kernel_path = self.extract_from_deb(deb_path, '/boot/kernel8.img') - dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2711-rpi-4-b.dtb') + kernel_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='/boot/kernel8.img') + dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='/boot/bcm2711-rpi-4-b.dtb') initrd_path_gz = self.ASSET_INITRD.fetch() initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) diff --git a/tests/functional/test_acpi_bits.py b/tests/functional/test_acpi_bits.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_acpi_bits.py +++ b/tests/functional/test_acpi_bits.py @@ -XXX,XX +XXX,XX @@ import re import shutil import subprocess -import tarfile import tempfile -import zipfile from typing import ( List, @@ -XXX,XX +XXX,XX @@ def setUp(self): # pylint: disable=arguments-differ %(self.BITS_INTERNAL_VER, self.BITS_COMMIT_HASH)) - bitsLocalArtLoc = self.ASSET_BITS.fetch() - self.logger.info("downloaded bits artifacts to %s", bitsLocalArtLoc) - # extract the bits artifact in the temp working directory - with zipfile.ZipFile(bitsLocalArtLoc, 'r') as zref: - zref.extractall(prebuiltDir) + self.archive_extract(self.ASSET_BITS, sub_dir='prebuilt', format='zip') # extract the bits software in the temp working directory - with zipfile.ZipFile(bits_zip_file, 'r') as zref: - zref.extractall(self.workdir) - - with tarfile.open(grub_tar_file, 'r', encoding='utf-8') as tarball: - tarball.extractall(self.workdir) + self.archive_extract(bits_zip_file) + self.archive_extract(grub_tar_file) self.copy_test_scripts() self.copy_bits_config() diff --git a/tests/functional/test_arm_aspeed.py b/tests/functional/test_arm_aspeed.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_aspeed.py +++ b/tests/functional/test_arm_aspeed.py @@ -XXX,XX +XXX,XX @@ interrupt_interactive_console_until_pattern, skipIfMissingCommands, ) -from qemu_test.utils import archive_extract -from zipfile import ZipFile from unittest import skipUnless class AST1030Machine(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class AST1030Machine(LinuxKernelTest): def test_ast1030_zephyros_1_04(self): self.set_machine('ast1030-evb') - zip_file = self.ASSET_ZEPHYR_1_04.fetch() - - kernel_name = "ast1030-evb-demo/zephyr.elf" - with ZipFile(zip_file, 'r') as zf: - zf.extract(kernel_name, path=self.workdir) - kernel_file = self.scratch_file(kernel_name) + kernel_file = self.archive_extract( + self.ASSET_ZEPHYR_1_04, member="ast1030-evb-demo/zephyr.elf") self.vm.set_console() self.vm.add_args('-kernel', kernel_file, '-nographic') @@ -XXX,XX +XXX,XX @@ def test_ast1030_zephyros_1_04(self): def test_ast1030_zephyros_1_07(self): self.set_machine('ast1030-evb') - zip_file = self.ASSET_ZEPHYR_1_07.fetch() - - kernel_name = "ast1030-evb-demo/zephyr.bin" - with ZipFile(zip_file, 'r') as zf: - zf.extract(kernel_name, path=self.workdir) - kernel_file = self.scratch_file(kernel_name) + kernel_file = self.archive_extract( + self.ASSET_ZEPHYR_1_07, member="ast1030-evb-demo/zephyr.bin") self.vm.set_console() self.vm.add_args('-kernel', kernel_file, '-nographic') @@ -XXX,XX +XXX,XX @@ def do_test_arm_aspeed_sdk_start(self, image): def test_arm_ast2500_evb_sdk(self): self.set_machine('ast2500-evb') - image_path = self.ASSET_SDK_V806_AST2500.fetch() - - archive_extract(image_path, self.workdir) + self.archive_extract(self.ASSET_SDK_V806_AST2500) self.do_test_arm_aspeed_sdk_start( self.scratch_file('ast2500-default', 'image-bmc')) @@ -XXX,XX +XXX,XX @@ def test_arm_ast2500_evb_sdk(self): def test_arm_ast2600_evb_sdk(self): self.set_machine('ast2600-evb') - image_path = self.ASSET_SDK_V806_AST2600_A2.fetch() - - archive_extract(image_path, self.workdir) + self.archive_extract(self.ASSET_SDK_V806_AST2600_A2) self.vm.add_args('-device', 'tmp105,bus=aspeed.i2c.bus.5,address=0x4d,id=tmp-test'); diff --git a/tests/functional/test_arm_bflt.py b/tests/functional/test_arm_bflt.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bflt.py +++ b/tests/functional/test_arm_bflt.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuUserTest, Asset, has_cmd from qemu_test import (QemuUserTest, Asset, skipIfMissingCommands, skipUntrustedTest) -from qemu_test.utils import cpio_extract class LoadBFLT(QemuUserTest): @@ -XXX,XX +XXX,XX @@ def test_stm32(self): busybox_path = self.scratch_file("bin", "busybox") with bz2.open(rootfs_path_bz2, 'rb') as cpio_handle: - cpio_extract(cpio_handle, self.workdir) + self.archive_extract(cpio_handle, format="cpio") res = self.run_cmd(busybox_path) ver = 'BusyBox v1.24.0.git (2015-02-03 22:17:13 CET) multi-call binary.' diff --git a/tests/functional/test_arm_bpim2u.py b/tests/functional/test_arm_bpim2u.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bpim2u.py +++ b/tests/functional/test_arm_bpim2u.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (LinuxKernelTest, exec_command_and_wait_for_pattern, Asset, interrupt_interactive_console_until_pattern, skipBigDataTest) -from qemu_test.utils import (archive_extract, gzip_uncompress, lzma_uncompress, +from qemu_test.utils import (gzip_uncompress, lzma_uncompress, image_pow2ceil_expand) @@ -XXX,XX +XXX,XX @@ class BananaPiMachine(LinuxKernelTest): def test_arm_bpim2u(self): self.set_machine('bpim2u') - deb_path = self.ASSET_DEB.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinuz-6.6.16-current-sunxi') + kernel_path = self.archive_extract( + self.ASSET_DEB, member='/boot/vmlinuz-6.6.16-current-sunxi') dtb_path = ('/usr/lib/linux-image-6.6.16-current-sunxi/' 'sun8i-r40-bananapi-m2-ultra.dtb') - dtb_path = self.extract_from_deb(deb_path, dtb_path) + dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u(self): def test_arm_bpim2u_initrd(self): self.set_machine('bpim2u') - deb_path = self.ASSET_DEB.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinuz-6.6.16-current-sunxi') + kernel_path = self.archive_extract( + self.ASSET_DEB, member='/boot/vmlinuz-6.6.16-current-sunxi') dtb_path = ('/usr/lib/linux-image-6.6.16-current-sunxi/' 'sun8i-r40-bananapi-m2-ultra.dtb') - dtb_path = self.extract_from_deb(deb_path, dtb_path) + dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) initrd_path_gz = self.ASSET_INITRD.fetch() initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_gmac(self): self.require_netdev('user') deb_path = self.ASSET_DEB.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinuz-6.6.16-current-sunxi') + kernel_path = self.archive_extract( + self.ASSET_DEB, member='/boot/vmlinuz-6.6.16-current-sunxi') dtb_path = ('/usr/lib/linux-image-6.6.16-current-sunxi/' 'sun8i-r40-bananapi-m2-ultra.dtb') - dtb_path = self.extract_from_deb(deb_path, dtb_path) + dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) rootfs_path_xz = self.ASSET_ROOTFS.fetch() rootfs_path = self.scratch_file('rootfs.cpio') lzma_uncompress(rootfs_path_xz, rootfs_path) diff --git a/tests/functional/test_arm_canona1100.py b/tests/functional/test_arm_canona1100.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_canona1100.py +++ b/tests/functional/test_arm_canona1100.py @@ -XXX,XX +XXX,XX @@ # later. See the COPYING file in the top-level directory. from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern -from qemu_test.utils import archive_extract class CanonA1100Machine(QemuSystemTest): """Boots the barebox firmware and checks that the console is operational""" @@ -XXX,XX +XXX,XX @@ class CanonA1100Machine(QemuSystemTest): def test_arm_canona1100(self): self.set_machine('canon-a1100') - file_path = self.ASSET_BIOS.fetch() - archive_extract(file_path, dest_dir=self.workdir, - member="day18/barebox.canon-a1100.bin") + bios = self.archive_extract(self.ASSET_BIOS, + member="day18/barebox.canon-a1100.bin") self.vm.set_console() - self.vm.add_args('-bios', - self.scratch_file('day18', - 'barebox.canon-a1100.bin')) + self.vm.add_args('-bios', bios) self.vm.launch() wait_for_console_pattern(self, 'running /env/bin/init') diff --git a/tests/functional/test_arm_collie.py b/tests/functional/test_arm_collie.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_collie.py +++ b/tests/functional/test_arm_collie.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract class CollieTest(LinuxKernelTest): diff --git a/tests/functional/test_arm_orangepi.py b/tests/functional/test_arm_orangepi.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_orangepi.py +++ b/tests/functional/test_arm_orangepi.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (LinuxKernelTest, exec_command_and_wait_for_pattern, Asset, interrupt_interactive_console_until_pattern, wait_for_console_pattern, skipBigDataTest) -from qemu_test.utils import (archive_extract, gzip_uncompress, lzma_uncompress, +from qemu_test.utils import (gzip_uncompress, lzma_uncompress, image_pow2ceil_expand) @@ -XXX,XX +XXX,XX @@ class BananaPiMachine(LinuxKernelTest): def test_arm_orangepi(self): self.set_machine('orangepi-pc') - deb_path = self.ASSET_DEB.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinuz-6.6.16-current-sunxi') + kernel_path = self.archive_extract( + self.ASSET_DEB, member='/boot/vmlinuz-6.6.16-current-sunxi') dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun8i-h3-orangepi-pc.dtb' - dtb_path = self.extract_from_deb(deb_path, dtb_path) + dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi(self): def test_arm_orangepi_initrd(self): self.set_machine('orangepi-pc') - deb_path = self.ASSET_DEB.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinuz-6.6.16-current-sunxi') + kernel_path = self.archive_extract( + self.ASSET_DEB, member='/boot/vmlinuz-6.6.16-current-sunxi') dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun8i-h3-orangepi-pc.dtb' - dtb_path = self.extract_from_deb(deb_path, dtb_path) + dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) initrd_path_gz = self.ASSET_INITRD.fetch() initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_initrd(self): def test_arm_orangepi_sd(self): self.set_machine('orangepi-pc') self.require_netdev('user') - deb_path = self.ASSET_DEB.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinuz-6.6.16-current-sunxi') + kernel_path = self.archive_extract( + self.ASSET_DEB, member='/boot/vmlinuz-6.6.16-current-sunxi') dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun8i-h3-orangepi-pc.dtb' - dtb_path = self.extract_from_deb(deb_path, dtb_path) + dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) rootfs_path_xz = self.ASSET_ROOTFS.fetch() rootfs_path = self.scratch_file('rootfs.cpio') lzma_uncompress(rootfs_path_xz, rootfs_path) @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_armbian(self): def test_arm_orangepi_uboot_netbsd9(self): self.set_machine('orangepi-pc') # This test download a 304MB compressed image and expand it to 2GB - deb_path = self.ASSET_UBOOT.fetch() # We use the common OrangePi PC 'plus' build of U-Boot for our secondary # program loader (SPL). We will then set the path to the more specific # OrangePi "PC" device tree blob with 'setenv fdtfile' in U-Boot prompt, # before to boot NetBSD. uboot_path = '/usr/lib/u-boot/orangepi_plus/u-boot-sunxi-with-spl.bin' - uboot_path = self.extract_from_deb(deb_path, uboot_path) + uboot_path = self.archive_extract(self.ASSET_UBOOT, member=uboot_path) image_path_gz = self.ASSET_NETBSD.fetch() image_path = self.scratch_file('armv7.img') gzip_uncompress(image_path_gz, image_path) diff --git a/tests/functional/test_arm_raspi2.py b/tests/functional/test_arm_raspi2.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_raspi2.py +++ b/tests/functional/test_arm_raspi2.py @@ -XXX,XX +XXX,XX @@ def do_test_arm_raspi2(self, uart_id): serial_kernel_cmdline = { 0: 'earlycon=pl011,0x3f201000 console=ttyAMA0', } - deb_path = self.ASSET_KERNEL_20190215.fetch() - kernel_path = self.extract_from_deb(deb_path, '/boot/kernel7.img') - dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2709-rpi-2-b.dtb') + kernel_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='/boot/kernel7.img') + dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='/boot/bcm2709-rpi-2-b.dtb') self.set_machine('raspi2b') self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_raspi2_uart0(self): self.do_test_arm_raspi2(0) def test_arm_raspi2_initrd(self): - deb_path = self.ASSET_KERNEL_20190215.fetch() - kernel_path = self.extract_from_deb(deb_path, '/boot/kernel7.img') - dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2709-rpi-2-b.dtb') + kernel_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='/boot/kernel7.img') + dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='/boot/bcm2709-rpi-2-b.dtb') initrd_path_gz = self.ASSET_INITRD.fetch() initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) diff --git a/tests/functional/test_arm_sx1.py b/tests/functional/test_arm_sx1.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_sx1.py +++ b/tests/functional/test_arm_sx1.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract class SX1Test(LinuxKernelTest): diff --git a/tests/functional/test_arm_vexpress.py b/tests/functional/test_arm_vexpress.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_vexpress.py +++ b/tests/functional/test_arm_vexpress.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract class VExpressTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class VExpressTest(LinuxKernelTest): def test_arm_vexpressa9(self): self.set_machine('vexpress-a9') - file_path = self.ASSET_DAY16.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY16) self.launch_kernel(self.scratch_file('day16', 'winter.zImage'), dtb=self.scratch_file('day16', 'vexpress-v2p-ca9.dtb'), diff --git a/tests/functional/test_m68k_mcf5208evb.py b/tests/functional/test_m68k_mcf5208evb.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_m68k_mcf5208evb.py +++ b/tests/functional/test_m68k_mcf5208evb.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract class Mcf5208EvbTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class Mcf5208EvbTest(LinuxKernelTest): def test_m68k_mcf5208evb(self): self.set_machine('mcf5208evb') - file_path = self.ASSET_DAY07.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY07) self.vm.set_console() self.vm.add_args('-kernel', self.scratch_file('day07', 'sanity-clause.elf')) diff --git a/tests/functional/test_m68k_q800.py b/tests/functional/test_m68k_q800.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_m68k_q800.py +++ b/tests/functional/test_m68k_q800.py @@ -XXX,XX +XXX,XX @@ class Q800MachineTest(LinuxKernelTest): def test_m68k_q800(self): self.set_machine('q800') - deb_path = self.ASSET_KERNEL.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinux-5.3.0-1-m68k') + kernel_path = self.archive_extract(self.ASSET_KERNEL, + member='/boot/vmlinux-5.3.0-1-m68k') self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + diff --git a/tests/functional/test_microblaze_s3adsp1800.py b/tests/functional/test_microblaze_s3adsp1800.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_microblaze_s3adsp1800.py +++ b/tests/functional/test_microblaze_s3adsp1800.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (exec_command, exec_command_and_wait_for_pattern, QemuSystemTest, Asset, wait_for_console_pattern) -from qemu_test.utils import archive_extract class MicroblazeMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ class MicroblazeMachine(QemuSystemTest): def test_microblaze_s3adsp1800(self): self.set_machine('petalogix-s3adsp1800') - file_path = self.ASSET_IMAGE.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_IMAGE) self.vm.set_console() self.vm.add_args('-kernel', self.scratch_file('day17', 'ballerina.bin')) diff --git a/tests/functional/test_microblazeel_s3adsp1800.py b/tests/functional/test_microblazeel_s3adsp1800.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_microblazeel_s3adsp1800.py +++ b/tests/functional/test_microblazeel_s3adsp1800.py @@ -XXX,XX +XXX,XX @@ import time from qemu_test import (exec_command, exec_command_and_wait_for_pattern, QemuSystemTest, Asset, wait_for_console_pattern) -from qemu_test.utils import archive_extract class MicroblazeelMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ class MicroblazeelMachine(QemuSystemTest): def test_microblazeel_s3adsp1800(self): self.require_netdev('user') self.set_machine('petalogix-s3adsp1800') - file_path = self.ASSET_IMAGE.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_IMAGE) self.vm.set_console() self.vm.add_args('-kernel', self.scratch_file('day13', 'xmaton.bin')) tftproot = self.scratch_file('day13') diff --git a/tests/functional/test_mips64el_fuloong2e.py b/tests/functional/test_mips64el_fuloong2e.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_fuloong2e.py +++ b/tests/functional/test_mips64el_fuloong2e.py @@ -XXX,XX +XXX,XX @@ class MipsFuloong2e(LinuxKernelTest): '2a70f15b397f4ced632b0c15cb22660394190644146d804d60a4796eefbe1f50') def test_linux_kernel_3_16(self): - deb_path = self.ASSET_KERNEL.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinux-3.16.0-6-loongson-2e') + kernel_path = self.archive_extract( + self.ASSET_KERNEL, + member='/boot/vmlinux-3.16.0-6-loongson-2e') self.set_machine('fuloong2e') self.vm.set_console() diff --git a/tests/functional/test_mips64el_malta.py b/tests/functional/test_mips64el_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_malta.py +++ b/tests/functional/test_mips64el_malta.py @@ -XXX,XX +XXX,XX @@ def test_mips64el_malta(self): [2] https://kernel-team.pages.debian.net/kernel-handbook/ ch-common-tasks.html#s-common-official """ - deb_path = self.ASSET_KERNEL_2_63_2.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinux-2.6.32-5-5kc-malta') + kernel_path = self.archive_extract( + self.ASSET_KERNEL_2_63_2, + member='/boot/vmlinux-2.6.32-5-5kc-malta') self.set_machine('malta') self.vm.set_console() diff --git a/tests/functional/test_mips_malta.py b/tests/functional/test_mips_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips_malta.py +++ b/tests/functional/test_mips_malta.py @@ -XXX,XX +XXX,XX @@ class MaltaMachineConsole(LinuxKernelTest): '16ca524148afb0626f483163e5edf352bc1ab0e4fc7b9f9d473252762f2c7a43') def test_mips_malta(self): - deb_path = self.ASSET_KERNEL_2_63_2.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinux-2.6.32-5-4kc-malta') + kernel_path = self.archive_extract( + self.ASSET_KERNEL_2_63_2, + member='/boot/vmlinux-2.6.32-5-4kc-malta') self.set_machine('malta') self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_mips_malta(self): 'dcfe3a7fe3200da3a00d176b95caaa086495eb158f2bff64afc67d7e1eb2cddc') def test_mips_malta_cpio(self): - deb_path = self.ASSET_KERNEL_4_5_0.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinux-4.5.0-2-4kc-malta') + kernel_path = self.archive_extract( + self.ASSET_KERNEL_4_5_0, + member='/boot/vmlinux-4.5.0-2-4kc-malta') initrd_path_gz = self.ASSET_INITRD.fetch() initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) diff --git a/tests/functional/test_mipsel_malta.py b/tests/functional/test_mipsel_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mipsel_malta.py +++ b/tests/functional/test_mipsel_malta.py @@ -XXX,XX +XXX,XX @@ interrupt_interactive_console_until_pattern, wait_for_console_pattern) from qemu_test.utils import lzma_uncompress -from zipfile import ZipFile class MaltaMachineConsole(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class MaltaMachineYAMON(QemuSystemTest): def test_mipsel_malta_yamon(self): yamon_bin = 'yamon-02.22.bin' - zip_path = self.ASSET_YAMON_ROM.fetch() - with ZipFile(zip_path, 'r') as zf: - zf.extract(yamon_bin, path=self.workdir) + self.archive_extract(self.ASSET_YAMON_ROM) yamon_path = self.scratch_file(yamon_bin) self.set_machine('malta') diff --git a/tests/functional/test_or1k_sim.py b/tests/functional/test_or1k_sim.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_or1k_sim.py +++ b/tests/functional/test_or1k_sim.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract class OpenRISC1kSimTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class OpenRISC1kSimTest(LinuxKernelTest): def test_or1k_sim(self): self.set_machine('or1k-sim') - file_path = self.ASSET_DAY20.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY20) self.vm.set_console() self.vm.add_args('-kernel', self.scratch_file('day20', 'vmlinux')) self.vm.launch() diff --git a/tests/functional/test_ppc64_e500.py b/tests/functional/test_ppc64_e500.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc64_e500.py +++ b/tests/functional/test_ppc64_e500.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract class E500Test(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class E500Test(LinuxKernelTest): def test_ppc64_e500(self): self.set_machine('ppce500') self.cpu = 'e5500' - file_path = self.ASSET_DAY19.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY19) self.launch_kernel(self.scratch_file('day19', 'uImage'), wait_for='QEMU advent calendar') diff --git a/tests/functional/test_ppc_amiga.py b/tests/functional/test_ppc_amiga.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_amiga.py +++ b/tests/functional/test_ppc_amiga.py @@ -XXX,XX +XXX,XX @@ import subprocess from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern, run_cmd -from zipfile import ZipFile class AmigaOneMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def test_ppc_amigaone(self): self.require_accelerator("tcg") self.set_machine('amigaone') tar_name = 'A1Firmware_Floppy_05-Mar-2005.zip' - zip_file = self.ASSET_IMAGE.fetch() - with ZipFile(zip_file, 'r') as zf: - zf.extractall(path=self.workdir) + self.archive_extract(self.ASSET_IMAGE, format="zip") bios = self.scratch_file("u-boot-amigaone.bin") with open(bios, "wb") as bios_fh: subprocess.run(['tail', '-c', '524288', diff --git a/tests/functional/test_ppc_bamboo.py b/tests/functional/test_ppc_bamboo.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_bamboo.py +++ b/tests/functional/test_ppc_bamboo.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from qemu_test.utils import archive_extract from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, exec_command_and_wait_for_pattern) @@ -XXX,XX +XXX,XX @@ def test_ppc_bamboo(self): self.set_machine('bamboo') self.require_accelerator("tcg") self.require_netdev('user') - file_path = self.ASSET_IMAGE.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_IMAGE) self.vm.set_console() self.vm.add_args('-kernel', self.scratch_file('system-image-powerpc-440fp', diff --git a/tests/functional/test_ppc_mac.py b/tests/functional/test_ppc_mac.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_mac.py +++ b/tests/functional/test_ppc_mac.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract class MacTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def do_day15_test(self): # we're running kvm_hv or kvm_pr. For now let's disable this test # if we don't have TCG support. self.require_accelerator("tcg") - - file_path = self.ASSET_DAY15.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY15) self.vm.add_args('-M', 'graphics=off') self.launch_kernel(self.scratch_file('day15', 'invaders.elf'), wait_for='QEMU advent calendar') diff --git a/tests/functional/test_ppc_mpc8544ds.py b/tests/functional/test_ppc_mpc8544ds.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_mpc8544ds.py +++ b/tests/functional/test_ppc_mpc8544ds.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from qemu_test.utils import archive_extract from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern class Mpc8544dsMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ class Mpc8544dsMachine(QemuSystemTest): def test_ppc_mpc8544ds(self): self.require_accelerator("tcg") self.set_machine('mpc8544ds') - file_path = self.ASSET_IMAGE.fetch() - archive_extract(file_path, self.workdir, member='creek/creek.bin') + kernel_file = self.archive_extract(self.ASSET_IMAGE, + member='creek/creek.bin') self.vm.set_console() - self.vm.add_args('-kernel', self.scratch_file('creek', 'creek.bin')) + self.vm.add_args('-kernel', kernel_file) self.vm.launch() wait_for_console_pattern(self, 'QEMU advent calendar 2020', self.panic_message) diff --git a/tests/functional/test_ppc_virtex_ml507.py b/tests/functional/test_ppc_virtex_ml507.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_virtex_ml507.py +++ b/tests/functional/test_ppc_virtex_ml507.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from qemu_test.utils import archive_extract from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern class VirtexMl507Machine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ class VirtexMl507Machine(QemuSystemTest): def test_ppc_virtex_ml507(self): self.require_accelerator("tcg") self.set_machine('virtex-ml507') - file_path = self.ASSET_IMAGE.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_IMAGE) self.vm.set_console() self.vm.add_args('-kernel', self.scratch_file('hippo', 'hippo.linux'), '-dtb', self.scratch_file('hippo', diff --git a/tests/functional/test_sh4_r2d.py b/tests/functional/test_sh4_r2d.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sh4_r2d.py +++ b/tests/functional/test_sh4_r2d.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset, skipFlakyTest -from qemu_test.utils import archive_extract + class R2dTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class R2dTest(LinuxKernelTest): @skipFlakyTest(bug_url=None) def test_r2d(self): self.set_machine('r2d') - file_path = self.ASSET_DAY09.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY09) self.vm.add_args('-append', 'console=ttySC1') self.launch_kernel(self.scratch_file('day09', 'zImage'), console_index=1, diff --git a/tests/functional/test_sh4eb_r2d.py b/tests/functional/test_sh4eb_r2d.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sh4eb_r2d.py +++ b/tests/functional/test_sh4eb_r2d.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (LinuxKernelTest, Asset, exec_command_and_wait_for_pattern) -from qemu_test.utils import archive_extract + class R2dEBTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class R2dEBTest(LinuxKernelTest): def test_sh4eb_r2d(self): self.set_machine('r2d') - file_path = self.ASSET_TGZ.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_TGZ) self.vm.add_args('-append', 'console=ttySC1 noiotrap') self.launch_kernel(self.scratch_file('sh4eb', 'linux-kernel'), initrd=self.scratch_file('sh4eb', 'initramfs.cpio.gz'), diff --git a/tests/functional/test_sparc64_sun4u.py b/tests/functional/test_sparc64_sun4u.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sparc64_sun4u.py +++ b/tests/functional/test_sparc64_sun4u.py @@ -XXX,XX +XXX,XX @@ # later. See the COPYING file in the top-level directory. from qemu_test import QemuSystemTest, Asset, wait_for_console_pattern -from qemu_test.utils import archive_extract class Sun4uMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ class Sun4uMachine(QemuSystemTest): def test_sparc64_sun4u(self): self.set_machine('sun4u') - file_path = self.ASSET_IMAGE.fetch() - kernel_name = 'day23/vmlinux' - archive_extract(file_path, self.workdir, kernel_name) + kernel_file = self.archive_extract(self.ASSET_IMAGE, + member='day23/vmlinux') self.vm.set_console() - self.vm.add_args('-kernel', self.scratch_file(kernel_name), + self.vm.add_args('-kernel', kernel_file, '-append', 'printk.time=0') self.vm.launch() wait_for_console_pattern(self, 'Starting logging: OK') diff --git a/tests/functional/test_sparc_sun4m.py b/tests/functional/test_sparc_sun4m.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sparc_sun4m.py +++ b/tests/functional/test_sparc_sun4m.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract class Sun4mTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class Sun4mTest(LinuxKernelTest): def test_sparc_ss20(self): self.set_machine('SS-20') - file_path = self.ASSET_DAY11.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY11) self.launch_kernel(self.scratch_file('day11', 'zImage.elf'), wait_for='QEMU advent calendar') diff --git a/tests/functional/test_xtensa_lx60.py b/tests/functional/test_xtensa_lx60.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_xtensa_lx60.py +++ b/tests/functional/test_xtensa_lx60.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract class XTensaLX60Test(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class XTensaLX60Test(LinuxKernelTest): def test_xtensa_lx60(self): self.set_machine('lx60') self.cpu = 'dc233c' - file_path = self.ASSET_DAY02.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY02) self.launch_kernel(self.scratch_file('day02', 'santas-sleigh-ride.elf'), wait_for='QEMU advent calendar') -- 2.46.0 There are many types of compression that the tests deal with, and it makes sense to have a single helper 'uncompress' that can deal with all. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/utils.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/functional/qemu_test/utils.py b/tests/functional/qemu_test/utils.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/utils.py +++ b/tests/functional/qemu_test/utils.py @@ -XXX,XX +XXX,XX @@ def lzma_uncompress(xz_path, output_path): os.remove(output_path) raise +def uncompress(input_path, output_path, format=None): + if format == "xz": + lzma_uncompress(input_path, output_path) + elif format == "gz": + gzip_uncompress(input_path, output_path) + else: + raise Exception(f"Unknown compression format {format}") + +def guess_uncompress_format(path): + (name, ext) = os.path.splitext(path) + if ext == ".xz": + return "xz" + elif ext == ".gz": + return "gz" + else: + raise Exception(f"Unknown compression format for {path}") + def cpio_extract(cpio_handle, output_path): cwd = os.getcwd() os.chdir(output_path) -- 2.46.0 This helper wrappers utils.uncompress, forcing the use of the scratch directory, to ensure any uncompressed files are cleaned at test termination. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/testcase.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -XXX,XX +XXX,XX @@ from .cmd import run_cmd from .config import BUILD_DIR from .utils import (archive_extract as utils_archive_extract, - guess_archive_format) + uncompress as utils_uncompress, + guess_archive_format, guess_uncompress_format) class QemuBaseTest(unittest.TestCase): @@ -XXX,XX +XXX,XX @@ class QemuBaseTest(unittest.TestCase): log = None logdir = None + def uncompress(self, input_path, format=None): + if type(input_path) == Asset: + if format is None: + format = guess_uncompress_format(input_path.url) + input_path = input_path.fetch() + elif format is None: + format = guess_uncompress_format(input_path) + + (name, ext) = os.path.splitext(input_path) + output_path = self.scratch_file(os.path.basename(name)) + + utils_uncompress(input_path, output_path) + + return output_path + ''' @params archive: filename, Asset, or file-like object to extract @params sub_dir: optional sub-directory to extract into -- 2.46.0 Replace use of lzma_uncompress and gzip_uncompress with the new uncompress helper. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/testcase.py | 2 +- tests/functional/test_aarch64_raspi4.py | 5 +---- tests/functional/test_aarch64_sbsaref.py | 9 ++------- tests/functional/test_alpha_clipper.py | 4 +--- tests/functional/test_arm_bpim2u.py | 15 ++++----------- tests/functional/test_arm_orangepi.py | 19 +++++-------------- tests/functional/test_arm_raspi2.py | 5 +---- tests/functional/test_mips64el_malta.py | 9 ++------- tests/functional/test_mips_malta.py | 5 +---- tests/functional/test_mipsel_malta.py | 15 +++++---------- tests/functional/test_rx_gdbsim.py | 5 +---- tests/functional/test_s390x_ccw_virtio.py | 6 ++---- tests/functional/test_s390x_topology.py | 5 +---- 13 files changed, 27 insertions(+), 77 deletions(-) diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -XXX,XX +XXX,XX @@ def uncompress(self, input_path, format=None): (name, ext) = os.path.splitext(input_path) output_path = self.scratch_file(os.path.basename(name)) - utils_uncompress(input_path, output_path) + utils_uncompress(input_path, output_path, format) return output_path diff --git a/tests/functional/test_aarch64_raspi4.py b/tests/functional/test_aarch64_raspi4.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_raspi4.py +++ b/tests/functional/test_aarch64_raspi4.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (LinuxKernelTest, Asset, exec_command_and_wait_for_pattern) -from qemu_test.utils import gzip_uncompress class Aarch64Raspi4Machine(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_arm_raspi4_initrd(self): member='/boot/kernel8.img') dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215, member='/boot/bcm2711-rpi-4-b.dtb') - initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = self.scratch_file('rootfs.cpio') - gzip_uncompress(initrd_path_gz, initrd_path) + initrd_path = self.uncompress(self.ASSET_INITRD) self.set_machine('raspi4b') self.vm.set_console() diff --git a/tests/functional/test_aarch64_sbsaref.py b/tests/functional/test_aarch64_sbsaref.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_sbsaref.py +++ b/tests/functional/test_aarch64_sbsaref.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (QemuSystemTest, Asset, wait_for_console_pattern, interrupt_interactive_console_until_pattern) -from qemu_test.utils import lzma_uncompress def fetch_firmware(test): @@ -XXX,XX +XXX,XX @@ def fetch_firmware(test): """ # Secure BootRom (TF-A code) - fs0_xz_path = Aarch64SbsarefMachine.ASSET_FLASH0.fetch() - fs0_path = test.scratch_file("SBSA_FLASH0.fd") - lzma_uncompress(fs0_xz_path, fs0_path) + fs0_path = test.uncompress(Aarch64SbsarefMachine.ASSET_FLASH0) # Non-secure rom (UEFI and EFI variables) - fs1_xz_path = Aarch64SbsarefMachine.ASSET_FLASH1.fetch() - fs1_path = test.scratch_file("SBSA_FLASH1.fd") - lzma_uncompress(fs1_xz_path, fs1_path) + fs1_path = test.uncompress(Aarch64SbsarefMachine.ASSET_FLASH1) for path in [fs0_path, fs1_path]: with open(path, "ab+") as fd: diff --git a/tests/functional/test_alpha_clipper.py b/tests/functional/test_alpha_clipper.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_alpha_clipper.py +++ b/tests/functional/test_alpha_clipper.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import gzip_uncompress class AlphaClipperTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_alpha_clipper(self): self.set_machine('clipper') kernel_path = self.ASSET_KERNEL.fetch() - uncompressed_kernel = self.scratch_file('vmlinux') - gzip_uncompress(kernel_path, uncompressed_kernel) + uncompressed_kernel = self.uncompress(self.ASSET_KERNEL, format="gz") self.vm.set_console() kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=ttyS0' diff --git a/tests/functional/test_arm_bpim2u.py b/tests/functional/test_arm_bpim2u.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bpim2u.py +++ b/tests/functional/test_arm_bpim2u.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (LinuxKernelTest, exec_command_and_wait_for_pattern, Asset, interrupt_interactive_console_until_pattern, skipBigDataTest) -from qemu_test.utils import (gzip_uncompress, lzma_uncompress, - image_pow2ceil_expand) +from qemu_test.utils import (image_pow2ceil_expand) class BananaPiMachine(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_initrd(self): dtb_path = ('/usr/lib/linux-image-6.6.16-current-sunxi/' 'sun8i-r40-bananapi-m2-ultra.dtb') dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) - initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = self.scratch_file('rootfs.cpio') - gzip_uncompress(initrd_path_gz, initrd_path) + initrd_path = self.uncompress(self.ASSET_INITRD) self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_gmac(self): dtb_path = ('/usr/lib/linux-image-6.6.16-current-sunxi/' 'sun8i-r40-bananapi-m2-ultra.dtb') dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) - rootfs_path_xz = self.ASSET_ROOTFS.fetch() - rootfs_path = self.scratch_file('rootfs.cpio') - lzma_uncompress(rootfs_path_xz, rootfs_path) + rootfs_path = self.uncompress(self.ASSET_ROOTFS) image_pow2ceil_expand(rootfs_path) self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_openwrt_22_03_3(self): self.set_machine('bpim2u') # This test download a 8.9 MiB compressed image and expand it # to 127 MiB. - image_path_gz = self.ASSET_SD_IMAGE.fetch() - image_path = self.scratch_file('sdcard.img') - gzip_uncompress(image_path_gz, image_path) + image_path = self.uncompress(self.ASSET_SD_IMAGE) image_pow2ceil_expand(image_path) self.vm.set_console() diff --git a/tests/functional/test_arm_orangepi.py b/tests/functional/test_arm_orangepi.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_orangepi.py +++ b/tests/functional/test_arm_orangepi.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (LinuxKernelTest, exec_command_and_wait_for_pattern, Asset, interrupt_interactive_console_until_pattern, wait_for_console_pattern, skipBigDataTest) -from qemu_test.utils import (gzip_uncompress, lzma_uncompress, - image_pow2ceil_expand) +from qemu_test.utils import image_pow2ceil_expand class BananaPiMachine(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_initrd(self): self.ASSET_DEB, member='/boot/vmlinuz-6.6.16-current-sunxi') dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun8i-h3-orangepi-pc.dtb' dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) - initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = self.scratch_file('rootfs.cpio') - gzip_uncompress(initrd_path_gz, initrd_path) + initrd_path = self.uncompress(self.ASSET_INITRD) self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_sd(self): self.ASSET_DEB, member='/boot/vmlinuz-6.6.16-current-sunxi') dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun8i-h3-orangepi-pc.dtb' dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) - rootfs_path_xz = self.ASSET_ROOTFS.fetch() - rootfs_path = self.scratch_file('rootfs.cpio') - lzma_uncompress(rootfs_path_xz, rootfs_path) + rootfs_path = self.uncompress(self.ASSET_ROOTFS) image_pow2ceil_expand(rootfs_path) self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_armbian(self): # This test download a 275 MiB compressed image and expand it # to 1036 MiB, but the underlying filesystem is 1552 MiB... # As we expand it to 2 GiB we are safe. - image_path_xz = self.ASSET_ARMBIAN.fetch() - image_path = self.scratch_file('armbian.img') - lzma_uncompress(image_path_xz, image_path) + iamge_path = self.uncompress(self.ASSET_ARMBIAN) image_pow2ceil_expand(image_path) self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_uboot_netbsd9(self): # before to boot NetBSD. uboot_path = '/usr/lib/u-boot/orangepi_plus/u-boot-sunxi-with-spl.bin' uboot_path = self.archive_extract(self.ASSET_UBOOT, member=uboot_path) - image_path_gz = self.ASSET_NETBSD.fetch() - image_path = self.scratch_file('armv7.img') - gzip_uncompress(image_path_gz, image_path) + image_path = self.uncompress(self.ASSET_NETBSD) image_pow2ceil_expand(image_path) image_drive_args = 'if=sd,format=raw,snapshot=on,file=' + image_path diff --git a/tests/functional/test_arm_raspi2.py b/tests/functional/test_arm_raspi2.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_raspi2.py +++ b/tests/functional/test_arm_raspi2.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (LinuxKernelTest, Asset, exec_command_and_wait_for_pattern) -from qemu_test.utils import gzip_uncompress class ArmRaspi2Machine(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_arm_raspi2_initrd(self): member='/boot/kernel7.img') dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215, member='/boot/bcm2709-rpi-2-b.dtb') - initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = self.scratch_file('rootfs.cpio') - gzip_uncompress(initrd_path_gz, initrd_path) + initrd_path = self.uncompress(self.ASSET_INITRD) self.set_machine('raspi2b') self.vm.set_console() diff --git a/tests/functional/test_mips64el_malta.py b/tests/functional/test_mips64el_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_malta.py +++ b/tests/functional/test_mips64el_malta.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (LinuxKernelTest, Asset, exec_command_and_wait_for_pattern, skipIfMissingImports, skipFlakyTest, skipUntrustedTest) -from qemu_test.utils import gzip_uncompress class MaltaMachineConsole(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_mips64el_malta(self): @skipUntrustedTest() def test_mips64el_malta_5KEc_cpio(self): kernel_path = self.ASSET_KERNEL_3_19_3.fetch() - initrd_path_gz = self.ASSET_CPIO_R1.fetch() - initrd_path = self.scratch_file('rootfs.cpio') - gzip_uncompress(initrd_path_gz, initrd_path) + initrd_path = self.uncompress(self.ASSET_CPIO_R1) self.set_machine('malta') self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def do_test_i6400_framebuffer_logo(self, cpu_cores_count): screendump_path = self.scratch_file('screendump.pbm') - kernel_path_gz = self.ASSET_KERNEL_4_7_0.fetch() - kernel_path = self.scratch_file("vmlinux") - gzip_uncompress(kernel_path_gz, kernel_path) + kernel_path = self.uncompress(self.ASSET_KERNEL_4_7_0) tuxlogo_path = self.ASSET_TUXLOGO.fetch() diff --git a/tests/functional/test_mips_malta.py b/tests/functional/test_mips_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips_malta.py +++ b/tests/functional/test_mips_malta.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern -from qemu_test.utils import gzip_uncompress class MaltaMachineConsole(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_mips_malta_cpio(self): kernel_path = self.archive_extract( self.ASSET_KERNEL_4_5_0, member='/boot/vmlinux-4.5.0-2-4kc-malta') - initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = self.scratch_file('rootfs.cpio') - gzip_uncompress(initrd_path_gz, initrd_path) + initrd_path = self.uncompress(self.ASSET_INITRD) self.set_machine('malta') self.vm.set_console() diff --git a/tests/functional/test_mipsel_malta.py b/tests/functional/test_mipsel_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mipsel_malta.py +++ b/tests/functional/test_mipsel_malta.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (QemuSystemTest, LinuxKernelTest, Asset, interrupt_interactive_console_until_pattern, wait_for_console_pattern) -from qemu_test.utils import lzma_uncompress class MaltaMachineConsole(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class MaltaMachineConsole(LinuxKernelTest): 'generic_nano32r6el_page64k_dbg.xz'), 'ce21ff4b07a981ecb8a39db2876616f5a2473eb2ab459c6f67465b9914b0c6b6') - def do_test_mips_malta32el_nanomips(self, kernel_path_xz): - kernel_path = self.scratch_file('kernel') - lzma_uncompress(kernel_path_xz, kernel_path) + def do_test_mips_malta32el_nanomips(self, kernel): + kernel_path = self.uncompress(kernel) self.set_machine('malta') self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def do_test_mips_malta32el_nanomips(self, kernel_path_xz): self.wait_for_console_pattern(console_pattern) def test_mips_malta32el_nanomips_4k(self): - kernel_path_xz = self.ASSET_KERNEL_4K.fetch() - self.do_test_mips_malta32el_nanomips(kernel_path_xz) + self.do_test_mips_malta32el_nanomips(self.ASSET_KERNEL_4K) def test_mips_malta32el_nanomips_16k_up(self): - kernel_path_xz = self.ASSET_KERNEL_16K.fetch() - self.do_test_mips_malta32el_nanomips(kernel_path_xz) + self.do_test_mips_malta32el_nanomips(self.ASSET_KERNEL_16K) def test_mips_malta32el_nanomips_64k_dbg(self): - kernel_path_xz = self.ASSET_KERNEL_16K.fetch() - self.do_test_mips_malta32el_nanomips(kernel_path_xz) + self.do_test_mips_malta32el_nanomips(self.ASSET_KERNEL_64K) class MaltaMachineYAMON(QemuSystemTest): diff --git a/tests/functional/test_rx_gdbsim.py b/tests/functional/test_rx_gdbsim.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_rx_gdbsim.py +++ b/tests/functional/test_rx_gdbsim.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (QemuSystemTest, Asset, exec_command_and_wait_for_pattern, wait_for_console_pattern, skipFlakyTest) -from qemu_test.utils import gzip_uncompress class RxGdbSimMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def test_uboot(self): """ self.set_machine('gdbsim-r5f562n8') - uboot_path_gz = self.ASSET_UBOOT.fetch() - uboot_path = self.scratch_file('u-boot.bin') - gzip_uncompress(uboot_path_gz, uboot_path) + uboot_path = self.uncompress(self.ASSET_UBOOT) self.vm.set_console() self.vm.add_args('-bios', uboot_path, diff --git a/tests/functional/test_s390x_ccw_virtio.py b/tests/functional/test_s390x_ccw_virtio.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_s390x_ccw_virtio.py +++ b/tests/functional/test_s390x_ccw_virtio.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (QemuSystemTest, Asset, exec_command_and_wait_for_pattern, wait_for_console_pattern) -from qemu_test.utils import lzma_uncompress + class S390CCWVirtioMachine(QemuSystemTest): KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 ' @@ -XXX,XX +XXX,XX @@ def test_s390x_fedora(self): kernel_path = self.ASSET_F31_KERNEL.fetch() - initrd_path_xz = self.ASSET_F31_INITRD.fetch() - initrd_path = self.scratch_file('initrd-raw.img') - lzma_uncompress(initrd_path_xz, initrd_path) + initrd_path = self.uncompress(self.ASSET_F31_INITRD, format="xz") self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + ' audit=0 ' diff --git a/tests/functional/test_s390x_topology.py b/tests/functional/test_s390x_topology.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_s390x_topology.py +++ b/tests/functional/test_s390x_topology.py @@ -XXX,XX +XXX,XX @@ from qemu_test import (QemuSystemTest, Asset, exec_command, exec_command_and_wait_for_pattern, wait_for_console_pattern) -from qemu_test.utils import lzma_uncompress class S390CPUTopology(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def kernel_init(self): """ self.require_accelerator("kvm") kernel_path = self.ASSET_F35_KERNEL.fetch() - initrd_path_xz = self.ASSET_F35_INITRD.fetch() - initrd_path = self.scratch_file('initrd-raw.img') - lzma_uncompress(initrd_path_xz, initrd_path) + initrd_path = self.uncompress(self.ASSET_F35_INITRD, format="xz") self.vm.set_console() kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE -- 2.46.0 This series is an attempt to bring a little more guaranteed order to asset and scratch file handling in the functional tests. The main highlights are: * Add custom @skipXXXXX decorators for common scenarios present in QEMU tests * Add helpers for creating file paths for various well known types of data, or well known locations, to avoid adhoc path manipulation * Add helpers to simplify uncompressing and extracting archives, from files downloaded as assets The series overall has a neutral diffstat, but if you look at just test files, as opposed to the shared infra, you'll see a significant reduction of lines of code in the tests, and I believe its easier to read them with less boilerplate. This is based on: https://gitlab.com/thuth/qemu.git tags/pull-request-2024-12-11 Changes in v3: * Rebase and update for recently changed/added tests * Drop mistaken submodule update * Fix syntax error with has_cmd usage * Fix undefined variable error in virtio gpu tests * Fix whitespace issues * Fix typos * Add patch to fix execute perms on some recent tests Changes in v2: * Put archive & uncompress helpers in their own files * Have archive_extract & uncompress directly handle assets and format detection * Drop has_cmd/has_cmds helpers in favour of 'which' * Drop obsolete tessract version check * Simplify 'which' impl * Replace 'run_cmd' with direct use of subprocess * Remove even more unused imports * Gracefully handle asset download failure by skipping tests * Drop redundant hardcoded ./contrib path in virtio GPU test * Various docs improvements to decorators * Drop formatting change to import statements Daniel P. Berrangé (32): tests/functional: add execute permission to aspeed tests tests/functional: remove many unused imports tests/functional: resolve str(Asset) to cache file path tests/functional: remove duplicated 'which' function impl tests/functional: simplify 'which' implementation tests/functional: drop 'tesseract_available' helper tests/functional: introduce some helpful decorators tests/functional: switch to new test skip decorators tests/functional: drop 'has_cmd' and 'has_cmds' helpers tests/functional: add helpers for building file paths tests/functional: switch over to using self.log_file(...) tests/functional: switch over to using self.build_file(...) tests/functional: switch over to using self.data_file(...) tests/functional: switch over to using self.scratch_file() tests/functional: remove redundant 'rmtree' call tests/functional: move archive handling into new archive.py file tests/functional: move uncompress handling into new uncompress.py file tests/functional: add common zip_extract helper tests/functional: add common deb_extract helper tests/functional: let cpio_extract accept filenames tests/functional: add a generalized archive_extract tests/functional: add 'archive_extract' to QemuBaseTest tests/functional: convert tests to new archive_extract helper tests/functional: add a generalized uncompress helper tests/functional: add 'uncompress' to QemuBaseTest tests/functional: convert tests to new uncompress helper tests/functional: drop back compat imports from utils.py tests/functional: replace 'run_cmd' with subprocess helpers tests/functional: remove now unused 'run_cmd' helper tests/functional: skip tests if assets are not available tests/functional: ignore errors when caching assets, except for 404 MAINTAINERS: add myself as reviewer for functional test suite MAINTAINERS | 1 + tests/functional/qemu_test/__init__.py | 9 +- tests/functional/qemu_test/archive.py | 117 ++++++++++ tests/functional/qemu_test/asset.py | 26 ++- tests/functional/qemu_test/cmd.py | 76 ++----- tests/functional/qemu_test/decorators.py | 107 +++++++++ tests/functional/qemu_test/linuxkernel.py | 29 +-- tests/functional/qemu_test/tesseract.py | 21 +- tests/functional/qemu_test/testcase.py | 205 ++++++++++++++++-- tests/functional/qemu_test/tuxruntest.py | 19 +- tests/functional/qemu_test/uncompress.py | 83 +++++++ tests/functional/qemu_test/utils.py | 45 ---- tests/functional/test_aarch64_aspeed.py | 23 +- tests/functional/test_aarch64_raspi3.py | 9 +- tests/functional/test_aarch64_raspi4.py | 21 +- tests/functional/test_aarch64_sbsaref.py | 12 +- .../functional/test_aarch64_sbsaref_alpine.py | 1 - .../test_aarch64_sbsaref_freebsd.py | 1 - tests/functional/test_aarch64_virt.py | 14 +- tests/functional/test_acpi_bits.py | 124 ++++------- tests/functional/test_alpha_clipper.py | 6 +- tests/functional/test_arm_aspeed_ast1030.py | 18 +- tests/functional/test_arm_aspeed_ast2500.py | 8 +- tests/functional/test_arm_aspeed_ast2600.py | 14 +- tests/functional/test_arm_aspeed_palmetto.py | 0 tests/functional/test_arm_aspeed_rainier.py | 11 +- tests/functional/test_arm_aspeed_romulus.py | 0 tests/functional/test_arm_bflt.py | 13 +- tests/functional/test_arm_bpim2u.py | 44 ++-- tests/functional/test_arm_canona1100.py | 10 +- tests/functional/test_arm_collie.py | 2 +- tests/functional/test_arm_cubieboard.py | 40 ++-- tests/functional/test_arm_emcraft_sf2.py | 2 +- tests/functional/test_arm_integratorcp.py | 28 +-- tests/functional/test_arm_orangepi.py | 60 +++-- tests/functional/test_arm_raspi2.py | 21 +- tests/functional/test_arm_smdkc210.py | 18 +- tests/functional/test_arm_sx1.py | 2 +- tests/functional/test_arm_vexpress.py | 10 +- tests/functional/test_linux_initrd.py | 7 +- tests/functional/test_m68k_mcf5208evb.py | 8 +- tests/functional/test_m68k_nextcube.py | 24 +- tests/functional/test_m68k_q800.py | 5 +- .../functional/test_microblaze_s3adsp1800.py | 9 +- .../test_microblazeel_s3adsp1800.py | 10 +- tests/functional/test_mips64el_fuloong2e.py | 10 +- tests/functional/test_mips64el_loongson3v.py | 8 +- tests/functional/test_mips64el_malta.py | 45 ++-- tests/functional/test_mips_malta.py | 19 +- tests/functional/test_mipsel_malta.py | 24 +- tests/functional/test_or1k_sim.py | 7 +- tests/functional/test_ppc64_e500.py | 7 +- tests/functional/test_ppc64_hv.py | 39 +--- tests/functional/test_ppc64_tuxrun.py | 7 +- tests/functional/test_ppc_40p.py | 7 +- tests/functional/test_ppc_amiga.py | 20 +- tests/functional/test_ppc_bamboo.py | 15 +- tests/functional/test_ppc_mac.py | 8 +- tests/functional/test_ppc_mpc8544ds.py | 8 +- tests/functional/test_ppc_virtex_ml507.py | 10 +- tests/functional/test_rx_gdbsim.py | 13 +- tests/functional/test_s390x_ccw_virtio.py | 6 +- tests/functional/test_s390x_topology.py | 7 +- tests/functional/test_sh4_r2d.py | 14 +- tests/functional/test_sh4eb_r2d.py | 14 +- tests/functional/test_sparc64_sun4u.py | 11 +- tests/functional/test_sparc_sun4m.py | 7 +- tests/functional/test_virtio_gpu.py | 16 +- tests/functional/test_xtensa_lx60.py | 8 +- 69 files changed, 887 insertions(+), 756 deletions(-) create mode 100644 tests/functional/qemu_test/archive.py create mode 100644 tests/functional/qemu_test/decorators.py create mode 100644 tests/functional/qemu_test/uncompress.py mode change 100644 => 100755 tests/functional/test_aarch64_aspeed.py mode change 100644 => 100755 tests/functional/test_arm_aspeed_ast1030.py mode change 100644 => 100755 tests/functional/test_arm_aspeed_ast2500.py mode change 100644 => 100755 tests/functional/test_arm_aspeed_ast2600.py mode change 100644 => 100755 tests/functional/test_arm_aspeed_palmetto.py mode change 100644 => 100755 tests/functional/test_arm_aspeed_rainier.py mode change 100644 => 100755 tests/functional/test_arm_aspeed_romulus.py -- 2.46.0 Tests are expected to be directly invoked when debugging so must have execute permission. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/test_aarch64_aspeed.py | 0 tests/functional/test_arm_aspeed_ast1030.py | 0 tests/functional/test_arm_aspeed_ast2500.py | 0 tests/functional/test_arm_aspeed_ast2600.py | 0 tests/functional/test_arm_aspeed_palmetto.py | 0 tests/functional/test_arm_aspeed_rainier.py | 0 tests/functional/test_arm_aspeed_romulus.py | 0 7 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/functional/test_aarch64_aspeed.py mode change 100644 => 100755 tests/functional/test_arm_aspeed_ast1030.py mode change 100644 => 100755 tests/functional/test_arm_aspeed_ast2500.py mode change 100644 => 100755 tests/functional/test_arm_aspeed_ast2600.py mode change 100644 => 100755 tests/functional/test_arm_aspeed_palmetto.py mode change 100644 => 100755 tests/functional/test_arm_aspeed_rainier.py mode change 100644 => 100755 tests/functional/test_arm_aspeed_romulus.py diff --git a/tests/functional/test_aarch64_aspeed.py b/tests/functional/test_aarch64_aspeed.py old mode 100644 new mode 100755 diff --git a/tests/functional/test_arm_aspeed_ast1030.py b/tests/functional/test_arm_aspeed_ast1030.py old mode 100644 new mode 100755 diff --git a/tests/functional/test_arm_aspeed_ast2500.py b/tests/functional/test_arm_aspeed_ast2500.py old mode 100644 new mode 100755 diff --git a/tests/functional/test_arm_aspeed_ast2600.py b/tests/functional/test_arm_aspeed_ast2600.py old mode 100644 new mode 100755 diff --git a/tests/functional/test_arm_aspeed_palmetto.py b/tests/functional/test_arm_aspeed_palmetto.py old mode 100644 new mode 100755 diff --git a/tests/functional/test_arm_aspeed_rainier.py b/tests/functional/test_arm_aspeed_rainier.py old mode 100644 new mode 100755 diff --git a/tests/functional/test_arm_aspeed_romulus.py b/tests/functional/test_arm_aspeed_romulus.py old mode 100644 new mode 100755 -- 2.46.0 Identified using 'pylint --disable=all --enable=W0611' Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/asset.py | 1 - tests/functional/qemu_test/tesseract.py | 1 - tests/functional/qemu_test/tuxruntest.py | 3 +-- tests/functional/test_aarch64_aspeed.py | 1 - tests/functional/test_aarch64_sbsaref_alpine.py | 1 - tests/functional/test_aarch64_sbsaref_freebsd.py | 1 - tests/functional/test_acpi_bits.py | 2 -- tests/functional/test_arm_bpim2u.py | 2 +- tests/functional/test_arm_collie.py | 2 +- tests/functional/test_arm_cubieboard.py | 1 - tests/functional/test_arm_orangepi.py | 2 +- tests/functional/test_arm_smdkc210.py | 3 +-- tests/functional/test_arm_sx1.py | 2 +- tests/functional/test_microblaze_s3adsp1800.py | 1 - tests/functional/test_ppc_amiga.py | 2 +- tests/functional/test_virtio_gpu.py | 1 - 16 files changed, 7 insertions(+), 19 deletions(-) diff --git a/tests/functional/qemu_test/asset.py b/tests/functional/qemu_test/asset.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/asset.py +++ b/tests/functional/qemu_test/asset.py @@ -XXX,XX +XXX,XX @@ import logging import os import stat -import subprocess import sys import unittest import urllib.request diff --git a/tests/functional/qemu_test/tesseract.py b/tests/functional/qemu_test/tesseract.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/tesseract.py +++ b/tests/functional/qemu_test/tesseract.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import re import logging from . import has_cmd, run_cmd diff --git a/tests/functional/qemu_test/tuxruntest.py b/tests/functional/qemu_test/tuxruntest.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/tuxruntest.py +++ b/tests/functional/qemu_test/tuxruntest.py @@ -XXX,XX +XXX,XX @@ import os import stat -import time from qemu_test import QemuSystemTest -from qemu_test import exec_command, exec_command_and_wait_for_pattern +from qemu_test import exec_command_and_wait_for_pattern from qemu_test import wait_for_console_pattern from qemu_test import has_cmd, run_cmd, get_qemu_img diff --git a/tests/functional/test_aarch64_aspeed.py b/tests/functional/test_aarch64_aspeed.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_aspeed.py +++ b/tests/functional/test_aarch64_aspeed.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import sys import os from qemu_test import QemuSystemTest, Asset diff --git a/tests/functional/test_aarch64_sbsaref_alpine.py b/tests/functional/test_aarch64_sbsaref_alpine.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_sbsaref_alpine.py +++ b/tests/functional/test_aarch64_sbsaref_alpine.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern -from qemu_test import interrupt_interactive_console_until_pattern from unittest import skipUnless from test_aarch64_sbsaref import fetch_firmware diff --git a/tests/functional/test_aarch64_sbsaref_freebsd.py b/tests/functional/test_aarch64_sbsaref_freebsd.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_sbsaref_freebsd.py +++ b/tests/functional/test_aarch64_sbsaref_freebsd.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern -from qemu_test import interrupt_interactive_console_until_pattern from unittest import skipUnless from test_aarch64_sbsaref import fetch_firmware diff --git a/tests/functional/test_acpi_bits.py b/tests/functional/test_acpi_bits.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_acpi_bits.py +++ b/tests/functional/test_acpi_bits.py @@ -XXX,XX +XXX,XX @@ https://gitlab.com/qemu-project/biosbits-bits . """ -import logging import os import platform import re import shutil import subprocess import tarfile -import tempfile import zipfile from pathlib import Path diff --git a/tests/functional/test_arm_bpim2u.py b/tests/functional/test_arm_bpim2u.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bpim2u.py +++ b/tests/functional/test_arm_bpim2u.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, exec_command_and_wait_for_pattern from qemu_test import Asset, interrupt_interactive_console_until_pattern -from qemu_test.utils import archive_extract, gzip_uncompress, lzma_uncompress +from qemu_test.utils import gzip_uncompress, lzma_uncompress from qemu_test.utils import image_pow2ceil_expand from unittest import skipUnless diff --git a/tests/functional/test_arm_collie.py b/tests/functional/test_arm_collie.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_collie.py +++ b/tests/functional/test_arm_collie.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract + class CollieTest(LinuxKernelTest): diff --git a/tests/functional/test_arm_cubieboard.py b/tests/functional/test_arm_cubieboard.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_cubieboard.py +++ b/tests/functional/test_arm_cubieboard.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later import os -import shutil from qemu_test import LinuxKernelTest, Asset, exec_command_and_wait_for_pattern from qemu_test import interrupt_interactive_console_until_pattern diff --git a/tests/functional/test_arm_orangepi.py b/tests/functional/test_arm_orangepi.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_orangepi.py +++ b/tests/functional/test_arm_orangepi.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, exec_command_and_wait_for_pattern from qemu_test import Asset, interrupt_interactive_console_until_pattern from qemu_test import wait_for_console_pattern -from qemu_test.utils import archive_extract, gzip_uncompress, lzma_uncompress +from qemu_test.utils import gzip_uncompress, lzma_uncompress from qemu_test.utils import image_pow2ceil_expand from unittest import skipUnless diff --git a/tests/functional/test_arm_smdkc210.py b/tests/functional/test_arm_smdkc210.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_smdkc210.py +++ b/tests/functional/test_arm_smdkc210.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later import os -import shutil -from qemu_test import LinuxKernelTest, Asset, exec_command_and_wait_for_pattern +from qemu_test import LinuxKernelTest, Asset from qemu_test.utils import gzip_uncompress class Smdkc210Machine(LinuxKernelTest): diff --git a/tests/functional/test_arm_sx1.py b/tests/functional/test_arm_sx1.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_sx1.py +++ b/tests/functional/test_arm_sx1.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract + class SX1Test(LinuxKernelTest): diff --git a/tests/functional/test_microblaze_s3adsp1800.py b/tests/functional/test_microblaze_s3adsp1800.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_microblaze_s3adsp1800.py +++ b/tests/functional/test_microblaze_s3adsp1800.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from qemu_test import exec_command, exec_command_and_wait_for_pattern from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern from qemu_test.utils import archive_extract diff --git a/tests/functional/test_ppc_amiga.py b/tests/functional/test_ppc_amiga.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_amiga.py +++ b/tests/functional/test_ppc_amiga.py @@ -XXX,XX +XXX,XX @@ import subprocess from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern, run_cmd +from qemu_test import wait_for_console_pattern from zipfile import ZipFile class AmigaOneMachine(QemuSystemTest): diff --git a/tests/functional/test_virtio_gpu.py b/tests/functional/test_virtio_gpu.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_virtio_gpu.py +++ b/tests/functional/test_virtio_gpu.py @@ -XXX,XX +XXX,XX @@ from qemu_test import exec_command_and_wait_for_pattern from qemu_test import is_readable_executable_file -from qemu.utils import kvm_available import os import socket -- 2.46.0 Allow an Asset object to be used in place of a filename but making its string representation resolve to the cache file path. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/asset.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/functional/qemu_test/asset.py b/tests/functional/qemu_test/asset.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/asset.py +++ b/tests/functional/qemu_test/asset.py @@ -XXX,XX +XXX,XX @@ def __repr__(self): return "Asset: url=%s hash=%s cache=%s" % ( self.url, self.hash, self.cache_file) + def __str__(self): + return str(self.cache_file) + def _check(self, cache_file): if self.hash is None: return True -- 2.46.0 Put the 'which' function into shared code. Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/__init__.py | 2 +- tests/functional/qemu_test/cmd.py | 10 ++++++++++ tests/functional/test_acpi_bits.py | 13 +------------ tests/functional/test_ppc64_hv.py | 13 +------------ 4 files changed, 13 insertions(+), 25 deletions(-) diff --git a/tests/functional/qemu_test/__init__.py b/tests/functional/qemu_test/__init__.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/__init__.py +++ b/tests/functional/qemu_test/__init__.py @@ -XXX,XX +XXX,XX @@ from .config import BUILD_DIR from .cmd import has_cmd, has_cmds, run_cmd, is_readable_executable_file, \ interrupt_interactive_console_until_pattern, wait_for_console_pattern, \ - exec_command, exec_command_and_wait_for_pattern, get_qemu_img + exec_command, exec_command_and_wait_for_pattern, get_qemu_img, which from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest from .linuxkernel import LinuxKernelTest diff --git a/tests/functional/qemu_test/cmd.py b/tests/functional/qemu_test/cmd.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/cmd.py +++ b/tests/functional/qemu_test/cmd.py @@ -XXX,XX +XXX,XX @@ from .config import BUILD_DIR +def which(tool): + """ looks up the full path for @tool, returns None if not found + or if @tool does not have executable permissions. + """ + paths=os.getenv('PATH') + for p in paths.split(os.path.pathsep): + p = os.path.join(p, tool) + if os.path.exists(p) and os.access(p, os.X_OK): + return p + return None def has_cmd(name, args=None): """ diff --git a/tests/functional/test_acpi_bits.py b/tests/functional/test_acpi_bits.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_acpi_bits.py +++ b/tests/functional/test_acpi_bits.py @@ -XXX,XX +XXX,XX @@ ) from qemu.machine import QEMUMachine from unittest import skipIf -from qemu_test import QemuSystemTest, Asset +from qemu_test import QemuSystemTest, Asset, which deps = ["xorriso", "mformat"] # dependent tools needed in the test setup/box. supported_platforms = ['x86_64'] # supported test platforms. @@ -XXX,XX +XXX,XX @@ # default timeout of 120 secs is sometimes not enough for bits test. BITS_TIMEOUT = 200 -def which(tool): - """ looks up the full path for @tool, returns None if not found - or if @tool does not have executable permissions. - """ - paths=os.getenv('PATH') - for p in paths.split(os.path.pathsep): - p = os.path.join(p, tool) - if os.path.exists(p) and os.access(p, os.X_OK): - return p - return None - def missing_deps(): """ returns True if any of the test dependent tools are absent. """ diff --git a/tests/functional/test_ppc64_hv.py b/tests/functional/test_ppc64_hv.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc64_hv.py +++ b/tests/functional/test_ppc64_hv.py @@ -XXX,XX +XXX,XX @@ from unittest import skipIf, skipUnless from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern, exec_command +from qemu_test import wait_for_console_pattern, exec_command, which import os import time import subprocess @@ -XXX,XX +XXX,XX @@ deps = ["xorriso"] # dependent tools needed in the test setup/box. -def which(tool): - """ looks up the full path for @tool, returns None if not found - or if @tool does not have executable permissions. - """ - paths=os.getenv('PATH') - for p in paths.split(os.path.pathsep): - p = os.path.join(p, tool) - if os.path.exists(p) and os.access(p, os.X_OK): - return p - return None - def missing_deps(): """ returns True if any of the test dependent tools are absent. """ -- 2.46.0 The 'access' check implies the file exists. Reviewed-by: Thomas Huth <thuth@redhat.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/qemu_test/cmd.py b/tests/functional/qemu_test/cmd.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/cmd.py +++ b/tests/functional/qemu_test/cmd.py @@ -XXX,XX +XXX,XX @@ def which(tool): paths=os.getenv('PATH') for p in paths.split(os.path.pathsep): p = os.path.join(p, tool) - if os.path.exists(p) and os.access(p, os.X_OK): + if os.access(p, os.X_OK): return p return None -- 2.46.0 Platforms we target have new enough tesseract that it suffices to merely check if the binary exists. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/tesseract.py | 12 +----------- tests/functional/test_m68k_nextcube.py | 8 +++----- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/tests/functional/qemu_test/tesseract.py b/tests/functional/qemu_test/tesseract.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/tesseract.py +++ b/tests/functional/qemu_test/tesseract.py @@ -XXX,XX +XXX,XX @@ import logging -from . import has_cmd, run_cmd - -def tesseract_available(expected_version): - (has_tesseract, _) = has_cmd('tesseract') - if not has_tesseract: - return False - (stdout, stderr, ret) = run_cmd([ 'tesseract', '--version']) - if ret: - return False - version = stdout.split()[1] - return int(version.split('.')[0]) >= expected_version +from . import run_cmd def tesseract_ocr(image_path, tesseract_args=''): console_logger = logging.getLogger('console') diff --git a/tests/functional/test_m68k_nextcube.py b/tests/functional/test_m68k_nextcube.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_m68k_nextcube.py +++ b/tests/functional/test_m68k_nextcube.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuSystemTest, Asset from unittest import skipUnless -from qemu_test.tesseract import tesseract_available, tesseract_ocr +from qemu_test import has_cmd +from qemu_test.tesseract import tesseract_ocr PIL_AVAILABLE = True try: @@ -XXX,XX +XXX,XX @@ def test_bootrom_framebuffer_size(self): self.assertEqual(width, 1120) self.assertEqual(height, 832) - # Tesseract 4 adds a new OCR engine based on LSTM neural networks. The - # new version is faster and more accurate than version 3. The drawback is - # that it is still alpha-level software. - @skipUnless(tesseract_available(4), 'tesseract OCR tool not available') + @skipUnless(*has_cmd('tesseract')) def test_bootrom_framebuffer_ocr_with_tesseract(self): self.set_machine('next-cube') screenshot_path = os.path.join(self.workdir, "dump.ppm") -- 2.46.0 Reduce repeated boilerplate with some helper decorators: @skipIfNotPlatform("x86_64", "aarch64") => Skip unless the build host platform matches @skipIfMissingCommands("mkisofs", "losetup") => Skips unless all listed commands are found in $PATH @skipIfMissingImports("numpy", "cv2") => Skips unless all listed modules can be imported @skipFlakyTest("https://gitlab.com/qemu-project/qemu/-/issues/NNN") => Skips unless env var requests flaky tests with the reason documented in the referenced gitlab bug @skipBigData => Skips unless env var permits tests creating big data files @skipUntrustedTest => Skips unless env var permits tests which are potentially dangerous to the host Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/__init__.py | 3 + tests/functional/qemu_test/decorators.py | 107 +++++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 tests/functional/qemu_test/decorators.py diff --git a/tests/functional/qemu_test/__init__.py b/tests/functional/qemu_test/__init__.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/__init__.py +++ b/tests/functional/qemu_test/__init__.py @@ -XXX,XX +XXX,XX @@ exec_command, exec_command_and_wait_for_pattern, get_qemu_img, which from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest from .linuxkernel import LinuxKernelTest +from .decorators import skipIfMissingCommands, skipIfNotMachine, \ + skipFlakyTest, skipUntrustedTest, skipBigDataTest, \ + skipIfMissingImports diff --git a/tests/functional/qemu_test/decorators.py b/tests/functional/qemu_test/decorators.py new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/functional/qemu_test/decorators.py @@ -XXX,XX +XXX,XX @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Decorators useful in functional tests + +import os +import platform +from unittest import skipUnless + +from .cmd import which + +''' +Decorator to skip execution of a test if the list +of command binaries is not available in $PATH. +Example: + + @skipIfMissingCommands("mkisofs", "losetup") +''' +def skipIfMissingCommands(*args): + def has_cmds(cmdlist): + for cmd in cmdlist: + if not which(cmd): + return False + return True + + return skipUnless(lambda: has_cmds(args), + 'required command(s) "%s" not installed' % + ", ".join(args)) + +''' +Decorator to skip execution of a test if the current +host machine does not match one of the permitted +machines. +Example + + @skipIfNotMachine("x86_64", "aarch64") +''' +def skipIfNotMachine(*args): + return skipUnless(lambda: platform.machine() in args, + 'not running on one of the required machine(s) "%s"' % + ", ".join(args)) + +''' +Decorator to skip execution of flaky tests, unless +the $QEMU_TEST_FLAKY_TESTS environment variable is set. +A bug URL must be provided that documents the observed +failure behaviour, so it can be tracked & re-evaluated +in future. + +Historical tests may be providing "None" as the bug_url +but this should not be done for new test. + +Example: + + @skipFlakyTest("https://gitlab.com/qemu-project/qemu/-/issues/NNN") +''' +def skipFlakyTest(bug_url): + if bug_url is None: + bug_url = "FIXME: reproduce flaky test and file bug report or remove" + return skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), + f'Test is unstable: {bug_url}') + +''' +Decorator to skip execution of tests which are likely +to execute untrusted commands on the host, or commands +which process untrusted code, unless the +$QEMU_TEST_ALLOW_UNTRUSTED_CODE env var is set. +Example: + + @skipUntrustedTest() +''' +def skipUntrustedTest(): + return skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), + 'Test runs untrusted code / processes untrusted data') + +''' +Decorator to skip execution of tests which need large +data storage (over around 500MB-1GB mark) on the host, +unless the $QEMU_TEST_ALLOW_LARGE_STORAGE environment +variable is set + +Example: + + @skipBigDataTest() +''' +def skipBigDataTest(): + return skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), + 'Test requires large host storage space') + +''' +Decorator to skip execution of a test if the list +of python imports is not available. +Example: + + @skipIfMissingImports("numpy", "cv2") +''' +def skipIfMissingImports(*args): + def has_imports(importlist): + for impname in importlist: + try: + import impname + except ImportError: + return False + return True + + return skipUnless(lambda: has_imports(args), + 'required import(s) "%s" not installed' % + ", ".join(args)) -- 2.46.0 This ensures consistency of behaviour across all the tests, and requires that we provide gitlab bug links when marking a test to be skipped due to unreliability. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/test_acpi_bits.py | 25 +++-------------- tests/functional/test_arm_aspeed_ast2600.py | 7 +++-- tests/functional/test_arm_bflt.py | 7 +++-- tests/functional/test_arm_bpim2u.py | 5 ++-- tests/functional/test_arm_cubieboard.py | 5 ++-- tests/functional/test_arm_integratorcp.py | 25 +++++------------ tests/functional/test_arm_orangepi.py | 8 +++--- tests/functional/test_linux_initrd.py | 7 +++-- tests/functional/test_m68k_nextcube.py | 15 +++-------- tests/functional/test_mips64el_fuloong2e.py | 4 +-- tests/functional/test_mips64el_loongson3v.py | 8 +++--- tests/functional/test_mips64el_malta.py | 28 +++++++------------- tests/functional/test_ppc64_hv.py | 18 +++---------- tests/functional/test_ppc_40p.py | 7 ++--- tests/functional/test_rx_gdbsim.py | 6 ++--- tests/functional/test_sh4_r2d.py | 8 +++--- 16 files changed, 61 insertions(+), 122 deletions(-) diff --git a/tests/functional/test_acpi_bits.py b/tests/functional/test_acpi_bits.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_acpi_bits.py +++ b/tests/functional/test_acpi_bits.py @@ -XXX,XX +XXX,XX @@ """ import os -import platform import re import shutil import subprocess @@ -XXX,XX +XXX,XX @@ Sequence, ) from qemu.machine import QEMUMachine -from unittest import skipIf -from qemu_test import QemuSystemTest, Asset, which +from qemu_test import (QemuSystemTest, Asset, skipIfMissingCommands, + skipIfNotMachine) -deps = ["xorriso", "mformat"] # dependent tools needed in the test setup/box. -supported_platforms = ['x86_64'] # supported test platforms. # default timeout of 120 secs is sometimes not enough for bits test. BITS_TIMEOUT = 200 -def missing_deps(): - """ returns True if any of the test dependent tools are absent. - """ - for dep in deps: - if which(dep) is None: - return True - return False - -def supported_platform(): - """ checks if the test is running on a supported platform. - """ - return platform.machine() in supported_platforms - class QEMUBitsMachine(QEMUMachine): # pylint: disable=too-few-public-methods """ A QEMU VM, with isa-debugcon enabled and bits iso passed @@ -XXX,XX +XXX,XX @@ def base_args(self): """return the base argument to QEMU binary""" return self._base_args -@skipIf(not supported_platform() or missing_deps(), - 'unsupported platform or dependencies (%s) not installed' \ - % ','.join(deps)) +@skipIfMissingCommands("xorriso", "mformat") +@skipIfNotMachine("x86_64") class AcpiBitsTest(QemuSystemTest): #pylint: disable=too-many-instance-attributes """ ACPI and SMBIOS tests using biosbits. diff --git a/tests/functional/test_arm_aspeed_ast2600.py b/tests/functional/test_arm_aspeed_ast2600.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_aspeed_ast2600.py +++ b/tests/functional/test_arm_aspeed_ast2600.py @@ -XXX,XX +XXX,XX @@ from qemu_test import Asset from aspeed import AspeedTest -from qemu_test import exec_command_and_wait_for_pattern -from qemu_test import has_cmd +from qemu_test import exec_command_and_wait_for_pattern, skipIfMissingCommands from qemu_test.utils import archive_extract -from unittest import skipUnless + class AST2600Machine(AspeedTest): @@ -XXX,XX +XXX,XX @@ def test_arm_ast2600_evb_buildroot(self): 'images/ast2600-evb/buildroot-2023.02-tpm/flash.img'), 'a46009ae8a5403a0826d607215e731a8c68d27c14c41e55331706b8f9c7bd997') - @skipUnless(*has_cmd('swtpm')) + @skipIfMissingCommands('swtpm') def test_arm_ast2600_evb_buildroot_tpm(self): self.set_machine('ast2600-evb') diff --git a/tests/functional/test_arm_bflt.py b/tests/functional/test_arm_bflt.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bflt.py +++ b/tests/functional/test_arm_bflt.py @@ -XXX,XX +XXX,XX @@ import bz2 from qemu_test import QemuUserTest, Asset -from qemu_test import has_cmd +from qemu_test import skipIfMissingCommands, skipUntrustedTest from qemu_test.utils import cpio_extract -from unittest import skipUnless class LoadBFLT(QemuUserTest): @@ -XXX,XX +XXX,XX @@ class LoadBFLT(QemuUserTest): ('https://elinux.org/images/5/51/Stm32_mini_rootfs.cpio.bz2'), 'eefb788e4980c9e8d6c9d60ce7d15d4da6bf4fbc6a80f487673824600d5ba9cc') - @skipUnless(*has_cmd('cpio')) - @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code') + @skipIfMissingCommands('cpio') + @skipUntrustedTest() def test_stm32(self): # See https://elinux.org/STM32#User_Space rootfs_path_bz2 = self.ASSET_ROOTFS.fetch() diff --git a/tests/functional/test_arm_bpim2u.py b/tests/functional/test_arm_bpim2u.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bpim2u.py +++ b/tests/functional/test_arm_bpim2u.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, exec_command_and_wait_for_pattern from qemu_test import Asset, interrupt_interactive_console_until_pattern +from qemu_test import skipBigDataTest from qemu_test.utils import gzip_uncompress, lzma_uncompress from qemu_test.utils import image_pow2ceil_expand -from unittest import skipUnless + class BananaPiMachine(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_gmac(self): os.remove(dtb_path) os.remove(rootfs_path) - @skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), 'storage limited') + @skipBigDataTest() def test_arm_bpim2u_openwrt_22_03_3(self): self.set_machine('bpim2u') # This test download a 8.9 MiB compressed image and expand it diff --git a/tests/functional/test_arm_cubieboard.py b/tests/functional/test_arm_cubieboard.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_cubieboard.py +++ b/tests/functional/test_arm_cubieboard.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, Asset, exec_command_and_wait_for_pattern from qemu_test import interrupt_interactive_console_until_pattern +from qemu_test import skipBigDataTest from qemu_test.utils import gzip_uncompress, image_pow2ceil_expand -from unittest import skipUnless + class CubieboardMachine(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_arm_cubieboard_sata(self): # Wait for VM to shut down gracefully self.vm.wait() - @skipUnless(os.getenv('AVOCADO_ALLOW_LARGE_STORAGE'), 'storage limited') + @skipBigDataTest() def test_arm_cubieboard_openwrt_22_03_2(self): # This test download a 7.5 MiB compressed image and expand it # to 126 MiB. diff --git a/tests/functional/test_arm_integratorcp.py b/tests/functional/test_arm_integratorcp.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_integratorcp.py +++ b/tests/functional/test_arm_integratorcp.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern -from unittest import skipUnless - - -NUMPY_AVAILABLE = True -try: - import numpy as np -except ImportError: - NUMPY_AVAILABLE = False - -CV2_AVAILABLE = True -try: - import cv2 -except ImportError: - CV2_AVAILABLE = False +from qemu_test import skipIfMissingImports, skipUntrustedTest class IntegratorMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def boot_integratorcp(self): '-append', 'printk.time=0 console=ttyAMA0') self.vm.launch() - @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code') + @skipUntrustedTest() def test_integratorcp_console(self): """ Boots the Linux kernel and checks that the console is operational @@ -XXX,XX +XXX,XX @@ def test_integratorcp_console(self): self.boot_integratorcp() wait_for_console_pattern(self, 'Log in as root') - @skipUnless(NUMPY_AVAILABLE, 'Python NumPy not installed') - @skipUnless(CV2_AVAILABLE, 'Python OpenCV not installed') - @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code') + @skipIfMissingImports("numpy", "cv2") + @skipUntrustedTest() def test_framebuffer_tux_logo(self): """ Boot Linux and verify the Tux logo is displayed on the framebuffer. """ + import numpy as np + import cv2 + screendump_path = os.path.join(self.workdir, "screendump.pbm") tuxlogo_path = self.ASSET_TUXLOGO.fetch() diff --git a/tests/functional/test_arm_orangepi.py b/tests/functional/test_arm_orangepi.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_orangepi.py +++ b/tests/functional/test_arm_orangepi.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, exec_command_and_wait_for_pattern from qemu_test import Asset, interrupt_interactive_console_until_pattern -from qemu_test import wait_for_console_pattern +from qemu_test import wait_for_console_pattern, skipBigDataTest from qemu_test.utils import gzip_uncompress, lzma_uncompress from qemu_test.utils import image_pow2ceil_expand -from unittest import skipUnless + class BananaPiMachine(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_sd(self): os.remove(dtb_path) os.remove(rootfs_path) - @skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), 'storage limited') + @skipBigDataTest() def test_arm_orangepi_armbian(self): self.set_machine('orangepi-pc') # This test download a 275 MiB compressed image and expand it @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_armbian(self): 'to <orangepipc>') self.wait_for_console_pattern('Starting Load Kernel Modules...') - @skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), 'storage limited') + @skipBigDataTest() def test_arm_orangepi_uboot_netbsd9(self): self.set_machine('orangepi-pc') # This test download a 304MB compressed image and expand it to 2GB diff --git a/tests/functional/test_linux_initrd.py b/tests/functional/test_linux_initrd.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_linux_initrd.py +++ b/tests/functional/test_linux_initrd.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import os import logging import tempfile -from qemu_test import QemuSystemTest, Asset -from unittest import skipUnless +from qemu_test import QemuSystemTest, Asset, skipFlakyTest class LinuxInitrd(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def test_with_2gib_file_should_exit_error_msg_with_linux_v3_6(self): max_size + 1) self.assertRegex(self.vm.get_log(), expected_msg) - @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab') + # XXX file tracking bug + @skipFlakyTest(bug_url=None) def test_with_2gib_file_should_work_with_linux_v4_16(self): """ QEMU has supported up to 4 GiB initrd for recent kernel diff --git a/tests/functional/test_m68k_nextcube.py b/tests/functional/test_m68k_nextcube.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_m68k_nextcube.py +++ b/tests/functional/test_m68k_nextcube.py @@ -XXX,XX +XXX,XX @@ import time from qemu_test import QemuSystemTest, Asset -from unittest import skipUnless - -from qemu_test import has_cmd +from qemu_test import skipIfMissingImports, skipIfMissingCommands from qemu_test.tesseract import tesseract_ocr -PIL_AVAILABLE = True -try: - from PIL import Image -except ImportError: - PIL_AVAILABLE = False - class NextCubeMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def check_bootrom_framebuffer(self, screenshot_path): self.vm.cmd('human-monitor-command', command_line='screendump %s' % screenshot_path) - @skipUnless(PIL_AVAILABLE, 'Python PIL not installed') + @skipIfMissingImports("PIL") def test_bootrom_framebuffer_size(self): self.set_machine('next-cube') screenshot_path = os.path.join(self.workdir, "dump.ppm") self.check_bootrom_framebuffer(screenshot_path) + from PIL import Image width, height = Image.open(screenshot_path).size self.assertEqual(width, 1120) self.assertEqual(height, 832) - @skipUnless(*has_cmd('tesseract')) + @skipIfMissingCommands('tesseract') def test_bootrom_framebuffer_ocr_with_tesseract(self): self.set_machine('next-cube') screenshot_path = os.path.join(self.workdir, "dump.ppm") diff --git a/tests/functional/test_mips64el_fuloong2e.py b/tests/functional/test_mips64el_fuloong2e.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_fuloong2e.py +++ b/tests/functional/test_mips64el_fuloong2e.py @@ -XXX,XX +XXX,XX @@ import subprocess from qemu_test import LinuxKernelTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import wait_for_console_pattern, skipUntrustedTest from unittest import skipUnless class MipsFuloong2e(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_linux_kernel_3_16(self): console_pattern = 'Kernel command line: %s' % kernel_command_line self.wait_for_console_pattern(console_pattern) - @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code') + @skipUntrustedTest() @skipUnless(os.getenv('RESCUE_YL_PATH'), 'RESCUE_YL_PATH not available') def test_linux_kernel_2_6_27_isa_serial(self): # Recovery system for the Yeeloong laptop diff --git a/tests/functional/test_mips64el_loongson3v.py b/tests/functional/test_mips64el_loongson3v.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_loongson3v.py +++ b/tests/functional/test_mips64el_loongson3v.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - -from unittest import skipUnless from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import wait_for_console_pattern, skipUntrustedTest + class MipsLoongson3v(QemuSystemTest): timeout = 60 @@ -XXX,XX +XXX,XX @@ class MipsLoongson3v(QemuSystemTest): 'releases/download/20210112/pmon-3avirt.bin'), 'fcdf6bb2cb7885a4a62f31fcb0d5e368bac7b6cea28f40c6dfa678af22fea20a') - @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code') + @skipUntrustedTest() def test_pmon_serial_console(self): self.set_machine('loongson3-virt') diff --git a/tests/functional/test_mips64el_malta.py b/tests/functional/test_mips64el_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_malta.py +++ b/tests/functional/test_mips64el_malta.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern +from qemu_test import skipIfMissingImports, skipFlakyTest, skipUntrustedTest from qemu_test.utils import gzip_uncompress -from unittest import skipUnless - -NUMPY_AVAILABLE = True -try: - import numpy as np -except ImportError: - NUMPY_AVAILABLE = False - -CV2_AVAILABLE = True -try: - import cv2 -except ImportError: - CV2_AVAILABLE = False class MaltaMachineConsole(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_mips64el_malta(self): 'rootfs.mipsel64r1.cpio.gz'), '75ba10cd35fb44e32948eeb26974f061b703c81c4ba2fab1ebcacf1d1bec3b61') - @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code') + @skipUntrustedTest() def test_mips64el_malta_5KEc_cpio(self): kernel_path = self.ASSET_KERNEL_3_19_3.fetch() initrd_path_gz = self.ASSET_CPIO_R1.fetch() @@ -XXX,XX +XXX,XX @@ def test_mips64el_malta_5KEc_cpio(self): self.vm.wait() -@skipUnless(NUMPY_AVAILABLE, 'Python NumPy not installed') -@skipUnless(CV2_AVAILABLE, 'Python OpenCV not installed') +@skipIfMissingImports('numpy', 'cv2') class MaltaMachineFramebuffer(LinuxKernelTest): timeout = 30 @@ -XXX,XX +XXX,XX @@ def do_test_i6400_framebuffer_logo(self, cpu_cores_count): """ Boot Linux kernel and check Tux logo is displayed on the framebuffer. """ + + import numpy as np + import cv2 + screendump_path = os.path.join(self.workdir, 'screendump.pbm') kernel_path_gz = self.ASSET_KERNEL_4_7_0.fetch() @@ -XXX,XX +XXX,XX @@ def do_test_i6400_framebuffer_logo(self, cpu_cores_count): def test_mips_malta_i6400_framebuffer_logo_1core(self): self.do_test_i6400_framebuffer_logo(1) - @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab') + # XXX file tracking bug + @skipFlakyTest(bug_url=None) def test_mips_malta_i6400_framebuffer_logo_7cores(self): self.do_test_i6400_framebuffer_logo(7) - @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab') + @skipFlakyTest(bug_url=None) def test_mips_malta_i6400_framebuffer_logo_8cores(self): self.do_test_i6400_framebuffer_logo(8) diff --git a/tests/functional/test_ppc64_hv.py b/tests/functional/test_ppc64_hv.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc64_hv.py +++ b/tests/functional/test_ppc64_hv.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from unittest import skipIf, skipUnless from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern, exec_command, which +from qemu_test import wait_for_console_pattern, exec_command +from qemu_test import skipIfMissingCommands, skipBigDataTest import os import time import subprocess from datetime import datetime -deps = ["xorriso"] # dependent tools needed in the test setup/box. - -def missing_deps(): - """ returns True if any of the test dependent tools are absent. - """ - for dep in deps: - if which(dep) is None: - return True - return False - # Alpine is a light weight distro that supports QEMU. These tests boot # that on the machine then run a QEMU guest inside it in KVM mode, # that runs the same Alpine distro image. @@ -XXX,XX +XXX,XX @@ def missing_deps(): # large download, but it may be more polite to create qcow2 image with # QEMU already installed and use that. # XXX: The order of these tests seems to matter, see git blame. -@skipIf(missing_deps(), 'dependencies (%s) not installed' % ','.join(deps)) -@skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), 'storage limited') +@skipIfMissingCommands("xorriso") +@skipBigDataTest() class HypervisorTest(QemuSystemTest): timeout = 1000 diff --git a/tests/functional/test_ppc_40p.py b/tests/functional/test_ppc_40p.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_40p.py +++ b/tests/functional/test_ppc_40p.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import os - -from unittest import skipUnless from qemu_test import QemuSystemTest, Asset -from qemu_test import wait_for_console_pattern +from qemu_test import wait_for_console_pattern, skipUntrustedTest class IbmPrep40pMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ class IbmPrep40pMachine(QemuSystemTest): # All rights reserved. # U.S. Government Users Restricted Rights - Use, duplication or disclosure # restricted by GSA ADP Schedule Contract with IBM Corp. - @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code') + @skipUntrustedTest() def test_factory_firmware_and_netbsd(self): self.set_machine('40p') self.require_accelerator("tcg") diff --git a/tests/functional/test_rx_gdbsim.py b/tests/functional/test_rx_gdbsim.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_rx_gdbsim.py +++ b/tests/functional/test_rx_gdbsim.py @@ -XXX,XX +XXX,XX @@ import os -from unittest import skipUnless from qemu_test import QemuSystemTest, Asset from qemu_test import exec_command_and_wait_for_pattern -from qemu_test import wait_for_console_pattern +from qemu_test import wait_for_console_pattern, skipFlakyTest from qemu_test.utils import gzip_uncompress @@ -XXX,XX +XXX,XX @@ def test_uboot(self): wait_for_console_pattern(self, uboot_version) gcc_version = 'rx-unknown-linux-gcc (GCC) 9.0.0 20181105 (experimental)' # FIXME limit baudrate on chardev, else we type too fast + # https://gitlab.com/qemu-project/qemu/-/issues/2691 #exec_command_and_wait_for_pattern(self, 'version', gcc_version) - @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab') + @skipFlakyTest(bug_url="https://gitlab.com/qemu-project/qemu/-/issues/2691") def test_linux_sash(self): """ Boots a Linux kernel and checks that the console is operational. diff --git a/tests/functional/test_sh4_r2d.py b/tests/functional/test_sh4_r2d.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sh4_r2d.py +++ b/tests/functional/test_sh4_r2d.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - -from qemu_test import LinuxKernelTest, Asset +from qemu_test import LinuxKernelTest, Asset, skipFlakyTest from qemu_test.utils import archive_extract -from unittest import skipUnless class R2dTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class R2dTest(LinuxKernelTest): # This test has a 6-10% failure rate on various hosts that look # like issues with a buggy kernel. - @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable') + # XXX file tracking bug + @skipFlakyTest(bug_url=None) def test_r2d(self): self.set_machine('r2d') file_path = self.ASSET_DAY09.fetch() -- 2.46.0 The 'which' helper is simpler, not depending on the external 'which' binary, and is sufficient for test needs. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/__init__.py | 2 +- tests/functional/qemu_test/cmd.py | 54 ++---------------------- tests/functional/qemu_test/tuxruntest.py | 10 ++--- 3 files changed, 9 insertions(+), 57 deletions(-) diff --git a/tests/functional/qemu_test/__init__.py b/tests/functional/qemu_test/__init__.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/__init__.py +++ b/tests/functional/qemu_test/__init__.py @@ -XXX,XX +XXX,XX @@ from .asset import Asset from .config import BUILD_DIR -from .cmd import has_cmd, has_cmds, run_cmd, is_readable_executable_file, \ +from .cmd import run_cmd, is_readable_executable_file, \ interrupt_interactive_console_until_pattern, wait_for_console_pattern, \ exec_command, exec_command_and_wait_for_pattern, get_qemu_img, which from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest diff --git a/tests/functional/qemu_test/cmd.py b/tests/functional/qemu_test/cmd.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/cmd.py +++ b/tests/functional/qemu_test/cmd.py @@ -XXX,XX +XXX,XX @@ def which(tool): return p return None -def has_cmd(name, args=None): - """ - This function is for use in a @skipUnless decorator, e.g.: - - @skipUnless(*has_cmd('sudo -n', ('sudo', '-n', 'true'))) - def test_something_that_needs_sudo(self): - ... - """ - - if args is None: - args = ('which', name) - - try: - _, stderr, exitcode = run_cmd(args) - except Exception as e: - exitcode = -1 - stderr = str(e) - - if exitcode != 0: - cmd_line = ' '.join(args) - err = f'{name} required, but "{cmd_line}" failed: {stderr.strip()}' - return (False, err) - else: - return (True, '') - -def has_cmds(*cmds): - """ - This function is for use in a @skipUnless decorator and - allows checking for the availability of multiple commands, e.g.: - - @skipUnless(*has_cmds(('cmd1', ('cmd1', '--some-parameter')), - 'cmd2', 'cmd3')) - def test_something_that_needs_cmd1_and_cmd2(self): - ... - """ - - for cmd in cmds: - if isinstance(cmd, str): - cmd = (cmd,) - - ok, errstr = has_cmd(*cmd) - if not ok: - return (False, errstr) - - return (True, '') - def run_cmd(args): subp = subprocess.Popen(args, stdout=subprocess.PIPE, @@ -XXX,XX +XXX,XX @@ def get_qemu_img(test): qemu_img = os.path.join(BUILD_DIR, 'qemu-img') if os.path.exists(qemu_img): return qemu_img - (has_system_qemu_img, errmsg) = has_cmd('qemu-img') - if has_system_qemu_img: - return 'qemu-img' - test.skipTest(errmsg) + qemu_img = which('qemu-img') + if qemu_img is not None: + return qemu_img + test.skipTest(f"qemu-img not found in {BUILD_DIR} or '$PATH'") diff --git a/tests/functional/qemu_test/tuxruntest.py b/tests/functional/qemu_test/tuxruntest.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/tuxruntest.py +++ b/tests/functional/qemu_test/tuxruntest.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuSystemTest from qemu_test import exec_command_and_wait_for_pattern from qemu_test import wait_for_console_pattern -from qemu_test import has_cmd, run_cmd, get_qemu_img +from qemu_test import which, run_cmd, get_qemu_img class TuxRunBaselineTest(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def setUp(self): super().setUp() # We need zstd for all the tuxrun tests - (has_zstd, msg) = has_cmd('zstd') - if has_zstd is False: - self.skipTest(msg) - self.zstd = 'zstd' + if which('zstd') is None: + self.skipTest("zstd not found in $PATH") # Pre-init TuxRun specific settings: Most machines work with # reasonable defaults but we sometimes need to tweak the @@ -XXX,XX +XXX,XX @@ def fetch_tuxrun_assets(self, kernel_asset, rootfs_asset, dtb_asset=None): disk_image = self.workdir + "/rootfs.ext4" - run_cmd([self.zstd, "-f", "-d", disk_image_zst, + run_cmd(['zstd', "-f", "-d", disk_image_zst, "-o", disk_image]) # zstd copies source archive permissions for the output # file, so must make this writable for QEMU -- 2.46.0 Add helper methods that construct paths for * log files - to be preserved at the end of a test * scratch files - to be purged at the end of a test * build files - anything relative to the build root * data files - anything relative to the functional test source root * socket files - a short temporary dir to avoid UNIX socket limits These are to be used instead of direct access to the self.workdir, or self.logdir variables, or any other place where paths are built manually. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/testcase.py | 95 ++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -XXX,XX +XXX,XX @@ import logging import os +from pathlib import Path import pycotap import shutil import subprocess import sys +import tempfile import unittest import uuid @@ -XXX,XX +XXX,XX @@ class QemuBaseTest(unittest.TestCase): log = None logdir = None + ''' + Create a temporary directory suitable for storing UNIX + socket paths. + + Returns: a tempfile.TemporaryDirectory instance + ''' + def socket_dir(self): + if self.socketdir is None: + self.socketdir = tempfile.TemporaryDirectory( + prefix="qemu_func_test_sock_") + return self.socketdir + + ''' + @params args list of zero or more subdirectories or file + + Construct a path for accessing a data file located + relative to the source directory that is the root for + functional tests. + + @args may be an empty list to reference the root dir + itself, may be a single element to reference a file in + the root directory, or may be multiple elements to + reference a file nested below. The path components + will be joined using the platform appropriate path + separator. + + Returns: string representing a file path + ''' + def data_file(self, *args): + return str(Path(Path(__file__).parent.parent, *args)) + + ''' + @params args list of zero or more subdirectories or file + + Construct a path for accessing a data file located + relative to the build directory root. + + @args may be an empty list to reference the build dir + itself, may be a single element to reference a file in + the build directory, or may be multiple elements to + reference a file nested below. The path components + will be joined using the platform appropriate path + separator. + + Returns: string representing a file path + ''' + def build_file(self, *args): + return str(Path(BUILD_DIR, *args)) + + ''' + @params args list of zero or more subdirectories or file + + Construct a path for accessing/creating a scratch file + located relative to a temporary directory dedicated to + this test case. The directory and its contents will be + purged upon completion of the test. + + @args may be an empty list to reference the scratch dir + itself, may be a single element to reference a file in + the scratch directory, or may be multiple elements to + reference a file nested below. The path components + will be joined using the platform appropriate path + separator. + + Returns: string representing a file path + ''' + def scratch_file(self, *args): + return str(Path(self.workdir, *args)) + + ''' + @params args list of zero or more subdirectories or file + + Construct a path for accessing/creating a log file + located relative to a temporary directory dedicated to + this test case. The directory and its log files will be + preserved upon completion of the test. + + @args may be an empty list to reference the log dir + itself, may be a single element to reference a file in + the log directory, or may be multiple elements to + reference a file nested below. The path components + will be joined using the platform appropriate path + separator. + + Returns: string representing a file path + ''' + def log_file(self, *args): + return str(Path(self.logdir, *args)) + def setUp(self, bin_prefix): self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set') self.arch = self.qemu_bin.split('-')[-1] + self.socketdir = None self.outputdir = os.path.join(BUILD_DIR, 'tests', 'functional', self.arch, self.id()) @@ -XXX,XX +XXX,XX @@ def setUp(self, bin_prefix): def tearDown(self): if "QEMU_TEST_KEEP_SCRATCH" not in os.environ: shutil.rmtree(self.workdir) + if self.socketdir is not None: + shutil.rmtree(self.socketdir.name) + self.socketdir = None self.machinelog.removeHandler(self._log_fh) self.log.removeHandler(self._log_fh) -- 2.46.0 This removes direct access of the 'self.logdir' variable. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/testcase.py | 9 ++++----- tests/functional/test_virtio_gpu.py | 4 +--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -XXX,XX +XXX,XX @@ def scratch_file(self, *args): Returns: string representing a file path ''' def log_file(self, *args): - return str(Path(self.logdir, *args)) + return str(Path(self.outputdir, *args)) def setUp(self, bin_prefix): self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set') @@ -XXX,XX +XXX,XX @@ def setUp(self, bin_prefix): self.workdir = os.path.join(self.outputdir, 'scratch') os.makedirs(self.workdir, exist_ok=True) - self.logdir = self.outputdir - self.log_filename = os.path.join(self.logdir, 'base.log') + self.log_filename = self.log_file('base.log') self.log = logging.getLogger('qemu-test') self.log.setLevel(logging.DEBUG) self._log_fh = logging.FileHandler(self.log_filename, mode='w') @@ -XXX,XX +XXX,XX @@ def setUp(self): console_log = logging.getLogger('console') console_log.setLevel(logging.DEBUG) - self.console_log_name = os.path.join(self.logdir, 'console.log') + self.console_log_name = self.log_file('console.log') self._console_log_fh = logging.FileHandler(self.console_log_name, mode='w') self._console_log_fh.setLevel(logging.DEBUG) @@ -XXX,XX +XXX,XX @@ def _new_vm(self, name, *args): vm = QEMUMachine(self.qemu_bin, name=name, base_temp_dir=self.workdir, - log_dir=self.logdir) + log_dir=self.log_file()) self.log.debug('QEMUMachine "%s" created', name) self.log.debug('QEMUMachine "%s" temp_dir: %s', name, vm.temp_dir) diff --git a/tests/functional/test_virtio_gpu.py b/tests/functional/test_virtio_gpu.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_virtio_gpu.py +++ b/tests/functional/test_virtio_gpu.py @@ -XXX,XX +XXX,XX @@ def test_vhost_user_vga_virgl(self): os.set_inheritable(qemu_sock.fileno(), True) os.set_inheritable(vug_sock.fileno(), True) - self._vug_log_path = os.path.join( - self.logdir, "vhost-user-gpu.log" - ) + self._vug_log_path = self.log_file("vhost-user-gpu.log") self._vug_log_file = open(self._vug_log_path, "wb") self.log.info('Complete vhost-user-gpu.log file can be ' 'found at %s', self._vug_log_path) -- 2.46.0 This removes direct access of the 'BUILD_DIR' variable. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/cmd.py | 5 ++--- tests/functional/qemu_test/testcase.py | 4 ++-- tests/functional/test_aarch64_virt.py | 5 ++--- tests/functional/test_virtio_gpu.py | 11 +++-------- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/tests/functional/qemu_test/cmd.py b/tests/functional/qemu_test/cmd.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/cmd.py +++ b/tests/functional/qemu_test/cmd.py @@ -XXX,XX +XXX,XX @@ import os.path import subprocess -from .config import BUILD_DIR def which(tool): """ looks up the full path for @tool, returns None if not found @@ -XXX,XX +XXX,XX @@ def get_qemu_img(test): # If qemu-img has been built, use it, otherwise the system wide one # will be used. - qemu_img = os.path.join(BUILD_DIR, 'qemu-img') + qemu_img = test.build_file('qemu-img') if os.path.exists(qemu_img): return qemu_img qemu_img = which('qemu-img') if qemu_img is not None: return qemu_img - test.skipTest(f"qemu-img not found in {BUILD_DIR} or '$PATH'") + test.skipTest(f"qemu-img not found in build dir or '$PATH'") diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -XXX,XX +XXX,XX @@ def setUp(self, bin_prefix): self.arch = self.qemu_bin.split('-')[-1] self.socketdir = None - self.outputdir = os.path.join(BUILD_DIR, 'tests', 'functional', - self.arch, self.id()) + self.outputdir = self.build_file('tests', 'functional', + self.arch, self.id()) self.workdir = os.path.join(self.outputdir, 'scratch') os.makedirs(self.workdir, exist_ok=True) diff --git a/tests/functional/test_aarch64_virt.py b/tests/functional/test_aarch64_virt.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_virt.py +++ b/tests/functional/test_aarch64_virt.py @@ -XXX,XX +XXX,XX @@ import os import logging -from qemu_test import BUILD_DIR from qemu_test import QemuSystemTest, Asset from qemu_test import exec_command, wait_for_console_pattern from qemu_test import get_qemu_img, run_cmd @@ -XXX,XX +XXX,XX @@ def test_alpine_virt_tcg_gic_max(self): "mte=on," "gic-version=max,iommu=smmuv3") self.vm.add_args("-smp", "2", "-m", "1024") - self.vm.add_args('-bios', os.path.join(BUILD_DIR, 'pc-bios', - 'edk2-aarch64-code.fd')) + self.vm.add_args('-bios', self.build_file('pc-bios', + 'edk2-aarch64-code.fd')) self.vm.add_args("-drive", f"file={iso_path},media=cdrom,format=raw") self.vm.add_args('-device', 'virtio-rng-pci,rng=rng0') self.vm.add_args('-object', 'rng-random,id=rng0,filename=/dev/urandom') diff --git a/tests/functional/test_virtio_gpu.py b/tests/functional/test_virtio_gpu.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_virtio_gpu.py +++ b/tests/functional/test_virtio_gpu.py @@ -XXX,XX +XXX,XX @@ # later. See the COPYING file in the top-level directory. -from qemu_test import BUILD_DIR from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern from qemu_test import exec_command_and_wait_for_pattern @@ -XXX,XX +XXX,XX @@ import subprocess -def pick_default_vug_bin(): - relative_path = "./contrib/vhost-user-gpu/vhost-user-gpu" - if is_readable_executable_file(relative_path): - return relative_path - - bld_dir_path = os.path.join(BUILD_DIR, relative_path) +def pick_default_vug_bin(test): + bld_dir_path = test.build_file("contrib", "vhost-user-gpu", "vhost-user-gpu") if is_readable_executable_file(bld_dir_path): return bld_dir_path @@ -XXX,XX +XXX,XX @@ def test_vhost_user_vga_virgl(self): # FIXME: should check presence of vhost-user-gpu, virgl, memfd etc self.require_accelerator('kvm') - vug = pick_default_vug_bin() + vug = pick_default_vug_bin(self) if not vug: self.skipTest("Could not find vhost-user-gpu") -- 2.46.0 This removes direct path manipulation to figure out the source dir Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/test_acpi_bits.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/tests/functional/test_acpi_bits.py b/tests/functional/test_acpi_bits.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_acpi_bits.py +++ b/tests/functional/test_acpi_bits.py @@ -XXX,XX +XXX,XX @@ import tarfile import zipfile -from pathlib import Path from typing import ( List, Optional, @@ -XXX,XX +XXX,XX @@ class AcpiBitsTest(QemuSystemTest): #pylint: disable=too-many-instance-attribute def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._vm = None - self._baseDir = None self._debugcon_addr = '0x403' self._debugcon_log = 'debugcon-log.txt' @@ -XXX,XX +XXX,XX @@ def _print_log(self, log): def copy_bits_config(self): """ copies the bios bits config file into bits. """ - config_file = 'bits-cfg.txt' - bits_config_dir = os.path.join(self._baseDir, 'acpi-bits', - 'bits-config') + bits_config_file = self.data_file('acpi-bits', + 'bits-config', + 'bits-cfg.txt') target_config_dir = os.path.join(self.workdir, 'bits-%d' %self.BITS_INTERNAL_VER, 'boot') - self.assertTrue(os.path.exists(bits_config_dir)) + self.assertTrue(os.path.exists(bits_config_file)) self.assertTrue(os.path.exists(target_config_dir)) - self.assertTrue(os.access(os.path.join(bits_config_dir, - config_file), os.R_OK)) - shutil.copy2(os.path.join(bits_config_dir, config_file), - target_config_dir) + shutil.copy2(bits_config_file, target_config_dir) self.logger.info('copied config file %s to %s', - config_file, target_config_dir) + bits_config_file, target_config_dir) def copy_test_scripts(self): """copies the python test scripts into bits. """ - bits_test_dir = os.path.join(self._baseDir, 'acpi-bits', - 'bits-tests') + bits_test_dir = self.data_file('acpi-bits', 'bits-tests') target_test_dir = os.path.join(self.workdir, 'bits-%d' %self.BITS_INTERNAL_VER, 'boot', 'python') @@ -XXX,XX +XXX,XX @@ def setUp(self): # pylint: disable=arguments-differ super().setUp() self.logger = self.log - self._baseDir = Path(__file__).parent - prebuiltDir = os.path.join(self.workdir, 'prebuilt') if not os.path.isdir(prebuiltDir): os.mkdir(prebuiltDir, mode=0o775) -- 2.46.0 Replace any instances of os.path.join(self.workdir, ".../...") self.workdir + "/.../..." with self.scratch_file("...", "...") which is more compact and portable Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/linuxkernel.py | 7 ++- tests/functional/qemu_test/tuxruntest.py | 2 +- tests/functional/test_aarch64_aspeed.py | 17 ++++--- tests/functional/test_aarch64_raspi3.py | 3 +- tests/functional/test_aarch64_raspi4.py | 4 +- tests/functional/test_aarch64_sbsaref.py | 6 +-- tests/functional/test_aarch64_virt.py | 3 +- tests/functional/test_acpi_bits.py | 49 +++++++++---------- tests/functional/test_alpha_clipper.py | 4 +- tests/functional/test_arm_aspeed_ast1030.py | 6 +-- tests/functional/test_arm_aspeed_ast2500.py | 2 +- tests/functional/test_arm_aspeed_ast2600.py | 2 +- tests/functional/test_arm_bflt.py | 3 +- tests/functional/test_arm_bpim2u.py | 6 +-- tests/functional/test_arm_canona1100.py | 3 +- tests/functional/test_arm_cubieboard.py | 6 +-- tests/functional/test_arm_emcraft_sf2.py | 2 +- tests/functional/test_arm_integratorcp.py | 3 +- tests/functional/test_arm_orangepi.py | 8 +-- tests/functional/test_arm_raspi2.py | 4 +- tests/functional/test_arm_smdkc210.py | 2 +- tests/functional/test_arm_vexpress.py | 5 +- tests/functional/test_m68k_mcf5208evb.py | 3 +- tests/functional/test_m68k_nextcube.py | 5 +- .../functional/test_microblaze_s3adsp1800.py | 3 +- .../test_microblazeel_s3adsp1800.py | 5 +- tests/functional/test_mips64el_malta.py | 6 +-- tests/functional/test_mips_malta.py | 4 +- tests/functional/test_mipsel_malta.py | 6 +-- tests/functional/test_or1k_sim.py | 2 +- tests/functional/test_ppc64_e500.py | 2 +- tests/functional/test_ppc64_hv.py | 12 +---- tests/functional/test_ppc_amiga.py | 12 +++-- tests/functional/test_ppc_bamboo.py | 10 ++-- tests/functional/test_ppc_mac.py | 2 +- tests/functional/test_ppc_mpc8544ds.py | 2 +- tests/functional/test_ppc_virtex_ml507.py | 5 +- tests/functional/test_rx_gdbsim.py | 4 +- tests/functional/test_s390x_ccw_virtio.py | 2 +- tests/functional/test_s390x_topology.py | 4 +- tests/functional/test_sh4_r2d.py | 3 +- tests/functional/test_sh4eb_r2d.py | 8 +-- tests/functional/test_sparc64_sun4u.py | 5 +- tests/functional/test_sparc_sun4m.py | 2 +- tests/functional/test_xtensa_lx60.py | 3 +- 45 files changed, 118 insertions(+), 139 deletions(-) diff --git a/tests/functional/qemu_test/linuxkernel.py b/tests/functional/qemu_test/linuxkernel.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/linuxkernel.py +++ b/tests/functional/qemu_test/linuxkernel.py @@ -XXX,XX +XXX,XX @@ def extract_from_deb(self, deb_path, path): os.chdir(cwd) # Return complete path to extracted file. Because callers to # extract_from_deb() specify 'path' with a leading slash, it is - # necessary to use os.path.relpath() as otherwise os.path.join() - # interprets it as an absolute path and drops the self.workdir part. - return os.path.normpath(os.path.join(self.workdir, - os.path.relpath(path, '/'))) + # necessary to use os.path.relpath() as otherwise scratch_file() + # interprets it as an absolute path and drops the required prefix + return os.path.normpath(self.scratch_file(os.path.relpath(path, '/'))) diff --git a/tests/functional/qemu_test/tuxruntest.py b/tests/functional/qemu_test/tuxruntest.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/tuxruntest.py +++ b/tests/functional/qemu_test/tuxruntest.py @@ -XXX,XX +XXX,XX @@ def fetch_tuxrun_assets(self, kernel_asset, rootfs_asset, dtb_asset=None): kernel_image = kernel_asset.fetch() disk_image_zst = rootfs_asset.fetch() - disk_image = self.workdir + "/rootfs.ext4" + disk_image = self.scratch_file("rootfs.ext4") run_cmd(['zstd', "-f", "-d", disk_image_zst, "-o", disk_image]) diff --git a/tests/functional/test_aarch64_aspeed.py b/tests/functional/test_aarch64_aspeed.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_aspeed.py +++ b/tests/functional/test_aarch64_aspeed.py @@ -XXX,XX +XXX,XX @@ def test_aarch64_ast2700_evb_sdk_v09_02(self): archive_extract(image_path, self.workdir) num_cpu = 4 - image_dir = self.workdir + '/ast2700-default/' - uboot_size = os.path.getsize(image_dir + 'u-boot-nodtb.bin') + uboot_size = os.path.getsize(self.scratch_file('ast2700-default', + 'u-boot-nodtb.bin')) uboot_dtb_load_addr = hex(0x400000000 + uboot_size) load_images_list = [ { 'addr': '0x400000000', - 'file': image_dir + 'u-boot-nodtb.bin' + 'file': self.scratch_file('ast2700-default', + 'u-boot-nodtb.bin') }, { 'addr': str(uboot_dtb_load_addr), - 'file': image_dir + 'u-boot.dtb' + 'file': self.scratch_file('ast2700-default', 'u-boot.dtb') }, { 'addr': '0x430000000', - 'file': image_dir + 'bl31.bin' + 'file': self.scratch_file('ast2700-default', 'bl31.bin') }, { 'addr': '0x430080000', - 'file': image_dir + 'optee/tee-raw.bin' + 'file': self.scratch_file('ast2700-default', 'optee', + 'tee-raw.bin') } ] @@ -XXX,XX +XXX,XX @@ def test_aarch64_ast2700_evb_sdk_v09_02(self): self.vm.add_args('-smp', str(num_cpu)) self.vm.add_args('-device', 'tmp105,bus=aspeed.i2c.bus.1,address=0x4d,id=tmp-test') - self.do_test_aarch64_aspeed_sdk_start(image_dir + 'image-bmc') + self.do_test_aarch64_aspeed_sdk_start( + self.scratch_file('ast2700-default', 'image-bmc')) wait_for_console_pattern(self, 'ast2700-default login:') diff --git a/tests/functional/test_aarch64_raspi3.py b/tests/functional/test_aarch64_raspi3.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_raspi3.py +++ b/tests/functional/test_aarch64_raspi3.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os from zipfile import ZipFile from qemu_test import LinuxKernelTest, Asset @@ -XXX,XX +XXX,XX @@ def test_aarch64_raspi3_atf(self): with ZipFile(zip_path, 'r') as zf: zf.extract(efi_name, path=self.workdir) - efi_fd = os.path.join(self.workdir, efi_name) + efi_fd = self.scratch_file(efi_name) self.set_machine('raspi3b') self.vm.set_console(console_index=1) diff --git a/tests/functional/test_aarch64_raspi4.py b/tests/functional/test_aarch64_raspi4.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_raspi4.py +++ b/tests/functional/test_aarch64_raspi4.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern from qemu_test.utils import gzip_uncompress @@ -XXX,XX +XXX,XX @@ def test_arm_raspi4_initrd(self): kernel_path = self.extract_from_deb(deb_path, '/boot/kernel8.img') dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2711-rpi-4-b.dtb') initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = os.path.join(self.workdir, 'rootfs.cpio') + initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) self.set_machine('raspi4b') diff --git a/tests/functional/test_aarch64_sbsaref.py b/tests/functional/test_aarch64_sbsaref.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_sbsaref.py +++ b/tests/functional/test_aarch64_sbsaref.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern from qemu_test import interrupt_interactive_console_until_pattern @@ -XXX,XX +XXX,XX @@ def fetch_firmware(test): # Secure BootRom (TF-A code) fs0_xz_path = Aarch64SbsarefMachine.ASSET_FLASH0.fetch() - fs0_path = os.path.join(test.workdir, "SBSA_FLASH0.fd") + fs0_path = test.scratch_file("SBSA_FLASH0.fd") lzma_uncompress(fs0_xz_path, fs0_path) # Non-secure rom (UEFI and EFI variables) fs1_xz_path = Aarch64SbsarefMachine.ASSET_FLASH1.fetch() - fs1_path = os.path.join(test.workdir, "SBSA_FLASH1.fd") + fs1_path = test.scratch_file("SBSA_FLASH1.fd") lzma_uncompress(fs1_xz_path, fs1_path) for path in [fs0_path, fs1_path]: diff --git a/tests/functional/test_aarch64_virt.py b/tests/functional/test_aarch64_virt.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_virt.py +++ b/tests/functional/test_aarch64_virt.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later import time -import os import logging from qemu_test import QemuSystemTest, Asset @@ -XXX,XX +XXX,XX @@ def common_aarch64_virt(self, machine): # Also add a scratch block device logger.info('creating scratch qcow2 image') - image_path = os.path.join(self.workdir, 'scratch.qcow2') + image_path = self.scratch_file('scratch.qcow2') qemu_img = get_qemu_img(self) run_cmd([qemu_img, 'create', '-f', 'qcow2', image_path, '8M']) diff --git a/tests/functional/test_acpi_bits.py b/tests/functional/test_acpi_bits.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_acpi_bits.py +++ b/tests/functional/test_acpi_bits.py @@ -XXX,XX +XXX,XX @@ def copy_bits_config(self): bits_config_file = self.data_file('acpi-bits', 'bits-config', 'bits-cfg.txt') - target_config_dir = os.path.join(self.workdir, - 'bits-%d' %self.BITS_INTERNAL_VER, - 'boot') + target_config_dir = self.scratch_file('bits-%d' % + self.BITS_INTERNAL_VER, + 'boot') self.assertTrue(os.path.exists(bits_config_file)) self.assertTrue(os.path.exists(target_config_dir)) shutil.copy2(bits_config_file, target_config_dir) @@ -XXX,XX +XXX,XX @@ def copy_test_scripts(self): """copies the python test scripts into bits. """ bits_test_dir = self.data_file('acpi-bits', 'bits-tests') - target_test_dir = os.path.join(self.workdir, - 'bits-%d' %self.BITS_INTERNAL_VER, - 'boot', 'python') + target_test_dir = self.scratch_file('bits-%d' % self.BITS_INTERNAL_VER, + 'boot', 'python') self.assertTrue(os.path.exists(bits_test_dir)) self.assertTrue(os.path.exists(target_test_dir)) @@ -XXX,XX +XXX,XX @@ def fix_mkrescue(self, mkrescue): the directory where we have extracted our pre-built bits grub tarball. """ - grub_x86_64_mods = os.path.join(self.workdir, 'grub-inst-x86_64-efi') - grub_i386_mods = os.path.join(self.workdir, 'grub-inst') + grub_x86_64_mods = self.scratch_file('grub-inst-x86_64-efi') + grub_i386_mods = self.scratch_file('grub-inst') self.assertTrue(os.path.exists(grub_x86_64_mods)) self.assertTrue(os.path.exists(grub_i386_mods)) @@ -XXX,XX +XXX,XX @@ def generate_bits_iso(self): """ Uses grub-mkrescue to generate a fresh bits iso with the python test scripts """ - bits_dir = os.path.join(self.workdir, - 'bits-%d' %self.BITS_INTERNAL_VER) - iso_file = os.path.join(self.workdir, - 'bits-%d.iso' %self.BITS_INTERNAL_VER) - mkrescue_script = os.path.join(self.workdir, - 'grub-inst-x86_64-efi', 'bin', - 'grub-mkrescue') + bits_dir = self.scratch_file('bits-%d' % self.BITS_INTERNAL_VER) + iso_file = self.scratch_file('bits-%d.iso' % self.BITS_INTERNAL_VER) + mkrescue_script = self.scratch_file('grub-inst-x86_64-efi', + 'bin', + 'grub-mkrescue') self.assertTrue(os.access(mkrescue_script, os.R_OK | os.W_OK | os.X_OK)) @@ -XXX,XX +XXX,XX @@ def setUp(self): # pylint: disable=arguments-differ super().setUp() self.logger = self.log - prebuiltDir = os.path.join(self.workdir, 'prebuilt') + prebuiltDir = self.scratch_file('prebuilt') if not os.path.isdir(prebuiltDir): os.mkdir(prebuiltDir, mode=0o775) - bits_zip_file = os.path.join(prebuiltDir, 'bits-%d-%s.zip' - %(self.BITS_INTERNAL_VER, - self.BITS_COMMIT_HASH)) - grub_tar_file = os.path.join(prebuiltDir, - 'bits-%d-%s-grub.tar.gz' - %(self.BITS_INTERNAL_VER, - self.BITS_COMMIT_HASH)) + bits_zip_file = self.scratch_file('prebuilt', + 'bits-%d-%s.zip' + %(self.BITS_INTERNAL_VER, + self.BITS_COMMIT_HASH)) + grub_tar_file = self.scratch_file('prebuilt', + 'bits-%d-%s-grub.tar.gz' + %(self.BITS_INTERNAL_VER, + self.BITS_COMMIT_HASH)) bitsLocalArtLoc = self.ASSET_BITS.fetch() self.logger.info("downloaded bits artifacts to %s", bitsLocalArtLoc) @@ -XXX,XX +XXX,XX @@ def parse_log(self): """parse the log generated by running bits tests and check for failures. """ - debugconf = os.path.join(self.workdir, self._debugcon_log) + debugconf = self.scratch_file(self._debugcon_log) log = "" with open(debugconf, 'r', encoding='utf-8') as filehandle: log = filehandle.read() @@ -XXX,XX +XXX,XX @@ def test_acpi_smbios_bits(self): """The main test case implementation.""" self.set_machine('pc') - iso_file = os.path.join(self.workdir, - 'bits-%d.iso' %self.BITS_INTERNAL_VER) + iso_file = self.scratch_file('bits-%d.iso' % self.BITS_INTERNAL_VER) self.assertTrue(os.access(iso_file, os.R_OK)) diff --git a/tests/functional/test_alpha_clipper.py b/tests/functional/test_alpha_clipper.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_alpha_clipper.py +++ b/tests/functional/test_alpha_clipper.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - from qemu_test import LinuxKernelTest, Asset from qemu_test.utils import gzip_uncompress @@ -XXX,XX +XXX,XX @@ def test_alpha_clipper(self): self.set_machine('clipper') kernel_path = self.ASSET_KERNEL.fetch() - uncompressed_kernel = os.path.join(self.workdir, 'vmlinux') + uncompressed_kernel = self.scratch_file('vmlinux') gzip_uncompress(kernel_path, uncompressed_kernel) self.vm.set_console() diff --git a/tests/functional/test_arm_aspeed_ast1030.py b/tests/functional/test_arm_aspeed_ast1030.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_aspeed_ast1030.py +++ b/tests/functional/test_arm_aspeed_ast1030.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern from zipfile import ZipFile @@ -XXX,XX +XXX,XX @@ def test_ast1030_zephyros_1_04(self): kernel_name = "ast1030-evb-demo/zephyr.elf" with ZipFile(zip_file, 'r') as zf: zf.extract(kernel_name, path=self.workdir) - kernel_file = os.path.join(self.workdir, kernel_name) + kernel_file = self.scratch_file(kernel_name) self.vm.set_console() self.vm.add_args('-kernel', kernel_file, '-nographic') @@ -XXX,XX +XXX,XX @@ def test_ast1030_zephyros_1_07(self): kernel_name = "ast1030-evb-demo/zephyr.bin" with ZipFile(zip_file, 'r') as zf: zf.extract(kernel_name, path=self.workdir) - kernel_file = os.path.join(self.workdir, kernel_name) + kernel_file = self.scratch_file(kernel_name) self.vm.set_console() self.vm.add_args('-kernel', kernel_file, '-nographic') diff --git a/tests/functional/test_arm_aspeed_ast2500.py b/tests/functional/test_arm_aspeed_ast2500.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_aspeed_ast2500.py +++ b/tests/functional/test_arm_aspeed_ast2500.py @@ -XXX,XX +XXX,XX @@ def test_arm_ast2500_evb_sdk(self): archive_extract(image_path, self.workdir) self.do_test_arm_aspeed_sdk_start( - self.workdir + '/ast2500-default/image-bmc') + self.scratch_file("ast2500-default", "image-bmc")) self.wait_for_console_pattern('ast2500-default login:') diff --git a/tests/functional/test_arm_aspeed_ast2600.py b/tests/functional/test_arm_aspeed_ast2600.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_aspeed_ast2600.py +++ b/tests/functional/test_arm_aspeed_ast2600.py @@ -XXX,XX +XXX,XX @@ def test_arm_ast2600_evb_sdk(self): self.vm.add_args('-device', 'ds1338,bus=aspeed.i2c.bus.5,address=0x32'); self.do_test_arm_aspeed_sdk_start( - self.workdir + '/ast2600-a2/image-bmc') + self.scratch_file("ast2600-a2", "image-bmc")) self.wait_for_console_pattern('ast2600-a2 login:') diff --git a/tests/functional/test_arm_bflt.py b/tests/functional/test_arm_bflt.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bflt.py +++ b/tests/functional/test_arm_bflt.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os import bz2 from qemu_test import QemuUserTest, Asset @@ -XXX,XX +XXX,XX @@ class LoadBFLT(QemuUserTest): def test_stm32(self): # See https://elinux.org/STM32#User_Space rootfs_path_bz2 = self.ASSET_ROOTFS.fetch() - busybox_path = os.path.join(self.workdir, "bin/busybox") + busybox_path = self.scratch_file("bin", "busybox") with bz2.open(rootfs_path_bz2, 'rb') as cpio_handle: cpio_extract(cpio_handle, self.workdir) diff --git a/tests/functional/test_arm_bpim2u.py b/tests/functional/test_arm_bpim2u.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bpim2u.py +++ b/tests/functional/test_arm_bpim2u.py @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_initrd(self): 'sun8i-r40-bananapi-m2-ultra.dtb') dtb_path = self.extract_from_deb(deb_path, dtb_path) initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = os.path.join(self.workdir, 'rootfs.cpio') + initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_gmac(self): 'sun8i-r40-bananapi-m2-ultra.dtb') dtb_path = self.extract_from_deb(deb_path, dtb_path) rootfs_path_xz = self.ASSET_ROOTFS.fetch() - rootfs_path = os.path.join(self.workdir, 'rootfs.cpio') + rootfs_path = self.scratch_file('rootfs.cpio') lzma_uncompress(rootfs_path_xz, rootfs_path) image_pow2ceil_expand(rootfs_path) @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_openwrt_22_03_3(self): # This test download a 8.9 MiB compressed image and expand it # to 127 MiB. image_path_gz = self.ASSET_SD_IMAGE.fetch() - image_path = os.path.join(self.workdir, 'sdcard.img') + image_path = self.scratch_file('sdcard.img') gzip_uncompress(image_path_gz, image_path) image_pow2ceil_expand(image_path) diff --git a/tests/functional/test_arm_canona1100.py b/tests/functional/test_arm_canona1100.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_canona1100.py +++ b/tests/functional/test_arm_canona1100.py @@ -XXX,XX +XXX,XX @@ def test_arm_canona1100(self): member="day18/barebox.canon-a1100.bin") self.vm.set_console() self.vm.add_args('-bios', - self.workdir + '/day18/barebox.canon-a1100.bin') + self.scratch_file('day18', + 'barebox.canon-a1100.bin')) self.vm.launch() wait_for_console_pattern(self, 'running /env/bin/init') diff --git a/tests/functional/test_arm_cubieboard.py b/tests/functional/test_arm_cubieboard.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_cubieboard.py +++ b/tests/functional/test_arm_cubieboard.py @@ -XXX,XX +XXX,XX @@ def test_arm_cubieboard_initrd(self): dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun4i-a10-cubieboard.dtb' dtb_path = self.extract_from_deb(deb_path, dtb_path) initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = os.path.join(self.workdir, 'rootfs.cpio') + initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_cubieboard_sata(self): dtb_path = self.extract_from_deb(deb_path, dtb_path) rootfs_path_gz = self.ASSET_SATA_ROOTFS.fetch() - rootfs_path = os.path.join(self.workdir, 'rootfs.cpio') + rootfs_path = self.scratch_file('rootfs.cpio') gzip_uncompress(rootfs_path_gz, rootfs_path) self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_cubieboard_openwrt_22_03_2(self): # to 126 MiB. self.set_machine('cubieboard') image_path_gz = self.ASSET_OPENWRT.fetch() - image_path = os.path.join(self.workdir, 'sdcard.img') + image_path = self.scratch_file('sdcard.img') gzip_uncompress(image_path_gz, image_path) image_pow2ceil_expand(image_path) diff --git a/tests/functional/test_arm_emcraft_sf2.py b/tests/functional/test_arm_emcraft_sf2.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_emcraft_sf2.py +++ b/tests/functional/test_arm_emcraft_sf2.py @@ -XXX,XX +XXX,XX @@ def test_arm_emcraft_sf2(self): uboot_path = self.ASSET_UBOOT.fetch() spi_path = self.ASSET_SPI.fetch() - spi_path_rw = os.path.join(self.workdir, 'spi.bin') + spi_path_rw = self.scratch_file('spi.bin') shutil.copy(spi_path, spi_path_rw) os.chmod(spi_path_rw, 0o600) diff --git a/tests/functional/test_arm_integratorcp.py b/tests/functional/test_arm_integratorcp.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_integratorcp.py +++ b/tests/functional/test_arm_integratorcp.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os import logging from qemu_test import QemuSystemTest, Asset @@ -XXX,XX +XXX,XX @@ def test_framebuffer_tux_logo(self): import numpy as np import cv2 - screendump_path = os.path.join(self.workdir, "screendump.pbm") + screendump_path = self.scratch_file("screendump.pbm") tuxlogo_path = self.ASSET_TUXLOGO.fetch() self.boot_integratorcp() diff --git a/tests/functional/test_arm_orangepi.py b/tests/functional/test_arm_orangepi.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_orangepi.py +++ b/tests/functional/test_arm_orangepi.py @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_initrd(self): dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun8i-h3-orangepi-pc.dtb' dtb_path = self.extract_from_deb(deb_path, dtb_path) initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = os.path.join(self.workdir, 'rootfs.cpio') + initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_sd(self): dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun8i-h3-orangepi-pc.dtb' dtb_path = self.extract_from_deb(deb_path, dtb_path) rootfs_path_xz = self.ASSET_ROOTFS.fetch() - rootfs_path = os.path.join(self.workdir, 'rootfs.cpio') + rootfs_path = self.scratch_file('rootfs.cpio') lzma_uncompress(rootfs_path_xz, rootfs_path) image_pow2ceil_expand(rootfs_path) @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_armbian(self): # to 1036 MiB, but the underlying filesystem is 1552 MiB... # As we expand it to 2 GiB we are safe. image_path_xz = self.ASSET_ARMBIAN.fetch() - image_path = os.path.join(self.workdir, 'armbian.img') + image_path = self.scratch_file('armbian.img') lzma_uncompress(image_path_xz, image_path) image_pow2ceil_expand(image_path) @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_uboot_netbsd9(self): uboot_path = '/usr/lib/u-boot/orangepi_plus/u-boot-sunxi-with-spl.bin' uboot_path = self.extract_from_deb(deb_path, uboot_path) image_path_gz = self.ASSET_NETBSD.fetch() - image_path = os.path.join(self.workdir, 'armv7.img') + image_path = self.scratch_file('armv7.img') gzip_uncompress(image_path_gz, image_path) image_pow2ceil_expand(image_path) image_drive_args = 'if=sd,format=raw,snapshot=on,file=' + image_path diff --git a/tests/functional/test_arm_raspi2.py b/tests/functional/test_arm_raspi2.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_raspi2.py +++ b/tests/functional/test_arm_raspi2.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern from qemu_test.utils import gzip_uncompress @@ -XXX,XX +XXX,XX @@ def test_arm_raspi2_initrd(self): kernel_path = self.extract_from_deb(deb_path, '/boot/kernel7.img') dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2709-rpi-2-b.dtb') initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = os.path.join(self.workdir, 'rootfs.cpio') + initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) self.set_machine('raspi2b') diff --git a/tests/functional/test_arm_smdkc210.py b/tests/functional/test_arm_smdkc210.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_smdkc210.py +++ b/tests/functional/test_arm_smdkc210.py @@ -XXX,XX +XXX,XX @@ def test_arm_exynos4210_initrd(self): dtb_path = self.extract_from_deb(deb_path, dtb_path) initrd_path_gz = self.ASSET_ROOTFS.fetch() - initrd_path = os.path.join(self.workdir, 'rootfs.cpio') + initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) self.vm.set_console() diff --git a/tests/functional/test_arm_vexpress.py b/tests/functional/test_arm_vexpress.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_vexpress.py +++ b/tests/functional/test_arm_vexpress.py @@ -XXX,XX +XXX,XX @@ def test_arm_vexpressa9(self): self.set_machine('vexpress-a9') file_path = self.ASSET_DAY16.fetch() archive_extract(file_path, self.workdir) - self.launch_kernel(self.workdir + '/day16/winter.zImage', - dtb=self.workdir + '/day16/vexpress-v2p-ca9.dtb', + self.launch_kernel(self.scratch_file('day16', 'winter.zImage'), + dtb=self.scratch_file('day16', + 'vexpress-v2p-ca9.dtb'), wait_for='QEMU advent calendar') if __name__ == '__main__': diff --git a/tests/functional/test_m68k_mcf5208evb.py b/tests/functional/test_m68k_mcf5208evb.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_m68k_mcf5208evb.py +++ b/tests/functional/test_m68k_mcf5208evb.py @@ -XXX,XX +XXX,XX @@ def test_m68k_mcf5208evb(self): file_path = self.ASSET_DAY07.fetch() archive_extract(file_path, self.workdir) self.vm.set_console() - self.vm.add_args('-kernel', self.workdir + '/day07/sanity-clause.elf') + self.vm.add_args('-kernel', + self.scratch_file('day07', 'sanity-clause.elf')) self.vm.launch() self.wait_for_console_pattern('QEMU advent calendar') diff --git a/tests/functional/test_m68k_nextcube.py b/tests/functional/test_m68k_nextcube.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_m68k_nextcube.py +++ b/tests/functional/test_m68k_nextcube.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import os import time from qemu_test import QemuSystemTest, Asset @@ -XXX,XX +XXX,XX @@ def check_bootrom_framebuffer(self, screenshot_path): @skipIfMissingImports("PIL") def test_bootrom_framebuffer_size(self): self.set_machine('next-cube') - screenshot_path = os.path.join(self.workdir, "dump.ppm") + screenshot_path = self.scratch_file("dump.ppm") self.check_bootrom_framebuffer(screenshot_path) from PIL import Image @@ -XXX,XX +XXX,XX @@ def test_bootrom_framebuffer_size(self): @skipIfMissingCommands('tesseract') def test_bootrom_framebuffer_ocr_with_tesseract(self): self.set_machine('next-cube') - screenshot_path = os.path.join(self.workdir, "dump.ppm") + screenshot_path = self.scratch_file("dump.ppm") self.check_bootrom_framebuffer(screenshot_path) lines = tesseract_ocr(screenshot_path) text = '\n'.join(lines) diff --git a/tests/functional/test_microblaze_s3adsp1800.py b/tests/functional/test_microblaze_s3adsp1800.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_microblaze_s3adsp1800.py +++ b/tests/functional/test_microblaze_s3adsp1800.py @@ -XXX,XX +XXX,XX @@ def test_microblaze_s3adsp1800(self): file_path = self.ASSET_IMAGE.fetch() archive_extract(file_path, self.workdir) self.vm.set_console() - self.vm.add_args('-kernel', self.workdir + '/day17/ballerina.bin') + self.vm.add_args('-kernel', + self.scratch_file('day17', 'ballerina.bin')) self.vm.launch() wait_for_console_pattern(self, 'This architecture does not have ' 'kernel memory protection') diff --git a/tests/functional/test_microblazeel_s3adsp1800.py b/tests/functional/test_microblazeel_s3adsp1800.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_microblazeel_s3adsp1800.py +++ b/tests/functional/test_microblazeel_s3adsp1800.py @@ -XXX,XX +XXX,XX @@ def test_microblazeel_s3adsp1800(self): file_path = self.ASSET_IMAGE.fetch() archive_extract(file_path, self.workdir) self.vm.set_console() - self.vm.add_args('-kernel', self.workdir + '/day13/xmaton.bin') - self.vm.add_args('-nic', 'user,tftp=' + self.workdir + '/day13/') + self.vm.add_args('-kernel', self.scratch_file('day13', 'xmaton.bin')) + tftproot = self.scratch_file('day13') + self.vm.add_args('-nic', f'user,tftp={tftproot}') self.vm.launch() wait_for_console_pattern(self, 'QEMU Advent Calendar 2023') time.sleep(0.1) diff --git a/tests/functional/test_mips64el_malta.py b/tests/functional/test_mips64el_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_malta.py +++ b/tests/functional/test_mips64el_malta.py @@ -XXX,XX +XXX,XX @@ def test_mips64el_malta(self): def test_mips64el_malta_5KEc_cpio(self): kernel_path = self.ASSET_KERNEL_3_19_3.fetch() initrd_path_gz = self.ASSET_CPIO_R1.fetch() - initrd_path = os.path.join(self.workdir, 'rootfs.cpio') + initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) self.set_machine('malta') @@ -XXX,XX +XXX,XX @@ def do_test_i6400_framebuffer_logo(self, cpu_cores_count): import numpy as np import cv2 - screendump_path = os.path.join(self.workdir, 'screendump.pbm') + screendump_path = self.scratch_file('screendump.pbm') kernel_path_gz = self.ASSET_KERNEL_4_7_0.fetch() - kernel_path = self.workdir + "/vmlinux" + kernel_path = self.scratch_file("vmlinux") gzip_uncompress(kernel_path_gz, kernel_path) tuxlogo_path = self.ASSET_TUXLOGO.fetch() diff --git a/tests/functional/test_mips_malta.py b/tests/functional/test_mips_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips_malta.py +++ b/tests/functional/test_mips_malta.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern from qemu_test.utils import gzip_uncompress @@ -XXX,XX +XXX,XX @@ def test_mips_malta_cpio(self): kernel_path = self.extract_from_deb(deb_path, '/boot/vmlinux-4.5.0-2-4kc-malta') initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = os.path.join(self.workdir, 'rootfs.cpio') + initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) self.set_machine('malta') diff --git a/tests/functional/test_mipsel_malta.py b/tests/functional/test_mipsel_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mipsel_malta.py +++ b/tests/functional/test_mipsel_malta.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os - from qemu_test import QemuSystemTest, LinuxKernelTest, Asset from qemu_test import interrupt_interactive_console_until_pattern from qemu_test import wait_for_console_pattern @@ -XXX,XX +XXX,XX @@ class MaltaMachineConsole(LinuxKernelTest): 'ce21ff4b07a981ecb8a39db2876616f5a2473eb2ab459c6f67465b9914b0c6b6') def do_test_mips_malta32el_nanomips(self, kernel_path_xz): - kernel_path = os.path.join(self.workdir, 'kernel') + kernel_path = self.scratch_file('kernel') lzma_uncompress(kernel_path_xz, kernel_path) self.set_machine('malta') @@ -XXX,XX +XXX,XX @@ def test_mipsel_malta_yamon(self): zip_path = self.ASSET_YAMON_ROM.fetch() with ZipFile(zip_path, 'r') as zf: zf.extract(yamon_bin, path=self.workdir) - yamon_path = os.path.join(self.workdir, yamon_bin) + yamon_path = self.scratch_file(yamon_bin) self.set_machine('malta') self.vm.set_console() diff --git a/tests/functional/test_or1k_sim.py b/tests/functional/test_or1k_sim.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_or1k_sim.py +++ b/tests/functional/test_or1k_sim.py @@ -XXX,XX +XXX,XX @@ def test_or1k_sim(self): file_path = self.ASSET_DAY20.fetch() archive_extract(file_path, self.workdir) self.vm.set_console() - self.vm.add_args('-kernel', self.workdir + '/day20/vmlinux') + self.vm.add_args('-kernel', self.scratch_file('day20', 'vmlinux')) self.vm.launch() self.wait_for_console_pattern('QEMU advent calendar') diff --git a/tests/functional/test_ppc64_e500.py b/tests/functional/test_ppc64_e500.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc64_e500.py +++ b/tests/functional/test_ppc64_e500.py @@ -XXX,XX +XXX,XX @@ def test_ppc64_e500(self): self.cpu = 'e5500' file_path = self.ASSET_DAY19.fetch() archive_extract(file_path, self.workdir) - self.launch_kernel(self.workdir + '/day19/uImage', + self.launch_kernel(self.scratch_file('day19', 'uImage'), wait_for='QEMU advent calendar') if __name__ == '__main__': diff --git a/tests/functional/test_ppc64_hv.py b/tests/functional/test_ppc64_hv.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc64_hv.py +++ b/tests/functional/test_ppc64_hv.py @@ -XXX,XX +XXX,XX @@ def extract_from_iso(self, iso, path): :param path: path within the iso file of the file to be extracted :returns: path of the extracted file """ - filename = os.path.basename(path) - - cwd = os.getcwd() - os.chdir(self.workdir) + filename = self.scratch_file(os.path.basename(path)) cmd = "xorriso -osirrox on -indev %s -cpx %s %s" % (iso, path, filename) subprocess.run(cmd.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) os.chmod(filename, 0o600) - os.chdir(cwd) - # Return complete path to extracted file. Because callers to - # extract_from_iso() specify 'path' with a leading slash, it is - # necessary to use os.path.relpath() as otherwise os.path.join() - # interprets it as an absolute path and drops the self.workdir part. - return os.path.normpath(os.path.join(self.workdir, filename)) + return filename def setUp(self): super().setUp() diff --git a/tests/functional/test_ppc_amiga.py b/tests/functional/test_ppc_amiga.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_amiga.py +++ b/tests/functional/test_ppc_amiga.py @@ -XXX,XX +XXX,XX @@ def test_ppc_amigaone(self): zip_file = self.ASSET_IMAGE.fetch() with ZipFile(zip_file, 'r') as zf: zf.extractall(path=self.workdir) - bios_fh = open(self.workdir + "/u-boot-amigaone.bin", "wb") - subprocess.run(['tail', '-c', '524288', - self.workdir + "/floppy_edition/updater.image"], - stdout=bios_fh) + bios = self.scratch_file("u-boot-amigaone.bin") + with open(bios, "wb") as bios_fh: + subprocess.run(['tail', '-c', '524288', + self.scratch_file("floppy_edition", + "updater.image")], + stdout=bios_fh) self.vm.set_console() - self.vm.add_args('-bios', self.workdir + '/u-boot-amigaone.bin') + self.vm.add_args('-bios', bios) self.vm.launch() wait_for_console_pattern(self, 'FLASH:') diff --git a/tests/functional/test_ppc_bamboo.py b/tests/functional/test_ppc_bamboo.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_bamboo.py +++ b/tests/functional/test_ppc_bamboo.py @@ -XXX,XX +XXX,XX @@ def test_ppc_bamboo(self): file_path = self.ASSET_IMAGE.fetch() archive_extract(file_path, self.workdir) self.vm.set_console() - self.vm.add_args('-kernel', self.workdir + - '/system-image-powerpc-440fp/linux', - '-initrd', self.workdir + - '/system-image-powerpc-440fp/rootfs.cpio.gz', + self.vm.add_args('-kernel', + self.scratch_file('system-image-powerpc-440fp', + 'linux'), + '-initrd', + self.scratch_file('system-image-powerpc-440fp', + 'rootfs.cpio.gz'), '-nic', 'user,model=rtl8139,restrict=on') self.vm.launch() wait_for_console_pattern(self, 'Type exit when done') diff --git a/tests/functional/test_ppc_mac.py b/tests/functional/test_ppc_mac.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_mac.py +++ b/tests/functional/test_ppc_mac.py @@ -XXX,XX +XXX,XX @@ def do_day15_test(self): file_path = self.ASSET_DAY15.fetch() archive_extract(file_path, self.workdir) self.vm.add_args('-M', 'graphics=off') - self.launch_kernel(self.workdir + '/day15/invaders.elf', + self.launch_kernel(self.scratch_file('day15', 'invaders.elf'), wait_for='QEMU advent calendar') def test_ppc_g3beige(self): diff --git a/tests/functional/test_ppc_mpc8544ds.py b/tests/functional/test_ppc_mpc8544ds.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_mpc8544ds.py +++ b/tests/functional/test_ppc_mpc8544ds.py @@ -XXX,XX +XXX,XX @@ def test_ppc_mpc8544ds(self): file_path = self.ASSET_IMAGE.fetch() archive_extract(file_path, self.workdir, member='creek/creek.bin') self.vm.set_console() - self.vm.add_args('-kernel', self.workdir + '/creek/creek.bin') + self.vm.add_args('-kernel', self.scratch_file('creek', 'creek.bin')) self.vm.launch() wait_for_console_pattern(self, 'QEMU advent calendar 2020', self.panic_message) diff --git a/tests/functional/test_ppc_virtex_ml507.py b/tests/functional/test_ppc_virtex_ml507.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_virtex_ml507.py +++ b/tests/functional/test_ppc_virtex_ml507.py @@ -XXX,XX +XXX,XX @@ def test_ppc_virtex_ml507(self): file_path = self.ASSET_IMAGE.fetch() archive_extract(file_path, self.workdir) self.vm.set_console() - self.vm.add_args('-kernel', self.workdir + '/hippo/hippo.linux', - '-dtb', self.workdir + '/hippo/virtex440-ml507.dtb', + self.vm.add_args('-kernel', self.scratch_file('hippo', 'hippo.linux'), + '-dtb', self.scratch_file('hippo', + 'virtex440-ml507.dtb'), '-m', '512') self.vm.launch() wait_for_console_pattern(self, 'QEMU advent calendar 2020', diff --git a/tests/functional/test_rx_gdbsim.py b/tests/functional/test_rx_gdbsim.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_rx_gdbsim.py +++ b/tests/functional/test_rx_gdbsim.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import os - from qemu_test import QemuSystemTest, Asset from qemu_test import exec_command_and_wait_for_pattern from qemu_test import wait_for_console_pattern, skipFlakyTest @@ -XXX,XX +XXX,XX @@ def test_uboot(self): self.set_machine('gdbsim-r5f562n8') uboot_path_gz = self.ASSET_UBOOT.fetch() - uboot_path = os.path.join(self.workdir, 'u-boot.bin') + uboot_path = self.scratch_file('u-boot.bin') gzip_uncompress(uboot_path_gz, uboot_path) self.vm.set_console() diff --git a/tests/functional/test_s390x_ccw_virtio.py b/tests/functional/test_s390x_ccw_virtio.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_s390x_ccw_virtio.py +++ b/tests/functional/test_s390x_ccw_virtio.py @@ -XXX,XX +XXX,XX @@ def test_s390x_fedora(self): kernel_path = self.ASSET_F31_KERNEL.fetch() initrd_path_xz = self.ASSET_F31_INITRD.fetch() - initrd_path = os.path.join(self.workdir, 'initrd-raw.img') + initrd_path = self.scratch_file('initrd-raw.img') lzma_uncompress(initrd_path_xz, initrd_path) self.vm.set_console() diff --git a/tests/functional/test_s390x_topology.py b/tests/functional/test_s390x_topology.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_s390x_topology.py +++ b/tests/functional/test_s390x_topology.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import os - from qemu_test import QemuSystemTest, Asset from qemu_test import exec_command from qemu_test import exec_command_and_wait_for_pattern @@ -XXX,XX +XXX,XX @@ def kernel_init(self): self.require_accelerator("kvm") kernel_path = self.ASSET_F35_KERNEL.fetch() initrd_path_xz = self.ASSET_F35_INITRD.fetch() - initrd_path = os.path.join(self.workdir, 'initrd-raw.img') + initrd_path = self.scratch_file('initrd-raw.img') lzma_uncompress(initrd_path_xz, initrd_path) self.vm.set_console() diff --git a/tests/functional/test_sh4_r2d.py b/tests/functional/test_sh4_r2d.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sh4_r2d.py +++ b/tests/functional/test_sh4_r2d.py @@ -XXX,XX +XXX,XX @@ def test_r2d(self): file_path = self.ASSET_DAY09.fetch() archive_extract(file_path, self.workdir) self.vm.add_args('-append', 'console=ttySC1') - self.launch_kernel(self.workdir + '/day09/zImage', console_index=1, + self.launch_kernel(self.scratch_file('day09', 'zImage'), + console_index=1, wait_for='QEMU advent calendar') if __name__ == '__main__': diff --git a/tests/functional/test_sh4eb_r2d.py b/tests/functional/test_sh4eb_r2d.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sh4eb_r2d.py +++ b/tests/functional/test_sh4eb_r2d.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import os import shutil from qemu_test import LinuxKernelTest, Asset @@ -XXX,XX +XXX,XX @@ def test_sh4eb_r2d(self): file_path = self.ASSET_TGZ.fetch() archive_extract(file_path, self.workdir) self.vm.add_args('-append', 'console=ttySC1 noiotrap') - self.launch_kernel(os.path.join(self.workdir, 'sh4eb/linux-kernel'), - initrd=os.path.join(self.workdir, 'sh4eb/initramfs.cpio.gz'), + self.launch_kernel(self.scratch_file('sh4eb', 'linux-kernel'), + initrd=self.scratch_file('sh4eb', + 'initramfs.cpio.gz'), console_index=1, wait_for='Type exit when done') exec_command_and_wait_for_pattern(self, 'exit', 'Restarting system') - shutil.rmtree(os.path.join(self.workdir, 'sh4eb')) + shutil.rmtree(self.scratch_file('sh4eb')) if __name__ == '__main__': LinuxKernelTest.main() diff --git a/tests/functional/test_sparc64_sun4u.py b/tests/functional/test_sparc64_sun4u.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sparc64_sun4u.py +++ b/tests/functional/test_sparc64_sun4u.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import os - from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern from qemu_test.utils import archive_extract + class Sun4uMachine(QemuSystemTest): """Boots the Linux kernel and checks that the console is operational""" @@ -XXX,XX +XXX,XX @@ def test_sparc64_sun4u(self): kernel_name = 'day23/vmlinux' archive_extract(file_path, self.workdir, kernel_name) self.vm.set_console() - self.vm.add_args('-kernel', os.path.join(self.workdir, kernel_name), + self.vm.add_args('-kernel', self.scratch_file(kernel_name), '-append', 'printk.time=0') self.vm.launch() wait_for_console_pattern(self, 'Starting logging: OK') diff --git a/tests/functional/test_sparc_sun4m.py b/tests/functional/test_sparc_sun4m.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sparc_sun4m.py +++ b/tests/functional/test_sparc_sun4m.py @@ -XXX,XX +XXX,XX @@ def test_sparc_ss20(self): self.set_machine('SS-20') file_path = self.ASSET_DAY11.fetch() archive_extract(file_path, self.workdir) - self.launch_kernel(self.workdir + '/day11/zImage.elf', + self.launch_kernel(self.scratch_file('day11', 'zImage.elf'), wait_for='QEMU advent calendar') if __name__ == '__main__': diff --git a/tests/functional/test_xtensa_lx60.py b/tests/functional/test_xtensa_lx60.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_xtensa_lx60.py +++ b/tests/functional/test_xtensa_lx60.py @@ -XXX,XX +XXX,XX @@ def test_xtensa_lx60(self): self.cpu = 'dc233c' file_path = self.ASSET_DAY02.fetch() archive_extract(file_path, self.workdir) - self.launch_kernel(self.workdir + '/day02/santas-sleigh-ride.elf', + self.launch_kernel(self.scratch_file('day02', + 'santas-sleigh-ride.elf'), wait_for='QEMU advent calendar') if __name__ == '__main__': -- 2.46.0 Everything in the scratch directory is automatically purged. Calling 'rmtree' again breaks the ability to optionally preserve the scratch directory contents. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/test_sh4eb_r2d.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/functional/test_sh4eb_r2d.py b/tests/functional/test_sh4eb_r2d.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sh4eb_r2d.py +++ b/tests/functional/test_sh4eb_r2d.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import shutil - from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern from qemu_test.utils import archive_extract @@ -XXX,XX +XXX,XX @@ def test_sh4eb_r2d(self): 'initramfs.cpio.gz'), console_index=1, wait_for='Type exit when done') exec_command_and_wait_for_pattern(self, 'exit', 'Restarting system') - shutil.rmtree(self.scratch_file('sh4eb')) if __name__ == '__main__': LinuxKernelTest.main() -- 2.46.0 More archive related code will be added shortly, so having a separate file makes more sense. The utils.py imports the functions from archive.py, so that existing callers don't need to be modified. This avoids redundant code churn until later in the series when all calls will be adapted for other reasons. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/archive.py | 31 +++++++++++++++++++++++++++ tests/functional/qemu_test/utils.py | 23 +++----------------- 2 files changed, 34 insertions(+), 20 deletions(-) create mode 100644 tests/functional/qemu_test/archive.py diff --git a/tests/functional/qemu_test/archive.py b/tests/functional/qemu_test/archive.py new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/functional/qemu_test/archive.py @@ -XXX,XX +XXX,XX @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Utilities for python-based QEMU tests +# +# Copyright 2024 Red Hat, Inc. +# +# Authors: +# Thomas Huth <thuth@redhat.com> + +import os +import subprocess +import tarfile + + +def tar_extract(archive, dest_dir, member=None): + with tarfile.open(archive) as tf: + if hasattr(tarfile, 'data_filter'): + tf.extraction_filter = getattr(tarfile, 'data_filter', + (lambda member, path: member)) + if member: + tf.extract(member=member, path=dest_dir) + else: + tf.extractall(path=dest_dir) + +def cpio_extract(cpio_handle, output_path): + cwd = os.getcwd() + os.chdir(output_path) + subprocess.run(['cpio', '-i'], + input=cpio_handle.read(), + stderr=subprocess.DEVNULL) + os.chdir(cwd) diff --git a/tests/functional/qemu_test/utils.py b/tests/functional/qemu_test/utils.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/utils.py +++ b/tests/functional/qemu_test/utils.py @@ -XXX,XX +XXX,XX @@ import lzma import os import shutil -import subprocess -import tarfile + +from .archive import tar_extract as archive_extract +from .archive import cpio_extract """ Round up to next power of 2 @@ -XXX,XX +XXX,XX @@ def image_pow2ceil_expand(path): with open(path, 'ab+') as fd: fd.truncate(size_aligned) -def archive_extract(archive, dest_dir, member=None): - with tarfile.open(archive) as tf: - if hasattr(tarfile, 'data_filter'): - tf.extraction_filter = getattr(tarfile, 'data_filter', - (lambda member, path: member)) - if member: - tf.extract(member=member, path=dest_dir) - else: - tf.extractall(path=dest_dir) - def gzip_uncompress(gz_path, output_path): if os.path.exists(output_path): return @@ -XXX,XX +XXX,XX @@ def lzma_uncompress(xz_path, output_path): except: os.remove(output_path) raise - -def cpio_extract(cpio_handle, output_path): - cwd = os.getcwd() - os.chdir(output_path) - subprocess.run(['cpio', '-i'], - input=cpio_handle.read(), - stderr=subprocess.DEVNULL) - os.chdir(cwd) -- 2.46.0 More uncompress related code will be added shortly, so having a separate file makes more sense. The utils.py imports the functions from archive.py, so that existing callers don't need to be modified. This avoids redundant code churn until later in the series when all calls will be adapted for other reasons. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/uncompress.py | 36 ++++++++++++++++++++++++ tests/functional/qemu_test/utils.py | 27 ++---------------- 2 files changed, 38 insertions(+), 25 deletions(-) create mode 100644 tests/functional/qemu_test/uncompress.py diff --git a/tests/functional/qemu_test/uncompress.py b/tests/functional/qemu_test/uncompress.py new file mode 100644 index XXXXXXX..XXXXXXX --- /dev/null +++ b/tests/functional/qemu_test/uncompress.py @@ -XXX,XX +XXX,XX @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Utilities for python-based QEMU tests +# +# Copyright 2024 Red Hat, Inc. +# +# Authors: +# Thomas Huth <thuth@redhat.com> + +import gzip +import lzma +import os +import shutil + + +def gzip_uncompress(gz_path, output_path): + if os.path.exists(output_path): + return + with gzip.open(gz_path, 'rb') as gz_in: + try: + with open(output_path, 'wb') as raw_out: + shutil.copyfileobj(gz_in, raw_out) + except: + os.remove(output_path) + raise + +def lzma_uncompress(xz_path, output_path): + if os.path.exists(output_path): + return + with lzma.open(xz_path, 'rb') as lzma_in: + try: + with open(output_path, 'wb') as raw_out: + shutil.copyfileobj(lzma_in, raw_out) + except: + os.remove(output_path) + raise diff --git a/tests/functional/qemu_test/utils.py b/tests/functional/qemu_test/utils.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/utils.py +++ b/tests/functional/qemu_test/utils.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import gzip -import lzma import os -import shutil from .archive import tar_extract as archive_extract from .archive import cpio_extract +from .uncompress import gzip_uncompress +from .uncompress import lzma_uncompress """ Round up to next power of 2 @@ -XXX,XX +XXX,XX @@ def image_pow2ceil_expand(path): if size != size_aligned: with open(path, 'ab+') as fd: fd.truncate(size_aligned) - -def gzip_uncompress(gz_path, output_path): - if os.path.exists(output_path): - return - with gzip.open(gz_path, 'rb') as gz_in: - try: - with open(output_path, 'wb') as raw_out: - shutil.copyfileobj(gz_in, raw_out) - except: - os.remove(output_path) - raise - -def lzma_uncompress(xz_path, output_path): - if os.path.exists(output_path): - return - with lzma.open(xz_path, 'rb') as lzma_in: - try: - with open(output_path, 'wb') as raw_out: - shutil.copyfileobj(lzma_in, raw_out) - except: - os.remove(output_path) - raise -- 2.46.0 This mirrors the existing archive_extract and cpio_extract helpers Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/archive.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/functional/qemu_test/archive.py b/tests/functional/qemu_test/archive.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/archive.py +++ b/tests/functional/qemu_test/archive.py @@ -XXX,XX +XXX,XX @@ import os import subprocess import tarfile +import zipfile def tar_extract(archive, dest_dir, member=None): @@ -XXX,XX +XXX,XX @@ def cpio_extract(cpio_handle, output_path): input=cpio_handle.read(), stderr=subprocess.DEVNULL) os.chdir(cwd) + +def zip_extract(archive, dest_dir, member=None): + with zipfile.ZipFile(archive, 'r') as zf: + if member: + zf.extract(member=member, path=dest_dir) + else: + zf.extractall(path=dest_dir) -- 2.46.0 This mirrors the existing archive_extract, cpio_extract and zip_extract helpers Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/archive.py | 13 +++++++++++++ tests/functional/qemu_test/linuxkernel.py | 13 ++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/tests/functional/qemu_test/archive.py b/tests/functional/qemu_test/archive.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/archive.py +++ b/tests/functional/qemu_test/archive.py @@ -XXX,XX +XXX,XX @@ import tarfile import zipfile +from .cmd import run_cmd + def tar_extract(archive, dest_dir, member=None): with tarfile.open(archive) as tf: @@ -XXX,XX +XXX,XX @@ def zip_extract(archive, dest_dir, member=None): zf.extract(member=member, path=dest_dir) else: zf.extractall(path=dest_dir) + +def deb_extract(archive, dest_dir, member=None): + cwd = os.getcwd() + os.chdir(dest_dir) + try: + (stdout, stderr, ret) = run_cmd(['ar', 't', archive]) + file_path = stdout.split()[2] + run_cmd(['ar', 'x', archive, file_path]) + tar_extract(file_path, dest_dir, member) + finally: + os.chdir(cwd) diff --git a/tests/functional/qemu_test/linuxkernel.py b/tests/functional/qemu_test/linuxkernel.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/linuxkernel.py +++ b/tests/functional/qemu_test/linuxkernel.py @@ -XXX,XX +XXX,XX @@ import os from .testcase import QemuSystemTest -from .cmd import run_cmd, wait_for_console_pattern -from .utils import archive_extract +from .cmd import wait_for_console_pattern +from .archive import deb_extract + class LinuxKernelTest(QemuSystemTest): KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 ' @@ -XXX,XX +XXX,XX @@ def extract_from_deb(self, deb_path, path): :param path: path within the deb archive of the file to be extracted :returns: path of the extracted file """ - cwd = os.getcwd() - os.chdir(self.workdir) - (stdout, stderr, ret) = run_cmd(['ar', 't', deb_path]) - file_path = stdout.split()[2] - run_cmd(['ar', 'x', deb_path, file_path]) - archive_extract(file_path, self.workdir) - os.chdir(cwd) + deb_extract(deb_path, self.workdir, member="." + path) # Return complete path to extracted file. Because callers to # extract_from_deb() specify 'path' with a leading slash, it is # necessary to use os.path.relpath() as otherwise scratch_file() -- 2.46.0 Currently cpio_extract differs from tar_extract/zip_extract in that it only allows a file-like object as input. Adapt it to also support filenames. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/archive.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/functional/qemu_test/archive.py b/tests/functional/qemu_test/archive.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/archive.py +++ b/tests/functional/qemu_test/archive.py @@ -XXX,XX +XXX,XX @@ # Thomas Huth <thuth@redhat.com> import os -import subprocess +from subprocess import check_call, run, DEVNULL import tarfile import zipfile @@ -XXX,XX +XXX,XX @@ def tar_extract(archive, dest_dir, member=None): else: tf.extractall(path=dest_dir) -def cpio_extract(cpio_handle, output_path): +def cpio_extract(archive, output_path): cwd = os.getcwd() os.chdir(output_path) - subprocess.run(['cpio', '-i'], - input=cpio_handle.read(), - stderr=subprocess.DEVNULL) + # Not passing 'check=True' as cpio exits with non-zero + # status if the archive contains any device nodes :-( + if type(archive) == str: + run(['cpio', '-i', '-F', archive], + stdout=DEVNULL, stderr=DEVNULL) + else: + run(['cpio', '-i'], + input=archive.read(), + stdout=DEVNULL, stderr=DEVNULL) os.chdir(cwd) def zip_extract(archive, dest_dir, member=None): -- 2.46.0 There are many types of archives that the tests deal with. Provide a generalized 'archive_extract' that can detect the format and delegate to the appropriate helper for extraction. This ensures that all archive extraction code follows the same design pattern. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/__init__.py | 1 + tests/functional/qemu_test/archive.py | 58 ++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/tests/functional/qemu_test/__init__.py b/tests/functional/qemu_test/__init__.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/__init__.py +++ b/tests/functional/qemu_test/__init__.py @@ -XXX,XX +XXX,XX @@ from .decorators import skipIfMissingCommands, skipIfNotMachine, \ skipFlakyTest, skipUntrustedTest, skipBigDataTest, \ skipIfMissingImports +from .archive import archive_extract diff --git a/tests/functional/qemu_test/archive.py b/tests/functional/qemu_test/archive.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/archive.py +++ b/tests/functional/qemu_test/archive.py @@ -XXX,XX +XXX,XX @@ import os from subprocess import check_call, run, DEVNULL import tarfile +from urllib.parse import urlparse import zipfile +from .asset import Asset from .cmd import run_cmd @@ -XXX,XX +XXX,XX @@ def deb_extract(archive, dest_dir, member=None): tar_extract(file_path, dest_dir, member) finally: os.chdir(cwd) + +''' +@params archive: filename, Asset, or file-like object to extract +@params dest_dir: target directory to extract into +@params member: optional member file to limit extraction to + +Extracts @archive into @dest_dir. All files are extracted +unless @member specifies a limit. + +If @format is None, heuristics will be applied to guess the format +from the filename or Asset URL. @format must be non-None if @archive +is a file-like object. +''' +def archive_extract(archive, dest_dir, format=None, member=None): + if format is None: + format = guess_archive_format(archive) + if type(archive) == Asset: + archive = str(archive) + + if format == "tar": + tar_extract(archive, dest_dir, member) + elif format == "zip": + zip_extract(archive, dest_dir, member) + elif format == "cpio": + if member is not None: + raise Exception("Unable to filter cpio extraction") + cpio_extract(archive, dest_dir) + elif format == "deb": + if type(archive) != str: + raise Exception("Unable to use file-like object with deb archives") + deb_extract(archive, dest_dir, "./" + member) + else: + raise Exception(f"Unknown archive format {format}") + +''' +@params archive: filename, or Asset to guess + +Guess the format of @compressed, raising an exception if +no format can be determined +''' +def guess_archive_format(archive): + if type(archive) == Asset: + archive = urlparse(archive.url).path + elif type(archive) != str: + raise Exception(f"Unable to guess archive format for {archive}") + + if ".tar." in archive or archive.endswith("tgz"): + return "tar" + elif archive.endswith(".zip"): + return "zip" + elif archive.endswith(".cpio"): + return "cpio" + elif archive.endswith(".deb") or archive.endswith(".udeb"): + return "deb" + else: + raise Exception(f"Unknown archive format for {archive}") -- 2.46.0 This helper wrappers archive.archive_extract, forcing the use of the scratch directory, to ensure any extracted files are cleaned at test termination. If a specific member is requested, then the path to the extracted file is also returned. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/testcase.py | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -XXX,XX +XXX,XX @@ from qemu.machine import QEMUMachine from qemu.utils import kvm_available, tcg_available +from .archive import archive_extract from .asset import Asset from .cmd import run_cmd from .config import BUILD_DIR @@ -XXX,XX +XXX,XX @@ class QemuBaseTest(unittest.TestCase): log = None logdir = None + ''' + @params archive: filename, Asset, or file-like object to extract + @params format: optional archive format (tar, zip, deb, cpio) + @params sub_dir: optional sub-directory to extract into + @params member: optional member file to limit extraction to + + Extracts @archive into the scratch directory, or a directory beneath + named by @sub_dir. All files are extracted unless @member specifies + a limit. + + If @format is None, heuristics will be applied to guess the format + from the filename or Asset URL. @format must be non-None if @archive + is a file-like object. + + If @member is non-None, returns the fully qualified path to @member + ''' + def archive_extract(self, archive, format=None, sub_dir=None, member=None): + self.log.debug(f"Extract {archive} format={format}" + + f"sub_dir={sub_dir} member={member}") + if type(archive) == Asset: + archive.fetch() + if sub_dir is None: + archive_extract(archive, self.scratch_file(), format, member) + else: + archive_extract(archive, self.scratch_file(sub_dir), + format, member) + + if member is not None: + return self.scratch_file(member) + return None + ''' Create a temporary directory suitable for storing UNIX socket paths. -- 2.46.0 Replace use of utils.archive_extract and extract_from_deb with the new archive_extract helper. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/linuxkernel.py | 19 ---------- tests/functional/test_aarch64_aspeed.py | 5 ++- tests/functional/test_aarch64_raspi3.py | 8 +---- tests/functional/test_aarch64_raspi4.py | 14 ++++---- tests/functional/test_acpi_bits.py | 15 ++------ tests/functional/test_arm_aspeed_ast1030.py | 16 +++------ tests/functional/test_arm_aspeed_ast2500.py | 6 ++-- tests/functional/test_arm_aspeed_ast2600.py | 5 +-- tests/functional/test_arm_aspeed_rainier.py | 11 +++--- tests/functional/test_arm_bflt.py | 3 +- tests/functional/test_arm_bpim2u.py | 26 +++++++------- tests/functional/test_arm_canona1100.py | 11 +++--- tests/functional/test_arm_cubieboard.py | 20 +++++------ tests/functional/test_arm_orangepi.py | 35 +++++++++---------- tests/functional/test_arm_raspi2.py | 14 ++++---- tests/functional/test_arm_smdkc210.py | 9 +++-- tests/functional/test_arm_vexpress.py | 5 ++- tests/functional/test_m68k_mcf5208evb.py | 5 ++- tests/functional/test_m68k_q800.py | 5 ++- .../functional/test_microblaze_s3adsp1800.py | 5 ++- .../test_microblazeel_s3adsp1800.py | 5 ++- tests/functional/test_mips64el_fuloong2e.py | 6 ++-- tests/functional/test_mips64el_malta.py | 6 ++-- tests/functional/test_mips_malta.py | 12 +++---- tests/functional/test_mipsel_malta.py | 5 +-- tests/functional/test_or1k_sim.py | 5 ++- tests/functional/test_ppc64_e500.py | 5 ++- tests/functional/test_ppc_amiga.py | 6 ++-- tests/functional/test_ppc_bamboo.py | 5 ++- tests/functional/test_ppc_mac.py | 6 ++-- tests/functional/test_ppc_mpc8544ds.py | 8 ++--- tests/functional/test_ppc_virtex_ml507.py | 5 ++- tests/functional/test_sh4_r2d.py | 5 ++- tests/functional/test_sh4eb_r2d.py | 5 ++- tests/functional/test_sparc64_sun4u.py | 8 ++--- tests/functional/test_sparc_sun4m.py | 5 ++- tests/functional/test_xtensa_lx60.py | 5 ++- 37 files changed, 134 insertions(+), 205 deletions(-) diff --git a/tests/functional/qemu_test/linuxkernel.py b/tests/functional/qemu_test/linuxkernel.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/linuxkernel.py +++ b/tests/functional/qemu_test/linuxkernel.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -import os - from .testcase import QemuSystemTest from .cmd import wait_for_console_pattern -from .archive import deb_extract class LinuxKernelTest(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def launch_kernel(self, kernel, initrd=None, dtb=None, console_index=0, self.vm.launch() if wait_for: self.wait_for_console_pattern(wait_for) - - def extract_from_deb(self, deb_path, path): - """ - Extracts a file from a deb package into the test workdir - - :param deb_path: path to the deb archive - :param path: path within the deb archive of the file to be extracted - :returns: path of the extracted file - """ - deb_extract(deb_path, self.workdir, member="." + path) - # Return complete path to extracted file. Because callers to - # extract_from_deb() specify 'path' with a leading slash, it is - # necessary to use os.path.relpath() as otherwise scratch_file() - # interprets it as an absolute path and drops the required prefix - return os.path.normpath(self.scratch_file(os.path.relpath(path, '/'))) - diff --git a/tests/functional/test_aarch64_aspeed.py b/tests/functional/test_aarch64_aspeed.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_aspeed.py +++ b/tests/functional/test_aarch64_aspeed.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern from qemu_test import exec_command_and_wait_for_pattern -from qemu_test.utils import archive_extract + class AST2x00MachineSDK(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def do_test_aarch64_aspeed_sdk_start(self, image): def test_aarch64_ast2700_evb_sdk_v09_02(self): self.set_machine('ast2700-evb') - image_path = self.ASSET_SDK_V902_AST2700.fetch() - archive_extract(image_path, self.workdir) + self.archive_extract(self.ASSET_SDK_V902_AST2700) num_cpu = 4 uboot_size = os.path.getsize(self.scratch_file('ast2700-default', diff --git a/tests/functional/test_aarch64_raspi3.py b/tests/functional/test_aarch64_raspi3.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_raspi3.py +++ b/tests/functional/test_aarch64_raspi3.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later -from zipfile import ZipFile - from qemu_test import LinuxKernelTest, Asset @@ -XXX,XX +XXX,XX @@ class Aarch64Raspi3Machine(LinuxKernelTest): def test_aarch64_raspi3_atf(self): efi_name = 'RPI_EFI.fd' - zip_path = self.ASSET_RPI3_UEFI.fetch() - - with ZipFile(zip_path, 'r') as zf: - zf.extract(efi_name, path=self.workdir) - efi_fd = self.scratch_file(efi_name) + efi_fd = self.archive_extract(self.ASSET_RPI3_UEFI, member=efi_name) self.set_machine('raspi3b') self.vm.set_console(console_index=1) diff --git a/tests/functional/test_aarch64_raspi4.py b/tests/functional/test_aarch64_raspi4.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_raspi4.py +++ b/tests/functional/test_aarch64_raspi4.py @@ -XXX,XX +XXX,XX @@ class Aarch64Raspi4Machine(LinuxKernelTest): '7c0b16d1853772f6f4c3ca63e789b3b9ff4936efac9c8a01fb0c98c05c7a7648') def test_arm_raspi4(self): - deb_path = self.ASSET_KERNEL_20190215.fetch() - kernel_path = self.extract_from_deb(deb_path, '/boot/kernel8.img') - dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2711-rpi-4-b.dtb') + kernel_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='boot/kernel8.img') + dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='boot/bcm2711-rpi-4-b.dtb') self.set_machine('raspi4b') self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_raspi4(self): def test_arm_raspi4_initrd(self): - deb_path = self.ASSET_KERNEL_20190215.fetch() - kernel_path = self.extract_from_deb(deb_path, '/boot/kernel8.img') - dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2711-rpi-4-b.dtb') + kernel_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='boot/kernel8.img') + dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='boot/bcm2711-rpi-4-b.dtb') initrd_path_gz = self.ASSET_INITRD.fetch() initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) diff --git a/tests/functional/test_acpi_bits.py b/tests/functional/test_acpi_bits.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_acpi_bits.py +++ b/tests/functional/test_acpi_bits.py @@ -XXX,XX +XXX,XX @@ import re import shutil import subprocess -import tarfile -import zipfile from typing import ( List, @@ -XXX,XX +XXX,XX @@ def setUp(self): # pylint: disable=arguments-differ %(self.BITS_INTERNAL_VER, self.BITS_COMMIT_HASH)) - bitsLocalArtLoc = self.ASSET_BITS.fetch() - self.logger.info("downloaded bits artifacts to %s", bitsLocalArtLoc) - # extract the bits artifact in the temp working directory - with zipfile.ZipFile(bitsLocalArtLoc, 'r') as zref: - zref.extractall(prebuiltDir) + self.archive_extract(self.ASSET_BITS, sub_dir='prebuilt', format='zip') # extract the bits software in the temp working directory - with zipfile.ZipFile(bits_zip_file, 'r') as zref: - zref.extractall(self.workdir) - - with tarfile.open(grub_tar_file, 'r', encoding='utf-8') as tarball: - tarball.extractall(self.workdir) + self.archive_extract(bits_zip_file) + self.archive_extract(grub_tar_file) self.copy_test_scripts() self.copy_bits_config() diff --git a/tests/functional/test_arm_aspeed_ast1030.py b/tests/functional/test_arm_aspeed_ast1030.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_aspeed_ast1030.py +++ b/tests/functional/test_arm_aspeed_ast1030.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern -from zipfile import ZipFile + class AST1030Machine(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class AST1030Machine(LinuxKernelTest): def test_ast1030_zephyros_1_04(self): self.set_machine('ast1030-evb') - zip_file = self.ASSET_ZEPHYR_1_04.fetch() - kernel_name = "ast1030-evb-demo/zephyr.elf" - with ZipFile(zip_file, 'r') as zf: - zf.extract(kernel_name, path=self.workdir) - kernel_file = self.scratch_file(kernel_name) + kernel_file = self.archive_extract( + self.ASSET_ZEPHYR_1_04, member=kernel_name) self.vm.set_console() self.vm.add_args('-kernel', kernel_file, '-nographic') @@ -XXX,XX +XXX,XX @@ def test_ast1030_zephyros_1_04(self): def test_ast1030_zephyros_1_07(self): self.set_machine('ast1030-evb') - zip_file = self.ASSET_ZEPHYR_1_07.fetch() - kernel_name = "ast1030-evb-demo/zephyr.bin" - with ZipFile(zip_file, 'r') as zf: - zf.extract(kernel_name, path=self.workdir) - kernel_file = self.scratch_file(kernel_name) + kernel_file = self.archive_extract( + self.ASSET_ZEPHYR_1_07, member=kernel_name) self.vm.set_console() self.vm.add_args('-kernel', kernel_file, '-nographic') diff --git a/tests/functional/test_arm_aspeed_ast2500.py b/tests/functional/test_arm_aspeed_ast2500.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_aspeed_ast2500.py +++ b/tests/functional/test_arm_aspeed_ast2500.py @@ -XXX,XX +XXX,XX @@ from qemu_test import Asset from aspeed import AspeedTest from qemu_test import exec_command_and_wait_for_pattern -from qemu_test.utils import archive_extract + class AST2500Machine(AspeedTest): @@ -XXX,XX +XXX,XX @@ def test_arm_ast2500_evb_buildroot(self): def test_arm_ast2500_evb_sdk(self): self.set_machine('ast2500-evb') - image_path = self.ASSET_SDK_V806_AST2500.fetch() - - archive_extract(image_path, self.workdir) + self.archive_extract(self.ASSET_SDK_V806_AST2500) self.do_test_arm_aspeed_sdk_start( self.scratch_file("ast2500-default", "image-bmc")) diff --git a/tests/functional/test_arm_aspeed_ast2600.py b/tests/functional/test_arm_aspeed_ast2600.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_aspeed_ast2600.py +++ b/tests/functional/test_arm_aspeed_ast2600.py @@ -XXX,XX +XXX,XX @@ from qemu_test import Asset from aspeed import AspeedTest from qemu_test import exec_command_and_wait_for_pattern, skipIfMissingCommands -from qemu_test.utils import archive_extract class AST2600Machine(AspeedTest): @@ -XXX,XX +XXX,XX @@ def test_arm_ast2600_evb_buildroot_tpm(self): def test_arm_ast2600_evb_sdk(self): self.set_machine('ast2600-evb') - image_path = self.ASSET_SDK_V806_AST2600_A2.fetch() - - archive_extract(image_path, self.workdir) + self.archive_extract(self.ASSET_SDK_V806_AST2600_A2) self.vm.add_args('-device', 'tmp105,bus=aspeed.i2c.bus.5,address=0x4d,id=tmp-test'); diff --git a/tests/functional/test_arm_aspeed_rainier.py b/tests/functional/test_arm_aspeed_rainier.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_aspeed_rainier.py +++ b/tests/functional/test_arm_aspeed_rainier.py @@ -XXX,XX +XXX,XX @@ def test_arm_aspeed_emmc_boot(self): def test_arm_debian_kernel_boot(self): self.set_machine('rainier-bmc') - deb_path = self.ASSET_DEBIAN_LINUX_ARMHF_DEB.fetch() - - kernel_path = self.extract_from_deb(deb_path, '/boot/vmlinuz-5.17.0-2-armmp') - dtb_path = self.extract_from_deb(deb_path, - '/usr/lib/linux-image-5.17.0-2-armmp/aspeed-bmc-ibm-rainier.dtb') + kernel_path = self.archive_extract( + self.ASSET_DEBIAN_LINUX_ARMHF_DEB, + member='boot/vmlinuz-5.17.0-2-armmp') + dtb_path = self.archive_extract( + self.ASSET_DEBIAN_LINUX_ARMHF_DEB, + member='usr/lib/linux-image-5.17.0-2-armmp/aspeed-bmc-ibm-rainier.dtb') self.vm.set_console() self.vm.add_args('-kernel', kernel_path, diff --git a/tests/functional/test_arm_bflt.py b/tests/functional/test_arm_bflt.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bflt.py +++ b/tests/functional/test_arm_bflt.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuUserTest, Asset from qemu_test import skipIfMissingCommands, skipUntrustedTest -from qemu_test.utils import cpio_extract class LoadBFLT(QemuUserTest): @@ -XXX,XX +XXX,XX @@ def test_stm32(self): busybox_path = self.scratch_file("bin", "busybox") with bz2.open(rootfs_path_bz2, 'rb') as cpio_handle: - cpio_extract(cpio_handle, self.workdir) + self.archive_extract(cpio_handle, format="cpio") res = self.run_cmd(busybox_path) ver = 'BusyBox v1.24.0.git (2015-02-03 22:17:13 CET) multi-call binary.' diff --git a/tests/functional/test_arm_bpim2u.py b/tests/functional/test_arm_bpim2u.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bpim2u.py +++ b/tests/functional/test_arm_bpim2u.py @@ -XXX,XX +XXX,XX @@ class BananaPiMachine(LinuxKernelTest): def test_arm_bpim2u(self): self.set_machine('bpim2u') - deb_path = self.ASSET_DEB.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinuz-6.6.16-current-sunxi') - dtb_path = ('/usr/lib/linux-image-6.6.16-current-sunxi/' + kernel_path = self.archive_extract( + self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi') + dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' 'sun8i-r40-bananapi-m2-ultra.dtb') - dtb_path = self.extract_from_deb(deb_path, dtb_path) + dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u(self): def test_arm_bpim2u_initrd(self): self.set_machine('bpim2u') - deb_path = self.ASSET_DEB.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinuz-6.6.16-current-sunxi') - dtb_path = ('/usr/lib/linux-image-6.6.16-current-sunxi/' + kernel_path = self.archive_extract( + self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi') + dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' 'sun8i-r40-bananapi-m2-ultra.dtb') - dtb_path = self.extract_from_deb(deb_path, dtb_path) + dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) initrd_path_gz = self.ASSET_INITRD.fetch() initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_gmac(self): self.require_netdev('user') deb_path = self.ASSET_DEB.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinuz-6.6.16-current-sunxi') - dtb_path = ('/usr/lib/linux-image-6.6.16-current-sunxi/' + kernel_path = self.archive_extract( + self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi') + dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' 'sun8i-r40-bananapi-m2-ultra.dtb') - dtb_path = self.extract_from_deb(deb_path, dtb_path) + dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) rootfs_path_xz = self.ASSET_ROOTFS.fetch() rootfs_path = self.scratch_file('rootfs.cpio') lzma_uncompress(rootfs_path_xz, rootfs_path) diff --git a/tests/functional/test_arm_canona1100.py b/tests/functional/test_arm_canona1100.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_canona1100.py +++ b/tests/functional/test_arm_canona1100.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern -from qemu_test.utils import archive_extract + class CanonA1100Machine(QemuSystemTest): """Boots the barebox firmware and checks that the console is operational""" @@ -XXX,XX +XXX,XX @@ class CanonA1100Machine(QemuSystemTest): def test_arm_canona1100(self): self.set_machine('canon-a1100') - file_path = self.ASSET_BIOS.fetch() - archive_extract(file_path, dest_dir=self.workdir, - member="day18/barebox.canon-a1100.bin") + bios = self.archive_extract(self.ASSET_BIOS, + member="day18/barebox.canon-a1100.bin") self.vm.set_console() - self.vm.add_args('-bios', - self.scratch_file('day18', - 'barebox.canon-a1100.bin')) + self.vm.add_args('-bios', bios) self.vm.launch() wait_for_console_pattern(self, 'running /env/bin/init') diff --git a/tests/functional/test_arm_cubieboard.py b/tests/functional/test_arm_cubieboard.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_cubieboard.py +++ b/tests/functional/test_arm_cubieboard.py @@ -XXX,XX +XXX,XX @@ class CubieboardMachine(LinuxKernelTest): def test_arm_cubieboard_initrd(self): self.set_machine('cubieboard') - deb_path = self.ASSET_DEB.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinuz-6.6.16-current-sunxi') - dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun4i-a10-cubieboard.dtb' - dtb_path = self.extract_from_deb(deb_path, dtb_path) + kernel_path = self.archive_extract( + self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi') + dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' + + 'sun4i-a10-cubieboard.dtb') + dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) initrd_path_gz = self.ASSET_INITRD.fetch() initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) @@ -XXX,XX +XXX,XX @@ def test_arm_cubieboard_initrd(self): def test_arm_cubieboard_sata(self): self.set_machine('cubieboard') - deb_path = self.ASSET_DEB.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinuz-6.6.16-current-sunxi') - dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun4i-a10-cubieboard.dtb' - dtb_path = self.extract_from_deb(deb_path, dtb_path) + kernel_path = self.archive_extract( + self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi') + dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' + + 'sun4i-a10-cubieboard.dtb') + dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) rootfs_path_gz = self.ASSET_SATA_ROOTFS.fetch() rootfs_path = self.scratch_file('rootfs.cpio') diff --git a/tests/functional/test_arm_orangepi.py b/tests/functional/test_arm_orangepi.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_orangepi.py +++ b/tests/functional/test_arm_orangepi.py @@ -XXX,XX +XXX,XX @@ class BananaPiMachine(LinuxKernelTest): def test_arm_orangepi(self): self.set_machine('orangepi-pc') - deb_path = self.ASSET_DEB.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinuz-6.6.16-current-sunxi') - dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun8i-h3-orangepi-pc.dtb' - dtb_path = self.extract_from_deb(deb_path, dtb_path) + kernel_path = self.archive_extract( + self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi') + dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' + + 'sun8i-h3-orangepi-pc.dtb') + dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi(self): def test_arm_orangepi_initrd(self): self.set_machine('orangepi-pc') - deb_path = self.ASSET_DEB.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinuz-6.6.16-current-sunxi') - dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun8i-h3-orangepi-pc.dtb' - dtb_path = self.extract_from_deb(deb_path, dtb_path) + kernel_path = self.archive_extract( + self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi') + dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' + + 'sun8i-h3-orangepi-pc.dtb') + dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) initrd_path_gz = self.ASSET_INITRD.fetch() initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_initrd(self): def test_arm_orangepi_sd(self): self.set_machine('orangepi-pc') self.require_netdev('user') - deb_path = self.ASSET_DEB.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinuz-6.6.16-current-sunxi') - dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun8i-h3-orangepi-pc.dtb' - dtb_path = self.extract_from_deb(deb_path, dtb_path) + kernel_path = self.archive_extract( + self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi') + dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' + + 'sun8i-h3-orangepi-pc.dtb') + dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) rootfs_path_xz = self.ASSET_ROOTFS.fetch() rootfs_path = self.scratch_file('rootfs.cpio') lzma_uncompress(rootfs_path_xz, rootfs_path) @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_armbian(self): def test_arm_orangepi_uboot_netbsd9(self): self.set_machine('orangepi-pc') # This test download a 304MB compressed image and expand it to 2GB - deb_path = self.ASSET_UBOOT.fetch() # We use the common OrangePi PC 'plus' build of U-Boot for our secondary # program loader (SPL). We will then set the path to the more specific # OrangePi "PC" device tree blob with 'setenv fdtfile' in U-Boot prompt, # before to boot NetBSD. - uboot_path = '/usr/lib/u-boot/orangepi_plus/u-boot-sunxi-with-spl.bin' - uboot_path = self.extract_from_deb(deb_path, uboot_path) + uboot_path = 'usr/lib/u-boot/orangepi_plus/u-boot-sunxi-with-spl.bin' + uboot_path = self.archive_extract(self.ASSET_UBOOT, member=uboot_path) image_path_gz = self.ASSET_NETBSD.fetch() image_path = self.scratch_file('armv7.img') gzip_uncompress(image_path_gz, image_path) diff --git a/tests/functional/test_arm_raspi2.py b/tests/functional/test_arm_raspi2.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_raspi2.py +++ b/tests/functional/test_arm_raspi2.py @@ -XXX,XX +XXX,XX @@ def do_test_arm_raspi2(self, uart_id): serial_kernel_cmdline = { 0: 'earlycon=pl011,0x3f201000 console=ttyAMA0', } - deb_path = self.ASSET_KERNEL_20190215.fetch() - kernel_path = self.extract_from_deb(deb_path, '/boot/kernel7.img') - dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2709-rpi-2-b.dtb') + kernel_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='boot/kernel7.img') + dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='boot/bcm2709-rpi-2-b.dtb') self.set_machine('raspi2b') self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_raspi2_uart0(self): self.do_test_arm_raspi2(0) def test_arm_raspi2_initrd(self): - deb_path = self.ASSET_KERNEL_20190215.fetch() - kernel_path = self.extract_from_deb(deb_path, '/boot/kernel7.img') - dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2709-rpi-2-b.dtb') + kernel_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='boot/kernel7.img') + dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215, + member='boot/bcm2709-rpi-2-b.dtb') initrd_path_gz = self.ASSET_INITRD.fetch() initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) diff --git a/tests/functional/test_arm_smdkc210.py b/tests/functional/test_arm_smdkc210.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_smdkc210.py +++ b/tests/functional/test_arm_smdkc210.py @@ -XXX,XX +XXX,XX @@ class Smdkc210Machine(LinuxKernelTest): def test_arm_exynos4210_initrd(self): self.set_machine('smdkc210') - deb_path = self.ASSET_DEB.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinuz-4.19.0-6-armmp') - dtb_path = '/usr/lib/linux-image-4.19.0-6-armmp/exynos4210-smdkv310.dtb' - dtb_path = self.extract_from_deb(deb_path, dtb_path) + kernel_path = self.archive_extract(self.ASSET_DEB, + member='boot/vmlinuz-4.19.0-6-armmp') + dtb_path = 'usr/lib/linux-image-4.19.0-6-armmp/exynos4210-smdkv310.dtb' + dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) initrd_path_gz = self.ASSET_ROOTFS.fetch() initrd_path = self.scratch_file('rootfs.cpio') diff --git a/tests/functional/test_arm_vexpress.py b/tests/functional/test_arm_vexpress.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_vexpress.py +++ b/tests/functional/test_arm_vexpress.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract + class VExpressTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class VExpressTest(LinuxKernelTest): def test_arm_vexpressa9(self): self.set_machine('vexpress-a9') - file_path = self.ASSET_DAY16.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY16) self.launch_kernel(self.scratch_file('day16', 'winter.zImage'), dtb=self.scratch_file('day16', 'vexpress-v2p-ca9.dtb'), diff --git a/tests/functional/test_m68k_mcf5208evb.py b/tests/functional/test_m68k_mcf5208evb.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_m68k_mcf5208evb.py +++ b/tests/functional/test_m68k_mcf5208evb.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract + class Mcf5208EvbTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class Mcf5208EvbTest(LinuxKernelTest): def test_m68k_mcf5208evb(self): self.set_machine('mcf5208evb') - file_path = self.ASSET_DAY07.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY07) self.vm.set_console() self.vm.add_args('-kernel', self.scratch_file('day07', 'sanity-clause.elf')) diff --git a/tests/functional/test_m68k_q800.py b/tests/functional/test_m68k_q800.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_m68k_q800.py +++ b/tests/functional/test_m68k_q800.py @@ -XXX,XX +XXX,XX @@ class Q800MachineTest(LinuxKernelTest): def test_m68k_q800(self): self.set_machine('q800') - deb_path = self.ASSET_KERNEL.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinux-5.3.0-1-m68k') + kernel_path = self.archive_extract(self.ASSET_KERNEL, + member='boot/vmlinux-5.3.0-1-m68k') self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + diff --git a/tests/functional/test_microblaze_s3adsp1800.py b/tests/functional/test_microblaze_s3adsp1800.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_microblaze_s3adsp1800.py +++ b/tests/functional/test_microblaze_s3adsp1800.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern -from qemu_test.utils import archive_extract + class MicroblazeMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ class MicroblazeMachine(QemuSystemTest): def test_microblaze_s3adsp1800(self): self.set_machine('petalogix-s3adsp1800') - file_path = self.ASSET_IMAGE.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_IMAGE) self.vm.set_console() self.vm.add_args('-kernel', self.scratch_file('day17', 'ballerina.bin')) diff --git a/tests/functional/test_microblazeel_s3adsp1800.py b/tests/functional/test_microblazeel_s3adsp1800.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_microblazeel_s3adsp1800.py +++ b/tests/functional/test_microblazeel_s3adsp1800.py @@ -XXX,XX +XXX,XX @@ from qemu_test import exec_command, exec_command_and_wait_for_pattern from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern -from qemu_test.utils import archive_extract + class MicroblazeelMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ class MicroblazeelMachine(QemuSystemTest): def test_microblazeel_s3adsp1800(self): self.require_netdev('user') self.set_machine('petalogix-s3adsp1800') - file_path = self.ASSET_IMAGE.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_IMAGE) self.vm.set_console() self.vm.add_args('-kernel', self.scratch_file('day13', 'xmaton.bin')) tftproot = self.scratch_file('day13') diff --git a/tests/functional/test_mips64el_fuloong2e.py b/tests/functional/test_mips64el_fuloong2e.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_fuloong2e.py +++ b/tests/functional/test_mips64el_fuloong2e.py @@ -XXX,XX +XXX,XX @@ class MipsFuloong2e(LinuxKernelTest): '2a70f15b397f4ced632b0c15cb22660394190644146d804d60a4796eefbe1f50') def test_linux_kernel_3_16(self): - deb_path = self.ASSET_KERNEL.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinux-3.16.0-6-loongson-2e') + kernel_path = self.archive_extract( + self.ASSET_KERNEL, + member='boot/vmlinux-3.16.0-6-loongson-2e') self.set_machine('fuloong2e') self.vm.set_console() diff --git a/tests/functional/test_mips64el_malta.py b/tests/functional/test_mips64el_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_malta.py +++ b/tests/functional/test_mips64el_malta.py @@ -XXX,XX +XXX,XX @@ def test_mips64el_malta(self): [2] https://kernel-team.pages.debian.net/kernel-handbook/ ch-common-tasks.html#s-common-official """ - deb_path = self.ASSET_KERNEL_2_63_2.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinux-2.6.32-5-5kc-malta') + kernel_path = self.archive_extract( + self.ASSET_KERNEL_2_63_2, + member='boot/vmlinux-2.6.32-5-5kc-malta') self.set_machine('malta') self.vm.set_console() diff --git a/tests/functional/test_mips_malta.py b/tests/functional/test_mips_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips_malta.py +++ b/tests/functional/test_mips_malta.py @@ -XXX,XX +XXX,XX @@ class MaltaMachineConsole(LinuxKernelTest): '16ca524148afb0626f483163e5edf352bc1ab0e4fc7b9f9d473252762f2c7a43') def test_mips_malta(self): - deb_path = self.ASSET_KERNEL_2_63_2.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinux-2.6.32-5-4kc-malta') + kernel_path = self.archive_extract( + self.ASSET_KERNEL_2_63_2, + member='boot/vmlinux-2.6.32-5-4kc-malta') self.set_machine('malta') self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_mips_malta(self): 'dcfe3a7fe3200da3a00d176b95caaa086495eb158f2bff64afc67d7e1eb2cddc') def test_mips_malta_cpio(self): - deb_path = self.ASSET_KERNEL_4_5_0.fetch() - kernel_path = self.extract_from_deb(deb_path, - '/boot/vmlinux-4.5.0-2-4kc-malta') + kernel_path = self.archive_extract( + self.ASSET_KERNEL_4_5_0, + member='boot/vmlinux-4.5.0-2-4kc-malta') initrd_path_gz = self.ASSET_INITRD.fetch() initrd_path = self.scratch_file('rootfs.cpio') gzip_uncompress(initrd_path_gz, initrd_path) diff --git a/tests/functional/test_mipsel_malta.py b/tests/functional/test_mipsel_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mipsel_malta.py +++ b/tests/functional/test_mipsel_malta.py @@ -XXX,XX +XXX,XX @@ from qemu_test import interrupt_interactive_console_until_pattern from qemu_test import wait_for_console_pattern from qemu_test.utils import lzma_uncompress -from zipfile import ZipFile class MaltaMachineConsole(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class MaltaMachineYAMON(QemuSystemTest): def test_mipsel_malta_yamon(self): yamon_bin = 'yamon-02.22.bin' - zip_path = self.ASSET_YAMON_ROM.fetch() - with ZipFile(zip_path, 'r') as zf: - zf.extract(yamon_bin, path=self.workdir) + self.archive_extract(self.ASSET_YAMON_ROM) yamon_path = self.scratch_file(yamon_bin) self.set_machine('malta') diff --git a/tests/functional/test_or1k_sim.py b/tests/functional/test_or1k_sim.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_or1k_sim.py +++ b/tests/functional/test_or1k_sim.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract + class OpenRISC1kSimTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class OpenRISC1kSimTest(LinuxKernelTest): def test_or1k_sim(self): self.set_machine('or1k-sim') - file_path = self.ASSET_DAY20.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY20) self.vm.set_console() self.vm.add_args('-kernel', self.scratch_file('day20', 'vmlinux')) self.vm.launch() diff --git a/tests/functional/test_ppc64_e500.py b/tests/functional/test_ppc64_e500.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc64_e500.py +++ b/tests/functional/test_ppc64_e500.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract + class E500Test(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class E500Test(LinuxKernelTest): def test_ppc64_e500(self): self.set_machine('ppce500') self.cpu = 'e5500' - file_path = self.ASSET_DAY19.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY19) self.launch_kernel(self.scratch_file('day19', 'uImage'), wait_for='QEMU advent calendar') diff --git a/tests/functional/test_ppc_amiga.py b/tests/functional/test_ppc_amiga.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_amiga.py +++ b/tests/functional/test_ppc_amiga.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern -from zipfile import ZipFile + class AmigaOneMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def test_ppc_amigaone(self): self.require_accelerator("tcg") self.set_machine('amigaone') tar_name = 'A1Firmware_Floppy_05-Mar-2005.zip' - zip_file = self.ASSET_IMAGE.fetch() - with ZipFile(zip_file, 'r') as zf: - zf.extractall(path=self.workdir) + self.archive_extract(self.ASSET_IMAGE, format="zip") bios = self.scratch_file("u-boot-amigaone.bin") with open(bios, "wb") as bios_fh: subprocess.run(['tail', '-c', '524288', diff --git a/tests/functional/test_ppc_bamboo.py b/tests/functional/test_ppc_bamboo.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_bamboo.py +++ b/tests/functional/test_ppc_bamboo.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from qemu_test.utils import archive_extract from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern from qemu_test import exec_command_and_wait_for_pattern + class BambooMachine(QemuSystemTest): timeout = 90 @@ -XXX,XX +XXX,XX @@ def test_ppc_bamboo(self): self.set_machine('bamboo') self.require_accelerator("tcg") self.require_netdev('user') - file_path = self.ASSET_IMAGE.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_IMAGE) self.vm.set_console() self.vm.add_args('-kernel', self.scratch_file('system-image-powerpc-440fp', diff --git a/tests/functional/test_ppc_mac.py b/tests/functional/test_ppc_mac.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_mac.py +++ b/tests/functional/test_ppc_mac.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract + class MacTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def do_day15_test(self): # we're running kvm_hv or kvm_pr. For now let's disable this test # if we don't have TCG support. self.require_accelerator("tcg") - - file_path = self.ASSET_DAY15.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY15) self.vm.add_args('-M', 'graphics=off') self.launch_kernel(self.scratch_file('day15', 'invaders.elf'), wait_for='QEMU advent calendar') diff --git a/tests/functional/test_ppc_mpc8544ds.py b/tests/functional/test_ppc_mpc8544ds.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_mpc8544ds.py +++ b/tests/functional/test_ppc_mpc8544ds.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from qemu_test.utils import archive_extract from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern + class Mpc8544dsMachine(QemuSystemTest): timeout = 90 @@ -XXX,XX +XXX,XX @@ class Mpc8544dsMachine(QemuSystemTest): def test_ppc_mpc8544ds(self): self.require_accelerator("tcg") self.set_machine('mpc8544ds') - file_path = self.ASSET_IMAGE.fetch() - archive_extract(file_path, self.workdir, member='creek/creek.bin') + kernel_file = self.archive_extract(self.ASSET_IMAGE, + member='creek/creek.bin') self.vm.set_console() - self.vm.add_args('-kernel', self.scratch_file('creek', 'creek.bin')) + self.vm.add_args('-kernel', kernel_file) self.vm.launch() wait_for_console_pattern(self, 'QEMU advent calendar 2020', self.panic_message) diff --git a/tests/functional/test_ppc_virtex_ml507.py b/tests/functional/test_ppc_virtex_ml507.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc_virtex_ml507.py +++ b/tests/functional/test_ppc_virtex_ml507.py @@ -XXX,XX +XXX,XX @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. -from qemu_test.utils import archive_extract from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern + class VirtexMl507Machine(QemuSystemTest): timeout = 90 @@ -XXX,XX +XXX,XX @@ class VirtexMl507Machine(QemuSystemTest): def test_ppc_virtex_ml507(self): self.require_accelerator("tcg") self.set_machine('virtex-ml507') - file_path = self.ASSET_IMAGE.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_IMAGE) self.vm.set_console() self.vm.add_args('-kernel', self.scratch_file('hippo', 'hippo.linux'), '-dtb', self.scratch_file('hippo', diff --git a/tests/functional/test_sh4_r2d.py b/tests/functional/test_sh4_r2d.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sh4_r2d.py +++ b/tests/functional/test_sh4_r2d.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset, skipFlakyTest -from qemu_test.utils import archive_extract + class R2dTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class R2dTest(LinuxKernelTest): @skipFlakyTest(bug_url=None) def test_r2d(self): self.set_machine('r2d') - file_path = self.ASSET_DAY09.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY09) self.vm.add_args('-append', 'console=ttySC1') self.launch_kernel(self.scratch_file('day09', 'zImage'), console_index=1, diff --git a/tests/functional/test_sh4eb_r2d.py b/tests/functional/test_sh4eb_r2d.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sh4eb_r2d.py +++ b/tests/functional/test_sh4eb_r2d.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern -from qemu_test.utils import archive_extract + class R2dEBTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class R2dEBTest(LinuxKernelTest): def test_sh4eb_r2d(self): self.set_machine('r2d') - file_path = self.ASSET_TGZ.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_TGZ) self.vm.add_args('-append', 'console=ttySC1 noiotrap') self.launch_kernel(self.scratch_file('sh4eb', 'linux-kernel'), initrd=self.scratch_file('sh4eb', diff --git a/tests/functional/test_sparc64_sun4u.py b/tests/functional/test_sparc64_sun4u.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sparc64_sun4u.py +++ b/tests/functional/test_sparc64_sun4u.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern -from qemu_test.utils import archive_extract class Sun4uMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ class Sun4uMachine(QemuSystemTest): def test_sparc64_sun4u(self): self.set_machine('sun4u') - file_path = self.ASSET_IMAGE.fetch() - kernel_name = 'day23/vmlinux' - archive_extract(file_path, self.workdir, kernel_name) + kernel_file = self.archive_extract(self.ASSET_IMAGE, + member='day23/vmlinux') self.vm.set_console() - self.vm.add_args('-kernel', self.scratch_file(kernel_name), + self.vm.add_args('-kernel', kernel_file, '-append', 'printk.time=0') self.vm.launch() wait_for_console_pattern(self, 'Starting logging: OK') diff --git a/tests/functional/test_sparc_sun4m.py b/tests/functional/test_sparc_sun4m.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_sparc_sun4m.py +++ b/tests/functional/test_sparc_sun4m.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract + class Sun4mTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class Sun4mTest(LinuxKernelTest): def test_sparc_ss20(self): self.set_machine('SS-20') - file_path = self.ASSET_DAY11.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY11) self.launch_kernel(self.scratch_file('day11', 'zImage.elf'), wait_for='QEMU advent calendar') diff --git a/tests/functional/test_xtensa_lx60.py b/tests/functional/test_xtensa_lx60.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_xtensa_lx60.py +++ b/tests/functional/test_xtensa_lx60.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import archive_extract + class XTensaLX60Test(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class XTensaLX60Test(LinuxKernelTest): def test_xtensa_lx60(self): self.set_machine('lx60') self.cpu = 'dc233c' - file_path = self.ASSET_DAY02.fetch() - archive_extract(file_path, self.workdir) + self.archive_extract(self.ASSET_DAY02) self.launch_kernel(self.scratch_file('day02', 'santas-sleigh-ride.elf'), wait_for='QEMU advent calendar') -- 2.46.0 There are many types of compression that the tests deal with, and it makes sense to have a single helper 'uncompress' that can deal with all. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/__init__.py | 1 + tests/functional/qemu_test/uncompress.py | 47 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/tests/functional/qemu_test/__init__.py b/tests/functional/qemu_test/__init__.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/__init__.py +++ b/tests/functional/qemu_test/__init__.py @@ -XXX,XX +XXX,XX @@ skipFlakyTest, skipUntrustedTest, skipBigDataTest, \ skipIfMissingImports from .archive import archive_extract +from .uncompress import uncompress diff --git a/tests/functional/qemu_test/uncompress.py b/tests/functional/qemu_test/uncompress.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/uncompress.py +++ b/tests/functional/qemu_test/uncompress.py @@ -XXX,XX +XXX,XX @@ import lzma import os import shutil +from urllib.parse import urlparse + +from .asset import Asset def gzip_uncompress(gz_path, output_path): @@ -XXX,XX +XXX,XX @@ def lzma_uncompress(xz_path, output_path): except: os.remove(output_path) raise + +''' +@params compressed: filename, Asset, or file-like object to uncompress +@params uncompressed: filename to uncompress into +@params format: optional compression format (gzip, lzma) + +Uncompresses @compressed into @uncompressed + +If @format is None, heuristics will be applied to guess the format +from the filename or Asset URL. @format must be non-None if @uncompressed +is a file-like object. + +Returns the fully qualified path to the uncompessed file +''' +def uncompress(compressed, uncompressed, format=None): + if format is None: + format = guess_uncompress_format(compressed) + + if format == "xz": + lzma_uncompress(str(compressed), uncompressed) + elif format == "gz": + gzip_uncompress(str(compressed), uncompressed) + else: + raise Exception(f"Unknown compression format {format}") + +''' +@params compressed: filename, Asset, or file-like object to guess + +Guess the format of @compressed, raising an exception if +no format can be determined +''' +def guess_uncompress_format(compressed): + if type(compressed) == Asset: + compressed = urlparse(compressed.url).path + elif type(compressed) != str: + raise Exception(f"Unable to guess compression cformat for {compressed}") + + (name, ext) = os.path.splitext(compressed) + if ext == ".xz": + return "xz" + elif ext == ".gz": + return "gz" + else: + raise Exception(f"Unknown compression format for {compressed}") -- 2.46.0 This helper wrappers utils.uncompress, forcing the use of the scratch directory, to ensure any uncompressed files are cleaned at test termination. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/testcase.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -XXX,XX +XXX,XX @@ from .asset import Asset from .cmd import run_cmd from .config import BUILD_DIR +from .uncompress import uncompress class QemuBaseTest(unittest.TestCase): @@ -XXX,XX +XXX,XX @@ class QemuBaseTest(unittest.TestCase): log = None logdir = None + ''' + @params compressed: filename, Asset, or file-like object to uncompress + @params format: optional compression format (gzip, lzma) + + Uncompresses @compressed into the scratch directory. + + If @format is None, heuristics will be applied to guess the format + from the filename or Asset URL. @format must be non-None if @uncompressed + is a file-like object. + + Returns the fully qualified path to the uncompressed file + ''' + def uncompress(self, compressed, format=None): + self.log.debug(f"Uncompress {compressed} format={format}") + if type(compressed) == Asset: + compressed.fetch() + + (name, ext) = os.path.splitext(str(compressed)) + uncompressed = self.scratch_file(os.path.basename(name)) + + uncompress(compressed, uncompressed, format) + + return uncompressed + ''' @params archive: filename, Asset, or file-like object to extract @params format: optional archive format (tar, zip, deb, cpio) -- 2.46.0 Replace use of lzma_uncompress and gzip_uncompress with the new uncompress helper. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/test_aarch64_raspi4.py | 5 +---- tests/functional/test_aarch64_sbsaref.py | 10 +++------- tests/functional/test_alpha_clipper.py | 4 +--- tests/functional/test_arm_bpim2u.py | 13 +++---------- tests/functional/test_arm_cubieboard.py | 14 ++++---------- tests/functional/test_arm_orangepi.py | 17 ++++------------- tests/functional/test_arm_raspi2.py | 5 +---- tests/functional/test_arm_smdkc210.py | 6 ++---- tests/functional/test_mips64el_malta.py | 9 ++------- tests/functional/test_mips_malta.py | 5 +---- tests/functional/test_mipsel_malta.py | 15 +++++---------- tests/functional/test_rx_gdbsim.py | 5 +---- tests/functional/test_s390x_ccw_virtio.py | 6 ++---- tests/functional/test_s390x_topology.py | 5 +---- 14 files changed, 31 insertions(+), 88 deletions(-) diff --git a/tests/functional/test_aarch64_raspi4.py b/tests/functional/test_aarch64_raspi4.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_raspi4.py +++ b/tests/functional/test_aarch64_raspi4.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern -from qemu_test.utils import gzip_uncompress class Aarch64Raspi4Machine(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_arm_raspi4_initrd(self): member='boot/kernel8.img') dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215, member='boot/bcm2711-rpi-4-b.dtb') - initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = self.scratch_file('rootfs.cpio') - gzip_uncompress(initrd_path_gz, initrd_path) + initrd_path = self.uncompress(self.ASSET_INITRD) self.set_machine('raspi4b') self.vm.set_console() diff --git a/tests/functional/test_aarch64_sbsaref.py b/tests/functional/test_aarch64_sbsaref.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_sbsaref.py +++ b/tests/functional/test_aarch64_sbsaref.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuSystemTest, Asset from qemu_test import wait_for_console_pattern from qemu_test import interrupt_interactive_console_until_pattern -from qemu_test.utils import lzma_uncompress + def fetch_firmware(test): """ @@ -XXX,XX +XXX,XX @@ def fetch_firmware(test): """ # Secure BootRom (TF-A code) - fs0_xz_path = Aarch64SbsarefMachine.ASSET_FLASH0.fetch() - fs0_path = test.scratch_file("SBSA_FLASH0.fd") - lzma_uncompress(fs0_xz_path, fs0_path) + fs0_path = test.uncompress(Aarch64SbsarefMachine.ASSET_FLASH0) # Non-secure rom (UEFI and EFI variables) - fs1_xz_path = Aarch64SbsarefMachine.ASSET_FLASH1.fetch() - fs1_path = test.scratch_file("SBSA_FLASH1.fd") - lzma_uncompress(fs1_xz_path, fs1_path) + fs1_path = test.uncompress(Aarch64SbsarefMachine.ASSET_FLASH1) for path in [fs0_path, fs1_path]: with open(path, "ab+") as fd: diff --git a/tests/functional/test_alpha_clipper.py b/tests/functional/test_alpha_clipper.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_alpha_clipper.py +++ b/tests/functional/test_alpha_clipper.py @@ -XXX,XX +XXX,XX @@ # SPDX-License-Identifier: GPL-2.0-or-later from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import gzip_uncompress class AlphaClipperTest(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_alpha_clipper(self): self.set_machine('clipper') kernel_path = self.ASSET_KERNEL.fetch() - uncompressed_kernel = self.scratch_file('vmlinux') - gzip_uncompress(kernel_path, uncompressed_kernel) + uncompressed_kernel = self.uncompress(self.ASSET_KERNEL, format="gz") self.vm.set_console() kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=ttyS0' diff --git a/tests/functional/test_arm_bpim2u.py b/tests/functional/test_arm_bpim2u.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_bpim2u.py +++ b/tests/functional/test_arm_bpim2u.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, exec_command_and_wait_for_pattern from qemu_test import Asset, interrupt_interactive_console_until_pattern from qemu_test import skipBigDataTest -from qemu_test.utils import gzip_uncompress, lzma_uncompress from qemu_test.utils import image_pow2ceil_expand @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_initrd(self): dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' 'sun8i-r40-bananapi-m2-ultra.dtb') dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) - initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = self.scratch_file('rootfs.cpio') - gzip_uncompress(initrd_path_gz, initrd_path) + initrd_path = self.uncompress(self.ASSET_INITRD) self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_gmac(self): dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' 'sun8i-r40-bananapi-m2-ultra.dtb') dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) - rootfs_path_xz = self.ASSET_ROOTFS.fetch() - rootfs_path = self.scratch_file('rootfs.cpio') - lzma_uncompress(rootfs_path_xz, rootfs_path) + rootfs_path = self.uncompress(self.ASSET_ROOTFS) image_pow2ceil_expand(rootfs_path) self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_bpim2u_openwrt_22_03_3(self): self.set_machine('bpim2u') # This test download a 8.9 MiB compressed image and expand it # to 127 MiB. - image_path_gz = self.ASSET_SD_IMAGE.fetch() - image_path = self.scratch_file('sdcard.img') - gzip_uncompress(image_path_gz, image_path) + image_path = self.uncompress(self.ASSET_SD_IMAGE) image_pow2ceil_expand(image_path) self.vm.set_console() diff --git a/tests/functional/test_arm_cubieboard.py b/tests/functional/test_arm_cubieboard.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_cubieboard.py +++ b/tests/functional/test_arm_cubieboard.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, Asset, exec_command_and_wait_for_pattern from qemu_test import interrupt_interactive_console_until_pattern from qemu_test import skipBigDataTest -from qemu_test.utils import gzip_uncompress, image_pow2ceil_expand +from qemu_test.utils import image_pow2ceil_expand class CubieboardMachine(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_arm_cubieboard_initrd(self): dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' + 'sun4i-a10-cubieboard.dtb') dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) - initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = self.scratch_file('rootfs.cpio') - gzip_uncompress(initrd_path_gz, initrd_path) + initrd_path = self.uncompress(self.ASSET_INITRD) self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + @@ -XXX,XX +XXX,XX @@ def test_arm_cubieboard_sata(self): 'sun4i-a10-cubieboard.dtb') dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) - rootfs_path_gz = self.ASSET_SATA_ROOTFS.fetch() - rootfs_path = self.scratch_file('rootfs.cpio') - gzip_uncompress(rootfs_path_gz, rootfs_path) + rootfs_path = self.uncompress(self.ASSET_SATA_ROOTFS) self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + @@ -XXX,XX +XXX,XX @@ def test_arm_cubieboard_openwrt_22_03_2(self): # This test download a 7.5 MiB compressed image and expand it # to 126 MiB. self.set_machine('cubieboard') - image_path_gz = self.ASSET_OPENWRT.fetch() - image_path = self.scratch_file('sdcard.img') - gzip_uncompress(image_path_gz, image_path) + image_path = self.uncompress(self.ASSET_OPENWRT) image_pow2ceil_expand(image_path) self.vm.set_console() diff --git a/tests/functional/test_arm_orangepi.py b/tests/functional/test_arm_orangepi.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_orangepi.py +++ b/tests/functional/test_arm_orangepi.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, exec_command_and_wait_for_pattern from qemu_test import Asset, interrupt_interactive_console_until_pattern from qemu_test import wait_for_console_pattern, skipBigDataTest -from qemu_test.utils import gzip_uncompress, lzma_uncompress from qemu_test.utils import image_pow2ceil_expand @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_initrd(self): dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' + 'sun8i-h3-orangepi-pc.dtb') dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) - initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = self.scratch_file('rootfs.cpio') - gzip_uncompress(initrd_path_gz, initrd_path) + initrd_path = self.uncompress(self.ASSET_INITRD) self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_sd(self): dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' + 'sun8i-h3-orangepi-pc.dtb') dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) - rootfs_path_xz = self.ASSET_ROOTFS.fetch() - rootfs_path = self.scratch_file('rootfs.cpio') - lzma_uncompress(rootfs_path_xz, rootfs_path) + rootfs_path = self.uncompress(self.ASSET_ROOTFS) image_pow2ceil_expand(rootfs_path) self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_armbian(self): # This test download a 275 MiB compressed image and expand it # to 1036 MiB, but the underlying filesystem is 1552 MiB... # As we expand it to 2 GiB we are safe. - image_path_xz = self.ASSET_ARMBIAN.fetch() - image_path = self.scratch_file('armbian.img') - lzma_uncompress(image_path_xz, image_path) + image_path = self.uncompress(self.ASSET_ARMBIAN) image_pow2ceil_expand(image_path) self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def test_arm_orangepi_uboot_netbsd9(self): # before to boot NetBSD. uboot_path = 'usr/lib/u-boot/orangepi_plus/u-boot-sunxi-with-spl.bin' uboot_path = self.archive_extract(self.ASSET_UBOOT, member=uboot_path) - image_path_gz = self.ASSET_NETBSD.fetch() - image_path = self.scratch_file('armv7.img') - gzip_uncompress(image_path_gz, image_path) + image_path = self.uncompress(self.ASSET_NETBSD) image_pow2ceil_expand(image_path) image_drive_args = 'if=sd,format=raw,snapshot=on,file=' + image_path diff --git a/tests/functional/test_arm_raspi2.py b/tests/functional/test_arm_raspi2.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_raspi2.py +++ b/tests/functional/test_arm_raspi2.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern -from qemu_test.utils import gzip_uncompress class ArmRaspi2Machine(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_arm_raspi2_initrd(self): member='boot/kernel7.img') dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215, member='boot/bcm2709-rpi-2-b.dtb') - initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = self.scratch_file('rootfs.cpio') - gzip_uncompress(initrd_path_gz, initrd_path) + initrd_path = self.uncompress(self.ASSET_INITRD) self.set_machine('raspi2b') self.vm.set_console() diff --git a/tests/functional/test_arm_smdkc210.py b/tests/functional/test_arm_smdkc210.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_arm_smdkc210.py +++ b/tests/functional/test_arm_smdkc210.py @@ -XXX,XX +XXX,XX @@ import os from qemu_test import LinuxKernelTest, Asset -from qemu_test.utils import gzip_uncompress + class Smdkc210Machine(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_arm_exynos4210_initrd(self): dtb_path = 'usr/lib/linux-image-4.19.0-6-armmp/exynos4210-smdkv310.dtb' dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path) - initrd_path_gz = self.ASSET_ROOTFS.fetch() - initrd_path = self.scratch_file('rootfs.cpio') - gzip_uncompress(initrd_path_gz, initrd_path) + initrd_path = self.uncompress(self.ASSET_ROOTFS) self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + diff --git a/tests/functional/test_mips64el_malta.py b/tests/functional/test_mips64el_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips64el_malta.py +++ b/tests/functional/test_mips64el_malta.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern from qemu_test import skipIfMissingImports, skipFlakyTest, skipUntrustedTest -from qemu_test.utils import gzip_uncompress class MaltaMachineConsole(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_mips64el_malta(self): @skipUntrustedTest() def test_mips64el_malta_5KEc_cpio(self): kernel_path = self.ASSET_KERNEL_3_19_3.fetch() - initrd_path_gz = self.ASSET_CPIO_R1.fetch() - initrd_path = self.scratch_file('rootfs.cpio') - gzip_uncompress(initrd_path_gz, initrd_path) + initrd_path = self.uncompress(self.ASSET_CPIO_R1) self.set_machine('malta') self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def do_test_i6400_framebuffer_logo(self, cpu_cores_count): screendump_path = self.scratch_file('screendump.pbm') - kernel_path_gz = self.ASSET_KERNEL_4_7_0.fetch() - kernel_path = self.scratch_file("vmlinux") - gzip_uncompress(kernel_path_gz, kernel_path) + kernel_path = self.uncompress(self.ASSET_KERNEL_4_7_0) tuxlogo_path = self.ASSET_TUXLOGO.fetch() diff --git a/tests/functional/test_mips_malta.py b/tests/functional/test_mips_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mips_malta.py +++ b/tests/functional/test_mips_malta.py @@ -XXX,XX +XXX,XX @@ from qemu_test import LinuxKernelTest, Asset from qemu_test import exec_command_and_wait_for_pattern -from qemu_test.utils import gzip_uncompress class MaltaMachineConsole(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ def test_mips_malta_cpio(self): kernel_path = self.archive_extract( self.ASSET_KERNEL_4_5_0, member='boot/vmlinux-4.5.0-2-4kc-malta') - initrd_path_gz = self.ASSET_INITRD.fetch() - initrd_path = self.scratch_file('rootfs.cpio') - gzip_uncompress(initrd_path_gz, initrd_path) + initrd_path = self.uncompress(self.ASSET_INITRD) self.set_machine('malta') self.vm.set_console() diff --git a/tests/functional/test_mipsel_malta.py b/tests/functional/test_mipsel_malta.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_mipsel_malta.py +++ b/tests/functional/test_mipsel_malta.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuSystemTest, LinuxKernelTest, Asset from qemu_test import interrupt_interactive_console_until_pattern from qemu_test import wait_for_console_pattern -from qemu_test.utils import lzma_uncompress class MaltaMachineConsole(LinuxKernelTest): @@ -XXX,XX +XXX,XX @@ class MaltaMachineConsole(LinuxKernelTest): 'generic_nano32r6el_page64k_dbg.xz'), 'ce21ff4b07a981ecb8a39db2876616f5a2473eb2ab459c6f67465b9914b0c6b6') - def do_test_mips_malta32el_nanomips(self, kernel_path_xz): - kernel_path = self.scratch_file('kernel') - lzma_uncompress(kernel_path_xz, kernel_path) + def do_test_mips_malta32el_nanomips(self, kernel): + kernel_path = self.uncompress(kernel) self.set_machine('malta') self.vm.set_console() @@ -XXX,XX +XXX,XX @@ def do_test_mips_malta32el_nanomips(self, kernel_path_xz): self.wait_for_console_pattern(console_pattern) def test_mips_malta32el_nanomips_4k(self): - kernel_path_xz = self.ASSET_KERNEL_4K.fetch() - self.do_test_mips_malta32el_nanomips(kernel_path_xz) + self.do_test_mips_malta32el_nanomips(self.ASSET_KERNEL_4K) def test_mips_malta32el_nanomips_16k_up(self): - kernel_path_xz = self.ASSET_KERNEL_16K.fetch() - self.do_test_mips_malta32el_nanomips(kernel_path_xz) + self.do_test_mips_malta32el_nanomips(self.ASSET_KERNEL_16K) def test_mips_malta32el_nanomips_64k_dbg(self): - kernel_path_xz = self.ASSET_KERNEL_16K.fetch() - self.do_test_mips_malta32el_nanomips(kernel_path_xz) + self.do_test_mips_malta32el_nanomips(self.ASSET_KERNEL_64K) class MaltaMachineYAMON(QemuSystemTest): diff --git a/tests/functional/test_rx_gdbsim.py b/tests/functional/test_rx_gdbsim.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_rx_gdbsim.py +++ b/tests/functional/test_rx_gdbsim.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuSystemTest, Asset from qemu_test import exec_command_and_wait_for_pattern from qemu_test import wait_for_console_pattern, skipFlakyTest -from qemu_test.utils import gzip_uncompress class RxGdbSimMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def test_uboot(self): """ self.set_machine('gdbsim-r5f562n8') - uboot_path_gz = self.ASSET_UBOOT.fetch() - uboot_path = self.scratch_file('u-boot.bin') - gzip_uncompress(uboot_path_gz, uboot_path) + uboot_path = self.uncompress(self.ASSET_UBOOT) self.vm.set_console() self.vm.add_args('-bios', uboot_path, diff --git a/tests/functional/test_s390x_ccw_virtio.py b/tests/functional/test_s390x_ccw_virtio.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_s390x_ccw_virtio.py +++ b/tests/functional/test_s390x_ccw_virtio.py @@ -XXX,XX +XXX,XX @@ from qemu_test import QemuSystemTest, Asset from qemu_test import exec_command_and_wait_for_pattern from qemu_test import wait_for_console_pattern -from qemu_test.utils import lzma_uncompress + class S390CCWVirtioMachine(QemuSystemTest): KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 ' @@ -XXX,XX +XXX,XX @@ def test_s390x_fedora(self): kernel_path = self.ASSET_F31_KERNEL.fetch() - initrd_path_xz = self.ASSET_F31_INITRD.fetch() - initrd_path = self.scratch_file('initrd-raw.img') - lzma_uncompress(initrd_path_xz, initrd_path) + initrd_path = self.uncompress(self.ASSET_F31_INITRD, format="xz") self.vm.set_console() kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + ' audit=0 ' diff --git a/tests/functional/test_s390x_topology.py b/tests/functional/test_s390x_topology.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_s390x_topology.py +++ b/tests/functional/test_s390x_topology.py @@ -XXX,XX +XXX,XX @@ from qemu_test import exec_command from qemu_test import exec_command_and_wait_for_pattern from qemu_test import wait_for_console_pattern -from qemu_test.utils import lzma_uncompress class S390CPUTopology(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def kernel_init(self): """ self.require_accelerator("kvm") kernel_path = self.ASSET_F35_KERNEL.fetch() - initrd_path_xz = self.ASSET_F35_INITRD.fetch() - initrd_path = self.scratch_file('initrd-raw.img') - lzma_uncompress(initrd_path_xz, initrd_path) + initrd_path = self.uncompress(self.ASSET_F35_INITRD, format="xz") self.vm.set_console() kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE -- 2.46.0 Now that all tests are converted over to the higher level wrapper functions, the back compat imports from utils.py are redundant. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/utils.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/functional/qemu_test/utils.py b/tests/functional/qemu_test/utils.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/utils.py +++ b/tests/functional/qemu_test/utils.py @@ -XXX,XX +XXX,XX @@ import os -from .archive import tar_extract as archive_extract -from .archive import cpio_extract -from .uncompress import gzip_uncompress -from .uncompress import lzma_uncompress - """ Round up to next power of 2 """ -- 2.46.0 The 'run_cmd' helper is re-implementing a convenient helper that already exists in the form of the 'run' and 'check_call' methods provided by 'subprocess'. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/archive.py | 9 ++++--- tests/functional/qemu_test/tesseract.py | 10 ++++---- tests/functional/qemu_test/testcase.py | 31 +++++++++++++----------- tests/functional/qemu_test/tuxruntest.py | 8 +++--- tests/functional/test_aarch64_virt.py | 6 +++-- tests/functional/test_ppc64_tuxrun.py | 7 ++++-- 6 files changed, 41 insertions(+), 30 deletions(-) diff --git a/tests/functional/qemu_test/archive.py b/tests/functional/qemu_test/archive.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/archive.py +++ b/tests/functional/qemu_test/archive.py @@ -XXX,XX +XXX,XX @@ import zipfile from .asset import Asset -from .cmd import run_cmd def tar_extract(archive, dest_dir, member=None): @@ -XXX,XX +XXX,XX @@ def deb_extract(archive, dest_dir, member=None): cwd = os.getcwd() os.chdir(dest_dir) try: - (stdout, stderr, ret) = run_cmd(['ar', 't', archive]) - file_path = stdout.split()[2] - run_cmd(['ar', 'x', archive, file_path]) + proc = run(['ar', 't', archive], + check=True, capture_output=True, encoding='utf8') + file_path = proc.stdout.split()[2] + check_call(['ar', 'x', archive, file_path], + stdout=DEVNULL, stderr=DEVNULL) tar_extract(file_path, dest_dir, member) finally: os.chdir(cwd) diff --git a/tests/functional/qemu_test/tesseract.py b/tests/functional/qemu_test/tesseract.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/tesseract.py +++ b/tests/functional/qemu_test/tesseract.py @@ -XXX,XX +XXX,XX @@ # later. See the COPYING file in the top-level directory. import logging +from subprocess import run -from . import run_cmd def tesseract_ocr(image_path, tesseract_args=''): console_logger = logging.getLogger('console') console_logger.debug(image_path) - (stdout, stderr, ret) = run_cmd(['tesseract', image_path, - 'stdout']) - if ret: + proc = run(['tesseract', image_path, 'stdout'], + capture_output=True, encoding='utf8') + if proc.returncode: return None lines = [] - for line in stdout.split('\n'): + for line in proc.stdout.split('\n'): sline = line.strip() if len(sline): console_logger.debug(sline) diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -XXX,XX +XXX,XX @@ from pathlib import Path import pycotap import shutil -import subprocess +from subprocess import run import sys import tempfile import unittest @@ -XXX,XX +XXX,XX @@ from .archive import archive_extract from .asset import Asset -from .cmd import run_cmd from .config import BUILD_DIR from .uncompress import uncompress @@ -XXX,XX +XXX,XX @@ def add_ldpath(self, ldpath): self._ldpath.append(os.path.abspath(ldpath)) def run_cmd(self, bin_path, args=[]): - return subprocess.run([self.qemu_bin] - + ["-L %s" % ldpath for ldpath in self._ldpath] - + [bin_path] - + args, - text=True, capture_output=True) + return run([self.qemu_bin] + + ["-L %s" % ldpath for ldpath in self._ldpath] + + [bin_path] + + args, + text=True, capture_output=True) class QemuSystemTest(QemuBaseTest): """Facilitates system emulation tests.""" @@ -XXX,XX +XXX,XX @@ def setUp(self): def set_machine(self, machinename): # TODO: We should use QMP to get the list of available machines if not self._machinehelp: - self._machinehelp = run_cmd([self.qemu_bin, '-M', 'help'])[0]; + self._machinehelp = run( + [self.qemu_bin, '-M', 'help'], + capture_output=True, check=True, encoding='utf8').stdout if self._machinehelp.find(machinename) < 0: self.skipTest('no support for machine ' + machinename) self.machine = machinename @@ -XXX,XX +XXX,XX @@ def require_accelerator(self, accelerator): "available" % accelerator) def require_netdev(self, netdevname): - netdevhelp = run_cmd([self.qemu_bin, - '-M', 'none', '-netdev', 'help'])[0]; - if netdevhelp.find('\n' + netdevname + '\n') < 0: + help = run([self.qemu_bin, + '-M', 'none', '-netdev', 'help'], + capture_output=True, check=True, encoding='utf8').stdout; + if help.find('\n' + netdevname + '\n') < 0: self.skipTest('no support for " + netdevname + " networking') def require_device(self, devicename): - devhelp = run_cmd([self.qemu_bin, - '-M', 'none', '-device', 'help'])[0]; - if devhelp.find(devicename) < 0: + help = run([self.qemu_bin, + '-M', 'none', '-device', 'help'], + capture_output=True, check=True, encoding='utf8').stdout; + if help.find(devicename) < 0: self.skipTest('no support for device ' + devicename) def _new_vm(self, name, *args): diff --git a/tests/functional/qemu_test/tuxruntest.py b/tests/functional/qemu_test/tuxruntest.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/tuxruntest.py +++ b/tests/functional/qemu_test/tuxruntest.py @@ -XXX,XX +XXX,XX @@ import os import stat +from subprocess import check_call, DEVNULL from qemu_test import QemuSystemTest from qemu_test import exec_command_and_wait_for_pattern from qemu_test import wait_for_console_pattern -from qemu_test import which, run_cmd, get_qemu_img +from qemu_test import which, get_qemu_img class TuxRunBaselineTest(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def fetch_tuxrun_assets(self, kernel_asset, rootfs_asset, dtb_asset=None): disk_image = self.scratch_file("rootfs.ext4") - run_cmd(['zstd', "-f", "-d", disk_image_zst, - "-o", disk_image]) + check_call(['zstd', "-f", "-d", disk_image_zst, + "-o", disk_image], + stdout=DEVNULL, stderr=DEVNULL) # zstd copies source archive permissions for the output # file, so must make this writable for QEMU os.chmod(disk_image, stat.S_IRUSR | stat.S_IWUSR) diff --git a/tests/functional/test_aarch64_virt.py b/tests/functional/test_aarch64_virt.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_aarch64_virt.py +++ b/tests/functional/test_aarch64_virt.py @@ -XXX,XX +XXX,XX @@ import time import logging +from subprocess import check_call, DEVNULL from qemu_test import QemuSystemTest, Asset from qemu_test import exec_command, wait_for_console_pattern -from qemu_test import get_qemu_img, run_cmd +from qemu_test import get_qemu_img class Aarch64VirtMachine(QemuSystemTest): @@ -XXX,XX +XXX,XX @@ def common_aarch64_virt(self, machine): logger.info('creating scratch qcow2 image') image_path = self.scratch_file('scratch.qcow2') qemu_img = get_qemu_img(self) - run_cmd([qemu_img, 'create', '-f', 'qcow2', image_path, '8M']) + check_call([qemu_img, 'create', '-f', 'qcow2', image_path, '8M'], + stdout=DEVNULL, stderr=DEVNULL) # Add the device self.vm.add_args('-blockdev', diff --git a/tests/functional/test_ppc64_tuxrun.py b/tests/functional/test_ppc64_tuxrun.py index XXXXXXX..XXXXXXX 100755 --- a/tests/functional/test_ppc64_tuxrun.py +++ b/tests/functional/test_ppc64_tuxrun.py @@ -XXX,XX +XXX,XX @@ # # SPDX-License-Identifier: GPL-2.0-or-later +from subprocess import check_call, DEVNULL import tempfile -from qemu_test import run_cmd, Asset +from qemu_test import Asset from qemu_test.tuxruntest import TuxRunBaselineTest class TuxRunPPC64Test(TuxRunBaselineTest): @@ -XXX,XX +XXX,XX @@ def ppc64_common_tuxrun(self, kernel_asset, rootfs_asset, prefix): # Create a temporary qcow2 and launch the test-case with tempfile.NamedTemporaryFile(prefix=prefix, suffix='.qcow2') as qcow2: - run_cmd([self.qemu_img, 'create', '-f', 'qcow2', qcow2.name, ' 1G']) + check_call([self.qemu_img, 'create', '-f', 'qcow2', + qcow2.name, ' 1G'], + stdout=DEVNULL, stderr=DEVNULL) self.vm.add_args('-drive', 'file=' + qcow2.name + ',format=qcow2,if=none,id=' -- 2.46.0 All usage has been replaced by direct 'subprocess' helpers. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/__init__.py | 2 +- tests/functional/qemu_test/cmd.py | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/functional/qemu_test/__init__.py b/tests/functional/qemu_test/__init__.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/__init__.py +++ b/tests/functional/qemu_test/__init__.py @@ -XXX,XX +XXX,XX @@ from .asset import Asset from .config import BUILD_DIR -from .cmd import run_cmd, is_readable_executable_file, \ +from .cmd import is_readable_executable_file, \ interrupt_interactive_console_until_pattern, wait_for_console_pattern, \ exec_command, exec_command_and_wait_for_pattern, get_qemu_img, which from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest diff --git a/tests/functional/qemu_test/cmd.py b/tests/functional/qemu_test/cmd.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/cmd.py +++ b/tests/functional/qemu_test/cmd.py @@ -XXX,XX +XXX,XX @@ import logging import os import os.path -import subprocess def which(tool): @@ -XXX,XX +XXX,XX @@ def which(tool): return p return None -def run_cmd(args): - subp = subprocess.Popen(args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) - stdout, stderr = subp.communicate() - ret = subp.returncode - - return (stdout, stderr, ret) - def is_readable_executable_file(path): return os.path.isfile(path) and os.access(path, os.R_OK | os.X_OK) -- 2.46.0 If downloading of assets has been disabled, then skip running a test if the assets it has registered are not already downloaded. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/asset.py | 8 +++++++- tests/functional/qemu_test/testcase.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/functional/qemu_test/asset.py b/tests/functional/qemu_test/asset.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/asset.py +++ b/tests/functional/qemu_test/asset.py @@ -XXX,XX +XXX,XX @@ def _check(self, cache_file): def valid(self): return self.cache_file.exists() and self._check(self.cache_file) + def fetchable(self): + return not os.environ.get("QEMU_TEST_NO_DOWNLOAD", False) + + def available(self): + return self.valid() or self.fetchable() + def _wait_for_other_download(self, tmp_cache_file): # Another thread already seems to download the asset, so wait until # it is done, while also checking the size to see whether it is stuck @@ -XXX,XX +XXX,XX @@ def fetch(self): self.cache_file, self.url) return str(self.cache_file) - if os.environ.get("QEMU_TEST_NO_DOWNLOAD", False): + if not self.fetchable(): raise Exception("Asset cache is invalid and downloads disabled") self.log.info("Downloading %s to %s...", self.url, self.cache_file) diff --git a/tests/functional/qemu_test/testcase.py b/tests/functional/qemu_test/testcase.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/testcase.py +++ b/tests/functional/qemu_test/testcase.py @@ -XXX,XX +XXX,XX @@ def scratch_file(self, *args): def log_file(self, *args): return str(Path(self.outputdir, *args)) + def assets_available(self): + for name, asset in vars(self.__class__).items(): + if name.startswith("ASSET_") and type(asset) == Asset: + if not asset.available(): + self.log.debug(f"Asset {asset.url} not available") + return False + return True + def setUp(self, bin_prefix): self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set') self.arch = self.qemu_bin.split('-')[-1] @@ -XXX,XX +XXX,XX @@ def setUp(self, bin_prefix): self.machinelog.setLevel(logging.DEBUG) self.machinelog.addHandler(self._log_fh) + if not self.assets_available(): + self.skipTest('One or more assets is not available') + def tearDown(self): if "QEMU_TEST_KEEP_SCRATCH" not in os.environ: shutil.rmtree(self.workdir) -- 2.46.0 We see periodic errors caching assets due to a combination of transient networking and server problems. With the previous patch to skip running a test when it has missing assets, we can now treat most cache download errors as non-fatal. Only HTTP 404 is retained as fatal, since it is a strong indicator of a fully broken test rather than a transient error. Reviewed-by: Thomas Huth <thuth@redhat.com> Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- tests/functional/qemu_test/asset.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/functional/qemu_test/asset.py b/tests/functional/qemu_test/asset.py index XXXXXXX..XXXXXXX 100644 --- a/tests/functional/qemu_test/asset.py +++ b/tests/functional/qemu_test/asset.py @@ -XXX,XX +XXX,XX @@ from time import sleep from pathlib import Path from shutil import copyfileobj +from urllib.error import HTTPError # Instances of this class must be declared as class level variables @@ -XXX,XX +XXX,XX @@ def precache_test(test): for name, asset in vars(test.__class__).items(): if name.startswith("ASSET_") and type(asset) == Asset: log.info("Attempting to cache '%s'" % asset) - asset.fetch() + try: + asset.fetch() + except HTTPError as e: + # Treat 404 as fatal, since it is highly likely to + # indicate a broken test rather than a transient + # server or networking problem + if e.code == 404: + raise + + log.debug(f"HTTP error {e.code} from {asset.url} " + + "skipping asset precache") + log.removeHandler(handler) def precache_suite(suite): -- 2.46.0 Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index XXXXXXX..XXXXXXX 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -XXX,XX +XXX,XX @@ W: https://cirrus-ci.com/github/qemu/qemu Functional testing framework M: Thomas Huth <thuth@redhat.com> R: Philippe Mathieu-Daudé <philmd@linaro.org> +R: Daniel P. Berrange <berrange@redhat.com> F: tests/functional/qemu_test/ Windows Hosted Continuous Integration -- 2.46.0