This extends the VBSA test to run the linux tests. The sysarch-acs
test suite does provide some pre-built images which is good because
the tests require a patched kernel. However due to the structure of
the image we need to jump one or two hoops to get something useful:
- download and double decompress (zip then xz) the image
- navigate grub to launch the Linux Execution Environment
- shutdown the system once tests are done
- extract the logs from the MSDOS file system and parse them
It does make the code a bit ugly but it works for me at least. So far
the subset of tests run is limited but that might be solved by adding
some more devices to the PCIe bus to exercise the SMMU behaviour.
Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
---
tests/functional/aarch64/test_virt_vbsa.py | 123 +++++++++++++++++++--
1 file changed, 112 insertions(+), 11 deletions(-)
diff --git a/tests/functional/aarch64/test_virt_vbsa.py b/tests/functional/aarch64/test_virt_vbsa.py
index 1dd4cecde1e..0456dfe9fd5 100755
--- a/tests/functional/aarch64/test_virt_vbsa.py
+++ b/tests/functional/aarch64/test_virt_vbsa.py
@@ -10,12 +10,15 @@
# SPDX-License-Identifier: GPL-2.0-or-later
import shutil
-from subprocess import check_call, DEVNULL
+import re
+from subprocess import check_call, check_output, DEVNULL
from qemu_test import QemuSystemTest, Asset
from qemu_test import get_qemu_img, skipIfMissingCommands
from qemu_test import wait_for_console_pattern
+from qemu_test import interrupt_interactive_console_until_pattern as int_until
from qemu_test import exec_command_and_wait_for_pattern as ec_and_wait
+from qemu_test import send_key
@skipIfMissingCommands("mformat", "mcopy", "mmd")
@@ -28,6 +31,20 @@ def wait_for_console_pattern(self, success_message, vm=None):
failure_message='FAILED',
vm=vm)
+ def append_firmware_blobs(self):
+ """
+ Setup QEMU firmware blobs for boot.
+ """
+ code_path = self.build_file('pc-bios', 'edk2-aarch64-code.fd')
+ vars_source = self.build_file('pc-bios', 'edk2-arm-vars.fd')
+ vars_path = self.scratch_file('vars.fd')
+ shutil.copy(vars_source, vars_path)
+
+ self.vm.add_args('-drive',
+ f'if=pflash,format=raw,readonly=on,file={code_path}')
+ self.vm.add_args('-drive', f'if=pflash,format=raw,file={vars_path}')
+
+
ASSET_VBSA_EFI = Asset(
'https://github.com/ARM-software/sysarch-acs/raw/refs/heads/main'
'/prebuilt_images/VBSA/v25.12_VBSA_0.7.0/Vbsa.efi',
@@ -45,20 +62,12 @@ def test_aarch64_vbsa_uefi_tests(self):
self.vm.set_console()
- # virt machine wi
+ # virt machine
self.set_machine('virt')
self.vm.add_args('-M', 'virt,gic-version=max,virtualization=on')
self.vm.add_args('-cpu', 'max', '-m', '1024')
- # We will use the QEMU firmware blobs to boot
- code_path = self.build_file('pc-bios', 'edk2-aarch64-code.fd')
- vars_source = self.build_file('pc-bios', 'edk2-arm-vars.fd')
- vars_path = self.scratch_file('vars.fd')
- shutil.copy(vars_source, vars_path)
-
- self.vm.add_args('-drive',
- f'if=pflash,format=raw,readonly=on,file={code_path}')
- self.vm.add_args('-drive', f'if=pflash,format=raw,file={vars_path}')
+ self.append_firmware_blobs()
# Build an EFI FAT32 file-system for the UEFI tests
vbsa_efi = self.ASSET_VBSA_EFI.fetch()
@@ -102,5 +111,97 @@ def test_aarch64_vbsa_uefi_tests(self):
self.wait_for_console_pattern('VBSA tests complete. Reset the system.')
+ ASSET_SYSREADY_IMAGE = Asset(
+ 'https://github.com/ARM-software/arm-systemready/'
+ 'releases/download/v25.10_SR_3.1.0/systemready_acs_live_image.img.xz.zip',
+ 'df2c359de15784b1da6a8e6f3c98a053ee38ac0b3f241ccea62e17db092eb03a')
+
+ ROOT_PROMPT = '/ # '
+ DOWN_KEY = "\x1b[B"
+
+ @skipIfMissingCommands("sfdisk", "jq")
+ def test_aarch64_vbsa_linux_tests(self):
+ """
+ Launch the Linux based VBSA test from the ACS prebuilt images.
+
+ We can use the pre-built images and then trigger the Linux
+ build and run the tests. We then need to slurp the results
+ from the partition.
+ """
+
+ self.vm.set_console()
+
+ # virt machine with SMMU
+ self.set_machine('virt')
+ self.vm.add_args('-M', 'virt,gic-version=max,virtualization=on,iommu=smmuv3')
+ self.vm.add_args('-cpu', 'max', '-m', '1024', '-smp', '4')
+
+ self.append_firmware_blobs()
+
+ # First fetch, decompress (twice) and prepare the disk image
+ # on an NVME device (the kernel only has drivers for that).
+ disk_image_zip = self.ASSET_SYSREADY_IMAGE.fetch()
+ disk_image_xz = self.uncompress(disk_image_zip,
+ target="sysacs.xz", format="zip")
+ disk_image = self.uncompress(disk_image_xz,
+ target="sysacs.img", format="xz")
+
+ self.vm.add_args('-device',
+ 'nvme,drive=hd,serial=QEMU_ROOT_SSD')
+ self.vm.add_args('-blockdev',
+ f'driver=raw,node-name=hd,file.driver=file,file.filename={disk_image}')
+
+ # Launch QEMU and wait for grub and select the "Linux
+ # Execution Environment" so we can launch the test.
+
+ self.vm.launch()
+ self.wait_for_console_pattern('Use the ^ and v keys to select which entry is highlighted')
+
+ # 3 down arrows then return to select entry
+ send_key(self, self.DOWN_KEY)
+ send_key(self, self.DOWN_KEY)
+ send_key(self, self.DOWN_KEY)
+
+ ec_and_wait(self, "\n", self.ROOT_PROMPT)
+
+ ec_and_wait(self, "/usr/bin/bsa.sh --skip B_REP_1,B_IEP_1,B_PCIe_11,B_MEM_06",
+ self.ROOT_PROMPT)
+
+ # Now we can shutdown
+ ec_and_wait(self, "halt -f", "reboot: System halted")
+ self.vm.shutdown()
+
+ # and extract the test logs
+ offset = int(check_output(f"sfdisk --json {disk_image} |"
+ "jq '.partitiontable.partitions[0].start * 512'",
+ shell = True))
+ bsa_app_res = self.scratch_file("BsaResultsApp.log")
+ bsa_kern_res = self.scratch_file("BsaResultsKernel.log")
+
+ check_call(["mcopy", "-i", f"{disk_image}@@{offset}",
+ f"::acs_results/Linux/BsaResultsApp.log", bsa_app_res])
+ check_call(["mcopy", "-i", f"{disk_image}@@{offset}",
+ f"::acs_results/Linux/BsaResultsKernel.log", bsa_kern_res])
+
+ # for now just check the kernel log for the result summary
+ test_result_re = re.compile(r"\[.*\]\s+(.+): Result:\s+(\w+)")
+ summary_re = re.compile(r"Total Tests Run =\s*(\d+), Tests Passed =\s*(\d+), Tests Failed =\s*(\d+)")
+
+ with open(bsa_kern_res, 'r') as f:
+ for line in f:
+ test_match = test_result_re.search(line)
+ if test_match:
+ desc = test_match.group(1)
+ status = test_match.group(2)
+ self.log.info(f"Test: {desc} status: {status}")
+
+ match = summary_re.search(line)
+ if match:
+ total, passed, failed = match.groups()
+
+ if int(failed) > 0:
+ self.fail(f"{failed} tests failed ({total})")
+
+
if __name__ == '__main__':
QemuSystemTest.main()
--
2.47.3
On 3/6/26 8:17 AM, Alex Bennée wrote:
> This extends the VBSA test to run the linux tests. The sysarch-acs
> test suite does provide some pre-built images which is good because
> the tests require a patched kernel. However due to the structure of
> the image we need to jump one or two hoops to get something useful:
>
> - download and double decompress (zip then xz) the image
> - navigate grub to launch the Linux Execution Environment
> - shutdown the system once tests are done
> - extract the logs from the MSDOS file system and parse them
>
> It does make the code a bit ugly but it works for me at least. So far
> the subset of tests run is limited but that might be solved by adding
> some more devices to the PCIe bus to exercise the SMMU behaviour.
>
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> ---
> tests/functional/aarch64/test_virt_vbsa.py | 123 +++++++++++++++++++--
> 1 file changed, 112 insertions(+), 11 deletions(-)
>
> diff --git a/tests/functional/aarch64/test_virt_vbsa.py b/tests/functional/aarch64/test_virt_vbsa.py
> index 1dd4cecde1e..0456dfe9fd5 100755
> --- a/tests/functional/aarch64/test_virt_vbsa.py
> +++ b/tests/functional/aarch64/test_virt_vbsa.py
> @@ -10,12 +10,15 @@
> # SPDX-License-Identifier: GPL-2.0-or-later
>
> import shutil
> -from subprocess import check_call, DEVNULL
> +import re
> +from subprocess import check_call, check_output, DEVNULL
>
> from qemu_test import QemuSystemTest, Asset
> from qemu_test import get_qemu_img, skipIfMissingCommands
> from qemu_test import wait_for_console_pattern
> +from qemu_test import interrupt_interactive_console_until_pattern as int_until
> from qemu_test import exec_command_and_wait_for_pattern as ec_and_wait
> +from qemu_test import send_key
>
>
> @skipIfMissingCommands("mformat", "mcopy", "mmd")
> @@ -28,6 +31,20 @@ def wait_for_console_pattern(self, success_message, vm=None):
> failure_message='FAILED',
> vm=vm)
>
> + def append_firmware_blobs(self):
> + """
> + Setup QEMU firmware blobs for boot.
> + """
> + code_path = self.build_file('pc-bios', 'edk2-aarch64-code.fd')
> + vars_source = self.build_file('pc-bios', 'edk2-arm-vars.fd')
> + vars_path = self.scratch_file('vars.fd')
> + shutil.copy(vars_source, vars_path)
> +
> + self.vm.add_args('-drive',
> + f'if=pflash,format=raw,readonly=on,file={code_path}')
> + self.vm.add_args('-drive', f'if=pflash,format=raw,file={vars_path}')
> +
> +
> ASSET_VBSA_EFI = Asset(
> 'https://github.com/ARM-software/sysarch-acs/raw/refs/heads/main'
> '/prebuilt_images/VBSA/v25.12_VBSA_0.7.0/Vbsa.efi',
> @@ -45,20 +62,12 @@ def test_aarch64_vbsa_uefi_tests(self):
>
> self.vm.set_console()
>
> - # virt machine wi
> + # virt machine
> self.set_machine('virt')
> self.vm.add_args('-M', 'virt,gic-version=max,virtualization=on')
> self.vm.add_args('-cpu', 'max', '-m', '1024')
>
> - # We will use the QEMU firmware blobs to boot
> - code_path = self.build_file('pc-bios', 'edk2-aarch64-code.fd')
> - vars_source = self.build_file('pc-bios', 'edk2-arm-vars.fd')
> - vars_path = self.scratch_file('vars.fd')
> - shutil.copy(vars_source, vars_path)
> -
> - self.vm.add_args('-drive',
> - f'if=pflash,format=raw,readonly=on,file={code_path}')
> - self.vm.add_args('-drive', f'if=pflash,format=raw,file={vars_path}')
> + self.append_firmware_blobs()
>
> # Build an EFI FAT32 file-system for the UEFI tests
> vbsa_efi = self.ASSET_VBSA_EFI.fetch()
> @@ -102,5 +111,97 @@ def test_aarch64_vbsa_uefi_tests(self):
> self.wait_for_console_pattern('VBSA tests complete. Reset the system.')
>
>
> + ASSET_SYSREADY_IMAGE = Asset(
> + 'https://github.com/ARM-software/arm-systemready/'
> + 'releases/download/v25.10_SR_3.1.0/systemready_acs_live_image.img.xz.zip',
> + 'df2c359de15784b1da6a8e6f3c98a053ee38ac0b3f241ccea62e17db092eb03a')
> +
> + ROOT_PROMPT = '/ # '
> + DOWN_KEY = "\x1b[B"
> +
> + @skipIfMissingCommands("sfdisk", "jq")
> + def test_aarch64_vbsa_linux_tests(self):
> + """
> + Launch the Linux based VBSA test from the ACS prebuilt images.
> +
> + We can use the pre-built images and then trigger the Linux
> + build and run the tests. We then need to slurp the results
> + from the partition.
> + """
> +
> + self.vm.set_console()
> +
> + # virt machine with SMMU
> + self.set_machine('virt')
> + self.vm.add_args('-M', 'virt,gic-version=max,virtualization=on,iommu=smmuv3')
> + self.vm.add_args('-cpu', 'max', '-m', '1024', '-smp', '4')
> +
> + self.append_firmware_blobs()
> +
> + # First fetch, decompress (twice) and prepare the disk image
> + # on an NVME device (the kernel only has drivers for that).
> + disk_image_zip = self.ASSET_SYSREADY_IMAGE.fetch()
> + disk_image_xz = self.uncompress(disk_image_zip,
> + target="sysacs.xz", format="zip")
> + disk_image = self.uncompress(disk_image_xz,
> + target="sysacs.img", format="xz")
> +
> + self.vm.add_args('-device',
> + 'nvme,drive=hd,serial=QEMU_ROOT_SSD')
> + self.vm.add_args('-blockdev',
> + f'driver=raw,node-name=hd,file.driver=file,file.filename={disk_image}')
> +
> + # Launch QEMU and wait for grub and select the "Linux
> + # Execution Environment" so we can launch the test.
> +
> + self.vm.launch()
> + self.wait_for_console_pattern('Use the ^ and v keys to select which entry is highlighted')
> +
> + # 3 down arrows then return to select entry
> + send_key(self, self.DOWN_KEY)
> + send_key(self, self.DOWN_KEY)
> + send_key(self, self.DOWN_KEY)
> +
As a suggestion,
Alternatively, is it possible to modify the image to select a different
kind of command?
I have no idea which bootloader they use, but if it's grub or simply a
UEFI shell script, it should be quite easy to generate the right entry.
> + ec_and_wait(self, "\n", self.ROOT_PROMPT)
> +
> + ec_and_wait(self, "/usr/bin/bsa.sh --skip B_REP_1,B_IEP_1,B_PCIe_11,B_MEM_06",
> + self.ROOT_PROMPT)
> +
> + # Now we can shutdown
> + ec_and_wait(self, "halt -f", "reboot: System halted")
> + self.vm.shutdown()
> +
> + # and extract the test logs
> + offset = int(check_output(f"sfdisk --json {disk_image} |"
> + "jq '.partitiontable.partitions[0].start * 512'",
> + shell = True))
> + bsa_app_res = self.scratch_file("BsaResultsApp.log")
> + bsa_kern_res = self.scratch_file("BsaResultsKernel.log")
> +
> + check_call(["mcopy", "-i", f"{disk_image}@@{offset}",
> + f"::acs_results/Linux/BsaResultsApp.log", bsa_app_res])
> + check_call(["mcopy", "-i", f"{disk_image}@@{offset}",
> + f"::acs_results/Linux/BsaResultsKernel.log", bsa_kern_res])
> +
> + # for now just check the kernel log for the result summary
> + test_result_re = re.compile(r"\[.*\]\s+(.+): Result:\s+(\w+)")
> + summary_re = re.compile(r"Total Tests Run =\s*(\d+), Tests Passed =\s*(\d+), Tests Failed =\s*(\d+)")
> +
> + with open(bsa_kern_res, 'r') as f:
> + for line in f:
> + test_match = test_result_re.search(line)
> + if test_match:
> + desc = test_match.group(1)
> + status = test_match.group(2)
> + self.log.info(f"Test: {desc} status: {status}")
> +
> + match = summary_re.search(line)
> + if match:
> + total, passed, failed = match.groups()
> +
> + if int(failed) > 0:
> + self.fail(f"{failed} tests failed ({total})")
> +
> +
> if __name__ == '__main__':
> QemuSystemTest.main()
Reviewed-by: Pierrick Bouvier <pierrick.bouvier@linaro.org>
© 2016 - 2026 Red Hat, Inc.