The powernv and pseries machines both provide hypervisor facilities,
which is a large and complicated set of features that don't get much
system testing in CI.

Add a new test case for these which runs QEMU KVM inside the target.
---
 tests/avocado/ppc_hv_tests.py | 173 ++++++++++++++++++++++++++++++++++
 1 file changed, 173 insertions(+)
 create mode 100644 tests/avocado/ppc_hv_tests.py

diff --git a/tests/avocado/ppc_hv_tests.py b/tests/avocado/ppc_hv_tests.py
new file mode 100644
index 0000000000..16aac1d35d
--- /dev/null
+++ b/tests/avocado/ppc_hv_tests.py
@@ -0,0 +1,173 @@
+# Tests that specifically try to exercise hypervisor features of the
+# target machines. powernv supports the Power hypervisor ISA, and
+# pseries supports the nested-HV hypervisor spec.
+#
+# Copyright (c) 2023 IBM Corporation
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or
+# later.  See the COPYING file in the top-level directory.
+
+from avocado import skipIf
+from avocado.utils import archive
+from avocado_qemu import QemuSystemTest
+from avocado_qemu import wait_for_console_pattern, exec_command
+import os
+import time
+import subprocess
+
+# Alpine is a light weight distro that supports QEMU. These tests boot
+# that on the machine then run a QEMU guest inside it in KVM mode,
+# that runs the same Alpine distro image.
+# QEMU packages are downloaded and installed on each test. That's not a
+# large download, but it may be more polite to create qcow2 image with
+# QEMU already installed and use that.
+class HypervisorTest(QemuSystemTest):
+
+    timeout = 600
+    KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 console=hvc0 '
+    panic_message = 'Kernel panic - not syncing'
+    good_message = 'VFS: Cannot open root device'
+
+    def extract_from_iso(self, iso, path):
+        """
+        Extracts a file from an iso file into the test workdir
+
+        :param iso: path to the iso file
+        :param path: path within the iso file of the file to be extracted
+        :returns: path of the extracted file
+        """
+        filename = os.path.basename(path)
+
+        cwd = os.getcwd()
+        os.chdir(self.workdir)
+
+        with open(filename, "w") as outfile:
+            cmd = "isoinfo -i %s -R -x %s" % (iso, path)
+            subprocess.run(cmd.split(), stdout=outfile)
+
+        os.chdir(cwd)
+
+        # Return complete path to extracted file.  Because callers to
+        # extract_from_iso() specify 'path' with a leading slash, it is
+        # necessary to use os.path.relpath() as otherwise os.path.join()
+        # interprets it as an absolute path and drops the self.workdir part.
+        return os.path.normpath(os.path.join(self.workdir, filename))
+
+    def do_start_alpine(self):
+        iso_url = ('https://dl-cdn.alpinelinux.org/alpine/v3.18/releases/ppc64le/alpine-standard-3.18.4-ppc64le.iso')
+
+        # Alpine use sha256 so I recalculated this myself
+        iso_sha256 = 'c26b8d3e17c2f3f0fed02b4b1296589c2390e6d5548610099af75300edd7b3ff'
+        iso_path = self.fetch_asset(iso_url, asset_hash=iso_sha256,
+                                    algorithm = "sha256")
+
+        vmlinuz = self.extract_from_iso(iso_path, '/boot/vmlinuz-lts')
+        initramfs = self.extract_from_iso(iso_path, '/boot/initramfs-lts')
+
+        self.vm.set_console()
+        self.vm.add_args("-kernel", vmlinuz)
+        self.vm.add_args("-initrd", initramfs)
+        kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
+                               'console=hvc0')
+        self.require_accelerator("tcg")
+
+        self.vm.add_args("-accel", "tcg,thread=multi")
+        self.vm.add_args("-smp", "4", "-m", "2g")
+        self.vm.add_args("-drive", f"file={iso_path},format=raw,if=none,id=drive0")
+#        self.vm.add_args("-drive", f"file={iso_path},format=raw")
+
+        self.vm.launch()
+        wait_for_console_pattern(self, 'Welcome to Alpine Linux 3.18')
+        exec_command(self, 'root')
+        wait_for_console_pattern(self, 'localhost login:')
+        wait_for_console_pattern(self, 'You may change this message by editing /etc/motd.')
+        exec_command(self, 'setup-alpine -qe')
+        wait_for_console_pattern(self, 'localhost:~#')
+
+    def do_stop_alpine(self):
+        exec_command(self, 'poweroff')
+        wait_for_console_pattern(self, 'alpine:~#')
+        self.vm.wait()
+
+    def do_setup_kvm(self):
+        exec_command(self, 'echo http://dl-cdn.alpinelinux.org/alpine/v3.18/main > /etc/apk/repositories')
+        wait_for_console_pattern(self, 'alpine:~#')
+        exec_command(self, 'echo http://dl-cdn.alpinelinux.org/alpine/v3.18/community >> /etc/apk/repositories')
+        wait_for_console_pattern(self, 'alpine:~#')
+        exec_command(self, 'apk update')
+        wait_for_console_pattern(self, 'alpine:~#')
+        exec_command(self, 'apk add qemu-system-ppc64')
+        wait_for_console_pattern(self, 'alpine:~#')
+        exec_command(self, 'modprobe kvm-hv')
+        wait_for_console_pattern(self, 'alpine:~#')
+
+    def do_test_kvm(self, hpt=False):
+        if hpt:
+            append = 'disable_radix'
+        else:
+            append = ''
+        exec_command(self, 'qemu-system-ppc64 -nographic -smp 2 -m 1g '
+                           '-machine pseries,x-vof=on,accel=kvm '
+                           '-machine cap-cfpc=broken,cap-sbbc=broken,'
+                                    'cap-ibs=broken,cap-ccf-assist=off '
+                           '-drive file=/dev/nvme0n1,format=raw,readonly=on '
+                           '-initrd /media/nvme0n1/boot/initramfs-lts '
+                           '-kernel /media/nvme0n1/boot/vmlinuz-lts '
+                           '-append \'usbcore.nousb ' + append + '\'')
+        # Alpine 3.18 kernel seems to crash in XHCI USB driver.
+        wait_for_console_pattern(self, 'Welcome to Alpine Linux 3.18')
+        exec_command(self, 'root')
+        wait_for_console_pattern(self, 'localhost login:')
+        wait_for_console_pattern(self, 'You may change this message by editing /etc/motd.')
+        exec_command(self, 'poweroff >& /dev/null')
+        wait_for_console_pattern(self, 'localhost:~#')
+        wait_for_console_pattern(self, 'reboot: Power down')
+        time.sleep(1)
+        exec_command(self, '') # console has strange issue after qemu exit
+        exec_command(self, 'reset')
+        exec_command(self, 'echo VM finished')
+        wait_for_console_pattern(self, 'VM finished')
+
+    @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
+    def test_hv_pseries(self):
+        """
+        :avocado: tags=arch:ppc64
+        :avocado: tags=machine:pseries
+        :avocado: tags=accel:tcg
+        """
+        self.vm.add_args('-device', 'nvme,serial=1234,drive=drive0')
+        self.vm.add_args("-machine", "x-vof=on,cap-nested-hv=on")
+        self.do_start_alpine()
+        self.do_setup_kvm()
+        self.do_test_kvm()
+        self.do_stop_alpine()
+
+    @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
+    def test_hv_pseries_kvm(self):
+        """
+        :avocado: tags=arch:ppc64
+        :avocado: tags=machine:pseries
+        :avocado: tags=accel:kvm
+        """
+        self.vm.add_args('-device', 'nvme,serial=1234,drive=drive0')
+        self.vm.add_args("-machine", "x-vof=on,cap-nested-hv=on")
+        self.do_start_alpine()
+        self.do_setup_kvm()
+        self.do_test_kvm()
+        self.do_stop_alpine()
+
+    @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
+    def test_hv_powernv(self):
+        """
+        :avocado: tags=arch:ppc64
+        :avocado: tags=machine:powernv
+        :avocado: tags=accel:tcg
+        """
+        self.vm.add_args('-device', 'nvme,bus=pcie.2,addr=0x0,serial=1234,drive=drive0',
+                         '-device', 'e1000e,netdev=net0,mac=C0:FF:EE:00:00:02,bus=pcie.0,addr=0x0',
+                         '-netdev', 'user,id=net0,hostfwd=::20022-:22,hostname=alpine')
+        self.do_start_alpine()
+        self.do_setup_kvm()
+        self.do_test_kvm()
+        self.do_test_kvm(True)
+        self.do_stop_alpine()
-- 
2.42.0

+ Anushree, Hari - FYI

On 10/10/23 13:22, Nicholas Piggin wrote:
> The powernv and pseries machines both provide hypervisor facilities,
> which is a large and complicated set of features that don't get much
> system testing in CI.
> 
> Add a new test case for these which runs QEMU KVM inside the target.
> ---
>   tests/avocado/ppc_hv_tests.py | 173 ++++++++++++++++++++++++++++++++++
>   1 file changed, 173 insertions(+)
>   create mode 100644 tests/avocado/ppc_hv_tests.py
> 
> diff --git a/tests/avocado/ppc_hv_tests.py b/tests/avocado/ppc_hv_tests.py
> new file mode 100644
> index 0000000000..16aac1d35d
> --- /dev/null
> +++ b/tests/avocado/ppc_hv_tests.py
> @@ -0,0 +1,173 @@
> +# Tests that specifically try to exercise hypervisor features of the
> +# target machines. powernv supports the Power hypervisor ISA, and
> +# pseries supports the nested-HV hypervisor spec.
> +#
> +# Copyright (c) 2023 IBM Corporation
> +#
> +# This work is licensed under the terms of the GNU GPL, version 2 or
> +# later.  See the COPYING file in the top-level directory.
> +
> +from avocado import skipIf
> +from avocado.utils import archive
> +from avocado_qemu import QemuSystemTest
> +from avocado_qemu import wait_for_console_pattern, exec_command
> +import os
> +import time
> +import subprocess
> +
> +# Alpine is a light weight distro that supports QEMU. These tests boot
> +# that on the machine then run a QEMU guest inside it in KVM mode,
> +# that runs the same Alpine distro image.
> +# QEMU packages are downloaded and installed on each test. That's not a
> +# large download, but it may be more polite to create qcow2 image with
> +# QEMU already installed and use that.
> +class HypervisorTest(QemuSystemTest):
> +
> +    timeout = 600
> +    KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 console=hvc0 '
> +    panic_message = 'Kernel panic - not syncing'
> +    good_message = 'VFS: Cannot open root device'
> +
> +    def extract_from_iso(self, iso, path):
> +        """
> +        Extracts a file from an iso file into the test workdir
> +
> +        :param iso: path to the iso file
> +        :param path: path within the iso file of the file to be extracted
> +        :returns: path of the extracted file
> +        """
> +        filename = os.path.basename(path)
> +
> +        cwd = os.getcwd()
> +        os.chdir(self.workdir)
> +
> +        with open(filename, "w") as outfile:
> +            cmd = "isoinfo -i %s -R -x %s" % (iso, path)
> +            subprocess.run(cmd.split(), stdout=outfile)
> +
> +        os.chdir(cwd)
> +
> +        # Return complete path to extracted file.  Because callers to
> +        # extract_from_iso() specify 'path' with a leading slash, it is
> +        # necessary to use os.path.relpath() as otherwise os.path.join()
> +        # interprets it as an absolute path and drops the self.workdir part.
> +        return os.path.normpath(os.path.join(self.workdir, filename))
> +
> +    def do_start_alpine(self):
> +        iso_url = ('https://dl-cdn.alpinelinux.org/alpine/v3.18/releases/ppc64le/alpine-standard-3.18.4-ppc64le.iso')
> +
> +        # Alpine use sha256 so I recalculated this myself
> +        iso_sha256 = 'c26b8d3e17c2f3f0fed02b4b1296589c2390e6d5548610099af75300edd7b3ff'
> +        iso_path = self.fetch_asset(iso_url, asset_hash=iso_sha256,
> +                                    algorithm = "sha256")
> +
> +        vmlinuz = self.extract_from_iso(iso_path, '/boot/vmlinuz-lts')
> +        initramfs = self.extract_from_iso(iso_path, '/boot/initramfs-lts')
> +
> +        self.vm.set_console()
> +        self.vm.add_args("-kernel", vmlinuz)
> +        self.vm.add_args("-initrd", initramfs)
> +        kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
> +                               'console=hvc0')

console=hvc0 was already added to KERNEL_COMMON_COMMAND_LINE during init.

> +        self.require_accelerator("tcg")
> +
> +        self.vm.add_args("-accel", "tcg,thread=multi")
> +        self.vm.add_args("-smp", "4", "-m", "2g")
> +        self.vm.add_args("-drive", f"file={iso_path},format=raw,if=none,id=drive0")
> +#        self.vm.add_args("-drive", f"file={iso_path},format=raw")

commented code can be removed.

> +
> +        self.vm.launch()
> +        wait_for_console_pattern(self, 'Welcome to Alpine Linux 3.18')
> +        exec_command(self, 'root')
> +        wait_for_console_pattern(self, 'localhost login:')
> +        wait_for_console_pattern(self, 'You may change this message by editing /etc/motd.')
> +        exec_command(self, 'setup-alpine -qe')
> +        wait_for_console_pattern(self, 'localhost:~#')
> +
> +    def do_stop_alpine(self):
> +        exec_command(self, 'poweroff')
> +        wait_for_console_pattern(self, 'alpine:~#')
> +        self.vm.wait()
> +
> +    def do_setup_kvm(self):
> +        exec_command(self, 'echo http://dl-cdn.alpinelinux.org/alpine/v3.18/main > /etc/apk/repositories')
> +        wait_for_console_pattern(self, 'alpine:~#')
> +        exec_command(self, 'echo http://dl-cdn.alpinelinux.org/alpine/v3.18/community >> /etc/apk/repositories')
> +        wait_for_console_pattern(self, 'alpine:~#')
> +        exec_command(self, 'apk update')
> +        wait_for_console_pattern(self, 'alpine:~#')
> +        exec_command(self, 'apk add qemu-system-ppc64')
> +        wait_for_console_pattern(self, 'alpine:~#')
> +        exec_command(self, 'modprobe kvm-hv')
> +        wait_for_console_pattern(self, 'alpine:~#')
> +
> +    def do_test_kvm(self, hpt=False):
> +        if hpt:
> +            append = 'disable_radix'
> +        else:
> +            append = ''
> +        exec_command(self, 'qemu-system-ppc64 -nographic -smp 2 -m 1g '
> +                           '-machine pseries,x-vof=on,accel=kvm '
> +                           '-machine cap-cfpc=broken,cap-sbbc=broken,'
> +                                    'cap-ibs=broken,cap-ccf-assist=off '
> +                           '-drive file=/dev/nvme0n1,format=raw,readonly=on '
> +                           '-initrd /media/nvme0n1/boot/initramfs-lts '
> +                           '-kernel /media/nvme0n1/boot/vmlinuz-lts '
> +                           '-append \'usbcore.nousb ' + append + '\'')
> +        # Alpine 3.18 kernel seems to crash in XHCI USB driver.
> +        wait_for_console_pattern(self, 'Welcome to Alpine Linux 3.18')
> +        exec_command(self, 'root')
> +        wait_for_console_pattern(self, 'localhost login:')
> +        wait_for_console_pattern(self, 'You may change this message by editing /etc/motd.')
> +        exec_command(self, 'poweroff >& /dev/null')
> +        wait_for_console_pattern(self, 'localhost:~#')
> +        wait_for_console_pattern(self, 'reboot: Power down')
> +        time.sleep(1)
> +        exec_command(self, '') # console has strange issue after qemu exit
> +        exec_command(self, 'reset')
> +        exec_command(self, 'echo VM finished')
> +        wait_for_console_pattern(self, 'VM finished')
> +
> +    @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
> +    def test_hv_pseries(self):
> +        """
> +        :avocado: tags=arch:ppc64
> +        :avocado: tags=machine:pseries
> +        :avocado: tags=accel:tcg
> +        """
> +        self.vm.add_args('-device', 'nvme,serial=1234,drive=drive0')
> +        self.vm.add_args("-machine", "x-vof=on,cap-nested-hv=on")
> +        self.do_start_alpine()
> +        self.do_setup_kvm()
> +        self.do_test_kvm()
> +        self.do_stop_alpine()
> +
> +    @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
> +    def test_hv_pseries_kvm(self):
> +        """
> +        :avocado: tags=arch:ppc64
> +        :avocado: tags=machine:pseries
> +        :avocado: tags=accel:kvm
> +        """
> +        self.vm.add_args('-device', 'nvme,serial=1234,drive=drive0')
> +        self.vm.add_args("-machine", "x-vof=on,cap-nested-hv=on")

Is this meant to launch an L0 with KVM accel on pseries with 
cap-nested-hv=on and then do another nested guest using KVM (with below 
steps). Is cap-nested-hv really supported on actual pseries LPARs yet?

regards,
Harsh

> +        self.do_start_alpine()
> +        self.do_setup_kvm()
> +        self.do_test_kvm()
> +        self.do_stop_alpine()
> +
> +    @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
> +    def test_hv_powernv(self):
> +        """
> +        :avocado: tags=arch:ppc64
> +        :avocado: tags=machine:powernv
> +        :avocado: tags=accel:tcg
> +        """
> +        self.vm.add_args('-device', 'nvme,bus=pcie.2,addr=0x0,serial=1234,drive=drive0',
> +                         '-device', 'e1000e,netdev=net0,mac=C0:FF:EE:00:00:02,bus=pcie.0,addr=0x0',
> +                         '-netdev', 'user,id=net0,hostfwd=::20022-:22,hostname=alpine')
> +        self.do_start_alpine()
> +        self.do_setup_kvm()
> +        self.do_test_kvm()
> +        self.do_test_kvm(True)
> +        self.do_stop_alpine()