1. Patchew
  2. Xen
  3. [PATCH v2 0/4] CI: updates to XTF CI runners
  4. View patch

[PATCH v2 1/4] CI: unify x86 XTF test runner

dmkhn@proton.me posted 4 patches 9 months, 3 weeks ago
There is a newer version of this series
[PATCH v2 1/4] CI: unify x86 XTF test runner
Posted by dmkhn@proton.me 9 months, 3 weeks ago 
From: Denis Mukhin <dmukhin@ford.com>

Add test runner script qemu-xtf.sh which is allows any XTF x86 test to be
easily executed. Test runner is invoked from the qemu-smoke* jobs with the
hardcoded parameters.

Each x86 XTF job lead time is reduced a bit since only the test-related code
is built, not the entire XTF project.

Add .gitignore to avoid committing test artifacts by mistake.

Signed-off-by: Denis Mukhin <dmukhin@ford.com>
---
Changes since v2:
- removed TOP
- dropped XEN_CONSOLE in favor of XEN_CMDLINE
- fixup for XTF_SRC_CONFIG
---
 automation/gitlab-ci/test.yaml                |   8 +-
 automation/scripts/.gitignore                 |   6 +
 .../scripts/include/configs/xtf-x86-64-config |   0
 automation/scripts/include/xtf-runner         | 131 ++++++++++++++++++
 automation/scripts/include/xtf-x86-64         |  31 +++++
 automation/scripts/qemu-smoke-x86-64.sh       |  26 ----
 automation/scripts/qemu-xtf.sh                |  28 ++++
 7 files changed, 200 insertions(+), 30 deletions(-)
 create mode 100644 automation/scripts/.gitignore
 create mode 100644 automation/scripts/include/configs/xtf-x86-64-config
 create mode 100644 automation/scripts/include/xtf-runner
 create mode 100644 automation/scripts/include/xtf-x86-64
 delete mode 100755 automation/scripts/qemu-smoke-x86-64.sh
 create mode 100755 automation/scripts/qemu-xtf.sh

diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
index 5ce445b78f..3adc841335 100644
--- a/automation/gitlab-ci/test.yaml
+++ b/automation/gitlab-ci/test.yaml
@@ -659,28 +659,28 @@ qemu-alpine-x86_64-gcc:
 qemu-smoke-x86-64-gcc:
   extends: .qemu-smoke-x86-64
   script:
-    - ./automation/scripts/qemu-smoke-x86-64.sh pv 2>&1 | tee ${LOGFILE}
+    - ./automation/scripts/qemu-xtf.sh x86-64 pv64 example 2>&1 | tee ${LOGFILE}
   needs:
     - debian-12-x86_64-gcc-debug
 
 qemu-smoke-x86-64-clang:
   extends: .qemu-smoke-x86-64
   script:
-    - ./automation/scripts/qemu-smoke-x86-64.sh pv 2>&1 | tee ${LOGFILE}
+    - ./automation/scripts/qemu-xtf.sh x86-64 pv64 example 2>&1 | tee ${LOGFILE}
   needs:
     - debian-12-x86_64-clang-debug
 
 qemu-smoke-x86-64-gcc-pvh:
   extends: .qemu-smoke-x86-64
   script:
-    - ./automation/scripts/qemu-smoke-x86-64.sh pvh 2>&1 | tee ${LOGFILE}
+    - ./automation/scripts/qemu-xtf.sh x86-64 hvm64 example 2>&1 | tee ${LOGFILE}
   needs:
     - debian-12-x86_64-gcc-debug
 
 qemu-smoke-x86-64-clang-pvh:
   extends: .qemu-smoke-x86-64
   script:
-    - ./automation/scripts/qemu-smoke-x86-64.sh pvh 2>&1 | tee ${LOGFILE}
+    - ./automation/scripts/qemu-xtf.sh x86-64 hvm64 example 2>&1 | tee ${LOGFILE}
   needs:
     - debian-12-x86_64-clang-debug
 
diff --git a/automation/scripts/.gitignore b/automation/scripts/.gitignore
new file mode 100644
index 0000000000..2f2d6e1ebd
--- /dev/null
+++ b/automation/scripts/.gitignore
@@ -0,0 +1,6 @@
+!include
+
+binaries
+smoke.serial
+xen
+xtf*/
diff --git a/automation/scripts/include/configs/xtf-x86-64-config b/automation/scripts/include/configs/xtf-x86-64-config
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/automation/scripts/include/xtf-runner b/automation/scripts/include/xtf-runner
new file mode 100644
index 0000000000..320eb65dc8
--- /dev/null
+++ b/automation/scripts/include/xtf-runner
@@ -0,0 +1,131 @@
+#!/bin/bash
+#
+# XTF test utilities.
+#
+# Environment variables:
+#   BOOT_MSG: Expected boot message
+#   FW_PREFIX: Firmware images path including '/' at the end
+#   PASSED: XTF test printout in case of a pass
+#   QEMU_PREFIX: QEMU path including '/' at the end
+#   TEST_LOG: Output log file
+#   UBOOT_CMD: U-Boot command line
+#   WORKDIR: Test working directory
+#   XEN_BINARY: Xen binary location
+#   XEN_CONSOLE: Xen console device name
+#   XTF_SRC_CONFIG: XTF config file
+#   XTF_SRC_BRANCH: XTF branch
+#   XTF_SRC_URI: XTF source code URI
+
+# Output log file
+TEST_LOG="${TEST_LOG:-${XEN_ROOT}/smoke.serial}"
+# XTF test printout in case of a pass
+PASSED="${PASSED:-Test result: SUCCESS}"
+# Expected boot message
+BOOT_MSG="${BOOT_MSG:-Latest ChangeSet: }"
+# Test working directory
+WORKDIR="${WORKDIR:-${XEN_ROOT}/binaries}"
+# XTF source code
+XTF_SRC_CONFIG="${XTF_SRC_CONFIG:-include/configs/xtf-${ARCH}-config}"
+
+function die()
+{
+    set +x
+    echo "FATAL: $*" >&2
+    exit 1
+}
+
+# Build an XTF test binary.
+# $1 Test variant.
+# $2 Test name.
+function xtf_build_binary()
+{
+    local xtf_variant=$1
+    local xtf_name=$2
+    local xtf_dir="xtf-${ARCH}"
+
+    # Crude check for local testing
+    if [ ! -d ${xtf_dir} ]; then
+        git clone ${XTF_SRC_URI} ${xtf_dir} -b ${XTF_SRC_BRANCH}
+    fi
+
+    make \
+        -C ${xtf_dir} \
+        -j$(nproc) \
+        $(tr '\n' ' ' < ${XTF_SRC_CONFIG}) \
+        TESTS=tests/${xtf_name}
+
+    export XTF_NAME="${xtf_name}"
+    export XTF_VARIANT="${xtf_variant}"
+    export XTF_WORKDIR="$(readlink -f ${xtf_dir})"
+    export XTF_BINARY="${XTF_WORKDIR}/tests/${xtf_name}/test-${xtf_variant}-${xtf_name}"
+}
+
+# Build Xen command line for running an XTF test.
+# $1 Test variant.
+# $2 Test name.
+function xtf_build_cmdline()
+{
+    local xtf_variant=$1
+    local xtf_name=$2
+    declare -a cmdline=()
+
+    cmdline+=("${XEN_CMDLINE}")
+
+    # NB: OK to have hvm64, which is x86-only variant
+    if [[ $xtf_variant == "hvm64" ]]; then
+        cmdline+=("dom0-iommu=none dom0=pvh")
+    fi
+
+    export XEN_CMDLINE="${cmdline[@]}"
+}
+
+# Build an XTF test environment.
+# $1 Test variant.
+# $2 Test name.
+function xtf_build_test()
+{
+    local v=$1
+    local xtf_name=$2
+    local xtf_variant=""
+
+    for x in ${XTF_SRC_VARIANTS}; do
+        if [[ "${x}" == "${v}" ]]; then
+            xtf_variant=${v}
+            break
+        fi
+    done
+    if [[ -z $xtf_variant ]]; then
+        die "unsupported test variant '$1', supported variants: ${XTF_SRC_VARIANTS}"
+    fi
+
+    xtf_build_binary ${xtf_variant} ${xtf_name}
+    xtf_build_cmdline ${xtf_variant} ${xtf_name}
+}
+
+# Execute an XTF test.
+function xtf_run_test()
+{
+    rm -f ${TEST_LOG}
+    export BOOT_MSG PASSED TEST_CMD TEST_LOG UBOOT_CMD
+    ./console.exp | sed 's/\r\+$//'
+}
+
+# Setup environment and run an XTF test.
+# $1 Test variant.
+# $2 Test name.
+function xtf_test()
+{
+    # Out: FW_*, QEMU_*, XEN_{BINARY,CONSOLE}, XTF_SRC_*
+    xtf_arch_prepare
+
+    # In: XTF_SRC_*
+    # OUt: XTF_{BINARY,NAME,VARIANT,WORKDIR} and XEN_CMDLINE
+    xtf_build_test $@
+
+    # In: FW_*, QEMU_*, XTF_*, XEN_*
+    # Out: BOOT_MSG, PASSED, TEST_{CMD,LOG}, UBOOT_CMD
+    xtf_arch_setup
+
+    # In: BOOT_MSG, PASSED, TEST_{CMD,LOG}, UBOOT_CMD
+    xtf_run_test
+}
diff --git a/automation/scripts/include/xtf-x86-64 b/automation/scripts/include/xtf-x86-64
new file mode 100644
index 0000000000..b1b0cc201e
--- /dev/null
+++ b/automation/scripts/include/xtf-x86-64
@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# XTF test utilities (x86_64).
+#
+
+# Arch-specific environment overrides.
+function xtf_arch_prepare()
+{
+    export FW_PREFIX="${FW_PREFIX:-}"
+    export QEMU_PREFIX="${QEMU_PREFIX:-}"
+    export XEN_BINARY="${XEN_BINARY:-${WORKDIR}/xen}"
+    export XEN_CMDLINE="${XEN_CMDLINE:-loglvl=all noreboot console_timestamps=boot console=com1}"
+    export XTF_SRC_BRANCH="${XTF_SRC_BRANCH:-master}"
+    export XTF_SRC_URI="${XTF_SRC_URI:-https://xenbits.xen.org/git-http/xtf.git}"
+    export XTF_SRC_VARIANTS="hvm64 pv64"
+}
+
+# Perform arch-specific XTF environment setup.
+function xtf_arch_setup()
+{
+    export TEST_CMD="${QEMU_PREFIX}qemu-system-x86_64 \
+        -no-reboot \
+        -nographic \
+        -monitor none \
+        -serial stdio \
+        -m 512 \
+        -kernel ${XEN_BINARY} \
+        -initrd ${XTF_BINARY} \
+        -append \"${XEN_CMDLINE}\" \
+    "
+}
diff --git a/automation/scripts/qemu-smoke-x86-64.sh b/automation/scripts/qemu-smoke-x86-64.sh
deleted file mode 100755
index da0c26cc2f..0000000000
--- a/automation/scripts/qemu-smoke-x86-64.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-set -ex -o pipefail
-
-# variant should be either pv or pvh
-variant=$1
-
-# Clone and build XTF
-git clone https://xenbits.xen.org/git-http/xtf.git
-cd xtf && make -j$(nproc) && cd -
-
-case $variant in
-    pvh) k=test-hvm64-example    extra="dom0-iommu=none dom0=pvh" ;;
-    *)   k=test-pv64-example     extra= ;;
-esac
-
-rm -f smoke.serial
-export TEST_CMD="qemu-system-x86_64 -nographic -kernel binaries/xen \
-        -initrd xtf/tests/example/$k \
-        -append \"loglvl=all console=com1 noreboot console_timestamps=boot $extra\" \
-        -m 512 -monitor none -serial stdio"
-
-export TEST_LOG="smoke.serial"
-export PASSED="Test result: SUCCESS"
-
-./automation/scripts/console.exp | sed 's/\r\+$//'
diff --git a/automation/scripts/qemu-xtf.sh b/automation/scripts/qemu-xtf.sh
new file mode 100755
index 0000000000..55c221b36d
--- /dev/null
+++ b/automation/scripts/qemu-xtf.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# XTF test runner (QEMU).
+#
+
+set -e -o pipefail
+
+if [ $# -lt 3 ]; then
+    echo "Usage: $(basename $0) ARCH XTF-VARIANT XTF-NAME"
+    exit 0
+fi
+
+export ARCH="$1"
+shift
+
+set -x
+
+export XEN_ROOT="${PWD}"
+cd $(dirname $0)
+
+source include/xtf-runner
+
+if [ ! -f "include/xtf-${ARCH}" ]; then
+    die "unsupported architecture '${ARCH}'"
+fi
+source include/xtf-${ARCH}
+
+xtf_test $@
-- 
2.34.1
Re: [PATCH v2 1/4] CI: unify x86 XTF test runner
Posted by Stefano Stabellini 9 months, 3 weeks ago 
On Sat, 19 Apr 2025, dmkhn@proton.me wrote:
> From: Denis Mukhin <dmukhin@ford.com>
> 
> Add test runner script qemu-xtf.sh which is allows any XTF x86 test to be
> easily executed. Test runner is invoked from the qemu-smoke* jobs with the
> hardcoded parameters.
> 
> Each x86 XTF job lead time is reduced a bit since only the test-related code
> is built, not the entire XTF project.
> 
> Add .gitignore to avoid committing test artifacts by mistake.
> 
> Signed-off-by: Denis Mukhin <dmukhin@ford.com>
> ---
> Changes since v2:
> - removed TOP
> - dropped XEN_CONSOLE in favor of XEN_CMDLINE
> - fixup for XTF_SRC_CONFIG

Much better!


> ---
>  automation/gitlab-ci/test.yaml                |   8 +-
>  automation/scripts/.gitignore                 |   6 +
>  .../scripts/include/configs/xtf-x86-64-config |   0
>  automation/scripts/include/xtf-runner         | 131 ++++++++++++++++++
>  automation/scripts/include/xtf-x86-64         |  31 +++++
>  automation/scripts/qemu-smoke-x86-64.sh       |  26 ----
>  automation/scripts/qemu-xtf.sh                |  28 ++++
>  7 files changed, 200 insertions(+), 30 deletions(-)
>  create mode 100644 automation/scripts/.gitignore
>  create mode 100644 automation/scripts/include/configs/xtf-x86-64-config
>  create mode 100644 automation/scripts/include/xtf-runner
>  create mode 100644 automation/scripts/include/xtf-x86-64
>  delete mode 100755 automation/scripts/qemu-smoke-x86-64.sh
>  create mode 100755 automation/scripts/qemu-xtf.sh
> 
> diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
> index 5ce445b78f..3adc841335 100644
> --- a/automation/gitlab-ci/test.yaml
> +++ b/automation/gitlab-ci/test.yaml
> @@ -659,28 +659,28 @@ qemu-alpine-x86_64-gcc:
>  qemu-smoke-x86-64-gcc:
>    extends: .qemu-smoke-x86-64
>    script:
> -    - ./automation/scripts/qemu-smoke-x86-64.sh pv 2>&1 | tee ${LOGFILE}
> +    - ./automation/scripts/qemu-xtf.sh x86-64 pv64 example 2>&1 | tee ${LOGFILE}
>    needs:
>      - debian-12-x86_64-gcc-debug
>  
>  qemu-smoke-x86-64-clang:
>    extends: .qemu-smoke-x86-64
>    script:
> -    - ./automation/scripts/qemu-smoke-x86-64.sh pv 2>&1 | tee ${LOGFILE}
> +    - ./automation/scripts/qemu-xtf.sh x86-64 pv64 example 2>&1 | tee ${LOGFILE}
>    needs:
>      - debian-12-x86_64-clang-debug
>  
>  qemu-smoke-x86-64-gcc-pvh:
>    extends: .qemu-smoke-x86-64
>    script:
> -    - ./automation/scripts/qemu-smoke-x86-64.sh pvh 2>&1 | tee ${LOGFILE}
> +    - ./automation/scripts/qemu-xtf.sh x86-64 hvm64 example 2>&1 | tee ${LOGFILE}
>    needs:
>      - debian-12-x86_64-gcc-debug
>  
>  qemu-smoke-x86-64-clang-pvh:
>    extends: .qemu-smoke-x86-64
>    script:
> -    - ./automation/scripts/qemu-smoke-x86-64.sh pvh 2>&1 | tee ${LOGFILE}
> +    - ./automation/scripts/qemu-xtf.sh x86-64 hvm64 example 2>&1 | tee ${LOGFILE}
>    needs:
>      - debian-12-x86_64-clang-debug
>  
> diff --git a/automation/scripts/.gitignore b/automation/scripts/.gitignore
> new file mode 100644
> index 0000000000..2f2d6e1ebd
> --- /dev/null
> +++ b/automation/scripts/.gitignore
> @@ -0,0 +1,6 @@
> +!include

Why !include ?


> +binaries
> +smoke.serial
> +xen
> +xtf*/
> diff --git a/automation/scripts/include/configs/xtf-x86-64-config b/automation/scripts/include/configs/xtf-x86-64-config
> new file mode 100644
> index 0000000000..e69de29bb2
> diff --git a/automation/scripts/include/xtf-runner b/automation/scripts/include/xtf-runner
> new file mode 100644
> index 0000000000..320eb65dc8
> --- /dev/null
> +++ b/automation/scripts/include/xtf-runner
> @@ -0,0 +1,131 @@
> +#!/bin/bash
> +#
> +# XTF test utilities.
> +#
> +# Environment variables:
> +#   BOOT_MSG: Expected boot message
> +#   FW_PREFIX: Firmware images path including '/' at the end
> +#   PASSED: XTF test printout in case of a pass
> +#   QEMU_PREFIX: QEMU path including '/' at the end
> +#   TEST_LOG: Output log file
> +#   UBOOT_CMD: U-Boot command line
> +#   WORKDIR: Test working directory
> +#   XEN_BINARY: Xen binary location
> +#   XEN_CONSOLE: Xen console device name
> +#   XTF_SRC_CONFIG: XTF config file
> +#   XTF_SRC_BRANCH: XTF branch
> +#   XTF_SRC_URI: XTF source code URI
> +
> +# Output log file
> +TEST_LOG="${TEST_LOG:-${XEN_ROOT}/smoke.serial}"
> +# XTF test printout in case of a pass
> +PASSED="${PASSED:-Test result: SUCCESS}"
> +# Expected boot message
> +BOOT_MSG="${BOOT_MSG:-Latest ChangeSet: }"
> +# Test working directory
> +WORKDIR="${WORKDIR:-${XEN_ROOT}/binaries}"
> +# XTF source code
> +XTF_SRC_CONFIG="${XTF_SRC_CONFIG:-include/configs/xtf-${ARCH}-config}"
> +
> +function die()
> +{
> +    set +x
> +    echo "FATAL: $*" >&2
> +    exit 1
> +}
> +
> +# Build an XTF test binary.
> +# $1 Test variant.
> +# $2 Test name.
> +function xtf_build_binary()
> +{
> +    local xtf_variant=$1
> +    local xtf_name=$2
> +    local xtf_dir="xtf-${ARCH}"
> +
> +    # Crude check for local testing
> +    if [ ! -d ${xtf_dir} ]; then
> +        git clone ${XTF_SRC_URI} ${xtf_dir} -b ${XTF_SRC_BRANCH}
> +    fi
> +
> +    make \
> +        -C ${xtf_dir} \
> +        -j$(nproc) \
> +        $(tr '\n' ' ' < ${XTF_SRC_CONFIG}) \
> +        TESTS=tests/${xtf_name}
> +
> +    export XTF_NAME="${xtf_name}"
> +    export XTF_VARIANT="${xtf_variant}"
> +    export XTF_WORKDIR="$(readlink -f ${xtf_dir})"
> +    export XTF_BINARY="${XTF_WORKDIR}/tests/${xtf_name}/test-${xtf_variant}-${xtf_name}"
> +}
> +
> +# Build Xen command line for running an XTF test.
> +# $1 Test variant.
> +# $2 Test name.
> +function xtf_build_cmdline()
> +{
> +    local xtf_variant=$1
> +    local xtf_name=$2
> +    declare -a cmdline=()
> +
> +    cmdline+=("${XEN_CMDLINE}")
> +
> +    # NB: OK to have hvm64, which is x86-only variant
> +    if [[ $xtf_variant == "hvm64" ]]; then
> +        cmdline+=("dom0-iommu=none dom0=pvh")
> +    fi
> +
> +    export XEN_CMDLINE="${cmdline[@]}"
> +}
> +
> +# Build an XTF test environment.
> +# $1 Test variant.
> +# $2 Test name.
> +function xtf_build_test()
> +{
> +    local v=$1
> +    local xtf_name=$2
> +    local xtf_variant=""
> +
> +    for x in ${XTF_SRC_VARIANTS}; do
> +        if [[ "${x}" == "${v}" ]]; then
> +            xtf_variant=${v}
> +            break
> +        fi
> +    done
> +    if [[ -z $xtf_variant ]]; then
> +        die "unsupported test variant '$1', supported variants: ${XTF_SRC_VARIANTS}"
> +    fi
> +
> +    xtf_build_binary ${xtf_variant} ${xtf_name}
> +    xtf_build_cmdline ${xtf_variant} ${xtf_name}
> +}
> +
> +# Execute an XTF test.
> +function xtf_run_test()
> +{
> +    rm -f ${TEST_LOG}
> +    export BOOT_MSG PASSED TEST_CMD TEST_LOG UBOOT_CMD
> +    ./console.exp | sed 's/\r\+$//'
> +}
> +
> +# Setup environment and run an XTF test.
> +# $1 Test variant.
> +# $2 Test name.
> +function xtf_test()
> +{
> +    # Out: FW_*, QEMU_*, XEN_{BINARY,CONSOLE}, XTF_SRC_*
> +    xtf_arch_prepare
> +
> +    # In: XTF_SRC_*
> +    # OUt: XTF_{BINARY,NAME,VARIANT,WORKDIR} and XEN_CMDLINE
> +    xtf_build_test $@
> +
> +    # In: FW_*, QEMU_*, XTF_*, XEN_*
> +    # Out: BOOT_MSG, PASSED, TEST_{CMD,LOG}, UBOOT_CMD
> +    xtf_arch_setup
> +
> +    # In: BOOT_MSG, PASSED, TEST_{CMD,LOG}, UBOOT_CMD
> +    xtf_run_test
> +}
> diff --git a/automation/scripts/include/xtf-x86-64 b/automation/scripts/include/xtf-x86-64
> new file mode 100644
> index 0000000000..b1b0cc201e
> --- /dev/null
> +++ b/automation/scripts/include/xtf-x86-64
> @@ -0,0 +1,31 @@
> +#!/bin/bash
> +#
> +# XTF test utilities (x86_64).
> +#
> +
> +# Arch-specific environment overrides.
> +function xtf_arch_prepare()
> +{
> +    export FW_PREFIX="${FW_PREFIX:-}"
> +    export QEMU_PREFIX="${QEMU_PREFIX:-}"
> +    export XEN_BINARY="${XEN_BINARY:-${WORKDIR}/xen}"
> +    export XEN_CMDLINE="${XEN_CMDLINE:-loglvl=all noreboot console_timestamps=boot console=com1}"
> +    export XTF_SRC_BRANCH="${XTF_SRC_BRANCH:-master}"
> +    export XTF_SRC_URI="${XTF_SRC_URI:-https://xenbits.xen.org/git-http/xtf.git}"
> +    export XTF_SRC_VARIANTS="hvm64 pv64"
> +}
> +
> +# Perform arch-specific XTF environment setup.
> +function xtf_arch_setup()
> +{
> +    export TEST_CMD="${QEMU_PREFIX}qemu-system-x86_64 \
> +        -no-reboot \
> +        -nographic \
> +        -monitor none \
> +        -serial stdio \
> +        -m 512 \
> +        -kernel ${XEN_BINARY} \
> +        -initrd ${XTF_BINARY} \
> +        -append \"${XEN_CMDLINE}\" \
> +    "
> +}
> diff --git a/automation/scripts/qemu-smoke-x86-64.sh b/automation/scripts/qemu-smoke-x86-64.sh
> deleted file mode 100755
> index da0c26cc2f..0000000000
> --- a/automation/scripts/qemu-smoke-x86-64.sh
> +++ /dev/null
> @@ -1,26 +0,0 @@
> -#!/bin/bash
> -
> -set -ex -o pipefail
> -
> -# variant should be either pv or pvh
> -variant=$1
> -
> -# Clone and build XTF
> -git clone https://xenbits.xen.org/git-http/xtf.git
> -cd xtf && make -j$(nproc) && cd -
> -
> -case $variant in
> -    pvh) k=test-hvm64-example    extra="dom0-iommu=none dom0=pvh" ;;
> -    *)   k=test-pv64-example     extra= ;;
> -esac
> -
> -rm -f smoke.serial
> -export TEST_CMD="qemu-system-x86_64 -nographic -kernel binaries/xen \
> -        -initrd xtf/tests/example/$k \
> -        -append \"loglvl=all console=com1 noreboot console_timestamps=boot $extra\" \
> -        -m 512 -monitor none -serial stdio"
> -
> -export TEST_LOG="smoke.serial"
> -export PASSED="Test result: SUCCESS"
> -
> -./automation/scripts/console.exp | sed 's/\r\+$//'
> diff --git a/automation/scripts/qemu-xtf.sh b/automation/scripts/qemu-xtf.sh
> new file mode 100755
> index 0000000000..55c221b36d
> --- /dev/null
> +++ b/automation/scripts/qemu-xtf.sh
> @@ -0,0 +1,28 @@
> +#!/bin/bash
> +#
> +# XTF test runner (QEMU).
> +#
> +
> +set -e -o pipefail
> +
> +if [ $# -lt 3 ]; then
> +    echo "Usage: $(basename $0) ARCH XTF-VARIANT XTF-NAME"
> +    exit 0
> +fi
> +
> +export ARCH="$1"
> +shift
> +
> +set -x
> +
> +export XEN_ROOT="${PWD}"
> +cd $(dirname $0)
> +
> +source include/xtf-runner
> +
> +if [ ! -f "include/xtf-${ARCH}" ]; then
> +    die "unsupported architecture '${ARCH}'"
> +fi
> +source include/xtf-${ARCH}
> +
> +xtf_test $@
> -- 
> 2.34.1
> 
>
Re: [PATCH v2 1/4] CI: unify x86 XTF test runner
Posted by dmkhn@proton.me 9 months, 3 weeks ago 
On Mon, Apr 21, 2025 at 01:39:21PM -0700, Stefano Stabellini wrote:
> On Sat, 19 Apr 2025, dmkhn@proton.me wrote:
> > From: Denis Mukhin <dmukhin@ford.com>
> >
> > Add test runner script qemu-xtf.sh which is allows any XTF x86 test to be
> > easily executed. Test runner is invoked from the qemu-smoke* jobs with the
> > hardcoded parameters.
> >
> > Each x86 XTF job lead time is reduced a bit since only the test-related code
> > is built, not the entire XTF project.
> >
> > Add .gitignore to avoid committing test artifacts by mistake.
> >
> > Signed-off-by: Denis Mukhin <dmukhin@ford.com>
> > ---
> > Changes since v2:
> > - removed TOP
> > - dropped XEN_CONSOLE in favor of XEN_CMDLINE
> > - fixup for XTF_SRC_CONFIG
> 
> Much better!
> 
> 
> > ---
> >  automation/gitlab-ci/test.yaml                |   8 +-
> >  automation/scripts/.gitignore                 |   6 +
> >  .../scripts/include/configs/xtf-x86-64-config |   0
> >  automation/scripts/include/xtf-runner         | 131 ++++++++++++++++++
> >  automation/scripts/include/xtf-x86-64         |  31 +++++
> >  automation/scripts/qemu-smoke-x86-64.sh       |  26 ----
> >  automation/scripts/qemu-xtf.sh                |  28 ++++
> >  7 files changed, 200 insertions(+), 30 deletions(-)
> >  create mode 100644 automation/scripts/.gitignore
> >  create mode 100644 automation/scripts/include/configs/xtf-x86-64-config
> >  create mode 100644 automation/scripts/include/xtf-runner
> >  create mode 100644 automation/scripts/include/xtf-x86-64
> >  delete mode 100755 automation/scripts/qemu-smoke-x86-64.sh
> >  create mode 100755 automation/scripts/qemu-xtf.sh
> >
> > diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
> > index 5ce445b78f..3adc841335 100644
> > --- a/automation/gitlab-ci/test.yaml
> > +++ b/automation/gitlab-ci/test.yaml
> > @@ -659,28 +659,28 @@ qemu-alpine-x86_64-gcc:
> >  qemu-smoke-x86-64-gcc:
> >    extends: .qemu-smoke-x86-64
> >    script:
> > -    - ./automation/scripts/qemu-smoke-x86-64.sh pv 2>&1 | tee ${LOGFILE}
> > +    - ./automation/scripts/qemu-xtf.sh x86-64 pv64 example 2>&1 | tee ${LOGFILE}
> >    needs:
> >      - debian-12-x86_64-gcc-debug
> >
> >  qemu-smoke-x86-64-clang:
> >    extends: .qemu-smoke-x86-64
> >    script:
> > -    - ./automation/scripts/qemu-smoke-x86-64.sh pv 2>&1 | tee ${LOGFILE}
> > +    - ./automation/scripts/qemu-xtf.sh x86-64 pv64 example 2>&1 | tee ${LOGFILE}
> >    needs:
> >      - debian-12-x86_64-clang-debug
> >
> >  qemu-smoke-x86-64-gcc-pvh:
> >    extends: .qemu-smoke-x86-64
> >    script:
> > -    - ./automation/scripts/qemu-smoke-x86-64.sh pvh 2>&1 | tee ${LOGFILE}
> > +    - ./automation/scripts/qemu-xtf.sh x86-64 hvm64 example 2>&1 | tee ${LOGFILE}
> >    needs:
> >      - debian-12-x86_64-gcc-debug
> >
> >  qemu-smoke-x86-64-clang-pvh:
> >    extends: .qemu-smoke-x86-64
> >    script:
> > -    - ./automation/scripts/qemu-smoke-x86-64.sh pvh 2>&1 | tee ${LOGFILE}
> > +    - ./automation/scripts/qemu-xtf.sh x86-64 hvm64 example 2>&1 | tee ${LOGFILE}
> >    needs:
> >      - debian-12-x86_64-clang-debug
> >
> > diff --git a/automation/scripts/.gitignore b/automation/scripts/.gitignore
> > new file mode 100644
> > index 0000000000..2f2d6e1ebd
> > --- /dev/null
> > +++ b/automation/scripts/.gitignore
> > @@ -0,0 +1,6 @@
> > +!include
> 
> Why !include ?

Forgot to clean this up, thanks.
In my local environment, I test like the following:

  cd automation/scripts
  WORKDIR=... ./qemu-xtf.sh x86-64 pv64 example

The XTF runner script will clone XTF repo at automation/scripts/xtf-$ARCH
which will clash with include/xtf-$ARCH fragments.

Fixed in v3.

> 
> 
> > +binaries
> > +smoke.serial
> > +xen
> > +xtf*/
> > diff --git a/automation/scripts/include/configs/xtf-x86-64-config b/automation/scripts/include/configs/xtf-x86-64-config
> > new file mode 100644
> > index 0000000000..e69de29bb2
> > diff --git a/automation/scripts/include/xtf-runner b/automation/scripts/include/xtf-runner
> > new file mode 100644
> > index 0000000000..320eb65dc8
> > --- /dev/null
> > +++ b/automation/scripts/include/xtf-runner
> > @@ -0,0 +1,131 @@
> > +#!/bin/bash
> > +#
> > +# XTF test utilities.
> > +#
> > +# Environment variables:
> > +#   BOOT_MSG: Expected boot message
> > +#   FW_PREFIX: Firmware images path including '/' at the end
> > +#   PASSED: XTF test printout in case of a pass
> > +#   QEMU_PREFIX: QEMU path including '/' at the end
> > +#   TEST_LOG: Output log file
> > +#   UBOOT_CMD: U-Boot command line
> > +#   WORKDIR: Test working directory
> > +#   XEN_BINARY: Xen binary location
> > +#   XEN_CONSOLE: Xen console device name
> > +#   XTF_SRC_CONFIG: XTF config file
> > +#   XTF_SRC_BRANCH: XTF branch
> > +#   XTF_SRC_URI: XTF source code URI
> > +
> > +# Output log file
> > +TEST_LOG="${TEST_LOG:-${XEN_ROOT}/smoke.serial}"
> > +# XTF test printout in case of a pass
> > +PASSED="${PASSED:-Test result: SUCCESS}"
> > +# Expected boot message
> > +BOOT_MSG="${BOOT_MSG:-Latest ChangeSet: }"
> > +# Test working directory
> > +WORKDIR="${WORKDIR:-${XEN_ROOT}/binaries}"
> > +# XTF source code
> > +XTF_SRC_CONFIG="${XTF_SRC_CONFIG:-include/configs/xtf-${ARCH}-config}"
> > +
> > +function die()
> > +{
> > +    set +x
> > +    echo "FATAL: $*" >&2
> > +    exit 1
> > +}
> > +
> > +# Build an XTF test binary.
> > +# $1 Test variant.
> > +# $2 Test name.
> > +function xtf_build_binary()
> > +{
> > +    local xtf_variant=$1
> > +    local xtf_name=$2
> > +    local xtf_dir="xtf-${ARCH}"
> > +
> > +    # Crude check for local testing
> > +    if [ ! -d ${xtf_dir} ]; then
> > +        git clone ${XTF_SRC_URI} ${xtf_dir} -b ${XTF_SRC_BRANCH}
> > +    fi
> > +
> > +    make \
> > +        -C ${xtf_dir} \
> > +        -j$(nproc) \
> > +        $(tr '\n' ' ' < ${XTF_SRC_CONFIG}) \
> > +        TESTS=tests/${xtf_name}
> > +
> > +    export XTF_NAME="${xtf_name}"
> > +    export XTF_VARIANT="${xtf_variant}"
> > +    export XTF_WORKDIR="$(readlink -f ${xtf_dir})"
> > +    export XTF_BINARY="${XTF_WORKDIR}/tests/${xtf_name}/test-${xtf_variant}-${xtf_name}"
> > +}
> > +
> > +# Build Xen command line for running an XTF test.
> > +# $1 Test variant.
> > +# $2 Test name.
> > +function xtf_build_cmdline()
> > +{
> > +    local xtf_variant=$1
> > +    local xtf_name=$2
> > +    declare -a cmdline=()
> > +
> > +    cmdline+=("${XEN_CMDLINE}")
> > +
> > +    # NB: OK to have hvm64, which is x86-only variant
> > +    if [[ $xtf_variant == "hvm64" ]]; then
> > +        cmdline+=("dom0-iommu=none dom0=pvh")
> > +    fi
> > +
> > +    export XEN_CMDLINE="${cmdline[@]}"
> > +}
> > +
> > +# Build an XTF test environment.
> > +# $1 Test variant.
> > +# $2 Test name.
> > +function xtf_build_test()
> > +{
> > +    local v=$1
> > +    local xtf_name=$2
> > +    local xtf_variant=""
> > +
> > +    for x in ${XTF_SRC_VARIANTS}; do
> > +        if [[ "${x}" == "${v}" ]]; then
> > +            xtf_variant=${v}
> > +            break
> > +        fi
> > +    done
> > +    if [[ -z $xtf_variant ]]; then
> > +        die "unsupported test variant '$1', supported variants: ${XTF_SRC_VARIANTS}"
> > +    fi
> > +
> > +    xtf_build_binary ${xtf_variant} ${xtf_name}
> > +    xtf_build_cmdline ${xtf_variant} ${xtf_name}
> > +}
> > +
> > +# Execute an XTF test.
> > +function xtf_run_test()
> > +{
> > +    rm -f ${TEST_LOG}
> > +    export BOOT_MSG PASSED TEST_CMD TEST_LOG UBOOT_CMD
> > +    ./console.exp | sed 's/\r\+$//'
> > +}
> > +
> > +# Setup environment and run an XTF test.
> > +# $1 Test variant.
> > +# $2 Test name.
> > +function xtf_test()
> > +{
> > +    # Out: FW_*, QEMU_*, XEN_{BINARY,CONSOLE}, XTF_SRC_*
> > +    xtf_arch_prepare
> > +
> > +    # In: XTF_SRC_*
> > +    # OUt: XTF_{BINARY,NAME,VARIANT,WORKDIR} and XEN_CMDLINE
> > +    xtf_build_test $@
> > +
> > +    # In: FW_*, QEMU_*, XTF_*, XEN_*
> > +    # Out: BOOT_MSG, PASSED, TEST_{CMD,LOG}, UBOOT_CMD
> > +    xtf_arch_setup
> > +
> > +    # In: BOOT_MSG, PASSED, TEST_{CMD,LOG}, UBOOT_CMD
> > +    xtf_run_test
> > +}
> > diff --git a/automation/scripts/include/xtf-x86-64 b/automation/scripts/include/xtf-x86-64
> > new file mode 100644
> > index 0000000000..b1b0cc201e
> > --- /dev/null
> > +++ b/automation/scripts/include/xtf-x86-64
> > @@ -0,0 +1,31 @@
> > +#!/bin/bash
> > +#
> > +# XTF test utilities (x86_64).
> > +#
> > +
> > +# Arch-specific environment overrides.
> > +function xtf_arch_prepare()
> > +{
> > +    export FW_PREFIX="${FW_PREFIX:-}"
> > +    export QEMU_PREFIX="${QEMU_PREFIX:-}"
> > +    export XEN_BINARY="${XEN_BINARY:-${WORKDIR}/xen}"
> > +    export XEN_CMDLINE="${XEN_CMDLINE:-loglvl=all noreboot console_timestamps=boot console=com1}"
> > +    export XTF_SRC_BRANCH="${XTF_SRC_BRANCH:-master}"
> > +    export XTF_SRC_URI="${XTF_SRC_URI:-https://xenbits.xen.org/git-http/xtf.git}"
> > +    export XTF_SRC_VARIANTS="hvm64 pv64"
> > +}
> > +
> > +# Perform arch-specific XTF environment setup.
> > +function xtf_arch_setup()
> > +{
> > +    export TEST_CMD="${QEMU_PREFIX}qemu-system-x86_64 \
> > +        -no-reboot \
> > +        -nographic \
> > +        -monitor none \
> > +        -serial stdio \
> > +        -m 512 \
> > +        -kernel ${XEN_BINARY} \
> > +        -initrd ${XTF_BINARY} \
> > +        -append \"${XEN_CMDLINE}\" \
> > +    "
> > +}
> > diff --git a/automation/scripts/qemu-smoke-x86-64.sh b/automation/scripts/qemu-smoke-x86-64.sh
> > deleted file mode 100755
> > index da0c26cc2f..0000000000
> > --- a/automation/scripts/qemu-smoke-x86-64.sh
> > +++ /dev/null
> > @@ -1,26 +0,0 @@
> > -#!/bin/bash
> > -
> > -set -ex -o pipefail
> > -
> > -# variant should be either pv or pvh
> > -variant=$1
> > -
> > -# Clone and build XTF
> > -git clone https://xenbits.xen.org/git-http/xtf.git
> > -cd xtf && make -j$(nproc) && cd -
> > -
> > -case $variant in
> > -    pvh) k=test-hvm64-example    extra="dom0-iommu=none dom0=pvh" ;;
> > -    *)   k=test-pv64-example     extra= ;;
> > -esac
> > -
> > -rm -f smoke.serial
> > -export TEST_CMD="qemu-system-x86_64 -nographic -kernel binaries/xen \
> > -        -initrd xtf/tests/example/$k \
> > -        -append \"loglvl=all console=com1 noreboot console_timestamps=boot $extra\" \
> > -        -m 512 -monitor none -serial stdio"
> > -
> > -export TEST_LOG="smoke.serial"
> > -export PASSED="Test result: SUCCESS"
> > -
> > -./automation/scripts/console.exp | sed 's/\r\+$//'
> > diff --git a/automation/scripts/qemu-xtf.sh b/automation/scripts/qemu-xtf.sh
> > new file mode 100755
> > index 0000000000..55c221b36d
> > --- /dev/null
> > +++ b/automation/scripts/qemu-xtf.sh
> > @@ -0,0 +1,28 @@
> > +#!/bin/bash
> > +#
> > +# XTF test runner (QEMU).
> > +#
> > +
> > +set -e -o pipefail
> > +
> > +if [ $# -lt 3 ]; then
> > +    echo "Usage: $(basename $0) ARCH XTF-VARIANT XTF-NAME"
> > +    exit 0
> > +fi
> > +
> > +export ARCH="$1"
> > +shift
> > +
> > +set -x
> > +
> > +export XEN_ROOT="${PWD}"
> > +cd $(dirname $0)
> > +
> > +source include/xtf-runner
> > +
> > +if [ ! -f "include/xtf-${ARCH}" ]; then
> > +    die "unsupported architecture '${ARCH}'"
> > +fi
> > +source include/xtf-${ARCH}
> > +
> > +xtf_test $@
> > --
> > 2.34.1
> >
> >

Patchew 2.1-devel-b67e4c2

© 2016 - 2026 Red Hat, Inc.