automation: use expect to run QEMU

[PATCH v2] automation: use expect to run QEMU

Posted by Stefano Stabellini 3 months, 1 week ago

Use expect to invoke QEMU so that we can terminate the test as soon as
we get the right string in the output instead of waiting until the
final timeout.

For timeout, instead of an hardcoding the value, use a Gitlab CI
variable "QEMU_TIMEOUT" that can be changed depending on the latest
status of the Gitlab CI runners.

Note that for simplicity in the case of dom0less test, the script only
checks for the $PASSED string. That is because the dom0 prompt and the
$PASSED string could happen in any order making it difficult to check
with expect which checks for strings in a specific order.

Signed-off-by: Stefano Stabellini <stefano.stabellini@amd.com>

---
Changes in v2:
- add empty lines for readability
- changes qemu-key file extension to exp
- drop xen_msg that is unused
- use capital letters for exported variables
- check for both dom0 and domU message on dom0less tests

https://gitlab.com/xen-project/people/sstabellini/xen/-/pipelines/1410820286
---
 automation/scripts/qemu-alpine-x86_64.sh      | 16 +++---
 automation/scripts/qemu-key.exp               | 51 +++++++++++++++++++
 automation/scripts/qemu-smoke-dom0-arm32.sh   | 15 +++---
 automation/scripts/qemu-smoke-dom0-arm64.sh   | 15 +++---
 .../scripts/qemu-smoke-dom0less-arm32.sh      | 17 +++----
 .../scripts/qemu-smoke-dom0less-arm64.sh      | 15 +++---
 automation/scripts/qemu-smoke-ppc64le.sh      | 12 ++---
 automation/scripts/qemu-smoke-riscv64.sh      | 12 ++---
 automation/scripts/qemu-smoke-x86-64.sh       | 14 ++---
 automation/scripts/qemu-xtf-dom0less-arm64.sh | 14 +++--
 10 files changed, 110 insertions(+), 71 deletions(-)
 create mode 100755 automation/scripts/qemu-key.exp

diff --git a/automation/scripts/qemu-alpine-x86_64.sh b/automation/scripts/qemu-alpine-x86_64.sh
index 8e398dcea3..5359e0820b 100755
--- a/automation/scripts/qemu-alpine-x86_64.sh
+++ b/automation/scripts/qemu-alpine-x86_64.sh
@@ -77,18 +77,16 @@ EOF
 # Run the test
 rm -f smoke.serial
 set +e
-timeout -k 1 720 \
-qemu-system-x86_64 \
+export QEMU_CMD="qemu-system-x86_64 \
     -cpu qemu64,+svm \
     -m 2G -smp 2 \
     -monitor none -serial stdio \
     -nographic \
     -device virtio-net-pci,netdev=n0 \
-    -netdev user,id=n0,tftp=binaries,bootfile=/pxelinux.0 |& \
-        # Remove carriage returns from the stdout output, as gitlab
-        # interface chokes on them
-        tee smoke.serial | sed 's/\r//'
+    -netdev user,id=n0,tftp=binaries,bootfile=/pxelinux.0"
 
-set -e
-(grep -q "Domain-0" smoke.serial && grep -q "BusyBox" smoke.serial) || exit 1
-exit 0
+export QEMU_LOG="smoke.serial"
+export LOG_MSG="Domain-0"
+export PASSED="BusyBox"
+
+./automation/scripts/qemu-key.exp
diff --git a/automation/scripts/qemu-key.exp b/automation/scripts/qemu-key.exp
new file mode 100755
index 0000000000..9f3a3364fa
--- /dev/null
+++ b/automation/scripts/qemu-key.exp
@@ -0,0 +1,51 @@
+#!/usr/bin/expect -f
+
+set timeout $env(QEMU_TIMEOUT)
+
+log_file -a $env(QEMU_LOG)
+
+match_max 10000
+
+eval spawn $env(QEMU_CMD)
+
+expect_after {
+    -re "(.*)\r" {
+        exp_continue
+    }
+    timeout {send_user "ERROR-Timeout!\n"; exit 1}
+    eof {send_user "ERROR-EOF!\n"; exit 1}
+}
+
+if {[info exists env(UBOOT_CMD)]} {
+    expect "=>"
+
+    send "$env(UBOOT_CMD)\r"
+}
+
+if {[info exists env(LOG_MSG)] && [info exists env(PASSED)]} {
+    expect {
+        "$env(PASSED)" {
+            expect "$env(LOG_MSG)"
+            exit 0
+        }
+        "$env(LOG_MSG)" {
+            expect "$env(PASSED)"
+            exit 0
+        }
+    }
+}
+
+if {[info exists env(LOG_MSG)]} {
+    expect "$env(LOG_MSG)"
+}
+
+if {[info exists env(PASSED)]} {
+    expect {
+        "$env(PASSED)" {
+            exit 0
+        }
+    }
+}
+
+expect eof
+
diff --git a/automation/scripts/qemu-smoke-dom0-arm32.sh b/automation/scripts/qemu-smoke-dom0-arm32.sh
index eaaea5a982..b235b1476a 100755
--- a/automation/scripts/qemu-smoke-dom0-arm32.sh
+++ b/automation/scripts/qemu-smoke-dom0-arm32.sh
@@ -78,9 +78,7 @@ bash imagebuilder/scripts/uboot-script-gen -t tftp -d . -c config
 
 rm -f ${serial_log}
 set +e
-echo "  virtio scan; dhcp; tftpb 0x40000000 boot.scr; source 0x40000000"| \
-timeout -k 1 720 \
-./qemu-system-arm \
+export QEMU_CMD="./qemu-system-arm \
    -machine virt \
    -machine virtualization=true \
    -smp 4 \
@@ -91,9 +89,10 @@ timeout -k 1 720 \
    -no-reboot \
    -device virtio-net-pci,netdev=n0 \
    -netdev user,id=n0,tftp=./ \
-   -bios /usr/lib/u-boot/qemu_arm/u-boot.bin |& \
-      tee ${serial_log} | sed 's/\r//'
+   -bios /usr/lib/u-boot/qemu_arm/u-boot.bin"
 
-set -e
-(grep -q "Domain-0" ${serial_log} && grep -q "^/ #" ${serial_log}) || exit 1
-exit 0
+export UBOOT_CMD="virtio scan; dhcp; tftpb 0x40000000 boot.scr; source 0x40000000"
+export QEMU_LOG="${serial_log}"
+export LOG_MSG="Domain-0"
+export PASSED="/ #"
+../automation/scripts/qemu-key.exp
diff --git a/automation/scripts/qemu-smoke-dom0-arm64.sh b/automation/scripts/qemu-smoke-dom0-arm64.sh
index 352963a741..5e12b136d5 100755
--- a/automation/scripts/qemu-smoke-dom0-arm64.sh
+++ b/automation/scripts/qemu-smoke-dom0-arm64.sh
@@ -94,9 +94,7 @@ bash imagebuilder/scripts/uboot-script-gen -t tftp -d binaries/ -c binaries/conf
 # Run the test
 rm -f smoke.serial
 set +e
-echo "  virtio scan; dhcp; tftpb 0x40000000 boot.scr; source 0x40000000"| \
-timeout -k 1 720 \
-./binaries/qemu-system-aarch64 \
+export QEMU_CMD="./binaries/qemu-system-aarch64 \
     -machine virtualization=true \
     -cpu cortex-a57 -machine type=virt \
     -m 2048 -monitor none -serial stdio \
@@ -104,9 +102,10 @@ timeout -k 1 720 \
     -no-reboot \
     -device virtio-net-pci,netdev=n0 \
     -netdev user,id=n0,tftp=binaries \
-    -bios /usr/lib/u-boot/qemu_arm64/u-boot.bin |& \
-        tee smoke.serial | sed 's/\r//'
+    -bios /usr/lib/u-boot/qemu_arm64/u-boot.bin"
 
-set -e
-(grep -q "Domain-0" smoke.serial && grep -q "BusyBox" smoke.serial) || exit 1
-exit 0
+export UBOOT_CMD="virtio scan; dhcp; tftpb 0x40000000 boot.scr; source 0x40000000"
+export QEMU_LOG="smoke.serial"
+export LOG_MSG="Domain-0"
+export PASSED="BusyBox"
+./automation/scripts/qemu-key.exp
diff --git a/automation/scripts/qemu-smoke-dom0less-arm32.sh b/automation/scripts/qemu-smoke-dom0less-arm32.sh
index c027c8c5c8..5cdf9c4365 100755
--- a/automation/scripts/qemu-smoke-dom0less-arm32.sh
+++ b/automation/scripts/qemu-smoke-dom0less-arm32.sh
@@ -5,7 +5,7 @@ set -ex
 test_variant=$1
 
 # Prompt to grep for to check if dom0 booted successfully
-dom0_prompt="^/ #"
+dom0_prompt="/ #"
 
 serial_log="$(pwd)/smoke.serial"
 
@@ -131,9 +131,7 @@ bash imagebuilder/scripts/uboot-script-gen -t tftp -d . -c config
 # Run the test
 rm -f ${serial_log}
 set +e
-echo "  virtio scan; dhcp; tftpb 0x40000000 boot.scr; source 0x40000000"| \
-timeout -k 1 240 \
-./qemu-system-arm \
+export QEMU_CMD="./qemu-system-arm \
     -machine virt \
     -machine virtualization=true \
     -smp 4 \
@@ -144,9 +142,10 @@ timeout -k 1 240 \
     -no-reboot \
     -device virtio-net-pci,netdev=n0 \
     -netdev user,id=n0,tftp=./ \
-    -bios /usr/lib/u-boot/qemu_arm/u-boot.bin |& \
-        tee ${serial_log} | sed 's/\r//'
+    -bios /usr/lib/u-boot/qemu_arm/u-boot.bin"
 
-set -e
-(grep -q "${dom0_prompt}" ${serial_log} && grep -q "${passed}" ${serial_log}) || exit 1
-exit 0
+export UBOOT_CMD="virtio scan; dhcp; tftpb 0x40000000 boot.scr; source 0x40000000"
+export QEMU_LOG="${serial_log}"
+export LOG_MSG="${dom0_prompt}"
+export PASSED="${passed}"
+../automation/scripts/qemu-key.exp
diff --git a/automation/scripts/qemu-smoke-dom0less-arm64.sh b/automation/scripts/qemu-smoke-dom0less-arm64.sh
index 15258692d5..1290ac95a6 100755
--- a/automation/scripts/qemu-smoke-dom0less-arm64.sh
+++ b/automation/scripts/qemu-smoke-dom0less-arm64.sh
@@ -205,9 +205,7 @@ bash imagebuilder/scripts/uboot-script-gen -t tftp -d binaries/ -c binaries/conf
 # Run the test
 rm -f smoke.serial
 set +e
-echo "  virtio scan; dhcp; tftpb 0x40000000 boot.scr; source 0x40000000"| \
-timeout -k 1 240 \
-./binaries/qemu-system-aarch64 \
+export QEMU_CMD="./binaries/qemu-system-aarch64 \
     -machine virtualization=true \
     -cpu cortex-a57 -machine type=virt,gic-version=$gic_version \
     -m 2048 -monitor none -serial stdio \
@@ -215,9 +213,10 @@ timeout -k 1 240 \
     -no-reboot \
     -device virtio-net-pci,netdev=n0 \
     -netdev user,id=n0,tftp=binaries \
-    -bios /usr/lib/u-boot/qemu_arm64/u-boot.bin |& \
-        tee smoke.serial | sed 's/\r//'
+    -bios /usr/lib/u-boot/qemu_arm64/u-boot.bin"
 
-set -e
-(grep -q "^Welcome to Alpine Linux" smoke.serial && grep -q "${passed}" smoke.serial) || exit 1
-exit 0
+export UBOOT_CMD="virtio scan; dhcp; tftpb 0x40000000 boot.scr; source 0x40000000"
+export QEMU_LOG="smoke.serial"
+export LOG_MSG="Welcome to Alpine Linux"
+export PASSED="${passed}"
+./automation/scripts/qemu-key.exp
diff --git a/automation/scripts/qemu-smoke-ppc64le.sh b/automation/scripts/qemu-smoke-ppc64le.sh
index 9088881b73..e411a6a504 100755
--- a/automation/scripts/qemu-smoke-ppc64le.sh
+++ b/automation/scripts/qemu-smoke-ppc64le.sh
@@ -11,8 +11,7 @@ machine=$1
 rm -f ${serial_log}
 set +e
 
-timeout -k 1 20 \
-qemu-system-ppc64 \
+export QEMU_CMD="qemu-system-ppc64 \
     -bios skiboot.lid \
     -M $machine \
     -m 2g \
@@ -21,9 +20,8 @@ qemu-system-ppc64 \
     -monitor none \
     -nographic \
     -serial stdio \
-    -kernel binaries/xen \
-    |& tee ${serial_log} | sed 's/\r//'
+    -kernel binaries/xen"
 
-set -e
-(grep -q "Hello, ppc64le!" ${serial_log}) || exit 1
-exit 0
+export QEMU_LOG="${serial_log}"
+export PASSED="Hello, ppc64le!"
+./automation/scripts/qemu-key.exp
diff --git a/automation/scripts/qemu-smoke-riscv64.sh b/automation/scripts/qemu-smoke-riscv64.sh
index f90df3c051..1b1f5b3bf2 100755
--- a/automation/scripts/qemu-smoke-riscv64.sh
+++ b/automation/scripts/qemu-smoke-riscv64.sh
@@ -6,15 +6,13 @@ set -ex
 rm -f smoke.serial
 set +e
 
-timeout -k 1 2 \
-qemu-system-riscv64 \
+export QEMU_CMD="qemu-system-riscv64 \
     -M virt \
     -smp 1 \
     -nographic \
     -m 2g \
-    -kernel binaries/xen \
-    |& tee smoke.serial | sed 's/\r//'
+    -kernel binaries/xen"
 
-set -e
-(grep -q "All set up" smoke.serial) || exit 1
-exit 0
+export QEMU_LOG="smoke.serial"
+export PASSED="All set up"
+./automation/scripts/qemu-key.exp
diff --git a/automation/scripts/qemu-smoke-x86-64.sh b/automation/scripts/qemu-smoke-x86-64.sh
index 3014d07314..c7fcf0ff6e 100755
--- a/automation/scripts/qemu-smoke-x86-64.sh
+++ b/automation/scripts/qemu-smoke-x86-64.sh
@@ -16,11 +16,11 @@ esac
 
 rm -f smoke.serial
 set +e
-timeout -k 1 30 \
-qemu-system-x86_64 -nographic -kernel binaries/xen \
+export QEMU_CMD="qemu-system-x86_64 -nographic -kernel binaries/xen \
         -initrd xtf/tests/example/$k \
-        -append "loglvl=all console=com1 noreboot console_timestamps=boot $extra" \
-        -m 512 -monitor none -serial file:smoke.serial
-set -e
-grep -q 'Test result: SUCCESS' smoke.serial || exit 1
-exit 0
+        -append \"loglvl=all console=com1 noreboot console_timestamps=boot $extra\" \
+        -m 512 -monitor none -serial stdio"
+
+export QEMU_LOG="smoke.serial"
+export PASSED="Test result: SUCCESS"
+./automation/scripts/qemu-key.exp
diff --git a/automation/scripts/qemu-xtf-dom0less-arm64.sh b/automation/scripts/qemu-xtf-dom0less-arm64.sh
index b08c2d44fb..9659adee2b 100755
--- a/automation/scripts/qemu-xtf-dom0less-arm64.sh
+++ b/automation/scripts/qemu-xtf-dom0less-arm64.sh
@@ -51,9 +51,7 @@ bash imagebuilder/scripts/uboot-script-gen -t tftp -d binaries/ -c binaries/conf
 # Run the test
 rm -f smoke.serial
 set +e
-echo "  virtio scan; dhcp; tftpb 0x40000000 boot.scr; source 0x40000000"| \
-timeout -k 1 120 \
-./binaries/qemu-system-aarch64 \
+export QEMU_CMD="./binaries/qemu-system-aarch64 \
     -machine virtualization=true \
     -cpu cortex-a57 -machine type=virt \
     -m 2048 -monitor none -serial stdio \
@@ -61,9 +59,9 @@ timeout -k 1 120 \
     -no-reboot \
     -device virtio-net-pci,netdev=n0 \
     -netdev user,id=n0,tftp=binaries \
-    -bios /usr/lib/u-boot/qemu_arm64/u-boot.bin |& \
-        tee smoke.serial | sed 's/\r//'
+    -bios /usr/lib/u-boot/qemu_arm64/u-boot.bin"
 
-set -e
-(grep -q "${passed}" smoke.serial) || exit 1
-exit 0
+export UBOOT_CMD="virtio scan; dhcp; tftpb 0x40000000 boot.scr; source 0x40000000"
+export QEMU_LOG="smoke.serial"
+export PASSED="${passed}"
+./automation/scripts/qemu-key.exp
-- 
2.25.1

Re: [PATCH v2] automation: use expect to run QEMU

Posted by Michal Orzel 3 months, 1 week ago

Hi Stefano,

On 13/08/2024 04:21, Stefano Stabellini wrote:
> 
> 
> Use expect to invoke QEMU so that we can terminate the test as soon as
> we get the right string in the output instead of waiting until the
> final timeout.
> 
> For timeout, instead of an hardcoding the value, use a Gitlab CI
> variable "QEMU_TIMEOUT" that can be changed depending on the latest
> status of the Gitlab CI runners.
> 
> Note that for simplicity in the case of dom0less test, the script only
> checks for the $PASSED string. That is because the dom0 prompt and the
> $PASSED string could happen in any order making it difficult to check
> with expect which checks for strings in a specific order.
I think this comment is no longer true and needs to be removed.

> 
> Signed-off-by: Stefano Stabellini <stefano.stabellini@amd.com>
> 
> ---
> Changes in v2:
> - add empty lines for readability
I can only see it being done for one script.

> - changes qemu-key file extension to exp
> - drop xen_msg that is unused
> - use capital letters for exported variables
> - check for both dom0 and domU message on dom0less tests
> 
> https://gitlab.com/xen-project/people/sstabellini/xen/-/pipelines/1410820286
> ---
>  automation/scripts/qemu-alpine-x86_64.sh      | 16 +++---
>  automation/scripts/qemu-key.exp               | 51 +++++++++++++++++++
>  automation/scripts/qemu-smoke-dom0-arm32.sh   | 15 +++---
>  automation/scripts/qemu-smoke-dom0-arm64.sh   | 15 +++---
>  .../scripts/qemu-smoke-dom0less-arm32.sh      | 17 +++----
>  .../scripts/qemu-smoke-dom0less-arm64.sh      | 15 +++---
>  automation/scripts/qemu-smoke-ppc64le.sh      | 12 ++---
>  automation/scripts/qemu-smoke-riscv64.sh      | 12 ++---
>  automation/scripts/qemu-smoke-x86-64.sh       | 14 ++---
>  automation/scripts/qemu-xtf-dom0less-arm64.sh | 14 +++--
>  10 files changed, 110 insertions(+), 71 deletions(-)
>  create mode 100755 automation/scripts/qemu-key.exp
> 
> diff --git a/automation/scripts/qemu-alpine-x86_64.sh b/automation/scripts/qemu-alpine-x86_64.sh
> index 8e398dcea3..5359e0820b 100755
> --- a/automation/scripts/qemu-alpine-x86_64.sh
> +++ b/automation/scripts/qemu-alpine-x86_64.sh
> @@ -77,18 +77,16 @@ EOF
>  # Run the test
>  rm -f smoke.serial
>  set +e
> -timeout -k 1 720 \
> -qemu-system-x86_64 \
> +export QEMU_CMD="qemu-system-x86_64 \
>      -cpu qemu64,+svm \
>      -m 2G -smp 2 \
>      -monitor none -serial stdio \
>      -nographic \
>      -device virtio-net-pci,netdev=n0 \
> -    -netdev user,id=n0,tftp=binaries,bootfile=/pxelinux.0 |& \
> -        # Remove carriage returns from the stdout output, as gitlab
> -        # interface chokes on them
> -        tee smoke.serial | sed 's/\r//'
> +    -netdev user,id=n0,tftp=binaries,bootfile=/pxelinux.0"
> 
> -set -e
> -(grep -q "Domain-0" smoke.serial && grep -q "BusyBox" smoke.serial) || exit 1
> -exit 0
> +export QEMU_LOG="smoke.serial"
> +export LOG_MSG="Domain-0"
> +export PASSED="BusyBox"
> +
> +./automation/scripts/qemu-key.exp
> diff --git a/automation/scripts/qemu-key.exp b/automation/scripts/qemu-key.exp
> new file mode 100755
> index 0000000000..9f3a3364fa
> --- /dev/null
> +++ b/automation/scripts/qemu-key.exp
> @@ -0,0 +1,51 @@
> +#!/usr/bin/expect -f
> +
> +set timeout $env(QEMU_TIMEOUT)
> +
> +log_file -a $env(QEMU_LOG)
> +
> +match_max 10000
> +
> +eval spawn $env(QEMU_CMD)
> +
> +expect_after {
> +    -re "(.*)\r" {
> +        exp_continue
> +    }
> +    timeout {send_user "ERROR-Timeout!\n"; exit 1}
It should be send_error to send the string to stderr

> +    eof {send_user "ERROR-EOF!\n"; exit 1}
> +}
> +
> +if {[info exists env(UBOOT_CMD)]} {
> +    expect "=>"
> +
> +    send "$env(UBOOT_CMD)\r"
> +}
> +
> +if {[info exists env(LOG_MSG)] && [info exists env(PASSED)]} {
> +    expect {
> +        "$env(PASSED)" {
> +            expect "$env(LOG_MSG)"
> +            exit 0
> +        }
> +        "$env(LOG_MSG)" {
> +            expect "$env(PASSED)"
> +            exit 0
> +        }
> +    }
> +}
> +
> +if {[info exists env(LOG_MSG)]} {
> +    expect "$env(LOG_MSG)"
> +}
> +
> +if {[info exists env(PASSED)]} {
> +    expect {
> +        "$env(PASSED)" {
> +            exit 0
> +        }
> +    }
> +}
Given that this script treats both LOG_MSG and PASSED as optional, if a script
does not define PASSED, it will just continue until timeout. Might be worth making
PASSED mandatory. Something like:

if {[info exists env(LOG_MSG)]} {
    expect {
        "$env(PASSED)" {
            expect "$env(LOG_MSG)"
            exit 0
        }
        "$env(LOG_MSG)" {
            expect "$env(PASSED)"
            exit 0
        }
    }
}

expect {
    "$env(PASSED)" {
        exit 0
    }
}

In any case, you can do the changes on commit:
Reviewed-by: Michal Orzel <michal.orzel@amd.com>

~Michal