This patch adds migration test support for aarch64. The test code, which
implements the same functionality as x86, is booted as a kernel in qemu.
Here are the design choices we make for aarch64:
* We choose this -kernel approach because aarch64 QEMU doesn't provide a
built-in fw like x86 does. So instead of relying on a boot loader, we
use -kernel approach for aarch64.
* The serial output is sent to PL011 directly.
* The physical memory base for mach-virt machine is 0x40000000. We change
the start_address and end_address for aarch64.
In addition to providing the binary, this patch also includes the source
code and the build script in tests/migration/. So users can change the
source and/or re-compile the binary as they wish.
Signed-off-by: Wei Huang <wei@redhat.com>
---
tests/Makefile.include | 1 +
tests/migration-test.c | 39 ++++++++++++++++---
tests/migration/Makefile | 12 +++++-
tests/migration/aarch64-a-b-kernel.h | 19 +++++++++
tests/migration/aarch64-a-b-kernel.s | 74 ++++++++++++++++++++++++++++++++++++
5 files changed, 137 insertions(+), 8 deletions(-)
create mode 100644 tests/migration/aarch64-a-b-kernel.h
create mode 100644 tests/migration/aarch64-a-b-kernel.s
diff --git a/tests/Makefile.include b/tests/Makefile.include
index 278c13a..5b1605a 100644
--- a/tests/Makefile.include
+++ b/tests/Makefile.include
@@ -372,6 +372,7 @@ check-qtest-arm-y += tests/sdhci-test$(EXESUF)
check-qtest-aarch64-y = tests/numa-test$(EXESUF)
check-qtest-aarch64-y += tests/sdhci-test$(EXESUF)
+check-qtest-aarch64-y += tests/migration-test$(EXESUF)
check-qtest-microblazeel-y = $(check-qtest-microblaze-y)
diff --git a/tests/migration-test.c b/tests/migration-test.c
index 97fdb19..0b6ab5c 100644
--- a/tests/migration-test.c
+++ b/tests/migration-test.c
@@ -23,8 +23,8 @@
#define MIN_NVRAM_SIZE 8192 /* from spapr_nvram.c */
-const unsigned start_address = 1024 * 1024;
-const unsigned end_address = 100 * 1024 * 1024;
+unsigned start_address = 1024 * 1024;
+unsigned end_address = 100 * 1024 * 1024;
bool got_stop;
#if defined(__linux__)
@@ -81,12 +81,13 @@ static const char *tmpfs;
* outputting a 'B' every so often if it's still running.
*/
#include "tests/migration/x86-a-b-bootblock.h"
+#include "tests/migration/aarch64-a-b-kernel.h"
-static void init_bootfile_x86(const char *bootpath)
+static void init_bootfile(const char *bootpath, void *content)
{
FILE *bootfile = fopen(bootpath, "wb");
- g_assert_cmpint(fwrite(x86_bootsect, 512, 1, bootfile), ==, 1);
+ g_assert_cmpint(fwrite(content, 512, 1, bootfile), ==, 1);
fclose(bootfile);
}
@@ -392,7 +393,7 @@ static void test_migrate_start(QTestState **from, QTestState **to,
got_stop = false;
if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) {
- init_bootfile_x86(bootpath);
+ init_bootfile(bootpath, x86_bootsect);
cmd_src = g_strdup_printf("-machine accel=%s -m 150M"
" -name source,debug-threads=on"
" -serial file:%s/src_serial"
@@ -421,6 +422,32 @@ static void test_migrate_start(QTestState **from, QTestState **to,
" -serial file:%s/dest_serial"
" -incoming %s",
accel, tmpfs, uri);
+ } else if (strcmp(arch, "aarch64") == 0) {
+ const char *cpu;
+
+ if (access("/dev/kvm", F_OK)) {
+ accel = "kvm";
+ cpu = "host";
+ } else {
+ accel = "tcg";
+ cpu = "cortex-a57";
+ }
+
+ init_bootfile(bootpath, aarch64_kernel);
+ cmd_src = g_strdup_printf("-machine virt,accel=%s -m 150M "
+ "-name vmsource,debug-threads=on -cpu %s "
+ "-serial file:%s/src_serial "
+ "-kernel %s ",
+ accel, cpu, tmpfs, bootpath);
+ cmd_dst = g_strdup_printf("-machine virt,accel=%s -m 150M "
+ "-name vmdest,debug-threads=on -cpu %s "
+ "-serial file:%s/dest_serial "
+ "-kernel %s "
+ "-incoming %s ",
+ accel, cpu, tmpfs, bootpath, uri);
+ /* aarch64 virt machine physical memory starts at 0x40000000 */
+ start_address += 0x40000000;
+ end_address += 0x40000000;
} else {
g_assert_not_reached();
}
@@ -502,7 +529,7 @@ static void test_deprecated(void)
{
QTestState *from;
- from = qtest_start("");
+ from = qtest_start("-machine none");
deprecated_set_downtime(from, 0.12345);
deprecated_set_speed(from, "12345");
diff --git a/tests/migration/Makefile b/tests/migration/Makefile
index e8e6026..00f854b 100644
--- a/tests/migration/Makefile
+++ b/tests/migration/Makefile
@@ -15,6 +15,7 @@ cross-gcc = $(firstword $(wildcard $(patsubst %ld,%gcc,$(call cross-ld,$(1)))))
find-cross-prefix = $(subst gcc,,$(notdir $(call cross-gcc,$(1))))
x86_64_cross_prefix := $(call find-cross-prefix,x86_64)
+aarch64_cross_prefix := $(call find-cross-prefix,aarch64)
export __note
override define __note
@@ -25,7 +26,7 @@ override define __note
*/
endef
-all: x86-a-b-bootblock.h
+all: x86-a-b-bootblock.h aarch64-a-b-kernel.h
x86-a-b-bootblock.h: x86-a-b-bootblock.s
$(x86_64_cross_prefix)as --32 -march=i486 $< -o x86.o
@@ -34,5 +35,12 @@ x86-a-b-bootblock.h: x86-a-b-bootblock.s
echo "$$__note" > $@
xxd -i x86.bootsect | sed -e 's/.*int.*//' >> $@
+aarch64-a-b-kernel.h: aarch64-a-b-kernel.s
+ $(aarch64_cross_prefix)gcc -o aarch64.elf -nostdlib \
+ -Wl,--build-id=none,-Ttext=40080000 $<
+ $(aarch64_cross_prefix)objcopy -O binary aarch64.elf aarch64.kernel
+ echo "$$__note" > $@
+ xxd -i aarch64.kernel | sed -e 's/.*int.*//' >> $@
+
clean:
- rm -rf *.bootsect *.boot *.o
+ rm -rf *.bootsect *.boot *.o *.elf *.kernel
diff --git a/tests/migration/aarch64-a-b-kernel.h b/tests/migration/aarch64-a-b-kernel.h
new file mode 100644
index 0000000..d294cf8
--- /dev/null
+++ b/tests/migration/aarch64-a-b-kernel.h
@@ -0,0 +1,19 @@
+/* This file is automatically generated from
+ * tests/migration/aarch64-a-b-kernel.s, edit that and then run
+ * "make aarch64-a-b-kernel.h" inside tests/migration to update,
+ * and then remember to send both in your patch submission.
+ */
+unsigned char aarch64_kernel[] = {
+ 0x00, 0x10, 0x38, 0xd5, 0x00, 0xf8, 0x7f, 0x92, 0x00, 0x10, 0x18, 0xd5,
+ 0xdf, 0x3f, 0x03, 0xd5, 0x00, 0x02, 0xa8, 0xd2, 0x01, 0xc8, 0xa8, 0xd2,
+ 0x23, 0x08, 0x80, 0x52, 0x02, 0x20, 0xa1, 0xd2, 0x43, 0x00, 0x00, 0x39,
+ 0x03, 0x00, 0x80, 0x52, 0xe4, 0x03, 0x00, 0xaa, 0x83, 0x00, 0x00, 0x39,
+ 0x84, 0x04, 0x40, 0x91, 0x9f, 0x00, 0x01, 0xeb, 0xad, 0xff, 0xff, 0x54,
+ 0x05, 0x00, 0x80, 0x52, 0xe4, 0x03, 0x00, 0xaa, 0x24, 0x7e, 0x0b, 0xd5,
+ 0x83, 0x00, 0x40, 0x39, 0x63, 0x04, 0x00, 0x11, 0x63, 0x1c, 0x00, 0x12,
+ 0x83, 0x00, 0x00, 0x39, 0x84, 0x04, 0x40, 0x91, 0x9f, 0x00, 0x01, 0xeb,
+ 0x2b, 0xff, 0xff, 0x54, 0xa5, 0x04, 0x00, 0x11, 0xa5, 0x1c, 0x00, 0x12,
+ 0xbf, 0x00, 0x00, 0x71, 0x81, 0xfe, 0xff, 0x54, 0x43, 0x08, 0x80, 0x52,
+ 0x43, 0x00, 0x00, 0x39, 0xf1, 0xff, 0xff, 0x97
+};
+
diff --git a/tests/migration/aarch64-a-b-kernel.s b/tests/migration/aarch64-a-b-kernel.s
new file mode 100644
index 0000000..713e7a8
--- /dev/null
+++ b/tests/migration/aarch64-a-b-kernel.s
@@ -0,0 +1,74 @@
+#
+# Copyright (c) 2018 Red Hat, Inc. and/or its affiliates
+#
+# Author:
+# Wei Huang <wei@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or later.
+# See the COPYING file in the top-level directory.
+
+.set ARM_MACH_VIRT_UART, 0x09000000
+.set ARM_MACH_VIRT_PHYS_BASE, 0x40000000
+.set ARM_MACH_VIRT_A_B_START, (0x40000000 + 1*1024*1024)
+.set ARM_MACH_VIRT_A_B_END, (0x40000000 + 100*1024*1024)
+
+.section .text
+
+ .globl _start
+
+_start:
+ /* disable MMU to use phys mem address */
+ mrs x0, sctlr_el1
+ bic x0, x0, #(1<<0)
+ msr sctlr_el1, x0
+ isb
+
+ /* traverse 1M-100M */
+ mov x0, ARM_MACH_VIRT_A_B_START
+ mov x1, ARM_MACH_VIRT_A_B_END
+
+ /* output char 'A' to PL011 */
+ mov w3, 'A'
+ mov x2, ARM_MACH_VIRT_UART
+ strb w3, [x2]
+
+ /* clean up memory */
+ mov w3, #0
+ mov x4, x0
+clean:
+ strb w3, [x4]
+ add x4, x4, #(4096)
+ cmp x4, x1
+ ble clean
+
+ /* w5 keeps a counter so we can limit the output speed */
+ mov w5, #0
+
+ /* main body */
+mainloop:
+ mov x4, x0
+
+innerloop:
+ /* clean cache because el2 might still cache guest data under KVM */
+ dc civac, x4
+
+ /* increment the first byte of each 4KB page by 1 */
+ ldrb w3, [x4]
+ add w3, w3, #1
+ and w3, w3, #(0xff)
+ strb w3, [x4]
+
+ add x4, x4, #(4096)
+ cmp x4, x1
+ blt innerloop
+
+ add w5, w5, #1
+ and w5, w5, #(0xff)
+ cmp w5, #0
+ bne mainloop
+
+ /* output char 'B' to PL011 */
+ mov w3, 'B'
+ strb w3, [x2]
+
+ bl mainloop
--
1.8.3.1
On Thu, Feb 15, 2018 at 01:07:25AM -0500, Wei Huang wrote:
> This patch adds migration test support for aarch64. The test code, which
> implements the same functionality as x86, is booted as a kernel in qemu.
> Here are the design choices we make for aarch64:
>
> * We choose this -kernel approach because aarch64 QEMU doesn't provide a
> built-in fw like x86 does. So instead of relying on a boot loader, we
> use -kernel approach for aarch64.
> * The serial output is sent to PL011 directly.
> * The physical memory base for mach-virt machine is 0x40000000. We change
> the start_address and end_address for aarch64.
>
> In addition to providing the binary, this patch also includes the source
> code and the build script in tests/migration/. So users can change the
> source and/or re-compile the binary as they wish.
>
> Signed-off-by: Wei Huang <wei@redhat.com>
> ---
> tests/Makefile.include | 1 +
> tests/migration-test.c | 39 ++++++++++++++++---
> tests/migration/Makefile | 12 +++++-
> tests/migration/aarch64-a-b-kernel.h | 19 +++++++++
> tests/migration/aarch64-a-b-kernel.s | 74 ++++++++++++++++++++++++++++++++++++
> 5 files changed, 137 insertions(+), 8 deletions(-)
> create mode 100644 tests/migration/aarch64-a-b-kernel.h
> create mode 100644 tests/migration/aarch64-a-b-kernel.s
>
> diff --git a/tests/Makefile.include b/tests/Makefile.include
> index 278c13a..5b1605a 100644
> --- a/tests/Makefile.include
> +++ b/tests/Makefile.include
> @@ -372,6 +372,7 @@ check-qtest-arm-y += tests/sdhci-test$(EXESUF)
>
> check-qtest-aarch64-y = tests/numa-test$(EXESUF)
> check-qtest-aarch64-y += tests/sdhci-test$(EXESUF)
> +check-qtest-aarch64-y += tests/migration-test$(EXESUF)
>
> check-qtest-microblazeel-y = $(check-qtest-microblaze-y)
>
> diff --git a/tests/migration-test.c b/tests/migration-test.c
> index 97fdb19..0b6ab5c 100644
> --- a/tests/migration-test.c
> +++ b/tests/migration-test.c
> @@ -23,8 +23,8 @@
>
> #define MIN_NVRAM_SIZE 8192 /* from spapr_nvram.c */
>
> -const unsigned start_address = 1024 * 1024;
> -const unsigned end_address = 100 * 1024 * 1024;
> +unsigned start_address = 1024 * 1024;
> +unsigned end_address = 100 * 1024 * 1024;
> bool got_stop;
>
> #if defined(__linux__)
> @@ -81,12 +81,13 @@ static const char *tmpfs;
> * outputting a 'B' every so often if it's still running.
> */
> #include "tests/migration/x86-a-b-bootblock.h"
> +#include "tests/migration/aarch64-a-b-kernel.h"
>
> -static void init_bootfile_x86(const char *bootpath)
> +static void init_bootfile(const char *bootpath, void *content)
> {
> FILE *bootfile = fopen(bootpath, "wb");
>
> - g_assert_cmpint(fwrite(x86_bootsect, 512, 1, bootfile), ==, 1);
> + g_assert_cmpint(fwrite(content, 512, 1, bootfile), ==, 1);
> fclose(bootfile);
> }
>
> @@ -392,7 +393,7 @@ static void test_migrate_start(QTestState **from, QTestState **to,
> got_stop = false;
>
> if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) {
> - init_bootfile_x86(bootpath);
> + init_bootfile(bootpath, x86_bootsect);
> cmd_src = g_strdup_printf("-machine accel=%s -m 150M"
> " -name source,debug-threads=on"
> " -serial file:%s/src_serial"
> @@ -421,6 +422,32 @@ static void test_migrate_start(QTestState **from, QTestState **to,
> " -serial file:%s/dest_serial"
> " -incoming %s",
> accel, tmpfs, uri);
> + } else if (strcmp(arch, "aarch64") == 0) {
> + const char *cpu;
> +
> + if (access("/dev/kvm", F_OK)) {
> + accel = "kvm";
> + cpu = "host";
This won't work. I have /dev/kvm on my x86 notebook, but I can't
run aarch64 kvm guests there. You also need to check if the host
is aarch64. You can do that with uname(3), checking uname.machine.
> + } else {
> + accel = "tcg";
> + cpu = "cortex-a57";
> + }
> +
> + init_bootfile(bootpath, aarch64_kernel);
> + cmd_src = g_strdup_printf("-machine virt,accel=%s -m 150M "
You should also add ,gic-version=%s, where %s is either 'host' when kvm
is in use, or 2 or 3 (whichever) when tcg is in use.
> + "-name vmsource,debug-threads=on -cpu %s "
> + "-serial file:%s/src_serial "
> + "-kernel %s ",
> + accel, cpu, tmpfs, bootpath);
> + cmd_dst = g_strdup_printf("-machine virt,accel=%s -m 150M "
> + "-name vmdest,debug-threads=on -cpu %s "
> + "-serial file:%s/dest_serial "
> + "-kernel %s "
> + "-incoming %s ",
> + accel, cpu, tmpfs, bootpath, uri);
> + /* aarch64 virt machine physical memory starts at 0x40000000 */
> + start_address += 0x40000000;
> + end_address += 0x40000000;
> } else {
> g_assert_not_reached();
> }
> @@ -502,7 +529,7 @@ static void test_deprecated(void)
> {
> QTestState *from;
>
> - from = qtest_start("");
> + from = qtest_start("-machine none");
>
> deprecated_set_downtime(from, 0.12345);
> deprecated_set_speed(from, "12345");
> diff --git a/tests/migration/Makefile b/tests/migration/Makefile
> index e8e6026..00f854b 100644
> --- a/tests/migration/Makefile
> +++ b/tests/migration/Makefile
> @@ -15,6 +15,7 @@ cross-gcc = $(firstword $(wildcard $(patsubst %ld,%gcc,$(call cross-ld,$(1)))))
> find-cross-prefix = $(subst gcc,,$(notdir $(call cross-gcc,$(1))))
>
> x86_64_cross_prefix := $(call find-cross-prefix,x86_64)
> +aarch64_cross_prefix := $(call find-cross-prefix,aarch64)
>
> export __note
> override define __note
> @@ -25,7 +26,7 @@ override define __note
> */
> endef
>
> -all: x86-a-b-bootblock.h
> +all: x86-a-b-bootblock.h aarch64-a-b-kernel.h
>
> x86-a-b-bootblock.h: x86-a-b-bootblock.s
> $(x86_64_cross_prefix)as --32 -march=i486 $< -o x86.o
> @@ -34,5 +35,12 @@ x86-a-b-bootblock.h: x86-a-b-bootblock.s
> echo "$$__note" > $@
> xxd -i x86.bootsect | sed -e 's/.*int.*//' >> $@
>
> +aarch64-a-b-kernel.h: aarch64-a-b-kernel.s
> + $(aarch64_cross_prefix)gcc -o aarch64.elf -nostdlib \
> + -Wl,--build-id=none,-Ttext=40080000 $<
> + $(aarch64_cross_prefix)objcopy -O binary aarch64.elf aarch64.kernel
> + echo "$$__note" > $@
> + xxd -i aarch64.kernel | sed -e 's/.*int.*//' >> $@
> +
> clean:
> - rm -rf *.bootsect *.boot *.o
> + rm -rf *.bootsect *.boot *.o *.elf *.kernel
I'm not sure you want the '-r' in this 'rm'.
> diff --git a/tests/migration/aarch64-a-b-kernel.h b/tests/migration/aarch64-a-b-kernel.h
> new file mode 100644
> index 0000000..d294cf8
> --- /dev/null
> +++ b/tests/migration/aarch64-a-b-kernel.h
> @@ -0,0 +1,19 @@
> +/* This file is automatically generated from
> + * tests/migration/aarch64-a-b-kernel.s, edit that and then run
> + * "make aarch64-a-b-kernel.h" inside tests/migration to update,
> + * and then remember to send both in your patch submission.
> + */
> +unsigned char aarch64_kernel[] = {
> + 0x00, 0x10, 0x38, 0xd5, 0x00, 0xf8, 0x7f, 0x92, 0x00, 0x10, 0x18, 0xd5,
> + 0xdf, 0x3f, 0x03, 0xd5, 0x00, 0x02, 0xa8, 0xd2, 0x01, 0xc8, 0xa8, 0xd2,
> + 0x23, 0x08, 0x80, 0x52, 0x02, 0x20, 0xa1, 0xd2, 0x43, 0x00, 0x00, 0x39,
> + 0x03, 0x00, 0x80, 0x52, 0xe4, 0x03, 0x00, 0xaa, 0x83, 0x00, 0x00, 0x39,
> + 0x84, 0x04, 0x40, 0x91, 0x9f, 0x00, 0x01, 0xeb, 0xad, 0xff, 0xff, 0x54,
> + 0x05, 0x00, 0x80, 0x52, 0xe4, 0x03, 0x00, 0xaa, 0x24, 0x7e, 0x0b, 0xd5,
> + 0x83, 0x00, 0x40, 0x39, 0x63, 0x04, 0x00, 0x11, 0x63, 0x1c, 0x00, 0x12,
> + 0x83, 0x00, 0x00, 0x39, 0x84, 0x04, 0x40, 0x91, 0x9f, 0x00, 0x01, 0xeb,
> + 0x2b, 0xff, 0xff, 0x54, 0xa5, 0x04, 0x00, 0x11, 0xa5, 0x1c, 0x00, 0x12,
> + 0xbf, 0x00, 0x00, 0x71, 0x81, 0xfe, 0xff, 0x54, 0x43, 0x08, 0x80, 0x52,
> + 0x43, 0x00, 0x00, 0x39, 0xf1, 0xff, 0xff, 0x97
> +};
> +
> diff --git a/tests/migration/aarch64-a-b-kernel.s b/tests/migration/aarch64-a-b-kernel.s
> new file mode 100644
> index 0000000..713e7a8
> --- /dev/null
> +++ b/tests/migration/aarch64-a-b-kernel.s
> @@ -0,0 +1,74 @@
> +#
> +# Copyright (c) 2018 Red Hat, Inc. and/or its affiliates
> +#
> +# Author:
> +# Wei Huang <wei@redhat.com>
> +#
> +# This work is licensed under the terms of the GNU GPL, version 2 or later.
> +# See the COPYING file in the top-level directory.
> +
> +.set ARM_MACH_VIRT_UART, 0x09000000
> +.set ARM_MACH_VIRT_PHYS_BASE, 0x40000000
> +.set ARM_MACH_VIRT_A_B_START, (0x40000000 + 1*1024*1024)
> +.set ARM_MACH_VIRT_A_B_END, (0x40000000 + 100*1024*1024)
These make this file nicer, but it doesn't avoid scattering
0x40000000 in two instead of one new place. I was thinking
a header file that would be included by this file and
tests/migration-test.c would be created.
> +
> +.section .text
> +
> + .globl _start
> +
> +_start:
> + /* disable MMU to use phys mem address */
> + mrs x0, sctlr_el1
> + bic x0, x0, #(1<<0)
> + msr sctlr_el1, x0
> + isb
> +
> + /* traverse 1M-100M */
> + mov x0, ARM_MACH_VIRT_A_B_START
> + mov x1, ARM_MACH_VIRT_A_B_END
> +
> + /* output char 'A' to PL011 */
> + mov w3, 'A'
> + mov x2, ARM_MACH_VIRT_UART
> + strb w3, [x2]
> +
> + /* clean up memory */
> + mov w3, #0
> + mov x4, x0
> +clean:
> + strb w3, [x4]
> + add x4, x4, #(4096)
> + cmp x4, x1
> + ble clean
> +
> + /* w5 keeps a counter so we can limit the output speed */
> + mov w5, #0
> +
> + /* main body */
> +mainloop:
> + mov x4, x0
> +
> +innerloop:
> + /* clean cache because el2 might still cache guest data under KVM */
> + dc civac, x4
> +
> + /* increment the first byte of each 4KB page by 1 */
> + ldrb w3, [x4]
> + add w3, w3, #1
> + and w3, w3, #(0xff)
> + strb w3, [x4]
> +
> + add x4, x4, #(4096)
> + cmp x4, x1
> + blt innerloop
> +
> + add w5, w5, #1
> + and w5, w5, #(0xff)
> + cmp w5, #0
> + bne mainloop
nit: no need for the () in the immediates.
> +
> + /* output char 'B' to PL011 */
> + mov w3, 'B'
> + strb w3, [x2]
> +
> + bl mainloop
> --
> 1.8.3.1
>
>
Thanks,
drew
© 2016 - 2025 Red Hat, Inc.