[v12] TDX KVM selftests

[PATCH v12 08/23] KVM: selftests: Define structs to pass parameters to TDX boot code

Posted by Sagi Shahar 1 month, 2 weeks ago

TDX registers are inaccessible to KVM. Therefore we need a different
mechanism to load boot parameters for TDX code. TDX boot code will read
the registers values from memory and set the registers manually.

This patch defines the data structures used to communicate between c
code and the TDX assembly boot code which will be added in a later
patch.

Use kbuild.h to expose the offsets into the structs from c code to
assembly code.

Reviewed-by: Binbin Wu <binbin.wu@linux.intel.com>
Co-developed-by: Ackerley Tng <ackerleytng@google.com>
Signed-off-by: Ackerley Tng <ackerleytng@google.com>
Signed-off-by: Sagi Shahar <sagis@google.com>
---
 tools/testing/selftests/kvm/Makefile.kvm      | 18 +++++
 .../selftests/kvm/include/x86/tdx/td_boot.h   | 69 +++++++++++++++++++
 .../kvm/lib/x86/tdx/td_boot_offsets.c         | 21 ++++++
 3 files changed, 108 insertions(+)
 create mode 100644 tools/testing/selftests/kvm/include/x86/tdx/td_boot.h
 create mode 100644 tools/testing/selftests/kvm/lib/x86/tdx/td_boot_offsets.c

diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
index 148d427ff24b..5e809064ff1c 100644
--- a/tools/testing/selftests/kvm/Makefile.kvm
+++ b/tools/testing/selftests/kvm/Makefile.kvm
@@ -19,6 +19,8 @@ LIBKVM += lib/userfaultfd_util.c
 
 LIBKVM_STRING += lib/string_override.c
 
+LIBKVM_ASM_DEFS += lib/x86/tdx/td_boot_offsets.c
+
 LIBKVM_x86 += lib/x86/apic.c
 LIBKVM_x86 += lib/x86/handlers.S
 LIBKVM_x86 += lib/x86/hyperv.c
@@ -239,6 +241,10 @@ OVERRIDE_TARGETS = 1
 include ../lib.mk
 include ../cgroup/lib/libcgroup.mk
 
+# Enable Kbuild tools.
+include $(top_srcdir)/scripts/Kbuild.include
+include $(top_srcdir)/scripts/Makefile.lib
+
 INSTALL_HDR_PATH = $(top_srcdir)/usr
 LINUX_HDR_PATH = $(INSTALL_HDR_PATH)/include/
 LINUX_TOOL_INCLUDE = $(top_srcdir)/tools/include
@@ -291,6 +297,7 @@ LIBKVM_S := $(filter %.S,$(LIBKVM))
 LIBKVM_C_OBJ := $(patsubst %.c, $(OUTPUT)/%.o, $(LIBKVM_C))
 LIBKVM_S_OBJ := $(patsubst %.S, $(OUTPUT)/%.o, $(LIBKVM_S))
 LIBKVM_STRING_OBJ := $(patsubst %.c, $(OUTPUT)/%.o, $(LIBKVM_STRING))
+LIBKVM_ASM_DEFS_OBJ += $(patsubst %.c, $(OUTPUT)/%.s, $(LIBKVM_ASM_DEFS))
 LIBKVM_OBJS = $(LIBKVM_C_OBJ) $(LIBKVM_S_OBJ) $(LIBKVM_STRING_OBJ) $(LIBCGROUP_O)
 SPLIT_TEST_GEN_PROGS := $(patsubst %, $(OUTPUT)/%, $(SPLIT_TESTS))
 SPLIT_TEST_GEN_OBJ := $(patsubst %, $(OUTPUT)/$(ARCH)/%.o, $(SPLIT_TESTS))
@@ -317,6 +324,7 @@ $(SPLIT_TEST_GEN_OBJ): $(OUTPUT)/$(ARCH)/%.o: $(ARCH)/%.c
 
 EXTRA_CLEAN += $(GEN_HDRS) \
 	       $(LIBKVM_OBJS) \
+	       $(LIBKVM_ASM_DEFS_OBJ) \
 	       $(SPLIT_TEST_GEN_OBJ) \
 	       $(TEST_DEP_FILES) \
 	       $(TEST_GEN_OBJ) \
@@ -328,18 +336,28 @@ $(LIBKVM_C_OBJ): $(OUTPUT)/%.o: %.c $(GEN_HDRS)
 $(LIBKVM_S_OBJ): $(OUTPUT)/%.o: %.S $(GEN_HDRS)
 	$(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c $< -o $@
 
+$(LIBKVM_ASM_DEFS_OBJ): $(OUTPUT)/%.s: %.c FORCE
+	$(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -S $< -o $@
+
 # Compile the string overrides as freestanding to prevent the compiler from
 # generating self-referential code, e.g. without "freestanding" the compiler may
 # "optimize" memcmp() by invoking memcmp(), thus causing infinite recursion.
 $(LIBKVM_STRING_OBJ): $(OUTPUT)/%.o: %.c
 	$(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c -ffreestanding $< -o $@
 
+$(OUTPUT)/include/x86/tdx/td_boot_offsets.h: $(OUTPUT)/lib/x86/tdx/td_boot_offsets.s FORCE
+	$(call filechk,offsets,__TDX_BOOT_OFFSETS_H__)
+
+EXTRA_CLEAN += $(OUTPUT)/include/x86/tdx/td_boot_offsets.h
+
 $(shell mkdir -p $(sort $(dir $(TEST_GEN_PROGS))))
 $(SPLIT_TEST_GEN_OBJ): $(GEN_HDRS)
 $(TEST_GEN_PROGS): $(LIBKVM_OBJS)
 $(TEST_GEN_PROGS_EXTENDED): $(LIBKVM_OBJS)
 $(TEST_GEN_OBJ): $(GEN_HDRS)
 
+FORCE:
+
 cscope: include_paths = $(LINUX_TOOL_INCLUDE) $(LINUX_HDR_PATH) include lib ..
 cscope:
 	$(RM) cscope.*
diff --git a/tools/testing/selftests/kvm/include/x86/tdx/td_boot.h b/tools/testing/selftests/kvm/include/x86/tdx/td_boot.h
new file mode 100644
index 000000000000..32631645fe13
--- /dev/null
+++ b/tools/testing/selftests/kvm/include/x86/tdx/td_boot.h
@@ -0,0 +1,69 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef SELFTEST_TDX_TD_BOOT_H
+#define SELFTEST_TDX_TD_BOOT_H
+
+#include <stdint.h>
+
+#include <linux/compiler.h>
+#include <linux/sizes.h>
+
+/*
+ * Layout for boot section (not to scale)
+ *
+ *                                   GPA
+ * _________________________________ 0x1_0000_0000 (4GB)
+ * |   Boot code trampoline    |
+ * |___________________________|____ 0x0_ffff_fff0: Reset vector (16B below 4GB)
+ * |   Boot code               |
+ * |___________________________|____ td_boot will be copied here, so that the
+ * |                           |     jmp to td_boot is exactly at the reset vector
+ * |   Empty space             |
+ * |                           |
+ * |───────────────────────────|
+ * |                           |
+ * |                           |
+ * |   Boot parameters         |
+ * |                           |
+ * |                           |
+ * |___________________________|____ 0x0_ffff_0000: TD_BOOT_PARAMETERS_GPA
+ */
+#define FOUR_GIGABYTES_GPA (SZ_4G)
+
+/*
+ * The exact memory layout for LGDT or LIDT instructions.
+ */
+struct __packed td_boot_parameters_dtr {
+	uint16_t limit;
+	uint32_t base;
+};
+
+/*
+ * Allows each vCPU to be initialized with different rip and esp.
+ */
+struct td_per_vcpu_parameters {
+	uint32_t esp_gva;
+	uint64_t guest_code;
+};
+
+/*
+ * Boot parameters for the TD.
+ *
+ * Unlike a regular VM, KVM cannot set registers such as esp, eip, etc
+ * before boot, so to run selftests, these registers' values have to be
+ * initialized by the TD.
+ *
+ * This struct is loaded in TD private memory at TD_BOOT_PARAMETERS_GPA.
+ *
+ * The TD boot code will read off parameters from this struct and set up the
+ * vCPU for executing selftests.
+ */
+struct td_boot_parameters {
+	uint32_t cr0;
+	uint32_t cr3;
+	uint32_t cr4;
+	struct td_boot_parameters_dtr gdtr;
+	struct td_boot_parameters_dtr idtr;
+	struct td_per_vcpu_parameters per_vcpu[];
+};
+
+#endif /* SELFTEST_TDX_TD_BOOT_H */
diff --git a/tools/testing/selftests/kvm/lib/x86/tdx/td_boot_offsets.c b/tools/testing/selftests/kvm/lib/x86/tdx/td_boot_offsets.c
new file mode 100644
index 000000000000..7f76a3585b99
--- /dev/null
+++ b/tools/testing/selftests/kvm/lib/x86/tdx/td_boot_offsets.c
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: GPL-2.0
+#define COMPILE_OFFSETS
+
+#include <linux/kbuild.h>
+
+#include "tdx/td_boot.h"
+
+static void __attribute__((used)) common(void)
+{
+	OFFSET(TD_BOOT_PARAMETERS_CR0, td_boot_parameters, cr0);
+	OFFSET(TD_BOOT_PARAMETERS_CR3, td_boot_parameters, cr3);
+	OFFSET(TD_BOOT_PARAMETERS_CR4, td_boot_parameters, cr4);
+	OFFSET(TD_BOOT_PARAMETERS_GDT, td_boot_parameters, gdtr);
+	OFFSET(TD_BOOT_PARAMETERS_IDT, td_boot_parameters, idtr);
+	OFFSET(TD_BOOT_PARAMETERS_PER_VCPU, td_boot_parameters, per_vcpu);
+	OFFSET(TD_PER_VCPU_PARAMETERS_ESP_GVA, td_per_vcpu_parameters, esp_gva);
+	OFFSET(TD_PER_VCPU_PARAMETERS_GUEST_CODE, td_per_vcpu_parameters,
+	       guest_code);
+	DEFINE(SIZEOF_TD_PER_VCPU_PARAMETERS,
+	       sizeof(struct td_per_vcpu_parameters));
+}
-- 
2.51.1.851.g4ebd6896fd-goog

Re: [PATCH v12 08/23] KVM: selftests: Define structs to pass parameters to TDX boot code

Posted by Ira Weiny 2 weeks, 6 days ago

Sagi Shahar wrote:

[snip]

> diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
> index 148d427ff24b..5e809064ff1c 100644
> --- a/tools/testing/selftests/kvm/Makefile.kvm
> +++ b/tools/testing/selftests/kvm/Makefile.kvm

[snip]

>  
> +$(OUTPUT)/include/x86/tdx/td_boot_offsets.h: $(OUTPUT)/lib/x86/tdx/td_boot_offsets.s FORCE
> +	$(call filechk,offsets,__TDX_BOOT_OFFSETS_H__)
> +
> +EXTRA_CLEAN += $(OUTPUT)/include/x86/tdx/td_boot_offsets.h
> +

I just noticed we should add this and tdcall_offsets.h to .gitignore since
they are not auto generated.

Ira

[snip]

Re: [PATCH v12 08/23] KVM: selftests: Define structs to pass parameters to TDX boot code

Posted by Ira Weiny 2 weeks, 1 day ago

Ira Weiny wrote:
> Sagi Shahar wrote:

[snip]

> >  
> > +$(OUTPUT)/include/x86/tdx/td_boot_offsets.h: $(OUTPUT)/lib/x86/tdx/td_boot_offsets.s FORCE
> > +	$(call filechk,offsets,__TDX_BOOT_OFFSETS_H__)
> > +
> > +EXTRA_CLEAN += $(OUTPUT)/include/x86/tdx/td_boot_offsets.h
> > +
> 
> I just noticed we should add this and tdcall_offsets.h to .gitignore since
> they are not auto generated.
           ^^^
	   now

Just to be clear.
Ira

[snip]

Re: [PATCH v12 08/23] KVM: selftests: Define structs to pass parameters to TDX boot code

Posted by Reinette Chatre 1 month, 2 weeks ago

Hi Sagi,

subject nit: Define -> Declare

On 10/28/25 2:20 PM, Sagi Shahar wrote:
> TDX registers are inaccessible to KVM. Therefore we need a different

Avoid impersonating code by using "we".

> mechanism to load boot parameters for TDX code. TDX boot code will read
> the registers values from memory and set the registers manually.

Above seems to be a mix of context and summary of change. It looks like
the changelogs of this series need to be reworked to meet the KVM requirements
documented in "Changelog" section of  Documentation/process/maintainer-kvm-x86.rst
highlighted by Sean.

> 
> This patch defines the data structures used to communicate between c

Avoid using "this patch" - it is redundant.

> code and the TDX assembly boot code which will be added in a later
> patch.
> 
> Use kbuild.h to expose the offsets into the structs from c code to
> assembly code.

Reinette

Re: [PATCH v12 08/23] KVM: selftests: Define structs to pass parameters to TDX boot code

Posted by Reinette Chatre 1 month, 2 weeks ago

Hi Sagi,

On 10/28/25 2:20 PM, Sagi Shahar wrote:
> TDX registers are inaccessible to KVM. Therefore we need a different
> mechanism to load boot parameters for TDX code. TDX boot code will read
> the registers values from memory and set the registers manually.
> 
> This patch defines the data structures used to communicate between c
> code and the TDX assembly boot code which will be added in a later
> patch.
> 

(sidenote: I do not know what the bar for this work is so I'll defer
comments related to local customs like using "we" and "this patch" in
changelog)

> Use kbuild.h to expose the offsets into the structs from c code to
> assembly code.
> 


> diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
> index 148d427ff24b..5e809064ff1c 100644
> --- a/tools/testing/selftests/kvm/Makefile.kvm
> +++ b/tools/testing/selftests/kvm/Makefile.kvm

...

> @@ -328,18 +336,28 @@ $(LIBKVM_C_OBJ): $(OUTPUT)/%.o: %.c $(GEN_HDRS)
>  $(LIBKVM_S_OBJ): $(OUTPUT)/%.o: %.S $(GEN_HDRS)
>  	$(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c $< -o $@
>  
> +$(LIBKVM_ASM_DEFS_OBJ): $(OUTPUT)/%.s: %.c FORCE
> +	$(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -S $< -o $@
> +
>  # Compile the string overrides as freestanding to prevent the compiler from
>  # generating self-referential code, e.g. without "freestanding" the compiler may
>  # "optimize" memcmp() by invoking memcmp(), thus causing infinite recursion.
>  $(LIBKVM_STRING_OBJ): $(OUTPUT)/%.o: %.c
>  	$(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c -ffreestanding $< -o $@
>  
> +$(OUTPUT)/include/x86/tdx/td_boot_offsets.h: $(OUTPUT)/lib/x86/tdx/td_boot_offsets.s FORCE
> +	$(call filechk,offsets,__TDX_BOOT_OFFSETS_H__)
> +

Some folks prefer to keep build output separate and may build tests using a command
line like:
	make O=<output dir> TARGETS=kvm -C tools/testing/selftests

This is a valid usage and will result in td_boot_offsets.h placed in <output dir> that
is not covered by current include path. A build with above command line thus fails:

lib/x86/tdx/td_boot.S:4:10: fatal error: tdx/td_boot_offsets.h: No such file or directory
    4 | #include "tdx/td_boot_offsets.h"
      |          ^~~~~~~~~~~~~~~~~~~~~~~
compilation terminated.


Something like below may be needed to add the output directory to the include path:

diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
index 2f49c8965df9..98bc40a7f069 100644
--- a/tools/testing/selftests/kvm/Makefile.kvm
+++ b/tools/testing/selftests/kvm/Makefile.kvm
@@ -262,7 +262,7 @@ CFLAGS += -Wall -Wstrict-prototypes -Wuninitialized -O2 -g -std=gnu99 \
 	-fno-stack-protector -fno-PIE -fno-strict-aliasing \
 	-I$(LINUX_TOOL_INCLUDE) -I$(LINUX_TOOL_ARCH_INCLUDE) \
 	-I$(LINUX_HDR_PATH) -Iinclude -I$(<D) -Iinclude/$(ARCH) \
-	-I ../rseq -I.. $(EXTRA_CFLAGS) $(KHDR_INCLUDES)
+	-I ../rseq -I.. -I$(OUTPUT)/include/$(ARCH) $(EXTRA_CFLAGS) $(KHDR_INCLUDES)
 ifeq ($(ARCH),s390)
 	CFLAGS += -march=z10
 endif

Reinette

Re: [PATCH v12 08/23] KVM: selftests: Define structs to pass parameters to TDX boot code

Posted by Sean Christopherson 1 month, 2 weeks ago

On Wed, Oct 29, 2025, Reinette Chatre wrote:
> Hi Sagi,
> 
> On 10/28/25 2:20 PM, Sagi Shahar wrote:
> > TDX registers are inaccessible to KVM. Therefore we need a different
> > mechanism to load boot parameters for TDX code. TDX boot code will read
> > the registers values from memory and set the registers manually.
> > 
> > This patch defines the data structures used to communicate between c
> > code and the TDX assembly boot code which will be added in a later
> > patch.
> > 
> 
> (sidenote: I do not know what the bar for this work is so I'll defer
> comments related to local customs like using "we" and "this patch" in
> changelog)

The same as KVM x86, which follows the same rules as the tip tree, with a few
intentional differences.  By all means, call out those things, it'll save me the
effort :-)

Documentation/process/maintainer-kvm-x86.rst

> 
> > Use kbuild.h to expose the offsets into the structs from c code to
> > assembly code.
> > 
> 
> 
> > diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
> > index 148d427ff24b..5e809064ff1c 100644
> > --- a/tools/testing/selftests/kvm/Makefile.kvm
> > +++ b/tools/testing/selftests/kvm/Makefile.kvm
> 
> ...
> 
> > @@ -328,18 +336,28 @@ $(LIBKVM_C_OBJ): $(OUTPUT)/%.o: %.c $(GEN_HDRS)
> >  $(LIBKVM_S_OBJ): $(OUTPUT)/%.o: %.S $(GEN_HDRS)
> >  	$(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c $< -o $@
> >  
> > +$(LIBKVM_ASM_DEFS_OBJ): $(OUTPUT)/%.s: %.c FORCE
> > +	$(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -S $< -o $@
> > +
> >  # Compile the string overrides as freestanding to prevent the compiler from
> >  # generating self-referential code, e.g. without "freestanding" the compiler may
> >  # "optimize" memcmp() by invoking memcmp(), thus causing infinite recursion.
> >  $(LIBKVM_STRING_OBJ): $(OUTPUT)/%.o: %.c
> >  	$(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c -ffreestanding $< -o $@
> >  
> > +$(OUTPUT)/include/x86/tdx/td_boot_offsets.h: $(OUTPUT)/lib/x86/tdx/td_boot_offsets.s FORCE
> > +	$(call filechk,offsets,__TDX_BOOT_OFFSETS_H__)

Presumably this needs to be guarded so that it's x86-only.  I can't tell for sure
as there are other problems in this series of a similar nature that prevent me from
getting far enough to see.  Please build test on at least one other architecture
before sending the next version.

lib/kvm_util.c:7:10: fatal error: tdx/tdx_util.h: No such file or directory
    7 | #include "tdx/tdx_util.h"
      |          ^~~~~~~~~~~~~~~~
compilation terminated.


If possible, I would also really like to see these programatically defined, e.g.
something like (I have no idea if this is remotely valid syntax):

  $(OUTPUT)/$(TEST_GEN_HEADERS): $(OUTPUT)/%.s FORCE
     $(call filechk,offsets,__%_h__)


> Some folks prefer to keep build output separate and may build tests using a command
> line like:
> 	make O=<output dir> TARGETS=kvm -C tools/testing/selftests

Ya, I exclusively build that way.

> This is a valid usage and will result in td_boot_offsets.h placed in <output dir> that
> is not covered by current include path. A build with above command line thus fails:
> 
> lib/x86/tdx/td_boot.S:4:10: fatal error: tdx/td_boot_offsets.h: No such file or directory
>     4 | #include "tdx/td_boot_offsets.h"
>       |          ^~~~~~~~~~~~~~~~~~~~~~~
> compilation terminated.
> 
> 
> Something like below may be needed to add the output directory to the include path:
> 
> diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
> index 2f49c8965df9..98bc40a7f069 100644
> --- a/tools/testing/selftests/kvm/Makefile.kvm
> +++ b/tools/testing/selftests/kvm/Makefile.kvm
> @@ -262,7 +262,7 @@ CFLAGS += -Wall -Wstrict-prototypes -Wuninitialized -O2 -g -std=gnu99 \
>  	-fno-stack-protector -fno-PIE -fno-strict-aliasing \
>  	-I$(LINUX_TOOL_INCLUDE) -I$(LINUX_TOOL_ARCH_INCLUDE) \
>  	-I$(LINUX_HDR_PATH) -Iinclude -I$(<D) -Iinclude/$(ARCH) \
> -	-I ../rseq -I.. $(EXTRA_CFLAGS) $(KHDR_INCLUDES)
> +	-I ../rseq -I.. -I$(OUTPUT)/include/$(ARCH) $(EXTRA_CFLAGS) $(KHDR_INCLUDES)

Hrm, ya, though I assume we want to define e.g. KVM_GEN_HDRS so that they can be
added to the csope.  Note, ARM already has some generated header stuff, but the
generated code comes from outside of KVM selftests, so we'll want to make sure to
avoid a collision with GEN_HDRS, thus the KVM_ prefix.

	ifeq ($(ARCH),arm64)
	tools_dir := $(top_srcdir)/tools
	arm64_tools_dir := $(tools_dir)/arch/arm64/tools/

	ifneq ($(abs_objdir),)
	arm64_hdr_outdir := $(abs_objdir)/tools/
	else
	arm64_hdr_outdir := $(tools_dir)/
	endif

	GEN_HDRS := $(arm64_hdr_outdir)arch/arm64/include/generated/
	CFLAGS += -I$(GEN_HDRS)

	$(GEN_HDRS): $(wildcard $(arm64_tools_dir)/*)
		$(MAKE) -C $(arm64_tools_dir) OUTPUT=$(arm64_hdr_outdir)
	endif