From: Ard Biesheuvel <ardb@kernel.org>

Merge the local include "pgtable.h" -which declares the API of the

5-level paging trampoline- into <asm/boot.h> so that its implementation

in la57toggle.S as well as the calling code can be decoupled from the

traditional decompressor.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>

---

arch/x86/boot/compressed/head_64.S | 1 -

arch/x86/boot/compressed/la57toggle.S | 1 -

arch/x86/boot/compressed/misc.c | 1 -

arch/x86/boot/compressed/pgtable.h | 18 ------------------

arch/x86/boot/compressed/pgtable_64.c | 1 -

arch/x86/include/asm/boot.h | 10 ++++++++++

6 files changed, 10 insertions(+), 22 deletions(-)

diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S

index XXXXXXX..XXXXXXX 100644

--- a/arch/x86/boot/compressed/head_64.S

+++ b/arch/x86/boot/compressed/head_64.S

@@ -XXX,XX +XXX,XX @@

#include <asm/bootparam.h>

#include <asm/desc_defs.h>

#include <asm/trapnr.h>

-#include "pgtable.h"

/*

* Fix alignment at 16 bytes. Following CONFIG_FUNCTION_ALIGNMENT will result

diff --git a/arch/x86/boot/compressed/la57toggle.S b/arch/x86/boot/compressed/la57toggle.S

index XXXXXXX..XXXXXXX 100644

--- a/arch/x86/boot/compressed/la57toggle.S

+++ b/arch/x86/boot/compressed/la57toggle.S

@@ -XXX,XX +XXX,XX @@

#include <asm/boot.h>

#include <asm/msr.h>

#include <asm/processor-flags.h>

-#include "pgtable.h"

/*

* This is the 32-bit trampoline that will be copied over to low memory. It

diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c

index XXXXXXX..XXXXXXX 100644

--- a/arch/x86/boot/compressed/misc.c

+++ b/arch/x86/boot/compressed/misc.c

@@ -XXX,XX +XXX,XX @@

#include "misc.h"

#include "error.h"

-#include "pgtable.h"

#include "../string.h"

#include "../voffset.h"

#include <asm/bootparam_utils.h>

diff --git a/arch/x86/boot/compressed/pgtable.h b/arch/x86/boot/compressed/pgtable.h

deleted file mode 100644

index XXXXXXX..XXXXXXX

--- a/arch/x86/boot/compressed/pgtable.h

+++ /dev/null

@@ -XXX,XX +XXX,XX @@

-#ifndef BOOT_COMPRESSED_PAGETABLE_H

-#define BOOT_COMPRESSED_PAGETABLE_H

-

-#define TRAMPOLINE_32BIT_SIZE        (2 * PAGE_SIZE)

-

-#define TRAMPOLINE_32BIT_CODE_OFFSET    PAGE_SIZE

-#define TRAMPOLINE_32BIT_CODE_SIZE    0xA0

-

-#ifndef __ASSEMBLER__

-

-extern unsigned long *trampoline_32bit;

-

-extern void trampoline_32bit_src(void *trampoline, bool enable_5lvl);

-

-extern const u16 trampoline_ljmp_imm_offset;

-

-#endif /* __ASSEMBLER__ */

-#endif /* BOOT_COMPRESSED_PAGETABLE_H */

diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c

index XXXXXXX..XXXXXXX 100644

--- a/arch/x86/boot/compressed/pgtable_64.c

+++ b/arch/x86/boot/compressed/pgtable_64.c

@@ -XXX,XX +XXX,XX @@

#include <asm/bootparam_utils.h>

#include <asm/e820/types.h>

#include <asm/processor.h>

-#include "pgtable.h"

#include "../string.h"

#include "efi.h"

diff --git a/arch/x86/include/asm/boot.h b/arch/x86/include/asm/boot.h

index XXXXXXX..XXXXXXX 100644

--- a/arch/x86/include/asm/boot.h

+++ b/arch/x86/include/asm/boot.h

@@ -XXX,XX +XXX,XX @@

# define BOOT_STACK_SIZE    0x1000

#endif

+#define TRAMPOLINE_32BIT_SIZE        (2 * PAGE_SIZE)

+

+#define TRAMPOLINE_32BIT_CODE_OFFSET    PAGE_SIZE

+#define TRAMPOLINE_32BIT_CODE_SIZE    0xA0

+

#ifndef __ASSEMBLER__

extern unsigned int output_len;

extern const unsigned long kernel_text_size;

@@ -XXX,XX +XXX,XX @@ unsigned long decompress_kernel(unsigned char *outbuf, unsigned long virt_addr,

                void (*error)(char *x));

extern struct boot_params *boot_params_ptr;

+extern unsigned long *trampoline_32bit;

+extern const u16 trampoline_ljmp_imm_offset;

+

+void trampoline_32bit_src(void *trampoline, bool enable_5lvl);

+

#endif

#endif /* _ASM_X86_BOOT_H */

--

2.49.0.472.ge94155a9ec-goog

From: Ard Biesheuvel <ardb@kernel.org>

The 5-level paging trampoline is used by both the EFI stub and the

traditional decompressor. Move it out of the decompressor sources into

the newly minted arch/x86/boot/startup/ sub-directory which will hold

startup code that may be shared between the decompressor, the EFI stub

and the kernel proper, and needs to tolerate being called during early

boot, before the kernel virtual mapping has been created.

This will allow the 5-level paging trampoline to be used by EFI boot

images such as zboot that omit the traditional decompressor entirely.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>

---

arch/x86/Makefile | 1 +

arch/x86/boot/compressed/Makefile | 2 +-

arch/x86/boot/startup/Makefile | 3 +++

arch/x86/boot/{compressed => startup}/la57toggle.S | 0

4 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/arch/x86/Makefile b/arch/x86/Makefile

index XXXXXXX..XXXXXXX 100644

--- a/arch/x86/Makefile

+++ b/arch/x86/Makefile

@@ -XXX,XX +XXX,XX @@ archprepare: $(cpufeaturemasks.hdr)

###

# Kernel objects

+core-y += arch/x86/boot/startup/

libs-y += arch/x86/lib/

# drivers-y are linked after core-y

diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile

index XXXXXXX..XXXXXXX 100644

--- a/arch/x86/boot/compressed/Makefile

+++ b/arch/x86/boot/compressed/Makefile

@@ -XXX,XX +XXX,XX @@ ifdef CONFIG_X86_64

    vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/mem_encrypt.o

    vmlinux-objs-y += $(obj)/pgtable_64.o

    vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev.o

-    vmlinux-objs-y += $(obj)/la57toggle.o

endif

vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o

@@ -XXX,XX +XXX,XX @@ vmlinux-objs-$(CONFIG_UNACCEPTED_MEMORY) += $(obj)/mem.o

vmlinux-objs-$(CONFIG_EFI) += $(obj)/efi.o

vmlinux-libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a

+vmlinux-libs-$(CONFIG_X86_64)    += $(objtree)/arch/x86/boot/startup/lib.a

$(obj)/vmlinux: $(vmlinux-objs-y) $(vmlinux-libs-y) FORCE

    $(call if_changed,ld)

diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/arch/x86/boot/startup/Makefile

@@ -XXX,XX +XXX,XX @@

+# SPDX-License-Identifier: GPL-2.0

+

+lib-$(CONFIG_X86_64)        += la57toggle.o

diff --git a/arch/x86/boot/compressed/la57toggle.S b/arch/x86/boot/startup/la57toggle.S

similarity index 100%

rename from arch/x86/boot/compressed/la57toggle.S

rename to arch/x86/boot/startup/la57toggle.S

--

2.49.0.472.ge94155a9ec-goog

From: Ard Biesheuvel <ardb@kernel.org>

Linus expressed a strong preference for arch-specific asm code (i.e.,

virtually all of it) to reside under arch/ rather than anywhere else.

So move the EFI mixed mode startup code back, and put it under

arch/x86/boot/startup/ where all shared x86 startup code is going to

live.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>

---

arch/x86/boot/startup/Makefile | 3 +++

drivers/firmware/efi/libstub/x86-mixed.S => arch/x86/boot/startup/efi-mixed.S | 0

drivers/firmware/efi/libstub/Makefile | 1 -

3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile

index XXXXXXX..XXXXXXX 100644

--- a/arch/x86/boot/startup/Makefile

+++ b/arch/x86/boot/startup/Makefile

@@ -XXX,XX +XXX,XX @@

# SPDX-License-Identifier: GPL-2.0

+KBUILD_AFLAGS        += -D__DISABLE_EXPORTS

+

lib-$(CONFIG_X86_64)        += la57toggle.o

+lib-$(CONFIG_EFI_MIXED)        += efi-mixed.o

diff --git a/drivers/firmware/efi/libstub/x86-mixed.S b/arch/x86/boot/startup/efi-mixed.S

similarity index 100%

rename from drivers/firmware/efi/libstub/x86-mixed.S

rename to arch/x86/boot/startup/efi-mixed.S

diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile

index XXXXXXX..XXXXXXX 100644

--- a/drivers/firmware/efi/libstub/Makefile

+++ b/drivers/firmware/efi/libstub/Makefile

@@ -XXX,XX +XXX,XX @@ lib-$(CONFIG_EFI_GENERIC_STUB)    += efi-stub.o string.o intrinsics.o systable.o \

lib-$(CONFIG_ARM)        += arm32-stub.o

lib-$(CONFIG_ARM64)        += kaslr.o arm64.o arm64-stub.o smbios.o

lib-$(CONFIG_X86)        += x86-stub.o smbios.o

-lib-$(CONFIG_EFI_MIXED)        += x86-mixed.o

lib-$(CONFIG_X86_64)        += x86-5lvl.o

lib-$(CONFIG_RISCV)        += kaslr.o riscv.o riscv-stub.o

lib-$(CONFIG_LOONGARCH)        += loongarch.o loongarch-stub.o

--

2.49.0.472.ge94155a9ec-goog

From: Ard Biesheuvel <ardb@kernel.org>

Move the early GDT/IDT setup code that runs long before the kernel

virtual mapping is up into arch/x86/boot/startup/, and build it in a way

that ensures that the code tolerates being called from the 1:1 mapping

of memory.

This allows the RIP_REL_REF() macro uses to be dropped, and removes the

need for emitting the code into the special .head.text section.

Also tweak the sed symbol matching pattern in the decompressor to match

on lower case 't' or 'b', as these will be emitted by Clang for symbols

with hidden linkage.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>

---

arch/x86/boot/compressed/Makefile | 2 +-

arch/x86/boot/startup/Makefile | 15 ++++

arch/x86/boot/startup/gdt_idt.c | 82 ++++++++++++++++++++

arch/x86/kernel/head64.c | 74 ------------------

4 files changed, 98 insertions(+), 75 deletions(-)

diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile

index XXXXXXX..XXXXXXX 100644

--- a/arch/x86/boot/compressed/Makefile

+++ b/arch/x86/boot/compressed/Makefile

@@ -XXX,XX +XXX,XX @@ LDFLAGS_vmlinux += -T

hostprogs    := mkpiggy

HOST_EXTRACFLAGS += -I$(srctree)/tools/include

-sed-voffset := -e 's/^\([0-9a-fA-F]*\) [ABCDGRSTVW] \(_text\|__start_rodata\|__bss_start\|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p'

+sed-voffset := -e 's/^\([0-9a-fA-F]*\) [ABbCDGRSTtVW] \(_text\|__start_rodata\|__bss_start\|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p'

quiet_cmd_voffset = VOFFSET $@

cmd_voffset = $(NM) $< | sed -n $(sed-voffset) > $@

diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile

index XXXXXXX..XXXXXXX 100644

--- a/arch/x86/boot/startup/Makefile

+++ b/arch/x86/boot/startup/Makefile

@@ -XXX,XX +XXX,XX @@

# SPDX-License-Identifier: GPL-2.0

KBUILD_AFLAGS        += -D__DISABLE_EXPORTS

+KBUILD_CFLAGS        += -D__DISABLE_EXPORTS -mcmodel=small -fPIC \

+             -Os -DDISABLE_BRANCH_PROFILING \

+             $(DISABLE_STACKLEAK_PLUGIN) \

+             -fno-stack-protector -D__NO_FORTIFY \

+             -include $(srctree)/include/linux/hidden.h

+

+# disable ftrace hooks

+KBUILD_CFLAGS    := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS))

+KASAN_SANITIZE    := n

+KCSAN_SANITIZE    := n

+KMSAN_SANITIZE    := n

+UBSAN_SANITIZE    := n

+KCOV_INSTRUMENT    := n

+

+obj-$(CONFIG_X86_64)        += gdt_idt.o

lib-$(CONFIG_X86_64)        += la57toggle.o

lib-$(CONFIG_EFI_MIXED)        += efi-mixed.o

diff --git a/arch/x86/boot/startup/gdt_idt.c b/arch/x86/boot/startup/gdt_idt.c

new file mode 100644

index XXXXXXX..XXXXXXX

--- /dev/null

+++ b/arch/x86/boot/startup/gdt_idt.c

@@ -XXX,XX +XXX,XX @@

+// SPDX-License-Identifier: GPL-2.0

+

+#include <linux/linkage.h>

+#include <linux/types.h>

+

+#include <asm/desc.h>

+#include <asm/setup.h>

+#include <asm/sev.h>

+#include <asm/trapnr.h>

+

+/*

+ * Data structures and code used for IDT setup in head_64.S. The bringup-IDT is

+ * used until the idt_table takes over. On the boot CPU this happens in

+ * x86_64_start_kernel(), on secondary CPUs in start_secondary(). In both cases

+ * this happens in the functions called from head_64.S.

+ *

+ * The idt_table can't be used that early because all the code modifying it is

+ * in idt.c and can be instrumented by tracing or KASAN, which both don't work

+ * during early CPU bringup. Also the idt_table has the runtime vectors

+ * configured which require certain CPU state to be setup already (like TSS),

+ * which also hasn't happened yet in early CPU bringup.

+ */

+static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;

+

+/* This may run while still in the direct mapping */

+static void startup_64_load_idt(void *vc_handler)

+{

+    struct desc_ptr desc = {

+        .address = (unsigned long)bringup_idt_table,

+        .size = sizeof(bringup_idt_table) - 1,

+    };

+    struct idt_data data;

+    gate_desc idt_desc;

+

+    /* @vc_handler is set only for a VMM Communication Exception */

+    if (vc_handler) {

+        init_idt_data(&data, X86_TRAP_VC, vc_handler);

+        idt_init_desc(&idt_desc, &data);

+        native_write_idt_entry((gate_desc *)desc.address, X86_TRAP_VC, &idt_desc);

+    }

+

+    native_load_idt(&desc);

+}

+

+/* This is used when running on kernel addresses */

+void early_setup_idt(void)

+{

+    void *handler = NULL;

+

+    if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) {

+        setup_ghcb();

+        handler = vc_boot_ghcb;

+    }

+

+    startup_64_load_idt(handler);

+}

+

+/*

+ * Setup boot CPU state needed before kernel switches to virtual addresses.

+ */

+void __init startup_64_setup_gdt_idt(void)

+{

+    void *handler = NULL;

+

+    struct desc_ptr startup_gdt_descr = {

+        .address = (__force unsigned long)gdt_page.gdt,

+        .size = GDT_SIZE - 1,

+    };

+

+    /* Load GDT */

+    native_load_gdt(&startup_gdt_descr);

+

+    /* New GDT is live - reload data segment registers */

+    asm volatile("movl %%eax, %%ds\n"

+         "movl %%eax, %%ss\n"

+         "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");

+

+    if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))

+        handler = vc_no_ghcb;

+

+    startup_64_load_idt(handler);

+}

diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c

index XXXXXXX..XXXXXXX 100644

--- a/arch/x86/kernel/head64.c

+++ b/arch/x86/kernel/head64.c

@@ -XXX,XX +XXX,XX @@ void __init __noreturn x86_64_start_reservations(char *real_mode_data)

    start_kernel();

}

-

-/*

- * Data structures and code used for IDT setup in head_64.S. The bringup-IDT is

- * used until the idt_table takes over. On the boot CPU this happens in

- * x86_64_start_kernel(), on secondary CPUs in start_secondary(). In both cases

- * this happens in the functions called from head_64.S.

- *

- * The idt_table can't be used that early because all the code modifying it is

- * in idt.c and can be instrumented by tracing or KASAN, which both don't work

- * during early CPU bringup. Also the idt_table has the runtime vectors

- * configured which require certain CPU state to be setup already (like TSS),

- * which also hasn't happened yet in early CPU bringup.

- */

-static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;

-

-/* This may run while still in the direct mapping */

-static void __head startup_64_load_idt(void *vc_handler)

-{

-    struct desc_ptr desc = {

-        .address = (unsigned long)&RIP_REL_REF(bringup_idt_table),

-        .size = sizeof(bringup_idt_table) - 1,

-    };

-    struct idt_data data;

-    gate_desc idt_desc;

-

-    /* @vc_handler is set only for a VMM Communication Exception */

-    if (vc_handler) {

-        init_idt_data(&data, X86_TRAP_VC, vc_handler);

-        idt_init_desc(&idt_desc, &data);

-        native_write_idt_entry((gate_desc *)desc.address, X86_TRAP_VC, &idt_desc);

-    }

-

-    native_load_idt(&desc);

-}

-

-/* This is used when running on kernel addresses */

-void early_setup_idt(void)

-{

-    void *handler = NULL;

-

-    if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) {

-        setup_ghcb();

-        handler = vc_boot_ghcb;

-    }

-

-    startup_64_load_idt(handler);

-}

-

-/*

- * Setup boot CPU state needed before kernel switches to virtual addresses.

- */

-void __head startup_64_setup_gdt_idt(void)

-{

-    struct desc_struct *gdt = (void *)(__force unsigned long)gdt_page.gdt;

-    void *handler = NULL;

-

-    struct desc_ptr startup_gdt_descr = {

-        .address = (unsigned long)&RIP_REL_REF(*gdt),

-        .size = GDT_SIZE - 1,

-    };

-

-    /* Load GDT */

-    native_load_gdt(&startup_gdt_descr);

-

-    /* New GDT is live - reload data segment registers */

-    asm volatile("movl %%eax, %%ds\n"

-         "movl %%eax, %%ss\n"

-         "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");

-

-    if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))

-        handler = &RIP_REL_REF(vc_no_ghcb);

-

-    startup_64_load_idt(handler);

-}

--

2.49.0.472.ge94155a9ec-goog