Series comparison

-[RFC PATCH 0/6] x86: Refactor and consolidate startup code
+[PATCH v3 0/7] x86: Refactor and consolidate startup code
 From: Ard Biesheuvel <ardb@kernel.org>
-Start refactoring the x86 startup code so we keep all the code that is
+Reorganize C code that is used during early boot, either in the
-shared between different boot stages (EFI stub, decompressor, early
+decompressor/EFI stub or the kernel proper, but before the kernel
-startup in the core kernel *) and/or needs to be built in a special way
+virtual mapping is up.
 (due to the fact that it is C code that runs from the 1:1 mapping of
 RAM) in a single place, sharing all the C flags and other runes that are
 needed to disable instrumentation, sanitizers, etc.
-This is an RFC so I have left some things for later, e.g., the SEV-SNP
+v3:
-init code in arch/x86/coco that is shared between all of the above [*]
+- keep rip_rel_ptr() around in PIC code - sadly, it is still needed in
-and will be tricky to disentangle; there are also some known issues in
+  some cases
-that code related to EFI boot that we are addressing in parallel.
+- remove RIP_REL_REF() uses in separate patches
 - keep __head annotations for now, they will all be removed later
 - disable objtool validation for library objects (i.e., pieces that are
   not linked into vmlinux)
 I will follow up with a series that gets rid of .head.text altogether,
 as it will no longer be needed at all once the startup code is checked
 for absolute relocations.
 The SEV startup code needs to be moved first, though, and this is a bit
 more complicated, so I will decouple that effort from this series, also
 because there is a known issue that needs to be fixed first related to
 memory acceptance from the EFI stub.
 Cc: Tom Lendacky <thomas.lendacky@amd.com>
 Cc: Dionna Amalie Glaze <dionnaglaze@google.com>
 Cc: Kevin Loughlin <kevinloughlin@google.com>
-Ard Biesheuvel (6):
+Ard Biesheuvel (7):
-  x86/boot/compressed: Merge local pgtable.h include into asm/boot.h
+  x86/boot/startup: Disable objtool validation for library code
-  x86/boot: Move 5-level paging trampoline into startup code
+  x86/asm: Make rip_rel_ptr() usable from fPIC code
-  x86/boot: Move EFI mixed mode startup code back under arch/x86
+  x86/boot: Move the early GDT/IDT setup code into startup/
   x86/boot: Move early GDT/IDT setup code into startup/
   x86/boot: Move early kernel mapping code into startup/
+  x86/boot: Drop RIP_REL_REF() uses from early mapping code
   x86/boot: Move early SME init code into startup/
+  x86/boot: Drop RIP_REL_REF() uses from SME startup code
- arch/x86/Makefile                                                             |   1 +
+ arch/x86/boot/compressed/Makefile                          |   2 +-
- arch/x86/boot/compressed/Makefile                                             |   4 +-
+ arch/x86/boot/startup/Makefile                             |  22 ++
- arch/x86/boot/compressed/head_64.S                                            |   1 -
+ arch/x86/boot/startup/gdt_idt.c                            |  83 ++++++
- arch/x86/boot/compressed/misc.c                                               |   1 -
+ arch/x86/boot/startup/map_kernel.c                         | 225 ++++++++++++++++
- arch/x86/boot/compressed/pgtable.h                                            |  18 --
+ arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} |  19 +-
- arch/x86/boot/compressed/pgtable_64.c                                         |   1 -
+ arch/x86/coco/sev/core.c                                   |   2 +-
- arch/x86/boot/startup/Makefile                                                |  22 ++
+ arch/x86/coco/sev/shared.c                                 |   4 +-
- drivers/firmware/efi/libstub/x86-mixed.S => arch/x86/boot/startup/efi-mixed.S |   0
+ arch/x86/include/asm/asm.h                                 |   2 +-
- arch/x86/boot/startup/gdt_idt.c                                               |  82 ++++++
+ arch/x86/include/asm/coco.h                                |   2 +-
- arch/x86/boot/{compressed => startup}/la57toggle.S                            |   1 -
+ arch/x86/include/asm/mem_encrypt.h                         |   2 +-
- arch/x86/boot/startup/map_kernel.c                                            | 232 +++++++++++++++
+ arch/x86/kernel/head64.c                                   | 285 +-------------------
- arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c}                    |  45 ++-
+ arch/x86/mm/Makefile                                       |   6 -
- arch/x86/include/asm/boot.h                                                   |  10 +
+files changed, 346 insertions(+), 308 deletions(-)
  arch/x86/include/asm/mem_encrypt.h                                            |   2 +-
  arch/x86/kernel/head64.c                                                      | 302 +-------------------
  arch/x86/mm/Makefile                                                          |   6 -
  drivers/firmware/efi/libstub/Makefile                                         |   1 -
 files changed, 372 insertions(+), 357 deletions(-)
  delete mode 100644 arch/x86/boot/compressed/pgtable.h
  create mode 100644 arch/x86/boot/startup/Makefile
  rename drivers/firmware/efi/libstub/x86-mixed.S => arch/x86/boot/startup/efi-mixed.S (100%)
  create mode 100644 arch/x86/boot/startup/gdt_idt.c
- rename arch/x86/boot/{compressed => startup}/la57toggle.S (99%)
  create mode 100644 arch/x86/boot/startup/map_kernel.c
- rename arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} (92%)
+ rename arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} (97%)
+base-commit: 4f2d1bbc2c92a32fd612e6c3b51832d5c1c3678e
 --
-.49.0.472.ge94155a9ec-goog
+.49.0.504.g3bcea36a83-goog

-[RFC PATCH 3/6] x86/boot: Move EFI mixed mode startup code back under arch/x86
+[PATCH v3 1/7] x86/boot/startup: Disable objtool validation for library code
 From: Ard Biesheuvel <ardb@kernel.org>
-Linus expressed a strong preference for arch-specific asm code (i.e.,
+The library code built under arch/x86/boot/startup is not intended to be
-virtually all of it) to reside under arch/ rather than anywhere else.
+linked into vmlinux but only into the decompressor and/or the EFI stub.
-So move the EFI mixed mode startup code back, and put it under
+This means objtool validation is not needed here, and may result in
-arch/x86/boot/startup/ where all shared x86 startup code is going to
+false positive errors for things like missing retpolines.
-live.
 So disable it for all objects added to lib-y
 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
 ---
- arch/x86/boot/startup/Makefile                                                | 3 +++
+ arch/x86/boot/startup/Makefile | 6 ++++++
- drivers/firmware/efi/libstub/x86-mixed.S => arch/x86/boot/startup/efi-mixed.S | 0
+file changed, 6 insertions(+)
  drivers/firmware/efi/libstub/Makefile                                         | 1 -
 files changed, 3 insertions(+), 1 deletion(-)
 diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/boot/startup/Makefile
 +++ b/arch/x86/boot/startup/Makefile
-@@ -XXX,XX +XXX,XX @@
+@@ -XXX,XX +XXX,XX @@ KBUILD_AFLAGS        += -D__DISABLE_EXPORTS
- # SPDX-License-Identifier: GPL-2.0
+ lib-$(CONFIG_X86_64)        += la57toggle.o
-+KBUILD_AFLAGS        += -D__DISABLE_EXPORTS
+ lib-$(CONFIG_EFI_MIXED)        += efi-mixed.o
 +
- lib-$(CONFIG_X86_64)        += la57toggle.o
++#
-+lib-$(CONFIG_EFI_MIXED)        += efi-mixed.o
++# Disable objtool validation for all library code, which is intended
-diff --git a/drivers/firmware/efi/libstub/x86-mixed.S b/arch/x86/boot/startup/efi-mixed.S
++# to be linked into the decompressor or the EFI stub but not vmlinux
-similarity index 100%
++#
-rename from drivers/firmware/efi/libstub/x86-mixed.S
++$(patsubst %.o,$(obj)/%.o,$(lib-y)): OBJECT_FILES_NON_STANDARD := y
 rename to arch/x86/boot/startup/efi-mixed.S
 diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
 index XXXXXXX..XXXXXXX 100644
 --- a/drivers/firmware/efi/libstub/Makefile
 +++ b/drivers/firmware/efi/libstub/Makefile
@@ -XXX,XX +XXX,XX @@ lib-$(CONFIG_EFI_GENERIC_STUB)    += efi-stub.o string.o intrinsics.o systable.o \
  lib-$(CONFIG_ARM)        += arm32-stub.o
  lib-$(CONFIG_ARM64)        += kaslr.o arm64.o arm64-stub.o smbios.o
  lib-$(CONFIG_X86)        += x86-stub.o smbios.o
 -lib-$(CONFIG_EFI_MIXED)        += x86-mixed.o
  lib-$(CONFIG_X86_64)        += x86-5lvl.o
  lib-$(CONFIG_RISCV)        += kaslr.o riscv.o riscv-stub.o
  lib-$(CONFIG_LOONGARCH)        += loongarch.o loongarch-stub.o
 --
-.49.0.472.ge94155a9ec-goog
+.49.0.504.g3bcea36a83-goog

-[RFC PATCH 1/6] x86/boot/compressed: Merge local pgtable.h include into asm/boot.h
+[PATCH v3 2/7] x86/asm: Make rip_rel_ptr() usable from fPIC code
 From: Ard Biesheuvel <ardb@kernel.org>
-Merge the local include "pgtable.h" -which declares the API of the
+RIP_REL_REF() is used in non-PIC C code that is called very early,
--level paging trampoline- into <asm/boot.h> so that its implementation
+before the kernel virtual mapping is up, which is the mapping that the
-in la57toggle.S as well as the calling code can be decoupled from the
+linker expects. It is currently used in two different ways:
-traditional decompressor.
+- to refer to the value of a global variable, including as an lvalue in
   assignments;
 - to take the address of a global variable via the mapping that the code
   currently executes at.
 The former case is only needed in non-PIC code, as PIC code will never
 use absolute symbol references when the address of the symbol is not
 being used. But taking the address of a variable in PIC code may still
 require extra care, as a stack allocated struct assignment may be
 emitted as a memcpy() from a statically allocated copy in .rodata.
 For instance, this
   void startup_64_setup_gdt_idt(void)
   {
         struct desc_ptr startup_gdt_descr = {
                 .address = (__force unsigned long)gdt_page.gdt,
                 .size    = GDT_SIZE - 1,
         };
 may result in an absolute symbol reference in PIC code, even though the
 struct is allocated on the stack and populated at runtime.
 To address this case, make rip_rel_ptr() accessible in PIC code, and
 update any existing uses where the address of a global variable is
 taken using RIP_REL_REF.
 Once all code of this nature has been moved into arch/x86/boot/startup
 and built with -fPIC, RIP_REL_REF() can be retired, and only
 rip_rel_ptr() will remain.
 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
 ---
- arch/x86/boot/compressed/head_64.S    |  1 -
+ arch/x86/coco/sev/core.c           |  2 +-
- arch/x86/boot/compressed/la57toggle.S |  1 -
+ arch/x86/coco/sev/shared.c         |  4 ++--
- arch/x86/boot/compressed/misc.c       |  1 -
+ arch/x86/include/asm/asm.h         |  2 +-
- arch/x86/boot/compressed/pgtable.h    | 18 ------------------
+ arch/x86/kernel/head64.c           | 23 ++++++++++----------
- arch/x86/boot/compressed/pgtable_64.c |  1 -
+ arch/x86/mm/mem_encrypt_identity.c |  6 ++---
- arch/x86/include/asm/boot.h           | 10 ++++++++++
+files changed, 18 insertions(+), 19 deletions(-)
-files changed, 10 insertions(+), 22 deletions(-)
+diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
-diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S
+index XXXXXXX..XXXXXXX 100644
-index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/coco/sev/core.c
---- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/coco/sev/core.c
-+++ b/arch/x86/boot/compressed/head_64.S
+@@ -XXX,XX +XXX,XX @@ static __head void svsm_setup(struct cc_blob_sev_info *cc_info)
       * kernel was loaded (physbase), so the get the CA address using
       * RIP-relative addressing.
       */
 -    pa = (u64)&RIP_REL_REF(boot_svsm_ca_page);
 +    pa = (u64)rip_rel_ptr(&boot_svsm_ca_page);
      /*
       * Switch over to the boot SVSM CA while the current CA is still
 diff --git a/arch/x86/coco/sev/shared.c b/arch/x86/coco/sev/shared.c
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/coco/sev/shared.c
 +++ b/arch/x86/coco/sev/shared.c
@@ -XXX,XX +XXX,XX @@ static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid
   */
  static const struct snp_cpuid_table *snp_cpuid_get_table(void)
  {
 -    return &RIP_REL_REF(cpuid_table_copy);
 +    return rip_rel_ptr(&cpuid_table_copy);
  }
  /*
@@ -XXX,XX +XXX,XX @@ static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info)
       * routine is running identity mapped when called, both by the decompressor
       * code and the early kernel code.
       */
 -    if (!rmpadjust((unsigned long)&RIP_REL_REF(boot_ghcb_page), RMP_PG_SIZE_4K, 1))
 +    if (!rmpadjust((unsigned long)rip_rel_ptr(&boot_ghcb_page), RMP_PG_SIZE_4K, 1))
          return false;
      /*
 diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/include/asm/asm.h
 +++ b/arch/x86/include/asm/asm.h
 @@ -XXX,XX +XXX,XX @@
- #include <asm/bootparam.h>
- #include <asm/desc_defs.h>
- #include <asm/trapnr.h>
--#include "pgtable.h"
- /*
-  * Fix alignment at 16 bytes. Following CONFIG_FUNCTION_ALIGNMENT will result
-diff --git a/arch/x86/boot/compressed/la57toggle.S b/arch/x86/boot/compressed/la57toggle.S
-index XXXXXXX..XXXXXXX 100644
---- a/arch/x86/boot/compressed/la57toggle.S
-+++ b/arch/x86/boot/compressed/la57toggle.S
-@@ -XXX,XX +XXX,XX @@
- #include <asm/boot.h>
- #include <asm/msr.h>
- #include <asm/processor-flags.h>
--#include "pgtable.h"
- /*
-  * This is the 32-bit trampoline that will be copied over to low memory. It
-diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
-index XXXXXXX..XXXXXXX 100644
---- a/arch/x86/boot/compressed/misc.c
-+++ b/arch/x86/boot/compressed/misc.c
-@@ -XXX,XX +XXX,XX @@
- #include "misc.h"
- #include "error.h"
--#include "pgtable.h"
- #include "../string.h"
- #include "../voffset.h"
- #include <asm/bootparam_utils.h>
-diff --git a/arch/x86/boot/compressed/pgtable.h b/arch/x86/boot/compressed/pgtable.h
-deleted file mode 100644
-index XXXXXXX..XXXXXXX
---- a/arch/x86/boot/compressed/pgtable.h
-+++ /dev/null
-@@ -XXX,XX +XXX,XX @@
--#ifndef BOOT_COMPRESSED_PAGETABLE_H
--#define BOOT_COMPRESSED_PAGETABLE_H
--
--#define TRAMPOLINE_32BIT_SIZE        (2 * PAGE_SIZE)
--
--#define TRAMPOLINE_32BIT_CODE_OFFSET    PAGE_SIZE
--#define TRAMPOLINE_32BIT_CODE_SIZE    0xA0
--
--#ifndef __ASSEMBLER__
--
--extern unsigned long *trampoline_32bit;
--
--extern void trampoline_32bit_src(void *trampoline, bool enable_5lvl);
--
--extern const u16 trampoline_ljmp_imm_offset;
--
--#endif /* __ASSEMBLER__ */
--#endif /* BOOT_COMPRESSED_PAGETABLE_H */
-diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c
-index XXXXXXX..XXXXXXX 100644
---- a/arch/x86/boot/compressed/pgtable_64.c
-+++ b/arch/x86/boot/compressed/pgtable_64.c
-@@ -XXX,XX +XXX,XX @@
- #include <asm/bootparam_utils.h>
- #include <asm/e820/types.h>
- #include <asm/processor.h>
--#include "pgtable.h"
- #include "../string.h"
- #include "efi.h"
-diff --git a/arch/x86/include/asm/boot.h b/arch/x86/include/asm/boot.h
-index XXXXXXX..XXXXXXX 100644
---- a/arch/x86/include/asm/boot.h
-+++ b/arch/x86/include/asm/boot.h
-@@ -XXX,XX +XXX,XX @@
- # define BOOT_STACK_SIZE    0x1000
  #endif
-+#define TRAMPOLINE_32BIT_SIZE        (2 * PAGE_SIZE)
-+
-+#define TRAMPOLINE_32BIT_CODE_OFFSET    PAGE_SIZE
-+#define TRAMPOLINE_32BIT_CODE_SIZE    0xA0
-+
  #ifndef __ASSEMBLER__
- extern unsigned int output_len;
+-#ifndef __pic__
- extern const unsigned long kernel_text_size;
+ static __always_inline __pure void *rip_rel_ptr(void *p)
-@@ -XXX,XX +XXX,XX @@ unsigned long decompress_kernel(unsigned char *outbuf, unsigned long virt_addr,
+ {
-                 void (*error)(char *x));
+     asm("leaq %c1(%%rip), %0" : "=r"(p) : "i"(p));
- extern struct boot_params *boot_params_ptr;
+     return p;
-+extern unsigned long *trampoline_32bit;
+ }
-+extern const u16 trampoline_ljmp_imm_offset;
++#ifndef __pic__
-+
+ #define RIP_REL_REF(var)    (*(typeof(&(var)))rip_rel_ptr(&(var)))
-+void trampoline_32bit_src(void *trampoline, bool enable_5lvl);
+ #else
-+
+ #define RIP_REL_REF(var)    (var)
- #endif
+diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
+index XXXXXXX..XXXXXXX 100644
- #endif /* _ASM_X86_BOOT_H */
+--- a/arch/x86/kernel/head64.c
 +++ b/arch/x86/kernel/head64.c
@@ -XXX,XX +XXX,XX @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
       * attribute.
       */
      if (sme_get_me_mask()) {
 -        paddr = (unsigned long)&RIP_REL_REF(__start_bss_decrypted);
 -        paddr_end = (unsigned long)&RIP_REL_REF(__end_bss_decrypted);
 +        paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted);
 +        paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted);
          for (; paddr < paddr_end; paddr += PMD_SIZE) {
              /*
@@ -XXX,XX +XXX,XX @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
  unsigned long __head __startup_64(unsigned long p2v_offset,
                    struct boot_params *bp)
  {
 -    pmd_t (*early_pgts)[PTRS_PER_PMD] = RIP_REL_REF(early_dynamic_pgts);
 -    unsigned long physaddr = (unsigned long)&RIP_REL_REF(_text);
 +    pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);
 +    unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);
      unsigned long va_text, va_end;
      unsigned long pgtable_flags;
      unsigned long load_delta;
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
          for (;;);
      va_text = physaddr - p2v_offset;
 -    va_end  = (unsigned long)&RIP_REL_REF(_end) - p2v_offset;
 +    va_end  = (unsigned long)rip_rel_ptr(_end) - p2v_offset;
      /* Include the SME encryption mask in the fixup value */
      load_delta += sme_get_me_mask();
      /* Fixup the physical addresses in the page table */
 -    pgd = &RIP_REL_REF(early_top_pgt)->pgd;
 +    pgd = rip_rel_ptr(early_top_pgt);
      pgd[pgd_index(__START_KERNEL_map)] += load_delta;
      if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
 -        p4d = (p4dval_t *)&RIP_REL_REF(level4_kernel_pgt);
 +        p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt);
          p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
          pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
       * error, causing the BIOS to halt the system.
       */
 -    pmd = &RIP_REL_REF(level2_kernel_pgt)->pmd;
 +    pmd = rip_rel_ptr(level2_kernel_pgt);
      /* invalidate pages before the kernel image */
      for (i = 0; i < pmd_index(va_text); i++)
@@ -XXX,XX +XXX,XX @@ static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
  static void __head startup_64_load_idt(void *vc_handler)
  {
      struct desc_ptr desc = {
 -        .address = (unsigned long)&RIP_REL_REF(bringup_idt_table),
 +        .address = (unsigned long)rip_rel_ptr(bringup_idt_table),
          .size    = sizeof(bringup_idt_table) - 1,
      };
      struct idt_data data;
@@ -XXX,XX +XXX,XX @@ void early_setup_idt(void)
   */
  void __head startup_64_setup_gdt_idt(void)
  {
 -    struct desc_struct *gdt = (void *)(__force unsigned long)gdt_page.gdt;
      void *handler = NULL;
      struct desc_ptr startup_gdt_descr = {
 -        .address = (unsigned long)&RIP_REL_REF(*gdt),
 +        .address = (unsigned long)rip_rel_ptr((__force void *)&gdt_page),
          .size    = GDT_SIZE - 1,
      };
@@ -XXX,XX +XXX,XX @@ void __head startup_64_setup_gdt_idt(void)
               "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
      if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
 -        handler = &RIP_REL_REF(vc_no_ghcb);
 +        handler = rip_rel_ptr(vc_no_ghcb);
      startup_64_load_idt(handler);
  }
 diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/mm/mem_encrypt_identity.c
 +++ b/arch/x86/mm/mem_encrypt_identity.c
@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
       *     memory from being cached.
       */
 -    kernel_start = (unsigned long)RIP_REL_REF(_text);
 -    kernel_end = ALIGN((unsigned long)RIP_REL_REF(_end), PMD_SIZE);
 +    kernel_start = (unsigned long)rip_rel_ptr(_text);
 +    kernel_end = ALIGN((unsigned long)rip_rel_ptr(_end), PMD_SIZE);
      kernel_len = kernel_end - kernel_start;
      initrd_start = 0;
@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
       *   pagetable structures for the encryption of the kernel
       *   pagetable structures for workarea (in case not currently mapped)
       */
 -    execute_start = workarea_start = (unsigned long)RIP_REL_REF(sme_workarea);
 +    execute_start = workarea_start = (unsigned long)rip_rel_ptr(sme_workarea);
      execute_end = execute_start + (PAGE_SIZE * 2) + PMD_SIZE;
      execute_len = execute_end - execute_start;
 --
-.49.0.472.ge94155a9ec-goog
+.49.0.504.g3bcea36a83-goog

-[RFC PATCH 4/6] x86/boot: Move early GDT/IDT setup code into startup/
+[PATCH v3 3/7] x86/boot: Move the early GDT/IDT setup code into startup/
 From: Ard Biesheuvel <ardb@kernel.org>
 Move the early GDT/IDT setup code that runs long before the kernel
 virtual mapping is up into arch/x86/boot/startup/, and build it in a way
 that ensures that the code tolerates being called from the 1:1 mapping
-of memory.
+of memory. The code itself is left unchanged by this patch.
 This allows the RIP_REL_REF() macro uses to be dropped, and removes the
 need for emitting the code into the special .head.text section.
 Also tweak the sed symbol matching pattern in the decompressor to match
 on lower case 't' or 'b', as these will be emitted by Clang for symbols
 with hidden linkage.
 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
 ---
  arch/x86/boot/compressed/Makefile |  2 +-
  arch/x86/boot/startup/Makefile    | 15 ++++
- arch/x86/boot/startup/gdt_idt.c   | 82 ++++++++++++++++++++
+ arch/x86/boot/startup/gdt_idt.c   | 83 ++++++++++++++++++++
- arch/x86/kernel/head64.c          | 74 ------------------
+ arch/x86/kernel/head64.c          | 73 -----------------
-files changed, 98 insertions(+), 75 deletions(-)
+files changed, 99 insertions(+), 74 deletions(-)
 diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/boot/compressed/Makefile
 +++ b/arch/x86/boot/compressed/Makefile
 ...
 +
 +#include <linux/linkage.h>
 +#include <linux/types.h>
 +
 +#include <asm/desc.h>
++#include <asm/init.h>
 +#include <asm/setup.h>
 +#include <asm/sev.h>
 +#include <asm/trapnr.h>
 +
 +/*
 ...
 + * which also hasn't happened yet in early CPU bringup.
 + */
 +static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
 +
 +/* This may run while still in the direct mapping */
-+static void startup_64_load_idt(void *vc_handler)
++static void __head startup_64_load_idt(void *vc_handler)
 +{
 +    struct desc_ptr desc = {
-+        .address = (unsigned long)bringup_idt_table,
++        .address = (unsigned long)rip_rel_ptr(bringup_idt_table),
 +        .size    = sizeof(bringup_idt_table) - 1,
 +    };
 +    struct idt_data data;
 +    gate_desc idt_desc;
 +
 ...
 +}
 +
 +/*
 + * Setup boot CPU state needed before kernel switches to virtual addresses.
 + */
-+void __init startup_64_setup_gdt_idt(void)
++void __head startup_64_setup_gdt_idt(void)
 +{
 +    void *handler = NULL;
 +
 +    struct desc_ptr startup_gdt_descr = {
-+        .address = (__force unsigned long)gdt_page.gdt,
++        .address = (unsigned long)rip_rel_ptr((__force void *)&gdt_page),
 +        .size    = GDT_SIZE - 1,
 +    };
 +
 +    /* Load GDT */
 +    native_load_gdt(&startup_gdt_descr);
 ...
 +    asm volatile("movl %%eax, %%ds\n"
 +             "movl %%eax, %%ss\n"
 +             "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
 +
 +    if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
-+        handler = vc_no_ghcb;
++        handler = rip_rel_ptr(vc_no_ghcb);
 +
 +    startup_64_load_idt(handler);
 +}
 diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
 index XXXXXXX..XXXXXXX 100644
 ...
 -
 -/* This may run while still in the direct mapping */
 -static void __head startup_64_load_idt(void *vc_handler)
 -{
 -    struct desc_ptr desc = {
--        .address = (unsigned long)&RIP_REL_REF(bringup_idt_table),
+-        .address = (unsigned long)rip_rel_ptr(bringup_idt_table),
 -        .size    = sizeof(bringup_idt_table) - 1,
 -    };
 -    struct idt_data data;
 -    gate_desc idt_desc;
 -
 ...
 -/*
 - * Setup boot CPU state needed before kernel switches to virtual addresses.
 - */
 -void __head startup_64_setup_gdt_idt(void)
 -{
--    struct desc_struct *gdt = (void *)(__force unsigned long)gdt_page.gdt;
 -    void *handler = NULL;
 -
 -    struct desc_ptr startup_gdt_descr = {
--        .address = (unsigned long)&RIP_REL_REF(*gdt),
+-        .address = (unsigned long)rip_rel_ptr((__force void *)&gdt_page),
 -        .size    = GDT_SIZE - 1,
 -    };
 -
 -    /* Load GDT */
 -    native_load_gdt(&startup_gdt_descr);
 ...
 -    asm volatile("movl %%eax, %%ds\n"
 -             "movl %%eax, %%ss\n"
 -             "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
 -
 -    if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
--        handler = &RIP_REL_REF(vc_no_ghcb);
+-        handler = rip_rel_ptr(vc_no_ghcb);
 -
 -    startup_64_load_idt(handler);
 -}
 --
-.49.0.472.ge94155a9ec-goog
+.49.0.504.g3bcea36a83-goog

-[RFC PATCH 5/6] x86/boot: Move early kernel mapping code into startup/
+[PATCH v3 4/7] x86/boot: Move early kernel mapping code into startup/
 From: Ard Biesheuvel <ardb@kernel.org>
 The startup code that constructs the kernel virtual mapping runs from
 the 1:1 mapping of memory itself, and therefore, cannot use absolute
-symbol references. Move this code into a separate source file under
+symbol references. Before making changes in subsequent patches, move
-arch/x86/boot/startup/ where all such code will be kept from now on.
+this code into a separate source file under arch/x86/boot/startup/ where
+all such code will be kept from now on.
 Since all code here is constructed in a manner that ensures that it
 tolerates running from the 1:1 mapping of memory, any uses of the
 RIP_REL_REF() macro can be dropped, along with __head annotations for
 placing this code in a dedicated startup section.
 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
 ---
  arch/x86/boot/startup/Makefile     |   2 +-
- arch/x86/boot/startup/map_kernel.c | 232 ++++++++++++++++++++
+ arch/x86/boot/startup/map_kernel.c | 224 ++++++++++++++++++++
- arch/x86/kernel/head64.c           | 228 +------------------
+ arch/x86/kernel/head64.c           | 211 +-----------------
-files changed, 234 insertions(+), 228 deletions(-)
+files changed, 226 insertions(+), 211 deletions(-)
 diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/boot/startup/Makefile
 +++ b/arch/x86/boot/startup/Makefile
 ...
 +#include <linux/linkage.h>
 +#include <linux/types.h>
 +#include <linux/kernel.h>
 +#include <linux/pgtable.h>
 +
++#include <asm/init.h>
 +#include <asm/sections.h>
 +#include <asm/setup.h>
 +#include <asm/sev.h>
 +
 +extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
 +extern unsigned int next_early_pgt;
-+
-+#ifdef CONFIG_X86_5LEVEL
-+unsigned int __pgtable_l5_enabled __ro_after_init;
-+unsigned int pgdir_shift __ro_after_init = 39;
-+EXPORT_SYMBOL(pgdir_shift);
-+unsigned int ptrs_per_p4d __ro_after_init = 1;
-+EXPORT_SYMBOL(ptrs_per_p4d);
-+#endif
-+
-+#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT
-+unsigned long page_offset_base __ro_after_init = __PAGE_OFFSET_BASE_L4;
-+EXPORT_SYMBOL(page_offset_base);
-+unsigned long vmalloc_base __ro_after_init = __VMALLOC_BASE_L4;
-+EXPORT_SYMBOL(vmalloc_base);
-+unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4;
-+EXPORT_SYMBOL(vmemmap_base);
-+#endif
 +
 +static inline bool check_la57_support(void)
 +{
 +    if (!IS_ENABLED(CONFIG_X86_5LEVEL))
 +        return false;
 ...
 +     * stage. Only check if it has been enabled there.
 +     */
 +    if (!(native_read_cr4() & X86_CR4_LA57))
 +        return false;
 +
-+    __pgtable_l5_enabled    = 1;
++    RIP_REL_REF(__pgtable_l5_enabled)    = 1;
-+    pgdir_shift        = 48;
++    RIP_REL_REF(pgdir_shift)        = 48;
-+    ptrs_per_p4d        = 512;
++    RIP_REL_REF(ptrs_per_p4d)        = 512;
-+    page_offset_base    = __PAGE_OFFSET_BASE_L5;
++    RIP_REL_REF(page_offset_base)        = __PAGE_OFFSET_BASE_L5;
-+    vmalloc_base        = __VMALLOC_BASE_L5;
++    RIP_REL_REF(vmalloc_base)        = __VMALLOC_BASE_L5;
-+    vmemmap_base        = __VMEMMAP_BASE_L5;
++    RIP_REL_REF(vmemmap_base)        = __VMEMMAP_BASE_L5;
 +
 +    return true;
 +}
 +
-+static unsigned long sme_postprocess_startup(struct boot_params *bp,
++static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
-+                         pmdval_t *pmd,
++                            pmdval_t *pmd,
-+                         unsigned long p2v_offset)
++                            unsigned long p2v_offset)
 +{
 +    unsigned long paddr, paddr_end;
 +    int i;
 +
 +    /* Encrypt the kernel and related (if SME is active) */
 ...
 +     * The bss section will be memset to zero later in the initialization so
 +     * there is no need to zero it after changing the memory encryption
 +     * attribute.
 +     */
 +    if (sme_get_me_mask()) {
-+        paddr = (unsigned long)__start_bss_decrypted;
++        paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted);
-+        paddr_end = (unsigned long)__end_bss_decrypted;
++        paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted);
 +
 +        for (; paddr < paddr_end; paddr += PMD_SIZE) {
 +            /*
 +             * On SNP, transition the page to shared in the RMP table so that
 +             * it is consistent with the page table attribute change.
 ...
 +     * modifier for the initial pgdir entry programmed into CR3.
 +     */
 +    return sme_get_me_mask();
 +}
 +
-+unsigned long __init __startup_64(unsigned long p2v_offset,
++/* Code in __startup_64() can be relocated during execution, but the compiler
 + * doesn't have to generate PC-relative relocations when accessing globals from
 + * that function. Clang actually does not generate them, which leads to
 + * boot-time crashes. To work around this problem, every global pointer must
 + * be accessed using RIP_REL_REF(). Kernel virtual addresses can be determined
 + * by subtracting p2v_offset from the RIP-relative address.
 + */
 +unsigned long __head __startup_64(unsigned long p2v_offset,
 +                  struct boot_params *bp)
 +{
-+    pmd_t (*early_pgts)[PTRS_PER_PMD] = early_dynamic_pgts;
++    pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);
-+    unsigned long physaddr = (unsigned long)_text;
++    unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);
 +    unsigned long va_text, va_end;
 +    unsigned long pgtable_flags;
 +    unsigned long load_delta;
 +    pgdval_t *pgd;
 +    p4dval_t *p4d;
 ...
 +
 +    /*
 +     * Compute the delta between the address I am compiled to run at
 +     * and the address I am actually running at.
 +     */
-+    phys_base = load_delta = __START_KERNEL_map + p2v_offset;
++    load_delta = __START_KERNEL_map + p2v_offset;
 +    RIP_REL_REF(phys_base) = load_delta;
 +
 +    /* Is the address not 2M aligned? */
 +    if (load_delta & ~PMD_MASK)
 +        for (;;);
 +
 +    va_text = physaddr - p2v_offset;
-+    va_end  = (unsigned long)_end - p2v_offset;
++    va_end  = (unsigned long)rip_rel_ptr(_end) - p2v_offset;
 +
 +    /* Include the SME encryption mask in the fixup value */
 +    load_delta += sme_get_me_mask();
 +
 +    /* Fixup the physical addresses in the page table */
 +
-+    pgd = &early_top_pgt[0].pgd;
++    pgd = rip_rel_ptr(early_top_pgt);
 +    pgd[pgd_index(__START_KERNEL_map)] += load_delta;
 +
 +    if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
-+        p4d = (p4dval_t *)level4_kernel_pgt;
++        p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt);
 +        p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
 +
 +        pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
 +    }
 +
-+    level3_kernel_pgt[PTRS_PER_PUD - 2].pud += load_delta;
++    RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 2].pud += load_delta;
-+    level3_kernel_pgt[PTRS_PER_PUD - 1].pud += load_delta;
++    RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 1].pud += load_delta;
 +
 +    for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
-+        level2_fixmap_pgt[i].pmd += load_delta;
++        RIP_REL_REF(level2_fixmap_pgt)[i].pmd += load_delta;
 +
 +    /*
 +     * Set up the identity mapping for the switchover.  These
 +     * entries should *NOT* have the global bit set!  This also
 +     * creates a bunch of nonsense entries but that is fine --
 +     * it avoids problems around wraparound.
 +     */
 +
 +    pud = &early_pgts[0]->pmd;
 +    pmd = &early_pgts[1]->pmd;
-+    next_early_pgt = 2;
++    RIP_REL_REF(next_early_pgt) = 2;
 +
 +    pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
 +
 +    if (la57) {
-+        p4d = &early_pgts[next_early_pgt++]->pmd;
++        p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd;
 +
 +        i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
 +        pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
 +        pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;
 +
 ...
 +    pud[(i + 0) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
 +    pud[(i + 1) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
 +
 +    pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
 +    /* Filter out unsupported __PAGE_KERNEL_* bits: */
-+    pmd_entry &= __supported_pte_mask;
++    pmd_entry &= RIP_REL_REF(__supported_pte_mask);
 +    pmd_entry += sme_get_me_mask();
 +    pmd_entry +=  physaddr;
 +
 +    for (i = 0; i < DIV_ROUND_UP(va_end - va_text, PMD_SIZE); i++) {
 +        int idx = i + (physaddr >> PMD_SHIFT);
 ...
 +     * and on some hardware (particularly the UV platform) even
 +     * speculative access to some reserved areas is caught as an
 +     * error, causing the BIOS to halt the system.
 +     */
 +
-+    pmd = &level2_kernel_pgt[0].pmd;
++    pmd = rip_rel_ptr(level2_kernel_pgt);
 +
 +    /* invalidate pages before the kernel image */
 +    for (i = 0; i < pmd_index(va_text); i++)
 +        pmd[i] &= ~_PAGE_PRESENT;
 +
 ...
  extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
 -static unsigned int __initdata next_early_pgt;
 +unsigned int __initdata next_early_pgt;
  pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
--#ifdef CONFIG_X86_5LEVEL
+ #ifdef CONFIG_X86_5LEVEL
--unsigned int __pgtable_l5_enabled __ro_after_init;
+@@ -XXX,XX +XXX,XX @@ unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4;
--unsigned int pgdir_shift __ro_after_init = 39;
+ EXPORT_SYMBOL(vmemmap_base);
--EXPORT_SYMBOL(pgdir_shift);
+ #endif
--unsigned int ptrs_per_p4d __ro_after_init = 1;
 -EXPORT_SYMBOL(ptrs_per_p4d);
 -#endif
 -
 -#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT
 -unsigned long page_offset_base __ro_after_init = __PAGE_OFFSET_BASE_L4;
 -EXPORT_SYMBOL(page_offset_base);
 -unsigned long vmalloc_base __ro_after_init = __VMALLOC_BASE_L4;
 -EXPORT_SYMBOL(vmalloc_base);
 -unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4;
 -EXPORT_SYMBOL(vmemmap_base);
 -#endif
 -
 -static inline bool check_la57_support(void)
 -{
 -    if (!IS_ENABLED(CONFIG_X86_5LEVEL))
 -        return false;
 -
 ...
 -     * The bss section will be memset to zero later in the initialization so
 -     * there is no need to zero it after changing the memory encryption
 -     * attribute.
 -     */
 -    if (sme_get_me_mask()) {
--        paddr = (unsigned long)&RIP_REL_REF(__start_bss_decrypted);
+-        paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted);
--        paddr_end = (unsigned long)&RIP_REL_REF(__end_bss_decrypted);
+-        paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted);
 -
 -        for (; paddr < paddr_end; paddr += PMD_SIZE) {
 -            /*
 -             * On SNP, transition the page to shared in the RMP table so that
 -             * it is consistent with the page table attribute change.
 ...
 - * by subtracting p2v_offset from the RIP-relative address.
 - */
 -unsigned long __head __startup_64(unsigned long p2v_offset,
 -                  struct boot_params *bp)
 -{
--    pmd_t (*early_pgts)[PTRS_PER_PMD] = RIP_REL_REF(early_dynamic_pgts);
+-    pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);
--    unsigned long physaddr = (unsigned long)&RIP_REL_REF(_text);
+-    unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);
 -    unsigned long va_text, va_end;
 -    unsigned long pgtable_flags;
 -    unsigned long load_delta;
 -    pgdval_t *pgd;
 -    p4dval_t *p4d;
 ...
 -    /* Is the address not 2M aligned? */
 -    if (load_delta & ~PMD_MASK)
 -        for (;;);
 -
 -    va_text = physaddr - p2v_offset;
--    va_end  = (unsigned long)&RIP_REL_REF(_end) - p2v_offset;
+-    va_end  = (unsigned long)rip_rel_ptr(_end) - p2v_offset;
 -
 -    /* Include the SME encryption mask in the fixup value */
 -    load_delta += sme_get_me_mask();
 -
 -    /* Fixup the physical addresses in the page table */
 -
--    pgd = &RIP_REL_REF(early_top_pgt)->pgd;
+-    pgd = rip_rel_ptr(early_top_pgt);
 -    pgd[pgd_index(__START_KERNEL_map)] += load_delta;
 -
 -    if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
--        p4d = (p4dval_t *)&RIP_REL_REF(level4_kernel_pgt);
+-        p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt);
 -        p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
 -
 -        pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
 -    }
 -
 ...
 -     * and on some hardware (particularly the UV platform) even
 -     * speculative access to some reserved areas is caught as an
 -     * error, causing the BIOS to halt the system.
 -     */
 -
--    pmd = &RIP_REL_REF(level2_kernel_pgt)->pmd;
+-    pmd = rip_rel_ptr(level2_kernel_pgt);
 -
 -    /* invalidate pages before the kernel image */
 -    for (i = 0; i < pmd_index(va_text); i++)
 -        pmd[i] &= ~_PAGE_PRESENT;
 -
 ...
 -
  /* Wipe all early page tables except for the kernel symbol map */
  static void __init reset_early_page_tables(void)
  {
 --
-.49.0.472.ge94155a9ec-goog
+.49.0.504.g3bcea36a83-goog

-New patch
+[PATCH v3 5/7] x86/boot: Drop RIP_REL_REF() uses from early mapping code
+From: Ard Biesheuvel <ardb@kernel.org>
+Now that __startup_64() is built using -fPIC, RIP_REL_REF() has become a
+NOP and can be removed. Only some occurrences of rip_rel_ptr() will
+remain, to explicitly take the address of certain global structures in
+the 1:1 mapping of memory.
+While at it, update the code comment to describe why this is needed.
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+---
+ arch/x86/boot/startup/map_kernel.c | 41 ++++++++++----------
+file changed, 21 insertions(+), 20 deletions(-)
+diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c
+index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/boot/startup/map_kernel.c
++++ b/arch/x86/boot/startup/map_kernel.c
+@@ -XXX,XX +XXX,XX @@ static inline bool check_la57_support(void)
+     if (!(native_read_cr4() & X86_CR4_LA57))
+         return false;
+-    RIP_REL_REF(__pgtable_l5_enabled)    = 1;
+-    RIP_REL_REF(pgdir_shift)        = 48;
+-    RIP_REL_REF(ptrs_per_p4d)        = 512;
+-    RIP_REL_REF(page_offset_base)        = __PAGE_OFFSET_BASE_L5;
+-    RIP_REL_REF(vmalloc_base)        = __VMALLOC_BASE_L5;
+-    RIP_REL_REF(vmemmap_base)        = __VMEMMAP_BASE_L5;
++    __pgtable_l5_enabled    = 1;
++    pgdir_shift        = 48;
++    ptrs_per_p4d        = 512;
++    page_offset_base    = __PAGE_OFFSET_BASE_L5;
++    vmalloc_base        = __VMALLOC_BASE_L5;
++    vmemmap_base        = __VMEMMAP_BASE_L5;
+     return true;
+ }
+@@ -XXX,XX +XXX,XX @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
+     return sme_get_me_mask();
+ }
+-/* Code in __startup_64() can be relocated during execution, but the compiler
+- * doesn't have to generate PC-relative relocations when accessing globals from
+- * that function. Clang actually does not generate them, which leads to
+- * boot-time crashes. To work around this problem, every global pointer must
+- * be accessed using RIP_REL_REF(). Kernel virtual addresses can be determined
+- * by subtracting p2v_offset from the RIP-relative address.
++/*
++ * This code is compiled using PIC codegen because it will execute from the
++ * early 1:1 mapping of memory, which deviates from the mapping expected by the
++ * linker. Due to this deviation, taking the address of a global variable will
++ * produce an ambiguous result when using the plain & operator.  Instead,
++ * rip_rel_ptr() must be used, which will return the RIP-relative address in
++ * the 1:1 mapping of memory. Kernel virtual addresses can be determined by
++ * subtracting p2v_offset from the RIP-relative address.
+  */
+ unsigned long __head __startup_64(unsigned long p2v_offset,
+                   struct boot_params *bp)
+@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
+      * Compute the delta between the address I am compiled to run at
+      * and the address I am actually running at.
+      */
+-    load_delta = __START_KERNEL_map + p2v_offset;
+-    RIP_REL_REF(phys_base) = load_delta;
++    phys_base = load_delta = __START_KERNEL_map + p2v_offset;
+     /* Is the address not 2M aligned? */
+     if (load_delta & ~PMD_MASK)
+@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
+         pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
+     }
+-    RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 2].pud += load_delta;
+-    RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 1].pud += load_delta;
++    level3_kernel_pgt[PTRS_PER_PUD - 2].pud += load_delta;
++    level3_kernel_pgt[PTRS_PER_PUD - 1].pud += load_delta;
+     for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
+-        RIP_REL_REF(level2_fixmap_pgt)[i].pmd += load_delta;
++        level2_fixmap_pgt[i].pmd += load_delta;
+     /*
+      * Set up the identity mapping for the switchover.  These
+@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
+     pud = &early_pgts[0]->pmd;
+     pmd = &early_pgts[1]->pmd;
+-    RIP_REL_REF(next_early_pgt) = 2;
++    next_early_pgt = 2;
+     pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
+     if (la57) {
+-        p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd;
++        p4d = &early_pgts[next_early_pgt++]->pmd;
+         i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
+         pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
+@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
+     pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
+     /* Filter out unsupported __PAGE_KERNEL_* bits: */
+-    pmd_entry &= RIP_REL_REF(__supported_pte_mask);
++    pmd_entry &= __supported_pte_mask;
+     pmd_entry += sme_get_me_mask();
+     pmd_entry +=  physaddr;
+--
+.49.0.504.g3bcea36a83-goog

-[RFC PATCH 6/6] x86/boot: Move early SME init code into startup/
+[PATCH v3 6/7] x86/boot: Move early SME init code into startup/
 ...
 Move the SME initialization code, which runs from the 1:1 mapping of
 memory as it operates on the kernel virtual mapping, into the new
 sub-directory arch/x86/boot/startup/ where all startup code will reside
 that needs to tolerate executing from the 1:1 mapping.
-This allows RIP_REL_REF() macro invocations and __head annotations to be
-dropped.
 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
 ---
- arch/x86/boot/startup/Makefile                             |  1 +
+ arch/x86/boot/startup/Makefile                             | 1 +
- arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} | 45 +++++++++-----------
+ arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} | 2 --
- arch/x86/include/asm/mem_encrypt.h                         |  2 +-
+ arch/x86/mm/Makefile                                       | 6 ------
- arch/x86/mm/Makefile                                       |  6 ---
+files changed, 1 insertion(+), 8 deletions(-)
 files changed, 23 insertions(+), 31 deletions(-)
 diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/boot/startup/Makefile
 +++ b/arch/x86/boot/startup/Makefile
 ...
 +obj-$(CONFIG_AMD_MEM_ENCRYPT)    += sme.o
  lib-$(CONFIG_X86_64)        += la57toggle.o
  lib-$(CONFIG_EFI_MIXED)        += efi-mixed.o
 diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/boot/startup/sme.c
-similarity index 92%
+similarity index 99%
 rename from arch/x86/mm/mem_encrypt_identity.c
 rename to arch/x86/boot/startup/sme.c
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/mm/mem_encrypt_identity.c
 +++ b/arch/x86/boot/startup/sme.c
 ...
 -#include "mm_internal.h"
 -
  #define PGD_FLAGS        _KERNPG_TABLE_NOENC
  #define P4D_FLAGS        _KERNPG_TABLE_NOENC
  #define PUD_FLAGS        _KERNPG_TABLE_NOENC
-@@ -XXX,XX +XXX,XX @@ struct sme_populate_pgd_data {
-  */
- static char sme_workarea[2 * PMD_SIZE] __section(".init.scratch");
--static void __head sme_clear_pgd(struct sme_populate_pgd_data *ppd)
-+static void __init sme_clear_pgd(struct sme_populate_pgd_data *ppd)
- {
-     unsigned long pgd_start, pgd_end, pgd_size;
-     pgd_t *pgd_p;
-@@ -XXX,XX +XXX,XX @@ static void __head sme_clear_pgd(struct sme_populate_pgd_data *ppd)
-     memset(pgd_p, 0, pgd_size);
- }
--static pud_t __head *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
-+static pud_t __init *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
- {
-     pgd_t *pgd;
-     p4d_t *p4d;
-@@ -XXX,XX +XXX,XX @@ static pud_t __head *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
-     return pud;
- }
--static void __head sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
-+static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
- {
-     pud_t *pud;
-     pmd_t *pmd;
-@@ -XXX,XX +XXX,XX @@ static void __head sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
-     set_pmd(pmd, __pmd(ppd->paddr | ppd->pmd_flags));
- }
--static void __head sme_populate_pgd(struct sme_populate_pgd_data *ppd)
-+static void __init sme_populate_pgd(struct sme_populate_pgd_data *ppd)
- {
-     pud_t *pud;
-     pmd_t *pmd;
-@@ -XXX,XX +XXX,XX @@ static void __head sme_populate_pgd(struct sme_populate_pgd_data *ppd)
-         set_pte(pte, __pte(ppd->paddr | ppd->pte_flags));
- }
--static void __head __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
-+static void __init __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
- {
-     while (ppd->vaddr < ppd->vaddr_end) {
-         sme_populate_pgd_large(ppd);
-@@ -XXX,XX +XXX,XX @@ static void __head __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
-     }
- }
--static void __head __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
-+static void __init __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
- {
-     while (ppd->vaddr < ppd->vaddr_end) {
-         sme_populate_pgd(ppd);
-@@ -XXX,XX +XXX,XX @@ static void __head __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
-     }
- }
--static void __head __sme_map_range(struct sme_populate_pgd_data *ppd,
-+static void __init __sme_map_range(struct sme_populate_pgd_data *ppd,
-                    pmdval_t pmd_flags, pteval_t pte_flags)
- {
-     unsigned long vaddr_end;
-@@ -XXX,XX +XXX,XX @@ static void __head __sme_map_range(struct sme_populate_pgd_data *ppd,
-     __sme_map_range_pte(ppd);
- }
--static void __head sme_map_range_encrypted(struct sme_populate_pgd_data *ppd)
-+static void __init sme_map_range_encrypted(struct sme_populate_pgd_data *ppd)
- {
-     __sme_map_range(ppd, PMD_FLAGS_ENC, PTE_FLAGS_ENC);
- }
--static void __head sme_map_range_decrypted(struct sme_populate_pgd_data *ppd)
-+static void __init sme_map_range_decrypted(struct sme_populate_pgd_data *ppd)
- {
-     __sme_map_range(ppd, PMD_FLAGS_DEC, PTE_FLAGS_DEC);
- }
--static void __head sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd)
-+static void __init sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd)
- {
-     __sme_map_range(ppd, PMD_FLAGS_DEC_WP, PTE_FLAGS_DEC_WP);
- }
--static unsigned long __head sme_pgtable_calc(unsigned long len)
-+static unsigned long __init sme_pgtable_calc(unsigned long len)
- {
-     unsigned long entries = 0, tables = 0;
-@@ -XXX,XX +XXX,XX @@ static unsigned long __head sme_pgtable_calc(unsigned long len)
-     return entries + tables;
- }
--void __head sme_encrypt_kernel(struct boot_params *bp)
-+void __init sme_encrypt_kernel(struct boot_params *bp)
- {
-     unsigned long workarea_start, workarea_end, workarea_len;
-     unsigned long execute_start, execute_end, execute_len;
-@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
-      * instrumentation or checking boot_cpu_data in the cc_platform_has()
-      * function.
-      */
--    if (!sme_get_me_mask() ||
--        RIP_REL_REF(sev_status) & MSR_AMD64_SEV_ENABLED)
-+    if (!sme_get_me_mask() || sev_status & MSR_AMD64_SEV_ENABLED)
-         return;
-     /*
-@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
-      *     memory from being cached.
-      */
--    kernel_start = (unsigned long)RIP_REL_REF(_text);
--    kernel_end = ALIGN((unsigned long)RIP_REL_REF(_end), PMD_SIZE);
-+    kernel_start = (unsigned long)_text;
-+    kernel_end = ALIGN((unsigned long)_end, PMD_SIZE);
-     kernel_len = kernel_end - kernel_start;
-     initrd_start = 0;
-@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
-      *   pagetable structures for the encryption of the kernel
-      *   pagetable structures for workarea (in case not currently mapped)
-      */
--    execute_start = workarea_start = (unsigned long)RIP_REL_REF(sme_workarea);
-+    execute_start = workarea_start = (unsigned long)sme_workarea;
-     execute_end = execute_start + (PAGE_SIZE * 2) + PMD_SIZE;
-     execute_len = execute_end - execute_start;
-@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
-     native_write_cr3(__native_read_cr3());
- }
--void __head sme_enable(struct boot_params *bp)
-+void __init sme_enable(struct boot_params *bp)
- {
-     unsigned int eax, ebx, ecx, edx;
-     unsigned long feature_mask;
-@@ -XXX,XX +XXX,XX @@ void __head sme_enable(struct boot_params *bp)
-     me_mask = 1UL << (ebx & 0x3f);
-     /* Check the SEV MSR whether SEV or SME is enabled */
--    RIP_REL_REF(sev_status) = msr = __rdmsr(MSR_AMD64_SEV);
-+    sev_status = msr = __rdmsr(MSR_AMD64_SEV);
-     feature_mask = (msr & MSR_AMD64_SEV_ENABLED) ? AMD_SEV_BIT : AMD_SME_BIT;
-     /*
-@@ -XXX,XX +XXX,XX @@ void __head sme_enable(struct boot_params *bp)
-             return;
-     }
--    RIP_REL_REF(sme_me_mask) = me_mask;
--    RIP_REL_REF(physical_mask) &= ~me_mask;
--    RIP_REL_REF(cc_vendor) = CC_VENDOR_AMD;
-+    sme_me_mask    = me_mask;
-+    physical_mask    &= ~me_mask;
-+    cc_vendor    = CC_VENDOR_AMD;
-     cc_set_mask(me_mask);
- }
-diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h
-index XXXXXXX..XXXXXXX 100644
---- a/arch/x86/include/asm/mem_encrypt.h
-+++ b/arch/x86/include/asm/mem_encrypt.h
-@@ -XXX,XX +XXX,XX @@ void __init sev_es_init_vc_handling(void);
- static inline u64 sme_get_me_mask(void)
- {
--    return RIP_REL_REF(sme_me_mask);
-+    return sme_me_mask;
- }
- #define __bss_decrypted __section(".bss..decrypted")
 diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/mm/Makefile
 +++ b/arch/x86/mm/Makefile
 @@ -XXX,XX +XXX,XX @@
 ...
  obj-$(CONFIG_AMD_MEM_ENCRYPT)    += mem_encrypt_amd.o
 -obj-$(CONFIG_AMD_MEM_ENCRYPT)    += mem_encrypt_identity.o
  obj-$(CONFIG_AMD_MEM_ENCRYPT)    += mem_encrypt_boot.o
 --
-.49.0.472.ge94155a9ec-goog
+.49.0.504.g3bcea36a83-goog

-[RFC PATCH 2/6] x86/boot: Move 5-level paging trampoline into startup code
+[PATCH v3 7/7] x86/boot: Drop RIP_REL_REF() uses from SME startup code
 From: Ard Biesheuvel <ardb@kernel.org>
-The 5-level paging trampoline is used by both the EFI stub and the
+RIP_REL_REF() has no effect on code residing in arch/x86/boot/startup,
-traditional decompressor. Move it out of the decompressor sources into
+as it is built with -fPIC. So remove any occurrences from the SME
-the newly minted arch/x86/boot/startup/ sub-directory which will hold
+startup code.
 startup code that may be shared between the decompressor, the EFI stub
 and the kernel proper, and needs to tolerate being called during early
 boot, before the kernel virtual mapping has been created.
-This will allow the 5-level paging trampoline to be used by EFI boot
+Note the SME is the only caller of cc_set_mask() that requires this, so
-images such as zboot that omit the traditional decompressor entirely.
+drop it from there as well.
 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
 ---
- arch/x86/Makefile                                  | 1 +
+ arch/x86/boot/startup/sme.c        | 11 +++++------
- arch/x86/boot/compressed/Makefile                  | 2 +-
+ arch/x86/include/asm/coco.h        |  2 +-
- arch/x86/boot/startup/Makefile                     | 3 +++
+ arch/x86/include/asm/mem_encrypt.h |  2 +-
- arch/x86/boot/{compressed => startup}/la57toggle.S | 0
+files changed, 7 insertions(+), 8 deletions(-)
 files changed, 5 insertions(+), 1 deletion(-)
-diff --git a/arch/x86/Makefile b/arch/x86/Makefile
+diff --git a/arch/x86/boot/startup/sme.c b/arch/x86/boot/startup/sme.c
 index XXXXXXX..XXXXXXX 100644
---- a/arch/x86/Makefile
+--- a/arch/x86/boot/startup/sme.c
-+++ b/arch/x86/Makefile
++++ b/arch/x86/boot/startup/sme.c
-@@ -XXX,XX +XXX,XX @@ archprepare: $(cpufeaturemasks.hdr)
+@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
- ###
+      * instrumentation or checking boot_cpu_data in the cc_platform_has()
- # Kernel objects
+      * function.
+      */
-+core-y  += arch/x86/boot/startup/
+-    if (!sme_get_me_mask() ||
- libs-y  += arch/x86/lib/
+-        RIP_REL_REF(sev_status) & MSR_AMD64_SEV_ENABLED)
++    if (!sme_get_me_mask() || sev_status & MSR_AMD64_SEV_ENABLED)
- # drivers-y are linked after core-y
+         return;
-diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
      /*
@@ -XXX,XX +XXX,XX @@ void __head sme_enable(struct boot_params *bp)
      me_mask = 1UL << (ebx & 0x3f);
      /* Check the SEV MSR whether SEV or SME is enabled */
 -    RIP_REL_REF(sev_status) = msr = __rdmsr(MSR_AMD64_SEV);
 +    sev_status = msr = __rdmsr(MSR_AMD64_SEV);
      feature_mask = (msr & MSR_AMD64_SEV_ENABLED) ? AMD_SEV_BIT : AMD_SME_BIT;
      /*
@@ -XXX,XX +XXX,XX @@ void __head sme_enable(struct boot_params *bp)
              return;
      }
 -    RIP_REL_REF(sme_me_mask) = me_mask;
 -    RIP_REL_REF(physical_mask) &= ~me_mask;
 -    RIP_REL_REF(cc_vendor) = CC_VENDOR_AMD;
 +    sme_me_mask    = me_mask;
 +    physical_mask    &= ~me_mask;
 +    cc_vendor    = CC_VENDOR_AMD;
      cc_set_mask(me_mask);
  }
 diff --git a/arch/x86/include/asm/coco.h b/arch/x86/include/asm/coco.h
 index XXXXXXX..XXXXXXX 100644
---- a/arch/x86/boot/compressed/Makefile
+--- a/arch/x86/include/asm/coco.h
-+++ b/arch/x86/boot/compressed/Makefile
++++ b/arch/x86/include/asm/coco.h
-@@ -XXX,XX +XXX,XX @@ ifdef CONFIG_X86_64
+@@ -XXX,XX +XXX,XX @@ static inline u64 cc_get_mask(void)
-     vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/mem_encrypt.o
-     vmlinux-objs-y += $(obj)/pgtable_64.o
+ static inline void cc_set_mask(u64 mask)
-     vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev.o
+ {
--    vmlinux-objs-y += $(obj)/la57toggle.o
+-    RIP_REL_REF(cc_mask) = mask;
- endif
++    cc_mask = mask;
+ }
- vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o
-@@ -XXX,XX +XXX,XX @@ vmlinux-objs-$(CONFIG_UNACCEPTED_MEMORY) += $(obj)/mem.o
+ u64 cc_mkenc(u64 val);
+diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h
- vmlinux-objs-$(CONFIG_EFI) += $(obj)/efi.o
+index XXXXXXX..XXXXXXX 100644
- vmlinux-libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a
+--- a/arch/x86/include/asm/mem_encrypt.h
-+vmlinux-libs-$(CONFIG_X86_64)    += $(objtree)/arch/x86/boot/startup/lib.a
++++ b/arch/x86/include/asm/mem_encrypt.h
+@@ -XXX,XX +XXX,XX @@ void __init sev_es_init_vc_handling(void);
- $(obj)/vmlinux: $(vmlinux-objs-y) $(vmlinux-libs-y) FORCE
-     $(call if_changed,ld)
+ static inline u64 sme_get_me_mask(void)
-diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
+ {
-new file mode 100644
+-    return RIP_REL_REF(sme_me_mask);
-index XXXXXXX..XXXXXXX
++    return sme_me_mask;
---- /dev/null
+ }
-+++ b/arch/x86/boot/startup/Makefile
-@@ -XXX,XX +XXX,XX @@
+ #define __bss_decrypted __section(".bss..decrypted")
 +# SPDX-License-Identifier: GPL-2.0
 +
 +lib-$(CONFIG_X86_64)        += la57toggle.o
 diff --git a/arch/x86/boot/compressed/la57toggle.S b/arch/x86/boot/startup/la57toggle.S
 similarity index 100%
 rename from arch/x86/boot/compressed/la57toggle.S
 rename to arch/x86/boot/startup/la57toggle.S
 --
-.49.0.472.ge94155a9ec-goog
+.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

Start refactoring the x86 startup code so we keep all the code that is
shared between different boot stages (EFI stub, decompressor, early
startup in the core kernel *) and/or needs to be built in a special way
(due to the fact that it is C code that runs from the 1:1 mapping of
RAM) in a single place, sharing all the C flags and other runes that are
needed to disable instrumentation, sanitizers, etc.

This is an RFC so I have left some things for later, e.g., the SEV-SNP
init code in arch/x86/coco that is shared between all of the above [*]
and will be tricky to disentangle; there are also some known issues in
that code related to EFI boot that we are addressing in parallel.

Cc: Tom Lendacky <thomas.lendacky@amd.com>
Cc: Dionna Amalie Glaze <dionnaglaze@google.com>
Cc: Kevin Loughlin <kevinloughlin@google.com>

Ard Biesheuvel (6):
  x86/boot/compressed: Merge local pgtable.h include into asm/boot.h
  x86/boot: Move 5-level paging trampoline into startup code
  x86/boot: Move EFI mixed mode startup code back under arch/x86
  x86/boot: Move early GDT/IDT setup code into startup/
  x86/boot: Move early kernel mapping code into startup/
  x86/boot: Move early SME init code into startup/

arch/x86/Makefile                                                             |   1 +
 arch/x86/boot/compressed/Makefile                                             |   4 +-
 arch/x86/boot/compressed/head_64.S                                            |   1 -
 arch/x86/boot/compressed/misc.c                                               |   1 -
 arch/x86/boot/compressed/pgtable.h                                            |  18 --
 arch/x86/boot/compressed/pgtable_64.c                                         |   1 -
 arch/x86/boot/startup/Makefile                                                |  22 ++
 drivers/firmware/efi/libstub/x86-mixed.S => arch/x86/boot/startup/efi-mixed.S |   0
 arch/x86/boot/startup/gdt_idt.c                                               |  82 ++++++
 arch/x86/boot/{compressed => startup}/la57toggle.S                            |   1 -
 arch/x86/boot/startup/map_kernel.c                                            | 232 +++++++++++++++
 arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c}                    |  45 ++-
 arch/x86/include/asm/boot.h                                                   |  10 +
 arch/x86/include/asm/mem_encrypt.h                                            |   2 +-
 arch/x86/kernel/head64.c                                                      | 302 +-------------------
 arch/x86/mm/Makefile                                                          |   6 -
 drivers/firmware/efi/libstub/Makefile                                         |   1 -
 17 files changed, 372 insertions(+), 357 deletions(-)
 delete mode 100644 arch/x86/boot/compressed/pgtable.h
 create mode 100644 arch/x86/boot/startup/Makefile
 rename drivers/firmware/efi/libstub/x86-mixed.S => arch/x86/boot/startup/efi-mixed.S (100%)
 create mode 100644 arch/x86/boot/startup/gdt_idt.c
 rename arch/x86/boot/{compressed => startup}/la57toggle.S (99%)
 create mode 100644 arch/x86/boot/startup/map_kernel.c
 rename arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} (92%)

-- 
2.49.0.472.ge94155a9ec-goog

From: Ard Biesheuvel <ardb@kernel.org>

Merge the local include "pgtable.h" -which declares the API of the
5-level paging trampoline- into <asm/boot.h> so that its implementation
in la57toggle.S as well as the calling code can be decoupled from the
traditional decompressor.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/compressed/head_64.S    |  1 -
 arch/x86/boot/compressed/la57toggle.S |  1 -
 arch/x86/boot/compressed/misc.c       |  1 -
 arch/x86/boot/compressed/pgtable.h    | 18 ------------------
 arch/x86/boot/compressed/pgtable_64.c |  1 -
 arch/x86/include/asm/boot.h           | 10 ++++++++++
 6 files changed, 10 insertions(+), 22 deletions(-)

diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/head_64.S
+++ b/arch/x86/boot/compressed/head_64.S
@@ -XXX,XX +XXX,XX @@
 #include <asm/bootparam.h>
 #include <asm/desc_defs.h>
 #include <asm/trapnr.h>
-#include "pgtable.h"
 
 /*
  * Fix alignment at 16 bytes. Following CONFIG_FUNCTION_ALIGNMENT will result
diff --git a/arch/x86/boot/compressed/la57toggle.S b/arch/x86/boot/compressed/la57toggle.S
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/la57toggle.S
+++ b/arch/x86/boot/compressed/la57toggle.S
@@ -XXX,XX +XXX,XX @@
 #include <asm/boot.h>
 #include <asm/msr.h>
 #include <asm/processor-flags.h>
-#include "pgtable.h"
 
 /*
  * This is the 32-bit trampoline that will be copied over to low memory. It
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -XXX,XX +XXX,XX @@
 
 #include "misc.h"
 #include "error.h"
-#include "pgtable.h"
 #include "../string.h"
 #include "../voffset.h"
 #include <asm/bootparam_utils.h>
diff --git a/arch/x86/boot/compressed/pgtable.h b/arch/x86/boot/compressed/pgtable.h
deleted file mode 100644
index XXXXXXX..XXXXXXX
--- a/arch/x86/boot/compressed/pgtable.h
+++ /dev/null
@@ -XXX,XX +XXX,XX @@
-#ifndef BOOT_COMPRESSED_PAGETABLE_H
-#define BOOT_COMPRESSED_PAGETABLE_H
-
-#define TRAMPOLINE_32BIT_SIZE		(2 * PAGE_SIZE)
-
-#define TRAMPOLINE_32BIT_CODE_OFFSET	PAGE_SIZE
-#define TRAMPOLINE_32BIT_CODE_SIZE	0xA0
-
-#ifndef __ASSEMBLER__
-
-extern unsigned long *trampoline_32bit;
-
-extern void trampoline_32bit_src(void *trampoline, bool enable_5lvl);
-
-extern const u16 trampoline_ljmp_imm_offset;
-
-#endif /* __ASSEMBLER__ */
-#endif /* BOOT_COMPRESSED_PAGETABLE_H */
diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/pgtable_64.c
+++ b/arch/x86/boot/compressed/pgtable_64.c
@@ -XXX,XX +XXX,XX @@
 #include <asm/bootparam_utils.h>
 #include <asm/e820/types.h>
 #include <asm/processor.h>
-#include "pgtable.h"
 #include "../string.h"
 #include "efi.h"
 
diff --git a/arch/x86/include/asm/boot.h b/arch/x86/include/asm/boot.h
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/include/asm/boot.h
+++ b/arch/x86/include/asm/boot.h
@@ -XXX,XX +XXX,XX @@
 # define BOOT_STACK_SIZE	0x1000
 #endif
 
+#define TRAMPOLINE_32BIT_SIZE		(2 * PAGE_SIZE)
+
+#define TRAMPOLINE_32BIT_CODE_OFFSET	PAGE_SIZE
+#define TRAMPOLINE_32BIT_CODE_SIZE	0xA0
+
 #ifndef __ASSEMBLER__
 extern unsigned int output_len;
 extern const unsigned long kernel_text_size;
@@ -XXX,XX +XXX,XX @@ unsigned long decompress_kernel(unsigned char *outbuf, unsigned long virt_addr,
 				void (*error)(char *x));
 
 extern struct boot_params *boot_params_ptr;
+extern unsigned long *trampoline_32bit;
+extern const u16 trampoline_ljmp_imm_offset;
+
+void trampoline_32bit_src(void *trampoline, bool enable_5lvl);
+
 #endif
 
 #endif /* _ASM_X86_BOOT_H */
-- 
2.49.0.472.ge94155a9ec-goog

From: Ard Biesheuvel <ardb@kernel.org>

The 5-level paging trampoline is used by both the EFI stub and the
traditional decompressor. Move it out of the decompressor sources into
the newly minted arch/x86/boot/startup/ sub-directory which will hold
startup code that may be shared between the decompressor, the EFI stub
and the kernel proper, and needs to tolerate being called during early
boot, before the kernel virtual mapping has been created.

This will allow the 5-level paging trampoline to be used by EFI boot
images such as zboot that omit the traditional decompressor entirely.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/Makefile                                  | 1 +
 arch/x86/boot/compressed/Makefile                  | 2 +-
 arch/x86/boot/startup/Makefile                     | 3 +++
 arch/x86/boot/{compressed => startup}/la57toggle.S | 0
 4 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -XXX,XX +XXX,XX @@ archprepare: $(cpufeaturemasks.hdr)
 ###
 # Kernel objects
 
+core-y  += arch/x86/boot/startup/
 libs-y  += arch/x86/lib/
 
 # drivers-y are linked after core-y
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -XXX,XX +XXX,XX @@ ifdef CONFIG_X86_64
 	vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/mem_encrypt.o
 	vmlinux-objs-y += $(obj)/pgtable_64.o
 	vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev.o
-	vmlinux-objs-y += $(obj)/la57toggle.o
 endif
 
 vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o
@@ -XXX,XX +XXX,XX @@ vmlinux-objs-$(CONFIG_UNACCEPTED_MEMORY) += $(obj)/mem.o
 
 vmlinux-objs-$(CONFIG_EFI) += $(obj)/efi.o
 vmlinux-libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a
+vmlinux-libs-$(CONFIG_X86_64)	+= $(objtree)/arch/x86/boot/startup/lib.a
 
 $(obj)/vmlinux: $(vmlinux-objs-y) $(vmlinux-libs-y) FORCE
 	$(call if_changed,ld)
diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/arch/x86/boot/startup/Makefile
@@ -XXX,XX +XXX,XX @@
+# SPDX-License-Identifier: GPL-2.0
+
+lib-$(CONFIG_X86_64)		+= la57toggle.o
diff --git a/arch/x86/boot/compressed/la57toggle.S b/arch/x86/boot/startup/la57toggle.S
similarity index 100%
rename from arch/x86/boot/compressed/la57toggle.S
rename to arch/x86/boot/startup/la57toggle.S
-- 
2.49.0.472.ge94155a9ec-goog

From: Ard Biesheuvel <ardb@kernel.org>

Linus expressed a strong preference for arch-specific asm code (i.e.,
virtually all of it) to reside under arch/ rather than anywhere else.

So move the EFI mixed mode startup code back, and put it under
arch/x86/boot/startup/ where all shared x86 startup code is going to
live.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/Makefile                                                | 3 +++
 drivers/firmware/efi/libstub/x86-mixed.S => arch/x86/boot/startup/efi-mixed.S | 0
 drivers/firmware/efi/libstub/Makefile                                         | 1 -
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/Makefile
+++ b/arch/x86/boot/startup/Makefile
@@ -XXX,XX +XXX,XX @@
 # SPDX-License-Identifier: GPL-2.0
 
+KBUILD_AFLAGS		+= -D__DISABLE_EXPORTS
+
 lib-$(CONFIG_X86_64)		+= la57toggle.o
+lib-$(CONFIG_EFI_MIXED)		+= efi-mixed.o
diff --git a/drivers/firmware/efi/libstub/x86-mixed.S b/arch/x86/boot/startup/efi-mixed.S
similarity index 100%
rename from drivers/firmware/efi/libstub/x86-mixed.S
rename to arch/x86/boot/startup/efi-mixed.S
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -XXX,XX +XXX,XX @@ lib-$(CONFIG_EFI_GENERIC_STUB)	+= efi-stub.o string.o intrinsics.o systable.o \
 lib-$(CONFIG_ARM)		+= arm32-stub.o
 lib-$(CONFIG_ARM64)		+= kaslr.o arm64.o arm64-stub.o smbios.o
 lib-$(CONFIG_X86)		+= x86-stub.o smbios.o
-lib-$(CONFIG_EFI_MIXED)		+= x86-mixed.o
 lib-$(CONFIG_X86_64)		+= x86-5lvl.o
 lib-$(CONFIG_RISCV)		+= kaslr.o riscv.o riscv-stub.o
 lib-$(CONFIG_LOONGARCH)		+= loongarch.o loongarch-stub.o
-- 
2.49.0.472.ge94155a9ec-goog

From: Ard Biesheuvel <ardb@kernel.org>

Move the early GDT/IDT setup code that runs long before the kernel
virtual mapping is up into arch/x86/boot/startup/, and build it in a way
that ensures that the code tolerates being called from the 1:1 mapping
of memory.

This allows the RIP_REL_REF() macro uses to be dropped, and removes the
need for emitting the code into the special .head.text section.

Also tweak the sed symbol matching pattern in the decompressor to match
on lower case 't' or 'b', as these will be emitted by Clang for symbols
with hidden linkage.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/compressed/Makefile |  2 +-
 arch/x86/boot/startup/Makefile    | 15 ++++
 arch/x86/boot/startup/gdt_idt.c   | 82 ++++++++++++++++++++
 arch/x86/kernel/head64.c          | 74 ------------------
 4 files changed, 98 insertions(+), 75 deletions(-)

diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -XXX,XX +XXX,XX @@ LDFLAGS_vmlinux += -T
 hostprogs	:= mkpiggy
 HOST_EXTRACFLAGS += -I$(srctree)/tools/include
 
-sed-voffset := -e 's/^$[0-9a-fA-F]*$ [ABCDGRSTVW] $_text\|__start_rodata\|__bss_start\|_end$$$/\#define VO_\2 _AC(0x\1,UL)/p'
+sed-voffset := -e 's/^$[0-9a-fA-F]*$ [ABbCDGRSTtVW] $_text\|__start_rodata\|__bss_start\|_end$$$/\#define VO_\2 _AC(0x\1,UL)/p'
 
 quiet_cmd_voffset = VOFFSET $@
       cmd_voffset = $(NM) $< | sed -n $(sed-voffset) > $@
diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/Makefile
+++ b/arch/x86/boot/startup/Makefile
@@ -XXX,XX +XXX,XX @@
 # SPDX-License-Identifier: GPL-2.0
 
 KBUILD_AFLAGS		+= -D__DISABLE_EXPORTS
+KBUILD_CFLAGS		+= -D__DISABLE_EXPORTS -mcmodel=small -fPIC \
+			   -Os -DDISABLE_BRANCH_PROFILING \
+			   $(DISABLE_STACKLEAK_PLUGIN) \
+			   -fno-stack-protector -D__NO_FORTIFY \
+			   -include $(srctree)/include/linux/hidden.h
+
+# disable ftrace hooks
+KBUILD_CFLAGS	:= $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS))
+KASAN_SANITIZE	:= n
+KCSAN_SANITIZE	:= n
+KMSAN_SANITIZE	:= n
+UBSAN_SANITIZE	:= n
+KCOV_INSTRUMENT	:= n
+
+obj-$(CONFIG_X86_64)		+= gdt_idt.o
 
 lib-$(CONFIG_X86_64)		+= la57toggle.o
 lib-$(CONFIG_EFI_MIXED)		+= efi-mixed.o
diff --git a/arch/x86/boot/startup/gdt_idt.c b/arch/x86/boot/startup/gdt_idt.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/arch/x86/boot/startup/gdt_idt.c
@@ -XXX,XX +XXX,XX @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/linkage.h>
+#include <linux/types.h>
+
+#include <asm/desc.h>
+#include <asm/setup.h>
+#include <asm/sev.h>
+#include <asm/trapnr.h>
+
+/*
+ * Data structures and code used for IDT setup in head_64.S. The bringup-IDT is
+ * used until the idt_table takes over. On the boot CPU this happens in
+ * x86_64_start_kernel(), on secondary CPUs in start_secondary(). In both cases
+ * this happens in the functions called from head_64.S.
+ *
+ * The idt_table can't be used that early because all the code modifying it is
+ * in idt.c and can be instrumented by tracing or KASAN, which both don't work
+ * during early CPU bringup. Also the idt_table has the runtime vectors
+ * configured which require certain CPU state to be setup already (like TSS),
+ * which also hasn't happened yet in early CPU bringup.
+ */
+static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
+
+/* This may run while still in the direct mapping */
+static void startup_64_load_idt(void *vc_handler)
+{
+	struct desc_ptr desc = {
+		.address = (unsigned long)bringup_idt_table,
+		.size    = sizeof(bringup_idt_table) - 1,
+	};
+	struct idt_data data;
+	gate_desc idt_desc;
+
+	/* @vc_handler is set only for a VMM Communication Exception */
+	if (vc_handler) {
+		init_idt_data(&data, X86_TRAP_VC, vc_handler);
+		idt_init_desc(&idt_desc, &data);
+		native_write_idt_entry((gate_desc *)desc.address, X86_TRAP_VC, &idt_desc);
+	}
+
+	native_load_idt(&desc);
+}
+
+/* This is used when running on kernel addresses */
+void early_setup_idt(void)
+{
+	void *handler = NULL;
+
+	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) {
+		setup_ghcb();
+		handler = vc_boot_ghcb;
+	}
+
+	startup_64_load_idt(handler);
+}
+
+/*
+ * Setup boot CPU state needed before kernel switches to virtual addresses.
+ */
+void __init startup_64_setup_gdt_idt(void)
+{
+	void *handler = NULL;
+
+	struct desc_ptr startup_gdt_descr = {
+		.address = (__force unsigned long)gdt_page.gdt,
+		.size    = GDT_SIZE - 1,
+	};
+
+	/* Load GDT */
+	native_load_gdt(&startup_gdt_descr);
+
+	/* New GDT is live - reload data segment registers */
+	asm volatile("movl %%eax, %%ds\n"
+		     "movl %%eax, %%ss\n"
+		     "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
+
+	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
+		handler = vc_no_ghcb;
+
+	startup_64_load_idt(handler);
+}
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -XXX,XX +XXX,XX @@ void __init __noreturn x86_64_start_reservations(char *real_mode_data)
 
 	start_kernel();
 }
-
-/*
- * Data structures and code used for IDT setup in head_64.S. The bringup-IDT is
- * used until the idt_table takes over. On the boot CPU this happens in
- * x86_64_start_kernel(), on secondary CPUs in start_secondary(). In both cases
- * this happens in the functions called from head_64.S.
- *
- * The idt_table can't be used that early because all the code modifying it is
- * in idt.c and can be instrumented by tracing or KASAN, which both don't work
- * during early CPU bringup. Also the idt_table has the runtime vectors
- * configured which require certain CPU state to be setup already (like TSS),
- * which also hasn't happened yet in early CPU bringup.
- */
-static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
-
-/* This may run while still in the direct mapping */
-static void __head startup_64_load_idt(void *vc_handler)
-{
-	struct desc_ptr desc = {
-		.address = (unsigned long)&RIP_REL_REF(bringup_idt_table),
-		.size    = sizeof(bringup_idt_table) - 1,
-	};
-	struct idt_data data;
-	gate_desc idt_desc;
-
-	/* @vc_handler is set only for a VMM Communication Exception */
-	if (vc_handler) {
-		init_idt_data(&data, X86_TRAP_VC, vc_handler);
-		idt_init_desc(&idt_desc, &data);
-		native_write_idt_entry((gate_desc *)desc.address, X86_TRAP_VC, &idt_desc);
-	}
-
-	native_load_idt(&desc);
-}
-
-/* This is used when running on kernel addresses */
-void early_setup_idt(void)
-{
-	void *handler = NULL;
-
-	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) {
-		setup_ghcb();
-		handler = vc_boot_ghcb;
-	}
-
-	startup_64_load_idt(handler);
-}
-
-/*
- * Setup boot CPU state needed before kernel switches to virtual addresses.
- */
-void __head startup_64_setup_gdt_idt(void)
-{
-	struct desc_struct *gdt = (void *)(__force unsigned long)gdt_page.gdt;
-	void *handler = NULL;
-
-	struct desc_ptr startup_gdt_descr = {
-		.address = (unsigned long)&RIP_REL_REF(*gdt),
-		.size    = GDT_SIZE - 1,
-	};
-
-	/* Load GDT */
-	native_load_gdt(&startup_gdt_descr);
-
-	/* New GDT is live - reload data segment registers */
-	asm volatile("movl %%eax, %%ds\n"
-		     "movl %%eax, %%ss\n"
-		     "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
-
-	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
-		handler = &RIP_REL_REF(vc_no_ghcb);
-
-	startup_64_load_idt(handler);
-}
-- 
2.49.0.472.ge94155a9ec-goog

From: Ard Biesheuvel <ardb@kernel.org>

The startup code that constructs the kernel virtual mapping runs from
the 1:1 mapping of memory itself, and therefore, cannot use absolute
symbol references. Move this code into a separate source file under
arch/x86/boot/startup/ where all such code will be kept from now on.

Since all code here is constructed in a manner that ensures that it
tolerates running from the 1:1 mapping of memory, any uses of the
RIP_REL_REF() macro can be dropped, along with __head annotations for
placing this code in a dedicated startup section.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/Makefile     |   2 +-
 arch/x86/boot/startup/map_kernel.c | 232 ++++++++++++++++++++
 arch/x86/kernel/head64.c           | 228 +------------------
 3 files changed, 234 insertions(+), 228 deletions(-)

diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/Makefile
+++ b/arch/x86/boot/startup/Makefile
@@ -XXX,XX +XXX,XX @@ KMSAN_SANITIZE	:= n
 UBSAN_SANITIZE	:= n
 KCOV_INSTRUMENT	:= n
 
-obj-$(CONFIG_X86_64)		+= gdt_idt.o
+obj-$(CONFIG_X86_64)		+= gdt_idt.o map_kernel.o
 
 lib-$(CONFIG_X86_64)		+= la57toggle.o
 lib-$(CONFIG_EFI_MIXED)		+= efi-mixed.o
diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/arch/x86/boot/startup/map_kernel.c
@@ -XXX,XX +XXX,XX @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/init.h>
+#include <linux/linkage.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/pgtable.h>
+
+#include <asm/sections.h>
+#include <asm/setup.h>
+#include <asm/sev.h>
+
+extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
+extern unsigned int next_early_pgt;
+
+#ifdef CONFIG_X86_5LEVEL
+unsigned int __pgtable_l5_enabled __ro_after_init;
+unsigned int pgdir_shift __ro_after_init = 39;
+EXPORT_SYMBOL(pgdir_shift);
+unsigned int ptrs_per_p4d __ro_after_init = 1;
+EXPORT_SYMBOL(ptrs_per_p4d);
+#endif
+
+#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT
+unsigned long page_offset_base __ro_after_init = __PAGE_OFFSET_BASE_L4;
+EXPORT_SYMBOL(page_offset_base);
+unsigned long vmalloc_base __ro_after_init = __VMALLOC_BASE_L4;
+EXPORT_SYMBOL(vmalloc_base);
+unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4;
+EXPORT_SYMBOL(vmemmap_base);
+#endif
+
+static inline bool check_la57_support(void)
+{
+	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
+		return false;
+
+	/*
+	 * 5-level paging is detected and enabled at kernel decompression
+	 * stage. Only check if it has been enabled there.
+	 */
+	if (!(native_read_cr4() & X86_CR4_LA57))
+		return false;
+
+	__pgtable_l5_enabled	= 1;
+	pgdir_shift		= 48;
+	ptrs_per_p4d		= 512;
+	page_offset_base	= __PAGE_OFFSET_BASE_L5;
+	vmalloc_base		= __VMALLOC_BASE_L5;
+	vmemmap_base		= __VMEMMAP_BASE_L5;
+
+	return true;
+}
+
+static unsigned long sme_postprocess_startup(struct boot_params *bp,
+					     pmdval_t *pmd,
+					     unsigned long p2v_offset)
+{
+	unsigned long paddr, paddr_end;
+	int i;
+
+	/* Encrypt the kernel and related (if SME is active) */
+	sme_encrypt_kernel(bp);
+
+	/*
+	 * Clear the memory encryption mask from the .bss..decrypted section.
+	 * The bss section will be memset to zero later in the initialization so
+	 * there is no need to zero it after changing the memory encryption
+	 * attribute.
+	 */
+	if (sme_get_me_mask()) {
+		paddr = (unsigned long)__start_bss_decrypted;
+		paddr_end = (unsigned long)__end_bss_decrypted;
+
+		for (; paddr < paddr_end; paddr += PMD_SIZE) {
+			/*
+			 * On SNP, transition the page to shared in the RMP table so that
+			 * it is consistent with the page table attribute change.
+			 *
+			 * __start_bss_decrypted has a virtual address in the high range
+			 * mapping (kernel .text). PVALIDATE, by way of
+			 * early_snp_set_memory_shared(), requires a valid virtual
+			 * address but the kernel is currently running off of the identity
+			 * mapping so use the PA to get a *currently* valid virtual address.
+			 */
+			early_snp_set_memory_shared(paddr, paddr, PTRS_PER_PMD);
+
+			i = pmd_index(paddr - p2v_offset);
+			pmd[i] -= sme_get_me_mask();
+		}
+	}
+
+	/*
+	 * Return the SME encryption mask (if SME is active) to be used as a
+	 * modifier for the initial pgdir entry programmed into CR3.
+	 */
+	return sme_get_me_mask();
+}
+
+unsigned long __init __startup_64(unsigned long p2v_offset,
+				  struct boot_params *bp)
+{
+	pmd_t (*early_pgts)[PTRS_PER_PMD] = early_dynamic_pgts;
+	unsigned long physaddr = (unsigned long)_text;
+	unsigned long va_text, va_end;
+	unsigned long pgtable_flags;
+	unsigned long load_delta;
+	pgdval_t *pgd;
+	p4dval_t *p4d;
+	pudval_t *pud;
+	pmdval_t *pmd, pmd_entry;
+	bool la57;
+	int i;
+
+	la57 = check_la57_support();
+
+	/* Is the address too large? */
+	if (physaddr >> MAX_PHYSMEM_BITS)
+		for (;;);
+
+	/*
+	 * Compute the delta between the address I am compiled to run at
+	 * and the address I am actually running at.
+	 */
+	phys_base = load_delta = __START_KERNEL_map + p2v_offset;
+
+	/* Is the address not 2M aligned? */
+	if (load_delta & ~PMD_MASK)
+		for (;;);
+
+	va_text = physaddr - p2v_offset;
+	va_end  = (unsigned long)_end - p2v_offset;
+
+	/* Include the SME encryption mask in the fixup value */
+	load_delta += sme_get_me_mask();
+
+	/* Fixup the physical addresses in the page table */
+
+	pgd = &early_top_pgt[0].pgd;
+	pgd[pgd_index(__START_KERNEL_map)] += load_delta;
+
+	if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
+		p4d = (p4dval_t *)level4_kernel_pgt;
+		p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
+
+		pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
+	}
+
+	level3_kernel_pgt[PTRS_PER_PUD - 2].pud += load_delta;
+	level3_kernel_pgt[PTRS_PER_PUD - 1].pud += load_delta;
+
+	for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
+		level2_fixmap_pgt[i].pmd += load_delta;
+
+	/*
+	 * Set up the identity mapping for the switchover.  These
+	 * entries should *NOT* have the global bit set!  This also
+	 * creates a bunch of nonsense entries but that is fine --
+	 * it avoids problems around wraparound.
+	 */
+
+	pud = &early_pgts[0]->pmd;
+	pmd = &early_pgts[1]->pmd;
+	next_early_pgt = 2;
+
+	pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
+
+	if (la57) {
+		p4d = &early_pgts[next_early_pgt++]->pmd;
+
+		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
+		pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
+		pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;
+
+		i = physaddr >> P4D_SHIFT;
+		p4d[(i + 0) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
+		p4d[(i + 1) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
+	} else {
+		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
+		pgd[i + 0] = (pgdval_t)pud + pgtable_flags;
+		pgd[i + 1] = (pgdval_t)pud + pgtable_flags;
+	}
+
+	i = physaddr >> PUD_SHIFT;
+	pud[(i + 0) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
+	pud[(i + 1) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
+
+	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
+	/* Filter out unsupported __PAGE_KERNEL_* bits: */
+	pmd_entry &= __supported_pte_mask;
+	pmd_entry += sme_get_me_mask();
+	pmd_entry +=  physaddr;
+
+	for (i = 0; i < DIV_ROUND_UP(va_end - va_text, PMD_SIZE); i++) {
+		int idx = i + (physaddr >> PMD_SHIFT);
+
+		pmd[idx % PTRS_PER_PMD] = pmd_entry + i * PMD_SIZE;
+	}
+
+	/*
+	 * Fixup the kernel text+data virtual addresses. Note that
+	 * we might write invalid pmds, when the kernel is relocated
+	 * cleanup_highmap() fixes this up along with the mappings
+	 * beyond _end.
+	 *
+	 * Only the region occupied by the kernel image has so far
+	 * been checked against the table of usable memory regions
+	 * provided by the firmware, so invalidate pages outside that
+	 * region. A page table entry that maps to a reserved area of
+	 * memory would allow processor speculation into that area,
+	 * and on some hardware (particularly the UV platform) even
+	 * speculative access to some reserved areas is caught as an
+	 * error, causing the BIOS to halt the system.
+	 */
+
+	pmd = &level2_kernel_pgt[0].pmd;
+
+	/* invalidate pages before the kernel image */
+	for (i = 0; i < pmd_index(va_text); i++)
+		pmd[i] &= ~_PAGE_PRESENT;
+
+	/* fixup pages that are part of the kernel image */
+	for (; i <= pmd_index(va_end); i++)
+		if (pmd[i] & _PAGE_PRESENT)
+			pmd[i] += load_delta;
+
+	/* invalidate pages after the kernel image */
+	for (; i < PTRS_PER_PMD; i++)
+		pmd[i] &= ~_PAGE_PRESENT;
+
+	return sme_postprocess_startup(bp, pmd, p2v_offset);
+}
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -XXX,XX +XXX,XX @@
  * Manage page tables very early on.
  */
 extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
-static unsigned int __initdata next_early_pgt;
+unsigned int __initdata next_early_pgt;
 pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
 
-#ifdef CONFIG_X86_5LEVEL
-unsigned int __pgtable_l5_enabled __ro_after_init;
-unsigned int pgdir_shift __ro_after_init = 39;
-EXPORT_SYMBOL(pgdir_shift);
-unsigned int ptrs_per_p4d __ro_after_init = 1;
-EXPORT_SYMBOL(ptrs_per_p4d);
-#endif
-
-#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT
-unsigned long page_offset_base __ro_after_init = __PAGE_OFFSET_BASE_L4;
-EXPORT_SYMBOL(page_offset_base);
-unsigned long vmalloc_base __ro_after_init = __VMALLOC_BASE_L4;
-EXPORT_SYMBOL(vmalloc_base);
-unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4;
-EXPORT_SYMBOL(vmemmap_base);
-#endif
-
-static inline bool check_la57_support(void)
-{
-	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
-		return false;
-
-	/*
-	 * 5-level paging is detected and enabled at kernel decompression
-	 * stage. Only check if it has been enabled there.
-	 */
-	if (!(native_read_cr4() & X86_CR4_LA57))
-		return false;
-
-	RIP_REL_REF(__pgtable_l5_enabled)	= 1;
-	RIP_REL_REF(pgdir_shift)		= 48;
-	RIP_REL_REF(ptrs_per_p4d)		= 512;
-	RIP_REL_REF(page_offset_base)		= __PAGE_OFFSET_BASE_L5;
-	RIP_REL_REF(vmalloc_base)		= __VMALLOC_BASE_L5;
-	RIP_REL_REF(vmemmap_base)		= __VMEMMAP_BASE_L5;
-
-	return true;
-}
-
-static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
-						    pmdval_t *pmd,
-						    unsigned long p2v_offset)
-{
-	unsigned long paddr, paddr_end;
-	int i;
-
-	/* Encrypt the kernel and related (if SME is active) */
-	sme_encrypt_kernel(bp);
-
-	/*
-	 * Clear the memory encryption mask from the .bss..decrypted section.
-	 * The bss section will be memset to zero later in the initialization so
-	 * there is no need to zero it after changing the memory encryption
-	 * attribute.
-	 */
-	if (sme_get_me_mask()) {
-		paddr = (unsigned long)&RIP_REL_REF(__start_bss_decrypted);
-		paddr_end = (unsigned long)&RIP_REL_REF(__end_bss_decrypted);
-
-		for (; paddr < paddr_end; paddr += PMD_SIZE) {
-			/*
-			 * On SNP, transition the page to shared in the RMP table so that
-			 * it is consistent with the page table attribute change.
-			 *
-			 * __start_bss_decrypted has a virtual address in the high range
-			 * mapping (kernel .text). PVALIDATE, by way of
-			 * early_snp_set_memory_shared(), requires a valid virtual
-			 * address but the kernel is currently running off of the identity
-			 * mapping so use the PA to get a *currently* valid virtual address.
-			 */
-			early_snp_set_memory_shared(paddr, paddr, PTRS_PER_PMD);
-
-			i = pmd_index(paddr - p2v_offset);
-			pmd[i] -= sme_get_me_mask();
-		}
-	}
-
-	/*
-	 * Return the SME encryption mask (if SME is active) to be used as a
-	 * modifier for the initial pgdir entry programmed into CR3.
-	 */
-	return sme_get_me_mask();
-}
-
-/* Code in __startup_64() can be relocated during execution, but the compiler
- * doesn't have to generate PC-relative relocations when accessing globals from
- * that function. Clang actually does not generate them, which leads to
- * boot-time crashes. To work around this problem, every global pointer must
- * be accessed using RIP_REL_REF(). Kernel virtual addresses can be determined
- * by subtracting p2v_offset from the RIP-relative address.
- */
-unsigned long __head __startup_64(unsigned long p2v_offset,
-				  struct boot_params *bp)
-{
-	pmd_t (*early_pgts)[PTRS_PER_PMD] = RIP_REL_REF(early_dynamic_pgts);
-	unsigned long physaddr = (unsigned long)&RIP_REL_REF(_text);
-	unsigned long va_text, va_end;
-	unsigned long pgtable_flags;
-	unsigned long load_delta;
-	pgdval_t *pgd;
-	p4dval_t *p4d;
-	pudval_t *pud;
-	pmdval_t *pmd, pmd_entry;
-	bool la57;
-	int i;
-
-	la57 = check_la57_support();
-
-	/* Is the address too large? */
-	if (physaddr >> MAX_PHYSMEM_BITS)
-		for (;;);
-
-	/*
-	 * Compute the delta between the address I am compiled to run at
-	 * and the address I am actually running at.
-	 */
-	load_delta = __START_KERNEL_map + p2v_offset;
-	RIP_REL_REF(phys_base) = load_delta;
-
-	/* Is the address not 2M aligned? */
-	if (load_delta & ~PMD_MASK)
-		for (;;);
-
-	va_text = physaddr - p2v_offset;
-	va_end  = (unsigned long)&RIP_REL_REF(_end) - p2v_offset;
-
-	/* Include the SME encryption mask in the fixup value */
-	load_delta += sme_get_me_mask();
-
-	/* Fixup the physical addresses in the page table */
-
-	pgd = &RIP_REL_REF(early_top_pgt)->pgd;
-	pgd[pgd_index(__START_KERNEL_map)] += load_delta;
-
-	if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
-		p4d = (p4dval_t *)&RIP_REL_REF(level4_kernel_pgt);
-		p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
-
-		pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
-	}
-
-	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 2].pud += load_delta;
-	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 1].pud += load_delta;
-
-	for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
-		RIP_REL_REF(level2_fixmap_pgt)[i].pmd += load_delta;
-
-	/*
-	 * Set up the identity mapping for the switchover.  These
-	 * entries should *NOT* have the global bit set!  This also
-	 * creates a bunch of nonsense entries but that is fine --
-	 * it avoids problems around wraparound.
-	 */
-
-	pud = &early_pgts[0]->pmd;
-	pmd = &early_pgts[1]->pmd;
-	RIP_REL_REF(next_early_pgt) = 2;
-
-	pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
-
-	if (la57) {
-		p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd;
-
-		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
-		pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
-		pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;
-
-		i = physaddr >> P4D_SHIFT;
-		p4d[(i + 0) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
-		p4d[(i + 1) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
-	} else {
-		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
-		pgd[i + 0] = (pgdval_t)pud + pgtable_flags;
-		pgd[i + 1] = (pgdval_t)pud + pgtable_flags;
-	}
-
-	i = physaddr >> PUD_SHIFT;
-	pud[(i + 0) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
-	pud[(i + 1) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
-
-	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
-	/* Filter out unsupported __PAGE_KERNEL_* bits: */
-	pmd_entry &= RIP_REL_REF(__supported_pte_mask);
-	pmd_entry += sme_get_me_mask();
-	pmd_entry +=  physaddr;
-
-	for (i = 0; i < DIV_ROUND_UP(va_end - va_text, PMD_SIZE); i++) {
-		int idx = i + (physaddr >> PMD_SHIFT);
-
-		pmd[idx % PTRS_PER_PMD] = pmd_entry + i * PMD_SIZE;
-	}
-
-	/*
-	 * Fixup the kernel text+data virtual addresses. Note that
-	 * we might write invalid pmds, when the kernel is relocated
-	 * cleanup_highmap() fixes this up along with the mappings
-	 * beyond _end.
-	 *
-	 * Only the region occupied by the kernel image has so far
-	 * been checked against the table of usable memory regions
-	 * provided by the firmware, so invalidate pages outside that
-	 * region. A page table entry that maps to a reserved area of
-	 * memory would allow processor speculation into that area,
-	 * and on some hardware (particularly the UV platform) even
-	 * speculative access to some reserved areas is caught as an
-	 * error, causing the BIOS to halt the system.
-	 */
-
-	pmd = &RIP_REL_REF(level2_kernel_pgt)->pmd;
-
-	/* invalidate pages before the kernel image */
-	for (i = 0; i < pmd_index(va_text); i++)
-		pmd[i] &= ~_PAGE_PRESENT;
-
-	/* fixup pages that are part of the kernel image */
-	for (; i <= pmd_index(va_end); i++)
-		if (pmd[i] & _PAGE_PRESENT)
-			pmd[i] += load_delta;
-
-	/* invalidate pages after the kernel image */
-	for (; i < PTRS_PER_PMD; i++)
-		pmd[i] &= ~_PAGE_PRESENT;
-
-	return sme_postprocess_startup(bp, pmd, p2v_offset);
-}
-
 /* Wipe all early page tables except for the kernel symbol map */
 static void __init reset_early_page_tables(void)
 {
-- 
2.49.0.472.ge94155a9ec-goog

From: Ard Biesheuvel <ardb@kernel.org>

Move the SME initialization code, which runs from the 1:1 mapping of
memory as it operates on the kernel virtual mapping, into the new
sub-directory arch/x86/boot/startup/ where all startup code will reside
that needs to tolerate executing from the 1:1 mapping.

This allows RIP_REL_REF() macro invocations and __head annotations to be
dropped.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/Makefile                             |  1 +
 arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} | 45 +++++++++-----------
 arch/x86/include/asm/mem_encrypt.h                         |  2 +-
 arch/x86/mm/Makefile                                       |  6 ---
 4 files changed, 23 insertions(+), 31 deletions(-)

From: Ard Biesheuvel <ardb@kernel.org>

Reorganize C code that is used during early boot, either in the
decompressor/EFI stub or the kernel proper, but before the kernel
virtual mapping is up.

v3:
- keep rip_rel_ptr() around in PIC code - sadly, it is still needed in
  some cases
- remove RIP_REL_REF() uses in separate patches
- keep __head annotations for now, they will all be removed later
- disable objtool validation for library objects (i.e., pieces that are
  not linked into vmlinux)

I will follow up with a series that gets rid of .head.text altogether,
as it will no longer be needed at all once the startup code is checked
for absolute relocations.

The SEV startup code needs to be moved first, though, and this is a bit
more complicated, so I will decouple that effort from this series, also
because there is a known issue that needs to be fixed first related to
memory acceptance from the EFI stub.

Cc: Tom Lendacky <thomas.lendacky@amd.com>
Cc: Dionna Amalie Glaze <dionnaglaze@google.com>
Cc: Kevin Loughlin <kevinloughlin@google.com>

Ard Biesheuvel (7):
  x86/boot/startup: Disable objtool validation for library code
  x86/asm: Make rip_rel_ptr() usable from fPIC code
  x86/boot: Move the early GDT/IDT setup code into startup/
  x86/boot: Move early kernel mapping code into startup/
  x86/boot: Drop RIP_REL_REF() uses from early mapping code
  x86/boot: Move early SME init code into startup/
  x86/boot: Drop RIP_REL_REF() uses from SME startup code

arch/x86/boot/compressed/Makefile                          |   2 +-
 arch/x86/boot/startup/Makefile                             |  22 ++
 arch/x86/boot/startup/gdt_idt.c                            |  83 ++++++
 arch/x86/boot/startup/map_kernel.c                         | 225 ++++++++++++++++
 arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} |  19 +-
 arch/x86/coco/sev/core.c                                   |   2 +-
 arch/x86/coco/sev/shared.c                                 |   4 +-
 arch/x86/include/asm/asm.h                                 |   2 +-
 arch/x86/include/asm/coco.h                                |   2 +-
 arch/x86/include/asm/mem_encrypt.h                         |   2 +-
 arch/x86/kernel/head64.c                                   | 285 +-------------------
 arch/x86/mm/Makefile                                       |   6 -
 12 files changed, 346 insertions(+), 308 deletions(-)
 create mode 100644 arch/x86/boot/startup/gdt_idt.c
 create mode 100644 arch/x86/boot/startup/map_kernel.c
 rename arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} (97%)

base-commit: 4f2d1bbc2c92a32fd612e6c3b51832d5c1c3678e
-- 
2.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

The library code built under arch/x86/boot/startup is not intended to be
linked into vmlinux but only into the decompressor and/or the EFI stub.

This means objtool validation is not needed here, and may result in
false positive errors for things like missing retpolines.

So disable it for all objects added to lib-y

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/Makefile | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/Makefile
+++ b/arch/x86/boot/startup/Makefile
@@ -XXX,XX +XXX,XX @@ KBUILD_AFLAGS		+= -D__DISABLE_EXPORTS
 
 lib-$(CONFIG_X86_64)		+= la57toggle.o
 lib-$(CONFIG_EFI_MIXED)		+= efi-mixed.o
+
+#
+# Disable objtool validation for all library code, which is intended
+# to be linked into the decompressor or the EFI stub but not vmlinux
+#
+$(patsubst %.o,$(obj)/%.o,$(lib-y)): OBJECT_FILES_NON_STANDARD := y
-- 
2.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

RIP_REL_REF() is used in non-PIC C code that is called very early,
before the kernel virtual mapping is up, which is the mapping that the
linker expects. It is currently used in two different ways:
- to refer to the value of a global variable, including as an lvalue in
  assignments;
- to take the address of a global variable via the mapping that the code
  currently executes at.

The former case is only needed in non-PIC code, as PIC code will never
use absolute symbol references when the address of the symbol is not
being used. But taking the address of a variable in PIC code may still
require extra care, as a stack allocated struct assignment may be
emitted as a memcpy() from a statically allocated copy in .rodata.

For instance, this

void startup_64_setup_gdt_idt(void)
  {
        struct desc_ptr startup_gdt_descr = {
                .address = (__force unsigned long)gdt_page.gdt,
                .size    = GDT_SIZE - 1,
        };

may result in an absolute symbol reference in PIC code, even though the
struct is allocated on the stack and populated at runtime.

To address this case, make rip_rel_ptr() accessible in PIC code, and
update any existing uses where the address of a global variable is
taken using RIP_REL_REF.

Once all code of this nature has been moved into arch/x86/boot/startup
and built with -fPIC, RIP_REL_REF() can be retired, and only
rip_rel_ptr() will remain.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/coco/sev/core.c           |  2 +-
 arch/x86/coco/sev/shared.c         |  4 ++--
 arch/x86/include/asm/asm.h         |  2 +-
 arch/x86/kernel/head64.c           | 23 ++++++++++----------
 arch/x86/mm/mem_encrypt_identity.c |  6 ++---
 5 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/coco/sev/core.c
+++ b/arch/x86/coco/sev/core.c
@@ -XXX,XX +XXX,XX @@ static __head void svsm_setup(struct cc_blob_sev_info *cc_info)
 	 * kernel was loaded (physbase), so the get the CA address using
 	 * RIP-relative addressing.
 	 */
-	pa = (u64)&RIP_REL_REF(boot_svsm_ca_page);
+	pa = (u64)rip_rel_ptr(&boot_svsm_ca_page);
 
 	/*
 	 * Switch over to the boot SVSM CA while the current CA is still
diff --git a/arch/x86/coco/sev/shared.c b/arch/x86/coco/sev/shared.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/coco/sev/shared.c
+++ b/arch/x86/coco/sev/shared.c
@@ -XXX,XX +XXX,XX @@ static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid
  */
 static const struct snp_cpuid_table *snp_cpuid_get_table(void)
 {
-	return &RIP_REL_REF(cpuid_table_copy);
+	return rip_rel_ptr(&cpuid_table_copy);
 }
 
 /*
@@ -XXX,XX +XXX,XX @@ static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info)
 	 * routine is running identity mapped when called, both by the decompressor
 	 * code and the early kernel code.
 	 */
-	if (!rmpadjust((unsigned long)&RIP_REL_REF(boot_ghcb_page), RMP_PG_SIZE_4K, 1))
+	if (!rmpadjust((unsigned long)rip_rel_ptr(&boot_ghcb_page), RMP_PG_SIZE_4K, 1))
 		return false;
 
 	/*
diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/include/asm/asm.h
+++ b/arch/x86/include/asm/asm.h
@@ -XXX,XX +XXX,XX @@
 #endif
 
 #ifndef __ASSEMBLER__
-#ifndef __pic__
 static __always_inline __pure void *rip_rel_ptr(void *p)
 {
 	asm("leaq %c1(%%rip), %0" : "=r"(p) : "i"(p));
 
 	return p;
 }
+#ifndef __pic__
 #define RIP_REL_REF(var)	(*(typeof(&(var)))rip_rel_ptr(&(var)))
 #else
 #define RIP_REL_REF(var)	(var)
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -XXX,XX +XXX,XX @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
 	 * attribute.
 	 */
 	if (sme_get_me_mask()) {
-		paddr = (unsigned long)&RIP_REL_REF(__start_bss_decrypted);
-		paddr_end = (unsigned long)&RIP_REL_REF(__end_bss_decrypted);
+		paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted);
+		paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted);
 
 		for (; paddr < paddr_end; paddr += PMD_SIZE) {
 			/*
@@ -XXX,XX +XXX,XX @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
 unsigned long __head __startup_64(unsigned long p2v_offset,
 				  struct boot_params *bp)
 {
-	pmd_t (*early_pgts)[PTRS_PER_PMD] = RIP_REL_REF(early_dynamic_pgts);
-	unsigned long physaddr = (unsigned long)&RIP_REL_REF(_text);
+	pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);
+	unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);
 	unsigned long va_text, va_end;
 	unsigned long pgtable_flags;
 	unsigned long load_delta;
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
 		for (;;);
 
 	va_text = physaddr - p2v_offset;
-	va_end  = (unsigned long)&RIP_REL_REF(_end) - p2v_offset;
+	va_end  = (unsigned long)rip_rel_ptr(_end) - p2v_offset;
 
 	/* Include the SME encryption mask in the fixup value */
 	load_delta += sme_get_me_mask();
 
 	/* Fixup the physical addresses in the page table */
 
-	pgd = &RIP_REL_REF(early_top_pgt)->pgd;
+	pgd = rip_rel_ptr(early_top_pgt);
 	pgd[pgd_index(__START_KERNEL_map)] += load_delta;
 
 	if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
-		p4d = (p4dval_t *)&RIP_REL_REF(level4_kernel_pgt);
+		p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt);
 		p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
 
 		pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
 	 * error, causing the BIOS to halt the system.
 	 */
 
-	pmd = &RIP_REL_REF(level2_kernel_pgt)->pmd;
+	pmd = rip_rel_ptr(level2_kernel_pgt);
 
 	/* invalidate pages before the kernel image */
 	for (i = 0; i < pmd_index(va_text); i++)
@@ -XXX,XX +XXX,XX @@ static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
 static void __head startup_64_load_idt(void *vc_handler)
 {
 	struct desc_ptr desc = {
-		.address = (unsigned long)&RIP_REL_REF(bringup_idt_table),
+		.address = (unsigned long)rip_rel_ptr(bringup_idt_table),
 		.size    = sizeof(bringup_idt_table) - 1,
 	};
 	struct idt_data data;
@@ -XXX,XX +XXX,XX @@ void early_setup_idt(void)
  */
 void __head startup_64_setup_gdt_idt(void)
 {
-	struct desc_struct *gdt = (void *)(__force unsigned long)gdt_page.gdt;
 	void *handler = NULL;
 
 	struct desc_ptr startup_gdt_descr = {
-		.address = (unsigned long)&RIP_REL_REF(*gdt),
+		.address = (unsigned long)rip_rel_ptr((__force void *)&gdt_page),
 		.size    = GDT_SIZE - 1,
 	};
 
@@ -XXX,XX +XXX,XX @@ void __head startup_64_setup_gdt_idt(void)
 		     "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
 
 	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
-		handler = &RIP_REL_REF(vc_no_ghcb);
+		handler = rip_rel_ptr(vc_no_ghcb);
 
 	startup_64_load_idt(handler);
 }
diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/mm/mem_encrypt_identity.c
+++ b/arch/x86/mm/mem_encrypt_identity.c
@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
 	 *     memory from being cached.
 	 */
 
-	kernel_start = (unsigned long)RIP_REL_REF(_text);
-	kernel_end = ALIGN((unsigned long)RIP_REL_REF(_end), PMD_SIZE);
+	kernel_start = (unsigned long)rip_rel_ptr(_text);
+	kernel_end = ALIGN((unsigned long)rip_rel_ptr(_end), PMD_SIZE);
 	kernel_len = kernel_end - kernel_start;
 
 	initrd_start = 0;
@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
 	 *   pagetable structures for the encryption of the kernel
 	 *   pagetable structures for workarea (in case not currently mapped)
 	 */
-	execute_start = workarea_start = (unsigned long)RIP_REL_REF(sme_workarea);
+	execute_start = workarea_start = (unsigned long)rip_rel_ptr(sme_workarea);
 	execute_end = execute_start + (PAGE_SIZE * 2) + PMD_SIZE;
 	execute_len = execute_end - execute_start;
 
-- 
2.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

Also tweak the sed symbol matching pattern in the decompressor to match
on lower case 't' or 'b', as these will be emitted by Clang for symbols
with hidden linkage.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/compressed/Makefile |  2 +-
 arch/x86/boot/startup/Makefile    | 15 ++++
 arch/x86/boot/startup/gdt_idt.c   | 83 ++++++++++++++++++++
 arch/x86/kernel/head64.c          | 73 -----------------
 4 files changed, 99 insertions(+), 74 deletions(-)

diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -XXX,XX +XXX,XX @@ LDFLAGS_vmlinux += -T
 hostprogs	:= mkpiggy
 HOST_EXTRACFLAGS += -I$(srctree)/tools/include
 
-sed-voffset := -e 's/^$[0-9a-fA-F]*$ [ABCDGRSTVW] $_text\|__start_rodata\|__bss_start\|_end$$$/\#define VO_\2 _AC(0x\1,UL)/p'
+sed-voffset := -e 's/^$[0-9a-fA-F]*$ [ABbCDGRSTtVW] $_text\|__start_rodata\|__bss_start\|_end$$$/\#define VO_\2 _AC(0x\1,UL)/p'
 
 quiet_cmd_voffset = VOFFSET $@
       cmd_voffset = $(NM) $< | sed -n $(sed-voffset) > $@
diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/Makefile
+++ b/arch/x86/boot/startup/Makefile
@@ -XXX,XX +XXX,XX @@
 # SPDX-License-Identifier: GPL-2.0
 
 KBUILD_AFLAGS		+= -D__DISABLE_EXPORTS
+KBUILD_CFLAGS		+= -D__DISABLE_EXPORTS -mcmodel=small -fPIC \
+			   -Os -DDISABLE_BRANCH_PROFILING \
+			   $(DISABLE_STACKLEAK_PLUGIN) \
+			   -fno-stack-protector -D__NO_FORTIFY \
+			   -include $(srctree)/include/linux/hidden.h
+
+# disable ftrace hooks
+KBUILD_CFLAGS	:= $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS))
+KASAN_SANITIZE	:= n
+KCSAN_SANITIZE	:= n
+KMSAN_SANITIZE	:= n
+UBSAN_SANITIZE	:= n
+KCOV_INSTRUMENT	:= n
+
+obj-$(CONFIG_X86_64)		+= gdt_idt.o
 
 lib-$(CONFIG_X86_64)		+= la57toggle.o
 lib-$(CONFIG_EFI_MIXED)		+= efi-mixed.o
diff --git a/arch/x86/boot/startup/gdt_idt.c b/arch/x86/boot/startup/gdt_idt.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/arch/x86/boot/startup/gdt_idt.c
@@ -XXX,XX +XXX,XX @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/linkage.h>
+#include <linux/types.h>
+
+#include <asm/desc.h>
+#include <asm/init.h>
+#include <asm/setup.h>
+#include <asm/sev.h>
+#include <asm/trapnr.h>
+
+/*
+ * Data structures and code used for IDT setup in head_64.S. The bringup-IDT is
+ * used until the idt_table takes over. On the boot CPU this happens in
+ * x86_64_start_kernel(), on secondary CPUs in start_secondary(). In both cases
+ * this happens in the functions called from head_64.S.
+ *
+ * The idt_table can't be used that early because all the code modifying it is
+ * in idt.c and can be instrumented by tracing or KASAN, which both don't work
+ * during early CPU bringup. Also the idt_table has the runtime vectors
+ * configured which require certain CPU state to be setup already (like TSS),
+ * which also hasn't happened yet in early CPU bringup.
+ */
+static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
+
+/* This may run while still in the direct mapping */
+static void __head startup_64_load_idt(void *vc_handler)
+{
+	struct desc_ptr desc = {
+		.address = (unsigned long)rip_rel_ptr(bringup_idt_table),
+		.size    = sizeof(bringup_idt_table) - 1,
+	};
+	struct idt_data data;
+	gate_desc idt_desc;
+
+	/* @vc_handler is set only for a VMM Communication Exception */
+	if (vc_handler) {
+		init_idt_data(&data, X86_TRAP_VC, vc_handler);
+		idt_init_desc(&idt_desc, &data);
+		native_write_idt_entry((gate_desc *)desc.address, X86_TRAP_VC, &idt_desc);
+	}
+
+	native_load_idt(&desc);
+}
+
+/* This is used when running on kernel addresses */
+void early_setup_idt(void)
+{
+	void *handler = NULL;
+
+	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) {
+		setup_ghcb();
+		handler = vc_boot_ghcb;
+	}
+
+	startup_64_load_idt(handler);
+}
+
+/*
+ * Setup boot CPU state needed before kernel switches to virtual addresses.
+ */
+void __head startup_64_setup_gdt_idt(void)
+{
+	void *handler = NULL;
+
+	struct desc_ptr startup_gdt_descr = {
+		.address = (unsigned long)rip_rel_ptr((__force void *)&gdt_page),
+		.size    = GDT_SIZE - 1,
+	};
+
+	/* Load GDT */
+	native_load_gdt(&startup_gdt_descr);
+
+	/* New GDT is live - reload data segment registers */
+	asm volatile("movl %%eax, %%ds\n"
+		     "movl %%eax, %%ss\n"
+		     "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
+
+	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
+		handler = rip_rel_ptr(vc_no_ghcb);
+
+	startup_64_load_idt(handler);
+}
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -XXX,XX +XXX,XX @@ void __init __noreturn x86_64_start_reservations(char *real_mode_data)
 
 	start_kernel();
 }
-
-/*
- * Data structures and code used for IDT setup in head_64.S. The bringup-IDT is
- * used until the idt_table takes over. On the boot CPU this happens in
- * x86_64_start_kernel(), on secondary CPUs in start_secondary(). In both cases
- * this happens in the functions called from head_64.S.
- *
- * The idt_table can't be used that early because all the code modifying it is
- * in idt.c and can be instrumented by tracing or KASAN, which both don't work
- * during early CPU bringup. Also the idt_table has the runtime vectors
- * configured which require certain CPU state to be setup already (like TSS),
- * which also hasn't happened yet in early CPU bringup.
- */
-static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
-
-/* This may run while still in the direct mapping */
-static void __head startup_64_load_idt(void *vc_handler)
-{
-	struct desc_ptr desc = {
-		.address = (unsigned long)rip_rel_ptr(bringup_idt_table),
-		.size    = sizeof(bringup_idt_table) - 1,
-	};
-	struct idt_data data;
-	gate_desc idt_desc;
-
-	/* @vc_handler is set only for a VMM Communication Exception */
-	if (vc_handler) {
-		init_idt_data(&data, X86_TRAP_VC, vc_handler);
-		idt_init_desc(&idt_desc, &data);
-		native_write_idt_entry((gate_desc *)desc.address, X86_TRAP_VC, &idt_desc);
-	}
-
-	native_load_idt(&desc);
-}
-
-/* This is used when running on kernel addresses */
-void early_setup_idt(void)
-{
-	void *handler = NULL;
-
-	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) {
-		setup_ghcb();
-		handler = vc_boot_ghcb;
-	}
-
-	startup_64_load_idt(handler);
-}
-
-/*
- * Setup boot CPU state needed before kernel switches to virtual addresses.
- */
-void __head startup_64_setup_gdt_idt(void)
-{
-	void *handler = NULL;
-
-	struct desc_ptr startup_gdt_descr = {
-		.address = (unsigned long)rip_rel_ptr((__force void *)&gdt_page),
-		.size    = GDT_SIZE - 1,
-	};
-
-	/* Load GDT */
-	native_load_gdt(&startup_gdt_descr);
-
-	/* New GDT is live - reload data segment registers */
-	asm volatile("movl %%eax, %%ds\n"
-		     "movl %%eax, %%ss\n"
-		     "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
-
-	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
-		handler = rip_rel_ptr(vc_no_ghcb);
-
-	startup_64_load_idt(handler);
-}
-- 
2.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

The startup code that constructs the kernel virtual mapping runs from
the 1:1 mapping of memory itself, and therefore, cannot use absolute
symbol references. Before making changes in subsequent patches, move
this code into a separate source file under arch/x86/boot/startup/ where
all such code will be kept from now on.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/Makefile     |   2 +-
 arch/x86/boot/startup/map_kernel.c | 224 ++++++++++++++++++++
 arch/x86/kernel/head64.c           | 211 +-----------------
 3 files changed, 226 insertions(+), 211 deletions(-)

diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/Makefile
+++ b/arch/x86/boot/startup/Makefile
@@ -XXX,XX +XXX,XX @@ KMSAN_SANITIZE	:= n
 UBSAN_SANITIZE	:= n
 KCOV_INSTRUMENT	:= n
 
-obj-$(CONFIG_X86_64)		+= gdt_idt.o
+obj-$(CONFIG_X86_64)		+= gdt_idt.o map_kernel.o
 
 lib-$(CONFIG_X86_64)		+= la57toggle.o
 lib-$(CONFIG_EFI_MIXED)		+= efi-mixed.o
diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/arch/x86/boot/startup/map_kernel.c
@@ -XXX,XX +XXX,XX @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/init.h>
+#include <linux/linkage.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/pgtable.h>
+
+#include <asm/init.h>
+#include <asm/sections.h>
+#include <asm/setup.h>
+#include <asm/sev.h>
+
+extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
+extern unsigned int next_early_pgt;
+
+static inline bool check_la57_support(void)
+{
+	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
+		return false;
+
+	/*
+	 * 5-level paging is detected and enabled at kernel decompression
+	 * stage. Only check if it has been enabled there.
+	 */
+	if (!(native_read_cr4() & X86_CR4_LA57))
+		return false;
+
+	RIP_REL_REF(__pgtable_l5_enabled)	= 1;
+	RIP_REL_REF(pgdir_shift)		= 48;
+	RIP_REL_REF(ptrs_per_p4d)		= 512;
+	RIP_REL_REF(page_offset_base)		= __PAGE_OFFSET_BASE_L5;
+	RIP_REL_REF(vmalloc_base)		= __VMALLOC_BASE_L5;
+	RIP_REL_REF(vmemmap_base)		= __VMEMMAP_BASE_L5;
+
+	return true;
+}
+
+static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
+						    pmdval_t *pmd,
+						    unsigned long p2v_offset)
+{
+	unsigned long paddr, paddr_end;
+	int i;
+
+	/* Encrypt the kernel and related (if SME is active) */
+	sme_encrypt_kernel(bp);
+
+	/*
+	 * Clear the memory encryption mask from the .bss..decrypted section.
+	 * The bss section will be memset to zero later in the initialization so
+	 * there is no need to zero it after changing the memory encryption
+	 * attribute.
+	 */
+	if (sme_get_me_mask()) {
+		paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted);
+		paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted);
+
+		for (; paddr < paddr_end; paddr += PMD_SIZE) {
+			/*
+			 * On SNP, transition the page to shared in the RMP table so that
+			 * it is consistent with the page table attribute change.
+			 *
+			 * __start_bss_decrypted has a virtual address in the high range
+			 * mapping (kernel .text). PVALIDATE, by way of
+			 * early_snp_set_memory_shared(), requires a valid virtual
+			 * address but the kernel is currently running off of the identity
+			 * mapping so use the PA to get a *currently* valid virtual address.
+			 */
+			early_snp_set_memory_shared(paddr, paddr, PTRS_PER_PMD);
+
+			i = pmd_index(paddr - p2v_offset);
+			pmd[i] -= sme_get_me_mask();
+		}
+	}
+
+	/*
+	 * Return the SME encryption mask (if SME is active) to be used as a
+	 * modifier for the initial pgdir entry programmed into CR3.
+	 */
+	return sme_get_me_mask();
+}
+
+/* Code in __startup_64() can be relocated during execution, but the compiler
+ * doesn't have to generate PC-relative relocations when accessing globals from
+ * that function. Clang actually does not generate them, which leads to
+ * boot-time crashes. To work around this problem, every global pointer must
+ * be accessed using RIP_REL_REF(). Kernel virtual addresses can be determined
+ * by subtracting p2v_offset from the RIP-relative address.
+ */
+unsigned long __head __startup_64(unsigned long p2v_offset,
+				  struct boot_params *bp)
+{
+	pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);
+	unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);
+	unsigned long va_text, va_end;
+	unsigned long pgtable_flags;
+	unsigned long load_delta;
+	pgdval_t *pgd;
+	p4dval_t *p4d;
+	pudval_t *pud;
+	pmdval_t *pmd, pmd_entry;
+	bool la57;
+	int i;
+
+	la57 = check_la57_support();
+
+	/* Is the address too large? */
+	if (physaddr >> MAX_PHYSMEM_BITS)
+		for (;;);
+
+	/*
+	 * Compute the delta between the address I am compiled to run at
+	 * and the address I am actually running at.
+	 */
+	load_delta = __START_KERNEL_map + p2v_offset;
+	RIP_REL_REF(phys_base) = load_delta;
+
+	/* Is the address not 2M aligned? */
+	if (load_delta & ~PMD_MASK)
+		for (;;);
+
+	va_text = physaddr - p2v_offset;
+	va_end  = (unsigned long)rip_rel_ptr(_end) - p2v_offset;
+
+	/* Include the SME encryption mask in the fixup value */
+	load_delta += sme_get_me_mask();
+
+	/* Fixup the physical addresses in the page table */
+
+	pgd = rip_rel_ptr(early_top_pgt);
+	pgd[pgd_index(__START_KERNEL_map)] += load_delta;
+
+	if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
+		p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt);
+		p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
+
+		pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
+	}
+
+	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 2].pud += load_delta;
+	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 1].pud += load_delta;
+
+	for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
+		RIP_REL_REF(level2_fixmap_pgt)[i].pmd += load_delta;
+
+	/*
+	 * Set up the identity mapping for the switchover.  These
+	 * entries should *NOT* have the global bit set!  This also
+	 * creates a bunch of nonsense entries but that is fine --
+	 * it avoids problems around wraparound.
+	 */
+
+	pud = &early_pgts[0]->pmd;
+	pmd = &early_pgts[1]->pmd;
+	RIP_REL_REF(next_early_pgt) = 2;
+
+	pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
+
+	if (la57) {
+		p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd;
+
+		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
+		pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
+		pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;
+
+		i = physaddr >> P4D_SHIFT;
+		p4d[(i + 0) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
+		p4d[(i + 1) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
+	} else {
+		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
+		pgd[i + 0] = (pgdval_t)pud + pgtable_flags;
+		pgd[i + 1] = (pgdval_t)pud + pgtable_flags;
+	}
+
+	i = physaddr >> PUD_SHIFT;
+	pud[(i + 0) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
+	pud[(i + 1) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
+
+	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
+	/* Filter out unsupported __PAGE_KERNEL_* bits: */
+	pmd_entry &= RIP_REL_REF(__supported_pte_mask);
+	pmd_entry += sme_get_me_mask();
+	pmd_entry +=  physaddr;
+
+	for (i = 0; i < DIV_ROUND_UP(va_end - va_text, PMD_SIZE); i++) {
+		int idx = i + (physaddr >> PMD_SHIFT);
+
+		pmd[idx % PTRS_PER_PMD] = pmd_entry + i * PMD_SIZE;
+	}
+
+	/*
+	 * Fixup the kernel text+data virtual addresses. Note that
+	 * we might write invalid pmds, when the kernel is relocated
+	 * cleanup_highmap() fixes this up along with the mappings
+	 * beyond _end.
+	 *
+	 * Only the region occupied by the kernel image has so far
+	 * been checked against the table of usable memory regions
+	 * provided by the firmware, so invalidate pages outside that
+	 * region. A page table entry that maps to a reserved area of
+	 * memory would allow processor speculation into that area,
+	 * and on some hardware (particularly the UV platform) even
+	 * speculative access to some reserved areas is caught as an
+	 * error, causing the BIOS to halt the system.
+	 */
+
+	pmd = rip_rel_ptr(level2_kernel_pgt);
+
+	/* invalidate pages before the kernel image */
+	for (i = 0; i < pmd_index(va_text); i++)
+		pmd[i] &= ~_PAGE_PRESENT;
+
+	/* fixup pages that are part of the kernel image */
+	for (; i <= pmd_index(va_end); i++)
+		if (pmd[i] & _PAGE_PRESENT)
+			pmd[i] += load_delta;
+
+	/* invalidate pages after the kernel image */
+	for (; i < PTRS_PER_PMD; i++)
+		pmd[i] &= ~_PAGE_PRESENT;
+
+	return sme_postprocess_startup(bp, pmd, p2v_offset);
+}
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -XXX,XX +XXX,XX @@
  * Manage page tables very early on.
  */
 extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
-static unsigned int __initdata next_early_pgt;
+unsigned int __initdata next_early_pgt;
 pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
 
 #ifdef CONFIG_X86_5LEVEL
@@ -XXX,XX +XXX,XX @@ unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4;
 EXPORT_SYMBOL(vmemmap_base);
 #endif
 
-static inline bool check_la57_support(void)
-{
-	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
-		return false;
-
-	/*
-	 * 5-level paging is detected and enabled at kernel decompression
-	 * stage. Only check if it has been enabled there.
-	 */
-	if (!(native_read_cr4() & X86_CR4_LA57))
-		return false;
-
-	RIP_REL_REF(__pgtable_l5_enabled)	= 1;
-	RIP_REL_REF(pgdir_shift)		= 48;
-	RIP_REL_REF(ptrs_per_p4d)		= 512;
-	RIP_REL_REF(page_offset_base)		= __PAGE_OFFSET_BASE_L5;
-	RIP_REL_REF(vmalloc_base)		= __VMALLOC_BASE_L5;
-	RIP_REL_REF(vmemmap_base)		= __VMEMMAP_BASE_L5;
-
-	return true;
-}
-
-static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
-						    pmdval_t *pmd,
-						    unsigned long p2v_offset)
-{
-	unsigned long paddr, paddr_end;
-	int i;
-
-	/* Encrypt the kernel and related (if SME is active) */
-	sme_encrypt_kernel(bp);
-
-	/*
-	 * Clear the memory encryption mask from the .bss..decrypted section.
-	 * The bss section will be memset to zero later in the initialization so
-	 * there is no need to zero it after changing the memory encryption
-	 * attribute.
-	 */
-	if (sme_get_me_mask()) {
-		paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted);
-		paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted);
-
-		for (; paddr < paddr_end; paddr += PMD_SIZE) {
-			/*
-			 * On SNP, transition the page to shared in the RMP table so that
-			 * it is consistent with the page table attribute change.
-			 *
-			 * __start_bss_decrypted has a virtual address in the high range
-			 * mapping (kernel .text). PVALIDATE, by way of
-			 * early_snp_set_memory_shared(), requires a valid virtual
-			 * address but the kernel is currently running off of the identity
-			 * mapping so use the PA to get a *currently* valid virtual address.
-			 */
-			early_snp_set_memory_shared(paddr, paddr, PTRS_PER_PMD);
-
-			i = pmd_index(paddr - p2v_offset);
-			pmd[i] -= sme_get_me_mask();
-		}
-	}
-
-	/*
-	 * Return the SME encryption mask (if SME is active) to be used as a
-	 * modifier for the initial pgdir entry programmed into CR3.
-	 */
-	return sme_get_me_mask();
-}
-
-/* Code in __startup_64() can be relocated during execution, but the compiler
- * doesn't have to generate PC-relative relocations when accessing globals from
- * that function. Clang actually does not generate them, which leads to
- * boot-time crashes. To work around this problem, every global pointer must
- * be accessed using RIP_REL_REF(). Kernel virtual addresses can be determined
- * by subtracting p2v_offset from the RIP-relative address.
- */
-unsigned long __head __startup_64(unsigned long p2v_offset,
-				  struct boot_params *bp)
-{
-	pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);
-	unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);
-	unsigned long va_text, va_end;
-	unsigned long pgtable_flags;
-	unsigned long load_delta;
-	pgdval_t *pgd;
-	p4dval_t *p4d;
-	pudval_t *pud;
-	pmdval_t *pmd, pmd_entry;
-	bool la57;
-	int i;
-
-	la57 = check_la57_support();
-
-	/* Is the address too large? */
-	if (physaddr >> MAX_PHYSMEM_BITS)
-		for (;;);
-
-	/*
-	 * Compute the delta between the address I am compiled to run at
-	 * and the address I am actually running at.
-	 */
-	load_delta = __START_KERNEL_map + p2v_offset;
-	RIP_REL_REF(phys_base) = load_delta;
-
-	/* Is the address not 2M aligned? */
-	if (load_delta & ~PMD_MASK)
-		for (;;);
-
-	va_text = physaddr - p2v_offset;
-	va_end  = (unsigned long)rip_rel_ptr(_end) - p2v_offset;
-
-	/* Include the SME encryption mask in the fixup value */
-	load_delta += sme_get_me_mask();
-
-	/* Fixup the physical addresses in the page table */
-
-	pgd = rip_rel_ptr(early_top_pgt);
-	pgd[pgd_index(__START_KERNEL_map)] += load_delta;
-
-	if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
-		p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt);
-		p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
-
-		pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
-	}
-
-	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 2].pud += load_delta;
-	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 1].pud += load_delta;
-
-	for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
-		RIP_REL_REF(level2_fixmap_pgt)[i].pmd += load_delta;
-
-	/*
-	 * Set up the identity mapping for the switchover.  These
-	 * entries should *NOT* have the global bit set!  This also
-	 * creates a bunch of nonsense entries but that is fine --
-	 * it avoids problems around wraparound.
-	 */
-
-	pud = &early_pgts[0]->pmd;
-	pmd = &early_pgts[1]->pmd;
-	RIP_REL_REF(next_early_pgt) = 2;
-
-	pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
-
-	if (la57) {
-		p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd;
-
-		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
-		pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
-		pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;
-
-		i = physaddr >> P4D_SHIFT;
-		p4d[(i + 0) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
-		p4d[(i + 1) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
-	} else {
-		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
-		pgd[i + 0] = (pgdval_t)pud + pgtable_flags;
-		pgd[i + 1] = (pgdval_t)pud + pgtable_flags;
-	}
-
-	i = physaddr >> PUD_SHIFT;
-	pud[(i + 0) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
-	pud[(i + 1) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
-
-	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
-	/* Filter out unsupported __PAGE_KERNEL_* bits: */
-	pmd_entry &= RIP_REL_REF(__supported_pte_mask);
-	pmd_entry += sme_get_me_mask();
-	pmd_entry +=  physaddr;
-
-	for (i = 0; i < DIV_ROUND_UP(va_end - va_text, PMD_SIZE); i++) {
-		int idx = i + (physaddr >> PMD_SHIFT);
-
-		pmd[idx % PTRS_PER_PMD] = pmd_entry + i * PMD_SIZE;
-	}
-
-	/*
-	 * Fixup the kernel text+data virtual addresses. Note that
-	 * we might write invalid pmds, when the kernel is relocated
-	 * cleanup_highmap() fixes this up along with the mappings
-	 * beyond _end.
-	 *
-	 * Only the region occupied by the kernel image has so far
-	 * been checked against the table of usable memory regions
-	 * provided by the firmware, so invalidate pages outside that
-	 * region. A page table entry that maps to a reserved area of
-	 * memory would allow processor speculation into that area,
-	 * and on some hardware (particularly the UV platform) even
-	 * speculative access to some reserved areas is caught as an
-	 * error, causing the BIOS to halt the system.
-	 */
-
-	pmd = rip_rel_ptr(level2_kernel_pgt);
-
-	/* invalidate pages before the kernel image */
-	for (i = 0; i < pmd_index(va_text); i++)
-		pmd[i] &= ~_PAGE_PRESENT;
-
-	/* fixup pages that are part of the kernel image */
-	for (; i <= pmd_index(va_end); i++)
-		if (pmd[i] & _PAGE_PRESENT)
-			pmd[i] += load_delta;
-
-	/* invalidate pages after the kernel image */
-	for (; i < PTRS_PER_PMD; i++)
-		pmd[i] &= ~_PAGE_PRESENT;
-
-	return sme_postprocess_startup(bp, pmd, p2v_offset);
-}
-
 /* Wipe all early page tables except for the kernel symbol map */
 static void __init reset_early_page_tables(void)
 {
-- 
2.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

Now that __startup_64() is built using -fPIC, RIP_REL_REF() has become a
NOP and can be removed. Only some occurrences of rip_rel_ptr() will
remain, to explicitly take the address of certain global structures in
the 1:1 mapping of memory.

While at it, update the code comment to describe why this is needed.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/map_kernel.c | 41 ++++++++++----------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/map_kernel.c
+++ b/arch/x86/boot/startup/map_kernel.c
@@ -XXX,XX +XXX,XX @@ static inline bool check_la57_support(void)
 	if (!(native_read_cr4() & X86_CR4_LA57))
 		return false;
 
-	RIP_REL_REF(__pgtable_l5_enabled)	= 1;
-	RIP_REL_REF(pgdir_shift)		= 48;
-	RIP_REL_REF(ptrs_per_p4d)		= 512;
-	RIP_REL_REF(page_offset_base)		= __PAGE_OFFSET_BASE_L5;
-	RIP_REL_REF(vmalloc_base)		= __VMALLOC_BASE_L5;
-	RIP_REL_REF(vmemmap_base)		= __VMEMMAP_BASE_L5;
+	__pgtable_l5_enabled	= 1;
+	pgdir_shift		= 48;
+	ptrs_per_p4d		= 512;
+	page_offset_base	= __PAGE_OFFSET_BASE_L5;
+	vmalloc_base		= __VMALLOC_BASE_L5;
+	vmemmap_base		= __VMEMMAP_BASE_L5;
 
 	return true;
 }
@@ -XXX,XX +XXX,XX @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
 	return sme_get_me_mask();
 }
 
-/* Code in __startup_64() can be relocated during execution, but the compiler
- * doesn't have to generate PC-relative relocations when accessing globals from
- * that function. Clang actually does not generate them, which leads to
- * boot-time crashes. To work around this problem, every global pointer must
- * be accessed using RIP_REL_REF(). Kernel virtual addresses can be determined
- * by subtracting p2v_offset from the RIP-relative address.
+/*
+ * This code is compiled using PIC codegen because it will execute from the
+ * early 1:1 mapping of memory, which deviates from the mapping expected by the
+ * linker. Due to this deviation, taking the address of a global variable will
+ * produce an ambiguous result when using the plain & operator.  Instead,
+ * rip_rel_ptr() must be used, which will return the RIP-relative address in
+ * the 1:1 mapping of memory. Kernel virtual addresses can be determined by
+ * subtracting p2v_offset from the RIP-relative address.
  */
 unsigned long __head __startup_64(unsigned long p2v_offset,
 				  struct boot_params *bp)
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
 	 * Compute the delta between the address I am compiled to run at
 	 * and the address I am actually running at.
 	 */
-	load_delta = __START_KERNEL_map + p2v_offset;
-	RIP_REL_REF(phys_base) = load_delta;
+	phys_base = load_delta = __START_KERNEL_map + p2v_offset;
 
 	/* Is the address not 2M aligned? */
 	if (load_delta & ~PMD_MASK)
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
 		pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
 	}
 
-	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 2].pud += load_delta;
-	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 1].pud += load_delta;
+	level3_kernel_pgt[PTRS_PER_PUD - 2].pud += load_delta;
+	level3_kernel_pgt[PTRS_PER_PUD - 1].pud += load_delta;
 
 	for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
-		RIP_REL_REF(level2_fixmap_pgt)[i].pmd += load_delta;
+		level2_fixmap_pgt[i].pmd += load_delta;
 
 	/*
 	 * Set up the identity mapping for the switchover.  These
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
 
 	pud = &early_pgts[0]->pmd;
 	pmd = &early_pgts[1]->pmd;
-	RIP_REL_REF(next_early_pgt) = 2;
+	next_early_pgt = 2;
 
 	pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
 
 	if (la57) {
-		p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd;
+		p4d = &early_pgts[next_early_pgt++]->pmd;
 
 		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
 		pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
 
 	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
 	/* Filter out unsupported __PAGE_KERNEL_* bits: */
-	pmd_entry &= RIP_REL_REF(__supported_pte_mask);
+	pmd_entry &= __supported_pte_mask;
 	pmd_entry += sme_get_me_mask();
 	pmd_entry +=  physaddr;
 
-- 
2.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/Makefile                             | 1 +
 arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} | 2 --
 arch/x86/mm/Makefile                                       | 6 ------
 3 files changed, 1 insertion(+), 8 deletions(-)

From: Ard Biesheuvel <ardb@kernel.org>

RIP_REL_REF() has no effect on code residing in arch/x86/boot/startup,
as it is built with -fPIC. So remove any occurrences from the SME
startup code.

Note the SME is the only caller of cc_set_mask() that requires this, so
drop it from there as well.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/sme.c        | 11 +++++------
 arch/x86/include/asm/coco.h        |  2 +-
 arch/x86/include/asm/mem_encrypt.h |  2 +-
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/arch/x86/boot/startup/sme.c b/arch/x86/boot/startup/sme.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/sme.c
+++ b/arch/x86/boot/startup/sme.c
@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
 	 * instrumentation or checking boot_cpu_data in the cc_platform_has()
 	 * function.
 	 */
-	if (!sme_get_me_mask() ||
-	    RIP_REL_REF(sev_status) & MSR_AMD64_SEV_ENABLED)
+	if (!sme_get_me_mask() || sev_status & MSR_AMD64_SEV_ENABLED)
 		return;
 
 	/*
@@ -XXX,XX +XXX,XX @@ void __head sme_enable(struct boot_params *bp)
 	me_mask = 1UL << (ebx & 0x3f);
 
 	/* Check the SEV MSR whether SEV or SME is enabled */
-	RIP_REL_REF(sev_status) = msr = __rdmsr(MSR_AMD64_SEV);
+	sev_status = msr = __rdmsr(MSR_AMD64_SEV);
 	feature_mask = (msr & MSR_AMD64_SEV_ENABLED) ? AMD_SEV_BIT : AMD_SME_BIT;
 
 	/*
@@ -XXX,XX +XXX,XX @@ void __head sme_enable(struct boot_params *bp)
 			return;
 	}
 
-	RIP_REL_REF(sme_me_mask) = me_mask;
-	RIP_REL_REF(physical_mask) &= ~me_mask;
-	RIP_REL_REF(cc_vendor) = CC_VENDOR_AMD;
+	sme_me_mask	= me_mask;
+	physical_mask	&= ~me_mask;
+	cc_vendor	= CC_VENDOR_AMD;
 	cc_set_mask(me_mask);
 }
diff --git a/arch/x86/include/asm/coco.h b/arch/x86/include/asm/coco.h
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/include/asm/coco.h
+++ b/arch/x86/include/asm/coco.h
@@ -XXX,XX +XXX,XX @@ static inline u64 cc_get_mask(void)
 
 static inline void cc_set_mask(u64 mask)
 {
-	RIP_REL_REF(cc_mask) = mask;
+	cc_mask = mask;
 }
 
 u64 cc_mkenc(u64 val);
diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/include/asm/mem_encrypt.h
+++ b/arch/x86/include/asm/mem_encrypt.h
@@ -XXX,XX +XXX,XX @@ void __init sev_es_init_vc_handling(void);
 
 static inline u64 sme_get_me_mask(void)
 {
-	return RIP_REL_REF(sme_me_mask);
+	return sme_me_mask;
 }
 
 #define __bss_decrypted __section(".bss..decrypted")
-- 
2.49.0.504.g3bcea36a83-goog