Series comparison

-[RFC PATCH 0/6] x86: Refactor and consolidate startup code
+[PATCH v4 00/11] x86: Refactor and consolidate startup code
 From: Ard Biesheuvel <ardb@kernel.org>
-Start refactoring the x86 startup code so we keep all the code that is
+!! NOTE: patches #7 - #10 depend on [0] !!
 shared between different boot stages (EFI stub, decompressor, early
 startup in the core kernel *) and/or needs to be built in a special way
 (due to the fact that it is C code that runs from the 1:1 mapping of
 RAM) in a single place, sharing all the C flags and other runes that are
 needed to disable instrumentation, sanitizers, etc.
-This is an RFC so I have left some things for later, e.g., the SEV-SNP
+Reorganize C code that is used during early boot, either in the
-init code in arch/x86/coco that is shared between all of the above [*]
+decompressor/EFI stub or the kernel proper, but before the kernel
-and will be tricky to disentangle; there are also some known issues in
+virtual mapping is up.
-that code related to EFI boot that we are addressing in parallel.
 v4:
 - drop patches that were queued up
 - fix address space error in patch #1
 - add patches for SEV-SNP boot code - these cannot be applied yet, but
   are included for completeness
 v3:
 - keep rip_rel_ptr() around in PIC code - sadly, it is still needed in
   some cases
 - remove RIP_REL_REF() uses in separate patches
 - keep __head annotations for now, they will all be removed later
 - disable objtool validation for library objects (i.e., pieces that are
   not linked into vmlinux)
 I will follow up with a series that gets rid of .head.text altogether,
 as it will no longer be needed at all once the startup code is checked
 for absolute relocations.
 Cc: Tom Lendacky <thomas.lendacky@amd.com>
 Cc: Dionna Amalie Glaze <dionnaglaze@google.com>
 Cc: Kevin Loughlin <kevinloughlin@google.com>
-Ard Biesheuvel (6):
+[0] https://lore.kernel.org/all/20250410132850.3708703-2-ardb+git@google.com/T/#u
-  x86/boot/compressed: Merge local pgtable.h include into asm/boot.h
-  x86/boot: Move 5-level paging trampoline into startup code
+Ard Biesheuvel (11):
-  x86/boot: Move EFI mixed mode startup code back under arch/x86
+  x86/asm: Make rip_rel_ptr() usable from fPIC code
-  x86/boot: Move early GDT/IDT setup code into startup/
+  x86/boot: Move the early GDT/IDT setup code into startup/
   x86/boot: Move early kernel mapping code into startup/
+  x86/boot: Drop RIP_REL_REF() uses from early mapping code
   x86/boot: Move early SME init code into startup/
+  x86/boot: Drop RIP_REL_REF() uses from SME startup code
+  x86/sev: Prepare for splitting off early SEV code
+  x86/sev: Split off startup code from core code
+  x86/boot: Move SEV startup code into startup/
+  x86/boot: Drop RIP_REL_REF() uses from early SEV code
+  x86/asm: Retire RIP_REL_REF()
- arch/x86/Makefile                                                             |   1 +
+ arch/x86/boot/compressed/Makefile                          |    2 +-
- arch/x86/boot/compressed/Makefile                                             |   4 +-
+ arch/x86/boot/compressed/sev.c                             |   17 +-
- arch/x86/boot/compressed/head_64.S                                            |   1 -
+ arch/x86/boot/startup/Makefile                             |   16 +
- arch/x86/boot/compressed/misc.c                                               |   1 -
+ arch/x86/boot/startup/gdt_idt.c                            |   84 +
- arch/x86/boot/compressed/pgtable.h                                            |  18 --
+ arch/x86/boot/startup/map_kernel.c                         |  225 +++
- arch/x86/boot/compressed/pgtable_64.c                                         |   1 -
+ arch/x86/{coco/sev/shared.c => boot/startup/sev-shared.c}  |  375 +----
- arch/x86/boot/startup/Makefile                                                |  22 ++
+ arch/x86/boot/startup/sev-startup.c                        | 1395 ++++++++++++++++
- drivers/firmware/efi/libstub/x86-mixed.S => arch/x86/boot/startup/efi-mixed.S |   0
+ arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} |   19 +-
- arch/x86/boot/startup/gdt_idt.c                                               |  82 ++++++
+ arch/x86/coco/sev/Makefile                                 |   19 -
- arch/x86/boot/{compressed => startup}/la57toggle.S                            |   1 -
+ arch/x86/coco/sev/core.c                                   | 1726 ++++----------------
- arch/x86/boot/startup/map_kernel.c                                            | 232 +++++++++++++++
+ arch/x86/include/asm/asm.h                                 |    5 -
- arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c}                    |  45 ++-
+ arch/x86/include/asm/coco.h                                |    2 +-
- arch/x86/include/asm/boot.h                                                   |  10 +
+ arch/x86/include/asm/mem_encrypt.h                         |    2 +-
- arch/x86/include/asm/mem_encrypt.h                                            |   2 +-
+ arch/x86/include/asm/sev-internal.h                        |  112 ++
- arch/x86/kernel/head64.c                                                      | 302 +-------------------
+ arch/x86/include/asm/sev.h                                 |   37 +
- arch/x86/mm/Makefile                                                          |   6 -
+ arch/x86/kernel/head64.c                                   |  285 +---
- drivers/firmware/efi/libstub/Makefile                                         |   1 -
+ arch/x86/mm/Makefile                                       |    6 -
-files changed, 372 insertions(+), 357 deletions(-)
+files changed, 2208 insertions(+), 2119 deletions(-)
  delete mode 100644 arch/x86/boot/compressed/pgtable.h
  create mode 100644 arch/x86/boot/startup/Makefile
  rename drivers/firmware/efi/libstub/x86-mixed.S => arch/x86/boot/startup/efi-mixed.S (100%)
  create mode 100644 arch/x86/boot/startup/gdt_idt.c
- rename arch/x86/boot/{compressed => startup}/la57toggle.S (99%)
  create mode 100644 arch/x86/boot/startup/map_kernel.c
- rename arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} (92%)
+ rename arch/x86/{coco/sev/shared.c => boot/startup/sev-shared.c} (78%)
  create mode 100644 arch/x86/boot/startup/sev-startup.c
  rename arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} (97%)
  create mode 100644 arch/x86/include/asm/sev-internal.h
 --
-.49.0.472.ge94155a9ec-goog
+.49.0.504.g3bcea36a83-goog

-New patch
+[PATCH v4 01/11] x86/asm: Make rip_rel_ptr() usable from fPIC code
+From: Ard Biesheuvel <ardb@kernel.org>
 RIP_REL_REF() is used in non-PIC C code that is called very early,
 before the kernel virtual mapping is up, which is the mapping that the
 linker expects. It is currently used in two different ways:
 - to refer to the value of a global variable, including as an lvalue in
   assignments;
 - to take the address of a global variable via the mapping that the code
   currently executes at.
 The former case is only needed in non-PIC code, as PIC code will never
 use absolute symbol references when the address of the symbol is not
 being used. But taking the address of a variable in PIC code may still
 require extra care, as a stack allocated struct assignment may be
 emitted as a memcpy() from a statically allocated copy in .rodata.
 For instance, this
   void startup_64_setup_gdt_idt(void)
   {
         struct desc_ptr startup_gdt_descr = {
                 .address = (__force unsigned long)gdt_page.gdt,
                 .size    = GDT_SIZE - 1,
         };
 may result in an absolute symbol reference in PIC code, even though the
 struct is allocated on the stack and populated at runtime.
 To address this case, make rip_rel_ptr() accessible in PIC code, and
 update any existing uses where the address of a global variable is
 taken using RIP_REL_REF.
 Once all code of this nature has been moved into arch/x86/boot/startup
 and built with -fPIC, RIP_REL_REF() can be retired, and only
 rip_rel_ptr() will remain.
 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
 ---
  arch/x86/coco/sev/core.c           |  2 +-
  arch/x86/coco/sev/shared.c         |  4 ++--
  arch/x86/include/asm/asm.h         |  2 +-
  arch/x86/kernel/head64.c           | 24 ++++++++++----------
  arch/x86/mm/mem_encrypt_identity.c |  6 ++---
 files changed, 19 insertions(+), 19 deletions(-)
 diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/coco/sev/core.c
 +++ b/arch/x86/coco/sev/core.c
@@ -XXX,XX +XXX,XX @@ static __head void svsm_setup(struct cc_blob_sev_info *cc_info)
       * kernel was loaded (physbase), so the get the CA address using
       * RIP-relative addressing.
       */
 -    pa = (u64)&RIP_REL_REF(boot_svsm_ca_page);
 +    pa = (u64)rip_rel_ptr(&boot_svsm_ca_page);
      /*
       * Switch over to the boot SVSM CA while the current CA is still
 diff --git a/arch/x86/coco/sev/shared.c b/arch/x86/coco/sev/shared.c
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/coco/sev/shared.c
 +++ b/arch/x86/coco/sev/shared.c
@@ -XXX,XX +XXX,XX @@ static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid
   */
  static const struct snp_cpuid_table *snp_cpuid_get_table(void)
  {
 -    return &RIP_REL_REF(cpuid_table_copy);
 +    return rip_rel_ptr(&cpuid_table_copy);
  }
  /*
@@ -XXX,XX +XXX,XX @@ static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info)
       * routine is running identity mapped when called, both by the decompressor
       * code and the early kernel code.
       */
 -    if (!rmpadjust((unsigned long)&RIP_REL_REF(boot_ghcb_page), RMP_PG_SIZE_4K, 1))
 +    if (!rmpadjust((unsigned long)rip_rel_ptr(&boot_ghcb_page), RMP_PG_SIZE_4K, 1))
          return false;
      /*
 diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/include/asm/asm.h
 +++ b/arch/x86/include/asm/asm.h
@@ -XXX,XX +XXX,XX @@
  #endif
  #ifndef __ASSEMBLER__
 -#ifndef __pic__
  static __always_inline __pure void *rip_rel_ptr(void *p)
  {
      asm("leaq %c1(%%rip), %0" : "=r"(p) : "i"(p));
      return p;
  }
 +#ifndef __pic__
  #define RIP_REL_REF(var)    (*(typeof(&(var)))rip_rel_ptr(&(var)))
  #else
  #define RIP_REL_REF(var)    (var)
 diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/kernel/head64.c
 +++ b/arch/x86/kernel/head64.c
@@ -XXX,XX +XXX,XX @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
       * attribute.
       */
      if (sme_get_me_mask()) {
 -        paddr = (unsigned long)&RIP_REL_REF(__start_bss_decrypted);
 -        paddr_end = (unsigned long)&RIP_REL_REF(__end_bss_decrypted);
 +        paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted);
 +        paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted);
          for (; paddr < paddr_end; paddr += PMD_SIZE) {
              /*
@@ -XXX,XX +XXX,XX @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
  unsigned long __head __startup_64(unsigned long p2v_offset,
                    struct boot_params *bp)
  {
 -    pmd_t (*early_pgts)[PTRS_PER_PMD] = RIP_REL_REF(early_dynamic_pgts);
 -    unsigned long physaddr = (unsigned long)&RIP_REL_REF(_text);
 +    pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);
 +    unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);
      unsigned long va_text, va_end;
      unsigned long pgtable_flags;
      unsigned long load_delta;
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
          for (;;);
      va_text = physaddr - p2v_offset;
 -    va_end  = (unsigned long)&RIP_REL_REF(_end) - p2v_offset;
 +    va_end  = (unsigned long)rip_rel_ptr(_end) - p2v_offset;
      /* Include the SME encryption mask in the fixup value */
      load_delta += sme_get_me_mask();
      /* Fixup the physical addresses in the page table */
 -    pgd = &RIP_REL_REF(early_top_pgt)->pgd;
 +    pgd = rip_rel_ptr(early_top_pgt);
      pgd[pgd_index(__START_KERNEL_map)] += load_delta;
      if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
 -        p4d = (p4dval_t *)&RIP_REL_REF(level4_kernel_pgt);
 +        p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt);
          p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
          pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
       * error, causing the BIOS to halt the system.
       */
 -    pmd = &RIP_REL_REF(level2_kernel_pgt)->pmd;
 +    pmd = rip_rel_ptr(level2_kernel_pgt);
      /* invalidate pages before the kernel image */
      for (i = 0; i < pmd_index(va_text); i++)
@@ -XXX,XX +XXX,XX @@ static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
  static void __head startup_64_load_idt(void *vc_handler)
  {
      struct desc_ptr desc = {
 -        .address = (unsigned long)&RIP_REL_REF(bringup_idt_table),
 +        .address = (unsigned long)rip_rel_ptr(bringup_idt_table),
          .size    = sizeof(bringup_idt_table) - 1,
      };
      struct idt_data data;
@@ -XXX,XX +XXX,XX @@ void early_setup_idt(void)
   */
  void __head startup_64_setup_gdt_idt(void)
  {
 -    struct desc_struct *gdt = (void *)(__force unsigned long)gdt_page.gdt;
 +    struct gdt_page *gp = rip_rel_ptr((void *)(__force unsigned long)&gdt_page);
      void *handler = NULL;
      struct desc_ptr startup_gdt_descr = {
 -        .address = (unsigned long)&RIP_REL_REF(*gdt),
 +        .address = (unsigned long)gp->gdt,
          .size    = GDT_SIZE - 1,
      };
@@ -XXX,XX +XXX,XX @@ void __head startup_64_setup_gdt_idt(void)
               "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
      if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
 -        handler = &RIP_REL_REF(vc_no_ghcb);
 +        handler = rip_rel_ptr(vc_no_ghcb);
      startup_64_load_idt(handler);
  }
 diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/mm/mem_encrypt_identity.c
 +++ b/arch/x86/mm/mem_encrypt_identity.c
@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
       *     memory from being cached.
       */
 -    kernel_start = (unsigned long)RIP_REL_REF(_text);
 -    kernel_end = ALIGN((unsigned long)RIP_REL_REF(_end), PMD_SIZE);
 +    kernel_start = (unsigned long)rip_rel_ptr(_text);
 +    kernel_end = ALIGN((unsigned long)rip_rel_ptr(_end), PMD_SIZE);
      kernel_len = kernel_end - kernel_start;
      initrd_start = 0;
@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
       *   pagetable structures for the encryption of the kernel
       *   pagetable structures for workarea (in case not currently mapped)
       */
 -    execute_start = workarea_start = (unsigned long)RIP_REL_REF(sme_workarea);
 +    execute_start = workarea_start = (unsigned long)rip_rel_ptr(sme_workarea);
      execute_end = execute_start + (PAGE_SIZE * 2) + PMD_SIZE;
      execute_len = execute_end - execute_start;
 --
 .49.0.504.g3bcea36a83-goog

-[RFC PATCH 4/6] x86/boot: Move early GDT/IDT setup code into startup/
+[PATCH v4 02/11] x86/boot: Move the early GDT/IDT setup code into startup/
 From: Ard Biesheuvel <ardb@kernel.org>
 Move the early GDT/IDT setup code that runs long before the kernel
 virtual mapping is up into arch/x86/boot/startup/, and build it in a way
 that ensures that the code tolerates being called from the 1:1 mapping
-of memory.
+of memory. The code itself is left unchanged by this patch.
 This allows the RIP_REL_REF() macro uses to be dropped, and removes the
 need for emitting the code into the special .head.text section.
 Also tweak the sed symbol matching pattern in the decompressor to match
 on lower case 't' or 'b', as these will be emitted by Clang for symbols
 with hidden linkage.
 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
 ---
  arch/x86/boot/compressed/Makefile |  2 +-
  arch/x86/boot/startup/Makefile    | 15 ++++
- arch/x86/boot/startup/gdt_idt.c   | 82 ++++++++++++++++++++
+ arch/x86/boot/startup/gdt_idt.c   | 84 ++++++++++++++++++++
- arch/x86/kernel/head64.c          | 74 ------------------
+ arch/x86/kernel/head64.c          | 74 -----------------
-files changed, 98 insertions(+), 75 deletions(-)
+files changed, 100 insertions(+), 75 deletions(-)
 diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/boot/compressed/Makefile
 +++ b/arch/x86/boot/compressed/Makefile
 ...
 +
 +#include <linux/linkage.h>
 +#include <linux/types.h>
 +
 +#include <asm/desc.h>
++#include <asm/init.h>
 +#include <asm/setup.h>
 +#include <asm/sev.h>
 +#include <asm/trapnr.h>
 +
 +/*
 ...
 + * which also hasn't happened yet in early CPU bringup.
 + */
 +static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
 +
 +/* This may run while still in the direct mapping */
-+static void startup_64_load_idt(void *vc_handler)
++static void __head startup_64_load_idt(void *vc_handler)
 +{
 +    struct desc_ptr desc = {
-+        .address = (unsigned long)bringup_idt_table,
++        .address = (unsigned long)rip_rel_ptr(bringup_idt_table),
 +        .size    = sizeof(bringup_idt_table) - 1,
 +    };
 +    struct idt_data data;
 +    gate_desc idt_desc;
 +
 ...
 +}
 +
 +/*
 + * Setup boot CPU state needed before kernel switches to virtual addresses.
 + */
-+void __init startup_64_setup_gdt_idt(void)
++void __head startup_64_setup_gdt_idt(void)
 +{
++    struct gdt_page *gp = rip_rel_ptr((void *)(__force unsigned long)&gdt_page);
 +    void *handler = NULL;
 +
 +    struct desc_ptr startup_gdt_descr = {
-+        .address = (__force unsigned long)gdt_page.gdt,
++        .address = (unsigned long)gp->gdt,
 +        .size    = GDT_SIZE - 1,
 +    };
 +
 +    /* Load GDT */
 +    native_load_gdt(&startup_gdt_descr);
 ...
 +    asm volatile("movl %%eax, %%ds\n"
 +             "movl %%eax, %%ss\n"
 +             "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
 +
 +    if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
-+        handler = vc_no_ghcb;
++        handler = rip_rel_ptr(vc_no_ghcb);
 +
 +    startup_64_load_idt(handler);
 +}
 diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
 index XXXXXXX..XXXXXXX 100644
 ...
 -
 -/* This may run while still in the direct mapping */
 -static void __head startup_64_load_idt(void *vc_handler)
 -{
 -    struct desc_ptr desc = {
--        .address = (unsigned long)&RIP_REL_REF(bringup_idt_table),
+-        .address = (unsigned long)rip_rel_ptr(bringup_idt_table),
 -        .size    = sizeof(bringup_idt_table) - 1,
 -    };
 -    struct idt_data data;
 -    gate_desc idt_desc;
 -
 ...
 -/*
 - * Setup boot CPU state needed before kernel switches to virtual addresses.
 - */
 -void __head startup_64_setup_gdt_idt(void)
 -{
--    struct desc_struct *gdt = (void *)(__force unsigned long)gdt_page.gdt;
+-    struct gdt_page *gp = rip_rel_ptr((void *)(__force unsigned long)&gdt_page);
 -    void *handler = NULL;
 -
 -    struct desc_ptr startup_gdt_descr = {
--        .address = (unsigned long)&RIP_REL_REF(*gdt),
+-        .address = (unsigned long)gp->gdt,
 -        .size    = GDT_SIZE - 1,
 -    };
 -
 -    /* Load GDT */
 -    native_load_gdt(&startup_gdt_descr);
 ...
 -    asm volatile("movl %%eax, %%ds\n"
 -             "movl %%eax, %%ss\n"
 -             "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
 -
 -    if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
--        handler = &RIP_REL_REF(vc_no_ghcb);
+-        handler = rip_rel_ptr(vc_no_ghcb);
 -
 -    startup_64_load_idt(handler);
 -}
 --
-.49.0.472.ge94155a9ec-goog
+.49.0.504.g3bcea36a83-goog

-[RFC PATCH 5/6] x86/boot: Move early kernel mapping code into startup/
+[PATCH v4 03/11] x86/boot: Move early kernel mapping code into startup/
 From: Ard Biesheuvel <ardb@kernel.org>
 The startup code that constructs the kernel virtual mapping runs from
 the 1:1 mapping of memory itself, and therefore, cannot use absolute
-symbol references. Move this code into a separate source file under
+symbol references. Before making changes in subsequent patches, move
-arch/x86/boot/startup/ where all such code will be kept from now on.
+this code into a separate source file under arch/x86/boot/startup/ where
+all such code will be kept from now on.
 Since all code here is constructed in a manner that ensures that it
 tolerates running from the 1:1 mapping of memory, any uses of the
 RIP_REL_REF() macro can be dropped, along with __head annotations for
 placing this code in a dedicated startup section.
 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
 ---
  arch/x86/boot/startup/Makefile     |   2 +-
- arch/x86/boot/startup/map_kernel.c | 232 ++++++++++++++++++++
+ arch/x86/boot/startup/map_kernel.c | 224 ++++++++++++++++++++
- arch/x86/kernel/head64.c           | 228 +------------------
+ arch/x86/kernel/head64.c           | 211 +-----------------
-files changed, 234 insertions(+), 228 deletions(-)
+files changed, 226 insertions(+), 211 deletions(-)
 diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/boot/startup/Makefile
 +++ b/arch/x86/boot/startup/Makefile
 ...
 +#include <linux/linkage.h>
 +#include <linux/types.h>
 +#include <linux/kernel.h>
 +#include <linux/pgtable.h>
 +
++#include <asm/init.h>
 +#include <asm/sections.h>
 +#include <asm/setup.h>
 +#include <asm/sev.h>
 +
 +extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
 +extern unsigned int next_early_pgt;
-+
-+#ifdef CONFIG_X86_5LEVEL
-+unsigned int __pgtable_l5_enabled __ro_after_init;
-+unsigned int pgdir_shift __ro_after_init = 39;
-+EXPORT_SYMBOL(pgdir_shift);
-+unsigned int ptrs_per_p4d __ro_after_init = 1;
-+EXPORT_SYMBOL(ptrs_per_p4d);
-+#endif
-+
-+#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT
-+unsigned long page_offset_base __ro_after_init = __PAGE_OFFSET_BASE_L4;
-+EXPORT_SYMBOL(page_offset_base);
-+unsigned long vmalloc_base __ro_after_init = __VMALLOC_BASE_L4;
-+EXPORT_SYMBOL(vmalloc_base);
-+unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4;
-+EXPORT_SYMBOL(vmemmap_base);
-+#endif
 +
 +static inline bool check_la57_support(void)
 +{
 +    if (!IS_ENABLED(CONFIG_X86_5LEVEL))
 +        return false;
 ...
 +     * stage. Only check if it has been enabled there.
 +     */
 +    if (!(native_read_cr4() & X86_CR4_LA57))
 +        return false;
 +
-+    __pgtable_l5_enabled    = 1;
++    RIP_REL_REF(__pgtable_l5_enabled)    = 1;
-+    pgdir_shift        = 48;
++    RIP_REL_REF(pgdir_shift)        = 48;
-+    ptrs_per_p4d        = 512;
++    RIP_REL_REF(ptrs_per_p4d)        = 512;
-+    page_offset_base    = __PAGE_OFFSET_BASE_L5;
++    RIP_REL_REF(page_offset_base)        = __PAGE_OFFSET_BASE_L5;
-+    vmalloc_base        = __VMALLOC_BASE_L5;
++    RIP_REL_REF(vmalloc_base)        = __VMALLOC_BASE_L5;
-+    vmemmap_base        = __VMEMMAP_BASE_L5;
++    RIP_REL_REF(vmemmap_base)        = __VMEMMAP_BASE_L5;
 +
 +    return true;
 +}
 +
-+static unsigned long sme_postprocess_startup(struct boot_params *bp,
++static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
-+                         pmdval_t *pmd,
++                            pmdval_t *pmd,
-+                         unsigned long p2v_offset)
++                            unsigned long p2v_offset)
 +{
 +    unsigned long paddr, paddr_end;
 +    int i;
 +
 +    /* Encrypt the kernel and related (if SME is active) */
 ...
 +     * The bss section will be memset to zero later in the initialization so
 +     * there is no need to zero it after changing the memory encryption
 +     * attribute.
 +     */
 +    if (sme_get_me_mask()) {
-+        paddr = (unsigned long)__start_bss_decrypted;
++        paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted);
-+        paddr_end = (unsigned long)__end_bss_decrypted;
++        paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted);
 +
 +        for (; paddr < paddr_end; paddr += PMD_SIZE) {
 +            /*
 +             * On SNP, transition the page to shared in the RMP table so that
 +             * it is consistent with the page table attribute change.
 ...
 +     * modifier for the initial pgdir entry programmed into CR3.
 +     */
 +    return sme_get_me_mask();
 +}
 +
-+unsigned long __init __startup_64(unsigned long p2v_offset,
++/* Code in __startup_64() can be relocated during execution, but the compiler
 + * doesn't have to generate PC-relative relocations when accessing globals from
 + * that function. Clang actually does not generate them, which leads to
 + * boot-time crashes. To work around this problem, every global pointer must
 + * be accessed using RIP_REL_REF(). Kernel virtual addresses can be determined
 + * by subtracting p2v_offset from the RIP-relative address.
 + */
 +unsigned long __head __startup_64(unsigned long p2v_offset,
 +                  struct boot_params *bp)
 +{
-+    pmd_t (*early_pgts)[PTRS_PER_PMD] = early_dynamic_pgts;
++    pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);
-+    unsigned long physaddr = (unsigned long)_text;
++    unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);
 +    unsigned long va_text, va_end;
 +    unsigned long pgtable_flags;
 +    unsigned long load_delta;
 +    pgdval_t *pgd;
 +    p4dval_t *p4d;
 ...
 +
 +    /*
 +     * Compute the delta between the address I am compiled to run at
 +     * and the address I am actually running at.
 +     */
-+    phys_base = load_delta = __START_KERNEL_map + p2v_offset;
++    load_delta = __START_KERNEL_map + p2v_offset;
 +    RIP_REL_REF(phys_base) = load_delta;
 +
 +    /* Is the address not 2M aligned? */
 +    if (load_delta & ~PMD_MASK)
 +        for (;;);
 +
 +    va_text = physaddr - p2v_offset;
-+    va_end  = (unsigned long)_end - p2v_offset;
++    va_end  = (unsigned long)rip_rel_ptr(_end) - p2v_offset;
 +
 +    /* Include the SME encryption mask in the fixup value */
 +    load_delta += sme_get_me_mask();
 +
 +    /* Fixup the physical addresses in the page table */
 +
-+    pgd = &early_top_pgt[0].pgd;
++    pgd = rip_rel_ptr(early_top_pgt);
 +    pgd[pgd_index(__START_KERNEL_map)] += load_delta;
 +
 +    if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
-+        p4d = (p4dval_t *)level4_kernel_pgt;
++        p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt);
 +        p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
 +
 +        pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
 +    }
 +
-+    level3_kernel_pgt[PTRS_PER_PUD - 2].pud += load_delta;
++    RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 2].pud += load_delta;
-+    level3_kernel_pgt[PTRS_PER_PUD - 1].pud += load_delta;
++    RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 1].pud += load_delta;
 +
 +    for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
-+        level2_fixmap_pgt[i].pmd += load_delta;
++        RIP_REL_REF(level2_fixmap_pgt)[i].pmd += load_delta;
 +
 +    /*
 +     * Set up the identity mapping for the switchover.  These
 +     * entries should *NOT* have the global bit set!  This also
 +     * creates a bunch of nonsense entries but that is fine --
 +     * it avoids problems around wraparound.
 +     */
 +
 +    pud = &early_pgts[0]->pmd;
 +    pmd = &early_pgts[1]->pmd;
-+    next_early_pgt = 2;
++    RIP_REL_REF(next_early_pgt) = 2;
 +
 +    pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
 +
 +    if (la57) {
-+        p4d = &early_pgts[next_early_pgt++]->pmd;
++        p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd;
 +
 +        i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
 +        pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
 +        pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;
 +
 ...
 +    pud[(i + 0) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
 +    pud[(i + 1) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
 +
 +    pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
 +    /* Filter out unsupported __PAGE_KERNEL_* bits: */
-+    pmd_entry &= __supported_pte_mask;
++    pmd_entry &= RIP_REL_REF(__supported_pte_mask);
 +    pmd_entry += sme_get_me_mask();
 +    pmd_entry +=  physaddr;
 +
 +    for (i = 0; i < DIV_ROUND_UP(va_end - va_text, PMD_SIZE); i++) {
 +        int idx = i + (physaddr >> PMD_SHIFT);
 ...
 +     * and on some hardware (particularly the UV platform) even
 +     * speculative access to some reserved areas is caught as an
 +     * error, causing the BIOS to halt the system.
 +     */
 +
-+    pmd = &level2_kernel_pgt[0].pmd;
++    pmd = rip_rel_ptr(level2_kernel_pgt);
 +
 +    /* invalidate pages before the kernel image */
 +    for (i = 0; i < pmd_index(va_text); i++)
 +        pmd[i] &= ~_PAGE_PRESENT;
 +
 ...
  extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
 -static unsigned int __initdata next_early_pgt;
 +unsigned int __initdata next_early_pgt;
  pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
--#ifdef CONFIG_X86_5LEVEL
+ #ifdef CONFIG_X86_5LEVEL
--unsigned int __pgtable_l5_enabled __ro_after_init;
+@@ -XXX,XX +XXX,XX @@ unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4;
--unsigned int pgdir_shift __ro_after_init = 39;
+ EXPORT_SYMBOL(vmemmap_base);
--EXPORT_SYMBOL(pgdir_shift);
+ #endif
--unsigned int ptrs_per_p4d __ro_after_init = 1;
 -EXPORT_SYMBOL(ptrs_per_p4d);
 -#endif
 -
 -#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT
 -unsigned long page_offset_base __ro_after_init = __PAGE_OFFSET_BASE_L4;
 -EXPORT_SYMBOL(page_offset_base);
 -unsigned long vmalloc_base __ro_after_init = __VMALLOC_BASE_L4;
 -EXPORT_SYMBOL(vmalloc_base);
 -unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4;
 -EXPORT_SYMBOL(vmemmap_base);
 -#endif
 -
 -static inline bool check_la57_support(void)
 -{
 -    if (!IS_ENABLED(CONFIG_X86_5LEVEL))
 -        return false;
 -
 ...
 -     * The bss section will be memset to zero later in the initialization so
 -     * there is no need to zero it after changing the memory encryption
 -     * attribute.
 -     */
 -    if (sme_get_me_mask()) {
--        paddr = (unsigned long)&RIP_REL_REF(__start_bss_decrypted);
+-        paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted);
--        paddr_end = (unsigned long)&RIP_REL_REF(__end_bss_decrypted);
+-        paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted);
 -
 -        for (; paddr < paddr_end; paddr += PMD_SIZE) {
 -            /*
 -             * On SNP, transition the page to shared in the RMP table so that
 -             * it is consistent with the page table attribute change.
 ...
 - * by subtracting p2v_offset from the RIP-relative address.
 - */
 -unsigned long __head __startup_64(unsigned long p2v_offset,
 -                  struct boot_params *bp)
 -{
--    pmd_t (*early_pgts)[PTRS_PER_PMD] = RIP_REL_REF(early_dynamic_pgts);
+-    pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);
--    unsigned long physaddr = (unsigned long)&RIP_REL_REF(_text);
+-    unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);
 -    unsigned long va_text, va_end;
 -    unsigned long pgtable_flags;
 -    unsigned long load_delta;
 -    pgdval_t *pgd;
 -    p4dval_t *p4d;
 ...
 -    /* Is the address not 2M aligned? */
 -    if (load_delta & ~PMD_MASK)
 -        for (;;);
 -
 -    va_text = physaddr - p2v_offset;
--    va_end  = (unsigned long)&RIP_REL_REF(_end) - p2v_offset;
+-    va_end  = (unsigned long)rip_rel_ptr(_end) - p2v_offset;
 -
 -    /* Include the SME encryption mask in the fixup value */
 -    load_delta += sme_get_me_mask();
 -
 -    /* Fixup the physical addresses in the page table */
 -
--    pgd = &RIP_REL_REF(early_top_pgt)->pgd;
+-    pgd = rip_rel_ptr(early_top_pgt);
 -    pgd[pgd_index(__START_KERNEL_map)] += load_delta;
 -
 -    if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
--        p4d = (p4dval_t *)&RIP_REL_REF(level4_kernel_pgt);
+-        p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt);
 -        p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
 -
 -        pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
 -    }
 -
 ...
 -     * and on some hardware (particularly the UV platform) even
 -     * speculative access to some reserved areas is caught as an
 -     * error, causing the BIOS to halt the system.
 -     */
 -
--    pmd = &RIP_REL_REF(level2_kernel_pgt)->pmd;
+-    pmd = rip_rel_ptr(level2_kernel_pgt);
 -
 -    /* invalidate pages before the kernel image */
 -    for (i = 0; i < pmd_index(va_text); i++)
 -        pmd[i] &= ~_PAGE_PRESENT;
 -
 ...
 -
  /* Wipe all early page tables except for the kernel symbol map */
  static void __init reset_early_page_tables(void)
  {
 --
-.49.0.472.ge94155a9ec-goog
+.49.0.504.g3bcea36a83-goog

-New patch
+[PATCH v4 04/11] x86/boot: Drop RIP_REL_REF() uses from early mapping code
+From: Ard Biesheuvel <ardb@kernel.org>
+Now that __startup_64() is built using -fPIC, RIP_REL_REF() has become a
+NOP and can be removed. Only some occurrences of rip_rel_ptr() will
+remain, to explicitly take the address of certain global structures in
+the 1:1 mapping of memory.
+While at it, update the code comment to describe why this is needed.
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+---
+ arch/x86/boot/startup/map_kernel.c | 41 ++++++++++----------
+file changed, 21 insertions(+), 20 deletions(-)
+diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c
+index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/boot/startup/map_kernel.c
++++ b/arch/x86/boot/startup/map_kernel.c
+@@ -XXX,XX +XXX,XX @@ static inline bool check_la57_support(void)
+     if (!(native_read_cr4() & X86_CR4_LA57))
+         return false;
+-    RIP_REL_REF(__pgtable_l5_enabled)    = 1;
+-    RIP_REL_REF(pgdir_shift)        = 48;
+-    RIP_REL_REF(ptrs_per_p4d)        = 512;
+-    RIP_REL_REF(page_offset_base)        = __PAGE_OFFSET_BASE_L5;
+-    RIP_REL_REF(vmalloc_base)        = __VMALLOC_BASE_L5;
+-    RIP_REL_REF(vmemmap_base)        = __VMEMMAP_BASE_L5;
++    __pgtable_l5_enabled    = 1;
++    pgdir_shift        = 48;
++    ptrs_per_p4d        = 512;
++    page_offset_base    = __PAGE_OFFSET_BASE_L5;
++    vmalloc_base        = __VMALLOC_BASE_L5;
++    vmemmap_base        = __VMEMMAP_BASE_L5;
+     return true;
+ }
+@@ -XXX,XX +XXX,XX @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
+     return sme_get_me_mask();
+ }
+-/* Code in __startup_64() can be relocated during execution, but the compiler
+- * doesn't have to generate PC-relative relocations when accessing globals from
+- * that function. Clang actually does not generate them, which leads to
+- * boot-time crashes. To work around this problem, every global pointer must
+- * be accessed using RIP_REL_REF(). Kernel virtual addresses can be determined
+- * by subtracting p2v_offset from the RIP-relative address.
++/*
++ * This code is compiled using PIC codegen because it will execute from the
++ * early 1:1 mapping of memory, which deviates from the mapping expected by the
++ * linker. Due to this deviation, taking the address of a global variable will
++ * produce an ambiguous result when using the plain & operator.  Instead,
++ * rip_rel_ptr() must be used, which will return the RIP-relative address in
++ * the 1:1 mapping of memory. Kernel virtual addresses can be determined by
++ * subtracting p2v_offset from the RIP-relative address.
+  */
+ unsigned long __head __startup_64(unsigned long p2v_offset,
+                   struct boot_params *bp)
+@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
+      * Compute the delta between the address I am compiled to run at
+      * and the address I am actually running at.
+      */
+-    load_delta = __START_KERNEL_map + p2v_offset;
+-    RIP_REL_REF(phys_base) = load_delta;
++    phys_base = load_delta = __START_KERNEL_map + p2v_offset;
+     /* Is the address not 2M aligned? */
+     if (load_delta & ~PMD_MASK)
+@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
+         pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
+     }
+-    RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 2].pud += load_delta;
+-    RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 1].pud += load_delta;
++    level3_kernel_pgt[PTRS_PER_PUD - 2].pud += load_delta;
++    level3_kernel_pgt[PTRS_PER_PUD - 1].pud += load_delta;
+     for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
+-        RIP_REL_REF(level2_fixmap_pgt)[i].pmd += load_delta;
++        level2_fixmap_pgt[i].pmd += load_delta;
+     /*
+      * Set up the identity mapping for the switchover.  These
+@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
+     pud = &early_pgts[0]->pmd;
+     pmd = &early_pgts[1]->pmd;
+-    RIP_REL_REF(next_early_pgt) = 2;
++    next_early_pgt = 2;
+     pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
+     if (la57) {
+-        p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd;
++        p4d = &early_pgts[next_early_pgt++]->pmd;
+         i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
+         pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
+@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
+     pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
+     /* Filter out unsupported __PAGE_KERNEL_* bits: */
+-    pmd_entry &= RIP_REL_REF(__supported_pte_mask);
++    pmd_entry &= __supported_pte_mask;
+     pmd_entry += sme_get_me_mask();
+     pmd_entry +=  physaddr;
+--
+.49.0.504.g3bcea36a83-goog

-[RFC PATCH 6/6] x86/boot: Move early SME init code into startup/
+[PATCH v4 05/11] x86/boot: Move early SME init code into startup/
 ...
 Move the SME initialization code, which runs from the 1:1 mapping of
 memory as it operates on the kernel virtual mapping, into the new
 sub-directory arch/x86/boot/startup/ where all startup code will reside
 that needs to tolerate executing from the 1:1 mapping.
-This allows RIP_REL_REF() macro invocations and __head annotations to be
-dropped.
 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
 ---
- arch/x86/boot/startup/Makefile                             |  1 +
+ arch/x86/boot/startup/Makefile                             | 1 +
- arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} | 45 +++++++++-----------
+ arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} | 2 --
- arch/x86/include/asm/mem_encrypt.h                         |  2 +-
+ arch/x86/mm/Makefile                                       | 6 ------
- arch/x86/mm/Makefile                                       |  6 ---
+files changed, 1 insertion(+), 8 deletions(-)
 files changed, 23 insertions(+), 31 deletions(-)
 diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/boot/startup/Makefile
 +++ b/arch/x86/boot/startup/Makefile
 ...
 +obj-$(CONFIG_AMD_MEM_ENCRYPT)    += sme.o
  lib-$(CONFIG_X86_64)        += la57toggle.o
  lib-$(CONFIG_EFI_MIXED)        += efi-mixed.o
 diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/boot/startup/sme.c
-similarity index 92%
+similarity index 99%
 rename from arch/x86/mm/mem_encrypt_identity.c
 rename to arch/x86/boot/startup/sme.c
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/mm/mem_encrypt_identity.c
 +++ b/arch/x86/boot/startup/sme.c
 ...
 -#include "mm_internal.h"
 -
  #define PGD_FLAGS        _KERNPG_TABLE_NOENC
  #define P4D_FLAGS        _KERNPG_TABLE_NOENC
  #define PUD_FLAGS        _KERNPG_TABLE_NOENC
-@@ -XXX,XX +XXX,XX @@ struct sme_populate_pgd_data {
-  */
- static char sme_workarea[2 * PMD_SIZE] __section(".init.scratch");
--static void __head sme_clear_pgd(struct sme_populate_pgd_data *ppd)
-+static void __init sme_clear_pgd(struct sme_populate_pgd_data *ppd)
- {
-     unsigned long pgd_start, pgd_end, pgd_size;
-     pgd_t *pgd_p;
-@@ -XXX,XX +XXX,XX @@ static void __head sme_clear_pgd(struct sme_populate_pgd_data *ppd)
-     memset(pgd_p, 0, pgd_size);
- }
--static pud_t __head *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
-+static pud_t __init *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
- {
-     pgd_t *pgd;
-     p4d_t *p4d;
-@@ -XXX,XX +XXX,XX @@ static pud_t __head *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
-     return pud;
- }
--static void __head sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
-+static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
- {
-     pud_t *pud;
-     pmd_t *pmd;
-@@ -XXX,XX +XXX,XX @@ static void __head sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
-     set_pmd(pmd, __pmd(ppd->paddr | ppd->pmd_flags));
- }
--static void __head sme_populate_pgd(struct sme_populate_pgd_data *ppd)
-+static void __init sme_populate_pgd(struct sme_populate_pgd_data *ppd)
- {
-     pud_t *pud;
-     pmd_t *pmd;
-@@ -XXX,XX +XXX,XX @@ static void __head sme_populate_pgd(struct sme_populate_pgd_data *ppd)
-         set_pte(pte, __pte(ppd->paddr | ppd->pte_flags));
- }
--static void __head __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
-+static void __init __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
- {
-     while (ppd->vaddr < ppd->vaddr_end) {
-         sme_populate_pgd_large(ppd);
-@@ -XXX,XX +XXX,XX @@ static void __head __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
-     }
- }
--static void __head __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
-+static void __init __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
- {
-     while (ppd->vaddr < ppd->vaddr_end) {
-         sme_populate_pgd(ppd);
-@@ -XXX,XX +XXX,XX @@ static void __head __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
-     }
- }
--static void __head __sme_map_range(struct sme_populate_pgd_data *ppd,
-+static void __init __sme_map_range(struct sme_populate_pgd_data *ppd,
-                    pmdval_t pmd_flags, pteval_t pte_flags)
- {
-     unsigned long vaddr_end;
-@@ -XXX,XX +XXX,XX @@ static void __head __sme_map_range(struct sme_populate_pgd_data *ppd,
-     __sme_map_range_pte(ppd);
- }
--static void __head sme_map_range_encrypted(struct sme_populate_pgd_data *ppd)
-+static void __init sme_map_range_encrypted(struct sme_populate_pgd_data *ppd)
- {
-     __sme_map_range(ppd, PMD_FLAGS_ENC, PTE_FLAGS_ENC);
- }
--static void __head sme_map_range_decrypted(struct sme_populate_pgd_data *ppd)
-+static void __init sme_map_range_decrypted(struct sme_populate_pgd_data *ppd)
- {
-     __sme_map_range(ppd, PMD_FLAGS_DEC, PTE_FLAGS_DEC);
- }
--static void __head sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd)
-+static void __init sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd)
- {
-     __sme_map_range(ppd, PMD_FLAGS_DEC_WP, PTE_FLAGS_DEC_WP);
- }
--static unsigned long __head sme_pgtable_calc(unsigned long len)
-+static unsigned long __init sme_pgtable_calc(unsigned long len)
- {
-     unsigned long entries = 0, tables = 0;
-@@ -XXX,XX +XXX,XX @@ static unsigned long __head sme_pgtable_calc(unsigned long len)
-     return entries + tables;
- }
--void __head sme_encrypt_kernel(struct boot_params *bp)
-+void __init sme_encrypt_kernel(struct boot_params *bp)
- {
-     unsigned long workarea_start, workarea_end, workarea_len;
-     unsigned long execute_start, execute_end, execute_len;
-@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
-      * instrumentation or checking boot_cpu_data in the cc_platform_has()
-      * function.
-      */
--    if (!sme_get_me_mask() ||
--        RIP_REL_REF(sev_status) & MSR_AMD64_SEV_ENABLED)
-+    if (!sme_get_me_mask() || sev_status & MSR_AMD64_SEV_ENABLED)
-         return;
-     /*
-@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
-      *     memory from being cached.
-      */
--    kernel_start = (unsigned long)RIP_REL_REF(_text);
--    kernel_end = ALIGN((unsigned long)RIP_REL_REF(_end), PMD_SIZE);
-+    kernel_start = (unsigned long)_text;
-+    kernel_end = ALIGN((unsigned long)_end, PMD_SIZE);
-     kernel_len = kernel_end - kernel_start;
-     initrd_start = 0;
-@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
-      *   pagetable structures for the encryption of the kernel
-      *   pagetable structures for workarea (in case not currently mapped)
-      */
--    execute_start = workarea_start = (unsigned long)RIP_REL_REF(sme_workarea);
-+    execute_start = workarea_start = (unsigned long)sme_workarea;
-     execute_end = execute_start + (PAGE_SIZE * 2) + PMD_SIZE;
-     execute_len = execute_end - execute_start;
-@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
-     native_write_cr3(__native_read_cr3());
- }
--void __head sme_enable(struct boot_params *bp)
-+void __init sme_enable(struct boot_params *bp)
- {
-     unsigned int eax, ebx, ecx, edx;
-     unsigned long feature_mask;
-@@ -XXX,XX +XXX,XX @@ void __head sme_enable(struct boot_params *bp)
-     me_mask = 1UL << (ebx & 0x3f);
-     /* Check the SEV MSR whether SEV or SME is enabled */
--    RIP_REL_REF(sev_status) = msr = __rdmsr(MSR_AMD64_SEV);
-+    sev_status = msr = __rdmsr(MSR_AMD64_SEV);
-     feature_mask = (msr & MSR_AMD64_SEV_ENABLED) ? AMD_SEV_BIT : AMD_SME_BIT;
-     /*
-@@ -XXX,XX +XXX,XX @@ void __head sme_enable(struct boot_params *bp)
-             return;
-     }
--    RIP_REL_REF(sme_me_mask) = me_mask;
--    RIP_REL_REF(physical_mask) &= ~me_mask;
--    RIP_REL_REF(cc_vendor) = CC_VENDOR_AMD;
-+    sme_me_mask    = me_mask;
-+    physical_mask    &= ~me_mask;
-+    cc_vendor    = CC_VENDOR_AMD;
-     cc_set_mask(me_mask);
- }
-diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h
-index XXXXXXX..XXXXXXX 100644
---- a/arch/x86/include/asm/mem_encrypt.h
-+++ b/arch/x86/include/asm/mem_encrypt.h
-@@ -XXX,XX +XXX,XX @@ void __init sev_es_init_vc_handling(void);
- static inline u64 sme_get_me_mask(void)
- {
--    return RIP_REL_REF(sme_me_mask);
-+    return sme_me_mask;
- }
- #define __bss_decrypted __section(".bss..decrypted")
 diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/mm/Makefile
 +++ b/arch/x86/mm/Makefile
 @@ -XXX,XX +XXX,XX @@
 ...
  obj-$(CONFIG_AMD_MEM_ENCRYPT)    += mem_encrypt_amd.o
 -obj-$(CONFIG_AMD_MEM_ENCRYPT)    += mem_encrypt_identity.o
  obj-$(CONFIG_AMD_MEM_ENCRYPT)    += mem_encrypt_boot.o
 --
-.49.0.472.ge94155a9ec-goog
+.49.0.504.g3bcea36a83-goog

-New patch
+[PATCH v4 06/11] x86/boot: Drop RIP_REL_REF() uses from SME startup code
+From: Ard Biesheuvel <ardb@kernel.org>
+RIP_REL_REF() has no effect on code residing in arch/x86/boot/startup,
+as it is built with -fPIC. So remove any occurrences from the SME
+startup code.
+Note the SME is the only caller of cc_set_mask() that requires this, so
+drop it from there as well.
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+---
+ arch/x86/boot/startup/sme.c        | 11 +++++------
+ arch/x86/include/asm/coco.h        |  2 +-
+ arch/x86/include/asm/mem_encrypt.h |  2 +-
+files changed, 7 insertions(+), 8 deletions(-)
+diff --git a/arch/x86/boot/startup/sme.c b/arch/x86/boot/startup/sme.c
+index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/boot/startup/sme.c
++++ b/arch/x86/boot/startup/sme.c
+@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
+      * instrumentation or checking boot_cpu_data in the cc_platform_has()
+      * function.
+      */
+-    if (!sme_get_me_mask() ||
+-        RIP_REL_REF(sev_status) & MSR_AMD64_SEV_ENABLED)
++    if (!sme_get_me_mask() || sev_status & MSR_AMD64_SEV_ENABLED)
+         return;
+     /*
+@@ -XXX,XX +XXX,XX @@ void __head sme_enable(struct boot_params *bp)
+     me_mask = 1UL << (ebx & 0x3f);
+     /* Check the SEV MSR whether SEV or SME is enabled */
+-    RIP_REL_REF(sev_status) = msr = __rdmsr(MSR_AMD64_SEV);
++    sev_status = msr = __rdmsr(MSR_AMD64_SEV);
+     feature_mask = (msr & MSR_AMD64_SEV_ENABLED) ? AMD_SEV_BIT : AMD_SME_BIT;
+     /*
+@@ -XXX,XX +XXX,XX @@ void __head sme_enable(struct boot_params *bp)
+             return;
+     }
+-    RIP_REL_REF(sme_me_mask) = me_mask;
+-    RIP_REL_REF(physical_mask) &= ~me_mask;
+-    RIP_REL_REF(cc_vendor) = CC_VENDOR_AMD;
++    sme_me_mask    = me_mask;
++    physical_mask    &= ~me_mask;
++    cc_vendor    = CC_VENDOR_AMD;
+     cc_set_mask(me_mask);
+ }
+diff --git a/arch/x86/include/asm/coco.h b/arch/x86/include/asm/coco.h
+index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/include/asm/coco.h
++++ b/arch/x86/include/asm/coco.h
+@@ -XXX,XX +XXX,XX @@ static inline u64 cc_get_mask(void)
+ static inline void cc_set_mask(u64 mask)
+ {
+-    RIP_REL_REF(cc_mask) = mask;
++    cc_mask = mask;
+ }
+ u64 cc_mkenc(u64 val);
+diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h
+index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/include/asm/mem_encrypt.h
++++ b/arch/x86/include/asm/mem_encrypt.h
+@@ -XXX,XX +XXX,XX @@ void __init sev_es_init_vc_handling(void);
+ static inline u64 sme_get_me_mask(void)
+ {
+-    return RIP_REL_REF(sme_me_mask);
++    return sme_me_mask;
+ }
+ #define __bss_decrypted __section(".bss..decrypted")
+--
+.49.0.504.g3bcea36a83-goog

-New patch
+[PATCH v4 07/11] x86/sev: Prepare for splitting off early SEV code
+From: Ard Biesheuvel <ardb@kernel.org>
+Prepare for splitting off parts of the SEV core.c source file into a
+file that carries code that must tolerate being called from the early
+:1 mapping. This will allow special build-time handling of thise code,
+to ensure that it gets generated in a way that is compatible with the
+early execution context.
+So create a de-facto internal SEV API and put the definitions into
+sev-internal.h. No attempt is made to allow this header file to be
+included in arbitrary other sources - this is explicitly not the intent.
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+---
+ arch/x86/boot/compressed/sev.c      |  15 ++-
+ arch/x86/coco/sev/core.c            | 108 +++--------------
+ arch/x86/coco/sev/shared.c          |  64 ++--------
+ arch/x86/include/asm/sev-internal.h | 122 ++++++++++++++++++++
+ arch/x86/include/asm/sev.h          |  37 ++++++
+files changed, 194 insertions(+), 152 deletions(-)
+diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c
+index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/boot/compressed/sev.c
++++ b/arch/x86/boot/compressed/sev.c
+@@ -XXX,XX +XXX,XX @@ static bool fault_in_kernel_space(unsigned long address)
+ #include "../../lib/inat.c"
+ #include "../../lib/insn.c"
+-/* Include code for early handlers */
+-#include "../../coco/sev/shared.c"
++extern struct svsm_ca *boot_svsm_caa;
++extern u64 boot_svsm_caa_pa;
+-static struct svsm_ca *svsm_get_caa(void)
++struct svsm_ca *svsm_get_caa(void)
+ {
+     return boot_svsm_caa;
+ }
+-static u64 svsm_get_caa_pa(void)
++u64 svsm_get_caa_pa(void)
+ {
+     return boot_svsm_caa_pa;
+ }
+-static int svsm_perform_call_protocol(struct svsm_call *call)
++int svsm_perform_call_protocol(struct svsm_call *call);
++
++/* Include code for early handlers */
++#include "../../coco/sev/shared.c"
++
++int svsm_perform_call_protocol(struct svsm_call *call)
+ {
+     struct ghcb *ghcb;
+     int ret;
+diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
+index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/coco/sev/core.c
++++ b/arch/x86/coco/sev/core.c
+@@ -XXX,XX +XXX,XX @@
+ #include <asm/cpu_entry_area.h>
+ #include <asm/stacktrace.h>
+ #include <asm/sev.h>
++#include <asm/sev-internal.h>
+ #include <asm/insn-eval.h>
+ #include <asm/fpu/xcr.h>
+ #include <asm/processor.h>
+@@ -XXX,XX +XXX,XX @@
+ #include <asm/cpuid.h>
+ #include <asm/cmdline.h>
+-#define DR7_RESET_VALUE        0x400
+-
+ /* AP INIT values as documented in the APM2  section "Processor Initialization State" */
+ #define AP_INIT_CS_LIMIT        0xffff
+ #define AP_INIT_DS_LIMIT        0xffff
+@@ -XXX,XX +XXX,XX @@ static const char * const sev_status_feat_names[] = {
+ };
+ /* For early boot hypervisor communication in SEV-ES enabled guests */
+-static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
++struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
+ /*
+  * Needs to be in the .data section because we need it NULL before bss is
+  * cleared
+  */
+-static struct ghcb *boot_ghcb __section(".data");
++struct ghcb *boot_ghcb __section(".data");
+ /* Bitmap of SEV features supported by the hypervisor */
+-static u64 sev_hv_features __ro_after_init;
++u64 sev_hv_features __ro_after_init;
+ /* Secrets page physical address from the CC blob */
+ static u64 secrets_pa __ro_after_init;
+@@ -XXX,XX +XXX,XX @@ static u64 snp_tsc_scale __ro_after_init;
+ static u64 snp_tsc_offset __ro_after_init;
+ static u64 snp_tsc_freq_khz __ro_after_init;
+-/* #VC handler runtime per-CPU data */
+-struct sev_es_runtime_data {
+-    struct ghcb ghcb_page;
+-
+-    /*
+-     * Reserve one page per CPU as backup storage for the unencrypted GHCB.
+-     * It is needed when an NMI happens while the #VC handler uses the real
+-     * GHCB, and the NMI handler itself is causing another #VC exception. In
+-     * that case the GHCB content of the first handler needs to be backed up
+-     * and restored.
+-     */
+-    struct ghcb backup_ghcb;
+-
+-    /*
+-     * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions.
+-     * There is no need for it to be atomic, because nothing is written to
+-     * the GHCB between the read and the write of ghcb_active. So it is safe
+-     * to use it when a nested #VC exception happens before the write.
+-     *
+-     * This is necessary for example in the #VC->NMI->#VC case when the NMI
+-     * happens while the first #VC handler uses the GHCB. When the NMI code
+-     * raises a second #VC handler it might overwrite the contents of the
+-     * GHCB written by the first handler. To avoid this the content of the
+-     * GHCB is saved and restored when the GHCB is detected to be in use
+-     * already.
+-     */
+-    bool ghcb_active;
+-    bool backup_ghcb_active;
+-
+-    /*
+-     * Cached DR7 value - write it on DR7 writes and return it on reads.
+-     * That value will never make it to the real hardware DR7 as debugging
+-     * is currently unsupported in SEV-ES guests.
+-     */
+-    unsigned long dr7;
+-};
+-
+-struct ghcb_state {
+-    struct ghcb *ghcb;
+-};
+ /* For early boot SVSM communication */
+-static struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE);
++struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE);
+-static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
+-static DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
+-static DEFINE_PER_CPU(struct svsm_ca *, svsm_caa);
+-static DEFINE_PER_CPU(u64, svsm_caa_pa);
++DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
++DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
++DEFINE_PER_CPU(struct svsm_ca *, svsm_caa);
++DEFINE_PER_CPU(u64, svsm_caa_pa);
+ static __always_inline bool on_vc_stack(struct pt_regs *regs)
+ {
+@@ -XXX,XX +XXX,XX @@ void noinstr __sev_es_ist_exit(void)
+  *
+  * Callers must disable local interrupts around it.
+  */
+-static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
++noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
+ {
+     struct sev_es_runtime_data *data;
+     struct ghcb *ghcb;
+@@ -XXX,XX +XXX,XX @@ static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
+     return ghcb;
+ }
+-static inline u64 sev_es_rd_ghcb_msr(void)
+-{
+-    return __rdmsr(MSR_AMD64_SEV_ES_GHCB);
+-}
+-
+-static __always_inline void sev_es_wr_ghcb_msr(u64 val)
+-{
+-    u32 low, high;
+-
+-    low  = (u32)(val);
+-    high = (u32)(val >> 32);
+-
+-    native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high);
+-}
+-
+ static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt,
+                 unsigned char *buffer)
+ {
+@@ -XXX,XX +XXX,XX @@ static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt)
+ /* Include code shared with pre-decompression boot stage */
+ #include "shared.c"
+-static inline struct svsm_ca *svsm_get_caa(void)
+-{
+-    /*
+-     * Use rIP-relative references when called early in the boot. If
+-     * ->use_cas is set, then it is late in the boot and no need
+-     * to worry about rIP-relative references.
+-     */
+-    if (RIP_REL_REF(sev_cfg).use_cas)
+-        return this_cpu_read(svsm_caa);
+-    else
+-        return RIP_REL_REF(boot_svsm_caa);
+-}
+-
+-static u64 svsm_get_caa_pa(void)
+-{
+-    /*
+-     * Use rIP-relative references when called early in the boot. If
+-     * ->use_cas is set, then it is late in the boot and no need
+-     * to worry about rIP-relative references.
+-     */
+-    if (RIP_REL_REF(sev_cfg).use_cas)
+-        return this_cpu_read(svsm_caa_pa);
+-    else
+-        return RIP_REL_REF(boot_svsm_caa_pa);
+-}
+-
+-static noinstr void __sev_put_ghcb(struct ghcb_state *state)
++noinstr void __sev_put_ghcb(struct ghcb_state *state)
+ {
+     struct sev_es_runtime_data *data;
+     struct ghcb *ghcb;
+@@ -XXX,XX +XXX,XX @@ static noinstr void __sev_put_ghcb(struct ghcb_state *state)
+     }
+ }
+-static int svsm_perform_call_protocol(struct svsm_call *call)
++int svsm_perform_call_protocol(struct svsm_call *call)
+ {
+     struct ghcb_state state;
+     unsigned long flags;
+@@ -XXX,XX +XXX,XX @@ static u64 __init get_jump_table_addr(void)
+     return ret;
+ }
+-static void __head
++void __head
+ early_set_pages_state(unsigned long vaddr, unsigned long paddr,
+               unsigned long npages, enum psc_op op)
+ {
+diff --git a/arch/x86/coco/sev/shared.c b/arch/x86/coco/sev/shared.c
+index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/coco/sev/shared.c
++++ b/arch/x86/coco/sev/shared.c
+@@ -XXX,XX +XXX,XX @@
+  */
+ u8 snp_vmpl __ro_after_init;
+ EXPORT_SYMBOL_GPL(snp_vmpl);
+-static struct svsm_ca *boot_svsm_caa __ro_after_init;
+-static u64 boot_svsm_caa_pa __ro_after_init;
+-
+-static struct svsm_ca *svsm_get_caa(void);
+-static u64 svsm_get_caa_pa(void);
+-static int svsm_perform_call_protocol(struct svsm_call *call);
++struct svsm_ca *boot_svsm_caa __ro_after_init;
++u64 boot_svsm_caa_pa __ro_after_init;
+ /* I/O parameters for CPUID-related helpers */
+ struct cpuid_leaf {
+@@ -XXX,XX +XXX,XX @@ struct cpuid_leaf {
+     u32 edx;
+ };
+-/*
+- * Individual entries of the SNP CPUID table, as defined by the SNP
+- * Firmware ABI, Revision 0.9, Section 7.1, Table 14.
+- */
+-struct snp_cpuid_fn {
+-    u32 eax_in;
+-    u32 ecx_in;
+-    u64 xcr0_in;
+-    u64 xss_in;
+-    u32 eax;
+-    u32 ebx;
+-    u32 ecx;
+-    u32 edx;
+-    u64 __reserved;
+-} __packed;
+-
+-/*
+- * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,
+- * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit
+- * of 64 entries per CPUID table.
+- */
+-#define SNP_CPUID_COUNT_MAX 64
+-
+-struct snp_cpuid_table {
+-    u32 count;
+-    u32 __reserved1;
+-    u64 __reserved2;
+-    struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];
+-} __packed;
+-
+ /*
+  * Since feature negotiation related variables are set early in the boot
+  * process they must reside in the .data section so as not to be zeroed
+@@ -XXX,XX +XXX,XX @@ static u32 cpuid_std_range_max __ro_after_init;
+ static u32 cpuid_hyp_range_max __ro_after_init;
+ static u32 cpuid_ext_range_max __ro_after_init;
+-static bool __init sev_es_check_cpu_features(void)
++bool __init sev_es_check_cpu_features(void)
+ {
+     if (!has_cpuflag(X86_FEATURE_RDRAND)) {
+         error("RDRAND instruction not supported - no trusted source of randomness available\n");
+@@ -XXX,XX +XXX,XX @@ static bool __init sev_es_check_cpu_features(void)
+     return true;
+ }
+-static void __head __noreturn
++void __head __noreturn
+ sev_es_terminate(unsigned int set, unsigned int reason)
+ {
+     u64 val = GHCB_MSR_TERM_REQ;
+@@ -XXX,XX +XXX,XX @@ sev_es_terminate(unsigned int set, unsigned int reason)
+ /*
+  * The hypervisor features are available from GHCB version 2 onward.
+  */
+-static u64 get_hv_features(void)
++u64 get_hv_features(void)
+ {
+     u64 val;
+@@ -XXX,XX +XXX,XX @@ static u64 get_hv_features(void)
+     return GHCB_MSR_HV_FT_RESP_VAL(val);
+ }
+-static void snp_register_ghcb_early(unsigned long paddr)
++void snp_register_ghcb_early(unsigned long paddr)
+ {
+     unsigned long pfn = paddr >> PAGE_SHIFT;
+     u64 val;
+@@ -XXX,XX +XXX,XX @@ static void snp_register_ghcb_early(unsigned long paddr)
+         sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER);
+ }
+-static bool sev_es_negotiate_protocol(void)
++bool sev_es_negotiate_protocol(void)
+ {
+     u64 val;
+@@ -XXX,XX +XXX,XX @@ static bool sev_es_negotiate_protocol(void)
+     return true;
+ }
+-static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
+-{
+-    ghcb->save.sw_exit_code = 0;
+-    __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
+-}
+-
+ static bool vc_decoding_needed(unsigned long exit_code)
+ {
+     /* Exceptions don't require to decode the instruction */
+@@ -XXX,XX +XXX,XX @@ static int svsm_perform_ghcb_protocol(struct ghcb *ghcb, struct svsm_call *call)
+     return svsm_process_result_codes(call);
+ }
+-static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
+-                      struct es_em_ctxt *ctxt,
+-                      u64 exit_code, u64 exit_info_1,
+-                      u64 exit_info_2)
++enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
++                   struct es_em_ctxt *ctxt,
++                   u64 exit_code, u64 exit_info_1,
++                   u64 exit_info_2)
+ {
+     /* Fill in protocol and format specifiers */
+     ghcb->protocol_version = ghcb_version;
+@@ -XXX,XX +XXX,XX @@ static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid
+  * while running with the initial identity mapping as well as the
+  * switch-over to kernel virtual addresses later.
+  */
+-static const struct snp_cpuid_table *snp_cpuid_get_table(void)
++const struct snp_cpuid_table *snp_cpuid_get_table(void)
+ {
+     return rip_rel_ptr(&cpuid_table_copy);
+ }
+diff --git a/arch/x86/include/asm/sev-internal.h b/arch/x86/include/asm/sev-internal.h
+new file mode 100644
+index XXXXXXX..XXXXXXX
+--- /dev/null
++++ b/arch/x86/include/asm/sev-internal.h
+@@ -XXX,XX +XXX,XX @@
++/* SPDX-License-Identifier: GPL-2.0 */
++
++#define DR7_RESET_VALUE        0x400
++
++extern struct ghcb boot_ghcb_page;
++extern struct ghcb *boot_ghcb;
++extern u64 sev_hv_features;
++
++/* #VC handler runtime per-CPU data */
++struct sev_es_runtime_data {
++    struct ghcb ghcb_page;
++
++    /*
++     * Reserve one page per CPU as backup storage for the unencrypted GHCB.
++     * It is needed when an NMI happens while the #VC handler uses the real
++     * GHCB, and the NMI handler itself is causing another #VC exception. In
++     * that case the GHCB content of the first handler needs to be backed up
++     * and restored.
++     */
++    struct ghcb backup_ghcb;
++
++    /*
++     * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions.
++     * There is no need for it to be atomic, because nothing is written to
++     * the GHCB between the read and the write of ghcb_active. So it is safe
++     * to use it when a nested #VC exception happens before the write.
++     *
++     * This is necessary for example in the #VC->NMI->#VC case when the NMI
++     * happens while the first #VC handler uses the GHCB. When the NMI code
++     * raises a second #VC handler it might overwrite the contents of the
++     * GHCB written by the first handler. To avoid this the content of the
++     * GHCB is saved and restored when the GHCB is detected to be in use
++     * already.
++     */
++    bool ghcb_active;
++    bool backup_ghcb_active;
++
++    /*
++     * Cached DR7 value - write it on DR7 writes and return it on reads.
++     * That value will never make it to the real hardware DR7 as debugging
++     * is currently unsupported in SEV-ES guests.
++     */
++    unsigned long dr7;
++};
++
++struct ghcb_state {
++    struct ghcb *ghcb;
++};
++
++extern struct svsm_ca boot_svsm_ca_page;
++
++struct ghcb *__sev_get_ghcb(struct ghcb_state *state);
++void __sev_put_ghcb(struct ghcb_state *state);
++
++DECLARE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
++DECLARE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
++
++void early_set_pages_state(unsigned long vaddr, unsigned long paddr,
++               unsigned long npages, enum psc_op op);
++
++void __noreturn sev_es_terminate(unsigned int set, unsigned int reason);
++
++DECLARE_PER_CPU(struct svsm_ca *, svsm_caa);
++DECLARE_PER_CPU(u64, svsm_caa_pa);
++
++extern struct svsm_ca *boot_svsm_caa;
++extern u64 boot_svsm_caa_pa;
++
++static __always_inline struct svsm_ca *svsm_get_caa(void)
++{
++    /*
++     * Use rIP-relative references when called early in the boot. If
++     * ->use_cas is set, then it is late in the boot and no need
++     * to worry about rIP-relative references.
++     */
++    if (RIP_REL_REF(sev_cfg).use_cas)
++        return this_cpu_read(svsm_caa);
++    else
++        return RIP_REL_REF(boot_svsm_caa);
++}
++
++static __always_inline u64 svsm_get_caa_pa(void)
++{
++    /*
++     * Use rIP-relative references when called early in the boot. If
++     * ->use_cas is set, then it is late in the boot and no need
++     * to worry about rIP-relative references.
++     */
++    if (RIP_REL_REF(sev_cfg).use_cas)
++        return this_cpu_read(svsm_caa_pa);
++    else
++        return RIP_REL_REF(boot_svsm_caa_pa);
++}
++
++int svsm_perform_call_protocol(struct svsm_call *call);
++
++static inline u64 sev_es_rd_ghcb_msr(void)
++{
++    return __rdmsr(MSR_AMD64_SEV_ES_GHCB);
++}
++
++static __always_inline void sev_es_wr_ghcb_msr(u64 val)
++{
++    u32 low, high;
++
++    low  = (u32)(val);
++    high = (u32)(val >> 32);
++
++    native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high);
++}
++
++enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
++                   struct es_em_ctxt *ctxt,
++                   u64 exit_code, u64 exit_info_1,
++                   u64 exit_info_2);
++
++void snp_register_ghcb_early(unsigned long paddr);
++bool sev_es_negotiate_protocol(void);
++bool sev_es_check_cpu_features(void);
++u64 get_hv_features(void);
++
++const struct snp_cpuid_table *snp_cpuid_get_table(void);
+diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h
+index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/include/asm/sev.h
++++ b/arch/x86/include/asm/sev.h
+@@ -XXX,XX +XXX,XX @@
+ #include <asm/sev-common.h>
+ #include <asm/coco.h>
+ #include <asm/set_memory.h>
++#include <asm/svm.h>
+ #define GHCB_PROTOCOL_MIN    1ULL
+ #define GHCB_PROTOCOL_MAX    2ULL
+@@ -XXX,XX +XXX,XX @@ extern void vc_no_ghcb(void);
+ extern void vc_boot_ghcb(void);
+ extern bool handle_vc_boot_ghcb(struct pt_regs *regs);
++/*
++ * Individual entries of the SNP CPUID table, as defined by the SNP
++ * Firmware ABI, Revision 0.9, Section 7.1, Table 14.
++ */
++struct snp_cpuid_fn {
++    u32 eax_in;
++    u32 ecx_in;
++    u64 xcr0_in;
++    u64 xss_in;
++    u32 eax;
++    u32 ebx;
++    u32 ecx;
++    u32 edx;
++    u64 __reserved;
++} __packed;
++
++/*
++ * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,
++ * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit
++ * of 64 entries per CPUID table.
++ */
++#define SNP_CPUID_COUNT_MAX 64
++
++struct snp_cpuid_table {
++    u32 count;
++    u32 __reserved1;
++    u64 __reserved2;
++    struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];
++} __packed;
++
+ /* PVALIDATE return codes */
+ #define PVALIDATE_FAIL_SIZEMISMATCH    6
+@@ -XXX,XX +XXX,XX @@ int snp_send_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req
+ void __init snp_secure_tsc_prepare(void);
+ void __init snp_secure_tsc_init(void);
++static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
++{
++    ghcb->save.sw_exit_code = 0;
++    __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
++}
++
+ #else    /* !CONFIG_AMD_MEM_ENCRYPT */
+ #define snp_vmpl 0
+--
+.49.0.504.g3bcea36a83-goog

-[RFC PATCH 2/6] x86/boot: Move 5-level paging trampoline into startup code
+[PATCH v4 08/11] x86/sev: Split off startup code from core code
 From: Ard Biesheuvel <ardb@kernel.org>
-The 5-level paging trampoline is used by both the EFI stub and the
+Disentangle the SEV core code and the SEV code that is called during
-traditional decompressor. Move it out of the decompressor sources into
+early boot. The latter piece will be moved into startup/ in a subsequent
-the newly minted arch/x86/boot/startup/ sub-directory which will hold
+patch.
 startup code that may be shared between the decompressor, the EFI stub
 and the kernel proper, and needs to tolerate being called during early
 boot, before the kernel virtual mapping has been created.
 This will allow the 5-level paging trampoline to be used by EFI boot
 images such as zboot that omit the traditional decompressor entirely.
 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
 ---
- arch/x86/Makefile                                  | 1 +
+ arch/x86/boot/compressed/sev.c |    2 +
- arch/x86/boot/compressed/Makefile                  | 2 +-
+ arch/x86/coco/sev/Makefile     |   12 +-
- arch/x86/boot/startup/Makefile                     | 3 +++
+ arch/x86/coco/sev/core.c       | 1574 ++++----------------
- arch/x86/boot/{compressed => startup}/la57toggle.S | 0
+ arch/x86/coco/sev/shared.c     |  281 ----
-files changed, 5 insertions(+), 1 deletion(-)
+ arch/x86/coco/sev/startup.c    | 1395 +++++++++++++++++
 files changed, 1658 insertions(+), 1606 deletions(-)
-diff --git a/arch/x86/Makefile b/arch/x86/Makefile
+diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c
 index XXXXXXX..XXXXXXX 100644
---- a/arch/x86/Makefile
+--- a/arch/x86/boot/compressed/sev.c
-+++ b/arch/x86/Makefile
++++ b/arch/x86/boot/compressed/sev.c
-@@ -XXX,XX +XXX,XX @@ archprepare: $(cpufeaturemasks.hdr)
+@@ -XXX,XX +XXX,XX @@ u64 svsm_get_caa_pa(void)
- ###
- # Kernel objects
+ int svsm_perform_call_protocol(struct svsm_call *call);
-+core-y  += arch/x86/boot/startup/
++u8 snp_vmpl;
- libs-y  += arch/x86/lib/
++
+ /* Include code for early handlers */
- # drivers-y are linked after core-y
+ #include "../../coco/sev/shared.c"
-diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
 diff --git a/arch/x86/coco/sev/Makefile b/arch/x86/coco/sev/Makefile
 index XXXXXXX..XXXXXXX 100644
---- a/arch/x86/boot/compressed/Makefile
+--- a/arch/x86/coco/sev/Makefile
-+++ b/arch/x86/boot/compressed/Makefile
++++ b/arch/x86/coco/sev/Makefile
-@@ -XXX,XX +XXX,XX @@ ifdef CONFIG_X86_64
+@@ -XXX,XX +XXX,XX @@
-     vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/mem_encrypt.o
+ # SPDX-License-Identifier: GPL-2.0
-     vmlinux-objs-y += $(obj)/pgtable_64.o
-     vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev.o
+-obj-y += core.o
--    vmlinux-objs-y += $(obj)/la57toggle.o
++obj-y += core.o startup.o
  # jump tables are emitted using absolute references in non-PIC code
  # so they cannot be used in the early SEV startup code
 -CFLAGS_core.o += -fno-jump-tables
 +CFLAGS_startup.o += -fno-jump-tables
  ifdef CONFIG_FUNCTION_TRACER
 -CFLAGS_REMOVE_core.o = -pg
 +CFLAGS_REMOVE_startup.o = -pg
  endif
- vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o
+-KASAN_SANITIZE_core.o    := n
-@@ -XXX,XX +XXX,XX @@ vmlinux-objs-$(CONFIG_UNACCEPTED_MEMORY) += $(obj)/mem.o
+-KMSAN_SANITIZE_core.o    := n
+-KCOV_INSTRUMENT_core.o    := n
- vmlinux-objs-$(CONFIG_EFI) += $(obj)/efi.o
++KASAN_SANITIZE_startup.o    := n
- vmlinux-libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a
++KMSAN_SANITIZE_startup.o    := n
-+vmlinux-libs-$(CONFIG_X86_64)    += $(objtree)/arch/x86/boot/startup/lib.a
++KCOV_INSTRUMENT_startup.o    := n
- $(obj)/vmlinux: $(vmlinux-objs-y) $(vmlinux-libs-y) FORCE
+ # With some compiler versions the generated code results in boot hangs, caused
-     $(call if_changed,ld)
+ # by several compilation units. To be safe, disable all instrumentation.
-diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
+diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/coco/sev/core.c
 +++ b/arch/x86/coco/sev/core.c
@@ -XXX,XX +XXX,XX @@ static const char * const sev_status_feat_names[] = {
      [MSR_AMD64_SNP_SMT_PROT_BIT]        = "SMTProt",
  };
 -/* For early boot hypervisor communication in SEV-ES enabled guests */
 -struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
 -
 -/*
 - * Needs to be in the .data section because we need it NULL before bss is
 - * cleared
 - */
 -struct ghcb *boot_ghcb __section(".data");
 -
 -/* Bitmap of SEV features supported by the hypervisor */
 -u64 sev_hv_features __ro_after_init;
 -
  /* Secrets page physical address from the CC blob */
  static u64 secrets_pa __ro_after_init;
@@ -XXX,XX +XXX,XX @@ static u64 snp_tsc_scale __ro_after_init;
  static u64 snp_tsc_offset __ro_after_init;
  static u64 snp_tsc_freq_khz __ro_after_init;
 -
 -/* For early boot SVSM communication */
 -struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE);
 -
  DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
  DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
 -DEFINE_PER_CPU(struct svsm_ca *, svsm_caa);
 -DEFINE_PER_CPU(u64, svsm_caa_pa);
 +
 +/*
 + * SVSM related information:
 + *   When running under an SVSM, the VMPL that Linux is executing at must be
 + *   non-zero. The VMPL is therefore used to indicate the presence of an SVSM.
 + */
 +u8 snp_vmpl __ro_after_init;
 +EXPORT_SYMBOL_GPL(snp_vmpl);
  static __always_inline bool on_vc_stack(struct pt_regs *regs)
  {
@@ -XXX,XX +XXX,XX @@ static __always_inline bool on_vc_stack(struct pt_regs *regs)
      return ((sp >= __this_cpu_ist_bottom_va(VC)) && (sp < __this_cpu_ist_top_va(VC)));
  }
 +
  /*
   * This function handles the case when an NMI is raised in the #VC
   * exception handler entry code, before the #VC handler has switched off
@@ -XXX,XX +XXX,XX @@ void noinstr __sev_es_ist_exit(void)
      this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist);
  }
 -/*
 - * Nothing shall interrupt this code path while holding the per-CPU
 - * GHCB. The backup GHCB is only for NMIs interrupting this path.
 - *
 - * Callers must disable local interrupts around it.
 - */
 -noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
 +static u64 __init get_snp_jump_table_addr(void)
  {
 -    struct sev_es_runtime_data *data;
 -    struct ghcb *ghcb;
 -
 -    WARN_ON(!irqs_disabled());
 -
 -    data = this_cpu_read(runtime_data);
 -    ghcb = &data->ghcb_page;
 -
 -    if (unlikely(data->ghcb_active)) {
 -        /* GHCB is already in use - save its contents */
 -
 -        if (unlikely(data->backup_ghcb_active)) {
 -            /*
 -             * Backup-GHCB is also already in use. There is no way
 -             * to continue here so just kill the machine. To make
 -             * panic() work, mark GHCBs inactive so that messages
 -             * can be printed out.
 -             */
 -            data->ghcb_active        = false;
 -            data->backup_ghcb_active = false;
 -
 -            instrumentation_begin();
 -            panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use");
 -            instrumentation_end();
 -        }
 -
 -        /* Mark backup_ghcb active before writing to it */
 -        data->backup_ghcb_active = true;
 -
 -        state->ghcb = &data->backup_ghcb;
 +    struct snp_secrets_page *secrets;
 +    void __iomem *mem;
 +    u64 addr;
 -        /* Backup GHCB content */
 -        *state->ghcb = *ghcb;
 -    } else {
 -        state->ghcb = NULL;
 -        data->ghcb_active = true;
 +    mem = ioremap_encrypted(secrets_pa, PAGE_SIZE);
 +    if (!mem) {
 +        pr_err("Unable to locate AP jump table address: failed to map the SNP secrets page.\n");
 +        return 0;
      }
 -    return ghcb;
 -}
 +    secrets = (__force struct snp_secrets_page *)mem;
 -static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt,
 -                unsigned char *buffer)
 -{
 -    return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
 +    addr = secrets->os_area.ap_jump_table_pa;
 +    iounmap(mem);
 +
 +    return addr;
  }
 -static enum es_result __vc_decode_user_insn(struct es_em_ctxt *ctxt)
 +void noinstr __sev_es_nmi_complete(void)
  {
 -    char buffer[MAX_INSN_SIZE];
 -    int insn_bytes;
 -
 -    insn_bytes = insn_fetch_from_user_inatomic(ctxt->regs, buffer);
 -    if (insn_bytes == 0) {
 -        /* Nothing could be copied */
 -        ctxt->fi.vector     = X86_TRAP_PF;
 -        ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER;
 -        ctxt->fi.cr2        = ctxt->regs->ip;
 -        return ES_EXCEPTION;
 -    } else if (insn_bytes == -EINVAL) {
 -        /* Effective RIP could not be calculated */
 -        ctxt->fi.vector     = X86_TRAP_GP;
 -        ctxt->fi.error_code = 0;
 -        ctxt->fi.cr2        = 0;
 -        return ES_EXCEPTION;
 -    }
 -
 -    if (!insn_decode_from_regs(&ctxt->insn, ctxt->regs, buffer, insn_bytes))
 -        return ES_DECODE_FAILED;
 +    struct ghcb_state state;
 +    struct ghcb *ghcb;
 -    if (ctxt->insn.immediate.got)
 -        return ES_OK;
 -    else
 -        return ES_DECODE_FAILED;
 -}
 +    ghcb = __sev_get_ghcb(&state);
 -static enum es_result __vc_decode_kern_insn(struct es_em_ctxt *ctxt)
 -{
 -    char buffer[MAX_INSN_SIZE];
 -    int res, ret;
 +    vc_ghcb_invalidate(ghcb);
 +    ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE);
 +    ghcb_set_sw_exit_info_1(ghcb, 0);
 +    ghcb_set_sw_exit_info_2(ghcb, 0);
 -    res = vc_fetch_insn_kernel(ctxt, buffer);
 -    if (res) {
 -        ctxt->fi.vector     = X86_TRAP_PF;
 -        ctxt->fi.error_code = X86_PF_INSTR;
 -        ctxt->fi.cr2        = ctxt->regs->ip;
 -        return ES_EXCEPTION;
 -    }
 +    sev_es_wr_ghcb_msr(__pa_nodebug(ghcb));
 +    VMGEXIT();
 -    ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
 -    if (ret < 0)
 -        return ES_DECODE_FAILED;
 -    else
 -        return ES_OK;
 +    __sev_put_ghcb(&state);
  }
 -static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
 +static u64 __init get_jump_table_addr(void)
  {
 -    if (user_mode(ctxt->regs))
 -        return __vc_decode_user_insn(ctxt);
 -    else
 -        return __vc_decode_kern_insn(ctxt);
 -}
 +    struct ghcb_state state;
 +    unsigned long flags;
 +    struct ghcb *ghcb;
 +    u64 ret = 0;
 -static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
 -                   char *dst, char *buf, size_t size)
 -{
 -    unsigned long error_code = X86_PF_PROT | X86_PF_WRITE;
 +    if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
 +        return get_snp_jump_table_addr();
 -    /*
 -     * This function uses __put_user() independent of whether kernel or user
 -     * memory is accessed. This works fine because __put_user() does no
 -     * sanity checks of the pointer being accessed. All that it does is
 -     * to report when the access failed.
 -     *
 -     * Also, this function runs in atomic context, so __put_user() is not
 -     * allowed to sleep. The page-fault handler detects that it is running
 -     * in atomic context and will not try to take mmap_sem and handle the
 -     * fault, so additional pagefault_enable()/disable() calls are not
 -     * needed.
 -     *
 -     * The access can't be done via copy_to_user() here because
 -     * vc_write_mem() must not use string instructions to access unsafe
 -     * memory. The reason is that MOVS is emulated by the #VC handler by
 -     * splitting the move up into a read and a write and taking a nested #VC
 -     * exception on whatever of them is the MMIO access. Using string
 -     * instructions here would cause infinite nesting.
 -     */
 -    switch (size) {
 -    case 1: {
 -        u8 d1;
 -        u8 __user *target = (u8 __user *)dst;
 -
 -        memcpy(&d1, buf, 1);
 -        if (__put_user(d1, target))
 -            goto fault;
 -        break;
 -    }
 -    case 2: {
 -        u16 d2;
 -        u16 __user *target = (u16 __user *)dst;
 +    local_irq_save(flags);
 -        memcpy(&d2, buf, 2);
 -        if (__put_user(d2, target))
 -            goto fault;
 -        break;
 -    }
 -    case 4: {
 -        u32 d4;
 -        u32 __user *target = (u32 __user *)dst;
 +    ghcb = __sev_get_ghcb(&state);
 -        memcpy(&d4, buf, 4);
 -        if (__put_user(d4, target))
 -            goto fault;
 -        break;
 -    }
 -    case 8: {
 -        u64 d8;
 -        u64 __user *target = (u64 __user *)dst;
 +    vc_ghcb_invalidate(ghcb);
 +    ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE);
 +    ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE);
 +    ghcb_set_sw_exit_info_2(ghcb, 0);
 -        memcpy(&d8, buf, 8);
 -        if (__put_user(d8, target))
 -            goto fault;
 -        break;
 -    }
 -    default:
 -        WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
 -        return ES_UNSUPPORTED;
 -    }
 +    sev_es_wr_ghcb_msr(__pa(ghcb));
 +    VMGEXIT();
 -    return ES_OK;
 +    if (ghcb_sw_exit_info_1_is_valid(ghcb) &&
 +        ghcb_sw_exit_info_2_is_valid(ghcb))
 +        ret = ghcb->save.sw_exit_info_2;
 -fault:
 -    if (user_mode(ctxt->regs))
 -        error_code |= X86_PF_USER;
 +    __sev_put_ghcb(&state);
 -    ctxt->fi.vector = X86_TRAP_PF;
 -    ctxt->fi.error_code = error_code;
 -    ctxt->fi.cr2 = (unsigned long)dst;
 +    local_irq_restore(flags);
 -    return ES_EXCEPTION;
 +    return ret;
  }
 -static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
 -                  char *src, char *buf, size_t size)
 +static inline void __pval_terminate(u64 pfn, bool action, unsigned int page_size,
 +                    int ret, u64 svsm_ret)
  {
 -    unsigned long error_code = X86_PF_PROT;
 -
 -    /*
 -     * This function uses __get_user() independent of whether kernel or user
 -     * memory is accessed. This works fine because __get_user() does no
 -     * sanity checks of the pointer being accessed. All that it does is
 -     * to report when the access failed.
 -     *
 -     * Also, this function runs in atomic context, so __get_user() is not
 -     * allowed to sleep. The page-fault handler detects that it is running
 -     * in atomic context and will not try to take mmap_sem and handle the
 -     * fault, so additional pagefault_enable()/disable() calls are not
 -     * needed.
 -     *
 -     * The access can't be done via copy_from_user() here because
 -     * vc_read_mem() must not use string instructions to access unsafe
 -     * memory. The reason is that MOVS is emulated by the #VC handler by
 -     * splitting the move up into a read and a write and taking a nested #VC
 -     * exception on whatever of them is the MMIO access. Using string
 -     * instructions here would cause infinite nesting.
 -     */
 -    switch (size) {
 -    case 1: {
 -        u8 d1;
 -        u8 __user *s = (u8 __user *)src;
 -
 -        if (__get_user(d1, s))
 -            goto fault;
 -        memcpy(buf, &d1, 1);
 -        break;
 -    }
 -    case 2: {
 -        u16 d2;
 -        u16 __user *s = (u16 __user *)src;
 -
 -        if (__get_user(d2, s))
 -            goto fault;
 -        memcpy(buf, &d2, 2);
 -        break;
 -    }
 -    case 4: {
 -        u32 d4;
 -        u32 __user *s = (u32 __user *)src;
 -
 -        if (__get_user(d4, s))
 -            goto fault;
 -        memcpy(buf, &d4, 4);
 -        break;
 -    }
 -    case 8: {
 -        u64 d8;
 -        u64 __user *s = (u64 __user *)src;
 -        if (__get_user(d8, s))
 -            goto fault;
 -        memcpy(buf, &d8, 8);
 -        break;
 -    }
 -    default:
 -        WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
 -        return ES_UNSUPPORTED;
 -    }
 +    WARN(1, "PVALIDATE failure: pfn: 0x%llx, action: %u, size: %u, ret: %d, svsm_ret: 0x%llx\n",
 +         pfn, action, page_size, ret, svsm_ret);
 -    return ES_OK;
 +    sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
 +}
 -fault:
 -    if (user_mode(ctxt->regs))
 -        error_code |= X86_PF_USER;
 +static void svsm_pval_terminate(struct svsm_pvalidate_call *pc, int ret, u64 svsm_ret)
 +{
 +    unsigned int page_size;
 +    bool action;
 +    u64 pfn;
 -    ctxt->fi.vector = X86_TRAP_PF;
 -    ctxt->fi.error_code = error_code;
 -    ctxt->fi.cr2 = (unsigned long)src;
 +    pfn = pc->entry[pc->cur_index].pfn;
 +    action = pc->entry[pc->cur_index].action;
 +    page_size = pc->entry[pc->cur_index].page_size;
 -    return ES_EXCEPTION;
 +    __pval_terminate(pfn, action, page_size, ret, svsm_ret);
  }
 -static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
 -                       unsigned long vaddr, phys_addr_t *paddr)
 +static void pval_pages(struct snp_psc_desc *desc)
  {
 -    unsigned long va = (unsigned long)vaddr;
 -    unsigned int level;
 -    phys_addr_t pa;
 -    pgd_t *pgd;
 -    pte_t *pte;
 -
 -    pgd = __va(read_cr3_pa());
 -    pgd = &pgd[pgd_index(va)];
 -    pte = lookup_address_in_pgd(pgd, va, &level);
 -    if (!pte) {
 -        ctxt->fi.vector     = X86_TRAP_PF;
 -        ctxt->fi.cr2        = vaddr;
 -        ctxt->fi.error_code = 0;
 -
 -        if (user_mode(ctxt->regs))
 -            ctxt->fi.error_code |= X86_PF_USER;
 +    struct psc_entry *e;
 +    unsigned long vaddr;
 +    unsigned int size;
 +    unsigned int i;
 +    bool validate;
 +    u64 pfn;
 +    int rc;
 -        return ES_EXCEPTION;
 -    }
 +    for (i = 0; i <= desc->hdr.end_entry; i++) {
 +        e = &desc->entries[i];
 -    if (WARN_ON_ONCE(pte_val(*pte) & _PAGE_ENC))
 -        /* Emulated MMIO to/from encrypted memory not supported */
 -        return ES_UNSUPPORTED;
 +        pfn = e->gfn;
 +        vaddr = (unsigned long)pfn_to_kaddr(pfn);
 +        size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
 +        validate = e->operation == SNP_PAGE_STATE_PRIVATE;
 -    pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
 -    pa |= va & ~page_level_mask(level);
 +        rc = pvalidate(vaddr, size, validate);
 +        if (!rc)
 +            continue;
 -    *paddr = pa;
 +        if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) {
 +            unsigned long vaddr_end = vaddr + PMD_SIZE;
 -    return ES_OK;
 +            for (; vaddr < vaddr_end; vaddr += PAGE_SIZE, pfn++) {
 +                rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
 +                if (rc)
 +                    __pval_terminate(pfn, validate, RMP_PG_SIZE_4K, rc, 0);
 +            }
 +        } else {
 +            __pval_terminate(pfn, validate, size, rc, 0);
 +        }
 +    }
  }
 -static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
 +static u64 svsm_build_ca_from_pfn_range(u64 pfn, u64 pfn_end, bool action,
 +                    struct svsm_pvalidate_call *pc)
  {
 -    BUG_ON(size > 4);
 +    struct svsm_pvalidate_entry *pe;
 -    if (user_mode(ctxt->regs)) {
 -        struct thread_struct *t = &current->thread;
 -        struct io_bitmap *iobm = t->io_bitmap;
 -        size_t idx;
 +    /* Nothing in the CA yet */
 +    pc->num_entries = 0;
 +    pc->cur_index   = 0;
 -        if (!iobm)
 -            goto fault;
 +    pe = &pc->entry[0];
 -        for (idx = port; idx < port + size; ++idx) {
 -            if (test_bit(idx, iobm->bitmap))
 -                goto fault;
 -        }
 -    }
 +    while (pfn < pfn_end) {
 +        pe->page_size = RMP_PG_SIZE_4K;
 +        pe->action    = action;
 +        pe->ignore_cf = 0;
 +        pe->pfn       = pfn;
 -    return ES_OK;
 +        pe++;
 +        pfn++;
 -fault:
 -    ctxt->fi.vector = X86_TRAP_GP;
 -    ctxt->fi.error_code = 0;
 +        pc->num_entries++;
 +        if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT)
 +            break;
 +    }
 -    return ES_EXCEPTION;
 +    return pfn;
  }
 -static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt)
 +static int svsm_build_ca_from_psc_desc(struct snp_psc_desc *desc, unsigned int desc_entry,
 +                       struct svsm_pvalidate_call *pc)
  {
 -    long error_code = ctxt->fi.error_code;
 -    int trapnr = ctxt->fi.vector;
 -
 -    ctxt->regs->orig_ax = ctxt->fi.error_code;
 -
 -    switch (trapnr) {
 -    case X86_TRAP_GP:
 -        exc_general_protection(ctxt->regs, error_code);
 -        break;
 -    case X86_TRAP_UD:
 -        exc_invalid_op(ctxt->regs);
 -        break;
 -    case X86_TRAP_PF:
 -        write_cr2(ctxt->fi.cr2);
 -        exc_page_fault(ctxt->regs, error_code);
 -        break;
 -    case X86_TRAP_AC:
 -        exc_alignment_check(ctxt->regs, error_code);
 -        break;
 -    default:
 -        pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n");
 -        BUG();
 -    }
 -}
 +    struct svsm_pvalidate_entry *pe;
 +    struct psc_entry *e;
 -/* Include code shared with pre-decompression boot stage */
 -#include "shared.c"
 +    /* Nothing in the CA yet */
 +    pc->num_entries = 0;
 +    pc->cur_index   = 0;
 -noinstr void __sev_put_ghcb(struct ghcb_state *state)
 -{
 -    struct sev_es_runtime_data *data;
 -    struct ghcb *ghcb;
 +    pe = &pc->entry[0];
 +    e  = &desc->entries[desc_entry];
 -    WARN_ON(!irqs_disabled());
 +    while (desc_entry <= desc->hdr.end_entry) {
 +        pe->page_size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
 +        pe->action    = e->operation == SNP_PAGE_STATE_PRIVATE;
 +        pe->ignore_cf = 0;
 +        pe->pfn       = e->gfn;
 -    data = this_cpu_read(runtime_data);
 -    ghcb = &data->ghcb_page;
 +        pe++;
 +        e++;
 -    if (state->ghcb) {
 -        /* Restore GHCB from Backup */
 -        *ghcb = *state->ghcb;
 -        data->backup_ghcb_active = false;
 -        state->ghcb = NULL;
 -    } else {
 -        /*
 -         * Invalidate the GHCB so a VMGEXIT instruction issued
 -         * from userspace won't appear to be valid.
 -         */
 -        vc_ghcb_invalidate(ghcb);
 -        data->ghcb_active = false;
 +        desc_entry++;
 +        pc->num_entries++;
 +        if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT)
 +            break;
      }
 +
 +    return desc_entry;
  }
 -int svsm_perform_call_protocol(struct svsm_call *call)
 +static void svsm_pval_pages(struct snp_psc_desc *desc)
  {
 -    struct ghcb_state state;
 +    struct svsm_pvalidate_entry pv_4k[VMGEXIT_PSC_MAX_ENTRY];
 +    unsigned int i, pv_4k_count = 0;
 +    struct svsm_pvalidate_call *pc;
 +    struct svsm_call call = {};
      unsigned long flags;
 -    struct ghcb *ghcb;
 +    bool action;
 +    u64 pc_pa;
      int ret;
      /*
@@ -XXX,XX +XXX,XX @@ int svsm_perform_call_protocol(struct svsm_call *call)
      flags = native_local_irq_save();
      /*
 -     * Use rip-relative references when called early in the boot. If
 -     * ghcbs_initialized is set, then it is late in the boot and no need
 -     * to worry about rip-relative references in called functions.
 +     * The SVSM calling area (CA) can support processing 510 entries at a
 +     * time. Loop through the Page State Change descriptor until the CA is
 +     * full or the last entry in the descriptor is reached, at which time
 +     * the SVSM is invoked. This repeats until all entries in the descriptor
 +     * are processed.
       */
 -    if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
 -        ghcb = __sev_get_ghcb(&state);
 -    else if (RIP_REL_REF(boot_ghcb))
 -        ghcb = RIP_REL_REF(boot_ghcb);
 -    else
 -        ghcb = NULL;
 +    call.caa = svsm_get_caa();
 -    do {
 -        ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call)
 -               : svsm_perform_msr_protocol(call);
 -    } while (ret == -EAGAIN);
 +    pc = (struct svsm_pvalidate_call *)call.caa->svsm_buffer;
 +    pc_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer);
 -    if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
 -        __sev_put_ghcb(&state);
 +    /* Protocol 0, Call ID 1 */
 +    call.rax = SVSM_CORE_CALL(SVSM_CORE_PVALIDATE);
 +    call.rcx = pc_pa;
 -    native_local_irq_restore(flags);
 +    for (i = 0; i <= desc->hdr.end_entry;) {
 +        i = svsm_build_ca_from_psc_desc(desc, i, pc);
 -    return ret;
 -}
 +        do {
 +            ret = svsm_perform_call_protocol(&call);
 +            if (!ret)
 +                continue;
 -void noinstr __sev_es_nmi_complete(void)
 -{
 -    struct ghcb_state state;
 -    struct ghcb *ghcb;
 +            /*
 +             * Check if the entry failed because of an RMP mismatch (a
 +             * PVALIDATE at 2M was requested, but the page is mapped in
 +             * the RMP as 4K).
 +             */
 -    ghcb = __sev_get_ghcb(&state);
 +            if (call.rax_out == SVSM_PVALIDATE_FAIL_SIZEMISMATCH &&
 +                pc->entry[pc->cur_index].page_size == RMP_PG_SIZE_2M) {
 +                /* Save this entry for post-processing at 4K */
 +                pv_4k[pv_4k_count++] = pc->entry[pc->cur_index];
 +
 +                /* Skip to the next one unless at the end of the list */
 +                pc->cur_index++;
 +                if (pc->cur_index < pc->num_entries)
 +                    ret = -EAGAIN;
 +                else
 +                    ret = 0;
 +            }
 +        } while (ret == -EAGAIN);
 -    vc_ghcb_invalidate(ghcb);
 -    ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE);
 -    ghcb_set_sw_exit_info_1(ghcb, 0);
 -    ghcb_set_sw_exit_info_2(ghcb, 0);
 +        if (ret)
 +            svsm_pval_terminate(pc, ret, call.rax_out);
 +    }
 -    sev_es_wr_ghcb_msr(__pa_nodebug(ghcb));
 -    VMGEXIT();
 +    /* Process any entries that failed to be validated at 2M and validate them at 4K */
 +    for (i = 0; i < pv_4k_count; i++) {
 +        u64 pfn, pfn_end;
 -    __sev_put_ghcb(&state);
 -}
 +        action  = pv_4k[i].action;
 +        pfn     = pv_4k[i].pfn;
 +        pfn_end = pfn + 512;
 -static u64 __init get_snp_jump_table_addr(void)
 -{
 -    struct snp_secrets_page *secrets;
 -    void __iomem *mem;
 -    u64 addr;
 +        while (pfn < pfn_end) {
 +            pfn = svsm_build_ca_from_pfn_range(pfn, pfn_end, action, pc);
 -    mem = ioremap_encrypted(secrets_pa, PAGE_SIZE);
 -    if (!mem) {
 -        pr_err("Unable to locate AP jump table address: failed to map the SNP secrets page.\n");
 -        return 0;
 +            ret = svsm_perform_call_protocol(&call);
 +            if (ret)
 +                svsm_pval_terminate(pc, ret, call.rax_out);
 +        }
      }
 -    secrets = (__force struct snp_secrets_page *)mem;
 -
 -    addr = secrets->os_area.ap_jump_table_pa;
 -    iounmap(mem);
 -
 -    return addr;
 +    native_local_irq_restore(flags);
  }
 -static u64 __init get_jump_table_addr(void)
 +static void pvalidate_pages(struct snp_psc_desc *desc)
  {
 -    struct ghcb_state state;
 -    unsigned long flags;
 -    struct ghcb *ghcb;
 -    u64 ret = 0;
 -
 -    if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
 -        return get_snp_jump_table_addr();
 -
 -    local_irq_save(flags);
 +    if (snp_vmpl)
 +        svsm_pval_pages(desc);
 +    else
 +        pval_pages(desc);
 +}
 -    ghcb = __sev_get_ghcb(&state);
 +static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)
 +{
 +    int cur_entry, end_entry, ret = 0;
 +    struct snp_psc_desc *data;
 +    struct es_em_ctxt ctxt;
      vc_ghcb_invalidate(ghcb);
 -    ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE);
 -    ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE);
 -    ghcb_set_sw_exit_info_2(ghcb, 0);
 -
 -    sev_es_wr_ghcb_msr(__pa(ghcb));
 -    VMGEXIT();
 -
 -    if (ghcb_sw_exit_info_1_is_valid(ghcb) &&
 -        ghcb_sw_exit_info_2_is_valid(ghcb))
 -        ret = ghcb->save.sw_exit_info_2;
 -
 -    __sev_put_ghcb(&state);
 -    local_irq_restore(flags);
 -
 -    return ret;
 -}
 +    /* Copy the input desc into GHCB shared buffer */
 +    data = (struct snp_psc_desc *)ghcb->shared_buffer;
 +    memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc)));
 -void __head
 -early_set_pages_state(unsigned long vaddr, unsigned long paddr,
 -              unsigned long npages, enum psc_op op)
 -{
 -    unsigned long paddr_end;
 -    u64 val;
 -
 -    vaddr = vaddr & PAGE_MASK;
 +    /*
 +     * As per the GHCB specification, the hypervisor can resume the guest
 +     * before processing all the entries. Check whether all the entries
 +     * are processed. If not, then keep retrying. Note, the hypervisor
 +     * will update the data memory directly to indicate the status, so
 +     * reference the data->hdr everywhere.
 +     *
 +     * The strategy here is to wait for the hypervisor to change the page
 +     * state in the RMP table before guest accesses the memory pages. If the
 +     * page state change was not successful, then later memory access will
 +     * result in a crash.
 +     */
 +    cur_entry = data->hdr.cur_entry;
 +    end_entry = data->hdr.end_entry;
 -    paddr = paddr & PAGE_MASK;
 -    paddr_end = paddr + (npages << PAGE_SHIFT);
 +    while (data->hdr.cur_entry <= data->hdr.end_entry) {
 +        ghcb_set_sw_scratch(ghcb, (u64)__pa(data));
 -    while (paddr < paddr_end) {
 -        /* Page validation must be rescinded before changing to shared */
 -        if (op == SNP_PAGE_STATE_SHARED)
 -            pvalidate_4k_page(vaddr, paddr, false);
 +        /* This will advance the shared buffer data points to. */
 +        ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0);
          /*
 -         * Use the MSR protocol because this function can be called before
 -         * the GHCB is established.
 +         * Page State Change VMGEXIT can pass error code through
 +         * exit_info_2.
           */
 -        sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
 -        VMGEXIT();
 -
 -        val = sev_es_rd_ghcb_msr();
 -
 -        if (GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP)
 -            goto e_term;
 -
 -        if (GHCB_MSR_PSC_RESP_VAL(val))
 -            goto e_term;
 +        if (WARN(ret || ghcb->save.sw_exit_info_2,
 +             "SNP: PSC failed ret=%d exit_info_2=%llx\n",
 +             ret, ghcb->save.sw_exit_info_2)) {
 +            ret = 1;
 +            goto out;
 +        }
 -        /* Page validation must be performed after changing to private */
 -        if (op == SNP_PAGE_STATE_PRIVATE)
 -            pvalidate_4k_page(vaddr, paddr, true);
 +        /* Verify that reserved bit is not set */
 +        if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) {
 +            ret = 1;
 +            goto out;
 +        }
 -        vaddr += PAGE_SIZE;
 -        paddr += PAGE_SIZE;
 +        /*
 +         * Sanity check that entry processing is not going backwards.
 +         * This will happen only if hypervisor is tricking us.
 +         */
 +        if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry,
 +"SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n",
 +             end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) {
 +            ret = 1;
 +            goto out;
 +        }
      }
 -    return;
 -
 -e_term:
 -    sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
 +out:
 +    return ret;
  }
 -void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr,
 -                     unsigned long npages)
 -{
 -    /*
 -     * This can be invoked in early boot while running identity mapped, so
 -     * use an open coded check for SNP instead of using cc_platform_has().
 -     * This eliminates worries about jump tables or checking boot_cpu_data
 -     * in the cc_platform_has() function.
 -     */
 -    if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
 -        return;
 -
 -     /*
 -      * Ask the hypervisor to mark the memory pages as private in the RMP
 -      * table.
 -      */
 -    early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_PRIVATE);
 -}
 -
 -void __head early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr,
 -                    unsigned long npages)
 -{
 -    /*
 -     * This can be invoked in early boot while running identity mapped, so
 -     * use an open coded check for SNP instead of using cc_platform_has().
 -     * This eliminates worries about jump tables or checking boot_cpu_data
 -     * in the cc_platform_has() function.
 -     */
 -    if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
 -        return;
 -
 -     /* Ask hypervisor to mark the memory pages shared in the RMP table. */
 -    early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_SHARED);
 -}
 -
 -static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr,
 -                       unsigned long vaddr_end, int op)
 +static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr,
 +                       unsigned long vaddr_end, int op)
  {
      struct ghcb_state state;
      bool use_large_entry;
@@ -XXX,XX +XXX,XX @@ int __init sev_es_efi_map_ghcbs(pgd_t *pgd)
      return 0;
  }
 -/* Writes to the SVSM CAA MSR are ignored */
 -static enum es_result __vc_handle_msr_caa(struct pt_regs *regs, bool write)
 -{
 -    if (write)
 -        return ES_OK;
 -
 -    regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa));
 -    regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa));
 -
 -    return ES_OK;
 -}
 -
 -/*
 - * TSC related accesses should not exit to the hypervisor when a guest is
 - * executing with Secure TSC enabled, so special handling is required for
 - * accesses of MSR_IA32_TSC and MSR_AMD64_GUEST_TSC_FREQ.
 - */
 -static enum es_result __vc_handle_secure_tsc_msrs(struct pt_regs *regs, bool write)
 -{
 -    u64 tsc;
 -
 -    /*
 -     * GUEST_TSC_FREQ should not be intercepted when Secure TSC is enabled.
 -     * Terminate the SNP guest when the interception is enabled.
 -     */
 -    if (regs->cx == MSR_AMD64_GUEST_TSC_FREQ)
 -        return ES_VMM_ERROR;
 -
 -    /*
 -     * Writes: Writing to MSR_IA32_TSC can cause subsequent reads of the TSC
 -     *         to return undefined values, so ignore all writes.
 -     *
 -     * Reads: Reads of MSR_IA32_TSC should return the current TSC value, use
 -     *        the value returned by rdtsc_ordered().
 -     */
 -    if (write) {
 -        WARN_ONCE(1, "TSC MSR writes are verboten!\n");
 -        return ES_OK;
 -    }
 -
 -    tsc = rdtsc_ordered();
 -    regs->ax = lower_32_bits(tsc);
 -    regs->dx = upper_32_bits(tsc);
 -
 -    return ES_OK;
 -}
 -
 -static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
 -{
 -    struct pt_regs *regs = ctxt->regs;
 -    enum es_result ret;
 -    bool write;
 -
 -    /* Is it a WRMSR? */
 -    write = ctxt->insn.opcode.bytes[1] == 0x30;
 -
 -    switch (regs->cx) {
 -    case MSR_SVSM_CAA:
 -        return __vc_handle_msr_caa(regs, write);
 -    case MSR_IA32_TSC:
 -    case MSR_AMD64_GUEST_TSC_FREQ:
 -        if (sev_status & MSR_AMD64_SNP_SECURE_TSC)
 -            return __vc_handle_secure_tsc_msrs(regs, write);
 -        break;
 -    default:
 -        break;
 -    }
 -
 -    ghcb_set_rcx(ghcb, regs->cx);
 -    if (write) {
 -        ghcb_set_rax(ghcb, regs->ax);
 -        ghcb_set_rdx(ghcb, regs->dx);
 -    }
 -
 -    ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, write, 0);
 -
 -    if ((ret == ES_OK) && !write) {
 -        regs->ax = ghcb->save.rax;
 -        regs->dx = ghcb->save.rdx;
 -    }
 -
 -    return ret;
 -}
 -
  static void snp_register_per_cpu_ghcb(void)
  {
      struct sev_es_runtime_data *data;
@@ -XXX,XX +XXX,XX @@ void __init sev_es_init_vc_handling(void)
      initial_vc_handler = (unsigned long)kernel_exc_vmm_communication;
  }
 -static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt)
 -{
 -    int trapnr = ctxt->fi.vector;
 -
 -    if (trapnr == X86_TRAP_PF)
 -        native_write_cr2(ctxt->fi.cr2);
 -
 -    ctxt->regs->orig_ax = ctxt->fi.error_code;
 -    do_early_exception(ctxt->regs, trapnr);
 -}
 -
 -static long *vc_insn_get_rm(struct es_em_ctxt *ctxt)
 -{
 -    long *reg_array;
 -    int offset;
 -
 -    reg_array = (long *)ctxt->regs;
 -    offset    = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs);
 -
 -    if (offset < 0)
 -        return NULL;
 -
 -    offset /= sizeof(long);
 -
 -    return reg_array + offset;
 -}
 -static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
 -                 unsigned int bytes, bool read)
 -{
 -    u64 exit_code, exit_info_1, exit_info_2;
 -    unsigned long ghcb_pa = __pa(ghcb);
 -    enum es_result res;
 -    phys_addr_t paddr;
 -    void __user *ref;
 -
 -    ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs);
 -    if (ref == (void __user *)-1L)
 -        return ES_UNSUPPORTED;
 -
 -    exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE;
 -
 -    res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr);
 -    if (res != ES_OK) {
 -        if (res == ES_EXCEPTION && !read)
 -            ctxt->fi.error_code |= X86_PF_WRITE;
 -
 -        return res;
 -    }
 -
 -    exit_info_1 = paddr;
 -    /* Can never be greater than 8 */
 -    exit_info_2 = bytes;
 -
 -    ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer));
 -
 -    return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2);
 -}
 -
 -/*
 - * The MOVS instruction has two memory operands, which raises the
 - * problem that it is not known whether the access to the source or the
 - * destination caused the #VC exception (and hence whether an MMIO read
 - * or write operation needs to be emulated).
 - *
 - * Instead of playing games with walking page-tables and trying to guess
 - * whether the source or destination is an MMIO range, split the move
 - * into two operations, a read and a write with only one memory operand.
 - * This will cause a nested #VC exception on the MMIO address which can
 - * then be handled.
 - *
 - * This implementation has the benefit that it also supports MOVS where
 - * source _and_ destination are MMIO regions.
 - *
 - * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a
 - * rare operation. If it turns out to be a performance problem the split
 - * operations can be moved to memcpy_fromio() and memcpy_toio().
 - */
 -static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt,
 -                      unsigned int bytes)
 -{
 -    unsigned long ds_base, es_base;
 -    unsigned char *src, *dst;
 -    unsigned char buffer[8];
 -    enum es_result ret;
 -    bool rep;
 -    int off;
 -
 -    ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS);
 -    es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
 -
 -    if (ds_base == -1L || es_base == -1L) {
 -        ctxt->fi.vector = X86_TRAP_GP;
 -        ctxt->fi.error_code = 0;
 -        return ES_EXCEPTION;
 -    }
 -
 -    src = ds_base + (unsigned char *)ctxt->regs->si;
 -    dst = es_base + (unsigned char *)ctxt->regs->di;
 -
 -    ret = vc_read_mem(ctxt, src, buffer, bytes);
 -    if (ret != ES_OK)
 -        return ret;
 -
 -    ret = vc_write_mem(ctxt, dst, buffer, bytes);
 -    if (ret != ES_OK)
 -        return ret;
 -
 -    if (ctxt->regs->flags & X86_EFLAGS_DF)
 -        off = -bytes;
 -    else
 -        off =  bytes;
 -
 -    ctxt->regs->si += off;
 -    ctxt->regs->di += off;
 -
 -    rep = insn_has_rep_prefix(&ctxt->insn);
 -    if (rep)
 -        ctxt->regs->cx -= 1;
 -
 -    if (!rep || ctxt->regs->cx == 0)
 -        return ES_OK;
 -    else
 -        return ES_RETRY;
 -}
 -
 -static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
 -{
 -    struct insn *insn = &ctxt->insn;
 -    enum insn_mmio_type mmio;
 -    unsigned int bytes = 0;
 -    enum es_result ret;
 -    u8 sign_byte;
 -    long *reg_data;
 -
 -    mmio = insn_decode_mmio(insn, &bytes);
 -    if (mmio == INSN_MMIO_DECODE_FAILED)
 -        return ES_DECODE_FAILED;
 -
 -    if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) {
 -        reg_data = insn_get_modrm_reg_ptr(insn, ctxt->regs);
 -        if (!reg_data)
 -            return ES_DECODE_FAILED;
 -    }
 -
 -    if (user_mode(ctxt->regs))
 -        return ES_UNSUPPORTED;
 -
 -    switch (mmio) {
 -    case INSN_MMIO_WRITE:
 -        memcpy(ghcb->shared_buffer, reg_data, bytes);
 -        ret = vc_do_mmio(ghcb, ctxt, bytes, false);
 -        break;
 -    case INSN_MMIO_WRITE_IMM:
 -        memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes);
 -        ret = vc_do_mmio(ghcb, ctxt, bytes, false);
 -        break;
 -    case INSN_MMIO_READ:
 -        ret = vc_do_mmio(ghcb, ctxt, bytes, true);
 -        if (ret)
 -            break;
 -
 -        /* Zero-extend for 32-bit operation */
 -        if (bytes == 4)
 -            *reg_data = 0;
 -
 -        memcpy(reg_data, ghcb->shared_buffer, bytes);
 -        break;
 -    case INSN_MMIO_READ_ZERO_EXTEND:
 -        ret = vc_do_mmio(ghcb, ctxt, bytes, true);
 -        if (ret)
 -            break;
 -
 -        /* Zero extend based on operand size */
 -        memset(reg_data, 0, insn->opnd_bytes);
 -        memcpy(reg_data, ghcb->shared_buffer, bytes);
 -        break;
 -    case INSN_MMIO_READ_SIGN_EXTEND:
 -        ret = vc_do_mmio(ghcb, ctxt, bytes, true);
 -        if (ret)
 -            break;
 -
 -        if (bytes == 1) {
 -            u8 *val = (u8 *)ghcb->shared_buffer;
 -
 -            sign_byte = (*val & 0x80) ? 0xff : 0x00;
 -        } else {
 -            u16 *val = (u16 *)ghcb->shared_buffer;
 -
 -            sign_byte = (*val & 0x8000) ? 0xff : 0x00;
 -        }
 -
 -        /* Sign extend based on operand size */
 -        memset(reg_data, sign_byte, insn->opnd_bytes);
 -        memcpy(reg_data, ghcb->shared_buffer, bytes);
 -        break;
 -    case INSN_MMIO_MOVS:
 -        ret = vc_handle_mmio_movs(ctxt, bytes);
 -        break;
 -    default:
 -        ret = ES_UNSUPPORTED;
 -        break;
 -    }
 -
 -    return ret;
 -}
 -
 -static enum es_result vc_handle_dr7_write(struct ghcb *ghcb,
 -                      struct es_em_ctxt *ctxt)
 -{
 -    struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
 -    long val, *reg = vc_insn_get_rm(ctxt);
 -    enum es_result ret;
 -
 -    if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
 -        return ES_VMM_ERROR;
 -
 -    if (!reg)
 -        return ES_DECODE_FAILED;
 -
 -    val = *reg;
 -
 -    /* Upper 32 bits must be written as zeroes */
 -    if (val >> 32) {
 -        ctxt->fi.vector = X86_TRAP_GP;
 -        ctxt->fi.error_code = 0;
 -        return ES_EXCEPTION;
 -    }
 -
 -    /* Clear out other reserved bits and set bit 10 */
 -    val = (val & 0xffff23ffL) | BIT(10);
 -
 -    /* Early non-zero writes to DR7 are not supported */
 -    if (!data && (val & ~DR7_RESET_VALUE))
 -        return ES_UNSUPPORTED;
 -
 -    /* Using a value of 0 for ExitInfo1 means RAX holds the value */
 -    ghcb_set_rax(ghcb, val);
 -    ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0);
 -    if (ret != ES_OK)
 -        return ret;
 -
 -    if (data)
 -        data->dr7 = val;
 -
 -    return ES_OK;
 -}
 -
 -static enum es_result vc_handle_dr7_read(struct ghcb *ghcb,
 -                     struct es_em_ctxt *ctxt)
 -{
 -    struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
 -    long *reg = vc_insn_get_rm(ctxt);
 -
 -    if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
 -        return ES_VMM_ERROR;
 -
 -    if (!reg)
 -        return ES_DECODE_FAILED;
 -
 -    if (data)
 -        *reg = data->dr7;
 -    else
 -        *reg = DR7_RESET_VALUE;
 -
 -    return ES_OK;
 -}
 -
 -static enum es_result vc_handle_wbinvd(struct ghcb *ghcb,
 -                       struct es_em_ctxt *ctxt)
 -{
 -    return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0);
 -}
 -
 -static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
 -{
 -    enum es_result ret;
 -
 -    ghcb_set_rcx(ghcb, ctxt->regs->cx);
 -
 -    ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0);
 -    if (ret != ES_OK)
 -        return ret;
 -
 -    if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb)))
 -        return ES_VMM_ERROR;
 -
 -    ctxt->regs->ax = ghcb->save.rax;
 -    ctxt->regs->dx = ghcb->save.rdx;
 -
 -    return ES_OK;
 -}
 -
 -static enum es_result vc_handle_monitor(struct ghcb *ghcb,
 -                    struct es_em_ctxt *ctxt)
 -{
 -    /*
 -     * Treat it as a NOP and do not leak a physical address to the
 -     * hypervisor.
 -     */
 -    return ES_OK;
 -}
 -
 -static enum es_result vc_handle_mwait(struct ghcb *ghcb,
 -                      struct es_em_ctxt *ctxt)
 -{
 -    /* Treat the same as MONITOR/MONITORX */
 -    return ES_OK;
 -}
 -
 -static enum es_result vc_handle_vmmcall(struct ghcb *ghcb,
 -                    struct es_em_ctxt *ctxt)
 -{
 -    enum es_result ret;
 -
 -    ghcb_set_rax(ghcb, ctxt->regs->ax);
 -    ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0);
 -
 -    if (x86_platform.hyper.sev_es_hcall_prepare)
 -        x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs);
 -
 -    ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0);
 -    if (ret != ES_OK)
 -        return ret;
 -
 -    if (!ghcb_rax_is_valid(ghcb))
 -        return ES_VMM_ERROR;
 -
 -    ctxt->regs->ax = ghcb->save.rax;
 -
 -    /*
 -     * Call sev_es_hcall_finish() after regs->ax is already set.
 -     * This allows the hypervisor handler to overwrite it again if
 -     * necessary.
 -     */
 -    if (x86_platform.hyper.sev_es_hcall_finish &&
 -        !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs))
 -        return ES_VMM_ERROR;
 -
 -    return ES_OK;
 -}
 -
 -static enum es_result vc_handle_trap_ac(struct ghcb *ghcb,
 -                    struct es_em_ctxt *ctxt)
 -{
 -    /*
 -     * Calling ecx_alignment_check() directly does not work, because it
 -     * enables IRQs and the GHCB is active. Forward the exception and call
 -     * it later from vc_forward_exception().
 -     */
 -    ctxt->fi.vector = X86_TRAP_AC;
 -    ctxt->fi.error_code = 0;
 -    return ES_EXCEPTION;
 -}
 -
 -static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
 -                     struct ghcb *ghcb,
 -                     unsigned long exit_code)
 -{
 -    enum es_result result = vc_check_opcode_bytes(ctxt, exit_code);
 -
 -    if (result != ES_OK)
 -        return result;
 -
 -    switch (exit_code) {
 -    case SVM_EXIT_READ_DR7:
 -        result = vc_handle_dr7_read(ghcb, ctxt);
 -        break;
 -    case SVM_EXIT_WRITE_DR7:
 -        result = vc_handle_dr7_write(ghcb, ctxt);
 -        break;
 -    case SVM_EXIT_EXCP_BASE + X86_TRAP_AC:
 -        result = vc_handle_trap_ac(ghcb, ctxt);
 -        break;
 -    case SVM_EXIT_RDTSC:
 -    case SVM_EXIT_RDTSCP:
 -        result = vc_handle_rdtsc(ghcb, ctxt, exit_code);
 -        break;
 -    case SVM_EXIT_RDPMC:
 -        result = vc_handle_rdpmc(ghcb, ctxt);
 -        break;
 -    case SVM_EXIT_INVD:
 -        pr_err_ratelimited("#VC exception for INVD??? Seriously???\n");
 -        result = ES_UNSUPPORTED;
 -        break;
 -    case SVM_EXIT_CPUID:
 -        result = vc_handle_cpuid(ghcb, ctxt);
 -        break;
 -    case SVM_EXIT_IOIO:
 -        result = vc_handle_ioio(ghcb, ctxt);
 -        break;
 -    case SVM_EXIT_MSR:
 -        result = vc_handle_msr(ghcb, ctxt);
 -        break;
 -    case SVM_EXIT_VMMCALL:
 -        result = vc_handle_vmmcall(ghcb, ctxt);
 -        break;
 -    case SVM_EXIT_WBINVD:
 -        result = vc_handle_wbinvd(ghcb, ctxt);
 -        break;
 -    case SVM_EXIT_MONITOR:
 -        result = vc_handle_monitor(ghcb, ctxt);
 -        break;
 -    case SVM_EXIT_MWAIT:
 -        result = vc_handle_mwait(ghcb, ctxt);
 -        break;
 -    case SVM_EXIT_NPF:
 -        result = vc_handle_mmio(ghcb, ctxt);
 -        break;
 -    default:
 -        /*
 -         * Unexpected #VC exception
 -         */
 -        result = ES_UNSUPPORTED;
 -    }
 -
 -    return result;
 -}
 -
 -static __always_inline bool is_vc2_stack(unsigned long sp)
 -{
 -    return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2));
 -}
 -
 -static __always_inline bool vc_from_invalid_context(struct pt_regs *regs)
 -{
 -    unsigned long sp, prev_sp;
 -
 -    sp      = (unsigned long)regs;
 -    prev_sp = regs->sp;
 -
 -    /*
 -     * If the code was already executing on the VC2 stack when the #VC
 -     * happened, let it proceed to the normal handling routine. This way the
 -     * code executing on the VC2 stack can cause #VC exceptions to get handled.
 -     */
 -    return is_vc2_stack(sp) && !is_vc2_stack(prev_sp);
 -}
 -
 -static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code)
 -{
 -    struct ghcb_state state;
 -    struct es_em_ctxt ctxt;
 -    enum es_result result;
 -    struct ghcb *ghcb;
 -    bool ret = true;
 -
 -    ghcb = __sev_get_ghcb(&state);
 -
 -    vc_ghcb_invalidate(ghcb);
 -    result = vc_init_em_ctxt(&ctxt, regs, error_code);
 -
 -    if (result == ES_OK)
 -        result = vc_handle_exitcode(&ctxt, ghcb, error_code);
 -
 -    __sev_put_ghcb(&state);
 -
 -    /* Done - now check the result */
 -    switch (result) {
 -    case ES_OK:
 -        vc_finish_insn(&ctxt);
 -        break;
 -    case ES_UNSUPPORTED:
 -        pr_err_ratelimited("Unsupported exit-code 0x%02lx in #VC exception (IP: 0x%lx)\n",
 -                   error_code, regs->ip);
 -        ret = false;
 -        break;
 -    case ES_VMM_ERROR:
 -        pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
 -                   error_code, regs->ip);
 -        ret = false;
 -        break;
 -    case ES_DECODE_FAILED:
 -        pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
 -                   error_code, regs->ip);
 -        ret = false;
 -        break;
 -    case ES_EXCEPTION:
 -        vc_forward_exception(&ctxt);
 -        break;
 -    case ES_RETRY:
 -        /* Nothing to do */
 -        break;
 -    default:
 -        pr_emerg("Unknown result in %s():%d\n", __func__, result);
 -        /*
 -         * Emulating the instruction which caused the #VC exception
 -         * failed - can't continue so print debug information
 -         */
 -        BUG();
 -    }
 -
 -    return ret;
 -}
 -
 -static __always_inline bool vc_is_db(unsigned long error_code)
 -{
 -    return error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB;
 -}
 -
 -/*
 - * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode
 - * and will panic when an error happens.
 - */
 -DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication)
 -{
 -    irqentry_state_t irq_state;
 -
 -    /*
 -     * With the current implementation it is always possible to switch to a
 -     * safe stack because #VC exceptions only happen at known places, like
 -     * intercepted instructions or accesses to MMIO areas/IO ports. They can
 -     * also happen with code instrumentation when the hypervisor intercepts
 -     * #DB, but the critical paths are forbidden to be instrumented, so #DB
 -     * exceptions currently also only happen in safe places.
 -     *
 -     * But keep this here in case the noinstr annotations are violated due
 -     * to bug elsewhere.
 -     */
 -    if (unlikely(vc_from_invalid_context(regs))) {
 -        instrumentation_begin();
 -        panic("Can't handle #VC exception from unsupported context\n");
 -        instrumentation_end();
 -    }
 -
 -    /*
 -     * Handle #DB before calling into !noinstr code to avoid recursive #DB.
 -     */
 -    if (vc_is_db(error_code)) {
 -        exc_debug(regs);
 -        return;
 -    }
 -
 -    irq_state = irqentry_nmi_enter(regs);
 -
 -    instrumentation_begin();
 -
 -    if (!vc_raw_handle_exception(regs, error_code)) {
 -        /* Show some debug info */
 -        show_regs(regs);
 -
 -        /* Ask hypervisor to sev_es_terminate */
 -        sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
 -
 -        /* If that fails and we get here - just panic */
 -        panic("Returned from Terminate-Request to Hypervisor\n");
 -    }
 -
 -    instrumentation_end();
 -    irqentry_nmi_exit(regs, irq_state);
 -}
 -
 -/*
 - * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode
 - * and will kill the current task with SIGBUS when an error happens.
 - */
 -DEFINE_IDTENTRY_VC_USER(exc_vmm_communication)
 -{
 -    /*
 -     * Handle #DB before calling into !noinstr code to avoid recursive #DB.
 -     */
 -    if (vc_is_db(error_code)) {
 -        noist_exc_debug(regs);
 -        return;
 -    }
 -
 -    irqentry_enter_from_user_mode(regs);
 -    instrumentation_begin();
 -
 -    if (!vc_raw_handle_exception(regs, error_code)) {
 -        /*
 -         * Do not kill the machine if user-space triggered the
 -         * exception. Send SIGBUS instead and let user-space deal with
 -         * it.
 -         */
 -        force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0);
 -    }
 -
 -    instrumentation_end();
 -    irqentry_exit_to_user_mode(regs);
 -}
 -
 -bool __init handle_vc_boot_ghcb(struct pt_regs *regs)
 -{
 -    unsigned long exit_code = regs->orig_ax;
 -    struct es_em_ctxt ctxt;
 -    enum es_result result;
 -
 -    vc_ghcb_invalidate(boot_ghcb);
 -
 -    result = vc_init_em_ctxt(&ctxt, regs, exit_code);
 -    if (result == ES_OK)
 -        result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code);
 -
 -    /* Done - now check the result */
 -    switch (result) {
 -    case ES_OK:
 -        vc_finish_insn(&ctxt);
 -        break;
 -    case ES_UNSUPPORTED:
 -        early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n",
 -                exit_code, regs->ip);
 -        goto fail;
 -    case ES_VMM_ERROR:
 -        early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
 -                exit_code, regs->ip);
 -        goto fail;
 -    case ES_DECODE_FAILED:
 -        early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
 -                exit_code, regs->ip);
 -        goto fail;
 -    case ES_EXCEPTION:
 -        vc_early_forward_exception(&ctxt);
 -        break;
 -    case ES_RETRY:
 -        /* Nothing to do */
 -        break;
 -    default:
 -        BUG();
 -    }
 -
 -    return true;
 -
 -fail:
 -    show_regs(regs);
 -
 -    sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
 -}
 -
 -/*
 - * Initial set up of SNP relies on information provided by the
 - * Confidential Computing blob, which can be passed to the kernel
 - * in the following ways, depending on how it is booted:
 - *
 - * - when booted via the boot/decompress kernel:
 - *   - via boot_params
 - *
 - * - when booted directly by firmware/bootloader (e.g. CONFIG_PVH):
 - *   - via a setup_data entry, as defined by the Linux Boot Protocol
 - *
 - * Scan for the blob in that order.
 - */
 -static __head struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
 -{
 -    struct cc_blob_sev_info *cc_info;
 -
 -    /* Boot kernel would have passed the CC blob via boot_params. */
 -    if (bp->cc_blob_address) {
 -        cc_info = (struct cc_blob_sev_info *)(unsigned long)bp->cc_blob_address;
 -        goto found_cc_info;
 -    }
 -
 -    /*
 -     * If kernel was booted directly, without the use of the
 -     * boot/decompression kernel, the CC blob may have been passed via
 -     * setup_data instead.
 -     */
 -    cc_info = find_cc_blob_setup_data(bp);
 -    if (!cc_info)
 -        return NULL;
 -
 -found_cc_info:
 -    if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
 -        snp_abort();
 -
 -    return cc_info;
 -}
 -
 -static __head void svsm_setup(struct cc_blob_sev_info *cc_info)
 -{
 -    struct svsm_call call = {};
 -    int ret;
 -    u64 pa;
 -
 -    /*
 -     * Record the SVSM Calling Area address (CAA) if the guest is not
 -     * running at VMPL0. The CA will be used to communicate with the
 -     * SVSM to perform the SVSM services.
 -     */
 -    if (!svsm_setup_ca(cc_info))
 -        return;
 -
 -    /*
 -     * It is very early in the boot and the kernel is running identity
 -     * mapped but without having adjusted the pagetables to where the
 -     * kernel was loaded (physbase), so the get the CA address using
 -     * RIP-relative addressing.
 -     */
 -    pa = (u64)rip_rel_ptr(&boot_svsm_ca_page);
 -
 -    /*
 -     * Switch over to the boot SVSM CA while the current CA is still
 -     * addressable. There is no GHCB at this point so use the MSR protocol.
 -     *
 -     * SVSM_CORE_REMAP_CA call:
 -     *   RAX = 0 (Protocol=0, CallID=0)
 -     *   RCX = New CA GPA
 -     */
 -    call.caa = svsm_get_caa();
 -    call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA);
 -    call.rcx = pa;
 -    ret = svsm_perform_call_protocol(&call);
 -    if (ret)
 -        sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CA_REMAP_FAIL);
 -
 -    RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)pa;
 -    RIP_REL_REF(boot_svsm_caa_pa) = pa;
 -}
 -
 -bool __head snp_init(struct boot_params *bp)
 -{
 -    struct cc_blob_sev_info *cc_info;
 -
 -    if (!bp)
 -        return false;
 -
 -    cc_info = find_cc_blob(bp);
 -    if (!cc_info)
 -        return false;
 -
 -    if (cc_info->secrets_phys && cc_info->secrets_len == PAGE_SIZE)
 -        secrets_pa = cc_info->secrets_phys;
 -    else
 -        return false;
 -
 -    setup_cpuid_table(cc_info);
 -
 -    svsm_setup(cc_info);
 -
 -    /*
 -     * The CC blob will be used later to access the secrets page. Cache
 -     * it here like the boot kernel does.
 -     */
 -    bp->cc_blob_address = (u32)(unsigned long)cc_info;
 -
 -    return true;
 -}
 -
 -void __head __noreturn snp_abort(void)
 -{
 -    sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
 -}
 -
  /*
   * SEV-SNP guests should only execute dmi_setup() if EFI_CONFIG_TABLES are
   * enabled, as the alternative (fallback) logic for DMI probing in the legacy
 diff --git a/arch/x86/coco/sev/shared.c b/arch/x86/coco/sev/shared.c
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/coco/sev/shared.c
 +++ b/arch/x86/coco/sev/shared.c
@@ -XXX,XX +XXX,XX @@
  /*
   * SVSM related information:
 - *   When running under an SVSM, the VMPL that Linux is executing at must be
 - *   non-zero. The VMPL is therefore used to indicate the presence of an SVSM.
 - *
   *   During boot, the page tables are set up as identity mapped and later
   *   changed to use kernel virtual addresses. Maintain separate virtual and
   *   physical addresses for the CAA to allow SVSM functions to be used during
   *   early boot, both with identity mapped virtual addresses and proper kernel
   *   virtual addresses.
   */
 -u8 snp_vmpl __ro_after_init;
 -EXPORT_SYMBOL_GPL(snp_vmpl);
  struct svsm_ca *boot_svsm_caa __ro_after_init;
  u64 boot_svsm_caa_pa __ro_after_init;
@@ -XXX,XX +XXX,XX @@ static void __head setup_cpuid_table(const struct cc_blob_sev_info *cc_info)
      }
  }
 -static inline void __pval_terminate(u64 pfn, bool action, unsigned int page_size,
 -                    int ret, u64 svsm_ret)
 -{
 -    WARN(1, "PVALIDATE failure: pfn: 0x%llx, action: %u, size: %u, ret: %d, svsm_ret: 0x%llx\n",
 -         pfn, action, page_size, ret, svsm_ret);
 -
 -    sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
 -}
 -
 -static void svsm_pval_terminate(struct svsm_pvalidate_call *pc, int ret, u64 svsm_ret)
 -{
 -    unsigned int page_size;
 -    bool action;
 -    u64 pfn;
 -
 -    pfn = pc->entry[pc->cur_index].pfn;
 -    action = pc->entry[pc->cur_index].action;
 -    page_size = pc->entry[pc->cur_index].page_size;
 -
 -    __pval_terminate(pfn, action, page_size, ret, svsm_ret);
 -}
 -
  static void __head svsm_pval_4k_page(unsigned long paddr, bool validate)
  {
      struct svsm_pvalidate_call *pc;
@@ -XXX,XX +XXX,XX @@ static void __head pvalidate_4k_page(unsigned long vaddr, unsigned long paddr,
      }
  }
 -static void pval_pages(struct snp_psc_desc *desc)
 -{
 -    struct psc_entry *e;
 -    unsigned long vaddr;
 -    unsigned int size;
 -    unsigned int i;
 -    bool validate;
 -    u64 pfn;
 -    int rc;
 -
 -    for (i = 0; i <= desc->hdr.end_entry; i++) {
 -        e = &desc->entries[i];
 -
 -        pfn = e->gfn;
 -        vaddr = (unsigned long)pfn_to_kaddr(pfn);
 -        size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
 -        validate = e->operation == SNP_PAGE_STATE_PRIVATE;
 -
 -        rc = pvalidate(vaddr, size, validate);
 -        if (!rc)
 -            continue;
 -
 -        if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) {
 -            unsigned long vaddr_end = vaddr + PMD_SIZE;
 -
 -            for (; vaddr < vaddr_end; vaddr += PAGE_SIZE, pfn++) {
 -                rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
 -                if (rc)
 -                    __pval_terminate(pfn, validate, RMP_PG_SIZE_4K, rc, 0);
 -            }
 -        } else {
 -            __pval_terminate(pfn, validate, size, rc, 0);
 -        }
 -    }
 -}
 -
 -static u64 svsm_build_ca_from_pfn_range(u64 pfn, u64 pfn_end, bool action,
 -                    struct svsm_pvalidate_call *pc)
 -{
 -    struct svsm_pvalidate_entry *pe;
 -
 -    /* Nothing in the CA yet */
 -    pc->num_entries = 0;
 -    pc->cur_index   = 0;
 -
 -    pe = &pc->entry[0];
 -
 -    while (pfn < pfn_end) {
 -        pe->page_size = RMP_PG_SIZE_4K;
 -        pe->action    = action;
 -        pe->ignore_cf = 0;
 -        pe->pfn       = pfn;
 -
 -        pe++;
 -        pfn++;
 -
 -        pc->num_entries++;
 -        if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT)
 -            break;
 -    }
 -
 -    return pfn;
 -}
 -
 -static int svsm_build_ca_from_psc_desc(struct snp_psc_desc *desc, unsigned int desc_entry,
 -                       struct svsm_pvalidate_call *pc)
 -{
 -    struct svsm_pvalidate_entry *pe;
 -    struct psc_entry *e;
 -
 -    /* Nothing in the CA yet */
 -    pc->num_entries = 0;
 -    pc->cur_index   = 0;
 -
 -    pe = &pc->entry[0];
 -    e  = &desc->entries[desc_entry];
 -
 -    while (desc_entry <= desc->hdr.end_entry) {
 -        pe->page_size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
 -        pe->action    = e->operation == SNP_PAGE_STATE_PRIVATE;
 -        pe->ignore_cf = 0;
 -        pe->pfn       = e->gfn;
 -
 -        pe++;
 -        e++;
 -
 -        desc_entry++;
 -        pc->num_entries++;
 -        if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT)
 -            break;
 -    }
 -
 -    return desc_entry;
 -}
 -
 -static void svsm_pval_pages(struct snp_psc_desc *desc)
 -{
 -    struct svsm_pvalidate_entry pv_4k[VMGEXIT_PSC_MAX_ENTRY];
 -    unsigned int i, pv_4k_count = 0;
 -    struct svsm_pvalidate_call *pc;
 -    struct svsm_call call = {};
 -    unsigned long flags;
 -    bool action;
 -    u64 pc_pa;
 -    int ret;
 -
 -    /*
 -     * This can be called very early in the boot, use native functions in
 -     * order to avoid paravirt issues.
 -     */
 -    flags = native_local_irq_save();
 -
 -    /*
 -     * The SVSM calling area (CA) can support processing 510 entries at a
 -     * time. Loop through the Page State Change descriptor until the CA is
 -     * full or the last entry in the descriptor is reached, at which time
 -     * the SVSM is invoked. This repeats until all entries in the descriptor
 -     * are processed.
 -     */
 -    call.caa = svsm_get_caa();
 -
 -    pc = (struct svsm_pvalidate_call *)call.caa->svsm_buffer;
 -    pc_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer);
 -
 -    /* Protocol 0, Call ID 1 */
 -    call.rax = SVSM_CORE_CALL(SVSM_CORE_PVALIDATE);
 -    call.rcx = pc_pa;
 -
 -    for (i = 0; i <= desc->hdr.end_entry;) {
 -        i = svsm_build_ca_from_psc_desc(desc, i, pc);
 -
 -        do {
 -            ret = svsm_perform_call_protocol(&call);
 -            if (!ret)
 -                continue;
 -
 -            /*
 -             * Check if the entry failed because of an RMP mismatch (a
 -             * PVALIDATE at 2M was requested, but the page is mapped in
 -             * the RMP as 4K).
 -             */
 -
 -            if (call.rax_out == SVSM_PVALIDATE_FAIL_SIZEMISMATCH &&
 -                pc->entry[pc->cur_index].page_size == RMP_PG_SIZE_2M) {
 -                /* Save this entry for post-processing at 4K */
 -                pv_4k[pv_4k_count++] = pc->entry[pc->cur_index];
 -
 -                /* Skip to the next one unless at the end of the list */
 -                pc->cur_index++;
 -                if (pc->cur_index < pc->num_entries)
 -                    ret = -EAGAIN;
 -                else
 -                    ret = 0;
 -            }
 -        } while (ret == -EAGAIN);
 -
 -        if (ret)
 -            svsm_pval_terminate(pc, ret, call.rax_out);
 -    }
 -
 -    /* Process any entries that failed to be validated at 2M and validate them at 4K */
 -    for (i = 0; i < pv_4k_count; i++) {
 -        u64 pfn, pfn_end;
 -
 -        action  = pv_4k[i].action;
 -        pfn     = pv_4k[i].pfn;
 -        pfn_end = pfn + 512;
 -
 -        while (pfn < pfn_end) {
 -            pfn = svsm_build_ca_from_pfn_range(pfn, pfn_end, action, pc);
 -
 -            ret = svsm_perform_call_protocol(&call);
 -            if (ret)
 -                svsm_pval_terminate(pc, ret, call.rax_out);
 -        }
 -    }
 -
 -    native_local_irq_restore(flags);
 -}
 -
 -static void pvalidate_pages(struct snp_psc_desc *desc)
 -{
 -    if (snp_vmpl)
 -        svsm_pval_pages(desc);
 -    else
 -        pval_pages(desc);
 -}
 -
 -static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)
 -{
 -    int cur_entry, end_entry, ret = 0;
 -    struct snp_psc_desc *data;
 -    struct es_em_ctxt ctxt;
 -
 -    vc_ghcb_invalidate(ghcb);
 -
 -    /* Copy the input desc into GHCB shared buffer */
 -    data = (struct snp_psc_desc *)ghcb->shared_buffer;
 -    memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc)));
 -
 -    /*
 -     * As per the GHCB specification, the hypervisor can resume the guest
 -     * before processing all the entries. Check whether all the entries
 -     * are processed. If not, then keep retrying. Note, the hypervisor
 -     * will update the data memory directly to indicate the status, so
 -     * reference the data->hdr everywhere.
 -     *
 -     * The strategy here is to wait for the hypervisor to change the page
 -     * state in the RMP table before guest accesses the memory pages. If the
 -     * page state change was not successful, then later memory access will
 -     * result in a crash.
 -     */
 -    cur_entry = data->hdr.cur_entry;
 -    end_entry = data->hdr.end_entry;
 -
 -    while (data->hdr.cur_entry <= data->hdr.end_entry) {
 -        ghcb_set_sw_scratch(ghcb, (u64)__pa(data));
 -
 -        /* This will advance the shared buffer data points to. */
 -        ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0);
 -
 -        /*
 -         * Page State Change VMGEXIT can pass error code through
 -         * exit_info_2.
 -         */
 -        if (WARN(ret || ghcb->save.sw_exit_info_2,
 -             "SNP: PSC failed ret=%d exit_info_2=%llx\n",
 -             ret, ghcb->save.sw_exit_info_2)) {
 -            ret = 1;
 -            goto out;
 -        }
 -
 -        /* Verify that reserved bit is not set */
 -        if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) {
 -            ret = 1;
 -            goto out;
 -        }
 -
 -        /*
 -         * Sanity check that entry processing is not going backwards.
 -         * This will happen only if hypervisor is tricking us.
 -         */
 -        if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry,
 -"SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n",
 -             end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) {
 -            ret = 1;
 -            goto out;
 -        }
 -    }
 -
 -out:
 -    return ret;
 -}
 -
  static enum es_result vc_check_opcode_bytes(struct es_em_ctxt *ctxt,
                          unsigned long exit_code)
  {
 diff --git a/arch/x86/coco/sev/startup.c b/arch/x86/coco/sev/startup.c
 new file mode 100644
 index XXXXXXX..XXXXXXX
 --- /dev/null
-+++ b/arch/x86/boot/startup/Makefile
++++ b/arch/x86/coco/sev/startup.c
 @@ -XXX,XX +XXX,XX @@
-+# SPDX-License-Identifier: GPL-2.0
++// SPDX-License-Identifier: GPL-2.0-only
-+
++/*
-+lib-$(CONFIG_X86_64)        += la57toggle.o
++ * AMD Memory Encryption Support
-diff --git a/arch/x86/boot/compressed/la57toggle.S b/arch/x86/boot/startup/la57toggle.S
++ *
-similarity index 100%
++ * Copyright (C) 2019 SUSE
-rename from arch/x86/boot/compressed/la57toggle.S
++ *
-rename to arch/x86/boot/startup/la57toggle.S
++ * Author: Joerg Roedel <jroedel@suse.de>
 + */
 +
 +#define pr_fmt(fmt)    "SEV: " fmt
 +
 +#include <linux/sched/debug.h>    /* For show_regs() */
 +#include <linux/percpu-defs.h>
 +#include <linux/cc_platform.h>
 +#include <linux/printk.h>
 +#include <linux/mm_types.h>
 +#include <linux/set_memory.h>
 +#include <linux/memblock.h>
 +#include <linux/kernel.h>
 +#include <linux/mm.h>
 +#include <linux/cpumask.h>
 +#include <linux/efi.h>
 +#include <linux/io.h>
 +#include <linux/psp-sev.h>
 +#include <uapi/linux/sev-guest.h>
 +
 +#include <asm/init.h>
 +#include <asm/cpu_entry_area.h>
 +#include <asm/stacktrace.h>
 +#include <asm/sev.h>
 +#include <asm/sev-internal.h>
 +#include <asm/insn-eval.h>
 +#include <asm/fpu/xcr.h>
 +#include <asm/processor.h>
 +#include <asm/realmode.h>
 +#include <asm/setup.h>
 +#include <asm/traps.h>
 +#include <asm/svm.h>
 +#include <asm/smp.h>
 +#include <asm/cpu.h>
 +#include <asm/apic.h>
 +#include <asm/cpuid.h>
 +#include <asm/cmdline.h>
 +
 +/* For early boot hypervisor communication in SEV-ES enabled guests */
 +struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
 +
 +/*
 + * Needs to be in the .data section because we need it NULL before bss is
 + * cleared
 + */
 +struct ghcb *boot_ghcb __section(".data");
 +
 +/* Bitmap of SEV features supported by the hypervisor */
 +u64 sev_hv_features __ro_after_init;
 +
 +/* Secrets page physical address from the CC blob */
 +static u64 secrets_pa __ro_after_init;
 +
 +/* For early boot SVSM communication */
 +struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE);
 +
 +DEFINE_PER_CPU(struct svsm_ca *, svsm_caa);
 +DEFINE_PER_CPU(u64, svsm_caa_pa);
 +
 +/*
 + * Nothing shall interrupt this code path while holding the per-CPU
 + * GHCB. The backup GHCB is only for NMIs interrupting this path.
 + *
 + * Callers must disable local interrupts around it.
 + */
 +noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
 +{
 +    struct sev_es_runtime_data *data;
 +    struct ghcb *ghcb;
 +
 +    WARN_ON(!irqs_disabled());
 +
 +    data = this_cpu_read(runtime_data);
 +    ghcb = &data->ghcb_page;
 +
 +    if (unlikely(data->ghcb_active)) {
 +        /* GHCB is already in use - save its contents */
 +
 +        if (unlikely(data->backup_ghcb_active)) {
 +            /*
 +             * Backup-GHCB is also already in use. There is no way
 +             * to continue here so just kill the machine. To make
 +             * panic() work, mark GHCBs inactive so that messages
 +             * can be printed out.
 +             */
 +            data->ghcb_active        = false;
 +            data->backup_ghcb_active = false;
 +
 +            instrumentation_begin();
 +            panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use");
 +            instrumentation_end();
 +        }
 +
 +        /* Mark backup_ghcb active before writing to it */
 +        data->backup_ghcb_active = true;
 +
 +        state->ghcb = &data->backup_ghcb;
 +
 +        /* Backup GHCB content */
 +        *state->ghcb = *ghcb;
 +    } else {
 +        state->ghcb = NULL;
 +        data->ghcb_active = true;
 +    }
 +
 +    return ghcb;
 +}
 +
 +static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt,
 +                unsigned char *buffer)
 +{
 +    return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
 +}
 +
 +static enum es_result __vc_decode_user_insn(struct es_em_ctxt *ctxt)
 +{
 +    char buffer[MAX_INSN_SIZE];
 +    int insn_bytes;
 +
 +    insn_bytes = insn_fetch_from_user_inatomic(ctxt->regs, buffer);
 +    if (insn_bytes == 0) {
 +        /* Nothing could be copied */
 +        ctxt->fi.vector     = X86_TRAP_PF;
 +        ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER;
 +        ctxt->fi.cr2        = ctxt->regs->ip;
 +        return ES_EXCEPTION;
 +    } else if (insn_bytes == -EINVAL) {
 +        /* Effective RIP could not be calculated */
 +        ctxt->fi.vector     = X86_TRAP_GP;
 +        ctxt->fi.error_code = 0;
 +        ctxt->fi.cr2        = 0;
 +        return ES_EXCEPTION;
 +    }
 +
 +    if (!insn_decode_from_regs(&ctxt->insn, ctxt->regs, buffer, insn_bytes))
 +        return ES_DECODE_FAILED;
 +
 +    if (ctxt->insn.immediate.got)
 +        return ES_OK;
 +    else
 +        return ES_DECODE_FAILED;
 +}
 +
 +static enum es_result __vc_decode_kern_insn(struct es_em_ctxt *ctxt)
 +{
 +    char buffer[MAX_INSN_SIZE];
 +    int res, ret;
 +
 +    res = vc_fetch_insn_kernel(ctxt, buffer);
 +    if (res) {
 +        ctxt->fi.vector     = X86_TRAP_PF;
 +        ctxt->fi.error_code = X86_PF_INSTR;
 +        ctxt->fi.cr2        = ctxt->regs->ip;
 +        return ES_EXCEPTION;
 +    }
 +
 +    ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
 +    if (ret < 0)
 +        return ES_DECODE_FAILED;
 +    else
 +        return ES_OK;
 +}
 +
 +static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
 +{
 +    if (user_mode(ctxt->regs))
 +        return __vc_decode_user_insn(ctxt);
 +    else
 +        return __vc_decode_kern_insn(ctxt);
 +}
 +
 +static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
 +                   char *dst, char *buf, size_t size)
 +{
 +    unsigned long error_code = X86_PF_PROT | X86_PF_WRITE;
 +
 +    /*
 +     * This function uses __put_user() independent of whether kernel or user
 +     * memory is accessed. This works fine because __put_user() does no
 +     * sanity checks of the pointer being accessed. All that it does is
 +     * to report when the access failed.
 +     *
 +     * Also, this function runs in atomic context, so __put_user() is not
 +     * allowed to sleep. The page-fault handler detects that it is running
 +     * in atomic context and will not try to take mmap_sem and handle the
 +     * fault, so additional pagefault_enable()/disable() calls are not
 +     * needed.
 +     *
 +     * The access can't be done via copy_to_user() here because
 +     * vc_write_mem() must not use string instructions to access unsafe
 +     * memory. The reason is that MOVS is emulated by the #VC handler by
 +     * splitting the move up into a read and a write and taking a nested #VC
 +     * exception on whatever of them is the MMIO access. Using string
 +     * instructions here would cause infinite nesting.
 +     */
 +    switch (size) {
 +    case 1: {
 +        u8 d1;
 +        u8 __user *target = (u8 __user *)dst;
 +
 +        memcpy(&d1, buf, 1);
 +        if (__put_user(d1, target))
 +            goto fault;
 +        break;
 +    }
 +    case 2: {
 +        u16 d2;
 +        u16 __user *target = (u16 __user *)dst;
 +
 +        memcpy(&d2, buf, 2);
 +        if (__put_user(d2, target))
 +            goto fault;
 +        break;
 +    }
 +    case 4: {
 +        u32 d4;
 +        u32 __user *target = (u32 __user *)dst;
 +
 +        memcpy(&d4, buf, 4);
 +        if (__put_user(d4, target))
 +            goto fault;
 +        break;
 +    }
 +    case 8: {
 +        u64 d8;
 +        u64 __user *target = (u64 __user *)dst;
 +
 +        memcpy(&d8, buf, 8);
 +        if (__put_user(d8, target))
 +            goto fault;
 +        break;
 +    }
 +    default:
 +        WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
 +        return ES_UNSUPPORTED;
 +    }
 +
 +    return ES_OK;
 +
 +fault:
 +    if (user_mode(ctxt->regs))
 +        error_code |= X86_PF_USER;
 +
 +    ctxt->fi.vector = X86_TRAP_PF;
 +    ctxt->fi.error_code = error_code;
 +    ctxt->fi.cr2 = (unsigned long)dst;
 +
 +    return ES_EXCEPTION;
 +}
 +
 +static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
 +                  char *src, char *buf, size_t size)
 +{
 +    unsigned long error_code = X86_PF_PROT;
 +
 +    /*
 +     * This function uses __get_user() independent of whether kernel or user
 +     * memory is accessed. This works fine because __get_user() does no
 +     * sanity checks of the pointer being accessed. All that it does is
 +     * to report when the access failed.
 +     *
 +     * Also, this function runs in atomic context, so __get_user() is not
 +     * allowed to sleep. The page-fault handler detects that it is running
 +     * in atomic context and will not try to take mmap_sem and handle the
 +     * fault, so additional pagefault_enable()/disable() calls are not
 +     * needed.
 +     *
 +     * The access can't be done via copy_from_user() here because
 +     * vc_read_mem() must not use string instructions to access unsafe
 +     * memory. The reason is that MOVS is emulated by the #VC handler by
 +     * splitting the move up into a read and a write and taking a nested #VC
 +     * exception on whatever of them is the MMIO access. Using string
 +     * instructions here would cause infinite nesting.
 +     */
 +    switch (size) {
 +    case 1: {
 +        u8 d1;
 +        u8 __user *s = (u8 __user *)src;
 +
 +        if (__get_user(d1, s))
 +            goto fault;
 +        memcpy(buf, &d1, 1);
 +        break;
 +    }
 +    case 2: {
 +        u16 d2;
 +        u16 __user *s = (u16 __user *)src;
 +
 +        if (__get_user(d2, s))
 +            goto fault;
 +        memcpy(buf, &d2, 2);
 +        break;
 +    }
 +    case 4: {
 +        u32 d4;
 +        u32 __user *s = (u32 __user *)src;
 +
 +        if (__get_user(d4, s))
 +            goto fault;
 +        memcpy(buf, &d4, 4);
 +        break;
 +    }
 +    case 8: {
 +        u64 d8;
 +        u64 __user *s = (u64 __user *)src;
 +        if (__get_user(d8, s))
 +            goto fault;
 +        memcpy(buf, &d8, 8);
 +        break;
 +    }
 +    default:
 +        WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
 +        return ES_UNSUPPORTED;
 +    }
 +
 +    return ES_OK;
 +
 +fault:
 +    if (user_mode(ctxt->regs))
 +        error_code |= X86_PF_USER;
 +
 +    ctxt->fi.vector = X86_TRAP_PF;
 +    ctxt->fi.error_code = error_code;
 +    ctxt->fi.cr2 = (unsigned long)src;
 +
 +    return ES_EXCEPTION;
 +}
 +
 +static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
 +                       unsigned long vaddr, phys_addr_t *paddr)
 +{
 +    unsigned long va = (unsigned long)vaddr;
 +    unsigned int level;
 +    phys_addr_t pa;
 +    pgd_t *pgd;
 +    pte_t *pte;
 +
 +    pgd = __va(read_cr3_pa());
 +    pgd = &pgd[pgd_index(va)];
 +    pte = lookup_address_in_pgd(pgd, va, &level);
 +    if (!pte) {
 +        ctxt->fi.vector     = X86_TRAP_PF;
 +        ctxt->fi.cr2        = vaddr;
 +        ctxt->fi.error_code = 0;
 +
 +        if (user_mode(ctxt->regs))
 +            ctxt->fi.error_code |= X86_PF_USER;
 +
 +        return ES_EXCEPTION;
 +    }
 +
 +    if (WARN_ON_ONCE(pte_val(*pte) & _PAGE_ENC))
 +        /* Emulated MMIO to/from encrypted memory not supported */
 +        return ES_UNSUPPORTED;
 +
 +    pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
 +    pa |= va & ~page_level_mask(level);
 +
 +    *paddr = pa;
 +
 +    return ES_OK;
 +}
 +
 +static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
 +{
 +    BUG_ON(size > 4);
 +
 +    if (user_mode(ctxt->regs)) {
 +        struct thread_struct *t = &current->thread;
 +        struct io_bitmap *iobm = t->io_bitmap;
 +        size_t idx;
 +
 +        if (!iobm)
 +            goto fault;
 +
 +        for (idx = port; idx < port + size; ++idx) {
 +            if (test_bit(idx, iobm->bitmap))
 +                goto fault;
 +        }
 +    }
 +
 +    return ES_OK;
 +
 +fault:
 +    ctxt->fi.vector = X86_TRAP_GP;
 +    ctxt->fi.error_code = 0;
 +
 +    return ES_EXCEPTION;
 +}
 +
 +static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt)
 +{
 +    long error_code = ctxt->fi.error_code;
 +    int trapnr = ctxt->fi.vector;
 +
 +    ctxt->regs->orig_ax = ctxt->fi.error_code;
 +
 +    switch (trapnr) {
 +    case X86_TRAP_GP:
 +        exc_general_protection(ctxt->regs, error_code);
 +        break;
 +    case X86_TRAP_UD:
 +        exc_invalid_op(ctxt->regs);
 +        break;
 +    case X86_TRAP_PF:
 +        write_cr2(ctxt->fi.cr2);
 +        exc_page_fault(ctxt->regs, error_code);
 +        break;
 +    case X86_TRAP_AC:
 +        exc_alignment_check(ctxt->regs, error_code);
 +        break;
 +    default:
 +        pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n");
 +        BUG();
 +    }
 +}
 +
 +/* Include code shared with pre-decompression boot stage */
 +#include "shared.c"
 +
 +noinstr void __sev_put_ghcb(struct ghcb_state *state)
 +{
 +    struct sev_es_runtime_data *data;
 +    struct ghcb *ghcb;
 +
 +    WARN_ON(!irqs_disabled());
 +
 +    data = this_cpu_read(runtime_data);
 +    ghcb = &data->ghcb_page;
 +
 +    if (state->ghcb) {
 +        /* Restore GHCB from Backup */
 +        *ghcb = *state->ghcb;
 +        data->backup_ghcb_active = false;
 +        state->ghcb = NULL;
 +    } else {
 +        /*
 +         * Invalidate the GHCB so a VMGEXIT instruction issued
 +         * from userspace won't appear to be valid.
 +         */
 +        vc_ghcb_invalidate(ghcb);
 +        data->ghcb_active = false;
 +    }
 +}
 +
 +int svsm_perform_call_protocol(struct svsm_call *call)
 +{
 +    struct ghcb_state state;
 +    unsigned long flags;
 +    struct ghcb *ghcb;
 +    int ret;
 +
 +    /*
 +     * This can be called very early in the boot, use native functions in
 +     * order to avoid paravirt issues.
 +     */
 +    flags = native_local_irq_save();
 +
 +    /*
 +     * Use rip-relative references when called early in the boot. If
 +     * ghcbs_initialized is set, then it is late in the boot and no need
 +     * to worry about rip-relative references in called functions.
 +     */
 +    if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
 +        ghcb = __sev_get_ghcb(&state);
 +    else if (RIP_REL_REF(boot_ghcb))
 +        ghcb = RIP_REL_REF(boot_ghcb);
 +    else
 +        ghcb = NULL;
 +
 +    do {
 +        ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call)
 +               : svsm_perform_msr_protocol(call);
 +    } while (ret == -EAGAIN);
 +
 +    if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
 +        __sev_put_ghcb(&state);
 +
 +    native_local_irq_restore(flags);
 +
 +    return ret;
 +}
 +
 +void __head
 +early_set_pages_state(unsigned long vaddr, unsigned long paddr,
 +              unsigned long npages, enum psc_op op)
 +{
 +    unsigned long paddr_end;
 +    u64 val;
 +
 +    vaddr = vaddr & PAGE_MASK;
 +
 +    paddr = paddr & PAGE_MASK;
 +    paddr_end = paddr + (npages << PAGE_SHIFT);
 +
 +    while (paddr < paddr_end) {
 +        /* Page validation must be rescinded before changing to shared */
 +        if (op == SNP_PAGE_STATE_SHARED)
 +            pvalidate_4k_page(vaddr, paddr, false);
 +
 +        /*
 +         * Use the MSR protocol because this function can be called before
 +         * the GHCB is established.
 +         */
 +        sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
 +        VMGEXIT();
 +
 +        val = sev_es_rd_ghcb_msr();
 +
 +        if (GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP)
 +            goto e_term;
 +
 +        if (GHCB_MSR_PSC_RESP_VAL(val))
 +            goto e_term;
 +
 +        /* Page validation must be performed after changing to private */
 +        if (op == SNP_PAGE_STATE_PRIVATE)
 +            pvalidate_4k_page(vaddr, paddr, true);
 +
 +        vaddr += PAGE_SIZE;
 +        paddr += PAGE_SIZE;
 +    }
 +
 +    return;
 +
 +e_term:
 +    sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
 +}
 +
 +void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr,
 +                     unsigned long npages)
 +{
 +    /*
 +     * This can be invoked in early boot while running identity mapped, so
 +     * use an open coded check for SNP instead of using cc_platform_has().
 +     * This eliminates worries about jump tables or checking boot_cpu_data
 +     * in the cc_platform_has() function.
 +     */
 +    if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
 +        return;
 +
 +     /*
 +      * Ask the hypervisor to mark the memory pages as private in the RMP
 +      * table.
 +      */
 +    early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_PRIVATE);
 +}
 +
 +void __head early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr,
 +                    unsigned long npages)
 +{
 +    /*
 +     * This can be invoked in early boot while running identity mapped, so
 +     * use an open coded check for SNP instead of using cc_platform_has().
 +     * This eliminates worries about jump tables or checking boot_cpu_data
 +     * in the cc_platform_has() function.
 +     */
 +    if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
 +        return;
 +
 +     /* Ask hypervisor to mark the memory pages shared in the RMP table. */
 +    early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_SHARED);
 +}
 +
 +/* Writes to the SVSM CAA MSR are ignored */
 +static enum es_result __vc_handle_msr_caa(struct pt_regs *regs, bool write)
 +{
 +    if (write)
 +        return ES_OK;
 +
 +    regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa));
 +    regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa));
 +
 +    return ES_OK;
 +}
 +
 +/*
 + * TSC related accesses should not exit to the hypervisor when a guest is
 + * executing with Secure TSC enabled, so special handling is required for
 + * accesses of MSR_IA32_TSC and MSR_AMD64_GUEST_TSC_FREQ.
 + */
 +static enum es_result __vc_handle_secure_tsc_msrs(struct pt_regs *regs, bool write)
 +{
 +    u64 tsc;
 +
 +    /*
 +     * GUEST_TSC_FREQ should not be intercepted when Secure TSC is enabled.
 +     * Terminate the SNP guest when the interception is enabled.
 +     */
 +    if (regs->cx == MSR_AMD64_GUEST_TSC_FREQ)
 +        return ES_VMM_ERROR;
 +
 +    /*
 +     * Writes: Writing to MSR_IA32_TSC can cause subsequent reads of the TSC
 +     *         to return undefined values, so ignore all writes.
 +     *
 +     * Reads: Reads of MSR_IA32_TSC should return the current TSC value, use
 +     *        the value returned by rdtsc_ordered().
 +     */
 +    if (write) {
 +        WARN_ONCE(1, "TSC MSR writes are verboten!\n");
 +        return ES_OK;
 +    }
 +
 +    tsc = rdtsc_ordered();
 +    regs->ax = lower_32_bits(tsc);
 +    regs->dx = upper_32_bits(tsc);
 +
 +    return ES_OK;
 +}
 +
 +static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
 +{
 +    struct pt_regs *regs = ctxt->regs;
 +    enum es_result ret;
 +    bool write;
 +
 +    /* Is it a WRMSR? */
 +    write = ctxt->insn.opcode.bytes[1] == 0x30;
 +
 +    switch (regs->cx) {
 +    case MSR_SVSM_CAA:
 +        return __vc_handle_msr_caa(regs, write);
 +    case MSR_IA32_TSC:
 +    case MSR_AMD64_GUEST_TSC_FREQ:
 +        if (sev_status & MSR_AMD64_SNP_SECURE_TSC)
 +            return __vc_handle_secure_tsc_msrs(regs, write);
 +        break;
 +    default:
 +        break;
 +    }
 +
 +    ghcb_set_rcx(ghcb, regs->cx);
 +    if (write) {
 +        ghcb_set_rax(ghcb, regs->ax);
 +        ghcb_set_rdx(ghcb, regs->dx);
 +    }
 +
 +    ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, write, 0);
 +
 +    if ((ret == ES_OK) && !write) {
 +        regs->ax = ghcb->save.rax;
 +        regs->dx = ghcb->save.rdx;
 +    }
 +
 +    return ret;
 +}
 +
 +static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt)
 +{
 +    int trapnr = ctxt->fi.vector;
 +
 +    if (trapnr == X86_TRAP_PF)
 +        native_write_cr2(ctxt->fi.cr2);
 +
 +    ctxt->regs->orig_ax = ctxt->fi.error_code;
 +    do_early_exception(ctxt->regs, trapnr);
 +}
 +
 +static long *vc_insn_get_rm(struct es_em_ctxt *ctxt)
 +{
 +    long *reg_array;
 +    int offset;
 +
 +    reg_array = (long *)ctxt->regs;
 +    offset    = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs);
 +
 +    if (offset < 0)
 +        return NULL;
 +
 +    offset /= sizeof(long);
 +
 +    return reg_array + offset;
 +}
 +static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
 +                 unsigned int bytes, bool read)
 +{
 +    u64 exit_code, exit_info_1, exit_info_2;
 +    unsigned long ghcb_pa = __pa(ghcb);
 +    enum es_result res;
 +    phys_addr_t paddr;
 +    void __user *ref;
 +
 +    ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs);
 +    if (ref == (void __user *)-1L)
 +        return ES_UNSUPPORTED;
 +
 +    exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE;
 +
 +    res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr);
 +    if (res != ES_OK) {
 +        if (res == ES_EXCEPTION && !read)
 +            ctxt->fi.error_code |= X86_PF_WRITE;
 +
 +        return res;
 +    }
 +
 +    exit_info_1 = paddr;
 +    /* Can never be greater than 8 */
 +    exit_info_2 = bytes;
 +
 +    ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer));
 +
 +    return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2);
 +}
 +
 +/*
 + * The MOVS instruction has two memory operands, which raises the
 + * problem that it is not known whether the access to the source or the
 + * destination caused the #VC exception (and hence whether an MMIO read
 + * or write operation needs to be emulated).
 + *
 + * Instead of playing games with walking page-tables and trying to guess
 + * whether the source or destination is an MMIO range, split the move
 + * into two operations, a read and a write with only one memory operand.
 + * This will cause a nested #VC exception on the MMIO address which can
 + * then be handled.
 + *
 + * This implementation has the benefit that it also supports MOVS where
 + * source _and_ destination are MMIO regions.
 + *
 + * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a
 + * rare operation. If it turns out to be a performance problem the split
 + * operations can be moved to memcpy_fromio() and memcpy_toio().
 + */
 +static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt,
 +                      unsigned int bytes)
 +{
 +    unsigned long ds_base, es_base;
 +    unsigned char *src, *dst;
 +    unsigned char buffer[8];
 +    enum es_result ret;
 +    bool rep;
 +    int off;
 +
 +    ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS);
 +    es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
 +
 +    if (ds_base == -1L || es_base == -1L) {
 +        ctxt->fi.vector = X86_TRAP_GP;
 +        ctxt->fi.error_code = 0;
 +        return ES_EXCEPTION;
 +    }
 +
 +    src = ds_base + (unsigned char *)ctxt->regs->si;
 +    dst = es_base + (unsigned char *)ctxt->regs->di;
 +
 +    ret = vc_read_mem(ctxt, src, buffer, bytes);
 +    if (ret != ES_OK)
 +        return ret;
 +
 +    ret = vc_write_mem(ctxt, dst, buffer, bytes);
 +    if (ret != ES_OK)
 +        return ret;
 +
 +    if (ctxt->regs->flags & X86_EFLAGS_DF)
 +        off = -bytes;
 +    else
 +        off =  bytes;
 +
 +    ctxt->regs->si += off;
 +    ctxt->regs->di += off;
 +
 +    rep = insn_has_rep_prefix(&ctxt->insn);
 +    if (rep)
 +        ctxt->regs->cx -= 1;
 +
 +    if (!rep || ctxt->regs->cx == 0)
 +        return ES_OK;
 +    else
 +        return ES_RETRY;
 +}
 +
 +static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
 +{
 +    struct insn *insn = &ctxt->insn;
 +    enum insn_mmio_type mmio;
 +    unsigned int bytes = 0;
 +    enum es_result ret;
 +    u8 sign_byte;
 +    long *reg_data;
 +
 +    mmio = insn_decode_mmio(insn, &bytes);
 +    if (mmio == INSN_MMIO_DECODE_FAILED)
 +        return ES_DECODE_FAILED;
 +
 +    if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) {
 +        reg_data = insn_get_modrm_reg_ptr(insn, ctxt->regs);
 +        if (!reg_data)
 +            return ES_DECODE_FAILED;
 +    }
 +
 +    if (user_mode(ctxt->regs))
 +        return ES_UNSUPPORTED;
 +
 +    switch (mmio) {
 +    case INSN_MMIO_WRITE:
 +        memcpy(ghcb->shared_buffer, reg_data, bytes);
 +        ret = vc_do_mmio(ghcb, ctxt, bytes, false);
 +        break;
 +    case INSN_MMIO_WRITE_IMM:
 +        memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes);
 +        ret = vc_do_mmio(ghcb, ctxt, bytes, false);
 +        break;
 +    case INSN_MMIO_READ:
 +        ret = vc_do_mmio(ghcb, ctxt, bytes, true);
 +        if (ret)
 +            break;
 +
 +        /* Zero-extend for 32-bit operation */
 +        if (bytes == 4)
 +            *reg_data = 0;
 +
 +        memcpy(reg_data, ghcb->shared_buffer, bytes);
 +        break;
 +    case INSN_MMIO_READ_ZERO_EXTEND:
 +        ret = vc_do_mmio(ghcb, ctxt, bytes, true);
 +        if (ret)
 +            break;
 +
 +        /* Zero extend based on operand size */
 +        memset(reg_data, 0, insn->opnd_bytes);
 +        memcpy(reg_data, ghcb->shared_buffer, bytes);
 +        break;
 +    case INSN_MMIO_READ_SIGN_EXTEND:
 +        ret = vc_do_mmio(ghcb, ctxt, bytes, true);
 +        if (ret)
 +            break;
 +
 +        if (bytes == 1) {
 +            u8 *val = (u8 *)ghcb->shared_buffer;
 +
 +            sign_byte = (*val & 0x80) ? 0xff : 0x00;
 +        } else {
 +            u16 *val = (u16 *)ghcb->shared_buffer;
 +
 +            sign_byte = (*val & 0x8000) ? 0xff : 0x00;
 +        }
 +
 +        /* Sign extend based on operand size */
 +        memset(reg_data, sign_byte, insn->opnd_bytes);
 +        memcpy(reg_data, ghcb->shared_buffer, bytes);
 +        break;
 +    case INSN_MMIO_MOVS:
 +        ret = vc_handle_mmio_movs(ctxt, bytes);
 +        break;
 +    default:
 +        ret = ES_UNSUPPORTED;
 +        break;
 +    }
 +
 +    return ret;
 +}
 +
 +static enum es_result vc_handle_dr7_write(struct ghcb *ghcb,
 +                      struct es_em_ctxt *ctxt)
 +{
 +    struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
 +    long val, *reg = vc_insn_get_rm(ctxt);
 +    enum es_result ret;
 +
 +    if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
 +        return ES_VMM_ERROR;
 +
 +    if (!reg)
 +        return ES_DECODE_FAILED;
 +
 +    val = *reg;
 +
 +    /* Upper 32 bits must be written as zeroes */
 +    if (val >> 32) {
 +        ctxt->fi.vector = X86_TRAP_GP;
 +        ctxt->fi.error_code = 0;
 +        return ES_EXCEPTION;
 +    }
 +
 +    /* Clear out other reserved bits and set bit 10 */
 +    val = (val & 0xffff23ffL) | BIT(10);
 +
 +    /* Early non-zero writes to DR7 are not supported */
 +    if (!data && (val & ~DR7_RESET_VALUE))
 +        return ES_UNSUPPORTED;
 +
 +    /* Using a value of 0 for ExitInfo1 means RAX holds the value */
 +    ghcb_set_rax(ghcb, val);
 +    ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0);
 +    if (ret != ES_OK)
 +        return ret;
 +
 +    if (data)
 +        data->dr7 = val;
 +
 +    return ES_OK;
 +}
 +
 +static enum es_result vc_handle_dr7_read(struct ghcb *ghcb,
 +                     struct es_em_ctxt *ctxt)
 +{
 +    struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
 +    long *reg = vc_insn_get_rm(ctxt);
 +
 +    if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
 +        return ES_VMM_ERROR;
 +
 +    if (!reg)
 +        return ES_DECODE_FAILED;
 +
 +    if (data)
 +        *reg = data->dr7;
 +    else
 +        *reg = DR7_RESET_VALUE;
 +
 +    return ES_OK;
 +}
 +
 +static enum es_result vc_handle_wbinvd(struct ghcb *ghcb,
 +                       struct es_em_ctxt *ctxt)
 +{
 +    return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0);
 +}
 +
 +static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
 +{
 +    enum es_result ret;
 +
 +    ghcb_set_rcx(ghcb, ctxt->regs->cx);
 +
 +    ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0);
 +    if (ret != ES_OK)
 +        return ret;
 +
 +    if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb)))
 +        return ES_VMM_ERROR;
 +
 +    ctxt->regs->ax = ghcb->save.rax;
 +    ctxt->regs->dx = ghcb->save.rdx;
 +
 +    return ES_OK;
 +}
 +
 +static enum es_result vc_handle_monitor(struct ghcb *ghcb,
 +                    struct es_em_ctxt *ctxt)
 +{
 +    /*
 +     * Treat it as a NOP and do not leak a physical address to the
 +     * hypervisor.
 +     */
 +    return ES_OK;
 +}
 +
 +static enum es_result vc_handle_mwait(struct ghcb *ghcb,
 +                      struct es_em_ctxt *ctxt)
 +{
 +    /* Treat the same as MONITOR/MONITORX */
 +    return ES_OK;
 +}
 +
 +static enum es_result vc_handle_vmmcall(struct ghcb *ghcb,
 +                    struct es_em_ctxt *ctxt)
 +{
 +    enum es_result ret;
 +
 +    ghcb_set_rax(ghcb, ctxt->regs->ax);
 +    ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0);
 +
 +    if (x86_platform.hyper.sev_es_hcall_prepare)
 +        x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs);
 +
 +    ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0);
 +    if (ret != ES_OK)
 +        return ret;
 +
 +    if (!ghcb_rax_is_valid(ghcb))
 +        return ES_VMM_ERROR;
 +
 +    ctxt->regs->ax = ghcb->save.rax;
 +
 +    /*
 +     * Call sev_es_hcall_finish() after regs->ax is already set.
 +     * This allows the hypervisor handler to overwrite it again if
 +     * necessary.
 +     */
 +    if (x86_platform.hyper.sev_es_hcall_finish &&
 +        !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs))
 +        return ES_VMM_ERROR;
 +
 +    return ES_OK;
 +}
 +
 +static enum es_result vc_handle_trap_ac(struct ghcb *ghcb,
 +                    struct es_em_ctxt *ctxt)
 +{
 +    /*
 +     * Calling ecx_alignment_check() directly does not work, because it
 +     * enables IRQs and the GHCB is active. Forward the exception and call
 +     * it later from vc_forward_exception().
 +     */
 +    ctxt->fi.vector = X86_TRAP_AC;
 +    ctxt->fi.error_code = 0;
 +    return ES_EXCEPTION;
 +}
 +
 +static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
 +                     struct ghcb *ghcb,
 +                     unsigned long exit_code)
 +{
 +    enum es_result result = vc_check_opcode_bytes(ctxt, exit_code);
 +
 +    if (result != ES_OK)
 +        return result;
 +
 +    switch (exit_code) {
 +    case SVM_EXIT_READ_DR7:
 +        result = vc_handle_dr7_read(ghcb, ctxt);
 +        break;
 +    case SVM_EXIT_WRITE_DR7:
 +        result = vc_handle_dr7_write(ghcb, ctxt);
 +        break;
 +    case SVM_EXIT_EXCP_BASE + X86_TRAP_AC:
 +        result = vc_handle_trap_ac(ghcb, ctxt);
 +        break;
 +    case SVM_EXIT_RDTSC:
 +    case SVM_EXIT_RDTSCP:
 +        result = vc_handle_rdtsc(ghcb, ctxt, exit_code);
 +        break;
 +    case SVM_EXIT_RDPMC:
 +        result = vc_handle_rdpmc(ghcb, ctxt);
 +        break;
 +    case SVM_EXIT_INVD:
 +        pr_err_ratelimited("#VC exception for INVD??? Seriously???\n");
 +        result = ES_UNSUPPORTED;
 +        break;
 +    case SVM_EXIT_CPUID:
 +        result = vc_handle_cpuid(ghcb, ctxt);
 +        break;
 +    case SVM_EXIT_IOIO:
 +        result = vc_handle_ioio(ghcb, ctxt);
 +        break;
 +    case SVM_EXIT_MSR:
 +        result = vc_handle_msr(ghcb, ctxt);
 +        break;
 +    case SVM_EXIT_VMMCALL:
 +        result = vc_handle_vmmcall(ghcb, ctxt);
 +        break;
 +    case SVM_EXIT_WBINVD:
 +        result = vc_handle_wbinvd(ghcb, ctxt);
 +        break;
 +    case SVM_EXIT_MONITOR:
 +        result = vc_handle_monitor(ghcb, ctxt);
 +        break;
 +    case SVM_EXIT_MWAIT:
 +        result = vc_handle_mwait(ghcb, ctxt);
 +        break;
 +    case SVM_EXIT_NPF:
 +        result = vc_handle_mmio(ghcb, ctxt);
 +        break;
 +    default:
 +        /*
 +         * Unexpected #VC exception
 +         */
 +        result = ES_UNSUPPORTED;
 +    }
 +
 +    return result;
 +}
 +
 +static __always_inline bool is_vc2_stack(unsigned long sp)
 +{
 +    return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2));
 +}
 +
 +static __always_inline bool vc_from_invalid_context(struct pt_regs *regs)
 +{
 +    unsigned long sp, prev_sp;
 +
 +    sp      = (unsigned long)regs;
 +    prev_sp = regs->sp;
 +
 +    /*
 +     * If the code was already executing on the VC2 stack when the #VC
 +     * happened, let it proceed to the normal handling routine. This way the
 +     * code executing on the VC2 stack can cause #VC exceptions to get handled.
 +     */
 +    return is_vc2_stack(sp) && !is_vc2_stack(prev_sp);
 +}
 +
 +static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code)
 +{
 +    struct ghcb_state state;
 +    struct es_em_ctxt ctxt;
 +    enum es_result result;
 +    struct ghcb *ghcb;
 +    bool ret = true;
 +
 +    ghcb = __sev_get_ghcb(&state);
 +
 +    vc_ghcb_invalidate(ghcb);
 +    result = vc_init_em_ctxt(&ctxt, regs, error_code);
 +
 +    if (result == ES_OK)
 +        result = vc_handle_exitcode(&ctxt, ghcb, error_code);
 +
 +    __sev_put_ghcb(&state);
 +
 +    /* Done - now check the result */
 +    switch (result) {
 +    case ES_OK:
 +        vc_finish_insn(&ctxt);
 +        break;
 +    case ES_UNSUPPORTED:
 +        pr_err_ratelimited("Unsupported exit-code 0x%02lx in #VC exception (IP: 0x%lx)\n",
 +                   error_code, regs->ip);
 +        ret = false;
 +        break;
 +    case ES_VMM_ERROR:
 +        pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
 +                   error_code, regs->ip);
 +        ret = false;
 +        break;
 +    case ES_DECODE_FAILED:
 +        pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
 +                   error_code, regs->ip);
 +        ret = false;
 +        break;
 +    case ES_EXCEPTION:
 +        vc_forward_exception(&ctxt);
 +        break;
 +    case ES_RETRY:
 +        /* Nothing to do */
 +        break;
 +    default:
 +        pr_emerg("Unknown result in %s():%d\n", __func__, result);
 +        /*
 +         * Emulating the instruction which caused the #VC exception
 +         * failed - can't continue so print debug information
 +         */
 +        BUG();
 +    }
 +
 +    return ret;
 +}
 +
 +static __always_inline bool vc_is_db(unsigned long error_code)
 +{
 +    return error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB;
 +}
 +
 +/*
 + * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode
 + * and will panic when an error happens.
 + */
 +DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication)
 +{
 +    irqentry_state_t irq_state;
 +
 +    /*
 +     * With the current implementation it is always possible to switch to a
 +     * safe stack because #VC exceptions only happen at known places, like
 +     * intercepted instructions or accesses to MMIO areas/IO ports. They can
 +     * also happen with code instrumentation when the hypervisor intercepts
 +     * #DB, but the critical paths are forbidden to be instrumented, so #DB
 +     * exceptions currently also only happen in safe places.
 +     *
 +     * But keep this here in case the noinstr annotations are violated due
 +     * to bug elsewhere.
 +     */
 +    if (unlikely(vc_from_invalid_context(regs))) {
 +        instrumentation_begin();
 +        panic("Can't handle #VC exception from unsupported context\n");
 +        instrumentation_end();
 +    }
 +
 +    /*
 +     * Handle #DB before calling into !noinstr code to avoid recursive #DB.
 +     */
 +    if (vc_is_db(error_code)) {
 +        exc_debug(regs);
 +        return;
 +    }
 +
 +    irq_state = irqentry_nmi_enter(regs);
 +
 +    instrumentation_begin();
 +
 +    if (!vc_raw_handle_exception(regs, error_code)) {
 +        /* Show some debug info */
 +        show_regs(regs);
 +
 +        /* Ask hypervisor to sev_es_terminate */
 +        sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
 +
 +        /* If that fails and we get here - just panic */
 +        panic("Returned from Terminate-Request to Hypervisor\n");
 +    }
 +
 +    instrumentation_end();
 +    irqentry_nmi_exit(regs, irq_state);
 +}
 +
 +/*
 + * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode
 + * and will kill the current task with SIGBUS when an error happens.
 + */
 +DEFINE_IDTENTRY_VC_USER(exc_vmm_communication)
 +{
 +    /*
 +     * Handle #DB before calling into !noinstr code to avoid recursive #DB.
 +     */
 +    if (vc_is_db(error_code)) {
 +        noist_exc_debug(regs);
 +        return;
 +    }
 +
 +    irqentry_enter_from_user_mode(regs);
 +    instrumentation_begin();
 +
 +    if (!vc_raw_handle_exception(regs, error_code)) {
 +        /*
 +         * Do not kill the machine if user-space triggered the
 +         * exception. Send SIGBUS instead and let user-space deal with
 +         * it.
 +         */
 +        force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0);
 +    }
 +
 +    instrumentation_end();
 +    irqentry_exit_to_user_mode(regs);
 +}
 +
 +bool __init handle_vc_boot_ghcb(struct pt_regs *regs)
 +{
 +    unsigned long exit_code = regs->orig_ax;
 +    struct es_em_ctxt ctxt;
 +    enum es_result result;
 +
 +    vc_ghcb_invalidate(boot_ghcb);
 +
 +    result = vc_init_em_ctxt(&ctxt, regs, exit_code);
 +    if (result == ES_OK)
 +        result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code);
 +
 +    /* Done - now check the result */
 +    switch (result) {
 +    case ES_OK:
 +        vc_finish_insn(&ctxt);
 +        break;
 +    case ES_UNSUPPORTED:
 +        early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n",
 +                exit_code, regs->ip);
 +        goto fail;
 +    case ES_VMM_ERROR:
 +        early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
 +                exit_code, regs->ip);
 +        goto fail;
 +    case ES_DECODE_FAILED:
 +        early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
 +                exit_code, regs->ip);
 +        goto fail;
 +    case ES_EXCEPTION:
 +        vc_early_forward_exception(&ctxt);
 +        break;
 +    case ES_RETRY:
 +        /* Nothing to do */
 +        break;
 +    default:
 +        BUG();
 +    }
 +
 +    return true;
 +
 +fail:
 +    show_regs(regs);
 +
 +    sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
 +}
 +
 +/*
 + * Initial set up of SNP relies on information provided by the
 + * Confidential Computing blob, which can be passed to the kernel
 + * in the following ways, depending on how it is booted:
 + *
 + * - when booted via the boot/decompress kernel:
 + *   - via boot_params
 + *
 + * - when booted directly by firmware/bootloader (e.g. CONFIG_PVH):
 + *   - via a setup_data entry, as defined by the Linux Boot Protocol
 + *
 + * Scan for the blob in that order.
 + */
 +static __head struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
 +{
 +    struct cc_blob_sev_info *cc_info;
 +
 +    /* Boot kernel would have passed the CC blob via boot_params. */
 +    if (bp->cc_blob_address) {
 +        cc_info = (struct cc_blob_sev_info *)(unsigned long)bp->cc_blob_address;
 +        goto found_cc_info;
 +    }
 +
 +    /*
 +     * If kernel was booted directly, without the use of the
 +     * boot/decompression kernel, the CC blob may have been passed via
 +     * setup_data instead.
 +     */
 +    cc_info = find_cc_blob_setup_data(bp);
 +    if (!cc_info)
 +        return NULL;
 +
 +found_cc_info:
 +    if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
 +        snp_abort();
 +
 +    return cc_info;
 +}
 +
 +static __head void svsm_setup(struct cc_blob_sev_info *cc_info)
 +{
 +    struct svsm_call call = {};
 +    int ret;
 +    u64 pa;
 +
 +    /*
 +     * Record the SVSM Calling Area address (CAA) if the guest is not
 +     * running at VMPL0. The CA will be used to communicate with the
 +     * SVSM to perform the SVSM services.
 +     */
 +    if (!svsm_setup_ca(cc_info))
 +        return;
 +
 +    /*
 +     * It is very early in the boot and the kernel is running identity
 +     * mapped but without having adjusted the pagetables to where the
 +     * kernel was loaded (physbase), so the get the CA address using
 +     * RIP-relative addressing.
 +     */
 +    pa = (u64)rip_rel_ptr(&boot_svsm_ca_page);
 +
 +    /*
 +     * Switch over to the boot SVSM CA while the current CA is still
 +     * addressable. There is no GHCB at this point so use the MSR protocol.
 +     *
 +     * SVSM_CORE_REMAP_CA call:
 +     *   RAX = 0 (Protocol=0, CallID=0)
 +     *   RCX = New CA GPA
 +     */
 +    call.caa = svsm_get_caa();
 +    call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA);
 +    call.rcx = pa;
 +    ret = svsm_perform_call_protocol(&call);
 +    if (ret)
 +        sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CA_REMAP_FAIL);
 +
 +    RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)pa;
 +    RIP_REL_REF(boot_svsm_caa_pa) = pa;
 +}
 +
 +bool __head snp_init(struct boot_params *bp)
 +{
 +    struct cc_blob_sev_info *cc_info;
 +
 +    if (!bp)
 +        return false;
 +
 +    cc_info = find_cc_blob(bp);
 +    if (!cc_info)
 +        return false;
 +
 +    if (cc_info->secrets_phys && cc_info->secrets_len == PAGE_SIZE)
 +        secrets_pa = cc_info->secrets_phys;
 +    else
 +        return false;
 +
 +    setup_cpuid_table(cc_info);
 +
 +    svsm_setup(cc_info);
 +
 +    /*
 +     * The CC blob will be used later to access the secrets page. Cache
 +     * it here like the boot kernel does.
 +     */
 +    bp->cc_blob_address = (u32)(unsigned long)cc_info;
 +
 +    return true;
 +}
 +
 +void __head __noreturn snp_abort(void)
 +{
 +    sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
 +}
 --
-.49.0.472.ge94155a9ec-goog
+.49.0.504.g3bcea36a83-goog

-[RFC PATCH 3/6] x86/boot: Move EFI mixed mode startup code back under arch/x86
+[PATCH v4 09/11] x86/boot: Move SEV startup code into startup/
 From: Ard Biesheuvel <ardb@kernel.org>
-Linus expressed a strong preference for arch-specific asm code (i.e.,
+Move the SEV startup code into arch/x86/boot/startup/, where it will
-virtually all of it) to reside under arch/ rather than anywhere else.
+reside along with other code that executes extremely early, and
+therefore needs to be built in a special manner.
 So move the EFI mixed mode startup code back, and put it under
 arch/x86/boot/startup/ where all shared x86 startup code is going to
 live.
 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
 ---
- arch/x86/boot/startup/Makefile                                                | 3 +++
+ arch/x86/boot/compressed/sev.c                              |  2 +-
- drivers/firmware/efi/libstub/x86-mixed.S => arch/x86/boot/startup/efi-mixed.S | 0
+ arch/x86/boot/startup/Makefile                              |  2 +-
- drivers/firmware/efi/libstub/Makefile                                         | 1 -
+ arch/x86/{coco/sev/shared.c => boot/startup/sev-shared.c}   |  0
-files changed, 3 insertions(+), 1 deletion(-)
+ arch/x86/{coco/sev/startup.c => boot/startup/sev-startup.c} |  2 +-
  arch/x86/coco/sev/Makefile                                  | 21 +-------------------
 files changed, 4 insertions(+), 23 deletions(-)
+diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c
+index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/boot/compressed/sev.c
++++ b/arch/x86/boot/compressed/sev.c
+@@ -XXX,XX +XXX,XX @@ int svsm_perform_call_protocol(struct svsm_call *call);
+ u8 snp_vmpl;
+ /* Include code for early handlers */
+-#include "../../coco/sev/shared.c"
++#include "../../boot/startup/sev-shared.c"
+ int svsm_perform_call_protocol(struct svsm_call *call)
+ {
 diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
 index XXXXXXX..XXXXXXX 100644
 --- a/arch/x86/boot/startup/Makefile
 +++ b/arch/x86/boot/startup/Makefile
+@@ -XXX,XX +XXX,XX @@ UBSAN_SANITIZE    := n
+ KCOV_INSTRUMENT    := n
+ obj-$(CONFIG_X86_64)        += gdt_idt.o map_kernel.o
+-obj-$(CONFIG_AMD_MEM_ENCRYPT)    += sme.o
++obj-$(CONFIG_AMD_MEM_ENCRYPT)    += sme.o sev-startup.o
+ lib-$(CONFIG_X86_64)        += la57toggle.o
+ lib-$(CONFIG_EFI_MIXED)        += efi-mixed.o
+diff --git a/arch/x86/coco/sev/shared.c b/arch/x86/boot/startup/sev-shared.c
+similarity index 100%
+rename from arch/x86/coco/sev/shared.c
+rename to arch/x86/boot/startup/sev-shared.c
+diff --git a/arch/x86/coco/sev/startup.c b/arch/x86/boot/startup/sev-startup.c
+similarity index 99%
+rename from arch/x86/coco/sev/startup.c
+rename to arch/x86/boot/startup/sev-startup.c
+index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/coco/sev/startup.c
++++ b/arch/x86/boot/startup/sev-startup.c
+@@ -XXX,XX +XXX,XX @@ static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt)
+ }
+ /* Include code shared with pre-decompression boot stage */
+-#include "shared.c"
++#include "sev-shared.c"
+ noinstr void __sev_put_ghcb(struct ghcb_state *state)
+ {
+diff --git a/arch/x86/coco/sev/Makefile b/arch/x86/coco/sev/Makefile
+index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/coco/sev/Makefile
++++ b/arch/x86/coco/sev/Makefile
 @@ -XXX,XX +XXX,XX @@
  # SPDX-License-Identifier: GPL-2.0
-+KBUILD_AFLAGS        += -D__DISABLE_EXPORTS
+-obj-y += core.o startup.o
-+
+-
- lib-$(CONFIG_X86_64)        += la57toggle.o
+-# jump tables are emitted using absolute references in non-PIC code
-+lib-$(CONFIG_EFI_MIXED)        += efi-mixed.o
+-# so they cannot be used in the early SEV startup code
-diff --git a/drivers/firmware/efi/libstub/x86-mixed.S b/arch/x86/boot/startup/efi-mixed.S
+-CFLAGS_startup.o += -fno-jump-tables
-similarity index 100%
+-
-rename from drivers/firmware/efi/libstub/x86-mixed.S
+-ifdef CONFIG_FUNCTION_TRACER
-rename to arch/x86/boot/startup/efi-mixed.S
+-CFLAGS_REMOVE_startup.o = -pg
-diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
+-endif
-index XXXXXXX..XXXXXXX 100644
+-
---- a/drivers/firmware/efi/libstub/Makefile
+-KASAN_SANITIZE_startup.o    := n
-+++ b/drivers/firmware/efi/libstub/Makefile
+-KMSAN_SANITIZE_startup.o    := n
-@@ -XXX,XX +XXX,XX @@ lib-$(CONFIG_EFI_GENERIC_STUB)    += efi-stub.o string.o intrinsics.o systable.o \
+-KCOV_INSTRUMENT_startup.o    := n
- lib-$(CONFIG_ARM)        += arm32-stub.o
+-
- lib-$(CONFIG_ARM64)        += kaslr.o arm64.o arm64-stub.o smbios.o
+-# With some compiler versions the generated code results in boot hangs, caused
- lib-$(CONFIG_X86)        += x86-stub.o smbios.o
+-# by several compilation units. To be safe, disable all instrumentation.
--lib-$(CONFIG_EFI_MIXED)        += x86-mixed.o
+-KCSAN_SANITIZE        := n
- lib-$(CONFIG_X86_64)        += x86-5lvl.o
+-
- lib-$(CONFIG_RISCV)        += kaslr.o riscv.o riscv-stub.o
+-# Clang 14 and older may fail to respect __no_sanitize_undefined when inlining
- lib-$(CONFIG_LOONGARCH)        += loongarch.o loongarch-stub.o
+-UBSAN_SANITIZE        := n
 +obj-y += core.o
 --
-.49.0.472.ge94155a9ec-goog
+.49.0.504.g3bcea36a83-goog

-New patch
+[PATCH v4 10/11] x86/boot: Drop RIP_REL_REF() uses from early SEV code
+From: Ard Biesheuvel <ardb@kernel.org>
+Now that the early SEV code is built with -fPIC, RIP_REL_REF() has no
+effect and can be dropped.
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+---
+ arch/x86/boot/startup/sev-shared.c  | 26 +++++++++-----------
+ arch/x86/boot/startup/sev-startup.c | 16 ++++++------
+ arch/x86/include/asm/sev-internal.h | 18 +++-----------
+files changed, 23 insertions(+), 37 deletions(-)
+diff --git a/arch/x86/boot/startup/sev-shared.c b/arch/x86/boot/startup/sev-shared.c
+index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/boot/startup/sev-shared.c
++++ b/arch/x86/boot/startup/sev-shared.c
+@@ -XXX,XX +XXX,XX @@ static int svsm_perform_ghcb_protocol(struct ghcb *ghcb, struct svsm_call *call)
+      * Fill in protocol and format specifiers. This can be called very early
+      * in the boot, so use rip-relative references as needed.
+      */
+-    ghcb->protocol_version = RIP_REL_REF(ghcb_version);
++    ghcb->protocol_version = ghcb_version;
+     ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
+     ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_SNP_RUN_VMPL);
+@@ -XXX,XX +XXX,XX @@ snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
+         leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0;
+         /* Skip post-processing for out-of-range zero leafs. */
+-        if (!(leaf->fn <= RIP_REL_REF(cpuid_std_range_max) ||
+-              (leaf->fn >= 0x40000000 && leaf->fn <= RIP_REL_REF(cpuid_hyp_range_max)) ||
+-              (leaf->fn >= 0x80000000 && leaf->fn <= RIP_REL_REF(cpuid_ext_range_max))))
++        if (!(leaf->fn <= cpuid_std_range_max ||
++              (leaf->fn >= 0x40000000 && leaf->fn <= cpuid_hyp_range_max) ||
++              (leaf->fn >= 0x80000000 && leaf->fn <= cpuid_ext_range_max)))
+             return 0;
+     }
+@@ -XXX,XX +XXX,XX @@ static void __head setup_cpuid_table(const struct cc_blob_sev_info *cc_info)
+         const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
+         if (fn->eax_in == 0x0)
+-            RIP_REL_REF(cpuid_std_range_max) = fn->eax;
++            cpuid_std_range_max = fn->eax;
+         else if (fn->eax_in == 0x40000000)
+-            RIP_REL_REF(cpuid_hyp_range_max) = fn->eax;
++            cpuid_hyp_range_max = fn->eax;
+         else if (fn->eax_in == 0x80000000)
+-            RIP_REL_REF(cpuid_ext_range_max) = fn->eax;
++            cpuid_ext_range_max = fn->eax;
+     }
+ }
+@@ -XXX,XX +XXX,XX @@ static void __head pvalidate_4k_page(unsigned long vaddr, unsigned long paddr,
+ {
+     int ret;
+-    /*
+-     * This can be called very early during boot, so use rIP-relative
+-     * references as needed.
+-     */
+-    if (RIP_REL_REF(snp_vmpl)) {
++    if (snp_vmpl) {
+         svsm_pval_4k_page(paddr, validate);
+     } else {
+         ret = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
+@@ -XXX,XX +XXX,XX @@ static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info)
+     if (!secrets_page->svsm_guest_vmpl)
+         sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_VMPL0);
+-    RIP_REL_REF(snp_vmpl) = secrets_page->svsm_guest_vmpl;
++    snp_vmpl = secrets_page->svsm_guest_vmpl;
+     caa = secrets_page->svsm_caa;
+@@ -XXX,XX +XXX,XX @@ static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info)
+      * The CA is identity mapped when this routine is called, both by the
+      * decompressor code and the early kernel code.
+      */
+-    RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)caa;
+-    RIP_REL_REF(boot_svsm_caa_pa) = caa;
++    boot_svsm_caa = (struct svsm_ca *)caa;
++    boot_svsm_caa_pa = caa;
+     /* Advertise the SVSM presence via CPUID. */
+     cpuid_table = (struct snp_cpuid_table *)snp_cpuid_get_table();
+diff --git a/arch/x86/boot/startup/sev-startup.c b/arch/x86/boot/startup/sev-startup.c
+index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/boot/startup/sev-startup.c
++++ b/arch/x86/boot/startup/sev-startup.c
+@@ -XXX,XX +XXX,XX @@ int svsm_perform_call_protocol(struct svsm_call *call)
+      * ghcbs_initialized is set, then it is late in the boot and no need
+      * to worry about rip-relative references in called functions.
+      */
+-    if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
++    if (sev_cfg.ghcbs_initialized)
+         ghcb = __sev_get_ghcb(&state);
+-    else if (RIP_REL_REF(boot_ghcb))
+-        ghcb = RIP_REL_REF(boot_ghcb);
++    else if (boot_ghcb)
++        ghcb = boot_ghcb;
+     else
+         ghcb = NULL;
+@@ -XXX,XX +XXX,XX @@ int svsm_perform_call_protocol(struct svsm_call *call)
+                : svsm_perform_msr_protocol(call);
+     } while (ret == -EAGAIN);
+-    if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
++    if (sev_cfg.ghcbs_initialized)
+         __sev_put_ghcb(&state);
+     native_local_irq_restore(flags);
+@@ -XXX,XX +XXX,XX @@ void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long padd
+      * This eliminates worries about jump tables or checking boot_cpu_data
+      * in the cc_platform_has() function.
+      */
+-    if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
++    if (!(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
+         return;
+      /*
+@@ -XXX,XX +XXX,XX @@ void __head early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr
+      * This eliminates worries about jump tables or checking boot_cpu_data
+      * in the cc_platform_has() function.
+      */
+-    if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
++    if (!(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
+         return;
+      /* Ask hypervisor to mark the memory pages shared in the RMP table. */
+@@ -XXX,XX +XXX,XX @@ static __head void svsm_setup(struct cc_blob_sev_info *cc_info)
+     if (ret)
+         sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CA_REMAP_FAIL);
+-    RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)pa;
+-    RIP_REL_REF(boot_svsm_caa_pa) = pa;
++    boot_svsm_caa = (struct svsm_ca *)pa;
++    boot_svsm_caa_pa = pa;
+ }
+ bool __head snp_init(struct boot_params *bp)
+diff --git a/arch/x86/include/asm/sev-internal.h b/arch/x86/include/asm/sev-internal.h
+index XXXXXXX..XXXXXXX 100644
+--- a/arch/x86/include/asm/sev-internal.h
++++ b/arch/x86/include/asm/sev-internal.h
+@@ -XXX,XX +XXX,XX @@ extern u64 boot_svsm_caa_pa;
+ static __always_inline struct svsm_ca *svsm_get_caa(void)
+ {
+-    /*
+-     * Use rIP-relative references when called early in the boot. If
+-     * ->use_cas is set, then it is late in the boot and no need
+-     * to worry about rIP-relative references.
+-     */
+-    if (RIP_REL_REF(sev_cfg).use_cas)
++    if (sev_cfg.use_cas)
+         return this_cpu_read(svsm_caa);
+     else
+-        return RIP_REL_REF(boot_svsm_caa);
++        return boot_svsm_caa;
+ }
+ static __always_inline u64 svsm_get_caa_pa(void)
+ {
+-    /*
+-     * Use rIP-relative references when called early in the boot. If
+-     * ->use_cas is set, then it is late in the boot and no need
+-     * to worry about rIP-relative references.
+-     */
+-    if (RIP_REL_REF(sev_cfg).use_cas)
++    if (sev_cfg.use_cas)
+         return this_cpu_read(svsm_caa_pa);
+     else
+-        return RIP_REL_REF(boot_svsm_caa_pa);
++        return boot_svsm_caa_pa;
+ }
+ int svsm_perform_call_protocol(struct svsm_call *call);
+--
+.49.0.504.g3bcea36a83-goog

-[RFC PATCH 1/6] x86/boot/compressed: Merge local pgtable.h include into asm/boot.h
+[PATCH v4 11/11] x86/asm: Retire RIP_REL_REF()
 From: Ard Biesheuvel <ardb@kernel.org>
-Merge the local include "pgtable.h" -which declares the API of the
+Now that all users have been moved into startup/ where PIC codegen is
--level paging trampoline- into <asm/boot.h> so that its implementation
+used, RIP_REL_REF() is no longer needed. Remove it.
 in la57toggle.S as well as the calling code can be decoupled from the
 traditional decompressor.
 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
 ---
- arch/x86/boot/compressed/head_64.S    |  1 -
+ arch/x86/include/asm/asm.h | 5 -----
- arch/x86/boot/compressed/la57toggle.S |  1 -
+file changed, 5 deletions(-)
  arch/x86/boot/compressed/misc.c       |  1 -
  arch/x86/boot/compressed/pgtable.h    | 18 ------------------
  arch/x86/boot/compressed/pgtable_64.c |  1 -
  arch/x86/include/asm/boot.h           | 10 ++++++++++
 files changed, 10 insertions(+), 22 deletions(-)
-diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S
+diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h
 index XXXXXXX..XXXXXXX 100644
---- a/arch/x86/boot/compressed/head_64.S
+--- a/arch/x86/include/asm/asm.h
-+++ b/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/include/asm/asm.h
-@@ -XXX,XX +XXX,XX @@
+@@ -XXX,XX +XXX,XX @@ static __always_inline __pure void *rip_rel_ptr(void *p)
- #include <asm/bootparam.h>
- #include <asm/desc_defs.h>
+     return p;
- #include <asm/trapnr.h>
+ }
--#include "pgtable.h"
+-#ifndef __pic__
 -#define RIP_REL_REF(var)    (*(typeof(&(var)))rip_rel_ptr(&(var)))
 -#else
 -#define RIP_REL_REF(var)    (var)
 -#endif
  #endif
  /*
-  * Fix alignment at 16 bytes. Following CONFIG_FUNCTION_ALIGNMENT will result
-diff --git a/arch/x86/boot/compressed/la57toggle.S b/arch/x86/boot/compressed/la57toggle.S
-index XXXXXXX..XXXXXXX 100644
---- a/arch/x86/boot/compressed/la57toggle.S
-+++ b/arch/x86/boot/compressed/la57toggle.S
-@@ -XXX,XX +XXX,XX @@
- #include <asm/boot.h>
- #include <asm/msr.h>
- #include <asm/processor-flags.h>
--#include "pgtable.h"
- /*
-  * This is the 32-bit trampoline that will be copied over to low memory. It
-diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
-index XXXXXXX..XXXXXXX 100644
---- a/arch/x86/boot/compressed/misc.c
-+++ b/arch/x86/boot/compressed/misc.c
-@@ -XXX,XX +XXX,XX @@
- #include "misc.h"
- #include "error.h"
--#include "pgtable.h"
- #include "../string.h"
- #include "../voffset.h"
- #include <asm/bootparam_utils.h>
-diff --git a/arch/x86/boot/compressed/pgtable.h b/arch/x86/boot/compressed/pgtable.h
-deleted file mode 100644
-index XXXXXXX..XXXXXXX
---- a/arch/x86/boot/compressed/pgtable.h
-+++ /dev/null
-@@ -XXX,XX +XXX,XX @@
--#ifndef BOOT_COMPRESSED_PAGETABLE_H
--#define BOOT_COMPRESSED_PAGETABLE_H
--
--#define TRAMPOLINE_32BIT_SIZE        (2 * PAGE_SIZE)
--
--#define TRAMPOLINE_32BIT_CODE_OFFSET    PAGE_SIZE
--#define TRAMPOLINE_32BIT_CODE_SIZE    0xA0
--
--#ifndef __ASSEMBLER__
--
--extern unsigned long *trampoline_32bit;
--
--extern void trampoline_32bit_src(void *trampoline, bool enable_5lvl);
--
--extern const u16 trampoline_ljmp_imm_offset;
--
--#endif /* __ASSEMBLER__ */
--#endif /* BOOT_COMPRESSED_PAGETABLE_H */
-diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c
-index XXXXXXX..XXXXXXX 100644
---- a/arch/x86/boot/compressed/pgtable_64.c
-+++ b/arch/x86/boot/compressed/pgtable_64.c
-@@ -XXX,XX +XXX,XX @@
- #include <asm/bootparam_utils.h>
- #include <asm/e820/types.h>
- #include <asm/processor.h>
--#include "pgtable.h"
- #include "../string.h"
- #include "efi.h"
-diff --git a/arch/x86/include/asm/boot.h b/arch/x86/include/asm/boot.h
-index XXXXXXX..XXXXXXX 100644
---- a/arch/x86/include/asm/boot.h
-+++ b/arch/x86/include/asm/boot.h
-@@ -XXX,XX +XXX,XX @@
- # define BOOT_STACK_SIZE    0x1000
- #endif
-+#define TRAMPOLINE_32BIT_SIZE        (2 * PAGE_SIZE)
-+
-+#define TRAMPOLINE_32BIT_CODE_OFFSET    PAGE_SIZE
-+#define TRAMPOLINE_32BIT_CODE_SIZE    0xA0
-+
- #ifndef __ASSEMBLER__
- extern unsigned int output_len;
- extern const unsigned long kernel_text_size;
-@@ -XXX,XX +XXX,XX @@ unsigned long decompress_kernel(unsigned char *outbuf, unsigned long virt_addr,
-                 void (*error)(char *x));
- extern struct boot_params *boot_params_ptr;
-+extern unsigned long *trampoline_32bit;
-+extern const u16 trampoline_ljmp_imm_offset;
-+
-+void trampoline_32bit_src(void *trampoline, bool enable_5lvl);
-+
- #endif
- #endif /* _ASM_X86_BOOT_H */
 --
-.49.0.472.ge94155a9ec-goog
+.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

Start refactoring the x86 startup code so we keep all the code that is
shared between different boot stages (EFI stub, decompressor, early
startup in the core kernel *) and/or needs to be built in a special way
(due to the fact that it is C code that runs from the 1:1 mapping of
RAM) in a single place, sharing all the C flags and other runes that are
needed to disable instrumentation, sanitizers, etc.

This is an RFC so I have left some things for later, e.g., the SEV-SNP
init code in arch/x86/coco that is shared between all of the above [*]
and will be tricky to disentangle; there are also some known issues in
that code related to EFI boot that we are addressing in parallel.

Cc: Tom Lendacky <thomas.lendacky@amd.com>
Cc: Dionna Amalie Glaze <dionnaglaze@google.com>
Cc: Kevin Loughlin <kevinloughlin@google.com>

Ard Biesheuvel (6):
  x86/boot/compressed: Merge local pgtable.h include into asm/boot.h
  x86/boot: Move 5-level paging trampoline into startup code
  x86/boot: Move EFI mixed mode startup code back under arch/x86
  x86/boot: Move early GDT/IDT setup code into startup/
  x86/boot: Move early kernel mapping code into startup/
  x86/boot: Move early SME init code into startup/

arch/x86/Makefile                                                             |   1 +
 arch/x86/boot/compressed/Makefile                                             |   4 +-
 arch/x86/boot/compressed/head_64.S                                            |   1 -
 arch/x86/boot/compressed/misc.c                                               |   1 -
 arch/x86/boot/compressed/pgtable.h                                            |  18 --
 arch/x86/boot/compressed/pgtable_64.c                                         |   1 -
 arch/x86/boot/startup/Makefile                                                |  22 ++
 drivers/firmware/efi/libstub/x86-mixed.S => arch/x86/boot/startup/efi-mixed.S |   0
 arch/x86/boot/startup/gdt_idt.c                                               |  82 ++++++
 arch/x86/boot/{compressed => startup}/la57toggle.S                            |   1 -
 arch/x86/boot/startup/map_kernel.c                                            | 232 +++++++++++++++
 arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c}                    |  45 ++-
 arch/x86/include/asm/boot.h                                                   |  10 +
 arch/x86/include/asm/mem_encrypt.h                                            |   2 +-
 arch/x86/kernel/head64.c                                                      | 302 +-------------------
 arch/x86/mm/Makefile                                                          |   6 -
 drivers/firmware/efi/libstub/Makefile                                         |   1 -
 17 files changed, 372 insertions(+), 357 deletions(-)
 delete mode 100644 arch/x86/boot/compressed/pgtable.h
 create mode 100644 arch/x86/boot/startup/Makefile
 rename drivers/firmware/efi/libstub/x86-mixed.S => arch/x86/boot/startup/efi-mixed.S (100%)
 create mode 100644 arch/x86/boot/startup/gdt_idt.c
 rename arch/x86/boot/{compressed => startup}/la57toggle.S (99%)
 create mode 100644 arch/x86/boot/startup/map_kernel.c
 rename arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} (92%)

-- 
2.49.0.472.ge94155a9ec-goog

From: Ard Biesheuvel <ardb@kernel.org>

Merge the local include "pgtable.h" -which declares the API of the
5-level paging trampoline- into <asm/boot.h> so that its implementation
in la57toggle.S as well as the calling code can be decoupled from the
traditional decompressor.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/compressed/head_64.S    |  1 -
 arch/x86/boot/compressed/la57toggle.S |  1 -
 arch/x86/boot/compressed/misc.c       |  1 -
 arch/x86/boot/compressed/pgtable.h    | 18 ------------------
 arch/x86/boot/compressed/pgtable_64.c |  1 -
 arch/x86/include/asm/boot.h           | 10 ++++++++++
 6 files changed, 10 insertions(+), 22 deletions(-)

diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/head_64.S
+++ b/arch/x86/boot/compressed/head_64.S
@@ -XXX,XX +XXX,XX @@
 #include <asm/bootparam.h>
 #include <asm/desc_defs.h>
 #include <asm/trapnr.h>
-#include "pgtable.h"
 
 /*
  * Fix alignment at 16 bytes. Following CONFIG_FUNCTION_ALIGNMENT will result
diff --git a/arch/x86/boot/compressed/la57toggle.S b/arch/x86/boot/compressed/la57toggle.S
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/la57toggle.S
+++ b/arch/x86/boot/compressed/la57toggle.S
@@ -XXX,XX +XXX,XX @@
 #include <asm/boot.h>
 #include <asm/msr.h>
 #include <asm/processor-flags.h>
-#include "pgtable.h"
 
 /*
  * This is the 32-bit trampoline that will be copied over to low memory. It
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -XXX,XX +XXX,XX @@
 
 #include "misc.h"
 #include "error.h"
-#include "pgtable.h"
 #include "../string.h"
 #include "../voffset.h"
 #include <asm/bootparam_utils.h>
diff --git a/arch/x86/boot/compressed/pgtable.h b/arch/x86/boot/compressed/pgtable.h
deleted file mode 100644
index XXXXXXX..XXXXXXX
--- a/arch/x86/boot/compressed/pgtable.h
+++ /dev/null
@@ -XXX,XX +XXX,XX @@
-#ifndef BOOT_COMPRESSED_PAGETABLE_H
-#define BOOT_COMPRESSED_PAGETABLE_H
-
-#define TRAMPOLINE_32BIT_SIZE		(2 * PAGE_SIZE)
-
-#define TRAMPOLINE_32BIT_CODE_OFFSET	PAGE_SIZE
-#define TRAMPOLINE_32BIT_CODE_SIZE	0xA0
-
-#ifndef __ASSEMBLER__
-
-extern unsigned long *trampoline_32bit;
-
-extern void trampoline_32bit_src(void *trampoline, bool enable_5lvl);
-
-extern const u16 trampoline_ljmp_imm_offset;
-
-#endif /* __ASSEMBLER__ */
-#endif /* BOOT_COMPRESSED_PAGETABLE_H */
diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/pgtable_64.c
+++ b/arch/x86/boot/compressed/pgtable_64.c
@@ -XXX,XX +XXX,XX @@
 #include <asm/bootparam_utils.h>
 #include <asm/e820/types.h>
 #include <asm/processor.h>
-#include "pgtable.h"
 #include "../string.h"
 #include "efi.h"
 
diff --git a/arch/x86/include/asm/boot.h b/arch/x86/include/asm/boot.h
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/include/asm/boot.h
+++ b/arch/x86/include/asm/boot.h
@@ -XXX,XX +XXX,XX @@
 # define BOOT_STACK_SIZE	0x1000
 #endif
 
+#define TRAMPOLINE_32BIT_SIZE		(2 * PAGE_SIZE)
+
+#define TRAMPOLINE_32BIT_CODE_OFFSET	PAGE_SIZE
+#define TRAMPOLINE_32BIT_CODE_SIZE	0xA0
+
 #ifndef __ASSEMBLER__
 extern unsigned int output_len;
 extern const unsigned long kernel_text_size;
@@ -XXX,XX +XXX,XX @@ unsigned long decompress_kernel(unsigned char *outbuf, unsigned long virt_addr,
 				void (*error)(char *x));
 
 extern struct boot_params *boot_params_ptr;
+extern unsigned long *trampoline_32bit;
+extern const u16 trampoline_ljmp_imm_offset;
+
+void trampoline_32bit_src(void *trampoline, bool enable_5lvl);
+
 #endif
 
 #endif /* _ASM_X86_BOOT_H */
-- 
2.49.0.472.ge94155a9ec-goog

From: Ard Biesheuvel <ardb@kernel.org>

The 5-level paging trampoline is used by both the EFI stub and the
traditional decompressor. Move it out of the decompressor sources into
the newly minted arch/x86/boot/startup/ sub-directory which will hold
startup code that may be shared between the decompressor, the EFI stub
and the kernel proper, and needs to tolerate being called during early
boot, before the kernel virtual mapping has been created.

This will allow the 5-level paging trampoline to be used by EFI boot
images such as zboot that omit the traditional decompressor entirely.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/Makefile                                  | 1 +
 arch/x86/boot/compressed/Makefile                  | 2 +-
 arch/x86/boot/startup/Makefile                     | 3 +++
 arch/x86/boot/{compressed => startup}/la57toggle.S | 0
 4 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -XXX,XX +XXX,XX @@ archprepare: $(cpufeaturemasks.hdr)
 ###
 # Kernel objects
 
+core-y  += arch/x86/boot/startup/
 libs-y  += arch/x86/lib/
 
 # drivers-y are linked after core-y
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -XXX,XX +XXX,XX @@ ifdef CONFIG_X86_64
 	vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/mem_encrypt.o
 	vmlinux-objs-y += $(obj)/pgtable_64.o
 	vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev.o
-	vmlinux-objs-y += $(obj)/la57toggle.o
 endif
 
 vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o
@@ -XXX,XX +XXX,XX @@ vmlinux-objs-$(CONFIG_UNACCEPTED_MEMORY) += $(obj)/mem.o
 
 vmlinux-objs-$(CONFIG_EFI) += $(obj)/efi.o
 vmlinux-libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a
+vmlinux-libs-$(CONFIG_X86_64)	+= $(objtree)/arch/x86/boot/startup/lib.a
 
 $(obj)/vmlinux: $(vmlinux-objs-y) $(vmlinux-libs-y) FORCE
 	$(call if_changed,ld)
diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/arch/x86/boot/startup/Makefile
@@ -XXX,XX +XXX,XX @@
+# SPDX-License-Identifier: GPL-2.0
+
+lib-$(CONFIG_X86_64)		+= la57toggle.o
diff --git a/arch/x86/boot/compressed/la57toggle.S b/arch/x86/boot/startup/la57toggle.S
similarity index 100%
rename from arch/x86/boot/compressed/la57toggle.S
rename to arch/x86/boot/startup/la57toggle.S
-- 
2.49.0.472.ge94155a9ec-goog

From: Ard Biesheuvel <ardb@kernel.org>

Linus expressed a strong preference for arch-specific asm code (i.e.,
virtually all of it) to reside under arch/ rather than anywhere else.

So move the EFI mixed mode startup code back, and put it under
arch/x86/boot/startup/ where all shared x86 startup code is going to
live.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/Makefile                                                | 3 +++
 drivers/firmware/efi/libstub/x86-mixed.S => arch/x86/boot/startup/efi-mixed.S | 0
 drivers/firmware/efi/libstub/Makefile                                         | 1 -
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/Makefile
+++ b/arch/x86/boot/startup/Makefile
@@ -XXX,XX +XXX,XX @@
 # SPDX-License-Identifier: GPL-2.0
 
+KBUILD_AFLAGS		+= -D__DISABLE_EXPORTS
+
 lib-$(CONFIG_X86_64)		+= la57toggle.o
+lib-$(CONFIG_EFI_MIXED)		+= efi-mixed.o
diff --git a/drivers/firmware/efi/libstub/x86-mixed.S b/arch/x86/boot/startup/efi-mixed.S
similarity index 100%
rename from drivers/firmware/efi/libstub/x86-mixed.S
rename to arch/x86/boot/startup/efi-mixed.S
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -XXX,XX +XXX,XX @@ lib-$(CONFIG_EFI_GENERIC_STUB)	+= efi-stub.o string.o intrinsics.o systable.o \
 lib-$(CONFIG_ARM)		+= arm32-stub.o
 lib-$(CONFIG_ARM64)		+= kaslr.o arm64.o arm64-stub.o smbios.o
 lib-$(CONFIG_X86)		+= x86-stub.o smbios.o
-lib-$(CONFIG_EFI_MIXED)		+= x86-mixed.o
 lib-$(CONFIG_X86_64)		+= x86-5lvl.o
 lib-$(CONFIG_RISCV)		+= kaslr.o riscv.o riscv-stub.o
 lib-$(CONFIG_LOONGARCH)		+= loongarch.o loongarch-stub.o
-- 
2.49.0.472.ge94155a9ec-goog

From: Ard Biesheuvel <ardb@kernel.org>

Move the early GDT/IDT setup code that runs long before the kernel
virtual mapping is up into arch/x86/boot/startup/, and build it in a way
that ensures that the code tolerates being called from the 1:1 mapping
of memory.

This allows the RIP_REL_REF() macro uses to be dropped, and removes the
need for emitting the code into the special .head.text section.

Also tweak the sed symbol matching pattern in the decompressor to match
on lower case 't' or 'b', as these will be emitted by Clang for symbols
with hidden linkage.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/compressed/Makefile |  2 +-
 arch/x86/boot/startup/Makefile    | 15 ++++
 arch/x86/boot/startup/gdt_idt.c   | 82 ++++++++++++++++++++
 arch/x86/kernel/head64.c          | 74 ------------------
 4 files changed, 98 insertions(+), 75 deletions(-)

diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -XXX,XX +XXX,XX @@ LDFLAGS_vmlinux += -T
 hostprogs	:= mkpiggy
 HOST_EXTRACFLAGS += -I$(srctree)/tools/include
 
-sed-voffset := -e 's/^$[0-9a-fA-F]*$ [ABCDGRSTVW] $_text\|__start_rodata\|__bss_start\|_end$$$/\#define VO_\2 _AC(0x\1,UL)/p'
+sed-voffset := -e 's/^$[0-9a-fA-F]*$ [ABbCDGRSTtVW] $_text\|__start_rodata\|__bss_start\|_end$$$/\#define VO_\2 _AC(0x\1,UL)/p'
 
 quiet_cmd_voffset = VOFFSET $@
       cmd_voffset = $(NM) $< | sed -n $(sed-voffset) > $@
diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/Makefile
+++ b/arch/x86/boot/startup/Makefile
@@ -XXX,XX +XXX,XX @@
 # SPDX-License-Identifier: GPL-2.0
 
 KBUILD_AFLAGS		+= -D__DISABLE_EXPORTS
+KBUILD_CFLAGS		+= -D__DISABLE_EXPORTS -mcmodel=small -fPIC \
+			   -Os -DDISABLE_BRANCH_PROFILING \
+			   $(DISABLE_STACKLEAK_PLUGIN) \
+			   -fno-stack-protector -D__NO_FORTIFY \
+			   -include $(srctree)/include/linux/hidden.h
+
+# disable ftrace hooks
+KBUILD_CFLAGS	:= $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS))
+KASAN_SANITIZE	:= n
+KCSAN_SANITIZE	:= n
+KMSAN_SANITIZE	:= n
+UBSAN_SANITIZE	:= n
+KCOV_INSTRUMENT	:= n
+
+obj-$(CONFIG_X86_64)		+= gdt_idt.o
 
 lib-$(CONFIG_X86_64)		+= la57toggle.o
 lib-$(CONFIG_EFI_MIXED)		+= efi-mixed.o
diff --git a/arch/x86/boot/startup/gdt_idt.c b/arch/x86/boot/startup/gdt_idt.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/arch/x86/boot/startup/gdt_idt.c
@@ -XXX,XX +XXX,XX @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/linkage.h>
+#include <linux/types.h>
+
+#include <asm/desc.h>
+#include <asm/setup.h>
+#include <asm/sev.h>
+#include <asm/trapnr.h>
+
+/*
+ * Data structures and code used for IDT setup in head_64.S. The bringup-IDT is
+ * used until the idt_table takes over. On the boot CPU this happens in
+ * x86_64_start_kernel(), on secondary CPUs in start_secondary(). In both cases
+ * this happens in the functions called from head_64.S.
+ *
+ * The idt_table can't be used that early because all the code modifying it is
+ * in idt.c and can be instrumented by tracing or KASAN, which both don't work
+ * during early CPU bringup. Also the idt_table has the runtime vectors
+ * configured which require certain CPU state to be setup already (like TSS),
+ * which also hasn't happened yet in early CPU bringup.
+ */
+static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
+
+/* This may run while still in the direct mapping */
+static void startup_64_load_idt(void *vc_handler)
+{
+	struct desc_ptr desc = {
+		.address = (unsigned long)bringup_idt_table,
+		.size    = sizeof(bringup_idt_table) - 1,
+	};
+	struct idt_data data;
+	gate_desc idt_desc;
+
+	/* @vc_handler is set only for a VMM Communication Exception */
+	if (vc_handler) {
+		init_idt_data(&data, X86_TRAP_VC, vc_handler);
+		idt_init_desc(&idt_desc, &data);
+		native_write_idt_entry((gate_desc *)desc.address, X86_TRAP_VC, &idt_desc);
+	}
+
+	native_load_idt(&desc);
+}
+
+/* This is used when running on kernel addresses */
+void early_setup_idt(void)
+{
+	void *handler = NULL;
+
+	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) {
+		setup_ghcb();
+		handler = vc_boot_ghcb;
+	}
+
+	startup_64_load_idt(handler);
+}
+
+/*
+ * Setup boot CPU state needed before kernel switches to virtual addresses.
+ */
+void __init startup_64_setup_gdt_idt(void)
+{
+	void *handler = NULL;
+
+	struct desc_ptr startup_gdt_descr = {
+		.address = (__force unsigned long)gdt_page.gdt,
+		.size    = GDT_SIZE - 1,
+	};
+
+	/* Load GDT */
+	native_load_gdt(&startup_gdt_descr);
+
+	/* New GDT is live - reload data segment registers */
+	asm volatile("movl %%eax, %%ds\n"
+		     "movl %%eax, %%ss\n"
+		     "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
+
+	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
+		handler = vc_no_ghcb;
+
+	startup_64_load_idt(handler);
+}
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -XXX,XX +XXX,XX @@ void __init __noreturn x86_64_start_reservations(char *real_mode_data)
 
 	start_kernel();
 }
-
-/*
- * Data structures and code used for IDT setup in head_64.S. The bringup-IDT is
- * used until the idt_table takes over. On the boot CPU this happens in
- * x86_64_start_kernel(), on secondary CPUs in start_secondary(). In both cases
- * this happens in the functions called from head_64.S.
- *
- * The idt_table can't be used that early because all the code modifying it is
- * in idt.c and can be instrumented by tracing or KASAN, which both don't work
- * during early CPU bringup. Also the idt_table has the runtime vectors
- * configured which require certain CPU state to be setup already (like TSS),
- * which also hasn't happened yet in early CPU bringup.
- */
-static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
-
-/* This may run while still in the direct mapping */
-static void __head startup_64_load_idt(void *vc_handler)
-{
-	struct desc_ptr desc = {
-		.address = (unsigned long)&RIP_REL_REF(bringup_idt_table),
-		.size    = sizeof(bringup_idt_table) - 1,
-	};
-	struct idt_data data;
-	gate_desc idt_desc;
-
-	/* @vc_handler is set only for a VMM Communication Exception */
-	if (vc_handler) {
-		init_idt_data(&data, X86_TRAP_VC, vc_handler);
-		idt_init_desc(&idt_desc, &data);
-		native_write_idt_entry((gate_desc *)desc.address, X86_TRAP_VC, &idt_desc);
-	}
-
-	native_load_idt(&desc);
-}
-
-/* This is used when running on kernel addresses */
-void early_setup_idt(void)
-{
-	void *handler = NULL;
-
-	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) {
-		setup_ghcb();
-		handler = vc_boot_ghcb;
-	}
-
-	startup_64_load_idt(handler);
-}
-
-/*
- * Setup boot CPU state needed before kernel switches to virtual addresses.
- */
-void __head startup_64_setup_gdt_idt(void)
-{
-	struct desc_struct *gdt = (void *)(__force unsigned long)gdt_page.gdt;
-	void *handler = NULL;
-
-	struct desc_ptr startup_gdt_descr = {
-		.address = (unsigned long)&RIP_REL_REF(*gdt),
-		.size    = GDT_SIZE - 1,
-	};
-
-	/* Load GDT */
-	native_load_gdt(&startup_gdt_descr);
-
-	/* New GDT is live - reload data segment registers */
-	asm volatile("movl %%eax, %%ds\n"
-		     "movl %%eax, %%ss\n"
-		     "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
-
-	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
-		handler = &RIP_REL_REF(vc_no_ghcb);
-
-	startup_64_load_idt(handler);
-}
-- 
2.49.0.472.ge94155a9ec-goog

From: Ard Biesheuvel <ardb@kernel.org>

The startup code that constructs the kernel virtual mapping runs from
the 1:1 mapping of memory itself, and therefore, cannot use absolute
symbol references. Move this code into a separate source file under
arch/x86/boot/startup/ where all such code will be kept from now on.

Since all code here is constructed in a manner that ensures that it
tolerates running from the 1:1 mapping of memory, any uses of the
RIP_REL_REF() macro can be dropped, along with __head annotations for
placing this code in a dedicated startup section.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/Makefile     |   2 +-
 arch/x86/boot/startup/map_kernel.c | 232 ++++++++++++++++++++
 arch/x86/kernel/head64.c           | 228 +------------------
 3 files changed, 234 insertions(+), 228 deletions(-)

diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/Makefile
+++ b/arch/x86/boot/startup/Makefile
@@ -XXX,XX +XXX,XX @@ KMSAN_SANITIZE	:= n
 UBSAN_SANITIZE	:= n
 KCOV_INSTRUMENT	:= n
 
-obj-$(CONFIG_X86_64)		+= gdt_idt.o
+obj-$(CONFIG_X86_64)		+= gdt_idt.o map_kernel.o
 
 lib-$(CONFIG_X86_64)		+= la57toggle.o
 lib-$(CONFIG_EFI_MIXED)		+= efi-mixed.o
diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/arch/x86/boot/startup/map_kernel.c
@@ -XXX,XX +XXX,XX @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/init.h>
+#include <linux/linkage.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/pgtable.h>
+
+#include <asm/sections.h>
+#include <asm/setup.h>
+#include <asm/sev.h>
+
+extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
+extern unsigned int next_early_pgt;
+
+#ifdef CONFIG_X86_5LEVEL
+unsigned int __pgtable_l5_enabled __ro_after_init;
+unsigned int pgdir_shift __ro_after_init = 39;
+EXPORT_SYMBOL(pgdir_shift);
+unsigned int ptrs_per_p4d __ro_after_init = 1;
+EXPORT_SYMBOL(ptrs_per_p4d);
+#endif
+
+#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT
+unsigned long page_offset_base __ro_after_init = __PAGE_OFFSET_BASE_L4;
+EXPORT_SYMBOL(page_offset_base);
+unsigned long vmalloc_base __ro_after_init = __VMALLOC_BASE_L4;
+EXPORT_SYMBOL(vmalloc_base);
+unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4;
+EXPORT_SYMBOL(vmemmap_base);
+#endif
+
+static inline bool check_la57_support(void)
+{
+	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
+		return false;
+
+	/*
+	 * 5-level paging is detected and enabled at kernel decompression
+	 * stage. Only check if it has been enabled there.
+	 */
+	if (!(native_read_cr4() & X86_CR4_LA57))
+		return false;
+
+	__pgtable_l5_enabled	= 1;
+	pgdir_shift		= 48;
+	ptrs_per_p4d		= 512;
+	page_offset_base	= __PAGE_OFFSET_BASE_L5;
+	vmalloc_base		= __VMALLOC_BASE_L5;
+	vmemmap_base		= __VMEMMAP_BASE_L5;
+
+	return true;
+}
+
+static unsigned long sme_postprocess_startup(struct boot_params *bp,
+					     pmdval_t *pmd,
+					     unsigned long p2v_offset)
+{
+	unsigned long paddr, paddr_end;
+	int i;
+
+	/* Encrypt the kernel and related (if SME is active) */
+	sme_encrypt_kernel(bp);
+
+	/*
+	 * Clear the memory encryption mask from the .bss..decrypted section.
+	 * The bss section will be memset to zero later in the initialization so
+	 * there is no need to zero it after changing the memory encryption
+	 * attribute.
+	 */
+	if (sme_get_me_mask()) {
+		paddr = (unsigned long)__start_bss_decrypted;
+		paddr_end = (unsigned long)__end_bss_decrypted;
+
+		for (; paddr < paddr_end; paddr += PMD_SIZE) {
+			/*
+			 * On SNP, transition the page to shared in the RMP table so that
+			 * it is consistent with the page table attribute change.
+			 *
+			 * __start_bss_decrypted has a virtual address in the high range
+			 * mapping (kernel .text). PVALIDATE, by way of
+			 * early_snp_set_memory_shared(), requires a valid virtual
+			 * address but the kernel is currently running off of the identity
+			 * mapping so use the PA to get a *currently* valid virtual address.
+			 */
+			early_snp_set_memory_shared(paddr, paddr, PTRS_PER_PMD);
+
+			i = pmd_index(paddr - p2v_offset);
+			pmd[i] -= sme_get_me_mask();
+		}
+	}
+
+	/*
+	 * Return the SME encryption mask (if SME is active) to be used as a
+	 * modifier for the initial pgdir entry programmed into CR3.
+	 */
+	return sme_get_me_mask();
+}
+
+unsigned long __init __startup_64(unsigned long p2v_offset,
+				  struct boot_params *bp)
+{
+	pmd_t (*early_pgts)[PTRS_PER_PMD] = early_dynamic_pgts;
+	unsigned long physaddr = (unsigned long)_text;
+	unsigned long va_text, va_end;
+	unsigned long pgtable_flags;
+	unsigned long load_delta;
+	pgdval_t *pgd;
+	p4dval_t *p4d;
+	pudval_t *pud;
+	pmdval_t *pmd, pmd_entry;
+	bool la57;
+	int i;
+
+	la57 = check_la57_support();
+
+	/* Is the address too large? */
+	if (physaddr >> MAX_PHYSMEM_BITS)
+		for (;;);
+
+	/*
+	 * Compute the delta between the address I am compiled to run at
+	 * and the address I am actually running at.
+	 */
+	phys_base = load_delta = __START_KERNEL_map + p2v_offset;
+
+	/* Is the address not 2M aligned? */
+	if (load_delta & ~PMD_MASK)
+		for (;;);
+
+	va_text = physaddr - p2v_offset;
+	va_end  = (unsigned long)_end - p2v_offset;
+
+	/* Include the SME encryption mask in the fixup value */
+	load_delta += sme_get_me_mask();
+
+	/* Fixup the physical addresses in the page table */
+
+	pgd = &early_top_pgt[0].pgd;
+	pgd[pgd_index(__START_KERNEL_map)] += load_delta;
+
+	if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
+		p4d = (p4dval_t *)level4_kernel_pgt;
+		p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
+
+		pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
+	}
+
+	level3_kernel_pgt[PTRS_PER_PUD - 2].pud += load_delta;
+	level3_kernel_pgt[PTRS_PER_PUD - 1].pud += load_delta;
+
+	for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
+		level2_fixmap_pgt[i].pmd += load_delta;
+
+	/*
+	 * Set up the identity mapping for the switchover.  These
+	 * entries should *NOT* have the global bit set!  This also
+	 * creates a bunch of nonsense entries but that is fine --
+	 * it avoids problems around wraparound.
+	 */
+
+	pud = &early_pgts[0]->pmd;
+	pmd = &early_pgts[1]->pmd;
+	next_early_pgt = 2;
+
+	pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
+
+	if (la57) {
+		p4d = &early_pgts[next_early_pgt++]->pmd;
+
+		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
+		pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
+		pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;
+
+		i = physaddr >> P4D_SHIFT;
+		p4d[(i + 0) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
+		p4d[(i + 1) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
+	} else {
+		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
+		pgd[i + 0] = (pgdval_t)pud + pgtable_flags;
+		pgd[i + 1] = (pgdval_t)pud + pgtable_flags;
+	}
+
+	i = physaddr >> PUD_SHIFT;
+	pud[(i + 0) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
+	pud[(i + 1) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
+
+	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
+	/* Filter out unsupported __PAGE_KERNEL_* bits: */
+	pmd_entry &= __supported_pte_mask;
+	pmd_entry += sme_get_me_mask();
+	pmd_entry +=  physaddr;
+
+	for (i = 0; i < DIV_ROUND_UP(va_end - va_text, PMD_SIZE); i++) {
+		int idx = i + (physaddr >> PMD_SHIFT);
+
+		pmd[idx % PTRS_PER_PMD] = pmd_entry + i * PMD_SIZE;
+	}
+
+	/*
+	 * Fixup the kernel text+data virtual addresses. Note that
+	 * we might write invalid pmds, when the kernel is relocated
+	 * cleanup_highmap() fixes this up along with the mappings
+	 * beyond _end.
+	 *
+	 * Only the region occupied by the kernel image has so far
+	 * been checked against the table of usable memory regions
+	 * provided by the firmware, so invalidate pages outside that
+	 * region. A page table entry that maps to a reserved area of
+	 * memory would allow processor speculation into that area,
+	 * and on some hardware (particularly the UV platform) even
+	 * speculative access to some reserved areas is caught as an
+	 * error, causing the BIOS to halt the system.
+	 */
+
+	pmd = &level2_kernel_pgt[0].pmd;
+
+	/* invalidate pages before the kernel image */
+	for (i = 0; i < pmd_index(va_text); i++)
+		pmd[i] &= ~_PAGE_PRESENT;
+
+	/* fixup pages that are part of the kernel image */
+	for (; i <= pmd_index(va_end); i++)
+		if (pmd[i] & _PAGE_PRESENT)
+			pmd[i] += load_delta;
+
+	/* invalidate pages after the kernel image */
+	for (; i < PTRS_PER_PMD; i++)
+		pmd[i] &= ~_PAGE_PRESENT;
+
+	return sme_postprocess_startup(bp, pmd, p2v_offset);
+}
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -XXX,XX +XXX,XX @@
  * Manage page tables very early on.
  */
 extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
-static unsigned int __initdata next_early_pgt;
+unsigned int __initdata next_early_pgt;
 pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
 
-#ifdef CONFIG_X86_5LEVEL
-unsigned int __pgtable_l5_enabled __ro_after_init;
-unsigned int pgdir_shift __ro_after_init = 39;
-EXPORT_SYMBOL(pgdir_shift);
-unsigned int ptrs_per_p4d __ro_after_init = 1;
-EXPORT_SYMBOL(ptrs_per_p4d);
-#endif
-
-#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT
-unsigned long page_offset_base __ro_after_init = __PAGE_OFFSET_BASE_L4;
-EXPORT_SYMBOL(page_offset_base);
-unsigned long vmalloc_base __ro_after_init = __VMALLOC_BASE_L4;
-EXPORT_SYMBOL(vmalloc_base);
-unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4;
-EXPORT_SYMBOL(vmemmap_base);
-#endif
-
-static inline bool check_la57_support(void)
-{
-	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
-		return false;
-
-	/*
-	 * 5-level paging is detected and enabled at kernel decompression
-	 * stage. Only check if it has been enabled there.
-	 */
-	if (!(native_read_cr4() & X86_CR4_LA57))
-		return false;
-
-	RIP_REL_REF(__pgtable_l5_enabled)	= 1;
-	RIP_REL_REF(pgdir_shift)		= 48;
-	RIP_REL_REF(ptrs_per_p4d)		= 512;
-	RIP_REL_REF(page_offset_base)		= __PAGE_OFFSET_BASE_L5;
-	RIP_REL_REF(vmalloc_base)		= __VMALLOC_BASE_L5;
-	RIP_REL_REF(vmemmap_base)		= __VMEMMAP_BASE_L5;
-
-	return true;
-}
-
-static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
-						    pmdval_t *pmd,
-						    unsigned long p2v_offset)
-{
-	unsigned long paddr, paddr_end;
-	int i;
-
-	/* Encrypt the kernel and related (if SME is active) */
-	sme_encrypt_kernel(bp);
-
-	/*
-	 * Clear the memory encryption mask from the .bss..decrypted section.
-	 * The bss section will be memset to zero later in the initialization so
-	 * there is no need to zero it after changing the memory encryption
-	 * attribute.
-	 */
-	if (sme_get_me_mask()) {
-		paddr = (unsigned long)&RIP_REL_REF(__start_bss_decrypted);
-		paddr_end = (unsigned long)&RIP_REL_REF(__end_bss_decrypted);
-
-		for (; paddr < paddr_end; paddr += PMD_SIZE) {
-			/*
-			 * On SNP, transition the page to shared in the RMP table so that
-			 * it is consistent with the page table attribute change.
-			 *
-			 * __start_bss_decrypted has a virtual address in the high range
-			 * mapping (kernel .text). PVALIDATE, by way of
-			 * early_snp_set_memory_shared(), requires a valid virtual
-			 * address but the kernel is currently running off of the identity
-			 * mapping so use the PA to get a *currently* valid virtual address.
-			 */
-			early_snp_set_memory_shared(paddr, paddr, PTRS_PER_PMD);
-
-			i = pmd_index(paddr - p2v_offset);
-			pmd[i] -= sme_get_me_mask();
-		}
-	}
-
-	/*
-	 * Return the SME encryption mask (if SME is active) to be used as a
-	 * modifier for the initial pgdir entry programmed into CR3.
-	 */
-	return sme_get_me_mask();
-}
-
-/* Code in __startup_64() can be relocated during execution, but the compiler
- * doesn't have to generate PC-relative relocations when accessing globals from
- * that function. Clang actually does not generate them, which leads to
- * boot-time crashes. To work around this problem, every global pointer must
- * be accessed using RIP_REL_REF(). Kernel virtual addresses can be determined
- * by subtracting p2v_offset from the RIP-relative address.
- */
-unsigned long __head __startup_64(unsigned long p2v_offset,
-				  struct boot_params *bp)
-{
-	pmd_t (*early_pgts)[PTRS_PER_PMD] = RIP_REL_REF(early_dynamic_pgts);
-	unsigned long physaddr = (unsigned long)&RIP_REL_REF(_text);
-	unsigned long va_text, va_end;
-	unsigned long pgtable_flags;
-	unsigned long load_delta;
-	pgdval_t *pgd;
-	p4dval_t *p4d;
-	pudval_t *pud;
-	pmdval_t *pmd, pmd_entry;
-	bool la57;
-	int i;
-
-	la57 = check_la57_support();
-
-	/* Is the address too large? */
-	if (physaddr >> MAX_PHYSMEM_BITS)
-		for (;;);
-
-	/*
-	 * Compute the delta between the address I am compiled to run at
-	 * and the address I am actually running at.
-	 */
-	load_delta = __START_KERNEL_map + p2v_offset;
-	RIP_REL_REF(phys_base) = load_delta;
-
-	/* Is the address not 2M aligned? */
-	if (load_delta & ~PMD_MASK)
-		for (;;);
-
-	va_text = physaddr - p2v_offset;
-	va_end  = (unsigned long)&RIP_REL_REF(_end) - p2v_offset;
-
-	/* Include the SME encryption mask in the fixup value */
-	load_delta += sme_get_me_mask();
-
-	/* Fixup the physical addresses in the page table */
-
-	pgd = &RIP_REL_REF(early_top_pgt)->pgd;
-	pgd[pgd_index(__START_KERNEL_map)] += load_delta;
-
-	if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
-		p4d = (p4dval_t *)&RIP_REL_REF(level4_kernel_pgt);
-		p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
-
-		pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
-	}
-
-	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 2].pud += load_delta;
-	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 1].pud += load_delta;
-
-	for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
-		RIP_REL_REF(level2_fixmap_pgt)[i].pmd += load_delta;
-
-	/*
-	 * Set up the identity mapping for the switchover.  These
-	 * entries should *NOT* have the global bit set!  This also
-	 * creates a bunch of nonsense entries but that is fine --
-	 * it avoids problems around wraparound.
-	 */
-
-	pud = &early_pgts[0]->pmd;
-	pmd = &early_pgts[1]->pmd;
-	RIP_REL_REF(next_early_pgt) = 2;
-
-	pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
-
-	if (la57) {
-		p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd;
-
-		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
-		pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
-		pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;
-
-		i = physaddr >> P4D_SHIFT;
-		p4d[(i + 0) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
-		p4d[(i + 1) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
-	} else {
-		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
-		pgd[i + 0] = (pgdval_t)pud + pgtable_flags;
-		pgd[i + 1] = (pgdval_t)pud + pgtable_flags;
-	}
-
-	i = physaddr >> PUD_SHIFT;
-	pud[(i + 0) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
-	pud[(i + 1) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
-
-	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
-	/* Filter out unsupported __PAGE_KERNEL_* bits: */
-	pmd_entry &= RIP_REL_REF(__supported_pte_mask);
-	pmd_entry += sme_get_me_mask();
-	pmd_entry +=  physaddr;
-
-	for (i = 0; i < DIV_ROUND_UP(va_end - va_text, PMD_SIZE); i++) {
-		int idx = i + (physaddr >> PMD_SHIFT);
-
-		pmd[idx % PTRS_PER_PMD] = pmd_entry + i * PMD_SIZE;
-	}
-
-	/*
-	 * Fixup the kernel text+data virtual addresses. Note that
-	 * we might write invalid pmds, when the kernel is relocated
-	 * cleanup_highmap() fixes this up along with the mappings
-	 * beyond _end.
-	 *
-	 * Only the region occupied by the kernel image has so far
-	 * been checked against the table of usable memory regions
-	 * provided by the firmware, so invalidate pages outside that
-	 * region. A page table entry that maps to a reserved area of
-	 * memory would allow processor speculation into that area,
-	 * and on some hardware (particularly the UV platform) even
-	 * speculative access to some reserved areas is caught as an
-	 * error, causing the BIOS to halt the system.
-	 */
-
-	pmd = &RIP_REL_REF(level2_kernel_pgt)->pmd;
-
-	/* invalidate pages before the kernel image */
-	for (i = 0; i < pmd_index(va_text); i++)
-		pmd[i] &= ~_PAGE_PRESENT;
-
-	/* fixup pages that are part of the kernel image */
-	for (; i <= pmd_index(va_end); i++)
-		if (pmd[i] & _PAGE_PRESENT)
-			pmd[i] += load_delta;
-
-	/* invalidate pages after the kernel image */
-	for (; i < PTRS_PER_PMD; i++)
-		pmd[i] &= ~_PAGE_PRESENT;
-
-	return sme_postprocess_startup(bp, pmd, p2v_offset);
-}
-
 /* Wipe all early page tables except for the kernel symbol map */
 static void __init reset_early_page_tables(void)
 {
-- 
2.49.0.472.ge94155a9ec-goog

From: Ard Biesheuvel <ardb@kernel.org>

Move the SME initialization code, which runs from the 1:1 mapping of
memory as it operates on the kernel virtual mapping, into the new
sub-directory arch/x86/boot/startup/ where all startup code will reside
that needs to tolerate executing from the 1:1 mapping.

This allows RIP_REL_REF() macro invocations and __head annotations to be
dropped.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/Makefile                             |  1 +
 arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} | 45 +++++++++-----------
 arch/x86/include/asm/mem_encrypt.h                         |  2 +-
 arch/x86/mm/Makefile                                       |  6 ---
 4 files changed, 23 insertions(+), 31 deletions(-)

From: Ard Biesheuvel <ardb@kernel.org>

!! NOTE: patches #7 - #10 depend on [0] !!

Reorganize C code that is used during early boot, either in the
decompressor/EFI stub or the kernel proper, but before the kernel
virtual mapping is up.

v4:
- drop patches that were queued up
- fix address space error in patch #1
- add patches for SEV-SNP boot code - these cannot be applied yet, but
  are included for completeness

v3:
- keep rip_rel_ptr() around in PIC code - sadly, it is still needed in
  some cases
- remove RIP_REL_REF() uses in separate patches
- keep __head annotations for now, they will all be removed later
- disable objtool validation for library objects (i.e., pieces that are
  not linked into vmlinux)

I will follow up with a series that gets rid of .head.text altogether,
as it will no longer be needed at all once the startup code is checked
for absolute relocations.

Cc: Tom Lendacky <thomas.lendacky@amd.com>
Cc: Dionna Amalie Glaze <dionnaglaze@google.com>
Cc: Kevin Loughlin <kevinloughlin@google.com>

[0] https://lore.kernel.org/all/20250410132850.3708703-2-ardb+git@google.com/T/#u

Ard Biesheuvel (11):
  x86/asm: Make rip_rel_ptr() usable from fPIC code
  x86/boot: Move the early GDT/IDT setup code into startup/
  x86/boot: Move early kernel mapping code into startup/
  x86/boot: Drop RIP_REL_REF() uses from early mapping code
  x86/boot: Move early SME init code into startup/
  x86/boot: Drop RIP_REL_REF() uses from SME startup code
  x86/sev: Prepare for splitting off early SEV code
  x86/sev: Split off startup code from core code
  x86/boot: Move SEV startup code into startup/
  x86/boot: Drop RIP_REL_REF() uses from early SEV code
  x86/asm: Retire RIP_REL_REF()

arch/x86/boot/compressed/Makefile                          |    2 +-
 arch/x86/boot/compressed/sev.c                             |   17 +-
 arch/x86/boot/startup/Makefile                             |   16 +
 arch/x86/boot/startup/gdt_idt.c                            |   84 +
 arch/x86/boot/startup/map_kernel.c                         |  225 +++
 arch/x86/{coco/sev/shared.c => boot/startup/sev-shared.c}  |  375 +----
 arch/x86/boot/startup/sev-startup.c                        | 1395 ++++++++++++++++
 arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} |   19 +-
 arch/x86/coco/sev/Makefile                                 |   19 -
 arch/x86/coco/sev/core.c                                   | 1726 ++++----------------
 arch/x86/include/asm/asm.h                                 |    5 -
 arch/x86/include/asm/coco.h                                |    2 +-
 arch/x86/include/asm/mem_encrypt.h                         |    2 +-
 arch/x86/include/asm/sev-internal.h                        |  112 ++
 arch/x86/include/asm/sev.h                                 |   37 +
 arch/x86/kernel/head64.c                                   |  285 +---
 arch/x86/mm/Makefile                                       |    6 -
 17 files changed, 2208 insertions(+), 2119 deletions(-)
 create mode 100644 arch/x86/boot/startup/gdt_idt.c
 create mode 100644 arch/x86/boot/startup/map_kernel.c
 rename arch/x86/{coco/sev/shared.c => boot/startup/sev-shared.c} (78%)
 create mode 100644 arch/x86/boot/startup/sev-startup.c
 rename arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} (97%)
 create mode 100644 arch/x86/include/asm/sev-internal.h

-- 
2.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

RIP_REL_REF() is used in non-PIC C code that is called very early,
before the kernel virtual mapping is up, which is the mapping that the
linker expects. It is currently used in two different ways:
- to refer to the value of a global variable, including as an lvalue in
  assignments;
- to take the address of a global variable via the mapping that the code
  currently executes at.

The former case is only needed in non-PIC code, as PIC code will never
use absolute symbol references when the address of the symbol is not
being used. But taking the address of a variable in PIC code may still
require extra care, as a stack allocated struct assignment may be
emitted as a memcpy() from a statically allocated copy in .rodata.

For instance, this

void startup_64_setup_gdt_idt(void)
  {
        struct desc_ptr startup_gdt_descr = {
                .address = (__force unsigned long)gdt_page.gdt,
                .size    = GDT_SIZE - 1,
        };

may result in an absolute symbol reference in PIC code, even though the
struct is allocated on the stack and populated at runtime.

To address this case, make rip_rel_ptr() accessible in PIC code, and
update any existing uses where the address of a global variable is
taken using RIP_REL_REF.

Once all code of this nature has been moved into arch/x86/boot/startup
and built with -fPIC, RIP_REL_REF() can be retired, and only
rip_rel_ptr() will remain.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/coco/sev/core.c           |  2 +-
 arch/x86/coco/sev/shared.c         |  4 ++--
 arch/x86/include/asm/asm.h         |  2 +-
 arch/x86/kernel/head64.c           | 24 ++++++++++----------
 arch/x86/mm/mem_encrypt_identity.c |  6 ++---
 5 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/coco/sev/core.c
+++ b/arch/x86/coco/sev/core.c
@@ -XXX,XX +XXX,XX @@ static __head void svsm_setup(struct cc_blob_sev_info *cc_info)
 	 * kernel was loaded (physbase), so the get the CA address using
 	 * RIP-relative addressing.
 	 */
-	pa = (u64)&RIP_REL_REF(boot_svsm_ca_page);
+	pa = (u64)rip_rel_ptr(&boot_svsm_ca_page);
 
 	/*
 	 * Switch over to the boot SVSM CA while the current CA is still
diff --git a/arch/x86/coco/sev/shared.c b/arch/x86/coco/sev/shared.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/coco/sev/shared.c
+++ b/arch/x86/coco/sev/shared.c
@@ -XXX,XX +XXX,XX @@ static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid
  */
 static const struct snp_cpuid_table *snp_cpuid_get_table(void)
 {
-	return &RIP_REL_REF(cpuid_table_copy);
+	return rip_rel_ptr(&cpuid_table_copy);
 }
 
 /*
@@ -XXX,XX +XXX,XX @@ static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info)
 	 * routine is running identity mapped when called, both by the decompressor
 	 * code and the early kernel code.
 	 */
-	if (!rmpadjust((unsigned long)&RIP_REL_REF(boot_ghcb_page), RMP_PG_SIZE_4K, 1))
+	if (!rmpadjust((unsigned long)rip_rel_ptr(&boot_ghcb_page), RMP_PG_SIZE_4K, 1))
 		return false;
 
 	/*
diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/include/asm/asm.h
+++ b/arch/x86/include/asm/asm.h
@@ -XXX,XX +XXX,XX @@
 #endif
 
 #ifndef __ASSEMBLER__
-#ifndef __pic__
 static __always_inline __pure void *rip_rel_ptr(void *p)
 {
 	asm("leaq %c1(%%rip), %0" : "=r"(p) : "i"(p));
 
 	return p;
 }
+#ifndef __pic__
 #define RIP_REL_REF(var)	(*(typeof(&(var)))rip_rel_ptr(&(var)))
 #else
 #define RIP_REL_REF(var)	(var)
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -XXX,XX +XXX,XX @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
 	 * attribute.
 	 */
 	if (sme_get_me_mask()) {
-		paddr = (unsigned long)&RIP_REL_REF(__start_bss_decrypted);
-		paddr_end = (unsigned long)&RIP_REL_REF(__end_bss_decrypted);
+		paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted);
+		paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted);
 
 		for (; paddr < paddr_end; paddr += PMD_SIZE) {
 			/*
@@ -XXX,XX +XXX,XX @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
 unsigned long __head __startup_64(unsigned long p2v_offset,
 				  struct boot_params *bp)
 {
-	pmd_t (*early_pgts)[PTRS_PER_PMD] = RIP_REL_REF(early_dynamic_pgts);
-	unsigned long physaddr = (unsigned long)&RIP_REL_REF(_text);
+	pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);
+	unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);
 	unsigned long va_text, va_end;
 	unsigned long pgtable_flags;
 	unsigned long load_delta;
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
 		for (;;);
 
 	va_text = physaddr - p2v_offset;
-	va_end  = (unsigned long)&RIP_REL_REF(_end) - p2v_offset;
+	va_end  = (unsigned long)rip_rel_ptr(_end) - p2v_offset;
 
 	/* Include the SME encryption mask in the fixup value */
 	load_delta += sme_get_me_mask();
 
 	/* Fixup the physical addresses in the page table */
 
-	pgd = &RIP_REL_REF(early_top_pgt)->pgd;
+	pgd = rip_rel_ptr(early_top_pgt);
 	pgd[pgd_index(__START_KERNEL_map)] += load_delta;
 
 	if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
-		p4d = (p4dval_t *)&RIP_REL_REF(level4_kernel_pgt);
+		p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt);
 		p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
 
 		pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
 	 * error, causing the BIOS to halt the system.
 	 */
 
-	pmd = &RIP_REL_REF(level2_kernel_pgt)->pmd;
+	pmd = rip_rel_ptr(level2_kernel_pgt);
 
 	/* invalidate pages before the kernel image */
 	for (i = 0; i < pmd_index(va_text); i++)
@@ -XXX,XX +XXX,XX @@ static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
 static void __head startup_64_load_idt(void *vc_handler)
 {
 	struct desc_ptr desc = {
-		.address = (unsigned long)&RIP_REL_REF(bringup_idt_table),
+		.address = (unsigned long)rip_rel_ptr(bringup_idt_table),
 		.size    = sizeof(bringup_idt_table) - 1,
 	};
 	struct idt_data data;
@@ -XXX,XX +XXX,XX @@ void early_setup_idt(void)
  */
 void __head startup_64_setup_gdt_idt(void)
 {
-	struct desc_struct *gdt = (void *)(__force unsigned long)gdt_page.gdt;
+	struct gdt_page *gp = rip_rel_ptr((void *)(__force unsigned long)&gdt_page);
 	void *handler = NULL;
 
 	struct desc_ptr startup_gdt_descr = {
-		.address = (unsigned long)&RIP_REL_REF(*gdt),
+		.address = (unsigned long)gp->gdt,
 		.size    = GDT_SIZE - 1,
 	};
 
@@ -XXX,XX +XXX,XX @@ void __head startup_64_setup_gdt_idt(void)
 		     "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
 
 	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
-		handler = &RIP_REL_REF(vc_no_ghcb);
+		handler = rip_rel_ptr(vc_no_ghcb);
 
 	startup_64_load_idt(handler);
 }
diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/mm/mem_encrypt_identity.c
+++ b/arch/x86/mm/mem_encrypt_identity.c
@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
 	 *     memory from being cached.
 	 */
 
-	kernel_start = (unsigned long)RIP_REL_REF(_text);
-	kernel_end = ALIGN((unsigned long)RIP_REL_REF(_end), PMD_SIZE);
+	kernel_start = (unsigned long)rip_rel_ptr(_text);
+	kernel_end = ALIGN((unsigned long)rip_rel_ptr(_end), PMD_SIZE);
 	kernel_len = kernel_end - kernel_start;
 
 	initrd_start = 0;
@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
 	 *   pagetable structures for the encryption of the kernel
 	 *   pagetable structures for workarea (in case not currently mapped)
 	 */
-	execute_start = workarea_start = (unsigned long)RIP_REL_REF(sme_workarea);
+	execute_start = workarea_start = (unsigned long)rip_rel_ptr(sme_workarea);
 	execute_end = execute_start + (PAGE_SIZE * 2) + PMD_SIZE;
 	execute_len = execute_end - execute_start;
 
-- 
2.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

Also tweak the sed symbol matching pattern in the decompressor to match
on lower case 't' or 'b', as these will be emitted by Clang for symbols
with hidden linkage.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/compressed/Makefile |  2 +-
 arch/x86/boot/startup/Makefile    | 15 ++++
 arch/x86/boot/startup/gdt_idt.c   | 84 ++++++++++++++++++++
 arch/x86/kernel/head64.c          | 74 -----------------
 4 files changed, 100 insertions(+), 75 deletions(-)

diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -XXX,XX +XXX,XX @@ LDFLAGS_vmlinux += -T
 hostprogs	:= mkpiggy
 HOST_EXTRACFLAGS += -I$(srctree)/tools/include
 
-sed-voffset := -e 's/^$[0-9a-fA-F]*$ [ABCDGRSTVW] $_text\|__start_rodata\|__bss_start\|_end$$$/\#define VO_\2 _AC(0x\1,UL)/p'
+sed-voffset := -e 's/^$[0-9a-fA-F]*$ [ABbCDGRSTtVW] $_text\|__start_rodata\|__bss_start\|_end$$$/\#define VO_\2 _AC(0x\1,UL)/p'
 
 quiet_cmd_voffset = VOFFSET $@
       cmd_voffset = $(NM) $< | sed -n $(sed-voffset) > $@
diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/Makefile
+++ b/arch/x86/boot/startup/Makefile
@@ -XXX,XX +XXX,XX @@
 # SPDX-License-Identifier: GPL-2.0
 
 KBUILD_AFLAGS		+= -D__DISABLE_EXPORTS
+KBUILD_CFLAGS		+= -D__DISABLE_EXPORTS -mcmodel=small -fPIC \
+			   -Os -DDISABLE_BRANCH_PROFILING \
+			   $(DISABLE_STACKLEAK_PLUGIN) \
+			   -fno-stack-protector -D__NO_FORTIFY \
+			   -include $(srctree)/include/linux/hidden.h
+
+# disable ftrace hooks
+KBUILD_CFLAGS	:= $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS))
+KASAN_SANITIZE	:= n
+KCSAN_SANITIZE	:= n
+KMSAN_SANITIZE	:= n
+UBSAN_SANITIZE	:= n
+KCOV_INSTRUMENT	:= n
+
+obj-$(CONFIG_X86_64)		+= gdt_idt.o
 
 lib-$(CONFIG_X86_64)		+= la57toggle.o
 lib-$(CONFIG_EFI_MIXED)		+= efi-mixed.o
diff --git a/arch/x86/boot/startup/gdt_idt.c b/arch/x86/boot/startup/gdt_idt.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/arch/x86/boot/startup/gdt_idt.c
@@ -XXX,XX +XXX,XX @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/linkage.h>
+#include <linux/types.h>
+
+#include <asm/desc.h>
+#include <asm/init.h>
+#include <asm/setup.h>
+#include <asm/sev.h>
+#include <asm/trapnr.h>
+
+/*
+ * Data structures and code used for IDT setup in head_64.S. The bringup-IDT is
+ * used until the idt_table takes over. On the boot CPU this happens in
+ * x86_64_start_kernel(), on secondary CPUs in start_secondary(). In both cases
+ * this happens in the functions called from head_64.S.
+ *
+ * The idt_table can't be used that early because all the code modifying it is
+ * in idt.c and can be instrumented by tracing or KASAN, which both don't work
+ * during early CPU bringup. Also the idt_table has the runtime vectors
+ * configured which require certain CPU state to be setup already (like TSS),
+ * which also hasn't happened yet in early CPU bringup.
+ */
+static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
+
+/* This may run while still in the direct mapping */
+static void __head startup_64_load_idt(void *vc_handler)
+{
+	struct desc_ptr desc = {
+		.address = (unsigned long)rip_rel_ptr(bringup_idt_table),
+		.size    = sizeof(bringup_idt_table) - 1,
+	};
+	struct idt_data data;
+	gate_desc idt_desc;
+
+	/* @vc_handler is set only for a VMM Communication Exception */
+	if (vc_handler) {
+		init_idt_data(&data, X86_TRAP_VC, vc_handler);
+		idt_init_desc(&idt_desc, &data);
+		native_write_idt_entry((gate_desc *)desc.address, X86_TRAP_VC, &idt_desc);
+	}
+
+	native_load_idt(&desc);
+}
+
+/* This is used when running on kernel addresses */
+void early_setup_idt(void)
+{
+	void *handler = NULL;
+
+	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) {
+		setup_ghcb();
+		handler = vc_boot_ghcb;
+	}
+
+	startup_64_load_idt(handler);
+}
+
+/*
+ * Setup boot CPU state needed before kernel switches to virtual addresses.
+ */
+void __head startup_64_setup_gdt_idt(void)
+{
+	struct gdt_page *gp = rip_rel_ptr((void *)(__force unsigned long)&gdt_page);
+	void *handler = NULL;
+
+	struct desc_ptr startup_gdt_descr = {
+		.address = (unsigned long)gp->gdt,
+		.size    = GDT_SIZE - 1,
+	};
+
+	/* Load GDT */
+	native_load_gdt(&startup_gdt_descr);
+
+	/* New GDT is live - reload data segment registers */
+	asm volatile("movl %%eax, %%ds\n"
+		     "movl %%eax, %%ss\n"
+		     "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
+
+	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
+		handler = rip_rel_ptr(vc_no_ghcb);
+
+	startup_64_load_idt(handler);
+}
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -XXX,XX +XXX,XX @@ void __init __noreturn x86_64_start_reservations(char *real_mode_data)
 
 	start_kernel();
 }
-
-/*
- * Data structures and code used for IDT setup in head_64.S. The bringup-IDT is
- * used until the idt_table takes over. On the boot CPU this happens in
- * x86_64_start_kernel(), on secondary CPUs in start_secondary(). In both cases
- * this happens in the functions called from head_64.S.
- *
- * The idt_table can't be used that early because all the code modifying it is
- * in idt.c and can be instrumented by tracing or KASAN, which both don't work
- * during early CPU bringup. Also the idt_table has the runtime vectors
- * configured which require certain CPU state to be setup already (like TSS),
- * which also hasn't happened yet in early CPU bringup.
- */
-static gate_desc bringup_idt_table[NUM_EXCEPTION_VECTORS] __page_aligned_data;
-
-/* This may run while still in the direct mapping */
-static void __head startup_64_load_idt(void *vc_handler)
-{
-	struct desc_ptr desc = {
-		.address = (unsigned long)rip_rel_ptr(bringup_idt_table),
-		.size    = sizeof(bringup_idt_table) - 1,
-	};
-	struct idt_data data;
-	gate_desc idt_desc;
-
-	/* @vc_handler is set only for a VMM Communication Exception */
-	if (vc_handler) {
-		init_idt_data(&data, X86_TRAP_VC, vc_handler);
-		idt_init_desc(&idt_desc, &data);
-		native_write_idt_entry((gate_desc *)desc.address, X86_TRAP_VC, &idt_desc);
-	}
-
-	native_load_idt(&desc);
-}
-
-/* This is used when running on kernel addresses */
-void early_setup_idt(void)
-{
-	void *handler = NULL;
-
-	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT)) {
-		setup_ghcb();
-		handler = vc_boot_ghcb;
-	}
-
-	startup_64_load_idt(handler);
-}
-
-/*
- * Setup boot CPU state needed before kernel switches to virtual addresses.
- */
-void __head startup_64_setup_gdt_idt(void)
-{
-	struct gdt_page *gp = rip_rel_ptr((void *)(__force unsigned long)&gdt_page);
-	void *handler = NULL;
-
-	struct desc_ptr startup_gdt_descr = {
-		.address = (unsigned long)gp->gdt,
-		.size    = GDT_SIZE - 1,
-	};
-
-	/* Load GDT */
-	native_load_gdt(&startup_gdt_descr);
-
-	/* New GDT is live - reload data segment registers */
-	asm volatile("movl %%eax, %%ds\n"
-		     "movl %%eax, %%ss\n"
-		     "movl %%eax, %%es\n" : : "a"(__KERNEL_DS) : "memory");
-
-	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT))
-		handler = rip_rel_ptr(vc_no_ghcb);
-
-	startup_64_load_idt(handler);
-}
-- 
2.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

The startup code that constructs the kernel virtual mapping runs from
the 1:1 mapping of memory itself, and therefore, cannot use absolute
symbol references. Before making changes in subsequent patches, move
this code into a separate source file under arch/x86/boot/startup/ where
all such code will be kept from now on.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/Makefile     |   2 +-
 arch/x86/boot/startup/map_kernel.c | 224 ++++++++++++++++++++
 arch/x86/kernel/head64.c           | 211 +-----------------
 3 files changed, 226 insertions(+), 211 deletions(-)

diff --git a/arch/x86/boot/startup/Makefile b/arch/x86/boot/startup/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/Makefile
+++ b/arch/x86/boot/startup/Makefile
@@ -XXX,XX +XXX,XX @@ KMSAN_SANITIZE	:= n
 UBSAN_SANITIZE	:= n
 KCOV_INSTRUMENT	:= n
 
-obj-$(CONFIG_X86_64)		+= gdt_idt.o
+obj-$(CONFIG_X86_64)		+= gdt_idt.o map_kernel.o
 
 lib-$(CONFIG_X86_64)		+= la57toggle.o
 lib-$(CONFIG_EFI_MIXED)		+= efi-mixed.o
diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/arch/x86/boot/startup/map_kernel.c
@@ -XXX,XX +XXX,XX @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/init.h>
+#include <linux/linkage.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/pgtable.h>
+
+#include <asm/init.h>
+#include <asm/sections.h>
+#include <asm/setup.h>
+#include <asm/sev.h>
+
+extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
+extern unsigned int next_early_pgt;
+
+static inline bool check_la57_support(void)
+{
+	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
+		return false;
+
+	/*
+	 * 5-level paging is detected and enabled at kernel decompression
+	 * stage. Only check if it has been enabled there.
+	 */
+	if (!(native_read_cr4() & X86_CR4_LA57))
+		return false;
+
+	RIP_REL_REF(__pgtable_l5_enabled)	= 1;
+	RIP_REL_REF(pgdir_shift)		= 48;
+	RIP_REL_REF(ptrs_per_p4d)		= 512;
+	RIP_REL_REF(page_offset_base)		= __PAGE_OFFSET_BASE_L5;
+	RIP_REL_REF(vmalloc_base)		= __VMALLOC_BASE_L5;
+	RIP_REL_REF(vmemmap_base)		= __VMEMMAP_BASE_L5;
+
+	return true;
+}
+
+static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
+						    pmdval_t *pmd,
+						    unsigned long p2v_offset)
+{
+	unsigned long paddr, paddr_end;
+	int i;
+
+	/* Encrypt the kernel and related (if SME is active) */
+	sme_encrypt_kernel(bp);
+
+	/*
+	 * Clear the memory encryption mask from the .bss..decrypted section.
+	 * The bss section will be memset to zero later in the initialization so
+	 * there is no need to zero it after changing the memory encryption
+	 * attribute.
+	 */
+	if (sme_get_me_mask()) {
+		paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted);
+		paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted);
+
+		for (; paddr < paddr_end; paddr += PMD_SIZE) {
+			/*
+			 * On SNP, transition the page to shared in the RMP table so that
+			 * it is consistent with the page table attribute change.
+			 *
+			 * __start_bss_decrypted has a virtual address in the high range
+			 * mapping (kernel .text). PVALIDATE, by way of
+			 * early_snp_set_memory_shared(), requires a valid virtual
+			 * address but the kernel is currently running off of the identity
+			 * mapping so use the PA to get a *currently* valid virtual address.
+			 */
+			early_snp_set_memory_shared(paddr, paddr, PTRS_PER_PMD);
+
+			i = pmd_index(paddr - p2v_offset);
+			pmd[i] -= sme_get_me_mask();
+		}
+	}
+
+	/*
+	 * Return the SME encryption mask (if SME is active) to be used as a
+	 * modifier for the initial pgdir entry programmed into CR3.
+	 */
+	return sme_get_me_mask();
+}
+
+/* Code in __startup_64() can be relocated during execution, but the compiler
+ * doesn't have to generate PC-relative relocations when accessing globals from
+ * that function. Clang actually does not generate them, which leads to
+ * boot-time crashes. To work around this problem, every global pointer must
+ * be accessed using RIP_REL_REF(). Kernel virtual addresses can be determined
+ * by subtracting p2v_offset from the RIP-relative address.
+ */
+unsigned long __head __startup_64(unsigned long p2v_offset,
+				  struct boot_params *bp)
+{
+	pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);
+	unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);
+	unsigned long va_text, va_end;
+	unsigned long pgtable_flags;
+	unsigned long load_delta;
+	pgdval_t *pgd;
+	p4dval_t *p4d;
+	pudval_t *pud;
+	pmdval_t *pmd, pmd_entry;
+	bool la57;
+	int i;
+
+	la57 = check_la57_support();
+
+	/* Is the address too large? */
+	if (physaddr >> MAX_PHYSMEM_BITS)
+		for (;;);
+
+	/*
+	 * Compute the delta between the address I am compiled to run at
+	 * and the address I am actually running at.
+	 */
+	load_delta = __START_KERNEL_map + p2v_offset;
+	RIP_REL_REF(phys_base) = load_delta;
+
+	/* Is the address not 2M aligned? */
+	if (load_delta & ~PMD_MASK)
+		for (;;);
+
+	va_text = physaddr - p2v_offset;
+	va_end  = (unsigned long)rip_rel_ptr(_end) - p2v_offset;
+
+	/* Include the SME encryption mask in the fixup value */
+	load_delta += sme_get_me_mask();
+
+	/* Fixup the physical addresses in the page table */
+
+	pgd = rip_rel_ptr(early_top_pgt);
+	pgd[pgd_index(__START_KERNEL_map)] += load_delta;
+
+	if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
+		p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt);
+		p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
+
+		pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
+	}
+
+	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 2].pud += load_delta;
+	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 1].pud += load_delta;
+
+	for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
+		RIP_REL_REF(level2_fixmap_pgt)[i].pmd += load_delta;
+
+	/*
+	 * Set up the identity mapping for the switchover.  These
+	 * entries should *NOT* have the global bit set!  This also
+	 * creates a bunch of nonsense entries but that is fine --
+	 * it avoids problems around wraparound.
+	 */
+
+	pud = &early_pgts[0]->pmd;
+	pmd = &early_pgts[1]->pmd;
+	RIP_REL_REF(next_early_pgt) = 2;
+
+	pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
+
+	if (la57) {
+		p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd;
+
+		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
+		pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
+		pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;
+
+		i = physaddr >> P4D_SHIFT;
+		p4d[(i + 0) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
+		p4d[(i + 1) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
+	} else {
+		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
+		pgd[i + 0] = (pgdval_t)pud + pgtable_flags;
+		pgd[i + 1] = (pgdval_t)pud + pgtable_flags;
+	}
+
+	i = physaddr >> PUD_SHIFT;
+	pud[(i + 0) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
+	pud[(i + 1) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
+
+	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
+	/* Filter out unsupported __PAGE_KERNEL_* bits: */
+	pmd_entry &= RIP_REL_REF(__supported_pte_mask);
+	pmd_entry += sme_get_me_mask();
+	pmd_entry +=  physaddr;
+
+	for (i = 0; i < DIV_ROUND_UP(va_end - va_text, PMD_SIZE); i++) {
+		int idx = i + (physaddr >> PMD_SHIFT);
+
+		pmd[idx % PTRS_PER_PMD] = pmd_entry + i * PMD_SIZE;
+	}
+
+	/*
+	 * Fixup the kernel text+data virtual addresses. Note that
+	 * we might write invalid pmds, when the kernel is relocated
+	 * cleanup_highmap() fixes this up along with the mappings
+	 * beyond _end.
+	 *
+	 * Only the region occupied by the kernel image has so far
+	 * been checked against the table of usable memory regions
+	 * provided by the firmware, so invalidate pages outside that
+	 * region. A page table entry that maps to a reserved area of
+	 * memory would allow processor speculation into that area,
+	 * and on some hardware (particularly the UV platform) even
+	 * speculative access to some reserved areas is caught as an
+	 * error, causing the BIOS to halt the system.
+	 */
+
+	pmd = rip_rel_ptr(level2_kernel_pgt);
+
+	/* invalidate pages before the kernel image */
+	for (i = 0; i < pmd_index(va_text); i++)
+		pmd[i] &= ~_PAGE_PRESENT;
+
+	/* fixup pages that are part of the kernel image */
+	for (; i <= pmd_index(va_end); i++)
+		if (pmd[i] & _PAGE_PRESENT)
+			pmd[i] += load_delta;
+
+	/* invalidate pages after the kernel image */
+	for (; i < PTRS_PER_PMD; i++)
+		pmd[i] &= ~_PAGE_PRESENT;
+
+	return sme_postprocess_startup(bp, pmd, p2v_offset);
+}
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -XXX,XX +XXX,XX @@
  * Manage page tables very early on.
  */
 extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
-static unsigned int __initdata next_early_pgt;
+unsigned int __initdata next_early_pgt;
 pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
 
 #ifdef CONFIG_X86_5LEVEL
@@ -XXX,XX +XXX,XX @@ unsigned long vmemmap_base __ro_after_init = __VMEMMAP_BASE_L4;
 EXPORT_SYMBOL(vmemmap_base);
 #endif
 
-static inline bool check_la57_support(void)
-{
-	if (!IS_ENABLED(CONFIG_X86_5LEVEL))
-		return false;
-
-	/*
-	 * 5-level paging is detected and enabled at kernel decompression
-	 * stage. Only check if it has been enabled there.
-	 */
-	if (!(native_read_cr4() & X86_CR4_LA57))
-		return false;
-
-	RIP_REL_REF(__pgtable_l5_enabled)	= 1;
-	RIP_REL_REF(pgdir_shift)		= 48;
-	RIP_REL_REF(ptrs_per_p4d)		= 512;
-	RIP_REL_REF(page_offset_base)		= __PAGE_OFFSET_BASE_L5;
-	RIP_REL_REF(vmalloc_base)		= __VMALLOC_BASE_L5;
-	RIP_REL_REF(vmemmap_base)		= __VMEMMAP_BASE_L5;
-
-	return true;
-}
-
-static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
-						    pmdval_t *pmd,
-						    unsigned long p2v_offset)
-{
-	unsigned long paddr, paddr_end;
-	int i;
-
-	/* Encrypt the kernel and related (if SME is active) */
-	sme_encrypt_kernel(bp);
-
-	/*
-	 * Clear the memory encryption mask from the .bss..decrypted section.
-	 * The bss section will be memset to zero later in the initialization so
-	 * there is no need to zero it after changing the memory encryption
-	 * attribute.
-	 */
-	if (sme_get_me_mask()) {
-		paddr = (unsigned long)rip_rel_ptr(__start_bss_decrypted);
-		paddr_end = (unsigned long)rip_rel_ptr(__end_bss_decrypted);
-
-		for (; paddr < paddr_end; paddr += PMD_SIZE) {
-			/*
-			 * On SNP, transition the page to shared in the RMP table so that
-			 * it is consistent with the page table attribute change.
-			 *
-			 * __start_bss_decrypted has a virtual address in the high range
-			 * mapping (kernel .text). PVALIDATE, by way of
-			 * early_snp_set_memory_shared(), requires a valid virtual
-			 * address but the kernel is currently running off of the identity
-			 * mapping so use the PA to get a *currently* valid virtual address.
-			 */
-			early_snp_set_memory_shared(paddr, paddr, PTRS_PER_PMD);
-
-			i = pmd_index(paddr - p2v_offset);
-			pmd[i] -= sme_get_me_mask();
-		}
-	}
-
-	/*
-	 * Return the SME encryption mask (if SME is active) to be used as a
-	 * modifier for the initial pgdir entry programmed into CR3.
-	 */
-	return sme_get_me_mask();
-}
-
-/* Code in __startup_64() can be relocated during execution, but the compiler
- * doesn't have to generate PC-relative relocations when accessing globals from
- * that function. Clang actually does not generate them, which leads to
- * boot-time crashes. To work around this problem, every global pointer must
- * be accessed using RIP_REL_REF(). Kernel virtual addresses can be determined
- * by subtracting p2v_offset from the RIP-relative address.
- */
-unsigned long __head __startup_64(unsigned long p2v_offset,
-				  struct boot_params *bp)
-{
-	pmd_t (*early_pgts)[PTRS_PER_PMD] = rip_rel_ptr(early_dynamic_pgts);
-	unsigned long physaddr = (unsigned long)rip_rel_ptr(_text);
-	unsigned long va_text, va_end;
-	unsigned long pgtable_flags;
-	unsigned long load_delta;
-	pgdval_t *pgd;
-	p4dval_t *p4d;
-	pudval_t *pud;
-	pmdval_t *pmd, pmd_entry;
-	bool la57;
-	int i;
-
-	la57 = check_la57_support();
-
-	/* Is the address too large? */
-	if (physaddr >> MAX_PHYSMEM_BITS)
-		for (;;);
-
-	/*
-	 * Compute the delta between the address I am compiled to run at
-	 * and the address I am actually running at.
-	 */
-	load_delta = __START_KERNEL_map + p2v_offset;
-	RIP_REL_REF(phys_base) = load_delta;
-
-	/* Is the address not 2M aligned? */
-	if (load_delta & ~PMD_MASK)
-		for (;;);
-
-	va_text = physaddr - p2v_offset;
-	va_end  = (unsigned long)rip_rel_ptr(_end) - p2v_offset;
-
-	/* Include the SME encryption mask in the fixup value */
-	load_delta += sme_get_me_mask();
-
-	/* Fixup the physical addresses in the page table */
-
-	pgd = rip_rel_ptr(early_top_pgt);
-	pgd[pgd_index(__START_KERNEL_map)] += load_delta;
-
-	if (IS_ENABLED(CONFIG_X86_5LEVEL) && la57) {
-		p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt);
-		p4d[MAX_PTRS_PER_P4D - 1] += load_delta;
-
-		pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
-	}
-
-	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 2].pud += load_delta;
-	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 1].pud += load_delta;
-
-	for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
-		RIP_REL_REF(level2_fixmap_pgt)[i].pmd += load_delta;
-
-	/*
-	 * Set up the identity mapping for the switchover.  These
-	 * entries should *NOT* have the global bit set!  This also
-	 * creates a bunch of nonsense entries but that is fine --
-	 * it avoids problems around wraparound.
-	 */
-
-	pud = &early_pgts[0]->pmd;
-	pmd = &early_pgts[1]->pmd;
-	RIP_REL_REF(next_early_pgt) = 2;
-
-	pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
-
-	if (la57) {
-		p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd;
-
-		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
-		pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
-		pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;
-
-		i = physaddr >> P4D_SHIFT;
-		p4d[(i + 0) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
-		p4d[(i + 1) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags;
-	} else {
-		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
-		pgd[i + 0] = (pgdval_t)pud + pgtable_flags;
-		pgd[i + 1] = (pgdval_t)pud + pgtable_flags;
-	}
-
-	i = physaddr >> PUD_SHIFT;
-	pud[(i + 0) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
-	pud[(i + 1) % PTRS_PER_PUD] = (pudval_t)pmd + pgtable_flags;
-
-	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
-	/* Filter out unsupported __PAGE_KERNEL_* bits: */
-	pmd_entry &= RIP_REL_REF(__supported_pte_mask);
-	pmd_entry += sme_get_me_mask();
-	pmd_entry +=  physaddr;
-
-	for (i = 0; i < DIV_ROUND_UP(va_end - va_text, PMD_SIZE); i++) {
-		int idx = i + (physaddr >> PMD_SHIFT);
-
-		pmd[idx % PTRS_PER_PMD] = pmd_entry + i * PMD_SIZE;
-	}
-
-	/*
-	 * Fixup the kernel text+data virtual addresses. Note that
-	 * we might write invalid pmds, when the kernel is relocated
-	 * cleanup_highmap() fixes this up along with the mappings
-	 * beyond _end.
-	 *
-	 * Only the region occupied by the kernel image has so far
-	 * been checked against the table of usable memory regions
-	 * provided by the firmware, so invalidate pages outside that
-	 * region. A page table entry that maps to a reserved area of
-	 * memory would allow processor speculation into that area,
-	 * and on some hardware (particularly the UV platform) even
-	 * speculative access to some reserved areas is caught as an
-	 * error, causing the BIOS to halt the system.
-	 */
-
-	pmd = rip_rel_ptr(level2_kernel_pgt);
-
-	/* invalidate pages before the kernel image */
-	for (i = 0; i < pmd_index(va_text); i++)
-		pmd[i] &= ~_PAGE_PRESENT;
-
-	/* fixup pages that are part of the kernel image */
-	for (; i <= pmd_index(va_end); i++)
-		if (pmd[i] & _PAGE_PRESENT)
-			pmd[i] += load_delta;
-
-	/* invalidate pages after the kernel image */
-	for (; i < PTRS_PER_PMD; i++)
-		pmd[i] &= ~_PAGE_PRESENT;
-
-	return sme_postprocess_startup(bp, pmd, p2v_offset);
-}
-
 /* Wipe all early page tables except for the kernel symbol map */
 static void __init reset_early_page_tables(void)
 {
-- 
2.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

Now that __startup_64() is built using -fPIC, RIP_REL_REF() has become a
NOP and can be removed. Only some occurrences of rip_rel_ptr() will
remain, to explicitly take the address of certain global structures in
the 1:1 mapping of memory.

While at it, update the code comment to describe why this is needed.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/map_kernel.c | 41 ++++++++++----------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/arch/x86/boot/startup/map_kernel.c b/arch/x86/boot/startup/map_kernel.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/map_kernel.c
+++ b/arch/x86/boot/startup/map_kernel.c
@@ -XXX,XX +XXX,XX @@ static inline bool check_la57_support(void)
 	if (!(native_read_cr4() & X86_CR4_LA57))
 		return false;
 
-	RIP_REL_REF(__pgtable_l5_enabled)	= 1;
-	RIP_REL_REF(pgdir_shift)		= 48;
-	RIP_REL_REF(ptrs_per_p4d)		= 512;
-	RIP_REL_REF(page_offset_base)		= __PAGE_OFFSET_BASE_L5;
-	RIP_REL_REF(vmalloc_base)		= __VMALLOC_BASE_L5;
-	RIP_REL_REF(vmemmap_base)		= __VMEMMAP_BASE_L5;
+	__pgtable_l5_enabled	= 1;
+	pgdir_shift		= 48;
+	ptrs_per_p4d		= 512;
+	page_offset_base	= __PAGE_OFFSET_BASE_L5;
+	vmalloc_base		= __VMALLOC_BASE_L5;
+	vmemmap_base		= __VMEMMAP_BASE_L5;
 
 	return true;
 }
@@ -XXX,XX +XXX,XX @@ static unsigned long __head sme_postprocess_startup(struct boot_params *bp,
 	return sme_get_me_mask();
 }
 
-/* Code in __startup_64() can be relocated during execution, but the compiler
- * doesn't have to generate PC-relative relocations when accessing globals from
- * that function. Clang actually does not generate them, which leads to
- * boot-time crashes. To work around this problem, every global pointer must
- * be accessed using RIP_REL_REF(). Kernel virtual addresses can be determined
- * by subtracting p2v_offset from the RIP-relative address.
+/*
+ * This code is compiled using PIC codegen because it will execute from the
+ * early 1:1 mapping of memory, which deviates from the mapping expected by the
+ * linker. Due to this deviation, taking the address of a global variable will
+ * produce an ambiguous result when using the plain & operator.  Instead,
+ * rip_rel_ptr() must be used, which will return the RIP-relative address in
+ * the 1:1 mapping of memory. Kernel virtual addresses can be determined by
+ * subtracting p2v_offset from the RIP-relative address.
  */
 unsigned long __head __startup_64(unsigned long p2v_offset,
 				  struct boot_params *bp)
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
 	 * Compute the delta between the address I am compiled to run at
 	 * and the address I am actually running at.
 	 */
-	load_delta = __START_KERNEL_map + p2v_offset;
-	RIP_REL_REF(phys_base) = load_delta;
+	phys_base = load_delta = __START_KERNEL_map + p2v_offset;
 
 	/* Is the address not 2M aligned? */
 	if (load_delta & ~PMD_MASK)
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
 		pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE;
 	}
 
-	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 2].pud += load_delta;
-	RIP_REL_REF(level3_kernel_pgt)[PTRS_PER_PUD - 1].pud += load_delta;
+	level3_kernel_pgt[PTRS_PER_PUD - 2].pud += load_delta;
+	level3_kernel_pgt[PTRS_PER_PUD - 1].pud += load_delta;
 
 	for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
-		RIP_REL_REF(level2_fixmap_pgt)[i].pmd += load_delta;
+		level2_fixmap_pgt[i].pmd += load_delta;
 
 	/*
 	 * Set up the identity mapping for the switchover.  These
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
 
 	pud = &early_pgts[0]->pmd;
 	pmd = &early_pgts[1]->pmd;
-	RIP_REL_REF(next_early_pgt) = 2;
+	next_early_pgt = 2;
 
 	pgtable_flags = _KERNPG_TABLE_NOENC + sme_get_me_mask();
 
 	if (la57) {
-		p4d = &early_pgts[RIP_REL_REF(next_early_pgt)++]->pmd;
+		p4d = &early_pgts[next_early_pgt++]->pmd;
 
 		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;
 		pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;
@@ -XXX,XX +XXX,XX @@ unsigned long __head __startup_64(unsigned long p2v_offset,
 
 	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;
 	/* Filter out unsupported __PAGE_KERNEL_* bits: */
-	pmd_entry &= RIP_REL_REF(__supported_pte_mask);
+	pmd_entry &= __supported_pte_mask;
 	pmd_entry += sme_get_me_mask();
 	pmd_entry +=  physaddr;
 
-- 
2.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/Makefile                             | 1 +
 arch/x86/{mm/mem_encrypt_identity.c => boot/startup/sme.c} | 2 --
 arch/x86/mm/Makefile                                       | 6 ------
 3 files changed, 1 insertion(+), 8 deletions(-)

From: Ard Biesheuvel <ardb@kernel.org>

RIP_REL_REF() has no effect on code residing in arch/x86/boot/startup,
as it is built with -fPIC. So remove any occurrences from the SME
startup code.

Note the SME is the only caller of cc_set_mask() that requires this, so
drop it from there as well.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/sme.c        | 11 +++++------
 arch/x86/include/asm/coco.h        |  2 +-
 arch/x86/include/asm/mem_encrypt.h |  2 +-
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/arch/x86/boot/startup/sme.c b/arch/x86/boot/startup/sme.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/sme.c
+++ b/arch/x86/boot/startup/sme.c
@@ -XXX,XX +XXX,XX @@ void __head sme_encrypt_kernel(struct boot_params *bp)
 	 * instrumentation or checking boot_cpu_data in the cc_platform_has()
 	 * function.
 	 */
-	if (!sme_get_me_mask() ||
-	    RIP_REL_REF(sev_status) & MSR_AMD64_SEV_ENABLED)
+	if (!sme_get_me_mask() || sev_status & MSR_AMD64_SEV_ENABLED)
 		return;
 
 	/*
@@ -XXX,XX +XXX,XX @@ void __head sme_enable(struct boot_params *bp)
 	me_mask = 1UL << (ebx & 0x3f);
 
 	/* Check the SEV MSR whether SEV or SME is enabled */
-	RIP_REL_REF(sev_status) = msr = __rdmsr(MSR_AMD64_SEV);
+	sev_status = msr = __rdmsr(MSR_AMD64_SEV);
 	feature_mask = (msr & MSR_AMD64_SEV_ENABLED) ? AMD_SEV_BIT : AMD_SME_BIT;
 
 	/*
@@ -XXX,XX +XXX,XX @@ void __head sme_enable(struct boot_params *bp)
 			return;
 	}
 
-	RIP_REL_REF(sme_me_mask) = me_mask;
-	RIP_REL_REF(physical_mask) &= ~me_mask;
-	RIP_REL_REF(cc_vendor) = CC_VENDOR_AMD;
+	sme_me_mask	= me_mask;
+	physical_mask	&= ~me_mask;
+	cc_vendor	= CC_VENDOR_AMD;
 	cc_set_mask(me_mask);
 }
diff --git a/arch/x86/include/asm/coco.h b/arch/x86/include/asm/coco.h
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/include/asm/coco.h
+++ b/arch/x86/include/asm/coco.h
@@ -XXX,XX +XXX,XX @@ static inline u64 cc_get_mask(void)
 
 static inline void cc_set_mask(u64 mask)
 {
-	RIP_REL_REF(cc_mask) = mask;
+	cc_mask = mask;
 }
 
 u64 cc_mkenc(u64 val);
diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/include/asm/mem_encrypt.h
+++ b/arch/x86/include/asm/mem_encrypt.h
@@ -XXX,XX +XXX,XX @@ void __init sev_es_init_vc_handling(void);
 
 static inline u64 sme_get_me_mask(void)
 {
-	return RIP_REL_REF(sme_me_mask);
+	return sme_me_mask;
 }
 
 #define __bss_decrypted __section(".bss..decrypted")
-- 
2.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

Prepare for splitting off parts of the SEV core.c source file into a
file that carries code that must tolerate being called from the early
1:1 mapping. This will allow special build-time handling of thise code,
to ensure that it gets generated in a way that is compatible with the
early execution context.

So create a de-facto internal SEV API and put the definitions into
sev-internal.h. No attempt is made to allow this header file to be
included in arbitrary other sources - this is explicitly not the intent.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/compressed/sev.c      |  15 ++-
 arch/x86/coco/sev/core.c            | 108 +++--------------
 arch/x86/coco/sev/shared.c          |  64 ++--------
 arch/x86/include/asm/sev-internal.h | 122 ++++++++++++++++++++
 arch/x86/include/asm/sev.h          |  37 ++++++
 5 files changed, 194 insertions(+), 152 deletions(-)

diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/sev.c
+++ b/arch/x86/boot/compressed/sev.c
@@ -XXX,XX +XXX,XX @@ static bool fault_in_kernel_space(unsigned long address)
 #include "../../lib/inat.c"
 #include "../../lib/insn.c"
 
-/* Include code for early handlers */
-#include "../../coco/sev/shared.c"
+extern struct svsm_ca *boot_svsm_caa;
+extern u64 boot_svsm_caa_pa;
 
-static struct svsm_ca *svsm_get_caa(void)
+struct svsm_ca *svsm_get_caa(void)
 {
 	return boot_svsm_caa;
 }
 
-static u64 svsm_get_caa_pa(void)
+u64 svsm_get_caa_pa(void)
 {
 	return boot_svsm_caa_pa;
 }
 
-static int svsm_perform_call_protocol(struct svsm_call *call)
+int svsm_perform_call_protocol(struct svsm_call *call);
+
+/* Include code for early handlers */
+#include "../../coco/sev/shared.c"
+
+int svsm_perform_call_protocol(struct svsm_call *call)
 {
 	struct ghcb *ghcb;
 	int ret;
diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/coco/sev/core.c
+++ b/arch/x86/coco/sev/core.c
@@ -XXX,XX +XXX,XX @@
 #include <asm/cpu_entry_area.h>
 #include <asm/stacktrace.h>
 #include <asm/sev.h>
+#include <asm/sev-internal.h>
 #include <asm/insn-eval.h>
 #include <asm/fpu/xcr.h>
 #include <asm/processor.h>
@@ -XXX,XX +XXX,XX @@
 #include <asm/cpuid.h>
 #include <asm/cmdline.h>
 
-#define DR7_RESET_VALUE        0x400
-
 /* AP INIT values as documented in the APM2  section "Processor Initialization State" */
 #define AP_INIT_CS_LIMIT		0xffff
 #define AP_INIT_DS_LIMIT		0xffff
@@ -XXX,XX +XXX,XX @@ static const char * const sev_status_feat_names[] = {
 };
 
 /* For early boot hypervisor communication in SEV-ES enabled guests */
-static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
+struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
 
 /*
  * Needs to be in the .data section because we need it NULL before bss is
  * cleared
  */
-static struct ghcb *boot_ghcb __section(".data");
+struct ghcb *boot_ghcb __section(".data");
 
 /* Bitmap of SEV features supported by the hypervisor */
-static u64 sev_hv_features __ro_after_init;
+u64 sev_hv_features __ro_after_init;
 
 /* Secrets page physical address from the CC blob */
 static u64 secrets_pa __ro_after_init;
@@ -XXX,XX +XXX,XX @@ static u64 snp_tsc_scale __ro_after_init;
 static u64 snp_tsc_offset __ro_after_init;
 static u64 snp_tsc_freq_khz __ro_after_init;
 
-/* #VC handler runtime per-CPU data */
-struct sev_es_runtime_data {
-	struct ghcb ghcb_page;
-
-	/*
-	 * Reserve one page per CPU as backup storage for the unencrypted GHCB.
-	 * It is needed when an NMI happens while the #VC handler uses the real
-	 * GHCB, and the NMI handler itself is causing another #VC exception. In
-	 * that case the GHCB content of the first handler needs to be backed up
-	 * and restored.
-	 */
-	struct ghcb backup_ghcb;
-
-	/*
-	 * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions.
-	 * There is no need for it to be atomic, because nothing is written to
-	 * the GHCB between the read and the write of ghcb_active. So it is safe
-	 * to use it when a nested #VC exception happens before the write.
-	 *
-	 * This is necessary for example in the #VC->NMI->#VC case when the NMI
-	 * happens while the first #VC handler uses the GHCB. When the NMI code
-	 * raises a second #VC handler it might overwrite the contents of the
-	 * GHCB written by the first handler. To avoid this the content of the
-	 * GHCB is saved and restored when the GHCB is detected to be in use
-	 * already.
-	 */
-	bool ghcb_active;
-	bool backup_ghcb_active;
-
-	/*
-	 * Cached DR7 value - write it on DR7 writes and return it on reads.
-	 * That value will never make it to the real hardware DR7 as debugging
-	 * is currently unsupported in SEV-ES guests.
-	 */
-	unsigned long dr7;
-};
-
-struct ghcb_state {
-	struct ghcb *ghcb;
-};
 
 /* For early boot SVSM communication */
-static struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE);
+struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE);
 
-static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
-static DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
-static DEFINE_PER_CPU(struct svsm_ca *, svsm_caa);
-static DEFINE_PER_CPU(u64, svsm_caa_pa);
+DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
+DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
+DEFINE_PER_CPU(struct svsm_ca *, svsm_caa);
+DEFINE_PER_CPU(u64, svsm_caa_pa);
 
 static __always_inline bool on_vc_stack(struct pt_regs *regs)
 {
@@ -XXX,XX +XXX,XX @@ void noinstr __sev_es_ist_exit(void)
  *
  * Callers must disable local interrupts around it.
  */
-static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
+noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
 {
 	struct sev_es_runtime_data *data;
 	struct ghcb *ghcb;
@@ -XXX,XX +XXX,XX @@ static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
 	return ghcb;
 }
 
-static inline u64 sev_es_rd_ghcb_msr(void)
-{
-	return __rdmsr(MSR_AMD64_SEV_ES_GHCB);
-}
-
-static __always_inline void sev_es_wr_ghcb_msr(u64 val)
-{
-	u32 low, high;
-
-	low  = (u32)(val);
-	high = (u32)(val >> 32);
-
-	native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high);
-}
-
 static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt,
 				unsigned char *buffer)
 {
@@ -XXX,XX +XXX,XX @@ static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt)
 /* Include code shared with pre-decompression boot stage */
 #include "shared.c"
 
-static inline struct svsm_ca *svsm_get_caa(void)
-{
-	/*
-	 * Use rIP-relative references when called early in the boot. If
-	 * ->use_cas is set, then it is late in the boot and no need
-	 * to worry about rIP-relative references.
-	 */
-	if (RIP_REL_REF(sev_cfg).use_cas)
-		return this_cpu_read(svsm_caa);
-	else
-		return RIP_REL_REF(boot_svsm_caa);
-}
-
-static u64 svsm_get_caa_pa(void)
-{
-	/*
-	 * Use rIP-relative references when called early in the boot. If
-	 * ->use_cas is set, then it is late in the boot and no need
-	 * to worry about rIP-relative references.
-	 */
-	if (RIP_REL_REF(sev_cfg).use_cas)
-		return this_cpu_read(svsm_caa_pa);
-	else
-		return RIP_REL_REF(boot_svsm_caa_pa);
-}
-
-static noinstr void __sev_put_ghcb(struct ghcb_state *state)
+noinstr void __sev_put_ghcb(struct ghcb_state *state)
 {
 	struct sev_es_runtime_data *data;
 	struct ghcb *ghcb;
@@ -XXX,XX +XXX,XX @@ static noinstr void __sev_put_ghcb(struct ghcb_state *state)
 	}
 }
 
-static int svsm_perform_call_protocol(struct svsm_call *call)
+int svsm_perform_call_protocol(struct svsm_call *call)
 {
 	struct ghcb_state state;
 	unsigned long flags;
@@ -XXX,XX +XXX,XX @@ static u64 __init get_jump_table_addr(void)
 	return ret;
 }
 
-static void __head
+void __head
 early_set_pages_state(unsigned long vaddr, unsigned long paddr,
 		      unsigned long npages, enum psc_op op)
 {
diff --git a/arch/x86/coco/sev/shared.c b/arch/x86/coco/sev/shared.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/coco/sev/shared.c
+++ b/arch/x86/coco/sev/shared.c
@@ -XXX,XX +XXX,XX @@
  */
 u8 snp_vmpl __ro_after_init;
 EXPORT_SYMBOL_GPL(snp_vmpl);
-static struct svsm_ca *boot_svsm_caa __ro_after_init;
-static u64 boot_svsm_caa_pa __ro_after_init;
-
-static struct svsm_ca *svsm_get_caa(void);
-static u64 svsm_get_caa_pa(void);
-static int svsm_perform_call_protocol(struct svsm_call *call);
+struct svsm_ca *boot_svsm_caa __ro_after_init;
+u64 boot_svsm_caa_pa __ro_after_init;
 
 /* I/O parameters for CPUID-related helpers */
 struct cpuid_leaf {
@@ -XXX,XX +XXX,XX @@ struct cpuid_leaf {
 	u32 edx;
 };
 
-/*
- * Individual entries of the SNP CPUID table, as defined by the SNP
- * Firmware ABI, Revision 0.9, Section 7.1, Table 14.
- */
-struct snp_cpuid_fn {
-	u32 eax_in;
-	u32 ecx_in;
-	u64 xcr0_in;
-	u64 xss_in;
-	u32 eax;
-	u32 ebx;
-	u32 ecx;
-	u32 edx;
-	u64 __reserved;
-} __packed;
-
-/*
- * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,
- * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit
- * of 64 entries per CPUID table.
- */
-#define SNP_CPUID_COUNT_MAX 64
-
-struct snp_cpuid_table {
-	u32 count;
-	u32 __reserved1;
-	u64 __reserved2;
-	struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];
-} __packed;
-
 /*
  * Since feature negotiation related variables are set early in the boot
  * process they must reside in the .data section so as not to be zeroed
@@ -XXX,XX +XXX,XX @@ static u32 cpuid_std_range_max __ro_after_init;
 static u32 cpuid_hyp_range_max __ro_after_init;
 static u32 cpuid_ext_range_max __ro_after_init;
 
-static bool __init sev_es_check_cpu_features(void)
+bool __init sev_es_check_cpu_features(void)
 {
 	if (!has_cpuflag(X86_FEATURE_RDRAND)) {
 		error("RDRAND instruction not supported - no trusted source of randomness available\n");
@@ -XXX,XX +XXX,XX @@ static bool __init sev_es_check_cpu_features(void)
 	return true;
 }
 
-static void __head __noreturn
+void __head __noreturn
 sev_es_terminate(unsigned int set, unsigned int reason)
 {
 	u64 val = GHCB_MSR_TERM_REQ;
@@ -XXX,XX +XXX,XX @@ sev_es_terminate(unsigned int set, unsigned int reason)
 /*
  * The hypervisor features are available from GHCB version 2 onward.
  */
-static u64 get_hv_features(void)
+u64 get_hv_features(void)
 {
 	u64 val;
 
@@ -XXX,XX +XXX,XX @@ static u64 get_hv_features(void)
 	return GHCB_MSR_HV_FT_RESP_VAL(val);
 }
 
-static void snp_register_ghcb_early(unsigned long paddr)
+void snp_register_ghcb_early(unsigned long paddr)
 {
 	unsigned long pfn = paddr >> PAGE_SHIFT;
 	u64 val;
@@ -XXX,XX +XXX,XX @@ static void snp_register_ghcb_early(unsigned long paddr)
 		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER);
 }
 
-static bool sev_es_negotiate_protocol(void)
+bool sev_es_negotiate_protocol(void)
 {
 	u64 val;
 
@@ -XXX,XX +XXX,XX @@ static bool sev_es_negotiate_protocol(void)
 	return true;
 }
 
-static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
-{
-	ghcb->save.sw_exit_code = 0;
-	__builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
-}
-
 static bool vc_decoding_needed(unsigned long exit_code)
 {
 	/* Exceptions don't require to decode the instruction */
@@ -XXX,XX +XXX,XX @@ static int svsm_perform_ghcb_protocol(struct ghcb *ghcb, struct svsm_call *call)
 	return svsm_process_result_codes(call);
 }
 
-static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
-					  struct es_em_ctxt *ctxt,
-					  u64 exit_code, u64 exit_info_1,
-					  u64 exit_info_2)
+enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
+				   struct es_em_ctxt *ctxt,
+				   u64 exit_code, u64 exit_info_1,
+				   u64 exit_info_2)
 {
 	/* Fill in protocol and format specifiers */
 	ghcb->protocol_version = ghcb_version;
@@ -XXX,XX +XXX,XX @@ static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid
  * while running with the initial identity mapping as well as the
  * switch-over to kernel virtual addresses later.
  */
-static const struct snp_cpuid_table *snp_cpuid_get_table(void)
+const struct snp_cpuid_table *snp_cpuid_get_table(void)
 {
 	return rip_rel_ptr(&cpuid_table_copy);
 }
diff --git a/arch/x86/include/asm/sev-internal.h b/arch/x86/include/asm/sev-internal.h
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/arch/x86/include/asm/sev-internal.h
@@ -XXX,XX +XXX,XX @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#define DR7_RESET_VALUE        0x400
+
+extern struct ghcb boot_ghcb_page;
+extern struct ghcb *boot_ghcb;
+extern u64 sev_hv_features;
+
+/* #VC handler runtime per-CPU data */
+struct sev_es_runtime_data {
+	struct ghcb ghcb_page;
+
+	/*
+	 * Reserve one page per CPU as backup storage for the unencrypted GHCB.
+	 * It is needed when an NMI happens while the #VC handler uses the real
+	 * GHCB, and the NMI handler itself is causing another #VC exception. In
+	 * that case the GHCB content of the first handler needs to be backed up
+	 * and restored.
+	 */
+	struct ghcb backup_ghcb;
+
+	/*
+	 * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions.
+	 * There is no need for it to be atomic, because nothing is written to
+	 * the GHCB between the read and the write of ghcb_active. So it is safe
+	 * to use it when a nested #VC exception happens before the write.
+	 *
+	 * This is necessary for example in the #VC->NMI->#VC case when the NMI
+	 * happens while the first #VC handler uses the GHCB. When the NMI code
+	 * raises a second #VC handler it might overwrite the contents of the
+	 * GHCB written by the first handler. To avoid this the content of the
+	 * GHCB is saved and restored when the GHCB is detected to be in use
+	 * already.
+	 */
+	bool ghcb_active;
+	bool backup_ghcb_active;
+
+	/*
+	 * Cached DR7 value - write it on DR7 writes and return it on reads.
+	 * That value will never make it to the real hardware DR7 as debugging
+	 * is currently unsupported in SEV-ES guests.
+	 */
+	unsigned long dr7;
+};
+
+struct ghcb_state {
+	struct ghcb *ghcb;
+};
+
+extern struct svsm_ca boot_svsm_ca_page;
+
+struct ghcb *__sev_get_ghcb(struct ghcb_state *state);
+void __sev_put_ghcb(struct ghcb_state *state);
+
+DECLARE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
+DECLARE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
+
+void early_set_pages_state(unsigned long vaddr, unsigned long paddr,
+			   unsigned long npages, enum psc_op op);
+
+void __noreturn sev_es_terminate(unsigned int set, unsigned int reason);
+
+DECLARE_PER_CPU(struct svsm_ca *, svsm_caa);
+DECLARE_PER_CPU(u64, svsm_caa_pa);
+
+extern struct svsm_ca *boot_svsm_caa;
+extern u64 boot_svsm_caa_pa;
+
+static __always_inline struct svsm_ca *svsm_get_caa(void)
+{
+	/*
+	 * Use rIP-relative references when called early in the boot. If
+	 * ->use_cas is set, then it is late in the boot and no need
+	 * to worry about rIP-relative references.
+	 */
+	if (RIP_REL_REF(sev_cfg).use_cas)
+		return this_cpu_read(svsm_caa);
+	else
+		return RIP_REL_REF(boot_svsm_caa);
+}
+
+static __always_inline u64 svsm_get_caa_pa(void)
+{
+	/*
+	 * Use rIP-relative references when called early in the boot. If
+	 * ->use_cas is set, then it is late in the boot and no need
+	 * to worry about rIP-relative references.
+	 */
+	if (RIP_REL_REF(sev_cfg).use_cas)
+		return this_cpu_read(svsm_caa_pa);
+	else
+		return RIP_REL_REF(boot_svsm_caa_pa);
+}
+
+int svsm_perform_call_protocol(struct svsm_call *call);
+
+static inline u64 sev_es_rd_ghcb_msr(void)
+{
+	return __rdmsr(MSR_AMD64_SEV_ES_GHCB);
+}
+
+static __always_inline void sev_es_wr_ghcb_msr(u64 val)
+{
+	u32 low, high;
+
+	low  = (u32)(val);
+	high = (u32)(val >> 32);
+
+	native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high);
+}
+
+enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
+				   struct es_em_ctxt *ctxt,
+				   u64 exit_code, u64 exit_info_1,
+				   u64 exit_info_2);
+
+void snp_register_ghcb_early(unsigned long paddr);
+bool sev_es_negotiate_protocol(void);
+bool sev_es_check_cpu_features(void);
+u64 get_hv_features(void);
+
+const struct snp_cpuid_table *snp_cpuid_get_table(void);
diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/include/asm/sev.h
+++ b/arch/x86/include/asm/sev.h
@@ -XXX,XX +XXX,XX @@
 #include <asm/sev-common.h>
 #include <asm/coco.h>
 #include <asm/set_memory.h>
+#include <asm/svm.h>
 
 #define GHCB_PROTOCOL_MIN	1ULL
 #define GHCB_PROTOCOL_MAX	2ULL
@@ -XXX,XX +XXX,XX @@ extern void vc_no_ghcb(void);
 extern void vc_boot_ghcb(void);
 extern bool handle_vc_boot_ghcb(struct pt_regs *regs);
 
+/*
+ * Individual entries of the SNP CPUID table, as defined by the SNP
+ * Firmware ABI, Revision 0.9, Section 7.1, Table 14.
+ */
+struct snp_cpuid_fn {
+	u32 eax_in;
+	u32 ecx_in;
+	u64 xcr0_in;
+	u64 xss_in;
+	u32 eax;
+	u32 ebx;
+	u32 ecx;
+	u32 edx;
+	u64 __reserved;
+} __packed;
+
+/*
+ * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,
+ * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit
+ * of 64 entries per CPUID table.
+ */
+#define SNP_CPUID_COUNT_MAX 64
+
+struct snp_cpuid_table {
+	u32 count;
+	u32 __reserved1;
+	u64 __reserved2;
+	struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];
+} __packed;
+
 /* PVALIDATE return codes */
 #define PVALIDATE_FAIL_SIZEMISMATCH	6
 
@@ -XXX,XX +XXX,XX @@ int snp_send_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req
 void __init snp_secure_tsc_prepare(void);
 void __init snp_secure_tsc_init(void);
 
+static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
+{
+	ghcb->save.sw_exit_code = 0;
+	__builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
+}
+
 #else	/* !CONFIG_AMD_MEM_ENCRYPT */
 
 #define snp_vmpl 0
-- 
2.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

Disentangle the SEV core code and the SEV code that is called during
early boot. The latter piece will be moved into startup/ in a subsequent
patch.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/compressed/sev.c |    2 +
 arch/x86/coco/sev/Makefile     |   12 +-
 arch/x86/coco/sev/core.c       | 1574 ++++----------------
 arch/x86/coco/sev/shared.c     |  281 ----
 arch/x86/coco/sev/startup.c    | 1395 +++++++++++++++++
 5 files changed, 1658 insertions(+), 1606 deletions(-)

diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/compressed/sev.c
+++ b/arch/x86/boot/compressed/sev.c
@@ -XXX,XX +XXX,XX @@ u64 svsm_get_caa_pa(void)
 
 int svsm_perform_call_protocol(struct svsm_call *call);
 
+u8 snp_vmpl;
+
 /* Include code for early handlers */
 #include "../../coco/sev/shared.c"
 
diff --git a/arch/x86/coco/sev/Makefile b/arch/x86/coco/sev/Makefile
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/coco/sev/Makefile
+++ b/arch/x86/coco/sev/Makefile
@@ -XXX,XX +XXX,XX @@
 # SPDX-License-Identifier: GPL-2.0
 
-obj-y += core.o
+obj-y += core.o startup.o
 
 # jump tables are emitted using absolute references in non-PIC code
 # so they cannot be used in the early SEV startup code
-CFLAGS_core.o += -fno-jump-tables
+CFLAGS_startup.o += -fno-jump-tables
 
 ifdef CONFIG_FUNCTION_TRACER
-CFLAGS_REMOVE_core.o = -pg
+CFLAGS_REMOVE_startup.o = -pg
 endif
 
-KASAN_SANITIZE_core.o	:= n
-KMSAN_SANITIZE_core.o	:= n
-KCOV_INSTRUMENT_core.o	:= n
+KASAN_SANITIZE_startup.o	:= n
+KMSAN_SANITIZE_startup.o	:= n
+KCOV_INSTRUMENT_startup.o	:= n
 
 # With some compiler versions the generated code results in boot hangs, caused
 # by several compilation units. To be safe, disable all instrumentation.
diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/coco/sev/core.c
+++ b/arch/x86/coco/sev/core.c
@@ -XXX,XX +XXX,XX @@ static const char * const sev_status_feat_names[] = {
 	[MSR_AMD64_SNP_SMT_PROT_BIT]		= "SMTProt",
 };
 
-/* For early boot hypervisor communication in SEV-ES enabled guests */
-struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
-
-/*
- * Needs to be in the .data section because we need it NULL before bss is
- * cleared
- */
-struct ghcb *boot_ghcb __section(".data");
-
-/* Bitmap of SEV features supported by the hypervisor */
-u64 sev_hv_features __ro_after_init;
-
 /* Secrets page physical address from the CC blob */
 static u64 secrets_pa __ro_after_init;
 
@@ -XXX,XX +XXX,XX @@ static u64 snp_tsc_scale __ro_after_init;
 static u64 snp_tsc_offset __ro_after_init;
 static u64 snp_tsc_freq_khz __ro_after_init;
 
-
-/* For early boot SVSM communication */
-struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE);
-
 DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
 DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
-DEFINE_PER_CPU(struct svsm_ca *, svsm_caa);
-DEFINE_PER_CPU(u64, svsm_caa_pa);
+
+/*
+ * SVSM related information:
+ *   When running under an SVSM, the VMPL that Linux is executing at must be
+ *   non-zero. The VMPL is therefore used to indicate the presence of an SVSM.
+ */
+u8 snp_vmpl __ro_after_init;
+EXPORT_SYMBOL_GPL(snp_vmpl);
 
 static __always_inline bool on_vc_stack(struct pt_regs *regs)
 {
@@ -XXX,XX +XXX,XX @@ static __always_inline bool on_vc_stack(struct pt_regs *regs)
 	return ((sp >= __this_cpu_ist_bottom_va(VC)) && (sp < __this_cpu_ist_top_va(VC)));
 }
 
+
 /*
  * This function handles the case when an NMI is raised in the #VC
  * exception handler entry code, before the #VC handler has switched off
@@ -XXX,XX +XXX,XX @@ void noinstr __sev_es_ist_exit(void)
 	this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist);
 }
 
-/*
- * Nothing shall interrupt this code path while holding the per-CPU
- * GHCB. The backup GHCB is only for NMIs interrupting this path.
- *
- * Callers must disable local interrupts around it.
- */
-noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
+static u64 __init get_snp_jump_table_addr(void)
 {
-	struct sev_es_runtime_data *data;
-	struct ghcb *ghcb;
-
-	WARN_ON(!irqs_disabled());
-
-	data = this_cpu_read(runtime_data);
-	ghcb = &data->ghcb_page;
-
-	if (unlikely(data->ghcb_active)) {
-		/* GHCB is already in use - save its contents */
-
-		if (unlikely(data->backup_ghcb_active)) {
-			/*
-			 * Backup-GHCB is also already in use. There is no way
-			 * to continue here so just kill the machine. To make
-			 * panic() work, mark GHCBs inactive so that messages
-			 * can be printed out.
-			 */
-			data->ghcb_active        = false;
-			data->backup_ghcb_active = false;
-
-			instrumentation_begin();
-			panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use");
-			instrumentation_end();
-		}
-
-		/* Mark backup_ghcb active before writing to it */
-		data->backup_ghcb_active = true;
-
-		state->ghcb = &data->backup_ghcb;
+	struct snp_secrets_page *secrets;
+	void __iomem *mem;
+	u64 addr;
 
-		/* Backup GHCB content */
-		*state->ghcb = *ghcb;
-	} else {
-		state->ghcb = NULL;
-		data->ghcb_active = true;
+	mem = ioremap_encrypted(secrets_pa, PAGE_SIZE);
+	if (!mem) {
+		pr_err("Unable to locate AP jump table address: failed to map the SNP secrets page.\n");
+		return 0;
 	}
 
-	return ghcb;
-}
+	secrets = (__force struct snp_secrets_page *)mem;
 
-static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt,
-				unsigned char *buffer)
-{
-	return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
+	addr = secrets->os_area.ap_jump_table_pa;
+	iounmap(mem);
+
+	return addr;
 }
 
-static enum es_result __vc_decode_user_insn(struct es_em_ctxt *ctxt)
+void noinstr __sev_es_nmi_complete(void)
 {
-	char buffer[MAX_INSN_SIZE];
-	int insn_bytes;
-
-	insn_bytes = insn_fetch_from_user_inatomic(ctxt->regs, buffer);
-	if (insn_bytes == 0) {
-		/* Nothing could be copied */
-		ctxt->fi.vector     = X86_TRAP_PF;
-		ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER;
-		ctxt->fi.cr2        = ctxt->regs->ip;
-		return ES_EXCEPTION;
-	} else if (insn_bytes == -EINVAL) {
-		/* Effective RIP could not be calculated */
-		ctxt->fi.vector     = X86_TRAP_GP;
-		ctxt->fi.error_code = 0;
-		ctxt->fi.cr2        = 0;
-		return ES_EXCEPTION;
-	}
-
-	if (!insn_decode_from_regs(&ctxt->insn, ctxt->regs, buffer, insn_bytes))
-		return ES_DECODE_FAILED;
+	struct ghcb_state state;
+	struct ghcb *ghcb;
 
-	if (ctxt->insn.immediate.got)
-		return ES_OK;
-	else
-		return ES_DECODE_FAILED;
-}
+	ghcb = __sev_get_ghcb(&state);
 
-static enum es_result __vc_decode_kern_insn(struct es_em_ctxt *ctxt)
-{
-	char buffer[MAX_INSN_SIZE];
-	int res, ret;
+	vc_ghcb_invalidate(ghcb);
+	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE);
+	ghcb_set_sw_exit_info_1(ghcb, 0);
+	ghcb_set_sw_exit_info_2(ghcb, 0);
 
-	res = vc_fetch_insn_kernel(ctxt, buffer);
-	if (res) {
-		ctxt->fi.vector     = X86_TRAP_PF;
-		ctxt->fi.error_code = X86_PF_INSTR;
-		ctxt->fi.cr2        = ctxt->regs->ip;
-		return ES_EXCEPTION;
-	}
+	sev_es_wr_ghcb_msr(__pa_nodebug(ghcb));
+	VMGEXIT();
 
-	ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
-	if (ret < 0)
-		return ES_DECODE_FAILED;
-	else
-		return ES_OK;
+	__sev_put_ghcb(&state);
 }
 
-static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
+static u64 __init get_jump_table_addr(void)
 {
-	if (user_mode(ctxt->regs))
-		return __vc_decode_user_insn(ctxt);
-	else
-		return __vc_decode_kern_insn(ctxt);
-}
+	struct ghcb_state state;
+	unsigned long flags;
+	struct ghcb *ghcb;
+	u64 ret = 0;
 
-static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
-				   char *dst, char *buf, size_t size)
-{
-	unsigned long error_code = X86_PF_PROT | X86_PF_WRITE;
+	if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
+		return get_snp_jump_table_addr();
 
-	/*
-	 * This function uses __put_user() independent of whether kernel or user
-	 * memory is accessed. This works fine because __put_user() does no
-	 * sanity checks of the pointer being accessed. All that it does is
-	 * to report when the access failed.
-	 *
-	 * Also, this function runs in atomic context, so __put_user() is not
-	 * allowed to sleep. The page-fault handler detects that it is running
-	 * in atomic context and will not try to take mmap_sem and handle the
-	 * fault, so additional pagefault_enable()/disable() calls are not
-	 * needed.
-	 *
-	 * The access can't be done via copy_to_user() here because
-	 * vc_write_mem() must not use string instructions to access unsafe
-	 * memory. The reason is that MOVS is emulated by the #VC handler by
-	 * splitting the move up into a read and a write and taking a nested #VC
-	 * exception on whatever of them is the MMIO access. Using string
-	 * instructions here would cause infinite nesting.
-	 */
-	switch (size) {
-	case 1: {
-		u8 d1;
-		u8 __user *target = (u8 __user *)dst;
-
-		memcpy(&d1, buf, 1);
-		if (__put_user(d1, target))
-			goto fault;
-		break;
-	}
-	case 2: {
-		u16 d2;
-		u16 __user *target = (u16 __user *)dst;
+	local_irq_save(flags);
 
-		memcpy(&d2, buf, 2);
-		if (__put_user(d2, target))
-			goto fault;
-		break;
-	}
-	case 4: {
-		u32 d4;
-		u32 __user *target = (u32 __user *)dst;
+	ghcb = __sev_get_ghcb(&state);
 
-		memcpy(&d4, buf, 4);
-		if (__put_user(d4, target))
-			goto fault;
-		break;
-	}
-	case 8: {
-		u64 d8;
-		u64 __user *target = (u64 __user *)dst;
+	vc_ghcb_invalidate(ghcb);
+	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE);
+	ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE);
+	ghcb_set_sw_exit_info_2(ghcb, 0);
 
-		memcpy(&d8, buf, 8);
-		if (__put_user(d8, target))
-			goto fault;
-		break;
-	}
-	default:
-		WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
-		return ES_UNSUPPORTED;
-	}
+	sev_es_wr_ghcb_msr(__pa(ghcb));
+	VMGEXIT();
 
-	return ES_OK;
+	if (ghcb_sw_exit_info_1_is_valid(ghcb) &&
+	    ghcb_sw_exit_info_2_is_valid(ghcb))
+		ret = ghcb->save.sw_exit_info_2;
 
-fault:
-	if (user_mode(ctxt->regs))
-		error_code |= X86_PF_USER;
+	__sev_put_ghcb(&state);
 
-	ctxt->fi.vector = X86_TRAP_PF;
-	ctxt->fi.error_code = error_code;
-	ctxt->fi.cr2 = (unsigned long)dst;
+	local_irq_restore(flags);
 
-	return ES_EXCEPTION;
+	return ret;
 }
 
-static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
-				  char *src, char *buf, size_t size)
+static inline void __pval_terminate(u64 pfn, bool action, unsigned int page_size,
+				    int ret, u64 svsm_ret)
 {
-	unsigned long error_code = X86_PF_PROT;
-
-	/*
-	 * This function uses __get_user() independent of whether kernel or user
-	 * memory is accessed. This works fine because __get_user() does no
-	 * sanity checks of the pointer being accessed. All that it does is
-	 * to report when the access failed.
-	 *
-	 * Also, this function runs in atomic context, so __get_user() is not
-	 * allowed to sleep. The page-fault handler detects that it is running
-	 * in atomic context and will not try to take mmap_sem and handle the
-	 * fault, so additional pagefault_enable()/disable() calls are not
-	 * needed.
-	 *
-	 * The access can't be done via copy_from_user() here because
-	 * vc_read_mem() must not use string instructions to access unsafe
-	 * memory. The reason is that MOVS is emulated by the #VC handler by
-	 * splitting the move up into a read and a write and taking a nested #VC
-	 * exception on whatever of them is the MMIO access. Using string
-	 * instructions here would cause infinite nesting.
-	 */
-	switch (size) {
-	case 1: {
-		u8 d1;
-		u8 __user *s = (u8 __user *)src;
-
-		if (__get_user(d1, s))
-			goto fault;
-		memcpy(buf, &d1, 1);
-		break;
-	}
-	case 2: {
-		u16 d2;
-		u16 __user *s = (u16 __user *)src;
-
-		if (__get_user(d2, s))
-			goto fault;
-		memcpy(buf, &d2, 2);
-		break;
-	}
-	case 4: {
-		u32 d4;
-		u32 __user *s = (u32 __user *)src;
-
-		if (__get_user(d4, s))
-			goto fault;
-		memcpy(buf, &d4, 4);
-		break;
-	}
-	case 8: {
-		u64 d8;
-		u64 __user *s = (u64 __user *)src;
-		if (__get_user(d8, s))
-			goto fault;
-		memcpy(buf, &d8, 8);
-		break;
-	}
-	default:
-		WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
-		return ES_UNSUPPORTED;
-	}
+	WARN(1, "PVALIDATE failure: pfn: 0x%llx, action: %u, size: %u, ret: %d, svsm_ret: 0x%llx\n",
+	     pfn, action, page_size, ret, svsm_ret);
 
-	return ES_OK;
+	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
+}
 
-fault:
-	if (user_mode(ctxt->regs))
-		error_code |= X86_PF_USER;
+static void svsm_pval_terminate(struct svsm_pvalidate_call *pc, int ret, u64 svsm_ret)
+{
+	unsigned int page_size;
+	bool action;
+	u64 pfn;
 
-	ctxt->fi.vector = X86_TRAP_PF;
-	ctxt->fi.error_code = error_code;
-	ctxt->fi.cr2 = (unsigned long)src;
+	pfn = pc->entry[pc->cur_index].pfn;
+	action = pc->entry[pc->cur_index].action;
+	page_size = pc->entry[pc->cur_index].page_size;
 
-	return ES_EXCEPTION;
+	__pval_terminate(pfn, action, page_size, ret, svsm_ret);
 }
 
-static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
-					   unsigned long vaddr, phys_addr_t *paddr)
+static void pval_pages(struct snp_psc_desc *desc)
 {
-	unsigned long va = (unsigned long)vaddr;
-	unsigned int level;
-	phys_addr_t pa;
-	pgd_t *pgd;
-	pte_t *pte;
-
-	pgd = __va(read_cr3_pa());
-	pgd = &pgd[pgd_index(va)];
-	pte = lookup_address_in_pgd(pgd, va, &level);
-	if (!pte) {
-		ctxt->fi.vector     = X86_TRAP_PF;
-		ctxt->fi.cr2        = vaddr;
-		ctxt->fi.error_code = 0;
-
-		if (user_mode(ctxt->regs))
-			ctxt->fi.error_code |= X86_PF_USER;
+	struct psc_entry *e;
+	unsigned long vaddr;
+	unsigned int size;
+	unsigned int i;
+	bool validate;
+	u64 pfn;
+	int rc;
 
-		return ES_EXCEPTION;
-	}
+	for (i = 0; i <= desc->hdr.end_entry; i++) {
+		e = &desc->entries[i];
 
-	if (WARN_ON_ONCE(pte_val(*pte) & _PAGE_ENC))
-		/* Emulated MMIO to/from encrypted memory not supported */
-		return ES_UNSUPPORTED;
+		pfn = e->gfn;
+		vaddr = (unsigned long)pfn_to_kaddr(pfn);
+		size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
+		validate = e->operation == SNP_PAGE_STATE_PRIVATE;
 
-	pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
-	pa |= va & ~page_level_mask(level);
+		rc = pvalidate(vaddr, size, validate);
+		if (!rc)
+			continue;
 
-	*paddr = pa;
+		if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) {
+			unsigned long vaddr_end = vaddr + PMD_SIZE;
 
-	return ES_OK;
+			for (; vaddr < vaddr_end; vaddr += PAGE_SIZE, pfn++) {
+				rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
+				if (rc)
+					__pval_terminate(pfn, validate, RMP_PG_SIZE_4K, rc, 0);
+			}
+		} else {
+			__pval_terminate(pfn, validate, size, rc, 0);
+		}
+	}
 }
 
-static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
+static u64 svsm_build_ca_from_pfn_range(u64 pfn, u64 pfn_end, bool action,
+					struct svsm_pvalidate_call *pc)
 {
-	BUG_ON(size > 4);
+	struct svsm_pvalidate_entry *pe;
 
-	if (user_mode(ctxt->regs)) {
-		struct thread_struct *t = &current->thread;
-		struct io_bitmap *iobm = t->io_bitmap;
-		size_t idx;
+	/* Nothing in the CA yet */
+	pc->num_entries = 0;
+	pc->cur_index   = 0;
 
-		if (!iobm)
-			goto fault;
+	pe = &pc->entry[0];
 
-		for (idx = port; idx < port + size; ++idx) {
-			if (test_bit(idx, iobm->bitmap))
-				goto fault;
-		}
-	}
+	while (pfn < pfn_end) {
+		pe->page_size = RMP_PG_SIZE_4K;
+		pe->action    = action;
+		pe->ignore_cf = 0;
+		pe->pfn       = pfn;
 
-	return ES_OK;
+		pe++;
+		pfn++;
 
-fault:
-	ctxt->fi.vector = X86_TRAP_GP;
-	ctxt->fi.error_code = 0;
+		pc->num_entries++;
+		if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT)
+			break;
+	}
 
-	return ES_EXCEPTION;
+	return pfn;
 }
 
-static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt)
+static int svsm_build_ca_from_psc_desc(struct snp_psc_desc *desc, unsigned int desc_entry,
+				       struct svsm_pvalidate_call *pc)
 {
-	long error_code = ctxt->fi.error_code;
-	int trapnr = ctxt->fi.vector;
-
-	ctxt->regs->orig_ax = ctxt->fi.error_code;
-
-	switch (trapnr) {
-	case X86_TRAP_GP:
-		exc_general_protection(ctxt->regs, error_code);
-		break;
-	case X86_TRAP_UD:
-		exc_invalid_op(ctxt->regs);
-		break;
-	case X86_TRAP_PF:
-		write_cr2(ctxt->fi.cr2);
-		exc_page_fault(ctxt->regs, error_code);
-		break;
-	case X86_TRAP_AC:
-		exc_alignment_check(ctxt->regs, error_code);
-		break;
-	default:
-		pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n");
-		BUG();
-	}
-}
+	struct svsm_pvalidate_entry *pe;
+	struct psc_entry *e;
 
-/* Include code shared with pre-decompression boot stage */
-#include "shared.c"
+	/* Nothing in the CA yet */
+	pc->num_entries = 0;
+	pc->cur_index   = 0;
 
-noinstr void __sev_put_ghcb(struct ghcb_state *state)
-{
-	struct sev_es_runtime_data *data;
-	struct ghcb *ghcb;
+	pe = &pc->entry[0];
+	e  = &desc->entries[desc_entry];
 
-	WARN_ON(!irqs_disabled());
+	while (desc_entry <= desc->hdr.end_entry) {
+		pe->page_size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
+		pe->action    = e->operation == SNP_PAGE_STATE_PRIVATE;
+		pe->ignore_cf = 0;
+		pe->pfn       = e->gfn;
 
-	data = this_cpu_read(runtime_data);
-	ghcb = &data->ghcb_page;
+		pe++;
+		e++;
 
-	if (state->ghcb) {
-		/* Restore GHCB from Backup */
-		*ghcb = *state->ghcb;
-		data->backup_ghcb_active = false;
-		state->ghcb = NULL;
-	} else {
-		/*
-		 * Invalidate the GHCB so a VMGEXIT instruction issued
-		 * from userspace won't appear to be valid.
-		 */
-		vc_ghcb_invalidate(ghcb);
-		data->ghcb_active = false;
+		desc_entry++;
+		pc->num_entries++;
+		if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT)
+			break;
 	}
+
+	return desc_entry;
 }
 
-int svsm_perform_call_protocol(struct svsm_call *call)
+static void svsm_pval_pages(struct snp_psc_desc *desc)
 {
-	struct ghcb_state state;
+	struct svsm_pvalidate_entry pv_4k[VMGEXIT_PSC_MAX_ENTRY];
+	unsigned int i, pv_4k_count = 0;
+	struct svsm_pvalidate_call *pc;
+	struct svsm_call call = {};
 	unsigned long flags;
-	struct ghcb *ghcb;
+	bool action;
+	u64 pc_pa;
 	int ret;
 
 	/*
@@ -XXX,XX +XXX,XX @@ int svsm_perform_call_protocol(struct svsm_call *call)
 	flags = native_local_irq_save();
 
 	/*
-	 * Use rip-relative references when called early in the boot. If
-	 * ghcbs_initialized is set, then it is late in the boot and no need
-	 * to worry about rip-relative references in called functions.
+	 * The SVSM calling area (CA) can support processing 510 entries at a
+	 * time. Loop through the Page State Change descriptor until the CA is
+	 * full or the last entry in the descriptor is reached, at which time
+	 * the SVSM is invoked. This repeats until all entries in the descriptor
+	 * are processed.
 	 */
-	if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
-		ghcb = __sev_get_ghcb(&state);
-	else if (RIP_REL_REF(boot_ghcb))
-		ghcb = RIP_REL_REF(boot_ghcb);
-	else
-		ghcb = NULL;
+	call.caa = svsm_get_caa();
 
-	do {
-		ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call)
-			   : svsm_perform_msr_protocol(call);
-	} while (ret == -EAGAIN);
+	pc = (struct svsm_pvalidate_call *)call.caa->svsm_buffer;
+	pc_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer);
 
-	if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
-		__sev_put_ghcb(&state);
+	/* Protocol 0, Call ID 1 */
+	call.rax = SVSM_CORE_CALL(SVSM_CORE_PVALIDATE);
+	call.rcx = pc_pa;
 
-	native_local_irq_restore(flags);
+	for (i = 0; i <= desc->hdr.end_entry;) {
+		i = svsm_build_ca_from_psc_desc(desc, i, pc);
 
-	return ret;
-}
+		do {
+			ret = svsm_perform_call_protocol(&call);
+			if (!ret)
+				continue;
 
-void noinstr __sev_es_nmi_complete(void)
-{
-	struct ghcb_state state;
-	struct ghcb *ghcb;
+			/*
+			 * Check if the entry failed because of an RMP mismatch (a
+			 * PVALIDATE at 2M was requested, but the page is mapped in
+			 * the RMP as 4K).
+			 */
 
-	ghcb = __sev_get_ghcb(&state);
+			if (call.rax_out == SVSM_PVALIDATE_FAIL_SIZEMISMATCH &&
+			    pc->entry[pc->cur_index].page_size == RMP_PG_SIZE_2M) {
+				/* Save this entry for post-processing at 4K */
+				pv_4k[pv_4k_count++] = pc->entry[pc->cur_index];
+
+				/* Skip to the next one unless at the end of the list */
+				pc->cur_index++;
+				if (pc->cur_index < pc->num_entries)
+					ret = -EAGAIN;
+				else
+					ret = 0;
+			}
+		} while (ret == -EAGAIN);
 
-	vc_ghcb_invalidate(ghcb);
-	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE);
-	ghcb_set_sw_exit_info_1(ghcb, 0);
-	ghcb_set_sw_exit_info_2(ghcb, 0);
+		if (ret)
+			svsm_pval_terminate(pc, ret, call.rax_out);
+	}
 
-	sev_es_wr_ghcb_msr(__pa_nodebug(ghcb));
-	VMGEXIT();
+	/* Process any entries that failed to be validated at 2M and validate them at 4K */
+	for (i = 0; i < pv_4k_count; i++) {
+		u64 pfn, pfn_end;
 
-	__sev_put_ghcb(&state);
-}
+		action  = pv_4k[i].action;
+		pfn     = pv_4k[i].pfn;
+		pfn_end = pfn + 512;
 
-static u64 __init get_snp_jump_table_addr(void)
-{
-	struct snp_secrets_page *secrets;
-	void __iomem *mem;
-	u64 addr;
+		while (pfn < pfn_end) {
+			pfn = svsm_build_ca_from_pfn_range(pfn, pfn_end, action, pc);
 
-	mem = ioremap_encrypted(secrets_pa, PAGE_SIZE);
-	if (!mem) {
-		pr_err("Unable to locate AP jump table address: failed to map the SNP secrets page.\n");
-		return 0;
+			ret = svsm_perform_call_protocol(&call);
+			if (ret)
+				svsm_pval_terminate(pc, ret, call.rax_out);
+		}
 	}
 
-	secrets = (__force struct snp_secrets_page *)mem;
-
-	addr = secrets->os_area.ap_jump_table_pa;
-	iounmap(mem);
-
-	return addr;
+	native_local_irq_restore(flags);
 }
 
-static u64 __init get_jump_table_addr(void)
+static void pvalidate_pages(struct snp_psc_desc *desc)
 {
-	struct ghcb_state state;
-	unsigned long flags;
-	struct ghcb *ghcb;
-	u64 ret = 0;
-
-	if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
-		return get_snp_jump_table_addr();
-
-	local_irq_save(flags);
+	if (snp_vmpl)
+		svsm_pval_pages(desc);
+	else
+		pval_pages(desc);
+}
 
-	ghcb = __sev_get_ghcb(&state);
+static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)
+{
+	int cur_entry, end_entry, ret = 0;
+	struct snp_psc_desc *data;
+	struct es_em_ctxt ctxt;
 
 	vc_ghcb_invalidate(ghcb);
-	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE);
-	ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE);
-	ghcb_set_sw_exit_info_2(ghcb, 0);
-
-	sev_es_wr_ghcb_msr(__pa(ghcb));
-	VMGEXIT();
-
-	if (ghcb_sw_exit_info_1_is_valid(ghcb) &&
-	    ghcb_sw_exit_info_2_is_valid(ghcb))
-		ret = ghcb->save.sw_exit_info_2;
-
-	__sev_put_ghcb(&state);
 
-	local_irq_restore(flags);
-
-	return ret;
-}
+	/* Copy the input desc into GHCB shared buffer */
+	data = (struct snp_psc_desc *)ghcb->shared_buffer;
+	memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc)));
 
-void __head
-early_set_pages_state(unsigned long vaddr, unsigned long paddr,
-		      unsigned long npages, enum psc_op op)
-{
-	unsigned long paddr_end;
-	u64 val;
-
-	vaddr = vaddr & PAGE_MASK;
+	/*
+	 * As per the GHCB specification, the hypervisor can resume the guest
+	 * before processing all the entries. Check whether all the entries
+	 * are processed. If not, then keep retrying. Note, the hypervisor
+	 * will update the data memory directly to indicate the status, so
+	 * reference the data->hdr everywhere.
+	 *
+	 * The strategy here is to wait for the hypervisor to change the page
+	 * state in the RMP table before guest accesses the memory pages. If the
+	 * page state change was not successful, then later memory access will
+	 * result in a crash.
+	 */
+	cur_entry = data->hdr.cur_entry;
+	end_entry = data->hdr.end_entry;
 
-	paddr = paddr & PAGE_MASK;
-	paddr_end = paddr + (npages << PAGE_SHIFT);
+	while (data->hdr.cur_entry <= data->hdr.end_entry) {
+		ghcb_set_sw_scratch(ghcb, (u64)__pa(data));
 
-	while (paddr < paddr_end) {
-		/* Page validation must be rescinded before changing to shared */
-		if (op == SNP_PAGE_STATE_SHARED)
-			pvalidate_4k_page(vaddr, paddr, false);
+		/* This will advance the shared buffer data points to. */
+		ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0);
 
 		/*
-		 * Use the MSR protocol because this function can be called before
-		 * the GHCB is established.
+		 * Page State Change VMGEXIT can pass error code through
+		 * exit_info_2.
 		 */
-		sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
-		VMGEXIT();
-
-		val = sev_es_rd_ghcb_msr();
-
-		if (GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP)
-			goto e_term;
-
-		if (GHCB_MSR_PSC_RESP_VAL(val))
-			goto e_term;
+		if (WARN(ret || ghcb->save.sw_exit_info_2,
+			 "SNP: PSC failed ret=%d exit_info_2=%llx\n",
+			 ret, ghcb->save.sw_exit_info_2)) {
+			ret = 1;
+			goto out;
+		}
 
-		/* Page validation must be performed after changing to private */
-		if (op == SNP_PAGE_STATE_PRIVATE)
-			pvalidate_4k_page(vaddr, paddr, true);
+		/* Verify that reserved bit is not set */
+		if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) {
+			ret = 1;
+			goto out;
+		}
 
-		vaddr += PAGE_SIZE;
-		paddr += PAGE_SIZE;
+		/*
+		 * Sanity check that entry processing is not going backwards.
+		 * This will happen only if hypervisor is tricking us.
+		 */
+		if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry,
+"SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n",
+			 end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) {
+			ret = 1;
+			goto out;
+		}
 	}
 
-	return;
-
-e_term:
-	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
+out:
+	return ret;
 }
 
-void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr,
-					 unsigned long npages)
-{
-	/*
-	 * This can be invoked in early boot while running identity mapped, so
-	 * use an open coded check for SNP instead of using cc_platform_has().
-	 * This eliminates worries about jump tables or checking boot_cpu_data
-	 * in the cc_platform_has() function.
-	 */
-	if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
-		return;
-
-	 /*
-	  * Ask the hypervisor to mark the memory pages as private in the RMP
-	  * table.
-	  */
-	early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_PRIVATE);
-}
-
-void __head early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr,
-					unsigned long npages)
-{
-	/*
-	 * This can be invoked in early boot while running identity mapped, so
-	 * use an open coded check for SNP instead of using cc_platform_has().
-	 * This eliminates worries about jump tables or checking boot_cpu_data
-	 * in the cc_platform_has() function.
-	 */
-	if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
-		return;
-
-	 /* Ask hypervisor to mark the memory pages shared in the RMP table. */
-	early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_SHARED);
-}
-
-static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr,
-				       unsigned long vaddr_end, int op)
+static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr,
+				       unsigned long vaddr_end, int op)
 {
 	struct ghcb_state state;
 	bool use_large_entry;
@@ -XXX,XX +XXX,XX @@ int __init sev_es_efi_map_ghcbs(pgd_t *pgd)
 	return 0;
 }
 
-/* Writes to the SVSM CAA MSR are ignored */
-static enum es_result __vc_handle_msr_caa(struct pt_regs *regs, bool write)
-{
-	if (write)
-		return ES_OK;
-
-	regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa));
-	regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa));
-
-	return ES_OK;
-}
-
-/*
- * TSC related accesses should not exit to the hypervisor when a guest is
- * executing with Secure TSC enabled, so special handling is required for
- * accesses of MSR_IA32_TSC and MSR_AMD64_GUEST_TSC_FREQ.
- */
-static enum es_result __vc_handle_secure_tsc_msrs(struct pt_regs *regs, bool write)
-{
-	u64 tsc;
-
-	/*
-	 * GUEST_TSC_FREQ should not be intercepted when Secure TSC is enabled.
-	 * Terminate the SNP guest when the interception is enabled.
-	 */
-	if (regs->cx == MSR_AMD64_GUEST_TSC_FREQ)
-		return ES_VMM_ERROR;
-
-	/*
-	 * Writes: Writing to MSR_IA32_TSC can cause subsequent reads of the TSC
-	 *         to return undefined values, so ignore all writes.
-	 *
-	 * Reads: Reads of MSR_IA32_TSC should return the current TSC value, use
-	 *        the value returned by rdtsc_ordered().
-	 */
-	if (write) {
-		WARN_ONCE(1, "TSC MSR writes are verboten!\n");
-		return ES_OK;
-	}
-
-	tsc = rdtsc_ordered();
-	regs->ax = lower_32_bits(tsc);
-	regs->dx = upper_32_bits(tsc);
-
-	return ES_OK;
-}
-
-static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
-{
-	struct pt_regs *regs = ctxt->regs;
-	enum es_result ret;
-	bool write;
-
-	/* Is it a WRMSR? */
-	write = ctxt->insn.opcode.bytes[1] == 0x30;
-
-	switch (regs->cx) {
-	case MSR_SVSM_CAA:
-		return __vc_handle_msr_caa(regs, write);
-	case MSR_IA32_TSC:
-	case MSR_AMD64_GUEST_TSC_FREQ:
-		if (sev_status & MSR_AMD64_SNP_SECURE_TSC)
-			return __vc_handle_secure_tsc_msrs(regs, write);
-		break;
-	default:
-		break;
-	}
-
-	ghcb_set_rcx(ghcb, regs->cx);
-	if (write) {
-		ghcb_set_rax(ghcb, regs->ax);
-		ghcb_set_rdx(ghcb, regs->dx);
-	}
-
-	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, write, 0);
-
-	if ((ret == ES_OK) && !write) {
-		regs->ax = ghcb->save.rax;
-		regs->dx = ghcb->save.rdx;
-	}
-
-	return ret;
-}
-
 static void snp_register_per_cpu_ghcb(void)
 {
 	struct sev_es_runtime_data *data;
@@ -XXX,XX +XXX,XX @@ void __init sev_es_init_vc_handling(void)
 	initial_vc_handler = (unsigned long)kernel_exc_vmm_communication;
 }
 
-static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt)
-{
-	int trapnr = ctxt->fi.vector;
-
-	if (trapnr == X86_TRAP_PF)
-		native_write_cr2(ctxt->fi.cr2);
-
-	ctxt->regs->orig_ax = ctxt->fi.error_code;
-	do_early_exception(ctxt->regs, trapnr);
-}
-
-static long *vc_insn_get_rm(struct es_em_ctxt *ctxt)
-{
-	long *reg_array;
-	int offset;
-
-	reg_array = (long *)ctxt->regs;
-	offset    = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs);
-
-	if (offset < 0)
-		return NULL;
-
-	offset /= sizeof(long);
-
-	return reg_array + offset;
-}
-static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
-				 unsigned int bytes, bool read)
-{
-	u64 exit_code, exit_info_1, exit_info_2;
-	unsigned long ghcb_pa = __pa(ghcb);
-	enum es_result res;
-	phys_addr_t paddr;
-	void __user *ref;
-
-	ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs);
-	if (ref == (void __user *)-1L)
-		return ES_UNSUPPORTED;
-
-	exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE;
-
-	res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr);
-	if (res != ES_OK) {
-		if (res == ES_EXCEPTION && !read)
-			ctxt->fi.error_code |= X86_PF_WRITE;
-
-		return res;
-	}
-
-	exit_info_1 = paddr;
-	/* Can never be greater than 8 */
-	exit_info_2 = bytes;
-
-	ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer));
-
-	return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2);
-}
-
-/*
- * The MOVS instruction has two memory operands, which raises the
- * problem that it is not known whether the access to the source or the
- * destination caused the #VC exception (and hence whether an MMIO read
- * or write operation needs to be emulated).
- *
- * Instead of playing games with walking page-tables and trying to guess
- * whether the source or destination is an MMIO range, split the move
- * into two operations, a read and a write with only one memory operand.
- * This will cause a nested #VC exception on the MMIO address which can
- * then be handled.
- *
- * This implementation has the benefit that it also supports MOVS where
- * source _and_ destination are MMIO regions.
- *
- * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a
- * rare operation. If it turns out to be a performance problem the split
- * operations can be moved to memcpy_fromio() and memcpy_toio().
- */
-static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt,
-					  unsigned int bytes)
-{
-	unsigned long ds_base, es_base;
-	unsigned char *src, *dst;
-	unsigned char buffer[8];
-	enum es_result ret;
-	bool rep;
-	int off;
-
-	ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS);
-	es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
-
-	if (ds_base == -1L || es_base == -1L) {
-		ctxt->fi.vector = X86_TRAP_GP;
-		ctxt->fi.error_code = 0;
-		return ES_EXCEPTION;
-	}
-
-	src = ds_base + (unsigned char *)ctxt->regs->si;
-	dst = es_base + (unsigned char *)ctxt->regs->di;
-
-	ret = vc_read_mem(ctxt, src, buffer, bytes);
-	if (ret != ES_OK)
-		return ret;
-
-	ret = vc_write_mem(ctxt, dst, buffer, bytes);
-	if (ret != ES_OK)
-		return ret;
-
-	if (ctxt->regs->flags & X86_EFLAGS_DF)
-		off = -bytes;
-	else
-		off =  bytes;
-
-	ctxt->regs->si += off;
-	ctxt->regs->di += off;
-
-	rep = insn_has_rep_prefix(&ctxt->insn);
-	if (rep)
-		ctxt->regs->cx -= 1;
-
-	if (!rep || ctxt->regs->cx == 0)
-		return ES_OK;
-	else
-		return ES_RETRY;
-}
-
-static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
-{
-	struct insn *insn = &ctxt->insn;
-	enum insn_mmio_type mmio;
-	unsigned int bytes = 0;
-	enum es_result ret;
-	u8 sign_byte;
-	long *reg_data;
-
-	mmio = insn_decode_mmio(insn, &bytes);
-	if (mmio == INSN_MMIO_DECODE_FAILED)
-		return ES_DECODE_FAILED;
-
-	if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) {
-		reg_data = insn_get_modrm_reg_ptr(insn, ctxt->regs);
-		if (!reg_data)
-			return ES_DECODE_FAILED;
-	}
-
-	if (user_mode(ctxt->regs))
-		return ES_UNSUPPORTED;
-
-	switch (mmio) {
-	case INSN_MMIO_WRITE:
-		memcpy(ghcb->shared_buffer, reg_data, bytes);
-		ret = vc_do_mmio(ghcb, ctxt, bytes, false);
-		break;
-	case INSN_MMIO_WRITE_IMM:
-		memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes);
-		ret = vc_do_mmio(ghcb, ctxt, bytes, false);
-		break;
-	case INSN_MMIO_READ:
-		ret = vc_do_mmio(ghcb, ctxt, bytes, true);
-		if (ret)
-			break;
-
-		/* Zero-extend for 32-bit operation */
-		if (bytes == 4)
-			*reg_data = 0;
-
-		memcpy(reg_data, ghcb->shared_buffer, bytes);
-		break;
-	case INSN_MMIO_READ_ZERO_EXTEND:
-		ret = vc_do_mmio(ghcb, ctxt, bytes, true);
-		if (ret)
-			break;
-
-		/* Zero extend based on operand size */
-		memset(reg_data, 0, insn->opnd_bytes);
-		memcpy(reg_data, ghcb->shared_buffer, bytes);
-		break;
-	case INSN_MMIO_READ_SIGN_EXTEND:
-		ret = vc_do_mmio(ghcb, ctxt, bytes, true);
-		if (ret)
-			break;
-
-		if (bytes == 1) {
-			u8 *val = (u8 *)ghcb->shared_buffer;
-
-			sign_byte = (*val & 0x80) ? 0xff : 0x00;
-		} else {
-			u16 *val = (u16 *)ghcb->shared_buffer;
-
-			sign_byte = (*val & 0x8000) ? 0xff : 0x00;
-		}
-
-		/* Sign extend based on operand size */
-		memset(reg_data, sign_byte, insn->opnd_bytes);
-		memcpy(reg_data, ghcb->shared_buffer, bytes);
-		break;
-	case INSN_MMIO_MOVS:
-		ret = vc_handle_mmio_movs(ctxt, bytes);
-		break;
-	default:
-		ret = ES_UNSUPPORTED;
-		break;
-	}
-
-	return ret;
-}
-
-static enum es_result vc_handle_dr7_write(struct ghcb *ghcb,
-					  struct es_em_ctxt *ctxt)
-{
-	struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
-	long val, *reg = vc_insn_get_rm(ctxt);
-	enum es_result ret;
-
-	if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
-		return ES_VMM_ERROR;
-
-	if (!reg)
-		return ES_DECODE_FAILED;
-
-	val = *reg;
-
-	/* Upper 32 bits must be written as zeroes */
-	if (val >> 32) {
-		ctxt->fi.vector = X86_TRAP_GP;
-		ctxt->fi.error_code = 0;
-		return ES_EXCEPTION;
-	}
-
-	/* Clear out other reserved bits and set bit 10 */
-	val = (val & 0xffff23ffL) | BIT(10);
-
-	/* Early non-zero writes to DR7 are not supported */
-	if (!data && (val & ~DR7_RESET_VALUE))
-		return ES_UNSUPPORTED;
-
-	/* Using a value of 0 for ExitInfo1 means RAX holds the value */
-	ghcb_set_rax(ghcb, val);
-	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0);
-	if (ret != ES_OK)
-		return ret;
-
-	if (data)
-		data->dr7 = val;
-
-	return ES_OK;
-}
-
-static enum es_result vc_handle_dr7_read(struct ghcb *ghcb,
-					 struct es_em_ctxt *ctxt)
-{
-	struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
-	long *reg = vc_insn_get_rm(ctxt);
-
-	if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
-		return ES_VMM_ERROR;
-
-	if (!reg)
-		return ES_DECODE_FAILED;
-
-	if (data)
-		*reg = data->dr7;
-	else
-		*reg = DR7_RESET_VALUE;
-
-	return ES_OK;
-}
-
-static enum es_result vc_handle_wbinvd(struct ghcb *ghcb,
-				       struct es_em_ctxt *ctxt)
-{
-	return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0);
-}
-
-static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
-{
-	enum es_result ret;
-
-	ghcb_set_rcx(ghcb, ctxt->regs->cx);
-
-	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0);
-	if (ret != ES_OK)
-		return ret;
-
-	if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb)))
-		return ES_VMM_ERROR;
-
-	ctxt->regs->ax = ghcb->save.rax;
-	ctxt->regs->dx = ghcb->save.rdx;
-
-	return ES_OK;
-}
-
-static enum es_result vc_handle_monitor(struct ghcb *ghcb,
-					struct es_em_ctxt *ctxt)
-{
-	/*
-	 * Treat it as a NOP and do not leak a physical address to the
-	 * hypervisor.
-	 */
-	return ES_OK;
-}
-
-static enum es_result vc_handle_mwait(struct ghcb *ghcb,
-				      struct es_em_ctxt *ctxt)
-{
-	/* Treat the same as MONITOR/MONITORX */
-	return ES_OK;
-}
-
-static enum es_result vc_handle_vmmcall(struct ghcb *ghcb,
-					struct es_em_ctxt *ctxt)
-{
-	enum es_result ret;
-
-	ghcb_set_rax(ghcb, ctxt->regs->ax);
-	ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0);
-
-	if (x86_platform.hyper.sev_es_hcall_prepare)
-		x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs);
-
-	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0);
-	if (ret != ES_OK)
-		return ret;
-
-	if (!ghcb_rax_is_valid(ghcb))
-		return ES_VMM_ERROR;
-
-	ctxt->regs->ax = ghcb->save.rax;
-
-	/*
-	 * Call sev_es_hcall_finish() after regs->ax is already set.
-	 * This allows the hypervisor handler to overwrite it again if
-	 * necessary.
-	 */
-	if (x86_platform.hyper.sev_es_hcall_finish &&
-	    !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs))
-		return ES_VMM_ERROR;
-
-	return ES_OK;
-}
-
-static enum es_result vc_handle_trap_ac(struct ghcb *ghcb,
-					struct es_em_ctxt *ctxt)
-{
-	/*
-	 * Calling ecx_alignment_check() directly does not work, because it
-	 * enables IRQs and the GHCB is active. Forward the exception and call
-	 * it later from vc_forward_exception().
-	 */
-	ctxt->fi.vector = X86_TRAP_AC;
-	ctxt->fi.error_code = 0;
-	return ES_EXCEPTION;
-}
-
-static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
-					 struct ghcb *ghcb,
-					 unsigned long exit_code)
-{
-	enum es_result result = vc_check_opcode_bytes(ctxt, exit_code);
-
-	if (result != ES_OK)
-		return result;
-
-	switch (exit_code) {
-	case SVM_EXIT_READ_DR7:
-		result = vc_handle_dr7_read(ghcb, ctxt);
-		break;
-	case SVM_EXIT_WRITE_DR7:
-		result = vc_handle_dr7_write(ghcb, ctxt);
-		break;
-	case SVM_EXIT_EXCP_BASE + X86_TRAP_AC:
-		result = vc_handle_trap_ac(ghcb, ctxt);
-		break;
-	case SVM_EXIT_RDTSC:
-	case SVM_EXIT_RDTSCP:
-		result = vc_handle_rdtsc(ghcb, ctxt, exit_code);
-		break;
-	case SVM_EXIT_RDPMC:
-		result = vc_handle_rdpmc(ghcb, ctxt);
-		break;
-	case SVM_EXIT_INVD:
-		pr_err_ratelimited("#VC exception for INVD??? Seriously???\n");
-		result = ES_UNSUPPORTED;
-		break;
-	case SVM_EXIT_CPUID:
-		result = vc_handle_cpuid(ghcb, ctxt);
-		break;
-	case SVM_EXIT_IOIO:
-		result = vc_handle_ioio(ghcb, ctxt);
-		break;
-	case SVM_EXIT_MSR:
-		result = vc_handle_msr(ghcb, ctxt);
-		break;
-	case SVM_EXIT_VMMCALL:
-		result = vc_handle_vmmcall(ghcb, ctxt);
-		break;
-	case SVM_EXIT_WBINVD:
-		result = vc_handle_wbinvd(ghcb, ctxt);
-		break;
-	case SVM_EXIT_MONITOR:
-		result = vc_handle_monitor(ghcb, ctxt);
-		break;
-	case SVM_EXIT_MWAIT:
-		result = vc_handle_mwait(ghcb, ctxt);
-		break;
-	case SVM_EXIT_NPF:
-		result = vc_handle_mmio(ghcb, ctxt);
-		break;
-	default:
-		/*
-		 * Unexpected #VC exception
-		 */
-		result = ES_UNSUPPORTED;
-	}
-
-	return result;
-}
-
-static __always_inline bool is_vc2_stack(unsigned long sp)
-{
-	return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2));
-}
-
-static __always_inline bool vc_from_invalid_context(struct pt_regs *regs)
-{
-	unsigned long sp, prev_sp;
-
-	sp      = (unsigned long)regs;
-	prev_sp = regs->sp;
-
-	/*
-	 * If the code was already executing on the VC2 stack when the #VC
-	 * happened, let it proceed to the normal handling routine. This way the
-	 * code executing on the VC2 stack can cause #VC exceptions to get handled.
-	 */
-	return is_vc2_stack(sp) && !is_vc2_stack(prev_sp);
-}
-
-static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code)
-{
-	struct ghcb_state state;
-	struct es_em_ctxt ctxt;
-	enum es_result result;
-	struct ghcb *ghcb;
-	bool ret = true;
-
-	ghcb = __sev_get_ghcb(&state);
-
-	vc_ghcb_invalidate(ghcb);
-	result = vc_init_em_ctxt(&ctxt, regs, error_code);
-
-	if (result == ES_OK)
-		result = vc_handle_exitcode(&ctxt, ghcb, error_code);
-
-	__sev_put_ghcb(&state);
-
-	/* Done - now check the result */
-	switch (result) {
-	case ES_OK:
-		vc_finish_insn(&ctxt);
-		break;
-	case ES_UNSUPPORTED:
-		pr_err_ratelimited("Unsupported exit-code 0x%02lx in #VC exception (IP: 0x%lx)\n",
-				   error_code, regs->ip);
-		ret = false;
-		break;
-	case ES_VMM_ERROR:
-		pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
-				   error_code, regs->ip);
-		ret = false;
-		break;
-	case ES_DECODE_FAILED:
-		pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
-				   error_code, regs->ip);
-		ret = false;
-		break;
-	case ES_EXCEPTION:
-		vc_forward_exception(&ctxt);
-		break;
-	case ES_RETRY:
-		/* Nothing to do */
-		break;
-	default:
-		pr_emerg("Unknown result in %s():%d\n", __func__, result);
-		/*
-		 * Emulating the instruction which caused the #VC exception
-		 * failed - can't continue so print debug information
-		 */
-		BUG();
-	}
-
-	return ret;
-}
-
-static __always_inline bool vc_is_db(unsigned long error_code)
-{
-	return error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB;
-}
-
-/*
- * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode
- * and will panic when an error happens.
- */
-DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication)
-{
-	irqentry_state_t irq_state;
-
-	/*
-	 * With the current implementation it is always possible to switch to a
-	 * safe stack because #VC exceptions only happen at known places, like
-	 * intercepted instructions or accesses to MMIO areas/IO ports. They can
-	 * also happen with code instrumentation when the hypervisor intercepts
-	 * #DB, but the critical paths are forbidden to be instrumented, so #DB
-	 * exceptions currently also only happen in safe places.
-	 *
-	 * But keep this here in case the noinstr annotations are violated due
-	 * to bug elsewhere.
-	 */
-	if (unlikely(vc_from_invalid_context(regs))) {
-		instrumentation_begin();
-		panic("Can't handle #VC exception from unsupported context\n");
-		instrumentation_end();
-	}
-
-	/*
-	 * Handle #DB before calling into !noinstr code to avoid recursive #DB.
-	 */
-	if (vc_is_db(error_code)) {
-		exc_debug(regs);
-		return;
-	}
-
-	irq_state = irqentry_nmi_enter(regs);
-
-	instrumentation_begin();
-
-	if (!vc_raw_handle_exception(regs, error_code)) {
-		/* Show some debug info */
-		show_regs(regs);
-
-		/* Ask hypervisor to sev_es_terminate */
-		sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
-
-		/* If that fails and we get here - just panic */
-		panic("Returned from Terminate-Request to Hypervisor\n");
-	}
-
-	instrumentation_end();
-	irqentry_nmi_exit(regs, irq_state);
-}
-
-/*
- * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode
- * and will kill the current task with SIGBUS when an error happens.
- */
-DEFINE_IDTENTRY_VC_USER(exc_vmm_communication)
-{
-	/*
-	 * Handle #DB before calling into !noinstr code to avoid recursive #DB.
-	 */
-	if (vc_is_db(error_code)) {
-		noist_exc_debug(regs);
-		return;
-	}
-
-	irqentry_enter_from_user_mode(regs);
-	instrumentation_begin();
-
-	if (!vc_raw_handle_exception(regs, error_code)) {
-		/*
-		 * Do not kill the machine if user-space triggered the
-		 * exception. Send SIGBUS instead and let user-space deal with
-		 * it.
-		 */
-		force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0);
-	}
-
-	instrumentation_end();
-	irqentry_exit_to_user_mode(regs);
-}
-
-bool __init handle_vc_boot_ghcb(struct pt_regs *regs)
-{
-	unsigned long exit_code = regs->orig_ax;
-	struct es_em_ctxt ctxt;
-	enum es_result result;
-
-	vc_ghcb_invalidate(boot_ghcb);
-
-	result = vc_init_em_ctxt(&ctxt, regs, exit_code);
-	if (result == ES_OK)
-		result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code);
-
-	/* Done - now check the result */
-	switch (result) {
-	case ES_OK:
-		vc_finish_insn(&ctxt);
-		break;
-	case ES_UNSUPPORTED:
-		early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n",
-				exit_code, regs->ip);
-		goto fail;
-	case ES_VMM_ERROR:
-		early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
-				exit_code, regs->ip);
-		goto fail;
-	case ES_DECODE_FAILED:
-		early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
-				exit_code, regs->ip);
-		goto fail;
-	case ES_EXCEPTION:
-		vc_early_forward_exception(&ctxt);
-		break;
-	case ES_RETRY:
-		/* Nothing to do */
-		break;
-	default:
-		BUG();
-	}
-
-	return true;
-
-fail:
-	show_regs(regs);
-
-	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
-}
-
-/*
- * Initial set up of SNP relies on information provided by the
- * Confidential Computing blob, which can be passed to the kernel
- * in the following ways, depending on how it is booted:
- *
- * - when booted via the boot/decompress kernel:
- *   - via boot_params
- *
- * - when booted directly by firmware/bootloader (e.g. CONFIG_PVH):
- *   - via a setup_data entry, as defined by the Linux Boot Protocol
- *
- * Scan for the blob in that order.
- */
-static __head struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
-{
-	struct cc_blob_sev_info *cc_info;
-
-	/* Boot kernel would have passed the CC blob via boot_params. */
-	if (bp->cc_blob_address) {
-		cc_info = (struct cc_blob_sev_info *)(unsigned long)bp->cc_blob_address;
-		goto found_cc_info;
-	}
-
-	/*
-	 * If kernel was booted directly, without the use of the
-	 * boot/decompression kernel, the CC blob may have been passed via
-	 * setup_data instead.
-	 */
-	cc_info = find_cc_blob_setup_data(bp);
-	if (!cc_info)
-		return NULL;
-
-found_cc_info:
-	if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
-		snp_abort();
-
-	return cc_info;
-}
-
-static __head void svsm_setup(struct cc_blob_sev_info *cc_info)
-{
-	struct svsm_call call = {};
-	int ret;
-	u64 pa;
-
-	/*
-	 * Record the SVSM Calling Area address (CAA) if the guest is not
-	 * running at VMPL0. The CA will be used to communicate with the
-	 * SVSM to perform the SVSM services.
-	 */
-	if (!svsm_setup_ca(cc_info))
-		return;
-
-	/*
-	 * It is very early in the boot and the kernel is running identity
-	 * mapped but without having adjusted the pagetables to where the
-	 * kernel was loaded (physbase), so the get the CA address using
-	 * RIP-relative addressing.
-	 */
-	pa = (u64)rip_rel_ptr(&boot_svsm_ca_page);
-
-	/*
-	 * Switch over to the boot SVSM CA while the current CA is still
-	 * addressable. There is no GHCB at this point so use the MSR protocol.
-	 *
-	 * SVSM_CORE_REMAP_CA call:
-	 *   RAX = 0 (Protocol=0, CallID=0)
-	 *   RCX = New CA GPA
-	 */
-	call.caa = svsm_get_caa();
-	call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA);
-	call.rcx = pa;
-	ret = svsm_perform_call_protocol(&call);
-	if (ret)
-		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CA_REMAP_FAIL);
-
-	RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)pa;
-	RIP_REL_REF(boot_svsm_caa_pa) = pa;
-}
-
-bool __head snp_init(struct boot_params *bp)
-{
-	struct cc_blob_sev_info *cc_info;
-
-	if (!bp)
-		return false;
-
-	cc_info = find_cc_blob(bp);
-	if (!cc_info)
-		return false;
-
-	if (cc_info->secrets_phys && cc_info->secrets_len == PAGE_SIZE)
-		secrets_pa = cc_info->secrets_phys;
-	else
-		return false;
-
-	setup_cpuid_table(cc_info);
-
-	svsm_setup(cc_info);
-
-	/*
-	 * The CC blob will be used later to access the secrets page. Cache
-	 * it here like the boot kernel does.
-	 */
-	bp->cc_blob_address = (u32)(unsigned long)cc_info;
-
-	return true;
-}
-
-void __head __noreturn snp_abort(void)
-{
-	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
-}
-
 /*
  * SEV-SNP guests should only execute dmi_setup() if EFI_CONFIG_TABLES are
  * enabled, as the alternative (fallback) logic for DMI probing in the legacy
diff --git a/arch/x86/coco/sev/shared.c b/arch/x86/coco/sev/shared.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/coco/sev/shared.c
+++ b/arch/x86/coco/sev/shared.c
@@ -XXX,XX +XXX,XX @@
 
 /*
  * SVSM related information:
- *   When running under an SVSM, the VMPL that Linux is executing at must be
- *   non-zero. The VMPL is therefore used to indicate the presence of an SVSM.
- *
  *   During boot, the page tables are set up as identity mapped and later
  *   changed to use kernel virtual addresses. Maintain separate virtual and
  *   physical addresses for the CAA to allow SVSM functions to be used during
  *   early boot, both with identity mapped virtual addresses and proper kernel
  *   virtual addresses.
  */
-u8 snp_vmpl __ro_after_init;
-EXPORT_SYMBOL_GPL(snp_vmpl);
 struct svsm_ca *boot_svsm_caa __ro_after_init;
 u64 boot_svsm_caa_pa __ro_after_init;
 
@@ -XXX,XX +XXX,XX @@ static void __head setup_cpuid_table(const struct cc_blob_sev_info *cc_info)
 	}
 }
 
-static inline void __pval_terminate(u64 pfn, bool action, unsigned int page_size,
-				    int ret, u64 svsm_ret)
-{
-	WARN(1, "PVALIDATE failure: pfn: 0x%llx, action: %u, size: %u, ret: %d, svsm_ret: 0x%llx\n",
-	     pfn, action, page_size, ret, svsm_ret);
-
-	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
-}
-
-static void svsm_pval_terminate(struct svsm_pvalidate_call *pc, int ret, u64 svsm_ret)
-{
-	unsigned int page_size;
-	bool action;
-	u64 pfn;
-
-	pfn = pc->entry[pc->cur_index].pfn;
-	action = pc->entry[pc->cur_index].action;
-	page_size = pc->entry[pc->cur_index].page_size;
-
-	__pval_terminate(pfn, action, page_size, ret, svsm_ret);
-}
-
 static void __head svsm_pval_4k_page(unsigned long paddr, bool validate)
 {
 	struct svsm_pvalidate_call *pc;
@@ -XXX,XX +XXX,XX @@ static void __head pvalidate_4k_page(unsigned long vaddr, unsigned long paddr,
 	}
 }
 
-static void pval_pages(struct snp_psc_desc *desc)
-{
-	struct psc_entry *e;
-	unsigned long vaddr;
-	unsigned int size;
-	unsigned int i;
-	bool validate;
-	u64 pfn;
-	int rc;
-
-	for (i = 0; i <= desc->hdr.end_entry; i++) {
-		e = &desc->entries[i];
-
-		pfn = e->gfn;
-		vaddr = (unsigned long)pfn_to_kaddr(pfn);
-		size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
-		validate = e->operation == SNP_PAGE_STATE_PRIVATE;
-
-		rc = pvalidate(vaddr, size, validate);
-		if (!rc)
-			continue;
-
-		if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) {
-			unsigned long vaddr_end = vaddr + PMD_SIZE;
-
-			for (; vaddr < vaddr_end; vaddr += PAGE_SIZE, pfn++) {
-				rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
-				if (rc)
-					__pval_terminate(pfn, validate, RMP_PG_SIZE_4K, rc, 0);
-			}
-		} else {
-			__pval_terminate(pfn, validate, size, rc, 0);
-		}
-	}
-}
-
-static u64 svsm_build_ca_from_pfn_range(u64 pfn, u64 pfn_end, bool action,
-					struct svsm_pvalidate_call *pc)
-{
-	struct svsm_pvalidate_entry *pe;
-
-	/* Nothing in the CA yet */
-	pc->num_entries = 0;
-	pc->cur_index   = 0;
-
-	pe = &pc->entry[0];
-
-	while (pfn < pfn_end) {
-		pe->page_size = RMP_PG_SIZE_4K;
-		pe->action    = action;
-		pe->ignore_cf = 0;
-		pe->pfn       = pfn;
-
-		pe++;
-		pfn++;
-
-		pc->num_entries++;
-		if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT)
-			break;
-	}
-
-	return pfn;
-}
-
-static int svsm_build_ca_from_psc_desc(struct snp_psc_desc *desc, unsigned int desc_entry,
-				       struct svsm_pvalidate_call *pc)
-{
-	struct svsm_pvalidate_entry *pe;
-	struct psc_entry *e;
-
-	/* Nothing in the CA yet */
-	pc->num_entries = 0;
-	pc->cur_index   = 0;
-
-	pe = &pc->entry[0];
-	e  = &desc->entries[desc_entry];
-
-	while (desc_entry <= desc->hdr.end_entry) {
-		pe->page_size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
-		pe->action    = e->operation == SNP_PAGE_STATE_PRIVATE;
-		pe->ignore_cf = 0;
-		pe->pfn       = e->gfn;
-
-		pe++;
-		e++;
-
-		desc_entry++;
-		pc->num_entries++;
-		if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT)
-			break;
-	}
-
-	return desc_entry;
-}
-
-static void svsm_pval_pages(struct snp_psc_desc *desc)
-{
-	struct svsm_pvalidate_entry pv_4k[VMGEXIT_PSC_MAX_ENTRY];
-	unsigned int i, pv_4k_count = 0;
-	struct svsm_pvalidate_call *pc;
-	struct svsm_call call = {};
-	unsigned long flags;
-	bool action;
-	u64 pc_pa;
-	int ret;
-
-	/*
-	 * This can be called very early in the boot, use native functions in
-	 * order to avoid paravirt issues.
-	 */
-	flags = native_local_irq_save();
-
-	/*
-	 * The SVSM calling area (CA) can support processing 510 entries at a
-	 * time. Loop through the Page State Change descriptor until the CA is
-	 * full or the last entry in the descriptor is reached, at which time
-	 * the SVSM is invoked. This repeats until all entries in the descriptor
-	 * are processed.
-	 */
-	call.caa = svsm_get_caa();
-
-	pc = (struct svsm_pvalidate_call *)call.caa->svsm_buffer;
-	pc_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer);
-
-	/* Protocol 0, Call ID 1 */
-	call.rax = SVSM_CORE_CALL(SVSM_CORE_PVALIDATE);
-	call.rcx = pc_pa;
-
-	for (i = 0; i <= desc->hdr.end_entry;) {
-		i = svsm_build_ca_from_psc_desc(desc, i, pc);
-
-		do {
-			ret = svsm_perform_call_protocol(&call);
-			if (!ret)
-				continue;
-
-			/*
-			 * Check if the entry failed because of an RMP mismatch (a
-			 * PVALIDATE at 2M was requested, but the page is mapped in
-			 * the RMP as 4K).
-			 */
-
-			if (call.rax_out == SVSM_PVALIDATE_FAIL_SIZEMISMATCH &&
-			    pc->entry[pc->cur_index].page_size == RMP_PG_SIZE_2M) {
-				/* Save this entry for post-processing at 4K */
-				pv_4k[pv_4k_count++] = pc->entry[pc->cur_index];
-
-				/* Skip to the next one unless at the end of the list */
-				pc->cur_index++;
-				if (pc->cur_index < pc->num_entries)
-					ret = -EAGAIN;
-				else
-					ret = 0;
-			}
-		} while (ret == -EAGAIN);
-
-		if (ret)
-			svsm_pval_terminate(pc, ret, call.rax_out);
-	}
-
-	/* Process any entries that failed to be validated at 2M and validate them at 4K */
-	for (i = 0; i < pv_4k_count; i++) {
-		u64 pfn, pfn_end;
-
-		action  = pv_4k[i].action;
-		pfn     = pv_4k[i].pfn;
-		pfn_end = pfn + 512;
-
-		while (pfn < pfn_end) {
-			pfn = svsm_build_ca_from_pfn_range(pfn, pfn_end, action, pc);
-
-			ret = svsm_perform_call_protocol(&call);
-			if (ret)
-				svsm_pval_terminate(pc, ret, call.rax_out);
-		}
-	}
-
-	native_local_irq_restore(flags);
-}
-
-static void pvalidate_pages(struct snp_psc_desc *desc)
-{
-	if (snp_vmpl)
-		svsm_pval_pages(desc);
-	else
-		pval_pages(desc);
-}
-
-static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)
-{
-	int cur_entry, end_entry, ret = 0;
-	struct snp_psc_desc *data;
-	struct es_em_ctxt ctxt;
-
-	vc_ghcb_invalidate(ghcb);
-
-	/* Copy the input desc into GHCB shared buffer */
-	data = (struct snp_psc_desc *)ghcb->shared_buffer;
-	memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc)));
-
-	/*
-	 * As per the GHCB specification, the hypervisor can resume the guest
-	 * before processing all the entries. Check whether all the entries
-	 * are processed. If not, then keep retrying. Note, the hypervisor
-	 * will update the data memory directly to indicate the status, so
-	 * reference the data->hdr everywhere.
-	 *
-	 * The strategy here is to wait for the hypervisor to change the page
-	 * state in the RMP table before guest accesses the memory pages. If the
-	 * page state change was not successful, then later memory access will
-	 * result in a crash.
-	 */
-	cur_entry = data->hdr.cur_entry;
-	end_entry = data->hdr.end_entry;
-
-	while (data->hdr.cur_entry <= data->hdr.end_entry) {
-		ghcb_set_sw_scratch(ghcb, (u64)__pa(data));
-
-		/* This will advance the shared buffer data points to. */
-		ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0);
-
-		/*
-		 * Page State Change VMGEXIT can pass error code through
-		 * exit_info_2.
-		 */
-		if (WARN(ret || ghcb->save.sw_exit_info_2,
-			 "SNP: PSC failed ret=%d exit_info_2=%llx\n",
-			 ret, ghcb->save.sw_exit_info_2)) {
-			ret = 1;
-			goto out;
-		}
-
-		/* Verify that reserved bit is not set */
-		if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) {
-			ret = 1;
-			goto out;
-		}
-
-		/*
-		 * Sanity check that entry processing is not going backwards.
-		 * This will happen only if hypervisor is tricking us.
-		 */
-		if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry,
-"SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n",
-			 end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) {
-			ret = 1;
-			goto out;
-		}
-	}
-
-out:
-	return ret;
-}
-
 static enum es_result vc_check_opcode_bytes(struct es_em_ctxt *ctxt,
 					    unsigned long exit_code)
 {
diff --git a/arch/x86/coco/sev/startup.c b/arch/x86/coco/sev/startup.c
new file mode 100644
index XXXXXXX..XXXXXXX
--- /dev/null
+++ b/arch/x86/coco/sev/startup.c
@@ -XXX,XX +XXX,XX @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AMD Memory Encryption Support
+ *
+ * Copyright (C) 2019 SUSE
+ *
+ * Author: Joerg Roedel <jroedel@suse.de>
+ */
+
+#define pr_fmt(fmt)	"SEV: " fmt
+
+#include <linux/sched/debug.h>	/* For show_regs() */
+#include <linux/percpu-defs.h>
+#include <linux/cc_platform.h>
+#include <linux/printk.h>
+#include <linux/mm_types.h>
+#include <linux/set_memory.h>
+#include <linux/memblock.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/cpumask.h>
+#include <linux/efi.h>
+#include <linux/io.h>
+#include <linux/psp-sev.h>
+#include <uapi/linux/sev-guest.h>
+
+#include <asm/init.h>
+#include <asm/cpu_entry_area.h>
+#include <asm/stacktrace.h>
+#include <asm/sev.h>
+#include <asm/sev-internal.h>
+#include <asm/insn-eval.h>
+#include <asm/fpu/xcr.h>
+#include <asm/processor.h>
+#include <asm/realmode.h>
+#include <asm/setup.h>
+#include <asm/traps.h>
+#include <asm/svm.h>
+#include <asm/smp.h>
+#include <asm/cpu.h>
+#include <asm/apic.h>
+#include <asm/cpuid.h>
+#include <asm/cmdline.h>
+
+/* For early boot hypervisor communication in SEV-ES enabled guests */
+struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
+
+/*
+ * Needs to be in the .data section because we need it NULL before bss is
+ * cleared
+ */
+struct ghcb *boot_ghcb __section(".data");
+
+/* Bitmap of SEV features supported by the hypervisor */
+u64 sev_hv_features __ro_after_init;
+
+/* Secrets page physical address from the CC blob */
+static u64 secrets_pa __ro_after_init;
+
+/* For early boot SVSM communication */
+struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE);
+
+DEFINE_PER_CPU(struct svsm_ca *, svsm_caa);
+DEFINE_PER_CPU(u64, svsm_caa_pa);
+
+/*
+ * Nothing shall interrupt this code path while holding the per-CPU
+ * GHCB. The backup GHCB is only for NMIs interrupting this path.
+ *
+ * Callers must disable local interrupts around it.
+ */
+noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
+{
+	struct sev_es_runtime_data *data;
+	struct ghcb *ghcb;
+
+	WARN_ON(!irqs_disabled());
+
+	data = this_cpu_read(runtime_data);
+	ghcb = &data->ghcb_page;
+
+	if (unlikely(data->ghcb_active)) {
+		/* GHCB is already in use - save its contents */
+
+		if (unlikely(data->backup_ghcb_active)) {
+			/*
+			 * Backup-GHCB is also already in use. There is no way
+			 * to continue here so just kill the machine. To make
+			 * panic() work, mark GHCBs inactive so that messages
+			 * can be printed out.
+			 */
+			data->ghcb_active        = false;
+			data->backup_ghcb_active = false;
+
+			instrumentation_begin();
+			panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use");
+			instrumentation_end();
+		}
+
+		/* Mark backup_ghcb active before writing to it */
+		data->backup_ghcb_active = true;
+
+		state->ghcb = &data->backup_ghcb;
+
+		/* Backup GHCB content */
+		*state->ghcb = *ghcb;
+	} else {
+		state->ghcb = NULL;
+		data->ghcb_active = true;
+	}
+
+	return ghcb;
+}
+
+static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt,
+				unsigned char *buffer)
+{
+	return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
+}
+
+static enum es_result __vc_decode_user_insn(struct es_em_ctxt *ctxt)
+{
+	char buffer[MAX_INSN_SIZE];
+	int insn_bytes;
+
+	insn_bytes = insn_fetch_from_user_inatomic(ctxt->regs, buffer);
+	if (insn_bytes == 0) {
+		/* Nothing could be copied */
+		ctxt->fi.vector     = X86_TRAP_PF;
+		ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER;
+		ctxt->fi.cr2        = ctxt->regs->ip;
+		return ES_EXCEPTION;
+	} else if (insn_bytes == -EINVAL) {
+		/* Effective RIP could not be calculated */
+		ctxt->fi.vector     = X86_TRAP_GP;
+		ctxt->fi.error_code = 0;
+		ctxt->fi.cr2        = 0;
+		return ES_EXCEPTION;
+	}
+
+	if (!insn_decode_from_regs(&ctxt->insn, ctxt->regs, buffer, insn_bytes))
+		return ES_DECODE_FAILED;
+
+	if (ctxt->insn.immediate.got)
+		return ES_OK;
+	else
+		return ES_DECODE_FAILED;
+}
+
+static enum es_result __vc_decode_kern_insn(struct es_em_ctxt *ctxt)
+{
+	char buffer[MAX_INSN_SIZE];
+	int res, ret;
+
+	res = vc_fetch_insn_kernel(ctxt, buffer);
+	if (res) {
+		ctxt->fi.vector     = X86_TRAP_PF;
+		ctxt->fi.error_code = X86_PF_INSTR;
+		ctxt->fi.cr2        = ctxt->regs->ip;
+		return ES_EXCEPTION;
+	}
+
+	ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
+	if (ret < 0)
+		return ES_DECODE_FAILED;
+	else
+		return ES_OK;
+}
+
+static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
+{
+	if (user_mode(ctxt->regs))
+		return __vc_decode_user_insn(ctxt);
+	else
+		return __vc_decode_kern_insn(ctxt);
+}
+
+static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
+				   char *dst, char *buf, size_t size)
+{
+	unsigned long error_code = X86_PF_PROT | X86_PF_WRITE;
+
+	/*
+	 * This function uses __put_user() independent of whether kernel or user
+	 * memory is accessed. This works fine because __put_user() does no
+	 * sanity checks of the pointer being accessed. All that it does is
+	 * to report when the access failed.
+	 *
+	 * Also, this function runs in atomic context, so __put_user() is not
+	 * allowed to sleep. The page-fault handler detects that it is running
+	 * in atomic context and will not try to take mmap_sem and handle the
+	 * fault, so additional pagefault_enable()/disable() calls are not
+	 * needed.
+	 *
+	 * The access can't be done via copy_to_user() here because
+	 * vc_write_mem() must not use string instructions to access unsafe
+	 * memory. The reason is that MOVS is emulated by the #VC handler by
+	 * splitting the move up into a read and a write and taking a nested #VC
+	 * exception on whatever of them is the MMIO access. Using string
+	 * instructions here would cause infinite nesting.
+	 */
+	switch (size) {
+	case 1: {
+		u8 d1;
+		u8 __user *target = (u8 __user *)dst;
+
+		memcpy(&d1, buf, 1);
+		if (__put_user(d1, target))
+			goto fault;
+		break;
+	}
+	case 2: {
+		u16 d2;
+		u16 __user *target = (u16 __user *)dst;
+
+		memcpy(&d2, buf, 2);
+		if (__put_user(d2, target))
+			goto fault;
+		break;
+	}
+	case 4: {
+		u32 d4;
+		u32 __user *target = (u32 __user *)dst;
+
+		memcpy(&d4, buf, 4);
+		if (__put_user(d4, target))
+			goto fault;
+		break;
+	}
+	case 8: {
+		u64 d8;
+		u64 __user *target = (u64 __user *)dst;
+
+		memcpy(&d8, buf, 8);
+		if (__put_user(d8, target))
+			goto fault;
+		break;
+	}
+	default:
+		WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
+		return ES_UNSUPPORTED;
+	}
+
+	return ES_OK;
+
+fault:
+	if (user_mode(ctxt->regs))
+		error_code |= X86_PF_USER;
+
+	ctxt->fi.vector = X86_TRAP_PF;
+	ctxt->fi.error_code = error_code;
+	ctxt->fi.cr2 = (unsigned long)dst;
+
+	return ES_EXCEPTION;
+}
+
+static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
+				  char *src, char *buf, size_t size)
+{
+	unsigned long error_code = X86_PF_PROT;
+
+	/*
+	 * This function uses __get_user() independent of whether kernel or user
+	 * memory is accessed. This works fine because __get_user() does no
+	 * sanity checks of the pointer being accessed. All that it does is
+	 * to report when the access failed.
+	 *
+	 * Also, this function runs in atomic context, so __get_user() is not
+	 * allowed to sleep. The page-fault handler detects that it is running
+	 * in atomic context and will not try to take mmap_sem and handle the
+	 * fault, so additional pagefault_enable()/disable() calls are not
+	 * needed.
+	 *
+	 * The access can't be done via copy_from_user() here because
+	 * vc_read_mem() must not use string instructions to access unsafe
+	 * memory. The reason is that MOVS is emulated by the #VC handler by
+	 * splitting the move up into a read and a write and taking a nested #VC
+	 * exception on whatever of them is the MMIO access. Using string
+	 * instructions here would cause infinite nesting.
+	 */
+	switch (size) {
+	case 1: {
+		u8 d1;
+		u8 __user *s = (u8 __user *)src;
+
+		if (__get_user(d1, s))
+			goto fault;
+		memcpy(buf, &d1, 1);
+		break;
+	}
+	case 2: {
+		u16 d2;
+		u16 __user *s = (u16 __user *)src;
+
+		if (__get_user(d2, s))
+			goto fault;
+		memcpy(buf, &d2, 2);
+		break;
+	}
+	case 4: {
+		u32 d4;
+		u32 __user *s = (u32 __user *)src;
+
+		if (__get_user(d4, s))
+			goto fault;
+		memcpy(buf, &d4, 4);
+		break;
+	}
+	case 8: {
+		u64 d8;
+		u64 __user *s = (u64 __user *)src;
+		if (__get_user(d8, s))
+			goto fault;
+		memcpy(buf, &d8, 8);
+		break;
+	}
+	default:
+		WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
+		return ES_UNSUPPORTED;
+	}
+
+	return ES_OK;
+
+fault:
+	if (user_mode(ctxt->regs))
+		error_code |= X86_PF_USER;
+
+	ctxt->fi.vector = X86_TRAP_PF;
+	ctxt->fi.error_code = error_code;
+	ctxt->fi.cr2 = (unsigned long)src;
+
+	return ES_EXCEPTION;
+}
+
+static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
+					   unsigned long vaddr, phys_addr_t *paddr)
+{
+	unsigned long va = (unsigned long)vaddr;
+	unsigned int level;
+	phys_addr_t pa;
+	pgd_t *pgd;
+	pte_t *pte;
+
+	pgd = __va(read_cr3_pa());
+	pgd = &pgd[pgd_index(va)];
+	pte = lookup_address_in_pgd(pgd, va, &level);
+	if (!pte) {
+		ctxt->fi.vector     = X86_TRAP_PF;
+		ctxt->fi.cr2        = vaddr;
+		ctxt->fi.error_code = 0;
+
+		if (user_mode(ctxt->regs))
+			ctxt->fi.error_code |= X86_PF_USER;
+
+		return ES_EXCEPTION;
+	}
+
+	if (WARN_ON_ONCE(pte_val(*pte) & _PAGE_ENC))
+		/* Emulated MMIO to/from encrypted memory not supported */
+		return ES_UNSUPPORTED;
+
+	pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
+	pa |= va & ~page_level_mask(level);
+
+	*paddr = pa;
+
+	return ES_OK;
+}
+
+static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
+{
+	BUG_ON(size > 4);
+
+	if (user_mode(ctxt->regs)) {
+		struct thread_struct *t = &current->thread;
+		struct io_bitmap *iobm = t->io_bitmap;
+		size_t idx;
+
+		if (!iobm)
+			goto fault;
+
+		for (idx = port; idx < port + size; ++idx) {
+			if (test_bit(idx, iobm->bitmap))
+				goto fault;
+		}
+	}
+
+	return ES_OK;
+
+fault:
+	ctxt->fi.vector = X86_TRAP_GP;
+	ctxt->fi.error_code = 0;
+
+	return ES_EXCEPTION;
+}
+
+static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt)
+{
+	long error_code = ctxt->fi.error_code;
+	int trapnr = ctxt->fi.vector;
+
+	ctxt->regs->orig_ax = ctxt->fi.error_code;
+
+	switch (trapnr) {
+	case X86_TRAP_GP:
+		exc_general_protection(ctxt->regs, error_code);
+		break;
+	case X86_TRAP_UD:
+		exc_invalid_op(ctxt->regs);
+		break;
+	case X86_TRAP_PF:
+		write_cr2(ctxt->fi.cr2);
+		exc_page_fault(ctxt->regs, error_code);
+		break;
+	case X86_TRAP_AC:
+		exc_alignment_check(ctxt->regs, error_code);
+		break;
+	default:
+		pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n");
+		BUG();
+	}
+}
+
+/* Include code shared with pre-decompression boot stage */
+#include "shared.c"
+
+noinstr void __sev_put_ghcb(struct ghcb_state *state)
+{
+	struct sev_es_runtime_data *data;
+	struct ghcb *ghcb;
+
+	WARN_ON(!irqs_disabled());
+
+	data = this_cpu_read(runtime_data);
+	ghcb = &data->ghcb_page;
+
+	if (state->ghcb) {
+		/* Restore GHCB from Backup */
+		*ghcb = *state->ghcb;
+		data->backup_ghcb_active = false;
+		state->ghcb = NULL;
+	} else {
+		/*
+		 * Invalidate the GHCB so a VMGEXIT instruction issued
+		 * from userspace won't appear to be valid.
+		 */
+		vc_ghcb_invalidate(ghcb);
+		data->ghcb_active = false;
+	}
+}
+
+int svsm_perform_call_protocol(struct svsm_call *call)
+{
+	struct ghcb_state state;
+	unsigned long flags;
+	struct ghcb *ghcb;
+	int ret;
+
+	/*
+	 * This can be called very early in the boot, use native functions in
+	 * order to avoid paravirt issues.
+	 */
+	flags = native_local_irq_save();
+
+	/*
+	 * Use rip-relative references when called early in the boot. If
+	 * ghcbs_initialized is set, then it is late in the boot and no need
+	 * to worry about rip-relative references in called functions.
+	 */
+	if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
+		ghcb = __sev_get_ghcb(&state);
+	else if (RIP_REL_REF(boot_ghcb))
+		ghcb = RIP_REL_REF(boot_ghcb);
+	else
+		ghcb = NULL;
+
+	do {
+		ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call)
+			   : svsm_perform_msr_protocol(call);
+	} while (ret == -EAGAIN);
+
+	if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
+		__sev_put_ghcb(&state);
+
+	native_local_irq_restore(flags);
+
+	return ret;
+}
+
+void __head
+early_set_pages_state(unsigned long vaddr, unsigned long paddr,
+		      unsigned long npages, enum psc_op op)
+{
+	unsigned long paddr_end;
+	u64 val;
+
+	vaddr = vaddr & PAGE_MASK;
+
+	paddr = paddr & PAGE_MASK;
+	paddr_end = paddr + (npages << PAGE_SHIFT);
+
+	while (paddr < paddr_end) {
+		/* Page validation must be rescinded before changing to shared */
+		if (op == SNP_PAGE_STATE_SHARED)
+			pvalidate_4k_page(vaddr, paddr, false);
+
+		/*
+		 * Use the MSR protocol because this function can be called before
+		 * the GHCB is established.
+		 */
+		sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
+		VMGEXIT();
+
+		val = sev_es_rd_ghcb_msr();
+
+		if (GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP)
+			goto e_term;
+
+		if (GHCB_MSR_PSC_RESP_VAL(val))
+			goto e_term;
+
+		/* Page validation must be performed after changing to private */
+		if (op == SNP_PAGE_STATE_PRIVATE)
+			pvalidate_4k_page(vaddr, paddr, true);
+
+		vaddr += PAGE_SIZE;
+		paddr += PAGE_SIZE;
+	}
+
+	return;
+
+e_term:
+	sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
+}
+
+void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr,
+					 unsigned long npages)
+{
+	/*
+	 * This can be invoked in early boot while running identity mapped, so
+	 * use an open coded check for SNP instead of using cc_platform_has().
+	 * This eliminates worries about jump tables or checking boot_cpu_data
+	 * in the cc_platform_has() function.
+	 */
+	if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
+		return;
+
+	 /*
+	  * Ask the hypervisor to mark the memory pages as private in the RMP
+	  * table.
+	  */
+	early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_PRIVATE);
+}
+
+void __head early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr,
+					unsigned long npages)
+{
+	/*
+	 * This can be invoked in early boot while running identity mapped, so
+	 * use an open coded check for SNP instead of using cc_platform_has().
+	 * This eliminates worries about jump tables or checking boot_cpu_data
+	 * in the cc_platform_has() function.
+	 */
+	if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
+		return;
+
+	 /* Ask hypervisor to mark the memory pages shared in the RMP table. */
+	early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_SHARED);
+}
+
+/* Writes to the SVSM CAA MSR are ignored */
+static enum es_result __vc_handle_msr_caa(struct pt_regs *regs, bool write)
+{
+	if (write)
+		return ES_OK;
+
+	regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa));
+	regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa));
+
+	return ES_OK;
+}
+
+/*
+ * TSC related accesses should not exit to the hypervisor when a guest is
+ * executing with Secure TSC enabled, so special handling is required for
+ * accesses of MSR_IA32_TSC and MSR_AMD64_GUEST_TSC_FREQ.
+ */
+static enum es_result __vc_handle_secure_tsc_msrs(struct pt_regs *regs, bool write)
+{
+	u64 tsc;
+
+	/*
+	 * GUEST_TSC_FREQ should not be intercepted when Secure TSC is enabled.
+	 * Terminate the SNP guest when the interception is enabled.
+	 */
+	if (regs->cx == MSR_AMD64_GUEST_TSC_FREQ)
+		return ES_VMM_ERROR;
+
+	/*
+	 * Writes: Writing to MSR_IA32_TSC can cause subsequent reads of the TSC
+	 *         to return undefined values, so ignore all writes.
+	 *
+	 * Reads: Reads of MSR_IA32_TSC should return the current TSC value, use
+	 *        the value returned by rdtsc_ordered().
+	 */
+	if (write) {
+		WARN_ONCE(1, "TSC MSR writes are verboten!\n");
+		return ES_OK;
+	}
+
+	tsc = rdtsc_ordered();
+	regs->ax = lower_32_bits(tsc);
+	regs->dx = upper_32_bits(tsc);
+
+	return ES_OK;
+}
+
+static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
+{
+	struct pt_regs *regs = ctxt->regs;
+	enum es_result ret;
+	bool write;
+
+	/* Is it a WRMSR? */
+	write = ctxt->insn.opcode.bytes[1] == 0x30;
+
+	switch (regs->cx) {
+	case MSR_SVSM_CAA:
+		return __vc_handle_msr_caa(regs, write);
+	case MSR_IA32_TSC:
+	case MSR_AMD64_GUEST_TSC_FREQ:
+		if (sev_status & MSR_AMD64_SNP_SECURE_TSC)
+			return __vc_handle_secure_tsc_msrs(regs, write);
+		break;
+	default:
+		break;
+	}
+
+	ghcb_set_rcx(ghcb, regs->cx);
+	if (write) {
+		ghcb_set_rax(ghcb, regs->ax);
+		ghcb_set_rdx(ghcb, regs->dx);
+	}
+
+	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, write, 0);
+
+	if ((ret == ES_OK) && !write) {
+		regs->ax = ghcb->save.rax;
+		regs->dx = ghcb->save.rdx;
+	}
+
+	return ret;
+}
+
+static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt)
+{
+	int trapnr = ctxt->fi.vector;
+
+	if (trapnr == X86_TRAP_PF)
+		native_write_cr2(ctxt->fi.cr2);
+
+	ctxt->regs->orig_ax = ctxt->fi.error_code;
+	do_early_exception(ctxt->regs, trapnr);
+}
+
+static long *vc_insn_get_rm(struct es_em_ctxt *ctxt)
+{
+	long *reg_array;
+	int offset;
+
+	reg_array = (long *)ctxt->regs;
+	offset    = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs);
+
+	if (offset < 0)
+		return NULL;
+
+	offset /= sizeof(long);
+
+	return reg_array + offset;
+}
+static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
+				 unsigned int bytes, bool read)
+{
+	u64 exit_code, exit_info_1, exit_info_2;
+	unsigned long ghcb_pa = __pa(ghcb);
+	enum es_result res;
+	phys_addr_t paddr;
+	void __user *ref;
+
+	ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs);
+	if (ref == (void __user *)-1L)
+		return ES_UNSUPPORTED;
+
+	exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE;
+
+	res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr);
+	if (res != ES_OK) {
+		if (res == ES_EXCEPTION && !read)
+			ctxt->fi.error_code |= X86_PF_WRITE;
+
+		return res;
+	}
+
+	exit_info_1 = paddr;
+	/* Can never be greater than 8 */
+	exit_info_2 = bytes;
+
+	ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer));
+
+	return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2);
+}
+
+/*
+ * The MOVS instruction has two memory operands, which raises the
+ * problem that it is not known whether the access to the source or the
+ * destination caused the #VC exception (and hence whether an MMIO read
+ * or write operation needs to be emulated).
+ *
+ * Instead of playing games with walking page-tables and trying to guess
+ * whether the source or destination is an MMIO range, split the move
+ * into two operations, a read and a write with only one memory operand.
+ * This will cause a nested #VC exception on the MMIO address which can
+ * then be handled.
+ *
+ * This implementation has the benefit that it also supports MOVS where
+ * source _and_ destination are MMIO regions.
+ *
+ * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a
+ * rare operation. If it turns out to be a performance problem the split
+ * operations can be moved to memcpy_fromio() and memcpy_toio().
+ */
+static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt,
+					  unsigned int bytes)
+{
+	unsigned long ds_base, es_base;
+	unsigned char *src, *dst;
+	unsigned char buffer[8];
+	enum es_result ret;
+	bool rep;
+	int off;
+
+	ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS);
+	es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
+
+	if (ds_base == -1L || es_base == -1L) {
+		ctxt->fi.vector = X86_TRAP_GP;
+		ctxt->fi.error_code = 0;
+		return ES_EXCEPTION;
+	}
+
+	src = ds_base + (unsigned char *)ctxt->regs->si;
+	dst = es_base + (unsigned char *)ctxt->regs->di;
+
+	ret = vc_read_mem(ctxt, src, buffer, bytes);
+	if (ret != ES_OK)
+		return ret;
+
+	ret = vc_write_mem(ctxt, dst, buffer, bytes);
+	if (ret != ES_OK)
+		return ret;
+
+	if (ctxt->regs->flags & X86_EFLAGS_DF)
+		off = -bytes;
+	else
+		off =  bytes;
+
+	ctxt->regs->si += off;
+	ctxt->regs->di += off;
+
+	rep = insn_has_rep_prefix(&ctxt->insn);
+	if (rep)
+		ctxt->regs->cx -= 1;
+
+	if (!rep || ctxt->regs->cx == 0)
+		return ES_OK;
+	else
+		return ES_RETRY;
+}
+
+static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
+{
+	struct insn *insn = &ctxt->insn;
+	enum insn_mmio_type mmio;
+	unsigned int bytes = 0;
+	enum es_result ret;
+	u8 sign_byte;
+	long *reg_data;
+
+	mmio = insn_decode_mmio(insn, &bytes);
+	if (mmio == INSN_MMIO_DECODE_FAILED)
+		return ES_DECODE_FAILED;
+
+	if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) {
+		reg_data = insn_get_modrm_reg_ptr(insn, ctxt->regs);
+		if (!reg_data)
+			return ES_DECODE_FAILED;
+	}
+
+	if (user_mode(ctxt->regs))
+		return ES_UNSUPPORTED;
+
+	switch (mmio) {
+	case INSN_MMIO_WRITE:
+		memcpy(ghcb->shared_buffer, reg_data, bytes);
+		ret = vc_do_mmio(ghcb, ctxt, bytes, false);
+		break;
+	case INSN_MMIO_WRITE_IMM:
+		memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes);
+		ret = vc_do_mmio(ghcb, ctxt, bytes, false);
+		break;
+	case INSN_MMIO_READ:
+		ret = vc_do_mmio(ghcb, ctxt, bytes, true);
+		if (ret)
+			break;
+
+		/* Zero-extend for 32-bit operation */
+		if (bytes == 4)
+			*reg_data = 0;
+
+		memcpy(reg_data, ghcb->shared_buffer, bytes);
+		break;
+	case INSN_MMIO_READ_ZERO_EXTEND:
+		ret = vc_do_mmio(ghcb, ctxt, bytes, true);
+		if (ret)
+			break;
+
+		/* Zero extend based on operand size */
+		memset(reg_data, 0, insn->opnd_bytes);
+		memcpy(reg_data, ghcb->shared_buffer, bytes);
+		break;
+	case INSN_MMIO_READ_SIGN_EXTEND:
+		ret = vc_do_mmio(ghcb, ctxt, bytes, true);
+		if (ret)
+			break;
+
+		if (bytes == 1) {
+			u8 *val = (u8 *)ghcb->shared_buffer;
+
+			sign_byte = (*val & 0x80) ? 0xff : 0x00;
+		} else {
+			u16 *val = (u16 *)ghcb->shared_buffer;
+
+			sign_byte = (*val & 0x8000) ? 0xff : 0x00;
+		}
+
+		/* Sign extend based on operand size */
+		memset(reg_data, sign_byte, insn->opnd_bytes);
+		memcpy(reg_data, ghcb->shared_buffer, bytes);
+		break;
+	case INSN_MMIO_MOVS:
+		ret = vc_handle_mmio_movs(ctxt, bytes);
+		break;
+	default:
+		ret = ES_UNSUPPORTED;
+		break;
+	}
+
+	return ret;
+}
+
+static enum es_result vc_handle_dr7_write(struct ghcb *ghcb,
+					  struct es_em_ctxt *ctxt)
+{
+	struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
+	long val, *reg = vc_insn_get_rm(ctxt);
+	enum es_result ret;
+
+	if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
+		return ES_VMM_ERROR;
+
+	if (!reg)
+		return ES_DECODE_FAILED;
+
+	val = *reg;
+
+	/* Upper 32 bits must be written as zeroes */
+	if (val >> 32) {
+		ctxt->fi.vector = X86_TRAP_GP;
+		ctxt->fi.error_code = 0;
+		return ES_EXCEPTION;
+	}
+
+	/* Clear out other reserved bits and set bit 10 */
+	val = (val & 0xffff23ffL) | BIT(10);
+
+	/* Early non-zero writes to DR7 are not supported */
+	if (!data && (val & ~DR7_RESET_VALUE))
+		return ES_UNSUPPORTED;
+
+	/* Using a value of 0 for ExitInfo1 means RAX holds the value */
+	ghcb_set_rax(ghcb, val);
+	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0);
+	if (ret != ES_OK)
+		return ret;
+
+	if (data)
+		data->dr7 = val;
+
+	return ES_OK;
+}
+
+static enum es_result vc_handle_dr7_read(struct ghcb *ghcb,
+					 struct es_em_ctxt *ctxt)
+{
+	struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
+	long *reg = vc_insn_get_rm(ctxt);
+
+	if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
+		return ES_VMM_ERROR;
+
+	if (!reg)
+		return ES_DECODE_FAILED;
+
+	if (data)
+		*reg = data->dr7;
+	else
+		*reg = DR7_RESET_VALUE;
+
+	return ES_OK;
+}
+
+static enum es_result vc_handle_wbinvd(struct ghcb *ghcb,
+				       struct es_em_ctxt *ctxt)
+{
+	return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0);
+}
+
+static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
+{
+	enum es_result ret;
+
+	ghcb_set_rcx(ghcb, ctxt->regs->cx);
+
+	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0);
+	if (ret != ES_OK)
+		return ret;
+
+	if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb)))
+		return ES_VMM_ERROR;
+
+	ctxt->regs->ax = ghcb->save.rax;
+	ctxt->regs->dx = ghcb->save.rdx;
+
+	return ES_OK;
+}
+
+static enum es_result vc_handle_monitor(struct ghcb *ghcb,
+					struct es_em_ctxt *ctxt)
+{
+	/*
+	 * Treat it as a NOP and do not leak a physical address to the
+	 * hypervisor.
+	 */
+	return ES_OK;
+}
+
+static enum es_result vc_handle_mwait(struct ghcb *ghcb,
+				      struct es_em_ctxt *ctxt)
+{
+	/* Treat the same as MONITOR/MONITORX */
+	return ES_OK;
+}
+
+static enum es_result vc_handle_vmmcall(struct ghcb *ghcb,
+					struct es_em_ctxt *ctxt)
+{
+	enum es_result ret;
+
+	ghcb_set_rax(ghcb, ctxt->regs->ax);
+	ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0);
+
+	if (x86_platform.hyper.sev_es_hcall_prepare)
+		x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs);
+
+	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0);
+	if (ret != ES_OK)
+		return ret;
+
+	if (!ghcb_rax_is_valid(ghcb))
+		return ES_VMM_ERROR;
+
+	ctxt->regs->ax = ghcb->save.rax;
+
+	/*
+	 * Call sev_es_hcall_finish() after regs->ax is already set.
+	 * This allows the hypervisor handler to overwrite it again if
+	 * necessary.
+	 */
+	if (x86_platform.hyper.sev_es_hcall_finish &&
+	    !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs))
+		return ES_VMM_ERROR;
+
+	return ES_OK;
+}
+
+static enum es_result vc_handle_trap_ac(struct ghcb *ghcb,
+					struct es_em_ctxt *ctxt)
+{
+	/*
+	 * Calling ecx_alignment_check() directly does not work, because it
+	 * enables IRQs and the GHCB is active. Forward the exception and call
+	 * it later from vc_forward_exception().
+	 */
+	ctxt->fi.vector = X86_TRAP_AC;
+	ctxt->fi.error_code = 0;
+	return ES_EXCEPTION;
+}
+
+static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
+					 struct ghcb *ghcb,
+					 unsigned long exit_code)
+{
+	enum es_result result = vc_check_opcode_bytes(ctxt, exit_code);
+
+	if (result != ES_OK)
+		return result;
+
+	switch (exit_code) {
+	case SVM_EXIT_READ_DR7:
+		result = vc_handle_dr7_read(ghcb, ctxt);
+		break;
+	case SVM_EXIT_WRITE_DR7:
+		result = vc_handle_dr7_write(ghcb, ctxt);
+		break;
+	case SVM_EXIT_EXCP_BASE + X86_TRAP_AC:
+		result = vc_handle_trap_ac(ghcb, ctxt);
+		break;
+	case SVM_EXIT_RDTSC:
+	case SVM_EXIT_RDTSCP:
+		result = vc_handle_rdtsc(ghcb, ctxt, exit_code);
+		break;
+	case SVM_EXIT_RDPMC:
+		result = vc_handle_rdpmc(ghcb, ctxt);
+		break;
+	case SVM_EXIT_INVD:
+		pr_err_ratelimited("#VC exception for INVD??? Seriously???\n");
+		result = ES_UNSUPPORTED;
+		break;
+	case SVM_EXIT_CPUID:
+		result = vc_handle_cpuid(ghcb, ctxt);
+		break;
+	case SVM_EXIT_IOIO:
+		result = vc_handle_ioio(ghcb, ctxt);
+		break;
+	case SVM_EXIT_MSR:
+		result = vc_handle_msr(ghcb, ctxt);
+		break;
+	case SVM_EXIT_VMMCALL:
+		result = vc_handle_vmmcall(ghcb, ctxt);
+		break;
+	case SVM_EXIT_WBINVD:
+		result = vc_handle_wbinvd(ghcb, ctxt);
+		break;
+	case SVM_EXIT_MONITOR:
+		result = vc_handle_monitor(ghcb, ctxt);
+		break;
+	case SVM_EXIT_MWAIT:
+		result = vc_handle_mwait(ghcb, ctxt);
+		break;
+	case SVM_EXIT_NPF:
+		result = vc_handle_mmio(ghcb, ctxt);
+		break;
+	default:
+		/*
+		 * Unexpected #VC exception
+		 */
+		result = ES_UNSUPPORTED;
+	}
+
+	return result;
+}
+
+static __always_inline bool is_vc2_stack(unsigned long sp)
+{
+	return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2));
+}
+
+static __always_inline bool vc_from_invalid_context(struct pt_regs *regs)
+{
+	unsigned long sp, prev_sp;
+
+	sp      = (unsigned long)regs;
+	prev_sp = regs->sp;
+
+	/*
+	 * If the code was already executing on the VC2 stack when the #VC
+	 * happened, let it proceed to the normal handling routine. This way the
+	 * code executing on the VC2 stack can cause #VC exceptions to get handled.
+	 */
+	return is_vc2_stack(sp) && !is_vc2_stack(prev_sp);
+}
+
+static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code)
+{
+	struct ghcb_state state;
+	struct es_em_ctxt ctxt;
+	enum es_result result;
+	struct ghcb *ghcb;
+	bool ret = true;
+
+	ghcb = __sev_get_ghcb(&state);
+
+	vc_ghcb_invalidate(ghcb);
+	result = vc_init_em_ctxt(&ctxt, regs, error_code);
+
+	if (result == ES_OK)
+		result = vc_handle_exitcode(&ctxt, ghcb, error_code);
+
+	__sev_put_ghcb(&state);
+
+	/* Done - now check the result */
+	switch (result) {
+	case ES_OK:
+		vc_finish_insn(&ctxt);
+		break;
+	case ES_UNSUPPORTED:
+		pr_err_ratelimited("Unsupported exit-code 0x%02lx in #VC exception (IP: 0x%lx)\n",
+				   error_code, regs->ip);
+		ret = false;
+		break;
+	case ES_VMM_ERROR:
+		pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
+				   error_code, regs->ip);
+		ret = false;
+		break;
+	case ES_DECODE_FAILED:
+		pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
+				   error_code, regs->ip);
+		ret = false;
+		break;
+	case ES_EXCEPTION:
+		vc_forward_exception(&ctxt);
+		break;
+	case ES_RETRY:
+		/* Nothing to do */
+		break;
+	default:
+		pr_emerg("Unknown result in %s():%d\n", __func__, result);
+		/*
+		 * Emulating the instruction which caused the #VC exception
+		 * failed - can't continue so print debug information
+		 */
+		BUG();
+	}
+
+	return ret;
+}
+
+static __always_inline bool vc_is_db(unsigned long error_code)
+{
+	return error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB;
+}
+
+/*
+ * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode
+ * and will panic when an error happens.
+ */
+DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication)
+{
+	irqentry_state_t irq_state;
+
+	/*
+	 * With the current implementation it is always possible to switch to a
+	 * safe stack because #VC exceptions only happen at known places, like
+	 * intercepted instructions or accesses to MMIO areas/IO ports. They can
+	 * also happen with code instrumentation when the hypervisor intercepts
+	 * #DB, but the critical paths are forbidden to be instrumented, so #DB
+	 * exceptions currently also only happen in safe places.
+	 *
+	 * But keep this here in case the noinstr annotations are violated due
+	 * to bug elsewhere.
+	 */
+	if (unlikely(vc_from_invalid_context(regs))) {
+		instrumentation_begin();
+		panic("Can't handle #VC exception from unsupported context\n");
+		instrumentation_end();
+	}
+
+	/*
+	 * Handle #DB before calling into !noinstr code to avoid recursive #DB.
+	 */
+	if (vc_is_db(error_code)) {
+		exc_debug(regs);
+		return;
+	}
+
+	irq_state = irqentry_nmi_enter(regs);
+
+	instrumentation_begin();
+
+	if (!vc_raw_handle_exception(regs, error_code)) {
+		/* Show some debug info */
+		show_regs(regs);
+
+		/* Ask hypervisor to sev_es_terminate */
+		sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
+
+		/* If that fails and we get here - just panic */
+		panic("Returned from Terminate-Request to Hypervisor\n");
+	}
+
+	instrumentation_end();
+	irqentry_nmi_exit(regs, irq_state);
+}
+
+/*
+ * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode
+ * and will kill the current task with SIGBUS when an error happens.
+ */
+DEFINE_IDTENTRY_VC_USER(exc_vmm_communication)
+{
+	/*
+	 * Handle #DB before calling into !noinstr code to avoid recursive #DB.
+	 */
+	if (vc_is_db(error_code)) {
+		noist_exc_debug(regs);
+		return;
+	}
+
+	irqentry_enter_from_user_mode(regs);
+	instrumentation_begin();
+
+	if (!vc_raw_handle_exception(regs, error_code)) {
+		/*
+		 * Do not kill the machine if user-space triggered the
+		 * exception. Send SIGBUS instead and let user-space deal with
+		 * it.
+		 */
+		force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0);
+	}
+
+	instrumentation_end();
+	irqentry_exit_to_user_mode(regs);
+}
+
+bool __init handle_vc_boot_ghcb(struct pt_regs *regs)
+{
+	unsigned long exit_code = regs->orig_ax;
+	struct es_em_ctxt ctxt;
+	enum es_result result;
+
+	vc_ghcb_invalidate(boot_ghcb);
+
+	result = vc_init_em_ctxt(&ctxt, regs, exit_code);
+	if (result == ES_OK)
+		result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code);
+
+	/* Done - now check the result */
+	switch (result) {
+	case ES_OK:
+		vc_finish_insn(&ctxt);
+		break;
+	case ES_UNSUPPORTED:
+		early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n",
+				exit_code, regs->ip);
+		goto fail;
+	case ES_VMM_ERROR:
+		early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
+				exit_code, regs->ip);
+		goto fail;
+	case ES_DECODE_FAILED:
+		early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
+				exit_code, regs->ip);
+		goto fail;
+	case ES_EXCEPTION:
+		vc_early_forward_exception(&ctxt);
+		break;
+	case ES_RETRY:
+		/* Nothing to do */
+		break;
+	default:
+		BUG();
+	}
+
+	return true;
+
+fail:
+	show_regs(regs);
+
+	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
+}
+
+/*
+ * Initial set up of SNP relies on information provided by the
+ * Confidential Computing blob, which can be passed to the kernel
+ * in the following ways, depending on how it is booted:
+ *
+ * - when booted via the boot/decompress kernel:
+ *   - via boot_params
+ *
+ * - when booted directly by firmware/bootloader (e.g. CONFIG_PVH):
+ *   - via a setup_data entry, as defined by the Linux Boot Protocol
+ *
+ * Scan for the blob in that order.
+ */
+static __head struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
+{
+	struct cc_blob_sev_info *cc_info;
+
+	/* Boot kernel would have passed the CC blob via boot_params. */
+	if (bp->cc_blob_address) {
+		cc_info = (struct cc_blob_sev_info *)(unsigned long)bp->cc_blob_address;
+		goto found_cc_info;
+	}
+
+	/*
+	 * If kernel was booted directly, without the use of the
+	 * boot/decompression kernel, the CC blob may have been passed via
+	 * setup_data instead.
+	 */
+	cc_info = find_cc_blob_setup_data(bp);
+	if (!cc_info)
+		return NULL;
+
+found_cc_info:
+	if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
+		snp_abort();
+
+	return cc_info;
+}
+
+static __head void svsm_setup(struct cc_blob_sev_info *cc_info)
+{
+	struct svsm_call call = {};
+	int ret;
+	u64 pa;
+
+	/*
+	 * Record the SVSM Calling Area address (CAA) if the guest is not
+	 * running at VMPL0. The CA will be used to communicate with the
+	 * SVSM to perform the SVSM services.
+	 */
+	if (!svsm_setup_ca(cc_info))
+		return;
+
+	/*
+	 * It is very early in the boot and the kernel is running identity
+	 * mapped but without having adjusted the pagetables to where the
+	 * kernel was loaded (physbase), so the get the CA address using
+	 * RIP-relative addressing.
+	 */
+	pa = (u64)rip_rel_ptr(&boot_svsm_ca_page);
+
+	/*
+	 * Switch over to the boot SVSM CA while the current CA is still
+	 * addressable. There is no GHCB at this point so use the MSR protocol.
+	 *
+	 * SVSM_CORE_REMAP_CA call:
+	 *   RAX = 0 (Protocol=0, CallID=0)
+	 *   RCX = New CA GPA
+	 */
+	call.caa = svsm_get_caa();
+	call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA);
+	call.rcx = pa;
+	ret = svsm_perform_call_protocol(&call);
+	if (ret)
+		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CA_REMAP_FAIL);
+
+	RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)pa;
+	RIP_REL_REF(boot_svsm_caa_pa) = pa;
+}
+
+bool __head snp_init(struct boot_params *bp)
+{
+	struct cc_blob_sev_info *cc_info;
+
+	if (!bp)
+		return false;
+
+	cc_info = find_cc_blob(bp);
+	if (!cc_info)
+		return false;
+
+	if (cc_info->secrets_phys && cc_info->secrets_len == PAGE_SIZE)
+		secrets_pa = cc_info->secrets_phys;
+	else
+		return false;
+
+	setup_cpuid_table(cc_info);
+
+	svsm_setup(cc_info);
+
+	/*
+	 * The CC blob will be used later to access the secrets page. Cache
+	 * it here like the boot kernel does.
+	 */
+	bp->cc_blob_address = (u32)(unsigned long)cc_info;
+
+	return true;
+}
+
+void __head __noreturn snp_abort(void)
+{
+	sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
+}
-- 
2.49.0.504.g3bcea36a83-goog

From: Ard Biesheuvel <ardb@kernel.org>

Move the SEV startup code into arch/x86/boot/startup/, where it will
reside along with other code that executes extremely early, and
therefore needs to be built in a special manner.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/compressed/sev.c                              |  2 +-
 arch/x86/boot/startup/Makefile                              |  2 +-
 arch/x86/{coco/sev/shared.c => boot/startup/sev-shared.c}   |  0
 arch/x86/{coco/sev/startup.c => boot/startup/sev-startup.c} |  2 +-
 arch/x86/coco/sev/Makefile                                  | 21 +-------------------
 5 files changed, 4 insertions(+), 23 deletions(-)

From: Ard Biesheuvel <ardb@kernel.org>

Now that the early SEV code is built with -fPIC, RIP_REL_REF() has no
effect and can be dropped.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/x86/boot/startup/sev-shared.c  | 26 +++++++++-----------
 arch/x86/boot/startup/sev-startup.c | 16 ++++++------
 arch/x86/include/asm/sev-internal.h | 18 +++-----------
 3 files changed, 23 insertions(+), 37 deletions(-)

diff --git a/arch/x86/boot/startup/sev-shared.c b/arch/x86/boot/startup/sev-shared.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/sev-shared.c
+++ b/arch/x86/boot/startup/sev-shared.c
@@ -XXX,XX +XXX,XX @@ static int svsm_perform_ghcb_protocol(struct ghcb *ghcb, struct svsm_call *call)
 	 * Fill in protocol and format specifiers. This can be called very early
 	 * in the boot, so use rip-relative references as needed.
 	 */
-	ghcb->protocol_version = RIP_REL_REF(ghcb_version);
+	ghcb->protocol_version = ghcb_version;
 	ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
 
 	ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_SNP_RUN_VMPL);
@@ -XXX,XX +XXX,XX @@ snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
 		leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0;
 
 		/* Skip post-processing for out-of-range zero leafs. */
-		if (!(leaf->fn <= RIP_REL_REF(cpuid_std_range_max) ||
-		      (leaf->fn >= 0x40000000 && leaf->fn <= RIP_REL_REF(cpuid_hyp_range_max)) ||
-		      (leaf->fn >= 0x80000000 && leaf->fn <= RIP_REL_REF(cpuid_ext_range_max))))
+		if (!(leaf->fn <= cpuid_std_range_max ||
+		      (leaf->fn >= 0x40000000 && leaf->fn <= cpuid_hyp_range_max) ||
+		      (leaf->fn >= 0x80000000 && leaf->fn <= cpuid_ext_range_max)))
 			return 0;
 	}
 
@@ -XXX,XX +XXX,XX @@ static void __head setup_cpuid_table(const struct cc_blob_sev_info *cc_info)
 		const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
 
 		if (fn->eax_in == 0x0)
-			RIP_REL_REF(cpuid_std_range_max) = fn->eax;
+			cpuid_std_range_max = fn->eax;
 		else if (fn->eax_in == 0x40000000)
-			RIP_REL_REF(cpuid_hyp_range_max) = fn->eax;
+			cpuid_hyp_range_max = fn->eax;
 		else if (fn->eax_in == 0x80000000)
-			RIP_REL_REF(cpuid_ext_range_max) = fn->eax;
+			cpuid_ext_range_max = fn->eax;
 	}
 }
 
@@ -XXX,XX +XXX,XX @@ static void __head pvalidate_4k_page(unsigned long vaddr, unsigned long paddr,
 {
 	int ret;
 
-	/*
-	 * This can be called very early during boot, so use rIP-relative
-	 * references as needed.
-	 */
-	if (RIP_REL_REF(snp_vmpl)) {
+	if (snp_vmpl) {
 		svsm_pval_4k_page(paddr, validate);
 	} else {
 		ret = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
@@ -XXX,XX +XXX,XX @@ static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info)
 	if (!secrets_page->svsm_guest_vmpl)
 		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_VMPL0);
 
-	RIP_REL_REF(snp_vmpl) = secrets_page->svsm_guest_vmpl;
+	snp_vmpl = secrets_page->svsm_guest_vmpl;
 
 	caa = secrets_page->svsm_caa;
 
@@ -XXX,XX +XXX,XX @@ static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info)
 	 * The CA is identity mapped when this routine is called, both by the
 	 * decompressor code and the early kernel code.
 	 */
-	RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)caa;
-	RIP_REL_REF(boot_svsm_caa_pa) = caa;
+	boot_svsm_caa = (struct svsm_ca *)caa;
+	boot_svsm_caa_pa = caa;
 
 	/* Advertise the SVSM presence via CPUID. */
 	cpuid_table = (struct snp_cpuid_table *)snp_cpuid_get_table();
diff --git a/arch/x86/boot/startup/sev-startup.c b/arch/x86/boot/startup/sev-startup.c
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/boot/startup/sev-startup.c
+++ b/arch/x86/boot/startup/sev-startup.c
@@ -XXX,XX +XXX,XX @@ int svsm_perform_call_protocol(struct svsm_call *call)
 	 * ghcbs_initialized is set, then it is late in the boot and no need
 	 * to worry about rip-relative references in called functions.
 	 */
-	if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
+	if (sev_cfg.ghcbs_initialized)
 		ghcb = __sev_get_ghcb(&state);
-	else if (RIP_REL_REF(boot_ghcb))
-		ghcb = RIP_REL_REF(boot_ghcb);
+	else if (boot_ghcb)
+		ghcb = boot_ghcb;
 	else
 		ghcb = NULL;
 
@@ -XXX,XX +XXX,XX @@ int svsm_perform_call_protocol(struct svsm_call *call)
 			   : svsm_perform_msr_protocol(call);
 	} while (ret == -EAGAIN);
 
-	if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
+	if (sev_cfg.ghcbs_initialized)
 		__sev_put_ghcb(&state);
 
 	native_local_irq_restore(flags);
@@ -XXX,XX +XXX,XX @@ void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long padd
 	 * This eliminates worries about jump tables or checking boot_cpu_data
 	 * in the cc_platform_has() function.
 	 */
-	if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
+	if (!(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
 		return;
 
 	 /*
@@ -XXX,XX +XXX,XX @@ void __head early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr
 	 * This eliminates worries about jump tables or checking boot_cpu_data
 	 * in the cc_platform_has() function.
 	 */
-	if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
+	if (!(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
 		return;
 
 	 /* Ask hypervisor to mark the memory pages shared in the RMP table. */
@@ -XXX,XX +XXX,XX @@ static __head void svsm_setup(struct cc_blob_sev_info *cc_info)
 	if (ret)
 		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CA_REMAP_FAIL);
 
-	RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)pa;
-	RIP_REL_REF(boot_svsm_caa_pa) = pa;
+	boot_svsm_caa = (struct svsm_ca *)pa;
+	boot_svsm_caa_pa = pa;
 }
 
 bool __head snp_init(struct boot_params *bp)
diff --git a/arch/x86/include/asm/sev-internal.h b/arch/x86/include/asm/sev-internal.h
index XXXXXXX..XXXXXXX 100644
--- a/arch/x86/include/asm/sev-internal.h
+++ b/arch/x86/include/asm/sev-internal.h
@@ -XXX,XX +XXX,XX @@ extern u64 boot_svsm_caa_pa;
 
 static __always_inline struct svsm_ca *svsm_get_caa(void)
 {
-	/*
-	 * Use rIP-relative references when called early in the boot. If
-	 * ->use_cas is set, then it is late in the boot and no need
-	 * to worry about rIP-relative references.
-	 */
-	if (RIP_REL_REF(sev_cfg).use_cas)
+	if (sev_cfg.use_cas)
 		return this_cpu_read(svsm_caa);
 	else
-		return RIP_REL_REF(boot_svsm_caa);
+		return boot_svsm_caa;
 }
 
 static __always_inline u64 svsm_get_caa_pa(void)
 {
-	/*
-	 * Use rIP-relative references when called early in the boot. If
-	 * ->use_cas is set, then it is late in the boot and no need
-	 * to worry about rIP-relative references.
-	 */
-	if (RIP_REL_REF(sev_cfg).use_cas)
+	if (sev_cfg.use_cas)
 		return this_cpu_read(svsm_caa_pa);
 	else
-		return RIP_REL_REF(boot_svsm_caa_pa);
+		return boot_svsm_caa_pa;
 }
 
 int svsm_perform_call_protocol(struct svsm_call *call);
-- 
2.49.0.504.g3bcea36a83-goog