We will want to use that value for call trace generation, and likely
also to eliminate the somewhat fragile shadow stack searching done in
fixup_exception_return(). For those purposes, guest-only entry points do
not need to record that value.

To keep the saving code simple, record our own SSP that corresponds to
an exception frame, pointing to the top of the shadow stack counterpart
of what the CPU has saved on the regular stack. Consuming code can then
work its way from there.

In SAVE_ALL / RESTORE_ALL simply drop the use of UREGS_r15. We want the
full size in all cases, so what exactly the top-of-stack field is going
to be (whose UREGS_* is 0 anyway) doesn't matter this much there. This
way we don't need to distinguish between XEN_SHSTK=y and XEN_SHSTK=n.

Signed-off-by: Jan Beulich <jbeulich@suse.com>
---
v3: Put new field at the front of struct cpu_user_regs.
v2: Add comment ahead of SAVE_ALL. Add comma between its parameters.
    Re-base.

--- a/xen/arch/x86/hvm/svm/entry.S
+++ b/xen/arch/x86/hvm/svm/entry.S
@@ -89,7 +89,7 @@ __UNLIKELY_END(nsvm_hap)
 
         vmrun
 
-        SAVE_ALL
+        SAVE_ALL ssp=0
 
         GET_CURRENT(bx)
 
--- a/xen/arch/x86/hvm/vmx/entry.S
+++ b/xen/arch/x86/hvm/vmx/entry.S
@@ -22,7 +22,7 @@
 #include <asm/page.h>
 
 FUNC(vmx_asm_vmexit_handler)
-        SAVE_ALL
+        SAVE_ALL ssp=0
 
         mov  %cr2,%rax
         GET_CURRENT(bx)
@@ -157,7 +157,7 @@ UNLIKELY_END(realmode)
 
 .Lvmx_vmentry_fail:
         sti
-        SAVE_ALL
+        SAVE_ALL ssp=0
 
         /*
          * SPEC_CTRL_ENTRY notes
--- a/xen/arch/x86/include/asm/asm_defns.h
+++ b/xen/arch/x86/include/asm/asm_defns.h
@@ -219,8 +219,12 @@ static always_inline void stac(void)
 #endif
 
 #ifdef __ASSEMBLER__
-.macro SAVE_ALL compat=0
-        addq  $-(UREGS_error_code-UREGS_r15), %rsp
+/*
+ * Use sites may override ssp to 0. It should never be overridden to 1.
+ * NB: compat=1 implies ssp=0.
+ */
+.macro SAVE_ALL compat=0, ssp=IS_ENABLED(CONFIG_XEN_SHSTK)
+        addq  $-UREGS_error_code, %rsp
         cld
         movq  %rdi,UREGS_rdi(%rsp)
         xor   %edi, %edi
@@ -233,6 +237,9 @@ static always_inline void stac(void)
         movq  %rax,UREGS_rax(%rsp)
         xor   %eax, %eax
 .if !\compat
+.if \ssp
+        rdsspq %rcx
+.endif
         movq  %r8,UREGS_r8(%rsp)
         movq  %r9,UREGS_r9(%rsp)
         movq  %r10,UREGS_r10(%rsp)
@@ -262,6 +269,9 @@ static always_inline void stac(void)
         xor   %r13d, %r13d
         xor   %r14d, %r14d
         xor   %r15d, %r15d
+#ifdef CONFIG_XEN_SHSTK
+        mov   %rcx, UREGS_entry_ssp(%rsp)
+#endif
 .endm
 
 #define LOAD_ONE_REG(reg, compat) \
@@ -309,13 +319,15 @@ static always_inline void stac(void)
         LOAD_ONE_REG(dx, \compat)
         LOAD_ONE_REG(si, \compat)
         LOAD_ONE_REG(di, \compat)
-        subq  $-(UREGS_error_code-UREGS_r15+\adj), %rsp
+        subq  $-(UREGS_error_code + \adj), %rsp
 .endm
 
 /*
- * Push and clear GPRs
+ * Push and clear GPRs.
+ *
+ * Use sites may override ssp to 0. It should never be overridden to 1.
  */
-.macro PUSH_AND_CLEAR_GPRS
+.macro PUSH_AND_CLEAR_GPRS ssp=IS_ENABLED(CONFIG_XEN_SHSTK)
         push  %rdi
         xor   %edi, %edi
         push  %rsi
@@ -326,6 +338,9 @@ static always_inline void stac(void)
         xor   %ecx, %ecx
         push  %rax
         xor   %eax, %eax
+ .if \ssp
+        rdsspq %rcx
+ .endif
         push  %r8
         xor   %r8d, %r8d
         push  %r9
@@ -352,12 +367,18 @@ static always_inline void stac(void)
         xor   %r14d, %r14d
         push  %r15
         xor   %r15d, %r15d
+#ifdef CONFIG_XEN_SHSTK
+        push  %rcx
+#endif
 .endm
 
 /*
  * POP GPRs from a UREGS_* frame on the stack.  Does not modify flags.
  */
 .macro POP_GPRS skip_rax=0
+#ifdef CONFIG_XEN_SHSTK
+        pop   %rcx
+#endif
         pop   %r15
         pop   %r14
         pop   %r13
--- a/xen/arch/x86/include/asm/cpu-user-regs.h
+++ b/xen/arch/x86/include/asm/cpu-user-regs.h
@@ -11,6 +11,15 @@
  */
 struct cpu_user_regs
 {
+#ifdef CONFIG_XEN_SHSTK
+    /*
+     * This points _at_ the corresponding shadow stack frame; it is _not_ the
+     * outer context's SSP.  That, if the outer context has CET-SS enabled,
+     * is stored in the top slot of the pointed to shadow stack.
+     */
+    uint64_t entry_ssp;
+#endif
+
     union { uint64_t r15;    uint32_t r15d;   uint16_t r15w;  uint8_t r15b; };
     union { uint64_t r14;    uint32_t r14d;   uint16_t r14w;  uint8_t r14b; };
     union { uint64_t r13;    uint32_t r13d;   uint16_t r13w;  uint8_t r13b; };
--- a/xen/arch/x86/x86_64/asm-offsets.c
+++ b/xen/arch/x86/x86_64/asm-offsets.c
@@ -53,6 +53,9 @@ void __dummy__(void)
     OFFSET(UREGS_eflags, struct cpu_user_regs, rflags);
     OFFSET(UREGS_rsp, struct cpu_user_regs, rsp);
     OFFSET(UREGS_ss, struct cpu_user_regs, ss);
+#ifdef CONFIG_XEN_SHSTK
+    OFFSET(UREGS_entry_ssp, struct cpu_user_regs, entry_ssp);
+#endif
     DEFINE(UREGS_kernel_sizeof, sizeof(struct cpu_user_regs));
     BLANK();
 
--- a/xen/arch/x86/x86_64/entry.S
+++ b/xen/arch/x86/x86_64/entry.S
@@ -304,7 +304,7 @@ FUNC(lstar_enter)
         pushq $0
         BUILD_BUG_ON(TRAP_syscall & 0xff)
         movb  $TRAP_syscall >> 8, EFRAME_entry_vector + 1(%rsp)
-        SAVE_ALL
+        SAVE_ALL ssp=0
 
         GET_STACK_END(14)
 
@@ -344,7 +344,7 @@ FUNC(cstar_enter)
         pushq $0
         BUILD_BUG_ON(TRAP_syscall & 0xff)
         movb  $TRAP_syscall >> 8, EFRAME_entry_vector + 1(%rsp)
-        SAVE_ALL
+        SAVE_ALL ssp=0
 
         GET_STACK_END(14)
 
@@ -388,7 +388,7 @@ LABEL(sysenter_eflags_saved, 0)
         pushq $0
         BUILD_BUG_ON(TRAP_syscall & 0xff)
         movb  $TRAP_syscall >> 8, EFRAME_entry_vector + 1(%rsp)
-        SAVE_ALL
+        SAVE_ALL ssp=0
 
         GET_STACK_END(14)
 
@@ -444,7 +444,7 @@ FUNC(entry_int80)
         ALTERNATIVE "", clac, X86_FEATURE_XEN_SMAP
         pushq $0
         movb  $0x80, EFRAME_entry_vector(%rsp)
-        SAVE_ALL
+        SAVE_ALL ssp=0
 
         GET_STACK_END(14)
 
--- a/xen/arch/x86/x86_64/entry-fred.S
+++ b/xen/arch/x86/x86_64/entry-fred.S
@@ -11,7 +11,7 @@
         /* The Ring3 entry point is required to be 4k aligned. */
 
 FUNC(entry_FRED_R3, 4096)
-        PUSH_AND_CLEAR_GPRS
+        PUSH_AND_CLEAR_GPRS ssp=0
 
         mov     %rsp, %rdi
         call    entry_from_pv
@@ -54,7 +54,7 @@ LABEL(eretu, 0)
 END(eretu_exit_to_guest)
 
 FUNC(eretu_error_dom_crash)
-        PUSH_AND_CLEAR_GPRS
+        PUSH_AND_CLEAR_GPRS ssp=0
         sti
         call    asm_domain_crash_synchronous  /* Does not return */
 END(eretu_error_dom_crash)