Patch 8 is the XSA-439 fix for the AMD DIV issue, disclosed insufficiently ahead of August 8th for us to prepare a fix for the embargo. Patches 1 thru 7 are prerequisites, identified while trying to write patch 8. All 8 patches are for all security trees. Andrew Cooper (8): x86/spec-ctrl: Fix confusion between SPEC_CTRL_EXIT_TO_XEN{,_IST} x86/spec-ctrl: Fold DO_SPEC_CTRL_EXIT_TO_XEN into it's single user x86/spec-ctrl: Turn the remaining SPEC_CTRL_{ENTRY,EXIT}_* into asm macros x86/spec-ctrl: Extend all SPEC_CTRL_{ENTER,EXIT}_* comments x86/entry: Adjust restore_all_xen to hold stack_end in %r14 x86/entry: Track the IST-ness of an entry for the exit paths x86/spec-ctrl: Issue VERW during IST exit to Xen x86/spec-ctrl: Mitigate the Zen1 DIV leakge docs/misc/xen-command-line.pandoc | 6 +- xen/arch/x86/hvm/svm/entry.S | 1 + xen/arch/x86/include/asm/cpufeatures.h | 2 +- xen/arch/x86/include/asm/spec_ctrl_asm.h | 150 ++++++++++++++++------- xen/arch/x86/spec_ctrl.c | 45 ++++++- xen/arch/x86/traps.c | 13 ++ xen/arch/x86/x86_64/compat/entry.S | 9 +- xen/arch/x86/x86_64/entry.S | 31 +++-- 8 files changed, 198 insertions(+), 59 deletions(-) base-commit: 6aa25c32180ab59081c73bae4c568367d9133a1f -- 2.30.2 c/s 3fffaf9c13e9 ("x86/entry: Avoid using alternatives in NMI/#MC paths") dropped the only user, leaving behind the (incorrect) implication that Xen had split exit paths. Delete the unused SPEC_CTRL_EXIT_TO_XEN and rename SPEC_CTRL_EXIT_TO_XEN_IST to SPEC_CTRL_EXIT_TO_XEN for consistency. No functional change. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> --- xen/arch/x86/include/asm/spec_ctrl_asm.h | 8 +------- xen/arch/x86/x86_64/entry.S | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/xen/arch/x86/include/asm/spec_ctrl_asm.h b/xen/arch/x86/include/asm/spec_ctrl_asm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/spec_ctrl_asm.h +++ b/xen/arch/x86/include/asm/spec_ctrl_asm.h @@ -XXX,XX +XXX,XX @@ * - SPEC_CTRL_ENTRY_FROM_PV * - SPEC_CTRL_ENTRY_FROM_INTR * - SPEC_CTRL_ENTRY_FROM_INTR_IST - * - SPEC_CTRL_EXIT_TO_XEN_IST * - SPEC_CTRL_EXIT_TO_XEN * - SPEC_CTRL_EXIT_TO_PV * @@ -XXX,XX +XXX,XX @@ ALTERNATIVE "", __stringify(DO_SPEC_CTRL_ENTRY maybexen=1), \ X86_FEATURE_SC_MSR_PV -/* Use when exiting to Xen context. */ -#define SPEC_CTRL_EXIT_TO_XEN \ - ALTERNATIVE "", \ - DO_SPEC_CTRL_EXIT_TO_XEN, X86_FEATURE_SC_MSR_PV - /* Use when exiting to PV guest context. */ #define SPEC_CTRL_EXIT_TO_PV \ ALTERNATIVE "", \ @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): .endm /* Use when exiting to Xen in IST context. */ -.macro SPEC_CTRL_EXIT_TO_XEN_IST +.macro SPEC_CTRL_EXIT_TO_XEN /* * Requires %rbx=stack_end * Clobbers %rax, %rcx, %rdx diff --git a/xen/arch/x86/x86_64/entry.S b/xen/arch/x86/x86_64/entry.S index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/x86_64/entry.S +++ b/xen/arch/x86/x86_64/entry.S @@ -XXX,XX +XXX,XX @@ UNLIKELY_START(ne, exit_cr3) UNLIKELY_END(exit_cr3) /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */ - SPEC_CTRL_EXIT_TO_XEN_IST /* Req: %rbx=end, Clob: acd */ + SPEC_CTRL_EXIT_TO_XEN /* Req: %rbx=end, Clob: acd */ RESTORE_ALL adj=8 iretq base-commit: 6aa25c32180ab59081c73bae4c568367d9133a1f -- 2.30.2 With the SPEC_CTRL_EXIT_TO_XEN{,_IST} confusion fixed, it's now obvious that there's only a single EXIT_TO_XEN path. Fold DO_SPEC_CTRL_EXIT_TO_XEN into SPEC_CTRL_EXIT_TO_XEN to simplify further fixes. When merging labels, switch the name to .L\@_skip_sc_msr as "skip" on its own is going to be too generic shortly. No functional change. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> --- xen/arch/x86/include/asm/spec_ctrl_asm.h | 40 ++++++++++-------------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/xen/arch/x86/include/asm/spec_ctrl_asm.h b/xen/arch/x86/include/asm/spec_ctrl_asm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/spec_ctrl_asm.h +++ b/xen/arch/x86/include/asm/spec_ctrl_asm.h @@ -XXX,XX +XXX,XX @@ wrmsr .endm -.macro DO_SPEC_CTRL_EXIT_TO_XEN -/* - * Requires %rbx=stack_end - * Clobbers %rax, %rcx, %rdx - * - * When returning to Xen context, look to see whether SPEC_CTRL shadowing is - * in effect, and reload the shadow value. This covers race conditions which - * exist with an NMI/MCE/etc hitting late in the return-to-guest path. - */ - xor %edx, %edx - - testb $SCF_use_shadow, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx) - jz .L\@_skip - - mov STACK_CPUINFO_FIELD(shadow_spec_ctrl)(%rbx), %eax - mov $MSR_SPEC_CTRL, %ecx - wrmsr - -.L\@_skip: -.endm - .macro DO_SPEC_CTRL_EXIT_TO_GUEST /* * Requires %eax=spec_ctrl, %rsp=regs/cpuinfo @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): * Clobbers %rax, %rcx, %rdx */ testb $SCF_ist_sc_msr, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx) - jz .L\@_skip + jz .L\@_skip_sc_msr - DO_SPEC_CTRL_EXIT_TO_XEN + /* + * When returning to Xen context, look to see whether SPEC_CTRL shadowing + * is in effect, and reload the shadow value. This covers race conditions + * which exist with an NMI/MCE/etc hitting late in the return-to-guest + * path. + */ + xor %edx, %edx -.L\@_skip: + testb $SCF_use_shadow, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx) + jz .L\@_skip_sc_msr + + mov STACK_CPUINFO_FIELD(shadow_spec_ctrl)(%rbx), %eax + mov $MSR_SPEC_CTRL, %ecx + wrmsr + +.L\@_skip_sc_msr: .endm #endif /* __ASSEMBLY__ */ -- 2.30.2 These have grown more complex over time, with some already having been converted. Provide full Requires/Clobbers comments, otherwise missing at this level of indirection. No functional change. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> --- xen/arch/x86/include/asm/spec_ctrl_asm.h | 37 ++++++++++++++++++------ 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/xen/arch/x86/include/asm/spec_ctrl_asm.h b/xen/arch/x86/include/asm/spec_ctrl_asm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/spec_ctrl_asm.h +++ b/xen/arch/x86/include/asm/spec_ctrl_asm.h @@ -XXX,XX +XXX,XX @@ .endm /* Use after an entry from PV context (syscall/sysenter/int80/int82/etc). */ -#define SPEC_CTRL_ENTRY_FROM_PV \ +.macro SPEC_CTRL_ENTRY_FROM_PV +/* + * Requires %rsp=regs/cpuinfo, %rdx=0 + * Clobbers %rax, %rcx, %rdx + */ ALTERNATIVE "", __stringify(DO_SPEC_CTRL_COND_IBPB maybexen=0), \ - X86_FEATURE_IBPB_ENTRY_PV; \ - ALTERNATIVE "", DO_OVERWRITE_RSB, X86_FEATURE_SC_RSB_PV; \ + X86_FEATURE_IBPB_ENTRY_PV + + ALTERNATIVE "", DO_OVERWRITE_RSB, X86_FEATURE_SC_RSB_PV + ALTERNATIVE "", __stringify(DO_SPEC_CTRL_ENTRY maybexen=0), \ X86_FEATURE_SC_MSR_PV +.endm /* Use in interrupt/exception context. May interrupt Xen or PV context. */ -#define SPEC_CTRL_ENTRY_FROM_INTR \ +.macro SPEC_CTRL_ENTRY_FROM_INTR +/* + * Requires %rsp=regs, %r14=stack_end, %rdx=0 + * Clobbers %rax, %rcx, %rdx + */ ALTERNATIVE "", __stringify(DO_SPEC_CTRL_COND_IBPB maybexen=1), \ - X86_FEATURE_IBPB_ENTRY_PV; \ - ALTERNATIVE "", DO_OVERWRITE_RSB, X86_FEATURE_SC_RSB_PV; \ + X86_FEATURE_IBPB_ENTRY_PV + + ALTERNATIVE "", DO_OVERWRITE_RSB, X86_FEATURE_SC_RSB_PV + ALTERNATIVE "", __stringify(DO_SPEC_CTRL_ENTRY maybexen=1), \ X86_FEATURE_SC_MSR_PV +.endm /* Use when exiting to PV guest context. */ -#define SPEC_CTRL_EXIT_TO_PV \ - ALTERNATIVE "", \ - DO_SPEC_CTRL_EXIT_TO_GUEST, X86_FEATURE_SC_MSR_PV; \ +.macro SPEC_CTRL_EXIT_TO_PV +/* + * Requires %rax=spec_ctrl, %rsp=regs/info + * Clobbers %rcx, %rdx + */ + ALTERNATIVE "", DO_SPEC_CTRL_EXIT_TO_GUEST, X86_FEATURE_SC_MSR_PV + DO_SPEC_CTRL_COND_VERW +.endm /* * Use in IST interrupt/exception context. May interrupt Xen or PV context. -- 2.30.2 ... to better explain how they're used. Doing so highlights that SPEC_CTRL_EXIT_TO_XEN is missing a VERW flush for the corner case when e.g. an NMI hits late in an exit-to-guest path. Leave a TODO, which will be addressed in subsequent patches which arrange for DO_COND_VERW to be safe within SPEC_CTRL_EXIT_TO_XEN. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> This was decided not to be XSA-worthy, as guests can't usefully control when IST events occur. --- xen/arch/x86/include/asm/spec_ctrl_asm.h | 35 ++++++++++++++++++++---- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/xen/arch/x86/include/asm/spec_ctrl_asm.h b/xen/arch/x86/include/asm/spec_ctrl_asm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/spec_ctrl_asm.h +++ b/xen/arch/x86/include/asm/spec_ctrl_asm.h @@ -XXX,XX +XXX,XX @@ wrmsr .endm -/* Use after an entry from PV context (syscall/sysenter/int80/int82/etc). */ +/* + * Used after a synchronous entry from PV context. SYSCALL, SYSENTER, INT, + * etc. Will always interrupt a guest speculation context. + */ .macro SPEC_CTRL_ENTRY_FROM_PV /* * Requires %rsp=regs/cpuinfo, %rdx=0 @@ -XXX,XX +XXX,XX @@ X86_FEATURE_SC_MSR_PV .endm -/* Use in interrupt/exception context. May interrupt Xen or PV context. */ +/* + * Used after a synchronous interrupt or exception. May interrupt Xen or PV + * context, but will not interrupt Xen with a guest speculation context, + * outside of fatal error cases. + */ .macro SPEC_CTRL_ENTRY_FROM_INTR /* * Requires %rsp=regs, %r14=stack_end, %rdx=0 @@ -XXX,XX +XXX,XX @@ X86_FEATURE_SC_MSR_PV .endm -/* Use when exiting to PV guest context. */ +/* + * Used when exiting from any entry context, back to PV context. This + * includes from an IST entry which moved onto the primary stack. + */ .macro SPEC_CTRL_EXIT_TO_PV /* * Requires %rax=spec_ctrl, %rsp=regs/info @@ -XXX,XX +XXX,XX @@ .endm /* - * Use in IST interrupt/exception context. May interrupt Xen or PV context. + * Used after an IST entry (i.e. needs special care, consider to be fully + * asynchronous with finding sane state). May interrupt PV or Xen context, + * including other SPEC_CTRL_{ENTRY,EXIT}_* regions with unsanitised state. + * + * An IST entry which interrupts PV context moves onto the primary stack and + * leaves via SPEC_CTRL_EXIT_TO_PV, *not* SPEC_CTRL_EXIT_TO_XEN. */ .macro SPEC_CTRL_ENTRY_FROM_INTR_IST /* @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): UNLIKELY_END(\@_serialise) .endm -/* Use when exiting to Xen in IST context. */ +/* + * Use when exiting from any entry context, back to Xen context. This + * includes returning to other SPEC_CTRL_{ENTRY,EXIT}_* regions with + * unsanitised state. + * + * Because we might have interrupted Xen beyond SPEC_CTRL_EXIT_TO_$GUEST, we + * must treat this as if it were an EXIT_TO_$GUEST case too. + */ .macro SPEC_CTRL_EXIT_TO_XEN /* * Requires %rbx=stack_end @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): wrmsr .L\@_skip_sc_msr: + + /* TODO VERW */ + .endm #endif /* __ASSEMBLY__ */ -- 2.30.2 All other SPEC_CTRL_{ENTRY,EXIT}_* helpers hold stack_end in %r14. Adjust it for consistency, freeing up %rbx to be used differently. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> --- xen/arch/x86/include/asm/spec_ctrl_asm.h | 8 ++++---- xen/arch/x86/x86_64/entry.S | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/xen/arch/x86/include/asm/spec_ctrl_asm.h b/xen/arch/x86/include/asm/spec_ctrl_asm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/spec_ctrl_asm.h +++ b/xen/arch/x86/include/asm/spec_ctrl_asm.h @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): */ .macro SPEC_CTRL_EXIT_TO_XEN /* - * Requires %rbx=stack_end + * Requires %r14=stack_end * Clobbers %rax, %rcx, %rdx */ - testb $SCF_ist_sc_msr, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx) + testb $SCF_ist_sc_msr, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14) jz .L\@_skip_sc_msr /* @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): */ xor %edx, %edx - testb $SCF_use_shadow, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx) + testb $SCF_use_shadow, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14) jz .L\@_skip_sc_msr - mov STACK_CPUINFO_FIELD(shadow_spec_ctrl)(%rbx), %eax + mov STACK_CPUINFO_FIELD(shadow_spec_ctrl)(%r14), %eax mov $MSR_SPEC_CTRL, %ecx wrmsr diff --git a/xen/arch/x86/x86_64/entry.S b/xen/arch/x86/x86_64/entry.S index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/x86_64/entry.S +++ b/xen/arch/x86/x86_64/entry.S @@ -XXX,XX +XXX,XX @@ restore_all_xen: * Check whether we need to switch to the per-CPU page tables, in * case we return to late PV exit code (from an NMI or #MC). */ - GET_STACK_END(bx) - cmpb $0, STACK_CPUINFO_FIELD(use_pv_cr3)(%rbx) + GET_STACK_END(14) + cmpb $0, STACK_CPUINFO_FIELD(use_pv_cr3)(%r14) UNLIKELY_START(ne, exit_cr3) - mov STACK_CPUINFO_FIELD(pv_cr3)(%rbx), %rax + mov STACK_CPUINFO_FIELD(pv_cr3)(%r14), %rax mov %rax, %cr3 UNLIKELY_END(exit_cr3) /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */ - SPEC_CTRL_EXIT_TO_XEN /* Req: %rbx=end, Clob: acd */ + SPEC_CTRL_EXIT_TO_XEN /* Req: %r14=end, Clob: acd */ RESTORE_ALL adj=8 iretq -- 2.30.2 Use %r12 to hold an ist_exit boolean. This register is zero elsewhere in the entry/exit asm, so it only needs setting in the IST path. As this is subtle and fragile, add check_ist_exit() to be used in debugging builds to cross-check that the ist_exit boolean matches the entry vector. Write check_ist_exit() it in C, because it's debug only and the logic more complicated than I care about maintaining in asm. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> --- xen/arch/x86/traps.c | 13 +++++++++++++ xen/arch/x86/x86_64/compat/entry.S | 9 ++++++++- xen/arch/x86/x86_64/entry.S | 23 ++++++++++++++++++++--- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/traps.c +++ b/xen/arch/x86/traps.c @@ -XXX,XX +XXX,XX @@ void asm_domain_crash_synchronous(unsigned long addr) do_softirq(); } +#ifdef CONFIG_DEBUG +void check_ist_exit(const struct cpu_user_regs *regs, bool ist_exit) +{ + const unsigned int ist_mask = + (1U << X86_EXC_NMI) | (1U << X86_EXC_DB) | + (1U << X86_EXC_DF) | (1U << X86_EXC_MC); + uint8_t ev = regs->entry_vector; + bool is_ist = (ev < X86_EXC_NUM) && ((1U << ev) & ist_mask); + + ASSERT(is_ist == ist_exit); +} +#endif + /* * Local variables: * mode: C diff --git a/xen/arch/x86/x86_64/compat/entry.S b/xen/arch/x86/x86_64/compat/entry.S index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/x86_64/compat/entry.S +++ b/xen/arch/x86/x86_64/compat/entry.S @@ -XXX,XX +XXX,XX @@ compat_process_trap: call compat_create_bounce_frame jmp compat_test_all_events -/* %rbx: struct vcpu, interrupts disabled */ +/* %rbx: struct vcpu, %r12: ist_exit, interrupts disabled */ ENTRY(compat_restore_all_guest) + +#ifdef CONFIG_DEBUG + mov %rsp, %rdi + mov %r12, %rsi + call check_ist_exit +#endif + ASSERT_INTERRUPTS_DISABLED mov $~(X86_EFLAGS_IOPL | X86_EFLAGS_VM), %r11d and UREGS_eflags(%rsp),%r11d diff --git a/xen/arch/x86/x86_64/entry.S b/xen/arch/x86/x86_64/entry.S index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/x86_64/entry.S +++ b/xen/arch/x86/x86_64/entry.S @@ -XXX,XX +XXX,XX @@ process_trap: .section .text.entry, "ax", @progbits -/* %rbx: struct vcpu, interrupts disabled */ +/* %rbx: struct vcpu, %r12: ist_exit, interrupts disabled */ restore_all_guest: - ASSERT_INTERRUPTS_DISABLED +#ifdef CONFIG_DEBUG + mov %rsp, %rdi + mov %r12, %rsi + call check_ist_exit +#endif + + ASSERT_INTERRUPTS_DISABLED /* Stash guest SPEC_CTRL value while we can read struct vcpu. */ mov VCPU_arch_msrs(%rbx), %rdx mov VCPUMSR_spec_ctrl_raw(%rdx), %r15d @@ -XXX,XX +XXX,XX @@ ENTRY(early_page_fault) .section .text.entry, "ax", @progbits ALIGN -/* No special register assumptions. */ +/* %r12=ist_exit */ restore_all_xen: + +#ifdef CONFIG_DEBUG + mov %rsp, %rdi + mov %r12, %rsi + call check_ist_exit +#endif + /* * Check whether we need to switch to the per-CPU page tables, in * case we return to late PV exit code (from an NMI or #MC). @@ -XXX,XX +XXX,XX @@ handle_ist_exception: .L_ist_dispatch_done: mov %r15, STACK_CPUINFO_FIELD(xen_cr3)(%r14) mov %bl, STACK_CPUINFO_FIELD(use_pv_cr3)(%r14) + + /* This is an IST exit */ + mov $1, %r12 + cmpb $X86_EXC_NMI, UREGS_entry_vector(%rsp) jne ret_from_intr -- 2.30.2 There is a corner case where e.g. an NMI hitting an exit-to-guest path after SPEC_CTRL_EXIT_TO_* would have run the entire NMI handler *after* the VERW flush to scrub potentially sensitive data from uarch buffers. In order to compensate, issue VERW when exiting to Xen from an IST entry. SPEC_CTRL_EXIT_TO_XEN already has two reads of spec_ctrl_flags off the stack, and we're about to add a third. Load the field into %ebx, and list the register as clobbered. %r12 has been arranged to be the ist_exit signal, so add this as an input dependency and use it to identify when to issue a VERW. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> --- xen/arch/x86/include/asm/spec_ctrl_asm.h | 20 +++++++++++++++----- xen/arch/x86/x86_64/entry.S | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/xen/arch/x86/include/asm/spec_ctrl_asm.h b/xen/arch/x86/include/asm/spec_ctrl_asm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/spec_ctrl_asm.h +++ b/xen/arch/x86/include/asm/spec_ctrl_asm.h @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): */ .macro SPEC_CTRL_EXIT_TO_XEN /* - * Requires %r14=stack_end - * Clobbers %rax, %rcx, %rdx + * Requires %r12=ist_exit, %r14=stack_end + * Clobbers %rax, %rbx, %rcx, %rdx */ - testb $SCF_ist_sc_msr, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14) + movzbl STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14), %ebx + + testb $SCF_ist_sc_msr, %bl jz .L\@_skip_sc_msr /* @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): */ xor %edx, %edx - testb $SCF_use_shadow, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14) + testb $SCF_use_shadow, %bl jz .L\@_skip_sc_msr mov STACK_CPUINFO_FIELD(shadow_spec_ctrl)(%r14), %eax @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): .L\@_skip_sc_msr: - /* TODO VERW */ + test %r12, %r12 + jz .L\@_skip_ist_exit + + /* Logically DO_SPEC_CTRL_COND_VERW but without the %rsp=cpuinfo dependency */ + testb $SCF_verw, %bl + jz .L\@_verw_skip + verw STACK_CPUINFO_FIELD(verw_sel)(%r14) +.L\@_verw_skip: +.L\@_skip_ist_exit: .endm #endif /* __ASSEMBLY__ */ diff --git a/xen/arch/x86/x86_64/entry.S b/xen/arch/x86/x86_64/entry.S index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/x86_64/entry.S +++ b/xen/arch/x86/x86_64/entry.S @@ -XXX,XX +XXX,XX @@ UNLIKELY_START(ne, exit_cr3) UNLIKELY_END(exit_cr3) /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */ - SPEC_CTRL_EXIT_TO_XEN /* Req: %r14=end, Clob: acd */ + SPEC_CTRL_EXIT_TO_XEN /* Req: %r12=ist_exit %r14=end, Clob: abcd */ RESTORE_ALL adj=8 iretq -- 2.30.2 In the Zen1 microarchitecure, there is one divider in the pipeline which services uops from both threads. In the case of #DE, the latched result from the previous DIV to execute will be forwarded speculatively. This is an interesting covert channel that allows two threads to communicate without any system calls. In also allows userspace to obtain the result of the most recent DIV instruction executed (even speculatively) in the core, which can be from a higher privilege context. Scrub the buffers in the divider unit by executing a non-faulting divide. This needs performing on the exit-to-guest paths, and ist_exit-to-Xen. This is XSA-439 / CVE-2023-20588. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> No embargo - this is already public. XSA paperwork to follow. --- docs/misc/xen-command-line.pandoc | 6 +++- xen/arch/x86/hvm/svm/entry.S | 1 + xen/arch/x86/include/asm/cpufeatures.h | 2 +- xen/arch/x86/include/asm/spec_ctrl_asm.h | 16 +++++++++ xen/arch/x86/spec_ctrl.c | 45 ++++++++++++++++++++++-- 5 files changed, 66 insertions(+), 4 deletions(-) diff --git a/docs/misc/xen-command-line.pandoc b/docs/misc/xen-command-line.pandoc index XXXXXXX..XXXXXXX 100644 --- a/docs/misc/xen-command-line.pandoc +++ b/docs/misc/xen-command-line.pandoc @@ -XXX,XX +XXX,XX @@ By default SSBD will be mitigated at runtime (i.e `ssbd=runtime`). > {msr-sc,rsb,md-clear,ibpb-entry}=<bool>|{pv,hvm}=<bool>, > bti-thunk=retpoline|lfence|jmp, {ibrs,ibpb,ssbd,psfd, > eager-fpu,l1d-flush,branch-harden,srb-lock, -> unpriv-mmio,gds-mit}=<bool> ]` +> unpriv-mmio,gds-mit,div-scrub}=<bool> ]` Controls for speculative execution sidechannel mitigations. By default, Xen will pick the most appropriate mitigations based on compiled in support, @@ -XXX,XX +XXX,XX @@ has elected not to lock the configuration, Xen will use GDS_CTRL to mitigate GDS with. Otherwise, Xen will mitigate by disabling AVX, which blocks the use of the AVX2 Gather instructions. +On all hardware, the `div-scrub=` option can be used to force or prevent Xen +from mitigating the DIV-leakage vulnerability. By default, Xen will mitigate +DIV-leakage on hardware believed to be vulnerable. + ### sync_console > `= <boolean>` diff --git a/xen/arch/x86/hvm/svm/entry.S b/xen/arch/x86/hvm/svm/entry.S index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/hvm/svm/entry.S +++ b/xen/arch/x86/hvm/svm/entry.S @@ -XXX,XX +XXX,XX @@ __UNLIKELY_END(nsvm_hap) 1: /* No Spectre v1 concerns. Execution will hit VMRUN imminently. */ .endm ALTERNATIVE "", svm_vmentry_spec_ctrl, X86_FEATURE_SC_MSR_HVM + ALTERNATIVE "", DO_SPEC_CTRL_DIV, X86_FEATURE_SC_DIV pop %r15 pop %r14 diff --git a/xen/arch/x86/include/asm/cpufeatures.h b/xen/arch/x86/include/asm/cpufeatures.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/cpufeatures.h +++ b/xen/arch/x86/include/asm/cpufeatures.h @@ -XXX,XX +XXX,XX @@ XEN_CPUFEATURE(SC_RSB_HVM, X86_SYNTH(19)) /* RSB overwrite needed for HVM XEN_CPUFEATURE(XEN_SELFSNOOP, X86_SYNTH(20)) /* SELFSNOOP gets used by Xen itself */ XEN_CPUFEATURE(SC_MSR_IDLE, X86_SYNTH(21)) /* Clear MSR_SPEC_CTRL on idle */ XEN_CPUFEATURE(XEN_LBR, X86_SYNTH(22)) /* Xen uses MSR_DEBUGCTL.LBR */ -/* Bits 23 unused. */ +XEN_CPUFEATURE(SC_DIV, X86_SYNTH(25)) /* DIV scrub needed */ XEN_CPUFEATURE(SC_RSB_IDLE, X86_SYNTH(24)) /* RSB overwrite needed for idle. */ XEN_CPUFEATURE(SC_VERW_IDLE, X86_SYNTH(25)) /* VERW used by Xen for idle */ XEN_CPUFEATURE(XEN_SHSTK, X86_SYNTH(26)) /* Xen uses CET Shadow Stacks */ diff --git a/xen/arch/x86/include/asm/spec_ctrl_asm.h b/xen/arch/x86/include/asm/spec_ctrl_asm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/spec_ctrl_asm.h +++ b/xen/arch/x86/include/asm/spec_ctrl_asm.h @@ -XXX,XX +XXX,XX @@ .L\@_verw_skip: .endm +.macro DO_SPEC_CTRL_DIV +/* + * Requires nothing + * Clobbers %rax + * + * Issue a DIV for its flushing side effect (Zen1 uarch specific). Any + * non-faulting DIV will do, and a byte DIV has least latency. + */ + mov $1, %eax + div %al +.endm + .macro DO_SPEC_CTRL_ENTRY maybexen:req /* * Requires %rsp=regs (also cpuinfo if !maybexen) @@ -XXX,XX +XXX,XX @@ ALTERNATIVE "", DO_SPEC_CTRL_EXIT_TO_GUEST, X86_FEATURE_SC_MSR_PV DO_SPEC_CTRL_COND_VERW + + ALTERNATIVE "", DO_SPEC_CTRL_DIV, X86_FEATURE_SC_DIV .endm /* @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): verw STACK_CPUINFO_FIELD(verw_sel)(%r14) .L\@_verw_skip: + ALTERNATIVE "", DO_SPEC_CTRL_DIV, X86_FEATURE_SC_DIV + .L\@_skip_ist_exit: .endm diff --git a/xen/arch/x86/spec_ctrl.c b/xen/arch/x86/spec_ctrl.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/spec_ctrl.c +++ b/xen/arch/x86/spec_ctrl.c @@ -XXX,XX +XXX,XX @@ static int8_t __initdata opt_srb_lock = -1; static bool __initdata opt_unpriv_mmio; static bool __ro_after_init opt_fb_clear_mmio; static int8_t __initdata opt_gds_mit = -1; +static int8_t __initdata opt_div_scrub = -1; static int __init cf_check parse_spec_ctrl(const char *s) { @@ -XXX,XX +XXX,XX @@ static int __init cf_check parse_spec_ctrl(const char *s) opt_srb_lock = 0; opt_unpriv_mmio = false; opt_gds_mit = 0; + opt_div_scrub = 0; } else if ( val > 0 ) rc = -EINVAL; @@ -XXX,XX +XXX,XX @@ static int __init cf_check parse_spec_ctrl(const char *s) opt_unpriv_mmio = val; else if ( (val = parse_boolean("gds-mit", s, ss)) >= 0 ) opt_gds_mit = val; + else if ( (val = parse_boolean("div-scrub", s, ss)) >= 0 ) + opt_div_scrub = val; else rc = -EINVAL; @@ -XXX,XX +XXX,XX @@ static void __init print_details(enum ind_thunk thunk) "\n"); /* Settings for Xen's protection, irrespective of guests. */ - printk(" Xen settings: BTI-Thunk %s, SPEC_CTRL: %s%s%s%s%s, Other:%s%s%s%s%s\n", + printk(" Xen settings: BTI-Thunk %s, SPEC_CTRL: %s%s%s%s%s, Other:%s%s%s%s%s%s\n", thunk == THUNK_NONE ? "N/A" : thunk == THUNK_RETPOLINE ? "RETPOLINE" : thunk == THUNK_LFENCE ? "LFENCE" : @@ -XXX,XX +XXX,XX @@ static void __init print_details(enum ind_thunk thunk) opt_l1d_flush ? " L1D_FLUSH" : "", opt_md_clear_pv || opt_md_clear_hvm || opt_fb_clear_mmio ? " VERW" : "", - opt_branch_harden ? " BRANCH_HARDEN" : ""); + opt_branch_harden ? " BRANCH_HARDEN" : "", + opt_div_scrub ? " DIV" : ""); /* L1TF diagnostics, printed if vulnerable or PV shadowing is in use. */ if ( cpu_has_bug_l1tf || opt_pv_l1tf_hwdom || opt_pv_l1tf_domu ) @@ -XXX,XX +XXX,XX @@ static void __init srso_calculations(bool hw_smt_enabled) setup_force_cpu_cap(X86_FEATURE_SRSO_NO); } +/* + * Div leakage is specific to the AMD Zen1 microarchitecure. Use STIBP as a + * heuristic to select between Zen1 and Zen2 uarches. + */ +static bool __init has_div_vuln(void) +{ + if ( !(boot_cpu_data.x86_vendor & + (X86_VENDOR_AMD | X86_VENDOR_HYGON)) ) + return false; + + if ( (boot_cpu_data.x86 != 0x17 && boot_cpu_data.x86 != 0x18) || + !boot_cpu_has(X86_FEATURE_AMD_STIBP) ) + return false; + + return true; +} + +static void __init div_calculations(bool hw_smt_enabled) +{ + bool cpu_bug_div = has_div_vuln(); + + if ( opt_div_scrub == -1 ) + opt_div_scrub = cpu_bug_div; + + if ( opt_div_scrub ) + setup_force_cpu_cap(X86_FEATURE_SC_DIV); + + if ( opt_smt == -1 && cpu_bug_div && hw_smt_enabled ) + warning_add( + "Booted on leaky-DIV hardware with SMT/Hyperthreading\n" + "enabled. Please assess your configuration and choose an\n" + "explicit 'smt=<bool>' setting. See XSA-439.\n"); +} + static void __init ibpb_calculations(void) { bool def_ibpb_entry = false; @@ -XXX,XX +XXX,XX @@ void __init init_speculation_mitigations(void) ibpb_calculations(); + div_calculations(hw_smt_enabled); + /* Check whether Eager FPU should be enabled by default. */ if ( opt_eager_fpu == -1 ) opt_eager_fpu = should_use_eager_fpu(); -- 2.30.2 Patch 9 is the XSA-439 fix for the AMD DIV issue, disclosed insufficiently ahead of August 8th for us to prepare a fix for the embargo. Patches 1 thru 8 are prerequisites, identified while trying to write patch 9. All 9 patches are for all security trees. Andrew Cooper (9): x86/spec-ctrl: Fix confusion between SPEC_CTRL_EXIT_TO_XEN{,_IST} x86/spec-ctrl: Fold DO_SPEC_CTRL_EXIT_TO_XEN into it's single user x86/spec-ctrl: Turn the remaining SPEC_CTRL_{ENTRY,EXIT}_* into asm macros x86/spec-ctrl: Improve all SPEC_CTRL_{ENTER,EXIT}_* comments x86/entry: Adjust restore_all_xen to hold stack_end in %r14 x86/entry: Track the IST-ness of an entry for the exit paths x86/spec-ctrl: Issue VERW during IST exit to Xen x86/amd: Introduce is_zen{1,2}_uarch() predicates x86/spec-ctrl: Mitigate the Zen1 DIV leakge docs/misc/xen-command-line.pandoc | 6 +- xen/arch/x86/cpu/amd.c | 18 +-- xen/arch/x86/hvm/svm/entry.S | 1 + xen/arch/x86/include/asm/amd.h | 11 ++ xen/arch/x86/include/asm/cpufeatures.h | 2 +- xen/arch/x86/include/asm/spec_ctrl_asm.h | 152 ++++++++++++++++------- xen/arch/x86/spec_ctrl.c | 49 +++++++- xen/arch/x86/traps.c | 13 ++ xen/arch/x86/x86_64/compat/entry.S | 9 +- xen/arch/x86/x86_64/entry.S | 30 ++++- 10 files changed, 220 insertions(+), 71 deletions(-) -- 2.30.2 c/s 3fffaf9c13e9 ("x86/entry: Avoid using alternatives in NMI/#MC paths") dropped the only user, leaving behind the (incorrect) implication that Xen had split exit paths. Delete the unused SPEC_CTRL_EXIT_TO_XEN and rename SPEC_CTRL_EXIT_TO_XEN_IST to SPEC_CTRL_EXIT_TO_XEN for consistency. No functional change. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> Reviewed-by: Jan Beulich <jbeulich@suse.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> v2: * Tweak comment. --- xen/arch/x86/include/asm/spec_ctrl_asm.h | 10 ++-------- xen/arch/x86/x86_64/entry.S | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/xen/arch/x86/include/asm/spec_ctrl_asm.h b/xen/arch/x86/include/asm/spec_ctrl_asm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/spec_ctrl_asm.h +++ b/xen/arch/x86/include/asm/spec_ctrl_asm.h @@ -XXX,XX +XXX,XX @@ * - SPEC_CTRL_ENTRY_FROM_PV * - SPEC_CTRL_ENTRY_FROM_INTR * - SPEC_CTRL_ENTRY_FROM_INTR_IST - * - SPEC_CTRL_EXIT_TO_XEN_IST * - SPEC_CTRL_EXIT_TO_XEN * - SPEC_CTRL_EXIT_TO_PV * @@ -XXX,XX +XXX,XX @@ ALTERNATIVE "", __stringify(DO_SPEC_CTRL_ENTRY maybexen=1), \ X86_FEATURE_SC_MSR_PV -/* Use when exiting to Xen context. */ -#define SPEC_CTRL_EXIT_TO_XEN \ - ALTERNATIVE "", \ - DO_SPEC_CTRL_EXIT_TO_XEN, X86_FEATURE_SC_MSR_PV - /* Use when exiting to PV guest context. */ #define SPEC_CTRL_EXIT_TO_PV \ ALTERNATIVE "", \ @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): UNLIKELY_END(\@_serialise) .endm -/* Use when exiting to Xen in IST context. */ -.macro SPEC_CTRL_EXIT_TO_XEN_IST +/* Use when exiting to Xen context. */ +.macro SPEC_CTRL_EXIT_TO_XEN /* * Requires %rbx=stack_end * Clobbers %rax, %rcx, %rdx diff --git a/xen/arch/x86/x86_64/entry.S b/xen/arch/x86/x86_64/entry.S index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/x86_64/entry.S +++ b/xen/arch/x86/x86_64/entry.S @@ -XXX,XX +XXX,XX @@ UNLIKELY_START(ne, exit_cr3) UNLIKELY_END(exit_cr3) /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */ - SPEC_CTRL_EXIT_TO_XEN_IST /* Req: %rbx=end, Clob: acd */ + SPEC_CTRL_EXIT_TO_XEN /* Req: %rbx=end, Clob: acd */ RESTORE_ALL adj=8 iretq -- 2.30.2 With the SPEC_CTRL_EXIT_TO_XEN{,_IST} confusion fixed, it's now obvious that there's only a single EXIT_TO_XEN path. Fold DO_SPEC_CTRL_EXIT_TO_XEN into SPEC_CTRL_EXIT_TO_XEN to simplify further fixes. When merging labels, switch the name to .L\@_skip_sc_msr as "skip" on its own is going to be too generic shortly. No functional change. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> Reviewed-by: Jan Beulich <jbeulich@suse.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> --- xen/arch/x86/include/asm/spec_ctrl_asm.h | 40 ++++++++++-------------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/xen/arch/x86/include/asm/spec_ctrl_asm.h b/xen/arch/x86/include/asm/spec_ctrl_asm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/spec_ctrl_asm.h +++ b/xen/arch/x86/include/asm/spec_ctrl_asm.h @@ -XXX,XX +XXX,XX @@ wrmsr .endm -.macro DO_SPEC_CTRL_EXIT_TO_XEN -/* - * Requires %rbx=stack_end - * Clobbers %rax, %rcx, %rdx - * - * When returning to Xen context, look to see whether SPEC_CTRL shadowing is - * in effect, and reload the shadow value. This covers race conditions which - * exist with an NMI/MCE/etc hitting late in the return-to-guest path. - */ - xor %edx, %edx - - testb $SCF_use_shadow, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx) - jz .L\@_skip - - mov STACK_CPUINFO_FIELD(shadow_spec_ctrl)(%rbx), %eax - mov $MSR_SPEC_CTRL, %ecx - wrmsr - -.L\@_skip: -.endm - .macro DO_SPEC_CTRL_EXIT_TO_GUEST /* * Requires %eax=spec_ctrl, %rsp=regs/cpuinfo @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): * Clobbers %rax, %rcx, %rdx */ testb $SCF_ist_sc_msr, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx) - jz .L\@_skip + jz .L\@_skip_sc_msr - DO_SPEC_CTRL_EXIT_TO_XEN + /* + * When returning to Xen context, look to see whether SPEC_CTRL shadowing + * is in effect, and reload the shadow value. This covers race conditions + * which exist with an NMI/MCE/etc hitting late in the return-to-guest + * path. + */ + xor %edx, %edx -.L\@_skip: + testb $SCF_use_shadow, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx) + jz .L\@_skip_sc_msr + + mov STACK_CPUINFO_FIELD(shadow_spec_ctrl)(%rbx), %eax + mov $MSR_SPEC_CTRL, %ecx + wrmsr + +.L\@_skip_sc_msr: .endm #endif /* __ASSEMBLY__ */ -- 2.30.2 These have grown more complex over time, with some already having been converted. Provide full Requires/Clobbers comments, otherwise missing at this level of indirection. No functional change. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> Reviewed-by: Jan Beulich <jbeulich@suse.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> --- xen/arch/x86/include/asm/spec_ctrl_asm.h | 37 ++++++++++++++++++------ 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/xen/arch/x86/include/asm/spec_ctrl_asm.h b/xen/arch/x86/include/asm/spec_ctrl_asm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/spec_ctrl_asm.h +++ b/xen/arch/x86/include/asm/spec_ctrl_asm.h @@ -XXX,XX +XXX,XX @@ .endm /* Use after an entry from PV context (syscall/sysenter/int80/int82/etc). */ -#define SPEC_CTRL_ENTRY_FROM_PV \ +.macro SPEC_CTRL_ENTRY_FROM_PV +/* + * Requires %rsp=regs/cpuinfo, %rdx=0 + * Clobbers %rax, %rcx, %rdx + */ ALTERNATIVE "", __stringify(DO_SPEC_CTRL_COND_IBPB maybexen=0), \ - X86_FEATURE_IBPB_ENTRY_PV; \ - ALTERNATIVE "", DO_OVERWRITE_RSB, X86_FEATURE_SC_RSB_PV; \ + X86_FEATURE_IBPB_ENTRY_PV + + ALTERNATIVE "", DO_OVERWRITE_RSB, X86_FEATURE_SC_RSB_PV + ALTERNATIVE "", __stringify(DO_SPEC_CTRL_ENTRY maybexen=0), \ X86_FEATURE_SC_MSR_PV +.endm /* Use in interrupt/exception context. May interrupt Xen or PV context. */ -#define SPEC_CTRL_ENTRY_FROM_INTR \ +.macro SPEC_CTRL_ENTRY_FROM_INTR +/* + * Requires %rsp=regs, %r14=stack_end, %rdx=0 + * Clobbers %rax, %rcx, %rdx + */ ALTERNATIVE "", __stringify(DO_SPEC_CTRL_COND_IBPB maybexen=1), \ - X86_FEATURE_IBPB_ENTRY_PV; \ - ALTERNATIVE "", DO_OVERWRITE_RSB, X86_FEATURE_SC_RSB_PV; \ + X86_FEATURE_IBPB_ENTRY_PV + + ALTERNATIVE "", DO_OVERWRITE_RSB, X86_FEATURE_SC_RSB_PV + ALTERNATIVE "", __stringify(DO_SPEC_CTRL_ENTRY maybexen=1), \ X86_FEATURE_SC_MSR_PV +.endm /* Use when exiting to PV guest context. */ -#define SPEC_CTRL_EXIT_TO_PV \ - ALTERNATIVE "", \ - DO_SPEC_CTRL_EXIT_TO_GUEST, X86_FEATURE_SC_MSR_PV; \ +.macro SPEC_CTRL_EXIT_TO_PV +/* + * Requires %rax=spec_ctrl, %rsp=regs/info + * Clobbers %rcx, %rdx + */ + ALTERNATIVE "", DO_SPEC_CTRL_EXIT_TO_GUEST, X86_FEATURE_SC_MSR_PV + DO_SPEC_CTRL_COND_VERW +.endm /* * Use in IST interrupt/exception context. May interrupt Xen or PV context. -- 2.30.2 ... to better explain how they're used. Doing so highlights that SPEC_CTRL_EXIT_TO_XEN is missing a VERW flush for the corner case when e.g. an NMI hits late in an exit-to-guest path. Leave a TODO, which will be addressed in subsequent patches which arrange for DO_COND_VERW to be safe within SPEC_CTRL_EXIT_TO_XEN. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> This was decided not to be XSA-worthy, as guests can't usefully control when IST events occur. v2: * Rewrite. --- xen/arch/x86/include/asm/spec_ctrl_asm.h | 36 ++++++++++++++++++++---- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/xen/arch/x86/include/asm/spec_ctrl_asm.h b/xen/arch/x86/include/asm/spec_ctrl_asm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/spec_ctrl_asm.h +++ b/xen/arch/x86/include/asm/spec_ctrl_asm.h @@ -XXX,XX +XXX,XX @@ wrmsr .endm -/* Use after an entry from PV context (syscall/sysenter/int80/int82/etc). */ +/* + * Used after an entry from PV context: SYSCALL, SYSENTER, INT, + * etc. There is always a guest speculation state in context. + */ .macro SPEC_CTRL_ENTRY_FROM_PV /* * Requires %rsp=regs/cpuinfo, %rdx=0 @@ -XXX,XX +XXX,XX @@ X86_FEATURE_SC_MSR_PV .endm -/* Use in interrupt/exception context. May interrupt Xen or PV context. */ +/* + * Used after an exception or maskable interrupt, hitting Xen or PV context. + * There will either be a guest speculation context, or (baring fatal + * exceptions) a well-formed Xen speculation context. + */ .macro SPEC_CTRL_ENTRY_FROM_INTR /* * Requires %rsp=regs, %r14=stack_end, %rdx=0 @@ -XXX,XX +XXX,XX @@ X86_FEATURE_SC_MSR_PV .endm -/* Use when exiting to PV guest context. */ +/* + * Used when exiting from any entry context, back to PV context. This + * includes from an IST entry which moved onto the primary stack. + */ .macro SPEC_CTRL_EXIT_TO_PV /* * Requires %rax=spec_ctrl, %rsp=regs/info @@ -XXX,XX +XXX,XX @@ .endm /* - * Use in IST interrupt/exception context. May interrupt Xen or PV context. + * Used after an IST entry hitting Xen or PV context. Special care is needed, + * because when hitting Xen context, there may not a well-formed speculation + * context. (i.e. it can hit in the middle of SPEC_CTRL_{ENTRY,EXIT}_* + * regions.) + * + * An IST entry which hits PV context moves onto the primary stack and leaves + * via SPEC_CTRL_EXIT_TO_PV, *not* SPEC_CTRL_EXIT_TO_XEN. */ .macro SPEC_CTRL_ENTRY_FROM_INTR_IST /* @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): UNLIKELY_END(\@_serialise) .endm -/* Use when exiting to Xen context. */ +/* + * Use when exiting from any entry context, back to Xen context. This + * includes returning to other SPEC_CTRL_{ENTRY,EXIT}_* regions with an + * incomplete speculation context. + * + * Because we might have interrupted Xen beyond SPEC_CTRL_EXIT_TO_$GUEST, we + * need to treat this as if it were an EXIT_TO_$GUEST case too. + */ .macro SPEC_CTRL_EXIT_TO_XEN /* * Requires %rbx=stack_end @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): wrmsr .L\@_skip_sc_msr: + + /* TODO VERW */ + .endm #endif /* __ASSEMBLY__ */ -- 2.30.2 All other SPEC_CTRL_{ENTRY,EXIT}_* helpers hold stack_end in %r14. Adjust it for consistency. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> Reviewed-by: Jan Beulich <jbeulich@suse.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> --- xen/arch/x86/include/asm/spec_ctrl_asm.h | 8 ++++---- xen/arch/x86/x86_64/entry.S | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/xen/arch/x86/include/asm/spec_ctrl_asm.h b/xen/arch/x86/include/asm/spec_ctrl_asm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/spec_ctrl_asm.h +++ b/xen/arch/x86/include/asm/spec_ctrl_asm.h @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): */ .macro SPEC_CTRL_EXIT_TO_XEN /* - * Requires %rbx=stack_end + * Requires %r14=stack_end * Clobbers %rax, %rcx, %rdx */ - testb $SCF_ist_sc_msr, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx) + testb $SCF_ist_sc_msr, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14) jz .L\@_skip_sc_msr /* @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): */ xor %edx, %edx - testb $SCF_use_shadow, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%rbx) + testb $SCF_use_shadow, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14) jz .L\@_skip_sc_msr - mov STACK_CPUINFO_FIELD(shadow_spec_ctrl)(%rbx), %eax + mov STACK_CPUINFO_FIELD(shadow_spec_ctrl)(%r14), %eax mov $MSR_SPEC_CTRL, %ecx wrmsr diff --git a/xen/arch/x86/x86_64/entry.S b/xen/arch/x86/x86_64/entry.S index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/x86_64/entry.S +++ b/xen/arch/x86/x86_64/entry.S @@ -XXX,XX +XXX,XX @@ restore_all_xen: * Check whether we need to switch to the per-CPU page tables, in * case we return to late PV exit code (from an NMI or #MC). */ - GET_STACK_END(bx) - cmpb $0, STACK_CPUINFO_FIELD(use_pv_cr3)(%rbx) + GET_STACK_END(14) + cmpb $0, STACK_CPUINFO_FIELD(use_pv_cr3)(%r14) UNLIKELY_START(ne, exit_cr3) - mov STACK_CPUINFO_FIELD(pv_cr3)(%rbx), %rax + mov STACK_CPUINFO_FIELD(pv_cr3)(%r14), %rax mov %rax, %cr3 UNLIKELY_END(exit_cr3) /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */ - SPEC_CTRL_EXIT_TO_XEN /* Req: %rbx=end, Clob: acd */ + SPEC_CTRL_EXIT_TO_XEN /* Req: %r14=end, Clob: acd */ RESTORE_ALL adj=8 iretq -- 2.30.2 Use %r12 to hold an ist_exit boolean. This register is zero elsewhere in the entry/exit asm, so it only needs setting in the IST path. As this is subtle and fragile, add check_ist_exit() to be used in debugging builds to cross-check that the ist_exit boolean matches the entry vector. Write check_ist_exit() it in C, because it's debug only and the logic more complicated than I care to maintain in asm. For now, we only need to use this signal in the exit-to-Xen path, but some exit-to-guest paths happen in IST context too. Check the correctness in all exit paths to avoid the logic bitrotting. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> v2: * %r12 -> %r12d * Extend commit message * Tweak surrounding context --- xen/arch/x86/traps.c | 13 +++++++++++++ xen/arch/x86/x86_64/compat/entry.S | 9 ++++++++- xen/arch/x86/x86_64/entry.S | 22 ++++++++++++++++++++-- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/traps.c +++ b/xen/arch/x86/traps.c @@ -XXX,XX +XXX,XX @@ void asm_domain_crash_synchronous(unsigned long addr) do_softirq(); } +#ifdef CONFIG_DEBUG +void check_ist_exit(const struct cpu_user_regs *regs, bool ist_exit) +{ + const unsigned int ist_mask = + (1U << X86_EXC_NMI) | (1U << X86_EXC_DB) | + (1U << X86_EXC_DF) | (1U << X86_EXC_MC); + uint8_t ev = regs->entry_vector; + bool is_ist = (ev < X86_EXC_NUM) && ((1U << ev) & ist_mask); + + ASSERT(is_ist == ist_exit); +} +#endif + /* * Local variables: * mode: C diff --git a/xen/arch/x86/x86_64/compat/entry.S b/xen/arch/x86/x86_64/compat/entry.S index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/x86_64/compat/entry.S +++ b/xen/arch/x86/x86_64/compat/entry.S @@ -XXX,XX +XXX,XX @@ compat_process_trap: call compat_create_bounce_frame jmp compat_test_all_events -/* %rbx: struct vcpu, interrupts disabled */ +/* %rbx: struct vcpu, %r12: ist_exit, interrupts disabled */ ENTRY(compat_restore_all_guest) + +#ifdef CONFIG_DEBUG + mov %rsp, %rdi + mov %r12, %rsi + call check_ist_exit +#endif + ASSERT_INTERRUPTS_DISABLED mov $~(X86_EFLAGS_IOPL | X86_EFLAGS_VM), %r11d and UREGS_eflags(%rsp),%r11d diff --git a/xen/arch/x86/x86_64/entry.S b/xen/arch/x86/x86_64/entry.S index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/x86_64/entry.S +++ b/xen/arch/x86/x86_64/entry.S @@ -XXX,XX +XXX,XX @@ process_trap: .section .text.entry, "ax", @progbits -/* %rbx: struct vcpu, interrupts disabled */ +/* %rbx: struct vcpu, %r12: ist_exit, interrupts disabled */ restore_all_guest: + +#ifdef CONFIG_DEBUG + mov %rsp, %rdi + mov %r12, %rsi + call check_ist_exit +#endif + ASSERT_INTERRUPTS_DISABLED /* Stash guest SPEC_CTRL value while we can read struct vcpu. */ @@ -XXX,XX +XXX,XX @@ ENTRY(early_page_fault) .section .text.entry, "ax", @progbits ALIGN -/* No special register assumptions. */ +/* %r12=ist_exit */ restore_all_xen: + +#ifdef CONFIG_DEBUG + mov %rsp, %rdi + mov %r12, %rsi + call check_ist_exit +#endif + /* * Check whether we need to switch to the per-CPU page tables, in * case we return to late PV exit code (from an NMI or #MC). @@ -XXX,XX +XXX,XX @@ handle_ist_exception: .L_ist_dispatch_done: mov %r15, STACK_CPUINFO_FIELD(xen_cr3)(%r14) mov %bl, STACK_CPUINFO_FIELD(use_pv_cr3)(%r14) + + /* This is an IST exit */ + mov $1, %r12d + cmpb $X86_EXC_NMI, UREGS_entry_vector(%rsp) jne ret_from_intr -- 2.30.2 There is a corner case where e.g. an NMI hitting an exit-to-guest path after SPEC_CTRL_EXIT_TO_* would have run the entire NMI handler *after* the VERW flush to scrub potentially sensitive data from uarch buffers. In order to compensate, issue VERW when exiting to Xen from an IST entry. SPEC_CTRL_EXIT_TO_XEN already has two reads of spec_ctrl_flags off the stack, and we're about to add a third. Load the field into %ebx, and list the register as clobbered. %r12 has been arranged to be the ist_exit signal, so add this as an input dependency and use it to identify when to issue a VERW. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> Reviewed-by: Jan Beulich <jbeulich@suse.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> Note to reviewers: .L\@_skip_verw and .L\@_skip_ist_exit are separate to reduce the churn in the following patch. v2: * Rename .L\@_skip_verw --- xen/arch/x86/include/asm/spec_ctrl_asm.h | 20 +++++++++++++++----- xen/arch/x86/x86_64/entry.S | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/xen/arch/x86/include/asm/spec_ctrl_asm.h b/xen/arch/x86/include/asm/spec_ctrl_asm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/spec_ctrl_asm.h +++ b/xen/arch/x86/include/asm/spec_ctrl_asm.h @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): */ .macro SPEC_CTRL_EXIT_TO_XEN /* - * Requires %r14=stack_end - * Clobbers %rax, %rcx, %rdx + * Requires %r12=ist_exit, %r14=stack_end + * Clobbers %rax, %rbx, %rcx, %rdx */ - testb $SCF_ist_sc_msr, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14) + movzbl STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14), %ebx + + testb $SCF_ist_sc_msr, %bl jz .L\@_skip_sc_msr /* @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): */ xor %edx, %edx - testb $SCF_use_shadow, STACK_CPUINFO_FIELD(spec_ctrl_flags)(%r14) + testb $SCF_use_shadow, %bl jz .L\@_skip_sc_msr mov STACK_CPUINFO_FIELD(shadow_spec_ctrl)(%r14), %eax @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): .L\@_skip_sc_msr: - /* TODO VERW */ + test %r12, %r12 + jz .L\@_skip_ist_exit + + /* Logically DO_SPEC_CTRL_COND_VERW but without the %rsp=cpuinfo dependency */ + testb $SCF_verw, %bl + jz .L\@_skip_verw + verw STACK_CPUINFO_FIELD(verw_sel)(%r14) +.L\@_skip_verw: +.L\@_skip_ist_exit: .endm #endif /* __ASSEMBLY__ */ diff --git a/xen/arch/x86/x86_64/entry.S b/xen/arch/x86/x86_64/entry.S index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/x86_64/entry.S +++ b/xen/arch/x86/x86_64/entry.S @@ -XXX,XX +XXX,XX @@ UNLIKELY_START(ne, exit_cr3) UNLIKELY_END(exit_cr3) /* WARNING! `ret`, `call *`, `jmp *` not safe beyond this point. */ - SPEC_CTRL_EXIT_TO_XEN /* Req: %r14=end, Clob: acd */ + SPEC_CTRL_EXIT_TO_XEN /* Req: %r12=ist_exit %r14=end, Clob: abcd */ RESTORE_ALL adj=8 iretq -- 2.30.2 We already have 3 cases using STIBP as a Zen1/2 heuristic, and are about to introduce a 4th. Wrap the heuristic into a pair of predictes rather than opencoding it, and the explaination of the heursitic, at each usage site. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> v2: * New --- xen/arch/x86/cpu/amd.c | 18 ++++-------------- xen/arch/x86/include/asm/amd.h | 11 +++++++++++ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/xen/arch/x86/cpu/amd.c b/xen/arch/x86/cpu/amd.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/cpu/amd.c +++ b/xen/arch/x86/cpu/amd.c @@ -XXX,XX +XXX,XX @@ void amd_set_legacy_ssbd(bool enable) * non-branch instructions to be ignored. It is to be set unilaterally in * newer microcode. * - * This chickenbit is something unrelated on Zen1, and Zen1 vs Zen2 isn't a - * simple model number comparison, so use STIBP as a heuristic to separate the - * two uarches in Fam17h(AMD)/18h(Hygon). + * This chickenbit is something unrelated on Zen1. */ void amd_init_spectral_chicken(void) { uint64_t val, chickenbit = 1 << 1; - if (cpu_has_hypervisor || !boot_cpu_has(X86_FEATURE_AMD_STIBP)) + if (cpu_has_hypervisor || !is_zen2_uarch()) return; if (rdmsr_safe(MSR_AMD64_DE_CFG2, val) == 0 && !(val & chickenbit)) @@ -XXX,XX +XXX,XX @@ void amd_check_zenbleed(void) * With the Fam17h check above, most parts getting here are * Zen1. They're not affected. Assume Zen2 ones making it * here are affected regardless of microcode version. - * - * Zen1 vs Zen2 isn't a simple model number comparison, so use - * STIBP as a heuristic to distinguish. */ - if (!boot_cpu_has(X86_FEATURE_AMD_STIBP)) + if (is_zen1_uarch()) return; good_rev = ~0U; break; @@ -XXX,XX +XXX,XX @@ static int __init cf_check zen2_c6_errata_check(void) */ s_time_t delta; - /* - * Zen1 vs Zen2 isn't a simple model number comparison, so use STIBP as - * a heuristic to separate the two uarches in Fam17h. - */ - if (cpu_has_hypervisor || boot_cpu_data.x86 != 0x17 || - !boot_cpu_has(X86_FEATURE_AMD_STIBP)) + if (cpu_has_hypervisor || boot_cpu_data.x86 != 0x17 || !is_zen2_uarch()) return 0; /* diff --git a/xen/arch/x86/include/asm/amd.h b/xen/arch/x86/include/asm/amd.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/amd.h +++ b/xen/arch/x86/include/asm/amd.h @@ -XXX,XX +XXX,XX @@ AMD_MODEL_RANGE(0x11, 0x0, 0x0, 0xff, 0xf), \ AMD_MODEL_RANGE(0x12, 0x0, 0x0, 0xff, 0xf)) +/* + * The Zen1 and Zen2 microarchitectures are implemented by AMD (Fam17h) and + * Hygon (Fam18h) but without simple model number rules. Instead, use STIBP + * as a heuristic that distinguishes the two. + * + * The caller is required to perform the appropriate vendor/family checks + * first. + */ +#define is_zen1_uarch() (!boot_cpu_has(X86_FEATURE_AMD_STIBP)) +#define is_zen2_uarch() boot_cpu_has(X86_FEATURE_AMD_STIBP) + struct cpuinfo_x86; int cpu_has_amd_erratum(const struct cpuinfo_x86 *, int, ...); -- 2.30.2 In the Zen1 microarchitecure, there is one divider in the pipeline which services uops from both threads. In the case of #DE, the latched result from the previous DIV to execute will be forwarded speculatively. This is an interesting covert channel that allows two threads to communicate without any system calls. In also allows userspace to obtain the result of the most recent DIV instruction executed (even speculatively) in the core, which can be from a higher privilege context. Scrub the result from the divider by executing a non-faulting divide. This needs performing on the exit-to-guest paths, and ist_exit-to-Xen. This is XSA-439 / CVE-2023-20588. Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com> --- CC: Jan Beulich <JBeulich@suse.com> CC: Roger Pau Monné <roger.pau@citrix.com> CC: Wei Liu <wl@xen.org> No embargo - this is already public. XSA paperwork to follow. v2: * Rebase over the introduction of is_zen1_uarch(). * Fix the SC_DIV bit not to alias SC_VERW_IDLE. * Extend comments. --- docs/misc/xen-command-line.pandoc | 6 ++- xen/arch/x86/hvm/svm/entry.S | 1 + xen/arch/x86/include/asm/cpufeatures.h | 2 +- xen/arch/x86/include/asm/spec_ctrl_asm.h | 17 ++++++++ xen/arch/x86/spec_ctrl.c | 49 +++++++++++++++++++++++- 5 files changed, 72 insertions(+), 3 deletions(-) diff --git a/docs/misc/xen-command-line.pandoc b/docs/misc/xen-command-line.pandoc index XXXXXXX..XXXXXXX 100644 --- a/docs/misc/xen-command-line.pandoc +++ b/docs/misc/xen-command-line.pandoc @@ -XXX,XX +XXX,XX @@ By default SSBD will be mitigated at runtime (i.e `ssbd=runtime`). > {msr-sc,rsb,md-clear,ibpb-entry}=<bool>|{pv,hvm}=<bool>, > bti-thunk=retpoline|lfence|jmp, {ibrs,ibpb,ssbd,psfd, > eager-fpu,l1d-flush,branch-harden,srb-lock, -> unpriv-mmio,gds-mit}=<bool> ]` +> unpriv-mmio,gds-mit,div-scrub}=<bool> ]` Controls for speculative execution sidechannel mitigations. By default, Xen will pick the most appropriate mitigations based on compiled in support, @@ -XXX,XX +XXX,XX @@ has elected not to lock the configuration, Xen will use GDS_CTRL to mitigate GDS with. Otherwise, Xen will mitigate by disabling AVX, which blocks the use of the AVX2 Gather instructions. +On all hardware, the `div-scrub=` option can be used to force or prevent Xen +from mitigating the DIV-leakage vulnerability. By default, Xen will mitigate +DIV-leakage on hardware believed to be vulnerable. + ### sync_console > `= <boolean>` diff --git a/xen/arch/x86/hvm/svm/entry.S b/xen/arch/x86/hvm/svm/entry.S index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/hvm/svm/entry.S +++ b/xen/arch/x86/hvm/svm/entry.S @@ -XXX,XX +XXX,XX @@ __UNLIKELY_END(nsvm_hap) 1: /* No Spectre v1 concerns. Execution will hit VMRUN imminently. */ .endm ALTERNATIVE "", svm_vmentry_spec_ctrl, X86_FEATURE_SC_MSR_HVM + ALTERNATIVE "", DO_SPEC_CTRL_DIV, X86_FEATURE_SC_DIV pop %r15 pop %r14 diff --git a/xen/arch/x86/include/asm/cpufeatures.h b/xen/arch/x86/include/asm/cpufeatures.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/cpufeatures.h +++ b/xen/arch/x86/include/asm/cpufeatures.h @@ -XXX,XX +XXX,XX @@ XEN_CPUFEATURE(SC_RSB_HVM, X86_SYNTH(19)) /* RSB overwrite needed for HVM XEN_CPUFEATURE(XEN_SELFSNOOP, X86_SYNTH(20)) /* SELFSNOOP gets used by Xen itself */ XEN_CPUFEATURE(SC_MSR_IDLE, X86_SYNTH(21)) /* Clear MSR_SPEC_CTRL on idle */ XEN_CPUFEATURE(XEN_LBR, X86_SYNTH(22)) /* Xen uses MSR_DEBUGCTL.LBR */ -/* Bits 23 unused. */ +XEN_CPUFEATURE(SC_DIV, X86_SYNTH(23)) /* DIV scrub needed */ XEN_CPUFEATURE(SC_RSB_IDLE, X86_SYNTH(24)) /* RSB overwrite needed for idle. */ XEN_CPUFEATURE(SC_VERW_IDLE, X86_SYNTH(25)) /* VERW used by Xen for idle */ XEN_CPUFEATURE(XEN_SHSTK, X86_SYNTH(26)) /* Xen uses CET Shadow Stacks */ diff --git a/xen/arch/x86/include/asm/spec_ctrl_asm.h b/xen/arch/x86/include/asm/spec_ctrl_asm.h index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/include/asm/spec_ctrl_asm.h +++ b/xen/arch/x86/include/asm/spec_ctrl_asm.h @@ -XXX,XX +XXX,XX @@ .L\@_verw_skip: .endm +.macro DO_SPEC_CTRL_DIV +/* + * Requires nothing + * Clobbers %rax + * + * Issue a DIV for its flushing side effect (Zen1 uarch specific). Any + * non-faulting DIV will do; a byte DIV has least latency, and doesn't clobber + * %rdx. + */ + mov $1, %eax + div %al +.endm + .macro DO_SPEC_CTRL_ENTRY maybexen:req /* * Requires %rsp=regs (also cpuinfo if !maybexen) @@ -XXX,XX +XXX,XX @@ ALTERNATIVE "", DO_SPEC_CTRL_EXIT_TO_GUEST, X86_FEATURE_SC_MSR_PV DO_SPEC_CTRL_COND_VERW + + ALTERNATIVE "", DO_SPEC_CTRL_DIV, X86_FEATURE_SC_DIV .endm /* @@ -XXX,XX +XXX,XX @@ UNLIKELY_DISPATCH_LABEL(\@_serialise): verw STACK_CPUINFO_FIELD(verw_sel)(%r14) .L\@_skip_verw: + ALTERNATIVE "", DO_SPEC_CTRL_DIV, X86_FEATURE_SC_DIV + .L\@_skip_ist_exit: .endm diff --git a/xen/arch/x86/spec_ctrl.c b/xen/arch/x86/spec_ctrl.c index XXXXXXX..XXXXXXX 100644 --- a/xen/arch/x86/spec_ctrl.c +++ b/xen/arch/x86/spec_ctrl.c @@ -XXX,XX +XXX,XX @@ static int8_t __initdata opt_srb_lock = -1; static bool __initdata opt_unpriv_mmio; static bool __ro_after_init opt_fb_clear_mmio; static int8_t __initdata opt_gds_mit = -1; +static int8_t __initdata opt_div_scrub = -1; static int __init cf_check parse_spec_ctrl(const char *s) { @@ -XXX,XX +XXX,XX @@ static int __init cf_check parse_spec_ctrl(const char *s) opt_srb_lock = 0; opt_unpriv_mmio = false; opt_gds_mit = 0; + opt_div_scrub = 0; } else if ( val > 0 ) rc = -EINVAL; @@ -XXX,XX +XXX,XX @@ static int __init cf_check parse_spec_ctrl(const char *s) opt_unpriv_mmio = val; else if ( (val = parse_boolean("gds-mit", s, ss)) >= 0 ) opt_gds_mit = val; + else if ( (val = parse_boolean("div-scrub", s, ss)) >= 0 ) + opt_div_scrub = val; else rc = -EINVAL; @@ -XXX,XX +XXX,XX @@ static void __init print_details(enum ind_thunk thunk) "\n"); /* Settings for Xen's protection, irrespective of guests. */ - printk(" Xen settings: BTI-Thunk %s, SPEC_CTRL: %s%s%s%s%s, Other:%s%s%s%s%s\n", + printk(" Xen settings: BTI-Thunk %s, SPEC_CTRL: %s%s%s%s%s, Other:%s%s%s%s%s%s\n", thunk == THUNK_NONE ? "N/A" : thunk == THUNK_RETPOLINE ? "RETPOLINE" : thunk == THUNK_LFENCE ? "LFENCE" : @@ -XXX,XX +XXX,XX @@ static void __init print_details(enum ind_thunk thunk) opt_l1d_flush ? " L1D_FLUSH" : "", opt_md_clear_pv || opt_md_clear_hvm || opt_fb_clear_mmio ? " VERW" : "", + opt_div_scrub ? " DIV" : "", opt_branch_harden ? " BRANCH_HARDEN" : ""); /* L1TF diagnostics, printed if vulnerable or PV shadowing is in use. */ @@ -XXX,XX +XXX,XX @@ static void __init srso_calculations(bool hw_smt_enabled) setup_force_cpu_cap(X86_FEATURE_SRSO_NO); } +/* + * The Div leakage issue is specific to the AMD Zen1 microarchitecure. + * + * However, there's no $FOO_NO bit defined, so if we're virtualised we have no + * hope of spotting the case where we might move to vulnerable hardware. We + * also can't make any useful conclusion about SMT-ness. + * + * Don't check the hypervisor bit, so at least we do the safe thing when + * booting on something that looks like a Zen1 CPU. + */ +static bool __init has_div_vuln(void) +{ + if ( !(boot_cpu_data.x86_vendor & + (X86_VENDOR_AMD | X86_VENDOR_HYGON)) ) + return false; + + if ( (boot_cpu_data.x86 != 0x17 && boot_cpu_data.x86 != 0x18) || + !is_zen1_uarch() ) + return false; + + return true; +} + +static void __init div_calculations(bool hw_smt_enabled) +{ + bool cpu_bug_div = has_div_vuln(); + + if ( opt_div_scrub == -1 ) + opt_div_scrub = cpu_bug_div; + + if ( opt_div_scrub ) + setup_force_cpu_cap(X86_FEATURE_SC_DIV); + + if ( opt_smt == -1 && !cpu_has_hypervisor && cpu_bug_div && hw_smt_enabled ) + warning_add( + "Booted on leaky-DIV hardware with SMT/Hyperthreading\n" + "enabled. Please assess your configuration and choose an\n" + "explicit 'smt=<bool>' setting. See XSA-439.\n"); +} + static void __init ibpb_calculations(void) { bool def_ibpb_entry = false; @@ -XXX,XX +XXX,XX @@ void __init init_speculation_mitigations(void) ibpb_calculations(); + div_calculations(hw_smt_enabled); + /* Check whether Eager FPU should be enabled by default. */ if ( opt_eager_fpu == -1 ) opt_eager_fpu = should_use_eager_fpu(); -- 2.30.2