The code generation for barrier_nospec_true() is not correct.  We are taking a
perf it from the added fences, but not gaining any speculative safety.

This is caused by inline assembly trying to fight the compiler optimiser, and
the optimiser winning.  There is no clear way to achieve safety, so turn the
perf hit off for now.

This also largely reverts 3860d5534df4.  The name 'l1tf-barrier', and making
barrier_nospec_true() depend on CONFIG_HVM was constrained by what could be
discussed publicly at the time.  Now that MDS is public, neither aspects are
correct.

As l1tf-barrier hasn't been in a release of Xen, and
CONFIG_SPECULATIVE_BRANCH_HARDEN is disabled until we can find a safe way of
implementing the functionality, remove the l1tf-barrier command line option.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
CC: Jan Beulich <JBeulich@suse.com>
CC: Wei Liu <wl@xen.org>
CC: Roger Pau Monné <roger.pau@citrix.com>
CC: Juergen Gross <jgross@suse.com>
CC: Norbert Manthey <nmanthey@amazon.de>
---
 docs/misc/xen-command-line.pandoc |  8 +-------
 xen/arch/x86/spec_ctrl.c          | 17 ++---------------
 xen/common/Kconfig                | 17 +++++++++++++++++
 xen/include/asm-x86/cpufeatures.h |  2 +-
 xen/include/asm-x86/nospec.h      |  4 ++--
 xen/include/asm-x86/spec_ctrl.h   |  1 -
 6 files changed, 23 insertions(+), 26 deletions(-)

diff --git a/docs/misc/xen-command-line.pandoc b/docs/misc/xen-command-line.pandoc
index fc64429064..b9c5b822ca 100644
--- a/docs/misc/xen-command-line.pandoc
+++ b/docs/misc/xen-command-line.pandoc
@@ -1932,7 +1932,7 @@ By default SSBD will be mitigated at runtime (i.e `ssbd=runtime`).
 ### spec-ctrl (x86)
 > `= List of [ <bool>, xen=<bool>, {pv,hvm,msr-sc,rsb,md-clear}=<bool>,
 >              bti-thunk=retpoline|lfence|jmp, {ibrs,ibpb,ssbd,eager-fpu,
->              l1d-flush,l1tf-barrier}=<bool> ]`
+>              l1d-flush}=<bool> ]`
 
 Controls for speculative execution sidechannel mitigations.  By default, Xen
 will pick the most appropriate mitigations based on compiled in support,
@@ -2004,12 +2004,6 @@ Irrespective of Xen's setting, the feature is virtualised for HVM guests to
 use.  By default, Xen will enable this mitigation on hardware believed to be
 vulnerable to L1TF.
 
-On hardware vulnerable to L1TF, the `l1tf-barrier=` option can be used to force
-or prevent Xen from protecting evaluations inside the hypervisor with a barrier
-instruction to not load potentially secret information into L1 cache.  By
-default, Xen will enable this mitigation on hardware believed to be vulnerable
-to L1TF.
-
 ### sync_console
 > `= <boolean>`
 
diff --git a/xen/arch/x86/spec_ctrl.c b/xen/arch/x86/spec_ctrl.c
index 4761be81bd..5ea8870981 100644
--- a/xen/arch/x86/spec_ctrl.c
+++ b/xen/arch/x86/spec_ctrl.c
@@ -21,7 +21,6 @@
 #include <xen/lib.h>
 #include <xen/warning.h>
 
-#include <asm/cpuid.h>
 #include <asm/microcode.h>
 #include <asm/msr.h>
 #include <asm/processor.h>
@@ -53,7 +52,6 @@ bool __read_mostly opt_ibpb = true;
 bool __read_mostly opt_ssbd = false;
 int8_t __read_mostly opt_eager_fpu = -1;
 int8_t __read_mostly opt_l1d_flush = -1;
-int8_t __read_mostly opt_l1tf_barrier = -1;
 
 bool __initdata bsp_delay_spec_ctrl;
 uint8_t __read_mostly default_xen_spec_ctrl;
@@ -98,8 +96,6 @@ static int __init parse_spec_ctrl(const char *s)
             if ( opt_pv_l1tf_domu < 0 )
                 opt_pv_l1tf_domu = 0;
 
-            opt_l1tf_barrier = 0;
-
         disable_common:
             opt_rsb_pv = false;
             opt_rsb_hvm = false;
@@ -175,8 +171,6 @@ static int __init parse_spec_ctrl(const char *s)
             opt_eager_fpu = val;
         else if ( (val = parse_boolean("l1d-flush", s, ss)) >= 0 )
             opt_l1d_flush = val;
-        else if ( (val = parse_boolean("l1tf-barrier", s, ss)) >= 0 )
-            opt_l1tf_barrier = val;
         else
             rc = -EINVAL;
 
@@ -337,7 +331,7 @@ static void __init print_details(enum ind_thunk thunk, uint64_t caps)
                "\n");
 
     /* Settings for Xen's protection, irrespective of guests. */
-    printk("  Xen settings: BTI-Thunk %s, SPEC_CTRL: %s%s, Other:%s%s%s%s\n",
+    printk("  Xen settings: BTI-Thunk %s, SPEC_CTRL: %s%s, Other:%s%s%s\n",
            thunk == THUNK_NONE      ? "N/A" :
            thunk == THUNK_RETPOLINE ? "RETPOLINE" :
            thunk == THUNK_LFENCE    ? "LFENCE" :
@@ -348,8 +342,7 @@ static void __init print_details(enum ind_thunk thunk, uint64_t caps)
            (default_xen_spec_ctrl & SPEC_CTRL_SSBD)  ? " SSBD+" : " SSBD-",
            opt_ibpb                                  ? " IBPB"  : "",
            opt_l1d_flush                             ? " L1D_FLUSH" : "",
-           opt_md_clear_pv || opt_md_clear_hvm       ? " VERW"  : "",
-           opt_l1tf_barrier                          ? " L1TF_BARRIER" : "");
+           opt_md_clear_pv || opt_md_clear_hvm       ? " VERW"  : "");
 
     /* L1TF diagnostics, printed if vulnerable or PV shadowing is in use. */
     if ( cpu_has_bug_l1tf || opt_pv_l1tf_hwdom || opt_pv_l1tf_domu )
@@ -1034,12 +1027,6 @@ void __init init_speculation_mitigations(void)
     else if ( opt_l1d_flush == -1 )
         opt_l1d_flush = cpu_has_bug_l1tf && !(caps & ARCH_CAPS_SKIP_L1DFL);
 
-    /* By default, enable L1TF_VULN on L1TF-vulnerable hardware */
-    if ( opt_l1tf_barrier == -1 )
-        opt_l1tf_barrier = cpu_has_bug_l1tf && (opt_smt || !opt_l1d_flush);
-    if ( opt_l1tf_barrier > 0 )
-        setup_force_cpu_cap(X86_FEATURE_SC_L1TF_VULN);
-
     /*
      * We do not disable HT by default on affected hardware.
      *
diff --git a/xen/common/Kconfig b/xen/common/Kconfig
index 9644cc9911..d851e63083 100644
--- a/xen/common/Kconfig
+++ b/xen/common/Kconfig
@@ -96,6 +96,23 @@ config SPECULATIVE_ARRAY_HARDEN
 
 	  If unsure, say Y.
 
+config SPECULATIVE_BRANCH_HARDEN
+	bool "Speculative Branch Hardening"
+	depends on BROKEN
+        ---help---
+	  Contemporary processors may use speculative execution as a
+	  performance optimisation, but this can potentially be abused by an
+	  attacker to leak data via speculative sidechannels.
+
+	  One source of misbehaviour is by executing the wrong basic block
+	  following a conditional jump.
+
+	  When enabled, specific conditions which have been deemed liable to
+	  be speculatively abused will be hardened to avoid entering the wrong
+	  basic block.
+
+	  !!! WARNING - This is broken and doesn't generate safe code !!!
+
 endmenu
 
 config KEXEC
diff --git a/xen/include/asm-x86/cpufeatures.h b/xen/include/asm-x86/cpufeatures.h
index 91eccf5161..ecb651c35d 100644
--- a/xen/include/asm-x86/cpufeatures.h
+++ b/xen/include/asm-x86/cpufeatures.h
@@ -27,7 +27,7 @@ XEN_CPUFEATURE(XEN_SMAP,          X86_SYNTH(11)) /* SMAP gets used by Xen itself
 XEN_CPUFEATURE(LFENCE_DISPATCH,   X86_SYNTH(12)) /* lfence set as Dispatch Serialising */
 XEN_CPUFEATURE(IND_THUNK_LFENCE,  X86_SYNTH(13)) /* Use IND_THUNK_LFENCE */
 XEN_CPUFEATURE(IND_THUNK_JMP,     X86_SYNTH(14)) /* Use IND_THUNK_JMP */
-XEN_CPUFEATURE(SC_L1TF_VULN,      X86_SYNTH(15)) /* L1TF protection required */
+/* 15 unused. */
 XEN_CPUFEATURE(SC_MSR_PV,         X86_SYNTH(16)) /* MSR_SPEC_CTRL used by Xen for PV */
 XEN_CPUFEATURE(SC_MSR_HVM,        X86_SYNTH(17)) /* MSR_SPEC_CTRL used by Xen for HVM */
 XEN_CPUFEATURE(SC_RSB_PV,         X86_SYNTH(18)) /* RSB overwrite needed for PV */
diff --git a/xen/include/asm-x86/nospec.h b/xen/include/asm-x86/nospec.h
index 2aa47b3455..df7f9b13d7 100644
--- a/xen/include/asm-x86/nospec.h
+++ b/xen/include/asm-x86/nospec.h
@@ -9,8 +9,8 @@
 /* Allow to insert a read memory barrier into conditionals */
 static always_inline bool barrier_nospec_true(void)
 {
-#ifdef CONFIG_HVM
-    alternative("", "lfence", X86_FEATURE_SC_L1TF_VULN);
+#ifdef CONFIG_SPECULATIVE_BRANCH_HARDEN
+    alternative("", "lfence", X86_FEATURE_ALWAYS);
 #endif
     return true;
 }
diff --git a/xen/include/asm-x86/spec_ctrl.h b/xen/include/asm-x86/spec_ctrl.h
index 1339ddd7ef..ba03bb42e5 100644
--- a/xen/include/asm-x86/spec_ctrl.h
+++ b/xen/include/asm-x86/spec_ctrl.h
@@ -37,7 +37,6 @@ extern bool opt_ibpb;
 extern bool opt_ssbd;
 extern int8_t opt_eager_fpu;
 extern int8_t opt_l1d_flush;
-extern int8_t opt_l1tf_barrier;
 
 extern bool bsp_delay_spec_ctrl;
 extern uint8_t default_xen_spec_ctrl;
-- 
2.11.0


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

Hi,

On 9/30/19 7:24 PM, Andrew Cooper wrote:
> The code generation for barrier_nospec_true() is not correct.  We are taking a
> perf it from the added fences, but not gaining any speculative safety.

s/it/hit/?

> 
> This is caused by inline assembly trying to fight the compiler optimiser, and
> the optimiser winning.  There is no clear way to achieve safety, so turn the
> perf hit off for now.
> 
> This also largely reverts 3860d5534df4.  The name 'l1tf-barrier', and making
> barrier_nospec_true() depend on CONFIG_HVM was constrained by what could be
> discussed publicly at the time.  Now that MDS is public, neither aspects are
> correct.
> 
> As l1tf-barrier hasn't been in a release of Xen, and
> CONFIG_SPECULATIVE_BRANCH_HARDEN is disabled until we can find a safe way of
> implementing the functionality, remove the l1tf-barrier command line option.
> 
> Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
> ---
> CC: Jan Beulich <JBeulich@suse.com>
> CC: Wei Liu <wl@xen.org>
> CC: Roger Pau Monné <roger.pau@citrix.com>
> CC: Juergen Gross <jgross@suse.com>
> CC: Norbert Manthey <nmanthey@amazon.de>
> ---
>   docs/misc/xen-command-line.pandoc |  8 +-------
>   xen/arch/x86/spec_ctrl.c          | 17 ++---------------
>   xen/common/Kconfig                | 17 +++++++++++++++++

I think this wanted to have "THE REST" CCed.

>   xen/include/asm-x86/cpufeatures.h |  2 +-
>   xen/include/asm-x86/nospec.h      |  4 ++--
>   xen/include/asm-x86/spec_ctrl.h   |  1 -
>   6 files changed, 23 insertions(+), 26 deletions(-)

[...]

> diff --git a/xen/common/Kconfig b/xen/common/Kconfig
> index 9644cc9911..d851e63083 100644
> --- a/xen/common/Kconfig
> +++ b/xen/common/Kconfig
> @@ -96,6 +96,23 @@ config SPECULATIVE_ARRAY_HARDEN
>   
>   	  If unsure, say Y.
>   
> +config SPECULATIVE_BRANCH_HARDEN
> +	bool "Speculative Branch Hardening"
> +	depends on BROKEN
> +        ---help---
> +	  Contemporary processors may use speculative execution as a
> +	  performance optimisation, but this can potentially be abused by an
> +	  attacker to leak data via speculative sidechannels.
> +
> +	  One source of misbehaviour is by executing the wrong basic block
> +	  following a conditional jump.
> +
> +	  When enabled, specific conditions which have been deemed liable to
> +	  be speculatively abused will be hardened to avoid entering the wrong
> +	  basic block.
> +
> +	  !!! WARNING - This is broken and doesn't generate safe code !!!

Any reason to add that in common code when this is x86 only? My worry is 
this gate config gate nothing on Arm so the user may have a false sense 
that it can be used (even though it is clearly BROKEN).

Also the name is quite close to the CONFIG_HARDEN_PREDICTOR on Arm and 
may confuse user. Although, I don't have a better name so far :/

Cheers,

-- 
Julien Grall

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

On 30/09/2019 21:17, Julien Grall wrote:
> Hi,
>
> On 9/30/19 7:24 PM, Andrew Cooper wrote:
>> The code generation for barrier_nospec_true() is not correct.  We are
>> taking a
>> perf it from the added fences, but not gaining any speculative safety.
>
> s/it/hit/?

Yes.

>
>>
>> This is caused by inline assembly trying to fight the compiler
>> optimiser, and
>> the optimiser winning.  There is no clear way to achieve safety, so
>> turn the
>> perf hit off for now.
>>
>> This also largely reverts 3860d5534df4.  The name 'l1tf-barrier', and
>> making
>> barrier_nospec_true() depend on CONFIG_HVM was constrained by what
>> could be
>> discussed publicly at the time.  Now that MDS is public, neither
>> aspects are
>> correct.
>>
>> As l1tf-barrier hasn't been in a release of Xen, and
>> CONFIG_SPECULATIVE_BRANCH_HARDEN is disabled until we can find a safe
>> way of
>> implementing the functionality, remove the l1tf-barrier command line
>> option.
>>
>> Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
>> ---
>> CC: Jan Beulich <JBeulich@suse.com>
>> CC: Wei Liu <wl@xen.org>
>> CC: Roger Pau Monné <roger.pau@citrix.com>
>> CC: Juergen Gross <jgross@suse.com>
>> CC: Norbert Manthey <nmanthey@amazon.de>
>> ---
>>   docs/misc/xen-command-line.pandoc |  8 +-------
>>   xen/arch/x86/spec_ctrl.c          | 17 ++---------------
>>   xen/common/Kconfig                | 17 +++++++++++++++++
>
> I think this wanted to have "THE REST" CCed.
>
>>   xen/include/asm-x86/cpufeatures.h |  2 +-
>>   xen/include/asm-x86/nospec.h      |  4 ++--
>>   xen/include/asm-x86/spec_ctrl.h   |  1 -
>>   6 files changed, 23 insertions(+), 26 deletions(-)
>
> [...]
>
>> diff --git a/xen/common/Kconfig b/xen/common/Kconfig
>> index 9644cc9911..d851e63083 100644
>> --- a/xen/common/Kconfig
>> +++ b/xen/common/Kconfig
>> @@ -96,6 +96,23 @@ config SPECULATIVE_ARRAY_HARDEN
>>           If unsure, say Y.
>>   +config SPECULATIVE_BRANCH_HARDEN
>> +    bool "Speculative Branch Hardening"
>> +    depends on BROKEN
>> +        ---help---
>> +      Contemporary processors may use speculative execution as a
>> +      performance optimisation, but this can potentially be abused
>> by an
>> +      attacker to leak data via speculative sidechannels.
>> +
>> +      One source of misbehaviour is by executing the wrong basic block
>> +      following a conditional jump.
>> +
>> +      When enabled, specific conditions which have been deemed
>> liable to
>> +      be speculatively abused will be hardened to avoid entering the
>> wrong
>> +      basic block.
>> +
>> +      !!! WARNING - This is broken and doesn't generate safe code !!!
>
> Any reason to add that in common code when this is x86 only?

In principle, its not x86 specific.

> My worry is this gate config gate nothing on Arm so the user may have
> a false sense that it can be used (even though it is clearly BROKEN).
>
> Also the name is quite close to the CONFIG_HARDEN_PREDICTOR on Arm and
> may confuse user. Although, I don't have a better name so far :/

The "depends on BROKEN" means it will never show up to a user in
menuconfig, which is why it was only CC to x86, and not to rest.

As for naming, I'm open to suggestions, but this was the best I could
come up with.

~Andrew

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

Hi Andrew,

On 01/10/2019 00:21, Andrew Cooper wrote:
> On 30/09/2019 21:17, Julien Grall wrote:
>> Hi,
>>
>> On 9/30/19 7:24 PM, Andrew Cooper wrote:
>>> The code generation for barrier_nospec_true() is not correct.  We are
>>> taking a
>>> perf it from the added fences, but not gaining any speculative safety.
>>
>> s/it/hit/?
> 
> Yes.
> 
>>
>>>
>>> This is caused by inline assembly trying to fight the compiler
>>> optimiser, and
>>> the optimiser winning.  There is no clear way to achieve safety, so
>>> turn the
>>> perf hit off for now.
>>>
>>> This also largely reverts 3860d5534df4.  The name 'l1tf-barrier', and
>>> making
>>> barrier_nospec_true() depend on CONFIG_HVM was constrained by what
>>> could be
>>> discussed publicly at the time.  Now that MDS is public, neither
>>> aspects are
>>> correct.
>>>
>>> As l1tf-barrier hasn't been in a release of Xen, and
>>> CONFIG_SPECULATIVE_BRANCH_HARDEN is disabled until we can find a safe
>>> way of
>>> implementing the functionality, remove the l1tf-barrier command line
>>> option.
>>>
>>> Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
>>> ---
>>> CC: Jan Beulich <JBeulich@suse.com>
>>> CC: Wei Liu <wl@xen.org>
>>> CC: Roger Pau Monné <roger.pau@citrix.com>
>>> CC: Juergen Gross <jgross@suse.com>
>>> CC: Norbert Manthey <nmanthey@amazon.de>
>>> ---
>>>    docs/misc/xen-command-line.pandoc |  8 +-------
>>>    xen/arch/x86/spec_ctrl.c          | 17 ++---------------
>>>    xen/common/Kconfig                | 17 +++++++++++++++++
>>
>> I think this wanted to have "THE REST" CCed.
>>
>>>    xen/include/asm-x86/cpufeatures.h |  2 +-
>>>    xen/include/asm-x86/nospec.h      |  4 ++--
>>>    xen/include/asm-x86/spec_ctrl.h   |  1 -
>>>    6 files changed, 23 insertions(+), 26 deletions(-)
>>
>> [...]
>>
>>> diff --git a/xen/common/Kconfig b/xen/common/Kconfig
>>> index 9644cc9911..d851e63083 100644
>>> --- a/xen/common/Kconfig
>>> +++ b/xen/common/Kconfig
>>> @@ -96,6 +96,23 @@ config SPECULATIVE_ARRAY_HARDEN
>>>            If unsure, say Y.
>>>    +config SPECULATIVE_BRANCH_HARDEN
>>> +    bool "Speculative Branch Hardening"
>>> +    depends on BROKEN
>>> +        ---help---
>>> +      Contemporary processors may use speculative execution as a
>>> +      performance optimisation, but this can potentially be abused
>>> by an
>>> +      attacker to leak data via speculative sidechannels.
>>> +
>>> +      One source of misbehaviour is by executing the wrong basic block
>>> +      following a conditional jump.
>>> +
>>> +      When enabled, specific conditions which have been deemed
>>> liable to
>>> +      be speculatively abused will be hardened to avoid entering the
>>> wrong
>>> +      basic block.
>>> +
>>> +      !!! WARNING - This is broken and doesn't generate safe code !!!
>>
>> Any reason to add that in common code when this is x86 only?
> 
> In principle, its not x86 specific.
> 
>> My worry is this gate config gate nothing on Arm so the user may have
>> a false sense that it can be used (even though it is clearly BROKEN).
>>
>> Also the name is quite close to the CONFIG_HARDEN_PREDICTOR on Arm and
>> may confuse user. Although, I don't have a better name so far :/
> 
> The "depends on BROKEN" means it will never show up to a user in
> menuconfig, which is why it was only CC to x86, and not to rest.

What's the long term plan for this option? Are you planning to remove it 
completely or just the dependencies on BROKEN?

My concern is if this option will ever become selectable then it will not doing 
what's expected on Arm.

So, even if in principle it is arch agnostic, it feels to me this option should 
better be implemented in x86/Kconfig.

Cheers,

-- 
Julien Grall

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

On 01.10.2019 11:17, Julien Grall wrote:
> On 01/10/2019 00:21, Andrew Cooper wrote:
>> On 30/09/2019 21:17, Julien Grall wrote:
>>> My worry is this gate config gate nothing on Arm so the user may have
>>> a false sense that it can be used (even though it is clearly BROKEN).
>>>
>>> Also the name is quite close to the CONFIG_HARDEN_PREDICTOR on Arm and
>>> may confuse user. Although, I don't have a better name so far :/
>>
>> The "depends on BROKEN" means it will never show up to a user in
>> menuconfig, which is why it was only CC to x86, and not to rest.
> 
> What's the long term plan for this option? Are you planning to remove it 
> completely or just the dependencies on BROKEN?
> 
> My concern is if this option will ever become selectable then it will not doing 
> what's expected on Arm.
> 
> So, even if in principle it is arch agnostic, it feels to me this option should 
> better be implemented in x86/Kconfig.

I don't think so, no. When BROKEN is to be removed, it ought to be
replaced by a suitable "depends on HAVE_*", which Arm could choose
to not select.

Jan

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

Hi Jan,

On 01/10/2019 10:22, Jan Beulich wrote:
> On 01.10.2019 11:17, Julien Grall wrote:
>> On 01/10/2019 00:21, Andrew Cooper wrote:
>>> On 30/09/2019 21:17, Julien Grall wrote:
>>>> My worry is this gate config gate nothing on Arm so the user may have
>>>> a false sense that it can be used (even though it is clearly BROKEN).
>>>>
>>>> Also the name is quite close to the CONFIG_HARDEN_PREDICTOR on Arm and
>>>> may confuse user. Although, I don't have a better name so far :/
>>>
>>> The "depends on BROKEN" means it will never show up to a user in
>>> menuconfig, which is why it was only CC to x86, and not to rest.
>>
>> What's the long term plan for this option? Are you planning to remove it
>> completely or just the dependencies on BROKEN?
>>
>> My concern is if this option will ever become selectable then it will not doing
>> what's expected on Arm.
>>
>> So, even if in principle it is arch agnostic, it feels to me this option should
>> better be implemented in x86/Kconfig.
> 
> I don't think so, no. When BROKEN is to be removed, it ought to be
> replaced by a suitable "depends on HAVE_*", which Arm could choose
> to not select.

So there are an expectation this option will be used by common code in the future?

Cheers,

-- 
Julien Grall

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

On 01/10/2019 10:26, Julien Grall wrote:
> Hi Jan,
>
> On 01/10/2019 10:22, Jan Beulich wrote:
>> On 01.10.2019 11:17, Julien Grall wrote:
>>> On 01/10/2019 00:21, Andrew Cooper wrote:
>>>> On 30/09/2019 21:17, Julien Grall wrote:
>>>>> My worry is this gate config gate nothing on Arm so the user may have
>>>>> a false sense that it can be used (even though it is clearly BROKEN).
>>>>>
>>>>> Also the name is quite close to the CONFIG_HARDEN_PREDICTOR on Arm
>>>>> and
>>>>> may confuse user. Although, I don't have a better name so far :/
>>>>
>>>> The "depends on BROKEN" means it will never show up to a user in
>>>> menuconfig, which is why it was only CC to x86, and not to rest.
>>>
>>> What's the long term plan for this option? Are you planning to
>>> remove it
>>> completely or just the dependencies on BROKEN?
>>>
>>> My concern is if this option will ever become selectable then it
>>> will not doing
>>> what's expected on Arm.
>>>
>>> So, even if in principle it is arch agnostic, it feels to me this
>>> option should
>>> better be implemented in x86/Kconfig.
>>
>> I don't think so, no. When BROKEN is to be removed, it ought to be
>> replaced by a suitable "depends on HAVE_*", which Arm could choose
>> to not select.
>
> So there are an expectation this option will be used by common code in
> the future?

It already is.  ARM has stubs for evaluate_nospec() etc.

My gut feeling is that the only way this is going to be resolved sanely
is with a compiler feature or plugin, at which point it reasonably can
be cross-arch.

~Andrew

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

Hi Andrew,

On 01/10/2019 10:41, Andrew Cooper wrote:
> On 01/10/2019 10:26, Julien Grall wrote:
>> Hi Jan,
>>
>> On 01/10/2019 10:22, Jan Beulich wrote:
>>> On 01.10.2019 11:17, Julien Grall wrote:
>>>> On 01/10/2019 00:21, Andrew Cooper wrote:
>>>>> On 30/09/2019 21:17, Julien Grall wrote:
>>>>>> My worry is this gate config gate nothing on Arm so the user may have
>>>>>> a false sense that it can be used (even though it is clearly BROKEN).
>>>>>>
>>>>>> Also the name is quite close to the CONFIG_HARDEN_PREDICTOR on Arm
>>>>>> and
>>>>>> may confuse user. Although, I don't have a better name so far :/
>>>>>
>>>>> The "depends on BROKEN" means it will never show up to a user in
>>>>> menuconfig, which is why it was only CC to x86, and not to rest.
>>>>
>>>> What's the long term plan for this option? Are you planning to
>>>> remove it
>>>> completely or just the dependencies on BROKEN?
>>>>
>>>> My concern is if this option will ever become selectable then it
>>>> will not doing
>>>> what's expected on Arm.
>>>>
>>>> So, even if in principle it is arch agnostic, it feels to me this
>>>> option should
>>>> better be implemented in x86/Kconfig.
>>>
>>> I don't think so, no. When BROKEN is to be removed, it ought to be
>>> replaced by a suitable "depends on HAVE_*", which Arm could choose
>>> to not select.
>>
>> So there are an expectation this option will be used by common code in
>> the future?
> 
> It already is.  ARM has stubs for evaluate_nospec() etc.
> 
> My gut feeling is that the only way this is going to be resolved sanely
> is with a compiler feature or plugin, at which point it reasonably can
> be cross-arch.

Fair enough. I don't have any more concern then. I will have a think about the 
naming or the possibly to merge the two configs (the common and arm ones) together.

Cheers,

-- 
Julien Grall

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

On 30.09.2019 20:24, Andrew Cooper wrote:
> The code generation for barrier_nospec_true() is not correct.  We are taking a
> perf it from the added fences, but not gaining any speculative safety.

You want to be more specific here, I think: ISTR you saying that the LFENCEs
get inserted at the wrong place. IIRC we want them on either side of a
conditional branch, i.e. immediately following a branch insn as well as right
at the branch target. I've taken, as a simple example,
p2m_mem_access_sanity_check(), and this looks to be the way gcc9 has generated
code (in a non-debug build). Hence either I'm mis-remembering what we want
things to look like, or there's more to it than code generation simply being
"not correct".

> --- a/xen/common/Kconfig
> +++ b/xen/common/Kconfig
> @@ -96,6 +96,23 @@ config SPECULATIVE_ARRAY_HARDEN
>  
>  	  If unsure, say Y.
>  
> +config SPECULATIVE_BRANCH_HARDEN
> +	bool "Speculative Branch Hardening"
> +	depends on BROKEN
> +        ---help---
> +	  Contemporary processors may use speculative execution as a
> +	  performance optimisation, but this can potentially be abused by an
> +	  attacker to leak data via speculative sidechannels.
> +
> +	  One source of misbehaviour is by executing the wrong basic block
> +	  following a conditional jump.
> +
> +	  When enabled, specific conditions which have been deemed liable to
> +	  be speculatively abused will be hardened to avoid entering the wrong
> +	  basic block.
> +
> +	  !!! WARNING - This is broken and doesn't generate safe code !!!

Not being a native speaker, this read to me as "is broken in general",
whereas the brokenness is that according to your analysis safe code
does not result. Therefore how about "This is broken in that it doesn't
generate safe code"?

> --- a/xen/include/asm-x86/nospec.h
> +++ b/xen/include/asm-x86/nospec.h
> @@ -9,8 +9,8 @@
>  /* Allow to insert a read memory barrier into conditionals */
>  static always_inline bool barrier_nospec_true(void)
>  {
> -#ifdef CONFIG_HVM
> -    alternative("", "lfence", X86_FEATURE_SC_L1TF_VULN);
> +#ifdef CONFIG_SPECULATIVE_BRANCH_HARDEN
> +    alternative("", "lfence", X86_FEATURE_ALWAYS);

Why alternative() then and not just asm()?

Jan

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

On 01/10/2019 13:21, Jan Beulich wrote:
> On 30.09.2019 20:24, Andrew Cooper wrote:
>> The code generation for barrier_nospec_true() is not correct.  We are taking a
>> perf it from the added fences, but not gaining any speculative safety.
> You want to be more specific here, I think: ISTR you saying that the LFENCEs
> get inserted at the wrong place.

Correct.

>  IIRC we want them on either side of a
> conditional branch, i.e. immediately following a branch insn as well as right
> at the branch target.

Specifically, they must be at the head of both basic blocks following
the conditional jump.

> I've taken, as a simple example,
> p2m_mem_access_sanity_check(), and this looks to be the way gcc9 has generated
> code (in a non-debug build). Hence either I'm mis-remembering what we want
> things to look like, or there's more to it than code generation simply being
> "not correct".

This example demonstrates the problem, and actually throws a further
spanner in the works of how make this safe, which hadn't occurred to me
before.

The instruction stream from a caller of p2m_mem_access_sanity_check()
will look something like this:

call p2m_mem_access_sanity_check
    ...
    lfence
    ...
    ret   
cmp $0, %eax
jne ...

Which is unsafe, because the only safe way to arrange this code would be:

call p2m_mem_access_sanity_check
    ...
    ret
cmp $0, %eax
jne 1f
lfence
...
1: lfence
...

There is absolutely no possible way for inline assembly to be used to
propagate this safety property across translation units.  This is going
to have to be an attribute of some form or another handled by the compiler.

>
>> --- a/xen/common/Kconfig
>> +++ b/xen/common/Kconfig
>> @@ -96,6 +96,23 @@ config SPECULATIVE_ARRAY_HARDEN
>>  
>>  	  If unsure, say Y.
>>  
>> +config SPECULATIVE_BRANCH_HARDEN
>> +	bool "Speculative Branch Hardening"
>> +	depends on BROKEN
>> +        ---help---
>> +	  Contemporary processors may use speculative execution as a
>> +	  performance optimisation, but this can potentially be abused by an
>> +	  attacker to leak data via speculative sidechannels.
>> +
>> +	  One source of misbehaviour is by executing the wrong basic block
>> +	  following a conditional jump.
>> +
>> +	  When enabled, specific conditions which have been deemed liable to
>> +	  be speculatively abused will be hardened to avoid entering the wrong
>> +	  basic block.
>> +
>> +	  !!! WARNING - This is broken and doesn't generate safe code !!!
> Not being a native speaker, this read to me as "is broken in general",
> whereas the brokenness is that according to your analysis safe code
> does not result. Therefore how about "This is broken in that it doesn't
> generate safe code"?

I wouldn't necessarily agree with the "in general" implication, but
given the lack of clarity, a better option would be:

!!! WARNING - This option doesn't work as intended. It does not generate
speculatively safe code !!!

>
>> --- a/xen/include/asm-x86/nospec.h
>> +++ b/xen/include/asm-x86/nospec.h
>> @@ -9,8 +9,8 @@
>>  /* Allow to insert a read memory barrier into conditionals */
>>  static always_inline bool barrier_nospec_true(void)
>>  {
>> -#ifdef CONFIG_HVM
>> -    alternative("", "lfence", X86_FEATURE_SC_L1TF_VULN);
>> +#ifdef CONFIG_SPECULATIVE_BRANCH_HARDEN
>> +    alternative("", "lfence", X86_FEATURE_ALWAYS);
> Why alternative() then and not just asm()?

Ok.

~Andrew

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

On 01.10.2019 14:51, Andrew Cooper wrote:
> On 01/10/2019 13:21, Jan Beulich wrote:
>> On 30.09.2019 20:24, Andrew Cooper wrote:
>>> The code generation for barrier_nospec_true() is not correct.  We are taking a
>>> perf it from the added fences, but not gaining any speculative safety.
>> You want to be more specific here, I think: ISTR you saying that the LFENCEs
>> get inserted at the wrong place.
> 
> Correct.
> 
>>  IIRC we want them on either side of a
>> conditional branch, i.e. immediately following a branch insn as well as right
>> at the branch target.
> 
> Specifically, they must be at the head of both basic blocks following
> the conditional jump.
> 
>> I've taken, as a simple example,
>> p2m_mem_access_sanity_check(), and this looks to be the way gcc9 has generated
>> code (in a non-debug build). Hence either I'm mis-remembering what we want
>> things to look like, or there's more to it than code generation simply being
>> "not correct".
> 
> This example demonstrates the problem, and actually throws a further
> spanner in the works of how make this safe, which hadn't occurred to me
> before.
> 
> The instruction stream from a caller of p2m_mem_access_sanity_check()
> will look something like this:
> 
> call p2m_mem_access_sanity_check
>     ...
>     lfence
>     ...
>     ret   
> cmp $0, %eax
> jne ...
> 
> Which is unsafe, because the only safe way to arrange this code would be:
> 
> call p2m_mem_access_sanity_check
>     ...
>     ret
> cmp $0, %eax
> jne 1f
> lfence
> ...
> 1: lfence
> ...
> 
> There is absolutely no possible way for inline assembly to be used to
> propagate this safety property across translation units.  This is going
> to have to be an attribute of some form or another handled by the compiler.

But you realize that this particular example is basically a more
complex is_XYZ() check, which could be dealt with by inlining the
function. Of course there are going to be larger functions where
the result wants to be guarded like you say. But just like the
addition of the nospec macros to various is_XYZ() functions is a
manual operation (as long the compiler doesn't help), it would in
that case be a matter of latching the return value into a local
variable and using an appropriate guarding construct when
evaluating it.

So I'm afraid for now I still can't agree with your "not correct"
assessment - the generated code in the example looks correct to
me, and if further guarding was needed in users of this particular
function, it would be those users which would need further
massaging.

>>> --- a/xen/common/Kconfig
>>> +++ b/xen/common/Kconfig
>>> @@ -96,6 +96,23 @@ config SPECULATIVE_ARRAY_HARDEN
>>>  
>>>  	  If unsure, say Y.
>>>  
>>> +config SPECULATIVE_BRANCH_HARDEN
>>> +	bool "Speculative Branch Hardening"
>>> +	depends on BROKEN
>>> +        ---help---
>>> +	  Contemporary processors may use speculative execution as a
>>> +	  performance optimisation, but this can potentially be abused by an
>>> +	  attacker to leak data via speculative sidechannels.
>>> +
>>> +	  One source of misbehaviour is by executing the wrong basic block
>>> +	  following a conditional jump.
>>> +
>>> +	  When enabled, specific conditions which have been deemed liable to
>>> +	  be speculatively abused will be hardened to avoid entering the wrong
>>> +	  basic block.
>>> +
>>> +	  !!! WARNING - This is broken and doesn't generate safe code !!!
>> Not being a native speaker, this read to me as "is broken in general",
>> whereas the brokenness is that according to your analysis safe code
>> does not result. Therefore how about "This is broken in that it doesn't
>> generate safe code"?
> 
> I wouldn't necessarily agree with the "in general" implication, but
> given the lack of clarity, a better option would be:
> 
> !!! WARNING - This option doesn't work as intended. It does not generate
> speculatively safe code !!!

Fine with me.

Jan

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

On 01/10/2019 15:32, Jan Beulich wrote:
> On 01.10.2019 14:51, Andrew Cooper wrote:
>> On 01/10/2019 13:21, Jan Beulich wrote:
>>> On 30.09.2019 20:24, Andrew Cooper wrote:
>>>> The code generation for barrier_nospec_true() is not correct.  We are taking a
>>>> perf it from the added fences, but not gaining any speculative safety.
>>> You want to be more specific here, I think: ISTR you saying that the LFENCEs
>>> get inserted at the wrong place.
>> Correct.
>>
>>>  IIRC we want them on either side of a
>>> conditional branch, i.e. immediately following a branch insn as well as right
>>> at the branch target.
>> Specifically, they must be at the head of both basic blocks following
>> the conditional jump.
>>
>>> I've taken, as a simple example,
>>> p2m_mem_access_sanity_check(), and this looks to be the way gcc9 has generated
>>> code (in a non-debug build). Hence either I'm mis-remembering what we want
>>> things to look like, or there's more to it than code generation simply being
>>> "not correct".
>> This example demonstrates the problem, and actually throws a further
>> spanner in the works of how make this safe, which hadn't occurred to me
>> before.
>>
>> The instruction stream from a caller of p2m_mem_access_sanity_check()
>> will look something like this:
>>
>> call p2m_mem_access_sanity_check
>>     ...
>>     lfence
>>     ...
>>     ret   
>> cmp $0, %eax
>> jne ...
>>
>> Which is unsafe, because the only safe way to arrange this code would be:
>>
>> call p2m_mem_access_sanity_check
>>     ...
>>     ret
>> cmp $0, %eax
>> jne 1f
>> lfence
>> ...
>> 1: lfence
>> ...
>>
>> There is absolutely no possible way for inline assembly to be used to
>> propagate this safety property across translation units.  This is going
>> to have to be an attribute of some form or another handled by the compiler.
> But you realize that this particular example is basically a more
> complex is_XYZ() check, which could be dealt with by inlining the
> function. Of course there are going to be larger functions where
> the result wants to be guarded like you say. But just like the
> addition of the nospec macros to various is_XYZ() functions is a
> manual operation (as long the compiler doesn't help), it would in
> that case be a matter of latching the return value into a local
> variable and using an appropriate guarding construct when
> evaluating it.

And this reasoning demonstrates yet another problem (this one was raised
at the meeting in Chicago).

evaluate_nospec() is not a useful construct if it needs inserting at
every higher level predicate to result in safe code.  This is
boarderline-impossible to review for, and extremely easy to break
accidentally.

>
> So I'm afraid for now I still can't agree with your "not correct"
> assessment - the generated code in the example looks correct to
> me, and if further guarding was needed in users of this particular
> function, it would be those users which would need further
> massaging.

Safety against spectre v1 is not a matter of opinion.

To protect against speculatively executing the wrong basic block, the
pipeline must execute the conditional jump first, *then* hit an lfence
to serialise the instruction stream and revector in the case of
incorrect speculation.

The other way around is not safe.  Serialising the instruction stream
doesn't do anything to protect against the attacker taking control of a
later branch.

The bigger problem is to do with classifying what we are protecting
against.  In the case of is_control_domain(), it is any action based on
the result of the decision.  For is_{pv,hvm}_domain(), is only (to a
first approximation) speculative type confusion into the pv/hvm unions
(which in practice extends to calling pv_/hvm_ functions as well).

As for the real concrete breakages.  In a staging build with GCC 6

$ objdump -d xen-syms | grep '<is_hvm_domain>:' | wc -l
18
$ objdump -d xen-syms | grep '<is_pv_domain>:' | wc -l
9

All of which have the lfence too early to protect against speculative
type confusion.

~Andrew

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

On 01.10.2019 17:37, Andrew Cooper wrote:
> On 01/10/2019 15:32, Jan Beulich wrote:
>> On 01.10.2019 14:51, Andrew Cooper wrote:
>>> On 01/10/2019 13:21, Jan Beulich wrote:
>>>> On 30.09.2019 20:24, Andrew Cooper wrote:
>>>>> The code generation for barrier_nospec_true() is not correct.  We are taking a
>>>>> perf it from the added fences, but not gaining any speculative safety.
>>>> You want to be more specific here, I think: ISTR you saying that the LFENCEs
>>>> get inserted at the wrong place.
>>> Correct.
>>>
>>>>  IIRC we want them on either side of a
>>>> conditional branch, i.e. immediately following a branch insn as well as right
>>>> at the branch target.
>>> Specifically, they must be at the head of both basic blocks following
>>> the conditional jump.
>>>
>>>> I've taken, as a simple example,
>>>> p2m_mem_access_sanity_check(), and this looks to be the way gcc9 has generated
>>>> code (in a non-debug build). Hence either I'm mis-remembering what we want
>>>> things to look like, or there's more to it than code generation simply being
>>>> "not correct".
>>> This example demonstrates the problem, and actually throws a further
>>> spanner in the works of how make this safe, which hadn't occurred to me
>>> before.
>>>
>>> The instruction stream from a caller of p2m_mem_access_sanity_check()
>>> will look something like this:
>>>
>>> call p2m_mem_access_sanity_check
>>>     ...
>>>     lfence
>>>     ...
>>>     ret   
>>> cmp $0, %eax
>>> jne ...
>>>
>>> Which is unsafe, because the only safe way to arrange this code would be:
>>>
>>> call p2m_mem_access_sanity_check
>>>     ...
>>>     ret
>>> cmp $0, %eax
>>> jne 1f
>>> lfence
>>> ...
>>> 1: lfence
>>> ...
>>>
>>> There is absolutely no possible way for inline assembly to be used to
>>> propagate this safety property across translation units.  This is going
>>> to have to be an attribute of some form or another handled by the compiler.
>> But you realize that this particular example is basically a more
>> complex is_XYZ() check, which could be dealt with by inlining the
>> function. Of course there are going to be larger functions where
>> the result wants to be guarded like you say. But just like the
>> addition of the nospec macros to various is_XYZ() functions is a
>> manual operation (as long the compiler doesn't help), it would in
>> that case be a matter of latching the return value into a local
>> variable and using an appropriate guarding construct when
>> evaluating it.
> 
> And this reasoning demonstrates yet another problem (this one was raised
> at the meeting in Chicago).
> 
> evaluate_nospec() is not a useful construct if it needs inserting at
> every higher level predicate to result in safe code.  This is
> boarderline-impossible to review for, and extremely easy to break
> accidentally.

I agree; since evaluate_nospec() insertion need is generally a hard
to investigate / review action, I don#t consider this unexpected.

>> So I'm afraid for now I still can't agree with your "not correct"
>> assessment - the generated code in the example looks correct to
>> me, and if further guarding was needed in users of this particular
>> function, it would be those users which would need further
>> massaging.
> 
> Safety against spectre v1 is not a matter of opinion.
> 
> To protect against speculatively executing the wrong basic block, the
> pipeline must execute the conditional jump first, *then* hit an lfence
> to serialise the instruction stream and revector in the case of
> incorrect speculation.
> 
> The other way around is not safe.  Serialising the instruction stream
> doesn't do anything to protect against the attacker taking control of a
> later branch.
> 
> The bigger problem is to do with classifying what we are protecting
> against.  In the case of is_control_domain(), it is any action based on
> the result of the decision.  For is_{pv,hvm}_domain(), is only (to a
> first approximation) speculative type confusion into the pv/hvm unions
> (which in practice extends to calling pv_/hvm_ functions as well).
> 
> As for the real concrete breakages.  In a staging build with GCC 6
> 
> $ objdump -d xen-syms | grep '<is_hvm_domain>:' | wc -l
> 18
> $ objdump -d xen-syms | grep '<is_pv_domain>:' | wc -l
> 9
> 
> All of which have the lfence too early to protect against speculative
> type confusion.

And all of which are because, other than I think it was originally
intended, the functions still aren't always_inline.

Jan

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

On 02/10/2019 09:24, Jan Beulich wrote:
> On 01.10.2019 17:37, Andrew Cooper wrote:
>> On 01/10/2019 15:32, Jan Beulich wrote:
>>> On 01.10.2019 14:51, Andrew Cooper wrote:
>>>> On 01/10/2019 13:21, Jan Beulich wrote:
>>>>> On 30.09.2019 20:24, Andrew Cooper wrote:
>>>>>> The code generation for barrier_nospec_true() is not correct.  We are taking a
>>>>>> perf it from the added fences, but not gaining any speculative safety.
>>>>> You want to be more specific here, I think: ISTR you saying that the LFENCEs
>>>>> get inserted at the wrong place.
>>>> Correct.
>>>>
>>>>>  IIRC we want them on either side of a
>>>>> conditional branch, i.e. immediately following a branch insn as well as right
>>>>> at the branch target.
>>>> Specifically, they must be at the head of both basic blocks following
>>>> the conditional jump.
>>>>
>>>>> I've taken, as a simple example,
>>>>> p2m_mem_access_sanity_check(), and this looks to be the way gcc9 has generated
>>>>> code (in a non-debug build). Hence either I'm mis-remembering what we want
>>>>> things to look like, or there's more to it than code generation simply being
>>>>> "not correct".
>>>> This example demonstrates the problem, and actually throws a further
>>>> spanner in the works of how make this safe, which hadn't occurred to me
>>>> before.
>>>>
>>>> The instruction stream from a caller of p2m_mem_access_sanity_check()
>>>> will look something like this:
>>>>
>>>> call p2m_mem_access_sanity_check
>>>>     ...
>>>>     lfence
>>>>     ...
>>>>     ret   
>>>> cmp $0, %eax
>>>> jne ...
>>>>
>>>> Which is unsafe, because the only safe way to arrange this code would be:
>>>>
>>>> call p2m_mem_access_sanity_check
>>>>     ...
>>>>     ret
>>>> cmp $0, %eax
>>>> jne 1f
>>>> lfence
>>>> ...
>>>> 1: lfence
>>>> ...
>>>>
>>>> There is absolutely no possible way for inline assembly to be used to
>>>> propagate this safety property across translation units.  This is going
>>>> to have to be an attribute of some form or another handled by the compiler.
>>> But you realize that this particular example is basically a more
>>> complex is_XYZ() check, which could be dealt with by inlining the
>>> function. Of course there are going to be larger functions where
>>> the result wants to be guarded like you say. But just like the
>>> addition of the nospec macros to various is_XYZ() functions is a
>>> manual operation (as long the compiler doesn't help), it would in
>>> that case be a matter of latching the return value into a local
>>> variable and using an appropriate guarding construct when
>>> evaluating it.
>> And this reasoning demonstrates yet another problem (this one was raised
>> at the meeting in Chicago).
>>
>> evaluate_nospec() is not a useful construct if it needs inserting at
>> every higher level predicate to result in safe code.  This is
>> boarderline-impossible to review for, and extremely easy to break
>> accidentally.
> I agree; since evaluate_nospec() insertion need is generally a hard
> to investigate / review action, I don#t consider this unexpected.
>
>>> So I'm afraid for now I still can't agree with your "not correct"
>>> assessment - the generated code in the example looks correct to
>>> me, and if further guarding was needed in users of this particular
>>> function, it would be those users which would need further
>>> massaging.
>> Safety against spectre v1 is not a matter of opinion.
>>
>> To protect against speculatively executing the wrong basic block, the
>> pipeline must execute the conditional jump first, *then* hit an lfence
>> to serialise the instruction stream and revector in the case of
>> incorrect speculation.
>>
>> The other way around is not safe.  Serialising the instruction stream
>> doesn't do anything to protect against the attacker taking control of a
>> later branch.
>>
>> The bigger problem is to do with classifying what we are protecting
>> against.  In the case of is_control_domain(), it is any action based on
>> the result of the decision.  For is_{pv,hvm}_domain(), is only (to a
>> first approximation) speculative type confusion into the pv/hvm unions
>> (which in practice extends to calling pv_/hvm_ functions as well).
>>
>> As for the real concrete breakages.  In a staging build with GCC 6
>>
>> $ objdump -d xen-syms | grep '<is_hvm_domain>:' | wc -l
>> 18
>> $ objdump -d xen-syms | grep '<is_pv_domain>:' | wc -l
>> 9
>>
>> All of which have the lfence too early to protect against speculative
>> type confusion.
> And all of which are because, other than I think it was originally
> intended, the functions still aren't always_inline.

Right, but if we force is_hvm_domain() to be always_inline, then
is_hvm_vcpu() gets out-of-lined.

This turns into a game of whack-a-mole, where any predicate wrapping
something with an embedded evaluate_nospec() breaks the safety.

~Andrew

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

On 02.10.2019 21:31, Andrew Cooper wrote:
> On 02/10/2019 09:24, Jan Beulich wrote:
>> On 01.10.2019 17:37, Andrew Cooper wrote:
>>> On 01/10/2019 15:32, Jan Beulich wrote:
>>>> On 01.10.2019 14:51, Andrew Cooper wrote:
>>>>> On 01/10/2019 13:21, Jan Beulich wrote:
>>>>>> On 30.09.2019 20:24, Andrew Cooper wrote:
>>>>>>> The code generation for barrier_nospec_true() is not correct.  We are taking a
>>>>>>> perf it from the added fences, but not gaining any speculative safety.
>>>>>> You want to be more specific here, I think: ISTR you saying that the LFENCEs
>>>>>> get inserted at the wrong place.
>>>>> Correct.
>>>>>
>>>>>>  IIRC we want them on either side of a
>>>>>> conditional branch, i.e. immediately following a branch insn as well as right
>>>>>> at the branch target.
>>>>> Specifically, they must be at the head of both basic blocks following
>>>>> the conditional jump.
>>>>>
>>>>>> I've taken, as a simple example,
>>>>>> p2m_mem_access_sanity_check(), and this looks to be the way gcc9 has generated
>>>>>> code (in a non-debug build). Hence either I'm mis-remembering what we want
>>>>>> things to look like, or there's more to it than code generation simply being
>>>>>> "not correct".
>>>>> This example demonstrates the problem, and actually throws a further
>>>>> spanner in the works of how make this safe, which hadn't occurred to me
>>>>> before.
>>>>>
>>>>> The instruction stream from a caller of p2m_mem_access_sanity_check()
>>>>> will look something like this:
>>>>>
>>>>> call p2m_mem_access_sanity_check
>>>>>     ...
>>>>>     lfence
>>>>>     ...
>>>>>     ret   
>>>>> cmp $0, %eax
>>>>> jne ...
>>>>>
>>>>> Which is unsafe, because the only safe way to arrange this code would be:
>>>>>
>>>>> call p2m_mem_access_sanity_check
>>>>>     ...
>>>>>     ret
>>>>> cmp $0, %eax
>>>>> jne 1f
>>>>> lfence
>>>>> ...
>>>>> 1: lfence
>>>>> ...
>>>>>
>>>>> There is absolutely no possible way for inline assembly to be used to
>>>>> propagate this safety property across translation units.  This is going
>>>>> to have to be an attribute of some form or another handled by the compiler.
>>>> But you realize that this particular example is basically a more
>>>> complex is_XYZ() check, which could be dealt with by inlining the
>>>> function. Of course there are going to be larger functions where
>>>> the result wants to be guarded like you say. But just like the
>>>> addition of the nospec macros to various is_XYZ() functions is a
>>>> manual operation (as long the compiler doesn't help), it would in
>>>> that case be a matter of latching the return value into a local
>>>> variable and using an appropriate guarding construct when
>>>> evaluating it.
>>> And this reasoning demonstrates yet another problem (this one was raised
>>> at the meeting in Chicago).
>>>
>>> evaluate_nospec() is not a useful construct if it needs inserting at
>>> every higher level predicate to result in safe code.  This is
>>> boarderline-impossible to review for, and extremely easy to break
>>> accidentally.
>> I agree; since evaluate_nospec() insertion need is generally a hard
>> to investigate / review action, I don#t consider this unexpected.
>>
>>>> So I'm afraid for now I still can't agree with your "not correct"
>>>> assessment - the generated code in the example looks correct to
>>>> me, and if further guarding was needed in users of this particular
>>>> function, it would be those users which would need further
>>>> massaging.
>>> Safety against spectre v1 is not a matter of opinion.
>>>
>>> To protect against speculatively executing the wrong basic block, the
>>> pipeline must execute the conditional jump first, *then* hit an lfence
>>> to serialise the instruction stream and revector in the case of
>>> incorrect speculation.
>>>
>>> The other way around is not safe.  Serialising the instruction stream
>>> doesn't do anything to protect against the attacker taking control of a
>>> later branch.
>>>
>>> The bigger problem is to do with classifying what we are protecting
>>> against.  In the case of is_control_domain(), it is any action based on
>>> the result of the decision.  For is_{pv,hvm}_domain(), is only (to a
>>> first approximation) speculative type confusion into the pv/hvm unions
>>> (which in practice extends to calling pv_/hvm_ functions as well).
>>>
>>> As for the real concrete breakages.  In a staging build with GCC 6
>>>
>>> $ objdump -d xen-syms | grep '<is_hvm_domain>:' | wc -l
>>> 18
>>> $ objdump -d xen-syms | grep '<is_pv_domain>:' | wc -l
>>> 9
>>>
>>> All of which have the lfence too early to protect against speculative
>>> type confusion.
>> And all of which are because, other than I think it was originally
>> intended, the functions still aren't always_inline.
> 
> Right, but if we force is_hvm_domain() to be always_inline, then
> is_hvm_vcpu() gets out-of-lined.
> 
> This turns into a game of whack-a-mole, where any predicate wrapping
> something with an embedded evaluate_nospec() breaks the safety.

That's understood, but what do you do? The consequence is that we'd
have to go through _all_ inline (predicate) functions, converting
them to always_inline as needed. Auditing non-inline ones (like
p2m_mem_access_sanity_check()) would need doing independently anyway,
and I'd consider this an independent aspect/issue.

Jan

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel