1. Patchew
  2. QEMU
  3. [PATCH 0/4] target/i386: add support for FRED
  4. View patch

[PATCH 4/4] target/i386: add live migration support for FRED

Xin Li posted 4 patches 2 years, 5 months ago
Maintainers: Paolo Bonzini <pbonzini@redhat.com>, Marcelo Tosatti <mtosatti@redhat.com>
There is a newer version of this series
[PATCH 4/4] target/i386: add live migration support for FRED
Posted by Xin Li 2 years, 5 months ago 
FRED CPU states are managed in 10 FRED MSRs, in addtion to a few existing
CPU registers and MSRs, e.g., the CR4.FRED bit.

Add the 10 new FRED MSRs to x86 CPUArchState for live migration support.

Tested-by: Shan Kang <shan.kang@intel.com>
Signed-off-by: Xin Li <xin3.li@intel.com>
---
 target/i386/cpu.h     | 24 +++++++++++++++++++
 target/i386/kvm/kvm.c | 54 +++++++++++++++++++++++++++++++++++++++++++
 target/i386/machine.c | 10 ++++++++
 3 files changed, 88 insertions(+)

diff --git a/target/i386/cpu.h b/target/i386/cpu.h
index 924819a64c..a36a1a58c4 100644
--- a/target/i386/cpu.h
+++ b/target/i386/cpu.h
@@ -529,6 +529,20 @@ typedef enum X86Seg {
 #define MSR_IA32_XFD                    0x000001c4
 #define MSR_IA32_XFD_ERR                0x000001c5
 
+#define MSR_IA32_PL0_SSP                0x000006a4       /* Stack level 0 shadow stack pointer in ring 0 */
+
+/* FRED MSRs */
+#define MSR_IA32_FRED_RSP0              0x000001cc       /* Stack level 0 regular stack pointer */
+#define MSR_IA32_FRED_RSP1              0x000001cd       /* Stack level 1 regular stack pointer */
+#define MSR_IA32_FRED_RSP2              0x000001ce       /* Stack level 2 regular stack pointer */
+#define MSR_IA32_FRED_RSP3              0x000001cf       /* Stack level 3 regular stack pointer */
+#define MSR_IA32_FRED_STKLVLS           0x000001d0       /* FRED exception stack levels */
+#define MSR_IA32_FRED_SSP0              MSR_IA32_PL0_SSP /* Stack level 0 shadow stack pointer in ring 0 */
+#define MSR_IA32_FRED_SSP1              0x000001d1       /* Stack level 1 shadow stack pointer in ring 0 */
+#define MSR_IA32_FRED_SSP2              0x000001d2       /* Stack level 2 shadow stack pointer in ring 0 */
+#define MSR_IA32_FRED_SSP3              0x000001d3       /* Stack level 3 shadow stack pointer in ring 0 */
+#define MSR_IA32_FRED_CONFIG            0x000001d4       /* FRED Entrypoint and interrupt stack level */
+
 #define MSR_IA32_BNDCFGS                0x00000d90
 #define MSR_IA32_XSS                    0x00000da0
 #define MSR_IA32_UMWAIT_CONTROL         0xe1
@@ -1680,6 +1694,16 @@ typedef struct CPUArchState {
     target_ulong cstar;
     target_ulong fmask;
     target_ulong kernelgsbase;
+    target_ulong fred_rsp0;
+    target_ulong fred_rsp1;
+    target_ulong fred_rsp2;
+    target_ulong fred_rsp3;
+    target_ulong fred_stklvls;
+    target_ulong fred_ssp0;
+    target_ulong fred_ssp1;
+    target_ulong fred_ssp2;
+    target_ulong fred_ssp3;
+    target_ulong fred_config;
 #endif
 
     uint64_t tsc_adjust;
diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c
index 639a242ad8..4b241c82d8 100644
--- a/target/i386/kvm/kvm.c
+++ b/target/i386/kvm/kvm.c
@@ -3401,6 +3401,18 @@ static int kvm_put_msrs(X86CPU *cpu, int level)
         kvm_msr_entry_add(cpu, MSR_KERNELGSBASE, env->kernelgsbase);
         kvm_msr_entry_add(cpu, MSR_FMASK, env->fmask);
         kvm_msr_entry_add(cpu, MSR_LSTAR, env->lstar);
+        if (env->features[FEAT_7_1_EAX] & CPUID_7_1_EAX_FRED) {
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP0, env->fred_rsp0);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP1, env->fred_rsp1);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP2, env->fred_rsp2);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP3, env->fred_rsp3);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_STKLVLS, env->fred_stklvls);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP0, env->fred_ssp0);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP1, env->fred_ssp1);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP2, env->fred_ssp2);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP3, env->fred_ssp3);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_CONFIG, env->fred_config);
+        }
     }
 #endif
 
@@ -3901,6 +3913,18 @@ static int kvm_get_msrs(X86CPU *cpu)
         kvm_msr_entry_add(cpu, MSR_KERNELGSBASE, 0);
         kvm_msr_entry_add(cpu, MSR_FMASK, 0);
         kvm_msr_entry_add(cpu, MSR_LSTAR, 0);
+        if (env->features[FEAT_7_1_EAX] & CPUID_7_1_EAX_FRED) {
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP0, 0);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP1, 0);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP2, 0);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP3, 0);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_STKLVLS, 0);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP0, 0);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP1, 0);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP2, 0);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP3, 0);
+            kvm_msr_entry_add(cpu, MSR_IA32_FRED_CONFIG, 0);
+        }
     }
 #endif
     kvm_msr_entry_add(cpu, MSR_KVM_SYSTEM_TIME, 0);
@@ -4123,6 +4147,36 @@ static int kvm_get_msrs(X86CPU *cpu)
         case MSR_LSTAR:
             env->lstar = msrs[i].data;
             break;
+        case MSR_IA32_FRED_RSP0:
+            env->fred_rsp0 = msrs[i].data;
+            break;
+        case MSR_IA32_FRED_RSP1:
+            env->fred_rsp1 = msrs[i].data;
+            break;
+        case MSR_IA32_FRED_RSP2:
+            env->fred_rsp2 = msrs[i].data;
+            break;
+        case MSR_IA32_FRED_RSP3:
+            env->fred_rsp3 = msrs[i].data;
+            break;
+        case MSR_IA32_FRED_STKLVLS:
+            env->fred_stklvls = msrs[i].data;
+            break;
+        case MSR_IA32_FRED_SSP0:
+            env->fred_ssp0 = msrs[i].data;
+            break;
+        case MSR_IA32_FRED_SSP1:
+            env->fred_ssp1 = msrs[i].data;
+            break;
+        case MSR_IA32_FRED_SSP2:
+            env->fred_ssp2 = msrs[i].data;
+            break;
+        case MSR_IA32_FRED_SSP3:
+            env->fred_ssp3 = msrs[i].data;
+            break;
+        case MSR_IA32_FRED_CONFIG:
+            env->fred_config = msrs[i].data;
+            break;
 #endif
         case MSR_IA32_TSC:
             env->tsc = msrs[i].data;
diff --git a/target/i386/machine.c b/target/i386/machine.c
index c7ac8084b2..5c722a49c5 100644
--- a/target/i386/machine.c
+++ b/target/i386/machine.c
@@ -1652,6 +1652,16 @@ const VMStateDescription vmstate_x86_cpu = {
         VMSTATE_UINT64(env.cstar, X86CPU),
         VMSTATE_UINT64(env.fmask, X86CPU),
         VMSTATE_UINT64(env.kernelgsbase, X86CPU),
+        VMSTATE_UINT64(env.fred_rsp0, X86CPU),
+        VMSTATE_UINT64(env.fred_rsp1, X86CPU),
+        VMSTATE_UINT64(env.fred_rsp2, X86CPU),
+        VMSTATE_UINT64(env.fred_rsp3, X86CPU),
+        VMSTATE_UINT64(env.fred_stklvls, X86CPU),
+        VMSTATE_UINT64(env.fred_ssp0, X86CPU),
+        VMSTATE_UINT64(env.fred_ssp1, X86CPU),
+        VMSTATE_UINT64(env.fred_ssp2, X86CPU),
+        VMSTATE_UINT64(env.fred_ssp3, X86CPU),
+        VMSTATE_UINT64(env.fred_config, X86CPU),
 #endif
         VMSTATE_UINT32(env.smbase, X86CPU),
 
-- 
2.34.1
Re: [PATCH 4/4] target/i386: add live migration support for FRED
Posted by Yang, Weijiang 2 years, 4 months ago 
On 9/1/2023 1:30 PM, Li, Xin3 wrote:
> FRED CPU states are managed in 10 FRED MSRs, in addtion to a few existing
> CPU registers and MSRs, e.g., the CR4.FRED bit.
>
> Add the 10 new FRED MSRs to x86 CPUArchState for live migration support.
>
> Tested-by: Shan Kang <shan.kang@intel.com>
> Signed-off-by: Xin Li <xin3.li@intel.com>
> ---
>   target/i386/cpu.h     | 24 +++++++++++++++++++
>   target/i386/kvm/kvm.c | 54 +++++++++++++++++++++++++++++++++++++++++++
>   target/i386/machine.c | 10 ++++++++
>   3 files changed, 88 insertions(+)
>
> diff --git a/target/i386/cpu.h b/target/i386/cpu.h
> index 924819a64c..a36a1a58c4 100644
> --- a/target/i386/cpu.h
> +++ b/target/i386/cpu.h
> @@ -529,6 +529,20 @@ typedef enum X86Seg {
>   #define MSR_IA32_XFD                    0x000001c4
>   #define MSR_IA32_XFD_ERR                0x000001c5
>   
> +#define MSR_IA32_PL0_SSP                0x000006a4       /* Stack level 0 shadow stack pointer in ring 0 */
> +
> +/* FRED MSRs */
> +#define MSR_IA32_FRED_RSP0              0x000001cc       /* Stack level 0 regular stack pointer */
> +#define MSR_IA32_FRED_RSP1              0x000001cd       /* Stack level 1 regular stack pointer */
> +#define MSR_IA32_FRED_RSP2              0x000001ce       /* Stack level 2 regular stack pointer */
> +#define MSR_IA32_FRED_RSP3              0x000001cf       /* Stack level 3 regular stack pointer */
> +#define MSR_IA32_FRED_STKLVLS           0x000001d0       /* FRED exception stack levels */
> +#define MSR_IA32_FRED_SSP0              MSR_IA32_PL0_SSP /* Stack level 0 shadow stack pointer in ring 0 */
> +#define MSR_IA32_FRED_SSP1              0x000001d1       /* Stack level 1 shadow stack pointer in ring 0 */
> +#define MSR_IA32_FRED_SSP2              0x000001d2       /* Stack level 2 shadow stack pointer in ring 0 */
> +#define MSR_IA32_FRED_SSP3              0x000001d3       /* Stack level 3 shadow stack pointer in ring 0 */
> +#define MSR_IA32_FRED_CONFIG            0x000001d4       /* FRED Entrypoint and interrupt stack level */
> +
>   #define MSR_IA32_BNDCFGS                0x00000d90
>   #define MSR_IA32_XSS                    0x00000da0
>   #define MSR_IA32_UMWAIT_CONTROL         0xe1
> @@ -1680,6 +1694,16 @@ typedef struct CPUArchState {
>       target_ulong cstar;
>       target_ulong fmask;
>       target_ulong kernelgsbase;
> +    target_ulong fred_rsp0;
> +    target_ulong fred_rsp1;
> +    target_ulong fred_rsp2;
> +    target_ulong fred_rsp3;
> +    target_ulong fred_stklvls;
> +    target_ulong fred_ssp0;
> +    target_ulong fred_ssp1;
> +    target_ulong fred_ssp2;
> +    target_ulong fred_ssp3;
> +    target_ulong fred_config;
>   #endif
>   
>       uint64_t tsc_adjust;
> diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c
> index 639a242ad8..4b241c82d8 100644
> --- a/target/i386/kvm/kvm.c
> +++ b/target/i386/kvm/kvm.c
> @@ -3401,6 +3401,18 @@ static int kvm_put_msrs(X86CPU *cpu, int level)
>           kvm_msr_entry_add(cpu, MSR_KERNELGSBASE, env->kernelgsbase);
>           kvm_msr_entry_add(cpu, MSR_FMASK, env->fmask);
>           kvm_msr_entry_add(cpu, MSR_LSTAR, env->lstar);
> +        if (env->features[FEAT_7_1_EAX] & CPUID_7_1_EAX_FRED) {
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP0, env->fred_rsp0);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP1, env->fred_rsp1);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP2, env->fred_rsp2);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP3, env->fred_rsp3);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_STKLVLS, env->fred_stklvls);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP0, env->fred_ssp0);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP1, env->fred_ssp1);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP2, env->fred_ssp2);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP3, env->fred_ssp3);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_CONFIG, env->fred_config);
> +        }
>       }
>   #endif
>   
> @@ -3901,6 +3913,18 @@ static int kvm_get_msrs(X86CPU *cpu)
>           kvm_msr_entry_add(cpu, MSR_KERNELGSBASE, 0);
>           kvm_msr_entry_add(cpu, MSR_FMASK, 0);
>           kvm_msr_entry_add(cpu, MSR_LSTAR, 0);
> +        if (env->features[FEAT_7_1_EAX] & CPUID_7_1_EAX_FRED) {
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP0, 0);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP1, 0);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP2, 0);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_RSP3, 0);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_STKLVLS, 0);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP0, 0);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP1, 0);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP2, 0);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_SSP3, 0);
> +            kvm_msr_entry_add(cpu, MSR_IA32_FRED_CONFIG, 0);
> +        }
>       }
>   #endif
>       kvm_msr_entry_add(cpu, MSR_KVM_SYSTEM_TIME, 0);
> @@ -4123,6 +4147,36 @@ static int kvm_get_msrs(X86CPU *cpu)
>           case MSR_LSTAR:
>               env->lstar = msrs[i].data;
>               break;
> +        case MSR_IA32_FRED_RSP0:
> +            env->fred_rsp0 = msrs[i].data;
> +            break;
> +        case MSR_IA32_FRED_RSP1:
> +            env->fred_rsp1 = msrs[i].data;
> +            break;
> +        case MSR_IA32_FRED_RSP2:
> +            env->fred_rsp2 = msrs[i].data;
> +            break;
> +        case MSR_IA32_FRED_RSP3:
> +            env->fred_rsp3 = msrs[i].data;
> +            break;
> +        case MSR_IA32_FRED_STKLVLS:
> +            env->fred_stklvls = msrs[i].data;
> +            break;
> +        case MSR_IA32_FRED_SSP0:
> +            env->fred_ssp0 = msrs[i].data;
> +            break;
> +        case MSR_IA32_FRED_SSP1:
> +            env->fred_ssp1 = msrs[i].data;
> +            break;
> +        case MSR_IA32_FRED_SSP2:
> +            env->fred_ssp2 = msrs[i].data;
> +            break;
> +        case MSR_IA32_FRED_SSP3:
> +            env->fred_ssp3 = msrs[i].data;
> +            break;
> +        case MSR_IA32_FRED_CONFIG:
> +            env->fred_config = msrs[i].data;
> +            break;
>   #endif
>           case MSR_IA32_TSC:
>               env->tsc = msrs[i].data;
> diff --git a/target/i386/machine.c b/target/i386/machine.c
> index c7ac8084b2..5c722a49c5 100644
> --- a/target/i386/machine.c
> +++ b/target/i386/machine.c
> @@ -1652,6 +1652,16 @@ const VMStateDescription vmstate_x86_cpu = {
>           VMSTATE_UINT64(env.cstar, X86CPU),
>           VMSTATE_UINT64(env.fmask, X86CPU),
>           VMSTATE_UINT64(env.kernelgsbase, X86CPU),
> +        VMSTATE_UINT64(env.fred_rsp0, X86CPU),
> +        VMSTATE_UINT64(env.fred_rsp1, X86CPU),
> +        VMSTATE_UINT64(env.fred_rsp2, X86CPU),
> +        VMSTATE_UINT64(env.fred_rsp3, X86CPU),
> +        VMSTATE_UINT64(env.fred_stklvls, X86CPU),
> +        VMSTATE_UINT64(env.fred_ssp0, X86CPU),
> +        VMSTATE_UINT64(env.fred_ssp1, X86CPU),
> +        VMSTATE_UINT64(env.fred_ssp2, X86CPU),
> +        VMSTATE_UINT64(env.fred_ssp3, X86CPU),
> +        VMSTATE_UINT64(env.fred_config, X86CPU),
>   #endif
>           VMSTATE_UINT32(env.smbase, X86CPU),
>   

IMO, it's better to split this patch into two, one is for FRED MSR access, the other is
for LM support.

And move the FRED MSRs from basic x86_cpu part to .subsections part because FRED
doesn't belong to basic CPU registers.  In the .subsections part, you may define a struct
to hold FRED MSRs then add  .needed callback helper for QEMU to query whether FRED
MSRs are necessary for live migration.

See other features, e.g., vmstate_msr_intel_sgx
RE: [PATCH 4/4] target/i386: add live migration support for FRED
Posted by Li, Xin3 2 years, 4 months ago 
> > FRED CPU states are managed in 10 FRED MSRs, in addtion to a few
> > existing CPU registers and MSRs, e.g., the CR4.FRED bit.
> >
> > Add the 10 new FRED MSRs to x86 CPUArchState for live migration support.
> >
> 
> IMO, it's better to split this patch into two, one is for FRED MSR access, the other
> is for LM support.
> 
> And move the FRED MSRs from basic x86_cpu part to .subsections part because
> FRED doesn't belong to basic CPU registers.  In the .subsections part, you may
> define a struct to hold FRED MSRs then add  .needed callback helper for QEMU to
> query whether FRED MSRs are necessary for live migration.

Good point!

> 
> See other features, e.g., vmstate_msr_intel_sgx

Patchew 2.1-devel-b67e4c2

© 2016 - 2026 Red Hat, Inc.