New kernels sometimes expose new registers in an unconditionnal
 manner.  This situation breaks backward migration as qemu notices
there are more registers to store on guest than supported in the
destination kerenl. This leads to a "failed to load
cpu:cpreg_vmstate_array_len" error.

A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
migration from a host kernel which features the commit to a destination
host that doesn't fail.

Currently QEMU is not using that feature so ignoring this latter
is not a problem. An easy way to fix the migration issue is to teach
qemu we don't care about that register and we can simply ignore it,
including its state migration.

This patch introduces a CPU property, under the form of an array of
reg indices which indicates which registers can be ignored.

The goal then is to set this property in machine type compats such
as:
static GlobalProperty arm_virt_kernel_compat_10_1[] = {
    /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
    { TYPE_ARM_CPU, "kvm-hidden-regs", "0x6030000000160003" },
}

Signed-off-by: Eric Auger <eric.auger@redhat.com>
---
 target/arm/cpu.h        |  4 ++++
 target/arm/kvm.c        | 36 ++++++++++++++++++++++++++++++++++--
 target/arm/trace-events |  2 ++
 3 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index c15d79a106..121b4372b2 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -1031,6 +1031,10 @@ struct ArchCPU {
     /* KVM steal time */
     OnOffAuto kvm_steal_time;
 
+    /* KVM registers that must be ignored/hidden */
+    uint64_t *kvm_hidden_regs;
+    uint32_t nr_kvm_hidden_regs;
+
     /* Uniprocessor system with MP extensions */
     bool mp_is_up;
 
diff --git a/target/arm/kvm.c b/target/arm/kvm.c
index 6672344855..67675781f4 100644
--- a/target/arm/kvm.c
+++ b/target/arm/kvm.c
@@ -39,6 +39,8 @@
 #include "qemu/log.h"
 #include "hw/acpi/acpi.h"
 #include "hw/acpi/ghes.h"
+#include "hw/qdev-properties.h"
+#include "hw/qdev-properties-system.h"
 #include "target/arm/gtimer.h"
 #include "migration/blocker.h"
 
@@ -483,6 +485,10 @@ static void kvm_steal_time_set(Object *obj, bool value, Error **errp)
     ARM_CPU(obj)->kvm_steal_time = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
 }
 
+static const Property arm_cpu_kvm_compat_hidden_regs_property =
+    DEFINE_PROP_ARRAY("kvm-hidden-regs", ARMCPU,
+                      nr_kvm_hidden_regs, kvm_hidden_regs, qdev_prop_uint64, uint64_t);
+
 /* KVM VCPU properties should be prefixed with "kvm-". */
 void kvm_arm_add_vcpu_properties(ARMCPU *cpu)
 {
@@ -504,6 +510,8 @@ void kvm_arm_add_vcpu_properties(ARMCPU *cpu)
                              kvm_steal_time_set);
     object_property_set_description(obj, "kvm-steal-time",
                                     "Set off to disable KVM steal time.");
+
+    qdev_property_add_static(DEVICE(obj), &arm_cpu_kvm_compat_hidden_regs_property);
 }
 
 bool kvm_arm_pmu_supported(void)
@@ -764,6 +772,26 @@ static bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx)
     }
 }
 
+/**
+ * kvm_vcpu_compat_hidden_reg:
+ * @cpu: ARMCPU
+ * @regidx: index of the register to check
+ *
+ * Depending on the CPU compat returns true if @regidx must be
+ * ignored during sync & migration
+ */
+static inline bool
+kvm_vcpu_compat_hidden_reg(ARMCPU *cpu, uint64_t regidx)
+{
+    for (int i = 0; i < cpu->nr_kvm_hidden_regs; i++) {
+        if (cpu->kvm_hidden_regs[i] == regidx) {
+            trace_kvm_vcpu_compat_hidden_reg(regidx);
+            return true;
+        }
+    }
+    return false;
+}
+
 /**
  * kvm_arm_init_cpreg_list:
  * @cpu: ARMCPU
@@ -798,7 +826,8 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu)
     qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64);
 
     for (i = 0, arraylen = 0; i < rlp->n; i++) {
-        if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) {
+        if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i]) ||
+            kvm_vcpu_compat_hidden_reg(cpu, rlp->reg[i])) {
             continue;
         }
         switch (rlp->reg[i] & KVM_REG_SIZE_MASK) {
@@ -814,6 +843,8 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu)
         arraylen++;
     }
 
+    trace_kvm_arm_init_cpreg_list_arraylen(arraylen);
+
     cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
     cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
     cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes,
@@ -825,7 +856,8 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu)
 
     for (i = 0, arraylen = 0; i < rlp->n; i++) {
         uint64_t regidx = rlp->reg[i];
-        if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) {
+        if (!kvm_arm_reg_syncs_via_cpreg_list(regidx) ||
+            kvm_vcpu_compat_hidden_reg(cpu, regidx)) {
             continue;
         }
         cpu->cpreg_indexes[arraylen] = regidx;
diff --git a/target/arm/trace-events b/target/arm/trace-events
index 4438dce7be..1b4ab0c683 100644
--- a/target/arm/trace-events
+++ b/target/arm/trace-events
@@ -13,3 +13,5 @@ arm_gt_update_irq(int timer, int irqstate) "gt_update_irq: timer %d irqstate %d"
 
 # kvm.c
 kvm_arm_fixup_msi_route(uint64_t iova, uint64_t gpa) "MSI iova = 0x%"PRIx64" is translated into 0x%"PRIx64
+kvm_arm_init_cpreg_list_arraylen(uint32_t arraylen) "arraylen=%d"
+kvm_vcpu_compat_hidden_reg(uint64_t regidx) "0x%"PRIx64" is hidden"
-- 
2.49.0

On Thu, Sep 11 2025, Eric Auger <eric.auger@redhat.com> wrote:

> New kernels sometimes expose new registers in an unconditionnal
>  manner.  This situation breaks backward migration as qemu notices
> there are more registers to store on guest than supported in the
> destination kerenl. This leads to a "failed to load
> cpu:cpreg_vmstate_array_len" error.
>
> A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
> pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
> Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
> migration from a host kernel which features the commit to a destination
> host that doesn't fail.
>
> Currently QEMU is not using that feature so ignoring this latter
> is not a problem. An easy way to fix the migration issue is to teach
> qemu we don't care about that register and we can simply ignore it,
> including its state migration.
>
> This patch introduces a CPU property, under the form of an array of
> reg indices which indicates which registers can be ignored.
>
> The goal then is to set this property in machine type compats such
> as:
> static GlobalProperty arm_virt_kernel_compat_10_1[] = {
>     /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
>     { TYPE_ARM_CPU, "kvm-hidden-regs", "0x6030000000160003" },
> }
>
> Signed-off-by: Eric Auger <eric.auger@redhat.com>
> ---
>  target/arm/cpu.h        |  4 ++++
>  target/arm/kvm.c        | 36 ++++++++++++++++++++++++++++++++++--
>  target/arm/trace-events |  2 ++
>  3 files changed, 40 insertions(+), 2 deletions(-)
>

(...)

> +/**
> + * kvm_vcpu_compat_hidden_reg:
> + * @cpu: ARMCPU
> + * @regidx: index of the register to check
> + *
> + * Depending on the CPU compat returns true if @regidx must be
> + * ignored during sync & migration

Maybe add some more explanation, either here or at the kvm_hidden_regs[]
def? So that people do not need to look at the introducing commit :)

"This is intended for when we know that we do not use the register to be
ignored, and want to keep compatibility."

> + */
> +static inline bool
> +kvm_vcpu_compat_hidden_reg(ARMCPU *cpu, uint64_t regidx)
> +{
> +    for (int i = 0; i < cpu->nr_kvm_hidden_regs; i++) {
> +        if (cpu->kvm_hidden_regs[i] == regidx) {
> +            trace_kvm_vcpu_compat_hidden_reg(regidx);
> +            return true;
> +        }
> +    }
> +    return false;
> +}

On 10/8/25 3:43 PM, Cornelia Huck wrote:
> On Thu, Sep 11 2025, Eric Auger <eric.auger@redhat.com> wrote:
>
>> New kernels sometimes expose new registers in an unconditionnal
>>  manner.  This situation breaks backward migration as qemu notices
>> there are more registers to store on guest than supported in the
>> destination kerenl. This leads to a "failed to load
>> cpu:cpreg_vmstate_array_len" error.
>>
>> A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
>> pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
>> Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
>> migration from a host kernel which features the commit to a destination
>> host that doesn't fail.
>>
>> Currently QEMU is not using that feature so ignoring this latter
>> is not a problem. An easy way to fix the migration issue is to teach
>> qemu we don't care about that register and we can simply ignore it,
>> including its state migration.
>>
>> This patch introduces a CPU property, under the form of an array of
>> reg indices which indicates which registers can be ignored.
>>
>> The goal then is to set this property in machine type compats such
>> as:
>> static GlobalProperty arm_virt_kernel_compat_10_1[] = {
>>     /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
>>     { TYPE_ARM_CPU, "kvm-hidden-regs", "0x6030000000160003" },
>> }
>>
>> Signed-off-by: Eric Auger <eric.auger@redhat.com>
>> ---
>>  target/arm/cpu.h        |  4 ++++
>>  target/arm/kvm.c        | 36 ++++++++++++++++++++++++++++++++++--
>>  target/arm/trace-events |  2 ++
>>  3 files changed, 40 insertions(+), 2 deletions(-)
>>
> (...)
>
>> +/**
>> + * kvm_vcpu_compat_hidden_reg:
>> + * @cpu: ARMCPU
>> + * @regidx: index of the register to check
>> + *
>> + * Depending on the CPU compat returns true if @regidx must be
>> + * ignored during sync & migration
> Maybe add some more explanation, either here or at the kvm_hidden_regs[]
> def? So that people do not need to look at the introducing commit :)
>
> "This is intended for when we know that we do not use the register to be
> ignored, and want to keep compatibility."

Added here and also in next patch, in cpu.h

Thanks!

Eric
>
>> + */
>> +static inline bool
>> +kvm_vcpu_compat_hidden_reg(ARMCPU *cpu, uint64_t regidx)
>> +{
>> +    for (int i = 0; i < cpu->nr_kvm_hidden_regs; i++) {
>> +        if (cpu->kvm_hidden_regs[i] == regidx) {
>> +            trace_kvm_vcpu_compat_hidden_reg(regidx);
>> +            return true;
>> +        }
>> +    }
>> +    return false;
>> +}

On Thu, 11 Sep 2025, Eric Auger wrote:
> New kernels sometimes expose new registers in an unconditionnal
> manner.  This situation breaks backward migration as qemu notices
> there are more registers to store on guest than supported in the
> destination kerenl. This leads to a "failed to load
> cpu:cpreg_vmstate_array_len" error.
>
> A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
> pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
> Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
> migration from a host kernel which features the commit to a destination
> host that doesn't fail.
>
> Currently QEMU is not using that feature so ignoring this latter
> is not a problem. An easy way to fix the migration issue is to teach
> qemu we don't care about that register and we can simply ignore it,
> including its state migration.
>
> This patch introduces a CPU property, under the form of an array of
> reg indices which indicates which registers can be ignored.
>
> The goal then is to set this property in machine type compats such
> as:
> static GlobalProperty arm_virt_kernel_compat_10_1[] = {
>    /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
>    { TYPE_ARM_CPU, "kvm-hidden-regs", "0x6030000000160003" },
> }

One thing worth noting - once this series lands:
https://lore.kernel.org/qemu-devel/20250801074730.28329-1-shameerkolothum@gmail.com/
we might need to add a bit more logic here. Either using the kvm
interfaces (only ignore KVM_REG_ARM_VENDOR_HYP_BMAP_2 when the register
value is 0) or qemu knowledge (only ignore KVM_REG_ARM_VENDOR_HYP_BMAP_2
when the impl-cpu property is not used).

Sebastian

Hi Sebastian,

On 9/18/25 6:16 PM, Sebastian Ott wrote:
> On Thu, 11 Sep 2025, Eric Auger wrote:
>> New kernels sometimes expose new registers in an unconditionnal
>> manner.  This situation breaks backward migration as qemu notices
>> there are more registers to store on guest than supported in the
>> destination kerenl. This leads to a "failed to load
>> cpu:cpreg_vmstate_array_len" error.
>>
>> A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
>> pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
>> Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
>> migration from a host kernel which features the commit to a destination
>> host that doesn't fail.
>>
>> Currently QEMU is not using that feature so ignoring this latter
>> is not a problem. An easy way to fix the migration issue is to teach
>> qemu we don't care about that register and we can simply ignore it,
>> including its state migration.
>>
>> This patch introduces a CPU property, under the form of an array of
>> reg indices which indicates which registers can be ignored.
>>
>> The goal then is to set this property in machine type compats such
>> as:
>> static GlobalProperty arm_virt_kernel_compat_10_1[] = {
>>    /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
>>    { TYPE_ARM_CPU, "kvm-hidden-regs", "0x6030000000160003" },
>> }
>
> One thing worth noting - once this series lands:
> https://lore.kernel.org/qemu-devel/20250801074730.28329-1-shameerkolothum@gmail.com/
>
> we might need to add a bit more logic here. Either using the kvm
> interfaces (only ignore KVM_REG_ARM_VENDOR_HYP_BMAP_2 when the register
> value is 0) or qemu knowledge (only ignore KVM_REG_ARM_VENDOR_HYP_BMAP_2
> when the impl-cpu property is not used).

Effectively if we "hide" KVM_REG_ARM_VENDOR_HYP_BMAP_2 on save/restore
we must enforce the reg is not used by userspace.

One way would be to test whether KVM_REG_ARM_VENDOR_HYP_BMAP_2 is hidden
in kvm_arm_target_impl_cpus_supported() and if it is, report false.
However for every new functionality in qemu it does not sound sensible
to check whether new KVM regs being used are anonymously hidden.

Another way could be to fail kvm_set_one_reg/kvm_get_one_reg in case the
register is hidden. That way in Shameer's series, kvm_arch_init_vcpu()
would fail if BMAP_2 is hidden, ie. in our example for all machines
types before 10.2. By the way adding Shameer.

Thanks

Eric


>
> Sebastian

On Fri, Oct 03 2025, Eric Auger <eric.auger@redhat.com> wrote:

> Hi Sebastian,
>
> On 9/18/25 6:16 PM, Sebastian Ott wrote:
>> On Thu, 11 Sep 2025, Eric Auger wrote:
>>> New kernels sometimes expose new registers in an unconditionnal
>>> manner.  This situation breaks backward migration as qemu notices
>>> there are more registers to store on guest than supported in the
>>> destination kerenl. This leads to a "failed to load
>>> cpu:cpreg_vmstate_array_len" error.
>>>
>>> A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
>>> pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
>>> Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
>>> migration from a host kernel which features the commit to a destination
>>> host that doesn't fail.
>>>
>>> Currently QEMU is not using that feature so ignoring this latter
>>> is not a problem. An easy way to fix the migration issue is to teach
>>> qemu we don't care about that register and we can simply ignore it,
>>> including its state migration.
>>>
>>> This patch introduces a CPU property, under the form of an array of
>>> reg indices which indicates which registers can be ignored.
>>>
>>> The goal then is to set this property in machine type compats such
>>> as:
>>> static GlobalProperty arm_virt_kernel_compat_10_1[] = {
>>>    /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
>>>    { TYPE_ARM_CPU, "kvm-hidden-regs", "0x6030000000160003" },
>>> }
>>
>> One thing worth noting - once this series lands:
>> https://lore.kernel.org/qemu-devel/20250801074730.28329-1-shameerkolothum@gmail.com/
>>
>> we might need to add a bit more logic here. Either using the kvm
>> interfaces (only ignore KVM_REG_ARM_VENDOR_HYP_BMAP_2 when the register
>> value is 0) or qemu knowledge (only ignore KVM_REG_ARM_VENDOR_HYP_BMAP_2
>> when the impl-cpu property is not used).
>
> Effectively if we "hide" KVM_REG_ARM_VENDOR_HYP_BMAP_2 on save/restore
> we must enforce the reg is not used by userspace.
>
> One way would be to test whether KVM_REG_ARM_VENDOR_HYP_BMAP_2 is hidden
> in kvm_arm_target_impl_cpus_supported() and if it is, report false.
> However for every new functionality in qemu it does not sound sensible
> to check whether new KVM regs being used are anonymously hidden.
>
> Another way could be to fail kvm_set_one_reg/kvm_get_one_reg in case the
> register is hidden. That way in Shameer's series, kvm_arch_init_vcpu()
> would fail if BMAP_2 is hidden, ie. in our example for all machines
> types before 10.2. By the way adding Shameer.

I think tying this to the state of the reg (hidden or not) is less
error-prone (I'd assume we'd have different ways of detecting whether
something is used for future cases, and "is the reg hidden?" would work
in all cases.) We'd need to tie migration to matching machine versions
anyway, I think.

On 10/8/25 3:49 PM, Cornelia Huck wrote:
> On Fri, Oct 03 2025, Eric Auger <eric.auger@redhat.com> wrote:
>
>> Hi Sebastian,
>>
>> On 9/18/25 6:16 PM, Sebastian Ott wrote:
>>> On Thu, 11 Sep 2025, Eric Auger wrote:
>>>> New kernels sometimes expose new registers in an unconditionnal
>>>> manner.  This situation breaks backward migration as qemu notices
>>>> there are more registers to store on guest than supported in the
>>>> destination kerenl. This leads to a "failed to load
>>>> cpu:cpreg_vmstate_array_len" error.
>>>>
>>>> A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
>>>> pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
>>>> Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
>>>> migration from a host kernel which features the commit to a destination
>>>> host that doesn't fail.
>>>>
>>>> Currently QEMU is not using that feature so ignoring this latter
>>>> is not a problem. An easy way to fix the migration issue is to teach
>>>> qemu we don't care about that register and we can simply ignore it,
>>>> including its state migration.
>>>>
>>>> This patch introduces a CPU property, under the form of an array of
>>>> reg indices which indicates which registers can be ignored.
>>>>
>>>> The goal then is to set this property in machine type compats such
>>>> as:
>>>> static GlobalProperty arm_virt_kernel_compat_10_1[] = {
>>>>    /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
>>>>    { TYPE_ARM_CPU, "kvm-hidden-regs", "0x6030000000160003" },
>>>> }
>>> One thing worth noting - once this series lands:
>>> https://lore.kernel.org/qemu-devel/20250801074730.28329-1-shameerkolothum@gmail.com/
>>>
>>> we might need to add a bit more logic here. Either using the kvm
>>> interfaces (only ignore KVM_REG_ARM_VENDOR_HYP_BMAP_2 when the register
>>> value is 0) or qemu knowledge (only ignore KVM_REG_ARM_VENDOR_HYP_BMAP_2
>>> when the impl-cpu property is not used).
>> Effectively if we "hide" KVM_REG_ARM_VENDOR_HYP_BMAP_2 on save/restore
>> we must enforce the reg is not used by userspace.
>>
>> One way would be to test whether KVM_REG_ARM_VENDOR_HYP_BMAP_2 is hidden
>> in kvm_arm_target_impl_cpus_supported() and if it is, report false.
>> However for every new functionality in qemu it does not sound sensible
>> to check whether new KVM regs being used are anonymously hidden.
>>
>> Another way could be to fail kvm_set_one_reg/kvm_get_one_reg in case the
>> register is hidden. That way in Shameer's series, kvm_arch_init_vcpu()
>> would fail if BMAP_2 is hidden, ie. in our example for all machines
>> types before 10.2. By the way adding Shameer.
> I think tying this to the state of the reg (hidden or not) is less
> error-prone (I'd assume we'd have different ways of detecting whether
> something is used for future cases, and "is the reg hidden?" would work
> in all cases.) We'd need to tie migration to matching machine versions
> anyway, I think.
>
I guess you suggest to check the hidden/fake state in

kvm_set_one_reg/kvm_get_one_reg too. One issue is those helpers are arch agnostic. I would need to either introduce a callback in the CPU class to check the actual status or add the props in the parent CPU object. Or introduce a KVM IOTCL to teach KVM a reg shall never be accessed.

Thoughts?

Thanks

Eric

On Tue, Oct 14 2025, Eric Auger <eric.auger@redhat.com> wrote:

> On 10/8/25 3:49 PM, Cornelia Huck wrote:
>> On Fri, Oct 03 2025, Eric Auger <eric.auger@redhat.com> wrote:
>>
>>> Hi Sebastian,
>>>
>>> On 9/18/25 6:16 PM, Sebastian Ott wrote:
>>>> On Thu, 11 Sep 2025, Eric Auger wrote:
>>>>> New kernels sometimes expose new registers in an unconditionnal
>>>>> manner.  This situation breaks backward migration as qemu notices
>>>>> there are more registers to store on guest than supported in the
>>>>> destination kerenl. This leads to a "failed to load
>>>>> cpu:cpreg_vmstate_array_len" error.
>>>>>
>>>>> A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
>>>>> pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
>>>>> Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
>>>>> migration from a host kernel which features the commit to a destination
>>>>> host that doesn't fail.
>>>>>
>>>>> Currently QEMU is not using that feature so ignoring this latter
>>>>> is not a problem. An easy way to fix the migration issue is to teach
>>>>> qemu we don't care about that register and we can simply ignore it,
>>>>> including its state migration.
>>>>>
>>>>> This patch introduces a CPU property, under the form of an array of
>>>>> reg indices which indicates which registers can be ignored.
>>>>>
>>>>> The goal then is to set this property in machine type compats such
>>>>> as:
>>>>> static GlobalProperty arm_virt_kernel_compat_10_1[] = {
>>>>>    /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
>>>>>    { TYPE_ARM_CPU, "kvm-hidden-regs", "0x6030000000160003" },
>>>>> }
>>>> One thing worth noting - once this series lands:
>>>> https://lore.kernel.org/qemu-devel/20250801074730.28329-1-shameerkolothum@gmail.com/
>>>>
>>>> we might need to add a bit more logic here. Either using the kvm
>>>> interfaces (only ignore KVM_REG_ARM_VENDOR_HYP_BMAP_2 when the register
>>>> value is 0) or qemu knowledge (only ignore KVM_REG_ARM_VENDOR_HYP_BMAP_2
>>>> when the impl-cpu property is not used).
>>> Effectively if we "hide" KVM_REG_ARM_VENDOR_HYP_BMAP_2 on save/restore
>>> we must enforce the reg is not used by userspace.
>>>
>>> One way would be to test whether KVM_REG_ARM_VENDOR_HYP_BMAP_2 is hidden
>>> in kvm_arm_target_impl_cpus_supported() and if it is, report false.
>>> However for every new functionality in qemu it does not sound sensible
>>> to check whether new KVM regs being used are anonymously hidden.
>>>
>>> Another way could be to fail kvm_set_one_reg/kvm_get_one_reg in case the
>>> register is hidden. That way in Shameer's series, kvm_arch_init_vcpu()
>>> would fail if BMAP_2 is hidden, ie. in our example for all machines
>>> types before 10.2. By the way adding Shameer.
>> I think tying this to the state of the reg (hidden or not) is less
>> error-prone (I'd assume we'd have different ways of detecting whether
>> something is used for future cases, and "is the reg hidden?" would work
>> in all cases.) We'd need to tie migration to matching machine versions
>> anyway, I think.
>>
> I guess you suggest to check the hidden/fake state in
>
> kvm_set_one_reg/kvm_get_one_reg too. One issue is those helpers are arch agnostic. I would need to either introduce a callback in the CPU class to check the actual status or add the props in the parent CPU object. Or introduce a KVM IOTCL to teach KVM a reg shall never be accessed.

Aren't they always called from arch code, though? We could easily wrap
them for arm, I think. (Unless there's some more generic interest in
conditional ONE_REG accesses.)

Hi Connie,

On 10/15/25 3:12 PM, Cornelia Huck wrote:
> On Tue, Oct 14 2025, Eric Auger <eric.auger@redhat.com> wrote:
>
>> On 10/8/25 3:49 PM, Cornelia Huck wrote:
>>> On Fri, Oct 03 2025, Eric Auger <eric.auger@redhat.com> wrote:
>>>
>>>> Hi Sebastian,
>>>>
>>>> On 9/18/25 6:16 PM, Sebastian Ott wrote:
>>>>> On Thu, 11 Sep 2025, Eric Auger wrote:
>>>>>> New kernels sometimes expose new registers in an unconditionnal
>>>>>> manner.  This situation breaks backward migration as qemu notices
>>>>>> there are more registers to store on guest than supported in the
>>>>>> destination kerenl. This leads to a "failed to load
>>>>>> cpu:cpreg_vmstate_array_len" error.
>>>>>>
>>>>>> A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
>>>>>> pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
>>>>>> Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
>>>>>> migration from a host kernel which features the commit to a destination
>>>>>> host that doesn't fail.
>>>>>>
>>>>>> Currently QEMU is not using that feature so ignoring this latter
>>>>>> is not a problem. An easy way to fix the migration issue is to teach
>>>>>> qemu we don't care about that register and we can simply ignore it,
>>>>>> including its state migration.
>>>>>>
>>>>>> This patch introduces a CPU property, under the form of an array of
>>>>>> reg indices which indicates which registers can be ignored.
>>>>>>
>>>>>> The goal then is to set this property in machine type compats such
>>>>>> as:
>>>>>> static GlobalProperty arm_virt_kernel_compat_10_1[] = {
>>>>>>    /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
>>>>>>    { TYPE_ARM_CPU, "kvm-hidden-regs", "0x6030000000160003" },
>>>>>> }
>>>>> One thing worth noting - once this series lands:
>>>>> https://lore.kernel.org/qemu-devel/20250801074730.28329-1-shameerkolothum@gmail.com/
>>>>>
>>>>> we might need to add a bit more logic here. Either using the kvm
>>>>> interfaces (only ignore KVM_REG_ARM_VENDOR_HYP_BMAP_2 when the register
>>>>> value is 0) or qemu knowledge (only ignore KVM_REG_ARM_VENDOR_HYP_BMAP_2
>>>>> when the impl-cpu property is not used).
>>>> Effectively if we "hide" KVM_REG_ARM_VENDOR_HYP_BMAP_2 on save/restore
>>>> we must enforce the reg is not used by userspace.
>>>>
>>>> One way would be to test whether KVM_REG_ARM_VENDOR_HYP_BMAP_2 is hidden
>>>> in kvm_arm_target_impl_cpus_supported() and if it is, report false.
>>>> However for every new functionality in qemu it does not sound sensible
>>>> to check whether new KVM regs being used are anonymously hidden.
>>>>
>>>> Another way could be to fail kvm_set_one_reg/kvm_get_one_reg in case the
>>>> register is hidden. That way in Shameer's series, kvm_arch_init_vcpu()
>>>> would fail if BMAP_2 is hidden, ie. in our example for all machines
>>>> types before 10.2. By the way adding Shameer.
>>> I think tying this to the state of the reg (hidden or not) is less
>>> error-prone (I'd assume we'd have different ways of detecting whether
>>> something is used for future cases, and "is the reg hidden?" would work
>>> in all cases.) We'd need to tie migration to matching machine versions
>>> anyway, I think.
>>>
>> I guess you suggest to check the hidden/fake state in
>>
>> kvm_set_one_reg/kvm_get_one_reg too. One issue is those helpers are arch agnostic. I would need to either introduce a callback in the CPU class to check the actual status or add the props in the parent CPU object. Or introduce a KVM IOTCL to teach KVM a reg shall never be accessed.
> Aren't they always called from arch code, though? We could easily wrap
> them for arm, I think. (Unless there's some more generic interest in
> conditional ONE_REG accesses.)

For the time being I chose to add a new CPU class callback to check
whether the reg is hidden in kvm_set/get_one_reg.
Let's wait for other comments ... ;-)

Thanks

Eric
>

On Thu, 11 Sep 2025, Eric Auger wrote:
> New kernels sometimes expose new registers in an unconditionnal
> manner.  This situation breaks backward migration as qemu notices
> there are more registers to store on guest than supported in the
> destination kerenl. This leads to a "failed to load
> cpu:cpreg_vmstate_array_len" error.
>
> A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
> pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
> Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
> migration from a host kernel which features the commit to a destination
> host that doesn't fail.
>
> Currently QEMU is not using that feature so ignoring this latter
> is not a problem. An easy way to fix the migration issue is to teach
> qemu we don't care about that register and we can simply ignore it,
> including its state migration.
>
> This patch introduces a CPU property, under the form of an array of
> reg indices which indicates which registers can be ignored.
>
> The goal then is to set this property in machine type compats such
> as:
> static GlobalProperty arm_virt_kernel_compat_10_1[] = {
>    /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
>    { TYPE_ARM_CPU, "kvm-hidden-regs", "0x6030000000160003" },
> }
>
> Signed-off-by: Eric Auger <eric.auger@redhat.com>
> ---
> target/arm/cpu.h        |  4 ++++
> target/arm/kvm.c        | 36 ++++++++++++++++++++++++++++++++++--
> target/arm/trace-events |  2 ++
> 3 files changed, 40 insertions(+), 2 deletions(-)
>

Reviewed-by: Sebastian Ott <sebott@redhat.com>