New kernels sometimes expose new registers in an unconditionnal
 manner.  This situation breaks backward migration as qemu notices
there are more registers to store on guest than supported in the
destination kerenl. This leads to a "failed to load
cpu:cpreg_vmstate_array_len" error.

A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
migration from a host kernel which features the commit to a destination
host that doesn't fail.

Currently QEMU is not using that feature so ignoring this latter
is not a problem. An easy way to fix the migration issue is to teach
qemu we don't care about that register and we can simply ignore it,
including its state migration.

This patch introduces a CPU property, under the form of an array of
reg indices which indicates which registers can be ignored.

The goal then is to set this property in machine type compats such
as:
static GlobalProperty arm_virt_kernel_compat_10_1[] = {
    /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
    { TYPE_ARM_CPU, "kvm-hidden-regs", "0x6030000000160003" },
}

Signed-off-by: Eric Auger <eric.auger@redhat.com>
---
 target/arm/cpu.h        |  4 ++++
 target/arm/kvm.c        | 36 ++++++++++++++++++++++++++++++++++--
 target/arm/trace-events |  2 ++
 3 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index c15d79a106..121b4372b2 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -1031,6 +1031,10 @@ struct ArchCPU {
     /* KVM steal time */
     OnOffAuto kvm_steal_time;
 
+    /* KVM registers that must be ignored/hidden */
+    uint64_t *kvm_hidden_regs;
+    uint32_t nr_kvm_hidden_regs;
+
     /* Uniprocessor system with MP extensions */
     bool mp_is_up;
 
diff --git a/target/arm/kvm.c b/target/arm/kvm.c
index 6672344855..67675781f4 100644
--- a/target/arm/kvm.c
+++ b/target/arm/kvm.c
@@ -39,6 +39,8 @@
 #include "qemu/log.h"
 #include "hw/acpi/acpi.h"
 #include "hw/acpi/ghes.h"
+#include "hw/qdev-properties.h"
+#include "hw/qdev-properties-system.h"
 #include "target/arm/gtimer.h"
 #include "migration/blocker.h"
 
@@ -483,6 +485,10 @@ static void kvm_steal_time_set(Object *obj, bool value, Error **errp)
     ARM_CPU(obj)->kvm_steal_time = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
 }
 
+static const Property arm_cpu_kvm_compat_hidden_regs_property =
+    DEFINE_PROP_ARRAY("kvm-hidden-regs", ARMCPU,
+                      nr_kvm_hidden_regs, kvm_hidden_regs, qdev_prop_uint64, uint64_t);
+
 /* KVM VCPU properties should be prefixed with "kvm-". */
 void kvm_arm_add_vcpu_properties(ARMCPU *cpu)
 {
@@ -504,6 +510,8 @@ void kvm_arm_add_vcpu_properties(ARMCPU *cpu)
                              kvm_steal_time_set);
     object_property_set_description(obj, "kvm-steal-time",
                                     "Set off to disable KVM steal time.");
+
+    qdev_property_add_static(DEVICE(obj), &arm_cpu_kvm_compat_hidden_regs_property);
 }
 
 bool kvm_arm_pmu_supported(void)
@@ -764,6 +772,26 @@ static bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx)
     }
 }
 
+/**
+ * kvm_vcpu_compat_hidden_reg:
+ * @cpu: ARMCPU
+ * @regidx: index of the register to check
+ *
+ * Depending on the CPU compat returns true if @regidx must be
+ * ignored during sync & migration
+ */
+static inline bool
+kvm_vcpu_compat_hidden_reg(ARMCPU *cpu, uint64_t regidx)
+{
+    for (int i = 0; i < cpu->nr_kvm_hidden_regs; i++) {
+        if (cpu->kvm_hidden_regs[i] == regidx) {
+            trace_kvm_vcpu_compat_hidden_reg(regidx);
+            return true;
+        }
+    }
+    return false;
+}
+
 /**
  * kvm_arm_init_cpreg_list:
  * @cpu: ARMCPU
@@ -798,7 +826,8 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu)
     qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64);
 
     for (i = 0, arraylen = 0; i < rlp->n; i++) {
-        if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) {
+        if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i]) ||
+            kvm_vcpu_compat_hidden_reg(cpu, rlp->reg[i])) {
             continue;
         }
         switch (rlp->reg[i] & KVM_REG_SIZE_MASK) {
@@ -814,6 +843,8 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu)
         arraylen++;
     }
 
+    trace_kvm_arm_init_cpreg_list_arraylen(arraylen);
+
     cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
     cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
     cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes,
@@ -825,7 +856,8 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu)
 
     for (i = 0, arraylen = 0; i < rlp->n; i++) {
         uint64_t regidx = rlp->reg[i];
-        if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) {
+        if (!kvm_arm_reg_syncs_via_cpreg_list(regidx) ||
+            kvm_vcpu_compat_hidden_reg(cpu, regidx)) {
             continue;
         }
         cpu->cpreg_indexes[arraylen] = regidx;
diff --git a/target/arm/trace-events b/target/arm/trace-events
index 4438dce7be..1b4ab0c683 100644
--- a/target/arm/trace-events
+++ b/target/arm/trace-events
@@ -13,3 +13,5 @@ arm_gt_update_irq(int timer, int irqstate) "gt_update_irq: timer %d irqstate %d"
 
 # kvm.c
 kvm_arm_fixup_msi_route(uint64_t iova, uint64_t gpa) "MSI iova = 0x%"PRIx64" is translated into 0x%"PRIx64
+kvm_arm_init_cpreg_list_arraylen(uint32_t arraylen) "arraylen=%d"
+kvm_vcpu_compat_hidden_reg(uint64_t regidx) "0x%"PRIx64" is hidden"
-- 
2.49.0

On Thu, 11 Sep 2025, Eric Auger wrote:
> New kernels sometimes expose new registers in an unconditionnal
> manner.  This situation breaks backward migration as qemu notices
> there are more registers to store on guest than supported in the
> destination kerenl. This leads to a "failed to load
> cpu:cpreg_vmstate_array_len" error.
>
> A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
> pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
> Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
> migration from a host kernel which features the commit to a destination
> host that doesn't fail.
>
> Currently QEMU is not using that feature so ignoring this latter
> is not a problem. An easy way to fix the migration issue is to teach
> qemu we don't care about that register and we can simply ignore it,
> including its state migration.
>
> This patch introduces a CPU property, under the form of an array of
> reg indices which indicates which registers can be ignored.
>
> The goal then is to set this property in machine type compats such
> as:
> static GlobalProperty arm_virt_kernel_compat_10_1[] = {
>    /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
>    { TYPE_ARM_CPU, "kvm-hidden-regs", "0x6030000000160003" },
> }

One thing worth noting - once this series lands:
https://lore.kernel.org/qemu-devel/20250801074730.28329-1-shameerkolothum@gmail.com/
we might need to add a bit more logic here. Either using the kvm
interfaces (only ignore KVM_REG_ARM_VENDOR_HYP_BMAP_2 when the register
value is 0) or qemu knowledge (only ignore KVM_REG_ARM_VENDOR_HYP_BMAP_2
when the impl-cpu property is not used).

Sebastian

On Thu, 11 Sep 2025, Eric Auger wrote:
> New kernels sometimes expose new registers in an unconditionnal
> manner.  This situation breaks backward migration as qemu notices
> there are more registers to store on guest than supported in the
> destination kerenl. This leads to a "failed to load
> cpu:cpreg_vmstate_array_len" error.
>
> A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
> pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
> Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
> migration from a host kernel which features the commit to a destination
> host that doesn't fail.
>
> Currently QEMU is not using that feature so ignoring this latter
> is not a problem. An easy way to fix the migration issue is to teach
> qemu we don't care about that register and we can simply ignore it,
> including its state migration.
>
> This patch introduces a CPU property, under the form of an array of
> reg indices which indicates which registers can be ignored.
>
> The goal then is to set this property in machine type compats such
> as:
> static GlobalProperty arm_virt_kernel_compat_10_1[] = {
>    /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
>    { TYPE_ARM_CPU, "kvm-hidden-regs", "0x6030000000160003" },
> }
>
> Signed-off-by: Eric Auger <eric.auger@redhat.com>
> ---
> target/arm/cpu.h        |  4 ++++
> target/arm/kvm.c        | 36 ++++++++++++++++++++++++++++++++++--
> target/arm/trace-events |  2 ++
> 3 files changed, 40 insertions(+), 2 deletions(-)
>

Reviewed-by: Sebastian Ott <sebott@redhat.com>