More recent kernels sometimes expose new registers in an
unconditionnal manner. This situation breaks backward migration
as qemu notices there are more registers in the input stream
than supported on the destination host. This leads to a
"failed to load cpu:cpreg_vmstate_array_len" error.

A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
migration from a host kernel that features the commit to a destination
host that doesn't, fail with above error.

Currently QEMU is not using that feature so ignoring this latter
is not a problem. An easy way to fix the migration issue is to teach
qemu we don't care about that register and we can simply ignore it
when syncing its state during migration.

This patch introduces an array of such hidden registers. Soon it will
be settable through an array property.

If hidden, the register is moved out of the array of cpreg which is
built in kvm_arm_init_cpreg_list(). That way their state won't be
synced.

To extend that functionality to TCG, do the same in add_cpreg_to_list()
and count_cpreg().

Signed-off-by: Eric Auger <eric.auger@redhat.com>
Reviewed-by: Sebastian Ott <sebott@redhat.com>

---
v2 -> v3:
- use kvm_regidx

v1 -> v2:
- Move the property in a separate patch
- improve the commit msg
- change the trace point to just print info in
  kvm_arm_init_cpreg_list()
- improve comment in cpu.h (Connie)

target/arm/helper: Skip hidden registers

In case a cpreg is hidden, skip it when initialing the cpreg
list.

Signed-off-by: Eric Auger <eric.auger@redhat.com>
Reviewed-by: Cornelia Huck <cohuck@redhat.com>
---
 target/arm/cpu.h        | 20 ++++++++++++++++++++
 target/arm/helper.c     | 12 +++++++++++-
 target/arm/kvm.c        | 12 +++++++++++-
 target/arm/trace-events |  2 ++
 4 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index e900ef7937b..e87f222e7be 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -1045,6 +1045,15 @@ struct ArchCPU {
     /* KVM steal time */
     OnOffAuto kvm_steal_time;
 
+    /*
+     * Array of register indexes that need to be hidden to allow migration
+     * in certain cases, i.e. when a register is exposed in KVM or defined
+     * in TCG but not actually used in QEMU. Indexes are described in Linux
+     * Documentation/virt/kvm/api.rst for both KVM and TCG.
+     */
+    uint64_t *hidden_regs;
+    uint32_t nr_hidden_regs;
+
     /* Uniprocessor system with MP extensions */
     bool mp_is_up;
 
@@ -1185,6 +1194,17 @@ struct ARMCPUClass {
     ResettablePhases parent_phases;
 };
 
+static inline bool
+arm_cpu_hidden_reg(ARMCPU *cpu, uint64_t regidx)
+{
+    for (int i = 0; i < cpu->nr_hidden_regs; i++) {
+        if (cpu->hidden_regs[i] == regidx) {
+            return true;
+        }
+    }
+    return false;
+}
+
 /* Callback functions for the generic timer's timers. */
 void arm_gt_ptimer_cb(void *opaque);
 void arm_gt_vtimer_cb(void *opaque);
diff --git a/target/arm/helper.c b/target/arm/helper.c
index dce648b4824..8217517150b 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -235,9 +235,13 @@ static void add_cpreg_to_list(gpointer key, gpointer value, gpointer opaque)
     ARMCPU *cpu = opaque;
     uint32_t regidx = (uintptr_t)key;
     const ARMCPRegInfo *ri = value;
+    uint64_t kvm_regidx = cpreg_to_kvm_id(regidx);
 
+    if (arm_cpu_hidden_reg(cpu, kvm_regidx)) {
+        return;
+    }
     if (!(ri->type & (ARM_CP_NO_RAW | ARM_CP_ALIAS))) {
-        cpu->cpreg_indexes[cpu->cpreg_array_len] = cpreg_to_kvm_id(regidx);
+        cpu->cpreg_indexes[cpu->cpreg_array_len] = kvm_regidx;
         /* The value array need not be initialized at this point */
         cpu->cpreg_array_len++;
     }
@@ -247,6 +251,12 @@ static void count_cpreg(gpointer key, gpointer value, gpointer opaque)
 {
     ARMCPU *cpu = opaque;
     const ARMCPRegInfo *ri = value;
+    uint32_t regidx = (uintptr_t)key;
+    uint64_t kvm_regidx = cpreg_to_kvm_id(regidx);
+
+    if (arm_cpu_hidden_reg(cpu, kvm_regidx)) {
+        return;
+    }
 
     if (!(ri->type & (ARM_CP_NO_RAW | ARM_CP_ALIAS))) {
         cpu->cpreg_array_len++;
diff --git a/target/arm/kvm.c b/target/arm/kvm.c
index c6f0d0fc4e1..7e0a3680748 100644
--- a/target/arm/kvm.c
+++ b/target/arm/kvm.c
@@ -789,7 +789,10 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu)
     qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64);
 
     for (i = 0, arraylen = 0; i < rlp->n; i++) {
-        if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) {
+        uint64_t regidx = rlp->reg[i];
+
+        if (!kvm_arm_reg_syncs_via_cpreg_list(regidx) ||
+            arm_cpu_hidden_reg(cpu, regidx)) {
             continue;
         }
         switch (rlp->reg[i] & KVM_REG_SIZE_MASK) {
@@ -805,6 +808,8 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu)
         arraylen++;
     }
 
+    trace_kvm_arm_init_cpreg_list_arraylen(arraylen);
+
     cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
     cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
     cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes,
@@ -816,9 +821,14 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu)
 
     for (i = 0, arraylen = 0; i < rlp->n; i++) {
         uint64_t regidx = rlp->reg[i];
+
         if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) {
             continue;
         }
+        if (arm_cpu_hidden_reg(cpu, regidx)) {
+            trace_kvm_arm_init_cpreg_list_skip_hidden_reg(rlp->reg[i]);
+            continue;
+        }
         cpu->cpreg_indexes[arraylen] = regidx;
         arraylen++;
     }
diff --git a/target/arm/trace-events b/target/arm/trace-events
index 0a5ed3e69d5..20f4b4f2cd0 100644
--- a/target/arm/trace-events
+++ b/target/arm/trace-events
@@ -14,6 +14,8 @@ arm_gt_update_irq(int timer, int irqstate) "gt_update_irq: timer %d irqstate %d"
 # kvm.c
 kvm_arm_fixup_msi_route(uint64_t iova, uint64_t gpa) "MSI iova = 0x%"PRIx64" is translated into 0x%"PRIx64
 kvm_arm_cpu_post_load_missing_reg(char *name) "Missing register in input stream: %s"
+kvm_arm_init_cpreg_list_arraylen(uint32_t arraylen) "arraylen=%d"
+kvm_arm_init_cpreg_list_skip_hidden_reg(uint64_t regidx) "hidden 0x%"PRIx64" is skipped"
 
 # cpu.c
 arm_cpu_reset(uint64_t mp_aff) "cpu %" PRIu64
-- 
2.52.0

Hi Alex,

On 1/26/26 5:53 PM, Eric Auger wrote:
> More recent kernels sometimes expose new registers in an
> unconditionnal manner. This situation breaks backward migration
> as qemu notices there are more registers in the input stream
> than supported on the destination host. This leads to a
> "failed to load cpu:cpreg_vmstate_array_len" error.
>
> A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
> pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
> Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
> migration from a host kernel that features the commit to a destination
> host that doesn't, fail with above error.

Please find below a follow up on your question regarding hidden reg,
during yesterday's qemu biweekly

Regarding this KVM_REG_ARM_VENDOR_HYP_BMAP_2 pseudo FW reg, this latter
was introduced by 

[PATCH v8 3/6] KVM: arm64: Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2 <https://lore.kernel.org/all/20250221140229.12588-4-shameerali.kolothum.thodi@huawei.com/#r>

https://lore.kernel.org/all/20250221140229.12588-4-shameerali.kolothum.thodi@huawei.com/
The purpose is to let userspace retrieve and negotiate available Vendor
Hyp services. So it allows qemu to opt in for specific services (the
first one being the MIDR based errata mgt). So the guest only reads this
reg and does not change it. So this sounds safe to ignore saving/restore
the reg state as long as qemu does not play with the reg.   QEMU does
not get/set this reg yet. Support may come later. Shameer posted an RFC:
[RFC PATCH RESEND 3/4] target/arm/kvm: Handle KVM Target Imp CPU
hypercalls
https://lore.kernel.org/all/3bf24eb7-b1cd-793c-158f-7b6e2aeab026@redhat.com/
There you can see how the reg is accessed. Besides, enlarging your
question to an hidden reg likely to be written by the guest and whose
state wouldn't be saved/restored due to the "hidden" nature of the reg,
we could add a check in qemu that compares the reg value against default
init one on migration source and in case it differs fail the migration.
Thanks Eric
> Currently QEMU is not using that feature so ignoring this latter
> is not a problem. An easy way to fix the migration issue is to teach
> qemu we don't care about that register and we can simply ignore it
> when syncing its state during migration.
>
> This patch introduces an array of such hidden registers. Soon it will
> be settable through an array property.
>
> If hidden, the register is moved out of the array of cpreg which is
> built in kvm_arm_init_cpreg_list(). That way their state won't be
> synced.
>
> To extend that functionality to TCG, do the same in add_cpreg_to_list()
> and count_cpreg().
>
> Signed-off-by: Eric Auger <eric.auger@redhat.com>
> Reviewed-by: Sebastian Ott <sebott@redhat.com>
>
> ---
> v2 -> v3:
> - use kvm_regidx
>
> v1 -> v2:
> - Move the property in a separate patch
> - improve the commit msg
> - change the trace point to just print info in
>   kvm_arm_init_cpreg_list()
> - improve comment in cpu.h (Connie)
>
> target/arm/helper: Skip hidden registers
>
> In case a cpreg is hidden, skip it when initialing the cpreg
> list.
>
> Signed-off-by: Eric Auger <eric.auger@redhat.com>
> Reviewed-by: Cornelia Huck <cohuck@redhat.com>
> ---
>  target/arm/cpu.h        | 20 ++++++++++++++++++++
>  target/arm/helper.c     | 12 +++++++++++-
>  target/arm/kvm.c        | 12 +++++++++++-
>  target/arm/trace-events |  2 ++
>  4 files changed, 44 insertions(+), 2 deletions(-)
>
> diff --git a/target/arm/cpu.h b/target/arm/cpu.h
> index e900ef7937b..e87f222e7be 100644
> --- a/target/arm/cpu.h
> +++ b/target/arm/cpu.h
> @@ -1045,6 +1045,15 @@ struct ArchCPU {
>      /* KVM steal time */
>      OnOffAuto kvm_steal_time;
>  
> +    /*
> +     * Array of register indexes that need to be hidden to allow migration
> +     * in certain cases, i.e. when a register is exposed in KVM or defined
> +     * in TCG but not actually used in QEMU. Indexes are described in Linux
> +     * Documentation/virt/kvm/api.rst for both KVM and TCG.
> +     */
> +    uint64_t *hidden_regs;
> +    uint32_t nr_hidden_regs;
> +
>      /* Uniprocessor system with MP extensions */
>      bool mp_is_up;
>  
> @@ -1185,6 +1194,17 @@ struct ARMCPUClass {
>      ResettablePhases parent_phases;
>  };
>  
> +static inline bool
> +arm_cpu_hidden_reg(ARMCPU *cpu, uint64_t regidx)
> +{
> +    for (int i = 0; i < cpu->nr_hidden_regs; i++) {
> +        if (cpu->hidden_regs[i] == regidx) {
> +            return true;
> +        }
> +    }
> +    return false;
> +}
> +
>  /* Callback functions for the generic timer's timers. */
>  void arm_gt_ptimer_cb(void *opaque);
>  void arm_gt_vtimer_cb(void *opaque);
> diff --git a/target/arm/helper.c b/target/arm/helper.c
> index dce648b4824..8217517150b 100644
> --- a/target/arm/helper.c
> +++ b/target/arm/helper.c
> @@ -235,9 +235,13 @@ static void add_cpreg_to_list(gpointer key, gpointer value, gpointer opaque)
>      ARMCPU *cpu = opaque;
>      uint32_t regidx = (uintptr_t)key;
>      const ARMCPRegInfo *ri = value;
> +    uint64_t kvm_regidx = cpreg_to_kvm_id(regidx);
>  
> +    if (arm_cpu_hidden_reg(cpu, kvm_regidx)) {
> +        return;
> +    }
>      if (!(ri->type & (ARM_CP_NO_RAW | ARM_CP_ALIAS))) {
> -        cpu->cpreg_indexes[cpu->cpreg_array_len] = cpreg_to_kvm_id(regidx);
> +        cpu->cpreg_indexes[cpu->cpreg_array_len] = kvm_regidx;
>          /* The value array need not be initialized at this point */
>          cpu->cpreg_array_len++;
>      }
> @@ -247,6 +251,12 @@ static void count_cpreg(gpointer key, gpointer value, gpointer opaque)
>  {
>      ARMCPU *cpu = opaque;
>      const ARMCPRegInfo *ri = value;
> +    uint32_t regidx = (uintptr_t)key;
> +    uint64_t kvm_regidx = cpreg_to_kvm_id(regidx);
> +
> +    if (arm_cpu_hidden_reg(cpu, kvm_regidx)) {
> +        return;
> +    }
>  
>      if (!(ri->type & (ARM_CP_NO_RAW | ARM_CP_ALIAS))) {
>          cpu->cpreg_array_len++;
> diff --git a/target/arm/kvm.c b/target/arm/kvm.c
> index c6f0d0fc4e1..7e0a3680748 100644
> --- a/target/arm/kvm.c
> +++ b/target/arm/kvm.c
> @@ -789,7 +789,10 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu)
>      qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64);
>  
>      for (i = 0, arraylen = 0; i < rlp->n; i++) {
> -        if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) {
> +        uint64_t regidx = rlp->reg[i];
> +
> +        if (!kvm_arm_reg_syncs_via_cpreg_list(regidx) ||
> +            arm_cpu_hidden_reg(cpu, regidx)) {
>              continue;
>          }
>          switch (rlp->reg[i] & KVM_REG_SIZE_MASK) {
> @@ -805,6 +808,8 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu)
>          arraylen++;
>      }
>  
> +    trace_kvm_arm_init_cpreg_list_arraylen(arraylen);
> +
>      cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
>      cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
>      cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes,
> @@ -816,9 +821,14 @@ static int kvm_arm_init_cpreg_list(ARMCPU *cpu)
>  
>      for (i = 0, arraylen = 0; i < rlp->n; i++) {
>          uint64_t regidx = rlp->reg[i];
> +
>          if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) {
>              continue;
>          }
> +        if (arm_cpu_hidden_reg(cpu, regidx)) {
> +            trace_kvm_arm_init_cpreg_list_skip_hidden_reg(rlp->reg[i]);
> +            continue;
> +        }
>          cpu->cpreg_indexes[arraylen] = regidx;
>          arraylen++;
>      }
> diff --git a/target/arm/trace-events b/target/arm/trace-events
> index 0a5ed3e69d5..20f4b4f2cd0 100644
> --- a/target/arm/trace-events
> +++ b/target/arm/trace-events
> @@ -14,6 +14,8 @@ arm_gt_update_irq(int timer, int irqstate) "gt_update_irq: timer %d irqstate %d"
>  # kvm.c
>  kvm_arm_fixup_msi_route(uint64_t iova, uint64_t gpa) "MSI iova = 0x%"PRIx64" is translated into 0x%"PRIx64
>  kvm_arm_cpu_post_load_missing_reg(char *name) "Missing register in input stream: %s"
> +kvm_arm_init_cpreg_list_arraylen(uint32_t arraylen) "arraylen=%d"
> +kvm_arm_init_cpreg_list_skip_hidden_reg(uint64_t regidx) "hidden 0x%"PRIx64" is skipped"
>  
>  # cpu.c
>  arm_cpu_reset(uint64_t mp_aff) "cpu %" PRIu64

On Mon, Jan 26 2026, Eric Auger <eric.auger@redhat.com> wrote:

> More recent kernels sometimes expose new registers in an
> unconditionnal manner. This situation breaks backward migration
> as qemu notices there are more registers in the input stream
> than supported on the destination host. This leads to a
> "failed to load cpu:cpreg_vmstate_array_len" error.
>
> A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
> pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
> Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
> migration from a host kernel that features the commit to a destination
> host that doesn't, fail with above error.
>
> Currently QEMU is not using that feature so ignoring this latter
> is not a problem. An easy way to fix the migration issue is to teach
> qemu we don't care about that register and we can simply ignore it
> when syncing its state during migration.
>
> This patch introduces an array of such hidden registers. Soon it will
> be settable through an array property.
>
> If hidden, the register is moved out of the array of cpreg which is
> built in kvm_arm_init_cpreg_list(). That way their state won't be
> synced.
>
> To extend that functionality to TCG, do the same in add_cpreg_to_list()
> and count_cpreg().
>
> Signed-off-by: Eric Auger <eric.auger@redhat.com>
> Reviewed-by: Sebastian Ott <sebott@redhat.com>
>
> ---
> v2 -> v3:
> - use kvm_regidx
>
> v1 -> v2:
> - Move the property in a separate patch
> - improve the commit msg
> - change the trace point to just print info in
>   kvm_arm_init_cpreg_list()
> - improve comment in cpu.h (Connie)
>
> target/arm/helper: Skip hidden registers
>
> In case a cpreg is hidden, skip it when initialing the cpreg
> list.
>
> Signed-off-by: Eric Auger <eric.auger@redhat.com>
> Reviewed-by: Cornelia Huck <cohuck@redhat.com>

This looks a bit weird, probably due to patch folding... but I'm happy
to have my R-b apply to the whole patch :)

> ---
>  target/arm/cpu.h        | 20 ++++++++++++++++++++
>  target/arm/helper.c     | 12 +++++++++++-
>  target/arm/kvm.c        | 12 +++++++++++-
>  target/arm/trace-events |  2 ++
>  4 files changed, 44 insertions(+), 2 deletions(-)

On 1/27/26 6:09 PM, Cornelia Huck wrote:
> On Mon, Jan 26 2026, Eric Auger <eric.auger@redhat.com> wrote:
>
>> More recent kernels sometimes expose new registers in an
>> unconditionnal manner. This situation breaks backward migration
>> as qemu notices there are more registers in the input stream
>> than supported on the destination host. This leads to a
>> "failed to load cpu:cpreg_vmstate_array_len" error.
>>
>> A good example is the introduction of KVM_REG_ARM_VENDOR_HYP_BMAP_2
>> pseudo FW register in v6.16 by commit C0000e58c74e (“KVM: arm64:
>> Introduce KVM_REG_ARM_VENDOR_HYP_BMAP_2”). Trying to do backward
>> migration from a host kernel that features the commit to a destination
>> host that doesn't, fail with above error.
>>
>> Currently QEMU is not using that feature so ignoring this latter
>> is not a problem. An easy way to fix the migration issue is to teach
>> qemu we don't care about that register and we can simply ignore it
>> when syncing its state during migration.
>>
>> This patch introduces an array of such hidden registers. Soon it will
>> be settable through an array property.
>>
>> If hidden, the register is moved out of the array of cpreg which is
>> built in kvm_arm_init_cpreg_list(). That way their state won't be
>> synced.
>>
>> To extend that functionality to TCG, do the same in add_cpreg_to_list()
>> and count_cpreg().
>>
>> Signed-off-by: Eric Auger <eric.auger@redhat.com>
>> Reviewed-by: Sebastian Ott <sebott@redhat.com>
>>
>> ---
>> v2 -> v3:
>> - use kvm_regidx
>>
>> v1 -> v2:
>> - Move the property in a separate patch
>> - improve the commit msg
>> - change the trace point to just print info in
>>   kvm_arm_init_cpreg_list()
>> - improve comment in cpu.h (Connie)
>>
>> target/arm/helper: Skip hidden registers
>>
>> In case a cpreg is hidden, skip it when initialing the cpreg
>> list.
>>
>> Signed-off-by: Eric Auger <eric.auger@redhat.com>
>> Reviewed-by: Cornelia Huck <cohuck@redhat.com>
> This looks a bit weird, probably due to patch folding... but I'm happy
> to have my R-b apply to the whole patch :)

Oh yes I missed that. Nevertheless it is below the --- so I guess it is
not that much an issue

Eric
>
>> ---
>>  target/arm/cpu.h        | 20 ++++++++++++++++++++
>>  target/arm/helper.c     | 12 +++++++++++-
>>  target/arm/kvm.c        | 12 +++++++++++-
>>  target/arm/trace-events |  2 ++
>>  4 files changed, 44 insertions(+), 2 deletions(-)