VM notification binding and pending tracking exist for non-secure
endpoints, but FFA_NOTIFICATION_SET still only forwards secure
destinations to the SPMC. Non-secure VMs therefore cannot receive
notifications from other VMs. Local NPI delivery also needs explicit
re-arm tracking so repeated raises are not lost while the interrupt is
already pending.
Add a local VM notification delivery path for non-secure destinations.
notification_set_vm() resolves the destination endpoint, verifies that
every requested bit is bound to the sender, sets the receiver's
vm_pending bitmap under notif_lock, and raises an NPI only when local
pending state is not already armed.
Track whether a local NPI is already armed with notif_irq_raised,
clear that state once both VM and hypervisor pending bitmaps are
drained, and keep notif_lock held across the VM notification injection
attempt. If no destination vCPU is online, leave the pending bits set
and keep notif_irq_raised clear so delivery can be retried later.
Also expose firmware notification availability so FFA_FEATURES only
advertises notification support when it is actually provided by the
firmware or by CONFIG_FFA_VM_TO_VM.
Functional impact: when CONFIG_FFA_VM_TO_VM is enabled, non-secure
FFA_NOTIFICATION_SET delivers VM-to-VM notifications locally and keeps
NPI delivery reliable across repeated raises.
Signed-off-by: Bertrand Marquis <bertrand.marquis@arm.com>
---
Changes since v1:
- serialize notification_set_vm() state updates with the NPI attempt
- keep pending VM notifications set when local injection fails
---
xen/arch/arm/tee/ffa.c | 24 ++++++++--
xen/arch/arm/tee/ffa_notif.c | 82 ++++++++++++++++++++++++++++++++--
xen/arch/arm/tee/ffa_private.h | 17 ++++---
3 files changed, 107 insertions(+), 16 deletions(-)
diff --git a/xen/arch/arm/tee/ffa.c b/xen/arch/arm/tee/ffa.c
index 1fe33f26454a..7fe021049cba 100644
--- a/xen/arch/arm/tee/ffa.c
+++ b/xen/arch/arm/tee/ffa.c
@@ -39,8 +39,13 @@
* o FFA_MSG_SEND_DIRECT_REQ:
* - only supported from a VM to an SP
* o FFA_NOTIFICATION_*:
+ * - only supported when firmware notifications are enabled or VM-to-VM
+ * support is built in
* - only supports global notifications, that is, per vCPU notifications
- * are not supported
+ * are not supported and secure per-vCPU notification information is
+ * not forwarded
+ * - the source endpoint ID reported for a notification may no longer
+ * exist by the time the receiver consumes it
* - doesn't support signalling the secondary scheduler of pending
* notification for secure partitions
* - doesn't support notifications for Xen itself
@@ -245,6 +250,8 @@ static void handle_features(struct cpu_user_regs *regs)
uint32_t a1 = get_user_reg(regs, 1);
struct domain *d = current->domain;
struct ffa_ctx *ctx = d->arch.tee;
+ bool notif_supported = IS_ENABLED(CONFIG_FFA_VM_TO_VM) ||
+ ffa_notif_fw_enabled();
/*
* FFA_FEATURES defines w2 as input properties only for specific
@@ -343,10 +350,16 @@ static void handle_features(struct cpu_user_regs *regs)
break;
case FFA_FEATURE_NOTIF_PEND_INTR:
- ffa_set_regs_success(regs, GUEST_FFA_NOTIF_PEND_INTR_ID, 0);
+ if ( notif_supported )
+ ffa_set_regs_success(regs, GUEST_FFA_NOTIF_PEND_INTR_ID, 0);
+ else
+ ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
break;
case FFA_FEATURE_SCHEDULE_RECV_INTR:
- ffa_set_regs_success(regs, GUEST_FFA_SCHEDULE_RECV_INTR_ID, 0);
+ if ( notif_supported )
+ ffa_set_regs_success(regs, GUEST_FFA_SCHEDULE_RECV_INTR_ID, 0);
+ else
+ ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
break;
case FFA_PARTITION_INFO_GET_REGS:
if ( ACCESS_ONCE(ctx->guest_vers) >= FFA_VERSION_1_2 )
@@ -361,7 +374,10 @@ static void handle_features(struct cpu_user_regs *regs)
case FFA_NOTIFICATION_SET:
case FFA_NOTIFICATION_INFO_GET_32:
case FFA_NOTIFICATION_INFO_GET_64:
- ffa_set_regs_success(regs, 0, 0);
+ if ( notif_supported )
+ ffa_set_regs_success(regs, 0, 0);
+ else
+ ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
break;
default:
ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
diff --git a/xen/arch/arm/tee/ffa_notif.c b/xen/arch/arm/tee/ffa_notif.c
index a841c8f8d747..b29d948a7110 100644
--- a/xen/arch/arm/tee/ffa_notif.c
+++ b/xen/arch/arm/tee/ffa_notif.c
@@ -21,6 +21,11 @@ static bool __ro_after_init fw_notif_enabled;
static unsigned int __ro_after_init notif_sri_irq;
static DEFINE_SPINLOCK(notif_info_lock);
+bool ffa_notif_fw_enabled(void)
+{
+ return fw_notif_enabled;
+}
+
static bool inject_notif_pending(struct domain *d)
{
struct vcpu *v;
@@ -107,6 +112,55 @@ out_unlock:
return ret;
}
+/*
+ * Deliver a VM-to-VM notification. ctx->notif.notif_lock protects
+ * vm_bind/vm_pending so callers must not hold it already.
+ */
+static int32_t notification_set_vm(uint16_t dst_id, uint16_t src_id,
+ uint32_t flags, uint64_t bitmap)
+{
+ struct domain *dst_d;
+ struct ffa_ctx *dst_ctx;
+ unsigned int id;
+ int32_t ret;
+
+ if ( flags )
+ return FFA_RET_INVALID_PARAMETERS;
+
+ ret = ffa_endpoint_domain_lookup(dst_id, &dst_d, &dst_ctx);
+ if ( ret )
+ return ret;
+
+ ret = FFA_RET_OK;
+
+ spin_lock(&dst_ctx->notif.notif_lock);
+
+ for ( id = 0; id < FFA_NUM_VM_NOTIF; id++ )
+ {
+ if ( !(bitmap & BIT(id, ULL)) )
+ continue;
+
+ if ( dst_ctx->notif.vm_bind[id] != src_id )
+ {
+ ret = FFA_RET_DENIED;
+ goto out_unlock;
+ }
+ }
+
+ dst_ctx->notif.vm_pending |= bitmap;
+ if ( !dst_ctx->notif.notif_irq_raised &&
+ (dst_ctx->notif.vm_pending || dst_ctx->notif.hyp_pending) &&
+ inject_notif_pending(dst_d) )
+ dst_ctx->notif.notif_irq_raised = true;
+
+out_unlock:
+ spin_unlock(&dst_ctx->notif.notif_lock);
+
+ rcu_unlock_domain(dst_d);
+
+ return ret;
+}
+
int32_t ffa_handle_notification_bind(struct cpu_user_regs *regs)
{
struct domain *d = current->domain;
@@ -288,6 +342,8 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
{
+ bool pending;
+
spin_lock(&ctx->notif.notif_lock);
if ( (flags & FFA_NOTIF_FLAG_BITMAP_HYP) && ctx->notif.hyp_pending )
@@ -298,6 +354,18 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
ctx->notif.notif_irq_raised = false;
}
+ if ( (flags & FFA_NOTIF_FLAG_BITMAP_VM) && ctx->notif.vm_pending )
+ {
+ w4 = (uint32_t)(ctx->notif.vm_pending & GENMASK(31, 0));
+ w5 = (uint32_t)((ctx->notif.vm_pending >> 32) & GENMASK(31, 0));
+ ctx->notif.vm_pending = 0;
+ }
+
+ pending = (ctx->notif.hyp_pending != 0) ||
+ (ctx->notif.vm_pending != 0);
+ if ( !pending )
+ ctx->notif.notif_irq_raised = false;
+
spin_unlock(&ctx->notif.notif_lock);
}
@@ -323,9 +391,17 @@ int32_t ffa_handle_notification_set(struct cpu_user_regs *regs)
if ( flags )
return FFA_RET_INVALID_PARAMETERS;
- if ( FFA_ID_IS_SECURE(dest_id) && fw_notif_enabled )
- return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
- bitmap_hi);
+ if ( FFA_ID_IS_SECURE(dest_id) )
+ {
+ if ( fw_notif_enabled )
+ return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags,
+ bitmap_lo, bitmap_hi);
+ }
+ else if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
+ {
+ return notification_set_vm(dest_id, caller_id, flags,
+ ((uint64_t)bitmap_hi << 32) | bitmap_lo);
+ }
return FFA_RET_NOT_SUPPORTED;
}
diff --git a/xen/arch/arm/tee/ffa_private.h b/xen/arch/arm/tee/ffa_private.h
index 78a0a9815d56..923a071a9d7c 100644
--- a/xen/arch/arm/tee/ffa_private.h
+++ b/xen/arch/arm/tee/ffa_private.h
@@ -340,20 +340,18 @@ struct ffa_ctx_notif {
uint64_t vm_pending;
/*
- * Source endpoint bound to each VM notification ID (0 means unbound).
+ * Tracks whether an NPI has been raised for local pending notifications.
+ * Protected by notif_lock.
*/
- uint16_t vm_bind[FFA_NUM_VM_NOTIF];
+ bool notif_irq_raised;
/*
- * Lock protecting the hypervisor-managed notification state.
+ * Source endpoint bound to each VM notification ID (0 means unbound).
*/
- spinlock_t notif_lock;
+ uint16_t vm_bind[FFA_NUM_VM_NOTIF];
- /*
- * Tracks whether a local notification pending interrupt was raised.
- * Protected by notif_lock.
- */
- bool notif_irq_raised;
+ /* Lock protecting local notification state. */
+ spinlock_t notif_lock;
/*
* Bitmap of pending hypervisor notifications (for HYP bitmap queries).
@@ -495,6 +493,7 @@ void ffa_notif_init(void);
void ffa_notif_init_interrupt(void);
int ffa_notif_domain_init(struct domain *d);
void ffa_notif_domain_destroy(struct domain *d);
+bool ffa_notif_fw_enabled(void);
int32_t ffa_handle_notification_bind(struct cpu_user_regs *regs);
int32_t ffa_handle_notification_unbind(struct cpu_user_regs *regs);
--
2.53.0
Hi Bertrand,
On Wed, Apr 29, 2026 at 7:44 AM Bertrand Marquis
<bertrand.marquis@arm.com> wrote:
>
> VM notification binding and pending tracking exist for non-secure
> endpoints, but FFA_NOTIFICATION_SET still only forwards secure
> destinations to the SPMC. Non-secure VMs therefore cannot receive
> notifications from other VMs. Local NPI delivery also needs explicit
> re-arm tracking so repeated raises are not lost while the interrupt is
> already pending.
>
> Add a local VM notification delivery path for non-secure destinations.
> notification_set_vm() resolves the destination endpoint, verifies that
> every requested bit is bound to the sender, sets the receiver's
> vm_pending bitmap under notif_lock, and raises an NPI only when local
> pending state is not already armed.
>
> Track whether a local NPI is already armed with notif_irq_raised,
> clear that state once both VM and hypervisor pending bitmaps are
> drained, and keep notif_lock held across the VM notification injection
> attempt. If no destination vCPU is online, leave the pending bits set
> and keep notif_irq_raised clear so delivery can be retried later.
> Also expose firmware notification availability so FFA_FEATURES only
> advertises notification support when it is actually provided by the
> firmware or by CONFIG_FFA_VM_TO_VM.
>
> Functional impact: when CONFIG_FFA_VM_TO_VM is enabled, non-secure
> FFA_NOTIFICATION_SET delivers VM-to-VM notifications locally and keeps
> NPI delivery reliable across repeated raises.
>
> Signed-off-by: Bertrand Marquis <bertrand.marquis@arm.com>
> ---
> Changes since v1:
> - serialize notification_set_vm() state updates with the NPI attempt
> - keep pending VM notifications set when local injection fails
> ---
> xen/arch/arm/tee/ffa.c | 24 ++++++++--
> xen/arch/arm/tee/ffa_notif.c | 82 ++++++++++++++++++++++++++++++++--
> xen/arch/arm/tee/ffa_private.h | 17 ++++---
> 3 files changed, 107 insertions(+), 16 deletions(-)
>
> diff --git a/xen/arch/arm/tee/ffa.c b/xen/arch/arm/tee/ffa.c
> index 1fe33f26454a..7fe021049cba 100644
> --- a/xen/arch/arm/tee/ffa.c
> +++ b/xen/arch/arm/tee/ffa.c
> @@ -39,8 +39,13 @@
> * o FFA_MSG_SEND_DIRECT_REQ:
> * - only supported from a VM to an SP
> * o FFA_NOTIFICATION_*:
> + * - only supported when firmware notifications are enabled or VM-to-VM
> + * support is built in
> * - only supports global notifications, that is, per vCPU notifications
> - * are not supported
> + * are not supported and secure per-vCPU notification information is
> + * not forwarded
> + * - the source endpoint ID reported for a notification may no longer
> + * exist by the time the receiver consumes it
> * - doesn't support signalling the secondary scheduler of pending
> * notification for secure partitions
> * - doesn't support notifications for Xen itself
> @@ -245,6 +250,8 @@ static void handle_features(struct cpu_user_regs *regs)
> uint32_t a1 = get_user_reg(regs, 1);
> struct domain *d = current->domain;
> struct ffa_ctx *ctx = d->arch.tee;
> + bool notif_supported = IS_ENABLED(CONFIG_FFA_VM_TO_VM) ||
> + ffa_notif_fw_enabled();
>
> /*
> * FFA_FEATURES defines w2 as input properties only for specific
> @@ -343,10 +350,16 @@ static void handle_features(struct cpu_user_regs *regs)
>
> break;
> case FFA_FEATURE_NOTIF_PEND_INTR:
> - ffa_set_regs_success(regs, GUEST_FFA_NOTIF_PEND_INTR_ID, 0);
> + if ( notif_supported )
> + ffa_set_regs_success(regs, GUEST_FFA_NOTIF_PEND_INTR_ID, 0);
> + else
> + ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
> break;
> case FFA_FEATURE_SCHEDULE_RECV_INTR:
> - ffa_set_regs_success(regs, GUEST_FFA_SCHEDULE_RECV_INTR_ID, 0);
> + if ( notif_supported )
> + ffa_set_regs_success(regs, GUEST_FFA_SCHEDULE_RECV_INTR_ID, 0);
> + else
> + ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
> break;
> case FFA_PARTITION_INFO_GET_REGS:
> if ( ACCESS_ONCE(ctx->guest_vers) >= FFA_VERSION_1_2 )
> @@ -361,7 +374,10 @@ static void handle_features(struct cpu_user_regs *regs)
> case FFA_NOTIFICATION_SET:
> case FFA_NOTIFICATION_INFO_GET_32:
> case FFA_NOTIFICATION_INFO_GET_64:
> - ffa_set_regs_success(regs, 0, 0);
> + if ( notif_supported )
> + ffa_set_regs_success(regs, 0, 0);
> + else
> + ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
> break;
> default:
> ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
> diff --git a/xen/arch/arm/tee/ffa_notif.c b/xen/arch/arm/tee/ffa_notif.c
> index a841c8f8d747..b29d948a7110 100644
> --- a/xen/arch/arm/tee/ffa_notif.c
> +++ b/xen/arch/arm/tee/ffa_notif.c
> @@ -21,6 +21,11 @@ static bool __ro_after_init fw_notif_enabled;
> static unsigned int __ro_after_init notif_sri_irq;
> static DEFINE_SPINLOCK(notif_info_lock);
>
> +bool ffa_notif_fw_enabled(void)
> +{
> + return fw_notif_enabled;
> +}
> +
> static bool inject_notif_pending(struct domain *d)
> {
> struct vcpu *v;
> @@ -107,6 +112,55 @@ out_unlock:
> return ret;
> }
>
> +/*
> + * Deliver a VM-to-VM notification. ctx->notif.notif_lock protects
> + * vm_bind/vm_pending so callers must not hold it already.
> + */
> +static int32_t notification_set_vm(uint16_t dst_id, uint16_t src_id,
> + uint32_t flags, uint64_t bitmap)
> +{
> + struct domain *dst_d;
> + struct ffa_ctx *dst_ctx;
> + unsigned int id;
> + int32_t ret;
> +
> + if ( flags )
> + return FFA_RET_INVALID_PARAMETERS;
> +
> + ret = ffa_endpoint_domain_lookup(dst_id, &dst_d, &dst_ctx);
> + if ( ret )
> + return ret;
> +
> + ret = FFA_RET_OK;
> +
> + spin_lock(&dst_ctx->notif.notif_lock);
> +
> + for ( id = 0; id < FFA_NUM_VM_NOTIF; id++ )
> + {
> + if ( !(bitmap & BIT(id, ULL)) )
> + continue;
> +
> + if ( dst_ctx->notif.vm_bind[id] != src_id )
> + {
> + ret = FFA_RET_DENIED;
> + goto out_unlock;
> + }
> + }
> +
> + dst_ctx->notif.vm_pending |= bitmap;
> + if ( !dst_ctx->notif.notif_irq_raised &&
> + (dst_ctx->notif.vm_pending || dst_ctx->notif.hyp_pending) &&
> + inject_notif_pending(dst_d) )
> + dst_ctx->notif.notif_irq_raised = true;
> +
> +out_unlock:
> + spin_unlock(&dst_ctx->notif.notif_lock);
> +
> + rcu_unlock_domain(dst_d);
> +
> + return ret;
> +}
> +
> int32_t ffa_handle_notification_bind(struct cpu_user_regs *regs)
> {
> struct domain *d = current->domain;
> @@ -288,6 +342,8 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
>
> if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
> {
> + bool pending;
> +
> spin_lock(&ctx->notif.notif_lock);
>
> if ( (flags & FFA_NOTIF_FLAG_BITMAP_HYP) && ctx->notif.hyp_pending )
> @@ -298,6 +354,18 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
> ctx->notif.notif_irq_raised = false;
> }
>
> + if ( (flags & FFA_NOTIF_FLAG_BITMAP_VM) && ctx->notif.vm_pending )
> + {
> + w4 = (uint32_t)(ctx->notif.vm_pending & GENMASK(31, 0));
> + w5 = (uint32_t)((ctx->notif.vm_pending >> 32) & GENMASK(31, 0));
> + ctx->notif.vm_pending = 0;
> + }
> +
> + pending = (ctx->notif.hyp_pending != 0) ||
> + (ctx->notif.vm_pending != 0);
> + if ( !pending )
> + ctx->notif.notif_irq_raised = false;
This seems to take care of clearing notif_irq_raised for all cases. Do
we still need the one just above this block (copied here):
if ( !ctx->notif.vm_pending )
ctx->notif.notif_irq_raised = false;
?
Cheers,
Jens
> +
> spin_unlock(&ctx->notif.notif_lock);
> }
>
> @@ -323,9 +391,17 @@ int32_t ffa_handle_notification_set(struct cpu_user_regs *regs)
> if ( flags )
> return FFA_RET_INVALID_PARAMETERS;
>
> - if ( FFA_ID_IS_SECURE(dest_id) && fw_notif_enabled )
> - return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
> - bitmap_hi);
> + if ( FFA_ID_IS_SECURE(dest_id) )
> + {
> + if ( fw_notif_enabled )
> + return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags,
> + bitmap_lo, bitmap_hi);
> + }
> + else if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
> + {
> + return notification_set_vm(dest_id, caller_id, flags,
> + ((uint64_t)bitmap_hi << 32) | bitmap_lo);
> + }
>
> return FFA_RET_NOT_SUPPORTED;
> }
> diff --git a/xen/arch/arm/tee/ffa_private.h b/xen/arch/arm/tee/ffa_private.h
> index 78a0a9815d56..923a071a9d7c 100644
> --- a/xen/arch/arm/tee/ffa_private.h
> +++ b/xen/arch/arm/tee/ffa_private.h
> @@ -340,20 +340,18 @@ struct ffa_ctx_notif {
> uint64_t vm_pending;
>
> /*
> - * Source endpoint bound to each VM notification ID (0 means unbound).
> + * Tracks whether an NPI has been raised for local pending notifications.
> + * Protected by notif_lock.
> */
> - uint16_t vm_bind[FFA_NUM_VM_NOTIF];
> + bool notif_irq_raised;
>
> /*
> - * Lock protecting the hypervisor-managed notification state.
> + * Source endpoint bound to each VM notification ID (0 means unbound).
> */
> - spinlock_t notif_lock;
> + uint16_t vm_bind[FFA_NUM_VM_NOTIF];
>
> - /*
> - * Tracks whether a local notification pending interrupt was raised.
> - * Protected by notif_lock.
> - */
> - bool notif_irq_raised;
> + /* Lock protecting local notification state. */
> + spinlock_t notif_lock;
>
> /*
> * Bitmap of pending hypervisor notifications (for HYP bitmap queries).
> @@ -495,6 +493,7 @@ void ffa_notif_init(void);
> void ffa_notif_init_interrupt(void);
> int ffa_notif_domain_init(struct domain *d);
> void ffa_notif_domain_destroy(struct domain *d);
> +bool ffa_notif_fw_enabled(void);
>
> int32_t ffa_handle_notification_bind(struct cpu_user_regs *regs);
> int32_t ffa_handle_notification_unbind(struct cpu_user_regs *regs);
> --
> 2.53.0
>
Hi Jens,
> On 5 May 2026, at 11:21, Jens Wiklander <jens.wiklander@linaro.org> wrote:
>
> Hi Bertrand,
>
> On Wed, Apr 29, 2026 at 7:44 AM Bertrand Marquis
> <bertrand.marquis@arm.com> wrote:
>>
>> VM notification binding and pending tracking exist for non-secure
>> endpoints, but FFA_NOTIFICATION_SET still only forwards secure
>> destinations to the SPMC. Non-secure VMs therefore cannot receive
>> notifications from other VMs. Local NPI delivery also needs explicit
>> re-arm tracking so repeated raises are not lost while the interrupt is
>> already pending.
>>
>> Add a local VM notification delivery path for non-secure destinations.
>> notification_set_vm() resolves the destination endpoint, verifies that
>> every requested bit is bound to the sender, sets the receiver's
>> vm_pending bitmap under notif_lock, and raises an NPI only when local
>> pending state is not already armed.
>>
>> Track whether a local NPI is already armed with notif_irq_raised,
>> clear that state once both VM and hypervisor pending bitmaps are
>> drained, and keep notif_lock held across the VM notification injection
>> attempt. If no destination vCPU is online, leave the pending bits set
>> and keep notif_irq_raised clear so delivery can be retried later.
>> Also expose firmware notification availability so FFA_FEATURES only
>> advertises notification support when it is actually provided by the
>> firmware or by CONFIG_FFA_VM_TO_VM.
>>
>> Functional impact: when CONFIG_FFA_VM_TO_VM is enabled, non-secure
>> FFA_NOTIFICATION_SET delivers VM-to-VM notifications locally and keeps
>> NPI delivery reliable across repeated raises.
>>
>> Signed-off-by: Bertrand Marquis <bertrand.marquis@arm.com>
>> ---
>> Changes since v1:
>> - serialize notification_set_vm() state updates with the NPI attempt
>> - keep pending VM notifications set when local injection fails
>> ---
>> xen/arch/arm/tee/ffa.c | 24 ++++++++--
>> xen/arch/arm/tee/ffa_notif.c | 82 ++++++++++++++++++++++++++++++++--
>> xen/arch/arm/tee/ffa_private.h | 17 ++++---
>> 3 files changed, 107 insertions(+), 16 deletions(-)
>>
>> diff --git a/xen/arch/arm/tee/ffa.c b/xen/arch/arm/tee/ffa.c
>> index 1fe33f26454a..7fe021049cba 100644
>> --- a/xen/arch/arm/tee/ffa.c
>> +++ b/xen/arch/arm/tee/ffa.c
>> @@ -39,8 +39,13 @@
>> * o FFA_MSG_SEND_DIRECT_REQ:
>> * - only supported from a VM to an SP
>> * o FFA_NOTIFICATION_*:
>> + * - only supported when firmware notifications are enabled or VM-to-VM
>> + * support is built in
>> * - only supports global notifications, that is, per vCPU notifications
>> - * are not supported
>> + * are not supported and secure per-vCPU notification information is
>> + * not forwarded
>> + * - the source endpoint ID reported for a notification may no longer
>> + * exist by the time the receiver consumes it
>> * - doesn't support signalling the secondary scheduler of pending
>> * notification for secure partitions
>> * - doesn't support notifications for Xen itself
>> @@ -245,6 +250,8 @@ static void handle_features(struct cpu_user_regs *regs)
>> uint32_t a1 = get_user_reg(regs, 1);
>> struct domain *d = current->domain;
>> struct ffa_ctx *ctx = d->arch.tee;
>> + bool notif_supported = IS_ENABLED(CONFIG_FFA_VM_TO_VM) ||
>> + ffa_notif_fw_enabled();
>>
>> /*
>> * FFA_FEATURES defines w2 as input properties only for specific
>> @@ -343,10 +350,16 @@ static void handle_features(struct cpu_user_regs *regs)
>>
>> break;
>> case FFA_FEATURE_NOTIF_PEND_INTR:
>> - ffa_set_regs_success(regs, GUEST_FFA_NOTIF_PEND_INTR_ID, 0);
>> + if ( notif_supported )
>> + ffa_set_regs_success(regs, GUEST_FFA_NOTIF_PEND_INTR_ID, 0);
>> + else
>> + ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
>> break;
>> case FFA_FEATURE_SCHEDULE_RECV_INTR:
>> - ffa_set_regs_success(regs, GUEST_FFA_SCHEDULE_RECV_INTR_ID, 0);
>> + if ( notif_supported )
>> + ffa_set_regs_success(regs, GUEST_FFA_SCHEDULE_RECV_INTR_ID, 0);
>> + else
>> + ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
>> break;
>> case FFA_PARTITION_INFO_GET_REGS:
>> if ( ACCESS_ONCE(ctx->guest_vers) >= FFA_VERSION_1_2 )
>> @@ -361,7 +374,10 @@ static void handle_features(struct cpu_user_regs *regs)
>> case FFA_NOTIFICATION_SET:
>> case FFA_NOTIFICATION_INFO_GET_32:
>> case FFA_NOTIFICATION_INFO_GET_64:
>> - ffa_set_regs_success(regs, 0, 0);
>> + if ( notif_supported )
>> + ffa_set_regs_success(regs, 0, 0);
>> + else
>> + ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
>> break;
>> default:
>> ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
>> diff --git a/xen/arch/arm/tee/ffa_notif.c b/xen/arch/arm/tee/ffa_notif.c
>> index a841c8f8d747..b29d948a7110 100644
>> --- a/xen/arch/arm/tee/ffa_notif.c
>> +++ b/xen/arch/arm/tee/ffa_notif.c
>> @@ -21,6 +21,11 @@ static bool __ro_after_init fw_notif_enabled;
>> static unsigned int __ro_after_init notif_sri_irq;
>> static DEFINE_SPINLOCK(notif_info_lock);
>>
>> +bool ffa_notif_fw_enabled(void)
>> +{
>> + return fw_notif_enabled;
>> +}
>> +
>> static bool inject_notif_pending(struct domain *d)
>> {
>> struct vcpu *v;
>> @@ -107,6 +112,55 @@ out_unlock:
>> return ret;
>> }
>>
>> +/*
>> + * Deliver a VM-to-VM notification. ctx->notif.notif_lock protects
>> + * vm_bind/vm_pending so callers must not hold it already.
>> + */
>> +static int32_t notification_set_vm(uint16_t dst_id, uint16_t src_id,
>> + uint32_t flags, uint64_t bitmap)
>> +{
>> + struct domain *dst_d;
>> + struct ffa_ctx *dst_ctx;
>> + unsigned int id;
>> + int32_t ret;
>> +
>> + if ( flags )
>> + return FFA_RET_INVALID_PARAMETERS;
>> +
>> + ret = ffa_endpoint_domain_lookup(dst_id, &dst_d, &dst_ctx);
>> + if ( ret )
>> + return ret;
>> +
>> + ret = FFA_RET_OK;
>> +
>> + spin_lock(&dst_ctx->notif.notif_lock);
>> +
>> + for ( id = 0; id < FFA_NUM_VM_NOTIF; id++ )
>> + {
>> + if ( !(bitmap & BIT(id, ULL)) )
>> + continue;
>> +
>> + if ( dst_ctx->notif.vm_bind[id] != src_id )
>> + {
>> + ret = FFA_RET_DENIED;
>> + goto out_unlock;
>> + }
>> + }
>> +
>> + dst_ctx->notif.vm_pending |= bitmap;
>> + if ( !dst_ctx->notif.notif_irq_raised &&
>> + (dst_ctx->notif.vm_pending || dst_ctx->notif.hyp_pending) &&
>> + inject_notif_pending(dst_d) )
>> + dst_ctx->notif.notif_irq_raised = true;
>> +
>> +out_unlock:
>> + spin_unlock(&dst_ctx->notif.notif_lock);
>> +
>> + rcu_unlock_domain(dst_d);
>> +
>> + return ret;
>> +}
>> +
>> int32_t ffa_handle_notification_bind(struct cpu_user_regs *regs)
>> {
>> struct domain *d = current->domain;
>> @@ -288,6 +342,8 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
>>
>> if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
>> {
>> + bool pending;
>> +
>> spin_lock(&ctx->notif.notif_lock);
>>
>> if ( (flags & FFA_NOTIF_FLAG_BITMAP_HYP) && ctx->notif.hyp_pending )
>> @@ -298,6 +354,18 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
>> ctx->notif.notif_irq_raised = false;
>> }
>>
>> + if ( (flags & FFA_NOTIF_FLAG_BITMAP_VM) && ctx->notif.vm_pending )
>> + {
>> + w4 = (uint32_t)(ctx->notif.vm_pending & GENMASK(31, 0));
>> + w5 = (uint32_t)((ctx->notif.vm_pending >> 32) & GENMASK(31, 0));
>> + ctx->notif.vm_pending = 0;
>> + }
>> +
>> + pending = (ctx->notif.hyp_pending != 0) ||
>> + (ctx->notif.vm_pending != 0);
>> + if ( !pending )
>> + ctx->notif.notif_irq_raised = false;
>
> This seems to take care of clearing notif_irq_raised for all cases. Do
> we still need the one just above this block (copied here):
> if ( !ctx->notif.vm_pending )
> ctx->notif.notif_irq_raised = false;
> ?
Yes you are right, this is now redundant.
I will drop it in v3.
Cheers
Bertrand
>
> Cheers,
> Jens
>
>> +
>> spin_unlock(&ctx->notif.notif_lock);
>> }
>>
>> @@ -323,9 +391,17 @@ int32_t ffa_handle_notification_set(struct cpu_user_regs *regs)
>> if ( flags )
>> return FFA_RET_INVALID_PARAMETERS;
>>
>> - if ( FFA_ID_IS_SECURE(dest_id) && fw_notif_enabled )
>> - return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
>> - bitmap_hi);
>> + if ( FFA_ID_IS_SECURE(dest_id) )
>> + {
>> + if ( fw_notif_enabled )
>> + return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags,
>> + bitmap_lo, bitmap_hi);
>> + }
>> + else if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
>> + {
>> + return notification_set_vm(dest_id, caller_id, flags,
>> + ((uint64_t)bitmap_hi << 32) | bitmap_lo);
>> + }
>>
>> return FFA_RET_NOT_SUPPORTED;
>> }
>> diff --git a/xen/arch/arm/tee/ffa_private.h b/xen/arch/arm/tee/ffa_private.h
>> index 78a0a9815d56..923a071a9d7c 100644
>> --- a/xen/arch/arm/tee/ffa_private.h
>> +++ b/xen/arch/arm/tee/ffa_private.h
>> @@ -340,20 +340,18 @@ struct ffa_ctx_notif {
>> uint64_t vm_pending;
>>
>> /*
>> - * Source endpoint bound to each VM notification ID (0 means unbound).
>> + * Tracks whether an NPI has been raised for local pending notifications.
>> + * Protected by notif_lock.
>> */
>> - uint16_t vm_bind[FFA_NUM_VM_NOTIF];
>> + bool notif_irq_raised;
>>
>> /*
>> - * Lock protecting the hypervisor-managed notification state.
>> + * Source endpoint bound to each VM notification ID (0 means unbound).
>> */
>> - spinlock_t notif_lock;
>> + uint16_t vm_bind[FFA_NUM_VM_NOTIF];
>>
>> - /*
>> - * Tracks whether a local notification pending interrupt was raised.
>> - * Protected by notif_lock.
>> - */
>> - bool notif_irq_raised;
>> + /* Lock protecting local notification state. */
>> + spinlock_t notif_lock;
>>
>> /*
>> * Bitmap of pending hypervisor notifications (for HYP bitmap queries).
>> @@ -495,6 +493,7 @@ void ffa_notif_init(void);
>> void ffa_notif_init_interrupt(void);
>> int ffa_notif_domain_init(struct domain *d);
>> void ffa_notif_domain_destroy(struct domain *d);
>> +bool ffa_notif_fw_enabled(void);
>>
>> int32_t ffa_handle_notification_bind(struct cpu_user_regs *regs);
>> int32_t ffa_handle_notification_unbind(struct cpu_user_regs *regs);
>> --
>> 2.53.0
>>
Hi Bertrand,
On Mon, May 11, 2026 at 3:17 PM Bertrand Marquis
<Bertrand.Marquis@arm.com> wrote:
>
> Hi Jens,
>
> > On 5 May 2026, at 11:21, Jens Wiklander <jens.wiklander@linaro.org> wrote:
> >
> > Hi Bertrand,
> >
> > On Wed, Apr 29, 2026 at 7:44 AM Bertrand Marquis
> > <bertrand.marquis@arm.com> wrote:
> >>
> >> VM notification binding and pending tracking exist for non-secure
> >> endpoints, but FFA_NOTIFICATION_SET still only forwards secure
> >> destinations to the SPMC. Non-secure VMs therefore cannot receive
> >> notifications from other VMs. Local NPI delivery also needs explicit
> >> re-arm tracking so repeated raises are not lost while the interrupt is
> >> already pending.
> >>
> >> Add a local VM notification delivery path for non-secure destinations.
> >> notification_set_vm() resolves the destination endpoint, verifies that
> >> every requested bit is bound to the sender, sets the receiver's
> >> vm_pending bitmap under notif_lock, and raises an NPI only when local
> >> pending state is not already armed.
> >>
> >> Track whether a local NPI is already armed with notif_irq_raised,
> >> clear that state once both VM and hypervisor pending bitmaps are
> >> drained, and keep notif_lock held across the VM notification injection
> >> attempt. If no destination vCPU is online, leave the pending bits set
> >> and keep notif_irq_raised clear so delivery can be retried later.
> >> Also expose firmware notification availability so FFA_FEATURES only
> >> advertises notification support when it is actually provided by the
> >> firmware or by CONFIG_FFA_VM_TO_VM.
> >>
> >> Functional impact: when CONFIG_FFA_VM_TO_VM is enabled, non-secure
> >> FFA_NOTIFICATION_SET delivers VM-to-VM notifications locally and keeps
> >> NPI delivery reliable across repeated raises.
> >>
> >> Signed-off-by: Bertrand Marquis <bertrand.marquis@arm.com>
> >> ---
> >> Changes since v1:
> >> - serialize notification_set_vm() state updates with the NPI attempt
> >> - keep pending VM notifications set when local injection fails
> >> ---
> >> xen/arch/arm/tee/ffa.c | 24 ++++++++--
> >> xen/arch/arm/tee/ffa_notif.c | 82 ++++++++++++++++++++++++++++++++--
> >> xen/arch/arm/tee/ffa_private.h | 17 ++++---
> >> 3 files changed, 107 insertions(+), 16 deletions(-)
> >>
> >> diff --git a/xen/arch/arm/tee/ffa.c b/xen/arch/arm/tee/ffa.c
> >> index 1fe33f26454a..7fe021049cba 100644
> >> --- a/xen/arch/arm/tee/ffa.c
> >> +++ b/xen/arch/arm/tee/ffa.c
> >> @@ -39,8 +39,13 @@
> >> * o FFA_MSG_SEND_DIRECT_REQ:
> >> * - only supported from a VM to an SP
> >> * o FFA_NOTIFICATION_*:
> >> + * - only supported when firmware notifications are enabled or VM-to-VM
> >> + * support is built in
> >> * - only supports global notifications, that is, per vCPU notifications
> >> - * are not supported
> >> + * are not supported and secure per-vCPU notification information is
> >> + * not forwarded
> >> + * - the source endpoint ID reported for a notification may no longer
> >> + * exist by the time the receiver consumes it
> >> * - doesn't support signalling the secondary scheduler of pending
> >> * notification for secure partitions
> >> * - doesn't support notifications for Xen itself
> >> @@ -245,6 +250,8 @@ static void handle_features(struct cpu_user_regs *regs)
> >> uint32_t a1 = get_user_reg(regs, 1);
> >> struct domain *d = current->domain;
> >> struct ffa_ctx *ctx = d->arch.tee;
> >> + bool notif_supported = IS_ENABLED(CONFIG_FFA_VM_TO_VM) ||
> >> + ffa_notif_fw_enabled();
> >>
> >> /*
> >> * FFA_FEATURES defines w2 as input properties only for specific
> >> @@ -343,10 +350,16 @@ static void handle_features(struct cpu_user_regs *regs)
> >>
> >> break;
> >> case FFA_FEATURE_NOTIF_PEND_INTR:
> >> - ffa_set_regs_success(regs, GUEST_FFA_NOTIF_PEND_INTR_ID, 0);
> >> + if ( notif_supported )
> >> + ffa_set_regs_success(regs, GUEST_FFA_NOTIF_PEND_INTR_ID, 0);
> >> + else
> >> + ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
> >> break;
> >> case FFA_FEATURE_SCHEDULE_RECV_INTR:
> >> - ffa_set_regs_success(regs, GUEST_FFA_SCHEDULE_RECV_INTR_ID, 0);
> >> + if ( notif_supported )
> >> + ffa_set_regs_success(regs, GUEST_FFA_SCHEDULE_RECV_INTR_ID, 0);
> >> + else
> >> + ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
> >> break;
> >> case FFA_PARTITION_INFO_GET_REGS:
> >> if ( ACCESS_ONCE(ctx->guest_vers) >= FFA_VERSION_1_2 )
> >> @@ -361,7 +374,10 @@ static void handle_features(struct cpu_user_regs *regs)
> >> case FFA_NOTIFICATION_SET:
> >> case FFA_NOTIFICATION_INFO_GET_32:
> >> case FFA_NOTIFICATION_INFO_GET_64:
> >> - ffa_set_regs_success(regs, 0, 0);
> >> + if ( notif_supported )
> >> + ffa_set_regs_success(regs, 0, 0);
> >> + else
> >> + ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
> >> break;
> >> default:
> >> ffa_set_regs_error(regs, FFA_RET_NOT_SUPPORTED);
> >> diff --git a/xen/arch/arm/tee/ffa_notif.c b/xen/arch/arm/tee/ffa_notif.c
> >> index a841c8f8d747..b29d948a7110 100644
> >> --- a/xen/arch/arm/tee/ffa_notif.c
> >> +++ b/xen/arch/arm/tee/ffa_notif.c
> >> @@ -21,6 +21,11 @@ static bool __ro_after_init fw_notif_enabled;
> >> static unsigned int __ro_after_init notif_sri_irq;
> >> static DEFINE_SPINLOCK(notif_info_lock);
> >>
> >> +bool ffa_notif_fw_enabled(void)
> >> +{
> >> + return fw_notif_enabled;
> >> +}
> >> +
> >> static bool inject_notif_pending(struct domain *d)
> >> {
> >> struct vcpu *v;
> >> @@ -107,6 +112,55 @@ out_unlock:
> >> return ret;
> >> }
> >>
> >> +/*
> >> + * Deliver a VM-to-VM notification. ctx->notif.notif_lock protects
> >> + * vm_bind/vm_pending so callers must not hold it already.
> >> + */
> >> +static int32_t notification_set_vm(uint16_t dst_id, uint16_t src_id,
> >> + uint32_t flags, uint64_t bitmap)
> >> +{
> >> + struct domain *dst_d;
> >> + struct ffa_ctx *dst_ctx;
> >> + unsigned int id;
> >> + int32_t ret;
> >> +
> >> + if ( flags )
> >> + return FFA_RET_INVALID_PARAMETERS;
> >> +
> >> + ret = ffa_endpoint_domain_lookup(dst_id, &dst_d, &dst_ctx);
> >> + if ( ret )
> >> + return ret;
> >> +
> >> + ret = FFA_RET_OK;
> >> +
> >> + spin_lock(&dst_ctx->notif.notif_lock);
> >> +
> >> + for ( id = 0; id < FFA_NUM_VM_NOTIF; id++ )
> >> + {
> >> + if ( !(bitmap & BIT(id, ULL)) )
> >> + continue;
> >> +
> >> + if ( dst_ctx->notif.vm_bind[id] != src_id )
> >> + {
> >> + ret = FFA_RET_DENIED;
> >> + goto out_unlock;
> >> + }
> >> + }
> >> +
> >> + dst_ctx->notif.vm_pending |= bitmap;
> >> + if ( !dst_ctx->notif.notif_irq_raised &&
> >> + (dst_ctx->notif.vm_pending || dst_ctx->notif.hyp_pending) &&
> >> + inject_notif_pending(dst_d) )
> >> + dst_ctx->notif.notif_irq_raised = true;
> >> +
> >> +out_unlock:
> >> + spin_unlock(&dst_ctx->notif.notif_lock);
> >> +
> >> + rcu_unlock_domain(dst_d);
> >> +
> >> + return ret;
> >> +}
> >> +
> >> int32_t ffa_handle_notification_bind(struct cpu_user_regs *regs)
> >> {
> >> struct domain *d = current->domain;
> >> @@ -288,6 +342,8 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
> >>
> >> if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
> >> {
> >> + bool pending;
> >> +
> >> spin_lock(&ctx->notif.notif_lock);
> >>
> >> if ( (flags & FFA_NOTIF_FLAG_BITMAP_HYP) && ctx->notif.hyp_pending )
> >> @@ -298,6 +354,18 @@ void ffa_handle_notification_get(struct cpu_user_regs *regs)
> >> ctx->notif.notif_irq_raised = false;
> >> }
> >>
> >> + if ( (flags & FFA_NOTIF_FLAG_BITMAP_VM) && ctx->notif.vm_pending )
> >> + {
> >> + w4 = (uint32_t)(ctx->notif.vm_pending & GENMASK(31, 0));
> >> + w5 = (uint32_t)((ctx->notif.vm_pending >> 32) & GENMASK(31, 0));
> >> + ctx->notif.vm_pending = 0;
> >> + }
> >> +
> >> + pending = (ctx->notif.hyp_pending != 0) ||
> >> + (ctx->notif.vm_pending != 0);
> >> + if ( !pending )
> >> + ctx->notif.notif_irq_raised = false;
> >
> > This seems to take care of clearing notif_irq_raised for all cases. Do
> > we still need the one just above this block (copied here):
> > if ( !ctx->notif.vm_pending )
> > ctx->notif.notif_irq_raised = false;
> > ?
>
> Yes you are right, this is now redundant.
> I will drop it in v3.
Good, thanks.
Cheers,
Jens
>
> Cheers
> Bertrand
>
> >
> > Cheers,
> > Jens
> >
> >> +
> >> spin_unlock(&ctx->notif.notif_lock);
> >> }
> >>
> >> @@ -323,9 +391,17 @@ int32_t ffa_handle_notification_set(struct cpu_user_regs *regs)
> >> if ( flags )
> >> return FFA_RET_INVALID_PARAMETERS;
> >>
> >> - if ( FFA_ID_IS_SECURE(dest_id) && fw_notif_enabled )
> >> - return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags, bitmap_lo,
> >> - bitmap_hi);
> >> + if ( FFA_ID_IS_SECURE(dest_id) )
> >> + {
> >> + if ( fw_notif_enabled )
> >> + return ffa_simple_call(FFA_NOTIFICATION_SET, src_dst, flags,
> >> + bitmap_lo, bitmap_hi);
> >> + }
> >> + else if ( IS_ENABLED(CONFIG_FFA_VM_TO_VM) )
> >> + {
> >> + return notification_set_vm(dest_id, caller_id, flags,
> >> + ((uint64_t)bitmap_hi << 32) | bitmap_lo);
> >> + }
> >>
> >> return FFA_RET_NOT_SUPPORTED;
> >> }
> >> diff --git a/xen/arch/arm/tee/ffa_private.h b/xen/arch/arm/tee/ffa_private.h
> >> index 78a0a9815d56..923a071a9d7c 100644
> >> --- a/xen/arch/arm/tee/ffa_private.h
> >> +++ b/xen/arch/arm/tee/ffa_private.h
> >> @@ -340,20 +340,18 @@ struct ffa_ctx_notif {
> >> uint64_t vm_pending;
> >>
> >> /*
> >> - * Source endpoint bound to each VM notification ID (0 means unbound).
> >> + * Tracks whether an NPI has been raised for local pending notifications.
> >> + * Protected by notif_lock.
> >> */
> >> - uint16_t vm_bind[FFA_NUM_VM_NOTIF];
> >> + bool notif_irq_raised;
> >>
> >> /*
> >> - * Lock protecting the hypervisor-managed notification state.
> >> + * Source endpoint bound to each VM notification ID (0 means unbound).
> >> */
> >> - spinlock_t notif_lock;
> >> + uint16_t vm_bind[FFA_NUM_VM_NOTIF];
> >>
> >> - /*
> >> - * Tracks whether a local notification pending interrupt was raised.
> >> - * Protected by notif_lock.
> >> - */
> >> - bool notif_irq_raised;
> >> + /* Lock protecting local notification state. */
> >> + spinlock_t notif_lock;
> >>
> >> /*
> >> * Bitmap of pending hypervisor notifications (for HYP bitmap queries).
> >> @@ -495,6 +493,7 @@ void ffa_notif_init(void);
> >> void ffa_notif_init_interrupt(void);
> >> int ffa_notif_domain_init(struct domain *d);
> >> void ffa_notif_domain_destroy(struct domain *d);
> >> +bool ffa_notif_fw_enabled(void);
> >>
> >> int32_t ffa_handle_notification_bind(struct cpu_user_regs *regs);
> >> int32_t ffa_handle_notification_unbind(struct cpu_user_regs *regs);
> >> --
> >> 2.53.0
> >>
>
© 2016 - 2026 Red Hat, Inc.