To allocate the xenstore event channel and initialize the grant table
entry, the xenstore domid is neeed.  A dom0 is created before the domUs,
so it is normally available through hardware_domain.  With capabilities
and dom0less, the xenstore domain may not be created first.

Keep the population of the page and HVM_PARAM_STORE_PFN in the normal
domain construction, but delay event channel creation and grant seeding
to after all domUs are created.  HVM_PARAM_STORE_PFN now serves as
indication to setup xenstore since the device tree is no longer
immediately available.  0 means no xenstore.  ~0ULL means legacy so only
the event channel needs setup, and any other value means to seed the
page.

dom0 needs to set xs_domid when it is serving as the xenstore domain.

The domain running xenstored needs to be the handler for VIRQ_DOM_EXC,
so set that as well - it otherwise defaults to hardware domain.

Signed-off-by: Jason Andryuk <jason.andryuk@amd.com>
---
v2:
Re-order ahead of seeding.
Fix created type in commit message
Change set_xs_domid to set_xs_domain
Set xenstore domain as VIRQ_DOM_EXC handler

Signed-off-by: Jason Andryuk <jason.andryuk@amd.com>
---
 xen/arch/arm/dom0less-build.c             | 67 +++++++++++++++++------
 xen/arch/arm/domain_build.c               |  2 +
 xen/arch/arm/include/asm/dom0less-build.h |  2 +
 3 files changed, 53 insertions(+), 18 deletions(-)

diff --git a/xen/arch/arm/dom0less-build.c b/xen/arch/arm/dom0less-build.c
index e25d7bd468..a46f292c1f 100644
--- a/xen/arch/arm/dom0less-build.c
+++ b/xen/arch/arm/dom0less-build.c
@@ -20,6 +20,15 @@
 #include <asm/static-memory.h>
 #include <asm/static-shmem.h>
 
+static domid_t __initdata xs_domid = DOMID_INVALID;
+static bool __initdata need_xenstore;
+
+void __init set_xs_domain(struct domain *d)
+{
+    xs_domid = d->domain_id;
+    set_global_virq_handler(d, VIRQ_DOM_EXC);
+}
+
 bool __init is_dom0less_mode(void)
 {
     struct bootmodules *mods = &bootinfo.modules;
@@ -694,7 +703,7 @@ static int __init alloc_xenstore_evtchn(struct domain *d)
     int rc;
 
     alloc.dom = d->domain_id;
-    alloc.remote_dom = hardware_domain->domain_id;
+    alloc.remote_dom = xs_domid;
     rc = evtchn_alloc_unbound(&alloc, 0);
     if ( rc )
     {
@@ -760,16 +769,10 @@ static int __init alloc_xenstore_params(struct kernel_info *kinfo)
     struct domain *d = kinfo->d;
     int rc = 0;
 
-    if ( kinfo->dom0less_feature & (DOM0LESS_XENSTORE | DOM0LESS_XS_LEGACY) )
-    {
-        ASSERT(hardware_domain);
-        rc = alloc_xenstore_evtchn(d);
-        if ( rc < 0 )
-            return rc;
+    if ( (kinfo->dom0less_feature & (DOM0LESS_XENSTORE | DOM0LESS_XS_LEGACY))
+                                 == (DOM0LESS_XENSTORE | DOM0LESS_XS_LEGACY) )
         d->arch.hvm.params[HVM_PARAM_STORE_PFN] = ~0ULL;
-    }
-
-    if ( kinfo->dom0less_feature & DOM0LESS_XENSTORE )
+    else if ( kinfo->dom0less_feature & DOM0LESS_XENSTORE )
     {
         rc = alloc_xenstore_page(d);
         if ( rc < 0 )
@@ -841,6 +844,30 @@ static void __init domain_vcpu_affinity(struct domain *d,
     }
 }
 
+static void __init initialize_domU_xenstore(void)
+{
+    struct domain *d;
+
+    if ( xs_domid == DOMID_INVALID )
+        return;
+
+    for_each_domain( d )
+    {
+        unsigned long gfn = d->arch.hvm.params[HVM_PARAM_STORE_PFN];
+        int rc;
+
+        if ( gfn == 0 )
+            continue;
+
+        if ( is_xenstore_domain(d) )
+            continue;
+
+        rc = alloc_xenstore_evtchn(d);
+        if ( rc < 0 )
+            panic("%pd: Failed to allocate xenstore_evtchn\n", d);
+    }
+}
+
 static int __init construct_domU(struct domain *d,
                                  const struct dt_device_node *node)
 {
@@ -881,17 +908,13 @@ static int __init construct_domU(struct domain *d,
          rc == -ENODATA ||
          (rc == 0 && !strcmp(dom0less_enhanced, "enabled")) )
     {
-        if ( hardware_domain )
-            kinfo.dom0less_feature = DOM0LESS_ENHANCED;
-        else
-            panic("At the moment, Xenstore support requires dom0 to be present\n");
+        need_xenstore = true;
+        kinfo.dom0less_feature = DOM0LESS_ENHANCED;
     }
     else if ( rc == 0 && !strcmp(dom0less_enhanced, "legacy") )
     {
-        if ( hardware_domain )
-            kinfo.dom0less_feature = DOM0LESS_ENHANCED_LEGACY;
-        else
-            panic("At the moment, Xenstore support requires dom0 to be present\n");
+        need_xenstore = true;
+        kinfo.dom0less_feature = DOM0LESS_ENHANCED_LEGACY;
     }
     else if ( rc == 0 && !strcmp(dom0less_enhanced, "no-xenstore") )
         kinfo.dom0less_feature = DOM0LESS_ENHANCED_NO_XS;
@@ -1138,7 +1161,15 @@ void __init create_domUs(void)
         if ( rc )
             panic("Could not set up domain %s (rc = %d)\n",
                   dt_node_name(node), rc);
+
+        if ( d_cfg.flags & XEN_DOMCTL_CDF_xs_domain )
+            set_xs_domain(d);
     }
+
+    if ( need_xenstore && xs_domid == DOMID_INVALID )
+        panic("xenstore requested, but xenstore domain not present\n");
+
+    initialize_domU_xenstore();
 }
 
 /*
diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c
index b3a396c2fc..d99ca7b43f 100644
--- a/xen/arch/arm/domain_build.c
+++ b/xen/arch/arm/domain_build.c
@@ -2408,6 +2408,8 @@ void __init create_dom0(void)
     rc = construct_dom0(dom0);
     if ( rc )
         panic("Could not set up DOM0 guest OS (rc = %d)\n", rc);
+
+    set_xs_domain(dom0);
 }
 
 /*
diff --git a/xen/arch/arm/include/asm/dom0less-build.h b/xen/arch/arm/include/asm/dom0less-build.h
index 5864944bda..b0e41a1954 100644
--- a/xen/arch/arm/include/asm/dom0less-build.h
+++ b/xen/arch/arm/include/asm/dom0less-build.h
@@ -9,6 +9,7 @@
 
 void create_domUs(void);
 bool is_dom0less_mode(void);
+void set_xs_domain(struct domain *d);
 
 #else /* !CONFIG_DOM0LESS_BOOT */
 
@@ -17,6 +18,7 @@ static inline bool is_dom0less_mode(void)
 {
     return false;
 }
+static inline void set_xs_domain(struct domain *d) {}
 
 #endif /* CONFIG_DOM0LESS_BOOT */
 
-- 
2.49.0

On Mon, 31 Mar 2025, Jason Andryuk wrote:
> To allocate the xenstore event channel and initialize the grant table
> entry, the xenstore domid is neeed.  A dom0 is created before the domUs,
> so it is normally available through hardware_domain.  With capabilities
> and dom0less, the xenstore domain may not be created first.
> 
> Keep the population of the page and HVM_PARAM_STORE_PFN in the normal
> domain construction, but delay event channel creation and grant seeding
> to after all domUs are created.  HVM_PARAM_STORE_PFN now serves as
> indication to setup xenstore since the device tree is no longer
> immediately available.  0 means no xenstore.  ~0ULL means legacy so only
> the event channel needs setup, and any other value means to seed the
> page.
> 
> dom0 needs to set xs_domid when it is serving as the xenstore domain.
> 
> The domain running xenstored needs to be the handler for VIRQ_DOM_EXC,
> so set that as well - it otherwise defaults to hardware domain.
> 
> Signed-off-by: Jason Andryuk <jason.andryuk@amd.com>
> ---
> v2:
> Re-order ahead of seeding.
> Fix created type in commit message
> Change set_xs_domid to set_xs_domain
> Set xenstore domain as VIRQ_DOM_EXC handler
> 
> Signed-off-by: Jason Andryuk <jason.andryuk@amd.com>
> ---
>  xen/arch/arm/dom0less-build.c             | 67 +++++++++++++++++------
>  xen/arch/arm/domain_build.c               |  2 +
>  xen/arch/arm/include/asm/dom0less-build.h |  2 +
>  3 files changed, 53 insertions(+), 18 deletions(-)
> 
> diff --git a/xen/arch/arm/dom0less-build.c b/xen/arch/arm/dom0less-build.c
> index e25d7bd468..a46f292c1f 100644
> --- a/xen/arch/arm/dom0less-build.c
> +++ b/xen/arch/arm/dom0less-build.c
> @@ -20,6 +20,15 @@
>  #include <asm/static-memory.h>
>  #include <asm/static-shmem.h>
>  
> +static domid_t __initdata xs_domid = DOMID_INVALID;
> +static bool __initdata need_xenstore;
> +
> +void __init set_xs_domain(struct domain *d)
> +{
> +    xs_domid = d->domain_id;
> +    set_global_virq_handler(d, VIRQ_DOM_EXC);
> +}
> +
>  bool __init is_dom0less_mode(void)
>  {
>      struct bootmodules *mods = &bootinfo.modules;
> @@ -694,7 +703,7 @@ static int __init alloc_xenstore_evtchn(struct domain *d)
>      int rc;
>  
>      alloc.dom = d->domain_id;
> -    alloc.remote_dom = hardware_domain->domain_id;
> +    alloc.remote_dom = xs_domid;
>      rc = evtchn_alloc_unbound(&alloc, 0);
>      if ( rc )
>      {
> @@ -760,16 +769,10 @@ static int __init alloc_xenstore_params(struct kernel_info *kinfo)
>      struct domain *d = kinfo->d;
>      int rc = 0;
>  
> -    if ( kinfo->dom0less_feature & (DOM0LESS_XENSTORE | DOM0LESS_XS_LEGACY) )
> -    {
> -        ASSERT(hardware_domain);
> -        rc = alloc_xenstore_evtchn(d);
> -        if ( rc < 0 )
> -            return rc;
> +    if ( (kinfo->dom0less_feature & (DOM0LESS_XENSTORE | DOM0LESS_XS_LEGACY))
> +                                 == (DOM0LESS_XENSTORE | DOM0LESS_XS_LEGACY) )
>          d->arch.hvm.params[HVM_PARAM_STORE_PFN] = ~0ULL;
> -    }
> -
> -    if ( kinfo->dom0less_feature & DOM0LESS_XENSTORE )
> +    else if ( kinfo->dom0less_feature & DOM0LESS_XENSTORE )
>      {
>          rc = alloc_xenstore_page(d);
>          if ( rc < 0 )
> @@ -841,6 +844,30 @@ static void __init domain_vcpu_affinity(struct domain *d,
>      }
>  }
>  
> +static void __init initialize_domU_xenstore(void)
> +{
> +    struct domain *d;
> +
> +    if ( xs_domid == DOMID_INVALID )
> +        return;
> +
> +    for_each_domain( d )
> +    {
> +        unsigned long gfn = d->arch.hvm.params[HVM_PARAM_STORE_PFN];

This should be uint64_t. With that change:

Reviewed-by: Stefano Stabellini <sstabellini@kernel.org>


> +        int rc;
> +
> +        if ( gfn == 0 )
> +            continue;
> +
> +        if ( is_xenstore_domain(d) )
> +            continue;
> +
> +        rc = alloc_xenstore_evtchn(d);
> +        if ( rc < 0 )
> +            panic("%pd: Failed to allocate xenstore_evtchn\n", d);
> +    }
> +}
> +
>  static int __init construct_domU(struct domain *d,
>                                   const struct dt_device_node *node)
>  {