When constructing domU, and specifically the event channels for their console
and xenstore event channels, the domid for the backing domain must be known.
Therefore, the control, hardware, and xenstore domains are deemed as core
domains, and must be constructed before any of the other domains.

This commit introduces the build_core_domains() function that will ensure the
core domains are constructed first.

Signed-off-by: Daniel P. Smith <dpsmith@apertussolutions.com>
---
 xen/arch/x86/domain-builder/core.c     | 68 ++++++++++++++++++++++++--
 xen/arch/x86/include/asm/boot-domain.h |  2 +
 2 files changed, 66 insertions(+), 4 deletions(-)

diff --git a/xen/arch/x86/domain-builder/core.c b/xen/arch/x86/domain-builder/core.c
index 901efce62a61..f693aa46d278 100644
--- a/xen/arch/x86/domain-builder/core.c
+++ b/xen/arch/x86/domain-builder/core.c
@@ -103,18 +103,78 @@ void __init builder_init(struct boot_info *bi)
     }
 }
 
+static int  __init build_core_domains(struct boot_info *bi)
+{
+    int count = 0;
+    struct boot_domain *bd;
+    int hw, cd, xs;
+
+    cd = first_boot_domain_index(bi, BUILD_CAPS_CONTROL);
+    if ( cd > MAX_NR_BOOTDOMS )
+        printk(XENLOG_WARNING "No control domain was defined\n");
+    else
+    {
+        bd = &bi->domains[cd];
+
+        arch_create_dom(bi, bd);
+        if ( bd->d )
+        {
+            bd->constructed = true;
+            count++;
+        }
+    }
+
+    hw = first_boot_domain_index(bi, BUILD_CAPS_HARDWARE);
+    if ( hw > MAX_NR_BOOTDOMS )
+        printk(XENLOG_WARNING "No hardware domain was defined\n");
+    else
+    {
+        if ( hw != cd )
+        {
+            bd = &bi->domains[hw];
+
+            arch_create_dom(bi, bd);
+            if ( bd->d )
+            {
+                bd->constructed = true;
+                count++;
+            }
+        }
+    }
+
+    xs = first_boot_domain_index(bi, BUILD_CAPS_XENSTORE);
+    if ( xs > MAX_NR_BOOTDOMS )
+        printk(XENLOG_WARNING "No xenstore domain was defined\n");
+    else
+    {
+        if ( xs != cd && xs != hw )
+        {
+            bd = &bi->domains[xs];
+
+            arch_create_dom(bi, bd);
+            if ( bd->d )
+            {
+                bd->constructed = true;
+                count++;
+            }
+        }
+    }
+
+    return count;
+}
+
 unsigned int __init builder_create_domains(struct boot_info *bi)
 {
     unsigned int build_count = 0;
     struct boot_domain *bd = &bi->domains[0];
 
+    if ( bi->nr_domains == 0 )
+        panic("%s: no domains defined\n", __func__);
+
     if ( bd->kernel == NULL && bd->capabilities & BUILD_CAPS_CONTROL )
         panic("%s: control domain missing kernel\n", __func__);
 
-
-    arch_create_dom(bi, bd);
-    if ( bd->d )
-        build_count++;
+    build_count = build_core_domains(bi);
 
     /* Free temporary buffers. */
     free_boot_modules();
diff --git a/xen/arch/x86/include/asm/boot-domain.h b/xen/arch/x86/include/asm/boot-domain.h
index df2bfa0c94fa..a574f4941ed3 100644
--- a/xen/arch/x86/include/asm/boot-domain.h
+++ b/xen/arch/x86/include/asm/boot-domain.h
@@ -42,6 +42,8 @@ struct boot_domain {
         xen_pfn_t gfn;
         evtchn_port_t evtchn;
     } console, store;
+
+    bool constructed;
 };
 
 #endif
-- 
2.30.2

On 2025-04-19 18:08, Daniel P. Smith wrote:
> When constructing domU, and specifically the event channels for their console
> and xenstore event channels, the domid for the backing domain must be known.
> Therefore, the control, hardware, and xenstore domains are deemed as core
> domains, and must be constructed before any of the other domains.
> 
> This commit introduces the build_core_domains() function that will ensure the
> core domains are constructed first.
> 
> Signed-off-by: Daniel P. Smith <dpsmith@apertussolutions.com>
> ---
>   xen/arch/x86/domain-builder/core.c     | 68 ++++++++++++++++++++++++--
>   xen/arch/x86/include/asm/boot-domain.h |  2 +
>   2 files changed, 66 insertions(+), 4 deletions(-)
> 
> diff --git a/xen/arch/x86/domain-builder/core.c b/xen/arch/x86/domain-builder/core.c
> index 901efce62a61..f693aa46d278 100644
> --- a/xen/arch/x86/domain-builder/core.c
> +++ b/xen/arch/x86/domain-builder/core.c
> @@ -103,18 +103,78 @@ void __init builder_init(struct boot_info *bi)
>       }
>   }
>   
> +static int  __init build_core_domains(struct boot_info *bi)
> +{
> +    int count = 0;
> +    struct boot_domain *bd;
> +    int hw, cd, xs;
> +
> +    cd = first_boot_domain_index(bi, BUILD_CAPS_CONTROL);

> +    hw = first_boot_domain_index(bi, BUILD_CAPS_HARDWARE);

> +    xs = first_boot_domain_index(bi, BUILD_CAPS_XENSTORE);

This order has issues if you actually have disaggregated domains.

Control and Hardware depend on Xenstore for Xenstore.

Control and Xenstore depend on Hardware for console support.

I re-worked the xenstore allocation to run after domain creation.  I've 
upstreamed that for dom0less (and ARM doesn't have to deal with consoles).

So if xenstore allocation is moved later, Hardware, Xenstore, then 
Control works.  But xenstore and console could both be handled after the 
fact and then the construction order doesn't matter.  The backend domid 
is needed to construct the event channel and grant entry.  With assigned 
domids, alloc_store_evtchn()/alloc_console_evtchn() can operate on the 
domids instead of expecting the domain to have been constructed.

Regards,
Jason

On 4/23/25 15:50, Jason Andryuk wrote:
> On 2025-04-19 18:08, Daniel P. Smith wrote:
>> When constructing domU, and specifically the event channels for their 
>> console
>> and xenstore event channels, the domid for the backing domain must be 
>> known.
>> Therefore, the control, hardware, and xenstore domains are deemed as core
>> domains, and must be constructed before any of the other domains.
>>
>> This commit introduces the build_core_domains() function that will 
>> ensure the
>> core domains are constructed first.
>>
>> Signed-off-by: Daniel P. Smith <dpsmith@apertussolutions.com>
>> ---
>>   xen/arch/x86/domain-builder/core.c     | 68 ++++++++++++++++++++++++--
>>   xen/arch/x86/include/asm/boot-domain.h |  2 +
>>   2 files changed, 66 insertions(+), 4 deletions(-)
>>
>> diff --git a/xen/arch/x86/domain-builder/core.c b/xen/arch/x86/domain- 
>> builder/core.c
>> index 901efce62a61..f693aa46d278 100644
>> --- a/xen/arch/x86/domain-builder/core.c
>> +++ b/xen/arch/x86/domain-builder/core.c
>> @@ -103,18 +103,78 @@ void __init builder_init(struct boot_info *bi)
>>       }
>>   }
>> +static int  __init build_core_domains(struct boot_info *bi)
>> +{
>> +    int count = 0;
>> +    struct boot_domain *bd;
>> +    int hw, cd, xs;
>> +
>> +    cd = first_boot_domain_index(bi, BUILD_CAPS_CONTROL);
> 
>> +    hw = first_boot_domain_index(bi, BUILD_CAPS_HARDWARE);
> 
>> +    xs = first_boot_domain_index(bi, BUILD_CAPS_XENSTORE);
> 
> This order has issues if you actually have disaggregated domains.
> 
> Control and Hardware depend on Xenstore for Xenstore.
> 
> Control and Xenstore depend on Hardware for console support.
> 
> I re-worked the xenstore allocation to run after domain creation.  I've 
> upstreamed that for dom0less (and ARM doesn't have to deal with consoles).
> 
> So if xenstore allocation is moved later, Hardware, Xenstore, then 
> Control works.  But xenstore and console could both be handled after the 
> fact and then the construction order doesn't matter.  The backend domid 
> is needed to construct the event channel and grant entry.  With assigned 
> domids, alloc_store_evtchn()/alloc_console_evtchn() can operate on the 
> domids instead of expecting the domain to have been constructed.

Actually, there is a larger issue here and this is going to be reworked.

V/r,
dps