Move static event channel feature related code to a separate module
(static-evtchn.{c,h}) in the spirit of fine granular configuration, so
that the feature can be disabled if not needed.

Introduce Kconfig option CONFIG_STATIC_EVTCHN, enabled by default (to
keep the current behavior) dependent on CONFIG_DOM0LESS. While it could
be possible to create a loopback connection for dom0 only, this use case
does not really need this feature and all the docs and commit messages
refer explicitly to the use in dom0less system.

The only function visible externally is alloc_static_evtchn(), so move
the prototype to static-evtchn.h and provide a stub in case a feature
is disabled. Guard static_evtchn_created in struct dt_device_node as
well as its helpers.

Signed-off-by: Michal Orzel <michal.orzel@amd.com>
---
This is a follow up to Luca's dom0less features modularization.

Changes in v2:
 - keep static_evtchn_created helpers in device_tree.h, move macro to *.c
---
 xen/arch/arm/Kconfig                     |   8 ++
 xen/arch/arm/Makefile                    |   1 +
 xen/arch/arm/domain_build.c              | 146 --------------------
 xen/arch/arm/include/asm/setup.h         |   1 -
 xen/arch/arm/include/asm/static-evtchn.h |  25 ++++
 xen/arch/arm/setup.c                     |   1 +
 xen/arch/arm/static-evtchn.c             | 161 +++++++++++++++++++++++
 xen/include/xen/device_tree.h            |   4 +
 8 files changed, 200 insertions(+), 147 deletions(-)
 create mode 100644 xen/arch/arm/include/asm/static-evtchn.h
 create mode 100644 xen/arch/arm/static-evtchn.c

diff --git a/xen/arch/arm/Kconfig b/xen/arch/arm/Kconfig
index f73b62e50d00..50e9bfae1ac8 100644
--- a/xen/arch/arm/Kconfig
+++ b/xen/arch/arm/Kconfig
@@ -217,6 +217,14 @@ config STATIC_SHM
 	help
 	  This option enables statically shared memory on a dom0less system.
 
+config STATIC_EVTCHN
+	bool "Static event channel support on a dom0less system"
+	depends on DOM0LESS_BOOT
+	default y
+	help
+	  This option enables establishing static event channel communication
+	  between domains on a dom0less system (domU-domU as well as domU-dom0).
+
 endmenu
 
 menu "ARM errata workaround via the alternative framework"
diff --git a/xen/arch/arm/Makefile b/xen/arch/arm/Makefile
index 809772417c14..33c677672fe6 100644
--- a/xen/arch/arm/Makefile
+++ b/xen/arch/arm/Makefile
@@ -51,6 +51,7 @@ obj-y += setup.o
 obj-y += shutdown.o
 obj-y += smp.o
 obj-y += smpboot.o
+obj-$(CONFIG_STATIC_EVTCHN) += static-evtchn.init.o
 obj-$(CONFIG_STATIC_MEMORY) += static-memory.init.o
 obj-$(CONFIG_STATIC_SHM) += static-shmem.init.o
 obj-y += sysctl.o
diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c
index df66fb88d8ec..613b2885cebb 100644
--- a/xen/arch/arm/domain_build.c
+++ b/xen/arch/arm/domain_build.c
@@ -38,8 +38,6 @@
 #include <xen/grant_table.h>
 #include <xen/serial.h>
 
-#define STATIC_EVTCHN_NODE_SIZE_CELLS 2
-
 static unsigned int __initdata opt_dom0_max_vcpus;
 integer_param("dom0_max_vcpus", opt_dom0_max_vcpus);
 
@@ -1919,150 +1917,6 @@ void __init evtchn_allocate(struct domain *d)
     d->arch.hvm.params[HVM_PARAM_CALLBACK_IRQ] = val;
 }
 
-static int __init get_evtchn_dt_property(const struct dt_device_node *np,
-                                         uint32_t *port, uint32_t *phandle)
-{
-    const __be32 *prop = NULL;
-    uint32_t len;
-
-    prop = dt_get_property(np, "xen,evtchn", &len);
-    if ( !prop )
-    {
-        printk(XENLOG_ERR "xen,evtchn property should not be empty.\n");
-        return -EINVAL;
-    }
-
-    if ( !len || len < dt_cells_to_size(STATIC_EVTCHN_NODE_SIZE_CELLS) )
-    {
-        printk(XENLOG_ERR "xen,evtchn property value is not valid.\n");
-        return -EINVAL;
-    }
-
-    *port = dt_next_cell(1, &prop);
-    *phandle = dt_next_cell(1, &prop);
-
-    return 0;
-}
-
-static int __init alloc_domain_evtchn(struct dt_device_node *node)
-{
-    int rc;
-    uint32_t domU1_port, domU2_port, remote_phandle;
-    struct dt_device_node *remote_node;
-    const struct dt_device_node *p1_node, *p2_node;
-    struct evtchn_alloc_unbound alloc_unbound;
-    struct evtchn_bind_interdomain bind_interdomain;
-    struct domain *d1 = NULL, *d2 = NULL;
-
-    if ( !dt_device_is_compatible(node, "xen,evtchn-v1") )
-        return 0;
-
-    /*
-     * Event channel is already created while parsing the other side of
-     * evtchn node.
-     */
-    if ( dt_device_static_evtchn_created(node) )
-        return 0;
-
-    rc = get_evtchn_dt_property(node, &domU1_port, &remote_phandle);
-    if ( rc )
-        return rc;
-
-    remote_node = dt_find_node_by_phandle(remote_phandle);
-    if ( !remote_node )
-    {
-        printk(XENLOG_ERR
-                "evtchn: could not find remote evtchn phandle\n");
-        return -EINVAL;
-    }
-
-    rc = get_evtchn_dt_property(remote_node, &domU2_port, &remote_phandle);
-    if ( rc )
-        return rc;
-
-    if ( node->phandle != remote_phandle )
-    {
-        printk(XENLOG_ERR "xen,evtchn property is not setup correctly.\n");
-        return -EINVAL;
-    }
-
-    p1_node = dt_get_parent(node);
-    if ( !p1_node )
-    {
-        printk(XENLOG_ERR "evtchn: evtchn parent node is NULL\n" );
-        return -EINVAL;
-    }
-
-    p2_node = dt_get_parent(remote_node);
-    if ( !p2_node )
-    {
-        printk(XENLOG_ERR "evtchn: remote parent node is NULL\n" );
-        return -EINVAL;
-    }
-
-    d1 = get_domain_by_id(p1_node->used_by);
-    d2 = get_domain_by_id(p2_node->used_by);
-
-    if ( !d1 || !d2 )
-    {
-        printk(XENLOG_ERR "evtchn: could not find domains\n" );
-        return -EINVAL;
-    }
-
-    alloc_unbound.dom = d1->domain_id;
-    alloc_unbound.remote_dom = d2->domain_id;
-
-    rc = evtchn_alloc_unbound(&alloc_unbound, domU1_port);
-    if ( rc < 0 )
-    {
-        printk(XENLOG_ERR
-                "evtchn_alloc_unbound() failure (Error %d) \n", rc);
-        return rc;
-    }
-
-    bind_interdomain.remote_dom  = d1->domain_id;
-    bind_interdomain.remote_port = domU1_port;
-
-    rc = evtchn_bind_interdomain(&bind_interdomain, d2, domU2_port);
-    if ( rc < 0 )
-    {
-        printk(XENLOG_ERR
-                "evtchn_bind_interdomain() failure (Error %d) \n", rc);
-        return rc;
-    }
-
-    dt_device_set_static_evtchn_created(node);
-    dt_device_set_static_evtchn_created(remote_node);
-
-    return 0;
-}
-
-void __init alloc_static_evtchn(void)
-{
-    struct dt_device_node *node, *evtchn_node;
-    struct dt_device_node *chosen = dt_find_node_by_path("/chosen");
-
-    BUG_ON(chosen == NULL);
-
-    if ( hardware_domain )
-        dt_device_set_used_by(chosen, hardware_domain->domain_id);
-
-    dt_for_each_child_node(chosen, node)
-    {
-        if ( hardware_domain )
-        {
-            if ( alloc_domain_evtchn(node) != 0 )
-                panic("Could not set up domains evtchn\n");
-        }
-
-        dt_for_each_child_node(node, evtchn_node)
-        {
-            if ( alloc_domain_evtchn(evtchn_node) != 0 )
-                panic("Could not set up domains evtchn\n");
-        }
-    }
-}
-
 static void __init find_gnttab_region(struct domain *d,
                                       struct kernel_info *kinfo)
 {
diff --git a/xen/arch/arm/include/asm/setup.h b/xen/arch/arm/include/asm/setup.h
index bda3c07b8797..d15a88d2e0d1 100644
--- a/xen/arch/arm/include/asm/setup.h
+++ b/xen/arch/arm/include/asm/setup.h
@@ -136,7 +136,6 @@ void acpi_create_efi_mmap_table(struct domain *d,
 int acpi_make_efi_nodes(void *fdt, struct membank tbl_add[]);
 
 void create_dom0(void);
-void alloc_static_evtchn(void);
 
 void discard_initial_modules(void);
 void fw_unreserved_regions(paddr_t s, paddr_t e,
diff --git a/xen/arch/arm/include/asm/static-evtchn.h b/xen/arch/arm/include/asm/static-evtchn.h
new file mode 100644
index 000000000000..f964522f6a62
--- /dev/null
+++ b/xen/arch/arm/include/asm/static-evtchn.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#ifndef __ASM_STATIC_EVTCHN_H_
+#define __ASM_STATIC_EVTCHN_H_
+
+#ifdef CONFIG_STATIC_EVTCHN
+
+void alloc_static_evtchn(void);
+
+#else /* !CONFIG_STATIC_EVTCHN */
+
+static inline void alloc_static_evtchn(void) {};
+
+#endif /* CONFIG_STATIC_EVTCHN */
+
+#endif /* __ASM_STATIC_EVTCHN_H_ */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
index d07ea044db48..59dd9bb25a9f 100644
--- a/xen/arch/arm/setup.c
+++ b/xen/arch/arm/setup.c
@@ -36,6 +36,7 @@
 #include <asm/alternative.h>
 #include <asm/dom0less-build.h>
 #include <asm/page.h>
+#include <asm/static-evtchn.h>
 #include <asm/current.h>
 #include <asm/setup.h>
 #include <asm/gic.h>
diff --git a/xen/arch/arm/static-evtchn.c b/xen/arch/arm/static-evtchn.c
new file mode 100644
index 000000000000..49db08d5c6fc
--- /dev/null
+++ b/xen/arch/arm/static-evtchn.c
@@ -0,0 +1,161 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#include <xen/event.h>
+
+#include <asm/static-evtchn.h>
+
+#define STATIC_EVTCHN_NODE_SIZE_CELLS 2
+
+static int __init get_evtchn_dt_property(const struct dt_device_node *np,
+                                         uint32_t *port, uint32_t *phandle)
+{
+    const __be32 *prop = NULL;
+    uint32_t len;
+
+    prop = dt_get_property(np, "xen,evtchn", &len);
+    if ( !prop )
+    {
+        printk(XENLOG_ERR "xen,evtchn property should not be empty.\n");
+        return -EINVAL;
+    }
+
+    if ( !len || len < dt_cells_to_size(STATIC_EVTCHN_NODE_SIZE_CELLS) )
+    {
+        printk(XENLOG_ERR "xen,evtchn property value is not valid.\n");
+        return -EINVAL;
+    }
+
+    *port = dt_next_cell(1, &prop);
+    *phandle = dt_next_cell(1, &prop);
+
+    return 0;
+}
+
+static int __init alloc_domain_evtchn(struct dt_device_node *node)
+{
+    int rc;
+    uint32_t domU1_port, domU2_port, remote_phandle;
+    struct dt_device_node *remote_node;
+    const struct dt_device_node *p1_node, *p2_node;
+    struct evtchn_alloc_unbound alloc_unbound;
+    struct evtchn_bind_interdomain bind_interdomain;
+    struct domain *d1 = NULL, *d2 = NULL;
+
+    if ( !dt_device_is_compatible(node, "xen,evtchn-v1") )
+        return 0;
+
+    /*
+     * Event channel is already created while parsing the other side of
+     * evtchn node.
+     */
+    if ( dt_device_static_evtchn_created(node) )
+        return 0;
+
+    rc = get_evtchn_dt_property(node, &domU1_port, &remote_phandle);
+    if ( rc )
+        return rc;
+
+    remote_node = dt_find_node_by_phandle(remote_phandle);
+    if ( !remote_node )
+    {
+        printk(XENLOG_ERR
+                "evtchn: could not find remote evtchn phandle\n");
+        return -EINVAL;
+    }
+
+    rc = get_evtchn_dt_property(remote_node, &domU2_port, &remote_phandle);
+    if ( rc )
+        return rc;
+
+    if ( node->phandle != remote_phandle )
+    {
+        printk(XENLOG_ERR "xen,evtchn property is not setup correctly.\n");
+        return -EINVAL;
+    }
+
+    p1_node = dt_get_parent(node);
+    if ( !p1_node )
+    {
+        printk(XENLOG_ERR "evtchn: evtchn parent node is NULL\n" );
+        return -EINVAL;
+    }
+
+    p2_node = dt_get_parent(remote_node);
+    if ( !p2_node )
+    {
+        printk(XENLOG_ERR "evtchn: remote parent node is NULL\n" );
+        return -EINVAL;
+    }
+
+    d1 = get_domain_by_id(p1_node->used_by);
+    d2 = get_domain_by_id(p2_node->used_by);
+
+    if ( !d1 || !d2 )
+    {
+        printk(XENLOG_ERR "evtchn: could not find domains\n" );
+        return -EINVAL;
+    }
+
+    alloc_unbound.dom = d1->domain_id;
+    alloc_unbound.remote_dom = d2->domain_id;
+
+    rc = evtchn_alloc_unbound(&alloc_unbound, domU1_port);
+    if ( rc < 0 )
+    {
+        printk(XENLOG_ERR
+                "evtchn_alloc_unbound() failure (Error %d) \n", rc);
+        return rc;
+    }
+
+    bind_interdomain.remote_dom  = d1->domain_id;
+    bind_interdomain.remote_port = domU1_port;
+
+    rc = evtchn_bind_interdomain(&bind_interdomain, d2, domU2_port);
+    if ( rc < 0 )
+    {
+        printk(XENLOG_ERR
+                "evtchn_bind_interdomain() failure (Error %d) \n", rc);
+        return rc;
+    }
+
+    dt_device_set_static_evtchn_created(node);
+    dt_device_set_static_evtchn_created(remote_node);
+
+    return 0;
+}
+
+void __init alloc_static_evtchn(void)
+{
+    struct dt_device_node *node, *evtchn_node;
+    struct dt_device_node *chosen = dt_find_node_by_path("/chosen");
+
+    BUG_ON(chosen == NULL);
+
+    if ( hardware_domain )
+        dt_device_set_used_by(chosen, hardware_domain->domain_id);
+
+    dt_for_each_child_node(chosen, node)
+    {
+        if ( hardware_domain )
+        {
+            if ( alloc_domain_evtchn(node) != 0 )
+                panic("Could not set up domains evtchn\n");
+        }
+
+        dt_for_each_child_node(node, evtchn_node)
+        {
+            if ( alloc_domain_evtchn(evtchn_node) != 0 )
+                panic("Could not set up domains evtchn\n");
+        }
+    }
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/xen/include/xen/device_tree.h b/xen/include/xen/device_tree.h
index 3ae7b45429b6..94a836cb4e38 100644
--- a/xen/include/xen/device_tree.h
+++ b/xen/include/xen/device_tree.h
@@ -94,8 +94,10 @@ struct dt_device_node {
     /* IOMMU specific fields */
     bool is_protected;
 
+#ifdef CONFIG_STATIC_EVTCHN
     /* HACK: Remove this if there is a need of space */
     bool static_evtchn_created;
+#endif
 
     /*
      * The main purpose of this list is to link the structure in the list
@@ -366,6 +368,7 @@ static inline bool dt_property_name_is_equal(const struct dt_property *pp,
     return !dt_prop_cmp(pp->name, name);
 }
 
+#ifdef CONFIG_STATIC_EVTCHN
 static inline void
 dt_device_set_static_evtchn_created(struct dt_device_node *device)
 {
@@ -377,6 +380,7 @@ dt_device_static_evtchn_created(const struct dt_device_node *device)
 {
     return device->static_evtchn_created;
 }
+#endif /* CONFIG_STATIC_EVTCHN */
 
 /**
  * dt_find_compatible_node - Find a node based on type and one of the
-- 
2.25.1

> On 30 Nov 2023, at 09:57, Michal Orzel <michal.orzel@amd.com> wrote:
> 
> Move static event channel feature related code to a separate module
> (static-evtchn.{c,h}) in the spirit of fine granular configuration, so
> that the feature can be disabled if not needed.
> 
> Introduce Kconfig option CONFIG_STATIC_EVTCHN, enabled by default (to
> keep the current behavior) dependent on CONFIG_DOM0LESS. While it could
> be possible to create a loopback connection for dom0 only, this use case
> does not really need this feature and all the docs and commit messages
> refer explicitly to the use in dom0less system.
> 
> The only function visible externally is alloc_static_evtchn(), so move
> the prototype to static-evtchn.h and provide a stub in case a feature
> is disabled. Guard static_evtchn_created in struct dt_device_node as
> well as its helpers.
> 
> Signed-off-by: Michal Orzel <michal.orzel@amd.com>
> ---

Hi Michal,

FWIW because it is already Ack-ed.

Reviewed-by: Luca Fancellu <luca.fancellu@arm.com>

Hi Luca,

On 30/11/2023 13:17, Luca Fancellu wrote:
>> On 30 Nov 2023, at 09:57, Michal Orzel <michal.orzel@amd.com> wrote:
>>
>> Move static event channel feature related code to a separate module
>> (static-evtchn.{c,h}) in the spirit of fine granular configuration, so
>> that the feature can be disabled if not needed.
>>
>> Introduce Kconfig option CONFIG_STATIC_EVTCHN, enabled by default (to
>> keep the current behavior) dependent on CONFIG_DOM0LESS. While it could
>> be possible to create a loopback connection for dom0 only, this use case
>> does not really need this feature and all the docs and commit messages
>> refer explicitly to the use in dom0less system.
>>
>> The only function visible externally is alloc_static_evtchn(), so move
>> the prototype to static-evtchn.h and provide a stub in case a feature
>> is disabled. Guard static_evtchn_created in struct dt_device_node as
>> well as its helpers.
>>
>> Signed-off-by: Michal Orzel <michal.orzel@amd.com>
>> ---
> 
> Hi Michal,
> 
> FWIW because it is already Ack-ed.
> 
> Reviewed-by: Luca Fancellu <luca.fancellu@arm.com>

Usually when I provide an Acked-by, it means I went through the code and 
generally happy with the patch, but I didn't review throroughly. Even if 
I provide a reviewed-by, it is always useful to have extra pair of eyes 
:). So thanks for looking at it!

It is now committed.

Cheers,

> 
> 
> 

-- 
Julien Grall

Hi Michal,

On 30/11/2023 09:57, Michal Orzel wrote:
> Move static event channel feature related code to a separate module
> (static-evtchn.{c,h}) in the spirit of fine granular configuration, so
> that the feature can be disabled if not needed.
> 
> Introduce Kconfig option CONFIG_STATIC_EVTCHN, enabled by default (to
> keep the current behavior) dependent on CONFIG_DOM0LESS. While it could
> be possible to create a loopback connection for dom0 only, this use case
> does not really need this feature and all the docs and commit messages
> refer explicitly to the use in dom0less system.
> 
> The only function visible externally is alloc_static_evtchn(), so move
> the prototype to static-evtchn.h and provide a stub in case a feature
> is disabled. Guard static_evtchn_created in struct dt_device_node as
> well as its helpers.
> 
> Signed-off-by: Michal Orzel <michal.orzel@amd.com>

Acked-by: Julien Grall <jgrall@amazon.com>

Cheers,

-- 
Julien Grall