From: "Edgar E. Iglesias" <edgar.iglesias@amd.com>

Add support for optionally creating a PCIe/GPEX controller.

Signed-off-by: Edgar E. Iglesias <edgar.iglesias@amd.com>
Reviewed-by: Stefano Stabellini <sstabellini@kernel.org>
---
 hw/xen/xen-pvh-common.c         | 76 +++++++++++++++++++++++++++++++++
 include/hw/xen/xen-pvh-common.h | 29 +++++++++++++
 2 files changed, 105 insertions(+)

diff --git a/hw/xen/xen-pvh-common.c b/hw/xen/xen-pvh-common.c
index 295f920442..28d7168446 100644
--- a/hw/xen/xen-pvh-common.c
+++ b/hw/xen/xen-pvh-common.c
@@ -122,6 +122,64 @@ static void xen_enable_tpm(XenPVHMachineState *s)
 }
 #endif
 
+/*
+ * We use the GPEX PCIe controller with its internal INTX PCI interrupt
+ * swizzling. This swizzling is emulated in QEMU and routes all INTX
+ * interrupts from endpoints down to only 4 INTX interrupts.
+ * See include/hw/pci/pci.h : pci_swizzle()
+ */
+static inline void xenpvh_gpex_init(XenPVHMachineState *s,
+                                    XenPVHMachineClass *xpc,
+                                    MemoryRegion *sysmem)
+{
+    MemoryRegion *ecam_reg;
+    MemoryRegion *mmio_reg;
+    DeviceState *dev;
+    int i;
+
+    object_initialize_child(OBJECT(s), "gpex", &s->pci.gpex,
+                            TYPE_GPEX_HOST);
+    dev = DEVICE(&s->pci.gpex);
+    sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal);
+
+    ecam_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 0);
+    memory_region_add_subregion(sysmem, s->cfg.pci_ecam.base, ecam_reg);
+
+    mmio_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 1);
+
+    if (s->cfg.pci_mmio.size) {
+        memory_region_init_alias(&s->pci.mmio_alias, OBJECT(dev), "pcie-mmio",
+                                 mmio_reg,
+                                 s->cfg.pci_mmio.base, s->cfg.pci_mmio.size);
+        memory_region_add_subregion(sysmem, s->cfg.pci_mmio.base,
+                                    &s->pci.mmio_alias);
+    }
+
+    if (s->cfg.pci_mmio_high.size) {
+        memory_region_init_alias(&s->pci.mmio_high_alias, OBJECT(dev),
+                "pcie-mmio-high",
+                mmio_reg, s->cfg.pci_mmio_high.base, s->cfg.pci_mmio_high.size);
+        memory_region_add_subregion(sysmem, s->cfg.pci_mmio_high.base,
+                &s->pci.mmio_high_alias);
+    }
+
+    /*
+     * PVH implementations with PCI enabled must provide set_pci_intx_irq()
+     * and optionally an implementation of set_pci_link_route().
+     */
+    assert(xpc->set_pci_intx_irq);
+
+    for (i = 0; i < GPEX_NUM_IRQS; i++) {
+        qemu_irq irq = qemu_allocate_irq(xpc->set_pci_intx_irq, s, i);
+
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), i, irq);
+        gpex_set_irq_num(GPEX_HOST(dev), i, s->cfg.pci_intx_irq_base + i);
+        if (xpc->set_pci_link_route) {
+            xpc->set_pci_link_route(i, s->cfg.pci_intx_irq_base + i);
+        }
+    }
+}
+
 static void xen_pvh_init(MachineState *ms)
 {
     XenPVHMachineState *s = XEN_PVH_MACHINE(ms);
@@ -152,6 +210,15 @@ static void xen_pvh_init(MachineState *ms)
     }
 #endif
 
+    /* Non-zero pci-ecam-size enables PCI.  */
+    if (s->cfg.pci_ecam.size) {
+        if (s->cfg.pci_ecam.size != 256 * MiB) {
+            error_report("pci-ecam-size only supports values 0 or 0x10000000");
+            exit(EXIT_FAILURE);
+        }
+        xenpvh_gpex_init(s, xpc, sysmem);
+    }
+
     /* Call the implementation specific init.  */
     if (xpc->init) {
         xpc->init(ms);
@@ -200,6 +267,9 @@ XEN_PVH_PROP_MEMMAP(ram_high)
 /* TPM only has a base-addr option.  */
 XEN_PVH_PROP_MEMMAP_BASE(tpm)
 XEN_PVH_PROP_MEMMAP(virtio_mmio)
+XEN_PVH_PROP_MEMMAP(pci_ecam)
+XEN_PVH_PROP_MEMMAP(pci_mmio)
+XEN_PVH_PROP_MEMMAP(pci_mmio_high)
 
 void xen_pvh_class_setup_common_props(XenPVHMachineClass *xpc)
 {
@@ -242,6 +312,12 @@ do {                                                                      \
         OC_MEMMAP_PROP(oc, "virtio-mmio", virtio_mmio);
     }
 
+    if (xpc->has_pci) {
+        OC_MEMMAP_PROP(oc, "pci-ecam", pci_ecam);
+        OC_MEMMAP_PROP(oc, "pci-mmio", pci_mmio);
+        OC_MEMMAP_PROP(oc, "pci-mmio-high", pci_mmio_high);
+    }
+
 #ifdef CONFIG_TPM
     if (xpc->has_tpm) {
         object_class_property_add(oc, "tpm-base-addr", "uint64_t",
diff --git a/include/hw/xen/xen-pvh-common.h b/include/hw/xen/xen-pvh-common.h
index 77fd98b9fe..bc09eea936 100644
--- a/include/hw/xen/xen-pvh-common.h
+++ b/include/hw/xen/xen-pvh-common.h
@@ -25,10 +25,29 @@ struct XenPVHMachineClass {
     /* PVH implementation specific init.  */
     void (*init)(MachineState *state);
 
+    /*
+     * set_pci_intx_irq - Deliver INTX irqs to the guest.
+     *
+     * @opaque: pointer to XenPVHMachineState.
+     * @irq: IRQ after swizzling, between 0-3.
+     * @level: IRQ level.
+     */
+    void (*set_pci_intx_irq)(void *opaque, int irq, int level);
+
+    /*
+     * set_pci_link_route: - optional implementation call to setup
+     * routing between INTX IRQ (0 - 3) and GSI's.
+     *
+     * @line: line the INTx line (0 => A .. 3 => B)
+     * @irq: GSI
+     */
+    int (*set_pci_link_route)(uint8_t line, uint8_t irq);
+
     /*
      * Each implementation can optionally enable features that it
      * supports and are known to work.
      */
+    bool has_pci;
     bool has_tpm;
     bool has_virtio_mmio;
 };
@@ -44,6 +63,12 @@ struct XenPVHMachineState {
         MemoryRegion high;
     } ram;
 
+    struct {
+        GPEXHost gpex;
+        MemoryRegion mmio_alias;
+        MemoryRegion mmio_high_alias;
+    } pci;
+
     struct {
         MemMapEntry ram_low, ram_high;
         MemMapEntry tpm;
@@ -52,6 +77,10 @@ struct XenPVHMachineState {
         MemMapEntry virtio_mmio;
         uint32_t virtio_mmio_num;
         uint32_t virtio_mmio_irq_base;
+
+        /* PCI */
+        MemMapEntry pci_ecam, pci_mmio, pci_mmio_high;
+        uint32_t pci_intx_irq_base;
     } cfg;
 };
 
-- 
2.43.0

Hi Edgar,

On 4/9/24 18:15, Edgar E. Iglesias wrote:
> From: "Edgar E. Iglesias" <edgar.iglesias@amd.com>
> 
> Add support for optionally creating a PCIe/GPEX controller.
> 
> Signed-off-by: Edgar E. Iglesias <edgar.iglesias@amd.com>
> Reviewed-by: Stefano Stabellini <sstabellini@kernel.org>
> ---
>   hw/xen/xen-pvh-common.c         | 76 +++++++++++++++++++++++++++++++++
>   include/hw/xen/xen-pvh-common.h | 29 +++++++++++++
>   2 files changed, 105 insertions(+)


> +/*
> + * We use the GPEX PCIe controller with its internal INTX PCI interrupt
> + * swizzling. This swizzling is emulated in QEMU and routes all INTX
> + * interrupts from endpoints down to only 4 INTX interrupts.
> + * See include/hw/pci/pci.h : pci_swizzle()
> + */
> +static inline void xenpvh_gpex_init(XenPVHMachineState *s,
> +                                    XenPVHMachineClass *xpc,
> +                                    MemoryRegion *sysmem)
> +{
> +    MemoryRegion *ecam_reg;
> +    MemoryRegion *mmio_reg;
> +    DeviceState *dev;
> +    int i;
> +
> +    object_initialize_child(OBJECT(s), "gpex", &s->pci.gpex,
> +                            TYPE_GPEX_HOST);
> +    dev = DEVICE(&s->pci.gpex);
> +    sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal);
> +
> +    ecam_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 0);
> +    memory_region_add_subregion(sysmem, s->cfg.pci_ecam.base, ecam_reg);
> +
> +    mmio_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 1);
> +
> +    if (s->cfg.pci_mmio.size) {
> +        memory_region_init_alias(&s->pci.mmio_alias, OBJECT(dev), "pcie-mmio",
> +                                 mmio_reg,
> +                                 s->cfg.pci_mmio.base, s->cfg.pci_mmio.size);
> +        memory_region_add_subregion(sysmem, s->cfg.pci_mmio.base,
> +                                    &s->pci.mmio_alias);
> +    }
> +
> +    if (s->cfg.pci_mmio_high.size) {
> +        memory_region_init_alias(&s->pci.mmio_high_alias, OBJECT(dev),
> +                "pcie-mmio-high",
> +                mmio_reg, s->cfg.pci_mmio_high.base, s->cfg.pci_mmio_high.size);
> +        memory_region_add_subregion(sysmem, s->cfg.pci_mmio_high.base,
> +                &s->pci.mmio_high_alias);
> +    }
> +
> +    /*
> +     * PVH implementations with PCI enabled must provide set_pci_intx_irq()
> +     * and optionally an implementation of set_pci_link_route().
> +     */
> +    assert(xpc->set_pci_intx_irq);
> +
> +    for (i = 0; i < GPEX_NUM_IRQS; i++) {
> +        qemu_irq irq = qemu_allocate_irq(xpc->set_pci_intx_irq, s, i);
> +
> +        sysbus_connect_irq(SYS_BUS_DEVICE(dev), i, irq);
> +        gpex_set_irq_num(GPEX_HOST(dev), i, s->cfg.pci_intx_irq_base + i);
> +        if (xpc->set_pci_link_route) {
> +            xpc->set_pci_link_route(i, s->cfg.pci_intx_irq_base + i);
> +        }
> +    }
> +}

Some Kconfig selector seems missing here:

/usr/bin/ld: libqemu-aarch64-softmmu.a.p/hw_xen_xen-pvh-common.c.o: in 
function `xenpvh_gpex_init':
hw/xen/xen-pvh-common.c:174: undefined reference to `gpex_set_irq_num'
/usr/bin/ld: libqemu-aarch64-softmmu.a.p/hw_xen_xen-hvm-common.c.o: in 
function `pci_dev_bus_num':
include/hw/pci/pci.h:337: undefined reference to `pci_bus_num'
/usr/bin/ld: include/hw/pci/pci.h:337: undefined reference to `pci_bus_num'
/usr/bin/ld: include/hw/pci/pci.h:337: undefined reference to `pci_bus_num'
/usr/bin/ld: include/hw/pci/pci.h:337: undefined reference to `pci_bus_num'
/usr/bin/ld: include/hw/pci/pci.h:337: undefined reference to `pci_bus_num'
/usr/bin/ld: libqemu-aarch64-softmmu.a.p/hw_xen_xen-hvm-common.c.o: in 
function `cpu_ioreq_config':
hw/xen/xen-hvm-common.c:412: undefined reference to 
`pci_host_config_read_common'
/usr/bin/ld: hw/xen/xen-hvm-common.c:428: undefined reference to 
`pci_host_config_read_common'
/usr/bin/ld: hw/xen/xen-hvm-common.c:438: undefined reference to 
`pci_host_config_write_common'

The current 'XEN' key represents both the "accelerator" part and
the common Xen HW, which isn't helping to follow. Anyhow, this
snippet fixes the build issue:

-- >8 --
diff --git a/accel/Kconfig b/accel/Kconfig
index 794e0d18d2..4263cab722 100644
--- a/accel/Kconfig
+++ b/accel/Kconfig
@@ -16,4 +16,5 @@ config KVM
  config XEN
      bool
      select FSDEV_9P if VIRTFS
+    select PCI_EXPRESS_GENERIC_BRIDGE
      select XEN_BUS
---

I'll post a patch later.

Regards,

Phil.

On Tue, Feb 18, 2025 at 6:02 AM Philippe Mathieu-Daudé <philmd@linaro.org>
wrote:

> Hi Edgar,
>
> On 4/9/24 18:15, Edgar E. Iglesias wrote:
> > From: "Edgar E. Iglesias" <edgar.iglesias@amd.com>
> >
> > Add support for optionally creating a PCIe/GPEX controller.
> >
> > Signed-off-by: Edgar E. Iglesias <edgar.iglesias@amd.com>
> > Reviewed-by: Stefano Stabellini <sstabellini@kernel.org>
> > ---
> >   hw/xen/xen-pvh-common.c         | 76 +++++++++++++++++++++++++++++++++
> >   include/hw/xen/xen-pvh-common.h | 29 +++++++++++++
> >   2 files changed, 105 insertions(+)
>
>
> > +/*
> > + * We use the GPEX PCIe controller with its internal INTX PCI interrupt
> > + * swizzling. This swizzling is emulated in QEMU and routes all INTX
> > + * interrupts from endpoints down to only 4 INTX interrupts.
> > + * See include/hw/pci/pci.h : pci_swizzle()
> > + */
> > +static inline void xenpvh_gpex_init(XenPVHMachineState *s,
> > +                                    XenPVHMachineClass *xpc,
> > +                                    MemoryRegion *sysmem)
> > +{
> > +    MemoryRegion *ecam_reg;
> > +    MemoryRegion *mmio_reg;
> > +    DeviceState *dev;
> > +    int i;
> > +
> > +    object_initialize_child(OBJECT(s), "gpex", &s->pci.gpex,
> > +                            TYPE_GPEX_HOST);
> > +    dev = DEVICE(&s->pci.gpex);
> > +    sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal);
> > +
> > +    ecam_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 0);
> > +    memory_region_add_subregion(sysmem, s->cfg.pci_ecam.base, ecam_reg);
> > +
> > +    mmio_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 1);
> > +
> > +    if (s->cfg.pci_mmio.size) {
> > +        memory_region_init_alias(&s->pci.mmio_alias, OBJECT(dev),
> "pcie-mmio",
> > +                                 mmio_reg,
> > +                                 s->cfg.pci_mmio.base,
> s->cfg.pci_mmio.size);
> > +        memory_region_add_subregion(sysmem, s->cfg.pci_mmio.base,
> > +                                    &s->pci.mmio_alias);
> > +    }
> > +
> > +    if (s->cfg.pci_mmio_high.size) {
> > +        memory_region_init_alias(&s->pci.mmio_high_alias, OBJECT(dev),
> > +                "pcie-mmio-high",
> > +                mmio_reg, s->cfg.pci_mmio_high.base,
> s->cfg.pci_mmio_high.size);
> > +        memory_region_add_subregion(sysmem, s->cfg.pci_mmio_high.base,
> > +                &s->pci.mmio_high_alias);
> > +    }
> > +
> > +    /*
> > +     * PVH implementations with PCI enabled must provide
> set_pci_intx_irq()
> > +     * and optionally an implementation of set_pci_link_route().
> > +     */
> > +    assert(xpc->set_pci_intx_irq);
> > +
> > +    for (i = 0; i < GPEX_NUM_IRQS; i++) {
> > +        qemu_irq irq = qemu_allocate_irq(xpc->set_pci_intx_irq, s, i);
> > +
> > +        sysbus_connect_irq(SYS_BUS_DEVICE(dev), i, irq);
> > +        gpex_set_irq_num(GPEX_HOST(dev), i, s->cfg.pci_intx_irq_base +
> i);
> > +        if (xpc->set_pci_link_route) {
> > +            xpc->set_pci_link_route(i, s->cfg.pci_intx_irq_base + i);
> > +        }
> > +    }
> > +}
>
> Some Kconfig selector seems missing here:
>
> /usr/bin/ld: libqemu-aarch64-softmmu.a.p/hw_xen_xen-pvh-common.c.o: in
> function `xenpvh_gpex_init':
> hw/xen/xen-pvh-common.c:174: undefined reference to `gpex_set_irq_num'
> /usr/bin/ld: libqemu-aarch64-softmmu.a.p/hw_xen_xen-hvm-common.c.o: in
> function `pci_dev_bus_num':
> include/hw/pci/pci.h:337: undefined reference to `pci_bus_num'
> /usr/bin/ld: include/hw/pci/pci.h:337: undefined reference to `pci_bus_num'
> /usr/bin/ld: include/hw/pci/pci.h:337: undefined reference to `pci_bus_num'
> /usr/bin/ld: include/hw/pci/pci.h:337: undefined reference to `pci_bus_num'
> /usr/bin/ld: include/hw/pci/pci.h:337: undefined reference to `pci_bus_num'
> /usr/bin/ld: libqemu-aarch64-softmmu.a.p/hw_xen_xen-hvm-common.c.o: in
> function `cpu_ioreq_config':
> hw/xen/xen-hvm-common.c:412: undefined reference to
> `pci_host_config_read_common'
> /usr/bin/ld: hw/xen/xen-hvm-common.c:428: undefined reference to
> `pci_host_config_read_common'
> /usr/bin/ld: hw/xen/xen-hvm-common.c:438: undefined reference to
> `pci_host_config_write_common'
>
> The current 'XEN' key represents both the "accelerator" part and
> the common Xen HW, which isn't helping to follow. Anyhow, this
> snippet fixes the build issue:
>
> -- >8 --
> diff --git a/accel/Kconfig b/accel/Kconfig
> index 794e0d18d2..4263cab722 100644
> --- a/accel/Kconfig
> +++ b/accel/Kconfig
> @@ -16,4 +16,5 @@ config KVM
>   config XEN
>       bool
>       select FSDEV_9P if VIRTFS
> +    select PCI_EXPRESS_GENERIC_BRIDGE
>       select XEN_BUS
> ---
>
> I'll post a patch later.
>


Sounds good, thanks Phil!

Cheers,
Edgar


>
> Regards,
>
> Phil.
>