With the soon to be introduced user-creatable SMMUv3 devices for
virt, it is possible to have multiple SMMUv3 devices associated
with different PCIe root complexes.
Update IORT nodes accordingly.
An example IORT Id mappings for a Qemu virt machine with two
PCIe Root Complexes each assocaited with a SMMUv3 will
be something like below,
-device arm-smmuv3,primary-bus=pcie.0,id=smmuv3.0
-device arm-smmuv3,primary-bus=pcie.1,id=smmuv3.1
...
+--------------------+ +--------------------+
| Root Complex 0 | | Root Complex 1 |
| | | |
| Requestor IDs | | Requestor IDs |
| 0x0000 - 0x00FF | | 0x0100 - 0x01FF |
+---------+----------+ +---------+----------+
| |
| |
| Stream ID Mapping |
v v
+--------------------+ +--------------------+
| SMMUv3 Node 0 | | SMMUv3 Node 1 |
| | | |
| Stream IDs 0x0000- | | Stream IDs 0x0100- |
| 0x00FF mapped from | | 0x01FF mapped from |
| RC0 Requestor IDs | | RC1 Requestor IDs |
+--------------------+ +--------------------+
| |
| |
+----------------+---------------+
|
|Device ID Mapping
v
+----------------------------+
| ITS Node 0 |
| |
| Device IDs: |
| 0x0000 - 0x00FF (from RC0) |
| 0x0100 - 0x01FF (from RC1) |
| 0x0200 - 0xFFFF (No SMMU) |
+----------------------------+
Tested-by: Nathan Chen <nathanc@nvidia.com>
Signed-off-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
---
hw/arm/virt-acpi-build.c | 55 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 55 insertions(+)
diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c
index d39506179a..72b79100ce 100644
--- a/hw/arm/virt-acpi-build.c
+++ b/hw/arm/virt-acpi-build.c
@@ -43,6 +43,7 @@
#include "hw/acpi/generic_event_device.h"
#include "hw/acpi/tpm.h"
#include "hw/acpi/hmat.h"
+#include "hw/arm/smmuv3.h"
#include "hw/pci/pcie_host.h"
#include "hw/pci/pci.h"
#include "hw/pci/pci_bus.h"
@@ -296,6 +297,58 @@ populate_smmuv3_legacy_dev(GArray *sdev_blob)
g_array_append_val(sdev_blob, sdev);
}
+static int smmuv3_dev_idmap_compare(gconstpointer a, gconstpointer b)
+{
+ AcpiIortSMMUv3Dev *sdev_a = (AcpiIortSMMUv3Dev *)a;
+ AcpiIortSMMUv3Dev *sdev_b = (AcpiIortSMMUv3Dev *)b;
+ AcpiIortIdMapping *map_a = &g_array_index(sdev_a->idmaps,
+ AcpiIortIdMapping, 0);
+ AcpiIortIdMapping *map_b = &g_array_index(sdev_b->idmaps,
+ AcpiIortIdMapping, 0);
+ return map_a->input_base - map_b->input_base;
+}
+
+static int iort_smmuv3_devices(Object *obj, void *opaque)
+{
+ VirtMachineState *vms = VIRT_MACHINE(qdev_get_machine());
+ GArray *sdev_blob = opaque;
+ AcpiIortIdMapping idmap;
+ PlatformBusDevice *pbus;
+ AcpiIortSMMUv3Dev sdev;
+ int min_bus, max_bus;
+ SysBusDevice *sbdev;
+ PCIBus *bus;
+
+ if (!object_dynamic_cast(obj, TYPE_ARM_SMMUV3)) {
+ return 0;
+ }
+
+ bus = PCI_BUS(object_property_get_link(obj, "primary-bus", &error_abort));
+ pbus = PLATFORM_BUS_DEVICE(vms->platform_bus_dev);
+ sbdev = SYS_BUS_DEVICE(obj);
+ sdev.base = platform_bus_get_mmio_addr(pbus, sbdev, 0);
+ sdev.base += vms->memmap[VIRT_PLATFORM_BUS].base;
+ sdev.irq = platform_bus_get_irqn(pbus, sbdev, 0);
+ sdev.irq += vms->irqmap[VIRT_PLATFORM_BUS];
+ sdev.irq += ARM_SPI_BASE;
+
+ pci_bus_range(bus, &min_bus, &max_bus);
+ sdev.idmaps = g_array_new(false, true, sizeof(AcpiIortIdMapping));
+ idmap.input_base = min_bus << 8,
+ idmap.id_count = (max_bus - min_bus + 1) << 8,
+ g_array_append_val(sdev.idmaps, idmap);
+ g_array_append_val(sdev_blob, sdev);
+ return 0;
+}
+
+static void populate_smmuv3_dev(GArray *sdev_blob)
+{
+ object_child_foreach_recursive(object_get_root(),
+ iort_smmuv3_devices, sdev_blob);
+ /* Sort the smmuv3 devices(if any) by smmu idmap input_base */
+ g_array_sort(sdev_blob, smmuv3_dev_idmap_compare);
+}
+
/*
* Input Output Remapping Table (IORT)
* Conforms to "IO Remapping Table System Software on ARM Platforms",
@@ -320,6 +373,8 @@ build_iort(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms)
nb_nodes = 2; /* RC, ITS */
if (vms->legacy_smmuv3_present) {
populate_smmuv3_legacy_dev(smmuv3_devs);
+ } else {
+ populate_smmuv3_dev(smmuv3_devs);
}
num_smmus = smmuv3_devs->len;
--
2.47.0
Hi Shameer,
On 6/13/25 4:44 PM, Shameer Kolothum wrote:
> With the soon to be introduced user-creatable SMMUv3 devices for
> virt, it is possible to have multiple SMMUv3 devices associated
> with different PCIe root complexes.
>
> Update IORT nodes accordingly.
>
> An example IORT Id mappings for a Qemu virt machine with two
> PCIe Root Complexes each assocaited with a SMMUv3 will
> be something like below,
>
> -device arm-smmuv3,primary-bus=pcie.0,id=smmuv3.0
> -device arm-smmuv3,primary-bus=pcie.1,id=smmuv3.1
> ...
>
> +--------------------+ +--------------------+
> | Root Complex 0 | | Root Complex 1 |
> | | | |
> | Requestor IDs | | Requestor IDs |
> | 0x0000 - 0x00FF | | 0x0100 - 0x01FF |
> +---------+----------+ +---------+----------+
> | |
> | |
> | Stream ID Mapping |
> v v
> +--------------------+ +--------------------+
> | SMMUv3 Node 0 | | SMMUv3 Node 1 |
> | | | |
> | Stream IDs 0x0000- | | Stream IDs 0x0100- |
> | 0x00FF mapped from | | 0x01FF mapped from |
> | RC0 Requestor IDs | | RC1 Requestor IDs |
> +--------------------+ +--------------------+
> | |
> | |
> +----------------+---------------+
> |
> |Device ID Mapping
> v
> +----------------------------+
> | ITS Node 0 |
> | |
> | Device IDs: |
> | 0x0000 - 0x00FF (from RC0) |
> | 0x0100 - 0x01FF (from RC1) |
> | 0x0200 - 0xFFFF (No SMMU) |
> +----------------------------+
>
> Tested-by: Nathan Chen <nathanc@nvidia.com>
> Signed-off-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
> ---
> hw/arm/virt-acpi-build.c | 55 ++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 55 insertions(+)
>
> diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c
> index d39506179a..72b79100ce 100644
> --- a/hw/arm/virt-acpi-build.c
> +++ b/hw/arm/virt-acpi-build.c
> @@ -43,6 +43,7 @@
> #include "hw/acpi/generic_event_device.h"
> #include "hw/acpi/tpm.h"
> #include "hw/acpi/hmat.h"
> +#include "hw/arm/smmuv3.h"
> #include "hw/pci/pcie_host.h"
> #include "hw/pci/pci.h"
> #include "hw/pci/pci_bus.h"
> @@ -296,6 +297,58 @@ populate_smmuv3_legacy_dev(GArray *sdev_blob)
> g_array_append_val(sdev_blob, sdev);
> }
>
> +static int smmuv3_dev_idmap_compare(gconstpointer a, gconstpointer b)
> +{
> + AcpiIortSMMUv3Dev *sdev_a = (AcpiIortSMMUv3Dev *)a;
> + AcpiIortSMMUv3Dev *sdev_b = (AcpiIortSMMUv3Dev *)b;
> + AcpiIortIdMapping *map_a = &g_array_index(sdev_a->idmaps,
> + AcpiIortIdMapping, 0);
> + AcpiIortIdMapping *map_b = &g_array_index(sdev_b->idmaps,
> + AcpiIortIdMapping, 0);
> + return map_a->input_base - map_b->input_base;
> +}
> +
> +static int iort_smmuv3_devices(Object *obj, void *opaque)
> +{
> + VirtMachineState *vms = VIRT_MACHINE(qdev_get_machine());
> + GArray *sdev_blob = opaque;
> + AcpiIortIdMapping idmap;
> + PlatformBusDevice *pbus;
> + AcpiIortSMMUv3Dev sdev;
> + int min_bus, max_bus;
> + SysBusDevice *sbdev;
> + PCIBus *bus;
> +
> + if (!object_dynamic_cast(obj, TYPE_ARM_SMMUV3)) {
> + return 0;
> + }
> +
> + bus = PCI_BUS(object_property_get_link(obj, "primary-bus", &error_abort));
> + pbus = PLATFORM_BUS_DEVICE(vms->platform_bus_dev);
> + sbdev = SYS_BUS_DEVICE(obj);
> + sdev.base = platform_bus_get_mmio_addr(pbus, sbdev, 0);
> + sdev.base += vms->memmap[VIRT_PLATFORM_BUS].base;
> + sdev.irq = platform_bus_get_irqn(pbus, sbdev, 0);
> + sdev.irq += vms->irqmap[VIRT_PLATFORM_BUS];
> + sdev.irq += ARM_SPI_BASE;
> +
> + pci_bus_range(bus, &min_bus, &max_bus);
> + sdev.idmaps = g_array_new(false, true, sizeof(AcpiIortIdMapping));
> + idmap.input_base = min_bus << 8,
> + idmap.id_count = (max_bus - min_bus + 1) << 8,
> + g_array_append_val(sdev.idmaps, idmap);
> + g_array_append_val(sdev_blob, sdev);
> + return 0;
> +}
> +
> +static void populate_smmuv3_dev(GArray *sdev_blob)
> +{
> + object_child_foreach_recursive(object_get_root(),
> + iort_smmuv3_devices, sdev_blob);
> + /* Sort the smmuv3 devices(if any) by smmu idmap input_base */
> + g_array_sort(sdev_blob, smmuv3_dev_idmap_compare);
> +}
> +
> /*
> * Input Output Remapping Table (IORT)
> * Conforms to "IO Remapping Table System Software on ARM Platforms",
> @@ -320,6 +373,8 @@ build_iort(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms)
> nb_nodes = 2; /* RC, ITS */
> if (vms->legacy_smmuv3_present) {
> populate_smmuv3_legacy_dev(smmuv3_devs);
> + } else {
> + populate_smmuv3_dev(smmuv3_devs);
> }
>
> num_smmus = smmuv3_devs->len;
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Eric
On Fri, 13 Jun 2025 15:44:45 +0100
Shameer Kolothum <shameerali.kolothum.thodi@huawei.com> wrote:
> With the soon to be introduced user-creatable SMMUv3 devices for
> virt, it is possible to have multiple SMMUv3 devices associated
> with different PCIe root complexes.
>
> Update IORT nodes accordingly.
>
> An example IORT Id mappings for a Qemu virt machine with two
> PCIe Root Complexes each assocaited with a SMMUv3 will
> be something like below,
>
> -device arm-smmuv3,primary-bus=pcie.0,id=smmuv3.0
> -device arm-smmuv3,primary-bus=pcie.1,id=smmuv3.1
> ...
>
> +--------------------+ +--------------------+
> | Root Complex 0 | | Root Complex 1 |
> | | | |
> | Requestor IDs | | Requestor IDs |
> | 0x0000 - 0x00FF | | 0x0100 - 0x01FF |
> +---------+----------+ +---------+----------+
> | |
> | |
> | Stream ID Mapping |
> v v
> +--------------------+ +--------------------+
> | SMMUv3 Node 0 | | SMMUv3 Node 1 |
> | | | |
> | Stream IDs 0x0000- | | Stream IDs 0x0100- |
> | 0x00FF mapped from | | 0x01FF mapped from |
> | RC0 Requestor IDs | | RC1 Requestor IDs |
> +--------------------+ +--------------------+
> | |
> | |
> +----------------+---------------+
> |
> |Device ID Mapping
> v
> +----------------------------+
> | ITS Node 0 |
> | |
> | Device IDs: |
> | 0x0000 - 0x00FF (from RC0) |
> | 0x0100 - 0x01FF (from RC1) |
> | 0x0200 - 0xFFFF (No SMMU) |
> +----------------------------+
>
> Tested-by: Nathan Chen <nathanc@nvidia.com>
> Signed-off-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>
Seems fine to me.
Reviewed-by: Jonathan Cameron <jonathan.cameron@huawei.com>
Other than needs a bios table test :)
> ---
> hw/arm/virt-acpi-build.c | 55 ++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 55 insertions(+)
>
> diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c
> index d39506179a..72b79100ce 100644
> --- a/hw/arm/virt-acpi-build.c
> +++ b/hw/arm/virt-acpi-build.c
> @@ -43,6 +43,7 @@
> #include "hw/acpi/generic_event_device.h"
> #include "hw/acpi/tpm.h"
> #include "hw/acpi/hmat.h"
> +#include "hw/arm/smmuv3.h"
> #include "hw/pci/pcie_host.h"
> #include "hw/pci/pci.h"
> #include "hw/pci/pci_bus.h"
> @@ -296,6 +297,58 @@ populate_smmuv3_legacy_dev(GArray *sdev_blob)
> g_array_append_val(sdev_blob, sdev);
> }
>
> +static int smmuv3_dev_idmap_compare(gconstpointer a, gconstpointer b)
> +{
> + AcpiIortSMMUv3Dev *sdev_a = (AcpiIortSMMUv3Dev *)a;
> + AcpiIortSMMUv3Dev *sdev_b = (AcpiIortSMMUv3Dev *)b;
> + AcpiIortIdMapping *map_a = &g_array_index(sdev_a->idmaps,
> + AcpiIortIdMapping, 0);
> + AcpiIortIdMapping *map_b = &g_array_index(sdev_b->idmaps,
> + AcpiIortIdMapping, 0);
> + return map_a->input_base - map_b->input_base;
> +}
> +
> +static int iort_smmuv3_devices(Object *obj, void *opaque)
> +{
> + VirtMachineState *vms = VIRT_MACHINE(qdev_get_machine());
> + GArray *sdev_blob = opaque;
> + AcpiIortIdMapping idmap;
> + PlatformBusDevice *pbus;
> + AcpiIortSMMUv3Dev sdev;
> + int min_bus, max_bus;
> + SysBusDevice *sbdev;
> + PCIBus *bus;
> +
> + if (!object_dynamic_cast(obj, TYPE_ARM_SMMUV3)) {
> + return 0;
> + }
> +
> + bus = PCI_BUS(object_property_get_link(obj, "primary-bus", &error_abort));
> + pbus = PLATFORM_BUS_DEVICE(vms->platform_bus_dev);
> + sbdev = SYS_BUS_DEVICE(obj);
> + sdev.base = platform_bus_get_mmio_addr(pbus, sbdev, 0);
> + sdev.base += vms->memmap[VIRT_PLATFORM_BUS].base;
> + sdev.irq = platform_bus_get_irqn(pbus, sbdev, 0);
> + sdev.irq += vms->irqmap[VIRT_PLATFORM_BUS];
> + sdev.irq += ARM_SPI_BASE;
> +
> + pci_bus_range(bus, &min_bus, &max_bus);
> + sdev.idmaps = g_array_new(false, true, sizeof(AcpiIortIdMapping));
> + idmap.input_base = min_bus << 8,
> + idmap.id_count = (max_bus - min_bus + 1) << 8,
> + g_array_append_val(sdev.idmaps, idmap);
> + g_array_append_val(sdev_blob, sdev);
> + return 0;
> +}
> -----Original Message----- > From: Jonathan Cameron <jonathan.cameron@huawei.com> > Sent: Monday, June 16, 2025 11:55 AM > To: Shameerali Kolothum Thodi > <shameerali.kolothum.thodi@huawei.com>; Linuxarm > <linuxarm@huawei.com> > Cc: qemu-arm@nongnu.org; qemu-devel@nongnu.org; > eric.auger@redhat.com; peter.maydell@linaro.org; jgg@nvidia.com; > nicolinc@nvidia.com; ddutile@redhat.com; berrange@redhat.com; > imammedo@redhat.com; nathanc@nvidia.com; mochs@nvidia.com; > smostafa@google.com; Wangzhou (B) <wangzhou1@hisilicon.com>; > jiangkunkun <jiangkunkun@huawei.com>; Jonathan Cameron > <jonathan.cameron@huawei.com>; zhangfei.gao@linaro.org > Subject: Re: [PATCH v4 3/7] hw/arm/virt-acpi-build: Update IORT for > multiple smmuv3 devices > > On Fri, 13 Jun 2025 15:44:45 +0100 > Shameer Kolothum <shameerali.kolothum.thodi@huawei.com> wrote: > > > With the soon to be introduced user-creatable SMMUv3 devices for > > virt, it is possible to have multiple SMMUv3 devices associated > > with different PCIe root complexes. > > > > Update IORT nodes accordingly. > > > > An example IORT Id mappings for a Qemu virt machine with two > > PCIe Root Complexes each assocaited with a SMMUv3 will > > be something like below, > > > > -device arm-smmuv3,primary-bus=pcie.0,id=smmuv3.0 > > -device arm-smmuv3,primary-bus=pcie.1,id=smmuv3.1 > > ... > > > > +--------------------+ +--------------------+ > > | Root Complex 0 | | Root Complex 1 | > > | | | | > > | Requestor IDs | | Requestor IDs | > > | 0x0000 - 0x00FF | | 0x0100 - 0x01FF | > > +---------+----------+ +---------+----------+ > > | | > > | | > > | Stream ID Mapping | > > v v > > +--------------------+ +--------------------+ > > | SMMUv3 Node 0 | | SMMUv3 Node 1 | > > | | | | > > | Stream IDs 0x0000- | | Stream IDs 0x0100- | > > | 0x00FF mapped from | | 0x01FF mapped from | > > | RC0 Requestor IDs | | RC1 Requestor IDs | > > +--------------------+ +--------------------+ > > | | > > | | > > +----------------+---------------+ > > | > > |Device ID Mapping > > v > > +----------------------------+ > > | ITS Node 0 | > > | | > > | Device IDs: | > > | 0x0000 - 0x00FF (from RC0) | > > | 0x0100 - 0x01FF (from RC1) | > > | 0x0200 - 0xFFFF (No SMMU) | > > +----------------------------+ > > > > Tested-by: Nathan Chen <nathanc@nvidia.com> > > Signed-off-by: Shameer Kolothum > <shameerali.kolothum.thodi@huawei.com> > Seems fine to me. > Reviewed-by: Jonathan Cameron <jonathan.cameron@huawei.com> > > Other than needs a bios table test :) Sure, Will add one. Thanks, Shameer
On Fri, Jun 13, 2025 at 03:44:45PM +0100, Shameer Kolothum wrote: > With the soon to be introduced user-creatable SMMUv3 devices for > virt, it is possible to have multiple SMMUv3 devices associated > with different PCIe root complexes. > > Update IORT nodes accordingly. > > An example IORT Id mappings for a Qemu virt machine with two > PCIe Root Complexes each assocaited with a SMMUv3 will > be something like below, > > -device arm-smmuv3,primary-bus=pcie.0,id=smmuv3.0 > -device arm-smmuv3,primary-bus=pcie.1,id=smmuv3.1 > ... > > +--------------------+ +--------------------+ > | Root Complex 0 | | Root Complex 1 | > | | | | > | Requestor IDs | | Requestor IDs | > | 0x0000 - 0x00FF | | 0x0100 - 0x01FF | > +---------+----------+ +---------+----------+ > | | > | | > | Stream ID Mapping | > v v > +--------------------+ +--------------------+ > | SMMUv3 Node 0 | | SMMUv3 Node 1 | > | | | | > | Stream IDs 0x0000- | | Stream IDs 0x0100- | > | 0x00FF mapped from | | 0x01FF mapped from | > | RC0 Requestor IDs | | RC1 Requestor IDs | > +--------------------+ +--------------------+ > | | > | | > +----------------+---------------+ > | > |Device ID Mapping > v > +----------------------------+ > | ITS Node 0 | > | | > | Device IDs: | > | 0x0000 - 0x00FF (from RC0) | > | 0x0100 - 0x01FF (from RC1) | > | 0x0200 - 0xFFFF (No SMMU) | > +----------------------------+ > > Tested-by: Nathan Chen <nathanc@nvidia.com> > Signed-off-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com> Reviewed-by: Nicolin Chen <nicolinc@nvidia.com>
© 2016 - 2025 Red Hat, Inc.