From: Mirsad Ostrakovic <mirsad_ostrakovic@epam.com>

The patch adds preliminary support for multiple instances of
SMMU on sysbus.
Introduce generic-bus-iommu-id for each IOMMU and check if it used
on init.

For now, each smmu can be created via static configuration in arm/virt.c.

Signed-off-by: Mirsad Ostrakovic <mirsad_ostrakovic@epam.com>
Signed-off-by: Ruslan Ruslichenko <Ruslan_Ruslichenko@epam.com>
---
 hw/arm/smmu-common.c         |  7 ++++---
 hw/arm/virt.c                |  3 ++-
 hw/core/bus.c                | 19 ++++++++++++++++---
 hw/virtio/virtio-mmio.c      | 10 +++++++---
 include/hw/arm/smmu-common.h |  3 ++-
 include/hw/qdev-core.h       | 23 ++++++++++++++++++-----
 include/qemu/typedefs.h      |  1 +
 7 files changed, 50 insertions(+), 16 deletions(-)

diff --git a/hw/arm/smmu-common.c b/hw/arm/smmu-common.c
index 52ef49d7f6..10a551b5bc 100644
--- a/hw/arm/smmu-common.c
+++ b/hw/arm/smmu-common.c
@@ -981,8 +981,8 @@ static void smmu_base_realize(DeviceState *dev, Error **errp)
                                      g_free, g_free);
     s->smmu_pcibus_by_busptr = g_hash_table_new(NULL, NULL);
 
-    if (s->generic_primary_bus ) {
-        bus_setup_iommu(s->generic_primary_bus, &bus_smmu_ops, s);
+    if (s->generic_bus) {
+        bus_setup_iommu(s->generic_bus, s->generic_bus_iommu_id, &bus_smmu_ops, s);
         return;
     }
 
@@ -1041,8 +1041,9 @@ static const Property smmu_dev_properties[] = {
     DEFINE_PROP_BOOL("pci_smmu_per_bus", SMMUState, pci_smmu_per_bus, false),
     DEFINE_PROP_LINK("pci-primary-bus", SMMUState, pci_primary_bus,
                      TYPE_PCI_BUS, PCIBus *),
-    DEFINE_PROP_LINK("generic-primary-bus", SMMUState, generic_primary_bus,
+    DEFINE_PROP_LINK("generic-bus", SMMUState, generic_bus,
                      TYPE_BUS, BusState *),
+    DEFINE_PROP_UINT8("generic-bus-iommu-id", SMMUState, generic_bus_iommu_id, 255u),
 };
 
 static void smmu_base_class_init(ObjectClass *klass, const void *data)
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index ff3eb95036..5ee21234aa 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -1575,8 +1575,9 @@ static void create_smmu_sysbus(VirtMachineState *vms)
     if (!vmc->no_nested_smmu) {
         object_property_set_str(OBJECT(dev), "stage", "nested", &error_fatal);
     }
-    object_property_set_link(OBJECT(dev), "generic-primary-bus", 
+    object_property_set_link(OBJECT(dev), "generic-bus", 
                              OBJECT(sysbus_get_default()), &error_abort);
+    object_property_set_int(OBJECT(dev), "generic-bus-iommu-id", 0u, &error_abort);
     sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal);
     sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
     for (i = 0; i < NUM_SMMU_IRQS; i++) {
diff --git a/hw/core/bus.c b/hw/core/bus.c
index 6d1483fdbd..c26a300546 100644
--- a/hw/core/bus.c
+++ b/hw/core/bus.c
@@ -80,7 +80,7 @@ bool bus_is_in_reset(BusState *bus)
     return resettable_is_in_reset(OBJECT(bus));
 }
 
-void bus_setup_iommu(BusState *bus, const BusIOMMUOps *ops, void *opaque)
+void bus_setup_iommu(BusState *bus, uint8_t iommu_id, const BusIOMMUOps *ops, void *opaque)
 {
     /*
      * If called, bus_setup_iommu() should provide a minimum set of
@@ -89,8 +89,17 @@ void bus_setup_iommu(BusState *bus, const BusIOMMUOps *ops, void *opaque)
     assert(ops);
     assert(ops->get_address_space);
 
-    bus->iommu_ops = ops;
-    bus->iommu_opaque = opaque;
+    /*
+     * Provided IOMMU index shall be in range of valid values.
+     */
+    assert(iommu_id < (sizeof(bus->iommu) / sizeof(bus->iommu[0])));
+    /*
+     * Allocated entry cannot be used!
+     */
+    assert(!bus->iommu[iommu_id].used);
+
+    bus->iommu[iommu_id].iommu_ops = ops;
+    bus->iommu[iommu_id].iommu_opaque = opaque;
 }
 
 static ResettableState *bus_get_reset_state(Object *obj)
@@ -230,6 +239,10 @@ static void qbus_initfn(Object *obj)
 {
     BusState *bus = BUS(obj);
 
+    for (int i = 0u; i < (sizeof(bus->iommu) / sizeof(bus->iommu[0])); i++) {
+        bus->iommu[i].used = false;
+    }
+
     QTAILQ_INIT(&bus->children);
     object_property_add_link(obj, QDEV_HOTPLUG_HANDLER_PROPERTY,
                              TYPE_HOTPLUG_HANDLER,
diff --git a/hw/virtio/virtio-mmio.c b/hw/virtio/virtio-mmio.c
index 8f19492e3f..214c090ec8 100644
--- a/hw/virtio/virtio-mmio.c
+++ b/hw/virtio/virtio-mmio.c
@@ -1,3 +1,4 @@
+
 /*
  * Virtio MMIO bindings
  *
@@ -37,6 +38,8 @@
 #include "trace.h"
 #include "qapi/error.h"
 
+#define VIRTIO_SYSBUS_IOMMU_ID (0u)
+
 static bool virtio_mmio_ioeventfd_enabled(DeviceState *d)
 {
     VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d);
@@ -874,13 +877,14 @@ static void virtio_mmio_vmstate_change(DeviceState *d, bool running)
 static AddressSpace *virtio_mmio_get_dma_as(DeviceState *parent)
 {
     // BusState *iommu_bus = qdev_get_parent_bus(parent);
+    const uint32_t iommu_id = VIRTIO_SYSBUS_IOMMU_ID;
     BusState *iommu_bus = sysbus_get_default();
     VirtIOMMIOProxy *proxy = VIRTIO_MMIO(parent);
     AddressSpace *as = NULL;
 
-    if (iommu_bus && iommu_bus->iommu_ops) {
-        as = iommu_bus->iommu_ops->get_address_space(
-            iommu_bus, iommu_bus->iommu_opaque, proxy->stream_id);
+    if (iommu_bus && iommu_bus->iommu[iommu_id].iommu_ops) {
+        as = iommu_bus->iommu[iommu_id].iommu_ops->get_address_space(
+                iommu_bus, iommu_bus->iommu[iommu_id].iommu_opaque, proxy->stream_id);
     }
 
     return as ? as : &address_space_memory;
diff --git a/include/hw/arm/smmu-common.h b/include/hw/arm/smmu-common.h
index 670ae46930..a44c73cf0c 100644
--- a/include/hw/arm/smmu-common.h
+++ b/include/hw/arm/smmu-common.h
@@ -171,7 +171,8 @@ struct SMMUState {
     uint8_t bus_num;
     PCIBus *pci_primary_bus;
     bool pci_smmu_per_bus; /* SMMU is specific to the pci_primary_bus */
-    BusState *generic_primary_bus;
+    BusState *generic_bus;
+    uint8_t generic_bus_iommu_id;
 };
 
 struct SMMUBaseClass {
diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h
index 2092450b90..f766213705 100644
--- a/include/hw/qdev-core.h
+++ b/include/hw/qdev-core.h
@@ -386,6 +386,20 @@ typedef struct BusIOMMUOps {
     AddressSpace * (*get_address_space)(BusState *bus, void *opaque, int devid);
 } BusIOMMUOps;
 
+/**
+ * struct BusIOMMU:
+ * @iommu_ops: TODO
+ * @iommu_opaque: TODO
+ * @used: TODO
+ */
+struct BusIOMMU {
+    const BusIOMMUOps *iommu_ops;
+    void *iommu_opaque;
+    bool used;
+};
+
+#define BUS_IOMMU_MAX 10
+
 /**
  * struct BusState:
  * @obj: parent object
@@ -396,8 +410,7 @@ typedef struct BusIOMMUOps {
  * @realized: is the bus itself realized?
  * @full: is the bus full?
  * @num_children: current number of child buses
- * @iommu_ops: TODO
- * @iommu_opaque: TODO
+ * @iommu: TODO
  */
 struct BusState {
     /* private: */
@@ -410,8 +423,7 @@ struct BusState {
     bool realized;
     bool full;
     int num_children;
-    const BusIOMMUOps *iommu_ops;
-    void *iommu_opaque;
+    BusIOMMU iommu[BUS_IOMMU_MAX];
 
     /**
      * @children: an RCU protected QTAILQ, thus readers must use RCU
@@ -958,13 +970,14 @@ bool bus_is_in_reset(BusState *bus);
  * bus_setup_iommu() - Set up IOMMU operations for a bus
  * @bus: the bus to configure
  * @ops: IOMMU operations structure containing callback functions
+ * @iommu_id: TODO
  * @opaque: opaque data passed to IOMMU operation callbacks
  *
  * Configure IOMMU operations for the specified bus. The ops structure
  * must contain at least the get_address_space callback. The opaque
  * parameter is passed through to the operation callbacks.
  */
-void bus_setup_iommu(BusState *bus, const BusIOMMUOps *ops, void *opaque);
+void bus_setup_iommu(BusState *bus, uint8_t iommu_id, const BusIOMMUOps *ops, void *opaque);
 
 /* This should go away once we get rid of the NULL bus hack */
 BusState *sysbus_get_default(void);
diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h
index 4a94af9665..b5d5f534f3 100644
--- a/include/qemu/typedefs.h
+++ b/include/qemu/typedefs.h
@@ -34,6 +34,7 @@ typedef struct BlockBackend BlockBackend;
 typedef struct BlockBackendRootState BlockBackendRootState;
 typedef struct BlockDriverState BlockDriverState;
 typedef struct BusClass BusClass;
+typedef struct BusIOMMU BusIOMMU;
 typedef struct BusState BusState;
 typedef struct Chardev Chardev;
 typedef struct Clock Clock;
-- 
2.43.0