Add support for "smmuv3Dev" IOMMU model, and add
support for parsing multiple IOMMU devices from
the VM definition when "smmuv3Dev" is the IOMMU model.
Enable plugging smmuv3Dev into pcie-root and
pcie-expander-bus.
Signed-off-by: Nathan Chen <nathanc@nvidia.com>
---
docs/formatdomain.rst | 5 +-
src/conf/domain_addr.c | 12 ++-
src/conf/domain_addr.h | 4 +-
src/conf/domain_conf.c | 153 ++++++++++++++++++++++++++----
src/conf/domain_conf.h | 11 ++-
src/conf/domain_validate.c | 38 +++++---
src/conf/schemas/domaincommon.rng | 8 +-
src/libvirt_private.syms | 2 +
src/qemu/qemu_alias.c | 15 ++-
src/qemu/qemu_command.c | 135 +++++++++++++++++---------
src/qemu/qemu_domain_address.c | 33 ++++---
src/qemu/qemu_driver.c | 8 +-
src/qemu/qemu_postparse.c | 11 ++-
src/qemu/qemu_validate.c | 22 ++++-
14 files changed, 342 insertions(+), 115 deletions(-)
diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
index c7c75ae219..f9b835da7d 100644
--- a/docs/formatdomain.rst
+++ b/docs/formatdomain.rst
@@ -9043,8 +9043,9 @@ Example:
...
``model``
- Supported values are ``intel`` (for Q35 guests) ``smmuv3``
- (:since:`since 5.5.0`, for ARM virt guests), and ``virtio``
+ Supported values are ``intel`` (for Q35 guests), ``smmuv3``
+ (:since:`since 5.5.0`, for ARM virt guests), ``smmuv3Dev``
+ (for ARM virt guests), and ``virtio``
(:since:`since 8.3.0`, for Q35 and ARM virt guests).
``driver``
diff --git a/src/conf/domain_addr.c b/src/conf/domain_addr.c
index 8dfa8feca0..6a75a7aee1 100644
--- a/src/conf/domain_addr.c
+++ b/src/conf/domain_addr.c
@@ -386,6 +386,8 @@ virDomainPCIAddressFlagsCompatible(virPCIDeviceAddress *addr,
connectStr = "pcie-expander-bus";
} else if (devFlags & VIR_PCI_CONNECT_TYPE_PCI_BRIDGE) {
connectStr = "pci-bridge";
+ } else if (devFlags & VIR_PCI_CONNECT_TYPE_SMMUV3_DEV) {
+ connectStr = "arm-smmuv3-dev";
} else {
/* this should never happen. If it does, there is a
* bug in the code that sets the flag bits for devices.
@@ -517,7 +519,8 @@ virDomainPCIAddressBusSetModel(virDomainPCIAddressBus *bus,
bus->flags = (VIR_PCI_CONNECT_TYPE_PCIE_DEVICE |
VIR_PCI_CONNECT_TYPE_PCIE_ROOT_PORT |
VIR_PCI_CONNECT_TYPE_DMI_TO_PCI_BRIDGE |
- VIR_PCI_CONNECT_TYPE_PCIE_EXPANDER_BUS);
+ VIR_PCI_CONNECT_TYPE_PCIE_EXPANDER_BUS |
+ VIR_PCI_CONNECT_TYPE_SMMUV3_DEV);
bus->minSlot = 1;
bus->maxSlot = VIR_PCI_ADDRESS_SLOT_LAST;
break;
@@ -561,11 +564,12 @@ virDomainPCIAddressBusSetModel(virDomainPCIAddressBus *bus,
bus->maxSlot = VIR_PCI_ADDRESS_SLOT_LAST;
break;
case VIR_DOMAIN_CONTROLLER_MODEL_PCIE_EXPANDER_BUS:
- /* 32 slots, no hotplug, only accepts pcie-root-port or
- * dmi-to-pci-bridge
+ /* 32 slots, no hotplug, only accepts pcie-root-port,
+ * dmi-to-pci-bridge, or arm-smmuv3-dev
*/
bus->flags = (VIR_PCI_CONNECT_TYPE_PCIE_ROOT_PORT |
- VIR_PCI_CONNECT_TYPE_DMI_TO_PCI_BRIDGE);
+ VIR_PCI_CONNECT_TYPE_DMI_TO_PCI_BRIDGE |
+ VIR_PCI_CONNECT_TYPE_SMMUV3_DEV);
bus->minSlot = 0;
bus->maxSlot = VIR_PCI_ADDRESS_SLOT_LAST;
break;
diff --git a/src/conf/domain_addr.h b/src/conf/domain_addr.h
index e72fb48847..906ba9a52d 100644
--- a/src/conf/domain_addr.h
+++ b/src/conf/domain_addr.h
@@ -57,6 +57,7 @@ typedef enum {
VIR_PCI_CONNECT_TYPE_PCIE_EXPANDER_BUS = 1 << 11,
VIR_PCI_CONNECT_TYPE_PCI_BRIDGE = 1 << 12,
VIR_PCI_CONNECT_TYPE_PCIE_TO_PCI_BRIDGE = 1 << 13,
+ VIR_PCI_CONNECT_TYPE_SMMUV3_DEV = 1 << 14,
} virDomainPCIConnectFlags;
/* a combination of all bits that describe the type of connections
@@ -71,7 +72,8 @@ typedef enum {
VIR_PCI_CONNECT_TYPE_PCI_EXPANDER_BUS | \
VIR_PCI_CONNECT_TYPE_PCIE_EXPANDER_BUS | \
VIR_PCI_CONNECT_TYPE_PCI_BRIDGE | \
- VIR_PCI_CONNECT_TYPE_PCIE_TO_PCI_BRIDGE)
+ VIR_PCI_CONNECT_TYPE_PCIE_TO_PCI_BRIDGE | \
+ VIR_PCI_CONNECT_TYPE_SMMUV3_DEV)
/* combination of all bits that could be used to connect a normal
* endpoint device (i.e. excluding the connection possible between an
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index b3b0bd7329..c8d481eb5c 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -1350,6 +1350,7 @@ VIR_ENUM_IMPL(virDomainIOMMUModel,
"intel",
"smmuv3",
"virtio",
+ "smmuv3Dev",
);
VIR_ENUM_IMPL(virDomainVsockModel,
@@ -4113,7 +4114,8 @@ void virDomainDefFree(virDomainDef *def)
virDomainCryptoDefFree(def->cryptos[i]);
g_free(def->cryptos);
- virDomainIOMMUDefFree(def->iommu);
+ for (i = 0; i < def->niommus; i++)
+ virDomainIOMMUDefFree(def->iommu[i]);
virDomainPstoreDefFree(def->pstore);
@@ -4985,9 +4987,9 @@ virDomainDeviceInfoIterateFlags(virDomainDef *def,
}
device.type = VIR_DOMAIN_DEVICE_IOMMU;
- if (def->iommu) {
- device.data.iommu = def->iommu;
- if ((rc = cb(def, &device, &def->iommu->info, opaque)) != 0)
+ for (i = 0; i < def->niommus; i++) {
+ device.data.iommu = def->iommu[i];
+ if ((rc = cb(def, &device, &def->iommu[i]->info, opaque)) != 0)
return rc;
}
@@ -16369,6 +16371,110 @@ virDomainInputDefFind(const virDomainDef *def,
}
+bool
+virDomainIOMMUDefEquals(const virDomainIOMMUDef *a,
+ const virDomainIOMMUDef *b)
+{
+ if (a->model != b->model ||
+ a->intremap != b->intremap ||
+ a->caching_mode != b->caching_mode ||
+ a->eim != b->eim ||
+ a->iotlb != b->iotlb ||
+ a->aw_bits != b->aw_bits ||
+ a->dma_translation != b->dma_translation)
+ return false;
+
+ switch (a->info.type) {
+ case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI:
+ if (a->info.addr.pci.domain != b->info.addr.pci.domain ||
+ a->info.addr.pci.bus != b->info.addr.pci.bus ||
+ a->info.addr.pci.slot != b->info.addr.pci.slot ||
+ a->info.addr.pci.function != b->info.addr.pci.function) {
+ return false;
+ }
+ break;
+
+ case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_DRIVE:
+ if (a->info.addr.drive.controller != b->info.addr.drive.controller ||
+ a->info.addr.drive.bus != b->info.addr.drive.bus ||
+ a->info.addr.drive.unit != b->info.addr.drive.unit) {
+ return false;
+ }
+ break;
+
+ case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_SERIAL:
+ if (a->info.addr.vioserial.controller != b->info.addr.vioserial.controller ||
+ a->info.addr.vioserial.bus != b->info.addr.vioserial.bus ||
+ a->info.addr.vioserial.port != b->info.addr.vioserial.port) {
+ return false;
+ }
+ break;
+
+ case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_CCID:
+ if (a->info.addr.ccid.controller != b->info.addr.ccid.controller ||
+ a->info.addr.ccid.slot != b->info.addr.ccid.slot) {
+ return false;
+ }
+ break;
+
+ case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_ISA:
+ if (a->info.addr.isa.iobase != b->info.addr.isa.iobase ||
+ a->info.addr.isa.irq != b->info.addr.isa.irq) {
+ return false;
+ }
+ break;
+
+ case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_DIMM:
+ if (a->info.addr.dimm.slot != b->info.addr.dimm.slot) {
+ return false;
+ }
+
+ if (a->info.addr.dimm.base != b->info.addr.dimm.base) {
+ return false;
+ }
+ break;
+
+ case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_CCW:
+ if (a->info.addr.ccw.cssid != b->info.addr.ccw.cssid ||
+ a->info.addr.ccw.ssid != b->info.addr.ccw.ssid ||
+ a->info.addr.ccw.devno != b->info.addr.ccw.devno) {
+ return false;
+ }
+ break;
+
+ case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_USB:
+ case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_SPAPRVIO:
+ case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_S390:
+ case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_MMIO:
+ case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE:
+ case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_UNASSIGNED:
+ case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_LAST:
+ break;
+ }
+
+ if (a->info.acpiIndex != b->info.acpiIndex) {
+ return false;
+ }
+
+ return true;
+}
+
+
+ssize_t
+virDomainIOMMUDefFind(const virDomainDef *def,
+ const virDomainIOMMUDef *iommu)
+{
+ size_t i;
+
+ for (i = 0; i < def->niommus; i++) {
+ if (virDomainIOMMUDefEquals(iommu, def->iommu[i]))
+ return i;
+ }
+
+ return -1;
+}
+
+
bool
virDomainVsockDefEquals(const virDomainVsockDef *a,
const virDomainVsockDef *b)
@@ -19375,6 +19481,7 @@ virDomainDefControllersParse(virDomainDef *def,
return 0;
}
+
static virDomainDef *
virDomainDefParseXML(xmlXPathContextPtr ctxt,
virDomainXMLOption *xmlopt,
@@ -20021,19 +20128,28 @@ virDomainDefParseXML(xmlXPathContextPtr ctxt,
}
VIR_FREE(nodes);
+ /* analysis of iommu devices */
if ((n = virXPathNodeSet("./devices/iommu", ctxt, &nodes)) < 0)
return NULL;
- if (n > 1) {
+ if (n > 1 && !virXPathBoolean("./devices/iommu/@model = 'smmuv3Dev'", ctxt)) {
virReportError(VIR_ERR_XML_ERROR, "%s",
- _("only a single IOMMU device is supported"));
+ _("multiple IOMMU devices are only supported with model smmuv3Dev"));
return NULL;
}
- if (n > 0) {
- if (!(def->iommu = virDomainIOMMUDefParseXML(xmlopt, nodes[0],
- ctxt, flags)))
+ if (n > 0)
+ def->iommu = g_new0(virDomainIOMMUDef *, n);
+
+ for (i = 0; i < n; i++) {
+ virDomainIOMMUDef *iommu;
+
+ iommu = virDomainIOMMUDefParseXML(xmlopt, nodes[i], ctxt, flags);
+
+ if (!iommu)
return NULL;
+
+ def->iommu[def->niommus++] = iommu;
}
VIR_FREE(nodes);
@@ -22455,15 +22571,17 @@ virDomainDefCheckABIStabilityFlags(virDomainDef *src,
goto error;
}
- if (!!src->iommu != !!dst->iommu) {
- virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
- _("Target domain IOMMU device count does not match source"));
+ if (src->niommus != dst->niommus) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Target domain IOMMU device count %1$zu does not match source %2$zu"),
+ dst->niommus, src->niommus);
goto error;
}
- if (src->iommu &&
- !virDomainIOMMUDefCheckABIStability(src->iommu, dst->iommu))
- goto error;
+ for (i = 0; i < src->niommus; i++) {
+ if (!virDomainIOMMUDefCheckABIStability(src->iommu[i], dst->iommu[i]))
+ goto error;
+ }
if (!!src->vsock != !!dst->vsock) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
@@ -29222,8 +29340,9 @@ virDomainDefFormatInternalSetRootName(virDomainDef *def,
for (n = 0; n < def->ncryptos; n++) {
virDomainCryptoDefFormat(buf, def->cryptos[n], flags);
}
- if (def->iommu)
- virDomainIOMMUDefFormat(buf, def->iommu);
+
+ for (n = 0; n < def->niommus; n++)
+ virDomainIOMMUDefFormat(buf, def->iommu[n]);
if (def->vsock)
virDomainVsockDefFormat(buf, def->vsock);
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index 58b97a2b54..75568ff50a 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -3005,6 +3005,7 @@ typedef enum {
VIR_DOMAIN_IOMMU_MODEL_INTEL,
VIR_DOMAIN_IOMMU_MODEL_SMMUV3,
VIR_DOMAIN_IOMMU_MODEL_VIRTIO,
+ VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV,
VIR_DOMAIN_IOMMU_MODEL_LAST
} virDomainIOMMUModel;
@@ -3258,6 +3259,9 @@ struct _virDomainDef {
size_t nwatchdogs;
virDomainWatchdogDef **watchdogs;
+ size_t niommus;
+ virDomainIOMMUDef **iommu;
+
/* At maximum 2 TPMs on the domain if a TPM Proxy is present. */
size_t ntpms;
virDomainTPMDef **tpms;
@@ -3267,7 +3271,6 @@ struct _virDomainDef {
virDomainNVRAMDef *nvram;
virCPUDef *cpu;
virDomainRedirFilterDef *redirfilter;
- virDomainIOMMUDef *iommu;
virDomainVsockDef *vsock;
virDomainPstoreDef *pstore;
@@ -4269,6 +4272,12 @@ virDomainShmemDef *virDomainShmemDefRemove(virDomainDef *def, size_t idx)
ssize_t virDomainInputDefFind(const virDomainDef *def,
const virDomainInputDef *input)
ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) G_GNUC_WARN_UNUSED_RESULT;
+
+bool virDomainIOMMUDefEquals(const virDomainIOMMUDef *a,
+ const virDomainIOMMUDef *b);
+ssize_t virDomainIOMMUDefFind(const virDomainDef *def,
+ const virDomainIOMMUDef *iommu)
+ ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) G_GNUC_WARN_UNUSED_RESULT;
bool virDomainVsockDefEquals(const virDomainVsockDef *a,
const virDomainVsockDef *b)
ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) G_GNUC_WARN_UNUSED_RESULT;
diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
index d0d4bc0bf4..4861b1c002 100644
--- a/src/conf/domain_validate.c
+++ b/src/conf/domain_validate.c
@@ -1838,21 +1838,32 @@ virDomainDefCputuneValidate(const virDomainDef *def)
static int
virDomainDefIOMMUValidate(const virDomainDef *def)
{
+ size_t i;
+
if (!def->iommu)
- return 0;
+ return 0;
- if (def->iommu->intremap == VIR_TRISTATE_SWITCH_ON &&
- def->features[VIR_DOMAIN_FEATURE_IOAPIC] != VIR_DOMAIN_IOAPIC_QEMU) {
- virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
- _("IOMMU interrupt remapping requires split I/O APIC (ioapic driver='qemu')"));
- return -1;
- }
+ for (i = 0; i < def->niommus; i++) {
+ virDomainIOMMUDef *iommu = def->iommu[i];
- if (def->iommu->eim == VIR_TRISTATE_SWITCH_ON &&
- def->iommu->intremap != VIR_TRISTATE_SWITCH_ON) {
- virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
- _("IOMMU eim requires interrupt remapping to be enabled"));
- return -1;
+ if (def->niommus > 1 && iommu->model != VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV) {
+ virReportError(VIR_ERR_XML_ERROR, "%s",
+ _("IOMMU model smmuv3Dev must be specified for multiple IOMMU definitions"));
+ }
+
+ if (iommu->intremap == VIR_TRISTATE_SWITCH_ON &&
+ def->features[VIR_DOMAIN_FEATURE_IOAPIC] != VIR_DOMAIN_IOAPIC_QEMU) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("IOMMU interrupt remapping requires split I/O APIC (ioapic driver='qemu')"));
+ return -1;
+ }
+
+ if (iommu->eim == VIR_TRISTATE_SWITCH_ON &&
+ iommu->intremap != VIR_TRISTATE_SWITCH_ON) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("IOMMU eim requires interrupt remapping to be enabled"));
+ return -1;
+ }
}
return 0;
@@ -3066,6 +3077,7 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu)
{
switch (iommu->model) {
case VIR_DOMAIN_IOMMU_MODEL_SMMUV3:
+ case VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV:
case VIR_DOMAIN_IOMMU_MODEL_VIRTIO:
if (iommu->intremap != VIR_TRISTATE_SWITCH_ABSENT ||
iommu->caching_mode != VIR_TRISTATE_SWITCH_ABSENT ||
@@ -3095,6 +3107,8 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu)
return -1;
}
break;
+ case VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV:
+ break;
case VIR_DOMAIN_IOMMU_MODEL_VIRTIO:
case VIR_DOMAIN_IOMMU_MODEL_LAST:
diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
index 5597d5a66b..8d1768b24f 100644
--- a/src/conf/schemas/domaincommon.rng
+++ b/src/conf/schemas/domaincommon.rng
@@ -6190,6 +6190,7 @@
<value>intel</value>
<value>smmuv3</value>
<value>virtio</value>
+ <value>smmuv3Dev</value>
</choice>
</attribute>
<interleave>
@@ -6233,9 +6234,6 @@
<optional>
<ref name="address"/>
</optional>
- <optional>
- <ref name="alias"/>
- </optional>
</interleave>
</element>
</define>
@@ -6867,9 +6865,9 @@
<zeroOrMore>
<ref name="panic"/>
</zeroOrMore>
- <optional>
+ <zeroOrMore>
<ref name="iommu"/>
- </optional>
+ </zeroOrMore>
<optional>
<ref name="vsock"/>
</optional>
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index a8ebf9efd8..143d3af8f3 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -491,6 +491,8 @@ virDomainInputSourceGrabToggleTypeToString;
virDomainInputSourceGrabTypeFromString;
virDomainInputSourceGrabTypeToString;
virDomainInputTypeToString;
+virDomainIOMMUDefEquals;
+virDomainIOMMUDefFind;
virDomainIOMMUDefFree;
virDomainIOMMUDefNew;
virDomainIOMMUModelTypeFromString;
diff --git a/src/qemu/qemu_alias.c b/src/qemu/qemu_alias.c
index 3e6bced4a8..d635eb0178 100644
--- a/src/qemu/qemu_alias.c
+++ b/src/qemu/qemu_alias.c
@@ -631,10 +631,14 @@ qemuAssignDeviceVsockAlias(virDomainVsockDef *vsock)
static void
-qemuAssignDeviceIOMMUAlias(virDomainIOMMUDef *iommu)
+qemuAssignDeviceIOMMUAlias(virDomainDef *def,
+ virDomainIOMMUDef **iommu)
{
- if (!iommu->info.alias)
- iommu->info.alias = g_strdup("iommu0");
+ size_t i;
+ for (i = 0; i < def->niommus; i++) {
+ if (!iommu[i]->info.alias)
+ iommu[i]->info.alias = g_strdup_printf("iommu%zu", i);
+ }
}
@@ -750,8 +754,9 @@ qemuAssignDeviceAliases(virDomainDef *def)
if (def->vsock) {
qemuAssignDeviceVsockAlias(def->vsock);
}
- if (def->iommu)
- qemuAssignDeviceIOMMUAlias(def->iommu);
+ if (def->iommu && def->niommus > 0) {
+ qemuAssignDeviceIOMMUAlias(def, def->iommu);
+ }
for (i = 0; i < def->ncryptos; i++) {
qemuAssignDeviceCryptoAlias(def, def->cryptos[i]);
}
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index bf03561fde..9deafefaad 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -6128,60 +6128,106 @@ qemuBuildBootCommandLine(virCommand *cmd,
}
+static virJSONValue *
+qemuBuildPCISmmuv3DevDevProps(const virDomainDef *def,
+ virDomainIOMMUDef *iommu)
+{
+ g_autoptr(virJSONValue) props = NULL;
+ g_autofree char *pciaddr = NULL;
+ g_autofree char *bus = qemuBuildDeviceAddressPCIGetBus(def, &iommu->info);
+
+ if (!bus)
+ return NULL;
+
+ if (iommu->info.addr.pci.function != 0) {
+ pciaddr = g_strdup_printf("0x%x.0x%x", iommu->info.addr.pci.slot,
+ iommu->info.addr.pci.function);
+ } else {
+ pciaddr = g_strdup_printf("0x%x", iommu->info.addr.pci.slot);
+ }
+ if (virJSONValueObjectAdd(&props,
+ "s:driver", "arm-smmuv3-accel",
+ "s:bus", bus,
+ NULL) < 0)
+ return NULL;
+
+ return g_steal_pointer(&props);
+}
+
+
static int
qemuBuildIOMMUCommandLine(virCommand *cmd,
const virDomainDef *def,
virQEMUCaps *qemuCaps)
{
+ size_t i;
g_autoptr(virJSONValue) props = NULL;
- const virDomainIOMMUDef *iommu = def->iommu;
- if (!iommu)
+ if (!def->iommu || def->niommus <= 0)
return 0;
- switch (iommu->model) {
- case VIR_DOMAIN_IOMMU_MODEL_INTEL:
- if (virJSONValueObjectAdd(&props,
- "s:driver", "intel-iommu",
- "s:id", iommu->info.alias,
- "S:intremap", qemuOnOffAuto(iommu->intremap),
- "T:caching-mode", iommu->caching_mode,
- "S:eim", qemuOnOffAuto(iommu->eim),
- "T:device-iotlb", iommu->iotlb,
- "z:aw-bits", iommu->aw_bits,
- "T:dma-translation", iommu->dma_translation,
- NULL) < 0)
- return -1;
+ for (i = 0; i < def->niommus; i++) {
+ virDomainIOMMUDef *iommu = def->iommu[i];
+ switch (iommu->model) {
+ case VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV:
+ /* Ignore unassigned devices */
+ if (iommu->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_UNASSIGNED)
+ continue;
- if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, qemuCaps) < 0)
- return -1;
+ if (qemuCommandAddExtDevice(cmd, &iommu->info, def, qemuCaps) < 0)
+ return -1;
- return 0;
+ if (!(props = qemuBuildPCISmmuv3DevDevProps(def, iommu)))
+ return -1;
- case VIR_DOMAIN_IOMMU_MODEL_VIRTIO:
- if (virJSONValueObjectAdd(&props,
- "s:driver", "virtio-iommu",
- "s:id", iommu->info.alias,
- NULL) < 0) {
- return -1;
- }
+ if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, qemuCaps) < 0)
+ return -1;
- if (qemuBuildDeviceAddressProps(props, def, &iommu->info) < 0)
- return -1;
+ break;
- if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, qemuCaps) < 0)
- return -1;
+ case VIR_DOMAIN_IOMMU_MODEL_INTEL:
+ if (virJSONValueObjectAdd(&props,
+ "s:driver", "intel-iommu",
+ "s:id", iommu->info.alias,
+ "S:intremap", qemuOnOffAuto(iommu->intremap),
+ "T:caching-mode", iommu->caching_mode,
+ "S:eim", qemuOnOffAuto(iommu->eim),
+ "T:device-iotlb", iommu->iotlb,
+ "z:aw-bits", iommu->aw_bits,
+ "T:dma-translation", iommu->dma_translation,
+ NULL) < 0)
+ return -1;
- return 0;
+ if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, qemuCaps) < 0)
+ return -1;
- case VIR_DOMAIN_IOMMU_MODEL_SMMUV3:
- /* There is no -device for SMMUv3, so nothing to be done here */
- return 0;
+ return 0;
- case VIR_DOMAIN_IOMMU_MODEL_LAST:
- default:
- virReportEnumRangeError(virDomainIOMMUModel, iommu->model);
- return -1;
+ case VIR_DOMAIN_IOMMU_MODEL_VIRTIO:
+ if (virJSONValueObjectAdd(&props,
+ "s:driver", "virtio-iommu",
+ "s:id", iommu->info.alias,
+ NULL) < 0) {
+ return -1;
+ }
+
+ if (qemuBuildDeviceAddressProps(props, def, &iommu->info) < 0)
+ return -1;
+
+ if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, qemuCaps) < 0)
+ return -1;
+
+ return 0;
+
+ case VIR_DOMAIN_IOMMU_MODEL_SMMUV3:
+ /* There is no -device for SMMUv3, so nothing to be done here */
+ return 0;
+
+ case VIR_DOMAIN_IOMMU_MODEL_LAST:
+ default:
+ virReportEnumRangeError(virDomainIOMMUModel, iommu->model);
+ return -1;
+ }
}
return 0;
@@ -7001,8 +7047,8 @@ qemuBuildMachineCommandLine(virCommand *cmd,
if (qemuAppendDomainFeaturesMachineParam(&buf, def, qemuCaps) < 0)
return -1;
- if (def->iommu) {
- switch (def->iommu->model) {
+ if (def->iommu && def->niommus == 1) {
+ switch (def->iommu[0]->model) {
case VIR_DOMAIN_IOMMU_MODEL_SMMUV3:
virBufferAddLit(&buf, ",iommu=smmuv3");
break;
@@ -7011,10 +7057,11 @@ qemuBuildMachineCommandLine(virCommand *cmd,
case VIR_DOMAIN_IOMMU_MODEL_VIRTIO:
/* These IOMMUs are formatted in qemuBuildIOMMUCommandLine */
break;
-
+ case VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV:
+ break;
case VIR_DOMAIN_IOMMU_MODEL_LAST:
default:
- virReportEnumRangeError(virDomainIOMMUModel, def->iommu->model);
+ virReportEnumRangeError(virDomainIOMMUModel, def->iommu[0]->model);
return -1;
}
}
@@ -10598,15 +10645,15 @@ qemuBuildCommandLine(virDomainObj *vm,
if (qemuBuildBootCommandLine(cmd, def) < 0)
return NULL;
- if (qemuBuildIOMMUCommandLine(cmd, def, qemuCaps) < 0)
- return NULL;
-
if (qemuBuildGlobalControllerCommandLine(cmd, def) < 0)
return NULL;
if (qemuBuildControllersCommandLine(cmd, def, qemuCaps) < 0)
return NULL;
+ if (qemuBuildIOMMUCommandLine(cmd, def, qemuCaps) < 0)
+ return NULL;
+
if (qemuBuildMemoryDeviceCommandLine(cmd, cfg, def, priv) < 0)
return NULL;
diff --git a/src/qemu/qemu_domain_address.c b/src/qemu/qemu_domain_address.c
index e89cdee487..aeacdbc9db 100644
--- a/src/qemu/qemu_domain_address.c
+++ b/src/qemu/qemu_domain_address.c
@@ -943,6 +943,8 @@ qemuDomainDeviceCalculatePCIConnectFlags(virDomainDeviceDef *dev,
case VIR_DOMAIN_IOMMU_MODEL_INTEL:
case VIR_DOMAIN_IOMMU_MODEL_SMMUV3:
+ case VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV:
+ return VIR_PCI_CONNECT_TYPE_SMMUV3_DEV;
case VIR_DOMAIN_IOMMU_MODEL_LAST:
/* These are not PCI devices */
return 0;
@@ -2354,22 +2356,25 @@ qemuDomainAssignDevicePCISlots(virDomainDef *def,
/* Nada - none are PCI based (yet) */
}
- if (def->iommu) {
- virDomainIOMMUDef *iommu = def->iommu;
+ if (def->iommu && def->niommus > 0) {
+ for (i = 0; i < def->niommus; i++) {
+ virDomainIOMMUDef *iommu = def->iommu[i];
- switch (iommu->model) {
- case VIR_DOMAIN_IOMMU_MODEL_VIRTIO:
- if (virDeviceInfoPCIAddressIsWanted(&iommu->info) &&
- qemuDomainPCIAddressReserveNextAddr(addrs, &iommu->info) < 0) {
- return -1;
- }
- break;
+ switch (iommu->model) {
+ case VIR_DOMAIN_IOMMU_MODEL_VIRTIO:
+ if (virDeviceInfoPCIAddressIsWanted(&iommu->info) &&
+ qemuDomainPCIAddressReserveNextAddr(addrs, &iommu->info) < 0) {
+ return -1;
+ }
+ break;
- case VIR_DOMAIN_IOMMU_MODEL_INTEL:
- case VIR_DOMAIN_IOMMU_MODEL_SMMUV3:
- case VIR_DOMAIN_IOMMU_MODEL_LAST:
- /* These are not PCI devices */
- break;
+ case VIR_DOMAIN_IOMMU_MODEL_INTEL:
+ case VIR_DOMAIN_IOMMU_MODEL_SMMUV3:
+ case VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV:
+ case VIR_DOMAIN_IOMMU_MODEL_LAST:
+ /* These are not PCI devices */
+ break;
+ }
}
}
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index a34d6f1437..8599761137 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -6894,12 +6894,12 @@ qemuDomainAttachDeviceConfig(virDomainDef *vmdef,
break;
case VIR_DOMAIN_DEVICE_IOMMU:
- if (vmdef->iommu) {
+ if (vmdef->iommu && vmdef->niommus > 0) {
virReportError(VIR_ERR_OPERATION_INVALID, "%s",
_("domain already has an iommu device"));
return -1;
}
- vmdef->iommu = g_steal_pointer(&dev->data.iommu);
+ VIR_APPEND_ELEMENT(vmdef->iommu, vmdef->niommus, dev->data.iommu);
break;
case VIR_DOMAIN_DEVICE_VIDEO:
@@ -7113,12 +7113,12 @@ qemuDomainDetachDeviceConfig(virDomainDef *vmdef,
break;
case VIR_DOMAIN_DEVICE_IOMMU:
- if (!vmdef->iommu) {
+ if ((idx = virDomainIOMMUDefFind(vmdef, dev->data.iommu)) < 0) {
virReportError(VIR_ERR_OPERATION_FAILED, "%s",
_("matching iommu device not found"));
return -1;
}
- g_clear_pointer(&vmdef->iommu, virDomainIOMMUDefFree);
+ VIR_DELETE_ELEMENT(vmdef->iommu, idx, vmdef->niommus);
break;
case VIR_DOMAIN_DEVICE_VIDEO:
diff --git a/src/qemu/qemu_postparse.c b/src/qemu/qemu_postparse.c
index ed4af9ca8e..f2df3cba73 100644
--- a/src/qemu/qemu_postparse.c
+++ b/src/qemu/qemu_postparse.c
@@ -1450,7 +1450,7 @@ qemuDomainDefAddDefaultDevices(virQEMUDriver *driver,
}
}
- if (addIOMMU && !def->iommu &&
+ if (addIOMMU && !def->iommu && def->niommus == 0 &&
virQEMUCapsGet(qemuCaps, QEMU_CAPS_DEVICE_INTEL_IOMMU) &&
virQEMUCapsGet(qemuCaps, QEMU_CAPS_INTEL_IOMMU_INTREMAP) &&
virQEMUCapsGet(qemuCaps, QEMU_CAPS_INTEL_IOMMU_EIM)) {
@@ -1462,7 +1462,8 @@ qemuDomainDefAddDefaultDevices(virQEMUDriver *driver,
iommu->intremap = VIR_TRISTATE_SWITCH_ON;
iommu->eim = VIR_TRISTATE_SWITCH_ON;
- def->iommu = g_steal_pointer(&iommu);
+ def->iommu = g_new0(virDomainIOMMUDef *, 1);
+ def->iommu[def->niommus++] = g_steal_pointer(&iommu);
}
if (qemuDomainDefAddDefaultAudioBackend(driver, def) < 0)
@@ -1538,9 +1539,9 @@ qemuDomainDefEnableDefaultFeatures(virDomainDef *def,
* domain already has IOMMU without inremap. This will be fixed in
* qemuDomainIOMMUDefPostParse() but there domain definition can't be
* modified so change it now. */
- if (def->iommu &&
- (def->iommu->intremap == VIR_TRISTATE_SWITCH_ON ||
- qemuDomainNeedsIOMMUWithEIM(def)) &&
+ if (def->iommu && def->niommus == 1 &&
+ (def->iommu[0]->intremap == VIR_TRISTATE_SWITCH_ON ||
+ qemuDomainNeedsIOMMUWithEIM(def)) &&
def->features[VIR_DOMAIN_FEATURE_IOAPIC] == VIR_DOMAIN_IOAPIC_NONE) {
def->features[VIR_DOMAIN_FEATURE_IOAPIC] = VIR_DOMAIN_IOAPIC_QEMU;
}
diff --git a/src/qemu/qemu_validate.c b/src/qemu/qemu_validate.c
index b2c3c9e2f6..f4431f1b2a 100644
--- a/src/qemu/qemu_validate.c
+++ b/src/qemu/qemu_validate.c
@@ -851,7 +851,12 @@ qemuValidateDomainVCpuTopology(const virDomainDef *def, virQEMUCaps *qemuCaps)
QEMU_MAX_VCPUS_WITHOUT_EIM);
return -1;
}
- if (!def->iommu || def->iommu->eim != VIR_TRISTATE_SWITCH_ON) {
+ if (!def->iommu || def->niommus == 0) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("iommu device not found. more than %1$d vCPUs require extended interrupt mode enabled on the iommu device"),
+ QEMU_MAX_VCPUS_WITHOUT_EIM);
+ return -1;
+ } else if (def->iommu[0]->eim != VIR_TRISTATE_SWITCH_ON) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
_("more than %1$d vCPUs require extended interrupt mode enabled on the iommu device"),
QEMU_MAX_VCPUS_WITHOUT_EIM);
@@ -5225,6 +5230,21 @@ qemuValidateDomainDeviceDefIOMMU(const virDomainIOMMUDef *iommu,
}
break;
+ case VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV:
+ if (!qemuDomainIsARMVirt(def)) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("IOMMU device: '%1$s' is only supported with ARM Virt machines"),
+ virDomainIOMMUModelTypeToString(iommu->model));
+ return -1;
+ }
+ if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_MACHINE_VIRT_IOMMU)) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("IOMMU device: '%1$s' is not supported with this QEMU binary"),
+ virDomainIOMMUModelTypeToString(iommu->model));
+ return -1;
+ }
+ break;
+
case VIR_DOMAIN_IOMMU_MODEL_LAST:
default:
virReportEnumRangeError(virDomainIOMMUModel, iommu->model);
--
2.43.0On Thu, May 15, 2025 at 01:36:39PM -0700, Nathan Chen via Devel wrote:
> Add support for "smmuv3Dev" IOMMU model, and add
> support for parsing multiple IOMMU devices from
> the VM definition when "smmuv3Dev" is the IOMMU model.
> Enable plugging smmuv3Dev into pcie-root and
> pcie-expander-bus.
>
> Signed-off-by: Nathan Chen <nathanc@nvidia.com>
> ---
> docs/formatdomain.rst | 5 +-
> src/conf/domain_addr.c | 12 ++-
> src/conf/domain_addr.h | 4 +-
> src/conf/domain_conf.c | 153 ++++++++++++++++++++++++++----
> src/conf/domain_conf.h | 11 ++-
> src/conf/domain_validate.c | 38 +++++---
> src/conf/schemas/domaincommon.rng | 8 +-
> src/libvirt_private.syms | 2 +
> src/qemu/qemu_alias.c | 15 ++-
> src/qemu/qemu_command.c | 135 +++++++++++++++++---------
> src/qemu/qemu_domain_address.c | 33 ++++---
> src/qemu/qemu_driver.c | 8 +-
> src/qemu/qemu_postparse.c | 11 ++-
> src/qemu/qemu_validate.c | 22 ++++-
> 14 files changed, 342 insertions(+), 115 deletions(-)
>
> diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
> index 58b97a2b54..75568ff50a 100644
> --- a/src/conf/domain_conf.h
> +++ b/src/conf/domain_conf.h
> @@ -3005,6 +3005,7 @@ typedef enum {
> VIR_DOMAIN_IOMMU_MODEL_INTEL,
> VIR_DOMAIN_IOMMU_MODEL_SMMUV3,
> VIR_DOMAIN_IOMMU_MODEL_VIRTIO,
> + VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV,
Can you remove this addition from this patch and put it
in a following commit...
>
> VIR_DOMAIN_IOMMU_MODEL_LAST
> } virDomainIOMMUModel;
> @@ -3258,6 +3259,9 @@ struct _virDomainDef {
> size_t nwatchdogs;
> virDomainWatchdogDef **watchdogs;
>
> + size_t niommus;
> + virDomainIOMMUDef **iommu;
...so we focus exclusively on this bit.
> +
> /* At maximum 2 TPMs on the domain if a TPM Proxy is present. */
> size_t ntpms;
> virDomainTPMDef **tpms;
> @@ -3267,7 +3271,6 @@ struct _virDomainDef {
> virDomainNVRAMDef *nvram;
> virCPUDef *cpu;
> virDomainRedirFilterDef *redirfilter;
> - virDomainIOMMUDef *iommu;
> virDomainVsockDef *vsock;
> virDomainPstoreDef *pstore;
>
With regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
On 5/20/2025 4:20 AM, Daniel P. Berrangé wrote:
>> Add support for "smmuv3Dev" IOMMU model, and add
>> support for parsing multiple IOMMU devices from
>> the VM definition when "smmuv3Dev" is the IOMMU model.
>> Enable plugging smmuv3Dev into pcie-root and
>> pcie-expander-bus.
>>
>> Signed-off-by: Nathan Chen<nathanc@nvidia.com>
>> ---
>> docs/formatdomain.rst | 5 +-
>> src/conf/domain_addr.c | 12 ++-
>> src/conf/domain_addr.h | 4 +-
>> src/conf/domain_conf.c | 153 ++++++++++++++++++++++++++----
>> src/conf/domain_conf.h | 11 ++-
>> src/conf/domain_validate.c | 38 +++++---
>> src/conf/schemas/domaincommon.rng | 8 +-
>> src/libvirt_private.syms | 2 +
>> src/qemu/qemu_alias.c | 15 ++-
>> src/qemu/qemu_command.c | 135 +++++++++++++++++---------
>> src/qemu/qemu_domain_address.c | 33 ++++---
>> src/qemu/qemu_driver.c | 8 +-
>> src/qemu/qemu_postparse.c | 11 ++-
>> src/qemu/qemu_validate.c | 22 ++++-
>> 14 files changed, 342 insertions(+), 115 deletions(-)
>>
>> diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
>> index 58b97a2b54..75568ff50a 100644
>> --- a/src/conf/domain_conf.h
>> +++ b/src/conf/domain_conf.h
>> @@ -3005,6 +3005,7 @@ typedef enum {
>> VIR_DOMAIN_IOMMU_MODEL_INTEL,
>> VIR_DOMAIN_IOMMU_MODEL_SMMUV3,
>> VIR_DOMAIN_IOMMU_MODEL_VIRTIO,
>> + VIR_DOMAIN_IOMMU_MODEL_SMMUV3_DEV,
> Can you remove this addition from this patch and put it
> in a following commit...
>
>>
>> VIR_DOMAIN_IOMMU_MODEL_LAST
>> } virDomainIOMMUModel;
>> @@ -3258,6 +3259,9 @@ struct _virDomainDef {
>> size_t nwatchdogs;
>> virDomainWatchdogDef **watchdogs;
>>
>> + size_t niommus;
>> + virDomainIOMMUDef **iommu;
> ...so we focus exclusively on this bit.
>
Yes, I will separate these changes out in the next revision so that we
have separate commits - one for multiple IOMMU definitions and one for
smmuv3Dev support.
>> +
>> /* At maximum 2 TPMs on the domain if a TPM Proxy is present. */
>> size_t ntpms;
>> virDomainTPMDef **tpms;
>> @@ -3267,7 +3271,6 @@ struct _virDomainDef {
>> virDomainNVRAMDef *nvram;
>> virCPUDef *cpu;
>> virDomainRedirFilterDef *redirfilter;
>> - virDomainIOMMUDef *iommu;
>> virDomainVsockDef *vsock;
>> virDomainPstoreDef *pstore;
>>
> With regards,
> Daniel
Thanks,
Nathan
© 2016 - 2025 Red Hat, Inc.