From: Nathan Chen <nathanc@nvidia.com>
Add support for enabling HW-accelerated nested SMMUv3
via <accel> attribute and its additional attributes for
ATS, SSID, RIL, and OAS configuration. Support element
for specifying PCI hostdev PASID capability offset.
Signed-off-by: Nathan Chen <nathanc@nvidia.com>
---
docs/formatdomain.rst | 36 +++++++++++
src/conf/domain_conf.c | 101 +++++++++++++++++++++++++++++-
src/conf/domain_conf.h | 6 ++
src/conf/domain_validate.c | 36 ++++++++++-
src/conf/schemas/domaincommon.rng | 32 ++++++++++
src/qemu/qemu_command.c | 23 +++++++
src/util/virpci.h | 4 ++
7 files changed, 233 insertions(+), 5 deletions(-)
diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
index 04ef319a73..998d289ebf 100644
--- a/docs/formatdomain.rst
+++ b/docs/formatdomain.rst
@@ -4880,6 +4880,13 @@ or:
device; if PCI ROM loading is disabled through this attribute, attempts to
tweak the loading process further using the ``bar`` or ``file`` attributes
will be rejected. :since:`Since 4.3.0 (QEMU and KVM only)`.
+``vpasidCapOffset``
+ The ``vpasidCapOffset`` element is used to change the offset at which a
+ PASID PCIe extended capability is placed in a vfio-pci device's PCIe
+ extended configuration space. If not specified or set to 0, the capability
+ is placed at the end of the extended configuration space when PASID is
+ supported. The offset must be 4-byte aligned and within the PCIe extended
+ configuration space.
``address``
The ``address`` element for USB devices has a ``bus`` and ``device``
attribute to specify the USB bus and device number the device appears at on
@@ -9264,6 +9271,35 @@ Example:
The ``pciBus`` attribute notes the index of the controller that an
IOMMU device is attached to. (QEMU/KVM and ``smmuv3`` model only)
+ ``accel``
+ The ``accel`` attribute with possible values ``on`` and ``off`` can be used
+ to enable hardware acceleration support for smmuv3Dev IOMMU devices.
+ (QEMU/KVM and ``smmuv3`` model only)
+
+ ``ats``
+ The ``ats`` attribute with possible values ``on`` and ``off`` can be used
+ to enable reporting Address Translation Services capability to the guest
+ for smmuv3Dev IOMMU devices with ``accel`` set to ``on``, if the host
+ SMMUv3 supports ATS and the associated passthrough device supports ATS.
+ (QEMU/KVM and ``smmuv3`` model only)
+
+ ``ril``
+ The ``ril`` attribute with possible values ``on`` and ``off`` can be used
+ to report whether Range Invalidation for smmuv3Dev IOMMU devices with
+ ``accel`` set to ``on`` is compatible with host SMMUv3 support.
+ (QEMU/KVM and ``smmuv3`` model only)
+
+ ``ssidSize``
+ The ``ssidSize`` attribute sets the number of bits used to represent
+ SubstreamIDs. A value of N allows SSIDs in the range [0 .. 2^N - 1].
+ The valid range is 0-20, and a value greater than 0 is required for
+ enabling PASID support, as doing so advertises PASID capability to
+ the vIOMMU. (QEMU/KVM and ``smmuv3`` model only)
+
+ ``oas``
+ The ``oas`` attribute sets the output address size in units of bits.
+ (QEMU/KVM and ``smmuv3`` model only)
+
The ``virtio`` IOMMU devices can further have ``address`` element as described
in `Device addresses`_ (address has to by type of ``pci``).
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 9672168df9..e64484a60d 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -13678,6 +13678,7 @@ virDomainHostdevDefParseXML(virDomainXMLOption *xmlopt,
virDomainHostdevDef *def;
VIR_XPATH_NODE_AUTORESTORE(ctxt)
unsigned int type;
+ int rc;
ctxt->node = node;
@@ -13731,8 +13732,16 @@ virDomainHostdevDefParseXML(virDomainXMLOption *xmlopt,
def->shareable = true;
break;
- case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB:
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI:
+ rc = virXPathUIntBase("string(./vpasidCapOffset)", ctxt,
+ 0, &def->vpasidCapOffset);
+ if (rc == -2) {
+ virReportError(VIR_ERR_XML_ERROR, "%s",
+ _("Invalid format for vpasidCapOffset"));
+ goto error;
+ }
+ break;
+ case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB:
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_SCSI_HOST:
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_MDEV:
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_LAST:
@@ -14514,6 +14523,26 @@ virDomainIOMMUDefParseXML(virDomainXMLOption *xmlopt,
if (virXMLPropInt(driver, "pciBus", 10, VIR_XML_PROP_NONE,
&iommu->pci_bus, -1) < 0)
return NULL;
+
+ if (virXMLPropTristateSwitch(driver, "accel", VIR_XML_PROP_NONE,
+ &iommu->accel) < 0)
+ return NULL;
+
+ if (virXMLPropTristateSwitch(driver, "ats", VIR_XML_PROP_NONE,
+ &iommu->ats) < 0)
+ return NULL;
+
+ if (virXMLPropTristateSwitch(driver, "ril", VIR_XML_PROP_NONE,
+ &iommu->ril) < 0)
+ return NULL;
+
+ if (virXMLPropUInt(driver, "ssidSize", 10, VIR_XML_PROP_NONE,
+ &iommu->ssid_size) < 0)
+ return NULL;
+
+ if (virXMLPropUInt(driver, "oas", 10, VIR_XML_PROP_NONE,
+ &iommu->oas) < 0)
+ return NULL;
}
if (virDomainDeviceInfoParseXML(xmlopt, node, ctxt,
@@ -16577,7 +16606,13 @@ virDomainIOMMUDefEquals(const virDomainIOMMUDef *a,
a->eim != b->eim ||
a->iotlb != b->iotlb ||
a->aw_bits != b->aw_bits ||
- a->dma_translation != b->dma_translation)
+ a->dma_translation != b->dma_translation ||
+ a->pci_bus != b->pci_bus ||
+ a->accel != b->accel ||
+ a->ats != b->ats ||
+ a->ril != b->ril ||
+ a->ssid_size != b->ssid_size ||
+ a->oas != b->oas)
return false;
if (a->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE &&
@@ -21349,6 +21384,14 @@ virDomainHostdevDefCheckABIStability(virDomainHostdevDef *src,
}
}
+ if (src->vpasidCapOffset != dst->vpasidCapOffset) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Target host device vPASID capability offset %1$s does not match source %2$s"),
+ virDomainHostdevModeTypeToString(dst->vpasidCapOffset),
+ virDomainHostdevModeTypeToString(src->vpasidCapOffset));
+ return false;
+ }
+
if (!virDomainDeviceInfoCheckABIStability(src->info, dst->info))
return false;
@@ -22311,6 +22354,36 @@ virDomainIOMMUDefCheckABIStability(virDomainIOMMUDef *src,
dst->pci_bus, src->pci_bus);
return false;
}
+ if (src->accel != dst->accel) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Target domain IOMMU device accel value '%1$d' does not match source '%2$d'"),
+ dst->accel, src->accel);
+ return false;
+ }
+ if (src->ats != dst->ats) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Target domain IOMMU device ATS value '%1$d' does not match source '%2$d'"),
+ dst->ats, src->ats);
+ return false;
+ }
+ if (src->ril != dst->ril) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Target domain IOMMU device ril value '%1$d' does not match source '%2$d'"),
+ dst->ril, src->ril);
+ return false;
+ }
+ if (src->ssid_size != dst->ssid_size) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Target domain IOMMU device ssid_size value '%1$d' does not match source '%2$d'"),
+ dst->ssid_size, src->ssid_size);
+ return false;
+ }
+ if (src->oas != dst->oas) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Target domain IOMMU device oas value '%1$d' does not match source '%2$d'"),
+ dst->oas, src->oas);
+ return false;
+ }
if (src->dma_translation != dst->dma_translation) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
_("Target domain IOMMU device dma translation '%1$s' does not match source '%2$s'"),
@@ -27732,6 +27805,10 @@ virDomainHostdevDefFormat(virBuffer *buf,
if (def->shareable)
virBufferAddLit(buf, "<shareable/>\n");
+ if (def->vpasidCapOffset)
+ virBufferAsprintf(buf, "<vpasidCapOffset>0x%x</vpasidCapOffset>\n",
+ def->vpasidCapOffset);
+
virDomainDeviceInfoFormat(buf, def->info, flags | VIR_DOMAIN_DEF_FORMAT_ALLOW_BOOT
| VIR_DOMAIN_DEF_FORMAT_ALLOW_ROM);
@@ -28657,6 +28734,26 @@ virDomainIOMMUDefFormat(virBuffer *buf,
virBufferAsprintf(&driverAttrBuf, " pciBus='%d'",
iommu->pci_bus);
}
+ if (iommu->accel != VIR_TRISTATE_SWITCH_ABSENT) {
+ virBufferAsprintf(&driverAttrBuf, " accel='%s'",
+ virTristateSwitchTypeToString(iommu->accel));
+ }
+ if (iommu->ats != VIR_TRISTATE_SWITCH_ABSENT) {
+ virBufferAsprintf(&driverAttrBuf, " ats='%s'",
+ virTristateSwitchTypeToString(iommu->ats));
+ }
+ if (iommu->ril != VIR_TRISTATE_SWITCH_ABSENT) {
+ virBufferAsprintf(&driverAttrBuf, " ril='%s'",
+ virTristateSwitchTypeToString(iommu->ril));
+ }
+ if (iommu->ssid_size > 0) {
+ virBufferAsprintf(&driverAttrBuf, " ssidSize='%d'",
+ iommu->ssid_size);
+ }
+ if (iommu->oas > 0) {
+ virBufferAsprintf(&driverAttrBuf, " oas='%d'",
+ iommu->oas);
+ }
virXMLFormatElement(&childBuf, "driver", &driverAttrBuf, NULL);
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index 83d49969d3..1396073678 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -370,6 +370,7 @@ struct _virDomainHostdevDef {
bool missing;
bool readonly;
bool shareable;
+ unsigned int vpasidCapOffset;
virTristateBool writeFiltering;
union {
virDomainHostdevSubsys subsys;
@@ -3062,6 +3063,11 @@ struct _virDomainIOMMUDef {
virTristateSwitch dma_translation;
virTristateSwitch xtsup;
virTristateSwitch pt;
+ virTristateSwitch accel;
+ virTristateSwitch ats;
+ virTristateSwitch ril;
+ unsigned int ssid_size;
+ unsigned int oas;
};
typedef enum {
diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
index 4482203087..7b2c32395d 100644
--- a/src/conf/domain_validate.c
+++ b/src/conf/domain_validate.c
@@ -29,6 +29,7 @@
#include "virutil.h"
#include "virstring.h"
#include "virhostmem.h"
+#include "virpci.h"
#define VIR_FROM_THIS VIR_FROM_DOMAIN
@@ -2432,6 +2433,20 @@ virDomainHostdevDefValidate(const virDomainHostdevDef *hostdev)
_("PCI host devices must use 'pci' or 'unassigned' address type"));
return -1;
}
+ if (hostdev->vpasidCapOffset) {
+ if (hostdev->vpasidCapOffset & 0x3) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("vpasidCapOffset must be 4-byte aligned"));
+ return -1;
+ }
+ /* PASID ECAP size of 0x8 */
+ if (hostdev->vpasidCapOffset < VIR_DOMAIN_PCI_CONFIG_SPACE_SIZE ||
+ hostdev->vpasidCapOffset > VIR_DOMAIN_PCIE_CONFIG_SPACE_SIZE - 0x8) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("vpasidCapOffset must be within PCIe extended configuration space (0x100-0xFFF)"));
+ return -1;
+ }
+ }
break;
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_SCSI:
if (hostdev->info->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE &&
@@ -3208,7 +3223,12 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu)
iommu->iotlb != VIR_TRISTATE_SWITCH_ABSENT ||
iommu->aw_bits != 0 ||
iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT ||
- iommu->pci_bus >= 0) {
+ iommu->pci_bus >= 0 ||
+ iommu->accel != VIR_TRISTATE_SWITCH_ABSENT ||
+ iommu->ats != VIR_TRISTATE_SWITCH_ABSENT ||
+ iommu->ril != VIR_TRISTATE_SWITCH_ABSENT ||
+ iommu->ssid_size != 0 ||
+ iommu->oas != 0) {
virReportError(VIR_ERR_XML_ERROR,
_("iommu model '%1$s' doesn't support additional attributes"),
virDomainIOMMUModelTypeToString(iommu->model));
@@ -3221,7 +3241,12 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu)
iommu->eim != VIR_TRISTATE_SWITCH_ABSENT ||
iommu->aw_bits != 0 ||
iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT ||
- iommu->pci_bus >= 0) {
+ iommu->pci_bus >= 0 ||
+ iommu->accel != VIR_TRISTATE_SWITCH_ABSENT ||
+ iommu->ats != VIR_TRISTATE_SWITCH_ABSENT ||
+ iommu->ril != VIR_TRISTATE_SWITCH_ABSENT ||
+ iommu->ssid_size != 0 ||
+ iommu->oas != 0) {
virReportError(VIR_ERR_XML_ERROR,
_("iommu model '%1$s' doesn't support some additional attributes"),
virDomainIOMMUModelTypeToString(iommu->model));
@@ -3232,7 +3257,12 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu)
case VIR_DOMAIN_IOMMU_MODEL_INTEL:
if (iommu->pt != VIR_TRISTATE_SWITCH_ABSENT ||
iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT ||
- iommu->pci_bus >= 0) {
+ iommu->pci_bus >= 0 ||
+ iommu->accel != VIR_TRISTATE_SWITCH_ABSENT ||
+ iommu->ats != VIR_TRISTATE_SWITCH_ABSENT ||
+ iommu->ril != VIR_TRISTATE_SWITCH_ABSENT ||
+ iommu->ssid_size != 0 ||
+ iommu->oas != 0) {
virReportError(VIR_ERR_XML_ERROR,
_("iommu model '%1$s' doesn't support some additional attributes"),
virDomainIOMMUModelTypeToString(iommu->model));
diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
index 114dd3f96f..234aae5459 100644
--- a/src/conf/schemas/domaincommon.rng
+++ b/src/conf/schemas/domaincommon.rng
@@ -6329,6 +6329,31 @@
<data type="unsignedInt"/>
</attribute>
</optional>
+ <optional>
+ <attribute name="accel">
+ <ref name="virOnOff"/>
+ </attribute>
+ </optional>
+ <optional>
+ <attribute name="ats">
+ <ref name="virOnOff"/>
+ </attribute>
+ </optional>
+ <optional>
+ <attribute name="ril">
+ <ref name="virOnOff"/>
+ </attribute>
+ </optional>
+ <optional>
+ <attribute name="ssidSize">
+ <data type="unsignedInt"/>
+ </attribute>
+ </optional>
+ <optional>
+ <attribute name="oas">
+ <data type="unsignedInt"/>
+ </attribute>
+ </optional>
</element>
</optional>
<optional>
@@ -6610,6 +6635,13 @@
<ref name="pciaddress"/>
</element>
</element>
+ <optional>
+ <element name="vpasidCapOffset">
+ <data type="string">
+ <param name="pattern">(0x[0-9a-fA-F]+|[0-9]+)</param>
+ </data>
+ </element>
+ </optional>
<ref name="hostdevsubsysvfiodisplay"/>
</interleave>
</define>
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index e81efdfde7..aa5ee2a787 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -4802,6 +4802,7 @@ qemuBuildPCIHostdevDevProps(const virDomainDef *def,
"S:failover_pair_id", failover_pair_id,
"S:display", qemuOnOffAuto(pcisrc->display),
"B:ramfb", ramfb,
+ "p:x-vpasid-cap-offset", dev->vpasidCapOffset,
NULL) < 0)
return NULL;
@@ -6267,9 +6268,31 @@ qemuBuildPCINestedSmmuv3DevProps(const virDomainDef *def,
"s:driver", "arm-smmuv3",
"s:primary-bus", bus,
"s:id", iommu->info.alias,
+ "B:accel", (iommu->accel == VIR_TRISTATE_SWITCH_ON),
+ "B:ats", (iommu->ats == VIR_TRISTATE_SWITCH_ON),
NULL) < 0)
return NULL;
+ /* QEMU SMMUv3 has RIL support by default; only emit when explicitly disabling */
+ if (iommu->ril == VIR_TRISTATE_SWITCH_OFF) {
+ if (virJSONValueObjectAppendBoolean(props, "ril", false) < 0)
+ return NULL;
+ }
+
+ if (iommu->ssid_size > 0) {
+ if (virJSONValueObjectAdd(&props,
+ "p:ssidsize", iommu->ssid_size,
+ NULL) < 0)
+ return NULL;
+ }
+
+ if (iommu->oas > 0) {
+ if (virJSONValueObjectAdd(&props,
+ "p:oas", iommu->oas,
+ NULL) < 0)
+ return NULL;
+ }
+
return g_steal_pointer(&props);
}
diff --git a/src/util/virpci.h b/src/util/virpci.h
index fc538566e1..7e78cb267c 100644
--- a/src/util/virpci.h
+++ b/src/util/virpci.h
@@ -35,6 +35,10 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC(virPCIDeviceList, virObjectUnref);
#define VIR_DOMAIN_DEVICE_ZPCI_MAX_UID UINT16_MAX
#define VIR_DOMAIN_DEVICE_ZPCI_MAX_FID UINT32_MAX
+/* Size of the standard PCI config space */
+#define VIR_DOMAIN_PCI_CONFIG_SPACE_SIZE 0x100
+/* Size of the standard PCIe config space: 4KB */
+#define VIR_DOMAIN_PCIE_CONFIG_SPACE_SIZE 0x1000
typedef struct _virZPCIDeviceAddressID virZPCIDeviceAddressID;
typedef struct _virZPCIDeviceAddress virZPCIDeviceAddress;
--
2.43.0© 2016 - 2026 Red Hat, Inc.