From: Nathan Chen <nathanc@nvidia.com>

Add support for enabling HW-accelerated nested SMMUv3
via <accel> attribute and its additional attributes for
ATS, PASID, RIL, and OAS configuration.

Signed-off-by: Nathan Chen <nathanc@nvidia.com>
---
 docs/formatdomain.rst                         | 27 +++++++
 src/conf/domain_conf.c                        | 78 ++++++++++++++++++-
 src/conf/domain_conf.h                        |  5 ++
 src/conf/domain_validate.c                    | 21 ++++-
 src/conf/schemas/domaincommon.rng             | 25 ++++++
 src/qemu/qemu_command.c                       | 11 +++
 ...-smmuv3-pci-bus-single.aarch64-latest.args |  2 +-
 .../iommu-smmuv3-pci-bus.aarch64-latest.args  |  4 +-
 8 files changed, 166 insertions(+), 7 deletions(-)

diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
index 04ef319a73..458a514b60 100644
--- a/docs/formatdomain.rst
+++ b/docs/formatdomain.rst
@@ -9264,6 +9264,33 @@ Example:
       The ``pciBus`` attribute notes the index of the controller that an
       IOMMU device is attached to. (QEMU/KVM and ``smmuv3`` model only)
 
+   ``accel``
+      The ``accel`` attribute with possible values ``on`` and ``off`` can be used
+      to enable hardware acceleration support for smmuv3Dev IOMMU devices.
+      (QEMU/KVM and ``smmuv3`` model only)
+
+   ``ats``
+      The ``ats`` attribute with possible values ``on`` and ``off`` can be used
+      to enable reporting Address Translation Services capability to the guest
+      for smmuv3Dev IOMMU devices with ``accel`` set to ``on``, if the host
+      SMMUv3 supports ATS and the associated passthrough device supports ATS.
+      (QEMU/KVM and ``smmuv3`` model only)
+
+   ``ril``
+      The ``ril`` attribute with possible values ``on`` and ``off`` can be used
+      to report whether Range Invalidation for smmuv3Dev IOMMU devices with
+      ``accel`` set to ``on`` is compatible with host SMMUv3 support.
+      (QEMU/KVM and ``smmuv3`` model only)
+
+   ``pasid``
+      The ``pasid`` attribute with possible values ``on`` and ``off`` can be
+      used to enable Process Address Space ID support for smmuv3Dev IOMMU devices
+      with ``accel`` set to ``on``. (QEMU/KVM and ``smmuv3`` model only)
+
+   ``oas``
+      The ``oas`` attribute sets the output address size in units of bits.
+      (QEMU/KVM and ``smmuv3`` model only)
+
 The ``virtio`` IOMMU devices can further have ``address`` element as described
 in `Device addresses`_ (address has to by type of ``pci``).
 
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index b85bbbcaf6..99183b5c82 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -14514,6 +14514,26 @@ virDomainIOMMUDefParseXML(virDomainXMLOption *xmlopt,
         if (virXMLPropInt(driver, "pciBus", 10, VIR_XML_PROP_NONE,
                           &iommu->pci_bus, -1) < 0)
             return NULL;
+
+        if (virXMLPropTristateSwitch(driver, "accel", VIR_XML_PROP_NONE,
+                                     &iommu->accel) < 0)
+            return NULL;
+
+        if (virXMLPropTristateSwitch(driver, "ats", VIR_XML_PROP_NONE,
+                                     &iommu->ats) < 0)
+            return NULL;
+
+        if (virXMLPropTristateSwitch(driver, "ril", VIR_XML_PROP_NONE,
+                                     &iommu->ril) < 0)
+            return NULL;
+
+        if (virXMLPropTristateSwitch(driver, "pasid", VIR_XML_PROP_NONE,
+                                     &iommu->pasid) < 0)
+            return NULL;
+
+        if (virXMLPropUInt(driver, "oas", 10, VIR_XML_PROP_NONE,
+                           &iommu->oas) < 0)
+            return NULL;
     }
 
     if (virDomainDeviceInfoParseXML(xmlopt, node, ctxt,
@@ -16565,7 +16585,13 @@ virDomainIOMMUDefEquals(const virDomainIOMMUDef *a,
         a->eim != b->eim ||
         a->iotlb != b->iotlb ||
         a->aw_bits != b->aw_bits ||
-        a->dma_translation != b->dma_translation)
+        a->dma_translation != b->dma_translation ||
+        a->pci_bus != b->pci_bus ||
+        a->accel != b->accel ||
+        a->ats != b->ats ||
+        a->ril != b->ril ||
+        a->pasid != b->pasid ||
+        a->oas != b->oas)
         return false;
 
     if (a->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE &&
@@ -22262,6 +22288,36 @@ virDomainIOMMUDefCheckABIStability(virDomainIOMMUDef *src,
                        dst->pci_bus, src->pci_bus);
         return false;
     }
+    if (src->accel != dst->accel) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("Target domain IOMMU device accel value '%1$d' does not match source '%2$d'"),
+                       dst->accel, src->accel);
+        return false;
+    }
+    if (src->ats != dst->ats) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("Target domain IOMMU device ATS value '%1$d' does not match source '%2$d'"),
+                       dst->ats, src->ats);
+        return false;
+    }
+    if (src->ril != dst->ril) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("Target domain IOMMU device ril value '%1$d' does not match source '%2$d'"),
+                       dst->ril, src->ril);
+        return false;
+    }
+    if (src->pasid != dst->pasid) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("Target domain IOMMU device pasid value '%1$d' does not match source '%2$d'"),
+                       dst->pasid, src->pasid);
+        return false;
+    }
+    if (src->oas != dst->oas) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("Target domain IOMMU device oas value '%1$d' does not match source '%2$d'"),
+                       dst->oas, src->oas);
+        return false;
+    }
     if (src->dma_translation != dst->dma_translation) {
         virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                        _("Target domain IOMMU device dma translation '%1$s' does not match source '%2$s'"),
@@ -28608,6 +28664,26 @@ virDomainIOMMUDefFormat(virBuffer *buf,
         virBufferAsprintf(&driverAttrBuf, " pciBus='%d'",
                           iommu->pci_bus);
     }
+    if (iommu->accel != VIR_TRISTATE_SWITCH_ABSENT) {
+        virBufferAsprintf(&driverAttrBuf, " accel='%s'",
+                          virTristateSwitchTypeToString(iommu->accel));
+    }
+    if (iommu->ats != VIR_TRISTATE_SWITCH_ABSENT) {
+        virBufferAsprintf(&driverAttrBuf, " ats='%s'",
+                          virTristateSwitchTypeToString(iommu->ats));
+    }
+    if (iommu->ril != VIR_TRISTATE_SWITCH_ABSENT) {
+        virBufferAsprintf(&driverAttrBuf, " ril='%s'",
+                          virTristateSwitchTypeToString(iommu->ril));
+    }
+    if (iommu->pasid != VIR_TRISTATE_SWITCH_ABSENT) {
+        virBufferAsprintf(&driverAttrBuf, " pasid='%s'",
+                          virTristateSwitchTypeToString(iommu->pasid));
+    }
+    if (iommu->oas > 0) {
+        virBufferAsprintf(&driverAttrBuf, " oas='%d'",
+                          iommu->oas);
+    }
 
     virXMLFormatElement(&childBuf, "driver", &driverAttrBuf, NULL);
 
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index cb35ff06bd..71ed4ce0ed 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -3062,6 +3062,11 @@ struct _virDomainIOMMUDef {
     virTristateSwitch dma_translation;
     virTristateSwitch xtsup;
     virTristateSwitch pt;
+    virTristateSwitch accel;
+    virTristateSwitch ats;
+    virTristateSwitch ril;
+    virTristateSwitch pasid;
+    unsigned int oas;
 };
 
 typedef enum {
diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
index 4558e7b210..0f84e9f237 100644
--- a/src/conf/domain_validate.c
+++ b/src/conf/domain_validate.c
@@ -3140,7 +3140,12 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu)
             iommu->iotlb != VIR_TRISTATE_SWITCH_ABSENT ||
             iommu->aw_bits != 0 ||
             iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT ||
-            iommu->pci_bus >= 0) {
+            iommu->pci_bus >= 0 ||
+            iommu->accel != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->ats != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->ril != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->pasid != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->oas != 0) {
             virReportError(VIR_ERR_XML_ERROR,
                            _("iommu model '%1$s' doesn't support additional attributes"),
                            virDomainIOMMUModelTypeToString(iommu->model));
@@ -3153,7 +3158,12 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu)
             iommu->eim != VIR_TRISTATE_SWITCH_ABSENT ||
             iommu->aw_bits != 0 ||
             iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT ||
-            iommu->pci_bus >= 0) {
+            iommu->pci_bus >= 0 ||
+            iommu->accel != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->ats != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->ril != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->pasid != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->oas != 0) {
             virReportError(VIR_ERR_XML_ERROR,
                            _("iommu model '%1$s' doesn't support some additional attributes"),
                            virDomainIOMMUModelTypeToString(iommu->model));
@@ -3164,7 +3174,12 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu)
     case VIR_DOMAIN_IOMMU_MODEL_INTEL:
         if (iommu->pt != VIR_TRISTATE_SWITCH_ABSENT ||
             iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT ||
-            iommu->pci_bus >= 0) {
+            iommu->pci_bus >= 0 ||
+            iommu->accel != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->ats != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->ril != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->pasid != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->oas != 0) {
             virReportError(VIR_ERR_XML_ERROR,
                            _("iommu model '%1$s' doesn't support some additional attributes"),
                            virDomainIOMMUModelTypeToString(iommu->model));
diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
index 5d5fd87f87..3d6dc695c4 100644
--- a/src/conf/schemas/domaincommon.rng
+++ b/src/conf/schemas/domaincommon.rng
@@ -6329,6 +6329,31 @@
                 <data type="unsignedInt"/>
               </attribute>
             </optional>
+            <optional>
+              <attribute name="accel">
+                <ref name="virOnOff"/>
+              </attribute>
+            </optional>
+            <optional>
+              <attribute name="ats">
+                <ref name="virOnOff"/>
+              </attribute>
+            </optional>
+            <optional>
+              <attribute name="ril">
+                <ref name="virOnOff"/>
+              </attribute>
+            </optional>
+            <optional>
+              <attribute name="pasid">
+                <ref name="virOnOff"/>
+              </attribute>
+            </optional>
+            <optional>
+              <attribute name="oas">
+                <data type="unsignedInt"/>
+              </attribute>
+            </optional>
           </element>
         </optional>
         <optional>
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index 0de0a79b46..c4a6ec7aa6 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -6267,9 +6267,20 @@ qemuBuildPCINestedSmmuv3DevProps(const virDomainDef *def,
                               "s:driver", "arm-smmuv3",
                               "s:primary-bus", bus,
                               "s:id", iommu->info.alias,
+                              "b:accel", (iommu->accel == VIR_TRISTATE_SWITCH_ON),
+                              "b:ats", (iommu->ats == VIR_TRISTATE_SWITCH_ON),
+                              "b:ril", (iommu->ril == VIR_TRISTATE_SWITCH_ON),
+                              "b:pasid", (iommu->pasid == VIR_TRISTATE_SWITCH_ON),
                               NULL) < 0)
         return NULL;
 
+    if (iommu->oas > 0) {
+        if (virJSONValueObjectAdd(&props,
+                                  "U:oas", (unsigned long long)iommu->oas,
+                                  NULL) < 0)
+            return NULL;
+    }
+
     return g_steal_pointer(&props);
 }
 
diff --git a/tests/qemuxmlconfdata/iommu-smmuv3-pci-bus-single.aarch64-latest.args b/tests/qemuxmlconfdata/iommu-smmuv3-pci-bus-single.aarch64-latest.args
index 34e7bda1c5..f5c76018e9 100644
--- a/tests/qemuxmlconfdata/iommu-smmuv3-pci-bus-single.aarch64-latest.args
+++ b/tests/qemuxmlconfdata/iommu-smmuv3-pci-bus-single.aarch64-latest.args
@@ -30,7 +30,7 @@ XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain--1-guest/.config \
 -device '{"driver":"pxb-pcie","bus_nr":248,"id":"pci.2","bus":"pcie.0","addr":"0x2"}' \
 -device '{"driver":"pcie-root-port","port":0,"chassis":21,"id":"pci.3","bus":"pci.1","addr":"0x0"}' \
 -device '{"driver":"pcie-root-port","port":168,"chassis":22,"id":"pci.4","bus":"pci.2","addr":"0x0"}' \
--device '{"driver":"arm-smmuv3","primary-bus":"pci.1","id":"iommu0"}' \
+-device '{"driver":"arm-smmuv3","primary-bus":"pci.1","id":"iommu0","accel":false,"ats":false,"ril":false,"pasid":false}' \
 -audiodev '{"id":"audio1","driver":"none"}' \
 -object '{"qom-type":"rng-random","id":"objrng0","filename":"/dev/urandom"}' \
 -device '{"driver":"virtio-rng-pci","rng":"objrng0","id":"rng0","bus":"pci.3","addr":"0x0"}' \
diff --git a/tests/qemuxmlconfdata/iommu-smmuv3-pci-bus.aarch64-latest.args b/tests/qemuxmlconfdata/iommu-smmuv3-pci-bus.aarch64-latest.args
index ff75b08944..839763e6d2 100644
--- a/tests/qemuxmlconfdata/iommu-smmuv3-pci-bus.aarch64-latest.args
+++ b/tests/qemuxmlconfdata/iommu-smmuv3-pci-bus.aarch64-latest.args
@@ -30,8 +30,8 @@ XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain--1-guest/.config \
 -device '{"driver":"pxb-pcie","bus_nr":248,"id":"pci.2","bus":"pcie.0","addr":"0x2"}' \
 -device '{"driver":"pcie-root-port","port":0,"chassis":21,"id":"pci.3","bus":"pci.1","addr":"0x0"}' \
 -device '{"driver":"pcie-root-port","port":168,"chassis":22,"id":"pci.4","bus":"pci.2","addr":"0x0"}' \
--device '{"driver":"arm-smmuv3","primary-bus":"pci.1","id":"iommu0"}' \
--device '{"driver":"arm-smmuv3","primary-bus":"pci.2","id":"iommu1"}' \
+-device '{"driver":"arm-smmuv3","primary-bus":"pci.1","id":"iommu0","accel":false,"ats":false,"ril":false,"pasid":false}' \
+-device '{"driver":"arm-smmuv3","primary-bus":"pci.2","id":"iommu1","accel":false,"ats":false,"ril":false,"pasid":false}' \
 -audiodev '{"id":"audio1","driver":"none"}' \
 -object '{"qom-type":"rng-random","id":"objrng0","filename":"/dev/urandom"}' \
 -device '{"driver":"virtio-rng-pci","rng":"objrng0","id":"rng0","bus":"pci.3","addr":"0x0"}' \
-- 
2.43.0