From: Michal Privoznik <mprivozn@redhat.com>

In PCI assignment scenario the virtio-iommu needs to know the
guest page size also known as granule. Expose it as an attribute
to the <driver/> element of a virtio-iommu.

This is possibly interesting only for aarch64 since it supports
virtio-iommu and also supports running guests with different page
size than the host.

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>
---
 docs/formatdomain.rst                         | 38 ++++++++-
 src/conf/domain_conf.c                        | 77 ++++++++++++++++++-
 src/conf/domain_conf.h                        |  1 +
 src/conf/domain_validate.c                    |  9 ++-
 src/conf/schemas/domaincommon.rng             | 22 ++++++
 src/qemu/qemu_validate.c                      | 11 +++
 .../virtio-iommu-aarch64.aarch64-latest.xml   |  4 +-
 .../qemuxmlconfdata/virtio-iommu-aarch64.xml  |  4 +-
 .../virtio-iommu-x86_64.x86_64-latest.xml     |  3 +
 tests/qemuxmlconfdata/virtio-iommu-x86_64.xml |  6 +-
 10 files changed, 166 insertions(+), 9 deletions(-)

diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
index 4b34a8a963..50563741a0 100644
--- a/docs/formatdomain.rst
+++ b/docs/formatdomain.rst
@@ -9194,7 +9194,7 @@ IOMMU devices
 
 The ``iommu`` element can be used to add an IOMMU device. :since:`Since 2.1.0`
 
-Example:
+Examples:
 
 ::
 
@@ -9206,6 +9206,17 @@ Example:
    </devices>
    ...
 
+
+   ...
+   <devices>
+     <iommu model='virtio'>
+       <driver aw_bits='48'>
+         <granule size='64' unit='KiB'/>
+       </driver>
+     </iommu>
+   </devices>
+   ...
+
 ``model``
    Supported values are ``intel`` (for Q35 guests) ``smmuv3``
    (:since:`since 5.5.0`, for ARM virt guests), ``virtio``
@@ -9264,6 +9275,31 @@ Example:
       The ``pciBus`` attribute notes the index of the controller that an
       IOMMU device is attached to. (QEMU/KVM and ``smmuv3`` model only)
 
+In case of ``virtio`` IOMMU device, the ``driver`` element can optionally
+contain ``granule`` subelement that allows to choose which granule will be
+used by default. It is useful when running guests with different page size
+than the host. :since:`Since 12.1.0` (QEMU/KVM and ``virtio`` model only).
+There are two possible options:
+
+::
+
+  <iommu model='virtio'>
+    <driver>
+      <granule mode='host'/>
+    </driver>
+  </iommu>
+
+  <iommu model='virtio'>
+    <driver>
+      <granule size='64' unit='KiB'/>
+    </driver>
+  </iommu>
+
+The ``mode='host'`` case matches the host page size, the other sets desired
+granule size. Please note that hypervisor might support only some selected
+values. For instance, QEMU supports only 4KiB, 8KiB, 16KiB and 64KiB large
+granules.
+
 The ``virtio`` IOMMU devices can further have ``address`` element as described
 in `Device addresses`_ (address has to by type of ``pci``).
 
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 1b2a439ca4..4503fc0095 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -1592,6 +1592,21 @@ VIR_ENUM_IMPL(virDomainChrSourceMode,
 );
 
 
+/*virDomainIOMMUGranuleModeTypeToString:
+ * @val: value to format
+ *
+ * Reuturns: an allocated string. Caller must free it.
+ */
+static char *
+virDomainIOMMUGranuleModeTypeToString(int val)
+{
+    if (val == -1)
+        return g_strdup("host");
+
+    return g_strdup_printf("%dKiB", val);
+}
+
+
 static virClass *virDomainObjClass;
 static virClass *virDomainXMLOptionClass;
 static void virDomainObjDispose(void *obj);
@@ -14487,6 +14502,8 @@ virDomainIOMMUDefParseXML(virDomainXMLOption *xmlopt,
         return NULL;
 
     if ((driver = virXPathNode("./driver", ctxt))) {
+        xmlNodePtr granule;
+
         if (virXMLPropTristateSwitch(driver, "intremap", VIR_XML_PROP_NONE,
                                      &iommu->intremap) < 0)
             return NULL;
@@ -14522,6 +14539,41 @@ virDomainIOMMUDefParseXML(virDomainXMLOption *xmlopt,
         if (virXMLPropInt(driver, "pciBus", 10, VIR_XML_PROP_NONE,
                           &iommu->pci_bus, -1) < 0)
             return NULL;
+
+        if ((granule = virXPathNode("./driver/granule", ctxt))) {
+            g_autofree char *mode = virXMLPropString(granule, "mode");
+            unsigned long long size;
+            int rc;
+
+            rc = virDomainParseMemory("./driver/granule/@size",
+                                      "./driver/granule/@unit",
+                                      ctxt, &size, false, false);
+            if (rc < 0) {
+                return NULL;
+            } else if (rc > 0) {
+                if (mode) {
+                    virReportError(VIR_ERR_XML_ERROR, "%s",
+                                   _("'mode' and 'size' can't be specified at the same time for 'granule'"));
+                    return NULL;
+                }
+
+                if (size != (int) size) {
+                    virReportError(VIR_ERR_OVERFLOW, "%s", _("size value too large"));
+                    return NULL;
+                }
+
+                iommu->granule = (int)size;
+            } else {
+                if (STREQ_NULLABLE(mode, "host")) {
+                    iommu->granule = -1;
+                } else if (mode) {
+                    virReportError(VIR_ERR_XML_ERROR,
+                                   _("Invalid value for attribute '%1$s' in element '%2$s': '%3$s'."),
+                                   "mode", "granule", mode);
+                    return NULL;
+                }
+            }
+        }
     }
 
     if (virDomainDeviceInfoParseXML(xmlopt, node, ctxt,
@@ -16585,7 +16637,8 @@ virDomainIOMMUDefEquals(const virDomainIOMMUDef *a,
         a->eim != b->eim ||
         a->iotlb != b->iotlb ||
         a->aw_bits != b->aw_bits ||
-        a->dma_translation != b->dma_translation)
+        a->dma_translation != b->dma_translation ||
+        a->granule != b->granule)
         return false;
 
     if (a->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE &&
@@ -22340,6 +22393,16 @@ virDomainIOMMUDefCheckABIStability(virDomainIOMMUDef *src,
                        virTristateSwitchTypeToString(src->xtsup));
         return false;
     }
+    if (src->granule != dst->granule) {
+        g_autofree char *src_granule = virDomainIOMMUGranuleModeTypeToString(src->granule);
+        g_autofree char *dst_granule = virDomainIOMMUGranuleModeTypeToString(dst->granule);
+
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("Target domain IOMMU device granule '%1$s' does not match source '%2$s'"),
+                       dst_granule,
+                       src_granule);
+        return false;
+    }
 
     return virDomainDeviceInfoCheckABIStability(&src->info, &dst->info);
 }
@@ -28628,6 +28691,7 @@ virDomainIOMMUDefFormat(virBuffer *buf,
     g_auto(virBuffer) childBuf = VIR_BUFFER_INIT_CHILD(buf);
     g_auto(virBuffer) attrBuf = VIR_BUFFER_INITIALIZER;
     g_auto(virBuffer) driverAttrBuf = VIR_BUFFER_INITIALIZER;
+    g_auto(virBuffer) driverChildBuf = VIR_BUFFER_INIT_CHILD(&childBuf);
 
     if (iommu->intremap != VIR_TRISTATE_SWITCH_ABSENT) {
         virBufferAsprintf(&driverAttrBuf, " intremap='%s'",
@@ -28665,8 +28729,17 @@ virDomainIOMMUDefFormat(virBuffer *buf,
         virBufferAsprintf(&driverAttrBuf, " pciBus='%d'",
                           iommu->pci_bus);
     }
+    if (iommu->granule != 0) {
+        if (iommu->granule == -1) {
+            virBufferAddLit(&driverChildBuf, "<granule mode='host'/>\n");
+        } else {
+            virBufferAsprintf(&driverChildBuf,
+                              "<granule size='%d' unit='KiB'/>\n",
+                              iommu->granule);
+        }
+    }
 
-    virXMLFormatElement(&childBuf, "driver", &driverAttrBuf, NULL);
+    virXMLFormatElement(&childBuf, "driver", &driverAttrBuf, &driverChildBuf);
 
     virDomainDeviceInfoFormat(&childBuf, &iommu->info, 0);
 
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index 83d49969d3..72badb32a6 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -3062,6 +3062,7 @@ struct _virDomainIOMMUDef {
     virTristateSwitch dma_translation;
     virTristateSwitch xtsup;
     virTristateSwitch pt;
+    int granule; /* -1 means 'host', 0 unset, page size in KiB otherwise */
 };
 
 typedef enum {
diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
index c83fff132b..1ad614935f 100644
--- a/src/conf/domain_validate.c
+++ b/src/conf/domain_validate.c
@@ -3194,7 +3194,8 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu)
             iommu->aw_bits != 0 ||
             iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT ||
             iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT ||
-            iommu->pt != VIR_TRISTATE_SWITCH_ABSENT) {
+            iommu->pt != VIR_TRISTATE_SWITCH_ABSENT ||
+            iommu->granule != 0) {
             virReportError(VIR_ERR_XML_ERROR,
                            _("iommu model '%1$s' doesn't support some additional attributes"),
                            virDomainIOMMUModelTypeToString(iommu->model));
@@ -3229,7 +3230,8 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu)
             iommu->eim != VIR_TRISTATE_SWITCH_ABSENT ||
             iommu->aw_bits != 0 ||
             iommu->dma_translation != VIR_TRISTATE_SWITCH_ABSENT ||
-            iommu->pci_bus >= 0) {
+            iommu->pci_bus >= 0 ||
+            iommu->granule != 0) {
             virReportError(VIR_ERR_XML_ERROR,
                            _("iommu model '%1$s' doesn't support some additional attributes"),
                            virDomainIOMMUModelTypeToString(iommu->model));
@@ -3240,7 +3242,8 @@ virDomainIOMMUDefValidate(const virDomainIOMMUDef *iommu)
     case VIR_DOMAIN_IOMMU_MODEL_INTEL:
         if (iommu->pt != VIR_TRISTATE_SWITCH_ABSENT ||
             iommu->xtsup != VIR_TRISTATE_SWITCH_ABSENT ||
-            iommu->pci_bus >= 0) {
+            iommu->pci_bus >= 0 ||
+            iommu->granule != 0) {
             virReportError(VIR_ERR_XML_ERROR,
                            _("iommu model '%1$s' doesn't support some additional attributes"),
                            virDomainIOMMUModelTypeToString(iommu->model));
diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
index 114dd3f96f..3d5eb7d4f3 100644
--- a/src/conf/schemas/domaincommon.rng
+++ b/src/conf/schemas/domaincommon.rng
@@ -6329,6 +6329,28 @@
                 <data type="unsignedInt"/>
               </attribute>
             </optional>
+            <optional>
+              <element name="granule">
+                <choice>
+                  <group>
+                    <attribute name="mode">
+                      <value>host</value>
+                    </attribute>
+                  </group>
+                  <group>
+                    <attribute name="size">
+                      <ref name="unsignedInt"/>
+                    </attribute>
+                    <optional>
+                      <attribute name="unit">
+                        <ref name="unit"/>
+                      </attribute>
+                    </optional>
+                  </group>
+                </choice>
+                <empty/>
+              </element>
+            </optional>
           </element>
         </optional>
         <optional>
diff --git a/src/qemu/qemu_validate.c b/src/qemu/qemu_validate.c
index ab8a1938c1..8f85334cf9 100644
--- a/src/qemu/qemu_validate.c
+++ b/src/qemu/qemu_validate.c
@@ -5685,6 +5685,17 @@ qemuValidateDomainDeviceDefIOMMU(const virDomainIOMMUDef *iommu,
         return -1;
     }
 
+    /* QEMU supports only 4KiB, 8KiB, 16KiB and 64KiB granule size */
+    if (iommu->granule > 0 &&
+        !(iommu->granule == 4 ||
+          iommu->granule == 8 ||
+          iommu->granule == 16 ||
+          iommu->granule == 64)) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                       _("iommu: unsupported granule size. Supported values are 4, 8, 16 and 64 KiB"));
+        return -1;
+    }
+
     return 0;
 }
 
diff --git a/tests/qemuxmlconfdata/virtio-iommu-aarch64.aarch64-latest.xml b/tests/qemuxmlconfdata/virtio-iommu-aarch64.aarch64-latest.xml
index 4ae628ab5a..ad2fedaab7 100644
--- a/tests/qemuxmlconfdata/virtio-iommu-aarch64.aarch64-latest.xml
+++ b/tests/qemuxmlconfdata/virtio-iommu-aarch64.aarch64-latest.xml
@@ -29,7 +29,9 @@
     <audio id='1' type='none'/>
     <memballoon model='none'/>
     <iommu model='virtio'>
-      <driver aw_bits='48'/>
+      <driver aw_bits='48'>
+        <granule size='64' unit='KiB'/>
+      </driver>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
     </iommu>
   </devices>
diff --git a/tests/qemuxmlconfdata/virtio-iommu-aarch64.xml b/tests/qemuxmlconfdata/virtio-iommu-aarch64.xml
index 96e5ea05ae..8d5f081b92 100644
--- a/tests/qemuxmlconfdata/virtio-iommu-aarch64.xml
+++ b/tests/qemuxmlconfdata/virtio-iommu-aarch64.xml
@@ -14,7 +14,9 @@
     <controller type='usb' model='none'/>
     <memballoon model='none'/>
     <iommu model='virtio'>
-      <driver aw_bits='48'/>
+      <driver aw_bits='48'>
+        <granule size='64' unit='KiB'/>
+      </driver>
     </iommu>
   </devices>
 </domain>
diff --git a/tests/qemuxmlconfdata/virtio-iommu-x86_64.x86_64-latest.xml b/tests/qemuxmlconfdata/virtio-iommu-x86_64.x86_64-latest.xml
index f458f9a706..56cee393d3 100644
--- a/tests/qemuxmlconfdata/virtio-iommu-x86_64.x86_64-latest.xml
+++ b/tests/qemuxmlconfdata/virtio-iommu-x86_64.x86_64-latest.xml
@@ -31,6 +31,9 @@
     <watchdog model='itco' action='reset'/>
     <memballoon model='none'/>
     <iommu model='virtio'>
+      <driver>
+        <granule mode='host'/>
+      </driver>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
     </iommu>
   </devices>
diff --git a/tests/qemuxmlconfdata/virtio-iommu-x86_64.xml b/tests/qemuxmlconfdata/virtio-iommu-x86_64.xml
index 51c13d2ef6..db0af57f69 100644
--- a/tests/qemuxmlconfdata/virtio-iommu-x86_64.xml
+++ b/tests/qemuxmlconfdata/virtio-iommu-x86_64.xml
@@ -13,6 +13,10 @@
     <emulator>/usr/bin/qemu-system-x86_64</emulator>
     <controller type='usb' model='none'/>
     <memballoon model='none'/>
-    <iommu model='virtio'/>
+    <iommu model='virtio'>
+      <driver>
+        <granule mode='host'/>
+      </driver>
+    </iommu>
   </devices>
 </domain>
-- 
2.52.0