QEMU smmuv3 accel does not support live migration yet, so dirty
tracking for the nesting parent HWPT is not useful.

Also, nested vIOMMU use cases can break on some platforms. For
example, SMMUv3 with HTTU may advertise dirty tracking capability,
but the kernel supports it only for stage-1. Requesting dirty
tracking for a nesting parent HWPT (stage-2) can fail.

Add a vIOMMU flag to explicitly request dirty tracking for the
nesting parent HWPT. For nested cases, dirty tracking is enabled
only when requested by the vIOMMU.

Non-nested cases and Intel vIOMMU keep the existing behavior.

Fixes: fc6dafb98cec ("hw/arm/smmuv3: Implement get_viommu_cap() callback")
Signed-off-by: Shameer Kolothum <skolothumtho@nvidia.com>
---
 include/hw/core/iommu.h       |  2 ++
 include/hw/vfio/vfio-device.h |  1 +
 hw/i386/intel_iommu.c         |  7 +++++--
 hw/vfio/device.c              | 11 +++++++++++
 hw/vfio/iommufd.c             | 11 +++++++++--
 5 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/include/hw/core/iommu.h b/include/hw/core/iommu.h
index 86af315c15..cd59a367ce 100644
--- a/include/hw/core/iommu.h
+++ b/include/hw/core/iommu.h
@@ -21,6 +21,8 @@ enum viommu_flags {
     /* vIOMMU needs nesting parent HWPT to create nested HWPT */
     VIOMMU_FLAG_WANT_NESTING_PARENT = BIT_ULL(0),
     VIOMMU_FLAG_PASID_SUPPORTED = BIT_ULL(1),
+    /* vIOMMU needs dirty tracking on the nesting parent HWPT for nested use */
+    VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING = BIT_ULL(2),
 };
 
 /* Host IOMMU quirks. Extracted from host IOMMU capabilities */
diff --git a/include/hw/vfio/vfio-device.h b/include/hw/vfio/vfio-device.h
index 828a31c006..a95c5bf503 100644
--- a/include/hw/vfio/vfio-device.h
+++ b/include/hw/vfio/vfio-device.h
@@ -268,6 +268,7 @@ void vfio_device_prepare(VFIODevice *vbasedev, VFIOContainer *bcontainer,
 void vfio_device_unprepare(VFIODevice *vbasedev);
 
 bool vfio_device_get_viommu_flags_want_nesting(VFIODevice *vbasedev);
+bool vfio_device_get_viommu_flags_want_nesting_dirty(VFIODevice *vbasedev);
 bool vfio_device_get_host_iommu_quirk_bypass_ro(VFIODevice *vbasedev,
                                                 uint32_t type, void *caps,
                                                 uint32_t size);
diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c
index 4924aa4375..5ec5ed82d7 100644
--- a/hw/i386/intel_iommu.c
+++ b/hw/i386/intel_iommu.c
@@ -4831,9 +4831,12 @@ static void vtd_dev_unset_iommu_device(PCIBus *bus, void *opaque, int devfn)
 static uint64_t vtd_get_viommu_flags(void *opaque)
 {
     IntelIOMMUState *s = opaque;
-    uint64_t flags;
+    uint64_t flags = 0;
 
-    flags = s->fsts ? VIOMMU_FLAG_WANT_NESTING_PARENT : 0;
+    if (s->fsts) {
+        flags = VIOMMU_FLAG_WANT_NESTING_PARENT |
+                VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING;
+    }
 
     return flags;
 }
diff --git a/hw/vfio/device.c b/hw/vfio/device.c
index 973fc35b59..8f7ae919a5 100644
--- a/hw/vfio/device.c
+++ b/hw/vfio/device.c
@@ -522,6 +522,17 @@ void vfio_device_unprepare(VFIODevice *vbasedev)
     vbasedev->bcontainer = NULL;
 }
 
+bool vfio_device_get_viommu_flags_want_nesting_dirty(VFIODevice *vbasedev)
+{
+    VFIOPCIDevice *vdev = vfio_pci_from_vfio_device(vbasedev);
+
+    if (vdev) {
+        return !!(pci_device_get_viommu_flags(PCI_DEVICE(vdev)) &
+                  VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING);
+    }
+    return false;
+}
+
 bool vfio_device_get_viommu_flags_want_nesting(VFIODevice *vbasedev)
 {
     VFIOPCIDevice *vdev = vfio_pci_from_vfio_device(vbasedev);
diff --git a/hw/vfio/iommufd.c b/hw/vfio/iommufd.c
index 131612eb83..231635cdc7 100644
--- a/hw/vfio/iommufd.c
+++ b/hw/vfio/iommufd.c
@@ -349,6 +349,7 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,
     ERRP_GUARD();
     IOMMUFDBackend *iommufd = vbasedev->iommufd;
     VFIOContainer *bcontainer = VFIO_IOMMU(container);
+    bool viommu_nesting, viommu_nesting_dirty;
     uint32_t type, flags = 0;
     uint64_t hw_caps;
     VendorCaps caps;
@@ -402,8 +403,14 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,
         return false;
     }
 
+    viommu_nesting = vfio_device_get_viommu_flags_want_nesting(vbasedev);
+    viommu_nesting_dirty =
+        vfio_device_get_viommu_flags_want_nesting_dirty(vbasedev);
+
     if (hw_caps & IOMMU_HW_CAP_DIRTY_TRACKING) {
-        flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
+        if (!viommu_nesting || viommu_nesting_dirty) {
+            flags |= IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
+        }
     }
 
     /*
@@ -411,7 +418,7 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,
      * force to create it so that it could be reused by vIOMMU to create
      * nested HWPT.
      */
-    if (vfio_device_get_viommu_flags_want_nesting(vbasedev)) {
+    if (viommu_nesting) {
         flags |= IOMMU_HWPT_ALLOC_NEST_PARENT;
 
         if (vfio_device_get_host_iommu_quirk_bypass_ro(vbasedev, type,
-- 
2.43.0