qemu: Implement support for iommufd

[PATCH v2 3/5] qemu: open iommufd FD from libvirt backend

Posted by Nathan Chen via Devel 2 weeks, 6 days ago

Open iommufd FD from libvirt backend without exposing
these FDs to XML users, i.e. one per domain for
/dev/iommu, and pass the FD to qemu command line.

Signed-off-by: Nathan Chen <nathanc@nvidia.com>
---
 src/qemu/qemu_command.c |  8 ++++--
 src/qemu/qemu_domain.c  |  1 +
 src/qemu/qemu_domain.h  |  2 ++
 src/qemu/qemu_process.c | 56 +++++++++++++++++++++++++++++++++++++++++
 4 files changed, 65 insertions(+), 2 deletions(-)

diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index 9b08f66175..99c310cf31 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -5229,12 +5229,14 @@ qemuBuildAcpiNodesetProps(virCommand *cmd,
 static int
 qemuBuildHostdevCommandLine(virCommand *cmd,
                             const virDomainDef *def,
-                            virQEMUCaps *qemuCaps)
+                            virQEMUCaps *qemuCaps,
+                            virDomainObj *vm)
 {
     size_t i;
     g_autoptr(virJSONValue) props = NULL;
     int iommufd = 0;
     const char * iommufdId = "iommufd0";
+    qemuDomainObjPrivate *priv = vm->privateData;
 
     for (i = 0; i < def->nhostdevs; i++) {
         virDomainHostdevDef *hostdev = def->hostdevs[i];
@@ -5265,8 +5267,10 @@ qemuBuildHostdevCommandLine(virCommand *cmd,
 
             if (subsys->u.pci.driver.iommufd == VIR_TRISTATE_BOOL_YES && iommufd == 0) {
                 iommufd = 1;
+                virCommandPassFD(cmd, priv->iommufd, VIR_COMMAND_PASS_FD_CLOSE_PARENT);
                 if (qemuMonitorCreateObjectProps(&props, "iommufd",
                                                  iommufdId,
+                                                 "S:fd", g_strdup_printf("%d", priv->iommufd),
                                                  NULL) < 0)
                     return -1;
 
@@ -10967,7 +10971,7 @@ qemuBuildCommandLine(virDomainObj *vm,
     if (qemuBuildRedirdevCommandLine(cmd, def, qemuCaps) < 0)
         return NULL;
 
-    if (qemuBuildHostdevCommandLine(cmd, def, qemuCaps) < 0)
+    if (qemuBuildHostdevCommandLine(cmd, def, qemuCaps, vm) < 0)
         return NULL;
 
     if (migrateURI)
diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index 7601bdbb2b..d569dd5ad9 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -2042,6 +2042,7 @@ qemuDomainObjPrivateAlloc(void *opaque)
     priv->blockjobs = virHashNew(virObjectUnref);
     priv->fds = virHashNew(g_object_unref);
 
+    priv->iommufd = -1;
     priv->pidMonitored = -1;
 
     /* agent commands block by default, user can choose different behavior */
diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h
index 4736f1ede5..e55ba1c968 100644
--- a/src/qemu/qemu_domain.h
+++ b/src/qemu/qemu_domain.h
@@ -264,6 +264,8 @@ struct _qemuDomainObjPrivate {
     /* named file descriptor groups associated with the VM */
     GHashTable *fds;
 
+    int iommufd;
+
     char *memoryBackingDir;
 };
 
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index bf245ee8af..83b8a586a1 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -10272,6 +10272,38 @@ qemuProcessHandleNbdkitExit(qemuNbdkitProcess *nbdkit,
     virObjectUnlock(vm);
 }
 
+/**
+ * qemuProcessOpenIommuFd:
+ * @vm: domain object
+ * @iommuFd: returned file descriptor
+ *
+ * Opens /dev/iommu file descriptor for the VM.
+ *
+ * Returns: 0 on success, -1 on failure
+ */
+static int
+qemuProcessOpenIommuFd(virDomainObj *vm, int *iommuFd)
+{
+    int fd = -1;
+
+    VIR_DEBUG("Opening IOMMU FD for domain %s", vm->def->name);
+
+    if ((fd = open("/dev/iommu", O_RDWR | O_CLOEXEC)) < 0) {
+        if (errno == ENOENT) {
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                           _("IOMMU FD support requires /dev/iommu device"));
+        } else {
+            virReportSystemError(errno, "%s",
+                                 _("cannot open /dev/iommu"));
+        }
+        return -1;
+    }
+
+    *iommuFd = fd;
+    VIR_DEBUG("Opened IOMMU FD %d for domain %s", fd, vm->def->name);
+    return 0;
+}
+
 /**
  * qemuProcessOpenVfioDeviceFd:
  * @hostdev: host device definition
@@ -10329,6 +10361,8 @@ qemuProcessOpenVfioDeviceFd(virDomainHostdevDef *hostdev,
 int
 qemuProcessOpenVfioFds(virDomainObj *vm)
 {
+    qemuDomainObjPrivate *priv = vm->privateData;
+    bool needsIommuFd = false;
     size_t i;
 
     /* Check if we have any hostdevs that need VFIO FDs */
@@ -10342,6 +10376,8 @@ qemuProcessOpenVfioFds(virDomainObj *vm)
             if (hostdev->source.subsys.u.pci.driver.name == VIR_DEVICE_HOSTDEV_PCI_DRIVER_NAME_VFIO &&
                 hostdev->source.subsys.u.pci.driver.iommufd == VIR_TRISTATE_BOOL_YES) {
 
+                needsIommuFd = true;
+
                 if (!hostdev->privateData) {
                     if (!(hostdev->privateData = qemuDomainHostdevPrivateNew()))
                         goto error;
@@ -10363,6 +10399,18 @@ qemuProcessOpenVfioFds(virDomainObj *vm)
         }
     }
 
+    /* Open IOMMU FD if needed */
+    if (needsIommuFd) {
+        int iommuFd = -1;
+
+        if (qemuProcessOpenIommuFd(vm, &iommuFd) < 0)
+            goto error;
+
+        priv->iommufd = iommuFd;
+
+        VIR_DEBUG("Stored IOMMU FD %d", priv->iommufd);
+    }
+
     return 0;
 
  error:
@@ -10379,6 +10427,7 @@ qemuProcessOpenVfioFds(virDomainObj *vm)
 void
 qemuProcessCloseVfioFds(virDomainObj *vm)
 {
+    qemuDomainObjPrivate *priv = vm->privateData;
     size_t i;
 
     /* Close all VFIO device FDs */
@@ -10396,4 +10445,11 @@ qemuProcessCloseVfioFds(virDomainObj *vm)
             VIR_FORCE_CLOSE(hostdevPriv->vfioDeviceFd);
         }
     }
+
+    /* Close IOMMU FD */
+    if (priv->iommufd >= 0) {
+        VIR_DEBUG("Closing IOMMU FD %d", priv->iommufd);
+        VIR_FORCE_CLOSE(priv->iommufd);
+        priv->iommufd = -1;
+    }
 }
-- 
2.43.0

Re: [PATCH v2 3/5] qemu: open iommufd FD from libvirt backend

Posted by Ján Tomko via Devel 3 days, 18 hours ago

On a Friday in 2025, Nathan Chen via Devel wrote:
>Open iommufd FD from libvirt backend without exposing
>these FDs to XML users, i.e. one per domain for
>/dev/iommu, and pass the FD to qemu command line.
>
>Signed-off-by: Nathan Chen <nathanc@nvidia.com>
>---
> src/qemu/qemu_command.c |  8 ++++--
> src/qemu/qemu_domain.c  |  1 +
> src/qemu/qemu_domain.h  |  2 ++
> src/qemu/qemu_process.c | 56 +++++++++++++++++++++++++++++++++++++++++
> 4 files changed, 65 insertions(+), 2 deletions(-)
>
>diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
>index 9b08f66175..99c310cf31 100644
>--- a/src/qemu/qemu_command.c
>+++ b/src/qemu/qemu_command.c
>@@ -5229,12 +5229,14 @@ qemuBuildAcpiNodesetProps(virCommand *cmd,
> static int
> qemuBuildHostdevCommandLine(virCommand *cmd,
>                             const virDomainDef *def,
>-                            virQEMUCaps *qemuCaps)
>+                            virQEMUCaps *qemuCaps,
>+                            virDomainObj *vm)
> {
>     size_t i;
>     g_autoptr(virJSONValue) props = NULL;
>     int iommufd = 0;
>     const char * iommufdId = "iommufd0";
>+    qemuDomainObjPrivate *priv = vm->privateData;
>
>     for (i = 0; i < def->nhostdevs; i++) {
>         virDomainHostdevDef *hostdev = def->hostdevs[i];
>@@ -5265,8 +5267,10 @@ qemuBuildHostdevCommandLine(virCommand *cmd,
>
>             if (subsys->u.pci.driver.iommufd == VIR_TRISTATE_BOOL_YES && iommufd == 0) {
>                 iommufd = 1;
>+                virCommandPassFD(cmd, priv->iommufd, VIR_COMMAND_PASS_FD_CLOSE_PARENT);
>                 if (qemuMonitorCreateObjectProps(&props, "iommufd",
>                                                  iommufdId,
>+                                                 "S:fd", g_strdup_printf("%d", priv->iommufd),

Same comment about g_strdup_printf as with the other fd: formatting.
Also, this change would be in a different function now.

>                                                  NULL) < 0)
>                     return -1;
>
>@@ -10967,7 +10971,7 @@ qemuBuildCommandLine(virDomainObj *vm,
>     if (qemuBuildRedirdevCommandLine(cmd, def, qemuCaps) < 0)
>         return NULL;
>
>-    if (qemuBuildHostdevCommandLine(cmd, def, qemuCaps) < 0)
>+    if (qemuBuildHostdevCommandLine(cmd, def, qemuCaps, vm) < 0)
>         return NULL;
>
>     if (migrateURI)
>diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
>index 7601bdbb2b..d569dd5ad9 100644
>--- a/src/qemu/qemu_domain.c
>+++ b/src/qemu/qemu_domain.c
>@@ -2042,6 +2042,7 @@ qemuDomainObjPrivateAlloc(void *opaque)
>     priv->blockjobs = virHashNew(virObjectUnref);
>     priv->fds = virHashNew(g_object_unref);
>
>+    priv->iommufd = -1;
>     priv->pidMonitored = -1;
>
>     /* agent commands block by default, user can choose different behavior */
>diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h
>index 4736f1ede5..e55ba1c968 100644
>--- a/src/qemu/qemu_domain.h
>+++ b/src/qemu/qemu_domain.h
>@@ -264,6 +264,8 @@ struct _qemuDomainObjPrivate {
>     /* named file descriptor groups associated with the VM */
>     GHashTable *fds;
>
>+    int iommufd;
>+
>     char *memoryBackingDir;
> };
>
>diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
>index bf245ee8af..83b8a586a1 100644
>--- a/src/qemu/qemu_process.c
>+++ b/src/qemu/qemu_process.c
>@@ -10272,6 +10272,38 @@ qemuProcessHandleNbdkitExit(qemuNbdkitProcess *nbdkit,
>     virObjectUnlock(vm);
> }
>
>+/**
>+ * qemuProcessOpenIommuFd:
>+ * @vm: domain object
>+ * @iommuFd: returned file descriptor
>+ *
>+ * Opens /dev/iommu file descriptor for the VM.
>+ *
>+ * Returns: 0 on success, -1 on failure
>+ */
>+static int
>+qemuProcessOpenIommuFd(virDomainObj *vm, int *iommuFd)
>+{
>+    int fd = -1;
>+
>+    VIR_DEBUG("Opening IOMMU FD for domain %s", vm->def->name);
>+
>+    if ((fd = open("/dev/iommu", O_RDWR | O_CLOEXEC)) < 0) {
>+        if (errno == ENOENT) {
>+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
>+                           _("IOMMU FD support requires /dev/iommu device"));
>+        } else {
>+            virReportSystemError(errno, "%s",
>+                                 _("cannot open /dev/iommu"));
>+        }
>+        return -1;
>+    }
>+
>+    *iommuFd = fd;
>+    VIR_DEBUG("Opened IOMMU FD %d for domain %s", fd, vm->def->name);
>+    return 0;
>+}
>+
> /**
>  * qemuProcessOpenVfioDeviceFd:
>  * @hostdev: host device definition
>@@ -10329,6 +10361,8 @@ qemuProcessOpenVfioDeviceFd(virDomainHostdevDef *hostdev,
> int
> qemuProcessOpenVfioFds(virDomainObj *vm)
> {
>+    qemuDomainObjPrivate *priv = vm->privateData;
>+    bool needsIommuFd = false;
>     size_t i;
>
>     /* Check if we have any hostdevs that need VFIO FDs */
>@@ -10342,6 +10376,8 @@ qemuProcessOpenVfioFds(virDomainObj *vm)
>             if (hostdev->source.subsys.u.pci.driver.name == VIR_DEVICE_HOSTDEV_PCI_DRIVER_NAME_VFIO &&
>                 hostdev->source.subsys.u.pci.driver.iommufd == VIR_TRISTATE_BOOL_YES) {
>
>+                needsIommuFd = true;
>+
>                 if (!hostdev->privateData) {
>                     if (!(hostdev->privateData = qemuDomainHostdevPrivateNew()))
>                         goto error;
>@@ -10363,6 +10399,18 @@ qemuProcessOpenVfioFds(virDomainObj *vm)
>         }
>     }
>
>+    /* Open IOMMU FD if needed */
>+    if (needsIommuFd) {
>+        int iommuFd = -1;
>+

No need for the intermediary variable,

>+        if (qemuProcessOpenIommuFd(vm, &iommuFd) < 0)
>+            goto error;
>+
>+        priv->iommufd = iommuFd;
>+
>+        VIR_DEBUG("Stored IOMMU FD %d", priv->iommufd);
>+    }
>+
>     return 0;
>
>  error:

Everything below can be dropped
Jano

>@@ -10379,6 +10427,7 @@ qemuProcessOpenVfioFds(virDomainObj *vm)
> void
> qemuProcessCloseVfioFds(virDomainObj *vm)
> {
>+    qemuDomainObjPrivate *priv = vm->privateData;
>     size_t i;
>
>     /* Close all VFIO device FDs */
>@@ -10396,4 +10445,11 @@ qemuProcessCloseVfioFds(virDomainObj *vm)
>             VIR_FORCE_CLOSE(hostdevPriv->vfioDeviceFd);
>         }
>     }
>+
>+    /* Close IOMMU FD */
>+    if (priv->iommufd >= 0) {
>+        VIR_DEBUG("Closing IOMMU FD %d", priv->iommufd);
>+        VIR_FORCE_CLOSE(priv->iommufd);
>+        priv->iommufd = -1;
>+    }
> }
>-- 
>2.43.0
>

[PATCH v2 1/5] qemu: Implement support for associating iommufd to hostdev
[PATCH v2 2/5] qemu: open VFIO FDs from libvirt backend
[PATCH v2 3/5] qemu: open iommufd FD from libvirt backend
[PATCH v2 4/5] qemu: Update Cgroup, namespace, and seclabel for iommufd
[PATCH v2 5/5] tests: qemuxmlconfdata: provide iommufd sample XML and CLI args