When using the mapped-ram migration capability, direct IO is
enabled by setting the "direct-io" migration parameter to
"true" and passing QEMU an additional fd with O_DIRECT set.

Signed-off-by: Jim Fehlig <jfehlig@suse.com>
---
 src/qemu/qemu_migration.c | 11 ++++++-----
 src/qemu/qemu_process.c   | 30 ++++++++++++++++++++++++++----
 src/qemu/qemu_process.h   |  6 ++++--
 3 files changed, 36 insertions(+), 11 deletions(-)

diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
index daa42bcfe4..56199c616b 100644
--- a/src/qemu/qemu_migration.c
+++ b/src/qemu/qemu_migration.c
@@ -2948,7 +2948,8 @@ qemuMigrationDstPrepareCleanup(virQEMUDriver *driver,
 }
 
 static qemuProcessIncomingDef *
-qemuMigrationDstPrepare(virDomainObj *vm,
+qemuMigrationDstPrepare(virQEMUDriver *driver,
+                        virDomainObj *vm,
                         bool tunnel,
                         const char *protocol,
                         const char *listenAddress,
@@ -3008,8 +3009,8 @@ qemuMigrationDstPrepare(virDomainObj *vm,
         migrateFrom = g_strdup_printf(incFormat, protocol, listenAddress, port);
     }
 
-    return qemuProcessIncomingDefNew(vm, listenAddress,
-                                     migrateFrom, fd, NULL, NULL);
+    return qemuProcessIncomingDefNew(driver, vm, listenAddress,
+                                     migrateFrom, fd, NULL, NULL, NULL);
 }
 
 
@@ -3151,7 +3152,7 @@ qemuMigrationDstPrepareActive(virQEMUDriver *driver,
         goto error;
     stopProcess = true;
 
-    if (!(incoming = qemuMigrationDstPrepare(vm, tunnel, protocol,
+    if (!(incoming = qemuMigrationDstPrepare(driver, vm, tunnel, protocol,
                                              listenAddress, port,
                                              &dataFD[0])))
         goto error;
@@ -3522,7 +3523,7 @@ qemuMigrationDstPrepareResume(virQEMUDriver *driver,
 
     priv->origname = g_strdup(origname);
 
-    if (!(incoming = qemuMigrationDstPrepare(vm, false, protocol,
+    if (!(incoming = qemuMigrationDstPrepare(driver, vm, false, protocol,
                                              listenAddress, port, NULL)))
         goto cleanup;
 
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index b02cd84aff..e10894d0f9 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -4694,13 +4694,16 @@ qemuProcessIncomingDefFree(qemuProcessIncomingDef *inc)
  * qemuProcessIncomingDefFree will NOT close it.
  */
 qemuProcessIncomingDef *
-qemuProcessIncomingDefNew(virDomainObj *vm,
+qemuProcessIncomingDefNew(virQEMUDriver *driver,
+                          virDomainObj *vm,
                           const char *listenAddress,
                           const char *migrateFrom,
                           int *fd,
                           const char *path,
-                          virQEMUSaveData *data)
+                          virQEMUSaveData *data,
+                          qemuMigrationParams *migParams)
 {
+    g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
     qemuDomainObjPrivate *priv = vm->privateData;
     qemuProcessIncomingDef *inc = NULL;
 
@@ -4715,10 +4718,28 @@ qemuProcessIncomingDefNew(virDomainObj *vm,
         size_t offset = sizeof(virQEMUSaveHeader) + data->header.data_len;
 
         if (data->header.features & QEMU_SAVE_FEATURE_MAPPED_RAM) {
+            bool directio = false;
             unsigned int fdsetId;
 
             inc->fdPassMigrate = qemuFDPassNew("migrate", priv);
-            qemuFDPassAddFD(inc->fdPassMigrate, fd, "-buffered-fd");
+            /* When using directio with mapped-ram, qemu needs an fd without
+             * O_DIRECT set for reading small bits of unaligned state. */
+            if (qemuMigrationParamsGetBool(migParams,
+                                           QEMU_MIGRATION_PARAM_DIRECT_IO,
+                                           &directio) < 0)
+                goto error;
+
+            if (directio) {
+                VIR_AUTOCLOSE bufferedFd = -1;
+
+                if ((bufferedFd = qemuDomainOpenFile(cfg, NULL, path, O_RDONLY, NULL)) < 0)
+                    goto error;
+
+                qemuFDPassAddFD(inc->fdPassMigrate, &bufferedFd, "-buffered-fd");
+                qemuFDPassAddFD(inc->fdPassMigrate, fd, "-directio-fd");
+            } else {
+                qemuFDPassAddFD(inc->fdPassMigrate, fd, "-buffered-fd");
+            }
             qemuFDPassGetId(inc->fdPassMigrate, &fdsetId);
             inc->uri = g_strdup_printf("file:/dev/fdset/%u,offset=%#lx", fdsetId, offset);
         } else {
@@ -8377,7 +8398,8 @@ qemuProcessStartWithMemoryState(virConnectPtr conn,
     int rc = 0;
     int ret = -1;
 
-    incoming = qemuProcessIncomingDefNew(vm, NULL, "stdio", fd, path, data);
+    incoming = qemuProcessIncomingDefNew(driver, vm, NULL, "stdio",
+                                         fd, path, data, migParams);
     if (!incoming)
         return -1;
 
diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h
index 93699da8bd..0e6aedf75d 100644
--- a/src/qemu/qemu_process.h
+++ b/src/qemu/qemu_process.h
@@ -58,12 +58,14 @@ struct _qemuProcessIncomingDef {
     const char *path; /* path associated with fd */
 };
 
-qemuProcessIncomingDef *qemuProcessIncomingDefNew(virDomainObj *vm,
+qemuProcessIncomingDef *qemuProcessIncomingDefNew(virQEMUDriver *driver,
+                                                  virDomainObj *vm,
                                                   const char *listenAddress,
                                                   const char *migrateFrom,
                                                   int *fd,
                                                   const char *path,
-                                                  virQEMUSaveData *data);
+                                                  virQEMUSaveData *data,
+                                                  qemuMigrationParams *migParams);
 
 void qemuProcessIncomingDefFree(qemuProcessIncomingDef *inc);
 
-- 
2.35.3