We support using O_DIRECT with multifd by isolating the main migration
channel which does unaligned IO from the multifd channels that do
aligned IO. When multifd is not enabled, we can still use O_DIRECT, if
we judiciously enable/disable the flag to avoid the unaligned IO.

Add helpers to enable/disable direct-io around the aligned parts.

Signed-off-by: Fabiano Rosas <farosas@suse.de>
---
 migration/migration.c | 31 +++++++++++++++++++++++++++++++
 migration/migration.h |  2 ++
 migration/qemu-file.c | 29 +++++++++++++++++++++++++++++
 migration/qemu-file.h |  2 +-
 4 files changed, 63 insertions(+), 1 deletion(-)

diff --git a/migration/migration.c b/migration/migration.c
index e03c80b3aa..be128f95da 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -420,6 +420,37 @@ static void migrate_generate_event(int new_state)
     }
 }
 
+bool migration_direct_io_start(QEMUFile *file, Error **errp)
+{
+    if (!migrate_direct_io() || migrate_multifd()) {
+        return true;
+    }
+
+    /* flush any potentially unaligned IO before setting O_DIRECT */
+    qemu_fflush(file);
+
+    if (!qemu_file_set_direct_io(file, true)) {
+        error_setg(errp, "Failed to enable direct-io");
+        return false;
+    }
+
+    return true;
+}
+
+bool migration_direct_io_finish(QEMUFile *file, Error **errp)
+{
+    if (!migrate_direct_io() || migrate_multifd()) {
+        return true;
+    }
+
+    if (!qemu_file_set_direct_io(file, false)) {
+        error_setg(errp, "Failed to disable direct-io");
+        return false;
+    }
+
+    return true;
+}
+
 /*
  * Send a message on the return channel back to the source
  * of the migration.
diff --git a/migration/migration.h b/migration/migration.h
index 6af01362d4..4d808563b5 100644
--- a/migration/migration.h
+++ b/migration/migration.h
@@ -536,4 +536,6 @@ int migration_rp_wait(MigrationState *s);
  */
 void migration_rp_kick(MigrationState *s);
 
+bool migration_direct_io_start(QEMUFile *file, Error **errp);
+bool migration_direct_io_finish(QEMUFile *file, Error **errp);
 #endif
diff --git a/migration/qemu-file.c b/migration/qemu-file.c
index 9ccbbb0099..4796be918f 100644
--- a/migration/qemu-file.c
+++ b/migration/qemu-file.c
@@ -52,6 +52,11 @@ struct QEMUFile {
 
     int last_error;
     Error *last_error_obj;
+    /*
+     * Whether O_DIRECT is in effect. Used to catch code attempting
+     * unaligned IO.
+     */
+    bool dio;
 };
 
 /*
@@ -281,6 +286,9 @@ int qemu_fflush(QEMUFile *f)
     }
     if (f->iovcnt > 0) {
         Error *local_error = NULL;
+
+        assert(!f->dio);
+
         if (qio_channel_writev_all(f->ioc,
                                    f->iov, f->iovcnt,
                                    &local_error) < 0) {
@@ -429,6 +437,8 @@ void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, size_t size,
         return;
     }
 
+    assert(!f->dio);
+
     add_to_iovec(f, buf, size, may_free);
 }
 
@@ -440,6 +450,8 @@ void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, size_t size)
         return;
     }
 
+    assert(!f->dio);
+
     while (size > 0) {
         l = IO_BUF_SIZE - f->buf_index;
         if (l > size) {
@@ -558,6 +570,8 @@ off_t qemu_get_offset(QEMUFile *f)
 
 void qemu_put_byte(QEMUFile *f, int v)
 {
+    assert(!f->dio);
+
     if (f->last_error) {
         return;
     }
@@ -865,3 +879,18 @@ int qemu_file_get_to_fd(QEMUFile *f, int fd, size_t size)
 
     return 0;
 }
+
+bool qemu_file_set_direct_io(QEMUFile *f, bool enabled)
+{
+    Error *local_err = NULL;
+
+    qio_channel_file_set_direct_io(f->ioc, enabled, &local_err);
+    if (local_err) {
+        qemu_file_set_error_obj(f, -EINVAL, local_err);
+        return false;
+    }
+
+    f->dio = enabled;
+
+    return true;
+}
diff --git a/migration/qemu-file.h b/migration/qemu-file.h
index 11c2120edd..118853b21c 100644
--- a/migration/qemu-file.h
+++ b/migration/qemu-file.h
@@ -79,5 +79,5 @@ size_t qemu_get_buffer_at(QEMUFile *f, const uint8_t *buf, size_t buflen,
                           off_t pos);
 
 QIOChannel *qemu_file_get_ioc(QEMUFile *file);
-
+bool qemu_file_set_direct_io(QEMUFile *f, bool enabled);
 #endif
-- 
2.35.3