During the multifd send phase, the multifd packet header is included
as the first element of the iovec, except in the special case of a
socket migration with zero copy enabled. In that case the packet
header is sent separately. To avoid the first position of the iovec
being empty, we play with the p->iovs_num value to make sure
send_prepare() fills the iovec from the correct position depending on
whether the header is at the first position or not.

This is confusing at first sight and could be made simpler if we
always put the header at the first position in the iovec and advance
the iovec pointer when sending with zero copy. That way we can keep
that logic restricted to the socket code.

Signed-off-by: Fabiano Rosas <farosas@suse.de>
---
 migration/multifd-zlib.c |  3 ---
 migration/multifd-zstd.c |  3 ---
 migration/multifd.c      | 11 +++++------
 migration/socket.c       | 19 +++++++++++--------
 4 files changed, 16 insertions(+), 20 deletions(-)

diff --git a/migration/multifd-zlib.c b/migration/multifd-zlib.c
index 0859efa60f..af6ef96670 100644
--- a/migration/multifd-zlib.c
+++ b/migration/multifd-zlib.c
@@ -176,9 +176,6 @@ static int zlib_send_prepare(MultiFDSendParams *p, Error **errp)
 
 static int zlib_send(MultiFDSendParams *p, Error **errp)
 {
-    p->iov[0].iov_len = p->packet_len;
-    p->iov[0].iov_base = p->packet;
-
     return qio_channel_writev_full_all(p->c, p->iov, p->iovs_num, NULL,
                                        0, p->write_flags, errp);
 }
diff --git a/migration/multifd-zstd.c b/migration/multifd-zstd.c
index ca0fc79fdd..6d533eaa54 100644
--- a/migration/multifd-zstd.c
+++ b/migration/multifd-zstd.c
@@ -165,9 +165,6 @@ static int zstd_send_prepare(MultiFDSendParams *p, Error **errp)
 
 static int zstd_send(MultiFDSendParams *p, Error **errp)
 {
-    p->iov[0].iov_len = p->packet_len;
-    p->iov[0].iov_base = p->packet;
-
     return qio_channel_writev_full_all(p->c, p->iov, p->iovs_num, NULL,
                                        0, p->write_flags, errp);
 }
diff --git a/migration/multifd.c b/migration/multifd.c
index 9f509699c2..358a4dbf8f 100644
--- a/migration/multifd.c
+++ b/migration/multifd.c
@@ -171,6 +171,9 @@ static void multifd_send_fill_packet(MultiFDSendParams *p)
 
         packet->offset[i] = cpu_to_be64(temp);
     }
+
+    p->iov[0].iov_len = p->packet_len;
+    p->iov[0].iov_base = p->packet;
 }
 
 static int multifd_recv_unfill_packet(MultiFDRecvParams *p, Error **errp)
@@ -546,7 +549,6 @@ static void *multifd_send_thread(void *opaque)
     MigrationThread *thread = NULL;
     Error *local_err = NULL;
     int ret = 0;
-    bool use_zero_copy_send = migrate_zero_copy_send();
 
     thread = migration_threads_add(p->name, qemu_get_thread_id());
 
@@ -574,11 +576,8 @@ static void *multifd_send_thread(void *opaque)
             uint32_t flags;
             p->normal_num = 0;
 
-            if (use_zero_copy_send) {
-                p->iovs_num = 0;
-            } else {
-                p->iovs_num = 1;
-            }
+            /* The header is always added to the vector */
+            p->iovs_num = 1;
 
             for (int i = 0; i < p->pages->num; i++) {
                 p->normal[p->normal_num] = p->pages->offset[i];
diff --git a/migration/socket.c b/migration/socket.c
index 608f30975e..af22ff7cc4 100644
--- a/migration/socket.c
+++ b/migration/socket.c
@@ -16,6 +16,7 @@
 
 #include "qemu/osdep.h"
 #include "qemu/cutils.h"
+#include "qemu/iov.h"
 #include "exec/ramblock.h"
 #include "qemu/error-report.h"
 #include "qapi/error.h"
@@ -230,22 +231,24 @@ static int multifd_socket_send_prepare(MultiFDSendParams *p, Error **errp)
 
 static int multifd_socket_send(MultiFDSendParams *p, Error **errp)
 {
+    struct iovec *iov = p->iov;
     int ret;
 
     if (migrate_zero_copy_send()) {
-        /* Send header first, without zerocopy */
-        ret = qio_channel_write_all(p->c, (void *)p->packet, p->packet_len,
-                                    errp);
+        /*
+         * The first iovec element is always the header. Sent it first
+         * without zerocopy.
+         */
+        ret = qio_channel_writev_all(p->c, iov, 1, errp);
         if (ret) {
             return ret;
         }
-    } else {
-        /* Send header using the same writev call */
-        p->iov[0].iov_len = p->packet_len;
-        p->iov[0].iov_base = p->packet;
+
+        /* header sent, discard it */
+        iov_discard_front(&iov, &p->iovs_num, iov[0].iov_len);
     }
 
-    return qio_channel_writev_full_all(p->c, p->iov, p->iovs_num, NULL,
+    return qio_channel_writev_full_all(p->c, iov, p->iovs_num, NULL,
                                        0, p->write_flags, errp);
 }
 
-- 
2.35.3