From: Peter Xu <peterx@redhat.com>

Introduce a helper multifd_send_prepare_header() to setup the header packet
for multifd sender.

It's fine to setup the IOV[0] _before_ send_prepare() because the packet
buffer is already ready, even if the content is to be filled in.

With this helper, we can already slightly clean up the zero copy path.

Note that I explicitly put it into multifd.h, because I want it inlined
directly into multifd*.c where necessary later.

Signed-off-by: Peter Xu <peterx@redhat.com>
---
 migration/multifd.h |  8 ++++++++
 migration/multifd.c | 16 ++++++++--------
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/migration/multifd.h b/migration/multifd.h
index 2e4ad0dc56..4ec005f53f 100644
--- a/migration/multifd.h
+++ b/migration/multifd.h
@@ -209,5 +209,13 @@ typedef struct {
 
 void multifd_register_ops(int method, MultiFDMethods *ops);
 
+static inline void multifd_send_prepare_header(MultiFDSendParams *p)
+{
+    p->iov[0].iov_len = p->packet_len;
+    p->iov[0].iov_base = p->packet;
+    p->iovs_num++;
+}
+
+
 #endif
 
diff --git a/migration/multifd.c b/migration/multifd.c
index 8d4b80f365..1b0035787e 100644
--- a/migration/multifd.c
+++ b/migration/multifd.c
@@ -707,10 +707,14 @@ static void *multifd_send_thread(void *opaque)
         if (p->pending_job) {
             MultiFDPages_t *pages = p->pages;
 
-            if (use_zero_copy_send) {
-                p->iovs_num = 0;
-            } else {
-                p->iovs_num = 1;
+            p->iovs_num = 0;
+
+            if (!use_zero_copy_send) {
+                /*
+                 * Only !zero_copy needs the header in IOV; zerocopy will
+                 * send it separately.
+                 */
+                multifd_send_prepare_header(p);
             }
 
             assert(pages->num);
@@ -730,10 +734,6 @@ static void *multifd_send_thread(void *opaque)
                 if (ret != 0) {
                     break;
                 }
-            } else {
-                /* Send header using the same writev call */
-                p->iov[0].iov_len = p->packet_len;
-                p->iov[0].iov_base = p->packet;
             }
 
             ret = qio_channel_writev_full_all(p->c, p->iov, p->iovs_num, NULL,
-- 
2.43.0

peterx@redhat.com writes:

> From: Peter Xu <peterx@redhat.com>
>
> Introduce a helper multifd_send_prepare_header() to setup the header packet
> for multifd sender.
>
> It's fine to setup the IOV[0] _before_ send_prepare() because the packet
> buffer is already ready, even if the content is to be filled in.
>
> With this helper, we can already slightly clean up the zero copy path.
>
> Note that I explicitly put it into multifd.h, because I want it inlined
> directly into multifd*.c where necessary later.
>
> Signed-off-by: Peter Xu <peterx@redhat.com>

Reviewed-by: Fabiano Rosas <farosas@suse.de>

nit below:

> ---
>  migration/multifd.h |  8 ++++++++
>  migration/multifd.c | 16 ++++++++--------
>  2 files changed, 16 insertions(+), 8 deletions(-)
>
> diff --git a/migration/multifd.h b/migration/multifd.h
> index 2e4ad0dc56..4ec005f53f 100644
> --- a/migration/multifd.h
> +++ b/migration/multifd.h
> @@ -209,5 +209,13 @@ typedef struct {
>  
>  void multifd_register_ops(int method, MultiFDMethods *ops);
>  
> +static inline void multifd_send_prepare_header(MultiFDSendParams *p)
> +{
> +    p->iov[0].iov_len = p->packet_len;
> +    p->iov[0].iov_base = p->packet;
> +    p->iovs_num++;
> +}
> +
> +
>  #endif
>  
> diff --git a/migration/multifd.c b/migration/multifd.c
> index 8d4b80f365..1b0035787e 100644
> --- a/migration/multifd.c
> +++ b/migration/multifd.c
> @@ -707,10 +707,14 @@ static void *multifd_send_thread(void *opaque)
>          if (p->pending_job) {
>              MultiFDPages_t *pages = p->pages;
>  
> -            if (use_zero_copy_send) {
> -                p->iovs_num = 0;
> -            } else {
> -                p->iovs_num = 1;
> +            p->iovs_num = 0;
> +
> +            if (!use_zero_copy_send) {
> +                /*
> +                 * Only !zero_copy needs the header in IOV; zerocopy will
> +                 * send it separately.

Could use the same spelling for both mentions to zero copy.

> +                 */
> +                multifd_send_prepare_header(p);
>              }
>  
>              assert(pages->num);
> @@ -730,10 +734,6 @@ static void *multifd_send_thread(void *opaque)
>                  if (ret != 0) {
>                      break;
>                  }
> -            } else {
> -                /* Send header using the same writev call */
> -                p->iov[0].iov_len = p->packet_len;
> -                p->iov[0].iov_base = p->packet;
>              }
>  
>              ret = qio_channel_writev_full_all(p->c, p->iov, p->iovs_num, NULL,

On Wed, Jan 31, 2024 at 06:32:54PM -0300, Fabiano Rosas wrote:
> > +            if (!use_zero_copy_send) {
> > +                /*
> > +                 * Only !zero_copy needs the header in IOV; zerocopy will
> > +                 * send it separately.
> 
> Could use the same spelling for both mentions to zero copy.

Will do.

-- 
Peter Xu