The old RDMA's io_create_watch() isn't really doing much work anyway.  For
G_IO_OUT, it already does return immediately.  For G_IO_IN, it will try to
detect some RDMA context length however normally nobody will be able to set
it at all.

Simplify the code so that RDMA iochannels simply always rely on synchronous
reads and writes.  It is highly likely what 6ddd2d76ca6f86f was talking
about, that the async model isn't really working well.

This helps because this is almost the only dependency that the migration
core would need a coroutine for rdma channels.

Signed-off-by: Peter Xu <peterx@redhat.com>
---
 migration/rdma.c | 69 +++---------------------------------------------
 1 file changed, 3 insertions(+), 66 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index ed4e20b988..bcd7aae2f2 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2789,56 +2789,14 @@ static gboolean
 qio_channel_rdma_source_prepare(GSource *source,
                                 gint *timeout)
 {
-    QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
-    RDMAContext *rdma;
-    GIOCondition cond = 0;
     *timeout = -1;
-
-    RCU_READ_LOCK_GUARD();
-    if (rsource->condition == G_IO_IN) {
-        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
-    } else {
-        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
-    }
-
-    if (!rdma) {
-        error_report("RDMAContext is NULL when prepare Gsource");
-        return FALSE;
-    }
-
-    if (rdma->wr_data[0].control_len) {
-        cond |= G_IO_IN;
-    }
-    cond |= G_IO_OUT;
-
-    return cond & rsource->condition;
+    return TRUE;
 }
 
 static gboolean
 qio_channel_rdma_source_check(GSource *source)
 {
-    QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
-    RDMAContext *rdma;
-    GIOCondition cond = 0;
-
-    RCU_READ_LOCK_GUARD();
-    if (rsource->condition == G_IO_IN) {
-        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
-    } else {
-        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
-    }
-
-    if (!rdma) {
-        error_report("RDMAContext is NULL when check Gsource");
-        return FALSE;
-    }
-
-    if (rdma->wr_data[0].control_len) {
-        cond |= G_IO_IN;
-    }
-    cond |= G_IO_OUT;
-
-    return cond & rsource->condition;
+    return TRUE;
 }
 
 static gboolean
@@ -2848,29 +2806,8 @@ qio_channel_rdma_source_dispatch(GSource *source,
 {
     QIOChannelFunc func = (QIOChannelFunc)callback;
     QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
-    RDMAContext *rdma;
-    GIOCondition cond = 0;
-
-    RCU_READ_LOCK_GUARD();
-    if (rsource->condition == G_IO_IN) {
-        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
-    } else {
-        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
-    }
-
-    if (!rdma) {
-        error_report("RDMAContext is NULL when dispatch Gsource");
-        return FALSE;
-    }
-
-    if (rdma->wr_data[0].control_len) {
-        cond |= G_IO_IN;
-    }
-    cond |= G_IO_OUT;
 
-    return (*func)(QIO_CHANNEL(rsource->rioc),
-                   (cond & rsource->condition),
-                   user_data);
+    return (*func)(QIO_CHANNEL(rsource->rioc), rsource->condition, user_data);
 }
 
 static void
-- 
2.50.1

On 28/08/2025 04:59, Peter Xu wrote:
> The old RDMA's io_create_watch() isn't really doing much work anyway.  For
> G_IO_OUT, it already does return immediately.  For G_IO_IN, it will try to
> detect some RDMA context length however normally nobody will be able to set
> it at all.
> 


First, RDMA migration works well with this patch applied.

Tested-by: Li Zhijian <lizhijian@fujitsu.com>


I have a small question. While testing, I didn't observe any callers to
qio_channel_rdma_create_watch() during a complete RDMA migration using
the default capabilities and parameters.
I was wondering in which case this function is expected to be called?
(I see io_create_watch() is mandatory for QIOChannelClass)


Thanks
Zhijian


> Simplify the code so that RDMA iochannels simply always rely on synchronous
> reads and writes.  It is highly likely what 6ddd2d76ca6f86f was talking
> about, that the async model isn't really working well.
> 
> This helps because this is almost the only dependency that the migration
> core would need a coroutine for rdma channels.
> 
> Signed-off-by: Peter Xu <peterx@redhat.com>
> ---
>   migration/rdma.c | 69 +++---------------------------------------------
>   1 file changed, 3 insertions(+), 66 deletions(-)
> 
> diff --git a/migration/rdma.c b/migration/rdma.c
> index ed4e20b988..bcd7aae2f2 100644
> --- a/migration/rdma.c
> +++ b/migration/rdma.c
> @@ -2789,56 +2789,14 @@ static gboolean
>   qio_channel_rdma_source_prepare(GSource *source,
>                                   gint *timeout)
>   {
> -    QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
> -    RDMAContext *rdma;
> -    GIOCondition cond = 0;
>       *timeout = -1;
> -
> -    RCU_READ_LOCK_GUARD();
> -    if (rsource->condition == G_IO_IN) {
> -        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
> -    } else {
> -        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
> -    }
> -
> -    if (!rdma) {
> -        error_report("RDMAContext is NULL when prepare Gsource");
> -        return FALSE;
> -    }
> -
> -    if (rdma->wr_data[0].control_len) {
> -        cond |= G_IO_IN;
> -    }
> -    cond |= G_IO_OUT;
> -
> -    return cond & rsource->condition;
> +    return TRUE;
>   }
>   
>   static gboolean
>   qio_channel_rdma_source_check(GSource *source)
>   {
> -    QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
> -    RDMAContext *rdma;
> -    GIOCondition cond = 0;
> -
> -    RCU_READ_LOCK_GUARD();
> -    if (rsource->condition == G_IO_IN) {
> -        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
> -    } else {
> -        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
> -    }
> -
> -    if (!rdma) {
> -        error_report("RDMAContext is NULL when check Gsource");
> -        return FALSE;
> -    }
> -
> -    if (rdma->wr_data[0].control_len) {
> -        cond |= G_IO_IN;
> -    }
> -    cond |= G_IO_OUT;
> -
> -    return cond & rsource->condition;
> +    return TRUE;
>   }
>   
>   static gboolean
> @@ -2848,29 +2806,8 @@ qio_channel_rdma_source_dispatch(GSource *source,
>   {
>       QIOChannelFunc func = (QIOChannelFunc)callback;
>       QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
> -    RDMAContext *rdma;
> -    GIOCondition cond = 0;
> -
> -    RCU_READ_LOCK_GUARD();
> -    if (rsource->condition == G_IO_IN) {
> -        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
> -    } else {
> -        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
> -    }
> -
> -    if (!rdma) {
> -        error_report("RDMAContext is NULL when dispatch Gsource");
> -        return FALSE;
> -    }
> -
> -    if (rdma->wr_data[0].control_len) {
> -        cond |= G_IO_IN;
> -    }
> -    cond |= G_IO_OUT;
>   
> -    return (*func)(QIO_CHANNEL(rsource->rioc),
> -                   (cond & rsource->condition),
> -                   user_data);
> +    return (*func)(QIO_CHANNEL(rsource->rioc), rsource->condition, user_data);
>   }
>   
>   static void

On Fri, Sep 26, 2025 at 02:39:43AM +0000, Zhijian Li (Fujitsu) wrote:
> 
> 
> On 28/08/2025 04:59, Peter Xu wrote:
> > The old RDMA's io_create_watch() isn't really doing much work anyway.  For
> > G_IO_OUT, it already does return immediately.  For G_IO_IN, it will try to
> > detect some RDMA context length however normally nobody will be able to set
> > it at all.
> > 
> 
> 
> First, RDMA migration works well with this patch applied.
> 
> Tested-by: Li Zhijian <lizhijian@fujitsu.com>

Thanks a lot, Zhijian.

> 
> 
> I have a small question. While testing, I didn't observe any callers to
> qio_channel_rdma_create_watch() during a complete RDMA migration using
> the default capabilities and parameters.
> I was wondering in which case this function is expected to be called?
> (I see io_create_watch() is mandatory for QIOChannelClass)

Yes, that's also my observation.  See my reply to Fabiano on the same patch
for some information.

A summary of what I said there but more focused to what you're asking: IIUC
currently we almost always rely on qemu_rdma_wait_comp_channel() to poll
the two rdma fds, and yield if necessary when in a coroutine.

IOW, I don't know when qio_channel_rdma_create_watch(), or in most cases,
qio_channel_wait(), will be used at all.  I had a feeling that if it's used
it might stuck forever (as the gsource will be monitoring control_len, see
below [1], while IIUC only the thread itself can update it, or am I
wrong?).  But I'm not fluent with the RDMA codebase.  Maybe you'll have a
better picture after seeing what I said here and there.

This patch is almost something I want to guarantee it won't happen, hence
for whatever could return QIO_CHANNEL_ERR_BLOCK for rdma channels I want to
make sure it immediately retries instead of hanging forever in the temp
main loop of qio_channel_wait().

> 
> 
> Thanks
> Zhijian
> 
> 
> > Simplify the code so that RDMA iochannels simply always rely on synchronous
> > reads and writes.  It is highly likely what 6ddd2d76ca6f86f was talking
> > about, that the async model isn't really working well.
> > 
> > This helps because this is almost the only dependency that the migration
> > core would need a coroutine for rdma channels.
> > 
> > Signed-off-by: Peter Xu <peterx@redhat.com>
> > ---
> >   migration/rdma.c | 69 +++---------------------------------------------
> >   1 file changed, 3 insertions(+), 66 deletions(-)
> > 
> > diff --git a/migration/rdma.c b/migration/rdma.c
> > index ed4e20b988..bcd7aae2f2 100644
> > --- a/migration/rdma.c
> > +++ b/migration/rdma.c
> > @@ -2789,56 +2789,14 @@ static gboolean
> >   qio_channel_rdma_source_prepare(GSource *source,
> >                                   gint *timeout)
> >   {
> > -    QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
> > -    RDMAContext *rdma;
> > -    GIOCondition cond = 0;
> >       *timeout = -1;
> > -
> > -    RCU_READ_LOCK_GUARD();
> > -    if (rsource->condition == G_IO_IN) {
> > -        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
> > -    } else {
> > -        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
> > -    }
> > -
> > -    if (!rdma) {
> > -        error_report("RDMAContext is NULL when prepare Gsource");
> > -        return FALSE;
> > -    }
> > -
> > -    if (rdma->wr_data[0].control_len) {
> > -        cond |= G_IO_IN;
> > -    }
> > -    cond |= G_IO_OUT;
> > -
> > -    return cond & rsource->condition;
> > +    return TRUE;
> >   }
> >   
> >   static gboolean
> >   qio_channel_rdma_source_check(GSource *source)
> >   {
> > -    QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
> > -    RDMAContext *rdma;
> > -    GIOCondition cond = 0;
> > -
> > -    RCU_READ_LOCK_GUARD();
> > -    if (rsource->condition == G_IO_IN) {
> > -        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
> > -    } else {
> > -        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
> > -    }
> > -
> > -    if (!rdma) {
> > -        error_report("RDMAContext is NULL when check Gsource");
> > -        return FALSE;
> > -    }
> > -
> > -    if (rdma->wr_data[0].control_len) {

[1]

> > -        cond |= G_IO_IN;
> > -    }
> > -    cond |= G_IO_OUT;
> > -
> > -    return cond & rsource->condition;
> > +    return TRUE;
> >   }
> >   
> >   static gboolean
> > @@ -2848,29 +2806,8 @@ qio_channel_rdma_source_dispatch(GSource *source,
> >   {
> >       QIOChannelFunc func = (QIOChannelFunc)callback;
> >       QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
> > -    RDMAContext *rdma;
> > -    GIOCondition cond = 0;
> > -
> > -    RCU_READ_LOCK_GUARD();
> > -    if (rsource->condition == G_IO_IN) {
> > -        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
> > -    } else {
> > -        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
> > -    }
> > -
> > -    if (!rdma) {
> > -        error_report("RDMAContext is NULL when dispatch Gsource");
> > -        return FALSE;
> > -    }
> > -
> > -    if (rdma->wr_data[0].control_len) {
> > -        cond |= G_IO_IN;
> > -    }
> > -    cond |= G_IO_OUT;
> >   
> > -    return (*func)(QIO_CHANNEL(rsource->rioc),
> > -                   (cond & rsource->condition),
> > -                   user_data);
> > +    return (*func)(QIO_CHANNEL(rsource->rioc), rsource->condition, user_data);
> >   }
> >   
> >   static void

-- 
Peter Xu

Peter Xu <peterx@redhat.com> writes:

> The old RDMA's io_create_watch() isn't really doing much work anyway.  For
> G_IO_OUT, it already does return immediately.  For G_IO_IN, it will try to
> detect some RDMA context length however normally nobody will be able to set
> it at all.
>
> Simplify the code so that RDMA iochannels simply always rely on synchronous
> reads and writes.  It is highly likely what 6ddd2d76ca6f86f was talking
> about, that the async model isn't really working well.
>
> This helps because this is almost the only dependency that the migration
> core would need a coroutine for rdma channels.
>

I don't understand this. How does this code require a coroutine? Isn't
the io_watch exactly the strategy used when there is no coroutine?

> Signed-off-by: Peter Xu <peterx@redhat.com>
> ---
>  migration/rdma.c | 69 +++---------------------------------------------
>  1 file changed, 3 insertions(+), 66 deletions(-)
>
> diff --git a/migration/rdma.c b/migration/rdma.c
> index ed4e20b988..bcd7aae2f2 100644
> --- a/migration/rdma.c
> +++ b/migration/rdma.c
> @@ -2789,56 +2789,14 @@ static gboolean
>  qio_channel_rdma_source_prepare(GSource *source,
>                                  gint *timeout)
>  {
> -    QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
> -    RDMAContext *rdma;
> -    GIOCondition cond = 0;
>      *timeout = -1;
> -
> -    RCU_READ_LOCK_GUARD();
> -    if (rsource->condition == G_IO_IN) {
> -        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
> -    } else {
> -        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
> -    }
> -
> -    if (!rdma) {
> -        error_report("RDMAContext is NULL when prepare Gsource");
> -        return FALSE;
> -    }
> -
> -    if (rdma->wr_data[0].control_len) {
> -        cond |= G_IO_IN;
> -    }
> -    cond |= G_IO_OUT;
> -
> -    return cond & rsource->condition;
> +    return TRUE;
>  }
>  
>  static gboolean
>  qio_channel_rdma_source_check(GSource *source)
>  {
> -    QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
> -    RDMAContext *rdma;
> -    GIOCondition cond = 0;
> -
> -    RCU_READ_LOCK_GUARD();
> -    if (rsource->condition == G_IO_IN) {
> -        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
> -    } else {
> -        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
> -    }
> -
> -    if (!rdma) {
> -        error_report("RDMAContext is NULL when check Gsource");
> -        return FALSE;
> -    }
> -
> -    if (rdma->wr_data[0].control_len) {
> -        cond |= G_IO_IN;
> -    }
> -    cond |= G_IO_OUT;
> -
> -    return cond & rsource->condition;
> +    return TRUE;

These are fine if we want the source to run as soon as possible, I
think. But then...

>  }
>  
>  static gboolean
> @@ -2848,29 +2806,8 @@ qio_channel_rdma_source_dispatch(GSource *source,
>  {
>      QIOChannelFunc func = (QIOChannelFunc)callback;
>      QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
> -    RDMAContext *rdma;
> -    GIOCondition cond = 0;
> -
> -    RCU_READ_LOCK_GUARD();
> -    if (rsource->condition == G_IO_IN) {
> -        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
> -    } else {
> -        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
> -    }
> -
> -    if (!rdma) {
> -        error_report("RDMAContext is NULL when dispatch Gsource");
> -        return FALSE;
> -    }
> -
> -    if (rdma->wr_data[0].control_len) {
> -        cond |= G_IO_IN;
> -    }
> -    cond |= G_IO_OUT;
>  
> -    return (*func)(QIO_CHANNEL(rsource->rioc),
> -                   (cond & rsource->condition),
> -                   user_data);
> +    return (*func)(QIO_CHANNEL(rsource->rioc), rsource->condition, user_data);

No idea who even calls g_source_set_callback() in this case. What is func?

>  }
>  
>  static void

On Tue, Sep 16, 2025 at 07:35:45PM -0300, Fabiano Rosas wrote:
> Peter Xu <peterx@redhat.com> writes:
> 
> > The old RDMA's io_create_watch() isn't really doing much work anyway.  For
> > G_IO_OUT, it already does return immediately.  For G_IO_IN, it will try to
> > detect some RDMA context length however normally nobody will be able to set
> > it at all.
> >
> > Simplify the code so that RDMA iochannels simply always rely on synchronous
> > reads and writes.  It is highly likely what 6ddd2d76ca6f86f was talking
> > about, that the async model isn't really working well.
> >
> > This helps because this is almost the only dependency that the migration
> > core would need a coroutine for rdma channels.
> >
> 
> I don't understand this. How does this code require a coroutine? Isn't
> the io_watch exactly the strategy used when there is no coroutine?

Good question.  I can't remember what I was picturing when writting it.

Here the rational should be, RDMA works slightly differently from other
iochannels, because its async model doesn't really work
asynchronously.. instead no matter whether the channel is in sync/async
mode, it always only work in a sync manner.

Here, when I was saying async I meant we currently set NONBLOCK always for
incoming main channel.

For non-RDMA channels, what happens with current master branch is when we
have nothing to read, we yield at qemu_fill_buffer().

For RDMA channels, what I see is it always polls on its own and it yields
at qemu_rdma_wait_comp_channel(). A sample stack:

#0  qemu_coroutine_yield
#1  0x0000562e46e51f77 in yield_until_fd_readable
#2  0x0000562e46927823 in qemu_rdma_wait_comp_channel
#3  0x0000562e46927b35 in qemu_rdma_block_for_wrid
#4  0x0000562e46927e6f in qemu_rdma_post_send_control
#5  0x0000562e4692857f in qemu_rdma_exchange_recv
#6  0x0000562e4692ab5e in qio_channel_rdma_readv
#7  0x0000562e46c1f2d7 in qio_channel_readv_full
#8  0x0000562e46c13a6e in qemu_fill_buffer
#9  0x0000562e46c14ba8 in qemu_peek_byte
#10 0x0000562e46c14c09 in qemu_get_byte
#11 0x0000562e46c14e2a in qemu_get_be32
#12 0x0000562e46c14e8a in qemu_get_be64
#13 0x0000562e46913f08 in ram_load_precopy
#14 0x0000562e46914448 in ram_load
#15 0x0000562e469186e3 in vmstate_load
#16 0x0000562e4691ce6d in qemu_loadvm_section_part_end
#17 0x0000562e4691d99b in qemu_loadvm_state_main
#18 0x0000562e4691db87 in qemu_loadvm_state
#19 0x0000562e468f2e87 in process_incoming_migration_co

AFAICT, this is the only channel that does explicit yields internally,
rather than relying on iochannel/qemufile framework, aka, qemu_fill_buffer().

IOW, I don't even know when RDMA's qemu_fill_buffer() internally will get a
retval of QIO_CHANNEL_ERR_BLOCK for its qio_channel_readv_full(), because
rdma's io_readv() ignors NONBLOCK always.. AFAIU.

Now, going back to this patch: since I never hit QIO_CHANNEL_ERR_BLOCK
before, I don't think I know when I'll need this patch, but I had this
patch to make sure after we switch to the thread model, we will never go
into qio_channel_wait(), because IIUC fundamentally it's broken.  After
this patch applied, it'll reliably retry immediately.  Again, I don't know
when it'll become useful, but I'm trying to make sure we stick with the
solo place (qemu_rdma_wait_comp_channel) for polling things.

So I plan to remove this sentence, which looks misleading.  Meanwhile I can
add some of above into it.

> 
> > Signed-off-by: Peter Xu <peterx@redhat.com>
> > ---
> >  migration/rdma.c | 69 +++---------------------------------------------
> >  1 file changed, 3 insertions(+), 66 deletions(-)
> >
> > diff --git a/migration/rdma.c b/migration/rdma.c
> > index ed4e20b988..bcd7aae2f2 100644
> > --- a/migration/rdma.c
> > +++ b/migration/rdma.c
> > @@ -2789,56 +2789,14 @@ static gboolean
> >  qio_channel_rdma_source_prepare(GSource *source,
> >                                  gint *timeout)
> >  {
> > -    QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
> > -    RDMAContext *rdma;
> > -    GIOCondition cond = 0;
> >      *timeout = -1;
> > -
> > -    RCU_READ_LOCK_GUARD();
> > -    if (rsource->condition == G_IO_IN) {
> > -        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
> > -    } else {
> > -        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
> > -    }
> > -
> > -    if (!rdma) {
> > -        error_report("RDMAContext is NULL when prepare Gsource");
> > -        return FALSE;
> > -    }
> > -
> > -    if (rdma->wr_data[0].control_len) {
> > -        cond |= G_IO_IN;
> > -    }
> > -    cond |= G_IO_OUT;
> > -
> > -    return cond & rsource->condition;
> > +    return TRUE;
> >  }
> >  
> >  static gboolean
> >  qio_channel_rdma_source_check(GSource *source)
> >  {
> > -    QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
> > -    RDMAContext *rdma;
> > -    GIOCondition cond = 0;
> > -
> > -    RCU_READ_LOCK_GUARD();
> > -    if (rsource->condition == G_IO_IN) {
> > -        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
> > -    } else {
> > -        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
> > -    }
> > -
> > -    if (!rdma) {
> > -        error_report("RDMAContext is NULL when check Gsource");
> > -        return FALSE;
> > -    }
> > -
> > -    if (rdma->wr_data[0].control_len) {
> > -        cond |= G_IO_IN;
> > -    }
> > -    cond |= G_IO_OUT;
> > -
> > -    return cond & rsource->condition;
> > +    return TRUE;
> 
> These are fine if we want the source to run as soon as possible, I
> think. But then...
> 
> >  }
> >  
> >  static gboolean
> > @@ -2848,29 +2806,8 @@ qio_channel_rdma_source_dispatch(GSource *source,
> >  {
> >      QIOChannelFunc func = (QIOChannelFunc)callback;
> >      QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
> > -    RDMAContext *rdma;
> > -    GIOCondition cond = 0;
> > -
> > -    RCU_READ_LOCK_GUARD();
> > -    if (rsource->condition == G_IO_IN) {
> > -        rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
> > -    } else {
> > -        rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
> > -    }
> > -
> > -    if (!rdma) {
> > -        error_report("RDMAContext is NULL when dispatch Gsource");
> > -        return FALSE;
> > -    }
> > -
> > -    if (rdma->wr_data[0].control_len) {
> > -        cond |= G_IO_IN;
> > -    }
> > -    cond |= G_IO_OUT;
> >  
> > -    return (*func)(QIO_CHANNEL(rsource->rioc),
> > -                   (cond & rsource->condition),
> > -                   user_data);
> > +    return (*func)(QIO_CHANNEL(rsource->rioc), rsource->condition, user_data);
> 
> No idea who even calls g_source_set_callback() in this case. What is func?

In terms of qio_channel_wait(), func is qio_channel_wait_complete().

After this patch, qio_channel_wait_complete() will be invoked immediately,
hence qio_channel_wait() will reliably return immediately for rdma channels.

-- 
Peter Xu