On 23/10/2025 03:26, Peter Xu wrote:
> The old RDMA's io_create_watch() isn't really doing much work anyway. For
> G_IO_OUT, it already does return immediately. For G_IO_IN, it will try to
> detect some RDMA context length however normally nobody will be able to set
> it at all.
>
> Simplify the code so that RDMA iochannels simply always rely on synchronous
> reads and writes. It is highly likely what 6ddd2d76ca6f86f was talking
> about, that the async model isn't really working well.
>
> To be eplicit, incoming migration should always have marked the iochannel
s/eplicit/explicit ?
> to be nonblocking. For non-RDMA channels, what happens with current master
> branch is when we have nothing to read, QEMU yields the coroutine at
> qemu_fill_buffer(). For RDMA, what I see is it always polls on its own and
> it yields at qemu_rdma_wait_comp_channel(). A sample stack:
>
> #0 qemu_coroutine_yield
> #1 0x0000562e46e51f77 in yield_until_fd_readable
> #2 0x0000562e46927823 in qemu_rdma_wait_comp_channel
> #3 0x0000562e46927b35 in qemu_rdma_block_for_wrid
> #4 0x0000562e46927e6f in qemu_rdma_post_send_control
> #5 0x0000562e4692857f in qemu_rdma_exchange_recv
> #6 0x0000562e4692ab5e in qio_channel_rdma_readv
> #7 0x0000562e46c1f2d7 in qio_channel_readv_full
> #8 0x0000562e46c13a6e in qemu_fill_buffer
> #9 0x0000562e46c14ba8 in qemu_peek_byte
> #10 0x0000562e46c14c09 in qemu_get_byte
> #11 0x0000562e46c14e2a in qemu_get_be32
> #12 0x0000562e46c14e8a in qemu_get_be64
> #13 0x0000562e46913f08 in ram_load_precopy
> #14 0x0000562e46914448 in ram_load
> #15 0x0000562e469186e3 in vmstate_load
> #16 0x0000562e4691ce6d in qemu_loadvm_section_part_end
> #17 0x0000562e4691d99b in qemu_loadvm_state_main
> #18 0x0000562e4691db87 in qemu_loadvm_state
> #19 0x0000562e468f2e87 in process_incoming_migration_co
>
> This patch may or may not help in reality, the whole IO watch may or may
> not be working at all for RDMA iochannels. In all cases, this patch makes
> sure above will be the only place that RDMA can poll on IOs.
make sense to me.
Acked-by: Li Zhijian <lizhijian@fujitsu.com>
>
> Tested-by: Zhijian Li (Fujitsu) <lizhijian@fujitsu.com>
> Signed-off-by: Peter Xu <peterx@redhat.com>
> ---
> migration/rdma.c | 69 +++---------------------------------------------
> 1 file changed, 3 insertions(+), 66 deletions(-)
>
> diff --git a/migration/rdma.c b/migration/rdma.c
> index 13dd391c14..0e5e02cdca 100644
> --- a/migration/rdma.c
> +++ b/migration/rdma.c
> @@ -2776,56 +2776,14 @@ static gboolean
> qio_channel_rdma_source_prepare(GSource *source,
> gint *timeout)
> {
> - QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
> - RDMAContext *rdma;
> - GIOCondition cond = 0;
> *timeout = -1;
> -
> - RCU_READ_LOCK_GUARD();
> - if (rsource->condition == G_IO_IN) {
> - rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
> - } else {
> - rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
> - }
> -
> - if (!rdma) {
> - error_report("RDMAContext is NULL when prepare Gsource");
> - return FALSE;
> - }
> -
> - if (rdma->wr_data[0].control_len) {
> - cond |= G_IO_IN;
> - }
> - cond |= G_IO_OUT;
> -
> - return cond & rsource->condition;
> + return TRUE;
> }
>
> static gboolean
> qio_channel_rdma_source_check(GSource *source)
> {
> - QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
> - RDMAContext *rdma;
> - GIOCondition cond = 0;
> -
> - RCU_READ_LOCK_GUARD();
> - if (rsource->condition == G_IO_IN) {
> - rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
> - } else {
> - rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
> - }
> -
> - if (!rdma) {
> - error_report("RDMAContext is NULL when check Gsource");
> - return FALSE;
> - }
> -
> - if (rdma->wr_data[0].control_len) {
> - cond |= G_IO_IN;
> - }
> - cond |= G_IO_OUT;
> -
> - return cond & rsource->condition;
> + return TRUE;
> }
>
> static gboolean
> @@ -2835,29 +2793,8 @@ qio_channel_rdma_source_dispatch(GSource *source,
> {
> QIOChannelFunc func = (QIOChannelFunc)callback;
> QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
> - RDMAContext *rdma;
> - GIOCondition cond = 0;
> -
> - RCU_READ_LOCK_GUARD();
> - if (rsource->condition == G_IO_IN) {
> - rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
> - } else {
> - rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
> - }
> -
> - if (!rdma) {
> - error_report("RDMAContext is NULL when dispatch Gsource");
> - return FALSE;
> - }
> -
> - if (rdma->wr_data[0].control_len) {
> - cond |= G_IO_IN;
> - }
> - cond |= G_IO_OUT;
>
> - return (*func)(QIO_CHANNEL(rsource->rioc),
> - (cond & rsource->condition),
> - user_data);
> + return (*func)(QIO_CHANNEL(rsource->rioc), rsource->condition, user_data);
> }
>
> static void