From nobody Fri May 17 01:43:41 2024 Received: from us-smtp-delivery-124.mimecast.com (us-smtp-delivery-124.mimecast.com [170.10.133.124]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id DC83621355 for ; Thu, 14 Sep 2023 17:02:24 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=redhat.com; s=mimecast20190719; t=1694710943; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=aEVBOaAzEqajZTBC12YcikiwAQ8spCarEy0dLAtSHWI=; b=RS5DNHONwTb714Lggrm3dkFu2XBiOcBwb0Tef5EW1+8LecJgbpIn5pqNlapNHxR6EMhit2 q6EqstEwwpAu3k1jSmOedJRFhOoNIETVmwiY/2qrVX9+mxnucE9fh0y32GiWEpDUmSFtrM 7HlZpMrYuFRTHoBSjjRJh0F5359Wy2g= Received: from mimecast-mx02.redhat.com (mimecast-mx02.redhat.com [66.187.233.88]) by relay.mimecast.com with ESMTP with STARTTLS (version=TLSv1.2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id us-mta-44-JOdX0ltkOQKmUOBqA_lwSw-1; Thu, 14 Sep 2023 13:02:21 -0400 X-MC-Unique: JOdX0ltkOQKmUOBqA_lwSw-1 Received: from smtp.corp.redhat.com (int-mx06.intmail.prod.int.rdu2.redhat.com [10.11.54.6]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mimecast-mx02.redhat.com (Postfix) with ESMTPS id E9FA31871DB6 for ; Thu, 14 Sep 2023 17:02:12 +0000 (UTC) Received: from gerbillo.redhat.com (unknown [10.45.224.154]) by smtp.corp.redhat.com (Postfix) with ESMTP id 7809A21B20B1 for ; Thu, 14 Sep 2023 17:02:12 +0000 (UTC) From: Paolo Abeni To: mptcp@lists.linux.dev Subject: [PATCH mptcp-next v2 1/3] mptcp: consolidate sockopt synchronization Date: Thu, 14 Sep 2023 19:02:04 +0200 Message-ID: <9ad5120b406ac73ea2ca65de714966dafad3dc3c.1694710538.git.pabeni@redhat.com> In-Reply-To: References: Precedence: bulk X-Mailing-List: mptcp@lists.linux.dev List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Scanned-By: MIMEDefang 3.1 on 10.11.54.6 X-Mimecast-Spam-Score: 0 X-Mimecast-Originator: redhat.com Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8"; x-default="true" Move the socket option synchronization for active subflows at subflow creation time. This allows removing the now unused unlocked variant of such helper. While at that, clean-up a bit the mptcp_subflow_create_socket() errors path. Signed-off-by: Paolo Abeni --- net/mptcp/protocol.c | 2 -- net/mptcp/sockopt.c | 22 ---------------------- net/mptcp/subflow.c | 18 +++++++++--------- 3 files changed, 9 insertions(+), 33 deletions(-) diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index 3facdc006adc..3a905c122306 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -121,8 +121,6 @@ struct sock *__mptcp_nmpc_sk(struct mptcp_sock *msk) ret =3D __mptcp_socket_create(msk); if (ret) return ERR_PTR(ret); - - mptcp_sockopt_sync(msk, msk->first); } =20 return msk->first; diff --git a/net/mptcp/sockopt.c b/net/mptcp/sockopt.c index 8260202c0066..f44b364b0055 100644 --- a/net/mptcp/sockopt.c +++ b/net/mptcp/sockopt.c @@ -1444,28 +1444,6 @@ static void sync_socket_options(struct mptcp_sock *m= sk, struct sock *ssk) inet_assign_bit(FREEBIND, ssk, inet_test_bit(FREEBIND, sk)); } =20 -static void __mptcp_sockopt_sync(struct mptcp_sock *msk, struct sock *ssk) -{ - bool slow =3D lock_sock_fast(ssk); - - sync_socket_options(msk, ssk); - - unlock_sock_fast(ssk, slow); -} - -void mptcp_sockopt_sync(struct mptcp_sock *msk, struct sock *ssk) -{ - struct mptcp_subflow_context *subflow =3D mptcp_subflow_ctx(ssk); - - msk_owned_by_me(msk); - - if (READ_ONCE(subflow->setsockopt_seq) !=3D msk->setsockopt_seq) { - __mptcp_sockopt_sync(msk, ssk); - - subflow->setsockopt_seq =3D msk->setsockopt_seq; - } -} - void mptcp_sockopt_sync_locked(struct mptcp_sock *msk, struct sock *ssk) { struct mptcp_subflow_context *subflow =3D mptcp_subflow_ctx(ssk); diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index 918c1a235790..1c7ff7247bc0 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -1525,8 +1525,6 @@ int __mptcp_subflow_connect(struct sock *sk, const st= ruct mptcp_addr_info *loc, if (addr.ss_family =3D=3D AF_INET6) addrlen =3D sizeof(struct sockaddr_in6); #endif - mptcp_sockopt_sync(msk, ssk); - ssk->sk_bound_dev_if =3D ifindex; err =3D kernel_bind(sf, (struct sockaddr *)&addr, addrlen); if (err) @@ -1637,7 +1635,7 @@ int mptcp_subflow_create_socket(struct sock *sk, unsi= gned short family, =20 err =3D security_mptcp_add_subflow(sk, sf->sk); if (err) - goto release_ssk; + goto err_free; =20 /* the newly created socket has to be in the same cgroup as its parent */ mptcp_attach_cgroup(sk, sf->sk); @@ -1651,15 +1649,12 @@ int mptcp_subflow_create_socket(struct sock *sk, un= signed short family, get_net_track(net, &sf->sk->ns_tracker, GFP_KERNEL); sock_inuse_add(net, 1); err =3D tcp_set_ulp(sf->sk, "mptcp"); + if (err) + goto err_free; =20 -release_ssk: + mptcp_sockopt_sync_locked(mptcp_sk(sk), sf->sk); release_sock(sf->sk); =20 - if (err) { - sock_release(sf); - return err; - } - /* the newly created socket really belongs to the owning MPTCP master * socket, even if for additional subflows the allocation is performed * by a kernel workqueue. Adjust inode references, so that the @@ -1679,6 +1674,11 @@ int mptcp_subflow_create_socket(struct sock *sk, uns= igned short family, mptcp_subflow_ops_override(sf->sk); =20 return 0; + +err_free: + release_sock(sf->sk); + sock_release(sf); + return err; } =20 static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk, --=20 2.41.0 From nobody Fri May 17 01:43:41 2024 Received: from us-smtp-delivery-124.mimecast.com (us-smtp-delivery-124.mimecast.com [170.10.129.124]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 30B8D21355 for ; Thu, 14 Sep 2023 17:02:21 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=redhat.com; s=mimecast20190719; t=1694710941; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=3spK7kfYXjBeLtRekBsY8TTdAohgmRCQPN1fCq5xwnY=; b=A7AfbwVYpfQU9ZwHBQkBEuiDQNBvfCgpX18YrrYRo+/WBCPIOrtEeLs6KAxC+jLwJxdFgL hWjTJRNHzrpT/RQeUgDSN52keZB2zlcBF7kFfULQ4V7RFbImG+sJIPlLdPtRZJxtiyvAxt Ku945LLCfZmSwjZ2LLj8/uU8neIphg4= Received: from mimecast-mx02.redhat.com (mimecast-mx02.redhat.com [66.187.233.88]) by relay.mimecast.com with ESMTP with STARTTLS (version=TLSv1.2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id us-mta-335-1TUeLC8BOAizQjtDGRkXWw-1; Thu, 14 Sep 2023 13:02:19 -0400 X-MC-Unique: 1TUeLC8BOAizQjtDGRkXWw-1 Received: from smtp.corp.redhat.com (int-mx06.intmail.prod.int.rdu2.redhat.com [10.11.54.6]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mimecast-mx02.redhat.com (Postfix) with ESMTPS id AAFA8931343 for ; Thu, 14 Sep 2023 17:02:13 +0000 (UTC) Received: from gerbillo.redhat.com (unknown [10.45.224.154]) by smtp.corp.redhat.com (Postfix) with ESMTP id 395EE2156A27 for ; Thu, 14 Sep 2023 17:02:13 +0000 (UTC) From: Paolo Abeni To: mptcp@lists.linux.dev Subject: [PATCH mptcp-next v2 2/3] mptcp: ignore notsent_lowat setting at the subflow level. Date: Thu, 14 Sep 2023 19:02:05 +0200 Message-ID: <5699711c3526920c738895685cd771501e1553b0.1694710538.git.pabeni@redhat.com> In-Reply-To: References: Precedence: bulk X-Mailing-List: mptcp@lists.linux.dev List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Scanned-By: MIMEDefang 3.1 on 10.11.54.6 X-Mimecast-Spam-Score: 0 X-Mimecast-Originator: redhat.com Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8"; x-default="true" Any latency related tuning taking action at the subflow level does not really affect the user-space, as only the main MPTCP socket is relevant. Anyway any limiting setting may foul the MPTCP scheduler, not being able to fully use the subflow-level cwin, leading to very poor b/w usage. Enforce notsent_lowat to be a no-op on every subflow. Note that TCP_NOTSENT_LOWAT is currently not supported, and properly dealing with that will require more invasive changes. Signed-off-by: Paolo Abeni --- net/mptcp/sockopt.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/net/mptcp/sockopt.c b/net/mptcp/sockopt.c index f44b364b0055..453d6c78c25c 100644 --- a/net/mptcp/sockopt.c +++ b/net/mptcp/sockopt.c @@ -1450,6 +1450,12 @@ void mptcp_sockopt_sync_locked(struct mptcp_sock *ms= k, struct sock *ssk) =20 msk_owned_by_me(msk); =20 + /* subflows must ignore any latency-related settings: will not affect + * the user-space - only the msk is relevant - but will foul the + * mptcp scheduler + */ + tcp_sk(ssk)->notsent_lowat =3D UINT_MAX; + if (READ_ONCE(subflow->setsockopt_seq) !=3D msk->setsockopt_seq) { sync_socket_options(msk, ssk); =20 --=20 2.41.0 From nobody Fri May 17 01:43:41 2024 Received: from us-smtp-delivery-124.mimecast.com (us-smtp-delivery-124.mimecast.com [170.10.133.124]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 3FF0921355 for ; Thu, 14 Sep 2023 17:02:27 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=redhat.com; s=mimecast20190719; t=1694710946; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=K2V6kA5BNS8EBaRzBYd95kZPjGOJA63K1doaYQebCj0=; b=aYymcA3Llo99fD2E2gaGBQVPEYknh9njP5TNT9+XeBLPaHEYqDA+ThFBPpkQHQGzhcn0iT ROkYMamCkP5Ha9nzy1QtMFiGaBkkBP1Z31hqzkdndOrwPXaCn25VYZlMWSACX4Li4nIe7J B97EUKAofrgG8XwrVwcuzCIGafHnfNM= Received: from mimecast-mx02.redhat.com (mimecast-mx02.redhat.com [66.187.233.88]) by relay.mimecast.com with ESMTP with STARTTLS (version=TLSv1.2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id us-mta-554-H7XB-Q10MV2umUtFBwgapw-1; Thu, 14 Sep 2023 13:02:22 -0400 X-MC-Unique: H7XB-Q10MV2umUtFBwgapw-1 Received: from smtp.corp.redhat.com (int-mx06.intmail.prod.int.rdu2.redhat.com [10.11.54.6]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mimecast-mx02.redhat.com (Postfix) with ESMTPS id 6C951105DAAC for ; Thu, 14 Sep 2023 17:02:14 +0000 (UTC) Received: from gerbillo.redhat.com (unknown [10.45.224.154]) by smtp.corp.redhat.com (Postfix) with ESMTP id EE674200A391 for ; Thu, 14 Sep 2023 17:02:13 +0000 (UTC) From: Paolo Abeni To: mptcp@lists.linux.dev Subject: [PATCH mptcp-next v2 3/3] mptcp: refactor sndbuf auto-tuning. Date: Thu, 14 Sep 2023 19:02:06 +0200 Message-ID: <91b50314a17d427087f718d4b77aa23dd07fcde4.1694710538.git.pabeni@redhat.com> In-Reply-To: References: Precedence: bulk X-Mailing-List: mptcp@lists.linux.dev List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 X-Scanned-By: MIMEDefang 3.1 on 10.11.54.6 X-Mimecast-Spam-Score: 0 X-Mimecast-Originator: redhat.com Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8"; x-default="true" The MPTCP protocol account for the data enqueued on all the subflows to the main socket send buffer, while the send buffer auto-tuning algorithm set the main socket send buffer size as the max size among the subflows. That causes bad performances when at least one subflow is sndbuf limited, e.g. due to very high latency, as the MPTCP scheduler can't even fill such buffer. Change the send-buffer auto-tuning algorithm to compute the main socket send buffer size as the sum of all the subflows buffer size. Signed-off-by: Paolo Abeni --- net/mptcp/protocol.c | 25 ++++++++++++++++++++++++- net/mptcp/protocol.h | 32 +++++++++++++++++++++++++++----- net/mptcp/subflow.c | 3 +-- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index 3a905c122306..180755c8c4d1 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -891,6 +891,7 @@ static bool __mptcp_finish_join(struct mptcp_sock *msk,= struct sock *ssk) mptcp_sockopt_sync_locked(msk, ssk); mptcp_subflow_joined(msk, ssk); mptcp_stop_tout_timer(sk); + __mptcp_propagate_sndbuf(sk, ssk); return true; } =20 @@ -2403,6 +2404,7 @@ static void __mptcp_close_ssk(struct sock *sk, struct= sock *ssk, * disconnect should never fail */ WARN_ON_ONCE(tcp_disconnect(ssk, 0)); + __mptcp_propagate_sndbuf(sk, ssk); mptcp_subflow_ctx_reset(subflow); release_sock(ssk); =20 @@ -2427,6 +2429,7 @@ static void __mptcp_close_ssk(struct sock *sk, struct= sock *ssk, } =20 out_release: + __mptcp_update_sndbuf(sk, -subflow->cached_sndbuf); __mptcp_subflow_error_report(sk, ssk); release_sock(ssk); =20 @@ -3214,7 +3217,7 @@ struct sock *mptcp_sk_clone_init(const struct sock *s= k, * uses the correct data */ mptcp_copy_inaddrs(nsk, ssk); - mptcp_propagate_sndbuf(nsk, ssk); + __mptcp_propagate_sndbuf(nsk, ssk); =20 mptcp_rcv_space_init(msk, ssk); bh_unlock_sock(nsk); @@ -3339,6 +3342,14 @@ void __mptcp_check_push(struct sock *sk, struct sock= *ssk) __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags); } =20 +static void __mptcp_tune_sndbuf(struct sock *sk) +{ + struct mptcp_sock *msk =3D mptcp_sk(sk); + + __mptcp_update_sndbuf(sk, msk->delta_sndbuf); + msk->delta_sndbuf =3D 0; +} + #define MPTCP_FLAGS_PROCESS_CTX_NEED (BIT(MPTCP_PUSH_PENDING) | \ BIT(MPTCP_RETRANSMIT) | \ BIT(MPTCP_FLUSH_JOIN_LIST)) @@ -3392,6 +3403,8 @@ static void mptcp_release_cb(struct sock *sk) __mptcp_set_connected(sk); if (__test_and_clear_bit(MPTCP_ERROR_REPORT, &msk->cb_flags)) __mptcp_error_report(sk); + if (__test_and_clear_bit(MPTCP_TUNE_SNDBUF, &msk->cb_flags)) + __mptcp_tune_sndbuf(sk); } =20 __mptcp_update_rmem(sk); @@ -3437,6 +3450,15 @@ void mptcp_subflow_process_delegated(struct sock *ss= k) mptcp_data_unlock(sk); mptcp_subflow_delegated_done(subflow, MPTCP_DELEGATE_SEND); } + if (test_bit(MPTCP_DELEGATE_SNDBUF, &subflow->delegated_status)) { + mptcp_data_lock(sk); + if (!sock_owned_by_user(sk)) + __mptcp_propagate_sndbuf(sk, ssk); + else + __set_bit(MPTCP_TUNE_SNDBUF, &mptcp_sk(sk)->cb_flags); + mptcp_data_unlock(sk); + mptcp_subflow_delegated_done(subflow, MPTCP_DELEGATE_SNDBUF); + } if (test_bit(MPTCP_DELEGATE_ACK, &subflow->delegated_status)) { schedule_3rdack_retransmission(ssk); mptcp_subflow_delegated_done(subflow, MPTCP_DELEGATE_ACK); @@ -3523,6 +3545,7 @@ bool mptcp_finish_join(struct sock *ssk) /* active subflow, already present inside the conn_list */ if (!list_empty(&subflow->node)) { mptcp_subflow_joined(msk, ssk); + mptcp_propagate_sndbuf(parent, ssk); return true; } =20 diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index aebc0cc24dad..02fb8b22c4ee 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -123,6 +123,7 @@ #define MPTCP_RETRANSMIT 4 #define MPTCP_FLUSH_JOIN_LIST 5 #define MPTCP_CONNECTED 6 +#define MPTCP_TUNE_SNDBUF 7 =20 struct mptcp_skb_cb { u64 map_seq; @@ -320,6 +321,7 @@ struct mptcp_sock { u64 rtt_us; /* last maximum rtt of subflows */ } rcvq_space; u8 scaling_ratio; + int delta_sndbuf; =20 u32 subflow_id; u32 setsockopt_seq; @@ -446,6 +448,7 @@ DECLARE_PER_CPU(struct mptcp_delegated_action, mptcp_de= legated_actions); =20 #define MPTCP_DELEGATE_SEND 0 #define MPTCP_DELEGATE_ACK 1 +#define MPTCP_DELEGATE_SNDBUF 2 =20 /* MPTCP subflow context */ struct mptcp_subflow_context { @@ -518,6 +521,7 @@ struct mptcp_subflow_context { =20 u32 setsockopt_seq; u32 stale_rcv_tstamp; + int cached_sndbuf; =20 struct sock *tcp_sock; /* tcp sk backpointer */ struct sock *conn; /* parent mptcp_sock */ @@ -780,13 +784,31 @@ static inline bool mptcp_data_fin_enabled(const struc= t mptcp_sock *msk) READ_ONCE(msk->write_seq) =3D=3D READ_ONCE(msk->snd_nxt); } =20 -static inline bool mptcp_propagate_sndbuf(struct sock *sk, struct sock *ss= k) +static inline void __mptcp_update_sndbuf(struct sock *sk, int delta) { - if ((sk->sk_userlocks & SOCK_SNDBUF_LOCK) || ssk->sk_sndbuf <=3D READ_ONC= E(sk->sk_sndbuf)) - return false; + if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK) && delta) + WRITE_ONCE(sk->sk_sndbuf, sk->sk_sndbuf + delta); +} + +static inline void __mptcp_propagate_sndbuf(struct sock *sk, struct sock *= ssk) +{ + struct mptcp_subflow_context *subflow =3D mptcp_subflow_ctx(ssk); + int ssk_sndbuf =3D READ_ONCE(ssk->sk_sndbuf); + + __mptcp_update_sndbuf(sk, ssk_sndbuf - subflow->cached_sndbuf); + subflow->cached_sndbuf =3D ssk_sndbuf; +} + +static inline void mptcp_propagate_sndbuf(struct sock *sk, struct sock *ss= k) +{ + struct mptcp_subflow_context *subflow =3D mptcp_subflow_ctx(ssk); + + if (likely(READ_ONCE(ssk->sk_sndbuf) =3D=3D subflow->cached_sndbuf)) + return; =20 - WRITE_ONCE(sk->sk_sndbuf, ssk->sk_sndbuf); - return true; + local_bh_disable(); + mptcp_subflow_delegate(subflow, MPTCP_DELEGATE_SNDBUF); + local_bh_enable(); } =20 static inline void mptcp_write_space(struct sock *sk) diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index 1c7ff7247bc0..5a901f14e1d7 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -421,6 +421,7 @@ static bool subflow_use_different_dport(struct mptcp_so= ck *msk, const struct soc =20 void __mptcp_set_connected(struct sock *sk) { + __mptcp_propagate_sndbuf(sk, mptcp_sk(sk)->first); if (sk->sk_state =3D=3D TCP_SYN_SENT) { inet_sk_state_store(sk, TCP_ESTABLISHED); sk->sk_state_change(sk); @@ -472,7 +473,6 @@ static void subflow_finish_connect(struct sock *sk, con= st struct sk_buff *skb) return; =20 msk =3D mptcp_sk(parent); - mptcp_propagate_sndbuf(parent, sk); subflow->rel_write_seq =3D 1; subflow->conn_finished =3D 1; subflow->ssn_offset =3D TCP_SKB_CB(skb)->seq; @@ -1728,7 +1728,6 @@ static void subflow_state_change(struct sock *sk) =20 msk =3D mptcp_sk(parent); if (subflow_simultaneous_connect(sk)) { - mptcp_propagate_sndbuf(parent, sk); mptcp_do_fallback(sk); mptcp_rcv_space_init(msk, sk); pr_fallback(msk); --=20 2.41.0