Set of helpers for mptcp_skb_add(). Some functions are inspired from tcp
fastopen.c file. This chain helps to add “skb” to 
“&msk->sk_receive_queue”
Example: “subflow_v4_conn_request”->”mptcp_conn_request”->
”mptcp_try_fastopen”->”mptcp_fastopen_create_child”->
”mptcp_fastopen_add_skb”

Signed-off-by: Dmytro Shytyi <dmytro@shytyi.net>
---
 include/net/tcp.h       |  27 +++++++
 net/ipv4/tcp_fastopen.c |  38 +++++-----
 net/ipv4/tcp_input.c    |  20 ++---
 net/mptcp/fastopen.c    | 163 ++++++++++++++++++++++++++++++++++++++++
 net/mptcp/protocol.h    |  39 ++++++++++
 net/mptcp/subflow.c     |   2 +-
 6 files changed, 261 insertions(+), 28 deletions(-)

diff --git a/include/net/tcp.h b/include/net/tcp.h
index 735e957f7f4b..dc072d120f32 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -1729,6 +1729,33 @@ int tcp_md5_hash_key(struct tcp_md5sig_pool *hp,
 		     const struct tcp_md5sig_key *key);
 
 /* From tcp_fastopen.c */
+struct sock *tcp_fastopen_create_child(struct sock *sk,
+				       struct sk_buff *skb,
+				       struct request_sock *req);
+bool tcp_fastopen_queue_check(struct sock *sk);
+int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk,
+			    u64 *key);
+void tcp_fastopen_cookie_gen(struct sock *sk,
+			     struct request_sock *req,
+			     struct sk_buff *syn,
+			     struct tcp_fastopen_cookie *foc);
+int tcp_fastopen_cookie_gen_check(struct sock *sk,
+				  struct request_sock *req,
+				  struct sk_buff *syn,
+				  struct tcp_fastopen_cookie *orig,
+				  struct tcp_fastopen_cookie *valid_foc);
+bool tcp_fastopen_no_cookie(const struct sock *sk,
+			    const struct dst_entry *dst,
+			    int flag);
+void tcp_reqsk_record_syn(const struct sock *sk,
+			  struct request_sock *req,
+			  const struct sk_buff *skb);
+void tcp_ecn_create_request(struct request_sock *req,
+			    const struct sk_buff *skb,
+			    const struct sock *listen_sk,
+			    const struct dst_entry *dst);
+void tcp_openreq_init(struct request_sock *req,
+		      const struct tcp_options_received *rx_opt,
+		      struct sk_buff *skb, const struct sock *sk);
 void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
 			    struct tcp_fastopen_cookie *cookie);
 void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
diff --git a/net/ipv4/tcp_fastopen.c b/net/ipv4/tcp_fastopen.c
index 45cc7f1ca296..0963d7cbc8df 100644
--- a/net/ipv4/tcp_fastopen.c
+++ b/net/ipv4/tcp_fastopen.c
@@ -3,6 +3,7 @@
 #include <linux/tcp.h>
 #include <linux/rcupdate.h>
 #include <net/tcp.h>
+#include "../mptcp/protocol.h"
 
 void tcp_fastopen_init_key_once(struct net *net)
 {
@@ -149,10 +150,10 @@ static bool __tcp_fastopen_cookie_gen_cipher(struct request_sock *req,
 /* Generate the fastopen cookie by applying SipHash to both the source and
  * destination addresses.
  */
-static void tcp_fastopen_cookie_gen(struct sock *sk,
-				    struct request_sock *req,
-				    struct sk_buff *syn,
-				    struct tcp_fastopen_cookie *foc)
+void tcp_fastopen_cookie_gen(struct sock *sk,
+			     struct request_sock *req,
+			     struct sk_buff *syn,
+			     struct tcp_fastopen_cookie *foc)
 {
 	struct tcp_fastopen_context *ctx;
 
@@ -207,11 +208,11 @@ void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb)
 }
 
 /* returns 0 - no key match, 1 for primary, 2 for backup */
-static int tcp_fastopen_cookie_gen_check(struct sock *sk,
-					 struct request_sock *req,
-					 struct sk_buff *syn,
-					 struct tcp_fastopen_cookie *orig,
-					 struct tcp_fastopen_cookie *valid_foc)
+int tcp_fastopen_cookie_gen_check(struct sock *sk,
+				  struct request_sock *req,
+				  struct sk_buff *syn,
+				  struct tcp_fastopen_cookie *orig,
+				  struct tcp_fastopen_cookie *valid_foc)
 {
 	struct tcp_fastopen_cookie search_foc = { .len = -1 };
 	struct tcp_fastopen_cookie *foc = valid_foc;
@@ -235,9 +236,9 @@ static int tcp_fastopen_cookie_gen_check(struct sock *sk,
 	return ret;
 }
 
-static struct sock *tcp_fastopen_create_child(struct sock *sk,
-					      struct sk_buff *skb,
-					      struct request_sock *req)
+struct sock *tcp_fastopen_create_child(struct sock *sk,
+				       struct sk_buff *skb,
+				       struct request_sock *req)
 {
 	struct tcp_sock *tp;
 	struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue;
@@ -283,7 +284,10 @@ static struct sock *tcp_fastopen_create_child(struct sock *sk,
 
 	tp->rcv_nxt = TCP_SKB_CB(skb)->seq + 1;
 
-	tcp_fastopen_add_skb(child, skb);
+	if (sk_is_mptcp(sk))
+		mptcp_fastopen_add_skb(child, skb, req);
+	else
+		tcp_fastopen_add_skb(child, skb);
 
 	tcp_rsk(req)->rcv_nxt = tp->rcv_nxt;
 	tp->rcv_wup = tp->rcv_nxt;
@@ -293,7 +297,7 @@ static struct sock *tcp_fastopen_create_child(struct sock *sk,
 	return child;
 }
 
-static bool tcp_fastopen_queue_check(struct sock *sk)
+bool tcp_fastopen_queue_check(struct sock *sk)
 {
 	struct fastopen_queue *fastopenq;
 
@@ -329,9 +333,9 @@ static bool tcp_fastopen_queue_check(struct sock *sk)
 	return true;
 }
 
-static bool tcp_fastopen_no_cookie(const struct sock *sk,
-				   const struct dst_entry *dst,
-				   int flag)
+bool tcp_fastopen_no_cookie(const struct sock *sk,
+			    const struct dst_entry *dst,
+			    int flag)
 {
 	return (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen) & flag) ||
 	       tcp_sk(sk)->fastopen_no_cookie ||
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index bc2ea12221f9..c82b3d0a801a 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -6730,10 +6730,10 @@ static inline void pr_drop_req(struct request_sock *req, __u16 port, int family)
  * RFC8311 §4.3 which updates RFC3168 to allow the development of such
  * extensions.
  */
-static void tcp_ecn_create_request(struct request_sock *req,
-				   const struct sk_buff *skb,
-				   const struct sock *listen_sk,
-				   const struct dst_entry *dst)
+void tcp_ecn_create_request(struct request_sock *req,
+			    const struct sk_buff *skb,
+			    const struct sock *listen_sk,
+			    const struct dst_entry *dst)
 {
 	const struct tcphdr *th = tcp_hdr(skb);
 	const struct net *net = sock_net(listen_sk);
@@ -6754,9 +6754,9 @@ static void tcp_ecn_create_request(struct request_sock *req,
 		inet_rsk(req)->ecn_ok = 1;
 }
 
-static void tcp_openreq_init(struct request_sock *req,
-			     const struct tcp_options_received *rx_opt,
-			     struct sk_buff *skb, const struct sock *sk)
+void tcp_openreq_init(struct request_sock *req,
+		      const struct tcp_options_received *rx_opt,
+		      struct sk_buff *skb, const struct sock *sk)
 {
 	struct inet_request_sock *ireq = inet_rsk(req);
 
@@ -6837,9 +6837,9 @@ static bool tcp_syn_flood_action(const struct sock *sk, const char *proto)
 	return want_cookie;
 }
 
-static void tcp_reqsk_record_syn(const struct sock *sk,
-				 struct request_sock *req,
-				 const struct sk_buff *skb)
+void tcp_reqsk_record_syn(const struct sock *sk,
+			  struct request_sock *req,
+			  const struct sk_buff *skb)
 {
 	if (tcp_sk(sk)->save_syn) {
 		u32 len = skb_network_header_len(skb) + tcp_hdrlen(skb);
diff --git a/net/mptcp/fastopen.c b/net/mptcp/fastopen.c
index 1b9caf56e02f..b7c1c4523c10 100644
--- a/net/mptcp/fastopen.c
+++ b/net/mptcp/fastopen.c
@@ -166,3 +166,166 @@ void mptcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb, struct request
 	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
 		tcp_fin(sk);
 }
+
+struct sock *mptcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
+				struct request_sock *req,
+				struct tcp_fastopen_cookie *foc,
+				const struct dst_entry *dst)
+{
+	bool syn_data_status = TCP_SKB_CB(skb)->end_seq != TCP_SKB_CB(skb)->seq + 1;
+	struct tcp_fastopen_cookie valid_mptcp_foc = { .len = -1 };
+	struct sock *child_sock;
+	int ret = 0;
+
+	if ((syn_data_status || foc->len >= 0) &&
+	    tcp_fastopen_queue_check(sk)) {
+		foc->len = -1;
+		return NULL;
+	}
+
+	if (tcp_fastopen_no_cookie(sk, dst, TFO_SERVER_COOKIE_NOT_REQD))
+		goto fastopen;
+
+	if (foc->len == 0) {
+		tcp_fastopen_cookie_gen(sk, req, skb, &valid_mptcp_foc);
+	} else if (foc->len > 0) {
+		ret = tcp_fastopen_cookie_gen_check(sk, req, skb, foc,
+						    &valid_mptcp_foc);
+		if (ret) {
+fastopen:
+			child_sock = tcp_fastopen_create_child(sk, skb, req);
+			if (child_sock) {
+				if (ret == 2) {
+					valid_mptcp_foc.exp = foc->exp;
+					*foc = valid_mptcp_foc;
+				} else {
+					foc->len = -1;
+				}
+				return child_sock;
+			}
+		}
+	}
+	valid_mptcp_foc.exp = foc->exp;
+	*foc = valid_mptcp_foc;
+	return NULL;
+}
+
+int mptcp_conn_request(struct request_sock_ops *rsk_ops,
+		       const struct tcp_request_sock_ops *af_ops,
+		       struct sock *sk, struct sk_buff *skb)
+{
+	struct tcp_fastopen_cookie mptcp_foc = { .len = -1 };
+	struct tcp_options_received tmp_opt_rcvd;
+	__u32 isn = TCP_SKB_CB(skb)->tcp_tw_isn;
+	struct tcp_sock *tp_sock = tcp_sk(sk);
+	struct sock *mptcp_fo_sk = NULL;
+	struct net *net = sock_net(sk);
+	struct request_sock *req_sock;
+	bool want_cookie = false;
+	struct dst_entry *dst;
+	struct flowi fl;
+
+	if (sk_acceptq_is_full(sk))
+		goto drop;
+
+	req_sock = inet_reqsk_alloc(rsk_ops, sk, !want_cookie);
+	if (!req_sock)
+		goto drop;
+
+	req_sock->syncookie = want_cookie;
+	tcp_rsk(req_sock)->af_specific = af_ops;
+	tcp_rsk(req_sock)->ts_off = 1;
+	tcp_rsk(req_sock)->is_mptcp = 1;
+
+	tcp_clear_options(&tmp_opt_rcvd);
+	tmp_opt_rcvd.mss_clamp = af_ops->mss_clamp;
+	tmp_opt_rcvd.user_mss  = tp_sock->rx_opt.user_mss;
+	tcp_parse_options(sock_net(sk), skb, &tmp_opt_rcvd, 0,
+			  want_cookie ? NULL : &mptcp_foc);
+
+	if (want_cookie && !tmp_opt_rcvd.saw_tstamp)
+		tcp_clear_options(&tmp_opt_rcvd);
+
+	if (IS_ENABLED(CONFIG_SMC) && want_cookie)
+		tmp_opt_rcvd.smc_ok = 0;
+
+	tmp_opt_rcvd.tstamp_ok = 0;
+	tcp_openreq_init(req_sock, &tmp_opt_rcvd, skb, sk);
+	inet_rsk(req_sock)->no_srccheck = inet_sk(sk)->transparent;
+
+	inet_rsk(req_sock)->ir_iif = inet_request_bound_dev_if(sk, skb);
+
+	dst = af_ops->route_req(sk, skb, &fl, req_sock);
+	if (!dst)
+		goto drop_and_free;
+
+	if (tmp_opt_rcvd.tstamp_ok)
+		tcp_rsk(req_sock)->ts_off = af_ops->init_ts_off(net, skb);
+
+	if (!want_cookie && !isn) {
+		if (!net->ipv4.sysctl_tcp_syncookies &&
+		    (net->ipv4.sysctl_max_syn_backlog - inet_csk_reqsk_queue_len(sk) <
+		     (net->ipv4.sysctl_max_syn_backlog >> 2)) &&
+		    !tcp_peer_is_proven(req_sock, dst)) {
+			goto drop_and_release;
+		}
+
+		isn = af_ops->init_seq(skb);
+	}
+
+	tcp_ecn_create_request(req_sock, skb, sk, dst);
+
+	if (want_cookie) {
+		isn = cookie_init_sequence(af_ops, sk, skb, &req_sock->mss);
+		if (!tmp_opt_rcvd.tstamp_ok)
+			inet_rsk(req_sock)->ecn_ok = 0;
+	}
+
+	tcp_rsk(req_sock)->snt_isn = isn;
+	tcp_rsk(req_sock)->txhash = net_tx_rndhash();
+	tcp_rsk(req_sock)->syn_tos = TCP_SKB_CB(skb)->ip_dsfield;
+
+	tcp_openreq_init_rwin(req_sock, sk, dst);
+	sk_rx_queue_set(req_to_sk(req_sock), skb);
+	if (!want_cookie) {
+		tcp_reqsk_record_syn(sk, req_sock, skb);
+		mptcp_fo_sk = mptcp_try_fastopen(sk, skb, req_sock, &mptcp_foc, dst);
+	}
+	if (mptcp_fo_sk) {
+		af_ops->send_synack(mptcp_fo_sk, dst, &fl, req_sock,
+				    &mptcp_foc, TCP_SYNACK_FASTOPEN, skb);
+		if (!inet_csk_reqsk_queue_add(sk, req_sock, mptcp_fo_sk)) {
+			reqsk_fastopen_remove(mptcp_fo_sk, req_sock, false);
+			bh_unlock_sock(mptcp_fo_sk);
+			sock_put(mptcp_fo_sk);
+			goto drop_and_free;
+		}
+		sk->sk_data_ready(sk);
+		bh_unlock_sock(mptcp_fo_sk);
+		sock_put(mptcp_fo_sk);
+	} else {
+		tcp_rsk(req_sock)->tfo_listener = false;
+		if (!want_cookie) {
+			req_sock->timeout = tcp_timeout_init((struct sock *)req_sock);
+			inet_csk_reqsk_queue_hash_add(sk, req_sock, req_sock->timeout);
+		}
+		af_ops->send_synack(sk, dst, &fl, req_sock, &mptcp_foc,
+				    !want_cookie ? TCP_SYNACK_NORMAL :
+						   TCP_SYNACK_COOKIE,
+				    skb);
+		if (want_cookie) {
+			reqsk_free(req_sock);
+			return 0;
+		}
+	}
+	reqsk_put(req_sock);
+	return 0;
+
+drop_and_release:
+	dst_release(dst);
+drop_and_free:
+	__reqsk_free(req_sock);
+drop:
+	tcp_listendrop(sk);
+	return 0;
+}
diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
index b90279c734ae..8d894a2d06b3 100644
--- a/net/mptcp/protocol.h
+++ b/net/mptcp/protocol.h
@@ -849,6 +849,45 @@ int mptcp_setsockopt_sol_tcp_fastopen(struct mptcp_sock *msk, sockptr_t optval,
 				      unsigned int optlen);
 void mptcp_treat_hshake_ack_fastopen(struct mptcp_sock *msk, struct mptcp_subflow_context *subflow,
 				     struct mptcp_options_received mp_opt);
+void mptcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb,
+			    struct request_sock *req);
+void mptcp_reqsk_record_syn(const struct sock *sk,
+			    struct request_sock *req,
+			    const struct sk_buff *skb);
+void mptcp_ecn_create_request(struct request_sock *req,
+			      const struct sk_buff *skb,
+			      const struct sock *listen_sk,
+			      const struct dst_entry *dst);
+void mptcp_openreq_init(struct request_sock *req,
+			const struct tcp_options_received *rx_opt,
+			struct sk_buff *skb, const struct sock *sk);
+struct sock *mptcp_fastopen_create_child(struct sock *sk,
+					 struct sk_buff *skb,
+					 struct request_sock *req);
+bool mptcp_fastopen_queue_check(struct sock *sk);
+bool mptcp_fastopen_cookie_gen_cipher(struct request_sock *req,
+				      struct sk_buff *syn,
+				      const siphash_key_t *key,
+				      struct tcp_fastopen_cookie *foc);
+void mptcp_fastopen_cookie_gen(struct sock *sk,
+			       struct request_sock *req,
+			       struct sk_buff *syn,
+			       struct tcp_fastopen_cookie *foc);
+int mptcp_fastopen_cookie_gen_check(struct sock *sk,
+				    struct request_sock *req,
+				    struct sk_buff *syn,
+				    struct tcp_fastopen_cookie *orig,
+				    struct tcp_fastopen_cookie *valid_foc);
+bool mptcp_fastopen_no_cookie(const struct sock *sk,
+			      const struct dst_entry *dst,
+			      int flag);
+struct sock *mptcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
+				struct request_sock *req,
+				struct tcp_fastopen_cookie *foc,
+				const struct dst_entry *dst);
+int mptcp_conn_request(struct request_sock_ops *rsk_ops,
+		       const struct tcp_request_sock_ops *af_ops,
+		       struct sock *sk, struct sk_buff *skb);
 // Fast Open Mechanism functions end
 
 static inline bool mptcp_pm_should_add_signal(struct mptcp_sock *msk)
diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c
index 075c61d1d37f..ff5fe4ff3d21 100644
--- a/net/mptcp/subflow.c
+++ b/net/mptcp/subflow.c
@@ -542,7 +542,7 @@ static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb)
 	if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
 		goto drop;
 
-	return tcp_conn_request(&mptcp_subflow_request_sock_ops,
+	return mptcp_conn_request(&mptcp_subflow_request_sock_ops,
 				&subflow_request_sock_ipv4_ops,
 				sk, skb);
 drop:
-- 
2.25.1

On Sun, 2022-09-18 at 00:28 +0200, Dmytro Shytyi wrote:
> Set of helpers for mptcp_skb_add(). Some functions are inspired from tcp
> fastopen.c file. This chain helps to add “skb” to 
> “&msk->sk_receive_queue”
> Example: “subflow_v4_conn_request”->”mptcp_conn_request”->
> ”mptcp_try_fastopen”->”mptcp_fastopen_create_child”->
> ”mptcp_fastopen_add_skb”
> 
> Signed-off-by: Dmytro Shytyi <dmytro@shytyi.net>

I think this hooking is not necessary. Leveraging the existing TFO
implementation and doing delayed ack_seq initialization, as in Benjamin
series should suffice. 

Thanks,

Paolo

This is fixed in v8.

We fully reuse the existing linux kernel code.

Best,
Dmytro

On 9/19/2022 4:06 PM, Paolo Abeni wrote:
> On Sun, 2022-09-18 at 00:28 +0200, Dmytro Shytyi wrote:
>> Set of helpers for mptcp_skb_add(). Some functions are inspired from tcp
>> fastopen.c file. This chain helps to add “skb” to
>> “&msk->sk_receive_queue”
>> Example: “subflow_v4_conn_request”->”mptcp_conn_request”->
>> ”mptcp_try_fastopen”->”mptcp_fastopen_create_child”->
>> ”mptcp_fastopen_add_skb”
>>
>> Signed-off-by: Dmytro Shytyi <dmytro@shytyi.net>
> I think this hooking is not necessary. Leveraging the existing TFO
> implementation and doing delayed ack_seq initialization, as in Benjamin
> series should suffice.
>
> Thanks,
>
> Paolo
>