From nobody Thu Apr 2 14:43:03 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id B2847C6FA83 for ; Fri, 23 Sep 2022 20:14:19 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S232526AbiIWUOQ (ORCPT ); Fri, 23 Sep 2022 16:14:16 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:39098 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S232399AbiIWUNk (ORCPT ); Fri, 23 Sep 2022 16:13:40 -0400 Received: from mail-wm1-x32f.google.com (mail-wm1-x32f.google.com [IPv6:2a00:1450:4864:20::32f]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 83EBD1231E4 for ; Fri, 23 Sep 2022 13:13:36 -0700 (PDT) Received: by mail-wm1-x32f.google.com with SMTP id t4so879264wmj.5 for ; Fri, 23 Sep 2022 13:13:36 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=arista.com; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date; bh=tR79mqk7FBZM/YFiGtYHosi4rMc2770EYwHkFS9K1LU=; b=DKxxKQwWnAC/GdnKrlwpJ5DdqvEiKi8+4fi1B3fAgfMf5Z5i2optbrh9zUp2fpksuq 2KYyXYZOp7j8wROv7vmJAgr9w8GrZsTl83yxjZCErdYDtV3z9QGiP8oJKbBq0o+GTooF LtRber0KCltj0/+Mu4tRuraYz8fqi++gU/bqg93iuP4mKf8jx8L02qUaGgx1cEe0MMSs 8uZ6Jb3w9RNFuYzHNRLrzO2nXRNMgaJVmm5Er8ElnDL0uZIqvWFdq3JiQKW4Xhfb3rnP 8qLjV//iqmf2k54WPUJeNpHa53cA84wWNiO1wMK/wX8JYDQOKMx35DHnd08H4/O2c9gs PT+g== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date; bh=tR79mqk7FBZM/YFiGtYHosi4rMc2770EYwHkFS9K1LU=; b=hAmgBPEVMbErcWtCQertDC+5SW1qZj2eV5iDOaiCs22W0kyOzqkCOli23PfPPjNvwm HxGroxYpogHX70q7GDFu16wVlPZTqZTTCFpCPuWHn+SgqxpxdOCN6VccmEBYaeC0SKL+ CpMpg9HVcDDmZ97pcPZrjSNRHQrunfgbzq6+gZz9eadyrajxYmwz21Q4s/v673oGemJr xO5A+XvhpNDkXA88ObkF9JrQzJrE1lsXFTv6a0JieLqJw+3kefMpiS3oCxeO5YD0MvOM eC8DhO06Z8eflvuE2f6/AcONlKclHn715XO+xCPJqsYvgZOHvwndijwCUhcykDQaq8Pl uVTQ== X-Gm-Message-State: ACrzQf3rRaIAVdpuGNds9SZT3p8svJS9WtmKNOxRDSuQzEqjGZf/gn4b LrmsAZKQF2c+aCyAVwtoc16AMTTTryYDpw== X-Google-Smtp-Source: AMsMyM7TtIOMm/eEr/a3QvK8xTcsLv5aVrNGimapV8YQe39vz6kskp/0Bq3s/Qc09g5W8OosPMamNg== X-Received: by 2002:a05:600c:4f82:b0:3b4:a6fc:89d2 with SMTP id n2-20020a05600c4f8200b003b4a6fc89d2mr6935343wmq.53.1663964014601; Fri, 23 Sep 2022 13:13:34 -0700 (PDT) Received: from Mindolluin.ire.aristanetworks.com ([217.173.96.166]) by smtp.gmail.com with ESMTPSA id k11-20020a05600c0b4b00b003b492753826sm3281056wmr.43.2022.09.23.13.13.33 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 23 Sep 2022 13:13:34 -0700 (PDT) From: Dmitry Safonov To: linux-kernel@vger.kernel.org, David Ahern , Eric Dumazet Cc: Dmitry Safonov , Andy Lutomirski , Ard Biesheuvel , Bob Gilligan , Dan Carpenter , "David S. Miller" , Dmitry Safonov <0x7f454c46@gmail.com>, Eric Biggers , "Eric W. Biederman" , Francesco Ruggeri , Herbert Xu , Hideaki YOSHIFUJI , Ivan Delalande , Jakub Kicinski , Leonard Crestez , Paolo Abeni , Salam Noureddine , Shuah Khan , netdev@vger.kernel.org, linux-crypto@vger.kernel.org Subject: [PATCH v2 05/35] net/tcp: Use crypto_pool for TCP-MD5 Date: Fri, 23 Sep 2022 21:12:49 +0100 Message-Id: <20220923201319.493208-6-dima@arista.com> X-Mailer: git-send-email 2.37.2 In-Reply-To: <20220923201319.493208-1-dima@arista.com> References: <20220923201319.493208-1-dima@arista.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Use crypto_pool API that was designed with tcp_md5sig_pool in mind. The conversion to use crypto_pool will allow: - to reuse ahash_request(s) for different users - to allocate only one per-CPU scratch buffer rather than a new one for each user - to have a common API for net/ users that need ahash on RX/TX fast path Signed-off-by: Dmitry Safonov --- include/net/tcp.h | 22 +++----- net/ipv4/Kconfig | 2 +- net/ipv4/tcp.c | 99 ++++++++++-------------------------- net/ipv4/tcp_ipv4.c | 105 +++++++++++++++++++++++---------------- net/ipv4/tcp_minisocks.c | 22 ++++++-- net/ipv6/tcp_ipv6.c | 53 +++++++++----------- 6 files changed, 138 insertions(+), 165 deletions(-) diff --git a/include/net/tcp.h b/include/net/tcp.h index 831cd1e24687..75bb817928c1 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -1654,12 +1654,6 @@ union tcp_md5sum_block { #endif }; =20 -/* - pool: digest algorithm, hash description and scratch buffer */ -struct tcp_md5sig_pool { - struct ahash_request *md5_req; - void *scratch; -}; - /* - functions */ int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key, const struct sock *sk, const struct sk_buff *skb); @@ -1715,17 +1709,15 @@ tcp_inbound_md5_hash(const struct sock *sk, const s= truct sk_buff *skb, #define tcp_twsk_md5_key(twsk) NULL #endif =20 -bool tcp_alloc_md5sig_pool(void); - -struct tcp_md5sig_pool *tcp_get_md5sig_pool(void); -static inline void tcp_put_md5sig_pool(void) -{ - local_bh_enable(); -} +struct crypto_pool_ahash; +int tcp_md5_alloc_crypto_pool(void); +void tcp_md5_release_crypto_pool(void); +void tcp_md5_add_crypto_pool(void); +extern int tcp_md5_crypto_pool_id; =20 -int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *, +int tcp_md5_hash_skb_data(struct crypto_pool_ahash *, const struct sk_buff= *, unsigned int header_len); -int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, +int tcp_md5_hash_key(struct crypto_pool_ahash *hp, const struct tcp_md5sig_key *key); =20 /* From tcp_fastopen.c */ diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig index e983bb0c5012..c341864e4398 100644 --- a/net/ipv4/Kconfig +++ b/net/ipv4/Kconfig @@ -733,7 +733,7 @@ config DEFAULT_TCP_CONG =20 config TCP_MD5SIG bool "TCP: MD5 Signature Option support (RFC2385)" - select CRYPTO + select CRYPTO_POOL select CRYPTO_MD5 help RFC2385 specifies a method of giving MD5 protection to TCP sessions. diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 0933701dc69c..b08487113c28 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -244,6 +244,7 @@ #define pr_fmt(fmt) "TCP: " fmt =20 #include +#include #include #include #include @@ -4401,92 +4402,45 @@ int tcp_getsockopt(struct sock *sk, int level, int = optname, char __user *optval, EXPORT_SYMBOL(tcp_getsockopt); =20 #ifdef CONFIG_TCP_MD5SIG -static DEFINE_PER_CPU(struct tcp_md5sig_pool, tcp_md5sig_pool); -static DEFINE_MUTEX(tcp_md5sig_mutex); -static bool tcp_md5sig_pool_populated =3D false; +int tcp_md5_crypto_pool_id =3D -1; +EXPORT_SYMBOL(tcp_md5_crypto_pool_id); =20 -static void __tcp_alloc_md5sig_pool(void) +int tcp_md5_alloc_crypto_pool(void) { - struct crypto_ahash *hash; - int cpu; - - hash =3D crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC); - if (IS_ERR(hash)) - return; - - for_each_possible_cpu(cpu) { - void *scratch =3D per_cpu(tcp_md5sig_pool, cpu).scratch; - struct ahash_request *req; - - if (!scratch) { - scratch =3D kmalloc_node(sizeof(union tcp_md5sum_block) + - sizeof(struct tcphdr), - GFP_KERNEL, - cpu_to_node(cpu)); - if (!scratch) - return; - per_cpu(tcp_md5sig_pool, cpu).scratch =3D scratch; - } - if (per_cpu(tcp_md5sig_pool, cpu).md5_req) - continue; - - req =3D ahash_request_alloc(hash, GFP_KERNEL); - if (!req) - return; + int ret; =20 - ahash_request_set_callback(req, 0, NULL, NULL); + ret =3D crypto_pool_reserve_scratch(sizeof(union tcp_md5sum_block) + + sizeof(struct tcphdr)); + if (ret) + return ret; =20 - per_cpu(tcp_md5sig_pool, cpu).md5_req =3D req; + ret =3D crypto_pool_alloc_ahash("md5"); + if (ret >=3D 0) { + tcp_md5_crypto_pool_id =3D ret; + return 0; } - /* before setting tcp_md5sig_pool_populated, we must commit all writes - * to memory. See smp_rmb() in tcp_get_md5sig_pool() - */ - smp_wmb(); - tcp_md5sig_pool_populated =3D true; + return ret; } +EXPORT_SYMBOL(tcp_md5_alloc_crypto_pool); =20 -bool tcp_alloc_md5sig_pool(void) +void tcp_md5_release_crypto_pool(void) { - if (unlikely(!tcp_md5sig_pool_populated)) { - mutex_lock(&tcp_md5sig_mutex); - - if (!tcp_md5sig_pool_populated) - __tcp_alloc_md5sig_pool(); - - mutex_unlock(&tcp_md5sig_mutex); - } - return tcp_md5sig_pool_populated; + crypto_pool_release(tcp_md5_crypto_pool_id); } -EXPORT_SYMBOL(tcp_alloc_md5sig_pool); +EXPORT_SYMBOL(tcp_md5_release_crypto_pool); =20 - -/** - * tcp_get_md5sig_pool - get md5sig_pool for this user - * - * We use percpu structure, so if we succeed, we exit with preemption - * and BH disabled, to make sure another thread or softirq handling - * wont try to get same context. - */ -struct tcp_md5sig_pool *tcp_get_md5sig_pool(void) +void tcp_md5_add_crypto_pool(void) { - local_bh_disable(); - - if (tcp_md5sig_pool_populated) { - /* coupled with smp_wmb() in __tcp_alloc_md5sig_pool() */ - smp_rmb(); - return this_cpu_ptr(&tcp_md5sig_pool); - } - local_bh_enable(); - return NULL; + crypto_pool_add(tcp_md5_crypto_pool_id); } -EXPORT_SYMBOL(tcp_get_md5sig_pool); +EXPORT_SYMBOL(tcp_md5_add_crypto_pool); =20 -int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *hp, +int tcp_md5_hash_skb_data(struct crypto_pool_ahash *hp, const struct sk_buff *skb, unsigned int header_len) { struct scatterlist sg; const struct tcphdr *tp =3D tcp_hdr(skb); - struct ahash_request *req =3D hp->md5_req; + struct ahash_request *req =3D hp->req; unsigned int i; const unsigned int head_data_len =3D skb_headlen(skb) > header_len ? skb_headlen(skb) - header_len : 0; @@ -4520,16 +4474,17 @@ int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *h= p, } EXPORT_SYMBOL(tcp_md5_hash_skb_data); =20 -int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, const struct tcp_md5sig_k= ey *key) +int tcp_md5_hash_key(struct crypto_pool_ahash *hp, + const struct tcp_md5sig_key *key) { u8 keylen =3D READ_ONCE(key->keylen); /* paired with WRITE_ONCE() in tcp_= md5_do_add */ struct scatterlist sg; =20 sg_init_one(&sg, key->key, keylen); - ahash_request_set_crypt(hp->md5_req, &sg, NULL, keylen); + ahash_request_set_crypt(hp->req, &sg, NULL, keylen); =20 /* We use data_race() because tcp_md5_do_add() might change key->key unde= r us */ - return data_race(crypto_ahash_update(hp->md5_req)); + return data_race(crypto_ahash_update(hp->req)); } EXPORT_SYMBOL(tcp_md5_hash_key); =20 diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index c8c3fbaf3651..4e1a54759137 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -79,6 +79,7 @@ #include =20 #include +#include #include =20 #include @@ -1157,9 +1158,6 @@ static int tcp_md5sig_info_add(struct sock *sk, gfp_t= gfp) struct tcp_sock *tp =3D tcp_sk(sk); struct tcp_md5sig_info *md5sig; =20 - if (rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) - return 0; - md5sig =3D kmalloc(sizeof(*md5sig), gfp); if (!md5sig) return -ENOMEM; @@ -1206,10 +1204,6 @@ static int __tcp_md5_do_add(struct sock *sk, const u= nion tcp_md5_addr *addr, key =3D sock_kmalloc(sk, sizeof(*key), gfp | __GFP_ZERO); if (!key) return -ENOMEM; - if (!tcp_alloc_md5sig_pool()) { - sock_kfree_s(sk, key, sizeof(*key)); - return -ENOMEM; - } =20 memcpy(key->key, newkey, newkeylen); key->keylen =3D newkeylen; @@ -1228,8 +1222,17 @@ int tcp_md5_do_add(struct sock *sk, const union tcp_= md5_addr *addr, int family, u8 prefixlen, int l3index, u8 flags, const u8 *newkey, u8 newkeylen) { - if (tcp_md5sig_info_add(sk, GFP_KERNEL)) - return -ENOMEM; + struct tcp_sock *tp =3D tcp_sk(sk); + + if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))= ) { + if (tcp_md5_alloc_crypto_pool()) + return -ENOMEM; + + if (tcp_md5sig_info_add(sk, GFP_KERNEL)) { + tcp_md5_release_crypto_pool(); + return -ENOMEM; + } + } =20 static_branch_inc(&tcp_md5_needed.key); =20 @@ -1242,8 +1245,16 @@ int tcp_md5_key_copy(struct sock *sk, const union tc= p_md5_addr *addr, int family, u8 prefixlen, int l3index, struct tcp_md5sig_key *key) { - if (tcp_md5sig_info_add(sk, sk_gfp_mask(sk, GFP_ATOMIC))) - return -ENOMEM; + struct tcp_sock *tp =3D tcp_sk(sk); + + if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))= ) { + tcp_md5_add_crypto_pool(); + + if (tcp_md5sig_info_add(sk, sk_gfp_mask(sk, GFP_ATOMIC))) { + tcp_md5_release_crypto_pool(); + return -ENOMEM; + } + } =20 atomic_inc(&tcp_md5_needed.key.key.enabled); =20 @@ -1342,7 +1353,7 @@ static int tcp_v4_parse_md5_keys(struct sock *sk, int= optname, cmd.tcpm_key, cmd.tcpm_keylen); } =20 -static int tcp_v4_md5_hash_headers(struct tcp_md5sig_pool *hp, +static int tcp_v4_md5_hash_headers(struct crypto_pool_ahash *hp, __be32 daddr, __be32 saddr, const struct tcphdr *th, int nbytes) { @@ -1350,7 +1361,7 @@ static int tcp_v4_md5_hash_headers(struct tcp_md5sig_= pool *hp, struct scatterlist sg; struct tcphdr *_th; =20 - bp =3D hp->scratch; + bp =3D hp->base.scratch; bp->saddr =3D saddr; bp->daddr =3D daddr; bp->pad =3D 0; @@ -1362,37 +1373,34 @@ static int tcp_v4_md5_hash_headers(struct tcp_md5si= g_pool *hp, _th->check =3D 0; =20 sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th)); - ahash_request_set_crypt(hp->md5_req, &sg, NULL, + ahash_request_set_crypt(hp->req, &sg, NULL, sizeof(*bp) + sizeof(*th)); - return crypto_ahash_update(hp->md5_req); + return crypto_ahash_update(hp->req); } =20 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key= *key, __be32 daddr, __be32 saddr, const struct tcphdr *th) { - struct tcp_md5sig_pool *hp; - struct ahash_request *req; + struct crypto_pool_ahash hp; =20 - hp =3D tcp_get_md5sig_pool(); - if (!hp) + if (crypto_pool_get(tcp_md5_crypto_pool_id, (struct crypto_pool *)&hp)) goto clear_hash_noput; - req =3D hp->md5_req; =20 - if (crypto_ahash_init(req)) + if (crypto_ahash_init(hp.req)) goto clear_hash; - if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, th->doff << 2)) + if (tcp_v4_md5_hash_headers(&hp, daddr, saddr, th, th->doff << 2)) goto clear_hash; - if (tcp_md5_hash_key(hp, key)) + if (tcp_md5_hash_key(&hp, key)) goto clear_hash; - ahash_request_set_crypt(req, NULL, md5_hash, 0); - if (crypto_ahash_final(req)) + ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); + if (crypto_ahash_final(hp.req)) goto clear_hash; =20 - tcp_put_md5sig_pool(); + crypto_pool_put(); return 0; =20 clear_hash: - tcp_put_md5sig_pool(); + crypto_pool_put(); clear_hash_noput: memset(md5_hash, 0, 16); return 1; @@ -1402,8 +1410,7 @@ int tcp_v4_md5_hash_skb(char *md5_hash, const struct = tcp_md5sig_key *key, const struct sock *sk, const struct sk_buff *skb) { - struct tcp_md5sig_pool *hp; - struct ahash_request *req; + struct crypto_pool_ahash hp; const struct tcphdr *th =3D tcp_hdr(skb); __be32 saddr, daddr; =20 @@ -1416,29 +1423,27 @@ int tcp_v4_md5_hash_skb(char *md5_hash, const struc= t tcp_md5sig_key *key, daddr =3D iph->daddr; } =20 - hp =3D tcp_get_md5sig_pool(); - if (!hp) + if (crypto_pool_get(tcp_md5_crypto_pool_id, (struct crypto_pool *)&hp)) goto clear_hash_noput; - req =3D hp->md5_req; =20 - if (crypto_ahash_init(req)) + if (crypto_ahash_init(hp.req)) goto clear_hash; =20 - if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, skb->len)) + if (tcp_v4_md5_hash_headers(&hp, daddr, saddr, th, skb->len)) goto clear_hash; - if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2)) + if (tcp_md5_hash_skb_data(&hp, skb, th->doff << 2)) goto clear_hash; - if (tcp_md5_hash_key(hp, key)) + if (tcp_md5_hash_key(&hp, key)) goto clear_hash; - ahash_request_set_crypt(req, NULL, md5_hash, 0); - if (crypto_ahash_final(req)) + ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); + if (crypto_ahash_final(hp.req)) goto clear_hash; =20 - tcp_put_md5sig_pool(); + crypto_pool_put(); return 0; =20 clear_hash: - tcp_put_md5sig_pool(); + crypto_pool_put(); clear_hash_noput: memset(md5_hash, 0, 16); return 1; @@ -2257,6 +2262,18 @@ static int tcp_v4_init_sock(struct sock *sk) return 0; } =20 +#ifdef CONFIG_TCP_MD5SIG +static void tcp_md5sig_info_free_rcu(struct rcu_head *head) +{ + struct tcp_md5sig_info *md5sig; + + md5sig =3D container_of(head, struct tcp_md5sig_info, rcu); + kfree(md5sig); + static_branch_slow_dec_deferred(&tcp_md5_needed); + tcp_md5_release_crypto_pool(); +} +#endif + void tcp_v4_destroy_sock(struct sock *sk) { struct tcp_sock *tp =3D tcp_sk(sk); @@ -2281,10 +2298,12 @@ void tcp_v4_destroy_sock(struct sock *sk) #ifdef CONFIG_TCP_MD5SIG /* Clean up the MD5 key list, if any */ if (tp->md5sig_info) { + struct tcp_md5sig_info *md5sig; + + md5sig =3D rcu_dereference_protected(tp->md5sig_info, 1); tcp_clear_md5_list(sk); - kfree_rcu(rcu_dereference_protected(tp->md5sig_info, 1), rcu); - tp->md5sig_info =3D NULL; - static_branch_slow_dec_deferred(&tcp_md5_needed); + call_rcu(&md5sig->rcu, tcp_md5sig_info_free_rcu); + rcu_assign_pointer(tp->md5sig_info, NULL); } #endif =20 diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index 5d475a45a478..d1d30337ffec 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -297,8 +297,10 @@ void tcp_time_wait(struct sock *sk, int state, int tim= eo) key =3D tp->af_specific->md5_lookup(sk, sk); if (key) { tcptw->tw_md5_key =3D kmemdup(key, sizeof(*key), GFP_ATOMIC); - BUG_ON(tcptw->tw_md5_key && !tcp_alloc_md5sig_pool()); + if (WARN_ON(!tcptw->tw_md5_key)) + break; atomic_inc(&tcp_md5_needed.key.key.enabled); + tcp_md5_add_crypto_pool(); } } } while (0); @@ -335,16 +337,26 @@ void tcp_time_wait(struct sock *sk, int state, int ti= meo) } EXPORT_SYMBOL(tcp_time_wait); =20 +#ifdef CONFIG_TCP_MD5SIG +static void tcp_md5_twsk_free_rcu(struct rcu_head *head) +{ + struct tcp_md5sig_key *key; + + key =3D container_of(head, struct tcp_md5sig_key, rcu); + kfree(key); + static_branch_slow_dec_deferred(&tcp_md5_needed); + tcp_md5_release_crypto_pool(); +} +#endif + void tcp_twsk_destructor(struct sock *sk) { #ifdef CONFIG_TCP_MD5SIG if (static_branch_unlikely(&tcp_md5_needed.key)) { struct tcp_timewait_sock *twsk =3D tcp_twsk(sk); =20 - if (twsk->tw_md5_key) { - kfree_rcu(twsk->tw_md5_key, rcu); - static_branch_slow_dec_deferred(&tcp_md5_needed); - } + if (twsk->tw_md5_key) + call_rcu(&twsk->tw_md5_key->rcu, tcp_md5_twsk_free_rcu); } #endif } diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index cb891a71db0d..f75569f889e7 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -64,6 +64,7 @@ #include =20 #include +#include #include =20 #include @@ -665,7 +666,7 @@ static int tcp_v6_parse_md5_keys(struct sock *sk, int o= ptname, cmd.tcpm_key, cmd.tcpm_keylen); } =20 -static int tcp_v6_md5_hash_headers(struct tcp_md5sig_pool *hp, +static int tcp_v6_md5_hash_headers(struct crypto_pool_ahash *hp, const struct in6_addr *daddr, const struct in6_addr *saddr, const struct tcphdr *th, int nbytes) @@ -674,7 +675,7 @@ static int tcp_v6_md5_hash_headers(struct tcp_md5sig_po= ol *hp, struct scatterlist sg; struct tcphdr *_th; =20 - bp =3D hp->scratch; + bp =3D hp->base.scratch; /* 1. TCP pseudo-header (RFC2460) */ bp->saddr =3D *saddr; bp->daddr =3D *daddr; @@ -686,38 +687,35 @@ static int tcp_v6_md5_hash_headers(struct tcp_md5sig_= pool *hp, _th->check =3D 0; =20 sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th)); - ahash_request_set_crypt(hp->md5_req, &sg, NULL, + ahash_request_set_crypt(hp->req, &sg, NULL, sizeof(*bp) + sizeof(*th)); - return crypto_ahash_update(hp->md5_req); + return crypto_ahash_update(hp->req); } =20 static int tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key= *key, const struct in6_addr *daddr, struct in6_addr *saddr, const struct tcphdr *th) { - struct tcp_md5sig_pool *hp; - struct ahash_request *req; + struct crypto_pool_ahash hp; =20 - hp =3D tcp_get_md5sig_pool(); - if (!hp) + if (crypto_pool_get(tcp_md5_crypto_pool_id, (struct crypto_pool *)&hp)) goto clear_hash_noput; - req =3D hp->md5_req; =20 - if (crypto_ahash_init(req)) + if (crypto_ahash_init(hp.req)) goto clear_hash; - if (tcp_v6_md5_hash_headers(hp, daddr, saddr, th, th->doff << 2)) + if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, th->doff << 2)) goto clear_hash; - if (tcp_md5_hash_key(hp, key)) + if (tcp_md5_hash_key(&hp, key)) goto clear_hash; - ahash_request_set_crypt(req, NULL, md5_hash, 0); - if (crypto_ahash_final(req)) + ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); + if (crypto_ahash_final(hp.req)) goto clear_hash; =20 - tcp_put_md5sig_pool(); + crypto_pool_put(); return 0; =20 clear_hash: - tcp_put_md5sig_pool(); + crypto_pool_put(); clear_hash_noput: memset(md5_hash, 0, 16); return 1; @@ -729,8 +727,7 @@ static int tcp_v6_md5_hash_skb(char *md5_hash, const struct sk_buff *skb) { const struct in6_addr *saddr, *daddr; - struct tcp_md5sig_pool *hp; - struct ahash_request *req; + struct crypto_pool_ahash hp; const struct tcphdr *th =3D tcp_hdr(skb); =20 if (sk) { /* valid for establish/request sockets */ @@ -742,29 +739,27 @@ static int tcp_v6_md5_hash_skb(char *md5_hash, daddr =3D &ip6h->daddr; } =20 - hp =3D tcp_get_md5sig_pool(); - if (!hp) + if (crypto_pool_get(tcp_md5_crypto_pool_id, (struct crypto_pool *)&hp)) goto clear_hash_noput; - req =3D hp->md5_req; =20 - if (crypto_ahash_init(req)) + if (crypto_ahash_init(hp.req)) goto clear_hash; =20 - if (tcp_v6_md5_hash_headers(hp, daddr, saddr, th, skb->len)) + if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, skb->len)) goto clear_hash; - if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2)) + if (tcp_md5_hash_skb_data(&hp, skb, th->doff << 2)) goto clear_hash; - if (tcp_md5_hash_key(hp, key)) + if (tcp_md5_hash_key(&hp, key)) goto clear_hash; - ahash_request_set_crypt(req, NULL, md5_hash, 0); - if (crypto_ahash_final(req)) + ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); + if (crypto_ahash_final(hp.req)) goto clear_hash; =20 - tcp_put_md5sig_pool(); + crypto_pool_put(); return 0; =20 clear_hash: - tcp_put_md5sig_pool(); + crypto_pool_put(); clear_hash_noput: memset(md5_hash, 0, 16); return 1; --=20 2.37.2