From nobody Wed Apr 15 04:24:52 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 780AFC19F29 for ; Tue, 26 Jul 2022 20:16:29 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S239885AbiGZUQ0 (ORCPT ); Tue, 26 Jul 2022 16:16:26 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:51798 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S239837AbiGZUQQ (ORCPT ); Tue, 26 Jul 2022 16:16:16 -0400 Received: from mail-wm1-x32f.google.com (mail-wm1-x32f.google.com [IPv6:2a00:1450:4864:20::32f]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 4937B222B4 for ; Tue, 26 Jul 2022 13:16:10 -0700 (PDT) Received: by mail-wm1-x32f.google.com with SMTP id c22so9390901wmr.2 for ; Tue, 26 Jul 2022 13:16:10 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=arista.com; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=pXanxk9kkSH5IzZhG+IrzCgXJT5SaMoTxGMdgKi88Ao=; b=aOTdaJR7DRCHTIhcRrNUKxttpjYsSYzUN56FfDWDFzWSCG98auk7DNhiske4HaC3zG cS6h+UesI+/EQl/sHZpkP7AsceKUhdlWOjCCO3rXGmOU7O6FegMwB4L8sFyYH9uL0KLU UKsGcQMbRLxqcQ+tphoCYKCx6EqnRaRP9idnG+bIn08ahovsjwxGm6YMATBHXTlbjgiS fWk6KsehukN3u1IN5gy6zWnI59fMaGzddwvFRpExwBo5b4rlTSqWtz8XwLEYXUS0WeZ/ kE0+yP1n5ukWYeqlSd0NDKImOFfrKY9RToA5QaWiBhAsjw6KRTL8ApD3o9t+OickTtBz ysBw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=pXanxk9kkSH5IzZhG+IrzCgXJT5SaMoTxGMdgKi88Ao=; b=0M53j/n1VbQmu2h14MSdQDpYNJFBBz8xzmwGxDrK9UzdghPlk62msFBM4/18i+HV1O k/RP2dky2zjpRdifluapiv3o1p6zzoGEXyAKqx7+gpwzYpZWDQR4TaqpE24K1DN2fsNQ OgVAo8+c0zmz8cwB27w+Wkmjx1lhGdv8K5yPK/9JBLxyuIeEEz24Lx4WxTHFONJJgF/2 XsIEv6NBDp+hNaVgW3ksS/T0h4lcAdo4e9VMIqdWIb2Qh6DFO9wPDMjQIphk8z6lONvb idGs8qpbIDfPdyXV0XStZLEAtP6WcMocwr4ZiDmw+aEZRedAKFyIkieMkXUlG3SL56pX LtnA== X-Gm-Message-State: AJIora/0BLBZ/kbPciyEOXOeaFE6LslAPj+JfWNNnreBOZLpqtIhEv3r 7mc1D4Kd+EhmQN4PhQ8aNdaj28MJlIGCPQ== X-Google-Smtp-Source: AGRyM1vN2dLizczxw8G9+ojw1TT85gS1AkDW1LlK5/c4Vuq3soc//mjpjF9+MdBOjk0uYiaWbIipjg== X-Received: by 2002:a05:600c:2212:b0:3a3:328:5bd2 with SMTP id z18-20020a05600c221200b003a303285bd2mr568174wml.146.1658866568443; Tue, 26 Jul 2022 13:16:08 -0700 (PDT) Received: from Mindolluin.ire.aristanetworks.com ([217.173.96.166]) by smtp.gmail.com with ESMTPSA id m6-20020a05600c3b0600b003a320e6f011sm28073wms.1.2022.07.26.13.16.07 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 26 Jul 2022 13:16:08 -0700 (PDT) From: Dmitry Safonov To: linux-kernel@vger.kernel.org Cc: Dmitry Safonov <0x7f454c46@gmail.com>, Dmitry Safonov , Andy Lutomirski , Ard Biesheuvel , David Ahern , "David S. Miller" , Eric Biggers , Eric Dumazet , Francesco Ruggeri , Herbert Xu , Hideaki YOSHIFUJI , Jakub Kicinski , Leonard Crestez , Paolo Abeni , Salam Noureddine , netdev@vger.kernel.org, linux-crypto@vger.kernel.org Subject: [PATCH 1/6] crypto: Introduce crypto_pool Date: Tue, 26 Jul 2022 21:15:55 +0100 Message-Id: <20220726201600.1715505-2-dima@arista.com> X-Mailer: git-send-email 2.36.1 In-Reply-To: <20220726201600.1715505-1-dima@arista.com> References: <20220726201600.1715505-1-dima@arista.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Introduce a per-CPU pool of async crypto requests that can be used in bh-disabled contexts (designed with net RX/TX softirqs as users in mind). Allocation can sleep and is a slow-path. Initial implementation has only ahash as a backend and a fix-sized array of possible algorithms used in parallel. Signed-off-by: Dmitry Safonov --- crypto/Kconfig | 6 + crypto/Makefile | 1 + crypto/crypto_pool.c | 287 ++++++++++++++++++++++++++++++++++++++++++ include/crypto/pool.h | 34 +++++ 4 files changed, 328 insertions(+) create mode 100644 crypto/crypto_pool.c create mode 100644 include/crypto/pool.h diff --git a/crypto/Kconfig b/crypto/Kconfig index bb427a835e44..aeddaa3dcc77 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -2128,6 +2128,12 @@ config CRYPTO_STATS config CRYPTO_HASH_INFO bool =20 +config CRYPTO_POOL + tristate "Per-CPU crypto pool" + default n + help + Per-CPU pool of crypto requests ready for usage in atomic contexts. + source "drivers/crypto/Kconfig" source "crypto/asymmetric_keys/Kconfig" source "certs/Kconfig" diff --git a/crypto/Makefile b/crypto/Makefile index 167c004dbf4f..6d1d9801b76b 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -63,6 +63,7 @@ obj-$(CONFIG_CRYPTO_ACOMP2) +=3D crypto_acompress.o cryptomgr-y :=3D algboss.o testmgr.o =20 obj-$(CONFIG_CRYPTO_MANAGER2) +=3D cryptomgr.o +obj-$(CONFIG_CRYPTO_POOL) +=3D crypto_pool.o obj-$(CONFIG_CRYPTO_USER) +=3D crypto_user.o crypto_user-y :=3D crypto_user_base.o crypto_user-$(CONFIG_CRYPTO_STATS) +=3D crypto_user_stat.o diff --git a/crypto/crypto_pool.c b/crypto/crypto_pool.c new file mode 100644 index 000000000000..c668c02499b7 --- /dev/null +++ b/crypto/crypto_pool.c @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include +#include +#include +#include +#include +#include + +static unsigned long scratch_size =3D DEFAULT_CRYPTO_POOL_SCRATCH_SZ; +static DEFINE_PER_CPU(void *, crypto_pool_scratch); + +struct crypto_pool_entry { + struct ahash_request * __percpu *req; + const char *alg; + struct kref kref; + bool needs_key; +}; + +#define CPOOL_SIZE (PAGE_SIZE/sizeof(struct crypto_pool_entry)) +static struct crypto_pool_entry cpool[CPOOL_SIZE]; +static int last_allocated; +static DEFINE_MUTEX(cpool_mutex); + +static int crypto_pool_scratch_alloc(void) +{ + int cpu; + + lockdep_assert_held(&cpool_mutex); + + for_each_possible_cpu(cpu) { + void *scratch =3D per_cpu(crypto_pool_scratch, cpu); + + if (scratch) + continue; + + scratch =3D kmalloc_node(scratch_size, GFP_KERNEL, + cpu_to_node(cpu)); + if (!scratch) + return -ENOMEM; + per_cpu(crypto_pool_scratch, cpu) =3D scratch; + } + return 0; +} + +static void crypto_pool_scratch_free(void) +{ + int cpu; + + lockdep_assert_held(&cpool_mutex); + + for_each_possible_cpu(cpu) { + void *scratch =3D per_cpu(crypto_pool_scratch, cpu); + + if (!scratch) + continue; + per_cpu(crypto_pool_scratch, cpu) =3D NULL; + kfree(scratch); + } +} + +static int __cpool_alloc_ahash(struct crypto_pool_entry *e, const char *al= g) +{ + struct crypto_ahash *hash; + int cpu, ret =3D -ENOMEM; + + e->alg =3D kstrdup(alg, GFP_KERNEL); + if (!e->alg) + return -ENOMEM; + + e->req =3D alloc_percpu(struct ahash_request *); + if (!e->req) + goto out_free_alg; + + hash =3D crypto_alloc_ahash(alg, 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(hash)) { + ret =3D PTR_ERR(hash); + goto out_free_req; + } + + /* If hash has .setkey(), allocate ahash per-cpu, not only request */ + e->needs_key =3D crypto_ahash_get_flags(hash) & CRYPTO_TFM_NEED_KEY; + + for_each_possible_cpu(cpu) { + struct ahash_request *req; + + if (!hash) + hash =3D crypto_alloc_ahash(alg, 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(hash)) + goto out_free; + + req =3D ahash_request_alloc(hash, GFP_KERNEL); + if (!req) + goto out_free; + + ahash_request_set_callback(req, 0, NULL, NULL); + + *per_cpu_ptr(e->req, cpu) =3D req; + + if (e->needs_key) + hash =3D NULL; + } + kref_init(&e->kref); + return 0; + +out_free: + if (!IS_ERR_OR_NULL(hash) && e->needs_key) + crypto_free_ahash(hash); + + for_each_possible_cpu(cpu) { + if (*per_cpu_ptr(e->req, cpu) =3D=3D NULL) + break; + hash =3D crypto_ahash_reqtfm(*per_cpu_ptr(e->req, cpu)); + ahash_request_free(*per_cpu_ptr(e->req, cpu)); + if (e->needs_key) { + crypto_free_ahash(hash); + hash =3D NULL; + } + } + + if (hash) + crypto_free_ahash(hash); +out_free_req: + free_percpu(e->req); +out_free_alg: + kfree(e->alg); + e->alg =3D NULL; + return ret; +} + +/** + * crypto_pool_alloc_ahash - allocates pool for ahash requests + * @alg: name of async hash algorithm + */ +int crypto_pool_alloc_ahash(const char *alg) +{ + unsigned int i; + int err; + + /* slow-path */ + mutex_lock(&cpool_mutex); + err =3D crypto_pool_scratch_alloc(); + if (err) + goto out; + + for (i =3D 0; i < last_allocated; i++) { + if (cpool[i].alg && !strcmp(cpool[i].alg, alg)) { + kref_get(&cpool[i].kref); + goto out; + } + } + + for (i =3D 0; i < last_allocated; i++) { + if (!cpool[i].alg) + break; + } + if (i >=3D CPOOL_SIZE) { + err =3D -ENOSPC; + goto out; + } + + err =3D __cpool_alloc_ahash(&cpool[i], alg); + if (!err && last_allocated <=3D i) + last_allocated++; +out: + mutex_unlock(&cpool_mutex); + return err ?: (int)i; +} +EXPORT_SYMBOL_GPL(crypto_pool_alloc_ahash); + +static void __cpool_free_entry(struct crypto_pool_entry *e) +{ + struct crypto_ahash *hash =3D NULL; + int cpu; + + for_each_possible_cpu(cpu) { + if (*per_cpu_ptr(e->req, cpu) =3D=3D NULL) + continue; + + hash =3D crypto_ahash_reqtfm(*per_cpu_ptr(e->req, cpu)); + ahash_request_free(*per_cpu_ptr(e->req, cpu)); + if (e->needs_key) { + crypto_free_ahash(hash); + hash =3D NULL; + } + } + if (hash) + crypto_free_ahash(hash); + free_percpu(e->req); + kfree(e->alg); + memset(e, 0, sizeof(*e)); +} + +static void cpool_cleanup_work_cb(struct work_struct *work) +{ + unsigned int i; + bool free_scratch =3D true; + + mutex_lock(&cpool_mutex); + for (i =3D 0; i < last_allocated; i++) { + if (kref_read(&cpool[i].kref) > 0) { + free_scratch =3D false; + continue; + } + if (!cpool[i].alg) + continue; + __cpool_free_entry(&cpool[i]); + } + if (free_scratch) + crypto_pool_scratch_free(); + mutex_unlock(&cpool_mutex); +} + +static DECLARE_WORK(cpool_cleanup_work, cpool_cleanup_work_cb); +static void cpool_schedule_cleanup(struct kref *kref) +{ + schedule_work(&cpool_cleanup_work); +} + +/** + * crypto_pool_release - decreases number of users for a pool. If it was + * the last user of the pool, releases any memory that was consumed. + * @id: crypto_pool that was previously allocated by crypto_pool_alloc_aha= sh() + */ +void crypto_pool_release(unsigned int id) +{ + if (WARN_ON_ONCE(id > last_allocated || !cpool[id].alg)) + return; + + /* slow-path */ + kref_put(&cpool[id].kref, cpool_schedule_cleanup); +} +EXPORT_SYMBOL_GPL(crypto_pool_release); + +/** + * crypto_pool_add - increases number of users (refcounter) for a pool + * @id: crypto_pool that was previously allocated by crypto_pool_alloc_aha= sh() + */ +void crypto_pool_add(unsigned int id) +{ + if (WARN_ON_ONCE(id > last_allocated || !cpool[id].alg)) + return; + kref_get(&cpool[id].kref); +} +EXPORT_SYMBOL_GPL(crypto_pool_add); + +/** + * crypto_pool_get - disable bh and start using crypto_pool + * @id: crypto_pool that was previously allocated by crypto_pool_alloc_aha= sh() + * @c: returned crypto_pool for usage (uninitialized on failure) + */ +int crypto_pool_get(unsigned int id, struct crypto_pool *c) +{ + struct crypto_pool_ahash *ret =3D (struct crypto_pool_ahash *)c; + + local_bh_disable(); + if (WARN_ON_ONCE(id > last_allocated || !cpool[id].alg)) { + local_bh_enable(); + return -EINVAL; + } + ret->req =3D *this_cpu_ptr(cpool[id].req); + ret->base.scratch =3D this_cpu_read(crypto_pool_scratch); + return 0; +} +EXPORT_SYMBOL_GPL(crypto_pool_get); + +/** + * crypto_pool_algo - return algorithm of crypto_pool + * @id: crypto_pool that was previously allocated by crypto_pool_alloc_aha= sh() + * @buf: buffer to return name of algorithm + * @buf_len: size of @buf + */ +size_t crypto_pool_algo(unsigned int id, char *buf, size_t buf_len) +{ + size_t ret =3D 0; + + /* slow-path */ + mutex_lock(&cpool_mutex); + if (cpool[id].alg) + ret =3D strscpy(buf, cpool[id].alg, buf_len); + mutex_unlock(&cpool_mutex); + return ret; +} +EXPORT_SYMBOL_GPL(crypto_pool_algo); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Per-CPU pool of crypto requests"); diff --git a/include/crypto/pool.h b/include/crypto/pool.h new file mode 100644 index 000000000000..2c61aa45faff --- /dev/null +++ b/include/crypto/pool.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _CRYPTO_POOL_H +#define _CRYPTO_POOL_H + +#include + +#define DEFAULT_CRYPTO_POOL_SCRATCH_SZ 128 + +struct crypto_pool { + void *scratch; +}; + +/* + * struct crypto_pool_ahash - per-CPU pool of ahash_requests + * @base: common members that can be used by any async crypto ops + * @req: pre-allocated ahash request + */ +struct crypto_pool_ahash { + struct crypto_pool base; + struct ahash_request *req; +}; + +int crypto_pool_alloc_ahash(const char *alg); +void crypto_pool_add(unsigned int id); +void crypto_pool_release(unsigned int id); + +int crypto_pool_get(unsigned int id, struct crypto_pool *c); +static inline void crypto_pool_put(void) +{ + local_bh_enable(); +} +size_t crypto_pool_algo(unsigned int id, char *buf, size_t buf_len); + +#endif /* _CRYPTO_POOL_H */ --=20 2.36.1 From nobody Wed Apr 15 04:24:53 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 0E942C00140 for ; Tue, 26 Jul 2022 20:16:32 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S239894AbiGZUQa (ORCPT ); Tue, 26 Jul 2022 16:16:30 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:51820 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S239849AbiGZUQR (ORCPT ); Tue, 26 Jul 2022 16:16:17 -0400 Received: from mail-wm1-x330.google.com (mail-wm1-x330.google.com [IPv6:2a00:1450:4864:20::330]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 90457326F7 for ; Tue, 26 Jul 2022 13:16:11 -0700 (PDT) Received: by mail-wm1-x330.google.com with SMTP id c22so9390919wmr.2 for ; Tue, 26 Jul 2022 13:16:11 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=arista.com; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=GWR8XbBS1oy+lmPSjXOdFIQ7ZMLcTchWH7fRG8D5Oe4=; b=LyKNGMWZNujT6s9/QUWJ2JVBwtpyqqQwZJlDQaKuI+upIlpRhAy3Nj9WwePSSu8pUl yA3q3wn8gtvE5mz8rffc9gM4xJMJrCu8GPLJvcn6Mn+O9ZZAgFc3GCzL64tC11ct6TmK PtQjdgAMGzKrapZECA0xSYFxrtDcXAsfd/IzTcjArOvGuVkEP4LGMSY4fWdrEnyQW4fN J2BOgLkTRSIKF+kKYCweRPV0ABuWIEOrUjWwqN3/Aaegd6rAWAOw6wGm0RtJdgPX/pJv 0RLBc24wp3AJaUS1Jd7q4puaPs4XkhODkccEz8DTMVRjj1HEvj3e5YP5EqWuR+eX339l li5A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=GWR8XbBS1oy+lmPSjXOdFIQ7ZMLcTchWH7fRG8D5Oe4=; b=GWC1l1JFdxsaG5tKRsUeZo9k4VMWrVM1e5I0KwQPFzMKEpCWN7xIvTt8YT5//eif7P bG/iOJYatbMsppRKEnT/zvqMNbyyKDDZeFvnTjpdoQWIoiwvbiyWzOtg9/ddTF9Wb9Fs 8hsXDEE//llwSq6h7yGj8TEv5X+KThwHMFSqeR8Zuk/V3t4o6k6jEhAxQgGcqTb1Qup6 6w80cat7GarBhDNoLFLRVhlk/zb+rg53UviB/QkZv3xnzmyT+gJdiPRNMIIWZdfuXHDL /UoLR298MqF83lw5TqyRHhb/2H5fqRvRseDKK19Fkdm6uFCphVl2ro3qe7EQrCdSA0Tn 5/aQ== X-Gm-Message-State: AJIora+SOWU4dwrUs0GmfgjIbaz+zJFIbjyFM00s54ffjGYBWG9kC3qr vcFSMq6xR32FG+Nf2kWnCiG7lXwU3IB72Q== X-Google-Smtp-Source: AGRyM1sTDJOnzvOn5GuKOaCCikjaJbFUgi0rTM+scygEVQdwUbMQzlcukhe0K/rXy0i8Vr8g6LcDsg== X-Received: by 2002:a7b:c7d1:0:b0:3a3:1890:3495 with SMTP id z17-20020a7bc7d1000000b003a318903495mr591785wmk.18.1658866569657; Tue, 26 Jul 2022 13:16:09 -0700 (PDT) Received: from Mindolluin.ire.aristanetworks.com ([217.173.96.166]) by smtp.gmail.com with ESMTPSA id m6-20020a05600c3b0600b003a320e6f011sm28073wms.1.2022.07.26.13.16.08 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 26 Jul 2022 13:16:09 -0700 (PDT) From: Dmitry Safonov To: linux-kernel@vger.kernel.org Cc: Dmitry Safonov <0x7f454c46@gmail.com>, Dmitry Safonov , Andy Lutomirski , Ard Biesheuvel , David Ahern , "David S. Miller" , Eric Biggers , Eric Dumazet , Francesco Ruggeri , Herbert Xu , Hideaki YOSHIFUJI , Jakub Kicinski , Leonard Crestez , Paolo Abeni , Salam Noureddine , netdev@vger.kernel.org, linux-crypto@vger.kernel.org Subject: [PATCH 2/6] crypto_pool: Add crypto_pool_reserve_scratch() Date: Tue, 26 Jul 2022 21:15:56 +0100 Message-Id: <20220726201600.1715505-3-dima@arista.com> X-Mailer: git-send-email 2.36.1 In-Reply-To: <20220726201600.1715505-1-dima@arista.com> References: <20220726201600.1715505-1-dima@arista.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Instead of having build-time hardcoded constant, reallocate scratch area, if needed by user. Different algos, different users may need different size of temp per-CPU buffer. Only up-sizing supported for simplicity. Signed-off-by: Dmitry Safonov --- crypto/Kconfig | 6 ++++ crypto/crypto_pool.c | 65 +++++++++++++++++++++++++++++++------------ include/crypto/pool.h | 3 +- 3 files changed, 54 insertions(+), 20 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index aeddaa3dcc77..e5865be483be 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -2134,6 +2134,12 @@ config CRYPTO_POOL help Per-CPU pool of crypto requests ready for usage in atomic contexts. =20 +config CRYPTO_POOL_DEFAULT_SCRATCH_SIZE + hex "Per-CPU default scratch area size" + depends on CRYPTO_POOL + default 0x100 + range 0x100 0x10000 + source "drivers/crypto/Kconfig" source "crypto/asymmetric_keys/Kconfig" source "certs/Kconfig" diff --git a/crypto/crypto_pool.c b/crypto/crypto_pool.c index c668c02499b7..8ad6415fa817 100644 --- a/crypto/crypto_pool.c +++ b/crypto/crypto_pool.c @@ -1,13 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-or-later =20 #include +#include #include #include #include #include #include =20 -static unsigned long scratch_size =3D DEFAULT_CRYPTO_POOL_SCRATCH_SZ; +static unsigned long scratch_size =3D CONFIG_CRYPTO_POOL_DEFAULT_SCRATCH_S= IZE; static DEFINE_PER_CPU(void *, crypto_pool_scratch); =20 struct crypto_pool_entry { @@ -19,28 +20,60 @@ struct crypto_pool_entry { =20 #define CPOOL_SIZE (PAGE_SIZE/sizeof(struct crypto_pool_entry)) static struct crypto_pool_entry cpool[CPOOL_SIZE]; -static int last_allocated; +static unsigned int last_allocated; static DEFINE_MUTEX(cpool_mutex); =20 -static int crypto_pool_scratch_alloc(void) +static void __set_scratch(void *scratch) { - int cpu; + kfree(this_cpu_read(crypto_pool_scratch)); + this_cpu_write(crypto_pool_scratch, scratch); +} =20 - lockdep_assert_held(&cpool_mutex); +/* Slow-path */ +/** + * crypto_pool_reserve_scratch - re-allocates scratch buffer, slow-path + * @size: request size for the scratch/temp buffer + */ +int crypto_pool_reserve_scratch(unsigned long size) +{ + int cpu, err =3D 0; =20 + mutex_lock(&cpool_mutex); + if (size <=3D scratch_size) { + for_each_possible_cpu(cpu) { + if (per_cpu(crypto_pool_scratch, cpu)) + continue; + goto allocate_scratch; + } + mutex_unlock(&cpool_mutex); + return 0; + } +allocate_scratch: + cpus_read_lock(); for_each_possible_cpu(cpu) { - void *scratch =3D per_cpu(crypto_pool_scratch, cpu); + void *scratch; =20 - if (scratch) - continue; + scratch =3D kmalloc_node(size, GFP_KERNEL, cpu_to_node(cpu)); + if (!scratch) { + err =3D -ENOMEM; + break; + } =20 - scratch =3D kmalloc_node(scratch_size, GFP_KERNEL, - cpu_to_node(cpu)); - if (!scratch) - return -ENOMEM; - per_cpu(crypto_pool_scratch, cpu) =3D scratch; + if (!cpu_online(cpu)) { + kfree(per_cpu(crypto_pool_scratch, cpu)); + per_cpu(crypto_pool_scratch, cpu) =3D scratch; + continue; + } + err =3D smp_call_function_single(cpu, __set_scratch, scratch, 1); + if (err) { + kfree(scratch); + break; + } } - return 0; + + cpus_read_unlock(); + mutex_unlock(&cpool_mutex); + return err; } =20 static void crypto_pool_scratch_free(void) @@ -139,10 +172,6 @@ int crypto_pool_alloc_ahash(const char *alg) =20 /* slow-path */ mutex_lock(&cpool_mutex); - err =3D crypto_pool_scratch_alloc(); - if (err) - goto out; - for (i =3D 0; i < last_allocated; i++) { if (cpool[i].alg && !strcmp(cpool[i].alg, alg)) { kref_get(&cpool[i].kref); diff --git a/include/crypto/pool.h b/include/crypto/pool.h index 2c61aa45faff..c7d817860cc3 100644 --- a/include/crypto/pool.h +++ b/include/crypto/pool.h @@ -4,8 +4,6 @@ =20 #include =20 -#define DEFAULT_CRYPTO_POOL_SCRATCH_SZ 128 - struct crypto_pool { void *scratch; }; @@ -20,6 +18,7 @@ struct crypto_pool_ahash { struct ahash_request *req; }; =20 +int crypto_pool_reserve_scratch(unsigned long size); int crypto_pool_alloc_ahash(const char *alg); void crypto_pool_add(unsigned int id); void crypto_pool_release(unsigned int id); --=20 2.36.1 From nobody Wed Apr 15 04:24:53 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id A2B5FC19F29 for ; Tue, 26 Jul 2022 20:16:36 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S239906AbiGZUQe (ORCPT ); Tue, 26 Jul 2022 16:16:34 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:51986 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S239854AbiGZUQV (ORCPT ); Tue, 26 Jul 2022 16:16:21 -0400 Received: from mail-wr1-x430.google.com (mail-wr1-x430.google.com [IPv6:2a00:1450:4864:20::430]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 0A6B821E16 for ; Tue, 26 Jul 2022 13:16:12 -0700 (PDT) Received: by mail-wr1-x430.google.com with SMTP id h8so21586878wrw.1 for ; Tue, 26 Jul 2022 13:16:12 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=arista.com; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc; bh=pdEmRQDkktxyj133IoEXqOneVb48tak5Ed7jC2tmPss=; b=NoSqp+M8tdLz1sYdFPgF7sEQXLSNybM3qMMiMI+8XdUdNnT/1ziUDTfJOPtzM2pf7B 60eDbGh5QPE2+3VRM5/liw90GlJa4dRAWD35fnQU46tW6uTxxrgvQzAkvdyK7YDJ2BLL S3yc9XpH15mLCEnOqP5tHgO4eztj0vRCO/Lc/AY/QTEDwvOWN66UwOj7wv9/l0cdHWwH Z1elrL0lZEiHkoxXg3msEDCkqQxfTT0XD+8ar1CJzEgclE0dYP0wH9K0jeeflUXT27Ct S51HGosABXnq1obGhEZfcrAIwK2QXaf24/I71b2B99nO+tCN3RNfD/A+47RYIDlq+xYO I1fw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc; bh=pdEmRQDkktxyj133IoEXqOneVb48tak5Ed7jC2tmPss=; b=TIbp7Nnf8YelSYxFF0+vaY80VYWRvyP+ICcR5+DFwVyWZqU7h+1tNjDbK9HPbaSbBX v9sv6PtIHXurxBBM+FVcBrQ/yTMUnGjX2Dq+108WkIK3caWaBex+YJM36FCkO3sSt8Gr L7uBorT+uBZluP12M4q6g64ohA8F6kEQ/XlRcfVYraB7k3HQcnL4PzGxiYmR/oGKHU7X Gjod6WxogkYTU0MrtynA7mJxKJX06sWE9fFgAHvbqhZm3XIyjR5w0urDRGH3R7DhGDNW TSI7Xd/uLLiL7fw6L0A3aKjLcMl/py7xnDgYm+WZ+t7OAbUdRRtDG0U8/4zW3EBkBuqo GuVQ== X-Gm-Message-State: AJIora/1Hn6rkOOh3+oq5+laGWDFffXbvLQm/E0N5g0eN+YhYMFNsZpz bXmFZSyhuZevMyzZamJxm4E6TEyq8eQ5Bg== X-Google-Smtp-Source: AGRyM1uQRBcHLiZ1Y9LPpdW7z50O+MQLPM9/unZYKIWc040bRCMVRnpd/+MrBgLnu3o3KKE4D15qKg== X-Received: by 2002:a5d:5581:0:b0:20f:fc51:7754 with SMTP id i1-20020a5d5581000000b0020ffc517754mr12412279wrv.413.1658866571059; Tue, 26 Jul 2022 13:16:11 -0700 (PDT) Received: from Mindolluin.ire.aristanetworks.com ([217.173.96.166]) by smtp.gmail.com with ESMTPSA id m6-20020a05600c3b0600b003a320e6f011sm28073wms.1.2022.07.26.13.16.09 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 26 Jul 2022 13:16:10 -0700 (PDT) From: Dmitry Safonov To: linux-kernel@vger.kernel.org Cc: Dmitry Safonov <0x7f454c46@gmail.com>, Dmitry Safonov , Andy Lutomirski , Ard Biesheuvel , David Ahern , "David S. Miller" , Eric Biggers , Eric Dumazet , Francesco Ruggeri , Herbert Xu , Hideaki YOSHIFUJI , Jakub Kicinski , Leonard Crestez , Paolo Abeni , Salam Noureddine , netdev@vger.kernel.org, linux-crypto@vger.kernel.org Subject: [PATCH 3/6] net/tcp: Separate tcp_md5sig_info allocation into tcp_md5sig_info_add() Date: Tue, 26 Jul 2022 21:15:57 +0100 Message-Id: <20220726201600.1715505-4-dima@arista.com> X-Mailer: git-send-email 2.36.1 In-Reply-To: <20220726201600.1715505-1-dima@arista.com> References: <20220726201600.1715505-1-dima@arista.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Add a helper to allocate tcp_md5sig_info, that will help later to do/allocate things when info allocated, once per socket. Signed-off-by: Dmitry Safonov --- net/ipv4/tcp_ipv4.c | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 0c83780dc9bf..55e4092209a5 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -1152,6 +1152,24 @@ struct tcp_md5sig_key *tcp_v4_md5_lookup(const struc= t sock *sk, } EXPORT_SYMBOL(tcp_v4_md5_lookup); =20 +static int tcp_md5sig_info_add(struct sock *sk, gfp_t gfp) +{ + struct tcp_sock *tp =3D tcp_sk(sk); + struct tcp_md5sig_info *md5sig; + + if (rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) + return 0; + + md5sig =3D kmalloc(sizeof(*md5sig), gfp); + if (!md5sig) + return -ENOMEM; + + sk_gso_disable(sk); + INIT_HLIST_HEAD(&md5sig->head); + rcu_assign_pointer(tp->md5sig_info, md5sig); + return 0; +} + /* This can be called on a newly created socket, from other files */ int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, int family, u8 prefixlen, int l3index, u8 flags, @@ -1182,17 +1200,11 @@ int tcp_md5_do_add(struct sock *sk, const union tcp= _md5_addr *addr, return 0; } =20 + if (tcp_md5sig_info_add(sk, gfp)) + return -ENOMEM; + md5sig =3D rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk)); - if (!md5sig) { - md5sig =3D kmalloc(sizeof(*md5sig), gfp); - if (!md5sig) - return -ENOMEM; - - sk_gso_disable(sk); - INIT_HLIST_HEAD(&md5sig->head); - rcu_assign_pointer(tp->md5sig_info, md5sig); - } =20 key =3D sock_kmalloc(sk, sizeof(*key), gfp | __GFP_ZERO); if (!key) --=20 2.36.1 From nobody Wed Apr 15 04:24:53 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id A8542C04A68 for ; Tue, 26 Jul 2022 20:16:56 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S239926AbiGZUQy (ORCPT ); Tue, 26 Jul 2022 16:16:54 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:52006 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S239865AbiGZUQW (ORCPT ); Tue, 26 Jul 2022 16:16:22 -0400 Received: from mail-wr1-x429.google.com (mail-wr1-x429.google.com [IPv6:2a00:1450:4864:20::429]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 2EEF832B85 for ; Tue, 26 Jul 2022 13:16:14 -0700 (PDT) Received: by mail-wr1-x429.google.com with SMTP id h9so21649824wrm.0 for ; Tue, 26 Jul 2022 13:16:14 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=arista.com; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc; bh=pmSO5o3Q44tqKwKFfKNMm0Ze8ef3CVpVV9CI9yXzviE=; b=D5lUTewwVLHHGjzMcNus5GsYhY5VOyJ5EbsBDumqrRq9zpSv8LL+NILNY+idc7V+jy gwssJlDW4770QDmmsH1d8Z2O7J/OOre7FLr4zdjpP6erd/N5/yx5S7DUdz/51yqjxP1F XkQpdIlAPcuuU9EK2Hp55iUVvyctYxw5oBCKEMSQJ7N5BNayUJNb8F5yjlmA8z28kUVP A/AzsOtOML497PIrxw9l3CfdJCNw9/gT3tP0R6Td5iUkT08C2NtmMEKETjiSeQZku+3s IoRefq1rlqZP+SVAhReLcNqCIZfD07wbzR0r9SkPpyEzb0AaR/oVqFPa8A4oqfVyQGOU pNtg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc; bh=pmSO5o3Q44tqKwKFfKNMm0Ze8ef3CVpVV9CI9yXzviE=; b=kvNNkSaV1CcossabpbhKPA2q2iVgk59G5W4G8vdavwgVh5vRXmKLhsPPOFDmO3ruRA fY3V5EW4rmIQjLvfD3j5EzTXNCNe2vceOjhJrob35NOiTI4tsj+0H+8k6Anxnr0mUwin pcyQWvo+SNJpLMff6Ixp7fsMoqCWW/5ld7pVbn1ztVdWs2SzNFu4oq/TVoLjucEFoKgO iHpzUulmf5+LnOV0MfzfGCgESfCakdFBE2JVaywcOHjz2JkpzqMvc3/JUIZKg5Po49Tw nfCHE7UTO5/rJj6uldDj+Uy1RMafeI1SPAcDN6hBdKTwB4gHpmR/x7VGHbEFOIVaTuK1 2llA== X-Gm-Message-State: AJIora/3vq3GSD5cmEGm1ECD/BPhY0MiAs3NFM9ZP7AllXAnM4nGopI1 1TAE+BDbThaHiFAcza8kG604KvDM/DJmsg== X-Google-Smtp-Source: AGRyM1uB79Y9Ri2seTNWcO7Gq0vlx1PC/AF/5zeWVVl++TFNfM6zhF2Jc/Q/fwlASGpGWU8eiPG99Q== X-Received: by 2002:a05:6000:986:b0:21e:939b:a71 with SMTP id by6-20020a056000098600b0021e939b0a71mr5780694wrb.256.1658866572371; Tue, 26 Jul 2022 13:16:12 -0700 (PDT) Received: from Mindolluin.ire.aristanetworks.com ([217.173.96.166]) by smtp.gmail.com with ESMTPSA id m6-20020a05600c3b0600b003a320e6f011sm28073wms.1.2022.07.26.13.16.11 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 26 Jul 2022 13:16:11 -0700 (PDT) From: Dmitry Safonov To: linux-kernel@vger.kernel.org Cc: Dmitry Safonov <0x7f454c46@gmail.com>, Dmitry Safonov , Andy Lutomirski , Ard Biesheuvel , David Ahern , "David S. Miller" , Eric Biggers , Eric Dumazet , Francesco Ruggeri , Herbert Xu , Hideaki YOSHIFUJI , Jakub Kicinski , Leonard Crestez , Paolo Abeni , Salam Noureddine , netdev@vger.kernel.org, linux-crypto@vger.kernel.org Subject: [PATCH 4/6] net/tcp: Disable TCP-MD5 static key on tcp_md5sig_info destruction Date: Tue, 26 Jul 2022 21:15:58 +0100 Message-Id: <20220726201600.1715505-5-dima@arista.com> X-Mailer: git-send-email 2.36.1 In-Reply-To: <20220726201600.1715505-1-dima@arista.com> References: <20220726201600.1715505-1-dima@arista.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" To do that, separate two scenarios: - where it's the first MD5 key on the system, which means that enabling of the static key may need to sleep; - copying of an existing key from a listening socket to the request socket upon receiving a signed TCP segment, where static key was already enabled (when the key was added to the listening socket). Now the life-time of the static branch for TCP-MD5 is until: - last tcp_md5sig_info is destroyed - last socket in time-wait state with MD5 key is closed. Which means that after all sockets with TCP-MD5 keys are gone, the system gets back the performance of disabled md5-key static branch. Signed-off-by: Dmitry Safonov Reported-by: kernel test robot --- include/net/tcp.h | 10 ++++++--- net/ipv4/tcp.c | 5 +---- net/ipv4/tcp_ipv4.c | 45 +++++++++++++++++++++++++++++++--------- net/ipv4/tcp_minisocks.c | 9 +++++--- net/ipv4/tcp_output.c | 4 ++-- net/ipv6/tcp_ipv6.c | 10 ++++----- 6 files changed, 55 insertions(+), 28 deletions(-) diff --git a/include/net/tcp.h b/include/net/tcp.h index f366626cbbba..aa735f963723 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -1663,7 +1663,11 @@ int tcp_v4_md5_hash_skb(char *md5_hash, const struct= tcp_md5sig_key *key, const struct sock *sk, const struct sk_buff *skb); int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, int family, u8 prefixlen, int l3index, u8 flags, - const u8 *newkey, u8 newkeylen, gfp_t gfp); + const u8 *newkey, u8 newkeylen); +int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr, + int family, u8 prefixlen, int l3index, + struct tcp_md5sig_key *key); + int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family, u8 prefixlen, int l3index, u8 flags); struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk, @@ -1671,7 +1675,7 @@ struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct= sock *sk, =20 #ifdef CONFIG_TCP_MD5SIG #include -extern struct static_key_false tcp_md5_needed; +extern struct static_key_false_deferred tcp_md5_needed; struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3in= dex, const union tcp_md5_addr *addr, int family); @@ -1679,7 +1683,7 @@ static inline struct tcp_md5sig_key * tcp_md5_do_lookup(const struct sock *sk, int l3index, const union tcp_md5_addr *addr, int family) { - if (!static_branch_unlikely(&tcp_md5_needed)) + if (!static_branch_unlikely(&tcp_md5_needed.key)) return NULL; return __tcp_md5_do_lookup(sk, l3index, addr, family); } diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 635c6782cdbb..300056008e90 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -4404,11 +4404,8 @@ bool tcp_alloc_md5sig_pool(void) if (unlikely(!tcp_md5sig_pool_populated)) { mutex_lock(&tcp_md5sig_mutex); =20 - if (!tcp_md5sig_pool_populated) { + if (!tcp_md5sig_pool_populated) __tcp_alloc_md5sig_pool(); - if (tcp_md5sig_pool_populated) - static_branch_inc(&tcp_md5_needed); - } =20 mutex_unlock(&tcp_md5sig_mutex); } diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 55e4092209a5..5b0caaea5029 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -1044,7 +1044,7 @@ static void tcp_v4_reqsk_destructor(struct request_so= ck *req) * We need to maintain these in the sk structure. */ =20 -DEFINE_STATIC_KEY_FALSE(tcp_md5_needed); +DEFINE_STATIC_KEY_DEFERRED_FALSE(tcp_md5_needed, HZ); EXPORT_SYMBOL(tcp_md5_needed); =20 static bool better_md5_match(struct tcp_md5sig_key *old, struct tcp_md5sig= _key *new) @@ -1171,9 +1171,9 @@ static int tcp_md5sig_info_add(struct sock *sk, gfp_t= gfp) } =20 /* This can be called on a newly created socket, from other files */ -int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, - int family, u8 prefixlen, int l3index, u8 flags, - const u8 *newkey, u8 newkeylen, gfp_t gfp) +int __tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, + int family, u8 prefixlen, int l3index, u8 flags, + const u8 *newkey, u8 newkeylen, gfp_t gfp) { /* Add Key to the list */ struct tcp_md5sig_key *key; @@ -1200,9 +1200,6 @@ int tcp_md5_do_add(struct sock *sk, const union tcp_m= d5_addr *addr, return 0; } =20 - if (tcp_md5sig_info_add(sk, gfp)) - return -ENOMEM; - md5sig =3D rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk)); =20 @@ -1226,8 +1223,36 @@ int tcp_md5_do_add(struct sock *sk, const union tcp_= md5_addr *addr, hlist_add_head_rcu(&key->node, &md5sig->head); return 0; } + +int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, + int family, u8 prefixlen, int l3index, u8 flags, + const u8 *newkey, u8 newkeylen) +{ + if (tcp_md5sig_info_add(sk, GFP_KERNEL)) + return -ENOMEM; + + static_branch_inc(&tcp_md5_needed.key); + + return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index, flags, + newkey, newkeylen, GFP_KERNEL); +} EXPORT_SYMBOL(tcp_md5_do_add); =20 +int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr, + int family, u8 prefixlen, int l3index, + struct tcp_md5sig_key *key) +{ + if (tcp_md5sig_info_add(sk, sk_gfp_mask(sk, GFP_ATOMIC))) + return -ENOMEM; + + atomic_inc(&tcp_md5_needed.key.key.enabled); + + return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index, + key->flags, key->key, key->keylen, + sk_gfp_mask(sk, GFP_ATOMIC)); +} +EXPORT_SYMBOL(tcp_md5_key_copy); + int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int fa= mily, u8 prefixlen, int l3index, u8 flags) { @@ -1314,7 +1339,7 @@ static int tcp_v4_parse_md5_keys(struct sock *sk, int= optname, return -EINVAL; =20 return tcp_md5_do_add(sk, addr, AF_INET, prefixlen, l3index, flags, - cmd.tcpm_key, cmd.tcpm_keylen, GFP_KERNEL); + cmd.tcpm_key, cmd.tcpm_keylen); } =20 static int tcp_v4_md5_hash_headers(struct tcp_md5sig_pool *hp, @@ -1571,8 +1596,7 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *= sk, struct sk_buff *skb, * memory, then we end up not copying the key * across. Shucks. */ - tcp_md5_do_add(newsk, addr, AF_INET, 32, l3index, key->flags, - key->key, key->keylen, GFP_ATOMIC); + tcp_md5_key_copy(newsk, addr, AF_INET, 32, l3index, key); sk_gso_disable(newsk); } #endif @@ -2260,6 +2284,7 @@ void tcp_v4_destroy_sock(struct sock *sk) tcp_clear_md5_list(sk); kfree_rcu(rcu_dereference_protected(tp->md5sig_info, 1), rcu); tp->md5sig_info =3D NULL; + static_branch_slow_dec_deferred(&tcp_md5_needed); } #endif =20 diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index cb95d88497ae..5d475a45a478 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -291,13 +291,14 @@ void tcp_time_wait(struct sock *sk, int state, int ti= meo) */ do { tcptw->tw_md5_key =3D NULL; - if (static_branch_unlikely(&tcp_md5_needed)) { + if (static_branch_unlikely(&tcp_md5_needed.key)) { struct tcp_md5sig_key *key; =20 key =3D tp->af_specific->md5_lookup(sk, sk); if (key) { tcptw->tw_md5_key =3D kmemdup(key, sizeof(*key), GFP_ATOMIC); BUG_ON(tcptw->tw_md5_key && !tcp_alloc_md5sig_pool()); + atomic_inc(&tcp_md5_needed.key.key.enabled); } } } while (0); @@ -337,11 +338,13 @@ EXPORT_SYMBOL(tcp_time_wait); void tcp_twsk_destructor(struct sock *sk) { #ifdef CONFIG_TCP_MD5SIG - if (static_branch_unlikely(&tcp_md5_needed)) { + if (static_branch_unlikely(&tcp_md5_needed.key)) { struct tcp_timewait_sock *twsk =3D tcp_twsk(sk); =20 - if (twsk->tw_md5_key) + if (twsk->tw_md5_key) { kfree_rcu(twsk->tw_md5_key, rcu); + static_branch_slow_dec_deferred(&tcp_md5_needed); + } } #endif } diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 78b654ff421b..9e12845a8758 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -766,7 +766,7 @@ static unsigned int tcp_syn_options(struct sock *sk, st= ruct sk_buff *skb, =20 *md5 =3D NULL; #ifdef CONFIG_TCP_MD5SIG - if (static_branch_unlikely(&tcp_md5_needed) && + if (static_branch_unlikely(&tcp_md5_needed.key) && rcu_access_pointer(tp->md5sig_info)) { *md5 =3D tp->af_specific->md5_lookup(sk, sk); if (*md5) { @@ -922,7 +922,7 @@ static unsigned int tcp_established_options(struct sock= *sk, struct sk_buff *skb =20 *md5 =3D NULL; #ifdef CONFIG_TCP_MD5SIG - if (static_branch_unlikely(&tcp_md5_needed) && + if (static_branch_unlikely(&tcp_md5_needed.key) && rcu_access_pointer(tp->md5sig_info)) { *md5 =3D tp->af_specific->md5_lookup(sk, sk); if (*md5) { diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index e54eee80ce5f..cb891a71db0d 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -658,12 +658,11 @@ static int tcp_v6_parse_md5_keys(struct sock *sk, int= optname, if (ipv6_addr_v4mapped(&sin6->sin6_addr)) return tcp_md5_do_add(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr= 32[3], AF_INET, prefixlen, l3index, flags, - cmd.tcpm_key, cmd.tcpm_keylen, - GFP_KERNEL); + cmd.tcpm_key, cmd.tcpm_keylen); =20 return tcp_md5_do_add(sk, (union tcp_md5_addr *)&sin6->sin6_addr, AF_INET6, prefixlen, l3index, flags, - cmd.tcpm_key, cmd.tcpm_keylen, GFP_KERNEL); + cmd.tcpm_key, cmd.tcpm_keylen); } =20 static int tcp_v6_md5_hash_headers(struct tcp_md5sig_pool *hp, @@ -1359,9 +1358,8 @@ static struct sock *tcp_v6_syn_recv_sock(const struct= sock *sk, struct sk_buff * * memory, then we end up not copying the key * across. Shucks. */ - tcp_md5_do_add(newsk, (union tcp_md5_addr *)&newsk->sk_v6_daddr, - AF_INET6, 128, l3index, key->flags, key->key, key->keylen, - sk_gfp_mask(sk, GFP_ATOMIC)); + tcp_md5_key_copy(newsk, (union tcp_md5_addr *)&newsk->sk_v6_daddr, + AF_INET6, 128, l3index, key); } #endif =20 --=20 2.36.1 From nobody Wed Apr 15 04:24:53 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id ED581C00140 for ; Tue, 26 Jul 2022 20:17:01 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S239898AbiGZURA (ORCPT ); Tue, 26 Jul 2022 16:17:00 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:52012 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S239868AbiGZUQW (ORCPT ); Tue, 26 Jul 2022 16:16:22 -0400 Received: from mail-wr1-x433.google.com (mail-wr1-x433.google.com [IPv6:2a00:1450:4864:20::433]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id C159D32B9F for ; Tue, 26 Jul 2022 13:16:15 -0700 (PDT) Received: by mail-wr1-x433.google.com with SMTP id g2so13550868wru.3 for ; Tue, 26 Jul 2022 13:16:15 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=arista.com; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc; bh=Eehr4I6LShNPnqC8iBpwixr+0n5YGJ6W1hkSoL6jCXQ=; b=g2/reOHW1eJbSXHaa1DvAMlFs2zXHAgHPrGCu/RS16SCytzEIvFndxzuYNXT4GWYfe V7pAkoe4ABpxg1IbikjFCoqATQNRCSO/z1Yycaq3lRakL9NxbpOBZzQxxdihVbMIkwtX NZ2E3K7Z+i7HMGZun2Ey/5Rsw4lkCRSO6L8D8y6GODSTOL9UY3Nb/xU6owmj9kwlIWTJ u/88DoY1abaAxAy4YWTbM9IAcSWc47wLhx+OKHJ/kHBAGV/lmdmtrAGusT6l3I9McN7x Ro+wsqoq7xN+hAUKg39zMUEoAyCGunQioHZSA9Q2ZojXCwRuG99EPd6HlNMM0yfh4FIV 84Ow== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc; bh=Eehr4I6LShNPnqC8iBpwixr+0n5YGJ6W1hkSoL6jCXQ=; b=C5yYSIG1QXozMJt8ktQZDt0OBhm84ihQyspY3IFs3ZviECRF2D3/JBBOif6SOM7MAU 55DEJxmn5nLIuDIUYK6nAnUDK8qPlgf69zO2NQhMKDigIjoeUVYtCDLYDvw2UigbdBv3 kS8K5Zj/ht652MtgiykMsm1k/0D2aGnwErdSPL6v/4YR6VLPsqN6KYC9xJMNes5C8G22 J2nu37Ab+IFHhcQx0fnqJkctv69eSOS1yKl0vTjm//syxnSGBcjYo2qrr7UrK1fPqFww frvOaKMmahVSh91X7vKSE8DdLYY2Nic1bmgeyX3W2mxBP1PdmEwZn+tklEfVT3it6Fhr MW7g== X-Gm-Message-State: AJIora/mVtLYxIwy3CsPZxW2SrLwb+Xk5YeRpivvFE5dJwv5n8op7DGf c787ZSGF2YuTrGzJIo+B8n+ot8aXnN/cAA== X-Google-Smtp-Source: AGRyM1uVbzFsXTbvL6A2nOYM6ttwODXmdQywk1yFcQ8V0rwXJHiB0bUjPmQaRKZ/gDlecdLa4vnigw== X-Received: by 2002:adf:facb:0:b0:21e:4f54:9651 with SMTP id a11-20020adffacb000000b0021e4f549651mr12470112wrs.378.1658866573724; Tue, 26 Jul 2022 13:16:13 -0700 (PDT) Received: from Mindolluin.ire.aristanetworks.com ([217.173.96.166]) by smtp.gmail.com with ESMTPSA id m6-20020a05600c3b0600b003a320e6f011sm28073wms.1.2022.07.26.13.16.12 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 26 Jul 2022 13:16:13 -0700 (PDT) From: Dmitry Safonov To: linux-kernel@vger.kernel.org Cc: Dmitry Safonov <0x7f454c46@gmail.com>, Dmitry Safonov , Andy Lutomirski , Ard Biesheuvel , David Ahern , "David S. Miller" , Eric Biggers , Eric Dumazet , Francesco Ruggeri , Herbert Xu , Hideaki YOSHIFUJI , Jakub Kicinski , Leonard Crestez , Paolo Abeni , Salam Noureddine , netdev@vger.kernel.org, linux-crypto@vger.kernel.org Subject: [PATCH 5/6] net/tcp: Use crypto_pool for TCP-MD5 Date: Tue, 26 Jul 2022 21:15:59 +0100 Message-Id: <20220726201600.1715505-6-dima@arista.com> X-Mailer: git-send-email 2.36.1 In-Reply-To: <20220726201600.1715505-1-dima@arista.com> References: <20220726201600.1715505-1-dima@arista.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" Use crypto_pool API that was designed with tcp_md5sig_pool in mind. The conversion to use crypto_pool will allow: - to reuse ahash_request(s) for different users - to allocate only one per-CPU scratch buffer rather than a new one for each user - to have a common API for net/ users that need ahash on RX/TX fast path Signed-off-by: Dmitry Safonov --- include/net/tcp.h | 22 +++------ net/ipv4/Kconfig | 2 +- net/ipv4/tcp.c | 99 +++++++++++----------------------------- net/ipv4/tcp_ipv4.c | 90 +++++++++++++++++++++--------------- net/ipv4/tcp_minisocks.c | 22 +++++++-- net/ipv6/tcp_ipv6.c | 53 ++++++++++----------- 6 files changed, 127 insertions(+), 161 deletions(-) diff --git a/include/net/tcp.h b/include/net/tcp.h index aa735f963723..7060dd1cf6cd 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -1652,12 +1652,6 @@ union tcp_md5sum_block { #endif }; =20 -/* - pool: digest algorithm, hash description and scratch buffer */ -struct tcp_md5sig_pool { - struct ahash_request *md5_req; - void *scratch; -}; - /* - functions */ int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key, const struct sock *sk, const struct sk_buff *skb); @@ -1713,17 +1707,15 @@ tcp_inbound_md5_hash(const struct sock *sk, const s= truct sk_buff *skb, #define tcp_twsk_md5_key(twsk) NULL #endif =20 -bool tcp_alloc_md5sig_pool(void); - -struct tcp_md5sig_pool *tcp_get_md5sig_pool(void); -static inline void tcp_put_md5sig_pool(void) -{ - local_bh_enable(); -} +struct crypto_pool_ahash; +int tcp_md5_alloc_crypto_pool(void); +void tcp_md5_release_crypto_pool(void); +void tcp_md5_add_crypto_pool(void); +extern int tcp_md5_crypto_pool_id; =20 -int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *, +int tcp_md5_hash_skb_data(struct crypto_pool_ahash *, const struct sk_buff= *, unsigned int header_len); -int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, +int tcp_md5_hash_key(struct crypto_pool_ahash *hp, const struct tcp_md5sig_key *key); =20 /* From tcp_fastopen.c */ diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig index e983bb0c5012..c341864e4398 100644 --- a/net/ipv4/Kconfig +++ b/net/ipv4/Kconfig @@ -733,7 +733,7 @@ config DEFAULT_TCP_CONG =20 config TCP_MD5SIG bool "TCP: MD5 Signature Option support (RFC2385)" - select CRYPTO + select CRYPTO_POOL select CRYPTO_MD5 help RFC2385 specifies a method of giving MD5 protection to TCP sessions. diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 300056008e90..73abb00e12bf 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -244,6 +244,7 @@ #define pr_fmt(fmt) "TCP: " fmt =20 #include +#include #include #include #include @@ -4355,92 +4356,43 @@ int tcp_getsockopt(struct sock *sk, int level, int = optname, char __user *optval, EXPORT_SYMBOL(tcp_getsockopt); =20 #ifdef CONFIG_TCP_MD5SIG -static DEFINE_PER_CPU(struct tcp_md5sig_pool, tcp_md5sig_pool); -static DEFINE_MUTEX(tcp_md5sig_mutex); -static bool tcp_md5sig_pool_populated =3D false; +int tcp_md5_crypto_pool_id =3D -1; +EXPORT_SYMBOL(tcp_md5_crypto_pool_id); =20 -static void __tcp_alloc_md5sig_pool(void) +int tcp_md5_alloc_crypto_pool(void) { - struct crypto_ahash *hash; - int cpu; - - hash =3D crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC); - if (IS_ERR(hash)) - return; - - for_each_possible_cpu(cpu) { - void *scratch =3D per_cpu(tcp_md5sig_pool, cpu).scratch; - struct ahash_request *req; - - if (!scratch) { - scratch =3D kmalloc_node(sizeof(union tcp_md5sum_block) + - sizeof(struct tcphdr), - GFP_KERNEL, - cpu_to_node(cpu)); - if (!scratch) - return; - per_cpu(tcp_md5sig_pool, cpu).scratch =3D scratch; - } - if (per_cpu(tcp_md5sig_pool, cpu).md5_req) - continue; - - req =3D ahash_request_alloc(hash, GFP_KERNEL); - if (!req) - return; + int ret; =20 - ahash_request_set_callback(req, 0, NULL, NULL); + ret =3D crypto_pool_reserve_scratch(sizeof(union tcp_md5sum_block) + + sizeof(struct tcphdr)); + if (ret) + return ret; =20 - per_cpu(tcp_md5sig_pool, cpu).md5_req =3D req; - } - /* before setting tcp_md5sig_pool_populated, we must commit all writes - * to memory. See smp_rmb() in tcp_get_md5sig_pool() - */ - smp_wmb(); - tcp_md5sig_pool_populated =3D true; + ret =3D crypto_pool_alloc_ahash("md5"); + if (ret >=3D 0) + tcp_md5_crypto_pool_id =3D ret; + return ret; } +EXPORT_SYMBOL(tcp_md5_alloc_crypto_pool); =20 -bool tcp_alloc_md5sig_pool(void) +void tcp_md5_release_crypto_pool(void) { - if (unlikely(!tcp_md5sig_pool_populated)) { - mutex_lock(&tcp_md5sig_mutex); - - if (!tcp_md5sig_pool_populated) - __tcp_alloc_md5sig_pool(); - - mutex_unlock(&tcp_md5sig_mutex); - } - return tcp_md5sig_pool_populated; + crypto_pool_release(tcp_md5_crypto_pool_id); } -EXPORT_SYMBOL(tcp_alloc_md5sig_pool); +EXPORT_SYMBOL(tcp_md5_release_crypto_pool); =20 - -/** - * tcp_get_md5sig_pool - get md5sig_pool for this user - * - * We use percpu structure, so if we succeed, we exit with preemption - * and BH disabled, to make sure another thread or softirq handling - * wont try to get same context. - */ -struct tcp_md5sig_pool *tcp_get_md5sig_pool(void) +void tcp_md5_add_crypto_pool(void) { - local_bh_disable(); - - if (tcp_md5sig_pool_populated) { - /* coupled with smp_wmb() in __tcp_alloc_md5sig_pool() */ - smp_rmb(); - return this_cpu_ptr(&tcp_md5sig_pool); - } - local_bh_enable(); - return NULL; + crypto_pool_add(tcp_md5_crypto_pool_id); } -EXPORT_SYMBOL(tcp_get_md5sig_pool); +EXPORT_SYMBOL(tcp_md5_add_crypto_pool); =20 -int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *hp, +int tcp_md5_hash_skb_data(struct crypto_pool_ahash *hp, const struct sk_buff *skb, unsigned int header_len) { struct scatterlist sg; const struct tcphdr *tp =3D tcp_hdr(skb); - struct ahash_request *req =3D hp->md5_req; + struct ahash_request *req =3D hp->req; unsigned int i; const unsigned int head_data_len =3D skb_headlen(skb) > header_len ? skb_headlen(skb) - header_len : 0; @@ -4474,16 +4426,17 @@ int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *h= p, } EXPORT_SYMBOL(tcp_md5_hash_skb_data); =20 -int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, const struct tcp_md5sig_k= ey *key) +int tcp_md5_hash_key(struct crypto_pool_ahash *hp, + const struct tcp_md5sig_key *key) { u8 keylen =3D READ_ONCE(key->keylen); /* paired with WRITE_ONCE() in tcp_= md5_do_add */ struct scatterlist sg; =20 sg_init_one(&sg, key->key, keylen); - ahash_request_set_crypt(hp->md5_req, &sg, NULL, keylen); + ahash_request_set_crypt(hp->req, &sg, NULL, keylen); =20 /* We use data_race() because tcp_md5_do_add() might change key->key unde= r us */ - return data_race(crypto_ahash_update(hp->md5_req)); + return data_race(crypto_ahash_update(hp->req)); } EXPORT_SYMBOL(tcp_md5_hash_key); =20 diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 5b0caaea5029..100e142ed03a 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -79,6 +79,7 @@ #include =20 #include +#include #include =20 #include @@ -1206,10 +1207,6 @@ int __tcp_md5_do_add(struct sock *sk, const union tc= p_md5_addr *addr, key =3D sock_kmalloc(sk, sizeof(*key), gfp | __GFP_ZERO); if (!key) return -ENOMEM; - if (!tcp_alloc_md5sig_pool()) { - sock_kfree_s(sk, key, sizeof(*key)); - return -ENOMEM; - } =20 memcpy(key->key, newkey, newkeylen); key->keylen =3D newkeylen; @@ -1228,8 +1225,13 @@ int tcp_md5_do_add(struct sock *sk, const union tcp_= md5_addr *addr, int family, u8 prefixlen, int l3index, u8 flags, const u8 *newkey, u8 newkeylen) { - if (tcp_md5sig_info_add(sk, GFP_KERNEL)) + if (tcp_md5_alloc_crypto_pool()) + return -ENOMEM; + + if (tcp_md5sig_info_add(sk, GFP_KERNEL)) { + tcp_md5_release_crypto_pool(); return -ENOMEM; + } =20 static_branch_inc(&tcp_md5_needed.key); =20 @@ -1242,8 +1244,12 @@ int tcp_md5_key_copy(struct sock *sk, const union tc= p_md5_addr *addr, int family, u8 prefixlen, int l3index, struct tcp_md5sig_key *key) { - if (tcp_md5sig_info_add(sk, sk_gfp_mask(sk, GFP_ATOMIC))) + tcp_md5_add_crypto_pool(); + + if (tcp_md5sig_info_add(sk, sk_gfp_mask(sk, GFP_ATOMIC))) { + tcp_md5_release_crypto_pool(); return -ENOMEM; + } =20 atomic_inc(&tcp_md5_needed.key.key.enabled); =20 @@ -1342,7 +1348,7 @@ static int tcp_v4_parse_md5_keys(struct sock *sk, int= optname, cmd.tcpm_key, cmd.tcpm_keylen); } =20 -static int tcp_v4_md5_hash_headers(struct tcp_md5sig_pool *hp, +static int tcp_v4_md5_hash_headers(struct crypto_pool_ahash *hp, __be32 daddr, __be32 saddr, const struct tcphdr *th, int nbytes) { @@ -1350,7 +1356,7 @@ static int tcp_v4_md5_hash_headers(struct tcp_md5sig_= pool *hp, struct scatterlist sg; struct tcphdr *_th; =20 - bp =3D hp->scratch; + bp =3D hp->base.scratch; bp->saddr =3D saddr; bp->daddr =3D daddr; bp->pad =3D 0; @@ -1362,37 +1368,34 @@ static int tcp_v4_md5_hash_headers(struct tcp_md5si= g_pool *hp, _th->check =3D 0; =20 sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th)); - ahash_request_set_crypt(hp->md5_req, &sg, NULL, + ahash_request_set_crypt(hp->req, &sg, NULL, sizeof(*bp) + sizeof(*th)); - return crypto_ahash_update(hp->md5_req); + return crypto_ahash_update(hp->req); } =20 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key= *key, __be32 daddr, __be32 saddr, const struct tcphdr *th) { - struct tcp_md5sig_pool *hp; - struct ahash_request *req; + struct crypto_pool_ahash hp; =20 - hp =3D tcp_get_md5sig_pool(); - if (!hp) + if (crypto_pool_get(tcp_md5_crypto_pool_id, (struct crypto_pool *)&hp)) goto clear_hash_noput; - req =3D hp->md5_req; =20 - if (crypto_ahash_init(req)) + if (crypto_ahash_init(hp.req)) goto clear_hash; - if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, th->doff << 2)) + if (tcp_v4_md5_hash_headers(&hp, daddr, saddr, th, th->doff << 2)) goto clear_hash; - if (tcp_md5_hash_key(hp, key)) + if (tcp_md5_hash_key(&hp, key)) goto clear_hash; - ahash_request_set_crypt(req, NULL, md5_hash, 0); - if (crypto_ahash_final(req)) + ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); + if (crypto_ahash_final(hp.req)) goto clear_hash; =20 - tcp_put_md5sig_pool(); + crypto_pool_put(); return 0; =20 clear_hash: - tcp_put_md5sig_pool(); + crypto_pool_put(); clear_hash_noput: memset(md5_hash, 0, 16); return 1; @@ -1402,8 +1405,7 @@ int tcp_v4_md5_hash_skb(char *md5_hash, const struct = tcp_md5sig_key *key, const struct sock *sk, const struct sk_buff *skb) { - struct tcp_md5sig_pool *hp; - struct ahash_request *req; + struct crypto_pool_ahash hp; const struct tcphdr *th =3D tcp_hdr(skb); __be32 saddr, daddr; =20 @@ -1416,29 +1418,27 @@ int tcp_v4_md5_hash_skb(char *md5_hash, const struc= t tcp_md5sig_key *key, daddr =3D iph->daddr; } =20 - hp =3D tcp_get_md5sig_pool(); - if (!hp) + if (crypto_pool_get(tcp_md5_crypto_pool_id, (struct crypto_pool *)&hp)) goto clear_hash_noput; - req =3D hp->md5_req; =20 - if (crypto_ahash_init(req)) + if (crypto_ahash_init(hp.req)) goto clear_hash; =20 - if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, skb->len)) + if (tcp_v4_md5_hash_headers(&hp, daddr, saddr, th, skb->len)) goto clear_hash; - if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2)) + if (tcp_md5_hash_skb_data(&hp, skb, th->doff << 2)) goto clear_hash; - if (tcp_md5_hash_key(hp, key)) + if (tcp_md5_hash_key(&hp, key)) goto clear_hash; - ahash_request_set_crypt(req, NULL, md5_hash, 0); - if (crypto_ahash_final(req)) + ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); + if (crypto_ahash_final(hp.req)) goto clear_hash; =20 - tcp_put_md5sig_pool(); + crypto_pool_put(); return 0; =20 clear_hash: - tcp_put_md5sig_pool(); + crypto_pool_put(); clear_hash_noput: memset(md5_hash, 0, 16); return 1; @@ -2257,6 +2257,18 @@ static int tcp_v4_init_sock(struct sock *sk) return 0; } =20 +#ifdef CONFIG_TCP_MD5SIG +static void tcp_md5sig_info_free_rcu(struct rcu_head *head) +{ + struct tcp_md5sig_info *md5sig; + + md5sig =3D container_of(head, struct tcp_md5sig_info, rcu); + kfree(md5sig); + static_branch_slow_dec_deferred(&tcp_md5_needed); + tcp_md5_release_crypto_pool(); +} +#endif + void tcp_v4_destroy_sock(struct sock *sk) { struct tcp_sock *tp =3D tcp_sk(sk); @@ -2281,10 +2293,12 @@ void tcp_v4_destroy_sock(struct sock *sk) #ifdef CONFIG_TCP_MD5SIG /* Clean up the MD5 key list, if any */ if (tp->md5sig_info) { + struct tcp_md5sig_info *md5sig; + + md5sig =3D rcu_dereference_protected(tp->md5sig_info, 1); tcp_clear_md5_list(sk); - kfree_rcu(rcu_dereference_protected(tp->md5sig_info, 1), rcu); - tp->md5sig_info =3D NULL; - static_branch_slow_dec_deferred(&tcp_md5_needed); + call_rcu(&md5sig->rcu, tcp_md5sig_info_free_rcu); + rcu_assign_pointer(tp->md5sig_info, NULL); } #endif =20 diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index 5d475a45a478..d1d30337ffec 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -297,8 +297,10 @@ void tcp_time_wait(struct sock *sk, int state, int tim= eo) key =3D tp->af_specific->md5_lookup(sk, sk); if (key) { tcptw->tw_md5_key =3D kmemdup(key, sizeof(*key), GFP_ATOMIC); - BUG_ON(tcptw->tw_md5_key && !tcp_alloc_md5sig_pool()); + if (WARN_ON(!tcptw->tw_md5_key)) + break; atomic_inc(&tcp_md5_needed.key.key.enabled); + tcp_md5_add_crypto_pool(); } } } while (0); @@ -335,16 +337,26 @@ void tcp_time_wait(struct sock *sk, int state, int ti= meo) } EXPORT_SYMBOL(tcp_time_wait); =20 +#ifdef CONFIG_TCP_MD5SIG +static void tcp_md5_twsk_free_rcu(struct rcu_head *head) +{ + struct tcp_md5sig_key *key; + + key =3D container_of(head, struct tcp_md5sig_key, rcu); + kfree(key); + static_branch_slow_dec_deferred(&tcp_md5_needed); + tcp_md5_release_crypto_pool(); +} +#endif + void tcp_twsk_destructor(struct sock *sk) { #ifdef CONFIG_TCP_MD5SIG if (static_branch_unlikely(&tcp_md5_needed.key)) { struct tcp_timewait_sock *twsk =3D tcp_twsk(sk); =20 - if (twsk->tw_md5_key) { - kfree_rcu(twsk->tw_md5_key, rcu); - static_branch_slow_dec_deferred(&tcp_md5_needed); - } + if (twsk->tw_md5_key) + call_rcu(&twsk->tw_md5_key->rcu, tcp_md5_twsk_free_rcu); } #endif } diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index cb891a71db0d..f75569f889e7 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -64,6 +64,7 @@ #include =20 #include +#include #include =20 #include @@ -665,7 +666,7 @@ static int tcp_v6_parse_md5_keys(struct sock *sk, int o= ptname, cmd.tcpm_key, cmd.tcpm_keylen); } =20 -static int tcp_v6_md5_hash_headers(struct tcp_md5sig_pool *hp, +static int tcp_v6_md5_hash_headers(struct crypto_pool_ahash *hp, const struct in6_addr *daddr, const struct in6_addr *saddr, const struct tcphdr *th, int nbytes) @@ -674,7 +675,7 @@ static int tcp_v6_md5_hash_headers(struct tcp_md5sig_po= ol *hp, struct scatterlist sg; struct tcphdr *_th; =20 - bp =3D hp->scratch; + bp =3D hp->base.scratch; /* 1. TCP pseudo-header (RFC2460) */ bp->saddr =3D *saddr; bp->daddr =3D *daddr; @@ -686,38 +687,35 @@ static int tcp_v6_md5_hash_headers(struct tcp_md5sig_= pool *hp, _th->check =3D 0; =20 sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th)); - ahash_request_set_crypt(hp->md5_req, &sg, NULL, + ahash_request_set_crypt(hp->req, &sg, NULL, sizeof(*bp) + sizeof(*th)); - return crypto_ahash_update(hp->md5_req); + return crypto_ahash_update(hp->req); } =20 static int tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key= *key, const struct in6_addr *daddr, struct in6_addr *saddr, const struct tcphdr *th) { - struct tcp_md5sig_pool *hp; - struct ahash_request *req; + struct crypto_pool_ahash hp; =20 - hp =3D tcp_get_md5sig_pool(); - if (!hp) + if (crypto_pool_get(tcp_md5_crypto_pool_id, (struct crypto_pool *)&hp)) goto clear_hash_noput; - req =3D hp->md5_req; =20 - if (crypto_ahash_init(req)) + if (crypto_ahash_init(hp.req)) goto clear_hash; - if (tcp_v6_md5_hash_headers(hp, daddr, saddr, th, th->doff << 2)) + if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, th->doff << 2)) goto clear_hash; - if (tcp_md5_hash_key(hp, key)) + if (tcp_md5_hash_key(&hp, key)) goto clear_hash; - ahash_request_set_crypt(req, NULL, md5_hash, 0); - if (crypto_ahash_final(req)) + ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); + if (crypto_ahash_final(hp.req)) goto clear_hash; =20 - tcp_put_md5sig_pool(); + crypto_pool_put(); return 0; =20 clear_hash: - tcp_put_md5sig_pool(); + crypto_pool_put(); clear_hash_noput: memset(md5_hash, 0, 16); return 1; @@ -729,8 +727,7 @@ static int tcp_v6_md5_hash_skb(char *md5_hash, const struct sk_buff *skb) { const struct in6_addr *saddr, *daddr; - struct tcp_md5sig_pool *hp; - struct ahash_request *req; + struct crypto_pool_ahash hp; const struct tcphdr *th =3D tcp_hdr(skb); =20 if (sk) { /* valid for establish/request sockets */ @@ -742,29 +739,27 @@ static int tcp_v6_md5_hash_skb(char *md5_hash, daddr =3D &ip6h->daddr; } =20 - hp =3D tcp_get_md5sig_pool(); - if (!hp) + if (crypto_pool_get(tcp_md5_crypto_pool_id, (struct crypto_pool *)&hp)) goto clear_hash_noput; - req =3D hp->md5_req; =20 - if (crypto_ahash_init(req)) + if (crypto_ahash_init(hp.req)) goto clear_hash; =20 - if (tcp_v6_md5_hash_headers(hp, daddr, saddr, th, skb->len)) + if (tcp_v6_md5_hash_headers(&hp, daddr, saddr, th, skb->len)) goto clear_hash; - if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2)) + if (tcp_md5_hash_skb_data(&hp, skb, th->doff << 2)) goto clear_hash; - if (tcp_md5_hash_key(hp, key)) + if (tcp_md5_hash_key(&hp, key)) goto clear_hash; - ahash_request_set_crypt(req, NULL, md5_hash, 0); - if (crypto_ahash_final(req)) + ahash_request_set_crypt(hp.req, NULL, md5_hash, 0); + if (crypto_ahash_final(hp.req)) goto clear_hash; =20 - tcp_put_md5sig_pool(); + crypto_pool_put(); return 0; =20 clear_hash: - tcp_put_md5sig_pool(); + crypto_pool_put(); clear_hash_noput: memset(md5_hash, 0, 16); return 1; --=20 2.36.1 From nobody Wed Apr 15 04:24:53 2026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id BF53DC00140 for ; Tue, 26 Jul 2022 20:17:07 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S239949AbiGZURF (ORCPT ); Tue, 26 Jul 2022 16:17:05 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:51978 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S239876AbiGZUQW (ORCPT ); Tue, 26 Jul 2022 16:16:22 -0400 Received: from mail-wr1-x434.google.com (mail-wr1-x434.google.com [IPv6:2a00:1450:4864:20::434]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id F124B32EC3 for ; Tue, 26 Jul 2022 13:16:16 -0700 (PDT) Received: by mail-wr1-x434.google.com with SMTP id v17so2669630wrr.10 for ; Tue, 26 Jul 2022 13:16:16 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=arista.com; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=awxy/BDZoT0mM/P9mIhPTjV07MQDeBS6ciLhmG/Wd6U=; b=Mas144nsu2lGiSPU7SoK2Dkvt5JZVg0bt5YZ5uyveNMSh2MeomG6b8c0PR8essfZ1Q OxyDDVWcaSYFog89+hcdJRrm9vRWNeQS2kulnHwpYs9bfwmBrZpQbiLlAzfg/r6kKxjn deh0mXYPSOEVWcT0Rv5X+0xnkkdE6DQud2sIVROP4iJtFKe2oscy989H6b8PQ0EQ87nc YedkvrWDJ9z1gCK4IoW2WjiPOJzIM2oBFEg3BsEAOvNLbZAoXufMAO2z/G6ge0mlrL+T eF6i+lgGk7MQNJYBklV8qUguOilc/LdaowwM2bRfxirx7W624ROvjBwwlsRDt3PR8Tm5 HQnA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=awxy/BDZoT0mM/P9mIhPTjV07MQDeBS6ciLhmG/Wd6U=; b=yBFGXH+z6VDk0HK52SveDZvp8gqUstl+TiU9daUNgWXqdMlUAWNd/nlMKNLYZZYrkL Md610NOwNqcTtHWO8cCmvYb2L0dL2gNmnOTj2tErCCRe6yHvzSxvchHWd+KurCvvdSXz ZH2Aj5KWLhy3Bwf11Pwhar7MSLTRr9iT+p7qb1gnwXpa+LvulXZXhLcNCo/9m+1EtPec dRlnov0z9snfqbvzCnA1x12vEjHK31FNp+yYpYXz3BDmMoRamEbPrLa/3/+ulQClCaqb wRYzi7UL74/FAwwVClopLWOHD6WG9Tmxa6FlB1Zmfx5Ebh6aSJU32uxGToWKK3oHNpFZ eiIQ== X-Gm-Message-State: AJIora/oFnnIhsAOIzFbkGlT+PNqVhjwjK3pvlpY/aI5oKb4wnh2EtpX p4faWoEp6Ttm61iHkUV2JJ4oZj1yE0OpEw== X-Google-Smtp-Source: AGRyM1suvy8hibTSugbZ0RLGIbu6B2puqQ/WjE13KKElm59HY9bzOPt2Uu/G6z8SjtU3/DaBfjhoYg== X-Received: by 2002:a05:6000:168e:b0:21d:ae03:49d9 with SMTP id y14-20020a056000168e00b0021dae0349d9mr12073735wrd.457.1658866575021; Tue, 26 Jul 2022 13:16:15 -0700 (PDT) Received: from Mindolluin.ire.aristanetworks.com ([217.173.96.166]) by smtp.gmail.com with ESMTPSA id m6-20020a05600c3b0600b003a320e6f011sm28073wms.1.2022.07.26.13.16.13 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 26 Jul 2022 13:16:14 -0700 (PDT) From: Dmitry Safonov To: linux-kernel@vger.kernel.org Cc: Dmitry Safonov <0x7f454c46@gmail.com>, Dmitry Safonov , Andy Lutomirski , Ard Biesheuvel , David Ahern , "David S. Miller" , Eric Biggers , Eric Dumazet , Francesco Ruggeri , Herbert Xu , Hideaki YOSHIFUJI , Jakub Kicinski , Leonard Crestez , Paolo Abeni , Salam Noureddine , netdev@vger.kernel.org, linux-crypto@vger.kernel.org Subject: [PATCH 6/6] net/ipv6: sr: Switch to using crypto_pool Date: Tue, 26 Jul 2022 21:16:00 +0100 Message-Id: <20220726201600.1715505-7-dima@arista.com> X-Mailer: git-send-email 2.36.1 In-Reply-To: <20220726201600.1715505-1-dima@arista.com> References: <20220726201600.1715505-1-dima@arista.com> MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Type: text/plain; charset="utf-8" The conversion to use crypto_pool has the following upsides: - now SR uses asynchronous API which may potentially free CPU cycles and improve performance for of CPU crypto algorithm providers; - hash descriptors now don't have to be allocated on boot, but only at the moment SR starts using HMAC and until the last HMAC secret is deleted; - potentially reuse ahash_request(s) for different users - allocate only one per-CPU scratch buffer rather than a new one for each user - have a common API for net/ users that need ahash on RX/TX fast path Signed-off-by: Dmitry Safonov --- include/net/seg6_hmac.h | 7 -- net/ipv6/Kconfig | 2 +- net/ipv6/seg6.c | 3 - net/ipv6/seg6_hmac.c | 204 ++++++++++++++++------------------------ 4 files changed, 80 insertions(+), 136 deletions(-) diff --git a/include/net/seg6_hmac.h b/include/net/seg6_hmac.h index 2b5d2ee5613e..d6b7820ecda2 100644 --- a/include/net/seg6_hmac.h +++ b/include/net/seg6_hmac.h @@ -32,13 +32,6 @@ struct seg6_hmac_info { u8 alg_id; }; =20 -struct seg6_hmac_algo { - u8 alg_id; - char name[64]; - struct crypto_shash * __percpu *tfms; - struct shash_desc * __percpu *shashs; -}; - extern int seg6_hmac_compute(struct seg6_hmac_info *hinfo, struct ipv6_sr_hdr *hdr, struct in6_addr *saddr, u8 *output); diff --git a/net/ipv6/Kconfig b/net/ipv6/Kconfig index 658bfed1df8b..5be1dab0f178 100644 --- a/net/ipv6/Kconfig +++ b/net/ipv6/Kconfig @@ -304,7 +304,7 @@ config IPV6_SEG6_LWTUNNEL config IPV6_SEG6_HMAC bool "IPv6: Segment Routing HMAC support" depends on IPV6 - select CRYPTO + select CRYPTO_POOL select CRYPTO_HMAC select CRYPTO_SHA1 select CRYPTO_SHA256 diff --git a/net/ipv6/seg6.c b/net/ipv6/seg6.c index 73aaabf0e966..96b80e1d04c1 100644 --- a/net/ipv6/seg6.c +++ b/net/ipv6/seg6.c @@ -552,9 +552,6 @@ int __init seg6_init(void) =20 void seg6_exit(void) { -#ifdef CONFIG_IPV6_SEG6_HMAC - seg6_hmac_exit(); -#endif #ifdef CONFIG_IPV6_SEG6_LWTUNNEL seg6_iptunnel_exit(); #endif diff --git a/net/ipv6/seg6_hmac.c b/net/ipv6/seg6_hmac.c index d43c50a7310d..3732dd993925 100644 --- a/net/ipv6/seg6_hmac.c +++ b/net/ipv6/seg6_hmac.c @@ -35,6 +35,7 @@ #include =20 #include +#include #include #include #include @@ -70,6 +71,12 @@ static const struct rhashtable_params rht_params =3D { .obj_cmpfn =3D seg6_hmac_cmpfn, }; =20 +struct seg6_hmac_algo { + u8 alg_id; + char name[64]; + int crypto_pool_id; +}; + static struct seg6_hmac_algo hmac_algos[] =3D { { .alg_id =3D SEG6_HMAC_ALGO_SHA1, @@ -115,55 +122,17 @@ static struct seg6_hmac_algo *__hmac_get_algo(u8 alg_= id) return NULL; } =20 -static int __do_hmac(struct seg6_hmac_info *hinfo, const char *text, u8 ps= ize, - u8 *output, int outlen) -{ - struct seg6_hmac_algo *algo; - struct crypto_shash *tfm; - struct shash_desc *shash; - int ret, dgsize; - - algo =3D __hmac_get_algo(hinfo->alg_id); - if (!algo) - return -ENOENT; - - tfm =3D *this_cpu_ptr(algo->tfms); - - dgsize =3D crypto_shash_digestsize(tfm); - if (dgsize > outlen) { - pr_debug("sr-ipv6: __do_hmac: digest size too big (%d / %d)\n", - dgsize, outlen); - return -ENOMEM; - } - - ret =3D crypto_shash_setkey(tfm, hinfo->secret, hinfo->slen); - if (ret < 0) { - pr_debug("sr-ipv6: crypto_shash_setkey failed: err %d\n", ret); - goto failed; - } - - shash =3D *this_cpu_ptr(algo->shashs); - shash->tfm =3D tfm; - - ret =3D crypto_shash_digest(shash, text, psize, output); - if (ret < 0) { - pr_debug("sr-ipv6: crypto_shash_digest failed: err %d\n", ret); - goto failed; - } - - return dgsize; - -failed: - return ret; -} - int seg6_hmac_compute(struct seg6_hmac_info *hinfo, struct ipv6_sr_hdr *hd= r, struct in6_addr *saddr, u8 *output) { __be32 hmackeyid =3D cpu_to_be32(hinfo->hmackeyid); - u8 tmp_out[SEG6_HMAC_MAX_DIGESTSIZE]; + struct crypto_pool_ahash hp; + struct seg6_hmac_algo *algo; int plen, i, dgsize, wrsize; + struct crypto_ahash *tfm; + struct scatterlist sg; char *ring, *off; + int err; =20 /* a 160-byte buffer for digest output allows to store highest known * hash function (RadioGatun) with up to 1216 bits @@ -176,6 +145,10 @@ int seg6_hmac_compute(struct seg6_hmac_info *hinfo, st= ruct ipv6_sr_hdr *hdr, if (plen >=3D SEG6_HMAC_RING_SIZE) return -EMSGSIZE; =20 + algo =3D __hmac_get_algo(hinfo->alg_id); + if (!algo) + return -ENOENT; + /* Let's build the HMAC text on the ring buffer. The text is composed * as follows, in order: * @@ -186,8 +159,36 @@ int seg6_hmac_compute(struct seg6_hmac_info *hinfo, st= ruct ipv6_sr_hdr *hdr, * 5. All segments in the segments list (n * 128 bits) */ =20 - local_bh_disable(); + err =3D crypto_pool_get(algo->crypto_pool_id, (struct crypto_pool *)&hp); + if (err) + return err; + ring =3D this_cpu_ptr(hmac_ring); + + sg_init_one(&sg, ring, plen); + + tfm =3D crypto_ahash_reqtfm(hp.req); + dgsize =3D crypto_ahash_digestsize(tfm); + if (dgsize > SEG6_HMAC_MAX_DIGESTSIZE) { + pr_debug("digest size too big (%d / %d)\n", + dgsize, SEG6_HMAC_MAX_DIGESTSIZE); + err =3D -ENOMEM; + goto err_put_pool; + } + + err =3D crypto_ahash_setkey(tfm, hinfo->secret, hinfo->slen); + if (err) { + pr_debug("crypto_ahash_setkey failed: err %d\n", err); + goto err_put_pool; + } + + err =3D crypto_ahash_init(hp.req); + if (err) + goto err_put_pool; + + ahash_request_set_crypt(hp.req, &sg, + hp.base.scratch, SEG6_HMAC_MAX_DIGESTSIZE); + off =3D ring; =20 /* source address */ @@ -210,21 +211,25 @@ int seg6_hmac_compute(struct seg6_hmac_info *hinfo, s= truct ipv6_sr_hdr *hdr, off +=3D 16; } =20 - dgsize =3D __do_hmac(hinfo, ring, plen, tmp_out, - SEG6_HMAC_MAX_DIGESTSIZE); - local_bh_enable(); + err =3D crypto_ahash_update(hp.req); + if (err) + goto err_put_pool; =20 - if (dgsize < 0) - return dgsize; + err =3D crypto_ahash_final(hp.req); + if (err) + goto err_put_pool; =20 wrsize =3D SEG6_HMAC_FIELD_LEN; if (wrsize > dgsize) wrsize =3D dgsize; =20 memset(output, 0, SEG6_HMAC_FIELD_LEN); - memcpy(output, tmp_out, wrsize); + memcpy(output, hp.base.scratch, wrsize); =20 - return 0; +err_put_pool: + crypto_pool_put(); + + return err; } EXPORT_SYMBOL(seg6_hmac_compute); =20 @@ -291,12 +296,24 @@ EXPORT_SYMBOL(seg6_hmac_info_lookup); int seg6_hmac_info_add(struct net *net, u32 key, struct seg6_hmac_info *hi= nfo) { struct seg6_pernet_data *sdata =3D seg6_pernet(net); - int err; + struct seg6_hmac_algo *algo; + int ret; =20 - err =3D rhashtable_lookup_insert_fast(&sdata->hmac_infos, &hinfo->node, + algo =3D __hmac_get_algo(hinfo->alg_id); + if (!algo) + return -ENOENT; + + ret =3D crypto_pool_alloc_ahash(algo->name); + if (ret < 0) + return ret; + algo->crypto_pool_id =3D ret; + + ret =3D rhashtable_lookup_insert_fast(&sdata->hmac_infos, &hinfo->node, rht_params); + if (ret) + crypto_pool_release(algo->crypto_pool_id); =20 - return err; + return ret; } EXPORT_SYMBOL(seg6_hmac_info_add); =20 @@ -304,6 +321,7 @@ int seg6_hmac_info_del(struct net *net, u32 key) { struct seg6_pernet_data *sdata =3D seg6_pernet(net); struct seg6_hmac_info *hinfo; + struct seg6_hmac_algo *algo; int err =3D -ENOENT; =20 hinfo =3D rhashtable_lookup_fast(&sdata->hmac_infos, &key, rht_params); @@ -315,6 +333,12 @@ int seg6_hmac_info_del(struct net *net, u32 key) if (err) goto out; =20 + algo =3D __hmac_get_algo(hinfo->alg_id); + if (algo) + crypto_pool_release(algo->crypto_pool_id); + else + WARN_ON_ONCE(1); + seg6_hinfo_release(hinfo); =20 out: @@ -348,56 +372,9 @@ int seg6_push_hmac(struct net *net, struct in6_addr *s= addr, } EXPORT_SYMBOL(seg6_push_hmac); =20 -static int seg6_hmac_init_algo(void) -{ - struct seg6_hmac_algo *algo; - struct crypto_shash *tfm; - struct shash_desc *shash; - int i, alg_count, cpu; - - alg_count =3D ARRAY_SIZE(hmac_algos); - - for (i =3D 0; i < alg_count; i++) { - struct crypto_shash **p_tfm; - int shsize; - - algo =3D &hmac_algos[i]; - algo->tfms =3D alloc_percpu(struct crypto_shash *); - if (!algo->tfms) - return -ENOMEM; - - for_each_possible_cpu(cpu) { - tfm =3D crypto_alloc_shash(algo->name, 0, 0); - if (IS_ERR(tfm)) - return PTR_ERR(tfm); - p_tfm =3D per_cpu_ptr(algo->tfms, cpu); - *p_tfm =3D tfm; - } - - p_tfm =3D raw_cpu_ptr(algo->tfms); - tfm =3D *p_tfm; - - shsize =3D sizeof(*shash) + crypto_shash_descsize(tfm); - - algo->shashs =3D alloc_percpu(struct shash_desc *); - if (!algo->shashs) - return -ENOMEM; - - for_each_possible_cpu(cpu) { - shash =3D kzalloc_node(shsize, GFP_KERNEL, - cpu_to_node(cpu)); - if (!shash) - return -ENOMEM; - *per_cpu_ptr(algo->shashs, cpu) =3D shash; - } - } - - return 0; -} - int __init seg6_hmac_init(void) { - return seg6_hmac_init_algo(); + return crypto_pool_reserve_scratch(SEG6_HMAC_MAX_DIGESTSIZE); } =20 int __net_init seg6_hmac_net_init(struct net *net) @@ -407,29 +384,6 @@ int __net_init seg6_hmac_net_init(struct net *net) return rhashtable_init(&sdata->hmac_infos, &rht_params); } =20 -void seg6_hmac_exit(void) -{ - struct seg6_hmac_algo *algo =3D NULL; - int i, alg_count, cpu; - - alg_count =3D ARRAY_SIZE(hmac_algos); - for (i =3D 0; i < alg_count; i++) { - algo =3D &hmac_algos[i]; - for_each_possible_cpu(cpu) { - struct crypto_shash *tfm; - struct shash_desc *shash; - - shash =3D *per_cpu_ptr(algo->shashs, cpu); - kfree(shash); - tfm =3D *per_cpu_ptr(algo->tfms, cpu); - crypto_free_shash(tfm); - } - free_percpu(algo->tfms); - free_percpu(algo->shashs); - } -} -EXPORT_SYMBOL(seg6_hmac_exit); - void __net_exit seg6_hmac_net_exit(struct net *net) { struct seg6_pernet_data *sdata =3D seg6_pernet(net); --=20 2.36.1