From nobody Fri Dec 19 17:35:49 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A82DE136A; Sat, 17 May 2025 02:25:17 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448717; cv=none; b=qGQ9s5RcMaMSfh4/oT8f+UmWz+XFGaU7x2bFhO9ruHt+nf5ayup23qzRqeBKL6oqflSrh1GOUwyoTHjo/K1qUpHAe6WGWDCNSFXvfmnWL1ZOZ7D3vo9R3rtq1M/nhLU6L+HLoyVptCIsVbi8Hx9F7esu2fdqXAZgqPA3BAcZKOE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448717; c=relaxed/simple; bh=rxExDmAR8OpsxVyOb93D2F15CH0EPCc4/kwgbjLbV9Y=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=AAhs4sA4K/X/AxWFgpIlc6ln49ZjyaFq+C3aq/fvUi+R40RgVW0WF2U5DTSFEG7HgI6lLv3r00QES6U8bNYuL0KV81RoNZ+w9nszHEf8srEzUFHb65A87vdbj/RHVr2r8EQttbfUZCLyMBXWjXP/nW60bBagNJhcofjbrZVXbtk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=P/l/zeea; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="P/l/zeea" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 1674EC4CEED; Sat, 17 May 2025 02:25:17 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1747448717; bh=rxExDmAR8OpsxVyOb93D2F15CH0EPCc4/kwgbjLbV9Y=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=P/l/zeeaXlGjk85ozjkiv28mbQYM7Eywozk64Ge4ByotlQz/Ay52oeNxkTuhNQ0Xs 0c1HUsfX8sbbds48bccgUqdXM9ET90XngyZ3FFa08krugskvGIL7AfLQHb/M9LhegA +lvnwjclZhv3Ppy+s8egKJGviQPxBfb9g/5ZemZQfiOEGN8mmJh5T49jzcp51DXR7A saT45Gbtey7sO3XQxvASua+4Ekpr7YlRTtQ45vhgu9X2UNEf7B1DAlmZ3WIleVTr/5 2ejbGtRYpPcMIkAt2tY7IKGdkm+oBjX/nonENKohJDyYd7Gmq3V+HJtEpoTN0cfvDF tZ7YaQIazksZA== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Subject: [PATCH 1/8] Revert "crypto: sha256 - Use the partial block API" Date: Fri, 16 May 2025 19:24:21 -0700 Message-ID: <20250517022428.401622-2-ebiggers@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250517022428.401622-1-ebiggers@kernel.org> References: <20250517022428.401622-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Eric Biggers This reverts commit 3bf5337879101166dfacfbc2a780d1a379c288ba which got pushed out despite being nacked. The library API already has to handle partial blocks, and it makes a lot more sense to just use that. Signed-off-by: Eric Biggers --- crypto/sha256.c | 81 ++++++++++++++----------------------------------- 1 file changed, 23 insertions(+), 58 deletions(-) diff --git a/crypto/sha256.c b/crypto/sha256.c index 4aeb213bab117..cf190114574ea 100644 --- a/crypto/sha256.c +++ b/crypto/sha256.c @@ -50,24 +50,18 @@ static int crypto_sha256_update_generic(struct shash_de= sc *desc, const u8 *data, unsigned int len) { return crypto_sha256_update(desc, data, len, true); } =20 -static int crypto_sha256_update_lib(struct shash_desc *desc, const u8 *dat= a, - unsigned int len) -{ - sha256_update(shash_desc_ctx(desc), data, len); - return 0; -} - static int crypto_sha256_update_arch(struct shash_desc *desc, const u8 *da= ta, unsigned int len) { - return crypto_sha256_update(desc, data, len, false); + sha256_update(shash_desc_ctx(desc), data, len); + return 0; } =20 -static int crypto_sha256_final_lib(struct shash_desc *desc, u8 *out) +static int crypto_sha256_final_arch(struct shash_desc *desc, u8 *out) { sha256_final(shash_desc_ctx(desc), out); return 0; } =20 @@ -97,41 +91,38 @@ static int crypto_sha256_finup_generic(struct shash_des= c *desc, const u8 *data, } =20 static int crypto_sha256_finup_arch(struct shash_desc *desc, const u8 *dat= a, unsigned int len, u8 *out) { - return crypto_sha256_finup(desc, data, len, out, false); + struct sha256_state *sctx =3D shash_desc_ctx(desc); + + sha256_update(sctx, data, len); + sha256_final(sctx, out); + return 0; } =20 static int crypto_sha256_digest_generic(struct shash_desc *desc, const u8 = *data, unsigned int len, u8 *out) { crypto_sha256_init(desc); return crypto_sha256_finup_generic(desc, data, len, out); } =20 -static int crypto_sha256_digest_lib(struct shash_desc *desc, const u8 *dat= a, - unsigned int len, u8 *out) -{ - sha256(data, len, out); - return 0; -} - static int crypto_sha256_digest_arch(struct shash_desc *desc, const u8 *da= ta, unsigned int len, u8 *out) { - crypto_sha256_init(desc); - return crypto_sha256_finup_arch(desc, data, len, out); + sha256(data, len, out); + return 0; } =20 static int crypto_sha224_init(struct shash_desc *desc) { sha224_block_init(shash_desc_ctx(desc)); return 0; } =20 -static int crypto_sha224_final_lib(struct shash_desc *desc, u8 *out) +static int crypto_sha224_final_arch(struct shash_desc *desc, u8 *out) { sha224_final(shash_desc_ctx(desc), out); return 0; } =20 @@ -191,79 +182,53 @@ static struct shash_alg algs[] =3D { .finup =3D crypto_sha256_finup_generic, .descsize =3D sizeof(struct crypto_sha256_state), }, { .base.cra_name =3D "sha256", - .base.cra_driver_name =3D "sha256-lib", + .base.cra_driver_name =3D "sha256-" __stringify(ARCH), + .base.cra_priority =3D 300, .base.cra_blocksize =3D SHA256_BLOCK_SIZE, .base.cra_module =3D THIS_MODULE, .digestsize =3D SHA256_DIGEST_SIZE, .init =3D crypto_sha256_init, - .update =3D crypto_sha256_update_lib, - .final =3D crypto_sha256_final_lib, - .digest =3D crypto_sha256_digest_lib, + .update =3D crypto_sha256_update_arch, + .final =3D crypto_sha256_final_arch, + .finup =3D crypto_sha256_finup_arch, + .digest =3D crypto_sha256_digest_arch, .descsize =3D sizeof(struct sha256_state), .statesize =3D sizeof(struct crypto_sha256_state) + SHA256_BLOCK_SIZE + 1, .import =3D crypto_sha256_import_lib, .export =3D crypto_sha256_export_lib, }, { .base.cra_name =3D "sha224", - .base.cra_driver_name =3D "sha224-lib", + .base.cra_driver_name =3D "sha224-" __stringify(ARCH), + .base.cra_priority =3D 300, .base.cra_blocksize =3D SHA224_BLOCK_SIZE, .base.cra_module =3D THIS_MODULE, .digestsize =3D SHA224_DIGEST_SIZE, .init =3D crypto_sha224_init, - .update =3D crypto_sha256_update_lib, - .final =3D crypto_sha224_final_lib, + .update =3D crypto_sha256_update_arch, + .final =3D crypto_sha224_final_arch, .descsize =3D sizeof(struct sha256_state), .statesize =3D sizeof(struct crypto_sha256_state) + SHA256_BLOCK_SIZE + 1, .import =3D crypto_sha256_import_lib, .export =3D crypto_sha256_export_lib, }, - { - .base.cra_name =3D "sha256", - .base.cra_driver_name =3D "sha256-" __stringify(ARCH), - .base.cra_priority =3D 300, - .base.cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize =3D SHA256_BLOCK_SIZE, - .base.cra_module =3D THIS_MODULE, - .digestsize =3D SHA256_DIGEST_SIZE, - .init =3D crypto_sha256_init, - .update =3D crypto_sha256_update_arch, - .finup =3D crypto_sha256_finup_arch, - .digest =3D crypto_sha256_digest_arch, - .descsize =3D sizeof(struct crypto_sha256_state), - }, - { - .base.cra_name =3D "sha224", - .base.cra_driver_name =3D "sha224-" __stringify(ARCH), - .base.cra_priority =3D 300, - .base.cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize =3D SHA224_BLOCK_SIZE, - .base.cra_module =3D THIS_MODULE, - .digestsize =3D SHA224_DIGEST_SIZE, - .init =3D crypto_sha224_init, - .update =3D crypto_sha256_update_arch, - .finup =3D crypto_sha256_finup_arch, - .descsize =3D sizeof(struct crypto_sha256_state), - }, }; =20 static unsigned int num_algs; =20 static int __init crypto_sha256_mod_init(void) { /* register the arch flavours only if they differ from generic */ num_algs =3D ARRAY_SIZE(algs); - BUILD_BUG_ON(ARRAY_SIZE(algs) <=3D 2); + BUILD_BUG_ON(ARRAY_SIZE(algs) % 2 !=3D 0); if (!sha256_is_arch_optimized()) - num_algs -=3D 2; + num_algs /=3D 2; return crypto_register_shashes(algs, ARRAY_SIZE(algs)); } module_init(crypto_sha256_mod_init); =20 static void __exit crypto_sha256_mod_exit(void) --=20 2.49.0 From nobody Fri Dec 19 17:35:49 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 38880155A30; Sat, 17 May 2025 02:25:17 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448718; cv=none; b=BtiGsgn5ZQYSHwzj058d/Yr857MBegCp0QvPEMOdIZkcqa2oXHZjainIe01vLu/GEj8+TB4TW6am+CemV0zSjHSXB4ttT98Ky9bfdHmvilzo0+OQZj+3Chmz+AMPeAS/8FZ6ZS8oD0JZGdS8z9yrYengVtbzeV+rVBQ9M1RbFfE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448718; c=relaxed/simple; bh=3e9dMx8TONdupA1e08zfixVPVRX4LqDVpRilevWdVaU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=eNlw1MaEVv48sk/vGjIz59AeLM6xUUqRBSXL6BY1xR51Ar7E9RVy6IBKjiiI0nLgV+aj5Ujk+LbbWMj9NFDYJuVsJ/WksOanjtArEXxRvaWZjXG1DTn0aFN/NwBXhmHFVs0AP0FbyACj8Y3I/mvZsA6JlZuekQK/19cJ5MWrfJw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=aJrdv3VM; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="aJrdv3VM" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 49FB0C4CEEF; Sat, 17 May 2025 02:25:17 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1747448717; bh=3e9dMx8TONdupA1e08zfixVPVRX4LqDVpRilevWdVaU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=aJrdv3VMNlbtKAARwnCNkOGb8l1N9/cWUK+uQ1Zpop9HmWeiJ4VzY6Uuap9mRxoao XkhXMvdATmK5j3P9WTUEBuc3E5x3i+Hw4N8yFmbbolhSTic7u3ZtdkclOZsa3icnfT VMV1z+kkqMLOSWJXo5GITfi7fv4hig+XQZCw1mZmIhU8uhPs1qURausHt2D5R9DtHB 4UKW1En6je8mfg0Z4X3JSjBuD8U6MpgXaRHIPfNNfiDhLf5IwgR/qUDrT/zozqwk4m 6h+0rB23OIOV4MecogJInh3pD3qBeEm48/YFiysMxvaGQXrJlwI7oPV5hDgqsaKRh9 ojDKbL4guIWtA== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Subject: [PATCH 2/8] Revert "crypto: lib/sha256 - Use generic block helper" Date: Fri, 16 May 2025 19:24:22 -0700 Message-ID: <20250517022428.401622-3-ebiggers@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250517022428.401622-1-ebiggers@kernel.org> References: <20250517022428.401622-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Eric Biggers This reverts commit 3007e90572d0c5fd409c3d2fa8cedcbd5cb06d4b which got pushed out despite being nacked. BLOCK_HASH_UPDATE_BLOCKS makes the code harder to read and isn't really worth it. The *_generic() functions are needed for shash. Signed-off-by: Eric Biggers --- include/crypto/internal/sha2.h | 7 ++++ lib/crypto/sha256.c | 71 +++++++++++++++++++++++++++++----- 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/include/crypto/internal/sha2.h b/include/crypto/internal/sha2.h index b9bccd3ff57fc..fff156f66edc3 100644 --- a/include/crypto/internal/sha2.h +++ b/include/crypto/internal/sha2.h @@ -8,10 +8,17 @@ #include #include #include #include =20 +void sha256_update_generic(struct sha256_state *sctx, + const u8 *data, size_t len); +void sha256_final_generic(struct sha256_state *sctx, + u8 out[SHA256_DIGEST_SIZE]); +void sha224_final_generic(struct sha256_state *sctx, + u8 out[SHA224_DIGEST_SIZE]); + #if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) bool sha256_is_arch_optimized(void); #else static inline bool sha256_is_arch_optimized(void) { diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 107e5162507a7..2ced29efa181c 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -9,11 +9,10 @@ * Copyright (c) Andrew McDonald * Copyright (c) 2002 James Morris * Copyright (c) 2014 Red Hat Inc. */ =20 -#include #include #include #include #include =20 @@ -30,44 +29,75 @@ static inline bool sha256_purgatory(void) { return __is_defined(__DISABLE_EXPORTS); } =20 static inline void sha256_blocks(u32 state[SHA256_STATE_WORDS], const u8 *= data, - size_t nblocks) + size_t nblocks, bool force_generic) { - sha256_choose_blocks(state, data, nblocks, sha256_purgatory(), false); + sha256_choose_blocks(state, data, nblocks, + force_generic || sha256_purgatory(), false); } =20 -void sha256_update(struct sha256_state *sctx, const u8 *data, size_t len) +static inline void __sha256_update(struct sha256_state *sctx, const u8 *da= ta, + size_t len, bool force_generic) { size_t partial =3D sctx->count % SHA256_BLOCK_SIZE; =20 sctx->count +=3D len; - BLOCK_HASH_UPDATE_BLOCKS(sha256_blocks, sctx->ctx.state, data, len, - SHA256_BLOCK_SIZE, sctx->buf, partial); + + if (partial + len >=3D SHA256_BLOCK_SIZE) { + size_t nblocks; + + if (partial) { + size_t l =3D SHA256_BLOCK_SIZE - partial; + + memcpy(&sctx->buf[partial], data, l); + data +=3D l; + len -=3D l; + + sha256_blocks(sctx->state, sctx->buf, 1, force_generic); + } + + nblocks =3D len / SHA256_BLOCK_SIZE; + len %=3D SHA256_BLOCK_SIZE; + + if (nblocks) { + sha256_blocks(sctx->state, data, nblocks, + force_generic); + data +=3D nblocks * SHA256_BLOCK_SIZE; + } + partial =3D 0; + } + if (len) + memcpy(&sctx->buf[partial], data, len); +} + +void sha256_update(struct sha256_state *sctx, const u8 *data, size_t len) +{ + __sha256_update(sctx, data, len, false); } EXPORT_SYMBOL(sha256_update); =20 static inline void __sha256_final(struct sha256_state *sctx, u8 *out, - size_t digest_size) + size_t digest_size, bool force_generic) { size_t partial =3D sctx->count % SHA256_BLOCK_SIZE; =20 sha256_finup(&sctx->ctx, sctx->buf, partial, out, digest_size, - sha256_purgatory(), false); + force_generic || sha256_purgatory(), false); memzero_explicit(sctx, sizeof(*sctx)); } =20 void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]) { - __sha256_final(sctx, out, SHA256_DIGEST_SIZE); + __sha256_final(sctx, out, SHA256_DIGEST_SIZE, false); } EXPORT_SYMBOL(sha256_final); =20 void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]) { - __sha256_final(sctx, out, SHA224_DIGEST_SIZE); + __sha256_final(sctx, out, SHA224_DIGEST_SIZE, false); } EXPORT_SYMBOL(sha224_final); =20 void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]) { @@ -77,7 +107,28 @@ void sha256(const u8 *data, size_t len, u8 out[SHA256_D= IGEST_SIZE]) sha256_update(&sctx, data, len); sha256_final(&sctx, out); } EXPORT_SYMBOL(sha256); =20 +#if IS_ENABLED(CONFIG_CRYPTO_SHA256) && !defined(__DISABLE_EXPORTS) +void sha256_update_generic(struct sha256_state *sctx, + const u8 *data, size_t len) +{ + __sha256_update(sctx, data, len, true); +} +EXPORT_SYMBOL(sha256_update_generic); + +void sha256_final_generic(struct sha256_state *sctx, u8 out[SHA256_DIGEST_= SIZE]) +{ + __sha256_final(sctx, out, SHA256_DIGEST_SIZE, true); +} +EXPORT_SYMBOL(sha256_final_generic); + +void sha224_final_generic(struct sha256_state *sctx, u8 out[SHA224_DIGEST_= SIZE]) +{ + __sha256_final(sctx, out, SHA224_DIGEST_SIZE, true); +} +EXPORT_SYMBOL(sha224_final_generic); +#endif + MODULE_DESCRIPTION("SHA-256 Algorithm"); MODULE_LICENSE("GPL"); --=20 2.49.0 From nobody Fri Dec 19 17:35:49 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 3883B1553AA; Sat, 17 May 2025 02:25:17 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448718; cv=none; b=CzR0s97ggI/rHy5cj4exzdb+d2GDgnzG95JyYI+XEWKe5GU4w3wew01agucUeN/aq0ch4OSYD/fAbUR1RPkcggy6kmH23QWg3HCP3VAy6jI8xuQTL2aMTYSW1JjKpz5PNGN5e4kj69Y8uxUHlAvfl7FwB8Nj3dZDw9eeB7GGWUk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448718; c=relaxed/simple; bh=zUUQ2TPc05qFjP/3dGiAormza2njLcbl0lYH1g6EJzM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=FSva9bnjLoo/vQLt9Fzi3U9aF7soRgBkXsPF+/zRt/OCfYM+EZ2zuIkanFhMXQbsOipV46ga/9KgczR5fsLkGIhf9Nw44OeoTJnUi4g0C5kwjPX/q0GU/cLZJKYggQav3eSF2HySfhi031RTVt3z1axQwEeZ0XOfcW5xeVyQGrA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Z2S/9K23; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Z2S/9K23" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 7D65BC4CEF0; Sat, 17 May 2025 02:25:17 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1747448717; bh=zUUQ2TPc05qFjP/3dGiAormza2njLcbl0lYH1g6EJzM=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Z2S/9K23SvOJkqAVecLH7gOCj9koGAHFiDlFoWZ2b+ne16GebeCSatT+reQ4uAp7R pX20NSu9qRDSfNTk657+KJE6lgHArP7KHk22E7agp7tA6OZuOKnbZ+JluzxAg8p5/w OsDz19ZO+3WbbSP0w71lA+bWAl30BS7oqaGF6UuteQlgD+pnxzvPmAn5LcBZFlDa5C sIwYNTLw9G2t9y1yC4FaDb5Vhfk2DB1tfT0WTNTeD0AN8LSx4SZJ69N6+uZU20xi4Y JZEQEialJvh7iGIXvjlOe7B5S0BlYUiAFh5tykVUacYbEdIY7Vp+RZ78o1LY8obUia 0e1tXmGNhptXA== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Subject: [PATCH 3/8] Revert "crypto: x86/sha256 - Add simd block function" Date: Fri, 16 May 2025 19:24:23 -0700 Message-ID: <20250517022428.401622-4-ebiggers@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250517022428.401622-1-ebiggers@kernel.org> References: <20250517022428.401622-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Eric Biggers This reverts commit ee8a720e39ceb7495ab639c1eb6d4987fb6a52bf which got pushed out despite being nacked. That commit added a special low-level interface to allow the crypto_shash API to bypass the safety check for using kernel-mode FPU. It could give a marginal performance benefit for crypto_shash, but just is not worth the complexity and footgun. Moreover, the distinction between "arch" and "simd" is confusing and is not something that really should exist in generic code, given that different architectures can mean different things by "simd". Signed-off-by: Eric Biggers --- arch/x86/lib/crypto/Kconfig | 1 - arch/x86/lib/crypto/sha256.c | 12 +++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/arch/x86/lib/crypto/Kconfig b/arch/x86/lib/crypto/Kconfig index 5e94cdee492c2..e344579db3d85 100644 --- a/arch/x86/lib/crypto/Kconfig +++ b/arch/x86/lib/crypto/Kconfig @@ -28,7 +28,6 @@ config CRYPTO_POLY1305_X86_64 config CRYPTO_SHA256_X86_64 tristate depends on 64BIT default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD select CRYPTO_LIB_SHA256_GENERIC diff --git a/arch/x86/lib/crypto/sha256.c b/arch/x86/lib/crypto/sha256.c index 80380f8fdcee4..baba74d7d26f2 100644 --- a/arch/x86/lib/crypto/sha256.c +++ b/arch/x86/lib/crypto/sha256.c @@ -4,10 +4,11 @@ * * Copyright 2025 Google LLC */ #include #include +#include #include #include #include =20 asmlinkage void sha256_transform_ssse3(u32 state[SHA256_STATE_WORDS], @@ -21,28 +22,21 @@ asmlinkage void sha256_ni_transform(u32 state[SHA256_ST= ATE_WORDS], =20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_x86); =20 DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_transform_ssse3); =20 -void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks) { - if (static_branch_likely(&have_sha256_x86)) { + if (static_branch_likely(&have_sha256_x86) && crypto_simd_usable()) { kernel_fpu_begin(); static_call(sha256_blocks_x86)(state, data, nblocks); kernel_fpu_end(); } else { sha256_blocks_generic(state, data, nblocks); } } -EXPORT_SYMBOL_GPL(sha256_blocks_simd); - -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], - const u8 *data, size_t nblocks) -{ - sha256_blocks_generic(state, data, nblocks); -} EXPORT_SYMBOL_GPL(sha256_blocks_arch); =20 bool sha256_is_arch_optimized(void) { return static_key_enabled(&have_sha256_x86); --=20 2.49.0 From nobody Fri Dec 19 17:35:49 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 387F5155335; Sat, 17 May 2025 02:25:17 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448718; cv=none; b=mCZrsMQHwQnw+pVLREoIi2Q6hltVusa/s9zStksSlryL65GbfoWvK6ENng/KXhGL6HPjovRaAGtA30IMqB/fE1lCqNSWPbVkGHD+HvPsCAWpIBPB2wL9/Zs3NtOu77t47HxT5R9kEewp93G2pUaTw6gP84/v632gOtt14PXIj6s= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448718; c=relaxed/simple; bh=RQzXMFdhT1ylCP6mlf030+K7wWNM4Zi/92juH8XlYe8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=chJYKPe4FdgjYXjb3AV5XG8Jrvyr50DI0yzgHCtwe0Jpc9ybEh7mAN6QPUiHNILB9X7JP2Yi0LeRn9usm8hOdmj6+wDBSZ37G/4pXNomfJGygthdYFRoqnW0c/sC6IrGWQVteVciuXa2vW84DPtlVcwNmvWGK3qJY0BbhJfC5rc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=sFPr5i8J; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="sFPr5i8J" Received: by smtp.kernel.org (Postfix) with ESMTPSA id B0CFAC4CEF1; Sat, 17 May 2025 02:25:17 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1747448717; bh=RQzXMFdhT1ylCP6mlf030+K7wWNM4Zi/92juH8XlYe8=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=sFPr5i8JqNit2AvE7B26blWgaOQSyYK6Z77ZN/1NHPIgldG1VZfO7/5lLjfr6IIiX 4FEkrlZuinxg+xR0JYu1dlYeAV13u/sOGtQ8pSAgbBVqbIHytuS4woVxrBhSdUvGvr wG0c0YkUDD4OnYnil/o5DXFlPvu8tGpJ/ZgK5nT8C5BwiAqipjFKNY3nUHW+6Qp74h Fsq76s8NAFBMBdOwE1mMj6Y2T48JODZyLRNkDmQJPE16Ew8IydUth7HQ1WNkpJ8Sdy brYy0fTE9pQg77kER9lEAAE/IvLnK88KnVIT8jRqCIKZ6xZ6SEh83CYN6KVEeac7tU aup4xbVNHpVqA== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Subject: [PATCH 4/8] Revert "crypto: riscv/sha256 - Add simd block function" Date: Fri, 16 May 2025 19:24:24 -0700 Message-ID: <20250517022428.401622-5-ebiggers@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250517022428.401622-1-ebiggers@kernel.org> References: <20250517022428.401622-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Eric Biggers This reverts commit 491d6024f2820c78216b07cec1cb47c87dcae077 which got pushed out despite being nacked. That commit added a special low-level interface to allow the crypto_shash API to bypass the safety check for using kernel-mode vector. It could give a marginal performance benefit for crypto_shash, but just is not worth the complexity and footgun. Moreover, the distinction between "arch" and "simd" is confusing and is not something that really should exist in generic code, given that different architectures can mean different things by "simd". Signed-off-by: Eric Biggers --- arch/riscv/lib/crypto/Kconfig | 1 - arch/riscv/lib/crypto/sha256.c | 13 ++++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/arch/riscv/lib/crypto/Kconfig b/arch/riscv/lib/crypto/Kconfig index 47c99ea97ce2c..c100571feb7e8 100644 --- a/arch/riscv/lib/crypto/Kconfig +++ b/arch/riscv/lib/crypto/Kconfig @@ -10,7 +10,6 @@ config CRYPTO_CHACHA_RISCV64 config CRYPTO_SHA256_RISCV64 tristate depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD select CRYPTO_LIB_SHA256_GENERIC diff --git a/arch/riscv/lib/crypto/sha256.c b/arch/riscv/lib/crypto/sha256.c index 71808397dff4c..4ad3ffb8e0a98 100644 --- a/arch/riscv/lib/crypto/sha256.c +++ b/arch/riscv/lib/crypto/sha256.c @@ -7,38 +7,33 @@ * * Copyright (C) 2023 SiFive, Inc. * Author: Jerry Shih */ =20 +#include #include #include +#include #include #include =20 asmlinkage void sha256_transform_zvknha_or_zvknhb_zvkb( u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); =20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions); =20 -void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks) { - if (static_branch_likely(&have_extensions)) { + if (static_branch_likely(&have_extensions) && crypto_simd_usable()) { kernel_vector_begin(); sha256_transform_zvknha_or_zvknhb_zvkb(state, data, nblocks); kernel_vector_end(); } else { sha256_blocks_generic(state, data, nblocks); } } -EXPORT_SYMBOL_GPL(sha256_blocks_simd); - -void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], - const u8 *data, size_t nblocks) -{ - sha256_blocks_generic(state, data, nblocks); -} EXPORT_SYMBOL_GPL(sha256_blocks_arch); =20 bool sha256_is_arch_optimized(void) { return static_key_enabled(&have_extensions); --=20 2.49.0 From nobody Fri Dec 19 17:35:49 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4FC081917E3; Sat, 17 May 2025 02:25:18 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448719; cv=none; b=Da0LMM3AP9Xw3euszd8xCqZahP9xoeTEWHVWsatVbAbKtpH+jc3xx1lrxkYTKIBpCiOJSwDSXUniy7nsh1Nbeth26qL5pK9FjgD0ifHC1Df5tnQbgDUwi/AxkYzQdO/9WE5Eu4ak+Hxfk4vtn6kkpybFkLwu1VT+fpcaQ+78G2g= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448719; c=relaxed/simple; bh=gUUxjIgDrRMsYzb7G3k3d2wphMLjs03o+vzyCoM8US0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=mT7hfBmsDMXwLbziFm0YxCJmXbQmkopIv3cq61S0zQKojxwAp4M5mir6U4T06RccECsNw+wBcQLXkt6OijNP15Nmo1E7i3TM2l0DNK69a2/6V1IpFtO7hRzPYq+4Lq5jbuzLGbZ0uK0eE3sR2Dv26eOMIIy7hwc9aaTTfloDIXQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=JofceYRe; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="JofceYRe" Received: by smtp.kernel.org (Postfix) with ESMTPSA id E465AC4CEED; Sat, 17 May 2025 02:25:17 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1747448718; bh=gUUxjIgDrRMsYzb7G3k3d2wphMLjs03o+vzyCoM8US0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=JofceYRedR4KKMmWWK2UdkTiGZryBVX5oeS8Z2nUEoeezPsU6MPL9xeFnsiuuHikW CvHrylhrWgono/CkurcSWIBK+iA7W9DD2gmzzzyxa8paJ4g6Oee8ancYIJiLyux+Jw u663COUXKo0v0ov9sEL1zyqNc5uwZwk8TCoGmK1H/6oJw633iCRg7g00LckNkTIgoI htL45QQCFPFKC7TrLQK6ucEZWQxX6Fj4xH4a6EAfN1U/lnx0E1bYAZnwG1/Y+2NFxY /VFKKtSnTZZiyCW5afXbhkALdEd7SIjk+6l6z0cn6qq9ghT/O0ryk44NVQ7qVfCUq/ d0L2UDmSYvoMA== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Subject: [PATCH 5/8] Revert "crypto: arm64/sha256 - Add simd block function" Date: Fri, 16 May 2025 19:24:25 -0700 Message-ID: <20250517022428.401622-6-ebiggers@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250517022428.401622-1-ebiggers@kernel.org> References: <20250517022428.401622-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Eric Biggers This reverts commit adcb9e32e5e28935ec1148e1a314282a7367428d which got pushed out despite being nacked. That commit added a special low-level interface to allow the crypto_shash API to bypass the safety check for using kernel-mode NEON. It could give a marginal performance benefit for crypto_shash, but just is not worth the complexity and footgun. Moreover, the distinction between "arch" and "simd" is confusing and is not something that really should exist in generic code, given that different architectures can mean different things by "simd". Signed-off-by: Eric Biggers --- arch/arm64/crypto/sha512-glue.c | 6 +++--- arch/arm64/lib/crypto/Kconfig | 1 - arch/arm64/lib/crypto/sha2-armv8.pl | 2 +- arch/arm64/lib/crypto/sha256.c | 14 +++++++------- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glu= e.c index 15aa9d8b7b2c4..ab2e1c13dfadc 100644 --- a/arch/arm64/crypto/sha512-glue.c +++ b/arch/arm64/crypto/sha512-glue.c @@ -16,17 +16,17 @@ MODULE_AUTHOR("Andy Polyakov "); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); MODULE_ALIAS_CRYPTO("sha384"); MODULE_ALIAS_CRYPTO("sha512"); =20 -asmlinkage void sha512_blocks_arch(u64 *digest, const void *data, - unsigned int num_blks); +asmlinkage void sha512_block_data_order(u64 *digest, const void *data, + unsigned int num_blks); =20 static void sha512_arm64_transform(struct sha512_state *sst, u8 const *src, int blocks) { - sha512_blocks_arch(sst->state, src, blocks); + sha512_block_data_order(sst->state, src, blocks); } =20 static int sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) { diff --git a/arch/arm64/lib/crypto/Kconfig b/arch/arm64/lib/crypto/Kconfig index 129a7685cb4c1..49e57bfdb5b52 100644 --- a/arch/arm64/lib/crypto/Kconfig +++ b/arch/arm64/lib/crypto/Kconfig @@ -15,6 +15,5 @@ config CRYPTO_POLY1305_NEON =20 config CRYPTO_SHA256_ARM64 tristate default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD diff --git a/arch/arm64/lib/crypto/sha2-armv8.pl b/arch/arm64/lib/crypto/sh= a2-armv8.pl index 4aebd20c498bc..35ec9ae99fe16 100644 --- a/arch/arm64/lib/crypto/sha2-armv8.pl +++ b/arch/arm64/lib/crypto/sha2-armv8.pl @@ -93,11 +93,11 @@ if ($output =3D~ /512/) { @sigma1=3D(17,19,10); $rounds=3D64; $reg_t=3D"w"; } =20 -$func=3D"sha${BITS}_blocks_arch"; +$func=3D"sha${BITS}_block_data_order"; =20 ($ctx,$inp,$num,$Ktbl)=3Dmap("x$_",(0..2,30)); =20 @X=3Dmap("$reg_t$_",(3..15,0..2)); @V=3D($A,$B,$C,$D,$E,$F,$G,$H)=3Dmap("$reg_t$_",(20..27)); diff --git a/arch/arm64/lib/crypto/sha256.c b/arch/arm64/lib/crypto/sha256.c index bcf7a3adc0c46..fb9bff40357be 100644 --- a/arch/arm64/lib/crypto/sha256.c +++ b/arch/arm64/lib/crypto/sha256.c @@ -4,29 +4,29 @@ * * Copyright 2025 Google LLC */ #include #include +#include #include #include =20 -asmlinkage void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], - const u8 *data, size_t nblocks); -EXPORT_SYMBOL_GPL(sha256_blocks_arch); +asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); asmlinkage void sha256_block_neon(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); asmlinkage size_t __sha256_ce_transform(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); =20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); =20 -void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks) { if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_neon)) { + static_branch_likely(&have_neon) && crypto_simd_usable()) { if (static_branch_likely(&have_ce)) { do { size_t rem; =20 kernel_neon_begin(); @@ -40,14 +40,14 @@ void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], kernel_neon_begin(); sha256_block_neon(state, data, nblocks); kernel_neon_end(); } } else { - sha256_blocks_arch(state, data, nblocks); + sha256_block_data_order(state, data, nblocks); } } -EXPORT_SYMBOL_GPL(sha256_blocks_simd); +EXPORT_SYMBOL_GPL(sha256_blocks_arch); =20 bool sha256_is_arch_optimized(void) { /* We always can use at least the ARM64 scalar implementation. */ return true; --=20 2.49.0 From nobody Fri Dec 19 17:35:49 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4FBBC1917D6; Sat, 17 May 2025 02:25:18 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448719; cv=none; b=Mhsdr1f2i0ICmeM1Cj1S7nKeUSrgDxVpMI0w8FyBR7LAyWIImsXI+JnHi0AHUSu4beicLNCQdzIVzn8MS4hTFGRHtXRUeSm+SI7MN9yh07nw7F7SepuN5rYPVlB9D8PNd9raQVscdqo6uizmxsqOfwfH8eubj14GLiVDhqrEEBg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448719; c=relaxed/simple; bh=XXmV0q88v0oxfMFM0WBoheMAne6q7Vmr/cC9Ycy/h/I=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=aBRSknrUSM9IRCCNTqvkRfvc8f/sV7CXwOpGkKx/IW+RS/e1lQzUpfLGDHF32K3+hF05ocJ4Q+R0hD+P5Pmx+UU0nJqgYBYLRlB2FzTlh3BKe37HiqcFr64uhop1A1/nWCsQpNm2yUnzBqAoXW6cehyOg9/aBib8iPaYhTDM+k0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=NbsDb+Ey; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="NbsDb+Ey" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 2366DC4CEF7; Sat, 17 May 2025 02:25:18 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1747448718; bh=XXmV0q88v0oxfMFM0WBoheMAne6q7Vmr/cC9Ycy/h/I=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=NbsDb+EyeGdD5IFUZxcSfeMjmMFLVf9z4vgyaYEOeoRZ0CwE/Kdn759lTNGuYS+MO x7GPcQE+IRTj55Q8xPdTruecGzcITYbdcBgwEcNiGiRGwHiYce0zFawqqTNADwPeDl KF12WMTAMLj4dwQWXAbJnEdoTIVBCpOcglrAiTxSzUUA0am3m850hCY0SVJgU5S6cS HnnXkkR3j+pxyyQK7KQvnPbuSprX9WVPQTvDQ25srEBFY7M2wEIDRnAdW0+KwiP6fT hUvfxt/S1ukmCwWQxsGEYa/I1w24KcnYGpcYk0x5O5OV/jNpjwgjq1NIPnZyIcvjOp 3g4cBAJkdaV4Q== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Subject: [PATCH 6/8] Revert "crypto: arm/sha256 - Add simd block function" Date: Fri, 16 May 2025 19:24:26 -0700 Message-ID: <20250517022428.401622-7-ebiggers@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250517022428.401622-1-ebiggers@kernel.org> References: <20250517022428.401622-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Eric Biggers This reverts commit 2e43fc6d79cb24e0fe34aa4c2229ab6c414d852a which got pushed out despite being nacked. That commit added a special low-level interface to allow the crypto_shash API to bypass the safety check for using kernel-mode NEON. It could give a marginal performance benefit for crypto_shash, but just is not worth the complexity and footgun. Moreover, the distinction between "arch" and "simd" is confusing and is not something that really should exist in generic code, given that different architectures can mean different things by "simd". Signed-off-by: Eric Biggers --- arch/arm/lib/crypto/Kconfig | 1 - arch/arm/lib/crypto/sha256-armv4.pl | 20 ++++++++++---------- arch/arm/lib/crypto/sha256.c | 14 +++++++------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/arch/arm/lib/crypto/Kconfig b/arch/arm/lib/crypto/Kconfig index d1ad664f0c674..9f3ff30f40328 100644 --- a/arch/arm/lib/crypto/Kconfig +++ b/arch/arm/lib/crypto/Kconfig @@ -26,6 +26,5 @@ config CRYPTO_POLY1305_ARM config CRYPTO_SHA256_ARM tristate depends on !CPU_V7M default CRYPTO_LIB_SHA256 select CRYPTO_ARCH_HAVE_LIB_SHA256 - select CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD diff --git a/arch/arm/lib/crypto/sha256-armv4.pl b/arch/arm/lib/crypto/sha2= 56-armv4.pl index 8122db7fd5990..f3a2b54efd4ee 100644 --- a/arch/arm/lib/crypto/sha256-armv4.pl +++ b/arch/arm/lib/crypto/sha256-armv4.pl @@ -202,22 +202,22 @@ K256: .word 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 .size K256,.-K256 .word 0 @ terminator #if __ARM_MAX_ARCH__>=3D7 && !defined(__KERNEL__) .LOPENSSL_armcap: -.word OPENSSL_armcap_P-sha256_blocks_arch +.word OPENSSL_armcap_P-sha256_block_data_order #endif .align 5 =20 -.global sha256_blocks_arch -.type sha256_blocks_arch,%function -sha256_blocks_arch: -.Lsha256_blocks_arch: +.global sha256_block_data_order +.type sha256_block_data_order,%function +sha256_block_data_order: +.Lsha256_block_data_order: #if __ARM_ARCH__<7 - sub r3,pc,#8 @ sha256_blocks_arch + sub r3,pc,#8 @ sha256_block_data_order #else - adr r3,.Lsha256_blocks_arch + adr r3,.Lsha256_block_data_order #endif #if __ARM_MAX_ARCH__>=3D7 && !defined(__KERNEL__) ldr r12,.LOPENSSL_armcap ldr r12,[r3,r12] @ OPENSSL_armcap_P tst r12,#ARMV8_SHA256 @@ -280,11 +280,11 @@ $code.=3D<<___; ldmia sp!,{r4-r11,lr} tst lr,#1 moveq pc,lr @ be binary compatible with V4, yet bx lr @ interoperable with Thumb ISA:-) #endif -.size sha256_blocks_arch,.-sha256_blocks_arch +.size sha256_block_data_order,.-sha256_block_data_order ___ ###################################################################### # NEON stuff # {{{ @@ -468,12 +468,12 @@ $code.=3D<<___; sha256_block_data_order_neon: .LNEON: stmdb sp!,{r4-r12,lr} =20 sub $H,sp,#16*4+16 - adr $Ktbl,.Lsha256_blocks_arch - sub $Ktbl,$Ktbl,#.Lsha256_blocks_arch-K256 + adr $Ktbl,.Lsha256_block_data_order + sub $Ktbl,$Ktbl,#.Lsha256_block_data_order-K256 bic $H,$H,#15 @ align for 128-bit stores mov $t2,sp mov sp,$H @ alloca add $len,$inp,$len,lsl#6 @ len to point at the end of inp =20 diff --git a/arch/arm/lib/crypto/sha256.c b/arch/arm/lib/crypto/sha256.c index 109192e54b0f0..2c9cfdaaa0691 100644 --- a/arch/arm/lib/crypto/sha256.c +++ b/arch/arm/lib/crypto/sha256.c @@ -4,40 +4,40 @@ * * Copyright 2025 Google LLC */ #include #include +#include #include #include =20 -asmlinkage void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], - const u8 *data, size_t nblocks); -EXPORT_SYMBOL_GPL(sha256_blocks_arch); +asmlinkage void sha256_block_data_order(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); asmlinkage void sha256_block_data_order_neon(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); asmlinkage void sha256_ce_transform(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); =20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); =20 -void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks) { if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_neon)) { + static_branch_likely(&have_neon) && crypto_simd_usable()) { kernel_neon_begin(); if (static_branch_likely(&have_ce)) sha256_ce_transform(state, data, nblocks); else sha256_block_data_order_neon(state, data, nblocks); kernel_neon_end(); } else { - sha256_blocks_arch(state, data, nblocks); + sha256_block_data_order(state, data, nblocks); } } -EXPORT_SYMBOL_GPL(sha256_blocks_simd); +EXPORT_SYMBOL_GPL(sha256_blocks_arch); =20 bool sha256_is_arch_optimized(void) { /* We always can use at least the ARM scalar implementation. */ return true; --=20 2.49.0 From nobody Fri Dec 19 17:35:49 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 8330415B135; Sat, 17 May 2025 02:25:18 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448718; cv=none; b=d4ujfF2fZX0DtrOOhk51YAKgYbb/QAbGaC3tUqP3n3vDcXUohDZWi1jGlTFzyB/bGKcTbl/vgTQ9ZjX1Ksm7momvauWkPEMfLUI6FB/HDHvAXGtVFu4WcMRKqhOchddSF/G4YZ5wpwT6cZtZtYhZhBfAvLdVU+AFXbKXk7lrklg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448718; c=relaxed/simple; bh=MQJYpC1dIjU+BppMsjIhJ5EZohiOBOMHzgF7MVI7/38=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=FXRSYvHa8TfG3URG6XQLOZ8twr9bMTjSGMLVnkkSO4AdCe0q//HizTv6AgF8AKJDrb++SB+OT0JyM97NT+iDWeKmPjnZsLOzs4D1b1C+n5G88U6MhNLU592o6AvQ/8WbckOlAoAxi6t2/9EL9Vq/hwsKSnBbhvEGjDh6u13C5BE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=GEuZHwQQ; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="GEuZHwQQ" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 571B7C4CEF9; Sat, 17 May 2025 02:25:18 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1747448718; bh=MQJYpC1dIjU+BppMsjIhJ5EZohiOBOMHzgF7MVI7/38=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=GEuZHwQQkEkvnz3EIPIZTvVxzLAZZ7mPVccjciQhCE/Jkoh7EI88DhZcs7n82EG8Y 8zt0dqtvNvA6tX7xTReehsFWWmjjMtt1quQETlilHWR5UG0bbQFhYLTwl2KO6zTR4j lQx1uEB7kPHNtzukiThtM/aqFfzyuBb0hBWG8J7ObHqCrN8hgVpmyXWnscw/pel46O E/puFsvxhxhOQOPyaHFzhxgQ6gTuDGKaF/DeAq3NaFamuoV8oARUgKcy4yV9u/e0W1 C6iMsQ+aqgyQS9I9cum9OERTZZHd83UHuOpL5txy4LrbdWc9b2rX4Fi407ItQKHU1T XWo2HFKMVcvUQ== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Subject: [PATCH 7/8] Revert "crypto: sha256 - Use the partial block API for generic" Date: Fri, 16 May 2025 19:24:27 -0700 Message-ID: <20250517022428.401622-8-ebiggers@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250517022428.401622-1-ebiggers@kernel.org> References: <20250517022428.401622-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Eric Biggers This reverts commit ff8f037d394f0900597ba527388a6eb95cd02695 which got pushed out despite being nacked. The library API already has to handle partial blocks, and it makes a lot more sense to just use that. Keep sha256_block_init() since drivers/crypto/padlock-sha.c is using it. Signed-off-by: Eric Biggers --- crypto/sha256.c | 79 ++++++++++++++++++++----------------------- include/crypto/sha2.h | 7 +--- 2 files changed, 38 insertions(+), 48 deletions(-) diff --git a/crypto/sha256.c b/crypto/sha256.c index cf190114574ea..47ad7e4cc55f7 100644 --- a/crypto/sha256.c +++ b/crypto/sha256.c @@ -28,68 +28,48 @@ const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = =3D { }; EXPORT_SYMBOL_GPL(sha256_zero_message_hash); =20 static int crypto_sha256_init(struct shash_desc *desc) { - sha256_block_init(shash_desc_ctx(desc)); + sha256_init(shash_desc_ctx(desc)); return 0; } =20 -static inline int crypto_sha256_update(struct shash_desc *desc, const u8 *= data, - unsigned int len, bool force_generic) -{ - struct crypto_sha256_state *sctx =3D shash_desc_ctx(desc); - int remain =3D len % SHA256_BLOCK_SIZE; - - sctx->count +=3D len - remain; - sha256_choose_blocks(sctx->state, data, len / SHA256_BLOCK_SIZE, - force_generic, !force_generic); - return remain; -} - static int crypto_sha256_update_generic(struct shash_desc *desc, const u8 = *data, unsigned int len) { - return crypto_sha256_update(desc, data, len, true); + sha256_update_generic(shash_desc_ctx(desc), data, len); + return 0; } =20 static int crypto_sha256_update_arch(struct shash_desc *desc, const u8 *da= ta, unsigned int len) { sha256_update(shash_desc_ctx(desc), data, len); return 0; } =20 -static int crypto_sha256_final_arch(struct shash_desc *desc, u8 *out) +static int crypto_sha256_final_generic(struct shash_desc *desc, u8 *out) { - sha256_final(shash_desc_ctx(desc), out); + sha256_final_generic(shash_desc_ctx(desc), out); return 0; } =20 -static __always_inline int crypto_sha256_finup(struct shash_desc *desc, - const u8 *data, - unsigned int len, u8 *out, - bool force_generic) +static int crypto_sha256_final_arch(struct shash_desc *desc, u8 *out) { - struct crypto_sha256_state *sctx =3D shash_desc_ctx(desc); - unsigned int remain =3D len; - u8 *buf; - - if (len >=3D SHA256_BLOCK_SIZE) - remain =3D crypto_sha256_update(desc, data, len, force_generic); - sctx->count +=3D remain; - buf =3D memcpy(sctx + 1, data + len - remain, remain); - sha256_finup(sctx, buf, remain, out, - crypto_shash_digestsize(desc->tfm), force_generic, - !force_generic); + sha256_final(shash_desc_ctx(desc), out); return 0; } =20 static int crypto_sha256_finup_generic(struct shash_desc *desc, const u8 *= data, unsigned int len, u8 *out) { - return crypto_sha256_finup(desc, data, len, out, true); + struct sha256_state *sctx =3D shash_desc_ctx(desc); + + sha256_update_generic(sctx, data, len); + sha256_final_generic(sctx, out); + return 0; } =20 static int crypto_sha256_finup_arch(struct shash_desc *desc, const u8 *dat= a, unsigned int len, u8 *out) { @@ -101,12 +81,16 @@ static int crypto_sha256_finup_arch(struct shash_desc = *desc, const u8 *data, } =20 static int crypto_sha256_digest_generic(struct shash_desc *desc, const u8 = *data, unsigned int len, u8 *out) { - crypto_sha256_init(desc); - return crypto_sha256_finup_generic(desc, data, len, out); + struct sha256_state *sctx =3D shash_desc_ctx(desc); + + sha256_init(sctx); + sha256_update_generic(sctx, data, len); + sha256_final_generic(sctx, out); + return 0; } =20 static int crypto_sha256_digest_arch(struct shash_desc *desc, const u8 *da= ta, unsigned int len, u8 *out) { @@ -114,11 +98,17 @@ static int crypto_sha256_digest_arch(struct shash_desc= *desc, const u8 *data, return 0; } =20 static int crypto_sha224_init(struct shash_desc *desc) { - sha224_block_init(shash_desc_ctx(desc)); + sha224_init(shash_desc_ctx(desc)); + return 0; +} + +static int crypto_sha224_final_generic(struct shash_desc *desc, u8 *out) +{ + sha224_final_generic(shash_desc_ctx(desc), out); return 0; } =20 static int crypto_sha224_final_arch(struct shash_desc *desc, u8 *out) { @@ -155,34 +145,39 @@ static int crypto_sha256_export_lib(struct shash_desc= *desc, void *out) static struct shash_alg algs[] =3D { { .base.cra_name =3D "sha256", .base.cra_driver_name =3D "sha256-generic", .base.cra_priority =3D 100, - .base.cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize =3D SHA256_BLOCK_SIZE, .base.cra_module =3D THIS_MODULE, .digestsize =3D SHA256_DIGEST_SIZE, .init =3D crypto_sha256_init, .update =3D crypto_sha256_update_generic, + .final =3D crypto_sha256_final_generic, .finup =3D crypto_sha256_finup_generic, .digest =3D crypto_sha256_digest_generic, - .descsize =3D sizeof(struct crypto_sha256_state), + .descsize =3D sizeof(struct sha256_state), + .statesize =3D sizeof(struct crypto_sha256_state) + + SHA256_BLOCK_SIZE + 1, + .import =3D crypto_sha256_import_lib, + .export =3D crypto_sha256_export_lib, }, { .base.cra_name =3D "sha224", .base.cra_driver_name =3D "sha224-generic", .base.cra_priority =3D 100, - .base.cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, .base.cra_blocksize =3D SHA224_BLOCK_SIZE, .base.cra_module =3D THIS_MODULE, .digestsize =3D SHA224_DIGEST_SIZE, .init =3D crypto_sha224_init, .update =3D crypto_sha256_update_generic, - .finup =3D crypto_sha256_finup_generic, - .descsize =3D sizeof(struct crypto_sha256_state), + .final =3D crypto_sha224_final_generic, + .descsize =3D sizeof(struct sha256_state), + .statesize =3D sizeof(struct crypto_sha256_state) + + SHA256_BLOCK_SIZE + 1, + .import =3D crypto_sha256_import_lib, + .export =3D crypto_sha256_export_lib, }, { .base.cra_name =3D "sha256", .base.cra_driver_name =3D "sha256-" __stringify(ARCH), .base.cra_priority =3D 300, diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index 4912572578dc2..f2df3bb90d11a 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -107,11 +107,11 @@ static inline void sha256_init(struct sha256_state *s= ctx) } void sha256_update(struct sha256_state *sctx, const u8 *data, size_t len); void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]); void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]); =20 -static inline void sha224_block_init(struct crypto_sha256_state *sctx) +static inline void sha224_init(struct sha256_state *sctx) { sctx->state[0] =3D SHA224_H0; sctx->state[1] =3D SHA224_H1; sctx->state[2] =3D SHA224_H2; sctx->state[3] =3D SHA224_H3; @@ -119,14 +119,9 @@ static inline void sha224_block_init(struct crypto_sha= 256_state *sctx) sctx->state[5] =3D SHA224_H5; sctx->state[6] =3D SHA224_H6; sctx->state[7] =3D SHA224_H7; sctx->count =3D 0; } - -static inline void sha224_init(struct sha256_state *sctx) -{ - sha224_block_init(&sctx->ctx); -} /* Simply use sha256_update as it is equivalent to sha224_update. */ void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]); =20 #endif /* _CRYPTO_SHA2_H */ --=20 2.49.0 From nobody Fri Dec 19 17:35:49 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id BFB0017B418; Sat, 17 May 2025 02:25:18 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448718; cv=none; b=M0EncSOzbnu7dUf3llKRof2x10UZJNLouQxyDJUqbtOSgHvh8jgXoF+dZ7RhW9S3UsWYmAcOknVc6RqWn1JUQsTRM7uvnEzfOqGeNIeIPwAbqL21nCJaGqoA5wFSmfoeaLbaXVbj4LRrznwyPwYkSgJhPeBZF65JdDuL8xjS9zo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1747448718; c=relaxed/simple; bh=zFcS6uR/dj+Xz3GyEef0LushmVtjQgiIhaKrPjHNxPI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=Spo12R7tcGjS9EEt3TMg2FPqyizEy2Xs8aPc3nFDOv0UqxDLgpUSEbZOtMtP6vI2zt3WYcIWt+hbxcsetaBpu1g6hEhHIdTOQLAPP8UgtP7v1v596iirjgHfYpO6fvQJ0/a8Lc27WHmD4JslTbNuB47PRle1k/ov1tlgOVZFyA0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=rjk/HbXw; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="rjk/HbXw" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 8A797C4CEF0; Sat, 17 May 2025 02:25:18 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1747448718; bh=zFcS6uR/dj+Xz3GyEef0LushmVtjQgiIhaKrPjHNxPI=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=rjk/HbXwyKzKwudbiPupxPDvIRdSBB13izcj3mHKpTZT6+6ZmDrbq4MZ67OEgsPXG U/9sDvGp7gvd2laSlHF+RvBOiFpVanpmbNaiMql+BwVGw1+HoYXflCrHY5smrSXsF6 OkqPCiX7qGwmEsAUogDDse6GXoit8QrTVCJNgOjx3FPfhTFYxkxPyiy/b5sB9tCSIQ Bo+LM/QyojDUhCvLGZ0acrbP/DtTZFM2/LTj5a2movkrAWgypEDO28e2e+92JIBGIs 0r93ikIEKBQXjvKnaF5CwTRAkZtDGb3gubz9XwuiThTQoG9mTugaTK9V5jPIMo9QEZ EVdUA+fvsOSQw== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Subject: [PATCH 8/8] Revert "crypto: lib/sha256 - Add helpers for block-based shash" Date: Fri, 16 May 2025 19:24:28 -0700 Message-ID: <20250517022428.401622-9-ebiggers@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250517022428.401622-1-ebiggers@kernel.org> References: <20250517022428.401622-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Eric Biggers This reverts commit 5b90a779bc547939421bfeb333e470658ba94fb6 which got pushed out despite being nacked. That commit added a special low-level interface to allow the crypto_shash API to bypass the safety check for using vector or SIMD registers. It could give a marginal performance benefit for crypto_shash, but just is not worth the complexity and footgun. Moreover, the distinction between "arch" and "simd" is confusing and is not something that really should exist in generic code, given that different architectures can mean different things by "simd". Signed-off-by: Eric Biggers --- include/crypto/internal/sha2.h | 45 ---------------------------------- lib/crypto/Kconfig | 8 ------ lib/crypto/sha256.c | 32 +++++++++++++++++------- 3 files changed, 23 insertions(+), 62 deletions(-) diff --git a/include/crypto/internal/sha2.h b/include/crypto/internal/sha2.h index fff156f66edc3..d641c67abcbc3 100644 --- a/include/crypto/internal/sha2.h +++ b/include/crypto/internal/sha2.h @@ -1,16 +1,11 @@ /* SPDX-License-Identifier: GPL-2.0-only */ =20 #ifndef _CRYPTO_INTERNAL_SHA2_H #define _CRYPTO_INTERNAL_SHA2_H =20 -#include #include -#include -#include -#include -#include =20 void sha256_update_generic(struct sha256_state *sctx, const u8 *data, size_t len); void sha256_final_generic(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]); @@ -27,47 +22,7 @@ static inline bool sha256_is_arch_optimized(void) #endif void sha256_blocks_generic(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks); -void sha256_blocks_simd(u32 state[SHA256_STATE_WORDS], - const u8 *data, size_t nblocks); - -static inline void sha256_choose_blocks( - u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks, - bool force_generic, bool force_simd) -{ - if (!IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) || force_generic) - sha256_blocks_generic(state, data, nblocks); - else if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD) && - (force_simd || crypto_simd_usable())) - sha256_blocks_simd(state, data, nblocks); - else - sha256_blocks_arch(state, data, nblocks); -} - -static __always_inline void sha256_finup( - struct crypto_sha256_state *sctx, u8 buf[SHA256_BLOCK_SIZE], - size_t len, u8 out[SHA256_DIGEST_SIZE], size_t digest_size, - bool force_generic, bool force_simd) -{ - const size_t bit_offset =3D SHA256_BLOCK_SIZE - 8; - __be64 *bits =3D (__be64 *)&buf[bit_offset]; - int i; - - buf[len++] =3D 0x80; - if (len > bit_offset) { - memset(&buf[len], 0, SHA256_BLOCK_SIZE - len); - sha256_choose_blocks(sctx->state, buf, 1, force_generic, - force_simd); - len =3D 0; - } - - memset(&buf[len], 0, bit_offset - len); - *bits =3D cpu_to_be64(sctx->count << 3); - sha256_choose_blocks(sctx->state, buf, 1, force_generic, force_simd); - - for (i =3D 0; i < digest_size; i +=3D 4) - put_unaligned_be32(sctx->state[i / 4], out + i); -} =20 #endif /* _CRYPTO_INTERNAL_SHA2_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 1ec1466108ccd..6319358b38c20 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -148,18 +148,10 @@ config CRYPTO_ARCH_HAVE_LIB_SHA256 bool help Declares whether the architecture provides an arch-specific accelerated implementation of the SHA-256 library interface. =20 -config CRYPTO_ARCH_HAVE_LIB_SHA256_SIMD - bool - help - Declares whether the architecture provides an arch-specific - accelerated implementation of the SHA-256 library interface - that is SIMD-based and therefore not usable in hardirq - context. - config CRYPTO_LIB_SHA256_GENERIC tristate default CRYPTO_LIB_SHA256 if !CRYPTO_ARCH_HAVE_LIB_SHA256 help This symbol can be selected by arch implementations of the SHA-256 diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 2ced29efa181c..563f09c9f3815 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -13,30 +13,29 @@ =20 #include #include #include #include +#include =20 /* * If __DISABLE_EXPORTS is defined, then this file is being compiled for a * pre-boot environment. In that case, ignore the kconfig options, pull t= he * generic code into the same translation unit, and use that only. */ #ifdef __DISABLE_EXPORTS #include "sha256-generic.c" #endif =20 -static inline bool sha256_purgatory(void) -{ - return __is_defined(__DISABLE_EXPORTS); -} - static inline void sha256_blocks(u32 state[SHA256_STATE_WORDS], const u8 *= data, size_t nblocks, bool force_generic) { - sha256_choose_blocks(state, data, nblocks, - force_generic || sha256_purgatory(), false); +#if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) && !defined(__DISABLE_E= XPORTS) + if (!force_generic) + return sha256_blocks_arch(state, data, nblocks); +#endif + sha256_blocks_generic(state, data, nblocks); } =20 static inline void __sha256_update(struct sha256_state *sctx, const u8 *da= ta, size_t len, bool force_generic) { @@ -78,14 +77,29 @@ void sha256_update(struct sha256_state *sctx, const u8 = *data, size_t len) EXPORT_SYMBOL(sha256_update); =20 static inline void __sha256_final(struct sha256_state *sctx, u8 *out, size_t digest_size, bool force_generic) { + const size_t bit_offset =3D SHA256_BLOCK_SIZE - sizeof(__be64); + __be64 *bits =3D (__be64 *)&sctx->buf[bit_offset]; size_t partial =3D sctx->count % SHA256_BLOCK_SIZE; + size_t i; + + sctx->buf[partial++] =3D 0x80; + if (partial > bit_offset) { + memset(&sctx->buf[partial], 0, SHA256_BLOCK_SIZE - partial); + sha256_blocks(sctx->state, sctx->buf, 1, force_generic); + partial =3D 0; + } + + memset(&sctx->buf[partial], 0, bit_offset - partial); + *bits =3D cpu_to_be64(sctx->count << 3); + sha256_blocks(sctx->state, sctx->buf, 1, force_generic); + + for (i =3D 0; i < digest_size; i +=3D 4) + put_unaligned_be32(sctx->state[i / 4], out + i); =20 - sha256_finup(&sctx->ctx, sctx->buf, partial, out, digest_size, - force_generic || sha256_purgatory(), false); memzero_explicit(sctx, sizeof(*sctx)); } =20 void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]) { --=20 2.49.0