From nobody Wed Oct 8 08:14:50 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id AA3A128DB5E; Mon, 30 Jun 2025 16:05:22 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299522; cv=none; b=pbF6lLJ8SU1GATSEe/Wo+8g9/E8ofrwu2cwLmflYWUMjyNvPaLszAkyFC3Cq4astr35Nil/VNWMGw3n9y/ykgLB5iozxB5jfA1iMSX6QJg2jxX30//KDSbNt7CjpOML5x1gfNppuqsMkb26kOMUWRaQ/Z2F36cH+ykgB5ADKOWI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299522; c=relaxed/simple; bh=m8b9QVkIhF8M/bhCYfEccNbG5xwp7BU+pMzzoTskfZk=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=q5/ZOeD5JBHdIFHBYDJae7OV5p8yZpH+eLNyfEwe2tjMn1e3kmP2VpHIYR1iVZY93iZfIR22A6oaMI/5IB2AZsMlRcdcyEbtRvItqGC5m8c0Kjsb+CNK0t081bAs6thuXSa+rggv1RGdR1P0d8J2m3bv+/TnwObBJr9n9Gy/PM0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Dr1yLhc6; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Dr1yLhc6" Received: by smtp.kernel.org (Postfix) with ESMTPSA id E9809C4CEF4; Mon, 30 Jun 2025 16:05:21 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299522; bh=m8b9QVkIhF8M/bhCYfEccNbG5xwp7BU+pMzzoTskfZk=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Dr1yLhc6Hd7kiH/o6qs2PCfo54LA/2VhAWi+gIrOvpsnwFWtyuvceA2rFCIC2ES3H GJGvIWtnt1Dgs7HBOshwa0iW152TaUyUZfkkS8uiKwmVIPvK5gia9LQnANcTFB0ouf lWlsy3E4vW+8KjhOX4F+xVxAUKRK85Uk3334zsgnM6TzADG0k4Zz2/a0s9DsMUzGgz n5ZzT1dJR4xFtHaApZ4q0BFe5KTfahX51omf89DdTOys8OrQvoQ0YVO9nBS6M1ClTT v3rr7GntE1rWyy0lycLESqSvTmKiWgiB+9jnmQpb/GLUB4S8bJr+yqT2C3WIP6sKlF UyfbKdNCR+Gcw== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 01/16] crypto: sha512 - Rename conflicting symbols Date: Mon, 30 Jun 2025 09:03:05 -0700 Message-ID: <20250630160320.2888-2-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Rename existing functions and structs in architecture-optimized SHA-512 code that had names conflicting with the upcoming library interface which will be added to : sha384_init, sha512_init, sha512_update, sha384, and sha512. Note: all affected code will be superseded by later commits that migrate the arch-optimized SHA-512 code into the library. This commit simply keeps the kernel building for the initial introduction of the library. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers --- arch/arm64/crypto/sha512-glue.c | 8 ++++---- arch/s390/crypto/sha512_s390.c | 8 ++++---- arch/sparc/crypto/sha512_glue.c | 14 +++++++------- arch/x86/crypto/sha512_ssse3_glue.c | 10 +++++----- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glu= e.c index 15aa9d8b7b2c4..a78e184c100fa 100644 --- a/arch/arm64/crypto/sha512-glue.c +++ b/arch/arm64/crypto/sha512-glue.c @@ -25,12 +25,12 @@ static void sha512_arm64_transform(struct sha512_state = *sst, u8 const *src, int blocks) { sha512_blocks_arch(sst->state, src, blocks); } =20 -static int sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int sha512_update_arm64(struct shash_desc *desc, const u8 *data, + unsigned int len) { return sha512_base_do_update_blocks(desc, data, len, sha512_arm64_transform); } =20 @@ -42,11 +42,11 @@ static int sha512_finup(struct shash_desc *desc, const = u8 *data, } =20 static struct shash_alg algs[] =3D { { .digestsize =3D SHA512_DIGEST_SIZE, .init =3D sha512_base_init, - .update =3D sha512_update, + .update =3D sha512_update_arm64, .finup =3D sha512_finup, .descsize =3D SHA512_STATE_SIZE, .base.cra_name =3D "sha512", .base.cra_driver_name =3D "sha512-arm64", .base.cra_priority =3D 150, @@ -55,11 +55,11 @@ static struct shash_alg algs[] =3D { { .base.cra_blocksize =3D SHA512_BLOCK_SIZE, .base.cra_module =3D THIS_MODULE, }, { .digestsize =3D SHA384_DIGEST_SIZE, .init =3D sha384_base_init, - .update =3D sha512_update, + .update =3D sha512_update_arm64, .finup =3D sha512_finup, .descsize =3D SHA512_STATE_SIZE, .base.cra_name =3D "sha384", .base.cra_driver_name =3D "sha384-arm64", .base.cra_priority =3D 150, diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index 33711a29618c3..e8bb172dbed75 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -15,11 +15,11 @@ #include #include =20 #include "sha.h" =20 -static int sha512_init(struct shash_desc *desc) +static int sha512_init_s390(struct shash_desc *desc) { struct s390_sha_ctx *ctx =3D shash_desc_ctx(desc); =20 ctx->sha512.state[0] =3D SHA512_H0; ctx->sha512.state[1] =3D SHA512_H1; @@ -60,11 +60,11 @@ static int sha512_import(struct shash_desc *desc, const= void *in) return 0; } =20 static struct shash_alg sha512_alg =3D { .digestsize =3D SHA512_DIGEST_SIZE, - .init =3D sha512_init, + .init =3D sha512_init_s390, .update =3D s390_sha_update_blocks, .finup =3D s390_sha_finup, .export =3D sha512_export, .import =3D sha512_import, .descsize =3D sizeof(struct s390_sha_ctx), @@ -80,11 +80,11 @@ static struct shash_alg sha512_alg =3D { } }; =20 MODULE_ALIAS_CRYPTO("sha512"); =20 -static int sha384_init(struct shash_desc *desc) +static int sha384_init_s390(struct shash_desc *desc) { struct s390_sha_ctx *ctx =3D shash_desc_ctx(desc); =20 ctx->sha512.state[0] =3D SHA384_H0; ctx->sha512.state[1] =3D SHA384_H1; @@ -101,11 +101,11 @@ static int sha384_init(struct shash_desc *desc) return 0; } =20 static struct shash_alg sha384_alg =3D { .digestsize =3D SHA384_DIGEST_SIZE, - .init =3D sha384_init, + .init =3D sha384_init_s390, .update =3D s390_sha_update_blocks, .finup =3D s390_sha_finup, .export =3D sha512_export, .import =3D sha512_import, .descsize =3D sizeof(struct s390_sha_ctx), diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glu= e.c index 47b9277b6877a..fb81c3290c8c0 100644 --- a/arch/sparc/crypto/sha512_glue.c +++ b/arch/sparc/crypto/sha512_glue.c @@ -38,11 +38,11 @@ static int sha512_sparc64_finup(struct shash_desc *desc= , const u8 *src, { sha512_base_do_finup(desc, src, len, sha512_block); return sha512_base_finish(desc, out); } =20 -static struct shash_alg sha512 =3D { +static struct shash_alg sha512_alg =3D { .digestsize =3D SHA512_DIGEST_SIZE, .init =3D sha512_base_init, .update =3D sha512_sparc64_update, .finup =3D sha512_sparc64_finup, .descsize =3D SHA512_STATE_SIZE, @@ -53,11 +53,11 @@ static struct shash_alg sha512 =3D { .cra_blocksize =3D SHA512_BLOCK_SIZE, .cra_module =3D THIS_MODULE, } }; =20 -static struct shash_alg sha384 =3D { +static struct shash_alg sha384_alg =3D { .digestsize =3D SHA384_DIGEST_SIZE, .init =3D sha384_base_init, .update =3D sha512_sparc64_update, .finup =3D sha512_sparc64_finup, .descsize =3D SHA512_STATE_SIZE, @@ -85,17 +85,17 @@ static bool __init sparc64_has_sha512_opcode(void) } =20 static int __init sha512_sparc64_mod_init(void) { if (sparc64_has_sha512_opcode()) { - int ret =3D crypto_register_shash(&sha384); + int ret =3D crypto_register_shash(&sha384_alg); if (ret < 0) return ret; =20 - ret =3D crypto_register_shash(&sha512); + ret =3D crypto_register_shash(&sha512_alg); if (ret < 0) { - crypto_unregister_shash(&sha384); + crypto_unregister_shash(&sha384_alg); return ret; } =20 pr_info("Using sparc64 sha512 opcode optimized SHA-512/SHA-384 implement= ation\n"); return 0; @@ -104,12 +104,12 @@ static int __init sha512_sparc64_mod_init(void) return -ENODEV; } =20 static void __exit sha512_sparc64_mod_fini(void) { - crypto_unregister_shash(&sha384); - crypto_unregister_shash(&sha512); + crypto_unregister_shash(&sha384_alg); + crypto_unregister_shash(&sha512_alg); } =20 module_init(sha512_sparc64_mod_init); module_exit(sha512_sparc64_mod_fini); =20 diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_s= sse3_glue.c index 067684c543952..97744b7d23817 100644 --- a/arch/x86/crypto/sha512_ssse3_glue.c +++ b/arch/x86/crypto/sha512_ssse3_glue.c @@ -36,12 +36,12 @@ #include =20 asmlinkage void sha512_transform_ssse3(struct sha512_state *state, const u8 *data, int blocks); =20 -static int sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len, sha512_block_fn *sha512_xform) +static int sha512_update_x86(struct shash_desc *desc, const u8 *data, + unsigned int len, sha512_block_fn *sha512_xform) { int remain; =20 /* * Make sure struct sha512_state begins directly with the SHA512 @@ -67,11 +67,11 @@ static int sha512_finup(struct shash_desc *desc, const = u8 *data, } =20 static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - return sha512_update(desc, data, len, sha512_transform_ssse3); + return sha512_update_x86(desc, data, len, sha512_transform_ssse3); } =20 static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { @@ -139,11 +139,11 @@ static bool avx_usable(void) } =20 static int sha512_avx_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - return sha512_update(desc, data, len, sha512_transform_avx); + return sha512_update_x86(desc, data, len, sha512_transform_avx); } =20 static int sha512_avx_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { @@ -201,11 +201,11 @@ asmlinkage void sha512_transform_rorx(struct sha512_s= tate *state, const u8 *data, int blocks); =20 static int sha512_avx2_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - return sha512_update(desc, data, len, sha512_transform_rorx); + return sha512_update_x86(desc, data, len, sha512_transform_rorx); } =20 static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { --=20 2.50.0 From nobody Wed Oct 8 08:14:50 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 266E528FAB9; Mon, 30 Jun 2025 16:05:22 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299523; cv=none; b=q2ig3wE8jExXZyhHRm5UjIfAZaNDWvMIheY2fCv1KT/Uk/mTK8/awy7R/upmk24c3D+8vXnRAFSGHtOALOl9fGU0Lvw19+mE0rncYx6N3P+AyGRa0frUJf5zyd1nvpkQhy7Z+bAaKU6Tw5WXUmPN+5VOSxQefQI+F1J66NmixKo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299523; c=relaxed/simple; bh=eAlFTDD9aXKbjy6HE+rdkdPi0a/NSm/FPRsme2GVGco=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=tt4VZQKBDhpKW9/hTPjeTNhyt8g1F/5B8kBePh7uy7Kdk9tYWVMwe5L2xRIyuCJdP0/3B3RFx/DXu1wpTbOhg1EAPVWDrpmADs3ODso9VZcwlOcPrF8348C+DHSXM9MPGom2nV9RbnEPVTtQsjXDIeTpzvA0rtQvUpfNHHAjbfw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=BHeiGcd6; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="BHeiGcd6" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 687F0C4CEF3; Mon, 30 Jun 2025 16:05:22 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299522; bh=eAlFTDD9aXKbjy6HE+rdkdPi0a/NSm/FPRsme2GVGco=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=BHeiGcd62HEts1mhgVsYRzmrDKMThTLQzMKUc3qgPUlWgBVbFY28hsE/C9laDkrqA 1fn7CxdoaA1kDYfO/WHa0e6r2flYkYp45CJ/FGbNVGvoGOsLPXajCcEifQUpKqqYxI LYhUp0t3ZC8Pb8KyuvEOzAyiQDaK3MHIa5dhFIi/ARrqmr5nYRJoXDsTWLcfB3f2mK fYz6j1Oz1yDGgCy/NcnAvRQc8dBkLBtGexLaNc/5a9l/rFP5gDrIxckZ/l2zvkBtip RgXDth/m6lpPUFzUXbbOICnj95qePHy4OqgkJ/3Gp2v7nBrlMpaoJG1tGJTQpK8Rem WJ3ICXpPZdbCQ== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 02/16] lib/crypto: sha512: Add support for SHA-384 and SHA-512 Date: Mon, 30 Jun 2025 09:03:06 -0700 Message-ID: <20250630160320.2888-3-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Add basic support for SHA-384 and SHA-512 to lib/crypto/. Various in-kernel users will be able to use this instead of the old-school crypto API, which is harder to use and has more overhead. The basic support added by this commit consists of the API and its documentation, backed by a C implementation of the algorithms. sha512_block_generic() is derived from crypto/sha512_generic.c. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers --- include/crypto/sha2.h | 128 ++++++++++++++++++++ lib/crypto/Kconfig | 10 ++ lib/crypto/Makefile | 6 + lib/crypto/sha512.c | 265 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 409 insertions(+) create mode 100644 lib/crypto/sha512.c diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index 4912572578dc2..f2a6e84b28400 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -127,6 +127,134 @@ static inline void sha224_init(struct sha256_state *s= ctx) sha224_block_init(&sctx->ctx); } /* Simply use sha256_update as it is equivalent to sha224_update. */ void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]); =20 +/* State for the SHA-512 (and SHA-384) compression function */ +struct sha512_block_state { + u64 h[8]; +}; + +/* + * Context structure, shared by SHA-384 and SHA-512. The sha384_ctx and + * sha512_ctx structs wrap this one so that the API has proper typing and + * doesn't allow mixing the SHA-384 and SHA-512 functions arbitrarily. + */ +struct __sha512_ctx { + struct sha512_block_state state; + u64 bytecount_lo; + u64 bytecount_hi; + u8 buf[SHA512_BLOCK_SIZE] __aligned(__alignof__(__be64)); +}; +void __sha512_update(struct __sha512_ctx *ctx, const u8 *data, size_t len); + +/** + * struct sha384_ctx - Context for hashing a message with SHA-384 + * @ctx: private + */ +struct sha384_ctx { + struct __sha512_ctx ctx; +}; + +/** + * sha384_init() - Initialize a SHA-384 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider sha384() instead. + * + * Context: Any context. + */ +void sha384_init(struct sha384_ctx *ctx); + +/** + * sha384_update() - Update a SHA-384 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void sha384_update(struct sha384_ctx *ctx, + const u8 *data, size_t len) +{ + __sha512_update(&ctx->ctx, data, len); +} + +/** + * sha384_final() - Finish computing a SHA-384 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting SHA-384 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do= it. + * + * Context: Any context. + */ +void sha384_final(struct sha384_ctx *ctx, u8 out[SHA384_DIGEST_SIZE]); + +/** + * sha384() - Compute SHA-384 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting SHA-384 message digest + * + * Context: Any context. + */ +void sha384(const u8 *data, size_t len, u8 out[SHA384_DIGEST_SIZE]); + +/** + * struct sha512_ctx - Context for hashing a message with SHA-512 + * @ctx: private + */ +struct sha512_ctx { + struct __sha512_ctx ctx; +}; + +/** + * sha512_init() - Initialize a SHA-512 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider sha512() instead. + * + * Context: Any context. + */ +void sha512_init(struct sha512_ctx *ctx); + +/** + * sha512_update() - Update a SHA-512 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void sha512_update(struct sha512_ctx *ctx, + const u8 *data, size_t len) +{ + __sha512_update(&ctx->ctx, data, len); +} + +/** + * sha512_final() - Finish computing a SHA-512 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting SHA-512 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do= it. + * + * Context: Any context. + */ +void sha512_final(struct sha512_ctx *ctx, u8 out[SHA512_DIGEST_SIZE]); + +/** + * sha512() - Compute SHA-512 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting SHA-512 message digest + * + * Context: Any context. + */ +void sha512(const u8 *data, size_t len, u8 out[SHA512_DIGEST_SIZE]); + #endif /* _CRYPTO_SHA2_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 1ec1466108ccd..2d295c0e0f792 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -165,10 +165,20 @@ config CRYPTO_LIB_SHA256_GENERIC This symbol can be selected by arch implementations of the SHA-256 library interface that require the generic code as a fallback, e.g., for SIMD implementations. If no arch specific implementation is enabled, this implementation serves the users of CRYPTO_LIB_SHA256. =20 +config CRYPTO_LIB_SHA512 + tristate + help + The SHA-384 and SHA-512 library functions. Select this if your module + uses any of these functions from . + +config CRYPTO_LIB_SHA512_ARCH + bool + depends on CRYPTO_LIB_SHA512 && !UML + config CRYPTO_LIB_SM3 tristate =20 if !KMSAN # avoid false positives from assembly if ARM diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index b0c0f8aea2695..f6b6f370451ec 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -62,10 +62,16 @@ obj-$(CONFIG_CRYPTO_LIB_SHA256) +=3D libsha256.o libsha256-y :=3D sha256.o =20 obj-$(CONFIG_CRYPTO_LIB_SHA256_GENERIC) +=3D libsha256-generic.o libsha256-generic-y :=3D sha256-generic.o =20 +obj-$(CONFIG_CRYPTO_LIB_SHA512) +=3D libsha512.o +libsha512-y :=3D sha512.o +ifeq ($(CONFIG_CRYPTO_LIB_SHA512_ARCH),y) +CFLAGS_sha512.o +=3D -I$(src)/$(SRCARCH) +endif # CONFIG_CRYPTO_LIB_SHA512_ARCH + obj-$(CONFIG_MPILIB) +=3D mpi/ =20 obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) +=3D simd.o =20 obj-$(CONFIG_CRYPTO_LIB_SM3) +=3D libsm3.o diff --git a/lib/crypto/sha512.c b/lib/crypto/sha512.c new file mode 100644 index 0000000000000..536b71481b1ca --- /dev/null +++ b/lib/crypto/sha512.c @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SHA-384 and SHA-512 library functions + * + * Copyright (c) Jean-Luc Cooke + * Copyright (c) Andrew McDonald + * Copyright (c) 2003 Kyle McMartin + * Copyright 2025 Google LLC + */ + +#include +#include +#include +#include +#include +#include +#include + +static const struct sha512_block_state sha384_iv =3D { + .h =3D { + SHA384_H0, SHA384_H1, SHA384_H2, SHA384_H3, + SHA384_H4, SHA384_H5, SHA384_H6, SHA384_H7, + }, +}; + +static const struct sha512_block_state sha512_iv =3D { + .h =3D { + SHA512_H0, SHA512_H1, SHA512_H2, SHA512_H3, + SHA512_H4, SHA512_H5, SHA512_H6, SHA512_H7, + }, +}; + +static const u64 sha512_K[80] =3D { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL, + 0xe9b5dba58189dbbcULL, 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, 0xd807aa98a3030242ULL, + 0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL, + 0xc19bf174cf692694ULL, 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, 0x2de92c6f592b0275ULL, + 0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL, + 0xbf597fc7beef0ee4ULL, 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 0x27b70a8546d22ffcULL, + 0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL, + 0x92722c851482353bULL, 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, 0xd192e819d6ef5218ULL, + 0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL, + 0x34b0bcb5e19b48a8ULL, 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, 0x748f82ee5defb2fcULL, + 0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL, + 0xc67178f2e372532bULL, 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, 0x06f067aa72176fbaULL, + 0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL, + 0x431d67c49c100d4cULL, 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, +}; + +#define Ch(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) +#define Maj(x, y, z) (((x) & (y)) | ((z) & ((x) | (y)))) +#define e0(x) (ror64((x), 28) ^ ror64((x), 34) ^ ror64((x), 39)) +#define e1(x) (ror64((x), 14) ^ ror64((x), 18) ^ ror64((x), 41)) +#define s0(x) (ror64((x), 1) ^ ror64((x), 8) ^ ((x) >> 7)) +#define s1(x) (ror64((x), 19) ^ ror64((x), 61) ^ ((x) >> 6)) + +static void sha512_block_generic(struct sha512_block_state *state, + const u8 *data) +{ + u64 a =3D state->h[0]; + u64 b =3D state->h[1]; + u64 c =3D state->h[2]; + u64 d =3D state->h[3]; + u64 e =3D state->h[4]; + u64 f =3D state->h[5]; + u64 g =3D state->h[6]; + u64 h =3D state->h[7]; + u64 t1, t2; + u64 W[16]; + + for (int j =3D 0; j < 16; j++) + W[j] =3D get_unaligned_be64(data + j * sizeof(u64)); + + for (int i =3D 0; i < 80; i +=3D 8) { + if ((i & 15) =3D=3D 0 && i !=3D 0) { + for (int j =3D 0; j < 16; j++) { + W[j & 15] +=3D s1(W[(j - 2) & 15]) + + W[(j - 7) & 15] + + s0(W[(j - 15) & 15]); + } + } + t1 =3D h + e1(e) + Ch(e, f, g) + sha512_K[i] + W[(i & 15)]; + t2 =3D e0(a) + Maj(a, b, c); d +=3D t1; h =3D t1 + t2; + t1 =3D g + e1(d) + Ch(d, e, f) + sha512_K[i+1] + W[(i & 15) + 1]; + t2 =3D e0(h) + Maj(h, a, b); c +=3D t1; g =3D t1 + t2; + t1 =3D f + e1(c) + Ch(c, d, e) + sha512_K[i+2] + W[(i & 15) + 2]; + t2 =3D e0(g) + Maj(g, h, a); b +=3D t1; f =3D t1 + t2; + t1 =3D e + e1(b) + Ch(b, c, d) + sha512_K[i+3] + W[(i & 15) + 3]; + t2 =3D e0(f) + Maj(f, g, h); a +=3D t1; e =3D t1 + t2; + t1 =3D d + e1(a) + Ch(a, b, c) + sha512_K[i+4] + W[(i & 15) + 4]; + t2 =3D e0(e) + Maj(e, f, g); h +=3D t1; d =3D t1 + t2; + t1 =3D c + e1(h) + Ch(h, a, b) + sha512_K[i+5] + W[(i & 15) + 5]; + t2 =3D e0(d) + Maj(d, e, f); g +=3D t1; c =3D t1 + t2; + t1 =3D b + e1(g) + Ch(g, h, a) + sha512_K[i+6] + W[(i & 15) + 6]; + t2 =3D e0(c) + Maj(c, d, e); f +=3D t1; b =3D t1 + t2; + t1 =3D a + e1(f) + Ch(f, g, h) + sha512_K[i+7] + W[(i & 15) + 7]; + t2 =3D e0(b) + Maj(b, c, d); e +=3D t1; a =3D t1 + t2; + } + + state->h[0] +=3D a; + state->h[1] +=3D b; + state->h[2] +=3D c; + state->h[3] +=3D d; + state->h[4] +=3D e; + state->h[5] +=3D f; + state->h[6] +=3D g; + state->h[7] +=3D h; +} + +static void __maybe_unused +sha512_blocks_generic(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + do { + sha512_block_generic(state, data); + data +=3D SHA512_BLOCK_SIZE; + } while (--nblocks); +} + +#ifdef CONFIG_CRYPTO_LIB_SHA512_ARCH +#include "sha512.h" /* $(SRCARCH)/sha512.h */ +#else +#define sha512_blocks sha512_blocks_generic +#endif + +static void __sha512_init(struct __sha512_ctx *ctx, + const struct sha512_block_state *iv, + u64 initial_bytecount) +{ + ctx->state =3D *iv; + ctx->bytecount_lo =3D initial_bytecount; + ctx->bytecount_hi =3D 0; +} + +void sha384_init(struct sha384_ctx *ctx) +{ + __sha512_init(&ctx->ctx, &sha384_iv, 0); +} +EXPORT_SYMBOL_GPL(sha384_init); + +void sha512_init(struct sha512_ctx *ctx) +{ + __sha512_init(&ctx->ctx, &sha512_iv, 0); +} +EXPORT_SYMBOL_GPL(sha512_init); + +void __sha512_update(struct __sha512_ctx *ctx, const u8 *data, size_t len) +{ + size_t partial =3D ctx->bytecount_lo % SHA512_BLOCK_SIZE; + + if (check_add_overflow(ctx->bytecount_lo, len, &ctx->bytecount_lo)) + ctx->bytecount_hi++; + + if (partial + len >=3D SHA512_BLOCK_SIZE) { + size_t nblocks; + + if (partial) { + size_t l =3D SHA512_BLOCK_SIZE - partial; + + memcpy(&ctx->buf[partial], data, l); + data +=3D l; + len -=3D l; + + sha512_blocks(&ctx->state, ctx->buf, 1); + } + + nblocks =3D len / SHA512_BLOCK_SIZE; + len %=3D SHA512_BLOCK_SIZE; + + if (nblocks) { + sha512_blocks(&ctx->state, data, nblocks); + data +=3D nblocks * SHA512_BLOCK_SIZE; + } + partial =3D 0; + } + if (len) + memcpy(&ctx->buf[partial], data, len); +} +EXPORT_SYMBOL_GPL(__sha512_update); + +static void __sha512_final(struct __sha512_ctx *ctx, + u8 *out, size_t digest_size) +{ + u64 bitcount_hi =3D (ctx->bytecount_hi << 3) | (ctx->bytecount_lo >> 61); + u64 bitcount_lo =3D ctx->bytecount_lo << 3; + size_t partial =3D ctx->bytecount_lo % SHA512_BLOCK_SIZE; + + ctx->buf[partial++] =3D 0x80; + if (partial > SHA512_BLOCK_SIZE - 16) { + memset(&ctx->buf[partial], 0, SHA512_BLOCK_SIZE - partial); + sha512_blocks(&ctx->state, ctx->buf, 1); + partial =3D 0; + } + memset(&ctx->buf[partial], 0, SHA512_BLOCK_SIZE - 16 - partial); + *(__be64 *)&ctx->buf[SHA512_BLOCK_SIZE - 16] =3D cpu_to_be64(bitcount_hi); + *(__be64 *)&ctx->buf[SHA512_BLOCK_SIZE - 8] =3D cpu_to_be64(bitcount_lo); + sha512_blocks(&ctx->state, ctx->buf, 1); + + for (size_t i =3D 0; i < digest_size; i +=3D 8) + put_unaligned_be64(ctx->state.h[i / 8], out + i); +} + +void sha384_final(struct sha384_ctx *ctx, u8 out[SHA384_DIGEST_SIZE]) +{ + __sha512_final(&ctx->ctx, out, SHA384_DIGEST_SIZE); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(sha384_final); + +void sha512_final(struct sha512_ctx *ctx, u8 out[SHA512_DIGEST_SIZE]) +{ + __sha512_final(&ctx->ctx, out, SHA512_DIGEST_SIZE); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(sha512_final); + +void sha384(const u8 *data, size_t len, u8 out[SHA384_DIGEST_SIZE]) +{ + struct sha384_ctx ctx; + + sha384_init(&ctx); + sha384_update(&ctx, data, len); + sha384_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(sha384); + +void sha512(const u8 *data, size_t len, u8 out[SHA512_DIGEST_SIZE]) +{ + struct sha512_ctx ctx; + + sha512_init(&ctx); + sha512_update(&ctx, data, len); + sha512_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(sha512); + +#ifdef sha512_mod_init_arch +static int __init sha512_mod_init(void) +{ + sha512_mod_init_arch(); + return 0; +} +subsys_initcall(sha512_mod_init); + +static void __exit sha512_mod_exit(void) +{ +} +module_exit(sha512_mod_exit); +#endif + +MODULE_DESCRIPTION("SHA-384 and SHA-512 library functions"); +MODULE_LICENSE("GPL"); --=20 2.50.0 From nobody Wed Oct 8 08:14:50 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 55D3E28FFE5; Mon, 30 Jun 2025 16:05:23 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299523; cv=none; b=bHw7m3LvFIE5w56YyytjIxCWKtW+K2AKbHMnZ3Hzl4c2obundeS+FXo4zdbBcOcyQZDGFHMwryZDtxyv+lFG5oy0nuWekp6YMVtzjo3Rhok2iipMnN3ovMYszxBfgdNei3gXDyf+EhH/nDVM289G+0oFjenGiPay4ZL/L1QF3Ow= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299523; c=relaxed/simple; bh=m2Tqpcxe392TMDWwIQDZ7diXyctLde0ZUhfAzh0L8O0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=S4ioiy5H6L1hVbcIv7vNOJk7TvYs95qA3MvreDUAXOkvcCXldOjF5wBEOFM5SIXQqbX/vqQZMgOP5VANrnNRMjavdyELPhe0B48yXjJ6/VYrHL9m7SqWD7O7U0z+DrXtZ7F5HD6ozNf0ZpqQvXnmhzRA+miSBZVxtvxpnHK61vQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=MKVr3R9j; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="MKVr3R9j" Received: by smtp.kernel.org (Postfix) with ESMTPSA id DAD15C4CEF1; Mon, 30 Jun 2025 16:05:22 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299523; bh=m2Tqpcxe392TMDWwIQDZ7diXyctLde0ZUhfAzh0L8O0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=MKVr3R9jxuc0YdBb9mUnB/ZaPlKMJ+oVKqpsx1/waJ9k+r0x3YEEgl5OPCUiF72Bs oERs2kKqM9P0C5S178q4XhoMZskxAqO3ubL7YrAaxDZhDZAvfcp7A263K4Ho4O4zR2 azodLw0bc57w/2+ir7MRKwakzvic358jY9oOKAd+YVf0znOaKyNyn7xIUE/RuhEaqQ 52HNCm3rLycIos/73ZbY5ePUhWOIoB3MKU22ykQ1/xEqbwj2h+4jqgPbrlalnYvoTo CETeimyGRrWz5sDz08Xp4Kp6VafpNE22PphuRbFUytZrT8SRz73k5lSFXWEZWlv2gn 0dy27oJAnharA== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 03/16] lib/crypto: sha512: Add HMAC-SHA384 and HMAC-SHA512 support Date: Mon, 30 Jun 2025 09:03:07 -0700 Message-ID: <20250630160320.2888-4-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Since HMAC support is commonly needed and is fairly simple, include it as a first-class citizen of the SHA-512 library. The API supports both incremental and one-shot computation, and either preparing the key ahead of time or just using a raw key. The implementation is much more streamlined than crypto/hmac.c. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers --- include/crypto/sha2.h | 222 ++++++++++++++++++++++++++++++++++++++++++ lib/crypto/Kconfig | 5 +- lib/crypto/sha512.c | 141 ++++++++++++++++++++++++++- 3 files changed, 364 insertions(+), 4 deletions(-) diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index f2a6e84b28400..296ce9d468bfc 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -145,10 +145,26 @@ struct __sha512_ctx { u64 bytecount_hi; u8 buf[SHA512_BLOCK_SIZE] __aligned(__alignof__(__be64)); }; void __sha512_update(struct __sha512_ctx *ctx, const u8 *data, size_t len); =20 +/* + * HMAC key and message context structs, shared by HMAC-SHA384 and HMAC-SH= A512. + * The hmac_sha384_* and hmac_sha512_* structs wrap this one so that the A= PI has + * proper typing and doesn't allow mixing the functions arbitrarily. + */ +struct __hmac_sha512_key { + struct sha512_block_state istate; + struct sha512_block_state ostate; +}; +struct __hmac_sha512_ctx { + struct __sha512_ctx sha_ctx; + struct sha512_block_state ostate; +}; +void __hmac_sha512_init(struct __hmac_sha512_ctx *ctx, + const struct __hmac_sha512_key *key); + /** * struct sha384_ctx - Context for hashing a message with SHA-384 * @ctx: private */ struct sha384_ctx { @@ -200,10 +216,113 @@ void sha384_final(struct sha384_ctx *ctx, u8 out[SHA= 384_DIGEST_SIZE]); * * Context: Any context. */ void sha384(const u8 *data, size_t len, u8 out[SHA384_DIGEST_SIZE]); =20 +/** + * struct hmac_sha384_key - Prepared key for HMAC-SHA384 + * @key: private + */ +struct hmac_sha384_key { + struct __hmac_sha512_key key; +}; + +/** + * struct hmac_sha384_ctx - Context for computing HMAC-SHA384 of a message + * @ctx: private + */ +struct hmac_sha384_ctx { + struct __hmac_sha512_ctx ctx; +}; + +/** + * hmac_sha384_preparekey() - Prepare a key for HMAC-SHA384 + * @key: (output) the key structure to initialize + * @raw_key: the raw HMAC-SHA384 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * Note: the caller is responsible for zeroizing both the struct hmac_sha3= 84_key + * and the raw key once they are no longer needed. + * + * Context: Any context. + */ +void hmac_sha384_preparekey(struct hmac_sha384_key *key, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha384_init() - Initialize an HMAC-SHA384 context for a new message + * @ctx: (output) the HMAC context to initialize + * @key: the prepared HMAC key + * + * If you don't need incremental computation, consider hmac_sha384() inste= ad. + * + * Context: Any context. + */ +static inline void hmac_sha384_init(struct hmac_sha384_ctx *ctx, + const struct hmac_sha384_key *key) +{ + __hmac_sha512_init(&ctx->ctx, &key->key); +} + +/** + * hmac_sha384_update() - Update an HMAC-SHA384 context with message data + * @ctx: the HMAC context to update; must have been initialized + * @data: the message data + * @data_len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void hmac_sha384_update(struct hmac_sha384_ctx *ctx, + const u8 *data, size_t data_len) +{ + __sha512_update(&ctx->ctx.sha_ctx, data, data_len); +} + +/** + * hmac_sha384_final() - Finish computing an HMAC-SHA384 value + * @ctx: the HMAC context to finalize; must have been initialized + * @out: (output) the resulting HMAC-SHA384 value + * + * After finishing, this zeroizes @ctx. So the caller does not need to do= it. + * + * Context: Any context. + */ +void hmac_sha384_final(struct hmac_sha384_ctx *ctx, u8 out[SHA384_DIGEST_S= IZE]); + +/** + * hmac_sha384() - Compute HMAC-SHA384 in one shot, using a prepared key + * @key: the prepared HMAC key + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA384 value + * + * If you're using the key only once, consider using hmac_sha384_usingrawk= ey(). + * + * Context: Any context. + */ +void hmac_sha384(const struct hmac_sha384_key *key, + const u8 *data, size_t data_len, u8 out[SHA384_DIGEST_SIZE]); + +/** + * hmac_sha384_usingrawkey() - Compute HMAC-SHA384 in one shot, using a ra= w key + * @raw_key: the raw HMAC-SHA384 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA384 value + * + * If you're using the key multiple times, prefer to use + * hmac_sha384_preparekey() followed by multiple calls to hmac_sha384() in= stead. + * + * Context: Any context. + */ +void hmac_sha384_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[SHA384_DIGEST_SIZE]); + /** * struct sha512_ctx - Context for hashing a message with SHA-512 * @ctx: private */ struct sha512_ctx { @@ -255,6 +374,109 @@ void sha512_final(struct sha512_ctx *ctx, u8 out[SHA5= 12_DIGEST_SIZE]); * * Context: Any context. */ void sha512(const u8 *data, size_t len, u8 out[SHA512_DIGEST_SIZE]); =20 +/** + * struct hmac_sha512_key - Prepared key for HMAC-SHA512 + * @key: private + */ +struct hmac_sha512_key { + struct __hmac_sha512_key key; +}; + +/** + * struct hmac_sha512_ctx - Context for computing HMAC-SHA512 of a message + * @ctx: private + */ +struct hmac_sha512_ctx { + struct __hmac_sha512_ctx ctx; +}; + +/** + * hmac_sha512_preparekey() - Prepare a key for HMAC-SHA512 + * @key: (output) the key structure to initialize + * @raw_key: the raw HMAC-SHA512 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * Note: the caller is responsible for zeroizing both the struct hmac_sha5= 12_key + * and the raw key once they are no longer needed. + * + * Context: Any context. + */ +void hmac_sha512_preparekey(struct hmac_sha512_key *key, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha512_init() - Initialize an HMAC-SHA512 context for a new message + * @ctx: (output) the HMAC context to initialize + * @key: the prepared HMAC key + * + * If you don't need incremental computation, consider hmac_sha512() inste= ad. + * + * Context: Any context. + */ +static inline void hmac_sha512_init(struct hmac_sha512_ctx *ctx, + const struct hmac_sha512_key *key) +{ + __hmac_sha512_init(&ctx->ctx, &key->key); +} + +/** + * hmac_sha512_update() - Update an HMAC-SHA512 context with message data + * @ctx: the HMAC context to update; must have been initialized + * @data: the message data + * @data_len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void hmac_sha512_update(struct hmac_sha512_ctx *ctx, + const u8 *data, size_t data_len) +{ + __sha512_update(&ctx->ctx.sha_ctx, data, data_len); +} + +/** + * hmac_sha512_final() - Finish computing an HMAC-SHA512 value + * @ctx: the HMAC context to finalize; must have been initialized + * @out: (output) the resulting HMAC-SHA512 value + * + * After finishing, this zeroizes @ctx. So the caller does not need to do= it. + * + * Context: Any context. + */ +void hmac_sha512_final(struct hmac_sha512_ctx *ctx, u8 out[SHA512_DIGEST_S= IZE]); + +/** + * hmac_sha512() - Compute HMAC-SHA512 in one shot, using a prepared key + * @key: the prepared HMAC key + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA512 value + * + * If you're using the key only once, consider using hmac_sha512_usingrawk= ey(). + * + * Context: Any context. + */ +void hmac_sha512(const struct hmac_sha512_key *key, + const u8 *data, size_t data_len, u8 out[SHA512_DIGEST_SIZE]); + +/** + * hmac_sha512_usingrawkey() - Compute HMAC-SHA512 in one shot, using a ra= w key + * @raw_key: the raw HMAC-SHA512 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA512 value + * + * If you're using the key multiple times, prefer to use + * hmac_sha512_preparekey() followed by multiple calls to hmac_sha512() in= stead. + * + * Context: Any context. + */ +void hmac_sha512_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[SHA512_DIGEST_SIZE]); + #endif /* _CRYPTO_SHA2_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 2d295c0e0f792..d1bee3787eb3c 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -168,12 +168,13 @@ config CRYPTO_LIB_SHA256_GENERIC enabled, this implementation serves the users of CRYPTO_LIB_SHA256. =20 config CRYPTO_LIB_SHA512 tristate help - The SHA-384 and SHA-512 library functions. Select this if your module - uses any of these functions from . + The SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512 library functions. + Select this if your module uses any of these functions from + . =20 config CRYPTO_LIB_SHA512_ARCH bool depends on CRYPTO_LIB_SHA512 && !UML =20 diff --git a/lib/crypto/sha512.c b/lib/crypto/sha512.c index 536b71481b1ca..d514721491ca0 100644 --- a/lib/crypto/sha512.c +++ b/lib/crypto/sha512.c @@ -1,22 +1,24 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * SHA-384 and SHA-512 library functions + * SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512 library functions * * Copyright (c) Jean-Luc Cooke * Copyright (c) Andrew McDonald * Copyright (c) 2003 Kyle McMartin * Copyright 2025 Google LLC */ =20 +#include #include #include #include #include #include #include #include +#include =20 static const struct sha512_block_state sha384_iv =3D { .h =3D { SHA384_H0, SHA384_H1, SHA384_H2, SHA384_H3, SHA384_H4, SHA384_H5, SHA384_H6, SHA384_H7, @@ -245,10 +247,145 @@ void sha512(const u8 *data, size_t len, u8 out[SHA51= 2_DIGEST_SIZE]) sha512_update(&ctx, data, len); sha512_final(&ctx, out); } EXPORT_SYMBOL_GPL(sha512); =20 +static void __hmac_sha512_preparekey(struct __hmac_sha512_key *key, + const u8 *raw_key, size_t raw_key_len, + const struct sha512_block_state *iv) +{ + union { + u8 b[SHA512_BLOCK_SIZE]; + unsigned long w[SHA512_BLOCK_SIZE / sizeof(unsigned long)]; + } derived_key =3D { 0 }; + + if (unlikely(raw_key_len > SHA512_BLOCK_SIZE)) { + if (iv =3D=3D &sha384_iv) + sha384(raw_key, raw_key_len, derived_key.b); + else + sha512(raw_key, raw_key_len, derived_key.b); + } else { + memcpy(derived_key.b, raw_key, raw_key_len); + } + + for (size_t i =3D 0; i < ARRAY_SIZE(derived_key.w); i++) + derived_key.w[i] ^=3D REPEAT_BYTE(HMAC_IPAD_VALUE); + key->istate =3D *iv; + sha512_blocks(&key->istate, derived_key.b, 1); + + for (size_t i =3D 0; i < ARRAY_SIZE(derived_key.w); i++) + derived_key.w[i] ^=3D REPEAT_BYTE(HMAC_OPAD_VALUE ^ + HMAC_IPAD_VALUE); + key->ostate =3D *iv; + sha512_blocks(&key->ostate, derived_key.b, 1); + + memzero_explicit(&derived_key, sizeof(derived_key)); +} + +void hmac_sha384_preparekey(struct hmac_sha384_key *key, + const u8 *raw_key, size_t raw_key_len) +{ + __hmac_sha512_preparekey(&key->key, raw_key, raw_key_len, &sha384_iv); +} +EXPORT_SYMBOL_GPL(hmac_sha384_preparekey); + +void hmac_sha512_preparekey(struct hmac_sha512_key *key, + const u8 *raw_key, size_t raw_key_len) +{ + __hmac_sha512_preparekey(&key->key, raw_key, raw_key_len, &sha512_iv); +} +EXPORT_SYMBOL_GPL(hmac_sha512_preparekey); + +void __hmac_sha512_init(struct __hmac_sha512_ctx *ctx, + const struct __hmac_sha512_key *key) +{ + __sha512_init(&ctx->sha_ctx, &key->istate, SHA512_BLOCK_SIZE); + ctx->ostate =3D key->ostate; +} +EXPORT_SYMBOL_GPL(__hmac_sha512_init); + +static void __hmac_sha512_final(struct __hmac_sha512_ctx *ctx, + u8 *out, size_t digest_size) +{ + /* Generate the padded input for the outer hash in ctx->sha_ctx.buf. */ + __sha512_final(&ctx->sha_ctx, ctx->sha_ctx.buf, digest_size); + memset(&ctx->sha_ctx.buf[digest_size], 0, + SHA512_BLOCK_SIZE - digest_size); + ctx->sha_ctx.buf[digest_size] =3D 0x80; + *(__be32 *)&ctx->sha_ctx.buf[SHA512_BLOCK_SIZE - 4] =3D + cpu_to_be32(8 * (SHA512_BLOCK_SIZE + digest_size)); + + /* Compute the outer hash, which gives the HMAC value. */ + sha512_blocks(&ctx->ostate, ctx->sha_ctx.buf, 1); + for (size_t i =3D 0; i < digest_size; i +=3D 8) + put_unaligned_be64(ctx->ostate.h[i / 8], out + i); + + memzero_explicit(ctx, sizeof(*ctx)); +} + +void hmac_sha384_final(struct hmac_sha384_ctx *ctx, + u8 out[SHA384_DIGEST_SIZE]) +{ + __hmac_sha512_final(&ctx->ctx, out, SHA384_DIGEST_SIZE); +} +EXPORT_SYMBOL_GPL(hmac_sha384_final); + +void hmac_sha512_final(struct hmac_sha512_ctx *ctx, + u8 out[SHA512_DIGEST_SIZE]) +{ + __hmac_sha512_final(&ctx->ctx, out, SHA512_DIGEST_SIZE); +} +EXPORT_SYMBOL_GPL(hmac_sha512_final); + +void hmac_sha384(const struct hmac_sha384_key *key, + const u8 *data, size_t data_len, u8 out[SHA384_DIGEST_SIZE]) +{ + struct hmac_sha384_ctx ctx; + + hmac_sha384_init(&ctx, key); + hmac_sha384_update(&ctx, data, data_len); + hmac_sha384_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(hmac_sha384); + +void hmac_sha512(const struct hmac_sha512_key *key, + const u8 *data, size_t data_len, u8 out[SHA512_DIGEST_SIZE]) +{ + struct hmac_sha512_ctx ctx; + + hmac_sha512_init(&ctx, key); + hmac_sha512_update(&ctx, data, data_len); + hmac_sha512_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(hmac_sha512); + +void hmac_sha384_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[SHA384_DIGEST_SIZE]) +{ + struct hmac_sha384_key key; + + hmac_sha384_preparekey(&key, raw_key, raw_key_len); + hmac_sha384(&key, data, data_len, out); + + memzero_explicit(&key, sizeof(key)); +} +EXPORT_SYMBOL_GPL(hmac_sha384_usingrawkey); + +void hmac_sha512_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[SHA512_DIGEST_SIZE]) +{ + struct hmac_sha512_key key; + + hmac_sha512_preparekey(&key, raw_key, raw_key_len); + hmac_sha512(&key, data, data_len, out); + + memzero_explicit(&key, sizeof(key)); +} +EXPORT_SYMBOL_GPL(hmac_sha512_usingrawkey); + #ifdef sha512_mod_init_arch static int __init sha512_mod_init(void) { sha512_mod_init_arch(); return 0; @@ -259,7 +396,7 @@ static void __exit sha512_mod_exit(void) { } module_exit(sha512_mod_exit); #endif =20 -MODULE_DESCRIPTION("SHA-384 and SHA-512 library functions"); +MODULE_DESCRIPTION("SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512 library= functions"); MODULE_LICENSE("GPL"); --=20 2.50.0 From nobody Wed Oct 8 08:14:50 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1CF43291861; Mon, 30 Jun 2025 16:05:23 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299524; cv=none; b=ejPfP8ilCuxIAUn4sPEEs+jpktKVcEbYE8RS++tUYN5XaJq/FobwnmkJUfnLphrzwY5Q4mpHknX/KNbt25s9A+PqtB0YEbm+eIROVILC/eSw7BpTHxTyrHzckZQZp35gVEdhN8HMKbwS0Lx36CjlYPrTV+Wu8CYKDX7yhcbJhn0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299524; c=relaxed/simple; bh=Weaemm5tVSSOITzoU3Exa8gIqcPO7agiHtgu1h0EiAY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=CR4bczUTzjkOC4klTVwohwIjmtLoXHnxyyDic+v7g/8DbxuD7u6yE4suanYRG8omHqkB+D1sgDr4D9PJJdQPdHbHmEvf66Lxx0qsh+ogfovxY0SubRhZMurViJP88O6wAcbYzk4mG+8pUZONeXiVSyIzdAHGtq2ES18SgNUS0sU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=G6SdNRxT; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="G6SdNRxT" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 5B2EFC4CEF3; Mon, 30 Jun 2025 16:05:23 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299523; bh=Weaemm5tVSSOITzoU3Exa8gIqcPO7agiHtgu1h0EiAY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=G6SdNRxTh3Ciw7tvEBvdMJDJ2pBTRqKZrXnc4VCIBytKFkTJpt3yLkXrBQ43W5Xfc MSaPHP2zGfS31WjYmnoGB/3etvzWfs59cG8i3V0bPCVh9lqS/ayx3PhzyweVM6SHpE a8K3fbneg6OFdo2NJwZPzGn0ji5Zt97pbqCwuASSUt+oAi8zMRh4Zok+CG2MjbAfLp kG5RyDFYNzMraJW04smN96tyaP7uFDnj9JCU1Qu8HE63kCi4qCBolaIG2pqAI1w2m+ mePoviuPsG+xzMDSE3fl0wEsBX9Y4iAWgwOE4kx+gzIx0neyORBtS1/NZ0dV58LGej DeRbkpYFvI67A== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 04/16] crypto: riscv/sha512 - Stop depending on sha512_generic_block_fn Date: Mon, 30 Jun 2025 09:03:08 -0700 Message-ID: <20250630160320.2888-5-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" sha512_generic_block_fn() will no longer be available when the SHA-512 support in the old-school crypto API is changed to just wrap the SHA-512 library. Replace the use of sha512_generic_block_fn() in sha512-riscv64-glue.c with temporary code that uses the library's __sha512_update(). This is just a temporary workaround to keep the kernel building and functional at each commit; this code gets superseded when the RISC-V optimized SHA-512 is migrated to lib/crypto/ anyway. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers --- arch/riscv/crypto/Kconfig | 1 + arch/riscv/crypto/sha512-riscv64-glue.c | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index cd9b776602f89..53e4e1eacf554 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -29,10 +29,11 @@ config CRYPTO_GHASH_RISCV64 - Zvkg vector crypto extension =20 config CRYPTO_SHA512_RISCV64 tristate "Hash functions: SHA-384 and SHA-512" depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO + select CRYPTO_LIB_SHA512 select CRYPTO_SHA512 help SHA-384 and SHA-512 secure hash algorithm (FIPS 180) =20 Architecture: riscv64 using: diff --git a/arch/riscv/crypto/sha512-riscv64-glue.c b/arch/riscv/crypto/sh= a512-riscv64-glue.c index 4634fca78ae24..b3dbc71de07b0 100644 --- a/arch/riscv/crypto/sha512-riscv64-glue.c +++ b/arch/riscv/crypto/sha512-riscv64-glue.c @@ -36,11 +36,17 @@ static void sha512_block(struct sha512_state *state, co= nst u8 *data, if (crypto_simd_usable()) { kernel_vector_begin(); sha512_transform_zvknhb_zvkb(state, data, num_blocks); kernel_vector_end(); } else { - sha512_generic_block_fn(state, data, num_blocks); + struct __sha512_ctx ctx =3D {}; + + static_assert(sizeof(ctx.state) =3D=3D sizeof(state->state)); + memcpy(&ctx.state, state->state, sizeof(ctx.state)); + __sha512_update(&ctx, data, + (size_t)num_blocks * SHA512_BLOCK_SIZE); + memcpy(state->state, &ctx.state, sizeof(state->state)); } } =20 static int riscv64_sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) --=20 2.50.0 From nobody Wed Oct 8 08:14:51 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 8A2DC291C0F; Mon, 30 Jun 2025 16:05:24 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299524; cv=none; b=nXa06VUYAJY9/5tx6YtCEk0ynehnDK74kqR6bv62WIOR76Bgqm7xOtUv6VMEZ6lYo6Tqlow8Vhv2CBfspai6LppJeGXLDBbnn5U8GB/xsB9tOI/TNxeEh4B8ZcqfB587kuC4jJ5ux3DWRo+CtHGQECBRK4oBdeur/EgneeHgoZA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299524; c=relaxed/simple; bh=yvRIGyqN+DTv9LrLTgtWFFljA03UEvTyM+LQvlz7nbU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=PTIAocjlbYyuMmWHqEIi8D5zkpuBQWrd2aQu+ZndKzJyhputl+pm1ac+1ROF5Oy+Lbr8zMFhf+d+85uxRmOhYMXsmPc7i8eKqB+/x3nZeTlbzEBdPesfabo3TGBraUaQf7EL2XJzyo9WysHi99OfvQt7PkhSF+SXxriZg+aDi7w= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=JF6sLCEV; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="JF6sLCEV" Received: by smtp.kernel.org (Postfix) with ESMTPSA id CE655C4CEE3; Mon, 30 Jun 2025 16:05:23 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299524; bh=yvRIGyqN+DTv9LrLTgtWFFljA03UEvTyM+LQvlz7nbU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=JF6sLCEVPHvLGEuPZjbm5dMHGbYO276013+KylR+lU+34N38RlvZ42RwM+jRFK1MV u8xjn1UFXDPm7PIL4nHCAfWMViFt004SOCu2qR7qavbU8C11cd3CBGh7LPhYCyHFdM S/Co0+XU+sCnghgFhbAuhSgRFLtRTHw+mEPcp03oplTgiHja9j4YZ9LrltsmXpHkdr BNep3sQ3JaRQ287J9p5zSP63hw//bT2hC0EoQ681rxu1ibjQQLgkkvWkI0FsqKDua9 i+KN+E94C2ETEGSMmp+Mc+NqY57WfjZMUy5hfLg+XX79HGBTXJdsZanl08l/oKY/+b wlGCRh4OPMe4g== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 05/16] crypto: sha512 - Replace sha512_generic with wrapper around SHA-512 library Date: Mon, 30 Jun 2025 09:03:09 -0700 Message-ID: <20250630160320.2888-6-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Delete crypto/sha512_generic.c, which provided "generic" SHA-384 and SHA-512 crypto_shash algorithms. Replace it with crypto/sha512.c which provides SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512 crypto_shash algorithms using the corresponding library functions. This is a prerequisite for migrating all the arch-optimized SHA-512 code (which is almost 3000 lines) to lib/crypto/ rather than duplicating it. Since the replacement crypto_shash algorithms are implemented using the (potentially arch-optimized) library functions, give them cra_driver_names ending with "-lib" rather than "-generic". Update crypto/testmgr.c and one odd driver to take this change in driver name into account. Besides these cases which are accounted for, there are no known cases where the cra_driver_name was being depended on. This change does mean that the abstract partial block handling code in crypto/shash.c, which got added in 6.16, no longer gets used. But that's fine; the library has to implement the partial block handling anyway, and it's better to do it in the library since the block size and other properties of the algorithm are all fixed at compile time there, resulting in more streamlined code. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers --- crypto/Kconfig | 4 +- crypto/Makefile | 2 +- crypto/sha512.c | 258 ++++++++++++++++++++++++++ crypto/sha512_generic.c | 217 ---------------------- crypto/testmgr.c | 10 + drivers/crypto/starfive/jh7110-hash.c | 8 +- include/crypto/sha512_base.h | 3 - 7 files changed, 276 insertions(+), 226 deletions(-) create mode 100644 crypto/sha512.c delete mode 100644 crypto/sha512_generic.c diff --git a/crypto/Kconfig b/crypto/Kconfig index e1cfd0d4cc8f8..cb40a9b469722 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1000,12 +1000,14 @@ config CRYPTO_SHA256 Used by the btrfs filesystem, Ceph, NFS, and SMB. =20 config CRYPTO_SHA512 tristate "SHA-384 and SHA-512" select CRYPTO_HASH + select CRYPTO_LIB_SHA512 help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180, ISO/IEC 10118-3) + SHA-384 and SHA-512 secure hash algorithms (FIPS 180, ISO/IEC + 10118-3), including HMAC support. =20 config CRYPTO_SHA3 tristate "SHA-3" select CRYPTO_HASH help diff --git a/crypto/Makefile b/crypto/Makefile index 017df3a2e4bb3..271c77462cec9 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -76,11 +76,11 @@ obj-$(CONFIG_CRYPTO_MD4) +=3D md4.o obj-$(CONFIG_CRYPTO_MD5) +=3D md5.o obj-$(CONFIG_CRYPTO_RMD160) +=3D rmd160.o obj-$(CONFIG_CRYPTO_SHA1) +=3D sha1_generic.o obj-$(CONFIG_CRYPTO_SHA256) +=3D sha256.o CFLAGS_sha256.o +=3D -DARCH=3D$(ARCH) -obj-$(CONFIG_CRYPTO_SHA512) +=3D sha512_generic.o +obj-$(CONFIG_CRYPTO_SHA512) +=3D sha512.o obj-$(CONFIG_CRYPTO_SHA3) +=3D sha3_generic.o obj-$(CONFIG_CRYPTO_SM3_GENERIC) +=3D sm3_generic.o obj-$(CONFIG_CRYPTO_STREEBOG) +=3D streebog_generic.o obj-$(CONFIG_CRYPTO_WP512) +=3D wp512.o CFLAGS_wp512.o :=3D $(call cc-option,-fno-schedule-insns) # https://gcc.g= nu.org/bugzilla/show_bug.cgi?id=3D79149 diff --git a/crypto/sha512.c b/crypto/sha512.c new file mode 100644 index 0000000000000..d1e5400fe5906 --- /dev/null +++ b/crypto/sha512.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Crypto API support for SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512 + * + * Copyright (c) Jean-Luc Cooke + * Copyright (c) Andrew McDonald + * Copyright (c) 2003 Kyle McMartin + * Copyright 2025 Google LLC + */ +#include +#include +#include +#include + +/* SHA-384 */ + +const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE] =3D { + 0x38, 0xb0, 0x60, 0xa7, 0x51, 0xac, 0x96, 0x38, + 0x4c, 0xd9, 0x32, 0x7e, 0xb1, 0xb1, 0xe3, 0x6a, + 0x21, 0xfd, 0xb7, 0x11, 0x14, 0xbe, 0x07, 0x43, + 0x4c, 0x0c, 0xc7, 0xbf, 0x63, 0xf6, 0xe1, 0xda, + 0x27, 0x4e, 0xde, 0xbf, 0xe7, 0x6f, 0x65, 0xfb, + 0xd5, 0x1a, 0xd2, 0xf1, 0x48, 0x98, 0xb9, 0x5b +}; +EXPORT_SYMBOL_GPL(sha384_zero_message_hash); + +#define SHA384_CTX(desc) ((struct sha384_ctx *)shash_desc_ctx(desc)) + +static int crypto_sha384_init(struct shash_desc *desc) +{ + sha384_init(SHA384_CTX(desc)); + return 0; +} + +static int crypto_sha384_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + sha384_update(SHA384_CTX(desc), data, len); + return 0; +} + +static int crypto_sha384_final(struct shash_desc *desc, u8 *out) +{ + sha384_final(SHA384_CTX(desc), out); + return 0; +} + +static int crypto_sha384_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, u8 *out) +{ + sha384(data, len, out); + return 0; +} + +/* SHA-512 */ + +const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE] =3D { + 0xcf, 0x83, 0xe1, 0x35, 0x7e, 0xef, 0xb8, 0xbd, + 0xf1, 0x54, 0x28, 0x50, 0xd6, 0x6d, 0x80, 0x07, + 0xd6, 0x20, 0xe4, 0x05, 0x0b, 0x57, 0x15, 0xdc, + 0x83, 0xf4, 0xa9, 0x21, 0xd3, 0x6c, 0xe9, 0xce, + 0x47, 0xd0, 0xd1, 0x3c, 0x5d, 0x85, 0xf2, 0xb0, + 0xff, 0x83, 0x18, 0xd2, 0x87, 0x7e, 0xec, 0x2f, + 0x63, 0xb9, 0x31, 0xbd, 0x47, 0x41, 0x7a, 0x81, + 0xa5, 0x38, 0x32, 0x7a, 0xf9, 0x27, 0xda, 0x3e +}; +EXPORT_SYMBOL_GPL(sha512_zero_message_hash); + +#define SHA512_CTX(desc) ((struct sha512_ctx *)shash_desc_ctx(desc)) + +static int crypto_sha512_init(struct shash_desc *desc) +{ + sha512_init(SHA512_CTX(desc)); + return 0; +} + +static int crypto_sha512_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + sha512_update(SHA512_CTX(desc), data, len); + return 0; +} + +static int crypto_sha512_final(struct shash_desc *desc, u8 *out) +{ + sha512_final(SHA512_CTX(desc), out); + return 0; +} + +static int crypto_sha512_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, u8 *out) +{ + sha512(data, len, out); + return 0; +} + +/* HMAC-SHA384 */ + +#define HMAC_SHA384_KEY(tfm) ((struct hmac_sha384_key *)crypto_shash_ctx(t= fm)) +#define HMAC_SHA384_CTX(desc) ((struct hmac_sha384_ctx *)shash_desc_ctx(de= sc)) + +static int crypto_hmac_sha384_setkey(struct crypto_shash *tfm, + const u8 *raw_key, unsigned int keylen) +{ + hmac_sha384_preparekey(HMAC_SHA384_KEY(tfm), raw_key, keylen); + return 0; +} + +static int crypto_hmac_sha384_init(struct shash_desc *desc) +{ + hmac_sha384_init(HMAC_SHA384_CTX(desc), HMAC_SHA384_KEY(desc->tfm)); + return 0; +} + +static int crypto_hmac_sha384_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + hmac_sha384_update(HMAC_SHA384_CTX(desc), data, len); + return 0; +} + +static int crypto_hmac_sha384_final(struct shash_desc *desc, u8 *out) +{ + hmac_sha384_final(HMAC_SHA384_CTX(desc), out); + return 0; +} + +static int crypto_hmac_sha384_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, + u8 *out) +{ + hmac_sha384(HMAC_SHA384_KEY(desc->tfm), data, len, out); + return 0; +} + +/* HMAC-SHA512 */ + +#define HMAC_SHA512_KEY(tfm) ((struct hmac_sha512_key *)crypto_shash_ctx(t= fm)) +#define HMAC_SHA512_CTX(desc) ((struct hmac_sha512_ctx *)shash_desc_ctx(de= sc)) + +static int crypto_hmac_sha512_setkey(struct crypto_shash *tfm, + const u8 *raw_key, unsigned int keylen) +{ + hmac_sha512_preparekey(HMAC_SHA512_KEY(tfm), raw_key, keylen); + return 0; +} + +static int crypto_hmac_sha512_init(struct shash_desc *desc) +{ + hmac_sha512_init(HMAC_SHA512_CTX(desc), HMAC_SHA512_KEY(desc->tfm)); + return 0; +} + +static int crypto_hmac_sha512_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + hmac_sha512_update(HMAC_SHA512_CTX(desc), data, len); + return 0; +} + +static int crypto_hmac_sha512_final(struct shash_desc *desc, u8 *out) +{ + hmac_sha512_final(HMAC_SHA512_CTX(desc), out); + return 0; +} + +static int crypto_hmac_sha512_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, + u8 *out) +{ + hmac_sha512(HMAC_SHA512_KEY(desc->tfm), data, len, out); + return 0; +} + +/* Algorithm definitions */ + +static struct shash_alg algs[] =3D { + { + .base.cra_name =3D "sha384", + .base.cra_driver_name =3D "sha384-lib", + .base.cra_priority =3D 300, + .base.cra_blocksize =3D SHA384_BLOCK_SIZE, + .base.cra_module =3D THIS_MODULE, + .digestsize =3D SHA384_DIGEST_SIZE, + .init =3D crypto_sha384_init, + .update =3D crypto_sha384_update, + .final =3D crypto_sha384_final, + .digest =3D crypto_sha384_digest, + .descsize =3D sizeof(struct sha384_ctx), + }, + { + .base.cra_name =3D "sha512", + .base.cra_driver_name =3D "sha512-lib", + .base.cra_priority =3D 300, + .base.cra_blocksize =3D SHA512_BLOCK_SIZE, + .base.cra_module =3D THIS_MODULE, + .digestsize =3D SHA512_DIGEST_SIZE, + .init =3D crypto_sha512_init, + .update =3D crypto_sha512_update, + .final =3D crypto_sha512_final, + .digest =3D crypto_sha512_digest, + .descsize =3D sizeof(struct sha512_ctx), + }, + { + .base.cra_name =3D "hmac(sha384)", + .base.cra_driver_name =3D "hmac-sha384-lib", + .base.cra_priority =3D 300, + .base.cra_blocksize =3D SHA384_BLOCK_SIZE, + .base.cra_ctxsize =3D sizeof(struct hmac_sha384_key), + .base.cra_module =3D THIS_MODULE, + .digestsize =3D SHA384_DIGEST_SIZE, + .setkey =3D crypto_hmac_sha384_setkey, + .init =3D crypto_hmac_sha384_init, + .update =3D crypto_hmac_sha384_update, + .final =3D crypto_hmac_sha384_final, + .digest =3D crypto_hmac_sha384_digest, + .descsize =3D sizeof(struct hmac_sha384_ctx), + }, + { + .base.cra_name =3D "hmac(sha512)", + .base.cra_driver_name =3D "hmac-sha512-lib", + .base.cra_priority =3D 300, + .base.cra_blocksize =3D SHA512_BLOCK_SIZE, + .base.cra_ctxsize =3D sizeof(struct hmac_sha512_key), + .base.cra_module =3D THIS_MODULE, + .digestsize =3D SHA512_DIGEST_SIZE, + .setkey =3D crypto_hmac_sha512_setkey, + .init =3D crypto_hmac_sha512_init, + .update =3D crypto_hmac_sha512_update, + .final =3D crypto_hmac_sha512_final, + .digest =3D crypto_hmac_sha512_digest, + .descsize =3D sizeof(struct hmac_sha512_ctx), + }, +}; + +static int __init crypto_sha512_mod_init(void) +{ + return crypto_register_shashes(algs, ARRAY_SIZE(algs)); +} +module_init(crypto_sha512_mod_init); + +static void __exit crypto_sha512_mod_exit(void) +{ + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); +} +module_exit(crypto_sha512_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Crypto API support for SHA-384, SHA-512, HMAC-SHA384, = and HMAC-SHA512"); + +MODULE_ALIAS_CRYPTO("sha384"); +MODULE_ALIAS_CRYPTO("sha384-lib"); +MODULE_ALIAS_CRYPTO("sha512"); +MODULE_ALIAS_CRYPTO("sha512-lib"); +MODULE_ALIAS_CRYPTO("hmac(sha384)"); +MODULE_ALIAS_CRYPTO("hmac-sha384-lib"); +MODULE_ALIAS_CRYPTO("hmac(sha512)"); +MODULE_ALIAS_CRYPTO("hmac-sha512-lib"); diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c deleted file mode 100644 index 7368173f545eb..0000000000000 --- a/crypto/sha512_generic.c +++ /dev/null @@ -1,217 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* SHA-512 code by Jean-Luc Cooke - * - * Copyright (c) Jean-Luc Cooke - * Copyright (c) Andrew McDonald - * Copyright (c) 2003 Kyle McMartin - */ -#include -#include -#include -#include -#include -#include - -const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE] =3D { - 0x38, 0xb0, 0x60, 0xa7, 0x51, 0xac, 0x96, 0x38, - 0x4c, 0xd9, 0x32, 0x7e, 0xb1, 0xb1, 0xe3, 0x6a, - 0x21, 0xfd, 0xb7, 0x11, 0x14, 0xbe, 0x07, 0x43, - 0x4c, 0x0c, 0xc7, 0xbf, 0x63, 0xf6, 0xe1, 0xda, - 0x27, 0x4e, 0xde, 0xbf, 0xe7, 0x6f, 0x65, 0xfb, - 0xd5, 0x1a, 0xd2, 0xf1, 0x48, 0x98, 0xb9, 0x5b -}; -EXPORT_SYMBOL_GPL(sha384_zero_message_hash); - -const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE] =3D { - 0xcf, 0x83, 0xe1, 0x35, 0x7e, 0xef, 0xb8, 0xbd, - 0xf1, 0x54, 0x28, 0x50, 0xd6, 0x6d, 0x80, 0x07, - 0xd6, 0x20, 0xe4, 0x05, 0x0b, 0x57, 0x15, 0xdc, - 0x83, 0xf4, 0xa9, 0x21, 0xd3, 0x6c, 0xe9, 0xce, - 0x47, 0xd0, 0xd1, 0x3c, 0x5d, 0x85, 0xf2, 0xb0, - 0xff, 0x83, 0x18, 0xd2, 0x87, 0x7e, 0xec, 0x2f, - 0x63, 0xb9, 0x31, 0xbd, 0x47, 0x41, 0x7a, 0x81, - 0xa5, 0x38, 0x32, 0x7a, 0xf9, 0x27, 0xda, 0x3e -}; -EXPORT_SYMBOL_GPL(sha512_zero_message_hash); - -static inline u64 Ch(u64 x, u64 y, u64 z) -{ - return z ^ (x & (y ^ z)); -} - -static inline u64 Maj(u64 x, u64 y, u64 z) -{ - return (x & y) | (z & (x | y)); -} - -static const u64 sha512_K[80] =3D { - 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fUL= L, - 0xe9b5dba58189dbbcULL, 0x3956c25bf348b538ULL, 0x59f111f1b605d019UL= L, - 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, 0xd807aa98a3030242UL= L, - 0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2UL= L, - 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235UL= L, - 0xc19bf174cf692694ULL, 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3UL= L, - 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, 0x2de92c6f592b0275UL= L, - 0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5UL= L, - 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fUL= L, - 0xbf597fc7beef0ee4ULL, 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725UL= L, - 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 0x27b70a8546d22ffcUL= L, - 0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfUL= L, - 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6UL= L, - 0x92722c851482353bULL, 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001UL= L, - 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, 0xd192e819d6ef5218UL= L, - 0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8UL= L, - 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99UL= L, - 0x34b0bcb5e19b48a8ULL, 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbUL= L, - 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, 0x748f82ee5defb2fcUL= L, - 0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecUL= L, - 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915UL= L, - 0xc67178f2e372532bULL, 0xca273eceea26619cULL, 0xd186b8c721c0c207UL= L, - 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, 0x06f067aa72176fbaUL= L, - 0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bUL= L, - 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcUL= L, - 0x431d67c49c100d4cULL, 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aUL= L, - 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, -}; - -#define e0(x) (ror64(x,28) ^ ror64(x,34) ^ ror64(x,39)) -#define e1(x) (ror64(x,14) ^ ror64(x,18) ^ ror64(x,41)) -#define s0(x) (ror64(x, 1) ^ ror64(x, 8) ^ (x >> 7)) -#define s1(x) (ror64(x,19) ^ ror64(x,61) ^ (x >> 6)) - -static inline void LOAD_OP(int I, u64 *W, const u8 *input) -{ - W[I] =3D get_unaligned_be64((__u64 *)input + I); -} - -static inline void BLEND_OP(int I, u64 *W) -{ - W[I & 15] +=3D s1(W[(I-2) & 15]) + W[(I-7) & 15] + s0(W[(I-15) & 15]); -} - -static void -sha512_transform(u64 *state, const u8 *input) -{ - u64 a, b, c, d, e, f, g, h, t1, t2; - - int i; - u64 W[16]; - - /* load the state into our registers */ - a=3Dstate[0]; b=3Dstate[1]; c=3Dstate[2]; d=3Dstate[3]; - e=3Dstate[4]; f=3Dstate[5]; g=3Dstate[6]; h=3Dstate[7]; - - /* now iterate */ - for (i=3D0; i<80; i+=3D8) { - if (!(i & 8)) { - int j; - - if (i < 16) { - /* load the input */ - for (j =3D 0; j < 16; j++) - LOAD_OP(i + j, W, input); - } else { - for (j =3D 0; j < 16; j++) { - BLEND_OP(i + j, W); - } - } - } - - t1 =3D h + e1(e) + Ch(e,f,g) + sha512_K[i ] + W[(i & 15)]; - t2 =3D e0(a) + Maj(a,b,c); d+=3Dt1; h=3Dt1+t2; - t1 =3D g + e1(d) + Ch(d,e,f) + sha512_K[i+1] + W[(i & 15) + 1]; - t2 =3D e0(h) + Maj(h,a,b); c+=3Dt1; g=3Dt1+t2; - t1 =3D f + e1(c) + Ch(c,d,e) + sha512_K[i+2] + W[(i & 15) + 2]; - t2 =3D e0(g) + Maj(g,h,a); b+=3Dt1; f=3Dt1+t2; - t1 =3D e + e1(b) + Ch(b,c,d) + sha512_K[i+3] + W[(i & 15) + 3]; - t2 =3D e0(f) + Maj(f,g,h); a+=3Dt1; e=3Dt1+t2; - t1 =3D d + e1(a) + Ch(a,b,c) + sha512_K[i+4] + W[(i & 15) + 4]; - t2 =3D e0(e) + Maj(e,f,g); h+=3Dt1; d=3Dt1+t2; - t1 =3D c + e1(h) + Ch(h,a,b) + sha512_K[i+5] + W[(i & 15) + 5]; - t2 =3D e0(d) + Maj(d,e,f); g+=3Dt1; c=3Dt1+t2; - t1 =3D b + e1(g) + Ch(g,h,a) + sha512_K[i+6] + W[(i & 15) + 6]; - t2 =3D e0(c) + Maj(c,d,e); f+=3Dt1; b=3Dt1+t2; - t1 =3D a + e1(f) + Ch(f,g,h) + sha512_K[i+7] + W[(i & 15) + 7]; - t2 =3D e0(b) + Maj(b,c,d); e+=3Dt1; a=3Dt1+t2; - } - - state[0] +=3D a; state[1] +=3D b; state[2] +=3D c; state[3] +=3D d; - state[4] +=3D e; state[5] +=3D f; state[6] +=3D g; state[7] +=3D h; -} - -void sha512_generic_block_fn(struct sha512_state *sst, u8 const *src, - int blocks) -{ - do { - sha512_transform(sst->state, src); - src +=3D SHA512_BLOCK_SIZE; - } while (--blocks); -} -EXPORT_SYMBOL_GPL(sha512_generic_block_fn); - -static int crypto_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_base_do_update_blocks(desc, data, len, - sha512_generic_block_fn); -} - -static int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash) -{ - sha512_base_do_finup(desc, data, len, sha512_generic_block_fn); - return sha512_base_finish(desc, hash); -} - -static struct shash_alg sha512_algs[2] =3D { { - .digestsize =3D SHA512_DIGEST_SIZE, - .init =3D sha512_base_init, - .update =3D crypto_sha512_update, - .finup =3D crypto_sha512_finup, - .descsize =3D SHA512_STATE_SIZE, - .base =3D { - .cra_name =3D "sha512", - .cra_driver_name =3D "sha512-generic", - .cra_priority =3D 100, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize =3D SHA512_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -}, { - .digestsize =3D SHA384_DIGEST_SIZE, - .init =3D sha384_base_init, - .update =3D crypto_sha512_update, - .finup =3D crypto_sha512_finup, - .descsize =3D SHA512_STATE_SIZE, - .base =3D { - .cra_name =3D "sha384", - .cra_driver_name =3D "sha384-generic", - .cra_priority =3D 100, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize =3D SHA384_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -} }; - -static int __init sha512_generic_mod_init(void) -{ - return crypto_register_shashes(sha512_algs, ARRAY_SIZE(sha512_algs)); -} - -static void __exit sha512_generic_mod_fini(void) -{ - crypto_unregister_shashes(sha512_algs, ARRAY_SIZE(sha512_algs)); -} - -module_init(sha512_generic_mod_init); -module_exit(sha512_generic_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-512 and SHA-384 Secure Hash Algorithms"); - -MODULE_ALIAS_CRYPTO("sha384"); -MODULE_ALIAS_CRYPTO("sha384-generic"); -MODULE_ALIAS_CRYPTO("sha512"); -MODULE_ALIAS_CRYPTO("sha512-generic"); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 32f753d6c4302..9d8b11ea4af7f 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4313,49 +4313,55 @@ static const struct alg_test_desc alg_test_descs[] = =3D { .alg =3D "authenc(hmac(sha256),rfc3686(ctr(aes)))", .test =3D alg_test_null, .fips_allowed =3D 1, }, { .alg =3D "authenc(hmac(sha384),cbc(des))", + .generic_driver =3D "authenc(hmac-sha384-lib,cbc(des-generic))", .test =3D alg_test_aead, .suite =3D { .aead =3D __VECS(hmac_sha384_des_cbc_tv_temp) } }, { .alg =3D "authenc(hmac(sha384),cbc(des3_ede))", + .generic_driver =3D "authenc(hmac-sha384-lib,cbc(des3_ede-generic))", .test =3D alg_test_aead, .suite =3D { .aead =3D __VECS(hmac_sha384_des3_ede_cbc_tv_temp) } }, { .alg =3D "authenc(hmac(sha384),ctr(aes))", .test =3D alg_test_null, .fips_allowed =3D 1, }, { .alg =3D "authenc(hmac(sha384),cts(cbc(aes)))", + .generic_driver =3D "authenc(hmac-sha384-lib,cts(cbc(aes-generic)))", .test =3D alg_test_aead, .suite =3D { .aead =3D __VECS(krb5_test_aes256_cts_hmac_sha384_192) } }, { .alg =3D "authenc(hmac(sha384),rfc3686(ctr(aes)))", .test =3D alg_test_null, .fips_allowed =3D 1, }, { .alg =3D "authenc(hmac(sha512),cbc(aes))", + .generic_driver =3D "authenc(hmac-sha512-lib,cbc(aes-generic))", .fips_allowed =3D 1, .test =3D alg_test_aead, .suite =3D { .aead =3D __VECS(hmac_sha512_aes_cbc_tv_temp) } }, { .alg =3D "authenc(hmac(sha512),cbc(des))", + .generic_driver =3D "authenc(hmac-sha512-lib,cbc(des-generic))", .test =3D alg_test_aead, .suite =3D { .aead =3D __VECS(hmac_sha512_des_cbc_tv_temp) } }, { .alg =3D "authenc(hmac(sha512),cbc(des3_ede))", + .generic_driver =3D "authenc(hmac-sha512-lib,cbc(des3_ede-generic))", .test =3D alg_test_aead, .suite =3D { .aead =3D __VECS(hmac_sha512_des3_ede_cbc_tv_temp) } }, { @@ -5155,17 +5161,19 @@ static const struct alg_test_desc alg_test_descs[] = =3D { .suite =3D { .hash =3D __VECS(hmac_sha3_512_tv_template) } }, { .alg =3D "hmac(sha384)", + .generic_driver =3D "hmac-sha384-lib", .test =3D alg_test_hash, .fips_allowed =3D 1, .suite =3D { .hash =3D __VECS(hmac_sha384_tv_template) } }, { .alg =3D "hmac(sha512)", + .generic_driver =3D "hmac-sha512-lib", .test =3D alg_test_hash, .fips_allowed =3D 1, .suite =3D { .hash =3D __VECS(hmac_sha512_tv_template) } @@ -5491,17 +5499,19 @@ static const struct alg_test_desc alg_test_descs[] = =3D { .suite =3D { .hash =3D __VECS(sha3_512_tv_template) } }, { .alg =3D "sha384", + .generic_driver =3D "sha384-lib", .test =3D alg_test_hash, .fips_allowed =3D 1, .suite =3D { .hash =3D __VECS(sha384_tv_template) } }, { .alg =3D "sha512", + .generic_driver =3D "sha512-lib", .test =3D alg_test_hash, .fips_allowed =3D 1, .suite =3D { .hash =3D __VECS(sha512_tv_template) } diff --git a/drivers/crypto/starfive/jh7110-hash.c b/drivers/crypto/starfiv= e/jh7110-hash.c index 2c60a1047bc39..4abbff07412ff 100644 --- a/drivers/crypto/starfive/jh7110-hash.c +++ b/drivers/crypto/starfive/jh7110-hash.c @@ -503,17 +503,17 @@ static int starfive_sha256_init_tfm(struct crypto_aha= sh *hash) STARFIVE_HASH_SHA256, 0); } =20 static int starfive_sha384_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "sha384-generic", + return starfive_hash_init_tfm(hash, "sha384-lib", STARFIVE_HASH_SHA384, 0); } =20 static int starfive_sha512_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "sha512-generic", + return starfive_hash_init_tfm(hash, "sha512-lib", STARFIVE_HASH_SHA512, 0); } =20 static int starfive_sm3_init_tfm(struct crypto_ahash *hash) { @@ -533,17 +533,17 @@ static int starfive_hmac_sha256_init_tfm(struct crypt= o_ahash *hash) STARFIVE_HASH_SHA256, 1); } =20 static int starfive_hmac_sha384_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "hmac(sha384-generic)", + return starfive_hash_init_tfm(hash, "hmac-sha384-lib", STARFIVE_HASH_SHA384, 1); } =20 static int starfive_hmac_sha512_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "hmac(sha512-generic)", + return starfive_hash_init_tfm(hash, "hmac-sha512-lib", STARFIVE_HASH_SHA512, 1); } =20 static int starfive_hmac_sm3_init_tfm(struct crypto_ahash *hash) { diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h index aa814bab442d4..d1361b3eb70b0 100644 --- a/include/crypto/sha512_base.h +++ b/include/crypto/sha512_base.h @@ -112,9 +112,6 @@ static inline int sha512_base_finish(struct shash_desc = *desc, u8 *out) for (i =3D 0; digest_size > 0; i++, digest_size -=3D sizeof(__be64)) put_unaligned_be64(sctx->state[i], digest++); return 0; } =20 -void sha512_generic_block_fn(struct sha512_state *sst, u8 const *src, - int blocks); - #endif /* _CRYPTO_SHA512_BASE_H */ --=20 2.50.0 From nobody Wed Oct 8 08:14:51 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C13C8291C34; Mon, 30 Jun 2025 16:05:24 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299524; cv=none; b=QmG0hW3MYPyIQuLCM4kt7/yPt6X1Agcu2nW0nsuvv3fTnsjslDjMdJHCzhP+jaEzEMM9a2DCQGK8VwNewkvuhqhUW7ypl3eij3raWpKU72f8c7SMnPS/kNNjjwzY6d7psvL7UKBSs0XwiDCqQ6XRPEPIwhQ5XoXXl5YsqQSLguA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299524; c=relaxed/simple; bh=YQHYLY6F1cq4RblU0sL33f+9Pc6WlUUBxaZXFrCosr4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=YgZ171M1V0NCCLa7qFGyHOl8zGxI/re49+a2wL089En2y8+NUGk2ACA9oceBh5Hwv4nZl7yyOTlZouQTbIAD4Geo8AbTXjKljpFfuh1Cs4CHK77TQD5q2JhsjAQjpdMa1dQaAejaqWxWWbBmBah1MFThi5mc4nd2NGRjv+n31yo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Mf7RO/Yg; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Mf7RO/Yg" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 536DBC4CEF3; Mon, 30 Jun 2025 16:05:24 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299524; bh=YQHYLY6F1cq4RblU0sL33f+9Pc6WlUUBxaZXFrCosr4=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Mf7RO/Yg88sVHOL8L6t+pEWuB/+/x8zfTDaX8FyJzC4xydr3sH/BRW3DoD+J4hbHX AoDCornq7fnZJ122yoech7YF28ah37XJlQunAm8FbpA0fdTkNA2ZcOugIj8p49SDKy j/I65KSb2/Ynk9wU2P031Xtzt8SxlaUMFoXUpVwuxb1NBJuBxaWRf1RrxfscyOPBSF AloMwdLGt/88dpVK3sPmS3a0rBI+lj53NTYYxdngzGDwVYti2lS/G0SHTwniZqPBov 4x1VpeHJ5XXtj0J3Fl7FEx1+A/1+Zj6OGtbNOb5Mqnnfm0ZrGJLe3Ym9F1SK8gL1da w94/olI10A8Vw== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 06/16] crypto: sha512 - Use same state format as legacy drivers Date: Mon, 30 Jun 2025 09:03:10 -0700 Message-ID: <20250630160320.2888-7-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Make the export and import functions for the sha384, sha512, hmac(sha384), and hmac(sha512) shash algorithms use the same format as the padlock-sha and nx-sha512 drivers, as required by Herbert. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers --- crypto/sha512.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/crypto/sha512.c b/crypto/sha512.c index d1e5400fe5906..fb1c520978ef3 100644 --- a/crypto/sha512.c +++ b/crypto/sha512.c @@ -10,10 +10,48 @@ #include #include #include #include =20 +/* + * Export and import functions. crypto_shash wants a particular format th= at + * matches that used by some legacy drivers. It currently is the same as = the + * library SHA context, except the value in bytecount_lo must be block-ali= gned + * and the remainder must be stored in an extra u8 appended to the struct. + */ + +#define SHA512_SHASH_STATE_SIZE 209 +static_assert(offsetof(struct __sha512_ctx, state) =3D=3D 0); +static_assert(offsetof(struct __sha512_ctx, bytecount_lo) =3D=3D 64); +static_assert(offsetof(struct __sha512_ctx, bytecount_hi) =3D=3D 72); +static_assert(offsetof(struct __sha512_ctx, buf) =3D=3D 80); +static_assert(sizeof(struct __sha512_ctx) + 1 =3D=3D SHA512_SHASH_STATE_SI= ZE); + +static int __crypto_sha512_export(const struct __sha512_ctx *ctx0, void *o= ut) +{ + struct __sha512_ctx ctx =3D *ctx0; + unsigned int partial; + u8 *p =3D out; + + partial =3D ctx.bytecount_lo % SHA512_BLOCK_SIZE; + ctx.bytecount_lo -=3D partial; + memcpy(p, &ctx, sizeof(ctx)); + p +=3D sizeof(ctx); + *p =3D partial; + return 0; +} + +static int __crypto_sha512_import(struct __sha512_ctx *ctx, const void *in) +{ + const u8 *p =3D in; + + memcpy(ctx, p, sizeof(*ctx)); + p +=3D sizeof(*ctx); + ctx->bytecount_lo +=3D *p; + return 0; +} + /* SHA-384 */ =20 const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE] =3D { 0x38, 0xb0, 0x60, 0xa7, 0x51, 0xac, 0x96, 0x38, 0x4c, 0xd9, 0x32, 0x7e, 0xb1, 0xb1, 0xe3, 0x6a, @@ -50,10 +88,20 @@ static int crypto_sha384_digest(struct shash_desc *desc, { sha384(data, len, out); return 0; } =20 +static int crypto_sha384_export(struct shash_desc *desc, void *out) +{ + return __crypto_sha512_export(&SHA384_CTX(desc)->ctx, out); +} + +static int crypto_sha384_import(struct shash_desc *desc, const void *in) +{ + return __crypto_sha512_import(&SHA384_CTX(desc)->ctx, in); +} + /* SHA-512 */ =20 const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE] =3D { 0xcf, 0x83, 0xe1, 0x35, 0x7e, 0xef, 0xb8, 0xbd, 0xf1, 0x54, 0x28, 0x50, 0xd6, 0x6d, 0x80, 0x07, @@ -92,10 +140,20 @@ static int crypto_sha512_digest(struct shash_desc *des= c, { sha512(data, len, out); return 0; } =20 +static int crypto_sha512_export(struct shash_desc *desc, void *out) +{ + return __crypto_sha512_export(&SHA512_CTX(desc)->ctx, out); +} + +static int crypto_sha512_import(struct shash_desc *desc, const void *in) +{ + return __crypto_sha512_import(&SHA512_CTX(desc)->ctx, in); +} + /* HMAC-SHA384 */ =20 #define HMAC_SHA384_KEY(tfm) ((struct hmac_sha384_key *)crypto_shash_ctx(t= fm)) #define HMAC_SHA384_CTX(desc) ((struct hmac_sha384_ctx *)shash_desc_ctx(de= sc)) =20 @@ -131,10 +189,23 @@ static int crypto_hmac_sha384_digest(struct shash_des= c *desc, { hmac_sha384(HMAC_SHA384_KEY(desc->tfm), data, len, out); return 0; } =20 +static int crypto_hmac_sha384_export(struct shash_desc *desc, void *out) +{ + return __crypto_sha512_export(&HMAC_SHA384_CTX(desc)->ctx.sha_ctx, out); +} + +static int crypto_hmac_sha384_import(struct shash_desc *desc, const void *= in) +{ + struct hmac_sha384_ctx *ctx =3D HMAC_SHA384_CTX(desc); + + ctx->ctx.ostate =3D HMAC_SHA384_KEY(desc->tfm)->key.ostate; + return __crypto_sha512_import(&ctx->ctx.sha_ctx, in); +} + /* HMAC-SHA512 */ =20 #define HMAC_SHA512_KEY(tfm) ((struct hmac_sha512_key *)crypto_shash_ctx(t= fm)) #define HMAC_SHA512_CTX(desc) ((struct hmac_sha512_ctx *)shash_desc_ctx(de= sc)) =20 @@ -170,10 +241,23 @@ static int crypto_hmac_sha512_digest(struct shash_des= c *desc, { hmac_sha512(HMAC_SHA512_KEY(desc->tfm), data, len, out); return 0; } =20 +static int crypto_hmac_sha512_export(struct shash_desc *desc, void *out) +{ + return __crypto_sha512_export(&HMAC_SHA512_CTX(desc)->ctx.sha_ctx, out); +} + +static int crypto_hmac_sha512_import(struct shash_desc *desc, const void *= in) +{ + struct hmac_sha512_ctx *ctx =3D HMAC_SHA512_CTX(desc); + + ctx->ctx.ostate =3D HMAC_SHA512_KEY(desc->tfm)->key.ostate; + return __crypto_sha512_import(&ctx->ctx.sha_ctx, in); +} + /* Algorithm definitions */ =20 static struct shash_alg algs[] =3D { { .base.cra_name =3D "sha384", @@ -184,11 +268,14 @@ static struct shash_alg algs[] =3D { .digestsize =3D SHA384_DIGEST_SIZE, .init =3D crypto_sha384_init, .update =3D crypto_sha384_update, .final =3D crypto_sha384_final, .digest =3D crypto_sha384_digest, + .export =3D crypto_sha384_export, + .import =3D crypto_sha384_import, .descsize =3D sizeof(struct sha384_ctx), + .statesize =3D SHA512_SHASH_STATE_SIZE, }, { .base.cra_name =3D "sha512", .base.cra_driver_name =3D "sha512-lib", .base.cra_priority =3D 300, @@ -197,11 +284,14 @@ static struct shash_alg algs[] =3D { .digestsize =3D SHA512_DIGEST_SIZE, .init =3D crypto_sha512_init, .update =3D crypto_sha512_update, .final =3D crypto_sha512_final, .digest =3D crypto_sha512_digest, + .export =3D crypto_sha512_export, + .import =3D crypto_sha512_import, .descsize =3D sizeof(struct sha512_ctx), + .statesize =3D SHA512_SHASH_STATE_SIZE, }, { .base.cra_name =3D "hmac(sha384)", .base.cra_driver_name =3D "hmac-sha384-lib", .base.cra_priority =3D 300, @@ -212,11 +302,14 @@ static struct shash_alg algs[] =3D { .setkey =3D crypto_hmac_sha384_setkey, .init =3D crypto_hmac_sha384_init, .update =3D crypto_hmac_sha384_update, .final =3D crypto_hmac_sha384_final, .digest =3D crypto_hmac_sha384_digest, + .export =3D crypto_hmac_sha384_export, + .import =3D crypto_hmac_sha384_import, .descsize =3D sizeof(struct hmac_sha384_ctx), + .statesize =3D SHA512_SHASH_STATE_SIZE, }, { .base.cra_name =3D "hmac(sha512)", .base.cra_driver_name =3D "hmac-sha512-lib", .base.cra_priority =3D 300, @@ -227,11 +320,14 @@ static struct shash_alg algs[] =3D { .setkey =3D crypto_hmac_sha512_setkey, .init =3D crypto_hmac_sha512_init, .update =3D crypto_hmac_sha512_update, .final =3D crypto_hmac_sha512_final, .digest =3D crypto_hmac_sha512_digest, + .export =3D crypto_hmac_sha512_export, + .import =3D crypto_hmac_sha512_import, .descsize =3D sizeof(struct hmac_sha512_ctx), + .statesize =3D SHA512_SHASH_STATE_SIZE, }, }; =20 static int __init crypto_sha512_mod_init(void) { --=20 2.50.0 From nobody Wed Oct 8 08:14:51 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 8CB02292B3E; Mon, 30 Jun 2025 16:05:25 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299525; cv=none; b=sNdzRjuSkBfOvnutMpDULDlzeHmjBZRwekY1xou9u5+iT6CC1hjal30sQEyrTuifCfgka6nvv9iuHjXmUI0uofXliQIqU4B+1hwYJKGDr/oAEiyTnGKjM1cz+Pj+GBs5+YEnGB4Bp/weXVmhFVkkcmYbVxPKYCS0nXAZY+jsbo4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299525; c=relaxed/simple; bh=/omQk0AnfhMvcqecsM2I25KnqR+dVzBwTNmX31ZnD+I=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=nb9yuRWUBZQb/WpKJx8K7UH7t2kmO4JTSwTI9PEa9g8nTPgS3PHwwxHYL1wGkxMGUGDttKobETwEEXWWbDPuiNmFRrtOB1HaRTAYCyBB/7Qrqj4ZYEXYCrRob5LbBQOWx/pihw9Se+itdMUs/5gMhp0I1EUZXx2OvcTIN7cjhJY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=NGd3wu7j; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="NGd3wu7j" Received: by smtp.kernel.org (Postfix) with ESMTPSA id C5FF6C4CEF2; Mon, 30 Jun 2025 16:05:24 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299525; bh=/omQk0AnfhMvcqecsM2I25KnqR+dVzBwTNmX31ZnD+I=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=NGd3wu7j7sKL/um4b0OWMF3aaE1syWgvDA5LSmZu/ZEMkWrHGFx5Flqi2slW0qtbL arqlfH9D5NG5oIEsZF9jmUx72q45G1nToNEKH6CG1qveaSA3P1JZqovBbs6uCVqwG8 OqsoJF4Hy1qvMOuDVU/BDXxNNdMtzYtc1hdNpwwTz+f8hXDYp3DuUl1UXPKANwrmCB Af4f9v7fy29lTX4Jkuy93oiWuCPdd6alLUPYcUtOvQrPfb8r9hyD8M2vNmiuIKurch AUD0EynZj16eH/R2qERsVARGCFnnpQFS9CoblAbmnWfdN8iJTB2MJupQE3JXx4Hld9 9q1aI4pemzf6A== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 07/16] lib/crypto: arm/sha512: Migrate optimized SHA-512 code to library Date: Mon, 30 Jun 2025 09:03:11 -0700 Message-ID: <20250630160320.2888-8-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Instead of exposing the arm-optimized SHA-512 code via arm-specific crypto_shash algorithms, instead just implement the sha512_blocks() library function. This is much simpler, it makes the SHA-512 (and SHA-384) library functions be arm-optimized, and it fixes the longstanding issue where the arm-optimized SHA-512 code was disabled by default. SHA-512 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha512_blocks(), change the type of the nblocks parameter of the assembly functions from int to size_t. The assembly functions actually already treated it as size_t. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers --- arch/arm/configs/exynos_defconfig | 1 - arch/arm/configs/milbeaut_m10v_defconfig | 1 - arch/arm/configs/multi_v7_defconfig | 1 - arch/arm/configs/omap2plus_defconfig | 1 - arch/arm/configs/pxa_defconfig | 1 - arch/arm/crypto/Kconfig | 10 -- arch/arm/crypto/Makefile | 15 --- arch/arm/crypto/sha512-glue.c | 110 ------------------ arch/arm/crypto/sha512-neon-glue.c | 75 ------------ arch/arm/crypto/sha512.h | 3 - lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 14 +++ lib/crypto/arm/.gitignore | 2 + .../crypto =3D> lib/crypto/arm}/sha512-armv4.pl | 0 lib/crypto/arm/sha512.h | 38 ++++++ 15 files changed, 55 insertions(+), 218 deletions(-) delete mode 100644 arch/arm/crypto/sha512-glue.c delete mode 100644 arch/arm/crypto/sha512-neon-glue.c delete mode 100644 arch/arm/crypto/sha512.h create mode 100644 lib/crypto/arm/.gitignore rename {arch/arm/crypto =3D> lib/crypto/arm}/sha512-armv4.pl (100%) create mode 100644 lib/crypto/arm/sha512.h diff --git a/arch/arm/configs/exynos_defconfig b/arch/arm/configs/exynos_de= fconfig index f71af368674cf..d58e300693045 100644 --- a/arch/arm/configs/exynos_defconfig +++ b/arch/arm/configs/exynos_defconfig @@ -362,11 +362,10 @@ CONFIG_CRYPTO_LZ4=3Dm CONFIG_CRYPTO_USER_API_HASH=3Dm CONFIG_CRYPTO_USER_API_SKCIPHER=3Dm CONFIG_CRYPTO_USER_API_RNG=3Dm CONFIG_CRYPTO_USER_API_AEAD=3Dm CONFIG_CRYPTO_SHA1_ARM_NEON=3Dm -CONFIG_CRYPTO_SHA512_ARM=3Dm CONFIG_CRYPTO_AES_ARM_BS=3Dm CONFIG_CRYPTO_CHACHA20_NEON=3Dm CONFIG_CRYPTO_DEV_EXYNOS_RNG=3Dy CONFIG_CRYPTO_DEV_S5P=3Dy CONFIG_DMA_CMA=3Dy diff --git a/arch/arm/configs/milbeaut_m10v_defconfig b/arch/arm/configs/mi= lbeaut_m10v_defconfig index 242e7d5a3f682..8ebf8bd872fe8 100644 --- a/arch/arm/configs/milbeaut_m10v_defconfig +++ b/arch/arm/configs/milbeaut_m10v_defconfig @@ -98,11 +98,10 @@ CONFIG_CRYPTO_SELFTESTS=3Dy CONFIG_CRYPTO_AES=3Dy CONFIG_CRYPTO_SEQIV=3Dm CONFIG_CRYPTO_GHASH_ARM_CE=3Dm CONFIG_CRYPTO_SHA1_ARM_NEON=3Dm CONFIG_CRYPTO_SHA1_ARM_CE=3Dm -CONFIG_CRYPTO_SHA512_ARM=3Dm CONFIG_CRYPTO_AES_ARM=3Dm CONFIG_CRYPTO_AES_ARM_BS=3Dm CONFIG_CRYPTO_AES_ARM_CE=3Dm CONFIG_CRYPTO_CHACHA20_NEON=3Dm # CONFIG_CRYPTO_HW is not set diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v= 7_defconfig index 50c170b4619f7..3fd07e864ca85 100644 --- a/arch/arm/configs/multi_v7_defconfig +++ b/arch/arm/configs/multi_v7_defconfig @@ -1280,11 +1280,10 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=3Dm CONFIG_CRYPTO_USER_API_RNG=3Dm CONFIG_CRYPTO_USER_API_AEAD=3Dm CONFIG_CRYPTO_GHASH_ARM_CE=3Dm CONFIG_CRYPTO_SHA1_ARM_NEON=3Dm CONFIG_CRYPTO_SHA1_ARM_CE=3Dm -CONFIG_CRYPTO_SHA512_ARM=3Dm CONFIG_CRYPTO_AES_ARM=3Dm CONFIG_CRYPTO_AES_ARM_BS=3Dm CONFIG_CRYPTO_AES_ARM_CE=3Dm CONFIG_CRYPTO_CHACHA20_NEON=3Dm CONFIG_CRYPTO_DEV_SUN4I_SS=3Dm diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2p= lus_defconfig index 9f9780c8e62aa..530dfb8338c98 100644 --- a/arch/arm/configs/omap2plus_defconfig +++ b/arch/arm/configs/omap2plus_defconfig @@ -703,11 +703,10 @@ CONFIG_NLS_CODEPAGE_437=3Dy CONFIG_NLS_ISO8859_1=3Dy CONFIG_SECURITY=3Dy CONFIG_CRYPTO_MICHAEL_MIC=3Dy CONFIG_CRYPTO_GHASH_ARM_CE=3Dm CONFIG_CRYPTO_SHA1_ARM_NEON=3Dm -CONFIG_CRYPTO_SHA512_ARM=3Dm CONFIG_CRYPTO_AES_ARM=3Dm CONFIG_CRYPTO_AES_ARM_BS=3Dm CONFIG_CRYPTO_CHACHA20_NEON=3Dm CONFIG_CRYPTO_DEV_OMAP=3Dm CONFIG_CRYPTO_DEV_OMAP_SHAM=3Dm diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig index ff29c5b0e9c93..eaa44574d4a64 100644 --- a/arch/arm/configs/pxa_defconfig +++ b/arch/arm/configs/pxa_defconfig @@ -657,11 +657,10 @@ CONFIG_CRYPTO_WP512=3Dm CONFIG_CRYPTO_ANUBIS=3Dm CONFIG_CRYPTO_XCBC=3Dm CONFIG_CRYPTO_DEFLATE=3Dy CONFIG_CRYPTO_LZO=3Dy CONFIG_CRYPTO_SHA1_ARM=3Dm -CONFIG_CRYPTO_SHA512_ARM=3Dm CONFIG_CRYPTO_AES_ARM=3Dm CONFIG_FONTS=3Dy CONFIG_FONT_8x8=3Dy CONFIG_FONT_8x16=3Dy CONFIG_FONT_6x11=3Dy diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index 7efb9a8596e4e..a18f97f1597cb 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -91,20 +91,10 @@ config CRYPTO_SHA1_ARM_CE help SHA-1 secure hash algorithm (FIPS 180) =20 Architecture: arm using ARMv8 Crypto Extensions =20 -config CRYPTO_SHA512_ARM - tristate "Hash functions: SHA-384 and SHA-512 (NEON)" - select CRYPTO_HASH - depends on !CPU_V7M - help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180) - - Architecture: arm using - - NEON (Advanced SIMD) extensions - config CRYPTO_AES_ARM tristate "Ciphers: AES" select CRYPTO_ALGAPI select CRYPTO_AES help diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index 8479137c6e800..78a4042d8761c 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -5,11 +5,10 @@ =20 obj-$(CONFIG_CRYPTO_AES_ARM) +=3D aes-arm.o obj-$(CONFIG_CRYPTO_AES_ARM_BS) +=3D aes-arm-bs.o obj-$(CONFIG_CRYPTO_SHA1_ARM) +=3D sha1-arm.o obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) +=3D sha1-arm-neon.o -obj-$(CONFIG_CRYPTO_SHA512_ARM) +=3D sha512-arm.o obj-$(CONFIG_CRYPTO_BLAKE2B_NEON) +=3D blake2b-neon.o obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) +=3D nhpoly1305-neon.o obj-$(CONFIG_CRYPTO_CURVE25519_NEON) +=3D curve25519-neon.o =20 obj-$(CONFIG_CRYPTO_AES_ARM_CE) +=3D aes-arm-ce.o @@ -18,25 +17,11 @@ obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) +=3D ghash-arm-ce.o =20 aes-arm-y :=3D aes-cipher-core.o aes-cipher-glue.o aes-arm-bs-y :=3D aes-neonbs-core.o aes-neonbs-glue.o sha1-arm-y :=3D sha1-armv4-large.o sha1_glue.o sha1-arm-neon-y :=3D sha1-armv7-neon.o sha1_neon_glue.o -sha512-arm-neon-$(CONFIG_KERNEL_MODE_NEON) :=3D sha512-neon-glue.o -sha512-arm-y :=3D sha512-core.o sha512-glue.o $(sha512-arm-neon-y) blake2b-neon-y :=3D blake2b-neon-core.o blake2b-neon-glue.o sha1-arm-ce-y :=3D sha1-ce-core.o sha1-ce-glue.o aes-arm-ce-y :=3D aes-ce-core.o aes-ce-glue.o ghash-arm-ce-y :=3D ghash-ce-core.o ghash-ce-glue.o nhpoly1305-neon-y :=3D nh-neon-core.o nhpoly1305-neon-glue.o curve25519-neon-y :=3D curve25519-core.o curve25519-glue.o - -quiet_cmd_perl =3D PERL $@ - cmd_perl =3D $(PERL) $(<) > $(@) - -$(obj)/%-core.S: $(src)/%-armv4.pl - $(call cmd,perl) - -clean-files +=3D sha512-core.S - -aflags-thumb2-$(CONFIG_THUMB2_KERNEL) :=3D -U__thumb2__ -D__thumb2__=3D1 - -AFLAGS_sha512-core.o +=3D $(aflags-thumb2-y) diff --git a/arch/arm/crypto/sha512-glue.c b/arch/arm/crypto/sha512-glue.c deleted file mode 100644 index f8a6480889b1b..0000000000000 --- a/arch/arm/crypto/sha512-glue.c +++ /dev/null @@ -1,110 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * sha512-glue.c - accelerated SHA-384/512 for ARM - * - * Copyright (C) 2015 Linaro Ltd - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "sha512.h" - -MODULE_DESCRIPTION("Accelerated SHA-384/SHA-512 secure hash for ARM"); -MODULE_AUTHOR("Ard Biesheuvel "); -MODULE_LICENSE("GPL v2"); - -MODULE_ALIAS_CRYPTO("sha384"); -MODULE_ALIAS_CRYPTO("sha512"); -MODULE_ALIAS_CRYPTO("sha384-arm"); -MODULE_ALIAS_CRYPTO("sha512-arm"); - -asmlinkage void sha512_block_data_order(struct sha512_state *state, - u8 const *src, int blocks); - -static int sha512_arm_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_base_do_update_blocks(desc, data, len, - sha512_block_data_order); -} - -static int sha512_arm_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha512_base_do_finup(desc, data, len, sha512_block_data_order); - return sha512_base_finish(desc, out); -} - -static struct shash_alg sha512_arm_algs[] =3D { { - .init =3D sha384_base_init, - .update =3D sha512_arm_update, - .finup =3D sha512_arm_finup, - .descsize =3D SHA512_STATE_SIZE, - .digestsize =3D SHA384_DIGEST_SIZE, - .base =3D { - .cra_name =3D "sha384", - .cra_driver_name =3D "sha384-arm", - .cra_priority =3D 250, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize =3D SHA512_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -}, { - .init =3D sha512_base_init, - .update =3D sha512_arm_update, - .finup =3D sha512_arm_finup, - .descsize =3D SHA512_STATE_SIZE, - .digestsize =3D SHA512_DIGEST_SIZE, - .base =3D { - .cra_name =3D "sha512", - .cra_driver_name =3D "sha512-arm", - .cra_priority =3D 250, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize =3D SHA512_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -} }; - -static int __init sha512_arm_mod_init(void) -{ - int err; - - err =3D crypto_register_shashes(sha512_arm_algs, - ARRAY_SIZE(sha512_arm_algs)); - if (err) - return err; - - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && cpu_has_neon()) { - err =3D crypto_register_shashes(sha512_neon_algs, - ARRAY_SIZE(sha512_neon_algs)); - if (err) - goto err_unregister; - } - return 0; - -err_unregister: - crypto_unregister_shashes(sha512_arm_algs, - ARRAY_SIZE(sha512_arm_algs)); - - return err; -} - -static void __exit sha512_arm_mod_fini(void) -{ - crypto_unregister_shashes(sha512_arm_algs, - ARRAY_SIZE(sha512_arm_algs)); - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && cpu_has_neon()) - crypto_unregister_shashes(sha512_neon_algs, - ARRAY_SIZE(sha512_neon_algs)); -} - -module_init(sha512_arm_mod_init); -module_exit(sha512_arm_mod_fini); diff --git a/arch/arm/crypto/sha512-neon-glue.c b/arch/arm/crypto/sha512-ne= on-glue.c deleted file mode 100644 index bd528077fefbf..0000000000000 --- a/arch/arm/crypto/sha512-neon-glue.c +++ /dev/null @@ -1,75 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * sha512-neon-glue.c - accelerated SHA-384/512 for ARM NEON - * - * Copyright (C) 2015 Linaro Ltd - */ - -#include -#include -#include -#include -#include -#include - -#include "sha512.h" - -MODULE_ALIAS_CRYPTO("sha384-neon"); -MODULE_ALIAS_CRYPTO("sha512-neon"); - -asmlinkage void sha512_block_data_order_neon(struct sha512_state *state, - const u8 *src, int blocks); - -static int sha512_neon_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - int remain; - - kernel_neon_begin(); - remain =3D sha512_base_do_update_blocks(desc, data, len, - sha512_block_data_order_neon); - kernel_neon_end(); - return remain; -} - -static int sha512_neon_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - kernel_neon_begin(); - sha512_base_do_finup(desc, data, len, sha512_block_data_order_neon); - kernel_neon_end(); - return sha512_base_finish(desc, out); -} - -struct shash_alg sha512_neon_algs[] =3D { { - .init =3D sha384_base_init, - .update =3D sha512_neon_update, - .finup =3D sha512_neon_finup, - .descsize =3D SHA512_STATE_SIZE, - .digestsize =3D SHA384_DIGEST_SIZE, - .base =3D { - .cra_name =3D "sha384", - .cra_driver_name =3D "sha384-neon", - .cra_priority =3D 300, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize =3D SHA384_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - - } -}, { - .init =3D sha512_base_init, - .update =3D sha512_neon_update, - .finup =3D sha512_neon_finup, - .descsize =3D SHA512_STATE_SIZE, - .digestsize =3D SHA512_DIGEST_SIZE, - .base =3D { - .cra_name =3D "sha512", - .cra_driver_name =3D "sha512-neon", - .cra_priority =3D 300, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize =3D SHA512_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -} }; diff --git a/arch/arm/crypto/sha512.h b/arch/arm/crypto/sha512.h deleted file mode 100644 index eeaee52cda69b..0000000000000 --- a/arch/arm/crypto/sha512.h +++ /dev/null @@ -1,3 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -extern struct shash_alg sha512_neon_algs[2]; diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index d1bee3787eb3c..dac6356ba0aac 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -175,10 +175,11 @@ config CRYPTO_LIB_SHA512 . =20 config CRYPTO_LIB_SHA512_ARCH bool depends on CRYPTO_LIB_SHA512 && !UML + default y if ARM && !CPU_V7M =20 config CRYPTO_LIB_SM3 tristate =20 if !KMSAN # avoid false positives from assembly diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index f6b6f370451ec..67008a1612c6f 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -1,7 +1,12 @@ # SPDX-License-Identifier: GPL-2.0 =20 +aflags-thumb2-$(CONFIG_THUMB2_KERNEL) :=3D -U__thumb2__ -D__thumb2__=3D1 + +quiet_cmd_perlasm =3D PERLASM $@ + cmd_perlasm =3D $(PERL) $(<) > $(@) + obj-$(CONFIG_CRYPTO_LIB_UTILS) +=3D libcryptoutils.o libcryptoutils-y :=3D memneq.o utils.o =20 # chacha is used by the /dev/random driver which is always builtin obj-y +=3D chacha.o @@ -66,10 +71,19 @@ libsha256-generic-y :=3D sha256-generic.o =20 obj-$(CONFIG_CRYPTO_LIB_SHA512) +=3D libsha512.o libsha512-y :=3D sha512.o ifeq ($(CONFIG_CRYPTO_LIB_SHA512_ARCH),y) CFLAGS_sha512.o +=3D -I$(src)/$(SRCARCH) + +ifeq ($(CONFIG_ARM),y) +libsha512-y +=3D arm/sha512-core.o +$(obj)/arm/sha512-core.S: $(src)/arm/sha512-armv4.pl + $(call cmd,perlasm) +clean-files +=3D arm/sha512-core.S +AFLAGS_arm/sha512-core.o +=3D $(aflags-thumb2-y) +endif + endif # CONFIG_CRYPTO_LIB_SHA512_ARCH =20 obj-$(CONFIG_MPILIB) +=3D mpi/ =20 obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) +=3D simd.o diff --git a/lib/crypto/arm/.gitignore b/lib/crypto/arm/.gitignore new file mode 100644 index 0000000000000..670a4d97b5684 --- /dev/null +++ b/lib/crypto/arm/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +sha512-core.S diff --git a/arch/arm/crypto/sha512-armv4.pl b/lib/crypto/arm/sha512-armv4.= pl similarity index 100% rename from arch/arm/crypto/sha512-armv4.pl rename to lib/crypto/arm/sha512-armv4.pl diff --git a/lib/crypto/arm/sha512.h b/lib/crypto/arm/sha512.h new file mode 100644 index 0000000000000..f147b6490d6cd --- /dev/null +++ b/lib/crypto/arm/sha512.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * arm32-optimized SHA-512 block function + * + * Copyright 2025 Google LLC + */ + +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); + +asmlinkage void sha512_block_data_order(struct sha512_block_state *state, + const u8 *data, size_t nblocks); +asmlinkage void sha512_block_data_order_neon(struct sha512_block_state *st= ate, + const u8 *data, size_t nblocks); + +static void sha512_blocks(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && + static_branch_likely(&have_neon) && likely(crypto_simd_usable())) { + kernel_neon_begin(); + sha512_block_data_order_neon(state, data, nblocks); + kernel_neon_end(); + } else { + sha512_block_data_order(state, data, nblocks); + } +} + +#ifdef CONFIG_KERNEL_MODE_NEON +#define sha512_mod_init_arch sha512_mod_init_arch +static inline void sha512_mod_init_arch(void) +{ + if (cpu_has_neon()) + static_branch_enable(&have_neon); +} +#endif /* CONFIG_KERNEL_MODE_NEON */ --=20 2.50.0 From nobody Wed Oct 8 08:14:51 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id F3DC4292B41; Mon, 30 Jun 2025 16:05:25 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299526; cv=none; b=cycICQwzkmk5KWMpAbSRJxmB83+1MLobERKpFu78/dte7hDnGNxNhyiYU9yN3ukEo/RDpmX/boqDl0YkV0+I+hboCv2RNeOtFXpCsG7dT2IO0dzuyEwBpmhNl41QKikUzSpzt0qOJR+7TBJwSgJ6QZngOS3A1sOQKBqf2nQQXfY= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299526; c=relaxed/simple; bh=ErEAgVMHNorw18U5pjaEJsohh5MEwMTo7uVgc9XOnpg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=B6LHZYV49t5jnqLGj1cut3j9Ura7H27St+p/2/ekB4WYt0Q1RGTH7O8OjP3ZuizPh7tSayc/58Sn132J/wuWoCSaYp1KQxmvOIX1WbKYww55km/UewUNapMr87KY7mg66AEj0ipzq5RwBycwgNMiEqIz/Hj87BUA3N7fXeQ2Wbg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=sPwA+9N/; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="sPwA+9N/" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 44CB1C4CEFB; Mon, 30 Jun 2025 16:05:25 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299525; bh=ErEAgVMHNorw18U5pjaEJsohh5MEwMTo7uVgc9XOnpg=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=sPwA+9N/PmZU/Vofk/2p3oPfabiSsCi43lD+tpzvnViI7FGuAct966GSovR56bj7v rIygd0nXEyabxAdfm7IBpN+A1mZHlUJDA4rAW2r6J7B2dF0bDHoGZuqeCgy2BAzwGH hkX9QtxPmR78J0O9QJAPb63D53v1CMIgixfJX7QNrEjJGNuVnI9akPZr3NI33feIw3 3rwLaJMmhwpJ3Rk0U9TAPMdA+DPhg6Tzqz6gpscv9ubfTC4ar9WpK5RXhb1+GROQ8e DvNwBfUuRJvUeZGw7So6MbbkWl1hU9yz2fq4mbJKKZ4eAWsaD5/7metybDNtJvDlJy chgfgo21R2Rrg== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 08/16] lib/crypto: arm64/sha512: Migrate optimized SHA-512 code to library Date: Mon, 30 Jun 2025 09:03:12 -0700 Message-ID: <20250630160320.2888-9-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Instead of exposing the arm64-optimized SHA-512 code via arm64-specific crypto_shash algorithms, instead just implement the sha512_blocks() library function. This is much simpler, it makes the SHA-512 (and SHA-384) library functions be arm64-optimized, and it fixes the longstanding issue where the arm64-optimized SHA-512 code was disabled by default. SHA-512 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha512_blocks(), change the type of the nblocks parameter of the assembly functions from int or 'unsigned int' to size_t. Update the ARMv8 CE assembly function accordingly. The scalar assembly function actually already treated it as size_t. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers --- arch/arm64/configs/defconfig | 1 - arch/arm64/crypto/Kconfig | 19 ---- arch/arm64/crypto/Makefile | 14 --- arch/arm64/crypto/sha512-ce-glue.c | 96 ------------------- arch/arm64/crypto/sha512-glue.c | 83 ---------------- lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 10 ++ lib/crypto/arm64/.gitignore | 2 + .../crypto/arm64}/sha512-ce-core.S | 10 +- lib/crypto/arm64/sha512.h | 46 +++++++++ 10 files changed, 64 insertions(+), 218 deletions(-) delete mode 100644 arch/arm64/crypto/sha512-ce-glue.c delete mode 100644 arch/arm64/crypto/sha512-glue.c create mode 100644 lib/crypto/arm64/.gitignore rename {arch/arm64/crypto =3D> lib/crypto/arm64}/sha512-ce-core.S (97%) create mode 100644 lib/crypto/arm64/sha512.h diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 897fc686e6a91..b612b78b3b091 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1742,11 +1742,10 @@ CONFIG_CRYPTO_ECHAINIV=3Dy CONFIG_CRYPTO_MICHAEL_MIC=3Dm CONFIG_CRYPTO_ANSI_CPRNG=3Dy CONFIG_CRYPTO_USER_API_RNG=3Dm CONFIG_CRYPTO_GHASH_ARM64_CE=3Dy CONFIG_CRYPTO_SHA1_ARM64_CE=3Dy -CONFIG_CRYPTO_SHA512_ARM64_CE=3Dm CONFIG_CRYPTO_SHA3_ARM64=3Dm CONFIG_CRYPTO_SM3_ARM64_CE=3Dm CONFIG_CRYPTO_AES_ARM64_CE_BLK=3Dy CONFIG_CRYPTO_AES_ARM64_BS=3Dm CONFIG_CRYPTO_AES_ARM64_CE_CCM=3Dy diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index c44b0f202a1f5..a9ead99f72c28 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -34,29 +34,10 @@ config CRYPTO_SHA1_ARM64_CE SHA-1 secure hash algorithm (FIPS 180) =20 Architecture: arm64 using: - ARMv8 Crypto Extensions =20 -config CRYPTO_SHA512_ARM64 - tristate "Hash functions: SHA-384 and SHA-512" - select CRYPTO_HASH - help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180) - - Architecture: arm64 - -config CRYPTO_SHA512_ARM64_CE - tristate "Hash functions: SHA-384 and SHA-512 (ARMv8 Crypto Extensions)" - depends on KERNEL_MODE_NEON - select CRYPTO_HASH - select CRYPTO_SHA512_ARM64 - help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180) - - Architecture: arm64 using: - - ARMv8 Crypto Extensions - config CRYPTO_SHA3_ARM64 tristate "Hash functions: SHA-3 (ARMv8.2 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_SHA3 diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index c231c980c5142..228101f125d50 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -6,13 +6,10 @@ # =20 obj-$(CONFIG_CRYPTO_SHA1_ARM64_CE) +=3D sha1-ce.o sha1-ce-y :=3D sha1-ce-glue.o sha1-ce-core.o =20 -obj-$(CONFIG_CRYPTO_SHA512_ARM64_CE) +=3D sha512-ce.o -sha512-ce-y :=3D sha512-ce-glue.o sha512-ce-core.o - obj-$(CONFIG_CRYPTO_SHA3_ARM64) +=3D sha3-ce.o sha3-ce-y :=3D sha3-ce-glue.o sha3-ce-core.o =20 obj-$(CONFIG_CRYPTO_SM3_NEON) +=3D sm3-neon.o sm3-neon-y :=3D sm3-neon-glue.o sm3-neon-core.o @@ -51,24 +48,13 @@ obj-$(CONFIG_CRYPTO_AES_ARM64_CE_BLK) +=3D aes-ce-blk.o aes-ce-blk-y :=3D aes-glue-ce.o aes-ce.o =20 obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) +=3D aes-neon-blk.o aes-neon-blk-y :=3D aes-glue-neon.o aes-neon.o =20 -obj-$(CONFIG_CRYPTO_SHA512_ARM64) +=3D sha512-arm64.o -sha512-arm64-y :=3D sha512-glue.o sha512-core.o - obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) +=3D nhpoly1305-neon.o nhpoly1305-neon-y :=3D nh-neon-core.o nhpoly1305-neon-glue.o =20 obj-$(CONFIG_CRYPTO_AES_ARM64) +=3D aes-arm64.o aes-arm64-y :=3D aes-cipher-core.o aes-cipher-glue.o =20 obj-$(CONFIG_CRYPTO_AES_ARM64_BS) +=3D aes-neon-bs.o aes-neon-bs-y :=3D aes-neonbs-core.o aes-neonbs-glue.o - -quiet_cmd_perlasm =3D PERLASM $@ - cmd_perlasm =3D $(PERL) $(<) void $(@) - -$(obj)/sha512-core.S: $(src)/../lib/crypto/sha2-armv8.pl - $(call cmd,perlasm) - -clean-files +=3D sha512-core.S diff --git a/arch/arm64/crypto/sha512-ce-glue.c b/arch/arm64/crypto/sha512-= ce-glue.c deleted file mode 100644 index 6fb3001fa2c9b..0000000000000 --- a/arch/arm64/crypto/sha512-ce-glue.c +++ /dev/null @@ -1,96 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * sha512-ce-glue.c - SHA-384/SHA-512 using ARMv8 Crypto Extensions - * - * Copyright (C) 2018 Linaro Ltd - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#include -#include -#include -#include -#include -#include -#include - -MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash using ARMv8 Crypto Extensi= ons"); -MODULE_AUTHOR("Ard Biesheuvel "); -MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("sha384"); -MODULE_ALIAS_CRYPTO("sha512"); - -asmlinkage int __sha512_ce_transform(struct sha512_state *sst, u8 const *s= rc, - int blocks); - -static void sha512_ce_transform(struct sha512_state *sst, u8 const *src, - int blocks) -{ - do { - int rem; - - kernel_neon_begin(); - rem =3D __sha512_ce_transform(sst, src, blocks); - kernel_neon_end(); - src +=3D (blocks - rem) * SHA512_BLOCK_SIZE; - blocks =3D rem; - } while (blocks); -} - -static int sha512_ce_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_base_do_update_blocks(desc, data, len, - sha512_ce_transform); -} - -static int sha512_ce_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha512_base_do_finup(desc, data, len, sha512_ce_transform); - return sha512_base_finish(desc, out); -} - -static struct shash_alg algs[] =3D { { - .init =3D sha384_base_init, - .update =3D sha512_ce_update, - .finup =3D sha512_ce_finup, - .descsize =3D SHA512_STATE_SIZE, - .digestsize =3D SHA384_DIGEST_SIZE, - .base.cra_name =3D "sha384", - .base.cra_driver_name =3D "sha384-ce", - .base.cra_priority =3D 200, - .base.cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize =3D SHA512_BLOCK_SIZE, - .base.cra_module =3D THIS_MODULE, -}, { - .init =3D sha512_base_init, - .update =3D sha512_ce_update, - .finup =3D sha512_ce_finup, - .descsize =3D SHA512_STATE_SIZE, - .digestsize =3D SHA512_DIGEST_SIZE, - .base.cra_name =3D "sha512", - .base.cra_driver_name =3D "sha512-ce", - .base.cra_priority =3D 200, - .base.cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize =3D SHA512_BLOCK_SIZE, - .base.cra_module =3D THIS_MODULE, -} }; - -static int __init sha512_ce_mod_init(void) -{ - return crypto_register_shashes(algs, ARRAY_SIZE(algs)); -} - -static void __exit sha512_ce_mod_fini(void) -{ - crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); -} - -module_cpu_feature_match(SHA512, sha512_ce_mod_init); -module_exit(sha512_ce_mod_fini); diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glu= e.c deleted file mode 100644 index a78e184c100fa..0000000000000 --- a/arch/arm64/crypto/sha512-glue.c +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Linux/arm64 port of the OpenSSL SHA512 implementation for AArch64 - * - * Copyright (c) 2016 Linaro Ltd. - */ - -#include -#include -#include -#include -#include - -MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash for arm64"); -MODULE_AUTHOR("Andy Polyakov "); -MODULE_AUTHOR("Ard Biesheuvel "); -MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("sha384"); -MODULE_ALIAS_CRYPTO("sha512"); - -asmlinkage void sha512_blocks_arch(u64 *digest, const void *data, - unsigned int num_blks); - -static void sha512_arm64_transform(struct sha512_state *sst, u8 const *src, - int blocks) -{ - sha512_blocks_arch(sst->state, src, blocks); -} - -static int sha512_update_arm64(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_base_do_update_blocks(desc, data, len, - sha512_arm64_transform); -} - -static int sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha512_base_do_finup(desc, data, len, sha512_arm64_transform); - return sha512_base_finish(desc, out); -} - -static struct shash_alg algs[] =3D { { - .digestsize =3D SHA512_DIGEST_SIZE, - .init =3D sha512_base_init, - .update =3D sha512_update_arm64, - .finup =3D sha512_finup, - .descsize =3D SHA512_STATE_SIZE, - .base.cra_name =3D "sha512", - .base.cra_driver_name =3D "sha512-arm64", - .base.cra_priority =3D 150, - .base.cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize =3D SHA512_BLOCK_SIZE, - .base.cra_module =3D THIS_MODULE, -}, { - .digestsize =3D SHA384_DIGEST_SIZE, - .init =3D sha384_base_init, - .update =3D sha512_update_arm64, - .finup =3D sha512_finup, - .descsize =3D SHA512_STATE_SIZE, - .base.cra_name =3D "sha384", - .base.cra_driver_name =3D "sha384-arm64", - .base.cra_priority =3D 150, - .base.cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize =3D SHA384_BLOCK_SIZE, - .base.cra_module =3D THIS_MODULE, -} }; - -static int __init sha512_mod_init(void) -{ - return crypto_register_shashes(algs, ARRAY_SIZE(algs)); -} - -static void __exit sha512_mod_fini(void) -{ - crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); -} - -module_init(sha512_mod_init); -module_exit(sha512_mod_fini); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index dac6356ba0aac..26413f679fab2 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -176,10 +176,11 @@ config CRYPTO_LIB_SHA512 =20 config CRYPTO_LIB_SHA512_ARCH bool depends on CRYPTO_LIB_SHA512 && !UML default y if ARM && !CPU_V7M + default y if ARM64 =20 config CRYPTO_LIB_SM3 tristate =20 if !KMSAN # avoid false positives from assembly diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 67008a1612c6f..22269ab06d700 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -3,10 +3,13 @@ aflags-thumb2-$(CONFIG_THUMB2_KERNEL) :=3D -U__thumb2__ -D__thumb2__=3D1 =20 quiet_cmd_perlasm =3D PERLASM $@ cmd_perlasm =3D $(PERL) $(<) > $(@) =20 +quiet_cmd_perlasm_with_args =3D PERLASM $@ + cmd_perlasm_with_args =3D $(PERL) $(<) void $(@) + obj-$(CONFIG_CRYPTO_LIB_UTILS) +=3D libcryptoutils.o libcryptoutils-y :=3D memneq.o utils.o =20 # chacha is used by the /dev/random driver which is always builtin obj-y +=3D chacha.o @@ -80,10 +83,17 @@ $(obj)/arm/sha512-core.S: $(src)/arm/sha512-armv4.pl $(call cmd,perlasm) clean-files +=3D arm/sha512-core.S AFLAGS_arm/sha512-core.o +=3D $(aflags-thumb2-y) endif =20 +ifeq ($(CONFIG_ARM64),y) +libsha512-y +=3D arm64/sha512-core.o +$(obj)/arm64/sha512-core.S: $(src)/../../arch/arm64/lib/crypto/sha2-armv8.= pl + $(call cmd,perlasm_with_args) +clean-files +=3D arm64/sha512-core.S +libsha512-$(CONFIG_KERNEL_MODE_NEON) +=3D arm64/sha512-ce-core.o +endif endif # CONFIG_CRYPTO_LIB_SHA512_ARCH =20 obj-$(CONFIG_MPILIB) +=3D mpi/ =20 obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) +=3D simd.o diff --git a/lib/crypto/arm64/.gitignore b/lib/crypto/arm64/.gitignore new file mode 100644 index 0000000000000..670a4d97b5684 --- /dev/null +++ b/lib/crypto/arm64/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +sha512-core.S diff --git a/arch/arm64/crypto/sha512-ce-core.S b/lib/crypto/arm64/sha512-c= e-core.S similarity index 97% rename from arch/arm64/crypto/sha512-ce-core.S rename to lib/crypto/arm64/sha512-ce-core.S index 91ef68b15fcc6..7d870a435ea38 100644 --- a/arch/arm64/crypto/sha512-ce-core.S +++ b/lib/crypto/arm64/sha512-ce-core.S @@ -100,12 +100,12 @@ add v\i4\().2d, v\i1\().2d, v\i3\().2d sha512h2 q\i3, q\i1, v\i0\().2d .endm =20 /* - * int __sha512_ce_transform(struct sha512_state *sst, u8 const *src, - * int blocks) + * size_t __sha512_ce_transform(struct sha512_block_state *state, + * const u8 *data, size_t nblocks); */ .text SYM_FUNC_START(__sha512_ce_transform) /* load state */ ld1 {v8.2d-v11.2d}, [x0] @@ -115,11 +115,11 @@ SYM_FUNC_START(__sha512_ce_transform) ld1 {v20.2d-v23.2d}, [x3], #64 =20 /* load input */ 0: ld1 {v12.2d-v15.2d}, [x1], #64 ld1 {v16.2d-v19.2d}, [x1], #64 - sub w2, w2, #1 + sub x2, x2, #1 =20 CPU_LE( rev64 v12.16b, v12.16b ) CPU_LE( rev64 v13.16b, v13.16b ) CPU_LE( rev64 v14.16b, v14.16b ) CPU_LE( rev64 v15.16b, v15.16b ) @@ -195,12 +195,12 @@ CPU_LE( rev64 v19.16b, v19.16b ) add v10.2d, v10.2d, v2.2d add v11.2d, v11.2d, v3.2d =20 cond_yield 3f, x4, x5 /* handled all input blocks? */ - cbnz w2, 0b + cbnz x2, 0b =20 /* store new state */ 3: st1 {v8.2d-v11.2d}, [x0] - mov w0, w2 + mov x0, x2 ret SYM_FUNC_END(__sha512_ce_transform) diff --git a/lib/crypto/arm64/sha512.h b/lib/crypto/arm64/sha512.h new file mode 100644 index 0000000000000..eae14f9752e0b --- /dev/null +++ b/lib/crypto/arm64/sha512.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * arm64-optimized SHA-512 block function + * + * Copyright 2025 Google LLC + */ + +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns); + +asmlinkage void sha512_blocks_arch(struct sha512_block_state *state, + const u8 *data, size_t nblocks); +asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state, + const u8 *data, size_t nblocks); + +static void sha512_blocks(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && + static_branch_likely(&have_sha512_insns) && + likely(crypto_simd_usable())) { + do { + size_t rem; + + kernel_neon_begin(); + rem =3D __sha512_ce_transform(state, data, nblocks); + kernel_neon_end(); + data +=3D (nblocks - rem) * SHA512_BLOCK_SIZE; + nblocks =3D rem; + } while (nblocks); + } else { + sha512_blocks_arch(state, data, nblocks); + } +} + +#ifdef CONFIG_KERNEL_MODE_NEON +#define sha512_mod_init_arch sha512_mod_init_arch +static inline void sha512_mod_init_arch(void) +{ + if (cpu_have_named_feature(SHA512)) + static_branch_enable(&have_sha512_insns); +} +#endif /* CONFIG_KERNEL_MODE_NEON */ --=20 2.50.0 From nobody Wed Oct 8 08:14:51 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 786E8293B53; Mon, 30 Jun 2025 16:05:26 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299526; cv=none; b=Ud2g2uKOtEyVp9VTDBHMu19jtcTmiXA/fm6fhXAzyEQlW4z1ih6Lg157J/d8PJBSNBXvI342G+9AjNizSAfk5INnkAkSKVieamlnNb1UOR7QW0c+p6ZOZLbM/gwP/LmwWX+nvswFwgN37T2RwaxziSqcPfrLY/xMKe7Kfp6uBmU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299526; c=relaxed/simple; bh=JG4DTgucZUrWoEuYJPtjg0ziG3DV7kKAWZi/WwQtPT8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=XGatXojMEtPDrWNVw7ZawJO4bIFUX947RMPJtAwRnb4zkArWckDWImx0+Uy04HiRt27eBoo0PLEMcVGa79xBLTqNAhHsKuce20RfGyy0PlQTYg/4YOxad1yvUfJdsRfoQsGC1thgia+Cuo3lUOKIxskwU1EH3tYSk8waZ3Zs0M0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=ApnEO8WU; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="ApnEO8WU" Received: by smtp.kernel.org (Postfix) with ESMTPSA id B72D0C4CEF5; Mon, 30 Jun 2025 16:05:25 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299526; bh=JG4DTgucZUrWoEuYJPtjg0ziG3DV7kKAWZi/WwQtPT8=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=ApnEO8WUpJ+HKPFHeRZUHExWJBFUYzvVnT6VdNOePMw9WGwx8hz6EmDTiW70uLQ1D JVwjyKEx1AEGxEvedB7Hsm1TGLFnWP+Iz1kWVUW13AlOPPasV+0VIpNabEw5Ztz996 gtgEP6gkKn8Lf+T7iqB71BKDIReiN/L0P5pUiEkmApW0eQ/JybpRt3W3xIAWXsjj8I +DZuchFl4lzxIpizrG+v3v3higRcxF2I1ZGnPKWcatTcoMhuC6kA4jMZ8jLlFtjV52 vQePNePrSq1BSGwtEjjE7ygZUS1d1mJMEWQcscWCIIH5OcSga0/DyKYR5gH6Yazmws FJ1WA8zdeA8nw== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 09/16] mips: cavium-octeon: Move octeon-crypto.h into asm directory Date: Mon, 30 Jun 2025 09:03:13 -0700 Message-ID: <20250630160320.2888-10-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Since arch/mips/cavium-octeon/crypto/octeon-crypto.h is now needed outside of its directory, move it to arch/mips/include/asm/octeon/crypto.h so that it can be included as . Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers --- arch/mips/cavium-octeon/crypto/octeon-crypto.c | 3 +-- arch/mips/cavium-octeon/crypto/octeon-md5.c | 3 +-- arch/mips/cavium-octeon/crypto/octeon-sha1.c | 3 +-- arch/mips/cavium-octeon/crypto/octeon-sha256.c | 3 +-- arch/mips/cavium-octeon/crypto/octeon-sha512.c | 3 +-- .../crypto/octeon-crypto.h =3D> include/asm/octeon/crypto.h} | 0 6 files changed, 5 insertions(+), 10 deletions(-) rename arch/mips/{cavium-octeon/crypto/octeon-crypto.h =3D> include/asm/oc= teon/crypto.h} (100%) diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.c b/arch/mips/cav= ium-octeon/crypto/octeon-crypto.c index cfb4a146cf178..0ff8559391f5b 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-crypto.c +++ b/arch/mips/cavium-octeon/crypto/octeon-crypto.c @@ -5,16 +5,15 @@ * * Copyright (C) 2004-2012 Cavium Networks */ =20 #include +#include #include #include #include =20 -#include "octeon-crypto.h" - /** * Enable access to Octeon's COP2 crypto hardware for kernel use. Wrap any * crypto operations in calls to octeon_crypto_enable/disable in order to = make * sure the state of COP2 isn't corrupted if userspace is also performing * hardware crypto operations. Allocate the state parameter on the stack. diff --git a/arch/mips/cavium-octeon/crypto/octeon-md5.c b/arch/mips/cavium= -octeon/crypto/octeon-md5.c index fbc84eb7fedf5..a8ce831e2cebd 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-md5.c +++ b/arch/mips/cavium-octeon/crypto/octeon-md5.c @@ -17,20 +17,19 @@ * under the terms of the GNU General Public License as published by the F= ree * Software Foundation; either version 2 of the License, or (at your optio= n) * any later version. */ =20 +#include #include #include #include #include #include #include #include =20 -#include "octeon-crypto.h" - struct octeon_md5_state { __le32 hash[MD5_HASH_WORDS]; u64 byte_count; }; =20 diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha1.c b/arch/mips/caviu= m-octeon/crypto/octeon-sha1.c index e70f21a473daf..e4a369a7764fb 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha1.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha1.c @@ -11,20 +11,19 @@ * Copyright (c) Alan Smithee. * Copyright (c) Andrew McDonald * Copyright (c) Jean-Francois Dive */ =20 +#include #include #include #include #include #include #include #include =20 -#include "octeon-crypto.h" - /* * We pass everything as 64-bit. OCTEON can handle misaligned data. */ =20 static void octeon_sha1_store_hash(struct sha1_state *sctx) diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cav= ium-octeon/crypto/octeon-sha256.c index f93faaf1f4af6..c20038239cb6b 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c @@ -10,17 +10,16 @@ * Copyright (c) Andrew McDonald * Copyright (c) 2002 James Morris * SHA224 Support Copyright 2007 Intel Corporation */ =20 +#include #include #include #include #include =20 -#include "octeon-crypto.h" - /* * We pass everything as 64-bit. OCTEON can handle misaligned data. */ =20 void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha512.c b/arch/mips/cav= ium-octeon/crypto/octeon-sha512.c index 215311053db3c..53de74f642db0 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha512.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha512.c @@ -11,19 +11,18 @@ * Copyright (c) Jean-Luc Cooke * Copyright (c) Andrew McDonald * Copyright (c) 2003 Kyle McMartin */ =20 +#include #include #include #include #include #include #include =20 -#include "octeon-crypto.h" - /* * We pass everything as 64-bit. OCTEON can handle misaligned data. */ =20 static void octeon_sha512_store_hash(struct sha512_state *sctx) diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.h b/arch/mips/inc= lude/asm/octeon/crypto.h similarity index 100% rename from arch/mips/cavium-octeon/crypto/octeon-crypto.h rename to arch/mips/include/asm/octeon/crypto.h --=20 2.50.0 From nobody Wed Oct 8 08:14:51 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id EA9E9293C6E; Mon, 30 Jun 2025 16:05:26 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299527; cv=none; b=hvsgP+3DBgBQOwWlUfRIWAAN1bzMAcCAKKh9u2npBfPHsUzkKOXVRkAXZpU2HRP9Lv6Lj7SPzkD6RBNX4N5ZMpfOxYDnRC0MgsNCNz9d5BwLgbv25GewASchT1l+efrT45QKMHvQfHb4AJFDzI3bpESUhTQIg7sKLuqOyEHK3X8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299527; c=relaxed/simple; bh=X4Q3jMJYTozhR3OhKfdtes+x8nVkSNX4ADLbk36WeBU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=LcVx6eE9Yv+ff8QaaKjleLhZs0vXya2eVR8WuqQ73ndJWhyBSbvQzR18WoPufXyHs/oOmawrKQQlq7V6cdjP/rEGWVdSRwwiox1C0BkbaxLdh3rSalfDB6vdev6lRrvJxLy+kwu6kLlISPqW517DV2GZ0g907loypWRdB39myeE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=dYiYAok4; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="dYiYAok4" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 35AEBC4CEF1; Mon, 30 Jun 2025 16:05:26 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299526; bh=X4Q3jMJYTozhR3OhKfdtes+x8nVkSNX4ADLbk36WeBU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=dYiYAok4mmLaetpLGKE+oZw1I0/fkjYiA/j49nYQFuOzTzq/jJpWRoVb1o/FTyew1 6sL+6zQWgmh/b9Y+Wy+ykcZYpn3th0dDdV2VkvEUunYlP3HHstk17V67zpdtS5mtkX 7den1CZqICJDRSqBAuq+q0jFQzB/ig3niN+n4ijj/VeK+xqbeh4aKvHmdz0EWsAh47 AgYxV2PvrQqY6RAH8itrWXXy55TAwpvLW9lAIMUosyitb6y/MXXN/wzkxd3wjgrbUO IX3P2YfU19lkDVrC6CJ9KbQJntwJhzvNojYvBKK2N7KI0hB0gBbFKyIWGicPSkj1Qu HOWI8rNld1nPQ== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 10/16] lib/crypto: mips/sha512: Migrate optimized SHA-512 code to library Date: Mon, 30 Jun 2025 09:03:14 -0700 Message-ID: <20250630160320.2888-11-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Instead of exposing the mips-optimized SHA-512 code via mips-specific crypto_shash algorithms, instead just implement the sha512_blocks() library function. This is much simpler, it makes the SHA-512 (and SHA-384) library functions be mips-optimized, and it fixes the longstanding issue where the mips-optimized SHA-512 code was disabled by default. SHA-512 still remains available through crypto_shash, but individual architectures no longer need to handle it. Note: to see the diff from arch/mips/cavium-octeon/crypto/octeon-sha512.c to lib/crypto/mips/sha512.h, view this commit with 'git show -M10'. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers --- arch/mips/cavium-octeon/crypto/Makefile | 1 - .../mips/cavium-octeon/crypto/octeon-sha512.c | 166 ------------------ arch/mips/configs/cavium_octeon_defconfig | 1 - arch/mips/crypto/Kconfig | 10 -- lib/crypto/Kconfig | 1 + lib/crypto/mips/sha512.h | 74 ++++++++ 6 files changed, 75 insertions(+), 178 deletions(-) delete mode 100644 arch/mips/cavium-octeon/crypto/octeon-sha512.c create mode 100644 lib/crypto/mips/sha512.h diff --git a/arch/mips/cavium-octeon/crypto/Makefile b/arch/mips/cavium-oct= eon/crypto/Makefile index db26c73fa0eda..168b19ef7ce89 100644 --- a/arch/mips/cavium-octeon/crypto/Makefile +++ b/arch/mips/cavium-octeon/crypto/Makefile @@ -6,6 +6,5 @@ obj-y +=3D octeon-crypto.o =20 obj-$(CONFIG_CRYPTO_MD5_OCTEON) +=3D octeon-md5.o obj-$(CONFIG_CRYPTO_SHA1_OCTEON) +=3D octeon-sha1.o obj-$(CONFIG_CRYPTO_SHA256_OCTEON) +=3D octeon-sha256.o -obj-$(CONFIG_CRYPTO_SHA512_OCTEON) +=3D octeon-sha512.o diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha512.c b/arch/mips/cav= ium-octeon/crypto/octeon-sha512.c deleted file mode 100644 index 53de74f642db0..0000000000000 --- a/arch/mips/cavium-octeon/crypto/octeon-sha512.c +++ /dev/null @@ -1,166 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * SHA-512 and SHA-384 Secure Hash Algorithm. - * - * Adapted for OCTEON by Aaro Koskinen . - * - * Based on crypto/sha512_generic.c, which is: - * - * Copyright (c) Jean-Luc Cooke - * Copyright (c) Andrew McDonald - * Copyright (c) 2003 Kyle McMartin - */ - -#include -#include -#include -#include -#include -#include -#include - -/* - * We pass everything as 64-bit. OCTEON can handle misaligned data. - */ - -static void octeon_sha512_store_hash(struct sha512_state *sctx) -{ - write_octeon_64bit_hash_sha512(sctx->state[0], 0); - write_octeon_64bit_hash_sha512(sctx->state[1], 1); - write_octeon_64bit_hash_sha512(sctx->state[2], 2); - write_octeon_64bit_hash_sha512(sctx->state[3], 3); - write_octeon_64bit_hash_sha512(sctx->state[4], 4); - write_octeon_64bit_hash_sha512(sctx->state[5], 5); - write_octeon_64bit_hash_sha512(sctx->state[6], 6); - write_octeon_64bit_hash_sha512(sctx->state[7], 7); -} - -static void octeon_sha512_read_hash(struct sha512_state *sctx) -{ - sctx->state[0] =3D read_octeon_64bit_hash_sha512(0); - sctx->state[1] =3D read_octeon_64bit_hash_sha512(1); - sctx->state[2] =3D read_octeon_64bit_hash_sha512(2); - sctx->state[3] =3D read_octeon_64bit_hash_sha512(3); - sctx->state[4] =3D read_octeon_64bit_hash_sha512(4); - sctx->state[5] =3D read_octeon_64bit_hash_sha512(5); - sctx->state[6] =3D read_octeon_64bit_hash_sha512(6); - sctx->state[7] =3D read_octeon_64bit_hash_sha512(7); -} - -static void octeon_sha512_transform(struct sha512_state *sctx, - const u8 *src, int blocks) -{ - do { - const u64 *block =3D (const u64 *)src; - - write_octeon_64bit_block_sha512(block[0], 0); - write_octeon_64bit_block_sha512(block[1], 1); - write_octeon_64bit_block_sha512(block[2], 2); - write_octeon_64bit_block_sha512(block[3], 3); - write_octeon_64bit_block_sha512(block[4], 4); - write_octeon_64bit_block_sha512(block[5], 5); - write_octeon_64bit_block_sha512(block[6], 6); - write_octeon_64bit_block_sha512(block[7], 7); - write_octeon_64bit_block_sha512(block[8], 8); - write_octeon_64bit_block_sha512(block[9], 9); - write_octeon_64bit_block_sha512(block[10], 10); - write_octeon_64bit_block_sha512(block[11], 11); - write_octeon_64bit_block_sha512(block[12], 12); - write_octeon_64bit_block_sha512(block[13], 13); - write_octeon_64bit_block_sha512(block[14], 14); - octeon_sha512_start(block[15]); - - src +=3D SHA512_BLOCK_SIZE; - } while (--blocks); -} - -static int octeon_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct sha512_state *sctx =3D shash_desc_ctx(desc); - struct octeon_cop2_state state; - unsigned long flags; - int remain; - - flags =3D octeon_crypto_enable(&state); - octeon_sha512_store_hash(sctx); - - remain =3D sha512_base_do_update_blocks(desc, data, len, - octeon_sha512_transform); - - octeon_sha512_read_hash(sctx); - octeon_crypto_disable(&state, flags); - return remain; -} - -static int octeon_sha512_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *hash) -{ - struct sha512_state *sctx =3D shash_desc_ctx(desc); - struct octeon_cop2_state state; - unsigned long flags; - - flags =3D octeon_crypto_enable(&state); - octeon_sha512_store_hash(sctx); - - sha512_base_do_finup(desc, src, len, octeon_sha512_transform); - - octeon_sha512_read_hash(sctx); - octeon_crypto_disable(&state, flags); - return sha512_base_finish(desc, hash); -} - -static struct shash_alg octeon_sha512_algs[2] =3D { { - .digestsize =3D SHA512_DIGEST_SIZE, - .init =3D sha512_base_init, - .update =3D octeon_sha512_update, - .finup =3D octeon_sha512_finup, - .descsize =3D SHA512_STATE_SIZE, - .base =3D { - .cra_name =3D "sha512", - .cra_driver_name=3D "octeon-sha512", - .cra_priority =3D OCTEON_CR_OPCODE_PRIORITY, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize =3D SHA512_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -}, { - .digestsize =3D SHA384_DIGEST_SIZE, - .init =3D sha384_base_init, - .update =3D octeon_sha512_update, - .finup =3D octeon_sha512_finup, - .descsize =3D SHA512_STATE_SIZE, - .base =3D { - .cra_name =3D "sha384", - .cra_driver_name=3D "octeon-sha384", - .cra_priority =3D OCTEON_CR_OPCODE_PRIORITY, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize =3D SHA384_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -} }; - -static int __init octeon_sha512_mod_init(void) -{ - if (!octeon_has_crypto()) - return -ENOTSUPP; - return crypto_register_shashes(octeon_sha512_algs, - ARRAY_SIZE(octeon_sha512_algs)); -} - -static void __exit octeon_sha512_mod_fini(void) -{ - crypto_unregister_shashes(octeon_sha512_algs, - ARRAY_SIZE(octeon_sha512_algs)); -} - -module_init(octeon_sha512_mod_init); -module_exit(octeon_sha512_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-512 and SHA-384 Secure Hash Algorithms (OCTEON)"); -MODULE_AUTHOR("Aaro Koskinen "); diff --git a/arch/mips/configs/cavium_octeon_defconfig b/arch/mips/configs/= cavium_octeon_defconfig index 88ae0aa85364b..effdfb2bb738b 100644 --- a/arch/mips/configs/cavium_octeon_defconfig +++ b/arch/mips/configs/cavium_octeon_defconfig @@ -155,11 +155,10 @@ CONFIG_SECURITY=3Dy CONFIG_SECURITY_NETWORK=3Dy CONFIG_CRYPTO_CBC=3Dy CONFIG_CRYPTO_HMAC=3Dy CONFIG_CRYPTO_MD5_OCTEON=3Dy CONFIG_CRYPTO_SHA1_OCTEON=3Dm -CONFIG_CRYPTO_SHA512_OCTEON=3Dm CONFIG_CRYPTO_DES=3Dy CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=3Dy CONFIG_DEBUG_FS=3Dy CONFIG_MAGIC_SYSRQ=3Dy # CONFIG_SCHED_DEBUG is not set diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig index 6bf073ae7613f..51a76a5ee3b16 100644 --- a/arch/mips/crypto/Kconfig +++ b/arch/mips/crypto/Kconfig @@ -20,16 +20,6 @@ config CRYPTO_SHA1_OCTEON help SHA-1 secure hash algorithm (FIPS 180) =20 Architecture: mips OCTEON =20 -config CRYPTO_SHA512_OCTEON - tristate "Hash functions: SHA-384 and SHA-512 (OCTEON)" - depends on CPU_CAVIUM_OCTEON - select CRYPTO_SHA512 - select CRYPTO_HASH - help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180) - - Architecture: mips OCTEON using crypto instructions, when available - endmenu diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 26413f679fab2..303ea15e3e900 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -177,10 +177,11 @@ config CRYPTO_LIB_SHA512 config CRYPTO_LIB_SHA512_ARCH bool depends on CRYPTO_LIB_SHA512 && !UML default y if ARM && !CPU_V7M default y if ARM64 + default y if MIPS && CPU_CAVIUM_OCTEON =20 config CRYPTO_LIB_SM3 tristate =20 if !KMSAN # avoid false positives from assembly diff --git a/lib/crypto/mips/sha512.h b/lib/crypto/mips/sha512.h new file mode 100644 index 0000000000000..b3ffbc1e8ca8e --- /dev/null +++ b/lib/crypto/mips/sha512.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Cryptographic API. + * + * SHA-512 and SHA-384 Secure Hash Algorithm. + * + * Adapted for OCTEON by Aaro Koskinen . + * + * Based on crypto/sha512_generic.c, which is: + * + * Copyright (c) Jean-Luc Cooke + * Copyright (c) Andrew McDonald + * Copyright (c) 2003 Kyle McMartin + */ + +#include +#include + +/* + * We pass everything as 64-bit. OCTEON can handle misaligned data. + */ + +static void sha512_blocks(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + struct octeon_cop2_state cop2_state; + unsigned long flags; + + if (!octeon_has_crypto()) + return sha512_blocks_generic(state, data, nblocks); + + flags =3D octeon_crypto_enable(&cop2_state); + write_octeon_64bit_hash_sha512(state->h[0], 0); + write_octeon_64bit_hash_sha512(state->h[1], 1); + write_octeon_64bit_hash_sha512(state->h[2], 2); + write_octeon_64bit_hash_sha512(state->h[3], 3); + write_octeon_64bit_hash_sha512(state->h[4], 4); + write_octeon_64bit_hash_sha512(state->h[5], 5); + write_octeon_64bit_hash_sha512(state->h[6], 6); + write_octeon_64bit_hash_sha512(state->h[7], 7); + + do { + const u64 *block =3D (const u64 *)data; + + write_octeon_64bit_block_sha512(block[0], 0); + write_octeon_64bit_block_sha512(block[1], 1); + write_octeon_64bit_block_sha512(block[2], 2); + write_octeon_64bit_block_sha512(block[3], 3); + write_octeon_64bit_block_sha512(block[4], 4); + write_octeon_64bit_block_sha512(block[5], 5); + write_octeon_64bit_block_sha512(block[6], 6); + write_octeon_64bit_block_sha512(block[7], 7); + write_octeon_64bit_block_sha512(block[8], 8); + write_octeon_64bit_block_sha512(block[9], 9); + write_octeon_64bit_block_sha512(block[10], 10); + write_octeon_64bit_block_sha512(block[11], 11); + write_octeon_64bit_block_sha512(block[12], 12); + write_octeon_64bit_block_sha512(block[13], 13); + write_octeon_64bit_block_sha512(block[14], 14); + octeon_sha512_start(block[15]); + + data +=3D SHA512_BLOCK_SIZE; + } while (--nblocks); + + state->h[0] =3D read_octeon_64bit_hash_sha512(0); + state->h[1] =3D read_octeon_64bit_hash_sha512(1); + state->h[2] =3D read_octeon_64bit_hash_sha512(2); + state->h[3] =3D read_octeon_64bit_hash_sha512(3); + state->h[4] =3D read_octeon_64bit_hash_sha512(4); + state->h[5] =3D read_octeon_64bit_hash_sha512(5); + state->h[6] =3D read_octeon_64bit_hash_sha512(6); + state->h[7] =3D read_octeon_64bit_hash_sha512(7); + octeon_crypto_disable(&cop2_state, flags); +} --=20 2.50.0 From nobody Wed Oct 8 08:14:51 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 20A9F294A14; Mon, 30 Jun 2025 16:05:27 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299527; cv=none; b=qi8caylQauYlCao7N2anb0lwv0zn2/b6TrELeCKyV8pGKgtpq9Va/LwUJcSoWLFoZGw789Pokq5hMPNFy0VqPiJn6VIsM+K3wuJmQQg6sU0oaND3SnpcmxNOhj3S5owBGhcztjNFda8qORgfOQtohgbCvBZnjaXcjn3XxQJSI9o= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299527; c=relaxed/simple; bh=sOr7n7osKoPo0iWtqMHhPhtDizNmc02j97iKsfhVDgI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=UFFV/AvSCNABzLvi3qFuQ/utIDonuLR2ziPSVFBDdKYZg2HJAhQ83BPu0yyuXUSg4S7hicFrXTt1SuFdXtaDlpqQ/1DRH3kNIN//2wF4cT7h1u7r76NBoXT6MjvimdKGDI0BmqmidWrJQwfzqF5t2ArCLfnG9rZZjw/0SenBjn8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=mnk5p5kH; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="mnk5p5kH" Received: by smtp.kernel.org (Postfix) with ESMTPSA id A7BB2C4AF09; Mon, 30 Jun 2025 16:05:26 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299527; bh=sOr7n7osKoPo0iWtqMHhPhtDizNmc02j97iKsfhVDgI=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=mnk5p5kHUp5XLzKIHdu9Xj6bGlhyzDSJSu0/mBqEEQhM3pcKMwOLzG1xvojCgM41R yeoKZUT8c6Cc/5eRXRs6VKuJhz37JPnhbR3KCcCZq36K6muCR15e0MO2UOGh+riYIu xtGcdMjmh+BhJKvOatGq58kCZ6Xkh2/2tQ3qn2todXp0KVQEoxwfxwtcJpx8i2DTNc vzBzPSz+agDzgQ9iQHG2j1gHbvXVsa649XkGvv8XsOY8s7rUXKiNYZomenlidkHvSN mPNS9IzZk5C+0EkCsQY5LKSXwER4Y8MdhGqvPxv/8UUmV8x/4q+g6ezeYlnZPtRk1G FSL+fDwuFL6Sg== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 11/16] lib/crypto: riscv/sha512: Migrate optimized SHA-512 code to library Date: Mon, 30 Jun 2025 09:03:15 -0700 Message-ID: <20250630160320.2888-12-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Instead of exposing the riscv-optimized SHA-512 code via riscv-specific crypto_shash algorithms, instead just implement the sha512_blocks() library function. This is much simpler, it makes the SHA-512 (and SHA-384) library functions be riscv-optimized, and it fixes the longstanding issue where the riscv-optimized SHA-512 code was disabled by default. SHA-512 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha512_blocks(), change the type of the nblocks parameter of the assembly function from int to size_t. The assembly function actually already treated it as size_t. Note: to see the diff from arch/riscv/crypto/sha512-riscv64-glue.c to lib/crypto/riscv/sha512.h, view this commit with 'git show -M10'. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers --- arch/riscv/crypto/Kconfig | 12 -- arch/riscv/crypto/Makefile | 3 - arch/riscv/crypto/sha512-riscv64-glue.c | 130 ------------------ lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 2 + .../riscv}/sha512-riscv64-zvknhb-zvkb.S | 4 +- lib/crypto/riscv/sha512.h | 41 ++++++ 7 files changed, 46 insertions(+), 147 deletions(-) delete mode 100644 arch/riscv/crypto/sha512-riscv64-glue.c rename {arch/riscv/crypto =3D> lib/crypto/riscv}/sha512-riscv64-zvknhb-zvk= b.S (98%) create mode 100644 lib/crypto/riscv/sha512.h diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index 53e4e1eacf554..a75d6325607b4 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -26,22 +26,10 @@ config CRYPTO_GHASH_RISCV64 GCM GHASH function (NIST SP 800-38D) =20 Architecture: riscv64 using: - Zvkg vector crypto extension =20 -config CRYPTO_SHA512_RISCV64 - tristate "Hash functions: SHA-384 and SHA-512" - depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO - select CRYPTO_LIB_SHA512 - select CRYPTO_SHA512 - help - SHA-384 and SHA-512 secure hash algorithm (FIPS 180) - - Architecture: riscv64 using: - - Zvknhb vector crypto extension - - Zvkb vector crypto extension - config CRYPTO_SM3_RISCV64 tristate "Hash functions: SM3 (ShangMi 3)" depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO select CRYPTO_HASH select CRYPTO_LIB_SM3 diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile index e10e8257734e3..183495a95cc0e 100644 --- a/arch/riscv/crypto/Makefile +++ b/arch/riscv/crypto/Makefile @@ -5,13 +5,10 @@ aes-riscv64-y :=3D aes-riscv64-glue.o aes-riscv64-zvkned.= o \ aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o =20 obj-$(CONFIG_CRYPTO_GHASH_RISCV64) +=3D ghash-riscv64.o ghash-riscv64-y :=3D ghash-riscv64-glue.o ghash-riscv64-zvkg.o =20 -obj-$(CONFIG_CRYPTO_SHA512_RISCV64) +=3D sha512-riscv64.o -sha512-riscv64-y :=3D sha512-riscv64-glue.o sha512-riscv64-zvknhb-zvkb.o - obj-$(CONFIG_CRYPTO_SM3_RISCV64) +=3D sm3-riscv64.o sm3-riscv64-y :=3D sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o =20 obj-$(CONFIG_CRYPTO_SM4_RISCV64) +=3D sm4-riscv64.o sm4-riscv64-y :=3D sm4-riscv64-glue.o sm4-riscv64-zvksed-zvkb.o diff --git a/arch/riscv/crypto/sha512-riscv64-glue.c b/arch/riscv/crypto/sh= a512-riscv64-glue.c deleted file mode 100644 index b3dbc71de07b0..0000000000000 --- a/arch/riscv/crypto/sha512-riscv64-glue.c +++ /dev/null @@ -1,130 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * SHA-512 and SHA-384 using the RISC-V vector crypto extensions - * - * Copyright (C) 2023 VRULL GmbH - * Author: Heiko Stuebner - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih - */ - -#include -#include -#include -#include -#include -#include -#include - -/* - * Note: the asm function only uses the 'state' field of struct sha512_sta= te. - * It is assumed to be the first field. - */ -asmlinkage void sha512_transform_zvknhb_zvkb( - struct sha512_state *state, const u8 *data, int num_blocks); - -static void sha512_block(struct sha512_state *state, const u8 *data, - int num_blocks) -{ - /* - * Ensure struct sha512_state begins directly with the SHA-512 - * 512-bit internal state, as this is what the asm function expects. - */ - BUILD_BUG_ON(offsetof(struct sha512_state, state) !=3D 0); - - if (crypto_simd_usable()) { - kernel_vector_begin(); - sha512_transform_zvknhb_zvkb(state, data, num_blocks); - kernel_vector_end(); - } else { - struct __sha512_ctx ctx =3D {}; - - static_assert(sizeof(ctx.state) =3D=3D sizeof(state->state)); - memcpy(&ctx.state, state->state, sizeof(ctx.state)); - __sha512_update(&ctx, data, - (size_t)num_blocks * SHA512_BLOCK_SIZE); - memcpy(state->state, &ctx.state, sizeof(state->state)); - } -} - -static int riscv64_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_base_do_update_blocks(desc, data, len, sha512_block); -} - -static int riscv64_sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sha512_base_do_finup(desc, data, len, sha512_block); - return sha512_base_finish(desc, out); -} - -static int riscv64_sha512_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha512_base_init(desc) ?: - riscv64_sha512_finup(desc, data, len, out); -} - -static struct shash_alg riscv64_sha512_algs[] =3D { - { - .init =3D sha512_base_init, - .update =3D riscv64_sha512_update, - .finup =3D riscv64_sha512_finup, - .digest =3D riscv64_sha512_digest, - .descsize =3D SHA512_STATE_SIZE, - .digestsize =3D SHA512_DIGEST_SIZE, - .base =3D { - .cra_blocksize =3D SHA512_BLOCK_SIZE, - .cra_priority =3D 300, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_name =3D "sha512", - .cra_driver_name =3D "sha512-riscv64-zvknhb-zvkb", - .cra_module =3D THIS_MODULE, - }, - }, { - .init =3D sha384_base_init, - .update =3D riscv64_sha512_update, - .finup =3D riscv64_sha512_finup, - .descsize =3D SHA512_STATE_SIZE, - .digestsize =3D SHA384_DIGEST_SIZE, - .base =3D { - .cra_blocksize =3D SHA384_BLOCK_SIZE, - .cra_priority =3D 300, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_name =3D "sha384", - .cra_driver_name =3D "sha384-riscv64-zvknhb-zvkb", - .cra_module =3D THIS_MODULE, - }, - }, -}; - -static int __init riscv64_sha512_mod_init(void) -{ - if (riscv_isa_extension_available(NULL, ZVKNHB) && - riscv_isa_extension_available(NULL, ZVKB) && - riscv_vector_vlen() >=3D 128) - return crypto_register_shashes(riscv64_sha512_algs, - ARRAY_SIZE(riscv64_sha512_algs)); - - return -ENODEV; -} - -static void __exit riscv64_sha512_mod_exit(void) -{ - crypto_unregister_shashes(riscv64_sha512_algs, - ARRAY_SIZE(riscv64_sha512_algs)); -} - -module_init(riscv64_sha512_mod_init); -module_exit(riscv64_sha512_mod_exit); - -MODULE_DESCRIPTION("SHA-512 (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner "); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("sha512"); -MODULE_ALIAS_CRYPTO("sha384"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 303ea15e3e900..656f2c08c46cd 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -178,10 +178,11 @@ config CRYPTO_LIB_SHA512_ARCH bool depends on CRYPTO_LIB_SHA512 && !UML default y if ARM && !CPU_V7M default y if ARM64 default y if MIPS && CPU_CAVIUM_OCTEON + default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO =20 config CRYPTO_LIB_SM3 tristate =20 if !KMSAN # avoid false positives from assembly diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 22269ab06d700..bc42464a279b8 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -90,10 +90,12 @@ libsha512-y +=3D arm64/sha512-core.o $(obj)/arm64/sha512-core.S: $(src)/../../arch/arm64/lib/crypto/sha2-armv8.= pl $(call cmd,perlasm_with_args) clean-files +=3D arm64/sha512-core.S libsha512-$(CONFIG_KERNEL_MODE_NEON) +=3D arm64/sha512-ce-core.o endif + +libsha512-$(CONFIG_RISCV) +=3D riscv/sha512-riscv64-zvknhb-zvkb.o endif # CONFIG_CRYPTO_LIB_SHA512_ARCH =20 obj-$(CONFIG_MPILIB) +=3D mpi/ =20 obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) +=3D simd.o diff --git a/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S b/lib/crypto/ri= scv/sha512-riscv64-zvknhb-zvkb.S similarity index 98% rename from arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S rename to lib/crypto/riscv/sha512-riscv64-zvknhb-zvkb.S index 89f4a10d12dd6..b41eebf605462 100644 --- a/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S +++ b/lib/crypto/riscv/sha512-riscv64-zvknhb-zvkb.S @@ -91,12 +91,12 @@ sha512_4rounds \last, W1, W2, W3, W0 sha512_4rounds \last, W2, W3, W0, W1 sha512_4rounds \last, W3, W0, W1, W2 .endm =20 -// void sha512_transform_zvknhb_zvkb(u64 state[8], const u8 *data, -// int num_blocks); +// void sha512_transform_zvknhb_zvkb(struct sha512_block_state *state, +// const u8 *data, size_t nblocks); SYM_FUNC_START(sha512_transform_zvknhb_zvkb) =20 // Setup mask for the vmerge to replace the first word (idx=3D=3D0) in // message scheduling. There are 4 words, so an 8-bit mask suffices. vsetivli zero, 1, e8, m1, ta, ma diff --git a/lib/crypto/riscv/sha512.h b/lib/crypto/riscv/sha512.h new file mode 100644 index 0000000000000..9d0abede322f7 --- /dev/null +++ b/lib/crypto/riscv/sha512.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SHA-512 and SHA-384 using the RISC-V vector crypto extensions + * + * Copyright (C) 2023 VRULL GmbH + * Author: Heiko Stuebner + * + * Copyright (C) 2023 SiFive, Inc. + * Author: Jerry Shih + */ + +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions); + +asmlinkage void sha512_transform_zvknhb_zvkb(struct sha512_block_state *st= ate, + const u8 *data, size_t nblocks); + +static void sha512_blocks(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_extensions) && + likely(crypto_simd_usable())) { + kernel_vector_begin(); + sha512_transform_zvknhb_zvkb(state, data, nblocks); + kernel_vector_end(); + } else { + sha512_blocks_generic(state, data, nblocks); + } +} + +#define sha512_mod_init_arch sha512_mod_init_arch +static inline void sha512_mod_init_arch(void) +{ + if (riscv_isa_extension_available(NULL, ZVKNHB) && + riscv_isa_extension_available(NULL, ZVKB) && + riscv_vector_vlen() >=3D 128) + static_branch_enable(&have_extensions); +} --=20 2.50.0 From nobody Wed Oct 8 08:14:51 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id DCEA22980B7; Mon, 30 Jun 2025 16:05:27 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299528; cv=none; b=mgRuu76rCQbawtI23P2on88lAfG6fOJL3PXQydvN/PSuhUhgLedjKViaHzQfrmfMLWWsgJaBWEsahzaU5qbL+x/ETbe37pxwox2ILlI8kOqpdprUZh2qTCLIuTGUHCNTwYg/xZwYR0mIF2518yfCyJ+JxWqZkAop3Gy0fztGBs0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299528; c=relaxed/simple; bh=Jo/0UhzzhJ/7lZ84MAFhrojZAjufvPuvMimGEBYnelk=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=IwO3ztaI4ujs7JM1Bnc+hKzQFOaAUwZh71y4FflzncfbRzlMSXwWq8J2CQIU2bFHn0g89XNYpNtfWn1I2yYni28YsnNof+kBVz5tgNGt3G42NgF9klDFh1OONWYUHpaXa3m9Nm7a+J1goVftvcPraDLoJINybS/afqxHy20lxd0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=bl+I+Kmf; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="bl+I+Kmf" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 27067C4CEF3; Mon, 30 Jun 2025 16:05:27 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299527; bh=Jo/0UhzzhJ/7lZ84MAFhrojZAjufvPuvMimGEBYnelk=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=bl+I+Kmfn0qy7Q42rhg0UnqBv77Ua4ihiN+nqsEiJbi1xYB0E/MbdocLHCE/jC7Kh TiBAT+qd+pq39ivQ0/R/WirORlwesD/by0NlZWrvEU5VbUim2n07JLweJDQZIVO+EQ sNvvG0NEh7jaunbfvbwXESJalktgUVsU88zjX4DspE+EXdy1mRoHRUAa38CFpAgsw1 yPEgk4rfJVN+IhQUZ+1fJv8y+Dc2ztIuyx9EL5WE3nHP2Q1PnEDYqoRCtXArdwolLP VLIOsFDXBKBpocs7EDPKSiPSFlcO8iw74nzyoPa4h2pJBlwbKGmjUDuosv8AlzhYno bdzmWY5ORjFRg== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 12/16] lib/crypto: s390/sha512: Migrate optimized SHA-512 code to library Date: Mon, 30 Jun 2025 09:03:16 -0700 Message-ID: <20250630160320.2888-13-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Instead of exposing the s390-optimized SHA-512 code via s390-specific crypto_shash algorithms, instead just implement the sha512_blocks() library function. This is much simpler, it makes the SHA-512 (and SHA-384) library functions be s390-optimized, and it fixes the longstanding issue where the s390-optimized SHA-512 code was disabled by default. SHA-512 still remains available through crypto_shash, but individual architectures no longer need to handle it. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers --- arch/s390/configs/debug_defconfig | 1 - arch/s390/configs/defconfig | 1 - arch/s390/crypto/Kconfig | 10 -- arch/s390/crypto/Makefile | 1 - arch/s390/crypto/sha512_s390.c | 151 ------------------------------ lib/crypto/Kconfig | 1 + lib/crypto/s390/sha512.h | 28 ++++++ 7 files changed, 29 insertions(+), 164 deletions(-) delete mode 100644 arch/s390/crypto/sha512_s390.c create mode 100644 lib/crypto/s390/sha512.h diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_de= fconfig index 8ecad727497e1..ef313c30b375c 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -802,11 +802,10 @@ CONFIG_CRYPTO_ZSTD=3Dm CONFIG_CRYPTO_ANSI_CPRNG=3Dm CONFIG_CRYPTO_USER_API_HASH=3Dm CONFIG_CRYPTO_USER_API_SKCIPHER=3Dm CONFIG_CRYPTO_USER_API_RNG=3Dm CONFIG_CRYPTO_USER_API_AEAD=3Dm -CONFIG_CRYPTO_SHA512_S390=3Dm CONFIG_CRYPTO_SHA1_S390=3Dm CONFIG_CRYPTO_SHA3_256_S390=3Dm CONFIG_CRYPTO_SHA3_512_S390=3Dm CONFIG_CRYPTO_GHASH_S390=3Dm CONFIG_CRYPTO_AES_S390=3Dm diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index c13a77765162a..b6fa341bb03b6 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -789,11 +789,10 @@ CONFIG_CRYPTO_ANSI_CPRNG=3Dm CONFIG_CRYPTO_JITTERENTROPY_OSR=3D1 CONFIG_CRYPTO_USER_API_HASH=3Dm CONFIG_CRYPTO_USER_API_SKCIPHER=3Dm CONFIG_CRYPTO_USER_API_RNG=3Dm CONFIG_CRYPTO_USER_API_AEAD=3Dm -CONFIG_CRYPTO_SHA512_S390=3Dm CONFIG_CRYPTO_SHA1_S390=3Dm CONFIG_CRYPTO_SHA3_256_S390=3Dm CONFIG_CRYPTO_SHA3_512_S390=3Dm CONFIG_CRYPTO_GHASH_S390=3Dm CONFIG_CRYPTO_AES_S390=3Dm diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index e2c27588b21a9..4557514fbac35 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -1,19 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 =20 menu "Accelerated Cryptographic Algorithms for CPU (s390)" =20 -config CRYPTO_SHA512_S390 - tristate "Hash functions: SHA-384 and SHA-512" - select CRYPTO_HASH - help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180) - - Architecture: s390 - - It is available as of z10. - config CRYPTO_SHA1_S390 tristate "Hash functions: SHA-1" select CRYPTO_HASH help SHA-1 secure hash algorithm (FIPS 180) diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index 21757d86cd499..473d64c0982af 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -2,11 +2,10 @@ # # Cryptographic API # =20 obj-$(CONFIG_CRYPTO_SHA1_S390) +=3D sha1_s390.o sha_common.o -obj-$(CONFIG_CRYPTO_SHA512_S390) +=3D sha512_s390.o sha_common.o obj-$(CONFIG_CRYPTO_SHA3_256_S390) +=3D sha3_256_s390.o sha_common.o obj-$(CONFIG_CRYPTO_SHA3_512_S390) +=3D sha3_512_s390.o sha_common.o obj-$(CONFIG_CRYPTO_DES_S390) +=3D des_s390.o obj-$(CONFIG_CRYPTO_AES_S390) +=3D aes_s390.o obj-$(CONFIG_CRYPTO_PAES_S390) +=3D paes_s390.o diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c deleted file mode 100644 index e8bb172dbed75..0000000000000 --- a/arch/s390/crypto/sha512_s390.c +++ /dev/null @@ -1,151 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Cryptographic API. - * - * s390 implementation of the SHA512 and SHA38 Secure Hash Algorithm. - * - * Copyright IBM Corp. 2007 - * Author(s): Jan Glauber (jang@de.ibm.com) - */ -#include -#include -#include -#include -#include -#include -#include - -#include "sha.h" - -static int sha512_init_s390(struct shash_desc *desc) -{ - struct s390_sha_ctx *ctx =3D shash_desc_ctx(desc); - - ctx->sha512.state[0] =3D SHA512_H0; - ctx->sha512.state[1] =3D SHA512_H1; - ctx->sha512.state[2] =3D SHA512_H2; - ctx->sha512.state[3] =3D SHA512_H3; - ctx->sha512.state[4] =3D SHA512_H4; - ctx->sha512.state[5] =3D SHA512_H5; - ctx->sha512.state[6] =3D SHA512_H6; - ctx->sha512.state[7] =3D SHA512_H7; - ctx->count =3D 0; - ctx->sha512.count_hi =3D 0; - ctx->func =3D CPACF_KIMD_SHA_512; - - return 0; -} - -static int sha512_export(struct shash_desc *desc, void *out) -{ - struct s390_sha_ctx *sctx =3D shash_desc_ctx(desc); - struct sha512_state *octx =3D out; - - octx->count[0] =3D sctx->count; - octx->count[1] =3D sctx->sha512.count_hi; - memcpy(octx->state, sctx->state, sizeof(octx->state)); - return 0; -} - -static int sha512_import(struct shash_desc *desc, const void *in) -{ - struct s390_sha_ctx *sctx =3D shash_desc_ctx(desc); - const struct sha512_state *ictx =3D in; - - sctx->count =3D ictx->count[0]; - sctx->sha512.count_hi =3D ictx->count[1]; - - memcpy(sctx->state, ictx->state, sizeof(ictx->state)); - sctx->func =3D CPACF_KIMD_SHA_512; - return 0; -} - -static struct shash_alg sha512_alg =3D { - .digestsize =3D SHA512_DIGEST_SIZE, - .init =3D sha512_init_s390, - .update =3D s390_sha_update_blocks, - .finup =3D s390_sha_finup, - .export =3D sha512_export, - .import =3D sha512_import, - .descsize =3D sizeof(struct s390_sha_ctx), - .statesize =3D SHA512_STATE_SIZE, - .base =3D { - .cra_name =3D "sha512", - .cra_driver_name=3D "sha512-s390", - .cra_priority =3D 300, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize =3D SHA512_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -}; - -MODULE_ALIAS_CRYPTO("sha512"); - -static int sha384_init_s390(struct shash_desc *desc) -{ - struct s390_sha_ctx *ctx =3D shash_desc_ctx(desc); - - ctx->sha512.state[0] =3D SHA384_H0; - ctx->sha512.state[1] =3D SHA384_H1; - ctx->sha512.state[2] =3D SHA384_H2; - ctx->sha512.state[3] =3D SHA384_H3; - ctx->sha512.state[4] =3D SHA384_H4; - ctx->sha512.state[5] =3D SHA384_H5; - ctx->sha512.state[6] =3D SHA384_H6; - ctx->sha512.state[7] =3D SHA384_H7; - ctx->count =3D 0; - ctx->sha512.count_hi =3D 0; - ctx->func =3D CPACF_KIMD_SHA_512; - - return 0; -} - -static struct shash_alg sha384_alg =3D { - .digestsize =3D SHA384_DIGEST_SIZE, - .init =3D sha384_init_s390, - .update =3D s390_sha_update_blocks, - .finup =3D s390_sha_finup, - .export =3D sha512_export, - .import =3D sha512_import, - .descsize =3D sizeof(struct s390_sha_ctx), - .statesize =3D SHA512_STATE_SIZE, - .base =3D { - .cra_name =3D "sha384", - .cra_driver_name=3D "sha384-s390", - .cra_priority =3D 300, - .cra_blocksize =3D SHA384_BLOCK_SIZE, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_ctxsize =3D sizeof(struct s390_sha_ctx), - .cra_module =3D THIS_MODULE, - } -}; - -MODULE_ALIAS_CRYPTO("sha384"); - -static int __init init(void) -{ - int ret; - - if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_512)) - return -ENODEV; - if ((ret =3D crypto_register_shash(&sha512_alg)) < 0) - goto out; - if ((ret =3D crypto_register_shash(&sha384_alg)) < 0) - crypto_unregister_shash(&sha512_alg); -out: - return ret; -} - -static void __exit fini(void) -{ - crypto_unregister_shash(&sha512_alg); - crypto_unregister_shash(&sha384_alg); -} - -module_cpu_feature_match(S390_CPU_FEATURE_MSA, init); -module_exit(fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA512 and SHA-384 Secure Hash Algorithm"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 656f2c08c46cd..6f1386b8d79d0 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -179,10 +179,11 @@ config CRYPTO_LIB_SHA512_ARCH depends on CRYPTO_LIB_SHA512 && !UML default y if ARM && !CPU_V7M default y if ARM64 default y if MIPS && CPU_CAVIUM_OCTEON default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO + default y if S390 =20 config CRYPTO_LIB_SM3 tristate =20 if !KMSAN # avoid false positives from assembly diff --git a/lib/crypto/s390/sha512.h b/lib/crypto/s390/sha512.h new file mode 100644 index 0000000000000..24744651550cb --- /dev/null +++ b/lib/crypto/s390/sha512.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SHA-512 optimized using the CP Assist for Cryptographic Functions (CPAC= F) + * + * Copyright 2025 Google LLC + */ +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_sha512); + +static void sha512_blocks(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_cpacf_sha512)) + cpacf_kimd(CPACF_KIMD_SHA_512, state, data, + nblocks * SHA512_BLOCK_SIZE); + else + sha512_blocks_generic(state, data, nblocks); +} + +#define sha512_mod_init_arch sha512_mod_init_arch +static inline void sha512_mod_init_arch(void) +{ + if (cpu_have_feature(S390_CPU_FEATURE_MSA) && + cpacf_query_func(CPACF_KIMD, CPACF_KIMD_SHA_512)) + static_branch_enable(&have_cpacf_sha512); +} --=20 2.50.0 From nobody Wed Oct 8 08:14:51 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1519E29824B; Mon, 30 Jun 2025 16:05:28 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299528; cv=none; b=QjQ0u/6Zyf/w9Uo6BAO93I9eUYamPeRs1jlO/oNGO5lIAh7PTe5/8cqImEm3MfutVE9eTo5hK9U5d8XrvX58WlZqn2dNWRSGvpIKk9TX6ceLpGJYwfC97PWUwRcE1NhFMjg6B+LXFx6SBimfRJVvET3Bp00VkHK/MxgXaFroKOM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299528; c=relaxed/simple; bh=v+iLgskUKUB4ERhDBhwL5wDztdbCghNyooubV00uWUc=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=RTYI/TzOZxUnO+/HW5V8mOPGaxQwlZ1EWokdXM//RgQjjoetiR0p0hB68EtTBvTLBZmddRWVPhSQ5RiMvm7mBAD4/mNRev0L1EPt4xJOGBEW7pTAYfY7noQcV/0Ykp283Noif1RCOOAHf/PpoHtKG/p3DCFkEvISym+tKxazefc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=nQN8GWJh; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="nQN8GWJh" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 99D45C4CEF9; Mon, 30 Jun 2025 16:05:27 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299527; bh=v+iLgskUKUB4ERhDBhwL5wDztdbCghNyooubV00uWUc=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=nQN8GWJh33531j+6WcexvXsn8c2LR1wL9al6DpjnW09B10g874Y4hDXXJpJ6R/Z33 meHQZ0K51w4NnGTMXxEd4iskMv8pn+kFgELqSzitUNrGZvsm+sWmpkEQN/Rm6QHaVn 6KQwsXNpxLqudy3Pfa02JOD3h+lkFM91p1a3WAva+8/WbosPEqJe/ypTLudS9ewPxG c3ootlBjj2p6TxMbysEUyCB6JYKTBbbARDba9mAagm6t75Z80hujPZ7FYjG6fbVTHq VZcboqRF96aLrOWzIVIegkZktD2PyMkHvnhjczXym7S++cnH2SdTKZK3rPWjGDCc3k yu8aySZBjvaAg== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 13/16] lib/crypto: sparc/sha512: Migrate optimized SHA-512 code to library Date: Mon, 30 Jun 2025 09:03:17 -0700 Message-ID: <20250630160320.2888-14-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Instead of exposing the sparc-optimized SHA-512 code via sparc-specific crypto_shash algorithms, instead just implement the sha512_blocks() library function. This is much simpler, it makes the SHA-512 (and SHA-384) library functions be sparc-optimized, and it fixes the longstanding issue where the sparc-optimized SHA-512 code was disabled by default. SHA-512 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha512_blocks(), change the type of the nblocks parameter of the assembly function from int to size_t. The assembly function actually already treated it as size_t. Note: to see the diff from arch/sparc/crypto/sha512_glue.c to lib/crypto/sparc/sha512.h, view this commit with 'git show -M10'. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers --- arch/sparc/crypto/Kconfig | 10 -- arch/sparc/crypto/Makefile | 2 - arch/sparc/crypto/sha512_glue.c | 122 ------------------ lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 1 + lib/crypto/sparc/sha512.h | 42 ++++++ .../crypto =3D> lib/crypto/sparc}/sha512_asm.S | 0 7 files changed, 44 insertions(+), 134 deletions(-) delete mode 100644 arch/sparc/crypto/sha512_glue.c create mode 100644 lib/crypto/sparc/sha512.h rename {arch/sparc/crypto =3D> lib/crypto/sparc}/sha512_asm.S (100%) diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig index a6ba319c42dce..9d8da9aef3a41 100644 --- a/arch/sparc/crypto/Kconfig +++ b/arch/sparc/crypto/Kconfig @@ -34,20 +34,10 @@ config CRYPTO_SHA1_SPARC64 help SHA-1 secure hash algorithm (FIPS 180) =20 Architecture: sparc64 =20 -config CRYPTO_SHA512_SPARC64 - tristate "Hash functions: SHA-384 and SHA-512" - depends on SPARC64 - select CRYPTO_SHA512 - select CRYPTO_HASH - help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180) - - Architecture: sparc64 using crypto instructions, when available - config CRYPTO_AES_SPARC64 tristate "Ciphers: AES, modes: ECB, CBC, CTR" depends on SPARC64 select CRYPTO_SKCIPHER help diff --git a/arch/sparc/crypto/Makefile b/arch/sparc/crypto/Makefile index 701c39edb0d73..99a7e8fd13bc9 100644 --- a/arch/sparc/crypto/Makefile +++ b/arch/sparc/crypto/Makefile @@ -2,19 +2,17 @@ # # Arch-specific CryptoAPI modules. # =20 obj-$(CONFIG_CRYPTO_SHA1_SPARC64) +=3D sha1-sparc64.o -obj-$(CONFIG_CRYPTO_SHA512_SPARC64) +=3D sha512-sparc64.o obj-$(CONFIG_CRYPTO_MD5_SPARC64) +=3D md5-sparc64.o =20 obj-$(CONFIG_CRYPTO_AES_SPARC64) +=3D aes-sparc64.o obj-$(CONFIG_CRYPTO_DES_SPARC64) +=3D des-sparc64.o obj-$(CONFIG_CRYPTO_CAMELLIA_SPARC64) +=3D camellia-sparc64.o =20 sha1-sparc64-y :=3D sha1_asm.o sha1_glue.o -sha512-sparc64-y :=3D sha512_asm.o sha512_glue.o md5-sparc64-y :=3D md5_asm.o md5_glue.o =20 aes-sparc64-y :=3D aes_asm.o aes_glue.o des-sparc64-y :=3D des_asm.o des_glue.o camellia-sparc64-y :=3D camellia_asm.o camellia_glue.o diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glu= e.c deleted file mode 100644 index fb81c3290c8c0..0000000000000 --- a/arch/sparc/crypto/sha512_glue.c +++ /dev/null @@ -1,122 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Glue code for SHA512 hashing optimized for sparc64 crypto opcodes. - * - * This is based largely upon crypto/sha512_generic.c - * - * Copyright (c) Jean-Luc Cooke - * Copyright (c) Andrew McDonald - * Copyright (c) 2003 Kyle McMartin - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include -#include - -asmlinkage void sha512_sparc64_transform(u64 *digest, const char *data, - unsigned int rounds); - -static void sha512_block(struct sha512_state *sctx, const u8 *src, int blo= cks) -{ - sha512_sparc64_transform(sctx->state, src, blocks); -} - -static int sha512_sparc64_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_base_do_update_blocks(desc, data, len, sha512_block); -} - -static int sha512_sparc64_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - sha512_base_do_finup(desc, src, len, sha512_block); - return sha512_base_finish(desc, out); -} - -static struct shash_alg sha512_alg =3D { - .digestsize =3D SHA512_DIGEST_SIZE, - .init =3D sha512_base_init, - .update =3D sha512_sparc64_update, - .finup =3D sha512_sparc64_finup, - .descsize =3D SHA512_STATE_SIZE, - .base =3D { - .cra_name =3D "sha512", - .cra_driver_name=3D "sha512-sparc64", - .cra_priority =3D SPARC_CR_OPCODE_PRIORITY, - .cra_blocksize =3D SHA512_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -}; - -static struct shash_alg sha384_alg =3D { - .digestsize =3D SHA384_DIGEST_SIZE, - .init =3D sha384_base_init, - .update =3D sha512_sparc64_update, - .finup =3D sha512_sparc64_finup, - .descsize =3D SHA512_STATE_SIZE, - .base =3D { - .cra_name =3D "sha384", - .cra_driver_name=3D "sha384-sparc64", - .cra_priority =3D SPARC_CR_OPCODE_PRIORITY, - .cra_blocksize =3D SHA384_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -}; - -static bool __init sparc64_has_sha512_opcode(void) -{ - unsigned long cfr; - - if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) - return false; - - __asm__ __volatile__("rd %%asr26, %0" : "=3Dr" (cfr)); - if (!(cfr & CFR_SHA512)) - return false; - - return true; -} - -static int __init sha512_sparc64_mod_init(void) -{ - if (sparc64_has_sha512_opcode()) { - int ret =3D crypto_register_shash(&sha384_alg); - if (ret < 0) - return ret; - - ret =3D crypto_register_shash(&sha512_alg); - if (ret < 0) { - crypto_unregister_shash(&sha384_alg); - return ret; - } - - pr_info("Using sparc64 sha512 opcode optimized SHA-512/SHA-384 implement= ation\n"); - return 0; - } - pr_info("sparc64 sha512 opcode not available.\n"); - return -ENODEV; -} - -static void __exit sha512_sparc64_mod_fini(void) -{ - crypto_unregister_shash(&sha384_alg); - crypto_unregister_shash(&sha512_alg); -} - -module_init(sha512_sparc64_mod_init); -module_exit(sha512_sparc64_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-384 and SHA-512 Secure Hash Algorithm, sparc64 sha= 512 opcode accelerated"); - -MODULE_ALIAS_CRYPTO("sha384"); -MODULE_ALIAS_CRYPTO("sha512"); - -#include "crop_devid.c" diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 6f1386b8d79d0..c3edb78484f0a 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -180,10 +180,11 @@ config CRYPTO_LIB_SHA512_ARCH default y if ARM && !CPU_V7M default y if ARM64 default y if MIPS && CPU_CAVIUM_OCTEON default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO default y if S390 + default y if SPARC64 =20 config CRYPTO_LIB_SM3 tristate =20 if !KMSAN # avoid false positives from assembly diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index bc42464a279b8..0fb93f2469b3c 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -92,10 +92,11 @@ $(obj)/arm64/sha512-core.S: $(src)/../../arch/arm64/lib= /crypto/sha2-armv8.pl clean-files +=3D arm64/sha512-core.S libsha512-$(CONFIG_KERNEL_MODE_NEON) +=3D arm64/sha512-ce-core.o endif =20 libsha512-$(CONFIG_RISCV) +=3D riscv/sha512-riscv64-zvknhb-zvkb.o +libsha512-$(CONFIG_SPARC) +=3D sparc/sha512_asm.o endif # CONFIG_CRYPTO_LIB_SHA512_ARCH =20 obj-$(CONFIG_MPILIB) +=3D mpi/ =20 obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) +=3D simd.o diff --git a/lib/crypto/sparc/sha512.h b/lib/crypto/sparc/sha512.h new file mode 100644 index 0000000000000..55303ab6b15f7 --- /dev/null +++ b/lib/crypto/sparc/sha512.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * SHA-512 accelerated using the sparc64 sha512 opcodes + * + * Copyright (c) Jean-Luc Cooke + * Copyright (c) Andrew McDonald + * Copyright (c) 2003 Kyle McMartin + */ + +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_opcodes); + +asmlinkage void sha512_sparc64_transform(struct sha512_block_state *state, + const u8 *data, size_t nblocks); + +static void sha512_blocks(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_sha512_opcodes)) + sha512_sparc64_transform(state, data, nblocks); + else + sha512_blocks_generic(state, data, nblocks); +} + +#define sha512_mod_init_arch sha512_mod_init_arch +static inline void sha512_mod_init_arch(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return; + + __asm__ __volatile__("rd %%asr26, %0" : "=3Dr" (cfr)); + if (!(cfr & CFR_SHA512)) + return; + + static_branch_enable(&have_sha512_opcodes); + pr_info("Using sparc64 sha512 opcode optimized SHA-512/SHA-384 implementa= tion\n"); +} diff --git a/arch/sparc/crypto/sha512_asm.S b/lib/crypto/sparc/sha512_asm.S similarity index 100% rename from arch/sparc/crypto/sha512_asm.S rename to lib/crypto/sparc/sha512_asm.S --=20 2.50.0 From nobody Wed Oct 8 08:14:51 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id BBF44299957; Mon, 30 Jun 2025 16:05:28 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299528; cv=none; b=g1rpU0qltdrYdYmpRjB00oM5PYbU9Q50e11MogwhPavkvDUwW8FhvDvshVnVcFMVf4j4CXKjVFxaY+WfazvmQ9yEHpxJ/KdvGri7TtmIeZAlNM/n6jfGc8rFtLEI/Jk154Ljeq42SSUN2vGTbT9NZD6tAE7/qXZNLyrO2ga/UgA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299528; c=relaxed/simple; bh=rSgoYaJG0c19UXiZMKnoCIyNff6j1yNcmTHQlGSjGTY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=ofS0aEOVtj5QjcX+KuSNGw8rul0DhsSbayFeGLCFYDLpmQa+XhcclPrI/VtOXe5Prv20TTG7oEYQTEKcynSomTQd2NRlZlkRKOglJn7nFnDEY5cQyMCWMsHfEzT2YOq3JZPrJDC8MLEJLi2OW6qEsbzpkiFdietwRNIlMDMD5ns= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=niOe2zDW; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="niOe2zDW" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 188E4C4CEF1; Mon, 30 Jun 2025 16:05:28 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299528; bh=rSgoYaJG0c19UXiZMKnoCIyNff6j1yNcmTHQlGSjGTY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=niOe2zDWcL54rqVQiu22V58aWoS7OSERtpshwG++TmB8HqX4KhG1nJ7jkLZn4lsju XlXOOJ8YCHfh4CTe96zmx94U6DlUdSwhyBK395udwjOkfChfrlK2F4S4H2VpS+U0Bn GWq0f8Gaam1El0QT2bIl5xATd7W4jYdIiRIVLDpUNr0Joip1z9c3o5PEmZcFrBhJYp wISFMMEKAj/u0u+pyfLwL4RiihxNbMrYLDNac53oK72YJp0E8fZp80BQtWRzoKYdR0 DocOx4ShwQLStJ33q1V0e3DTKNvtmSUNuPKflTVDPNRkiTjgkLW2BQ31lA0RB9clN7 12a43IPPnZqXQ== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 14/16] lib/crypto: x86/sha512: Migrate optimized SHA-512 code to library Date: Mon, 30 Jun 2025 09:03:18 -0700 Message-ID: <20250630160320.2888-15-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Instead of exposing the x86-optimized SHA-512 code via x86-specific crypto_shash algorithms, instead just implement the sha512_blocks() library function. This is much simpler, it makes the SHA-512 (and SHA-384) library functions be x86-optimized, and it fixes the longstanding issue where the x86-optimized SHA-512 code was disabled by default. SHA-512 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha512_blocks(), change the type of the nblocks parameter of the assembly functions from int to size_t. The assembly functions actually already treated it as size_t. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers --- arch/x86/crypto/Kconfig | 13 - arch/x86/crypto/Makefile | 3 - arch/x86/crypto/sha512_ssse3_glue.c | 322 ------------------ lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 3 + .../crypto/x86}/sha512-avx-asm.S | 9 +- .../crypto/x86}/sha512-avx2-asm.S | 9 +- .../crypto/x86}/sha512-ssse3-asm.S | 10 +- lib/crypto/x86/sha512.h | 54 +++ 9 files changed, 72 insertions(+), 352 deletions(-) delete mode 100644 arch/x86/crypto/sha512_ssse3_glue.c rename {arch/x86/crypto =3D> lib/crypto/x86}/sha512-avx-asm.S (98%) rename {arch/x86/crypto =3D> lib/crypto/x86}/sha512-avx2-asm.S (99%) rename {arch/x86/crypto =3D> lib/crypto/x86}/sha512-ssse3-asm.S (97%) create mode 100644 lib/crypto/x86/sha512.h diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 56cfdc79e2c66..eb641a300154e 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -388,23 +388,10 @@ config CRYPTO_SHA1_SSSE3 - SSSE3 (Supplemental SSE3) - AVX (Advanced Vector Extensions) - AVX2 (Advanced Vector Extensions 2) - SHA-NI (SHA Extensions New Instructions) =20 -config CRYPTO_SHA512_SSSE3 - tristate "Hash functions: SHA-384 and SHA-512 (SSSE3/AVX/AVX2)" - depends on 64BIT - select CRYPTO_SHA512 - select CRYPTO_HASH - help - SHA-384 and SHA-512 secure hash algorithms (FIPS 180) - - Architecture: x86_64 using: - - SSSE3 (Supplemental SSE3) - - AVX (Advanced Vector Extensions) - - AVX2 (Advanced Vector Extensions 2) - config CRYPTO_SM3_AVX_X86_64 tristate "Hash functions: SM3 (AVX)" depends on 64BIT select CRYPTO_HASH select CRYPTO_LIB_SM3 diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index aa289a9e0153b..d31348be83704 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -52,13 +52,10 @@ aesni-intel-$(CONFIG_64BIT) +=3D aes-gcm-avx10-x86_64.o endif =20 obj-$(CONFIG_CRYPTO_SHA1_SSSE3) +=3D sha1-ssse3.o sha1-ssse3-y :=3D sha1_avx2_x86_64_asm.o sha1_ssse3_asm.o sha1_ni_asm.o sh= a1_ssse3_glue.o =20 -obj-$(CONFIG_CRYPTO_SHA512_SSSE3) +=3D sha512-ssse3.o -sha512-ssse3-y :=3D sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o = sha512_ssse3_glue.o - obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) +=3D ghash-clmulni-intel.o ghash-clmulni-intel-y :=3D ghash-clmulni-intel_asm.o ghash-clmulni-intel_g= lue.o =20 obj-$(CONFIG_CRYPTO_POLYVAL_CLMUL_NI) +=3D polyval-clmulni.o polyval-clmulni-y :=3D polyval-clmulni_asm.o polyval-clmulni_glue.o diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_s= sse3_glue.c deleted file mode 100644 index 97744b7d23817..0000000000000 --- a/arch/x86/crypto/sha512_ssse3_glue.c +++ /dev/null @@ -1,322 +0,0 @@ -/* - * Cryptographic API. - * - * Glue code for the SHA512 Secure Hash Algorithm assembler - * implementation using supplemental SSE3 / AVX / AVX2 instructions. - * - * This file is based on sha512_generic.c - * - * Copyright (C) 2013 Intel Corporation - * Author: Tim Chen - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the F= ree - * Software Foundation; either version 2 of the License, or (at your optio= n) - * any later version. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include - -asmlinkage void sha512_transform_ssse3(struct sha512_state *state, - const u8 *data, int blocks); - -static int sha512_update_x86(struct shash_desc *desc, const u8 *data, - unsigned int len, sha512_block_fn *sha512_xform) -{ - int remain; - - /* - * Make sure struct sha512_state begins directly with the SHA512 - * 512-bit internal state, as this is what the asm functions expect. - */ - BUILD_BUG_ON(offsetof(struct sha512_state, state) !=3D 0); - - kernel_fpu_begin(); - remain =3D sha512_base_do_update_blocks(desc, data, len, sha512_xform); - kernel_fpu_end(); - - return remain; -} - -static int sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out, sha512_block_fn *sha512_xform) -{ - kernel_fpu_begin(); - sha512_base_do_finup(desc, data, len, sha512_xform); - kernel_fpu_end(); - - return sha512_base_finish(desc, out); -} - -static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_update_x86(desc, data, len, sha512_transform_ssse3); -} - -static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha512_finup(desc, data, len, out, sha512_transform_ssse3); -} - -static struct shash_alg sha512_ssse3_algs[] =3D { { - .digestsize =3D SHA512_DIGEST_SIZE, - .init =3D sha512_base_init, - .update =3D sha512_ssse3_update, - .finup =3D sha512_ssse3_finup, - .descsize =3D SHA512_STATE_SIZE, - .base =3D { - .cra_name =3D "sha512", - .cra_driver_name =3D "sha512-ssse3", - .cra_priority =3D 150, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize =3D SHA512_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -}, { - .digestsize =3D SHA384_DIGEST_SIZE, - .init =3D sha384_base_init, - .update =3D sha512_ssse3_update, - .finup =3D sha512_ssse3_finup, - .descsize =3D SHA512_STATE_SIZE, - .base =3D { - .cra_name =3D "sha384", - .cra_driver_name =3D "sha384-ssse3", - .cra_priority =3D 150, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize =3D SHA384_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -} }; - -static int register_sha512_ssse3(void) -{ - if (boot_cpu_has(X86_FEATURE_SSSE3)) - return crypto_register_shashes(sha512_ssse3_algs, - ARRAY_SIZE(sha512_ssse3_algs)); - return 0; -} - -static void unregister_sha512_ssse3(void) -{ - if (boot_cpu_has(X86_FEATURE_SSSE3)) - crypto_unregister_shashes(sha512_ssse3_algs, - ARRAY_SIZE(sha512_ssse3_algs)); -} - -asmlinkage void sha512_transform_avx(struct sha512_state *state, - const u8 *data, int blocks); -static bool avx_usable(void) -{ - if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { - if (boot_cpu_has(X86_FEATURE_AVX)) - pr_info("AVX detected but unusable.\n"); - return false; - } - - return true; -} - -static int sha512_avx_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_update_x86(desc, data, len, sha512_transform_avx); -} - -static int sha512_avx_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha512_finup(desc, data, len, out, sha512_transform_avx); -} - -static struct shash_alg sha512_avx_algs[] =3D { { - .digestsize =3D SHA512_DIGEST_SIZE, - .init =3D sha512_base_init, - .update =3D sha512_avx_update, - .finup =3D sha512_avx_finup, - .descsize =3D SHA512_STATE_SIZE, - .base =3D { - .cra_name =3D "sha512", - .cra_driver_name =3D "sha512-avx", - .cra_priority =3D 160, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize =3D SHA512_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -}, { - .digestsize =3D SHA384_DIGEST_SIZE, - .init =3D sha384_base_init, - .update =3D sha512_avx_update, - .finup =3D sha512_avx_finup, - .descsize =3D SHA512_STATE_SIZE, - .base =3D { - .cra_name =3D "sha384", - .cra_driver_name =3D "sha384-avx", - .cra_priority =3D 160, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize =3D SHA384_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -} }; - -static int register_sha512_avx(void) -{ - if (avx_usable()) - return crypto_register_shashes(sha512_avx_algs, - ARRAY_SIZE(sha512_avx_algs)); - return 0; -} - -static void unregister_sha512_avx(void) -{ - if (avx_usable()) - crypto_unregister_shashes(sha512_avx_algs, - ARRAY_SIZE(sha512_avx_algs)); -} - -asmlinkage void sha512_transform_rorx(struct sha512_state *state, - const u8 *data, int blocks); - -static int sha512_avx2_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha512_update_x86(desc, data, len, sha512_transform_rorx); -} - -static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha512_finup(desc, data, len, out, sha512_transform_rorx); -} - -static struct shash_alg sha512_avx2_algs[] =3D { { - .digestsize =3D SHA512_DIGEST_SIZE, - .init =3D sha512_base_init, - .update =3D sha512_avx2_update, - .finup =3D sha512_avx2_finup, - .descsize =3D SHA512_STATE_SIZE, - .base =3D { - .cra_name =3D "sha512", - .cra_driver_name =3D "sha512-avx2", - .cra_priority =3D 170, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize =3D SHA512_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -}, { - .digestsize =3D SHA384_DIGEST_SIZE, - .init =3D sha384_base_init, - .update =3D sha512_avx2_update, - .finup =3D sha512_avx2_finup, - .descsize =3D SHA512_STATE_SIZE, - .base =3D { - .cra_name =3D "sha384", - .cra_driver_name =3D "sha384-avx2", - .cra_priority =3D 170, - .cra_flags =3D CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize =3D SHA384_BLOCK_SIZE, - .cra_module =3D THIS_MODULE, - } -} }; - -static bool avx2_usable(void) -{ - if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_BMI2)) - return true; - - return false; -} - -static int register_sha512_avx2(void) -{ - if (avx2_usable()) - return crypto_register_shashes(sha512_avx2_algs, - ARRAY_SIZE(sha512_avx2_algs)); - return 0; -} -static const struct x86_cpu_id module_cpu_ids[] =3D { - X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), - X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), - X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), - {} -}; -MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); - -static void unregister_sha512_avx2(void) -{ - if (avx2_usable()) - crypto_unregister_shashes(sha512_avx2_algs, - ARRAY_SIZE(sha512_avx2_algs)); -} - -static int __init sha512_ssse3_mod_init(void) -{ - if (!x86_match_cpu(module_cpu_ids)) - return -ENODEV; - - if (register_sha512_ssse3()) - goto fail; - - if (register_sha512_avx()) { - unregister_sha512_ssse3(); - goto fail; - } - - if (register_sha512_avx2()) { - unregister_sha512_avx(); - unregister_sha512_ssse3(); - goto fail; - } - - return 0; -fail: - return -ENODEV; -} - -static void __exit sha512_ssse3_mod_fini(void) -{ - unregister_sha512_avx2(); - unregister_sha512_avx(); - unregister_sha512_ssse3(); -} - -module_init(sha512_ssse3_mod_init); -module_exit(sha512_ssse3_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accele= rated"); - -MODULE_ALIAS_CRYPTO("sha512"); -MODULE_ALIAS_CRYPTO("sha512-ssse3"); -MODULE_ALIAS_CRYPTO("sha512-avx"); -MODULE_ALIAS_CRYPTO("sha512-avx2"); -MODULE_ALIAS_CRYPTO("sha384"); -MODULE_ALIAS_CRYPTO("sha384-ssse3"); -MODULE_ALIAS_CRYPTO("sha384-avx"); -MODULE_ALIAS_CRYPTO("sha384-avx2"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index c3edb78484f0a..dce127a69f131 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -181,10 +181,11 @@ config CRYPTO_LIB_SHA512_ARCH default y if ARM64 default y if MIPS && CPU_CAVIUM_OCTEON default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO default y if S390 default y if SPARC64 + default y if X86_64 =20 config CRYPTO_LIB_SM3 tristate =20 if !KMSAN # avoid false positives from assembly diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 0fb93f2469b3c..aaf445a85384c 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -93,10 +93,13 @@ clean-files +=3D arm64/sha512-core.S libsha512-$(CONFIG_KERNEL_MODE_NEON) +=3D arm64/sha512-ce-core.o endif =20 libsha512-$(CONFIG_RISCV) +=3D riscv/sha512-riscv64-zvknhb-zvkb.o libsha512-$(CONFIG_SPARC) +=3D sparc/sha512_asm.o +libsha512-$(CONFIG_X86) +=3D x86/sha512-ssse3-asm.o \ + x86/sha512-avx-asm.o \ + x86/sha512-avx2-asm.o endif # CONFIG_CRYPTO_LIB_SHA512_ARCH =20 obj-$(CONFIG_MPILIB) +=3D mpi/ =20 obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) +=3D simd.o diff --git a/arch/x86/crypto/sha512-avx-asm.S b/lib/crypto/x86/sha512-avx-a= sm.S similarity index 98% rename from arch/x86/crypto/sha512-avx-asm.S rename to lib/crypto/x86/sha512-avx-asm.S index 5bfce4b045fdf..0b5f69179d624 100644 --- a/arch/x86/crypto/sha512-avx-asm.S +++ b/lib/crypto/x86/sha512-avx-asm.S @@ -46,11 +46,10 @@ # and search for that title. # ######################################################################## =20 #include -#include =20 .text =20 # Virtual Registers # ARG1 @@ -265,18 +264,20 @@ frame_size =3D frame_WK + WK_SIZE add tmp0, h_64 RotateState .endm =20 ######################################################################## -# void sha512_transform_avx(sha512_state *state, const u8 *data, int block= s) +# void sha512_transform_avx(struct sha512_block_state *state, +# const u8 *data, size_t nblocks); # Purpose: Updates the SHA512 digest stored at "state" with the message # stored in "data". # The size of the message pointed to by "data" must be an integer multiple # of SHA512 message blocks. -# "blocks" is the message length in SHA512 blocks +# "nblocks" is the message length in SHA512 blocks ######################################################################## -SYM_TYPED_FUNC_START(sha512_transform_avx) +SYM_FUNC_START(sha512_transform_avx) + test msglen, msglen je .Lnowork =20 # Save GPRs push %rbx diff --git a/arch/x86/crypto/sha512-avx2-asm.S b/lib/crypto/x86/sha512-avx2= -asm.S similarity index 99% rename from arch/x86/crypto/sha512-avx2-asm.S rename to lib/crypto/x86/sha512-avx2-asm.S index 24973f42c43ff..2309c01e316b9 100644 --- a/arch/x86/crypto/sha512-avx2-asm.S +++ b/lib/crypto/x86/sha512-avx2-asm.S @@ -48,11 +48,10 @@ ######################################################################## # This code schedules 1 blocks at a time, with 4 lanes per block ######################################################################## =20 #include -#include =20 .text =20 # Virtual Registers Y_0 =3D %ymm4 @@ -557,18 +556,20 @@ frame_size =3D frame_CTX + CTX_SIZE RotateState =20 .endm =20 ######################################################################## -# void sha512_transform_rorx(sha512_state *state, const u8 *data, int bloc= ks) +# void sha512_transform_rorx(struct sha512_block_state *state, +# const u8 *data, size_t nblocks); # Purpose: Updates the SHA512 digest stored at "state" with the message # stored in "data". # The size of the message pointed to by "data" must be an integer multiple # of SHA512 message blocks. -# "blocks" is the message length in SHA512 blocks +# "nblocks" is the message length in SHA512 blocks ######################################################################## -SYM_TYPED_FUNC_START(sha512_transform_rorx) +SYM_FUNC_START(sha512_transform_rorx) + # Save GPRs push %rbx push %r12 push %r13 push %r14 diff --git a/arch/x86/crypto/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-sss= e3-asm.S similarity index 97% rename from arch/x86/crypto/sha512-ssse3-asm.S rename to lib/crypto/x86/sha512-ssse3-asm.S index 30a2c4777f9d5..12e78142f2e38 100644 --- a/arch/x86/crypto/sha512-ssse3-asm.S +++ b/lib/crypto/x86/sha512-ssse3-asm.S @@ -46,11 +46,10 @@ # and search for that title. # ######################################################################## =20 #include -#include =20 .text =20 # Virtual Registers # ARG1 @@ -264,20 +263,19 @@ frame_size =3D frame_WK + WK_SIZE lea (T1, T2), h_64 RotateState .endm =20 ######################################################################## -## void sha512_transform_ssse3(struct sha512_state *state, const u8 *data, -## int blocks); -# (struct sha512_state is assumed to begin with u64 state[8]) +# void sha512_transform_ssse3(struct sha512_block_state *state, +# const u8 *data, size_t nblocks); # Purpose: Updates the SHA512 digest stored at "state" with the message # stored in "data". # The size of the message pointed to by "data" must be an integer multiple # of SHA512 message blocks. -# "blocks" is the message length in SHA512 blocks. +# "nblocks" is the message length in SHA512 blocks ######################################################################## -SYM_TYPED_FUNC_START(sha512_transform_ssse3) +SYM_FUNC_START(sha512_transform_ssse3) =20 test msglen, msglen je .Lnowork =20 # Save GPRs diff --git a/lib/crypto/x86/sha512.h b/lib/crypto/x86/sha512.h new file mode 100644 index 0000000000000..c13503d9d57d9 --- /dev/null +++ b/lib/crypto/x86/sha512.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * x86-optimized SHA-512 block function + * + * Copyright 2025 Google LLC + */ + +#include +#include +#include + +DEFINE_STATIC_CALL(sha512_blocks_x86, sha512_blocks_generic); + +#define DEFINE_X86_SHA512_FN(c_fn, asm_fn) = \ + asmlinkage void asm_fn(struct sha512_block_state *state, \ + const u8 *data, size_t nblocks); \ + static void c_fn(struct sha512_block_state *state, const u8 *data, \ + size_t nblocks) \ + { \ + if (likely(crypto_simd_usable())) { \ + kernel_fpu_begin(); \ + asm_fn(state, data, nblocks); \ + kernel_fpu_end(); \ + } else { \ + sha512_blocks_generic(state, data, nblocks); \ + } \ + } + +DEFINE_X86_SHA512_FN(sha512_blocks_ssse3, sha512_transform_ssse3); +DEFINE_X86_SHA512_FN(sha512_blocks_avx, sha512_transform_avx); +DEFINE_X86_SHA512_FN(sha512_blocks_avx2, sha512_transform_rorx); + +static void sha512_blocks(struct sha512_block_state *state, + const u8 *data, size_t nblocks) +{ + static_call(sha512_blocks_x86)(state, data, nblocks); +} + +#define sha512_mod_init_arch sha512_mod_init_arch +static inline void sha512_mod_init_arch(void) +{ + if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL) && + boot_cpu_has(X86_FEATURE_AVX)) { + if (boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_BMI2)) + static_call_update(sha512_blocks_x86, + sha512_blocks_avx2); + else + static_call_update(sha512_blocks_x86, + sha512_blocks_avx); + } else if (boot_cpu_has(X86_FEATURE_SSSE3)) { + static_call_update(sha512_blocks_x86, sha512_blocks_ssse3); + } +} --=20 2.50.0 From nobody Wed Oct 8 08:14:51 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4EEF929C32B; Mon, 30 Jun 2025 16:05:29 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299529; cv=none; b=VQJDeB+VSuYt1JFAwOhMrLhwY5hCNdFNNGVTBbYeE9HL5wBYvCV694WRkEMSwBXoTkdVAkKFtjx5JISKRHyglB2H7SxJKswpO9IdUZ+jXSBu9xyxAhXVOQfK5wc2AKWuyJMWjQ3UM7uF7K6qJLzKMszoA9HvoLpXMBGn9a4dBCc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299529; c=relaxed/simple; bh=x0Z9wiotmf8jNMw5UZ8kqkasukLKKyE27l+fQpQiAtU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=GFFvxuuGKV04Wbc56Bd7SIOTUTrh0tC2EcT0XLPOKEgMLjdybN5oXDpghcWh6Lyoqfta8EtfanxS3jw5+dRgdHgNXt6eJmjbxQULRb9m6XWrjuQoTvQH8ZD/z+4J0Wydx4JI8JSI54AYNnmuqtsVfqPWnPRcK4OQEX2fpftXZqA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=t3JvUmlQ; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="t3JvUmlQ" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 8B986C4CEF3; Mon, 30 Jun 2025 16:05:28 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299528; bh=x0Z9wiotmf8jNMw5UZ8kqkasukLKKyE27l+fQpQiAtU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=t3JvUmlQQ4INyHMwXd8NDAuXQfblNIw7bb2A+AkbC1bEfjMvbGr9VVWYBufCEWja+ vRPcUj3Eh3HA5CCYjFNEPIvrkd5V+fCYMMRfLjgfRMBAOIDzx501JkRTOw24STASVN GlXEfFZ4QhPwvdzhL7Brq7NcN40QmKVHP3lciFUL9Eh22O7hkrRSsErLEcExvfFcRk RtTIKQjSwaglVhNVylmaJW3TUINxj06OACBE4anK4ueu1Kh2nnmHyNEfuaK4GIntCE GlPmSy5LDlWkdSrGWFPNlZVad3KFf1R78b5H9wRTMsFkiIVpiIi4UYjq+LAS4uKKW9 oLn9LSYQOwH8Q== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 15/16] lib/crypto: x86/sha512: Remove unnecessary checks for nblocks==0 Date: Mon, 30 Jun 2025 09:03:19 -0700 Message-ID: <20250630160320.2888-16-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Since sha512_blocks() is called only with nblocks >=3D 1, remove unnecessary checks for nblocks =3D=3D 0 from the x86 SHA-512 assembly code. Signed-off-by: Eric Biggers --- lib/crypto/x86/sha512-avx-asm.S | 6 +----- lib/crypto/x86/sha512-avx2-asm.S | 5 +---- lib/crypto/x86/sha512-ssse3-asm.S | 6 +----- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/lib/crypto/x86/sha512-avx-asm.S b/lib/crypto/x86/sha512-avx-as= m.S index 0b5f69179d624..7732aa8fd8506 100644 --- a/lib/crypto/x86/sha512-avx-asm.S +++ b/lib/crypto/x86/sha512-avx-asm.S @@ -270,17 +270,14 @@ frame_size =3D frame_WK + WK_SIZE # const u8 *data, size_t nblocks); # Purpose: Updates the SHA512 digest stored at "state" with the message # stored in "data". # The size of the message pointed to by "data" must be an integer multiple # of SHA512 message blocks. -# "nblocks" is the message length in SHA512 blocks +# "nblocks" is the message length in SHA512 blocks. Must be >=3D 1. ######################################################################## SYM_FUNC_START(sha512_transform_avx) =20 - test msglen, msglen - je .Lnowork - # Save GPRs push %rbx push %r12 push %r13 push %r14 @@ -360,11 +357,10 @@ SYM_FUNC_START(sha512_transform_avx) pop %r14 pop %r13 pop %r12 pop %rbx =20 -.Lnowork: RET SYM_FUNC_END(sha512_transform_avx) =20 ######################################################################## ### Binary Data diff --git a/lib/crypto/x86/sha512-avx2-asm.S b/lib/crypto/x86/sha512-avx2-= asm.S index 2309c01e316b9..22bdbfd899d0f 100644 --- a/lib/crypto/x86/sha512-avx2-asm.S +++ b/lib/crypto/x86/sha512-avx2-asm.S @@ -562,11 +562,11 @@ frame_size =3D frame_CTX + CTX_SIZE # const u8 *data, size_t nblocks); # Purpose: Updates the SHA512 digest stored at "state" with the message # stored in "data". # The size of the message pointed to by "data" must be an integer multiple # of SHA512 message blocks. -# "nblocks" is the message length in SHA512 blocks +# "nblocks" is the message length in SHA512 blocks. Must be >=3D 1. ######################################################################## SYM_FUNC_START(sha512_transform_rorx) =20 # Save GPRs push %rbx @@ -580,11 +580,10 @@ SYM_FUNC_START(sha512_transform_rorx) mov %rsp, %rbp sub $frame_size, %rsp and $~(0x20 - 1), %rsp =20 shl $7, NUM_BLKS # convert to bytes - jz .Ldone_hash add INP, NUM_BLKS # pointer to end of data mov NUM_BLKS, frame_INPEND(%rsp) =20 ## load initial digest mov 8*0(CTX1), a @@ -666,12 +665,10 @@ SYM_FUNC_START(sha512_transform_rorx) mov frame_INP(%rsp), INP add $128, INP cmp frame_INPEND(%rsp), INP jne .Lloop0 =20 -.Ldone_hash: - # Restore Stack Pointer mov %rbp, %rsp pop %rbp =20 # Restore GPRs diff --git a/lib/crypto/x86/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse= 3-asm.S index 12e78142f2e38..4cae7445b2a86 100644 --- a/lib/crypto/x86/sha512-ssse3-asm.S +++ b/lib/crypto/x86/sha512-ssse3-asm.S @@ -269,17 +269,14 @@ frame_size =3D frame_WK + WK_SIZE # const u8 *data, size_t nblocks); # Purpose: Updates the SHA512 digest stored at "state" with the message # stored in "data". # The size of the message pointed to by "data" must be an integer multiple # of SHA512 message blocks. -# "nblocks" is the message length in SHA512 blocks +# "nblocks" is the message length in SHA512 blocks. Must be >=3D 1. ######################################################################## SYM_FUNC_START(sha512_transform_ssse3) =20 - test msglen, msglen - je .Lnowork - # Save GPRs push %rbx push %r12 push %r13 push %r14 @@ -359,11 +356,10 @@ SYM_FUNC_START(sha512_transform_ssse3) pop %r14 pop %r13 pop %r12 pop %rbx =20 -.Lnowork: RET SYM_FUNC_END(sha512_transform_ssse3) =20 ######################################################################## ### Binary Data --=20 2.50.0 From nobody Wed Oct 8 08:14:51 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B95AA2BD5AB; Mon, 30 Jun 2025 16:05:29 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299529; cv=none; b=OSpSgpcWVM6Yqk8hRbChkPZGshW2FudL9qXdJj2rk3aJAC4NBStB28WE78/kVJWpkfUkXHPtGCKvt1OUKbzZSNcxI9prQIFaXKK9tfe8cYlonHuIN47At8e57XNeNFS6BqYlbYdoisEpYcFsw1VkaYaX7pGnRQaWHHf+GJVK3PA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751299529; c=relaxed/simple; bh=Re8d5TuPUDJdqxyH89C7qlYblbZnrA5EjF86KsH1XwY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=bCupi5/YrAnmdOaaKnBw22Da6MA5Z5mKy+kRF8lgcZ1mcYDeqlud4sVwlv9TSxIoY8z5S5hlispsx3d9DkSGkMcmAC65KWiD2gOg4VW9f9IP2Nb2XWExw+3O+j3FV/5Tuw21+q+78755UQmXtmGhNc7+1T+WGiHPEmlqRuKdjps= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=No3QeIRm; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="No3QeIRm" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 0AA2BC4CEF2; Mon, 30 Jun 2025 16:05:29 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1751299529; bh=Re8d5TuPUDJdqxyH89C7qlYblbZnrA5EjF86KsH1XwY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=No3QeIRmm7nlNERtKsn+ZyD7ApGiWkaIJk7FG/8b02RBvVt8BnEm4mMr1kGFRguX7 bZse8gOMDl2RguH9Al5RMXRfo4nMMB50ohKZVkdfhLnFRIrOxW9gvzUC+n00XH0+iY 9TH1JM6PrKklHfOntc9odhZozTrfFL7mhOzBliq2FCV5TlP968eIkkL1Ao0AXuC2Xd aiiB+i1f+mH0ulGtZTu7oHtA749yaAWdo2f/7TRILzujisMvwU7MP9ogNQ3M17vTMJ 4Qx9NLxIq4BVav5bn31e4LZnHmRGclQY/IXM+CjEJBahQ5XnqBxt8dyb/U4BzbSsRz tM1D5LSdhxLZg== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, sparclinux@vger.kernel.org, x86@kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Eric Biggers Subject: [PATCH v3 16/16] crypto: sha512 - Remove sha512_base.h Date: Mon, 30 Jun 2025 09:03:20 -0700 Message-ID: <20250630160320.2888-17-ebiggers@kernel.org> X-Mailer: git-send-email 2.50.0 In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org> References: <20250630160320.2888-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" sha512_base.h is no longer used, so remove it. Acked-by: Ard Biesheuvel Signed-off-by: Eric Biggers --- include/crypto/sha512_base.h | 117 ----------------------------------- 1 file changed, 117 deletions(-) delete mode 100644 include/crypto/sha512_base.h diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h deleted file mode 100644 index d1361b3eb70b0..0000000000000 --- a/include/crypto/sha512_base.h +++ /dev/null @@ -1,117 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * sha512_base.h - core logic for SHA-512 implementations - * - * Copyright (C) 2015 Linaro Ltd - */ - -#ifndef _CRYPTO_SHA512_BASE_H -#define _CRYPTO_SHA512_BASE_H - -#include -#include -#include -#include -#include -#include -#include - -typedef void (sha512_block_fn)(struct sha512_state *sst, u8 const *src, - int blocks); - -static inline int sha384_base_init(struct shash_desc *desc) -{ - struct sha512_state *sctx =3D shash_desc_ctx(desc); - - sctx->state[0] =3D SHA384_H0; - sctx->state[1] =3D SHA384_H1; - sctx->state[2] =3D SHA384_H2; - sctx->state[3] =3D SHA384_H3; - sctx->state[4] =3D SHA384_H4; - sctx->state[5] =3D SHA384_H5; - sctx->state[6] =3D SHA384_H6; - sctx->state[7] =3D SHA384_H7; - sctx->count[0] =3D sctx->count[1] =3D 0; - - return 0; -} - -static inline int sha512_base_init(struct shash_desc *desc) -{ - struct sha512_state *sctx =3D shash_desc_ctx(desc); - - sctx->state[0] =3D SHA512_H0; - sctx->state[1] =3D SHA512_H1; - sctx->state[2] =3D SHA512_H2; - sctx->state[3] =3D SHA512_H3; - sctx->state[4] =3D SHA512_H4; - sctx->state[5] =3D SHA512_H5; - sctx->state[6] =3D SHA512_H6; - sctx->state[7] =3D SHA512_H7; - sctx->count[0] =3D sctx->count[1] =3D 0; - - return 0; -} - -static inline int sha512_base_do_update_blocks(struct shash_desc *desc, - const u8 *data, - unsigned int len, - sha512_block_fn *block_fn) -{ - unsigned int remain =3D len - round_down(len, SHA512_BLOCK_SIZE); - struct sha512_state *sctx =3D shash_desc_ctx(desc); - - len -=3D remain; - sctx->count[0] +=3D len; - if (sctx->count[0] < len) - sctx->count[1]++; - block_fn(sctx, data, len / SHA512_BLOCK_SIZE); - return remain; -} - -static inline int sha512_base_do_finup(struct shash_desc *desc, const u8 *= src, - unsigned int len, - sha512_block_fn *block_fn) -{ - unsigned int bit_offset =3D SHA512_BLOCK_SIZE / 8 - 2; - struct sha512_state *sctx =3D shash_desc_ctx(desc); - union { - __be64 b64[SHA512_BLOCK_SIZE / 4]; - u8 u8[SHA512_BLOCK_SIZE * 2]; - } block =3D {}; - - if (len >=3D SHA512_BLOCK_SIZE) { - int remain; - - remain =3D sha512_base_do_update_blocks(desc, src, len, block_fn); - src +=3D len - remain; - len =3D remain; - } - - if (len >=3D bit_offset * 8) - bit_offset +=3D SHA512_BLOCK_SIZE / 8; - memcpy(&block, src, len); - block.u8[len] =3D 0x80; - sctx->count[0] +=3D len; - block.b64[bit_offset] =3D cpu_to_be64(sctx->count[1] << 3 | - sctx->count[0] >> 61); - block.b64[bit_offset + 1] =3D cpu_to_be64(sctx->count[0] << 3); - block_fn(sctx, block.u8, (bit_offset + 2) * 8 / SHA512_BLOCK_SIZE); - memzero_explicit(&block, sizeof(block)); - - return 0; -} - -static inline int sha512_base_finish(struct shash_desc *desc, u8 *out) -{ - unsigned int digest_size =3D crypto_shash_digestsize(desc->tfm); - struct sha512_state *sctx =3D shash_desc_ctx(desc); - __be64 *digest =3D (__be64 *)out; - int i; - - for (i =3D 0; digest_size > 0; i++, digest_size -=3D sizeof(__be64)) - put_unaligned_be64(sctx->state[i], digest++); - return 0; -} - -#endif /* _CRYPTO_SHA512_BASE_H */ --=20 2.50.0