From nobody Wed Apr 1 08:37:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2C80EDDC5; Wed, 1 Apr 2026 00:07:49 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002069; cv=none; b=H7EWjzusvfo5kqryF5u4ZkCxVlM280hob5I2R6ojHx8Us3c4vudVYDFblvj9+r+yplVjXzeFsNJtwiaLim6o/6Y53GujBpZwtzYGK79DuiPQUsSjo891ucaqjC2Q16sKv79Ixm8Q0IluJmf/MuKT2Qz6zCCTfMz1JgWNNu38Uuc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002069; c=relaxed/simple; bh=QQ5mFKI5+uBNS64ELsUrgq6oqh/+Q+BmfI0QIv2/GuM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=f/ujE3VRO+KgzOi/LKqax6rH+tn0ilCijuT6aIeYUOonxiCLZfcXTjK/hKocSemYHdbFjmd5R73KV125PaPUXWbtJ6KKA/5jueOvHdn8zcSmIbrGIyYjmKPgq57d8ybHRHtTCunJVbvpqZdmWthf7LcVm3jIQkfy+ijRjPlq2fk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=aqjb0dqG; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="aqjb0dqG" Received: by smtp.kernel.org (Postfix) with ESMTPSA id A9A12C2BCB0; Wed, 1 Apr 2026 00:07:48 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1775002069; bh=QQ5mFKI5+uBNS64ELsUrgq6oqh/+Q+BmfI0QIv2/GuM=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=aqjb0dqGJVSdbX84cN3kcRLDWKnpsqEutYUmzk2VWkQ1HxO+/cWz7jD0nZ8R20CQi /ai8DJSkK9Q7jCRTan2YBsl6kRpeW7Efc2O26yT9l9Zl8qn8Hd99kUiYnCOeNd4O0M rGwLz4WeTTT4/9kixddS/iB/pHnXZUtlyy/wq01qwZd0pvpFj0Rw9NLP8TCHUA5vWe P3/RqWKkvs8pCUUaLCUwWDOlJUzDrLkjjOXnivDXGDWSY5kRWngnaef8lfexYA/2dr EghDBbuGLMCmrGkfQTe1SvFs93Cmc6Vth6toFWDyVgUBs80BLlvqo5bTF7WKUzphPc /6BWsJhgQjTwg== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , linux-arm-kernel@lists.infradead.org, Eric Biggers Subject: [PATCH 1/9] lib/crypto: arm64/aes: Remove obsolete chunking logic Date: Tue, 31 Mar 2026 17:05:40 -0700 Message-ID: <20260401000548.133151-2-ebiggers@kernel.org> X-Mailer: git-send-email 2.53.0 In-Reply-To: <20260401000548.133151-1-ebiggers@kernel.org> References: <20260401000548.133151-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the AES-CBC-MAC code accordingly. Signed-off-by: Eric Biggers Reviewed-by: Ard Biesheuvel --- arch/arm64/crypto/aes-ce-ccm-glue.c | 13 ++++------- include/crypto/aes.h | 6 ++--- lib/crypto/arm64/aes-modes.S | 8 +++---- lib/crypto/arm64/aes.h | 35 +++++++++++------------------ 4 files changed, 23 insertions(+), 39 deletions(-) diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce= -ccm-glue.c index 45aed0073283..a304375ce724 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -99,20 +99,15 @@ static u32 ce_aes_ccm_auth_data(u8 mac[], u8 const in[]= , u32 abytes, =20 do { u32 blocks =3D abytes / AES_BLOCK_SIZE; =20 if (macp =3D=3D AES_BLOCK_SIZE || (!macp && blocks > 0)) { - u32 rem =3D ce_aes_mac_update(in, rk, rounds, blocks, mac, - macp, enc_after); - u32 adv =3D (blocks - rem) * AES_BLOCK_SIZE; - + ce_aes_mac_update(in, rk, rounds, blocks, mac, macp, + enc_after); macp =3D enc_after ? 0 : AES_BLOCK_SIZE; - in +=3D adv; - abytes -=3D adv; - - if (unlikely(rem)) - macp =3D 0; + in +=3D blocks * AES_BLOCK_SIZE; + abytes -=3D blocks * AES_BLOCK_SIZE; } else { u32 l =3D min(AES_BLOCK_SIZE - macp, abytes); =20 crypto_xor(&mac[macp], in, l); in +=3D l; diff --git a/include/crypto/aes.h b/include/crypto/aes.h index 3feb4105c2a2..16fbfd93e2bd 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -228,13 +228,13 @@ asmlinkage void ce_aes_essiv_cbc_encrypt(u8 out[], u8= const in[], u32 const rk1[], int rounds, int blocks, u8 iv[], u32 const rk2[]); asmlinkage void ce_aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[], int rounds, int blocks, u8 iv[], u32 const rk2[]); -asmlinkage size_t ce_aes_mac_update(u8 const in[], u32 const rk[], int rou= nds, - size_t blocks, u8 dg[], int enc_before, - int enc_after); +asmlinkage void ce_aes_mac_update(u8 const in[], u32 const rk[], int round= s, + size_t blocks, u8 dg[], int enc_before, + int enc_after); #elif defined(CONFIG_PPC) void ppc_expand_key_128(u32 *key_enc, const u8 *key); void ppc_expand_key_192(u32 *key_enc, const u8 *key); void ppc_expand_key_256(u32 *key_enc, const u8 *key); void ppc_generate_decrypt_key(u32 *key_dec, u32 *key_enc, unsigned int key= _len); diff --git a/lib/crypto/arm64/aes-modes.S b/lib/crypto/arm64/aes-modes.S index fc89cd02b642..10e537317eaf 100644 --- a/lib/crypto/arm64/aes-modes.S +++ b/lib/crypto/arm64/aes-modes.S @@ -815,13 +815,13 @@ AES_FUNC_START(aes_xts_decrypt) b .Lxtsdecctsout AES_FUNC_END(aes_xts_decrypt) =20 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) /* - * size_t aes_mac_update(u8 const in[], u32 const rk[], int rounds, - * size_t blocks, u8 dg[], int enc_before, - * int enc_after); + * void aes_mac_update(u8 const in[], u32 const rk[], int rounds, + * size_t blocks, u8 dg[], int enc_before, + * int enc_after); */ AES_FUNC_START(aes_mac_update) ld1 {v0.16b}, [x4] /* get dg */ enc_prepare w2, x1, x7 cbz w5, .Lmacloop4x @@ -842,11 +842,10 @@ AES_FUNC_START(aes_mac_update) cmp x3, xzr csinv w5, w6, wzr, eq cbz w5, .Lmacout encrypt_block v0, w2, x1, x7, w8 st1 {v0.16b}, [x4] /* return dg */ - cond_yield .Lmacout, x7, x8 b .Lmacloop4x .Lmac1x: add x3, x3, #4 .Lmacloop: cbz x3, .Lmacout @@ -861,9 +860,8 @@ AES_FUNC_START(aes_mac_update) encrypt_block v0, w2, x1, x7, w8 b .Lmacloop =20 .Lmacout: st1 {v0.16b}, [x4] /* return dg */ - mov x0, x3 ret AES_FUNC_END(aes_mac_update) #endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ diff --git a/lib/crypto/arm64/aes.h b/lib/crypto/arm64/aes.h index 135d3324a30a..9e9e45a6f787 100644 --- a/lib/crypto/arm64/aes.h +++ b/lib/crypto/arm64/aes.h @@ -27,13 +27,13 @@ asmlinkage void __aes_ce_encrypt(const u32 rk[], u8 out= [AES_BLOCK_SIZE], asmlinkage void __aes_ce_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE= ], const u8 in[AES_BLOCK_SIZE], int rounds); asmlinkage u32 __aes_ce_sub(u32 l); asmlinkage void __aes_ce_invert(struct aes_block *out, const struct aes_block *in); -asmlinkage size_t neon_aes_mac_update(u8 const in[], u32 const rk[], int r= ounds, - size_t blocks, u8 dg[], int enc_before, - int enc_after); +asmlinkage void neon_aes_mac_update(u8 const in[], u32 const rk[], int rou= nds, + size_t blocks, u8 dg[], int enc_before, + int enc_after); =20 /* * Expand an AES key using the crypto extensions if supported and usable or * generic code otherwise. The expanded key format is compatible between = the * two cases. The outputs are @rndkeys (required) and @inv_rndkeys (optio= nal). @@ -190,29 +190,20 @@ static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZ= E], const struct aes_enckey *key, const u8 *data, size_t nblocks, bool enc_before, bool enc_after) { if (static_branch_likely(&have_neon) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() { - if (static_branch_likely(&have_aes)) - rem =3D ce_aes_mac_update( - data, key->k.rndkeys, - key->nrounds, nblocks, h, - enc_before, enc_after); - else - rem =3D neon_aes_mac_update( - data, key->k.rndkeys, - key->nrounds, nblocks, h, - enc_before, enc_after); - } - data +=3D (nblocks - rem) * AES_BLOCK_SIZE; - nblocks =3D rem; - enc_before =3D false; - } while (nblocks); + scoped_ksimd() { + if (static_branch_likely(&have_aes)) + ce_aes_mac_update(data, key->k.rndkeys, + key->nrounds, nblocks, h, + enc_before, enc_after); + else + neon_aes_mac_update(data, key->k.rndkeys, + key->nrounds, nblocks, h, + enc_before, enc_after); + } return true; } return false; } #endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ --=20 2.53.0 From nobody Wed Apr 1 08:37:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 88A3A1B7F4; Wed, 1 Apr 2026 00:07:49 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002069; cv=none; b=XJrv4SIxlWmDlonht4lnWYJmmj1izNOAA88yoWJI4+BlnorilsWJVjqRxOHjyZOT745J5by7XErEvoFDYZNEv8lGgozbXbL7YJC7cYFF5+J5WgiGyNgzB+jfDCSHe63+f9ke6JOgJzli1Zqc/eiSohXeMgq3VAdMm6dqMZBS8aw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002069; c=relaxed/simple; bh=j3vg8XQEajSovL+s9UwD7AoOOlA0CLrK0i32Wlp9kDk=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=XZiTBGawfrNjHPlwnGJ1VkCMobb/MxqhlWGnvel6U4SqFW+yPR44jG5kHHFDoxpSpyHS5WlYKhrtntgn764onx+xxdjfHpRRBr/R2MTJGo/H0VIvZx7raQO9sCdeJohJK7/fLfDECH3rirDSpi6B3WcybAYxwv4NjMWHPb7bURo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=IlL9LCLq; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="IlL9LCLq" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 1D1F0C2BCB1; Wed, 1 Apr 2026 00:07:49 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1775002069; bh=j3vg8XQEajSovL+s9UwD7AoOOlA0CLrK0i32Wlp9kDk=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=IlL9LCLqxyWcvhbtkZ+pUsJlNFKEdgIXGBUuGkbM8x5PWdMr//mGBvVTrMU7+PWJU TG0/NYWkDnFcBJ0gn3kXWVNPvL94J9VpzT1QxSD3/axKGDwL+BSSkwsplfQU5rPOv7 xK2F97nl7PBY3wT5A45aJl7PQNLsCxkWUoSC+wO8QrWFW6uILXrhKnRJ9Nn//9GE2G 5IAA3JxG2A1qhIKUyrczQIrIZSYSFfSWQn+Hwq+RmeMEBCK6M0pupF7zk33iQFLxjG U9zTcQnZXBmD/cAGJrfoR1VVZrVMn9hNGwGQ1v7YlHuUkieKEauI41PsUQUlFrKaWR 0BuA/tIEDGIkg== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , linux-arm-kernel@lists.infradead.org, Eric Biggers Subject: [PATCH 2/9] lib/crypto: arm64/chacha: Remove obsolete chunking logic Date: Tue, 31 Mar 2026 17:05:41 -0700 Message-ID: <20260401000548.133151-3-ebiggers@kernel.org> X-Mailer: git-send-email 2.53.0 In-Reply-To: <20260401000548.133151-1-ebiggers@kernel.org> References: <20260401000548.133151-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the ChaCha code accordingly. Signed-off-by: Eric Biggers Reviewed-by: Ard Biesheuvel --- lib/crypto/arm64/chacha.h | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/lib/crypto/arm64/chacha.h b/lib/crypto/arm64/chacha.h index ca8c6a8b0578..c6f8ddf98e2d 100644 --- a/lib/crypto/arm64/chacha.h +++ b/lib/crypto/arm64/chacha.h @@ -34,13 +34,13 @@ asmlinkage void hchacha_block_neon(const struct chacha_= state *state, u32 out[HCHACHA_OUT_WORDS], int nrounds); =20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); =20 static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *s= rc, - int bytes, int nrounds) + unsigned int bytes, int nrounds) { - while (bytes > 0) { + while (bytes) { int l =3D min(bytes, CHACHA_BLOCK_SIZE * 5); =20 if (l <=3D CHACHA_BLOCK_SIZE) { u8 buf[CHACHA_BLOCK_SIZE]; =20 @@ -74,20 +74,12 @@ static void chacha_crypt_arch(struct chacha_state *stat= e, u8 *dst, { if (!static_branch_likely(&have_neon) || bytes <=3D CHACHA_BLOCK_SIZE || !crypto_simd_usable()) return chacha_crypt_generic(state, dst, src, bytes, nrounds); =20 - do { - unsigned int todo =3D min_t(unsigned int, bytes, SZ_4K); - - scoped_ksimd() - chacha_doneon(state, dst, src, todo, nrounds); - - bytes -=3D todo; - src +=3D todo; - dst +=3D todo; - } while (bytes); + scoped_ksimd() + chacha_doneon(state, dst, src, bytes, nrounds); } =20 #define chacha_mod_init_arch chacha_mod_init_arch static void chacha_mod_init_arch(void) { --=20 2.53.0 From nobody Wed Apr 1 08:37:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id DEECC29408; Wed, 1 Apr 2026 00:07:49 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002069; cv=none; b=Kue69HaqIMZV9HenJO/ewkaorba9FBEhncLi8jtnfAasBqG5mlihves+yggtTjThkakdIAW8yZYHfGwHx2MPaM5XX5ZtrdNj7fzwAVr/NboSyYA8HrY4FYDUlcT3C56tO4VYWpP1vk31txzx1qulAImMgQZV9fZHXocZ8ReSRzs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002069; c=relaxed/simple; bh=Xl+MsNWe1/r1+hTDGoE1qKiqas9gp7xrKlVQjVUdCyU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=WYrMnwmBeGCqKNU8YzpqH1UAVs7v1RDmtipfrrOm5WETHu3ERODKntWslK+mC6EhHNXDu14mQs+kLetjaB6id3awy22jZ0ye71bN85vACZ+emt7NX+nqq9m6Ol2J0huS4B0AWfpTe4yI3zTl1XTZgWTAuwcDYJWpotKauYZCEZw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=IHCmaKJn; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="IHCmaKJn" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 82D76C19423; Wed, 1 Apr 2026 00:07:49 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1775002069; bh=Xl+MsNWe1/r1+hTDGoE1qKiqas9gp7xrKlVQjVUdCyU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=IHCmaKJnLmp8V/FAp6FPJYNBZjtGPcpCLhe/EPJAX/ErJ9mrsrR4bH+PtTECPSDQ7 VtVox40YNbtQ4mSLUgeDVtJzRU9/o5p/B2Ni9x0yb+l8aFFVJxKCV4OBF5FweuKeSU A8JkWfVi+pFwRvLsBohwrVDy6MpSNBPL6kbmw5l/tZ+azQAGqCd+H2PYW1rG+hUEWY VRWnz+dY56ME2qUUwYCFtOnsshwg86V5LCvEL+xQ5PeI5d6zVbbpEUZ4s3kWfhJnnU k3Sh0vpSqpxN8F4rtoqlEUeBzbSnG72unkrZwYpF84hqfMMvnxfjbsmBhE/Xzjnw+8 UUHFjmaB5I3wQ== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , linux-arm-kernel@lists.infradead.org, Eric Biggers Subject: [PATCH 3/9] lib/crypto: arm64/gf128hash: Remove obsolete chunking logic Date: Tue, 31 Mar 2026 17:05:42 -0700 Message-ID: <20260401000548.133151-4-ebiggers@kernel.org> X-Mailer: git-send-email 2.53.0 In-Reply-To: <20260401000548.133151-1-ebiggers@kernel.org> References: <20260401000548.133151-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the GHASH and POLYVAL code accordingly. Signed-off-by: Eric Biggers Reviewed-by: Ard Biesheuvel --- lib/crypto/arm64/gf128hash.h | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/lib/crypto/arm64/gf128hash.h b/lib/crypto/arm64/gf128hash.h index b2c85585b758..1d1179f87f8d 100644 --- a/lib/crypto/arm64/gf128hash.h +++ b/lib/crypto/arm64/gf128hash.h @@ -87,20 +87,12 @@ static void polyval_mul_arch(struct polyval_elem *acc, static void ghash_blocks_arch(struct polyval_elem *acc, const struct ghash_key *key, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_asimd) && may_use_simd()) { - do { - /* Allow rescheduling every 4 KiB. */ - size_t n =3D min_t(size_t, nblocks, - 4096 / GHASH_BLOCK_SIZE); - - scoped_ksimd() - pmull_ghash_update_p8(n, acc, data, &key->h); - data +=3D n * GHASH_BLOCK_SIZE; - nblocks -=3D n; - } while (nblocks); + scoped_ksimd() + pmull_ghash_update_p8(nblocks, acc, data, &key->h); } else { ghash_blocks_generic(acc, &key->h, data, nblocks); } } =20 @@ -108,20 +100,12 @@ static void ghash_blocks_arch(struct polyval_elem *ac= c, static void polyval_blocks_arch(struct polyval_elem *acc, const struct polyval_key *key, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_pmull) && may_use_simd()) { - do { - /* Allow rescheduling every 4 KiB. */ - size_t n =3D min_t(size_t, nblocks, - 4096 / POLYVAL_BLOCK_SIZE); - - scoped_ksimd() - polyval_blocks_pmull(acc, key, data, n); - data +=3D n * POLYVAL_BLOCK_SIZE; - nblocks -=3D n; - } while (nblocks); + scoped_ksimd() + polyval_blocks_pmull(acc, key, data, nblocks); } else { polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], data, nblocks); } } --=20 2.53.0 From nobody Wed Apr 1 08:37:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 50C0C322A; Wed, 1 Apr 2026 00:07:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002070; cv=none; b=lI2uWB8rJ1tZ/LBSj6JHu2gwANUMXRCXIwAp9KxnHCToz0DsIPBXSWRwU30utcq9pI6fBZAJGPkru/URJsLBRqJ2Ioj0Mfy5qU2U9YMwxZMZaLYtqwSlQAHk78lNBKT2M/TZMk1uQZzet7U3ZqeJDyq2ZnPohJkQD4lKEzcV2Vk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002070; c=relaxed/simple; bh=IDxofDV1l6itbnGBXpDqaJyt+/X6blNaQzeWk25J7Ts=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=pSOjU9RFHkjl/SYVHAEZ2JcQONC1589QQVQwVftgIulr+44OS3YxpRdA8dm5yxOTrtm7f6MLfNRiAKzDmcg2qkTW+AakYa9CECSEhDZ5D4RBniRLTo9U9my8Oqj8WHrna02DZSS4F2mMIVrc3dYgnnw9UvYQMUCUBfbh/MoUAF8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=MLnFZ9ZG; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="MLnFZ9ZG" Received: by smtp.kernel.org (Postfix) with ESMTPSA id E8D7EC2BCB4; Wed, 1 Apr 2026 00:07:49 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1775002070; bh=IDxofDV1l6itbnGBXpDqaJyt+/X6blNaQzeWk25J7Ts=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=MLnFZ9ZGjNSPQd/8I/31b5XsbGh4co6ArbnJCiY8mDm0jMZkPalDqjTX5wNAnkBNo hjJnO+ANvexDSdzaAxZhxTYJT4F61XpgLpkvVPd++oiCnG57LPEr22T2+SeYSLVSel +nxzysiwnMsFChEc1Ho0QgXp9yeVk/usNn8bz9/UWDc6YeZz/6eQ2Zsj8UPVCTyG9s L+pFRYyUMQBPa8wRe89v3DROFp5Njl2tucohO6Y4LV9uzwAnal0dfP3mAtew+i9cn1 9VFc6z+HXIUkGapBP9Oq/Da6Eszzw0zImOdMXcqwYswJQtxXFqzM6ehAQcwbB4FP5V yX1vWGuxuQibQ== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , linux-arm-kernel@lists.infradead.org, Eric Biggers Subject: [PATCH 4/9] lib/crypto: arm64/poly1305: Remove obsolete chunking logic Date: Tue, 31 Mar 2026 17:05:43 -0700 Message-ID: <20260401000548.133151-5-ebiggers@kernel.org> X-Mailer: git-send-email 2.53.0 In-Reply-To: <20260401000548.133151-1-ebiggers@kernel.org> References: <20260401000548.133151-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the Poly1305 code accordingly. Signed-off-by: Eric Biggers Reviewed-by: Ard Biesheuvel --- lib/crypto/arm64/poly1305.h | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/crypto/arm64/poly1305.h b/lib/crypto/arm64/poly1305.h index b77669767cd6..3d4bde857699 100644 --- a/lib/crypto/arm64/poly1305.h +++ b/lib/crypto/arm64/poly1305.h @@ -25,21 +25,15 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neo= n); =20 static void poly1305_blocks(struct poly1305_block_state *state, const u8 *= src, unsigned int len, u32 padbit) { if (static_branch_likely(&have_neon) && likely(may_use_simd())) { - do { - unsigned int todo =3D min_t(unsigned int, len, SZ_4K); - - scoped_ksimd() - poly1305_blocks_neon(state, src, todo, padbit); - - len -=3D todo; - src +=3D todo; - } while (len); - } else + scoped_ksimd() + poly1305_blocks_neon(state, src, len, padbit); + } else { poly1305_blocks_arm64(state, src, len, padbit); + } } =20 #define poly1305_mod_init_arch poly1305_mod_init_arch static void poly1305_mod_init_arch(void) { --=20 2.53.0 From nobody Wed Apr 1 08:37:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C3F275C613; Wed, 1 Apr 2026 00:07:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002070; cv=none; b=pGlufmnYmNzOgKjOQcuYYYEfBpMLZbNszJbGMTo8KgHV8yNdwwa7fqXrQfjzYvDR9onn/IZ554VUNGEG2cgjQbq/ICVi1N12ITbKDwcXQaOkZomjYlpPeqv1SiQSjqakastOLMtdnGzIfrzy9uSe80AR8wmdYtxPebYW3Y/YZbk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002070; c=relaxed/simple; bh=lMl6Llb6SKdL6D1zgOk+w6G/KGWKdefJM1MFe8cedTY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=fjVlgHZkY85f1zWGaqCp7I/wWWmLjZz/Xoa7imQXsVenAwvgTTmZWaE3lEhiX1HulRd5wbl6jTcy3Ogm+Lk1fiLMT7hAKkBQVK4gATgRxxG2o4kEtyH7qZvJMDHFpFt/UAXWQJP96dSKL2i3NgPyehtFYKZOMH81k6DImZRj19o= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=d2PAyarZ; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="d2PAyarZ" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 5B457C2BCB3; Wed, 1 Apr 2026 00:07:50 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1775002070; bh=lMl6Llb6SKdL6D1zgOk+w6G/KGWKdefJM1MFe8cedTY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=d2PAyarZnxFpl6bvbCZYNUP1+rkfXxWgt+4MxWncO6fhwgsOwE0CXriXvLc61Szij dJETPQA9L+XuNjEWDoShv+7YLByys1tmpYDprg9gI0AAIysOozOoItbBgdnwFO3+gf v4cAKKnTtHLNmcNSTAEgXuaaXnKXsgPGvYcN6yGQxFj8mMxSuzOSMnwoonoyEytV8L YmhkDE/F71A/Jon5LY7To5UGsD2tPacVEUrmIa27jDukiHbrisVrvlX+Ba+RtwvfpJ EbAQ6FxbglMwrf+IdHlvOLgvUZ+otsqBgN/zcpKPPO8T+ihi1sHAaTePb29Pegbxet LVnHbHCPmhAwA== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , linux-arm-kernel@lists.infradead.org, Eric Biggers Subject: [PATCH 5/9] lib/crypto: arm64/sha1: Remove obsolete chunking logic Date: Tue, 31 Mar 2026 17:05:44 -0700 Message-ID: <20260401000548.133151-6-ebiggers@kernel.org> X-Mailer: git-send-email 2.53.0 In-Reply-To: <20260401000548.133151-1-ebiggers@kernel.org> References: <20260401000548.133151-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the SHA-1 code accordingly. Signed-off-by: Eric Biggers Reviewed-by: Ard Biesheuvel --- lib/crypto/arm64/sha1-ce-core.S | 14 +++++--------- lib/crypto/arm64/sha1.h | 15 ++++----------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/lib/crypto/arm64/sha1-ce-core.S b/lib/crypto/arm64/sha1-ce-cor= e.S index 8fbd4767f0f0..59d27fda0714 100644 --- a/lib/crypto/arm64/sha1-ce-core.S +++ b/lib/crypto/arm64/sha1-ce-core.S @@ -60,14 +60,14 @@ movk \tmp, :abs_g1:\val dup \k, \tmp .endm =20 /* - * size_t __sha1_ce_transform(struct sha1_block_state *state, - * const u8 *data, size_t nblocks); + * void sha1_ce_transform(struct sha1_block_state *state, + * const u8 *data, size_t nblocks); */ -SYM_FUNC_START(__sha1_ce_transform) +SYM_FUNC_START(sha1_ce_transform) /* load round constants */ loadrc k0.4s, 0x5a827999, w6 loadrc k1.4s, 0x6ed9eba1, w6 loadrc k2.4s, 0x8f1bbcdc, w6 loadrc k3.4s, 0xca62c1d6, w6 @@ -114,17 +114,13 @@ CPU_LE( rev32 v11.16b, v11.16b ) =20 /* update state */ add dgbv.2s, dgbv.2s, dg1v.2s add dgav.4s, dgav.4s, dg0v.4s =20 - /* return early if voluntary preemption is needed */ - cond_yield 1f, x5, x6 - /* handled all input blocks? */ cbnz x2, 0b =20 /* store new state */ -1: st1 {dgav.4s}, [x0] + st1 {dgav.4s}, [x0] str dgb, [x0, #16] - mov x0, x2 ret -SYM_FUNC_END(__sha1_ce_transform) +SYM_FUNC_END(sha1_ce_transform) diff --git a/lib/crypto/arm64/sha1.h b/lib/crypto/arm64/sha1.h index bc7071f1be09..112c5d443c56 100644 --- a/lib/crypto/arm64/sha1.h +++ b/lib/crypto/arm64/sha1.h @@ -7,26 +7,19 @@ #include #include =20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); =20 -asmlinkage size_t __sha1_ce_transform(struct sha1_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha1_ce_transform(struct sha1_block_state *state, + const u8 *data, size_t nblocks); =20 static void sha1_blocks(struct sha1_block_state *state, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_ce) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() - rem =3D __sha1_ce_transform(state, data, nblocks); - - data +=3D (nblocks - rem) * SHA1_BLOCK_SIZE; - nblocks =3D rem; - } while (nblocks); + scoped_ksimd() + sha1_ce_transform(state, data, nblocks); } else { sha1_blocks_generic(state, data, nblocks); } } =20 --=20 2.53.0 From nobody Wed Apr 1 08:37:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 79030175A86; Wed, 1 Apr 2026 00:07:51 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002071; cv=none; b=WW2G7e4lrp/AAvNPs8xm9ID3vMMneqF8gsZ0piUvkJoCDbmW3Ylb6JQytPyNtZSzjUM0PgddbY1jM6NLnJh/SYEkjH8ArJLZ18zkB89+PZhinbTGAmAaWgDWkZycKPb2nM46qVGsk8awUFoEoCain//rMTmmezsapT4lAgeGOxA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002071; c=relaxed/simple; bh=ZA5dMcNqb9sYb9aCW3C/EswuZIOxlWzjmUurDOE2JwA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=p9Jmw0TYQPZEDfUyIg94pDVfeh9Lp6wZp4IMurZAYkOOcPluBpzmd4ZERrfGEard3qkXPAc3sDI1v6aJp4vnVNWa3r6IAOtE5zmVcdDBNEZjl9HJ4EIbdP2S1oASPS12cnS2cC7kz9n8wdXwRz94VCafGO95m7KjxBnNRAW+H/g= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=aDXeKjFz; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="aDXeKjFz" Received: by smtp.kernel.org (Postfix) with ESMTPSA id C4F56C2BCB2; Wed, 1 Apr 2026 00:07:50 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1775002071; bh=ZA5dMcNqb9sYb9aCW3C/EswuZIOxlWzjmUurDOE2JwA=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=aDXeKjFzMbS83kJZgGBc/8YbdAGt2Mjt+ZfjZ9NIyP/zZsi9C/zmByZ9MEvYVeevw 00d2HqMUl8048AGKXjQWvQgPDnF5rX7UBPp6/rNPSnmGJUyvRfa14v5XKfU1RpWYHp zDOPFEzZ08waJL67H5BsERhRgbdGBb4hk7A2KCxuEpMA3jqjjJSicwIrRdc9rVLHPP nCvr2+oR8ML+0w/7boK/g9lxVkeJ4Gr1XseIhDOri32rorcA7zNVX4sG6ExNaRFQkZ LeB/qaJxuWKiidlYRxK+N+kncfZUgg0OGUo5M9Yntqfaf0Dt/ZyTdOLI5yK9USVIip r1raMpEq/HwQQ== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , linux-arm-kernel@lists.infradead.org, Eric Biggers Subject: [PATCH 6/9] lib/crypto: arm64/sha256: Remove obsolete chunking logic Date: Tue, 31 Mar 2026 17:05:45 -0700 Message-ID: <20260401000548.133151-7-ebiggers@kernel.org> X-Mailer: git-send-email 2.53.0 In-Reply-To: <20260401000548.133151-1-ebiggers@kernel.org> References: <20260401000548.133151-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the SHA-256 code accordingly. Signed-off-by: Eric Biggers Reviewed-by: Ard Biesheuvel --- lib/crypto/arm64/sha256-ce.S | 14 +++++--------- lib/crypto/arm64/sha256.h | 29 ++++++++--------------------- 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/lib/crypto/arm64/sha256-ce.S b/lib/crypto/arm64/sha256-ce.S index e4bfe42a61a9..b54ad977afa3 100644 --- a/lib/crypto/arm64/sha256-ce.S +++ b/lib/crypto/arm64/sha256-ce.S @@ -77,15 +77,15 @@ ld1 { v8.4s-v11.4s}, [\tmp], #64 ld1 {v12.4s-v15.4s}, [\tmp] .endm =20 /* - * size_t __sha256_ce_transform(struct sha256_block_state *state, - * const u8 *data, size_t nblocks); + * void sha256_ce_transform(struct sha256_block_state *state, + * const u8 *data, size_t nblocks); */ .text -SYM_FUNC_START(__sha256_ce_transform) +SYM_FUNC_START(sha256_ce_transform) =20 load_round_constants x8 =20 /* load state */ ld1 {dgav.4s, dgbv.4s}, [x0] @@ -125,21 +125,17 @@ CPU_LE( rev32 v19.16b, v19.16b ) =20 /* update state */ add dgav.4s, dgav.4s, dg0v.4s add dgbv.4s, dgbv.4s, dg1v.4s =20 - /* return early if voluntary preemption is needed */ - cond_yield 1f, x5, x6 - /* handled all input blocks? */ cbnz x2, 0b =20 /* store new state */ -1: st1 {dgav.4s, dgbv.4s}, [x0] - mov x0, x2 + st1 {dgav.4s, dgbv.4s}, [x0] ret -SYM_FUNC_END(__sha256_ce_transform) +SYM_FUNC_END(sha256_ce_transform) =20 .unreq dga .unreq dgav .unreq dgb .unreq dgbv diff --git a/lib/crypto/arm64/sha256.h b/lib/crypto/arm64/sha256.h index 1fad3d7baa9a..b4353d3c4dd0 100644 --- a/lib/crypto/arm64/sha256.h +++ b/lib/crypto/arm64/sha256.h @@ -12,30 +12,21 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); =20 asmlinkage void sha256_block_data_order(struct sha256_block_state *state, const u8 *data, size_t nblocks); asmlinkage void sha256_block_neon(struct sha256_block_state *state, const u8 *data, size_t nblocks); -asmlinkage size_t __sha256_ce_transform(struct sha256_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha256_ce_transform(struct sha256_block_state *state, + const u8 *data, size_t nblocks); =20 static void sha256_blocks(struct sha256_block_state *state, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_neon) && likely(may_use_simd())) { - if (static_branch_likely(&have_ce)) { - do { - size_t rem; - - scoped_ksimd() - rem =3D __sha256_ce_transform(state, data, - nblocks); - - data +=3D (nblocks - rem) * SHA256_BLOCK_SIZE; - nblocks =3D rem; - } while (nblocks); - } else { - scoped_ksimd() + scoped_ksimd() { + if (static_branch_likely(&have_ce)) + sha256_ce_transform(state, data, nblocks); + else sha256_block_neon(state, data, nblocks); } } else { sha256_block_data_order(state, data, nblocks); } @@ -53,17 +44,13 @@ asmlinkage void sha256_ce_finup2x(const struct __sha256= _ctx *ctx, static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx, const u8 *data1, const u8 *data2, size_t len, u8 out1[SHA256_DIGEST_SIZE], u8 out2[SHA256_DIGEST_SIZE]) { - /* - * The assembly requires len >=3D SHA256_BLOCK_SIZE && len <=3D INT_MAX. - * Further limit len to 65536 to avoid spending too long with preemption - * disabled. (Of course, in practice len is nearly always 4096 anyway.) - */ + /* The assembly requires len >=3D SHA256_BLOCK_SIZE && len <=3D INT_MAX. = */ if (static_branch_likely(&have_ce) && len >=3D SHA256_BLOCK_SIZE && - len <=3D 65536 && likely(may_use_simd())) { + len <=3D INT_MAX && likely(may_use_simd())) { scoped_ksimd() sha256_ce_finup2x(ctx, data1, data2, len, out1, out2); kmsan_unpoison_memory(out1, SHA256_DIGEST_SIZE); kmsan_unpoison_memory(out2, SHA256_DIGEST_SIZE); return true; --=20 2.53.0 From nobody Wed Apr 1 08:37:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 93F6617A586; Wed, 1 Apr 2026 00:07:51 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002071; cv=none; b=ARtq3mBk6X8QSSS1NIiWNWagK08zbOHQfQfx6IgmfjRHHAjLXg5FU8AiPP2iIo/8YNvkMM/XKEPs7i9geIsMk4XETulJBntWQpe7Hv6AJZon+FZvfjWWpAObncR1jzCLRApv/P+ACoq+1nEojQdI5ICEhu2M3zZCmgvPY18PxqI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002071; c=relaxed/simple; bh=CAC+qe7Z/7a3GD63JAT9HC8eCyiKaVWZzQ/HUGxe4cI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=kyeRdH2dBLF/iGWVO83Qt0W+gLYtGXwc1ALem1tkXfwnmZUyRQUBUNrLKaE1fGNUaIEO6oUQfgNUn5Nz56xWE2VSfwIKbNVBgPTeW1NERbSEp35O2u9wnasEYV2WEX9xKS7yMQ8KO+6XKaxUe6j5L6RLxQs0P7FUYUwRna1bELI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=VZ9K754L; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="VZ9K754L" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 39FBBC2BCB0; Wed, 1 Apr 2026 00:07:51 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1775002071; bh=CAC+qe7Z/7a3GD63JAT9HC8eCyiKaVWZzQ/HUGxe4cI=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=VZ9K754Lb9Mn8acJgg8xXygVIgFVgCsEu0WSKlrDoqPc+oJmZADyrnyGot+A4tQYL e6FIk6AaPSQLKatwj5hdQuC/e7vptVT2X94JvAi95l8BZOMa4ab82cvrhlhC3DHEZl waBKkLqjrt94Snwpxm3GMEKnAIb1+RzO9GJztMwQGaSXK23XMgqPsXeFd6VWst/DC4 Txi+ECzcEPZg33nWyU7fl3ybs6mzWdrFdLvSmhcL+++lWYtXaksOKTv/WYMe7T+yj4 CYykjpmjrcSOZL2v0B2jJsjrVkWG3dSGOTyGeNiCTOakRuf5qnURmOOkqc4GmQU1nY 1hcjJRa8UULoA== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , linux-arm-kernel@lists.infradead.org, Eric Biggers Subject: [PATCH 7/9] lib/crypto: arm64/sha512: Remove obsolete chunking logic Date: Tue, 31 Mar 2026 17:05:46 -0700 Message-ID: <20260401000548.133151-8-ebiggers@kernel.org> X-Mailer: git-send-email 2.53.0 In-Reply-To: <20260401000548.133151-1-ebiggers@kernel.org> References: <20260401000548.133151-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the SHA-512 code accordingly. Signed-off-by: Eric Biggers Reviewed-by: Ard Biesheuvel --- lib/crypto/arm64/sha512-ce-core.S | 12 +++++------- lib/crypto/arm64/sha512.h | 15 ++++----------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/lib/crypto/arm64/sha512-ce-core.S b/lib/crypto/arm64/sha512-ce= -core.S index ffd51acfd1ee..26834921e8d6 100644 --- a/lib/crypto/arm64/sha512-ce-core.S +++ b/lib/crypto/arm64/sha512-ce-core.S @@ -91,15 +91,15 @@ add v\i4\().2d, v\i1\().2d, v\i3\().2d sha512h2 q\i3, q\i1, v\i0\().2d .endm =20 /* - * size_t __sha512_ce_transform(struct sha512_block_state *state, - * const u8 *data, size_t nblocks); + * void sha512_ce_transform(struct sha512_block_state *state, + * const u8 *data, size_t nblocks); */ .text -SYM_FUNC_START(__sha512_ce_transform) +SYM_FUNC_START(sha512_ce_transform) /* load state */ ld1 {v8.2d-v11.2d}, [x0] =20 /* load first 4 round constants */ adr_l x3, .Lsha512_rcon @@ -184,14 +184,12 @@ CPU_LE( rev64 v19.16b, v19.16b ) add v8.2d, v8.2d, v0.2d add v9.2d, v9.2d, v1.2d add v10.2d, v10.2d, v2.2d add v11.2d, v11.2d, v3.2d =20 - cond_yield 3f, x4, x5 /* handled all input blocks? */ cbnz x2, 0b =20 /* store new state */ -3: st1 {v8.2d-v11.2d}, [x0] - mov x0, x2 + st1 {v8.2d-v11.2d}, [x0] ret -SYM_FUNC_END(__sha512_ce_transform) +SYM_FUNC_END(sha512_ce_transform) diff --git a/lib/crypto/arm64/sha512.h b/lib/crypto/arm64/sha512.h index d978c4d07e90..5da27e6e23ea 100644 --- a/lib/crypto/arm64/sha512.h +++ b/lib/crypto/arm64/sha512.h @@ -10,27 +10,20 @@ =20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns); =20 asmlinkage void sha512_block_data_order(struct sha512_block_state *state, const u8 *data, size_t nblocks); -asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha512_ce_transform(struct sha512_block_state *state, + const u8 *data, size_t nblocks); =20 static void sha512_blocks(struct sha512_block_state *state, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_sha512_insns) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() - rem =3D __sha512_ce_transform(state, data, nblocks); - - data +=3D (nblocks - rem) * SHA512_BLOCK_SIZE; - nblocks =3D rem; - } while (nblocks); + scoped_ksimd() + sha512_ce_transform(state, data, nblocks); } else { sha512_block_data_order(state, data, nblocks); } } =20 --=20 2.53.0 From nobody Wed Apr 1 08:37:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 07F9319AD5C; Wed, 1 Apr 2026 00:07:52 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002072; cv=none; b=QSHY7faLhHJmqS7+TcLVlFLGXKpjs6t62ABK3hRfhED/Ky0HkmWfNzKwJLznuXu7Yw5fgy7Q/FsOxtiRmNkS941BN58aDtHMnC7EMKisFpEQidOT66bK5O77nJUmZgRlBlXI1WAc7A3MpJ3B8H0JUpuTsIlW7p5L3iDDX8Uqc/0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002072; c=relaxed/simple; bh=07ofvOpbjy7Rvwy7/y+ptg/LOHhp6n7ORQdHYjZws/0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=fCVpgaKO/QRGRrnA/R3rh9YhUBtvUMs161a4zYohPOye9zSSDP1apbZPXc0tKZhWAC3W6FDyVFqwOn847eIMhOCmZ0Av6xUoKdgszyXXEnSWEJ7pr5Ml0UzCpJ3YlLvlKsIYyFzmcZeee4xWQMBgMvMNoxbHpJQaNBM230Xf1+U= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=CCTICZVm; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="CCTICZVm" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 9DE63C19423; Wed, 1 Apr 2026 00:07:51 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1775002071; bh=07ofvOpbjy7Rvwy7/y+ptg/LOHhp6n7ORQdHYjZws/0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=CCTICZVm5Xek9G7SrzaQRLeTtYiCK2S8jTgni2L9zEre5E/hNHHe0G0r5BKyKKsBQ 8su2oWId/cDBZrb8PihZ4wlC4Ac+nRiCWw+1LxhfViVGIWYY1icoXdLeqVK3g+FkBk P0yrqKIyWvZVB8mGQ8DuE8EOCmCScQnIMzOziVA5frx4YczjVAvnFFBFKpWfLTG023 q8TcH0TbNobEHDIDB7Z+twjZCqMhogkSVZVDaYZqQogaimEHgJSgQlca67UfaVop4t 8fDcs/RlYsUuhCvb4Az988H0Ee8WzbBZ1vMFIVv6Me66kbvftzVGlnqUEvK2SQGZys raP8llq4AjL+Q== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , linux-arm-kernel@lists.infradead.org, Eric Biggers Subject: [PATCH 8/9] lib/crypto: arm64/sha3: Remove obsolete chunking logic Date: Tue, 31 Mar 2026 17:05:47 -0700 Message-ID: <20260401000548.133151-9-ebiggers@kernel.org> X-Mailer: git-send-email 2.53.0 In-Reply-To: <20260401000548.133151-1-ebiggers@kernel.org> References: <20260401000548.133151-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the SHA-3 code accordingly. Signed-off-by: Eric Biggers Reviewed-by: Ard Biesheuvel --- lib/crypto/arm64/sha3-ce-core.S | 8 +++----- lib/crypto/arm64/sha3.h | 15 ++++----------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/lib/crypto/arm64/sha3-ce-core.S b/lib/crypto/arm64/sha3-ce-cor= e.S index ace90b506490..b8ab01987ae0 100644 --- a/lib/crypto/arm64/sha3-ce-core.S +++ b/lib/crypto/arm64/sha3-ce-core.S @@ -35,12 +35,12 @@ .macro xar, rd, rn, rm, imm6 .inst 0xce800000 | .L\rd | (.L\rn << 5) | ((\imm6) << 10) | (.L\rm << 16) .endm =20 /* - * size_t sha3_ce_transform(struct sha3_state *state, const u8 *data, - * size_t nblocks, size_t block_size) + * void sha3_ce_transform(struct sha3_state *state, const u8 *data, + * size_t nblocks, size_t block_size) * * block_size is assumed to be one of 72 (SHA3-512), 104 (SHA3-384), 136 * (SHA3-256 and SHAKE256), 144 (SHA3-224), or 168 (SHAKE128). */ .text @@ -183,22 +183,20 @@ SYM_FUNC_START(sha3_ce_transform) bcax v2.16b, v2.16b, v28.16b, v27.16b =20 eor v0.16b, v0.16b, v31.16b =20 cbnz w8, 3b - cond_yield 4f, x8, x9 cbnz x2, 0b =20 /* save state */ -4: st1 { v0.1d- v3.1d}, [x0], #32 + st1 { v0.1d- v3.1d}, [x0], #32 st1 { v4.1d- v7.1d}, [x0], #32 st1 { v8.1d-v11.1d}, [x0], #32 st1 {v12.1d-v15.1d}, [x0], #32 st1 {v16.1d-v19.1d}, [x0], #32 st1 {v20.1d-v23.1d}, [x0], #32 st1 {v24.1d}, [x0] - mov x0, x2 ret SYM_FUNC_END(sha3_ce_transform) =20 .section ".rodata", "a" .align 8 diff --git a/lib/crypto/arm64/sha3.h b/lib/crypto/arm64/sha3.h index b602f1b3b282..eaaba3224acc 100644 --- a/lib/crypto/arm64/sha3.h +++ b/lib/crypto/arm64/sha3.h @@ -10,26 +10,19 @@ #include #include =20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3); =20 -asmlinkage size_t sha3_ce_transform(struct sha3_state *state, const u8 *da= ta, - size_t nblocks, size_t block_size); +asmlinkage void sha3_ce_transform(struct sha3_state *state, const u8 *data, + size_t nblocks, size_t block_size); =20 static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data, size_t nblocks, size_t block_size) { if (static_branch_likely(&have_sha3) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() - rem =3D sha3_ce_transform(state, data, nblocks, - block_size); - data +=3D (nblocks - rem) * block_size; - nblocks =3D rem; - } while (nblocks); + scoped_ksimd() + sha3_ce_transform(state, data, nblocks, block_size); } else { sha3_absorb_blocks_generic(state, data, nblocks, block_size); } } =20 --=20 2.53.0 From nobody Wed Apr 1 08:37:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 74500E555; Wed, 1 Apr 2026 00:07:52 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002072; cv=none; b=TwvCC5LeMpB91s7lx+KahD9ljbMfdXUK8DWmG3uf2EXdCIdcW7RqtuoCNRDgOb482/ASpKiA/eCQYt9NQmkjT4DMmW0fJR5TwVWSvJdANDLSifzWdzL27tS6v8c4yjjyqgkv4aGgLiuHFRIV7OLLyswDesHynX3bPPGBjTaRBXk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1775002072; c=relaxed/simple; bh=uSa6SngRywqNPRUM39lw0q3opK7TOlmwGZRujqiCUMA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=JgyUmBUYkZDw/rS6z1zI8aOG2aWRBYstm+wkHk99Xx6NzTMQ+zfFLLbCyTMXsfTrdHvpHdMLOb6n92pcuK0wV8nPAcdYTxBfoCLAtvgpdqdavKyWmCm3gNtT2R5YQ1lDtgN9SCQBvbxqNNaYu6Aws7oVgsiNGiQONFziZR2jNEM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=XJ8IBEgw; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="XJ8IBEgw" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 14283C2BCB0; Wed, 1 Apr 2026 00:07:52 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1775002072; bh=uSa6SngRywqNPRUM39lw0q3opK7TOlmwGZRujqiCUMA=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=XJ8IBEgwo/GNi2M4/AW7mMQF5q1QaE9GwYnJLyqk0QVPdcttvrfH9iHO9ftvLcXha YMkLOI/ph8J5n+oMb9+ULFr9x4oClKY/MahoP9bGJlC69DSIQFiDBGuTvaZvGqw7E7 6cgGuVHozNi/rOUQ1l/5p3iSPvSQLcnnwjlk9ds50UnM9GQaiM+aTtqALZ3/cuHe9m UDYA2Umm8/xO/PsgwXPcxgVcxoEOypHfV6cZ0Iwxg/C+ibQPckec8J5OooB3VTubcu qHsDhGupLNfBHSFQcXx24DVcymcp7IAb7dKl6y47ojrUHy6mot1pd41ou5ngm5j9Bo Y+c3nekPZ1vTg== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , linux-arm-kernel@lists.infradead.org, Eric Biggers Subject: [PATCH 9/9] arm64: fpsimd: Remove obsolete cond_yield macro Date: Tue, 31 Mar 2026 17:05:48 -0700 Message-ID: <20260401000548.133151-10-ebiggers@kernel.org> X-Mailer: git-send-email 2.53.0 In-Reply-To: <20260401000548.133151-1-ebiggers@kernel.org> References: <20260401000548.133151-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" All invocations of the cond_yield macro have been removed, so remove the macro definition as well. Signed-off-by: Eric Biggers Reviewed-by: Ard Biesheuvel --- arch/arm64/include/asm/assembler.h | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/as= sembler.h index d3d46e5f7188..9d7c9ae5ac96 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -746,32 +746,10 @@ alternative_else_nop_endif =20 .macro set_sctlr_el2, reg set_sctlr sctlr_el2, \reg .endm =20 - /* - * Check whether asm code should yield as soon as it is able. This is - * the case if we are currently running in task context, and the - * TIF_NEED_RESCHED flag is set. (Note that the TIF_NEED_RESCHED flag - * is stored negated in the top word of the thread_info::preempt_count - * field) - */ - .macro cond_yield, lbl:req, tmp:req, tmp2 -#ifdef CONFIG_PREEMPT_VOLUNTARY - get_current_task \tmp - ldr \tmp, [\tmp, #TSK_TI_PREEMPT] - /* - * If we are serving a softirq, there is no point in yielding: the - * softirq will not be preempted no matter what we do, so we should - * run to completion as quickly as we can. The preempt_count field will - * have BIT(SOFTIRQ_SHIFT) set in this case, so the zero check will - * catch this case too. - */ - cbz \tmp, \lbl -#endif - .endm - /* * Branch Target Identifier (BTI) */ .macro bti, targets .equ .L__bti_targets_c, 34 --=20 2.53.0