From nobody Tue Dec 16 08:20:33 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B7962264A63; Mon, 5 May 2025 18:19:23 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746469163; cv=none; b=FsnD6tWOnj8mVJ1LorwPFzISGACgchb75QoLEj1PHEHwu7K+c2K09ZbWF1PezxTWOEEMFnQW/jNvA+UbZV7UCsHG2prSp2Lb4CsxqmBAni29C9tCOtkGH9vlzu0NdYpP/Ki/2aKeKWv6Q1VDNp4z0eBExZcQB41JVuSztpRZURE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746469163; c=relaxed/simple; bh=1lm36BmT2ciIHxgq9Faa5xgZTZ2tCFnxS4PPU4+4ko0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=PnpCBitJ5ltl7DsvyMT8znm+Tr85JFK1D7Py2z4pzQNfWb4Btsqwf3BKOlLateTMuWLY7zgYCDee/m/d5k89lqoQI0j91vYnrE2/KSjzv0XlTMajgPwz2x5O9v72qn/TN7uRwIm0fxgVvaGtfPWzKjutiIaVrD7nZlXMezBC8N0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=UnbGymxH; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="UnbGymxH" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 0AE50C4CEEF; Mon, 5 May 2025 18:19:23 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1746469163; bh=1lm36BmT2ciIHxgq9Faa5xgZTZ2tCFnxS4PPU4+4ko0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=UnbGymxHsW930DMEikNtN5ahidUeA7yTnvxSPUZ4bYSzsz3g+G34jfavUajcUbAcS xZUaXn65OsM1R8p3jAnEoaQV/FxMyzNnDjybfExtz3SrcfsKRBmZ+gNgs0GU0zCyvs JiMMI53pDNvsZRqCkn8K0T2Z/lZUIEqG6JP3hQ1e2L1IA306aMumL7Vog2W/EA/5As +d/ng2CyGIWu1SiqIWKx8jTTG9+W6bRWANyG/CJxorvLxYHgDS1WRfkXPhbRiqykm5 lHT+O0xD7Ihj1GVqF5OHDES1yb9kRPhnuNbirtSn8GNCyX5zp0YlijiMoVWelN4Ht/ 4XPeyygeRLgbg== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-bcachefs@vger.kernel.org, "Jason A . Donenfeld " , Theodore Ts'o Subject: [PATCH 1/4] crypto: lib/chacha - strongly type the ChaCha state Date: Mon, 5 May 2025 11:18:21 -0700 Message-ID: <20250505181824.647138-2-ebiggers@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250505181824.647138-1-ebiggers@kernel.org> References: <20250505181824.647138-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Eric Biggers The ChaCha state matrix is 16 32-bit words. Currently it is represented in the code as a raw u32 array, or even just a pointer to u32. This weak typing is error-prone. Instead, introduce struct chacha_state: struct chacha_state { u32 x[16]; }; Convert all ChaCha and HChaCha functions to use struct chacha_state. No functional changes. Signed-off-by: Eric Biggers Acked-by: Kent Overstreet --- arch/arm/lib/crypto/chacha-glue.c | 30 +++++---- arch/arm/lib/crypto/chacha-scalar-core.S | 5 +- arch/arm64/lib/crypto/chacha-neon-glue.c | 23 ++++--- arch/mips/lib/crypto/chacha-glue.c | 6 +- arch/powerpc/lib/crypto/chacha-p10-glue.c | 15 +++-- arch/powerpc/lib/crypto/chacha-p10le-8x.S | 6 +- arch/riscv/lib/crypto/chacha-riscv64-glue.c | 8 +-- arch/riscv/lib/crypto/chacha-riscv64-zvkb.S | 10 +-- arch/s390/lib/crypto/chacha-glue.c | 11 ++-- arch/x86/lib/crypto/chacha_glue.c | 58 ++++++++++------- crypto/chacha.c | 16 ++--- drivers/char/random.c | 41 ++++++------ fs/bcachefs/checksum.c | 18 ++--- include/crypto/chacha.h | 65 +++++++++++-------- lib/crypto/chacha.c | 35 +++++----- lib/crypto/chacha20poly1305-selftest.c | 8 +-- lib/crypto/chacha20poly1305.c | 51 ++++++++------- lib/crypto/libchacha.c | 2 +- .../crypto/chacha20-s390/test-cipher.c | 10 +-- 19 files changed, 228 insertions(+), 190 deletions(-) diff --git a/arch/arm/lib/crypto/chacha-glue.c b/arch/arm/lib/crypto/chacha= -glue.c index 1e28736834a0..0c2b4c62d484 100644 --- a/arch/arm/lib/crypto/chacha-glue.c +++ b/arch/arm/lib/crypto/chacha-glue.c @@ -15,28 +15,31 @@ #include #include #include #include =20 -asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 = *src, - int nrounds); -asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8= *src, +asmlinkage void chacha_block_xor_neon(const struct chacha_state *state, + u8 *dst, const u8 *src, int nrounds); +asmlinkage void chacha_4block_xor_neon(const struct chacha_state *state, + u8 *dst, const u8 *src, int nrounds, unsigned int nbytes); -asmlinkage void hchacha_block_arm(const u32 *state, u32 *out, int nrounds); -asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds= ); +asmlinkage void hchacha_block_arm(const struct chacha_state *state, + u32 *out, int nrounds); +asmlinkage void hchacha_block_neon(const struct chacha_state *state, + u32 *out, int nrounds); =20 asmlinkage void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes, - const u32 *state, int nrounds); + const struct chacha_state *state, int nrounds); =20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_neon); =20 static inline bool neon_usable(void) { return static_branch_likely(&use_neon) && crypto_simd_usable(); } =20 -static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, +static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *s= rc, unsigned int bytes, int nrounds) { u8 buf[CHACHA_BLOCK_SIZE]; =20 while (bytes > CHACHA_BLOCK_SIZE) { @@ -44,26 +47,27 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8= *src, =20 chacha_4block_xor_neon(state, dst, src, nrounds, l); bytes -=3D l; src +=3D l; dst +=3D l; - state[12] +=3D DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); + state->x[12] +=3D DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); } if (bytes) { const u8 *s =3D src; u8 *d =3D dst; =20 if (bytes !=3D CHACHA_BLOCK_SIZE) s =3D d =3D memcpy(buf, src, bytes); chacha_block_xor_neon(state, d, s, nrounds); if (d !=3D dst) memcpy(dst, buf, bytes); - state[12]++; + state->x[12]++; } } =20 -void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) +void hchacha_block_arch(const struct chacha_state *state, u32 *stream, + int nrounds) { if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) { hchacha_block_arm(state, stream, nrounds); } else { kernel_neon_begin(); @@ -71,17 +75,17 @@ void hchacha_block_arch(const u32 *state, u32 *stream, = int nrounds) kernel_neon_end(); } } EXPORT_SYMBOL(hchacha_block_arch); =20 -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int by= tes, - int nrounds) +void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, + unsigned int bytes, int nrounds) { if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable() || bytes <=3D CHACHA_BLOCK_SIZE) { chacha_doarm(dst, src, bytes, state, nrounds); - state[12] +=3D DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE); + state->x[12] +=3D DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE); return; } =20 do { unsigned int todo =3D min_t(unsigned int, bytes, SZ_4K); diff --git a/arch/arm/lib/crypto/chacha-scalar-core.S b/arch/arm/lib/crypto= /chacha-scalar-core.S index 083fe1ab96d0..d20b5de755cc 100644 --- a/arch/arm/lib/crypto/chacha-scalar-core.S +++ b/arch/arm/lib/crypto/chacha-scalar-core.S @@ -365,11 +365,11 @@ .Ldone\@: .endm // _chacha =20 /* * void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes, - * const u32 *state, int nrounds); + * const struct chacha_state *state, int nrounds); */ ENTRY(chacha_doarm) cmp r2, #0 // len =3D=3D 0? reteq lr =20 @@ -405,11 +405,12 @@ ENTRY(chacha_doarm) 1: _chacha 12 b 0b ENDPROC(chacha_doarm) =20 /* - * void hchacha_block_arm(const u32 state[16], u32 out[8], int nrounds); + * void hchacha_block_arm(const struct chacha_state *state, + * u32 out[8], int nrounds); */ ENTRY(hchacha_block_arm) push {r1,r4-r11,lr} =20 cmp r2, #12 // ChaCha12 ? diff --git a/arch/arm64/lib/crypto/chacha-neon-glue.c b/arch/arm64/lib/cryp= to/chacha-neon-glue.c index 2b0de97a6daf..7b451b3c7240 100644 --- a/arch/arm64/lib/crypto/chacha-neon-glue.c +++ b/arch/arm64/lib/crypto/chacha-neon-glue.c @@ -26,19 +26,21 @@ =20 #include #include #include =20 -asmlinkage void chacha_block_xor_neon(u32 *state, u8 *dst, const u8 *src, - int nrounds); -asmlinkage void chacha_4block_xor_neon(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_block_xor_neon(const struct chacha_state *state, + u8 *dst, const u8 *src, int nrounds); +asmlinkage void chacha_4block_xor_neon(const struct chacha_state *state, + u8 *dst, const u8 *src, int nrounds, int bytes); -asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds= ); +asmlinkage void hchacha_block_neon(const struct chacha_state *state, + u32 *out, int nrounds); =20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); =20 -static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, +static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *s= rc, int bytes, int nrounds) { while (bytes > 0) { int l =3D min(bytes, CHACHA_BLOCK_SIZE * 5); =20 @@ -46,22 +48,23 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8= *src, u8 buf[CHACHA_BLOCK_SIZE]; =20 memcpy(buf, src, l); chacha_block_xor_neon(state, buf, buf, nrounds); memcpy(dst, buf, l); - state[12] +=3D 1; + state->x[12] +=3D 1; break; } chacha_4block_xor_neon(state, dst, src, nrounds, l); bytes -=3D l; src +=3D l; dst +=3D l; - state[12] +=3D DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); + state->x[12] +=3D DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); } } =20 -void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) +void hchacha_block_arch(const struct chacha_state *state, u32 *stream, + int nrounds) { if (!static_branch_likely(&have_neon) || !crypto_simd_usable()) { hchacha_block_generic(state, stream, nrounds); } else { kernel_neon_begin(); @@ -69,12 +72,12 @@ void hchacha_block_arch(const u32 *state, u32 *stream, = int nrounds) kernel_neon_end(); } } EXPORT_SYMBOL(hchacha_block_arch); =20 -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int by= tes, - int nrounds) +void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, + unsigned int bytes, int nrounds) { if (!static_branch_likely(&have_neon) || bytes <=3D CHACHA_BLOCK_SIZE || !crypto_simd_usable()) return chacha_crypt_generic(state, dst, src, bytes, nrounds); =20 diff --git a/arch/mips/lib/crypto/chacha-glue.c b/arch/mips/lib/crypto/chac= ha-glue.c index 334ecb29fb8f..75df4040cded 100644 --- a/arch/mips/lib/crypto/chacha-glue.c +++ b/arch/mips/lib/crypto/chacha-glue.c @@ -7,15 +7,17 @@ =20 #include #include #include =20 -asmlinkage void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_crypt_arch(struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int bytes, int nrounds); EXPORT_SYMBOL(chacha_crypt_arch); =20 -asmlinkage void hchacha_block_arch(const u32 *state, u32 *stream, int nrou= nds); +asmlinkage void hchacha_block_arch(const struct chacha_state *state, + u32 *stream, int nrounds); EXPORT_SYMBOL(hchacha_block_arch); =20 bool chacha_is_arch_optimized(void) { return true; diff --git a/arch/powerpc/lib/crypto/chacha-p10-glue.c b/arch/powerpc/lib/c= rypto/chacha-p10-glue.c index 51daeaf5d26e..a6e6a8da1b8b 100644 --- a/arch/powerpc/lib/crypto/chacha-p10-glue.c +++ b/arch/powerpc/lib/crypto/chacha-p10-glue.c @@ -12,12 +12,12 @@ #include #include #include #include =20 -asmlinkage void chacha_p10le_8x(u32 *state, u8 *dst, const u8 *src, - unsigned int len, int nrounds); +asmlinkage void chacha_p10le_8x(const struct chacha_state *state, u8 *dst, + const u8 *src, unsigned int len, int nrounds); =20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10); =20 static void vsx_begin(void) { @@ -29,35 +29,36 @@ static void vsx_end(void) { disable_kernel_vsx(); preempt_enable(); } =20 -static void chacha_p10_do_8x(u32 *state, u8 *dst, const u8 *src, +static void chacha_p10_do_8x(struct chacha_state *state, u8 *dst, const u8= *src, unsigned int bytes, int nrounds) { unsigned int l =3D bytes & ~0x0FF; =20 if (l > 0) { chacha_p10le_8x(state, dst, src, l, nrounds); bytes -=3D l; src +=3D l; dst +=3D l; - state[12] +=3D l / CHACHA_BLOCK_SIZE; + state->x[12] +=3D l / CHACHA_BLOCK_SIZE; } =20 if (bytes > 0) chacha_crypt_generic(state, dst, src, bytes, nrounds); } =20 -void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) +void hchacha_block_arch(const struct chacha_state *state, + u32 *stream, int nrounds) { hchacha_block_generic(state, stream, nrounds); } EXPORT_SYMBOL(hchacha_block_arch); =20 -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int by= tes, - int nrounds) +void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, + unsigned int bytes, int nrounds) { if (!static_branch_likely(&have_p10) || bytes <=3D CHACHA_BLOCK_SIZE || !crypto_simd_usable()) return chacha_crypt_generic(state, dst, src, bytes, nrounds); =20 diff --git a/arch/powerpc/lib/crypto/chacha-p10le-8x.S b/arch/powerpc/lib/c= rypto/chacha-p10le-8x.S index 17bedb66b822..b29562bd5d40 100644 --- a/arch/powerpc/lib/crypto/chacha-p10le-8x.S +++ b/arch/powerpc/lib/crypto/chacha-p10le-8x.S @@ -5,13 +5,10 @@ # Copyright 2023- IBM Corp. All rights reserved # #=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D # Written by Danny Tsen # -# chacha_p10le_8x(u32 *state, byte *dst, const byte *src, -# size_t len, int nrounds); -# # do rounds, 8 quarter rounds # 1. a +=3D b; d ^=3D a; d <<<=3D 16; # 2. c +=3D d; b ^=3D c; b <<<=3D 12; # 3. a +=3D b; d ^=3D a; d <<<=3D 8; # 4. c +=3D d; b ^=3D c; b <<<=3D 7 @@ -573,11 +570,12 @@ stxvw4x \S+47, 31, 16 =20 .endm =20 # -# chacha20_p10le_8x(u32 *state, byte *dst, const byte *src, size_t len, in= t nrounds); +# void chacha_p10le_8x(const struct chacha_state *state, u8 *dst, const u8= *src, +# unsigned int len, int nrounds); # SYM_FUNC_START(chacha_p10le_8x) .align 5 cmpdi 6, 0 ble Out_no_chacha diff --git a/arch/riscv/lib/crypto/chacha-riscv64-glue.c b/arch/riscv/lib/c= rypto/chacha-riscv64-glue.c index 1740e1ca3a94..57541621981e 100644 --- a/arch/riscv/lib/crypto/chacha-riscv64-glue.c +++ b/arch/riscv/lib/crypto/chacha-riscv64-glue.c @@ -13,21 +13,21 @@ #include #include =20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_zvkb); =20 -asmlinkage void chacha_zvkb(u32 state[16], const u8 *in, u8 *out, +asmlinkage void chacha_zvkb(struct chacha_state *state, const u8 *in, u8 *= out, size_t nblocks, int nrounds); =20 -void hchacha_block_arch(const u32 *state, u32 *out, int nrounds) +void hchacha_block_arch(const struct chacha_state *state, u32 *out, int nr= ounds) { hchacha_block_generic(state, out, nrounds); } EXPORT_SYMBOL(hchacha_block_arch); =20 -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int by= tes, - int nrounds) +void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, + unsigned int bytes, int nrounds) { u8 block_buffer[CHACHA_BLOCK_SIZE]; unsigned int full_blocks =3D bytes / CHACHA_BLOCK_SIZE; unsigned int tail_bytes =3D bytes % CHACHA_BLOCK_SIZE; =20 diff --git a/arch/riscv/lib/crypto/chacha-riscv64-zvkb.S b/arch/riscv/lib/c= rypto/chacha-riscv64-zvkb.S index ab4423b3880e..b777d0b4e379 100644 --- a/arch/riscv/lib/crypto/chacha-riscv64-zvkb.S +++ b/arch/riscv/lib/crypto/chacha-riscv64-zvkb.S @@ -130,19 +130,19 @@ vror.vi \b1, \b1, 32 - 7 vror.vi \b2, \b2, 32 - 7 vror.vi \b3, \b3, 32 - 7 .endm =20 -// void chacha_zvkb(u32 state[16], const u8 *in, u8 *out, size_t nblocks, -// int nrounds); +// void chacha_zvkb(struct chacha_state *state, const u8 *in, u8 *out, +// size_t nblocks, int nrounds); // // |nblocks| is the number of 64-byte blocks to process, and must be nonze= ro. // // |state| gives the ChaCha state matrix, including the 32-bit counter in -// state[12] following the RFC7539 convention; note that this differs from= the -// original Salsa20 paper which uses a 64-bit counter in state[12..13]. T= he -// updated 32-bit counter is written back to state[12] before returning. +// state->x[12] following the RFC7539 convention; note that this differs f= rom +// the original Salsa20 paper which uses a 64-bit counter in state->x[12..= 13]. +// The updated 32-bit counter is written back to state->x[12] before retur= ning. SYM_FUNC_START(chacha_zvkb) addi sp, sp, -96 sd s0, 0(sp) sd s1, 8(sp) sd s2, 16(sp) diff --git a/arch/s390/lib/crypto/chacha-glue.c b/arch/s390/lib/crypto/chac= ha-glue.c index b3ffaa555385..0a9fd50c1bd8 100644 --- a/arch/s390/lib/crypto/chacha-glue.c +++ b/arch/s390/lib/crypto/chacha-glue.c @@ -14,18 +14,19 @@ #include #include #include #include "chacha-s390.h" =20 -void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) +void hchacha_block_arch(const struct chacha_state *state, + u32 *stream, int nrounds) { /* TODO: implement hchacha_block_arch() in assembly */ hchacha_block_generic(state, stream, nrounds); } EXPORT_SYMBOL(hchacha_block_arch); =20 -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, +void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { /* s390 chacha20 implementation has 20 rounds hard-coded, * it cannot handle a block of data or less, but otherwise * it can handle data of arbitrary size @@ -34,15 +35,15 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *s= rc, chacha_crypt_generic(state, dst, src, bytes, nrounds); } else { DECLARE_KERNEL_FPU_ONSTACK32(vxstate); =20 kernel_fpu_begin(&vxstate, KERNEL_VXR); - chacha20_vx(dst, src, bytes, &state[4], &state[12]); + chacha20_vx(dst, src, bytes, &state->x[4], &state->x[12]); kernel_fpu_end(&vxstate, KERNEL_VXR); =20 - state[12] +=3D round_up(bytes, CHACHA_BLOCK_SIZE) / - CHACHA_BLOCK_SIZE; + state->x[12] +=3D round_up(bytes, CHACHA_BLOCK_SIZE) / + CHACHA_BLOCK_SIZE; } } EXPORT_SYMBOL(chacha_crypt_arch); =20 bool chacha_is_arch_optimized(void) diff --git a/arch/x86/lib/crypto/chacha_glue.c b/arch/x86/lib/crypto/chacha= _glue.c index 94fcefbc8827..6f00a56e3e9a 100644 --- a/arch/x86/lib/crypto/chacha_glue.c +++ b/arch/x86/lib/crypto/chacha_glue.c @@ -10,28 +10,37 @@ #include #include #include #include =20 -asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_block_xor_ssse3(const struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int len, int nrounds); -asmlinkage void chacha_4block_xor_ssse3(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_4block_xor_ssse3(const struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int len, int nrounds); -asmlinkage void hchacha_block_ssse3(const u32 *state, u32 *out, int nround= s); +asmlinkage void hchacha_block_ssse3(const struct chacha_state *state, + u32 *out, int nrounds); =20 -asmlinkage void chacha_2block_xor_avx2(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_2block_xor_avx2(const struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int len, int nrounds); -asmlinkage void chacha_4block_xor_avx2(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_4block_xor_avx2(const struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int len, int nrounds); -asmlinkage void chacha_8block_xor_avx2(u32 *state, u8 *dst, const u8 *src, +asmlinkage void chacha_8block_xor_avx2(const struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int len, int nrounds); =20 -asmlinkage void chacha_2block_xor_avx512vl(u32 *state, u8 *dst, const u8 *= src, +asmlinkage void chacha_2block_xor_avx512vl(const struct chacha_state *stat= e, + u8 *dst, const u8 *src, unsigned int len, int nrounds); -asmlinkage void chacha_4block_xor_avx512vl(u32 *state, u8 *dst, const u8 *= src, +asmlinkage void chacha_4block_xor_avx512vl(const struct chacha_state *stat= e, + u8 *dst, const u8 *src, unsigned int len, int nrounds); -asmlinkage void chacha_8block_xor_avx512vl(u32 *state, u8 *dst, const u8 *= src, +asmlinkage void chacha_8block_xor_avx512vl(const struct chacha_state *stat= e, + u8 *dst, const u8 *src, unsigned int len, int nrounds); =20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_simd); static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx2); static __ro_after_init DEFINE_STATIC_KEY_FALSE(chacha_use_avx512vl); @@ -40,86 +49,87 @@ static unsigned int chacha_advance(unsigned int len, un= signed int maxblocks) { len =3D min(len, maxblocks * CHACHA_BLOCK_SIZE); return round_up(len, CHACHA_BLOCK_SIZE) / CHACHA_BLOCK_SIZE; } =20 -static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src, +static void chacha_dosimd(struct chacha_state *state, u8 *dst, const u8 *s= rc, unsigned int bytes, int nrounds) { if (static_branch_likely(&chacha_use_avx512vl)) { while (bytes >=3D CHACHA_BLOCK_SIZE * 8) { chacha_8block_xor_avx512vl(state, dst, src, bytes, nrounds); bytes -=3D CHACHA_BLOCK_SIZE * 8; src +=3D CHACHA_BLOCK_SIZE * 8; dst +=3D CHACHA_BLOCK_SIZE * 8; - state[12] +=3D 8; + state->x[12] +=3D 8; } if (bytes > CHACHA_BLOCK_SIZE * 4) { chacha_8block_xor_avx512vl(state, dst, src, bytes, nrounds); - state[12] +=3D chacha_advance(bytes, 8); + state->x[12] +=3D chacha_advance(bytes, 8); return; } if (bytes > CHACHA_BLOCK_SIZE * 2) { chacha_4block_xor_avx512vl(state, dst, src, bytes, nrounds); - state[12] +=3D chacha_advance(bytes, 4); + state->x[12] +=3D chacha_advance(bytes, 4); return; } if (bytes) { chacha_2block_xor_avx512vl(state, dst, src, bytes, nrounds); - state[12] +=3D chacha_advance(bytes, 2); + state->x[12] +=3D chacha_advance(bytes, 2); return; } } =20 if (static_branch_likely(&chacha_use_avx2)) { while (bytes >=3D CHACHA_BLOCK_SIZE * 8) { chacha_8block_xor_avx2(state, dst, src, bytes, nrounds); bytes -=3D CHACHA_BLOCK_SIZE * 8; src +=3D CHACHA_BLOCK_SIZE * 8; dst +=3D CHACHA_BLOCK_SIZE * 8; - state[12] +=3D 8; + state->x[12] +=3D 8; } if (bytes > CHACHA_BLOCK_SIZE * 4) { chacha_8block_xor_avx2(state, dst, src, bytes, nrounds); - state[12] +=3D chacha_advance(bytes, 8); + state->x[12] +=3D chacha_advance(bytes, 8); return; } if (bytes > CHACHA_BLOCK_SIZE * 2) { chacha_4block_xor_avx2(state, dst, src, bytes, nrounds); - state[12] +=3D chacha_advance(bytes, 4); + state->x[12] +=3D chacha_advance(bytes, 4); return; } if (bytes > CHACHA_BLOCK_SIZE) { chacha_2block_xor_avx2(state, dst, src, bytes, nrounds); - state[12] +=3D chacha_advance(bytes, 2); + state->x[12] +=3D chacha_advance(bytes, 2); return; } } =20 while (bytes >=3D CHACHA_BLOCK_SIZE * 4) { chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds); bytes -=3D CHACHA_BLOCK_SIZE * 4; src +=3D CHACHA_BLOCK_SIZE * 4; dst +=3D CHACHA_BLOCK_SIZE * 4; - state[12] +=3D 4; + state->x[12] +=3D 4; } if (bytes > CHACHA_BLOCK_SIZE) { chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds); - state[12] +=3D chacha_advance(bytes, 4); + state->x[12] +=3D chacha_advance(bytes, 4); return; } if (bytes) { chacha_block_xor_ssse3(state, dst, src, bytes, nrounds); - state[12]++; + state->x[12]++; } } =20 -void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) +void hchacha_block_arch(const struct chacha_state *state, + u32 *stream, int nrounds) { if (!static_branch_likely(&chacha_use_simd)) { hchacha_block_generic(state, stream, nrounds); } else { kernel_fpu_begin(); @@ -127,12 +137,12 @@ void hchacha_block_arch(const u32 *state, u32 *stream= , int nrounds) kernel_fpu_end(); } } EXPORT_SYMBOL(hchacha_block_arch); =20 -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int by= tes, - int nrounds) +void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, + unsigned int bytes, int nrounds) { if (!static_branch_likely(&chacha_use_simd) || bytes <=3D CHACHA_BLOCK_SIZE) return chacha_crypt_generic(state, dst, src, bytes, nrounds); =20 diff --git a/crypto/chacha.c b/crypto/chacha.c index 28a8ad6197ab..73ce62a9ac22 100644 --- a/crypto/chacha.c +++ b/crypto/chacha.c @@ -48,28 +48,28 @@ static int chacha12_setkey(struct crypto_skcipher *tfm, static int chacha_stream_xor(struct skcipher_request *req, const struct chacha_ctx *ctx, const u8 *iv, bool arch) { struct skcipher_walk walk; - u32 state[16]; + struct chacha_state state; int err; =20 err =3D skcipher_walk_virt(&walk, req, false); =20 - chacha_init(state, ctx->key, iv); + chacha_init(&state, ctx->key, iv); =20 while (walk.nbytes > 0) { unsigned int nbytes =3D walk.nbytes; =20 if (nbytes < walk.total) nbytes =3D round_down(nbytes, CHACHA_BLOCK_SIZE); =20 if (arch) - chacha_crypt(state, walk.dst.virt.addr, + chacha_crypt(&state, walk.dst.virt.addr, walk.src.virt.addr, nbytes, ctx->nrounds); else - chacha_crypt_generic(state, walk.dst.virt.addr, + chacha_crypt_generic(&state, walk.dst.virt.addr, walk.src.virt.addr, nbytes, ctx->nrounds); err =3D skcipher_walk_done(&walk, walk.nbytes - nbytes); } =20 @@ -95,19 +95,19 @@ static int crypto_chacha_crypt_arch(struct skcipher_req= uest *req) static int crypto_xchacha_crypt(struct skcipher_request *req, bool arch) { struct crypto_skcipher *tfm =3D crypto_skcipher_reqtfm(req); const struct chacha_ctx *ctx =3D crypto_skcipher_ctx(tfm); struct chacha_ctx subctx; - u32 state[16]; + struct chacha_state state; u8 real_iv[16]; =20 /* Compute the subkey given the original key and first 128 nonce bits */ - chacha_init(state, ctx->key, req->iv); + chacha_init(&state, ctx->key, req->iv); if (arch) - hchacha_block(state, subctx.key, ctx->nrounds); + hchacha_block(&state, subctx.key, ctx->nrounds); else - hchacha_block_generic(state, subctx.key, ctx->nrounds); + hchacha_block_generic(&state, subctx.key, ctx->nrounds); subctx.nrounds =3D ctx->nrounds; =20 /* Build the real IV */ memcpy(&real_iv[0], req->iv + 24, 8); /* stream position */ memcpy(&real_iv[8], req->iv + 16, 8); /* remaining 64 nonce bits */ diff --git a/drivers/char/random.c b/drivers/char/random.c index 38f2fab29c56..9f876ed2655b 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -307,24 +307,24 @@ static void crng_reseed(struct work_struct *work) * * The returned ChaCha state contains within it a copy of the old * key value, at index 4, so the state should always be zeroed out * immediately after using in order to maintain forward secrecy. * If the state cannot be erased in a timely manner, then it is - * safer to set the random_data parameter to &chacha_state[4] so - * that this function overwrites it before returning. + * safer to set the random_data parameter to &chacha_state->x[4] + * so that this function overwrites it before returning. */ static void crng_fast_key_erasure(u8 key[CHACHA_KEY_SIZE], - u32 chacha_state[CHACHA_STATE_WORDS], + struct chacha_state *chacha_state, u8 *random_data, size_t random_data_len) { u8 first_block[CHACHA_BLOCK_SIZE]; =20 BUG_ON(random_data_len > 32); =20 chacha_init_consts(chacha_state); - memcpy(&chacha_state[4], key, CHACHA_KEY_SIZE); - memset(&chacha_state[12], 0, sizeof(u32) * 4); + memcpy(&chacha_state->x[4], key, CHACHA_KEY_SIZE); + memset(&chacha_state->x[12], 0, sizeof(u32) * 4); chacha20_block(chacha_state, first_block); =20 memcpy(key, first_block, CHACHA_KEY_SIZE); memcpy(random_data, first_block + CHACHA_KEY_SIZE, random_data_len); memzero_explicit(first_block, sizeof(first_block)); @@ -333,11 +333,11 @@ static void crng_fast_key_erasure(u8 key[CHACHA_KEY_S= IZE], /* * This function returns a ChaCha state that you may use for generating * random data. It also returns up to 32 bytes on its own of random data * that may be used; random_data_len may not be greater than 32. */ -static void crng_make_state(u32 chacha_state[CHACHA_STATE_WORDS], +static void crng_make_state(struct chacha_state *chacha_state, u8 *random_data, size_t random_data_len) { unsigned long flags; struct crng *crng; =20 @@ -393,38 +393,38 @@ static void crng_make_state(u32 chacha_state[CHACHA_S= TATE_WORDS], local_unlock_irqrestore(&crngs.lock, flags); } =20 static void _get_random_bytes(void *buf, size_t len) { - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; u8 tmp[CHACHA_BLOCK_SIZE]; size_t first_block_len; =20 if (!len) return; =20 first_block_len =3D min_t(size_t, 32, len); - crng_make_state(chacha_state, buf, first_block_len); + crng_make_state(&chacha_state, buf, first_block_len); len -=3D first_block_len; buf +=3D first_block_len; =20 while (len) { if (len < CHACHA_BLOCK_SIZE) { - chacha20_block(chacha_state, tmp); + chacha20_block(&chacha_state, tmp); memcpy(buf, tmp, len); memzero_explicit(tmp, sizeof(tmp)); break; } =20 - chacha20_block(chacha_state, buf); - if (unlikely(chacha_state[12] =3D=3D 0)) - ++chacha_state[13]; + chacha20_block(&chacha_state, buf); + if (unlikely(chacha_state.x[12] =3D=3D 0)) + ++chacha_state.x[13]; len -=3D CHACHA_BLOCK_SIZE; buf +=3D CHACHA_BLOCK_SIZE; } =20 - memzero_explicit(chacha_state, sizeof(chacha_state)); + memzero_explicit(&chacha_state, sizeof(chacha_state)); } =20 /* * This returns random bytes in arbitrary quantities. The quality of the * random bytes is good as /dev/urandom. In order to ensure that the @@ -439,11 +439,11 @@ void get_random_bytes(void *buf, size_t len) } EXPORT_SYMBOL(get_random_bytes); =20 static ssize_t get_random_bytes_user(struct iov_iter *iter) { - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; u8 block[CHACHA_BLOCK_SIZE]; size_t ret =3D 0, copied; =20 if (unlikely(!iov_iter_count(iter))) return 0; @@ -451,25 +451,26 @@ static ssize_t get_random_bytes_user(struct iov_iter = *iter) /* * Immediately overwrite the ChaCha key at index 4 with random * bytes, in case userspace causes copy_to_iter() below to sleep * forever, so that we still retain forward secrecy in that case. */ - crng_make_state(chacha_state, (u8 *)&chacha_state[4], CHACHA_KEY_SIZE); + crng_make_state(&chacha_state, (u8 *)&chacha_state.x[4], + CHACHA_KEY_SIZE); /* * However, if we're doing a read of len <=3D 32, we don't need to * use chacha_state after, so we can simply return those bytes to * the user directly. */ if (iov_iter_count(iter) <=3D CHACHA_KEY_SIZE) { - ret =3D copy_to_iter(&chacha_state[4], CHACHA_KEY_SIZE, iter); + ret =3D copy_to_iter(&chacha_state.x[4], CHACHA_KEY_SIZE, iter); goto out_zero_chacha; } =20 for (;;) { - chacha20_block(chacha_state, block); - if (unlikely(chacha_state[12] =3D=3D 0)) - ++chacha_state[13]; + chacha20_block(&chacha_state, block); + if (unlikely(chacha_state.x[12] =3D=3D 0)) + ++chacha_state.x[13]; =20 copied =3D copy_to_iter(block, sizeof(block), iter); ret +=3D copied; if (!iov_iter_count(iter) || copied !=3D sizeof(block)) break; @@ -482,11 +483,11 @@ static ssize_t get_random_bytes_user(struct iov_iter = *iter) } } =20 memzero_explicit(block, sizeof(block)); out_zero_chacha: - memzero_explicit(chacha_state, sizeof(chacha_state)); + memzero_explicit(&chacha_state, sizeof(chacha_state)); return ret ? ret : -EFAULT; } =20 /* * Batched entropy returns random integers. The quality of the random diff --git a/fs/bcachefs/checksum.c b/fs/bcachefs/checksum.c index d0a34a097b80..312fda4bb1b5 100644 --- a/fs/bcachefs/checksum.c +++ b/fs/bcachefs/checksum.c @@ -89,11 +89,11 @@ static void bch2_checksum_update(struct bch2_checksum_s= tate *state, const void * default: BUG(); } } =20 -static void bch2_chacha20_init(u32 state[CHACHA_STATE_WORDS], +static void bch2_chacha20_init(struct chacha_state *state, const struct bch_key *key, struct nonce nonce) { u32 key_words[CHACHA_KEY_SIZE / sizeof(u32)]; =20 BUILD_BUG_ON(sizeof(key_words) !=3D sizeof(*key)); @@ -107,15 +107,15 @@ static void bch2_chacha20_init(u32 state[CHACHA_STATE= _WORDS], } =20 static void bch2_chacha20(const struct bch_key *key, struct nonce nonce, void *data, size_t len) { - u32 state[CHACHA_STATE_WORDS]; + struct chacha_state state; =20 - bch2_chacha20_init(state, key, nonce); - chacha20_crypt(state, data, data, len); - memzero_explicit(state, sizeof(state)); + bch2_chacha20_init(&state, key, nonce); + chacha20_crypt(&state, data, data, len); + memzero_explicit(&state, sizeof(state)); } =20 static void bch2_poly1305_init(struct poly1305_desc_ctx *desc, struct bch_fs *c, struct nonce nonce) { @@ -255,18 +255,18 @@ struct bch_csum bch2_checksum_bio(struct bch_fs *c, u= nsigned type, int __bch2_encrypt_bio(struct bch_fs *c, unsigned type, struct nonce nonce, struct bio *bio) { struct bio_vec bv; struct bvec_iter iter; - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; int ret =3D 0; =20 if (bch2_fs_inconsistent_on(!c->chacha20_key_set, c, "attempting to encrypt without encryption key")) return -BCH_ERR_no_encryption_key; =20 - bch2_chacha20_init(chacha_state, &c->chacha20_key, nonce); + bch2_chacha20_init(&chacha_state, &c->chacha20_key, nonce); =20 bio_for_each_segment(bv, bio, iter) { void *p; =20 /* @@ -278,14 +278,14 @@ int __bch2_encrypt_bio(struct bch_fs *c, unsigned typ= e, ret =3D -EIO; break; } =20 p =3D bvec_kmap_local(&bv); - chacha20_crypt(chacha_state, p, p, bv.bv_len); + chacha20_crypt(&chacha_state, p, p, bv.bv_len); kunmap_local(p); } - memzero_explicit(chacha_state, sizeof(chacha_state)); + memzero_explicit(&chacha_state, sizeof(chacha_state)); return ret; } =20 struct bch_csum bch2_checksum_merge(unsigned type, struct bch_csum a, struct bch_csum b, size_t b_len) diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h index 58129e18cc31..64fb270f2bfc 100644 --- a/include/crypto/chacha.h +++ b/include/crypto/chacha.h @@ -28,20 +28,27 @@ #define CHACHA_STATE_WORDS (CHACHA_BLOCK_SIZE / sizeof(u32)) =20 /* 192-bit nonce, then 64-bit stream position */ #define XCHACHA_IV_SIZE 32 =20 -void chacha_block_generic(u32 *state, u8 *stream, int nrounds); -static inline void chacha20_block(u32 *state, u8 *stream) +struct chacha_state { + u32 x[CHACHA_STATE_WORDS]; +}; + +void chacha_block_generic(struct chacha_state *state, u8 *stream, int nrou= nds); +static inline void chacha20_block(struct chacha_state *state, u8 *stream) { chacha_block_generic(state, stream, 20); } =20 -void hchacha_block_arch(const u32 *state, u32 *out, int nrounds); -void hchacha_block_generic(const u32 *state, u32 *out, int nrounds); +void hchacha_block_arch(const struct chacha_state *state, u32 *out, + int nrounds); +void hchacha_block_generic(const struct chacha_state *state, u32 *out, + int nrounds); =20 -static inline void hchacha_block(const u32 *state, u32 *out, int nrounds) +static inline void hchacha_block(const struct chacha_state *state, u32 *ou= t, + int nrounds) { if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA)) hchacha_block_arch(state, out, nrounds); else hchacha_block_generic(state, out, nrounds); @@ -52,51 +59,53 @@ enum chacha_constants { /* expand 32-byte k */ CHACHA_CONSTANT_ND_3 =3D 0x3320646eU, CHACHA_CONSTANT_2_BY =3D 0x79622d32U, CHACHA_CONSTANT_TE_K =3D 0x6b206574U }; =20 -static inline void chacha_init_consts(u32 *state) +static inline void chacha_init_consts(struct chacha_state *state) { - state[0] =3D CHACHA_CONSTANT_EXPA; - state[1] =3D CHACHA_CONSTANT_ND_3; - state[2] =3D CHACHA_CONSTANT_2_BY; - state[3] =3D CHACHA_CONSTANT_TE_K; + state->x[0] =3D CHACHA_CONSTANT_EXPA; + state->x[1] =3D CHACHA_CONSTANT_ND_3; + state->x[2] =3D CHACHA_CONSTANT_2_BY; + state->x[3] =3D CHACHA_CONSTANT_TE_K; } =20 -static inline void chacha_init(u32 *state, const u32 *key, const u8 *iv) +static inline void chacha_init(struct chacha_state *state, + const u32 *key, const u8 *iv) { chacha_init_consts(state); - state[4] =3D key[0]; - state[5] =3D key[1]; - state[6] =3D key[2]; - state[7] =3D key[3]; - state[8] =3D key[4]; - state[9] =3D key[5]; - state[10] =3D key[6]; - state[11] =3D key[7]; - state[12] =3D get_unaligned_le32(iv + 0); - state[13] =3D get_unaligned_le32(iv + 4); - state[14] =3D get_unaligned_le32(iv + 8); - state[15] =3D get_unaligned_le32(iv + 12); + state->x[4] =3D key[0]; + state->x[5] =3D key[1]; + state->x[6] =3D key[2]; + state->x[7] =3D key[3]; + state->x[8] =3D key[4]; + state->x[9] =3D key[5]; + state->x[10] =3D key[6]; + state->x[11] =3D key[7]; + state->x[12] =3D get_unaligned_le32(iv + 0); + state->x[13] =3D get_unaligned_le32(iv + 4); + state->x[14] =3D get_unaligned_le32(iv + 8); + state->x[15] =3D get_unaligned_le32(iv + 12); } =20 -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, +void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds); -void chacha_crypt_generic(u32 *state, u8 *dst, const u8 *src, +void chacha_crypt_generic(struct chacha_state *state, u8 *dst, const u8 *s= rc, unsigned int bytes, int nrounds); =20 -static inline void chacha_crypt(u32 *state, u8 *dst, const u8 *src, +static inline void chacha_crypt(struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA)) chacha_crypt_arch(state, dst, src, bytes, nrounds); else chacha_crypt_generic(state, dst, src, bytes, nrounds); } =20 -static inline void chacha20_crypt(u32 *state, u8 *dst, const u8 *src, - unsigned int bytes) +static inline void chacha20_crypt(struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int bytes) { chacha_crypt(state, dst, src, bytes, 20); } =20 #if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA) diff --git a/lib/crypto/chacha.c b/lib/crypto/chacha.c index 3cdda3b5ee06..a7f5eb091839 100644 --- a/lib/crypto/chacha.c +++ b/lib/crypto/chacha.c @@ -11,12 +11,13 @@ #include #include #include #include =20 -static void chacha_permute(u32 *x, int nrounds) +static void chacha_permute(struct chacha_state *state, int nrounds) { + u32 *x =3D state->x; int i; =20 /* whitelist the allowed round counts */ WARN_ON_ONCE(nrounds !=3D 20 && nrounds !=3D 12); =20 @@ -63,52 +64,54 @@ static void chacha_permute(u32 *x, int nrounds) } } =20 /** * chacha_block_generic - generate one keystream block and increment block= counter - * @state: input state matrix (16 32-bit words) + * @state: input state matrix * @stream: output keystream block (64 bytes) * @nrounds: number of rounds (20 or 12; 20 is recommended) * * This is the ChaCha core, a function from 64-byte strings to 64-byte str= ings. * The caller has already converted the endianness of the input. This fun= ction * also handles incrementing the block counter in the input matrix. */ -void chacha_block_generic(u32 *state, u8 *stream, int nrounds) +void chacha_block_generic(struct chacha_state *state, u8 *stream, int nrou= nds) { - u32 x[16]; + struct chacha_state permuted_state; int i; =20 - memcpy(x, state, 64); + memcpy(permuted_state.x, state->x, 64); =20 - chacha_permute(x, nrounds); + chacha_permute(&permuted_state, nrounds); =20 - for (i =3D 0; i < ARRAY_SIZE(x); i++) - put_unaligned_le32(x[i] + state[i], &stream[i * sizeof(u32)]); + for (i =3D 0; i < ARRAY_SIZE(state->x); i++) + put_unaligned_le32(permuted_state.x[i] + state->x[i], + &stream[i * sizeof(u32)]); =20 - state[12]++; + state->x[12]++; } EXPORT_SYMBOL(chacha_block_generic); =20 /** * hchacha_block_generic - abbreviated ChaCha core, for XChaCha - * @state: input state matrix (16 32-bit words) + * @state: input state matrix * @stream: output (8 32-bit words) * @nrounds: number of rounds (20 or 12; 20 is recommended) * * HChaCha is the ChaCha equivalent of HSalsa and is an intermediate step * towards XChaCha (see https://cr.yp.to/snuffle/xsalsa-20081128.pdf). HC= haCha * skips the final addition of the initial state, and outputs only certain= words * of the state. It should not be used for streaming directly. */ -void hchacha_block_generic(const u32 *state, u32 *stream, int nrounds) +void hchacha_block_generic(const struct chacha_state *state, + u32 *stream, int nrounds) { - u32 x[16]; + struct chacha_state permuted_state; =20 - memcpy(x, state, 64); + memcpy(permuted_state.x, state->x, 64); =20 - chacha_permute(x, nrounds); + chacha_permute(&permuted_state, nrounds); =20 - memcpy(&stream[0], &x[0], 16); - memcpy(&stream[4], &x[12], 16); + memcpy(&stream[0], &permuted_state.x[0], 16); + memcpy(&stream[4], &permuted_state.x[12], 16); } EXPORT_SYMBOL(hchacha_block_generic); diff --git a/lib/crypto/chacha20poly1305-selftest.c b/lib/crypto/chacha20po= ly1305-selftest.c index 2ea61c28be4f..e4c85bc5a6d7 100644 --- a/lib/crypto/chacha20poly1305-selftest.c +++ b/lib/crypto/chacha20poly1305-selftest.c @@ -8830,11 +8830,11 @@ chacha20poly1305_encrypt_bignonce(u8 *dst, const u8= *src, const size_t src_len, const u8 nonce[12], const u8 key[CHACHA20POLY1305_KEY_SIZE]) { const u8 *pad0 =3D page_address(ZERO_PAGE(0)); struct poly1305_desc_ctx poly1305_state; - u32 chacha20_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha20_state; union { u8 block0[POLY1305_KEY_SIZE]; __le64 lens[2]; } b =3D {{ 0 }}; u8 bottom_row[16] =3D { 0 }; @@ -8842,16 +8842,16 @@ chacha20poly1305_encrypt_bignonce(u8 *dst, const u8= *src, const size_t src_len, int i; =20 memcpy(&bottom_row[4], nonce, 12); for (i =3D 0; i < 8; ++i) le_key[i] =3D get_unaligned_le32(key + sizeof(le_key[i]) * i); - chacha_init(chacha20_state, le_key, bottom_row); - chacha20_crypt(chacha20_state, b.block0, b.block0, sizeof(b.block0)); + chacha_init(&chacha20_state, le_key, bottom_row); + chacha20_crypt(&chacha20_state, b.block0, b.block0, sizeof(b.block0)); poly1305_init(&poly1305_state, b.block0); poly1305_update(&poly1305_state, ad, ad_len); poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf); - chacha20_crypt(chacha20_state, dst, src, src_len); + chacha20_crypt(&chacha20_state, dst, src, src_len); poly1305_update(&poly1305_state, dst, src_len); poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf); b.lens[0] =3D cpu_to_le64(ad_len); b.lens[1] =3D cpu_to_le64(src_len); poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index 9cfa886f1f89..ed81f0658956 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -30,11 +30,12 @@ static void chacha_load_key(u32 *k, const u8 *in) k[5] =3D get_unaligned_le32(in + 20); k[6] =3D get_unaligned_le32(in + 24); k[7] =3D get_unaligned_le32(in + 28); } =20 -static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce) +static void xchacha_init(struct chacha_state *chacha_state, + const u8 *key, const u8 *nonce) { u32 k[CHACHA_KEY_WORDS]; u8 iv[CHACHA_IV_SIZE]; =20 memset(iv, 0, 8); @@ -52,11 +53,12 @@ static void xchacha_init(u32 *chacha_state, const u8 *k= ey, const u8 *nonce) memzero_explicit(iv, sizeof(iv)); } =20 static void __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, - const u8 *ad, const size_t ad_len, u32 *chacha_state) + const u8 *ad, const size_t ad_len, + struct chacha_state *chacha_state) { const u8 *pad0 =3D page_address(ZERO_PAGE(0)); struct poly1305_desc_ctx poly1305_state; union { u8 block0[POLY1305_KEY_SIZE]; @@ -80,30 +82,31 @@ __chacha20poly1305_encrypt(u8 *dst, const u8 *src, cons= t size_t src_len, b.lens[1] =3D cpu_to_le64(src_len); poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); =20 poly1305_final(&poly1305_state, dst + src_len); =20 - memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32)); + memzero_explicit(chacha_state, sizeof(*chacha_state)); memzero_explicit(&b, sizeof(b)); } =20 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, const size_t ad_len, const u64 nonce, const u8 key[CHACHA20POLY1305_KEY_SIZE]) { - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; u32 k[CHACHA_KEY_WORDS]; __le64 iv[2]; =20 chacha_load_key(k, key); =20 iv[0] =3D 0; iv[1] =3D cpu_to_le64(nonce); =20 - chacha_init(chacha_state, k, (u8 *)iv); - __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); + chacha_init(&chacha_state, k, (u8 *)iv); + __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, + &chacha_state); =20 memzero_explicit(iv, sizeof(iv)); memzero_explicit(k, sizeof(k)); } EXPORT_SYMBOL(chacha20poly1305_encrypt); @@ -111,20 +114,22 @@ EXPORT_SYMBOL(chacha20poly1305_encrypt); void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_le= n, const u8 *ad, const size_t ad_len, const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], const u8 key[CHACHA20POLY1305_KEY_SIZE]) { - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; =20 - xchacha_init(chacha_state, key, nonce); - __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); + xchacha_init(&chacha_state, key, nonce); + __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, + &chacha_state); } EXPORT_SYMBOL(xchacha20poly1305_encrypt); =20 static bool __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, - const u8 *ad, const size_t ad_len, u32 *chacha_state) + const u8 *ad, const size_t ad_len, + struct chacha_state *chacha_state) { const u8 *pad0 =3D page_address(ZERO_PAGE(0)); struct poly1305_desc_ctx poly1305_state; size_t dst_len; int ret; @@ -167,25 +172,25 @@ __chacha20poly1305_decrypt(u8 *dst, const u8 *src, co= nst size_t src_len, bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, const size_t ad_len, const u64 nonce, const u8 key[CHACHA20POLY1305_KEY_SIZE]) { - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; u32 k[CHACHA_KEY_WORDS]; __le64 iv[2]; bool ret; =20 chacha_load_key(k, key); =20 iv[0] =3D 0; iv[1] =3D cpu_to_le64(nonce); =20 - chacha_init(chacha_state, k, (u8 *)iv); + chacha_init(&chacha_state, k, (u8 *)iv); ret =3D __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, - chacha_state); + &chacha_state); =20 - memzero_explicit(chacha_state, sizeof(chacha_state)); + memzero_explicit(&chacha_state, sizeof(chacha_state)); memzero_explicit(iv, sizeof(iv)); memzero_explicit(k, sizeof(k)); return ret; } EXPORT_SYMBOL(chacha20poly1305_decrypt); @@ -193,15 +198,15 @@ EXPORT_SYMBOL(chacha20poly1305_decrypt); bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_le= n, const u8 *ad, const size_t ad_len, const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], const u8 key[CHACHA20POLY1305_KEY_SIZE]) { - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; =20 - xchacha_init(chacha_state, key, nonce); + xchacha_init(&chacha_state, key, nonce); return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, - chacha_state); + &chacha_state); } EXPORT_SYMBOL(xchacha20poly1305_decrypt); =20 static bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, @@ -211,11 +216,11 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatter= list *src, const u8 key[CHACHA20POLY1305_KEY_SIZE], int encrypt) { const u8 *pad0 =3D page_address(ZERO_PAGE(0)); struct poly1305_desc_ctx poly1305_state; - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; struct sg_mapping_iter miter; size_t partial =3D 0; unsigned int flags; bool ret =3D true; int sl; @@ -238,12 +243,12 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatter= list *src, chacha_load_key(b.k, key); =20 b.iv[0] =3D 0; b.iv[1] =3D cpu_to_le64(nonce); =20 - chacha_init(chacha_state, b.k, (u8 *)b.iv); - chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); + chacha_init(&chacha_state, b.k, (u8 *)b.iv); + chacha20_crypt(&chacha_state, b.block0, pad0, sizeof(b.block0)); poly1305_init(&poly1305_state, b.block0); =20 if (unlikely(ad_len)) { poly1305_update(&poly1305_state, ad, ad_len); if (ad_len & 0xf) @@ -274,17 +279,17 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatter= list *src, if (likely(length >=3D CHACHA_BLOCK_SIZE || length =3D=3D sl)) { size_t l =3D length; =20 if (unlikely(length < sl)) l &=3D ~(CHACHA_BLOCK_SIZE - 1); - chacha20_crypt(chacha_state, addr, addr, l); + chacha20_crypt(&chacha_state, addr, addr, l); addr +=3D l; length -=3D l; } =20 if (unlikely(length > 0)) { - chacha20_crypt(chacha_state, b.chacha_stream, pad0, + chacha20_crypt(&chacha_state, b.chacha_stream, pad0, CHACHA_BLOCK_SIZE); crypto_xor(addr, b.chacha_stream, length); partial =3D length; } =20 @@ -321,11 +326,11 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatter= list *src, sizeof(b.mac[1]), src_len, !encrypt); ret =3D encrypt || !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE); } =20 - memzero_explicit(chacha_state, sizeof(chacha_state)); + memzero_explicit(&chacha_state, sizeof(chacha_state)); memzero_explicit(&b, sizeof(b)); =20 return ret; } =20 diff --git a/lib/crypto/libchacha.c b/lib/crypto/libchacha.c index cc1be0496eb9..ebcca381e248 100644 --- a/lib/crypto/libchacha.c +++ b/lib/crypto/libchacha.c @@ -10,11 +10,11 @@ #include =20 #include // for crypto_xor_cpy #include =20 -void chacha_crypt_generic(u32 *state, u8 *dst, const u8 *src, +void chacha_crypt_generic(struct chacha_state *state, u8 *dst, const u8 *s= rc, unsigned int bytes, int nrounds) { /* aligned to potentially speed up crypto_xor() */ u8 stream[CHACHA_BLOCK_SIZE] __aligned(sizeof(long)); =20 diff --git a/tools/testing/crypto/chacha20-s390/test-cipher.c b/tools/testi= ng/crypto/chacha20-s390/test-cipher.c index 35ea65c54ffa..827507844e8f 100644 --- a/tools/testing/crypto/chacha20-s390/test-cipher.c +++ b/tools/testing/crypto/chacha20-s390/test-cipher.c @@ -48,11 +48,11 @@ struct skcipher_def { }; =20 /* Perform cipher operations with the chacha lib */ static int test_lib_chacha(u8 *revert, u8 *cipher, u8 *plain) { - u32 chacha_state[CHACHA_STATE_WORDS]; + struct chacha_state chacha_state; u8 iv[16], key[32]; u64 start, end; =20 memset(key, 'X', sizeof(key)); memset(iv, 'I', sizeof(iv)); @@ -64,14 +64,14 @@ static int test_lib_chacha(u8 *revert, u8 *cipher, u8 *= plain) print_hex_dump(KERN_INFO, "iv: ", DUMP_PREFIX_OFFSET, 16, 1, iv, 16, 1); } =20 /* Encrypt */ - chacha_init(chacha_state, (u32 *)key, iv); + chacha_init(&chacha_state, (u32 *)key, iv); =20 start =3D ktime_get_ns(); - chacha_crypt_arch(chacha_state, cipher, plain, data_size, 20); + chacha_crypt_arch(&chacha_state, cipher, plain, data_size, 20); end =3D ktime_get_ns(); =20 =20 if (debug) print_hex_dump(KERN_INFO, "encr:", DUMP_PREFIX_OFFSET, @@ -79,14 +79,14 @@ static int test_lib_chacha(u8 *revert, u8 *cipher, u8 *= plain) (data_size > 64 ? 64 : data_size), 1); =20 pr_info("lib encryption took: %lld nsec", end - start); =20 /* Decrypt */ - chacha_init(chacha_state, (u32 *)key, iv); + chacha_init(&chacha_state, (u32 *)key, iv); =20 start =3D ktime_get_ns(); - chacha_crypt_arch(chacha_state, revert, cipher, data_size, 20); + chacha_crypt_arch(&chacha_state, revert, cipher, data_size, 20); end =3D ktime_get_ns(); =20 if (debug) print_hex_dump(KERN_INFO, "decr:", DUMP_PREFIX_OFFSET, 16, 1, revert, --=20 2.49.0 From nobody Tue Dec 16 08:20:33 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4F3F626F46F; Mon, 5 May 2025 18:19:23 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746469164; cv=none; b=TtAMRXFfkXdY8xjIaunTdULMwMR1uqKDqO2fcF7Ew6lbV+EmQp9US71sQiEwRwk/goCidiMzIuHhOfTfwltNNWKPkc8sof7oIPBKB/Crnncc/8afuJ+dW4SoGJ4fmiuYq/Bh3K+qkb8vopdfcyAkwKWERx4BzlZwK8NYUP8l4Lg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746469164; c=relaxed/simple; bh=b/9JYhuN7J7zBR7ary1s1WvggcmHZ7V9W2w6tJgLRtw=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=hFInGnphNeiTFw/MsKURA2nMFKmqWqjLTtemwDft6VcxsnBMf0Qwx1S1FEsdXYqG2K7EVyg8SgbE4W/KB6cF70SgFhu7i59eqSw1hd4XcymtdAELIvsnjCbEf0LUlYntDnFFahWtT9c3CTgh3fnx07HDPohptMWulaRzau4bcIQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=pDa53t6D; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="pDa53t6D" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 7E8EEC4CEEE; Mon, 5 May 2025 18:19:23 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1746469163; bh=b/9JYhuN7J7zBR7ary1s1WvggcmHZ7V9W2w6tJgLRtw=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=pDa53t6DCjWYM/LmZqGF9bnJJ573ftIBSUQsp8IeHuT5N7tEcbBprRacM6EAolgWa KnM4f1Ecr4wsjUKSjvQrsrEOI61Z1Nenrn/tYHyD0VxpliwIloUv60+380sygQKeFn FxwIMuAGqXJ/SCu+mhdkAvNtZ/VV+Hpcs9PRhT0j3hKSDgORzS83PV0rP1IGISbSpL aTZfyBqlR/WkgLr3KGW3DlBmln+QJkH1npbWh+vtLcej0hH78R6O4qdEbRHUlUrbSw ke0NPkusn8yL0Rp2/trfxhRJ3n8yZzmTAV2C0r7cHVP5MRtIDrFszeZGaEVvf/LP32 dPGvxobHLcyGg== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-bcachefs@vger.kernel.org, "Jason A . Donenfeld " , Theodore Ts'o Subject: [PATCH 2/4] crypto: lib/chacha - use struct assignment to copy state Date: Mon, 5 May 2025 11:18:22 -0700 Message-ID: <20250505181824.647138-3-ebiggers@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250505181824.647138-1-ebiggers@kernel.org> References: <20250505181824.647138-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Eric Biggers Use struct assignment instead of memcpy() in lib/crypto/chacha.c where appropriate. No functional change. Signed-off-by: Eric Biggers --- lib/crypto/chacha.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/crypto/chacha.c b/lib/crypto/chacha.c index a7f5eb091839..ae50e441f9fb 100644 --- a/lib/crypto/chacha.c +++ b/lib/crypto/chacha.c @@ -74,15 +74,13 @@ static void chacha_permute(struct chacha_state *state, = int nrounds) * The caller has already converted the endianness of the input. This fun= ction * also handles incrementing the block counter in the input matrix. */ void chacha_block_generic(struct chacha_state *state, u8 *stream, int nrou= nds) { - struct chacha_state permuted_state; + struct chacha_state permuted_state =3D *state; int i; =20 - memcpy(permuted_state.x, state->x, 64); - chacha_permute(&permuted_state, nrounds); =20 for (i =3D 0; i < ARRAY_SIZE(state->x); i++) put_unaligned_le32(permuted_state.x[i] + state->x[i], &stream[i * sizeof(u32)]); @@ -103,13 +101,11 @@ EXPORT_SYMBOL(chacha_block_generic); * of the state. It should not be used for streaming directly. */ void hchacha_block_generic(const struct chacha_state *state, u32 *stream, int nrounds) { - struct chacha_state permuted_state; - - memcpy(permuted_state.x, state->x, 64); + struct chacha_state permuted_state =3D *state; =20 chacha_permute(&permuted_state, nrounds); =20 memcpy(&stream[0], &permuted_state.x[0], 16); memcpy(&stream[4], &permuted_state.x[12], 16); --=20 2.49.0 From nobody Tue Dec 16 08:20:33 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 5CCC12566D9; Mon, 5 May 2025 18:19:24 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746469164; cv=none; b=E9dmV5/FKJQaGjTSmx0M+jBsi5MesgpKmA8Uxzk6ffqqWBvDhkVoDWs9kH0We6E2xa66woMwFRYpUNognritkWIivN5l2zvhLGiWy9Z3j7pFVEabNPjeaoWNNh62iH2yhBgL2vbNC0q+Ohu7Q6/BbMrJhzJyNrHO4zvzaPKJ/fw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746469164; c=relaxed/simple; bh=mMc5BO3KaWzzAhqaUs9saVNrK9FbauW+oehe6PKy+Aw=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=ARe09G9pKwPVroAza6KARXGdcSXuCW08kjG0+mg5IAendOput5Dzpg9etv8AfMTGA6rVjNXIb40zcnm4O7MgFM2OgFPHZOwjSu+9PN97gCPkzpQXlTkgIqz0h1V8PNIgA6kVwfXXhKWFpeOKuvvT0siNhm5Pjw21tMrTmqzmZdE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=IdccGkzZ; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="IdccGkzZ" Received: by smtp.kernel.org (Postfix) with ESMTPSA id EA062C4CEF2; Mon, 5 May 2025 18:19:23 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1746469164; bh=mMc5BO3KaWzzAhqaUs9saVNrK9FbauW+oehe6PKy+Aw=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=IdccGkzZQnmV4j3zL0qW2HEPMFQO7Lbww2PjWdjgi17/TjJjHo3Rabs0jOB2pLBFR BNystxBX4qdoUELqjrc8NRgh4C6DjNKebmGQQv4FS+8mziYF0YiD5rENIf9z+kTYQo d/5anHb8+6JLhMlascA+6GaHteo22tUvh09FwyEJKqzORMaKb3DgTTSNzhVmuXmmE0 Jr1NQbeZonSiGvqI9p9VDbFjLVPnTv/FDbVSc+/MEpQj9N6o9vOdldAqt/tkvpP1fX RlHYn2sj2qI+hhlBZdf50Md2DXuONGO0ckmyZ6fIjPKP/cDcPY76pUPCDIqrLnfjvV E3yOy0SyJb5hA== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-bcachefs@vger.kernel.org, "Jason A . Donenfeld " , Theodore Ts'o Subject: [PATCH 3/4] crypto: lib/chacha - add strongly-typed state zeroization Date: Mon, 5 May 2025 11:18:23 -0700 Message-ID: <20250505181824.647138-4-ebiggers@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250505181824.647138-1-ebiggers@kernel.org> References: <20250505181824.647138-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Eric Biggers Now that the ChaCha state matrix is strongly-typed, add a helper function chacha_zeroize_state() which zeroizes it. Then convert all applicable callers to use it instead of direct memzero_explicit. No functional changes. Signed-off-by: Eric Biggers --- drivers/char/random.c | 4 ++-- fs/bcachefs/checksum.c | 4 ++-- include/crypto/chacha.h | 6 ++++++ lib/crypto/chacha20poly1305.c | 6 +++--- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/drivers/char/random.c b/drivers/char/random.c index 9f876ed2655b..5f22a08101f6 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -420,11 +420,11 @@ static void _get_random_bytes(void *buf, size_t len) ++chacha_state.x[13]; len -=3D CHACHA_BLOCK_SIZE; buf +=3D CHACHA_BLOCK_SIZE; } =20 - memzero_explicit(&chacha_state, sizeof(chacha_state)); + chacha_zeroize_state(&chacha_state); } =20 /* * This returns random bytes in arbitrary quantities. The quality of the * random bytes is good as /dev/urandom. In order to ensure that the @@ -483,11 +483,11 @@ static ssize_t get_random_bytes_user(struct iov_iter = *iter) } } =20 memzero_explicit(block, sizeof(block)); out_zero_chacha: - memzero_explicit(&chacha_state, sizeof(chacha_state)); + chacha_zeroize_state(&chacha_state); return ret ? ret : -EFAULT; } =20 /* * Batched entropy returns random integers. The quality of the random diff --git a/fs/bcachefs/checksum.c b/fs/bcachefs/checksum.c index 312fda4bb1b5..a4df8eba75f3 100644 --- a/fs/bcachefs/checksum.c +++ b/fs/bcachefs/checksum.c @@ -111,11 +111,11 @@ static void bch2_chacha20(const struct bch_key *key, = struct nonce nonce, { struct chacha_state state; =20 bch2_chacha20_init(&state, key, nonce); chacha20_crypt(&state, data, data, len); - memzero_explicit(&state, sizeof(state)); + chacha_zeroize_state(&state); } =20 static void bch2_poly1305_init(struct poly1305_desc_ctx *desc, struct bch_fs *c, struct nonce nonce) { @@ -281,11 +281,11 @@ int __bch2_encrypt_bio(struct bch_fs *c, unsigned typ= e, =20 p =3D bvec_kmap_local(&bv); chacha20_crypt(&chacha_state, p, p, bv.bv_len); kunmap_local(p); } - memzero_explicit(&chacha_state, sizeof(chacha_state)); + chacha_zeroize_state(&chacha_state); return ret; } =20 struct bch_csum bch2_checksum_merge(unsigned type, struct bch_csum a, struct bch_csum b, size_t b_len) diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h index 64fb270f2bfc..7c2e6c68919b 100644 --- a/include/crypto/chacha.h +++ b/include/crypto/chacha.h @@ -14,10 +14,11 @@ =20 #ifndef _CRYPTO_CHACHA_H #define _CRYPTO_CHACHA_H =20 #include +#include #include =20 /* 32-bit stream position, then 96-bit nonce (RFC7539 convention) */ #define CHACHA_IV_SIZE 16 =20 @@ -106,10 +107,15 @@ static inline void chacha20_crypt(struct chacha_state= *state, u8 *dst, const u8 *src, unsigned int bytes) { chacha_crypt(state, dst, src, bytes, 20); } =20 +static inline void chacha_zeroize_state(struct chacha_state *state) +{ + memzero_explicit(state, sizeof(*state)); +} + #if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA) bool chacha_is_arch_optimized(void); #else static inline bool chacha_is_arch_optimized(void) { diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index ed81f0658956..2e7bbc1a67ea 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -82,11 +82,11 @@ __chacha20poly1305_encrypt(u8 *dst, const u8 *src, cons= t size_t src_len, b.lens[1] =3D cpu_to_le64(src_len); poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); =20 poly1305_final(&poly1305_state, dst + src_len); =20 - memzero_explicit(chacha_state, sizeof(*chacha_state)); + chacha_zeroize_state(chacha_state); memzero_explicit(&b, sizeof(b)); } =20 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, const size_t ad_len, @@ -186,11 +186,11 @@ bool chacha20poly1305_decrypt(u8 *dst, const u8 *src,= const size_t src_len, =20 chacha_init(&chacha_state, k, (u8 *)iv); ret =3D __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, &chacha_state); =20 - memzero_explicit(&chacha_state, sizeof(chacha_state)); + chacha_zeroize_state(&chacha_state); memzero_explicit(iv, sizeof(iv)); memzero_explicit(k, sizeof(k)); return ret; } EXPORT_SYMBOL(chacha20poly1305_decrypt); @@ -326,11 +326,11 @@ bool chacha20poly1305_crypt_sg_inplace(struct scatter= list *src, sizeof(b.mac[1]), src_len, !encrypt); ret =3D encrypt || !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE); } =20 - memzero_explicit(&chacha_state, sizeof(chacha_state)); + chacha_zeroize_state(&chacha_state); memzero_explicit(&b, sizeof(b)); =20 return ret; } =20 --=20 2.49.0 From nobody Tue Dec 16 08:20:33 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 33BE92701C9; Mon, 5 May 2025 18:19:24 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746469166; cv=none; b=ehCmW45Y3DS3hn1n6/Y//hEfjGR0OIrHW/ZmOlm+cMddBCqC2FKOxD0FuR0toAf73+pW1GdTbsksO4DdWMeG0ixu3BtmDwip8GsanFunltWNtrpG0/sABVHTXuLM1YrYsMDfAP0T9neFUgalV2cxuDJJrA96e/zybghAoKnWqrU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746469166; c=relaxed/simple; bh=uslrcW8Rjh8Vtp1jwkYS+gns1NocaM1ysr80C17p+10=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=pEPv8byS10PIOTLiXhNxV42YstkyOUn+jyXvUACFDYmBFKPVUFXSumU/KtRTy7l3Fy2TlYnQWPKJxgxzbt+bMch7HmBl6H/nisBPuZSGHcrFupynaMyPXu73hHbqGjROkHIpS+1qViyCPkzeaUnnqOQpCgy+gu6Le8bgkUYMoRs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=qjLazIDP; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="qjLazIDP" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 6333BC4CEEF; Mon, 5 May 2025 18:19:24 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1746469164; bh=uslrcW8Rjh8Vtp1jwkYS+gns1NocaM1ysr80C17p+10=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=qjLazIDPDkm/C095UjRbYZiH2s5N7gpqYt3hMK3JSviK+dnBWkx8DT34stTwvRWa1 uUtdO0i0UEOfXdoCL9knyjf2nfgVeTT2W+a8+QuZeFSaGqfhUQssF3B+Zu2avSGq67 xd9zRgby65l75mxRUMgJ8mIB34uhHYW9XqvipSYtBOSCKEkOTEuzevxNKHxZbkUy8c kr0qCDllV/5OLtzYqjvDqCzj+9Le6N5jMBFbPAIGwstCX3DSTeBpVC8eixdn3W1XoB 9UB09lQN5DAc8rMGmN5uWYfVcXSoWJTv53n9I1Z2RSVh3pov8ByWjjIvVLik3+NFBX foUo9yue0AqVg== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org, linux-bcachefs@vger.kernel.org, "Jason A . Donenfeld " , Theodore Ts'o Subject: [PATCH 4/4] crypto: lib/chacha - add array bounds to function prototypes Date: Mon, 5 May 2025 11:18:24 -0700 Message-ID: <20250505181824.647138-5-ebiggers@kernel.org> X-Mailer: git-send-email 2.49.0 In-Reply-To: <20250505181824.647138-1-ebiggers@kernel.org> References: <20250505181824.647138-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Eric Biggers Add explicit array bounds to the function prototypes for the parameters that didn't already get handled by the conversion to use chacha_state: - chacha_block_*(): Change 'u8 *out' or 'u8 *stream' to u8 out[CHACHA_BLOCK_SIZE]. - hchacha_block_*(): Change 'u32 *out' or 'u32 *stream' to u32 out[HCHACHA_OUT_WORDS]. - chacha_init(): Change 'const u32 *key' to 'const u32 key[CHACHA_KEY_WORDS]'. Change 'const u8 *iv' to 'const u8 iv[CHACHA_IV_SIZE]'. No functional changes. This just makes it clear when fixed-size arrays are expected. Signed-off-by: Eric Biggers --- arch/arm/lib/crypto/chacha-glue.c | 12 ++++----- arch/arm/lib/crypto/chacha-scalar-core.S | 2 +- arch/arm64/lib/crypto/chacha-neon-glue.c | 10 ++++---- arch/mips/lib/crypto/chacha-glue.c | 2 +- arch/powerpc/lib/crypto/chacha-p10-glue.c | 4 +-- arch/riscv/lib/crypto/chacha-riscv64-glue.c | 3 ++- arch/s390/lib/crypto/chacha-glue.c | 4 +-- arch/x86/lib/crypto/chacha_glue.c | 8 +++--- crypto/chacha.c | 4 +-- include/crypto/chacha.h | 27 ++++++++++++--------- lib/crypto/chacha.c | 15 ++++++------ lib/crypto/chacha20poly1305.c | 2 -- 12 files changed, 49 insertions(+), 44 deletions(-) diff --git a/arch/arm/lib/crypto/chacha-glue.c b/arch/arm/lib/crypto/chacha= -glue.c index 0c2b4c62d484..88ec96415283 100644 --- a/arch/arm/lib/crypto/chacha-glue.c +++ b/arch/arm/lib/crypto/chacha-glue.c @@ -21,13 +21,13 @@ asmlinkage void chacha_block_xor_neon(const struct chac= ha_state *state, u8 *dst, const u8 *src, int nrounds); asmlinkage void chacha_4block_xor_neon(const struct chacha_state *state, u8 *dst, const u8 *src, int nrounds, unsigned int nbytes); asmlinkage void hchacha_block_arm(const struct chacha_state *state, - u32 *out, int nrounds); + u32 out[HCHACHA_OUT_WORDS], int nrounds); asmlinkage void hchacha_block_neon(const struct chacha_state *state, - u32 *out, int nrounds); + u32 out[HCHACHA_OUT_WORDS], int nrounds); =20 asmlinkage void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes, const struct chacha_state *state, int nrounds); =20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_neon); @@ -62,18 +62,18 @@ static void chacha_doneon(struct chacha_state *state, u= 8 *dst, const u8 *src, memcpy(dst, buf, bytes); state->x[12]++; } } =20 -void hchacha_block_arch(const struct chacha_state *state, u32 *stream, - int nrounds) +void hchacha_block_arch(const struct chacha_state *state, + u32 out[HCHACHA_OUT_WORDS], int nrounds) { if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) { - hchacha_block_arm(state, stream, nrounds); + hchacha_block_arm(state, out, nrounds); } else { kernel_neon_begin(); - hchacha_block_neon(state, stream, nrounds); + hchacha_block_neon(state, out, nrounds); kernel_neon_end(); } } EXPORT_SYMBOL(hchacha_block_arch); =20 diff --git a/arch/arm/lib/crypto/chacha-scalar-core.S b/arch/arm/lib/crypto= /chacha-scalar-core.S index d20b5de755cc..4951df05c158 100644 --- a/arch/arm/lib/crypto/chacha-scalar-core.S +++ b/arch/arm/lib/crypto/chacha-scalar-core.S @@ -406,11 +406,11 @@ ENTRY(chacha_doarm) b 0b ENDPROC(chacha_doarm) =20 /* * void hchacha_block_arm(const struct chacha_state *state, - * u32 out[8], int nrounds); + * u32 out[HCHACHA_OUT_WORDS], int nrounds); */ ENTRY(hchacha_block_arm) push {r1,r4-r11,lr} =20 cmp r2, #12 // ChaCha12 ? diff --git a/arch/arm64/lib/crypto/chacha-neon-glue.c b/arch/arm64/lib/cryp= to/chacha-neon-glue.c index 7b451b3c7240..d0188f974ca5 100644 --- a/arch/arm64/lib/crypto/chacha-neon-glue.c +++ b/arch/arm64/lib/crypto/chacha-neon-glue.c @@ -32,11 +32,11 @@ asmlinkage void chacha_block_xor_neon(const struct chac= ha_state *state, u8 *dst, const u8 *src, int nrounds); asmlinkage void chacha_4block_xor_neon(const struct chacha_state *state, u8 *dst, const u8 *src, int nrounds, int bytes); asmlinkage void hchacha_block_neon(const struct chacha_state *state, - u32 *out, int nrounds); + u32 out[HCHACHA_OUT_WORDS], int nrounds); =20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); =20 static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *s= rc, int bytes, int nrounds) @@ -59,18 +59,18 @@ static void chacha_doneon(struct chacha_state *state, u= 8 *dst, const u8 *src, dst +=3D l; state->x[12] +=3D DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); } } =20 -void hchacha_block_arch(const struct chacha_state *state, u32 *stream, - int nrounds) +void hchacha_block_arch(const struct chacha_state *state, + u32 out[HCHACHA_OUT_WORDS], int nrounds) { if (!static_branch_likely(&have_neon) || !crypto_simd_usable()) { - hchacha_block_generic(state, stream, nrounds); + hchacha_block_generic(state, out, nrounds); } else { kernel_neon_begin(); - hchacha_block_neon(state, stream, nrounds); + hchacha_block_neon(state, out, nrounds); kernel_neon_end(); } } EXPORT_SYMBOL(hchacha_block_arch); =20 diff --git a/arch/mips/lib/crypto/chacha-glue.c b/arch/mips/lib/crypto/chac= ha-glue.c index 75df4040cded..88c097594eb0 100644 --- a/arch/mips/lib/crypto/chacha-glue.c +++ b/arch/mips/lib/crypto/chacha-glue.c @@ -13,11 +13,11 @@ asmlinkage void chacha_crypt_arch(struct chacha_state *= state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds); EXPORT_SYMBOL(chacha_crypt_arch); =20 asmlinkage void hchacha_block_arch(const struct chacha_state *state, - u32 *stream, int nrounds); + u32 out[HCHACHA_OUT_WORDS], int nrounds); EXPORT_SYMBOL(hchacha_block_arch); =20 bool chacha_is_arch_optimized(void) { return true; diff --git a/arch/powerpc/lib/crypto/chacha-p10-glue.c b/arch/powerpc/lib/c= rypto/chacha-p10-glue.c index a6e6a8da1b8b..fcd23c6f1590 100644 --- a/arch/powerpc/lib/crypto/chacha-p10-glue.c +++ b/arch/powerpc/lib/crypto/chacha-p10-glue.c @@ -47,13 +47,13 @@ static void chacha_p10_do_8x(struct chacha_state *state= , u8 *dst, const u8 *src, if (bytes > 0) chacha_crypt_generic(state, dst, src, bytes, nrounds); } =20 void hchacha_block_arch(const struct chacha_state *state, - u32 *stream, int nrounds) + u32 out[HCHACHA_OUT_WORDS], int nrounds) { - hchacha_block_generic(state, stream, nrounds); + hchacha_block_generic(state, out, nrounds); } EXPORT_SYMBOL(hchacha_block_arch); =20 void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) diff --git a/arch/riscv/lib/crypto/chacha-riscv64-glue.c b/arch/riscv/lib/c= rypto/chacha-riscv64-glue.c index 57541621981e..8c3f11d79be3 100644 --- a/arch/riscv/lib/crypto/chacha-riscv64-glue.c +++ b/arch/riscv/lib/crypto/chacha-riscv64-glue.c @@ -16,11 +16,12 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_zvkb); =20 asmlinkage void chacha_zvkb(struct chacha_state *state, const u8 *in, u8 *= out, size_t nblocks, int nrounds); =20 -void hchacha_block_arch(const struct chacha_state *state, u32 *out, int nr= ounds) +void hchacha_block_arch(const struct chacha_state *state, + u32 out[HCHACHA_OUT_WORDS], int nrounds) { hchacha_block_generic(state, out, nrounds); } EXPORT_SYMBOL(hchacha_block_arch); =20 diff --git a/arch/s390/lib/crypto/chacha-glue.c b/arch/s390/lib/crypto/chac= ha-glue.c index 0a9fd50c1bd8..f95ba3483bbc 100644 --- a/arch/s390/lib/crypto/chacha-glue.c +++ b/arch/s390/lib/crypto/chacha-glue.c @@ -15,14 +15,14 @@ #include #include #include "chacha-s390.h" =20 void hchacha_block_arch(const struct chacha_state *state, - u32 *stream, int nrounds) + u32 out[HCHACHA_OUT_WORDS], int nrounds) { /* TODO: implement hchacha_block_arch() in assembly */ - hchacha_block_generic(state, stream, nrounds); + hchacha_block_generic(state, out, nrounds); } EXPORT_SYMBOL(hchacha_block_arch); =20 void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) diff --git a/arch/x86/lib/crypto/chacha_glue.c b/arch/x86/lib/crypto/chacha= _glue.c index 6f00a56e3e9a..10b2c945f541 100644 --- a/arch/x86/lib/crypto/chacha_glue.c +++ b/arch/x86/lib/crypto/chacha_glue.c @@ -17,11 +17,11 @@ asmlinkage void chacha_block_xor_ssse3(const struct cha= cha_state *state, unsigned int len, int nrounds); asmlinkage void chacha_4block_xor_ssse3(const struct chacha_state *state, u8 *dst, const u8 *src, unsigned int len, int nrounds); asmlinkage void hchacha_block_ssse3(const struct chacha_state *state, - u32 *out, int nrounds); + u32 out[HCHACHA_OUT_WORDS], int nrounds); =20 asmlinkage void chacha_2block_xor_avx2(const struct chacha_state *state, u8 *dst, const u8 *src, unsigned int len, int nrounds); asmlinkage void chacha_4block_xor_avx2(const struct chacha_state *state, @@ -125,17 +125,17 @@ static void chacha_dosimd(struct chacha_state *state,= u8 *dst, const u8 *src, state->x[12]++; } } =20 void hchacha_block_arch(const struct chacha_state *state, - u32 *stream, int nrounds) + u32 out[HCHACHA_OUT_WORDS], int nrounds) { if (!static_branch_likely(&chacha_use_simd)) { - hchacha_block_generic(state, stream, nrounds); + hchacha_block_generic(state, out, nrounds); } else { kernel_fpu_begin(); - hchacha_block_ssse3(state, stream, nrounds); + hchacha_block_ssse3(state, out, nrounds); kernel_fpu_end(); } } EXPORT_SYMBOL(hchacha_block_arch); =20 diff --git a/crypto/chacha.c b/crypto/chacha.c index 73ce62a9ac22..c3a11f4e2d13 100644 --- a/crypto/chacha.c +++ b/crypto/chacha.c @@ -44,12 +44,12 @@ static int chacha12_setkey(struct crypto_skcipher *tfm, { return chacha_setkey(tfm, key, keysize, 12); } =20 static int chacha_stream_xor(struct skcipher_request *req, - const struct chacha_ctx *ctx, const u8 *iv, - bool arch) + const struct chacha_ctx *ctx, + const u8 iv[CHACHA_IV_SIZE], bool arch) { struct skcipher_walk walk; struct chacha_state state; int err; =20 diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h index 7c2e6c68919b..91f6b4cf561c 100644 --- a/include/crypto/chacha.h +++ b/include/crypto/chacha.h @@ -24,32 +24,36 @@ =20 #define CHACHA_KEY_SIZE 32 #define CHACHA_BLOCK_SIZE 64 #define CHACHAPOLY_IV_SIZE 12 =20 -#define CHACHA_STATE_WORDS (CHACHA_BLOCK_SIZE / sizeof(u32)) +#define CHACHA_KEY_WORDS 8 +#define CHACHA_STATE_WORDS 16 +#define HCHACHA_OUT_WORDS 8 =20 /* 192-bit nonce, then 64-bit stream position */ #define XCHACHA_IV_SIZE 32 =20 struct chacha_state { u32 x[CHACHA_STATE_WORDS]; }; =20 -void chacha_block_generic(struct chacha_state *state, u8 *stream, int nrou= nds); -static inline void chacha20_block(struct chacha_state *state, u8 *stream) +void chacha_block_generic(struct chacha_state *state, + u8 out[CHACHA_BLOCK_SIZE], int nrounds); +static inline void chacha20_block(struct chacha_state *state, + u8 out[CHACHA_BLOCK_SIZE]) { - chacha_block_generic(state, stream, 20); + chacha_block_generic(state, out, 20); } =20 -void hchacha_block_arch(const struct chacha_state *state, u32 *out, - int nrounds); -void hchacha_block_generic(const struct chacha_state *state, u32 *out, - int nrounds); +void hchacha_block_arch(const struct chacha_state *state, + u32 out[HCHACHA_OUT_WORDS], int nrounds); +void hchacha_block_generic(const struct chacha_state *state, + u32 out[HCHACHA_OUT_WORDS], int nrounds); =20 -static inline void hchacha_block(const struct chacha_state *state, u32 *ou= t, - int nrounds) +static inline void hchacha_block(const struct chacha_state *state, + u32 out[HCHACHA_OUT_WORDS], int nrounds) { if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA)) hchacha_block_arch(state, out, nrounds); else hchacha_block_generic(state, out, nrounds); @@ -69,11 +73,12 @@ static inline void chacha_init_consts(struct chacha_sta= te *state) state->x[2] =3D CHACHA_CONSTANT_2_BY; state->x[3] =3D CHACHA_CONSTANT_TE_K; } =20 static inline void chacha_init(struct chacha_state *state, - const u32 *key, const u8 *iv) + const u32 key[CHACHA_KEY_WORDS], + const u8 iv[CHACHA_IV_SIZE]) { chacha_init_consts(state); state->x[4] =3D key[0]; state->x[5] =3D key[1]; state->x[6] =3D key[2]; diff --git a/lib/crypto/chacha.c b/lib/crypto/chacha.c index ae50e441f9fb..ced87dd31a97 100644 --- a/lib/crypto/chacha.c +++ b/lib/crypto/chacha.c @@ -65,49 +65,50 @@ static void chacha_permute(struct chacha_state *state, = int nrounds) } =20 /** * chacha_block_generic - generate one keystream block and increment block= counter * @state: input state matrix - * @stream: output keystream block (64 bytes) + * @out: output keystream block * @nrounds: number of rounds (20 or 12; 20 is recommended) * * This is the ChaCha core, a function from 64-byte strings to 64-byte str= ings. * The caller has already converted the endianness of the input. This fun= ction * also handles incrementing the block counter in the input matrix. */ -void chacha_block_generic(struct chacha_state *state, u8 *stream, int nrou= nds) +void chacha_block_generic(struct chacha_state *state, + u8 out[CHACHA_BLOCK_SIZE], int nrounds) { struct chacha_state permuted_state =3D *state; int i; =20 chacha_permute(&permuted_state, nrounds); =20 for (i =3D 0; i < ARRAY_SIZE(state->x); i++) put_unaligned_le32(permuted_state.x[i] + state->x[i], - &stream[i * sizeof(u32)]); + &out[i * sizeof(u32)]); =20 state->x[12]++; } EXPORT_SYMBOL(chacha_block_generic); =20 /** * hchacha_block_generic - abbreviated ChaCha core, for XChaCha * @state: input state matrix - * @stream: output (8 32-bit words) + * @out: the output words * @nrounds: number of rounds (20 or 12; 20 is recommended) * * HChaCha is the ChaCha equivalent of HSalsa and is an intermediate step * towards XChaCha (see https://cr.yp.to/snuffle/xsalsa-20081128.pdf). HC= haCha * skips the final addition of the initial state, and outputs only certain= words * of the state. It should not be used for streaming directly. */ void hchacha_block_generic(const struct chacha_state *state, - u32 *stream, int nrounds) + u32 out[HCHACHA_OUT_WORDS], int nrounds) { struct chacha_state permuted_state =3D *state; =20 chacha_permute(&permuted_state, nrounds); =20 - memcpy(&stream[0], &permuted_state.x[0], 16); - memcpy(&stream[4], &permuted_state.x[12], 16); + memcpy(&out[0], &permuted_state.x[0], 16); + memcpy(&out[4], &permuted_state.x[12], 16); } EXPORT_SYMBOL(hchacha_block_generic); diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index 2e7bbc1a67ea..fbd3690e2531 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -16,12 +16,10 @@ #include #include #include #include =20 -#define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) - static void chacha_load_key(u32 *k, const u8 *in) { k[0] =3D get_unaligned_le32(in); k[1] =3D get_unaligned_le32(in + 4); k[2] =3D get_unaligned_le32(in + 8); --=20 2.49.0