From nobody Sat Jul 25 16:48:31 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E565B3CA4B5; Wed, 15 Jul 2026 22:12:41 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153563; cv=none; b=tUXhzMb6dKzCOsAU7WOqR3NtwpX80nsH/jrrVdo7fRKexjgkAbDISDe0wOWRScCrnBlez9fZo7xpFdAUkzFWBGyjFV8mz3iGDKrZichnR+gXSVgRl+uQjbKM7BwPmUCtkijS5Pxs2Tbx7uxNZXcfPm4Lt9hfuUCo92cynnjyptQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153563; c=relaxed/simple; bh=vcyKlJQHPIPuRbCVeQ/g/VqSOzD+7aQkkW0DwKizObg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=O2kKf4puyG3xMNNnG4q6QFEFhECR/7YM7vVNuEy423rSKXQyfPQ6M7u/5n+jNgmd4MZRt4QwTrqVIn2Ng6SqQUtFlBABnfqXNMGjVt6owt4u5O5tqC2DKwvkzT+3HgGa+nhraT5UyWJef+Z8cjNGOvcof7eh98HbVSnlahF7boM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=OZsEF732; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="OZsEF732" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 906D81F00A3A; Wed, 15 Jul 2026 22:12:41 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784153561; bh=KBF7xnXXskNhZdtypClNKzXj4L01VOmK1znJnUbNEc4=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=OZsEF732wTphtRuoaSdJLJMHBxSAdepl1Y32kmvMGU3AY62vT4/B2FxgjWJUkDAeK WT1LK2epy6ps6Eoar15cew3GEIz1Kw4l3jwtCBnoVygNP38/YHnH4MgD0cF/934GOV AjwLQlQVFgRMHt8Y3PxSmLHCzRXttlrsTMP+ytkq1lQM+bp1p8t43CGCnUS8I14NAA +Y3596xHsFPISX1YBQR3eIX0BOPJq6Cit4XE0TdW2mC1RH/M29jdmshJFEQD5dKzL+ Dt/E1BPIuhAd2vYHGakRUrEE0LA8eK2mw64Fa+ZQIM31p9HDeMpDWREMXUh1b0OOMa oG3atdCgIqMCQ== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , Thomas Huth , Eric Biggers Subject: [PATCH v2 01/13] crypto: xts - Split out __xts_verify_key() helper Date: Wed, 15 Jul 2026 15:11:41 -0700 Message-ID: <20260715221153.246410-2-ebiggers@kernel.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260715221153.246410-1-ebiggers@kernel.org> References: <20260715221153.246410-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Make the AES-XTS key verification code callable by the crypto library by splitting out a helper function that doesn't use crypto_skcipher. Reviewed-by: Thomas Huth Signed-off-by: Eric Biggers --- include/crypto/xts.h | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/include/crypto/xts.h b/include/crypto/xts.h index 15b16c4853d8..16aef89f021f 100644 --- a/include/crypto/xts.h +++ b/include/crypto/xts.h @@ -7,9 +7,9 @@ #include =20 #define XTS_BLOCK_SIZE 16 +#define XTS_FORBID_WEAK_KEYS (1 << 0) =20 -static inline int xts_verify_key(struct crypto_skcipher *tfm, - const u8 *key, unsigned int keylen) +static inline int __xts_verify_key(const u8 *key, size_t keylen, int flags) { /* * key consists of keys of equal size concatenated, therefore @@ -29,12 +29,22 @@ static inline int xts_verify_key(struct crypto_skcipher= *tfm, * Ensure that the AES and tweak key are not identical when * in FIPS mode or the FORBID_WEAK_KEYS flag is set. */ - if ((fips_enabled || (crypto_skcipher_get_flags(tfm) & - CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) && + if ((fips_enabled || (flags & XTS_FORBID_WEAK_KEYS)) && !crypto_memneq(key, key + (keylen / 2), keylen / 2)) return -EINVAL; =20 return 0; } =20 +static inline int xts_verify_key(struct crypto_skcipher *tfm, const u8 *ke= y, + unsigned int keylen) +{ + int flags =3D (crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) ? + XTS_FORBID_WEAK_KEYS : + 0; + + return __xts_verify_key(key, keylen, flags); +} + #endif /* _CRYPTO_XTS_H */ --=20 2.55.0 From nobody Sat Jul 25 16:48:31 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 8C68C3CF04C; Wed, 15 Jul 2026 22:12:42 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153563; cv=none; b=CtPP/9El8QJcjiBbXk7pTcpM9YRvdhBe6hF+pNPl/3Te1ERj4Dq5Pdy+O+T+aBThLDlDrZfTzfeS8s0bXVNL4ee5o3sfJWQfTVGBL12L80L6AXTG6pSnQeJIgPITmFqQFlGy3ZxA+eGFXnz3WdJ5XJ5w9vfoog3czBJDxvWhn5U= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153563; c=relaxed/simple; bh=ABorqzBlOOPp3QQWidBwfsEeS3oqORKRTy7z0HfEP5o=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=j89hF91iw5UguV/Ab9W+7Cbp8lIuuDSJEkBQoA4jQOHbP2PWCg8abCTWwap5HuR65IegAmdULSxAGwPS7lyd7CDWXjqrqmywpqNJ5Js/rmoD2v3UW6FaEsEQJjJN5bjGQFTBC2ADO8ne5oB0HTW51mQkgVQR2oXglwjL/7An8Ps= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=kNDMR2dP; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="kNDMR2dP" Received: by smtp.kernel.org (Postfix) with ESMTPSA id E86231F00A3D; Wed, 15 Jul 2026 22:12:41 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784153562; bh=13QMH3FKypAK5+AwFIW+R1CUkb3y40Q+4DTaJMAQIW4=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=kNDMR2dPOjUGJNm8Q7ncEtGMuxdB3xkREV85JdFK01JU0uNeiojJ13ZxKl0rKM1GS Dh92kCoPKJhYxRp9c+vJX6saK+wTfmODzA3jEwlRfLWCxSOjK4JdOzGdq4jxCReJTI MCa6j2enHUkER7wFnZCKDzp2FZlyj6YuJLfKogzUsZanQ5z0biTsJnc07Aygr5uMkJ 5kHYUc1PKpnCjFiU2TXEIk3+5fFDJGC224n0SxB5vAtNyCayWUYguLwar+iAjo9fuQ 2QsdzSCpcMHmiUN7WOSSF82ur7omavhzgbMgwFWTFihfFp4KpJYxxmYz+5O2KJioEX h0O+I1PCH5q1Q== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , Thomas Huth , Eric Biggers Subject: [PATCH v2 02/13] lib/crypto: aes: Add ECB support Date: Wed, 15 Jul 2026 15:11:42 -0700 Message-ID: <20260715221153.246410-3-ebiggers@kernel.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260715221153.246410-1-ebiggers@kernel.org> References: <20260715221153.246410-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Add support for AES-ECB to the crypto library. This will be used to provide a streamlined implementation of the "ecb(aes)" crypto_skcipher algorithm. fs/crypto/keysetup_v1.c will also use aes_ecb_encrypt() directly. As usual, the architecture-optimized AES-ECB code will be migrated into the library as well (using the hooks provided in this commit), eliminating lots of repetitive boilerplate code. ECB is obsolete of course, but we need this for parity with the traditional API and to support some odd users of ECB in the kernel. Initial test coverage is provided by the crypto_skcipher support added in a later commit. I'm planning a KUnit test suite as well. Create a documentation file libcrypto-unauth-encryption.rst to hold the documentation for this and other unauthenticated encryption modes. Reviewed-by: Thomas Huth Signed-off-by: Eric Biggers --- .../crypto/libcrypto-unauth-encryption.rst | 28 ++++++++++ Documentation/crypto/libcrypto.rst | 1 + include/crypto/aes-ecb.h | 49 ++++++++++++++++ lib/crypto/Kconfig | 6 ++ lib/crypto/aes.c | 56 +++++++++++++++++++ lib/crypto/tests/Kconfig | 1 + 6 files changed, 141 insertions(+) create mode 100644 Documentation/crypto/libcrypto-unauth-encryption.rst create mode 100644 include/crypto/aes-ecb.h diff --git a/Documentation/crypto/libcrypto-unauth-encryption.rst b/Documen= tation/crypto/libcrypto-unauth-encryption.rst new file mode 100644 index 000000000000..6a3397a8adae --- /dev/null +++ b/Documentation/crypto/libcrypto-unauth-encryption.rst @@ -0,0 +1,28 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Unauthenticated encryption +=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D + +These APIs provide support for unauthenticated encryption and decryption, +including bare stream ciphers and other length-preserving algorithms such = as +block ciphers in XTS mode. The legitimate use cases for these algorithms = are: + +- Support for legacy protocols that really should have chosen an authentic= ated + mode (or even another primitive entirely) but didn't. + +- Internal components of authenticated modes. For example, AES-CTR is use= d by + AES-GCM and AES-CCM internally. + +- Storage encryption that cannot accommodate ciphertext expansion. Usually + AES-XTS is used for this. + +- Stream ciphers for key derivation and random number generation. + +Besides the above, these shouldn't be used. + +AES-ECB +------- + +This API provides support for AES in the ECB mode of operation. + +.. kernel-doc:: include/crypto/aes-ecb.h diff --git a/Documentation/crypto/libcrypto.rst b/Documentation/crypto/libc= rypto.rst index 0733e603d229..33312c3ffad4 100644 --- a/Documentation/crypto/libcrypto.rst +++ b/Documentation/crypto/libcrypto.rst @@ -162,5 +162,6 @@ API documentation libcrypto-blockcipher libcrypto-hash libcrypto-signature + libcrypto-unauth-encryption libcrypto-utils sha3 diff --git a/include/crypto/aes-ecb.h b/include/crypto/aes-ecb.h new file mode 100644 index 000000000000..8cac1f8794c7 --- /dev/null +++ b/include/crypto/aes-ecb.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-ECB unauthenticated encryption and decryption + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_ECB_H +#define _CRYPTO_AES_ECB_H + +#include + +/** + * aes_ecb_encrypt() - Encrypt data using AES-ECB + * @dst: The destination buffer. Can be in-place or out-of-place. For ot= her + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to encrypt. Must be a multiple of AES_BLOCK_SIZE. + * @key: The key, already prepared using aes_preparekey() or aes_prepareen= ckey() + * + * ECB mode is insecure by itself. This function exists only for compatib= ility + * with legacy protocols and for internal use by other modes. + * + * This supports incremental encryption, but the length of each chunk must= be a + * multiple of AES_BLOCK_SIZE. + * + * Context: Any context. + */ +void aes_ecb_encrypt(u8 *dst, const u8 *src, size_t len, aes_encrypt_arg k= ey); + +/** + * aes_ecb_decrypt() - Decrypt data using AES-ECB + * @dst: The destination buffer. Can be in-place or out-of-place. For ot= her + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to decrypt. Must be a multiple of AES_BLOCK_SIZE. + * @key: The key, already prepared using aes_preparekey() + * + * ECB mode is insecure by itself. This function exists only for compatib= ility + * with legacy protocols and for internal use by other modes. + * + * This supports incremental decryption, but the length of each chunk must= be a + * multiple of AES_BLOCK_SIZE. + * + * Context: Any context. + */ +void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t len, + const struct aes_key *key); + +#endif /* _CRYPTO_AES_ECB_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 83d4c95e079e..26514c181a7f 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -35,6 +35,12 @@ config CRYPTO_LIB_AES_CBC_MACS this if your module uses any of the functions from . =20 +config CRYPTO_LIB_AES_ECB + tristate + select CRYPTO_LIB_AES + help + The AES-ECB library functions. + config CRYPTO_LIB_AESGCM tristate select CRYPTO_LIB_AES diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index ca733f15b2a8..e2f1ebf81405 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -5,6 +5,7 @@ */ =20 #include +#include #include #include #include @@ -737,6 +738,61 @@ static inline void aes_cmac_fips_test(void) } #endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */ =20 +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB) +/* + * Hooks for optimized AES-ECB implementations, overridable by the archite= cture. + * They are called with len > 0 && len % AES_BLOCK_SIZE =3D=3D 0. Returni= ng false + * causes the fallback implementation to be used instead. + */ +#ifndef aes_ecb_encrypt_arch +static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len, + const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_ecb_decrypt_arch +static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len, + const struct aes_key *key) +{ + return false; +} +#endif + +void aes_ecb_encrypt(u8 *dst, const u8 *src, size_t len, aes_encrypt_arg k= ey) +{ + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len =3D round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len =3D=3D 0)) + return; + + if (likely(aes_ecb_encrypt_arch(dst, src, len, key.enc_key))) + return; + + for (size_t i =3D 0; i < len; i +=3D AES_BLOCK_SIZE) + aes_encrypt(key, &dst[i], &src[i]); +} +EXPORT_SYMBOL_GPL(aes_ecb_encrypt); + +void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t len, + const struct aes_key *key) +{ + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len =3D round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len =3D=3D 0)) + return; + + if (likely(aes_ecb_decrypt_arch(dst, src, len, key))) + return; + + for (size_t i =3D 0; i < len; i +=3D AES_BLOCK_SIZE) + aes_decrypt(key, &dst[i], &src[i]); +} +EXPORT_SYMBOL_GPL(aes_ecb_decrypt); +#endif /* CONFIG_CRYPTO_LIB_AES_ECB */ + static int __init aes_mod_init(void) { #ifdef aes_mod_init_arch diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index 9409c1a935c3..a57e87dbb1b1 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -145,6 +145,7 @@ config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT tristate "Enable all crypto library code for KUnit tests" depends on KUNIT select CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_AES_ECB select CRYPTO_LIB_BLAKE2B select CRYPTO_LIB_CHACHA20POLY1305 select CRYPTO_LIB_CURVE25519 --=20 2.55.0 From nobody Sat Jul 25 16:48:31 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 9D7F33CC7EB; Wed, 15 Jul 2026 22:12:42 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153564; cv=none; b=ZS3Q0xB5s8WRtPdj68ANy+7IPkHEZH9egforUWVE8sA15Tm9nhyK6dNeU0txMUwVwLb6IgMvSWTXpCePW2EFlalZhlMqt09CsUmSQ68wnESSQ/sO8aBoNWnY6CCoZgE/7tYAj7+e9VTIrQddwFXGjG62Wxi2Zdaj8ig44luc7KY= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153564; c=relaxed/simple; bh=bEc4j+/Hpz0s8hDCfudInMgzO3POsBWfyo85PjPnav0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=QOFNJWl3ZLWPy9joMb+EFGnLlXEGaFs8FE26gXMyi+ZDXZVZ900yDqrC/a1rVEAcpcvvap6aFaWu6yliIhZ4/SkLedsKzMmEQ3IYreTwwgu6OyDAbMYIQ4DBZtM7TD82ZCBQ8GO1eBZK4rvpKdKkTwK5qn3dpD4W9goo/mQkomo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=kQmsvCgG; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="kQmsvCgG" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 49A8D1F00AC4; Wed, 15 Jul 2026 22:12:42 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784153562; bh=f0SwIMBeUDReOcK1TiWFlGVQvWlJUbvTC+WpayPEZ6U=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=kQmsvCgGRiK441RtuMFrIAUbOIA6xmP+va40TSkbp+u1q72VWi0lnMClASvHUZ0TQ TdDC8FY62XmChDw6T6NOurit7hr0RDxOZECBigTlp7L02yJUTYBF/f9oQSF3AZuhGl kVe9XlLOfQtvIuRXUv/chc0tkCqVTlDU+WtCvk2Ywm4Y+2Ijdy+SXluBPQ3s+PtAyr Xvj52qcxnyPpdwTe9bCrGc4lNwSNCpzI9+X9phkcvrY0aAZVOjJ3F94bPeOFKSJzgS EwU4TKq+84dfT0KpI22pxCeQNfdQI3D+dJX7BFdHqYVNGrOwotP01R5CTALgSmIb7R 9/+LMxUm824Nw== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , Thomas Huth , Eric Biggers Subject: [PATCH v2 03/13] lib/crypto: aes: Add CBC and CBC-CTS support Date: Wed, 15 Jul 2026 15:11:43 -0700 Message-ID: <20260715221153.246410-4-ebiggers@kernel.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260715221153.246410-1-ebiggers@kernel.org> References: <20260715221153.246410-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Add support for AES-CBC and AES-CBC-CTS to the crypto library. These will be used to provide streamlined implementations of the "cbc(aes)" and "cts(cbc(aes))" crypto_skcipher algorithms. Most users of these crypto_skcipher algorithms will also be able to switch to the library, which as usual will be simpler and faster, e.g.: - block/blk-crypto-fallback.c (for AES-128-CBC-ESSIV) - fs/crypto/crypto.c (for AES-128-CBC-ESSIV) - fs/crypto/fname.c (for AES-256-CTS and AES-128-CBC) - kernel/bpf/crypto.c - net/ceph/crypto.c - security/keys/encrypted-keys/encrypted.c As usual, the architecture-optimized AES-CBC and AES-CBC-CTS code will be migrated into the library as well (using the hooks provided in this commit), eliminating lots of repetitive boilerplate code. Initial test coverage is provided by the crypto_skcipher support added in a later commit. I'm planning a KUnit test suite as well. Reviewed-by: Thomas Huth Signed-off-by: Eric Biggers --- .../crypto/libcrypto-unauth-encryption.rst | 7 + include/crypto/aes-cbc.h | 77 ++++++++ lib/crypto/Kconfig | 6 + lib/crypto/aes.c | 187 ++++++++++++++++++ lib/crypto/tests/Kconfig | 1 + 5 files changed, 278 insertions(+) create mode 100644 include/crypto/aes-cbc.h diff --git a/Documentation/crypto/libcrypto-unauth-encryption.rst b/Documen= tation/crypto/libcrypto-unauth-encryption.rst index 6a3397a8adae..82bdb0d1b93e 100644 --- a/Documentation/crypto/libcrypto-unauth-encryption.rst +++ b/Documentation/crypto/libcrypto-unauth-encryption.rst @@ -20,6 +20,13 @@ block ciphers in XTS mode. The legitimate use cases for= these algorithms are: =20 Besides the above, these shouldn't be used. =20 +AES-CBC and AES-CBC-CTS +----------------------- + +This API provides support for AES in the CBC and CBC-CTS modes of operatio= n. + +.. kernel-doc:: include/crypto/aes-cbc.h + AES-ECB ------- =20 diff --git a/include/crypto/aes-cbc.h b/include/crypto/aes-cbc.h new file mode 100644 index 000000000000..7bae340935f9 --- /dev/null +++ b/include/crypto/aes-cbc.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-CBC and AES-CBC-CTS unauthenticated encryption and decryption + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_CBC_H +#define _CRYPTO_AES_CBC_H + +#include + +/** + * aes_cbc_encrypt() - Encrypt data using AES-CBC + * @dst: The destination buffer. Can be in-place or out-of-place. For ot= her + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to encrypt. Must be a multiple of AES_BLOCK_SIZE. + * @iv: The initialization vector. It is updated with the next value, i.e= . the + * last ciphertext block (or left unchanged if @len =3D=3D 0). + * @key: The key, already prepared using aes_preparekey() or aes_prepareen= ckey() + * + * This supports incremental encryption. The length of each chunk must be= a + * multiple of AES_BLOCK_SIZE, and the updated @iv must be passed in each = time. + * + * Context: Any context. + */ +void aes_cbc_encrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[at_least AES_BLOCK_SIZE], aes_encrypt_arg key); + +/** + * aes_cbc_decrypt() - Decrypt data using AES-CBC + * @dst: The destination buffer. Can be in-place or out-of-place. For ot= her + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to decrypt. Must be a multiple of AES_BLOCK_SIZE. + * @iv: The initialization vector. It is updated with the next value, i.e= . the + * last ciphertext block (or left unchanged if @len =3D=3D 0). + * @key: The key, already prepared using aes_preparekey() + * + * This supports incremental decryption. The length of each chunk must be= a + * multiple of AES_BLOCK_SIZE, and the updated @iv must be passed in each = time. + * + * Context: Any context. + */ +void aes_cbc_decrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[at_least AES_BLOCK_SIZE], const struct aes_key *key); + +/** + * aes_cbc_cts_encrypt() - Encrypt data using AES-CBC-CTS (CS3 variant) + * @dst: The destination buffer. Can be in-place or out-of-place. For ot= her + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to encrypt, at least AES_BLOCK_SIZE + * @iv: The initialization vector, clobbered by this function + * @key: The key, already prepared using aes_preparekey() or aes_prepareen= ckey() + * + * Context: Any context. + */ +void aes_cbc_cts_encrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[at_least AES_BLOCK_SIZE], aes_encrypt_arg key); + +/** + * aes_cbc_cts_decrypt() - Decrypt data using AES-CBC-CTS (CS3 variant) + * @dst: The destination buffer. Can be in-place or out-of-place. For ot= her + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to decrypt, at least AES_BLOCK_SIZE + * @iv: The initialization vector, clobbered by this function + * @key: The key, already prepared using aes_preparekey() + * + * Context: Any context. + */ +void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[at_least AES_BLOCK_SIZE], + const struct aes_key *key); + +#endif /* _CRYPTO_AES_CBC_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 26514c181a7f..c64cc3e12b57 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -27,6 +27,12 @@ config CRYPTO_LIB_AESCFB select CRYPTO_LIB_AES select CRYPTO_LIB_UTILS =20 +config CRYPTO_LIB_AES_CBC + tristate + select CRYPTO_LIB_AES + help + The AES-CBC and AES-CBC-CTS library functions. + config CRYPTO_LIB_AES_CBC_MACS tristate select CRYPTO_LIB_AES diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index e2f1ebf81405..6710e8291504 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -5,6 +5,7 @@ */ =20 #include +#include #include #include #include @@ -793,6 +794,192 @@ void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t l= en, EXPORT_SYMBOL_GPL(aes_ecb_decrypt); #endif /* CONFIG_CRYPTO_LIB_AES_ECB */ =20 +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC) +/* + * Hooks for optimized AES-CBC implementations, overridable by the archite= cture. + * They are called with len > 0 && len % AES_BLOCK_SIZE =3D=3D 0. Returni= ng false + * causes the fallback implementation to be used instead. + */ +#ifndef aes_cbc_encrypt_arch +static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_cbc_decrypt_arch +static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_key *key) +{ + return false; +} +#endif + +void aes_cbc_encrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_S= IZE], + aes_encrypt_arg key) +{ + const u8 *prev =3D iv; + + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len =3D round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len =3D=3D 0)) + return; + + if (likely(aes_cbc_encrypt_arch(dst, src, len, iv, key.enc_key))) + return; + + do { + crypto_xor_cpy(dst, src, prev, AES_BLOCK_SIZE); + aes_encrypt(key, dst, dst); + prev =3D dst; + dst +=3D AES_BLOCK_SIZE; + src +=3D AES_BLOCK_SIZE; + len -=3D AES_BLOCK_SIZE; + } while (len); + memcpy(iv, prev, AES_BLOCK_SIZE); +} +EXPORT_SYMBOL_GPL(aes_cbc_encrypt); + +void aes_cbc_decrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_S= IZE], + const struct aes_key *key) +{ + u8 next_iv[AES_BLOCK_SIZE]; + + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len =3D round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len =3D=3D 0)) + return; + + if (likely(aes_cbc_decrypt_arch(dst, src, len, iv, key))) + return; + + len -=3D AES_BLOCK_SIZE; + dst +=3D len; + src +=3D len; + memcpy(next_iv, src, AES_BLOCK_SIZE); + for (;;) { + aes_decrypt(key, dst, src); + if (len =3D=3D 0) + break; + src -=3D AES_BLOCK_SIZE; + crypto_xor(dst, src, AES_BLOCK_SIZE); + dst -=3D AES_BLOCK_SIZE; + len -=3D AES_BLOCK_SIZE; + } + crypto_xor(dst, iv, AES_BLOCK_SIZE); + memcpy(iv, next_iv, AES_BLOCK_SIZE); +} +EXPORT_SYMBOL_GPL(aes_cbc_decrypt); + +/* + * Hooks for optimized AES-CBC-CTS implementations, overridable by the + * architecture. They are called with len > AES_BLOCK_SIZE. Returning fa= lse + * causes the fallback implementation to be used instead. The fallback + * implementation still uses the arch-optimized AES-CBC code if available,= but + * direct implementation of AES-CBC-CTS is helpful on short messages. + */ +#ifndef aes_cbc_cts_encrypt_arch +static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_cbc_cts_decrypt_arch +static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_key *key) +{ + return false; +} +#endif + +void aes_cbc_cts_encrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) +{ + /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ + size_t pn_offset =3D round_down(len - 1, AES_BLOCK_SIZE); + /* Length of P[n] and C[n], 1 <=3D pn_len <=3D AES_BLOCK_SIZE */ + size_t pn_len =3D len - pn_offset; + u8 tmp[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + u8 *pad; + + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (len =3D=3D AES_BLOCK_SIZE) { + aes_cbc_encrypt(dst, src, len, iv, key); + return; + } + if (likely(aes_cbc_cts_encrypt_arch(dst, src, len, iv, key.enc_key))) + return; + + /* CBC-encrypt all blocks except the last. */ + aes_cbc_encrypt(dst, src, pn_offset, iv, key); + + /* + * Compute C[n] and C[n - 1]. + * + * Careful: src may equal dst (i.e., the encryption can be in-place), so + * src[pn_offset..] can't be read after dst[pn_offset..] is written. + */ + pad =3D &dst[pn_offset - AES_BLOCK_SIZE]; + memcpy(tmp, pad, AES_BLOCK_SIZE); + crypto_xor(tmp, &src[pn_offset], pn_len); + memcpy(&dst[pn_offset], pad, pn_len); /* C[n] */ + aes_encrypt(key, pad, tmp); /* C[n - 1] */ + + memzero_explicit(tmp, sizeof(tmp)); +} +EXPORT_SYMBOL_GPL(aes_cbc_cts_encrypt); + +void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], const struct aes_key *key) +{ + /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ + size_t pn_offset =3D round_down(len - 1, AES_BLOCK_SIZE); + /* Length of P[n] and C[n], 1 <=3D pn_len <=3D AES_BLOCK_SIZE */ + size_t pn_len =3D len - pn_offset; + u8 *pad; + + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (len =3D=3D AES_BLOCK_SIZE) { + aes_cbc_decrypt(dst, src, len, iv, key); + return; + } + if (likely(aes_cbc_cts_decrypt_arch(dst, src, len, iv, key))) + return; + + /* Compute P[0]..P[n - 2]. */ + aes_cbc_decrypt(dst, src, pn_offset - AES_BLOCK_SIZE, iv, key); + + /* + * Compute P[n] and P[n - 1]. + * + * Careful: src may equal dst (i.e., the decryption can be in-place), so + * src[pn_offset..] can't be read after dst[pn_offset..] is written. + * + * To avoid needing a temporary buffer, do a "redundant" XOR to recover + * src[pn_offset..] from dst[pn_offset..] after the latter is written. + */ + pad =3D &dst[pn_offset - AES_BLOCK_SIZE]; + aes_decrypt(key, pad, &src[pn_offset - AES_BLOCK_SIZE]); + crypto_xor_cpy(&dst[pn_offset], &src[pn_offset], pad, + pn_len); /* P[n] */ + crypto_xor(pad, &dst[pn_offset], pn_len); + aes_decrypt(key, pad, pad); + crypto_xor(pad, iv, AES_BLOCK_SIZE); /* P[n - 1] */ +} +EXPORT_SYMBOL_GPL(aes_cbc_cts_decrypt); +#endif /* CONFIG_CRYPTO_LIB_AES_CBC */ + static int __init aes_mod_init(void) { #ifdef aes_mod_init_arch diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index a57e87dbb1b1..e78086f3c954 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -144,6 +144,7 @@ config CRYPTO_LIB_SM3_KUNIT_TEST config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT tristate "Enable all crypto library code for KUnit tests" depends on KUNIT + select CRYPTO_LIB_AES_CBC select CRYPTO_LIB_AES_CBC_MACS select CRYPTO_LIB_AES_ECB select CRYPTO_LIB_BLAKE2B --=20 2.55.0 From nobody Sat Jul 25 16:48:31 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E69B43CFF6C; Wed, 15 Jul 2026 22:12:42 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153564; cv=none; b=sE0Rhzp3jtv8wNzfmqISxAjpsTkyxLzrPmB2XyDeHBkeLBHYhieO9/mLJsiq+TO2DathdNTSxmEvuypRuTaU4bRTMItrkxOMFplbsGX34V/gT4TTH6LpS8NlUADN3mMkTPDNHGhZccIMVJa8WeXSltkPo0k0l3bl8B8vMUWFicY= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153564; c=relaxed/simple; bh=B0fztXzIR2xwOw4aMHpGcVof/C+9ZXCQWotLBdlAd4g=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=ldyxnOp5fZK/z81VgNOOxKMUWzajejj/3HpnSXEteE5hh+aCUD5b316pg3QjGfN28uxl//bA/fe02SyxZ11ie6KhIIJEOXj8sXi0aetVwHXREsYtyvNm7W1ijcO+Nze1kveYHf1WfiTd7iXC8JVz756CpG8KRzwj8njeI7q8uYE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=olgoeN/9; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="olgoeN/9" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 9D9A71F00A3E; Wed, 15 Jul 2026 22:12:42 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784153562; bh=oayr9qowYYpJzWDTfZs2bh+aLpd0OH9jYde0Q3W/KBg=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=olgoeN/9A5CnE6COO8KTgqa8zMNhGHzThbXIldWdbBeZEGYFA3qc8dZti8z2W2zht ew2Hg/1BFP1OkD7DlJwAS+3Ru6h2LNsm/ULLX/tjkqnrZQnOOTvUBt3m8ON4Y0Oux4 m6nKN+ecmLrF1LJWYbIXPWd42wvvPGczQiFaT9VJek8/CyQwXr4VcEzTCNF5+JiPWv pB42SqIy1nNXeBgM5HbWMKo68xhfUSdLkenjybdA2RUYa4qcirAu3GlppuUEd+aniC JZluR0aF4PVyybPxbpcX7D4jjM6oKhMnRUp13ZyYIOOS02nCk+YbtEvfZkHbxXJZG/ UEtyqD7LDhIYQ== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , Thomas Huth , Eric Biggers Subject: [PATCH v2 04/13] lib/crypto: aes: Add CTR and XCTR support Date: Wed, 15 Jul 2026 15:11:44 -0700 Message-ID: <20260715221153.246410-5-ebiggers@kernel.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260715221153.246410-1-ebiggers@kernel.org> References: <20260715221153.246410-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Add support for AES-CTR and AES-XCTR to the crypto library. These will be used to provide streamlined implementations of the "ctr(aes)" and "xctr(aes)" crypto_skcipher algorithms. Most users of "ctr(aes)" will also be able to switch to the library, which as usual will be simpler and faster, e.g.: - net/mac80211/fils_aead.c - net/mac802154/llsec.c As usual, the architecture-optimized AES-CTR and AES-XCTR code will be migrated into the library as well (using the hooks provided in this commit), eliminating lots of repetitive boilerplate code. This is also a prerequisite for supporting AES-GCM, AES-CCM, and AES-HCTR2 in the crypto library. Initial test coverage is provided by the crypto_skcipher support added in a later commit. I'm planning a KUnit test suite as well. Signed-off-by: Eric Biggers Reviewed-by: Thomas Huth --- .../crypto/libcrypto-unauth-encryption.rst | 7 ++ include/crypto/aes-ctr.h | 65 +++++++++++++ lib/crypto/Kconfig | 6 ++ lib/crypto/aes.c | 96 +++++++++++++++++++ lib/crypto/tests/Kconfig | 1 + 5 files changed, 175 insertions(+) create mode 100644 include/crypto/aes-ctr.h diff --git a/Documentation/crypto/libcrypto-unauth-encryption.rst b/Documen= tation/crypto/libcrypto-unauth-encryption.rst index 82bdb0d1b93e..4d3555ee81b7 100644 --- a/Documentation/crypto/libcrypto-unauth-encryption.rst +++ b/Documentation/crypto/libcrypto-unauth-encryption.rst @@ -27,6 +27,13 @@ This API provides support for AES in the CBC and CBC-CTS= modes of operation. =20 .. kernel-doc:: include/crypto/aes-cbc.h =20 +AES-CTR and AES-XCTR +-------------------- + +This API provides support for AES in the CTR and XCTR modes of operation. + +.. kernel-doc:: include/crypto/aes-ctr.h + AES-ECB ------- =20 diff --git a/include/crypto/aes-ctr.h b/include/crypto/aes-ctr.h new file mode 100644 index 000000000000..8e214a85716b --- /dev/null +++ b/include/crypto/aes-ctr.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-CTR and AES-XCTR stream ciphers + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_CTR_H +#define _CRYPTO_AES_CTR_H + +#include + +/** + * aes_ctr() - AES-CTR en/decryption + * @dst: The destination buffer. Can be in-place or out-of-place. For ot= her + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to en/decrypt + * @ctr: The counter. It will be incremented by ceil(@len / AES_BLOCK_SIZ= E). + * @key: The key, already prepared using aes_preparekey() or aes_prepareen= ckey() + * + * This implements AES in counter mode with a 128-bit big endian counter. + * + * This exists only for use by the implementation of modes built on top of= CTR + * (e.g., GCM and CCM) and some legacy protocols that use CTR mode directl= y. + * Callers are expected to know how to use CTR mode appropriately, includi= ng + * choosing (key, counter) pairs appropriately to avoid keystream reuse. + * + * This supports incremental en/decryption. The length of each non-final = chunk + * must be a multiple of AES_BLOCK_SIZE, and the updated @ctr must be pass= ed in + * each time. + * + * Context: Any context. + */ +void aes_ctr(u8 *dst, const u8 *src, size_t len, + u8 ctr[at_least AES_BLOCK_SIZE], aes_encrypt_arg key); + +/** + * aes_xctr() - AES-XCTR en/decryption + * @dst: The destination buffer. Can be in-place or out-of-place. For ot= her + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to en/decrypt + * @ctr: The block counter (in host endianness). For the first call, set = it to + * 1. It will be incremented by ceil(@len / AES_BLOCK_SIZE). + * @iv: The initialization vector + * @key: The key, already prepared using aes_preparekey() or aes_prepareen= ckey() + * + * This implements AES in XOR Counter mode, as specified in the paper + * "Length-preserving encryption with HCTR2" + * (https://eprint.iacr.org/2021/1441.pdf). + * + * This exists only for use by the implementation of modes built on top of= XCTR. + * Callers are expected to know how to use XCTR mode appropriately, includ= ing + * choosing (key, IV) pairs appropriately to avoid keystream reuse. + * + * This supports incremental en/decryption. The length of each non-final = chunk + * must be a multiple of AES_BLOCK_SIZE, and the updated @ctr must be pass= ed in + * each time. + * + * Context: Any context. + */ +void aes_xctr(u8 *dst, const u8 *src, size_t len, u64 *ctr, + const u8 iv[at_least AES_BLOCK_SIZE], aes_encrypt_arg key); + +#endif /* _CRYPTO_AES_CTR_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index c64cc3e12b57..67a44e82309d 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -41,6 +41,12 @@ config CRYPTO_LIB_AES_CBC_MACS this if your module uses any of the functions from . =20 +config CRYPTO_LIB_AES_CTR + tristate + select CRYPTO_LIB_AES + help + The AES-CTR and AES-XCTR library functions. + config CRYPTO_LIB_AES_ECB tristate select CRYPTO_LIB_AES diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index 6710e8291504..84384582b5bb 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -6,6 +6,7 @@ =20 #include #include +#include #include #include #include @@ -980,6 +981,101 @@ void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size= _t len, EXPORT_SYMBOL_GPL(aes_cbc_cts_decrypt); #endif /* CONFIG_CRYPTO_LIB_AES_CBC */ =20 +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) +/* + * Hooks for optimized AES-CTR and AES-XCTR implementations, overridable b= y the + * architecture. They are called with any len >=3D 0. Returning false ca= uses the + * fallback implementation to be used instead. + */ +#ifndef aes_ctr_arch +static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len, + u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_xctr_arch +static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 *ctr, + const u8 iv[AES_BLOCK_SIZE], + const struct aes_enckey *key) +{ + return false; +} +#endif + +static __always_inline void inc_be128_ctr(u8 ctr[AES_BLOCK_SIZE]) +{ + /* + * 255 times out of 256 the first iteration is enough, so unroll the + * first iteration as a micro-optimization. + */ + if ((++ctr[AES_BLOCK_SIZE - 1]) !=3D 0) + return; + for (int i =3D AES_BLOCK_SIZE - 2; i >=3D 0; i--) { + if (++ctr[i] !=3D 0) + break; + } +} + +void aes_ctr(u8 *dst, const u8 *src, size_t len, u8 ctr[AES_BLOCK_SIZE], + aes_encrypt_arg key) +{ + u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + + if (likely(aes_ctr_arch(dst, src, len, ctr, key.enc_key))) + return; + + /* Handle the full blocks. */ + for (; len >=3D AES_BLOCK_SIZE; len -=3D AES_BLOCK_SIZE) { + aes_encrypt(key, keystream, ctr); + crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); + inc_be128_ctr(ctr); + dst +=3D AES_BLOCK_SIZE; + src +=3D AES_BLOCK_SIZE; + } + /* Handle any partial block at the end. */ + if (len) { + aes_encrypt(key, keystream, ctr); + crypto_xor_cpy(dst, src, keystream, len); + /* Counter is incremented even with just a partial block. */ + inc_be128_ctr(ctr); + } + memzero_explicit(keystream, sizeof(keystream)); +} +EXPORT_SYMBOL_GPL(aes_ctr); + +void aes_xctr(u8 *dst, const u8 *src, size_t len, u64 *ctr, + const u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) +{ + const __le64 iv0 =3D get_unaligned((const __le64 *)&iv[0]); + __le64 aes_input[2]; + u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + + if (likely(aes_xctr_arch(dst, src, len, ctr, iv, key.enc_key))) + return; + + aes_input[1] =3D get_unaligned((const __le64 *)&iv[8]); + /* Handle the full blocks. */ + for (; len >=3D AES_BLOCK_SIZE; len -=3D AES_BLOCK_SIZE) { + aes_input[0] =3D iv0 ^ cpu_to_le64((*ctr)++); + aes_encrypt(key, keystream, (const u8 *)aes_input); + crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); + dst +=3D AES_BLOCK_SIZE; + src +=3D AES_BLOCK_SIZE; + } + /* Handle any partial block at the end. */ + if (len) { + /* Counter is incremented even with just a partial block. */ + aes_input[0] =3D iv0 ^ cpu_to_le64((*ctr)++); + aes_encrypt(key, keystream, (const u8 *)aes_input); + crypto_xor_cpy(dst, src, keystream, len); + } + memzero_explicit(keystream, sizeof(keystream)); + memzero_explicit(aes_input, sizeof(aes_input)); +} +EXPORT_SYMBOL_GPL(aes_xctr); +#endif /* CONFIG_CRYPTO_LIB_AES_CTR */ + static int __init aes_mod_init(void) { #ifdef aes_mod_init_arch diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index e78086f3c954..9284d0134d77 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -146,6 +146,7 @@ config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT depends on KUNIT select CRYPTO_LIB_AES_CBC select CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_AES_CTR select CRYPTO_LIB_AES_ECB select CRYPTO_LIB_BLAKE2B select CRYPTO_LIB_CHACHA20POLY1305 --=20 2.55.0 From nobody Sat Jul 25 16:48:31 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 92EDD3D3CF0; Wed, 15 Jul 2026 22:12:43 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153565; cv=none; b=dWePCowd7HOgDDyIIsPOFC+T7zBl7jhQpyw8RNOGYQZuWmgUAdqI5f/EP8VCjOJZI5xcQFKRrwyuZcn5SkUxCeGSscjTLlECT8qXLlJ/r49VaM48wgAhzf1vvU/qu84C2OS3ILGhYjKhhhurVOlC9DlcjtI+q+ASGXVJrWItY4U= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153565; c=relaxed/simple; bh=Ca4CrJrkQ2Bnfl9e7FCuhFHjdYOZoOCc5V58ff4gtiw=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=DSFtnAQF7/3KUfiUj0x76ynGLzjpNESO/+KX/INiIE4HH3yO7vbziHpV/BXk2R2xOUIHZxPU9U0vfixap5Ma2s8y+5djxPuVhVqhFIF8coJtxSa+arTEV785jxUvuE6JweOEEASOb1rAjqFRmH+FhFfIhG4x15RUyTWkFrrvWFg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=GKKOZfgb; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="GKKOZfgb" Received: by smtp.kernel.org (Postfix) with ESMTPSA id F03A11F00A3F; Wed, 15 Jul 2026 22:12:42 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784153563; bh=nObfJ38BsTk+XbkXUp3ucFttRRIcq2nTdzH7HOOKPCU=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=GKKOZfgbPbAxqseG7LbdxZXDqLX7OYZ5OtbNdH6vgrCzRzNJcR2nCFA+OQcEINk/s TqQkp3d5YquBQarL1BnArPx1UDjrEu9Ccp7JTH2Af9J5MrN8jubci7dSi0KT+14SZb 5ywDJiHGGcDeH+o2Tnr6gt6x9F2uOvNJgJbFDLgtPA9PoO5EvP5pWInSUrWT6Fp02z nNp6meQ1+YyXumc34IuFhcscdRnza41jFD15bZ0XYRpAjwsl/YwptVwYhyVwtn8pKQ pgaYU4wJRUUGejEyp566Lx/8BFHsFMHN51MFKmfLaJLv/8yCYPOSB0AQZ4ciA+mOXR aiCqubvi5gdPA== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , Thomas Huth , Eric Biggers Subject: [PATCH v2 05/13] lib/crypto: aes: Add XTS support Date: Wed, 15 Jul 2026 15:11:45 -0700 Message-ID: <20260715221153.246410-6-ebiggers@kernel.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260715221153.246410-1-ebiggers@kernel.org> References: <20260715221153.246410-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Add support for AES-XTS to the crypto library. This will be used to provide a streamlined implementation of the "xts(aes)" crypto_skcipher algorithm. I'm also planning to use this directly in fscrypt and blk-crypto-fallback. As usual, the architecture-optimized AES-XTS code will be migrated into the library as well (using the hooks provided in this commit), eliminating lots of repetitive boilerplate code. Compared to direct implementation of "xts(aes)", I've also eliminated the requirement for architectures to implement ciphertext stealing, as the library just handles it portably instead. That will simplify things considerably. Initial test coverage is provided by the crypto_skcipher support added in a later commit. I'm planning a KUnit test suite as well. Signed-off-by: Eric Biggers --- .../crypto/libcrypto-unauth-encryption.rst | 7 + include/crypto/aes-xts.h | 94 +++++++ lib/crypto/Kconfig | 6 + lib/crypto/aes.c | 231 ++++++++++++++++++ lib/crypto/tests/Kconfig | 1 + 5 files changed, 339 insertions(+) create mode 100644 include/crypto/aes-xts.h diff --git a/Documentation/crypto/libcrypto-unauth-encryption.rst b/Documen= tation/crypto/libcrypto-unauth-encryption.rst index 4d3555ee81b7..b4639b8927c6 100644 --- a/Documentation/crypto/libcrypto-unauth-encryption.rst +++ b/Documentation/crypto/libcrypto-unauth-encryption.rst @@ -40,3 +40,10 @@ AES-ECB This API provides support for AES in the ECB mode of operation. =20 .. kernel-doc:: include/crypto/aes-ecb.h + +AES-XTS +------- + +This API provides support for AES in the XTS mode of operation. + +.. kernel-doc:: include/crypto/aes-xts.h diff --git a/include/crypto/aes-xts.h b/include/crypto/aes-xts.h new file mode 100644 index 000000000000..b9e828265e58 --- /dev/null +++ b/include/crypto/aes-xts.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-XTS unauthenticated encryption and decryption + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_XTS_H +#define _CRYPTO_AES_XTS_H + +#include +#include + +/** + * struct aes_xts_key - A key prepared for AES-XTS encryption and decrypti= on + * + * Note that (depending on the architecture) this typically is around 768 = bytes, + * which makes it a bit too large to allocate on the stack in most cases. + */ +struct aes_xts_key { + /* private: */ + struct aes_key main_key; + struct aes_enckey tweak_key; +}; + +/** + * aes_xts_preparekey() - Prepare a key for AES-XTS encryption and decrypt= ion + * @key: (output) The key structure to initialize + * @in_key: The raw AES-XTS key + * @key_len: Length of the raw key in bytes + * @flags: Optional flag XTS_FORBID_WEAK_KEYS to forbid keys whose two hal= ves + * are the same. + * + * Users should use memzero_explicit() to zeroize the key struct at the en= d of + * its lifetime. (But if this function fails, zeroization is unnecessary.) + * + * Context: Any context. + * Return: + * * 0 on success + * * -EINVAL if the key is rejected because its length isn't 32, 64, or (w= hen + * FIPS mode isn't enabled) 48; or because its two halves are the same a= nd + * either XTS_FORBID_WEAK_KEYS is given or FIPS mode is enabled. + */ +int __must_check aes_xts_preparekey(struct aes_xts_key *key, const u8 *in_= key, + size_t key_len, int flags); + +/** + * aes_xts_encrypt() - Encrypt data using AES-XTS + * @dst: The destination buffer. Can be in-place or out-of-place. For ot= her + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to encrypt. On non-final calls it must be a nonz= ero + * multiple of AES_BLOCK_SIZE. On the final call it can be any value >= =3D + * AES_BLOCK_SIZE, i.e. ciphertext stealing is supported. + * @tweak: The tweak. It is updated with the next value, unless @len isn'= t a + * multiple of AES_BLOCK_SIZE in which case the value is unspecified. + * @key: The key, already prepared using aes_xts_preparekey() + * @cont: %false to begin encrypting a new message (do the tweak encryptio= n); + * %true to continue encrypting a message (skip tweak encryption) + * + * This supports both one-shot and incremental encryption. On the first c= all, + * pass @cont =3D %false. On any later calls, pass @cont =3D %true and th= e updated + * @tweak; all earlier @len must have been multiples of AES_BLOCK_SIZE. + * + * Context: Any context. + */ +void aes_xts_encrypt(u8 *dst, const u8 *src, size_t len, + u8 tweak[at_least AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont); + +/** + * aes_xts_decrypt() - Decrypt data using AES-XTS + * @dst: The destination buffer. Can be in-place or out-of-place. For ot= her + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to decrypt. On non-final calls it must be a nonz= ero + * multiple of AES_BLOCK_SIZE. On the final call it can be any value >= =3D + * AES_BLOCK_SIZE, i.e. ciphertext stealing is supported. + * @tweak: The tweak. It is updated with the next value, unless @len isn'= t a + * multiple of AES_BLOCK_SIZE in which case the value is unspecified. + * @key: The key, already prepared using aes_xts_preparekey() + * @cont: %false to begin decrypting a new message (do the tweak encryptio= n); + * %true to continue decrypting a message (skip tweak encryption) + * + * This supports both one-shot and incremental decryption. On the first c= all, + * pass @cont =3D %false. On any later calls, pass @cont =3D %true and th= e updated + * @tweak; all earlier @len must have been multiples of AES_BLOCK_SIZE. + * + * Context: Any context. + */ +void aes_xts_decrypt(u8 *dst, const u8 *src, size_t len, + u8 tweak[at_least AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont); + +#endif /* _CRYPTO_AES_XTS_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 67a44e82309d..6ec47cc328c8 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -53,6 +53,12 @@ config CRYPTO_LIB_AES_ECB help The AES-ECB library functions. =20 +config CRYPTO_LIB_AES_XTS + tristate + select CRYPTO_LIB_AES + help + The AES-XTS library functions. + config CRYPTO_LIB_AESGCM tristate select CRYPTO_LIB_AES diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index 84384582b5bb..03c80f4fe176 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -8,7 +8,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -1076,6 +1078,235 @@ void aes_xctr(u8 *dst, const u8 *src, size_t len, u= 64 *ctr, EXPORT_SYMBOL_GPL(aes_xctr); #endif /* CONFIG_CRYPTO_LIB_AES_CTR */ =20 +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS) +int aes_xts_preparekey(struct aes_xts_key *key, const u8 *in_key, + size_t key_len, int flags) +{ + int err; + + err =3D __xts_verify_key(in_key, key_len, flags); + if (unlikely(err)) + goto out_zeroize; + /* First half of XTS key is the main key */ + err =3D aes_preparekey(&key->main_key, in_key, key_len / 2); + if (unlikely(err)) + goto out_zeroize; + /* Second half of XTS key is the tweak key */ + err =3D aes_prepareenckey(&key->tweak_key, &in_key[key_len / 2], + key_len / 2); + if (unlikely(err)) + goto out_zeroize; + return 0; + +out_zeroize: + memzero_explicit(key, sizeof(*key)); + return err; +} +EXPORT_SYMBOL_GPL(aes_xts_preparekey); + +/* + * Hooks for optimized AES-XTS implementations, overridable by the archite= cture. + * They are called with len > 0 && len % AES_BLOCK_SIZE =3D=3D 0. In othe= r words, + * they aren't expected to handle ciphertext stealing or empty inputs. + * Returning false causes the fallback implementation to be used instead. + * + * (Currently, all users of AES-XTS in the kernel seem to en/decrypt whole + * numbers of blocks anyway, with len >=3D 512. So there's no need to hea= vily + * optimize ciphertext stealing for short messages.) + */ +#ifndef aes_xts_encrypt_arch +static bool aes_xts_encrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont) +{ + return false; +} +#endif +#ifndef aes_xts_decrypt_arch +static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont) +{ + return false; +} +#endif + +static noinline void aes_xts_crypt_nocts_blockbyblock( + u8 *dst, const u8 *src, size_t len, u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont, bool enc) +{ + le128 t; + + if (cont) + memcpy(&t, tweak, sizeof(t)); + else + aes_encrypt(&key->tweak_key, (u8 *)&t, tweak); + do { + crypto_xor_cpy(dst, src, (const u8 *)&t, AES_BLOCK_SIZE); + if (enc) + aes_encrypt(&key->main_key, dst, dst); + else + aes_decrypt(&key->main_key, dst, dst); + crypto_xor(dst, (const u8 *)&t, AES_BLOCK_SIZE); + gf128mul_x_ble(&t, &t); + dst +=3D AES_BLOCK_SIZE; + src +=3D AES_BLOCK_SIZE; + len -=3D AES_BLOCK_SIZE; + } while (len); + memcpy(tweak, &t, sizeof(t)); + memzero_explicit(&t, sizeof(t)); +} + +/* Requires len > 0 && len % AES_BLOCK_SIZE =3D=3D 0 */ +static __always_inline void aes_xts_encrypt_nocts(u8 *dst, const u8 *src, + size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + if (likely(aes_xts_encrypt_arch(dst, src, len, tweak, key, cont))) + return; + + /* + * For the fallback, just go block-by-block. It could be implemented on + * top of AES-ECB, which could be significantly faster than this if the + * arch has optimized AES-ECB code but not AES-XTS. However, AES-XTS + * performance is important enough that it needs to be (and has been) + * implemented directly by every non-obsolete arch anyway. + */ + aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, + /* enc=3D */ true); +} + +/* Requires len > 0 && len % AES_BLOCK_SIZE =3D=3D 0 */ +static __always_inline void aes_xts_decrypt_nocts(u8 *dst, const u8 *src, + size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + if (likely(aes_xts_decrypt_arch(dst, src, len, tweak, key, cont))) + return; + + /* Just go block-by-block. See comment in aes_xts_encrypt_nocts(). */ + aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, + /* enc=3D */ false); +} + +static noinline void aes_xts_encrypt_cts(u8 *dst, const u8 *src, size_t le= n, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + size_t partial_len =3D len % AES_BLOCK_SIZE; /* Length of partial block */ + size_t nocts_len =3D round_down(len, AES_BLOCK_SIZE); + u8 tmp_block[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + + /* Encrypt all full blocks. */ + aes_xts_encrypt_nocts(dst, src, nocts_len, tweak, key, cont); + dst +=3D nocts_len - AES_BLOCK_SIZE; + src +=3D nocts_len - AES_BLOCK_SIZE; + + /* + * Swap the partial block with the first 'partial_len' bytes of the + * encrypted last full block. Note that a temporary buffer is needed to + * support in-place encryption. + */ + memcpy(tmp_block, src + AES_BLOCK_SIZE, partial_len); + memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); + memcpy(dst, tmp_block, partial_len); + + /* Encrypt the last full block again. */ + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + aes_encrypt(&key->main_key, dst, dst); + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + memzero_explicit(tmp_block, sizeof(tmp_block)); +} + +static noinline void aes_xts_decrypt_cts(u8 *dst, const u8 *src, size_t le= n, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + size_t partial_len =3D len % AES_BLOCK_SIZE; /* Length of partial block */ + size_t nocts_len =3D round_down(len, AES_BLOCK_SIZE) - AES_BLOCK_SIZE; + union { + u8 block[AES_BLOCK_SIZE]; + le128 tweak; + } tmp __aligned(__alignof__(long)); + + /* + * Decrypt all blocks except the last full block and the partial block. + * The last full block has to be handled specially because decryption + * ciphertext stealing uses the last two tweaks in reverse order. + * + * nocts_len =3D=3D 0 is possible here, which aes_xts_decrypt_nocts() + * doesn't handle (so that the length doesn't get checked redundantly in + * the fast path). So handle that case specially as well. + */ + if (nocts_len) + aes_xts_decrypt_nocts(dst, src, nocts_len, tweak, key, cont); + else if (!cont) + aes_encrypt(&key->tweak_key, tweak, tweak); + dst +=3D nocts_len; + src +=3D nocts_len; + + /* Copy the tweak, advance it again, then decrypt last full block. */ + memcpy(&tmp.tweak, tweak, AES_BLOCK_SIZE); + gf128mul_x_ble(&tmp.tweak, &tmp.tweak); + crypto_xor_cpy(dst, src, tmp.block, AES_BLOCK_SIZE); + aes_decrypt(&key->main_key, dst, dst); + crypto_xor(dst, tmp.block, AES_BLOCK_SIZE); + + /* + * Swap the partial block with the first 'partial_len' bytes of the + * decrypted last full block. Note that a temporary buffer is needed to + * support in-place decryption. + */ + memcpy(tmp.block, src + AES_BLOCK_SIZE, partial_len); + memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); + memcpy(dst, tmp.block, partial_len); + + /* Decrypt the last full block again. */ + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + aes_decrypt(&key->main_key, dst, dst); + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + memzero_explicit(&tmp, sizeof(tmp)); +} + +void aes_xts_encrypt(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, + bool cont) +{ + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (unlikely(len % AES_BLOCK_SIZE)) { + aes_xts_encrypt_cts(dst, src, len, tweak, key, cont); + return; + } + + aes_xts_encrypt_nocts(dst, src, len, tweak, key, cont); +} +EXPORT_SYMBOL_GPL(aes_xts_encrypt); + +void aes_xts_decrypt(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, + bool cont) +{ + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (unlikely(len % AES_BLOCK_SIZE)) { + aes_xts_decrypt_cts(dst, src, len, tweak, key, cont); + return; + } + + aes_xts_decrypt_nocts(dst, src, len, tweak, key, cont); +} +EXPORT_SYMBOL_GPL(aes_xts_decrypt); +#endif /* CONFIG_CRYPTO_LIB_AES_XTS */ + static int __init aes_mod_init(void) { #ifdef aes_mod_init_arch diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index 9284d0134d77..b559e7c79e76 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -148,6 +148,7 @@ config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT select CRYPTO_LIB_AES_CBC_MACS select CRYPTO_LIB_AES_CTR select CRYPTO_LIB_AES_ECB + select CRYPTO_LIB_AES_XTS select CRYPTO_LIB_BLAKE2B select CRYPTO_LIB_CHACHA20POLY1305 select CRYPTO_LIB_CURVE25519 --=20 2.55.0 From nobody Sat Jul 25 16:48:31 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B7BF53D3D11; Wed, 15 Jul 2026 22:12:43 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153566; cv=none; b=jvlud4R0M3aQenFlTkw1Jdg/oMw1Ac5/on4I1MBFK3QmN+kLuM+rFISQsQoN93vu7NJadsPRcYpmmkmyb93y69hRFH1hzCGixY/y6JIhNCG5D1UTK2Iv9DHf/T74K+YBsh4s4RbEBkT9ONBMl7NM4+u01e+cTCyIqGFZ1lUouwM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153566; c=relaxed/simple; bh=ZYnI2vykuZWNMgwSOIdxUUmz8p84CxKndMoqkaWjJb0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=ml/tnZjpulL91SQaMMOhNqMoRypQL4Uih3VkdcOOMTLOM3Kc/JLbqCz00Z8swZbQBNr1kBRnZzd99N6s5YmL/5YbitMDuzh0jY1kMlOC0UAZMXbHXmA9HIIvHy3qpZwiqRfYABaiwOq/LO7roN3He9pn894tR/MVdoDlvWF3NzQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=HOjbsaD/; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="HOjbsaD/" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 61DA11F00ACF; Wed, 15 Jul 2026 22:12:43 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784153563; bh=Od/wSK3eJIlKS67EdeBYfeVj+o5UpF68V4i+ayxx9BU=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=HOjbsaD/S+RP8aAOaYI5X8gX3m9gVf3PE69vQNt8Ah7ILb2Yhi3cLl55IUpM5JXgF JWk5e8mvtxvaP3GoHH0slebk12eu+u88xk7o90RpxMNPAnFvS9RbMyTNH5gO7GVGNT 2Ts/CV/sS/qFJPRsunvpVKNjvHZ788KSIgi7QIa01D2dBB/zXQ59YryX22QGw/2TCU P0nJg7ZP27i15fl0uT+ZgDjbos9bbPoN18jGi5llkin5+TIvrqVEA3CTUE4lL6hw+R sV75KXXiefnedfk3neEAj0xxcQnNdJpms6nxXkS2dzKES8/FXLoWBAf//QmofLmKRT +3AbIgDcR/zpQ== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , Thomas Huth , Eric Biggers Subject: [PATCH v2 06/13] lib/crypto: aes: Add GCM support Date: Wed, 15 Jul 2026 15:11:46 -0700 Message-ID: <20260715221153.246410-7-ebiggers@kernel.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260715221153.246410-1-ebiggers@kernel.org> References: <20260715221153.246410-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Add support for AES-GCM to the crypto library. This will be used to provide streamlined implementations of the "gcm(aes)" and "rfc4106(gcm(aes))" crypto_aead algorithms. Most users of these will also be able to switch to the library, which as usual will be faster and simpler, e.g.: - drivers/net/macsec.c - fs/smb/client/ - fs/smb/server/ - net/ceph/messenger_v2.c - net/mac80211/ (for both GMAC and GCMP) - net/tipc/crypto.c - security/keys/trusted-keys/trusted_dcp.c (I've already written proof-of-concept patches for all the above, and they helped inform the API design.) As usual, the architecture-optimized AES-GCM code will be migrated into the library as well (using the hooks provided in this commit as well as the GHASH ones), eliminating lots of repetitive boilerplate code. Incremental en/decryption is supported. Incremental operation is a bit controversial in AEAD APIs because users have to be careful not to consume any decrypted data that hasn't been authenticated yet. But I do think it's the right choice here. It's not fundamentally different from the existing incremental MAC APIs, and it's the only approach that's general enough to work well for all users in the kernel: - An array of virtually-addressed buffers (like that used by BoringSSL's EVP_AEAD_CTX_sealv() and EVP_AEAD_CTX_openv()) doesn't work in the kernel in general, since in some cases the data for a single AES-GCM message is contained in a large number of highmem pages that each need to be mapped into memory individually. That can be done efficiently only by using CPU-local mappings, but there is a limited number of those. Ceph messenger v2 is a great example, as it can send or receive up to 32 MiB in a single AES-GCM message. And it needs the en/decrypted data to go into a (potentially large) number of bvecs provided by a custom iterator, as well as into four virtually-addressed buffers, two of which can be large buffers in the vmalloc region. Even just allocating an array big enough to store all the pointers can be problematic in the kernel. There are cases in which decryption runs in GFP_NOIO context or even in softirq context, where memory allocations are not as reliable as they normally are. - Meanwhile, 'struct scatterlist' (the choice of crypto_aead) has turned out to be really inconvenient for anyone who *does* just have virtually-addressed buffers. This is especially true if they can be in the vmalloc region, including the stack, as in that case the conversion to a scatterlist has to be done page-by-page. And even for users who have all of their data in bare 'struct page', none of them actually use 'struct scatterlist' as their native data structure anyway. They actually use skbs, bvecs, or other formats. - iov_iter is attractive, but ultimately not general enough either (considering the Ceph case for example), but also too general in some ways (like having support for userspace addresses). Additional iter types like ITER_SKB would help a bit, but bloating iov_iter with more types would reduce performance elsewhere in the kernel. Initial test coverage is provided by the crypto_aead support added in a later commit. I'm planning a KUnit test suite as well. Signed-off-by: Eric Biggers --- .../crypto/libcrypto-auth-encryption.rst | 13 + Documentation/crypto/libcrypto.rst | 1 + include/crypto/aes-gcm.h | 260 ++++++++++++++++ include/crypto/gcm.h | 4 +- lib/crypto/Kconfig | 8 + lib/crypto/aes.c | 280 ++++++++++++++++++ lib/crypto/tests/Kconfig | 1 + 7 files changed, 565 insertions(+), 2 deletions(-) create mode 100644 Documentation/crypto/libcrypto-auth-encryption.rst create mode 100644 include/crypto/aes-gcm.h diff --git a/Documentation/crypto/libcrypto-auth-encryption.rst b/Documenta= tion/crypto/libcrypto-auth-encryption.rst new file mode 100644 index 000000000000..06c94796b5f4 --- /dev/null +++ b/Documentation/crypto/libcrypto-auth-encryption.rst @@ -0,0 +1,13 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Authenticated encryption +=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D + +These APIs provide support for authenticated encryption and decryption. + +AES-GCM +------- + +This API provides support for AES in the GCM mode of operation. + +.. kernel-doc:: include/crypto/aes-gcm.h diff --git a/Documentation/crypto/libcrypto.rst b/Documentation/crypto/libc= rypto.rst index 33312c3ffad4..e911e0521597 100644 --- a/Documentation/crypto/libcrypto.rst +++ b/Documentation/crypto/libcrypto.rst @@ -159,6 +159,7 @@ API documentation .. toctree:: :maxdepth: 2 =20 + libcrypto-auth-encryption libcrypto-blockcipher libcrypto-hash libcrypto-signature diff --git a/include/crypto/aes-gcm.h b/include/crypto/aes-gcm.h new file mode 100644 index 000000000000..2aee62f01989 --- /dev/null +++ b/include/crypto/aes-gcm.h @@ -0,0 +1,260 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-GCM authenticated encryption and decryption + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_GCM_H +#define _CRYPTO_AES_GCM_H + +#include +#include +#include + +/** + * struct aes_gcm_key - A key prepared for AES-GCM encryption and decrypti= on + */ +struct aes_gcm_key { + /* private: */ + struct aes_enckey aes; + struct ghash_key ghash; + size_t authtag_len; /* Length of authentication tags in bytes */ +}; + +/** + * struct aes_gcm_ctx - Context for incrementally en/decrypting a message + */ +struct aes_gcm_ctx { + /* private: */ + /* + * Pointer to the key, which is assumed to live at least as long as this + * struct. + */ + const struct aes_gcm_key *key; + /* The current GHASH context */ + struct ghash_ctx ghash; + /* + * The current counter. This can be viewed as either a 128-bit big + * endian counter, or as a 96-bit nonce followed by a 32-bit big endian + * counter; it doesn't matter, since the last 32-bit word starts at 1, + * and AES-GCM is undefined for messages that would overflow that part. + * In practice this means that code optimized for AES-GCM can just + * increment the last 32-bit word (wrapping at 2^32), but when needed it + * can still call AES-CTR code that does a 128-bit increment. + * + * 'long' alignment is for crypto_xor() to work more efficiently. + */ + union { + u8 ctr[AES_BLOCK_SIZE]; + __be32 ctr32[AES_BLOCK_SIZE / 4]; + } __aligned(__alignof__(long)); + /* Buffered keystream for partial block updates */ + u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + /* Encrypted counter of 1. This gets XOR'ed with the tag at the end. */ + u8 j0_enc[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + /* Number of associated data bytes processed so far */ + u64 ad_len; + /* Number of en/decrypted bytes processed so far */ + u64 data_len; +}; + +/** + * aes_gcm_preparekey() - Prepare a key for AES-GCM encryption and decrypt= ion + * @key: (output) The key structure to initialize + * @in_key: The raw AES-GCM key + * @key_len: Length of the raw key in bytes: 16, 24, or 32 + * @authtag_len: Length of the authentication tag in bytes: + * 4, 8, 12, 13, 14, 15, or 16. 16 is recommended. + * + * Users should use memzero_explicit() to zeroize the key struct at the en= d of + * its lifetime. (But if this function fails, zeroization is unnecessary.) + * + * Context: Any context. + * Return: + * * 0 on success + * * -EINVAL if either of the lengths is invalid + */ +int __must_check aes_gcm_preparekey(struct aes_gcm_key *key, const u8 *in_= key, + size_t key_len, size_t authtag_len); + +/** + * aes_gcm_encrypt() - Encrypt a message with AES-GCM + * @dst: The destination ciphertext data. Can be in-place or out-of-place. + * For other overlaps the behavior is unspecified. + * @src: The source plaintext data + * @data_len: Length of plaintext in bytes (and ciphertext excluding the t= ag): + * at most 2^36 - 32 + * @authtag: The output authentication tag. Length is the authtag_len tha= t was + * passed to aes_gcm_preparekey(). Usually protocols using AES-GCM + * put the tag at the end of the ciphertext, in which case this should + * be set to @dst + @data_len and @dst must have room for the tag. + * @ad: The associated data + * @ad_len: Length of associated data in bytes: at most 2^61 - 1 + * @nonce: The 12-byte nonce. All (key, nonce) pairs used MUST be distinc= t. + * @key: The key, already prepared using aes_gcm_preparekey() + * + * For AES-GMAC (i.e., AES-GCM without any data en/decrypted), use dst=3DN= ULL, + * src=3DNULL, and data_len=3D0 to generate the AES-GMAC value. + * + * Context: Any context. + */ +void aes_gcm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, + const u8 *ad, size_t ad_len, const u8 nonce[at_least 12], + const struct aes_gcm_key *key); + +/** + * aes_gcm_decrypt() - Decrypt a message with AES-GCM + * @dst: The destination plaintext data. Can be in-place or out-of-place. + * For other overlaps the behavior is unspecified. + * @src: The source ciphertext data + * @data_len: Length of plaintext in bytes (and ciphertext excluding the t= ag): + * at most 2^36 - 32 + * @authtag: The stored authentication tag. Length is the authtag_len tha= t was + * passed to aes_gcm_preparekey(). Usually protocols using AES-GCM + * put the tag at the end of the ciphertext, in which case this should + * be set to @src + @data_len and @src must have room for the tag. + * @ad: The associated data + * @ad_len: Length of associated data in bytes: at most 2^61 - 1 + * @nonce: The 12-byte nonce + * @key: The key, already prepared using aes_gcm_preparekey() + * + * For AES-GMAC (i.e., AES-GCM without any data en/decrypted), use dst=3DN= ULL, + * src=3DNULL, and data_len=3D0 to verify the AES-GMAC value. + * + * Context: Any context. + * Return: + * * 0 on success. This is the only case where any decrypted or associate= d data + * can be used. + * * -EBADMSG if the message is inauthentic + */ +int __must_check aes_gcm_decrypt(u8 *dst, const u8 *src, size_t data_len, + const u8 *authtag, const u8 *ad, size_t ad_len, + const u8 nonce[at_least 12], + const struct aes_gcm_key *key); + +/** + * aes_gcm_init() - Initialize context for incremental AES-GCM encryption = or + * decryption, or for AES-GMAC computation + * @ctx: The context to initialize + * @nonce: The 12-byte nonce. All (key, nonce) pairs used for encryption = or MAC + * generation MUST be distinct. + * @key: The key, already prepared using aes_gcm_preparekey(). Note that a + * pointer to the key is saved in the context, so the key must live at + * least as long as the context. + * + * The context should be zeroized at the end of its lifetime. Normally th= at + * happens in aes_gcm_encrypt_final() or aes_gcm_decrypt_final(), but call= ers + * that abandon a context without finalizing it should explicitly zeroize = it. + * + * IMPORTANT: Callers that are decrypting data or computing a GMAC value f= or + * verification MUST NOT assume that any decrypted or associated data is + * authentic until the authentication tag has been verified. This increme= ntal + * API is provided solely to support callers that can't efficiently use the + * one-shot functions due to using a nonlinear data layout. + * + * For incremental AES-GCM encryption, use: + * + * 1. aes_gcm_init() + * 2. aes_gcm_auth_update() (any number of times) + * 3. aes_gcm_encrypt_update() (any number of times) + * 4. aes_gcm_encrypt_final() + * + * For incremental AES-GCM decryption, use: + * + * 1. aes_gcm_init() + * 2. aes_gcm_auth_update() (any number of times) + * 3. aes_gcm_decrypt_update() (any number of times) + * 4. aes_gcm_decrypt_final() + * + * AES-GMAC is just AES-GCM with zero bytes en/decrypted. For incremental + * AES-GMAC computation, use: + * + * 1. aes_gcm_init() + * 2. aes_gcm_auth_update() (any number of times) + * 3. aes_gcm_encrypt_final() to return the computed tag to the caller, or + * aes_gcm_decrypt_final() to directly verify the computed tag + * + * Context: Any context. + */ +void aes_gcm_init(struct aes_gcm_ctx *ctx, const u8 nonce[at_least 12], + const struct aes_gcm_key *key); + +/** + * aes_gcm_auth_update() - Incrementally process AES-GCM associated data + * @ctx: An AES-GCM context + * @ad: The associated data + * @len: Number of bytes provided. The caller must ensure that the total + * associated data length doesn't exceed GCM's limit of 2^61 - 1. + * + * IMPORTANT: Callers MUST NOT assume that any decrypted or associated dat= a is + * authentic until the authentication tag has been verified. + * + * Context: Any context. + */ +void aes_gcm_auth_update(struct aes_gcm_ctx *ctx, const u8 *ad, size_t len= ); + +/** + * aes_gcm_encrypt_update() - Incrementally encrypt data with AES-GCM + * @ctx: An AES-GCM context + * @dst: The destination buffer. Can be in-place or out-of-place. For ot= her + * overlaps the behavior is unspecified. + * @src: The source plaintext data + * @len: Number of bytes to encrypt. The caller must ensure that the total + * number of bytes encrypted doesn't exceed GCM's limit of 2^36 - 32. + * + * This can be called only after all associated data has been processed. + * + * Context: Any context. + */ +void aes_gcm_encrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *sr= c, + size_t len); + +/** + * aes_gcm_decrypt_update() - Incrementally decrypt data with AES-GCM + * @ctx: An AES-GCM context + * @dst: The destination buffer. Can be in-place or out-of-place. For ot= her + * overlaps the behavior is unspecified. + * @src: The source ciphertext data (not including auth tag) + * @len: Number of bytes to decrypt. The caller must ensure that the total + * number of bytes decrypted doesn't exceed GCM's limit of 2^36 - 32. + * + * This can be called only after all associated data has been processed. + * + * IMPORTANT: Callers MUST NOT assume that any decrypted or associated dat= a is + * authentic until the authentication tag has been verified. + * + * Context: Any context. + */ +void aes_gcm_decrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *sr= c, + size_t len); + +/** + * aes_gcm_encrypt_final() - Finish encrypting a message with AES-GCM + * @ctx: An AES-GCM context + * @authtag: The output authentication tag. Length is the authtag_len tha= t was + * passed to aes_gcm_preparekey(). + * + * This also zeroizes @ctx, so the caller doesn't need to do it. + * + * Context: Any context. + */ +void aes_gcm_encrypt_final(struct aes_gcm_ctx *ctx, u8 *authtag); + +/** + * aes_gcm_decrypt_final() - Finish decrypting a message with AES-GCM + * @ctx: An AES-GCM context + * @authtag: The stored authentication tag. Length is the authtag_len tha= t was + * passed to aes_gcm_preparekey(). + * + * This also zeroizes @ctx, so the caller doesn't need to do it. + * + * Context: Any context. + * Return: + * * 0 on success. This is the only case where any decrypted or associate= d data + * can be used. + * * -EBADMSG if the message is inauthentic + */ +int __must_check aes_gcm_decrypt_final(struct aes_gcm_ctx *ctx, + const u8 *authtag); + +#endif /* _CRYPTO_AES_GCM_H */ diff --git a/include/crypto/gcm.h b/include/crypto/gcm.h index 1d5f39ff1dc4..7fd7892ad818 100644 --- a/include/crypto/gcm.h +++ b/include/crypto/gcm.h @@ -13,7 +13,7 @@ /* * validate authentication tag for GCM */ -static inline int crypto_gcm_check_authsize(unsigned int authsize) +static inline int crypto_gcm_check_authsize(size_t authsize) { switch (authsize) { case 4: @@ -34,7 +34,7 @@ static inline int crypto_gcm_check_authsize(unsigned int = authsize) /* * validate authentication tag for RFC4106 */ -static inline int crypto_rfc4106_check_authsize(unsigned int authsize) +static inline int crypto_rfc4106_check_authsize(size_t authsize) { switch (authsize) { case 8: diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 6ec47cc328c8..b48b0b2299a8 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -53,6 +53,14 @@ config CRYPTO_LIB_AES_ECB help The AES-ECB library functions. =20 +config CRYPTO_LIB_AES_GCM + tristate + select CRYPTO_LIB_AES + select CRYPTO_LIB_AES_CTR + select CRYPTO_LIB_GF128HASH + help + The AES-GCM library functions. + config CRYPTO_LIB_AES_XTS tristate select CRYPTO_LIB_AES diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index 03c80f4fe176..473c0c16bfa2 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -1307,6 +1308,285 @@ void aes_xts_decrypt(u8 *dst, const u8 *src, size_t= len, EXPORT_SYMBOL_GPL(aes_xts_decrypt); #endif /* CONFIG_CRYPTO_LIB_AES_XTS */ =20 +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_GCM) +/* + * Hooks for optimized AES-GCM implementations, overridable by the archite= cture. + * They are called with len > 0 && len % AES_BLOCK_SIZE =3D=3D 0. I.e. th= ey aren't + * expected to handle empty inputs or partial blocks, as those cases are h= andled + * by non-arch-specific code instead. + * + * The GHASH accumulator is provided in POLYVAL format. The counter is pr= ovided + * in big endian format, and it's read-only, as the caller handles updatin= g it. + * + * Returning false causes the fallback implementation to be used instead. + * + * These hooks are used only for en/decrypted data. For the associated da= ta the + * GHASH functions are called instead, so those should be implemented too. + */ +#ifndef aes_gcm_encrypt_update_arch +static bool aes_gcm_encrypt_update_arch(u8 *dst, const u8 *src, size_t len, + struct polyval_elem *ghash_acc, + const __be32 ctr32[4], + const struct aes_enckey *aes_key, + const struct ghash_key *ghash_key) +{ + return false; +} +#endif +#ifndef aes_gcm_decrypt_update_arch +static bool aes_gcm_decrypt_update_arch(u8 *dst, const u8 *src, size_t len, + struct polyval_elem *ghash_acc, + const __be32 ctr32[4], + const struct aes_enckey *aes_key, + const struct ghash_key *ghash_key) +{ + return false; +} +#endif + +int aes_gcm_preparekey(struct aes_gcm_key *key, const u8 *in_key, + size_t key_len, size_t authtag_len) +{ + u8 h[AES_BLOCK_SIZE] =3D { 0 }; + int err; + + err =3D crypto_gcm_check_authsize(authtag_len); + if (unlikely(err)) + return err; + + err =3D aes_prepareenckey(&key->aes, in_key, key_len); + if (unlikely(err)) + return err; + + aes_encrypt(&key->aes, h, h); + ghash_preparekey(&key->ghash, h); + + key->authtag_len =3D authtag_len; + + memzero_explicit(h, sizeof(h)); + return 0; +} +EXPORT_SYMBOL_GPL(aes_gcm_preparekey); + +void aes_gcm_init(struct aes_gcm_ctx *ctx, const u8 nonce[12], + const struct aes_gcm_key *key) +{ + ctx->key =3D key; + ctx->ad_len =3D 0; + ctx->data_len =3D 0; + ghash_init(&ctx->ghash, &key->ghash); + memset(ctx->keystream, 0, sizeof(ctx->keystream)); + + memcpy(ctx->ctr32, nonce, 12); + ctx->ctr32[3] =3D cpu_to_be32(1); + + aes_encrypt(&key->aes, ctx->j0_enc, ctx->ctr); + ctx->ctr32[3] =3D cpu_to_be32(2); +} +EXPORT_SYMBOL_GPL(aes_gcm_init); + +void aes_gcm_auth_update(struct aes_gcm_ctx *ctx, const u8 *ad, size_t len) +{ + WARN_ON_ONCE(ctx->data_len !=3D 0); + if (len) { + ghash_update(&ctx->ghash, ad, len); + ctx->ad_len +=3D len; + } +} +EXPORT_SYMBOL_GPL(aes_gcm_auth_update); + +static const u8 gcm_zeroes[AES_BLOCK_SIZE]; + +static __always_inline void ghash_pad(struct ghash_ctx *ghash, u64 len) +{ + if (len % AES_BLOCK_SIZE) + ghash_update(ghash, gcm_zeroes, -len % AES_BLOCK_SIZE); +} + +static __always_inline void aes_gcm_crypt_update(struct aes_gcm_ctx *ctx, + u8 *dst, const u8 *src, + size_t len, bool enc) +{ + size_t partial_len, n; + + if (unlikely(len =3D=3D 0)) + return; + + partial_len =3D ctx->data_len % AES_BLOCK_SIZE; + if (ctx->data_len =3D=3D 0) + ghash_pad(&ctx->ghash, ctx->ad_len); + ctx->data_len +=3D len; + + if (unlikely(partial_len !=3D 0)) { + /* + * The previous call ended on a non-block-aligned data_len, so + * continue using a previously-generated keystream block. + */ + n =3D min(len, AES_BLOCK_SIZE - partial_len); + if (enc) { + crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], + n); + ghash_update(&ctx->ghash, dst, n); + } else { + ghash_update(&ctx->ghash, src, n); + crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], + n); + } + dst +=3D n; + src +=3D n; + len -=3D n; + } + + if (len >=3D AES_BLOCK_SIZE) { + n =3D round_down(len, AES_BLOCK_SIZE); + if (enc) { + if (likely(aes_gcm_encrypt_update_arch( + dst, src, n, &ctx->ghash.acc, ctx->ctr32, + &ctx->key->aes, &ctx->key->ghash))) { + be32_add_cpu(&ctx->ctr32[3], + n / AES_BLOCK_SIZE); + } else { + aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); + ghash_update(&ctx->ghash, dst, n); + } + } else { + if (likely(aes_gcm_decrypt_update_arch( + dst, src, n, &ctx->ghash.acc, ctx->ctr32, + &ctx->key->aes, &ctx->key->ghash))) { + be32_add_cpu(&ctx->ctr32[3], + n / AES_BLOCK_SIZE); + } else { + ghash_update(&ctx->ghash, src, n); + aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); + } + } + dst +=3D n; + src +=3D n; + len -=3D n; + } + + if (len !=3D 0) { + /* + * Ending on a non-block aligned data_len. Generate the next + * keystream block, use the needed portion of it, and leave it + * cached in ctx->keystream in case this isn't the final call. + */ + aes_encrypt(&ctx->key->aes, ctx->keystream, ctx->ctr); + be32_add_cpu(&ctx->ctr32[3], 1); + if (enc) { + crypto_xor_cpy(dst, src, ctx->keystream, len); + ghash_update(&ctx->ghash, dst, len); + } else { + ghash_update(&ctx->ghash, src, len); + crypto_xor_cpy(dst, src, ctx->keystream, len); + } + } +} + +void aes_gcm_encrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *sr= c, + size_t len) +{ + aes_gcm_crypt_update(ctx, dst, src, len, /* enc=3D */ true); +} +EXPORT_SYMBOL_GPL(aes_gcm_encrypt_update); + +void aes_gcm_decrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *sr= c, + size_t len) +{ + aes_gcm_crypt_update(ctx, dst, src, len, /* enc=3D */ false); +} +EXPORT_SYMBOL_GPL(aes_gcm_decrypt_update); + +/* Maximum AES-GCM associated data length in bytes */ +#define AES_GCM_MAX_AD_LEN ((1ULL << 61) - 1) +/* Maximum AES-GCM en/decrypted data length in bytes */ +#define AES_GCM_MAX_DATA_LEN ((1ULL << 36) - 32) + +void aes_gcm_encrypt_final(struct aes_gcm_ctx *ctx, u8 *authtag) +{ + __be64 tail[2]; + + WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN); + WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN); + + ghash_pad(&ctx->ghash, + ctx->data_len =3D=3D 0 ? ctx->ad_len : ctx->data_len); + + tail[0] =3D cpu_to_be64(ctx->ad_len * 8); + tail[1] =3D cpu_to_be64(ctx->data_len * 8); + ghash_update(&ctx->ghash, (const u8 *)tail, 16); + ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ + + crypto_xor_cpy(authtag, ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(aes_gcm_encrypt_final); + +int aes_gcm_decrypt_final(struct aes_gcm_ctx *ctx, const u8 *authtag) +{ + __be64 tail[2]; + int err; + + if (WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN) || + WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN)) { + err =3D -EBADMSG; + goto out; + } + + ghash_pad(&ctx->ghash, + ctx->data_len =3D=3D 0 ? ctx->ad_len : ctx->data_len); + + tail[0] =3D cpu_to_be64(ctx->ad_len * 8); + tail[1] =3D cpu_to_be64(ctx->data_len * 8); + ghash_update(&ctx->ghash, (const u8 *)tail, 16); + ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ + crypto_xor(ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); + err =3D crypto_memneq(ctx->ctr, authtag, ctx->key->authtag_len) ? + -EBADMSG : + 0; +out: + memzero_explicit(ctx, sizeof(*ctx)); + return err; +} +EXPORT_SYMBOL_GPL(aes_gcm_decrypt_final); + +void aes_gcm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, + const u8 *ad, size_t ad_len, const u8 nonce[12], + const struct aes_gcm_key *key) +{ + struct aes_gcm_ctx ctx; + + aes_gcm_init(&ctx, nonce, key); + aes_gcm_auth_update(&ctx, ad, ad_len); + aes_gcm_encrypt_update(&ctx, dst, src, data_len); + aes_gcm_encrypt_final(&ctx, authtag); +} +EXPORT_SYMBOL_GPL(aes_gcm_encrypt); + +int aes_gcm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *aut= htag, + const u8 *ad, size_t ad_len, const u8 nonce[12], + const struct aes_gcm_key *key) +{ + struct aes_gcm_ctx ctx; + int err; + + aes_gcm_init(&ctx, nonce, key); + aes_gcm_auth_update(&ctx, ad, ad_len); + aes_gcm_decrypt_update(&ctx, dst, src, data_len); + err =3D aes_gcm_decrypt_final(&ctx, authtag); + if (unlikely(err) && data_len) { + /* + * Clear the inauthentic decrypted data so that callers won't + * receive it even if they fail to correctly handle errors. + */ + memset(dst, 0, data_len); + } + return err; +} +EXPORT_SYMBOL_GPL(aes_gcm_decrypt); + +#endif /* CONFIG_CRYPTO_LIB_AES_GCM */ + static int __init aes_mod_init(void) { #ifdef aes_mod_init_arch diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index b559e7c79e76..51183ffabbef 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -148,6 +148,7 @@ config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT select CRYPTO_LIB_AES_CBC_MACS select CRYPTO_LIB_AES_CTR select CRYPTO_LIB_AES_ECB + select CRYPTO_LIB_AES_GCM select CRYPTO_LIB_AES_XTS select CRYPTO_LIB_BLAKE2B select CRYPTO_LIB_CHACHA20POLY1305 --=20 2.55.0 From nobody Sat Jul 25 16:48:31 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2194A3D4128; Wed, 15 Jul 2026 22:12:44 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153566; cv=none; b=H9ie+2/cJOtxr37eVO9mBaWlQxTYHoB0U1OsXe19+/Lc8XMpts3ZUWOmQ1ixs4qjF6DPtB54WCI3Ahoz9XQwv8Tg2g4Ge1ri168w06zYKwV+/+xUqd4KKx4Amj7Xa+zSf4R9skjggODRJ7fLabAnJnqN/pLJqaFKbpF9lEbcXAo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153566; c=relaxed/simple; bh=YiYrVApb87gWl8FHXLt8kK0sChpEdt5lHy3qC2E/duU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=tHHQiWDVXyNHsjcb7rWkKsed535PqCjg+xMDV55E5DNypFnGk+VAiWwKO67vBCakxugjMrFtSkj/l0nSuTBKPH6CtBq24zXv2k+C+sNGKQxexi5TmhSKHUB1/upbf5zXgLZpMoHylg/izwK3SR5uAj6+31lQ8yvJhLgLxyVJPeE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=K8JrzRB1; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="K8JrzRB1" Received: by smtp.kernel.org (Postfix) with ESMTPSA id B45F91F000E9; Wed, 15 Jul 2026 22:12:43 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784153563; bh=uxnqtyOJEN4mAKj+NucrzZthvNBnbWV7u4U8T2tktyA=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=K8JrzRB1DhZs/a8x9sgqf+Oi3ROKpxwfC0fghuG7F+I04tZQ3YiTtiMGIDnJdwsTJ v0rEgjeZgsdDryaRhrku1otP5yLP9U5AhzCNc6RGf7lBMAHdxp5M5S1b+BDerraAcI sZrKvlytxT8s+2Qr28FxVcBqVq7fJQRoCEy+Cjh//W8MK5XDruSeOiwl1L+Ae90Igs scQTV1xtAJEgg3YCPb5eJg7eqJOeskCQjxGZmQO5RT8txXFeshHy801yP1DijDUUTV FnOTEBQhbHeqKrYEIpWA9xtwtyYpC4MS4NaN6VwsmkpP+HLx2laR8gI57BW+0KAGgB wQOiPuxVtxuug== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , Thomas Huth , Eric Biggers Subject: [PATCH v2 07/13] lib/crypto: aes: Add CCM support Date: Wed, 15 Jul 2026 15:11:47 -0700 Message-ID: <20260715221153.246410-8-ebiggers@kernel.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260715221153.246410-1-ebiggers@kernel.org> References: <20260715221153.246410-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Add support for AES-CCM to the crypto library. This will be used to provide a streamlined implementation of the "ccm(aes)" crypto_aead algorithm. Most users of "ccm(aes)" will also be able to switch to the library, which as usual will be faster and simpler, e.g.: - fs/smb/client/ - fs/smb/server/ - net/mac80211/ - net/mac802154/ (I've already written proof-of-concept patches for all the above, and they helped inform the API design.) As in the AES-GCM API, incremental operation is supported. It has to be used carefully, especially when decrypting, but it makes the API general enough to work well for all users. The AES-CCM library code calls aes_cbcmac_blocks() directly, bypassing the higher-level aes_cbcmac_init(), aes_cbcmac_update(), and aes_cbcmac_final(). The latter set of functions is useful only for AES-CCM, so they don't make sense to keep around and will be removed once the "ccm(aes)" crypto_aead starts using the AES-CCM library. Initial test coverage is provided by the crypto_aead support added in a later commit. I'm planning a KUnit test suite as well. Signed-off-by: Eric Biggers --- .../crypto/libcrypto-auth-encryption.rst | 7 + include/crypto/aes-ccm.h | 266 +++++++++++++++ lib/crypto/Kconfig | 8 + lib/crypto/aes.c | 313 ++++++++++++++++++ lib/crypto/tests/Kconfig | 1 + 5 files changed, 595 insertions(+) create mode 100644 include/crypto/aes-ccm.h diff --git a/Documentation/crypto/libcrypto-auth-encryption.rst b/Documenta= tion/crypto/libcrypto-auth-encryption.rst index 06c94796b5f4..1e527685a42f 100644 --- a/Documentation/crypto/libcrypto-auth-encryption.rst +++ b/Documentation/crypto/libcrypto-auth-encryption.rst @@ -5,6 +5,13 @@ Authenticated encryption =20 These APIs provide support for authenticated encryption and decryption. =20 +AES-CCM +------- + +This API provides support for AES in the CCM mode of operation. + +.. kernel-doc:: include/crypto/aes-ccm.h + AES-GCM ------- =20 diff --git a/include/crypto/aes-ccm.h b/include/crypto/aes-ccm.h new file mode 100644 index 000000000000..8b00859ac4d6 --- /dev/null +++ b/include/crypto/aes-ccm.h @@ -0,0 +1,266 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-CCM authenticated encryption and decryption + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_CCM_H +#define _CRYPTO_AES_CCM_H + +#include + +/** + * struct aes_ccm_key - A key prepared for AES-CCM encryption and decrypti= on + */ +struct aes_ccm_key { + /* private: */ + struct aes_enckey aes; + size_t authtag_len; /* Length of authentication tags in bytes */ +}; + +/** + * struct aes_ccm_ctx - Context for incrementally en/decrypting a message + */ +struct aes_ccm_ctx { + /* private: */ + /* + * Pointer to the key, which is assumed to live at least as long as this + * struct. + */ + const struct aes_ccm_key *key; + /* + * The current CBC-MAC chaining value. When not on a block boundary, + * the partial block has been XOR'ed into this. The number of partial + * bytes is 'partial_len'. + */ + u8 mac[AES_BLOCK_SIZE] __aligned(__alignof__(__be64)); + /* The current counter, a 128-bit big endian value */ + u8 ctr[AES_BLOCK_SIZE] __aligned(__alignof__(__be64)); + /* Buffered keystream for partial block updates */ + u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(__be64)); + /* Encrypted counter of 0. This gets XOR'ed with the tag at the end. */ + u8 s0[AES_BLOCK_SIZE] __aligned(__alignof__(__be64)); + /* Number of associated data bytes remaining to be provided */ + u64 ad_remaining; + /* Number of en/decrypted data bytes remaining to be provided */ + u64 data_remaining; + /* Current partial block length, 0 <=3D partial_len < AES_BLOCK_SIZE */ + u32 partial_len; + /* True if associated data padding has been done */ + bool ad_padded; +}; + +/** + * aes_ccm_preparekey() - Prepare a key for AES-CCM encryption and decrypt= ion + * @key: (output) The key structure to initialize + * @in_key: The raw AES-CCM key + * @key_len: Length of the raw key in bytes: 16, 24, or 32 + * @authtag_len: Length of the authentication tag in bytes: + * 4, 6, 8, 10, 12, 14, or 16. 16 is recommended. + * + * Users should use memzero_explicit() to zeroize the key struct at the en= d of + * its lifetime. (But if this function fails, zeroization is unnecessary.) + * + * Context: Any context. + * Return: + * * 0 on success + * * -EINVAL if either of the lengths is invalid + */ +int __must_check aes_ccm_preparekey(struct aes_ccm_key *key, const u8 *in_= key, + size_t key_len, size_t authtag_len); + +/** + * aes_ccm_encrypt() - Encrypt a message with AES-CCM + * @dst: The destination ciphertext data. Can be in-place or out-of-place. + * For other overlaps the behavior is unspecified. + * @src: The source plaintext data + * @data_len: Length of plaintext in bytes (and ciphertext excluding the t= ag): + * at most 2^(120 - (8 * @nonce_len)) - 1 + * @authtag: The output authentication tag. Length is the authtag_len tha= t was + * passed to aes_ccm_preparekey(). Usually protocols using AES-CCM + * put the tag at the end of the ciphertext, in which case this should + * be set to @dst + @data_len and @dst must have room for the tag. + * @ad: The associated data + * @ad_len: Length of associated data in bytes + * @nonce: The nonce. All (key, nonce) pairs used MUST be distinct. + * @nonce_len: Length of the nonce in bytes: between 7 and 13 inclusive + * @key: The key, already prepared using aes_ccm_preparekey() + * + * Context: Any context. + * Return: + * * 0 on success + * * -EINVAL if @nonce_len is invalid + * * -EOVERFLOW if @data_len is too large for the selected @nonce_len + */ +int __must_check aes_ccm_encrypt(u8 *dst, const u8 *src, size_t data_len, + u8 *authtag, const u8 *ad, size_t ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_ccm_key *key); + +/** + * aes_ccm_decrypt() - Decrypt a message with AES-CCM + * @dst: The destination plaintext data. Can be in-place or out-of-place. + * For other overlaps the behavior is unspecified. + * @src: The source ciphertext data + * @data_len: Length of plaintext in bytes (and ciphertext excluding the t= ag): + * at most 2^(120 - (8 * @nonce_len)) - 1 + * @authtag: The stored authentication tag. Length is the authtag_len tha= t was + * passed to aes_ccm_preparekey(). Usually protocols using AES-CCM + * put the tag at the end of the ciphertext, in which case this should + * be set to @src + @data_len and @src must have room for the tag. + * @ad: The associated data + * @ad_len: Length of associated data in bytes + * @nonce: The nonce + * @nonce_len: Length of the nonce in bytes: between 7 and 13 inclusive + * @key: The key, already prepared using aes_ccm_preparekey() + * + * Context: Any context. + * Return: + * * 0 on success. This is the only case where any decrypted or associate= d data + * can be used. + * * -EBADMSG if the message is inauthentic + * * -EINVAL if @nonce_len is invalid + * * -EOVERFLOW if @data_len is too large for the selected @nonce_len + */ +int __must_check aes_ccm_decrypt(u8 *dst, const u8 *src, size_t data_len, + const u8 *authtag, const u8 *ad, size_t ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_ccm_key *key); + +/** + * aes_ccm_init() - Initialize context for incremental AES-CCM encryption = or + * decryption + * @ctx: The context to initialize + * @data_len: Length of the en/decrypted data that will be provided in byt= es: + * at most 2^(120 - (8 * @nonce_len)) - 1 + * @ad_len: Length of the associated data that will be provided in bytes + * @nonce: The nonce. All (key, nonce) pairs used for encryption MUST be + * distinct. + * @nonce_len: Length of the nonce in bytes: between 7 and 13 inclusive + * @key: The key, already prepared using aes_ccm_preparekey(). Note that a + * pointer to the key is saved in the context, so the key must live at + * least as long as the context. + * + * Unlike AES-GCM, AES-CCM requires the total lengths of the associated da= ta and + * the en/decrypted data to be known during initialization. Callers MUST = ensure + * that these lengths are correct. + * + * If this function returns success, the context should be zeroized at the= end + * of its lifetime. Normally that happens in aes_ccm_encrypt_final() or + * aes_ccm_decrypt_final(), but callers that abandon a context without + * finalizing it should explicitly zeroize it. + * + * IMPORTANT: Callers that are decrypting MUST NOT assume that any decrypt= ed or + * associated data is authentic until the authentication tag has been veri= fied. + * This incremental API is provided solely to support callers that can't + * efficiently use the one-shot functions due to using a nonlinear data la= yout. + * + * For incremental AES-CCM encryption, use: + * + * 1. aes_ccm_init() + * 2. aes_ccm_auth_update() (any number of times) + * 3. aes_ccm_encrypt_update() (any number of times) + * 4. aes_ccm_encrypt_final() + * + * For incremental AES-CCM decryption, use: + * + * 1. aes_ccm_init() + * 2. aes_ccm_auth_update() (any number of times) + * 3. aes_ccm_decrypt_update() (any number of times) + * 4. aes_ccm_decrypt_final() + * + * Context: Any context. + * Return: + * * 0 on success + * * -EINVAL if @nonce_len is invalid + * * -EOVERFLOW if @data_len is too large for the selected @nonce_len + */ +int __must_check aes_ccm_init(struct aes_ccm_ctx *ctx, u64 data_len, u64 a= d_len, + const u8 *nonce, size_t nonce_len, + const struct aes_ccm_key *key); + +/** + * aes_ccm_auth_update() - Incrementally process AES-CCM associated data + * @ctx: An AES-CCM context + * @ad: The associated data + * @len: Length of the associated data in bytes + * + * IMPORTANT: Callers MUST NOT assume that any decrypted or associated dat= a is + * authentic until the authentication tag has been verified. + * + * The total length of the associated data (over all calls to this functio= n) + * MUST match the ad_len that was passed to aes_ccm_init(). + * + * Context: Any context. + */ +void aes_ccm_auth_update(struct aes_ccm_ctx *ctx, const u8 *ad, size_t len= ); + +/** + * aes_ccm_encrypt_update() - Incrementally encrypt data with AES-CCM + * @ctx: An AES-CCM context + * @dst: The destination buffer. Can be in-place or out-of-place. For ot= her + * overlaps the behavior is unspecified. + * @src: The source plaintext data + * @len: Number of bytes to encrypt + * + * This can be called only after all associated data has been processed. + * + * The total length of the encrypted data (over all calls to this function= ) MUST + * match the data_len that was passed to aes_ccm_init(). + * + * Context: Any context. + */ +void aes_ccm_encrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *sr= c, + size_t len); + +/** + * aes_ccm_decrypt_update() - Incrementally decrypt data with AES-CCM + * @ctx: An AES-CCM context + * @dst: The destination buffer. Can be in-place or out-of-place. For ot= her + * overlaps the behavior is unspecified. + * @src: The source ciphertext data (not including auth tag) + * @len: Number of bytes to decrypt + * + * This can be called only after all associated data has been processed. + * + * The total length of the decrypted data (over all calls to this function= ) MUST + * match the data_len that was passed to aes_ccm_init(). + * + * IMPORTANT: Callers MUST NOT assume that any decrypted or associated dat= a is + * authentic until the authentication tag has been verified. + * + * Context: Any context. + */ +void aes_ccm_decrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *sr= c, + size_t len); + +/** + * aes_ccm_encrypt_final() - Finish encrypting a message with AES-CCM + * @ctx: An AES-CCM context + * @authtag: The output authentication tag. Length is the authtag_len tha= t was + * passed to aes_ccm_preparekey(). + * + * This also zeroizes @ctx, so the caller doesn't need to do it. + * + * Context: Any context. + */ +void aes_ccm_encrypt_final(struct aes_ccm_ctx *ctx, u8 *authtag); + +/** + * aes_ccm_decrypt_final() - Finish decrypting a message with AES-CCM + * @ctx: An AES-CCM context + * @authtag: The stored authentication tag. Length is the authtag_len tha= t was + * passed to aes_ccm_preparekey(). + * + * This also zeroizes @ctx, so the caller doesn't need to do it. + * + * Context: Any context. + * Return: + * * 0 on success. This is the only case where any decrypted or associate= d data + * can be used. + * * -EBADMSG if the message is inauthentic + */ +int __must_check aes_ccm_decrypt_final(struct aes_ccm_ctx *ctx, + const u8 *authtag); + +#endif /* _CRYPTO_AES_CCM_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index b48b0b2299a8..65a478f69715 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -41,6 +41,14 @@ config CRYPTO_LIB_AES_CBC_MACS this if your module uses any of the functions from . =20 +config CRYPTO_LIB_AES_CCM + tristate + select CRYPTO_LIB_AES + select CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_AES_CTR + help + The AES-CCM library functions. + config CRYPTO_LIB_AES_CTR tristate select CRYPTO_LIB_AES diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index 473c0c16bfa2..4222a4cec2f2 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -6,6 +6,7 @@ =20 #include #include +#include #include #include #include @@ -1587,6 +1588,318 @@ EXPORT_SYMBOL_GPL(aes_gcm_decrypt); =20 #endif /* CONFIG_CRYPTO_LIB_AES_GCM */ =20 +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CCM) +int aes_ccm_preparekey(struct aes_ccm_key *key, const u8 *in_key, + size_t key_len, size_t authtag_len) +{ + int err; + + if (unlikely(authtag_len < 4 || authtag_len > 16 || authtag_len % 2)) + return -EINVAL; + + err =3D aes_prepareenckey(&key->aes, in_key, key_len); + if (unlikely(err)) + return err; + + key->authtag_len =3D authtag_len; + return 0; +} +EXPORT_SYMBOL_GPL(aes_ccm_preparekey); + +int aes_ccm_init(struct aes_ccm_ctx *ctx, u64 data_len, u64 ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_ccm_key *key) +{ + /* + * This is the value L defined in the CCM specification. It determines + * the maximum allowed message length, and it is itself determined by + * the nonce length. They are inversely related, i.e. the longer the + * nonce the smaller the maximum message length is. + */ + unsigned int l =3D 15 - nonce_len; + + if (unlikely(nonce_len < 7 || nonce_len > 13)) + return -EINVAL; + /* Thus 2 <=3D l <=3D 8. */ + + /* Check whether data_len can be represented in 'l' bytes. */ + if (unlikely(data_len > U64_MAX >> (64 - 8 * l))) + return -EOVERFLOW; + + ctx->key =3D key; + ctx->ad_remaining =3D ad_len; + ctx->data_remaining =3D data_len; + ctx->ad_padded =3D false; + + /* + * Initialize the zero-th counter block to: + * + * L - 1 || nonce || 0 + * + * ... and the zero-th CBC-MAC block to: + * + * Flags || nonce || data_len + */ + *(__be64 *)&ctx->ctr[8] =3D 0; + *(__be64 *)&ctx->mac[8] =3D cpu_to_be64(data_len); + ctx->ctr[0] =3D l - 1; + ctx->mac[0] =3D (ad_len ? 0x40 : 0) | + (((key->authtag_len - 2) / 2) << 3) | (l - 1); + memcpy(&ctx->ctr[1], nonce, nonce_len); /* Overlapping store */ + memcpy(&ctx->mac[1], nonce, nonce_len); /* Overlapping store */ + + /* + * Generate S_0 by encrypting the counter (this is used to encrypt the + * auth tag later), and encrypt the zero-th CBC-MAC block. + */ + aes_encrypt(&key->aes, ctx->s0, ctx->ctr); + aes_encrypt(&key->aes, ctx->mac, ctx->mac); + + /* Increment the counter from 0 to 1. */ + ctx->ctr[15] =3D 1; + + if (ad_len) { + /* + * Update CBC-MAC with the associated data length, represented + * using either 2, 6, or 10 bytes depending on the length. + */ + if (likely(ad_len < 0xff00)) { + *(__be16 *)&ctx->mac[0] ^=3D cpu_to_be16(ad_len); + ctx->partial_len =3D 2; + } else if (ad_len <=3D U32_MAX) { + __be32 *p =3D (__be32 *)&ctx->mac[2]; + + *(__be16 *)&ctx->mac[0] ^=3D cpu_to_be16(0xfffe); + put_unaligned(get_unaligned(p) ^ cpu_to_be32(ad_len), + p); + ctx->partial_len =3D 6; + } else { + __be64 *p =3D (__be64 *)&ctx->mac[2]; + + *(__be16 *)&ctx->mac[0] ^=3D cpu_to_be16(0xffff); + put_unaligned(get_unaligned(p) ^ cpu_to_be64(ad_len), + p); + ctx->partial_len =3D 10; + } + } else { + ctx->partial_len =3D 0; + } + return 0; +} +EXPORT_SYMBOL_GPL(aes_ccm_init); + +void aes_ccm_auth_update(struct aes_ccm_ctx *ctx, const u8 *ad, size_t len) +{ + size_t partial_len =3D ctx->partial_len; + bool enc_before =3D false; + size_t nblocks; + + WARN_ON_ONCE(ctx->ad_padded); + + /* + * We could warn on len > ad_remaining here, but underflow will be + * caught by the !=3D 0 check at the end anyway. (It's a u64, so it isn't + * going to underflow all the way back to 0.) + */ + ctx->ad_remaining -=3D len; + + if (partial_len) { + size_t n =3D min(len, AES_BLOCK_SIZE - partial_len); + + crypto_xor(&ctx->mac[partial_len], ad, n); + ad +=3D n; + len -=3D n; + partial_len +=3D n; + if (partial_len < AES_BLOCK_SIZE) { + ctx->partial_len =3D partial_len; + return; + } + enc_before =3D true; + } + + nblocks =3D len / AES_BLOCK_SIZE; + len %=3D AES_BLOCK_SIZE; + if (nblocks =3D=3D 0) { + if (enc_before) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + } else { + aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, ad, nblocks, + enc_before, /* enc_after=3D */ true); + ad +=3D nblocks * AES_BLOCK_SIZE; + } + crypto_xor(ctx->mac, ad, len); + ctx->partial_len =3D len; +} +EXPORT_SYMBOL_GPL(aes_ccm_auth_update); + +static __always_inline void aes_ccm_crypt_update(struct aes_ccm_ctx *ctx, + u8 *dst, const u8 *src, + size_t len, bool enc) +{ + size_t partial_len =3D ctx->partial_len; + size_t n, nblocks; + + if (unlikely(len =3D=3D 0)) + return; + + WARN_ON_ONCE(ctx->ad_remaining !=3D 0); + + /* + * We could warn on len > data_remaining here, but underflow will be + * caught by the !=3D 0 check at the end anyway. (It's a u64, so it isn't + * going to underflow all the way back to 0.) + */ + ctx->data_remaining -=3D len; + + if (!ctx->ad_padded) { + ctx->ad_padded =3D true; + if (partial_len) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + } else if (partial_len) { + /* + * The previous call ended on a non-block-aligned data_len, so + * continue using a previously-generated keystream block. + */ + n =3D min(len, AES_BLOCK_SIZE - partial_len); + if (enc) + crypto_xor(&ctx->mac[partial_len], src, n); + crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], n); + if (!enc) + crypto_xor(&ctx->mac[partial_len], dst, n); + dst +=3D n; + src +=3D n; + len -=3D n; + partial_len +=3D n; + if (partial_len < AES_BLOCK_SIZE) { + ctx->partial_len =3D partial_len; + return; + } + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + } + + if (len >=3D AES_BLOCK_SIZE) { + n =3D round_down(len, AES_BLOCK_SIZE); + nblocks =3D len / AES_BLOCK_SIZE; + if (enc) + aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, src, + nblocks, /* enc_before=3D */ false, + /* enc_after=3D */ true); + aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); + if (!enc) + aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, dst, + nblocks, /* enc_before=3D */ false, + /* enc_after=3D */ true); + dst +=3D n; + src +=3D n; + len -=3D n; + } + + if (len) { + /* + * Ending on a non-block aligned data_len. Generate the next + * keystream block, use the needed portion of it, and leave it + * cached in ctx->keystream in case this isn't the final call. + */ + aes_encrypt(&ctx->key->aes, ctx->keystream, ctx->ctr); + inc_be128_ctr(ctx->ctr); + if (enc) + crypto_xor(ctx->mac, src, len); + crypto_xor_cpy(dst, src, ctx->keystream, len); + if (!enc) + crypto_xor(ctx->mac, dst, len); + } + ctx->partial_len =3D len; +} + +void aes_ccm_encrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *sr= c, + size_t len) +{ + aes_ccm_crypt_update(ctx, dst, src, len, /* enc=3D */ true); +} +EXPORT_SYMBOL_GPL(aes_ccm_encrypt_update); + +void aes_ccm_decrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *sr= c, + size_t len) +{ + aes_ccm_crypt_update(ctx, dst, src, len, /* enc=3D */ false); +} +EXPORT_SYMBOL_GPL(aes_ccm_decrypt_update); + +void aes_ccm_encrypt_final(struct aes_ccm_ctx *ctx, u8 *authtag) +{ + WARN_ON_ONCE(ctx->ad_remaining !=3D 0); + WARN_ON_ONCE(ctx->data_remaining !=3D 0); + if (ctx->partial_len) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + crypto_xor_cpy(authtag, ctx->mac, ctx->s0, ctx->key->authtag_len); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(aes_ccm_encrypt_final); + +int aes_ccm_decrypt_final(struct aes_ccm_ctx *ctx, const u8 *authtag) +{ + int err; + + if (WARN_ON_ONCE(ctx->ad_remaining !=3D 0) || + WARN_ON_ONCE(ctx->data_remaining !=3D 0)) { + err =3D -EBADMSG; + goto out; + } + + if (ctx->partial_len) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + crypto_xor(ctx->mac, ctx->s0, ctx->key->authtag_len); + err =3D crypto_memneq(ctx->mac, authtag, ctx->key->authtag_len) ? + -EBADMSG : + 0; +out: + memzero_explicit(ctx, sizeof(*ctx)); + return err; +} +EXPORT_SYMBOL_GPL(aes_ccm_decrypt_final); + +int aes_ccm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, + const u8 *ad, size_t ad_len, const u8 *nonce, + size_t nonce_len, const struct aes_ccm_key *key) +{ + struct aes_ccm_ctx ctx; + int err; + + err =3D aes_ccm_init(&ctx, data_len, ad_len, nonce, nonce_len, key); + if (unlikely(err)) + return err; + aes_ccm_auth_update(&ctx, ad, ad_len); + aes_ccm_encrypt_update(&ctx, dst, src, data_len); + aes_ccm_encrypt_final(&ctx, authtag); + return 0; +} +EXPORT_SYMBOL_GPL(aes_ccm_encrypt); + +int aes_ccm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *aut= htag, + const u8 *ad, size_t ad_len, const u8 *nonce, + size_t nonce_len, const struct aes_ccm_key *key) +{ + struct aes_ccm_ctx ctx; + int err; + + err =3D aes_ccm_init(&ctx, data_len, ad_len, nonce, nonce_len, key); + if (unlikely(err)) + return err; + aes_ccm_auth_update(&ctx, ad, ad_len); + aes_ccm_decrypt_update(&ctx, dst, src, data_len); + err =3D aes_ccm_decrypt_final(&ctx, authtag); + if (unlikely(err) && data_len) { + /* + * Clear the inauthentic decrypted data so that callers won't + * receive it even if they fail to correctly handle errors. + */ + memset(dst, 0, data_len); + } + return err; +} +EXPORT_SYMBOL_GPL(aes_ccm_decrypt); +#endif /* CONFIG_CRYPTO_LIB_AES_CCM */ + static int __init aes_mod_init(void) { #ifdef aes_mod_init_arch diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index 51183ffabbef..bc084dde424f 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -146,6 +146,7 @@ config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT depends on KUNIT select CRYPTO_LIB_AES_CBC select CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_AES_CCM select CRYPTO_LIB_AES_CTR select CRYPTO_LIB_AES_ECB select CRYPTO_LIB_AES_GCM --=20 2.55.0 From nobody Sat Jul 25 16:48:31 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 80CD03D45FA; Wed, 15 Jul 2026 22:12:44 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153566; cv=none; b=IaHACMQa9IyVRZr/O7Uo49Pngq5srykwRWL6pyCDbRQSL0KCkFVNCV4B+LSb0RqYqeB1lKud8Rk93DRuNVyk9DtWEoZq4CGOTaYk+9bvcbDiOzq2bK+Hrvft50fBdHMD3znJLCENcTKFPmxDyawRXYVWC5D7uKR6n1rNtqfWjCg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153566; c=relaxed/simple; bh=o1T6CNtpqoEZjNDMs3JIIhmKy57hp1Orqv6AVBQKOf4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=fVmtxsVsOZGogclOpH7OjGnHGMryFm8bmCGTmTHACe9LQsyzQ46LZe23vTfwH4EQQeL1kp1vsoNWtc2o89LaoWPt2xPi/nNBNDv9F+VwCM7K/8HzyG1MaytfL9QzGMzmIWYNjAoGAmGgkOU2MGYt8pagvpiduUeFjPD7a1AYprw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=jnPYg0Jq; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="jnPYg0Jq" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 121D71F00A3A; Wed, 15 Jul 2026 22:12:44 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784153564; bh=dc25d1R8r153FAI3T2MEXdGhmLTeOSPAa0Ovax6Q57w=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=jnPYg0JqdWtWgvNPxUvkWprVI8yJh4wbGD2cB8hXJrXCtRutiLhukcfUaPRqQeGQf Y2uCBwI0Wv/zZvNXZ7KDsvBYMgbsregxEDrOvMvBj592JaWzu0VQPJdrl8ylLV6Chj X5Y4kHdaxgfkBJZAvzCR2Zu7cIJfITkWbTup4jWqfDiM+aED71oqIA7dpfVOpJey4a Zyl6H+UxQOTiubzvNqjXrHAXJlaMR4R79NOpPJGtphs6qcL7gbd8iakjR/3O5ueMdZ LrstwOFjZIDUu4ZvQLG22PZyLR58NiyNaLnWRE+nte0MvTZUAnarsWobsjE9DcZyaQ nmIZTVmFEpHFQ== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , Thomas Huth , Eric Biggers Subject: [PATCH v2 08/13] crypto: aes - Add ECB support using library Date: Wed, 15 Jul 2026 15:11:48 -0700 Message-ID: <20260715221153.246410-9-ebiggers@kernel.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260715221153.246410-1-ebiggers@kernel.org> References: <20260715221153.246410-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Implement the "ecb(aes)" crypto_skcipher algorithm using the corresponding library functions. Among other benefits, this allows the architecture-optimized AES-ECB code to be migrated into the library while still leaving it accessible via crypto_skcipher, eliminating lots of boilerplate code. For now the cra_priority is set to just 110, since the architecture-optimized implementations of this algorithm haven't yet been migrated into the library. It will be boosted once that happens. Signed-off-by: Eric Biggers --- crypto/Kconfig | 5 ++ crypto/aes.c | 164 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 168 insertions(+), 1 deletion(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index b61401bd3ef6..1888ae9d3fa3 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -359,7 +359,12 @@ config CRYPTO_AES select CRYPTO_ALGAPI select CRYPTO_LIB_AES select CRYPTO_LIB_AES_CBC_MACS if CRYPTO_CMAC !=3D n || CRYPTO_XCBC !=3D = n || CRYPTO_CCM !=3D n + select CRYPTO_LIB_AES_ECB if CRYPTO_ECB !=3D n select CRYPTO_HASH if CRYPTO_CMAC !=3D n || CRYPTO_XCBC !=3D n || CRYPTO_= CCM !=3D n + # CRYPTO_SKCIPHER should be selected only if a mode that needs it is + # enabled, but that doesn't work due to a recursive dependency caused by + # CRYPTO_SKCIPHER selecting CRYPTO_ECB. So just always select it. + select CRYPTO_SKCIPHER help AES cipher algorithms (Rijndael)(FIPS-197, ISO/IEC 18033-3) =20 diff --git a/crypto/aes.c b/crypto/aes.c index 6bf23eb0503f..5fc487e584c4 100644 --- a/crypto/aes.c +++ b/crypto/aes.c @@ -6,12 +6,16 @@ */ =20 #include +#include #include #include #include +#include +#include #include =20 static_assert(__alignof__(struct aes_key) <=3D CRYPTO_MINALIGN); +static_assert(__alignof__(struct aes_enckey) <=3D CRYPTO_MINALIGN); =20 static int crypto_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len) @@ -85,7 +89,6 @@ static int __maybe_unused crypto_aes_cmac_digest(struct s= hash_desc *desc, return 0; } =20 -static_assert(__alignof__(struct aes_enckey) <=3D CRYPTO_MINALIGN); #define AES_CBCMAC_KEY(tfm) ((struct aes_enckey *)crypto_shash_ctx(tfm)) #define AES_CBCMAC_CTX(desc) ((struct aes_cbcmac_ctx *)shash_desc_ctx(desc= )) =20 @@ -200,6 +203,148 @@ static struct shash_alg mac_algs[] =3D { #endif }; =20 +static __maybe_unused int +crypto_aes_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct aes_key *key =3D crypto_skcipher_ctx(tfm); + + return aes_preparekey(key, in_key, key_len); +} + +static __maybe_unused int +crypto_aes_skcipher_setenckey(struct crypto_skcipher *tfm, const u8 *in_ke= y, + unsigned int key_len) +{ + struct aes_enckey *key =3D crypto_skcipher_ctx(tfm); + + return aes_prepareenckey(key, in_key, key_len); +} + +/* + * Call crypt_func() (a function that operates on simple virtual addresses= ) zero + * or more times to en/decrypt 'cryptlen' bytes of data from the source + * scatterlist 'src' and write it into the destination scatterlist 'dst', + * starting at 'start_pos' bytes into both. + * + * This always calls crypt_func() with a length that's a multiple of + * AES_BLOCK_SIZE, except the last call which includes any remainder. Thi= s is + * implemented by using an on-stack bounce buffer when necessary. The cur= rent + * implementation also tries to prefer passing at least 4 blocks, so e.g. + * scatterlist entries [16,16,16,16] result in a single 64-byte call. + * + * The scatterlists must describe either entirely different memory + * (out-of-place) or entirely the same memory (in-place). In the latter c= ase, + * crypt_func() is always called with the source and dest pointers the sam= e. + */ +#define AES_CRYPT_SG(crypt_func, dst, src, cryptlen, start_pos, ...) = \ + ({ \ + unsigned int remaining =3D (cryptlen); \ + unsigned int spos =3D (start_pos); \ + = \ + if (remaining !=3D 0) { \ + struct scatter_walk dst_walk, src_walk; \ + u8 tmp[4 * AES_BLOCK_SIZE] __aligned( \ + __alignof__(long)); \ + = \ + scatterwalk_start_at_pos(&dst_walk, (dst), spos); \ + scatterwalk_start_at_pos(&src_walk, (src), spos); \ + do { \ + unsigned int dst_avail =3D scatterwalk_clamp( \ + &dst_walk, remaining); \ + unsigned int src_avail =3D scatterwalk_clamp( \ + &src_walk, remaining); \ + unsigned int n =3D min(dst_avail, src_avail); \ + u8 *dst_virt; \ + const u8 *src_virt; \ + = \ + if (n < remaining) { \ + if (n < sizeof(tmp)) { \ + n =3D min(remaining, \ + sizeof(tmp)); \ + memcpy_from_scatterwalk( \ + tmp, &src_walk, n); \ + crypt_func(tmp, tmp, n, \ + ##__VA_ARGS__); \ + memcpy_to_scatterwalk( \ + &dst_walk, tmp, n); \ + remaining -=3D n; \ + continue; \ + } \ + n =3D round_down(n, AES_BLOCK_SIZE); \ + } \ + = \ + scatterwalk_map(&dst_walk); \ + dst_virt =3D dst_walk.addr; \ + if (IS_ENABLED(CONFIG_HIGHMEM) && \ + offset_in_page(src_walk.offset) =3D=3D \ + offset_in_page(dst_walk.offset) && \ + sg_page(src_walk.sg) + (src_walk.offset / \ + PAGE_SIZE) =3D=3D \ + sg_page(dst_walk.sg) + \ + (dst_walk.offset / \ + PAGE_SIZE)) { \ + src_virt =3D dst_virt; \ + } else { \ + scatterwalk_map(&src_walk); \ + src_virt =3D src_walk.addr; \ + } \ + crypt_func(dst_virt, src_virt, n, \ + ##__VA_ARGS__); \ + if (src_virt !=3D dst_virt) \ + scatterwalk_unmap(&src_walk); \ + scatterwalk_advance(&src_walk, n); \ + scatterwalk_done_dst(&dst_walk, n); \ + remaining -=3D n; \ + } while (remaining); \ + memzero_explicit(tmp, sizeof(tmp)); \ + } \ + }) + +/* AES-ECB */ + +static __maybe_unused int crypto_aes_ecb_encrypt(struct skcipher_request *= req) +{ + const struct aes_key *key =3D + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen % AES_BLOCK_SIZE)) + return -EINVAL; + AES_CRYPT_SG(aes_ecb_encrypt, req->dst, req->src, req->cryptlen, 0, + key); + return 0; +} + +static __maybe_unused int crypto_aes_ecb_decrypt(struct skcipher_request *= req) +{ + const struct aes_key *key =3D + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen % AES_BLOCK_SIZE)) + return -EINVAL; + AES_CRYPT_SG(aes_ecb_decrypt, req->dst, req->src, req->cryptlen, 0, + key); + return 0; +} + +static struct skcipher_alg skcipher_algs[] =3D { +#if IS_ENABLED(CONFIG_CRYPTO_ECB) + { + .base.cra_name =3D "ecb(aes)", + .base.cra_driver_name =3D "ecb-aes-lib", + .base.cra_priority =3D 110, + .base.cra_blocksize =3D AES_BLOCK_SIZE, + .base.cra_ctxsize =3D sizeof(struct aes_key), + .base.cra_module =3D THIS_MODULE, + .min_keysize =3D AES_MIN_KEY_SIZE, + .max_keysize =3D AES_MAX_KEY_SIZE, + .setkey =3D crypto_aes_skcipher_setkey, + .encrypt =3D crypto_aes_ecb_encrypt, + .decrypt =3D crypto_aes_ecb_decrypt, + }, +#endif +}; + static int __init crypto_aes_mod_init(void) { int err =3D crypto_register_alg(&alg); @@ -212,8 +357,18 @@ static int __init crypto_aes_mod_init(void) if (err) goto err_unregister_alg; } /* Else, CONFIG_CRYPTO_HASH might not be enabled. */ + + if (ARRAY_SIZE(skcipher_algs) > 0) { + err =3D crypto_register_skciphers(skcipher_algs, + ARRAY_SIZE(skcipher_algs)); + if (err) + goto err_unregister_macs; + } return 0; =20 +err_unregister_macs: + if (ARRAY_SIZE(mac_algs) > 0) + crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); err_unregister_alg: crypto_unregister_alg(&alg); return err; @@ -222,6 +377,9 @@ module_init(crypto_aes_mod_init); =20 static void __exit crypto_aes_mod_exit(void) { + if (ARRAY_SIZE(skcipher_algs) > 0) + crypto_unregister_skciphers(skcipher_algs, + ARRAY_SIZE(skcipher_algs)); if (ARRAY_SIZE(mac_algs) > 0) crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); crypto_unregister_alg(&alg); @@ -245,3 +403,7 @@ MODULE_ALIAS_CRYPTO("xcbc-aes-lib"); MODULE_ALIAS_CRYPTO("cbcmac(aes)"); MODULE_ALIAS_CRYPTO("cbcmac-aes-lib"); #endif +#if IS_ENABLED(CONFIG_CRYPTO_ECB) +MODULE_ALIAS_CRYPTO("ecb(aes)"); +MODULE_ALIAS_CRYPTO("ecb-aes-lib"); +#endif --=20 2.55.0 From nobody Sat Jul 25 16:48:31 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C41F93D47D9; Wed, 15 Jul 2026 22:12:44 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153566; cv=none; b=DEKPr/N5S2lPK/3cQFyYSUm96Yn1g4TZUb57DXti4J1d54qNI1A4KjaBK8DBuGLTOqo2/7k6TrpdVklDTZvS/LFLgHyp83Wo71W7K9uwatA8/tPZADhgbHWD2h0KStVmZifnVzeZXgU11dXhvBSYMa4bU1kJkFZuJk4mFkcCNJ4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153566; c=relaxed/simple; bh=emc4ZzynoiIxASrNcQvknVIfR6XSRRDto7re5ntlg6Q=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=L+WQvtrpL6m0Ih0KQPf9LTyItArpYwMZPwq1tiHeuvL/aSSFrtL2ryRgrffrIXFVuTpv8pp4+tKtk4UEFhNyyyk6SttctsctFJoLSpkUVWBvhVzj0XQ+nAAq+OfIFBCdQYqA7OAjaI4HPp1t38fAnUiv2QK2G4sLXbKTGxZTw30= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=bGEAqaBR; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="bGEAqaBR" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 770CC1F00ACA; Wed, 15 Jul 2026 22:12:44 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784153564; bh=aJSWNLerwUGZGtDyOQn2ByaQCAig9/sHV/RcIsrhaXI=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=bGEAqaBR4n1zDC9UIQJtoJEeN8ECKY3CUbdBgO5ugOE0Sy1Yvpc1e1JRY8LGcQBK6 fmzfY4sgRSggrLTmjFHm+K4dMqHa9iMeESX+mg2RNZ66CZhgkLI1tVaA6X4GS2XUWZ RtkFNdbUQbDrdk4s6MsnhFIErbrnlMxOVv2DvadHlBgT/JWv+Q8Y7MmRZH/RRPsmxx mnTXBeWbnEdsH9Pp2bqvDOtymERBBu3Dp5MeNWVdcdicar064FWIM5HqOUEShPVtFK p7fkFF74CdNv7yfAfcZHGJoeACzuY7sr8S/X6PwtGRpzHU9+6ycBskiPuv2iJHXF1T NLhC4G3yqnobg== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , Thomas Huth , Eric Biggers Subject: [PATCH v2 09/13] crypto: aes - Add CBC and CBC-CTS support using library Date: Wed, 15 Jul 2026 15:11:49 -0700 Message-ID: <20260715221153.246410-10-ebiggers@kernel.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260715221153.246410-1-ebiggers@kernel.org> References: <20260715221153.246410-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Implement the "cbc(aes)" and "cts(cbc(aes))" crypto_skcipher algorithms using the corresponding library functions. Among other benefits, this allows the architecture-optimized AES-CBC and AES-CBC-CTS code to be migrated into the library while still leaving it accessible via crypto_skcipher, eliminating lots of boilerplate code. For now the cra_priority is set to just 110, since the architecture-optimized implementations of these algorithms haven't yet been migrated into the library. It will be boosted once that happens. Signed-off-by: Eric Biggers --- crypto/Kconfig | 1 + crypto/aes.c | 169 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+) diff --git a/crypto/Kconfig b/crypto/Kconfig index 1888ae9d3fa3..f413cfc9a3e2 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -358,6 +358,7 @@ config CRYPTO_AES tristate "AES (Advanced Encryption Standard)" select CRYPTO_ALGAPI select CRYPTO_LIB_AES + select CRYPTO_LIB_AES_CBC if CRYPTO_CBC !=3D n || CRYPTO_CTS !=3D n select CRYPTO_LIB_AES_CBC_MACS if CRYPTO_CMAC !=3D n || CRYPTO_XCBC !=3D = n || CRYPTO_CCM !=3D n select CRYPTO_LIB_AES_ECB if CRYPTO_ECB !=3D n select CRYPTO_HASH if CRYPTO_CMAC !=3D n || CRYPTO_XCBC !=3D n || CRYPTO_= CCM !=3D n diff --git a/crypto/aes.c b/crypto/aes.c index 5fc487e584c4..2455abc29252 100644 --- a/crypto/aes.c +++ b/crypto/aes.c @@ -6,6 +6,7 @@ */ =20 #include +#include #include #include #include @@ -221,6 +222,19 @@ crypto_aes_skcipher_setenckey(struct crypto_skcipher *= tfm, const u8 *in_key, return aes_prepareenckey(key, in_key, key_len); } =20 +/* + * Return true if the request uses only a single scatterlist element and h= igh + * memory isn't enabled. This assumes that both scatterlists are non-NULL= , i.e. + * the caller must have handled the cryptlen =3D=3D 0 case already. + */ +static inline bool +skcipher_request_is_linear_lowmem(const struct skcipher_request *req) +{ + return !IS_ENABLED(CONFIG_HIGHMEM) && + req->dst->length >=3D req->cryptlen && + req->src->length >=3D req->cryptlen; +} + /* * Call crypt_func() (a function that operates on simple virtual addresses= ) zero * or more times to en/decrypt 'cryptlen' bytes of data from the source @@ -327,6 +341,121 @@ static __maybe_unused int crypto_aes_ecb_decrypt(stru= ct skcipher_request *req) return 0; } =20 +/* AES-CBC */ + +static void crypto_aes_cbc_encrypt_sg(struct skcipher_request *req, + unsigned int cryptlen, + const struct aes_key *key) +{ + AES_CRYPT_SG(aes_cbc_encrypt, req->dst, req->src, cryptlen, 0, req->iv, + key); +} + +static void crypto_aes_cbc_decrypt_sg(struct skcipher_request *req, + unsigned int cryptlen, + const struct aes_key *key) +{ + AES_CRYPT_SG(aes_cbc_decrypt, req->dst, req->src, cryptlen, 0, req->iv, + key); +} + +static __maybe_unused int crypto_aes_cbc_encrypt(struct skcipher_request *= req) +{ + const struct aes_key *key =3D + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen % AES_BLOCK_SIZE)) + return -EINVAL; + crypto_aes_cbc_encrypt_sg(req, req->cryptlen, key); + return 0; +} + +static __maybe_unused int crypto_aes_cbc_decrypt(struct skcipher_request *= req) +{ + const struct aes_key *key =3D + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen % AES_BLOCK_SIZE)) + return -EINVAL; + crypto_aes_cbc_decrypt_sg(req, req->cryptlen, key); + return 0; +} + +/* AES-CBC-CTS */ + +/* + * This handles AES-CBC-CTS en/decryption requests that use a nonlinear + * scatterlist layout or where HIGHMEM is enabled. It is explicitly 'noin= line' + * to keep the temporary buffer out of the stack frame of the fast path. + */ +static noinline int +crypto_aes_cbc_cts_crypt_nonlinear(struct skcipher_request *req, bool enc) +{ + const struct aes_key *key =3D + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + unsigned int main_len =3D req->cryptlen; + unsigned int tail_len; + u8 tmp[2 * AES_BLOCK_SIZE] __aligned(__alignof__(long)); + + if (main_len =3D=3D AES_BLOCK_SIZE) { + /* Single block is a special case that just does CBC. */ + if (enc) + crypto_aes_cbc_encrypt_sg(req, main_len, key); + else + crypto_aes_cbc_decrypt_sg(req, main_len, key); + return 0; + } + /* Just do the last two blocks separately. */ + tail_len =3D AES_BLOCK_SIZE + ((main_len - 1) % AES_BLOCK_SIZE) + 1; + main_len -=3D tail_len; + if (enc) + crypto_aes_cbc_encrypt_sg(req, main_len, key); + else + crypto_aes_cbc_decrypt_sg(req, main_len, key); + memcpy_from_sglist(tmp, req->src, main_len, tail_len); + if (enc) + aes_cbc_cts_encrypt(tmp, tmp, tail_len, req->iv, key); + else + aes_cbc_cts_decrypt(tmp, tmp, tail_len, req->iv, key); + memcpy_to_sglist(req->dst, main_len, tmp, tail_len); + memzero_explicit(tmp, sizeof(tmp)); + return 0; +} + +static __maybe_unused int +crypto_aes_cbc_cts_encrypt(struct skcipher_request *req) +{ + const struct aes_key *key =3D + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen < AES_BLOCK_SIZE)) + return -EINVAL; + if (likely(skcipher_request_is_linear_lowmem(req))) { + /* Fast path */ + aes_cbc_cts_encrypt(sg_virt(req->dst), sg_virt(req->src), + req->cryptlen, req->iv, key); + return 0; + } + return crypto_aes_cbc_cts_crypt_nonlinear(req, /* enc=3D */ true); +} + +static __maybe_unused int +crypto_aes_cbc_cts_decrypt(struct skcipher_request *req) +{ + const struct aes_key *key =3D + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen < AES_BLOCK_SIZE)) + return -EINVAL; + if (likely(skcipher_request_is_linear_lowmem(req))) { + /* Fast path */ + aes_cbc_cts_decrypt(sg_virt(req->dst), sg_virt(req->src), + req->cryptlen, req->iv, key); + return 0; + } + return crypto_aes_cbc_cts_crypt_nonlinear(req, /* enc=3D */ false); +} + static struct skcipher_alg skcipher_algs[] =3D { #if IS_ENABLED(CONFIG_CRYPTO_ECB) { @@ -343,6 +472,38 @@ static struct skcipher_alg skcipher_algs[] =3D { .decrypt =3D crypto_aes_ecb_decrypt, }, #endif +#if IS_ENABLED(CONFIG_CRYPTO_CBC) + { + .base.cra_name =3D "cbc(aes)", + .base.cra_driver_name =3D "cbc-aes-lib", + .base.cra_priority =3D 110, + .base.cra_blocksize =3D AES_BLOCK_SIZE, + .base.cra_ctxsize =3D sizeof(struct aes_key), + .base.cra_module =3D THIS_MODULE, + .min_keysize =3D AES_MIN_KEY_SIZE, + .max_keysize =3D AES_MAX_KEY_SIZE, + .ivsize =3D AES_BLOCK_SIZE, + .setkey =3D crypto_aes_skcipher_setkey, + .encrypt =3D crypto_aes_cbc_encrypt, + .decrypt =3D crypto_aes_cbc_decrypt, + }, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_CTS) + { + .base.cra_name =3D "cts(cbc(aes))", + .base.cra_driver_name =3D "cts-cbc-aes-lib", + .base.cra_priority =3D 110, + .base.cra_blocksize =3D AES_BLOCK_SIZE, + .base.cra_ctxsize =3D sizeof(struct aes_key), + .base.cra_module =3D THIS_MODULE, + .min_keysize =3D AES_MIN_KEY_SIZE, + .max_keysize =3D AES_MAX_KEY_SIZE, + .ivsize =3D AES_BLOCK_SIZE, + .setkey =3D crypto_aes_skcipher_setkey, + .encrypt =3D crypto_aes_cbc_cts_encrypt, + .decrypt =3D crypto_aes_cbc_cts_decrypt, + }, +#endif }; =20 static int __init crypto_aes_mod_init(void) @@ -407,3 +568,11 @@ MODULE_ALIAS_CRYPTO("cbcmac-aes-lib"); MODULE_ALIAS_CRYPTO("ecb(aes)"); MODULE_ALIAS_CRYPTO("ecb-aes-lib"); #endif +#if IS_ENABLED(CONFIG_CRYPTO_CBC) +MODULE_ALIAS_CRYPTO("cbc(aes)"); +MODULE_ALIAS_CRYPTO("cbc-aes-lib"); +#endif +#if IS_ENABLED(CONFIG_CRYPTO_CTS) +MODULE_ALIAS_CRYPTO("cts(cbc(aes))"); +MODULE_ALIAS_CRYPTO("cts-cbc-aes-lib"); +#endif --=20 2.55.0 From nobody Sat Jul 25 16:48:31 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 22EDE3D5252; Wed, 15 Jul 2026 22:12:45 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153568; cv=none; b=u/4TttXZ/AHvbgMIglsEBTqZQV8WgMaZ1cAuTHYA9eTMKF44OEKkIfrWSoaswscwdJDJ3Xca6qj6CTQtMHIouvS1xSXSk9XoxmqoyVIeslwqhV0ao61yfV3wjGaT3DHUcSwuLqapazZc7TRdN/k1enC1v07pzH1t8DkGqPIEDFk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153568; c=relaxed/simple; bh=/Prcikhmu8XrhKbJiWPHIr0zTI9gWz+xF6iukhXfuYY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=F+gneWrXMzwP/I+S26r0FENi04mI3V3DoxoEKsRTL5Q6tVBjbbhPd5fVCv8rXMrYD6mEfoCu7S5W7WWLekpgKdUYZT88YwvHblIjol/IkJytLx8pCjk7QlhLHqv/kES8qxxY/3yj03zr5nlQ1ULgSdpmcgT6VAxLIhXfhpMkxiM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=K5CG0aQq; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="K5CG0aQq" Received: by smtp.kernel.org (Postfix) with ESMTPSA id CC19A1F00A3D; Wed, 15 Jul 2026 22:12:44 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784153565; bh=mfbbgvlQ+9Z+7biX+EDzESayj6dZLFr8DxKzrfLdc1c=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=K5CG0aQqH5AwZLR+rXc1a5Ud3YY2lhdxYoQj9JxSCS+8xj/AAc/S7+wBe7UmUUsrD DqqfyzGhPWixRvdYVaZYcndje8t7uYmzMuEet5mbmBQSSDSOhKwaAMX6/ccCmY3Nim lwxd7yDchsuSeWyuLZHBLOFPhfucLni7kEtlhRuy8k5NzkYAq24+Cq0vqewBFlLcqj Gv/TU33RfAToNleAUEagFuS6/dabQHlii3wS9qLWnplfG6aL48w4om7MoLuF8QMxYG sbFpAUZZe73oAjXuGdlEaayw3Sz16kJGSBptDe6gVG2PaEFl2KZdgWphH36r3nJwWs 5Ne39dx7ZEOsg== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , Thomas Huth , Eric Biggers Subject: [PATCH v2 10/13] crypto: aes - Add CTR and XCTR support using library Date: Wed, 15 Jul 2026 15:11:50 -0700 Message-ID: <20260715221153.246410-11-ebiggers@kernel.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260715221153.246410-1-ebiggers@kernel.org> References: <20260715221153.246410-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Implement the "ctr(aes)" and "xctr(aes)" crypto_skcipher algorithms using the corresponding library functions. Among other benefits, this allows the architecture-optimized AES-CTR and AES-XCTR code to be migrated into the library while still leaving it accessible via crypto_skcipher, eliminating lots of boilerplate code. For now the cra_priority is set to just 110, since the architecture-optimized implementations of these algorithms haven't yet been migrated into the library. It will be boosted once that happens. Signed-off-by: Eric Biggers --- crypto/Kconfig | 1 + crypto/aes.c | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/crypto/Kconfig b/crypto/Kconfig index f413cfc9a3e2..5a198275e4fc 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -360,6 +360,7 @@ config CRYPTO_AES select CRYPTO_LIB_AES select CRYPTO_LIB_AES_CBC if CRYPTO_CBC !=3D n || CRYPTO_CTS !=3D n select CRYPTO_LIB_AES_CBC_MACS if CRYPTO_CMAC !=3D n || CRYPTO_XCBC !=3D = n || CRYPTO_CCM !=3D n + select CRYPTO_LIB_AES_CTR if CRYPTO_CTR !=3D n || CRYPTO_XCTR !=3D n select CRYPTO_LIB_AES_ECB if CRYPTO_ECB !=3D n select CRYPTO_HASH if CRYPTO_CMAC !=3D n || CRYPTO_XCBC !=3D n || CRYPTO_= CCM !=3D n # CRYPTO_SKCIPHER should be selected only if a mode that needs it is diff --git a/crypto/aes.c b/crypto/aes.c index 2455abc29252..6b298e788630 100644 --- a/crypto/aes.c +++ b/crypto/aes.c @@ -7,6 +7,7 @@ =20 #include #include +#include #include #include #include @@ -456,6 +457,31 @@ crypto_aes_cbc_cts_decrypt(struct skcipher_request *re= q) return crypto_aes_cbc_cts_crypt_nonlinear(req, /* enc=3D */ false); } =20 +/* AES-CTR */ + +static __maybe_unused int crypto_aes_ctr_crypt(struct skcipher_request *re= q) +{ + const struct aes_enckey *key =3D + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + AES_CRYPT_SG(aes_ctr, req->dst, req->src, req->cryptlen, 0, req->iv, + key); + return 0; +} + +/* AES-XCTR */ + +static __maybe_unused int crypto_aes_xctr_crypt(struct skcipher_request *r= eq) +{ + const struct aes_enckey *key =3D + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + u64 ctr =3D 1; + + AES_CRYPT_SG(aes_xctr, req->dst, req->src, req->cryptlen, 0, &ctr, + req->iv, key); + return 0; +} + static struct skcipher_alg skcipher_algs[] =3D { #if IS_ENABLED(CONFIG_CRYPTO_ECB) { @@ -504,6 +530,40 @@ static struct skcipher_alg skcipher_algs[] =3D { .decrypt =3D crypto_aes_cbc_cts_decrypt, }, #endif +#if IS_ENABLED(CONFIG_CRYPTO_CTR) + { + .base.cra_name =3D "ctr(aes)", + .base.cra_driver_name =3D "ctr-aes-lib", + .base.cra_priority =3D 110, + .base.cra_blocksize =3D 1, + .base.cra_ctxsize =3D sizeof(struct aes_enckey), + .base.cra_module =3D THIS_MODULE, + .min_keysize =3D AES_MIN_KEY_SIZE, + .max_keysize =3D AES_MAX_KEY_SIZE, + .ivsize =3D AES_BLOCK_SIZE, + .chunksize =3D AES_BLOCK_SIZE, + .setkey =3D crypto_aes_skcipher_setenckey, + .encrypt =3D crypto_aes_ctr_crypt, + .decrypt =3D crypto_aes_ctr_crypt, + }, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_XCTR) + { + .base.cra_name =3D "xctr(aes)", + .base.cra_driver_name =3D "xctr-aes-lib", + .base.cra_priority =3D 110, + .base.cra_blocksize =3D 1, + .base.cra_ctxsize =3D sizeof(struct aes_enckey), + .base.cra_module =3D THIS_MODULE, + .min_keysize =3D AES_MIN_KEY_SIZE, + .max_keysize =3D AES_MAX_KEY_SIZE, + .ivsize =3D AES_BLOCK_SIZE, + .chunksize =3D AES_BLOCK_SIZE, + .setkey =3D crypto_aes_skcipher_setenckey, + .encrypt =3D crypto_aes_xctr_crypt, + .decrypt =3D crypto_aes_xctr_crypt, + }, +#endif }; =20 static int __init crypto_aes_mod_init(void) @@ -576,3 +636,11 @@ MODULE_ALIAS_CRYPTO("cbc-aes-lib"); MODULE_ALIAS_CRYPTO("cts(cbc(aes))"); MODULE_ALIAS_CRYPTO("cts-cbc-aes-lib"); #endif +#if IS_ENABLED(CONFIG_CRYPTO_CTR) +MODULE_ALIAS_CRYPTO("ctr(aes)"); +MODULE_ALIAS_CRYPTO("ctr-aes-lib"); +#endif +#if IS_ENABLED(CONFIG_CRYPTO_XCTR) +MODULE_ALIAS_CRYPTO("xctr(aes)"); +MODULE_ALIAS_CRYPTO("xctr-aes-lib"); +#endif --=20 2.55.0 From nobody Sat Jul 25 16:48:31 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 936823D565C; Wed, 15 Jul 2026 22:12:45 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153567; cv=none; b=O/olFGVOir31+BzRPV+IbgjYu7i/gr4W1rsiooUeML8XQqko0aFGSgDxyfZIgHJLzCA4X0NKDt6NQdstRrbJKzaN8yFl+9SBHMgaBW3RF1uR4+cnSATm9USqVSL3CEKazX4KxM5R0xT2uswaT/XZOkYYotPObi1fsVztgW/0vj0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153567; c=relaxed/simple; bh=vcGKxXmkS6XVthF37CKhxW3nqxFgjm/e5+nXfHGT4EU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=pYKhtir2PKb0hOA3jDEzpweqOBlgR6vtAxmVchBDbbasf6uigTO1lisxxd/fHIdEMDtw7D40YhGp94Z1pbtWhhM0k0zHvpo1jSgHJ6SM1viNBRXQVexRuHfU4810ub/qKX3P/IzjPjImNBxF2C+jJlMlP+WlY50EYTv15KJzatY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=imBYCprs; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="imBYCprs" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 2D1CC1F00A3E; Wed, 15 Jul 2026 22:12:45 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784153565; bh=XeOTaAPmNI8eOCg7n0Yrq+hUMtt5GyeW653IENyPOdQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=imBYCprsRhyZms9JtAU+tcapzOBJ7GFKqHxV4yyHi+l608hREChmOp5ZcJbkgITuO 0NyaiOOXIrNPloSYY4KdYGtmvzLzKriCKnhhhYjAH9YmUKFzY4+gSelf2VfBoAWvJD sY9VA47fxTEW8ZpRG4IbWsWYKarXmuODVJrkvCfj2A2UlNk3XYAMi5miTCLFEYHaRO zsbUUhCSOsPLsP6ANaEUj8HKr0/dU2gKRW4Msmw2CTMUd0EFnstlL7sPkYKvej4U8S HXS+u3DYAryO/Ht+97V/G7lXFbuw8GZVT9RvwqITcFwdx7tuwdt7RQBlexO3W+Q65K XMtbnJ7cXtMgA== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , Thomas Huth , Eric Biggers Subject: [PATCH v2 11/13] crypto: aes - Add XTS support using library Date: Wed, 15 Jul 2026 15:11:51 -0700 Message-ID: <20260715221153.246410-12-ebiggers@kernel.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260715221153.246410-1-ebiggers@kernel.org> References: <20260715221153.246410-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Implement the "xts(aes)" crypto_skcipher algorithm using the corresponding library functions. Among other benefits, this allows the architecture-optimized AES-XTS code to be migrated into the library while still leaving it accessible via crypto_skcipher, eliminating lots of boilerplate code. Fast paths similar to what x86_64 uses (to eliminate the scatterlist walking overhead) are included. So we'll get that optimization for all architectures. For now the cra_priority is set to just 110, since the architecture-optimized implementations of this algorithm haven't yet been migrated into the library. It will be boosted once that happens. Signed-off-by: Eric Biggers --- crypto/Kconfig | 1 + crypto/aes.c | 117 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) diff --git a/crypto/Kconfig b/crypto/Kconfig index 5a198275e4fc..567b719c8a44 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -362,6 +362,7 @@ config CRYPTO_AES select CRYPTO_LIB_AES_CBC_MACS if CRYPTO_CMAC !=3D n || CRYPTO_XCBC !=3D = n || CRYPTO_CCM !=3D n select CRYPTO_LIB_AES_CTR if CRYPTO_CTR !=3D n || CRYPTO_XCTR !=3D n select CRYPTO_LIB_AES_ECB if CRYPTO_ECB !=3D n + select CRYPTO_LIB_AES_XTS if CRYPTO_XTS !=3D n select CRYPTO_HASH if CRYPTO_CMAC !=3D n || CRYPTO_XCBC !=3D n || CRYPTO_= CCM !=3D n # CRYPTO_SKCIPHER should be selected only if a mode that needs it is # enabled, but that doesn't work due to a recursive dependency caused by diff --git a/crypto/aes.c b/crypto/aes.c index 6b298e788630..eb22840a68f0 100644 --- a/crypto/aes.c +++ b/crypto/aes.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -482,6 +483,102 @@ static __maybe_unused int crypto_aes_xctr_crypt(struc= t skcipher_request *req) return 0; } =20 +/* AES-XTS */ + +static __maybe_unused int crypto_aes_xts_setkey(struct crypto_skcipher *tf= m, + const u8 *in_key, + unsigned int key_len) +{ + struct aes_xts_key *key =3D crypto_skcipher_ctx(tfm); + int flags =3D (crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) ? + XTS_FORBID_WEAK_KEYS : + 0; + + return aes_xts_preparekey(key, in_key, key_len, flags); +} + +static void aes_xts_crypt_wrapper(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool enc, + bool *cont) +{ + if (enc) + aes_xts_encrypt(dst, src, len, iv, key, *cont); + else + aes_xts_decrypt(dst, src, len, iv, key, *cont); + *cont =3D true; +} + +/* + * This handles AES-XTS en/decryption requests that use a nonlinear scatte= rlist + * layout or where HIGHMEM is enabled. It is explicitly 'noinline' to kee= p the + * temporary buffer out of the stack frame of the fast path. + */ +static noinline int crypto_aes_xts_crypt_nonlinear(struct skcipher_request= *req, + bool enc) +{ + const struct aes_xts_key *key =3D + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + u8 tmp[2 * AES_BLOCK_SIZE] __aligned(__alignof__(long)); + unsigned int main_len =3D req->cryptlen; + unsigned int tail_len =3D main_len % AES_BLOCK_SIZE; + bool cont =3D false; + + if (unlikely(tail_len)) { + /* + * Ciphertext stealing is needed. + * Just do the last two blocks separately. + */ + tail_len +=3D AES_BLOCK_SIZE; + main_len -=3D tail_len; + } + + AES_CRYPT_SG(aes_xts_crypt_wrapper, req->dst, req->src, main_len, 0, + req->iv, key, enc, &cont); + + if (unlikely(tail_len)) { + memcpy_from_sglist(tmp, req->src, main_len, tail_len); + aes_xts_crypt_wrapper(tmp, tmp, tail_len, req->iv, key, enc, + &cont); + memcpy_to_sglist(req->dst, main_len, tmp, tail_len); + memzero_explicit(tmp, sizeof(tmp)); + } + return 0; +} + +static __maybe_unused int crypto_aes_xts_encrypt(struct skcipher_request *= req) +{ + const struct aes_xts_key *key =3D + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen < AES_BLOCK_SIZE)) + return -EINVAL; + if (likely(skcipher_request_is_linear_lowmem(req))) { + /* Fast path */ + aes_xts_encrypt(sg_virt(req->dst), sg_virt(req->src), + req->cryptlen, req->iv, key, /* cont=3D */ false); + return 0; + } + return crypto_aes_xts_crypt_nonlinear(req, /* enc=3D */ true); +} + +static __maybe_unused int crypto_aes_xts_decrypt(struct skcipher_request *= req) +{ + const struct aes_xts_key *key =3D + crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + + if (unlikely(req->cryptlen < AES_BLOCK_SIZE)) + return -EINVAL; + if (likely(skcipher_request_is_linear_lowmem(req))) { + /* Fast path */ + aes_xts_decrypt(sg_virt(req->dst), sg_virt(req->src), + req->cryptlen, req->iv, key, /* cont=3D */ false); + return 0; + } + return crypto_aes_xts_crypt_nonlinear(req, /* enc=3D */ false); +} + static struct skcipher_alg skcipher_algs[] =3D { #if IS_ENABLED(CONFIG_CRYPTO_ECB) { @@ -564,6 +661,22 @@ static struct skcipher_alg skcipher_algs[] =3D { .decrypt =3D crypto_aes_xctr_crypt, }, #endif +#if IS_ENABLED(CONFIG_CRYPTO_XTS) + { + .base.cra_name =3D "xts(aes)", + .base.cra_driver_name =3D "xts-aes-lib", + .base.cra_priority =3D 110, + .base.cra_blocksize =3D AES_BLOCK_SIZE, + .base.cra_ctxsize =3D sizeof(struct aes_xts_key), + .base.cra_module =3D THIS_MODULE, + .min_keysize =3D 2 * AES_MIN_KEY_SIZE, + .max_keysize =3D 2 * AES_MAX_KEY_SIZE, + .ivsize =3D AES_BLOCK_SIZE, + .setkey =3D crypto_aes_xts_setkey, + .encrypt =3D crypto_aes_xts_encrypt, + .decrypt =3D crypto_aes_xts_decrypt, + }, +#endif }; =20 static int __init crypto_aes_mod_init(void) @@ -644,3 +757,7 @@ MODULE_ALIAS_CRYPTO("ctr-aes-lib"); MODULE_ALIAS_CRYPTO("xctr(aes)"); MODULE_ALIAS_CRYPTO("xctr-aes-lib"); #endif +#if IS_ENABLED(CONFIG_CRYPTO_XTS) +MODULE_ALIAS_CRYPTO("xts(aes)"); +MODULE_ALIAS_CRYPTO("xts-aes-lib"); +#endif --=20 2.55.0 From nobody Sat Jul 25 16:48:31 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id CA4543D5663; Wed, 15 Jul 2026 22:12:45 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153567; cv=none; b=ouIvtHliRt4wXhxj7cNS08wZUw5G25wAYXoG0lLs/pWgpDhjIIrvLuWMsykxgG5psSH/NbXCUpOaOA4rxTiZqxYp9Mo0GRW6MrtlUg6Ub1OpJ5f5YOssAJ9kvBxo5b0cP08k9N2y6u/L7Yio/FUyZBtHwkbdfqSevpt+g+5gu8U= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153567; c=relaxed/simple; bh=X6vENT9dx1rFIo2kQYTywDestj0u143qX8SOr4Y4NoM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=HRGy206inas4dIkdeWaZQbBOY8BC3X+eR3//TcwaJRvXlJqPaVh3+kzgLwgMVa3UPugSN1QawXS9IYjewtynUHyhQtAj6thUyn+s3J5t0rFODK2ByDj/F4i+5HEtVdKnJz/Iw79U9JL/vqbjdeL09VjrU9R9pCxY1iGUwuIcCjw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=kjun2y32; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="kjun2y32" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 802141F00ADE; Wed, 15 Jul 2026 22:12:45 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784153565; bh=+8XuiVdWQPQpsTfF27WgiLYO/8C64lfIEZbcz5HvIiQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=kjun2y32mswm/LgFeCsPkHyc4bbNK7bAKe/0ehjSzufNYBTxIQUS6sCXljLZ9eYaL a/gyqUoYMnEkekTfF3klKyzYVOURqulNapgKGdwLvXkkd/3hocoOH7RXKW1ES4rkNg DkPJExlEhLGCtpA4quoDGdmJAPRgqpNrpOwhhJonQQf7x2GsATYLqDSNfrWtp1gXxG FTmby8Ye6rakCs/yKBLk+f1o2IlF1jCvlnG1vHR9H9p6CL5Knh0uRzLzBgh8Uul2KT c48o69s8ISy+HlEMQ9PdplzI8ZpIs/CTFLG2uGsVLKIlNWb9NRWzBnzgNTwprgBhxN twgzsn7d05miA== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , Thomas Huth , Eric Biggers Subject: [PATCH v2 12/13] crypto: aes - Add GCM support using library Date: Wed, 15 Jul 2026 15:11:52 -0700 Message-ID: <20260715221153.246410-13-ebiggers@kernel.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260715221153.246410-1-ebiggers@kernel.org> References: <20260715221153.246410-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Implement the "gcm(aes)" and "rfc4106(gcm(aes))" crypto_aead algorithms using the corresponding library functions. Among other benefits, this allows the architecture-optimized AES-GCM code to be migrated into the library while still leaving it accessible via crypto_aead, eliminating lots of boilerplate code. For now the cra_priority is set to just 110, since the architecture-optimized implementations of these algorithms haven't yet been migrated into the library. It will be boosted once that happens. Signed-off-by: Eric Biggers --- crypto/Kconfig | 2 + crypto/aes.c | 245 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 247 insertions(+) diff --git a/crypto/Kconfig b/crypto/Kconfig index 567b719c8a44..bbd314c07fd7 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -362,7 +362,9 @@ config CRYPTO_AES select CRYPTO_LIB_AES_CBC_MACS if CRYPTO_CMAC !=3D n || CRYPTO_XCBC !=3D = n || CRYPTO_CCM !=3D n select CRYPTO_LIB_AES_CTR if CRYPTO_CTR !=3D n || CRYPTO_XCTR !=3D n select CRYPTO_LIB_AES_ECB if CRYPTO_ECB !=3D n + select CRYPTO_LIB_AES_GCM if CRYPTO_GCM !=3D n select CRYPTO_LIB_AES_XTS if CRYPTO_XTS !=3D n + select CRYPTO_AEAD if CRYPTO_GCM !=3D n select CRYPTO_HASH if CRYPTO_CMAC !=3D n || CRYPTO_XCBC !=3D n || CRYPTO_= CCM !=3D n # CRYPTO_SKCIPHER should be selected only if a mode that needs it is # enabled, but that doesn't work due to a recursive dependency caused by diff --git a/crypto/aes.c b/crypto/aes.c index eb22840a68f0..9fe106c9eed2 100644 --- a/crypto/aes.c +++ b/crypto/aes.c @@ -9,9 +9,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -317,6 +319,29 @@ skcipher_request_is_linear_lowmem(const struct skciphe= r_request *req) } \ }) =20 +/* + * Call ad_func() as needed to process the associated data in the first + * 'assoclen' bytes of the scatterlist 'src'. + */ +#define AES_PROCESS_ASSOC_DATA(ad_func, src, assoclen, ctx) = \ + ({ \ + unsigned int remaining =3D (assoclen); \ + = \ + if (remaining !=3D 0) { \ + struct scatter_walk walk; \ + = \ + scatterwalk_start(&walk, (src)); \ + do { \ + unsigned int n =3D \ + scatterwalk_next(&walk, remaining); \ + = \ + ad_func((ctx), walk.addr, n); \ + scatterwalk_done_src(&walk, n); \ + remaining -=3D n; \ + } while (remaining); \ + } \ + }) + /* AES-ECB */ =20 static __maybe_unused int crypto_aes_ecb_encrypt(struct skcipher_request *= req) @@ -679,6 +704,208 @@ static struct skcipher_alg skcipher_algs[] =3D { #endif }; =20 +/* AES-GCM */ + +static __maybe_unused int crypto_aes_gcm_setkey(struct crypto_aead *tfm, + const u8 *in_key, + unsigned int key_len) +{ + struct aes_gcm_key *key =3D crypto_aead_ctx(tfm); + + return aes_gcm_preparekey(key, in_key, key_len, + crypto_aead_authsize(tfm)); +} + +static __maybe_unused int crypto_aes_gcm_setauthsize(struct crypto_aead *t= fm, + unsigned int authsize) +{ + struct aes_gcm_key *key =3D crypto_aead_ctx(tfm); + + if (crypto_gcm_check_authsize(authsize) !=3D 0) + return -EINVAL; + /* Synchronize the tag length to the struct aes_gcm_key. */ + key->authtag_len =3D authsize; + return 0; +} + +static void crypto_aes_gcm_auth_update(struct aes_gcm_ctx *ctx, + struct scatterlist *src, + unsigned int assoclen) +{ + AES_PROCESS_ASSOC_DATA(aes_gcm_auth_update, src, assoclen, ctx); +} + +static void aes_gcm_encrypt_update_helper(u8 *dst, const u8 *src, + unsigned int len, + struct aes_gcm_ctx *ctx) +{ + aes_gcm_encrypt_update(ctx, dst, src, len); +} + +static void aes_gcm_decrypt_update_helper(u8 *dst, const u8 *src, + unsigned int len, + struct aes_gcm_ctx *ctx) +{ + aes_gcm_decrypt_update(ctx, dst, src, len); +} + +static int crypto_aes_gcm_encrypt_common(struct aead_request *req, + const struct aes_gcm_key *key, + u8 iv[12], unsigned int assoclen) +{ + struct aes_gcm_ctx ctx; + u8 authtag[16]; + + aes_gcm_init(&ctx, iv, key); + crypto_aes_gcm_auth_update(&ctx, req->src, assoclen); + AES_CRYPT_SG(aes_gcm_encrypt_update_helper, req->dst, req->src, + req->cryptlen, req->assoclen, &ctx); + aes_gcm_encrypt_final(&ctx, authtag); + memcpy_to_sglist(req->dst, req->assoclen + req->cryptlen, authtag, + key->authtag_len); + memzero_explicit(authtag, sizeof(authtag)); + return 0; +} + +static int crypto_aes_gcm_decrypt_common(struct aead_request *req, + const struct aes_gcm_key *key, + u8 iv[12], unsigned int assoclen) +{ + struct aes_gcm_ctx ctx; + unsigned int data_len; + u8 authtag[16]; + int err; + + aes_gcm_init(&ctx, iv, key); + crypto_aes_gcm_auth_update(&ctx, req->src, assoclen); + + /* crypto_aead_decrypt() already checked cryptlen >=3D authtag_len. */ + data_len =3D req->cryptlen - key->authtag_len; + AES_CRYPT_SG(aes_gcm_decrypt_update_helper, req->dst, req->src, + data_len, req->assoclen, &ctx); + + memcpy_from_sglist(authtag, req->src, req->assoclen + data_len, + key->authtag_len); + err =3D aes_gcm_decrypt_final(&ctx, authtag); + memzero_explicit(authtag, sizeof(authtag)); + return err; +} + +static __maybe_unused int crypto_aes_gcm_encrypt(struct aead_request *req) +{ + struct crypto_aead *tfm =3D crypto_aead_reqtfm(req); + const struct aes_gcm_key *key =3D crypto_aead_ctx(tfm); + + return crypto_aes_gcm_encrypt_common(req, key, req->iv, req->assoclen); +} + +static __maybe_unused int crypto_aes_gcm_decrypt(struct aead_request *req) +{ + struct crypto_aead *tfm =3D crypto_aead_reqtfm(req); + const struct aes_gcm_key *key =3D crypto_aead_ctx(tfm); + + return crypto_aes_gcm_decrypt_common(req, key, req->iv, req->assoclen); +} + +struct aes_rfc4106_key { + struct aes_gcm_key gcm; + u8 nonce[4]; +}; + +static __maybe_unused int crypto_aes_rfc4106_setkey(struct crypto_aead *tf= m, + const u8 *in_key, + unsigned int key_len) +{ + struct aes_rfc4106_key *key =3D crypto_aead_ctx(tfm); + + if (key_len < 4) + return -EINVAL; + + key_len -=3D 4; + memcpy(key->nonce, in_key + key_len, 4); + + return aes_gcm_preparekey(&key->gcm, in_key, key_len, + crypto_aead_authsize(tfm)); +} + +static __maybe_unused int +crypto_aes_rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int auths= ize) +{ + struct aes_rfc4106_key *key =3D crypto_aead_ctx(tfm); + + if (crypto_rfc4106_check_authsize(authsize) !=3D 0) + return -EINVAL; + + /* Synchronize the tag length to the struct aes_gcm_key. */ + key->gcm.authtag_len =3D authsize; + return 0; +} + +static __maybe_unused int crypto_aes_rfc4106_encrypt(struct aead_request *= req) +{ + struct crypto_aead *tfm =3D crypto_aead_reqtfm(req); + const struct aes_rfc4106_key *key =3D crypto_aead_ctx(tfm); + u8 iv[12]; + + if (crypto_ipsec_check_assoclen(req->assoclen) !=3D 0) + return -EINVAL; + memcpy(iv, key->nonce, 4); + memcpy(&iv[4], req->iv, 8); + + return crypto_aes_gcm_encrypt_common(req, &key->gcm, iv, + req->assoclen - 8); +} + +static __maybe_unused int crypto_aes_rfc4106_decrypt(struct aead_request *= req) +{ + struct crypto_aead *tfm =3D crypto_aead_reqtfm(req); + const struct aes_rfc4106_key *key =3D crypto_aead_ctx(tfm); + u8 iv[12]; + + if (crypto_ipsec_check_assoclen(req->assoclen) !=3D 0) + return -EINVAL; + memcpy(iv, key->nonce, 4); + memcpy(&iv[4], req->iv, 8); + + return crypto_aes_gcm_decrypt_common(req, &key->gcm, iv, + req->assoclen - 8); +} + +static struct aead_alg aead_algs[] =3D { +#if IS_ENABLED(CONFIG_CRYPTO_GCM) + { + .base.cra_name =3D "gcm(aes)", + .base.cra_driver_name =3D "gcm-aes-lib", + .base.cra_priority =3D 110, + .base.cra_blocksize =3D 1, + .base.cra_ctxsize =3D sizeof(struct aes_gcm_key), + .base.cra_module =3D THIS_MODULE, + .setkey =3D crypto_aes_gcm_setkey, + .setauthsize =3D crypto_aes_gcm_setauthsize, + .encrypt =3D crypto_aes_gcm_encrypt, + .decrypt =3D crypto_aes_gcm_decrypt, + .ivsize =3D GCM_AES_IV_SIZE, + .maxauthsize =3D AES_BLOCK_SIZE, + .chunksize =3D AES_BLOCK_SIZE, + }, + { + .base.cra_name =3D "rfc4106(gcm(aes))", + .base.cra_driver_name =3D "rfc4106-gcm-aes-lib", + .base.cra_priority =3D 110, + .base.cra_blocksize =3D 1, + .base.cra_ctxsize =3D sizeof(struct aes_rfc4106_key), + .base.cra_module =3D THIS_MODULE, + .setkey =3D crypto_aes_rfc4106_setkey, + .setauthsize =3D crypto_aes_rfc4106_setauthsize, + .encrypt =3D crypto_aes_rfc4106_encrypt, + .decrypt =3D crypto_aes_rfc4106_decrypt, + .ivsize =3D GCM_RFC4106_IV_SIZE, + .maxauthsize =3D AES_BLOCK_SIZE, + .chunksize =3D AES_BLOCK_SIZE, + }, +#endif /* CONFIG_CRYPTO_GCM */ +}; + static int __init crypto_aes_mod_init(void) { int err =3D crypto_register_alg(&alg); @@ -698,8 +925,18 @@ static int __init crypto_aes_mod_init(void) if (err) goto err_unregister_macs; } + + if (ARRAY_SIZE(aead_algs) > 0) { + err =3D crypto_register_aeads(aead_algs, ARRAY_SIZE(aead_algs)); + if (err) + goto err_unregister_skciphers; + } /* Else, CONFIG_CRYPTO_AEAD might not be enabled. */ return 0; =20 +err_unregister_skciphers: + if (ARRAY_SIZE(skcipher_algs) > 0) + crypto_unregister_skciphers(skcipher_algs, + ARRAY_SIZE(skcipher_algs)); err_unregister_macs: if (ARRAY_SIZE(mac_algs) > 0) crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); @@ -711,6 +948,8 @@ module_init(crypto_aes_mod_init); =20 static void __exit crypto_aes_mod_exit(void) { + if (ARRAY_SIZE(aead_algs) > 0) + crypto_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs)); if (ARRAY_SIZE(skcipher_algs) > 0) crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); @@ -761,3 +1000,9 @@ MODULE_ALIAS_CRYPTO("xctr-aes-lib"); MODULE_ALIAS_CRYPTO("xts(aes)"); MODULE_ALIAS_CRYPTO("xts-aes-lib"); #endif +#if IS_ENABLED(CONFIG_CRYPTO_GCM) +MODULE_ALIAS_CRYPTO("gcm(aes)"); +MODULE_ALIAS_CRYPTO("gcm-aes-lib"); +MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); +MODULE_ALIAS_CRYPTO("rfc4106-gcm-aes-lib"); +#endif --=20 2.55.0 From nobody Sat Jul 25 16:48:31 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 6373D3D3CFD; Wed, 15 Jul 2026 22:12:46 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153568; cv=none; b=L2zOVu1W0Mb8IcS+0U7gXL1NH8hW5mXpQTgselqNu8LSoLnhho3xZZADV5GNt0alIQCOVXBvxciWYH72mIc2EsuJAd6heT335Uktk5Oo30ohATyHGcdrrfSAAlJqfy0xC6z2c5X0TxDo/zHUWPX9hkZmgrAEuDHB+sgwm9HH3hg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784153568; c=relaxed/simple; bh=LXRYEjEzCMaqVvQ4aQujapjQiDkKpoeTjbjaY8NFUBE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=sUYH2qqi1G9cPKOwVx/oIubZZeU3v16Wkt13ore9UFB3nri2WLjw6b2LpUZU05Nyk+ueJhCrfM+C5sx4YPS5GuxUxMQl5IPUhCmzEIAwzMQHNcpyQYCrfFJexLDMWWM+Q+YA08mtkrr1KzU09k9O/03z7RHS1brq8Y5/DQ4dAnY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=emgVijv+; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="emgVijv+" Received: by smtp.kernel.org (Postfix) with ESMTPSA id D53C31F00ADB; Wed, 15 Jul 2026 22:12:45 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1784153566; bh=lDP4bJTGEiLkb+s3fAkSj4o0g5a8olZiVFLBdfwFwQY=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=emgVijv+lG4GjOk8aghzyjzivsazFcML4TZsAymItywM0kkbK21o27QD7YUrJzMqh axruznZo+f56+c7j5n8YvYdwWbQxPqK/5Z0hyO/h4eYqquHc8u9/6E8kqn9nZBD+Cg GZVO98tSz4ciEVa3auGum60788A5ucdbqlfdU2UARdamkRA9K2N/EUid0LxgOcOBKn c1HPGyXP03QgjYGUhM6N8IdxxRwKJ/aFWnfHm6vWOOBolWRmAaLRiEQv6n0xYuppx0 nvbcjFkL04VGOgVMsb9q/S7oDseMIEsf0PlI6bAuXubrGXcvG2jdbMWw3NNuHqNo7a Asa1N4xqI6MqQ== From: Eric Biggers To: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel , "Jason A . Donenfeld" , Herbert Xu , Thomas Huth , Eric Biggers Subject: [PATCH v2 13/13] crypto: aes - Add CCM support using library Date: Wed, 15 Jul 2026 15:11:53 -0700 Message-ID: <20260715221153.246410-14-ebiggers@kernel.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260715221153.246410-1-ebiggers@kernel.org> References: <20260715221153.246410-1-ebiggers@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Implement the "ccm(aes)" crypto_aead algorithm using the corresponding library functions. Among other benefits, this allows the architecture-optimized AES-CCM code to be migrated into the library while still leaving it accessible via crypto_aead, eliminating lots of boilerplate code. For now the cra_priority is set to just 110, since the architecture-optimized implementations of this algorithm haven't yet been migrated into the library. It will be boosted once that happens. Signed-off-by: Eric Biggers --- crypto/Kconfig | 3 +- crypto/aes.c | 129 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 1 deletion(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index bbd314c07fd7..981d5dc422d4 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -360,11 +360,12 @@ config CRYPTO_AES select CRYPTO_LIB_AES select CRYPTO_LIB_AES_CBC if CRYPTO_CBC !=3D n || CRYPTO_CTS !=3D n select CRYPTO_LIB_AES_CBC_MACS if CRYPTO_CMAC !=3D n || CRYPTO_XCBC !=3D = n || CRYPTO_CCM !=3D n + select CRYPTO_LIB_AES_CCM if CRYPTO_CCM !=3D n select CRYPTO_LIB_AES_CTR if CRYPTO_CTR !=3D n || CRYPTO_XCTR !=3D n select CRYPTO_LIB_AES_ECB if CRYPTO_ECB !=3D n select CRYPTO_LIB_AES_GCM if CRYPTO_GCM !=3D n select CRYPTO_LIB_AES_XTS if CRYPTO_XTS !=3D n - select CRYPTO_AEAD if CRYPTO_GCM !=3D n + select CRYPTO_AEAD if CRYPTO_GCM !=3D n || CRYPTO_CCM !=3D n select CRYPTO_HASH if CRYPTO_CMAC !=3D n || CRYPTO_XCBC !=3D n || CRYPTO_= CCM !=3D n # CRYPTO_SKCIPHER should be selected only if a mode that needs it is # enabled, but that doesn't work due to a recursive dependency caused by diff --git a/crypto/aes.c b/crypto/aes.c index 9fe106c9eed2..94791f481e98 100644 --- a/crypto/aes.c +++ b/crypto/aes.c @@ -7,6 +7,7 @@ =20 #include #include +#include #include #include #include @@ -871,6 +872,113 @@ static __maybe_unused int crypto_aes_rfc4106_decrypt(= struct aead_request *req) req->assoclen - 8); } =20 +/* AES-CCM */ + +static __maybe_unused int crypto_aes_ccm_setkey(struct crypto_aead *tfm, + const u8 *in_key, + unsigned int key_len) +{ + struct aes_ccm_key *key =3D crypto_aead_ctx(tfm); + + return aes_ccm_preparekey(key, in_key, key_len, + crypto_aead_authsize(tfm)); +} + +static __maybe_unused int crypto_aes_ccm_setauthsize(struct crypto_aead *t= fm, + unsigned int authsize) +{ + struct aes_ccm_key *key =3D crypto_aead_ctx(tfm); + + if (authsize < 4 || authsize > 16 || authsize % 2) + return -EINVAL; + /* Synchronize the tag length to the struct aes_ccm_key. */ + key->authtag_len =3D authsize; + return 0; +} + +static int crypto_aes_ccm_init(struct aes_ccm_ctx *ctx, + struct aead_request *req, unsigned int data_len, + const struct aes_ccm_key *key) +{ + int nonce_len; + const u8 *nonce; + int err; + + /* + * CCM accepts a variable-length nonce between 7 and 13 bytes + * inclusively, while crypto_aead assumes a fixed-length IV. This is + * worked around by requiring that iv[0] contain '14 - nonce_len' and + * iv[1..] contain the actual nonce. Extra bytes at the end are unused. + */ + nonce_len =3D 14 - (int)req->iv[0]; + if (unlikely(nonce_len < 7 || nonce_len > 13)) + return -EINVAL; + nonce =3D &req->iv[1]; + err =3D aes_ccm_init(ctx, data_len, req->assoclen, nonce, nonce_len, key); + if (unlikely(err)) + return err; + AES_PROCESS_ASSOC_DATA(aes_ccm_auth_update, req->src, req->assoclen, + ctx); + return 0; +} + +static void aes_ccm_encrypt_update_helper(u8 *dst, const u8 *src, + unsigned int len, + struct aes_ccm_ctx *ctx) +{ + aes_ccm_encrypt_update(ctx, dst, src, len); +} + +static void aes_ccm_decrypt_update_helper(u8 *dst, const u8 *src, + unsigned int len, + struct aes_ccm_ctx *ctx) +{ + aes_ccm_decrypt_update(ctx, dst, src, len); +} + +static __maybe_unused int crypto_aes_ccm_encrypt(struct aead_request *req) +{ + struct crypto_aead *tfm =3D crypto_aead_reqtfm(req); + const struct aes_ccm_key *key =3D crypto_aead_ctx(tfm); + struct aes_ccm_ctx ctx; + u8 authtag[16]; + int err; + + err =3D crypto_aes_ccm_init(&ctx, req, req->cryptlen, key); + if (unlikely(err)) + return err; + AES_CRYPT_SG(aes_ccm_encrypt_update_helper, req->dst, req->src, + req->cryptlen, req->assoclen, &ctx); + aes_ccm_encrypt_final(&ctx, authtag); + memcpy_to_sglist(req->dst, req->assoclen + req->cryptlen, authtag, + key->authtag_len); + memzero_explicit(authtag, sizeof(authtag)); + return 0; +} + +static __maybe_unused int crypto_aes_ccm_decrypt(struct aead_request *req) +{ + struct crypto_aead *tfm =3D crypto_aead_reqtfm(req); + const struct aes_ccm_key *key =3D crypto_aead_ctx(tfm); + unsigned int data_len; + struct aes_ccm_ctx ctx; + u8 authtag[16]; + int err; + + /* crypto_aead_decrypt() already checked cryptlen >=3D authtag_len. */ + data_len =3D req->cryptlen - key->authtag_len; + err =3D crypto_aes_ccm_init(&ctx, req, data_len, key); + if (unlikely(err)) + return err; + AES_CRYPT_SG(aes_ccm_decrypt_update_helper, req->dst, req->src, + data_len, req->assoclen, &ctx); + memcpy_from_sglist(authtag, req->src, req->assoclen + data_len, + key->authtag_len); + err =3D aes_ccm_decrypt_final(&ctx, authtag); + memzero_explicit(authtag, sizeof(authtag)); + return err; +} + static struct aead_alg aead_algs[] =3D { #if IS_ENABLED(CONFIG_CRYPTO_GCM) { @@ -904,6 +1012,23 @@ static struct aead_alg aead_algs[] =3D { .chunksize =3D AES_BLOCK_SIZE, }, #endif /* CONFIG_CRYPTO_GCM */ +#if IS_ENABLED(CONFIG_CRYPTO_CCM) + { + .base.cra_name =3D "ccm(aes)", + .base.cra_driver_name =3D "ccm-aes-lib", + .base.cra_priority =3D 110, + .base.cra_blocksize =3D 1, + .base.cra_ctxsize =3D sizeof(struct aes_ccm_key), + .base.cra_module =3D THIS_MODULE, + .setkey =3D crypto_aes_ccm_setkey, + .setauthsize =3D crypto_aes_ccm_setauthsize, + .encrypt =3D crypto_aes_ccm_encrypt, + .decrypt =3D crypto_aes_ccm_decrypt, + .ivsize =3D 16, + .maxauthsize =3D 16, + .chunksize =3D AES_BLOCK_SIZE, + }, +#endif /* CONFIG_CRYPTO_CCM */ }; =20 static int __init crypto_aes_mod_init(void) @@ -1006,3 +1131,7 @@ MODULE_ALIAS_CRYPTO("gcm-aes-lib"); MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); MODULE_ALIAS_CRYPTO("rfc4106-gcm-aes-lib"); #endif +#if IS_ENABLED(CONFIG_CRYPTO_CCM) +MODULE_ALIAS_CRYPTO("ccm(aes)"); +MODULE_ALIAS_CRYPTO("ccm-aes-lib"); +#endif --=20 2.55.0