From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 702812D5C6B; Fri, 15 Aug 2025 08:55:37 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248141; cv=none; b=rfg7Fi0xqusP6Jw0nfaU5iCHaSvrUtCSpCvEOu5mCFe9ffsQ56uXgBjnTMp2MdAKdraETuQmPtlS/J/x7oxpt13ZYshCjrV9cJXRPGqGv1vGkJR+EoXU4TKERVylxJPCE4yYY7N8NIjC92bu2umozJpB21sfQb+qECheB1bM5RM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248141; c=relaxed/simple; bh=zgWSS17PfBHNbGnl+c/SycLggSOAwyGWBJpBi2JRpr8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=PB6+ltM7OfSzpLADshgblCc2h2GvMF/OHUkSK70iwfLABlTtuouYz0NcvGa6NOy/KGM4gv6MEy1I/Wq3CaNWj9HEBoe5KVDYY6fXeJ2IUzAGLmvVmUeHDnLwX43kPxYC9PEEVkbVnBc6qESf1dSkkdw/VCc+yY4m79GvhfOGtbg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 1CB461E32; Fri, 15 Aug 2025 01:55:23 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id B58A63F63F; Fri, 15 Aug 2025 01:55:26 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 01/18] mm: Introduce kpkeys Date: Fri, 15 Aug 2025 09:54:55 +0100 Message-ID: <20250815085512.2182322-2-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" kpkeys is a simple framework to enable the use of protection keys (pkeys) to harden the kernel itself. This patch introduces the basic API in : a couple of functions to set and restore the pkey register and macros to define guard objects. kpkeys introduces a new concept on top of pkeys: the kpkeys level. Each level is associated to a set of permissions for the pkeys managed by the kpkeys framework. kpkeys_set_level(lvl) sets those permissions according to lvl, and returns the original pkey register, to be later restored by kpkeys_restore_pkey_reg(). To start with, only KPKEYS_LVL_DEFAULT is available, which is meant to grant RW access to KPKEYS_PKEY_DEFAULT (i.e. all memory since this is the only available pkey for now). Because each architecture implementing pkeys uses a different representation for the pkey register, and may reserve certain pkeys for specific uses, support for kpkeys must be explicitly indicated by selecting ARCH_HAS_KPKEYS and defining the following functions in , in addition to the macros provided in : - arch_kpkeys_set_level() - arch_kpkeys_restore_pkey_reg() - arch_kpkeys_enabled() Signed-off-by: Kevin Brodsky --- include/asm-generic/kpkeys.h | 17 ++++++ include/linux/kpkeys.h | 113 +++++++++++++++++++++++++++++++++++ mm/Kconfig | 2 + 3 files changed, 132 insertions(+) create mode 100644 include/asm-generic/kpkeys.h create mode 100644 include/linux/kpkeys.h diff --git a/include/asm-generic/kpkeys.h b/include/asm-generic/kpkeys.h new file mode 100644 index 000000000000..ab819f157d6a --- /dev/null +++ b/include/asm-generic/kpkeys.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_GENERIC_KPKEYS_H +#define __ASM_GENERIC_KPKEYS_H + +#ifndef KPKEYS_PKEY_DEFAULT +#define KPKEYS_PKEY_DEFAULT 0 +#endif + +/* + * Represents a pkey register value that cannot be used, typically disabli= ng + * access to all keys. + */ +#ifndef KPKEYS_PKEY_REG_INVAL +#define KPKEYS_PKEY_REG_INVAL 0 +#endif + +#endif /* __ASM_GENERIC_KPKEYS_H */ diff --git a/include/linux/kpkeys.h b/include/linux/kpkeys.h new file mode 100644 index 000000000000..faa6e2615798 --- /dev/null +++ b/include/linux/kpkeys.h @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_KPKEYS_H +#define _LINUX_KPKEYS_H + +#include +#include + +#define KPKEYS_LVL_DEFAULT 0 + +#define KPKEYS_LVL_MIN KPKEYS_LVL_DEFAULT +#define KPKEYS_LVL_MAX KPKEYS_LVL_DEFAULT + +#define __KPKEYS_GUARD(name, set_level, restore_pkey_reg, set_arg, ...) \ + __DEFINE_CLASS_IS_CONDITIONAL(name, false); \ + DEFINE_CLASS(name, u64, \ + restore_pkey_reg, set_level, set_arg); \ + static inline void *class_##name##_lock_ptr(u64 *_T) \ + { return _T; } + +/** + * KPKEYS_GUARD_NOOP() - define a guard type that does nothing + * @name: the name of the guard type + * @cond_arg: an argument specification (optional) + * + * Define a guard type that does nothing, useful to match a real guard type + * that is defined under an #ifdef. @cond_arg may optionally be passed to = match + * a guard defined using KPKEYS_GUARD_COND(). + */ +#define KPKEYS_GUARD_NOOP(name, ...) \ + __KPKEYS_GUARD(name, 0, (void)_T, ##__VA_ARGS__, void) + +#ifdef CONFIG_ARCH_HAS_KPKEYS + +#include + +/** + * KPKEYS_GUARD_COND() - define a guard type that conditionally switches to + * a given kpkeys level + * @name: the name of the guard type + * @level: the kpkeys level to switch to + * @cond: an expression that is evaluated as condition + * @cond_arg: an argument specification for the condition (optional) + * + * Define a guard type that switches to @level if @cond evaluates to true,= and + * does nothing otherwise. @cond_arg may be specified to give access to a + * caller-defined argument to @cond. + */ +#define KPKEYS_GUARD_COND(name, level, cond, ...) \ + __KPKEYS_GUARD(name, \ + cond ? kpkeys_set_level(level) \ + : KPKEYS_PKEY_REG_INVAL, \ + kpkeys_restore_pkey_reg(_T), \ + ##__VA_ARGS__, void) + +/** + * KPKEYS_GUARD() - define a guard type that switches to a given kpkeys le= vel + * if kpkeys are enabled + * @name: the name of the guard type + * @level: the kpkeys level to switch to + * + * Define a guard type that switches to @level if the system supports kpke= ys. + */ +#define KPKEYS_GUARD(name, level) \ + KPKEYS_GUARD_COND(name, level, arch_kpkeys_enabled()) + +/** + * kpkeys_set_level() - switch kpkeys level + * @level: the level to switch to + * + * Switches the kpkeys level to the specified value. @level must be a + * compile-time constant. The arch-specific pkey register will be updated + * accordingly, and the original value returned. + * + * Return: the original pkey register value if the register was written to= , or + * KPKEYS_PKEY_REG_INVAL otherwise (no write to the register was + * required). + */ +static __always_inline u64 kpkeys_set_level(int level) +{ + BUILD_BUG_ON_MSG(!__builtin_constant_p(level), + "kpkeys_set_level() only takes constant levels"); + BUILD_BUG_ON_MSG(level < KPKEYS_LVL_MIN || level > KPKEYS_LVL_MAX, + "Invalid level passed to kpkeys_set_level()"); + + return arch_kpkeys_set_level(level); +} + +/** + * kpkeys_restore_pkey_reg() - restores a pkey register value + * @pkey_reg: the pkey register value to restore + * + * This function is meant to be passed the value returned by kpkeys_set_le= vel(), + * in order to restore the pkey register to its original value (thus resto= ring + * the original kpkeys level). + */ +static __always_inline void kpkeys_restore_pkey_reg(u64 pkey_reg) +{ + if (pkey_reg !=3D KPKEYS_PKEY_REG_INVAL) + arch_kpkeys_restore_pkey_reg(pkey_reg); +} + +#else /* CONFIG_ARCH_HAS_KPKEYS */ + +#include + +static inline bool arch_kpkeys_enabled(void) +{ + return false; +} + +#endif /* CONFIG_ARCH_HAS_KPKEYS */ + +#endif /* _LINUX_KPKEYS_H */ diff --git a/mm/Kconfig b/mm/Kconfig index e443fe8cd6cf..90f2e5c381a6 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1173,6 +1173,8 @@ config ARCH_USES_HIGH_VMA_FLAGS bool config ARCH_HAS_PKEYS bool +config ARCH_HAS_KPKEYS + bool =20 config ARCH_USES_PG_ARCH_2 bool --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 13DF92D3A71; Fri, 15 Aug 2025 08:55:36 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248138; cv=none; b=ektqoGxBsYBESfKR1leyV7r9Sgez0jlFKhzaX8PmAO0quB9cdmXNSbuSUvxyyKvc6Ju9v5YgaHxTzQKf4OplsUyS9gE0hAAq3tIz11kyd7c6A/iWMs1hXvfi8bTvsax0Alw53z+yC5NKDWPnD5z/5ERbzy5HqNhbVqnTv3bF5V0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248138; c=relaxed/simple; bh=4aDjkHibEaVadfQGPlSSmLSpJv3EcwnykRxixv3rl/8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=HA7ZKfVwWSIZa1vc0LYwpNUEfY6a2pvB6JfjjJOx5sz9wo8qwqSLkzV11u7Hnk3RmY9fwxL7xJAVcZcH0pSc3Mda+TcYi14Sq8R80sgHz14ne/3nNYROVBrzwVigDSWX3kFS8jCwWUisteI9iJsNsi+VrFwgqod5xFqoMjI78KM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 00C5E1688; Fri, 15 Aug 2025 01:55:28 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 9326C3F63F; Fri, 15 Aug 2025 01:55:31 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 02/18] set_memory: Introduce set_memory_pkey() stub Date: Fri, 15 Aug 2025 09:54:56 +0100 Message-ID: <20250815085512.2182322-3-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Introduce a new function, set_memory_pkey(), which sets the protection key (pkey) of pages in the specified linear mapping range. Architectures implementing kernel pkeys (kpkeys) must provide a suitable implementation; an empty stub is added as fallback. Signed-off-by: Kevin Brodsky --- include/linux/set_memory.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/linux/set_memory.h b/include/linux/set_memory.h index 3030d9245f5a..7b3a8bfde3c6 100644 --- a/include/linux/set_memory.h +++ b/include/linux/set_memory.h @@ -84,4 +84,11 @@ static inline int set_memory_decrypted(unsigned long add= r, int numpages) } #endif /* CONFIG_ARCH_HAS_MEM_ENCRYPT */ =20 +#ifndef CONFIG_ARCH_HAS_KPKEYS +static inline int set_memory_pkey(unsigned long addr, int numpages, int pk= ey) +{ + return 0; +} +#endif + #endif /* _LINUX_SET_MEMORY_H_ */ --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 964F12D8363; Fri, 15 Aug 2025 08:55:41 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248144; cv=none; b=gtYDQHSuQXuh3i7igNLbvZzkpbPtDP/cJfV8mbXVm/NP81VniQrFPZ78+OEpPhkAt3LxjbMFv5V8ERXz6GTlfG0cbqlJK+/q/0kepzEMUcEVT5uR6HPrb25CyN9QXrmp090LF3dCbiY8kozXw6pfMAChCBdWIhUsVqmBF0S2p5U= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248144; c=relaxed/simple; bh=LZEP7J4G+85IvqSzGsfTqFlhUb51ATFhLdKeN8moA+s=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=CcWb0euvRB3uibKkYqMaG8Ceya98sLguE8oWuOUHElhuRDM4DAAnWTCHczeFhzccbzmJIdjRTeJbaZL1T/zhBgeFlJkd/N9rLn42NQCmO+N2SvhNxY3Gddw1q/ukSrewoB0Km9mV/WjMduAEKwTvmQ6p8jgamHWr53E1S37jRLo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id C86171E5E; Fri, 15 Aug 2025 01:55:32 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 703713F63F; Fri, 15 Aug 2025 01:55:36 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 03/18] arm64: mm: Enable overlays for all EL1 indirect permissions Date: Fri, 15 Aug 2025 09:54:57 +0100 Message-ID: <20250815085512.2182322-4-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In preparation of using POE inside the kernel, enable "Overlay applied" for all stage 1 base permissions in PIR_EL1. This ensures that the permissions set in POR_EL1 affect all kernel mappings. Signed-off-by: Kevin Brodsky --- arch/arm64/include/asm/pgtable-prot.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm= /pgtable-prot.h index 85dceb1c66f4..0ab4ac214b06 100644 --- a/arch/arm64/include/asm/pgtable-prot.h +++ b/arch/arm64/include/asm/pgtable-prot.h @@ -180,13 +180,13 @@ static inline bool __pure lpa2_is_enabled(void) PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_GCS), PIE_NONE_O) | \ PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_GCS_RO), PIE_NONE_O) | \ PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_EXECONLY), PIE_NONE_O) | \ - PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_READONLY_EXEC), PIE_R) | \ - PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_SHARED_EXEC), PIE_RW) | \ - PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_READONLY), PIE_R) | \ - PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_SHARED), PIE_RW) | \ - PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_KERNEL_ROX), PIE_RX) | \ - PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_KERNEL_EXEC), PIE_RWX) | \ - PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_KERNEL_RO), PIE_R) | \ - PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_KERNEL), PIE_RW)) + PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_READONLY_EXEC), PIE_R_O) | \ + PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_SHARED_EXEC), PIE_RW_O) | \ + PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_READONLY), PIE_R_O) | \ + PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_SHARED), PIE_RW_O) | \ + PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_KERNEL_ROX), PIE_RX_O) | \ + PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_KERNEL_EXEC), PIE_RWX_O) | \ + PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_KERNEL_RO), PIE_R_O) | \ + PIRx_ELx_PERM_PREP(pte_pi_index(_PAGE_KERNEL), PIE_RW_O)) =20 #endif /* __ASM_PGTABLE_PROT_H */ --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id B07702E1C5B; Fri, 15 Aug 2025 08:55:46 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248149; cv=none; b=pBy9IZUOJ1ZxXinhGWIba7ClUigclLDqt1DRTlKxzlvKSHZ/H+65EOh5Bv3stH9k4gU5R64QGEhkWGxSjXblF1AZhOu6NNqAjX8SLC+gZzYwzF9fQIe06coX3lT3T6nNDfSDe4v1cqsVU+6nlxLY5UZVSZK+j7iJpcMiwVDtt1Q= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248149; c=relaxed/simple; bh=7iruIE0Meb58he1IeFy/w/2y/XvZVdjS3WuTdETTx7Y=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=aDcO4hgF+2H2IZXjVjeO0sNjYYKNgS/i5fSNyvxr+LGNwYtxQoiXBsxXJJ9zG8MNLoGtqB93OBZPMBTF+3IZEeh8B3RUMPkbMW0WwK55HpTuWs5Z+9QwKkHdTazB8IiHXvdThwXlqfe974wvfB5NP481p1CJ7rVbMJ8NLeA6a5s= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id A4FEC1E32; Fri, 15 Aug 2025 01:55:37 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 4DA8C3F63F; Fri, 15 Aug 2025 01:55:41 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 04/18] arm64: Introduce por_elx_set_pkey_perms() helper Date: Fri, 15 Aug 2025 09:54:58 +0100 Message-ID: <20250815085512.2182322-5-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Introduce a helper that sets the permissions of a given pkey (POIndex) in the POR_ELx format, and make use of it in arch_set_user_pkey_access(). Signed-off-by: Kevin Brodsky --- arch/arm64/include/asm/por.h | 7 +++++++ arch/arm64/mm/mmu.c | 26 ++++++++++---------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/arch/arm64/include/asm/por.h b/arch/arm64/include/asm/por.h index d913d5b529e4..bffb4d2b1246 100644 --- a/arch/arm64/include/asm/por.h +++ b/arch/arm64/include/asm/por.h @@ -31,4 +31,11 @@ static inline bool por_elx_allows_exec(u64 por, u8 pkey) return perm & POE_X; } =20 +static inline u64 por_elx_set_pkey_perms(u64 por, u8 pkey, u64 perms) +{ + u64 shift =3D POR_ELx_PERM_SHIFT(pkey); + + return (por & ~(POE_MASK << shift)) | (perms << shift); +} + #endif /* _ASM_ARM64_POR_H */ diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 34e5d78af076..e41ed9e0d799 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1597,8 +1597,8 @@ void __cpu_replace_ttbr1(pgd_t *pgdp, bool cnp) #ifdef CONFIG_ARCH_HAS_PKEYS int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, unsigned = long init_val) { - u64 new_por; - u64 old_por; + u64 new_perms; + u64 por; =20 if (!system_supports_poe()) return -ENOSPC; @@ -1612,25 +1612,19 @@ int arch_set_user_pkey_access(struct task_struct *t= sk, int pkey, unsigned long i return -EINVAL; =20 /* Set the bits we need in POR: */ - new_por =3D POE_RWX; + new_perms =3D POE_RWX; if (init_val & PKEY_DISABLE_WRITE) - new_por &=3D ~POE_W; + new_perms &=3D ~POE_W; if (init_val & PKEY_DISABLE_ACCESS) - new_por &=3D ~POE_RW; + new_perms &=3D ~POE_RW; if (init_val & PKEY_DISABLE_READ) - new_por &=3D ~POE_R; + new_perms &=3D ~POE_R; if (init_val & PKEY_DISABLE_EXECUTE) - new_por &=3D ~POE_X; + new_perms &=3D ~POE_X; =20 - /* Shift the bits in to the correct place in POR for pkey: */ - new_por =3D POR_ELx_PERM_PREP(pkey, new_por); - - /* Get old POR and mask off any old bits in place: */ - old_por =3D read_sysreg_s(SYS_POR_EL0); - old_por &=3D ~(POE_MASK << POR_ELx_PERM_SHIFT(pkey)); - - /* Write old part along with new part: */ - write_sysreg_s(old_por | new_por, SYS_POR_EL0); + por =3D read_sysreg_s(SYS_POR_EL0); + por =3D por_elx_set_pkey_perms(por, pkey, new_perms); + write_sysreg_s(por, SYS_POR_EL0); =20 return 0; } --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 59FC72E1C5B; Fri, 15 Aug 2025 08:55:51 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248152; cv=none; b=L3AO2KoIU88eH1+Nq58VfD/ZHXlWTDVuXUQB0zRowBl9Mg83YzD0gZyoXivTu9+IKtn0/+AnrQwfVCneccPcQFg/+DGCRd7i3aRf37/g2oVqkEWXaIFhWKOZCzrUsEGoSkLlABvUoeSCGgond/9taGxVArh+SuY0DDAtfGaCISc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248152; c=relaxed/simple; bh=9x3EvjJGaWsAV1W1BN8oGAFzljiLdcdF+3EG4lSSIHw=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=sFY+uO9xO0E9YIpTifzxHjSwR6hu2Uc+S/6Dy0G2BEVykWNlXOj0vFbYShoFSZNBgdzV0gKM61gAmnhiCgbGNKoxdg3V0nMxurWWaq269+uXWm40Pl7iseTqwD5H2mxYc3rTnHL76bEXqB75lxr8K4C0lJAAU4CEwmXnn5ozDmc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 8534D1E32; Fri, 15 Aug 2025 01:55:42 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 2A4FC3F63F; Fri, 15 Aug 2025 01:55:46 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 05/18] arm64: Implement asm/kpkeys.h using POE Date: Fri, 15 Aug 2025 09:54:59 +0100 Message-ID: <20250815085512.2182322-6-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Implement the kpkeys interface if CONFIG_ARM64_POE is enabled. The permissions for KPKEYS_PKEY_DEFAULT (pkey 0) are set to RWX as this pkey is also used for code mappings. Signed-off-by: Kevin Brodsky --- arch/arm64/include/asm/kpkeys.h | 49 +++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 arch/arm64/include/asm/kpkeys.h diff --git a/arch/arm64/include/asm/kpkeys.h b/arch/arm64/include/asm/kpkey= s.h new file mode 100644 index 000000000000..3b0ab5e7dd22 --- /dev/null +++ b/arch/arm64/include/asm/kpkeys.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_KPKEYS_H +#define __ASM_KPKEYS_H + +#include +#include +#include + +#include + +static inline bool arch_kpkeys_enabled(void) +{ + return system_supports_poe(); +} + +#ifdef CONFIG_ARM64_POE + +static inline u64 por_set_kpkeys_level(u64 por, int level) +{ + por =3D por_elx_set_pkey_perms(por, KPKEYS_PKEY_DEFAULT, POE_RWX); + + return por; +} + +static __always_inline void __kpkeys_set_pkey_reg_nosync(u64 pkey_reg) +{ + write_sysreg_s(pkey_reg, SYS_POR_EL1); +} + +static __always_inline int arch_kpkeys_set_level(int level) +{ + u64 prev_por =3D read_sysreg_s(SYS_POR_EL1); + u64 new_por =3D por_set_kpkeys_level(prev_por, level); + + __kpkeys_set_pkey_reg_nosync(new_por); + isb(); + + return prev_por; +} + +static __always_inline void arch_kpkeys_restore_pkey_reg(u64 pkey_reg) +{ + __kpkeys_set_pkey_reg_nosync(pkey_reg); + isb(); +} + +#endif /* CONFIG_ARM64_POE */ + +#endif /* __ASM_KPKEYS_H */ --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 40B982DC326; Fri, 15 Aug 2025 08:55:55 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248157; cv=none; b=DGH+fP32BwKFdKBdfeAPF/tzArOpX0/g4N5eGVLBeGuYMV3feQAk8UOQrxONM9Fk7jZbsdDVuInQq0+tgnn+wIjY3W9FzHMScCF7a5CFd84m6AMphbF4ns9k+6lgcVr3u9QeX8m0U0YuWBiacMyIO9JU4K+ICvnXmKI5MNE3vNs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248157; c=relaxed/simple; bh=P2f0g0pfPqMGxvKXFobQpWgGSAUwBOVDNqc/pTSJKNA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=dSmTlFvYVE1DBG5+PG7KcDYlhRV4MLJ2D0s1ScB43rbIBoZKpPMSmfuoQaKuif+yy9Nx7iBDYgLdwO5HeEx7uRCiLdJp07XqY1UNgHPUwC/sFG0HcIR++MbppctFTSB08d4fEuVE4wWD9HGFVWp0YDDeFdwaHle02Wuab4Zy/v0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 619F51E32; Fri, 15 Aug 2025 01:55:47 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 076A53F63F; Fri, 15 Aug 2025 01:55:50 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 06/18] arm64: set_memory: Implement set_memory_pkey() Date: Fri, 15 Aug 2025 09:55:00 +0100 Message-ID: <20250815085512.2182322-7-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Implement set_memory_pkey() using POE if supported. Signed-off-by: Kevin Brodsky --- arch/arm64/include/asm/set_memory.h | 4 ++++ arch/arm64/mm/pageattr.c | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/arch/arm64/include/asm/set_memory.h b/arch/arm64/include/asm/s= et_memory.h index 90f61b17275e..b6cd6de34abf 100644 --- a/arch/arm64/include/asm/set_memory.h +++ b/arch/arm64/include/asm/set_memory.h @@ -19,4 +19,8 @@ bool kernel_page_present(struct page *page); int set_memory_encrypted(unsigned long addr, int numpages); int set_memory_decrypted(unsigned long addr, int numpages); =20 +#ifdef CONFIG_ARCH_HAS_KPKEYS +int set_memory_pkey(unsigned long addr, int numpages, int pkey); +#endif + #endif /* _ASM_ARM64_SET_MEMORY_H */ diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index 04d4a8f676db..41d87c2880fe 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -8,6 +8,7 @@ #include #include #include +#include =20 #include #include @@ -292,6 +293,30 @@ int set_direct_map_valid_noflush(struct page *page, un= signed nr, bool valid) return set_memory_valid(addr, nr, valid); } =20 +#ifdef CONFIG_ARCH_HAS_KPKEYS +int set_memory_pkey(unsigned long addr, int numpages, int pkey) +{ + unsigned long set_prot =3D 0; + + if (!system_supports_poe()) + return 0; + + if (!__is_lm_address(addr)) + return -EINVAL; + + if (pkey >=3D arch_max_pkey()) + return -EINVAL; + + set_prot |=3D pkey & BIT(0) ? PTE_PO_IDX_0 : 0; + set_prot |=3D pkey & BIT(1) ? PTE_PO_IDX_1 : 0; + set_prot |=3D pkey & BIT(2) ? PTE_PO_IDX_2 : 0; + + return __change_memory_common(addr, PAGE_SIZE * numpages, + __pgprot(set_prot), + __pgprot(PTE_PO_IDX_MASK)); +} +#endif + #ifdef CONFIG_DEBUG_PAGEALLOC /* * This is - apart from the return value - doing the same --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id ECB2E2E36F3; Fri, 15 Aug 2025 08:56:00 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248162; cv=none; b=cc6AGDRaJB9JXHl3vT6njL+y/2xCXRKayHzBmSz4GabsxzsymJnBF+gtVFDuMEPaJu1rw1US2OtBtFX5+CFfGiaJTxXyYLzcDl5wEyrOQ23+Dz4JiBK0J3YV+GUtwgUz30p99jmn+lWQcEW0zqnH/HNqdieyXRoVC/hkeTxa0aM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248162; c=relaxed/simple; bh=q03YaUF8+LrADLsj+GAMBkSRmOXh1tilKr7kebO7eUg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=tyxihR+BYloPK4SzSxdz7fzT8uHIjltJ2vzDtZJu2UMP1PO3/J3f7N2QEzcibT63g/xk353Js01koiPWQlsu+mpBjLx61P1RYKJxMCQLUdkj3rpYtTFzZiQijZsUZ8guHumJ+JBhaOA8mMQrtZvPMd5sXMe+i1HduEldML9jpHc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 3B7B31E32; Fri, 15 Aug 2025 01:55:52 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id D84FA3F63F; Fri, 15 Aug 2025 01:55:55 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 07/18] arm64: Reset POR_EL1 on exception entry Date: Fri, 15 Aug 2025 09:55:01 +0100 Message-ID: <20250815085512.2182322-8-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" POR_EL1 will be modified, through the kpkeys framework, in order to grant temporary RW access to certain keys. If an exception occurs in the middle of a "critical section" where POR_EL1 is set to a privileged value, it is preferable to reset it to its default value upon taking the exception to minimise the amount of code running at higher kpkeys level. This patch implements the reset of POR_EL1 on exception entry, storing the original value in a new pt_regs field and restoring on exception return. To avoid an expensive ISB, the register is only reset if the interrupted value isn't the default. No check is made on the return path as an ISB occurs anyway as part of ERET. Signed-off-by: Kevin Brodsky --- arch/arm64/include/asm/kpkeys.h | 10 ++++++++++ arch/arm64/include/asm/por.h | 4 ++++ arch/arm64/include/asm/ptrace.h | 4 ++++ arch/arm64/kernel/asm-offsets.c | 3 +++ arch/arm64/kernel/entry.S | 24 +++++++++++++++++++++++- 5 files changed, 44 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kpkeys.h b/arch/arm64/include/asm/kpkey= s.h index 3b0ab5e7dd22..79ae33388088 100644 --- a/arch/arm64/include/asm/kpkeys.h +++ b/arch/arm64/include/asm/kpkeys.h @@ -8,6 +8,14 @@ =20 #include =20 +/* + * Equivalent to por_set_kpkeys_level(0, KPKEYS_LVL_DEFAULT), but can also= be + * used in assembly. + */ +#define POR_EL1_INIT POR_ELx_PERM_PREP(KPKEYS_PKEY_DEFAULT, POE_RWX) + +#ifndef __ASSEMBLY__ + static inline bool arch_kpkeys_enabled(void) { return system_supports_poe(); @@ -46,4 +54,6 @@ static __always_inline void arch_kpkeys_restore_pkey_reg(= u64 pkey_reg) =20 #endif /* CONFIG_ARM64_POE */ =20 +#endif /* __ASSEMBLY__ */ + #endif /* __ASM_KPKEYS_H */ diff --git a/arch/arm64/include/asm/por.h b/arch/arm64/include/asm/por.h index bffb4d2b1246..58dce4b8021b 100644 --- a/arch/arm64/include/asm/por.h +++ b/arch/arm64/include/asm/por.h @@ -10,6 +10,8 @@ =20 #define POR_EL0_INIT POR_ELx_PERM_PREP(0, POE_RWX) =20 +#ifndef __ASSEMBLY__ + static inline bool por_elx_allows_read(u64 por, u8 pkey) { u8 perm =3D POR_ELx_PERM_GET(pkey, por); @@ -38,4 +40,6 @@ static inline u64 por_elx_set_pkey_perms(u64 por, u8 pkey= , u64 perms) return (por & ~(POE_MASK << shift)) | (perms << shift); } =20 +#endif /* __ASSEMBLY__ */ + #endif /* _ASM_ARM64_POR_H */ diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrac= e.h index 47ff8654c5ec..e907df4225d4 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -166,6 +166,10 @@ struct pt_regs { u64 orig_x0; s32 syscallno; u32 pmr; +#ifdef CONFIG_ARM64_POE + u64 por_el1; + u64 __unused; +#endif =20 u64 sdei_ttbr1; struct frame_record_meta stackframe; diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offset= s.c index 30d4bbe68661..8ae5cc3c203b 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -75,6 +75,9 @@ int main(void) DEFINE(S_SYSCALLNO, offsetof(struct pt_regs, syscallno)); DEFINE(S_SDEI_TTBR1, offsetof(struct pt_regs, sdei_ttbr1)); DEFINE(S_PMR, offsetof(struct pt_regs, pmr)); +#ifdef CONFIG_ARM64_POE + DEFINE(S_POR_EL1, offsetof(struct pt_regs, por_el1)); +#endif DEFINE(S_STACKFRAME, offsetof(struct pt_regs, stackframe)); DEFINE(S_STACKFRAME_TYPE, offsetof(struct pt_regs, stackframe.type)); DEFINE(PT_REGS_SIZE, sizeof(struct pt_regs)); diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index f8018b5c1f9a..0dd6f7fbb669 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -277,6 +278,19 @@ alternative_else_nop_endif .else add x21, sp, #PT_REGS_SIZE get_current_task tsk +#ifdef CONFIG_ARM64_POE +alternative_if_not ARM64_HAS_S1POE + b 1f +alternative_else_nop_endif + mrs_s x0, SYS_POR_EL1 + str x0, [sp, #S_POR_EL1] + mov x1, #POR_EL1_INIT + cmp x0, x1 + b.eq 1f + msr_s SYS_POR_EL1, x1 + isb +1: +#endif /* CONFIG_ARM64_POE */ .endif /* \el =3D=3D 0 */ mrs x22, elr_el1 mrs x23, spsr_el1 @@ -407,7 +421,15 @@ alternative_else_nop_endif mte_set_user_gcr tsk, x0, x1 =20 apply_ssbd 0, x0, x1 - .endif + .else +#ifdef CONFIG_ARM64_POE +alternative_if ARM64_HAS_S1POE + ldr x0, [sp, #S_POR_EL1] + msr_s SYS_POR_EL1, x0 + /* No explicit ISB; we rely on ERET */ +alternative_else_nop_endif +#endif /* CONFIG_ARM64_POE */ + .endif /* \el =3D=3D 0 */ =20 msr elr_el1, x21 // set up the return data msr spsr_el1, x22 --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id EE5BB2E7173; Fri, 15 Aug 2025 08:56:05 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248167; cv=none; b=fObiAXcHczJStGxxd9L/ubFcxJd+R8BVFt3C0cpOoZcRjbDzLHBbighVlhMguOmCuIT87Hngf4qFBnR3AASxfj38KjtXo8bDKYjYuQIG64OTaSwS+kmaHPzo3pcKIIfKnzjpmD7OW86rcP26Gigyl461p/CcsYGp7flePEvXstE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248167; c=relaxed/simple; bh=L4s+cOmjrRIeXaCKlqYsCN5eaWROkhsJoYnk1vEKkOM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=OJ0XcLVKnImJHBA9b4LDskSRdsQ0XWvEuMQHS3y+lESMnfIub681XvKCl7+mMSoNsjFqCpgpxmqw2nDWuB2jEFaDkP+yk39bwgsivOQq788TVE3kxjV1+KdflNAHNuRQg217qnDj2sPFTsgPsD3TSEzPM9ADVI9kjLwoF1Zp+HA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 1BC321688; Fri, 15 Aug 2025 01:55:57 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id B4F573F63F; Fri, 15 Aug 2025 01:56:00 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 08/18] arm64: Context-switch POR_EL1 Date: Fri, 15 Aug 2025 09:55:02 +0100 Message-ID: <20250815085512.2182322-9-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" POR_EL1 is about to be used by the kpkeys framework, modifying it for (typically small) sections of code. If an exception occurs during that window and scheduling occurs, we must ensure that POR_EL1 is context-switched as needed (saving the old value and restoring the new one). An ISB is needed to ensure the write takes effect, so we skip it if the new value is the same as the old, like for POR_EL0. Signed-off-by: Kevin Brodsky --- arch/arm64/include/asm/processor.h | 1 + arch/arm64/kernel/process.c | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/pr= ocessor.h index 61d62bfd5a7b..9340e94a27f6 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -187,6 +187,7 @@ struct thread_struct { u64 svcr; u64 tpidr2_el0; u64 por_el0; + u64 por_el1; #ifdef CONFIG_ARM64_GCS unsigned int gcs_el0_mode; unsigned int gcs_el0_locked; diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 96482a1412c6..f698839f018f 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -428,6 +428,9 @@ int copy_thread(struct task_struct *p, const struct ker= nel_clone_args *args) =20 ptrauth_thread_init_kernel(p); =20 + if (system_supports_poe()) + p->thread.por_el1 =3D read_sysreg_s(SYS_POR_EL1); + if (likely(!args->fn)) { *childregs =3D *current_pt_regs(); childregs->regs[0] =3D 0; @@ -678,6 +681,12 @@ static void permission_overlay_switch(struct task_stru= ct *next) * of POR_EL0. */ } + + current->thread.por_el1 =3D read_sysreg_s(SYS_POR_EL1); + if (current->thread.por_el1 !=3D next->thread.por_el1) { + write_sysreg_s(next->thread.por_el1, SYS_POR_EL1); + isb(); + } } =20 /* --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id BCD90279794; Fri, 15 Aug 2025 08:56:10 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248173; cv=none; b=fMvVs+iT+/H5vpY3ihXzGO73dWHH3yYKbiY4n8UzPNNBbqUk4WpoDFKa1oZ51F6hc3uTfz0W6XfknvQt46rIiilPQbc6Zw4viflB+84HcOYuMJ0MaQvrvF7M4pf40dStj6Ah8vjnkRuN0PwE+do9WvKXdfTatVSVXgp7BKyfW2c= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248173; c=relaxed/simple; bh=tmZRCNowstZAI1V2bmLIFPQpUCMRJW13eBZ4xESbbJA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=C3aWVJg1m7xqnJKLNv02FS9uQkwEhMUh52R84J7C9l4ChtQi75LynDOP6/xBikUiX5IxETh1+v946uHj2GlKTRsRm2FmS7orU2nDeURFzjlunNurbfvWdC0r9uOvIAdWsA0lju1vMLB4vSHtF1fkspPRWD41tpndMcWM7xABaLo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id ECC4F1688; Fri, 15 Aug 2025 01:56:01 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 924E93F63F; Fri, 15 Aug 2025 01:56:05 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 09/18] arm64: Enable kpkeys Date: Fri, 15 Aug 2025 09:55:03 +0100 Message-ID: <20250815085512.2182322-10-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" This is the final step to enable kpkeys on arm64. We enable POE at EL1 by setting TCR2_EL1.POE, and initialise POR_EL1 to the default value, enabling access to the default pkey/POIndex (0). An ISB is added so that POE restrictions are enforced immediately. Having done this, we can now select ARCH_HAS_KPKEYS if ARM64_POE is enabled. Signed-off-by: Kevin Brodsky --- arch/arm64/Kconfig | 1 + arch/arm64/kernel/cpufeature.c | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index e9bbfacc35a6..88b544244829 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2187,6 +2187,7 @@ config ARM64_POE def_bool y select ARCH_USES_HIGH_VMA_FLAGS select ARCH_HAS_PKEYS + select ARCH_HAS_KPKEYS help The Permission Overlay Extension is used to implement Memory Protection Keys. Memory Protection Keys provides a mechanism for diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 9ad065f15f1d..4a631115341a 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -76,6 +76,7 @@ #include #include #include +#include =20 #include #include @@ -2458,8 +2459,10 @@ static void cpu_enable_mops(const struct arm64_cpu_c= apabilities *__unused) #ifdef CONFIG_ARM64_POE static void cpu_enable_poe(const struct arm64_cpu_capabilities *__unused) { - sysreg_clear_set(REG_TCR2_EL1, 0, TCR2_EL1_E0POE); + write_sysreg_s(POR_EL1_INIT, SYS_POR_EL1); + sysreg_clear_set(REG_TCR2_EL1, 0, TCR2_EL1_E0POE | TCR2_EL1_POE); sysreg_clear_set(CPACR_EL1, 0, CPACR_EL1_E0POE); + isb(); } #endif =20 --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 2BF3828751D; Fri, 15 Aug 2025 08:56:15 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248178; cv=none; b=ZE01iSM7rTSBrsyjZJThreTcJsUgvzJhIj2a+8Ot/o70wMFpAauoewcc1DM9suEka9xqDPgkCMoORK7dZBGnzEe79PREblWLzKT50OuNQ8Djyj1YFFEzBOfrAH57UYJm+VWA39GZWyTF7JqgqVQWnerqW2tssSB5rSQ4+E0Ubug= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248178; c=relaxed/simple; bh=PbxzDzQ+4d4PrMMgh9IiYEZHqq8hGIwh0q0NJV1kFh0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=kK7lVlb4tJKGZ8vx3WXi6YdVk1rtrupn1nbMjeAJ8t/trQd0WTbk552K1zYvcFGlrxYRZCiXHsiwS6W+w2SHuJ9trc/XD6HzBIFU03KwmwxnLrM4T6vjrP11WgOaShJzsXuZgxKQ+OLGjRxzr/SGsG/iJ2k8A7G6kUnBhPn+LDM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id C99AA1688; Fri, 15 Aug 2025 01:56:06 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 6ED623F63F; Fri, 15 Aug 2025 01:56:10 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 10/18] mm: Introduce kernel_pgtables_set_pkey() Date: Fri, 15 Aug 2025 09:55:04 +0100 Message-ID: <20250815085512.2182322-11-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" kernel_pgtables_set_pkey() allows setting the pkey of all page table pages in swapper_pg_dir, recursively. This will be needed by kpkeys_hardened_pgtables, as it relies on all PTPs being mapped with a non-default pkey. Those initial kernel page tables cannot practically be assigned a non-default pkey right when they are allocated, so mutating them during (early) boot is required. Signed-off-by: Kevin Brodsky --- include/linux/mm.h | 2 + mm/memory.c | 137 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/include/linux/mm.h b/include/linux/mm.h index 1ae97a0b8ec7..f4dd96f3db91 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4134,6 +4134,8 @@ int arch_get_shadow_stack_status(struct task_struct *= t, unsigned long __user *st int arch_set_shadow_stack_status(struct task_struct *t, unsigned long stat= us); int arch_lock_shadow_stack_status(struct task_struct *t, unsigned long sta= tus); =20 +int kernel_pgtables_set_pkey(int pkey); + =20 /* * mseal of userspace process's system mappings. diff --git a/mm/memory.c b/mm/memory.c index 0ba4f6b71847..4f144abf5fc3 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -75,6 +75,8 @@ #include #include #include +#include +#include =20 #include =20 @@ -7183,3 +7185,138 @@ void vma_pgtable_walk_end(struct vm_area_struct *vm= a) if (is_vm_hugetlb_page(vma)) hugetlb_vma_unlock_read(vma); } + +static int __init set_page_pkey(void *p, int pkey) +{ + unsigned long addr =3D (unsigned long)p; + + /* + * swapper_pg_dir itself will be made read-only by mark_rodata_ro() + * so there is no point in changing its pkey. + */ + if (p =3D=3D swapper_pg_dir) + return 0; + + return set_memory_pkey(addr, 1, pkey); +} + +static int __init set_pkey_pte(pmd_t *pmd, int pkey) +{ + pte_t *pte; + int err; + + pte =3D pte_offset_kernel(pmd, 0); + err =3D set_page_pkey(pte, pkey); + + return err; +} + +static int __init set_pkey_pmd(pud_t *pud, int pkey) +{ + pmd_t *pmd; + int i, err =3D 0; + + pmd =3D pmd_offset(pud, 0); + + err =3D set_page_pkey(pmd, pkey); + if (err) + return err; + + for (i =3D 0; i < PTRS_PER_PMD; i++) { + if (pmd_none(pmd[i]) || pmd_bad(pmd[i]) || pmd_leaf(pmd[i])) + continue; + err =3D set_pkey_pte(&pmd[i], pkey); + if (err) + break; + } + + return err; +} + +static int __init set_pkey_pud(p4d_t *p4d, int pkey) +{ + pud_t *pud; + int i, err =3D 0; + + if (mm_pmd_folded(&init_mm)) + return set_pkey_pmd((pud_t *)p4d, pkey); + + pud =3D pud_offset(p4d, 0); + + err =3D set_page_pkey(pud, pkey); + if (err) + return err; + + for (i =3D 0; i < PTRS_PER_PUD; i++) { + if (pud_none(pud[i]) || pud_bad(pud[i]) || pud_leaf(pud[i])) + continue; + err =3D set_pkey_pmd(&pud[i], pkey); + if (err) + break; + } + + return err; +} + +static int __init set_pkey_p4d(pgd_t *pgd, int pkey) +{ + p4d_t *p4d; + int i, err =3D 0; + + if (mm_pud_folded(&init_mm)) + return set_pkey_pud((p4d_t *)pgd, pkey); + + p4d =3D p4d_offset(pgd, 0); + + err =3D set_page_pkey(p4d, pkey); + if (err) + return err; + + for (i =3D 0; i < PTRS_PER_P4D; i++) { + if (p4d_none(p4d[i]) || p4d_bad(p4d[i]) || p4d_leaf(p4d[i])) + continue; + err =3D set_pkey_pud(&p4d[i], pkey); + if (err) + break; + } + + return err; +} + +/** + * kernel_pgtables_set_pkey - set pkey for all kernel page table pages + * @pkey: pkey to set the page table pages to + * + * Walks swapper_pg_dir setting the protection key of every page table pag= e (at + * all levels) to @pkey. swapper_pg_dir itself is left untouched as it is + * expected to be mapped read-only by mark_rodata_ro(). + * + * No-op if the architecture does not support kpkeys. + */ +int __init kernel_pgtables_set_pkey(int pkey) +{ + pgd_t *pgd =3D swapper_pg_dir; + int i, err =3D 0; + + if (!arch_kpkeys_enabled()) + return 0; + + spin_lock(&init_mm.page_table_lock); + + if (mm_p4d_folded(&init_mm)) { + err =3D set_pkey_p4d(pgd, pkey); + goto out; + } + + for (i =3D 0; i < PTRS_PER_PGD; i++) { + if (pgd_none(pgd[i]) || pgd_bad(pgd[i]) || pgd_leaf(pgd[i])) + continue; + err =3D set_pkey_p4d(&pgd[i], pkey); + if (err) + break; + } + +out: + spin_unlock(&init_mm.page_table_lock); + return err; +} --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 640172F0669; Fri, 15 Aug 2025 08:56:20 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248182; cv=none; b=F7gccMEXREopI2K1wGfy+Z5cOYzXDdvfVdtfYj40eqkf9tD53PoRhanj/ehrQycSCzWx6+qBy3UNd9KG0O4geX9ECh0439DQC2lXXUKcbov3j13cBINXyWBAKAj79t23W1yIiuPCq0SAQ80RNLZEW4a0ubzYEBxUgUveVklvqSo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248182; c=relaxed/simple; bh=r+JSQKdi5dzAjbtVEKpEWaEwHP8e7BXDXVkpz1ngfiU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=KiRmvr4bHTxW6d+bGRv4RwEG3QLw0fEldVGv1BzF5l0SS4oYSntCLkt7E5YuRdhbgb6lcHMeHPEzbRlAd2t8OX0trsdM+PkIIOXc1lO4UKELlhZuvS+wxcatxHhRyNyhQxcABvbcKD4mdVDK2Sbx4inVUDFhDT6b3icupeLd9wo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id A38611688; Fri, 15 Aug 2025 01:56:11 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 4C4DA3F63F; Fri, 15 Aug 2025 01:56:15 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 11/18] mm: Introduce kpkeys_hardened_pgtables Date: Fri, 15 Aug 2025 09:55:05 +0100 Message-ID: <20250815085512.2182322-12-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" kpkeys_hardened_pgtables is a hardening feature based on kpkeys. It aims to prevent the corruption of page tables by: 1. mapping all page table pages, both kernel and user, with a privileged pkey (KPKEYS_PKEY_PGTABLES), and 2. granting write access to that pkey only when running at a higher kpkeys level (KPKEYS_LVL_PGTABLES). The feature is exposed as CONFIG_KPKEYS_HARDENED_PGTABLES; it requires explicit architecture opt-in by selecting ARCH_HAS_KPKEYS_HARDENED_PGTABLES, since much of the page table handling is arch-specific. This patch introduces an API to modify the PTPs' pkey. Because this API is going to be called from low-level pgtable helpers, it must be inactive on boot and explicitly switched on if and when kpkeys become available. A static key is used for that purpose; it is the responsibility of each architecture supporting kpkeys_hardened_pgtables to call kpkeys_hardened_pgtables_enable() as early as possible to switch on that static key. The initial kernel page tables are also walked to set their pkey, since they have already been allocated at that point. Signed-off-by: Kevin Brodsky --- include/asm-generic/kpkeys.h | 4 +++ include/linux/kpkeys.h | 46 ++++++++++++++++++++++++++++++++++- mm/Kconfig | 3 +++ mm/Makefile | 1 + mm/kpkeys_hardened_pgtables.c | 44 +++++++++++++++++++++++++++++++++ security/Kconfig.hardening | 12 +++++++++ 6 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 mm/kpkeys_hardened_pgtables.c diff --git a/include/asm-generic/kpkeys.h b/include/asm-generic/kpkeys.h index ab819f157d6a..cec92334a9f3 100644 --- a/include/asm-generic/kpkeys.h +++ b/include/asm-generic/kpkeys.h @@ -2,6 +2,10 @@ #ifndef __ASM_GENERIC_KPKEYS_H #define __ASM_GENERIC_KPKEYS_H =20 +#ifndef KPKEYS_PKEY_PGTABLES +#define KPKEYS_PKEY_PGTABLES 1 +#endif + #ifndef KPKEYS_PKEY_DEFAULT #define KPKEYS_PKEY_DEFAULT 0 #endif diff --git a/include/linux/kpkeys.h b/include/linux/kpkeys.h index faa6e2615798..5f4b096374ba 100644 --- a/include/linux/kpkeys.h +++ b/include/linux/kpkeys.h @@ -4,11 +4,15 @@ =20 #include #include +#include + +struct folio; =20 #define KPKEYS_LVL_DEFAULT 0 +#define KPKEYS_LVL_PGTABLES 1 =20 #define KPKEYS_LVL_MIN KPKEYS_LVL_DEFAULT -#define KPKEYS_LVL_MAX KPKEYS_LVL_DEFAULT +#define KPKEYS_LVL_MAX KPKEYS_LVL_PGTABLES =20 #define __KPKEYS_GUARD(name, set_level, restore_pkey_reg, set_arg, ...) \ __DEFINE_CLASS_IS_CONDITIONAL(name, false); \ @@ -110,4 +114,44 @@ static inline bool arch_kpkeys_enabled(void) =20 #endif /* CONFIG_ARCH_HAS_KPKEYS */ =20 +#ifdef CONFIG_KPKEYS_HARDENED_PGTABLES + +DECLARE_STATIC_KEY_FALSE(kpkeys_hardened_pgtables_key); + +static inline bool kpkeys_hardened_pgtables_enabled(void) +{ + return static_branch_unlikely(&kpkeys_hardened_pgtables_key); +} + +int kpkeys_protect_pgtable_memory(struct folio *folio); +int kpkeys_unprotect_pgtable_memory(struct folio *folio); + +/* + * Enables kpkeys_hardened_pgtables and switches existing kernel page tabl= es to + * a privileged pkey (KPKEYS_PKEY_PGTABLES). + * + * Should be called as early as possible by architecture code, after (k)pk= eys + * are initialised and before any user task is spawned. + */ +void kpkeys_hardened_pgtables_enable(void); + +#else /* CONFIG_KPKEYS_HARDENED_PGTABLES */ + +static inline bool kpkeys_hardened_pgtables_enabled(void) +{ + return false; +} + +static inline int kpkeys_protect_pgtable_memory(struct folio *folio) +{ + return 0; +} +static inline int kpkeys_unprotect_pgtable_memory(struct folio *folio) +{ + return 0; +} +static inline void kpkeys_hardened_pgtables_enable(void) {} + +#endif /* CONFIG_KPKEYS_HARDENED_PGTABLES */ + #endif /* _LINUX_KPKEYS_H */ diff --git a/mm/Kconfig b/mm/Kconfig index 90f2e5c381a6..e34edf5c41e7 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1175,6 +1175,9 @@ config ARCH_HAS_PKEYS bool config ARCH_HAS_KPKEYS bool +# ARCH_HAS_KPKEYS must be selected when selecting this option +config ARCH_HAS_KPKEYS_HARDENED_PGTABLES + bool =20 config ARCH_USES_PG_ARCH_2 bool diff --git a/mm/Makefile b/mm/Makefile index ef54aa615d9d..10848df0ca85 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -147,3 +147,4 @@ obj-$(CONFIG_SHRINKER_DEBUG) +=3D shrinker_debug.o obj-$(CONFIG_EXECMEM) +=3D execmem.o obj-$(CONFIG_TMPFS_QUOTA) +=3D shmem_quota.o obj-$(CONFIG_PT_RECLAIM) +=3D pt_reclaim.o +obj-$(CONFIG_KPKEYS_HARDENED_PGTABLES) +=3D kpkeys_hardened_pgtables.o diff --git a/mm/kpkeys_hardened_pgtables.c b/mm/kpkeys_hardened_pgtables.c new file mode 100644 index 000000000000..931fa97bc8a7 --- /dev/null +++ b/mm/kpkeys_hardened_pgtables.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include + +DEFINE_STATIC_KEY_FALSE(kpkeys_hardened_pgtables_key); + +int kpkeys_protect_pgtable_memory(struct folio *folio) +{ + unsigned long addr =3D (unsigned long)folio_address(folio); + unsigned int order =3D folio_order(folio); + int ret =3D 0; + + if (kpkeys_hardened_pgtables_enabled()) + ret =3D set_memory_pkey(addr, 1 << order, KPKEYS_PKEY_PGTABLES); + + WARN_ON(ret); + return ret; +} + +int kpkeys_unprotect_pgtable_memory(struct folio *folio) +{ + unsigned long addr =3D (unsigned long)folio_address(folio); + unsigned int order =3D folio_order(folio); + int ret =3D 0; + + if (kpkeys_hardened_pgtables_enabled()) + ret =3D set_memory_pkey(addr, 1 << order, KPKEYS_PKEY_DEFAULT); + + WARN_ON(ret); + return ret; +} + +void __init kpkeys_hardened_pgtables_enable(void) +{ + int ret; + + if (!arch_kpkeys_enabled()) + return; + + static_branch_enable(&kpkeys_hardened_pgtables_key); + ret =3D kernel_pgtables_set_pkey(KPKEYS_PKEY_PGTABLES); + WARN_ON(ret); +} diff --git a/security/Kconfig.hardening b/security/Kconfig.hardening index b9a5bc3430aa..41b7530530b7 100644 --- a/security/Kconfig.hardening +++ b/security/Kconfig.hardening @@ -265,6 +265,18 @@ config BUG_ON_DATA_CORRUPTION =20 If unsure, say N. =20 +config KPKEYS_HARDENED_PGTABLES + bool "Harden page tables using kernel pkeys" + depends on ARCH_HAS_KPKEYS_HARDENED_PGTABLES + help + This option makes all page tables mostly read-only by + allocating them with a non-default protection key (pkey) and + only enabling write access to that pkey in routines that are + expected to write to page table entries. + + This option has no effect if the system does not support + kernel pkeys. + endmenu =20 config CC_HAS_RANDSTRUCT --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 28DD22D876C; Fri, 15 Aug 2025 08:56:25 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248186; cv=none; b=eS/vk7rFHWVzOVE0NlHOFOLWaPzDY1xJy+2wh11DSkWsCAM3hRKrfV1Lsw7WEesUgpgOS7P0CksV0DLl6tfnlOtS+ggCpEmJZy//gQuY1R84AKiJs0TsdEcjJCQ5Lem6O2q49F261p+hmhuuKovF9nyENrCELYavrgen0ddzwV8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248186; c=relaxed/simple; bh=E5zQYM0CFx26SS/9pDiSxy4PulH4x1INJKJj8zOqUXk=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=Ya2ZsEz9SIVXaGc7ztRmZVMxHb0OeU6thKzdMbicIbvjcM+6v7NtjpWBoUtQlNEziVh422vgJ90ZvggL1nWlueQu7sc5n4SrGVr4BbAOqpZS6GtmWPdVsudZmREawMy52RuN3FjOGfwJuB4nh/7gnbnkSWQY+Vr1yFjKx5nZyLc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 834CF497; Fri, 15 Aug 2025 01:56:16 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 28E7A3F63F; Fri, 15 Aug 2025 01:56:20 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 12/18] mm: Allow __pagetable_ctor() to fail Date: Fri, 15 Aug 2025 09:55:06 +0100 Message-ID: <20250815085512.2182322-13-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In preparation for adding construction hooks (that may fail) to __pagetable_ctor(), make __pagetable_ctor() return a bool, propagate it to pagetable_*_ctor() and handle failure in the generic {pud,p4d,pgd}_alloc. Signed-off-by: Kevin Brodsky --- include/asm-generic/pgalloc.h | 15 ++++++++++++--- include/linux/mm.h | 21 ++++++++++----------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h index 3c8ec3bfea44..3e184f3ca37a 100644 --- a/include/asm-generic/pgalloc.h +++ b/include/asm-generic/pgalloc.h @@ -178,7 +178,10 @@ static inline pud_t *__pud_alloc_one_noprof(struct mm_= struct *mm, unsigned long if (!ptdesc) return NULL; =20 - pagetable_pud_ctor(ptdesc); + if (!pagetable_pud_ctor(ptdesc)) { + pagetable_free(ptdesc); + return NULL; + } return ptdesc_address(ptdesc); } #define __pud_alloc_one(...) alloc_hooks(__pud_alloc_one_noprof(__VA_ARGS_= _)) @@ -232,7 +235,10 @@ static inline p4d_t *__p4d_alloc_one_noprof(struct mm_= struct *mm, unsigned long if (!ptdesc) return NULL; =20 - pagetable_p4d_ctor(ptdesc); + if (!pagetable_p4d_ctor(ptdesc)) { + pagetable_free(ptdesc); + return NULL; + } return ptdesc_address(ptdesc); } #define __p4d_alloc_one(...) alloc_hooks(__p4d_alloc_one_noprof(__VA_ARGS_= _)) @@ -276,7 +282,10 @@ static inline pgd_t *__pgd_alloc_noprof(struct mm_stru= ct *mm, unsigned int order if (!ptdesc) return NULL; =20 - pagetable_pgd_ctor(ptdesc); + if (!pagetable_pgd_ctor(ptdesc)) { + pagetable_free(ptdesc); + return NULL; + } return ptdesc_address(ptdesc); } #define __pgd_alloc(...) alloc_hooks(__pgd_alloc_noprof(__VA_ARGS__)) diff --git a/include/linux/mm.h b/include/linux/mm.h index f4dd96f3db91..d9371d992033 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2973,12 +2973,13 @@ static inline bool ptlock_init(struct ptdesc *ptdes= c) { return true; } static inline void ptlock_free(struct ptdesc *ptdesc) {} #endif /* defined(CONFIG_SPLIT_PTE_PTLOCKS) */ =20 -static inline void __pagetable_ctor(struct ptdesc *ptdesc) +static inline bool __pagetable_ctor(struct ptdesc *ptdesc) { struct folio *folio =3D ptdesc_folio(ptdesc); =20 __folio_set_pgtable(folio); lruvec_stat_add_folio(folio, NR_PAGETABLE); + return true; } =20 static inline void pagetable_dtor(struct ptdesc *ptdesc) @@ -3001,8 +3002,7 @@ static inline bool pagetable_pte_ctor(struct mm_struc= t *mm, { if (mm !=3D &init_mm && !ptlock_init(ptdesc)) return false; - __pagetable_ctor(ptdesc); - return true; + return __pagetable_ctor(ptdesc); } =20 pte_t *___pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp); @@ -3109,8 +3109,7 @@ static inline bool pagetable_pmd_ctor(struct mm_struc= t *mm, if (mm !=3D &init_mm && !pmd_ptlock_init(ptdesc)) return false; ptdesc_pmd_pts_init(ptdesc); - __pagetable_ctor(ptdesc); - return true; + return __pagetable_ctor(ptdesc); } =20 /* @@ -3132,19 +3131,19 @@ static inline spinlock_t *pud_lock(struct mm_struct= *mm, pud_t *pud) return ptl; } =20 -static inline void pagetable_pud_ctor(struct ptdesc *ptdesc) +static inline bool pagetable_pud_ctor(struct ptdesc *ptdesc) { - __pagetable_ctor(ptdesc); + return __pagetable_ctor(ptdesc); } =20 -static inline void pagetable_p4d_ctor(struct ptdesc *ptdesc) +static inline bool pagetable_p4d_ctor(struct ptdesc *ptdesc) { - __pagetable_ctor(ptdesc); + return __pagetable_ctor(ptdesc); } =20 -static inline void pagetable_pgd_ctor(struct ptdesc *ptdesc) +static inline bool pagetable_pgd_ctor(struct ptdesc *ptdesc) { - __pagetable_ctor(ptdesc); + return __pagetable_ctor(ptdesc); } =20 extern void __init pagecache_init(void); --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 422AF2EA17D; Fri, 15 Aug 2025 08:56:30 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248191; cv=none; b=E2jJ52JY2qKE3mGIst39e2xtiaxJhc8aYNhnn0BUWGBPtwJD/YyodV34yOqrE86ZrM6a5bScN2NmO7mJ9A8Bs3p2uuK2E6DGLj2fBBHwzEe51lRTDEQn9naESBM1l6jt3EYMH5+KVJKx05KcDQMkg9o8fHGW6o5aHAXN8H4eQzY= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248191; c=relaxed/simple; bh=ToXBWOgGTz8chhNsAvevUA/c/HXTQ5KBWXpirOIkp70=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=gZCGi4A7u4lptLxgUvSGDFL4LEDer2wXLVOjOESnd94YWqPcEw2ZT45V29TgdJDsQYWRMZl7kzz92LNljxFUB1A5Fn/fZD3OIVsmIYf5REGuwj/nPjmThSMKsjM5NeXENcrJZrwXZt/Zjiy7G77QSLOeOV8upFDnW8N5RaFljA8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 72FCD1688; Fri, 15 Aug 2025 01:56:21 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 053E83F63F; Fri, 15 Aug 2025 01:56:24 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 13/18] mm: Map page tables with privileged pkey Date: Fri, 15 Aug 2025 09:55:07 +0100 Message-ID: <20250815085512.2182322-14-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" If CONFIG_KPKEYS_HARDENED_PGTABLES is enabled, map allocated page table pages using a privileged pkey (KPKEYS_PKEY_PGTABLES), so that page tables can only be written under guard(kpkeys_hardened_pgtables). This patch is a no-op if CONFIG_KPKEYS_HARDENED_PGTABLES is disabled (default). Signed-off-by: Kevin Brodsky --- include/linux/mm.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/linux/mm.h b/include/linux/mm.h index d9371d992033..4880cb7a4cb9 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -34,6 +34,7 @@ #include #include #include +#include =20 struct mempolicy; struct anon_vma; @@ -2979,6 +2980,8 @@ static inline bool __pagetable_ctor(struct ptdesc *pt= desc) =20 __folio_set_pgtable(folio); lruvec_stat_add_folio(folio, NR_PAGETABLE); + if (kpkeys_protect_pgtable_memory(folio)) + return false; return true; } =20 @@ -2989,6 +2992,7 @@ static inline void pagetable_dtor(struct ptdesc *ptde= sc) ptlock_free(ptdesc); __folio_clear_pgtable(folio); lruvec_stat_sub_folio(folio, NR_PAGETABLE); + kpkeys_unprotect_pgtable_memory(folio); } =20 static inline void pagetable_dtor_free(struct ptdesc *ptdesc) --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id ECDDC2EA142; Fri, 15 Aug 2025 08:56:34 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248196; cv=none; b=IASHqtFmY7OluAs6+nnZsi1XUEAxUskEUfgPIHneV5+0RsylBKUuSSLRxrwDc+IOz46zhbysA2CY0u92YoZ2UEEWAXirSB7dr8R+lDi78jDf1Hx3OLycrEbLMhmnRfM2+iRnENVmX6SsBOe9lZGlvJgTwbEiq9zYXrsq7/NqRb8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248196; c=relaxed/simple; bh=Pp53llZfJy7BQH6YugsN1HLR+mGCMps3QxoJJ+hk3i4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=LutMjOlzTiBSKYKioW8pxVO7Ju61/FH7L7l7SwDggnPgpDu1nQdec7URymWg6G2bJG0E2oPMxT1C3rCDjZRh4mhknntJAUy6WAglTZmasmPv/+2It8OCmt0XUZjnZUp6lJgcPKHKpxMWZKTez6dDbigPCSNxZd3w0h6eQ8sYXpE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 58492497; Fri, 15 Aug 2025 01:56:26 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id D5F693F63F; Fri, 15 Aug 2025 01:56:29 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 14/18] arm64: kpkeys: Support KPKEYS_LVL_PGTABLES Date: Fri, 15 Aug 2025 09:55:08 +0100 Message-ID: <20250815085512.2182322-15-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Enable RW access to KPKEYS_PKEY_PGTABLES (used to map page table pages) if switching to KPKEYS_LVL_PGTABLES, otherwise only grant RO access. Signed-off-by: Kevin Brodsky --- arch/arm64/include/asm/kpkeys.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kpkeys.h b/arch/arm64/include/asm/kpkey= s.h index 79ae33388088..64d6e22740ec 100644 --- a/arch/arm64/include/asm/kpkeys.h +++ b/arch/arm64/include/asm/kpkeys.h @@ -12,7 +12,8 @@ * Equivalent to por_set_kpkeys_level(0, KPKEYS_LVL_DEFAULT), but can also= be * used in assembly. */ -#define POR_EL1_INIT POR_ELx_PERM_PREP(KPKEYS_PKEY_DEFAULT, POE_RWX) +#define POR_EL1_INIT (POR_ELx_PERM_PREP(KPKEYS_PKEY_DEFAULT, POE_RWX) | \ + POR_ELx_PERM_PREP(KPKEYS_PKEY_PGTABLES, POE_R)) =20 #ifndef __ASSEMBLY__ =20 @@ -26,6 +27,8 @@ static inline bool arch_kpkeys_enabled(void) static inline u64 por_set_kpkeys_level(u64 por, int level) { por =3D por_elx_set_pkey_perms(por, KPKEYS_PKEY_DEFAULT, POE_RWX); + por =3D por_elx_set_pkey_perms(por, KPKEYS_PKEY_PGTABLES, + level =3D=3D KPKEYS_LVL_PGTABLES ? POE_RW : POE_R); =20 return por; } --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id A18902F5460; Fri, 15 Aug 2025 08:56:39 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248201; cv=none; b=KcxZwCA19BMS1XwfAWb5603S12f6u84ewFwbjB2tlY28Pq5tLUDixDUJ5Q0b4dRs7EhOXxF0ZJ7Bj0FwOeRd/Ivz6UX6mo0LaMBTcQQ/uuVZmnQ8CavVCVaAJN04l/fCAOMypZY7bpMQzEe99nDWRzMeeQt7Z0FltloAHO9Ma0E= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248201; c=relaxed/simple; bh=6oYYOMV2lXb1C0hqEreglgTFkhcTGJwexsDlWJHyrnA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=qbSShfv2PJqfgB47WUeTUOiyQ3FuMUrtyPunVgJjs1fUuJs2idvb7oz3yp95VfqO2B5JCEpkO+Mpj4Un3q0Z9/FFMOTV9KLZIxWbUQ5EsvaKHb9YQodQbrJtQaWw40mNjdk489K5yGYrERhVwknWT+nWzsxkSgOLe0dqyjecyJE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 1963E497; Fri, 15 Aug 2025 01:56:31 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id B32173F63F; Fri, 15 Aug 2025 01:56:34 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 15/18] arm64: mm: Guard page table writes with kpkeys Date: Fri, 15 Aug 2025 09:55:09 +0100 Message-ID: <20250815085512.2182322-16-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" When CONFIG_KPKEYS_HARDENED_PGTABLES is enabled, page tables (both user and kernel) are mapped with a privileged pkey in the linear mapping. As a result, they can only be written at an elevated kpkeys level. Introduce a kpkeys guard that sets POR_EL1 appropriately to allow writing to page tables, and use this guard wherever necessary. The scope is kept as small as possible, so that POR_EL1 is quickly reset to its default value. Where atomics are involved, the guard's scope encompasses the whole loop to avoid switching POR_EL1 unnecessarily. This patch is a no-op if CONFIG_KPKEYS_HARDENED_PGTABLES is disabled (default). Signed-off-by: Kevin Brodsky --- arch/arm64/include/asm/pgtable.h | 22 +++++++++++++++++++++- arch/arm64/mm/fault.c | 2 ++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgta= ble.h index abd2dee416b3..1694fb839854 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -39,6 +39,14 @@ #include #include #include +#include + +#ifdef CONFIG_KPKEYS_HARDENED_PGTABLES +KPKEYS_GUARD_COND(kpkeys_hardened_pgtables, KPKEYS_LVL_PGTABLES, + kpkeys_hardened_pgtables_enabled()) +#else +KPKEYS_GUARD_NOOP(kpkeys_hardened_pgtables) +#endif =20 static inline void emit_pte_barriers(void) { @@ -390,6 +398,7 @@ static inline pte_t pte_clear_uffd_wp(pte_t pte) =20 static inline void __set_pte_nosync(pte_t *ptep, pte_t pte) { + guard(kpkeys_hardened_pgtables)(); WRITE_ONCE(*ptep, pte); } =20 @@ -858,6 +867,7 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) } #endif /* __PAGETABLE_PMD_FOLDED */ =20 + guard(kpkeys_hardened_pgtables)(); WRITE_ONCE(*pmdp, pmd); =20 if (pmd_valid(pmd)) @@ -918,6 +928,7 @@ static inline void set_pud(pud_t *pudp, pud_t pud) return; } =20 + guard(kpkeys_hardened_pgtables)(); WRITE_ONCE(*pudp, pud); =20 if (pud_valid(pud)) @@ -999,6 +1010,7 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) return; } =20 + guard(kpkeys_hardened_pgtables)(); WRITE_ONCE(*p4dp, p4d); queue_pte_barriers(); } @@ -1127,6 +1139,7 @@ static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) return; } =20 + guard(kpkeys_hardened_pgtables)(); WRITE_ONCE(*pgdp, pgd); queue_pte_barriers(); } @@ -1316,6 +1329,7 @@ static inline int __ptep_test_and_clear_young(struct = vm_area_struct *vma, { pte_t old_pte, pte; =20 + guard(kpkeys_hardened_pgtables)(); pte =3D __ptep_get(ptep); do { old_pte =3D pte; @@ -1363,7 +1377,10 @@ static inline pte_t __ptep_get_and_clear_anysz(struc= t mm_struct *mm, pte_t *ptep, unsigned long pgsize) { - pte_t pte =3D __pte(xchg_relaxed(&pte_val(*ptep), 0)); + pte_t pte; + + scoped_guard(kpkeys_hardened_pgtables) + pte =3D __pte(xchg_relaxed(&pte_val(*ptep), 0)); =20 switch (pgsize) { case PAGE_SIZE: @@ -1436,6 +1453,7 @@ static inline void ___ptep_set_wrprotect(struct mm_st= ruct *mm, { pte_t old_pte; =20 + guard(kpkeys_hardened_pgtables)(); do { old_pte =3D pte; pte =3D pte_wrprotect(pte); @@ -1469,6 +1487,7 @@ static inline void __clear_young_dirty_pte(struct vm_= area_struct *vma, { pte_t old_pte; =20 + guard(kpkeys_hardened_pgtables)(); do { old_pte =3D pte; =20 @@ -1516,6 +1535,7 @@ static inline pmd_t pmdp_establish(struct vm_area_str= uct *vma, unsigned long address, pmd_t *pmdp, pmd_t pmd) { page_table_check_pmd_set(vma->vm_mm, pmdp, pmd); + guard(kpkeys_hardened_pgtables)(); return __pmd(xchg_relaxed(&pmd_val(*pmdp), pmd_val(pmd))); } #endif diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index d816ff44faff..c4ab361bba72 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -214,6 +214,8 @@ int __ptep_set_access_flags(struct vm_area_struct *vma, if (pte_same(pte, entry)) return 0; =20 + guard(kpkeys_hardened_pgtables)(); + /* only preserve the access flags and write permission */ pte_val(entry) &=3D PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY; =20 --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 1D5D42BDC21; Fri, 15 Aug 2025 08:56:44 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248207; cv=none; b=OYQ5PYNlwJugZ/IizIbBoa0SnIWLBPr+14Y7a5rwcN5Sw3ivZI8bPUGDJgu1I9EA/sICPB2RakC7HgboOneEyVofrsXgWOIKuuLHRZesUKq/J31rHK5OD1YSIqenXLlLMWawm/jPmR80Jw5698W+6kfLHi2JWEzYhrH27oWP1tg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248207; c=relaxed/simple; bh=gZc01VqmLu2SeIXQ3oXUtQLV6w2AHHzW2S/UA9eoVuQ=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=pM7m0WB0p4tCH04W+88eSEHyKKv7FEYyNE0disvqdkSUAtLE6SaPWX4RJOKhS7DW62H2Pf5xwz/QiygeVjeU80RpD3rChCsdCTTEgOsjc688Mo2WPCcnsQwGFub1UZcJAlwemaIpoZp3j3gID2pCqcU4x7a0Mt+Z0F+e1BxGzvQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 4BD21497; Fri, 15 Aug 2025 01:56:36 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 900D83F63F; Fri, 15 Aug 2025 01:56:39 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 16/18] arm64: Enable kpkeys_hardened_pgtables support Date: Fri, 15 Aug 2025 09:55:10 +0100 Message-ID: <20250815085512.2182322-17-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" kpkeys_hardened_pgtables should be enabled as early as possible (if selected). It does however require kpkeys being available, which means on arm64 POE being detected and enabled. POE is a boot feature, so calling kpkeys_hardened_pgtables_enable() just after setup_boot_cpu_features() in smp_prepare_boot_cpu() is the best we can do. With that done, all the bits are in place and we can advertise support for kpkeys_hardened_pgtables by selecting ARCH_HAS_KPKEYS_HARDENED_PGTABLES if ARM64_POE is enabled. Signed-off-by: Kevin Brodsky --- arch/arm64/Kconfig | 1 + arch/arm64/kernel/smp.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 88b544244829..6c77f446ab09 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2188,6 +2188,7 @@ config ARM64_POE select ARCH_USES_HIGH_VMA_FLAGS select ARCH_HAS_PKEYS select ARCH_HAS_KPKEYS + select ARCH_HAS_KPKEYS_HARDENED_PGTABLES help The Permission Overlay Extension is used to implement Memory Protection Keys. Memory Protection Keys provides a mechanism for diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 68cea3a4a35c..04ce6b2fd884 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -35,6 +35,7 @@ #include #include #include +#include =20 #include #include @@ -460,6 +461,7 @@ void __init smp_prepare_boot_cpu(void) if (system_uses_irq_prio_masking()) init_gic_priority_masking(); =20 + kpkeys_hardened_pgtables_enable(); kasan_init_hw_tags(); /* Init percpu seeds for random tags after cpus are set up. */ kasan_init_sw_tags(); --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id EB6332BDC21; Fri, 15 Aug 2025 08:56:49 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248211; cv=none; b=ag8XtarJKuDvZK/176wATB9BnEwVTvZpzYIvm2xPO7uOgxG5ja29Zzr3MTrERLVrXI8bvtw3VQu+6g/HWs6YKNyAXQcJAz8FXltBx8YHr49vqOYPmF+hEpM6EhQ94oKChgGkImbPGHBmN2MbVi9HsETW/Yj28jyGfP7jkXzBsGA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248211; c=relaxed/simple; bh=6BdOHuGR84+0veFfdcaAbFAWk3TWkmEjJXfe4lBxfqM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=SODPuaZpg1msErTIgQQVurPaPsHHjWsb0z5b549wOjIxAqEtYRP0iBb4qmjBKVjTBusYnGxePv6s7Q/S5NQZthpySPqqaAe5ia8isyKvb6Zs0IUXyi3BoMc78F6w78cablZsNwq4Ww3UgDi2rHm+SzcFu+NoikVwPnrca34BUz4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 2900D497; Fri, 15 Aug 2025 01:56:41 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id C52313F63F; Fri, 15 Aug 2025 01:56:44 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 17/18] mm: Add basic tests for kpkeys_hardened_pgtables Date: Fri, 15 Aug 2025 09:55:11 +0100 Message-ID: <20250815085512.2182322-18-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Add basic tests for the kpkeys_hardened_pgtables feature: try to perform a direct write to kernel and user page table entries and ensure it fails. Signed-off-by: Kevin Brodsky --- mm/Makefile | 1 + mm/tests/kpkeys_hardened_pgtables_kunit.c | 106 ++++++++++++++++++++++ security/Kconfig.hardening | 12 +++ 3 files changed, 119 insertions(+) create mode 100644 mm/tests/kpkeys_hardened_pgtables_kunit.c diff --git a/mm/Makefile b/mm/Makefile index 10848df0ca85..b1e6cf7f753c 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -148,3 +148,4 @@ obj-$(CONFIG_EXECMEM) +=3D execmem.o obj-$(CONFIG_TMPFS_QUOTA) +=3D shmem_quota.o obj-$(CONFIG_PT_RECLAIM) +=3D pt_reclaim.o obj-$(CONFIG_KPKEYS_HARDENED_PGTABLES) +=3D kpkeys_hardened_pgtables.o +obj-$(CONFIG_KPKEYS_HARDENED_PGTABLES_KUNIT_TEST) +=3D tests/kpkeys_harden= ed_pgtables_kunit.o diff --git a/mm/tests/kpkeys_hardened_pgtables_kunit.c b/mm/tests/kpkeys_ha= rdened_pgtables_kunit.c new file mode 100644 index 000000000000..3d916f0719d0 --- /dev/null +++ b/mm/tests/kpkeys_hardened_pgtables_kunit.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include + +KUNIT_DEFINE_ACTION_WRAPPER(vfree_wrapper, vfree, const void *); + +static inline pte_t *get_kernel_pte(unsigned long addr) +{ + pmd_t *pmdp =3D pmd_off_k(addr); + + if (!pmdp || pmd_leaf(*pmdp)) + return NULL; + + return pte_offset_kernel(pmdp, addr); +} + +static void write_linear_map_pte(struct kunit *test) +{ + pte_t *ptep; + pte_t pte; + int ret; + + if (!arch_kpkeys_enabled()) + kunit_skip(test, "kpkeys are not supported"); + + /* + * The choice of address is mostly arbitrary - we just need something + * that is PTE-mapped, such as a global variable. + */ + ptep =3D get_kernel_pte((unsigned long)&init_mm); + KUNIT_ASSERT_NOT_NULL_MSG(test, ptep, "Failed to get PTE"); + + pte =3D ptep_get(ptep); + pte =3D set_pte_bit(pte, __pgprot(PTE_WRITE)); + ret =3D copy_to_kernel_nofault(ptep, &pte, sizeof(pte)); + KUNIT_EXPECT_EQ_MSG(test, ret, -EFAULT, + "Direct PTE write wasn't prevented"); +} + +static void write_kernel_vmalloc_pte(struct kunit *test) +{ + void *mem; + pte_t *ptep; + pte_t pte; + int ret; + + if (!arch_kpkeys_enabled()) + kunit_skip(test, "kpkeys are not supported"); + + mem =3D vmalloc(PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, mem); + ret =3D kunit_add_action_or_reset(test, vfree_wrapper, mem); + KUNIT_ASSERT_EQ(test, ret, 0); + + ptep =3D get_kernel_pte((unsigned long)mem); + KUNIT_ASSERT_NOT_NULL_MSG(test, ptep, "Failed to get PTE"); + + pte =3D ptep_get(ptep); + pte =3D set_pte_bit(pte, __pgprot(PTE_WRITE)); + ret =3D copy_to_kernel_nofault(ptep, &pte, sizeof(pte)); + KUNIT_EXPECT_EQ_MSG(test, ret, -EFAULT, + "Direct PTE write wasn't prevented"); +} + +static void write_user_pmd(struct kunit *test) +{ + pmd_t *pmdp; + pmd_t pmd; + unsigned long uaddr; + int ret; + + if (!arch_kpkeys_enabled()) + kunit_skip(test, "kpkeys are not supported"); + + uaddr =3D kunit_vm_mmap(test, NULL, 0, PAGE_SIZE, PROT_READ, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_POPULATE, 0); + KUNIT_ASSERT_NE_MSG(test, uaddr, 0, "Could not create userspace mm"); + + /* We passed MAP_POPULATE so a PMD should already be allocated */ + pmdp =3D pmd_off(current->mm, uaddr); + KUNIT_ASSERT_NOT_NULL_MSG(test, pmdp, "Failed to get PMD"); + + pmd =3D pmdp_get(pmdp); + pmd =3D set_pmd_bit(pmd, __pgprot(PROT_SECT_NORMAL)); + ret =3D copy_to_kernel_nofault(pmdp, &pmd, sizeof(pmd)); + KUNIT_EXPECT_EQ_MSG(test, ret, -EFAULT, + "Direct PMD write wasn't prevented"); +} + +static struct kunit_case kpkeys_hardened_pgtables_test_cases[] =3D { + KUNIT_CASE(write_linear_map_pte), + KUNIT_CASE(write_kernel_vmalloc_pte), + KUNIT_CASE(write_user_pmd), + {} +}; + +static struct kunit_suite kpkeys_hardened_pgtables_test_suite =3D { + .name =3D "Hardened pgtables using kpkeys", + .test_cases =3D kpkeys_hardened_pgtables_test_cases, +}; +kunit_test_suite(kpkeys_hardened_pgtables_test_suite); + +MODULE_DESCRIPTION("Tests for the kpkeys_hardened_pgtables feature"); +MODULE_LICENSE("GPL"); diff --git a/security/Kconfig.hardening b/security/Kconfig.hardening index 41b7530530b7..653663008096 100644 --- a/security/Kconfig.hardening +++ b/security/Kconfig.hardening @@ -277,6 +277,18 @@ config KPKEYS_HARDENED_PGTABLES This option has no effect if the system does not support kernel pkeys. =20 +config KPKEYS_HARDENED_PGTABLES_KUNIT_TEST + tristate "KUnit tests for kpkeys_hardened_pgtables" if !KUNIT_ALL_TESTS + depends on KPKEYS_HARDENED_PGTABLES + depends on KUNIT + default KUNIT_ALL_TESTS + help + Enable this option to check that the kpkeys_hardened_pgtables feature + functions as intended, i.e. prevents arbitrary writes to user and + kernel page tables. + + If unsure, say N. + endmenu =20 config CC_HAS_RANDSTRUCT --=20 2.47.0 From nobody Sat Oct 4 14:13:12 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id CBFE12C2AA2; Fri, 15 Aug 2025 08:56:54 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248216; cv=none; b=Wvp7NiXtplteeHIM25OPLMLRy9CmiYIqnZ7zuyY1Yk9V6PluvhH+4X5epYbOreGvKgCIE4KFur0bt5Vu4TfnIVUxlKf7wIX8j4FwUIebHqK3uIDNKzQSye2T34Z7UEfm4ynfhqclbW7DExw2UqaER+VEDxTDOXfb9QO/xuHFTFI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755248216; c=relaxed/simple; bh=wrO/Sqh4R4Wsq/c46G/Xd9F9F2nr7GeOkIWqlY2LDrU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=MuaIBvjaYIj4j/PKlLaXDnSpCR6HiDdglvQmDBCQF91fzfFTgDkCruBY+yfjfzMOd3Ft+gBOoN5zeSTgtfpqMpoLrXRpke70E9TO+LIFx9NKtvWt3eO4YrHGNya5pYL08yssbkrnAQ44cNpyEHQjNx8jlLKvVeNDcT/Xjb9cCwc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 0891F497; Fri, 15 Aug 2025 01:56:46 -0700 (PDT) Received: from e123572-lin.arm.com (e123572-lin.cambridge.arm.com [10.1.194.54]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id A20AA3F63F; Fri, 15 Aug 2025 01:56:49 -0700 (PDT) From: Kevin Brodsky To: linux-hardening@vger.kernel.org Cc: linux-kernel@vger.kernel.org, Kevin Brodsky , Andrew Morton , Andy Lutomirski , Catalin Marinas , Dave Hansen , David Hildenbrand , Ira Weiny , Jann Horn , Jeff Xu , Joey Gouly , Kees Cook , Linus Walleij , Lorenzo Stoakes , Marc Zyngier , Mark Brown , Matthew Wilcox , Maxwell Bland , "Mike Rapoport (IBM)" , Peter Zijlstra , Pierre Langlois , Quentin Perret , Rick Edgecombe , Ryan Roberts , Thomas Gleixner , Vlastimil Babka , Will Deacon , linux-arm-kernel@lists.infradead.org, linux-mm@kvack.org, x86@kernel.org Subject: [RFC PATCH v5 18/18] arm64: mm: Batch kpkeys level switches Date: Fri, 15 Aug 2025 09:55:12 +0100 Message-ID: <20250815085512.2182322-19-kevin.brodsky@arm.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20250815085512.2182322-1-kevin.brodsky@arm.com> References: <20250815085512.2182322-1-kevin.brodsky@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The kpkeys_hardened_pgtables feature currently switches kpkeys level in every helper that writes to page tables, such as set_pte(). With kpkeys implemented using POE, this entails a pair of ISBs whenever such helper is called. A simple way to reduce this overhead is to make use of the lazy_mmu mode, which has recently been adopted on arm64 to batch barriers (DSB/ISB) when updating kernel pgtables [1]. Reusing the TIF_LAZY_MMU flag introduced by this series, we amend the kpkeys_hardened_pgtables guard so that no level switch (i.e. POR_EL1 update) is issued while that flag is set. Instead, we switch to KPKEYS_LVL_PGTABLES when entering lazy_mmu mode, and restore the previous level when exiting it. The optimisation is disabled while in interrupt as POR_EL1 is reset on exception entry, i.e. switching is not batched in that case. Restoring the previous kpkeys level requires storing the original value of POR_EL1 somewhere. This is a full 64-bit value so we cannot simply use a TIF flag, but since lazy_mmu sections cannot nest, some sort of thread-local variable would do the trick. There is no straightforward way to reuse current->thread.por_el1 for that purpose - this is where the current value of POR_EL1 is stored on a context switch, i.e. the value corresponding to KPKEYS_LVL_PGTABLES inside a lazy_mmu section. Instead, we add a new member to thread_struct to hold that value temporarily. This isn't optimal as that member is unused outside of lazy_mmu sections, but it is the simplest option. A further optimisation this patch makes is to merge the ISBs when exiting lazy_mmu mode. That is, if an ISB is going to be issued by emit_pte_barriers() because kernel pgtables were modified in the lazy_mmu section, we skip the ISB after restoring POR_EL1. This is done by checking TIF_LAZY_MMU_PENDING and ensuring that POR_EL1 is restored before emit_pte_barriers() is called. [1] https://lore.kernel.org/all/20250422081822.1836315-12-ryan.roberts@arm.= com/ Signed-off-by: Kevin Brodsky --- Unfortunately lazy_mmu sections can in fact nest under certain circumstances [2], which means that storing the original value of POR_EL1 in thread_struct is not always safe. I am working on modifying the lazy_mmu API to handle nesting gracefully, which should also help with restoring POR_EL1 without using thread_struct. See also the discussion in [3]. [2] https://lore.kernel.org/all/20250512150333.5589-1-ryan.roberts@arm.com/ [3] https://lore.kernel.org/all/20250606135654.178300-1-ryan.roberts@arm.co= m/t/#u arch/arm64/include/asm/pgtable.h | 37 +++++++++++++++++++++++++++++- arch/arm64/include/asm/processor.h | 1 + 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgta= ble.h index 1694fb839854..35d15b9722e4 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -43,11 +43,40 @@ =20 #ifdef CONFIG_KPKEYS_HARDENED_PGTABLES KPKEYS_GUARD_COND(kpkeys_hardened_pgtables, KPKEYS_LVL_PGTABLES, - kpkeys_hardened_pgtables_enabled()) + kpkeys_hardened_pgtables_enabled() && + (in_interrupt() || !test_thread_flag(TIF_LAZY_MMU))) #else KPKEYS_GUARD_NOOP(kpkeys_hardened_pgtables) #endif =20 +static void kpkeys_lazy_mmu_enter(void) +{ + if (!kpkeys_hardened_pgtables_enabled()) + return; + + current->thread.por_el1_lazy_mmu =3D kpkeys_set_level(KPKEYS_LVL_PGTABLES= ); +} + +static void kpkeys_lazy_mmu_exit(void) +{ + u64 saved_por_el1; + + if (!kpkeys_hardened_pgtables_enabled()) + return; + + saved_por_el1 =3D current->thread.por_el1_lazy_mmu; + + /* + * We skip any barrier if TIF_LAZY_MMU_PENDING is set: + * emit_pte_barriers() will issue an ISB just after this function + * returns. + */ + if (test_thread_flag(TIF_LAZY_MMU_PENDING)) + __kpkeys_set_pkey_reg_nosync(saved_por_el1); + else + arch_kpkeys_restore_pkey_reg(saved_por_el1); +} + static inline void emit_pte_barriers(void) { /* @@ -107,6 +136,7 @@ static inline void arch_enter_lazy_mmu_mode(void) return; =20 set_thread_flag(TIF_LAZY_MMU); + kpkeys_lazy_mmu_enter(); } =20 static inline void arch_flush_lazy_mmu_mode(void) @@ -123,6 +153,11 @@ static inline void arch_leave_lazy_mmu_mode(void) if (in_interrupt()) return; =20 + /* + * The ordering should be preserved to allow kpkeys_lazy_mmu_exit() + * to skip any barrier when TIF_LAZY_MMU_PENDING is set. + */ + kpkeys_lazy_mmu_exit(); arch_flush_lazy_mmu_mode(); clear_thread_flag(TIF_LAZY_MMU); } diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/pr= ocessor.h index 9340e94a27f6..7b20eedfe2fe 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -188,6 +188,7 @@ struct thread_struct { u64 tpidr2_el0; u64 por_el0; u64 por_el1; + u64 por_el1_lazy_mmu; #ifdef CONFIG_ARM64_GCS unsigned int gcs_el0_mode; unsigned int gcs_el0_locked; --=20 2.47.0