From nobody Sun Dec 14 06:21:57 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id B3F5BC76195 for ; Wed, 22 Mar 2023 17:04:54 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S230362AbjCVREx (ORCPT ); Wed, 22 Mar 2023 13:04:53 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:59410 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S231436AbjCVREb (ORCPT ); Wed, 22 Mar 2023 13:04:31 -0400 Received: from galois.linutronix.de (Galois.linutronix.de [IPv6:2a0a:51c0:0:12e:550::1]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 3941C3E1CF; Wed, 22 Mar 2023 10:03:50 -0700 (PDT) Date: Wed, 22 Mar 2023 17:03:07 -0000 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linutronix.de; s=2020; t=1679504588; h=from:from:sender:sender:reply-to:reply-to:subject:subject:date:date: message-id:message-id:to:to:cc:cc:mime-version:mime-version: content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=EM2B2gXKteL/cc8JOZxRU68arE6jgAKqClUGr7Cvy5c=; b=Fy7ISBw1RO3mtEOQOnH5tW8nhbLisBLFTkdmgeYnDILlVDQ0/CsCd0pRbIN8Q4dVGNq/4M pXsJW0wCwKzszDPWEp6LMs3fjzjuYf0IPUlbiL2J3I7fxfmwHFjRqnM07rGKTVCbZMB2pU V9MG0L9se1dy6gjOYSeNAPDV/pYgSHI7BfUsnv8z9qKrhCNZ/91zbTK8CiCN2g8csHVZ6f HzWsg6k0tKXdqGLCIRGfDdBz08GvnHMqX/2+cMKR9HFXVv5+5kIeGxlpVJPBWkFUi/WB5w wuFCa5dRIt+KLBqazrf311XDKDtqXnQDCa5Nu6fCBM4ynvtCaJP1p3QkUUaV4Q== DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed; d=linutronix.de; s=2020e; t=1679504588; h=from:from:sender:sender:reply-to:reply-to:subject:subject:date:date: message-id:message-id:to:to:cc:cc:mime-version:mime-version: content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=EM2B2gXKteL/cc8JOZxRU68arE6jgAKqClUGr7Cvy5c=; b=ov1gCtiqPEIyExIlS+VNEqUhyZD5CHh889vj39Je0Yq7yV4qNdSXcu8yP7cKJjgR0/7wPb gSpnPoh3enh3WABg== From: "tip-bot2 for Kirill A. Shutemov" Sender: tip-bot2@linutronix.de Reply-to: linux-kernel@vger.kernel.org To: linux-tip-commits@vger.kernel.org Subject: [tip: x86/mm] x86/uaccess: Provide untagged_addr() and remove tags before address check Cc: "Kirill A. Shutemov" , Dave Hansen , "Peter Zijlstra (Intel)" , Alexander Potapenko , x86@kernel.org, linux-kernel@vger.kernel.org MIME-Version: 1.0 Message-ID: <167950458759.5837.3318176853736322830.tip-bot2@tip-bot2> Robot-ID: Robot-Unsubscribe: Contact to get blacklisted from these emails Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org The following commit has been merged into the x86/mm branch of tip: Commit-ID: 74c228d20a51ddb1354409fdbed7b72427339d7b Gitweb: https://git.kernel.org/tip/74c228d20a51ddb1354409fdbed7b7242= 7339d7b Author: Kirill A. Shutemov AuthorDate: Sun, 12 Mar 2023 14:26:01 +03:00 Committer: Dave Hansen CommitterDate: Thu, 16 Mar 2023 13:08:39 -07:00 x86/uaccess: Provide untagged_addr() and remove tags before address check untagged_addr() is a helper used by the core-mm to strip tag bits and get the address to the canonical shape based on rules of the current thread. It only handles userspace addresses. The untagging mask is stored in per-CPU variable and set on context switching to the task. The tags must not be included into check whether it's okay to access the userspace address. Strip tags in access_ok(). Signed-off-by: Kirill A. Shutemov Signed-off-by: Dave Hansen Acked-by: Peter Zijlstra (Intel) Tested-by: Alexander Potapenko Link: https://lore.kernel.org/all/20230312112612.31869-7-kirill.shutemov%40= linux.intel.com --- arch/x86/include/asm/mmu.h | 3 ++- arch/x86/include/asm/mmu_context.h | 11 ++++++++- arch/x86/include/asm/tlbflush.h | 10 +++++++- arch/x86/include/asm/uaccess.h | 39 +++++++++++++++++++++++++++-- arch/x86/kernel/process.c | 3 ++- arch/x86/mm/init.c | 5 ++++- 6 files changed, 69 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/mmu.h b/arch/x86/include/asm/mmu.h index 22fc9fb..9cac8c4 100644 --- a/arch/x86/include/asm/mmu.h +++ b/arch/x86/include/asm/mmu.h @@ -45,6 +45,9 @@ typedef struct { #ifdef CONFIG_ADDRESS_MASKING /* Active LAM mode: X86_CR3_LAM_U48 or X86_CR3_LAM_U57 or 0 (disabled) */ unsigned long lam_cr3_mask; + + /* Significant bits of the virtual address. Excludes tag bits. */ + u64 untag_mask; #endif =20 struct mutex lock; diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_= context.h index 0295c38..eb1387a 100644 --- a/arch/x86/include/asm/mmu_context.h +++ b/arch/x86/include/asm/mmu_context.h @@ -101,6 +101,12 @@ static inline unsigned long mm_lam_cr3_mask(struct mm_= struct *mm) static inline void dup_lam(struct mm_struct *oldmm, struct mm_struct *mm) { mm->context.lam_cr3_mask =3D oldmm->context.lam_cr3_mask; + mm->context.untag_mask =3D oldmm->context.untag_mask; +} + +static inline void mm_reset_untag_mask(struct mm_struct *mm) +{ + mm->context.untag_mask =3D -1UL; } =20 #else @@ -113,6 +119,10 @@ static inline unsigned long mm_lam_cr3_mask(struct mm_= struct *mm) static inline void dup_lam(struct mm_struct *oldmm, struct mm_struct *mm) { } + +static inline void mm_reset_untag_mask(struct mm_struct *mm) +{ +} #endif =20 #define enter_lazy_tlb enter_lazy_tlb @@ -139,6 +149,7 @@ static inline int init_new_context(struct task_struct *= tsk, mm->context.execute_only_pkey =3D -1; } #endif + mm_reset_untag_mask(mm); init_new_context_ldt(mm); return 0; } diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflus= h.h index e8b47f5..75bfaa4 100644 --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h @@ -54,6 +54,15 @@ static inline void cr4_clear_bits(unsigned long mask) local_irq_restore(flags); } =20 +#ifdef CONFIG_ADDRESS_MASKING +DECLARE_PER_CPU(u64, tlbstate_untag_mask); + +static inline u64 current_untag_mask(void) +{ + return this_cpu_read(tlbstate_untag_mask); +} +#endif + #ifndef MODULE /* * 6 because 6 should be plenty and struct tlb_state will fit in two cache @@ -380,6 +389,7 @@ static inline void set_tlbstate_lam_mode(struct mm_stru= ct *mm) { this_cpu_write(cpu_tlbstate.lam, mm->context.lam_cr3_mask >> X86_CR3_LAM_U57_BIT); + this_cpu_write(tlbstate_untag_mask, mm->context.untag_mask); } =20 #else diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 1cc756e..c79ebdb 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -7,11 +7,13 @@ #include #include #include +#include #include #include #include #include #include +#include =20 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP static inline bool pagefault_disabled(void); @@ -21,6 +23,39 @@ static inline bool pagefault_disabled(void); # define WARN_ON_IN_IRQ() #endif =20 +#ifdef CONFIG_ADDRESS_MASKING +/* + * Mask out tag bits from the address. + * + * Magic with the 'sign' allows to untag userspace pointer without any bra= nches + * while leaving kernel addresses intact. + */ +static inline unsigned long __untagged_addr(unsigned long addr, + unsigned long mask) +{ + long sign =3D addr >> 63; + + addr &=3D mask | sign; + return addr; +} + +#define untagged_addr(addr) ({ \ + u64 __addr =3D (__force u64)(addr); \ + __addr =3D __untagged_addr(__addr, current_untag_mask()); \ + (__force __typeof__(addr))__addr; \ +}) + +#define untagged_addr_remote(mm, addr) ({ \ + u64 __addr =3D (__force u64)(addr); \ + mmap_assert_locked(mm); \ + __addr =3D __untagged_addr(__addr, (mm)->context.untag_mask); \ + (__force __typeof__(addr))__addr; \ +}) + +#else +#define untagged_addr(addr) (addr) +#endif + /** * access_ok - Checks if a user space pointer is valid * @addr: User space pointer to start of block to check @@ -38,10 +73,10 @@ static inline bool pagefault_disabled(void); * Return: true (nonzero) if the memory block may be valid, false (zero) * if it is definitely invalid. */ -#define access_ok(addr, size) \ +#define access_ok(addr, size) \ ({ \ WARN_ON_IN_IRQ(); \ - likely(__access_ok(addr, size)); \ + likely(__access_ok(untagged_addr(addr), size)); \ }) =20 #include diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index b650cde..bbc8c4c 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -48,6 +48,7 @@ #include #include #include +#include =20 #include "process.h" =20 @@ -368,6 +369,8 @@ void arch_setup_new_exec(void) task_clear_spec_ssb_noexec(current); speculation_ctrl_update(read_thread_flags()); } + + mm_reset_untag_mask(current->mm); } =20 #ifdef CONFIG_X86_IOPL_IOPERM diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index cb258f5..659b6c0 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -1048,6 +1048,11 @@ __visible DEFINE_PER_CPU_ALIGNED(struct tlb_state, c= pu_tlbstate) =3D { .cr4 =3D ~0UL, /* fail hard if we screw up cr4 shadow initialization */ }; =20 +#ifdef CONFIG_ADDRESS_MASKING +DEFINE_PER_CPU(u64, tlbstate_untag_mask); +EXPORT_PER_CPU_SYMBOL(tlbstate_untag_mask); +#endif + void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache) { /* entry 0 MUST be WB (hardwired to speed up translations) */