From nobody Wed Apr 15 17:26:29 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 827523E0C7A for ; Mon, 2 Mar 2026 13:56:19 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459780; cv=none; b=imvlxI6CyzI5+yGplh3bC0O/d9IPwqRPDpaaWsGblTST594+YpiqWvbltaVtZXABR1pNh7VT/os5RthywPt3ZAGZd33ys8K9Ac3byn6Cf4XX6Um/KjOpHgYP8rhELVS6F7FNm4dYx3H3XpBrJqAvNiXqTcl/zs3MRVkOmGIKA2s= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459780; c=relaxed/simple; bh=BlkBPNqrRDZstq0T6y+NnMltVILw+C86Pp9OZBtgjac=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=btKLnYegwsTDGab3AGJZpwt8fWM6hX5N8m40VmDrEdmc1sR01XoWDcUrWYjWioIO5fYmMRti80qlK0G8ACZMNPp8vh8lwGG7sjD6Q2F839q5O6brV9+zbtWwJf8DVmz6QNYz8/Z2WDfVF2/UwQkhUu2i5KsVJxDqWS5rz4hIoL8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id AE379150C; Mon, 2 Mar 2026 05:56:12 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 2F2A63F73B; Mon, 2 Mar 2026 05:56:17 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian , Jonathan Cameron Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v3 01/13] arm64: mm: Re-implement the __tlbi_level macro as a C function Date: Mon, 2 Mar 2026 13:55:48 +0000 Message-ID: <20260302135602.3716920-2-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260302135602.3716920-1-ryan.roberts@arm.com> References: <20260302135602.3716920-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" As part of efforts to reduce our reliance on complex preprocessor macros for TLB invalidation routines, convert the __tlbi_level macro to a C function for by-level TLB invalidation. Each specific tlbi level op is implemented as a C function and the appropriate function pointer is passed to __tlbi_level(). Since everything is declared inline and is statically resolvable, the compiler will convert the indirect function call to a direct inline execution. Suggested-by: Linus Torvalds Reviewed-by: Jonathan Cameron Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 67 +++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 13 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 1416e652612b7..a0e3ebe299864 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -97,19 +97,60 @@ static inline unsigned long get_trans_granule(void) =20 #define TLBI_TTL_UNKNOWN INT_MAX =20 -#define __tlbi_level(op, addr, level) do { \ - u64 arg =3D addr; \ - \ - if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && \ - level >=3D 0 && level <=3D 3) { \ - u64 ttl =3D level & 3; \ - ttl |=3D get_trans_granule() << 2; \ - arg &=3D ~TLBI_TTL_MASK; \ - arg |=3D FIELD_PREP(TLBI_TTL_MASK, ttl); \ - } \ - \ - __tlbi(op, arg); \ -} while(0) +typedef void (*tlbi_op)(u64 arg); + +static __always_inline void vae1is(u64 arg) +{ + __tlbi(vae1is, arg); +} + +static __always_inline void vae2is(u64 arg) +{ + __tlbi(vae2is, arg); +} + +static __always_inline void vale1(u64 arg) +{ + __tlbi(vale1, arg); +} + +static __always_inline void vale1is(u64 arg) +{ + __tlbi(vale1is, arg); +} + +static __always_inline void vale2is(u64 arg) +{ + __tlbi(vale2is, arg); +} + +static __always_inline void vaale1is(u64 arg) +{ + __tlbi(vaale1is, arg); +} + +static __always_inline void ipas2e1(u64 arg) +{ + __tlbi(ipas2e1, arg); +} + +static __always_inline void ipas2e1is(u64 arg) +{ + __tlbi(ipas2e1is, arg); +} + +static __always_inline void __tlbi_level(tlbi_op op, u64 addr, u32 level) +{ + u64 arg =3D addr; + + if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && level <=3D 3) { + u64 ttl =3D level | (get_trans_granule() << 2); + + FIELD_MODIFY(TLBI_TTL_MASK, &arg, ttl); + } + + op(arg); +} =20 #define __tlbi_user_level(op, arg, level) do { \ if (arm64_kernel_unmapped_at_el0()) \ --=20 2.43.0 From nobody Wed Apr 15 17:26:29 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id C01B13F23DD for ; Mon, 2 Mar 2026 13:56:21 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459783; cv=none; b=aAdv1VGC20sIu4Pl6qDEO2TVB+JCGtp9DEOSVhmnL2YRuegAxisTZIengdGXVjcH6gbNITN8CxjxI11jZIYA5EpWY9QBsDZFNRr07L9E2AaYu50FbZuQLHgxpmbfE+mPs4M6wkCUJobbb05cpYireC5zKrFQ/ZCOwdIXMjX4L+A= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459783; c=relaxed/simple; bh=vG8eFT7OKxhqphm4VuA3ncUtoTG9OcBna7We7UIyelM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=M9Z88mzv/ZTglplrWwGUmf6ZFRpsIcLBbkJr8h4H8JJSqSKUwyqYPQAr1fmsv/Vg3cg7+yTejPbuelDZpzLStIbLwMC7ish91SjAyE5w27FtYeCOQsKffkWvUuBBcH9uVWkfnc9BF+6kQ9T7AJ5430DVMhaii+2JJMML3v94Ee8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id D80601570; Mon, 2 Mar 2026 05:56:14 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 5644A3F73B; Mon, 2 Mar 2026 05:56:19 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian , Jonathan Cameron Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v3 02/13] arm64: mm: Introduce a C wrapper for by-range TLB invalidation Date: Mon, 2 Mar 2026 13:55:49 +0000 Message-ID: <20260302135602.3716920-3-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260302135602.3716920-1-ryan.roberts@arm.com> References: <20260302135602.3716920-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" As part of efforts to reduce our reliance on complex preprocessor macros for TLB invalidation routines, introduce a new C wrapper for by-range TLB invalidation which can be used instead of the __tlbi() macro and can additionally be called from C code. Each specific tlbi range op is implemented as a C function and the appropriate function pointer is passed to __tlbi_range(). Since everything is declared inline and is statically resolvable, the compiler will convert the indirect function call to a direct inline execution. Suggested-by: Linus Torvalds Reviewed-by: Jonathan Cameron Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 32 ++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index a0e3ebe299864..b3b86e5f7034e 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -468,6 +468,36 @@ static inline void arch_tlbbatch_flush(struct arch_tlb= flush_unmap_batch *batch) * operations can only span an even number of pages. We save this for l= ast to * ensure 64KB start alignment is maintained for the LPA2 case. */ +static __always_inline void rvae1is(u64 arg) +{ + __tlbi(rvae1is, arg); +} + +static __always_inline void rvale1(u64 arg) +{ + __tlbi(rvale1, arg); +} + +static __always_inline void rvale1is(u64 arg) +{ + __tlbi(rvale1is, arg); +} + +static __always_inline void rvaale1is(u64 arg) +{ + __tlbi(rvaale1is, arg); +} + +static __always_inline void ripas2e1is(u64 arg) +{ + __tlbi(ripas2e1is, arg); +} + +static __always_inline void __tlbi_range(tlbi_op op, u64 arg) +{ + op(arg); +} + #define __flush_tlb_range_op(op, start, pages, stride, \ asid, tlb_level, tlbi_user, lpa2) \ do { \ @@ -495,7 +525,7 @@ do { \ if (num >=3D 0) { \ addr =3D __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \ scale, num, tlb_level); \ - __tlbi(r##op, addr); \ + __tlbi_range(r##op, addr); \ if (tlbi_user) \ __tlbi_user(r##op, addr); \ __flush_start +=3D __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ --=20 2.43.0 From nobody Wed Apr 15 17:26:29 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id D443D3603DE for ; Mon, 2 Mar 2026 13:56:23 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459788; cv=none; b=tGNz6OaOarEriHURZvvx+Urg+8GMNer6pEtRL9RWV+05MTjQNYQN1LLvKgjqiYr5M4UNjK3yls2RXS+JSSZlnwzWktqrZywTmrhA/LiS5WRDyf6wCjeEQQEKCJs9A+ubnzq1D7ihEXKLFKFF3S9Ga5GDayTL74yXlW2OMEsaPTI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459788; c=relaxed/simple; bh=fw36dY7qf6xw1hfsUmdXFWPqE0SHuxznBj2EQTdyV7I=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=su5FylmIRtdtzkHuQWisq7Qnz8AeJv4mLALH+J+F6b5G166dVGP/X5OtqusnvTC7/K8CyrtQyjVubLUxXSbb0d2p09CYadndlwLqHhcD5CYDspIrS3Je7MZwL1pMxFI3Lx3XizQ3meZRF0B4GYJXFFHQzwAPzP8ev/c/4wHklhA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 0B6CD1655; Mon, 2 Mar 2026 05:56:17 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 7DC4E3F73B; Mon, 2 Mar 2026 05:56:21 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian , Jonathan Cameron Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v3 03/13] arm64: mm: Implicitly invalidate user ASID based on TLBI operation Date: Mon, 2 Mar 2026 13:55:50 +0000 Message-ID: <20260302135602.3716920-4-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260302135602.3716920-1-ryan.roberts@arm.com> References: <20260302135602.3716920-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" When kpti is enabled, separate ASIDs are used for userspace and kernelspace, requiring ASID-qualified TLB invalidation by virtual address to invalidate both of them. Push the logic for invalidating the two ASIDs down into the low-level tlbi-op-specific functions and remove the burden from the caller to handle the kpti-specific behaviour. Co-developed-by: Will Deacon Signed-off-by: Will Deacon Reviewed-by: Jonathan Cameron Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index b3b86e5f7034e..e586d9b71ea2d 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -102,6 +102,7 @@ typedef void (*tlbi_op)(u64 arg); static __always_inline void vae1is(u64 arg) { __tlbi(vae1is, arg); + __tlbi_user(vae1is, arg); } =20 static __always_inline void vae2is(u64 arg) @@ -112,11 +113,13 @@ static __always_inline void vae2is(u64 arg) static __always_inline void vale1(u64 arg) { __tlbi(vale1, arg); + __tlbi_user(vale1, arg); } =20 static __always_inline void vale1is(u64 arg) { __tlbi(vale1is, arg); + __tlbi_user(vale1is, arg); } =20 static __always_inline void vale2is(u64 arg) @@ -152,11 +155,6 @@ static __always_inline void __tlbi_level(tlbi_op op, u= 64 addr, u32 level) op(arg); } =20 -#define __tlbi_user_level(op, arg, level) do { \ - if (arm64_kernel_unmapped_at_el0()) \ - __tlbi_level(op, (arg | USER_ASID_FLAG), level); \ -} while (0) - /* * This macro creates a properly formatted VA operand for the TLB RANGE. T= he * value bit assignments are: @@ -444,8 +442,6 @@ static inline void arch_tlbbatch_flush(struct arch_tlbf= lush_unmap_batch *batch) * @stride: Flush granularity * @asid: The ASID of the task (0 for IPA instructions) * @tlb_level: Translation Table level hint, if known - * @tlbi_user: If 'true', call an additional __tlbi_user() - * (typically for user ASIDs). 'flase' for IPA instructions * @lpa2: If 'true', the lpa2 scheme is used as set out below * * When the CPU does not support TLB range operations, flush the TLB @@ -471,16 +467,19 @@ static inline void arch_tlbbatch_flush(struct arch_tl= bflush_unmap_batch *batch) static __always_inline void rvae1is(u64 arg) { __tlbi(rvae1is, arg); + __tlbi_user(rvae1is, arg); } =20 static __always_inline void rvale1(u64 arg) { __tlbi(rvale1, arg); + __tlbi_user(rvale1, arg); } =20 static __always_inline void rvale1is(u64 arg) { __tlbi(rvale1is, arg); + __tlbi_user(rvale1is, arg); } =20 static __always_inline void rvaale1is(u64 arg) @@ -499,7 +498,7 @@ static __always_inline void __tlbi_range(tlbi_op op, u6= 4 arg) } =20 #define __flush_tlb_range_op(op, start, pages, stride, \ - asid, tlb_level, tlbi_user, lpa2) \ + asid, tlb_level, lpa2) \ do { \ typeof(start) __flush_start =3D start; \ typeof(pages) __flush_pages =3D pages; \ @@ -514,8 +513,6 @@ do { \ (lpa2 && __flush_start !=3D ALIGN(__flush_start, SZ_64K))) { \ addr =3D __TLBI_VADDR(__flush_start, asid); \ __tlbi_level(op, addr, tlb_level); \ - if (tlbi_user) \ - __tlbi_user_level(op, addr, tlb_level); \ __flush_start +=3D stride; \ __flush_pages -=3D stride >> PAGE_SHIFT; \ continue; \ @@ -526,8 +523,6 @@ do { \ addr =3D __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \ scale, num, tlb_level); \ __tlbi_range(r##op, addr); \ - if (tlbi_user) \ - __tlbi_user(r##op, addr); \ __flush_start +=3D __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ __flush_pages -=3D __TLBI_RANGE_PAGES(num, scale);\ } \ @@ -536,7 +531,7 @@ do { \ } while (0) =20 #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ - __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false, kvm_l= pa2_is_enabled()); + __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_= enabled()); =20 static inline bool __flush_tlb_range_limit_excess(unsigned long start, unsigned long end, unsigned long pages, unsigned long stride) @@ -576,10 +571,10 @@ static inline void __flush_tlb_range_nosync(struct mm= _struct *mm, =20 if (last_level) __flush_tlb_range_op(vale1is, start, pages, stride, asid, - tlb_level, true, lpa2_is_enabled()); + tlb_level, lpa2_is_enabled()); else __flush_tlb_range_op(vae1is, start, pages, stride, asid, - tlb_level, true, lpa2_is_enabled()); + tlb_level, lpa2_is_enabled()); =20 mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); } @@ -604,7 +599,7 @@ static inline void local_flush_tlb_contpte(struct vm_ar= ea_struct *vma, dsb(nshst); asid =3D ASID(vma->vm_mm); __flush_tlb_range_op(vale1, addr, CONT_PTES, PAGE_SIZE, asid, - 3, true, lpa2_is_enabled()); + 3, lpa2_is_enabled()); mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, addr, addr + CONT_PTE_SIZE); dsb(nsh); @@ -638,7 +633,7 @@ static inline void flush_tlb_kernel_range(unsigned long= start, unsigned long end =20 dsb(ishst); __flush_tlb_range_op(vaale1is, start, pages, stride, 0, - TLBI_TTL_UNKNOWN, false, lpa2_is_enabled()); + TLBI_TTL_UNKNOWN, lpa2_is_enabled()); __tlbi_sync_s1ish(); isb(); } @@ -689,6 +684,7 @@ static inline bool huge_pmd_needs_flush(pmd_t oldpmd, p= md_t newpmd) } #define huge_pmd_needs_flush huge_pmd_needs_flush =20 +#undef __tlbi_user #endif =20 #endif --=20 2.43.0 From nobody Wed Apr 15 17:26:29 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 19367304BB3 for ; Mon, 2 Mar 2026 13:56:25 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459787; cv=none; b=DOyLpEd9TyUaDjjYb82dBFMFe/wDZ2Csp4qRNdmhZ+0v80x5FwnhQWmhbH8sbt1kgIe1qK5jQ8qkpgD5mYTfNwS35RgtE8m4+LtWXTrnKGaxMufDy+lK2qzw6iIs9mDwRK3IQ7yLZtVZtq9mEvXA/fWPtKQjpurTZMyOzf40KHg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459787; c=relaxed/simple; bh=xRCO9o4moicUEZiQ6OHjLIAhY38t25tbYpduMQqSP28=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=Q+cchB5oHaqRxD8driF+mjrU2hmzlPXQxajgIeLy20F4QKzZkgrMLIIXo7NQXKUYBqj6ii8Mn5tbQzY2yizQKf0sCDQME3g+33KBBUsmdwhcR8lK9WtD6uEC0/Io0Og4NxXDxo1GMC+jHzozM0BE5R6VbcRK8CMi5ICl5vC92YQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 5167A165C; Mon, 2 Mar 2026 05:56:19 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id A51953F73B; Mon, 2 Mar 2026 05:56:23 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian , Jonathan Cameron Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, Linu Cherian Subject: [PATCH v3 04/13] arm64: mm: Push __TLBI_VADDR() into __tlbi_level() Date: Mon, 2 Mar 2026 13:55:51 +0000 Message-ID: <20260302135602.3716920-5-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260302135602.3716920-1-ryan.roberts@arm.com> References: <20260302135602.3716920-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Will Deacon The __TLBI_VADDR() macro takes an ASID and an address and converts them into a single argument formatted correctly for a TLB invalidation instruction. Rather than have callers worry about this (especially in the case where the ASID is zero), push the macro down into __tlbi_level() via a new __tlbi_level_asid() helper. Signed-off-by: Will Deacon Reviewed-by: Linu Cherian Reviewed-by: Jonathan Cameron Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 14 ++++++++++---- arch/arm64/kernel/sys_compat.c | 2 +- arch/arm64/kvm/hyp/nvhe/mm.c | 2 +- arch/arm64/kvm/hyp/nvhe/tlb.c | 2 -- arch/arm64/kvm/hyp/pgtable.c | 4 ++-- arch/arm64/kvm/hyp/vhe/tlb.c | 2 -- 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index e586d9b71ea2d..2832305606b72 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -142,9 +142,10 @@ static __always_inline void ipas2e1is(u64 arg) __tlbi(ipas2e1is, arg); } =20 -static __always_inline void __tlbi_level(tlbi_op op, u64 addr, u32 level) +static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 le= vel, + u16 asid) { - u64 arg =3D addr; + u64 arg =3D __TLBI_VADDR(addr, asid); =20 if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && level <=3D 3) { u64 ttl =3D level | (get_trans_granule() << 2); @@ -155,6 +156,11 @@ static __always_inline void __tlbi_level(tlbi_op op, u= 64 addr, u32 level) op(arg); } =20 +static inline void __tlbi_level(tlbi_op op, u64 addr, u32 level) +{ + __tlbi_level_asid(op, addr, level, 0); +} + /* * This macro creates a properly formatted VA operand for the TLB RANGE. T= he * value bit assignments are: @@ -511,8 +517,7 @@ do { \ if (!system_supports_tlb_range() || \ __flush_pages =3D=3D 1 || \ (lpa2 && __flush_start !=3D ALIGN(__flush_start, SZ_64K))) { \ - addr =3D __TLBI_VADDR(__flush_start, asid); \ - __tlbi_level(op, addr, tlb_level); \ + __tlbi_level_asid(op, __flush_start, tlb_level, asid); \ __flush_start +=3D stride; \ __flush_pages -=3D stride >> PAGE_SHIFT; \ continue; \ @@ -685,6 +690,7 @@ static inline bool huge_pmd_needs_flush(pmd_t oldpmd, p= md_t newpmd) #define huge_pmd_needs_flush huge_pmd_needs_flush =20 #undef __tlbi_user +#undef __TLBI_VADDR #endif =20 #endif diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c index b9d4998c97efa..7e9860143add8 100644 --- a/arch/arm64/kernel/sys_compat.c +++ b/arch/arm64/kernel/sys_compat.c @@ -36,7 +36,7 @@ __do_compat_cache_op(unsigned long start, unsigned long e= nd) * The workaround requires an inner-shareable tlbi. * We pick the reserved-ASID to minimise the impact. */ - __tlbi(aside1is, __TLBI_VADDR(0, 0)); + __tlbi(aside1is, 0UL); __tlbi_sync_s1ish(); } =20 diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 218976287d3fe..4d8fcc7a3a41e 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -270,7 +270,7 @@ static void fixmap_clear_slot(struct hyp_fixmap_slot *s= lot) * https://lore.kernel.org/kvm/20221017115209.2099-1-will@kernel.org/T/#m= f10dfbaf1eaef9274c581b81c53758918c1d0f03 */ dsb(ishst); - __tlbi_level(vale2is, __TLBI_VADDR(addr, 0), level); + __tlbi_level(vale2is, addr, level); __tlbi_sync_s1ish_hyp(); isb(); } diff --git a/arch/arm64/kvm/hyp/nvhe/tlb.c b/arch/arm64/kvm/hyp/nvhe/tlb.c index 3dc1ce0d27fe6..b29140995d484 100644 --- a/arch/arm64/kvm/hyp/nvhe/tlb.c +++ b/arch/arm64/kvm/hyp/nvhe/tlb.c @@ -158,7 +158,6 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, * Instead, we invalidate Stage-2 for this IPA, and the * whole of Stage-1. Weep... */ - ipa >>=3D 12; __tlbi_level(ipas2e1is, ipa, level); =20 /* @@ -188,7 +187,6 @@ void __kvm_tlb_flush_vmid_ipa_nsh(struct kvm_s2_mmu *mm= u, * Instead, we invalidate Stage-2 for this IPA, and the * whole of Stage-1. Weep... */ - ipa >>=3D 12; __tlbi_level(ipas2e1, ipa, level); =20 /* diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 9b480f947da26..30226f2d5564a 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -490,14 +490,14 @@ static int hyp_unmap_walker(const struct kvm_pgtable_= visit_ctx *ctx, =20 kvm_clear_pte(ctx->ptep); dsb(ishst); - __tlbi_level(vae2is, __TLBI_VADDR(ctx->addr, 0), TLBI_TTL_UNKNOWN); + __tlbi_level(vae2is, ctx->addr, TLBI_TTL_UNKNOWN); } else { if (ctx->end - ctx->addr < granule) return -EINVAL; =20 kvm_clear_pte(ctx->ptep); dsb(ishst); - __tlbi_level(vale2is, __TLBI_VADDR(ctx->addr, 0), ctx->level); + __tlbi_level(vale2is, ctx->addr, ctx->level); *unmapped +=3D granule; } =20 diff --git a/arch/arm64/kvm/hyp/vhe/tlb.c b/arch/arm64/kvm/hyp/vhe/tlb.c index 35855dadfb1b3..f7b9dfe3f3a5a 100644 --- a/arch/arm64/kvm/hyp/vhe/tlb.c +++ b/arch/arm64/kvm/hyp/vhe/tlb.c @@ -104,7 +104,6 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, * Instead, we invalidate Stage-2 for this IPA, and the * whole of Stage-1. Weep... */ - ipa >>=3D 12; __tlbi_level(ipas2e1is, ipa, level); =20 /* @@ -136,7 +135,6 @@ void __kvm_tlb_flush_vmid_ipa_nsh(struct kvm_s2_mmu *mm= u, * Instead, we invalidate Stage-2 for this IPA, and the * whole of Stage-1. Weep... */ - ipa >>=3D 12; __tlbi_level(ipas2e1, ipa, level); =20 /* --=20 2.43.0 From nobody Wed Apr 15 17:26:29 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 81E7B3FB041 for ; Mon, 2 Mar 2026 13:56:28 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459789; cv=none; b=Qkn+QLS176k037HKBUc0OMU5hZs0H7b9BS/60foWpFhNIzx+035iGAK5i7bs/QEdcPvCU4aFWEei6jylWIZzUCuZVemS0iJjA6RPsTye/cMW4DV9ipk9qoZA2BOKGY3gpRwKjIK6YrdZYAU4YJWp6Hd3yBDgKwF68Q78bsjBcYs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459789; c=relaxed/simple; bh=Hzeolif7wQfK66Wk/R5ZTn5r6rEi24G9AaOm7QljhHg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=ciO/RVlE7tGuH7aV7el4Xo3VEl6qwdUfdgur1zD3/OmnEN5IfN582zBrZADw9wfxkSoVmcOnMGkhMQIN34r9e9CcLB9kH2PQCT77pTl6vvCEMnWftwJNAEeFsJ8XaYDkhvm74GucSF1c7db9sudYuN/BOg6R8QHnfsjD3KFjklM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 972C11684; Mon, 2 Mar 2026 05:56:21 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id EB0293F73B; Mon, 2 Mar 2026 05:56:25 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian , Jonathan Cameron Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, Linu Cherian Subject: [PATCH v3 05/13] arm64: mm: Inline __TLBI_VADDR_RANGE() into __tlbi_range() Date: Mon, 2 Mar 2026 13:55:52 +0000 Message-ID: <20260302135602.3716920-6-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260302135602.3716920-1-ryan.roberts@arm.com> References: <20260302135602.3716920-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Will Deacon The __TLBI_VADDR_RANGE() macro is only used in one place and isn't something that's generally useful outside of the low-level range invalidation gubbins. Inline __TLBI_VADDR_RANGE() into the __tlbi_range() function so that the macro can be removed entirely. Signed-off-by: Will Deacon Reviewed-by: Linu Cherian Reviewed-by: Jonathan Cameron Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 32 +++++++++++++------------------ 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 2832305606b72..0f81547470ea2 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -186,19 +186,6 @@ static inline void __tlbi_level(tlbi_op op, u64 addr, = u32 level) #define TLBIR_TTL_MASK GENMASK_ULL(38, 37) #define TLBIR_BADDR_MASK GENMASK_ULL(36, 0) =20 -#define __TLBI_VADDR_RANGE(baddr, asid, scale, num, ttl) \ - ({ \ - unsigned long __ta =3D 0; \ - unsigned long __ttl =3D (ttl >=3D 1 && ttl <=3D 3) ? ttl : 0; \ - __ta |=3D FIELD_PREP(TLBIR_BADDR_MASK, baddr); \ - __ta |=3D FIELD_PREP(TLBIR_TTL_MASK, __ttl); \ - __ta |=3D FIELD_PREP(TLBIR_NUM_MASK, num); \ - __ta |=3D FIELD_PREP(TLBIR_SCALE_MASK, scale); \ - __ta |=3D FIELD_PREP(TLBIR_TG_MASK, get_trans_granule()); \ - __ta |=3D FIELD_PREP(TLBIR_ASID_MASK, asid); \ - __ta; \ - }) - /* These macros are used by the TLBI RANGE feature. */ #define __TLBI_RANGE_PAGES(num, scale) \ ((unsigned long)((num) + 1) << (5 * (scale) + 1)) @@ -498,8 +485,19 @@ static __always_inline void ripas2e1is(u64 arg) __tlbi(ripas2e1is, arg); } =20 -static __always_inline void __tlbi_range(tlbi_op op, u64 arg) +static __always_inline void __tlbi_range(tlbi_op op, u64 addr, + u16 asid, int scale, int num, + u32 level, bool lpa2) { + u64 arg =3D 0; + + arg |=3D FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : PAGE_SHIFT)); + arg |=3D FIELD_PREP(TLBIR_TTL_MASK, level > 3 ? 0 : level); + arg |=3D FIELD_PREP(TLBIR_NUM_MASK, num); + arg |=3D FIELD_PREP(TLBIR_SCALE_MASK, scale); + arg |=3D FIELD_PREP(TLBIR_TG_MASK, get_trans_granule()); + arg |=3D FIELD_PREP(TLBIR_ASID_MASK, asid); + op(arg); } =20 @@ -510,8 +508,6 @@ do { \ typeof(pages) __flush_pages =3D pages; \ int num =3D 0; \ int scale =3D 3; \ - int shift =3D lpa2 ? 16 : PAGE_SHIFT; \ - unsigned long addr; \ \ while (__flush_pages > 0) { \ if (!system_supports_tlb_range() || \ @@ -525,9 +521,7 @@ do { \ \ num =3D __TLBI_RANGE_NUM(__flush_pages, scale); \ if (num >=3D 0) { \ - addr =3D __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \ - scale, num, tlb_level); \ - __tlbi_range(r##op, addr); \ + __tlbi_range(r##op, __flush_start, asid, scale, num, tlb_level, lpa2); \ __flush_start +=3D __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ __flush_pages -=3D __TLBI_RANGE_PAGES(num, scale);\ } \ --=20 2.43.0 From nobody Wed Apr 15 17:26:29 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 7FC5E3E5564 for ; Mon, 2 Mar 2026 13:56:30 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459791; cv=none; b=WWWyUE+CNhaAN+Z4YhOcTR53VT29sBi0V8bA3iAsgr9n0E8aZqPP91mkbRuej1ySpKzoEEKSHPmrLhsRuzlCEyLjcXp3RN4rZ3lH6TyCo/y91x7SI+lGZebTBmL6495jOm/sP0zBgpZ0DYouVLMIZMz29bSbAgHFDbgZbWryEXY= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459791; c=relaxed/simple; bh=4CNgPTNnVhJX0M8ZxhE9iZi/1mi1FZ4Xvtw1yCcCjvc=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=bHt+mbKHa7fDmlngSj5VVZYFm6+jQ/6F8dk7Tv5C5sFUarxceCV/Q9gcPT6WeZAMpFZOPnoaRyflR7OZQJteCLilTRAho1j5PDCMc6NMmjWyJmIjn46l7cSw8ypyJeFLy+YFGIfOWNVr33KSiYTa6LYEEOMzJUXTo/+Emt9SF6U= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id BEB9714BF; Mon, 2 Mar 2026 05:56:23 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 3CEA93F73B; Mon, 2 Mar 2026 05:56:28 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian , Jonathan Cameron Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v3 06/13] arm64: mm: Re-implement the __flush_tlb_range_op macro in C Date: Mon, 2 Mar 2026 13:55:53 +0000 Message-ID: <20260302135602.3716920-7-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260302135602.3716920-1-ryan.roberts@arm.com> References: <20260302135602.3716920-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The __flush_tlb_range_op() macro is horrible and has been a previous source of bugs thanks to multiple expansions of its arguments (see commit f7edb07ad7c6 ("Fix mmu notifiers for range-based invalidates")). Rewrite the thing in C. Suggested-by: Linus Torvalds Co-developed-by: Will Deacon Signed-off-by: Will Deacon Reviewed-by: Jonathan Cameron Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 84 +++++++++++++++++-------------- 1 file changed, 46 insertions(+), 38 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 0f81547470ea2..3c05afdbe3a69 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -429,12 +429,13 @@ static inline void arch_tlbbatch_flush(struct arch_tl= bflush_unmap_batch *batch) /* * __flush_tlb_range_op - Perform TLBI operation upon a range * - * @op: TLBI instruction that operates on a range (has 'r' prefix) + * @lop: TLBI level operation to perform + * @rop: TLBI range operation to perform * @start: The start address of the range * @pages: Range as the number of pages from 'start' * @stride: Flush granularity * @asid: The ASID of the task (0 for IPA instructions) - * @tlb_level: Translation Table level hint, if known + * @level: Translation Table level hint, if known * @lpa2: If 'true', the lpa2 scheme is used as set out below * * When the CPU does not support TLB range operations, flush the TLB @@ -501,36 +502,44 @@ static __always_inline void __tlbi_range(tlbi_op op, = u64 addr, op(arg); } =20 -#define __flush_tlb_range_op(op, start, pages, stride, \ - asid, tlb_level, lpa2) \ -do { \ - typeof(start) __flush_start =3D start; \ - typeof(pages) __flush_pages =3D pages; \ - int num =3D 0; \ - int scale =3D 3; \ - \ - while (__flush_pages > 0) { \ - if (!system_supports_tlb_range() || \ - __flush_pages =3D=3D 1 || \ - (lpa2 && __flush_start !=3D ALIGN(__flush_start, SZ_64K))) { \ - __tlbi_level_asid(op, __flush_start, tlb_level, asid); \ - __flush_start +=3D stride; \ - __flush_pages -=3D stride >> PAGE_SHIFT; \ - continue; \ - } \ - \ - num =3D __TLBI_RANGE_NUM(__flush_pages, scale); \ - if (num >=3D 0) { \ - __tlbi_range(r##op, __flush_start, asid, scale, num, tlb_level, lpa2); \ - __flush_start +=3D __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ - __flush_pages -=3D __TLBI_RANGE_PAGES(num, scale);\ - } \ - scale--; \ - } \ -} while (0) +static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, + u64 start, size_t pages, + u64 stride, u16 asid, + u32 level, bool lpa2) +{ + u64 addr =3D start, end =3D start + pages * PAGE_SIZE; + int scale =3D 3; + + while (addr !=3D end) { + int num; + + pages =3D (end - addr) >> PAGE_SHIFT; + + if (!system_supports_tlb_range() || pages =3D=3D 1) + goto invalidate_one; + + if (lpa2 && !IS_ALIGNED(addr, SZ_64K)) + goto invalidate_one; + + num =3D __TLBI_RANGE_NUM(pages, scale); + if (num >=3D 0) { + __tlbi_range(rop, addr, asid, scale, num, level, lpa2); + addr +=3D __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; + } + + scale--; + continue; +invalidate_one: + __tlbi_level_asid(lop, addr, level, asid); + addr +=3D stride; + } +} + +#define __flush_s1_tlb_range_op(op, start, pages, stride, asid, tlb_level)= \ + __flush_tlb_range_op(op, r##op, start, pages, stride, asid, tlb_level, lp= a2_is_enabled()) =20 #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ - __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_= enabled()); + __flush_tlb_range_op(op, r##op, start, pages, stride, 0, tlb_level, kvm_l= pa2_is_enabled()) =20 static inline bool __flush_tlb_range_limit_excess(unsigned long start, unsigned long end, unsigned long pages, unsigned long stride) @@ -569,11 +578,11 @@ static inline void __flush_tlb_range_nosync(struct mm= _struct *mm, asid =3D ASID(mm); =20 if (last_level) - __flush_tlb_range_op(vale1is, start, pages, stride, asid, - tlb_level, lpa2_is_enabled()); + __flush_s1_tlb_range_op(vale1is, start, pages, stride, + asid, tlb_level); else - __flush_tlb_range_op(vae1is, start, pages, stride, asid, - tlb_level, lpa2_is_enabled()); + __flush_s1_tlb_range_op(vae1is, start, pages, stride, + asid, tlb_level); =20 mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); } @@ -597,8 +606,7 @@ static inline void local_flush_tlb_contpte(struct vm_ar= ea_struct *vma, =20 dsb(nshst); asid =3D ASID(vma->vm_mm); - __flush_tlb_range_op(vale1, addr, CONT_PTES, PAGE_SIZE, asid, - 3, lpa2_is_enabled()); + __flush_s1_tlb_range_op(vale1, addr, CONT_PTES, PAGE_SIZE, asid, 3); mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, addr, addr + CONT_PTE_SIZE); dsb(nsh); @@ -631,8 +639,8 @@ static inline void flush_tlb_kernel_range(unsigned long= start, unsigned long end } =20 dsb(ishst); - __flush_tlb_range_op(vaale1is, start, pages, stride, 0, - TLBI_TTL_UNKNOWN, lpa2_is_enabled()); + __flush_s1_tlb_range_op(vaale1is, start, pages, stride, 0, + TLBI_TTL_UNKNOWN); __tlbi_sync_s1ish(); isb(); } --=20 2.43.0 From nobody Wed Apr 15 17:26:29 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id B994C3FB04F for ; Mon, 2 Mar 2026 13:56:32 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459793; cv=none; b=iJ7fkIgEtomvY99zGOu6ImzxY7goYYXoeaHGd/2AIf4FmNqfWtiInHFthxZMRqpLNeQlQMpc63GzK5svA/VIsDdARGGiWsMs7LFeV+qR7X2zQIjlJLYa+YpcZXqsfClPVTMwvUdeMZIba5g90sow1MCsMmuRKH7/ogYZyjNCkUE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459793; c=relaxed/simple; bh=iePEu5r3hE4sZ3Q438GrvO2I+PvCDppz8xg3Wu9e0FA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=X6x4xypazGkeow630fQ2PUnbM5xMkPL4IrZABInURRtxe6sadHfCmK1CkcX22cnfz5ivmFDkwzpa/fmonrwstIiqahSbWy7jtEAiZZx2/KI6TOByi+4YbmmMzG4ss8rkyTKSQhKXeme6mGK4VkymRuay/7B6r55CTwhIpbzX03U= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id E65FB150C; Mon, 2 Mar 2026 05:56:25 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 64A7C3F73B; Mon, 2 Mar 2026 05:56:30 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian , Jonathan Cameron Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v3 07/13] arm64: mm: Simplify __TLBI_RANGE_NUM() macro Date: Mon, 2 Mar 2026 13:55:54 +0000 Message-ID: <20260302135602.3716920-8-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260302135602.3716920-1-ryan.roberts@arm.com> References: <20260302135602.3716920-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Will Deacon Since commit e2768b798a19 ("arm64/mm: Modify range-based tlbi to decrement scale"), we don't need to clamp the 'pages' argument to fit the range for the specified 'scale' as we know that the upper bits will have been processed in a prior iteration. Drop the clamping and simplify the __TLBI_RANGE_NUM() macro. Signed-off-by: Will Deacon Reviewed-by: Ryan Roberts Reviewed-by: Dev Jain Reviewed-by: Jonathan Cameron Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 3c05afdbe3a69..fb7e541cfdfd9 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -199,11 +199,7 @@ static inline void __tlbi_level(tlbi_op op, u64 addr, = u32 level) * range. */ #define __TLBI_RANGE_NUM(pages, scale) \ - ({ \ - int __pages =3D min((pages), \ - __TLBI_RANGE_PAGES(31, (scale))); \ - (__pages >> (5 * (scale) + 1)) - 1; \ - }) + (((pages) >> (5 * (scale) + 1)) - 1) =20 #define __repeat_tlbi_sync(op, arg...) \ do { \ --=20 2.43.0 From nobody Wed Apr 15 17:26:29 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id E936A3FB042 for ; Mon, 2 Mar 2026 13:56:34 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459798; cv=none; b=qjZLlwo1tGYypOwRF/3kKPUrcOmJflBOQdBZenxHHGGceZCdxtAP1uMwcwyOeVcdoYCY8wlFIaOdNqptz9HfPQOf4Qqn+K2R/dpPyz7UsboP4qVm6H+3/El0tqZsTPEjuwH/4w21E5hXJRev2HCVlliLL3U8QZb0JHZld50iC/g= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459798; c=relaxed/simple; bh=C7wUI3dqCVdWASTjn+7LD4EnuChzjHegY79R5xOr0Us=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=D5yWZPpYGgySjLXBdBQL3Xd/LGmllzR358r5fkEYM5MMz28tnyEI26yxqU0RNWWty1lTaayOR0lXq3gsq6z+sAdrqd17a8a25DRiA8c7VBkov0vKiTM0CO3fSnM1J1+933Vg5t/6Hr5/4xiFobkXbYT8Yjfw6k3JiPUTyveqcHU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 166FB1570; Mon, 2 Mar 2026 05:56:28 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 8BB6A3F73B; Mon, 2 Mar 2026 05:56:32 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian , Jonathan Cameron Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v3 08/13] arm64: mm: Simplify __flush_tlb_range_limit_excess() Date: Mon, 2 Mar 2026 13:55:55 +0000 Message-ID: <20260302135602.3716920-9-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260302135602.3716920-1-ryan.roberts@arm.com> References: <20260302135602.3716920-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Will Deacon __flush_tlb_range_limit_excess() is unnecessarily complicated: - It takes a 'start', 'end' and 'pages' argument, whereas it only needs 'pages' (which the caller has computed from the other two arguments!). - It erroneously compares 'pages' with MAX_TLBI_RANGE_PAGES when the system doesn't support range-based invalidation but the range to be invalidated would result in fewer than MAX_DVM_OPS invalidations. Simplify the function so that it no longer takes the 'start' and 'end' arguments and only considers the MAX_TLBI_RANGE_PAGES threshold on systems that implement range-based invalidation. Signed-off-by: Will Deacon Reviewed-by: Jonathan Cameron Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index fb7e541cfdfd9..fd86647db24bb 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -537,21 +537,19 @@ static __always_inline void __flush_tlb_range_op(tlbi= _op lop, tlbi_op rop, #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ __flush_tlb_range_op(op, r##op, start, pages, stride, 0, tlb_level, kvm_l= pa2_is_enabled()) =20 -static inline bool __flush_tlb_range_limit_excess(unsigned long start, - unsigned long end, unsigned long pages, unsigned long stride) +static inline bool __flush_tlb_range_limit_excess(unsigned long pages, + unsigned long stride) { /* - * When the system does not support TLB range based flush - * operation, (MAX_DVM_OPS - 1) pages can be handled. But - * with TLB range based operation, MAX_TLBI_RANGE_PAGES - * pages can be handled. + * Assume that the worst case number of DVM ops required to flush a + * given range on a system that supports tlb-range is 20 (4 scales, 1 + * final page, 15 for alignment on LPA2 systems), which is much smaller + * than MAX_DVM_OPS. */ - if ((!system_supports_tlb_range() && - (end - start) >=3D (MAX_DVM_OPS * stride)) || - pages > MAX_TLBI_RANGE_PAGES) - return true; + if (system_supports_tlb_range()) + return pages > MAX_TLBI_RANGE_PAGES; =20 - return false; + return pages >=3D (MAX_DVM_OPS * stride) >> PAGE_SHIFT; } =20 static inline void __flush_tlb_range_nosync(struct mm_struct *mm, @@ -565,7 +563,7 @@ static inline void __flush_tlb_range_nosync(struct mm_s= truct *mm, end =3D round_up(end, stride); pages =3D (end - start) >> PAGE_SHIFT; =20 - if (__flush_tlb_range_limit_excess(start, end, pages, stride)) { + if (__flush_tlb_range_limit_excess(pages, stride)) { flush_tlb_mm(mm); return; } @@ -629,7 +627,7 @@ static inline void flush_tlb_kernel_range(unsigned long= start, unsigned long end end =3D round_up(end, stride); pages =3D (end - start) >> PAGE_SHIFT; =20 - if (__flush_tlb_range_limit_excess(start, end, pages, stride)) { + if (__flush_tlb_range_limit_excess(pages, stride)) { flush_tlb_all(); return; } --=20 2.43.0 From nobody Wed Apr 15 17:26:29 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 009CC3603DE for ; Mon, 2 Mar 2026 13:56:37 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459798; cv=none; b=TNJ/teqqxbToXlxapsFcwlOl50RRIuBQs00omaEzN4+mGu2motHwp+0sSgI33Pwwg3xWjbN4S0AFGyu3qK7gBesimzCkiG/d6pY6q5XQCqFTDLSA5bygQ250G60zdi4wTmNp1xOnAJUcVThi+2l9ekKnJpdd1iop9r3WtOhDJbw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459798; c=relaxed/simple; bh=lq0MBrGCPUaAHgyH4QL4LTUOZ8b/DmARfrB/MxgysUE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=V2dVKm/3X9qCcoimiqWqnWa5EWGqZURN1nnkX4GjYebXE4FQGsOppX05DzcwTbp/hx2ZFRPvJ66v8lzKmiV6fcEhLUeazQ2oh6wsdNSqLpyoV4CkD+LE5a3U3HRdzz8rq0JIyuu7sA7ZV0rLcA7Ybm+4swZxepfdycurr8y/cGM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 5FEBE1655; Mon, 2 Mar 2026 05:56:30 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id B30913F73B; Mon, 2 Mar 2026 05:56:34 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian , Jonathan Cameron Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, Linu Cherian Subject: [PATCH v3 09/13] arm64: mm: Refactor flush_tlb_page() to use __tlbi_level_asid() Date: Mon, 2 Mar 2026 13:55:56 +0000 Message-ID: <20260302135602.3716920-10-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260302135602.3716920-1-ryan.roberts@arm.com> References: <20260302135602.3716920-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Now that we have __tlbi_level_asid(), let's refactor the *flush_tlb_page*() variants to use it rather than open coding. The emitted tlbi(s) is/are intended to be exactly the same as before; no TTL hint is provided. Although the spec for flush_tlb_page() allows for setting the TTL hint to 3, it turns out that flush_tlb_fix_spurious_fault_pmd() depends on local_flush_tlb_page_nonotify() to invalidate the level 2 entry. This will be fixed separately. Reviewed-by: Linu Cherian Reviewed-by: Jonathan Cameron Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index fd86647db24bb..0a49a25a4fdc8 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -346,12 +346,8 @@ static inline void flush_tlb_mm(struct mm_struct *mm) static inline void __local_flush_tlb_page_nonotify_nosync(struct mm_struct= *mm, unsigned long uaddr) { - unsigned long addr; - dsb(nshst); - addr =3D __TLBI_VADDR(uaddr, ASID(mm)); - __tlbi(vale1, addr); - __tlbi_user(vale1, addr); + __tlbi_level_asid(vale1, uaddr, TLBI_TTL_UNKNOWN, ASID(mm)); } =20 static inline void local_flush_tlb_page_nonotify(struct vm_area_struct *vm= a, @@ -373,12 +369,8 @@ static inline void local_flush_tlb_page(struct vm_area= _struct *vma, static inline void __flush_tlb_page_nosync(struct mm_struct *mm, unsigned long uaddr) { - unsigned long addr; - dsb(ishst); - addr =3D __TLBI_VADDR(uaddr, ASID(mm)); - __tlbi(vale1is, addr); - __tlbi_user(vale1is, addr); + __tlbi_level_asid(vale1is, uaddr, TLBI_TTL_UNKNOWN, ASID(mm)); mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK, (uaddr & PAGE_MASK) + PAGE_SIZE); } --=20 2.43.0 From nobody Wed Apr 15 17:26:29 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 718D43FD121 for ; Mon, 2 Mar 2026 13:56:39 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459801; cv=none; b=S6fyw0ieK5i9LViMrmvW9QtzpczKT0fpsBuma5yVFqG5mxiKco7OEq/wie0S2LU5mucGCpzAjgr5dwdyBw4BIJnzQuNIByXz9pG1bQJl0TACdyLfJh6zsB47eQndcCh+pIUvS+i2e1DGewcEttV+b6+RqNrgWZZkyHkp4wq8HRM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459801; c=relaxed/simple; bh=cjcIRkbhrD0UdSJv1jKCxn60RMwRZbeWHSqvE/kDv1s=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=fbXmjEfseLIKpkINt9Zd7uu67PwfTXVscomgPczG1iZrZaF/L4RbKYn6kx1ewZCTS3vz74JgQQuwNRnR0KQuzddMwXPfx1vER04v5upQHL67Kq2R+jM892/9lNpk300cDq+YaVf+L1r8V6mSJZ0RtItbgu4g4TlLaG+amX5HFVs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id C1FC114BF; Mon, 2 Mar 2026 05:56:32 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 062253F73B; Mon, 2 Mar 2026 05:56:36 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian , Jonathan Cameron Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, Linu Cherian Subject: [PATCH v3 10/13] arm64: mm: Refactor __flush_tlb_range() to take flags Date: Mon, 2 Mar 2026 13:55:57 +0000 Message-ID: <20260302135602.3716920-11-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260302135602.3716920-1-ryan.roberts@arm.com> References: <20260302135602.3716920-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" We have function variants with "_nosync", "_local", "_nonotify" as well as the "last_level" parameter. Let's generalize and simplify by using a flags parameter to encode all these variants. As a first step, convert the "last_level" boolean parameter to a flags parameter and create the first flag, TLBF_NOWALKCACHE. When present, walk cache entries are not evicted, which is the same as the old last_level=3Dtrue. Reviewed-by: Linu Cherian Reviewed-by: Jonathan Cameron Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/hugetlb.h | 12 ++++++------ arch/arm64/include/asm/pgtable.h | 4 ++-- arch/arm64/include/asm/tlb.h | 6 +++--- arch/arm64/include/asm/tlbflush.h | 32 +++++++++++++++++++------------ arch/arm64/mm/contpte.c | 5 +++-- arch/arm64/mm/hugetlbpage.c | 4 ++-- arch/arm64/mm/mmu.c | 2 +- 7 files changed, 37 insertions(+), 28 deletions(-) diff --git a/arch/arm64/include/asm/hugetlb.h b/arch/arm64/include/asm/huge= tlb.h index e6f8ff3cc6306..d038ff14d16ca 100644 --- a/arch/arm64/include/asm/hugetlb.h +++ b/arch/arm64/include/asm/hugetlb.h @@ -71,23 +71,23 @@ static inline void __flush_hugetlb_tlb_range(struct vm_= area_struct *vma, unsigned long start, unsigned long end, unsigned long stride, - bool last_level) + tlbf_t flags) { switch (stride) { #ifndef __PAGETABLE_PMD_FOLDED case PUD_SIZE: - __flush_tlb_range(vma, start, end, PUD_SIZE, last_level, 1); + __flush_tlb_range(vma, start, end, PUD_SIZE, 1, flags); break; #endif case CONT_PMD_SIZE: case PMD_SIZE: - __flush_tlb_range(vma, start, end, PMD_SIZE, last_level, 2); + __flush_tlb_range(vma, start, end, PMD_SIZE, 2, flags); break; case CONT_PTE_SIZE: - __flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, 3); + __flush_tlb_range(vma, start, end, PAGE_SIZE, 3, flags); break; default: - __flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, TLBI_TTL_UNKNO= WN); + __flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, flags); } } =20 @@ -98,7 +98,7 @@ static inline void flush_hugetlb_tlb_range(struct vm_area= _struct *vma, { unsigned long stride =3D huge_page_size(hstate_vma(vma)); =20 - __flush_hugetlb_tlb_range(vma, start, end, stride, false); + __flush_hugetlb_tlb_range(vma, start, end, stride, TLBF_NONE); } =20 #endif /* __ASM_HUGETLB_H */ diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgta= ble.h index b3e58735c49bd..88bb9275ac898 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -89,9 +89,9 @@ static inline void arch_leave_lazy_mmu_mode(void) =20 /* Set stride and tlb_level in flush_*_tlb_range */ #define flush_pmd_tlb_range(vma, addr, end) \ - __flush_tlb_range(vma, addr, end, PMD_SIZE, false, 2) + __flush_tlb_range(vma, addr, end, PMD_SIZE, 2, TLBF_NONE) #define flush_pud_tlb_range(vma, addr, end) \ - __flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1) + __flush_tlb_range(vma, addr, end, PUD_SIZE, 1, TLBF_NONE) #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ =20 /* diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index 8d762607285cc..10869d7731b83 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -53,7 +53,7 @@ static inline int tlb_get_level(struct mmu_gather *tlb) static inline void tlb_flush(struct mmu_gather *tlb) { struct vm_area_struct vma =3D TLB_FLUSH_VMA(tlb->mm, 0); - bool last_level =3D !tlb->freed_tables; + tlbf_t flags =3D tlb->freed_tables ? TLBF_NONE : TLBF_NOWALKCACHE; unsigned long stride =3D tlb_get_unmap_size(tlb); int tlb_level =3D tlb_get_level(tlb); =20 @@ -63,13 +63,13 @@ static inline void tlb_flush(struct mmu_gather *tlb) * reallocate our ASID without invalidating the entire TLB. */ if (tlb->fullmm) { - if (!last_level) + if (tlb->freed_tables) flush_tlb_mm(tlb->mm); return; } =20 __flush_tlb_range(&vma, tlb->start, tlb->end, stride, - last_level, tlb_level); + tlb_level, flags); } =20 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 0a49a25a4fdc8..d134824ea5daa 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -286,16 +286,16 @@ static inline void __tlbi_sync_s1ish_hyp(void) * CPUs, ensuring that any walk-cache entries associated with the * translation are also invalidated. * - * __flush_tlb_range(vma, start, end, stride, last_level, tlb_level) + * __flush_tlb_range(vma, start, end, stride, tlb_level, flags) * Invalidate the virtual-address range '[start, end)' on all * CPUs for the user address space corresponding to 'vma->mm'. * The invalidation operations are issued at a granularity - * determined by 'stride' and only affect any walk-cache entries - * if 'last_level' is equal to false. tlb_level is the level at + * determined by 'stride'. tlb_level is the level at * which the invalidation must take place. If the level is wrong, * no invalidation may take place. In the case where the level * cannot be easily determined, the value TLBI_TTL_UNKNOWN will - * perform a non-hinted invalidation. + * perform a non-hinted invalidation. flags may be TLBF_NONE (0) or + * TLBF_NOWALKCACHE (elide eviction of walk cache entries). * * local_flush_tlb_page(vma, addr) * Local variant of flush_tlb_page(). Stale TLB entries may @@ -544,10 +544,18 @@ static inline bool __flush_tlb_range_limit_excess(uns= igned long pages, return pages >=3D (MAX_DVM_OPS * stride) >> PAGE_SHIFT; } =20 +typedef unsigned __bitwise tlbf_t; + +/* No special behaviour. */ +#define TLBF_NONE ((__force tlbf_t)0) + +/* Invalidate tlb entries only, leaving the page table walk cache intact. = */ +#define TLBF_NOWALKCACHE ((__force tlbf_t)BIT(0)) + static inline void __flush_tlb_range_nosync(struct mm_struct *mm, unsigned long start, unsigned long end, - unsigned long stride, bool last_level, - int tlb_level) + unsigned long stride, int tlb_level, + tlbf_t flags) { unsigned long asid, pages; =20 @@ -563,7 +571,7 @@ static inline void __flush_tlb_range_nosync(struct mm_s= truct *mm, dsb(ishst); asid =3D ASID(mm); =20 - if (last_level) + if (flags & TLBF_NOWALKCACHE) __flush_s1_tlb_range_op(vale1is, start, pages, stride, asid, tlb_level); else @@ -575,11 +583,11 @@ static inline void __flush_tlb_range_nosync(struct mm= _struct *mm, =20 static inline void __flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, - unsigned long stride, bool last_level, - int tlb_level) + unsigned long stride, int tlb_level, + tlbf_t flags) { __flush_tlb_range_nosync(vma->vm_mm, start, end, stride, - last_level, tlb_level); + tlb_level, flags); __tlbi_sync_s1ish(); } =20 @@ -607,7 +615,7 @@ static inline void flush_tlb_range(struct vm_area_struc= t *vma, * Set the tlb_level to TLBI_TTL_UNKNOWN because we can not get enough * information here. */ - __flush_tlb_range(vma, start, end, PAGE_SIZE, false, TLBI_TTL_UNKNOWN); + __flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE= ); } =20 static inline void flush_tlb_kernel_range(unsigned long start, unsigned lo= ng end) @@ -648,7 +656,7 @@ static inline void __flush_tlb_kernel_pgtable(unsigned = long kaddr) static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_ba= tch *batch, struct mm_struct *mm, unsigned long start, unsigned long end) { - __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3); + __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, 3, TLBF_NOWALKCACHE); } =20 static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval) diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index b929a455103f8..681f22fac52a1 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -225,7 +225,8 @@ static void contpte_convert(struct mm_struct *mm, unsig= ned long addr, */ =20 if (!system_supports_bbml2_noabort()) - __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, true, 3); + __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, 3, + TLBF_NOWALKCACHE); =20 __set_ptes(mm, start_addr, start_ptep, pte, CONT_PTES); } @@ -552,7 +553,7 @@ int contpte_clear_flush_young_ptes(struct vm_area_struc= t *vma, * eliding the trailing DSB applies here. */ __flush_tlb_range_nosync(vma->vm_mm, addr, end, - PAGE_SIZE, true, 3); + PAGE_SIZE, 3, TLBF_NOWALKCACHE); } =20 return young; diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index a42c05cf56408..0b7ccd0cbb9ec 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -181,7 +181,7 @@ static pte_t get_clear_contig_flush(struct mm_struct *m= m, struct vm_area_struct vma =3D TLB_FLUSH_VMA(mm, 0); unsigned long end =3D addr + (pgsize * ncontig); =20 - __flush_hugetlb_tlb_range(&vma, addr, end, pgsize, true); + __flush_hugetlb_tlb_range(&vma, addr, end, pgsize, TLBF_NOWALKCACHE); return orig_pte; } =20 @@ -209,7 +209,7 @@ static void clear_flush(struct mm_struct *mm, if (mm =3D=3D &init_mm) flush_tlb_kernel_range(saddr, addr); else - __flush_hugetlb_tlb_range(&vma, saddr, addr, pgsize, true); + __flush_hugetlb_tlb_range(&vma, saddr, addr, pgsize, TLBF_NOWALKCACHE); } =20 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index a6a00accf4f93..054df431846fd 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -2149,7 +2149,7 @@ pte_t modify_prot_start_ptes(struct vm_area_struct *v= ma, unsigned long addr, */ if (pte_accessible(vma->vm_mm, pte) && pte_user_exec(pte)) __flush_tlb_range(vma, addr, nr * PAGE_SIZE, - PAGE_SIZE, true, 3); + PAGE_SIZE, 3, TLBF_NOWALKCACHE); } =20 return pte; --=20 2.43.0 From nobody Wed Apr 15 17:26:29 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id BA3623FD142 for ; Mon, 2 Mar 2026 13:56:41 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459803; cv=none; b=AwZ+ctE9vshigR3hkVnIVaraHlNc2L8bKaYX/0UC+9Tp4la036oKiWzGMHETN5Rt/++m6xhmY/mSaA6wl5bh3nLSAfNh4uGtcwmbhp2e9+vl1XNEjMNwKAIzJ+OltS1A98Y8WUcVO2l6payHWeyS0XxcICDVhViiT3AN7GIYNRI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459803; c=relaxed/simple; bh=Di1x8C9Iz5b7B1CJFuSa3wjnKuPjkTxSNMxgsCXdxsM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=IskdrWkNE6ysHvddaOmiJHMekktC4MD7x8Yz1X5QVlLMoADNzHV/BHDJFmOvxAHTBOE3XYtBqV8+9iqtRyerHkhHsw0RnsYNGeNRhmtu2+UJjYpv922XIFWRRwUvQ21MN+R/v7i+qzUs8uG41LIyyerb05MWRAyuCa88U/BrYpU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id E943A150C; Mon, 2 Mar 2026 05:56:34 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 6ADA93F73B; Mon, 2 Mar 2026 05:56:39 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian , Jonathan Cameron Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v3 11/13] arm64: mm: More flags for __flush_tlb_range() Date: Mon, 2 Mar 2026 13:55:58 +0000 Message-ID: <20260302135602.3716920-12-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260302135602.3716920-1-ryan.roberts@arm.com> References: <20260302135602.3716920-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Refactor function variants with "_nosync", "_local" and "_nonotify" into a single __always_inline implementation that takes flags and rely on constant folding to select the parts that are actually needed at any given callsite, based on the provided flags. Flags all live in the tlbf_t (TLB flags) type; TLBF_NONE (0) continues to provide the strongest semantics (i.e. evict from walk cache, broadcast, synchronise and notify). Each flag reduces the strength in some way; TLBF_NONOTIFY, TLBF_NOSYNC and TLBF_NOBROADCAST are added to complement the existing TLBF_NOWALKCACHE. There are no users that require TLBF_NOBROADCAST without TLBF_NOWALKCACHE so implement that as BUILD_BUG() to avoid needing to introduce dead code for vae1 invalidations. The result is a clearer, simpler, more powerful API. Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 95 ++++++++++++++++++------------- arch/arm64/mm/contpte.c | 9 ++- 2 files changed, 62 insertions(+), 42 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index d134824ea5daa..5509927e45b93 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -295,7 +295,10 @@ static inline void __tlbi_sync_s1ish_hyp(void) * no invalidation may take place. In the case where the level * cannot be easily determined, the value TLBI_TTL_UNKNOWN will * perform a non-hinted invalidation. flags may be TLBF_NONE (0) or - * TLBF_NOWALKCACHE (elide eviction of walk cache entries). + * any combination of TLBF_NOWALKCACHE (elide eviction of walk + * cache entries), TLBF_NONOTIFY (don't call mmu notifiers), + * TLBF_NOSYNC (don't issue trailing dsb) and TLBF_NOBROADCAST + * (only perform the invalidation for the local cpu). * * local_flush_tlb_page(vma, addr) * Local variant of flush_tlb_page(). Stale TLB entries may @@ -305,12 +308,6 @@ static inline void __tlbi_sync_s1ish_hyp(void) * Same as local_flush_tlb_page() except MMU notifier will not be * called. * - * local_flush_tlb_contpte(vma, addr) - * Invalidate the virtual-address range - * '[addr, addr+CONT_PTE_SIZE)' mapped with contpte on local CPU - * for the user address space corresponding to 'vma->mm'. Stale - * TLB entries may remain in remote CPUs. - * * Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented * on top of these routines, since that is our interface to the mmu_gather * API as used by munmap() and friends. @@ -552,15 +549,23 @@ typedef unsigned __bitwise tlbf_t; /* Invalidate tlb entries only, leaving the page table walk cache intact. = */ #define TLBF_NOWALKCACHE ((__force tlbf_t)BIT(0)) =20 -static inline void __flush_tlb_range_nosync(struct mm_struct *mm, - unsigned long start, unsigned long end, - unsigned long stride, int tlb_level, - tlbf_t flags) +/* Skip the trailing dsb after issuing tlbi. */ +#define TLBF_NOSYNC ((__force tlbf_t)BIT(1)) + +/* Suppress tlb notifier callbacks for this flush operation. */ +#define TLBF_NONOTIFY ((__force tlbf_t)BIT(2)) + +/* Perform the tlbi locally without broadcasting to other CPUs. */ +#define TLBF_NOBROADCAST ((__force tlbf_t)BIT(3)) + +static __always_inline void __do_flush_tlb_range(struct vm_area_struct *vm= a, + unsigned long start, unsigned long end, + unsigned long stride, int tlb_level, + tlbf_t flags) { + struct mm_struct *mm =3D vma->vm_mm; unsigned long asid, pages; =20 - start =3D round_down(start, stride); - end =3D round_up(end, stride); pages =3D (end - start) >> PAGE_SHIFT; =20 if (__flush_tlb_range_limit_excess(pages, stride)) { @@ -568,17 +573,41 @@ static inline void __flush_tlb_range_nosync(struct mm= _struct *mm, return; } =20 - dsb(ishst); + if (!(flags & TLBF_NOBROADCAST)) + dsb(ishst); + else + dsb(nshst); + asid =3D ASID(mm); =20 - if (flags & TLBF_NOWALKCACHE) - __flush_s1_tlb_range_op(vale1is, start, pages, stride, - asid, tlb_level); - else + switch (flags & (TLBF_NOWALKCACHE | TLBF_NOBROADCAST)) { + case TLBF_NONE: __flush_s1_tlb_range_op(vae1is, start, pages, stride, - asid, tlb_level); + asid, tlb_level); + break; + case TLBF_NOWALKCACHE: + __flush_s1_tlb_range_op(vale1is, start, pages, stride, + asid, tlb_level); + break; + case TLBF_NOBROADCAST: + /* Combination unused */ + BUG(); + break; + case TLBF_NOWALKCACHE | TLBF_NOBROADCAST: + __flush_s1_tlb_range_op(vale1, start, pages, stride, + asid, tlb_level); + break; + } + + if (!(flags & TLBF_NONOTIFY)) + mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); =20 - mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); + if (!(flags & TLBF_NOSYNC)) { + if (!(flags & TLBF_NOBROADCAST)) + __tlbi_sync_s1ish(); + else + dsb(nsh); + } } =20 static inline void __flush_tlb_range(struct vm_area_struct *vma, @@ -586,24 +615,9 @@ static inline void __flush_tlb_range(struct vm_area_st= ruct *vma, unsigned long stride, int tlb_level, tlbf_t flags) { - __flush_tlb_range_nosync(vma->vm_mm, start, end, stride, - tlb_level, flags); - __tlbi_sync_s1ish(); -} - -static inline void local_flush_tlb_contpte(struct vm_area_struct *vma, - unsigned long addr) -{ - unsigned long asid; - - addr =3D round_down(addr, CONT_PTE_SIZE); - - dsb(nshst); - asid =3D ASID(vma->vm_mm); - __flush_s1_tlb_range_op(vale1, addr, CONT_PTES, PAGE_SIZE, asid, 3); - mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, addr, - addr + CONT_PTE_SIZE); - dsb(nsh); + start =3D round_down(start, stride); + end =3D round_up(end, stride); + __do_flush_tlb_range(vma, start, end, stride, tlb_level, flags); } =20 static inline void flush_tlb_range(struct vm_area_struct *vma, @@ -656,7 +670,10 @@ static inline void __flush_tlb_kernel_pgtable(unsigned= long kaddr) static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_ba= tch *batch, struct mm_struct *mm, unsigned long start, unsigned long end) { - __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, 3, TLBF_NOWALKCACHE); + struct vm_area_struct vma =3D { .vm_mm =3D mm, .vm_flags =3D 0 }; + + __flush_tlb_range(&vma, start, end, PAGE_SIZE, 3, + TLBF_NOWALKCACHE | TLBF_NOSYNC); } =20 static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval) diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index 681f22fac52a1..3f1a3e86353de 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -552,8 +552,8 @@ int contpte_clear_flush_young_ptes(struct vm_area_struc= t *vma, * See comment in __ptep_clear_flush_young(); same rationale for * eliding the trailing DSB applies here. */ - __flush_tlb_range_nosync(vma->vm_mm, addr, end, - PAGE_SIZE, 3, TLBF_NOWALKCACHE); + __flush_tlb_range(vma, addr, end, PAGE_SIZE, 3, + TLBF_NOWALKCACHE | TLBF_NOSYNC); } =20 return young; @@ -641,7 +641,10 @@ int contpte_ptep_set_access_flags(struct vm_area_struc= t *vma, __ptep_set_access_flags(vma, addr, ptep, entry, 0); =20 if (dirty) - local_flush_tlb_contpte(vma, start_addr); + __flush_tlb_range(vma, start_addr, + start_addr + CONT_PTE_SIZE, + PAGE_SIZE, 3, + TLBF_NOWALKCACHE | TLBF_NOBROADCAST); } else { __contpte_try_unfold(vma->vm_mm, addr, ptep, orig_pte); __ptep_set_access_flags(vma, addr, ptep, entry, dirty); --=20 2.43.0 From nobody Wed Apr 15 17:26:29 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 1203D3FB053 for ; Mon, 2 Mar 2026 13:56:43 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459805; cv=none; b=bTc8TXVc6/zFxIdLbF3Y0ymm/OqpF3osT9yB/N6kk15iUx9tfdxAszH6XdejgWjRdEPE6g/9TWrWqR0iUPpjGkBvnRJMJuVWx7nL+GsK9Xlx4cteQHsc6sbnTCmzb+VQdvXiHAcib9+6TY2M6HJjM1rWI8tJs76vmIwFbtNBIK8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459805; c=relaxed/simple; bh=ke9A+Fnz/vMjVxpNFA66iwPoLFdvPTmsi68x7t1N61c=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=KXA1GsohV1ZGXoETEQIsZmE/Z/LfZ9ZvkghphNzfVntCez03+sSbFI6WRbilbJXNRlp1Bfm6sVq/kI8WgxMW5kbdkGPQCp0/TzGerDA627Gu3ecsdVyMe/8xQ6S4SmrrqxaOuxn/43ccCMDt1LuM3g8XrjwTAliSHtt51MsXAP0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 3E4A91570; Mon, 2 Mar 2026 05:56:37 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 922513F73B; Mon, 2 Mar 2026 05:56:41 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian , Jonathan Cameron Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, Linu Cherian Subject: [PATCH v3 12/13] arm64: mm: Wrap flush_tlb_page() around __do_flush_tlb_range() Date: Mon, 2 Mar 2026 13:55:59 +0000 Message-ID: <20260302135602.3716920-13-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260302135602.3716920-1-ryan.roberts@arm.com> References: <20260302135602.3716920-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Flushing a page from the tlb is just a special case of flushing a range. So let's rework flush_tlb_page() so that it simply wraps __do_flush_tlb_range(). While at it, let's also update the API to take the same flags that we use when flushing a range. This allows us to delete all the ugly "_nosync", "_local" and "_nonotify" variants. Thanks to constant folding, all of the complex looping and tlbi-by-range options get eliminated so that the generated code for flush_tlb_page() looks very similar to the previous version. Reviewed-by: Linu Cherian Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/pgtable.h | 6 +-- arch/arm64/include/asm/tlbflush.h | 81 ++++++++++--------------------- arch/arm64/mm/fault.c | 2 +- 3 files changed, 29 insertions(+), 60 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgta= ble.h index 88bb9275ac898..7039931df4622 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -101,10 +101,10 @@ static inline void arch_leave_lazy_mmu_mode(void) * entries exist. */ #define flush_tlb_fix_spurious_fault(vma, address, ptep) \ - local_flush_tlb_page_nonotify(vma, address) + __flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY) =20 #define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp) \ - local_flush_tlb_page_nonotify(vma, address) + __flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY) =20 /* * ZERO_PAGE is a global shared page that is always zero: used @@ -1320,7 +1320,7 @@ static inline int __ptep_clear_flush_young(struct vm_= area_struct *vma, * context-switch, which provides a DSB to complete the TLB * invalidation. */ - flush_tlb_page_nosync(vma, address); + __flush_tlb_page(vma, address, TLBF_NOSYNC); } =20 return young; diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 5509927e45b93..5096ec7ab8650 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -269,10 +269,7 @@ static inline void __tlbi_sync_s1ish_hyp(void) * unmapping pages from vmalloc/io space. * * flush_tlb_page(vma, addr) - * Invalidate a single user mapping for address 'addr' in the - * address space corresponding to 'vma->mm'. Note that this - * operation only invalidates a single, last-level page-table - * entry and therefore does not affect any walk-caches. + * Equivalent to __flush_tlb_page(..., flags=3DTLBF_NONE) * * * Next, we have some undocumented invalidation routines that you probably @@ -300,13 +297,14 @@ static inline void __tlbi_sync_s1ish_hyp(void) * TLBF_NOSYNC (don't issue trailing dsb) and TLBF_NOBROADCAST * (only perform the invalidation for the local cpu). * - * local_flush_tlb_page(vma, addr) - * Local variant of flush_tlb_page(). Stale TLB entries may - * remain in remote CPUs. - * - * local_flush_tlb_page_nonotify(vma, addr) - * Same as local_flush_tlb_page() except MMU notifier will not be - * called. + * __flush_tlb_page(vma, addr, flags) + * Invalidate a single user mapping for address 'addr' in the + * address space corresponding to 'vma->mm'. Note that this + * operation only invalidates a single, last-level page-table entry + * and therefore does not affect any walk-caches. flags may contain + * any combination of TLBF_NONOTIFY (don't call mmu notifiers), + * TLBF_NOSYNC (don't issue trailing dsb) and TLBF_NOBROADCAST + * (only perform the invalidation for the local cpu). * * Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented * on top of these routines, since that is our interface to the mmu_gather @@ -340,51 +338,6 @@ static inline void flush_tlb_mm(struct mm_struct *mm) mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL); } =20 -static inline void __local_flush_tlb_page_nonotify_nosync(struct mm_struct= *mm, - unsigned long uaddr) -{ - dsb(nshst); - __tlbi_level_asid(vale1, uaddr, TLBI_TTL_UNKNOWN, ASID(mm)); -} - -static inline void local_flush_tlb_page_nonotify(struct vm_area_struct *vm= a, - unsigned long uaddr) -{ - __local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr); - dsb(nsh); -} - -static inline void local_flush_tlb_page(struct vm_area_struct *vma, - unsigned long uaddr) -{ - __local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr); - mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, uaddr & PAGE_MASK, - (uaddr & PAGE_MASK) + PAGE_SIZE); - dsb(nsh); -} - -static inline void __flush_tlb_page_nosync(struct mm_struct *mm, - unsigned long uaddr) -{ - dsb(ishst); - __tlbi_level_asid(vale1is, uaddr, TLBI_TTL_UNKNOWN, ASID(mm)); - mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK, - (uaddr & PAGE_MASK) + PAGE_SIZE); -} - -static inline void flush_tlb_page_nosync(struct vm_area_struct *vma, - unsigned long uaddr) -{ - return __flush_tlb_page_nosync(vma->vm_mm, uaddr); -} - -static inline void flush_tlb_page(struct vm_area_struct *vma, - unsigned long uaddr) -{ - flush_tlb_page_nosync(vma, uaddr); - __tlbi_sync_s1ish(); -} - static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) { return true; @@ -632,6 +585,22 @@ static inline void flush_tlb_range(struct vm_area_stru= ct *vma, __flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE= ); } =20 +static inline void __flush_tlb_page(struct vm_area_struct *vma, + unsigned long uaddr, tlbf_t flags) +{ + unsigned long start =3D round_down(uaddr, PAGE_SIZE); + unsigned long end =3D start + PAGE_SIZE; + + __do_flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, + TLBF_NOWALKCACHE | flags); +} + +static inline void flush_tlb_page(struct vm_area_struct *vma, + unsigned long uaddr) +{ + __flush_tlb_page(vma, uaddr, TLBF_NONE); +} + static inline void flush_tlb_kernel_range(unsigned long start, unsigned lo= ng end) { const unsigned long stride =3D PAGE_SIZE; diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index be9dab2c7d6a8..f91aa686f1428 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -239,7 +239,7 @@ int __ptep_set_access_flags(struct vm_area_struct *vma, * flush_tlb_fix_spurious_fault(). */ if (dirty) - local_flush_tlb_page(vma, address); + __flush_tlb_page(vma, address, TLBF_NOBROADCAST); return 1; } =20 --=20 2.43.0 From nobody Wed Apr 15 17:26:29 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 26B523FFAC3 for ; Mon, 2 Mar 2026 13:56:46 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459807; cv=none; b=pjGinPmDHrKabgSZTROxq20K2qI665Ao1qMv6TNf+icRkhPuf1wucoRjDjJCoMHdqlvgEnKlgGspvHE4wQNc3UxQs/MZNLPo7h3mTaUw8NAnxyjugZ2WgsZbmei7kWnA+ggQd9AkdDT4s+ORS0eUQD9jMK7RYQdRjP84SvXYBjk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1772459807; c=relaxed/simple; bh=tnNqa7ushA822zc8oDPeX4TdG+6k5C7EB64A/E8kZno=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=XSy3dXtm3Sia8sU90VJ1irpF7cFp4HxRf5XnbpXTgEXmekRF5vVzAaasBnKGCBfqrxg2jeUqULTtn5g56pbyRod28g7x95jxnFg9uJkxZ0cTy/kuEH6teS5e5Gkqv94g2q/0d9Jsaj/5F54P662yJ9XX6wD9vsWhj3V2p0VFmC0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 8183C1655; Mon, 2 Mar 2026 05:56:39 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id D80C23F73B; Mon, 2 Mar 2026 05:56:43 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian , Jonathan Cameron Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, Linu Cherian Subject: [PATCH v3 13/13] arm64: mm: Provide level hint for flush_tlb_page() Date: Mon, 2 Mar 2026 13:56:00 +0000 Message-ID: <20260302135602.3716920-14-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260302135602.3716920-1-ryan.roberts@arm.com> References: <20260302135602.3716920-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Previously tlb invalidations issued by __flush_tlb_page() did not contain a level hint. But the function is clearly only ever targeting level 3 tlb entries and its documentation agrees: | this operation only invalidates a single, last-level page-table | entry and therefore does not affect any walk-caches However, it turns out that the function was actually being used to invalidate a level 2 mapping via flush_tlb_fix_spurious_fault_pmd(). The bug was benign because the level hint was not set so the HW would still invalidate the PMD mapping, and also because the TLBF_NONOTIFY flag was set, the bounds of the mapping were never used for anything else. Now that we have the new and improved range-invalidation API, it is trival to fix flush_tlb_fix_spurious_fault_pmd() to explicitly flush the whole range (locally, without notification and last level only). So let's do that, and then update __flush_tlb_page() to hint level 3. Reviewed-by: Linu Cherian Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/pgtable.h | 5 +++-- arch/arm64/include/asm/tlbflush.h | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgta= ble.h index 7039931df4622..b1a96a8f2b17e 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -103,8 +103,9 @@ static inline void arch_leave_lazy_mmu_mode(void) #define flush_tlb_fix_spurious_fault(vma, address, ptep) \ __flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY) =20 -#define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp) \ - __flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY) +#define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp) \ + __flush_tlb_range(vma, address, address + PMD_SIZE, PMD_SIZE, 2, \ + TLBF_NOBROADCAST | TLBF_NONOTIFY | TLBF_NOWALKCACHE) =20 /* * ZERO_PAGE is a global shared page that is always zero: used diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 5096ec7ab8650..958fe97b744e5 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -591,7 +591,7 @@ static inline void __flush_tlb_page(struct vm_area_stru= ct *vma, unsigned long start =3D round_down(uaddr, PAGE_SIZE); unsigned long end =3D start + PAGE_SIZE; =20 - __do_flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, + __do_flush_tlb_range(vma, start, end, PAGE_SIZE, 3, TLBF_NOWALKCACHE | flags); } =20 --=20 2.43.0