From nobody Thu Dec 18 23:23:41 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 56AAA34679D for ; Tue, 16 Dec 2025 14:46:13 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896375; cv=none; b=ZZ2XFluBD8vRe7OC9sp0QaHRjqneup2bxFr2Rjpvaoknxp7jd2Z21orcwA4xFpvybdnxAMBaezUKjJhKNSH2A55UQGk5KhiDxrO/7gdu6G1u10DrtA6MFVt26ONoxDvVCeqmZDr6n5Im0p6dUEGXNszs47XbdKNNBSZ3TVQNQc4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896375; c=relaxed/simple; bh=FtgpUqryn+VfcIDzrL/xlcw9+yf9sTYav4Mn0cj51io=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=fg5C5W3+vbn8MGEJRmeuOMHPKoCxW2WZM5F6hV2CTScbTtCYfIzMpjoEbetk9Qqz2Mh/wjR1DtLQ8fa5lV1xFLjR6EDCEBNwTIkKDSDY859e5CSfbYO22ILuoHcVmg4EMFmb5WxBh9fS8a40nO5Med2LQpk7zNe5nQUVsIEGlss= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 8917C1516; Tue, 16 Dec 2025 06:46:05 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 3BD143F73F; Tue, 16 Dec 2025 06:46:11 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v1 01/13] arm64: mm: Re-implement the __tlbi_level macro as a C function Date: Tue, 16 Dec 2025 14:45:46 +0000 Message-ID: <20251216144601.2106412-2-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20251216144601.2106412-1-ryan.roberts@arm.com> References: <20251216144601.2106412-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" As part of efforts to reduce our reliance on complex preprocessor macros for TLB invalidation routines, convert the __tlbi_level macro to a C function for by-level TLB invalidation. Each specific tlbi level op is implemented as a C function and the appropriate function pointer is passed to __tlbi_level(). Since everything is declared inline and is statically resolvable, the compiler will convert the indirect function call to a direct inline execution. Suggested-by: Linus Torvalds Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 69 +++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index a2d65d7d6aae..13a59cf28943 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -105,19 +105,62 @@ static inline unsigned long get_trans_granule(void) =20 #define TLBI_TTL_UNKNOWN INT_MAX =20 -#define __tlbi_level(op, addr, level) do { \ - u64 arg =3D addr; \ - \ - if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && \ - level >=3D 0 && level <=3D 3) { \ - u64 ttl =3D level & 3; \ - ttl |=3D get_trans_granule() << 2; \ - arg &=3D ~TLBI_TTL_MASK; \ - arg |=3D FIELD_PREP(TLBI_TTL_MASK, ttl); \ - } \ - \ - __tlbi(op, arg); \ -} while(0) +typedef void (*tlbi_op)(u64 arg); + +static __always_inline void vae1is(u64 arg) +{ + __tlbi(vae1is, arg); +} + +static __always_inline void vae2is(u64 arg) +{ + __tlbi(vae2is, arg); +} + +static __always_inline void vale1(u64 arg) +{ + __tlbi(vale1, arg); + __tlbi_user(vale1, arg); +} + +static __always_inline void vale1is(u64 arg) +{ + __tlbi(vale1is, arg); +} + +static __always_inline void vale2is(u64 arg) +{ + __tlbi(vale2is, arg); +} + +static __always_inline void vaale1is(u64 arg) +{ + __tlbi(vaale1is, arg); +} + +static __always_inline void ipas2e1(u64 arg) +{ + __tlbi(ipas2e1, arg); +} + +static __always_inline void ipas2e1is(u64 arg) +{ + __tlbi(ipas2e1is, arg); +} + +static __always_inline void __tlbi_level(tlbi_op op, u64 addr, u32 level) +{ + u64 arg =3D addr; + + if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && level <=3D 3) { + u64 ttl =3D level | (get_trans_granule() << 2); + + arg &=3D ~TLBI_TTL_MASK; + arg |=3D FIELD_PREP(TLBI_TTL_MASK, ttl); + } + + op(arg); +} =20 #define __tlbi_user_level(op, arg, level) do { \ if (arm64_kernel_unmapped_at_el0()) \ --=20 2.43.0 From nobody Thu Dec 18 23:23:41 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 438043469E6 for ; Tue, 16 Dec 2025 14:46:14 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896377; cv=none; b=n/D4rlKZz+CdcZX/lVBNQbdgWIa7mtrVHpB/bOagjz/itCe++R+aTV01cOt5aiHYbfhTZWPMfn2Xs9WUdEcvOIuxhZZ24zvRKZCZBYMI92qs9jxa7FI3uApRr+JoEJ3EucNL7fTo/y/h0GQbxHtdDqcbyLdplmpnGTztnKVGpBM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896377; c=relaxed/simple; bh=O2qZ6OZ7h4Nw9edqoxMG9imTzBoaPD6JrT3OeLH5ze0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=Ko3wYXqCZE6GaTW3BV09JE33Ox+Nn12JHAq5rz6zm+ggu97hVU6vKyDgCD2vbOPBUlJySGBB9zNcfpq4URbv27tz/xQ3gf4av9xzoYLalmQ8KnxL+PImRALvzzDHNrVwNiq5D+Z0sVjDcPrnYAOQvwl/dJcYFDnsuCebAhA3TRA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 4B2BC1655; Tue, 16 Dec 2025 06:46:07 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id EEC223F73F; Tue, 16 Dec 2025 06:46:12 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v1 02/13] arm64: mm: Introduce a C wrapper for by-range TLB invalidation Date: Tue, 16 Dec 2025 14:45:47 +0000 Message-ID: <20251216144601.2106412-3-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20251216144601.2106412-1-ryan.roberts@arm.com> References: <20251216144601.2106412-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" As part of efforts to reduce our reliance on complex preprocessor macros for TLB invalidation routines, introduce a new C wrapper for by-range TLB invalidation which can be used instead of the __tlbi() macro and can additionally be called from C code. Each specific tlbi range op is implemented as a C function and the appropriate function pointer is passed to __tlbi_range(). Since everything is declared inline and is statically resolvable, the compiler will convert the indirect function call to a direct inline execution. Suggested-by: Linus Torvalds Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 33 ++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 13a59cf28943..c5111d2afc66 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -459,6 +459,37 @@ static inline void arch_tlbbatch_flush(struct arch_tlb= flush_unmap_batch *batch) * operations can only span an even number of pages. We save this for l= ast to * ensure 64KB start alignment is maintained for the LPA2 case. */ +static __always_inline void rvae1is(u64 arg) +{ + __tlbi(rvae1is, arg); +} + +static __always_inline void rvale1(u64 arg) +{ + __tlbi(rvale1, arg); + __tlbi_user(rvale1, arg); +} + +static __always_inline void rvale1is(u64 arg) +{ + __tlbi(rvale1is, arg); +} + +static __always_inline void rvaale1is(u64 arg) +{ + __tlbi(rvaale1is, arg); +} + +static __always_inline void ripas2e1is(u64 arg) +{ + __tlbi(ripas2e1is, arg); +} + +static __always_inline void __tlbi_range(tlbi_op op, u64 arg) +{ + op(arg); +} + #define __flush_tlb_range_op(op, start, pages, stride, \ asid, tlb_level, tlbi_user, lpa2) \ do { \ @@ -486,7 +517,7 @@ do { \ if (num >=3D 0) { \ addr =3D __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \ scale, num, tlb_level); \ - __tlbi(r##op, addr); \ + __tlbi_range(r##op, addr); \ if (tlbi_user) \ __tlbi_user(r##op, addr); \ __flush_start +=3D __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ --=20 2.43.0 From nobody Thu Dec 18 23:23:41 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id CF26B34AAE3 for ; Tue, 16 Dec 2025 14:46:16 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896378; cv=none; b=i0kbEtD9aABXeiV/C2TLlKj6qbiLpjZs6Rh1EuBICIKuESfghr3FOMIbLn396PRMvO7GVDY/eQ5tQdxTbl1V1+5LxPAQd8zFMZ6cZY4odUfvM/UGDKuiRaH9Bh/1v4CMMYXc9FxIE5JTII+gFJp/zwXYeES8vuKeGdpAivF/Li0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896378; c=relaxed/simple; bh=Rt8IXD1w9orXK7J7oeBz6X4kSUQE5Zbrwz94/3U7VY4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=ScS09RIbPs86NzOpDjj94+20ZyMeOn1ao9bObIDpQtm2QmQLdgnWiGRZVVzdja90q+r2nkWXuGxHyB6r/QZZC8edCoCzRd2I1jIwZcSCEjG0QNiRkBCeVSecCTJZwTMCPHzWwUNNLB81Djhqv82d40+jSwVuSpu2/Ar/7c+liFA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 0C6BF1682; Tue, 16 Dec 2025 06:46:09 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id AE3353F73F; Tue, 16 Dec 2025 06:46:14 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v1 03/13] arm64: mm: Implicitly invalidate user ASID based on TLBI operation Date: Tue, 16 Dec 2025 14:45:48 +0000 Message-ID: <20251216144601.2106412-4-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20251216144601.2106412-1-ryan.roberts@arm.com> References: <20251216144601.2106412-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" When kpti is enabled, separate ASIDs are used for userspace and kernelspace, requiring ASID-qualified TLB invalidation by virtual address to invalidate both of them. Push the logic for invalidating the two ASIDs down into the low-level tlbi-op-specific functions and remove the burden from the caller to handle the kpti-specific behaviour. Co-developed-by: Will Deacon Signed-off-by: Will Deacon Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index c5111d2afc66..31f43d953ce2 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -110,6 +110,7 @@ typedef void (*tlbi_op)(u64 arg); static __always_inline void vae1is(u64 arg) { __tlbi(vae1is, arg); + __tlbi_user(vae1is, arg); } =20 static __always_inline void vae2is(u64 arg) @@ -126,6 +127,7 @@ static __always_inline void vale1(u64 arg) static __always_inline void vale1is(u64 arg) { __tlbi(vale1is, arg); + __tlbi_user(vale1is, arg); } =20 static __always_inline void vale2is(u64 arg) @@ -162,11 +164,6 @@ static __always_inline void __tlbi_level(tlbi_op op, u= 64 addr, u32 level) op(arg); } =20 -#define __tlbi_user_level(op, arg, level) do { \ - if (arm64_kernel_unmapped_at_el0()) \ - __tlbi_level(op, (arg | USER_ASID_FLAG), level); \ -} while (0) - /* * This macro creates a properly formatted VA operand for the TLB RANGE. T= he * value bit assignments are: @@ -435,8 +432,6 @@ static inline void arch_tlbbatch_flush(struct arch_tlbf= lush_unmap_batch *batch) * @stride: Flush granularity * @asid: The ASID of the task (0 for IPA instructions) * @tlb_level: Translation Table level hint, if known - * @tlbi_user: If 'true', call an additional __tlbi_user() - * (typically for user ASIDs). 'flase' for IPA instructions * @lpa2: If 'true', the lpa2 scheme is used as set out below * * When the CPU does not support TLB range operations, flush the TLB @@ -462,6 +457,7 @@ static inline void arch_tlbbatch_flush(struct arch_tlbf= lush_unmap_batch *batch) static __always_inline void rvae1is(u64 arg) { __tlbi(rvae1is, arg); + __tlbi_user(rvae1is, arg); } =20 static __always_inline void rvale1(u64 arg) @@ -473,6 +469,7 @@ static __always_inline void rvale1(u64 arg) static __always_inline void rvale1is(u64 arg) { __tlbi(rvale1is, arg); + __tlbi_user(rvale1is, arg); } =20 static __always_inline void rvaale1is(u64 arg) @@ -491,7 +488,7 @@ static __always_inline void __tlbi_range(tlbi_op op, u6= 4 arg) } =20 #define __flush_tlb_range_op(op, start, pages, stride, \ - asid, tlb_level, tlbi_user, lpa2) \ + asid, tlb_level, lpa2) \ do { \ typeof(start) __flush_start =3D start; \ typeof(pages) __flush_pages =3D pages; \ @@ -506,8 +503,6 @@ do { \ (lpa2 && __flush_start !=3D ALIGN(__flush_start, SZ_64K))) { \ addr =3D __TLBI_VADDR(__flush_start, asid); \ __tlbi_level(op, addr, tlb_level); \ - if (tlbi_user) \ - __tlbi_user_level(op, addr, tlb_level); \ __flush_start +=3D stride; \ __flush_pages -=3D stride >> PAGE_SHIFT; \ continue; \ @@ -518,8 +513,6 @@ do { \ addr =3D __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \ scale, num, tlb_level); \ __tlbi_range(r##op, addr); \ - if (tlbi_user) \ - __tlbi_user(r##op, addr); \ __flush_start +=3D __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ __flush_pages -=3D __TLBI_RANGE_PAGES(num, scale);\ } \ @@ -528,7 +521,7 @@ do { \ } while (0) =20 #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ - __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false, kvm_l= pa2_is_enabled()); + __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_= enabled()); =20 static inline bool __flush_tlb_range_limit_excess(unsigned long start, unsigned long end, unsigned long pages, unsigned long stride) @@ -568,10 +561,10 @@ static inline void __flush_tlb_range_nosync(struct mm= _struct *mm, =20 if (last_level) __flush_tlb_range_op(vale1is, start, pages, stride, asid, - tlb_level, true, lpa2_is_enabled()); + tlb_level, lpa2_is_enabled()); else __flush_tlb_range_op(vae1is, start, pages, stride, asid, - tlb_level, true, lpa2_is_enabled()); + tlb_level, lpa2_is_enabled()); =20 mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); } @@ -630,7 +623,7 @@ static inline void flush_tlb_kernel_range(unsigned long= start, unsigned long end =20 dsb(ishst); __flush_tlb_range_op(vaale1is, start, pages, stride, 0, - TLBI_TTL_UNKNOWN, false, lpa2_is_enabled()); + TLBI_TTL_UNKNOWN, lpa2_is_enabled()); dsb(ish); isb(); } @@ -681,6 +674,6 @@ static inline bool huge_pmd_needs_flush(pmd_t oldpmd, p= md_t newpmd) } #define huge_pmd_needs_flush huge_pmd_needs_flush =20 +#undef __tlbi_user #endif - #endif --=20 2.43.0 From nobody Thu Dec 18 23:23:41 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 34B9334B1A2 for ; Tue, 16 Dec 2025 14:46:18 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896380; cv=none; b=NQybGux/8+swjl0+I8ob+tFGNjaTZVzn5DWS4lLgx/4XtjOeBXan07vc48uCkiFX+oT+lqqMTRWeZQTz/LBHnv7y2fQbZlqcHRDZGLPCe0KTNSBo6auv3wDnEo9jUt0E8pqQpuIZX3DXNn/ks4/1VGvOIOW7LYTqXWfa2f5HwrM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896380; c=relaxed/simple; bh=6KJ9ZdJCyrYu0tSoIv3vYSs0YUs0z/Rh9pDuIhs2sN0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=jFtovh6F92IgDHiPwKRDZEfeU3IkDJwFn2S8PskCDZfo9HAyBh+qFxH1Q9xUNeuwPPhrDspKr9DAIwLofw59LADHF2aNGy0E8CuU2eAplLBI89k1kUtUOdXvZcCI+Spa9zHQjbi734BaJKCvZxGZEcNpHxWr8++Km2uozCMgscY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id C2434FEC; Tue, 16 Dec 2025 06:46:10 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 71F473F73F; Tue, 16 Dec 2025 06:46:16 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v1 04/13] arm64: mm: Push __TLBI_VADDR() into __tlbi_level() Date: Tue, 16 Dec 2025 14:45:49 +0000 Message-ID: <20251216144601.2106412-5-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20251216144601.2106412-1-ryan.roberts@arm.com> References: <20251216144601.2106412-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Will Deacon The __TLBI_VADDR() macro takes an ASID and an address and converts them into a single argument formatted correctly for a TLB invalidation instruction. Rather than have callers worry about this (especially in the case where the ASID is zero), push the macro down into __tlbi_level() via a new __tlbi_level_asid() helper. Signed-off-by: Will Deacon Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 14 ++++++++++---- arch/arm64/kernel/sys_compat.c | 2 +- arch/arm64/kvm/hyp/nvhe/mm.c | 2 +- arch/arm64/kvm/hyp/pgtable.c | 4 ++-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 31f43d953ce2..39717f98c31e 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -150,9 +150,10 @@ static __always_inline void ipas2e1is(u64 arg) __tlbi(ipas2e1is, arg); } =20 -static __always_inline void __tlbi_level(tlbi_op op, u64 addr, u32 level) +static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 le= vel, + u16 asid) { - u64 arg =3D addr; + u64 arg =3D __TLBI_VADDR(addr, asid); =20 if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && level <=3D 3) { u64 ttl =3D level | (get_trans_granule() << 2); @@ -164,6 +165,11 @@ static __always_inline void __tlbi_level(tlbi_op op, u= 64 addr, u32 level) op(arg); } =20 +static inline void __tlbi_level(tlbi_op op, u64 addr, u32 level) +{ + __tlbi_level_asid(op, addr, level, 0); +} + /* * This macro creates a properly formatted VA operand for the TLB RANGE. T= he * value bit assignments are: @@ -501,8 +507,7 @@ do { \ if (!system_supports_tlb_range() || \ __flush_pages =3D=3D 1 || \ (lpa2 && __flush_start !=3D ALIGN(__flush_start, SZ_64K))) { \ - addr =3D __TLBI_VADDR(__flush_start, asid); \ - __tlbi_level(op, addr, tlb_level); \ + __tlbi_level_asid(op, __flush_start, tlb_level, asid); \ __flush_start +=3D stride; \ __flush_pages -=3D stride >> PAGE_SHIFT; \ continue; \ @@ -675,5 +680,6 @@ static inline bool huge_pmd_needs_flush(pmd_t oldpmd, p= md_t newpmd) #define huge_pmd_needs_flush huge_pmd_needs_flush =20 #undef __tlbi_user +#undef __TLBI_VADDR #endif #endif diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c index 4a609e9b65de..ad4857df4830 100644 --- a/arch/arm64/kernel/sys_compat.c +++ b/arch/arm64/kernel/sys_compat.c @@ -36,7 +36,7 @@ __do_compat_cache_op(unsigned long start, unsigned long e= nd) * The workaround requires an inner-shareable tlbi. * We pick the reserved-ASID to minimise the impact. */ - __tlbi(aside1is, __TLBI_VADDR(0, 0)); + __tlbi(aside1is, 0UL); dsb(ish); } =20 diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index ae8391baebc3..581385b21826 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -270,7 +270,7 @@ static void fixmap_clear_slot(struct hyp_fixmap_slot *s= lot) * https://lore.kernel.org/kvm/20221017115209.2099-1-will@kernel.org/T/#m= f10dfbaf1eaef9274c581b81c53758918c1d0f03 */ dsb(ishst); - __tlbi_level(vale2is, __TLBI_VADDR(addr, 0), level); + __tlbi_level(vale2is, addr, level); dsb(ish); isb(); } diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 947ac1a951a5..9292c569afe6 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -472,14 +472,14 @@ static int hyp_unmap_walker(const struct kvm_pgtable_= visit_ctx *ctx, =20 kvm_clear_pte(ctx->ptep); dsb(ishst); - __tlbi_level(vae2is, __TLBI_VADDR(ctx->addr, 0), TLBI_TTL_UNKNOWN); + __tlbi_level(vae2is, ctx->addr, TLBI_TTL_UNKNOWN); } else { if (ctx->end - ctx->addr < granule) return -EINVAL; =20 kvm_clear_pte(ctx->ptep); dsb(ishst); - __tlbi_level(vale2is, __TLBI_VADDR(ctx->addr, 0), ctx->level); + __tlbi_level(vale2is, ctx->addr, ctx->level); *unmapped +=3D granule; } =20 --=20 2.43.0 From nobody Thu Dec 18 23:23:41 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id E56FF34BA5B for ; Tue, 16 Dec 2025 14:46:19 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896381; cv=none; b=hq8aRHk3kpzZyK4uuQbGDNhl3bCMDQbxlzNNtt3kHc0hh2+Qm2iKCXffi+etDq+GnJVyQBiPZRq7axmyJkQymMKsr04LthKgrMcelKOinbmw2bJW9mphYvk92JL65WEBMSEPihNfEk1PUSXZFm5hOxAslyAUbTHrQUt06bWCiQw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896381; c=relaxed/simple; bh=jmVPGmZ+90L9/D0w3QtNrGVf6f6Iwf0zgkxhMHOXNAY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=mK2dWI9cytr+rWmTIPckX1iWDSjmcajoAWa9B6lnw4kZx5HHKpfclqXvHiKZzde+wcMCP2TizG2653OC0OV54ucee4tENlJYi8qvUQNf6/j+U5ZkmQwlmPmuzKE/IE1wocBsgXETDiFZGBCe2zPXh1PU6x0xDaVf6bUbgVtErVY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 811D61684; Tue, 16 Dec 2025 06:46:12 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 30D133F73F; Tue, 16 Dec 2025 06:46:18 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v1 05/13] arm64: mm: Inline __TLBI_VADDR_RANGE() into __tlbi_range() Date: Tue, 16 Dec 2025 14:45:50 +0000 Message-ID: <20251216144601.2106412-6-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20251216144601.2106412-1-ryan.roberts@arm.com> References: <20251216144601.2106412-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Will Deacon The __TLBI_VADDR_RANGE() macro is only used in one place and isn't something that's generally useful outside of the low-level range invalidation gubbins. Inline __TLBI_VADDR_RANGE() into the __tlbi_range() function so that the macro can be removed entirely. Signed-off-by: Will Deacon Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 32 +++++++++++++------------------ 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 39717f98c31e..887dd1f05a89 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -195,19 +195,6 @@ static inline void __tlbi_level(tlbi_op op, u64 addr, = u32 level) #define TLBIR_TTL_MASK GENMASK_ULL(38, 37) #define TLBIR_BADDR_MASK GENMASK_ULL(36, 0) =20 -#define __TLBI_VADDR_RANGE(baddr, asid, scale, num, ttl) \ - ({ \ - unsigned long __ta =3D 0; \ - unsigned long __ttl =3D (ttl >=3D 1 && ttl <=3D 3) ? ttl : 0; \ - __ta |=3D FIELD_PREP(TLBIR_BADDR_MASK, baddr); \ - __ta |=3D FIELD_PREP(TLBIR_TTL_MASK, __ttl); \ - __ta |=3D FIELD_PREP(TLBIR_NUM_MASK, num); \ - __ta |=3D FIELD_PREP(TLBIR_SCALE_MASK, scale); \ - __ta |=3D FIELD_PREP(TLBIR_TG_MASK, get_trans_granule()); \ - __ta |=3D FIELD_PREP(TLBIR_ASID_MASK, asid); \ - __ta; \ - }) - /* These macros are used by the TLBI RANGE feature. */ #define __TLBI_RANGE_PAGES(num, scale) \ ((unsigned long)((num) + 1) << (5 * (scale) + 1)) @@ -488,8 +475,19 @@ static __always_inline void ripas2e1is(u64 arg) __tlbi(ripas2e1is, arg); } =20 -static __always_inline void __tlbi_range(tlbi_op op, u64 arg) +static __always_inline void __tlbi_range(tlbi_op op, u64 addr, + u16 asid, int scale, int num, + u32 level, bool lpa2) { + u64 arg =3D 0; + + arg |=3D FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : PAGE_SHIFT)); + arg |=3D FIELD_PREP(TLBIR_TTL_MASK, level > 3 ? 0 : level); + arg |=3D FIELD_PREP(TLBIR_NUM_MASK, num); + arg |=3D FIELD_PREP(TLBIR_SCALE_MASK, scale); + arg |=3D FIELD_PREP(TLBIR_TG_MASK, get_trans_granule()); + arg |=3D FIELD_PREP(TLBIR_ASID_MASK, asid); + op(arg); } =20 @@ -500,8 +498,6 @@ do { \ typeof(pages) __flush_pages =3D pages; \ int num =3D 0; \ int scale =3D 3; \ - int shift =3D lpa2 ? 16 : PAGE_SHIFT; \ - unsigned long addr; \ \ while (__flush_pages > 0) { \ if (!system_supports_tlb_range() || \ @@ -515,9 +511,7 @@ do { \ \ num =3D __TLBI_RANGE_NUM(__flush_pages, scale); \ if (num >=3D 0) { \ - addr =3D __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \ - scale, num, tlb_level); \ - __tlbi_range(r##op, addr); \ + __tlbi_range(r##op, __flush_start, asid, scale, num, tlb_level, lpa2); \ __flush_start +=3D __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ __flush_pages -=3D __TLBI_RANGE_PAGES(num, scale);\ } \ --=20 2.43.0 From nobody Thu Dec 18 23:23:41 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 97BBD34C9AB for ; Tue, 16 Dec 2025 14:46:21 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896383; cv=none; b=JxEqGVR+CGlzM5gEuTaP2A8wFY3kkK/iO4on+0qpt8f4Am6jHElXLxwCfweKSdwmVkSs3Od9m+H+fh40eK+x7tCbjDZdP4R9UZlziG4FrNPBmp7DlEPefXIjApFF2Zw+Bwqp+42POJk60UZouM3DAmw8XqR2siKzdl2e2/UwzNc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896383; c=relaxed/simple; bh=hbZvsdl75oi0GW1FQDlf1P8U3yeh/ivdZiAqt9xkZE4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=Z1Flbluw47Kr+8AgOT1zxfI5WYqkaDnRyiyVvt1VBOFGwj8eMY0X9nMiqjE700zKfDYeJqtSeQ9aVtKc+x482lQQQFja0OCx+6YjNchDIPNgkC1j4Javk5x4jQXK90iCwM5xebCqBktXss8XvfkL5F1rQCfH6WPDdgDC9Egea5I= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 3CD72168F; Tue, 16 Dec 2025 06:46:14 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id E43D93F73F; Tue, 16 Dec 2025 06:46:19 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v1 06/13] arm64: mm: Re-implement the __flush_tlb_range_op macro in C Date: Tue, 16 Dec 2025 14:45:51 +0000 Message-ID: <20251216144601.2106412-7-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20251216144601.2106412-1-ryan.roberts@arm.com> References: <20251216144601.2106412-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The __flush_tlb_range_op() macro is horrible and has been a previous source of bugs thanks to multiple expansions of its arguments (see commit f7edb07ad7c6 ("Fix mmu notifiers for range-based invalidates")). Rewrite the thing in C. Suggested-by: Linus Torvalds Co-developed-by: Will Deacon Signed-off-by: Will Deacon Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 84 +++++++++++++++++-------------- 1 file changed, 46 insertions(+), 38 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 887dd1f05a89..d2a144a09a8f 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -419,12 +419,13 @@ static inline void arch_tlbbatch_flush(struct arch_tl= bflush_unmap_batch *batch) /* * __flush_tlb_range_op - Perform TLBI operation upon a range * - * @op: TLBI instruction that operates on a range (has 'r' prefix) + * @lop: TLBI level operation to perform + * @rop: TLBI range operation to perform * @start: The start address of the range * @pages: Range as the number of pages from 'start' * @stride: Flush granularity * @asid: The ASID of the task (0 for IPA instructions) - * @tlb_level: Translation Table level hint, if known + * @level: Translation Table level hint, if known * @lpa2: If 'true', the lpa2 scheme is used as set out below * * When the CPU does not support TLB range operations, flush the TLB @@ -491,36 +492,44 @@ static __always_inline void __tlbi_range(tlbi_op op, = u64 addr, op(arg); } =20 -#define __flush_tlb_range_op(op, start, pages, stride, \ - asid, tlb_level, lpa2) \ -do { \ - typeof(start) __flush_start =3D start; \ - typeof(pages) __flush_pages =3D pages; \ - int num =3D 0; \ - int scale =3D 3; \ - \ - while (__flush_pages > 0) { \ - if (!system_supports_tlb_range() || \ - __flush_pages =3D=3D 1 || \ - (lpa2 && __flush_start !=3D ALIGN(__flush_start, SZ_64K))) { \ - __tlbi_level_asid(op, __flush_start, tlb_level, asid); \ - __flush_start +=3D stride; \ - __flush_pages -=3D stride >> PAGE_SHIFT; \ - continue; \ - } \ - \ - num =3D __TLBI_RANGE_NUM(__flush_pages, scale); \ - if (num >=3D 0) { \ - __tlbi_range(r##op, __flush_start, asid, scale, num, tlb_level, lpa2); \ - __flush_start +=3D __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ - __flush_pages -=3D __TLBI_RANGE_PAGES(num, scale);\ - } \ - scale--; \ - } \ -} while (0) +static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, + u64 start, size_t pages, + u64 stride, u16 asid, + u32 level, bool lpa2) +{ + u64 addr =3D start, end =3D start + pages * PAGE_SIZE; + int scale =3D 3; + + while (addr !=3D end) { + int num; + + pages =3D (end - addr) >> PAGE_SHIFT; + + if (!system_supports_tlb_range() || pages =3D=3D 1) + goto invalidate_one; + + if (lpa2 && !IS_ALIGNED(addr, SZ_64K)) + goto invalidate_one; + + num =3D __TLBI_RANGE_NUM(pages, scale); + if (num >=3D 0) { + __tlbi_range(rop, addr, asid, scale, num, level, lpa2); + addr +=3D __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; + } + + scale--; + continue; +invalidate_one: + __tlbi_level_asid(lop, addr, level, asid); + addr +=3D stride; + } +} + +#define __flush_s1_tlb_range_op(op, start, pages, stride, asid, tlb_level)= \ + __flush_tlb_range_op(op, r##op, start, pages, stride, asid, tlb_level, lp= a2_is_enabled()) =20 #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ - __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_= enabled()); + __flush_tlb_range_op(op, r##op, start, pages, stride, 0, tlb_level, kvm_l= pa2_is_enabled()) =20 static inline bool __flush_tlb_range_limit_excess(unsigned long start, unsigned long end, unsigned long pages, unsigned long stride) @@ -559,11 +568,11 @@ static inline void __flush_tlb_range_nosync(struct mm= _struct *mm, asid =3D ASID(mm); =20 if (last_level) - __flush_tlb_range_op(vale1is, start, pages, stride, asid, - tlb_level, lpa2_is_enabled()); + __flush_s1_tlb_range_op(vale1is, start, pages, stride, + asid, tlb_level); else - __flush_tlb_range_op(vae1is, start, pages, stride, asid, - tlb_level, lpa2_is_enabled()); + __flush_s1_tlb_range_op(vae1is, start, pages, stride, + asid, tlb_level); =20 mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); } @@ -587,8 +596,7 @@ static inline void local_flush_tlb_contpte(struct vm_ar= ea_struct *vma, =20 dsb(nshst); asid =3D ASID(vma->vm_mm); - __flush_tlb_range_op(vale1, addr, CONT_PTES, PAGE_SIZE, asid, - 3, true, lpa2_is_enabled()); + __flush_s1_tlb_range_op(vale1, addr, CONT_PTES, PAGE_SIZE, asid, 3); mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, addr, addr + CONT_PTE_SIZE); dsb(nsh); @@ -621,8 +629,8 @@ static inline void flush_tlb_kernel_range(unsigned long= start, unsigned long end } =20 dsb(ishst); - __flush_tlb_range_op(vaale1is, start, pages, stride, 0, - TLBI_TTL_UNKNOWN, lpa2_is_enabled()); + __flush_s1_tlb_range_op(vaale1is, start, pages, stride, 0, + TLBI_TTL_UNKNOWN); dsb(ish); isb(); } --=20 2.43.0 From nobody Thu Dec 18 23:23:41 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id B623934D4CA for ; Tue, 16 Dec 2025 14:46:23 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896385; cv=none; b=iGYkx27yzpimM9uB4e69iCg29M0vxv4UCjYkTDgydATt7y2CCIGP5M6hPYxB/DQ2OXhnopAKpPshBeH/iKoEU/AbrtUvCrAT2yJ0v+cG1Jp2ll/uVf3H6/zujafgTcE8TuNHCU3XL+wA8oz6SQRY6b9o+01/JJbffdEqsD7pxOg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896385; c=relaxed/simple; bh=gAIubC0DksD1OzYXQeCMVrJfdUI4I7LTg5qvfpFqonc=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=iBnP/R4MgXcp59GVvd7vlbug9eG7BoAXbYPO6xcBhphj4PoKx+caLvF9edGADl7r7Zp3d9Qemz9UTnbFdUpc6+RbUIYlMkErbTcmtXFdVy/+6SXVQ3CutKqjmZBe7NevKd31WOHrJt37fwsd9SG2b7EHEgqq5GjVgUPKr5zmjdo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id F23A91655; Tue, 16 Dec 2025 06:46:15 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id A2F173F73F; Tue, 16 Dec 2025 06:46:21 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v1 07/13] arm64: mm: Simplify __TLBI_RANGE_NUM() macro Date: Tue, 16 Dec 2025 14:45:52 +0000 Message-ID: <20251216144601.2106412-8-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20251216144601.2106412-1-ryan.roberts@arm.com> References: <20251216144601.2106412-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Will Deacon Since commit e2768b798a19 ("arm64/mm: Modify range-based tlbi to decrement scale"), we don't need to clamp the 'pages' argument to fit the range for the specified 'scale' as we know that the upper bits will have been processed in a prior iteration. Drop the clamping and simplify the __TLBI_RANGE_NUM() macro. Signed-off-by: Will Deacon Reviewed-by: Ryan Roberts Reviewed-by: Dev Jain Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index d2a144a09a8f..0e1902f66e01 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -208,11 +208,7 @@ static inline void __tlbi_level(tlbi_op op, u64 addr, = u32 level) * range. */ #define __TLBI_RANGE_NUM(pages, scale) \ - ({ \ - int __pages =3D min((pages), \ - __TLBI_RANGE_PAGES(31, (scale))); \ - (__pages >> (5 * (scale) + 1)) - 1; \ - }) + (((pages) >> (5 * (scale) + 1)) - 1) =20 /* * TLB Invalidation --=20 2.43.0 From nobody Thu Dec 18 23:23:41 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 501EB34D4F5 for ; Tue, 16 Dec 2025 14:46:25 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896388; cv=none; b=mPKufzgYMTlxLGEYnYdyaMZfAJtwhAHjGqCrCrg3DXyfiSkegHCPM5wSs0/ik3cyPyJQ4eNi+Ll4EqoeO6IFORIrqP2Cbvqtw0tLTWhsOGojr968ndeSMg0Au+KgkaVruQ/PRGD8e8B7tj9JpqK/sRBGNCiCEBcOW5pdLzK3FLc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896388; c=relaxed/simple; bh=x6HDdQOqqbTmmUwD8mIyoGgVh3LJqaeQY3P+aCh6RKw=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=EL3OQEIIGwqUO2b8hFuw3TaVskWWP5kPIHV1XnF+zTB+XYIHrqIpem0R0WoeegUuv79aszJqyqsL5mShhP8k2Y9//FjsJLTY5dCJgIORepF6EZQwAoyG9MZNpzSuV3ibbjTyO2ANEPReMTV3Q/c/qPlPdLg2JZT36mayp8MwfG4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id B55271691; Tue, 16 Dec 2025 06:46:17 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 612B13F73F; Tue, 16 Dec 2025 06:46:23 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v1 08/13] arm64: mm: Simplify __flush_tlb_range_limit_excess() Date: Tue, 16 Dec 2025 14:45:53 +0000 Message-ID: <20251216144601.2106412-9-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20251216144601.2106412-1-ryan.roberts@arm.com> References: <20251216144601.2106412-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: Will Deacon __flush_tlb_range_limit_excess() is unnecessarily complicated: - It takes a 'start', 'end' and 'pages' argument, whereas it only needs 'pages' (which the caller has computed from the other two arguments!). - It erroneously compares 'pages' with MAX_TLBI_RANGE_PAGES when the system doesn't support range-based invalidation but the range to be invalidated would result in fewer than MAX_DVM_OPS invalidations. Simplify the function so that it no longer takes the 'start' and 'end' arguments and only considers the MAX_TLBI_RANGE_PAGES threshold on systems that implement range-based invalidation. Signed-off-by: Will Deacon Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 0e1902f66e01..3b72a71feac0 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -527,21 +527,13 @@ static __always_inline void __flush_tlb_range_op(tlbi= _op lop, tlbi_op rop, #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ __flush_tlb_range_op(op, r##op, start, pages, stride, 0, tlb_level, kvm_l= pa2_is_enabled()) =20 -static inline bool __flush_tlb_range_limit_excess(unsigned long start, - unsigned long end, unsigned long pages, unsigned long stride) +static inline bool __flush_tlb_range_limit_excess(unsigned long pages, + unsigned long stride) { - /* - * When the system does not support TLB range based flush - * operation, (MAX_DVM_OPS - 1) pages can be handled. But - * with TLB range based operation, MAX_TLBI_RANGE_PAGES - * pages can be handled. - */ - if ((!system_supports_tlb_range() && - (end - start) >=3D (MAX_DVM_OPS * stride)) || - pages > MAX_TLBI_RANGE_PAGES) + if (system_supports_tlb_range() && pages > MAX_TLBI_RANGE_PAGES) return true; =20 - return false; + return pages >=3D (MAX_DVM_OPS * stride) >> PAGE_SHIFT; } =20 static inline void __flush_tlb_range_nosync(struct mm_struct *mm, @@ -555,7 +547,7 @@ static inline void __flush_tlb_range_nosync(struct mm_s= truct *mm, end =3D round_up(end, stride); pages =3D (end - start) >> PAGE_SHIFT; =20 - if (__flush_tlb_range_limit_excess(start, end, pages, stride)) { + if (__flush_tlb_range_limit_excess(pages, stride)) { flush_tlb_mm(mm); return; } @@ -619,7 +611,7 @@ static inline void flush_tlb_kernel_range(unsigned long= start, unsigned long end end =3D round_up(end, stride); pages =3D (end - start) >> PAGE_SHIFT; =20 - if (__flush_tlb_range_limit_excess(start, end, pages, stride)) { + if (__flush_tlb_range_limit_excess(pages, stride)) { flush_tlb_all(); return; } --=20 2.43.0 From nobody Thu Dec 18 23:23:41 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 4A82934D91D for ; Tue, 16 Dec 2025 14:46:27 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896389; cv=none; b=ClhCKSF7m7WrE5gHATHViGPCv9ij7CVHSMQHvBUF5AaLd/6Oy4k5PFKBJvAT8FmYyVz7R71xjNNzTV5BOkivQ0h/Mn5xCMpG0wjMO1K5RcE3maExfxH9fpzKfGylQgNhWcosQqyoXXKdE5k27MNSu0Y75EZXzCkXC4EC5c3i2F0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896389; c=relaxed/simple; bh=C9wgH+fvywkbgyqcmHut+F7YFtPR8WBM2s9b+vkDiy0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=uIA2YXzio5prCOt6jxfmLSOo+TEM+2QQHbVnPCnakLa6QofeE8kUKL1kKNcXXLgVXorQ82Zuo93YWNl1mhR2yHL2SdVAkZfYfiqgenjQo3Js8dNX66y+MLh+d8xzQZBFc8yYPUy7ZUvCoGeacBGu0U/Y8lcnr/X1BxQr5ghiwBg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 73F281516; Tue, 16 Dec 2025 06:46:19 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 23F2B3F73F; Tue, 16 Dec 2025 06:46:25 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v1 09/13] arm64: mm: Refactor flush_tlb_page() to use __tlbi_level_asid() Date: Tue, 16 Dec 2025 14:45:54 +0000 Message-ID: <20251216144601.2106412-10-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20251216144601.2106412-1-ryan.roberts@arm.com> References: <20251216144601.2106412-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Now that we have __tlbi_level_asid(), let's refactor the *flush_tlb_page*() variants to use it rather than open coding. The emitted tlbi(s) is/are intended to be exactly the same as before; no TTL hint is provided. Although the spec for flush_tlb_page() allows for setting the TTL hint to 3, it turns out that flush_tlb_fix_spurious_fault_pmd() depends on local_flush_tlb_page_nonotify() to invalidate the level 2 entry. This will be fixed separately. Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 3b72a71feac0..37c782ddc149 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -327,12 +327,8 @@ static inline void flush_tlb_mm(struct mm_struct *mm) static inline void __local_flush_tlb_page_nonotify_nosync(struct mm_struct= *mm, unsigned long uaddr) { - unsigned long addr; - dsb(nshst); - addr =3D __TLBI_VADDR(uaddr, ASID(mm)); - __tlbi(vale1, addr); - __tlbi_user(vale1, addr); + __tlbi_level_asid(vale1, uaddr, TLBI_TTL_UNKNOWN, ASID(mm)); } =20 static inline void local_flush_tlb_page_nonotify(struct vm_area_struct *vm= a, @@ -354,12 +350,8 @@ static inline void local_flush_tlb_page(struct vm_area= _struct *vma, static inline void __flush_tlb_page_nosync(struct mm_struct *mm, unsigned long uaddr) { - unsigned long addr; - dsb(ishst); - addr =3D __TLBI_VADDR(uaddr, ASID(mm)); - __tlbi(vale1is, addr); - __tlbi_user(vale1is, addr); + __tlbi_level_asid(vale1is, uaddr, TLBI_TTL_UNKNOWN, ASID(mm)); mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK, (uaddr & PAGE_MASK) + PAGE_SIZE); } --=20 2.43.0 From nobody Thu Dec 18 23:23:41 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 8DF6C34D93A for ; Tue, 16 Dec 2025 14:46:29 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896392; cv=none; b=iuA7w5+fznshuwuaity3pERVE7UmvhyO5iMeMDl0dJZTfG2qMZ7fpz2X92y567ChMYDdpxSK2RpzN+6vilZm7QkG6+C4y3f0X2HZxBdMPxM9hBkrpn7CDePic9jTDFf/3fqF+BSc37KAGh9EpZd2rTZ91kOOP9tjfVxaxqH14tI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896392; c=relaxed/simple; bh=9odixVVGGsqqGd54zDTUdYqgEXxkXpXo5Sntibel13I=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=MFOHqJmBMXka11Xrd/KuzlsBKtlGBhw5u/+Xsh7Sw7PfyI1uUSmIGDaEhpfpTjBA3fcxhIprZniMvldYLUgzHn12JLHJBeToSFPmRocwG22OUWOLz9Rz4MES4FVwnRn1KrH3dBuJHyPfPKTbh3gUUhSWKxyaUxsIb0BiDDk0HoU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 4F2491682; Tue, 16 Dec 2025 06:46:21 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id D99463F73F; Tue, 16 Dec 2025 06:46:26 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v1 10/13] arm64: mm: Refactor __flush_tlb_range() to take flags Date: Tue, 16 Dec 2025 14:45:55 +0000 Message-ID: <20251216144601.2106412-11-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20251216144601.2106412-1-ryan.roberts@arm.com> References: <20251216144601.2106412-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" We have function variants with "_nosync", "_local", "_nonotify" as well as the "last_level" parameter. Let's generalize and simplify by using a flags parameter to encode all these variants. As a first step, convert the "last_level" boolean parameter to a flags parameter and create the first flag, TLBF_NOWALKCACHE. When present, walk cache entries are not evicted, which is the same as the old last_level=3Dtrue. Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/hugetlb.h | 12 ++++++------ arch/arm64/include/asm/pgtable.h | 4 ++-- arch/arm64/include/asm/tlb.h | 6 +++--- arch/arm64/include/asm/tlbflush.h | 28 ++++++++++++++++------------ arch/arm64/mm/contpte.c | 5 +++-- arch/arm64/mm/hugetlbpage.c | 4 ++-- arch/arm64/mm/mmu.c | 2 +- 7 files changed, 33 insertions(+), 28 deletions(-) diff --git a/arch/arm64/include/asm/hugetlb.h b/arch/arm64/include/asm/huge= tlb.h index 44c1f757bfcf..04af9499faf2 100644 --- a/arch/arm64/include/asm/hugetlb.h +++ b/arch/arm64/include/asm/hugetlb.h @@ -73,23 +73,23 @@ static inline void __flush_hugetlb_tlb_range(struct vm_= area_struct *vma, unsigned long start, unsigned long end, unsigned long stride, - bool last_level) + tlbf_t flags) { switch (stride) { #ifndef __PAGETABLE_PMD_FOLDED case PUD_SIZE: - __flush_tlb_range(vma, start, end, PUD_SIZE, last_level, 1); + __flush_tlb_range(vma, start, end, PUD_SIZE, 1, flags); break; #endif case CONT_PMD_SIZE: case PMD_SIZE: - __flush_tlb_range(vma, start, end, PMD_SIZE, last_level, 2); + __flush_tlb_range(vma, start, end, PMD_SIZE, 2, flags); break; case CONT_PTE_SIZE: - __flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, 3); + __flush_tlb_range(vma, start, end, PAGE_SIZE, 3, flags); break; default: - __flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, TLBI_TTL_UNKNO= WN); + __flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, flags); } } =20 @@ -100,7 +100,7 @@ static inline void flush_hugetlb_tlb_range(struct vm_ar= ea_struct *vma, { unsigned long stride =3D huge_page_size(hstate_vma(vma)); =20 - __flush_hugetlb_tlb_range(vma, start, end, stride, false); + __flush_hugetlb_tlb_range(vma, start, end, stride, TLBF_NONE); } =20 #endif /* __ASM_HUGETLB_H */ diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgta= ble.h index 64d5f1d9cce9..736747fbc843 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -124,9 +124,9 @@ static inline void arch_leave_lazy_mmu_mode(void) =20 /* Set stride and tlb_level in flush_*_tlb_range */ #define flush_pmd_tlb_range(vma, addr, end) \ - __flush_tlb_range(vma, addr, end, PMD_SIZE, false, 2) + __flush_tlb_range(vma, addr, end, PMD_SIZE, 2, TLBF_NONE) #define flush_pud_tlb_range(vma, addr, end) \ - __flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1) + __flush_tlb_range(vma, addr, end, PUD_SIZE, 1, TLBF_NONE) #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ =20 /* diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index 8d762607285c..10869d7731b8 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -53,7 +53,7 @@ static inline int tlb_get_level(struct mmu_gather *tlb) static inline void tlb_flush(struct mmu_gather *tlb) { struct vm_area_struct vma =3D TLB_FLUSH_VMA(tlb->mm, 0); - bool last_level =3D !tlb->freed_tables; + tlbf_t flags =3D tlb->freed_tables ? TLBF_NONE : TLBF_NOWALKCACHE; unsigned long stride =3D tlb_get_unmap_size(tlb); int tlb_level =3D tlb_get_level(tlb); =20 @@ -63,13 +63,13 @@ static inline void tlb_flush(struct mmu_gather *tlb) * reallocate our ASID without invalidating the entire TLB. */ if (tlb->fullmm) { - if (!last_level) + if (tlb->freed_tables) flush_tlb_mm(tlb->mm); return; } =20 __flush_tlb_range(&vma, tlb->start, tlb->end, stride, - last_level, tlb_level); + tlb_level, flags); } =20 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 37c782ddc149..9a37a6a014dc 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -267,16 +267,16 @@ static inline void __tlbi_level(tlbi_op op, u64 addr,= u32 level) * CPUs, ensuring that any walk-cache entries associated with the * translation are also invalidated. * - * __flush_tlb_range(vma, start, end, stride, last_level, tlb_level) + * __flush_tlb_range(vma, start, end, stride, last_level, tlb_level, flags) * Invalidate the virtual-address range '[start, end)' on all * CPUs for the user address space corresponding to 'vma->mm'. * The invalidation operations are issued at a granularity - * determined by 'stride' and only affect any walk-cache entries - * if 'last_level' is equal to false. tlb_level is the level at + * determined by 'stride'. tlb_level is the level at * which the invalidation must take place. If the level is wrong, * no invalidation may take place. In the case where the level * cannot be easily determined, the value TLBI_TTL_UNKNOWN will - * perform a non-hinted invalidation. + * perform a non-hinted invalidation. flags may be TLBF_NONE (0) or + * TLBF_NOWALKCACHE (elide eviction of walk cache entries). * * local_flush_tlb_page(vma, addr) * Local variant of flush_tlb_page(). Stale TLB entries may @@ -528,10 +528,14 @@ static inline bool __flush_tlb_range_limit_excess(uns= igned long pages, return pages >=3D (MAX_DVM_OPS * stride) >> PAGE_SHIFT; } =20 +typedef unsigned __bitwise tlbf_t; +#define TLBF_NONE ((__force tlbf_t)0) +#define TLBF_NOWALKCACHE ((__force tlbf_t)BIT(0)) + static inline void __flush_tlb_range_nosync(struct mm_struct *mm, unsigned long start, unsigned long end, - unsigned long stride, bool last_level, - int tlb_level) + unsigned long stride, int tlb_level, + tlbf_t flags) { unsigned long asid, pages; =20 @@ -547,7 +551,7 @@ static inline void __flush_tlb_range_nosync(struct mm_s= truct *mm, dsb(ishst); asid =3D ASID(mm); =20 - if (last_level) + if (flags & TLBF_NOWALKCACHE) __flush_s1_tlb_range_op(vale1is, start, pages, stride, asid, tlb_level); else @@ -559,11 +563,11 @@ static inline void __flush_tlb_range_nosync(struct mm= _struct *mm, =20 static inline void __flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, - unsigned long stride, bool last_level, - int tlb_level) + unsigned long stride, int tlb_level, + tlbf_t flags) { __flush_tlb_range_nosync(vma->vm_mm, start, end, stride, - last_level, tlb_level); + tlb_level, flags); dsb(ish); } =20 @@ -591,7 +595,7 @@ static inline void flush_tlb_range(struct vm_area_struc= t *vma, * Set the tlb_level to TLBI_TTL_UNKNOWN because we can not get enough * information here. */ - __flush_tlb_range(vma, start, end, PAGE_SIZE, false, TLBI_TTL_UNKNOWN); + __flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE= ); } =20 static inline void flush_tlb_kernel_range(unsigned long start, unsigned lo= ng end) @@ -632,7 +636,7 @@ static inline void __flush_tlb_kernel_pgtable(unsigned = long kaddr) static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_ba= tch *batch, struct mm_struct *mm, unsigned long start, unsigned long end) { - __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3); + __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, 3, TLBF_NOWALKCACHE); } =20 static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval) diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index 589bcf878938..1a12bb728ee1 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -205,7 +205,8 @@ static void contpte_convert(struct mm_struct *mm, unsig= ned long addr, */ =20 if (!system_supports_bbml2_noabort()) - __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, true, 3); + __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, 3, + TLBF_NOWALKCACHE); =20 __set_ptes(mm, start_addr, start_ptep, pte, CONT_PTES); } @@ -527,7 +528,7 @@ int contpte_ptep_clear_flush_young(struct vm_area_struc= t *vma, */ addr =3D ALIGN_DOWN(addr, CONT_PTE_SIZE); __flush_tlb_range_nosync(vma->vm_mm, addr, addr + CONT_PTE_SIZE, - PAGE_SIZE, true, 3); + PAGE_SIZE, 3, TLBF_NOWALKCACHE); } =20 return young; diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index 1d90a7e75333..7b95663f8c76 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -184,7 +184,7 @@ static pte_t get_clear_contig_flush(struct mm_struct *m= m, struct vm_area_struct vma =3D TLB_FLUSH_VMA(mm, 0); unsigned long end =3D addr + (pgsize * ncontig); =20 - __flush_hugetlb_tlb_range(&vma, addr, end, pgsize, true); + __flush_hugetlb_tlb_range(&vma, addr, end, pgsize, TLBF_NOWALKCACHE); return orig_pte; } =20 @@ -212,7 +212,7 @@ static void clear_flush(struct mm_struct *mm, if (mm =3D=3D &init_mm) flush_tlb_kernel_range(saddr, addr); else - __flush_hugetlb_tlb_range(&vma, saddr, addr, pgsize, true); + __flush_hugetlb_tlb_range(&vma, saddr, addr, pgsize, TLBF_NOWALKCACHE); } =20 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 9ae7ce00a7ef..a17d617a959a 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -2150,7 +2150,7 @@ pte_t modify_prot_start_ptes(struct vm_area_struct *v= ma, unsigned long addr, */ if (pte_accessible(vma->vm_mm, pte) && pte_user_exec(pte)) __flush_tlb_range(vma, addr, nr * PAGE_SIZE, - PAGE_SIZE, true, 3); + PAGE_SIZE, 3, TLBF_NOWALKCACHE); } =20 return pte; --=20 2.43.0 From nobody Thu Dec 18 23:23:41 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 7E8E2347FEA for ; Tue, 16 Dec 2025 14:46:30 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896393; cv=none; b=Za7Ox/CNtX3m+Y/Opi+MZ999M3r9cyZdhKywfLEpkmISScl1vGc7TpI3B+hOzA727t8LTbkJy7AsnFLxMik+CZVGpdnCNJgurL5P0LFekX50nYokkwTQ2xvng6MvkXsHE/qCIPxW0L+Qw8fDTphhQv4XL1US5Fl07XDCVbMVQ0k= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896393; c=relaxed/simple; bh=8yXJQ/B0lInYzZUTkpMXr4o2aC/2IU3A56X/bgvJJp8=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=BF7pqhwbjIuLtw1z5bNdXYe3FLN7ERdCsNuY9LfZur1xIwNmxujfZcaxeOWwoKIZzzahb7g5QIKi1Zq6T5zchgafE1XFRgUuqotz2hpm8boA4nm9hAbtowWDnbUrhreJ2iejg5jdGuRt/L3+aDIMM+JXqx2OTI5ylpFEfAI464o= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 0EBC9FEC; Tue, 16 Dec 2025 06:46:23 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id B28803F73F; Tue, 16 Dec 2025 06:46:28 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v1 11/13] arm64: mm: More flags for __flush_tlb_range() Date: Tue, 16 Dec 2025 14:45:56 +0000 Message-ID: <20251216144601.2106412-12-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20251216144601.2106412-1-ryan.roberts@arm.com> References: <20251216144601.2106412-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Refactor function variants with "_nosync", "_local" and "_nonotify" into a single __always_inline implementation that takes flags and rely on constant folding to select the parts that are actually needed at any given callsite, based on the provided flags. Flags all live in the tlbf_t (TLB flags) type; TLBF_NONE (0) continues to provide the strongest semantics (i.e. evict from walk cache, broadcast, synchronise and notify). Each flag reduces the strength in some way; TLBF_NONOTIFY, TLBF_NOSYNC and TLBF_NOBROADCAST are added to complement the existing TLBF_NOWALKCACHE. The result is a clearer, simpler, more powerful API. Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/tlbflush.h | 101 ++++++++++++++++++------------ arch/arm64/mm/contpte.c | 9 ++- 2 files changed, 68 insertions(+), 42 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index 9a37a6a014dc..ee747e66bbef 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -107,6 +107,12 @@ static inline unsigned long get_trans_granule(void) =20 typedef void (*tlbi_op)(u64 arg); =20 +static __always_inline void vae1(u64 arg) +{ + __tlbi(vae1, arg); + __tlbi_user(vae1, arg); +} + static __always_inline void vae1is(u64 arg) { __tlbi(vae1is, arg); @@ -276,7 +282,10 @@ static inline void __tlbi_level(tlbi_op op, u64 addr, = u32 level) * no invalidation may take place. In the case where the level * cannot be easily determined, the value TLBI_TTL_UNKNOWN will * perform a non-hinted invalidation. flags may be TLBF_NONE (0) or - * TLBF_NOWALKCACHE (elide eviction of walk cache entries). + * any combination of TLBF_NOWALKCACHE (elide eviction of walk + * cache entries), TLBF_NONOTIFY (don't call mmu notifiers), + * TLBF_NOSYNC (don't issue trailing dsb) and TLBF_NOBROADCAST + * (only perform the invalidation for the local cpu). * * local_flush_tlb_page(vma, addr) * Local variant of flush_tlb_page(). Stale TLB entries may @@ -286,12 +295,6 @@ static inline void __tlbi_level(tlbi_op op, u64 addr, = u32 level) * Same as local_flush_tlb_page() except MMU notifier will not be * called. * - * local_flush_tlb_contpte(vma, addr) - * Invalidate the virtual-address range - * '[addr, addr+CONT_PTE_SIZE)' mapped with contpte on local CPU - * for the user address space corresponding to 'vma->mm'. Stale - * TLB entries may remain in remote CPUs. - * * Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented * on top of these routines, since that is our interface to the mmu_gather * API as used by munmap() and friends. @@ -436,6 +439,12 @@ static inline void arch_tlbbatch_flush(struct arch_tlb= flush_unmap_batch *batch) * operations can only span an even number of pages. We save this for l= ast to * ensure 64KB start alignment is maintained for the LPA2 case. */ +static __always_inline void rvae1(u64 arg) +{ + __tlbi(rvae1, arg); + __tlbi_user(rvae1, arg); +} + static __always_inline void rvae1is(u64 arg) { __tlbi(rvae1is, arg); @@ -531,16 +540,18 @@ static inline bool __flush_tlb_range_limit_excess(uns= igned long pages, typedef unsigned __bitwise tlbf_t; #define TLBF_NONE ((__force tlbf_t)0) #define TLBF_NOWALKCACHE ((__force tlbf_t)BIT(0)) +#define TLBF_NOSYNC ((__force tlbf_t)BIT(1)) +#define TLBF_NONOTIFY ((__force tlbf_t)BIT(2)) +#define TLBF_NOBROADCAST ((__force tlbf_t)BIT(3)) =20 -static inline void __flush_tlb_range_nosync(struct mm_struct *mm, - unsigned long start, unsigned long end, - unsigned long stride, int tlb_level, - tlbf_t flags) +static __always_inline void ___flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end, + unsigned long stride, int tlb_level, + tlbf_t flags) { + struct mm_struct *mm =3D vma->vm_mm; unsigned long asid, pages; =20 - start =3D round_down(start, stride); - end =3D round_up(end, stride); pages =3D (end - start) >> PAGE_SHIFT; =20 if (__flush_tlb_range_limit_excess(pages, stride)) { @@ -548,17 +559,41 @@ static inline void __flush_tlb_range_nosync(struct mm= _struct *mm, return; } =20 - dsb(ishst); + if (!(flags & TLBF_NOBROADCAST)) + dsb(ishst); + else + dsb(nshst); + asid =3D ASID(mm); =20 - if (flags & TLBF_NOWALKCACHE) - __flush_s1_tlb_range_op(vale1is, start, pages, stride, - asid, tlb_level); - else + switch (flags & (TLBF_NOWALKCACHE | TLBF_NOBROADCAST)) { + case TLBF_NONE: __flush_s1_tlb_range_op(vae1is, start, pages, stride, - asid, tlb_level); + asid, tlb_level); + break; + case TLBF_NOWALKCACHE: + __flush_s1_tlb_range_op(vale1is, start, pages, stride, + asid, tlb_level); + break; + case TLBF_NOBROADCAST: + __flush_s1_tlb_range_op(vae1, start, pages, stride, + asid, tlb_level); + break; + case TLBF_NOWALKCACHE | TLBF_NOBROADCAST: + __flush_s1_tlb_range_op(vale1, start, pages, stride, + asid, tlb_level); + break; + } =20 - mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); + if (!(flags & TLBF_NONOTIFY)) + mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); + + if (!(flags & TLBF_NOSYNC)) { + if (!(flags & TLBF_NOBROADCAST)) + dsb(ish); + else + dsb(nsh); + } } =20 static inline void __flush_tlb_range(struct vm_area_struct *vma, @@ -566,24 +601,9 @@ static inline void __flush_tlb_range(struct vm_area_st= ruct *vma, unsigned long stride, int tlb_level, tlbf_t flags) { - __flush_tlb_range_nosync(vma->vm_mm, start, end, stride, - tlb_level, flags); - dsb(ish); -} - -static inline void local_flush_tlb_contpte(struct vm_area_struct *vma, - unsigned long addr) -{ - unsigned long asid; - - addr =3D round_down(addr, CONT_PTE_SIZE); - - dsb(nshst); - asid =3D ASID(vma->vm_mm); - __flush_s1_tlb_range_op(vale1, addr, CONT_PTES, PAGE_SIZE, asid, 3); - mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, addr, - addr + CONT_PTE_SIZE); - dsb(nsh); + start =3D round_down(start, stride); + end =3D round_up(end, stride); + ___flush_tlb_range(vma, start, end, stride, tlb_level, flags); } =20 static inline void flush_tlb_range(struct vm_area_struct *vma, @@ -636,7 +656,10 @@ static inline void __flush_tlb_kernel_pgtable(unsigned= long kaddr) static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_ba= tch *batch, struct mm_struct *mm, unsigned long start, unsigned long end) { - __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, 3, TLBF_NOWALKCACHE); + struct vm_area_struct vma =3D { .vm_mm =3D mm, .vm_flags =3D 0 }; + + __flush_tlb_range(&vma, start, end, PAGE_SIZE, 3, + TLBF_NOWALKCACHE | TLBF_NOSYNC); } =20 static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval) diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index 1a12bb728ee1..ec17a0e70415 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -527,8 +527,8 @@ int contpte_ptep_clear_flush_young(struct vm_area_struc= t *vma, * eliding the trailing DSB applies here. */ addr =3D ALIGN_DOWN(addr, CONT_PTE_SIZE); - __flush_tlb_range_nosync(vma->vm_mm, addr, addr + CONT_PTE_SIZE, - PAGE_SIZE, 3, TLBF_NOWALKCACHE); + __flush_tlb_range(vma, addr, addr + CONT_PTE_SIZE, + PAGE_SIZE, 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); } =20 return young; @@ -623,7 +623,10 @@ int contpte_ptep_set_access_flags(struct vm_area_struc= t *vma, __ptep_set_access_flags(vma, addr, ptep, entry, 0); =20 if (dirty) - local_flush_tlb_contpte(vma, start_addr); + __flush_tlb_range(vma, start_addr, + start_addr + CONT_PTE_SIZE, + PAGE_SIZE, 3, + TLBF_NOWALKCACHE | TLBF_NOBROADCAST); } else { __contpte_try_unfold(vma->vm_mm, addr, ptep, orig_pte); __ptep_set_access_flags(vma, addr, ptep, entry, dirty); --=20 2.43.0 From nobody Thu Dec 18 23:23:41 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 549AD34AB18 for ; Tue, 16 Dec 2025 14:46:32 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896396; cv=none; b=sTBfIsvRS+axksZK8chZSCCCLnAAs8WSNgD80gSK2fP0ucljHLwVkon3T4XVHfsWIUoVArLS/DuOeOxffl8PL0xOeZpFNx7zYMceHo5zm7/IyOnvUmu+VKFKTvCd+1j/X5qqqmDPuEAIGLLjNp2BMKQ2FWCXCJ25fgV0HX6bVk4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896396; c=relaxed/simple; bh=XSPqmMt9aMDGbBKRlOV+7Uqk66HWvqXZs5j5UkdkEjk=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=jeyyMOSvvMxvEhtYhDD09hmbus27MUfSn/OtNfxIpSeOsThcWmiLLPKCca1oVeuC0Yw21p3Qi+rfDZzWA9zwXoNLikTISwqfdNSOxJ+zc8My7q8loctAZKXJovh9euMLLGtv1DpJfe/vSjfsbL+bj3FqvFsMHn0beTtnAOSGTsU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 18BA7168F; Tue, 16 Dec 2025 06:46:25 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 71C223F73F; Tue, 16 Dec 2025 06:46:30 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v1 12/13] arm64: mm: Wrap flush_tlb_page() around ___flush_tlb_range() Date: Tue, 16 Dec 2025 14:45:57 +0000 Message-ID: <20251216144601.2106412-13-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20251216144601.2106412-1-ryan.roberts@arm.com> References: <20251216144601.2106412-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Flushing a page from the tlb is just a special case of flushing a range. So let's rework flush_tlb_page() so that it simply wraps ___flush_tlb_range(). While at it, let's also update the API to take the same flags that we use when flushing a range. This allows us to delete all the ugly "_nosync", "_local" and "_nonotify" variants. Thanks to constant folding, all of the complex looping and tlbi-by-range options get eliminated so that the generated code for flush_tlb_page() looks very similar to the previous version. Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/pgtable.h | 6 +-- arch/arm64/include/asm/tlbflush.h | 81 ++++++++++--------------------- arch/arm64/mm/fault.c | 2 +- 3 files changed, 29 insertions(+), 60 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgta= ble.h index 736747fbc843..b96a7ca465a1 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -136,10 +136,10 @@ static inline void arch_leave_lazy_mmu_mode(void) * entries exist. */ #define flush_tlb_fix_spurious_fault(vma, address, ptep) \ - local_flush_tlb_page_nonotify(vma, address) + __flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY) =20 #define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp) \ - local_flush_tlb_page_nonotify(vma, address) + __flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY) =20 /* * ZERO_PAGE is a global shared page that is always zero: used @@ -1351,7 +1351,7 @@ static inline int __ptep_clear_flush_young(struct vm_= area_struct *vma, * context-switch, which provides a DSB to complete the TLB * invalidation. */ - flush_tlb_page_nosync(vma, address); + __flush_tlb_page(vma, address, TLBF_NOSYNC); } =20 return young; diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index ee747e66bbef..fa5aee990742 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -256,10 +256,7 @@ static inline void __tlbi_level(tlbi_op op, u64 addr, = u32 level) * unmapping pages from vmalloc/io space. * * flush_tlb_page(vma, addr) - * Invalidate a single user mapping for address 'addr' in the - * address space corresponding to 'vma->mm'. Note that this - * operation only invalidates a single, last-level page-table - * entry and therefore does not affect any walk-caches. + * Equivalent to __flush_tlb_page(..., flags=3DTLBF_NONE) * * * Next, we have some undocumented invalidation routines that you probably @@ -287,13 +284,14 @@ static inline void __tlbi_level(tlbi_op op, u64 addr,= u32 level) * TLBF_NOSYNC (don't issue trailing dsb) and TLBF_NOBROADCAST * (only perform the invalidation for the local cpu). * - * local_flush_tlb_page(vma, addr) - * Local variant of flush_tlb_page(). Stale TLB entries may - * remain in remote CPUs. - * - * local_flush_tlb_page_nonotify(vma, addr) - * Same as local_flush_tlb_page() except MMU notifier will not be - * called. + * __flush_tlb_page(vma, addr, flags) + * Invalidate a single user mapping for address 'addr' in the + * address space corresponding to 'vma->mm'. Note that this + * operation only invalidates a single, last-level page-table entry + * and therefore does not affect any walk-caches. flags may contain + * any combination of TLBF_NONOTIFY (don't call mmu notifiers), + * TLBF_NOSYNC (don't issue trailing dsb) and TLBF_NOBROADCAST + * (only perform the invalidation for the local cpu). * * Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented * on top of these routines, since that is our interface to the mmu_gather @@ -327,51 +325,6 @@ static inline void flush_tlb_mm(struct mm_struct *mm) mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL); } =20 -static inline void __local_flush_tlb_page_nonotify_nosync(struct mm_struct= *mm, - unsigned long uaddr) -{ - dsb(nshst); - __tlbi_level_asid(vale1, uaddr, TLBI_TTL_UNKNOWN, ASID(mm)); -} - -static inline void local_flush_tlb_page_nonotify(struct vm_area_struct *vm= a, - unsigned long uaddr) -{ - __local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr); - dsb(nsh); -} - -static inline void local_flush_tlb_page(struct vm_area_struct *vma, - unsigned long uaddr) -{ - __local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr); - mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, uaddr & PAGE_MASK, - (uaddr & PAGE_MASK) + PAGE_SIZE); - dsb(nsh); -} - -static inline void __flush_tlb_page_nosync(struct mm_struct *mm, - unsigned long uaddr) -{ - dsb(ishst); - __tlbi_level_asid(vale1is, uaddr, TLBI_TTL_UNKNOWN, ASID(mm)); - mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK, - (uaddr & PAGE_MASK) + PAGE_SIZE); -} - -static inline void flush_tlb_page_nosync(struct vm_area_struct *vma, - unsigned long uaddr) -{ - return __flush_tlb_page_nosync(vma->vm_mm, uaddr); -} - -static inline void flush_tlb_page(struct vm_area_struct *vma, - unsigned long uaddr) -{ - flush_tlb_page_nosync(vma, uaddr); - dsb(ish); -} - static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) { /* @@ -618,6 +571,22 @@ static inline void flush_tlb_range(struct vm_area_stru= ct *vma, __flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE= ); } =20 +static inline void __flush_tlb_page(struct vm_area_struct *vma, + unsigned long uaddr, tlbf_t flags) +{ + unsigned long start =3D round_down(uaddr, PAGE_SIZE); + unsigned long end =3D start + PAGE_SIZE; + + ___flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, + TLBF_NOWALKCACHE | flags); +} + +static inline void flush_tlb_page(struct vm_area_struct *vma, + unsigned long uaddr) +{ + __flush_tlb_page(vma, uaddr, TLBF_NONE); +} + static inline void flush_tlb_kernel_range(unsigned long start, unsigned lo= ng end) { const unsigned long stride =3D PAGE_SIZE; diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index be9dab2c7d6a..f91aa686f142 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -239,7 +239,7 @@ int __ptep_set_access_flags(struct vm_area_struct *vma, * flush_tlb_fix_spurious_fault(). */ if (dirty) - local_flush_tlb_page(vma, address); + __flush_tlb_page(vma, address, TLBF_NOBROADCAST); return 1; } =20 --=20 2.43.0 From nobody Thu Dec 18 23:23:41 2025 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id E86D734DB5F for ; Tue, 16 Dec 2025 14:46:34 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896397; cv=none; b=I0Zt1GOnT6Ziv/f3GcFRqwaGXHEXVngubEbRsLWVDndh1Vd0SGFi0nNpjR5MV9j224lzR2jIuSz0rmCvh/IsZZ6IJw6gM8ejQwbq8zNHAyOmfYHz1Ez86WAzWDqxyFOsFo8mfDndZqEPD+b+4L0ouj/MKNYxFUhw4BdFTiIG5Bg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1765896397; c=relaxed/simple; bh=y8aTYMZA5YV99XSnf+mSwccANYKfRqrljoaBJ6X2zko=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=a4+wZFDYOGYff7iax19O0KRod+OAFFrc6O4bxqmUY0olRk6ckDyH+H8f0x0gFM5qoYGnl5OuLkdCpnB8mvFtkMw9Y1FSAW4ln1hAFsjIYpIbdNKB2FK1rnGiEwdXxfHmkhsfRNo0isJwTtf8WsKKSQwPQ7geiLWM+xi0tsG8PZE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id CC2741516; Tue, 16 Dec 2025 06:46:26 -0800 (PST) Received: from e125769.cambridge.arm.com (e125769.cambridge.arm.com [10.1.196.27]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 7E1F53F73F; Tue, 16 Dec 2025 06:46:32 -0800 (PST) From: Ryan Roberts To: Will Deacon , Ard Biesheuvel , Catalin Marinas , Mark Rutland , Linus Torvalds , Oliver Upton , Marc Zyngier , Dev Jain , Linu Cherian Cc: Ryan Roberts , linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Subject: [PATCH v1 13/13] arm64: mm: Provide level hint for flush_tlb_page() Date: Tue, 16 Dec 2025 14:45:58 +0000 Message-ID: <20251216144601.2106412-14-ryan.roberts@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20251216144601.2106412-1-ryan.roberts@arm.com> References: <20251216144601.2106412-1-ryan.roberts@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Previously tlb invalidations issued by __flush_tlb_page() did not contain a level hint. But the function is clearly only ever targeting level 3 tlb entries and its documentation agrees: | this operation only invalidates a single, last-level page-table | entry and therefore does not affect any walk-caches However, it turns out that the function was actually being used to invalidate a level 2 mapping via flush_tlb_fix_spurious_fault_pmd(). The bug was benign because the level hint was not set so the HW would still invalidate the PMD mapping, and also because the TLBF_NONOTIFY flag was set, the bounds of the mapping were never used for anything else. Now that we have the new and improved range-invalidation API, it is trival to fix flush_tlb_fix_spurious_fault_pmd() to explicitly flush the whole range (locally, without notification and last level only). So let's do that, and then update __flush_tlb_page() to hint level 3. Signed-off-by: Ryan Roberts --- arch/arm64/include/asm/pgtable.h | 5 +++-- arch/arm64/include/asm/tlbflush.h | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgta= ble.h index b96a7ca465a1..61f57647361a 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -138,8 +138,9 @@ static inline void arch_leave_lazy_mmu_mode(void) #define flush_tlb_fix_spurious_fault(vma, address, ptep) \ __flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY) =20 -#define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp) \ - __flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY) +#define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp) \ + __flush_tlb_range(vma, address, address + PMD_SIZE, PMD_SIZE, 2, \ + TLBF_NOBROADCAST | TLBF_NONOTIFY | TLBF_NOWALKCACHE) =20 /* * ZERO_PAGE is a global shared page that is always zero: used diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlb= flush.h index fa5aee990742..f24211b51df3 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -577,7 +577,7 @@ static inline void __flush_tlb_page(struct vm_area_stru= ct *vma, unsigned long start =3D round_down(uaddr, PAGE_SIZE); unsigned long end =3D start + PAGE_SIZE; =20 - ___flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, + ___flush_tlb_range(vma, start, end, PAGE_SIZE, 3, TLBF_NOWALKCACHE | flags); } =20 --=20 2.43.0