1. Patchew
  2. linux
  3. [PATCH 00/10] arm64: Replace TLB invalidation preprocessor macros with C functions
  4. View patch

[PATCH 09/10] arm64: mm: Simplify __flush_tlb_range_limit_excess()

Will Deacon posted 10 patches 2 months, 3 weeks ago
[PATCH 09/10] arm64: mm: Simplify __flush_tlb_range_limit_excess()
Posted by Will Deacon 2 months, 3 weeks ago 
__flush_tlb_range_limit_excess() is unnecessarily complicated:

  - It takes a 'start', 'end' and 'pages' argument, whereas it only
    needs 'pages' (which the caller has computed from the other two
    arguments!).

  - It erroneously compares 'pages' with MAX_TLBI_RANGE_PAGES when
    the system doesn't support range-based invalidation but the range to
    be invalidated would result in fewer than MAX_DVM_OPS invalidations.

Simplify the function so that it no longer takes the 'start' and 'end'
arguments and only considers the MAX_TLBI_RANGE_PAGES threshold on
systems that implement range-based invalidation.

Signed-off-by: Will Deacon <will@kernel.org>
---
 arch/arm64/include/asm/tlbflush.h | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index 8618a85d5cd3..2541863721af 100644
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -470,21 +470,13 @@ do {									\
 #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \
 	__flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_enabled());
 
-static inline bool __flush_tlb_range_limit_excess(unsigned long start,
-		unsigned long end, unsigned long pages, unsigned long stride)
+static inline bool __flush_tlb_range_limit_excess(unsigned long pages,
+						  unsigned long stride)
 {
-	/*
-	 * When the system does not support TLB range based flush
-	 * operation, (MAX_DVM_OPS - 1) pages can be handled. But
-	 * with TLB range based operation, MAX_TLBI_RANGE_PAGES
-	 * pages can be handled.
-	 */
-	if ((!system_supports_tlb_range() &&
-	     (end - start) >= (MAX_DVM_OPS * stride)) ||
-	    pages > MAX_TLBI_RANGE_PAGES)
+	if (system_supports_tlb_range() && pages > MAX_TLBI_RANGE_PAGES)
 		return true;
 
-	return false;
+	return pages >= (MAX_DVM_OPS * stride) >> PAGE_SHIFT;
 }
 
 static inline void __flush_tlb_range_nosync(struct mm_struct *mm,
@@ -498,7 +490,7 @@ static inline void __flush_tlb_range_nosync(struct mm_struct *mm,
 	end = round_up(end, stride);
 	pages = (end - start) >> PAGE_SHIFT;
 
-	if (__flush_tlb_range_limit_excess(start, end, pages, stride)) {
+	if (__flush_tlb_range_limit_excess(pages, stride)) {
 		flush_tlb_mm(mm);
 		return;
 	}
@@ -547,7 +539,7 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end
 	end = round_up(end, stride);
 	pages = (end - start) >> PAGE_SHIFT;
 
-	if (__flush_tlb_range_limit_excess(start, end, pages, stride)) {
+	if (__flush_tlb_range_limit_excess(pages, stride)) {
 		flush_tlb_all();
 		return;
 	}
-- 
2.50.0.727.gbf7dc18ff4-goog
Re: [PATCH 09/10] arm64: mm: Simplify __flush_tlb_range_limit_excess()
Posted by Ryan Roberts 2 months, 3 weeks ago 
On 11/07/2025 17:17, Will Deacon wrote:
> __flush_tlb_range_limit_excess() is unnecessarily complicated:
> 
>   - It takes a 'start', 'end' and 'pages' argument, whereas it only
>     needs 'pages' (which the caller has computed from the other two
>     arguments!).
> 
>   - It erroneously compares 'pages' with MAX_TLBI_RANGE_PAGES when
>     the system doesn't support range-based invalidation but the range to
>     be invalidated would result in fewer than MAX_DVM_OPS invalidations.
> 
> Simplify the function so that it no longer takes the 'start' and 'end'
> arguments and only considers the MAX_TLBI_RANGE_PAGES threshold on
> systems that implement range-based invalidation.
> 
> Signed-off-by: Will Deacon <will@kernel.org>

Does this warrant a Fixes: tag?

> ---
>  arch/arm64/include/asm/tlbflush.h | 20 ++++++--------------
>  1 file changed, 6 insertions(+), 14 deletions(-)
> 
> diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
> index 8618a85d5cd3..2541863721af 100644
> --- a/arch/arm64/include/asm/tlbflush.h
> +++ b/arch/arm64/include/asm/tlbflush.h
> @@ -470,21 +470,13 @@ do {									\
>  #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \
>  	__flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_enabled());
>  
> -static inline bool __flush_tlb_range_limit_excess(unsigned long start,
> -		unsigned long end, unsigned long pages, unsigned long stride)
> +static inline bool __flush_tlb_range_limit_excess(unsigned long pages,
> +						  unsigned long stride)
>  {
> -	/*
> -	 * When the system does not support TLB range based flush
> -	 * operation, (MAX_DVM_OPS - 1) pages can be handled. But
> -	 * with TLB range based operation, MAX_TLBI_RANGE_PAGES
> -	 * pages can be handled.
> -	 */
> -	if ((!system_supports_tlb_range() &&
> -	     (end - start) >= (MAX_DVM_OPS * stride)) ||
> -	    pages > MAX_TLBI_RANGE_PAGES)
> +	if (system_supports_tlb_range() && pages > MAX_TLBI_RANGE_PAGES)
>  		return true;
>  
> -	return false;
> +	return pages >= (MAX_DVM_OPS * stride) >> PAGE_SHIFT;
>  }

I'm still not sure I totally get this... Aren't these really 2 separate
concepts? MAX_TLBI_RANGE_PAGES is the max amount of VA that can be handled by a
single tlbi-by-range (and due to implementation, the largest range that can be
handled by the loop in __flush_tlb_range_op()). Whereas MAX_DVM_OPS is the max
number of tlbi instrcutions you want to issue with the PTL held? Perhaps it is
better to split these out; For the range case, calculate the number of ops you
actually need and compare with MAX_DVM_OPS?


>  
>  static inline void __flush_tlb_range_nosync(struct mm_struct *mm,
> @@ -498,7 +490,7 @@ static inline void __flush_tlb_range_nosync(struct mm_struct *mm,
>  	end = round_up(end, stride);
>  	pages = (end - start) >> PAGE_SHIFT;
>  
> -	if (__flush_tlb_range_limit_excess(start, end, pages, stride)) {
> +	if (__flush_tlb_range_limit_excess(pages, stride)) {
>  		flush_tlb_mm(mm);
>  		return;
>  	}
> @@ -547,7 +539,7 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end
>  	end = round_up(end, stride);
>  	pages = (end - start) >> PAGE_SHIFT;
>  
> -	if (__flush_tlb_range_limit_excess(start, end, pages, stride)) {
> +	if (__flush_tlb_range_limit_excess(pages, stride)) {
>  		flush_tlb_all();
>  		return;
>  	}
Re: [PATCH 09/10] arm64: mm: Simplify __flush_tlb_range_limit_excess()
Posted by Dev Jain 2 months, 3 weeks ago 
On 14/07/25 3:00 pm, Ryan Roberts wrote:
> On 11/07/2025 17:17, Will Deacon wrote:
>> __flush_tlb_range_limit_excess() is unnecessarily complicated:
>>
>>    - It takes a 'start', 'end' and 'pages' argument, whereas it only
>>      needs 'pages' (which the caller has computed from the other two
>>      arguments!).
>>
>>    - It erroneously compares 'pages' with MAX_TLBI_RANGE_PAGES when
>>      the system doesn't support range-based invalidation but the range to
>>      be invalidated would result in fewer than MAX_DVM_OPS invalidations.
>>
>> Simplify the function so that it no longer takes the 'start' and 'end'
>> arguments and only considers the MAX_TLBI_RANGE_PAGES threshold on
>> systems that implement range-based invalidation.
>>
>> Signed-off-by: Will Deacon <will@kernel.org>
> Does this warrant a Fixes: tag?
>
>> ---
>>   arch/arm64/include/asm/tlbflush.h | 20 ++++++--------------
>>   1 file changed, 6 insertions(+), 14 deletions(-)
>>
>> diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
>> index 8618a85d5cd3..2541863721af 100644
>> --- a/arch/arm64/include/asm/tlbflush.h
>> +++ b/arch/arm64/include/asm/tlbflush.h
>> @@ -470,21 +470,13 @@ do {									\
>>   #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \
>>   	__flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_enabled());
>>   
>> -static inline bool __flush_tlb_range_limit_excess(unsigned long start,
>> -		unsigned long end, unsigned long pages, unsigned long stride)
>> +static inline bool __flush_tlb_range_limit_excess(unsigned long pages,
>> +						  unsigned long stride)
>>   {
>> -	/*
>> -	 * When the system does not support TLB range based flush
>> -	 * operation, (MAX_DVM_OPS - 1) pages can be handled. But
>> -	 * with TLB range based operation, MAX_TLBI_RANGE_PAGES
>> -	 * pages can be handled.
>> -	 */
>> -	if ((!system_supports_tlb_range() &&
>> -	     (end - start) >= (MAX_DVM_OPS * stride)) ||
>> -	    pages > MAX_TLBI_RANGE_PAGES)
>> +	if (system_supports_tlb_range() && pages > MAX_TLBI_RANGE_PAGES)
>>   		return true;
>>   
>> -	return false;
>> +	return pages >= (MAX_DVM_OPS * stride) >> PAGE_SHIFT;
>>   }
> I'm still not sure I totally get this... Aren't these really 2 separate
> concepts? MAX_TLBI_RANGE_PAGES is the max amount of VA that can be handled by a
> single tlbi-by-range (and due to implementation, the largest range that can be
> handled by the loop in __flush_tlb_range_op()). Whereas MAX_DVM_OPS is the max
> number of tlbi instrcutions you want to issue with the PTL held? Perhaps it is
> better to split these out; For the range case, calculate the number of ops you
> actually need and compare with MAX_DVM_OPS?

If system_supports_tlb_range() returns true and pages <= MAX_TLBI_RANGE_PAGES,
then it is guaranteed that the number of tlbi range operations issued is bounded
by 4 -> we start from scale = 3, and by patch 6 of this series (and the loop
itself btw) it is guaranteed that the scale will be decremented, so the worst
case is that all scales are used, so at most 4 operations will be issued, so
we are safe.

So if my reasoning is correct, I think what we need is something like:

diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index aa9efee17277..53591caf3793 100644
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -427,21 +427,19 @@ do {									\
  #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \
  	__flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false, kvm_lpa2_is_enabled());
  
-static inline bool __flush_tlb_range_limit_excess(unsigned long start,
-		unsigned long end, unsigned long pages, unsigned long stride)
+static inline bool __flush_tlb_range_limit_excess(unsigned long pages,
+						  unsigned long stride)
  {
  	/*
-	 * When the system does not support TLB range based flush
-	 * operation, (MAX_DVM_OPS - 1) pages can be handled. But
-	 * with TLB range based operation, MAX_TLBI_RANGE_PAGES
-	 * pages can be handled.
+	 * If a TLBI range op has pages under MAX_TLBI_RANGE_PAGES, it
+	 * is guaranteed that the loop in __flush_tlb_range_op shall
+	 * terminate in at most 4 iterations, so we do not need to
+	 * check with MAX_DVM_OPS in this case.
  	 */
-	if ((!system_supports_tlb_range() &&
-	     (end - start) >= (MAX_DVM_OPS * stride)) ||
-	    pages > MAX_TLBI_RANGE_PAGES)
-		return true;
+	if (system_supports_tlb_range())
+		return pages > MAX_TLBI_RANGE_PAGES;
  
-	return false;
+	return pages >= (MAX_DVM_OPS * stride) >> PAGE_SHIFT;
  }
  
  static inline void __flush_tlb_range_nosync(struct mm_struct *mm,


where the comment can be worded better.


>
>
>>   
>>   static inline void __flush_tlb_range_nosync(struct mm_struct *mm,
>> @@ -498,7 +490,7 @@ static inline void __flush_tlb_range_nosync(struct mm_struct *mm,
>>   	end = round_up(end, stride);
>>   	pages = (end - start) >> PAGE_SHIFT;
>>   
>> -	if (__flush_tlb_range_limit_excess(start, end, pages, stride)) {
>> +	if (__flush_tlb_range_limit_excess(pages, stride)) {
>>   		flush_tlb_mm(mm);
>>   		return;
>>   	}
>> @@ -547,7 +539,7 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end
>>   	end = round_up(end, stride);
>>   	pages = (end - start) >> PAGE_SHIFT;
>>   
>> -	if (__flush_tlb_range_limit_excess(start, end, pages, stride)) {
>> +	if (__flush_tlb_range_limit_excess(pages, stride)) {
>>   		flush_tlb_all();
>>   		return;
>>   	}
>

Patchew 2.1-devel-b67e4c2

© 2016 - 2025 Red Hat, Inc.