Refactor the huge_pte helpers to use the new common set_ptes_anysz() and
ptep_get_and_clear_anysz() APIs.
This provides 2 benefits; First, when page_table_check=on, hugetlb is
now properly/fully checked. Previously only the first page of a hugetlb
folio was checked. Second, instead of having to call __set_ptes(nr=1)
for each pte in a loop, the whole contiguous batch can now be set in one
go, which enables some efficiencies and cleans up the code.
One detail to note is that huge_ptep_clear_flush() was previously
calling ptep_clear_flush() for a non-contiguous pte (i.e. a pud or pmd
block mapping). This has a couple of disadvantages; first
ptep_clear_flush() calls ptep_get_and_clear() which transparently
handles contpte. Given we only call for non-contiguous ptes, it would be
safe, but a waste of effort. It's preferable to go straight to the layer
below. However, more problematic is that ptep_get_and_clear() is for
PAGE_SIZE entries so it calls page_table_check_pte_clear() and would not
clear the whole hugetlb folio. So let's stop special-casing the non-cont
case and just rely on get_clear_contig_flush() to do the right thing for
non-cont entries.
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
---
arch/arm64/mm/hugetlbpage.c | 52 +++++++------------------------------
1 file changed, 10 insertions(+), 42 deletions(-)
diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
index 065be8650aa5..efd18bd1eae3 100644
--- a/arch/arm64/mm/hugetlbpage.c
+++ b/arch/arm64/mm/hugetlbpage.c
@@ -159,12 +159,12 @@ static pte_t get_clear_contig(struct mm_struct *mm,
pte_t pte, tmp_pte;
bool present;
- pte = __ptep_get_and_clear(mm, addr, ptep);
+ pte = ptep_get_and_clear_anysz(mm, ptep, pgsize);
present = pte_present(pte);
while (--ncontig) {
ptep++;
addr += pgsize;
- tmp_pte = __ptep_get_and_clear(mm, addr, ptep);
+ tmp_pte = ptep_get_and_clear_anysz(mm, ptep, pgsize);
if (present) {
if (pte_dirty(tmp_pte))
pte = pte_mkdirty(pte);
@@ -208,7 +208,7 @@ static void clear_flush(struct mm_struct *mm,
unsigned long i, saddr = addr;
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
- __ptep_get_and_clear(mm, addr, ptep);
+ ptep_get_and_clear_anysz(mm, ptep, pgsize);
__flush_hugetlb_tlb_range(&vma, saddr, addr, pgsize, true);
}
@@ -219,32 +219,20 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
size_t pgsize;
int i;
int ncontig;
- unsigned long pfn, dpfn;
- pgprot_t hugeprot;
ncontig = num_contig_ptes(sz, &pgsize);
if (!pte_present(pte)) {
for (i = 0; i < ncontig; i++, ptep++, addr += pgsize)
- __set_ptes(mm, addr, ptep, pte, 1);
+ set_ptes_anysz(mm, ptep, pte, 1, pgsize);
return;
}
- if (!pte_cont(pte)) {
- __set_ptes(mm, addr, ptep, pte, 1);
- return;
- }
-
- pfn = pte_pfn(pte);
- dpfn = pgsize >> PAGE_SHIFT;
- hugeprot = pte_pgprot(pte);
-
/* Only need to "break" if transitioning valid -> valid. */
- if (pte_valid(__ptep_get(ptep)))
+ if (pte_cont(pte) && pte_valid(__ptep_get(ptep)))
clear_flush(mm, addr, ptep, pgsize, ncontig);
- for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
- __set_ptes(mm, addr, ptep, pfn_pte(pfn, hugeprot), 1);
+ set_ptes_anysz(mm, ptep, pte, ncontig, pgsize);
}
pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
@@ -434,11 +422,9 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep,
pte_t pte, int dirty)
{
- int ncontig, i;
+ int ncontig;
size_t pgsize = 0;
- unsigned long pfn = pte_pfn(pte), dpfn;
struct mm_struct *mm = vma->vm_mm;
- pgprot_t hugeprot;
pte_t orig_pte;
VM_WARN_ON(!pte_present(pte));
@@ -447,7 +433,6 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma,
return __ptep_set_access_flags(vma, addr, ptep, pte, dirty);
ncontig = num_contig_ptes(huge_page_size(hstate_vma(vma)), &pgsize);
- dpfn = pgsize >> PAGE_SHIFT;
if (!__cont_access_flags_changed(ptep, pte, ncontig))
return 0;
@@ -462,19 +447,14 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma,
if (pte_young(orig_pte))
pte = pte_mkyoung(pte);
- hugeprot = pte_pgprot(pte);
- for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
- __set_ptes(mm, addr, ptep, pfn_pte(pfn, hugeprot), 1);
-
+ set_ptes_anysz(mm, ptep, pte, ncontig, pgsize);
return 1;
}
void huge_ptep_set_wrprotect(struct mm_struct *mm,
unsigned long addr, pte_t *ptep)
{
- unsigned long pfn, dpfn;
- pgprot_t hugeprot;
- int ncontig, i;
+ int ncontig;
size_t pgsize;
pte_t pte;
@@ -487,16 +467,11 @@ void huge_ptep_set_wrprotect(struct mm_struct *mm,
}
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
- dpfn = pgsize >> PAGE_SHIFT;
pte = get_clear_contig_flush(mm, addr, ptep, pgsize, ncontig);
pte = pte_wrprotect(pte);
- hugeprot = pte_pgprot(pte);
- pfn = pte_pfn(pte);
-
- for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
- __set_ptes(mm, addr, ptep, pfn_pte(pfn, hugeprot), 1);
+ set_ptes_anysz(mm, ptep, pte, ncontig, pgsize);
}
pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
@@ -505,13 +480,6 @@ pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
struct mm_struct *mm = vma->vm_mm;
size_t pgsize;
int ncontig;
- pte_t pte;
-
- pte = __ptep_get(ptep);
- VM_WARN_ON(!pte_present(pte));
-
- if (!pte_cont(pte))
- return ptep_clear_flush(vma, addr, ptep);
ncontig = num_contig_ptes(huge_page_size(hstate_vma(vma)), &pgsize);
return get_clear_contig_flush(mm, addr, ptep, pgsize, ncontig);
--
2.43.0On Tue, Mar 04, 2025 at 03:04:35PM +0000, Ryan Roberts wrote: > Refactor the huge_pte helpers to use the new common set_ptes_anysz() and > ptep_get_and_clear_anysz() APIs. Nitpick: maybe add underscores to this new API to suggest it's private. Up to you. > This provides 2 benefits; First, when page_table_check=on, hugetlb is > now properly/fully checked. Previously only the first page of a hugetlb > folio was checked. Second, instead of having to call __set_ptes(nr=1) > for each pte in a loop, the whole contiguous batch can now be set in one > go, which enables some efficiencies and cleans up the code. > > One detail to note is that huge_ptep_clear_flush() was previously > calling ptep_clear_flush() for a non-contiguous pte (i.e. a pud or pmd > block mapping). This has a couple of disadvantages; first > ptep_clear_flush() calls ptep_get_and_clear() which transparently > handles contpte. Given we only call for non-contiguous ptes, it would be > safe, but a waste of effort. It's preferable to go straight to the layer > below. However, more problematic is that ptep_get_and_clear() is for > PAGE_SIZE entries so it calls page_table_check_pte_clear() and would not > clear the whole hugetlb folio. So let's stop special-casing the non-cont > case and just rely on get_clear_contig_flush() to do the right thing for > non-cont entries. > > Signed-off-by: Ryan Roberts <ryan.roberts@arm.com> Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Hi Ryan,
kernel test robot noticed the following build warnings:
[auto build test WARNING on linus/master]
[also build test WARNING on v6.14-rc5 next-20250305]
[cannot apply to arm64/for-next/core akpm-mm/mm-everything arm-perf/for-next/perf]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Ryan-Roberts/arm64-hugetlb-Cleanup-huge_pte-size-discovery-mechanisms/20250304-230647
base: linus/master
patch link: https://lore.kernel.org/r/20250304150444.3788920-6-ryan.roberts%40arm.com
patch subject: [PATCH v3 05/11] arm64: hugetlb: Use set_ptes_anysz() and ptep_get_and_clear_anysz()
config: arm64-randconfig-003-20250305 (https://download.01.org/0day-ci/archive/20250305/202503052315.vk7m958M-lkp@intel.com/config)
compiler: clang version 21.0.0git (https://github.com/llvm/llvm-project 14170b16028c087ca154878f5ed93d3089a965c6)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250305/202503052315.vk7m958M-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202503052315.vk7m958M-lkp@intel.com/
All warnings (new ones prefixed by >>):
In file included from arch/arm64/mm/hugetlbpage.c:12:
In file included from include/linux/mm.h:2224:
include/linux/vmstat.h:504:43: warning: arithmetic between different enumeration types ('enum zone_stat_item' and 'enum numa_stat_item') [-Wenum-enum-conversion]
504 | return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
| ~~~~~~~~~~~~~~~~~~~~~ ^
505 | item];
| ~~~~
include/linux/vmstat.h:511:43: warning: arithmetic between different enumeration types ('enum zone_stat_item' and 'enum numa_stat_item') [-Wenum-enum-conversion]
511 | return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
| ~~~~~~~~~~~~~~~~~~~~~ ^
512 | NR_VM_NUMA_EVENT_ITEMS +
| ~~~~~~~~~~~~~~~~~~~~~~
>> arch/arm64/mm/hugetlbpage.c:154:23: warning: parameter 'addr' set but not used [-Wunused-but-set-parameter]
154 | unsigned long addr,
| ^
3 warnings generated.
vim +/addr +154 arch/arm64/mm/hugetlbpage.c
bc5dfb4fd7bd471 Baolin Wang 2022-05-16 144
d8bdcff2876424d Steve Capper 2017-08-22 145 /*
d8bdcff2876424d Steve Capper 2017-08-22 146 * Changing some bits of contiguous entries requires us to follow a
d8bdcff2876424d Steve Capper 2017-08-22 147 * Break-Before-Make approach, breaking the whole contiguous set
d8bdcff2876424d Steve Capper 2017-08-22 148 * before we can change any entries. See ARM DDI 0487A.k_iss10775,
d8bdcff2876424d Steve Capper 2017-08-22 149 * "Misprogramming of the Contiguous bit", page D4-1762.
d8bdcff2876424d Steve Capper 2017-08-22 150 *
d8bdcff2876424d Steve Capper 2017-08-22 151 * This helper performs the break step.
d8bdcff2876424d Steve Capper 2017-08-22 152 */
fb396bb459c1fa3 Anshuman Khandual 2022-05-10 153 static pte_t get_clear_contig(struct mm_struct *mm,
d8bdcff2876424d Steve Capper 2017-08-22 @154 unsigned long addr,
d8bdcff2876424d Steve Capper 2017-08-22 155 pte_t *ptep,
d8bdcff2876424d Steve Capper 2017-08-22 156 unsigned long pgsize,
d8bdcff2876424d Steve Capper 2017-08-22 157 unsigned long ncontig)
d8bdcff2876424d Steve Capper 2017-08-22 158 {
49c87f7677746f3 Ryan Roberts 2025-02-26 159 pte_t pte, tmp_pte;
49c87f7677746f3 Ryan Roberts 2025-02-26 160 bool present;
49c87f7677746f3 Ryan Roberts 2025-02-26 161
66251d3eadf78e2 Ryan Roberts 2025-03-04 162 pte = ptep_get_and_clear_anysz(mm, ptep, pgsize);
49c87f7677746f3 Ryan Roberts 2025-02-26 163 present = pte_present(pte);
49c87f7677746f3 Ryan Roberts 2025-02-26 164 while (--ncontig) {
49c87f7677746f3 Ryan Roberts 2025-02-26 165 ptep++;
49c87f7677746f3 Ryan Roberts 2025-02-26 166 addr += pgsize;
66251d3eadf78e2 Ryan Roberts 2025-03-04 167 tmp_pte = ptep_get_and_clear_anysz(mm, ptep, pgsize);
49c87f7677746f3 Ryan Roberts 2025-02-26 168 if (present) {
49c87f7677746f3 Ryan Roberts 2025-02-26 169 if (pte_dirty(tmp_pte))
49c87f7677746f3 Ryan Roberts 2025-02-26 170 pte = pte_mkdirty(pte);
49c87f7677746f3 Ryan Roberts 2025-02-26 171 if (pte_young(tmp_pte))
49c87f7677746f3 Ryan Roberts 2025-02-26 172 pte = pte_mkyoung(pte);
d8bdcff2876424d Steve Capper 2017-08-22 173 }
49c87f7677746f3 Ryan Roberts 2025-02-26 174 }
49c87f7677746f3 Ryan Roberts 2025-02-26 175 return pte;
d8bdcff2876424d Steve Capper 2017-08-22 176 }
d8bdcff2876424d Steve Capper 2017-08-22 177
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
On 05/03/2025 16:00, kernel test robot wrote:
> Hi Ryan,
>
> kernel test robot noticed the following build warnings:
>
> [auto build test WARNING on linus/master]
> [also build test WARNING on v6.14-rc5 next-20250305]
> [cannot apply to arm64/for-next/core akpm-mm/mm-everything arm-perf/for-next/perf]
> [If your patch is applied to the wrong git tree, kindly drop us a note.
> And when submitting patch, we suggest to use '--base' as documented in
> https://git-scm.com/docs/git-format-patch#_base_tree_information]
>
> url: https://github.com/intel-lab-lkp/linux/commits/Ryan-Roberts/arm64-hugetlb-Cleanup-huge_pte-size-discovery-mechanisms/20250304-230647
> base: linus/master
> patch link: https://lore.kernel.org/r/20250304150444.3788920-6-ryan.roberts%40arm.com
> patch subject: [PATCH v3 05/11] arm64: hugetlb: Use set_ptes_anysz() and ptep_get_and_clear_anysz()
> config: arm64-randconfig-003-20250305 (https://download.01.org/0day-ci/archive/20250305/202503052315.vk7m958M-lkp@intel.com/config)
> compiler: clang version 21.0.0git (https://github.com/llvm/llvm-project 14170b16028c087ca154878f5ed93d3089a965c6)
> reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250305/202503052315.vk7m958M-lkp@intel.com/reproduce)
>
> If you fix the issue in a separate patch/commit (i.e. not just a new version of
> the same patch/commit), kindly add following tags
> | Reported-by: kernel test robot <lkp@intel.com>
> | Closes: https://lore.kernel.org/oe-kbuild-all/202503052315.vk7m958M-lkp@intel.com/
>
> All warnings (new ones prefixed by >>):
>
> In file included from arch/arm64/mm/hugetlbpage.c:12:
> In file included from include/linux/mm.h:2224:
> include/linux/vmstat.h:504:43: warning: arithmetic between different enumeration types ('enum zone_stat_item' and 'enum numa_stat_item') [-Wenum-enum-conversion]
> 504 | return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
> | ~~~~~~~~~~~~~~~~~~~~~ ^
> 505 | item];
> | ~~~~
> include/linux/vmstat.h:511:43: warning: arithmetic between different enumeration types ('enum zone_stat_item' and 'enum numa_stat_item') [-Wenum-enum-conversion]
> 511 | return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
> | ~~~~~~~~~~~~~~~~~~~~~ ^
> 512 | NR_VM_NUMA_EVENT_ITEMS +
> | ~~~~~~~~~~~~~~~~~~~~~~
>>> arch/arm64/mm/hugetlbpage.c:154:23: warning: parameter 'addr' set but not used [-Wunused-but-set-parameter]
> 154 | unsigned long addr,
> | ^
> 3 warnings generated.
>
>
> vim +/addr +154 arch/arm64/mm/hugetlbpage.c
>
> bc5dfb4fd7bd471 Baolin Wang 2022-05-16 144
> d8bdcff2876424d Steve Capper 2017-08-22 145 /*
> d8bdcff2876424d Steve Capper 2017-08-22 146 * Changing some bits of contiguous entries requires us to follow a
> d8bdcff2876424d Steve Capper 2017-08-22 147 * Break-Before-Make approach, breaking the whole contiguous set
> d8bdcff2876424d Steve Capper 2017-08-22 148 * before we can change any entries. See ARM DDI 0487A.k_iss10775,
> d8bdcff2876424d Steve Capper 2017-08-22 149 * "Misprogramming of the Contiguous bit", page D4-1762.
> d8bdcff2876424d Steve Capper 2017-08-22 150 *
> d8bdcff2876424d Steve Capper 2017-08-22 151 * This helper performs the break step.
> d8bdcff2876424d Steve Capper 2017-08-22 152 */
> fb396bb459c1fa3 Anshuman Khandual 2022-05-10 153 static pte_t get_clear_contig(struct mm_struct *mm,
> d8bdcff2876424d Steve Capper 2017-08-22 @154 unsigned long addr,
> d8bdcff2876424d Steve Capper 2017-08-22 155 pte_t *ptep,
> d8bdcff2876424d Steve Capper 2017-08-22 156 unsigned long pgsize,
> d8bdcff2876424d Steve Capper 2017-08-22 157 unsigned long ncontig)
> d8bdcff2876424d Steve Capper 2017-08-22 158 {
> 49c87f7677746f3 Ryan Roberts 2025-02-26 159 pte_t pte, tmp_pte;
> 49c87f7677746f3 Ryan Roberts 2025-02-26 160 bool present;
> 49c87f7677746f3 Ryan Roberts 2025-02-26 161
> 66251d3eadf78e2 Ryan Roberts 2025-03-04 162 pte = ptep_get_and_clear_anysz(mm, ptep, pgsize);
> 49c87f7677746f3 Ryan Roberts 2025-02-26 163 present = pte_present(pte);
> 49c87f7677746f3 Ryan Roberts 2025-02-26 164 while (--ncontig) {
> 49c87f7677746f3 Ryan Roberts 2025-02-26 165 ptep++;
> 49c87f7677746f3 Ryan Roberts 2025-02-26 166 addr += pgsize;
Ahh yes, thanks! Looks like this line can be removed since we no longer need the
address.
Catalin, I was optimistically hoping this might be the final version. If it is,
are you happy to fold this in? Or do you want me to re-spin regardless?
> 66251d3eadf78e2 Ryan Roberts 2025-03-04 167 tmp_pte = ptep_get_and_clear_anysz(mm, ptep, pgsize);
> 49c87f7677746f3 Ryan Roberts 2025-02-26 168 if (present) {
> 49c87f7677746f3 Ryan Roberts 2025-02-26 169 if (pte_dirty(tmp_pte))
> 49c87f7677746f3 Ryan Roberts 2025-02-26 170 pte = pte_mkdirty(pte);
> 49c87f7677746f3 Ryan Roberts 2025-02-26 171 if (pte_young(tmp_pte))
> 49c87f7677746f3 Ryan Roberts 2025-02-26 172 pte = pte_mkyoung(pte);
> d8bdcff2876424d Steve Capper 2017-08-22 173 }
> 49c87f7677746f3 Ryan Roberts 2025-02-26 174 }
> 49c87f7677746f3 Ryan Roberts 2025-02-26 175 return pte;
> d8bdcff2876424d Steve Capper 2017-08-22 176 }
> d8bdcff2876424d Steve Capper 2017-08-22 177
>
© 2016 - 2026 Red Hat, Inc.