Refactor __set_ptes(), set_pmd_at() and set_pud_at() so that they are
all a thin wrapper around a new common set_ptes_anysz(), which takes
pgsize parameter. Additionally, refactor __ptep_get_and_clear() and
pmdp_huge_get_and_clear() to use a new common ptep_get_and_clear_anysz()
which also takes a pgsize parameter.
These changes will permit the huge_pte API to efficiently batch-set
pgtable entries and take advantage of the future barrier optimizations.
Additionally since the new *_anysz() helpers call the correct
page_table_check_*_set() API based on pgsize, this means that huge_ptes
will be able to get proper coverage. Currently the huge_pte API always
uses the pte API which assumes an entry only covers a single page.
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
---
arch/arm64/include/asm/pgtable.h | 108 +++++++++++++++++++------------
1 file changed, 67 insertions(+), 41 deletions(-)
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 0b2a2ad1b9e8..e255a36380dc 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -420,23 +420,6 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr)
return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte));
}
-static inline void __set_ptes(struct mm_struct *mm,
- unsigned long __always_unused addr,
- pte_t *ptep, pte_t pte, unsigned int nr)
-{
- page_table_check_ptes_set(mm, ptep, pte, nr);
- __sync_cache_and_tags(pte, nr);
-
- for (;;) {
- __check_safe_pte_update(mm, ptep, pte);
- __set_pte(ptep, pte);
- if (--nr == 0)
- break;
- ptep++;
- pte = pte_advance_pfn(pte, 1);
- }
-}
-
/*
* Hugetlb definitions.
*/
@@ -641,30 +624,59 @@ static inline pgprot_t pud_pgprot(pud_t pud)
return __pgprot(pud_val(pfn_pud(pfn, __pgprot(0))) ^ pud_val(pud));
}
-static inline void __set_pte_at(struct mm_struct *mm,
- unsigned long __always_unused addr,
- pte_t *ptep, pte_t pte, unsigned int nr)
+static inline void set_ptes_anysz(struct mm_struct *mm, pte_t *ptep, pte_t pte,
+ unsigned int nr, unsigned long pgsize)
{
- __sync_cache_and_tags(pte, nr);
- __check_safe_pte_update(mm, ptep, pte);
- __set_pte(ptep, pte);
+ unsigned long stride = pgsize >> PAGE_SHIFT;
+
+ switch (pgsize) {
+ case PAGE_SIZE:
+ page_table_check_ptes_set(mm, ptep, pte, nr);
+ break;
+ case PMD_SIZE:
+ page_table_check_pmds_set(mm, (pmd_t *)ptep, pte_pmd(pte), nr);
+ break;
+ case PUD_SIZE:
+ page_table_check_puds_set(mm, (pud_t *)ptep, pte_pud(pte), nr);
+ break;
+ default:
+ VM_WARN_ON(1);
+ }
+
+ __sync_cache_and_tags(pte, nr * stride);
+
+ for (;;) {
+ __check_safe_pte_update(mm, ptep, pte);
+ __set_pte(ptep, pte);
+ if (--nr == 0)
+ break;
+ ptep++;
+ pte = pte_advance_pfn(pte, stride);
+ }
}
-static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
- pmd_t *pmdp, pmd_t pmd)
+static inline void __set_ptes(struct mm_struct *mm,
+ unsigned long __always_unused addr,
+ pte_t *ptep, pte_t pte, unsigned int nr)
{
- page_table_check_pmd_set(mm, pmdp, pmd);
- return __set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd),
- PMD_SIZE >> PAGE_SHIFT);
+ set_ptes_anysz(mm, ptep, pte, nr, PAGE_SIZE);
}
-static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
- pud_t *pudp, pud_t pud)
+static inline void __set_pmds(struct mm_struct *mm,
+ unsigned long __always_unused addr,
+ pmd_t *pmdp, pmd_t pmd, unsigned int nr)
+{
+ set_ptes_anysz(mm, (pte_t *)pmdp, pmd_pte(pmd), nr, PMD_SIZE);
+}
+#define set_pmd_at(mm, addr, pmdp, pmd) __set_pmds(mm, addr, pmdp, pmd, 1)
+
+static inline void __set_puds(struct mm_struct *mm,
+ unsigned long __always_unused addr,
+ pud_t *pudp, pud_t pud, unsigned int nr)
{
- page_table_check_pud_set(mm, pudp, pud);
- return __set_pte_at(mm, addr, (pte_t *)pudp, pud_pte(pud),
- PUD_SIZE >> PAGE_SHIFT);
+ set_ptes_anysz(mm, (pte_t *)pudp, pud_pte(pud), nr, PUD_SIZE);
}
+#define set_pud_at(mm, addr, pudp, pud) __set_puds(mm, addr, pudp, pud, 1)
#define __p4d_to_phys(p4d) __pte_to_phys(p4d_pte(p4d))
#define __phys_to_p4d_val(phys) __phys_to_pte_val(phys)
@@ -1276,16 +1288,34 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */
-static inline pte_t __ptep_get_and_clear(struct mm_struct *mm,
- unsigned long address, pte_t *ptep)
+static inline pte_t ptep_get_and_clear_anysz(struct mm_struct *mm, pte_t *ptep,
+ unsigned long pgsize)
{
pte_t pte = __pte(xchg_relaxed(&pte_val(*ptep), 0));
- page_table_check_pte_clear(mm, pte);
+ switch (pgsize) {
+ case PAGE_SIZE:
+ page_table_check_pte_clear(mm, pte);
+ break;
+ case PMD_SIZE:
+ page_table_check_pmd_clear(mm, pte_pmd(pte));
+ break;
+ case PUD_SIZE:
+ page_table_check_pud_clear(mm, pte_pud(pte));
+ break;
+ default:
+ VM_WARN_ON(1);
+ }
return pte;
}
+static inline pte_t __ptep_get_and_clear(struct mm_struct *mm,
+ unsigned long address, pte_t *ptep)
+{
+ return ptep_get_and_clear_anysz(mm, ptep, PAGE_SIZE);
+}
+
static inline void __clear_full_ptes(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, unsigned int nr, int full)
{
@@ -1322,11 +1352,7 @@ static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm,
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long address, pmd_t *pmdp)
{
- pmd_t pmd = __pmd(xchg_relaxed(&pmd_val(*pmdp), 0));
-
- page_table_check_pmd_clear(mm, pmd);
-
- return pmd;
+ return pte_pmd(ptep_get_and_clear_anysz(mm, (pte_t *)pmdp, PMD_SIZE));
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
--
2.43.0On Tue, Mar 04, 2025 at 03:04:34PM +0000, Ryan Roberts wrote:
> +static inline void set_ptes_anysz(struct mm_struct *mm, pte_t *ptep, pte_t pte,
> + unsigned int nr, unsigned long pgsize)
> {
> - __sync_cache_and_tags(pte, nr);
> - __check_safe_pte_update(mm, ptep, pte);
> - __set_pte(ptep, pte);
> + unsigned long stride = pgsize >> PAGE_SHIFT;
> +
> + switch (pgsize) {
> + case PAGE_SIZE:
> + page_table_check_ptes_set(mm, ptep, pte, nr);
> + break;
> + case PMD_SIZE:
> + page_table_check_pmds_set(mm, (pmd_t *)ptep, pte_pmd(pte), nr);
> + break;
> + case PUD_SIZE:
> + page_table_check_puds_set(mm, (pud_t *)ptep, pte_pud(pte), nr);
> + break;
> + default:
> + VM_WARN_ON(1);
> + }
> +
> + __sync_cache_and_tags(pte, nr * stride);
> +
> + for (;;) {
> + __check_safe_pte_update(mm, ptep, pte);
> + __set_pte(ptep, pte);
> + if (--nr == 0)
> + break;
> + ptep++;
> + pte = pte_advance_pfn(pte, stride);
> + }
> }
I thought I replied to this one but somehow failed to send. The only
comment I have is that I'd add a double underscore in front of the anysz
functions to imply it's a private API. Otherwise it looks fine.
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Hi Ryan,
kernel test robot noticed the following build errors:
[auto build test ERROR on linus/master]
[also build test ERROR on v6.14-rc5 next-20250305]
[cannot apply to arm64/for-next/core akpm-mm/mm-everything arm-perf/for-next/perf]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Ryan-Roberts/arm64-hugetlb-Cleanup-huge_pte-size-discovery-mechanisms/20250304-230647
base: linus/master
patch link: https://lore.kernel.org/r/20250304150444.3788920-5-ryan.roberts%40arm.com
patch subject: [PATCH v3 04/11] arm64/mm: Refactor __set_ptes() and __ptep_get_and_clear()
config: arm64-randconfig-001-20250305 (https://download.01.org/0day-ci/archive/20250306/202503061237.QurSXHSC-lkp@intel.com/config)
compiler: clang version 15.0.7 (https://github.com/llvm/llvm-project 8dfdcc7b7bf66834a761bd8de445840ef68e4d1a)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250306/202503061237.QurSXHSC-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202503061237.QurSXHSC-lkp@intel.com/
All errors (new ones prefixed by >>):
In file included from arch/arm64/kernel/asm-offsets.c:12:
In file included from include/linux/ftrace.h:10:
In file included from include/linux/trace_recursion.h:5:
In file included from include/linux/interrupt.h:11:
In file included from include/linux/hardirq.h:11:
In file included from arch/arm64/include/asm/hardirq.h:17:
In file included from include/asm-generic/hardirq.h:17:
In file included from include/linux/irq.h:20:
In file included from include/linux/io.h:14:
In file included from arch/arm64/include/asm/io.h:12:
In file included from include/linux/pgtable.h:6:
>> arch/arm64/include/asm/pgtable.h:639:7: error: duplicate case value '536870912'
case PUD_SIZE:
^
include/asm-generic/pgtable-nopud.h:20:20: note: expanded from macro 'PUD_SIZE'
#define PUD_SIZE (1UL << PUD_SHIFT)
^
arch/arm64/include/asm/pgtable.h:636:7: note: previous case defined here
case PMD_SIZE:
^
include/asm-generic/pgtable-nopmd.h:22:20: note: expanded from macro 'PMD_SIZE'
#define PMD_SIZE (1UL << PMD_SHIFT)
^
In file included from arch/arm64/kernel/asm-offsets.c:12:
In file included from include/linux/ftrace.h:10:
In file included from include/linux/trace_recursion.h:5:
In file included from include/linux/interrupt.h:11:
In file included from include/linux/hardirq.h:11:
In file included from arch/arm64/include/asm/hardirq.h:17:
In file included from include/asm-generic/hardirq.h:17:
In file included from include/linux/irq.h:20:
In file included from include/linux/io.h:14:
In file included from arch/arm64/include/asm/io.h:12:
In file included from include/linux/pgtable.h:6:
arch/arm64/include/asm/pgtable.h:1303:7: error: duplicate case value '536870912'
case PUD_SIZE:
^
include/asm-generic/pgtable-nopud.h:20:20: note: expanded from macro 'PUD_SIZE'
#define PUD_SIZE (1UL << PUD_SHIFT)
^
arch/arm64/include/asm/pgtable.h:1300:7: note: previous case defined here
case PMD_SIZE:
^
include/asm-generic/pgtable-nopmd.h:22:20: note: expanded from macro 'PMD_SIZE'
#define PMD_SIZE (1UL << PMD_SHIFT)
^
2 errors generated.
make[3]: *** [scripts/Makefile.build:102: arch/arm64/kernel/asm-offsets.s] Error 1 shuffle=4064171735
make[3]: Target 'prepare' not remade because of errors.
make[2]: *** [Makefile:1264: prepare0] Error 2 shuffle=4064171735
make[2]: Target 'prepare' not remade because of errors.
make[1]: *** [Makefile:251: __sub-make] Error 2 shuffle=4064171735
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:251: __sub-make] Error 2 shuffle=4064171735
make: Target 'prepare' not remade because of errors.
vim +/536870912 +639 arch/arm64/include/asm/pgtable.h
626
627 static inline void set_ptes_anysz(struct mm_struct *mm, pte_t *ptep, pte_t pte,
628 unsigned int nr, unsigned long pgsize)
629 {
630 unsigned long stride = pgsize >> PAGE_SHIFT;
631
632 switch (pgsize) {
633 case PAGE_SIZE:
634 page_table_check_ptes_set(mm, ptep, pte, nr);
635 break;
636 case PMD_SIZE:
637 page_table_check_pmds_set(mm, (pmd_t *)ptep, pte_pmd(pte), nr);
638 break;
> 639 case PUD_SIZE:
640 page_table_check_puds_set(mm, (pud_t *)ptep, pte_pud(pte), nr);
641 break;
642 default:
643 VM_WARN_ON(1);
644 }
645
646 __sync_cache_and_tags(pte, nr * stride);
647
648 for (;;) {
649 __check_safe_pte_update(mm, ptep, pte);
650 __set_pte(ptep, pte);
651 if (--nr == 0)
652 break;
653 ptep++;
654 pte = pte_advance_pfn(pte, stride);
655 }
656 }
657
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
On 06/03/2025 05:08, kernel test robot wrote:
> Hi Ryan,
>
> kernel test robot noticed the following build errors:
>
> [auto build test ERROR on linus/master]
> [also build test ERROR on v6.14-rc5 next-20250305]
> [cannot apply to arm64/for-next/core akpm-mm/mm-everything arm-perf/for-next/perf]
> [If your patch is applied to the wrong git tree, kindly drop us a note.
> And when submitting patch, we suggest to use '--base' as documented in
> https://git-scm.com/docs/git-format-patch#_base_tree_information]
>
> url: https://github.com/intel-lab-lkp/linux/commits/Ryan-Roberts/arm64-hugetlb-Cleanup-huge_pte-size-discovery-mechanisms/20250304-230647
> base: linus/master
> patch link: https://lore.kernel.org/r/20250304150444.3788920-5-ryan.roberts%40arm.com
> patch subject: [PATCH v3 04/11] arm64/mm: Refactor __set_ptes() and __ptep_get_and_clear()
> config: arm64-randconfig-001-20250305 (https://download.01.org/0day-ci/archive/20250306/202503061237.QurSXHSC-lkp@intel.com/config)
> compiler: clang version 15.0.7 (https://github.com/llvm/llvm-project 8dfdcc7b7bf66834a761bd8de445840ef68e4d1a)
> reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250306/202503061237.QurSXHSC-lkp@intel.com/reproduce)
>
> If you fix the issue in a separate patch/commit (i.e. not just a new version of
> the same patch/commit), kindly add following tags
> | Reported-by: kernel test robot <lkp@intel.com>
> | Closes: https://lore.kernel.org/oe-kbuild-all/202503061237.QurSXHSC-lkp@intel.com/
>
> All errors (new ones prefixed by >>):
>
> In file included from arch/arm64/kernel/asm-offsets.c:12:
> In file included from include/linux/ftrace.h:10:
> In file included from include/linux/trace_recursion.h:5:
> In file included from include/linux/interrupt.h:11:
> In file included from include/linux/hardirq.h:11:
> In file included from arch/arm64/include/asm/hardirq.h:17:
> In file included from include/asm-generic/hardirq.h:17:
> In file included from include/linux/irq.h:20:
> In file included from include/linux/io.h:14:
> In file included from arch/arm64/include/asm/io.h:12:
> In file included from include/linux/pgtable.h:6:
>>> arch/arm64/include/asm/pgtable.h:639:7: error: duplicate case value '536870912'
> case PUD_SIZE:
> ^
> include/asm-generic/pgtable-nopud.h:20:20: note: expanded from macro 'PUD_SIZE'
> #define PUD_SIZE (1UL << PUD_SHIFT)
> ^
> arch/arm64/include/asm/pgtable.h:636:7: note: previous case defined here
> case PMD_SIZE:
> ^
> include/asm-generic/pgtable-nopmd.h:22:20: note: expanded from macro 'PMD_SIZE'
> #define PMD_SIZE (1UL << PMD_SHIFT)
> ^
> In file included from arch/arm64/kernel/asm-offsets.c:12:
> In file included from include/linux/ftrace.h:10:
> In file included from include/linux/trace_recursion.h:5:
> In file included from include/linux/interrupt.h:11:
> In file included from include/linux/hardirq.h:11:
> In file included from arch/arm64/include/asm/hardirq.h:17:
> In file included from include/asm-generic/hardirq.h:17:
> In file included from include/linux/irq.h:20:
> In file included from include/linux/io.h:14:
> In file included from arch/arm64/include/asm/io.h:12:
> In file included from include/linux/pgtable.h:6:
> arch/arm64/include/asm/pgtable.h:1303:7: error: duplicate case value '536870912'
> case PUD_SIZE:
> ^
> include/asm-generic/pgtable-nopud.h:20:20: note: expanded from macro 'PUD_SIZE'
> #define PUD_SIZE (1UL << PUD_SHIFT)
> ^
> arch/arm64/include/asm/pgtable.h:1300:7: note: previous case defined here
> case PMD_SIZE:
> ^
> include/asm-generic/pgtable-nopmd.h:22:20: note: expanded from macro 'PMD_SIZE'
> #define PMD_SIZE (1UL << PMD_SHIFT)
> ^
> 2 errors generated.
> make[3]: *** [scripts/Makefile.build:102: arch/arm64/kernel/asm-offsets.s] Error 1 shuffle=4064171735
> make[3]: Target 'prepare' not remade because of errors.
> make[2]: *** [Makefile:1264: prepare0] Error 2 shuffle=4064171735
> make[2]: Target 'prepare' not remade because of errors.
> make[1]: *** [Makefile:251: __sub-make] Error 2 shuffle=4064171735
> make[1]: Target 'prepare' not remade because of errors.
> make: *** [Makefile:251: __sub-make] Error 2 shuffle=4064171735
> make: Target 'prepare' not remade because of errors.
>
>
> vim +/536870912 +639 arch/arm64/include/asm/pgtable.h
>
> 626
> 627 static inline void set_ptes_anysz(struct mm_struct *mm, pte_t *ptep, pte_t pte,
> 628 unsigned int nr, unsigned long pgsize)
> 629 {
> 630 unsigned long stride = pgsize >> PAGE_SHIFT;
> 631
> 632 switch (pgsize) {
> 633 case PAGE_SIZE:
> 634 page_table_check_ptes_set(mm, ptep, pte, nr);
> 635 break;
> 636 case PMD_SIZE:
> 637 page_table_check_pmds_set(mm, (pmd_t *)ptep, pte_pmd(pte), nr);
> 638 break;
> > 639 case PUD_SIZE:
> 640 page_table_check_puds_set(mm, (pud_t *)ptep, pte_pud(pte), nr);
> 641 break;
Looks like this needs to be wrapped in `#ifndef __PAGETABLE_PMD_FOLDED`. This
failing config folds the PMD so PMD_SIZE and PUD_SIZE are the same.
Given there are now 2 kernel robot reports, I'll respin the series next week,
giving time for any interim review comments.
Thanks,
Ryan
> 642 default:
> 643 VM_WARN_ON(1);
> 644 }
> 645
> 646 __sync_cache_and_tags(pte, nr * stride);
> 647
> 648 for (;;) {
> 649 __check_safe_pte_update(mm, ptep, pte);
> 650 __set_pte(ptep, pte);
> 651 if (--nr == 0)
> 652 break;
> 653 ptep++;
> 654 pte = pte_advance_pfn(pte, stride);
> 655 }
> 656 }
> 657
>
© 2016 - 2026 Red Hat, Inc.