This is done by marking the region for which to skip initialization
with the MEMBLOCK_RSRV_NOINIT flag.
If the region is for hugepages and if HVO is enabled, then those
struct pages which will be freed later don't need to be initialized.
This can save significant time when a large number of hugepages are
allocated at boot time. HUGETLB_VMEMMAP_RESERVE_SIZE
struct pages at the start of hugepage still need to be initialized.
Signed-off-by: Usama Arif <usama.arif@bytedance.com>
---
mm/hugetlb.c | 21 +++++++++++++++++++++
mm/hugetlb_vmemmap.c | 2 +-
mm/hugetlb_vmemmap.h | 3 +++
3 files changed, 25 insertions(+), 1 deletion(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index c1fcf2af591a..bb2b12f41026 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3166,6 +3166,9 @@ int __alloc_bootmem_huge_page(struct hstate *h, int nid)
{
struct huge_bootmem_page *m = NULL; /* initialize for clang */
int nr_nodes, node;
+ phys_addr_t hugetlb_vmemmap_reserve_size =
+ HUGETLB_VMEMMAP_RESERVE_SIZE * sizeof(struct page);
+ phys_addr_t noinit_base;
/* do node specific alloc */
if (nid != NUMA_NO_NODE) {
@@ -3173,6 +3176,15 @@ int __alloc_bootmem_huge_page(struct hstate *h, int nid)
0, MEMBLOCK_ALLOC_ACCESSIBLE, nid);
if (!m)
return 0;
+
+ if (vmemmap_optimize_enabled && hugetlb_vmemmap_optimizable(h)) {
+ noinit_base = virt_to_phys(
+ (void *)((phys_addr_t) m + hugetlb_vmemmap_reserve_size));
+ memblock_rsrv_mark_noinit(
+ noinit_base,
+ huge_page_size(h) - hugetlb_vmemmap_reserve_size);
+ }
+
goto found;
}
/* allocate from next node when distributing huge pages */
@@ -3187,6 +3199,15 @@ int __alloc_bootmem_huge_page(struct hstate *h, int nid)
*/
if (!m)
return 0;
+
+ if (vmemmap_optimize_enabled && hugetlb_vmemmap_optimizable(h)) {
+ noinit_base = virt_to_phys(
+ (void *)((phys_addr_t) m + hugetlb_vmemmap_reserve_size));
+ memblock_rsrv_mark_noinit(
+ noinit_base,
+ huge_page_size(h) - hugetlb_vmemmap_reserve_size);
+ }
+
goto found;
}
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index bdf750a4786b..b5b7834e0f42 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -443,7 +443,7 @@ static int vmemmap_remap_alloc(unsigned long start, unsigned long end,
DEFINE_STATIC_KEY_FALSE(hugetlb_optimize_vmemmap_key);
EXPORT_SYMBOL(hugetlb_optimize_vmemmap_key);
-static bool vmemmap_optimize_enabled = IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON);
+bool vmemmap_optimize_enabled = IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON);
core_param(hugetlb_free_vmemmap, vmemmap_optimize_enabled, bool, 0);
/**
diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h
index 07555d2dc0cb..cb5171abe683 100644
--- a/mm/hugetlb_vmemmap.h
+++ b/mm/hugetlb_vmemmap.h
@@ -64,4 +64,7 @@ static inline bool hugetlb_vmemmap_optimizable(const struct hstate *h)
{
return hugetlb_vmemmap_optimizable_size(h) != 0;
}
+
+extern bool vmemmap_optimize_enabled;
+
#endif /* _LINUX_HUGETLB_VMEMMAP_H */
--
2.25.1Hi Usama,
kernel test robot noticed the following build errors:
[auto build test ERROR on akpm-mm/mm-everything]
url: https://github.com/intel-lab-lkp/linux/commits/Usama-Arif/mm-hugetlb-Skip-prep-of-tail-pages-when-HVO-is-enabled/20230728-044839
base: https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-everything
patch link: https://lore.kernel.org/r/20230727204624.1942372-7-usama.arif%40bytedance.com
patch subject: [v1 6/6] mm: hugetlb: Skip initialization of struct pages freed later by HVO
config: arm64-randconfig-r032-20230727 (https://download.01.org/0day-ci/archive/20230729/202307290124.suQ4U8Y4-lkp@intel.com/config)
compiler: clang version 17.0.0 (https://github.com/llvm/llvm-project.git 4a5ac14ee968ff0ad5d2cc1ffa0299048db4c88a)
reproduce: (https://download.01.org/0day-ci/archive/20230729/202307290124.suQ4U8Y4-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202307290124.suQ4U8Y4-lkp@intel.com/
All errors (new ones prefixed by >>):
In file included from mm/hugetlb.c:49:
mm/hugetlb_vmemmap.h:56:6: warning: no previous prototype for function 'vmemmap_should_optimize' [-Wmissing-prototypes]
56 | bool vmemmap_should_optimize(const struct hstate *h, const struct page *head)
| ^
mm/hugetlb_vmemmap.h:56:1: note: declare 'static' if the function is not intended to be used outside of this translation unit
56 | bool vmemmap_should_optimize(const struct hstate *h, const struct page *head)
| ^
| static
>> mm/hugetlb.c:3198:3: error: use of undeclared identifier 'HUGETLB_VMEMMAP_RESERVE_SIZE'
3198 | HUGETLB_VMEMMAP_RESERVE_SIZE * sizeof(struct page);
| ^
1 warning and 1 error generated.
vim +/HUGETLB_VMEMMAP_RESERVE_SIZE +3198 mm/hugetlb.c
3190
3191 int alloc_bootmem_huge_page(struct hstate *h, int nid)
3192 __attribute__ ((weak, alias("__alloc_bootmem_huge_page")));
3193 int __alloc_bootmem_huge_page(struct hstate *h, int nid)
3194 {
3195 struct huge_bootmem_page *m = NULL; /* initialize for clang */
3196 int nr_nodes, node;
3197 phys_addr_t hugetlb_vmemmap_reserve_size =
> 3198 HUGETLB_VMEMMAP_RESERVE_SIZE * sizeof(struct page);
3199 phys_addr_t noinit_base;
3200
3201 /* do node specific alloc */
3202 if (nid != NUMA_NO_NODE) {
3203 m = memblock_alloc_try_nid_raw(huge_page_size(h), huge_page_size(h),
3204 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid);
3205 if (!m)
3206 return 0;
3207
3208 if (vmemmap_optimize_enabled && hugetlb_vmemmap_optimizable(h)) {
3209 noinit_base = virt_to_phys(
3210 (void *)((phys_addr_t) m + hugetlb_vmemmap_reserve_size));
3211 memblock_rsrv_mark_noinit(
3212 noinit_base,
3213 huge_page_size(h) - hugetlb_vmemmap_reserve_size);
3214 }
3215
3216 goto found;
3217 }
3218 /* allocate from next node when distributing huge pages */
3219 for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
3220 m = memblock_alloc_try_nid_raw(
3221 huge_page_size(h), huge_page_size(h),
3222 0, MEMBLOCK_ALLOC_ACCESSIBLE, node);
3223 /*
3224 * Use the beginning of the huge page to store the
3225 * huge_bootmem_page struct (until gather_bootmem
3226 * puts them into the mem_map).
3227 */
3228 if (!m)
3229 return 0;
3230
3231 if (vmemmap_optimize_enabled && hugetlb_vmemmap_optimizable(h)) {
3232 noinit_base = virt_to_phys(
3233 (void *)((phys_addr_t) m + hugetlb_vmemmap_reserve_size));
3234 memblock_rsrv_mark_noinit(
3235 noinit_base,
3236 huge_page_size(h) - hugetlb_vmemmap_reserve_size);
3237 }
3238
3239 goto found;
3240 }
3241
3242 found:
3243 /* Put them into a private list first because mem_map is not up yet */
3244 INIT_LIST_HEAD(&m->list);
3245 list_add(&m->list, &huge_boot_pages);
3246 m->hstate = h;
3247 return 1;
3248 }
3249
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
Hi Usama,
kernel test robot noticed the following build warnings:
[auto build test WARNING on akpm-mm/mm-everything]
url: https://github.com/intel-lab-lkp/linux/commits/Usama-Arif/mm-hugetlb-Skip-prep-of-tail-pages-when-HVO-is-enabled/20230728-044839
base: https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-everything
patch link: https://lore.kernel.org/r/20230727204624.1942372-7-usama.arif%40bytedance.com
patch subject: [v1 6/6] mm: hugetlb: Skip initialization of struct pages freed later by HVO
config: i386-debian-10.3 (https://download.01.org/0day-ci/archive/20230729/202307290029.Kr5EEBeY-lkp@intel.com/config)
compiler: gcc-12 (Debian 12.2.0-14) 12.2.0
reproduce: (https://download.01.org/0day-ci/archive/20230729/202307290029.Kr5EEBeY-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202307290029.Kr5EEBeY-lkp@intel.com/
All warnings (new ones prefixed by >>):
In file included from mm/hugetlb.c:49:
mm/hugetlb_vmemmap.h:56:6: warning: no previous prototype for 'vmemmap_should_optimize' [-Wmissing-prototypes]
56 | bool vmemmap_should_optimize(const struct hstate *h, const struct page *head)
| ^~~~~~~~~~~~~~~~~~~~~~~
mm/hugetlb.c: In function '__alloc_bootmem_huge_page':
mm/hugetlb.c:3198:17: error: 'HUGETLB_VMEMMAP_RESERVE_SIZE' undeclared (first use in this function)
3198 | HUGETLB_VMEMMAP_RESERVE_SIZE * sizeof(struct page);
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~
mm/hugetlb.c:3198:17: note: each undeclared identifier is reported only once for each function it appears in
>> mm/hugetlb.c:3210:42: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
3210 | (void *)((phys_addr_t) m + hugetlb_vmemmap_reserve_size));
| ^
>> mm/hugetlb.c:3210:33: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
3210 | (void *)((phys_addr_t) m + hugetlb_vmemmap_reserve_size));
| ^
mm/hugetlb.c:3233:42: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast]
3233 | (void *)((phys_addr_t) m + hugetlb_vmemmap_reserve_size));
| ^
mm/hugetlb.c:3233:33: warning: cast to pointer from integer of different size [-Wint-to-pointer-cast]
3233 | (void *)((phys_addr_t) m + hugetlb_vmemmap_reserve_size));
| ^
vim +3210 mm/hugetlb.c
3190
3191 int alloc_bootmem_huge_page(struct hstate *h, int nid)
3192 __attribute__ ((weak, alias("__alloc_bootmem_huge_page")));
3193 int __alloc_bootmem_huge_page(struct hstate *h, int nid)
3194 {
3195 struct huge_bootmem_page *m = NULL; /* initialize for clang */
3196 int nr_nodes, node;
3197 phys_addr_t hugetlb_vmemmap_reserve_size =
3198 HUGETLB_VMEMMAP_RESERVE_SIZE * sizeof(struct page);
3199 phys_addr_t noinit_base;
3200
3201 /* do node specific alloc */
3202 if (nid != NUMA_NO_NODE) {
3203 m = memblock_alloc_try_nid_raw(huge_page_size(h), huge_page_size(h),
3204 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid);
3205 if (!m)
3206 return 0;
3207
3208 if (vmemmap_optimize_enabled && hugetlb_vmemmap_optimizable(h)) {
3209 noinit_base = virt_to_phys(
> 3210 (void *)((phys_addr_t) m + hugetlb_vmemmap_reserve_size));
3211 memblock_rsrv_mark_noinit(
3212 noinit_base,
3213 huge_page_size(h) - hugetlb_vmemmap_reserve_size);
3214 }
3215
3216 goto found;
3217 }
3218 /* allocate from next node when distributing huge pages */
3219 for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
3220 m = memblock_alloc_try_nid_raw(
3221 huge_page_size(h), huge_page_size(h),
3222 0, MEMBLOCK_ALLOC_ACCESSIBLE, node);
3223 /*
3224 * Use the beginning of the huge page to store the
3225 * huge_bootmem_page struct (until gather_bootmem
3226 * puts them into the mem_map).
3227 */
3228 if (!m)
3229 return 0;
3230
3231 if (vmemmap_optimize_enabled && hugetlb_vmemmap_optimizable(h)) {
3232 noinit_base = virt_to_phys(
3233 (void *)((phys_addr_t) m + hugetlb_vmemmap_reserve_size));
3234 memblock_rsrv_mark_noinit(
3235 noinit_base,
3236 huge_page_size(h) - hugetlb_vmemmap_reserve_size);
3237 }
3238
3239 goto found;
3240 }
3241
3242 found:
3243 /* Put them into a private list first because mem_map is not up yet */
3244 INIT_LIST_HEAD(&m->list);
3245 list_add(&m->list, &huge_boot_pages);
3246 m->hstate = h;
3247 return 1;
3248 }
3249
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
© 2016 - 2025 Red Hat, Inc.