From: Pankaj Raghav <p.raghav@samsung.com>
There are many places in the kernel where we need to zeroout larger
chunks but the maximum segment we can zeroout at a time by ZERO_PAGE
is limited by PAGE_SIZE.
This is especially annoying in block devices and filesystems where we
attach multiple ZERO_PAGEs to the bio in different bvecs. With multipage
bvec support in block layer, it is much more efficient to send out
larger zero pages as a part of single bvec.
This concern was raised during the review of adding LBS support to
XFS[1][2].
Usually huge_zero_folio is allocated on demand, and it will be
deallocated by the shrinker if there are no users of it left. At moment,
huge_zero_folio infrastructure refcount is tied to the process lifetime
that created it. This might not work for bio layer as the completitions
can be async and the process that created the huge_zero_folio might no
longer be alive.
Add a config option STATIC_PMD_ZERO_PAGE that will always allocate
the huge_zero_folio, and it will never be freed. This makes using the
huge_zero_folio without having to pass any mm struct and does not tie
the lifetime of the zero folio to anything.
memblock is used to allocated this PMD zero page during early boot.
If STATIC_PMD_ZERO_PAGE config option is enabled, then
mm_get_huge_zero_folio() will simply return this page instead of
dynamically allocating a new PMD page.
As STATIC_PMD_ZERO_PAGE does not depend on THP, declare huge_zero_folio
and huge_zero_pfn outside the THP config.
[1] https://lore.kernel.org/linux-xfs/20231027051847.GA7885@lst.de/
[2] https://lore.kernel.org/linux-xfs/ZitIK5OnR7ZNY0IG@infradead.org/
Suggested-by: David Hildenbrand <david@redhat.com>
Signed-off-by: Pankaj Raghav <p.raghav@samsung.com>
---
include/linux/mm.h | 25 ++++++++++++++++++++++++-
mm/Kconfig | 9 +++++++++
mm/huge_memory.c | 24 ++++++++++++++++++++----
mm/memory.c | 25 +++++++++++++++++++++++++
mm/mm_init.c | 1 +
5 files changed, 79 insertions(+), 5 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index c8fbeaacf896..428fe6d36b3c 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -4018,10 +4018,19 @@ static inline bool vma_is_special_huge(const struct vm_area_struct *vma)
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+#ifdef CONFIG_STATIC_PMD_ZERO_PAGE
+extern void __init static_pmd_zero_init(void);
+#else
+static inline void __init static_pmd_zero_init(void)
+{
+ return;
+}
+#endif
+
extern struct folio *huge_zero_folio;
extern unsigned long huge_zero_pfn;
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline bool is_huge_zero_folio(const struct folio *folio)
{
return READ_ONCE(huge_zero_folio) == folio;
@@ -4032,9 +4041,23 @@ static inline bool is_huge_zero_pmd(pmd_t pmd)
return pmd_present(pmd) && READ_ONCE(huge_zero_pfn) == pmd_pfn(pmd);
}
+#ifdef CONFIG_STATIC_PMD_ZERO_PAGE
+static inline struct folio *mm_get_huge_zero_folio(struct mm_struct *mm)
+{
+ return READ_ONCE(huge_zero_folio);
+}
+
+static inline void mm_put_huge_zero_folio(struct mm_struct *mm)
+{
+ return;
+}
+
+#else
struct folio *mm_get_huge_zero_folio(struct mm_struct *mm);
void mm_put_huge_zero_folio(struct mm_struct *mm);
+#endif /* CONFIG_STATIC_PMD_ZERO_PAGE */
+
#else
static inline bool is_huge_zero_folio(const struct folio *folio)
{
diff --git a/mm/Kconfig b/mm/Kconfig
index 781be3240e21..89d5971cf180 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -826,6 +826,15 @@ config ARCH_WANTS_THP_SWAP
config MM_ID
def_bool n
+config STATIC_PMD_ZERO_PAGE
+ bool "Allocate a PMD page for zeroing"
+ help
+ Typically huge_zero_folio, which is a PMD page of zeroes, is allocated
+ on demand and deallocated when not in use. This option will
+ allocate a PMD sized zero page during early boot and huge_zero_folio will
+ use it instead allocating dynamically.
+ Not suitable for memory constrained systems.
+
menuconfig TRANSPARENT_HUGEPAGE
bool "Transparent Hugepage Support"
depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE && !PREEMPT_RT
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 101b67ab2eb6..c12ca7134e88 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -75,9 +75,6 @@ static unsigned long deferred_split_scan(struct shrinker *shrink,
struct shrink_control *sc);
static bool split_underused_thp = true;
-static atomic_t huge_zero_refcount;
-struct folio *huge_zero_folio __read_mostly;
-unsigned long huge_zero_pfn __read_mostly = ~0UL;
unsigned long huge_anon_orders_always __read_mostly;
unsigned long huge_anon_orders_madvise __read_mostly;
unsigned long huge_anon_orders_inherit __read_mostly;
@@ -208,6 +205,23 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
return orders;
}
+#ifdef CONFIG_STATIC_PMD_ZERO_PAGE
+static int huge_zero_page_shrinker_init(void)
+{
+ return 0;
+}
+
+static void huge_zero_page_shrinker_exit(void)
+{
+ return;
+}
+#else
+
+static struct shrinker *huge_zero_page_shrinker;
+static atomic_t huge_zero_refcount;
+struct folio *huge_zero_folio __read_mostly;
+unsigned long huge_zero_pfn __read_mostly = ~0UL;
+
static bool get_huge_zero_page(void)
{
struct folio *zero_folio;
@@ -288,7 +302,6 @@ static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
return 0;
}
-static struct shrinker *huge_zero_page_shrinker;
static int huge_zero_page_shrinker_init(void)
{
huge_zero_page_shrinker = shrinker_alloc(0, "thp-zero");
@@ -307,6 +320,7 @@ static void huge_zero_page_shrinker_exit(void)
return;
}
+#endif
#ifdef CONFIG_SYSFS
static ssize_t enabled_show(struct kobject *kobj,
@@ -2843,6 +2857,8 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
pte_t *pte;
int i;
+ // FIXME: can this be called with static zero page?
+ VM_BUG_ON(IS_ENABLED(CONFIG_STATIC_PMD_ZERO_PAGE));
/*
* Leave pmd empty until pte is filled note that it is fine to delay
* notification until mmu_notifier_invalidate_range_end() as we are
diff --git a/mm/memory.c b/mm/memory.c
index b0cda5aab398..42c4c31ad14c 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -42,6 +42,7 @@
#include <linux/kernel_stat.h>
#include <linux/mm.h>
#include <linux/mm_inline.h>
+#include <linux/memblock.h>
#include <linux/sched/mm.h>
#include <linux/sched/numa_balancing.h>
#include <linux/sched/task.h>
@@ -159,6 +160,30 @@ static int __init init_zero_pfn(void)
}
early_initcall(init_zero_pfn);
+#ifdef CONFIG_STATIC_PMD_ZERO_PAGE
+struct folio *huge_zero_folio __read_mostly = NULL;
+unsigned long huge_zero_pfn __read_mostly = ~0UL;
+
+void __init static_pmd_zero_init(void)
+{
+ void *alloc = memblock_alloc(PMD_SIZE, PAGE_SIZE);
+
+ if (!alloc)
+ return;
+
+ huge_zero_folio = virt_to_folio(alloc);
+ huge_zero_pfn = page_to_pfn(virt_to_page(alloc));
+
+ __folio_set_head(huge_zero_folio);
+ prep_compound_head((struct page *)huge_zero_folio, PMD_ORDER);
+ /* Ensure zero folio won't have large_rmappable flag set. */
+ folio_clear_large_rmappable(huge_zero_folio);
+ folio_zero_range(huge_zero_folio, 0, PMD_SIZE);
+
+ return;
+}
+#endif
+
void mm_trace_rss_stat(struct mm_struct *mm, int member)
{
trace_rss_stat(mm, member);
diff --git a/mm/mm_init.c b/mm/mm_init.c
index f2944748f526..56d7ec372af1 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -2765,6 +2765,7 @@ void __init mm_core_init(void)
*/
kho_memory_init();
+ static_pmd_zero_init();
memblock_free_all();
mem_init();
kmem_cache_init();
--
2.49.0On Mon, Jul 07, 2025 at 04:23:17PM +0200, Pankaj Raghav (Samsung) wrote:
> From: Pankaj Raghav <p.raghav@samsung.com>
>
> There are many places in the kernel where we need to zeroout larger
> chunks but the maximum segment we can zeroout at a time by ZERO_PAGE
> is limited by PAGE_SIZE.
>
> This is especially annoying in block devices and filesystems where we
> attach multiple ZERO_PAGEs to the bio in different bvecs. With multipage
> bvec support in block layer, it is much more efficient to send out
> larger zero pages as a part of single bvec.
>
> This concern was raised during the review of adding LBS support to
> XFS[1][2].
Nit, but maybe worth spelling out LBS = (presumably :P) Large Block
Support.
>
> Usually huge_zero_folio is allocated on demand, and it will be
> deallocated by the shrinker if there are no users of it left. At moment,
> huge_zero_folio infrastructure refcount is tied to the process lifetime
> that created it. This might not work for bio layer as the completitions
> can be async and the process that created the huge_zero_folio might no
> longer be alive.
>
> Add a config option STATIC_PMD_ZERO_PAGE that will always allocate
> the huge_zero_folio, and it will never be freed. This makes using the
> huge_zero_folio without having to pass any mm struct and does not tie
> the lifetime of the zero folio to anything.
Can we in that case #ifndef CONFIG_STATIC_PMD_ZERO_PAGE around the refcount
logic?
And surely we should additionally update mm_get_huge_zero_folio() etc. to
account for this?
>
> memblock is used to allocated this PMD zero page during early boot.
>
> If STATIC_PMD_ZERO_PAGE config option is enabled, then
> mm_get_huge_zero_folio() will simply return this page instead of
> dynamically allocating a new PMD page.
>
> As STATIC_PMD_ZERO_PAGE does not depend on THP, declare huge_zero_folio
> and huge_zero_pfn outside the THP config.
>
> [1] https://lore.kernel.org/linux-xfs/20231027051847.GA7885@lst.de/
> [2] https://lore.kernel.org/linux-xfs/ZitIK5OnR7ZNY0IG@infradead.org/
>
> Suggested-by: David Hildenbrand <david@redhat.com>
> Signed-off-by: Pankaj Raghav <p.raghav@samsung.com>
> ---
> include/linux/mm.h | 25 ++++++++++++++++++++++++-
> mm/Kconfig | 9 +++++++++
> mm/huge_memory.c | 24 ++++++++++++++++++++----
> mm/memory.c | 25 +++++++++++++++++++++++++
> mm/mm_init.c | 1 +
> 5 files changed, 79 insertions(+), 5 deletions(-)
>
> diff --git a/include/linux/mm.h b/include/linux/mm.h
> index c8fbeaacf896..428fe6d36b3c 100644
> --- a/include/linux/mm.h
> +++ b/include/linux/mm.h
> @@ -4018,10 +4018,19 @@ static inline bool vma_is_special_huge(const struct vm_area_struct *vma)
>
> #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
>
> -#ifdef CONFIG_TRANSPARENT_HUGEPAGE
> +#ifdef CONFIG_STATIC_PMD_ZERO_PAGE
> +extern void __init static_pmd_zero_init(void);
We don't use extern for this kind of function declaration, and actually try
to remove extern's as we touch header decls that have them as we go.
> +#else
> +static inline void __init static_pmd_zero_init(void)
> +{
> + return;
This return is redundant.
> +}
> +#endif
> +
> extern struct folio *huge_zero_folio;
> extern unsigned long huge_zero_pfn;
>
> +#ifdef CONFIG_TRANSPARENT_HUGEPAGE
OK I guess the point here is to make huge_zero_folio, huge_zero_pfn
available regardless of whether THP is enabled.
Again, I really think this should live in huge_mm.h and any place that
doesn't include it needs to like, just include it :)
I really don't want these randomly placed in mm.h if we can avoid it.
Can we also add a comment saying 'this is used for both static huge PMD and THP
> static inline bool is_huge_zero_folio(const struct folio *folio)
> {
> return READ_ONCE(huge_zero_folio) == folio;
> @@ -4032,9 +4041,23 @@ static inline bool is_huge_zero_pmd(pmd_t pmd)
> return pmd_present(pmd) && READ_ONCE(huge_zero_pfn) == pmd_pfn(pmd);
> }
>
> +#ifdef CONFIG_STATIC_PMD_ZERO_PAGE
> +static inline struct folio *mm_get_huge_zero_folio(struct mm_struct *mm)
> +{
> + return READ_ONCE(huge_zero_folio);
> +}
> +
> +static inline void mm_put_huge_zero_folio(struct mm_struct *mm)
> +{
> + return;
This return is redundant.
> +}
> +
> +#else
> struct folio *mm_get_huge_zero_folio(struct mm_struct *mm);
> void mm_put_huge_zero_folio(struct mm_struct *mm);
>
> +#endif /* CONFIG_STATIC_PMD_ZERO_PAGE */
> +
> #else
> static inline bool is_huge_zero_folio(const struct folio *folio)
> {
> diff --git a/mm/Kconfig b/mm/Kconfig
> index 781be3240e21..89d5971cf180 100644
> --- a/mm/Kconfig
> +++ b/mm/Kconfig
> @@ -826,6 +826,15 @@ config ARCH_WANTS_THP_SWAP
> config MM_ID
> def_bool n
>
> +config STATIC_PMD_ZERO_PAGE
> + bool "Allocate a PMD page for zeroing"
> + help
> + Typically huge_zero_folio, which is a PMD page of zeroes, is allocated
> + on demand and deallocated when not in use. This option will
> + allocate a PMD sized zero page during early boot and huge_zero_folio will
> + use it instead allocating dynamically.
> + Not suitable for memory constrained systems.
Would have to be pretty constrained to not spare 2 MiB :P but I accept of
course these devices do exist...
> +
> menuconfig TRANSPARENT_HUGEPAGE
> bool "Transparent Hugepage Support"
> depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE && !PREEMPT_RT
> diff --git a/mm/huge_memory.c b/mm/huge_memory.c
> index 101b67ab2eb6..c12ca7134e88 100644
> --- a/mm/huge_memory.c
> +++ b/mm/huge_memory.c
> @@ -75,9 +75,6 @@ static unsigned long deferred_split_scan(struct shrinker *shrink,
> struct shrink_control *sc);
> static bool split_underused_thp = true;
>
> -static atomic_t huge_zero_refcount;
> -struct folio *huge_zero_folio __read_mostly;
> -unsigned long huge_zero_pfn __read_mostly = ~0UL;
Ugh yeah this is a mess.
I see you're moving this to mm/memory.c because we only compile
huge_memory.c if THP is enabled.
Are there any circumstances where it makes sense to want to use static PMD
page and NOT have THP enabled?
It'd just be simpler if we could have CONFIG_STATIC_PMD_ZERO_PAGE depend on
CONFIG_TRANSPARENT_HUGEPAGE.
Why can't we do that?
> unsigned long huge_anon_orders_always __read_mostly;
> unsigned long huge_anon_orders_madvise __read_mostly;
> unsigned long huge_anon_orders_inherit __read_mostly;
> @@ -208,6 +205,23 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
> return orders;
> }
>
> +#ifdef CONFIG_STATIC_PMD_ZERO_PAGE
> +static int huge_zero_page_shrinker_init(void)
> +{
> + return 0;
> +}
> +
> +static void huge_zero_page_shrinker_exit(void)
> +{
> + return;
You seem to love putting return statements in void functions like this :P
you don't need to, please remove.
> +}
> +#else
> +
> +static struct shrinker *huge_zero_page_shrinker;
> +static atomic_t huge_zero_refcount;
> +struct folio *huge_zero_folio __read_mostly;
> +unsigned long huge_zero_pfn __read_mostly = ~0UL;
> +
> static bool get_huge_zero_page(void)
> {
> struct folio *zero_folio;
> @@ -288,7 +302,6 @@ static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
> return 0;
> }
>
> -static struct shrinker *huge_zero_page_shrinker;
> static int huge_zero_page_shrinker_init(void)
> {
> huge_zero_page_shrinker = shrinker_alloc(0, "thp-zero");
> @@ -307,6 +320,7 @@ static void huge_zero_page_shrinker_exit(void)
> return;
> }
>
> +#endif
>
> #ifdef CONFIG_SYSFS
> static ssize_t enabled_show(struct kobject *kobj,
> @@ -2843,6 +2857,8 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
> pte_t *pte;
> int i;
>
> + // FIXME: can this be called with static zero page?
This shouldn't be in upstream code, it's up to you to determine this. And
please don't use //.
> + VM_BUG_ON(IS_ENABLED(CONFIG_STATIC_PMD_ZERO_PAGE));
Also [VM_]BUG_ON() is _entirely_ deprecated. This should be
VM_WARN_ON_ONCE().
> /*
> * Leave pmd empty until pte is filled note that it is fine to delay
> * notification until mmu_notifier_invalidate_range_end() as we are
> diff --git a/mm/memory.c b/mm/memory.c
> index b0cda5aab398..42c4c31ad14c 100644
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -42,6 +42,7 @@
> #include <linux/kernel_stat.h>
> #include <linux/mm.h>
> #include <linux/mm_inline.h>
> +#include <linux/memblock.h>
> #include <linux/sched/mm.h>
> #include <linux/sched/numa_balancing.h>
> #include <linux/sched/task.h>
> @@ -159,6 +160,30 @@ static int __init init_zero_pfn(void)
> }
> early_initcall(init_zero_pfn);
>
> +#ifdef CONFIG_STATIC_PMD_ZERO_PAGE
> +struct folio *huge_zero_folio __read_mostly = NULL;
> +unsigned long huge_zero_pfn __read_mostly = ~0UL;
> +
> +void __init static_pmd_zero_init(void)
> +{
> + void *alloc = memblock_alloc(PMD_SIZE, PAGE_SIZE);
> +
> + if (!alloc)
> + return;
Ummm... so we're fine with just having huge_zero_folio, huge_zero_pfn
unintialised if the allocation fails?
This seems to be to be a rare case where we should panic the kernel?
Because everything's broken now.
There's actually a memblock_alloc_or_panic() function you could use for
this.
> +
> + huge_zero_folio = virt_to_folio(alloc);
> + huge_zero_pfn = page_to_pfn(virt_to_page(alloc));
> +
> + __folio_set_head(huge_zero_folio);
> + prep_compound_head((struct page *)huge_zero_folio, PMD_ORDER);
What will the reference count be on the folio here? Might something
acccidentally put this somewhere if we're not careful?
> + /* Ensure zero folio won't have large_rmappable flag set. */
> + folio_clear_large_rmappable(huge_zero_folio);
Why? What would set it?
I'm a little concerned as to whether this folio is correctly initialised,
need to be careful here.
> + folio_zero_range(huge_zero_folio, 0, PMD_SIZE);
> +
> + return;
You don't need to put returns at the end of void functions.
> +}
> +#endif
> +
> void mm_trace_rss_stat(struct mm_struct *mm, int member)
> {
> trace_rss_stat(mm, member);
> diff --git a/mm/mm_init.c b/mm/mm_init.c
> index f2944748f526..56d7ec372af1 100644
> --- a/mm/mm_init.c
> +++ b/mm/mm_init.c
> @@ -2765,6 +2765,7 @@ void __init mm_core_init(void)
> */
> kho_memory_init();
>
> + static_pmd_zero_init();
> memblock_free_all();
> mem_init();
> kmem_cache_init();
> --
> 2.49.0
>
On 07.07.25 16:23, Pankaj Raghav (Samsung) wrote: > From: Pankaj Raghav <p.raghav@samsung.com> > > There are many places in the kernel where we need to zeroout larger > chunks but the maximum segment we can zeroout at a time by ZERO_PAGE > is limited by PAGE_SIZE. > > This is especially annoying in block devices and filesystems where we > attach multiple ZERO_PAGEs to the bio in different bvecs. With multipage > bvec support in block layer, it is much more efficient to send out > larger zero pages as a part of single bvec. > > This concern was raised during the review of adding LBS support to > XFS[1][2]. > > Usually huge_zero_folio is allocated on demand, and it will be > deallocated by the shrinker if there are no users of it left. At moment, > huge_zero_folio infrastructure refcount is tied to the process lifetime > that created it. This might not work for bio layer as the completitions > can be async and the process that created the huge_zero_folio might no > longer be alive. Of course, what we could do is indicating that there is any untracked reference to the huge zero folio, and then simply refuse to free it for all eternity. Essentially, every any non-mm reference -> un-shrinkable. We'd still be allocating the huge zero folio dynamically. We could try allocating it on first usage either from memblock, or from the buddy if already around. Then, we'd only need a config option to allow for that to happen. -- Cheers, David / dhildenb
On 15.07.25 16:21, David Hildenbrand wrote:
> On 07.07.25 16:23, Pankaj Raghav (Samsung) wrote:
>> From: Pankaj Raghav <p.raghav@samsung.com>
>>
>> There are many places in the kernel where we need to zeroout larger
>> chunks but the maximum segment we can zeroout at a time by ZERO_PAGE
>> is limited by PAGE_SIZE.
>>
>> This is especially annoying in block devices and filesystems where we
>> attach multiple ZERO_PAGEs to the bio in different bvecs. With multipage
>> bvec support in block layer, it is much more efficient to send out
>> larger zero pages as a part of single bvec.
>>
>> This concern was raised during the review of adding LBS support to
>> XFS[1][2].
>>
>> Usually huge_zero_folio is allocated on demand, and it will be
>> deallocated by the shrinker if there are no users of it left. At moment,
>> huge_zero_folio infrastructure refcount is tied to the process lifetime
>> that created it. This might not work for bio layer as the completitions
>> can be async and the process that created the huge_zero_folio might no
>> longer be alive.
>
> Of course, what we could do is indicating that there is any untracked
> reference to the huge zero folio, and then simply refuse to free it for
> all eternity.
>
> Essentially, every any non-mm reference -> un-shrinkable.
>
> We'd still be allocating the huge zero folio dynamically. We could try
> allocating it on first usage either from memblock, or from the buddy if
> already around.
>
> Then, we'd only need a config option to allow for that to happen.
Something incomplete and very hacky just to give an idea. It would try allocating
it if there is actual code running that would need it, and then have it
stick around forever.
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index e0a27f80f390d..357e29e98d8d2 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -481,6 +481,7 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf);
extern struct folio *huge_zero_folio;
extern unsigned long huge_zero_pfn;
+atomic_t huge_zero_folio_is_static;
static inline bool is_huge_zero_folio(const struct folio *folio)
{
@@ -499,6 +500,16 @@ static inline bool is_huge_zero_pmd(pmd_t pmd)
struct folio *mm_get_huge_zero_folio(struct mm_struct *mm);
void mm_put_huge_zero_folio(struct mm_struct *mm);
+struct folio *__get_static_huge_zero_folio(void);
+
+static inline struct folio *get_static_huge_zero_folio(void)
+{
+ if (!IS_ENMABLED(CONFIG_STATIC_HUGE_ZERO_FOLIO))
+ return NULL;
+ if (likely(atomic_read(&huge_zero_folio_is_static)))
+ return huge_zero_folio;
+ return get_static_huge_zero_folio();
+}
static inline bool thp_migration_supported(void)
{
@@ -509,7 +520,6 @@ void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmd, bool freeze);
bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr,
pmd_t *pmdp, struct folio *folio);
-
#else /* CONFIG_TRANSPARENT_HUGEPAGE */
static inline bool folio_test_pmd_mappable(struct folio *folio)
@@ -690,6 +700,11 @@ static inline int change_huge_pud(struct mmu_gather *tlb,
{
return 0;
}
+
+static inline struct folio *static_huge_zero_folio(void)
+{
+ return NULL;
+}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
static inline int split_folio_to_list_to_order(struct folio *folio,
@@ -703,4 +718,14 @@ static inline int split_folio_to_order(struct folio *folio, int new_order)
return split_folio_to_list_to_order(folio, NULL, new_order);
}
+static inline struct folio *largest_zero_folio(void)
+{
+ struct folio *folio;
+
+ folio = get_static_huge_zero_folio();
+ if (folio)
+ return folio;
+ return page_folio(ZERO_PAGE(0));
+}
+
#endif /* _LINUX_HUGE_MM_H */
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 31b5c4e61a574..eb49c69f9c8e2 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -77,6 +77,7 @@ static bool split_underused_thp = true;
static atomic_t huge_zero_refcount;
struct folio *huge_zero_folio __read_mostly;
unsigned long huge_zero_pfn __read_mostly = ~0UL;
+atomic_t huge_zero_folio_is_static __read_mostly;
unsigned long huge_anon_orders_always __read_mostly;
unsigned long huge_anon_orders_madvise __read_mostly;
unsigned long huge_anon_orders_inherit __read_mostly;
@@ -266,6 +267,25 @@ void mm_put_huge_zero_folio(struct mm_struct *mm)
put_huge_zero_page();
}
+#ifdef CONFIG_STATIC_HUGE_ZERO_FOLIO
+struct folio *__get_static_huge_zero_folio(void)
+{
+ /*
+ * Our raised reference will prevent the shrinker from ever having
+ * success -> static.
+ */
+ if (atomic_read(&huge_zero_folio_is_static))
+ return huge_zero_folio;
+ /* TODO: memblock allocation if buddy is not up yet? Or Reject that earlier. */
+ if (!get_huge_zero_page())
+ return NULL;
+ if (atomic_cmpxchg(&huge_zero_folio_is_static, 0, 1) != 0)
+ put_huge_zero_page();
+ return huge_zero_folio;
+
+}
+#endif /* CONFIG_STATIC_HUGE_ZERO_FOLIO */
+
static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
struct shrink_control *sc)
{
--
Cheers,
David / dhildenb
> > Then, we'd only need a config option to allow for that to happen.
>
> Something incomplete and very hacky just to give an idea. It would try allocating
> it if there is actual code running that would need it, and then have it
> stick around forever.
>
Thanks a lot for this David :) I think this is a much better idea and
reduces the amount code and reuse the existing infrastructure.
I will try this approach in the next version.
<snip>
> + /*
> + * Our raised reference will prevent the shrinker from ever having
> + * success -> static.
> + */
> + if (atomic_read(&huge_zero_folio_is_static))
> + return huge_zero_folio;
> + /* TODO: memblock allocation if buddy is not up yet? Or Reject that earlier. */
Do we need memblock allocation? At least the use cases I forsee for
static pmd zero page are all after the mm is up. So I don't see why we
need to allocate it via memblock.
> + if (!get_huge_zero_page())
> + return NULL;
> + if (atomic_cmpxchg(&huge_zero_folio_is_static, 0, 1) != 0)
> + put_huge_zero_page();
> + return huge_zero_folio;
> +
> +}
> +#endif /* CONFIG_STATIC_HUGE_ZERO_FOLIO */
> +
> static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
> struct shrink_control *sc)
> {
>
--
Pankaj
On 17.07.25 12:34, Pankaj Raghav (Samsung) wrote: >>> Then, we'd only need a config option to allow for that to happen. >> >> Something incomplete and very hacky just to give an idea. It would try allocating >> it if there is actual code running that would need it, and then have it >> stick around forever. >> > Thanks a lot for this David :) I think this is a much better idea and > reduces the amount code and reuse the existing infrastructure. > > I will try this approach in the next version. > > <snip> >> + /* >> + * Our raised reference will prevent the shrinker from ever having >> + * success -> static. >> + */ >> + if (atomic_read(&huge_zero_folio_is_static)) >> + return huge_zero_folio; >> + /* TODO: memblock allocation if buddy is not up yet? Or Reject that earlier. */ > > Do we need memblock allocation? At least the use cases I forsee for > static pmd zero page are all after the mm is up. So I don't see why we > need to allocate it via memblock. Even better! We might want to detect whether allocation of the huge zeropage failed a couple of times and then just give up. Otherwise, each and every user of the largest zero folio will keep allocating it. -- Cheers, David / dhildenb
On Thu, Jul 17, 2025 at 01:46:03PM +0200, David Hildenbrand wrote: > On 17.07.25 12:34, Pankaj Raghav (Samsung) wrote: > > > > Then, we'd only need a config option to allow for that to happen. > > > > > > Something incomplete and very hacky just to give an idea. It would try allocating > > > it if there is actual code running that would need it, and then have it > > > stick around forever. > > > > > Thanks a lot for this David :) I think this is a much better idea and > > reduces the amount code and reuse the existing infrastructure. > > > > I will try this approach in the next version. > > > > <snip> > > > + /* > > > + * Our raised reference will prevent the shrinker from ever having > > > + * success -> static. > > > + */ > > > + if (atomic_read(&huge_zero_folio_is_static)) > > > + return huge_zero_folio; > > > + /* TODO: memblock allocation if buddy is not up yet? Or Reject that earlier. */ > > > > Do we need memblock allocation? At least the use cases I forsee for > > static pmd zero page are all after the mm is up. So I don't see why we > > need to allocate it via memblock. > > Even better! > > We might want to detect whether allocation of the huge zeropage failed a > couple of times and then just give up. Otherwise, each and every user of the > largest zero folio will keep allocating it. Yes, that makes sense. We need sort of like a global counter to count the nr of failures and then give up trying to allocate it if it goes above a threshold. -- Pankaj
© 2016 - 2025 Red Hat, Inc.