Introduce LARGE_ZERO_PAGE of size 2M as an alternative to ZERO_PAGE of
size PAGE_SIZE.

There are many places in the kernel where we need to zeroout larger
chunks but the maximum segment we can zeroout at a time is limited by
PAGE_SIZE.

This is especially annoying in block devices and filesystems where we
attach multiple ZERO_PAGEs to the bio in different bvecs. With multipage
bvec support in block layer, it is much more efficient to send out
larger zero pages as a part of single bvec.

While there are other options such as huge_zero_page, they can fail
based on the system memory pressure requiring a fallback to ZERO_PAGE[3].

This idea (but not the implementation) was suggested during the review of
adding LBS support to XFS[1][2].

LARGE_ZERO_PAGE is added behind a config option so that systems that are
constrained by memory are not forced to use it.

[1] https://lore.kernel.org/linux-xfs/20231027051847.GA7885@lst.de/
[2] https://lore.kernel.org/linux-xfs/ZitIK5OnR7ZNY0IG@infradead.org/
[3] https://lore.kernel.org/linux-xfs/3pqmgrlewo6ctcwakdvbvjqixac5en6irlipe5aiz6vkylfyni@2luhrs36ke5r/

Suggested-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Pankaj Raghav <p.raghav@samsung.com>
---
 arch/Kconfig                   |  8 ++++++++
 arch/x86/include/asm/pgtable.h | 20 +++++++++++++++++++-
 arch/x86/kernel/head_64.S      |  9 ++++++++-
 3 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/arch/Kconfig b/arch/Kconfig
index b0adb665041f..aefa519cb211 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -218,6 +218,14 @@ config USER_RETURN_NOTIFIER
 	  Provide a kernel-internal notification when a cpu is about to
 	  switch to user mode.
 
+config LARGE_ZERO_PAGE
+	bool "Large zero pages"
+	def_bool n
+	help
+	  2M sized zero pages for zeroing. This will reserve 2M sized
+	  physical pages for zeroing. Not suitable for memory constrained
+	  systems.
+
 config HAVE_IOREMAP_PROT
 	bool
 
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index 3f59d7a16010..78eb83f2da34 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -17,6 +17,7 @@
 
 #ifndef __ASSEMBLER__
 #include <linux/spinlock.h>
+#include <linux/sizes.h>
 #include <asm/x86_init.h>
 #include <asm/pkru.h>
 #include <asm/fpu/api.h>
@@ -47,14 +48,31 @@ void ptdump_walk_user_pgd_level_checkwx(void);
 #define debug_checkwx_user()	do { } while (0)
 #endif
 
+#ifdef CONFIG_LARGE_ZERO_PAGE
+/*
+ * LARGE_ZERO_PAGE is a global shared page that is always zero: used
+ * for zero-mapped memory areas etc..
+ */
+extern unsigned long empty_large_zero_page[(SZ_2M) / sizeof(unsigned long)]
+	__visible;
+#define ZERO_LARGE_PAGE(vaddr) ((void)(vaddr),virt_to_page(empty_large_zero_page))
+
+#define ZERO_PAGE(vaddr) ZERO_LARGE_PAGE(vaddr)
+#define ZERO_LARGE_PAGE_SIZE SZ_2M
+#else
 /*
  * ZERO_PAGE is a global shared page that is always zero: used
  * for zero-mapped memory areas etc..
  */
-extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
+extern unsigned long empty_zero_page[(PAGE_SIZE) / sizeof(unsigned long)]
 	__visible;
 #define ZERO_PAGE(vaddr) ((void)(vaddr),virt_to_page(empty_zero_page))
 
+#define ZERO_LARGE_PAGE(vaddr) ZERO_PAGE(vaddr)
+
+#define ZERO_LARGE_PAGE_SIZE PAGE_SIZE
+#endif
+
 extern spinlock_t pgd_lock;
 extern struct list_head pgd_list;
 
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index fefe2a25cf02..ebcd12f72966 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -14,6 +14,7 @@
 #include <linux/threads.h>
 #include <linux/init.h>
 #include <linux/pgtable.h>
+#include <linux/sizes.h>
 #include <asm/segment.h>
 #include <asm/page.h>
 #include <asm/msr.h>
@@ -708,8 +709,14 @@ EXPORT_SYMBOL(phys_base)
 #include "../xen/xen-head.S"
 
 	__PAGE_ALIGNED_BSS
+#ifdef CONFIG_LARGE_ZERO_PAGE
+SYM_DATA_START_PAGE_ALIGNED(empty_large_zero_page)
+	.skip SZ_2M
+SYM_DATA_END(empty_large_zero_page)
+EXPORT_SYMBOL(empty_large_zero_page)
+#else
 SYM_DATA_START_PAGE_ALIGNED(empty_zero_page)
 	.skip PAGE_SIZE
 SYM_DATA_END(empty_zero_page)
 EXPORT_SYMBOL(empty_zero_page)
-
+#endif
-- 
2.47.2

On 16.05.25 12:10, Pankaj Raghav wrote:
> Introduce LARGE_ZERO_PAGE of size 2M as an alternative to ZERO_PAGE of
> size PAGE_SIZE.
> 
> There are many places in the kernel where we need to zeroout larger
> chunks but the maximum segment we can zeroout at a time is limited by
> PAGE_SIZE.
> 
> This is especially annoying in block devices and filesystems where we
> attach multiple ZERO_PAGEs to the bio in different bvecs. With multipage
> bvec support in block layer, it is much more efficient to send out
> larger zero pages as a part of single bvec.
> 
> While there are other options such as huge_zero_page, they can fail
> based on the system memory pressure requiring a fallback to ZERO_PAGE[3].

Instead of adding another one, why not have a config option that will 
always allocate the huge zeropage, and never free it?

I mean, the whole thing about dynamically allocating/freeing it was for 
memory-constrained systems. For large systems, we just don't care.

-- 
Cheers,

David / dhildenb

On Fri, May 16, 2025 at 02:21:04PM +0200, David Hildenbrand wrote:
> On 16.05.25 12:10, Pankaj Raghav wrote:
> > Introduce LARGE_ZERO_PAGE of size 2M as an alternative to ZERO_PAGE of
> > size PAGE_SIZE.
> > 
> > There are many places in the kernel where we need to zeroout larger
> > chunks but the maximum segment we can zeroout at a time is limited by
> > PAGE_SIZE.
> > 
> > This is especially annoying in block devices and filesystems where we
> > attach multiple ZERO_PAGEs to the bio in different bvecs. With multipage
> > bvec support in block layer, it is much more efficient to send out
> > larger zero pages as a part of single bvec.
> > 
> > While there are other options such as huge_zero_page, they can fail
> > based on the system memory pressure requiring a fallback to ZERO_PAGE[3].
> 
> Instead of adding another one, why not have a config option that will always
> allocate the huge zeropage, and never free it?
> 
> I mean, the whole thing about dynamically allocating/freeing it was for
> memory-constrained systems. For large systems, we just don't care.

That sounds like a good idea. I was just worried about wasting too much
memory with a huge page in systems with 64k page size. But it can always be
disabled by putting it behind a config.

Thanks, David. I will wait to see what others think but what you
suggested sounds like a good idea on how to proceed.

-- 
Pankaj Raghav

On 16.05.25 15:03, Pankaj Raghav (Samsung) wrote:
> On Fri, May 16, 2025 at 02:21:04PM +0200, David Hildenbrand wrote:
>> On 16.05.25 12:10, Pankaj Raghav wrote:
>>> Introduce LARGE_ZERO_PAGE of size 2M as an alternative to ZERO_PAGE of
>>> size PAGE_SIZE.
>>>
>>> There are many places in the kernel where we need to zeroout larger
>>> chunks but the maximum segment we can zeroout at a time is limited by
>>> PAGE_SIZE.
>>>
>>> This is especially annoying in block devices and filesystems where we
>>> attach multiple ZERO_PAGEs to the bio in different bvecs. With multipage
>>> bvec support in block layer, it is much more efficient to send out
>>> larger zero pages as a part of single bvec.
>>>
>>> While there are other options such as huge_zero_page, they can fail
>>> based on the system memory pressure requiring a fallback to ZERO_PAGE[3].
>>
>> Instead of adding another one, why not have a config option that will always
>> allocate the huge zeropage, and never free it?
>>
>> I mean, the whole thing about dynamically allocating/freeing it was for
>> memory-constrained systems. For large systems, we just don't care.
> 
> That sounds like a good idea. I was just worried about wasting too much
> memory with a huge page in systems with 64k page size. But it can always be
> disabled by putting it behind a config.

Exactly. If the huge zero page is larger than 2M, we probably don't want 
it in any case.

On arm64k it could be 512 of MiBs. Full of zeroes.

I'm wondering why nobody ever complained about that before, and I don't 
see anything immediate that would disable the huge zero page in such 
environments. Well, we can just leave that as it is.

In any case, the idea would be to have a Kconfig where we statically 
allocate the huge zero page and disable all the refcounting / shrinking.

Then, we can make this Kconfig specific to sane environments (e.g., 4 
KiB page size).

 From other MM code, we can then simply reuse that single huge zero page.

> 
> Thanks, David. I will wait to see what others think but what you
> suggested sounds like a good idea on how to proceed.

In particular, it wouldn't be arch specific, and we wouldn't waste on 
x86 2x 2MB for storing zeroes ...

-- 
Cheers,

David / dhildenb