This patch makes __vmalloc_area_node() to correctly handle non-blocking
allocation requests, such as GFP_ATOMIC and GFP_NOWAIT. Main changes:
- Add a __GFP_HIGHMEM to gfp_mask only for blocking requests
if there are no DMA constraints.
- vmap_page_range() is wrapped by memalloc_noreclaim_save/restore()
to avoid memory reclaim related operations that could sleep during
page table setup or mapping pages.
This is particularly important for page table allocations that
internally use GFP_PGTABLE_KERNEL, which may sleep unless such
scope restrictions are applied. For example:
<snip>
__pte_alloc_kernel()
pte_alloc_one_kernel(&init_mm);
pagetable_alloc_noprof(GFP_PGTABLE_KERNEL & ~__GFP_HIGHMEM, 0);
<snip>
Note: in most cases, PTE entries are established only up to the level
required by current vmap space usage, meaning the page tables are typically
fully populated during the mapping process.
Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com>
---
mm/vmalloc.c | 20 ++++++++++++++++----
1 file changed, 16 insertions(+), 4 deletions(-)
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 2424f80d524a..8a7eab810561 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -3721,12 +3721,20 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
unsigned int nr_small_pages = size >> PAGE_SHIFT;
unsigned int page_order;
unsigned int flags;
+ bool noblock;
int ret;
array_size = (unsigned long)nr_small_pages * sizeof(struct page *);
+ noblock = !gfpflags_allow_blocking(gfp_mask);
- if (!(gfp_mask & (GFP_DMA | GFP_DMA32)))
- gfp_mask |= __GFP_HIGHMEM;
+ if (noblock) {
+ /* __GFP_NOFAIL and "noblock" flags are mutually exclusive. */
+ nofail = false;
+ } else {
+ /* Allow highmem allocations if there are no DMA constraints. */
+ if (!(gfp_mask & (GFP_DMA | GFP_DMA32)))
+ gfp_mask |= __GFP_HIGHMEM;
+ }
/* Please note that the recursion is strictly bounded. */
if (array_size > PAGE_SIZE) {
@@ -3790,7 +3798,9 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
* page tables allocations ignore external gfp mask, enforce it
* by the scope API
*/
- if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
+ if (noblock)
+ flags = memalloc_noreclaim_save();
+ else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
flags = memalloc_nofs_save();
else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0)
flags = memalloc_noio_save();
@@ -3802,7 +3812,9 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
schedule_timeout_uninterruptible(1);
} while (nofail && (ret < 0));
- if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
+ if (noblock)
+ memalloc_noreclaim_restore(flags);
+ else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
memalloc_nofs_restore(flags);
else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0)
memalloc_noio_restore(flags);
--
2.39.5On 08/07/25 at 09:58am, Uladzislau Rezki (Sony) wrote:
> This patch makes __vmalloc_area_node() to correctly handle non-blocking
> allocation requests, such as GFP_ATOMIC and GFP_NOWAIT. Main changes:
>
> - Add a __GFP_HIGHMEM to gfp_mask only for blocking requests
> if there are no DMA constraints.
>
> - vmap_page_range() is wrapped by memalloc_noreclaim_save/restore()
> to avoid memory reclaim related operations that could sleep during
> page table setup or mapping pages.
>
> This is particularly important for page table allocations that
> internally use GFP_PGTABLE_KERNEL, which may sleep unless such
> scope restrictions are applied. For example:
>
> <snip>
> __pte_alloc_kernel()
> pte_alloc_one_kernel(&init_mm);
> pagetable_alloc_noprof(GFP_PGTABLE_KERNEL & ~__GFP_HIGHMEM, 0);
> <snip>
>
> Note: in most cases, PTE entries are established only up to the level
> required by current vmap space usage, meaning the page tables are typically
> fully populated during the mapping process.
>
> Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com>
> ---
> mm/vmalloc.c | 20 ++++++++++++++++----
> 1 file changed, 16 insertions(+), 4 deletions(-)
>
> diff --git a/mm/vmalloc.c b/mm/vmalloc.c
> index 2424f80d524a..8a7eab810561 100644
> --- a/mm/vmalloc.c
> +++ b/mm/vmalloc.c
> @@ -3721,12 +3721,20 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
> unsigned int nr_small_pages = size >> PAGE_SHIFT;
> unsigned int page_order;
> unsigned int flags;
> + bool noblock;
> int ret;
>
> array_size = (unsigned long)nr_small_pages * sizeof(struct page *);
> + noblock = !gfpflags_allow_blocking(gfp_mask);
>
> - if (!(gfp_mask & (GFP_DMA | GFP_DMA32)))
> - gfp_mask |= __GFP_HIGHMEM;
> + if (noblock) {
> + /* __GFP_NOFAIL and "noblock" flags are mutually exclusive. */
> + nofail = false;
> + } else {
> + /* Allow highmem allocations if there are no DMA constraints. */
> + if (!(gfp_mask & (GFP_DMA | GFP_DMA32)))
> + gfp_mask |= __GFP_HIGHMEM;
> + }
>
> /* Please note that the recursion is strictly bounded. */
> if (array_size > PAGE_SIZE) {
> @@ -3790,7 +3798,9 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
> * page tables allocations ignore external gfp mask, enforce it
> * by the scope API
> */
> - if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> + if (noblock)
> + flags = memalloc_noreclaim_save();
> + else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> flags = memalloc_nofs_save();
> else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0)
> flags = memalloc_noio_save();
> @@ -3802,7 +3812,9 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
> schedule_timeout_uninterruptible(1);
> } while (nofail && (ret < 0));
>
> - if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> + if (noblock)
> + memalloc_noreclaim_restore(flags);
> + else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> memalloc_nofs_restore(flags);
> else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0)
> memalloc_noio_restore(flags);
Can we use memalloc_flags_restore(flags) directly to replace above if
else checking? It can reduce LOC, might be not as readable as the change
in patch surely. Not strong opinion.
memalloc_flags_restore(flags);
On Mon, Aug 18, 2025 at 12:35:16PM +0800, Baoquan He wrote:
> On 08/07/25 at 09:58am, Uladzislau Rezki (Sony) wrote:
> > This patch makes __vmalloc_area_node() to correctly handle non-blocking
> > allocation requests, such as GFP_ATOMIC and GFP_NOWAIT. Main changes:
> >
> > - Add a __GFP_HIGHMEM to gfp_mask only for blocking requests
> > if there are no DMA constraints.
> >
> > - vmap_page_range() is wrapped by memalloc_noreclaim_save/restore()
> > to avoid memory reclaim related operations that could sleep during
> > page table setup or mapping pages.
> >
> > This is particularly important for page table allocations that
> > internally use GFP_PGTABLE_KERNEL, which may sleep unless such
> > scope restrictions are applied. For example:
> >
> > <snip>
> > __pte_alloc_kernel()
> > pte_alloc_one_kernel(&init_mm);
> > pagetable_alloc_noprof(GFP_PGTABLE_KERNEL & ~__GFP_HIGHMEM, 0);
> > <snip>
> >
> > Note: in most cases, PTE entries are established only up to the level
> > required by current vmap space usage, meaning the page tables are typically
> > fully populated during the mapping process.
> >
> > Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com>
> > ---
> > mm/vmalloc.c | 20 ++++++++++++++++----
> > 1 file changed, 16 insertions(+), 4 deletions(-)
> >
> > diff --git a/mm/vmalloc.c b/mm/vmalloc.c
> > index 2424f80d524a..8a7eab810561 100644
> > --- a/mm/vmalloc.c
> > +++ b/mm/vmalloc.c
> > @@ -3721,12 +3721,20 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
> > unsigned int nr_small_pages = size >> PAGE_SHIFT;
> > unsigned int page_order;
> > unsigned int flags;
> > + bool noblock;
> > int ret;
> >
> > array_size = (unsigned long)nr_small_pages * sizeof(struct page *);
> > + noblock = !gfpflags_allow_blocking(gfp_mask);
> >
> > - if (!(gfp_mask & (GFP_DMA | GFP_DMA32)))
> > - gfp_mask |= __GFP_HIGHMEM;
> > + if (noblock) {
> > + /* __GFP_NOFAIL and "noblock" flags are mutually exclusive. */
> > + nofail = false;
> > + } else {
> > + /* Allow highmem allocations if there are no DMA constraints. */
> > + if (!(gfp_mask & (GFP_DMA | GFP_DMA32)))
> > + gfp_mask |= __GFP_HIGHMEM;
> > + }
> >
> > /* Please note that the recursion is strictly bounded. */
> > if (array_size > PAGE_SIZE) {
> > @@ -3790,7 +3798,9 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
> > * page tables allocations ignore external gfp mask, enforce it
> > * by the scope API
> > */
> > - if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> > + if (noblock)
> > + flags = memalloc_noreclaim_save();
> > + else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> > flags = memalloc_nofs_save();
> > else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0)
> > flags = memalloc_noio_save();
> > @@ -3802,7 +3812,9 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
> > schedule_timeout_uninterruptible(1);
> > } while (nofail && (ret < 0));
> >
> > - if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> > + if (noblock)
> > + memalloc_noreclaim_restore(flags);
> > + else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> > memalloc_nofs_restore(flags);
> > else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0)
> > memalloc_noio_restore(flags);
>
> Can we use memalloc_flags_restore(flags) directly to replace above if
> else checking? It can reduce LOC, might be not as readable as the change
> in patch surely. Not strong opinion.
>
> memalloc_flags_restore(flags);
>
I agree, those if/else cases looks ugly. Maybe adding two save/restore
functions are worth doing specifically for vmalloc part.
--
Uladzislau Rezki
On 08/18/25 at 03:08pm, Uladzislau Rezki wrote:
> On Mon, Aug 18, 2025 at 12:35:16PM +0800, Baoquan He wrote:
> > On 08/07/25 at 09:58am, Uladzislau Rezki (Sony) wrote:
> > > This patch makes __vmalloc_area_node() to correctly handle non-blocking
> > > allocation requests, such as GFP_ATOMIC and GFP_NOWAIT. Main changes:
> > >
> > > - Add a __GFP_HIGHMEM to gfp_mask only for blocking requests
> > > if there are no DMA constraints.
> > >
> > > - vmap_page_range() is wrapped by memalloc_noreclaim_save/restore()
> > > to avoid memory reclaim related operations that could sleep during
> > > page table setup or mapping pages.
> > >
> > > This is particularly important for page table allocations that
> > > internally use GFP_PGTABLE_KERNEL, which may sleep unless such
> > > scope restrictions are applied. For example:
> > >
> > > <snip>
> > > __pte_alloc_kernel()
> > > pte_alloc_one_kernel(&init_mm);
> > > pagetable_alloc_noprof(GFP_PGTABLE_KERNEL & ~__GFP_HIGHMEM, 0);
> > > <snip>
> > >
> > > Note: in most cases, PTE entries are established only up to the level
> > > required by current vmap space usage, meaning the page tables are typically
> > > fully populated during the mapping process.
> > >
> > > Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com>
> > > ---
> > > mm/vmalloc.c | 20 ++++++++++++++++----
> > > 1 file changed, 16 insertions(+), 4 deletions(-)
> > >
> > > diff --git a/mm/vmalloc.c b/mm/vmalloc.c
> > > index 2424f80d524a..8a7eab810561 100644
> > > --- a/mm/vmalloc.c
> > > +++ b/mm/vmalloc.c
> > > @@ -3721,12 +3721,20 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
> > > unsigned int nr_small_pages = size >> PAGE_SHIFT;
> > > unsigned int page_order;
> > > unsigned int flags;
> > > + bool noblock;
> > > int ret;
> > >
> > > array_size = (unsigned long)nr_small_pages * sizeof(struct page *);
> > > + noblock = !gfpflags_allow_blocking(gfp_mask);
> > >
> > > - if (!(gfp_mask & (GFP_DMA | GFP_DMA32)))
> > > - gfp_mask |= __GFP_HIGHMEM;
> > > + if (noblock) {
> > > + /* __GFP_NOFAIL and "noblock" flags are mutually exclusive. */
> > > + nofail = false;
> > > + } else {
> > > + /* Allow highmem allocations if there are no DMA constraints. */
> > > + if (!(gfp_mask & (GFP_DMA | GFP_DMA32)))
> > > + gfp_mask |= __GFP_HIGHMEM;
> > > + }
> > >
> > > /* Please note that the recursion is strictly bounded. */
> > > if (array_size > PAGE_SIZE) {
> > > @@ -3790,7 +3798,9 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
> > > * page tables allocations ignore external gfp mask, enforce it
> > > * by the scope API
> > > */
> > > - if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> > > + if (noblock)
> > > + flags = memalloc_noreclaim_save();
> > > + else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> > > flags = memalloc_nofs_save();
> > > else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0)
> > > flags = memalloc_noio_save();
> > > @@ -3802,7 +3812,9 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
> > > schedule_timeout_uninterruptible(1);
> > > } while (nofail && (ret < 0));
> > >
> > > - if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> > > + if (noblock)
> > > + memalloc_noreclaim_restore(flags);
> > > + else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> > > memalloc_nofs_restore(flags);
> > > else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0)
> > > memalloc_noio_restore(flags);
> >
> > Can we use memalloc_flags_restore(flags) directly to replace above if
> > else checking? It can reduce LOC, might be not as readable as the change
> > in patch surely. Not strong opinion.
> >
> > memalloc_flags_restore(flags);
> >
> I agree, those if/else cases looks ugly. Maybe adding two save/restore
> functions are worth doing specifically for vmalloc part.
Yeah, that is also great idea.
On Thu 07-08-25 09:58:09, Uladzislau Rezki wrote:
> This patch makes __vmalloc_area_node() to correctly handle non-blocking
> allocation requests, such as GFP_ATOMIC and GFP_NOWAIT. Main changes:
>
> - Add a __GFP_HIGHMEM to gfp_mask only for blocking requests
> if there are no DMA constraints.
This begs for a more explanation. Why does __GFP_HIGHMEM matters? I
suspect this is due to kmapping of those pages but that could be done in
an atomic way. But in practice I do not think we really care about
highmem all that much for vmalloc. The vmalloc space is really tiny for
32b systems where highmem matters and failing vmalloc allocations due to
lack is of __GFP_HIGHMEM is hard to consider important if relevant at
all.
> - vmap_page_range() is wrapped by memalloc_noreclaim_save/restore()
> to avoid memory reclaim related operations that could sleep during
> page table setup or mapping pages.
>
> This is particularly important for page table allocations that
> internally use GFP_PGTABLE_KERNEL, which may sleep unless such
> scope restrictions are applied. For example:
>
> <snip>
> __pte_alloc_kernel()
> pte_alloc_one_kernel(&init_mm);
> pagetable_alloc_noprof(GFP_PGTABLE_KERNEL & ~__GFP_HIGHMEM, 0);
> <snip>
As I've said in several occations, I am not entirely happy about this
approach because it doesn't really guarantee atomicty. If any
architecture decides to use some sleeping locking down that path then
the whole thing just blows up. On the other hand this is mostly a
theoretical concern at this stage and this is a feature people have
been asking for a long time (especially from kvmalloc side) so better
good than perfect that his.
That being said, you are missing __kvmalloc_node_noprof,
__vmalloc_node_range_noprof (and maybe some more places) documentation
update.
> Note: in most cases, PTE entries are established only up to the level
> required by current vmap space usage, meaning the page tables are typically
> fully populated during the mapping process.
>
> Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com>
With the doc part fixed
Acked-by: Michal Hocko <mhocko@suse.com>
Thanks!
> ---
> mm/vmalloc.c | 20 ++++++++++++++++----
> 1 file changed, 16 insertions(+), 4 deletions(-)
>
> diff --git a/mm/vmalloc.c b/mm/vmalloc.c
> index 2424f80d524a..8a7eab810561 100644
> --- a/mm/vmalloc.c
> +++ b/mm/vmalloc.c
> @@ -3721,12 +3721,20 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
> unsigned int nr_small_pages = size >> PAGE_SHIFT;
> unsigned int page_order;
> unsigned int flags;
> + bool noblock;
> int ret;
>
> array_size = (unsigned long)nr_small_pages * sizeof(struct page *);
> + noblock = !gfpflags_allow_blocking(gfp_mask);
>
> - if (!(gfp_mask & (GFP_DMA | GFP_DMA32)))
> - gfp_mask |= __GFP_HIGHMEM;
> + if (noblock) {
> + /* __GFP_NOFAIL and "noblock" flags are mutually exclusive. */
> + nofail = false;
> + } else {
> + /* Allow highmem allocations if there are no DMA constraints. */
> + if (!(gfp_mask & (GFP_DMA | GFP_DMA32)))
> + gfp_mask |= __GFP_HIGHMEM;
> + }
>
> /* Please note that the recursion is strictly bounded. */
> if (array_size > PAGE_SIZE) {
> @@ -3790,7 +3798,9 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
> * page tables allocations ignore external gfp mask, enforce it
> * by the scope API
> */
> - if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> + if (noblock)
> + flags = memalloc_noreclaim_save();
> + else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> flags = memalloc_nofs_save();
> else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0)
> flags = memalloc_noio_save();
> @@ -3802,7 +3812,9 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
> schedule_timeout_uninterruptible(1);
> } while (nofail && (ret < 0));
>
> - if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> + if (noblock)
> + memalloc_noreclaim_restore(flags);
> + else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO)
> memalloc_nofs_restore(flags);
> else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0)
> memalloc_noio_restore(flags);
> --
> 2.39.5
>
--
Michal Hocko
SUSE Labs
On Thu, Aug 07, 2025 at 01:54:21PM +0200, Michal Hocko wrote: > On Thu 07-08-25 09:58:09, Uladzislau Rezki wrote: > > This patch makes __vmalloc_area_node() to correctly handle non-blocking > > allocation requests, such as GFP_ATOMIC and GFP_NOWAIT. Main changes: > > > > - Add a __GFP_HIGHMEM to gfp_mask only for blocking requests > > if there are no DMA constraints. > > This begs for a more explanation. Why does __GFP_HIGHMEM matters? I > suspect this is due to kmapping of those pages but that could be done in > an atomic way. But in practice I do not think we really care about > highmem all that much for vmalloc. The vmalloc space is really tiny for > 32b systems where highmem matters and failing vmalloc allocations due to > lack is of __GFP_HIGHMEM is hard to consider important if relevant at > all. > Thank you for this note. Yes, __GFP_HIGHMEM is about 32 bit systems. Initially, in the RFC series, during testing i saw some incompatibility kernel splats when use together with non-blocking flags. Whereas i do not see it anymore and now. It looks like i messed up something when testing pre-RFC series. I will not touch it. I mean i will keep it as it used to be, i.e. apply it if no (GFP_DMA | GFP_DMA32). > > - vmap_page_range() is wrapped by memalloc_noreclaim_save/restore() > > to avoid memory reclaim related operations that could sleep during > > page table setup or mapping pages. > > > > This is particularly important for page table allocations that > > internally use GFP_PGTABLE_KERNEL, which may sleep unless such > > scope restrictions are applied. For example: > > > > <snip> > > __pte_alloc_kernel() > > pte_alloc_one_kernel(&init_mm); > > pagetable_alloc_noprof(GFP_PGTABLE_KERNEL & ~__GFP_HIGHMEM, 0); > > <snip> > > As I've said in several occations, I am not entirely happy about this > approach because it doesn't really guarantee atomicty. If any > architecture decides to use some sleeping locking down that path then > the whole thing just blows up. On the other hand this is mostly a > theoretical concern at this stage and this is a feature people have > been asking for a long time (especially from kvmalloc side) so better > good than perfect that his. > I agree with it. Unfortunately i can not control the PTE kernel allocation layer. > > That being said, you are missing __kvmalloc_node_noprof, > __vmalloc_node_range_noprof (and maybe some more places) documentation > update. > I would like to fix documentation in separate patch. That was deliberately. > > Note: in most cases, PTE entries are established only up to the level > > required by current vmap space usage, meaning the page tables are typically > > fully populated during the mapping process. > > > > Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com> > > With the doc part fixed > Acked-by: Michal Hocko <mhocko@suse.com> > Thanks! Thanks! Applied. -- Uladzislau Rezki
© 2016 - 2025 Red Hat, Inc.