mm/folio.c | 24 +++++++++++++++++++----- mm/internal.h | 1 + mm/vmscan.c | 3 ++- 3 files changed, 22 insertions(+), 6 deletions(-)
move_folios_to_lru(), folio_batch_move_lru() and folios_put_refs() call
folio_unqueue_deferred_split() on folios where the refcount has dropped
to zero. In these paths, folio_unqueue_deferred_split() is called with
the lruvec lock often held (move_folios_to_lru() always holds it and
folio_batch_move_lru()/folios_put_refs() usually hold it for every folio
in the batch except the first). However, it's not needed to hold the
lruvec lock.
The lruvec lock protects folio->lru and the reclaim LRU lists whereas
the deferred split queue lives on folio->_deferred_list in
deferred_split_lru. The deferred_split_lru is a list_lru with its own
per-memcg per-node lock.
There are already other existing callers that call
folio_unqueue_deferred_split() without holding the lruvec lock (eg
__folio_put(), shrink_folio_list(), __folio_migrate_mapping(), ...).
Add folio_batch_unqueue_deferred_split() and use it in
move_folios_to_lru(), folio_batch_move_lru(), and folios_put_refs() to
unqueue each batch of zero-refcount folios from the deferred split list
after the lruvec lock is released and before
mem_cgroup_uncharge_folios(). This helps shorten the lruvec lock's
critical sections, which run with IRQs disabled and are highly
contended.
Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
---
mm/folio.c | 24 +++++++++++++++++++-----
mm/internal.h | 1 +
mm/vmscan.c | 3 ++-
3 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/mm/folio.c b/mm/folio.c
index 35e242b48870..85144078e635 100644
--- a/mm/folio.c
+++ b/mm/folio.c
@@ -109,6 +109,19 @@ EXPORT_SYMBOL(__folio_put);
typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio);
+/*
+ * Unqueue a batch of folios from the deferred split queue. This must be
+ * called after the folios' refcounts have dropped to zero and before they
+ * are uncharged (see comment above __folio_unqueue_deferred_split() for why).
+ */
+void folio_batch_unqueue_deferred_split(struct folio_batch *fbatch)
+{
+ unsigned int i;
+
+ for (i = 0; i < folio_batch_count(fbatch); i++)
+ folio_unqueue_deferred_split(fbatch->folios[i]);
+}
+
static void lru_add(struct lruvec *lruvec, struct folio *folio)
{
int was_unevictable = folio_test_clear_unevictable(folio);
@@ -177,14 +190,10 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
*
* We're bypassing normal cleanup. Clear flags that are not
* applicable to dead folios.
- *
- * Since the folio may be part of a huge page, unqueue from
- * deferred split list to avoid a dangling list entry.
*/
if (is_lru_add && folio_ref_freeze(folio, 1)) {
__folio_clear_active(folio);
__folio_clear_unevictable(folio);
- folio_unqueue_deferred_split(folio);
fbatch->folios[i] = NULL;
folio_batch_add(&free_fbatch, folio);
continue;
@@ -201,6 +210,11 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
/* Cleanup filtered dead folios. */
if (is_lru_add) {
+ /*
+ * Since the folios may be part of a huge page, unqueue them
+ * from the deferred split list to avoid a dangling list entry.
+ */
+ folio_batch_unqueue_deferred_split(&free_fbatch);
mem_cgroup_uncharge_folios(&free_fbatch);
free_unref_folios(&free_fbatch);
}
@@ -1020,7 +1034,6 @@ void folios_put_refs(struct folio_batch *folios, unsigned int *refs)
free_huge_folio(folio);
continue;
}
- folio_unqueue_deferred_split(folio);
__page_cache_release(folio, &lruvec, &flags);
if (j != i)
@@ -1035,6 +1048,7 @@ void folios_put_refs(struct folio_batch *folios, unsigned int *refs)
}
folios->nr = j;
+ folio_batch_unqueue_deferred_split(folios);
mem_cgroup_uncharge_folios(folios);
free_unref_folios(folios);
}
diff --git a/mm/internal.h b/mm/internal.h
index 0ca863f2639a..ac72577a6e3e 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -760,6 +760,7 @@ static inline bool folio_unqueue_deferred_split(struct folio *folio)
return __folio_unqueue_deferred_split(folio);
}
+void folio_batch_unqueue_deferred_split(struct folio_batch *fbatch);
static inline struct folio *page_rmappable_folio(struct page *page)
{
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 29b31446fc11..94859a830fb0 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2007,9 +2007,9 @@ static unsigned int move_folios_to_lru(struct list_head *list, bool do_rotate)
if (unlikely(folio_put_testzero(folio))) {
__folio_clear_lru_flags(folio);
- folio_unqueue_deferred_split(folio);
if (folio_batch_add(&free_folios, folio) == 0) {
lruvec_unlock_irq(lruvec);
+ folio_batch_unqueue_deferred_split(&free_folios);
mem_cgroup_uncharge_folios(&free_folios);
free_unref_folios(&free_folios);
lruvec = NULL;
@@ -2041,6 +2041,7 @@ static unsigned int move_folios_to_lru(struct list_head *list, bool do_rotate)
lruvec_unlock_irq(lruvec);
if (free_folios.nr) {
+ folio_batch_unqueue_deferred_split(&free_folios);
mem_cgroup_uncharge_folios(&free_folios);
free_unref_folios(&free_folios);
}
--
2.52.0On 10/14/26 00:13, Joanne Koong wrote:
> move_folios_to_lru(), folio_batch_move_lru() and folios_put_refs() call
> folio_unqueue_deferred_split() on folios where the refcount has dropped
> to zero. In these paths, folio_unqueue_deferred_split() is called with
> the lruvec lock often held (move_folios_to_lru() always holds it and
> folio_batch_move_lru()/folios_put_refs() usually hold it for every folio
> in the batch except the first). However, it's not needed to hold the
> lruvec lock.
>
> The lruvec lock protects folio->lru and the reclaim LRU lists whereas
> the deferred split queue lives on folio->_deferred_list in
> deferred_split_lru. The deferred_split_lru is a list_lru with its own
> per-memcg per-node lock.
>
> There are already other existing callers that call
> folio_unqueue_deferred_split() without holding the lruvec lock (eg
> __folio_put(), shrink_folio_list(), __folio_migrate_mapping(), ...).
>
> Add folio_batch_unqueue_deferred_split() and use it in
> move_folios_to_lru(), folio_batch_move_lru(), and folios_put_refs() to
> unqueue each batch of zero-refcount folios from the deferred split list
> after the lruvec lock is released and before
> mem_cgroup_uncharge_folios(). This helps shorten the lruvec lock's
> critical sections, which run with IRQs disabled and are highly
> contended.
>
> Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
> ---
> mm/folio.c | 24 +++++++++++++++++++-----
> mm/internal.h | 1 +
> mm/vmscan.c | 3 ++-
> 3 files changed, 22 insertions(+), 6 deletions(-)
>
> diff --git a/mm/folio.c b/mm/folio.c
> index 35e242b48870..85144078e635 100644
> --- a/mm/folio.c
> +++ b/mm/folio.c
> @@ -109,6 +109,19 @@ EXPORT_SYMBOL(__folio_put);
>
> typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio);
>
> +/*
> + * Unqueue a batch of folios from the deferred split queue. This must be
> + * called after the folios' refcounts have dropped to zero and before they
> + * are uncharged (see comment above __folio_unqueue_deferred_split() for why).
> + */
> +void folio_batch_unqueue_deferred_split(struct folio_batch *fbatch)
> +{
> + unsigned int i;
> +
> + for (i = 0; i < folio_batch_count(fbatch); i++)
> + folio_unqueue_deferred_split(fbatch->folios[i]);
> +}
> +
> static void lru_add(struct lruvec *lruvec, struct folio *folio)
> {
> int was_unevictable = folio_test_clear_unevictable(folio);
> @@ -177,14 +190,10 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
> *
> * We're bypassing normal cleanup. Clear flags that are not
> * applicable to dead folios.
> - *
> - * Since the folio may be part of a huge page, unqueue from
> - * deferred split list to avoid a dangling list entry.
> */
> if (is_lru_add && folio_ref_freeze(folio, 1)) {
> __folio_clear_active(folio);
> __folio_clear_unevictable(folio);
> - folio_unqueue_deferred_split(folio);
> fbatch->folios[i] = NULL;
> folio_batch_add(&free_fbatch, folio);
> continue;
> @@ -201,6 +210,11 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
>
> /* Cleanup filtered dead folios. */
> if (is_lru_add) {
> + /*
> + * Since the folios may be part of a huge page, unqueue them
> + * from the deferred split list to avoid a dangling list entry.
> + */
In general, this lgtm, thanks. I am confused about this existing comment you are
moving.
folios being *part of* a huge page? Huh? A folio might represent a huge page,
but what does the "part of" thing mean here.
Is it simply "Dequeue any large folios" ?
--
Cheers,
David
On Wed, Oct 14, 2026 at 2:31 AM David Hildenbrand (Arm)
<david@kernel.org> wrote:
>
> > @@ -177,14 +190,10 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
> > *
> > * We're bypassing normal cleanup. Clear flags that are not
> > * applicable to dead folios.
> > - *
> > - * Since the folio may be part of a huge page, unqueue from
> > - * deferred split list to avoid a dangling list entry.
> > */
> > if (is_lru_add && folio_ref_freeze(folio, 1)) {
> > __folio_clear_active(folio);
> > __folio_clear_unevictable(folio);
> > - folio_unqueue_deferred_split(folio);
> > fbatch->folios[i] = NULL;
> > folio_batch_add(&free_fbatch, folio);
> > continue;
> > @@ -201,6 +210,11 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
> >
> > /* Cleanup filtered dead folios. */
> > if (is_lru_add) {
> > + /*
> > + * Since the folios may be part of a huge page, unqueue them
> > + * from the deferred split list to avoid a dangling list entry.
> > + */
>
> In general, this lgtm, thanks. I am confused about this existing comment you are
> moving.
>
> folios being *part of* a huge page? Huh? A folio might represent a huge page,
> but what does the "part of" thing mean here.
>
> Is it simply "Dequeue any large folios" ?
Agreed, I'll reword this to "Unqueue any large folios from the
deferred split queue to avoid a dangling list entry once the folios
are freed".
Thanks,
Joanne
On Tue, Oct 13, 2026 at 03:13:07PM -0700, Joanne Koong wrote:
>move_folios_to_lru(), folio_batch_move_lru() and folios_put_refs() call
>folio_unqueue_deferred_split() on folios where the refcount has dropped
>to zero. In these paths, folio_unqueue_deferred_split() is called with
>the lruvec lock often held (move_folios_to_lru() always holds it and
>folio_batch_move_lru()/folios_put_refs() usually hold it for every folio
>in the batch except the first). However, it's not needed to hold the
>lruvec lock.
>
>The lruvec lock protects folio->lru and the reclaim LRU lists whereas
>the deferred split queue lives on folio->_deferred_list in
>deferred_split_lru. The deferred_split_lru is a list_lru with its own
>per-memcg per-node lock.
>
>There are already other existing callers that call
>folio_unqueue_deferred_split() without holding the lruvec lock (eg
>__folio_put(), shrink_folio_list(), __folio_migrate_mapping(), ...).
>
>Add folio_batch_unqueue_deferred_split() and use it in
>move_folios_to_lru(), folio_batch_move_lru(), and folios_put_refs() to
>unqueue each batch of zero-refcount folios from the deferred split list
>after the lruvec lock is released and before
>mem_cgroup_uncharge_folios(). This helps shorten the lruvec lock's
>critical sections, which run with IRQs disabled and are highly
>contended.
Nice work! Thanks!
BTW, have you seen measurable overhead from unqueueing deferred split
folios under the lruvec lock?
Any before/after numbers for this change, including the cost of the extra
pass over the batch?
Cheers, Lance
>Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
>---
> mm/folio.c | 24 +++++++++++++++++++-----
> mm/internal.h | 1 +
> mm/vmscan.c | 3 ++-
> 3 files changed, 22 insertions(+), 6 deletions(-)
>
>diff --git a/mm/folio.c b/mm/folio.c
>index 35e242b48870..85144078e635 100644
>--- a/mm/folio.c
>+++ b/mm/folio.c
>@@ -109,6 +109,19 @@ EXPORT_SYMBOL(__folio_put);
>
> typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio);
>
>+/*
>+ * Unqueue a batch of folios from the deferred split queue. This must be
>+ * called after the folios' refcounts have dropped to zero and before they
>+ * are uncharged (see comment above __folio_unqueue_deferred_split() for why).
>+ */
>+void folio_batch_unqueue_deferred_split(struct folio_batch *fbatch)
>+{
>+ unsigned int i;
>+
>+ for (i = 0; i < folio_batch_count(fbatch); i++)
>+ folio_unqueue_deferred_split(fbatch->folios[i]);
>+}
>+
> static void lru_add(struct lruvec *lruvec, struct folio *folio)
> {
> int was_unevictable = folio_test_clear_unevictable(folio);
>@@ -177,14 +190,10 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
> *
> * We're bypassing normal cleanup. Clear flags that are not
> * applicable to dead folios.
>- *
>- * Since the folio may be part of a huge page, unqueue from
>- * deferred split list to avoid a dangling list entry.
> */
> if (is_lru_add && folio_ref_freeze(folio, 1)) {
> __folio_clear_active(folio);
> __folio_clear_unevictable(folio);
>- folio_unqueue_deferred_split(folio);
> fbatch->folios[i] = NULL;
> folio_batch_add(&free_fbatch, folio);
> continue;
>@@ -201,6 +210,11 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
>
> /* Cleanup filtered dead folios. */
> if (is_lru_add) {
>+ /*
>+ * Since the folios may be part of a huge page, unqueue them
>+ * from the deferred split list to avoid a dangling list entry.
>+ */
>+ folio_batch_unqueue_deferred_split(&free_fbatch);
> mem_cgroup_uncharge_folios(&free_fbatch);
> free_unref_folios(&free_fbatch);
> }
>@@ -1020,7 +1034,6 @@ void folios_put_refs(struct folio_batch *folios, unsigned int *refs)
> free_huge_folio(folio);
> continue;
> }
>- folio_unqueue_deferred_split(folio);
> __page_cache_release(folio, &lruvec, &flags);
>
> if (j != i)
>@@ -1035,6 +1048,7 @@ void folios_put_refs(struct folio_batch *folios, unsigned int *refs)
> }
>
> folios->nr = j;
>+ folio_batch_unqueue_deferred_split(folios);
> mem_cgroup_uncharge_folios(folios);
> free_unref_folios(folios);
> }
>diff --git a/mm/internal.h b/mm/internal.h
>index 0ca863f2639a..ac72577a6e3e 100644
>--- a/mm/internal.h
>+++ b/mm/internal.h
>@@ -760,6 +760,7 @@ static inline bool folio_unqueue_deferred_split(struct folio *folio)
>
> return __folio_unqueue_deferred_split(folio);
> }
>+void folio_batch_unqueue_deferred_split(struct folio_batch *fbatch);
>
> static inline struct folio *page_rmappable_folio(struct page *page)
> {
>diff --git a/mm/vmscan.c b/mm/vmscan.c
>index 29b31446fc11..94859a830fb0 100644
>--- a/mm/vmscan.c
>+++ b/mm/vmscan.c
>@@ -2007,9 +2007,9 @@ static unsigned int move_folios_to_lru(struct list_head *list, bool do_rotate)
> if (unlikely(folio_put_testzero(folio))) {
> __folio_clear_lru_flags(folio);
>
>- folio_unqueue_deferred_split(folio);
> if (folio_batch_add(&free_folios, folio) == 0) {
> lruvec_unlock_irq(lruvec);
>+ folio_batch_unqueue_deferred_split(&free_folios);
> mem_cgroup_uncharge_folios(&free_folios);
> free_unref_folios(&free_folios);
> lruvec = NULL;
>@@ -2041,6 +2041,7 @@ static unsigned int move_folios_to_lru(struct list_head *list, bool do_rotate)
> lruvec_unlock_irq(lruvec);
>
> if (free_folios.nr) {
>+ folio_batch_unqueue_deferred_split(&free_folios);
> mem_cgroup_uncharge_folios(&free_folios);
> free_unref_folios(&free_folios);
> }
>--
>2.52.0
>
>
On Wed, Oct 14, 2026 at 11:07 AM Lance Yang <lance.yang@linux.dev> wrote: > > > On Tue, Oct 13, 2026 at 03:13:07PM -0700, Joanne Koong wrote: > >move_folios_to_lru(), folio_batch_move_lru() and folios_put_refs() call > >folio_unqueue_deferred_split() on folios where the refcount has dropped > >to zero. In these paths, folio_unqueue_deferred_split() is called with > >the lruvec lock often held (move_folios_to_lru() always holds it and > >folio_batch_move_lru()/folios_put_refs() usually hold it for every folio > >in the batch except the first). However, it's not needed to hold the > >lruvec lock. > > > >The lruvec lock protects folio->lru and the reclaim LRU lists whereas > >the deferred split queue lives on folio->_deferred_list in > >deferred_split_lru. The deferred_split_lru is a list_lru with its own > >per-memcg per-node lock. > > > >There are already other existing callers that call > >folio_unqueue_deferred_split() without holding the lruvec lock (eg > >__folio_put(), shrink_folio_list(), __folio_migrate_mapping(), ...). > > > >Add folio_batch_unqueue_deferred_split() and use it in > >move_folios_to_lru(), folio_batch_move_lru(), and folios_put_refs() to > >unqueue each batch of zero-refcount folios from the deferred split list > >after the lruvec lock is released and before > >mem_cgroup_uncharge_folios(). This helps shorten the lruvec lock's > >critical sections, which run with IRQs disabled and are highly > >contended. > > Nice work! Thanks! > > BTW, have you seen measurable overhead from unqueueing deferred split > folios under the lruvec lock? > > Any before/after numbers for this change, including the cost of the extra > pass over the batch? I recall Joanne mentioning some data on this in the link below. Is that right? https://lore.kernel.org/linux-mm/CAJnrk1a2PpUGTsCgS7jct2KET4K2zAL+_h+te8G1MaU9bXs99w@mail.gmail.com/ Best Regards Barry
On 2026/10/14 11:13, Barry Song wrote: > On Wed, Oct 14, 2026 at 11:07 AM Lance Yang <lance.yang@linux.dev> wrote: >> >> >> On Tue, Oct 13, 2026 at 03:13:07PM -0700, Joanne Koong wrote: >>> move_folios_to_lru(), folio_batch_move_lru() and folios_put_refs() call >>> folio_unqueue_deferred_split() on folios where the refcount has dropped >>> to zero. In these paths, folio_unqueue_deferred_split() is called with >>> the lruvec lock often held (move_folios_to_lru() always holds it and >>> folio_batch_move_lru()/folios_put_refs() usually hold it for every folio >>> in the batch except the first). However, it's not needed to hold the >>> lruvec lock. >>> >>> The lruvec lock protects folio->lru and the reclaim LRU lists whereas >>> the deferred split queue lives on folio->_deferred_list in >>> deferred_split_lru. The deferred_split_lru is a list_lru with its own >>> per-memcg per-node lock. >>> >>> There are already other existing callers that call >>> folio_unqueue_deferred_split() without holding the lruvec lock (eg >>> __folio_put(), shrink_folio_list(), __folio_migrate_mapping(), ...). >>> >>> Add folio_batch_unqueue_deferred_split() and use it in >>> move_folios_to_lru(), folio_batch_move_lru(), and folios_put_refs() to >>> unqueue each batch of zero-refcount folios from the deferred split list >>> after the lruvec lock is released and before >>> mem_cgroup_uncharge_folios(). This helps shorten the lruvec lock's >>> critical sections, which run with IRQs disabled and are highly >>> contended. >> >> Nice work! Thanks! >> >> BTW, have you seen measurable overhead from unqueueing deferred split >> folios under the lruvec lock? >> >> Any before/after numbers for this change, including the cost of the extra >> pass over the batch? > > I recall Joanne mentioning some data on this in the link below. Is that right? > > https://lore.kernel.org/linux-mm/CAJnrk1a2PpUGTsCgS7jct2KET4K2zAL+_h+te8G1MaU9bXs99w@mail.gmail.com/ Thanks, Barry! Yes, those numbers answer my question about the observed overhead :) Joanne, have you rerun that benchmark with this patch? I'd be interested in the before/after runtime and lruvec lock wait. Cheers, Lance
On Tue, Oct 13, 2026 at 9:24 PM Lance Yang <lance.yang@linux.dev> wrote: > > > > On 2026/10/14 11:13, Barry Song wrote: > > On Wed, Oct 14, 2026 at 11:07 AM Lance Yang <lance.yang@linux.dev> wrote: > >> > >> > >> On Tue, Oct 13, 2026 at 03:13:07PM -0700, Joanne Koong wrote: > >>> move_folios_to_lru(), folio_batch_move_lru() and folios_put_refs() call > >>> folio_unqueue_deferred_split() on folios where the refcount has dropped > >>> to zero. In these paths, folio_unqueue_deferred_split() is called with > >>> the lruvec lock often held (move_folios_to_lru() always holds it and > >>> folio_batch_move_lru()/folios_put_refs() usually hold it for every folio > >>> in the batch except the first). However, it's not needed to hold the > >>> lruvec lock. > >>> > >>> The lruvec lock protects folio->lru and the reclaim LRU lists whereas > >>> the deferred split queue lives on folio->_deferred_list in > >>> deferred_split_lru. The deferred_split_lru is a list_lru with its own > >>> per-memcg per-node lock. > >>> > >>> There are already other existing callers that call > >>> folio_unqueue_deferred_split() without holding the lruvec lock (eg > >>> __folio_put(), shrink_folio_list(), __folio_migrate_mapping(), ...). > >>> > >>> Add folio_batch_unqueue_deferred_split() and use it in > >>> move_folios_to_lru(), folio_batch_move_lru(), and folios_put_refs() to > >>> unqueue each batch of zero-refcount folios from the deferred split list > >>> after the lruvec lock is released and before > >>> mem_cgroup_uncharge_folios(). This helps shorten the lruvec lock's > >>> critical sections, which run with IRQs disabled and are highly > >>> contended. > >> > >> Nice work! Thanks! > >> > >> BTW, have you seen measurable overhead from unqueueing deferred split > >> folios under the lruvec lock? > >> > >> Any before/after numbers for this change, including the cost of the extra > >> pass over the batch? > > > > I recall Joanne mentioning some data on this in the link below. Is that right? > > > > https://lore.kernel.org/linux-mm/CAJnrk1a2PpUGTsCgS7jct2KET4K2zAL+_h+te8G1MaU9bXs99w@mail.gmail.com/ > > Thanks, Barry! Yes, those numbers answer my question about the > observed overhead :) > > Joanne, have you rerun that benchmark with this patch? I'd be > interested in the before/after runtime and lruvec lock wait. Hi Lance, (Thanks Barry for providing the link to the earlier discussion!) I have not run those benchmarks with this patch yet but I will get those numbers and add them to v2 when I send that out. Thanks, Joanne
© 2016 - 2026 Red Hat, Inc.