include/linux/memcontrol.h | 9 +++- include/linux/mm_inline.h | 115 +++++++++++++++++++++++++++++++++++++++----- include/linux/mmzone.h | 3 ++ mm/folio.c | 19 ++++---- mm/memcontrol.c | 18 +------ mm/migrate.c | 6 +-- mm/vmscan.c | 117 +++++++++++++++++++++++---------------------- 7 files changed, 188 insertions(+), 99 deletions(-)
This is a cleanup series separated out from the MGLRU-FG series [1]. As
that series is getting too long in following updates, seperate out the
clean up part for easier review and merge.
No feature change is intended, except one bugfix. It mostly replaces
the open-coded bit operations scattered throughout the MGLRU code with
new helpers, with proper kdocs, sanity debug checks, and hardens a few
MGLRU functions.
A subtle generation counter leak is also found during the refactoring
and the fix is included.
Also collected review feedbacks on the cleanup part from the posted
series.
Link: https://lore.kernel.org/linux-mm/20260804-mglru-fg-v1-0-4d8dad39dad6@tencent.com/ [1]
Signed-off-by: Kairui Song <kasong@tencent.com>
---
Changes in v5:
- Change atomic long to plain long in patch 1, and adjust commit message.
- Rename is_file to type, and avoid touching folio_is_file_lru in patch 7/7.
- Link to v4: https://patch.msgid.link/20260831-mglru-flags-cleanup-v4-0-2d15dde0d7ee@tencent.com
Changes in v4:
- Rename lru_gen_set_flags/lru_gen_from_flags to
lru_set_gen_flags/lru_get_gen_flags, similiar with lru refs helpers,
as suggested by [ Barry Song ].
- Added some comments to cover concerned raised by [ Barry Song ] and [
Baolin Wang ].
- A little code shuffling and better sanity check for concerns raised by
[ Ridong Chen ].
- Collect tags.
- Link to v3: https://patch.msgid.link/20260826-mglru-flags-cleanup-v3-0-d9f1c75549c8@tencent.com
Changes in v3:
- Set PG_workingset before setting LRU refs bit to avoid any potential
under report of folio tier.
- Migrate folio's referenced status before PG_update so lockless readers
won't set stale referenced status.
- Link to v2: https://patch.msgid.link/20260824-mglru-flags-cleanup-v2-0-0104132114ae@tencent.com
Changes in v2:
- Dropped the redundant LRU_GEN_MAX macro.
- Renamed folio_migrate_refs() to folio_migrate_lru_refs() and reworded
its commit message to describe the referenced state being transferred.
- Rewrote patch 6's commit message with a detailed analysis of the
anon/file accounting race.
- Dropped patch 7, it doesn't really fix anything, and might be in
conflict with other updating series.
- Collected Reviewed-by tags.
- Link to v1: https://patch.msgid.link/20260818-mglru-flags-cleanup-v1-0-8dbbdac0d28c@tencent.com
---
Kairui Song (6):
mm/memcontrol: move the lru_zone_size sanity check to the reader side
mm/mglru: introduce helpers for manipulating gen and refs flags
mm/migrate: copy all referenced state via folio_migrate_lru_refs
mm/mglru: move max_seq read into walk_update_folio
mm/mglru: use explicit tier range in read_ctrl_pos()
mm/mglru: fix potential generation folio number leak
include/linux/memcontrol.h | 9 +++-
include/linux/mm_inline.h | 115 +++++++++++++++++++++++++++++++++++++++-----
include/linux/mmzone.h | 3 ++
mm/folio.c | 19 ++++----
mm/memcontrol.c | 18 +------
mm/migrate.c | 6 +--
mm/vmscan.c | 117 +++++++++++++++++++++++----------------------
7 files changed, 188 insertions(+), 99 deletions(-)
---
base-commit: 893f78beb0477671991ccaae87df75acc484162f
change-id: 20260818-mglru-flags-cleanup-cc49cfad654f
Best regards,
--
Kairui Song <kasong@tencent.com>
On Wed, 02 Sep 2026 17:50:53 +0800 Kairui Song via B4 Relay <devnull+kasong.tencent.com@kernel.org> wrote:
> This is a cleanup series separated out from the MGLRU-FG series [1]. As
> that series is getting too long in following updates, seperate out the
> clean up part for easier review and merge.
>
> No feature change is intended, except one bugfix. It mostly replaces
> the open-coded bit operations scattered throughout the MGLRU code with
> new helpers, with proper kdocs, sanity debug checks, and hardens a few
> MGLRU functions.
>
> A subtle generation counter leak is also found during the refactoring
> and the fix is included.
>
> Also collected review feedbacks on the cleanup part from the posted
> series.
Thanks. I hit a non-trivial reject in [2/6] presumably thanks to
mm.git (mm-new) race conditions (appended).
The patchset comes nicely review by humans, but AI is less happy:
https://sashiko.dev/#/patchset/20260902-mglru-flags-cleanup-v5-0-9db761d779ef@tencent.com
So please take a look at all that and retry in a few days?
Thanks.
--- mm/vmscan.c
+++ mm/vmscan.c
@@ -3307,21 +3313,20 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio)
int type = folio_is_file_lru(folio);
struct lru_gen_folio *lrugen = &lruvec->lrugen;
int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
- unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
-
- VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio);
+ unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
do {
- new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+ new_gen = lru_get_gen_flags(old_flags);
+
/* folio_update_gen() has promoted this page? */
if (new_gen >= 0 && new_gen != old_gen)
return new_gen;
+ new_flags = old_flags;
new_gen = (old_gen + 1) % MAX_NR_GENS;
-
- new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS);
- new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF;
- } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
+ lru_set_gen_flags(&new_flags, new_gen);
+ lru_set_refs_flags(&new_flags, 0);
+ } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
lru_gen_update_size(lruvec, folio, old_gen, new_gen);
On Thu, Sep 3, 2026 at 5:11 AM Andrew Morton <akpm@linux-foundation.org> wrote: > > On Wed, 02 Sep 2026 17:50:53 +0800 Kairui Song via B4 Relay <devnull+kasong.tencent.com@kernel.org> wrote: > > > This is a cleanup series separated out from the MGLRU-FG series [1]. As > > that series is getting too long in following updates, seperate out the > > clean up part for easier review and merge. > > > > No feature change is intended, except one bugfix. It mostly replaces > > the open-coded bit operations scattered throughout the MGLRU code with > > new helpers, with proper kdocs, sanity debug checks, and hardens a few > > MGLRU functions. > > > > A subtle generation counter leak is also found during the refactoring > > and the fix is included. > > > > Also collected review feedbacks on the cleanup part from the posted > > series. > > Thanks. I hit a non-trivial reject in [2/6] presumably thanks to > mm.git (mm-new) race conditions (appended). > > The patchset comes nicely review by humans, but AI is less happy: > https://sashiko.dev/#/patchset/20260902-mglru-flags-cleanup-v5-0-9db761d779ef@tencent.com > > So please take a look at all that and retry in a few days? Right, I checked both already, sashiko has beem compalining about the same issue again and again since V2: https://lore.kernel.org/all/CAMgjq7BheDgQZa0=KiRnxVwky9by5LA=iaexfPfT0WftgfL=qQ@mail.gmail.com/ It's safe to ignore: not introduced by this series, and not a real problem either. The refs part is also fine, it barely has any real effect. Should I just rebase on top of mm-new? I also saw Barry's rebased version of Patch 2, that looks good to me.
On Thu, Sep 3, 2026 at 5:11 AM Andrew Morton <akpm@linux-foundation.org> wrote:
>
> On Wed, 02 Sep 2026 17:50:53 +0800 Kairui Song via B4 Relay <devnull+kasong.tencent.com@kernel.org> wrote:
>
> > This is a cleanup series separated out from the MGLRU-FG series [1]. As
> > that series is getting too long in following updates, seperate out the
> > clean up part for easier review and merge.
> >
> > No feature change is intended, except one bugfix. It mostly replaces
> > the open-coded bit operations scattered throughout the MGLRU code with
> > new helpers, with proper kdocs, sanity debug checks, and hardens a few
> > MGLRU functions.
> >
> > A subtle generation counter leak is also found during the refactoring
> > and the fix is included.
> >
> > Also collected review feedbacks on the cleanup part from the posted
> > series.
>
> Thanks. I hit a non-trivial reject in [2/6] presumably thanks to
> mm.git (mm-new) race conditions (appended).
>
> The patchset comes nicely review by humans, but AI is less happy:
> https://sashiko.dev/#/patchset/20260902-mglru-flags-cleanup-v5-0-9db761d779ef@tencent.com
>
> So please take a look at all that and retry in a few days?
Sorry for the merge conflicts.
I rebased Kairui's series on top of `mm-new`. I guess all we need is to
make patch 2 look like this:
From 01e2013722443f685d47b42e7d9b11aadef58722 Mon Sep 17 00:00:00 2001
From: Kairui Song <kasong@tencent.com>
Date: Wed, 2 Sep 2026 17:50:55 +0800
Subject: [PATCH 2/6] mm/mglru: introduce helpers for manipulating gen and refs
flags
Instead of doing bit ops on folio->flags.f, introduce helpers for
adjusting a folio's refs and generation info, making the code easier
to debug and understand.
No functional change is intended: some combined atomic operations are
split into two, which only creates harmless transient states. There is
no measurable performance impact, and some paths even look slightly
better in the generated assembly.
Acked-by: Qi Zheng <qi.zheng@linux.dev>
Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Reviewed-by: Barry Song <baohua@kernel.org>
Signed-off-by: Kairui Song <kasong@tencent.com>
Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
---
include/linux/mm_inline.h | 84 +++++++++++++++++++++++++++++++++++----
include/linux/mmzone.h | 1 +
mm/folio.c | 19 +++++----
mm/vmscan.c | 60 ++++++++++++++++------------
4 files changed, 122 insertions(+), 42 deletions(-)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 621c8653d8f7..3f4bd5b02b54 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -142,10 +142,66 @@ static inline int lru_tier_from_refs(int refs, bool workingset)
return workingset ? MAX_NR_TIERS - 1 : order_base_2(refs);
}
-static inline int folio_lru_refs(const struct folio *folio)
+/**
+ * lru_set_gen_flags - Set the LRU generation number to specified folio flags.
+ * @flags: pointer to the folio flags
+ * @gen: generation number, between 0 and (MAX_NR_GENS - 1), inclusive.
+ */
+static inline void lru_set_gen_flags(unsigned long *flags, int gen)
+{
+ BUILD_BUG_ON(LRU_GEN_MASK & LRU_REFS_MASK);
+ VM_WARN_ON_ONCE(gen >= MAX_NR_GENS || gen < 0);
+ /* Store gen offset by 1, zero means the folio is off-list. */
+ *flags &= ~LRU_GEN_MASK;
+ *flags |= (gen + 1UL) << LRU_GEN_PGOFF;
+}
+
+/**
+ * lru_get_gen_flags - Return the LRU generation number from folio flags.
+ * @flags: folio flags
+ *
+ * Returns: A number between 0 and (MAX_NR_GENS - 1), inclusive. Returns
+ * -1 if the flags indicate the folio is off the list (e.g., isolated).
+ */
+static inline int lru_get_gen_flags(unsigned long flags)
{
- unsigned long flags = READ_ONCE(folio->flags.f);
+ int gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+ /* Exclude the legal -1 from the unsigned MAX_NR_GENS comparison */
+ VM_WARN_ON_ONCE(gen != -1 && gen >= MAX_NR_GENS);
+ return gen;
+}
+
+/**
+ * lru_set_refs_flags - Set the LRU referenced count to folio flags.
+ * @flags: pointer to the folio flags
+ * @refs: referenced / access count number, between 0 and LRU_REFS_MAX, inclusive.
+ *
+ * For MGLRU, PG_referenced holds the first ref, and the extra bits hold the
+ * remaining refs. For classical LRU the extra bits are not used, so it can
+ * also be seen as the refs count never exceeds 1. In both cases, refs == 1
+ * means PG_referenced is set and the extra bits are zero, and refs == 0 means
+ * PG_referenced and the extra bits are all unset.
+ */
+static inline void lru_set_refs_flags(unsigned long *flags, unsigned int refs)
+{
+ VM_WARN_ON_ONCE(refs > LRU_REFS_MAX);
+ BUILD_BUG_ON(LRU_REFS_MAX != (LRU_REFS_MASK >> LRU_REFS_PGOFF) + 1);
+
+ *flags &= ~LRU_REFS_FLAGS;
+ if (!refs)
+ return;
+ *flags |= (BIT(PG_referenced) | ((refs - 1UL) << LRU_REFS_PGOFF));
+}
+
+/**
+ * lru_get_refs_flags - Return LRU referenced / access count from folio flags.
+ * @flags: folio flags
+ *
+ * Reads the LRU referenced count set by lru_set_refs_flags().
+ */
+static inline int lru_get_refs_flags(unsigned long flags)
+{
if (!(flags & BIT(PG_referenced)))
return 0;
/*
@@ -155,11 +211,24 @@ static inline int folio_lru_refs(const struct folio *folio)
return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + 1;
}
-static inline int folio_lru_gen(const struct folio *folio)
+static inline int folio_lru_refs(const struct folio *folio)
{
- unsigned long flags = READ_ONCE(folio->flags.f);
+ return lru_get_refs_flags(READ_ONCE(*const_folio_flags(folio, 0)));
+}
+
+static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs)
+{
+ unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
+
+ do {
+ new_flags = old_flags;
+ lru_set_refs_flags(&new_flags, refs);
+ } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
+}
- return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+static inline int folio_lru_gen(const struct folio *folio)
+{
+ return lru_get_gen_flags(READ_ONCE(*const_folio_flags(folio, 0)));
}
static inline bool lru_gen_is_active(const struct lruvec *lruvec, int gen)
@@ -270,7 +339,7 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
gen = lru_gen_from_seq(seq);
flags = (gen + 1UL) << LRU_GEN_PGOFF;
/* see the comment on MIN_NR_GENS about PG_active */
- set_mask_bits(&folio->flags.f, LRU_GEN_MASK | BIT(PG_active), flags);
+ set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK | BIT(PG_active), flags);
lru_gen_update_size(lruvec, folio, -1, gen);
/* for folio_rotate_reclaimable() */
@@ -295,7 +364,7 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
/* for folio_migrate_flags() */
flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;
- flags = set_mask_bits(&folio->flags.f, LRU_GEN_MASK, flags);
+ flags = set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK, flags);
gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
lru_gen_update_size(lruvec, folio, gen, -1);
@@ -339,7 +408,6 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
static inline void folio_migrate_refs(struct folio *new, const struct folio *old)
{
-
}
#endif /* CONFIG_LRU_GEN */
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 84e237f2c17d..775d6279dfea 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -500,6 +500,7 @@ enum lruvec_flags {
#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)
#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)
+#define LRU_REFS_MAX BIT(LRU_REFS_WIDTH)
/*
* For folios accessed multiple times through file descriptors,
diff --git a/mm/folio.c b/mm/folio.c
index c02dcea9c03c..fb874fe492b2 100644
--- a/mm/folio.c
+++ b/mm/folio.c
@@ -353,26 +353,28 @@ static void __lru_cache_activate_folio(struct folio *folio)
static void lru_gen_inc_refs(struct folio *folio)
{
- unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
+ unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
+ int refs;
if (folio_test_unevictable(folio))
return;
/* see the comment on LRU_REFS_FLAGS */
- if (!folio_test_referenced(folio)) {
- set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
+ if (!folio_lru_refs(folio)) {
+ folio_set_lru_refs(folio, 1);
return;
}
do {
- if ((old_flags & LRU_REFS_MASK) == LRU_REFS_MASK) {
+ new_flags = old_flags;
+ refs = lru_get_refs_flags(old_flags);
+ if (refs == LRU_REFS_MAX) {
if (!folio_test_workingset(folio))
folio_set_workingset(folio);
return;
}
-
- new_flags = old_flags + BIT(LRU_REFS_PGOFF);
- } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
+ lru_set_refs_flags(&new_flags, refs + 1);
+ } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
}
static bool lru_gen_clear_refs(struct folio *folio)
@@ -384,7 +386,8 @@ static bool lru_gen_clear_refs(struct folio *folio)
if (gen < 0)
return true;
- set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS | BIT(PG_workingset), 0);
+ folio_set_lru_refs(folio, 0);
+ folio_clear_workingset(folio);
rcu_read_lock();
seq = READ_ONCE(folio_lruvec(folio)->lrugen.min_seq[type]);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index bf2786c7247d..71adf4bf5074 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -863,19 +863,22 @@ static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags)
if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {
/* Activate file-backed executable folios after first usage. */
if (is_exec_file_folio(folio, vma_flags)) {
- set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset));
+ folio_set_workingset(folio);
+ folio_set_lru_refs(folio, 0);
return true;
}
- set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
+ folio_set_lru_refs(folio, 1);
return false;
}
/* Promote on second access */
- if (folio_lru_refs(folio) > 1)
- set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset));
- else
+ if (folio_lru_refs(folio) > 1) {
+ folio_set_workingset(folio);
+ folio_set_lru_refs(folio, 0);
+ } else {
folio_mark_accessed(folio);
+ }
return true;
}
#else
@@ -3291,11 +3294,10 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv)
******************************************************************************/
/* promote pages accessed through page tables */
-static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma_flags)
+static int folio_update_gen(struct folio *folio, int new_gen, const vma_flags_t *vma_flags)
{
- unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
-
- VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
+ unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
+ int old_gen;
/*
* See the comment on LRU_REFS_FLAGS, and activate file-backed
@@ -3304,31 +3306,34 @@ static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma
*/
if (!folio_test_referenced(folio) && !folio_test_workingset(folio) &&
!is_exec_file_folio(folio, vma_flags)) {
- set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
+ folio_set_lru_refs(folio, 1);
return -1;
}
do {
+ old_gen = lru_get_gen_flags(old_flags);
+ new_flags = old_flags;
+
/* lru_gen_del_folio() has isolated this page? */
- if (!(old_flags & LRU_GEN_MASK))
- return -1;
+ if (old_gen < 0)
+ break;
- new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS);
- new_flags |= ((gen + 1UL) << LRU_GEN_PGOFF) | BIT(PG_workingset);
- } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
+ lru_set_gen_flags(&new_flags, new_gen);
+ lru_set_refs_flags(&new_flags, 0);
+ new_flags |= BIT(PG_workingset);
+ } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
- return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+ return old_gen;
}
static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased)
{
- unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
+ unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
int new_gen;
- VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio);
-
do {
- new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+ new_gen = lru_get_gen_flags(old_flags);
+
/* folio_update_gen() has promoted this page? */
if (new_gen >= 0 && new_gen != old_gen) {
if (increased)
@@ -3336,11 +3341,12 @@ static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased)
return new_gen;
}
+ new_flags = old_flags;
new_gen = (old_gen + 1) % MAX_NR_GENS;
- new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS);
- new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF;
- } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
+ lru_set_gen_flags(&new_flags, new_gen);
+ lru_set_refs_flags(&new_flags, 0);
+ } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
if (increased)
*increased = true;
@@ -4785,7 +4791,7 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
/* see the comment on LRU_REFS_FLAGS */
if (!folio_test_referenced(folio))
- set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0);
+ folio_set_lru_refs(folio, 0);
success = lru_gen_del_folio(lruvec, folio, true);
VM_WARN_ON_ONCE_FOLIO(!success, folio);
@@ -5017,8 +5023,10 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
}
/* don't add rejected folios to the oldest generation */
- if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type])
- set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_active));
+ if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) {
+ folio_set_lru_refs(folio, 0);
+ folio_set_active(folio);
+ }
}
move_folios_to_lru(&list);
--
2.39.3 (Apple Git-146)
>
> Thanks.
On Thu, Sep 3, 2026 at 6:31 AM Barry Song (Xiaomi) <baohua@kernel.org> wrote:
>
> I rebased Kairui's series on top of `mm-new`. I guess all we need is to
> make patch 2 look like this:
>
> From 01e2013722443f685d47b42e7d9b11aadef58722 Mon Sep 17 00:00:00 2001
> From: Kairui Song <kasong@tencent.com>
> Date: Wed, 2 Sep 2026 17:50:55 +0800
> Subject: [PATCH 2/6] mm/mglru: introduce helpers for manipulating gen and refs
> flags
>
> Instead of doing bit ops on folio->flags.f, introduce helpers for
> adjusting a folio's refs and generation info, making the code easier
> to debug and understand.
>
> No functional change is intended: some combined atomic operations are
> split into two, which only creates harmless transient states. There is
> no measurable performance impact, and some paths even look slightly
> better in the generated assembly.
>
> Acked-by: Qi Zheng <qi.zheng@linux.dev>
> Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com>
> Reviewed-by: Barry Song <baohua@kernel.org>
> Signed-off-by: Kairui Song <kasong@tencent.com>
> Signed-off-by: Barry Song (Xiaomi) <baohua@kernel.org>
> ---
> include/linux/mm_inline.h | 84 +++++++++++++++++++++++++++++++++++----
> include/linux/mmzone.h | 1 +
> mm/folio.c | 19 +++++----
> mm/vmscan.c | 60 ++++++++++++++++------------
> 4 files changed, 122 insertions(+), 42 deletions(-)
>
> diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
> index 621c8653d8f7..3f4bd5b02b54 100644
> --- a/include/linux/mm_inline.h
> +++ b/include/linux/mm_inline.h
> @@ -142,10 +142,66 @@ static inline int lru_tier_from_refs(int refs, bool workingset)
> return workingset ? MAX_NR_TIERS - 1 : order_base_2(refs);
> }
>
> -static inline int folio_lru_refs(const struct folio *folio)
> +/**
> + * lru_set_gen_flags - Set the LRU generation number to specified folio flags.
> + * @flags: pointer to the folio flags
> + * @gen: generation number, between 0 and (MAX_NR_GENS - 1), inclusive.
> + */
> +static inline void lru_set_gen_flags(unsigned long *flags, int gen)
> +{
> + BUILD_BUG_ON(LRU_GEN_MASK & LRU_REFS_MASK);
> + VM_WARN_ON_ONCE(gen >= MAX_NR_GENS || gen < 0);
> + /* Store gen offset by 1, zero means the folio is off-list. */
> + *flags &= ~LRU_GEN_MASK;
> + *flags |= (gen + 1UL) << LRU_GEN_PGOFF;
> +}
> +
> +/**
> + * lru_get_gen_flags - Return the LRU generation number from folio flags.
> + * @flags: folio flags
> + *
> + * Returns: A number between 0 and (MAX_NR_GENS - 1), inclusive. Returns
> + * -1 if the flags indicate the folio is off the list (e.g., isolated).
> + */
> +static inline int lru_get_gen_flags(unsigned long flags)
> {
> - unsigned long flags = READ_ONCE(folio->flags.f);
> + int gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
>
> + /* Exclude the legal -1 from the unsigned MAX_NR_GENS comparison */
> + VM_WARN_ON_ONCE(gen != -1 && gen >= MAX_NR_GENS);
> + return gen;
> +}
> +
> +/**
> + * lru_set_refs_flags - Set the LRU referenced count to folio flags.
> + * @flags: pointer to the folio flags
> + * @refs: referenced / access count number, between 0 and LRU_REFS_MAX, inclusive.
> + *
> + * For MGLRU, PG_referenced holds the first ref, and the extra bits hold the
> + * remaining refs. For classical LRU the extra bits are not used, so it can
> + * also be seen as the refs count never exceeds 1. In both cases, refs == 1
> + * means PG_referenced is set and the extra bits are zero, and refs == 0 means
> + * PG_referenced and the extra bits are all unset.
> + */
> +static inline void lru_set_refs_flags(unsigned long *flags, unsigned int refs)
> +{
> + VM_WARN_ON_ONCE(refs > LRU_REFS_MAX);
> + BUILD_BUG_ON(LRU_REFS_MAX != (LRU_REFS_MASK >> LRU_REFS_PGOFF) + 1);
> +
> + *flags &= ~LRU_REFS_FLAGS;
> + if (!refs)
> + return;
> + *flags |= (BIT(PG_referenced) | ((refs - 1UL) << LRU_REFS_PGOFF));
> +}
> +
> +/**
> + * lru_get_refs_flags - Return LRU referenced / access count from folio flags.
> + * @flags: folio flags
> + *
> + * Reads the LRU referenced count set by lru_set_refs_flags().
> + */
> +static inline int lru_get_refs_flags(unsigned long flags)
> +{
> if (!(flags & BIT(PG_referenced)))
> return 0;
> /*
> @@ -155,11 +211,24 @@ static inline int folio_lru_refs(const struct folio *folio)
> return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + 1;
> }
>
> -static inline int folio_lru_gen(const struct folio *folio)
> +static inline int folio_lru_refs(const struct folio *folio)
> {
> - unsigned long flags = READ_ONCE(folio->flags.f);
> + return lru_get_refs_flags(READ_ONCE(*const_folio_flags(folio, 0)));
> +}
> +
> +static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs)
> +{
> + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
> +
> + do {
> + new_flags = old_flags;
> + lru_set_refs_flags(&new_flags, refs);
> + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
> +}
>
> - return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
> +static inline int folio_lru_gen(const struct folio *folio)
> +{
> + return lru_get_gen_flags(READ_ONCE(*const_folio_flags(folio, 0)));
> }
>
> static inline bool lru_gen_is_active(const struct lruvec *lruvec, int gen)
> @@ -270,7 +339,7 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
> gen = lru_gen_from_seq(seq);
> flags = (gen + 1UL) << LRU_GEN_PGOFF;
> /* see the comment on MIN_NR_GENS about PG_active */
> - set_mask_bits(&folio->flags.f, LRU_GEN_MASK | BIT(PG_active), flags);
> + set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK | BIT(PG_active), flags);
>
> lru_gen_update_size(lruvec, folio, -1, gen);
> /* for folio_rotate_reclaimable() */
> @@ -295,7 +364,7 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
>
> /* for folio_migrate_flags() */
> flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;
> - flags = set_mask_bits(&folio->flags.f, LRU_GEN_MASK, flags);
> + flags = set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK, flags);
> gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
>
> lru_gen_update_size(lruvec, folio, gen, -1);
> @@ -339,7 +408,6 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
>
> static inline void folio_migrate_refs(struct folio *new, const struct folio *old)
> {
> -
> }
> #endif /* CONFIG_LRU_GEN */
>
> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
> index 84e237f2c17d..775d6279dfea 100644
> --- a/include/linux/mmzone.h
> +++ b/include/linux/mmzone.h
> @@ -500,6 +500,7 @@ enum lruvec_flags {
>
> #define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)
> #define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)
> +#define LRU_REFS_MAX BIT(LRU_REFS_WIDTH)
>
> /*
> * For folios accessed multiple times through file descriptors,
> diff --git a/mm/folio.c b/mm/folio.c
> index c02dcea9c03c..fb874fe492b2 100644
> --- a/mm/folio.c
> +++ b/mm/folio.c
> @@ -353,26 +353,28 @@ static void __lru_cache_activate_folio(struct folio *folio)
>
> static void lru_gen_inc_refs(struct folio *folio)
> {
> - unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
> + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
> + int refs;
>
> if (folio_test_unevictable(folio))
> return;
>
> /* see the comment on LRU_REFS_FLAGS */
> - if (!folio_test_referenced(folio)) {
> - set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
> + if (!folio_lru_refs(folio)) {
> + folio_set_lru_refs(folio, 1);
> return;
> }
>
> do {
> - if ((old_flags & LRU_REFS_MASK) == LRU_REFS_MASK) {
> + new_flags = old_flags;
> + refs = lru_get_refs_flags(old_flags);
> + if (refs == LRU_REFS_MAX) {
> if (!folio_test_workingset(folio))
> folio_set_workingset(folio);
> return;
> }
> -
> - new_flags = old_flags + BIT(LRU_REFS_PGOFF);
> - } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
> + lru_set_refs_flags(&new_flags, refs + 1);
> + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
> }
>
> static bool lru_gen_clear_refs(struct folio *folio)
> @@ -384,7 +386,8 @@ static bool lru_gen_clear_refs(struct folio *folio)
> if (gen < 0)
> return true;
>
> - set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS | BIT(PG_workingset), 0);
> + folio_set_lru_refs(folio, 0);
> + folio_clear_workingset(folio);
>
> rcu_read_lock();
> seq = READ_ONCE(folio_lruvec(folio)->lrugen.min_seq[type]);
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index bf2786c7247d..71adf4bf5074 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -863,19 +863,22 @@ static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags)
> if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {
> /* Activate file-backed executable folios after first usage. */
> if (is_exec_file_folio(folio, vma_flags)) {
> - set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset));
> + folio_set_workingset(folio);
> + folio_set_lru_refs(folio, 0);
> return true;
> }
>
> - set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
> + folio_set_lru_refs(folio, 1);
> return false;
> }
>
> /* Promote on second access */
> - if (folio_lru_refs(folio) > 1)
> - set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset));
> - else
> + if (folio_lru_refs(folio) > 1) {
> + folio_set_workingset(folio);
> + folio_set_lru_refs(folio, 0);
> + } else {
> folio_mark_accessed(folio);
> + }
> return true;
> }
> #else
> @@ -3291,11 +3294,10 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv)
> ******************************************************************************/
>
> /* promote pages accessed through page tables */
> -static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma_flags)
> +static int folio_update_gen(struct folio *folio, int new_gen, const vma_flags_t *vma_flags)
> {
> - unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
> -
> - VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
> + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
> + int old_gen;
>
> /*
> * See the comment on LRU_REFS_FLAGS, and activate file-backed
> @@ -3304,31 +3306,34 @@ static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma
> */
> if (!folio_test_referenced(folio) && !folio_test_workingset(folio) &&
> !is_exec_file_folio(folio, vma_flags)) {
> - set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
> + folio_set_lru_refs(folio, 1);
> return -1;
> }
>
> do {
> + old_gen = lru_get_gen_flags(old_flags);
> + new_flags = old_flags;
> +
> /* lru_gen_del_folio() has isolated this page? */
> - if (!(old_flags & LRU_GEN_MASK))
> - return -1;
> + if (old_gen < 0)
> + break;
>
> - new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS);
> - new_flags |= ((gen + 1UL) << LRU_GEN_PGOFF) | BIT(PG_workingset);
> - } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
> + lru_set_gen_flags(&new_flags, new_gen);
> + lru_set_refs_flags(&new_flags, 0);
> + new_flags |= BIT(PG_workingset);
> + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
>
> - return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
> + return old_gen;
> }
>
> static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased)
> {
> - unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
> + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
> int new_gen;
>
> - VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio);
> -
> do {
> - new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
> + new_gen = lru_get_gen_flags(old_flags);
> +
> /* folio_update_gen() has promoted this page? */
> if (new_gen >= 0 && new_gen != old_gen) {
> if (increased)
> @@ -3336,11 +3341,12 @@ static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased)
> return new_gen;
> }
>
> + new_flags = old_flags;
> new_gen = (old_gen + 1) % MAX_NR_GENS;
>
> - new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS);
> - new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF;
> - } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
> + lru_set_gen_flags(&new_flags, new_gen);
> + lru_set_refs_flags(&new_flags, 0);
> + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
>
> if (increased)
> *increased = true;
> @@ -4785,7 +4791,7 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
>
> /* see the comment on LRU_REFS_FLAGS */
> if (!folio_test_referenced(folio))
> - set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0);
> + folio_set_lru_refs(folio, 0);
>
> success = lru_gen_del_folio(lruvec, folio, true);
> VM_WARN_ON_ONCE_FOLIO(!success, folio);
> @@ -5017,8 +5023,10 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
> }
>
> /* don't add rejected folios to the oldest generation */
> - if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type])
> - set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_active));
> + if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) {
> + folio_set_lru_refs(folio, 0);
> + folio_set_active(folio);
> + }
> }
>
> move_folios_to_lru(&list);
> --
> 2.39.3 (Apple Git-146)
>
>
> >
> > Thanks.
>
Thanks a lot! This looks good to me. I tried a rebase locally, which
resulted in the same thing. I think we can continue the merge with
this version.
© 2016 - 2026 Red Hat, Inc.