include/linux/memcontrol.h | 3 +++ mm/memcontrol.c | 41 +++++++++++++++++++++++++++++++++++--- mm/workingset.c | 8 ++++++++ 3 files changed, 49 insertions(+), 3 deletions(-)
From: Jiayuan Chen <jiayuan.chen@shopee.com>
Problem
-------
We observed an issue in production where a workload continuously
triggering memory.high also generates massive disk IO READ, causing
system-wide performance degradation.
This happens because memory.high penalty is currently based solely on
the overage amount, not the actual impact of that overage:
1. A memcg over memory.high reclaiming cold/unused pages
→ minimal system impact, light penalty is appropriate
2. A memcg over memory.high with hot pages being continuously
reclaimed and refaulted → severe IO pressure, needs heavy penalty
Both cases receive identical penalties today. Users are forced to
combine memory.high with io.max as a workaround, but this is:
- The wrong abstraction level (memory policy shouldn't require IO tuning)
- Hard to configure correctly across different storage devices
- Unintuitive for users who only want memory control
Reproduction
------------
A simple test program demonstrates the issue:
int fd = open("./200MB.file", O_RDWR|O_CREAT, 777);
char *mem = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
while (1) {
for (size_t i = 0; i < size; i += 4096) {
if (mem[rand() % size] != 0)
return -1;
}
}
Run with memory.high constraint:
cgcreate -g io,cpu,cpuset,memory:/always_high
cgset -r cpuset.cpus=0 always_high
cgset -r memory.high=150M always_high
cgexec -g cpu,cpuset,memory:/always_high ./high_test 200 &
Solution
--------
Incorporate refault recency into the penalty calculation. If a refault
occurred recently when memory.high is triggered, it indicates active
thrashing and warrants additional throttling.
Why not use refault counters directly?
- Refault statistics (WORKINGSET_REFAULT_*) are aggregated periodically,
not available in real-time for accurate delta calculation
- Calling mem_cgroup_flush_stats() on every charge would be prohibitively
expensive in the hot path
- Due to readahead, the same refault count can represent vastly different
IO loads, making counter-based estimation unreliable
The timestamp-based approach is:
- O(1) cost: single timestamp read and comparison
- Self-calibrating: penalty scales naturally with refault frequency
- Conservative: only triggers when refault and memory.high event
occur in close temporal proximity
When refault_penalty is active:
- Skip the "reclaim made progress" retry loop to apply throttling sooner
- Skip the "penalty too small" bypass to ensure some delay is applied
- Add refault-based delay to the overage-based delay
Results
-------
Before this patch (memory.high triggered, severe thrashing):
sar -d 1
Time DEV tps rkB/s %util
04:17:42 sda 3242.00 272684.00 89.60
04:17:43 sda 3412.00 251160.00 91.60
04:17:44 sda 3185.00 254532.00 88.00
04:17:45 sda 3230.00 253332.00 88.40
04:17:46 sda 3416.00 224712.00 92.40
04:17:47 sda 3613.00 206612.00 94.40
After this patch (memory.high triggered, thrashing mitigated):
sar -d 1
Time DEV tps rkB/s %util
04:08:57 sda 512.00 2048.00 5.60
04:08:58 sda 576.00 2304.00 6.80
04:08:59 sda 512.00 2048.00 6.80
04:09:00 sda 536.00 2144.00 4.80
04:09:01 sda 552.00 2208.00 10.40
04:09:02 sda 512.00 2048.00 9.20
After this patch with MADV_RANDOM (no readahead):
sar -d 1
Time DEV tps rkB/s %util
04:27:03 sda 40.00 5880.00 0.00
04:27:04 sda 41.00 6472.00 0.00
04:27:05 sda 37.00 4716.00 0.00
04:27:06 sda 48.00 8512.00 0.00
04:27:07 sda 33.00 4556.00 0.00
The patch reduces disk utilization from ~90% to ~6-10%, effectively
preventing memory.high-induced thrashing from overwhelming the IO
subsystem.
Signed-off-by: Jiayuan Chen <jiayuan.chen@shopee.com>
---
include/linux/memcontrol.h | 3 +++
mm/memcontrol.c | 41 +++++++++++++++++++++++++++++++++++---
mm/workingset.c | 8 ++++++++
3 files changed, 49 insertions(+), 3 deletions(-)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index fd400082313a..fc53de2485d6 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -321,6 +321,9 @@ struct mem_cgroup {
spinlock_t event_list_lock;
#endif /* CONFIG_MEMCG_V1 */
+ /* Timestamp of most recent refault, for thrashing detection */
+ u64 last_refault;
+
struct mem_cgroup_per_node *nodeinfo[];
};
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 75fc22a33b28..0dd42cce6926 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2226,6 +2226,38 @@ static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH;
}
+/*
+ * Check if a refault occurred recently, indicating active thrashing.
+ * Returns additional penalty jiffies based on refault recency.
+ *
+ * We use timestamp rather than refault counters because:
+ * 1. Counter aggregation is periodic and expensive to flush
+ * 2. Readahead makes counter-to-IO correlation unreliable
+ * 3. Timestamp gives us recency which directly reflects thrashing intensity
+ */
+static unsigned long calculate_refault(struct mem_cgroup *memcg)
+{
+ unsigned long last_refault = READ_ONCE(memcg->last_refault);
+ unsigned long now = jiffies;
+ long diff;
+
+ /*
+ * Only care about refaults within the last second. The closer
+ * the refault is to now, the higher the penalty:
+ *
+ * diff = 1 tick -> penalty = HZ (capped to HZ/10 = 100ms)
+ * diff = HZ/10 -> penalty = 10 ticks = 10ms
+ * diff = HZ/2 -> penalty = 2 ticks = 2ms
+ * diff >= HZ -> penalty = 0 (too old, not thrashing)
+ */
+ if (last_refault && time_before(now, last_refault + HZ)) {
+ diff = max((long)now - (long)last_refault, 1L);
+ /* Cap at 100ms to avoid excessive delays */
+ return min(HZ / diff, HZ / 10);
+ }
+ return 0;
+}
+
/*
* Reclaims memory over the high limit. Called directly from
* try_charge() (context permitting), as well as from the userland
@@ -2233,6 +2265,7 @@ static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
*/
void __mem_cgroup_handle_over_high(gfp_t gfp_mask)
{
+ unsigned long refault_penalty;
unsigned long penalty_jiffies;
unsigned long pflags;
unsigned long nr_reclaimed;
@@ -2279,12 +2312,14 @@ void __mem_cgroup_handle_over_high(gfp_t gfp_mask)
penalty_jiffies += calculate_high_delay(memcg, nr_pages,
swap_find_max_overage(memcg));
+ refault_penalty = calculate_refault(memcg);
+
/*
* Clamp the max delay per usermode return so as to still keep the
* application moving forwards and also permit diagnostics, albeit
* extremely slowly.
*/
- penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES);
+ penalty_jiffies = min(penalty_jiffies + refault_penalty, MEMCG_MAX_HIGH_DELAY_JIFFIES);
/*
* Don't sleep if the amount of jiffies this memcg owes us is so low
@@ -2292,7 +2327,7 @@ void __mem_cgroup_handle_over_high(gfp_t gfp_mask)
* go only a small amount over their memory.high value and maybe haven't
* been aggressively reclaimed enough yet.
*/
- if (penalty_jiffies <= HZ / 100)
+ if (!refault_penalty && penalty_jiffies <= HZ / 100)
goto out;
/*
@@ -2300,7 +2335,7 @@ void __mem_cgroup_handle_over_high(gfp_t gfp_mask)
* memory.high, we want to encourage that rather than doing allocator
* throttling.
*/
- if (nr_reclaimed || nr_retries--) {
+ if (!refault_penalty && (nr_reclaimed || nr_retries--)) {
in_retry = true;
goto retry_reclaim;
}
diff --git a/mm/workingset.c b/mm/workingset.c
index e9f05634747a..96e3c07e38ad 100644
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -290,6 +290,7 @@ static void lru_gen_refault(struct folio *folio, void *shadow)
struct lru_gen_folio *lrugen;
int type = folio_is_file_lru(folio);
int delta = folio_nr_pages(folio);
+ struct mem_cgroup *memcg;
rcu_read_lock();
@@ -297,6 +298,10 @@ static void lru_gen_refault(struct folio *folio, void *shadow)
if (lruvec != folio_lruvec(folio))
goto unlock;
+ memcg = folio_memcg(folio);
+ if (memcg)
+ WRITE_ONCE(memcg->last_refault, jiffies);
+
mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta);
if (!recent)
@@ -561,6 +566,9 @@ void workingset_refault(struct folio *folio, void *shadow)
pgdat = folio_pgdat(folio);
lruvec = mem_cgroup_lruvec(memcg, pgdat);
+ if (memcg)
+ WRITE_ONCE(memcg->last_refault, jiffies);
+
mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + file, nr);
if (!workingset_test_recent(shadow, file, &workingset, true))
--
2.43.0
Hi Jiayuan,
kernel test robot noticed the following build errors:
[auto build test ERROR on akpm-mm/mm-everything]
url: https://github.com/intel-lab-lkp/linux/commits/Jiayuan-Chen/mm-memcg-scale-memory-high-penalty-based-on-refault-recency/20251226-144331
base: https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-everything
patch link: https://lore.kernel.org/r/20251226064257.245581-1-jiayuan.chen%40linux.dev
patch subject: [PATCH v1] mm/memcg: scale memory.high penalty based on refault recency
config: x86_64-allnoconfig (https://download.01.org/0day-ci/archive/20251227/202512270405.sx7TY5MG-lkp@intel.com/config)
compiler: clang version 20.1.8 (https://github.com/llvm/llvm-project 87f0227cb60147a26a1eeb4fb06e3b505e9c7261)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20251227/202512270405.sx7TY5MG-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202512270405.sx7TY5MG-lkp@intel.com/
All errors (new ones prefixed by >>):
>> mm/workingset.c:570:19: error: incomplete definition of type 'struct mem_cgroup'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ~~~~~^
include/asm-generic/rwonce.h:60:33: note: expanded from macro 'WRITE_ONCE'
60 | compiletime_assert_rwonce_type(x); \
| ^
include/asm-generic/rwonce.h:36:35: note: expanded from macro 'compiletime_assert_rwonce_type'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^
include/linux/compiler_types.h:592:10: note: expanded from macro '__native_word'
592 | (sizeof(t) == sizeof(char) || sizeof(t) == sizeof(short) || \
| ^
include/linux/compiler_types.h:631:22: note: expanded from macro 'compiletime_assert'
631 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~
include/linux/compiler_types.h:619:23: note: expanded from macro '_compiletime_assert'
619 | __compiletime_assert(condition, msg, prefix, suffix)
| ^~~~~~~~~
include/linux/compiler_types.h:611:9: note: expanded from macro '__compiletime_assert'
611 | if (!(condition)) \
| ^~~~~~~~~
include/linux/shrinker.h:55:9: note: forward declaration of 'struct mem_cgroup'
55 | struct mem_cgroup *memcg;
| ^
>> mm/workingset.c:570:19: error: incomplete definition of type 'struct mem_cgroup'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ~~~~~^
include/asm-generic/rwonce.h:60:33: note: expanded from macro 'WRITE_ONCE'
60 | compiletime_assert_rwonce_type(x); \
| ^
include/asm-generic/rwonce.h:36:35: note: expanded from macro 'compiletime_assert_rwonce_type'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^
include/linux/compiler_types.h:592:39: note: expanded from macro '__native_word'
592 | (sizeof(t) == sizeof(char) || sizeof(t) == sizeof(short) || \
| ^
include/linux/compiler_types.h:631:22: note: expanded from macro 'compiletime_assert'
631 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~
include/linux/compiler_types.h:619:23: note: expanded from macro '_compiletime_assert'
619 | __compiletime_assert(condition, msg, prefix, suffix)
| ^~~~~~~~~
include/linux/compiler_types.h:611:9: note: expanded from macro '__compiletime_assert'
611 | if (!(condition)) \
| ^~~~~~~~~
include/linux/shrinker.h:55:9: note: forward declaration of 'struct mem_cgroup'
55 | struct mem_cgroup *memcg;
| ^
>> mm/workingset.c:570:19: error: incomplete definition of type 'struct mem_cgroup'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ~~~~~^
include/asm-generic/rwonce.h:60:33: note: expanded from macro 'WRITE_ONCE'
60 | compiletime_assert_rwonce_type(x); \
| ^
include/asm-generic/rwonce.h:36:35: note: expanded from macro 'compiletime_assert_rwonce_type'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^
include/linux/compiler_types.h:593:10: note: expanded from macro '__native_word'
593 | sizeof(t) == sizeof(int) || sizeof(t) == sizeof(long))
| ^
include/linux/compiler_types.h:631:22: note: expanded from macro 'compiletime_assert'
631 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~
include/linux/compiler_types.h:619:23: note: expanded from macro '_compiletime_assert'
619 | __compiletime_assert(condition, msg, prefix, suffix)
| ^~~~~~~~~
include/linux/compiler_types.h:611:9: note: expanded from macro '__compiletime_assert'
611 | if (!(condition)) \
| ^~~~~~~~~
include/linux/shrinker.h:55:9: note: forward declaration of 'struct mem_cgroup'
55 | struct mem_cgroup *memcg;
| ^
>> mm/workingset.c:570:19: error: incomplete definition of type 'struct mem_cgroup'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ~~~~~^
include/asm-generic/rwonce.h:60:33: note: expanded from macro 'WRITE_ONCE'
60 | compiletime_assert_rwonce_type(x); \
| ^
include/asm-generic/rwonce.h:36:35: note: expanded from macro 'compiletime_assert_rwonce_type'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^
include/linux/compiler_types.h:593:38: note: expanded from macro '__native_word'
593 | sizeof(t) == sizeof(int) || sizeof(t) == sizeof(long))
| ^
include/linux/compiler_types.h:631:22: note: expanded from macro 'compiletime_assert'
631 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~
include/linux/compiler_types.h:619:23: note: expanded from macro '_compiletime_assert'
619 | __compiletime_assert(condition, msg, prefix, suffix)
| ^~~~~~~~~
include/linux/compiler_types.h:611:9: note: expanded from macro '__compiletime_assert'
611 | if (!(condition)) \
| ^~~~~~~~~
include/linux/shrinker.h:55:9: note: forward declaration of 'struct mem_cgroup'
55 | struct mem_cgroup *memcg;
| ^
>> mm/workingset.c:570:19: error: incomplete definition of type 'struct mem_cgroup'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ~~~~~^
include/asm-generic/rwonce.h:60:33: note: expanded from macro 'WRITE_ONCE'
60 | compiletime_assert_rwonce_type(x); \
| ^
include/asm-generic/rwonce.h:36:48: note: expanded from macro 'compiletime_assert_rwonce_type'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^
include/linux/compiler_types.h:631:22: note: expanded from macro 'compiletime_assert'
631 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~
include/linux/compiler_types.h:619:23: note: expanded from macro '_compiletime_assert'
619 | __compiletime_assert(condition, msg, prefix, suffix)
| ^~~~~~~~~
include/linux/compiler_types.h:611:9: note: expanded from macro '__compiletime_assert'
611 | if (!(condition)) \
| ^~~~~~~~~
include/linux/shrinker.h:55:9: note: forward declaration of 'struct mem_cgroup'
55 | struct mem_cgroup *memcg;
| ^
>> mm/workingset.c:570:19: error: incomplete definition of type 'struct mem_cgroup'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ~~~~~^
include/asm-generic/rwonce.h:61:15: note: expanded from macro 'WRITE_ONCE'
61 | __WRITE_ONCE(x, val); \
| ^
include/asm-generic/rwonce.h:55:20: note: expanded from macro '__WRITE_ONCE'
55 | *(volatile typeof(x) *)&(x) = (val); \
| ^
include/linux/shrinker.h:55:9: note: forward declaration of 'struct mem_cgroup'
55 | struct mem_cgroup *memcg;
| ^
>> mm/workingset.c:570:19: error: incomplete definition of type 'struct mem_cgroup'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ~~~~~^
include/asm-generic/rwonce.h:61:15: note: expanded from macro 'WRITE_ONCE'
61 | __WRITE_ONCE(x, val); \
| ^
include/asm-generic/rwonce.h:55:27: note: expanded from macro '__WRITE_ONCE'
55 | *(volatile typeof(x) *)&(x) = (val); \
| ^
include/linux/shrinker.h:55:9: note: forward declaration of 'struct mem_cgroup'
55 | struct mem_cgroup *memcg;
| ^
7 errors generated.
vim +570 mm/workingset.c
529
530 /**
531 * workingset_refault - Evaluate the refault of a previously evicted folio.
532 * @folio: The freshly allocated replacement folio.
533 * @shadow: Shadow entry of the evicted folio.
534 *
535 * Calculates and evaluates the refault distance of the previously
536 * evicted folio in the context of the node and the memcg whose memory
537 * pressure caused the eviction.
538 */
539 void workingset_refault(struct folio *folio, void *shadow)
540 {
541 bool file = folio_is_file_lru(folio);
542 struct pglist_data *pgdat;
543 struct mem_cgroup *memcg;
544 struct lruvec *lruvec;
545 bool workingset;
546 long nr;
547
548 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
549
550 if (lru_gen_enabled()) {
551 lru_gen_refault(folio, shadow);
552 return;
553 }
554
555 /*
556 * The activation decision for this folio is made at the level
557 * where the eviction occurred, as that is where the LRU order
558 * during folio reclaim is being determined.
559 *
560 * However, the cgroup that will own the folio is the one that
561 * is actually experiencing the refault event. Make sure the folio is
562 * locked to guarantee folio_memcg() stability throughout.
563 */
564 nr = folio_nr_pages(folio);
565 memcg = folio_memcg(folio);
566 pgdat = folio_pgdat(folio);
567 lruvec = mem_cgroup_lruvec(memcg, pgdat);
568
569 if (memcg)
> 570 WRITE_ONCE(memcg->last_refault, jiffies);
571
572 mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + file, nr);
573
574 if (!workingset_test_recent(shadow, file, &workingset, true))
575 return;
576
577 folio_set_active(folio);
578 workingset_age_nonresident(lruvec, nr);
579 mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + file, nr);
580
581 /* Folio was active prior to eviction */
582 if (workingset) {
583 folio_set_workingset(folio);
584 /*
585 * XXX: Move to folio_add_lru() when it supports new vs
586 * putback
587 */
588 lru_note_cost_refault(folio);
589 mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + file, nr);
590 }
591 }
592
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
Hi Jiayuan,
kernel test robot noticed the following build errors:
[auto build test ERROR on akpm-mm/mm-everything]
url: https://github.com/intel-lab-lkp/linux/commits/Jiayuan-Chen/mm-memcg-scale-memory-high-penalty-based-on-refault-recency/20251226-144331
base: https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-everything
patch link: https://lore.kernel.org/r/20251226064257.245581-1-jiayuan.chen%40linux.dev
patch subject: [PATCH v1] mm/memcg: scale memory.high penalty based on refault recency
config: nios2-allnoconfig (https://download.01.org/0day-ci/archive/20251227/202512270457.MunhjmYM-lkp@intel.com/config)
compiler: nios2-linux-gcc (GCC) 11.5.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20251227/202512270457.MunhjmYM-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202512270457.MunhjmYM-lkp@intel.com/
All errors (new ones prefixed by >>):
In file included from <command-line>:
mm/workingset.c: In function 'workingset_refault':
>> mm/workingset.c:570:33: error: invalid use of undefined type 'struct mem_cgroup'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ^~
include/linux/compiler_types.h:611:23: note: in definition of macro '__compiletime_assert'
611 | if (!(condition)) \
| ^~~~~~~~~
include/linux/compiler_types.h:631:9: note: in expansion of macro '_compiletime_assert'
631 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:9: note: in expansion of macro 'compiletime_assert'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:28: note: in expansion of macro '__native_word'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~
include/asm-generic/rwonce.h:60:9: note: in expansion of macro 'compiletime_assert_rwonce_type'
60 | compiletime_assert_rwonce_type(x); \
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
mm/workingset.c:570:17: note: in expansion of macro 'WRITE_ONCE'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ^~~~~~~~~~
>> mm/workingset.c:570:33: error: invalid use of undefined type 'struct mem_cgroup'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ^~
include/linux/compiler_types.h:611:23: note: in definition of macro '__compiletime_assert'
611 | if (!(condition)) \
| ^~~~~~~~~
include/linux/compiler_types.h:631:9: note: in expansion of macro '_compiletime_assert'
631 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:9: note: in expansion of macro 'compiletime_assert'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:28: note: in expansion of macro '__native_word'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~
include/asm-generic/rwonce.h:60:9: note: in expansion of macro 'compiletime_assert_rwonce_type'
60 | compiletime_assert_rwonce_type(x); \
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
mm/workingset.c:570:17: note: in expansion of macro 'WRITE_ONCE'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ^~~~~~~~~~
>> mm/workingset.c:570:33: error: invalid use of undefined type 'struct mem_cgroup'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ^~
include/linux/compiler_types.h:611:23: note: in definition of macro '__compiletime_assert'
611 | if (!(condition)) \
| ^~~~~~~~~
include/linux/compiler_types.h:631:9: note: in expansion of macro '_compiletime_assert'
631 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:9: note: in expansion of macro 'compiletime_assert'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:28: note: in expansion of macro '__native_word'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~
include/asm-generic/rwonce.h:60:9: note: in expansion of macro 'compiletime_assert_rwonce_type'
60 | compiletime_assert_rwonce_type(x); \
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
mm/workingset.c:570:17: note: in expansion of macro 'WRITE_ONCE'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ^~~~~~~~~~
>> mm/workingset.c:570:33: error: invalid use of undefined type 'struct mem_cgroup'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ^~
include/linux/compiler_types.h:611:23: note: in definition of macro '__compiletime_assert'
611 | if (!(condition)) \
| ^~~~~~~~~
include/linux/compiler_types.h:631:9: note: in expansion of macro '_compiletime_assert'
631 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:9: note: in expansion of macro 'compiletime_assert'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:28: note: in expansion of macro '__native_word'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~
include/asm-generic/rwonce.h:60:9: note: in expansion of macro 'compiletime_assert_rwonce_type'
60 | compiletime_assert_rwonce_type(x); \
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
mm/workingset.c:570:17: note: in expansion of macro 'WRITE_ONCE'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ^~~~~~~~~~
>> mm/workingset.c:570:33: error: invalid use of undefined type 'struct mem_cgroup'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ^~
include/linux/compiler_types.h:611:23: note: in definition of macro '__compiletime_assert'
611 | if (!(condition)) \
| ^~~~~~~~~
include/linux/compiler_types.h:631:9: note: in expansion of macro '_compiletime_assert'
631 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:36:9: note: in expansion of macro 'compiletime_assert'
36 | compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \
| ^~~~~~~~~~~~~~~~~~
include/asm-generic/rwonce.h:60:9: note: in expansion of macro 'compiletime_assert_rwonce_type'
60 | compiletime_assert_rwonce_type(x); \
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
mm/workingset.c:570:17: note: in expansion of macro 'WRITE_ONCE'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ^~~~~~~~~~
In file included from ./arch/nios2/include/generated/asm/rwonce.h:1,
from include/linux/compiler.h:380,
from include/asm-generic/bug.h:5,
from ./arch/nios2/include/generated/asm/bug.h:1,
from include/linux/bug.h:5,
from include/linux/thread_info.h:13,
from include/asm-generic/current.h:6,
from ./arch/nios2/include/generated/asm/current.h:1,
from include/linux/sched.h:12,
from include/linux/cgroup.h:12,
from include/linux/memcontrol.h:13,
from mm/workingset.c:8:
>> mm/workingset.c:570:33: error: invalid use of undefined type 'struct mem_cgroup'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ^~
include/asm-generic/rwonce.h:55:27: note: in definition of macro '__WRITE_ONCE'
55 | *(volatile typeof(x) *)&(x) = (val); \
| ^
mm/workingset.c:570:17: note: in expansion of macro 'WRITE_ONCE'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ^~~~~~~~~~
>> mm/workingset.c:570:33: error: invalid use of undefined type 'struct mem_cgroup'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ^~
include/asm-generic/rwonce.h:55:34: note: in definition of macro '__WRITE_ONCE'
55 | *(volatile typeof(x) *)&(x) = (val); \
| ^
mm/workingset.c:570:17: note: in expansion of macro 'WRITE_ONCE'
570 | WRITE_ONCE(memcg->last_refault, jiffies);
| ^~~~~~~~~~
vim +570 mm/workingset.c
529
530 /**
531 * workingset_refault - Evaluate the refault of a previously evicted folio.
532 * @folio: The freshly allocated replacement folio.
533 * @shadow: Shadow entry of the evicted folio.
534 *
535 * Calculates and evaluates the refault distance of the previously
536 * evicted folio in the context of the node and the memcg whose memory
537 * pressure caused the eviction.
538 */
539 void workingset_refault(struct folio *folio, void *shadow)
540 {
541 bool file = folio_is_file_lru(folio);
542 struct pglist_data *pgdat;
543 struct mem_cgroup *memcg;
544 struct lruvec *lruvec;
545 bool workingset;
546 long nr;
547
548 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
549
550 if (lru_gen_enabled()) {
551 lru_gen_refault(folio, shadow);
552 return;
553 }
554
555 /*
556 * The activation decision for this folio is made at the level
557 * where the eviction occurred, as that is where the LRU order
558 * during folio reclaim is being determined.
559 *
560 * However, the cgroup that will own the folio is the one that
561 * is actually experiencing the refault event. Make sure the folio is
562 * locked to guarantee folio_memcg() stability throughout.
563 */
564 nr = folio_nr_pages(folio);
565 memcg = folio_memcg(folio);
566 pgdat = folio_pgdat(folio);
567 lruvec = mem_cgroup_lruvec(memcg, pgdat);
568
569 if (memcg)
> 570 WRITE_ONCE(memcg->last_refault, jiffies);
571
572 mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + file, nr);
573
574 if (!workingset_test_recent(shadow, file, &workingset, true))
575 return;
576
577 folio_set_active(folio);
578 workingset_age_nonresident(lruvec, nr);
579 mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + file, nr);
580
581 /* Folio was active prior to eviction */
582 if (workingset) {
583 folio_set_workingset(folio);
584 /*
585 * XXX: Move to folio_add_lru() when it supports new vs
586 * putback
587 */
588 lru_note_cost_refault(folio);
589 mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + file, nr);
590 }
591 }
592
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
© 2016 - 2026 Red Hat, Inc.