fs/exec.c | 1 +- include/linux/sched.h | 14 +++- kernel/exit.c | 33 +------- kernel/fork.c | 2 +- kernel/sched/fair.c | 196 ++++++++++++++++++++++++++++------------- 5 files changed, 154 insertions(+), 92 deletions(-)
The following commit has been merged into the sched/urgent branch of tip:
Commit-ID: b636fef85bda7d1bab9c0a45067ab1508d79d946
Gitweb: https://git.kernel.org/tip/b636fef85bda7d1bab9c0a45067ab1508d79d946
Author: Tim Chen <tim.c.chen@linux.intel.com>
AuthorDate: Mon, 21 Sep 2026 17:37:25 -07:00
Committer: Ingo Molnar <mingo@kernel.org>
CommitterDate: Tue, 22 Sep 2026 10:50:43 +02:00
sched/cache: Introduce task_struct->sched_cache_grp to fix UAF
Add a sched_cache_grp pointer to task_struct so that scheduler code
can access the cache group directly via the task, without going
through mm->sched_cache_grp. This decouples the scheduler's hot-path
accesses from the mm_struct.
Each task holds its own refcount on the sched_cache_group, separate
from the reference held by its mm_struct. The reference is acquired
in copy_mm() (fork) and exec_mmap() (exec), and released in exit_mm().
This fixes the use-after-free when account_mm_sched() reaches the group
through a task whose mm is being switched, as reported by Hyunwoo:
https://lore.kernel.org/lkml/apPb-Dr4nPYuHQOK@v4bel/
Convert all scheduler code in fair.c and exit.c to use
p->sched_cache_grp instead of p->mm->sched_cache_grp.
Keep the fork/exec/exit reference management out of the generic mm
paths: add sched_cache_fork(), sched_cache_fork_cleanup(),
sched_cache_exec_mmap() and sched_cache_exit_mm() in
kernel/sched/cache_sched.c (with empty stubs for !CONFIG_SCHED_CACHE),
so fs/exec.c, kernel/fork.c and kernel/exit.c each call one helper
instead of open-coding the refcounting under #ifdef. Also add
sched_cache_group_get() and task_cache_group_get().
Fixes: df0d98475954 ("sched/cache: Introduce infrastructure for cache-aware load balancing")
Closes: https://lore.kernel.org/lkml/apPb-Dr4nPYuHQOK@v4bel/
Closes: https://lore.kernel.org/all/343a7e07-7fad-4979-9c9b-82ec038c293c@linux.dev/
Reported-by: Hyunwoo Kim <imv4bel@gmail.com>
Reported-by: Zenghui Yu (Huawei) <zenghui.yu@linux.dev>
Co-developed-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: <stable@kernel.org> #7.2.x
Link: https://patch.msgid.link/ae7081dc54736bf115215f9867abb2711a7403fb.1790035273.git.tim.c.chen@linux.intel.com
---
fs/exec.c | 1 +-
include/linux/sched.h | 14 +++-
kernel/exit.c | 33 +-------
kernel/fork.c | 2 +-
kernel/sched/fair.c | 196 ++++++++++++++++++++++++++++-------------
5 files changed, 154 insertions(+), 92 deletions(-)
diff --git a/fs/exec.c b/fs/exec.c
index 8196434..a5269b5 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -882,6 +882,7 @@ static int exec_mmap(struct linux_binprm *bprm)
active_mm = tsk->active_mm;
tsk->active_mm = mm;
tsk->mm = mm;
+ sched_cache_exec_mmap(tsk, mm);
mm_init_cid(mm, tsk);
exec_state = task_exec_state_replace(tsk, exec_state);
/*
diff --git a/include/linux/sched.h b/include/linux/sched.h
index e14ad43..d35ae49 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1433,6 +1433,7 @@ struct task_struct {
#ifdef CONFIG_SCHED_CACHE
struct callback_head cache_work;
+ struct sched_cache_group __rcu *sched_cache_grp;
int preferred_llc;
/* 1: task was enqueued to its preferred LLC, 0 otherwise */
int pref_llc_queued;
@@ -2417,10 +2418,23 @@ struct sched_cache_group {
struct rcu_head rcu;
} ____cacheline_aligned_in_smp;
+struct sched_cache_group *sched_cache_group_get(struct sched_cache_group *grp);
+struct sched_cache_group *task_cache_group_get(struct task_struct *p);
+
+void sched_cache_fork(struct task_struct *p);
+void sched_cache_fork_cleanup(struct task_struct *p);
+void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm);
+void sched_cache_exit_mm(struct task_struct *p);
+
#else
struct sched_cache_group { };
+static inline void sched_cache_fork(struct task_struct *p) { }
+static inline void sched_cache_fork_cleanup(struct task_struct *p) { }
+static inline void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm) { }
+static inline void sched_cache_exit_mm(struct task_struct *p) { }
+
#endif
#ifndef MODULE
diff --git a/kernel/exit.c b/kernel/exit.c
index 024350e..282328d 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -551,37 +551,6 @@ void mm_update_next_owner(struct mm_struct *mm)
}
#endif /* CONFIG_MEMCG */
-#if defined(CONFIG_SCHED_CACHE) && defined(CONFIG_NUMA_BALANCING)
-/*
- * Subtract the memory footprint of the current task from
- * mm.
- */
-static void exit_mm_sched_cache(struct mm_struct *mm)
-{
- struct sched_cache_group *grp;
- unsigned long fp, sub;
-
- if (!current->total_numa_faults)
- return;
- /*
- * No lock protection due to performance considerations.
- * Make sure the group footprint does not become
- * negative.
- */
- grp = READ_ONCE(mm->sched_cache_grp);
- if (!grp)
- return;
-
- fp = READ_ONCE(grp->footprint);
- sub = min(fp, current->total_numa_faults);
- WRITE_ONCE(grp->footprint, fp - sub);
-}
-#else
-static inline void exit_mm_sched_cache(struct mm_struct *mm)
-{
-}
-#endif /* CONFIG_SCHED_CACHE CONFIG_NUMA_BALANCING */
-
/*
* Turn us into a lazy TLB process if we
* aren't already..
@@ -594,7 +563,7 @@ static void exit_mm(void)
if (!mm)
return;
- exit_mm_sched_cache(mm);
+ sched_cache_exit_mm(current);
mmap_read_lock(mm);
mmgrab_lazy_tlb(mm);
diff --git a/kernel/fork.c b/kernel/fork.c
index 5ef4133..10f2d05 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1599,6 +1599,7 @@ static int copy_mm(u64 clone_flags, struct task_struct *tsk)
tsk->mm = mm;
tsk->active_mm = mm;
+ sched_cache_fork(tsk);
return 0;
}
@@ -2602,6 +2603,7 @@ bad_fork_cleanup_io:
bad_fork_cleanup_namespaces:
exit_nsproxy_namespaces(p);
bad_fork_cleanup_mm:
+ sched_cache_fork_cleanup(p);
if (p->mm) {
mm_clear_owner(p->mm, p);
mmput(p->mm);
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index f0a9586..974a7df 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1478,7 +1478,7 @@ static inline int get_sched_cache_scale(int mul)
return (1 + (tol - 1) * mul);
}
-static bool exceed_llc_capacity(struct mm_struct *mm, int cpu)
+static bool exceed_llc_capacity(struct sched_cache_group *grp, int cpu)
{
#ifdef CONFIG_NUMA_BALANCING
unsigned long llc, footprint;
@@ -1492,11 +1492,6 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu)
return true;
if (static_branch_likely(&sched_numa_balancing)) {
- struct sched_cache_group *grp = READ_ONCE(mm->sched_cache_grp);
-
- if (!grp)
- return true;
-
/*
* TBD: RDT exclusive LLC ways reserved should be
* excluded.
@@ -1531,10 +1526,9 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu)
return false;
}
-static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p,
+static bool invalid_llc_nr(struct sched_cache_group *grp, struct task_struct *p,
int cpu)
{
- struct sched_cache_group *grp;
int scale;
if (get_nr_threads(p) <= 1)
@@ -1548,10 +1542,6 @@ static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p,
if (scale == INT_MAX)
return false;
- grp = READ_ONCE(mm->sched_cache_grp);
- if (!grp)
- return true;
-
return !fits_capacity((READ_ONCE(grp->nr_running_avg) * cpu_smt_num_threads),
(scale * per_cpu(sd_llc_size, cpu)));
}
@@ -1723,6 +1713,96 @@ static void sched_cache_group_put(struct sched_cache_group *grp)
call_rcu(&grp->rcu, sched_cache_group_free_rcu);
}
+DEFINE_FREE(sched_cache_group_put, struct sched_cache_group *,
+ sched_cache_group_put(_T));
+
+#define rcu_deref_sched_cache_grp(tsk) \
+ rcu_dereference_check((tsk)->sched_cache_grp, (tsk) == current)
+
+static struct sched_cache_group *sched_cache_replace_grp(struct task_struct *p,
+ struct sched_cache_group *new)
+{
+ struct sched_cache_group *old;
+
+ old = rcu_deref_sched_cache_grp(p);
+ rcu_assign_pointer(p->sched_cache_grp, new);
+
+ return old;
+}
+
+struct sched_cache_group *sched_cache_group_get(struct sched_cache_group *grp)
+{
+ /*
+ * refcount_inc_not_zero() is the acquire primitive for lockless
+ * (RCU) lookups; plain refcount_inc() would scribble the count if
+ * it already reached zero. Return NULL in that case.
+ */
+ if (grp && !refcount_inc_not_zero(&grp->refcnt))
+ grp = NULL;
+
+ return grp;
+}
+
+struct sched_cache_group *task_cache_group_get(struct task_struct *p)
+{
+ guard(rcu)();
+ return sched_cache_group_get(rcu_dereference(p->sched_cache_grp));
+}
+
+void sched_cache_fork(struct task_struct *p)
+{
+ /*
+ * The child takes its own reference on the mm's cache group, separate
+ * from the reference held by the mm. @p is not yet visible to readers,
+ * so a plain initializing store is enough.
+ */
+ RCU_INIT_POINTER(p->sched_cache_grp,
+ sched_cache_group_get(p->mm->sched_cache_grp));
+}
+
+void sched_cache_fork_cleanup(struct task_struct *p)
+{
+ /*
+ * A fork that fails after sched_cache_fork() never reaches exit_mm(),
+ * so drop the reference here. @p never became visible, so there are no
+ * concurrent readers and the reference we hold keeps the group alive.
+ */
+ sched_cache_group_put(rcu_access_pointer(p->sched_cache_grp));
+ RCU_INIT_POINTER(p->sched_cache_grp, NULL);
+}
+
+void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm)
+{
+ struct sched_cache_group *old;
+
+ /*
+ * Acquire the new reference before publishing the pointer, then drop
+ * the old one. @p is current and the only writer of its own pointer.
+ */
+ old = sched_cache_replace_grp(p, sched_cache_group_get(mm->sched_cache_grp));
+ sched_cache_group_put(old);
+}
+
+void sched_cache_exit_mm(struct task_struct *p)
+{
+ struct sched_cache_group *grp = sched_cache_replace_grp(p, NULL);
+
+#ifdef CONFIG_NUMA_BALANCING
+ /*
+ * Subtract this task's footprint from the group before dropping the
+ * reference, so the group footprint converges as its threads exit.
+ * Unlocked for performance; clamp to avoid underflow.
+ */
+ if (grp && p->total_numa_faults) {
+ unsigned long fp = READ_ONCE(grp->footprint);
+ unsigned long sub = min(fp, p->total_numa_faults);
+
+ WRITE_ONCE(grp->footprint, fp - sub);
+ }
+#endif
+ sched_cache_group_put(grp);
+}
+
void mm_destroy_sched(struct mm_struct *mm)
{
sched_cache_group_put(mm->sched_cache_grp);
@@ -1777,15 +1857,10 @@ static unsigned long fraction_mm_sched(struct rq *rq,
return div64_u64(NICE_0_LOAD * pcpu_sched->runtime, rq->cpu_runtime + 1);
}
-static int get_pref_llc(struct task_struct *p, struct mm_struct *mm)
+static int get_pref_llc(struct task_struct *p, struct sched_cache_group *grp)
{
int mm_sched_llc = -1, mm_sched_cpu;
- struct sched_cache_group *grp;
- if (!mm)
- return -1;
-
- grp = READ_ONCE(mm->sched_cache_grp);
if (!grp)
return -1;
@@ -1819,9 +1894,8 @@ static unsigned int task_running_on_cpu(int cpu, struct task_struct *p);
static inline
void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
{
+ struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp);
struct sched_cache_time *pcpu_sched;
- struct sched_cache_group *grp;
- struct mm_struct *mm = p->mm;
int mm_sched_llc = -1;
unsigned long epoch;
@@ -1832,12 +1906,8 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
return;
/*
* init_task, kthreads and user thread created
- * by user_mode_thread() don't have mm.
+ * by user_mode_thread() don't have a cache group.
*/
- if (!mm)
- return;
-
- grp = READ_ONCE(mm->sched_cache_grp);
if (!grp || !grp->pcpu_sched)
return;
@@ -1855,13 +1925,13 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
* its preferred state.
*/
if ((long)(epoch - READ_ONCE(grp->epoch)) > llc_epoch_affinity_timeout ||
- invalid_llc_nr(mm, p, cpu_of(rq)) ||
- exceed_llc_capacity(mm, cpu_of(rq))) {
+ invalid_llc_nr(grp, p, cpu_of(rq)) ||
+ exceed_llc_capacity(grp, cpu_of(rq))) {
if (READ_ONCE(grp->cpu) != -1)
WRITE_ONCE(grp->cpu, -1);
}
- mm_sched_llc = get_pref_llc(p, mm);
+ mm_sched_llc = get_pref_llc(p, grp);
/* task not on rq accounted later in account_entity_enqueue() */
if (task_running_on_cpu(rq->cpu, p) &&
@@ -1874,19 +1944,15 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
static void task_tick_cache(struct rq *rq, struct task_struct *p)
{
+ struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp);
struct callback_head *work = &p->cache_work;
- struct sched_cache_group *grp;
- struct mm_struct *mm = p->mm;
unsigned long epoch;
if (!sched_cache_enabled())
return;
- if (!mm || p->flags & PF_KTHREAD)
- return;
-
- grp = READ_ONCE(mm->sched_cache_grp);
- if (!grp || !grp->pcpu_sched)
+ if (!grp || p->flags & PF_KTHREAD ||
+ !grp->pcpu_sched)
return;
epoch = rq->cpu_epoch;
@@ -1968,14 +2034,13 @@ static inline void update_avg_scale(u64 *avg, u64 sample)
static void task_cache_work(struct callback_head *work)
{
+ struct sched_cache_group *grp __free(sched_cache_group_put) = NULL;
+ cpumask_var_t cpus __free(free_cpumask_var) = CPUMASK_VAR_NULL;
int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu;
unsigned long next_scan, now = jiffies;
struct task_struct *p = current, *cur;
unsigned long curr_m_a_occ = 0;
- struct sched_cache_group *grp;
- struct mm_struct *mm = p->mm;
unsigned long m_a_occ = 0;
- cpumask_var_t cpus;
WARN_ON_ONCE(work != &p->cache_work);
@@ -1984,7 +2049,12 @@ static void task_cache_work(struct callback_head *work)
if (p->flags & PF_EXITING)
return;
- grp = READ_ONCE(mm->sched_cache_grp);
+ /*
+ * A reference makes sure grp is not released by others. The rcu
+ * lock can not be held till after zalloc_cpumask_var() below,
+ * because the latter might sleep.
+ */
+ grp = task_cache_group_get(p);
if (!grp)
return;
@@ -1999,8 +2069,8 @@ static void task_cache_work(struct callback_head *work)
return;
curr_cpu = task_cpu(p);
- if (invalid_llc_nr(mm, p, curr_cpu) ||
- exceed_llc_capacity(mm, curr_cpu)) {
+ if (invalid_llc_nr(grp, p, curr_cpu) ||
+ exceed_llc_capacity(grp, curr_cpu)) {
if (READ_ONCE(grp->cpu) != -1)
WRITE_ONCE(grp->cpu, -1);
@@ -2033,9 +2103,13 @@ static void task_cache_work(struct callback_head *work)
m_cpu = i;
}
+ /*
+ * rcu_access_pointer() is used because the
+ * pointer is only compared, never dereferenced.
+ */
cur = rcu_dereference_all(cpu_rq(i)->curr);
if (cur && !(cur->flags & (PF_EXITING | PF_KTHREAD)) &&
- cur->mm == mm)
+ rcu_access_pointer(cur->sched_cache_grp) == grp)
nr_running++;
}
@@ -2081,7 +2155,6 @@ static void task_cache_work(struct callback_head *work)
}
update_avg_scale(&grp->nr_running_avg, nr_running);
- free_cpumask_var(cpus);
}
void init_sched_mm(struct task_struct *p)
@@ -2091,6 +2164,13 @@ void init_sched_mm(struct task_struct *p)
init_task_work(work, task_cache_work);
work->next = work;
/*
+ * dup_task_struct() copies the parent's task_struct, including its
+ * sched_cache_grp, for which the child holds no reference. Clear it
+ * here - before copy_mm() runs - so the child never carries a
+ * borrowed pointer that the fork error path would put.
+ */
+ RCU_INIT_POINTER(p->sched_cache_grp, NULL);
+ /*
* Reset new task's preference to avoid
* polluting account_llc_enqueue().
*/
@@ -3890,10 +3970,9 @@ static void task_numa_placement(struct task_struct *p)
* heuristic and occasional lost updates are tolerable.
*
* If a task exits, its corresponding footprint must
- * be subtracted from the mm->sched_cache_grp->footprint,
- * otherwise the mm->sched_cache_grp->footprint will not
- * converge: the exiting thread's footprint remains
- * unchanged/undecayed in mm->sched_cache_grp->footprint.
+ * be subtracted from p->sched_cache_grp->footprint,
+ * otherwise the footprint will not converge: the
+ * exiting thread's footprint remains unchanged/undecayed.
* See exit_mm().
*
* Lost updates and unsynchronized subtraction
@@ -3901,12 +3980,14 @@ static void task_numa_placement(struct task_struct *p)
* go negative. Clamp to zero to prevent the
* unsigned footprint from wrapping.
*/
- grp = READ_ONCE(p->mm->sched_cache_grp);
- if (!grp)
- continue;
+ scoped_guard(rcu) {
+ grp = rcu_dereference(p->sched_cache_grp);
- new_fp = (long)READ_ONCE(grp->footprint) + diff;
- WRITE_ONCE(grp->footprint, max(new_fp, 0L));
+ if (grp) {
+ new_fp = (long)READ_ONCE(grp->footprint) + diff;
+ WRITE_ONCE(grp->footprint, max(new_fp, 0L));
+ }
+ }
#endif
}
@@ -10855,7 +10936,6 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env,
struct task_struct *p)
{
struct sched_cache_group *grp;
- struct mm_struct *mm;
bool to_pref;
int cpu, src_cpu, dst_cpu;
@@ -10864,11 +10944,7 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env,
src_cpu = env->src_cpu;
dst_cpu = env->dst_cpu;
- mm = p->mm;
- if (!mm)
- return mig_unrestricted;
-
- grp = READ_ONCE(mm->sched_cache_grp);
+ grp = rcu_dereference_all(p->sched_cache_grp);
if (!grp)
return mig_unrestricted;
@@ -10877,8 +10953,8 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env,
return mig_unrestricted;
/* skip cache aware load balance for too many threads */
- if (invalid_llc_nr(mm, p, dst_cpu) ||
- exceed_llc_capacity(mm, dst_cpu)) {
+ if (invalid_llc_nr(grp, p, dst_cpu) ||
+ exceed_llc_capacity(grp, dst_cpu)) {
if (READ_ONCE(grp->cpu) != -1)
WRITE_ONCE(grp->cpu, -1);
return mig_unrestricted;© 2016 - 2026 Red Hat, Inc.