This is a follow up on the V4 series which can be found here:
https://lore.kernel.org/20251104075053.700034556@linutronix.de
The V1 cover letter contains a detailed analyisis of the issues:
https://lore.kernel.org/20251015164952.694882104@linutronix.de
TLDR: The CID management is way to complex and adds significant overhead
into scheduler hotpaths.
The series rewrites MM CID management in a more simplistic way which
focusses on low overhead in the scheduler while maintaining per task CIDs
as long as the number of threads is not exceeding the number of possible
CPUs.
The series is based on the V6 series of the rseq rewrite:
https://lore.kernel.org/20251027084220.785525188@linutronix.de
which is also available from git:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git core/rseq
The series on top of the tip core/rseq branch is available from git as
well:
git://git.kernel.org/pub/scm/linux/kernel/git/tglx/devel.git rseq/cid
Changes vs. V4:
- Zero mm_cid::transit on init - Mathieu
- Rephrase change logs - Mathiue
- Picked up Reviewed/Acked-by tags
Delta patch to V4 below.
Thanks,
tglx
---
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 93a2523908c4..72f368f7d6e2 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10848,6 +10848,7 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
{
mm->mm_cid.max_cids = 0;
mm->mm_cid.percpu = 0;
+ mm->mm_cid.transit = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
mm->mm_cid.pcpu_thrs = 0;The CID management is a complex beast, which affects both scheduling and
task migration. The compaction mechanism forces random tasks of a process
into task work on exit to user space causing latency spikes.
Revert back to the initial simple bitmap allocating mechanics, which are
known to have scalability issues as that allows to gradually build up a
replacement functionality in a reviewable way.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
V4: Remove barrier comment - Mathieu
---
include/linux/mm_types.h | 53 ----
kernel/fork.c | 5
kernel/sched/core.c | 517 +----------------------------------------------
kernel/sched/sched.h | 289 +++-----------------------
4 files changed, 64 insertions(+), 800 deletions(-)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -922,13 +922,9 @@ struct vm_area_struct {
#define vma_policy(vma) NULL
#endif
-#ifdef CONFIG_SCHED_MM_CID
struct mm_cid {
- u64 time;
- int cid;
- int recent_cid;
+ unsigned int cid;
};
-#endif
/*
* Opaque type representing current mm_struct flag state. Must be accessed via
@@ -1000,12 +996,6 @@ struct mm_struct {
* runqueue locks.
*/
struct mm_cid __percpu *pcpu_cid;
- /*
- * @mm_cid_next_scan: Next mm_cid scan (in jiffies).
- *
- * When the next mm_cid scan is due (in jiffies).
- */
- unsigned long mm_cid_next_scan;
/**
* @nr_cpus_allowed: Number of CPUs allowed for mm.
*
@@ -1014,14 +1004,6 @@ struct mm_struct {
*/
unsigned int nr_cpus_allowed;
/**
- * @max_nr_cid: Maximum number of allowed concurrency
- * IDs allocated.
- *
- * Track the highest number of allowed concurrency IDs
- * allocated for the mm.
- */
- atomic_t max_nr_cid;
- /**
* @cpus_allowed_lock: Lock protecting mm cpus_allowed.
*
* Provide mutual exclusion for mm cpus_allowed and
@@ -1371,35 +1353,7 @@ static inline void vma_iter_init(struct
#ifdef CONFIG_SCHED_MM_CID
-enum mm_cid_state {
- MM_CID_UNSET = -1U, /* Unset state has lazy_put flag set. */
- MM_CID_LAZY_PUT = (1U << 31),
-};
-
-static inline bool mm_cid_is_unset(int cid)
-{
- return cid == MM_CID_UNSET;
-}
-
-static inline bool mm_cid_is_lazy_put(int cid)
-{
- return !mm_cid_is_unset(cid) && (cid & MM_CID_LAZY_PUT);
-}
-
-static inline bool mm_cid_is_valid(int cid)
-{
- return !(cid & MM_CID_LAZY_PUT);
-}
-
-static inline int mm_cid_set_lazy_put(int cid)
-{
- return cid | MM_CID_LAZY_PUT;
-}
-
-static inline int mm_cid_clear_lazy_put(int cid)
-{
- return cid & ~MM_CID_LAZY_PUT;
-}
+#define MM_CID_UNSET (~0U)
/*
* mm_cpus_allowed: Union of all mm's threads allowed CPUs.
@@ -1432,11 +1386,8 @@ static inline void mm_init_cid(struct mm
struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i);
pcpu_cid->cid = MM_CID_UNSET;
- pcpu_cid->recent_cid = MM_CID_UNSET;
- pcpu_cid->time = 0;
}
mm->nr_cpus_allowed = p->nr_cpus_allowed;
- atomic_set(&mm->max_nr_cid, 0);
raw_spin_lock_init(&mm->cpus_allowed_lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
cpumask_clear(mm_cidmask(mm));
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -955,10 +955,9 @@ static struct task_struct *dup_task_stru
#endif
#ifdef CONFIG_SCHED_MM_CID
- tsk->mm_cid = -1;
- tsk->last_mm_cid = -1;
+ tsk->mm_cid = MM_CID_UNSET;
+ tsk->last_mm_cid = MM_CID_UNSET;
tsk->mm_cid_active = 0;
- tsk->migrate_from_cpu = -1;
#endif
return tsk;
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2128,8 +2128,6 @@ void activate_task(struct rq *rq, struct
{
if (task_on_rq_migrating(p))
flags |= ENQUEUE_MIGRATED;
- if (flags & ENQUEUE_MIGRATED)
- sched_mm_cid_migrate_to(rq, p);
enqueue_task(rq, p, flags);
@@ -3329,7 +3327,6 @@ void set_task_cpu(struct task_struct *p,
if (p->sched_class->migrate_task_rq)
p->sched_class->migrate_task_rq(p, new_cpu);
p->se.nr_migrations++;
- sched_mm_cid_migrate_from(p);
perf_event_task_migrate(p);
}
@@ -5280,9 +5277,6 @@ context_switch(struct rq *rq, struct tas
*
* kernel -> user switch + mmdrop_lazy_tlb() active
* user -> user switch
- *
- * switch_mm_cid() needs to be updated if the barriers provided
- * by context_switch() are modified.
*/
if (!next->mm) { // to kernel
enter_lazy_tlb(prev->active_mm, next);
@@ -5312,8 +5306,7 @@ context_switch(struct rq *rq, struct tas
}
}
- /* switch_mm_cid() requires the memory barriers above. */
- switch_mm_cid(rq, prev, next);
+ switch_mm_cid(prev, next);
/*
* Tell rseq that the task was scheduled in. Must be after
@@ -5604,7 +5597,6 @@ void sched_tick(void)
resched_latency = cpu_resched_latency(rq);
calc_global_load_tick(rq);
sched_core_tick(rq);
- task_tick_mm_cid(rq, donor);
scx_tick(rq);
rq_unlock(rq, &rf);
@@ -10376,522 +10368,47 @@ void call_trace_sched_update_nr_running(
}
#ifdef CONFIG_SCHED_MM_CID
-
-/*
- * @cid_lock: Guarantee forward-progress of cid allocation.
- *
- * Concurrency ID allocation within a bitmap is mostly lock-free. The cid_lock
- * is only used when contention is detected by the lock-free allocation so
- * forward progress can be guaranteed.
- */
-DEFINE_RAW_SPINLOCK(cid_lock);
-
-/*
- * @use_cid_lock: Select cid allocation behavior: lock-free vs spinlock.
- *
- * When @use_cid_lock is 0, the cid allocation is lock-free. When contention is
- * detected, it is set to 1 to ensure that all newly coming allocations are
- * serialized by @cid_lock until the allocation which detected contention
- * completes and sets @use_cid_lock back to 0. This guarantees forward progress
- * of a cid allocation.
- */
-int use_cid_lock;
-
-/*
- * mm_cid remote-clear implements a lock-free algorithm to clear per-mm/cpu cid
- * concurrently with respect to the execution of the source runqueue context
- * switch.
- *
- * There is one basic properties we want to guarantee here:
- *
- * (1) Remote-clear should _never_ mark a per-cpu cid UNSET when it is actively
- * used by a task. That would lead to concurrent allocation of the cid and
- * userspace corruption.
- *
- * Provide this guarantee by introducing a Dekker memory ordering to guarantee
- * that a pair of loads observe at least one of a pair of stores, which can be
- * shown as:
- *
- * X = Y = 0
- *
- * w[X]=1 w[Y]=1
- * MB MB
- * r[Y]=y r[X]=x
- *
- * Which guarantees that x==0 && y==0 is impossible. But rather than using
- * values 0 and 1, this algorithm cares about specific state transitions of the
- * runqueue current task (as updated by the scheduler context switch), and the
- * per-mm/cpu cid value.
- *
- * Let's introduce task (Y) which has task->mm == mm and task (N) which has
- * task->mm != mm for the rest of the discussion. There are two scheduler state
- * transitions on context switch we care about:
- *
- * (TSA) Store to rq->curr with transition from (N) to (Y)
- *
- * (TSB) Store to rq->curr with transition from (Y) to (N)
- *
- * On the remote-clear side, there is one transition we care about:
- *
- * (TMA) cmpxchg to *pcpu_cid to set the LAZY flag
- *
- * There is also a transition to UNSET state which can be performed from all
- * sides (scheduler, remote-clear). It is always performed with a cmpxchg which
- * guarantees that only a single thread will succeed:
- *
- * (TMB) cmpxchg to *pcpu_cid to mark UNSET
- *
- * Just to be clear, what we do _not_ want to happen is a transition to UNSET
- * when a thread is actively using the cid (property (1)).
- *
- * Let's looks at the relevant combinations of TSA/TSB, and TMA transitions.
- *
- * Scenario A) (TSA)+(TMA) (from next task perspective)
- *
- * CPU0 CPU1
- *
- * Context switch CS-1 Remote-clear
- * - store to rq->curr: (N)->(Y) (TSA) - cmpxchg to *pcpu_id to LAZY (TMA)
- * (implied barrier after cmpxchg)
- * - switch_mm_cid()
- * - memory barrier (see switch_mm_cid()
- * comment explaining how this barrier
- * is combined with other scheduler
- * barriers)
- * - mm_cid_get (next)
- * - READ_ONCE(*pcpu_cid) - rcu_dereference(src_rq->curr)
- *
- * This Dekker ensures that either task (Y) is observed by the
- * rcu_dereference() or the LAZY flag is observed by READ_ONCE(), or both are
- * observed.
- *
- * If task (Y) store is observed by rcu_dereference(), it means that there is
- * still an active task on the cpu. Remote-clear will therefore not transition
- * to UNSET, which fulfills property (1).
- *
- * If task (Y) is not observed, but the lazy flag is observed by READ_ONCE(),
- * it will move its state to UNSET, which clears the percpu cid perhaps
- * uselessly (which is not an issue for correctness). Because task (Y) is not
- * observed, CPU1 can move ahead to set the state to UNSET. Because moving
- * state to UNSET is done with a cmpxchg expecting that the old state has the
- * LAZY flag set, only one thread will successfully UNSET.
- *
- * If both states (LAZY flag and task (Y)) are observed, the thread on CPU0
- * will observe the LAZY flag and transition to UNSET (perhaps uselessly), and
- * CPU1 will observe task (Y) and do nothing more, which is fine.
- *
- * What we are effectively preventing with this Dekker is a scenario where
- * neither LAZY flag nor store (Y) are observed, which would fail property (1)
- * because this would UNSET a cid which is actively used.
- */
-
-void sched_mm_cid_migrate_from(struct task_struct *t)
-{
- t->migrate_from_cpu = task_cpu(t);
-}
-
-static
-int __sched_mm_cid_migrate_from_fetch_cid(struct rq *src_rq,
- struct task_struct *t,
- struct mm_cid *src_pcpu_cid)
-{
- struct mm_struct *mm = t->mm;
- struct task_struct *src_task;
- int src_cid, last_mm_cid;
-
- if (!mm)
- return -1;
-
- last_mm_cid = t->last_mm_cid;
- /*
- * If the migrated task has no last cid, or if the current
- * task on src rq uses the cid, it means the source cid does not need
- * to be moved to the destination cpu.
- */
- if (last_mm_cid == -1)
- return -1;
- src_cid = READ_ONCE(src_pcpu_cid->cid);
- if (!mm_cid_is_valid(src_cid) || last_mm_cid != src_cid)
- return -1;
-
- /*
- * If we observe an active task using the mm on this rq, it means we
- * are not the last task to be migrated from this cpu for this mm, so
- * there is no need to move src_cid to the destination cpu.
- */
- guard(rcu)();
- src_task = rcu_dereference(src_rq->curr);
- if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) {
- t->last_mm_cid = -1;
- return -1;
- }
-
- return src_cid;
-}
-
-static
-int __sched_mm_cid_migrate_from_try_steal_cid(struct rq *src_rq,
- struct task_struct *t,
- struct mm_cid *src_pcpu_cid,
- int src_cid)
-{
- struct task_struct *src_task;
- struct mm_struct *mm = t->mm;
- int lazy_cid;
-
- if (src_cid == -1)
- return -1;
-
- /*
- * Attempt to clear the source cpu cid to move it to the destination
- * cpu.
- */
- lazy_cid = mm_cid_set_lazy_put(src_cid);
- if (!try_cmpxchg(&src_pcpu_cid->cid, &src_cid, lazy_cid))
- return -1;
-
- /*
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm matches the scheduler barrier in context_switch()
- * between store to rq->curr and load of prev and next task's
- * per-mm/cpu cid.
- *
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm_cid_active matches the barrier in
- * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and
- * sched_mm_cid_after_execve() between store to t->mm_cid_active and
- * load of per-mm/cpu cid.
- */
-
- /*
- * If we observe an active task using the mm on this rq after setting
- * the lazy-put flag, this task will be responsible for transitioning
- * from lazy-put flag set to MM_CID_UNSET.
- */
- scoped_guard (rcu) {
- src_task = rcu_dereference(src_rq->curr);
- if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) {
- /*
- * We observed an active task for this mm, there is therefore
- * no point in moving this cid to the destination cpu.
- */
- t->last_mm_cid = -1;
- return -1;
- }
- }
-
- /*
- * The src_cid is unused, so it can be unset.
- */
- if (!try_cmpxchg(&src_pcpu_cid->cid, &lazy_cid, MM_CID_UNSET))
- return -1;
- WRITE_ONCE(src_pcpu_cid->recent_cid, MM_CID_UNSET);
- return src_cid;
-}
-
/*
- * Migration to dst cpu. Called with dst_rq lock held.
- * Interrupts are disabled, which keeps the window of cid ownership without the
- * source rq lock held small.
+ * When a task exits, the MM CID held by the task is not longer required as
+ * the task cannot return to user space.
*/
-void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t)
-{
- struct mm_cid *src_pcpu_cid, *dst_pcpu_cid;
- struct mm_struct *mm = t->mm;
- int src_cid, src_cpu;
- bool dst_cid_is_set;
- struct rq *src_rq;
-
- lockdep_assert_rq_held(dst_rq);
-
- if (!mm)
- return;
- src_cpu = t->migrate_from_cpu;
- if (src_cpu == -1) {
- t->last_mm_cid = -1;
- return;
- }
- /*
- * Move the src cid if the dst cid is unset. This keeps id
- * allocation closest to 0 in cases where few threads migrate around
- * many CPUs.
- *
- * If destination cid or recent cid is already set, we may have
- * to just clear the src cid to ensure compactness in frequent
- * migrations scenarios.
- *
- * It is not useful to clear the src cid when the number of threads is
- * greater or equal to the number of allowed CPUs, because user-space
- * can expect that the number of allowed cids can reach the number of
- * allowed CPUs.
- */
- dst_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(dst_rq));
- dst_cid_is_set = !mm_cid_is_unset(READ_ONCE(dst_pcpu_cid->cid)) ||
- !mm_cid_is_unset(READ_ONCE(dst_pcpu_cid->recent_cid));
- if (dst_cid_is_set && atomic_read(&mm->mm_users) >= READ_ONCE(mm->nr_cpus_allowed))
- return;
- src_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, src_cpu);
- src_rq = cpu_rq(src_cpu);
- src_cid = __sched_mm_cid_migrate_from_fetch_cid(src_rq, t, src_pcpu_cid);
- if (src_cid == -1)
- return;
- src_cid = __sched_mm_cid_migrate_from_try_steal_cid(src_rq, t, src_pcpu_cid,
- src_cid);
- if (src_cid == -1)
- return;
- if (dst_cid_is_set) {
- __mm_cid_put(mm, src_cid);
- return;
- }
- /* Move src_cid to dst cpu. */
- mm_cid_snapshot_time(dst_rq, mm);
- WRITE_ONCE(dst_pcpu_cid->cid, src_cid);
- WRITE_ONCE(dst_pcpu_cid->recent_cid, src_cid);
-}
-
-static void sched_mm_cid_remote_clear(struct mm_struct *mm, struct mm_cid *pcpu_cid,
- int cpu)
-{
- struct rq *rq = cpu_rq(cpu);
- struct task_struct *t;
- int cid, lazy_cid;
-
- cid = READ_ONCE(pcpu_cid->cid);
- if (!mm_cid_is_valid(cid))
- return;
-
- /*
- * Clear the cpu cid if it is set to keep cid allocation compact. If
- * there happens to be other tasks left on the source cpu using this
- * mm, the next task using this mm will reallocate its cid on context
- * switch.
- */
- lazy_cid = mm_cid_set_lazy_put(cid);
- if (!try_cmpxchg(&pcpu_cid->cid, &cid, lazy_cid))
- return;
-
- /*
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm matches the scheduler barrier in context_switch()
- * between store to rq->curr and load of prev and next task's
- * per-mm/cpu cid.
- *
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm_cid_active matches the barrier in
- * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and
- * sched_mm_cid_after_execve() between store to t->mm_cid_active and
- * load of per-mm/cpu cid.
- */
-
- /*
- * If we observe an active task using the mm on this rq after setting
- * the lazy-put flag, that task will be responsible for transitioning
- * from lazy-put flag set to MM_CID_UNSET.
- */
- scoped_guard (rcu) {
- t = rcu_dereference(rq->curr);
- if (READ_ONCE(t->mm_cid_active) && t->mm == mm)
- return;
- }
-
- /*
- * The cid is unused, so it can be unset.
- * Disable interrupts to keep the window of cid ownership without rq
- * lock small.
- */
- scoped_guard (irqsave) {
- if (try_cmpxchg(&pcpu_cid->cid, &lazy_cid, MM_CID_UNSET))
- __mm_cid_put(mm, cid);
- }
-}
-
-static void sched_mm_cid_remote_clear_old(struct mm_struct *mm, int cpu)
-{
- struct rq *rq = cpu_rq(cpu);
- struct mm_cid *pcpu_cid;
- struct task_struct *curr;
- u64 rq_clock;
-
- /*
- * rq->clock load is racy on 32-bit but one spurious clear once in a
- * while is irrelevant.
- */
- rq_clock = READ_ONCE(rq->clock);
- pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu);
-
- /*
- * In order to take care of infrequently scheduled tasks, bump the time
- * snapshot associated with this cid if an active task using the mm is
- * observed on this rq.
- */
- scoped_guard (rcu) {
- curr = rcu_dereference(rq->curr);
- if (READ_ONCE(curr->mm_cid_active) && curr->mm == mm) {
- WRITE_ONCE(pcpu_cid->time, rq_clock);
- return;
- }
- }
-
- if (rq_clock < pcpu_cid->time + SCHED_MM_CID_PERIOD_NS)
- return;
- sched_mm_cid_remote_clear(mm, pcpu_cid, cpu);
-}
-
-static void sched_mm_cid_remote_clear_weight(struct mm_struct *mm, int cpu,
- int weight)
-{
- struct mm_cid *pcpu_cid;
- int cid;
-
- pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu);
- cid = READ_ONCE(pcpu_cid->cid);
- if (!mm_cid_is_valid(cid) || cid < weight)
- return;
- sched_mm_cid_remote_clear(mm, pcpu_cid, cpu);
-}
-
-static void task_mm_cid_work(struct callback_head *work)
-{
- unsigned long now = jiffies, old_scan, next_scan;
- struct task_struct *t = current;
- struct cpumask *cidmask;
- struct mm_struct *mm;
- int weight, cpu;
-
- WARN_ON_ONCE(t != container_of(work, struct task_struct, cid_work));
-
- work->next = work; /* Prevent double-add */
- if (t->flags & PF_EXITING)
- return;
- mm = t->mm;
- if (!mm)
- return;
- old_scan = READ_ONCE(mm->mm_cid_next_scan);
- next_scan = now + msecs_to_jiffies(MM_CID_SCAN_DELAY);
- if (!old_scan) {
- unsigned long res;
-
- res = cmpxchg(&mm->mm_cid_next_scan, old_scan, next_scan);
- if (res != old_scan)
- old_scan = res;
- else
- old_scan = next_scan;
- }
- if (time_before(now, old_scan))
- return;
- if (!try_cmpxchg(&mm->mm_cid_next_scan, &old_scan, next_scan))
- return;
- cidmask = mm_cidmask(mm);
- /* Clear cids that were not recently used. */
- for_each_possible_cpu(cpu)
- sched_mm_cid_remote_clear_old(mm, cpu);
- weight = cpumask_weight(cidmask);
- /*
- * Clear cids that are greater or equal to the cidmask weight to
- * recompact it.
- */
- for_each_possible_cpu(cpu)
- sched_mm_cid_remote_clear_weight(mm, cpu, weight);
-}
-
-void init_sched_mm_cid(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- int mm_users = 0;
-
- if (mm) {
- mm_users = atomic_read(&mm->mm_users);
- if (mm_users == 1)
- mm->mm_cid_next_scan = jiffies + msecs_to_jiffies(MM_CID_SCAN_DELAY);
- }
- t->cid_work.next = &t->cid_work; /* Protect against double add */
- init_task_work(&t->cid_work, task_mm_cid_work);
-}
-
-void task_tick_mm_cid(struct rq *rq, struct task_struct *curr)
-{
- struct callback_head *work = &curr->cid_work;
- unsigned long now = jiffies;
-
- if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) ||
- work->next != work)
- return;
- if (time_before(now, READ_ONCE(curr->mm->mm_cid_next_scan)))
- return;
-
- /* No page allocation under rq lock */
- task_work_add(curr, work, TWA_RESUME);
-}
-
void sched_mm_cid_exit_signals(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
- struct rq *rq;
- if (!mm)
+ if (!mm || !t->mm_cid_active)
return;
- preempt_disable();
- rq = this_rq();
- guard(rq_lock_irqsave)(rq);
- preempt_enable_no_resched(); /* holding spinlock */
- WRITE_ONCE(t->mm_cid_active, 0);
- /*
- * Store t->mm_cid_active before loading per-mm/cpu cid.
- * Matches barrier in sched_mm_cid_remote_clear_old().
- */
- smp_mb();
- mm_cid_put(mm);
- t->last_mm_cid = t->mm_cid = -1;
+ guard(preempt)();
+ t->mm_cid_active = 0;
+ if (t->mm_cid != MM_CID_UNSET) {
+ cpumask_clear_cpu(t->mm_cid, mm_cidmask(mm));
+ t->mm_cid = MM_CID_UNSET;
+ }
}
+/* Deactivate MM CID allocation across execve() */
void sched_mm_cid_before_execve(struct task_struct *t)
{
- struct mm_struct *mm = t->mm;
- struct rq *rq;
-
- if (!mm)
- return;
-
- preempt_disable();
- rq = this_rq();
- guard(rq_lock_irqsave)(rq);
- preempt_enable_no_resched(); /* holding spinlock */
- WRITE_ONCE(t->mm_cid_active, 0);
- /*
- * Store t->mm_cid_active before loading per-mm/cpu cid.
- * Matches barrier in sched_mm_cid_remote_clear_old().
- */
- smp_mb();
- mm_cid_put(mm);
- t->last_mm_cid = t->mm_cid = -1;
+ sched_mm_cid_exit_signals(t);
}
+/* Reactivate MM CID after successful execve() */
void sched_mm_cid_after_execve(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
- struct rq *rq;
if (!mm)
return;
- preempt_disable();
- rq = this_rq();
- scoped_guard (rq_lock_irqsave, rq) {
- preempt_enable_no_resched(); /* holding spinlock */
- WRITE_ONCE(t->mm_cid_active, 1);
- /*
- * Store t->mm_cid_active before loading per-mm/cpu cid.
- * Matches barrier in sched_mm_cid_remote_clear_old().
- */
- smp_mb();
- t->last_mm_cid = t->mm_cid = mm_cid_get(rq, t, mm);
- }
+ guard(preempt)();
+ t->mm_cid_active = 1;
+ mm_cid_select(t);
}
void sched_mm_cid_fork(struct task_struct *t)
{
- WARN_ON_ONCE(!t->mm || t->mm_cid != -1);
+ WARN_ON_ONCE(!t->mm || t->mm_cid != MM_CID_UNSET);
t->mm_cid_active = 1;
}
#endif /* CONFIG_SCHED_MM_CID */
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3540,286 +3540,83 @@ extern void sched_dynamic_update(int mod
extern const char *preempt_modes[];
#ifdef CONFIG_SCHED_MM_CID
-
-#define SCHED_MM_CID_PERIOD_NS (100ULL * 1000000) /* 100ms */
-#define MM_CID_SCAN_DELAY 100 /* 100ms */
-
-extern raw_spinlock_t cid_lock;
-extern int use_cid_lock;
-
-extern void sched_mm_cid_migrate_from(struct task_struct *t);
-extern void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t);
-extern void task_tick_mm_cid(struct rq *rq, struct task_struct *curr);
-extern void init_sched_mm_cid(struct task_struct *t);
-
-static inline void __mm_cid_put(struct mm_struct *mm, int cid)
-{
- if (cid < 0)
- return;
- cpumask_clear_cpu(cid, mm_cidmask(mm));
-}
-
-/*
- * The per-mm/cpu cid can have the MM_CID_LAZY_PUT flag set or transition to
- * the MM_CID_UNSET state without holding the rq lock, but the rq lock needs to
- * be held to transition to other states.
- *
- * State transitions synchronized with cmpxchg or try_cmpxchg need to be
- * consistent across CPUs, which prevents use of this_cpu_cmpxchg.
- */
-static inline void mm_cid_put_lazy(struct task_struct *t)
+static inline void init_sched_mm_cid(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid;
+ unsigned int max_cid;
- lockdep_assert_irqs_disabled();
- cid = __this_cpu_read(pcpu_cid->cid);
- if (!mm_cid_is_lazy_put(cid) ||
- !try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET))
+ if (!mm)
return;
- __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
-}
-
-static inline int mm_cid_pcpu_unset(struct mm_struct *mm)
-{
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid, res;
- lockdep_assert_irqs_disabled();
- cid = __this_cpu_read(pcpu_cid->cid);
- for (;;) {
- if (mm_cid_is_unset(cid))
- return MM_CID_UNSET;
- /*
- * Attempt transition from valid or lazy-put to unset.
- */
- res = cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, cid, MM_CID_UNSET);
- if (res == cid)
- break;
- cid = res;
- }
- return cid;
+ /* Preset last_mm_cid */
+ max_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
+ t->last_mm_cid = max_cid - 1;
}
-static inline void mm_cid_put(struct mm_struct *mm)
+static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
{
- int cid;
+ struct mm_struct *mm = t->mm;
- lockdep_assert_irqs_disabled();
- cid = mm_cid_pcpu_unset(mm);
- if (cid == MM_CID_UNSET)
- return;
- __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
+ if (cid >= max_cids)
+ return false;
+ if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
+ return false;
+ t->mm_cid = t->last_mm_cid = cid;
+ __this_cpu_write(mm->pcpu_cid->cid, cid);
+ return true;
}
-static inline int __mm_cid_try_get(struct task_struct *t, struct mm_struct *mm)
+static inline bool mm_cid_get(struct task_struct *t)
{
- struct cpumask *cidmask = mm_cidmask(mm);
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid, max_nr_cid, allowed_max_nr_cid;
+ struct mm_struct *mm = t->mm;
+ unsigned int max_cids;
- /*
- * After shrinking the number of threads or reducing the number
- * of allowed cpus, reduce the value of max_nr_cid so expansion
- * of cid allocation will preserve cache locality if the number
- * of threads or allowed cpus increase again.
- */
- max_nr_cid = atomic_read(&mm->max_nr_cid);
- while ((allowed_max_nr_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed),
- atomic_read(&mm->mm_users))),
- max_nr_cid > allowed_max_nr_cid) {
- /* atomic_try_cmpxchg loads previous mm->max_nr_cid into max_nr_cid. */
- if (atomic_try_cmpxchg(&mm->max_nr_cid, &max_nr_cid, allowed_max_nr_cid)) {
- max_nr_cid = allowed_max_nr_cid;
- break;
- }
- }
- /* Try to re-use recent cid. This improves cache locality. */
- cid = __this_cpu_read(pcpu_cid->recent_cid);
- if (!mm_cid_is_unset(cid) && cid < max_nr_cid &&
- !cpumask_test_and_set_cpu(cid, cidmask))
- return cid;
- /*
- * Expand cid allocation if the maximum number of concurrency
- * IDs allocated (max_nr_cid) is below the number cpus allowed
- * and number of threads. Expanding cid allocation as much as
- * possible improves cache locality.
- */
- cid = max_nr_cid;
- while (cid < READ_ONCE(mm->nr_cpus_allowed) && cid < atomic_read(&mm->mm_users)) {
- /* atomic_try_cmpxchg loads previous mm->max_nr_cid into cid. */
- if (!atomic_try_cmpxchg(&mm->max_nr_cid, &cid, cid + 1))
- continue;
- if (!cpumask_test_and_set_cpu(cid, cidmask))
- return cid;
- }
- /*
- * Find the first available concurrency id.
- * Retry finding first zero bit if the mask is temporarily
- * filled. This only happens during concurrent remote-clear
- * which owns a cid without holding a rq lock.
- */
- for (;;) {
- cid = cpumask_first_zero(cidmask);
- if (cid < READ_ONCE(mm->nr_cpus_allowed))
- break;
- cpu_relax();
- }
- if (cpumask_test_and_set_cpu(cid, cidmask))
- return -1;
+ max_cids = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
- return cid;
-}
+ /* Try to reuse the last CID of this task */
+ if (__mm_cid_get(t, t->last_mm_cid, max_cids))
+ return true;
-/*
- * Save a snapshot of the current runqueue time of this cpu
- * with the per-cpu cid value, allowing to estimate how recently it was used.
- */
-static inline void mm_cid_snapshot_time(struct rq *rq, struct mm_struct *mm)
-{
- struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(rq));
+ /* Try to reuse the last CID of this mm on this CPU */
+ if (__mm_cid_get(t, __this_cpu_read(mm->pcpu_cid->cid), max_cids))
+ return true;
- lockdep_assert_rq_held(rq);
- WRITE_ONCE(pcpu_cid->time, rq->clock);
+ /* Try the first zero bit in the cidmask. */
+ return __mm_cid_get(t, cpumask_first_zero(mm_cidmask(mm)), max_cids);
}
-static inline int __mm_cid_get(struct rq *rq, struct task_struct *t,
- struct mm_struct *mm)
+static inline void mm_cid_select(struct task_struct *t)
{
- int cid;
-
- /*
- * All allocations (even those using the cid_lock) are lock-free. If
- * use_cid_lock is set, hold the cid_lock to perform cid allocation to
- * guarantee forward progress.
- */
- if (!READ_ONCE(use_cid_lock)) {
- cid = __mm_cid_try_get(t, mm);
- if (cid >= 0)
- goto end;
- raw_spin_lock(&cid_lock);
- } else {
- raw_spin_lock(&cid_lock);
- cid = __mm_cid_try_get(t, mm);
- if (cid >= 0)
- goto unlock;
- }
-
- /*
- * cid concurrently allocated. Retry while forcing following
- * allocations to use the cid_lock to ensure forward progress.
- */
- WRITE_ONCE(use_cid_lock, 1);
- /*
- * Set use_cid_lock before allocation. Only care about program order
- * because this is only required for forward progress.
- */
- barrier();
- /*
- * Retry until it succeeds. It is guaranteed to eventually succeed once
- * all newcoming allocations observe the use_cid_lock flag set.
- */
- do {
- cid = __mm_cid_try_get(t, mm);
- cpu_relax();
- } while (cid < 0);
/*
- * Allocate before clearing use_cid_lock. Only care about
- * program order because this is for forward progress.
+ * mm_cid_get() can fail when the maximum CID, which is determined
+ * by min(mm->nr_cpus_allowed, mm->mm_users) changes concurrently.
+ * That's a transient failure as there cannot be more tasks
+ * concurrently on a CPU (or about to be scheduled in) than that.
*/
- barrier();
- WRITE_ONCE(use_cid_lock, 0);
-unlock:
- raw_spin_unlock(&cid_lock);
-end:
- mm_cid_snapshot_time(rq, mm);
-
- return cid;
-}
-
-static inline int mm_cid_get(struct rq *rq, struct task_struct *t,
- struct mm_struct *mm)
-{
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid;
-
- lockdep_assert_rq_held(rq);
- cid = __this_cpu_read(pcpu_cid->cid);
- if (mm_cid_is_valid(cid)) {
- mm_cid_snapshot_time(rq, mm);
- return cid;
- }
- if (mm_cid_is_lazy_put(cid)) {
- if (try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET))
- __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
+ for (;;) {
+ if (mm_cid_get(t))
+ break;
}
- cid = __mm_cid_get(rq, t, mm);
- __this_cpu_write(pcpu_cid->cid, cid);
- __this_cpu_write(pcpu_cid->recent_cid, cid);
-
- return cid;
}
-static inline void switch_mm_cid(struct rq *rq,
- struct task_struct *prev,
- struct task_struct *next)
+static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
{
- /*
- * Provide a memory barrier between rq->curr store and load of
- * {prev,next}->mm->pcpu_cid[cpu] on rq->curr->mm transition.
- *
- * Should be adapted if context_switch() is modified.
- */
- if (!next->mm) { // to kernel
- /*
- * user -> kernel transition does not guarantee a barrier, but
- * we can use the fact that it performs an atomic operation in
- * mmgrab().
- */
- if (prev->mm) // from user
- smp_mb__after_mmgrab();
- /*
- * kernel -> kernel transition does not change rq->curr->mm
- * state. It stays NULL.
- */
- } else { // to user
- /*
- * kernel -> user transition does not provide a barrier
- * between rq->curr store and load of {prev,next}->mm->pcpu_cid[cpu].
- * Provide it here.
- */
- if (!prev->mm) { // from kernel
- smp_mb();
- } else { // from user
- /*
- * user->user transition relies on an implicit
- * memory barrier in switch_mm() when
- * current->mm changes. If the architecture
- * switch_mm() does not have an implicit memory
- * barrier, it is emitted here. If current->mm
- * is unchanged, no barrier is needed.
- */
- smp_mb__after_switch_mm();
- }
- }
if (prev->mm_cid_active) {
- mm_cid_snapshot_time(rq, prev->mm);
- mm_cid_put_lazy(prev);
- prev->mm_cid = -1;
+ if (prev->mm_cid != MM_CID_UNSET)
+ cpumask_clear_cpu(prev->mm_cid, mm_cidmask(prev->mm));
+ prev->mm_cid = MM_CID_UNSET;
}
+
if (next->mm_cid_active) {
- next->last_mm_cid = next->mm_cid = mm_cid_get(rq, next, next->mm);
+ mm_cid_select(next);
rseq_sched_set_task_mm_cid(next, next->mm_cid);
}
}
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void switch_mm_cid(struct rq *rq, struct task_struct *prev, struct task_struct *next) { }
-static inline void sched_mm_cid_migrate_from(struct task_struct *t) { }
-static inline void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) { }
-static inline void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) { }
static inline void init_sched_mm_cid(struct task_struct *t) { }
+static inline void mm_cid_select(struct task_struct *t) { }
+static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
extern u64 avg_vruntime(struct cfs_rq *cfs_rq);The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 77d7dc8bef482e987036bc204136bbda552d95cd
Gitweb: https://git.kernel.org/tip/77d7dc8bef482e987036bc204136bbda552d95cd
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:45 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:52 +01:00
sched/mmcid: Revert the complex CID management
The CID management is a complex beast, which affects both scheduling and
task migration. The compaction mechanism forces random tasks of a process
into task work on exit to user space causing latency spikes.
Revert back to the initial simple bitmap allocating mechanics, which are
known to have scalability issues as that allows to gradually build up a
replacement functionality in a reviewable way.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.068197830@linutronix.de
---
include/linux/mm_types.h | 53 +----
kernel/fork.c | 5 +-
kernel/sched/core.c | 517 +-------------------------------------
kernel/sched/sched.h | 289 +++------------------
4 files changed, 64 insertions(+), 800 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 90e5790..63b8c12 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -922,13 +922,9 @@ struct vm_area_struct {
#define vma_policy(vma) NULL
#endif
-#ifdef CONFIG_SCHED_MM_CID
struct mm_cid {
- u64 time;
- int cid;
- int recent_cid;
+ unsigned int cid;
};
-#endif
/*
* Opaque type representing current mm_struct flag state. Must be accessed via
@@ -1000,12 +996,6 @@ struct mm_struct {
* runqueue locks.
*/
struct mm_cid __percpu *pcpu_cid;
- /*
- * @mm_cid_next_scan: Next mm_cid scan (in jiffies).
- *
- * When the next mm_cid scan is due (in jiffies).
- */
- unsigned long mm_cid_next_scan;
/**
* @nr_cpus_allowed: Number of CPUs allowed for mm.
*
@@ -1014,14 +1004,6 @@ struct mm_struct {
*/
unsigned int nr_cpus_allowed;
/**
- * @max_nr_cid: Maximum number of allowed concurrency
- * IDs allocated.
- *
- * Track the highest number of allowed concurrency IDs
- * allocated for the mm.
- */
- atomic_t max_nr_cid;
- /**
* @cpus_allowed_lock: Lock protecting mm cpus_allowed.
*
* Provide mutual exclusion for mm cpus_allowed and
@@ -1371,35 +1353,7 @@ static inline void vma_iter_init(struct vma_iterator *vmi,
#ifdef CONFIG_SCHED_MM_CID
-enum mm_cid_state {
- MM_CID_UNSET = -1U, /* Unset state has lazy_put flag set. */
- MM_CID_LAZY_PUT = (1U << 31),
-};
-
-static inline bool mm_cid_is_unset(int cid)
-{
- return cid == MM_CID_UNSET;
-}
-
-static inline bool mm_cid_is_lazy_put(int cid)
-{
- return !mm_cid_is_unset(cid) && (cid & MM_CID_LAZY_PUT);
-}
-
-static inline bool mm_cid_is_valid(int cid)
-{
- return !(cid & MM_CID_LAZY_PUT);
-}
-
-static inline int mm_cid_set_lazy_put(int cid)
-{
- return cid | MM_CID_LAZY_PUT;
-}
-
-static inline int mm_cid_clear_lazy_put(int cid)
-{
- return cid & ~MM_CID_LAZY_PUT;
-}
+#define MM_CID_UNSET (~0U)
/*
* mm_cpus_allowed: Union of all mm's threads allowed CPUs.
@@ -1432,11 +1386,8 @@ static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i);
pcpu_cid->cid = MM_CID_UNSET;
- pcpu_cid->recent_cid = MM_CID_UNSET;
- pcpu_cid->time = 0;
}
mm->nr_cpus_allowed = p->nr_cpus_allowed;
- atomic_set(&mm->max_nr_cid, 0);
raw_spin_lock_init(&mm->cpus_allowed_lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
cpumask_clear(mm_cidmask(mm));
diff --git a/kernel/fork.c b/kernel/fork.c
index 3da0f08..9d9afe4 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -955,10 +955,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
#endif
#ifdef CONFIG_SCHED_MM_CID
- tsk->mm_cid = -1;
- tsk->last_mm_cid = -1;
+ tsk->mm_cid = MM_CID_UNSET;
+ tsk->last_mm_cid = MM_CID_UNSET;
tsk->mm_cid_active = 0;
- tsk->migrate_from_cpu = -1;
#endif
return tsk;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 579a8e9..11a1735 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2128,8 +2128,6 @@ void activate_task(struct rq *rq, struct task_struct *p, int flags)
{
if (task_on_rq_migrating(p))
flags |= ENQUEUE_MIGRATED;
- if (flags & ENQUEUE_MIGRATED)
- sched_mm_cid_migrate_to(rq, p);
enqueue_task(rq, p, flags);
@@ -3329,7 +3327,6 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
if (p->sched_class->migrate_task_rq)
p->sched_class->migrate_task_rq(p, new_cpu);
p->se.nr_migrations++;
- sched_mm_cid_migrate_from(p);
perf_event_task_migrate(p);
}
@@ -5280,9 +5277,6 @@ context_switch(struct rq *rq, struct task_struct *prev,
*
* kernel -> user switch + mmdrop_lazy_tlb() active
* user -> user switch
- *
- * switch_mm_cid() needs to be updated if the barriers provided
- * by context_switch() are modified.
*/
if (!next->mm) { // to kernel
enter_lazy_tlb(prev->active_mm, next);
@@ -5312,8 +5306,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
}
}
- /* switch_mm_cid() requires the memory barriers above. */
- switch_mm_cid(rq, prev, next);
+ switch_mm_cid(prev, next);
/*
* Tell rseq that the task was scheduled in. Must be after
@@ -5604,7 +5597,6 @@ void sched_tick(void)
resched_latency = cpu_resched_latency(rq);
calc_global_load_tick(rq);
sched_core_tick(rq);
- task_tick_mm_cid(rq, donor);
scx_tick(rq);
rq_unlock(rq, &rf);
@@ -10376,522 +10368,47 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
}
#ifdef CONFIG_SCHED_MM_CID
-
/*
- * @cid_lock: Guarantee forward-progress of cid allocation.
- *
- * Concurrency ID allocation within a bitmap is mostly lock-free. The cid_lock
- * is only used when contention is detected by the lock-free allocation so
- * forward progress can be guaranteed.
- */
-DEFINE_RAW_SPINLOCK(cid_lock);
-
-/*
- * @use_cid_lock: Select cid allocation behavior: lock-free vs spinlock.
- *
- * When @use_cid_lock is 0, the cid allocation is lock-free. When contention is
- * detected, it is set to 1 to ensure that all newly coming allocations are
- * serialized by @cid_lock until the allocation which detected contention
- * completes and sets @use_cid_lock back to 0. This guarantees forward progress
- * of a cid allocation.
- */
-int use_cid_lock;
-
-/*
- * mm_cid remote-clear implements a lock-free algorithm to clear per-mm/cpu cid
- * concurrently with respect to the execution of the source runqueue context
- * switch.
- *
- * There is one basic properties we want to guarantee here:
- *
- * (1) Remote-clear should _never_ mark a per-cpu cid UNSET when it is actively
- * used by a task. That would lead to concurrent allocation of the cid and
- * userspace corruption.
- *
- * Provide this guarantee by introducing a Dekker memory ordering to guarantee
- * that a pair of loads observe at least one of a pair of stores, which can be
- * shown as:
- *
- * X = Y = 0
- *
- * w[X]=1 w[Y]=1
- * MB MB
- * r[Y]=y r[X]=x
- *
- * Which guarantees that x==0 && y==0 is impossible. But rather than using
- * values 0 and 1, this algorithm cares about specific state transitions of the
- * runqueue current task (as updated by the scheduler context switch), and the
- * per-mm/cpu cid value.
- *
- * Let's introduce task (Y) which has task->mm == mm and task (N) which has
- * task->mm != mm for the rest of the discussion. There are two scheduler state
- * transitions on context switch we care about:
- *
- * (TSA) Store to rq->curr with transition from (N) to (Y)
- *
- * (TSB) Store to rq->curr with transition from (Y) to (N)
- *
- * On the remote-clear side, there is one transition we care about:
- *
- * (TMA) cmpxchg to *pcpu_cid to set the LAZY flag
- *
- * There is also a transition to UNSET state which can be performed from all
- * sides (scheduler, remote-clear). It is always performed with a cmpxchg which
- * guarantees that only a single thread will succeed:
- *
- * (TMB) cmpxchg to *pcpu_cid to mark UNSET
- *
- * Just to be clear, what we do _not_ want to happen is a transition to UNSET
- * when a thread is actively using the cid (property (1)).
- *
- * Let's looks at the relevant combinations of TSA/TSB, and TMA transitions.
- *
- * Scenario A) (TSA)+(TMA) (from next task perspective)
- *
- * CPU0 CPU1
- *
- * Context switch CS-1 Remote-clear
- * - store to rq->curr: (N)->(Y) (TSA) - cmpxchg to *pcpu_id to LAZY (TMA)
- * (implied barrier after cmpxchg)
- * - switch_mm_cid()
- * - memory barrier (see switch_mm_cid()
- * comment explaining how this barrier
- * is combined with other scheduler
- * barriers)
- * - mm_cid_get (next)
- * - READ_ONCE(*pcpu_cid) - rcu_dereference(src_rq->curr)
- *
- * This Dekker ensures that either task (Y) is observed by the
- * rcu_dereference() or the LAZY flag is observed by READ_ONCE(), or both are
- * observed.
- *
- * If task (Y) store is observed by rcu_dereference(), it means that there is
- * still an active task on the cpu. Remote-clear will therefore not transition
- * to UNSET, which fulfills property (1).
- *
- * If task (Y) is not observed, but the lazy flag is observed by READ_ONCE(),
- * it will move its state to UNSET, which clears the percpu cid perhaps
- * uselessly (which is not an issue for correctness). Because task (Y) is not
- * observed, CPU1 can move ahead to set the state to UNSET. Because moving
- * state to UNSET is done with a cmpxchg expecting that the old state has the
- * LAZY flag set, only one thread will successfully UNSET.
- *
- * If both states (LAZY flag and task (Y)) are observed, the thread on CPU0
- * will observe the LAZY flag and transition to UNSET (perhaps uselessly), and
- * CPU1 will observe task (Y) and do nothing more, which is fine.
- *
- * What we are effectively preventing with this Dekker is a scenario where
- * neither LAZY flag nor store (Y) are observed, which would fail property (1)
- * because this would UNSET a cid which is actively used.
+ * When a task exits, the MM CID held by the task is not longer required as
+ * the task cannot return to user space.
*/
-
-void sched_mm_cid_migrate_from(struct task_struct *t)
-{
- t->migrate_from_cpu = task_cpu(t);
-}
-
-static
-int __sched_mm_cid_migrate_from_fetch_cid(struct rq *src_rq,
- struct task_struct *t,
- struct mm_cid *src_pcpu_cid)
-{
- struct mm_struct *mm = t->mm;
- struct task_struct *src_task;
- int src_cid, last_mm_cid;
-
- if (!mm)
- return -1;
-
- last_mm_cid = t->last_mm_cid;
- /*
- * If the migrated task has no last cid, or if the current
- * task on src rq uses the cid, it means the source cid does not need
- * to be moved to the destination cpu.
- */
- if (last_mm_cid == -1)
- return -1;
- src_cid = READ_ONCE(src_pcpu_cid->cid);
- if (!mm_cid_is_valid(src_cid) || last_mm_cid != src_cid)
- return -1;
-
- /*
- * If we observe an active task using the mm on this rq, it means we
- * are not the last task to be migrated from this cpu for this mm, so
- * there is no need to move src_cid to the destination cpu.
- */
- guard(rcu)();
- src_task = rcu_dereference(src_rq->curr);
- if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) {
- t->last_mm_cid = -1;
- return -1;
- }
-
- return src_cid;
-}
-
-static
-int __sched_mm_cid_migrate_from_try_steal_cid(struct rq *src_rq,
- struct task_struct *t,
- struct mm_cid *src_pcpu_cid,
- int src_cid)
-{
- struct task_struct *src_task;
- struct mm_struct *mm = t->mm;
- int lazy_cid;
-
- if (src_cid == -1)
- return -1;
-
- /*
- * Attempt to clear the source cpu cid to move it to the destination
- * cpu.
- */
- lazy_cid = mm_cid_set_lazy_put(src_cid);
- if (!try_cmpxchg(&src_pcpu_cid->cid, &src_cid, lazy_cid))
- return -1;
-
- /*
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm matches the scheduler barrier in context_switch()
- * between store to rq->curr and load of prev and next task's
- * per-mm/cpu cid.
- *
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm_cid_active matches the barrier in
- * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and
- * sched_mm_cid_after_execve() between store to t->mm_cid_active and
- * load of per-mm/cpu cid.
- */
-
- /*
- * If we observe an active task using the mm on this rq after setting
- * the lazy-put flag, this task will be responsible for transitioning
- * from lazy-put flag set to MM_CID_UNSET.
- */
- scoped_guard (rcu) {
- src_task = rcu_dereference(src_rq->curr);
- if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) {
- /*
- * We observed an active task for this mm, there is therefore
- * no point in moving this cid to the destination cpu.
- */
- t->last_mm_cid = -1;
- return -1;
- }
- }
-
- /*
- * The src_cid is unused, so it can be unset.
- */
- if (!try_cmpxchg(&src_pcpu_cid->cid, &lazy_cid, MM_CID_UNSET))
- return -1;
- WRITE_ONCE(src_pcpu_cid->recent_cid, MM_CID_UNSET);
- return src_cid;
-}
-
-/*
- * Migration to dst cpu. Called with dst_rq lock held.
- * Interrupts are disabled, which keeps the window of cid ownership without the
- * source rq lock held small.
- */
-void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t)
-{
- struct mm_cid *src_pcpu_cid, *dst_pcpu_cid;
- struct mm_struct *mm = t->mm;
- int src_cid, src_cpu;
- bool dst_cid_is_set;
- struct rq *src_rq;
-
- lockdep_assert_rq_held(dst_rq);
-
- if (!mm)
- return;
- src_cpu = t->migrate_from_cpu;
- if (src_cpu == -1) {
- t->last_mm_cid = -1;
- return;
- }
- /*
- * Move the src cid if the dst cid is unset. This keeps id
- * allocation closest to 0 in cases where few threads migrate around
- * many CPUs.
- *
- * If destination cid or recent cid is already set, we may have
- * to just clear the src cid to ensure compactness in frequent
- * migrations scenarios.
- *
- * It is not useful to clear the src cid when the number of threads is
- * greater or equal to the number of allowed CPUs, because user-space
- * can expect that the number of allowed cids can reach the number of
- * allowed CPUs.
- */
- dst_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(dst_rq));
- dst_cid_is_set = !mm_cid_is_unset(READ_ONCE(dst_pcpu_cid->cid)) ||
- !mm_cid_is_unset(READ_ONCE(dst_pcpu_cid->recent_cid));
- if (dst_cid_is_set && atomic_read(&mm->mm_users) >= READ_ONCE(mm->nr_cpus_allowed))
- return;
- src_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, src_cpu);
- src_rq = cpu_rq(src_cpu);
- src_cid = __sched_mm_cid_migrate_from_fetch_cid(src_rq, t, src_pcpu_cid);
- if (src_cid == -1)
- return;
- src_cid = __sched_mm_cid_migrate_from_try_steal_cid(src_rq, t, src_pcpu_cid,
- src_cid);
- if (src_cid == -1)
- return;
- if (dst_cid_is_set) {
- __mm_cid_put(mm, src_cid);
- return;
- }
- /* Move src_cid to dst cpu. */
- mm_cid_snapshot_time(dst_rq, mm);
- WRITE_ONCE(dst_pcpu_cid->cid, src_cid);
- WRITE_ONCE(dst_pcpu_cid->recent_cid, src_cid);
-}
-
-static void sched_mm_cid_remote_clear(struct mm_struct *mm, struct mm_cid *pcpu_cid,
- int cpu)
-{
- struct rq *rq = cpu_rq(cpu);
- struct task_struct *t;
- int cid, lazy_cid;
-
- cid = READ_ONCE(pcpu_cid->cid);
- if (!mm_cid_is_valid(cid))
- return;
-
- /*
- * Clear the cpu cid if it is set to keep cid allocation compact. If
- * there happens to be other tasks left on the source cpu using this
- * mm, the next task using this mm will reallocate its cid on context
- * switch.
- */
- lazy_cid = mm_cid_set_lazy_put(cid);
- if (!try_cmpxchg(&pcpu_cid->cid, &cid, lazy_cid))
- return;
-
- /*
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm matches the scheduler barrier in context_switch()
- * between store to rq->curr and load of prev and next task's
- * per-mm/cpu cid.
- *
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm_cid_active matches the barrier in
- * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and
- * sched_mm_cid_after_execve() between store to t->mm_cid_active and
- * load of per-mm/cpu cid.
- */
-
- /*
- * If we observe an active task using the mm on this rq after setting
- * the lazy-put flag, that task will be responsible for transitioning
- * from lazy-put flag set to MM_CID_UNSET.
- */
- scoped_guard (rcu) {
- t = rcu_dereference(rq->curr);
- if (READ_ONCE(t->mm_cid_active) && t->mm == mm)
- return;
- }
-
- /*
- * The cid is unused, so it can be unset.
- * Disable interrupts to keep the window of cid ownership without rq
- * lock small.
- */
- scoped_guard (irqsave) {
- if (try_cmpxchg(&pcpu_cid->cid, &lazy_cid, MM_CID_UNSET))
- __mm_cid_put(mm, cid);
- }
-}
-
-static void sched_mm_cid_remote_clear_old(struct mm_struct *mm, int cpu)
-{
- struct rq *rq = cpu_rq(cpu);
- struct mm_cid *pcpu_cid;
- struct task_struct *curr;
- u64 rq_clock;
-
- /*
- * rq->clock load is racy on 32-bit but one spurious clear once in a
- * while is irrelevant.
- */
- rq_clock = READ_ONCE(rq->clock);
- pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu);
-
- /*
- * In order to take care of infrequently scheduled tasks, bump the time
- * snapshot associated with this cid if an active task using the mm is
- * observed on this rq.
- */
- scoped_guard (rcu) {
- curr = rcu_dereference(rq->curr);
- if (READ_ONCE(curr->mm_cid_active) && curr->mm == mm) {
- WRITE_ONCE(pcpu_cid->time, rq_clock);
- return;
- }
- }
-
- if (rq_clock < pcpu_cid->time + SCHED_MM_CID_PERIOD_NS)
- return;
- sched_mm_cid_remote_clear(mm, pcpu_cid, cpu);
-}
-
-static void sched_mm_cid_remote_clear_weight(struct mm_struct *mm, int cpu,
- int weight)
-{
- struct mm_cid *pcpu_cid;
- int cid;
-
- pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu);
- cid = READ_ONCE(pcpu_cid->cid);
- if (!mm_cid_is_valid(cid) || cid < weight)
- return;
- sched_mm_cid_remote_clear(mm, pcpu_cid, cpu);
-}
-
-static void task_mm_cid_work(struct callback_head *work)
-{
- unsigned long now = jiffies, old_scan, next_scan;
- struct task_struct *t = current;
- struct cpumask *cidmask;
- struct mm_struct *mm;
- int weight, cpu;
-
- WARN_ON_ONCE(t != container_of(work, struct task_struct, cid_work));
-
- work->next = work; /* Prevent double-add */
- if (t->flags & PF_EXITING)
- return;
- mm = t->mm;
- if (!mm)
- return;
- old_scan = READ_ONCE(mm->mm_cid_next_scan);
- next_scan = now + msecs_to_jiffies(MM_CID_SCAN_DELAY);
- if (!old_scan) {
- unsigned long res;
-
- res = cmpxchg(&mm->mm_cid_next_scan, old_scan, next_scan);
- if (res != old_scan)
- old_scan = res;
- else
- old_scan = next_scan;
- }
- if (time_before(now, old_scan))
- return;
- if (!try_cmpxchg(&mm->mm_cid_next_scan, &old_scan, next_scan))
- return;
- cidmask = mm_cidmask(mm);
- /* Clear cids that were not recently used. */
- for_each_possible_cpu(cpu)
- sched_mm_cid_remote_clear_old(mm, cpu);
- weight = cpumask_weight(cidmask);
- /*
- * Clear cids that are greater or equal to the cidmask weight to
- * recompact it.
- */
- for_each_possible_cpu(cpu)
- sched_mm_cid_remote_clear_weight(mm, cpu, weight);
-}
-
-void init_sched_mm_cid(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- int mm_users = 0;
-
- if (mm) {
- mm_users = atomic_read(&mm->mm_users);
- if (mm_users == 1)
- mm->mm_cid_next_scan = jiffies + msecs_to_jiffies(MM_CID_SCAN_DELAY);
- }
- t->cid_work.next = &t->cid_work; /* Protect against double add */
- init_task_work(&t->cid_work, task_mm_cid_work);
-}
-
-void task_tick_mm_cid(struct rq *rq, struct task_struct *curr)
-{
- struct callback_head *work = &curr->cid_work;
- unsigned long now = jiffies;
-
- if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) ||
- work->next != work)
- return;
- if (time_before(now, READ_ONCE(curr->mm->mm_cid_next_scan)))
- return;
-
- /* No page allocation under rq lock */
- task_work_add(curr, work, TWA_RESUME);
-}
-
void sched_mm_cid_exit_signals(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
- struct rq *rq;
- if (!mm)
+ if (!mm || !t->mm_cid_active)
return;
- preempt_disable();
- rq = this_rq();
- guard(rq_lock_irqsave)(rq);
- preempt_enable_no_resched(); /* holding spinlock */
- WRITE_ONCE(t->mm_cid_active, 0);
- /*
- * Store t->mm_cid_active before loading per-mm/cpu cid.
- * Matches barrier in sched_mm_cid_remote_clear_old().
- */
- smp_mb();
- mm_cid_put(mm);
- t->last_mm_cid = t->mm_cid = -1;
+ guard(preempt)();
+ t->mm_cid_active = 0;
+ if (t->mm_cid != MM_CID_UNSET) {
+ cpumask_clear_cpu(t->mm_cid, mm_cidmask(mm));
+ t->mm_cid = MM_CID_UNSET;
+ }
}
+/* Deactivate MM CID allocation across execve() */
void sched_mm_cid_before_execve(struct task_struct *t)
{
- struct mm_struct *mm = t->mm;
- struct rq *rq;
-
- if (!mm)
- return;
-
- preempt_disable();
- rq = this_rq();
- guard(rq_lock_irqsave)(rq);
- preempt_enable_no_resched(); /* holding spinlock */
- WRITE_ONCE(t->mm_cid_active, 0);
- /*
- * Store t->mm_cid_active before loading per-mm/cpu cid.
- * Matches barrier in sched_mm_cid_remote_clear_old().
- */
- smp_mb();
- mm_cid_put(mm);
- t->last_mm_cid = t->mm_cid = -1;
+ sched_mm_cid_exit_signals(t);
}
+/* Reactivate MM CID after successful execve() */
void sched_mm_cid_after_execve(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
- struct rq *rq;
if (!mm)
return;
- preempt_disable();
- rq = this_rq();
- scoped_guard (rq_lock_irqsave, rq) {
- preempt_enable_no_resched(); /* holding spinlock */
- WRITE_ONCE(t->mm_cid_active, 1);
- /*
- * Store t->mm_cid_active before loading per-mm/cpu cid.
- * Matches barrier in sched_mm_cid_remote_clear_old().
- */
- smp_mb();
- t->last_mm_cid = t->mm_cid = mm_cid_get(rq, t, mm);
- }
+ guard(preempt)();
+ t->mm_cid_active = 1;
+ mm_cid_select(t);
}
void sched_mm_cid_fork(struct task_struct *t)
{
- WARN_ON_ONCE(!t->mm || t->mm_cid != -1);
+ WARN_ON_ONCE(!t->mm || t->mm_cid != MM_CID_UNSET);
t->mm_cid_active = 1;
}
#endif /* CONFIG_SCHED_MM_CID */
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 4838dda..bf227c2 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3540,286 +3540,83 @@ extern void sched_dynamic_update(int mode);
extern const char *preempt_modes[];
#ifdef CONFIG_SCHED_MM_CID
-
-#define SCHED_MM_CID_PERIOD_NS (100ULL * 1000000) /* 100ms */
-#define MM_CID_SCAN_DELAY 100 /* 100ms */
-
-extern raw_spinlock_t cid_lock;
-extern int use_cid_lock;
-
-extern void sched_mm_cid_migrate_from(struct task_struct *t);
-extern void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t);
-extern void task_tick_mm_cid(struct rq *rq, struct task_struct *curr);
-extern void init_sched_mm_cid(struct task_struct *t);
-
-static inline void __mm_cid_put(struct mm_struct *mm, int cid)
-{
- if (cid < 0)
- return;
- cpumask_clear_cpu(cid, mm_cidmask(mm));
-}
-
-/*
- * The per-mm/cpu cid can have the MM_CID_LAZY_PUT flag set or transition to
- * the MM_CID_UNSET state without holding the rq lock, but the rq lock needs to
- * be held to transition to other states.
- *
- * State transitions synchronized with cmpxchg or try_cmpxchg need to be
- * consistent across CPUs, which prevents use of this_cpu_cmpxchg.
- */
-static inline void mm_cid_put_lazy(struct task_struct *t)
+static inline void init_sched_mm_cid(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid;
+ unsigned int max_cid;
- lockdep_assert_irqs_disabled();
- cid = __this_cpu_read(pcpu_cid->cid);
- if (!mm_cid_is_lazy_put(cid) ||
- !try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET))
+ if (!mm)
return;
- __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
-}
-static inline int mm_cid_pcpu_unset(struct mm_struct *mm)
-{
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid, res;
-
- lockdep_assert_irqs_disabled();
- cid = __this_cpu_read(pcpu_cid->cid);
- for (;;) {
- if (mm_cid_is_unset(cid))
- return MM_CID_UNSET;
- /*
- * Attempt transition from valid or lazy-put to unset.
- */
- res = cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, cid, MM_CID_UNSET);
- if (res == cid)
- break;
- cid = res;
- }
- return cid;
+ /* Preset last_mm_cid */
+ max_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
+ t->last_mm_cid = max_cid - 1;
}
-static inline void mm_cid_put(struct mm_struct *mm)
+static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
{
- int cid;
+ struct mm_struct *mm = t->mm;
- lockdep_assert_irqs_disabled();
- cid = mm_cid_pcpu_unset(mm);
- if (cid == MM_CID_UNSET)
- return;
- __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
+ if (cid >= max_cids)
+ return false;
+ if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
+ return false;
+ t->mm_cid = t->last_mm_cid = cid;
+ __this_cpu_write(mm->pcpu_cid->cid, cid);
+ return true;
}
-static inline int __mm_cid_try_get(struct task_struct *t, struct mm_struct *mm)
+static inline bool mm_cid_get(struct task_struct *t)
{
- struct cpumask *cidmask = mm_cidmask(mm);
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid, max_nr_cid, allowed_max_nr_cid;
+ struct mm_struct *mm = t->mm;
+ unsigned int max_cids;
- /*
- * After shrinking the number of threads or reducing the number
- * of allowed cpus, reduce the value of max_nr_cid so expansion
- * of cid allocation will preserve cache locality if the number
- * of threads or allowed cpus increase again.
- */
- max_nr_cid = atomic_read(&mm->max_nr_cid);
- while ((allowed_max_nr_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed),
- atomic_read(&mm->mm_users))),
- max_nr_cid > allowed_max_nr_cid) {
- /* atomic_try_cmpxchg loads previous mm->max_nr_cid into max_nr_cid. */
- if (atomic_try_cmpxchg(&mm->max_nr_cid, &max_nr_cid, allowed_max_nr_cid)) {
- max_nr_cid = allowed_max_nr_cid;
- break;
- }
- }
- /* Try to re-use recent cid. This improves cache locality. */
- cid = __this_cpu_read(pcpu_cid->recent_cid);
- if (!mm_cid_is_unset(cid) && cid < max_nr_cid &&
- !cpumask_test_and_set_cpu(cid, cidmask))
- return cid;
- /*
- * Expand cid allocation if the maximum number of concurrency
- * IDs allocated (max_nr_cid) is below the number cpus allowed
- * and number of threads. Expanding cid allocation as much as
- * possible improves cache locality.
- */
- cid = max_nr_cid;
- while (cid < READ_ONCE(mm->nr_cpus_allowed) && cid < atomic_read(&mm->mm_users)) {
- /* atomic_try_cmpxchg loads previous mm->max_nr_cid into cid. */
- if (!atomic_try_cmpxchg(&mm->max_nr_cid, &cid, cid + 1))
- continue;
- if (!cpumask_test_and_set_cpu(cid, cidmask))
- return cid;
- }
- /*
- * Find the first available concurrency id.
- * Retry finding first zero bit if the mask is temporarily
- * filled. This only happens during concurrent remote-clear
- * which owns a cid without holding a rq lock.
- */
- for (;;) {
- cid = cpumask_first_zero(cidmask);
- if (cid < READ_ONCE(mm->nr_cpus_allowed))
- break;
- cpu_relax();
- }
- if (cpumask_test_and_set_cpu(cid, cidmask))
- return -1;
+ max_cids = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
- return cid;
-}
+ /* Try to reuse the last CID of this task */
+ if (__mm_cid_get(t, t->last_mm_cid, max_cids))
+ return true;
-/*
- * Save a snapshot of the current runqueue time of this cpu
- * with the per-cpu cid value, allowing to estimate how recently it was used.
- */
-static inline void mm_cid_snapshot_time(struct rq *rq, struct mm_struct *mm)
-{
- struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(rq));
+ /* Try to reuse the last CID of this mm on this CPU */
+ if (__mm_cid_get(t, __this_cpu_read(mm->pcpu_cid->cid), max_cids))
+ return true;
- lockdep_assert_rq_held(rq);
- WRITE_ONCE(pcpu_cid->time, rq->clock);
+ /* Try the first zero bit in the cidmask. */
+ return __mm_cid_get(t, cpumask_first_zero(mm_cidmask(mm)), max_cids);
}
-static inline int __mm_cid_get(struct rq *rq, struct task_struct *t,
- struct mm_struct *mm)
+static inline void mm_cid_select(struct task_struct *t)
{
- int cid;
-
/*
- * All allocations (even those using the cid_lock) are lock-free. If
- * use_cid_lock is set, hold the cid_lock to perform cid allocation to
- * guarantee forward progress.
+ * mm_cid_get() can fail when the maximum CID, which is determined
+ * by min(mm->nr_cpus_allowed, mm->mm_users) changes concurrently.
+ * That's a transient failure as there cannot be more tasks
+ * concurrently on a CPU (or about to be scheduled in) than that.
*/
- if (!READ_ONCE(use_cid_lock)) {
- cid = __mm_cid_try_get(t, mm);
- if (cid >= 0)
- goto end;
- raw_spin_lock(&cid_lock);
- } else {
- raw_spin_lock(&cid_lock);
- cid = __mm_cid_try_get(t, mm);
- if (cid >= 0)
- goto unlock;
- }
-
- /*
- * cid concurrently allocated. Retry while forcing following
- * allocations to use the cid_lock to ensure forward progress.
- */
- WRITE_ONCE(use_cid_lock, 1);
- /*
- * Set use_cid_lock before allocation. Only care about program order
- * because this is only required for forward progress.
- */
- barrier();
- /*
- * Retry until it succeeds. It is guaranteed to eventually succeed once
- * all newcoming allocations observe the use_cid_lock flag set.
- */
- do {
- cid = __mm_cid_try_get(t, mm);
- cpu_relax();
- } while (cid < 0);
- /*
- * Allocate before clearing use_cid_lock. Only care about
- * program order because this is for forward progress.
- */
- barrier();
- WRITE_ONCE(use_cid_lock, 0);
-unlock:
- raw_spin_unlock(&cid_lock);
-end:
- mm_cid_snapshot_time(rq, mm);
-
- return cid;
-}
-
-static inline int mm_cid_get(struct rq *rq, struct task_struct *t,
- struct mm_struct *mm)
-{
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid;
-
- lockdep_assert_rq_held(rq);
- cid = __this_cpu_read(pcpu_cid->cid);
- if (mm_cid_is_valid(cid)) {
- mm_cid_snapshot_time(rq, mm);
- return cid;
- }
- if (mm_cid_is_lazy_put(cid)) {
- if (try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET))
- __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
+ for (;;) {
+ if (mm_cid_get(t))
+ break;
}
- cid = __mm_cid_get(rq, t, mm);
- __this_cpu_write(pcpu_cid->cid, cid);
- __this_cpu_write(pcpu_cid->recent_cid, cid);
-
- return cid;
}
-static inline void switch_mm_cid(struct rq *rq,
- struct task_struct *prev,
- struct task_struct *next)
+static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
{
- /*
- * Provide a memory barrier between rq->curr store and load of
- * {prev,next}->mm->pcpu_cid[cpu] on rq->curr->mm transition.
- *
- * Should be adapted if context_switch() is modified.
- */
- if (!next->mm) { // to kernel
- /*
- * user -> kernel transition does not guarantee a barrier, but
- * we can use the fact that it performs an atomic operation in
- * mmgrab().
- */
- if (prev->mm) // from user
- smp_mb__after_mmgrab();
- /*
- * kernel -> kernel transition does not change rq->curr->mm
- * state. It stays NULL.
- */
- } else { // to user
- /*
- * kernel -> user transition does not provide a barrier
- * between rq->curr store and load of {prev,next}->mm->pcpu_cid[cpu].
- * Provide it here.
- */
- if (!prev->mm) { // from kernel
- smp_mb();
- } else { // from user
- /*
- * user->user transition relies on an implicit
- * memory barrier in switch_mm() when
- * current->mm changes. If the architecture
- * switch_mm() does not have an implicit memory
- * barrier, it is emitted here. If current->mm
- * is unchanged, no barrier is needed.
- */
- smp_mb__after_switch_mm();
- }
- }
if (prev->mm_cid_active) {
- mm_cid_snapshot_time(rq, prev->mm);
- mm_cid_put_lazy(prev);
- prev->mm_cid = -1;
+ if (prev->mm_cid != MM_CID_UNSET)
+ cpumask_clear_cpu(prev->mm_cid, mm_cidmask(prev->mm));
+ prev->mm_cid = MM_CID_UNSET;
}
+
if (next->mm_cid_active) {
- next->last_mm_cid = next->mm_cid = mm_cid_get(rq, next, next->mm);
+ mm_cid_select(next);
rseq_sched_set_task_mm_cid(next, next->mm_cid);
}
}
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void switch_mm_cid(struct rq *rq, struct task_struct *prev, struct task_struct *next) { }
-static inline void sched_mm_cid_migrate_from(struct task_struct *t) { }
-static inline void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) { }
-static inline void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) { }
static inline void init_sched_mm_cid(struct task_struct *t) { }
+static inline void mm_cid_select(struct task_struct *t) { }
+static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
extern u64 avg_vruntime(struct cfs_rq *cfs_rq);Having a lot of CID functionality specific members in struct task_struct
and struct mm_struct is not really making the code easier to read.
Encapsulate the CID specific parts in data structures and keep them
separate from the stuff they are embedded in.
No functional change.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/mm_types.h | 56 +++++++++++----------------------------------
include/linux/rseq_types.h | 42 +++++++++++++++++++++++++++++++++
include/linux/sched.h | 11 +-------
init/init_task.c | 3 ++
kernel/fork.c | 6 ++--
kernel/sched/core.c | 16 ++++++------
kernel/sched/sched.h | 26 ++++++++++----------
7 files changed, 85 insertions(+), 75 deletions(-)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -20,6 +20,7 @@
#include <linux/seqlock.h>
#include <linux/percpu_counter.h>
#include <linux/types.h>
+#include <linux/rseq_types.h>
#include <linux/bitmap.h>
#include <asm/mmu.h>
@@ -922,10 +923,6 @@ struct vm_area_struct {
#define vma_policy(vma) NULL
#endif
-struct mm_cid {
- unsigned int cid;
-};
-
/*
* Opaque type representing current mm_struct flag state. Must be accessed via
* mm_flags_xxx() helper functions.
@@ -987,30 +984,9 @@ struct mm_struct {
*/
atomic_t mm_users;
-#ifdef CONFIG_SCHED_MM_CID
- /**
- * @pcpu_cid: Per-cpu current cid.
- *
- * Keep track of the currently allocated mm_cid for each cpu.
- * The per-cpu mm_cid values are serialized by their respective
- * runqueue locks.
- */
- struct mm_cid __percpu *pcpu_cid;
- /**
- * @nr_cpus_allowed: Number of CPUs allowed for mm.
- *
- * Number of CPUs allowed in the union of all mm's
- * threads allowed CPUs.
- */
- unsigned int nr_cpus_allowed;
- /**
- * @cpus_allowed_lock: Lock protecting mm cpus_allowed.
- *
- * Provide mutual exclusion for mm cpus_allowed and
- * mm nr_cpus_allowed updates.
- */
- raw_spinlock_t cpus_allowed_lock;
-#endif
+ /* MM CID related storage */
+ struct mm_mm_cid mm_cid;
+
#ifdef CONFIG_MMU
atomic_long_t pgtables_bytes; /* size of all page tables */
#endif
@@ -1352,9 +1328,6 @@ static inline void vma_iter_init(struct
}
#ifdef CONFIG_SCHED_MM_CID
-
-#define MM_CID_UNSET (~0U)
-
/*
* mm_cpus_allowed: Union of all mm's threads allowed CPUs.
*/
@@ -1383,20 +1356,20 @@ static inline void mm_init_cid(struct mm
int i;
for_each_possible_cpu(i) {
- struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i);
+ struct mm_cid_pcpu *pcpu = per_cpu_ptr(mm->mm_cid.pcpu, i);
- pcpu_cid->cid = MM_CID_UNSET;
+ pcpu->cid = MM_CID_UNSET;
}
- mm->nr_cpus_allowed = p->nr_cpus_allowed;
- raw_spin_lock_init(&mm->cpus_allowed_lock);
+ mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
cpumask_clear(mm_cidmask(mm));
}
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
{
- mm->pcpu_cid = alloc_percpu_noprof(struct mm_cid);
- if (!mm->pcpu_cid)
+ mm->mm_cid.pcpu = alloc_percpu_noprof(struct mm_cid_pcpu);
+ if (!mm->mm_cid.pcpu)
return -ENOMEM;
mm_init_cid(mm, p);
return 0;
@@ -1405,8 +1378,8 @@ static inline int mm_alloc_cid_noprof(st
static inline void mm_destroy_cid(struct mm_struct *mm)
{
- free_percpu(mm->pcpu_cid);
- mm->pcpu_cid = NULL;
+ free_percpu(mm->mm_cid.pcpu);
+ mm->mm_cid.pcpu = NULL;
}
static inline unsigned int mm_cid_size(void)
@@ -1421,10 +1394,9 @@ static inline void mm_set_cpus_allowed(s
if (!mm)
return;
/* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
- raw_spin_lock(&mm->cpus_allowed_lock);
+ guard(raw_spinlock)(&mm->mm_cid.lock);
cpumask_or(mm_allowed, mm_allowed, cpumask);
- WRITE_ONCE(mm->nr_cpus_allowed, cpumask_weight(mm_allowed));
- raw_spin_unlock(&mm->cpus_allowed_lock);
+ WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p) { }
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -90,4 +90,46 @@ struct rseq_data {
struct rseq_data { };
#endif /* !CONFIG_RSEQ */
+#ifdef CONFIG_SCHED_MM_CID
+
+#define MM_CID_UNSET (~0U)
+
+/**
+ * struct sched_mm_cid - Storage for per task MM CID data
+ * @active: MM CID is active for the task
+ * @cid: The CID associated to the task
+ * @last_cid: The last CID associated to the task
+ */
+struct sched_mm_cid {
+ unsigned int active;
+ unsigned int cid;
+ unsigned int last_cid;
+};
+
+/**
+ * struct mm_cid_pcpu - Storage for per CPU MM_CID data
+ * @cid: The CID associated to the CPU
+ */
+struct mm_cid_pcpu {
+ unsigned int cid;
+};
+
+/**
+ * struct mm_mm_cid - Storage for per MM CID data
+ * @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
+ * is growth only.
+ * @lock: Spinlock to protect all fields except @pcpu. It also protects
+ * the MM cid cpumask and the MM cidmask bitmap.
+ */
+struct mm_mm_cid {
+ struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int nr_cpus_allowed;
+ raw_spinlock_t lock;
+};
+#else /* CONFIG_SCHED_MM_CID */
+struct mm_mm_cid { };
+struct sched_mm_cid { };
+#endif /* !CONFIG_SCHED_MM_CID */
+
#endif
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1407,14 +1407,7 @@ struct task_struct {
#endif /* CONFIG_NUMA_BALANCING */
struct rseq_data rseq;
-
-#ifdef CONFIG_SCHED_MM_CID
- int mm_cid; /* Current cid in mm */
- int last_mm_cid; /* Most recent cid in mm */
- int migrate_from_cpu;
- int mm_cid_active; /* Whether cid bitmap is active */
- struct callback_head cid_work;
-#endif
+ struct sched_mm_cid mm_cid;
struct tlbflush_unmap_batch tlb_ubc;
@@ -2308,7 +2301,7 @@ void sched_mm_cid_fork(struct task_struc
void sched_mm_cid_exit_signals(struct task_struct *t);
static inline int task_mm_cid(struct task_struct *t)
{
- return t->mm_cid;
+ return t->mm_cid.cid;
}
#else
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -223,6 +223,9 @@ struct task_struct init_task __aligned(L
#ifdef CONFIG_SECCOMP_FILTER
.seccomp = { .filter_count = ATOMIC_INIT(0) },
#endif
+#ifdef CONFIG_SCHED_MM_CID
+ .mm_cid = { .cid = MM_CID_UNSET, },
+#endif
};
EXPORT_SYMBOL(init_task);
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -955,9 +955,9 @@ static struct task_struct *dup_task_stru
#endif
#ifdef CONFIG_SCHED_MM_CID
- tsk->mm_cid = MM_CID_UNSET;
- tsk->last_mm_cid = MM_CID_UNSET;
- tsk->mm_cid_active = 0;
+ tsk->mm_cid.cid = MM_CID_UNSET;
+ tsk->mm_cid.last_cid = MM_CID_UNSET;
+ tsk->mm_cid.active = 0;
#endif
return tsk;
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10376,14 +10376,14 @@ void sched_mm_cid_exit_signals(struct ta
{
struct mm_struct *mm = t->mm;
- if (!mm || !t->mm_cid_active)
+ if (!mm || !t->mm_cid.active)
return;
guard(preempt)();
- t->mm_cid_active = 0;
- if (t->mm_cid != MM_CID_UNSET) {
- cpumask_clear_cpu(t->mm_cid, mm_cidmask(mm));
- t->mm_cid = MM_CID_UNSET;
+ t->mm_cid.active = 0;
+ if (t->mm_cid.cid != MM_CID_UNSET) {
+ cpumask_clear_cpu(t->mm_cid.cid, mm_cidmask(mm));
+ t->mm_cid.cid = MM_CID_UNSET;
}
}
@@ -10402,14 +10402,14 @@ void sched_mm_cid_after_execve(struct ta
return;
guard(preempt)();
- t->mm_cid_active = 1;
+ t->mm_cid.active = 1;
mm_cid_select(t);
}
void sched_mm_cid_fork(struct task_struct *t)
{
- WARN_ON_ONCE(!t->mm || t->mm_cid != MM_CID_UNSET);
- t->mm_cid_active = 1;
+ WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
+ t->mm_cid.active = 1;
}
#endif /* CONFIG_SCHED_MM_CID */
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3549,8 +3549,8 @@ static inline void init_sched_mm_cid(str
return;
/* Preset last_mm_cid */
- max_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
- t->last_mm_cid = max_cid - 1;
+ max_cid = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
+ t->mm_cid.last_cid = max_cid - 1;
}
static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
@@ -3561,8 +3561,8 @@ static inline bool __mm_cid_get(struct t
return false;
if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
return false;
- t->mm_cid = t->last_mm_cid = cid;
- __this_cpu_write(mm->pcpu_cid->cid, cid);
+ t->mm_cid.cid = t->mm_cid.last_cid = cid;
+ __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
return true;
}
@@ -3571,14 +3571,14 @@ static inline bool mm_cid_get(struct tas
struct mm_struct *mm = t->mm;
unsigned int max_cids;
- max_cids = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
+ max_cids = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
/* Try to reuse the last CID of this task */
- if (__mm_cid_get(t, t->last_mm_cid, max_cids))
+ if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
return true;
/* Try to reuse the last CID of this mm on this CPU */
- if (__mm_cid_get(t, __this_cpu_read(mm->pcpu_cid->cid), max_cids))
+ if (__mm_cid_get(t, __this_cpu_read(mm->mm_cid.pcpu->cid), max_cids))
return true;
/* Try the first zero bit in the cidmask. */
@@ -3601,15 +3601,15 @@ static inline void mm_cid_select(struct
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
{
- if (prev->mm_cid_active) {
- if (prev->mm_cid != MM_CID_UNSET)
- cpumask_clear_cpu(prev->mm_cid, mm_cidmask(prev->mm));
- prev->mm_cid = MM_CID_UNSET;
+ if (prev->mm_cid.active) {
+ if (prev->mm_cid.cid != MM_CID_UNSET)
+ cpumask_clear_cpu(prev->mm_cid.cid, mm_cidmask(prev->mm));
+ prev->mm_cid.cid = MM_CID_UNSET;
}
- if (next->mm_cid_active) {
+ if (next->mm_cid.active) {
mm_cid_select(next);
- rseq_sched_set_task_mm_cid(next, next->mm_cid);
+ rseq_sched_set_task_mm_cid(next, next->mm_cid.cid);
}
}The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 8cea569ca785060b8c5cc7800713ddc3b1548a94
Gitweb: https://git.kernel.org/tip/8cea569ca785060b8c5cc7800713ddc3b1548a94
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:47 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:52 +01:00
sched/mmcid: Use proper data structures
Having a lot of CID functionality specific members in struct task_struct
and struct mm_struct is not really making the code easier to read.
Encapsulate the CID specific parts in data structures and keep them
separate from the stuff they are embedded in.
No functional change.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.131573768@linutronix.de
---
include/linux/mm_types.h | 56 +++++++++----------------------------
include/linux/rseq_types.h | 42 ++++++++++++++++++++++++++++-
include/linux/sched.h | 11 +------
init/init_task.c | 3 ++-
kernel/fork.c | 6 ++--
kernel/sched/core.c | 16 +++++------
kernel/sched/sched.h | 26 ++++++++---------
7 files changed, 85 insertions(+), 75 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 63b8c12..e4818e9 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -20,6 +20,7 @@
#include <linux/seqlock.h>
#include <linux/percpu_counter.h>
#include <linux/types.h>
+#include <linux/rseq_types.h>
#include <linux/bitmap.h>
#include <asm/mmu.h>
@@ -922,10 +923,6 @@ struct vm_area_struct {
#define vma_policy(vma) NULL
#endif
-struct mm_cid {
- unsigned int cid;
-};
-
/*
* Opaque type representing current mm_struct flag state. Must be accessed via
* mm_flags_xxx() helper functions.
@@ -987,30 +984,9 @@ struct mm_struct {
*/
atomic_t mm_users;
-#ifdef CONFIG_SCHED_MM_CID
- /**
- * @pcpu_cid: Per-cpu current cid.
- *
- * Keep track of the currently allocated mm_cid for each cpu.
- * The per-cpu mm_cid values are serialized by their respective
- * runqueue locks.
- */
- struct mm_cid __percpu *pcpu_cid;
- /**
- * @nr_cpus_allowed: Number of CPUs allowed for mm.
- *
- * Number of CPUs allowed in the union of all mm's
- * threads allowed CPUs.
- */
- unsigned int nr_cpus_allowed;
- /**
- * @cpus_allowed_lock: Lock protecting mm cpus_allowed.
- *
- * Provide mutual exclusion for mm cpus_allowed and
- * mm nr_cpus_allowed updates.
- */
- raw_spinlock_t cpus_allowed_lock;
-#endif
+ /* MM CID related storage */
+ struct mm_mm_cid mm_cid;
+
#ifdef CONFIG_MMU
atomic_long_t pgtables_bytes; /* size of all page tables */
#endif
@@ -1352,9 +1328,6 @@ static inline void vma_iter_init(struct vma_iterator *vmi,
}
#ifdef CONFIG_SCHED_MM_CID
-
-#define MM_CID_UNSET (~0U)
-
/*
* mm_cpus_allowed: Union of all mm's threads allowed CPUs.
*/
@@ -1383,20 +1356,20 @@ static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
int i;
for_each_possible_cpu(i) {
- struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i);
+ struct mm_cid_pcpu *pcpu = per_cpu_ptr(mm->mm_cid.pcpu, i);
- pcpu_cid->cid = MM_CID_UNSET;
+ pcpu->cid = MM_CID_UNSET;
}
- mm->nr_cpus_allowed = p->nr_cpus_allowed;
- raw_spin_lock_init(&mm->cpus_allowed_lock);
+ mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
cpumask_clear(mm_cidmask(mm));
}
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
{
- mm->pcpu_cid = alloc_percpu_noprof(struct mm_cid);
- if (!mm->pcpu_cid)
+ mm->mm_cid.pcpu = alloc_percpu_noprof(struct mm_cid_pcpu);
+ if (!mm->mm_cid.pcpu)
return -ENOMEM;
mm_init_cid(mm, p);
return 0;
@@ -1405,8 +1378,8 @@ static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *
static inline void mm_destroy_cid(struct mm_struct *mm)
{
- free_percpu(mm->pcpu_cid);
- mm->pcpu_cid = NULL;
+ free_percpu(mm->mm_cid.pcpu);
+ mm->mm_cid.pcpu = NULL;
}
static inline unsigned int mm_cid_size(void)
@@ -1421,10 +1394,9 @@ static inline void mm_set_cpus_allowed(struct mm_struct *mm, const struct cpumas
if (!mm)
return;
/* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
- raw_spin_lock(&mm->cpus_allowed_lock);
+ guard(raw_spinlock)(&mm->mm_cid.lock);
cpumask_or(mm_allowed, mm_allowed, cpumask);
- WRITE_ONCE(mm->nr_cpus_allowed, cpumask_weight(mm_allowed));
- raw_spin_unlock(&mm->cpus_allowed_lock);
+ WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p) { }
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 9c7a341..e444dd2 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -90,4 +90,46 @@ struct rseq_data {
struct rseq_data { };
#endif /* !CONFIG_RSEQ */
+#ifdef CONFIG_SCHED_MM_CID
+
+#define MM_CID_UNSET (~0U)
+
+/**
+ * struct sched_mm_cid - Storage for per task MM CID data
+ * @active: MM CID is active for the task
+ * @cid: The CID associated to the task
+ * @last_cid: The last CID associated to the task
+ */
+struct sched_mm_cid {
+ unsigned int active;
+ unsigned int cid;
+ unsigned int last_cid;
+};
+
+/**
+ * struct mm_cid_pcpu - Storage for per CPU MM_CID data
+ * @cid: The CID associated to the CPU
+ */
+struct mm_cid_pcpu {
+ unsigned int cid;
+};
+
+/**
+ * struct mm_mm_cid - Storage for per MM CID data
+ * @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
+ * is growth only.
+ * @lock: Spinlock to protect all fields except @pcpu. It also protects
+ * the MM cid cpumask and the MM cidmask bitmap.
+ */
+struct mm_mm_cid {
+ struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int nr_cpus_allowed;
+ raw_spinlock_t lock;
+};
+#else /* CONFIG_SCHED_MM_CID */
+struct mm_mm_cid { };
+struct sched_mm_cid { };
+#endif /* !CONFIG_SCHED_MM_CID */
+
#endif
diff --git a/include/linux/sched.h b/include/linux/sched.h
index e47abc8..64f080d 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1407,14 +1407,7 @@ struct task_struct {
#endif /* CONFIG_NUMA_BALANCING */
struct rseq_data rseq;
-
-#ifdef CONFIG_SCHED_MM_CID
- int mm_cid; /* Current cid in mm */
- int last_mm_cid; /* Most recent cid in mm */
- int migrate_from_cpu;
- int mm_cid_active; /* Whether cid bitmap is active */
- struct callback_head cid_work;
-#endif
+ struct sched_mm_cid mm_cid;
struct tlbflush_unmap_batch tlb_ubc;
@@ -2308,7 +2301,7 @@ void sched_mm_cid_fork(struct task_struct *t);
void sched_mm_cid_exit_signals(struct task_struct *t);
static inline int task_mm_cid(struct task_struct *t)
{
- return t->mm_cid;
+ return t->mm_cid.cid;
}
#else
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
diff --git a/init/init_task.c b/init/init_task.c
index a55e218..5d12269 100644
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -223,6 +223,9 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
#ifdef CONFIG_SECCOMP_FILTER
.seccomp = { .filter_count = ATOMIC_INIT(0) },
#endif
+#ifdef CONFIG_SCHED_MM_CID
+ .mm_cid = { .cid = MM_CID_UNSET, },
+#endif
};
EXPORT_SYMBOL(init_task);
diff --git a/kernel/fork.c b/kernel/fork.c
index 9d9afe4..74bc7c9 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -955,9 +955,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
#endif
#ifdef CONFIG_SCHED_MM_CID
- tsk->mm_cid = MM_CID_UNSET;
- tsk->last_mm_cid = MM_CID_UNSET;
- tsk->mm_cid_active = 0;
+ tsk->mm_cid.cid = MM_CID_UNSET;
+ tsk->mm_cid.last_cid = MM_CID_UNSET;
+ tsk->mm_cid.active = 0;
#endif
return tsk;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 11a1735..b1aa7d1 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10376,14 +10376,14 @@ void sched_mm_cid_exit_signals(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
- if (!mm || !t->mm_cid_active)
+ if (!mm || !t->mm_cid.active)
return;
guard(preempt)();
- t->mm_cid_active = 0;
- if (t->mm_cid != MM_CID_UNSET) {
- cpumask_clear_cpu(t->mm_cid, mm_cidmask(mm));
- t->mm_cid = MM_CID_UNSET;
+ t->mm_cid.active = 0;
+ if (t->mm_cid.cid != MM_CID_UNSET) {
+ cpumask_clear_cpu(t->mm_cid.cid, mm_cidmask(mm));
+ t->mm_cid.cid = MM_CID_UNSET;
}
}
@@ -10402,14 +10402,14 @@ void sched_mm_cid_after_execve(struct task_struct *t)
return;
guard(preempt)();
- t->mm_cid_active = 1;
+ t->mm_cid.active = 1;
mm_cid_select(t);
}
void sched_mm_cid_fork(struct task_struct *t)
{
- WARN_ON_ONCE(!t->mm || t->mm_cid != MM_CID_UNSET);
- t->mm_cid_active = 1;
+ WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
+ t->mm_cid.active = 1;
}
#endif /* CONFIG_SCHED_MM_CID */
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index bf227c2..a17f04f 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3549,8 +3549,8 @@ static inline void init_sched_mm_cid(struct task_struct *t)
return;
/* Preset last_mm_cid */
- max_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
- t->last_mm_cid = max_cid - 1;
+ max_cid = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
+ t->mm_cid.last_cid = max_cid - 1;
}
static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
@@ -3561,8 +3561,8 @@ static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigne
return false;
if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
return false;
- t->mm_cid = t->last_mm_cid = cid;
- __this_cpu_write(mm->pcpu_cid->cid, cid);
+ t->mm_cid.cid = t->mm_cid.last_cid = cid;
+ __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
return true;
}
@@ -3571,14 +3571,14 @@ static inline bool mm_cid_get(struct task_struct *t)
struct mm_struct *mm = t->mm;
unsigned int max_cids;
- max_cids = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
+ max_cids = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
/* Try to reuse the last CID of this task */
- if (__mm_cid_get(t, t->last_mm_cid, max_cids))
+ if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
return true;
/* Try to reuse the last CID of this mm on this CPU */
- if (__mm_cid_get(t, __this_cpu_read(mm->pcpu_cid->cid), max_cids))
+ if (__mm_cid_get(t, __this_cpu_read(mm->mm_cid.pcpu->cid), max_cids))
return true;
/* Try the first zero bit in the cidmask. */
@@ -3601,15 +3601,15 @@ static inline void mm_cid_select(struct task_struct *t)
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
{
- if (prev->mm_cid_active) {
- if (prev->mm_cid != MM_CID_UNSET)
- cpumask_clear_cpu(prev->mm_cid, mm_cidmask(prev->mm));
- prev->mm_cid = MM_CID_UNSET;
+ if (prev->mm_cid.active) {
+ if (prev->mm_cid.cid != MM_CID_UNSET)
+ cpumask_clear_cpu(prev->mm_cid.cid, mm_cidmask(prev->mm));
+ prev->mm_cid.cid = MM_CID_UNSET;
}
- if (next->mm_cid_active) {
+ if (next->mm_cid.active) {
mm_cid_select(next);
- rseq_sched_set_task_mm_cid(next, next->mm_cid);
+ rseq_sched_set_task_mm_cid(next, next->mm_cid.cid);
}
}
Both the per CPU storage and the data in mm_struct are heavily used in
context switch. As they can end up next to other frequently modified data,
they are subject to false sharing.
Make them cache line aligned.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/rseq_types.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -112,7 +112,7 @@ struct sched_mm_cid {
*/
struct mm_cid_pcpu {
unsigned int cid;
-};
+}____cacheline_aligned_in_smp;
/**
* struct mm_mm_cid - Storage for per MM CID data
@@ -126,7 +126,7 @@ struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
unsigned int nr_cpus_allowed;
raw_spinlock_t lock;
-};
+}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
struct sched_mm_cid { };The following commit has been merged into the core/rseq branch of tip:
Commit-ID: be4463fa2c7185823d2989562162d578b45a89ae
Gitweb: https://git.kernel.org/tip/be4463fa2c7185823d2989562162d578b45a89ae
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:49 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:53 +01:00
sched/mmcid: Cacheline align MM CID storage
Both the per CPU storage and the data in mm_struct are heavily used in
context switch. As they can end up next to other frequently modified data,
they are subject to false sharing.
Make them cache line aligned.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.194111661@linutronix.de
---
include/linux/rseq_types.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index e444dd2..d7e8071 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -112,7 +112,7 @@ struct sched_mm_cid {
*/
struct mm_cid_pcpu {
unsigned int cid;
-};
+}____cacheline_aligned_in_smp;
/**
* struct mm_mm_cid - Storage for per MM CID data
@@ -126,7 +126,7 @@ struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
unsigned int nr_cpus_allowed;
raw_spinlock_t lock;
-};
+}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
struct sched_mm_cid { };With whitespace checks enabled in the editor this makes eyes bleed.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
kernel/sched/core.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5278,15 +5278,15 @@ context_switch(struct rq *rq, struct tas
* kernel -> user switch + mmdrop_lazy_tlb() active
* user -> user switch
*/
- if (!next->mm) { // to kernel
+ if (!next->mm) { // to kernel
enter_lazy_tlb(prev->active_mm, next);
next->active_mm = prev->active_mm;
- if (prev->mm) // from user
+ if (prev->mm) // from user
mmgrab_lazy_tlb(prev->active_mm);
else
prev->active_mm = NULL;
- } else { // to user
+ } else { // to user
membarrier_switch_mm(rq, prev->active_mm, next->mm);
/*
* sys_membarrier() requires an smp_mb() between setting
@@ -5299,7 +5299,7 @@ context_switch(struct rq *rq, struct tas
switch_mm_irqs_off(prev->active_mm, next->mm, next);
lru_gen_use_mm(next->mm);
- if (!prev->mm) { // from kernel
+ if (!prev->mm) { // from kernel
/* will mmdrop_lazy_tlb() in finish_task_switch(). */
rq->prev_mm = prev->active_mm;
prev->active_mm = NULL;The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 925b7847bb7d4eb523a7698b309e8441647796f2
Gitweb: https://git.kernel.org/tip/925b7847bb7d4eb523a7698b309e8441647796f2
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:51 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:53 +01:00
sched: Fixup whitespace damage
With whitespace checks enabled in the editor this makes eyes bleed.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.258651925@linutronix.de
---
kernel/sched/core.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index b1aa7d1..b667171 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5278,15 +5278,15 @@ context_switch(struct rq *rq, struct task_struct *prev,
* kernel -> user switch + mmdrop_lazy_tlb() active
* user -> user switch
*/
- if (!next->mm) { // to kernel
+ if (!next->mm) { // to kernel
enter_lazy_tlb(prev->active_mm, next);
next->active_mm = prev->active_mm;
- if (prev->mm) // from user
+ if (prev->mm) // from user
mmgrab_lazy_tlb(prev->active_mm);
else
prev->active_mm = NULL;
- } else { // to user
+ } else { // to user
membarrier_switch_mm(rq, prev->active_mm, next->mm);
/*
* sys_membarrier() requires an smp_mb() between setting
@@ -5299,7 +5299,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
switch_mm_irqs_off(prev->active_mm, next->mm, next);
lru_gen_use_mm(next->mm);
- if (!prev->mm) { // from kernel
+ if (!prev->mm) { // from kernel
/* will mmdrop_lazy_tlb() in finish_task_switch(). */
rq->prev_mm = prev->active_mm;
prev->active_mm = NULL;This is only used in the scheduler core code, so there is no point to have
it in a global header.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Acked-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
include/linux/mm_types.h | 13 -------------
kernel/sched/core.c | 20 ++++++++++++++++++--
2 files changed, 18 insertions(+), 15 deletions(-)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1387,27 +1387,14 @@ static inline unsigned int mm_cid_size(v
return 2 * cpumask_size(); /* mm_cpus_allowed(), mm_cidmask(). */
}
-static inline void mm_set_cpus_allowed(struct mm_struct *mm, const struct cpumask *cpumask)
-{
- struct cpumask *mm_allowed = mm_cpus_allowed(mm);
-
- if (!mm)
- return;
- /* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
- guard(raw_spinlock)(&mm->mm_cid.lock);
- cpumask_or(mm_allowed, mm_allowed, cpumask);
- WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
-}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p) { }
static inline int mm_alloc_cid(struct mm_struct *mm, struct task_struct *p) { return 0; }
static inline void mm_destroy_cid(struct mm_struct *mm) { }
-
static inline unsigned int mm_cid_size(void)
{
return 0;
}
-static inline void mm_set_cpus_allowed(struct mm_struct *mm, const struct cpumask *cpumask) { }
#endif /* CONFIG_SCHED_MM_CID */
struct mmu_gather;
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2669,6 +2669,8 @@ int push_cpu_stop(void *arg)
return 0;
}
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const cpumask_t *affmask);
+
/*
* sched_class::set_cpus_allowed must do the below, but is not required to
* actually call this function.
@@ -2728,7 +2730,7 @@ static void
put_prev_task(rq, p);
p->sched_class->set_cpus_allowed(p, ctx);
- mm_set_cpus_allowed(p->mm, ctx->new_mask);
+ mm_update_cpus_allowed(p->mm, ctx->new_mask);
if (queued)
enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
@@ -10372,6 +10374,18 @@ void call_trace_sched_update_nr_running(
* When a task exits, the MM CID held by the task is not longer required as
* the task cannot return to user space.
*/
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
+{
+ struct cpumask *mm_allowed = mm_cpus_allowed(mm);
+
+ if (!mm)
+ return;
+ /* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
+ guard(raw_spinlock)(&mm->mm_cid.lock);
+ cpumask_or(mm_allowed, mm_allowed, affmsk);
+ WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
+}
+
void sched_mm_cid_exit_signals(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10411,7 +10425,9 @@ void sched_mm_cid_fork(struct task_struc
WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
t->mm_cid.active = 1;
}
-#endif /* CONFIG_SCHED_MM_CID */
+#else /* CONFIG_SCHED_MM_CID */
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }
+#endif /* !CONFIG_SCHED_MM_CID */
#ifdef CONFIG_SCHED_CLASS_EXT
void sched_deq_and_put_task(struct task_struct *p, int queue_flags,The following commit has been merged into the core/rseq branch of tip:
Commit-ID: b08ef5fc8fa01ae5285bef5ff783bbb425d1fb08
Gitweb: https://git.kernel.org/tip/b08ef5fc8fa01ae5285bef5ff783bbb425d1fb08
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:53 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:53 +01:00
sched/mmcid: Move scheduler code out of global header
This is only used in the scheduler core code, so there is no point to have
it in a global header.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Acked-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
Link: https://patch.msgid.link/20251119172549.321259077@linutronix.de
---
include/linux/mm_types.h | 13 -------------
kernel/sched/core.c | 20 ++++++++++++++++++--
2 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index e4818e9..67a7bdf 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1387,27 +1387,14 @@ static inline unsigned int mm_cid_size(void)
return 2 * cpumask_size(); /* mm_cpus_allowed(), mm_cidmask(). */
}
-static inline void mm_set_cpus_allowed(struct mm_struct *mm, const struct cpumask *cpumask)
-{
- struct cpumask *mm_allowed = mm_cpus_allowed(mm);
-
- if (!mm)
- return;
- /* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
- guard(raw_spinlock)(&mm->mm_cid.lock);
- cpumask_or(mm_allowed, mm_allowed, cpumask);
- WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
-}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p) { }
static inline int mm_alloc_cid(struct mm_struct *mm, struct task_struct *p) { return 0; }
static inline void mm_destroy_cid(struct mm_struct *mm) { }
-
static inline unsigned int mm_cid_size(void)
{
return 0;
}
-static inline void mm_set_cpus_allowed(struct mm_struct *mm, const struct cpumask *cpumask) { }
#endif /* CONFIG_SCHED_MM_CID */
struct mmu_gather;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index b667171..f5e37c2 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2669,6 +2669,8 @@ out_unlock:
return 0;
}
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const cpumask_t *affmask);
+
/*
* sched_class::set_cpus_allowed must do the below, but is not required to
* actually call this function.
@@ -2728,7 +2730,7 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx)
put_prev_task(rq, p);
p->sched_class->set_cpus_allowed(p, ctx);
- mm_set_cpus_allowed(p->mm, ctx->new_mask);
+ mm_update_cpus_allowed(p->mm, ctx->new_mask);
if (queued)
enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
@@ -10372,6 +10374,18 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
* When a task exits, the MM CID held by the task is not longer required as
* the task cannot return to user space.
*/
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
+{
+ struct cpumask *mm_allowed = mm_cpus_allowed(mm);
+
+ if (!mm)
+ return;
+ /* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
+ guard(raw_spinlock)(&mm->mm_cid.lock);
+ cpumask_or(mm_allowed, mm_allowed, affmsk);
+ WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
+}
+
void sched_mm_cid_exit_signals(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10411,7 +10425,9 @@ void sched_mm_cid_fork(struct task_struct *t)
WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
t->mm_cid.active = 1;
}
-#endif /* CONFIG_SCHED_MM_CID */
+#else /* CONFIG_SCHED_MM_CID */
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }
+#endif /* !CONFIG_SCHED_MM_CID */
#ifdef CONFIG_SCHED_CLASS_EXT
void sched_deq_and_put_task(struct task_struct *p, int queue_flags,mm_update_cpus_allowed() is not required to be invoked for affinity changes
due to migrate_disable() and migrate_enable().
migrate_disable() restricts the task temporarily to a CPU on which the task
was already allowed to run, so nothing changes. migrate_enable() restores
the actual task affinity mask.
If that mask changed between migrate_disable() and migrate_enable() then
that change was already accounted for.
Move the invocation to the proper place to avoid that.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
V2: Remove the nr_cpu_ids optimization which does not really work - PeterZ
---
kernel/sched/core.c | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2684,6 +2684,7 @@ void set_cpus_allowed_common(struct task
cpumask_copy(&p->cpus_mask, ctx->new_mask);
p->nr_cpus_allowed = cpumask_weight(ctx->new_mask);
+ mm_update_cpus_allowed(p->mm, ctx->new_mask);
/*
* Swap in a new user_cpus_ptr if SCA_USER flag set
@@ -2730,7 +2731,6 @@ static void
put_prev_task(rq, p);
p->sched_class->set_cpus_allowed(p, ctx);
- mm_update_cpus_allowed(p->mm, ctx->new_mask);
if (queued)
enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
@@ -10376,12 +10376,17 @@ void call_trace_sched_update_nr_running(
*/
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
- struct cpumask *mm_allowed = mm_cpus_allowed(mm);
+ struct cpumask *mm_allowed;
if (!mm)
return;
- /* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
+
+ /*
+ * mm::mm_cid::mm_cpus_allowed is the superset of each threads
+ * allowed CPUs mask which means it can only grow.
+ */
guard(raw_spinlock)(&mm->mm_cid.lock);
+ mm_allowed = mm_cpus_allowed(mm);
cpumask_or(mm_allowed, mm_allowed, affmsk);
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
}The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 0d032a43ebeb9bf255cd7e3dad5f7a6371571648
Gitweb: https://git.kernel.org/tip/0d032a43ebeb9bf255cd7e3dad5f7a6371571648
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:55 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:54 +01:00
sched/mmcid: Prevent pointless work in mm_update_cpus_allowed()
mm_update_cpus_allowed() is not required to be invoked for affinity changes
due to migrate_disable() and migrate_enable().
migrate_disable() restricts the task temporarily to a CPU on which the task
was already allowed to run, so nothing changes. migrate_enable() restores
the actual task affinity mask.
If that mask changed between migrate_disable() and migrate_enable() then
that change was already accounted for.
Move the invocation to the proper place to avoid that.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.385208276@linutronix.de
---
kernel/sched/core.c | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f5e37c2..2ea77e7 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2684,6 +2684,7 @@ void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx
cpumask_copy(&p->cpus_mask, ctx->new_mask);
p->nr_cpus_allowed = cpumask_weight(ctx->new_mask);
+ mm_update_cpus_allowed(p->mm, ctx->new_mask);
/*
* Swap in a new user_cpus_ptr if SCA_USER flag set
@@ -2730,7 +2731,6 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx)
put_prev_task(rq, p);
p->sched_class->set_cpus_allowed(p, ctx);
- mm_update_cpus_allowed(p->mm, ctx->new_mask);
if (queued)
enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
@@ -10376,12 +10376,17 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
*/
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
- struct cpumask *mm_allowed = mm_cpus_allowed(mm);
+ struct cpumask *mm_allowed;
if (!mm)
return;
- /* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
+
+ /*
+ * mm::mm_cid::mm_cpus_allowed is the superset of each threads
+ * allowed CPUs mask which means it can only grow.
+ */
guard(raw_spinlock)(&mm->mm_cid.lock);
+ mm_allowed = mm_cpus_allowed(mm);
cpumask_or(mm_allowed, mm_allowed, affmsk);
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
}CID management OR's two cpumasks and then calculates the weight on the
result. That's inefficient as that has to walk the same stuff twice. As
this is done with runqueue lock held, there is a real benefit of speeding
this up. Depending on the system this results in 10-20% less cycles spent
with runqueue lock held for a 4K cpumask.
Provide cpumask_weighted_or() and the corresponding bitmap functions which
return the weight of the OR result right away.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
V3: Rename again - Yury
V2: Rename and use the BITMAP_WEIGHT() macro - Yury
---
include/linux/bitmap.h | 15 +++++++++++++++
include/linux/cpumask.h | 16 ++++++++++++++++
lib/bitmap.c | 6 ++++++
3 files changed, 37 insertions(+)
--- a/include/linux/bitmap.h
+++ b/include/linux/bitmap.h
@@ -45,6 +45,7 @@ struct device;
* bitmap_copy(dst, src, nbits) *dst = *src
* bitmap_and(dst, src1, src2, nbits) *dst = *src1 & *src2
* bitmap_or(dst, src1, src2, nbits) *dst = *src1 | *src2
+ * bitmap_weighted_or(dst, src1, src2, nbits) *dst = *src1 | *src2. Returns Hamming Weight of dst
* bitmap_xor(dst, src1, src2, nbits) *dst = *src1 ^ *src2
* bitmap_andnot(dst, src1, src2, nbits) *dst = *src1 & ~(*src2)
* bitmap_complement(dst, src, nbits) *dst = ~(*src)
@@ -165,6 +166,8 @@ bool __bitmap_and(unsigned long *dst, co
const unsigned long *bitmap2, unsigned int nbits);
void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
+unsigned int __bitmap_weighted_or(unsigned long *dst, const unsigned long *bitmap1,
+ const unsigned long *bitmap2, unsigned int nbits);
void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
@@ -338,6 +341,18 @@ void bitmap_or(unsigned long *dst, const
}
static __always_inline
+unsigned int bitmap_weighted_or(unsigned long *dst, const unsigned long *src1,
+ const unsigned long *src2, unsigned int nbits)
+{
+ if (small_const_nbits(nbits)) {
+ *dst = *src1 | *src2;
+ return hweight_long(*dst & BITMAP_LAST_WORD_MASK(nbits));
+ } else {
+ return __bitmap_weighted_or(dst, src1, src2, nbits);
+ }
+}
+
+static __always_inline
void bitmap_xor(unsigned long *dst, const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -729,6 +729,22 @@ void cpumask_or(struct cpumask *dstp, co
}
/**
+ * cpumask_weighted_or - *dstp = *src1p | *src2p and return the weight of the result
+ * @dstp: the cpumask result
+ * @src1p: the first input
+ * @src2p: the second input
+ *
+ * Return: The number of bits set in the resulting cpumask @dstp
+ */
+static __always_inline
+unsigned int cpumask_weighted_or(struct cpumask *dstp, const struct cpumask *src1p,
+ const struct cpumask *src2p)
+{
+ return bitmap_weighted_or(cpumask_bits(dstp), cpumask_bits(src1p),
+ cpumask_bits(src2p), small_cpumask_bits);
+}
+
+/**
* cpumask_xor - *dstp = *src1p ^ *src2p
* @dstp: the cpumask result
* @src1p: the first input
--- a/lib/bitmap.c
+++ b/lib/bitmap.c
@@ -355,6 +355,12 @@ unsigned int __bitmap_weight_andnot(cons
}
EXPORT_SYMBOL(__bitmap_weight_andnot);
+unsigned int __bitmap_weighted_or(unsigned long *dst, const unsigned long *bitmap1,
+ const unsigned long *bitmap2, unsigned int bits)
+{
+ return BITMAP_WEIGHT(({dst[idx] = bitmap1[idx] | bitmap2[idx]; dst[idx]; }), bits);
+}
+
void __bitmap_set(unsigned long *map, unsigned int start, int len)
{
unsigned long *p = map + BIT_WORD(start);The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 437cb3ded25038d5280d21de489ce78c745118d5
Gitweb: https://git.kernel.org/tip/437cb3ded25038d5280d21de489ce78c745118d5
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:57 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:54 +01:00
cpumask: Introduce cpumask_weighted_or()
CID management OR's two cpumasks and then calculates the weight on the
result. That's inefficient as that has to walk the same stuff twice. As
this is done with runqueue lock held, there is a real benefit of speeding
this up. Depending on the system this results in 10-20% less cycles spent
with runqueue lock held for a 4K cpumask.
Provide cpumask_weighted_or() and the corresponding bitmap functions which
return the weight of the OR result right away.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.448263340@linutronix.de
---
include/linux/bitmap.h | 15 +++++++++++++++
include/linux/cpumask.h | 16 ++++++++++++++++
lib/bitmap.c | 6 ++++++
3 files changed, 37 insertions(+)
diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h
index 595217b..b0395e4 100644
--- a/include/linux/bitmap.h
+++ b/include/linux/bitmap.h
@@ -45,6 +45,7 @@ struct device;
* bitmap_copy(dst, src, nbits) *dst = *src
* bitmap_and(dst, src1, src2, nbits) *dst = *src1 & *src2
* bitmap_or(dst, src1, src2, nbits) *dst = *src1 | *src2
+ * bitmap_weighted_or(dst, src1, src2, nbits) *dst = *src1 | *src2. Returns Hamming Weight of dst
* bitmap_xor(dst, src1, src2, nbits) *dst = *src1 ^ *src2
* bitmap_andnot(dst, src1, src2, nbits) *dst = *src1 & ~(*src2)
* bitmap_complement(dst, src, nbits) *dst = ~(*src)
@@ -165,6 +166,8 @@ bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
+unsigned int __bitmap_weighted_or(unsigned long *dst, const unsigned long *bitmap1,
+ const unsigned long *bitmap2, unsigned int nbits);
void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
@@ -338,6 +341,18 @@ void bitmap_or(unsigned long *dst, const unsigned long *src1,
}
static __always_inline
+unsigned int bitmap_weighted_or(unsigned long *dst, const unsigned long *src1,
+ const unsigned long *src2, unsigned int nbits)
+{
+ if (small_const_nbits(nbits)) {
+ *dst = *src1 | *src2;
+ return hweight_long(*dst & BITMAP_LAST_WORD_MASK(nbits));
+ } else {
+ return __bitmap_weighted_or(dst, src1, src2, nbits);
+ }
+}
+
+static __always_inline
void bitmap_xor(unsigned long *dst, const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index ff8f41a..feba06e 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -729,6 +729,22 @@ void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
}
/**
+ * cpumask_weighted_or - *dstp = *src1p | *src2p and return the weight of the result
+ * @dstp: the cpumask result
+ * @src1p: the first input
+ * @src2p: the second input
+ *
+ * Return: The number of bits set in the resulting cpumask @dstp
+ */
+static __always_inline
+unsigned int cpumask_weighted_or(struct cpumask *dstp, const struct cpumask *src1p,
+ const struct cpumask *src2p)
+{
+ return bitmap_weighted_or(cpumask_bits(dstp), cpumask_bits(src1p),
+ cpumask_bits(src2p), small_cpumask_bits);
+}
+
+/**
* cpumask_xor - *dstp = *src1p ^ *src2p
* @dstp: the cpumask result
* @src1p: the first input
diff --git a/lib/bitmap.c b/lib/bitmap.c
index b976928..9dc5265 100644
--- a/lib/bitmap.c
+++ b/lib/bitmap.c
@@ -355,6 +355,12 @@ unsigned int __bitmap_weight_andnot(const unsigned long *bitmap1,
}
EXPORT_SYMBOL(__bitmap_weight_andnot);
+unsigned int __bitmap_weighted_or(unsigned long *dst, const unsigned long *bitmap1,
+ const unsigned long *bitmap2, unsigned int bits)
+{
+ return BITMAP_WEIGHT(({dst[idx] = bitmap1[idx] | bitmap2[idx]; dst[idx]; }), bits);
+}
+
void __bitmap_set(unsigned long *map, unsigned int start, int len)
{
unsigned long *p = map + BIT_WORD(start);Use cpumask_weighted_or() instead of cpumask_or() and cpumask_weight() on
the result, which walks the same bitmap twice. Results in 10-20% less
cycles, which reduces the runqueue lock hold time.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Acked-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
kernel/sched/core.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10377,6 +10377,7 @@ void call_trace_sched_update_nr_running(
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
+ unsigned int weight;
if (!mm)
return;
@@ -10387,8 +10388,8 @@ static inline void mm_update_cpus_allowe
*/
guard(raw_spinlock)(&mm->mm_cid.lock);
mm_allowed = mm_cpus_allowed(mm);
- cpumask_or(mm_allowed, mm_allowed, affmsk);
- WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
+ weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
+ WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
}
void sched_mm_cid_exit_signals(struct task_struct *t)The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 79c11fb3da8581a2f222b290ce62a153ab1108fc
Gitweb: https://git.kernel.org/tip/79c11fb3da8581a2f222b290ce62a153ab1108fc
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:59 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:54 +01:00
sched/mmcid: Use cpumask_weighted_or()
Use cpumask_weighted_or() instead of cpumask_or() and cpumask_weight() on
the result, which walks the same bitmap twice. Results in 10-20% less
cycles, which reduces the runqueue lock hold time.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Acked-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
Link: https://patch.msgid.link/20251119172549.511736272@linutronix.de
---
kernel/sched/core.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 2ea77e7..f6bbfa1 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10377,6 +10377,7 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
+ unsigned int weight;
if (!mm)
return;
@@ -10387,8 +10388,8 @@ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpu
*/
guard(raw_spinlock)(&mm->mm_cid.lock);
mm_allowed = mm_cpus_allowed(mm);
- cpumask_or(mm_allowed, mm_allowed, affmsk);
- WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
+ weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
+ WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
}
void sched_mm_cid_exit_signals(struct task_struct *t)Reevaluating num_possible_cpus() over and over does not make sense. That
becomes a constant after init as cpu_possible_mask is marked ro_after_init.
Cache the value during initialization and provide that for consumption.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Yury Norov <yury.norov@gmail.com>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Reviewed-by: Shrikanth Hegde <sshegde@linux.ibm.com>
---
V4: Add comment why this is not marked __init ....
V2: New patch
---
include/linux/cpumask.h | 10 ++++++++--
kernel/cpu.c | 19 +++++++++++++++++++
2 files changed, 27 insertions(+), 2 deletions(-)
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -126,6 +126,7 @@ extern struct cpumask __cpu_dying_mask;
#define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
extern atomic_t __num_online_cpus;
+extern unsigned int __num_possible_cpus;
extern cpumask_t cpus_booted_once_mask;
@@ -1152,13 +1153,13 @@ void init_cpu_possible(const struct cpum
#define __assign_cpu(cpu, mask, val) \
__assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
-#define set_cpu_possible(cpu, possible) assign_cpu((cpu), &__cpu_possible_mask, (possible))
#define set_cpu_enabled(cpu, enabled) assign_cpu((cpu), &__cpu_enabled_mask, (enabled))
#define set_cpu_present(cpu, present) assign_cpu((cpu), &__cpu_present_mask, (present))
#define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active))
#define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying))
void set_cpu_online(unsigned int cpu, bool online);
+void set_cpu_possible(unsigned int cpu, bool possible);
/**
* to_cpumask - convert a NR_CPUS bitmap to a struct cpumask *
@@ -1211,7 +1212,12 @@ static __always_inline unsigned int num_
{
return raw_atomic_read(&__num_online_cpus);
}
-#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
+
+static __always_inline unsigned int num_possible_cpus(void)
+{
+ return __num_possible_cpus;
+}
+
#define num_enabled_cpus() cpumask_weight(cpu_enabled_mask)
#define num_present_cpus() cpumask_weight(cpu_present_mask)
#define num_active_cpus() cpumask_weight(cpu_active_mask)
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -3108,6 +3108,9 @@ EXPORT_SYMBOL(__cpu_dying_mask);
atomic_t __num_online_cpus __read_mostly;
EXPORT_SYMBOL(__num_online_cpus);
+unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
+EXPORT_SYMBOL(__num_possible_cpus);
+
void init_cpu_present(const struct cpumask *src)
{
cpumask_copy(&__cpu_present_mask, src);
@@ -3116,6 +3119,7 @@ void init_cpu_present(const struct cpuma
void init_cpu_possible(const struct cpumask *src)
{
cpumask_copy(&__cpu_possible_mask, src);
+ __num_possible_cpus = cpumask_weight(&__cpu_possible_mask);
}
void set_cpu_online(unsigned int cpu, bool online)
@@ -3139,6 +3143,21 @@ void set_cpu_online(unsigned int cpu, bo
}
}
+/*
+ * This should be marked __init, but there is a boatload of call sites
+ * which need to be fixed up to do so. Sigh...
+ */
+void set_cpu_possible(unsigned int cpu, bool possible)
+{
+ if (possible) {
+ if (!cpumask_test_and_set_cpu(cpu, &__cpu_possible_mask))
+ __num_possible_cpus++;
+ } else {
+ if (cpumask_test_and_clear_cpu(cpu, &__cpu_possible_mask))
+ __num_possible_cpus--;
+ }
+}
+
/*
* Activate the first processor.
*/Hi Thomas,
On Wed, Nov 19, 2025 at 06:27:01PM +0100, Thomas Gleixner wrote:
> Reevaluating num_possible_cpus() over and over does not make sense. That
> becomes a constant after init as cpu_possible_mask is marked ro_after_init.
>
> Cache the value during initialization and provide that for consumption.
>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Reviewed-by: Yury Norov <yury.norov@gmail.com>
> Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
> Reviewed-by: Shrikanth Hegde <sshegde@linux.ibm.com>
After this change in -next as commit d0f23ccf6ba9 ("cpumask: Cache
num_possible_cpus()"), I am seeing boot failures on MIPS and PowerPC in
QEMU.
For MIPS using malta_defconfig:
$ make -skj"$(nproc)" ARCH=mips CROSS_COMPILE=mips-linux- clean malta_defconfig vmlinux
$ curl -LSs https://github.com/ClangBuiltLinux/boot-utils/releases/download/20241120-044434/mipsel-rootfs.cpio.zst | zstd -d >rootfs.cpio
$ qemu-system-mipsel \
-display none \
-nodefaults \
-cpu 24Kf \
-machine malta \
-kernel vmlinux \
-initrd rootfs.cpio \
-m 512m \
-serial mon:stdio
...
Linux version 6.18.0-rc4-00055-gd0f23ccf6ba9 (nathan@ax162) (mips-linux-gcc (GCC) 15.2.0, GNU ld (GNU Binutils) 2.45) #1 SMP Fri Nov 21 17:16:39 MST 2025
...
Kernel bug detected[#1]:
CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.18.0-rc4-00055-gd0f23ccf6ba9 #1 NONE
Hardware name: mti,malta
$ 0 : 00000000 00000001 00000001 00000008
$ 4 : 00000044 82090000 00000000 00000001
$ 8 : 00000004 808e1b98 80140000 80140000
$12 : 00000121 00000001 00000122 82090080
$16 : 00000000 80a17c7c 80a80000 80b10000
$20 : 00000000 00000000 00000000 00000000
$24 : 00000008 82095780
$28 : 82084000 82087db0 00000000 80122358
Hi : 51eb5fc3
Lo : 28f5a1df
epc : 80122360 mips_smp_ipi_allocate+0x200/0x214
ra : 80122358 mips_smp_ipi_allocate+0x1f8/0x214
Status: 1000a403 KERNEL EXL IE
Cause : 00800034 (ExcCode 0d)
PrId : 00019300 (MIPS 24Kc)
Modules linked in:
Process swapper/0 (pid: 1, threadinfo=(ptrval), task=(ptrval), tls=00000000)
Stack : 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
00000000 9cc98073 80abb628 80a80000 00000000 80b10000 00000000 80abb650
8200e500 80abb628 80a80000 00000000 80b10000 80110144 8200e500 808f5ae0
...
Call Trace:
[<80122360>] mips_smp_ipi_allocate+0x200/0x214
[<80abb650>] _gp+0x10/0x50
[<80110144>] do_one_initcall+0x64/0x2d0
[<80ab4fc8>] kernel_init_freeable+0x100/0x2f0
[<808e3980>] kernel_init+0x24/0x118
[<80112258>] ret_from_kernel_thread+0x14/0x1c
Code: 9044ac0c 2c420002 38420001 <00020336> 1000ffd1 00000000 0c238c2c 00000000 27bdff80
---[ end trace 0000000000000000 ]---
...
For PowerPC using Debian's configuration:
$ curl -LSs https://github.com/nathanchance/llvm-kernel-testing/raw/61ffff42ede84576a9f8721d06765d8de36a912f/configs/debian/powerpc64le.config >.config
$ make -skj"$(nproc)" ARCH=powerpc CROSS_COMPILE=powerpc64-linux- olddefconfig zImage.epapr
$ curl -LSs https://github.com/ClangBuiltLinux/boot-utils/releases/download/20241120-044434/ppc64le-rootfs.cpio.zst | zstd -d >rootfs.cpio
$ qemu-system-ppc64 \
-display none \
-nodefaults \
-device ipmi-bmc-sim,id=bmc0 \
-device isa-ipmi-bt,bmc=bmc0,irq=10 \
-machine powernv \
-kernel arch/powerpc/boot/zImage.epapr \
-initrd rootfs.cpio \
-m 2G \
-serial mon:stdio
...
[ 0.000000] Linux version 6.18.0-rc4+ (nathan@ax162) (powerpc64-linux-gcc (GCC) 15.2.0, GNU ld (GNU Binutils) 2.45) #1 SMP PREEMPT Fri Nov 21 17:05:23 MST 2025
...
[ 0.615917] BUG: Kernel NULL pointer dereference on read at 0x00000130
[ 0.616333] Faulting instruction address: 0xc000000000328804
[ 0.616885] Oops: Kernel access of bad area, sig: 7 [#1]
[ 0.617185] LE PAGE_SIZE=4K MMU=Radix SMP NR_CPUS=2048 NUMA PowerNV
[ 0.617618] Modules linked in:
[ 0.618051] CPU: 0 UID: 0 PID: 23 Comm: rcu_tasks_kthre Not tainted 6.18.0-rc4+ #1 PREEMPTLAZY
[ 0.618557] Hardware name: IBM PowerNV (emulated by qemu) POWER10 0x801200 opal:v7.1-106-g785a5e307 PowerNV
[ 0.619137] NIP: c000000000328804 LR: c00000000032a9f0 CTR: 0000000000000000
[ 0.619478] REGS: c0000000034ffb00 TRAP: 0300 Not tainted (6.18.0-rc4+)
[ 0.619817] MSR: 9000000000009033 <SF,HV,EE,ME,IR,DR,RI,LE> CR: 84000884 XER: 20040000
[ 0.620396] CFAR: c00000000032a9ec DAR: 0000000000000130 DSISR: 00080000 IRQMASK: 0
[ 0.620396] GPR00: c00000000032a9ac c0000000034ffda0 c000000001cb7100 c000000002764430
[ 0.620396] GPR04: c00000007f33fb90 0000000000000000 1db5beac45697d00 c00000000344aa00
[ 0.620396] GPR08: c000000003238000 0000000000000008 0000000000000001 0000000000000002
[ 0.620396] GPR12: 0000000000000000 c000000002a7d000 c000000000269438 c00000000349bc40
[ 0.620396] GPR16: 0000000000000000 0000000000000000 c00000000290ea80 0000000000000000
[ 0.620396] GPR20: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 0.620396] GPR24: c00000000291e460 c000000000328270 c00000000291e460 c000000001779218
[ 0.620396] GPR28: c000000002764430 0000000000000000 c00000007f33fb90 c0000000034ffdd0
[ 0.623797] NIP [c000000000328804] rcu_tasks_invoke_cbs+0x84/0x320
[ 0.624497] LR [c00000000032a9f0] rcu_tasks_one_gp+0x450/0x680
[ 0.624851] Call Trace:
[ 0.625036] [c0000000034ffda0] [c00000000290ea80] jiffies+0x0/0x80 (unreliable)
[ 0.625517] [c0000000034ffe20] [c00000000032a9ac] rcu_tasks_one_gp+0x40c/0x680
[ 0.625869] [c0000000034fff20] [c00000000032aebc] rcu_tasks_kthread+0x10c/0x140
[ 0.626294] [c0000000034fff90] [c0000000002695bc] kthread+0x18c/0x1a0
[ 0.626608] [c0000000034fffe0] [c00000000000de58] start_kernel_thread+0x14/0x18
[ 0.627031] Code: f9210030 fbe10038 f9210040 57bd083c 393d0001 7c095040 40800028 e90300c0 7d2907b4 814300d0 79291f24 7ca8482a <81250130> 7c2004ac 7c095000 418001a0
[ 0.628259] ---[ end trace 0000000000000000 ]---
...
At the parent change, neither of these problems exist. If there is any
other information I can provide or patches I can test, please let me
know.
Cheers,
Nathan
Hi
On 19.11.2025 18:27, Thomas Gleixner wrote:
> Reevaluating num_possible_cpus() over and over does not make sense. That
> becomes a constant after init as cpu_possible_mask is marked ro_after_init.
>
> Cache the value during initialization and provide that for consumption.
>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Reviewed-by: Yury Norov <yury.norov@gmail.com>
> Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
> Reviewed-by: Shrikanth Hegde <sshegde@linux.ibm.com>
This patch landed recently in linux-next as commit d0f23ccf6ba9
("cpumask: Cache num_possible_cpus()"). I found that it triggers the
following warning during boot on some of my test systems (namely
Raspberry Pi 3 and 4):
kvm [1]: nv: 568 coarse grained trap handlers
kvm [1]: IPA Size Limit: 40 bits
------------[ cut here ]------------
WARNING: CPU: 1 PID: 1 at arch/arm64/kvm/vmid.c:183
kvm_arm_vmid_alloc_init+0x98/0xac
Modules linked in:
CPU: 1 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.18.0-rc4+ #16195 PREEMPT
Hardware name: Raspberry Pi 3 Model B (DT)
pstate: 20000005 (nzCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
pc : kvm_arm_vmid_alloc_init+0x98/0xac
lr : kvm_arm_vmid_alloc_init+0x1c/0xac
...
Call trace:
kvm_arm_vmid_alloc_init+0x98/0xac (P)
kvm_arm_init+0x144/0x153c
do_one_initcall+0x64/0x308
kernel_init_freeable+0x280/0x4fc
kernel_init+0x20/0x1d8
ret_from_fork+0x10/0x20
irq event stamp: 65606
hardirqs last enabled at (65605): [<ffff8000801534e4>]
__up_console_sem+0x6c/0x80
hardirqs last disabled at (65606): [<ffff800081358f7c>] el1_brk64+0x20/0x60
softirqs last enabled at (65262): [<ffff8000800c4b1c>]
handle_softirqs+0x4c4/0x4dc
softirqs last disabled at (65257): [<ffff8000800106a0>]
__do_softirq+0x14/0x20
---[ end trace 0000000000000000 ]---
kvm [1]: Hyp nVHE mode initialized successfully
Reverting it on top of linux-next fixes the issue. Let me know how can I
help debugging it.
> ---
> V4: Add comment why this is not marked __init ....
> V2: New patch
> ---
> include/linux/cpumask.h | 10 ++++++++--
> kernel/cpu.c | 19 +++++++++++++++++++
> 2 files changed, 27 insertions(+), 2 deletions(-)
> --- a/include/linux/cpumask.h
> +++ b/include/linux/cpumask.h
> @@ -126,6 +126,7 @@ extern struct cpumask __cpu_dying_mask;
> #define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
>
> extern atomic_t __num_online_cpus;
> +extern unsigned int __num_possible_cpus;
>
> extern cpumask_t cpus_booted_once_mask;
>
> @@ -1152,13 +1153,13 @@ void init_cpu_possible(const struct cpum
> #define __assign_cpu(cpu, mask, val) \
> __assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
>
> -#define set_cpu_possible(cpu, possible) assign_cpu((cpu), &__cpu_possible_mask, (possible))
> #define set_cpu_enabled(cpu, enabled) assign_cpu((cpu), &__cpu_enabled_mask, (enabled))
> #define set_cpu_present(cpu, present) assign_cpu((cpu), &__cpu_present_mask, (present))
> #define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active))
> #define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying))
>
> void set_cpu_online(unsigned int cpu, bool online);
> +void set_cpu_possible(unsigned int cpu, bool possible);
>
> /**
> * to_cpumask - convert a NR_CPUS bitmap to a struct cpumask *
> @@ -1211,7 +1212,12 @@ static __always_inline unsigned int num_
> {
> return raw_atomic_read(&__num_online_cpus);
> }
> -#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
> +
> +static __always_inline unsigned int num_possible_cpus(void)
> +{
> + return __num_possible_cpus;
> +}
> +
> #define num_enabled_cpus() cpumask_weight(cpu_enabled_mask)
> #define num_present_cpus() cpumask_weight(cpu_present_mask)
> #define num_active_cpus() cpumask_weight(cpu_active_mask)
> --- a/kernel/cpu.c
> +++ b/kernel/cpu.c
> @@ -3108,6 +3108,9 @@ EXPORT_SYMBOL(__cpu_dying_mask);
> atomic_t __num_online_cpus __read_mostly;
> EXPORT_SYMBOL(__num_online_cpus);
>
> +unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
> +EXPORT_SYMBOL(__num_possible_cpus);
> +
> void init_cpu_present(const struct cpumask *src)
> {
> cpumask_copy(&__cpu_present_mask, src);
> @@ -3116,6 +3119,7 @@ void init_cpu_present(const struct cpuma
> void init_cpu_possible(const struct cpumask *src)
> {
> cpumask_copy(&__cpu_possible_mask, src);
> + __num_possible_cpus = cpumask_weight(&__cpu_possible_mask);
> }
>
> void set_cpu_online(unsigned int cpu, bool online)
> @@ -3139,6 +3143,21 @@ void set_cpu_online(unsigned int cpu, bo
> }
> }
>
> +/*
> + * This should be marked __init, but there is a boatload of call sites
> + * which need to be fixed up to do so. Sigh...
> + */
> +void set_cpu_possible(unsigned int cpu, bool possible)
> +{
> + if (possible) {
> + if (!cpumask_test_and_set_cpu(cpu, &__cpu_possible_mask))
> + __num_possible_cpus++;
> + } else {
> + if (cpumask_test_and_clear_cpu(cpu, &__cpu_possible_mask))
> + __num_possible_cpus--;
> + }
> +}
> +
> /*
> * Activate the first processor.
> */
>
>
>
Best regards
--
Marek Szyprowski, PhD
Samsung R&D Institute Poland
On Fri, Nov 21, 2025 at 11:56:44PM +0100, Marek Szyprowski wrote:
> Hi
>
> On 19.11.2025 18:27, Thomas Gleixner wrote:
> > Reevaluating num_possible_cpus() over and over does not make sense. That
> > becomes a constant after init as cpu_possible_mask is marked ro_after_init.
> >
> > Cache the value during initialization and provide that for consumption.
> >
> > Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> > Reviewed-by: Yury Norov <yury.norov@gmail.com>
> > Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
> > Reviewed-by: Shrikanth Hegde <sshegde@linux.ibm.com>
>
> This patch landed recently in linux-next as commit d0f23ccf6ba9
> ("cpumask: Cache num_possible_cpus()"). I found that it triggers the
> following warning during boot on some of my test systems (namely
> Raspberry Pi 3 and 4):
I also bisected to this commit on an ARM Neoverse V2. It happens twice
at about the same time, so the splats are interleaved, but they start
with an almost-NULL pointer dereference during either TASKS01 or TRACE02
callback processing. Things go surprisingly normally after that, at
least until a hang on shutdown. This sometimes happens quite late, as in
*minutes* after boot.
I do not see this on x86, nor on any rcutorture scenarios other
than TASKS01 and TRACE02. These are unusual in that they are the
only variants of the common-code tasks-RCU flavors that build with
CONFIG_PROVE_LOCKING=y.
My reproducer is:
tools/testing/selftests/rcutorture/bin/torture.sh --duration 10 --do-none --do-rcutorture --configs-rcutorture "TRACE01 TRACE02"
My current guess is that the snapshot is taken too early, though I would
be more confident of that if it happened on TREE01, in which CPUs come
online quite late.
Thanx, Paul
> kvm [1]: nv: 568 coarse grained trap handlers
> kvm [1]: IPA Size Limit: 40 bits
> ------------[ cut here ]------------
> WARNING: CPU: 1 PID: 1 at arch/arm64/kvm/vmid.c:183
> kvm_arm_vmid_alloc_init+0x98/0xac
> Modules linked in:
> CPU: 1 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.18.0-rc4+ #16195 PREEMPT
> Hardware name: Raspberry Pi 3 Model B (DT)
> pstate: 20000005 (nzCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
> pc : kvm_arm_vmid_alloc_init+0x98/0xac
> lr : kvm_arm_vmid_alloc_init+0x1c/0xac
> ...
> Call trace:
> kvm_arm_vmid_alloc_init+0x98/0xac (P)
> kvm_arm_init+0x144/0x153c
> do_one_initcall+0x64/0x308
> kernel_init_freeable+0x280/0x4fc
> kernel_init+0x20/0x1d8
> ret_from_fork+0x10/0x20
> irq event stamp: 65606
> hardirqs last enabled at (65605): [<ffff8000801534e4>]
> __up_console_sem+0x6c/0x80
> hardirqs last disabled at (65606): [<ffff800081358f7c>] el1_brk64+0x20/0x60
> softirqs last enabled at (65262): [<ffff8000800c4b1c>]
> handle_softirqs+0x4c4/0x4dc
> softirqs last disabled at (65257): [<ffff8000800106a0>]
> __do_softirq+0x14/0x20
> ---[ end trace 0000000000000000 ]---
> kvm [1]: Hyp nVHE mode initialized successfully
>
>
> Reverting it on top of linux-next fixes the issue. Let me know how can I
> help debugging it.
>
>
> > ---
> > V4: Add comment why this is not marked __init ....
> > V2: New patch
> > ---
> > include/linux/cpumask.h | 10 ++++++++--
> > kernel/cpu.c | 19 +++++++++++++++++++
> > 2 files changed, 27 insertions(+), 2 deletions(-)
> > --- a/include/linux/cpumask.h
> > +++ b/include/linux/cpumask.h
> > @@ -126,6 +126,7 @@ extern struct cpumask __cpu_dying_mask;
> > #define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
> >
> > extern atomic_t __num_online_cpus;
> > +extern unsigned int __num_possible_cpus;
> >
> > extern cpumask_t cpus_booted_once_mask;
> >
> > @@ -1152,13 +1153,13 @@ void init_cpu_possible(const struct cpum
> > #define __assign_cpu(cpu, mask, val) \
> > __assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
> >
> > -#define set_cpu_possible(cpu, possible) assign_cpu((cpu), &__cpu_possible_mask, (possible))
> > #define set_cpu_enabled(cpu, enabled) assign_cpu((cpu), &__cpu_enabled_mask, (enabled))
> > #define set_cpu_present(cpu, present) assign_cpu((cpu), &__cpu_present_mask, (present))
> > #define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active))
> > #define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying))
> >
> > void set_cpu_online(unsigned int cpu, bool online);
> > +void set_cpu_possible(unsigned int cpu, bool possible);
> >
> > /**
> > * to_cpumask - convert a NR_CPUS bitmap to a struct cpumask *
> > @@ -1211,7 +1212,12 @@ static __always_inline unsigned int num_
> > {
> > return raw_atomic_read(&__num_online_cpus);
> > }
> > -#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
> > +
> > +static __always_inline unsigned int num_possible_cpus(void)
> > +{
> > + return __num_possible_cpus;
> > +}
> > +
> > #define num_enabled_cpus() cpumask_weight(cpu_enabled_mask)
> > #define num_present_cpus() cpumask_weight(cpu_present_mask)
> > #define num_active_cpus() cpumask_weight(cpu_active_mask)
> > --- a/kernel/cpu.c
> > +++ b/kernel/cpu.c
> > @@ -3108,6 +3108,9 @@ EXPORT_SYMBOL(__cpu_dying_mask);
> > atomic_t __num_online_cpus __read_mostly;
> > EXPORT_SYMBOL(__num_online_cpus);
> >
> > +unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
> > +EXPORT_SYMBOL(__num_possible_cpus);
> > +
> > void init_cpu_present(const struct cpumask *src)
> > {
> > cpumask_copy(&__cpu_present_mask, src);
> > @@ -3116,6 +3119,7 @@ void init_cpu_present(const struct cpuma
> > void init_cpu_possible(const struct cpumask *src)
> > {
> > cpumask_copy(&__cpu_possible_mask, src);
> > + __num_possible_cpus = cpumask_weight(&__cpu_possible_mask);
> > }
> >
> > void set_cpu_online(unsigned int cpu, bool online)
> > @@ -3139,6 +3143,21 @@ void set_cpu_online(unsigned int cpu, bo
> > }
> > }
> >
> > +/*
> > + * This should be marked __init, but there is a boatload of call sites
> > + * which need to be fixed up to do so. Sigh...
> > + */
> > +void set_cpu_possible(unsigned int cpu, bool possible)
> > +{
> > + if (possible) {
> > + if (!cpumask_test_and_set_cpu(cpu, &__cpu_possible_mask))
> > + __num_possible_cpus++;
> > + } else {
> > + if (cpumask_test_and_clear_cpu(cpu, &__cpu_possible_mask))
> > + __num_possible_cpus--;
> > + }
> > +}
> > +
> > /*
> > * Activate the first processor.
> > */
> >
> >
> >
> Best regards
> --
> Marek Szyprowski, PhD
> Samsung R&D Institute Poland
>
On Sat, Nov 22 2025 at 10:47, Paul E. McKenney wrote: > On Fri, Nov 21, 2025 at 11:56:44PM +0100, Marek Szyprowski wrote: >> On 19.11.2025 18:27, Thomas Gleixner wrote: > My current guess is that the snapshot is taken too early, though I would > be more confident of that if it happened on TREE01, in which CPUs come > online quite late. https://lore.kernel.org/all/87zf8ehyf7.ffs@tglx
On Fri, Nov 21 2025 at 23:56, Marek Szyprowski wrote:
> Reverting it on top of linux-next fixes the issue. Let me know how can I
> help debugging it.
Can you test the fix below please?
Thanks,
tglx
---
Subject: cpu: Initialize __num_possible_cpus correctly
From: Thomas Gleixner <tglx@linutronix.de>
Date: Sat, 22 Nov 2025 16:19:18 +0100
The variable to cache the number of possible CPUs is initialized to NR_CPUS
at build time, but that's only correct when cpu_possible_mask is
initialized with CPU_BITS_ALL. That's only the case on PARISC.
On x86 and some other architectures this does not matter because they
initialize cpu_possible_mask via init_cpu_possible() which does a proper
weight calculation. Though on architectures which do not, this results
in a completely wrong cached value 'NR_CPUS + actual possible CPUs'.
Initialize it correctly to 0 when CONFIG_INIT_ALL_POSSIBLE=n and move the
NR_CPUS initialization into the PARISC specific section.
Fixes: d0f23ccf6ba9 ("cpumask: Cache num_possible_cpus()")
Reported-by: Marek Szyprowski <m.szyprowski@samsung.com>
Reported-by: Nathan Chancellor <nathan@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Closes: https://lore.kernel.org/all/89c7106e-a431-443a-9527-3d5fbce77fe1@samsung.com
Closes: https://lore.kernel.org/all/20251122002755.GA2682494@ax162
---
kernel/cpu.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -3085,10 +3085,13 @@ EXPORT_SYMBOL(cpu_all_bits);
#ifdef CONFIG_INIT_ALL_POSSIBLE
struct cpumask __cpu_possible_mask __ro_after_init
= {CPU_BITS_ALL};
+unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
#else
struct cpumask __cpu_possible_mask __ro_after_init;
+unsigned int __num_possible_cpus __ro_after_init;
#endif
EXPORT_SYMBOL(__cpu_possible_mask);
+EXPORT_SYMBOL(__num_possible_cpus);
struct cpumask __cpu_online_mask __read_mostly;
EXPORT_SYMBOL(__cpu_online_mask);
@@ -3108,9 +3111,6 @@ EXPORT_SYMBOL(__cpu_dying_mask);
atomic_t __num_online_cpus __read_mostly;
EXPORT_SYMBOL(__num_online_cpus);
-unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
-EXPORT_SYMBOL(__num_possible_cpus);
-
void init_cpu_present(const struct cpumask *src)
{
cpumask_copy(&__cpu_present_mask, src);
On 22.11.2025 16:36, Thomas Gleixner wrote:
> On Fri, Nov 21 2025 at 23:56, Marek Szyprowski wrote:
>> Reverting it on top of linux-next fixes the issue. Let me know how can I
>> help debugging it.
> Can you test the fix below please?
This fixes the observed issue. Thanks!
Tested-by: Marek Szyprowski <m.szyprowski@samsung.com>
> Thanks,
>
> tglx
> ---
> Subject: cpu: Initialize __num_possible_cpus correctly
> From: Thomas Gleixner <tglx@linutronix.de>
> Date: Sat, 22 Nov 2025 16:19:18 +0100
>
> The variable to cache the number of possible CPUs is initialized to NR_CPUS
> at build time, but that's only correct when cpu_possible_mask is
> initialized with CPU_BITS_ALL. That's only the case on PARISC.
>
> On x86 and some other architectures this does not matter because they
> initialize cpu_possible_mask via init_cpu_possible() which does a proper
> weight calculation. Though on architectures which do not, this results
> in a completely wrong cached value 'NR_CPUS + actual possible CPUs'.
>
> Initialize it correctly to 0 when CONFIG_INIT_ALL_POSSIBLE=n and move the
> NR_CPUS initialization into the PARISC specific section.
>
> Fixes: d0f23ccf6ba9 ("cpumask: Cache num_possible_cpus()")
> Reported-by: Marek Szyprowski <m.szyprowski@samsung.com>
> Reported-by: Nathan Chancellor <nathan@kernel.org>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Closes: https://lore.kernel.org/all/89c7106e-a431-443a-9527-3d5fbce77fe1@samsung.com
> Closes: https://lore.kernel.org/all/20251122002755.GA2682494@ax162
> ---
> kernel/cpu.c | 6 +++---
> 1 file changed, 3 insertions(+), 3 deletions(-)
>
> --- a/kernel/cpu.c
> +++ b/kernel/cpu.c
> @@ -3085,10 +3085,13 @@ EXPORT_SYMBOL(cpu_all_bits);
> #ifdef CONFIG_INIT_ALL_POSSIBLE
> struct cpumask __cpu_possible_mask __ro_after_init
> = {CPU_BITS_ALL};
> +unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
> #else
> struct cpumask __cpu_possible_mask __ro_after_init;
> +unsigned int __num_possible_cpus __ro_after_init;
> #endif
> EXPORT_SYMBOL(__cpu_possible_mask);
> +EXPORT_SYMBOL(__num_possible_cpus);
>
> struct cpumask __cpu_online_mask __read_mostly;
> EXPORT_SYMBOL(__cpu_online_mask);
> @@ -3108,9 +3111,6 @@ EXPORT_SYMBOL(__cpu_dying_mask);
> atomic_t __num_online_cpus __read_mostly;
> EXPORT_SYMBOL(__num_online_cpus);
>
> -unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
> -EXPORT_SYMBOL(__num_possible_cpus);
> -
> void init_cpu_present(const struct cpumask *src)
> {
> cpumask_copy(&__cpu_present_mask, src);
>
Best regards
--
Marek Szyprowski, PhD
Samsung R&D Institute Poland
On Sat, Nov 22, 2025 at 05:24:22PM +0100, Marek Szyprowski wrote:
>
> On 22.11.2025 16:36, Thomas Gleixner wrote:
> > On Fri, Nov 21 2025 at 23:56, Marek Szyprowski wrote:
> >> Reverting it on top of linux-next fixes the issue. Let me know how can I
> >> help debugging it.
> > Can you test the fix below please?
>
> This fixes the observed issue. Thanks!
>
> Tested-by: Marek Szyprowski <m.szyprowski@samsung.com>
And I clearly should have read ahead, but thank you for the quick fix!
Tested-by: Paul E. McKenney <paulmck@kernel.org>
> > Thanks,
> >
> > tglx
> > ---
> > Subject: cpu: Initialize __num_possible_cpus correctly
> > From: Thomas Gleixner <tglx@linutronix.de>
> > Date: Sat, 22 Nov 2025 16:19:18 +0100
> >
> > The variable to cache the number of possible CPUs is initialized to NR_CPUS
> > at build time, but that's only correct when cpu_possible_mask is
> > initialized with CPU_BITS_ALL. That's only the case on PARISC.
> >
> > On x86 and some other architectures this does not matter because they
> > initialize cpu_possible_mask via init_cpu_possible() which does a proper
> > weight calculation. Though on architectures which do not, this results
> > in a completely wrong cached value 'NR_CPUS + actual possible CPUs'.
> >
> > Initialize it correctly to 0 when CONFIG_INIT_ALL_POSSIBLE=n and move the
> > NR_CPUS initialization into the PARISC specific section.
> >
> > Fixes: d0f23ccf6ba9 ("cpumask: Cache num_possible_cpus()")
> > Reported-by: Marek Szyprowski <m.szyprowski@samsung.com>
> > Reported-by: Nathan Chancellor <nathan@kernel.org>
> > Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> > Closes: https://lore.kernel.org/all/89c7106e-a431-443a-9527-3d5fbce77fe1@samsung.com
> > Closes: https://lore.kernel.org/all/20251122002755.GA2682494@ax162
> > ---
> > kernel/cpu.c | 6 +++---
> > 1 file changed, 3 insertions(+), 3 deletions(-)
> >
> > --- a/kernel/cpu.c
> > +++ b/kernel/cpu.c
> > @@ -3085,10 +3085,13 @@ EXPORT_SYMBOL(cpu_all_bits);
> > #ifdef CONFIG_INIT_ALL_POSSIBLE
> > struct cpumask __cpu_possible_mask __ro_after_init
> > = {CPU_BITS_ALL};
> > +unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
> > #else
> > struct cpumask __cpu_possible_mask __ro_after_init;
> > +unsigned int __num_possible_cpus __ro_after_init;
> > #endif
> > EXPORT_SYMBOL(__cpu_possible_mask);
> > +EXPORT_SYMBOL(__num_possible_cpus);
> >
> > struct cpumask __cpu_online_mask __read_mostly;
> > EXPORT_SYMBOL(__cpu_online_mask);
> > @@ -3108,9 +3111,6 @@ EXPORT_SYMBOL(__cpu_dying_mask);
> > atomic_t __num_online_cpus __read_mostly;
> > EXPORT_SYMBOL(__num_online_cpus);
> >
> > -unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
> > -EXPORT_SYMBOL(__num_possible_cpus);
> > -
> > void init_cpu_present(const struct cpumask *src)
> > {
> > cpumask_copy(&__cpu_present_mask, src);
> >
> Best regards
> --
> Marek Szyprowski, PhD
> Samsung R&D Institute Poland
>
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 21782b3a5cd40892cb2995aa1ec3e74dd1112f1d
Gitweb: https://git.kernel.org/tip/21782b3a5cd40892cb2995aa1ec3e74dd1112f1d
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Sat, 22 Nov 2025 16:19:18 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Sun, 23 Nov 2025 19:59:30 +01:00
cpu: Initialize __num_possible_cpus correctly
The variable to cache the number of possible CPUs is initialized to NR_CPUS
at build time, but that's only correct when cpu_possible_mask is
initialized with CPU_BITS_ALL. That's only the case on PARISC.
On x86 and some other architectures this does not matter because they
initialize cpu_possible_mask via init_cpu_possible() which does a proper
weight calculation. Though on architectures which do not, this results
in a completely wrong cached value 'NR_CPUS + actual possible CPUs'.
Initialize it correctly to 0 when CONFIG_INIT_ALL_POSSIBLE=n and move the
NR_CPUS initialization into the PARISC specific section.
Fixes: d0f23ccf6ba9 ("cpumask: Cache num_possible_cpus()")
Reported-by: Marek Szyprowski <m.szyprowski@samsung.com>
Reported-by: Nathan Chancellor <nathan@kernel.org>
Reported-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Marek Szyprowski <m.szyprowski@samsung.com>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://patch.msgid.link/87zf8ehyf7.ffs@tglx
Closes: https://lore.kernel.org/all/89c7106e-a431-443a-9527-3d5fbce77fe1@samsung.com
Closes: https://lore.kernel.org/all/20251122002755.GA2682494@ax162
Closes: https://lore.kernel.org/all/ad2a55f9-9bca-41bf-a6ec-efb23eaa778f@paulmck-laptop
---
kernel/cpu.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 18e530d..b674fdf 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -3085,10 +3085,13 @@ EXPORT_SYMBOL(cpu_all_bits);
#ifdef CONFIG_INIT_ALL_POSSIBLE
struct cpumask __cpu_possible_mask __ro_after_init
= {CPU_BITS_ALL};
+unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
#else
struct cpumask __cpu_possible_mask __ro_after_init;
+unsigned int __num_possible_cpus __ro_after_init;
#endif
EXPORT_SYMBOL(__cpu_possible_mask);
+EXPORT_SYMBOL(__num_possible_cpus);
struct cpumask __cpu_online_mask __read_mostly;
EXPORT_SYMBOL(__cpu_online_mask);
@@ -3108,9 +3111,6 @@ EXPORT_SYMBOL(__cpu_dying_mask);
atomic_t __num_online_cpus __read_mostly;
EXPORT_SYMBOL(__num_online_cpus);
-unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
-EXPORT_SYMBOL(__num_possible_cpus);
-
void init_cpu_present(const struct cpumask *src)
{
cpumask_copy(&__cpu_present_mask, src);The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 35a5c37cb9f1f947dff18e7cfc75a8cfcfd557ca
Gitweb: https://git.kernel.org/tip/35a5c37cb9f1f947dff18e7cfc75a8cfcfd557ca
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:01 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:40 +01:00
cpumask: Cache num_possible_cpus()
Reevaluating num_possible_cpus() over and over does not make sense. That
becomes a constant after init as cpu_possible_mask is marked ro_after_init.
Cache the value during initialization and provide that for consumption.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Yury Norov <yury.norov@gmail.com>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Reviewed-by: Shrikanth Hegde <sshegde@linux.ibm.com>
Link: https://patch.msgid.link/20251119172549.578653738@linutronix.de
---
include/linux/cpumask.h | 10 ++++++++--
kernel/cpu.c | 19 +++++++++++++++++++
2 files changed, 27 insertions(+), 2 deletions(-)
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index feba06e..66694ee 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -126,6 +126,7 @@ extern struct cpumask __cpu_dying_mask;
#define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
extern atomic_t __num_online_cpus;
+extern unsigned int __num_possible_cpus;
extern cpumask_t cpus_booted_once_mask;
@@ -1152,13 +1153,13 @@ void init_cpu_possible(const struct cpumask *src);
#define __assign_cpu(cpu, mask, val) \
__assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
-#define set_cpu_possible(cpu, possible) assign_cpu((cpu), &__cpu_possible_mask, (possible))
#define set_cpu_enabled(cpu, enabled) assign_cpu((cpu), &__cpu_enabled_mask, (enabled))
#define set_cpu_present(cpu, present) assign_cpu((cpu), &__cpu_present_mask, (present))
#define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active))
#define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying))
void set_cpu_online(unsigned int cpu, bool online);
+void set_cpu_possible(unsigned int cpu, bool possible);
/**
* to_cpumask - convert a NR_CPUS bitmap to a struct cpumask *
@@ -1211,7 +1212,12 @@ static __always_inline unsigned int num_online_cpus(void)
{
return raw_atomic_read(&__num_online_cpus);
}
-#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
+
+static __always_inline unsigned int num_possible_cpus(void)
+{
+ return __num_possible_cpus;
+}
+
#define num_enabled_cpus() cpumask_weight(cpu_enabled_mask)
#define num_present_cpus() cpumask_weight(cpu_present_mask)
#define num_active_cpus() cpumask_weight(cpu_active_mask)
diff --git a/kernel/cpu.c b/kernel/cpu.c
index db9f6c5..b674fdf 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -3085,10 +3085,13 @@ EXPORT_SYMBOL(cpu_all_bits);
#ifdef CONFIG_INIT_ALL_POSSIBLE
struct cpumask __cpu_possible_mask __ro_after_init
= {CPU_BITS_ALL};
+unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
#else
struct cpumask __cpu_possible_mask __ro_after_init;
+unsigned int __num_possible_cpus __ro_after_init;
#endif
EXPORT_SYMBOL(__cpu_possible_mask);
+EXPORT_SYMBOL(__num_possible_cpus);
struct cpumask __cpu_online_mask __read_mostly;
EXPORT_SYMBOL(__cpu_online_mask);
@@ -3116,6 +3119,7 @@ void init_cpu_present(const struct cpumask *src)
void init_cpu_possible(const struct cpumask *src)
{
cpumask_copy(&__cpu_possible_mask, src);
+ __num_possible_cpus = cpumask_weight(&__cpu_possible_mask);
}
void set_cpu_online(unsigned int cpu, bool online)
@@ -3140,6 +3144,21 @@ void set_cpu_online(unsigned int cpu, bool online)
}
/*
+ * This should be marked __init, but there is a boatload of call sites
+ * which need to be fixed up to do so. Sigh...
+ */
+void set_cpu_possible(unsigned int cpu, bool possible)
+{
+ if (possible) {
+ if (!cpumask_test_and_set_cpu(cpu, &__cpu_possible_mask))
+ __num_possible_cpus++;
+ } else {
+ if (cpumask_test_and_clear_cpu(cpu, &__cpu_possible_mask))
+ __num_possible_cpus--;
+ }
+}
+
+/*
* Activate the first processor.
*/
void __init boot_cpu_init(void)The following commit has been merged into the core/rseq branch of tip:
Commit-ID: d0f23ccf6ba9e2cc202e9ad25a427b8e5ea3ca1e
Gitweb: https://git.kernel.org/tip/d0f23ccf6ba9e2cc202e9ad25a427b8e5ea3ca1e
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:01 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:54 +01:00
cpumask: Cache num_possible_cpus()
Reevaluating num_possible_cpus() over and over does not make sense. That
becomes a constant after init as cpu_possible_mask is marked ro_after_init.
Cache the value during initialization and provide that for consumption.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Yury Norov <yury.norov@gmail.com>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Reviewed-by: Shrikanth Hegde <sshegde@linux.ibm.com>
Link: https://patch.msgid.link/20251119172549.578653738@linutronix.de
---
include/linux/cpumask.h | 10 ++++++++--
kernel/cpu.c | 19 +++++++++++++++++++
2 files changed, 27 insertions(+), 2 deletions(-)
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index feba06e..66694ee 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -126,6 +126,7 @@ extern struct cpumask __cpu_dying_mask;
#define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
extern atomic_t __num_online_cpus;
+extern unsigned int __num_possible_cpus;
extern cpumask_t cpus_booted_once_mask;
@@ -1152,13 +1153,13 @@ void init_cpu_possible(const struct cpumask *src);
#define __assign_cpu(cpu, mask, val) \
__assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
-#define set_cpu_possible(cpu, possible) assign_cpu((cpu), &__cpu_possible_mask, (possible))
#define set_cpu_enabled(cpu, enabled) assign_cpu((cpu), &__cpu_enabled_mask, (enabled))
#define set_cpu_present(cpu, present) assign_cpu((cpu), &__cpu_present_mask, (present))
#define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active))
#define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying))
void set_cpu_online(unsigned int cpu, bool online);
+void set_cpu_possible(unsigned int cpu, bool possible);
/**
* to_cpumask - convert a NR_CPUS bitmap to a struct cpumask *
@@ -1211,7 +1212,12 @@ static __always_inline unsigned int num_online_cpus(void)
{
return raw_atomic_read(&__num_online_cpus);
}
-#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
+
+static __always_inline unsigned int num_possible_cpus(void)
+{
+ return __num_possible_cpus;
+}
+
#define num_enabled_cpus() cpumask_weight(cpu_enabled_mask)
#define num_present_cpus() cpumask_weight(cpu_present_mask)
#define num_active_cpus() cpumask_weight(cpu_active_mask)
diff --git a/kernel/cpu.c b/kernel/cpu.c
index db9f6c5..18e530d 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -3108,6 +3108,9 @@ EXPORT_SYMBOL(__cpu_dying_mask);
atomic_t __num_online_cpus __read_mostly;
EXPORT_SYMBOL(__num_online_cpus);
+unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
+EXPORT_SYMBOL(__num_possible_cpus);
+
void init_cpu_present(const struct cpumask *src)
{
cpumask_copy(&__cpu_present_mask, src);
@@ -3116,6 +3119,7 @@ void init_cpu_present(const struct cpumask *src)
void init_cpu_possible(const struct cpumask *src)
{
cpumask_copy(&__cpu_possible_mask, src);
+ __num_possible_cpus = cpumask_weight(&__cpu_possible_mask);
}
void set_cpu_online(unsigned int cpu, bool online)
@@ -3140,6 +3144,21 @@ void set_cpu_online(unsigned int cpu, bool online)
}
/*
+ * This should be marked __init, but there is a boatload of call sites
+ * which need to be fixed up to do so. Sigh...
+ */
+void set_cpu_possible(unsigned int cpu, bool possible)
+{
+ if (possible) {
+ if (!cpumask_test_and_set_cpu(cpu, &__cpu_possible_mask))
+ __num_possible_cpus++;
+ } else {
+ if (cpumask_test_and_clear_cpu(cpu, &__cpu_possible_mask))
+ __num_possible_cpus--;
+ }
+}
+
+/*
* Activate the first processor.
*/
void __init boot_cpu_init(void)This is truly a bitmap and just conveniently uses a cpumask because the
maximum size of the bitmap is nr_cpu_ids.
But that prevents to do searches for a zero bit in a limited range, which
is helpful to provide an efficient mechanism to consolidate the CID space
when the number of users decreases.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Acked-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
V4: Allocate bitmap_size(nr_possible_cpus()) - Mathieu
V4: Zero the bitmap with nrbits=nr_possible_cpus() - Brown paperbag
V3: Zero the bitmap with length of bitmap_size(nr_possible_cpus()) - Shrikanth
---
include/linux/mm_types.h | 9 +++++----
kernel/sched/core.c | 2 +-
kernel/sched/sched.h | 6 +++---
3 files changed, 9 insertions(+), 8 deletions(-)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1342,13 +1342,13 @@ static inline cpumask_t *mm_cpus_allowed
}
/* Accessor for struct mm_struct's cidmask. */
-static inline cpumask_t *mm_cidmask(struct mm_struct *mm)
+static inline unsigned long *mm_cidmask(struct mm_struct *mm)
{
unsigned long cid_bitmap = (unsigned long)mm_cpus_allowed(mm);
/* Skip mm_cpus_allowed */
cid_bitmap += cpumask_size();
- return (struct cpumask *)cid_bitmap;
+ return (unsigned long *)cid_bitmap;
}
static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
@@ -1363,7 +1363,7 @@ static inline void mm_init_cid(struct mm
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
- cpumask_clear(mm_cidmask(mm));
+ bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
@@ -1384,7 +1384,8 @@ static inline void mm_destroy_cid(struct
static inline unsigned int mm_cid_size(void)
{
- return 2 * cpumask_size(); /* mm_cpus_allowed(), mm_cidmask(). */
+ /* mm_cpus_allowed(), mm_cidmask(). */
+ return cpumask_size() + bitmap_size(num_possible_cpus());
}
#else /* CONFIG_SCHED_MM_CID */
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10402,7 +10402,7 @@ void sched_mm_cid_exit_signals(struct ta
guard(preempt)();
t->mm_cid.active = 0;
if (t->mm_cid.cid != MM_CID_UNSET) {
- cpumask_clear_cpu(t->mm_cid.cid, mm_cidmask(mm));
+ clear_bit(t->mm_cid.cid, mm_cidmask(mm));
t->mm_cid.cid = MM_CID_UNSET;
}
}
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3559,7 +3559,7 @@ static inline bool __mm_cid_get(struct t
if (cid >= max_cids)
return false;
- if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
+ if (test_and_set_bit(cid, mm_cidmask(mm)))
return false;
t->mm_cid.cid = t->mm_cid.last_cid = cid;
__this_cpu_write(mm->mm_cid.pcpu->cid, cid);
@@ -3582,7 +3582,7 @@ static inline bool mm_cid_get(struct tas
return true;
/* Try the first zero bit in the cidmask. */
- return __mm_cid_get(t, cpumask_first_zero(mm_cidmask(mm)), max_cids);
+ return __mm_cid_get(t, find_first_zero_bit(mm_cidmask(mm), num_possible_cpus()), max_cids);
}
static inline void mm_cid_select(struct task_struct *t)
@@ -3603,7 +3603,7 @@ static inline void switch_mm_cid(struct
{
if (prev->mm_cid.active) {
if (prev->mm_cid.cid != MM_CID_UNSET)
- cpumask_clear_cpu(prev->mm_cid.cid, mm_cidmask(prev->mm));
+ clear_bit(prev->mm_cid.cid, mm_cidmask(prev->mm));
prev->mm_cid.cid = MM_CID_UNSET;
}The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 539115f08cf850b9fdc6526b31da0839ff6c1631
Gitweb: https://git.kernel.org/tip/539115f08cf850b9fdc6526b31da0839ff6c1631
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:03 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:40 +01:00
sched/mmcid: Convert mm CID mask to a bitmap
This is truly a bitmap and just conveniently uses a cpumask because the
maximum size of the bitmap is nr_cpu_ids.
But that prevents to do searches for a zero bit in a limited range, which
is helpful to provide an efficient mechanism to consolidate the CID space
when the number of users decreases.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Acked-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
Link: https://patch.msgid.link/20251119172549.642866767@linutronix.de
---
include/linux/mm_types.h | 9 +++++----
kernel/sched/core.c | 2 +-
kernel/sched/sched.h | 6 +++---
3 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 67a7bdf..bafb81b 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1342,13 +1342,13 @@ static inline cpumask_t *mm_cpus_allowed(struct mm_struct *mm)
}
/* Accessor for struct mm_struct's cidmask. */
-static inline cpumask_t *mm_cidmask(struct mm_struct *mm)
+static inline unsigned long *mm_cidmask(struct mm_struct *mm)
{
unsigned long cid_bitmap = (unsigned long)mm_cpus_allowed(mm);
/* Skip mm_cpus_allowed */
cid_bitmap += cpumask_size();
- return (struct cpumask *)cid_bitmap;
+ return (unsigned long *)cid_bitmap;
}
static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
@@ -1363,7 +1363,7 @@ static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
- cpumask_clear(mm_cidmask(mm));
+ bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
@@ -1384,7 +1384,8 @@ static inline void mm_destroy_cid(struct mm_struct *mm)
static inline unsigned int mm_cid_size(void)
{
- return 2 * cpumask_size(); /* mm_cpus_allowed(), mm_cidmask(). */
+ /* mm_cpus_allowed(), mm_cidmask(). */
+ return cpumask_size() + bitmap_size(num_possible_cpus());
}
#else /* CONFIG_SCHED_MM_CID */
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f6bbfa1..9a114b6 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10402,7 +10402,7 @@ void sched_mm_cid_exit_signals(struct task_struct *t)
guard(preempt)();
t->mm_cid.active = 0;
if (t->mm_cid.cid != MM_CID_UNSET) {
- cpumask_clear_cpu(t->mm_cid.cid, mm_cidmask(mm));
+ clear_bit(t->mm_cid.cid, mm_cidmask(mm));
t->mm_cid.cid = MM_CID_UNSET;
}
}
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index a17f04f..31f2e43 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3559,7 +3559,7 @@ static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigne
if (cid >= max_cids)
return false;
- if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
+ if (test_and_set_bit(cid, mm_cidmask(mm)))
return false;
t->mm_cid.cid = t->mm_cid.last_cid = cid;
__this_cpu_write(mm->mm_cid.pcpu->cid, cid);
@@ -3582,7 +3582,7 @@ static inline bool mm_cid_get(struct task_struct *t)
return true;
/* Try the first zero bit in the cidmask. */
- return __mm_cid_get(t, cpumask_first_zero(mm_cidmask(mm)), max_cids);
+ return __mm_cid_get(t, find_first_zero_bit(mm_cidmask(mm), num_possible_cpus()), max_cids);
}
static inline void mm_cid_select(struct task_struct *t)
@@ -3603,7 +3603,7 @@ static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *n
{
if (prev->mm_cid.active) {
if (prev->mm_cid.cid != MM_CID_UNSET)
- cpumask_clear_cpu(prev->mm_cid.cid, mm_cidmask(prev->mm));
+ clear_bit(prev->mm_cid.cid, mm_cidmask(prev->mm));
prev->mm_cid.cid = MM_CID_UNSET;
}
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 472931e757fb3dfad1f78ce6f5abd821155433b2
Gitweb: https://git.kernel.org/tip/472931e757fb3dfad1f78ce6f5abd821155433b2
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:03 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:55 +01:00
sched/mmcid: Convert mm CID mask to a bitmap
This is truly a bitmap and just conveniently uses a cpumask because the
maximum size of the bitmap is nr_cpu_ids.
But that prevents to do searches for a zero bit in a limited range, which
is helpful to provide an efficient mechanism to consolidate the CID space
when the number of users decreases.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Acked-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
Link: https://patch.msgid.link/20251119172549.642866767@linutronix.de
---
include/linux/mm_types.h | 9 +++++----
kernel/sched/core.c | 2 +-
kernel/sched/sched.h | 6 +++---
3 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 67a7bdf..bafb81b 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1342,13 +1342,13 @@ static inline cpumask_t *mm_cpus_allowed(struct mm_struct *mm)
}
/* Accessor for struct mm_struct's cidmask. */
-static inline cpumask_t *mm_cidmask(struct mm_struct *mm)
+static inline unsigned long *mm_cidmask(struct mm_struct *mm)
{
unsigned long cid_bitmap = (unsigned long)mm_cpus_allowed(mm);
/* Skip mm_cpus_allowed */
cid_bitmap += cpumask_size();
- return (struct cpumask *)cid_bitmap;
+ return (unsigned long *)cid_bitmap;
}
static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
@@ -1363,7 +1363,7 @@ static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
- cpumask_clear(mm_cidmask(mm));
+ bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
@@ -1384,7 +1384,8 @@ static inline void mm_destroy_cid(struct mm_struct *mm)
static inline unsigned int mm_cid_size(void)
{
- return 2 * cpumask_size(); /* mm_cpus_allowed(), mm_cidmask(). */
+ /* mm_cpus_allowed(), mm_cidmask(). */
+ return cpumask_size() + bitmap_size(num_possible_cpus());
}
#else /* CONFIG_SCHED_MM_CID */
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f6bbfa1..9a114b6 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10402,7 +10402,7 @@ void sched_mm_cid_exit_signals(struct task_struct *t)
guard(preempt)();
t->mm_cid.active = 0;
if (t->mm_cid.cid != MM_CID_UNSET) {
- cpumask_clear_cpu(t->mm_cid.cid, mm_cidmask(mm));
+ clear_bit(t->mm_cid.cid, mm_cidmask(mm));
t->mm_cid.cid = MM_CID_UNSET;
}
}
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index a17f04f..31f2e43 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3559,7 +3559,7 @@ static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigne
if (cid >= max_cids)
return false;
- if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
+ if (test_and_set_bit(cid, mm_cidmask(mm)))
return false;
t->mm_cid.cid = t->mm_cid.last_cid = cid;
__this_cpu_write(mm->mm_cid.pcpu->cid, cid);
@@ -3582,7 +3582,7 @@ static inline bool mm_cid_get(struct task_struct *t)
return true;
/* Try the first zero bit in the cidmask. */
- return __mm_cid_get(t, cpumask_first_zero(mm_cidmask(mm)), max_cids);
+ return __mm_cid_get(t, find_first_zero_bit(mm_cidmask(mm), num_possible_cpus()), max_cids);
}
static inline void mm_cid_select(struct task_struct *t)
@@ -3603,7 +3603,7 @@ static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *n
{
if (prev->mm_cid.active) {
if (prev->mm_cid.cid != MM_CID_UNSET)
- cpumask_clear_cpu(prev->mm_cid.cid, mm_cidmask(prev->mm));
+ clear_bit(prev->mm_cid.cid, mm_cidmask(prev->mm));
prev->mm_cid.cid = MM_CID_UNSET;
}
There is no need anymore to keep this under sighand lock as the current
code and the upcoming replacement are not depending on the exit state of a
task anymore.
That allows to use a mutex in the exit path.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/sched.h | 4 ++--
kernel/exit.c | 1 +
kernel/sched/core.c | 4 ++--
kernel/signal.c | 2 --
4 files changed, 5 insertions(+), 6 deletions(-)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2298,7 +2298,7 @@ static __always_inline void alloc_tag_re
void sched_mm_cid_before_execve(struct task_struct *t);
void sched_mm_cid_after_execve(struct task_struct *t);
void sched_mm_cid_fork(struct task_struct *t);
-void sched_mm_cid_exit_signals(struct task_struct *t);
+void sched_mm_cid_exit(struct task_struct *t);
static inline int task_mm_cid(struct task_struct *t)
{
return t->mm_cid.cid;
@@ -2307,7 +2307,7 @@ static inline int task_mm_cid(struct tas
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
static inline void sched_mm_cid_after_execve(struct task_struct *t) { }
static inline void sched_mm_cid_fork(struct task_struct *t) { }
-static inline void sched_mm_cid_exit_signals(struct task_struct *t) { }
+static inline void sched_mm_cid_exit(struct task_struct *t) { }
static inline int task_mm_cid(struct task_struct *t)
{
/*
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -910,6 +910,7 @@ void __noreturn do_exit(long code)
user_events_exit(tsk);
io_uring_files_cancel();
+ sched_mm_cid_exit(tsk);
exit_signals(tsk); /* sets PF_EXITING */
seccomp_filter_release(tsk);
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10392,7 +10392,7 @@ static inline void mm_update_cpus_allowe
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
}
-void sched_mm_cid_exit_signals(struct task_struct *t)
+void sched_mm_cid_exit(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10410,7 +10410,7 @@ void sched_mm_cid_exit_signals(struct ta
/* Deactivate MM CID allocation across execve() */
void sched_mm_cid_before_execve(struct task_struct *t)
{
- sched_mm_cid_exit_signals(t);
+ sched_mm_cid_exit(t);
}
/* Reactivate MM CID after successful execve() */
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -3125,7 +3125,6 @@ void exit_signals(struct task_struct *ts
cgroup_threadgroup_change_begin(tsk);
if (thread_group_empty(tsk) || (tsk->signal->flags & SIGNAL_GROUP_EXIT)) {
- sched_mm_cid_exit_signals(tsk);
tsk->flags |= PF_EXITING;
cgroup_threadgroup_change_end(tsk);
return;
@@ -3136,7 +3135,6 @@ void exit_signals(struct task_struct *ts
* From now this task is not visible for group-wide signals,
* see wants_signal(), do_signal_stop().
*/
- sched_mm_cid_exit_signals(tsk);
tsk->flags |= PF_EXITING;
cgroup_threadgroup_change_end(tsk);The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 2b1642b881088bbf73fcb1147c474a198ec46729
Gitweb: https://git.kernel.org/tip/2b1642b881088bbf73fcb1147c474a198ec46729
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:05 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:40 +01:00
signal: Move MMCID exit out of sighand lock
There is no need anymore to keep this under sighand lock as the current
code and the upcoming replacement are not depending on the exit state of a
task anymore.
That allows to use a mutex in the exit path.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.706439391@linutronix.de
---
include/linux/sched.h | 4 ++--
kernel/exit.c | 1 +
kernel/sched/core.c | 4 ++--
kernel/signal.c | 2 --
4 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 64f080d..c411ae0 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2298,7 +2298,7 @@ static __always_inline void alloc_tag_restore(struct alloc_tag *tag, struct allo
void sched_mm_cid_before_execve(struct task_struct *t);
void sched_mm_cid_after_execve(struct task_struct *t);
void sched_mm_cid_fork(struct task_struct *t);
-void sched_mm_cid_exit_signals(struct task_struct *t);
+void sched_mm_cid_exit(struct task_struct *t);
static inline int task_mm_cid(struct task_struct *t)
{
return t->mm_cid.cid;
@@ -2307,7 +2307,7 @@ static inline int task_mm_cid(struct task_struct *t)
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
static inline void sched_mm_cid_after_execve(struct task_struct *t) { }
static inline void sched_mm_cid_fork(struct task_struct *t) { }
-static inline void sched_mm_cid_exit_signals(struct task_struct *t) { }
+static inline void sched_mm_cid_exit(struct task_struct *t) { }
static inline int task_mm_cid(struct task_struct *t)
{
/*
diff --git a/kernel/exit.c b/kernel/exit.c
index 9f74e8f..324616f 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -910,6 +910,7 @@ void __noreturn do_exit(long code)
user_events_exit(tsk);
io_uring_files_cancel();
+ sched_mm_cid_exit(tsk);
exit_signals(tsk); /* sets PF_EXITING */
seccomp_filter_release(tsk);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9a114b6..3fdf90a 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10392,7 +10392,7 @@ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpu
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
}
-void sched_mm_cid_exit_signals(struct task_struct *t)
+void sched_mm_cid_exit(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10410,7 +10410,7 @@ void sched_mm_cid_exit_signals(struct task_struct *t)
/* Deactivate MM CID allocation across execve() */
void sched_mm_cid_before_execve(struct task_struct *t)
{
- sched_mm_cid_exit_signals(t);
+ sched_mm_cid_exit(t);
}
/* Reactivate MM CID after successful execve() */
diff --git a/kernel/signal.c b/kernel/signal.c
index fe9190d..e42b8bd 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -3125,7 +3125,6 @@ void exit_signals(struct task_struct *tsk)
cgroup_threadgroup_change_begin(tsk);
if (thread_group_empty(tsk) || (tsk->signal->flags & SIGNAL_GROUP_EXIT)) {
- sched_mm_cid_exit_signals(tsk);
tsk->flags |= PF_EXITING;
cgroup_threadgroup_change_end(tsk);
return;
@@ -3136,7 +3135,6 @@ void exit_signals(struct task_struct *tsk)
* From now this task is not visible for group-wide signals,
* see wants_signal(), do_signal_stop().
*/
- sched_mm_cid_exit_signals(tsk);
tsk->flags |= PF_EXITING;
cgroup_threadgroup_change_end(tsk);The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 1497ac189541df3c7c344bca723a7597009a99f4
Gitweb: https://git.kernel.org/tip/1497ac189541df3c7c344bca723a7597009a99f4
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:05 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:55 +01:00
signal: Move MMCID exit out of sighand lock
There is no need anymore to keep this under sighand lock as the current
code and the upcoming replacement are not depending on the exit state of a
task anymore.
That allows to use a mutex in the exit path.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.706439391@linutronix.de
---
include/linux/sched.h | 4 ++--
kernel/exit.c | 1 +
kernel/sched/core.c | 4 ++--
kernel/signal.c | 2 --
4 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 64f080d..c411ae0 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2298,7 +2298,7 @@ static __always_inline void alloc_tag_restore(struct alloc_tag *tag, struct allo
void sched_mm_cid_before_execve(struct task_struct *t);
void sched_mm_cid_after_execve(struct task_struct *t);
void sched_mm_cid_fork(struct task_struct *t);
-void sched_mm_cid_exit_signals(struct task_struct *t);
+void sched_mm_cid_exit(struct task_struct *t);
static inline int task_mm_cid(struct task_struct *t)
{
return t->mm_cid.cid;
@@ -2307,7 +2307,7 @@ static inline int task_mm_cid(struct task_struct *t)
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
static inline void sched_mm_cid_after_execve(struct task_struct *t) { }
static inline void sched_mm_cid_fork(struct task_struct *t) { }
-static inline void sched_mm_cid_exit_signals(struct task_struct *t) { }
+static inline void sched_mm_cid_exit(struct task_struct *t) { }
static inline int task_mm_cid(struct task_struct *t)
{
/*
diff --git a/kernel/exit.c b/kernel/exit.c
index 9f74e8f..324616f 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -910,6 +910,7 @@ void __noreturn do_exit(long code)
user_events_exit(tsk);
io_uring_files_cancel();
+ sched_mm_cid_exit(tsk);
exit_signals(tsk); /* sets PF_EXITING */
seccomp_filter_release(tsk);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9a114b6..3fdf90a 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10392,7 +10392,7 @@ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpu
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
}
-void sched_mm_cid_exit_signals(struct task_struct *t)
+void sched_mm_cid_exit(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10410,7 +10410,7 @@ void sched_mm_cid_exit_signals(struct task_struct *t)
/* Deactivate MM CID allocation across execve() */
void sched_mm_cid_before_execve(struct task_struct *t)
{
- sched_mm_cid_exit_signals(t);
+ sched_mm_cid_exit(t);
}
/* Reactivate MM CID after successful execve() */
diff --git a/kernel/signal.c b/kernel/signal.c
index fe9190d..e42b8bd 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -3125,7 +3125,6 @@ void exit_signals(struct task_struct *tsk)
cgroup_threadgroup_change_begin(tsk);
if (thread_group_empty(tsk) || (tsk->signal->flags & SIGNAL_GROUP_EXIT)) {
- sched_mm_cid_exit_signals(tsk);
tsk->flags |= PF_EXITING;
cgroup_threadgroup_change_end(tsk);
return;
@@ -3136,7 +3135,6 @@ void exit_signals(struct task_struct *tsk)
* From now this task is not visible for group-wide signals,
* see wants_signal(), do_signal_stop().
*/
- sched_mm_cid_exit_signals(tsk);
tsk->flags |= PF_EXITING;
cgroup_threadgroup_change_end(tsk);It's getting bigger soon, so just move it out of line to the rest of the
code.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/mm_types.h | 15 +--------------
kernel/sched/core.c | 14 ++++++++++++++
2 files changed, 15 insertions(+), 14 deletions(-)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1351,20 +1351,7 @@ static inline unsigned long *mm_cidmask(
return (unsigned long *)cid_bitmap;
}
-static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
-{
- int i;
-
- for_each_possible_cpu(i) {
- struct mm_cid_pcpu *pcpu = per_cpu_ptr(mm->mm_cid.pcpu, i);
-
- pcpu->cid = MM_CID_UNSET;
- }
- mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
- raw_spin_lock_init(&mm->mm_cid.lock);
- cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
- bitmap_zero(mm_cidmask(mm), num_possible_cpus());
-}
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p);
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
{
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10431,6 +10431,20 @@ void sched_mm_cid_fork(struct task_struc
WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
t->mm_cid.active = 1;
}
+
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+{
+ struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+
+ mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ raw_spin_lock_init(&mm->mm_cid.lock);
+ cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
+ bitmap_zero(mm_cidmask(mm), num_possible_cpus());
+}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }
#endif /* !CONFIG_SCHED_MM_CID */The following commit has been merged into the core/rseq branch of tip:
Commit-ID: bf070520e398679cd582b3c3e44107bf22c143ba
Gitweb: https://git.kernel.org/tip/bf070520e398679cd582b3c3e44107bf22c143ba
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:07 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:40 +01:00
sched/mmcid: Move initialization out of line
It's getting bigger soon, so just move it out of line to the rest of the
code.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.769636491@linutronix.de
---
include/linux/mm_types.h | 15 +--------------
kernel/sched/core.c | 14 ++++++++++++++
2 files changed, 15 insertions(+), 14 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index bafb81b..3b7d05e 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1351,20 +1351,7 @@ static inline unsigned long *mm_cidmask(struct mm_struct *mm)
return (unsigned long *)cid_bitmap;
}
-static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
-{
- int i;
-
- for_each_possible_cpu(i) {
- struct mm_cid_pcpu *pcpu = per_cpu_ptr(mm->mm_cid.pcpu, i);
-
- pcpu->cid = MM_CID_UNSET;
- }
- mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
- raw_spin_lock_init(&mm->mm_cid.lock);
- cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
- bitmap_zero(mm_cidmask(mm), num_possible_cpus());
-}
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p);
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
{
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3fdf90a..34b6c31 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10431,6 +10431,20 @@ void sched_mm_cid_fork(struct task_struct *t)
WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
t->mm_cid.active = 1;
}
+
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+{
+ struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+
+ mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ raw_spin_lock_init(&mm->mm_cid.lock);
+ cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
+ bitmap_zero(mm_cidmask(mm), num_possible_cpus());
+}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }
#endif /* !CONFIG_SCHED_MM_CID */The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 61c7810a91fb15cffaa0b44810b8074c5e8a125d
Gitweb: https://git.kernel.org/tip/61c7810a91fb15cffaa0b44810b8074c5e8a125d
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:07 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:55 +01:00
sched/mmcid: Move initialization out of line
It's getting bigger soon, so just move it out of line to the rest of the
code.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.769636491@linutronix.de
---
include/linux/mm_types.h | 15 +--------------
kernel/sched/core.c | 14 ++++++++++++++
2 files changed, 15 insertions(+), 14 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index bafb81b..3b7d05e 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1351,20 +1351,7 @@ static inline unsigned long *mm_cidmask(struct mm_struct *mm)
return (unsigned long *)cid_bitmap;
}
-static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
-{
- int i;
-
- for_each_possible_cpu(i) {
- struct mm_cid_pcpu *pcpu = per_cpu_ptr(mm->mm_cid.pcpu, i);
-
- pcpu->cid = MM_CID_UNSET;
- }
- mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
- raw_spin_lock_init(&mm->mm_cid.lock);
- cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
- bitmap_zero(mm_cidmask(mm), num_possible_cpus());
-}
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p);
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
{
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3fdf90a..34b6c31 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10431,6 +10431,20 @@ void sched_mm_cid_fork(struct task_struct *t)
WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
t->mm_cid.active = 1;
}
+
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+{
+ struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+
+ mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ raw_spin_lock_init(&mm->mm_cid.lock);
+ cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
+ bitmap_zero(mm_cidmask(mm), num_possible_cpus());
+}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }
#endif /* !CONFIG_SCHED_MM_CID */Reading mm::mm_users and mm:::mm_cid::nr_cpus_allowed every time to compute
the maximal CID value is just wasteful as that value is only changing on
fork(), exit() and eventually when the affinity changes.
So it can be easily precomputed at those points and provided in mm::mm_cid
for consumption in the hot path.
But there is an issue with using mm::mm_users for accounting because that
does not necessarily reflect the number of user space tasks as other kernel
code can take temporary references on the MM which skew the picture.
Solve that by adding a users counter to struct mm_mm_cid, which is modified
by fork() and exit() and used for precomputing under mm_mm_cid::lock.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/rseq_types.h | 6 ++++
kernel/fork.c | 1
kernel/sched/core.c | 59 ++++++++++++++++++++++++++++++++-------------
kernel/sched/sched.h | 3 --
4 files changed, 50 insertions(+), 19 deletions(-)
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -117,14 +117,20 @@ struct mm_cid_pcpu {
/**
* struct mm_mm_cid - Storage for per MM CID data
* @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @max_cids: The exclusive maximum CID value for allocation and convergence
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
+ * @users: The number of tasks sharing this MM. Separate from mm::mm_users
+ * as that is modified by mmget()/mm_put() by other entities which
+ * do not actually share the MM.
* @lock: Spinlock to protect all fields except @pcpu. It also protects
* the MM cid cpumask and the MM cidmask bitmap.
*/
struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int max_cids;
unsigned int nr_cpus_allowed;
+ unsigned int users;
raw_spinlock_t lock;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -2455,6 +2455,7 @@ static bool need_futex_hash_allocate_def
exit_task_namespaces(p);
bad_fork_cleanup_mm:
if (p->mm) {
+ sched_mm_cid_exit(p);
mm_clear_owner(p->mm, p);
mmput(p->mm);
}
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -4485,7 +4485,6 @@ static void __sched_fork(u64 clone_flags
init_numa_balancing(clone_flags, p);
p->wake_entry.u_flags = CSD_TYPE_TTWU;
p->migration_pending = NULL;
- init_sched_mm_cid(p);
}
DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
@@ -10371,15 +10370,27 @@ void call_trace_sched_update_nr_running(
#ifdef CONFIG_SCHED_MM_CID
/*
- * When a task exits, the MM CID held by the task is not longer required as
- * the task cannot return to user space.
+ * Update the CID range properties when the constraints change. Invoked via
+ * fork(), exit() and affinity changes
*/
+static void mm_update_max_cids(struct mm_struct *mm)
+{
+ struct mm_mm_cid *mc = &mm->mm_cid;
+ unsigned int max_cids;
+
+ lockdep_assert_held(&mm->mm_cid.lock);
+
+ /* Calculate the new maximum constraint */
+ max_cids = min(mc->nr_cpus_allowed, mc->users);
+ WRITE_ONCE(mc->max_cids, max_cids);
+}
+
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
unsigned int weight;
- if (!mm)
+ if (!mm || !READ_ONCE(mm->mm_cid.users))
return;
/*
@@ -10389,9 +10400,30 @@ static inline void mm_update_cpus_allowe
guard(raw_spinlock)(&mm->mm_cid.lock);
mm_allowed = mm_cpus_allowed(mm);
weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
+ if (weight == mm->mm_cid.nr_cpus_allowed)
+ return;
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
+ mm_update_max_cids(mm);
+}
+
+void sched_mm_cid_fork(struct task_struct *t)
+{
+ struct mm_struct *mm = t->mm;
+
+ WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
+
+ guard(raw_spinlock)(&mm->mm_cid.lock);
+ t->mm_cid.active = 1;
+ mm->mm_cid.users++;
+ /* Preset last_cid for mm_cid_select() */
+ t->mm_cid.last_cid = READ_ONCE(mm->mm_cid.max_cids) - 1;
+ mm_update_max_cids(mm);
}
+/*
+ * When a task exits, the MM CID held by the task is not longer required as
+ * the task cannot return to user space.
+ */
void sched_mm_cid_exit(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10399,12 +10431,14 @@ void sched_mm_cid_exit(struct task_struc
if (!mm || !t->mm_cid.active)
return;
- guard(preempt)();
+ guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 0;
+ mm->mm_cid.users--;
if (t->mm_cid.cid != MM_CID_UNSET) {
clear_bit(t->mm_cid.cid, mm_cidmask(mm));
t->mm_cid.cid = MM_CID_UNSET;
}
+ mm_update_max_cids(mm);
}
/* Deactivate MM CID allocation across execve() */
@@ -10416,22 +10450,11 @@ void sched_mm_cid_before_execve(struct t
/* Reactivate MM CID after successful execve() */
void sched_mm_cid_after_execve(struct task_struct *t)
{
- struct mm_struct *mm = t->mm;
-
- if (!mm)
- return;
-
+ sched_mm_cid_fork(t);
guard(preempt)();
- t->mm_cid.active = 1;
mm_cid_select(t);
}
-void sched_mm_cid_fork(struct task_struct *t)
-{
- WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
- t->mm_cid.active = 1;
-}
-
void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
{
struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
@@ -10440,7 +10463,9 @@ void mm_init_cid(struct mm_struct *mm, s
for_each_possible_cpu(cpu)
per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+ mm->mm_cid.max_cids = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3571,7 +3571,7 @@ static inline bool mm_cid_get(struct tas
struct mm_struct *mm = t->mm;
unsigned int max_cids;
- max_cids = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
/* Try to reuse the last CID of this task */
if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
@@ -3614,7 +3614,6 @@ static inline void switch_mm_cid(struct
}
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void init_sched_mm_cid(struct task_struct *t) { }
static inline void mm_cid_select(struct task_struct *t) { }
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */The following commit has been merged into the core/rseq branch of tip:
Commit-ID: b0c3d51b54f8a4f4c809432d210c0c983d5cd97e
Gitweb: https://git.kernel.org/tip/b0c3d51b54f8a4f4c809432d210c0c983d5cd97e
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:09 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:40 +01:00
sched/mmcid: Provide precomputed maximal value
Reading mm::mm_users and mm:::mm_cid::nr_cpus_allowed every time to compute
the maximal CID value is just wasteful as that value is only changing on
fork(), exit() and eventually when the affinity changes.
So it can be easily precomputed at those points and provided in mm::mm_cid
for consumption in the hot path.
But there is an issue with using mm::mm_users for accounting because that
does not necessarily reflect the number of user space tasks as other kernel
code can take temporary references on the MM which skew the picture.
Solve that by adding a users counter to struct mm_mm_cid, which is modified
by fork() and exit() and used for precomputing under mm_mm_cid::lock.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.832764634@linutronix.de
---
include/linux/rseq_types.h | 6 ++++-
kernel/fork.c | 1 +-
kernel/sched/core.c | 59 ++++++++++++++++++++++++++-----------
kernel/sched/sched.h | 3 +--
4 files changed, 50 insertions(+), 19 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index d7e8071..0fab369 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -117,14 +117,20 @@ struct mm_cid_pcpu {
/**
* struct mm_mm_cid - Storage for per MM CID data
* @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @max_cids: The exclusive maximum CID value for allocation and convergence
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
+ * @users: The number of tasks sharing this MM. Separate from mm::mm_users
+ * as that is modified by mmget()/mm_put() by other entities which
+ * do not actually share the MM.
* @lock: Spinlock to protect all fields except @pcpu. It also protects
* the MM cid cpumask and the MM cidmask bitmap.
*/
struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int max_cids;
unsigned int nr_cpus_allowed;
+ unsigned int users;
raw_spinlock_t lock;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
diff --git a/kernel/fork.c b/kernel/fork.c
index 74bc7c9..6c23219 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -2455,6 +2455,7 @@ bad_fork_cleanup_namespaces:
exit_task_namespaces(p);
bad_fork_cleanup_mm:
if (p->mm) {
+ sched_mm_cid_exit(p);
mm_clear_owner(p->mm, p);
mmput(p->mm);
}
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 34b6c31..f9295c4 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -4485,7 +4485,6 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p)
init_numa_balancing(clone_flags, p);
p->wake_entry.u_flags = CSD_TYPE_TTWU;
p->migration_pending = NULL;
- init_sched_mm_cid(p);
}
DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
@@ -10371,15 +10370,27 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
#ifdef CONFIG_SCHED_MM_CID
/*
- * When a task exits, the MM CID held by the task is not longer required as
- * the task cannot return to user space.
+ * Update the CID range properties when the constraints change. Invoked via
+ * fork(), exit() and affinity changes
*/
+static void mm_update_max_cids(struct mm_struct *mm)
+{
+ struct mm_mm_cid *mc = &mm->mm_cid;
+ unsigned int max_cids;
+
+ lockdep_assert_held(&mm->mm_cid.lock);
+
+ /* Calculate the new maximum constraint */
+ max_cids = min(mc->nr_cpus_allowed, mc->users);
+ WRITE_ONCE(mc->max_cids, max_cids);
+}
+
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
unsigned int weight;
- if (!mm)
+ if (!mm || !READ_ONCE(mm->mm_cid.users))
return;
/*
@@ -10389,9 +10400,30 @@ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpu
guard(raw_spinlock)(&mm->mm_cid.lock);
mm_allowed = mm_cpus_allowed(mm);
weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
+ if (weight == mm->mm_cid.nr_cpus_allowed)
+ return;
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
+ mm_update_max_cids(mm);
+}
+
+void sched_mm_cid_fork(struct task_struct *t)
+{
+ struct mm_struct *mm = t->mm;
+
+ WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
+
+ guard(raw_spinlock)(&mm->mm_cid.lock);
+ t->mm_cid.active = 1;
+ mm->mm_cid.users++;
+ /* Preset last_cid for mm_cid_select() */
+ t->mm_cid.last_cid = READ_ONCE(mm->mm_cid.max_cids) - 1;
+ mm_update_max_cids(mm);
}
+/*
+ * When a task exits, the MM CID held by the task is not longer required as
+ * the task cannot return to user space.
+ */
void sched_mm_cid_exit(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10399,12 +10431,14 @@ void sched_mm_cid_exit(struct task_struct *t)
if (!mm || !t->mm_cid.active)
return;
- guard(preempt)();
+ guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 0;
+ mm->mm_cid.users--;
if (t->mm_cid.cid != MM_CID_UNSET) {
clear_bit(t->mm_cid.cid, mm_cidmask(mm));
t->mm_cid.cid = MM_CID_UNSET;
}
+ mm_update_max_cids(mm);
}
/* Deactivate MM CID allocation across execve() */
@@ -10416,22 +10450,11 @@ void sched_mm_cid_before_execve(struct task_struct *t)
/* Reactivate MM CID after successful execve() */
void sched_mm_cid_after_execve(struct task_struct *t)
{
- struct mm_struct *mm = t->mm;
-
- if (!mm)
- return;
-
+ sched_mm_cid_fork(t);
guard(preempt)();
- t->mm_cid.active = 1;
mm_cid_select(t);
}
-void sched_mm_cid_fork(struct task_struct *t)
-{
- WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
- t->mm_cid.active = 1;
-}
-
void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
{
struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
@@ -10440,7 +10463,9 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
for_each_possible_cpu(cpu)
per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+ mm->mm_cid.max_cids = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 31f2e43..d539fb2 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3571,7 +3571,7 @@ static inline bool mm_cid_get(struct task_struct *t)
struct mm_struct *mm = t->mm;
unsigned int max_cids;
- max_cids = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
/* Try to reuse the last CID of this task */
if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
@@ -3614,7 +3614,6 @@ static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *n
}
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void init_sched_mm_cid(struct task_struct *t) { }
static inline void mm_cid_select(struct task_struct *t) { }
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 3c7a7327102ca35b3081d1a7dde5819d0f5a5e08
Gitweb: https://git.kernel.org/tip/3c7a7327102ca35b3081d1a7dde5819d0f5a5e08
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:09 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:56 +01:00
sched/mmcid: Provide precomputed maximal value
Reading mm::mm_users and mm:::mm_cid::nr_cpus_allowed every time to compute
the maximal CID value is just wasteful as that value is only changing on
fork(), exit() and eventually when the affinity changes.
So it can be easily precomputed at those points and provided in mm::mm_cid
for consumption in the hot path.
But there is an issue with using mm::mm_users for accounting because that
does not necessarily reflect the number of user space tasks as other kernel
code can take temporary references on the MM which skew the picture.
Solve that by adding a users counter to struct mm_mm_cid, which is modified
by fork() and exit() and used for precomputing under mm_mm_cid::lock.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.832764634@linutronix.de
---
include/linux/rseq_types.h | 6 ++++-
kernel/fork.c | 1 +-
kernel/sched/core.c | 59 ++++++++++++++++++++++++++-----------
kernel/sched/sched.h | 3 +--
4 files changed, 50 insertions(+), 19 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index d7e8071..0fab369 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -117,14 +117,20 @@ struct mm_cid_pcpu {
/**
* struct mm_mm_cid - Storage for per MM CID data
* @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @max_cids: The exclusive maximum CID value for allocation and convergence
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
+ * @users: The number of tasks sharing this MM. Separate from mm::mm_users
+ * as that is modified by mmget()/mm_put() by other entities which
+ * do not actually share the MM.
* @lock: Spinlock to protect all fields except @pcpu. It also protects
* the MM cid cpumask and the MM cidmask bitmap.
*/
struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int max_cids;
unsigned int nr_cpus_allowed;
+ unsigned int users;
raw_spinlock_t lock;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
diff --git a/kernel/fork.c b/kernel/fork.c
index 74bc7c9..6c23219 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -2455,6 +2455,7 @@ bad_fork_cleanup_namespaces:
exit_task_namespaces(p);
bad_fork_cleanup_mm:
if (p->mm) {
+ sched_mm_cid_exit(p);
mm_clear_owner(p->mm, p);
mmput(p->mm);
}
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 34b6c31..f9295c4 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -4485,7 +4485,6 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p)
init_numa_balancing(clone_flags, p);
p->wake_entry.u_flags = CSD_TYPE_TTWU;
p->migration_pending = NULL;
- init_sched_mm_cid(p);
}
DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
@@ -10371,15 +10370,27 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
#ifdef CONFIG_SCHED_MM_CID
/*
- * When a task exits, the MM CID held by the task is not longer required as
- * the task cannot return to user space.
+ * Update the CID range properties when the constraints change. Invoked via
+ * fork(), exit() and affinity changes
*/
+static void mm_update_max_cids(struct mm_struct *mm)
+{
+ struct mm_mm_cid *mc = &mm->mm_cid;
+ unsigned int max_cids;
+
+ lockdep_assert_held(&mm->mm_cid.lock);
+
+ /* Calculate the new maximum constraint */
+ max_cids = min(mc->nr_cpus_allowed, mc->users);
+ WRITE_ONCE(mc->max_cids, max_cids);
+}
+
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
unsigned int weight;
- if (!mm)
+ if (!mm || !READ_ONCE(mm->mm_cid.users))
return;
/*
@@ -10389,9 +10400,30 @@ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpu
guard(raw_spinlock)(&mm->mm_cid.lock);
mm_allowed = mm_cpus_allowed(mm);
weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
+ if (weight == mm->mm_cid.nr_cpus_allowed)
+ return;
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
+ mm_update_max_cids(mm);
+}
+
+void sched_mm_cid_fork(struct task_struct *t)
+{
+ struct mm_struct *mm = t->mm;
+
+ WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
+
+ guard(raw_spinlock)(&mm->mm_cid.lock);
+ t->mm_cid.active = 1;
+ mm->mm_cid.users++;
+ /* Preset last_cid for mm_cid_select() */
+ t->mm_cid.last_cid = READ_ONCE(mm->mm_cid.max_cids) - 1;
+ mm_update_max_cids(mm);
}
+/*
+ * When a task exits, the MM CID held by the task is not longer required as
+ * the task cannot return to user space.
+ */
void sched_mm_cid_exit(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10399,12 +10431,14 @@ void sched_mm_cid_exit(struct task_struct *t)
if (!mm || !t->mm_cid.active)
return;
- guard(preempt)();
+ guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 0;
+ mm->mm_cid.users--;
if (t->mm_cid.cid != MM_CID_UNSET) {
clear_bit(t->mm_cid.cid, mm_cidmask(mm));
t->mm_cid.cid = MM_CID_UNSET;
}
+ mm_update_max_cids(mm);
}
/* Deactivate MM CID allocation across execve() */
@@ -10416,22 +10450,11 @@ void sched_mm_cid_before_execve(struct task_struct *t)
/* Reactivate MM CID after successful execve() */
void sched_mm_cid_after_execve(struct task_struct *t)
{
- struct mm_struct *mm = t->mm;
-
- if (!mm)
- return;
-
+ sched_mm_cid_fork(t);
guard(preempt)();
- t->mm_cid.active = 1;
mm_cid_select(t);
}
-void sched_mm_cid_fork(struct task_struct *t)
-{
- WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
- t->mm_cid.active = 1;
-}
-
void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
{
struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
@@ -10440,7 +10463,9 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
for_each_possible_cpu(cpu)
per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+ mm->mm_cid.max_cids = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 31f2e43..d539fb2 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3571,7 +3571,7 @@ static inline bool mm_cid_get(struct task_struct *t)
struct mm_struct *mm = t->mm;
unsigned int max_cids;
- max_cids = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
/* Try to reuse the last CID of this task */
if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
@@ -3614,7 +3614,6 @@ static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *n
}
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void init_sched_mm_cid(struct task_struct *t) { }
static inline void mm_cid_select(struct task_struct *t) { }
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */Prepare for the new CID management scheme which puts the CID ownership
transition into the fork() and exit() slow path by serializing
sched_mm_cid_fork()/exit() with it, so task list and cpu mask walks can be
done in interruptible and preemptible code.
The contention on it is not worse than on other concurrency controls in the
fork()/exit() machinery.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/rseq_types.h | 2 ++
kernel/sched/core.c | 22 ++++++++++++++++++++++
2 files changed, 24 insertions(+)
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -125,6 +125,7 @@ struct mm_cid_pcpu {
* do not actually share the MM.
* @lock: Spinlock to protect all fields except @pcpu. It also protects
* the MM cid cpumask and the MM cidmask bitmap.
+ * @mutex: Mutex to serialize forks and exits related to this mm
*/
struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
@@ -132,6 +133,7 @@ struct mm_mm_cid {
unsigned int nr_cpus_allowed;
unsigned int users;
raw_spinlock_t lock;
+ struct mutex mutex;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10370,6 +10370,25 @@ void call_trace_sched_update_nr_running(
#ifdef CONFIG_SCHED_MM_CID
/*
+ * Concurrency IDentifier management
+ *
+ * Serialization rules:
+ *
+ * mm::mm_cid::mutex: Serializes fork() and exit() and therefore
+ * protects mm::mm_cid::users.
+ *
+ * mm::mm_cid::lock: Serializes mm_update_max_cids() and
+ * mm_update_cpus_allowed(). Nests in mm_cid::mutex
+ * and runqueue lock.
+ *
+ * The mm_cidmask bitmap is not protected by any of the mm::mm_cid locks
+ * and can only be modified with atomic operations.
+ *
+ * The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
+ * lock.
+ */
+
+/*
* Update the CID range properties when the constraints change. Invoked via
* fork(), exit() and affinity changes
*/
@@ -10412,6 +10431,7 @@ void sched_mm_cid_fork(struct task_struc
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
+ guard(mutex)(&mm->mm_cid.mutex);
guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 1;
mm->mm_cid.users++;
@@ -10431,6 +10451,7 @@ void sched_mm_cid_exit(struct task_struc
if (!mm || !t->mm_cid.active)
return;
+ guard(mutex)(&mm->mm_cid.mutex);
guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 0;
mm->mm_cid.users--;
@@ -10467,6 +10488,7 @@ void mm_init_cid(struct mm_struct *mm, s
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
+ mutex_init(&mm->mm_cid.mutex);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 51dd92c71a38647803478fb81e1812286a8998b1
Gitweb: https://git.kernel.org/tip/51dd92c71a38647803478fb81e1812286a8998b1
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:11 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:41 +01:00
sched/mmcid: Serialize sched_mm_cid_fork()/exit() with a mutex
Prepare for the new CID management scheme which puts the CID ownership
transition into the fork() and exit() slow path by serializing
sched_mm_cid_fork()/exit() with it, so task list and cpu mask walks can be
done in interruptible and preemptible code.
The contention on it is not worse than on other concurrency controls in the
fork()/exit() machinery.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.895826703@linutronix.de
---
include/linux/rseq_types.h | 2 ++
kernel/sched/core.c | 22 ++++++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 0fab369..574aba6 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -125,6 +125,7 @@ struct mm_cid_pcpu {
* do not actually share the MM.
* @lock: Spinlock to protect all fields except @pcpu. It also protects
* the MM cid cpumask and the MM cidmask bitmap.
+ * @mutex: Mutex to serialize forks and exits related to this mm
*/
struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
@@ -132,6 +133,7 @@ struct mm_mm_cid {
unsigned int nr_cpus_allowed;
unsigned int users;
raw_spinlock_t lock;
+ struct mutex mutex;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f9295c4..01903cf 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10370,6 +10370,25 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
#ifdef CONFIG_SCHED_MM_CID
/*
+ * Concurrency IDentifier management
+ *
+ * Serialization rules:
+ *
+ * mm::mm_cid::mutex: Serializes fork() and exit() and therefore
+ * protects mm::mm_cid::users.
+ *
+ * mm::mm_cid::lock: Serializes mm_update_max_cids() and
+ * mm_update_cpus_allowed(). Nests in mm_cid::mutex
+ * and runqueue lock.
+ *
+ * The mm_cidmask bitmap is not protected by any of the mm::mm_cid locks
+ * and can only be modified with atomic operations.
+ *
+ * The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
+ * lock.
+ */
+
+/*
* Update the CID range properties when the constraints change. Invoked via
* fork(), exit() and affinity changes
*/
@@ -10412,6 +10431,7 @@ void sched_mm_cid_fork(struct task_struct *t)
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
+ guard(mutex)(&mm->mm_cid.mutex);
guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 1;
mm->mm_cid.users++;
@@ -10431,6 +10451,7 @@ void sched_mm_cid_exit(struct task_struct *t)
if (!mm || !t->mm_cid.active)
return;
+ guard(mutex)(&mm->mm_cid.mutex);
guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 0;
mm->mm_cid.users--;
@@ -10467,6 +10488,7 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
+ mutex_init(&mm->mm_cid.mutex);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 4c42755fe64a411d9f299c1f97e2d114698b6759
Gitweb: https://git.kernel.org/tip/4c42755fe64a411d9f299c1f97e2d114698b6759
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:11 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:56 +01:00
sched/mmcid: Serialize sched_mm_cid_fork()/exit() with a mutex
Prepare for the new CID management scheme which puts the CID ownership
transition into the fork() and exit() slow path by serializing
sched_mm_cid_fork()/exit() with it, so task list and cpu mask walks can be
done in interruptible and preemptible code.
The contention on it is not worse than on other concurrency controls in the
fork()/exit() machinery.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.895826703@linutronix.de
---
include/linux/rseq_types.h | 2 ++
kernel/sched/core.c | 22 ++++++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 0fab369..574aba6 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -125,6 +125,7 @@ struct mm_cid_pcpu {
* do not actually share the MM.
* @lock: Spinlock to protect all fields except @pcpu. It also protects
* the MM cid cpumask and the MM cidmask bitmap.
+ * @mutex: Mutex to serialize forks and exits related to this mm
*/
struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
@@ -132,6 +133,7 @@ struct mm_mm_cid {
unsigned int nr_cpus_allowed;
unsigned int users;
raw_spinlock_t lock;
+ struct mutex mutex;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f9295c4..01903cf 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10370,6 +10370,25 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
#ifdef CONFIG_SCHED_MM_CID
/*
+ * Concurrency IDentifier management
+ *
+ * Serialization rules:
+ *
+ * mm::mm_cid::mutex: Serializes fork() and exit() and therefore
+ * protects mm::mm_cid::users.
+ *
+ * mm::mm_cid::lock: Serializes mm_update_max_cids() and
+ * mm_update_cpus_allowed(). Nests in mm_cid::mutex
+ * and runqueue lock.
+ *
+ * The mm_cidmask bitmap is not protected by any of the mm::mm_cid locks
+ * and can only be modified with atomic operations.
+ *
+ * The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
+ * lock.
+ */
+
+/*
* Update the CID range properties when the constraints change. Invoked via
* fork(), exit() and affinity changes
*/
@@ -10412,6 +10431,7 @@ void sched_mm_cid_fork(struct task_struct *t)
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
+ guard(mutex)(&mm->mm_cid.mutex);
guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 1;
mm->mm_cid.users++;
@@ -10431,6 +10451,7 @@ void sched_mm_cid_exit(struct task_struct *t)
if (!mm || !t->mm_cid.active)
return;
+ guard(mutex)(&mm->mm_cid.mutex);
guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 0;
mm->mm_cid.users--;
@@ -10467,6 +10488,7 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
+ mutex_init(&mm->mm_cid.mutex);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}The MM CID management has two fundamental requirements:
1) It has to guarantee that at no given point in time the same CID is
used by concurrent tasks in userspace.
2) The CID space must not exceed the number of possible CPUs in a
system. While most allocators (glibc, tcmalloc, jemalloc) do not care
about that, there seems to be at least librseq depending on it.
The CID space compaction itself is not a functional correctness
requirement, it is only a useful optimization mechanism to reduce the
memory foot print in unused user space pools.
The optimal CID space is:
min(nr_tasks, nr_cpus_allowed);
Where @nr_tasks is the number of actual user space threads associated to
the mm and @nr_cpus_allowed is the superset of all task affinities. It is
growth only as it would be insane to take a racy snapshot of all task
affinities when the affinity of one task changes just do redo it 2
milliseconds later when the next task changes its affinity.
That means that as long as the number of tasks is lower or equal than the
number of CPUs allowed, each task owns a CID. If the number of tasks
exceeds the number of CPUs allowed it switches to per CPU mode, where the
CPUs own the CIDs and the tasks borrow them as long as they are scheduled
in.
For transition periods CIDs can go beyond the optimal space as long as they
don't go beyond the number of possible CPUs.
The current upstream implementation adds overhead into task migration to
keep the CID with the task. It also has to do the CID space consolidation
work from a task work in the exit to user space path. As that work is
assigned to a random task related to a MM this can inflict unwanted exit
latencies.
This can be done differently by implementing a strict CID ownership
mechanism. Either the CIDs are owned by the tasks or by the CPUs. The
latter provides less locality when tasks are heavily migrating, but there
is no justification to optimize for overcommit scenarios and thereby
penalizing everyone else.
Provide the basic infrastructure to implement this:
- Change the UNSET marker to BIT(31) from ~0U
- Add the ONCPU marker as BIT(30)
- Add the TRANSIT marker as BIT(29)
That allows to check for ownership trivially and provides a simple check for
UNSET as well. The TRANSIT marker is required to prevent CID space
exhaustion when switching from per CPU to per task mode.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
V5: Reword change log
V2: Add the TRANSIT(ion) bit
---
include/linux/rseq_types.h | 4 ++-
include/linux/sched.h | 6 ++--
kernel/sched/core.c | 10 +++++++
kernel/sched/sched.h | 59 +++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 75 insertions(+), 4 deletions(-)
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -92,7 +92,9 @@ struct rseq_data { };
#ifdef CONFIG_SCHED_MM_CID
-#define MM_CID_UNSET (~0U)
+#define MM_CID_UNSET BIT(31)
+#define MM_CID_ONCPU BIT(30)
+#define MM_CID_TRANSIT BIT(29)
/**
* struct sched_mm_cid - Storage for per task MM CID data
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2299,16 +2299,16 @@ void sched_mm_cid_before_execve(struct t
void sched_mm_cid_after_execve(struct task_struct *t);
void sched_mm_cid_fork(struct task_struct *t);
void sched_mm_cid_exit(struct task_struct *t);
-static inline int task_mm_cid(struct task_struct *t)
+static __always_inline int task_mm_cid(struct task_struct *t)
{
- return t->mm_cid.cid;
+ return t->mm_cid.cid & ~(MM_CID_ONCPU | MM_CID_TRANSIT);
}
#else
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
static inline void sched_mm_cid_after_execve(struct task_struct *t) { }
static inline void sched_mm_cid_fork(struct task_struct *t) { }
static inline void sched_mm_cid_exit(struct task_struct *t) { }
-static inline int task_mm_cid(struct task_struct *t)
+static __always_inline int task_mm_cid(struct task_struct *t)
{
/*
* Use the processor id as a fall-back when the mm cid feature is
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10386,6 +10386,16 @@ void call_trace_sched_update_nr_running(
*
* The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
* lock.
+ *
+ * CID ownership:
+ *
+ * A CID is either owned by a task (stored in task_struct::mm_cid.cid) or
+ * by a CPU (stored in mm::mm_cid.pcpu::cid). CIDs owned by CPUs have the
+ * MM_CID_ONCPU bit set. During transition from CPU to task ownership mode,
+ * MM_CID_TRANSIT is set on the per task CIDs. When this bit is set the
+ * task needs to drop the CID into the pool when scheduling out. Both bits
+ * (ONCPU and TRANSIT) are filtered out by task_cid() when the CID is
+ * actually handed over to user space in the RSEQ memory.
*/
/*
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3540,6 +3540,65 @@ extern void sched_dynamic_update(int mod
extern const char *preempt_modes[];
#ifdef CONFIG_SCHED_MM_CID
+
+static __always_inline bool cid_on_cpu(unsigned int cid)
+{
+ return cid & MM_CID_ONCPU;
+}
+
+static __always_inline bool cid_in_transit(unsigned int cid)
+{
+ return cid & MM_CID_TRANSIT;
+}
+
+static __always_inline unsigned int cpu_cid_to_cid(unsigned int cid)
+{
+ return cid & ~MM_CID_ONCPU;
+}
+
+static __always_inline unsigned int cid_to_cpu_cid(unsigned int cid)
+{
+ return cid | MM_CID_ONCPU;
+}
+
+static __always_inline unsigned int cid_to_transit_cid(unsigned int cid)
+{
+ return cid | MM_CID_TRANSIT;
+}
+
+static __always_inline unsigned int cid_from_transit_cid(unsigned int cid)
+{
+ return cid & ~MM_CID_TRANSIT;
+}
+
+static __always_inline bool cid_on_task(unsigned int cid)
+{
+ /* True if none of the MM_CID_ONCPU, MM_CID_TRANSIT, MM_CID_UNSET bits is set */
+ return cid < MM_CID_TRANSIT;
+}
+
+static __always_inline void mm_drop_cid(struct mm_struct *mm, unsigned int cid)
+{
+ clear_bit(cid, mm_cidmask(mm));
+}
+
+static __always_inline void mm_unset_cid_on_task(struct task_struct *t)
+{
+ unsigned int cid = t->mm_cid.cid;
+
+ t->mm_cid.cid = MM_CID_UNSET;
+ if (cid_on_task(cid))
+ mm_drop_cid(t->mm, cid);
+}
+
+static __always_inline void mm_drop_cid_on_cpu(struct mm_struct *mm, struct mm_cid_pcpu *pcp)
+{
+ /* Clear the ONCPU bit, but do not set UNSET in the per CPU storage */
+ pcp->cid = cpu_cid_to_cid(pcp->cid);
+ mm_drop_cid(mm, pcp->cid);
+}
+
+/* Active implementation */
static inline void init_sched_mm_cid(struct task_struct *t)
{
struct mm_struct *mm = t->mm;The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 23343b6b09acb4bf97f34ed60e135000ca57ede1
Gitweb: https://git.kernel.org/tip/23343b6b09acb4bf97f34ed60e135000ca57ede1
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:12 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:41 +01:00
sched/mmcid: Introduce per task/CPU ownership infrastructure
The MM CID management has two fundamental requirements:
1) It has to guarantee that at no given point in time the same CID is
used by concurrent tasks in userspace.
2) The CID space must not exceed the number of possible CPUs in a
system. While most allocators (glibc, tcmalloc, jemalloc) do not care
about that, there seems to be at least librseq depending on it.
The CID space compaction itself is not a functional correctness
requirement, it is only a useful optimization mechanism to reduce the
memory foot print in unused user space pools.
The optimal CID space is:
min(nr_tasks, nr_cpus_allowed);
Where @nr_tasks is the number of actual user space threads associated to
the mm and @nr_cpus_allowed is the superset of all task affinities. It is
growth only as it would be insane to take a racy snapshot of all task
affinities when the affinity of one task changes just do redo it 2
milliseconds later when the next task changes its affinity.
That means that as long as the number of tasks is lower or equal than the
number of CPUs allowed, each task owns a CID. If the number of tasks
exceeds the number of CPUs allowed it switches to per CPU mode, where the
CPUs own the CIDs and the tasks borrow them as long as they are scheduled
in.
For transition periods CIDs can go beyond the optimal space as long as they
don't go beyond the number of possible CPUs.
The current upstream implementation adds overhead into task migration to
keep the CID with the task. It also has to do the CID space consolidation
work from a task work in the exit to user space path. As that work is
assigned to a random task related to a MM this can inflict unwanted exit
latencies.
This can be done differently by implementing a strict CID ownership
mechanism. Either the CIDs are owned by the tasks or by the CPUs. The
latter provides less locality when tasks are heavily migrating, but there
is no justification to optimize for overcommit scenarios and thereby
penalizing everyone else.
Provide the basic infrastructure to implement this:
- Change the UNSET marker to BIT(31) from ~0U
- Add the ONCPU marker as BIT(30)
- Add the TRANSIT marker as BIT(29)
That allows to check for ownership trivially and provides a simple check for
UNSET as well. The TRANSIT marker is required to prevent CID space
exhaustion when switching from per CPU to per task mode.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://patch.msgid.link/20251119172549.960252358@linutronix.de
---
include/linux/rseq_types.h | 4 ++-
include/linux/sched.h | 6 ++--
kernel/sched/core.c | 10 ++++++-
kernel/sched/sched.h | 59 +++++++++++++++++++++++++++++++++++++-
4 files changed, 75 insertions(+), 4 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 574aba6..87854ef 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -92,7 +92,9 @@ struct rseq_data { };
#ifdef CONFIG_SCHED_MM_CID
-#define MM_CID_UNSET (~0U)
+#define MM_CID_UNSET BIT(31)
+#define MM_CID_ONCPU BIT(30)
+#define MM_CID_TRANSIT BIT(29)
/**
* struct sched_mm_cid - Storage for per task MM CID data
diff --git a/include/linux/sched.h b/include/linux/sched.h
index c411ae0..9eec409 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2299,16 +2299,16 @@ void sched_mm_cid_before_execve(struct task_struct *t);
void sched_mm_cid_after_execve(struct task_struct *t);
void sched_mm_cid_fork(struct task_struct *t);
void sched_mm_cid_exit(struct task_struct *t);
-static inline int task_mm_cid(struct task_struct *t)
+static __always_inline int task_mm_cid(struct task_struct *t)
{
- return t->mm_cid.cid;
+ return t->mm_cid.cid & ~(MM_CID_ONCPU | MM_CID_TRANSIT);
}
#else
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
static inline void sched_mm_cid_after_execve(struct task_struct *t) { }
static inline void sched_mm_cid_fork(struct task_struct *t) { }
static inline void sched_mm_cid_exit(struct task_struct *t) { }
-static inline int task_mm_cid(struct task_struct *t)
+static __always_inline int task_mm_cid(struct task_struct *t)
{
/*
* Use the processor id as a fall-back when the mm cid feature is
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 01903cf..55bb9c9 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10386,6 +10386,16 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
*
* The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
* lock.
+ *
+ * CID ownership:
+ *
+ * A CID is either owned by a task (stored in task_struct::mm_cid.cid) or
+ * by a CPU (stored in mm::mm_cid.pcpu::cid). CIDs owned by CPUs have the
+ * MM_CID_ONCPU bit set. During transition from CPU to task ownership mode,
+ * MM_CID_TRANSIT is set on the per task CIDs. When this bit is set the
+ * task needs to drop the CID into the pool when scheduling out. Both bits
+ * (ONCPU and TRANSIT) are filtered out by task_cid() when the CID is
+ * actually handed over to user space in the RSEQ memory.
*/
/*
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index d539fb2..4b49284 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3540,6 +3540,65 @@ extern void sched_dynamic_update(int mode);
extern const char *preempt_modes[];
#ifdef CONFIG_SCHED_MM_CID
+
+static __always_inline bool cid_on_cpu(unsigned int cid)
+{
+ return cid & MM_CID_ONCPU;
+}
+
+static __always_inline bool cid_in_transit(unsigned int cid)
+{
+ return cid & MM_CID_TRANSIT;
+}
+
+static __always_inline unsigned int cpu_cid_to_cid(unsigned int cid)
+{
+ return cid & ~MM_CID_ONCPU;
+}
+
+static __always_inline unsigned int cid_to_cpu_cid(unsigned int cid)
+{
+ return cid | MM_CID_ONCPU;
+}
+
+static __always_inline unsigned int cid_to_transit_cid(unsigned int cid)
+{
+ return cid | MM_CID_TRANSIT;
+}
+
+static __always_inline unsigned int cid_from_transit_cid(unsigned int cid)
+{
+ return cid & ~MM_CID_TRANSIT;
+}
+
+static __always_inline bool cid_on_task(unsigned int cid)
+{
+ /* True if none of the MM_CID_ONCPU, MM_CID_TRANSIT, MM_CID_UNSET bits is set */
+ return cid < MM_CID_TRANSIT;
+}
+
+static __always_inline void mm_drop_cid(struct mm_struct *mm, unsigned int cid)
+{
+ clear_bit(cid, mm_cidmask(mm));
+}
+
+static __always_inline void mm_unset_cid_on_task(struct task_struct *t)
+{
+ unsigned int cid = t->mm_cid.cid;
+
+ t->mm_cid.cid = MM_CID_UNSET;
+ if (cid_on_task(cid))
+ mm_drop_cid(t->mm, cid);
+}
+
+static __always_inline void mm_drop_cid_on_cpu(struct mm_struct *mm, struct mm_cid_pcpu *pcp)
+{
+ /* Clear the ONCPU bit, but do not set UNSET in the per CPU storage */
+ pcp->cid = cpu_cid_to_cid(pcp->cid);
+ mm_drop_cid(mm, pcp->cid);
+}
+
+/* Active implementation */
static inline void init_sched_mm_cid(struct task_struct *t)
{
struct mm_struct *mm = t->mm;The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 1bcc335bb29457d402b4fb033673aa89eca8187b
Gitweb: https://git.kernel.org/tip/1bcc335bb29457d402b4fb033673aa89eca8187b
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:12 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:57 +01:00
sched/mmcid: Introduce per task/CPU ownership infrastructure
The MM CID management has two fundamental requirements:
1) It has to guarantee that at no given point in time the same CID is
used by concurrent tasks in userspace.
2) The CID space must not exceed the number of possible CPUs in a
system. While most allocators (glibc, tcmalloc, jemalloc) do not care
about that, there seems to be at least librseq depending on it.
The CID space compaction itself is not a functional correctness
requirement, it is only a useful optimization mechanism to reduce the
memory foot print in unused user space pools.
The optimal CID space is:
min(nr_tasks, nr_cpus_allowed);
Where @nr_tasks is the number of actual user space threads associated to
the mm and @nr_cpus_allowed is the superset of all task affinities. It is
growth only as it would be insane to take a racy snapshot of all task
affinities when the affinity of one task changes just do redo it 2
milliseconds later when the next task changes its affinity.
That means that as long as the number of tasks is lower or equal than the
number of CPUs allowed, each task owns a CID. If the number of tasks
exceeds the number of CPUs allowed it switches to per CPU mode, where the
CPUs own the CIDs and the tasks borrow them as long as they are scheduled
in.
For transition periods CIDs can go beyond the optimal space as long as they
don't go beyond the number of possible CPUs.
The current upstream implementation adds overhead into task migration to
keep the CID with the task. It also has to do the CID space consolidation
work from a task work in the exit to user space path. As that work is
assigned to a random task related to a MM this can inflict unwanted exit
latencies.
This can be done differently by implementing a strict CID ownership
mechanism. Either the CIDs are owned by the tasks or by the CPUs. The
latter provides less locality when tasks are heavily migrating, but there
is no justification to optimize for overcommit scenarios and thereby
penalizing everyone else.
Provide the basic infrastructure to implement this:
- Change the UNSET marker to BIT(31) from ~0U
- Add the ONCPU marker as BIT(30)
- Add the TRANSIT marker as BIT(29)
That allows to check for ownership trivially and provides a simple check for
UNSET as well. The TRANSIT marker is required to prevent CID space
exhaustion when switching from per CPU to per task mode.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20251119172549.960252358@linutronix.de
---
include/linux/rseq_types.h | 4 ++-
include/linux/sched.h | 6 ++--
kernel/sched/core.c | 10 ++++++-
kernel/sched/sched.h | 59 +++++++++++++++++++++++++++++++++++++-
4 files changed, 75 insertions(+), 4 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 574aba6..87854ef 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -92,7 +92,9 @@ struct rseq_data { };
#ifdef CONFIG_SCHED_MM_CID
-#define MM_CID_UNSET (~0U)
+#define MM_CID_UNSET BIT(31)
+#define MM_CID_ONCPU BIT(30)
+#define MM_CID_TRANSIT BIT(29)
/**
* struct sched_mm_cid - Storage for per task MM CID data
diff --git a/include/linux/sched.h b/include/linux/sched.h
index c411ae0..9eec409 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2299,16 +2299,16 @@ void sched_mm_cid_before_execve(struct task_struct *t);
void sched_mm_cid_after_execve(struct task_struct *t);
void sched_mm_cid_fork(struct task_struct *t);
void sched_mm_cid_exit(struct task_struct *t);
-static inline int task_mm_cid(struct task_struct *t)
+static __always_inline int task_mm_cid(struct task_struct *t)
{
- return t->mm_cid.cid;
+ return t->mm_cid.cid & ~(MM_CID_ONCPU | MM_CID_TRANSIT);
}
#else
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
static inline void sched_mm_cid_after_execve(struct task_struct *t) { }
static inline void sched_mm_cid_fork(struct task_struct *t) { }
static inline void sched_mm_cid_exit(struct task_struct *t) { }
-static inline int task_mm_cid(struct task_struct *t)
+static __always_inline int task_mm_cid(struct task_struct *t)
{
/*
* Use the processor id as a fall-back when the mm cid feature is
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 01903cf..55bb9c9 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10386,6 +10386,16 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
*
* The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
* lock.
+ *
+ * CID ownership:
+ *
+ * A CID is either owned by a task (stored in task_struct::mm_cid.cid) or
+ * by a CPU (stored in mm::mm_cid.pcpu::cid). CIDs owned by CPUs have the
+ * MM_CID_ONCPU bit set. During transition from CPU to task ownership mode,
+ * MM_CID_TRANSIT is set on the per task CIDs. When this bit is set the
+ * task needs to drop the CID into the pool when scheduling out. Both bits
+ * (ONCPU and TRANSIT) are filtered out by task_cid() when the CID is
+ * actually handed over to user space in the RSEQ memory.
*/
/*
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index d539fb2..4b49284 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3540,6 +3540,65 @@ extern void sched_dynamic_update(int mode);
extern const char *preempt_modes[];
#ifdef CONFIG_SCHED_MM_CID
+
+static __always_inline bool cid_on_cpu(unsigned int cid)
+{
+ return cid & MM_CID_ONCPU;
+}
+
+static __always_inline bool cid_in_transit(unsigned int cid)
+{
+ return cid & MM_CID_TRANSIT;
+}
+
+static __always_inline unsigned int cpu_cid_to_cid(unsigned int cid)
+{
+ return cid & ~MM_CID_ONCPU;
+}
+
+static __always_inline unsigned int cid_to_cpu_cid(unsigned int cid)
+{
+ return cid | MM_CID_ONCPU;
+}
+
+static __always_inline unsigned int cid_to_transit_cid(unsigned int cid)
+{
+ return cid | MM_CID_TRANSIT;
+}
+
+static __always_inline unsigned int cid_from_transit_cid(unsigned int cid)
+{
+ return cid & ~MM_CID_TRANSIT;
+}
+
+static __always_inline bool cid_on_task(unsigned int cid)
+{
+ /* True if none of the MM_CID_ONCPU, MM_CID_TRANSIT, MM_CID_UNSET bits is set */
+ return cid < MM_CID_TRANSIT;
+}
+
+static __always_inline void mm_drop_cid(struct mm_struct *mm, unsigned int cid)
+{
+ clear_bit(cid, mm_cidmask(mm));
+}
+
+static __always_inline void mm_unset_cid_on_task(struct task_struct *t)
+{
+ unsigned int cid = t->mm_cid.cid;
+
+ t->mm_cid.cid = MM_CID_UNSET;
+ if (cid_on_task(cid))
+ mm_drop_cid(t->mm, cid);
+}
+
+static __always_inline void mm_drop_cid_on_cpu(struct mm_struct *mm, struct mm_cid_pcpu *pcp)
+{
+ /* Clear the ONCPU bit, but do not set UNSET in the per CPU storage */
+ pcp->cid = cpu_cid_to_cid(pcp->cid);
+ mm_drop_cid(mm, pcp->cid);
+}
+
+/* Active implementation */
static inline void init_sched_mm_cid(struct task_struct *t)
{
struct mm_struct *mm = t->mm;The MM CID management has two fundamental requirements:
1) It has to guarantee that at no given point in time the same CID is
used by concurrent tasks in userspace.
2) The CID space must not exceed the number of possible CPUs in a
system. While most allocators (glibc, tcmalloc, jemalloc) do not
care about that, there seems to be at least some LTTng library
depending on it.
The CID space compaction itself is not a functional correctness
requirement, it is only a useful optimization mechanism to reduce the
memory foot print in unused user space pools.
The optimal CID space is:
min(nr_tasks, nr_cpus_allowed);
Where @nr_tasks is the number of actual user space threads associated to
the mm and @nr_cpus_allowed is the superset of all task affinities. It is
growth only as it would be insane to take a racy snapshot of all task
affinities when the affinity of one task changes just do redo it 2
milliseconds later when the next task changes it's affinity.
That means that as long as the number of tasks is lower or equal than the
number of CPUs allowed, each task owns a CID. If the number of tasks
exceeds the number of CPUs allowed it switches to per CPU mode, where the
CPUs own the CIDs and the tasks borrow them as long as they are scheduled
in.
For transition periods CIDs can go beyond the optimal space as long as they
don't go beyond the number of possible CPUs.
The current upstream implementation adds overhead into task migration to
keep the CID with the task. It also has to do the CID space consolidation
work from a task work in the exit to user space path. As that work is
assigned to a random task related to a MM this can inflict unwanted exit
latencies.
Implement the context switch parts of a strict ownership mechanism to
address this.
This removes most of the work from the task which schedules out. Only
during transitioning from per CPU to per task ownership it is required to
drop the CID when leaving the CPU to prevent CID space exhaustion. Other
than that scheduling out is just a single check and branch.
The task which schedules in has to check whether:
1) The ownership mode changed
2) The CID is within the optimal CID space
In stable situations this results in zero work. The only short disruption
is when ownership mode changes or when the associated CID is not in the
optimal CID space. The latter only happens when tasks exit and therefore
the optimal CID space shrinks.
That mechanism is strictly optimized for the common case where no change
happens. The only case where it actually causes a temporary one time spike
is on mode changes when and only when a lot of tasks related to a MM
schedule exactly at the same time and have eventually to compete on
allocating a CID from the bitmap.
In the sysbench test case which triggered the spinlock contention in the
initial CID code, __schedule() drops significantly in perf top on a 128
Core (256 threads) machine when running sysbench with 255 threads, which
fits into the task mode limit of 256 together with the parent thread:
Upstream rseq/perf branch +CID rework
0.42% 0.37% 0.32% [k] __schedule
Increasing the number of threads to 256, which puts the test process into
per CPU mode looks about the same.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
V5: Polish changelog and initialize transit - Mathieu
V3: Move cpu_relax() out of the for() construct - Shrikanth
V2: Adjust to the mode transition changes
---
include/linux/rseq.h | 8 +-
include/linux/rseq_types.h | 18 +++--
kernel/sched/core.c | 2
kernel/sched/sched.h | 150 ++++++++++++++++++++++++++++++++++++++++++++-
4 files changed, 168 insertions(+), 10 deletions(-)
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -73,13 +73,13 @@ static __always_inline void rseq_sched_s
}
/*
- * Invoked from __set_task_cpu() when a task migrates to enforce an IDs
- * update.
+ * Invoked from __set_task_cpu() when a task migrates or from
+ * mm_cid_schedin() when the CID changes to enforce an IDs update.
*
* This does not raise TIF_NOTIFY_RESUME as that happens in
* rseq_sched_switch_event().
*/
-static __always_inline void rseq_sched_set_task_cpu(struct task_struct *t, unsigned int cpu)
+static __always_inline void rseq_sched_set_ids_changed(struct task_struct *t)
{
t->rseq.event.ids_changed = true;
}
@@ -169,7 +169,7 @@ static inline void rseq_fork(struct task
static inline void rseq_handle_slowpath(struct pt_regs *regs) { }
static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }
static inline void rseq_sched_switch_event(struct task_struct *t) { }
-static inline void rseq_sched_set_task_cpu(struct task_struct *t, unsigned int cpu) { }
+static inline void rseq_sched_set_ids_changed(struct task_struct *t) { }
static inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid) { }
static inline void rseq_force_update(void) { }
static inline void rseq_virt_userspace_exit(void) { }
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -119,23 +119,31 @@ struct mm_cid_pcpu {
/**
* struct mm_mm_cid - Storage for per MM CID data
* @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @percpu: Set, when CIDs are in per CPU mode
+ * @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
+ * @lock: Spinlock to protect all fields except @pcpu. It also protects
+ * the MM cid cpumask and the MM cidmask bitmap.
+ * @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
* @users: The number of tasks sharing this MM. Separate from mm::mm_users
* as that is modified by mmget()/mm_put() by other entities which
* do not actually share the MM.
- * @lock: Spinlock to protect all fields except @pcpu. It also protects
- * the MM cid cpumask and the MM cidmask bitmap.
- * @mutex: Mutex to serialize forks and exits related to this mm
*/
struct mm_mm_cid {
+ /* Hotpath read mostly members */
struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int percpu;
+ unsigned int transit;
unsigned int max_cids;
- unsigned int nr_cpus_allowed;
- unsigned int users;
+
raw_spinlock_t lock;
struct mutex mutex;
+
+ /* Low frequency modified */
+ unsigned int nr_cpus_allowed;
+ unsigned int users;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10495,6 +10495,8 @@ void mm_init_cid(struct mm_struct *mm, s
per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
mm->mm_cid.max_cids = 0;
+ mm->mm_cid.percpu = 0;
+ mm->mm_cid.transit = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2209,7 +2209,7 @@ static inline void __set_task_cpu(struct
smp_wmb();
WRITE_ONCE(task_thread_info(p)->cpu, cpu);
p->wake_cpu = cpu;
- rseq_sched_set_task_cpu(p, cpu);
+ rseq_sched_set_ids_changed(p);
#endif /* CONFIG_SMP */
}
@@ -3598,6 +3598,153 @@ static __always_inline void mm_drop_cid_
mm_drop_cid(mm, pcp->cid);
}
+static inline unsigned int __mm_get_cid(struct mm_struct *mm, unsigned int max_cids)
+{
+ unsigned int cid = find_first_zero_bit(mm_cidmask(mm), max_cids);
+
+ if (cid >= max_cids)
+ return MM_CID_UNSET;
+ if (test_and_set_bit(cid, mm_cidmask(mm)))
+ return MM_CID_UNSET;
+ return cid;
+}
+
+static inline unsigned int mm_get_cid(struct mm_struct *mm)
+{
+ unsigned int cid = __mm_get_cid(mm, READ_ONCE(mm->mm_cid.max_cids));
+
+ while (cid == MM_CID_UNSET) {
+ cpu_relax();
+ cid = __mm_get_cid(mm, num_possible_cpus());
+ }
+ return cid;
+}
+
+static inline unsigned int mm_cid_converge(struct mm_struct *mm, unsigned int orig_cid,
+ unsigned int max_cids)
+{
+ unsigned int new_cid, cid = cpu_cid_to_cid(orig_cid);
+
+ /* Is it in the optimal CID space? */
+ if (likely(cid < max_cids))
+ return orig_cid;
+
+ /* Try to find one in the optimal space. Otherwise keep the provided. */
+ new_cid = __mm_get_cid(mm, max_cids);
+ if (new_cid != MM_CID_UNSET) {
+ mm_drop_cid(mm, cid);
+ /* Preserve the ONCPU mode of the original CID */
+ return new_cid | (orig_cid & MM_CID_ONCPU);
+ }
+ return orig_cid;
+}
+
+static __always_inline void mm_cid_update_task_cid(struct task_struct *t, unsigned int cid)
+{
+ if (t->mm_cid.cid != cid) {
+ t->mm_cid.cid = cid;
+ rseq_sched_set_ids_changed(t);
+ }
+}
+
+static __always_inline void mm_cid_update_pcpu_cid(struct mm_struct *mm, unsigned int cid)
+{
+ __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
+}
+
+static __always_inline void mm_cid_from_cpu(struct task_struct *t, unsigned int cpu_cid)
+{
+ unsigned int max_cids, tcid = t->mm_cid.cid;
+ struct mm_struct *mm = t->mm;
+
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
+ /* Optimize for the common case where both have the ONCPU bit set */
+ if (likely(cid_on_cpu(cpu_cid & tcid))) {
+ if (likely(cpu_cid_to_cid(cpu_cid) < max_cids)) {
+ mm_cid_update_task_cid(t, cpu_cid);
+ return;
+ }
+ /* Try to converge into the optimal CID space */
+ cpu_cid = mm_cid_converge(mm, cpu_cid, max_cids);
+ } else {
+ /* Hand over or drop the task owned CID */
+ if (cid_on_task(tcid)) {
+ if (cid_on_cpu(cpu_cid))
+ mm_unset_cid_on_task(t);
+ else
+ cpu_cid = cid_to_cpu_cid(tcid);
+ }
+ /* Still nothing, allocate a new one */
+ if (!cid_on_cpu(cpu_cid))
+ cpu_cid = cid_to_cpu_cid(mm_get_cid(mm));
+ }
+ mm_cid_update_pcpu_cid(mm, cpu_cid);
+ mm_cid_update_task_cid(t, cpu_cid);
+}
+
+static __always_inline void mm_cid_from_task(struct task_struct *t, unsigned int cpu_cid)
+{
+ unsigned int max_cids, tcid = t->mm_cid.cid;
+ struct mm_struct *mm = t->mm;
+
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
+ /* Optimize for the common case, where both have the ONCPU bit clear */
+ if (likely(cid_on_task(tcid | cpu_cid))) {
+ if (likely(tcid < max_cids)) {
+ mm_cid_update_pcpu_cid(mm, tcid);
+ return;
+ }
+ /* Try to converge into the optimal CID space */
+ tcid = mm_cid_converge(mm, tcid, max_cids);
+ } else {
+ /* Hand over or drop the CPU owned CID */
+ if (cid_on_cpu(cpu_cid)) {
+ if (cid_on_task(tcid))
+ mm_drop_cid_on_cpu(mm, this_cpu_ptr(mm->mm_cid.pcpu));
+ else
+ tcid = cpu_cid_to_cid(cpu_cid);
+ }
+ /* Still nothing, allocate a new one */
+ if (!cid_on_task(tcid))
+ tcid = mm_get_cid(mm);
+ /* Set the transition mode flag if required */
+ tcid |= READ_ONCE(mm->mm_cid.transit);
+ }
+ mm_cid_update_pcpu_cid(mm, tcid);
+ mm_cid_update_task_cid(t, tcid);
+}
+
+static __always_inline void mm_cid_schedin(struct task_struct *next)
+{
+ struct mm_struct *mm = next->mm;
+ unsigned int cpu_cid;
+
+ if (!next->mm_cid.active)
+ return;
+
+ cpu_cid = __this_cpu_read(mm->mm_cid.pcpu->cid);
+ if (likely(!READ_ONCE(mm->mm_cid.percpu)))
+ mm_cid_from_task(next, cpu_cid);
+ else
+ mm_cid_from_cpu(next, cpu_cid);
+}
+
+static __always_inline void mm_cid_schedout(struct task_struct *prev)
+{
+ /* During mode transitions CIDs are temporary and need to be dropped */
+ if (likely(!cid_in_transit(prev->mm_cid.cid)))
+ return;
+
+ mm_drop_cid(prev->mm, cid_from_transit_cid(prev->mm_cid.cid));
+ prev->mm_cid.cid = MM_CID_UNSET;
+}
+
+static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next)
+{
+ mm_cid_schedout(prev);
+ mm_cid_schedin(next);
+}
+
/* Active implementation */
static inline void init_sched_mm_cid(struct task_struct *t)
{
@@ -3675,6 +3822,7 @@ static inline void switch_mm_cid(struct
#else /* !CONFIG_SCHED_MM_CID: */
static inline void mm_cid_select(struct task_struct *t) { }
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
+static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
extern u64 avg_vruntime(struct cfs_rq *cfs_rq);The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 9a723ed7facff6955da8d64cc9de7066038036c1
Gitweb: https://git.kernel.org/tip/9a723ed7facff6955da8d64cc9de7066038036c1
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:14 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:41 +01:00
sched/mmcid: Provide new scheduler CID mechanism
The MM CID management has two fundamental requirements:
1) It has to guarantee that at no given point in time the same CID is
used by concurrent tasks in userspace.
2) The CID space must not exceed the number of possible CPUs in a
system. While most allocators (glibc, tcmalloc, jemalloc) do not
care about that, there seems to be at least some LTTng library
depending on it.
The CID space compaction itself is not a functional correctness
requirement, it is only a useful optimization mechanism to reduce the
memory foot print in unused user space pools.
The optimal CID space is:
min(nr_tasks, nr_cpus_allowed);
Where @nr_tasks is the number of actual user space threads associated to
the mm and @nr_cpus_allowed is the superset of all task affinities. It is
growth only as it would be insane to take a racy snapshot of all task
affinities when the affinity of one task changes just do redo it 2
milliseconds later when the next task changes it's affinity.
That means that as long as the number of tasks is lower or equal than the
number of CPUs allowed, each task owns a CID. If the number of tasks
exceeds the number of CPUs allowed it switches to per CPU mode, where the
CPUs own the CIDs and the tasks borrow them as long as they are scheduled
in.
For transition periods CIDs can go beyond the optimal space as long as they
don't go beyond the number of possible CPUs.
The current upstream implementation adds overhead into task migration to
keep the CID with the task. It also has to do the CID space consolidation
work from a task work in the exit to user space path. As that work is
assigned to a random task related to a MM this can inflict unwanted exit
latencies.
Implement the context switch parts of a strict ownership mechanism to
address this.
This removes most of the work from the task which schedules out. Only
during transitioning from per CPU to per task ownership it is required to
drop the CID when leaving the CPU to prevent CID space exhaustion. Other
than that scheduling out is just a single check and branch.
The task which schedules in has to check whether:
1) The ownership mode changed
2) The CID is within the optimal CID space
In stable situations this results in zero work. The only short disruption
is when ownership mode changes or when the associated CID is not in the
optimal CID space. The latter only happens when tasks exit and therefore
the optimal CID space shrinks.
That mechanism is strictly optimized for the common case where no change
happens. The only case where it actually causes a temporary one time spike
is on mode changes when and only when a lot of tasks related to a MM
schedule exactly at the same time and have eventually to compete on
allocating a CID from the bitmap.
In the sysbench test case which triggered the spinlock contention in the
initial CID code, __schedule() drops significantly in perf top on a 128
Core (256 threads) machine when running sysbench with 255 threads, which
fits into the task mode limit of 256 together with the parent thread:
Upstream rseq/perf branch +CID rework
0.42% 0.37% 0.32% [k] __schedule
Increasing the number of threads to 256, which puts the test process into
per CPU mode looks about the same.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.023984859@linutronix.de
---
include/linux/rseq.h | 8 +-
include/linux/rseq_types.h | 18 ++--
kernel/sched/core.c | 2 +-
kernel/sched/sched.h | 150 +++++++++++++++++++++++++++++++++++-
4 files changed, 168 insertions(+), 10 deletions(-)
diff --git a/include/linux/rseq.h b/include/linux/rseq.h
index bf8a6bf..4c0e8bd 100644
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -73,13 +73,13 @@ static __always_inline void rseq_sched_switch_event(struct task_struct *t)
}
/*
- * Invoked from __set_task_cpu() when a task migrates to enforce an IDs
- * update.
+ * Invoked from __set_task_cpu() when a task migrates or from
+ * mm_cid_schedin() when the CID changes to enforce an IDs update.
*
* This does not raise TIF_NOTIFY_RESUME as that happens in
* rseq_sched_switch_event().
*/
-static __always_inline void rseq_sched_set_task_cpu(struct task_struct *t, unsigned int cpu)
+static __always_inline void rseq_sched_set_ids_changed(struct task_struct *t)
{
t->rseq.event.ids_changed = true;
}
@@ -168,7 +168,7 @@ static inline void rseq_fork(struct task_struct *t, u64 clone_flags)
static inline void rseq_handle_slowpath(struct pt_regs *regs) { }
static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }
static inline void rseq_sched_switch_event(struct task_struct *t) { }
-static inline void rseq_sched_set_task_cpu(struct task_struct *t, unsigned int cpu) { }
+static inline void rseq_sched_set_ids_changed(struct task_struct *t) { }
static inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid) { }
static inline void rseq_force_update(void) { }
static inline void rseq_virt_userspace_exit(void) { }
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 87854ef..66b1482 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -119,23 +119,31 @@ struct mm_cid_pcpu {
/**
* struct mm_mm_cid - Storage for per MM CID data
* @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @percpu: Set, when CIDs are in per CPU mode
+ * @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
+ * @lock: Spinlock to protect all fields except @pcpu. It also protects
+ * the MM cid cpumask and the MM cidmask bitmap.
+ * @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
* @users: The number of tasks sharing this MM. Separate from mm::mm_users
* as that is modified by mmget()/mm_put() by other entities which
* do not actually share the MM.
- * @lock: Spinlock to protect all fields except @pcpu. It also protects
- * the MM cid cpumask and the MM cidmask bitmap.
- * @mutex: Mutex to serialize forks and exits related to this mm
*/
struct mm_mm_cid {
+ /* Hotpath read mostly members */
struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int percpu;
+ unsigned int transit;
unsigned int max_cids;
- unsigned int nr_cpus_allowed;
- unsigned int users;
+
raw_spinlock_t lock;
struct mutex mutex;
+
+ /* Low frequency modified */
+ unsigned int nr_cpus_allowed;
+ unsigned int users;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 55bb9c9..659ae56 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10495,6 +10495,8 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
mm->mm_cid.max_cids = 0;
+ mm->mm_cid.percpu = 0;
+ mm->mm_cid.transit = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 4b49284..82c7978 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2209,7 +2209,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
smp_wmb();
WRITE_ONCE(task_thread_info(p)->cpu, cpu);
p->wake_cpu = cpu;
- rseq_sched_set_task_cpu(p, cpu);
+ rseq_sched_set_ids_changed(p);
#endif /* CONFIG_SMP */
}
@@ -3598,6 +3598,153 @@ static __always_inline void mm_drop_cid_on_cpu(struct mm_struct *mm, struct mm_c
mm_drop_cid(mm, pcp->cid);
}
+static inline unsigned int __mm_get_cid(struct mm_struct *mm, unsigned int max_cids)
+{
+ unsigned int cid = find_first_zero_bit(mm_cidmask(mm), max_cids);
+
+ if (cid >= max_cids)
+ return MM_CID_UNSET;
+ if (test_and_set_bit(cid, mm_cidmask(mm)))
+ return MM_CID_UNSET;
+ return cid;
+}
+
+static inline unsigned int mm_get_cid(struct mm_struct *mm)
+{
+ unsigned int cid = __mm_get_cid(mm, READ_ONCE(mm->mm_cid.max_cids));
+
+ while (cid == MM_CID_UNSET) {
+ cpu_relax();
+ cid = __mm_get_cid(mm, num_possible_cpus());
+ }
+ return cid;
+}
+
+static inline unsigned int mm_cid_converge(struct mm_struct *mm, unsigned int orig_cid,
+ unsigned int max_cids)
+{
+ unsigned int new_cid, cid = cpu_cid_to_cid(orig_cid);
+
+ /* Is it in the optimal CID space? */
+ if (likely(cid < max_cids))
+ return orig_cid;
+
+ /* Try to find one in the optimal space. Otherwise keep the provided. */
+ new_cid = __mm_get_cid(mm, max_cids);
+ if (new_cid != MM_CID_UNSET) {
+ mm_drop_cid(mm, cid);
+ /* Preserve the ONCPU mode of the original CID */
+ return new_cid | (orig_cid & MM_CID_ONCPU);
+ }
+ return orig_cid;
+}
+
+static __always_inline void mm_cid_update_task_cid(struct task_struct *t, unsigned int cid)
+{
+ if (t->mm_cid.cid != cid) {
+ t->mm_cid.cid = cid;
+ rseq_sched_set_ids_changed(t);
+ }
+}
+
+static __always_inline void mm_cid_update_pcpu_cid(struct mm_struct *mm, unsigned int cid)
+{
+ __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
+}
+
+static __always_inline void mm_cid_from_cpu(struct task_struct *t, unsigned int cpu_cid)
+{
+ unsigned int max_cids, tcid = t->mm_cid.cid;
+ struct mm_struct *mm = t->mm;
+
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
+ /* Optimize for the common case where both have the ONCPU bit set */
+ if (likely(cid_on_cpu(cpu_cid & tcid))) {
+ if (likely(cpu_cid_to_cid(cpu_cid) < max_cids)) {
+ mm_cid_update_task_cid(t, cpu_cid);
+ return;
+ }
+ /* Try to converge into the optimal CID space */
+ cpu_cid = mm_cid_converge(mm, cpu_cid, max_cids);
+ } else {
+ /* Hand over or drop the task owned CID */
+ if (cid_on_task(tcid)) {
+ if (cid_on_cpu(cpu_cid))
+ mm_unset_cid_on_task(t);
+ else
+ cpu_cid = cid_to_cpu_cid(tcid);
+ }
+ /* Still nothing, allocate a new one */
+ if (!cid_on_cpu(cpu_cid))
+ cpu_cid = cid_to_cpu_cid(mm_get_cid(mm));
+ }
+ mm_cid_update_pcpu_cid(mm, cpu_cid);
+ mm_cid_update_task_cid(t, cpu_cid);
+}
+
+static __always_inline void mm_cid_from_task(struct task_struct *t, unsigned int cpu_cid)
+{
+ unsigned int max_cids, tcid = t->mm_cid.cid;
+ struct mm_struct *mm = t->mm;
+
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
+ /* Optimize for the common case, where both have the ONCPU bit clear */
+ if (likely(cid_on_task(tcid | cpu_cid))) {
+ if (likely(tcid < max_cids)) {
+ mm_cid_update_pcpu_cid(mm, tcid);
+ return;
+ }
+ /* Try to converge into the optimal CID space */
+ tcid = mm_cid_converge(mm, tcid, max_cids);
+ } else {
+ /* Hand over or drop the CPU owned CID */
+ if (cid_on_cpu(cpu_cid)) {
+ if (cid_on_task(tcid))
+ mm_drop_cid_on_cpu(mm, this_cpu_ptr(mm->mm_cid.pcpu));
+ else
+ tcid = cpu_cid_to_cid(cpu_cid);
+ }
+ /* Still nothing, allocate a new one */
+ if (!cid_on_task(tcid))
+ tcid = mm_get_cid(mm);
+ /* Set the transition mode flag if required */
+ tcid |= READ_ONCE(mm->mm_cid.transit);
+ }
+ mm_cid_update_pcpu_cid(mm, tcid);
+ mm_cid_update_task_cid(t, tcid);
+}
+
+static __always_inline void mm_cid_schedin(struct task_struct *next)
+{
+ struct mm_struct *mm = next->mm;
+ unsigned int cpu_cid;
+
+ if (!next->mm_cid.active)
+ return;
+
+ cpu_cid = __this_cpu_read(mm->mm_cid.pcpu->cid);
+ if (likely(!READ_ONCE(mm->mm_cid.percpu)))
+ mm_cid_from_task(next, cpu_cid);
+ else
+ mm_cid_from_cpu(next, cpu_cid);
+}
+
+static __always_inline void mm_cid_schedout(struct task_struct *prev)
+{
+ /* During mode transitions CIDs are temporary and need to be dropped */
+ if (likely(!cid_in_transit(prev->mm_cid.cid)))
+ return;
+
+ mm_drop_cid(prev->mm, cid_from_transit_cid(prev->mm_cid.cid));
+ prev->mm_cid.cid = MM_CID_UNSET;
+}
+
+static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next)
+{
+ mm_cid_schedout(prev);
+ mm_cid_schedin(next);
+}
+
/* Active implementation */
static inline void init_sched_mm_cid(struct task_struct *t)
{
@@ -3675,6 +3822,7 @@ static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *n
#else /* !CONFIG_SCHED_MM_CID: */
static inline void mm_cid_select(struct task_struct *t) { }
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
+static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
extern u64 avg_vruntime(struct cfs_rq *cfs_rq);The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 6fa1e9565d81095ac1c085601a4af8b18a04db1d
Gitweb: https://git.kernel.org/tip/6fa1e9565d81095ac1c085601a4af8b18a04db1d
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:14 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:57 +01:00
sched/mmcid: Provide new scheduler CID mechanism
The MM CID management has two fundamental requirements:
1) It has to guarantee that at no given point in time the same CID is
used by concurrent tasks in userspace.
2) The CID space must not exceed the number of possible CPUs in a
system. While most allocators (glibc, tcmalloc, jemalloc) do not
care about that, there seems to be at least some LTTng library
depending on it.
The CID space compaction itself is not a functional correctness
requirement, it is only a useful optimization mechanism to reduce the
memory foot print in unused user space pools.
The optimal CID space is:
min(nr_tasks, nr_cpus_allowed);
Where @nr_tasks is the number of actual user space threads associated to
the mm and @nr_cpus_allowed is the superset of all task affinities. It is
growth only as it would be insane to take a racy snapshot of all task
affinities when the affinity of one task changes just do redo it 2
milliseconds later when the next task changes it's affinity.
That means that as long as the number of tasks is lower or equal than the
number of CPUs allowed, each task owns a CID. If the number of tasks
exceeds the number of CPUs allowed it switches to per CPU mode, where the
CPUs own the CIDs and the tasks borrow them as long as they are scheduled
in.
For transition periods CIDs can go beyond the optimal space as long as they
don't go beyond the number of possible CPUs.
The current upstream implementation adds overhead into task migration to
keep the CID with the task. It also has to do the CID space consolidation
work from a task work in the exit to user space path. As that work is
assigned to a random task related to a MM this can inflict unwanted exit
latencies.
Implement the context switch parts of a strict ownership mechanism to
address this.
This removes most of the work from the task which schedules out. Only
during transitioning from per CPU to per task ownership it is required to
drop the CID when leaving the CPU to prevent CID space exhaustion. Other
than that scheduling out is just a single check and branch.
The task which schedules in has to check whether:
1) The ownership mode changed
2) The CID is within the optimal CID space
In stable situations this results in zero work. The only short disruption
is when ownership mode changes or when the associated CID is not in the
optimal CID space. The latter only happens when tasks exit and therefore
the optimal CID space shrinks.
That mechanism is strictly optimized for the common case where no change
happens. The only case where it actually causes a temporary one time spike
is on mode changes when and only when a lot of tasks related to a MM
schedule exactly at the same time and have eventually to compete on
allocating a CID from the bitmap.
In the sysbench test case which triggered the spinlock contention in the
initial CID code, __schedule() drops significantly in perf top on a 128
Core (256 threads) machine when running sysbench with 255 threads, which
fits into the task mode limit of 256 together with the parent thread:
Upstream rseq/perf branch +CID rework
0.42% 0.37% 0.32% [k] __schedule
Increasing the number of threads to 256, which puts the test process into
per CPU mode looks about the same.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.023984859@linutronix.de
---
include/linux/rseq.h | 8 +-
include/linux/rseq_types.h | 18 ++--
kernel/sched/core.c | 2 +-
kernel/sched/sched.h | 150 +++++++++++++++++++++++++++++++++++-
4 files changed, 168 insertions(+), 10 deletions(-)
diff --git a/include/linux/rseq.h b/include/linux/rseq.h
index bf8a6bf..4c0e8bd 100644
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -73,13 +73,13 @@ static __always_inline void rseq_sched_switch_event(struct task_struct *t)
}
/*
- * Invoked from __set_task_cpu() when a task migrates to enforce an IDs
- * update.
+ * Invoked from __set_task_cpu() when a task migrates or from
+ * mm_cid_schedin() when the CID changes to enforce an IDs update.
*
* This does not raise TIF_NOTIFY_RESUME as that happens in
* rseq_sched_switch_event().
*/
-static __always_inline void rseq_sched_set_task_cpu(struct task_struct *t, unsigned int cpu)
+static __always_inline void rseq_sched_set_ids_changed(struct task_struct *t)
{
t->rseq.event.ids_changed = true;
}
@@ -168,7 +168,7 @@ static inline void rseq_fork(struct task_struct *t, u64 clone_flags)
static inline void rseq_handle_slowpath(struct pt_regs *regs) { }
static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }
static inline void rseq_sched_switch_event(struct task_struct *t) { }
-static inline void rseq_sched_set_task_cpu(struct task_struct *t, unsigned int cpu) { }
+static inline void rseq_sched_set_ids_changed(struct task_struct *t) { }
static inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid) { }
static inline void rseq_force_update(void) { }
static inline void rseq_virt_userspace_exit(void) { }
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 87854ef..66b1482 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -119,23 +119,31 @@ struct mm_cid_pcpu {
/**
* struct mm_mm_cid - Storage for per MM CID data
* @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @percpu: Set, when CIDs are in per CPU mode
+ * @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
+ * @lock: Spinlock to protect all fields except @pcpu. It also protects
+ * the MM cid cpumask and the MM cidmask bitmap.
+ * @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
* @users: The number of tasks sharing this MM. Separate from mm::mm_users
* as that is modified by mmget()/mm_put() by other entities which
* do not actually share the MM.
- * @lock: Spinlock to protect all fields except @pcpu. It also protects
- * the MM cid cpumask and the MM cidmask bitmap.
- * @mutex: Mutex to serialize forks and exits related to this mm
*/
struct mm_mm_cid {
+ /* Hotpath read mostly members */
struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int percpu;
+ unsigned int transit;
unsigned int max_cids;
- unsigned int nr_cpus_allowed;
- unsigned int users;
+
raw_spinlock_t lock;
struct mutex mutex;
+
+ /* Low frequency modified */
+ unsigned int nr_cpus_allowed;
+ unsigned int users;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 55bb9c9..659ae56 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10495,6 +10495,8 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
mm->mm_cid.max_cids = 0;
+ mm->mm_cid.percpu = 0;
+ mm->mm_cid.transit = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 4b49284..82c7978 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2209,7 +2209,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
smp_wmb();
WRITE_ONCE(task_thread_info(p)->cpu, cpu);
p->wake_cpu = cpu;
- rseq_sched_set_task_cpu(p, cpu);
+ rseq_sched_set_ids_changed(p);
#endif /* CONFIG_SMP */
}
@@ -3598,6 +3598,153 @@ static __always_inline void mm_drop_cid_on_cpu(struct mm_struct *mm, struct mm_c
mm_drop_cid(mm, pcp->cid);
}
+static inline unsigned int __mm_get_cid(struct mm_struct *mm, unsigned int max_cids)
+{
+ unsigned int cid = find_first_zero_bit(mm_cidmask(mm), max_cids);
+
+ if (cid >= max_cids)
+ return MM_CID_UNSET;
+ if (test_and_set_bit(cid, mm_cidmask(mm)))
+ return MM_CID_UNSET;
+ return cid;
+}
+
+static inline unsigned int mm_get_cid(struct mm_struct *mm)
+{
+ unsigned int cid = __mm_get_cid(mm, READ_ONCE(mm->mm_cid.max_cids));
+
+ while (cid == MM_CID_UNSET) {
+ cpu_relax();
+ cid = __mm_get_cid(mm, num_possible_cpus());
+ }
+ return cid;
+}
+
+static inline unsigned int mm_cid_converge(struct mm_struct *mm, unsigned int orig_cid,
+ unsigned int max_cids)
+{
+ unsigned int new_cid, cid = cpu_cid_to_cid(orig_cid);
+
+ /* Is it in the optimal CID space? */
+ if (likely(cid < max_cids))
+ return orig_cid;
+
+ /* Try to find one in the optimal space. Otherwise keep the provided. */
+ new_cid = __mm_get_cid(mm, max_cids);
+ if (new_cid != MM_CID_UNSET) {
+ mm_drop_cid(mm, cid);
+ /* Preserve the ONCPU mode of the original CID */
+ return new_cid | (orig_cid & MM_CID_ONCPU);
+ }
+ return orig_cid;
+}
+
+static __always_inline void mm_cid_update_task_cid(struct task_struct *t, unsigned int cid)
+{
+ if (t->mm_cid.cid != cid) {
+ t->mm_cid.cid = cid;
+ rseq_sched_set_ids_changed(t);
+ }
+}
+
+static __always_inline void mm_cid_update_pcpu_cid(struct mm_struct *mm, unsigned int cid)
+{
+ __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
+}
+
+static __always_inline void mm_cid_from_cpu(struct task_struct *t, unsigned int cpu_cid)
+{
+ unsigned int max_cids, tcid = t->mm_cid.cid;
+ struct mm_struct *mm = t->mm;
+
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
+ /* Optimize for the common case where both have the ONCPU bit set */
+ if (likely(cid_on_cpu(cpu_cid & tcid))) {
+ if (likely(cpu_cid_to_cid(cpu_cid) < max_cids)) {
+ mm_cid_update_task_cid(t, cpu_cid);
+ return;
+ }
+ /* Try to converge into the optimal CID space */
+ cpu_cid = mm_cid_converge(mm, cpu_cid, max_cids);
+ } else {
+ /* Hand over or drop the task owned CID */
+ if (cid_on_task(tcid)) {
+ if (cid_on_cpu(cpu_cid))
+ mm_unset_cid_on_task(t);
+ else
+ cpu_cid = cid_to_cpu_cid(tcid);
+ }
+ /* Still nothing, allocate a new one */
+ if (!cid_on_cpu(cpu_cid))
+ cpu_cid = cid_to_cpu_cid(mm_get_cid(mm));
+ }
+ mm_cid_update_pcpu_cid(mm, cpu_cid);
+ mm_cid_update_task_cid(t, cpu_cid);
+}
+
+static __always_inline void mm_cid_from_task(struct task_struct *t, unsigned int cpu_cid)
+{
+ unsigned int max_cids, tcid = t->mm_cid.cid;
+ struct mm_struct *mm = t->mm;
+
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
+ /* Optimize for the common case, where both have the ONCPU bit clear */
+ if (likely(cid_on_task(tcid | cpu_cid))) {
+ if (likely(tcid < max_cids)) {
+ mm_cid_update_pcpu_cid(mm, tcid);
+ return;
+ }
+ /* Try to converge into the optimal CID space */
+ tcid = mm_cid_converge(mm, tcid, max_cids);
+ } else {
+ /* Hand over or drop the CPU owned CID */
+ if (cid_on_cpu(cpu_cid)) {
+ if (cid_on_task(tcid))
+ mm_drop_cid_on_cpu(mm, this_cpu_ptr(mm->mm_cid.pcpu));
+ else
+ tcid = cpu_cid_to_cid(cpu_cid);
+ }
+ /* Still nothing, allocate a new one */
+ if (!cid_on_task(tcid))
+ tcid = mm_get_cid(mm);
+ /* Set the transition mode flag if required */
+ tcid |= READ_ONCE(mm->mm_cid.transit);
+ }
+ mm_cid_update_pcpu_cid(mm, tcid);
+ mm_cid_update_task_cid(t, tcid);
+}
+
+static __always_inline void mm_cid_schedin(struct task_struct *next)
+{
+ struct mm_struct *mm = next->mm;
+ unsigned int cpu_cid;
+
+ if (!next->mm_cid.active)
+ return;
+
+ cpu_cid = __this_cpu_read(mm->mm_cid.pcpu->cid);
+ if (likely(!READ_ONCE(mm->mm_cid.percpu)))
+ mm_cid_from_task(next, cpu_cid);
+ else
+ mm_cid_from_cpu(next, cpu_cid);
+}
+
+static __always_inline void mm_cid_schedout(struct task_struct *prev)
+{
+ /* During mode transitions CIDs are temporary and need to be dropped */
+ if (likely(!cid_in_transit(prev->mm_cid.cid)))
+ return;
+
+ mm_drop_cid(prev->mm, cid_from_transit_cid(prev->mm_cid.cid));
+ prev->mm_cid.cid = MM_CID_UNSET;
+}
+
+static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next)
+{
+ mm_cid_schedout(prev);
+ mm_cid_schedin(next);
+}
+
/* Active implementation */
static inline void init_sched_mm_cid(struct task_struct *t)
{
@@ -3675,6 +3822,7 @@ static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *n
#else /* !CONFIG_SCHED_MM_CID: */
static inline void mm_cid_select(struct task_struct *t) { }
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
+static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
extern u64 avg_vruntime(struct cfs_rq *cfs_rq);CIDs are either owned by tasks or by CPUs. The ownership mode depends on
the number of tasks related to a MM and the number of CPUs on which these
tasks are theoretically allowed to run on. Theoretically because that
number is the superset of CPU affinities of all tasks which only grows and
never shrinks.
Switching to per CPU mode happens when the user count becomes greater than
the maximum number of CIDs, which is calculated by:
opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users);
max_cids = min(1.25 * opt_cids, nr_cpu_ids);
The +25% allowance is useful for tight CPU masks in scenarios where only a
few threads are created and destroyed to avoid frequent mode
switches. Though this allowance shrinks, the closer opt_cids becomes to
nr_cpu_ids, which is the (unfortunate) hard ABI limit.
At the point of switching to per CPU mode the new user is not yet visible
in the system, so the task which initiated the fork() runs the fixup
function: mm_cid_fixup_tasks_to_cpu() walks the thread list and either
transfers each tasks owned CID to the CPU the task runs on or drops it into
the CID pool if a task is not on a CPU at that point in time. Tasks which
schedule in before the task walk reaches them do the handover in
mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus() completes it's
guaranteed that no task related to that MM owns a CID anymore.
Switching back to task mode happens when the user count goes below the
threshold which was recorded on the per CPU mode switch:
pcpu_thrs = min(opt_cids - (opt_cids / 4), nr_cpu_ids / 2);
This threshold is updated when a affinity change increases the number of
allowed CPUs for the MM, which might cause a switch back to per task mode.
If the switch back was initiated by a exiting task, then that task runs the
fixup function. If it was initiated by a affinity change, then it's run
either in the deferred update function in context of a workqueue or by a
task which forks a new one or by a task which exits. Whatever happens
first. mm_cid_fixup_cpus_to_task() walks through the possible CPUs and
either transfers the CPU owned CIDs to a related task which runs on the CPU
or drops it into the pool. Tasks which schedule in on a CPU which the walk
did not cover yet do the handover themselves.
This transition from CPU to per task ownership happens in two phases:
1) mm:mm_cid.transit contains MM_CID_TRANSIT. This is OR'ed on the task
CID and denotes that the CID is only temporarily owned by the
task. When it schedules out the task drops the CID back into the
pool if this bit is set.
2) The initiating context walks the per CPU space and after completion
clears mm:mm_cid.transit. After that point the CIDs are strictly
task owned again.
This two phase transition is required to prevent CID space exhaustion
during the transition as a direct transfer of ownership would fail if
two tasks are scheduled in on the same CPU before the fixup freed per
CPU CIDs.
When mm_cid_fixup_cpus_to_tasks() completes it's guaranteed that no CID
related to that MM is owned by a CPU anymore.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
V2: Fix CPU to task owner mode
---
include/linux/rseq_types.h | 7 -
kernel/sched/core.c | 277 +++++++++++++++++++++++++++++++++++++++++----
2 files changed, 258 insertions(+), 26 deletions(-)
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -122,14 +122,15 @@ struct mm_cid_pcpu {
* @percpu: Set, when CIDs are in per CPU mode
* @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
- * @lock: Spinlock to protect all fields except @pcpu. It also protects
- * the MM cid cpumask and the MM cidmask bitmap.
+ * @lock: Spinlock to protect against affinity setting which can't take @mutex
* @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
* @users: The number of tasks sharing this MM. Separate from mm::mm_users
* as that is modified by mmget()/mm_put() by other entities which
* do not actually share the MM.
+ * @pcpu_thrs: Threshold for switching back from per CPU mode
+ * @update_deferred: A deferred switch back to per task mode is pending.
*/
struct mm_mm_cid {
/* Hotpath read mostly members */
@@ -144,6 +145,8 @@ struct mm_mm_cid {
/* Low frequency modified */
unsigned int nr_cpus_allowed;
unsigned int users;
+ unsigned int pcpu_thrs;
+ unsigned int update_deferred;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10396,43 +10396,269 @@ void call_trace_sched_update_nr_running(
* task needs to drop the CID into the pool when scheduling out. Both bits
* (ONCPU and TRANSIT) are filtered out by task_cid() when the CID is
* actually handed over to user space in the RSEQ memory.
+ *
+ * Mode switching:
+ *
+ * Switching to per CPU mode happens when the user count becomes greater
+ * than the maximum number of CIDs, which is calculated by:
+ *
+ * opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users);
+ * max_cids = min(1.25 * opt_cids, num_possible_cpus());
+ *
+ * The +25% allowance is useful for tight CPU masks in scenarios where only
+ * a few threads are created and destroyed to avoid frequent mode
+ * switches. Though this allowance shrinks, the closer opt_cids becomes to
+ * num_possible_cpus(), which is the (unfortunate) hard ABI limit.
+ *
+ * At the point of switching to per CPU mode the new user is not yet
+ * visible in the system, so the task which initiated the fork() runs the
+ * fixup function: mm_cid_fixup_tasks_to_cpu() walks the thread list and
+ * either transfers each tasks owned CID to the CPU the task runs on or
+ * drops it into the CID pool if a task is not on a CPU at that point in
+ * time. Tasks which schedule in before the task walk reaches them do the
+ * handover in mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus() completes
+ * it's guaranteed that no task related to that MM owns a CID anymore.
+ *
+ * Switching back to task mode happens when the user count goes below the
+ * threshold which was recorded on the per CPU mode switch:
+ *
+ * pcpu_thrs = min(opt_cids - (opt_cids / 4), num_possible_cpus() / 2);
+ *
+ * This threshold is updated when a affinity change increases the number of
+ * allowed CPUs for the MM, which might cause a switch back to per task
+ * mode.
+ *
+ * If the switch back was initiated by a exiting task, then that task runs
+ * the fixup function. If it was initiated by a affinity change, then it's
+ * run either in the deferred update function in context of a workqueue or
+ * by a task which forks a new one or by a task which exits. Whatever
+ * happens first. mm_cid_fixup_cpus_to_task() walks through the possible
+ * CPUs and either transfers the CPU owned CIDs to a related task which
+ * runs on the CPU or drops it into the pool. Tasks which schedule in on a
+ * CPU which the walk did not cover yet do the handover themself.
+ *
+ * This transition from CPU to per task ownership happens in two phases:
+ *
+ * 1) mm:mm_cid.transit contains MM_CID_TRANSIT This is OR'ed on the task
+ * CID and denotes that the CID is only temporarily owned by the
+ * task. When it schedules out the task drops the CID back into the
+ * pool if this bit is set.
+ *
+ * 2) The initiating context walks the per CPU space and after completion
+ * clears mm:mm_cid.transit. So after that point the CIDs are strictly
+ * task owned again.
+ *
+ * This two phase transition is required to prevent CID space exhaustion
+ * during the transition as a direct transfer of ownership would fail if
+ * two tasks are scheduled in on the same CPU before the fixup freed per
+ * CPU CIDs.
+ *
+ * When mm_cid_fixup_cpus_to_tasks() completes it's guaranteed that no CID
+ * related to that MM is owned by a CPU anymore.
*/
/*
* Update the CID range properties when the constraints change. Invoked via
* fork(), exit() and affinity changes
*/
-static void mm_update_max_cids(struct mm_struct *mm)
+static void __mm_update_max_cids(struct mm_mm_cid *mc)
+{
+ unsigned int opt_cids, max_cids;
+
+ /* Calculate the new optimal constraint */
+ opt_cids = min(mc->nr_cpus_allowed, mc->users);
+
+ /* Adjust the maximum CIDs to +25% limited by the number of possible CPUs */
+ max_cids = min(opt_cids + (opt_cids / 4), num_possible_cpus());
+ WRITE_ONCE(mc->max_cids, max_cids);
+}
+
+static inline unsigned int mm_cid_calc_pcpu_thrs(struct mm_mm_cid *mc)
+{
+ unsigned int opt_cids;
+
+ opt_cids = min(mc->nr_cpus_allowed, mc->users);
+ return min(opt_cids - opt_cids / 4, num_possible_cpus() / 2);
+}
+
+static bool mm_update_max_cids(struct mm_struct *mm)
{
struct mm_mm_cid *mc = &mm->mm_cid;
- unsigned int max_cids;
lockdep_assert_held(&mm->mm_cid.lock);
- /* Calculate the new maximum constraint */
- max_cids = min(mc->nr_cpus_allowed, mc->users);
- WRITE_ONCE(mc->max_cids, max_cids);
+ /* Clear deferred mode switch flag. A change is handled by the caller */
+ mc->update_deferred = false;
+ __mm_update_max_cids(mc);
+
+ /* Check whether owner mode must be changed */
+ if (!mc->percpu) {
+ /* Enable per CPU mode when the number of users is above max_cids */
+ if (mc->users > mc->max_cids)
+ mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+ } else {
+ /* Switch back to per task if user count under threshold */
+ if (mc->users < mc->pcpu_thrs)
+ mc->pcpu_thrs = 0;
+ }
+
+ /* Mode change required? */
+ if (!!mc->percpu == !!mc->pcpu_thrs)
+ return false;
+ /* When switching back to per TASK mode, set the transition flag */
+ if (!mc->pcpu_thrs)
+ WRITE_ONCE(mc->transit, MM_CID_TRANSIT);
+ WRITE_ONCE(mc->percpu, !!mc->pcpu_thrs);
+ return true;
}
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
+ struct mm_mm_cid *mc;
unsigned int weight;
if (!mm || !READ_ONCE(mm->mm_cid.users))
return;
-
/*
* mm::mm_cid::mm_cpus_allowed is the superset of each threads
* allowed CPUs mask which means it can only grow.
*/
- guard(raw_spinlock)(&mm->mm_cid.lock);
+ mc = &mm->mm_cid;
+ guard(raw_spinlock)(&mc->lock);
mm_allowed = mm_cpus_allowed(mm);
weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
- if (weight == mm->mm_cid.nr_cpus_allowed)
+ if (weight == mc->nr_cpus_allowed)
+ return;
+
+ WRITE_ONCE(mc->nr_cpus_allowed, weight);
+ __mm_update_max_cids(mc);
+ if (!mc->percpu)
return;
- WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
- mm_update_max_cids(mm);
+
+ /* Adjust the threshold to the wider set */
+ mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+
+ /* Scheduling of deferred mode switch goes here */
+}
+
+static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
+{
+ if (cid_on_cpu(t->mm_cid.cid)) {
+ unsigned int cid = cpu_cid_to_cid(t->mm_cid.cid);
+
+ t->mm_cid.cid = cid_to_transit_cid(cid);
+ pcp->cid = t->mm_cid.cid;
+ }
+}
+
+static void __maybe_unused mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
+{
+ unsigned int cpu;
+
+ /* Walk the CPUs and fixup all stale CIDs */
+ for_each_possible_cpu(cpu) {
+ struct mm_cid_pcpu *pcp = per_cpu_ptr(mm->mm_cid.pcpu, cpu);
+ struct rq *rq = cpu_rq(cpu);
+
+ /* Remote access to mm::mm_cid::pcpu requires rq_lock */
+ guard(rq_lock_irq)(rq);
+ /* Is the CID still owned by the CPU? */
+ if (cid_on_cpu(pcp->cid)) {
+ /*
+ * If rq->curr has @mm, transfer it with the
+ * transition bit set. Otherwise drop it.
+ */
+ if (rq->curr->mm == mm && rq->curr->mm_cid.active)
+ mm_cid_transit_to_task(rq->curr, pcp);
+ else
+ mm_drop_cid_on_cpu(mm, pcp);
+
+ } else if (rq->curr->mm == mm && rq->curr->mm_cid.active) {
+ unsigned int cid = rq->curr->mm_cid.cid;
+
+ /* Ensure it has the transition bit set */
+ if (!cid_in_transit(cid)) {
+ cid = cid_to_transit_cid(cid);
+ rq->curr->mm_cid.cid = cid;
+ pcp->cid = cid;
+ }
+ }
+ }
+ /* Clear the transition bit */
+ WRITE_ONCE(mm->mm_cid.transit, 0);
+}
+
+static inline void mm_cid_transfer_to_cpu(struct task_struct *t, struct mm_cid_pcpu *pcp)
+{
+ if (cid_on_task(t->mm_cid.cid)) {
+ t->mm_cid.cid = cid_to_cpu_cid(t->mm_cid.cid);
+ pcp->cid = t->mm_cid.cid;
+ }
+}
+
+static bool mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm)
+{
+ /* Remote access to mm::mm_cid::pcpu requires rq_lock */
+ guard(task_rq_lock)(t);
+ /* If the task is not active it is not in the users count */
+ if (!t->mm_cid.active)
+ return false;
+ if (cid_on_task(t->mm_cid.cid)) {
+ /* If running on the CPU, transfer the CID, otherwise drop it */
+ if (task_rq(t)->curr == t)
+ mm_cid_transfer_to_cpu(t, per_cpu_ptr(mm->mm_cid.pcpu, task_cpu(t)));
+ else
+ mm_unset_cid_on_task(t);
+ }
+ return true;
+}
+
+static void __maybe_unused mm_cid_fixup_tasks_to_cpus(void)
+{
+ struct mm_struct *mm = current->mm;
+ struct task_struct *p, *t;
+ unsigned int users;
+
+ /*
+ * This can obviously race with a concurrent affinity change, which
+ * increases the number of allowed CPUs for this mm, but that does
+ * not affect the mode and only changes the CID constraints. A
+ * possible switch back to per task mode happens either in the
+ * deferred handler function or in the next fork()/exit().
+ *
+ * The caller has already transferred. The newly incoming task is
+ * already accounted for, but not yet visible.
+ */
+ users = mm->mm_cid.users - 2;
+ if (!users)
+ return;
+
+ guard(rcu)();
+ for_other_threads(current, t) {
+ if (mm_cid_fixup_task_to_cpu(t, mm))
+ users--;
+ }
+
+ if (!users)
+ return;
+
+ /* Happens only for VM_CLONE processes. */
+ for_each_process_thread(p, t) {
+ if (t == current || t->mm != mm)
+ continue;
+ if (mm_cid_fixup_task_to_cpu(t, mm)) {
+ if (--users == 0)
+ return;
+ }
+ }
+}
+
+static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm)
+{
+ t->mm_cid.active = 1;
+ mm->mm_cid.users++;
+ return mm_update_max_cids(mm);
}
void sched_mm_cid_fork(struct task_struct *t)
@@ -10442,12 +10668,19 @@ void sched_mm_cid_fork(struct task_struc
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
guard(mutex)(&mm->mm_cid.mutex);
- guard(raw_spinlock)(&mm->mm_cid.lock);
- t->mm_cid.active = 1;
- mm->mm_cid.users++;
- /* Preset last_cid for mm_cid_select() */
- t->mm_cid.last_cid = READ_ONCE(mm->mm_cid.max_cids) - 1;
- mm_update_max_cids(mm);
+ scoped_guard(raw_spinlock, &mm->mm_cid.lock) {
+ sched_mm_cid_add_user(t, mm);
+ /* Preset last_cid for mm_cid_select() */
+ t->mm_cid.last_cid = mm->mm_cid.max_cids - 1;
+ }
+}
+
+static bool sched_mm_cid_remove_user(struct task_struct *t)
+{
+ t->mm_cid.active = 0;
+ mm_unset_cid_on_task(t);
+ t->mm->mm_cid.users--;
+ return mm_update_max_cids(t->mm);
}
/*
@@ -10462,14 +10695,8 @@ void sched_mm_cid_exit(struct task_struc
return;
guard(mutex)(&mm->mm_cid.mutex);
- guard(raw_spinlock)(&mm->mm_cid.lock);
- t->mm_cid.active = 0;
- mm->mm_cid.users--;
- if (t->mm_cid.cid != MM_CID_UNSET) {
- clear_bit(t->mm_cid.cid, mm_cidmask(mm));
- t->mm_cid.cid = MM_CID_UNSET;
- }
- mm_update_max_cids(mm);
+ scoped_guard(raw_spinlock, &mm->mm_cid.lock)
+ sched_mm_cid_remove_user(t);
}
/* Deactivate MM CID allocation across execve() */
@@ -10499,6 +10726,8 @@ void mm_init_cid(struct mm_struct *mm, s
mm->mm_cid.transit = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
+ mm->mm_cid.pcpu_thrs = 0;
+ mm->mm_cid.update_deferred = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
mutex_init(&mm->mm_cid.mutex);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);The following commit has been merged into the core/rseq branch of tip:
Commit-ID: fbd0e71dc370af73f6b316e4de9eed273dd90340
Gitweb: https://git.kernel.org/tip/fbd0e71dc370af73f6b316e4de9eed273dd90340
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:16 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:41 +01:00
sched/mmcid: Provide CID ownership mode fixup functions
CIDs are either owned by tasks or by CPUs. The ownership mode depends on
the number of tasks related to a MM and the number of CPUs on which these
tasks are theoretically allowed to run on. Theoretically because that
number is the superset of CPU affinities of all tasks which only grows and
never shrinks.
Switching to per CPU mode happens when the user count becomes greater than
the maximum number of CIDs, which is calculated by:
opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users);
max_cids = min(1.25 * opt_cids, nr_cpu_ids);
The +25% allowance is useful for tight CPU masks in scenarios where only a
few threads are created and destroyed to avoid frequent mode
switches. Though this allowance shrinks, the closer opt_cids becomes to
nr_cpu_ids, which is the (unfortunate) hard ABI limit.
At the point of switching to per CPU mode the new user is not yet visible
in the system, so the task which initiated the fork() runs the fixup
function: mm_cid_fixup_tasks_to_cpu() walks the thread list and either
transfers each tasks owned CID to the CPU the task runs on or drops it into
the CID pool if a task is not on a CPU at that point in time. Tasks which
schedule in before the task walk reaches them do the handover in
mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus() completes it's
guaranteed that no task related to that MM owns a CID anymore.
Switching back to task mode happens when the user count goes below the
threshold which was recorded on the per CPU mode switch:
pcpu_thrs = min(opt_cids - (opt_cids / 4), nr_cpu_ids / 2);
This threshold is updated when a affinity change increases the number of
allowed CPUs for the MM, which might cause a switch back to per task mode.
If the switch back was initiated by a exiting task, then that task runs the
fixup function. If it was initiated by a affinity change, then it's run
either in the deferred update function in context of a workqueue or by a
task which forks a new one or by a task which exits. Whatever happens
first. mm_cid_fixup_cpus_to_task() walks through the possible CPUs and
either transfers the CPU owned CIDs to a related task which runs on the CPU
or drops it into the pool. Tasks which schedule in on a CPU which the walk
did not cover yet do the handover themselves.
This transition from CPU to per task ownership happens in two phases:
1) mm:mm_cid.transit contains MM_CID_TRANSIT. This is OR'ed on the task
CID and denotes that the CID is only temporarily owned by the
task. When it schedules out the task drops the CID back into the
pool if this bit is set.
2) The initiating context walks the per CPU space and after completion
clears mm:mm_cid.transit. After that point the CIDs are strictly
task owned again.
This two phase transition is required to prevent CID space exhaustion
during the transition as a direct transfer of ownership would fail if
two tasks are scheduled in on the same CPU before the fixup freed per
CPU CIDs.
When mm_cid_fixup_cpus_to_tasks() completes it's guaranteed that no CID
related to that MM is owned by a CPU anymore.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.088189028@linutronix.de
---
include/linux/rseq_types.h | 7 +-
kernel/sched/core.c | 278 ++++++++++++++++++++++++++++++++----
2 files changed, 259 insertions(+), 26 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 66b1482..a3a4f3f 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -122,14 +122,15 @@ struct mm_cid_pcpu {
* @percpu: Set, when CIDs are in per CPU mode
* @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
- * @lock: Spinlock to protect all fields except @pcpu. It also protects
- * the MM cid cpumask and the MM cidmask bitmap.
+ * @lock: Spinlock to protect against affinity setting which can't take @mutex
* @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
* @users: The number of tasks sharing this MM. Separate from mm::mm_users
* as that is modified by mmget()/mm_put() by other entities which
* do not actually share the MM.
+ * @pcpu_thrs: Threshold for switching back from per CPU mode
+ * @update_deferred: A deferred switch back to per task mode is pending.
*/
struct mm_mm_cid {
/* Hotpath read mostly members */
@@ -144,6 +145,8 @@ struct mm_mm_cid {
/* Low frequency modified */
unsigned int nr_cpus_allowed;
unsigned int users;
+ unsigned int pcpu_thrs;
+ unsigned int update_deferred;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 659ae56..eb0d59d 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10396,43 +10396,270 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
* task needs to drop the CID into the pool when scheduling out. Both bits
* (ONCPU and TRANSIT) are filtered out by task_cid() when the CID is
* actually handed over to user space in the RSEQ memory.
+ *
+ * Mode switching:
+ *
+ * Switching to per CPU mode happens when the user count becomes greater
+ * than the maximum number of CIDs, which is calculated by:
+ *
+ * opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users);
+ * max_cids = min(1.25 * opt_cids, num_possible_cpus());
+ *
+ * The +25% allowance is useful for tight CPU masks in scenarios where only
+ * a few threads are created and destroyed to avoid frequent mode
+ * switches. Though this allowance shrinks, the closer opt_cids becomes to
+ * num_possible_cpus(), which is the (unfortunate) hard ABI limit.
+ *
+ * At the point of switching to per CPU mode the new user is not yet
+ * visible in the system, so the task which initiated the fork() runs the
+ * fixup function: mm_cid_fixup_tasks_to_cpu() walks the thread list and
+ * either transfers each tasks owned CID to the CPU the task runs on or
+ * drops it into the CID pool if a task is not on a CPU at that point in
+ * time. Tasks which schedule in before the task walk reaches them do the
+ * handover in mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus() completes
+ * it's guaranteed that no task related to that MM owns a CID anymore.
+ *
+ * Switching back to task mode happens when the user count goes below the
+ * threshold which was recorded on the per CPU mode switch:
+ *
+ * pcpu_thrs = min(opt_cids - (opt_cids / 4), num_possible_cpus() / 2);
+ *
+ * This threshold is updated when a affinity change increases the number of
+ * allowed CPUs for the MM, which might cause a switch back to per task
+ * mode.
+ *
+ * If the switch back was initiated by a exiting task, then that task runs
+ * the fixup function. If it was initiated by a affinity change, then it's
+ * run either in the deferred update function in context of a workqueue or
+ * by a task which forks a new one or by a task which exits. Whatever
+ * happens first. mm_cid_fixup_cpus_to_task() walks through the possible
+ * CPUs and either transfers the CPU owned CIDs to a related task which
+ * runs on the CPU or drops it into the pool. Tasks which schedule in on a
+ * CPU which the walk did not cover yet do the handover themself.
+ *
+ * This transition from CPU to per task ownership happens in two phases:
+ *
+ * 1) mm:mm_cid.transit contains MM_CID_TRANSIT This is OR'ed on the task
+ * CID and denotes that the CID is only temporarily owned by the
+ * task. When it schedules out the task drops the CID back into the
+ * pool if this bit is set.
+ *
+ * 2) The initiating context walks the per CPU space and after completion
+ * clears mm:mm_cid.transit. So after that point the CIDs are strictly
+ * task owned again.
+ *
+ * This two phase transition is required to prevent CID space exhaustion
+ * during the transition as a direct transfer of ownership would fail if
+ * two tasks are scheduled in on the same CPU before the fixup freed per
+ * CPU CIDs.
+ *
+ * When mm_cid_fixup_cpus_to_tasks() completes it's guaranteed that no CID
+ * related to that MM is owned by a CPU anymore.
*/
/*
* Update the CID range properties when the constraints change. Invoked via
* fork(), exit() and affinity changes
*/
-static void mm_update_max_cids(struct mm_struct *mm)
+static void __mm_update_max_cids(struct mm_mm_cid *mc)
+{
+ unsigned int opt_cids, max_cids;
+
+ /* Calculate the new optimal constraint */
+ opt_cids = min(mc->nr_cpus_allowed, mc->users);
+
+ /* Adjust the maximum CIDs to +25% limited by the number of possible CPUs */
+ max_cids = min(opt_cids + (opt_cids / 4), num_possible_cpus());
+ WRITE_ONCE(mc->max_cids, max_cids);
+}
+
+static inline unsigned int mm_cid_calc_pcpu_thrs(struct mm_mm_cid *mc)
+{
+ unsigned int opt_cids;
+
+ opt_cids = min(mc->nr_cpus_allowed, mc->users);
+ /* Has to be at least 1 because 0 indicates PCPU mode off */
+ return max(min(opt_cids - opt_cids / 4, num_possible_cpus() / 2), 1);
+}
+
+static bool mm_update_max_cids(struct mm_struct *mm)
{
struct mm_mm_cid *mc = &mm->mm_cid;
- unsigned int max_cids;
lockdep_assert_held(&mm->mm_cid.lock);
- /* Calculate the new maximum constraint */
- max_cids = min(mc->nr_cpus_allowed, mc->users);
- WRITE_ONCE(mc->max_cids, max_cids);
+ /* Clear deferred mode switch flag. A change is handled by the caller */
+ mc->update_deferred = false;
+ __mm_update_max_cids(mc);
+
+ /* Check whether owner mode must be changed */
+ if (!mc->percpu) {
+ /* Enable per CPU mode when the number of users is above max_cids */
+ if (mc->users > mc->max_cids)
+ mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+ } else {
+ /* Switch back to per task if user count under threshold */
+ if (mc->users < mc->pcpu_thrs)
+ mc->pcpu_thrs = 0;
+ }
+
+ /* Mode change required? */
+ if (!!mc->percpu == !!mc->pcpu_thrs)
+ return false;
+ /* When switching back to per TASK mode, set the transition flag */
+ if (!mc->pcpu_thrs)
+ WRITE_ONCE(mc->transit, MM_CID_TRANSIT);
+ WRITE_ONCE(mc->percpu, !!mc->pcpu_thrs);
+ return true;
}
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
+ struct mm_mm_cid *mc;
unsigned int weight;
if (!mm || !READ_ONCE(mm->mm_cid.users))
return;
-
/*
* mm::mm_cid::mm_cpus_allowed is the superset of each threads
* allowed CPUs mask which means it can only grow.
*/
- guard(raw_spinlock)(&mm->mm_cid.lock);
+ mc = &mm->mm_cid;
+ guard(raw_spinlock)(&mc->lock);
mm_allowed = mm_cpus_allowed(mm);
weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
- if (weight == mm->mm_cid.nr_cpus_allowed)
+ if (weight == mc->nr_cpus_allowed)
+ return;
+
+ WRITE_ONCE(mc->nr_cpus_allowed, weight);
+ __mm_update_max_cids(mc);
+ if (!mc->percpu)
return;
- WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
- mm_update_max_cids(mm);
+
+ /* Adjust the threshold to the wider set */
+ mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+
+ /* Scheduling of deferred mode switch goes here */
+}
+
+static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
+{
+ if (cid_on_cpu(t->mm_cid.cid)) {
+ unsigned int cid = cpu_cid_to_cid(t->mm_cid.cid);
+
+ t->mm_cid.cid = cid_to_transit_cid(cid);
+ pcp->cid = t->mm_cid.cid;
+ }
+}
+
+static void __maybe_unused mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
+{
+ unsigned int cpu;
+
+ /* Walk the CPUs and fixup all stale CIDs */
+ for_each_possible_cpu(cpu) {
+ struct mm_cid_pcpu *pcp = per_cpu_ptr(mm->mm_cid.pcpu, cpu);
+ struct rq *rq = cpu_rq(cpu);
+
+ /* Remote access to mm::mm_cid::pcpu requires rq_lock */
+ guard(rq_lock_irq)(rq);
+ /* Is the CID still owned by the CPU? */
+ if (cid_on_cpu(pcp->cid)) {
+ /*
+ * If rq->curr has @mm, transfer it with the
+ * transition bit set. Otherwise drop it.
+ */
+ if (rq->curr->mm == mm && rq->curr->mm_cid.active)
+ mm_cid_transit_to_task(rq->curr, pcp);
+ else
+ mm_drop_cid_on_cpu(mm, pcp);
+
+ } else if (rq->curr->mm == mm && rq->curr->mm_cid.active) {
+ unsigned int cid = rq->curr->mm_cid.cid;
+
+ /* Ensure it has the transition bit set */
+ if (!cid_in_transit(cid)) {
+ cid = cid_to_transit_cid(cid);
+ rq->curr->mm_cid.cid = cid;
+ pcp->cid = cid;
+ }
+ }
+ }
+ /* Clear the transition bit */
+ WRITE_ONCE(mm->mm_cid.transit, 0);
+}
+
+static inline void mm_cid_transfer_to_cpu(struct task_struct *t, struct mm_cid_pcpu *pcp)
+{
+ if (cid_on_task(t->mm_cid.cid)) {
+ t->mm_cid.cid = cid_to_cpu_cid(t->mm_cid.cid);
+ pcp->cid = t->mm_cid.cid;
+ }
+}
+
+static bool mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm)
+{
+ /* Remote access to mm::mm_cid::pcpu requires rq_lock */
+ guard(task_rq_lock)(t);
+ /* If the task is not active it is not in the users count */
+ if (!t->mm_cid.active)
+ return false;
+ if (cid_on_task(t->mm_cid.cid)) {
+ /* If running on the CPU, transfer the CID, otherwise drop it */
+ if (task_rq(t)->curr == t)
+ mm_cid_transfer_to_cpu(t, per_cpu_ptr(mm->mm_cid.pcpu, task_cpu(t)));
+ else
+ mm_unset_cid_on_task(t);
+ }
+ return true;
+}
+
+static void __maybe_unused mm_cid_fixup_tasks_to_cpus(void)
+{
+ struct mm_struct *mm = current->mm;
+ struct task_struct *p, *t;
+ unsigned int users;
+
+ /*
+ * This can obviously race with a concurrent affinity change, which
+ * increases the number of allowed CPUs for this mm, but that does
+ * not affect the mode and only changes the CID constraints. A
+ * possible switch back to per task mode happens either in the
+ * deferred handler function or in the next fork()/exit().
+ *
+ * The caller has already transferred. The newly incoming task is
+ * already accounted for, but not yet visible.
+ */
+ users = mm->mm_cid.users - 2;
+ if (!users)
+ return;
+
+ guard(rcu)();
+ for_other_threads(current, t) {
+ if (mm_cid_fixup_task_to_cpu(t, mm))
+ users--;
+ }
+
+ if (!users)
+ return;
+
+ /* Happens only for VM_CLONE processes. */
+ for_each_process_thread(p, t) {
+ if (t == current || t->mm != mm)
+ continue;
+ if (mm_cid_fixup_task_to_cpu(t, mm)) {
+ if (--users == 0)
+ return;
+ }
+ }
+}
+
+static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm)
+{
+ t->mm_cid.active = 1;
+ mm->mm_cid.users++;
+ return mm_update_max_cids(mm);
}
void sched_mm_cid_fork(struct task_struct *t)
@@ -10442,12 +10669,19 @@ void sched_mm_cid_fork(struct task_struct *t)
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
guard(mutex)(&mm->mm_cid.mutex);
- guard(raw_spinlock)(&mm->mm_cid.lock);
- t->mm_cid.active = 1;
- mm->mm_cid.users++;
- /* Preset last_cid for mm_cid_select() */
- t->mm_cid.last_cid = READ_ONCE(mm->mm_cid.max_cids) - 1;
- mm_update_max_cids(mm);
+ scoped_guard(raw_spinlock, &mm->mm_cid.lock) {
+ sched_mm_cid_add_user(t, mm);
+ /* Preset last_cid for mm_cid_select() */
+ t->mm_cid.last_cid = mm->mm_cid.max_cids - 1;
+ }
+}
+
+static bool sched_mm_cid_remove_user(struct task_struct *t)
+{
+ t->mm_cid.active = 0;
+ mm_unset_cid_on_task(t);
+ t->mm->mm_cid.users--;
+ return mm_update_max_cids(t->mm);
}
/*
@@ -10462,14 +10696,8 @@ void sched_mm_cid_exit(struct task_struct *t)
return;
guard(mutex)(&mm->mm_cid.mutex);
- guard(raw_spinlock)(&mm->mm_cid.lock);
- t->mm_cid.active = 0;
- mm->mm_cid.users--;
- if (t->mm_cid.cid != MM_CID_UNSET) {
- clear_bit(t->mm_cid.cid, mm_cidmask(mm));
- t->mm_cid.cid = MM_CID_UNSET;
- }
- mm_update_max_cids(mm);
+ scoped_guard(raw_spinlock, &mm->mm_cid.lock)
+ sched_mm_cid_remove_user(t);
}
/* Deactivate MM CID allocation across execve() */
@@ -10499,6 +10727,8 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.transit = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
+ mm->mm_cid.pcpu_thrs = 0;
+ mm->mm_cid.update_deferred = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
mutex_init(&mm->mm_cid.mutex);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 340af997d25dab0f05c4de8399d656b112592a93
Gitweb: https://git.kernel.org/tip/340af997d25dab0f05c4de8399d656b112592a93
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:16 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:57 +01:00
sched/mmcid: Provide CID ownership mode fixup functions
CIDs are either owned by tasks or by CPUs. The ownership mode depends on
the number of tasks related to a MM and the number of CPUs on which these
tasks are theoretically allowed to run on. Theoretically because that
number is the superset of CPU affinities of all tasks which only grows and
never shrinks.
Switching to per CPU mode happens when the user count becomes greater than
the maximum number of CIDs, which is calculated by:
opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users);
max_cids = min(1.25 * opt_cids, nr_cpu_ids);
The +25% allowance is useful for tight CPU masks in scenarios where only a
few threads are created and destroyed to avoid frequent mode
switches. Though this allowance shrinks, the closer opt_cids becomes to
nr_cpu_ids, which is the (unfortunate) hard ABI limit.
At the point of switching to per CPU mode the new user is not yet visible
in the system, so the task which initiated the fork() runs the fixup
function: mm_cid_fixup_tasks_to_cpu() walks the thread list and either
transfers each tasks owned CID to the CPU the task runs on or drops it into
the CID pool if a task is not on a CPU at that point in time. Tasks which
schedule in before the task walk reaches them do the handover in
mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus() completes it's
guaranteed that no task related to that MM owns a CID anymore.
Switching back to task mode happens when the user count goes below the
threshold which was recorded on the per CPU mode switch:
pcpu_thrs = min(opt_cids - (opt_cids / 4), nr_cpu_ids / 2);
This threshold is updated when a affinity change increases the number of
allowed CPUs for the MM, which might cause a switch back to per task mode.
If the switch back was initiated by a exiting task, then that task runs the
fixup function. If it was initiated by a affinity change, then it's run
either in the deferred update function in context of a workqueue or by a
task which forks a new one or by a task which exits. Whatever happens
first. mm_cid_fixup_cpus_to_task() walks through the possible CPUs and
either transfers the CPU owned CIDs to a related task which runs on the CPU
or drops it into the pool. Tasks which schedule in on a CPU which the walk
did not cover yet do the handover themselves.
This transition from CPU to per task ownership happens in two phases:
1) mm:mm_cid.transit contains MM_CID_TRANSIT. This is OR'ed on the task
CID and denotes that the CID is only temporarily owned by the
task. When it schedules out the task drops the CID back into the
pool if this bit is set.
2) The initiating context walks the per CPU space and after completion
clears mm:mm_cid.transit. After that point the CIDs are strictly
task owned again.
This two phase transition is required to prevent CID space exhaustion
during the transition as a direct transfer of ownership would fail if
two tasks are scheduled in on the same CPU before the fixup freed per
CPU CIDs.
When mm_cid_fixup_cpus_to_tasks() completes it's guaranteed that no CID
related to that MM is owned by a CPU anymore.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.088189028@linutronix.de
---
include/linux/rseq_types.h | 7 +-
kernel/sched/core.c | 277 ++++++++++++++++++++++++++++++++----
2 files changed, 258 insertions(+), 26 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 66b1482..a3a4f3f 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -122,14 +122,15 @@ struct mm_cid_pcpu {
* @percpu: Set, when CIDs are in per CPU mode
* @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
- * @lock: Spinlock to protect all fields except @pcpu. It also protects
- * the MM cid cpumask and the MM cidmask bitmap.
+ * @lock: Spinlock to protect against affinity setting which can't take @mutex
* @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
* @users: The number of tasks sharing this MM. Separate from mm::mm_users
* as that is modified by mmget()/mm_put() by other entities which
* do not actually share the MM.
+ * @pcpu_thrs: Threshold for switching back from per CPU mode
+ * @update_deferred: A deferred switch back to per task mode is pending.
*/
struct mm_mm_cid {
/* Hotpath read mostly members */
@@ -144,6 +145,8 @@ struct mm_mm_cid {
/* Low frequency modified */
unsigned int nr_cpus_allowed;
unsigned int users;
+ unsigned int pcpu_thrs;
+ unsigned int update_deferred;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 659ae56..f781d59 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10396,43 +10396,269 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
* task needs to drop the CID into the pool when scheduling out. Both bits
* (ONCPU and TRANSIT) are filtered out by task_cid() when the CID is
* actually handed over to user space in the RSEQ memory.
+ *
+ * Mode switching:
+ *
+ * Switching to per CPU mode happens when the user count becomes greater
+ * than the maximum number of CIDs, which is calculated by:
+ *
+ * opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users);
+ * max_cids = min(1.25 * opt_cids, num_possible_cpus());
+ *
+ * The +25% allowance is useful for tight CPU masks in scenarios where only
+ * a few threads are created and destroyed to avoid frequent mode
+ * switches. Though this allowance shrinks, the closer opt_cids becomes to
+ * num_possible_cpus(), which is the (unfortunate) hard ABI limit.
+ *
+ * At the point of switching to per CPU mode the new user is not yet
+ * visible in the system, so the task which initiated the fork() runs the
+ * fixup function: mm_cid_fixup_tasks_to_cpu() walks the thread list and
+ * either transfers each tasks owned CID to the CPU the task runs on or
+ * drops it into the CID pool if a task is not on a CPU at that point in
+ * time. Tasks which schedule in before the task walk reaches them do the
+ * handover in mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus() completes
+ * it's guaranteed that no task related to that MM owns a CID anymore.
+ *
+ * Switching back to task mode happens when the user count goes below the
+ * threshold which was recorded on the per CPU mode switch:
+ *
+ * pcpu_thrs = min(opt_cids - (opt_cids / 4), num_possible_cpus() / 2);
+ *
+ * This threshold is updated when a affinity change increases the number of
+ * allowed CPUs for the MM, which might cause a switch back to per task
+ * mode.
+ *
+ * If the switch back was initiated by a exiting task, then that task runs
+ * the fixup function. If it was initiated by a affinity change, then it's
+ * run either in the deferred update function in context of a workqueue or
+ * by a task which forks a new one or by a task which exits. Whatever
+ * happens first. mm_cid_fixup_cpus_to_task() walks through the possible
+ * CPUs and either transfers the CPU owned CIDs to a related task which
+ * runs on the CPU or drops it into the pool. Tasks which schedule in on a
+ * CPU which the walk did not cover yet do the handover themself.
+ *
+ * This transition from CPU to per task ownership happens in two phases:
+ *
+ * 1) mm:mm_cid.transit contains MM_CID_TRANSIT This is OR'ed on the task
+ * CID and denotes that the CID is only temporarily owned by the
+ * task. When it schedules out the task drops the CID back into the
+ * pool if this bit is set.
+ *
+ * 2) The initiating context walks the per CPU space and after completion
+ * clears mm:mm_cid.transit. So after that point the CIDs are strictly
+ * task owned again.
+ *
+ * This two phase transition is required to prevent CID space exhaustion
+ * during the transition as a direct transfer of ownership would fail if
+ * two tasks are scheduled in on the same CPU before the fixup freed per
+ * CPU CIDs.
+ *
+ * When mm_cid_fixup_cpus_to_tasks() completes it's guaranteed that no CID
+ * related to that MM is owned by a CPU anymore.
*/
/*
* Update the CID range properties when the constraints change. Invoked via
* fork(), exit() and affinity changes
*/
-static void mm_update_max_cids(struct mm_struct *mm)
+static void __mm_update_max_cids(struct mm_mm_cid *mc)
+{
+ unsigned int opt_cids, max_cids;
+
+ /* Calculate the new optimal constraint */
+ opt_cids = min(mc->nr_cpus_allowed, mc->users);
+
+ /* Adjust the maximum CIDs to +25% limited by the number of possible CPUs */
+ max_cids = min(opt_cids + (opt_cids / 4), num_possible_cpus());
+ WRITE_ONCE(mc->max_cids, max_cids);
+}
+
+static inline unsigned int mm_cid_calc_pcpu_thrs(struct mm_mm_cid *mc)
+{
+ unsigned int opt_cids;
+
+ opt_cids = min(mc->nr_cpus_allowed, mc->users);
+ return min(opt_cids - opt_cids / 4, num_possible_cpus() / 2);
+}
+
+static bool mm_update_max_cids(struct mm_struct *mm)
{
struct mm_mm_cid *mc = &mm->mm_cid;
- unsigned int max_cids;
lockdep_assert_held(&mm->mm_cid.lock);
- /* Calculate the new maximum constraint */
- max_cids = min(mc->nr_cpus_allowed, mc->users);
- WRITE_ONCE(mc->max_cids, max_cids);
+ /* Clear deferred mode switch flag. A change is handled by the caller */
+ mc->update_deferred = false;
+ __mm_update_max_cids(mc);
+
+ /* Check whether owner mode must be changed */
+ if (!mc->percpu) {
+ /* Enable per CPU mode when the number of users is above max_cids */
+ if (mc->users > mc->max_cids)
+ mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+ } else {
+ /* Switch back to per task if user count under threshold */
+ if (mc->users < mc->pcpu_thrs)
+ mc->pcpu_thrs = 0;
+ }
+
+ /* Mode change required? */
+ if (!!mc->percpu == !!mc->pcpu_thrs)
+ return false;
+ /* When switching back to per TASK mode, set the transition flag */
+ if (!mc->pcpu_thrs)
+ WRITE_ONCE(mc->transit, MM_CID_TRANSIT);
+ WRITE_ONCE(mc->percpu, !!mc->pcpu_thrs);
+ return true;
}
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
+ struct mm_mm_cid *mc;
unsigned int weight;
if (!mm || !READ_ONCE(mm->mm_cid.users))
return;
-
/*
* mm::mm_cid::mm_cpus_allowed is the superset of each threads
* allowed CPUs mask which means it can only grow.
*/
- guard(raw_spinlock)(&mm->mm_cid.lock);
+ mc = &mm->mm_cid;
+ guard(raw_spinlock)(&mc->lock);
mm_allowed = mm_cpus_allowed(mm);
weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
- if (weight == mm->mm_cid.nr_cpus_allowed)
+ if (weight == mc->nr_cpus_allowed)
+ return;
+
+ WRITE_ONCE(mc->nr_cpus_allowed, weight);
+ __mm_update_max_cids(mc);
+ if (!mc->percpu)
return;
- WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
- mm_update_max_cids(mm);
+
+ /* Adjust the threshold to the wider set */
+ mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+
+ /* Scheduling of deferred mode switch goes here */
+}
+
+static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
+{
+ if (cid_on_cpu(t->mm_cid.cid)) {
+ unsigned int cid = cpu_cid_to_cid(t->mm_cid.cid);
+
+ t->mm_cid.cid = cid_to_transit_cid(cid);
+ pcp->cid = t->mm_cid.cid;
+ }
+}
+
+static void __maybe_unused mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
+{
+ unsigned int cpu;
+
+ /* Walk the CPUs and fixup all stale CIDs */
+ for_each_possible_cpu(cpu) {
+ struct mm_cid_pcpu *pcp = per_cpu_ptr(mm->mm_cid.pcpu, cpu);
+ struct rq *rq = cpu_rq(cpu);
+
+ /* Remote access to mm::mm_cid::pcpu requires rq_lock */
+ guard(rq_lock_irq)(rq);
+ /* Is the CID still owned by the CPU? */
+ if (cid_on_cpu(pcp->cid)) {
+ /*
+ * If rq->curr has @mm, transfer it with the
+ * transition bit set. Otherwise drop it.
+ */
+ if (rq->curr->mm == mm && rq->curr->mm_cid.active)
+ mm_cid_transit_to_task(rq->curr, pcp);
+ else
+ mm_drop_cid_on_cpu(mm, pcp);
+
+ } else if (rq->curr->mm == mm && rq->curr->mm_cid.active) {
+ unsigned int cid = rq->curr->mm_cid.cid;
+
+ /* Ensure it has the transition bit set */
+ if (!cid_in_transit(cid)) {
+ cid = cid_to_transit_cid(cid);
+ rq->curr->mm_cid.cid = cid;
+ pcp->cid = cid;
+ }
+ }
+ }
+ /* Clear the transition bit */
+ WRITE_ONCE(mm->mm_cid.transit, 0);
+}
+
+static inline void mm_cid_transfer_to_cpu(struct task_struct *t, struct mm_cid_pcpu *pcp)
+{
+ if (cid_on_task(t->mm_cid.cid)) {
+ t->mm_cid.cid = cid_to_cpu_cid(t->mm_cid.cid);
+ pcp->cid = t->mm_cid.cid;
+ }
+}
+
+static bool mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm)
+{
+ /* Remote access to mm::mm_cid::pcpu requires rq_lock */
+ guard(task_rq_lock)(t);
+ /* If the task is not active it is not in the users count */
+ if (!t->mm_cid.active)
+ return false;
+ if (cid_on_task(t->mm_cid.cid)) {
+ /* If running on the CPU, transfer the CID, otherwise drop it */
+ if (task_rq(t)->curr == t)
+ mm_cid_transfer_to_cpu(t, per_cpu_ptr(mm->mm_cid.pcpu, task_cpu(t)));
+ else
+ mm_unset_cid_on_task(t);
+ }
+ return true;
+}
+
+static void __maybe_unused mm_cid_fixup_tasks_to_cpus(void)
+{
+ struct mm_struct *mm = current->mm;
+ struct task_struct *p, *t;
+ unsigned int users;
+
+ /*
+ * This can obviously race with a concurrent affinity change, which
+ * increases the number of allowed CPUs for this mm, but that does
+ * not affect the mode and only changes the CID constraints. A
+ * possible switch back to per task mode happens either in the
+ * deferred handler function or in the next fork()/exit().
+ *
+ * The caller has already transferred. The newly incoming task is
+ * already accounted for, but not yet visible.
+ */
+ users = mm->mm_cid.users - 2;
+ if (!users)
+ return;
+
+ guard(rcu)();
+ for_other_threads(current, t) {
+ if (mm_cid_fixup_task_to_cpu(t, mm))
+ users--;
+ }
+
+ if (!users)
+ return;
+
+ /* Happens only for VM_CLONE processes. */
+ for_each_process_thread(p, t) {
+ if (t == current || t->mm != mm)
+ continue;
+ if (mm_cid_fixup_task_to_cpu(t, mm)) {
+ if (--users == 0)
+ return;
+ }
+ }
+}
+
+static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm)
+{
+ t->mm_cid.active = 1;
+ mm->mm_cid.users++;
+ return mm_update_max_cids(mm);
}
void sched_mm_cid_fork(struct task_struct *t)
@@ -10442,12 +10668,19 @@ void sched_mm_cid_fork(struct task_struct *t)
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
guard(mutex)(&mm->mm_cid.mutex);
- guard(raw_spinlock)(&mm->mm_cid.lock);
- t->mm_cid.active = 1;
- mm->mm_cid.users++;
- /* Preset last_cid for mm_cid_select() */
- t->mm_cid.last_cid = READ_ONCE(mm->mm_cid.max_cids) - 1;
- mm_update_max_cids(mm);
+ scoped_guard(raw_spinlock, &mm->mm_cid.lock) {
+ sched_mm_cid_add_user(t, mm);
+ /* Preset last_cid for mm_cid_select() */
+ t->mm_cid.last_cid = mm->mm_cid.max_cids - 1;
+ }
+}
+
+static bool sched_mm_cid_remove_user(struct task_struct *t)
+{
+ t->mm_cid.active = 0;
+ mm_unset_cid_on_task(t);
+ t->mm->mm_cid.users--;
+ return mm_update_max_cids(t->mm);
}
/*
@@ -10462,14 +10695,8 @@ void sched_mm_cid_exit(struct task_struct *t)
return;
guard(mutex)(&mm->mm_cid.mutex);
- guard(raw_spinlock)(&mm->mm_cid.lock);
- t->mm_cid.active = 0;
- mm->mm_cid.users--;
- if (t->mm_cid.cid != MM_CID_UNSET) {
- clear_bit(t->mm_cid.cid, mm_cidmask(mm));
- t->mm_cid.cid = MM_CID_UNSET;
- }
- mm_update_max_cids(mm);
+ scoped_guard(raw_spinlock, &mm->mm_cid.lock)
+ sched_mm_cid_remove_user(t);
}
/* Deactivate MM CID allocation across execve() */
@@ -10499,6 +10726,8 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.transit = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
+ mm->mm_cid.pcpu_thrs = 0;
+ mm->mm_cid.update_deferred = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
mutex_init(&mm->mm_cid.mutex);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);... to avoid header recursion hell.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/irq_work.h | 9 ++-------
include/linux/irq_work_types.h | 14 ++++++++++++++
2 files changed, 16 insertions(+), 7 deletions(-)
--- a/include/linux/irq_work.h
+++ b/include/linux/irq_work.h
@@ -2,8 +2,9 @@
#ifndef _LINUX_IRQ_WORK_H
#define _LINUX_IRQ_WORK_H
-#include <linux/smp_types.h>
+#include <linux/irq_work_types.h>
#include <linux/rcuwait.h>
+#include <linux/smp_types.h>
/*
* An entry can be in one of four states:
@@ -14,12 +15,6 @@
* busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
*/
-struct irq_work {
- struct __call_single_node node;
- void (*func)(struct irq_work *);
- struct rcuwait irqwait;
-};
-
#define __IRQ_WORK_INIT(_func, _flags) (struct irq_work){ \
.node = { .u_flags = (_flags), }, \
.func = (_func), \
--- /dev/null
+++ b/include/linux/irq_work_types.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_IRQ_WORK_TYPES_H
+#define _LINUX_IRQ_WORK_TYPES_H
+
+#include <linux/smp_types.h>
+#include <linux/types.h>
+
+struct irq_work {
+ struct __call_single_node node;
+ void (*func)(struct irq_work *);
+ struct rcuwait irqwait;
+};
+
+#endifThe following commit has been merged into the core/rseq branch of tip:
Commit-ID: c809f081fe400cb1b9898f4791c0d33146315161
Gitweb: https://git.kernel.org/tip/c809f081fe400cb1b9898f4791c0d33146315161
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:18 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:41 +01:00
irqwork: Move data struct to a types header
... to avoid header recursion hell.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.152813625@linutronix.de
---
include/linux/irq_work.h | 9 ++-------
include/linux/irq_work_types.h | 14 ++++++++++++++
2 files changed, 16 insertions(+), 7 deletions(-)
create mode 100644 include/linux/irq_work_types.h
diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h
index 136f298..c5afd05 100644
--- a/include/linux/irq_work.h
+++ b/include/linux/irq_work.h
@@ -2,8 +2,9 @@
#ifndef _LINUX_IRQ_WORK_H
#define _LINUX_IRQ_WORK_H
-#include <linux/smp_types.h>
+#include <linux/irq_work_types.h>
#include <linux/rcuwait.h>
+#include <linux/smp_types.h>
/*
* An entry can be in one of four states:
@@ -14,12 +15,6 @@
* busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
*/
-struct irq_work {
- struct __call_single_node node;
- void (*func)(struct irq_work *);
- struct rcuwait irqwait;
-};
-
#define __IRQ_WORK_INIT(_func, _flags) (struct irq_work){ \
.node = { .u_flags = (_flags), }, \
.func = (_func), \
diff --git a/include/linux/irq_work_types.h b/include/linux/irq_work_types.h
new file mode 100644
index 0000000..73abec5
--- /dev/null
+++ b/include/linux/irq_work_types.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_IRQ_WORK_TYPES_H
+#define _LINUX_IRQ_WORK_TYPES_H
+
+#include <linux/smp_types.h>
+#include <linux/types.h>
+
+struct irq_work {
+ struct __call_single_node node;
+ void (*func)(struct irq_work *);
+ struct rcuwait irqwait;
+};
+
+#endifThe following commit has been merged into the core/rseq branch of tip:
Commit-ID: 2644779ec144d3e8cce5fed9623b47e70b3e0422
Gitweb: https://git.kernel.org/tip/2644779ec144d3e8cce5fed9623b47e70b3e0422
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:18 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:58 +01:00
irqwork: Move data struct to a types header
... to avoid header recursion hell.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.152813625@linutronix.de
---
include/linux/irq_work.h | 9 ++-------
include/linux/irq_work_types.h | 14 ++++++++++++++
2 files changed, 16 insertions(+), 7 deletions(-)
create mode 100644 include/linux/irq_work_types.h
diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h
index 136f298..c5afd05 100644
--- a/include/linux/irq_work.h
+++ b/include/linux/irq_work.h
@@ -2,8 +2,9 @@
#ifndef _LINUX_IRQ_WORK_H
#define _LINUX_IRQ_WORK_H
-#include <linux/smp_types.h>
+#include <linux/irq_work_types.h>
#include <linux/rcuwait.h>
+#include <linux/smp_types.h>
/*
* An entry can be in one of four states:
@@ -14,12 +15,6 @@
* busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
*/
-struct irq_work {
- struct __call_single_node node;
- void (*func)(struct irq_work *);
- struct rcuwait irqwait;
-};
-
#define __IRQ_WORK_INIT(_func, _flags) (struct irq_work){ \
.node = { .u_flags = (_flags), }, \
.func = (_func), \
diff --git a/include/linux/irq_work_types.h b/include/linux/irq_work_types.h
new file mode 100644
index 0000000..73abec5
--- /dev/null
+++ b/include/linux/irq_work_types.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_IRQ_WORK_TYPES_H
+#define _LINUX_IRQ_WORK_TYPES_H
+
+#include <linux/smp_types.h>
+#include <linux/types.h>
+
+struct irq_work {
+ struct __call_single_node node;
+ void (*func)(struct irq_work *);
+ struct rcuwait irqwait;
+};
+
+#endifWhen affinity changes cause an increase of the number of CPUs allowed for
tasks which are related to a MM, that might results in a situation where
the ownership mode can go back from per CPU mode to per task mode.
As affinity changes happen with runqueue lock held there is no way to do
the actual mode change and required fixup right there.
Add the infrastructure to defer it to a workqueue. The scheduled work can
race with a fork() or exit(). Whatever happens first takes care of it.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/rseq_types.h | 8 ++++++
kernel/sched/core.c | 58 +++++++++++++++++++++++++++++++++++++++------
2 files changed, 59 insertions(+), 7 deletions(-)
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -2,7 +2,9 @@
#ifndef _LINUX_RSEQ_TYPES_H
#define _LINUX_RSEQ_TYPES_H
+#include <linux/irq_work_types.h>
#include <linux/types.h>
+#include <linux/workqueue_types.h>
#ifdef CONFIG_RSEQ
struct rseq;
@@ -122,6 +124,8 @@ struct mm_cid_pcpu {
* @percpu: Set, when CIDs are in per CPU mode
* @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
+ * @irq_work: irq_work to handle the affinity mode change case
+ * @work: Regular work to handle the affinity mode change case
* @lock: Spinlock to protect against affinity setting which can't take @mutex
* @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
@@ -139,6 +143,10 @@ struct mm_mm_cid {
unsigned int transit;
unsigned int max_cids;
+ /* Rarely used. Moves @lock and @mutex into the second cacheline */
+ struct irq_work irq_work;
+ struct work_struct work;
+
raw_spinlock_t lock;
struct mutex mutex;
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10538,8 +10538,17 @@ static inline void mm_update_cpus_allowe
/* Adjust the threshold to the wider set */
mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+ /* Switch back to per task mode? */
+ if (mc->users >= mc->pcpu_thrs)
+ return;
+
+ /* Don't queue twice */
+ if (mc->update_deferred)
+ return;
- /* Scheduling of deferred mode switch goes here */
+ /* Queue the irq work, which schedules the real work */
+ mc->update_deferred = true;
+ irq_work_queue(&mc->irq_work);
}
static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
@@ -10552,7 +10561,7 @@ static inline void mm_cid_transit_to_tas
}
}
-static void __maybe_unused mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
+static void mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
{
unsigned int cpu;
@@ -10713,14 +10722,47 @@ void sched_mm_cid_after_execve(struct ta
mm_cid_select(t);
}
-void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+static void mm_cid_work_fn(struct work_struct *work)
{
- struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
- int cpu;
+ struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
+
+ /* Make it compile, but not functional yet */
+ if (!IS_ENABLED(CONFIG_NEW_MM_CID))
+ return;
- for_each_possible_cpu(cpu)
- per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+ guard(mutex)(&mm->mm_cid.mutex);
+ /* Did the last user task exit already? */
+ if (!mm->mm_cid.users)
+ return;
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ /* Have fork() or exit() handled it already? */
+ if (!mm->mm_cid.update_deferred)
+ return;
+ /* This clears mm_cid::update_deferred */
+ if (!mm_update_max_cids(mm))
+ return;
+ /* Affinity changes can only switch back to task mode */
+ if (WARN_ON_ONCE(mm->mm_cid.percpu))
+ return;
+ }
+ mm_cid_fixup_cpus_to_tasks(mm);
+}
+
+static void mm_cid_irq_work(struct irq_work *work)
+{
+ struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.irq_work);
+
+ /*
+ * Needs to be unconditional because mm_cid::lock cannot be held
+ * when scheduling work as mm_update_cpus_allowed() nests inside
+ * rq::lock and schedule_work() might end up in wakeup...
+ */
+ schedule_work(&mm->mm_cid.work);
+}
+
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+{
mm->mm_cid.max_cids = 0;
mm->mm_cid.percpu = 0;
mm->mm_cid.transit = 0;
@@ -10730,6 +10772,8 @@ void mm_init_cid(struct mm_struct *mm, s
mm->mm_cid.update_deferred = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
mutex_init(&mm->mm_cid.mutex);
+ mm->mm_cid.irq_work = IRQ_WORK_INIT_HARD(mm_cid_irq_work);
+ INIT_WORK(&mm->mm_cid.work, mm_cid_work_fn);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 9da6ccbcea3de1fa704202e3346fe6c0226bfc18
Gitweb: https://git.kernel.org/tip/9da6ccbcea3de1fa704202e3346fe6c0226bfc18
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:20 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:42 +01:00
sched/mmcid: Implement deferred mode change
When affinity changes cause an increase of the number of CPUs allowed for
tasks which are related to a MM, that might results in a situation where
the ownership mode can go back from per CPU mode to per task mode.
As affinity changes happen with runqueue lock held there is no way to do
the actual mode change and required fixup right there.
Add the infrastructure to defer it to a workqueue. The scheduled work can
race with a fork() or exit(). Whatever happens first takes care of it.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.216484739@linutronix.de
---
include/linux/rseq_types.h | 8 +++++-
kernel/sched/core.c | 58 ++++++++++++++++++++++++++++++++-----
2 files changed, 59 insertions(+), 7 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index a3a4f3f..81fbb88 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -2,7 +2,9 @@
#ifndef _LINUX_RSEQ_TYPES_H
#define _LINUX_RSEQ_TYPES_H
+#include <linux/irq_work_types.h>
#include <linux/types.h>
+#include <linux/workqueue_types.h>
#ifdef CONFIG_RSEQ
struct rseq;
@@ -122,6 +124,8 @@ struct mm_cid_pcpu {
* @percpu: Set, when CIDs are in per CPU mode
* @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
+ * @irq_work: irq_work to handle the affinity mode change case
+ * @work: Regular work to handle the affinity mode change case
* @lock: Spinlock to protect against affinity setting which can't take @mutex
* @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
@@ -139,6 +143,10 @@ struct mm_mm_cid {
unsigned int transit;
unsigned int max_cids;
+ /* Rarely used. Moves @lock and @mutex into the second cacheline */
+ struct irq_work irq_work;
+ struct work_struct work;
+
raw_spinlock_t lock;
struct mutex mutex;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index eb0d59d..cbb543a 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10539,8 +10539,17 @@ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpu
/* Adjust the threshold to the wider set */
mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+ /* Switch back to per task mode? */
+ if (mc->users >= mc->pcpu_thrs)
+ return;
+
+ /* Don't queue twice */
+ if (mc->update_deferred)
+ return;
- /* Scheduling of deferred mode switch goes here */
+ /* Queue the irq work, which schedules the real work */
+ mc->update_deferred = true;
+ irq_work_queue(&mc->irq_work);
}
static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
@@ -10553,7 +10562,7 @@ static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_p
}
}
-static void __maybe_unused mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
+static void mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
{
unsigned int cpu;
@@ -10714,14 +10723,47 @@ void sched_mm_cid_after_execve(struct task_struct *t)
mm_cid_select(t);
}
-void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+static void mm_cid_work_fn(struct work_struct *work)
{
- struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
- int cpu;
+ struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
- for_each_possible_cpu(cpu)
- per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+ /* Make it compile, but not functional yet */
+ if (!IS_ENABLED(CONFIG_NEW_MM_CID))
+ return;
+
+ guard(mutex)(&mm->mm_cid.mutex);
+ /* Did the last user task exit already? */
+ if (!mm->mm_cid.users)
+ return;
+
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ /* Have fork() or exit() handled it already? */
+ if (!mm->mm_cid.update_deferred)
+ return;
+ /* This clears mm_cid::update_deferred */
+ if (!mm_update_max_cids(mm))
+ return;
+ /* Affinity changes can only switch back to task mode */
+ if (WARN_ON_ONCE(mm->mm_cid.percpu))
+ return;
+ }
+ mm_cid_fixup_cpus_to_tasks(mm);
+}
+
+static void mm_cid_irq_work(struct irq_work *work)
+{
+ struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.irq_work);
+ /*
+ * Needs to be unconditional because mm_cid::lock cannot be held
+ * when scheduling work as mm_update_cpus_allowed() nests inside
+ * rq::lock and schedule_work() might end up in wakeup...
+ */
+ schedule_work(&mm->mm_cid.work);
+}
+
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+{
mm->mm_cid.max_cids = 0;
mm->mm_cid.percpu = 0;
mm->mm_cid.transit = 0;
@@ -10731,6 +10773,8 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.update_deferred = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
mutex_init(&mm->mm_cid.mutex);
+ mm->mm_cid.irq_work = IRQ_WORK_INIT_HARD(mm_cid_irq_work);
+ INIT_WORK(&mm->mm_cid.work, mm_cid_work_fn);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}The following commit has been merged into the core/rseq branch of tip:
Commit-ID: cba5e581161e379037a94f5a75d1a61bd1ccce3b
Gitweb: https://git.kernel.org/tip/cba5e581161e379037a94f5a75d1a61bd1ccce3b
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:20 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:58 +01:00
sched/mmcid: Implement deferred mode change
When affinity changes cause an increase of the number of CPUs allowed for
tasks which are related to a MM, that might results in a situation where
the ownership mode can go back from per CPU mode to per task mode.
As affinity changes happen with runqueue lock held there is no way to do
the actual mode change and required fixup right there.
Add the infrastructure to defer it to a workqueue. The scheduled work can
race with a fork() or exit(). Whatever happens first takes care of it.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.216484739@linutronix.de
---
include/linux/rseq_types.h | 8 +++++-
kernel/sched/core.c | 58 ++++++++++++++++++++++++++++++++-----
2 files changed, 59 insertions(+), 7 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index a3a4f3f..81fbb88 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -2,7 +2,9 @@
#ifndef _LINUX_RSEQ_TYPES_H
#define _LINUX_RSEQ_TYPES_H
+#include <linux/irq_work_types.h>
#include <linux/types.h>
+#include <linux/workqueue_types.h>
#ifdef CONFIG_RSEQ
struct rseq;
@@ -122,6 +124,8 @@ struct mm_cid_pcpu {
* @percpu: Set, when CIDs are in per CPU mode
* @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
+ * @irq_work: irq_work to handle the affinity mode change case
+ * @work: Regular work to handle the affinity mode change case
* @lock: Spinlock to protect against affinity setting which can't take @mutex
* @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
@@ -139,6 +143,10 @@ struct mm_mm_cid {
unsigned int transit;
unsigned int max_cids;
+ /* Rarely used. Moves @lock and @mutex into the second cacheline */
+ struct irq_work irq_work;
+ struct work_struct work;
+
raw_spinlock_t lock;
struct mutex mutex;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f781d59..938757e 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10538,8 +10538,17 @@ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpu
/* Adjust the threshold to the wider set */
mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+ /* Switch back to per task mode? */
+ if (mc->users >= mc->pcpu_thrs)
+ return;
+
+ /* Don't queue twice */
+ if (mc->update_deferred)
+ return;
- /* Scheduling of deferred mode switch goes here */
+ /* Queue the irq work, which schedules the real work */
+ mc->update_deferred = true;
+ irq_work_queue(&mc->irq_work);
}
static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
@@ -10552,7 +10561,7 @@ static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_p
}
}
-static void __maybe_unused mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
+static void mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
{
unsigned int cpu;
@@ -10713,14 +10722,47 @@ void sched_mm_cid_after_execve(struct task_struct *t)
mm_cid_select(t);
}
-void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+static void mm_cid_work_fn(struct work_struct *work)
{
- struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
- int cpu;
+ struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
- for_each_possible_cpu(cpu)
- per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+ /* Make it compile, but not functional yet */
+ if (!IS_ENABLED(CONFIG_NEW_MM_CID))
+ return;
+
+ guard(mutex)(&mm->mm_cid.mutex);
+ /* Did the last user task exit already? */
+ if (!mm->mm_cid.users)
+ return;
+
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ /* Have fork() or exit() handled it already? */
+ if (!mm->mm_cid.update_deferred)
+ return;
+ /* This clears mm_cid::update_deferred */
+ if (!mm_update_max_cids(mm))
+ return;
+ /* Affinity changes can only switch back to task mode */
+ if (WARN_ON_ONCE(mm->mm_cid.percpu))
+ return;
+ }
+ mm_cid_fixup_cpus_to_tasks(mm);
+}
+
+static void mm_cid_irq_work(struct irq_work *work)
+{
+ struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.irq_work);
+ /*
+ * Needs to be unconditional because mm_cid::lock cannot be held
+ * when scheduling work as mm_update_cpus_allowed() nests inside
+ * rq::lock and schedule_work() might end up in wakeup...
+ */
+ schedule_work(&mm->mm_cid.work);
+}
+
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+{
mm->mm_cid.max_cids = 0;
mm->mm_cid.percpu = 0;
mm->mm_cid.transit = 0;
@@ -10730,6 +10772,8 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.update_deferred = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
mutex_init(&mm->mm_cid.mutex);
+ mm->mm_cid.irq_work = IRQ_WORK_INIT_HARD(mm_cid_irq_work);
+ INIT_WORK(&mm->mm_cid.work, mm_cid_work_fn);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}Now that all pieces are in place, change the implementations of
sched_mm_cid_fork() and sched_mm_cid_exit() to adhere to the new strict
ownership scheme and switch context_switch() over to use the new
mm_cid_schedin() functionality.
The common case is that there is no mode change required, which makes
fork() and exit() just update the user count and the constraints.
In case that a new user would exceed the CID space limit the fork() context
handles the transition to per CPU mode with mm::mm_cid::mutex held. exit()
handles the transition back to per task mode when the user count drops
below the switch back threshold. fork() might also be forced to handle a
deferred switch back to per task mode, when a affinity change increased the
number of allowed CPUs enough.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/rseq.h | 19 -------
include/linux/rseq_types.h | 8 +--
kernel/fork.c | 1
kernel/sched/core.c | 115 ++++++++++++++++++++++++++++++++++++++-------
kernel/sched/sched.h | 76 -----------------------------
5 files changed, 103 insertions(+), 116 deletions(-)
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -84,24 +84,6 @@ static __always_inline void rseq_sched_s
t->rseq.event.ids_changed = true;
}
-/*
- * Invoked from switch_mm_cid() in context switch when the task gets a MM
- * CID assigned.
- *
- * This does not raise TIF_NOTIFY_RESUME as that happens in
- * rseq_sched_switch_event().
- */
-static __always_inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid)
-{
- /*
- * Requires a comparison as the switch_mm_cid() code does not
- * provide a conditional for it readily. So avoid excessive updates
- * when nothing changes.
- */
- if (t->rseq.ids.mm_cid != cid)
- t->rseq.event.ids_changed = true;
-}
-
/* Enforce a full update after RSEQ registration and when execve() failed */
static inline void rseq_force_update(void)
{
@@ -170,7 +152,6 @@ static inline void rseq_handle_slowpath(
static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }
static inline void rseq_sched_switch_event(struct task_struct *t) { }
static inline void rseq_sched_set_ids_changed(struct task_struct *t) { }
-static inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid) { }
static inline void rseq_force_update(void) { }
static inline void rseq_virt_userspace_exit(void) { }
static inline void rseq_fork(struct task_struct *t, u64 clone_flags) { }
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -101,18 +101,18 @@ struct rseq_data { };
/**
* struct sched_mm_cid - Storage for per task MM CID data
* @active: MM CID is active for the task
- * @cid: The CID associated to the task
- * @last_cid: The last CID associated to the task
+ * @cid: The CID associated to the task either permanently or
+ * borrowed from the CPU
*/
struct sched_mm_cid {
unsigned int active;
unsigned int cid;
- unsigned int last_cid;
};
/**
* struct mm_cid_pcpu - Storage for per CPU MM_CID data
- * @cid: The CID associated to the CPU
+ * @cid: The CID associated to the CPU either permanently or
+ * while a task with a CID is running
*/
struct mm_cid_pcpu {
unsigned int cid;
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -956,7 +956,6 @@ static struct task_struct *dup_task_stru
#ifdef CONFIG_SCHED_MM_CID
tsk->mm_cid.cid = MM_CID_UNSET;
- tsk->mm_cid.last_cid = MM_CID_UNSET;
tsk->mm_cid.active = 0;
#endif
return tsk;
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5307,7 +5307,7 @@ context_switch(struct rq *rq, struct tas
}
}
- switch_mm_cid(prev, next);
+ mm_cid_switch_to(prev, next);
/*
* Tell rseq that the task was scheduled in. Must be after
@@ -10623,7 +10623,7 @@ static bool mm_cid_fixup_task_to_cpu(str
return true;
}
-static void __maybe_unused mm_cid_fixup_tasks_to_cpus(void)
+static void mm_cid_fixup_tasks_to_cpus(void)
{
struct mm_struct *mm = current->mm;
struct task_struct *p, *t;
@@ -10673,25 +10673,81 @@ static bool sched_mm_cid_add_user(struct
void sched_mm_cid_fork(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
+ bool percpu;
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
guard(mutex)(&mm->mm_cid.mutex);
- scoped_guard(raw_spinlock, &mm->mm_cid.lock) {
- sched_mm_cid_add_user(t, mm);
- /* Preset last_cid for mm_cid_select() */
- t->mm_cid.last_cid = mm->mm_cid.max_cids - 1;
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ struct mm_cid_pcpu *pcp = this_cpu_ptr(mm->mm_cid.pcpu);
+
+ /* First user ? */
+ if (!mm->mm_cid.users) {
+ sched_mm_cid_add_user(t, mm);
+ t->mm_cid.cid = mm_get_cid(mm);
+ /* Required for execve() */
+ pcp->cid = t->mm_cid.cid;
+ return;
+ }
+
+ if (!sched_mm_cid_add_user(t, mm)) {
+ if (!mm->mm_cid.percpu)
+ t->mm_cid.cid = mm_get_cid(mm);
+ return;
+ }
+
+ /* Handle the mode change and transfer current's CID */
+ percpu = !!mm->mm_cid.percpu;
+ if (!percpu)
+ mm_cid_transit_to_task(current, pcp);
+ else
+ mm_cid_transfer_to_cpu(current, pcp);
+ }
+
+ if (percpu) {
+ mm_cid_fixup_tasks_to_cpus();
+ } else {
+ mm_cid_fixup_cpus_to_tasks(mm);
+ t->mm_cid.cid = mm_get_cid(mm);
}
}
static bool sched_mm_cid_remove_user(struct task_struct *t)
{
t->mm_cid.active = 0;
- mm_unset_cid_on_task(t);
+ scoped_guard(preempt) {
+ /* Clear the transition bit */
+ t->mm_cid.cid = cid_from_transit_cid(t->mm_cid.cid);
+ mm_unset_cid_on_task(t);
+ }
t->mm->mm_cid.users--;
return mm_update_max_cids(t->mm);
}
+static bool __sched_mm_cid_exit(struct task_struct *t)
+{
+ struct mm_struct *mm = t->mm;
+
+ if (!sched_mm_cid_remove_user(t))
+ return false;
+ /*
+ * Contrary to fork() this only deals with a switch back to per
+ * task mode either because the above decreased users or an
+ * affinity change increased the number of allowed CPUs and the
+ * deferred fixup did not run yet.
+ */
+ if (WARN_ON_ONCE(mm->mm_cid.percpu))
+ return false;
+ /*
+ * A failed fork(2) cleanup never gets here, so @current must have
+ * the same MM as @t. That's true for exit() and the failed
+ * pthread_create() cleanup case.
+ */
+ if (WARN_ON_ONCE(current->mm != mm))
+ return false;
+ return true;
+}
+
/*
* When a task exits, the MM CID held by the task is not longer required as
* the task cannot return to user space.
@@ -10702,10 +10758,43 @@ void sched_mm_cid_exit(struct task_struc
if (!mm || !t->mm_cid.active)
return;
+ /*
+ * Ensure that only one instance is doing MM CID operations within
+ * a MM. The common case is uncontended. The rare fixup case adds
+ * some overhead.
+ */
+ scoped_guard(mutex, &mm->mm_cid.mutex) {
+ /* mm_cid::mutex is sufficient to protect mm_cid::users */
+ if (likely(mm->mm_cid.users > 1)) {
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ if (!__sched_mm_cid_exit(t))
+ return;
+ /* Mode change required. Transfer currents CID */
+ mm_cid_transit_to_task(current, this_cpu_ptr(mm->mm_cid.pcpu));
+ }
+ mm_cid_fixup_cpus_to_tasks(mm);
+ return;
+ }
+ /* Last user */
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ /* Required across execve() */
+ if (t == current)
+ mm_cid_transit_to_task(t, this_cpu_ptr(mm->mm_cid.pcpu));
+ /* Ignore mode change. There is nothing to do. */
+ sched_mm_cid_remove_user(t);
+ }
+ }
- guard(mutex)(&mm->mm_cid.mutex);
- scoped_guard(raw_spinlock, &mm->mm_cid.lock)
- sched_mm_cid_remove_user(t);
+ /*
+ * As this is the last user (execve(), process exit or failed
+ * fork(2)) there is no concurrency anymore.
+ *
+ * Synchronize eventually pending work to ensure that there are no
+ * dangling references left. @t->mm_cid.users is zero so nothing
+ * can queue this work anymore.
+ */
+ irq_work_sync(&mm->mm_cid.irq_work);
+ cancel_work_sync(&mm->mm_cid.work);
}
/* Deactivate MM CID allocation across execve() */
@@ -10718,18 +10807,12 @@ void sched_mm_cid_before_execve(struct t
void sched_mm_cid_after_execve(struct task_struct *t)
{
sched_mm_cid_fork(t);
- guard(preempt)();
- mm_cid_select(t);
}
static void mm_cid_work_fn(struct work_struct *work)
{
struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
- /* Make it compile, but not functional yet */
- if (!IS_ENABLED(CONFIG_NEW_MM_CID))
- return;
-
guard(mutex)(&mm->mm_cid.mutex);
/* Did the last user task exit already? */
if (!mm->mm_cid.users)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3745,83 +3745,7 @@ static inline void mm_cid_switch_to(stru
mm_cid_schedin(next);
}
-/* Active implementation */
-static inline void init_sched_mm_cid(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- unsigned int max_cid;
-
- if (!mm)
- return;
-
- /* Preset last_mm_cid */
- max_cid = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
- t->mm_cid.last_cid = max_cid - 1;
-}
-
-static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
-{
- struct mm_struct *mm = t->mm;
-
- if (cid >= max_cids)
- return false;
- if (test_and_set_bit(cid, mm_cidmask(mm)))
- return false;
- t->mm_cid.cid = t->mm_cid.last_cid = cid;
- __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
- return true;
-}
-
-static inline bool mm_cid_get(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- unsigned int max_cids;
-
- max_cids = READ_ONCE(mm->mm_cid.max_cids);
-
- /* Try to reuse the last CID of this task */
- if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
- return true;
-
- /* Try to reuse the last CID of this mm on this CPU */
- if (__mm_cid_get(t, __this_cpu_read(mm->mm_cid.pcpu->cid), max_cids))
- return true;
-
- /* Try the first zero bit in the cidmask. */
- return __mm_cid_get(t, find_first_zero_bit(mm_cidmask(mm), num_possible_cpus()), max_cids);
-}
-
-static inline void mm_cid_select(struct task_struct *t)
-{
- /*
- * mm_cid_get() can fail when the maximum CID, which is determined
- * by min(mm->nr_cpus_allowed, mm->mm_users) changes concurrently.
- * That's a transient failure as there cannot be more tasks
- * concurrently on a CPU (or about to be scheduled in) than that.
- */
- for (;;) {
- if (mm_cid_get(t))
- break;
- }
-}
-
-static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
-{
- if (prev->mm_cid.active) {
- if (prev->mm_cid.cid != MM_CID_UNSET)
- clear_bit(prev->mm_cid.cid, mm_cidmask(prev->mm));
- prev->mm_cid.cid = MM_CID_UNSET;
- }
-
- if (next->mm_cid.active) {
- mm_cid_select(next);
- rseq_sched_set_task_mm_cid(next, next->mm_cid.cid);
- }
-}
-
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void mm_cid_select(struct task_struct *t) { }
-static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */Hi Thomas,
On Wed, Nov 19, 2025 at 06:27:22PM +0100, Thomas Gleixner wrote:
> Now that all pieces are in place, change the implementations of
> sched_mm_cid_fork() and sched_mm_cid_exit() to adhere to the new strict
> ownership scheme and switch context_switch() over to use the new
> mm_cid_schedin() functionality.
>
> The common case is that there is no mode change required, which makes
> fork() and exit() just update the user count and the constraints.
>
> In case that a new user would exceed the CID space limit the fork() context
> handles the transition to per CPU mode with mm::mm_cid::mutex held. exit()
> handles the transition back to per task mode when the user count drops
> below the switch back threshold. fork() might also be forced to handle a
> deferred switch back to per task mode, when a affinity change increased the
> number of allowed CPUs enough.
>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Our CI started seeing a hang in QEMU after getting to userspace when
booting with a single CPU (our default logic when using TCG instead of
KVM) that I bisected to this change as commit 2635fb0f0973
("sched/mmcid: Switch over to the new mechanism") in -next.
https://github.com/ClangBuiltLinux/continuous-integration2/actions/runs/19557323049/job/56003155890
$ make -skj"$(nproc)" ARCH=x86_64 CROSS_COMPILE=x86_64-linux- clean defconfig bzImage
$ curl -LSs https://github.com/ClangBuiltLinux/boot-utils/releases/download/20241120-044434/x86_64-rootfs.cpio.zst | zstd -d >rootfs.cpio
$ qemu-system-x86_64 \
-display none \
-nodefaults \
-M q35 \
-d unimp,guest_errors \
-append 'console=ttyS0 earlycon=uart8250,io,0x3f8' \
-kernel arch/x86/boot/bzImage \
-initrd rootfs.cpio \
-cpu host \
-enable-kvm \
-m 512m \
-smp 1 \
-serial mon:stdio
[ 0.000000] Linux version 6.18.0-rc4-00066-g2635fb0f0973 (nathan@ax162) (x86_64-linux-gcc (GCC) 15.2.0, GNU ld (GNU Binutils) 2.45) #1 SMP PREEMPT_DYNAMIC Fri Nov 21 17:30:21 MST 2025
...
[ 0.946720] Freeing unused kernel image (initmem) memory: 2812K
[ 0.947864] Write protecting the kernel read-only data: 28672k
[ 0.949257] Freeing unused kernel image (text/rodata gap) memory: 1540K
[ 0.950579] Freeing unused kernel image (rodata/data gap) memory: 760K
[ 0.991764] x86/mm: Checked W+X mappings: passed, no W+X pages found.
[ 0.993000] Run /init as init process
<hangs>
If I change '-smp 1' to '-smp 2', userspace runs properly. At the parent
change, this issue does not exist but it is obviously possible that this
change exposes a bug from earlier in the series, I did not test.
Cheers,
Nathan
# bad: [d724c6f85e80a23ed46b7ebc6e38b527c09d64f5] Add linux-next specific files for 20251121
# good: [fd95357fd8c6778ac7dea6c57a19b8b182b6e91f] Merge tag 'sched_ext-for-6.18-rc6-fixes-2' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext
git bisect start 'd724c6f85e80a23ed46b7ebc6e38b527c09d64f5' 'fd95357fd8c6778ac7dea6c57a19b8b182b6e91f'
# good: [d8d0238c7ea639aa8c9f8c76d089e684e283ea87] Merge branch 'libcrypto-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git
git bisect good d8d0238c7ea639aa8c9f8c76d089e684e283ea87
# good: [84abc3686f608efd4914a5b4d651307e32ea4000] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git
git bisect good 84abc3686f608efd4914a5b4d651307e32ea4000
# bad: [c709a6758d17f1f2bccadb18cc82bccecdf216ff] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git
git bisect bad c709a6758d17f1f2bccadb18cc82bccecdf216ff
# skip: [8d00858e9326dafe5e27072e3cee5c23b60e7281] Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git
git bisect skip 8d00858e9326dafe5e27072e3cee5c23b60e7281
# good: [8be00d1ba3a1305469f6747d0285c4496a6855ee] KVM: arm64: GICv3: Completely disable trapping on vcpu exit
git bisect good 8be00d1ba3a1305469f6747d0285c4496a6855ee
# good: [4daaead949b331c271f6df4162f39589088fe27b] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git
git bisect good 4daaead949b331c271f6df4162f39589088fe27b
# bad: [b1a2fa6d34e27225c2b3f79840f198227af1f53d] Merge branch into tip/master: 'x86/sev'
git bisect bad b1a2fa6d34e27225c2b3f79840f198227af1f53d
# bad: [468f14a4c5c9def982ae9a74344fa6f905a9b725] Merge branch into tip/master: 'locking/core'
git bisect bad 468f14a4c5c9def982ae9a74344fa6f905a9b725
# bad: [705d7cad382be9b3b52b89b220e77b17cf24322a] Merge branch 'sched/core' into core/rseq, to resolve conflict
git bisect bad 705d7cad382be9b3b52b89b220e77b17cf24322a
# good: [472931e757fb3dfad1f78ce6f5abd821155433b2] sched/mmcid: Convert mm CID mask to a bitmap
git bisect good 472931e757fb3dfad1f78ce6f5abd821155433b2
# good: [7f829bde94b1c97b1804fa5860e066ea49dbfca3] sched/core: Optimize core cookie matching check
git bisect good 7f829bde94b1c97b1804fa5860e066ea49dbfca3
# good: [d206fbad9328ddb68ebabd7cf7413392acd38081] sched/fair: Revert max_newidle_lb_cost bump
git bisect good d206fbad9328ddb68ebabd7cf7413392acd38081
# good: [340af997d25dab0f05c4de8399d656b112592a93] sched/mmcid: Provide CID ownership mode fixup functions
git bisect good 340af997d25dab0f05c4de8399d656b112592a93
# bad: [2635fb0f0973c57c45f03708d52e827ec99ac78e] sched/mmcid: Switch over to the new mechanism
git bisect bad 2635fb0f0973c57c45f03708d52e827ec99ac78e
# good: [2644779ec144d3e8cce5fed9623b47e70b3e0422] irqwork: Move data struct to a types header
git bisect good 2644779ec144d3e8cce5fed9623b47e70b3e0422
# good: [cba5e581161e379037a94f5a75d1a61bd1ccce3b] sched/mmcid: Implement deferred mode change
git bisect good cba5e581161e379037a94f5a75d1a61bd1ccce3b
# first bad commit: [2635fb0f0973c57c45f03708d52e827ec99ac78e] sched/mmcid: Switch over to the new mechanism
On Fri, Nov 21 2025 at 17:43, Nathan Chancellor wrote:
> Our CI started seeing a hang in QEMU after getting to userspace when
> booting with a single CPU (our default logic when using TCG instead of
> KVM) that I bisected to this change as commit 2635fb0f0973
> ("sched/mmcid: Switch over to the new mechanism") in -next.
>
> If I change '-smp 1' to '-smp 2', userspace runs properly. At the parent
> change, this issue does not exist but it is obviously possible that this
> change exposes a bug from earlier in the series, I did not test.
Duh. Never tested with smp 1 :(
Fix is below. Now let me stare at that num_possible_cpus() fallout.
Thanks,
tglx
----
Subject: sched/mmcid: Ensure that per CPU threshold is > 0
From: Thomas Gleixner <tglx@linutronix.de>
Date: Sat, 22 Nov 2025 15:54:39 +0100
When num_possible_cpus() == 1 then the calculation for the threshold to
switch back from per CPU mode to per task mode results in 0, which
indicates that the per CPU mode is disabled.
Ensure that the threshold is > 0 to prevent that.
Fixes: 340af997d25d ("sched/mmcid: Provide CID ownership mode fixup functions")
Reported-by: Nathan Chancellor <nathan@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Closes: https://lore.kernel.org/all/20251122004358.GB2682494@ax162
---
kernel/sched/core.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10364,7 +10364,8 @@ static inline unsigned int mm_cid_calc_p
unsigned int opt_cids;
opt_cids = min(mc->nr_cpus_allowed, mc->users);
- return min(opt_cids - opt_cids / 4, num_possible_cpus() / 2);
+ /* Has to be at least 1 because 0 indicates PCPU mode off */
+ return max(min(opt_cids - opt_cids / 4, num_possible_cpus() / 2), 1);
}
static bool mm_update_max_cids(struct mm_struct *mm)
On 11/22/25 8:32 PM, Thomas Gleixner wrote:
> On Fri, Nov 21 2025 at 17:43, Nathan Chancellor wrote:
>> Our CI started seeing a hang in QEMU after getting to userspace when
>> booting with a single CPU (our default logic when using TCG instead of
>> KVM) that I bisected to this change as commit 2635fb0f0973
>> ("sched/mmcid: Switch over to the new mechanism") in -next.
>>
>> If I change '-smp 1' to '-smp 2', userspace runs properly. At the parent
>> change, this issue does not exist but it is obviously possible that this
>> change exposes a bug from earlier in the series, I did not test.
>
> Duh. Never tested with smp 1 :(
>
> Fix is below. Now let me stare at that num_possible_cpus() fallout.
>
> Thanks,
>
> tglx
> ----
> Subject: sched/mmcid: Ensure that per CPU threshold is > 0
> From: Thomas Gleixner <tglx@linutronix.de>
> Date: Sat, 22 Nov 2025 15:54:39 +0100
>
> When num_possible_cpus() == 1 then the calculation for the threshold to
> switch back from per CPU mode to per task mode results in 0, which
> indicates that the per CPU mode is disabled.
>
> Ensure that the threshold is > 0 to prevent that.
>
> Fixes: 340af997d25d ("sched/mmcid: Provide CID ownership mode fixup functions")
> Reported-by: Nathan Chancellor <nathan@kernel.org>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Closes: https://lore.kernel.org/all/20251122004358.GB2682494@ax162
> ---
> kernel/sched/core.c | 3 ++-
> 1 file changed, 2 insertions(+), 1 deletion(-)
>
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -10364,7 +10364,8 @@ static inline unsigned int mm_cid_calc_p
> unsigned int opt_cids;
>
> opt_cids = min(mc->nr_cpus_allowed, mc->users);
> - return min(opt_cids - opt_cids / 4, num_possible_cpus() / 2);
> + /* Has to be at least 1 because 0 indicates PCPU mode off */
> + return max(min(opt_cids - opt_cids / 4, num_possible_cpus() / 2), 1);
> }
>
> static bool mm_update_max_cids(struct mm_struct *mm)
Having both patches (this patch and num_possible_cpus fix) patch was
necessary for successful boot of smp=1 on qemu.
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 14b2b17cca541bf46341beb912cf4420ff27a0c7
Gitweb: https://git.kernel.org/tip/14b2b17cca541bf46341beb912cf4420ff27a0c7
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Sat, 22 Nov 2025 15:54:39 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Sun, 23 Nov 2025 19:59:30 +01:00
sched/mmcid: Ensure that per CPU threshold is > 0
When num_possible_cpus() == 1 then the calculation for the threshold to
switch back from per CPU mode to per task mode results in 0, which
indicates that the per CPU mode is disabled.
Ensure that the threshold is > 0 to prevent that.
Fixes: 340af997d25d ("sched/mmcid: Provide CID ownership mode fixup functions")
Reported-by: Nathan Chancellor <nathan@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://patch.msgid.link/873466jekm.ffs@tglx
Closes: https://lore.kernel.org/all/20251122004358.GB2682494@ax162
---
kernel/sched/core.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 5821968..58b4dbe 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10364,7 +10364,8 @@ static inline unsigned int mm_cid_calc_pcpu_thrs(struct mm_mm_cid *mc)
unsigned int opt_cids;
opt_cids = min(mc->nr_cpus_allowed, mc->users);
- return min(opt_cids - opt_cids / 4, num_possible_cpus() / 2);
+ /* Has to be at least 1 because 0 indicates PCPU mode off */
+ return max(min(opt_cids - opt_cids / 4, num_possible_cpus() / 2), 1);
}
static bool mm_update_max_cids(struct mm_struct *mm)The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 653fda7ae73d8033dedb65537acac0c2c287dc3f
Gitweb: https://git.kernel.org/tip/653fda7ae73d8033dedb65537acac0c2c287dc3f
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:22 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:42 +01:00
sched/mmcid: Switch over to the new mechanism
Now that all pieces are in place, change the implementations of
sched_mm_cid_fork() and sched_mm_cid_exit() to adhere to the new strict
ownership scheme and switch context_switch() over to use the new
mm_cid_schedin() functionality.
The common case is that there is no mode change required, which makes
fork() and exit() just update the user count and the constraints.
In case that a new user would exceed the CID space limit the fork() context
handles the transition to per CPU mode with mm::mm_cid::mutex held. exit()
handles the transition back to per task mode when the user count drops
below the switch back threshold. fork() might also be forced to handle a
deferred switch back to per task mode, when a affinity change increased the
number of allowed CPUs enough.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.280380631@linutronix.de
---
include/linux/rseq.h | 19 +------
include/linux/rseq_types.h | 8 +--
kernel/fork.c | 1 +-
kernel/sched/core.c | 115 ++++++++++++++++++++++++++++++------
kernel/sched/sched.h | 76 +------------------------
5 files changed, 103 insertions(+), 116 deletions(-)
diff --git a/include/linux/rseq.h b/include/linux/rseq.h
index 4c0e8bd..2266f4d 100644
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -84,24 +84,6 @@ static __always_inline void rseq_sched_set_ids_changed(struct task_struct *t)
t->rseq.event.ids_changed = true;
}
-/*
- * Invoked from switch_mm_cid() in context switch when the task gets a MM
- * CID assigned.
- *
- * This does not raise TIF_NOTIFY_RESUME as that happens in
- * rseq_sched_switch_event().
- */
-static __always_inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid)
-{
- /*
- * Requires a comparison as the switch_mm_cid() code does not
- * provide a conditional for it readily. So avoid excessive updates
- * when nothing changes.
- */
- if (t->rseq.ids.mm_cid != cid)
- t->rseq.event.ids_changed = true;
-}
-
/* Enforce a full update after RSEQ registration and when execve() failed */
static inline void rseq_force_update(void)
{
@@ -169,7 +151,6 @@ static inline void rseq_handle_slowpath(struct pt_regs *regs) { }
static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }
static inline void rseq_sched_switch_event(struct task_struct *t) { }
static inline void rseq_sched_set_ids_changed(struct task_struct *t) { }
-static inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid) { }
static inline void rseq_force_update(void) { }
static inline void rseq_virt_userspace_exit(void) { }
static inline void rseq_fork(struct task_struct *t, u64 clone_flags) { }
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 81fbb88..332dc14 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -101,18 +101,18 @@ struct rseq_data { };
/**
* struct sched_mm_cid - Storage for per task MM CID data
* @active: MM CID is active for the task
- * @cid: The CID associated to the task
- * @last_cid: The last CID associated to the task
+ * @cid: The CID associated to the task either permanently or
+ * borrowed from the CPU
*/
struct sched_mm_cid {
unsigned int active;
unsigned int cid;
- unsigned int last_cid;
};
/**
* struct mm_cid_pcpu - Storage for per CPU MM_CID data
- * @cid: The CID associated to the CPU
+ * @cid: The CID associated to the CPU either permanently or
+ * while a task with a CID is running
*/
struct mm_cid_pcpu {
unsigned int cid;
diff --git a/kernel/fork.c b/kernel/fork.c
index 6c23219..8475958 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -956,7 +956,6 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
#ifdef CONFIG_SCHED_MM_CID
tsk->mm_cid.cid = MM_CID_UNSET;
- tsk->mm_cid.last_cid = MM_CID_UNSET;
tsk->mm_cid.active = 0;
#endif
return tsk;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index cbb543a..62235f1 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5307,7 +5307,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
}
}
- switch_mm_cid(prev, next);
+ mm_cid_switch_to(prev, next);
/*
* Tell rseq that the task was scheduled in. Must be after
@@ -10624,7 +10624,7 @@ static bool mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm
return true;
}
-static void __maybe_unused mm_cid_fixup_tasks_to_cpus(void)
+static void mm_cid_fixup_tasks_to_cpus(void)
{
struct mm_struct *mm = current->mm;
struct task_struct *p, *t;
@@ -10674,25 +10674,81 @@ static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm)
void sched_mm_cid_fork(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
+ bool percpu;
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
guard(mutex)(&mm->mm_cid.mutex);
- scoped_guard(raw_spinlock, &mm->mm_cid.lock) {
- sched_mm_cid_add_user(t, mm);
- /* Preset last_cid for mm_cid_select() */
- t->mm_cid.last_cid = mm->mm_cid.max_cids - 1;
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ struct mm_cid_pcpu *pcp = this_cpu_ptr(mm->mm_cid.pcpu);
+
+ /* First user ? */
+ if (!mm->mm_cid.users) {
+ sched_mm_cid_add_user(t, mm);
+ t->mm_cid.cid = mm_get_cid(mm);
+ /* Required for execve() */
+ pcp->cid = t->mm_cid.cid;
+ return;
+ }
+
+ if (!sched_mm_cid_add_user(t, mm)) {
+ if (!mm->mm_cid.percpu)
+ t->mm_cid.cid = mm_get_cid(mm);
+ return;
+ }
+
+ /* Handle the mode change and transfer current's CID */
+ percpu = !!mm->mm_cid.percpu;
+ if (!percpu)
+ mm_cid_transit_to_task(current, pcp);
+ else
+ mm_cid_transfer_to_cpu(current, pcp);
+ }
+
+ if (percpu) {
+ mm_cid_fixup_tasks_to_cpus();
+ } else {
+ mm_cid_fixup_cpus_to_tasks(mm);
+ t->mm_cid.cid = mm_get_cid(mm);
}
}
static bool sched_mm_cid_remove_user(struct task_struct *t)
{
t->mm_cid.active = 0;
- mm_unset_cid_on_task(t);
+ scoped_guard(preempt) {
+ /* Clear the transition bit */
+ t->mm_cid.cid = cid_from_transit_cid(t->mm_cid.cid);
+ mm_unset_cid_on_task(t);
+ }
t->mm->mm_cid.users--;
return mm_update_max_cids(t->mm);
}
+static bool __sched_mm_cid_exit(struct task_struct *t)
+{
+ struct mm_struct *mm = t->mm;
+
+ if (!sched_mm_cid_remove_user(t))
+ return false;
+ /*
+ * Contrary to fork() this only deals with a switch back to per
+ * task mode either because the above decreased users or an
+ * affinity change increased the number of allowed CPUs and the
+ * deferred fixup did not run yet.
+ */
+ if (WARN_ON_ONCE(mm->mm_cid.percpu))
+ return false;
+ /*
+ * A failed fork(2) cleanup never gets here, so @current must have
+ * the same MM as @t. That's true for exit() and the failed
+ * pthread_create() cleanup case.
+ */
+ if (WARN_ON_ONCE(current->mm != mm))
+ return false;
+ return true;
+}
+
/*
* When a task exits, the MM CID held by the task is not longer required as
* the task cannot return to user space.
@@ -10703,10 +10759,43 @@ void sched_mm_cid_exit(struct task_struct *t)
if (!mm || !t->mm_cid.active)
return;
+ /*
+ * Ensure that only one instance is doing MM CID operations within
+ * a MM. The common case is uncontended. The rare fixup case adds
+ * some overhead.
+ */
+ scoped_guard(mutex, &mm->mm_cid.mutex) {
+ /* mm_cid::mutex is sufficient to protect mm_cid::users */
+ if (likely(mm->mm_cid.users > 1)) {
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ if (!__sched_mm_cid_exit(t))
+ return;
+ /* Mode change required. Transfer currents CID */
+ mm_cid_transit_to_task(current, this_cpu_ptr(mm->mm_cid.pcpu));
+ }
+ mm_cid_fixup_cpus_to_tasks(mm);
+ return;
+ }
+ /* Last user */
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ /* Required across execve() */
+ if (t == current)
+ mm_cid_transit_to_task(t, this_cpu_ptr(mm->mm_cid.pcpu));
+ /* Ignore mode change. There is nothing to do. */
+ sched_mm_cid_remove_user(t);
+ }
+ }
- guard(mutex)(&mm->mm_cid.mutex);
- scoped_guard(raw_spinlock, &mm->mm_cid.lock)
- sched_mm_cid_remove_user(t);
+ /*
+ * As this is the last user (execve(), process exit or failed
+ * fork(2)) there is no concurrency anymore.
+ *
+ * Synchronize eventually pending work to ensure that there are no
+ * dangling references left. @t->mm_cid.users is zero so nothing
+ * can queue this work anymore.
+ */
+ irq_work_sync(&mm->mm_cid.irq_work);
+ cancel_work_sync(&mm->mm_cid.work);
}
/* Deactivate MM CID allocation across execve() */
@@ -10719,18 +10808,12 @@ void sched_mm_cid_before_execve(struct task_struct *t)
void sched_mm_cid_after_execve(struct task_struct *t)
{
sched_mm_cid_fork(t);
- guard(preempt)();
- mm_cid_select(t);
}
static void mm_cid_work_fn(struct work_struct *work)
{
struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
- /* Make it compile, but not functional yet */
- if (!IS_ENABLED(CONFIG_NEW_MM_CID))
- return;
-
guard(mutex)(&mm->mm_cid.mutex);
/* Did the last user task exit already? */
if (!mm->mm_cid.users)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 82c7978..f9d0515 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3745,83 +3745,7 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct
mm_cid_schedin(next);
}
-/* Active implementation */
-static inline void init_sched_mm_cid(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- unsigned int max_cid;
-
- if (!mm)
- return;
-
- /* Preset last_mm_cid */
- max_cid = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
- t->mm_cid.last_cid = max_cid - 1;
-}
-
-static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
-{
- struct mm_struct *mm = t->mm;
-
- if (cid >= max_cids)
- return false;
- if (test_and_set_bit(cid, mm_cidmask(mm)))
- return false;
- t->mm_cid.cid = t->mm_cid.last_cid = cid;
- __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
- return true;
-}
-
-static inline bool mm_cid_get(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- unsigned int max_cids;
-
- max_cids = READ_ONCE(mm->mm_cid.max_cids);
-
- /* Try to reuse the last CID of this task */
- if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
- return true;
-
- /* Try to reuse the last CID of this mm on this CPU */
- if (__mm_cid_get(t, __this_cpu_read(mm->mm_cid.pcpu->cid), max_cids))
- return true;
-
- /* Try the first zero bit in the cidmask. */
- return __mm_cid_get(t, find_first_zero_bit(mm_cidmask(mm), num_possible_cpus()), max_cids);
-}
-
-static inline void mm_cid_select(struct task_struct *t)
-{
- /*
- * mm_cid_get() can fail when the maximum CID, which is determined
- * by min(mm->nr_cpus_allowed, mm->mm_users) changes concurrently.
- * That's a transient failure as there cannot be more tasks
- * concurrently on a CPU (or about to be scheduled in) than that.
- */
- for (;;) {
- if (mm_cid_get(t))
- break;
- }
-}
-
-static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
-{
- if (prev->mm_cid.active) {
- if (prev->mm_cid.cid != MM_CID_UNSET)
- clear_bit(prev->mm_cid.cid, mm_cidmask(prev->mm));
- prev->mm_cid.cid = MM_CID_UNSET;
- }
-
- if (next->mm_cid.active) {
- mm_cid_select(next);
- rseq_sched_set_task_mm_cid(next, next->mm_cid.cid);
- }
-}
-
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void mm_cid_select(struct task_struct *t) { }
-static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 2635fb0f0973c57c45f03708d52e827ec99ac78e
Gitweb: https://git.kernel.org/tip/2635fb0f0973c57c45f03708d52e827ec99ac78e
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:22 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:58 +01:00
sched/mmcid: Switch over to the new mechanism
Now that all pieces are in place, change the implementations of
sched_mm_cid_fork() and sched_mm_cid_exit() to adhere to the new strict
ownership scheme and switch context_switch() over to use the new
mm_cid_schedin() functionality.
The common case is that there is no mode change required, which makes
fork() and exit() just update the user count and the constraints.
In case that a new user would exceed the CID space limit the fork() context
handles the transition to per CPU mode with mm::mm_cid::mutex held. exit()
handles the transition back to per task mode when the user count drops
below the switch back threshold. fork() might also be forced to handle a
deferred switch back to per task mode, when a affinity change increased the
number of allowed CPUs enough.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.280380631@linutronix.de
---
include/linux/rseq.h | 19 +------
include/linux/rseq_types.h | 8 +--
kernel/fork.c | 1 +-
kernel/sched/core.c | 115 ++++++++++++++++++++++++++++++------
kernel/sched/sched.h | 76 +------------------------
5 files changed, 103 insertions(+), 116 deletions(-)
diff --git a/include/linux/rseq.h b/include/linux/rseq.h
index 4c0e8bd..2266f4d 100644
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -84,24 +84,6 @@ static __always_inline void rseq_sched_set_ids_changed(struct task_struct *t)
t->rseq.event.ids_changed = true;
}
-/*
- * Invoked from switch_mm_cid() in context switch when the task gets a MM
- * CID assigned.
- *
- * This does not raise TIF_NOTIFY_RESUME as that happens in
- * rseq_sched_switch_event().
- */
-static __always_inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid)
-{
- /*
- * Requires a comparison as the switch_mm_cid() code does not
- * provide a conditional for it readily. So avoid excessive updates
- * when nothing changes.
- */
- if (t->rseq.ids.mm_cid != cid)
- t->rseq.event.ids_changed = true;
-}
-
/* Enforce a full update after RSEQ registration and when execve() failed */
static inline void rseq_force_update(void)
{
@@ -169,7 +151,6 @@ static inline void rseq_handle_slowpath(struct pt_regs *regs) { }
static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }
static inline void rseq_sched_switch_event(struct task_struct *t) { }
static inline void rseq_sched_set_ids_changed(struct task_struct *t) { }
-static inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid) { }
static inline void rseq_force_update(void) { }
static inline void rseq_virt_userspace_exit(void) { }
static inline void rseq_fork(struct task_struct *t, u64 clone_flags) { }
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 81fbb88..332dc14 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -101,18 +101,18 @@ struct rseq_data { };
/**
* struct sched_mm_cid - Storage for per task MM CID data
* @active: MM CID is active for the task
- * @cid: The CID associated to the task
- * @last_cid: The last CID associated to the task
+ * @cid: The CID associated to the task either permanently or
+ * borrowed from the CPU
*/
struct sched_mm_cid {
unsigned int active;
unsigned int cid;
- unsigned int last_cid;
};
/**
* struct mm_cid_pcpu - Storage for per CPU MM_CID data
- * @cid: The CID associated to the CPU
+ * @cid: The CID associated to the CPU either permanently or
+ * while a task with a CID is running
*/
struct mm_cid_pcpu {
unsigned int cid;
diff --git a/kernel/fork.c b/kernel/fork.c
index 6c23219..8475958 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -956,7 +956,6 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
#ifdef CONFIG_SCHED_MM_CID
tsk->mm_cid.cid = MM_CID_UNSET;
- tsk->mm_cid.last_cid = MM_CID_UNSET;
tsk->mm_cid.active = 0;
#endif
return tsk;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 938757e..72f368f 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5307,7 +5307,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
}
}
- switch_mm_cid(prev, next);
+ mm_cid_switch_to(prev, next);
/*
* Tell rseq that the task was scheduled in. Must be after
@@ -10623,7 +10623,7 @@ static bool mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm
return true;
}
-static void __maybe_unused mm_cid_fixup_tasks_to_cpus(void)
+static void mm_cid_fixup_tasks_to_cpus(void)
{
struct mm_struct *mm = current->mm;
struct task_struct *p, *t;
@@ -10673,25 +10673,81 @@ static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm)
void sched_mm_cid_fork(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
+ bool percpu;
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
guard(mutex)(&mm->mm_cid.mutex);
- scoped_guard(raw_spinlock, &mm->mm_cid.lock) {
- sched_mm_cid_add_user(t, mm);
- /* Preset last_cid for mm_cid_select() */
- t->mm_cid.last_cid = mm->mm_cid.max_cids - 1;
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ struct mm_cid_pcpu *pcp = this_cpu_ptr(mm->mm_cid.pcpu);
+
+ /* First user ? */
+ if (!mm->mm_cid.users) {
+ sched_mm_cid_add_user(t, mm);
+ t->mm_cid.cid = mm_get_cid(mm);
+ /* Required for execve() */
+ pcp->cid = t->mm_cid.cid;
+ return;
+ }
+
+ if (!sched_mm_cid_add_user(t, mm)) {
+ if (!mm->mm_cid.percpu)
+ t->mm_cid.cid = mm_get_cid(mm);
+ return;
+ }
+
+ /* Handle the mode change and transfer current's CID */
+ percpu = !!mm->mm_cid.percpu;
+ if (!percpu)
+ mm_cid_transit_to_task(current, pcp);
+ else
+ mm_cid_transfer_to_cpu(current, pcp);
+ }
+
+ if (percpu) {
+ mm_cid_fixup_tasks_to_cpus();
+ } else {
+ mm_cid_fixup_cpus_to_tasks(mm);
+ t->mm_cid.cid = mm_get_cid(mm);
}
}
static bool sched_mm_cid_remove_user(struct task_struct *t)
{
t->mm_cid.active = 0;
- mm_unset_cid_on_task(t);
+ scoped_guard(preempt) {
+ /* Clear the transition bit */
+ t->mm_cid.cid = cid_from_transit_cid(t->mm_cid.cid);
+ mm_unset_cid_on_task(t);
+ }
t->mm->mm_cid.users--;
return mm_update_max_cids(t->mm);
}
+static bool __sched_mm_cid_exit(struct task_struct *t)
+{
+ struct mm_struct *mm = t->mm;
+
+ if (!sched_mm_cid_remove_user(t))
+ return false;
+ /*
+ * Contrary to fork() this only deals with a switch back to per
+ * task mode either because the above decreased users or an
+ * affinity change increased the number of allowed CPUs and the
+ * deferred fixup did not run yet.
+ */
+ if (WARN_ON_ONCE(mm->mm_cid.percpu))
+ return false;
+ /*
+ * A failed fork(2) cleanup never gets here, so @current must have
+ * the same MM as @t. That's true for exit() and the failed
+ * pthread_create() cleanup case.
+ */
+ if (WARN_ON_ONCE(current->mm != mm))
+ return false;
+ return true;
+}
+
/*
* When a task exits, the MM CID held by the task is not longer required as
* the task cannot return to user space.
@@ -10702,10 +10758,43 @@ void sched_mm_cid_exit(struct task_struct *t)
if (!mm || !t->mm_cid.active)
return;
+ /*
+ * Ensure that only one instance is doing MM CID operations within
+ * a MM. The common case is uncontended. The rare fixup case adds
+ * some overhead.
+ */
+ scoped_guard(mutex, &mm->mm_cid.mutex) {
+ /* mm_cid::mutex is sufficient to protect mm_cid::users */
+ if (likely(mm->mm_cid.users > 1)) {
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ if (!__sched_mm_cid_exit(t))
+ return;
+ /* Mode change required. Transfer currents CID */
+ mm_cid_transit_to_task(current, this_cpu_ptr(mm->mm_cid.pcpu));
+ }
+ mm_cid_fixup_cpus_to_tasks(mm);
+ return;
+ }
+ /* Last user */
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ /* Required across execve() */
+ if (t == current)
+ mm_cid_transit_to_task(t, this_cpu_ptr(mm->mm_cid.pcpu));
+ /* Ignore mode change. There is nothing to do. */
+ sched_mm_cid_remove_user(t);
+ }
+ }
- guard(mutex)(&mm->mm_cid.mutex);
- scoped_guard(raw_spinlock, &mm->mm_cid.lock)
- sched_mm_cid_remove_user(t);
+ /*
+ * As this is the last user (execve(), process exit or failed
+ * fork(2)) there is no concurrency anymore.
+ *
+ * Synchronize eventually pending work to ensure that there are no
+ * dangling references left. @t->mm_cid.users is zero so nothing
+ * can queue this work anymore.
+ */
+ irq_work_sync(&mm->mm_cid.irq_work);
+ cancel_work_sync(&mm->mm_cid.work);
}
/* Deactivate MM CID allocation across execve() */
@@ -10718,18 +10807,12 @@ void sched_mm_cid_before_execve(struct task_struct *t)
void sched_mm_cid_after_execve(struct task_struct *t)
{
sched_mm_cid_fork(t);
- guard(preempt)();
- mm_cid_select(t);
}
static void mm_cid_work_fn(struct work_struct *work)
{
struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
- /* Make it compile, but not functional yet */
- if (!IS_ENABLED(CONFIG_NEW_MM_CID))
- return;
-
guard(mutex)(&mm->mm_cid.mutex);
/* Did the last user task exit already? */
if (!mm->mm_cid.users)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 82c7978..f9d0515 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3745,83 +3745,7 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct
mm_cid_schedin(next);
}
-/* Active implementation */
-static inline void init_sched_mm_cid(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- unsigned int max_cid;
-
- if (!mm)
- return;
-
- /* Preset last_mm_cid */
- max_cid = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
- t->mm_cid.last_cid = max_cid - 1;
-}
-
-static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
-{
- struct mm_struct *mm = t->mm;
-
- if (cid >= max_cids)
- return false;
- if (test_and_set_bit(cid, mm_cidmask(mm)))
- return false;
- t->mm_cid.cid = t->mm_cid.last_cid = cid;
- __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
- return true;
-}
-
-static inline bool mm_cid_get(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- unsigned int max_cids;
-
- max_cids = READ_ONCE(mm->mm_cid.max_cids);
-
- /* Try to reuse the last CID of this task */
- if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
- return true;
-
- /* Try to reuse the last CID of this mm on this CPU */
- if (__mm_cid_get(t, __this_cpu_read(mm->mm_cid.pcpu->cid), max_cids))
- return true;
-
- /* Try the first zero bit in the cidmask. */
- return __mm_cid_get(t, find_first_zero_bit(mm_cidmask(mm), num_possible_cpus()), max_cids);
-}
-
-static inline void mm_cid_select(struct task_struct *t)
-{
- /*
- * mm_cid_get() can fail when the maximum CID, which is determined
- * by min(mm->nr_cpus_allowed, mm->mm_users) changes concurrently.
- * That's a transient failure as there cannot be more tasks
- * concurrently on a CPU (or about to be scheduled in) than that.
- */
- for (;;) {
- if (mm_cid_get(t))
- break;
- }
-}
-
-static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
-{
- if (prev->mm_cid.active) {
- if (prev->mm_cid.cid != MM_CID_UNSET)
- clear_bit(prev->mm_cid.cid, mm_cidmask(prev->mm));
- prev->mm_cid.cid = MM_CID_UNSET;
- }
-
- if (next->mm_cid.active) {
- mm_cid_select(next);
- rseq_sched_set_task_mm_cid(next, next->mm_cid.cid);
- }
-}
-
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void mm_cid_select(struct task_struct *t) { }
-static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
© 2016 - 2025 Red Hat, Inc.