From: Zecheng Li <zecheng@google.com>

To remove the cfs_rq pointer array in task_group, allocate the combined
cfs_rq and sched_entity using the per-cpu allocator.

This patch implements the following:

- Changes task_group->cfs_rq from struct cfs_rq ** to struct cfs_rq
__percpu *.

- Updates memory allocation in alloc_fair_sched_group() and
free_fair_sched_group() to use alloc_percpu() and free_percpu()
respectively.

- Uses the inline accessor tg_cfs_rq(tg, cpu) with per_cpu_ptr() to
retrieve the pointer to cfs_rq for the given task group and CPU.

- Replaces direct accesses tg->cfs_rq[cpu] with calls to the new
tg_cfs_rq(tg, cpu) helper.

- Handles the root_task_group: since struct rq is already a per-cpu
variable (runqueues), its embedded cfs_rq (rq->cfs) is also per-cpu.
Therefore, we assign root_task_group.cfs_rq = &runqueues.cfs.

- Cleanup the code in initializing the root task group.

This change places each CPU's cfs_rq and sched_entity in its local
per-cpu memory area to remove the per-task_group pointer arrays.

Signed-off-by: Zecheng Li <zecheng@google.com>
Signed-off-by: Zecheng Li <zli94@ncsu.edu>
---
 kernel/sched/core.c  | 35 ++++++++++-----------------
 kernel/sched/fair.c  | 57 +++++++++++++++++---------------------------
 kernel/sched/sched.h | 14 +++++++----
 3 files changed, 45 insertions(+), 61 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 8e2a67cecee9..80e8f4eb3f87 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -8549,7 +8549,7 @@ static struct kmem_cache *task_group_cache __ro_after_init;
 
 void __init sched_init(void)
 {
-	unsigned long ptr = 0;
+	unsigned long __maybe_unused ptr = 0;
 	int i;
 
 	/* Make sure the linker didn't screw up */
@@ -8565,33 +8565,24 @@ void __init sched_init(void)
 	wait_bit_init();
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-	ptr += nr_cpu_ids * sizeof(void **);
-#endif
-#ifdef CONFIG_RT_GROUP_SCHED
-	ptr += 2 * nr_cpu_ids * sizeof(void **);
-#endif
-	if (ptr) {
-		ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
+	root_task_group.cfs_rq = &runqueues.cfs;
 
-#ifdef CONFIG_FAIR_GROUP_SCHED
-		root_task_group.cfs_rq = (struct cfs_rq **)ptr;
-		ptr += nr_cpu_ids * sizeof(void **);
-
-		root_task_group.shares = ROOT_TASK_GROUP_LOAD;
-		init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL);
+	root_task_group.shares = ROOT_TASK_GROUP_LOAD;
+	init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL);
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 #ifdef CONFIG_EXT_GROUP_SCHED
-		scx_tg_init(&root_task_group);
+	scx_tg_init(&root_task_group);
 #endif /* CONFIG_EXT_GROUP_SCHED */
 #ifdef CONFIG_RT_GROUP_SCHED
-		root_task_group.rt_se = (struct sched_rt_entity **)ptr;
-		ptr += nr_cpu_ids * sizeof(void **);
+	ptr += 2 * nr_cpu_ids * sizeof(void **);
+	ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
+	root_task_group.rt_se = (struct sched_rt_entity **)ptr;
+	ptr += nr_cpu_ids * sizeof(void **);
 
-		root_task_group.rt_rq = (struct rt_rq **)ptr;
-		ptr += nr_cpu_ids * sizeof(void **);
+	root_task_group.rt_rq = (struct rt_rq **)ptr;
+	ptr += nr_cpu_ids * sizeof(void **);
 
 #endif /* CONFIG_RT_GROUP_SCHED */
-	}
 
 	init_defrootdomain();
 
@@ -9492,7 +9483,7 @@ static int tg_set_cfs_bandwidth(struct task_group *tg,
 	}
 
 	for_each_online_cpu(i) {
-		struct cfs_rq *cfs_rq = tg->cfs_rq[i];
+		struct cfs_rq *cfs_rq = tg_cfs_rq(tg, i);
 		struct rq *rq = cfs_rq->rq;
 
 		guard(rq_lock_irq)(rq);
@@ -9660,7 +9651,7 @@ static u64 throttled_time_self(struct task_group *tg)
 	u64 total = 0;
 
 	for_each_possible_cpu(i) {
-		total += READ_ONCE(tg->cfs_rq[i]->throttled_clock_self_time);
+		total += READ_ONCE(tg_cfs_rq(tg, i)->throttled_clock_self_time);
 	}
 
 	return total;
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index e44bd5448fa5..bc023704acd1 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -327,7 +327,7 @@ static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
 	 * to a tree or when we reach the top of the tree
 	 */
 	if (cfs_rq->tg->parent &&
-	    cfs_rq->tg->parent->cfs_rq[cpu]->on_list) {
+	    tg_cfs_rq(cfs_rq->tg->parent, cpu)->on_list) {
 		/*
 		 * If parent is already on the list, we add the child
 		 * just before. Thanks to circular linked property of
@@ -335,7 +335,7 @@ static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
 		 * of the list that starts by parent.
 		 */
 		list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
-			&(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list));
+			&(tg_cfs_rq(cfs_rq->tg->parent, cpu)->leaf_cfs_rq_list));
 		/*
 		 * The branch is now connected to its tree so we can
 		 * reset tmp_alone_branch to the beginning of the
@@ -4153,7 +4153,7 @@ static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq)
 
 	rcu_read_lock();
 	list_for_each_entry_rcu(tg, &task_groups, list) {
-		struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
+		struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
 
 		clear_tg_load_avg(cfs_rq);
 	}
@@ -5689,7 +5689,7 @@ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
 
 static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu)
 {
-	return throttled_hierarchy(task_group(p)->cfs_rq[dst_cpu]);
+	return throttled_hierarchy(tg_cfs_rq(task_group(p), dst_cpu));
 }
 
 static inline bool task_is_throttled(struct task_struct *p)
@@ -5835,7 +5835,7 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags);
 static int tg_unthrottle_up(struct task_group *tg, void *data)
 {
 	struct rq *rq = data;
-	struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
+	struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
 	struct task_struct *p, *tmp;
 
 	if (--cfs_rq->throttle_count)
@@ -5906,7 +5906,7 @@ static void record_throttle_clock(struct cfs_rq *cfs_rq)
 static int tg_throttle_down(struct task_group *tg, void *data)
 {
 	struct rq *rq = data;
-	struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
+	struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
 
 	if (cfs_rq->throttle_count++)
 		return 0;
@@ -6379,8 +6379,8 @@ static void sync_throttle(struct task_group *tg, int cpu)
 	if (!tg->parent)
 		return;
 
-	cfs_rq = tg->cfs_rq[cpu];
-	pcfs_rq = tg->parent->cfs_rq[cpu];
+	cfs_rq = tg_cfs_rq(tg, cpu);
+	pcfs_rq = tg_cfs_rq(tg->parent, cpu);
 
 	cfs_rq->throttle_count = pcfs_rq->throttle_count;
 	cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu));
@@ -6572,7 +6572,7 @@ static void __maybe_unused update_runtime_enabled(struct rq *rq)
 	rcu_read_lock();
 	list_for_each_entry_rcu(tg, &task_groups, list) {
 		struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
-		struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
+		struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
 
 		raw_spin_lock(&cfs_b->lock);
 		cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
@@ -6601,7 +6601,7 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
 
 	rcu_read_lock();
 	list_for_each_entry_rcu(tg, &task_groups, list) {
-		struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
+		struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
 
 		if (!cfs_rq->runtime_enabled)
 			continue;
@@ -9408,7 +9408,7 @@ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_
 	struct cfs_rq *dst_cfs_rq;
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-	dst_cfs_rq = task_group(p)->cfs_rq[dest_cpu];
+	dst_cfs_rq = tg_cfs_rq(task_group(p), dest_cpu);
 #else
 	dst_cfs_rq = &cpu_rq(dest_cpu)->cfs;
 #endif
@@ -13346,7 +13346,7 @@ static int task_is_throttled_fair(struct task_struct *p, int cpu)
 	struct cfs_rq *cfs_rq;
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-	cfs_rq = task_group(p)->cfs_rq[cpu];
+	cfs_rq = tg_cfs_rq(task_group(p), cpu);
 #else
 	cfs_rq = &cpu_rq(cpu)->cfs;
 #endif
@@ -13612,42 +13612,31 @@ static void task_change_group_fair(struct task_struct *p)
 
 void free_fair_sched_group(struct task_group *tg)
 {
-	int i;
-
-	for_each_possible_cpu(i) {
-		if (tg->cfs_rq && tg->cfs_rq[i]) {
-			struct cfs_tg_state *state =
-				container_of(tg->cfs_rq[i], struct cfs_tg_state, cfs_rq);
-			kfree(state);
-		}
-	}
-
-	kfree(tg->cfs_rq);
+	free_percpu(tg->cfs_rq);
 }
 
 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
 {
-	struct cfs_tg_state *state;
+	struct cfs_tg_state __percpu *state;
 	struct sched_entity *se;
 	struct cfs_rq *cfs_rq;
 	int i;
 
-	tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL);
-	if (!tg->cfs_rq)
+	state = alloc_percpu_gfp(struct cfs_tg_state, GFP_KERNEL);
+	if (!state)
 		goto err;
 
+	tg->cfs_rq = &state->cfs_rq;
 	tg->shares = NICE_0_LOAD;
 
 	init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent));
 
 	for_each_possible_cpu(i) {
-		state = kzalloc_node(sizeof(*state),
-				     GFP_KERNEL, cpu_to_node(i));
-		if (!state)
+		cfs_rq = tg_cfs_rq(tg, i);
+		if (!cfs_rq)
 			goto err;
 
-		cfs_rq = &state->cfs_rq;
-		se = &state->se;
+		se = tg_se(tg, i);
 		init_cfs_rq(cfs_rq);
 		init_tg_cfs_entry(tg, cfs_rq, se, i, tg_se(parent, i));
 		init_entity_runnable_average(se);
@@ -13684,7 +13673,7 @@ void unregister_fair_sched_group(struct task_group *tg)
 	destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
 
 	for_each_possible_cpu(cpu) {
-		struct cfs_rq *cfs_rq = tg->cfs_rq[cpu];
+		struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu);
 		struct sched_entity *se = tg_se(tg, cpu);
 		struct rq *rq = cpu_rq(cpu);
 
@@ -13721,8 +13710,6 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
 	cfs_rq->rq = rq;
 	init_cfs_rq_runtime(cfs_rq);
 
-	tg->cfs_rq[cpu] = cfs_rq;
-
 	/* se could be NULL for root_task_group */
 	if (!se)
 		return;
@@ -13815,7 +13802,7 @@ int sched_group_set_idle(struct task_group *tg, long idle)
 	for_each_possible_cpu(i) {
 		struct rq *rq = cpu_rq(i);
 		struct sched_entity *se = tg_se(tg, i);
-		struct cfs_rq *grp_cfs_rq = tg->cfs_rq[i];
+		struct cfs_rq *grp_cfs_rq = tg_cfs_rq(tg, i);
 		bool was_idle = cfs_rq_is_idle(grp_cfs_rq);
 		long idle_task_delta;
 		struct rq_flags rf;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index bb6daf2c1e79..8ad1780edfe8 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -477,7 +477,7 @@ struct task_group {
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
 	/* runqueue "owned" by this group on each CPU */
-	struct cfs_rq		**cfs_rq;
+	struct cfs_rq __percpu	*cfs_rq;
 	unsigned long		shares;
 	/*
 	 * load_avg can be heavily contended at clock tick time, so put
@@ -2201,13 +2201,19 @@ struct cfs_tg_state {
 	struct sched_statistics	stats;
 } __no_randomize_layout;
 
+/* Access a specific CPU's cfs_rq from a task group */
+static inline struct cfs_rq *tg_cfs_rq(struct task_group *tg, int cpu)
+{
+	return per_cpu_ptr(tg->cfs_rq, cpu);
+}
+
 static inline struct sched_entity *tg_se(struct task_group *tg, int cpu)
 {
 	if (is_root_task_group(tg))
 		return NULL;
 
 	struct cfs_tg_state *state =
-		container_of(tg->cfs_rq[cpu], struct cfs_tg_state, cfs_rq);
+		container_of(tg_cfs_rq(tg, cpu), struct cfs_tg_state, cfs_rq);
 	return &state->se;
 }
 
@@ -2230,8 +2236,8 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
 #endif
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-	set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]);
-	p->se.cfs_rq = tg->cfs_rq[cpu];
+	set_task_rq_fair(&p->se, p->se.cfs_rq, tg_cfs_rq(tg, cpu));
+	p->se.cfs_rq = tg_cfs_rq(tg, cpu);
 	p->se.parent = tg_se(tg, cpu);
 	p->se.depth = p->se.parent ? p->se.parent->depth + 1 : 0;
 #endif
-- 
2.52.0

Hello Zecheng,

On 1/18/2026 9:04 AM, Zecheng Li wrote:
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -8549,7 +8549,7 @@ static struct kmem_cache *task_group_cache __ro_after_init;
>  
>  void __init sched_init(void)
>  {
> -	unsigned long ptr = 0;
> +	unsigned long __maybe_unused ptr = 0;

Since "ptr" is now only used for CONFIG_RT_GROUP_SCHED ...

>  	int i;
>  
>  	/* Make sure the linker didn't screw up */
> @@ -8565,33 +8565,24 @@ void __init sched_init(void)
>  	wait_bit_init();
>  
>  #ifdef CONFIG_FAIR_GROUP_SCHED
> -	ptr += nr_cpu_ids * sizeof(void **);
> -#endif
> -#ifdef CONFIG_RT_GROUP_SCHED
> -	ptr += 2 * nr_cpu_ids * sizeof(void **);
> -#endif
> -	if (ptr) {
> -		ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
> +	root_task_group.cfs_rq = &runqueues.cfs;
>  
> -#ifdef CONFIG_FAIR_GROUP_SCHED
> -		root_task_group.cfs_rq = (struct cfs_rq **)ptr;
> -		ptr += nr_cpu_ids * sizeof(void **);
> -
> -		root_task_group.shares = ROOT_TASK_GROUP_LOAD;
> -		init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL);
> +	root_task_group.shares = ROOT_TASK_GROUP_LOAD;
> +	init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL);
>  #endif /* CONFIG_FAIR_GROUP_SCHED */
>  #ifdef CONFIG_EXT_GROUP_SCHED
> -		scx_tg_init(&root_task_group);
> +	scx_tg_init(&root_task_group);
>  #endif /* CONFIG_EXT_GROUP_SCHED */
>  #ifdef CONFIG_RT_GROUP_SCHED
> -		root_task_group.rt_se = (struct sched_rt_entity **)ptr;
> -		ptr += nr_cpu_ids * sizeof(void **);
> +	ptr += 2 * nr_cpu_ids * sizeof(void **);
> +	ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
> +	root_task_group.rt_se = (struct sched_rt_entity **)ptr;
> +	ptr += nr_cpu_ids * sizeof(void **);
>  
> -		root_task_group.rt_rq = (struct rt_rq **)ptr;
> -		ptr += nr_cpu_ids * sizeof(void **);
> +	root_task_group.rt_rq = (struct rt_rq **)ptr;
> +	ptr += nr_cpu_ids * sizeof(void **);

Can we also optimize the CONFIG_RT_GROUP_SCHED stuff in the same way
although I'm assuming the benefit is far less since they aren't
accessed as frequently as the cfs bits. Something like:

(Only build and boot tested on top of your series)

diff --git a/kernel/sched/autogroup.c b/kernel/sched/autogroup.c
index 954137775f38..7135f19d5fa2 100644
--- a/kernel/sched/autogroup.c
+++ b/kernel/sched/autogroup.c
@@ -52,7 +52,6 @@ static inline void autogroup_destroy(struct kref *kref)
 
 #ifdef CONFIG_RT_GROUP_SCHED
 	/* We've redirected RT tasks to the root task group... */
-	ag->tg->rt_se = NULL;
 	ag->tg->rt_rq = NULL;
 #endif
 	sched_release_group(ag->tg);
@@ -109,7 +108,6 @@ static inline struct autogroup *autogroup_create(void)
 	 * the policy change to proceed.
 	 */
 	free_rt_sched_group(tg);
-	tg->rt_se = root_task_group.rt_se;
 	tg->rt_rq = root_task_group.rt_rq;
 #endif /* CONFIG_RT_GROUP_SCHED */
 	tg->autogroup = ag;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 80e8f4eb3f87..cad8e8a1f519 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -8549,7 +8549,6 @@ static struct kmem_cache *task_group_cache __ro_after_init;
 
 void __init sched_init(void)
 {
-	unsigned long __maybe_unused ptr = 0;
 	int i;
 
 	/* Make sure the linker didn't screw up */
@@ -8574,14 +8573,7 @@ void __init sched_init(void)
 	scx_tg_init(&root_task_group);
 #endif /* CONFIG_EXT_GROUP_SCHED */
 #ifdef CONFIG_RT_GROUP_SCHED
-	ptr += 2 * nr_cpu_ids * sizeof(void **);
-	ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
-	root_task_group.rt_se = (struct sched_rt_entity **)ptr;
-	ptr += nr_cpu_ids * sizeof(void **);
-
-	root_task_group.rt_rq = (struct rt_rq **)ptr;
-	ptr += nr_cpu_ids * sizeof(void **);
-
+	root_task_group.rt_rq = &runqueues.rt;
 #endif /* CONFIG_RT_GROUP_SCHED */
 
 	init_defrootdomain();
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 0a9b2cd6da72..963004905a7d 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -201,26 +201,16 @@ void unregister_rt_sched_group(struct task_group *tg)
 	if (!rt_group_sched_enabled())
 		return;
 
-	if (tg->rt_se)
+	if (!is_root_task_group(tg))
 		destroy_rt_bandwidth(&tg->rt_bandwidth);
 }
 
 void free_rt_sched_group(struct task_group *tg)
 {
-	int i;
-
 	if (!rt_group_sched_enabled())
 		return;
 
-	for_each_possible_cpu(i) {
-		if (tg->rt_rq)
-			kfree(tg->rt_rq[i]);
-		if (tg->rt_se)
-			kfree(tg->rt_se[i]);
-	}
-
-	kfree(tg->rt_rq);
-	kfree(tg->rt_se);
+	free_percpu(tg->rt_rq);
 }
 
 void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
@@ -234,9 +224,6 @@ void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
 	rt_rq->rq = rq;
 	rt_rq->tg = tg;
 
-	tg->rt_rq[cpu] = rt_rq;
-	tg->rt_se[cpu] = rt_se;
-
 	if (!rt_se)
 		return;
 
@@ -252,42 +239,32 @@ void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
 
 int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
 {
-	struct rt_rq *rt_rq;
+	struct rt_tg_state __percpu *state;
 	struct sched_rt_entity *rt_se;
+	struct rt_rq *rt_rq;
 	int i;
 
 	if (!rt_group_sched_enabled())
 		return 1;
 
-	tg->rt_rq = kcalloc(nr_cpu_ids, sizeof(rt_rq), GFP_KERNEL);
-	if (!tg->rt_rq)
-		goto err;
-	tg->rt_se = kcalloc(nr_cpu_ids, sizeof(rt_se), GFP_KERNEL);
-	if (!tg->rt_se)
+	state = alloc_percpu_gfp(struct rt_tg_state, GFP_KERNEL);
+	if (!state)
 		goto err;
 
+	tg->rt_rq = &state->rt_rq;
 	init_rt_bandwidth(&tg->rt_bandwidth, ktime_to_ns(global_rt_period()), 0);
 
 	for_each_possible_cpu(i) {
-		rt_rq = kzalloc_node(sizeof(struct rt_rq),
-				     GFP_KERNEL, cpu_to_node(i));
-		if (!rt_rq)
-			goto err;
-
-		rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
-				     GFP_KERNEL, cpu_to_node(i));
-		if (!rt_se)
-			goto err_free_rq;
+		rt_rq = tg_rt_rq(tg, i);
+		rt_se = rt_rq_se(rt_rq);
 
 		init_rt_rq(rt_rq);
 		rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
-		init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);
+		init_tg_rt_entry(tg, rt_rq, rt_se, i, tg_rt_se(parent, i));
 	}
 
 	return 1;
 
-err_free_rq:
-	kfree(rt_rq);
 err:
 	return 0;
 }
@@ -510,7 +487,7 @@ static inline struct task_group *next_task_group(struct task_group *tg)
 
 #define for_each_rt_rq(rt_rq, iter, rq)					\
 	for (iter = &root_task_group;					\
-		iter && (rt_rq = iter->rt_rq[cpu_of(rq)]);		\
+		iter && (rt_rq = tg_rt_rq(iter, cpu_of(rq)));		\
 		iter = next_task_group(iter))
 
 #define for_each_sched_rt_entity(rt_se) \
@@ -528,11 +505,7 @@ static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
 {
 	struct task_struct *donor = rq_of_rt_rq(rt_rq)->donor;
 	struct rq *rq = rq_of_rt_rq(rt_rq);
-	struct sched_rt_entity *rt_se;
-
-	int cpu = cpu_of(rq);
-
-	rt_se = rt_rq->tg->rt_se[cpu];
+	struct sched_rt_entity *rt_se = rt_rq_se(rt_rq);
 
 	if (rt_rq->rt_nr_running) {
 		if (!rt_se)
@@ -547,10 +520,7 @@ static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
 
 static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
 {
-	struct sched_rt_entity *rt_se;
-	int cpu = cpu_of(rq_of_rt_rq(rt_rq));
-
-	rt_se = rt_rq->tg->rt_se[cpu];
+	struct sched_rt_entity *rt_se = rt_rq_se(rt_rq);
 
 	if (!rt_se) {
 		dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);
@@ -586,7 +556,7 @@ static inline const struct cpumask *sched_rt_period_mask(void)
 static inline
 struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)
 {
-	return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];
+	return tg_rt_rq(container_of(rt_b, struct task_group, rt_bandwidth), cpu);
 }
 
 static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
@@ -2563,7 +2533,7 @@ static int task_is_throttled_rt(struct task_struct *p, int cpu)
 	struct rt_rq *rt_rq;
 
 #ifdef CONFIG_RT_GROUP_SCHED // XXX maybe add task_rt_rq(), see also sched_rt_period_rt_rq
-	rt_rq = task_group(p)->rt_rq[cpu];
+	rt_rq = tg_rt_rq(task_group(p), cpu);
 	WARN_ON(!rt_group_sched_enabled() && rt_rq->tg != &root_task_group);
 #else
 	rt_rq = &cpu_rq(cpu)->rt;
@@ -2752,7 +2722,7 @@ static int tg_set_rt_bandwidth(struct task_group *tg,
 	tg->rt_bandwidth.rt_runtime = rt_runtime;
 
 	for_each_possible_cpu(i) {
-		struct rt_rq *rt_rq = tg->rt_rq[i];
+		struct rt_rq *rt_rq = tg_rt_rq(tg, i);
 
 		raw_spin_lock(&rt_rq->rt_runtime_lock);
 		rt_rq->rt_runtime = rt_runtime;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 8ad1780edfe8..62d7c89c74d8 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -488,8 +488,7 @@ struct task_group {
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #ifdef CONFIG_RT_GROUP_SCHED
-	struct sched_rt_entity	**rt_se;
-	struct rt_rq		**rt_rq;
+	struct rt_rq __percpu	*rt_rq;
 
 	struct rt_bandwidth	rt_bandwidth;
 #endif
@@ -2228,6 +2227,41 @@ static inline struct sched_entity *cfs_rq_se(struct cfs_rq *cfs_rq)
 }
 #endif
 
+#ifdef CONFIG_RT_GROUP_SCHED
+struct rt_tg_state {
+	struct rt_rq		rt_rq;
+	struct sched_rt_entity	rt_se;
+} __no_randomize_layout;
+
+/* Access a specific CPU's rt_rq from a task group */
+static inline struct rt_rq *tg_rt_rq(struct task_group *tg, int cpu)
+{
+	return per_cpu_ptr(tg->rt_rq, cpu);
+}
+
+static inline struct sched_rt_entity *tg_rt_se(struct task_group *tg, int cpu)
+{
+	struct rt_tg_state *state;
+
+	if (is_root_task_group(tg))
+		return NULL;
+
+	state = container_of(tg_rt_rq(tg, cpu), struct rt_tg_state, rt_rq);
+	return &state->rt_se;
+}
+
+static inline struct sched_rt_entity *rt_rq_se(struct rt_rq *rt_rq)
+{
+	struct rt_tg_state *state;
+
+	if (is_root_task_group(rt_rq->tg))
+		return NULL;
+
+	state = container_of(rt_rq, struct rt_tg_state, rt_rq);
+	return &state->rt_se;
+}
+#endif /* CONFIG_RT_GROUP_SCHED */
+
 /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
 static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
 {
@@ -2250,8 +2284,8 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
 	 */
 	if (!rt_group_sched_enabled())
 		tg = &root_task_group;
-	p->rt.rt_rq  = tg->rt_rq[cpu];
-	p->rt.parent = tg->rt_se[cpu];
+	p->rt.rt_rq  = tg_rt_rq(tg, cpu);
+	p->rt.parent = tg_rt_se(tg, cpu);
 #endif /* CONFIG_RT_GROUP_SCHED */
 }
 
-- 
Thanks and Regards,
Prateek

Hi Prateek,

On Mon, Jan 19, 2026 at 2:17 AM K Prateek Nayak <kprateek.nayak@amd.com> wrote:
>
> Can we also optimize the CONFIG_RT_GROUP_SCHED stuff in the same way
> although I'm assuming the benefit is far less since they aren't
> accessed as frequently as the cfs bits. Something like:
> ...

Thanks for the review. For patch 1 and 2, I'll fix those in v8.

For the RT structs optimization, I agree this would be a nice
improvement for consistency. I'd like to propose handling it as a
follow-up series after this CFS series. I don't currently have a good
test environment for RT workloads to validate performance impact.

Thanks,
Zecheng