The slow path for waking tasks currently discards all potential targets
when no cookie-matching CPU is found, leading to suboptimal task placement.
Fall back to selecting the idlest CPU when no cookie-matching target is
available, ensuring better CPU utilization while maintaining the preference
for cookie-compatible placements.
Signed-off-by: Fernand Sieber <sieberf@amazon.com>
---
kernel/sched/fair.c | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 43ddfc25af99..67746899809e 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -7335,7 +7335,8 @@ sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int
* sched_balance_find_dst_group_cpu - find the idlest CPU among the CPUs in the group.
*/
static int
-sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
+__sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *p,
+ int this_cpu, bool cookie_match)
{
unsigned long load, min_load = ULONG_MAX;
unsigned int min_exit_latency = UINT_MAX;
@@ -7352,7 +7353,8 @@ sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *
for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) {
struct rq *rq = cpu_rq(i);
- if (!sched_core_cookie_match(rq, p))
+ /* Only matching tasks if cookie_match, else only unmatching tasks */
+ if (cookie_match ^ sched_core_cookie_match(rq, p))
continue;
if (sched_idle_cpu(i))
@@ -7391,6 +7393,17 @@ sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *
return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu;
}
+/*
+ * sched_balance_find_dst_group_cpu - find the idlest CPU among the CPUs in the group.
+ */
+static inline int
+sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
+{
+ int cpu = __sched_balance_find_dst_group_cpu(group, p, this_cpu, true);
+
+ return cpu >= 0 ? cpu : __sched_balance_find_dst_group_cpu(group, p, this_cpu, false);
+}
+
static inline int sched_balance_find_dst_cpu(struct sched_domain *sd, struct task_struct *p,
int cpu, int prev_cpu, int sd_flag)
{
--
2.43.0
Amazon Development Centre (South Africa) (Proprietary) Limited
29 Gogosoa Street, Observatory, Cape Town, Western Cape, 7925, South Africa
Registration Number: 2004 / 034463 / 07Hello Fernand,
On 9/22/2025 6:09 PM, Fernand Sieber wrote:
> @@ -7391,6 +7393,17 @@ sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *
> return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu;
Based on the above return in __sched_balance_find_dst_group_cpu(), it
should always return a valid CPU since "least_loaded_cpu" is initialized
to "this_cpu".
> }
>
> +/*
> + * sched_balance_find_dst_group_cpu - find the idlest CPU among the CPUs in the group.
> + */
> +static inline int
> +sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
> +{
> + int cpu = __sched_balance_find_dst_group_cpu(group, p, this_cpu, true);
> +
> + return cpu >= 0 ? cpu : __sched_balance_find_dst_group_cpu(group, p, this_cpu, false);
So, under what circumstance does "cpu" here turns out to be < 0?
Am I missing something?
> +}
> +
> static inline int sched_balance_find_dst_cpu(struct sched_domain *sd, struct task_struct *p,
> int cpu, int prev_cpu, int sd_flag)
> {
--
Thanks and Regards,
Prateek
Hi Prateek,
On 9/23/2025 7:21 AM, K Prateek Nayak wrote:
> Based on the above return in __sched_balance_find_dst_group_cpu(), it
> should always return a valid CPU since "least_loaded_cpu" is initialized
> to "this_cpu".
>
> So, under what circumstance does "cpu" here turns out to be < 0?
> Am I missing something?
Hey Prateek. Thanks for the catch. I'll fix as follows for next rev:
+/*
+ * sched_balance_find_dst_group_cpu - find the idlest CPU among the CPUs in the group.
+ */
+static inline int
+sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
+{
+ int cpu = __sched_balance_find_dst_group_cpu(group, p, -1, true);
+ return cpu >= 0 ? cpu : __sched_balance_find_dst_group_cpu(group, p, this_cpu, false);
+}
+
Thanks,
Fernand
Amazon Development Centre (South Africa) (Proprietary) Limited
29 Gogosoa Street, Observatory, Cape Town, Western Cape, 7925, South Africa
Registration Number: 2004 / 034463 / 07
On 9/23/2025 9:32 AM, Fernand Sieber wrote:
> Hey Prateek. Thanks for the catch. I'll fix as follows for next rev:
>
> +/*
> + * sched_balance_find_dst_group_cpu - find the idlest CPU among the CPUs in the group.
> + */
> +static inline int
> +sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
> +{
> + int cpu = __sched_balance_find_dst_group_cpu(group, p, -1, true);
> + return cpu >= 0 ? cpu : __sched_balance_find_dst_group_cpu(group, p, this_cpu, false);
> +}
On second thoughts, this doesn't work as it breaks the original fallback path.
A better modification would be this:
@@ -7341,7 +7341,7 @@ __sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct
unsigned long load, min_load = ULONG_MAX;
unsigned int min_exit_latency = UINT_MAX;
u64 latest_idle_timestamp = 0;
- int least_loaded_cpu = this_cpu;
+ int least_loaded_cpu = -1;
int shallowest_idle_cpu = -1;
int i;
@@ -7357,6 +7357,9 @@ __sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct
if (cookie_match ^ sched_core_cookie_match(rq, p))
continue;
+ if (least_loaded_cpu < 0)
+ least_loaded_cpu = this_cpu;
+
if (sched_idle_cpu(i))
return i;
Thanks,
Fernand
Amazon Development Centre (South Africa) (Proprietary) Limited
29 Gogosoa Street, Observatory, Cape Town, Western Cape, 7925, South Africa
Registration Number: 2004 / 034463 / 07
© 2016 - 2026 Red Hat, Inc.