From nobody Sat Sep 26 22:55:39 2026 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4868340B0E9 for ; Fri, 28 Aug 2026 10:41:32 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1787913697; cv=none; b=mrJKsyhq7IIqdrIuyqAeO6xGTxKSOeQIficbG6H6b32hRM+/hx0iboMYQqOyXo9DnXUMWmbb5mI3DNzAKrAqp+uV8BNlrF8TxiWH/nQQsINS1RKF1SYl69Me/7PtrekKT/GHwaMcZ+PtDTeRmaXnS/r0j1ksYOWILJkHoGV/Tys= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1787913697; c=relaxed/simple; bh=dFrvw7iTZKmBzwV15Z42JzPl17IL7wCOJV+mM/6ClzE=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=msky4EsGohylGr6KqlGULtzLTLobCLTD8KUSCXS+u7UjdZqlyieRNjss/K8uUapXe0wIDa0oEsBN3aS7QN/l3x5PPKoQPOWyNidxqzHJ5H4ClIcdH1UC3c1sj3tdGZa+beOOnUgw5uO83wveaLO0FwaU52TU5gcZAG+Wo1wQGfE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=infradead.org; spf=pass smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=vlVJ5sm9; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="vlVJ5sm9" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=gTNkLy3rzRLHf/Y7wVAVEU5GaJWvM4O0ubkGWvEOWH0=; b=vlVJ5sm9iNp72Nr+n07hW3uG3l GLQVkPhOkoCFkd/zjrNB9Kygwc0bxp83QEOlggthTQEp0R1G3lT6DQ2/ZHYxE50dnE/5L+kR9FHLQ VY9vz5lbNvpjoyHW5xTG76Yte0UAtGYmbxriCzZs2d8I6KNo7qK41loRYVng5d/l8DsbYTaYQa8h3 oUx7su7K71eKqJrypbo53c4A0aneIGuSG5WkGyEFwKB2My5wgJbKQ9Fft59xymd6qFO+XCsHIn9M8 cvZFtzv+a+ykODKR78VCg7eEzLDQr+srhMb0i8shBAwlEC76vpLs8tCTNw3blE8cs6lT7pAaHb2YX bCN6JCNg==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.99.1 #2 (Red Hat Linux)) id 1wzu1S-0000000EWAf-43jM; Fri, 28 Aug 2026 10:41:23 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id D31FC30116A; Fri, 28 Aug 2026 12:41:21 +0200 (CEST) Message-ID: <20260828104018.378378994@infradead.org> User-Agent: quilt/0.68 Date: Fri, 28 Aug 2026 12:17:00 +0200 From: Peter Zijlstra To: mingo@kernel.org Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, kprateek.nayak@amd.com, linux-kernel@vger.kernel.org, tj@kernel.org Subject: [PATCH 1/7] sched/core: Fix pick_next_task() self recursion References: <20260828101659.812011872@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" It is possible for another sibling to end up in pick_next_task() when: pick_next_task() pick_task() sched_class::pick_task() drops the core wide rq->lock. In this case they end up trampling the core w= ide task selection state, possibly leading to NULL derefs. Detect this case by keeping a local copy of core_task_seq, a value that is incremented on {en,de}queue and schedule. Since RETRY_TASK is only possible when a higher priority task gets enqueued during the lock break, this must mean core_task_seq will also be incremented and is thus completely covered by the seq number mismatch. XXX words on forward progress go here Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/core.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6232,14 +6232,14 @@ static struct task_struct * pick_next_task(struct rq *rq, struct rq_flags *rf) __must_hold(__rq_lockp(rq)) { + bool core_clock_updated =3D (rq =3D=3D rq->core); struct task_struct *next, *p, *max; const struct cpumask *smt_mask; + int i, cpu, seq, occ =3D 0; bool fi_before =3D false; - bool core_clock_updated =3D (rq =3D=3D rq->core); + bool need_sync =3D false; unsigned long cookie; - int i, cpu, occ =3D 0; struct rq *rq_i; - bool need_sync =3D false; =20 if (!sched_core_enabled(rq)) return __pick_next_task(rq, rf); @@ -6314,7 +6314,7 @@ pick_next_task(struct rq *rq, struct rq_ * However, preemptions can cause multiple picks on the same task set. * 'Fix' this by also increasing @task_seq for every pick. */ - rq->core->core_task_seq++; + seq =3D ++rq->core->core_task_seq; =20 /* * Optimize for common case where this CPU has no cookies @@ -6362,7 +6362,8 @@ pick_next_task(struct rq *rq, struct rq_ update_rq_clock(rq_i); =20 p =3D pick_task(rq_i, rf); - if (unlikely(p =3D=3D RETRY_TASK)) { + if (unlikely(seq !=3D rq->core->core_task_seq || + WARN_ON_ONCE(p =3D=3D RETRY_TASK))) { /* rq lock may have been dropped, clocks invalidated */ core_clock_updated =3D false; if (!(rq->clock_update_flags & RQCF_UPDATED)) @@ -6392,7 +6393,7 @@ pick_next_task(struct rq *rq, struct rq_ if (cookie) p =3D sched_core_find(rq_i, cookie); if (!p) - p =3D idle_sched_class.pick_task(rq_i, rf); + p =3D idle_sched_class.pick_task(rq_i, NULL); } =20 rq_i->core_pick =3D p; From nobody Sat Sep 26 22:55:39 2026 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 482B740B0E0 for ; Fri, 28 Aug 2026 10:41:32 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1787913698; cv=none; b=Ay4Z3iNwKrsuE4bWia/BFD+8u5R/xhJDIobqmRlw0pGdW9/k86InF38PNHFw7k6l+YhD6Rjj28YxstoY/QmRETR+fhM2I0XeWm051uKyezkfQQLXuKSJ2CxjafnZpTku10VHhH+WP+lU6OrTPZSnkN2JBGYE8LJgqXJ2o/5CfAg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1787913698; c=relaxed/simple; bh=G789OFSfy/zeFOuHjvxGhSXQbcu+r91cSrKTd2413RU=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=dulb/QPdwe5tBydToEpaOBFW0UlieCgUH96/G/gk5ZUzVsbGHVtUzqy6QEgcb9W6NsdCGHIqfG+Qs6qmOTmp43IsCLpjrZamo+PHVkkqypk3mEjqLmHcgX/Kl/V3X6pHFB6Bj/GYYk24INitQZR4lud77xEMOR8VHAWQKrMY8Kg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=infradead.org; spf=pass smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=pyfNR8EJ; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="pyfNR8EJ" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=yJF0pF69jvkgkwzXDva8zPof6zLxO8fmfRsAa0Wb640=; b=pyfNR8EJHoXm+j81EThV0X+Ia6 zNCPX3FHMM3PHafl5mQbGJguSzpfKhyuny7TeImyx5BBOLcOpS8dM/fvTJnhjrqmr4PXfViGXRk/w l5WLgUz1msN9dj5kcyUd4mcXmVgVUIcastvo3na2sqoqarIOnqEz9eSA4AGz2gKsawCCq0AEHcEr+ wfUXlY3ruNz3R2VrhfbKjSn1Y8F+UWSJPG+gTceSDK/6JtHqgB63I8Ai+zpuaZ2Hl9VD2PFoOEq97 zIXHzGLHWr4FHu7xnSP9Ro4FFcL8VqZjjqGwLac4HdLEgHxzO7JWMhy1c1sb6hfu9y3rQjHtP3aMs 0UgzV9gg==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.99.1 #2 (Red Hat Linux)) id 1wzu1S-0000000EWAg-43j3; Fri, 28 Aug 2026 10:41:23 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id D6ED8303046; Fri, 28 Aug 2026 12:41:21 +0200 (CEST) Message-ID: <20260828104018.483560652@infradead.org> User-Agent: quilt/0.68 Date: Fri, 28 Aug 2026 12:17:01 +0200 From: Peter Zijlstra To: mingo@kernel.org Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, kprateek.nayak@amd.com, linux-kernel@vger.kernel.org, tj@kernel.org Subject: [PATCH 2/7] sched/core: Simplify/fix time updates References: <20260828101659.812011872@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Directly test RQCF_UPDATED instead of using convoluted logic to try and divinate the same. Notably, the multi-pick loop's pick_task() can, when it ends up balancing, = lock and unlock the calling CPUs RQ and 'lose' the RQCF_UPDATED tag, which then trips set_next_task(). Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/core.c | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6208,11 +6208,16 @@ extern void task_vruntime_update(struct =20 static void queue_core_balance(struct rq *rq); =20 +static void opt_update_rq_clock(struct rq *rq) +{ + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); +} + static struct task_struct * pick_next_task(struct rq *rq, struct rq_flags *rf) __must_hold(__rq_lockp(rq)) { - bool core_clock_updated =3D (rq =3D=3D rq->core); struct task_struct *next, *p, *max; const struct cpumask *smt_mask; int i, cpu, seq, occ =3D 0; @@ -6269,10 +6274,7 @@ pick_next_task(struct rq *rq, struct rq_ /* reset state */ rq->core->core_cookie =3D 0UL; if (rq->core->core_forceidle_count) { - if (!core_clock_updated) { - update_rq_clock(rq->core); - core_clock_updated =3D true; - } + opt_update_rq_clock(rq->core); sched_core_account_forceidle(rq); /* reset after accounting force idle */ rq->core->core_forceidle_start =3D 0; @@ -6299,14 +6301,11 @@ pick_next_task(struct rq *rq, struct rq_ * and there are no cookied tasks running on siblings. */ if (!need_sync) { + opt_update_rq_clock(rq); + next =3D pick_task(rq, rf); - if (unlikely(next =3D=3D RETRY_TASK)) { - /* rq lock may have been dropped, clocks invalidated */ - core_clock_updated =3D false; - if (!(rq->clock_update_flags & RQCF_UPDATED)) - update_rq_clock(rq); + if (unlikely(next =3D=3D RETRY_TASK)) goto restart; - } =20 if (!next->core_cookie) { rq->core_pick =3D NULL; @@ -6329,6 +6328,7 @@ pick_next_task(struct rq *rq, struct rq_ */ max =3D NULL; for_each_cpu_wrap(i, smt_mask, cpu) { + struct rq_flags rf_i =3D *rf; rq_i =3D cpu_rq(i); =20 /* @@ -6336,18 +6336,12 @@ pick_next_task(struct rq *rq, struct rq_ * pick_next_task(). If the current cpu is not the core, * the core may also have been updated above. */ - if (i !=3D cpu && (rq_i !=3D rq->core || !core_clock_updated)) - update_rq_clock(rq_i); + opt_update_rq_clock(rq_i); =20 - p =3D pick_task(rq_i, rf); + p =3D pick_task(rq_i, &rf_i); if (unlikely(seq !=3D rq->core->core_task_seq || - WARN_ON_ONCE(p =3D=3D RETRY_TASK))) { - /* rq lock may have been dropped, clocks invalidated */ - core_clock_updated =3D false; - if (!(rq->clock_update_flags & RQCF_UPDATED)) - update_rq_clock(rq); + WARN_ON_ONCE(p =3D=3D RETRY_TASK))) goto restart; - } =20 rq_i->core_pick =3D p; rq_i->core_dl_server =3D rq_i->dl_server; @@ -6356,6 +6350,13 @@ pick_next_task(struct rq *rq, struct rq_ max =3D p; } =20 + /* + * The above loop does @cpu first, if any sibling (which comes later) + * does a LOCK+UNLOCK of @rq in order to (try) steal a task, our + * RQCF_UPDATED got lost. + */ + rq->clock_update_flags |=3D RQCF_UPDATED; + cookie =3D rq->core->core_cookie =3D max->core_cookie; =20 /* From nobody Sat Sep 26 22:55:39 2026 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 49711412BFA for ; Fri, 28 Aug 2026 10:41:32 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1787913699; cv=none; b=QJFmab/I2B6/oey2zCJdUF/bzyGFKp3nswMJJmiFe4RKSI8YXfShwfcJ43dgBWu51fWNGI4tHvZtAeIj49PWr6HU4fc31rwo5tb2tD6KRXYg4clwAHNUGr8OJNTuxuOYoUIQBSWdCKO6nS/9tPiPLZ98XgMk7dOGT3fHgzGYbN0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1787913699; c=relaxed/simple; bh=n2FwlhhN0bBbNwBLFjEW4/4SeO+G8UfQSZ+nbFA1GTs=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=EEw4RdL9VuyzlYRmZ6hEEtVe4ay03UKTUd6LwVieMn/Wc4KoKy1pNtvAFvQ6ZSCzSSg1TgU7ptpgfyNbT4Nj/wk3hvLW6QnkIPB/8X9ThavM+AcXQs1oKoG+2OkSprbRYy8Y58dk3ss1OQ7YKEBmMZCC6BPD+ltLVp0PyVoaN48= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=infradead.org; spf=pass smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=rvcW1jt5; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="rvcW1jt5" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=UIjmU2oXAMcjl83pKKX4YVTcYXES3RkU5FaOkjGU5BQ=; b=rvcW1jt5WVE2YLC1z4L8OQqKgL DqyjUnzP5Vz6q36e3qXhG7bcVWzMTdx+GVfx2fs5psvWplLoYAkp7gIps3nrxR2TE37hk0t5ENLUt UbRV+CZ4J1+gpzbEgyhJ3XgOKp3P2xbGDvMUuHybo7as3hnQ0VT9VmJUuR9zE2zAb6riIpRqKbJpT 8BDN6wY+bHPoF2Of/eXNRpfU6JOIv5qGczDftr6OLRYqQoozQPGgeap5K1PX7JAro5aKD6TeAv7R0 Ub4nUSTBOoGXK+xnia/KsX2k3gfHxgRdyL8A36jkkLsgELbqlwRb23D5a15/Zg8/iDvSFHudRWUAs PtrEG43Q==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.99.1 #2 (Red Hat Linux)) id 1wzu1S-0000000EWAe-3xCZ; Fri, 28 Aug 2026 10:41:26 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id DAB3C303071; Fri, 28 Aug 2026 12:41:21 +0200 (CEST) Message-ID: <20260828104018.583549667@infradead.org> User-Agent: quilt/0.68 Date: Fri, 28 Aug 2026 12:17:02 +0200 From: Peter Zijlstra To: mingo@kernel.org Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, kprateek.nayak@amd.com, linux-kernel@vger.kernel.org, tj@kernel.org Subject: [PATCH 3/7] sched/core: Allow newidle for core-sched References: <20260828101659.812011872@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" As reported [1], it is possible for newidle, as called from pick_task_fair(= ), to migrate a task that was already selected for the SMT sibling. That is, in case of core scheduling (pick_next_task()'s restart_multi label= ), pick_task() is called for each SMT sibling. Suppose SMT0 picks task A, and = SMT1 has no tasks and does newidle, it must not be possible (but currently is) to migrate A to SMT1 and also select A. Avoids the 'stealing', but doesn't yet enable newidle just yet. [1] https://patch.msgid.link/20260603095108.GA1684319@bytedance.com Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/core.c | 3 ++- kernel/sched/fair.c | 6 ++---- kernel/sched/sched.h | 15 ++++++++++++++- 3 files changed, 18 insertions(+), 6 deletions(-) --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3049,7 +3049,8 @@ static int affine_move_task(struct rq *r return -EINVAL; } =20 - if (task_on_cpu(rq, p) || READ_ONCE(p->__state) =3D=3D TASK_WAKING) { + if (task_on_cpu(rq, p) || task_on_core(rq, p) || + READ_ONCE(p->__state) =3D=3D TASK_WAKING) { /* * MIGRATE_ENABLE gets here because 'p =3D=3D current', but for * anything else we cannot do is_migration_disabled(), punt --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10906,7 +10903,8 @@ int can_migrate_task(struct task_struct env->flags &=3D ~LBF_ALL_PINNED; =20 if (task_on_cpu(env->src_rq, p) || - task_current_donor(env->src_rq, p)) { + task_current_donor(env->src_rq, p) || + task_on_core(env->src_rq, p)) { schedstat_inc(p->stats.nr_failed_migrations_running); return 0; } --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1573,6 +1573,14 @@ static inline bool task_has_sched_core(s return !!p->core_cookie; } =20 +static inline bool task_on_core(struct rq *rq, struct task_struct *p) +{ + if (sched_core_disabled()) + return false; + + return rq->core_pick =3D=3D p; +} + #else /* !CONFIG_SCHED_CORE: */ =20 static inline bool sched_core_enabled(struct rq *rq) @@ -1618,6 +1626,11 @@ static inline bool task_has_sched_core(s return false; } =20 +static inline bool task_on_core(struct rq *rq, struct task_struct *p) +{ + return false; +} + #endif /* !CONFIG_SCHED_CORE */ =20 #ifdef CONFIG_RT_GROUP_SCHED @@ -4148,7 +4161,7 @@ void move_queued_task_locked(struct rq * static inline bool task_is_pushable(struct rq *rq, struct task_struct *p, int cpu) { - if (!task_on_cpu(rq, p) && + if (!task_on_cpu(rq, p) && !task_on_core(rq, p) && cpumask_test_cpu(cpu, &p->cpus_mask)) return true; From nobody Sat Sep 26 22:55:39 2026 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 49097412269 for ; Fri, 28 Aug 2026 10:41:32 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1787913698; cv=none; b=HfwCmw8BxuDnN1Vy3cWCzAKhnjhbBH3Oc1LQ/qxA883edpUEmsNx8LRBFIR2uOhYnfq5HtKKIcvwUbGyf4xf3wmtbEKZ+NMovK73n3uiu9YZt2JmyebmF91qHk0IOctjOtusgR549/OR+MCTqZ4dyRkpnbIPFepniNwKY9gJBJ0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1787913698; c=relaxed/simple; bh=JMBO9qmA5Afi8bgoVEUXd5gFEHvi1M2OJqPUFjg48/4=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=Jm5SEX6NszvRmMJ3LlOPSAm8W/Ch26MJtYdhJhAyTU9/cb4VyvE3gxU4ZZu0sJyd4yz6MESL/xBT0fMMZOa6XV236YSwcjoyZrwoyj7eG7pREBFoGepzqJLl9aeKMi9DyJuvSGokPpFURvxxPKCFD+QeVrh/qtlrpBslFb+Q/y8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=infradead.org; spf=pass smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=mWKtdMuS; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="mWKtdMuS" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=Prd8jwZtb58rorDXxvOktZtmgJu0zmfUkoaaXtifMuc=; b=mWKtdMuSVQVBXsHipw5PIy4OTv fn4Uckfbedg3u3A/5iBtD5Jn1e41XGOAMpzB2OEuwkgKRb9ixHn6zU3mclFBAyImB5n/1YNijSBYS cNrpAXc0327SqYOc5pNGjGXrjUv6zye3Yn7V42+0ysUpqovjaAGTgUT/Pl4+Dsxd6lDQwpS3KFnrJ sWr2f1EfL5tl6nqMaFsw+t4F0ZqIlVqVvMTh210gheT3IlAqUe0srOvhVZwZQvUho+gabvsL0Wb7c kfSx7QjwjW9rY1xnfmATj6ELWEm8H7tBco4FD579mXbhTA7N4mVYwLAlUL7Ts+pc7Sn7uOD9XHkHV KeK6AatQ==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.99.2 #2 (Red Hat Linux)) id 1wzu1T-00000008CEc-1JwK; Fri, 28 Aug 2026 10:41:23 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id DE3543030A6; Fri, 28 Aug 2026 12:41:21 +0200 (CEST) Message-ID: <20260828104018.681741017@infradead.org> User-Agent: quilt/0.68 Date: Fri, 28 Aug 2026 12:17:03 +0200 From: Peter Zijlstra To: mingo@kernel.org Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, kprateek.nayak@amd.com, linux-kernel@vger.kernel.org, tj@kernel.org Subject: [PATCH 4/7] sched/rt: Add early exit on balance path References: <20260828101659.812011872@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Add/make consistent an early exit on the balance_{rt,dl}() path. When there= are no pushable tasks, there is no point in taking the rq lock and all that jaz= z. Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/deadline.c | 10 +++------- kernel/sched/rt.c | 3 +++ 2 files changed, 6 insertions(+), 7 deletions(-) --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -3250,17 +3250,13 @@ static void pull_dl_task(struct rq *this src_rq->dl.earliest_dl.next)) continue; =20 + if (!has_pushable_dl_tasks(src_rq)) + continue; + /* Might drop this_rq->lock */ push_task =3D NULL; double_lock_balance(this_rq, src_rq); =20 - /* - * If there are no more pullable tasks on the - * rq, we're done with it. - */ - if (src_rq->dl.dl_nr_running <=3D 1) - goto skip; - p =3D pick_earliest_pushable_dl_task(src_rq, this_cpu); =20 /* --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -2303,6 +2303,9 @@ static void pull_rt_task(struct rq *this this_rq->rt.highest_prio.curr) continue; =20 + if (!has_pushable_tasks(src_rq)) + continue; + /* * We can potentially drop this_rq's lock in * double_lock_balance, and another CPU could From nobody Sat Sep 26 22:55:39 2026 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 9B344411A12 for ; Fri, 28 Aug 2026 10:41:32 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1787913697; cv=none; b=o0tdf9tSrVCJSUZSP34QrDPFbUYN9VgfR7QiMrmB4WeFkNko31gj1ssPk9J0lsdcU7fVFh1K3aWX0HHi2xB9EON6ycNVWheJ1DYUKXBdELZOq+vqS/9UnuZJjNdT10JnASFg6RokjMu8jYUFnG4e/MwUpqARq7CGU1UZgcDWJAo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1787913697; c=relaxed/simple; bh=x75Nn42OGuPkUaph2bNV9DA0Be9pp6Tx1RtXKAiVkqM=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=eVvNK9fkgpJmL8G5Ty3aepUCMjb8R5TPtBa/dihQzg4+V6kFvZBvt/cW3QhwqEqbvE1dSHjUVJAptHfIOm6J07Fj1Rz1COpsUFJ4pSEFSJeC2mSwh4of+/r9e2N01wAzWgHBTQoUqwWAZma2xtoM4vFl2SK3cbfEPvw2U5vFiDA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=infradead.org; spf=pass smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=N3x4KzAS; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="N3x4KzAS" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=lB9EtaL/P3ii3s+ArDjR6Oxn8NL0qwk8vBA/yDHTIYs=; b=N3x4KzASY7fX2zS07zZ7frLIi0 PxDxOsYcSyFViwC0D8U+N0hi0ZWRCxfhQHdY15UfrVedakj+0RJ6242d1hXtxOhCyRk/XogR7R4am g9OUcSn7pj3Nl8q6jhxV5tjsuKILiQ/zdXMaVrsLqSa5wG+Pmkt5UJM/nhXOX2Uh7vcGs05Pk2bMc ThC/DSemkG7ETU1TQyKOXz/Ajehknf+j4vHUZTMOiH7m86Wp4oewHW/3mTOZxfRSE4/p5DznmmWby qhscRJiPp7FXepQGrBlXSoZQXXE9V92yL9AqT1oHd7QcBQUCVtmREH8obz9mvVsMD2e6r3yapRtaS QlRQWXeA==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.99.1 #2 (Red Hat Linux)) id 1wzu1T-0000000EWAo-2VWH; Fri, 28 Aug 2026 10:41:26 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id E29C73030AD; Fri, 28 Aug 2026 12:41:21 +0200 (CEST) Message-ID: <20260828104018.787096901@infradead.org> User-Agent: quilt/0.68 Date: Fri, 28 Aug 2026 12:17:04 +0200 From: Peter Zijlstra To: mingo@kernel.org Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, kprateek.nayak@amd.com, linux-kernel@vger.kernel.org, tj@kernel.org Subject: [PATCH 5/7] sched/fair: Reflow pick_task_fair() / newidle References: <20260828101659.812011872@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" This is all histerical artifacts. Clean up the code to be less convoluted. Note: this also re-enables balance for core-sched. Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/fair.c | 51 +++++++++++++----------------------------------= ---- 1 file changed, 13 insertions(+), 38 deletions(-) --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5942,7 +5942,7 @@ static inline unsigned long cfs_rq_load_ return cfs_rq->avg.load_avg; } =20 -static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) +static void sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) __must_hold(__rq_lockp(this_rq)); =20 static inline unsigned long task_util(struct task_struct *p) @@ -10049,11 +10049,17 @@ struct task_struct *pick_task_fair(struc struct cfs_rq *cfs_rq =3D &rq->cfs; struct sched_entity *se; struct task_struct *p; - int new_tasks; =20 again: - if (!cfs_rq->h_nr_queued) - goto idle; + if (unlikely(!cfs_rq->h_nr_queued)) { + rq_modified_begin(rq, &fair_sched_class); + sched_balance_newidle(rq, rf); + if (rq_modified_above(rq, &fair_sched_class)) + return RETRY_TASK; + + if (!cfs_rq->h_nr_queued) + return NULL; + } =20 /* Might not have done put_prev_entity() */ if (cfs_rq->curr && cfs_rq->curr->on_rq) @@ -10065,17 +10071,6 @@ struct task_struct *pick_task_fair(struc =20 p =3D task_of(se); return p; - -idle: - if (sched_core_enabled(rq)) - return NULL; - - new_tasks =3D sched_balance_newidle(rq, rf); - if (new_tasks < 0) - return RETRY_TASK; - if (new_tasks > 0) - goto again; - return NULL; } =20 static struct task_struct * @@ -14531,13 +14526,8 @@ static inline void nohz_newidle_balance( /* * sched_balance_newidle is called by schedule() if this_cpu is about to b= ecome * idle. Attempts to pull tasks from other CPUs. - * - * Returns: - * < 0 - we released the lock and there are !fair tasks present - * 0 - failed, no new tasks - * > 0 - success, new (fair) tasks present */ -static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) +static void sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) __must_hold(__rq_lockp(this_rq)) { unsigned long next_balance =3D jiffies + HZ; @@ -14554,7 +14544,7 @@ static int sched_balance_newidle(struct * Return 0; the task will be enqueued when switching to idle. */ if (this_rq->ttwu_pending) - return 0; + return; =20 /* * We must set idle_stamp _before_ calling sched_balance_rq() @@ -14567,7 +14557,7 @@ static int sched_balance_newidle(struct * Do not pull tasks towards !active CPUs... */ if (!cpu_active(this_cpu)) - return 0; + return; =20 /* * This is OK, because current is on_cpu, which avoids it being picked @@ -14596,7 +14586,6 @@ static int sched_balance_newidle(struct t0 =3D sched_clock_cpu(this_cpu); __sched_balance_update_blocked_averages(this_rq); =20 - rq_modified_begin(this_rq, &fair_sched_class); raw_spin_rq_unlock(this_rq); =20 for_each_domain(this_cpu, sd) { @@ -14654,18 +14643,6 @@ static int sched_balance_newidle(struct if (curr_cost > this_rq->max_idle_balance_cost) this_rq->max_idle_balance_cost =3D curr_cost; =20 - /* - * While browsing the domains, we released the rq lock, a task could - * have been enqueued in the meantime. Since we're not going idle, - * pretend we pulled a task. - */ - if (this_rq->cfs.h_nr_queued && !pulled_task) - pulled_task =3D 1; - - /* If a higher prio class was modified, restart the pick */ - if (rq_modified_above(this_rq, &fair_sched_class)) - pulled_task =3D -1; - out: /* Move the next balance forward */ if (time_after(this_rq->next_balance, next_balance)) @@ -14677,8 +14654,6 @@ static int sched_balance_newidle(struct nohz_newidle_balance(this_rq); =20 rq_repin_lock(this_rq, rf); - - return pulled_task; } =20 /* From nobody Sat Sep 26 22:55:39 2026 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 85DB740EB91 for ; Fri, 28 Aug 2026 10:41:33 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1787913697; cv=none; b=GGAYembMM3J+iADWyii4yNjCvu7H1UBhSSe+5j3KP+0kAtQ0UxJaJ3mu+f49ilXyv4AJHZF6HIXHJWGLQ93ymyzp+pa0yVO1w8zMFhp+L8X6Y6Zyxa1AOUH++XVPy76/ayckQqwyiAKh4cHwGZ1/iYj7nZGVX6e1XkTeCUkg3Co= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1787913697; c=relaxed/simple; bh=M9L/8ers4EBHSjNxIucwGQmOBSlS1KyQ7jpC7KaSLxg=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=rv+5TcPbeMDVuoeKjcrRMv25OpwjIEnmGfAkjBdpfYFCGvhp1H18RdTagE3w7iDkFhjA4Wp9SGuyIEqDvm2fpIdW2Bj4PMrY529i1j5E1b5oYNC6ixZSJetcZ5neBJOcR3UqfZmIqIsfowyxtn3u0d78MkmDw3XOpG+cKQL1AuA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=infradead.org; spf=pass smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=sobi9/wC; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="sobi9/wC" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=ikZKHh438sPD3ZO6RnnPs393nybtZ4oMoD2bs/hJ8uU=; b=sobi9/wC5OMW+9t5zSar0pPlJJ ePHujL/11xNyklWIMzregODg3mZPV/rjMLdysupHc4YlAIyg3c+OHLQ96ZoUZ5ClomhXzjqib2gIO hTnc99arszPDLL09BycYNGE7vb6cxrtSoy9xF0/kr404aQBfH9T8BLBgUVjBDMAdp6xf96q2Vu63v QLtYmw52l1aF/YsUPQEMFltGp6EI2ecb7fI8iTz3mNltYmzIcCiyk8phdL+bcaTq4lXTa3BKum5AC GXpsTbx9Jw9RGfFesx2n8CYYSbFj49AReOqE5ZjsP43Clwq5I894CFIozAnxbx40sRtDol1EPIuRi RF9vwCWA==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.99.1 #2 (Red Hat Linux)) id 1wzu1T-0000000EWAn-2X1j; Fri, 28 Aug 2026 10:41:24 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id E697C303152; Fri, 28 Aug 2026 12:41:21 +0200 (CEST) Message-ID: <20260828104018.890793421@infradead.org> User-Agent: quilt/0.68 Date: Fri, 28 Aug 2026 12:17:05 +0200 From: Peter Zijlstra To: mingo@kernel.org Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, kprateek.nayak@amd.com, linux-kernel@vger.kernel.org, tj@kernel.org Subject: [PATCH 6/7] sched/fair: Push sched_balance_newidle() unlock down References: <20260828101659.812011872@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" By pushing down the this_rq unlock, there are more cases where the lock isn= 't dropped. XXX needs numbers Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/fair.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -13417,6 +13417,7 @@ static int sched_balance_rq(int this_cpu struct sched_domain *sd, enum cpu_idle_type idle, int *continue_balancing) { + bool lock_rq =3D false, unlock_rq =3D idle =3D=3D CPU_NEWLY_IDLE; int ld_moved, cur_ld_moved, active_balance =3D 0; struct sched_domain *sd_parent =3D sd->parent; struct sched_group *group; @@ -13485,6 +13486,12 @@ static int sched_balance_rq(int this_cpu */ env.loop_max =3D min(sysctl_sched_nr_migrate, busiest->nr_running); =20 + if (unlock_rq) { + raw_spin_rq_unlock(this_rq); + unlock_rq =3D false; + lock_rq =3D true; + } + more_balance: rq_lock_irqsave(busiest, &rf); update_rq_clock(busiest); @@ -13613,6 +13620,12 @@ static int sched_balance_rq(int this_cpu if (!need_active_balance(&env)) goto out_unbalanced; =20 + if (unlock_rq) { + raw_spin_rq_unlock(this_rq); + unlock_rq =3D false; + lock_rq =3D true; + } + scoped_guard (raw_spin_rq_lock_irqsave, busiest) { /* * Don't kick the active_load_balance_cpu_stop, @@ -13706,6 +13719,8 @@ static int sched_balance_rq(int this_cpu out: if (need_unlock) atomic_set_release(&sched_balance_running, 0); + if (lock_rq) + raw_spin_rq_lock(this_rq); =20 return ld_moved; } @@ -14582,8 +14597,6 @@ static void sched_balance_newidle(struct t0 =3D sched_clock_cpu(this_cpu); __sched_balance_update_blocked_averages(this_rq); =20 - raw_spin_rq_unlock(this_rq); - for_each_domain(this_cpu, sd) { u64 domain_cost; =20 @@ -14634,8 +14647,6 @@ static void sched_balance_newidle(struct break; } =20 - raw_spin_rq_lock(this_rq); - if (curr_cost > this_rq->max_idle_balance_cost) this_rq->max_idle_balance_cost =3D curr_cost; From nobody Sat Sep 26 22:55:39 2026 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C26E34028D9 for ; Fri, 28 Aug 2026 10:41:32 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1787913698; cv=none; b=X1TkjfwN1IgPFeRNpcWxzgvrDng0g2/MokxBhRVBEB1aApEjfNasb5bzalzPjS3fv4P1a17LwyCfLnMsgBn/v0xRzlLB/qJYKpY/2VPghteTQk6sK591SrEcBqpTMJJg98EsedLVRw6Oe/MdG2hnB+7fiEMU9axB6O8WvJI4CkY= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1787913698; c=relaxed/simple; bh=GT2keGz9a11wGv3Pu54CfaY09O9hMb4Kov+LmHnLmmU=; h=Message-ID:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=hXjBhsn3GQNIhBGOAB4GMEjQGvmzPU8PX+/EviFYkVW/7Faxbm3nJTqb20D5k7iipuX8yc8rY4QUq7i5TUm9E53ydQi67B+wZNkIfbJvXi2PD5GSYULRmt5RyVj+729XTk85q797yt4zDyQ/VAwD80K1NNpNFTAuwhOdV/ybCb8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=infradead.org; spf=pass smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=a5u48e+2; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="a5u48e+2" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-ID:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=mHZz+wX0gouUyTzWxfn+wXM4U+Ln9tJes1IvBO2JVTM=; b=a5u48e+2ntdf+4L0WShPMKVzX8 VaS+2UlaEPxOWrl7rNV3/6qhqZ5a+VJIgLMgaqPS9kHeMyAwUuuYP09SGdt8Bt6RXUfergkenY6st MeOvEK0pl6UzP8ecv7s8kWgJI5RScu+e/gb0oyL6wwxwS0KiyQneFvsPCLmi9jMHHrwMXnscN03ev OnGLjnTnNFApB8GJA4I9Wle/8ZXL8Nys7FexO+jb0TbYpk01tQu1WjNaFi6z6asmzkFB/OlmWf8xp 50TWOLLjwggXgC9GARS3cmZpFRw0yftP3Jk9ioxFkoq/3lls9f04RWWK0WA/ztLoywIyIv+Tjiyrs PNQKmnrg==; Received: from 77-249-17-252.cable.dynamic.v4.ziggo.nl ([77.249.17.252] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.99.1 #2 (Red Hat Linux)) id 1wzu1T-0000000EWAm-2RN9; Fri, 28 Aug 2026 10:41:24 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id EAAA13031C8; Fri, 28 Aug 2026 12:41:21 +0200 (CEST) Message-ID: <20260828104018.996963405@infradead.org> User-Agent: quilt/0.68 Date: Fri, 28 Aug 2026 12:17:06 +0200 From: Peter Zijlstra To: mingo@kernel.org Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, kprateek.nayak@amd.com, linux-kernel@vger.kernel.org, tj@kernel.org Subject: [PATCH 7/7] sched: Remove sched_class::balance() References: <20260828101659.812011872@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Ever since commit 50653216e4ff ("sched: Add support to pick functions to ta= ke rf"), we have the unfortunate situation that both sched_class::balance() and sched_class::pick_task() have overlapping functionality in that they drop rq->lock and balance tasks. Additionally, prev_balance() is only called for a single RQ in the core-sch= ed case, resulting in 'missed' balance opportunities in this case. Just like pick_task(), prev_balance() would iterate the classes and stop wh= en there is a runnable task found. The only difference is that pick_task() sta= rts from the top class, while prev_balance() would start from prev->class. This means that if prev was an RT task, we'd get an extra visit to balance_= dl() -- and similary, if prev is fair, we'd visit balance_{dl,rt}() both. Howeve= r, this is not a problem, because both need_pull_{dl,rt}_task() will DTRT. Therefore, move balance_{rt,dl}() into pick_task_{rt,dl}() and remove sched_class::balance(). Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/core.c | 23 ----------------------- kernel/sched/deadline.c | 10 ++++++---- kernel/sched/fair.c | 47 +++++++++++++-----------------------------= ----- kernel/sched/idle.c | 7 ------- kernel/sched/rt.c | 14 ++++++-------- kernel/sched/sched.h | 5 ----- kernel/sched/stop_task.c | 7 ------- 7 files changed, 25 insertions(+), 88 deletions(-) --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6115,25 +6115,6 @@ static inline void schedule_debug(struct schedstat_inc(this_rq()->sched_count); } =20 -static void prev_balance(struct rq *rq, struct rq_flags *rf) -{ - const struct sched_class *start_class =3D rq->donor->sched_class; - const struct sched_class *class; - - /* - * We must do the balancing pass before put_prev_task(), such - * that when we release the rq->lock the task is in the same - * state as before we took rq->lock. - * - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ - for_active_class_range(class, start_class, &idle_sched_class) { - if (class->balance && class->balance(rq, rf)) - break; - } -} - /* * Pick up the highest-prio task: */ @@ -6171,8 +6152,6 @@ __pick_next_task(struct rq *rq, struct r } =20 restart: - prev_balance(rq, rf); - for_each_active_class(class) { p =3D class->pick_task(rq, rf); if (unlikely(p =3D=3D RETRY_TASK)) @@ -6282,8 +6261,6 @@ pick_next_task(struct rq *rq, struct rq_ goto out_set_next; } =20 - prev_balance(rq, rf); - smt_mask =3D cpu_smt_mask(cpu); =20 restart: --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2705,7 +2705,7 @@ static void check_preempt_equal_dl(struc resched_curr(rq); } =20 -static int balance_dl(struct rq *rq, struct rq_flags *rf) +static void balance_dl(struct rq *rq, struct rq_flags *rf) { /* * Note, rq->donor may change during rq lock drops, @@ -2724,8 +2724,6 @@ static int balance_dl(struct rq *rq, str pull_dl_task(rq); rq_repin_lock(rq, rf); } - - return sched_stop_runnable(rq) || sched_dl_runnable(rq); } =20 /* @@ -2820,6 +2818,11 @@ static struct task_struct *__pick_task_d struct dl_rq *dl_rq =3D &rq->dl; struct task_struct *p; =20 + rq_modified_begin(rq, &dl_sched_class); + balance_dl(rq, rf); + if (rq_modified_above(rq, &dl_sched_class)) + return RETRY_TASK; + again: if (!sched_dl_runnable(rq)) return NULL; @@ -3655,7 +3658,6 @@ DEFINE_SCHED_CLASS(dl) =3D { .put_prev_task =3D put_prev_task_dl, .set_next_task =3D set_next_task_dl, =20 - .balance =3D balance_dl, .select_task_rq =3D select_task_rq_dl, .migrate_task_rq =3D migrate_task_rq_dl, .set_cpus_allowed =3D set_cpus_allowed_dl, --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -464,12 +464,6 @@ select_task_rq_idle(struct task_struct * return task_cpu(p); /* IDLE tasks as never migrated */ } =20 -static int -balance_idle(struct rq *rq, struct rq_flags *rf) -{ - return WARN_ON_ONCE(1); -} - /* * Idle tasks are unconditionally rescheduled: */ @@ -581,7 +575,6 @@ DEFINE_SCHED_CLASS(idle) =3D { .put_prev_task =3D put_prev_task_idle, .set_next_task =3D set_next_task_idle, =20 - .balance =3D balance_idle, .select_task_rq =3D select_task_rq_idle, .set_cpus_allowed =3D set_cpus_allowed_common, =20 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1596,7 +1596,7 @@ static void check_preempt_equal_prio(str resched_curr(rq); } =20 -static int balance_rt(struct rq *rq, struct rq_flags *rf) +static void balance_rt(struct rq *rq, struct rq_flags *rf) { /* * Note, rq->donor may change during rq lock drops, @@ -1615,8 +1615,6 @@ static int balance_rt(struct rq *rq, str pull_rt_task(rq); rq_repin_lock(rq, rf); } - - return sched_stop_runnable(rq) || sched_dl_runnable(rq) || sched_rt_runna= ble(rq); } =20 /* @@ -1715,14 +1713,15 @@ static struct task_struct *_pick_next_ta =20 static struct task_struct *pick_task_rt(struct rq *rq, struct rq_flags *rf) { - struct task_struct *p; + rq_modified_begin(rq, &rt_sched_class); + balance_rt(rq, rf); + if (rq_modified_above(rq, &rt_sched_class)) + return RETRY_TASK; =20 if (!sched_rt_runnable(rq)) return NULL; =20 - p =3D _pick_next_task_rt(rq); - - return p; + return _pick_next_task_rt(rq); } =20 static void put_prev_task_rt(struct rq *rq, struct task_struct *p, struct = task_struct *next) @@ -2610,7 +2609,6 @@ DEFINE_SCHED_CLASS(rt) =3D { .put_prev_task =3D put_prev_task_rt, .set_next_task =3D set_next_task_rt, =20 - .balance =3D balance_rt, .select_task_rq =3D select_task_rq_rt, .set_cpus_allowed =3D set_cpus_allowed_common, .rq_online =3D rq_online_rt, --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2658,11 +2658,6 @@ struct sched_class { void (*wakeup_preempt)(struct rq *rq, struct task_struct *p, int flags); =20 /* - * schedule/pick_next_task/prev_balance: rq->lock - */ - int (*balance)(struct rq *rq, struct rq_flags *rf); - - /* * schedule/pick_next_task: rq->lock */ struct task_struct *(*pick_task)(struct rq *rq, struct rq_flags *rf); --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -15,12 +15,6 @@ select_task_rq_stop(struct task_struct * return task_cpu(p); /* stop tasks as never migrate */ } =20 -static int -balance_stop(struct rq *rq, struct rq_flags *rf) -{ - return sched_stop_runnable(rq); -} - static void wakeup_preempt_stop(struct rq *rq, struct task_struct *p, int flags) { @@ -107,7 +101,6 @@ DEFINE_SCHED_CLASS(stop) =3D { .put_prev_task =3D put_prev_task_stop, .set_next_task =3D set_next_task_stop, =20 - .balance =3D balance_stop, .select_task_rq =3D select_task_rq_stop, .set_cpus_allowed =3D set_cpus_allowed_common,