From nobody Sat Feb 7 19:45:44 2026 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 633861AB512 for ; Tue, 13 Aug 2024 22:50:39 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589440; cv=none; b=IU649i0gffxT9sIiKOsQCbjudgMiMT8DXBlpn2JBQ5JBMI87ozKuvS3vvszQhzq088oPjozY8exViA+T1HtZFEc0rPoepR2YbNjodYSAUzYFXEynnYg/+tmIVp7NwYsafowErIHD7tJhQcEbTvsgdXYa92Dx4YvHu8SlmfW1QeY= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589440; c=relaxed/simple; bh=IjGyIR1ybwl2f+NOIyS2e5TbqO3Xbxq3C7XGh+QuLF0=; h=Message-Id:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=iA0jhnUaxt9cvyG3ybmz+zIji37XzLeD/zhWrEYfibm983J2xuOiEmSjmPDDKlfUOygN7AlCPm/N0MqdCY7rF5o9i8D5LZU1qcc4SozTBNtLfuMQdMy86bLH23FAi/koIv43DhQwkrQu1NgTDUKx9zWExsGhfBaAEC9mOkLdR9w= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=k71KYgKA; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="k71KYgKA" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=3hSSlKjAiFgV8m6vC2/ajtpmW/+xwn+/awkIsESMp/4=; b=k71KYgKAgU0+6lt5+AqrNwQpt0 VpQ+Q0wLE8Vy+TZ3HLPgiXbqprNM44oi6CCKpsIYTxorxdRQ2teq4dSuiO8wN2rhryQbdFUWGRKbq SGC11grIk3HHZwXnX33Y3MlUI0mntawW7C4nV4iavQSnOX+FDHI6ssJiGEftr8BUtmjYlPTshpS4V yUyvbkI0q9Fz3YY25m7zw3iSUPFDNe5M6OZAqf9+mqCoCzjBL3ONSqc37cCHDJRCIb4jN6eTc97vp sGKkSYlW7w0+LfHGFyBJgRjA/Ss0JshjCWGgIp9b06BBusECKY4vi9sLxPFg51L7p60Ggo7ksRYjW m0fnH4Pg==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.97.1 #2 (Red Hat Linux)) id 1se0LW-0000000H9ZU-0KWH; Tue, 13 Aug 2024 22:50:30 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 314FA300230; Wed, 14 Aug 2024 00:50:29 +0200 (CEST) Message-Id: <20240813224015.614146342@infradead.org> User-Agent: quilt/0.65 Date: Wed, 14 Aug 2024 00:25:49 +0200 From: Peter Zijlstra To: mingo@kernel.org, tj@kernel.org, void@manifault.com Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, linux-kernel@vger.kernel.org, joelaf@google.com Subject: [PATCH 1/9] sched: Use set_next_task(.first) where required References: <20240813222548.049744955@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Turns out the core_sched bits forgot to use the set_next_task(.first=3Dtrue) variant. Notably: pick_next_task() :=3D pick_task() + set_next_task(.first =3D true) Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/core.c | 4 ++-- kernel/sched/sched.h | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6010,7 +6010,7 @@ pick_next_task(struct rq *rq, struct tas next =3D rq->core_pick; if (next !=3D prev) { put_prev_task(rq, prev); - set_next_task(rq, next); + set_next_task_first(rq, next); } =20 rq->core_pick =3D NULL; @@ -6184,7 +6184,7 @@ pick_next_task(struct rq *rq, struct tas } =20 out_set_next: - set_next_task(rq, next); + set_next_task_first(rq, next); out: if (rq->core->core_forceidle_count && next =3D=3D rq->idle) queue_core_balance(rq); --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2363,6 +2363,10 @@ static inline void set_next_task(struct next->sched_class->set_next_task(rq, next, false); } =20 +static inline void set_next_task_first(struct rq *rq, struct task_struct *= next) +{ + next->sched_class->set_next_task(rq, next, true); +} =20 /* * Helper to define a sched_class instance; each one is placed in a separa= te From nobody Sat Feb 7 19:45:44 2026 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id BCD5B17B515 for ; Tue, 13 Aug 2024 22:50:38 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589441; cv=none; b=e4sncx7u6cRtoQgTFRvKImLfNsSRtS4bYj1G2RE6CrYDHFuZqHQrjuS6KjStSfo8pKTAtWQzyeYN/ZiysBfny9dk7Q2WTlEQd8xFa6UUh/7x/JAuBGmYU3AcvbSs834wFdSgX9q71DdQW28IINe3vnLfQVOhMkeijhIoyfD0BFg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589441; c=relaxed/simple; bh=oVc9IqFKqy7QuIG8uxkR7uWuhN+j12qwydbAA7W7k00=; h=Message-Id:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=eo6bwcYrkhPMRCgbwKhVek3gjoiKbSXGxBq8B1hIzeT0MfcwZqbJ4KC3H1QKIBF7jHLO1O+IuLFtqMtux23uBRBL5+RZSLiZTaXRRyXlxRe3FFMnuDLfH9GaynsMNGJ9Ur0QJcVJuvsDB/MXhBfru+gAXyk/b4nXve0JYbPfB6U= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=OSSijcG0; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="OSSijcG0" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=LdwJUkU98p+KR13h+P1Cw+rkaPkvIEoyHv2nL1rKICw=; b=OSSijcG01R0yC0N4CYpxpyKazi GIgd0Ri5vIdc9iq+flh4mZFdwBrlxbi3Ue/gg8Zegqhs/gPlm4AewYodP/tI/eOQn5nlYMsWMP//U Oe8kbWZJrG9bm8uJTynM4nLmVc8J0kg/Ses3YCuluWHbgo56KnMLnmvStyuwT98/OMxsANGoQDwHV lX17GToXxFaCkX6fsmY1PgbOCHBs/vQXTq++ffAZeQ73svDJSruurfmMoZL9eFKjoCk0N+rQAbzK9 R0RHb1aZhlwgobBUBPVvCk6RyBGKqwoh+bjqdtocy7ktnzaPRV4eIzTCftS7URvPoA7s72MTG7FU6 ygDT5icA==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.97.1 #2 (Red Hat Linux)) id 1se0LW-00000007s10-0hqd; Tue, 13 Aug 2024 22:50:30 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 348C3301195; Wed, 14 Aug 2024 00:50:29 +0200 (CEST) Message-Id: <20240813224015.724111109@infradead.org> User-Agent: quilt/0.65 Date: Wed, 14 Aug 2024 00:25:50 +0200 From: Peter Zijlstra To: mingo@kernel.org, tj@kernel.org, void@manifault.com Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, linux-kernel@vger.kernel.org, joelaf@google.com Subject: [PATCH 2/9] sched: Fixup set_next_task() implementations References: <20240813222548.049744955@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The rule is that: pick_next_task() :=3D pick_task() + set_next_task(.first =3D true) Turns out, there's still a few things in pick_next_task() that are missing from that combination. Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/deadline.c | 6 ++-- kernel/sched/fair.c | 62 ++++++++++++++++++++++++-------------------= ----- 2 files changed, 34 insertions(+), 34 deletions(-) --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2392,6 +2392,9 @@ static void set_next_task_dl(struct rq * update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 0); =20 deadline_queue_push_tasks(rq); + + if (hrtick_enabled(rq)) + start_hrtick_dl(rq, &p->dl); } =20 static struct sched_dl_entity *pick_next_dl_entity(struct dl_rq *dl_rq) @@ -2458,9 +2461,6 @@ static struct task_struct *pick_next_tas if (!p->dl_server) set_next_task_dl(rq, p, true); =20 - if (hrtick_enabled(rq)) - start_hrtick_dl(rq, &p->dl); - return p; } =20 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8758,6 +8758,9 @@ static struct task_struct *pick_task_fai return task_of(se); } =20 +static void __set_next_task_fair(struct rq *rq, struct task_struct *p, boo= l first); +static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool = first); + struct task_struct * pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_fla= gs *rf) { @@ -8806,33 +8809,17 @@ pick_next_task_fair(struct rq *rq, struc =20 put_prev_entity(cfs_rq, pse); set_next_entity(cfs_rq, se); + + __set_next_task_fair(rq, p, true); } =20 - goto done; + return p; + simple: #endif if (prev) put_prev_task(rq, prev); - - for_each_sched_entity(se) - set_next_entity(cfs_rq_of(se), se); - -done: __maybe_unused; -#ifdef CONFIG_SMP - /* - * Move the next running task to the front of - * the list, so our cfs_tasks list becomes MRU - * one. - */ - list_move(&p->se.group_node, &rq->cfs_tasks); -#endif - - if (hrtick_enabled_fair(rq)) - hrtick_start_fair(rq, p); - - update_misfit_status(p, rq); - sched_fair_update_stop_tick(rq, p); - + set_next_task_fair(rq, p, true); return p; =20 idle: @@ -13142,12 +13129,7 @@ static void switched_to_fair(struct rq * } } =20 -/* Account for a task changing its policy or group. - * - * This routine is mostly called to set cfs_rq->curr field when a task - * migrates between groups/classes. - */ -static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool = first) +static void __set_next_task_fair(struct rq *rq, struct task_struct *p, boo= l first) { struct sched_entity *se =3D &p->se; =20 @@ -13160,6 +13142,27 @@ static void set_next_task_fair(struct rq list_move(&se->group_node, &rq->cfs_tasks); } #endif + if (!first) + return; + + SCHED_WARN_ON(se->sched_delayed); + + if (hrtick_enabled_fair(rq)) + hrtick_start_fair(rq, p); + + update_misfit_status(p, rq); + sched_fair_update_stop_tick(rq, p); +} + +/* + * Account for a task changing its policy or group. + * + * This routine is mostly called to set cfs_rq->curr field when a task + * migrates between groups/classes. + */ +static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool = first) +{ + struct sched_entity *se =3D &p->se; =20 for_each_sched_entity(se) { struct cfs_rq *cfs_rq =3D cfs_rq_of(se); @@ -13169,10 +13172,7 @@ static void set_next_task_fair(struct rq account_cfs_rq_runtime(cfs_rq, 0); } =20 - if (!first) - return; - - SCHED_WARN_ON(se->sched_delayed); + __set_next_task_fair(rq, p, first); } =20 void init_cfs_rq(struct cfs_rq *cfs_rq) From nobody Sat Feb 7 19:45:44 2026 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A5A341AB522 for ; Tue, 13 Aug 2024 22:50:39 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589441; cv=none; b=DGJEUVOw4XtY0l9/RPIP4DJhhXArWmpcGHxCoav9R6jSvb0JgzTZ65m9UJKlXddnodlVrbVQ05RTtmh9WKxQuilhc/Gf2qw/lieFKMyHvwYunpk4TO5FpfuX27ujW1wJvZazaSjRQd8eKYiIA3YtHnRPCeYJhYwN9bijBEpIVN4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589441; c=relaxed/simple; bh=i8a8A/VUcSd/UmUIDuxLs+UOZIAkea+Jb6r9d160Nu4=; h=Message-Id:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=I9yKuzK0/rRpJLZG/qLl7Uh6AgCzwMs9dMHO4FkKWVYxprugB53hNa1THMOfXY1KTSZyD20do9boQjt73qQMy1KSx7PG7HFNqzXOU5G3pPyBc6/+zzGLxDPLsbGlKYcUMB7VRmyleEAFV7+/u5XRVE9TaADw8EFLPd5NDL5YoXM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=m39qjLeK; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="m39qjLeK" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=5S1HikZ9q6jC+sQEh9d0ZhP7cP9EgQzT1qMsQkQw+30=; b=m39qjLeKFe/0SPTBUNhSlnNVyb HFvuJuOb4SzhJWYsksysTQHB4wjK4FVHCNyjBe6hB+wtAr1ruUP8ZH3WbVaOc5y73md0lRIlRKFfU flU5b8VcbRpwtdaP5mArzA4XvIVrgWTRCSVuPEv298B+gOaiUKYR3qcSFDLpPt1vXj+QviuHveMSP TRxSlFkIJhbzzWEc5AtMTkL99VAif0/FqxkAqH0uDF+rYXEZxZxgkKdzcw8PSWCqh1dt3s6ai6Ncm 9OX+FoeDjPIfBeBU60QS5fsMrXMy6oTxTplxpeS/DUrLXt0IR5wEsGBH64N+LV1EQEzXF20qC+lru K/EvXtRw==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.97.1 #2 (Red Hat Linux)) id 1se0LW-00000007s11-0aZo; Tue, 13 Aug 2024 22:50:30 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 39750301D1D; Wed, 14 Aug 2024 00:50:29 +0200 (CEST) Message-Id: <20240813224015.837303391@infradead.org> User-Agent: quilt/0.65 Date: Wed, 14 Aug 2024 00:25:51 +0200 From: Peter Zijlstra To: mingo@kernel.org, tj@kernel.org, void@manifault.com Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, linux-kernel@vger.kernel.org, joelaf@google.com Subject: [PATCH 3/9] sched: Clean up DL server vs core sched References: <20240813222548.049744955@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Abide by the simple rule: pick_next_task() :=3D pick_task() + set_next_task(.first =3D true) This allows us to trivially get rid of server_pick_next() and things collapse nicely. Signed-off-by: Peter Zijlstra (Intel) --- include/linux/sched.h | 1 - kernel/sched/deadline.c | 18 +++++++----------- kernel/sched/fair.c | 13 +------------ kernel/sched/sched.h | 1 - 4 files changed, 8 insertions(+), 25 deletions(-) --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -692,7 +692,6 @@ struct sched_dl_entity { */ struct rq *rq; dl_server_has_tasks_f server_has_tasks; - dl_server_pick_f server_pick_next; dl_server_pick_f server_pick_task; =20 #ifdef CONFIG_RT_MUTEXES --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -1665,12 +1665,10 @@ void dl_server_stop(struct sched_dl_enti =20 void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq, dl_server_has_tasks_f has_tasks, - dl_server_pick_f pick_next, dl_server_pick_f pick_task) { dl_se->rq =3D rq; dl_se->server_has_tasks =3D has_tasks; - dl_se->server_pick_next =3D pick_next; dl_se->server_pick_task =3D pick_task; } =20 @@ -2410,9 +2408,8 @@ static struct sched_dl_entity *pick_next /* * __pick_next_task_dl - Helper to pick the next -deadline task to run. * @rq: The runqueue to pick the next task from. - * @peek: If true, just peek at the next task. Only relevant for dlserver. */ -static struct task_struct *__pick_next_task_dl(struct rq *rq, bool peek) +static struct task_struct *__pick_task_dl(struct rq *rq) { struct sched_dl_entity *dl_se; struct dl_rq *dl_rq =3D &rq->dl; @@ -2426,10 +2423,7 @@ static struct task_struct *__pick_next_t WARN_ON_ONCE(!dl_se); =20 if (dl_server(dl_se)) { - if (IS_ENABLED(CONFIG_SMP) && peek) - p =3D dl_se->server_pick_task(dl_se); - else - p =3D dl_se->server_pick_next(dl_se); + p =3D dl_se->server_pick_task(dl_se); if (!p) { dl_se->dl_yielded =3D 1; update_curr_dl_se(rq, dl_se, 0); @@ -2446,7 +2440,7 @@ static struct task_struct *__pick_next_t #ifdef CONFIG_SMP static struct task_struct *pick_task_dl(struct rq *rq) { - return __pick_next_task_dl(rq, true); + return __pick_task_dl(rq); } #endif =20 @@ -2454,11 +2448,13 @@ static struct task_struct *pick_next_tas { struct task_struct *p; =20 - p =3D __pick_next_task_dl(rq, false); + p =3D __pick_task_dl(rq); if (!p) return p; =20 - if (!p->dl_server) + if (p->dl_server) + p->sched_class->set_next_task(rq, p, true); + else set_next_task_dl(rq, p, true); =20 return p; --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8860,16 +8860,7 @@ static bool fair_server_has_tasks(struct =20 static struct task_struct *fair_server_pick_task(struct sched_dl_entity *d= l_se) { -#ifdef CONFIG_SMP return pick_task_fair(dl_se->rq); -#else - return NULL; -#endif -} - -static struct task_struct *fair_server_pick_next(struct sched_dl_entity *d= l_se) -{ - return pick_next_task_fair(dl_se->rq, NULL, NULL); } =20 void fair_server_init(struct rq *rq) @@ -8878,9 +8869,7 @@ void fair_server_init(struct rq *rq) =20 init_dl_entity(dl_se); =20 - dl_server_init(dl_se, rq, fair_server_has_tasks, fair_server_pick_next, - fair_server_pick_task); - + dl_server_init(dl_se, rq, fair_server_has_tasks, fair_server_pick_task); } =20 /* --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -362,7 +362,6 @@ extern void dl_server_start(struct sched extern void dl_server_stop(struct sched_dl_entity *dl_se); extern void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq, dl_server_has_tasks_f has_tasks, - dl_server_pick_f pick_next, dl_server_pick_f pick_task); =20 extern void dl_server_update_idle_time(struct rq *rq, From nobody Sat Feb 7 19:45:44 2026 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D7E2F1A76B6 for ; Tue, 13 Aug 2024 22:50:38 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589440; cv=none; b=Dv92OQUJ/9ubhHRhbw3ewKJ6I7E9NfAiEtazMsMgweUlqN9FsgWT4r3P2nBs7lkYRif5JnvmE9mb9mw4FARoYaJvv5e3gDdbBwFMl936OmC4kM2suR5nFTyuFiI69FN6VMrFZMwv1+3kKqdRAv7L0LioYbzXsVxV/SsHcUz++Vs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589440; c=relaxed/simple; bh=TnZwOCNsTC50oF6WkkQSYwA75XDmDW0q3W7IG+O8RYo=; h=Message-Id:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=kNoYPjnFmwqJu/YGt+4pk7tT40CrDrU9mlJ+UxYzrvxYLWqk28yXfqYeEi7UKYBZf1nV67Z5Ogx2Fhw88OjDfHlnOhEeCWCuRn/Qdd33dj9bcilkJPdj2lYlk60Wa/BXHkLu8jaZaS3ab9oajLHLBgjwi1znCa46K2jU33/8o28= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=gnZy3Asg; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="gnZy3Asg" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=N4qqyYvidrFryHZvi0gNFY6ukjdqN3urPMEKouQkyVY=; b=gnZy3AsgiNu6CZhX9QSQDLum5D QmIvz0JJsAzn+5rF+ctYWSVlzoBUEXMfCh1kNXgN6fv5hUla+VfLd3buhYOPz5g6bG79nDdCYFhc0 QE3ziTwq968d+Z/CV+3UsGcrWUocYZ17ZRP2NVEkviLkNiazba8IvXndudVJ4KS8/8AZOsi9TVhfz aCPXvM0bNlYT855FcAydppQc1bSu2MctyNqXfpPiwIYRKyNRnAlRHxfREqTLdHoNIe9AlXvXiTdrV Vu6z7yWAvTRc9WgM/isZPgnz3dVAAzuM42QBEkxUjI7203O5k17FKsC1kI3odMvUsod0DqnUkImiu WPYr4wKg==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.97.1 #2 (Red Hat Linux)) id 1se0LW-00000007s12-0hti; Tue, 13 Aug 2024 22:50:30 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 3DDE8302440; Wed, 14 Aug 2024 00:50:29 +0200 (CEST) Message-Id: <20240813224015.943143811@infradead.org> User-Agent: quilt/0.65 Date: Wed, 14 Aug 2024 00:25:52 +0200 From: Peter Zijlstra To: mingo@kernel.org, tj@kernel.org, void@manifault.com Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, linux-kernel@vger.kernel.org, joelaf@google.com Subject: [PATCH 4/9] sched: Split up put_prev_task_balance() References: <20240813222548.049744955@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" With the goal of pushing put_prev_task() after pick_task() / into pick_next_task(). Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/core.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5841,8 +5841,8 @@ static inline void schedule_debug(struct schedstat_inc(this_rq()->sched_count); } =20 -static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, - struct rq_flags *rf) +static void prev_balance(struct rq *rq, struct task_struct *prev, + struct rq_flags *rf) { #ifdef CONFIG_SMP const struct sched_class *class; @@ -5860,8 +5860,6 @@ static void put_prev_task_balance(struct } #endif =20 - put_prev_task(rq, prev); - /* * We've updated @prev and no longer need the server link, clear it. * Must be done before ->pick_next_task() because that can (re)set @@ -5917,7 +5915,8 @@ __pick_next_task(struct rq *rq, struct t } =20 restart: - put_prev_task_balance(rq, prev, rf); + prev_balance(rq, prev, rf); + put_prev_task(rq, prev); =20 for_each_class(class) { p =3D class->pick_next_task(rq); @@ -6017,7 +6016,8 @@ pick_next_task(struct rq *rq, struct tas goto out; } =20 - put_prev_task_balance(rq, prev, rf); + prev_balance(rq, prev, rf); + put_prev_task(rq, prev); =20 smt_mask =3D cpu_smt_mask(cpu); need_sync =3D !!rq->core->core_cookie; From nobody Sat Feb 7 19:45:44 2026 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id AD53236B11 for ; Tue, 13 Aug 2024 22:50:38 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589440; cv=none; b=OZift0rqJqN1/lVqrmtGb8UaD1uWION6IPTu0CMCh2hDDVEtyUcp+hOcZB8f+MSR0aQETK76hUjdlbTFxLF70oG8BPCSWtOFxw2Cp4MDfWucTjDF3LgkhiXLSQlpPISyKlF5iJmllBv99DP9LSoySobISYqaUULOYfMUTDRD8aU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589440; c=relaxed/simple; bh=d8GrdyfSbKrHISgF/oZceKT2zieCpn6boTgGMTZdomw=; h=Message-Id:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=u2rcb/e4yhbSE5jkajooN9aNshipFN512TyuhjuoLgK6bHPlkbfnnY52cBrSdOOiIruba3VkLC72FX80PkrQqWtmM4tvmCgsrEhEWxoa1RB5Gc/6+SvfQPSaM73t+NgAoOpjpWDfGBg9XLPC/IQLD428NK2k4D2z/1VE+Uwml5w= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=KJjRKkXx; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="KJjRKkXx" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=+MmaUoREfWUZek2lYGzLDHTOR/aEFKfS9H3PDIashAs=; b=KJjRKkXx+ue9TCa7ifRvrwpLSW amegoihVqXqx1boyYik+zVw2fT0N1DfOJl7qm82TfUCaL9hXPF3Q2oGC7ASxViysR9XHmJkcE0u5Y 8ecKTLwRqwnk6rGOFOBNTv2qPTNezqWnCaGCuBxqG4i0YJD12vmH4oiF6BPL/+9VJ2n/EroTnxymZ /9NfWwdMBiGAX3/fnhpoqHInVDTlJUO5JMQZDyh14JysXgz079usq6LCJw8VvArV0eBZX1aUkI7Mh Ph5OrRwQfjnmQ9utjOJ5XNIpkYcAfuBt7ccc2xsPcLQZ78oFFpjXdCkyOuC/ilcB4i+Ca70Ub0rJM LVobT0qg==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.97.1 #2 (Red Hat Linux)) id 1se0LW-00000007s19-3TWm; Tue, 13 Aug 2024 22:50:31 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 42918302D54; Wed, 14 Aug 2024 00:50:29 +0200 (CEST) Message-Id: <20240813224016.051225657@infradead.org> User-Agent: quilt/0.65 Date: Wed, 14 Aug 2024 00:25:53 +0200 From: Peter Zijlstra To: mingo@kernel.org, tj@kernel.org, void@manifault.com Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, linux-kernel@vger.kernel.org, joelaf@google.com Subject: [PATCH 5/9] sched: Rework pick_next_task() References: <20240813222548.049744955@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The current rule is that: pick_next_task() :=3D pick_task() + set_next_task(.first =3D true) And many classes implement it directly as such. Change things around to make pick_next_task() optional while also changing the definition to: pick_next_task(prev) :=3D pick_task() + put_prev_task() + set_next_task(.= first =3D true) The reason is that sched_ext would like to have a 'final' call that knows the next task. By placing put_prev_task() right next to set_next_task() (as it already is for sched_core) this becomes trivial. As a bonus, this is a nice cleanup on its own. Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/core.c | 21 +++++++++++++++------ kernel/sched/deadline.c | 21 +-------------------- kernel/sched/fair.c | 11 +++++------ kernel/sched/idle.c | 16 ++-------------- kernel/sched/rt.c | 13 +------------ kernel/sched/sched.h | 16 ++++++++++++---- kernel/sched/stop_task.c | 13 +------------ 7 files changed, 37 insertions(+), 74 deletions(-) --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5893,8 +5893,9 @@ __pick_next_task(struct rq *rq, struct t =20 /* Assume the next prioritized class is idle_sched_class */ if (!p) { + p =3D pick_task_idle(rq); put_prev_task(rq, prev); - p =3D pick_next_task_idle(rq); + set_next_task_first(rq, p); } =20 /* @@ -5916,12 +5917,20 @@ __pick_next_task(struct rq *rq, struct t =20 restart: prev_balance(rq, prev, rf); - put_prev_task(rq, prev); =20 for_each_class(class) { - p =3D class->pick_next_task(rq); - if (p) - return p; + if (class->pick_next_task) { + p =3D class->pick_next_task(rq, prev); + if (p) + return p; + } else { + p =3D class->pick_task(rq); + if (p) { + put_prev_task(rq, prev); + set_next_task_first(rq, p); + return p; + } + } } =20 BUG(); /* The idle class should always have a runnable task. */ @@ -6017,7 +6026,6 @@ pick_next_task(struct rq *rq, struct tas } =20 prev_balance(rq, prev, rf); - put_prev_task(rq, prev); =20 smt_mask =3D cpu_smt_mask(cpu); need_sync =3D !!rq->core->core_cookie; @@ -6184,6 +6192,7 @@ pick_next_task(struct rq *rq, struct tas } =20 out_set_next: + put_prev_task(rq, prev); set_next_task_first(rq, next); out: if (rq->core->core_forceidle_count && next =3D=3D rq->idle) --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2437,28 +2437,10 @@ static struct task_struct *__pick_task_d return p; } =20 -#ifdef CONFIG_SMP static struct task_struct *pick_task_dl(struct rq *rq) { return __pick_task_dl(rq); } -#endif - -static struct task_struct *pick_next_task_dl(struct rq *rq) -{ - struct task_struct *p; - - p =3D __pick_task_dl(rq); - if (!p) - return p; - - if (p->dl_server) - p->sched_class->set_next_task(rq, p, true); - else - set_next_task_dl(rq, p, true); - - return p; -} =20 static void put_prev_task_dl(struct rq *rq, struct task_struct *p) { @@ -3152,13 +3134,12 @@ DEFINE_SCHED_CLASS(dl) =3D { =20 .wakeup_preempt =3D wakeup_preempt_dl, =20 - .pick_next_task =3D pick_next_task_dl, + .pick_task =3D pick_task_dl, .put_prev_task =3D put_prev_task_dl, .set_next_task =3D set_next_task_dl, =20 #ifdef CONFIG_SMP .balance =3D balance_dl, - .pick_task =3D pick_task_dl, .select_task_rq =3D select_task_rq_dl, .migrate_task_rq =3D migrate_task_rq_dl, .set_cpus_allowed =3D set_cpus_allowed_dl, --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8775,7 +8775,7 @@ pick_next_task_fair(struct rq *rq, struc se =3D &p->se; =20 #ifdef CONFIG_FAIR_GROUP_SCHED - if (!prev || prev->sched_class !=3D &fair_sched_class) + if (prev->sched_class !=3D &fair_sched_class) goto simple; =20 /* @@ -8817,8 +8817,7 @@ pick_next_task_fair(struct rq *rq, struc =20 simple: #endif - if (prev) - put_prev_task(rq, prev); + put_prev_task(rq, prev); set_next_task_fair(rq, p, true); return p; =20 @@ -8848,9 +8847,9 @@ pick_next_task_fair(struct rq *rq, struc return NULL; } =20 -static struct task_struct *__pick_next_task_fair(struct rq *rq) +static struct task_struct *__pick_next_task_fair(struct rq *rq, struct tas= k_struct *prev) { - return pick_next_task_fair(rq, NULL, NULL); + return pick_next_task_fair(rq, prev, NULL); } =20 static bool fair_server_has_tasks(struct sched_dl_entity *dl_se) @@ -13487,13 +13486,13 @@ DEFINE_SCHED_CLASS(fair) =3D { =20 .wakeup_preempt =3D check_preempt_wakeup_fair, =20 + .pick_task =3D pick_task_fair, .pick_next_task =3D __pick_next_task_fair, .put_prev_task =3D put_prev_task_fair, .set_next_task =3D set_next_task_fair, =20 #ifdef CONFIG_SMP .balance =3D balance_fair, - .pick_task =3D pick_task_fair, .select_task_rq =3D select_task_rq_fair, .migrate_task_rq =3D migrate_task_rq_fair, =20 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -462,21 +462,10 @@ static void set_next_task_idle(struct rq next->se.exec_start =3D rq_clock_task(rq); } =20 -#ifdef CONFIG_SMP -static struct task_struct *pick_task_idle(struct rq *rq) +struct task_struct *pick_task_idle(struct rq *rq) { return rq->idle; } -#endif - -struct task_struct *pick_next_task_idle(struct rq *rq) -{ - struct task_struct *next =3D rq->idle; - - set_next_task_idle(rq, next, true); - - return next; -} =20 /* * It is not legal to sleep in the idle task - print a warning @@ -531,13 +520,12 @@ DEFINE_SCHED_CLASS(idle) =3D { =20 .wakeup_preempt =3D wakeup_preempt_idle, =20 - .pick_next_task =3D pick_next_task_idle, + .pick_task =3D pick_task_idle, .put_prev_task =3D put_prev_task_idle, .set_next_task =3D set_next_task_idle, =20 #ifdef CONFIG_SMP .balance =3D balance_idle, - .pick_task =3D pick_task_idle, .select_task_rq =3D select_task_rq_idle, .set_cpus_allowed =3D set_cpus_allowed_common, #endif --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1748,16 +1748,6 @@ static struct task_struct *pick_task_rt( return p; } =20 -static struct task_struct *pick_next_task_rt(struct rq *rq) -{ - struct task_struct *p =3D pick_task_rt(rq); - - if (p) - set_next_task_rt(rq, p, true); - - return p; -} - static void put_prev_task_rt(struct rq *rq, struct task_struct *p) { struct sched_rt_entity *rt_se =3D &p->rt; @@ -2645,13 +2635,12 @@ DEFINE_SCHED_CLASS(rt) =3D { =20 .wakeup_preempt =3D wakeup_preempt_rt, =20 - .pick_next_task =3D pick_next_task_rt, + .pick_task =3D pick_task_rt, .put_prev_task =3D put_prev_task_rt, .set_next_task =3D set_next_task_rt, =20 #ifdef CONFIG_SMP .balance =3D balance_rt, - .pick_task =3D pick_task_rt, .select_task_rq =3D select_task_rq_rt, .set_cpus_allowed =3D set_cpus_allowed_common, .rq_online =3D rq_online_rt, --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2300,7 +2300,17 @@ struct sched_class { =20 void (*wakeup_preempt)(struct rq *rq, struct task_struct *p, int flags); =20 - struct task_struct *(*pick_next_task)(struct rq *rq); + struct task_struct *(*pick_task)(struct rq *rq); + /* + * Optional! When implemented pick_next_task() should be equivalent to: + * + * next =3D pick_task(); + * if (next) { + * put_prev_task(prev); + * set_next_task_first(next); + * } + */ + struct task_struct *(*pick_next_task)(struct rq *rq, struct task_struct *= prev); =20 void (*put_prev_task)(struct rq *rq, struct task_struct *p); void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first); @@ -2309,8 +2319,6 @@ struct sched_class { int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *= rf); int (*select_task_rq)(struct task_struct *p, int task_cpu, int flags); =20 - struct task_struct * (*pick_task)(struct rq *rq); - void (*migrate_task_rq)(struct task_struct *p, int new_cpu); =20 void (*task_woken)(struct rq *this_rq, struct task_struct *task); @@ -2421,7 +2429,7 @@ static inline bool sched_fair_runnable(s } =20 extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_= struct *prev, struct rq_flags *rf); -extern struct task_struct *pick_next_task_idle(struct rq *rq); +extern struct task_struct *pick_task_idle(struct rq *rq); =20 #define SCA_CHECK 0x01 #define SCA_MIGRATE_DISABLE 0x02 --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -41,16 +41,6 @@ static struct task_struct *pick_task_sto return rq->stop; } =20 -static struct task_struct *pick_next_task_stop(struct rq *rq) -{ - struct task_struct *p =3D pick_task_stop(rq); - - if (p) - set_next_task_stop(rq, p, true); - - return p; -} - static void enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags) { @@ -112,13 +102,12 @@ DEFINE_SCHED_CLASS(stop) =3D { =20 .wakeup_preempt =3D wakeup_preempt_stop, =20 - .pick_next_task =3D pick_next_task_stop, + .pick_task =3D pick_task_stop, .put_prev_task =3D put_prev_task_stop, .set_next_task =3D set_next_task_stop, =20 #ifdef CONFIG_SMP .balance =3D balance_stop, - .pick_task =3D pick_task_stop, .select_task_rq =3D select_task_rq_stop, .set_cpus_allowed =3D set_cpus_allowed_common, #endif From nobody Sat Feb 7 19:45:44 2026 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C0C911A707C for ; Tue, 13 Aug 2024 22:50:38 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589442; cv=none; b=ELdaXPSdw1Kyohfwam0lP5oL3vnssGD0MfTzj4U6f3duQjgRL20ejbmpTIWEMhnhy/Z9ZKyDCcdc9ADJfqsq/ztQXJ/YS2Z3VTfuw80LTYFWKJN+tR68xJTwtKmlrnK9T4psKhKcLduuDvfIhQFnsqpcC/2AssfqjcoPewAxdxk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589442; c=relaxed/simple; bh=TxWCzl4+hNsTYCaW4f6uKRhQxDd87wdqETPidEJ/etU=; h=Message-Id:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=OAyu+IVQnEtHMNRUi1iYn9wCbXSszpnpUi2vf5ZRTjpA0xu2kFaK1jvyC6MivMyI0wrSrMP3EBwdr76GFMTJrUPvRrRLSm8UG4IpqzJfyEcB3VTDtzMR9UcpjDp5AMr9pvsT/bZpvjd5EpphCR6qQopSdtM7vfrzFx199lMkOZ4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=rXhvFuwe; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="rXhvFuwe" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=92noLzsJHL60m/QmsKdMpJSMk3hBlxpRjHA3o4EaS/s=; b=rXhvFuweWltESHRag9cir4ZZYI S1cQCp18WDW4+qjpmRUT93VDm+r3TDL18afGrCgtaLUEXkgw6Ub4FqasqvlLjHIi/RTla7uoXx6/N JBitBIa1UX/u/iXKBWmSYcQ8dlCRU3X5+W25vc9GzVufY1LAH9636fUVEze2rWT1QNlFAHhApVPGh hNwbDBpmgNPrGtec8E29BbP94Ly3pwQeCF3S9zn3TCB84E3CGmrOXXKoah4AsRTbCyrp2s00cKepn yjixoo3kN1hJyZ6PwZ+6ULCXMl4G17wUcuYAcDqjFWiswwjFxdDT+hd9fim4SmqFVV6OPYFjzC5z7 AWSEeCSw==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.97.1 #2 (Red Hat Linux)) id 1se0LW-00000007s1A-3W6Q; Tue, 13 Aug 2024 22:50:31 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 48BBB302D64; Wed, 14 Aug 2024 00:50:29 +0200 (CEST) Message-Id: <20240813224016.158454756@infradead.org> User-Agent: quilt/0.65 Date: Wed, 14 Aug 2024 00:25:54 +0200 From: Peter Zijlstra To: mingo@kernel.org, tj@kernel.org, void@manifault.com Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, linux-kernel@vger.kernel.org, joelaf@google.com Subject: [PATCH 6/9] sched: Combine the last put_prev_task() and the first set_next_task() References: <20240813222548.049744955@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Ensure the last put_prev_task() and the first set_next_task() always go together. Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/core.c | 17 ++++------------- kernel/sched/fair.c | 3 +-- kernel/sched/sched.h | 10 +++++++++- 3 files changed, 14 insertions(+), 16 deletions(-) --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5894,8 +5894,7 @@ __pick_next_task(struct rq *rq, struct t /* Assume the next prioritized class is idle_sched_class */ if (!p) { p =3D pick_task_idle(rq); - put_prev_task(rq, prev); - set_next_task_first(rq, p); + put_prev_set_next_task(rq, prev, p); } =20 /* @@ -5926,8 +5925,7 @@ __pick_next_task(struct rq *rq, struct t } else { p =3D class->pick_task(rq); if (p) { - put_prev_task(rq, prev); - set_next_task_first(rq, p); + put_prev_set_next_task(rq, prev, p); return p; } } @@ -6016,13 +6014,8 @@ pick_next_task(struct rq *rq, struct tas WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); =20 next =3D rq->core_pick; - if (next !=3D prev) { - put_prev_task(rq, prev); - set_next_task_first(rq, next); - } - rq->core_pick =3D NULL; - goto out; + goto out_set_next; } =20 prev_balance(rq, prev, rf); @@ -6192,9 +6185,7 @@ pick_next_task(struct rq *rq, struct tas } =20 out_set_next: - put_prev_task(rq, prev); - set_next_task_first(rq, next); -out: + put_prev_set_next_task(rq, prev, next); if (rq->core->core_forceidle_count && next =3D=3D rq->idle) queue_core_balance(rq); =20 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8817,8 +8817,7 @@ pick_next_task_fair(struct rq *rq, struc =20 simple: #endif - put_prev_task(rq, prev); - set_next_task_fair(rq, p, true); + put_prev_set_next_task(rq, prev, p); return p; =20 idle: --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2370,8 +2370,16 @@ static inline void set_next_task(struct next->sched_class->set_next_task(rq, next, false); } =20 -static inline void set_next_task_first(struct rq *rq, struct task_struct *= next) +static inline void put_prev_set_next_task(struct rq *rq, + struct task_struct *prev, + struct task_struct *next) { + WARN_ON_ONCE(rq->curr !=3D prev); + + if (next =3D=3D prev) + return; + + prev->sched_class->put_prev_task(rq, prev); next->sched_class->set_next_task(rq, next, true); } From nobody Sat Feb 7 19:45:44 2026 Received: from desiato.infradead.org (desiato.infradead.org [90.155.92.199]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A3AB71AB521 for ; Tue, 13 Aug 2024 22:50:39 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.92.199 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589441; cv=none; b=d+zeVjL5VGZZD5f25NCePBbYGuoSvxahpO+NC/yVkTwkOEbnlpRAu8Op/vfOi4Izp5YlbDmHBbpL3d7CrhdD9eL42SvY5df6jQ0qgtTTW0FqB8gczTRcnExBddGHILEfUE7B3DyVX3l4PUGgh/B1PJaMz6t85IloxZ3FBHSO/BQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589441; c=relaxed/simple; bh=hs0O2c+oaqUZ5z6NJS1vZ9jFt7Y19FGt3qinKMe6eb8=; h=Message-Id:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=uVESUlNV18Q2V8DjULu4GUCtII4IIqJZlcAx41YJWnQ92dbC7L+EykWUAWeCO5f9X8oPwhySoBBhKi0qGUNoMHL67vpulvUtd/Am2EOjbWD7CqxuA9fdVZL+0OyhdIN7zQ9K6+JiRT6wEbmI531IedCcTKwq5GchH4WgbG2IZzs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=CjZmpRHR; arc=none smtp.client-ip=90.155.92.199 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="CjZmpRHR" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=desiato.20200630; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=K6xq1Qbw3/PGWUvKoHwS4tDLQnVlHu3ZZwEzlKsVe/E=; b=CjZmpRHRUh2Zjmcc4oIDFX9gst PWQ6T6mwuInZ/Q0oPA1RhgspojtGGN83m8AIJR2p6uvar82Db2saB73zcNxPdeA13AFGJb40BJvOF UwA5e67BL0oeS9Z0RcsrMOHBMgturCoGGVx7CjLC8A1rbXS4qnD/JqCXot9c5Uwtwsw/bfkQNqImR Fpxog/tmecXvRm1nuhOJOASL/jlKiiR2PPacR7oUOlN1MTU98S+P1KI9iBI6Y2du4Lqvi+7LXssRV roEA1+nQ6W9a8pQ30/OnaIbmEawa7+VlkFqF0Yz8rnwtDgVob0LNt2UpxZ3No4AFDsuCB18oxwg7n jCKbRnVQ==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by desiato.infradead.org with esmtpsa (Exim 4.97.1 #2 (Red Hat Linux)) id 1se0LX-00000007s1F-3ZNJ; Tue, 13 Aug 2024 22:50:32 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 4EDFA302D79; Wed, 14 Aug 2024 00:50:29 +0200 (CEST) Message-Id: <20240813224016.259853414@infradead.org> User-Agent: quilt/0.65 Date: Wed, 14 Aug 2024 00:25:55 +0200 From: Peter Zijlstra To: mingo@kernel.org, tj@kernel.org, void@manifault.com Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, linux-kernel@vger.kernel.org, joelaf@google.com Subject: [PATCH 7/9] sched: Rework dl_server References: <20240813222548.049744955@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" When a task is selected through a dl_server, it will have p->dl_server set, such that it can account runtime to the dl_server, see update_curr_task(). Currently p->dl_server is set in pick*task() whenever it goes through the dl_server, clearing it is a bit of a mess though. The trivial solution is clearing it on the final put (now that we have this location). However, this gives a problem when: p =3D pick_task(rq); if (p) put_prev_set_next_task(rq, prev, next); picks the same task but through a different path, notably when it goes from picking through the dl_server to a direct pick or vice-versa. In that case we cannot readily determine wether we should clear or preserve p->dl_server. An additional complication is pick_*task() setting p->dl_server for a remote pick, it might still need to update runtime before it schedules the core_pick. Close all these holes and remove all the random clearing of p->dl_server by: - having pick_*task() manage rq->dl_server - having the final put_prev_task() clear p->dl_server - having the first set_next_task() set p->dl_server =3D rq->dl_server - complicate the core_sched code to save/restore rq->dl_server where appropriate. Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/core.c | 40 +++++++++++++++------------------------- kernel/sched/deadline.c | 2 +- kernel/sched/fair.c | 10 ++-------- kernel/sched/sched.h | 14 ++++++++++++++ 4 files changed, 32 insertions(+), 34 deletions(-) --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3668,8 +3668,6 @@ ttwu_do_activate(struct rq *rq, struct t rq->idle_stamp =3D 0; } #endif - - p->dl_server =3D NULL; } =20 /* @@ -5859,14 +5857,6 @@ static void prev_balance(struct rq *rq, break; } #endif - - /* - * We've updated @prev and no longer need the server link, clear it. - * Must be done before ->pick_next_task() because that can (re)set - * ->dl_server. - */ - if (prev->dl_server) - prev->dl_server =3D NULL; } =20 /* @@ -5878,6 +5868,8 @@ __pick_next_task(struct rq *rq, struct t const struct sched_class *class; struct task_struct *p; =20 + rq->dl_server =3D NULL; + /* * Optimization: we know that if all tasks are in the fair class we can * call that function directly, but only if the @prev task wasn't of a @@ -5897,20 +5889,6 @@ __pick_next_task(struct rq *rq, struct t put_prev_set_next_task(rq, prev, p); } =20 - /* - * This is a normal CFS pick, but the previous could be a DL pick. - * Clear it as previous is no longer picked. - */ - if (prev->dl_server) - prev->dl_server =3D NULL; - - /* - * This is the fast path; it cannot be a DL server pick; - * therefore even if @p =3D=3D @prev, ->dl_server must be NULL. - */ - if (p->dl_server) - p->dl_server =3D NULL; - return p; } =20 @@ -5958,6 +5936,8 @@ static inline struct task_struct *pick_t const struct sched_class *class; struct task_struct *p; =20 + rq->dl_server =3D NULL; + for_each_class(class) { p =3D class->pick_task(rq); if (p) @@ -5996,6 +5976,7 @@ pick_next_task(struct rq *rq, struct tas * another cpu during offline. */ rq->core_pick =3D NULL; + rq->core_dl_server =3D NULL; return __pick_next_task(rq, prev, rf); } =20 @@ -6014,7 +5995,9 @@ pick_next_task(struct rq *rq, struct tas WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); =20 next =3D rq->core_pick; + rq->dl_server =3D rq->core_dl_server; rq->core_pick =3D NULL; + rq->core_dl_server =3D NULL; goto out_set_next; } =20 @@ -6059,6 +6042,7 @@ pick_next_task(struct rq *rq, struct tas next =3D pick_task(rq); if (!next->core_cookie) { rq->core_pick =3D NULL; + rq->core_dl_server =3D NULL; /* * For robustness, update the min_vruntime_fi for * unconstrained picks as well. @@ -6086,7 +6070,9 @@ pick_next_task(struct rq *rq, struct tas if (i !=3D cpu && (rq_i !=3D rq->core || !core_clock_updated)) update_rq_clock(rq_i); =20 - p =3D rq_i->core_pick =3D pick_task(rq_i); + rq_i->core_pick =3D p =3D pick_task(rq_i); + rq_i->core_dl_server =3D rq_i->dl_server; + if (!max || prio_less(max, p, fi_before)) max =3D p; } @@ -6110,6 +6096,7 @@ pick_next_task(struct rq *rq, struct tas } =20 rq_i->core_pick =3D p; + rq_i->core_dl_server =3D NULL; =20 if (p =3D=3D rq_i->idle) { if (rq_i->nr_running) { @@ -6170,6 +6157,7 @@ pick_next_task(struct rq *rq, struct tas =20 if (i =3D=3D cpu) { rq_i->core_pick =3D NULL; + rq_i->core_dl_server =3D NULL; continue; } =20 @@ -6178,6 +6166,7 @@ pick_next_task(struct rq *rq, struct tas =20 if (rq_i->curr =3D=3D rq_i->core_pick) { rq_i->core_pick =3D NULL; + rq_i->core_dl_server =3D NULL; continue; } =20 @@ -8401,6 +8390,7 @@ void __init sched_init(void) #ifdef CONFIG_SCHED_CORE rq->core =3D rq; rq->core_pick =3D NULL; + rq->core_dl_server =3D NULL; rq->core_enabled =3D 0; rq->core_tree =3D RB_ROOT; rq->core_forceidle_count =3D 0; --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2429,7 +2429,7 @@ static struct task_struct *__pick_task_d update_curr_dl_se(rq, dl_se, 0); goto again; } - p->dl_server =3D dl_se; + rq->dl_server =3D dl_se; } else { p =3D dl_task_of(dl_se); } --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8747,14 +8747,6 @@ static struct task_struct *pick_task_fai cfs_rq =3D group_cfs_rq(se); } while (cfs_rq); =20 - /* - * This can be called from directly from CFS's ->pick_task() or indirectly - * from DL's ->pick_task when fair server is enabled. In the indirect cas= e, - * DL will set ->dl_server just after this function is called, so its Ok = to - * clear. In the direct case, we are picking directly so we must clear it. - */ - task_of(se)->dl_server =3D NULL; - return task_of(se); } =20 @@ -8778,6 +8770,8 @@ pick_next_task_fair(struct rq *rq, struc if (prev->sched_class !=3D &fair_sched_class) goto simple; =20 + __put_prev_set_next_dl_server(rq, prev, p); + /* * Because of the set_next_buddy() in dequeue_task_fair() it is rather * likely that a next task is from the same cgroup as the current. --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1066,6 +1066,7 @@ struct rq { unsigned int nr_uninterruptible; =20 struct task_struct __rcu *curr; + struct sched_dl_entity *dl_server; struct task_struct *idle; struct task_struct *stop; unsigned long next_balance; @@ -1193,6 +1194,7 @@ struct rq { /* per rq */ struct rq *core; struct task_struct *core_pick; + struct sched_dl_entity *core_dl_server; unsigned int core_enabled; unsigned int core_sched_seq; struct rb_root core_tree; @@ -2370,12 +2372,24 @@ static inline void set_next_task(struct next->sched_class->set_next_task(rq, next, false); } =20 +static inline void +__put_prev_set_next_dl_server(struct rq *rq, + struct task_struct *prev, + struct task_struct *next) +{ + prev->dl_server =3D NULL; + next->dl_server =3D rq->dl_server; + rq->dl_server =3D NULL; +} + static inline void put_prev_set_next_task(struct rq *rq, struct task_struct *prev, struct task_struct *next) { WARN_ON_ONCE(rq->curr !=3D prev); =20 + __put_prev_set_next_dl_server(rq, prev, next); + if (next =3D=3D prev) return; From nobody Sat Feb 7 19:45:44 2026 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 23D791AB524 for ; Tue, 13 Aug 2024 22:50:39 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589441; cv=none; b=uyB+NZGwWrywTLtqCqzFx4YiqGcOFvq8Xv+f6oarA1Atc/A/so990JXqgJfnFLJABz1BzsP4X99SobItMY+kuIlowhfVDt9H8iHoQewGLYCM6hITSeqw+1++19Kr43uXvHtYwvkHEgAV6lwICLPZ7X1kSzaWAcDbEgwtu/F3ogE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589441; c=relaxed/simple; bh=4EoFx1f69wqqPGysup50POHm+uC/DFxaLP2Z3d8HMD4=; h=Message-Id:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=QgPKCxzSbOhJS09MFzGAcysSLsbvjWjMenrS2U08m1BrMMXA497AuRvEAomieANS9rRzCOELNyM9IlR8h/52JUcHQklkJSr9kJxppN5jf2oHjvKBc8utUmsrAccYqbzjZpsYfQSpMEq1CMDxXbB4FEp5cl5FjB5wd04o1Gd+Oqk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=AugU10fV; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="AugU10fV" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=4g36lKE3UumaT2D2H6UQxYvi04wWUVzx4R2hCuMHUWw=; b=AugU10fVMPB10fW3/qQjEvoAKF vaEv0MWfZ5IXLz0RJdM+gfKIzYYxS0O5V7bkLj4yl0Clvh9oQplpRW2xmhb8dIwxdEl3D1eBV5Qwh aegGBP7eOatbBXSX359ZXcpHcT9qdrJZ7QQGEOMHL+3OqGmEkfRBRaMyxFH40SnwfaDl7y1DIWN7U 0q1+XLYATrEQDOvJP48Ia3XF+ZjPKMiEo3Djbjj/LBqbaK+YVDhEYOW0cWww74jHiJM/EAkt3/zVH Zl2Yg4Jptg5RRm5ZJc2bn7cunHoR6kiGVU4dC5ER2Z5gW0iBMedE+e2em9+Rm8/B2ATyySUL1FXbA KLbmY/dg==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.97.1 #2 (Red Hat Linux)) id 1se0LX-0000000H9ZW-3eU1; Tue, 13 Aug 2024 22:50:31 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 54E47302DAB; Wed, 14 Aug 2024 00:50:29 +0200 (CEST) Message-Id: <20240813224016.367421076@infradead.org> User-Agent: quilt/0.65 Date: Wed, 14 Aug 2024 00:25:56 +0200 From: Peter Zijlstra To: mingo@kernel.org, tj@kernel.org, void@manifault.com Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, linux-kernel@vger.kernel.org, joelaf@google.com Subject: [PATCH 8/9] sched: Add put_prev_task(.next) References: <20240813222548.049744955@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In order to tell the previous sched_class what the next task is, add put_prev_task(.next). Notable SCX will use this to: 1) determine the next task will leave the SCX sched class and push the current task to another CPU if possible. 2) statistics on how often and which other classes preempt it Signed-off-by: Peter Zijlstra (Intel) --- kernel/sched/deadline.c | 2 +- kernel/sched/fair.c | 2 +- kernel/sched/idle.c | 2 +- kernel/sched/rt.c | 2 +- kernel/sched/sched.h | 6 +++--- kernel/sched/stop_task.c | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2442,7 +2442,7 @@ static struct task_struct *pick_task_dl( return __pick_task_dl(rq); } =20 -static void put_prev_task_dl(struct rq *rq, struct task_struct *p) +static void put_prev_task_dl(struct rq *rq, struct task_struct *p, struct = task_struct *next) { struct sched_dl_entity *dl_se =3D &p->dl; struct dl_rq *dl_rq =3D &rq->dl; --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8867,7 +8867,7 @@ void fair_server_init(struct rq *rq) /* * Account for a descheduled task: */ -static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) +static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, st= ruct task_struct *next) { struct sched_entity *se =3D &prev->se; struct cfs_rq *cfs_rq; --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -450,7 +450,7 @@ static void wakeup_preempt_idle(struct r resched_curr(rq); } =20 -static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) +static void put_prev_task_idle(struct rq *rq, struct task_struct *prev, st= ruct task_struct *next) { dl_server_update_idle_time(rq, prev); } --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1748,7 +1748,7 @@ static struct task_struct *pick_task_rt( return p; } =20 -static void put_prev_task_rt(struct rq *rq, struct task_struct *p) +static void put_prev_task_rt(struct rq *rq, struct task_struct *p, struct = task_struct *next) { struct sched_rt_entity *rt_se =3D &p->rt; struct rt_rq *rt_rq =3D &rq->rt; --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2314,7 +2314,7 @@ struct sched_class { */ struct task_struct *(*pick_next_task)(struct rq *rq, struct task_struct *= prev); =20 - void (*put_prev_task)(struct rq *rq, struct task_struct *p); + void (*put_prev_task)(struct rq *rq, struct task_struct *p, struct task_s= truct *next); void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first); =20 #ifdef CONFIG_SMP @@ -2364,7 +2364,7 @@ struct sched_class { static inline void put_prev_task(struct rq *rq, struct task_struct *prev) { WARN_ON_ONCE(rq->curr !=3D prev); - prev->sched_class->put_prev_task(rq, prev); + prev->sched_class->put_prev_task(rq, prev, NULL); } =20 static inline void set_next_task(struct rq *rq, struct task_struct *next) @@ -2393,7 +2393,7 @@ static inline void put_prev_set_next_tas if (next =3D=3D prev) return; =20 - prev->sched_class->put_prev_task(rq, prev); + prev->sched_class->put_prev_task(rq, prev, next); next->sched_class->set_next_task(rq, next, true); } =20 --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -59,7 +59,7 @@ static void yield_task_stop(struct rq *r BUG(); /* the stop task should never yield, its pointless. */ } =20 -static void put_prev_task_stop(struct rq *rq, struct task_struct *prev) +static void put_prev_task_stop(struct rq *rq, struct task_struct *prev, st= ruct task_struct *next) { update_curr_common(rq); } From nobody Sat Feb 7 19:45:44 2026 Received: from casper.infradead.org (casper.infradead.org [90.155.50.34]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 64C351AB51F for ; Tue, 13 Aug 2024 22:50:39 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=90.155.50.34 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589441; cv=none; b=PE7MFeS0RFyHI2SiyJSldBWFicGxGv91I5QeLqeZNkkGwLs/GdcMLdUhaWTrkDNUTKToKzbAunGPuyRuF1a9KCksRJmVsCp0hiJJgNA5m799QfG7yHd4Pt7S3un1DmD6Di71YOr/A+1D9cJw8O6SDZCi//I7SqDeR4p4pDf1oug= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1723589441; c=relaxed/simple; bh=JDVcqKByKpkk/XFjkq6PeFMESkaTJiG8CDpqXb8Zuic=; h=Message-Id:Date:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=kPQqUXHGUyxa9rTPOept8RCZvdXIcHdvghzG/NwoRXVztteG7OsOgvZk7aLhhJcS91AgHgoJakR/DsCcVxQ09/i/9EWcS3Kga++1Kr2FLNPRz19pP1+namuRMtvxlr/v/pPYdCE8Alc/ui4SnacLPdmR90q/uFdR9iCUDqLvwxE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org; spf=none smtp.mailfrom=infradead.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b=hMIDXkBx; arc=none smtp.client-ip=90.155.50.34 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=infradead.org Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="hMIDXkBx" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Type:MIME-Version:References: Subject:Cc:To:From:Date:Message-Id:Sender:Reply-To:Content-Transfer-Encoding: Content-ID:Content-Description:In-Reply-To; bh=rPs5bSfp4cnPNS65oXy8urZjJQLEvY/WVkoVOqdQkmU=; b=hMIDXkBxwZ/gQ6mOaj4i3nTxRx COOjDGjg4iYVbxD4v2AX9hdLiS33irehZ7nb+ir4bHvqlzBEH8o/pMrxBNSwLVuwTFCSjdEOu3FW3 TvIfmaXc5+O11grusj4G/6A14YuuH7zMzGCRBwHLB0T0R8/CSGL19e0lKNn4GZE1AszDZXP2PM//W 00cx6W0ywkTfydlJAXXfZ7ecptxLfxYES/AvrRdm78mz1LBmdUfZcz8TsrStJnIFugNp0FdGQlfgs 4u7fPGfWnCHYGvPpi6ySgABRlY7IBVIQVc19GW9jg0hvyusJ0pspsHzcdtQB9Jg8Pt1nMNGvZht+8 KV+dHgNg==; Received: from j130084.upc-j.chello.nl ([24.132.130.84] helo=noisy.programming.kicks-ass.net) by casper.infradead.org with esmtpsa (Exim 4.97.1 #2 (Red Hat Linux)) id 1se0LY-0000000H9ZX-097E; Tue, 13 Aug 2024 22:50:32 +0000 Received: by noisy.programming.kicks-ass.net (Postfix, from userid 0) id 59F17302DB1; Wed, 14 Aug 2024 00:50:29 +0200 (CEST) Message-Id: <20240813224016.471745809@infradead.org> User-Agent: quilt/0.65 Date: Wed, 14 Aug 2024 00:25:57 +0200 From: Peter Zijlstra To: mingo@kernel.org, tj@kernel.org, void@manifault.com Cc: peterz@infradead.org, juri.lelli@redhat.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, rostedt@goodmis.org, bsegall@google.com, mgorman@suse.de, vschneid@redhat.com, linux-kernel@vger.kernel.org, joelaf@google.com Subject: [PATCH 9/9] sched: Add pick_task(.core) References: <20240813222548.049744955@infradead.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In order to distinguish between a regular vs a core pick_task() invocation, add a boolean argument. Notably SCX seems to need this, since its core pick Signed-off-by: Peter Zijlstra (Intel) --- include/linux/sched.h | 2 +- kernel/sched/core.c | 8 ++++---- kernel/sched/deadline.c | 9 ++------- kernel/sched/fair.c | 9 +++++---- kernel/sched/idle.c | 2 +- kernel/sched/rt.c | 2 +- kernel/sched/sched.h | 4 ++-- kernel/sched/stop_task.c | 2 +- 8 files changed, 17 insertions(+), 21 deletions(-) --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -601,7 +601,7 @@ struct sched_rt_entity { } __randomize_layout; =20 typedef bool (*dl_server_has_tasks_f)(struct sched_dl_entity *); -typedef struct task_struct *(*dl_server_pick_f)(struct sched_dl_entity *); +typedef struct task_struct *(*dl_server_pick_f)(struct sched_dl_entity *, = bool); =20 struct sched_dl_entity { struct rb_node rb_node; --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5885,7 +5885,7 @@ __pick_next_task(struct rq *rq, struct t =20 /* Assume the next prioritized class is idle_sched_class */ if (!p) { - p =3D pick_task_idle(rq); + p =3D pick_task_idle(rq, false); put_prev_set_next_task(rq, prev, p); } =20 @@ -5901,7 +5901,7 @@ __pick_next_task(struct rq *rq, struct t if (p) return p; } else { - p =3D class->pick_task(rq); + p =3D class->pick_task(rq, false); if (p) { put_prev_set_next_task(rq, prev, p); return p; @@ -5939,7 +5939,7 @@ static inline struct task_struct *pick_t rq->dl_server =3D NULL; =20 for_each_class(class) { - p =3D class->pick_task(rq); + p =3D class->pick_task(rq, true); if (p) return p; } @@ -6092,7 +6092,7 @@ pick_next_task(struct rq *rq, struct tas if (cookie) p =3D sched_core_find(rq_i, cookie); if (!p) - p =3D idle_sched_class.pick_task(rq_i); + p =3D idle_sched_class.pick_task(rq_i, true); } =20 rq_i->core_pick =3D p; --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2409,7 +2409,7 @@ static struct sched_dl_entity *pick_next * __pick_next_task_dl - Helper to pick the next -deadline task to run. * @rq: The runqueue to pick the next task from. */ -static struct task_struct *__pick_task_dl(struct rq *rq) +static struct task_struct *pick_task_dl(struct rq *rq, bool core) { struct sched_dl_entity *dl_se; struct dl_rq *dl_rq =3D &rq->dl; @@ -2423,7 +2423,7 @@ static struct task_struct *__pick_task_d WARN_ON_ONCE(!dl_se); =20 if (dl_server(dl_se)) { - p =3D dl_se->server_pick_task(dl_se); + p =3D dl_se->server_pick_task(dl_se, core); if (!p) { dl_se->dl_yielded =3D 1; update_curr_dl_se(rq, dl_se, 0); @@ -2437,11 +2437,6 @@ static struct task_struct *__pick_task_d return p; } =20 -static struct task_struct *pick_task_dl(struct rq *rq) -{ - return __pick_task_dl(rq); -} - static void put_prev_task_dl(struct rq *rq, struct task_struct *p, struct = task_struct *next) { struct sched_dl_entity *dl_se =3D &p->dl; --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8723,7 +8723,7 @@ static void check_preempt_wakeup_fair(st resched_curr(rq); } =20 -static struct task_struct *pick_task_fair(struct rq *rq) +static struct task_struct *pick_task_fair(struct rq *rq, bool core) { struct sched_entity *se; struct cfs_rq *cfs_rq; @@ -8761,7 +8761,7 @@ pick_next_task_fair(struct rq *rq, struc int new_tasks; =20 again: - p =3D pick_task_fair(rq); + p =3D pick_task_fair(rq, false); if (!p) goto idle; se =3D &p->se; @@ -8850,9 +8850,10 @@ static bool fair_server_has_tasks(struct return !!dl_se->rq->cfs.nr_running; } =20 -static struct task_struct *fair_server_pick_task(struct sched_dl_entity *d= l_se) +static struct task_struct * +fair_server_pick_task(struct sched_dl_entity *dl_se, bool core) { - return pick_task_fair(dl_se->rq); + return pick_task_fair(dl_se->rq, core); } =20 void fair_server_init(struct rq *rq) --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -462,7 +462,7 @@ static void set_next_task_idle(struct rq next->se.exec_start =3D rq_clock_task(rq); } =20 -struct task_struct *pick_task_idle(struct rq *rq) +struct task_struct *pick_task_idle(struct rq *rq, bool core) { return rq->idle; } --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1736,7 +1736,7 @@ static struct task_struct *_pick_next_ta return rt_task_of(rt_se); } =20 -static struct task_struct *pick_task_rt(struct rq *rq) +static struct task_struct *pick_task_rt(struct rq *rq, bool core) { struct task_struct *p; =20 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2302,7 +2302,7 @@ struct sched_class { =20 void (*wakeup_preempt)(struct rq *rq, struct task_struct *p, int flags); =20 - struct task_struct *(*pick_task)(struct rq *rq); + struct task_struct *(*pick_task)(struct rq *rq, bool core); /* * Optional! When implemented pick_next_task() should be equivalent to: * @@ -2451,7 +2451,7 @@ static inline bool sched_fair_runnable(s } =20 extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_= struct *prev, struct rq_flags *rf); -extern struct task_struct *pick_task_idle(struct rq *rq); +extern struct task_struct *pick_task_idle(struct rq *rq, bool core); =20 #define SCA_CHECK 0x01 #define SCA_MIGRATE_DISABLE 0x02 --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -33,7 +33,7 @@ static void set_next_task_stop(struct rq stop->se.exec_start =3D rq_clock_task(rq); } =20 -static struct task_struct *pick_task_stop(struct rq *rq) +static struct task_struct *pick_task_stop(struct rq *rq, bool core) { if (!sched_stop_runnable(rq)) return NULL;