The idea here is we want to charge the scheduler-context task's
vruntime but charge the execution-context task's sum_exec_runtime.
This way cputime accounting goes against the task actually running
but vruntime accounting goes against the rq->donor task so we get
proper fairness.
Cc: Joel Fernandes <joelaf@google.com>
Cc: Qais Yousef <qyousef@layalina.io>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Juri Lelli <juri.lelli@redhat.com>
Cc: Vincent Guittot <vincent.guittot@linaro.org>
Cc: Dietmar Eggemann <dietmar.eggemann@arm.com>
Cc: Valentin Schneider <vschneid@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Ben Segall <bsegall@google.com>
Cc: Zimuzo Ezeozue <zezeozue@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Will Deacon <will@kernel.org>
Cc: Waiman Long <longman@redhat.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: "Paul E. McKenney" <paulmck@kernel.org>
Cc: Metin Kaya <Metin.Kaya@arm.com>
Cc: Xuewen Yan <xuewen.yan94@gmail.com>
Cc: K Prateek Nayak <kprateek.nayak@amd.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Daniel Lezcano <daniel.lezcano@linaro.org>
Cc: kernel-team@android.com
Signed-off-by: John Stultz <jstultz@google.com>
---
kernel/sched/fair.c | 21 ++++++++++++++++-----
1 file changed, 16 insertions(+), 5 deletions(-)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index fbdca89c677f4..ebde314e151f1 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1131,22 +1131,33 @@ static void update_tg_load_avg(struct cfs_rq *cfs_rq)
}
#endif /* CONFIG_SMP */
-static s64 update_curr_se(struct rq *rq, struct sched_entity *curr)
+static s64 update_curr_se(struct rq *rq, struct sched_entity *se)
{
u64 now = rq_clock_task(rq);
s64 delta_exec;
- delta_exec = now - curr->exec_start;
+ delta_exec = now - se->exec_start;
if (unlikely(delta_exec <= 0))
return delta_exec;
- curr->exec_start = now;
- curr->sum_exec_runtime += delta_exec;
+ se->exec_start = now;
+ if (entity_is_task(se)) {
+ struct task_struct *running = rq->curr;
+ /*
+ * If se is a task, we account the time against the running
+ * task, as w/ proxy-exec they may not be the same.
+ */
+ running->se.exec_start = now;
+ running->se.sum_exec_runtime += delta_exec;
+ } else {
+ /* If not task, account the time against se */
+ se->sum_exec_runtime += delta_exec;
+ }
if (schedstat_enabled()) {
struct sched_statistics *stats;
- stats = __schedstats_from_se(curr);
+ stats = __schedstats_from_se(se);
__schedstat_set(stats->exec_max,
max(delta_exec, stats->exec_max));
}
--
2.47.0.371.ga323438b13-googOn Mon, Nov 25, 2024 at 11:51:57AM -0800, John Stultz wrote:
> -static s64 update_curr_se(struct rq *rq, struct sched_entity *curr)
> +static s64 update_curr_se(struct rq *rq, struct sched_entity *se)
> {
> u64 now = rq_clock_task(rq);
> s64 delta_exec;
>
> - delta_exec = now - curr->exec_start;
> + delta_exec = now - se->exec_start;
> if (unlikely(delta_exec <= 0))
> return delta_exec;
>
> - curr->exec_start = now;
> - curr->sum_exec_runtime += delta_exec;
> + se->exec_start = now;
> + if (entity_is_task(se)) {
> + struct task_struct *running = rq->curr;
> + /*
> + * If se is a task, we account the time against the running
> + * task, as w/ proxy-exec they may not be the same.
> + */
> + running->se.exec_start = now;
> + running->se.sum_exec_runtime += delta_exec;
> + } else {
> + /* If not task, account the time against se */
> + se->sum_exec_runtime += delta_exec;
> + }
>
> if (schedstat_enabled()) {
> struct sched_statistics *stats;
>
> - stats = __schedstats_from_se(curr);
> + stats = __schedstats_from_se(se);
> __schedstat_set(stats->exec_max,
> max(delta_exec, stats->exec_max));
> }
Would it not be *much* clearer if we do it like:
static s64 update_curr_se(struct rq *rq, struct sched_entity *donor,
struct sched_entity *curr)
{
...
donor->exec_start = now;
curr->exec_start = now;
curr->sum_exec_runtime += delta_exec;
...
}
and update the callsites like so:
update_curr_common()
update_curr_se(rq, &donor->se, &rq->curr.se)
update_curr()
update_curr_se(rq, &curr->se, &curr->se);
except, now I'm confused about the update_curr() case. That seems to
always update the execution context, rather than the donor ?
On Sat, Dec 14, 2024 at 12:37:40AM +0100, Peter Zijlstra wrote:
> On Mon, Nov 25, 2024 at 11:51:57AM -0800, John Stultz wrote:
>
>
>
> > -static s64 update_curr_se(struct rq *rq, struct sched_entity *curr)
> > +static s64 update_curr_se(struct rq *rq, struct sched_entity *se)
> > {
> > u64 now = rq_clock_task(rq);
> > s64 delta_exec;
> >
> > - delta_exec = now - curr->exec_start;
> > + delta_exec = now - se->exec_start;
> > if (unlikely(delta_exec <= 0))
> > return delta_exec;
> >
> > - curr->exec_start = now;
> > - curr->sum_exec_runtime += delta_exec;
> > + se->exec_start = now;
> > + if (entity_is_task(se)) {
> > + struct task_struct *running = rq->curr;
> > + /*
> > + * If se is a task, we account the time against the running
> > + * task, as w/ proxy-exec they may not be the same.
> > + */
> > + running->se.exec_start = now;
> > + running->se.sum_exec_runtime += delta_exec;
> > + } else {
> > + /* If not task, account the time against se */
> > + se->sum_exec_runtime += delta_exec;
> > + }
> >
> > if (schedstat_enabled()) {
> > struct sched_statistics *stats;
> >
> > - stats = __schedstats_from_se(curr);
> > + stats = __schedstats_from_se(se);
> > __schedstat_set(stats->exec_max,
> > max(delta_exec, stats->exec_max));
> > }
>
> Would it not be *much* clearer if we do it like:
>
> static s64 update_curr_se(struct rq *rq, struct sched_entity *donor,
> struct sched_entity *curr)
> {
> ...
> donor->exec_start = now;
> curr->exec_start = now;
> curr->sum_exec_runtime += delta_exec;
> ...
> }
>
> and update the callsites like so:
>
> update_curr_common()
> update_curr_se(rq, &donor->se, &rq->curr.se)
>
> update_curr()
> update_curr_se(rq, &curr->se, &curr->se);
>
>
> except, now I'm confused about the update_curr() case. That seems to
> always update the execution context, rather than the donor ?
Ah no, cfs_rq->curr is the donor.
I'll try again later; or risk keyboard face.. Zzzz
On Fri, Dec 13, 2024 at 4:10 PM Peter Zijlstra <peterz@infradead.org> wrote:
> On Sat, Dec 14, 2024 at 12:37:40AM +0100, Peter Zijlstra wrote:
> > Would it not be *much* clearer if we do it like:
> >
> > static s64 update_curr_se(struct rq *rq, struct sched_entity *donor,
> > struct sched_entity *curr)
> > {
> > ...
> > donor->exec_start = now;
> > curr->exec_start = now;
> > curr->sum_exec_runtime += delta_exec;
> > ...
> > }
> >
> > and update the callsites like so:
> >
> > update_curr_common()
> > update_curr_se(rq, &donor->se, &rq->curr.se)
> >
> > update_curr()
> > update_curr_se(rq, &curr->se, &curr->se);
> >
> >
> > except, now I'm confused about the update_curr() case. That seems to
> > always update the execution context, rather than the donor ?
>
> Ah no, cfs_rq->curr is the donor.
Yeah. That is one detail in the current series where the naming can be
particularly confusing.
I can go through and rename cfs_rq->curr to cfs_rq->donor (or some
other name) to make it more clear, but it seems like a ton of churn,
so I've been hesitant to do so until there was stronger consensus to
taking the patch series, but maybe we're at that point now?
But maybe a simpler and more isolated fix is I could just rework
update_curr_se to just take the rq* and we can derive the donor.se and
curr.se from that.
Thoughts?
thanks
-john
On Mon, Dec 16, 2024 at 10:09:16PM -0800, John Stultz wrote:
> On Fri, Dec 13, 2024 at 4:10 PM Peter Zijlstra <peterz@infradead.org> wrote:
> > On Sat, Dec 14, 2024 at 12:37:40AM +0100, Peter Zijlstra wrote:
> > > Would it not be *much* clearer if we do it like:
> > >
> > > static s64 update_curr_se(struct rq *rq, struct sched_entity *donor,
> > > struct sched_entity *curr)
> > > {
> > > ...
> > > donor->exec_start = now;
> > > curr->exec_start = now;
> > > curr->sum_exec_runtime += delta_exec;
> > > ...
> > > }
> > >
> > > and update the callsites like so:
> > >
> > > update_curr_common()
> > > update_curr_se(rq, &donor->se, &rq->curr.se)
> > >
> > > update_curr()
> > > update_curr_se(rq, &curr->se, &curr->se);
> > >
> > >
> > > except, now I'm confused about the update_curr() case. That seems to
> > > always update the execution context, rather than the donor ?
> >
> > Ah no, cfs_rq->curr is the donor.
>
> Yeah. That is one detail in the current series where the naming can be
> particularly confusing.
>
> I can go through and rename cfs_rq->curr to cfs_rq->donor (or some
> other name) to make it more clear, but it seems like a ton of churn,
> so I've been hesitant to do so until there was stronger consensus to
> taking the patch series, but maybe we're at that point now?
Nah, it was just me being confused, lets keep down the curn for now.
> But maybe a simpler and more isolated fix is I could just rework
> update_curr_se to just take the rq* and we can derive the donor.se and
> curr.se from that.
You can't; rq only has tasks, while cfs_rq is a hierarchy with many se's
backing a single task :/
© 2016 - 2026 Red Hat, Inc.