On 20/12/2023 12:18 am, John Stultz wrote:
> This pulls re-validation logic done in find_lowest_rq
> and find_latest_rq after re-acquiring the rq locks out into its
> own function.
>
> This allows us to later use a more complicated validation
> check for chain-migration when using proxy-exectuion.
execution
>
> TODO: It seems likely we could consolidate this two functions
> further and leave the task_is_rt()/task_is_dl() checks externally?
Agreed.
>
> Cc: Joel Fernandes <joelaf@google.com>
> Cc: Qais Yousef <qyousef@google.com>
> Cc: Ingo Molnar <mingo@redhat.com>
> Cc: Peter Zijlstra <peterz@infradead.org>
> Cc: Juri Lelli <juri.lelli@redhat.com>
> Cc: Vincent Guittot <vincent.guittot@linaro.org>
> Cc: Dietmar Eggemann <dietmar.eggemann@arm.com>
> Cc: Valentin Schneider <vschneid@redhat.com>
> Cc: Steven Rostedt <rostedt@goodmis.org>
> Cc: Ben Segall <bsegall@google.com>
> Cc: Zimuzo Ezeozue <zezeozue@google.com>
> Cc: Youssef Esmat <youssefesmat@google.com>
> Cc: Mel Gorman <mgorman@suse.de>
> Cc: Daniel Bristot de Oliveira <bristot@redhat.com>
> Cc: Will Deacon <will@kernel.org>
> Cc: Waiman Long <longman@redhat.com>
> Cc: Boqun Feng <boqun.feng@gmail.com>
> Cc: "Paul E. McKenney" <paulmck@kernel.org>
> Cc: Metin Kaya <Metin.Kaya@arm.com>
> Cc: Xuewen Yan <xuewen.yan94@gmail.com>
> Cc: K Prateek Nayak <kprateek.nayak@amd.com>
> Cc: Thomas Gleixner <tglx@linutronix.de>
> Cc: kernel-team@android.com
> Signed-off-by: John Stultz <jstultz@google.com>
> ---
> kernel/sched/deadline.c | 31 ++++++++++++++++++++-----
> kernel/sched/rt.c | 50 ++++++++++++++++++++++++++++-------------
> 2 files changed, 59 insertions(+), 22 deletions(-)
>
> diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
> index 21e56ac58e32..8b5701727342 100644
> --- a/kernel/sched/deadline.c
> +++ b/kernel/sched/deadline.c
> @@ -2172,6 +2172,30 @@ static int find_later_rq(struct task_struct *sched_ctx, struct task_struct *exec
> return -1;
> }
>
> +static inline bool dl_revalidate_rq_state(struct task_struct *task, struct rq *rq,
> + struct rq *later)
> +{
> + if (task_rq(task) != rq)
> + return false;
> +
> + if (!cpumask_test_cpu(later->cpu, &task->cpus_mask))
> + return false;
> +
> + if (task_on_cpu(rq, task))
> + return false;
> +
> + if (!dl_task(task))
> + return false;
> +
> + if (is_migration_disabled(task))
> + return false;
> +
> + if (!task_on_rq_queued(task))
> + return false;
> +
> + return true;
> +}
> +
> /* Locks the rq it finds */
> static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq)
> {
> @@ -2204,12 +2228,7 @@ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq)
>
> /* Retry if something changed. */
> if (double_lock_balance(rq, later_rq)) {
> - if (unlikely(task_rq(task) != rq ||
> - !cpumask_test_cpu(later_rq->cpu, &task->cpus_mask) ||
> - task_on_cpu(rq, task) ||
> - !dl_task(task) ||
> - is_migration_disabled(task) ||
> - !task_on_rq_queued(task))) {
> + if (unlikely(!dl_revalidate_rq_state(task, rq, later_rq))) {
> double_unlock_balance(rq, later_rq);
> later_rq = NULL;
> break;
> diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
> index f8134d062fa3..fabb19891e95 100644
> --- a/kernel/sched/rt.c
> +++ b/kernel/sched/rt.c
> @@ -1935,6 +1935,39 @@ static int find_lowest_rq(struct task_struct *sched_ctx, struct task_struct *exe
> return -1;
> }
>
> +static inline bool rt_revalidate_rq_state(struct task_struct *task, struct rq *rq,
> + struct rq *lowest)
> +{
> + /*
> + * We had to unlock the run queue. In
> + * the mean time, task could have
> + * migrated already or had its affinity changed.
> + * Also make sure that it wasn't scheduled on its rq.
> + * It is possible the task was scheduled, set
> + * "migrate_disabled" and then got preempted, so we must
> + * check the task migration disable flag here too.
> + */
> + if (task_rq(task) != rq)
> + return false;
> +
> + if (!cpumask_test_cpu(lowest->cpu, &task->cpus_mask))
> + return false;
> +
> + if (task_on_cpu(rq, task))
> + return false;
> +
> + if (!rt_task(task))
> + return false;
> +
> + if (is_migration_disabled(task))
> + return false;
> +
> + if (!task_on_rq_queued(task))
> + return false;
> +
> + return true;
> +}
> +
> /* Will lock the rq it finds */
> static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
> {
> @@ -1964,22 +1997,7 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
>
> /* if the prio of this runqueue changed, try again */
> if (double_lock_balance(rq, lowest_rq)) {
> - /*
> - * We had to unlock the run queue. In
> - * the mean time, task could have
> - * migrated already or had its affinity changed.
> - * Also make sure that it wasn't scheduled on its rq.
> - * It is possible the task was scheduled, set
> - * "migrate_disabled" and then got preempted, so we must
> - * check the task migration disable flag here too.
> - */
> - if (unlikely(task_rq(task) != rq ||
> - !cpumask_test_cpu(lowest_rq->cpu, &task->cpus_mask) ||
> - task_on_cpu(rq, task) ||
> - !rt_task(task) ||
> - is_migration_disabled(task) ||
> - !task_on_rq_queued(task))) {
> -
> + if (unlikely(!rt_revalidate_rq_state(task, rq, lowest_rq))) {
> double_unlock_balance(rq, lowest_rq);
> lowest_rq = NULL;
> break;