From nobody Sun Feb 8 10:25:57 2026 Received: from fanzine2.igalia.com (fanzine.igalia.com [178.60.130.6]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A97FF218ACB for ; Thu, 9 Jan 2025 13:15:20 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=178.60.130.6 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1736428523; cv=none; b=F5aWfUzxNJP82VjwXncQEBCiEW9vbqd4la1Wf4La/LDf7ZlQLevP5TVmgkBlMOCZntd+zH6g0svpJ8+RUTG4sN12YWPmv+dKaj2PXK69tMRvpfweB1te6/QH8n2vnPeGhXpQ/8nF1xgZG5ElyjEoFwH/R49Sjs6xuUKdrRtscM4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1736428523; c=relaxed/simple; bh=TlSqGNDpjyEQ/sAS+WG9j8ZbPrrXCvTJG01tJ8bLoZ0=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=BuWLbA+4yDv2wKicvB69kvAbNmPkZvc4ZHP1fVsvPv3uKBU3bG6h5eMyrrttQv7cKodTDmF3xICO2NAIkkn5DQKYYq41OT7i27GujNP3+noipdjsQcWBM2fg/NnfhtP3IGF0oouqk9lkmMnxxsg6+JUJH/yC2YBJYg/TjtrqFOo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=igalia.com; spf=pass smtp.mailfrom=igalia.com; dkim=pass (2048-bit key) header.d=igalia.com header.i=@igalia.com header.b=cIqraQJr; arc=none smtp.client-ip=178.60.130.6 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=igalia.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=igalia.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=igalia.com header.i=@igalia.com header.b="cIqraQJr" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=igalia.com; s=20170329; h=Content-Transfer-Encoding:MIME-Version:References:In-Reply-To: Message-ID:Date:Subject:Cc:To:From:Sender:Reply-To:Content-Type:Content-ID: Content-Description:Resent-Date:Resent-From:Resent-Sender:Resent-To:Resent-Cc :Resent-Message-ID:List-Id:List-Help:List-Unsubscribe:List-Subscribe: List-Post:List-Owner:List-Archive; bh=ot9Lx1zg/XLL07layOuusQRDG2mU5e3t2AM5FahhmOs=; b=cIqraQJrFMmyw6RyyKD/nnyw3I /GqPP3Xvf7H8bIGU/4/eYHFgyadMqqnCdOhGo9MmGTDzE8w+VATdtTyvPjMQD7J+YcR2dcIUfK6Yp eBSI3sqikdLm7wewk3WuAj5vJ4hbBDz3b+f+/hSOlxXzIuCF3jyopXAwg2g6bPfRdcapiJ3OFBYQ0 Txi9stj4ZgLmIahAyyUpfBmGVv2u0jTnncp3+PafVpT/eb98chz+k65oNRQ5tvfa3lXx0rHINRSoC LJUBx64cdx5J4UFJeWMtaFoed83M6Mgo0oHUiuN0/TF9qr7uWLATXwH7wMhijWGRR+kJR4QXVMCzU vfmW/G/A==; Received: from [58.29.143.236] (helo=localhost) by fanzine2.igalia.com with esmtpsa (Cipher TLS1.3:ECDHE_SECP256R1__RSA_PSS_RSAE_SHA256__AES_256_GCM:256) (Exim) id 1tVsNQ-00DZ9N-0T; Thu, 09 Jan 2025 14:15:08 +0100 From: Changwoo Min To: tj@kernel.org, void@manifault.com, arighi@nvidia.com, mingo@redhat.com, peterz@infradead.org Cc: kernel-dev@igalia.com, linux-kernel@vger.kernel.org, Changwoo Min Subject: [PATCH v8 1/6] sched_ext: Relocate scx_enabled() related code Date: Thu, 9 Jan 2025 22:14:51 +0900 Message-ID: <20250109131456.7055-2-changwoo@igalia.com> X-Mailer: git-send-email 2.47.1 In-Reply-To: <20250109131456.7055-1-changwoo@igalia.com> References: <20250109131456.7055-1-changwoo@igalia.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" scx_enabled() will be used in scx_rq_clock_update/invalidate() in the following patch, so relocate the scx_enabled() related code to the proper location. Signed-off-by: Changwoo Min Acked-by: Andrea Righi --- kernel/sched/sched.h | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 76f5f53a645f..440ecedf871b 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1717,6 +1717,19 @@ struct rq_flags { =20 extern struct balance_callback balance_push_callback; =20 +#ifdef CONFIG_SCHED_CLASS_EXT +extern const struct sched_class ext_sched_class; + +DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); /* SCX BPF scheduler loaded */ +DECLARE_STATIC_KEY_FALSE(__scx_switched_all); /* all fair class tasks on S= CX */ + +#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) +#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) +#else /* !CONFIG_SCHED_CLASS_EXT */ +#define scx_enabled() false +#define scx_switched_all() false +#endif /* !CONFIG_SCHED_CLASS_EXT */ + /* * Lockdep annotation that avoids accidental unlocks; it's like a * sticky/continuous lockdep_assert_held(). @@ -2505,19 +2518,6 @@ extern const struct sched_class rt_sched_class; extern const struct sched_class fair_sched_class; extern const struct sched_class idle_sched_class; =20 -#ifdef CONFIG_SCHED_CLASS_EXT -extern const struct sched_class ext_sched_class; - -DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); /* SCX BPF scheduler loaded */ -DECLARE_STATIC_KEY_FALSE(__scx_switched_all); /* all fair class tasks on S= CX */ - -#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) -#define scx_switched_all() static_branch_unlikely(&__scx_switched_all) -#else /* !CONFIG_SCHED_CLASS_EXT */ -#define scx_enabled() false -#define scx_switched_all() false -#endif /* !CONFIG_SCHED_CLASS_EXT */ - /* * Iterate only active classes. SCX can take over all fair tasks or be * completely disabled. If the former, skip fair. If the latter, skip SCX. --=20 2.47.1 From nobody Sun Feb 8 10:25:57 2026 Received: from fanzine2.igalia.com (fanzine.igalia.com [178.60.130.6]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2D691216E11 for ; Thu, 9 Jan 2025 13:15:23 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=178.60.130.6 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1736428526; cv=none; b=OSFs9Z2USiza9ael04FgZAgaJ3dGgE92tgJzoxCOBx/UIKSWDNR/ulw5YnZ+sdCO/PEslUUY6QJI+pQY+AzbLUlmaFq0XVNthdD1CX/Y2diasoikEkcSHZTLUr//nxuG2F8k6VXS2natx32Oteo7lQe46Dwt+814GMAexeIhbCg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1736428526; c=relaxed/simple; bh=+i/gMExXL3vJFzOpcGc2gohM1cn+Gbmb2ZkFJYYKQrE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=KRmCAnofvhuq5PvbBWfQKPuhAA0NAQ2itzH+rndRNNSSsKreg6Mb4qiPM1Pyth2TBOfOpr6Bn4F8pYkAFXhkU0RyOw8nTrMbWdj2CebWFzAKJyzrtP9ejJ4seNo+bgpbccWz8vFaJhYXlVRcI64w/yuhtJay4eyWqTza7/2tcHY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=igalia.com; spf=pass smtp.mailfrom=igalia.com; dkim=pass (2048-bit key) header.d=igalia.com header.i=@igalia.com header.b=Mfo27Ldl; arc=none smtp.client-ip=178.60.130.6 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=igalia.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=igalia.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=igalia.com header.i=@igalia.com header.b="Mfo27Ldl" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=igalia.com; s=20170329; h=Content-Transfer-Encoding:MIME-Version:References:In-Reply-To: Message-ID:Date:Subject:Cc:To:From:Sender:Reply-To:Content-Type:Content-ID: Content-Description:Resent-Date:Resent-From:Resent-Sender:Resent-To:Resent-Cc :Resent-Message-ID:List-Id:List-Help:List-Unsubscribe:List-Subscribe: List-Post:List-Owner:List-Archive; bh=WI5vYCfDt5JrxhtrDQHH7+RPhKJjIVbJXQIs9Sudfu0=; b=Mfo27Ldlrx8rpYDnD+gfWYGAUV RZBUmK3DEXXyB+VkbV2n61ituPQux1iTK4rD6LMkVlTLK+3IZ+qcSKV0ognFk3htqVTXs9DlcDex8 4a4gU/YV8taVSCwOHVh+Slo2Ws4cRPOJ/R8RjVh+ml8hG2fYPkLZjroijR1IvE7A6sIm1fmPmJsSC kuaAdlIEU1SsZETPlzu7O7GaYuPFWSOgCLp3rhxCwp/AHDN4UnXsgVPhrvj/EwwgIJozRSswSmuHj HnfzWQwEX+WZigUD99r0Rp+wdRmwJq2SmtQQHOzjZTOH3OfO22N4d1EdllVd/mR5Lu7ak6McdsLOV xvSiKhnw==; Received: from [58.29.143.236] (helo=localhost) by fanzine2.igalia.com with esmtpsa (Cipher TLS1.3:ECDHE_SECP256R1__RSA_PSS_RSAE_SHA256__AES_256_GCM:256) (Exim) id 1tVsNV-00DZ9Y-Nn; Thu, 09 Jan 2025 14:15:14 +0100 From: Changwoo Min To: tj@kernel.org, void@manifault.com, arighi@nvidia.com, mingo@redhat.com, peterz@infradead.org Cc: kernel-dev@igalia.com, linux-kernel@vger.kernel.org, Changwoo Min Subject: [PATCH v8 2/6] sched_ext: Implement scx_bpf_now() Date: Thu, 9 Jan 2025 22:14:52 +0900 Message-ID: <20250109131456.7055-3-changwoo@igalia.com> X-Mailer: git-send-email 2.47.1 In-Reply-To: <20250109131456.7055-1-changwoo@igalia.com> References: <20250109131456.7055-1-changwoo@igalia.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Returns a high-performance monotonically non-decreasing clock for the curre= nt CPU. The clock returned is in nanoseconds. It provides the following properties: 1) High performance: Many BPF schedulers call bpf_ktime_get_ns() frequently to account for execution time and track tasks' runtime properties. Unfortunately, in some hardware platforms, bpf_ktime_get_ns() -- which eventually reads a hardware timestamp counter -- is neither performant nor scalable. scx_bpf_now() aims to provide a high-performance clock by using the rq clock in the scheduler core whenever possible. 2) High enough resolution for the BPF scheduler use cases: In most BPF scheduler use cases, the required clock resolution is lower than the most accurate hardware clock (e.g., rdtsc in x86). scx_bpf_now() basically uses the rq clock in the scheduler core whenever it is valid. It considers that the rq clock is valid from the time the rq clock is updated (update_rq_clock) until the rq is unlocked (rq_unpin_lock). 3) Monotonically non-decreasing clock for the same CPU: scx_bpf_now() guarantees the clock never goes backward when comparing them in the same CPU. On the other hand, when comparing clocks in different CPUs, there is no such guarantee -- the clock can go backward. It provides a monotonically *non-decreasing* clock so that it would provide the same clock values in two different scx_bpf_now() calls in the same CPU during the same period of when the rq clock is valid. An rq clock becomes valid when it is updated using update_rq_clock() and invalidated when the rq is unlocked using rq_unpin_lock(). Let's suppose the following timeline in the scheduler core: T1. rq_lock(rq) T2. update_rq_clock(rq) T3. a sched_ext BPF operation T4. rq_unlock(rq) T5. a sched_ext BPF operation T6. rq_lock(rq) T7. update_rq_clock(rq) For [T2, T4), we consider that rq clock is valid (SCX_RQ_CLK_VALID is set), so scx_bpf_now() calls during [T2, T4) (including T3) will return the rq clock updated at T2. For duration [T4, T7), when a BPF scheduler can still call scx_bpf_now() (T5), we consider the rq clock is invalid (SCX_RQ_CLK_VALID is unset at T4). So when calling scx_bpf_now() at T5, we will return a fresh clock value by calling sched_clock_cpu() internally. Also, to prevent getting outdated rq clocks from a previous scx scheduler, invalidate all the rq clocks when unloading a BPF scheduler. One example of calling scx_bpf_now(), when the rq clock is invalid (like T5), is in scx_central [1]. The scx_central scheduler uses a BPF timer for preemptive scheduling. In every msec, the timer callback checks if the currently running tasks exceed their timeslice. At the beginning of the BPF timer callback (central_timerfn in scx_central.bpf.c), scx_central gets the current time. When the BPF timer callback runs, the rq clock could be invalid, the same as T5. In this case, scx_bpf_now() returns a fresh clock value rather than returning the old one (T2). [1] https://github.com/sched-ext/scx/blob/main/scheds/c/scx_central.bpf.c Signed-off-by: Changwoo Min Acked-by: Andrea Righi Acked-by: Peter Zijlstra (Intel) --- kernel/sched/core.c | 6 +++- kernel/sched/ext.c | 74 +++++++++++++++++++++++++++++++++++++++++++- kernel/sched/sched.h | 25 +++++++++++++-- 3 files changed, 101 insertions(+), 4 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 95e40895a519..ab8015c8cab4 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -789,6 +789,7 @@ static void update_rq_clock_task(struct rq *rq, s64 del= ta) void update_rq_clock(struct rq *rq) { s64 delta; + u64 clock; =20 lockdep_assert_rq_held(rq); =20 @@ -800,11 +801,14 @@ void update_rq_clock(struct rq *rq) SCHED_WARN_ON(rq->clock_update_flags & RQCF_UPDATED); rq->clock_update_flags |=3D RQCF_UPDATED; #endif + clock =3D sched_clock_cpu(cpu_of(rq)); + scx_rq_clock_update(rq, clock); =20 - delta =3D sched_clock_cpu(cpu_of(rq)) - rq->clock; + delta =3D clock - rq->clock; if (delta < 0) return; rq->clock +=3D delta; + update_rq_clock_task(rq, delta); } =20 diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index ec91a4d95722..9a568ee8e1f5 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -4915,7 +4915,7 @@ static void scx_ops_disable_workfn(struct kthread_wor= k *work) struct task_struct *p; struct rhashtable_iter rht_iter; struct scx_dispatch_q *dsq; - int i, kind; + int i, kind, cpu; =20 kind =3D atomic_read(&scx_exit_kind); while (true) { @@ -4998,6 +4998,15 @@ static void scx_ops_disable_workfn(struct kthread_wo= rk *work) scx_task_iter_stop(&sti); percpu_up_write(&scx_fork_rwsem); =20 + /* + * Invalidate all the rq clocks to prevent getting outdated + * rq clocks from a previous scx scheduler. + */ + for_each_possible_cpu(cpu) { + struct rq *rq =3D cpu_rq(cpu); + scx_rq_clock_invalidate(rq); + } + /* no task is on scx, turn off all the switches and flush in-progress cal= ls */ static_branch_disable(&__scx_ops_enabled); for (i =3D SCX_OPI_BEGIN; i < SCX_OPI_END; i++) @@ -7603,6 +7612,68 @@ __bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struc= t task_struct *p) } #endif =20 +/** + * scx_bpf_now - Returns a high-performance monotonically non-decreasing + * clock for the current CPU. The clock returned is in nanoseconds. + * + * It provides the following properties: + * + * 1) High performance: Many BPF schedulers call bpf_ktime_get_ns() freque= ntly + * to account for execution time and track tasks' runtime properties. + * Unfortunately, in some hardware platforms, bpf_ktime_get_ns() -- which + * eventually reads a hardware timestamp counter -- is neither performant= nor + * scalable. scx_bpf_now() aims to provide a high-performance clock by + * using the rq clock in the scheduler core whenever possible. + * + * 2) High enough resolution for the BPF scheduler use cases: In most BPF + * scheduler use cases, the required clock resolution is lower than the m= ost + * accurate hardware clock (e.g., rdtsc in x86). scx_bpf_now() basically + * uses the rq clock in the scheduler core whenever it is valid. It consi= ders + * that the rq clock is valid from the time the rq clock is updated + * (update_rq_clock) until the rq is unlocked (rq_unpin_lock). + * + * 3) Monotonically non-decreasing clock for the same CPU: scx_bpf_now() + * guarantees the clock never goes backward when comparing them in the sa= me + * CPU. On the other hand, when comparing clocks in different CPUs, there + * is no such guarantee -- the clock can go backward. It provides a + * monotonically *non-decreasing* clock so that it would provide the same + * clock values in two different scx_bpf_now() calls in the same CPU + * during the same period of when the rq clock is valid. + */ +__bpf_kfunc u64 scx_bpf_now(void) +{ + struct rq *rq; + u64 clock; + + preempt_disable(); + + rq =3D this_rq(); + if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) { + /* + * If the rq clock is valid, use the cached rq clock. + * + * Note that scx_bpf_now() is re-entrant between a process + * context and an interrupt context (e.g., timer interrupt). + * However, we don't need to consider the race between them + * because such race is not observable from a caller. + */ + clock =3D READ_ONCE(rq->scx.clock); + } else { + /* + * Otherwise, return a fresh rq clock. + * + * The rq clock is updated outside of the rq lock. + * In this case, keep the updated rq clock invalid so the next + * kfunc call outside the rq lock gets a fresh rq clock. + */ + clock =3D sched_clock_cpu(cpu_of(rq)); + } + + preempt_enable(); + + return clock; +} + __bpf_kfunc_end_defs(); =20 BTF_KFUNCS_START(scx_kfunc_ids_any) @@ -7634,6 +7705,7 @@ BTF_ID_FLAGS(func, scx_bpf_cpu_rq) #ifdef CONFIG_CGROUP_SCHED BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_RCU | KF_ACQUIRE) #endif +BTF_ID_FLAGS(func, scx_bpf_now) BTF_KFUNCS_END(scx_kfunc_ids_any) =20 static const struct btf_kfunc_id_set scx_kfunc_set_any =3D { diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 440ecedf871b..a97f96a06615 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -754,6 +754,7 @@ enum scx_rq_flags { SCX_RQ_BAL_PENDING =3D 1 << 2, /* balance hasn't run yet */ SCX_RQ_BAL_KEEP =3D 1 << 3, /* balance decided to keep current */ SCX_RQ_BYPASSING =3D 1 << 4, + SCX_RQ_CLK_VALID =3D 1 << 5, /* RQ clock is fresh and valid */ =20 SCX_RQ_IN_WAKEUP =3D 1 << 16, SCX_RQ_IN_BALANCE =3D 1 << 17, @@ -766,9 +767,10 @@ struct scx_rq { unsigned long ops_qseq; u64 extra_enq_flags; /* see move_task_to_local_dsq() */ u32 nr_running; - u32 flags; u32 cpuperf_target; /* [0, SCHED_CAPACITY_SCALE] */ bool cpu_released; + u32 flags; + u64 clock; /* current per-rq clock -- see scx_bpf_now() */ cpumask_var_t cpus_to_kick; cpumask_var_t cpus_to_kick_if_idle; cpumask_var_t cpus_to_preempt; @@ -1725,9 +1727,28 @@ DECLARE_STATIC_KEY_FALSE(__scx_switched_all); /* all= fair class tasks on SCX */ =20 #define scx_enabled() static_branch_unlikely(&__scx_ops_enabled) #define scx_switched_all() static_branch_unlikely(&__scx_switched_all) + +static inline void scx_rq_clock_update(struct rq *rq, u64 clock) +{ + if (!scx_enabled()) + return; + WRITE_ONCE(rq->scx.clock, clock); + smp_store_release(&rq->scx.flags, rq->scx.flags | SCX_RQ_CLK_VALID); +} + +static inline void scx_rq_clock_invalidate(struct rq *rq) +{ + if (!scx_enabled()) + return; + WRITE_ONCE(rq->scx.flags, rq->scx.flags & ~SCX_RQ_CLK_VALID); +} + #else /* !CONFIG_SCHED_CLASS_EXT */ #define scx_enabled() false #define scx_switched_all() false + +static inline void scx_rq_clock_update(struct rq *rq, u64 clock) {} +static inline void scx_rq_clock_invalidate(struct rq *rq) {} #endif /* !CONFIG_SCHED_CLASS_EXT */ =20 /* @@ -1759,7 +1780,7 @@ static inline void rq_unpin_lock(struct rq *rq, struc= t rq_flags *rf) if (rq->clock_update_flags > RQCF_ACT_SKIP) rf->clock_update_flags =3D RQCF_UPDATED; #endif - + scx_rq_clock_invalidate(rq); lockdep_unpin_lock(__rq_lockp(rq), rf->cookie); } =20 --=20 2.47.1 From nobody Sun Feb 8 10:25:57 2026 Received: from fanzine2.igalia.com (fanzine.igalia.com [178.60.130.6]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4510C218AA0 for ; Thu, 9 Jan 2025 13:15:28 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=178.60.130.6 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1736428531; cv=none; b=SIyTXV3h1zDpy/MMv5ltDLTzeD7y6MX9ucU5b+60rc7gSqZ20a+yHxm3guPYKFjpc+sEoQxk09sBAqRDou5wDwUIInOzJIbrM4PldhEa39ZpcQIa40nodpmSYw8XjYqXLrLvHV91zQPganxL+hVUG3D4h1o0W0yFINYZsbVsTT4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1736428531; c=relaxed/simple; bh=quj8prKTf0T2wGBr3bhLbhhITyZHTAO5tc+XMf0VBZI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=d4HtkdkwmR01fOEUeN1JQbWTbR487pJZ3eP0RyNaVcQTEpdkH7YYPTljKrPEyS0aXyRcQOyhw0ejeNrQ5kWlJqlRhxyTGjnyt59fil2Nwre4S/jeNhwaxKVdJA2psQVCkz72E/b8e8j6XpTtc3U1AfM6Xfwg0XK7pASaBsh09s8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=igalia.com; spf=pass smtp.mailfrom=igalia.com; dkim=pass (2048-bit key) header.d=igalia.com header.i=@igalia.com header.b=Z59W5DOo; arc=none smtp.client-ip=178.60.130.6 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=igalia.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=igalia.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=igalia.com header.i=@igalia.com header.b="Z59W5DOo" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=igalia.com; s=20170329; h=Content-Transfer-Encoding:MIME-Version:References:In-Reply-To: Message-ID:Date:Subject:Cc:To:From:Sender:Reply-To:Content-Type:Content-ID: Content-Description:Resent-Date:Resent-From:Resent-Sender:Resent-To:Resent-Cc :Resent-Message-ID:List-Id:List-Help:List-Unsubscribe:List-Subscribe: List-Post:List-Owner:List-Archive; bh=Wow0f2L0ocjtG3ziib/G7eNZl8dh1HZAWkEJfvze4q0=; b=Z59W5DOoyvzLt1Jc7LdDt1PgoG MmyBCO0m6iUXrXTpNtb8tCJ+soiyNMWEROdnRpS+L7HdyK93dl9MAAUCqASatwKoJSE6iLVI7WBnu 3LdHLEQfgjh8qabUXdclL9NapJ3HY054TELrIb/JrCxRJXfKutXxOBgun4ArcfeTBAlHQu8WPFrYx pzkhGT+kjqhFBGcSBaW9dtLgTkz2Je5aHTlNLtPQkpNfEB5iXsOic/Nze6pu50DI/pqbFF5ZwqlqN rjwX/dpqDnbUoX2DNPo+/NW5Rn4U64iFxD+/+qiuRtGyVfqoO6KSHP+7Q23Sy0/1l8lz56klZda1I o0MVMFkA==; Received: from [58.29.143.236] (helo=localhost) by fanzine2.igalia.com with esmtpsa (Cipher TLS1.3:ECDHE_SECP256R1__RSA_PSS_RSAE_SHA256__AES_256_GCM:256) (Exim) id 1tVsNb-00DZ9l-At; Thu, 09 Jan 2025 14:15:20 +0100 From: Changwoo Min To: tj@kernel.org, void@manifault.com, arighi@nvidia.com, mingo@redhat.com, peterz@infradead.org Cc: kernel-dev@igalia.com, linux-kernel@vger.kernel.org, Changwoo Min Subject: [PATCH v8 3/6] sched_ext: Add scx_bpf_now() for BPF scheduler Date: Thu, 9 Jan 2025 22:14:53 +0900 Message-ID: <20250109131456.7055-4-changwoo@igalia.com> X-Mailer: git-send-email 2.47.1 In-Reply-To: <20250109131456.7055-1-changwoo@igalia.com> References: <20250109131456.7055-1-changwoo@igalia.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" scx_bpf_now() is added to the header files so the BPF scheduler can use it. Signed-off-by: Changwoo Min Acked-by: Andrea Righi --- tools/sched_ext/include/scx/common.bpf.h | 1 + tools/sched_ext/include/scx/compat.bpf.h | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/inc= lude/scx/common.bpf.h index 858ba1f438f6..5c9517190713 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -76,6 +76,7 @@ bool scx_bpf_task_running(const struct task_struct *p) __= ksym; s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; struct rq *scx_bpf_cpu_rq(s32 cpu) __ksym; struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym __weak; +u64 scx_bpf_now(void) __ksym __weak; =20 /* * Use the following as @it__iter when calling scx_bpf_dsq_move[_vtime]() = from diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/inc= lude/scx/compat.bpf.h index d56520100a26..50e1499ae093 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -125,6 +125,11 @@ bool scx_bpf_dispatch_vtime_from_dsq___compat(struct b= pf_iter_scx_dsq *it__iter, false; \ }) =20 +#define scx_bpf_now() \ + (bpf_ksym_exists(scx_bpf_now) ? \ + scx_bpf_now() : \ + bpf_ktime_get_ns()) + /* * Define sched_ext_ops. This may be expanded to define multiple variants = for * backward compatibility. See compat.h::SCX_OPS_LOAD/ATTACH(). --=20 2.47.1 From nobody Sun Feb 8 10:25:57 2026 Received: from fanzine2.igalia.com (fanzine.igalia.com [178.60.130.6]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 3EE0726AD9 for ; Thu, 9 Jan 2025 13:15:33 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=178.60.130.6 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1736428538; cv=none; b=ssouJfqm9kewV2NoeoNYDkC84TsGijC4uW91GENyltSJPmX50CS63Y12ZvhPTsYDcDPl+LyQi75ncgnMSGpT94epiW76bvHCRG1oQj4+c+ueiBf1hpVVqMdxXKNz/Oha6zrQ5CGAegBMGUauQAjs5jZGGnIxw7/sP3TZ/12OBIc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1736428538; c=relaxed/simple; bh=vr8H5CtZ05R7JvmYMoz5R6sWD6J+X4CMRtIVG4kOE2o=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=RkjxHvKt5ZKIrXElXQ/0trv8Mk6NgZlna790nmJNCiPLZoJuxRnuMWCwpxLiRBR703B7bjos6/nO7zzCmu6rlu5+wW5nXsRKvQ5NJ4HFujcRc6SRF0x9TlQzZYfJQ4Ei7dseq/lELncvO2Rec9apqbhcZ/1ZMj0EPhu6GMTwybQ= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=igalia.com; spf=pass smtp.mailfrom=igalia.com; dkim=pass (2048-bit key) header.d=igalia.com header.i=@igalia.com header.b=ZbYKTcPe; arc=none smtp.client-ip=178.60.130.6 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=igalia.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=igalia.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=igalia.com header.i=@igalia.com header.b="ZbYKTcPe" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=igalia.com; s=20170329; h=Content-Transfer-Encoding:MIME-Version:References:In-Reply-To: Message-ID:Date:Subject:Cc:To:From:Sender:Reply-To:Content-Type:Content-ID: Content-Description:Resent-Date:Resent-From:Resent-Sender:Resent-To:Resent-Cc :Resent-Message-ID:List-Id:List-Help:List-Unsubscribe:List-Subscribe: List-Post:List-Owner:List-Archive; bh=aQh2aQFFIOtlO17ZAyGjilQkwjee2R8E6fn9IlabB6s=; b=ZbYKTcPeQg9zj/9MmYEOZUWquh 8lpm8NESeMMy8w3aFbmvv8JsGIUStmXECpTTbpQAF+FPBeTsToo2Umge4TRokLmlqnIESj4WgAZlk gaa4xdxkZP0HL8qNvQNfqRv/MNX2w2PDAmipmnNFCcKbskYPxM2z0jaBFUuuib88ilxQJGpqULVf7 eY0KsLiM5McIG+3Jg9DZVMpff6hzF05QP3BEIVG+mfCyUj84xzCmr9hN3CrMQdGL2INb7WrI+RDdI HCat0tWRoKPCYUdTCiLKtvJswR0p4bMb8Gsx4nCPVY2rxTniWZE03RHUWbxth3PqZWXyrSID9rOmT 9wbsvCMg==; Received: from [58.29.143.236] (helo=localhost) by fanzine2.igalia.com with esmtpsa (Cipher TLS1.3:ECDHE_SECP256R1__RSA_PSS_RSAE_SHA256__AES_256_GCM:256) (Exim) id 1tVsNg-00DZA1-Q6; Thu, 09 Jan 2025 14:15:25 +0100 From: Changwoo Min To: tj@kernel.org, void@manifault.com, arighi@nvidia.com, mingo@redhat.com, peterz@infradead.org Cc: kernel-dev@igalia.com, linux-kernel@vger.kernel.org, Changwoo Min Subject: [PATCH v8 4/6] sched_ext: Add time helpers for BPF schedulers Date: Thu, 9 Jan 2025 22:14:54 +0900 Message-ID: <20250109131456.7055-5-changwoo@igalia.com> X-Mailer: git-send-email 2.47.1 In-Reply-To: <20250109131456.7055-1-changwoo@igalia.com> References: <20250109131456.7055-1-changwoo@igalia.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The following functions are added for BPF schedulers: - time_delta(after, before) - time_after(a, b) - time_before(a, b) - time_after_eq(a, b) - time_before_eq(a, b) - time_in_range(a, b, c) - time_in_range_open(a, b, c) Signed-off-by: Changwoo Min Acked-by: Andrea Righi --- tools/sched_ext/include/scx/common.bpf.h | 94 ++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/inc= lude/scx/common.bpf.h index 5c9517190713..f3e15e9efa76 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -408,6 +408,100 @@ static __always_inline const struct cpumask *cast_mas= k(struct bpf_cpumask *mask) void bpf_rcu_read_lock(void) __ksym; void bpf_rcu_read_unlock(void) __ksym; =20 +/* + * Time helpers, most of which are from jiffies.h. + */ + +/** + * time_delta - Calculate the delta between new and old time stamp + * @after: first comparable as u64 + * @before: second comparable as u64 + * + * Return: the time difference, which is >=3D 0 + */ +static inline s64 time_delta(u64 after, u64 before) +{ + return (s64)(after - before) > 0 ? : 0; +} + +/** + * time_after - returns true if the time a is after time b. + * @a: first comparable as u64 + * @b: second comparable as u64 + * + * Do this with "<0" and ">=3D0" to only test the sign of the result. A + * good compiler would generate better code (and a really good compiler + * wouldn't care). Gcc is currently neither. + * + * Return: %true is time a is after time b, otherwise %false. + */ +static inline bool time_after(u64 a, u64 b) +{ + return (s64)(b - a) < 0; +} + +/** + * time_before - returns true if the time a is before time b. + * @a: first comparable as u64 + * @b: second comparable as u64 + * + * Return: %true is time a is before time b, otherwise %false. + */ +static inline bool time_before(u64 a, u64 b) +{ + return time_after(b, a); +} + +/** + * time_after_eq - returns true if the time a is after or the same as time= b. + * @a: first comparable as u64 + * @b: second comparable as u64 + * + * Return: %true is time a is after or the same as time b, otherwise %fals= e. + */ +static inline bool time_after_eq(u64 a, u64 b) +{ + return (s64)(a - b) >=3D 0; +} + +/** + * time_before_eq - returns true if the time a is before or the same as ti= me b. + * @a: first comparable as u64 + * @b: second comparable as u64 + * + * Return: %true is time a is before or the same as time b, otherwise %fal= se. + */ +static inline bool time_before_eq(u64 a, u64 b) +{ + return time_after_eq(b, a); +} + +/** + * time_in_range - Calculate whether a is in the range of [b, c]. + * @a: time to test + * @b: beginning of the range + * @c: end of the range + * + * Return: %true is time a is in the range [b, c], otherwise %false. + */ +static inline bool time_in_range(u64 a, u64 b, u64 c) +{ + return time_after_eq(a, b) && time_before_eq(a, c); +} + +/** + * time_in_range_open - Calculate whether a is in the range of [b, c). + * @a: time to test + * @b: beginning of the range + * @c: end of the range + * + * Return: %true is time a is in the range [b, c), otherwise %false. + */ +static inline bool time_in_range_open(u64 a, u64 b, u64 c) +{ + return time_after_eq(a, b) && time_before(a, c); +} + =20 /* * Other helpers --=20 2.47.1 From nobody Sun Feb 8 10:25:57 2026 Received: from fanzine2.igalia.com (fanzine.igalia.com [178.60.130.6]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 8E2EA219A8B for ; Thu, 9 Jan 2025 13:15:40 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=178.60.130.6 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1736428542; cv=none; b=R2kDTdz953VSJy1y6eqCzUpruQyyLcbbW6yQfOoMjeg2Uy7QYZewlz9uVk1uylS5LYajy+y1fR35wzhC9kdtxRMKuBh58USmxJZKfuzbo7oVYL5J4gNg9X4ygQCiHFxpReWrznoof2lgQH0eVIYE/9/B1jakfgY+GbccvdKKYCk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1736428542; c=relaxed/simple; bh=3KWIG2mmKhaV/gJkcgP4kuB7LjMEjgZ5X9Pyt9i8Jhg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=I4Etn4zrABeL21kbDK/mneF/lLhfLAmBbZI4WKz4P7oHQcE1OyaeVn5OUGdLEI5rslwRkRRN56OXf2NYI4JwVTiNIZB+Xw7jFEpsSeTOkYcEF7KFjB+RSjwqU0xXGrsqV1mXZ+0l4aKk6RHTzH4BfsXHaPosDROtKGD3xF7EkCk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=igalia.com; spf=pass smtp.mailfrom=igalia.com; dkim=pass (2048-bit key) header.d=igalia.com header.i=@igalia.com header.b=RXHNAdf/; arc=none smtp.client-ip=178.60.130.6 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=igalia.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=igalia.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=igalia.com header.i=@igalia.com header.b="RXHNAdf/" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=igalia.com; s=20170329; h=Content-Transfer-Encoding:MIME-Version:References:In-Reply-To: Message-ID:Date:Subject:Cc:To:From:Sender:Reply-To:Content-Type:Content-ID: Content-Description:Resent-Date:Resent-From:Resent-Sender:Resent-To:Resent-Cc :Resent-Message-ID:List-Id:List-Help:List-Unsubscribe:List-Subscribe: List-Post:List-Owner:List-Archive; bh=l78V4ESiWJCLASUcLE3njLXpFe+ymfOJGKR+C9fLFbw=; b=RXHNAdf/GpFCFejDid3S6ez4ov TxF44EF0GyENndLEl41Mu7gH53gBRgQpLLyGR++FwoKf1pNlocNJFvaNlboWeKtZPNJ5x8A9peyDD 8NsFi+uM6BgWdrsJZrq9GwowiRaemx6VZhEAPqh9FtlPEp7+tljT3T1UGtalKohKiBm9QzzcS3vLP J+FFQeLmnPFb7G2F38OiKpRasUc+h4go0sFMZiINgJlYtHjPYEMgyr7cwbNumhJug+evKDQLna2t5 YSqOVIuH9lmOX4RZPFCHYsFIbJdauffnFIVnPRjwq7I2kX7AzyzIU/qjW80TeRk1nOw+hYDLix3uO FsiKD4DQ==; Received: from [58.29.143.236] (helo=localhost) by fanzine2.igalia.com with esmtpsa (Cipher TLS1.3:ECDHE_SECP256R1__RSA_PSS_RSAE_SHA256__AES_256_GCM:256) (Exim) id 1tVsNm-00DZAD-E7; Thu, 09 Jan 2025 14:15:31 +0100 From: Changwoo Min To: tj@kernel.org, void@manifault.com, arighi@nvidia.com, mingo@redhat.com, peterz@infradead.org Cc: kernel-dev@igalia.com, linux-kernel@vger.kernel.org, Changwoo Min Subject: [PATCH v8 5/6] sched_ext: Replace bpf_ktime_get_ns() to scx_bpf_now() Date: Thu, 9 Jan 2025 22:14:55 +0900 Message-ID: <20250109131456.7055-6-changwoo@igalia.com> X-Mailer: git-send-email 2.47.1 In-Reply-To: <20250109131456.7055-1-changwoo@igalia.com> References: <20250109131456.7055-1-changwoo@igalia.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" In the BPF schedulers that use bpf_ktime_get_ns() -- scx_central and scx_flatcg, replace bpf_ktime_get_ns() calls to scx_bpf_now(). Signed-off-by: Changwoo Min Acked-by: Andrea Righi --- tools/sched_ext/scx_central.bpf.c | 4 ++-- tools/sched_ext/scx_flatcg.bpf.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/sched_ext/scx_central.bpf.c b/tools/sched_ext/scx_centra= l.bpf.c index 2907df78241e..4239034ad593 100644 --- a/tools/sched_ext/scx_central.bpf.c +++ b/tools/sched_ext/scx_central.bpf.c @@ -245,7 +245,7 @@ void BPF_STRUCT_OPS(central_running, struct task_struct= *p) s32 cpu =3D scx_bpf_task_cpu(p); u64 *started_at =3D ARRAY_ELEM_PTR(cpu_started_at, cpu, nr_cpu_ids); if (started_at) - *started_at =3D bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */ + *started_at =3D scx_bpf_now() ?: 1; /* 0 indicates idle */ } =20 void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable) @@ -258,7 +258,7 @@ void BPF_STRUCT_OPS(central_stopping, struct task_struc= t *p, bool runnable) =20 static int central_timerfn(void *map, int *key, struct bpf_timer *timer) { - u64 now =3D bpf_ktime_get_ns(); + u64 now =3D scx_bpf_now(); u64 nr_to_kick =3D nr_queued; s32 i, curr_cpu; =20 diff --git a/tools/sched_ext/scx_flatcg.bpf.c b/tools/sched_ext/scx_flatcg.= bpf.c index 3dbfa82883be..5f588963fb2f 100644 --- a/tools/sched_ext/scx_flatcg.bpf.c +++ b/tools/sched_ext/scx_flatcg.bpf.c @@ -734,7 +734,7 @@ void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_= struct *prev) struct fcg_cpu_ctx *cpuc; struct fcg_cgrp_ctx *cgc; struct cgroup *cgrp; - u64 now =3D bpf_ktime_get_ns(); + u64 now =3D scx_bpf_now(); bool picked_next =3D false; =20 cpuc =3D find_cpu_ctx(); --=20 2.47.1 From nobody Sun Feb 8 10:25:57 2026 Received: from fanzine2.igalia.com (fanzine.igalia.com [178.60.130.6]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A6EDD218EAD for ; Thu, 9 Jan 2025 13:15:45 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=178.60.130.6 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1736428547; cv=none; b=HfuaTN6+i89dDxkWTRaBppsUmrGOtpl3zZBtZtBbI6YuTaNiItW+q+P34RNZgohbxO7+R9SWVweZsIrXIWAiNs4B6hKsphf+6CPDXpOeJG051GxZIZbop8QuVdt69DCV55T8qTjGE6AfWZLRFcWw4dKRaZU+IqLohGdm1SBoJIo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1736428547; c=relaxed/simple; bh=x8H6DVoIMlNDpp4A25ArxzyiMF448S9aPxeDVLI8SLU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=TH5OdAGffwByh5hV37Exo1j9X6vnGvZSVEtmlOOs/5mAllwLeE776/yv65v7aYCiZymXw9Jyplec07Bz+ClX4rkHBH3oOuym4Yx/54pGMXN0Z8Qc6SO07NeSCERCAvqdZqyJ7PspmgfEjbcSXm07COSLvj8wL8QBHUjBOLiBcX8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=igalia.com; spf=pass smtp.mailfrom=igalia.com; dkim=pass (2048-bit key) header.d=igalia.com header.i=@igalia.com header.b=Szh+jHul; arc=none smtp.client-ip=178.60.130.6 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=igalia.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=igalia.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=igalia.com header.i=@igalia.com header.b="Szh+jHul" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=igalia.com; s=20170329; h=Content-Transfer-Encoding:MIME-Version:References:In-Reply-To: Message-ID:Date:Subject:Cc:To:From:Sender:Reply-To:Content-Type:Content-ID: Content-Description:Resent-Date:Resent-From:Resent-Sender:Resent-To:Resent-Cc :Resent-Message-ID:List-Id:List-Help:List-Unsubscribe:List-Subscribe: List-Post:List-Owner:List-Archive; bh=rmZA4vQufn+h6OnUPeuDpOx8Poi6ZPcPUwnKJ0yEQjk=; b=Szh+jHul7y+xBpH7vfJWQ58/ZQ gU5m9rq2kdJHCISE2aLQBoz1i1F6bnK2P9xMJYFSo2aBNMeFTI7XOkXnAzO2MmUlhX+jsy37G1am6 RCG/ZcBIpkWbbrRw1hlGGYLuV0cUk/F9ifRzILk3iwEk28LhYRq5pN7gIfdK3ORTskdVp0paRC24z dx/J1Zy+KLry8hc2cvKm7lMNUMUjDzI1KhQKVnFllOT78Ob8OnuPvfHhlrFygeSSxqUkfomO/Xg6v ehTQ775RQsQKAeU7gzV0F5k9nORfh6Gs7AAwY7wkXGRguXlkVPZFBIW7ry8eDrcAj9wCdZx3guZqm U5oPN26A==; Received: from [58.29.143.236] (helo=localhost) by fanzine2.igalia.com with esmtpsa (Cipher TLS1.3:ECDHE_SECP256R1__RSA_PSS_RSAE_SHA256__AES_256_GCM:256) (Exim) id 1tVsNr-00DZAT-NJ; Thu, 09 Jan 2025 14:15:36 +0100 From: Changwoo Min To: tj@kernel.org, void@manifault.com, arighi@nvidia.com, mingo@redhat.com, peterz@infradead.org Cc: kernel-dev@igalia.com, linux-kernel@vger.kernel.org, Changwoo Min Subject: [PATCH v8 6/6] sched_ext: Use time helpers in BPF schedulers Date: Thu, 9 Jan 2025 22:14:56 +0900 Message-ID: <20250109131456.7055-7-changwoo@igalia.com> X-Mailer: git-send-email 2.47.1 In-Reply-To: <20250109131456.7055-1-changwoo@igalia.com> References: <20250109131456.7055-1-changwoo@igalia.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Modify the BPF schedulers to use time helpers defined in common.bpf.h Signed-off-by: Changwoo Min Acked-by: Andrea Righi --- tools/sched_ext/scx_central.bpf.c | 7 +------ tools/sched_ext/scx_flatcg.bpf.c | 21 ++++++++------------- tools/sched_ext/scx_simple.bpf.c | 9 ++------- 3 files changed, 11 insertions(+), 26 deletions(-) diff --git a/tools/sched_ext/scx_central.bpf.c b/tools/sched_ext/scx_centra= l.bpf.c index 4239034ad593..50bc1737c167 100644 --- a/tools/sched_ext/scx_central.bpf.c +++ b/tools/sched_ext/scx_central.bpf.c @@ -87,11 +87,6 @@ struct { __type(value, struct central_timer); } central_timer SEC(".maps"); =20 -static bool vtime_before(u64 a, u64 b) -{ - return (s64)(a - b) < 0; -} - s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags) { @@ -279,7 +274,7 @@ static int central_timerfn(void *map, int *key, struct = bpf_timer *timer) /* kick iff the current one exhausted its slice */ started_at =3D ARRAY_ELEM_PTR(cpu_started_at, cpu, nr_cpu_ids); if (started_at && *started_at && - vtime_before(now, *started_at + slice_ns)) + time_before(now, *started_at + slice_ns)) continue; =20 /* and there's something pending */ diff --git a/tools/sched_ext/scx_flatcg.bpf.c b/tools/sched_ext/scx_flatcg.= bpf.c index 5f588963fb2f..2c720e3ecad5 100644 --- a/tools/sched_ext/scx_flatcg.bpf.c +++ b/tools/sched_ext/scx_flatcg.bpf.c @@ -137,11 +137,6 @@ static u64 div_round_up(u64 dividend, u64 divisor) return (dividend + divisor - 1) / divisor; } =20 -static bool vtime_before(u64 a, u64 b) -{ - return (s64)(a - b) < 0; -} - static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node = *b) { struct cgv_node *cgc_a, *cgc_b; @@ -271,7 +266,7 @@ static void cgrp_cap_budget(struct cgv_node *cgv_node, = struct fcg_cgrp_ctx *cgc) */ max_budget =3D (cgrp_slice_ns * nr_cpus * cgc->hweight) / (2 * FCG_HWEIGHT_ONE); - if (vtime_before(cvtime, cvtime_now - max_budget)) + if (time_before(cvtime, cvtime_now - max_budget)) cvtime =3D cvtime_now - max_budget; =20 cgv_node->cvtime =3D cvtime; @@ -401,7 +396,7 @@ void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p,= u64 enq_flags) * Limit the amount of budget that an idling task can accumulate * to one slice. */ - if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) + if (time_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL)) tvtime =3D cgc->tvtime_now - SCX_SLICE_DFL; =20 scx_bpf_dsq_insert_vtime(p, cgrp->kn->id, SCX_SLICE_DFL, @@ -535,7 +530,7 @@ void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) * from multiple CPUs and thus racy. Any error should be * contained and temporary. Let's just live with it. */ - if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime)) + if (time_before(cgc->tvtime_now, p->scx.dsq_vtime)) cgc->tvtime_now =3D p->scx.dsq_vtime; } bpf_cgroup_release(cgrp); @@ -645,7 +640,7 @@ static bool try_pick_next_cgroup(u64 *cgidp) cgv_node =3D container_of(rb_node, struct cgv_node, rb_node); cgid =3D cgv_node->cgid; =20 - if (vtime_before(cvtime_now, cgv_node->cvtime)) + if (time_before(cvtime_now, cgv_node->cvtime)) cvtime_now =3D cgv_node->cvtime; =20 /* @@ -744,7 +739,7 @@ void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_= struct *prev) if (!cpuc->cur_cgid) goto pick_next_cgroup; =20 - if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) { + if (time_before(now, cpuc->cur_at + cgrp_slice_ns)) { if (scx_bpf_dsq_move_to_local(cpuc->cur_cgid)) { stat_inc(FCG_STAT_CNS_KEEP); return; @@ -920,14 +915,14 @@ void BPF_STRUCT_OPS(fcg_cgroup_move, struct task_stru= ct *p, struct cgroup *from, struct cgroup *to) { struct fcg_cgrp_ctx *from_cgc, *to_cgc; - s64 vtime_delta; + s64 delta; =20 /* find_cgrp_ctx() triggers scx_ops_error() on lookup failures */ if (!(from_cgc =3D find_cgrp_ctx(from)) || !(to_cgc =3D find_cgrp_ctx(to)= )) return; =20 - vtime_delta =3D p->scx.dsq_vtime - from_cgc->tvtime_now; - p->scx.dsq_vtime =3D to_cgc->tvtime_now + vtime_delta; + delta =3D time_delta(p->scx.dsq_vtime, from_cgc->tvtime_now); + p->scx.dsq_vtime =3D to_cgc->tvtime_now + delta; } =20 s32 BPF_STRUCT_OPS_SLEEPABLE(fcg_init) diff --git a/tools/sched_ext/scx_simple.bpf.c b/tools/sched_ext/scx_simple.= bpf.c index 31f915b286c6..e6de99dba7db 100644 --- a/tools/sched_ext/scx_simple.bpf.c +++ b/tools/sched_ext/scx_simple.bpf.c @@ -52,11 +52,6 @@ static void stat_inc(u32 idx) (*cnt_p)++; } =20 -static inline bool vtime_before(u64 a, u64 b) -{ - return (s64)(a - b) < 0; -} - s32 BPF_STRUCT_OPS(simple_select_cpu, struct task_struct *p, s32 prev_cpu,= u64 wake_flags) { bool is_idle =3D false; @@ -84,7 +79,7 @@ void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p= , u64 enq_flags) * Limit the amount of budget that an idling task can accumulate * to one slice. */ - if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL)) + if (time_before(vtime, vtime_now - SCX_SLICE_DFL)) vtime =3D vtime_now - SCX_SLICE_DFL; =20 scx_bpf_dsq_insert_vtime(p, SHARED_DSQ, SCX_SLICE_DFL, vtime, @@ -108,7 +103,7 @@ void BPF_STRUCT_OPS(simple_running, struct task_struct = *p) * thus racy. Any error should be contained and temporary. Let's just * live with it. */ - if (vtime_before(vtime_now, p->scx.dsq_vtime)) + if (time_before(vtime_now, p->scx.dsq_vtime)) vtime_now =3D p->scx.dsq_vtime; } =20 --=20 2.47.1