From nobody Mon Feb 9 07:32:46 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id C2CAA149DE9 for ; Wed, 3 Apr 2024 15:06:10 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712156772; cv=none; b=NQjePjfPLQhOVhEvyO9lJG6G36/Phaw9buBc4AhctJz1BQon/U2wWy3cH3wDGD4Qid6AwGZi7hTwSXOk3Fz7/MTtc9QWhAUgvc80lJ8AGT3l91DTM3aS+naeLeEIbG4s7okQme6ZIDdftOvUG/jRm6IdqXcvl6YrdP4tDo163yc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712156772; c=relaxed/simple; bh=KeJBZfvQpzst1Q8dnNMU32tcHHCcw1VBPsqiOrQaTrg=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=JTkG59YKOHNAHHxn24Oz90dxzgrsF/h11FnfwHQjU0EbBwjLSo8catuvhW8XI476Is/Ch38gc6rvIqltxc+7P2dIK6GsUq5Mjy8vLbhi/2tAxXRgfIYW4QwaiXYtq8Vzhkcf0xD+MPOFY/7ULFU/wfie2ggFjCuEAT8Rs+P1VpM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 492B01650; Wed, 3 Apr 2024 08:06:41 -0700 (PDT) Received: from e126645.arm.com (unknown [10.57.74.15]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 64B0E3F7B4; Wed, 3 Apr 2024 08:06:06 -0700 (PDT) From: Pierre Gondois To: linux-kernel@vger.kernel.org Cc: Aaron Lu , Rui Zhang , Pierre Gondois , Anna-Maria Behnsen , Frederic Weisbecker , Ingo Molnar , Thomas Gleixner , Peter Zijlstra , Juri Lelli , Vincent Guittot , Dietmar Eggemann , Steven Rostedt , Ben Segall , Mel Gorman , Daniel Bristot de Oliveira , Valentin Schneider , Tejun Heo , Michal Hocko , Waiman Long , Andrew Morton Subject: [PATCH 1/7] sched/isolation: Introduce housekeeping_runtime isolation Date: Wed, 3 Apr 2024 17:05:33 +0200 Message-Id: <20240403150543.2793354-2-pierre.gondois@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20240403150543.2793354-1-pierre.gondois@arm.com> References: <20240403150543.2793354-1-pierre.gondois@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" CONFIG_CPU_ISOLATION allows to setup various cpu masks to exclude CPUs from some activities. Masks that are not modified default to cpu_online_mask. Masks are set at boot time. If no mask is modified, the static key 'housekeeping_overridden' is left to false, allowing to minimize the cost of calls to housekeeping_*() functions. Create a new housekeeping runtime type, whose isolation masks can be modified at runtime. Also add a set of functions around this new type. This type is independent from the 'housekeeping_overridden' static key. Signed-off-by: Pierre Gondois --- include/linux/sched/isolation.h | 28 +++++++++++++++++++++ kernel/sched/isolation.c | 43 +++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolatio= n.h index 2b461129d1fa..5d2f40c6f04c 100644 --- a/include/linux/sched/isolation.h +++ b/include/linux/sched/isolation.h @@ -6,6 +6,10 @@ #include #include =20 +enum hkr_type { + HKR_TYPE_MAX +}; + enum hk_type { HK_TYPE_TIMER, HK_TYPE_RCU, @@ -26,6 +30,12 @@ extern const struct cpumask *housekeeping_cpumask(enum h= k_type type); extern bool housekeeping_enabled(enum hk_type type); extern void housekeeping_affine(struct task_struct *t, enum hk_type type); extern bool housekeeping_test_cpu(int cpu, enum hk_type type); + +extern const struct cpumask *housekeeping_runtime_cpumask(enum hkr_type ty= pe); +extern bool housekeeping_runtime_test_cpu(int cpu, enum hkr_type type); +extern void housekeeping_runtime_set_cpu(int cpu, enum hkr_type type); +extern void housekeeping_runtime_clear_cpu(int cpu, enum hkr_type type); + extern void __init housekeeping_init(void); =20 #else @@ -53,6 +63,24 @@ static inline bool housekeeping_test_cpu(int cpu, enum h= k_type type) return true; } =20 +static inline const struct cpumask *housekeeping_runtime_cpumask(enum hkr_= type type) +{ + return cpu_possible_mask; +} + +static inline bool housekeeping_runtime_test_cpu(int cpu, enum hkr_type ty= pe) +{ + return true; +} + +static inline void housekeeping_runtime_set_cpu(int cpu, enum hkr_type typ= e) +{ +} + +static inline void housekeeping_runtime_clear_cpu(int cpu, enum hkr_type t= ype) +{ +} + static inline void housekeeping_init(void) { } #endif /* CONFIG_CPU_ISOLATION */ =20 diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index 373d42c707bc..5acbed870c28 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -23,6 +23,13 @@ enum hk_flags { DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); EXPORT_SYMBOL_GPL(housekeeping_overridden); =20 +struct housekeeping_runtime { + cpumask_var_t cpumasks[HKR_TYPE_MAX]; + unsigned long flags; +}; + +static struct housekeeping_runtime housekeeping_runtime; + struct housekeeping { cpumask_var_t cpumasks[HK_TYPE_MAX]; unsigned long flags; @@ -79,10 +86,46 @@ bool housekeeping_test_cpu(int cpu, enum hk_type type) } EXPORT_SYMBOL_GPL(housekeeping_test_cpu); =20 +const struct cpumask *housekeeping_runtime_cpumask(enum hkr_type type) +{ + if (housekeeping_runtime.cpumasks[type]) + return housekeeping_runtime.cpumasks[type]; + return cpu_possible_mask; +} + +bool housekeeping_runtime_test_cpu(int cpu, enum hkr_type type) +{ + if (housekeeping_runtime.cpumasks[type]) + return cpumask_test_cpu(cpu, housekeeping_runtime.cpumasks[type]); + return true; +} + +void housekeeping_runtime_set_cpu(int cpu, enum hkr_type type) +{ + if (housekeeping_runtime.cpumasks[type]) + cpumask_set_cpu(cpu, housekeeping_runtime.cpumasks[type]); +} + +void housekeeping_runtime_clear_cpu(int cpu, enum hkr_type type) +{ + if (housekeeping_runtime.cpumasks[type]) + cpumask_clear_cpu(cpu, housekeeping_runtime.cpumasks[type]); +} + +static void __init housekeeping_runtime_init(void) +{ + enum hkr_type type; + + for (type =3D 0; type < HKR_TYPE_MAX; type++) + alloc_cpumask_var(&housekeeping_runtime.cpumasks[type], GFP_KERNEL); +} + void __init housekeeping_init(void) { enum hk_type type; =20 + housekeeping_runtime_init(); + if (!housekeeping.flags) return; =20 --=20 2.25.1 From nobody Mon Feb 9 07:32:46 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 3EA8213A272 for ; Wed, 3 Apr 2024 15:06:22 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712156784; cv=none; b=I9+xA6PyAqY5lXHlP2B66DaNX7435p9h+Z9d43VixDrYgBOAooggrReShZJHvEMO63PivRL03FTJv57MaE8D2p+dQm4zkt8EOFB8ugdH8xO8dEVhzI5UyM93NN/Q1S57zrurm2mgXsC25DAG47j0wVyyrpdF5CehOHDIN/Zhy6I= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712156784; c=relaxed/simple; bh=YrK76JRYQ9vB2jJg41sUj5FbQqSSvT+rqTxOnYT1bKE=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=dmTcSe3kj+nvCPETPlLkj2TN9j0nKAGGGLtoLcypLZXPZnbPAQfn8i+pV63MYo2Ar65UEr7iYhwG5Tq+kACSBK9Z04VCbm0mHP0+pL0PB+yqhAPLZ8xfgHoQEJgmN882YS2zFBlKxuSMbYMtfMUNcymguUY1F5BYepuLjDdLFfg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id AE0B91655; Wed, 3 Apr 2024 08:06:52 -0700 (PDT) Received: from e126645.arm.com (unknown [10.57.74.15]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 486973F7B4; Wed, 3 Apr 2024 08:06:17 -0700 (PDT) From: Pierre Gondois To: linux-kernel@vger.kernel.org Cc: Aaron Lu , Rui Zhang , Pierre Gondois , Anna-Maria Behnsen , Frederic Weisbecker , Ingo Molnar , Thomas Gleixner , Peter Zijlstra , Juri Lelli , Vincent Guittot , Dietmar Eggemann , Steven Rostedt , Ben Segall , Mel Gorman , Daniel Bristot de Oliveira , Valentin Schneider , Michal Hocko , Waiman Long , Tejun Heo , Andrew Morton Subject: [PATCH 2/7] sched/isolation: Move HK_TYPE_SCHED to housekeeping runtime Date: Wed, 3 Apr 2024 17:05:34 +0200 Message-Id: <20240403150543.2793354-3-pierre.gondois@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20240403150543.2793354-1-pierre.gondois@arm.com> References: <20240403150543.2793354-1-pierre.gondois@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The HK_TYPE_SCHED isolation mask is never modified. It is however referenced in the scheduler code to ignore CPUs not taking part in load balancing for instance. Move the HK_TYPE_SCHED to the newly created housekeeping runtime type. Places where HK_TYPE_SCHED is used are not impacted as: - the HKR_FLAG_SCHED flag isn't used by the isolcpus/nohz_full kernel parameters - masks not set though kernel parameters default to the cpu_online_mask Signed-off-by: Pierre Gondois --- include/linux/sched/isolation.h | 2 +- kernel/sched/fair.c | 4 ++-- kernel/sched/isolation.c | 5 ++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolatio= n.h index 5d2f40c6f04c..80b4e26a1b73 100644 --- a/include/linux/sched/isolation.h +++ b/include/linux/sched/isolation.h @@ -7,6 +7,7 @@ #include =20 enum hkr_type { + HKR_TYPE_SCHED, HKR_TYPE_MAX }; =20 @@ -14,7 +15,6 @@ enum hk_type { HK_TYPE_TIMER, HK_TYPE_RCU, HK_TYPE_MISC, - HK_TYPE_SCHED, HK_TYPE_TICK, HK_TYPE_DOMAIN, HK_TYPE_WQ, diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 1dd37168da50..e3d707243ca7 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -12083,7 +12083,7 @@ void nohz_balance_enter_idle(int cpu) return; =20 /* Spare idle load balancing on CPUs that don't want to be disturbed: */ - if (!housekeeping_cpu(cpu, HK_TYPE_SCHED)) + if (!housekeeping_runtime_test_cpu(cpu, HKR_TYPE_SCHED)) return; =20 /* @@ -12309,7 +12309,7 @@ static void nohz_newidle_balance(struct rq *this_rq) * This CPU doesn't want to be disturbed by scheduler * housekeeping */ - if (!housekeeping_cpu(this_cpu, HK_TYPE_SCHED)) + if (!housekeeping_runtime_test_cpu(this_cpu, HKR_TYPE_SCHED)) return; =20 /* Will wake up very soon. No time for doing anything else*/ diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index 5acbed870c28..735925578815 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -8,11 +8,14 @@ * */ =20 +enum hk_runtime_flags { + HKR_FLAG_SCHED =3D BIT(HKR_TYPE_SCHED), +}; + enum hk_flags { HK_FLAG_TIMER =3D BIT(HK_TYPE_TIMER), HK_FLAG_RCU =3D BIT(HK_TYPE_RCU), HK_FLAG_MISC =3D BIT(HK_TYPE_MISC), - HK_FLAG_SCHED =3D BIT(HK_TYPE_SCHED), HK_FLAG_TICK =3D BIT(HK_TYPE_TICK), HK_FLAG_DOMAIN =3D BIT(HK_TYPE_DOMAIN), HK_FLAG_WQ =3D BIT(HK_TYPE_WQ), --=20 2.25.1 From nobody Mon Feb 9 07:32:46 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 6797B149DE3 for ; Wed, 3 Apr 2024 15:06:31 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712156792; cv=none; b=Gq/cix349ETa+DDdUG96/VzGsU9+FWjCBpV2uMhKa5ezJvV8rqx2t0pd5NZxAfHFgchRUQc9B//A6vYm9ODCW5QLgz/g5v+7sbzRT6WDubV7Bh4Sk1CuOfv6I04APL6XTyPaorh95tmY3LfXTCZluUz9LwQfCed+a/i6Jc6LkqM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712156792; c=relaxed/simple; bh=54N//QKAL73VOVwcfjNuqoX3Jag5no7obFOKhG6hRyI=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=Ol19/pK/hhYXctx/r1lfxPor/zpIJhOXWGqdc+MCUTFZbQRJmaW5pnaImHxSMkkQ4jE7hNJ8DeMANTVkmaawQ9SVuJzF2pCfuyIoQfrYEgz41t1xxsTED5qMRolNHYAieI8RbaGUIe+4HXhGFpzJ6r2p9ECeoPJT+1ohPwHn8RY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id E2D7E1684; Wed, 3 Apr 2024 08:07:01 -0700 (PDT) Received: from e126645.arm.com (unknown [10.57.74.15]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id A7F1D3F7B4; Wed, 3 Apr 2024 08:06:26 -0700 (PDT) From: Pierre Gondois To: linux-kernel@vger.kernel.org Cc: Aaron Lu , Rui Zhang , Pierre Gondois , Anna-Maria Behnsen , Frederic Weisbecker , Ingo Molnar , Thomas Gleixner , Peter Zijlstra , Juri Lelli , Vincent Guittot , Dietmar Eggemann , Steven Rostedt , Ben Segall , Mel Gorman , Daniel Bristot de Oliveira , Valentin Schneider , Andrew Morton , Michal Hocko , Waiman Long Subject: [PATCH 3/7] sched/isolation: Use HKR_TYPE_SCHED in find_new_ilb() Date: Wed, 3 Apr 2024 17:05:35 +0200 Message-Id: <20240403150543.2793354-4-pierre.gondois@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20240403150543.2793354-1-pierre.gondois@arm.com> References: <20240403150543.2793354-1-pierre.gondois@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Replace the HK_TYPE_MISC isolation mask with HKR_TYPE_SCHED in find_new_ilb(). This ultimately resolves [1], i.e. selecting a CPU for the nohz idle balance which cannot pull tasks through: sched_balance_trigger() \-nohz_balancer_kick() \-kick_ilb() \-find_new_ilb() \-smp_call_function_single_async() \-nohz_csd_cpu() \-raise_softirq_irqoff() sched_balance_softirq() \-_nohz_idle_balance() \-sched_balance_domains() Indeed, a following patch will add/remove CPUs with a NULL sched domain to the HKR_TYPE_SCHED cpumask. [1] https://lore.kernel.org/all/20230804090858.7605-1-rui.zhang@intel.com/ Reported-by: Zhang Rui Signed-off-by: Pierre Gondois --- kernel/sched/fair.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index e3d707243ca7..0665f5eb4703 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -11842,16 +11842,13 @@ static inline int on_null_domain(struct rq *rq) * - When one of the busy CPUs notices that there may be an idle rebalanci= ng * needed, they will kick the idle load balancer, which then does idle * load balancing for all the idle CPUs. - * - * - HK_TYPE_MISC CPUs are used for this task, because HK_TYPE_SCHED is no= t set - * anywhere yet. */ static inline int find_new_ilb(void) { const struct cpumask *hk_mask; int ilb_cpu; =20 - hk_mask =3D housekeeping_cpumask(HK_TYPE_MISC); + hk_mask =3D housekeeping_runtime_cpumask(HKR_TYPE_SCHED); =20 for_each_cpu_and(ilb_cpu, nohz.idle_cpus_mask, hk_mask) { =20 --=20 2.25.1 From nobody Mon Feb 9 07:32:46 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 9D64A14A0A6 for ; Wed, 3 Apr 2024 15:06:41 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712156803; cv=none; b=Y0js+CFdbQzhJQbEMaBEGxO3DiTWUSDQ5xtTdMGNS0gPz14M6dPRiEGrwTkrWVWkVpqBOyKgNb7AG7Sez1FEjuDkKQW/KGMH7zYPDqLslgZ66A3OPrCn+CV6muOJyl50+/WXz6pM0aZNFgE17RDz+JBCv/hrCg/B8pLbl5KLKx8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712156803; c=relaxed/simple; bh=pimyCXvc7YX8asor/As9BfBUS8PADGX9CEj0YFa8W2Y=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=RvDbhsV1C/cdlKIxWB4Gm/bKwdNFIUn0oNNbYd1LefIjgv6SUjp5DJcCtX5vV7Jq+Ug1E24Z5GXDMLkRTsnQOlgj6uAJPRo0OhK08L/+HOD1QWbcCN4Fj9bCTSaKHTcnQsZKrGZSdbzbJexqswLeTqdp7a5JklXxqlFexMYfyac= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id EDA771688; Wed, 3 Apr 2024 08:07:11 -0700 (PDT) Received: from e126645.arm.com (unknown [10.57.74.15]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 338893F7B4; Wed, 3 Apr 2024 08:06:36 -0700 (PDT) From: Pierre Gondois To: linux-kernel@vger.kernel.org Cc: Aaron Lu , Rui Zhang , Pierre Gondois , Anna-Maria Behnsen , Frederic Weisbecker , Ingo Molnar , Thomas Gleixner , Peter Zijlstra , Juri Lelli , Vincent Guittot , Dietmar Eggemann , Steven Rostedt , Ben Segall , Mel Gorman , Daniel Bristot de Oliveira , Valentin Schneider , Andrew Morton , Michal Hocko , Tejun Heo , Waiman Long Subject: [PATCH 4/7] sched/fair: Move/add on_null_domain()/housekeeping_cpu() checks Date: Wed, 3 Apr 2024 17:05:36 +0200 Message-Id: <20240403150543.2793354-5-pierre.gondois@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20240403150543.2793354-1-pierre.gondois@arm.com> References: <20240403150543.2793354-1-pierre.gondois@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Prepare a following patch by moving/adding on_null_domain() and housekeeping_runtime_cpu() checks. In nohz_balance_enter_idle(): - The housekeeping_runtime_cpu(cpu, HKR_TYPE_SCHED) call is currently a no-op as HKR_TYPE_SCHED is never configured. The call can thus be moved down. - In the current code, an isolated CPU sets nohz.has_blocked, but isn't set in nohz.idle_cpus_mask. However, _nohz_idle_balance::for_each_cpu_wrap() iterates over nohz.idle_cpus_mask cpus. Move the check up to avoid this. In nohz_balance_exit_idle(): - The check against a NULL sd in: nohz_balance_enter_idle() \-if (on_null_domain()) \-[returning here] \-rq->nohz_tick_stopped =3D 1 prevents from setting the rq's nohz_tick_stopped, and sched_balance_trigger() \-if (on_null_domain()) \-[returning here] \-nohz_balancer_kick() \-nohz_balance_exit_idle() prevents from resetting the nohz.[nr_cpus|idle_cpus_mask] variables. So the newly added on_null_domain() check doesn't change current behaviour. It however prepares: - the use of the HKR_TYPE_SCHED isolation mask - the removal of on_null_domain() in a later patch. Signed-off-by: Pierre Gondois --- kernel/sched/fair.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 0665f5eb4703..3e0f2a0f153f 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -12039,6 +12039,10 @@ void nohz_balance_exit_idle(struct rq *rq) { SCHED_WARN_ON(rq !=3D this_rq()); =20 + /* If we're a completely isolated CPU, we don't play: */ + if (on_null_domain(rq)) + return; + if (likely(!rq->nohz_tick_stopped)) return; =20 @@ -12079,10 +12083,6 @@ void nohz_balance_enter_idle(int cpu) if (!cpu_active(cpu)) return; =20 - /* Spare idle load balancing on CPUs that don't want to be disturbed: */ - if (!housekeeping_runtime_test_cpu(cpu, HKR_TYPE_SCHED)) - return; - /* * Can be set safely without rq->lock held * If a clear happens, it will have evaluated last additions because @@ -12090,6 +12090,14 @@ void nohz_balance_enter_idle(int cpu) */ rq->has_blocked_load =3D 1; =20 + /* Spare idle load balancing on CPUs that don't want to be disturbed: */ + if (!housekeeping_runtime_test_cpu(cpu, HKR_TYPE_SCHED)) + return; + + /* If we're a completely isolated CPU, we don't play: */ + if (on_null_domain(rq)) + return; + /* * The tick is still stopped but load could have been added in the * meantime. We set the nohz.has_blocked flag to trig a check of the @@ -12099,10 +12107,6 @@ void nohz_balance_enter_idle(int cpu) if (rq->nohz_tick_stopped) goto out; =20 - /* If we're a completely isolated CPU, we don't play: */ - if (on_null_domain(rq)) - return; - rq->nohz_tick_stopped =3D 1; =20 cpumask_set_cpu(cpu, nohz.idle_cpus_mask); --=20 2.25.1 From nobody Mon Feb 9 07:32:46 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id AD1B614AD12 for ; Wed, 3 Apr 2024 15:06:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712156812; cv=none; b=awX1nzxY90tSVmSzEu2ur8h3sra5G81SVfnzmPf9gxo4VDD1Kr9B/Yc/uEIEzFKkTNjiSL/5eSTLpoeitBiRasVq+hA6lMeUS20wUmYTvRTxIA3KE3/e++iqieSKIlarIfV4b0rg8j8+vgzeZuODM4ouiUNXe0x/uYnkqIPzZtE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712156812; c=relaxed/simple; bh=KP3KQkWwmKSjgJr+k2KTuvgcjSIjuAYMNmf/QgiXrPU=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=brImyySHLoX3Qp84yqT5kfX/wFTsTtxdIXCGwcXnISxRIKiSX48TZ8e1jDBujL5iznENCSZcP7T4CZrsdivkNPTlIVHl0b6WNV7yyrLdlP0HcgehkoAcQsMr4hPkYby7Ynj0/19DmtOwQMvdJyPNei3lH//FlSxgwCJA89Tc6V4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 5301F168F; Wed, 3 Apr 2024 08:07:21 -0700 (PDT) Received: from e126645.arm.com (unknown [10.57.74.15]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id B37DA3F7B4; Wed, 3 Apr 2024 08:06:45 -0700 (PDT) From: Pierre Gondois To: linux-kernel@vger.kernel.org Cc: Aaron Lu , Rui Zhang , Pierre Gondois , Anna-Maria Behnsen , Frederic Weisbecker , Ingo Molnar , Thomas Gleixner , Peter Zijlstra , Juri Lelli , Vincent Guittot , Dietmar Eggemann , Steven Rostedt , Ben Segall , Mel Gorman , Daniel Bristot de Oliveira , Valentin Schneider , Tejun Heo , Michal Hocko , Andrew Morton , Waiman Long Subject: [PATCH 5/7] sched/topology: Remove CPUs with NULL sd from HKR_TYPE_SCHED mask Date: Wed, 3 Apr 2024 17:05:37 +0200 Message-Id: <20240403150543.2793354-6-pierre.gondois@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20240403150543.2793354-1-pierre.gondois@arm.com> References: <20240403150543.2793354-1-pierre.gondois@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Upon attaching a NULL sched domain to a CPU, remove the CPU from the HKR_TYPE_SCHED isolation mask. CPUs present in this mask are prevented from being added to the fair.c variables: - nohz.idle_cpus_mask - nohz.nr_cpus Signed-off-by: Pierre Gondois --- kernel/sched/topology.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 63aecd2a7a9f..b4fc212ccfb0 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -775,6 +775,11 @@ cpu_attach_domain(struct sched_domain *sd, struct root= _domain *rd, int cpu) =20 sched_domain_debug(sd, cpu); =20 + if (sd) + housekeeping_runtime_set_cpu(cpu, HKR_TYPE_SCHED); + else + housekeeping_runtime_clear_cpu(cpu, HKR_TYPE_SCHED); + rq_attach_root(rq, rd); tmp =3D rq->sd; rcu_assign_pointer(rq->sd, sd); --=20 2.25.1 From nobody Mon Feb 9 07:32:46 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 2ED7F149E0A for ; Wed, 3 Apr 2024 15:07:00 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712156821; cv=none; b=jN71auGUHpWPpZdJu3OaIA8E2ePdHwKwRJsC3iVwhwOOhF13xTtTaCqB/sdUxwX/rqBXLlztZKfy3Cc3oQskzk+pYyj0n+d1JstNF0bn4ctwLJR5rS2qpgL5eGmhxscIudh+rXG+VLdPSmc2rcALvGsbHwvj2NqUgmADOcDhqhc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712156821; c=relaxed/simple; bh=7a1odYqmKjFL1UduF7vAKTCHsTnZIxWKisHGINaxoB0=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=ra6n+9pqIaoyihtvoaUK1rjc071SMsLv7k0MUe987BEX8hlqY7JWfQNkTNTLdJTPnL3T5GmDnRRu+8TLYgR03MnUCBdlIUXwWjxtjREbeUVaLaUnFCPrC0gGmzLSKHnHHQAchVeopeF20RTvy1S3JCgwJaknHV6GPoe+Llh/bWo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id AC1231691; Wed, 3 Apr 2024 08:07:30 -0700 (PDT) Received: from e126645.arm.com (unknown [10.57.74.15]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 71E283F7B4; Wed, 3 Apr 2024 08:06:55 -0700 (PDT) From: Pierre Gondois To: linux-kernel@vger.kernel.org Cc: Aaron Lu , Rui Zhang , Pierre Gondois , Anna-Maria Behnsen , Frederic Weisbecker , Ingo Molnar , Thomas Gleixner , Peter Zijlstra , Juri Lelli , Vincent Guittot , Dietmar Eggemann , Steven Rostedt , Ben Segall , Mel Gorman , Daniel Bristot de Oliveira , Valentin Schneider , Waiman Long , Andrew Morton , Michal Hocko Subject: [PATCH 6/7] sched/fair: Remove on_null_domain() and redundant checks Date: Wed, 3 Apr 2024 17:05:38 +0200 Message-Id: <20240403150543.2793354-7-pierre.gondois@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20240403150543.2793354-1-pierre.gondois@arm.com> References: <20240403150543.2793354-1-pierre.gondois@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" CPUs with a NULL sched domain are removed from the HKR_TYPE_SCHED isolation mask. The two following checks are equialent: - !housekeeping_runtime_test_cpu(cpu, HKR_TYPE_SCHED) - on_null_domain(rq) Remove on_null_domain() and the redundant checks. Signed-off-by: Pierre Gondois --- kernel/sched/fair.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 3e0f2a0f153f..9657c8f2176b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -11830,11 +11830,6 @@ static void sched_balance_domains(struct rq *rq, e= num cpu_idle_type idle) =20 } =20 -static inline int on_null_domain(struct rq *rq) -{ - return unlikely(!rcu_dereference_sched(rq->sd)); -} - #ifdef CONFIG_NO_HZ_COMMON /* * NOHZ idle load balancing (ILB) details: @@ -12040,7 +12035,7 @@ void nohz_balance_exit_idle(struct rq *rq) SCHED_WARN_ON(rq !=3D this_rq()); =20 /* If we're a completely isolated CPU, we don't play: */ - if (on_null_domain(rq)) + if (!housekeeping_runtime_test_cpu(cpu_of(rq), HKR_TYPE_SCHED)) return; =20 if (likely(!rq->nohz_tick_stopped)) @@ -12090,12 +12085,8 @@ void nohz_balance_enter_idle(int cpu) */ rq->has_blocked_load =3D 1; =20 - /* Spare idle load balancing on CPUs that don't want to be disturbed: */ - if (!housekeeping_runtime_test_cpu(cpu, HKR_TYPE_SCHED)) - return; - /* If we're a completely isolated CPU, we don't play: */ - if (on_null_domain(rq)) + if (!housekeeping_runtime_test_cpu(cpu, HKR_TYPE_SCHED)) return; =20 /* @@ -12504,11 +12495,14 @@ static __latent_entropy void sched_balance_softir= q(struct softirq_action *h) */ void sched_balance_trigger(struct rq *rq) { + int cpu =3D cpu_of(rq); + /* * Don't need to rebalance while attached to NULL domain or * runqueue CPU is not active */ - if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq)))) + if (unlikely(!housekeeping_runtime_test_cpu(cpu, HKR_TYPE_SCHED)) || + !cpu_active(cpu)) return; =20 if (time_after_eq(jiffies, rq->next_balance)) --=20 2.25.1 From nobody Mon Feb 9 07:32:46 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id F1C4B149E0A for ; Wed, 3 Apr 2024 15:07:08 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712156830; cv=none; b=CXR+3PhdlcY904SSaygKOrhTCAoBj3Rli4KBNvuGfowZYoKvhqrche0S/aNJWE3Uzx72FEvmJtIHtmSI0FGQJGLM//GDi6ixkKTDNBMYZ8Y3hlRAOGbimsHkgmTClpgXufYoF6S46NngErqL0DdUNAOdEbZLeJfYOT+u4RS2s5I= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1712156830; c=relaxed/simple; bh=W3JT2si9ImtOXLQWk2Xzg1pUyMSMDml9LjW4wwC97aw=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=nTuC6jyOIJGZpP8M5Wz/FMYBMkdP6qOYBp4Dm3YCfXdQ5BpPazQk/WvuEpL9fb0WETRSJsIfQ+0EF3JS5OI5oP/DTPmReVXJQUFeJGiMI9sFeFC2F8O2X4K3qi6ubv2Q4zFH/GQxcnHX1kygtJMPJHszU//Uply/DpvryLfwPUc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id AF09F1691; Wed, 3 Apr 2024 08:07:39 -0700 (PDT) Received: from e126645.arm.com (unknown [10.57.74.15]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 4DFCF3F7B4; Wed, 3 Apr 2024 08:07:04 -0700 (PDT) From: Pierre Gondois To: linux-kernel@vger.kernel.org Cc: Aaron Lu , Rui Zhang , Pierre Gondois , Anna-Maria Behnsen , Frederic Weisbecker , Ingo Molnar , Thomas Gleixner , Peter Zijlstra , Juri Lelli , Vincent Guittot , Dietmar Eggemann , Steven Rostedt , Ben Segall , Mel Gorman , Daniel Bristot de Oliveira , Valentin Schneider , Andrew Morton , Waiman Long , Tejun Heo Subject: [PATCH 7/7] sched/fair: Clear idle_cpus_mask for CPUs with NULL sd Date: Wed, 3 Apr 2024 17:05:39 +0200 Message-Id: <20240403150543.2793354-8-pierre.gondois@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20240403150543.2793354-1-pierre.gondois@arm.com> References: <20240403150543.2793354-1-pierre.gondois@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" As reported in [1], an isolated CPU keeps the values of: - rq->nohz_tick_stopped - nohz.idle_cpus_mask - nohz.nr_cpus when a NULL sd is attached to the CPU. Clear the values. [1] https://lore.kernel.org/all/20230804090858.7605-1-rui.zhang@intel.com/ Signed-off-by: Pierre Gondois --- include/linux/sched/nohz.h | 2 ++ kernel/sched/fair.c | 11 +++++++++++ kernel/sched/topology.c | 6 ++++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/include/linux/sched/nohz.h b/include/linux/sched/nohz.h index 6d67e9a5af6b..18e620715c9d 100644 --- a/include/linux/sched/nohz.h +++ b/include/linux/sched/nohz.h @@ -9,8 +9,10 @@ #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) extern void nohz_balance_enter_idle(int cpu); extern int get_nohz_timer_target(void); +extern void nohz_clear_state(int cpu); #else static inline void nohz_balance_enter_idle(int cpu) { } +static void nohz_clear_state(int cpu) { } #endif =20 #ifdef CONFIG_NO_HZ_COMMON diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 9657c8f2176b..6786d4d78e41 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -12014,6 +12014,17 @@ static void nohz_balancer_kick(struct rq *rq) kick_ilb(flags); } =20 +void nohz_clear_state(int cpu) +{ + struct rq *rq =3D cpu_rq(cpu); + + if (rq->nohz_tick_stopped) { + rq->nohz_tick_stopped =3D 0; + cpumask_clear_cpu(cpu, nohz.idle_cpus_mask); + atomic_dec(&nohz.nr_cpus); + } +} + static void set_cpu_sd_state_busy(int cpu) { struct sched_domain *sd; diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index b4fc212ccfb0..e8e40b7d964b 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -775,10 +775,12 @@ cpu_attach_domain(struct sched_domain *sd, struct roo= t_domain *rd, int cpu) =20 sched_domain_debug(sd, cpu); =20 - if (sd) + if (sd) { housekeeping_runtime_set_cpu(cpu, HKR_TYPE_SCHED); - else + } else { housekeeping_runtime_clear_cpu(cpu, HKR_TYPE_SCHED); + nohz_clear_state(cpu); + } =20 rq_attach_root(rq, rd); tmp =3D rq->sd; --=20 2.25.1