From nobody Fri Oct 3 15:35:59 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7F060322DCE for ; Fri, 29 Aug 2025 15:48:23 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1756482503; cv=none; b=IByCYGgs1UUXc/jKEFxonD2vZtVeDOr20xA6rEW/nQXaD4tBajQ71D6dDr7Js9+KRy1W7vhTSG3lt75xFpt8CABd9FUIgoCkDH0kgKdO4wxU1Xi1RViove3QLJKds1BM04af/TWqR0pjPHtYGLme0ob1LoEzQrLlEu8fb5Cifu0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1756482503; c=relaxed/simple; bh=1NsscIZdy+zc/Q1UQVJIqNLBjDKmHl6bm+ib3E4jCdY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=MNkcCYg2RqaLWrD9/gKq2pI5eNfcbmTgbkbycAXpY79C+PxwLlqXKI1uIc3TtT4xO/7mJIZysGTm/rC9Re4OCr6YqRo+FQ789wPEjMMocRlk0RzfDSrr5jyx9RWEHz2w7N5wbYzZnuP9t9IJ30dZEDWqUu6PJNjjTtSpCBP0EVA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=moJhYSwv; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="moJhYSwv" Received: by smtp.kernel.org (Postfix) with ESMTPSA id BB9CCC4CEF5; Fri, 29 Aug 2025 15:48:20 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1756482503; bh=1NsscIZdy+zc/Q1UQVJIqNLBjDKmHl6bm+ib3E4jCdY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=moJhYSwvkJTae0v8crpHZX4NyzTE8GHMQGDWQusW0tf/X9qrGl0rTpLFXZFdtrMC2 WfQSZPTnxE5QMn0Eh6uC/M22U4IebQeVbi5bAAD5V+1Xom+DRJtcCfuizaG70sftou +la+F9CZjXfeU09KCcR/rpn4L3loSCW0TB4swqDmHGYd0iO9UdnmDxbyUS52NBOCBR tKKgJBf/8IQvYwe+lLU04FGc4/0TWy/JeQBFo3sqUKPm296IRoPbk4Cpqny+TG6qVg JTqtHNAHFmAG1l4r/rV+Flr8AeDCsNyQbkPMzFgifjigDfnj4GQeVCqo1VVygbZift 9bD3RksIsGYlw== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , Ingo Molnar , Marco Crivellari , Michal Hocko , Peter Zijlstra , Tejun Heo , Thomas Gleixner , Vlastimil Babka , Waiman Long Subject: [PATCH 01/33] sched/isolation: Remove housekeeping static key Date: Fri, 29 Aug 2025 17:47:42 +0200 Message-ID: <20250829154814.47015-2-frederic@kernel.org> X-Mailer: git-send-email 2.51.0 In-Reply-To: <20250829154814.47015-1-frederic@kernel.org> References: <20250829154814.47015-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The housekeeping static key in its current use is mostly irrelevant. Most of the time, a housekeeping function call had already been issued before the static call got a chance to be evaluated, defeating the initial call optimization purpose. housekeeping_cpu() is the sole correct user performing the static call before the actual slow-path function call. But it's seldom used in fast-path. Finally the static call prevents from synchronizing correctly against dynamic updates of the housekeeping cpumasks through cpusets. Get away with a simple flag test instead. Signed-off-by: Frederic Weisbecker Reviewed-by: Phil Auld --- include/linux/sched/isolation.h | 25 +++++---- kernel/sched/isolation.c | 90 ++++++++++++++------------------- 2 files changed, 55 insertions(+), 60 deletions(-) diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolatio= n.h index d8501f4709b5..f98ba0d71c52 100644 --- a/include/linux/sched/isolation.h +++ b/include/linux/sched/isolation.h @@ -25,12 +25,22 @@ enum hk_type { }; =20 #ifdef CONFIG_CPU_ISOLATION -DECLARE_STATIC_KEY_FALSE(housekeeping_overridden); +extern unsigned long housekeeping_flags; + extern int housekeeping_any_cpu(enum hk_type type); extern const struct cpumask *housekeeping_cpumask(enum hk_type type); extern bool housekeeping_enabled(enum hk_type type); extern void housekeeping_affine(struct task_struct *t, enum hk_type type); extern bool housekeeping_test_cpu(int cpu, enum hk_type type); + +static inline bool housekeeping_cpu(int cpu, enum hk_type type) +{ + if (housekeeping_flags & BIT(type)) + return housekeeping_test_cpu(cpu, type); + else + return true; +} + extern void __init housekeeping_init(void); =20 #else @@ -58,17 +68,14 @@ static inline bool housekeeping_test_cpu(int cpu, enum = hk_type type) return true; } =20 +static inline bool housekeeping_cpu(int cpu, enum hk_type type) +{ + return true; +} + static inline void housekeeping_init(void) { } #endif /* CONFIG_CPU_ISOLATION */ =20 -static inline bool housekeeping_cpu(int cpu, enum hk_type type) -{ -#ifdef CONFIG_CPU_ISOLATION - if (static_branch_unlikely(&housekeeping_overridden)) - return housekeeping_test_cpu(cpu, type); -#endif - return true; -} =20 static inline bool cpu_is_isolated(int cpu) { diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index a4cf17b1fab0..2a6fc6fc46fb 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -16,19 +16,13 @@ enum hk_flags { HK_FLAG_KERNEL_NOISE =3D BIT(HK_TYPE_KERNEL_NOISE), }; =20 -DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); -EXPORT_SYMBOL_GPL(housekeeping_overridden); - -struct housekeeping { - cpumask_var_t cpumasks[HK_TYPE_MAX]; - unsigned long flags; -}; - -static struct housekeeping housekeeping; +static cpumask_var_t housekeeping_cpumasks[HK_TYPE_MAX]; +unsigned long housekeeping_flags; +EXPORT_SYMBOL_GPL(housekeeping_flags); =20 bool housekeeping_enabled(enum hk_type type) { - return !!(housekeeping.flags & BIT(type)); + return !!(housekeeping_flags & BIT(type)); } EXPORT_SYMBOL_GPL(housekeeping_enabled); =20 @@ -36,50 +30,46 @@ int housekeeping_any_cpu(enum hk_type type) { int cpu; =20 - if (static_branch_unlikely(&housekeeping_overridden)) { - if (housekeeping.flags & BIT(type)) { - cpu =3D sched_numa_find_closest(housekeeping.cpumasks[type], smp_proces= sor_id()); - if (cpu < nr_cpu_ids) - return cpu; + if (housekeeping_flags & BIT(type)) { + cpu =3D sched_numa_find_closest(housekeeping_cpumasks[type], smp_process= or_id()); + if (cpu < nr_cpu_ids) + return cpu; =20 - cpu =3D cpumask_any_and_distribute(housekeeping.cpumasks[type], cpu_onl= ine_mask); - if (likely(cpu < nr_cpu_ids)) - return cpu; - /* - * Unless we have another problem this can only happen - * at boot time before start_secondary() brings the 1st - * housekeeping CPU up. - */ - WARN_ON_ONCE(system_state =3D=3D SYSTEM_RUNNING || - type !=3D HK_TYPE_TIMER); - } + cpu =3D cpumask_any_and_distribute(housekeeping_cpumasks[type], cpu_onli= ne_mask); + if (likely(cpu < nr_cpu_ids)) + return cpu; + /* + * Unless we have another problem this can only happen + * at boot time before start_secondary() brings the 1st + * housekeeping CPU up. + */ + WARN_ON_ONCE(system_state =3D=3D SYSTEM_RUNNING || + type !=3D HK_TYPE_TIMER); } + return smp_processor_id(); } EXPORT_SYMBOL_GPL(housekeeping_any_cpu); =20 const struct cpumask *housekeeping_cpumask(enum hk_type type) { - if (static_branch_unlikely(&housekeeping_overridden)) - if (housekeeping.flags & BIT(type)) - return housekeeping.cpumasks[type]; + if (housekeeping_flags & BIT(type)) + return housekeeping_cpumasks[type]; return cpu_possible_mask; } EXPORT_SYMBOL_GPL(housekeeping_cpumask); =20 void housekeeping_affine(struct task_struct *t, enum hk_type type) { - if (static_branch_unlikely(&housekeeping_overridden)) - if (housekeeping.flags & BIT(type)) - set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]); + if (housekeeping_flags & BIT(type)) + set_cpus_allowed_ptr(t, housekeeping_cpumasks[type]); } EXPORT_SYMBOL_GPL(housekeeping_affine); =20 bool housekeeping_test_cpu(int cpu, enum hk_type type) { - if (static_branch_unlikely(&housekeeping_overridden)) - if (housekeeping.flags & BIT(type)) - return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]); + if (housekeeping_flags & BIT(type)) + return cpumask_test_cpu(cpu, housekeeping_cpumasks[type]); return true; } EXPORT_SYMBOL_GPL(housekeeping_test_cpu); @@ -88,17 +78,15 @@ void __init housekeeping_init(void) { enum hk_type type; =20 - if (!housekeeping.flags) + if (!housekeeping_flags) return; =20 - static_branch_enable(&housekeeping_overridden); - - if (housekeeping.flags & HK_FLAG_KERNEL_NOISE) + if (housekeeping_flags & HK_FLAG_KERNEL_NOISE) sched_tick_offload_init(); =20 - for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) { + for_each_set_bit(type, &housekeeping_flags, HK_TYPE_MAX) { /* We need at least one CPU to handle housekeeping work */ - WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type])); + WARN_ON_ONCE(cpumask_empty(housekeeping_cpumasks[type])); } } =20 @@ -106,8 +94,8 @@ static void __init housekeeping_setup_type(enum hk_type = type, cpumask_var_t housekeeping_staging) { =20 - alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]); - cpumask_copy(housekeeping.cpumasks[type], + alloc_bootmem_cpumask_var(&housekeeping_cpumasks[type]); + cpumask_copy(housekeeping_cpumasks[type], housekeeping_staging); } =20 @@ -117,7 +105,7 @@ static int __init housekeeping_setup(char *str, unsigne= d long flags) unsigned int first_cpu; int err =3D 0; =20 - if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERN= EL_NOISE)) { + if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping_flags & HK_FLAG_KERN= EL_NOISE)) { if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) { pr_warn("Housekeeping: nohz unsupported." " Build with CONFIG_NO_HZ_FULL\n"); @@ -139,7 +127,7 @@ static int __init housekeeping_setup(char *str, unsigne= d long flags) if (first_cpu >=3D nr_cpu_ids || first_cpu >=3D setup_max_cpus) { __cpumask_set_cpu(smp_processor_id(), housekeeping_staging); __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); - if (!housekeeping.flags) { + if (!housekeeping_flags) { pr_warn("Housekeeping: must include one present CPU, " "using boot CPU:%d\n", smp_processor_id()); } @@ -148,7 +136,7 @@ static int __init housekeeping_setup(char *str, unsigne= d long flags) if (cpumask_empty(non_housekeeping_mask)) goto free_housekeeping_staging; =20 - if (!housekeeping.flags) { + if (!housekeeping_flags) { /* First setup call ("nohz_full=3D" or "isolcpus=3D") */ enum hk_type type; =20 @@ -157,26 +145,26 @@ static int __init housekeeping_setup(char *str, unsig= ned long flags) } else { /* Second setup call ("nohz_full=3D" after "isolcpus=3D" or the reverse)= */ enum hk_type type; - unsigned long iter_flags =3D flags & housekeeping.flags; + unsigned long iter_flags =3D flags & housekeeping_flags; =20 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) { if (!cpumask_equal(housekeeping_staging, - housekeeping.cpumasks[type])) { + housekeeping_cpumasks[type])) { pr_warn("Housekeeping: nohz_full=3D must match isolcpus=3D\n"); goto free_housekeeping_staging; } } =20 - iter_flags =3D flags & ~housekeeping.flags; + iter_flags =3D flags & ~housekeeping_flags; =20 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) housekeeping_setup_type(type, housekeeping_staging); } =20 - if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERN= EL_NOISE)) + if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping_flags & HK_FLAG_KERN= EL_NOISE)) tick_nohz_full_setup(non_housekeeping_mask); =20 - housekeeping.flags |=3D flags; + housekeeping_flags |=3D flags; err =3D 1; =20 free_housekeeping_staging: --=20 2.51.0