From nobody Thu Sep 19 00:59:39 2024 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 6B95E1534EC; Fri, 26 Jul 2024 21:57:58 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1722031078; cv=none; b=ZUqU9yl/z/HKKXTVIVPdZ6qDbeJJnB04PVh1qHpFPm2glRo3j1CbHCqZy6iB7JfR1/PfR2gDUSIdlJNCfwJXxGKSa9UrPx0xJDgEjtuLPF4IIttizSbOSDh9n3oy/BwjnzC7jolXROvG8XU5CGDMEs50UcHEXlE1fP3HTcJ4+a8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1722031078; c=relaxed/simple; bh=Yz6bJUlYF+iMHAiEYs34cVgoTt8huaFYwVmLFroPfBE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=g2EI3H7nGDSeo/3ng3NHpCd6bJmLa5CpBapHHaEvH3wSJn1jxoD8ELgOTgKtTbfpggDRXXkDVXZvcS/RP9MYmUhr0NJ5Uc6HVimB/X3L2sKd3nzf9pRW/Zebu2VzGFjgRFjaIWWsK9tboW1XAqkMvhO4Fz0txZAYTLyII3OrFpE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=FSXgr8gw; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="FSXgr8gw" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 92415C4AF0B; Fri, 26 Jul 2024 21:57:55 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1722031078; bh=Yz6bJUlYF+iMHAiEYs34cVgoTt8huaFYwVmLFroPfBE=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=FSXgr8gw6zdBoVhwLdneQntEUu+abZ/Ach0L65MTN35uk9wogOwhZRMYHMrrSycTe 8bReYUBFBSCMGDCLGsJC3msiEp1UPf3bZyTCXOdn8LewsSAVlW10jJGMoXf2MSFyjd ooyyzp2RbVRoCsIV8aioV2Y6Clqu3KYfkw/VLLSRND9qUsKV67KNsD/cnRSBUd2eNH sOcjXGTvGjvScLH8fRZ5kkorPEcw5EIDkHQOJ44QRciDWxf7N/BTSwMijjpFEzKTmL HXVRWEvtGYwvqZ8Q4oxxOm+5m0/7w6rbNJMejSNP9FruDG91/dwpGj6DqKPsH6k3Yi H447CgOiKGM3g== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Paul E. McKenney" , Neeraj Upadhyay , Joel Fernandes , Boqun Feng , Zqiang , rcu@vger.kernel.org, Andrew Morton , Peter Zijlstra , Thomas Gleixner Subject: [RFC PATCH 20/20] rcu: Use kthread preferred affinity for RCU exp kworkers Date: Fri, 26 Jul 2024 23:56:56 +0200 Message-ID: <20240726215701.19459-21-frederic@kernel.org> X-Mailer: git-send-email 2.45.2 In-Reply-To: <20240726215701.19459-1-frederic@kernel.org> References: <20240726215701.19459-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Now that kthreads have an infrastructure to handle preferred affinity against CPU hotplug and housekeeping cpumask, convert RCU exp workers to use it instead of handling all the constraints by itself. Signed-off-by: Frederic Weisbecker Acked-by: Paul E. McKenney --- kernel/rcu/tree.c | 105 +++++++++------------------------------------- 1 file changed, 19 insertions(+), 86 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index e038f4abb872..f3e40a1dea65 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4777,6 +4777,22 @@ rcu_boot_init_percpu_data(int cpu) rcu_boot_init_nocb_percpu_data(rdp); } =20 +static void rcu_thread_affine_rnp(struct task_struct *t, struct rcu_node *= rnp) +{ + cpumask_var_t affinity; + int cpu; + + if (!zalloc_cpumask_var(&affinity, GFP_KERNEL)) + return; + + for_each_leaf_node_possible_cpu(rnp, cpu) + cpumask_set_cpu(cpu, affinity); + + kthread_affine_preferred(t, affinity); + + free_cpumask_var(affinity); +} + struct kthread_worker *rcu_exp_gp_kworker; =20 static void rcu_spawn_exp_par_gp_kworker(struct rcu_node *rnp) @@ -4789,7 +4805,7 @@ static void rcu_spawn_exp_par_gp_kworker(struct rcu_n= ode *rnp) if (rnp->exp_kworker) return; =20 - kworker =3D kthread_run_worker(0, name, rnp_index); + kworker =3D kthread_create_worker(0, name, rnp_index); if (IS_ERR_OR_NULL(kworker)) { pr_err("Failed to create par gp kworker on %d/%d\n", rnp->grplo, rnp->grphi); @@ -4799,16 +4815,9 @@ static void rcu_spawn_exp_par_gp_kworker(struct rcu_= node *rnp) =20 if (IS_ENABLED(CONFIG_RCU_EXP_KTHREAD)) sched_setscheduler_nocheck(kworker->task, SCHED_FIFO, ¶m); -} =20 -static struct task_struct *rcu_exp_par_gp_task(struct rcu_node *rnp) -{ - struct kthread_worker *kworker =3D READ_ONCE(rnp->exp_kworker); - - if (!kworker) - return NULL; - - return kworker->task; + rcu_thread_affine_rnp(kworker->task, rnp); + wake_up_process(kworker->task); } =20 static void __init rcu_start_exp_gp_kworker(void) @@ -4893,79 +4902,6 @@ int rcutree_prepare_cpu(unsigned int cpu) return 0; } =20 -static void rcu_thread_affine_rnp(struct task_struct *t, struct rcu_node *= rnp) -{ - cpumask_var_t affinity; - int cpu; - - if (!zalloc_cpumask_var(&affinity, GFP_KERNEL)) - return; - - for_each_leaf_node_possible_cpu(rnp, cpu) - cpumask_set_cpu(cpu, affinity); - - kthread_affine_preferred(t, affinity); - - free_cpumask_var(affinity); -} - -/* - * Update kthreads affinity during CPU-hotplug changes. - * - * Set the per-rcu_node kthread's affinity to cover all CPUs that are - * served by the rcu_node in question. The CPU hotplug lock is still - * held, so the value of rnp->qsmaskinit will be stable. - * - * We don't include outgoingcpu in the affinity set, use -1 if there is - * no outgoing CPU. If there are no CPUs left in the affinity set, - * this function allows the kthread to execute on any CPU. - * - * Any future concurrent calls are serialized via ->kthread_mutex. - */ -static void rcutree_affinity_setting(unsigned int cpu, int outgoingcpu) -{ - cpumask_var_t cm; - unsigned long mask; - struct rcu_data *rdp; - struct rcu_node *rnp; - struct task_struct *task_exp; - - rdp =3D per_cpu_ptr(&rcu_data, cpu); - rnp =3D rdp->mynode; - - task_exp =3D rcu_exp_par_gp_task(rnp); - - /* - * If CPU is the boot one, this task is created later from early - * initcall since kthreadd must be created first. - */ - if (!task_exp) - return; - - if (!zalloc_cpumask_var(&cm, GFP_KERNEL)) - return; - - mutex_lock(&rnp->kthread_mutex); - mask =3D rcu_rnp_online_cpus(rnp); - for_each_leaf_node_possible_cpu(rnp, cpu) - if ((mask & leaf_node_cpu_bit(rnp, cpu)) && - cpu !=3D outgoingcpu) - cpumask_set_cpu(cpu, cm); - cpumask_and(cm, cm, housekeeping_cpumask(HK_TYPE_RCU)); - if (cpumask_empty(cm)) { - cpumask_copy(cm, housekeeping_cpumask(HK_TYPE_RCU)); - if (outgoingcpu >=3D 0) - cpumask_clear_cpu(outgoingcpu, cm); - } - - if (task_exp) - set_cpus_allowed_ptr(task_exp, cm); - - mutex_unlock(&rnp->kthread_mutex); - - free_cpumask_var(cm); -} - /* * Has the specified (known valid) CPU ever been fully online? */ @@ -4994,7 +4930,6 @@ int rcutree_online_cpu(unsigned int cpu) if (rcu_scheduler_active =3D=3D RCU_SCHEDULER_INACTIVE) return 0; /* Too early in boot for scheduler work. */ sync_sched_exp_online_cleanup(cpu); - rcutree_affinity_setting(cpu, -1); =20 // Stop-machine done, so allow nohz_full to disable tick. tick_dep_clear(TICK_DEP_BIT_RCU); @@ -5207,8 +5142,6 @@ int rcutree_offline_cpu(unsigned int cpu) rnp->ffmask &=3D ~rdp->grpmask; raw_spin_unlock_irqrestore_rcu_node(rnp, flags); =20 - rcutree_affinity_setting(cpu, cpu); - // nohz_full CPUs need the tick for stop-machine to work quickly tick_dep_set(TICK_DEP_BIT_RCU); return 0; --=20 2.45.2