[v1] rcu: Fix expedited GP deadlock (and cleanup some nocb stuff)

[PATCH 4/8] rcu/exp: Handle RCU expedited grace period kworker allocation failure

Posted by Frederic Weisbecker 2 years ago

Just like is done for the kworker performing nodes initialization,
gracefully handle the possible allocation failure of the RCU expedited
grace period main kworker.

While at it perform a rename of the related checking functions to better
reflect the expedited specifics.

Fixes: 9621fbee44df ("rcu: Move expedited grace period (GP) work to RT kthread_worker")
Cc: Kalesh Singh <kaleshsingh@google.com>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
---
 kernel/rcu/tree.c     |  2 ++
 kernel/rcu/tree_exp.h | 25 +++++++++++++++++++------
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 055f4817bc70..39679cf66c3a 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -4717,6 +4717,7 @@ static void __init rcu_start_exp_gp_kworkers(void)
 	rcu_exp_gp_kworker = kthread_create_worker(0, gp_kworker_name);
 	if (IS_ERR_OR_NULL(rcu_exp_gp_kworker)) {
 		pr_err("Failed to create %s!\n", gp_kworker_name);
+		rcu_exp_gp_kworker = NULL;
 		return;
 	}
 
@@ -4725,6 +4726,7 @@ static void __init rcu_start_exp_gp_kworkers(void)
 		pr_err("Failed to create %s!\n", par_gp_kworker_name);
 		rcu_exp_par_gp_kworker = NULL;
 		kthread_destroy_worker(rcu_exp_gp_kworker);
+		rcu_exp_gp_kworker = NULL;
 		return;
 	}
 
diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
index 6d7cea5d591f..cb31f4fb4b36 100644
--- a/kernel/rcu/tree_exp.h
+++ b/kernel/rcu/tree_exp.h
@@ -429,7 +429,12 @@ static void sync_rcu_exp_select_node_cpus(struct kthread_work *wp)
 	__sync_rcu_exp_select_node_cpus(rewp);
 }
 
-static inline bool rcu_gp_par_worker_started(void)
+static inline bool rcu_exp_worker_started(void)
+{
+	return !!READ_ONCE(rcu_exp_gp_kworker);
+}
+
+static inline bool rcu_exp_par_worker_started(void)
 {
 	return !!READ_ONCE(rcu_exp_par_gp_kworker);
 }
@@ -479,7 +484,12 @@ static void sync_rcu_exp_select_node_cpus(struct work_struct *wp)
 	__sync_rcu_exp_select_node_cpus(rewp);
 }
 
-static inline bool rcu_gp_par_worker_started(void)
+static inline bool rcu_exp_worker_started(void)
+{
+	return !!READ_ONCE(rcu_gp_wq);
+}
+
+static inline bool rcu_exp_par_worker_started(void)
 {
 	return !!READ_ONCE(rcu_par_gp_wq);
 }
@@ -542,7 +552,7 @@ static void sync_rcu_exp_select_cpus(void)
 		rnp->exp_need_flush = false;
 		if (!READ_ONCE(rnp->expmask))
 			continue; /* Avoid early boot non-existent wq. */
-		if (!rcu_gp_par_worker_started() ||
+		if (!rcu_exp_par_worker_started() ||
 		    rcu_scheduler_active != RCU_SCHEDULER_RUNNING ||
 		    rcu_is_last_leaf_node(rnp)) {
 			/* No worker started yet or last leaf, do direct call. */
@@ -957,7 +967,7 @@ static void rcu_exp_print_detail_task_stall_rnp(struct rcu_node *rnp)
  */
 void synchronize_rcu_expedited(void)
 {
-	bool boottime = (rcu_scheduler_active == RCU_SCHEDULER_INIT);
+	bool can_queue;
 	unsigned long flags;
 	struct rcu_exp_work rew;
 	struct rcu_node *rnp;
@@ -968,6 +978,9 @@ void synchronize_rcu_expedited(void)
 			 lock_is_held(&rcu_sched_lock_map),
 			 "Illegal synchronize_rcu_expedited() in RCU read-side critical section");
 
+	can_queue = (rcu_scheduler_active != RCU_SCHEDULER_INIT) &&
+		    rcu_exp_worker_started();
+
 	/* Is the state is such that the call is a grace period? */
 	if (rcu_blocking_is_gp()) {
 		// Note well that this code runs with !PREEMPT && !SMP.
@@ -997,7 +1010,7 @@ void synchronize_rcu_expedited(void)
 		return;  /* Someone else did our work for us. */
 
 	/* Ensure that load happens before action based on it. */
-	if (unlikely(boottime)) {
+	if (unlikely(!can_queue)) {
 		/* Direct call during scheduler init and early_initcalls(). */
 		rcu_exp_sel_wait_wake(s);
 	} else {
@@ -1015,7 +1028,7 @@ void synchronize_rcu_expedited(void)
 	/* Let the next expedited grace period start. */
 	mutex_unlock(&rcu_state.exp_mutex);
 
-	if (likely(!boottime))
+	if (likely(can_queue))
 		synchronize_rcu_expedited_destroy_work(&rew);
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
-- 
2.42.1

Re: [PATCH 4/8] rcu/exp: Handle RCU expedited grace period kworker allocation failure

Posted by Kalesh Singh 2 years ago

On Fri, Dec 8, 2023 at 5:06 PM Frederic Weisbecker <frederic@kernel.org> wrote:
>
> Just like is done for the kworker performing nodes initialization,
> gracefully handle the possible allocation failure of the RCU expedited
> grace period main kworker.
>
> While at it perform a rename of the related checking functions to better
> reflect the expedited specifics.
>
> Fixes: 9621fbee44df ("rcu: Move expedited grace period (GP) work to RT kthread_worker")
> Cc: Kalesh Singh <kaleshsingh@google.com>
> Signed-off-by: Frederic Weisbecker <frederic@kernel.org>

Reviewed-by: Kalesh Singh <kaleshsingh@google.com>

Thanks,
Kalesh

> ---
>  kernel/rcu/tree.c     |  2 ++
>  kernel/rcu/tree_exp.h | 25 +++++++++++++++++++------
>  2 files changed, 21 insertions(+), 6 deletions(-)
>
> diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> index 055f4817bc70..39679cf66c3a 100644
> --- a/kernel/rcu/tree.c
> +++ b/kernel/rcu/tree.c
> @@ -4717,6 +4717,7 @@ static void __init rcu_start_exp_gp_kworkers(void)
>         rcu_exp_gp_kworker = kthread_create_worker(0, gp_kworker_name);
>         if (IS_ERR_OR_NULL(rcu_exp_gp_kworker)) {
>                 pr_err("Failed to create %s!\n", gp_kworker_name);
> +               rcu_exp_gp_kworker = NULL;
>                 return;
>         }
>
> @@ -4725,6 +4726,7 @@ static void __init rcu_start_exp_gp_kworkers(void)
>                 pr_err("Failed to create %s!\n", par_gp_kworker_name);
>                 rcu_exp_par_gp_kworker = NULL;
>                 kthread_destroy_worker(rcu_exp_gp_kworker);
> +               rcu_exp_gp_kworker = NULL;
>                 return;
>         }
>
> diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
> index 6d7cea5d591f..cb31f4fb4b36 100644
> --- a/kernel/rcu/tree_exp.h
> +++ b/kernel/rcu/tree_exp.h
> @@ -429,7 +429,12 @@ static void sync_rcu_exp_select_node_cpus(struct kthread_work *wp)
>         __sync_rcu_exp_select_node_cpus(rewp);
>  }
>
> -static inline bool rcu_gp_par_worker_started(void)
> +static inline bool rcu_exp_worker_started(void)
> +{
> +       return !!READ_ONCE(rcu_exp_gp_kworker);
> +}
> +
> +static inline bool rcu_exp_par_worker_started(void)
>  {
>         return !!READ_ONCE(rcu_exp_par_gp_kworker);
>  }
> @@ -479,7 +484,12 @@ static void sync_rcu_exp_select_node_cpus(struct work_struct *wp)
>         __sync_rcu_exp_select_node_cpus(rewp);
>  }
>
> -static inline bool rcu_gp_par_worker_started(void)
> +static inline bool rcu_exp_worker_started(void)
> +{
> +       return !!READ_ONCE(rcu_gp_wq);
> +}
> +
> +static inline bool rcu_exp_par_worker_started(void)
>  {
>         return !!READ_ONCE(rcu_par_gp_wq);
>  }
> @@ -542,7 +552,7 @@ static void sync_rcu_exp_select_cpus(void)
>                 rnp->exp_need_flush = false;
>                 if (!READ_ONCE(rnp->expmask))
>                         continue; /* Avoid early boot non-existent wq. */
> -               if (!rcu_gp_par_worker_started() ||
> +               if (!rcu_exp_par_worker_started() ||
>                     rcu_scheduler_active != RCU_SCHEDULER_RUNNING ||
>                     rcu_is_last_leaf_node(rnp)) {
>                         /* No worker started yet or last leaf, do direct call. */
> @@ -957,7 +967,7 @@ static void rcu_exp_print_detail_task_stall_rnp(struct rcu_node *rnp)
>   */
>  void synchronize_rcu_expedited(void)
>  {
> -       bool boottime = (rcu_scheduler_active == RCU_SCHEDULER_INIT);
> +       bool can_queue;
>         unsigned long flags;
>         struct rcu_exp_work rew;
>         struct rcu_node *rnp;
> @@ -968,6 +978,9 @@ void synchronize_rcu_expedited(void)
>                          lock_is_held(&rcu_sched_lock_map),
>                          "Illegal synchronize_rcu_expedited() in RCU read-side critical section");
>
> +       can_queue = (rcu_scheduler_active != RCU_SCHEDULER_INIT) &&
> +                   rcu_exp_worker_started();
> +
>         /* Is the state is such that the call is a grace period? */
>         if (rcu_blocking_is_gp()) {
>                 // Note well that this code runs with !PREEMPT && !SMP.
> @@ -997,7 +1010,7 @@ void synchronize_rcu_expedited(void)
>                 return;  /* Someone else did our work for us. */
>
>         /* Ensure that load happens before action based on it. */
> -       if (unlikely(boottime)) {
> +       if (unlikely(!can_queue)) {
>                 /* Direct call during scheduler init and early_initcalls(). */
>                 rcu_exp_sel_wait_wake(s);
>         } else {
> @@ -1015,7 +1028,7 @@ void synchronize_rcu_expedited(void)
>         /* Let the next expedited grace period start. */
>         mutex_unlock(&rcu_state.exp_mutex);
>
> -       if (likely(!boottime))
> +       if (likely(can_queue))
>                 synchronize_rcu_expedited_destroy_work(&rew);
>  }
>  EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
> --
> 2.42.1
>

[PATCH 1/8] rcu/nocb: Make IRQs disablement symetric
[PATCH 2/8] rcu/nocb: Re-arrange call_rcu() NOCB specific code
[PATCH 3/8] rcu/exp: Fix RCU expedited parallel grace period kworker allocation failure recovery
[PATCH 4/8] rcu/exp: Handle RCU expedited grace period kworker allocation failure
[PATCH 5/8] rcu: s/boost_kthread_mutex/kthread_mutex
[PATCH 6/8] rcu/exp: Make parallel exp gp kworker per rcu node
[PATCH 7/8] rcu/exp: Handle parallel exp gp kworkers affinity
[PATCH 8/8] rcu/exp: Remove rcu_par_gp_wq