From nobody Fri Sep 25 21:36:01 2026 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) client-ip=192.237.175.120; envelope-from=xen-devel-bounces@lists.xenproject.org; helo=lists.xenproject.org; Authentication-Results: mx.zohomail.com; dkim=pass; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=pass(p=none dis=none) header.from=xenproject.org ARC-Seal: i=1; a=rsa-sha256; t=1790353946; cv=none; d=zohomail.com; s=zohoarc; b=LqPg417BYxmvHAruhgAoHB16swMbH3OqPhSMvgrmVF0JK331ZGBaSHFb9HvpFgmsNTXWbJ6EhirLJ2GCbTFkL3OGOLqA6UhuuUAwvK+QKA77SzV8zr0RuGvd6DSVw0q0o6ncWmCn7bzrCFuJO3sUcSNZuvyTnOHCGNV5wAyI6LE= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1790353946; h=Content-Type:Content-Transfer-Encoding:Cc:Cc:Date:Date:From:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Help:List-Unsubscribe:MIME-Version:Message-ID:Sender:Subject:Subject:To:To:Message-Id:Reply-To; bh=3fSl0Zn3gA99zlkuSaBDz0i8XUehSOrrwfN84LQISAA=; b=UsqIZgBkDpp4gNfzBdU7gzkyAR/Xat+CyCP+HZnDkYAJ2v0lIrVuVM3zANqL3uCz86OEY/ZbH7lgoqdKsfXmXo+qcrvGIIcPOg6oDSZj0t2Jwykjow0iqIL7LJ1IhuFXtbibbgyRvPf4LqqzIgkFwSGun9fzVQ4mwE0syZUyBrs= ARC-Authentication-Results: i=1; mx.zohomail.com; dkim=pass; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=pass header.from= (p=none dis=none) Return-Path: Received: from lists.xenproject.org (lists.xenproject.org [192.237.175.120]) by mx.zohomail.com with SMTPS id 1790353946123146.06567101069118; Fri, 25 Sep 2026 09:32:26 -0700 (PDT) Received: from list by lists.xenproject.org with outflank-mailman.1434213.1654319 (Exim 4.92) (envelope-from ) id 1xA8qH-0001zY-Oz; Fri, 25 Sep 2026 16:32:09 +0000 Received: by outflank-mailman (output) from mailman id 1434213.1654319; Fri, 25 Sep 2026 16:32:09 +0000 Received: from localhost ([127.0.0.1] helo=lists.xenproject.org) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1xA8qH-0001zR-M0; Fri, 25 Sep 2026 16:32:09 +0000 Received: by outflank-mailman (input) for mailman id 1434213; Fri, 25 Sep 2026 16:32:09 +0000 Received: from mail.xenproject.org ([104.130.215.37]) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1xA8qH-0001zL-Ej for xen-devel@lists.xenproject.org; Fri, 25 Sep 2026 16:32:09 +0000 Received: from xenbits.xenproject.org ([104.239.192.120]) by mail.xenproject.org with esmtp (Exim 4.96) (envelope-from ) id 1xA8qG-00HMc5-0D; Fri, 25 Sep 2026 16:32:08 +0000 Received: from 224.pool85-54-217.dynamic.orange.es ([85.54.217.224] helo=localhost) by xenbits.xenproject.org with esmtpsa (TLS1.3) tls TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 (Exim 4.96) (envelope-from ) id 1xA8qG-00HO9j-1p; Fri, 25 Sep 2026 16:32:08 +0000 X-Outflank-Mailman: Message body and most headers restored to incoming version X-BeenThere: xen-devel@lists.xenproject.org List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Errors-To: xen-devel-bounces@lists.xenproject.org Precedence: list Sender: "Xen-devel" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=xenproject.org; s=20200302mail; h=Content-Transfer-Encoding:Content-Type: MIME-Version:References:In-Reply-To:Message-ID:Date:Subject:Cc:To:From; bh=3fSl0Zn3gA99zlkuSaBDz0i8XUehSOrrwfN84LQISAA=; b=jtDXBQN3krH/BsJZvQ8vpUjFR+ 8B8aN85NtNgIW8FXJAArDzunPN/D/x6ls8JgtBh9O5JjQckGprgQ4SPKYIEHLfRXlc0n68JUsuVAx OtiJfNwCH9hhRqU3eryj+jbkZZMzHPeoZX7CIs53PXSFHJe5Mak4gpXshwpcTlQFVu3M=; From: Roger Pau Monne To: xen-devel@lists.xenproject.org Cc: alejandro.garciavallejo@amd.com, Roger Pau Monne , Andrew Cooper , Anthony PERARD , Michal Orzel , Jan Beulich , Julien Grall , Stefano Stabellini Subject: [PATCH v2 1/3] xen/rcu: introduce the concept of RCU epoch Date: Fri, 25 Sep 2026 18:30:23 +0200 Message-ID: <20260925163025.65217-2-roger@xenproject.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260925163025.65217-1-roger@xenproject.org> References: <20260925163025.65217-1-roger@xenproject.org> MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable X-ZohoMail-DKIM: pass (identity @xenproject.org) X-ZM-MESSAGEID: 1790353948841158500 An RCU epoch signals the lifetime of RCU references. Each CPU records the epoch in force when it enters an RCU critical section. When a callback is queued the current epoch is stored in the callback and the global epoch is then bumped, as a way to know when all CPUs have moved past a specific epoch. At that point there can be no remaining references to objects fetched during the callback's epoch. The compiler barrier is switched to a full memory barrier, as future uses of rcu_lock_cnt must ensure the count is increased before taking a reference to any RCU protected object. Access rcu_lock_cnt, rcu_lock_epoch and rcu_epoch through ACCESS_ONCE() to stop the compiler shattering the loads and stores. The reordering prevention aspect of ACCESS_ONCE() is not what we rely on here (the memory barriers cover that); what matters is that these variables now have remote consumers, so each access must be a single, non-torn memory operation. Signed-off-by: Roger Pau Monn=C3=A9 --- Can possibly be folded into the next patch, as it's lacking context on its own to understand the need to introduce the logic. --- Changes since v1: - Do the setting of head->added in the interrupt disabled section. - Unconditionally do a memory barrier when enter/exit RCU critical regions. Attempting to do selectively is too complex. --- xen/common/rcupdate.c | 6 ++++++ xen/include/xen/rcupdate.h | 17 +++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/xen/common/rcupdate.c b/xen/common/rcupdate.c index c1b6b2ae768b..babaabbbeefe 100644 --- a/xen/common/rcupdate.c +++ b/xen/common/rcupdate.c @@ -49,6 +49,11 @@ #include =20 DEFINE_PER_CPU(unsigned int, rcu_lock_cnt); +/* Store epoch when CPU entered the RCU critical section. */ +DEFINE_PER_CPU(unsigned int, rcu_lock_epoch); + +/* Current RCU epoch, bumped every time a new callback is queued. */ +unsigned int rcu_epoch; =20 /* Global control variables for rcupdate callback mechanism. */ static struct rcu_ctrlblk { @@ -283,6 +288,7 @@ void call_rcu(struct rcu_head *head, head->func =3D func; head->next =3D NULL; local_irq_save(flags); + head->added =3D arch_fetch_and_add(&rcu_epoch, 1); rdp =3D &this_cpu(rcu_data); *rdp->nxttail =3D head; rdp->nxttail =3D &head->next; diff --git a/xen/include/xen/rcupdate.h b/xen/include/xen/rcupdate.h index c57f628107cf..5846e8c169d7 100644 --- a/xen/include/xen/rcupdate.h +++ b/xen/include/xen/rcupdate.h @@ -34,24 +34,32 @@ #include #include #include +#include #include #include =20 #define __rcu =20 DECLARE_PER_CPU(unsigned int, rcu_lock_cnt); +DECLARE_PER_CPU(unsigned int, rcu_lock_epoch); + +extern unsigned int rcu_epoch; =20 static inline void rcu_quiesce_disable(void) { + unsigned int cpu =3D smp_processor_id(); + preempt_disable(); - this_cpu(rcu_lock_cnt)++; - barrier(); + if ( !ACCESS_ONCE(per_cpu(rcu_lock_cnt, cpu))++ ) + ACCESS_ONCE(per_cpu(rcu_lock_epoch, cpu)) =3D ACCESS_ONCE(rcu_epoc= h); + smp_mb(); } =20 static inline void rcu_quiesce_enable(void) { - barrier(); - this_cpu(rcu_lock_cnt)--; + + smp_mb(); + ACCESS_ONCE(this_cpu(rcu_lock_cnt))--; preempt_enable(); } =20 @@ -68,6 +76,7 @@ static inline bool rcu_quiesce_allowed(void) struct rcu_head { struct rcu_head *next; void (*func)(struct rcu_head *head); + unsigned int added; }; =20 #define RCU_HEAD_INIT { .next =3D NULL, .func =3D NULL } --=20 2.55.0 From nobody Fri Sep 25 21:36:01 2026 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) client-ip=192.237.175.120; envelope-from=xen-devel-bounces@lists.xenproject.org; helo=lists.xenproject.org; Authentication-Results: mx.zohomail.com; dkim=pass; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=pass(p=none dis=none) header.from=xenproject.org ARC-Seal: i=1; a=rsa-sha256; t=1790353953; cv=none; d=zohomail.com; s=zohoarc; b=HY9+TDaAbz/uIOMj2CeSZDXdshYB75LSv8boYJvZHXI10r+HUzBGgcvqfrlhw0IU6WPpntdE3/toUrw7lQvWbM/MJoJ+k2NJQ3jDv6m0AJWVlFrKyWdz2NdbRjXrjMOzhS6jw2ZuYUCayQ0iwgNe+bnugSuv6cWyTzN/nv9RudM= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1790353953; h=Content-Type:Content-Transfer-Encoding:Cc:Cc:Date:Date:From:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Help:List-Unsubscribe:MIME-Version:Message-ID:Sender:Subject:Subject:To:To:Message-Id:Reply-To; bh=K95soxrXBGb9jEgINYkPiROFI00/lAb0CVue7bIFFjI=; b=Eu2MEljjt66R6S3g62lXG/1y2g2s/7V5nN91qYFMJtNAz2LCOLGv++y574YHGtlNYn72O014ZAD4EZHlTFRmYdHBO0f2LldhmdiLPkpRH+v3t8LXSKjj0hHtrF3vuVzVp8sM/+lNfh3EuCc6N1zQD9sItsbuMt5wSB9iU4M9fSw= ARC-Authentication-Results: i=1; mx.zohomail.com; dkim=pass; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=pass header.from= (p=none dis=none) Return-Path: Received: from lists.xenproject.org (lists.xenproject.org [192.237.175.120]) by mx.zohomail.com with SMTPS id 1790353953478499.43398301243315; Fri, 25 Sep 2026 09:32:33 -0700 (PDT) Received: from list by lists.xenproject.org with outflank-mailman.1434214.1654328 (Exim 4.92) (envelope-from ) id 1xA8qJ-0002Cl-WF; Fri, 25 Sep 2026 16:32:12 +0000 Received: by outflank-mailman (output) from mailman id 1434214.1654328; Fri, 25 Sep 2026 16:32:11 +0000 Received: from localhost ([127.0.0.1] helo=lists.xenproject.org) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1xA8qJ-0002Ce-SO; Fri, 25 Sep 2026 16:32:11 +0000 Received: by outflank-mailman (input) for mailman id 1434214; Fri, 25 Sep 2026 16:32:11 +0000 Received: from mail.xenproject.org ([104.130.215.37]) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1xA8qJ-0002CO-Dz for xen-devel@lists.xenproject.org; Fri, 25 Sep 2026 16:32:11 +0000 Received: from xenbits.xenproject.org ([104.239.192.120]) by mail.xenproject.org with esmtp (Exim 4.96) (envelope-from ) id 1xA8qI-00HMcH-08; Fri, 25 Sep 2026 16:32:10 +0000 Received: from 224.pool85-54-217.dynamic.orange.es ([85.54.217.224] helo=localhost) by xenbits.xenproject.org with esmtpsa (TLS1.3) tls TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 (Exim 4.96) (envelope-from ) id 1xA8qI-00HODg-1L; Fri, 25 Sep 2026 16:32:10 +0000 X-Outflank-Mailman: Message body and most headers restored to incoming version X-BeenThere: xen-devel@lists.xenproject.org List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Errors-To: xen-devel-bounces@lists.xenproject.org Precedence: list Sender: "Xen-devel" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=xenproject.org; s=20200302mail; h=Content-Transfer-Encoding:Content-Type: MIME-Version:References:In-Reply-To:Message-ID:Date:Subject:Cc:To:From; bh=K95soxrXBGb9jEgINYkPiROFI00/lAb0CVue7bIFFjI=; b=bXUvOSqwYSoLyVMOunlbJSzhjp jlETP4NUYbB7XmFCOBp2bZiT4CjCjEyBhts08jlNvSjwUB0H1Jq4RO8YnZz2yV5rJ22jXS9lzpxcn UisHHfajJmplYE0dUrhanmFJnG4g7qihpOpBTOxGfUPw6WbL2RA04N4DWVJZNErPNErQ=; From: Roger Pau Monne To: xen-devel@lists.xenproject.org Cc: alejandro.garciavallejo@amd.com, Roger Pau Monne , Andrew Cooper , Anthony PERARD , Michal Orzel , Jan Beulich , Julien Grall , Stefano Stabellini Subject: [PATCH v2 2/3] xen/rcu: simplify RCU implementation Date: Fri, 25 Sep 2026 18:30:24 +0200 Message-ID: <20260925163025.65217-3-roger@xenproject.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260925163025.65217-1-roger@xenproject.org> References: <20260925163025.65217-1-roger@xenproject.org> MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable X-ZohoMail-DKIM: pass (identity @xenproject.org) X-ZM-MESSAGEID: 1790353954959158500 The current implementation has two shortcomings for certain Xen usages: * When using the null scheduler it's possible for a CPU to never enter Xen context. A CPU not entering Xen context can block other CPUs from executing RCU callbacks, as there will be no quiescent state observed if the CPU doesn't enter Xen context. * If a certain amount of callbacks are pending, RCU will try to force a quiescent state, by sending an IPI to remote CPUs. This causes unwanted interference. Keep track of the RCU epoch when a callback was added, and only execute it once all CPUs are either outside of RCU critical regions, or any CPUs inside of RCU critical regions have entered such past the epoch when the callback was queued. Knowing whether a CPU is inside a RCU critical region is done based on the CPU rcu_lock_cnt value. This removes the grace/quiescent period state tracking entirely: the control block, the per-CPU quiescent bookkeeping, the batch lists and the forced-quiescent-state IPI all go away. As a result no RCU-driven IPIs are sent to remote CPUs, and callback progress no longer depends on every CPU passing through Xen context. Signed-off-by: Roger Pau Monn=C3=A9 --- The maximum batch of callbacks processed is limited to 10, this is bit arbitrary, but I think matches what the current logic attempts does. There's a bit more pruning to do regarding the usage of grace and quiesce in comments or functions names - I've done that as a followup change to reduce the size of this change. I think the manual raising of an RCU softirq is not really needed when the queue exceeds a certain length - the softirq loop will already attempt RCU callback processing if possible. --- Changes since v1: - Rework rcu_pending() so it reports whether there are callback ready to be pruned, instead of simply reporting whether there are any callbacks pending. This matches the previous implementation. - Force a barrier around entry/exit to each critical region, instead of doing so only in the first and last entry/exit respectively. This makes the logic more obviously correct. --- xen/common/rcupdate.c | 493 +++++++++++-------------------------- xen/include/xen/rcupdate.h | 14 +- 2 files changed, 146 insertions(+), 361 deletions(-) diff --git a/xen/common/rcupdate.c b/xen/common/rcupdate.c index babaabbbeefe..79d25c958f0a 100644 --- a/xen/common/rcupdate.c +++ b/xen/common/rcupdate.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -55,74 +56,31 @@ DEFINE_PER_CPU(unsigned int, rcu_lock_epoch); /* Current RCU epoch, bumped every time a new callback is queued. */ unsigned int rcu_epoch; =20 -/* Global control variables for rcupdate callback mechanism. */ -static struct rcu_ctrlblk { - long cur; /* Current batch number. */ - long completed; /* Number of the last completed batch */ - int next_pending; /* Is the next batch already waiting? */ - - spinlock_t lock __cacheline_aligned; - cpumask_t cpumask; /* CPUs that need to switch in order ... */ - cpumask_t idle_cpumask; /* ... unless they are already idle */ - /* for current batch to proceed. */ -} __cacheline_aligned rcu_ctrlblk =3D { - .cur =3D -300, - .completed =3D -300, - .lock =3D SPIN_LOCK_UNLOCKED, -}; - -/* - * Per-CPU data for Read-Copy Update. - * nxtlist - new callbacks are added here - * curlist - current batch for which quiescent cycle started if any - */ +/* Per-CPU data for Read-Copy Update. */ struct rcu_data { - /* 1) quiescent state handling : */ - long quiescbatch; /* Batch # for grace period */ - int qs_pending; /* core waits for quiesc state */ - - /* 2) batch handling */ - long batch; /* Batch # for current RCU batch */ - struct rcu_head *nxtlist; - struct rcu_head **nxttail; - long qlen; /* # of queued callbacks */ - struct rcu_head *curlist; - struct rcu_head **curtail; - struct rcu_head *donelist; - struct rcu_head **donetail; - long blimit; /* Upper limit on a processed batch = */ - int cpu; - long last_rs_qlen; /* qlen during the last resched */ - - /* 3) idle CPUs handling */ + /* + * List of pending callbacks, sorted by ascending epoch. Use a thresh= old + * value to raise an RCU softirq if the queue exceeds a given length. + */ + struct list_head pending; + unsigned int nr; +#define RCU_QUEUE_THRESHOLD 100 + + /* Keep track of CPUs blocking first RCU callback execution. */ + cpumask_t rcu_blocking; + + /* Idle CPU handling */ struct timer idle_timer; bool idle_timer_active; =20 - bool process_callbacks; + /* Barrier handling. */ bool barrier_active; }; =20 /* - * If a CPU with RCU callbacks queued goes idle, when the grace period is - * not finished yet, how can we make sure that the callbacks will eventual= ly - * be executed? In Linux (2.6.21, the first "tickless idle" Linux kernel), - * the periodic timer tick would not be stopped for such CPU. Here in Xen, - * we (may) don't even have a periodic timer tick, so we need to use a - * special purpose timer. - * - * Such timer: - * 1) is armed only when a CPU with an RCU callback(s) queued goes idle - * before the end of the current grace period (_not_ for any CPUs that - * go idle!); - * 2) when it fires, it is only re-armed if the grace period is still - * running; - * 3) it is stopped immediately, if the CPU wakes up from idle and - * resumes 'normal' execution. - * - * About how far in the future the timer should be programmed each time, - * it's hard to tell (guess!!). Since this mimics Linux's periodic timer - * tick, take values used there as an indication. In Linux 2.6.21, tick - * period can be 10ms, 4ms, 3.33ms or 1ms. + * If a CPU with RCU callbacks queued goes idle before the callbacks can be + * drained use a timer to ensure the CPU is woken up to process the remain= ing + * callback queue. * * By default, we use 10ms, to enable at least some power saving on the * CPU that is going idle. The user can change this, via a boot time @@ -137,21 +95,18 @@ static s_time_t __read_mostly idle_timer_period; /* * Increment and decrement values for the idle timer handler. The algorithm * works as follows: - * - if the timer actually fires, and it finds out that the grace period i= sn't - * over yet, we add IDLE_TIMER_PERIOD_INCR to the timer's period; - * - if the timer actually fires and it finds the grace period over, we - * subtract IDLE_TIMER_PERIOD_DECR from the timer's period. + * - If the timer actually fires, and it finds out there are CPUs still in= RCU + * critical regions, we add IDLE_TIMER_PERIOD_INCR to the timer's period. + * Note this is not very accurate, as the CPUs in those RCU critical reg= ions + * might not be holding back the execution of the local callbacks. + * - If the timer actually fires and it finds no CPUs in critical RCU regi= ons, + * we subtract IDLE_TIMER_PERIOD_DECR from the timer's period. */ #define IDLE_TIMER_PERIOD_INCR MILLISECS(10) #define IDLE_TIMER_PERIOD_DECR MICROSECS(100) =20 static DEFINE_PER_CPU(struct rcu_data, rcu_data); =20 -static int blimit =3D 10; -static int qhimark =3D 10000; -static int qlowmark =3D 100; -static int rsinterval =3D 1000; - /* * rcu_barrier() handling: * Two counters are used to synchronize rcu_barrier() work: @@ -246,35 +201,12 @@ void rcu_barrier(void) put_cpu_maps(); } =20 -/* Is batch a before batch b ? */ -static inline int rcu_batch_before(long a, long b) -{ - return (a - b) < 0; -} - -static void force_quiescent_state(struct rcu_data *rdp, - struct rcu_ctrlblk *rcp) -{ - cpumask_t cpumask; - raise_softirq(RCU_SOFTIRQ); - if (unlikely(rdp->qlen - rdp->last_rs_qlen > rsinterval)) { - rdp->last_rs_qlen =3D rdp->qlen; - /* - * Don't send IPI to itself. With irqs disabled, - * rdp->cpu is the current cpu. - */ - cpumask_andnot(&cpumask, &rcp->cpumask, cpumask_of(rdp->cpu)); - cpumask_raise_softirq(&cpumask, RCU_SOFTIRQ); - } -} - /** * call_rcu - Queue an RCU callback for invocation after a grace period. * @head: structure to be used for queueing the RCU updates. * @func: actual update function to be invoked after the grace period * - * The update function will be invoked some time after a full grace - * period elapses, in other words after all currently executing RCU + * The update function will be invoked after all currently executing RCU * read-side critical sections have completed. RCU read-side critical * sections are delimited by rcu_read_lock() and rcu_read_unlock(), * and may be nested. @@ -283,205 +215,88 @@ void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) { unsigned long flags; - struct rcu_data *rdp; + struct rcu_data *rdp =3D &this_cpu(rcu_data); =20 head->func =3D func; - head->next =3D NULL; local_irq_save(flags); head->added =3D arch_fetch_and_add(&rcu_epoch, 1); - rdp =3D &this_cpu(rcu_data); - *rdp->nxttail =3D head; - rdp->nxttail =3D &head->next; - if (unlikely(++rdp->qlen > qhimark)) { - rdp->blimit =3D INT_MAX; - force_quiescent_state(rdp, &rcu_ctrlblk); - } - local_irq_restore(flags); -} - -/* - * Invoke the completed RCU callbacks. They are expected to be in - * a per-cpu list. - */ -static void rcu_do_batch(struct rcu_data *rdp) -{ - struct rcu_head *next, *list; - int count =3D 0; - - list =3D rdp->donelist; - while (list) { - next =3D rdp->donelist =3D list->next; - list->func(list); - list =3D next; - rdp->qlen--; - if (++count >=3D rdp->blimit) - break; - } - if (rdp->blimit =3D=3D INT_MAX && rdp->qlen <=3D qlowmark) - rdp->blimit =3D blimit; - if (!rdp->donelist) - rdp->donetail =3D &rdp->donelist; - else - { - rdp->process_callbacks =3D true; - raise_softirq(RCU_SOFTIRQ); - } -} - -/* - * Grace period handling: - * The grace period handling consists out of two steps: - * - A new grace period is started. - * This is done by rcu_start_batch. The start is not broadcasted to - * all cpus, they must pick this up by comparing rcp->cur with - * rdp->quiescbatch. All cpus are recorded in the - * rcu_ctrlblk.cpumask bitmap. - * - All cpus must go through a quiescent state. - * Since the start of the grace period is not broadcasted, at least two - * calls to rcu_check_quiescent_state are required: - * The first call just notices that a new grace period is running. The - * following calls check if there was a quiescent state since the beginn= ing - * of the grace period. If so, it updates rcu_ctrlblk.cpumask. If - * the bitmap is empty, then the grace period is completed. - * rcu_check_quiescent_state calls rcu_start_batch(0) to start the next = grace - * period (if necessary). - */ -/* - * Register a new batch of callbacks, and start it up if there is currentl= y no - * active batch and the batch to be registered has not already occurred. - * Caller must hold rcu_ctrlblk.lock. - */ -static void rcu_start_batch(struct rcu_ctrlblk *rcp) -{ - if (rcp->next_pending && - rcp->completed =3D=3D rcp->cur) { - rcp->next_pending =3D 0; + list_add_tail(&head->list, &rdp->pending); + if ( ++rdp->nr > RCU_QUEUE_THRESHOLD ) /* - * next_pending =3D=3D 0 must be visible in - * __rcu_process_callbacks() before it can see new value of cur. + * Raise a softirq to attempt to force draining the queue, albeit + * there's no guarantee. */ - smp_wmb(); - rcp->cur++; - - /* - * Make sure the increment of rcp->cur is visible so, even if a - * CPU that is about to go idle, is captured inside rcp->cpumask, - * rcu_pending() will return false, which then means cpu_quiet() - * will be invoked, before the CPU would actually enter idle. - * - * This barrier is paired with the one in rcu_idle_enter(). - */ - smp_mb(); - cpumask_andnot(&rcp->cpumask, &cpu_online_map, &rcp->idle_cpumask); - } -} - -/* - * cpu went through a quiescent state since the beginning of the grace per= iod. - * Clear it from the cpu mask and complete the grace period if it was the = last - * cpu. Start another grace period if someone has further entries pending - */ -static void cpu_quiet(int cpu, struct rcu_ctrlblk *rcp) -{ - cpumask_clear_cpu(cpu, &rcp->cpumask); - if (cpumask_empty(&rcp->cpumask)) { - /* batch completed ! */ - rcp->completed =3D rcp->cur; - rcu_start_batch(rcp); - } + raise_softirq(RCU_SOFTIRQ); + local_irq_restore(flags); } =20 -/* - * Check if the cpu has gone through a quiescent state (say context - * switch). If so and if it already hasn't done so in this RCU - * quiescent cycle, then indicate that it has done so. - */ -static void rcu_check_quiescent_state(struct rcu_ctrlblk *rcp, - struct rcu_data *rdp) +#define RCU_MAX_BATCH 10 +static void cf_check rcu_process_callbacks(void) { - if (rdp->quiescbatch !=3D rcp->cur) { - /* start new grace period: */ - rdp->qs_pending =3D 1; - rdp->quiescbatch =3D rcp->cur; - return; - } - - /* Grace period already completed for this cpu? - * qs_pending is checked instead of the actual bitmap to avoid - * cacheline trashing. - */ - if (!rdp->qs_pending) - return; + static DEFINE_PER_CPU(cpumask_t, rcu_scratch); + cpumask_t *in_rcu =3D &this_cpu(rcu_scratch); + struct rcu_data *rdp =3D &this_cpu(rcu_data); + unsigned long flags; + unsigned int queued =3D 0, cpu; + LIST_HEAD(expired); + struct rcu_head *rcu; =20 - rdp->qs_pending =3D 0; + /* Clear any existing blocking mask - we will redo it if needed. */ + cpumask_clear(&rdp->rcu_blocking); =20 - spin_lock(&rcp->lock); /* - * rdp->quiescbatch/rcp->cur and the cpu bitmap can come out of sync - * during cpu startup. Ignore the quiescent state. + * Populate a cpumask with any CPUs inside RCU critical regions. Note= that + * CPUs entering past this point are of no interest, they will certain= ly + * use an epoch past any queued callbacks here. */ - if (likely(rdp->quiescbatch =3D=3D rcp->cur)) - cpu_quiet(rdp->cpu, rcp); - - spin_unlock(&rcp->lock); -} - - -/* - * This does the RCU processing work from softirq context.=20 - */ -static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp, - struct rcu_data *rdp) -{ - if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch)) { - *rdp->donetail =3D rdp->curlist; - rdp->donetail =3D rdp->curtail; - rdp->curlist =3D NULL; - rdp->curtail =3D &rdp->curlist; - } - - local_irq_disable(); - if (rdp->nxtlist && !rdp->curlist) { - rdp->curlist =3D rdp->nxtlist; - rdp->curtail =3D rdp->nxttail; - rdp->nxtlist =3D NULL; - rdp->nxttail =3D &rdp->nxtlist; - local_irq_enable(); + cpumask_clear(in_rcu); + for_each_online_cpu ( cpu ) + if ( ACCESS_ONCE(per_cpu(rcu_lock_cnt, cpu)) ) + __cpumask_set_cpu(cpu, in_rcu); =20 + local_irq_save(flags); + while ( queued < RCU_MAX_BATCH && + (rcu =3D list_first_entry_or_null(&rdp->pending, struct rcu_he= ad, + list)) ) + { /* - * start the next batch of callbacks + * Fetching rcu_lock_epoch out of order is not a concern here: in = the + * worst case it's going to result in an older more restrictive ep= och + * being checked against. Note the adding of a callback issues a + * arch_fetch_and_add() which is a barrier on itself, and guarante= es + * remote changes to the CPU mask to be visible here. */ - - /* determine batch number */ - rdp->batch =3D rcp->cur + 1; - /* see the comment and corresponding wmb() in - * the rcu_start_batch() - */ - smp_rmb(); - - if (!rcp->next_pending) { - /* and start it/schedule start if it's a new batch */ - spin_lock(&rcp->lock); - rcp->next_pending =3D 1; - rcu_start_batch(rcp); - spin_unlock(&rcp->lock); - } - } else { - local_irq_enable(); + for_each_cpu ( cpu, in_rcu ) + if ( (int)(ACCESS_ONCE(per_cpu(rcu_lock_epoch, cpu)) - + rcu->added) <=3D 0 ) + /* Record CPUs blocking the execution of the callback. */ + __cpumask_set_cpu(cpu, &rdp->rcu_blocking); + + if ( !cpumask_empty(&rdp->rcu_blocking) ) + /* + * Callbacks are sorted, exit loop as soon as we find one that + * can't be processed yet. + */ + goto process; + + list_del(&rcu->list); + list_add_tail(&rcu->list, &expired); + ASSERT(rdp->nr); + rdp->nr--; + queued++; } - rcu_check_quiescent_state(rcp, rdp); - if (rdp->donelist) - rcu_do_batch(rdp); -} =20 -static void cf_check rcu_process_callbacks(void) -{ - struct rcu_data *rdp =3D &this_cpu(rcu_data); + if ( queued =3D=3D RCU_MAX_BATCH && rdp->nr ) + /* There's more work to do, yield and raise a softirq to come back= . */ + raise_softirq(RCU_SOFTIRQ); =20 - if ( rdp->process_callbacks ) + process: + local_irq_restore(flags); + + while ( (rcu =3D list_first_entry_or_null(&expired, struct rcu_head, l= ist)) ) { - rdp->process_callbacks =3D false; - __rcu_process_callbacks(&rcu_ctrlblk, rdp); + list_del(&rcu->list); + rcu->func(rcu); } =20 if ( atomic_read(&cpu_count) && !rdp->barrier_active ) @@ -492,33 +307,26 @@ static void cf_check rcu_process_callbacks(void) } } =20 -static int __rcu_pending(struct rcu_ctrlblk *rcp, struct rcu_data *rdp) +bool rcu_pending(unsigned int cpu) { - /* This cpu has pending rcu entries and the grace period - * for them has completed. - */ - if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch)) - return 1; + struct rcu_data *rdp =3D &per_cpu(rcu_data, cpu); + const struct rcu_head *rcu =3D + list_first_entry_or_null(&rdp->pending, struct rcu_head, list); + unsigned int rcpu; =20 - /* This cpu has no pending entries, but there are new entries */ - if (!rdp->curlist && rdp->nxtlist) - return 1; + /* Modifying rcu_blocking per-CPU mask is only safe if done locally. */ + ASSERT(smp_processor_id() =3D=3D cpu); =20 - /* This cpu has finished callbacks to invoke */ - if (rdp->donelist) - return 1; + if ( !rcu ) + return false; =20 - /* The rcu core waits for a quiescent state from the cpu */ - if (rdp->quiescbatch !=3D rcp->cur || rdp->qs_pending) - return 1; + for_each_cpu ( rcpu, &rdp->rcu_blocking ) + if ( (int)(ACCESS_ONCE(per_cpu(rcu_lock_epoch, rcpu)) - + rcu->added) > 0 ) + /* Record CPUs blocking the execution of the callback. */ + __cpumask_clear_cpu(rcpu, &rdp->rcu_blocking); =20 - /* nothing to do */ - return 0; -} - -bool rcu_pending(unsigned int cpu) -{ - return !!__rcu_pending(&rcu_ctrlblk, &per_cpu(rcu_data, cpu)); + return cpumask_empty(&rdp->rcu_blocking); } =20 /* @@ -529,15 +337,13 @@ bool rcu_pending(unsigned int cpu) */ bool rcu_needs_cpu(unsigned int cpu) { - struct rcu_data *rdp =3D &per_cpu(rcu_data, cpu); - - return (rdp->curlist && !rdp->idle_timer_active) || rcu_pending(cpu); + return !!per_cpu(rcu_data, cpu).nr; } =20 /* * Timer for making sure the CPU where a callback is queued does * periodically poke rcu_pending(), so that it will invoke the callback - * not too late after the end of the grace period. + * not too late. */ static void rcu_idle_timer_start(void) { @@ -545,10 +351,9 @@ static void rcu_idle_timer_start(void) =20 /* * Note that we don't check rcu_pending() here. In fact, we don't want - * the timer armed on CPUs that are in the process of quiescing while - * going idle, unless they really are the ones with a queued callback. + * the timer armed on CPUs that don't have pending callbacks. */ - if (likely(!rdp->curlist)) + if (likely(!rdp->nr)) return; =20 set_timer(&rdp->idle_timer, NOW() + idle_timer_period); @@ -587,7 +392,7 @@ static void cf_check rcu_idle_timer_handler(void* data) { perfc_incr(rcu_idle_timer); =20 - if ( !cpumask_empty(&rcu_ctrlblk.cpumask) ) + if ( this_cpu(rcu_data).nr ) idle_timer_period =3D min(idle_timer_period + IDLE_TIMER_PERIOD_IN= CR, IDLE_TIMER_PERIOD_MAX); else @@ -597,55 +402,51 @@ static void cf_check rcu_idle_timer_handler(void* dat= a) =20 void rcu_check_callbacks(unsigned int cpu) { - struct rcu_data *rdp =3D &this_cpu(rcu_data); - - rdp->process_callbacks =3D true; raise_softirq(RCU_SOFTIRQ); } =20 -static void rcu_move_batch(struct rcu_data *this_rdp, struct rcu_head *lis= t, - struct rcu_head **tail) +/* Sorting functions for RCU list concatenation when a CPU goes offline. */ +static int cf_check cmp_rcu(void *priv, struct list_head *a, + struct list_head *b) { - local_irq_disable(); - *this_rdp->nxttail =3D list; - if (list) - this_rdp->nxttail =3D tail; - local_irq_enable(); + const struct rcu_head *l =3D container_of(a, struct rcu_head, list), + *r =3D container_of(b, struct rcu_head, list); + + return (int)(l->added - r->added); } =20 static void rcu_offline_cpu(struct rcu_data *this_rdp, - struct rcu_ctrlblk *rcp, struct rcu_data *rdp) + struct rcu_data *rdp) { kill_timer(&rdp->idle_timer); =20 - /* If the cpu going offline owns the grace period we can block - * indefinitely waiting for it, so flush it here. - */ - spin_lock(&rcp->lock); - if (rcp->cur !=3D rcp->completed) - cpu_quiet(rdp->cpu, rcp); - spin_unlock(&rcp->lock); - - rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail); - rcu_move_batch(this_rdp, rdp->curlist, rdp->curtail); - rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail); + if ( !rdp->nr ) + return; =20 + /* + * Append pending callbacks to the current CPU. By the time this is + * executed the CPU going offline cannot be in any RCU critical sectio= n or + * queue any more RCU work. + */ local_irq_disable(); - this_rdp->qlen +=3D rdp->qlen; + list_splice(&rdp->pending, &this_rdp->pending); + this_rdp->nr +=3D rdp->nr; + list_sort(NULL, &this_rdp->pending, cmp_rcu); + /* Clear blocking CPU mask: the top callback might have changed. */ + cpumask_clear(&this_rdp->rcu_blocking); local_irq_enable(); + + if ( this_rdp->nr > RCU_QUEUE_THRESHOLD ) + raise_softirq(RCU_SOFTIRQ); + + INIT_LIST_HEAD(&rdp->pending); + rdp->nr =3D 0; } =20 -static void rcu_init_percpu_data(int cpu, struct rcu_ctrlblk *rcp, - struct rcu_data *rdp) +static void rcu_init_percpu_data(int cpu, struct rcu_data *rdp) { memset(rdp, 0, sizeof(*rdp)); - rdp->curtail =3D &rdp->curlist; - rdp->nxttail =3D &rdp->nxtlist; - rdp->donetail =3D &rdp->donelist; - rdp->quiescbatch =3D rcp->completed; - rdp->qs_pending =3D 0; - rdp->cpu =3D cpu; - rdp->blimit =3D blimit; + INIT_LIST_HEAD(&rdp->pending); init_timer(&rdp->idle_timer, rcu_idle_timer_handler, rdp, cpu); } =20 @@ -658,11 +459,11 @@ static int cf_check cpu_callback( switch ( action ) { case CPU_UP_PREPARE: - rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp); + rcu_init_percpu_data(cpu, rdp); break; case CPU_UP_CANCELED: case CPU_DEAD: - rcu_offline_cpu(&this_cpu(rcu_data), &rcu_ctrlblk, rdp); + rcu_offline_cpu(&this_cpu(rcu_data), rdp); break; default: break; @@ -693,36 +494,18 @@ void __init rcu_init(void) } idle_timer_period =3D MILLISECS(idle_timer_period_ms); =20 - cpumask_clear(&rcu_ctrlblk.idle_cpumask); cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu); register_cpu_notifier(&cpu_nfb); open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); } =20 -/* - * The CPU is becoming idle, so no more read side critical - * sections, and one more step toward grace period. - */ +/* The CPU is becoming idle, ensure pending RCU work will get processed. */ void rcu_idle_enter(unsigned int cpu) { - ASSERT(!cpumask_test_cpu(cpu, &rcu_ctrlblk.idle_cpumask)); - cpumask_set_cpu(cpu, &rcu_ctrlblk.idle_cpumask); - /* - * If some other CPU is starting a new grace period, we'll notice that - * by seeing a new value in rcp->cur (different than our quiescbatch). - * That will force us all the way until cpu_quiet(), clearing our bit - * in rcp->cpumask, even in case we managed to get in there. - * - * Se the comment before cpumask_andnot() in rcu_start_batch(). - */ - smp_mb(); - rcu_idle_timer_start(); } =20 void rcu_idle_exit(unsigned int cpu) { rcu_idle_timer_stop(); - ASSERT(cpumask_test_cpu(cpu, &rcu_ctrlblk.idle_cpumask)); - cpumask_clear_cpu(cpu, &rcu_ctrlblk.idle_cpumask); } diff --git a/xen/include/xen/rcupdate.h b/xen/include/xen/rcupdate.h index 5846e8c169d7..312551d5b089 100644 --- a/xen/include/xen/rcupdate.h +++ b/xen/include/xen/rcupdate.h @@ -35,6 +35,7 @@ #include #include #include +#include #include #include =20 @@ -57,7 +58,6 @@ static inline void rcu_quiesce_disable(void) =20 static inline void rcu_quiesce_enable(void) { - smp_mb(); ACCESS_ONCE(this_cpu(rcu_lock_cnt))--; preempt_enable(); @@ -70,19 +70,21 @@ static inline bool rcu_quiesce_allowed(void) =20 /** * struct rcu_head - callback structure for use with RCU - * @next: next update requests in a list + * @list: list anchor. * @func: actual update function to call after the grace period. + * @added: epoch when the callback was added. */ struct rcu_head { - struct rcu_head *next; + struct list_head list; void (*func)(struct rcu_head *head); unsigned int added; }; =20 -#define RCU_HEAD_INIT { .next =3D NULL, .func =3D NULL } -#define RCU_HEAD(head) struct rcu_head head =3D RCU_HEAD_INIT +#define RCU_HEAD_INIT(head) { .list =3D LIST_HEAD_INIT((head).list), \ + .func =3D NULL } +#define RCU_HEAD(head) struct rcu_head head =3D RCU_HEAD_INIT(head) #define INIT_RCU_HEAD(ptr) do { \ - (ptr)->next =3D NULL; (ptr)->func =3D NULL; \ + INIT_LIST_HEAD(&(ptr)->list); (ptr)->func =3D NULL; \ } while (0) =20 =20 --=20 2.55.0 From nobody Fri Sep 25 21:36:01 2026 Delivered-To: importer@patchew.org Received-SPF: pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) client-ip=192.237.175.120; envelope-from=xen-devel-bounces@lists.xenproject.org; helo=lists.xenproject.org; Authentication-Results: mx.zohomail.com; dkim=pass; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=pass(p=none dis=none) header.from=xenproject.org ARC-Seal: i=1; a=rsa-sha256; t=1790353954; cv=none; d=zohomail.com; s=zohoarc; b=SOPuviy2DDF4X1UIjSHKAByvgObSW77L6imhHvgQlAUzzYL8AExq7yeKG2I3ow7YYtR0vMbEoY9uzPuOwQOogghdR95+0fMp6Dd/tj3mQaf6kOaV+5V5dYYF9slR9gW0p6mI2+zwaLFUdFAxloqAwfJWzZcoqOI2qFU6zoxdpC4= ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=zohomail.com; s=zohoarc; t=1790353954; h=Content-Type:Content-Transfer-Encoding:Cc:Cc:Date:Date:From:From:In-Reply-To:List-Subscribe:List-Post:List-Id:List-Help:List-Unsubscribe:MIME-Version:Message-ID:Sender:Subject:Subject:To:To:Message-Id:Reply-To; bh=VkJ8mIf2NudvkCJyhJ6QlIMNIJWqXQLcqbsgpgSyxgI=; b=Gdfe/RHFpCF8bP/BLsUnQxC50LQdYG3hwFShnTDmYz7xMZvAv/0IH7aapTDeD8xeNYMtIeViijnbnTCnMxqrqtYCZJjQ9Xu/PcA7TORarTjQSXhzilrjvWFcP+NJ+jNIGUwo7w/mEg7LCg+611djQaXQfWk77yVDzVLrPZJNOFE= ARC-Authentication-Results: i=1; mx.zohomail.com; dkim=pass; spf=pass (zohomail.com: domain of lists.xenproject.org designates 192.237.175.120 as permitted sender) smtp.mailfrom=xen-devel-bounces@lists.xenproject.org; dmarc=pass header.from= (p=none dis=none) Return-Path: Received: from lists.xenproject.org (lists.xenproject.org [192.237.175.120]) by mx.zohomail.com with SMTPS id 1790353954234896.8819883012999; Fri, 25 Sep 2026 09:32:34 -0700 (PDT) Received: from list by lists.xenproject.org with outflank-mailman.1434215.1654337 (Exim 4.92) (envelope-from ) id 1xA8qM-0002Ra-9J; Fri, 25 Sep 2026 16:32:14 +0000 Received: by outflank-mailman (output) from mailman id 1434215.1654337; Fri, 25 Sep 2026 16:32:14 +0000 Received: from localhost ([127.0.0.1] helo=lists.xenproject.org) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1xA8qM-0002RP-5l; Fri, 25 Sep 2026 16:32:14 +0000 Received: by outflank-mailman (input) for mailman id 1434215; Fri, 25 Sep 2026 16:32:13 +0000 Received: from mail.xenproject.org ([104.130.215.37]) by lists.xenproject.org with esmtp (Exim 4.92) (envelope-from ) id 1xA8qL-0002Q6-Cl for xen-devel@lists.xenproject.org; Fri, 25 Sep 2026 16:32:13 +0000 Received: from xenbits.xenproject.org ([104.239.192.120]) by mail.xenproject.org with esmtp (Exim 4.96) (envelope-from ) id 1xA8qK-00HMcV-0T; Fri, 25 Sep 2026 16:32:12 +0000 Received: from 224.pool85-54-217.dynamic.orange.es ([85.54.217.224] helo=localhost) by xenbits.xenproject.org with esmtpsa (TLS1.3) tls TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 (Exim 4.96) (envelope-from ) id 1xA8qK-00HOHQ-23; Fri, 25 Sep 2026 16:32:12 +0000 X-Outflank-Mailman: Message body and most headers restored to incoming version X-BeenThere: xen-devel@lists.xenproject.org List-Id: Xen developer discussion List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Errors-To: xen-devel-bounces@lists.xenproject.org Precedence: list Sender: "Xen-devel" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=xenproject.org; s=20200302mail; h=Content-Transfer-Encoding:Content-Type: MIME-Version:References:In-Reply-To:Message-ID:Date:Subject:Cc:To:From; bh=VkJ8mIf2NudvkCJyhJ6QlIMNIJWqXQLcqbsgpgSyxgI=; b=mRiBu/5p/+YeDVJvcC1E7V/tzN AGaTgP7akk0RuU7XfvGsKqim0GLjtICUX+mhXTbnEiQuK0a9yhl6W4Uafm2rLXW5slmQLzWxMLY5e Fe2zjNmR1NivN73K54o+BLN+ZXC0u39Rv+QnqoDKJb+xPQil686KJGd/khBWvgcW4PHk=; From: Roger Pau Monne To: xen-devel@lists.xenproject.org Cc: alejandro.garciavallejo@amd.com, Roger Pau Monne , Andrew Cooper , Anthony PERARD , Michal Orzel , Jan Beulich , Julien Grall , Stefano Stabellini Subject: [PATCH v2 3/3] xen/rcu: remove rcu_quiesce_{disable,enable}() Date: Fri, 25 Sep 2026 18:30:25 +0200 Message-ID: <20260925163025.65217-4-roger@xenproject.org> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20260925163025.65217-1-roger@xenproject.org> References: <20260925163025.65217-1-roger@xenproject.org> MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: quoted-printable X-ZohoMail-DKIM: pass (identity @xenproject.org) X-ZM-MESSAGEID: 1790353954879158500 Fold the code into its callers, having them as separate function just makes the code harder to read. Rename rcu_quiesce_allowed() as the RCU implementation no longer has the concept of quiesce periods. No functional change intended. Signed-off-by: Roger Pau Monn=C3=A9 --- Changes since v2: - New in this version. --- xen/common/softirq.c | 4 ++-- xen/include/xen/rcupdate.h | 32 +++++++++++--------------------- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/xen/common/softirq.c b/xen/common/softirq.c index dc3aabce3330..39e1a9844afa 100644 --- a/xen/common/softirq.c +++ b/xen/common/softirq.c @@ -31,7 +31,7 @@ static void __do_softirq(unsigned long ignore_mask) unsigned long pending; bool rcu_allowed =3D !(ignore_mask & (1UL << RCU_SOFTIRQ)); =20 - ASSERT(!rcu_allowed || rcu_quiesce_allowed()); + ASSERT(!rcu_allowed || rcu_processing_allowed()); =20 for ( ; ; ) { @@ -61,7 +61,7 @@ void process_pending_softirqs(void) (1UL << SCHED_SLAVE_SOFTIRQ); =20 /* Block RCU processing in case of rcu_read_lock() held. */ - if ( !rcu_quiesce_allowed() ) + if ( !rcu_processing_allowed() ) ignore_mask |=3D 1UL << RCU_SOFTIRQ; =20 ASSERT(!in_irq() && local_irq_is_enabled()); diff --git a/xen/include/xen/rcupdate.h b/xen/include/xen/rcupdate.h index 312551d5b089..cb16d7f79ae8 100644 --- a/xen/include/xen/rcupdate.h +++ b/xen/include/xen/rcupdate.h @@ -46,24 +46,7 @@ DECLARE_PER_CPU(unsigned int, rcu_lock_epoch); =20 extern unsigned int rcu_epoch; =20 -static inline void rcu_quiesce_disable(void) -{ - unsigned int cpu =3D smp_processor_id(); - - preempt_disable(); - if ( !ACCESS_ONCE(per_cpu(rcu_lock_cnt, cpu))++ ) - ACCESS_ONCE(per_cpu(rcu_lock_epoch, cpu)) =3D ACCESS_ONCE(rcu_epoc= h); - smp_mb(); -} - -static inline void rcu_quiesce_enable(void) -{ - smp_mb(); - ACCESS_ONCE(this_cpu(rcu_lock_cnt))--; - preempt_enable(); -} - -static inline bool rcu_quiesce_allowed(void) +static inline bool rcu_processing_allowed(void) { return !this_cpu(rcu_lock_cnt); } @@ -129,7 +112,12 @@ typedef struct _rcu_read_lock rcu_read_lock_t; */ static inline void rcu_read_lock(rcu_read_lock_t *lock) { - rcu_quiesce_disable(); + unsigned int cpu =3D smp_processor_id(); + + preempt_disable(); + if ( !ACCESS_ONCE(per_cpu(rcu_lock_cnt, cpu))++ ) + ACCESS_ONCE(per_cpu(rcu_lock_epoch, cpu)) =3D ACCESS_ONCE(rcu_epoc= h); + smp_mb(); } =20 /** @@ -139,8 +127,10 @@ static inline void rcu_read_lock(rcu_read_lock_t *lock) */ static inline void rcu_read_unlock(rcu_read_lock_t *lock) { - ASSERT(!rcu_quiesce_allowed()); - rcu_quiesce_enable(); + ASSERT(!rcu_processing_allowed()); + smp_mb(); + ACCESS_ONCE(this_cpu(rcu_lock_cnt))--; + preempt_enable(); } =20 /* --=20 2.55.0