From nobody Sat Feb 7 21:14:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 90E523EFD3D; Fri, 6 Feb 2026 14:23:05 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387785; cv=none; b=Kt74/rDbvI73T3ZZCAc4IDjFZ6IMWALwR5CnYLi3OeTRAIR95WGTkGFE7GNxtS1xd88NdEbHDpnu9qBY6H528tU8Y5bpCqsiY/Y6Jo9debIUG6jbSW2mAwkHTLag1eVdt0g9rxM4yKL61/BBOSdlwpiRfZtIEqLpBOHgsq/px6o= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387785; c=relaxed/simple; bh=wLAGVyuXhhrWKSeELGYkoOFpqB5ysD0I9rHCulvW2nY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=uX5dxLDGmJSb6b2eVaX6OclAqqZVkXVt5YnFQHWpfn1DIOGxj3Ou8HZKUmXkFdZM5B362052ONrgd7OwX9hx8EPDTyW6Q7IRFrqxKrLRwZaOhPiVfalPO96fIQe+seqZVsASa9jrKM6JXALPcyG46dLYWw1fpnQu33VxOVHISSM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=JpZ8uh3d; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="JpZ8uh3d" Received: by smtp.kernel.org (Postfix) with ESMTPSA id B52E9C19422; Fri, 6 Feb 2026 14:22:57 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1770387785; bh=wLAGVyuXhhrWKSeELGYkoOFpqB5ysD0I9rHCulvW2nY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=JpZ8uh3dVO70F+77got86Wunt3Up/fK/qFnnYDGHTtdQmqrjOvVd3KVzTRYvBu0y+ hdptMlW1CyrwZUjMxaknor8oyK3IRqdqjaqtM22j0CKyPl24pF+YtrXA7/ywoUsqVc N9FX/G/KlvtZP8UxRFrnyUBU06Yv5WyoSVVEeBxk/62132uvMF9T6F+EG0VdunYtzv rnhxjkXAi7wdshbMhVW+Nut7nB0ETScRmNlE8rJMSW5BA8jxbGhTLWuoA+ffTzdJxY eKxCztC7aCrqJmnr9bEPApU27MVgTDBeostanKewHhWrIQeSoqcWzG7/MaOwP3EPwu wQo4EUCnj7jcg== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Christophe Leroy (CS GROUP)" , "Rafael J. Wysocki" , Alexander Gordeev , Anna-Maria Behnsen , Ben Segall , Boqun Feng , Christian Borntraeger , Dietmar Eggemann , Heiko Carstens , Ingo Molnar , Jan Kiszka , Joel Fernandes , Juri Lelli , Kieran Bingham , Madhavan Srinivasan , Mel Gorman , Michael Ellerman , Neeraj Upadhyay , Nicholas Piggin , "Paul E . McKenney" , Peter Zijlstra , Steven Rostedt , Sven Schnelle , Thomas Gleixner , Uladzislau Rezki , Valentin Schneider , Vasily Gorbik , Vincent Guittot , Viresh Kumar , Xin Zhao , linux-pm@vger.kernel.org, linux-s390@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, Shrikanth Hegde Subject: [PATCH 01/15] sched/idle: Handle offlining first in idle loop Date: Fri, 6 Feb 2026 15:22:31 +0100 Message-ID: <20260206142245.58987-2-frederic@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20260206142245.58987-1-frederic@kernel.org> References: <20260206142245.58987-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Offline handling happens from within the inner idle loop, after the beginning of dyntick cputime accounting, nohz idle load balancing and TIF_NEED_RESCHED polling. This is not necessary and even buggy because: * There is no dyntick handling to do. And calling tick_nohz_idle_enter() messes up with the struct tick_sched reset that was performed on tick_sched_timer_dying(). * There is no nohz idle balancing to do. * Polling on TIF_RESCHED is irrelevant at this stage, there are no more tasks allowed to run. * No need to check if need_resched() before offline handling since stop_machine is done and all per-cpu kthread should be done with their job. Therefore move the offline handling at the beginning of the idle loop. This will also ease the idle cputime unification later by not elapsing idle time while offline through the call to: tick_nohz_idle_enter() -> tick_nohz_start_idle() Signed-off-by: Frederic Weisbecker --- kernel/sched/idle.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index c174afe1dd17..51764cbec6f3 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -260,6 +260,14 @@ static void do_idle(void) { int cpu =3D smp_processor_id(); =20 + if (cpu_is_offline(cpu)) { + local_irq_disable(); + /* All per-CPU kernel threads should be done by now. */ + WARN_ON_ONCE(need_resched()); + cpuhp_report_idle_dead(); + arch_cpu_idle_dead(); + } + /* * Check if we need to update blocked load */ @@ -311,11 +319,6 @@ static void do_idle(void) */ local_irq_disable(); =20 - if (cpu_is_offline(cpu)) { - cpuhp_report_idle_dead(); - arch_cpu_idle_dead(); - } - arch_cpu_idle_enter(); rcu_nocb_flush_deferred_wakeup(); =20 --=20 2.51.1 From nobody Sat Feb 7 21:14:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4F83B212542; Fri, 6 Feb 2026 14:23:13 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387793; cv=none; b=ciEm0OhyKZi62Z4wZa+dH0NWbMyb1+ZCqkufu5cOC7mBYXw1J7XCtgo0XDTrxEpT2rq2srR8wh6Bq9dtc0E5948dmIjKTuqHu4H6LYd8Fo8Jm7SD4Dhohjy+bvbIFWEOBjaAtQEQRv4lKYXgycLM8vwQ/3tQDKC8AXTh6amHWcQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387793; c=relaxed/simple; bh=LzZmy6jqMt6NobQKkguWtqCpma0/ISXupISgS1lOgXU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=Q6TW6lsZb3lYuja59uOj3QuqAwOCzHb1HR4c+it/+tEPcfcwovkBKoJMHpjwT978rv9NgvZp1Voic6G+hanB3XZkN/EIOZMLN0Z0siYJHTUZ3ScZV3zaoEkZkZxyhkAsxPQ0Wkjo8KpxNSJSj75sN90eog40nG0UkuDJDCsn8yE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=U/R+hWp2; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="U/R+hWp2" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 9F6FDC116C6; Fri, 6 Feb 2026 14:23:05 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1770387792; bh=LzZmy6jqMt6NobQKkguWtqCpma0/ISXupISgS1lOgXU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=U/R+hWp20w6WR9wbISxrYpIFFmsig+ITL1xt4J7YCi9YrVgSWZVjaE/RifELVM5NW w6p6Q+TwOhS+atviKJwHZcnXE5cweK8V8QlaQ+a0Hv0OJ7MC9iRcs9Lih+6MAqwMUc s9ok9Hr2/T7R6lOhqa4tuoX5hwm42XPcb5mB+wm6VRVep3WawJI5hMBwWZEExEGe5X V4cS4jtoRLfQIaW7ivr+gMO7a1qdbqWAxL1gw4/2Lw5h8A0+NffM9QNY2bJHk+Gfi5 /0dXEgD1/rFpNVbOPPFhtUPFxTe8o652LxIUa+gyc8Sl+9hBv1NWzdn9h1GlOrjOew 89qITlYW13c2g== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Christophe Leroy (CS GROUP)" , "Rafael J. Wysocki" , Alexander Gordeev , Anna-Maria Behnsen , Ben Segall , Boqun Feng , Christian Borntraeger , Dietmar Eggemann , Heiko Carstens , Ingo Molnar , Jan Kiszka , Joel Fernandes , Juri Lelli , Kieran Bingham , Madhavan Srinivasan , Mel Gorman , Michael Ellerman , Neeraj Upadhyay , Nicholas Piggin , "Paul E . McKenney" , Peter Zijlstra , Steven Rostedt , Sven Schnelle , Thomas Gleixner , Uladzislau Rezki , Valentin Schneider , Vasily Gorbik , Vincent Guittot , Viresh Kumar , Xin Zhao , linux-pm@vger.kernel.org, linux-s390@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, Shrikanth Hegde Subject: [PATCH 02/15] sched/cputime: Remove superfluous and error prone kcpustat_field() parameter Date: Fri, 6 Feb 2026 15:22:32 +0100 Message-ID: <20260206142245.58987-3-frederic@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20260206142245.58987-1-frederic@kernel.org> References: <20260206142245.58987-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The first parameter to kcpustat_field() is a pointer to the cpu kcpustat to be fetched from. This parameter is error prone because a copy to a kcpustat could be passed by accident instead of the original one. Also the kcpustat structure can already be retrieved with the help of the mandatory CPU argument. Remove the needless paramater. Signed-off-by: Frederic Weisbecker --- drivers/cpufreq/cpufreq_governor.c | 6 +++--- drivers/macintosh/rack-meter.c | 2 +- include/linux/kernel_stat.h | 8 +++----- kernel/rcu/tree.c | 9 +++------ kernel/rcu/tree_stall.h | 7 +++---- kernel/sched/cputime.c | 5 ++--- 6 files changed, 15 insertions(+), 22 deletions(-) diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_g= overnor.c index 1a7fcaf39cc9..b6683628091d 100644 --- a/drivers/cpufreq/cpufreq_governor.c +++ b/drivers/cpufreq/cpufreq_governor.c @@ -105,7 +105,7 @@ void gov_update_cpu_data(struct dbs_data *dbs_data) j_cdbs->prev_cpu_idle =3D get_cpu_idle_time(j, &j_cdbs->prev_update_tim= e, dbs_data->io_is_busy); if (dbs_data->ignore_nice_load) - j_cdbs->prev_cpu_nice =3D kcpustat_field(&kcpustat_cpu(j), CPUTIME_NIC= E, j); + j_cdbs->prev_cpu_nice =3D kcpustat_field(CPUTIME_NICE, j); } } } @@ -165,7 +165,7 @@ unsigned int dbs_update(struct cpufreq_policy *policy) j_cdbs->prev_cpu_idle =3D cur_idle_time; =20 if (ignore_nice) { - u64 cur_nice =3D kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j); + u64 cur_nice =3D kcpustat_field(CPUTIME_NICE, j); =20 idle_time +=3D div_u64(cur_nice - j_cdbs->prev_cpu_nice, NSEC_PER_USEC); j_cdbs->prev_cpu_nice =3D cur_nice; @@ -539,7 +539,7 @@ int cpufreq_dbs_governor_start(struct cpufreq_policy *p= olicy) j_cdbs->prev_load =3D 0; =20 if (ignore_nice) - j_cdbs->prev_cpu_nice =3D kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE= , j); + j_cdbs->prev_cpu_nice =3D kcpustat_field(CPUTIME_NICE, j); } =20 gov->start(policy); diff --git a/drivers/macintosh/rack-meter.c b/drivers/macintosh/rack-meter.c index 896a43bd819f..20b2ecd32340 100644 --- a/drivers/macintosh/rack-meter.c +++ b/drivers/macintosh/rack-meter.c @@ -87,7 +87,7 @@ static inline u64 get_cpu_idle_time(unsigned int cpu) kcpustat->cpustat[CPUTIME_IOWAIT]; =20 if (rackmeter_ignore_nice) - retval +=3D kcpustat_field(kcpustat, CPUTIME_NICE, cpu); + retval +=3D kcpustat_field(CPUTIME_NICE, cpu); =20 return retval; } diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index b97ce2df376f..dd020ecaf67b 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -100,14 +100,12 @@ static inline unsigned long kstat_cpu_irqs_sum(unsign= ed int cpu) } =20 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN -extern u64 kcpustat_field(struct kernel_cpustat *kcpustat, - enum cpu_usage_stat usage, int cpu); +extern u64 kcpustat_field(enum cpu_usage_stat usage, int cpu); extern void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu); #else -static inline u64 kcpustat_field(struct kernel_cpustat *kcpustat, - enum cpu_usage_stat usage, int cpu) +static inline u64 kcpustat_field(enum cpu_usage_stat usage, int cpu) { - return kcpustat->cpustat[usage]; + return kcpustat_cpu(cpu).cpustat[usage]; } =20 static inline void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 293bbd9ac3f4..ceea4b2f755b 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -968,14 +968,11 @@ static int rcu_watching_snap_recheck(struct rcu_data = *rdp) if (rcu_cpu_stall_cputime && rdp->snap_record.gp_seq !=3D rdp->gp_seq) { int cpu =3D rdp->cpu; struct rcu_snap_record *rsrp; - struct kernel_cpustat *kcsp; - - kcsp =3D &kcpustat_cpu(cpu); =20 rsrp =3D &rdp->snap_record; - rsrp->cputime_irq =3D kcpustat_field(kcsp, CPUTIME_IRQ, cpu); - rsrp->cputime_softirq =3D kcpustat_field(kcsp, CPUTIME_SOFTIRQ, cpu); - rsrp->cputime_system =3D kcpustat_field(kcsp, CPUTIME_SYSTEM, cpu); + rsrp->cputime_irq =3D kcpustat_field(CPUTIME_IRQ, cpu); + rsrp->cputime_softirq =3D kcpustat_field(CPUTIME_SOFTIRQ, cpu); + rsrp->cputime_system =3D kcpustat_field(CPUTIME_SYSTEM, cpu); rsrp->nr_hardirqs =3D kstat_cpu_irqs_sum(cpu) + arch_irq_stat_cpu(cpu); rsrp->nr_softirqs =3D kstat_cpu_softirqs_sum(cpu); rsrp->nr_csw =3D nr_context_switches_cpu(cpu); diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index b67532cb8770..cf7ae51cba40 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -479,7 +479,6 @@ static void print_cpu_stat_info(int cpu) { struct rcu_snap_record rsr, *rsrp; struct rcu_data *rdp =3D per_cpu_ptr(&rcu_data, cpu); - struct kernel_cpustat *kcsp =3D &kcpustat_cpu(cpu); =20 if (!rcu_cpu_stall_cputime) return; @@ -488,9 +487,9 @@ static void print_cpu_stat_info(int cpu) if (rsrp->gp_seq !=3D rdp->gp_seq) return; =20 - rsr.cputime_irq =3D kcpustat_field(kcsp, CPUTIME_IRQ, cpu); - rsr.cputime_softirq =3D kcpustat_field(kcsp, CPUTIME_SOFTIRQ, cpu); - rsr.cputime_system =3D kcpustat_field(kcsp, CPUTIME_SYSTEM, cpu); + rsr.cputime_irq =3D kcpustat_field(CPUTIME_IRQ, cpu); + rsr.cputime_softirq =3D kcpustat_field(CPUTIME_SOFTIRQ, cpu); + rsr.cputime_system =3D kcpustat_field(CPUTIME_SYSTEM, cpu); =20 pr_err("\t hardirqs softirqs csw/system\n"); pr_err("\t number: %8lld %10d %12lld\n", diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 4f97896887ec..5dcb0f2e01bc 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -961,10 +961,9 @@ static int kcpustat_field_vtime(u64 *cpustat, return 0; } =20 -u64 kcpustat_field(struct kernel_cpustat *kcpustat, - enum cpu_usage_stat usage, int cpu) +u64 kcpustat_field(enum cpu_usage_stat usage, int cpu) { - u64 *cpustat =3D kcpustat->cpustat; + u64 *cpustat =3D kcpustat_cpu(cpu).cpustat; u64 val =3D cpustat[usage]; struct rq *rq; int err; --=20 2.51.1 From nobody Sat Feb 7 21:14:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4E412212542; Fri, 6 Feb 2026 14:23:21 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387801; cv=none; b=nouReTpBo2z779ZBZomihMEC/mqNHWWAx95wFmuW9P8XIpflJ8DKTp3eCUyx0a7zKSMriACkRrON/Jpnupr/ez0Ux/NqdG25YPoW5xoSPVu/kGoXdl4H9MnQ0u0PyQnOEo9uzBq0xJBKfJF1FYMkwJ3ai8JDF3zyjbXIYoad2pU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387801; c=relaxed/simple; bh=qfdvJvQBGQZlUpvIquFdfzN94e1PNCF4663MJivSVPs=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=eo39hRiRHTrmjlmvFSzVmh92dEnA9xLLBtim8C8QSpK8lELlyHsrY1dmO/f0TbGxOWfd894ptzF1lJo6UCeELuWRKKdsg+zIxyfLq/SxmKiZAKgCDgevGiLkPj8jK9Roh4bx2x2SVfEncwENNR4CtaMuG5Ot+CJQsfDTbn/841w= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=OqKfkvWI; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="OqKfkvWI" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 591C7C19421; Fri, 6 Feb 2026 14:23:13 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1770387800; bh=qfdvJvQBGQZlUpvIquFdfzN94e1PNCF4663MJivSVPs=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=OqKfkvWIMKXMf0cmdOvt2Y10zm2nPU6tJSh5T6lbhjlSIW+OOuxr7rUhCcemxU0X8 V6fhXfiKi4KkXNj6wWqqOttvsoIh3wfATrAYXH2IJmXf8fuNQYHqUgGDrUGzx3/83Z 3Z48d8HPnEykVqhoj8FWVRuKEMqRnEIqzJ7bQQqzfd2yVDKWGUV2utENJq1VKVphxm pNfksmCGAEhEBqZPnlOhgu66iHn+zFTuP1wVNuCdb0X15WfEO30qGMC9VmiMxs3Kly wjtB/M13j+cX7HCX5MWqiT9ynyCP7VU+0MKdGBpSu65QwaPFBV/gs7hkFpMa6dehle 7idBsyhkmkOJg== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Christophe Leroy (CS GROUP)" , "Rafael J. Wysocki" , Alexander Gordeev , Anna-Maria Behnsen , Ben Segall , Boqun Feng , Christian Borntraeger , Dietmar Eggemann , Heiko Carstens , Ingo Molnar , Jan Kiszka , Joel Fernandes , Juri Lelli , Kieran Bingham , Madhavan Srinivasan , Mel Gorman , Michael Ellerman , Neeraj Upadhyay , Nicholas Piggin , "Paul E . McKenney" , Peter Zijlstra , Steven Rostedt , Sven Schnelle , Thomas Gleixner , Uladzislau Rezki , Valentin Schneider , Vasily Gorbik , Vincent Guittot , Viresh Kumar , Xin Zhao , linux-pm@vger.kernel.org, linux-s390@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, Shrikanth Hegde Subject: [PATCH 03/15] sched/cputime: Correctly support generic vtime idle time Date: Fri, 6 Feb 2026 15:22:33 +0100 Message-ID: <20260206142245.58987-4-frederic@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20260206142245.58987-1-frederic@kernel.org> References: <20260206142245.58987-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Currently whether generic vtime is running or not, the idle cputime is fetched from the nohz accounting. However generic vtime already does its own idle cputime accounting. Only the kernel stat accessors are not plugged to support it. Read the idle generic vtime cputime when it's running, this will allow to later more clearly split nohz and vtime cputime accounting. Signed-off-by: Frederic Weisbecker --- include/linux/vtime.h | 9 +++++++-- kernel/sched/cputime.c | 38 +++++++++++++++++++++++++++++--------- kernel/time/tick-sched.c | 12 +++++++++--- 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/include/linux/vtime.h b/include/linux/vtime.h index 29dd5b91dd7d..336875bea767 100644 --- a/include/linux/vtime.h +++ b/include/linux/vtime.h @@ -10,7 +10,6 @@ */ #ifdef CONFIG_VIRT_CPU_ACCOUNTING extern void vtime_account_kernel(struct task_struct *tsk); -extern void vtime_account_idle(struct task_struct *tsk); #endif /* !CONFIG_VIRT_CPU_ACCOUNTING */ =20 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN @@ -27,7 +26,13 @@ static inline void vtime_guest_exit(struct task_struct *= tsk) { } static inline void vtime_init_idle(struct task_struct *tsk, int cpu) { } #endif =20 +static inline bool vtime_generic_enabled_cpu(int cpu) +{ + return context_tracking_enabled_cpu(cpu); +} + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE +extern void vtime_account_idle(struct task_struct *tsk); extern void vtime_account_irq(struct task_struct *tsk, unsigned int offset= ); extern void vtime_account_softirq(struct task_struct *tsk); extern void vtime_account_hardirq(struct task_struct *tsk); @@ -74,7 +79,7 @@ static inline bool vtime_accounting_enabled(void) =20 static inline bool vtime_accounting_enabled_cpu(int cpu) { - return context_tracking_enabled_cpu(cpu); + return vtime_generic_enabled_cpu(cpu); } =20 static inline bool vtime_accounting_enabled_this_cpu(void) diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 5dcb0f2e01bc..5613838d0307 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -759,9 +759,9 @@ void vtime_guest_exit(struct task_struct *tsk) } EXPORT_SYMBOL_GPL(vtime_guest_exit); =20 -void vtime_account_idle(struct task_struct *tsk) +static void __vtime_account_idle(struct vtime *vtime) { - account_idle_time(get_vtime_delta(&tsk->vtime)); + account_idle_time(get_vtime_delta(vtime)); } =20 void vtime_task_switch_generic(struct task_struct *prev) @@ -770,7 +770,7 @@ void vtime_task_switch_generic(struct task_struct *prev) =20 write_seqcount_begin(&vtime->seqcount); if (vtime->state =3D=3D VTIME_IDLE) - vtime_account_idle(prev); + __vtime_account_idle(vtime); else __vtime_account_kernel(prev, vtime); vtime->state =3D VTIME_INACTIVE; @@ -912,6 +912,7 @@ static int kcpustat_field_vtime(u64 *cpustat, int cpu, u64 *val) { struct vtime *vtime =3D &tsk->vtime; + struct rq *rq =3D cpu_rq(cpu); unsigned int seq; =20 do { @@ -953,6 +954,14 @@ static int kcpustat_field_vtime(u64 *cpustat, if (state =3D=3D VTIME_GUEST && task_nice(tsk) > 0) *val +=3D vtime->gtime + vtime_delta(vtime); break; + case CPUTIME_IDLE: + if (state =3D=3D VTIME_IDLE && !atomic_read(&rq->nr_iowait)) + *val +=3D vtime_delta(vtime); + break; + case CPUTIME_IOWAIT: + if (state =3D=3D VTIME_IDLE && atomic_read(&rq->nr_iowait) > 0) + *val +=3D vtime_delta(vtime); + break; default: break; } @@ -1015,8 +1024,8 @@ static int kcpustat_cpu_fetch_vtime(struct kernel_cpu= stat *dst, *dst =3D *src; cpustat =3D dst->cpustat; =20 - /* Task is sleeping, dead or idle, nothing to add */ - if (state < VTIME_SYS) + /* Task is sleeping or dead, nothing to add */ + if (state < VTIME_IDLE) continue; =20 delta =3D vtime_delta(vtime); @@ -1025,15 +1034,17 @@ static int kcpustat_cpu_fetch_vtime(struct kernel_c= pustat *dst, * Task runs either in user (including guest) or kernel space, * add pending nohz time to the right place. */ - if (state =3D=3D VTIME_SYS) { + switch (vtime->state) { + case VTIME_SYS: cpustat[CPUTIME_SYSTEM] +=3D vtime->stime + delta; - } else if (state =3D=3D VTIME_USER) { + break; + case VTIME_USER: if (task_nice(tsk) > 0) cpustat[CPUTIME_NICE] +=3D vtime->utime + delta; else cpustat[CPUTIME_USER] +=3D vtime->utime + delta; - } else { - WARN_ON_ONCE(state !=3D VTIME_GUEST); + break; + case VTIME_GUEST: if (task_nice(tsk) > 0) { cpustat[CPUTIME_GUEST_NICE] +=3D vtime->gtime + delta; cpustat[CPUTIME_NICE] +=3D vtime->gtime + delta; @@ -1041,6 +1052,15 @@ static int kcpustat_cpu_fetch_vtime(struct kernel_cp= ustat *dst, cpustat[CPUTIME_GUEST] +=3D vtime->gtime + delta; cpustat[CPUTIME_USER] +=3D vtime->gtime + delta; } + break; + case VTIME_IDLE: + if (atomic_read(&cpu_rq(cpu)->nr_iowait) > 0) + cpustat[CPUTIME_IOWAIT] +=3D delta; + else + cpustat[CPUTIME_IDLE] +=3D delta; + break; + default: + WARN_ON_ONCE(1); } } while (read_seqcount_retry(&vtime->seqcount, seq)); =20 diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 8ddf74e705d3..9632066aea4d 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -774,9 +774,10 @@ static void tick_nohz_start_idle(struct tick_sched *ts) sched_clock_idle_sleep_event(); } =20 -static u64 get_cpu_sleep_time_us(struct tick_sched *ts, ktime_t *sleeptime, +static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, ktime_t= *sleeptime, bool compute_delta, u64 *last_update_time) { + struct tick_sched *ts =3D &per_cpu(tick_cpu_sched, cpu); ktime_t now, idle; unsigned int seq; =20 @@ -787,6 +788,11 @@ static u64 get_cpu_sleep_time_us(struct tick_sched *ts= , ktime_t *sleeptime, if (last_update_time) *last_update_time =3D ktime_to_us(now); =20 + if (vtime_generic_enabled_cpu(cpu)) { + idle =3D kcpustat_field(idx, cpu); + return ktime_to_us(idle); + } + do { seq =3D read_seqcount_begin(&ts->idle_sleeptime_seq); =20 @@ -824,7 +830,7 @@ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) { struct tick_sched *ts =3D &per_cpu(tick_cpu_sched, cpu); =20 - return get_cpu_sleep_time_us(ts, &ts->idle_sleeptime, + return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE, &ts->idle_sleeptime, !nr_iowait_cpu(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); @@ -850,7 +856,7 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_ti= me) { struct tick_sched *ts =3D &per_cpu(tick_cpu_sched, cpu); =20 - return get_cpu_sleep_time_us(ts, &ts->iowait_sleeptime, + return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT, &ts->iowait_sleeptime, nr_iowait_cpu(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); --=20 2.51.1 From nobody Sat Feb 7 21:14:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1E155212542; Fri, 6 Feb 2026 14:23:28 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387809; cv=none; b=sgpHrnoM9wkq3DZTVqhcxPRRGRPBAOLusjkhXkI7RiLZDyYBsMtUWSNHBQctYDK0NokbmccoxgPF5AlgyaJ6vOS5tVvVZciRNslHGcPcATS2vubh7SuzgLNmIRTMVJ62KuTVfbGf5+Skf4UsI4ves7UaWwcjqsxS5FAc6gQipkM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387809; c=relaxed/simple; bh=g8tVOSmKODdWDGu0H8GcuoZ/GglJDm/0ZHioXuiWVfI=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=G6c2OhrdPiD+IUM/FZsdeVI60f4hJZHjfIc+NL72x8IoqqZSSgAnqAG7fooKZ5WDPdzVNWtMsPog5D/5fudyUJfga7fMQuoKBliCi+Q97vq+5ohyZP+7vkknL9eab/6uiMZb/oSjIyJD7UM4QwKnEKssOy0KGpvqIuHbApgMIHE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=biDG7Lt0; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="biDG7Lt0" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 56251C116C6; Fri, 6 Feb 2026 14:23:21 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1770387808; bh=g8tVOSmKODdWDGu0H8GcuoZ/GglJDm/0ZHioXuiWVfI=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=biDG7Lt0Ev43G9yPf/tgbfgfwOIveTEAb5DEh8dtk+U1XX8gz0Q/dtBZNfgOtMJEl kNfCLFIfPB6t6uP61zZj+shuMOeqZpOC31b4NHcrsKWospmBqyGW9A6Qq3TQJ4oPWp 70x4mH4gbM+55OkCaL8A7oEHXbswaRXY+tnV/SzSPKOeUNh4EUFhAowwgCz9W2ycM3 qgFsurW+hcFm187yRedu7bgw/O83UsalM4x9AAr3yvdOKALeJ/Det5+MMsFD6sxceg aGycEmOIKs+IR8sy/n14LeKQpTkOc2iVjLeUzbFPrBqK/lSA0Anv/bcODsH4+wsvyN oR5GpdjQzF1XQ== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Christophe Leroy (CS GROUP)" , "Rafael J. Wysocki" , Alexander Gordeev , Anna-Maria Behnsen , Ben Segall , Boqun Feng , Christian Borntraeger , Dietmar Eggemann , Heiko Carstens , Ingo Molnar , Jan Kiszka , Joel Fernandes , Juri Lelli , Kieran Bingham , Madhavan Srinivasan , Mel Gorman , Michael Ellerman , Neeraj Upadhyay , Nicholas Piggin , "Paul E . McKenney" , Peter Zijlstra , Steven Rostedt , Sven Schnelle , Thomas Gleixner , Uladzislau Rezki , Valentin Schneider , Vasily Gorbik , Vincent Guittot , Viresh Kumar , Xin Zhao , linux-pm@vger.kernel.org, linux-s390@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, Shrikanth Hegde Subject: [PATCH 04/15] powerpc/time: Prepare to stop elapsing in dynticks-idle Date: Fri, 6 Feb 2026 15:22:34 +0100 Message-ID: <20260206142245.58987-5-frederic@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20260206142245.58987-1-frederic@kernel.org> References: <20260206142245.58987-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Currently the tick subsystem stores the idle cputime accounting in private fields, allowing cohabitation with architecture idle vtime accounting. The former is fetched on online CPUs, the latter on offline CPUs. For consolidation purpose, architecture vtime accounting will continue to account the cputime but will make a break when the idle tick is stopped. The dyntick cputime accounting will then be relayed by the tick subsystem so that the idle cputime is still seen advancing coherently even when the tick isn't there to flush the idle vtime. Prepare for that and introduce three new APIs which will be used in subsequent patches: _ vtime_dynticks_start() is deemed to be called when idle enters in dyntick mode. The idle cputime that elapsed so far is accumulated. - vtime_dynticks_stop() is deemed to be called when idle exits from dyntick mode. The vtime entry clocks are fast-forward to current time so that idle accounting restarts elapsing from now. - vtime_reset() is deemed to be called from dynticks idle IRQ entry to fast-forward the clock to current time so that the IRQ time is still accounted by vtime while nohz cputime is paused. Also accumulated vtime won't be flushed from dyntick-idle ticks to avoid accounting twice the idle cputime, along with nohz accounting. Signed-off-by: Frederic Weisbecker --- arch/powerpc/kernel/time.c | 41 ++++++++++++++++++++++++++++++++++++++ include/linux/vtime.h | 6 ++++++ 2 files changed, 47 insertions(+) diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 4bbeb8644d3d..18506740f4a4 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -376,6 +376,47 @@ void vtime_task_switch(struct task_struct *prev) acct->starttime =3D acct0->starttime; } } + +#ifdef CONFIG_NO_HZ_COMMON +/** + * vtime_reset - Fast forward vtime entry clocks + * + * Called from dynticks idle IRQ entry to fast-forward the clocks to curre= nt time + * so that the IRQ time is still accounted by vtime while nohz cputime is = paused. + */ +void vtime_reset(void) +{ + struct cpu_accounting_data *acct =3D get_accounting(current); + + acct->starttime =3D mftb(); +#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME + acct->startspurr =3D read_spurr(acct->starttime); +#endif +} + +/** + * vtime_dyntick_start - Inform vtime about entry to idle-dynticks + * + * Called when idle enters in dyntick mode. The idle cputime that elapsed = so far + * is accumulated and the tick subsystem takes over the idle cputime accou= nting. + */ +void vtime_dyntick_start(void) +{ + vtime_account_idle(current); +} + +/** + * vtime_dyntick_stop - Inform vtime about exit from idle-dynticks + * + * Called when idle exits from dyntick mode. The vtime entry clocks are + * fast-forward to current time so that idle accounting restarts elapsing = from + * now. + */ +void vtime_dyntick_stop(void) +{ + vtime_reset(); +} +#endif /* CONFIG_NO_HZ_COMMON */ #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ =20 void __no_kcsan __delay(unsigned long loops) diff --git a/include/linux/vtime.h b/include/linux/vtime.h index 336875bea767..61b94c12d7dd 100644 --- a/include/linux/vtime.h +++ b/include/linux/vtime.h @@ -37,11 +37,17 @@ extern void vtime_account_irq(struct task_struct *tsk, = unsigned int offset); extern void vtime_account_softirq(struct task_struct *tsk); extern void vtime_account_hardirq(struct task_struct *tsk); extern void vtime_flush(struct task_struct *tsk); +extern void vtime_reset(void); +extern void vtime_dyntick_start(void); +extern void vtime_dyntick_stop(void); #else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ static inline void vtime_account_irq(struct task_struct *tsk, unsigned int= offset) { } static inline void vtime_account_softirq(struct task_struct *tsk) { } static inline void vtime_account_hardirq(struct task_struct *tsk) { } static inline void vtime_flush(struct task_struct *tsk) { } +static inline void vtime_reset(void) { } +static inline void vtime_dyntick_start(void) { } +extern inline void vtime_dyntick_stop(void) { } #endif =20 /* --=20 2.51.1 From nobody Sat Feb 7 21:14:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B48CE3EF0A4; Fri, 6 Feb 2026 14:23:36 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387816; cv=none; b=f7eX+vgGLKxuVjIQ7odAgmnHXp0hhDUjERzl2p15kJN+Az1WBDtYRnGZO9paHsDGLafzIjYzQCNTAO5PDrMpve4F4jlg+F24Ji0iCEUZHWJSSjAwV8wx48PiGzFFNrGGUh9WoCDSJGbwYi7EIgtTbeCssOm24f/J4HkxzV9S+zE= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387816; c=relaxed/simple; bh=DaRfqtPxemmkp+aTTTQoiIZqc3SEzpEw+S8gq3cyoPY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=EbuDHij1Sn1FA8lQ8FQNcyHxe1abnePMkhueH8jn3RWTMpuEuo/eYaLLj5O8bTmAtBn/E8+GLAnePZyKUEpCAUnqCUAX4X8WGSTJ4UbmOBp/7YHplnN4jH+QQyU5kl2K6SmfYzXaFguDjuAsrlz6VnYH8ZPMiyosmr9iTUeigFU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=BU84qQiq; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="BU84qQiq" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 223F1C19421; Fri, 6 Feb 2026 14:23:28 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1770387816; bh=DaRfqtPxemmkp+aTTTQoiIZqc3SEzpEw+S8gq3cyoPY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=BU84qQiqPenzVeN7b6iVXZU3Aq98TI8RnVWp5ThVnZaQ61YRQiT8g8RJ4n31aWvM+ Bc8FPIVGx6kobB/Pk0WfSAe1Q7nrqN3RQ2LAeyq3lQJyFcTZVmJaB8Dkd4LJpIa0cb F6F5ZKuxGoI8CQ7TisaeuHWEm46TNDEGIRT6xlIGKC63IuNB282k3riJV1x8sy8HsH dNuPM3JGMwVOfV3JK8DMe7cYWVli/dyd2Fdo5kUBNs3VoO/CrsP/Y2zKlpbxEq11o1 r2W+zflG36srXMqKgrj9zAMyOgMwEEFX0e0e/CiLQdsskyHwQsb2+jC82mVhAVo5pF dfIwwG2nb3dNw== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Christophe Leroy (CS GROUP)" , "Rafael J. Wysocki" , Alexander Gordeev , Anna-Maria Behnsen , Ben Segall , Boqun Feng , Christian Borntraeger , Dietmar Eggemann , Heiko Carstens , Ingo Molnar , Jan Kiszka , Joel Fernandes , Juri Lelli , Kieran Bingham , Madhavan Srinivasan , Mel Gorman , Michael Ellerman , Neeraj Upadhyay , Nicholas Piggin , "Paul E . McKenney" , Peter Zijlstra , Steven Rostedt , Sven Schnelle , Thomas Gleixner , Uladzislau Rezki , Valentin Schneider , Vasily Gorbik , Vincent Guittot , Viresh Kumar , Xin Zhao , linux-pm@vger.kernel.org, linux-s390@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, Shrikanth Hegde Subject: [PATCH 05/15] s390/time: Prepare to stop elapsing in dynticks-idle Date: Fri, 6 Feb 2026 15:22:35 +0100 Message-ID: <20260206142245.58987-6-frederic@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20260206142245.58987-1-frederic@kernel.org> References: <20260206142245.58987-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Currently the tick subsystem stores the idle cputime accounting in private fields, allowing cohabitation with architecture idle vtime accounting. The former is fetched on online CPUs, the latter on offline CPUs. For consolidation purpose, architecture vtime accounting will continue to account the cputime but will make a break when the idle tick is stopped. The dyntick cputime accounting will then be relayed by the tick subsystem so that the idle cputime is still seen advancing coherently even when the tick isn't there to flush the idle vtime. Prepare for that and introduce three new APIs which will be used in subsequent patches: _ vtime_dynticks_start() is deemed to be called when idle enters in dyntick mode. The idle cputime that elapsed so far is accumulated and accounted. Also idle time accounting is ignored. - vtime_dynticks_stop() is deemed to be called when idle exits from dyntick mode. The vtime entry clocks are fast-forward to current time so that idle accounting restarts elapsing from now. Also idle time accounting is resumed. - vtime_reset() is deemed to be called from dynticks idle IRQ entry to fast-forward the clock to current time so that the IRQ time is still accounted by vtime while nohz cputime is paused. Also accumulated vtime won't be flushed from dyntick-idle ticks to avoid accounting twice the idle cputime, along with nohz accounting. Signed-off-by: Frederic Weisbecker --- arch/s390/include/asm/idle.h | 14 +++++--- arch/s390/kernel/idle.c | 19 +++++++---- arch/s390/kernel/vtime.c | 65 ++++++++++++++++++++++++++++++------ 3 files changed, 77 insertions(+), 21 deletions(-) diff --git a/arch/s390/include/asm/idle.h b/arch/s390/include/asm/idle.h index 09f763b9eb40..285b3da318d6 100644 --- a/arch/s390/include/asm/idle.h +++ b/arch/s390/include/asm/idle.h @@ -8,17 +8,21 @@ #ifndef _S390_IDLE_H #define _S390_IDLE_H =20 +#include #include #include =20 struct s390_idle_data { - unsigned long idle_count; - unsigned long idle_time; - unsigned long clock_idle_enter; - unsigned long timer_idle_enter; - unsigned long mt_cycles_enter[8]; + bool idle_dyntick; + unsigned long idle_count; + unsigned long idle_time; + unsigned long clock_idle_enter; + unsigned long timer_idle_enter; + unsigned long mt_cycles_enter[8]; }; =20 +DECLARE_PER_CPU(struct s390_idle_data, s390_idle); + extern struct device_attribute dev_attr_idle_count; extern struct device_attribute dev_attr_idle_time_us; =20 diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c index 39cb8d0ae348..614db5ea6ea3 100644 --- a/arch/s390/kernel/idle.c +++ b/arch/s390/kernel/idle.c @@ -19,7 +19,7 @@ #include #include "entry.h" =20 -static DEFINE_PER_CPU(struct s390_idle_data, s390_idle); +DEFINE_PER_CPU(struct s390_idle_data, s390_idle); =20 void account_idle_time_irq(void) { @@ -35,7 +35,15 @@ void account_idle_time_irq(void) this_cpu_add(mt_cycles[i], cycles_new[i] - idle->mt_cycles_enter[i]); } =20 + WRITE_ONCE(idle->idle_count, READ_ONCE(idle->idle_count) + 1); + + /* Account time spent with enabled wait psw loaded as idle time. */ idle_time =3D lc->int_clock - idle->clock_idle_enter; + WRITE_ONCE(idle->idle_time, READ_ONCE(idle->idle_time) + idle_time); + + /* Dyntick idle time accounted by nohz/scheduler */ + if (idle->idle_dyntick) + return; =20 lc->steal_timer +=3D idle->clock_idle_enter - lc->last_update_clock; lc->last_update_clock =3D lc->int_clock; @@ -43,9 +51,6 @@ void account_idle_time_irq(void) lc->system_timer +=3D lc->last_update_timer - idle->timer_idle_enter; lc->last_update_timer =3D lc->sys_enter_timer; =20 - /* Account time spent with enabled wait psw loaded as idle time. */ - WRITE_ONCE(idle->idle_time, READ_ONCE(idle->idle_time) + idle_time); - WRITE_ONCE(idle->idle_count, READ_ONCE(idle->idle_count) + 1); account_idle_time(cputime_to_nsecs(idle_time)); } =20 @@ -61,8 +66,10 @@ void noinstr arch_cpu_idle(void) set_cpu_flag(CIF_ENABLED_WAIT); if (smp_cpu_mtid) stcctm(MT_DIAG, smp_cpu_mtid, (u64 *)&idle->mt_cycles_enter); - idle->clock_idle_enter =3D get_tod_clock_fast(); - idle->timer_idle_enter =3D get_cpu_timer(); + if (!idle->idle_dyntick) { + idle->clock_idle_enter =3D get_tod_clock_fast(); + idle->timer_idle_enter =3D get_cpu_timer(); + } bpon(); __load_psw_mask(psw_mask); } diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index 234a0ba30510..c19528eb4ee3 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -17,6 +17,7 @@ #include #include #include +#include #include =20 #include "entry.h" @@ -111,23 +112,30 @@ static void account_system_index_scaled(struct task_s= truct *p, u64 cputime, account_system_index_time(p, cputime_to_nsecs(cputime), index); } =20 -/* - * Update process times based on virtual cpu times stored by entry.S - * to the lowcore fields user_timer, system_timer & steal_clock. - */ -static int do_account_vtime(struct task_struct *tsk) +static inline void vtime_reset_last_update(struct lowcore *lc) { - u64 timer, clock, user, guest, system, hardirq, softirq; - struct lowcore *lc =3D get_lowcore(); - - timer =3D lc->last_update_timer; - clock =3D lc->last_update_clock; asm volatile( " stpt %0\n" /* Store current cpu timer value */ " stckf %1" /* Store current tod clock value */ : "=3DQ" (lc->last_update_timer), "=3DQ" (lc->last_update_clock) : : "cc"); +} + +/* + * Update process times based on virtual cpu times stored by entry.S + * to the lowcore fields user_timer, system_timer & steal_clock. + */ +static int do_account_vtime(struct task_struct *tsk) +{ + u64 timer, clock, user, guest, system, hardirq, softirq; + struct lowcore *lc =3D get_lowcore(); + + timer =3D lc->last_update_timer; + clock =3D lc->last_update_clock; + + vtime_reset_last_update(lc); + clock =3D lc->last_update_clock - clock; timer -=3D lc->last_update_timer; =20 @@ -261,6 +269,43 @@ void vtime_account_hardirq(struct task_struct *tsk) virt_timer_forward(delta); } =20 +#ifdef CONFIG_NO_HZ_COMMON +/** + * vtime_reset - Fast forward vtime entry clocks + * + * Called from dynticks idle IRQ entry to fast-forward the clocks to curre= nt time + * so that the IRQ time is still accounted by vtime while nohz cputime is = paused. + */ +void vtime_reset(void) +{ + vtime_reset_last_update(get_lowcore()); +} + +/** + * vtime_dyntick_start - Inform vtime about entry to idle-dynticks + * + * Called when idle enters in dyntick mode. The idle cputime that elapsed = so far + * is flushed and the tick subsystem takes over the idle cputime accountin= g. + */ +void vtime_dyntick_start(void) +{ + __this_cpu_write(s390_idle.idle_dyntick, true); + vtime_flush(current); +} + +/** + * vtime_dyntick_stop - Inform vtime about exit from idle-dynticks + * + * Called when idle exits from dyntick mode. The vtime entry clocks are + * fast-forward to current time and idle accounting resumes. + */ +void vtime_dyntick_stop(void) +{ + vtime_reset_last_update(get_lowcore()); + __this_cpu_write(s390_idle.idle_dyntick, false); +} +#endif /* CONFIG_NO_HZ_COMMON */ + /* * Sorted add to a list. List is linear searched until first bigger * element is found. --=20 2.51.1 From nobody Sat Feb 7 21:14:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id B8D3C212542; Fri, 6 Feb 2026 14:23:45 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387825; cv=none; b=l/CN2F051bj3W2+EO5/xzaLN+Z2EvhsakjAMf3bJrWfG4X9GsKaJgQSS76OAi1srhO9AJMVouzuUBN8yIpcStGYcWdjXh5zvXm8BnbOjNDX3ohojgOme8RBfZAX9MktU9YyjFMza1/zEHdCpo+nSYHabIA971IDVMlzOtqfiecs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387825; c=relaxed/simple; bh=IbgsIe5v0OELV1evugyCjy9XF5xOTfRaayo/oBM79xA=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=srS2tHEw2Y1Q9D5v6iQogBNCV7FimXSghHS9rVbweEJxgk2TE31zwQJd9/olM2sgFYWz2nJnOBTc3NbjsBjT3zVE+fVXTawRY8VM6UDxYOqUxnJkVY2JOjMecVpA0TNaOocBoDgrMq+ym/7i9kQ28y1ramWs+TAAIVoNugkl7Mo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=QuJHwKim; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="QuJHwKim" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 0D1A3C19421; Fri, 6 Feb 2026 14:23:36 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1770387825; bh=IbgsIe5v0OELV1evugyCjy9XF5xOTfRaayo/oBM79xA=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=QuJHwKimvlBEaQIGoSfFSWw9NDTB7xFmowSy62HChbpxS8iy3G+mEUj/wfuxFrlaE 5hlfMtG5hGCbjdTWRYDmBMTIihoVaqilYpYvD9QU9kQ6AachZN8IyGaj2td+xbxux/ ziugIIhogKJsFFPVS0cGD14tRHv0HmYpbgHrfxT2azEbcVAR/exVFgxbXwtKy0bFfy 3jx44jWh2QRHMP8YVaVpd14C/ZoX7UPxx8SCWU2xJKxGCnq98MSRk000APsyz7ADnJ CprSaCtnWDCODM1IuvJgTrafHPo/iNsOQioAH7uPH4P3yz/+Mc6pMcntIGR8IqS19d p5OM+KulhxrQg== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Christophe Leroy (CS GROUP)" , "Rafael J. Wysocki" , Alexander Gordeev , Anna-Maria Behnsen , Ben Segall , Boqun Feng , Christian Borntraeger , Dietmar Eggemann , Heiko Carstens , Ingo Molnar , Jan Kiszka , Joel Fernandes , Juri Lelli , Kieran Bingham , Madhavan Srinivasan , Mel Gorman , Michael Ellerman , Neeraj Upadhyay , Nicholas Piggin , "Paul E . McKenney" , Peter Zijlstra , Steven Rostedt , Sven Schnelle , Thomas Gleixner , Uladzislau Rezki , Valentin Schneider , Vasily Gorbik , Vincent Guittot , Viresh Kumar , Xin Zhao , linux-pm@vger.kernel.org, linux-s390@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, Shrikanth Hegde Subject: [PATCH 06/15] tick/sched: Unify idle cputime accounting Date: Fri, 6 Feb 2026 15:22:36 +0100 Message-ID: <20260206142245.58987-7-frederic@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20260206142245.58987-1-frederic@kernel.org> References: <20260206142245.58987-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The non-vtime dynticks-idle cputime accounting is a big mess that accumulates within two concurrent statistics, each having their own shortcomings: * The accounting for online CPUs which is based on the delta between tick_nohz_start_idle() and tick_nohz_stop_idle(). Pros: - Works when the tick is off - Has nsecs granularity Cons: - Account idle steal time but doesn't substract it from idle cputime. - Assumes CONFIG_IRQ_TIME_ACCOUNTING by not accounting IRQs but the IRQ time is simply ignored when CONFIG_IRQ_TIME_ACCOUNTING=3Dn - The windows between 1) idle task scheduling and the first call to tick_nohz_start_idle() and 2) idle task between the last tick_nohz_stop_idle() and the rest of the idle time are blindspots wrt. cputime accounting (though mostly insignificant amount) - Relies on private fields outside of kernel stats, with specific accessors. * The accounting for offline CPUs which is based on ticks and the jiffies delta during which the tick was stopped. Pros: - Handles steal time correctly - Handle CONFIG_IRQ_TIME_ACCOUNTING=3Dy and CONFIG_IRQ_TIME_ACCOUNTING=3Dn correctly. - Handles the whole idle task - Accounts directly to kernel stats, without midlayer accumulator. Cons: - Doesn't elapse when the tick is off, which doesn't make it suitable for online CPUs. - Has TICK_NSEC granularity (jiffies) - Needs to track the dyntick-idle ticks that were accounted and substract them from the total jiffies time spent while the tick was stopped. This is an ugly workaround. Having two different accounting for a single context is not the only problem: since those accountings are of different natures, it is possible to observe the global idle time going backward after a CPU goes offline. Clean up the situation with introducing a hybrid approach that stays coherent and works for both online and offline CPUs: * Tick based or native vtime accounting operate before the idle loop is entered and resume once the idle loop prepares to exit. * When the idle loop starts, switch to dynticks-idle accounting as is done currently, except that the statistics accumulate directly to the relevant kernel stat fields. * Private dyntick cputime accounting fields are removed. * Works on both online and offline case. Further improvement will include: * Only switch to dynticks-idle cputime accounting when the tick actually goes in dynticks mode. * Handle CONFIG_IRQ_TIME_ACCOUNTING=3Dn correctly such that the dynticks-idle accounting still elapses while on IRQs. * Correctly substract idle steal cputime from idle time Reported-by: Xin Zhao Signed-off-by: Frederic Weisbecker --- include/linux/kernel_stat.h | 24 ++++++++++--- include/linux/vtime.h | 7 +++- kernel/sched/cputime.c | 62 ++++++++++++++++---------------- kernel/time/tick-sched.c | 71 +++++++++++-------------------------- 4 files changed, 76 insertions(+), 88 deletions(-) diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index dd020ecaf67b..ba65aad308a1 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -34,6 +34,9 @@ enum cpu_usage_stat { }; =20 struct kernel_cpustat { +#ifdef CONFIG_NO_HZ_COMMON + int idle_dyntick; +#endif u64 cpustat[NR_STATS]; }; =20 @@ -99,6 +102,20 @@ static inline unsigned long kstat_cpu_irqs_sum(unsigned= int cpu) return kstat_cpu(cpu).irqs_sum; } =20 +#ifdef CONFIG_NO_HZ_COMMON +extern void kcpustat_dyntick_start(void); +extern void kcpustat_dyntick_stop(void); +static inline bool kcpustat_idle_dyntick(void) +{ + return __this_cpu_read(kernel_cpustat.idle_dyntick); +} +#else +static inline bool kcpustat_idle_dyntick(void) +{ + return false; +} +#endif /* CONFIG_NO_HZ_COMMON */ + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN extern u64 kcpustat_field(enum cpu_usage_stat usage, int cpu); extern void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu); @@ -113,7 +130,7 @@ static inline void kcpustat_cpu_fetch(struct kernel_cpu= stat *dst, int cpu) *dst =3D kcpustat_cpu(cpu); } =20 -#endif +#endif /* !CONFIG_VIRT_CPU_ACCOUNTING_GEN */ =20 extern void account_user_time(struct task_struct *, u64); extern void account_guest_time(struct task_struct *, u64); @@ -127,14 +144,13 @@ extern u64 get_idle_time(struct kernel_cpustat *kcs, = int cpu); #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE static inline void account_process_tick(struct task_struct *tsk, int user) { - vtime_flush(tsk); + if (!kcpustat_idle_dyntick()) + vtime_flush(tsk); } #else extern void account_process_tick(struct task_struct *, int user); #endif =20 -extern void account_idle_ticks(unsigned long ticks); - #ifdef CONFIG_SCHED_CORE extern void __account_forceidle_time(struct task_struct *tsk, u64 delta); #endif diff --git a/include/linux/vtime.h b/include/linux/vtime.h index 61b94c12d7dd..a4506336002d 100644 --- a/include/linux/vtime.h +++ b/include/linux/vtime.h @@ -31,6 +31,11 @@ static inline bool vtime_generic_enabled_cpu(int cpu) return context_tracking_enabled_cpu(cpu); } =20 +static inline bool vtime_generic_enabled_this_cpu(void) +{ + return context_tracking_enabled_this_cpu(); +} + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE extern void vtime_account_idle(struct task_struct *tsk); extern void vtime_account_irq(struct task_struct *tsk, unsigned int offset= ); @@ -90,7 +95,7 @@ static inline bool vtime_accounting_enabled_cpu(int cpu) =20 static inline bool vtime_accounting_enabled_this_cpu(void) { - return context_tracking_enabled_this_cpu(); + return vtime_generic_enabled_this_cpu(); } =20 extern void vtime_task_switch_generic(struct task_struct *prev); diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 5613838d0307..d67f93e845a7 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -400,16 +400,30 @@ static void irqtime_account_process_tick(struct task_= struct *p, int user_tick, } } =20 -static void irqtime_account_idle_ticks(int ticks) -{ - irqtime_account_process_tick(current, 0, ticks); -} #else /* !CONFIG_IRQ_TIME_ACCOUNTING: */ -static inline void irqtime_account_idle_ticks(int ticks) { } static inline void irqtime_account_process_tick(struct task_struct *p, int= user_tick, int nr_ticks) { } #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */ =20 +#ifdef CONFIG_NO_HZ_COMMON +void kcpustat_dyntick_start(void) +{ + if (!vtime_generic_enabled_this_cpu()) { + vtime_dyntick_start(); + __this_cpu_write(kernel_cpustat.idle_dyntick, 1); + } +} + +void kcpustat_dyntick_stop(void) +{ + if (!vtime_generic_enabled_this_cpu()) { + __this_cpu_write(kernel_cpustat.idle_dyntick, 0); + vtime_dyntick_stop(); + steal_account_process_time(ULONG_MAX); + } +} +#endif /* CONFIG_NO_HZ_COMMON */ + /* * Use precise platform statistics if available: */ @@ -423,11 +437,15 @@ void vtime_account_irq(struct task_struct *tsk, unsig= ned int offset) vtime_account_hardirq(tsk); } else if (pc & SOFTIRQ_OFFSET) { vtime_account_softirq(tsk); - } else if (!IS_ENABLED(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) && - is_idle_task(tsk)) { - vtime_account_idle(tsk); + } else if (!kcpustat_idle_dyntick()) { + if (!IS_ENABLED(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) && + is_idle_task(tsk)) { + vtime_account_idle(tsk); + } else { + vtime_account_kernel(tsk); + } } else { - vtime_account_kernel(tsk); + vtime_reset(); } } =20 @@ -469,6 +487,9 @@ void account_process_tick(struct task_struct *p, int us= er_tick) if (vtime_accounting_enabled_this_cpu()) return; =20 + if (kcpustat_idle_dyntick()) + return; + if (irqtime_enabled()) { irqtime_account_process_tick(p, user_tick, 1); return; @@ -490,29 +511,6 @@ void account_process_tick(struct task_struct *p, int u= ser_tick) account_idle_time(cputime); } =20 -/* - * Account multiple ticks of idle time. - * @ticks: number of stolen ticks - */ -void account_idle_ticks(unsigned long ticks) -{ - u64 cputime, steal; - - if (irqtime_enabled()) { - irqtime_account_idle_ticks(ticks); - return; - } - - cputime =3D ticks * TICK_NSEC; - steal =3D steal_account_process_time(ULONG_MAX); - - if (steal >=3D cputime) - return; - - cputime -=3D steal; - account_idle_time(cputime); -} - /* * Adjust tick based cputime random precision against scheduler runtime * accounting. diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 9632066aea4d..21ac561a8545 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -285,8 +285,6 @@ static void tick_sched_handle(struct tick_sched *ts, st= ruct pt_regs *regs) if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && tick_sched_flag_test(ts, TS_FLAG_STOPPED)) { touch_softlockup_watchdog_sched(); - if (is_idle_task(current)) - ts->idle_jiffies++; /* * In case the current tick fired too early past its expected * expiration, make sure we don't bypass the next clock reprogramming @@ -744,8 +742,12 @@ static void tick_nohz_update_jiffies(ktime_t now) =20 static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now) { + u64 *cpustat =3D kcpustat_this_cpu->cpustat; ktime_t delta; =20 + if (vtime_generic_enabled_this_cpu()) + return; + if (WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE))) return; =20 @@ -753,9 +755,9 @@ static void tick_nohz_stop_idle(struct tick_sched *ts, = ktime_t now) =20 write_seqcount_begin(&ts->idle_sleeptime_seq); if (nr_iowait_cpu(smp_processor_id()) > 0) - ts->iowait_sleeptime =3D ktime_add(ts->iowait_sleeptime, delta); + cpustat[CPUTIME_IOWAIT] =3D ktime_add(cpustat[CPUTIME_IOWAIT], delta); else - ts->idle_sleeptime =3D ktime_add(ts->idle_sleeptime, delta); + cpustat[CPUTIME_IDLE] =3D ktime_add(cpustat[CPUTIME_IDLE], delta); =20 ts->idle_entrytime =3D now; tick_sched_flag_clear(ts, TS_FLAG_IDLE_ACTIVE); @@ -766,18 +768,21 @@ static void tick_nohz_stop_idle(struct tick_sched *ts= , ktime_t now) =20 static void tick_nohz_start_idle(struct tick_sched *ts) { + if (vtime_generic_enabled_this_cpu()) + return; + write_seqcount_begin(&ts->idle_sleeptime_seq); ts->idle_entrytime =3D ktime_get(); tick_sched_flag_set(ts, TS_FLAG_IDLE_ACTIVE); write_seqcount_end(&ts->idle_sleeptime_seq); - sched_clock_idle_sleep_event(); } =20 -static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, ktime_t= *sleeptime, +static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, bool compute_delta, u64 *last_update_time) { struct tick_sched *ts =3D &per_cpu(tick_cpu_sched, cpu); + u64 *cpustat =3D kcpustat_cpu(cpu).cpustat; ktime_t now, idle; unsigned int seq; =20 @@ -799,9 +804,9 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usag= e_stat idx, ktime_t *slee if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE) && compute_delta) { ktime_t delta =3D ktime_sub(now, ts->idle_entrytime); =20 - idle =3D ktime_add(*sleeptime, delta); + idle =3D ktime_add(cpustat[idx], delta); } else { - idle =3D *sleeptime; + idle =3D cpustat[idx]; } } while (read_seqcount_retry(&ts->idle_sleeptime_seq, seq)); =20 @@ -828,9 +833,7 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usag= e_stat idx, ktime_t *slee */ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) { - struct tick_sched *ts =3D &per_cpu(tick_cpu_sched, cpu); - - return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE, &ts->idle_sleeptime, + return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE, !nr_iowait_cpu(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); @@ -854,9 +857,7 @@ EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); */ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) { - struct tick_sched *ts =3D &per_cpu(tick_cpu_sched, cpu); - - return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT, &ts->iowait_sleeptime, + return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT, nr_iowait_cpu(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); @@ -1256,10 +1257,8 @@ void tick_nohz_idle_stop_tick(void) ts->idle_sleeps++; ts->idle_expires =3D expires; =20 - if (!was_stopped && tick_sched_flag_test(ts, TS_FLAG_STOPPED)) { - ts->idle_jiffies =3D ts->last_jiffies; + if (!was_stopped && tick_sched_flag_test(ts, TS_FLAG_STOPPED)) nohz_balance_enter_idle(cpu); - } } else { tick_nohz_retain_tick(ts); } @@ -1288,6 +1287,7 @@ void tick_nohz_idle_enter(void) WARN_ON_ONCE(ts->timer_expires_base); =20 tick_sched_flag_set(ts, TS_FLAG_INIDLE); + kcpustat_dyntick_start(); tick_nohz_start_idle(ts); =20 local_irq_enable(); @@ -1413,37 +1413,12 @@ unsigned long tick_nohz_get_idle_calls_cpu(int cpu) return ts->idle_calls; } =20 -static void tick_nohz_account_idle_time(struct tick_sched *ts, - ktime_t now) -{ - unsigned long ticks; - - ts->idle_exittime =3D now; - - if (vtime_accounting_enabled_this_cpu()) - return; - /* - * We stopped the tick in idle. update_process_times() would miss the - * time we slept, as it does only a 1 tick accounting. - * Enforce that this is accounted to idle ! - */ - ticks =3D jiffies - ts->idle_jiffies; - /* - * We might be one off. Do not randomly account a huge number of ticks! - */ - if (ticks && ticks < LONG_MAX) - account_idle_ticks(ticks); -} - void tick_nohz_idle_restart_tick(void) { struct tick_sched *ts =3D this_cpu_ptr(&tick_cpu_sched); =20 - if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) { - ktime_t now =3D ktime_get(); - tick_nohz_restart_sched_tick(ts, now); - tick_nohz_account_idle_time(ts, now); - } + if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) + tick_nohz_restart_sched_tick(ts, ktime_get()); } =20 static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now) @@ -1452,8 +1427,6 @@ static void tick_nohz_idle_update_tick(struct tick_sc= hed *ts, ktime_t now) __tick_nohz_full_update_tick(ts, now); else tick_nohz_restart_sched_tick(ts, now); - - tick_nohz_account_idle_time(ts, now); } =20 /** @@ -1495,6 +1468,7 @@ void tick_nohz_idle_exit(void) =20 if (tick_stopped) tick_nohz_idle_update_tick(ts, now); + kcpustat_dyntick_stop(); =20 local_irq_enable(); } @@ -1631,20 +1605,15 @@ void tick_setup_sched_timer(bool hrtimer) void tick_sched_timer_dying(int cpu) { struct tick_sched *ts =3D &per_cpu(tick_cpu_sched, cpu); - ktime_t idle_sleeptime, iowait_sleeptime; unsigned long idle_calls, idle_sleeps; =20 /* This must happen before hrtimers are migrated! */ if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) hrtimer_cancel(&ts->sched_timer); =20 - idle_sleeptime =3D ts->idle_sleeptime; - iowait_sleeptime =3D ts->iowait_sleeptime; idle_calls =3D ts->idle_calls; idle_sleeps =3D ts->idle_sleeps; memset(ts, 0, sizeof(*ts)); - ts->idle_sleeptime =3D idle_sleeptime; - ts->iowait_sleeptime =3D iowait_sleeptime; ts->idle_calls =3D idle_calls; ts->idle_sleeps =3D idle_sleeps; } --=20 2.51.1 From nobody Sat Feb 7 21:14:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D57E33F0763; Fri, 6 Feb 2026 14:23:53 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387833; cv=none; b=QxagZzRDte+WT3A2AX+Jmi4e9pvGemoCnGOzQOCWuCQEEKOSg4lvPOZA2VRISzeDXXyZU/zxdcItO4rJOGNPmi5qMPOudoSaZ811yr1PGc124c4+8xGz0Q4UypB89uvHV7hjL3iaq82tzn0LEQg8YtbU6PwoMWEhHNr6HJRsmfI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387833; c=relaxed/simple; bh=E9zTeO5x+KjyNgewXK9Mi3ZXHv+NiD0HDMZoN+zshlE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=uFV+moBnlBrwy60AFYGuyuNJ7irhfnjYso1cPH9GWQCibsxTEqJbQDKERgljWikPA9nGq0ru/mqOG6DrV1BY0OgqCjTtIvac9RJoBaH1tLdoj0B15TCkLUriu37X9WxKPnhxMXkf84MtCtm1KzEyZ72eizcvxI2Kbw8R7bb45QE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=kFmGLTxE; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="kFmGLTxE" Received: by smtp.kernel.org (Postfix) with ESMTPSA id D0AE5C116C6; Fri, 6 Feb 2026 14:23:45 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1770387833; bh=E9zTeO5x+KjyNgewXK9Mi3ZXHv+NiD0HDMZoN+zshlE=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=kFmGLTxEaIrIJJzRiPyrmfO9obNPSzQIgdiRScbMZu6qmx1Q6FtQ3R9Aw+6E8sRP1 3HtC8F8wo9w5y4hyRZiXDWe6Yoq2bgTp2n8jPUF3eJ6HQBvK+/pJ9k1rbP08KaVmzm ZlV4GHCqJz3b4bCL4bpJyQm/uQ7FP1PYUfH4bOW6rvOZ1dWtTlSgNXYl+BX1RnSr67 yzI4+xZbT6c/vJnJYNf9zWE2vAXmLgUjA2AWtc/XVqEC4McNjBgteT+FxFDp9kUWts KcmTfBVhmkCrD0QQgxfkHNsLefiRvPn55fMJFTNwmnJgP5m43MOuO0Hw8OxB1AGwKO cytjZ4krynFmA== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Christophe Leroy (CS GROUP)" , "Rafael J. Wysocki" , Alexander Gordeev , Anna-Maria Behnsen , Ben Segall , Boqun Feng , Christian Borntraeger , Dietmar Eggemann , Heiko Carstens , Ingo Molnar , Jan Kiszka , Joel Fernandes , Juri Lelli , Kieran Bingham , Madhavan Srinivasan , Mel Gorman , Michael Ellerman , Neeraj Upadhyay , Nicholas Piggin , "Paul E . McKenney" , Peter Zijlstra , Steven Rostedt , Sven Schnelle , Thomas Gleixner , Uladzislau Rezki , Valentin Schneider , Vasily Gorbik , Vincent Guittot , Viresh Kumar , Xin Zhao , linux-pm@vger.kernel.org, linux-s390@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, Shrikanth Hegde Subject: [PATCH 07/15] cpufreq: ondemand: Simplify idle cputime granularity test Date: Fri, 6 Feb 2026 15:22:37 +0100 Message-ID: <20260206142245.58987-8-frederic@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20260206142245.58987-1-frederic@kernel.org> References: <20260206142245.58987-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" cpufreq calls get_cpu_idle_time_us() just to know if idle cputime accounting has a nanoseconds granularity. Use the appropriate indicator instead to make that deduction. Signed-off-by: Frederic Weisbecker --- drivers/cpufreq/cpufreq_ondemand.c | 7 +------ include/linux/tick.h | 2 ++ kernel/time/hrtimer.c | 2 +- kernel/time/tick-internal.h | 2 -- kernel/time/tick-sched.c | 8 +++++++- kernel/time/timer.c | 2 +- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_o= ndemand.c index a6ecc203f7b7..bb7db82930e4 100644 --- a/drivers/cpufreq/cpufreq_ondemand.c +++ b/drivers/cpufreq/cpufreq_ondemand.c @@ -334,17 +334,12 @@ static void od_free(struct policy_dbs_info *policy_db= s) static int od_init(struct dbs_data *dbs_data) { struct od_dbs_tuners *tuners; - u64 idle_time; - int cpu; =20 tuners =3D kzalloc(sizeof(*tuners), GFP_KERNEL); if (!tuners) return -ENOMEM; =20 - cpu =3D get_cpu(); - idle_time =3D get_cpu_idle_time_us(cpu, NULL); - put_cpu(); - if (idle_time !=3D -1ULL) { + if (tick_nohz_is_active()) { /* Idle micro accounting is supported. Use finer thresholds */ dbs_data->up_threshold =3D MICRO_FREQUENCY_UP_THRESHOLD; } else { diff --git a/include/linux/tick.h b/include/linux/tick.h index ac76ae9fa36d..738007d6f577 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -126,6 +126,7 @@ enum tick_dep_bits { =20 #ifdef CONFIG_NO_HZ_COMMON extern bool tick_nohz_enabled; +extern bool tick_nohz_is_active(void); extern bool tick_nohz_tick_stopped(void); extern bool tick_nohz_tick_stopped_cpu(int cpu); extern void tick_nohz_idle_stop_tick(void); @@ -142,6 +143,7 @@ extern u64 get_cpu_idle_time_us(int cpu, u64 *last_upda= te_time); extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time); #else /* !CONFIG_NO_HZ_COMMON */ #define tick_nohz_enabled (0) +static inline bool tick_nohz_is_active(void) { return false; } static inline int tick_nohz_tick_stopped(void) { return 0; } static inline int tick_nohz_tick_stopped_cpu(int cpu) { return 0; } static inline void tick_nohz_idle_stop_tick(void) { } diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index f8ea8c8fc895..e1bbf883dfa8 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -943,7 +943,7 @@ void clock_was_set(unsigned int bases) cpumask_var_t mask; int cpu; =20 - if (!hrtimer_hres_active(cpu_base) && !tick_nohz_active) + if (!hrtimer_hres_active(cpu_base) && !tick_nohz_is_active()) goto out_timerfd; =20 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index 4e4f7bbe2a64..597d816d22e8 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h @@ -156,7 +156,6 @@ static inline void tick_nohz_init(void) { } #endif =20 #ifdef CONFIG_NO_HZ_COMMON -extern unsigned long tick_nohz_active; extern void timers_update_nohz(void); extern u64 get_jiffies_update(unsigned long *basej); # ifdef CONFIG_SMP @@ -171,7 +170,6 @@ extern void timer_expire_remote(unsigned int cpu); # endif #else /* CONFIG_NO_HZ_COMMON */ static inline void timers_update_nohz(void) { } -#define tick_nohz_active (0) #endif =20 DECLARE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases); diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 21ac561a8545..81c619bf662c 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -691,7 +691,7 @@ void __init tick_nohz_init(void) * NO HZ enabled ? */ bool tick_nohz_enabled __read_mostly =3D true; -unsigned long tick_nohz_active __read_mostly; +static unsigned long tick_nohz_active __read_mostly; /* * Enable / Disable tickless mode */ @@ -702,6 +702,12 @@ static int __init setup_tick_nohz(char *str) =20 __setup("nohz=3D", setup_tick_nohz); =20 +bool tick_nohz_is_active(void) +{ + return tick_nohz_active; +} +EXPORT_SYMBOL_GPL(tick_nohz_is_active); + bool tick_nohz_tick_stopped(void) { struct tick_sched *ts =3D this_cpu_ptr(&tick_cpu_sched); diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 1f2364126894..7e1e3bde6b8b 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -281,7 +281,7 @@ DEFINE_STATIC_KEY_FALSE(timers_migration_enabled); =20 static void timers_update_migration(void) { - if (sysctl_timer_migration && tick_nohz_active) + if (sysctl_timer_migration && tick_nohz_is_active()) static_branch_enable(&timers_migration_enabled); else static_branch_disable(&timers_migration_enabled); --=20 2.51.1 From nobody Sat Feb 7 21:14:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id AD0F0212542; Fri, 6 Feb 2026 14:24:01 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387841; cv=none; b=dMikEO7Tm4fgKGiQBdnwUBkArJGpiLFiUWQdqSjVkbuJ5HmQJ0NOr/qJ8VnhudjActyZkyBSkbOnhvK7NKew3JRAQy0IiupIScie531UGqfZ91VB2Okfq2FaLC0cKUSZVhsmZ80j9aGYkm1O3xlBsZHPGkjyyZMy6kCBTf2QvEk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387841; c=relaxed/simple; bh=Do+hwWPxjZYbNWyybj7BAgIt5vnOwzvEtgVy4FMIXwU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=fu0V6TSQ3IpmIgByPJisjs2G0SLAM6o7O4gV9gtodISsohfOnapRovh2iS+vCpcukw4uaVBWMkDB3e0W4D2t9ZB3uvTomlRVBC7ZCOOY/AXDoSLJ9bM/hl3m1mrijgF6MWX++sSSAx6WGDDpIEJ7+cZp/VBSetsiw9i91vKi3Sw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=MNCOcSTU; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="MNCOcSTU" Received: by smtp.kernel.org (Postfix) with ESMTPSA id E0C84C19424; Fri, 6 Feb 2026 14:23:53 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1770387841; bh=Do+hwWPxjZYbNWyybj7BAgIt5vnOwzvEtgVy4FMIXwU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=MNCOcSTUpyOl9Q4mQMbIw/URy1l4D0S4asdOyftfsDcewue8Dao7li7nylw6jdcwE 5xcCiyjI3UiF0TauFM9tLjJ8X4dezh3KXpax440RTfP2Ql0TwnHiTyko6XU/bojUvv sZme44B3wIxiPO5Konropk5kzhzrEJrsDg19F4SG4dMN9FqNlYgMtRFZ7sPQ4GZuum GwLqscapKQe5/ksPhSAwloBYLxXkckaoYnECQws7iJQYpvxbnyf/1LOy58zv7lUvmI 2qoQ3CyzqCo7aEPiDlZx6WHpY6T2bdtoHv3tqUL6o4SEHwUMgwS/fDuTASEFBuahwR tdeMxn8n7tTnw== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Christophe Leroy (CS GROUP)" , "Rafael J. Wysocki" , Alexander Gordeev , Anna-Maria Behnsen , Ben Segall , Boqun Feng , Christian Borntraeger , Dietmar Eggemann , Heiko Carstens , Ingo Molnar , Jan Kiszka , Joel Fernandes , Juri Lelli , Kieran Bingham , Madhavan Srinivasan , Mel Gorman , Michael Ellerman , Neeraj Upadhyay , Nicholas Piggin , "Paul E . McKenney" , Peter Zijlstra , Steven Rostedt , Sven Schnelle , Thomas Gleixner , Uladzislau Rezki , Valentin Schneider , Vasily Gorbik , Vincent Guittot , Viresh Kumar , Xin Zhao , linux-pm@vger.kernel.org, linux-s390@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, Shrikanth Hegde Subject: [PATCH 08/15] tick/sched: Remove nohz disabled special case in cputime fetch Date: Fri, 6 Feb 2026 15:22:38 +0100 Message-ID: <20260206142245.58987-9-frederic@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20260206142245.58987-1-frederic@kernel.org> References: <20260206142245.58987-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Even when nohz is not runtime enabled, the dynticks idle cputime accounting can run and the common idle cputime accessors are still relevant. Remove the nohz disabled special case accordingly. Signed-off-by: Frederic Weisbecker --- kernel/time/tick-sched.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 81c619bf662c..2b58cdc326c3 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -792,9 +792,6 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usag= e_stat idx, ktime_t now, idle; unsigned int seq; =20 - if (!tick_nohz_active) - return -1; - now =3D ktime_get(); if (last_update_time) *last_update_time =3D ktime_to_us(now); @@ -835,7 +832,7 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usag= e_stat idx, * This time is measured via accounting rather than sampling, * and is as accurate as ktime_get() is. * - * Return: -1 if NOHZ is not enabled, else total idle time of the @cpu + * Return: -1 if generic vtime is enabled, else total idle time of the @cpu */ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) { @@ -859,7 +856,7 @@ EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); * This time is measured via accounting rather than sampling, * and is as accurate as ktime_get() is. * - * Return: -1 if NOHZ is not enabled, else total iowait time of @cpu + * Return: -1 if generic vtime is enabled, else total iowait time of @cpu */ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) { --=20 2.51.1 From nobody Sat Feb 7 21:14:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A02813EF0A4; Fri, 6 Feb 2026 14:24:09 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387849; cv=none; b=uqcXKNoAfjdWBs3ulDOYzZ7iRzleTBL5beJFr+7wJ9+CMpMAra/IBUdtmVctGcAnxnNvzQbRQfULGUn3OH/P9ojzJmuU7yqk/2IJ/Eh7JfIhg6ObNEI0IqqaOa4nNBRZXkhlIOduxW+4oSJM4/QX3bqIB5JXeeLlj9LMM7PQbas= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387849; c=relaxed/simple; bh=Zut58oWwnAdSAa/LElaysKfGWrc+d2Pi1wvax1K4ey4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=bBuNYTaePDQRt/S7ks3AymILzFxvtxoUuxLHd2S1ruMFLCjjGgZkAenXnvHKvZvZvSZWHpUv4MpIL7BGRuOBs2JnqmvOjtjXVBWByCihHrP9hRXpJpEIueluS/MEzvKBv/o64L7zKVHZpDDhZRjOMeWPEneG0AVdIZQr0o0QAjM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=XZ3eKQ+3; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="XZ3eKQ+3" Received: by smtp.kernel.org (Postfix) with ESMTPSA id A9207C116C6; Fri, 6 Feb 2026 14:24:01 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1770387849; bh=Zut58oWwnAdSAa/LElaysKfGWrc+d2Pi1wvax1K4ey4=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=XZ3eKQ+3XefXB2htQQ0iVuhOST4+54PDYJpfCvGoyyq5B3LlcDa4KaugtG96dNXSP wHL2s6yqK44c5buBck5RKPK7ziHGVaoMSn/KD0Blxo9osLaexiinxGjQJq6G+yniZd H1CRhuDQ3XoGM/kST+ZqrKsEDagIcL9+7qE6TUa+krwM0oHhekBA6S5EPL0M3TDwZg 5016h3t6IhYDY4weyAyBkF1NeYUVkGN1+Dnm8QfGI+dVQN2qQkL77JwjJeHWtC6xWH XyTQTPhLN4nhg+2IEjIKdv7f5lKW3x7D7qI1vmWsB9tgwYzvaXyjfcGa12kt5xoDd4 HZb2e58GATc5w== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Christophe Leroy (CS GROUP)" , "Rafael J. Wysocki" , Alexander Gordeev , Anna-Maria Behnsen , Ben Segall , Boqun Feng , Christian Borntraeger , Dietmar Eggemann , Heiko Carstens , Ingo Molnar , Jan Kiszka , Joel Fernandes , Juri Lelli , Kieran Bingham , Madhavan Srinivasan , Mel Gorman , Michael Ellerman , Neeraj Upadhyay , Nicholas Piggin , "Paul E . McKenney" , Peter Zijlstra , Steven Rostedt , Sven Schnelle , Thomas Gleixner , Uladzislau Rezki , Valentin Schneider , Vasily Gorbik , Vincent Guittot , Viresh Kumar , Xin Zhao , linux-pm@vger.kernel.org, linux-s390@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, Shrikanth Hegde Subject: [PATCH 09/15] tick/sched: Move dyntick-idle cputime accounting to cputime code Date: Fri, 6 Feb 2026 15:22:39 +0100 Message-ID: <20260206142245.58987-10-frederic@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20260206142245.58987-1-frederic@kernel.org> References: <20260206142245.58987-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Although the dynticks-idle cputime accounting is necessarily tied to the tick subsystem, the actual related accounting code has no business residing there and should be part of the scheduler cputime code. Move away the relevant pieces and state machine to where they belong. Signed-off-by: Frederic Weisbecker --- include/linux/kernel_stat.h | 14 +++- kernel/sched/cputime.c | 149 +++++++++++++++++++++++++++++++-- kernel/time/tick-sched.c | 162 +++++++----------------------------- 3 files changed, 184 insertions(+), 141 deletions(-) diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index ba65aad308a1..9343353ac7a3 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -35,9 +35,12 @@ enum cpu_usage_stat { =20 struct kernel_cpustat { #ifdef CONFIG_NO_HZ_COMMON - int idle_dyntick; + bool idle_dyntick; + bool idle_elapse; + seqcount_t idle_sleeptime_seq; + u64 idle_entrytime; #endif - u64 cpustat[NR_STATS]; + u64 cpustat[NR_STATS]; }; =20 struct kernel_stat { @@ -103,8 +106,11 @@ static inline unsigned long kstat_cpu_irqs_sum(unsigne= d int cpu) } =20 #ifdef CONFIG_NO_HZ_COMMON -extern void kcpustat_dyntick_start(void); -extern void kcpustat_dyntick_stop(void); +extern void kcpustat_dyntick_start(u64 now); +extern void kcpustat_dyntick_stop(u64 now); +extern void kcpustat_irq_enter(u64 now); +extern void kcpustat_irq_exit(u64 now); + static inline bool kcpustat_idle_dyntick(void) { return __this_cpu_read(kernel_cpustat.idle_dyntick); diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index d67f93e845a7..d2cad4d8dc10 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -2,6 +2,7 @@ /* * Simple CPU accounting cgroup controller */ +#include #include #include #include "sched.h" @@ -406,22 +407,156 @@ static inline void irqtime_account_process_tick(stru= ct task_struct *p, int user_ #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */ =20 #ifdef CONFIG_NO_HZ_COMMON -void kcpustat_dyntick_start(void) +static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now) { - if (!vtime_generic_enabled_this_cpu()) { - vtime_dyntick_start(); - __this_cpu_write(kernel_cpustat.idle_dyntick, 1); - } + u64 *cpustat =3D kc->cpustat; + u64 delta; + + if (!kc->idle_elapse) + return; + + delta =3D now - kc->idle_entrytime; + + write_seqcount_begin(&kc->idle_sleeptime_seq); + if (nr_iowait_cpu(smp_processor_id()) > 0) + cpustat[CPUTIME_IOWAIT] +=3D delta; + else + cpustat[CPUTIME_IDLE] +=3D delta; + + kc->idle_entrytime =3D now; + kc->idle_elapse =3D false; + write_seqcount_end(&kc->idle_sleeptime_seq); } =20 -void kcpustat_dyntick_stop(void) +static void kcpustat_idle_start(struct kernel_cpustat *kc, u64 now) { + write_seqcount_begin(&kc->idle_sleeptime_seq); + kc->idle_entrytime =3D now; + kc->idle_elapse =3D true; + write_seqcount_end(&kc->idle_sleeptime_seq); +} + +void kcpustat_dyntick_stop(u64 now) +{ + struct kernel_cpustat *kc =3D kcpustat_this_cpu; + if (!vtime_generic_enabled_this_cpu()) { - __this_cpu_write(kernel_cpustat.idle_dyntick, 0); + WARN_ON_ONCE(!kc->idle_dyntick); + kcpustat_idle_stop(kc, now); + kc->idle_dyntick =3D false; vtime_dyntick_stop(); steal_account_process_time(ULONG_MAX); } } + +void kcpustat_dyntick_start(u64 now) +{ + struct kernel_cpustat *kc =3D kcpustat_this_cpu; + + if (!vtime_generic_enabled_this_cpu()) { + vtime_dyntick_start(); + kc->idle_dyntick =3D true; + kcpustat_idle_start(kc, now); + } +} + +void kcpustat_irq_enter(u64 now) +{ + struct kernel_cpustat *kc =3D kcpustat_this_cpu; + + if (!vtime_generic_enabled_this_cpu()) + kcpustat_idle_stop(kc, now); +} + +void kcpustat_irq_exit(u64 now) +{ + struct kernel_cpustat *kc =3D kcpustat_this_cpu; + + if (!vtime_generic_enabled_this_cpu()) + kcpustat_idle_start(kc, now); +} + +static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, + bool compute_delta, u64 *last_update_time) +{ + struct kernel_cpustat *kc =3D &kcpustat_cpu(cpu); + u64 *cpustat =3D kc->cpustat; + unsigned int seq; + ktime_t now; + u64 idle; + + now =3D ktime_get(); + if (last_update_time) + *last_update_time =3D ktime_to_us(now); + + if (vtime_generic_enabled_cpu(cpu)) { + idle =3D kcpustat_field(idx, cpu); + goto to_us; + } + + do { + seq =3D read_seqcount_begin(&kc->idle_sleeptime_seq); + + if (kc->idle_elapse && compute_delta) + idle =3D cpustat[idx] + (now - kc->idle_entrytime); + else + idle =3D cpustat[idx]; + } while (read_seqcount_retry(&kc->idle_sleeptime_seq, seq)); + +to_us: + do_div(idle, NSEC_PER_USEC); + + return idle; +} + +/** + * get_cpu_idle_time_us - get the total idle time of a CPU + * @cpu: CPU number to query + * @last_update_time: variable to store update time in. Do not update + * counters if NULL. + * + * Return the cumulative idle time (since boot) for a given + * CPU, in microseconds. Note that this is partially broken due to + * the counter of iowait tasks that can be remotely updated without + * any synchronization. Therefore it is possible to observe backward + * values within two consecutive reads. + * + * This time is measured via accounting rather than sampling, + * and is as accurate as ktime_get() is. + * + * Return: -1 if generic vtime is enabled, else total idle time of the @cpu + */ +u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) +{ + return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE, + !nr_iowait_cpu(cpu), last_update_time); +} +EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); + +/** + * get_cpu_iowait_time_us - get the total iowait time of a CPU + * @cpu: CPU number to query + * @last_update_time: variable to store update time in. Do not update + * counters if NULL. + * + * Return the cumulative iowait time (since boot) for a given + * CPU, in microseconds. Note this is partially broken due to + * the counter of iowait tasks that can be remotely updated without + * any synchronization. Therefore it is possible to observe backward + * values within two consecutive reads. + * + * This time is measured via accounting rather than sampling, + * and is as accurate as ktime_get() is. + * + * Return: -1 if generic vtime is enabled, else total iowait time of @cpu + */ +u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) +{ + return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT, + nr_iowait_cpu(cpu), last_update_time); +} +EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); + #endif /* CONFIG_NO_HZ_COMMON */ =20 /* diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 2b58cdc326c3..aa36c8d218e2 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -746,125 +746,6 @@ static void tick_nohz_update_jiffies(ktime_t now) touch_softlockup_watchdog_sched(); } =20 -static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now) -{ - u64 *cpustat =3D kcpustat_this_cpu->cpustat; - ktime_t delta; - - if (vtime_generic_enabled_this_cpu()) - return; - - if (WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE))) - return; - - delta =3D ktime_sub(now, ts->idle_entrytime); - - write_seqcount_begin(&ts->idle_sleeptime_seq); - if (nr_iowait_cpu(smp_processor_id()) > 0) - cpustat[CPUTIME_IOWAIT] =3D ktime_add(cpustat[CPUTIME_IOWAIT], delta); - else - cpustat[CPUTIME_IDLE] =3D ktime_add(cpustat[CPUTIME_IDLE], delta); - - ts->idle_entrytime =3D now; - tick_sched_flag_clear(ts, TS_FLAG_IDLE_ACTIVE); - write_seqcount_end(&ts->idle_sleeptime_seq); - - sched_clock_idle_wakeup_event(); -} - -static void tick_nohz_start_idle(struct tick_sched *ts) -{ - if (vtime_generic_enabled_this_cpu()) - return; - - write_seqcount_begin(&ts->idle_sleeptime_seq); - ts->idle_entrytime =3D ktime_get(); - tick_sched_flag_set(ts, TS_FLAG_IDLE_ACTIVE); - write_seqcount_end(&ts->idle_sleeptime_seq); - sched_clock_idle_sleep_event(); -} - -static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, - bool compute_delta, u64 *last_update_time) -{ - struct tick_sched *ts =3D &per_cpu(tick_cpu_sched, cpu); - u64 *cpustat =3D kcpustat_cpu(cpu).cpustat; - ktime_t now, idle; - unsigned int seq; - - now =3D ktime_get(); - if (last_update_time) - *last_update_time =3D ktime_to_us(now); - - if (vtime_generic_enabled_cpu(cpu)) { - idle =3D kcpustat_field(idx, cpu); - return ktime_to_us(idle); - } - - do { - seq =3D read_seqcount_begin(&ts->idle_sleeptime_seq); - - if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE) && compute_delta) { - ktime_t delta =3D ktime_sub(now, ts->idle_entrytime); - - idle =3D ktime_add(cpustat[idx], delta); - } else { - idle =3D cpustat[idx]; - } - } while (read_seqcount_retry(&ts->idle_sleeptime_seq, seq)); - - return ktime_to_us(idle); - -} - -/** - * get_cpu_idle_time_us - get the total idle time of a CPU - * @cpu: CPU number to query - * @last_update_time: variable to store update time in. Do not update - * counters if NULL. - * - * Return the cumulative idle time (since boot) for a given - * CPU, in microseconds. Note that this is partially broken due to - * the counter of iowait tasks that can be remotely updated without - * any synchronization. Therefore it is possible to observe backward - * values within two consecutive reads. - * - * This time is measured via accounting rather than sampling, - * and is as accurate as ktime_get() is. - * - * Return: -1 if generic vtime is enabled, else total idle time of the @cpu - */ -u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) -{ - return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE, - !nr_iowait_cpu(cpu), last_update_time); -} -EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); - -/** - * get_cpu_iowait_time_us - get the total iowait time of a CPU - * @cpu: CPU number to query - * @last_update_time: variable to store update time in. Do not update - * counters if NULL. - * - * Return the cumulative iowait time (since boot) for a given - * CPU, in microseconds. Note this is partially broken due to - * the counter of iowait tasks that can be remotely updated without - * any synchronization. Therefore it is possible to observe backward - * values within two consecutive reads. - * - * This time is measured via accounting rather than sampling, - * and is as accurate as ktime_get() is. - * - * Return: -1 if generic vtime is enabled, else total iowait time of @cpu - */ -u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) -{ - return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT, - nr_iowait_cpu(cpu), last_update_time); -} -EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); - static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) { hrtimer_cancel(&ts->sched_timer); @@ -1272,6 +1153,20 @@ void tick_nohz_idle_retain_tick(void) tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched)); } =20 +static void tick_nohz_clock_sleep(struct tick_sched *ts) +{ + tick_sched_flag_set(ts, TS_FLAG_IDLE_ACTIVE); + sched_clock_idle_sleep_event(); +} + +static void tick_nohz_clock_wakeup(struct tick_sched *ts) +{ + if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE)) { + tick_sched_flag_clear(ts, TS_FLAG_IDLE_ACTIVE); + sched_clock_idle_wakeup_event(); + } +} + /** * tick_nohz_idle_enter - prepare for entering idle on the current CPU * @@ -1286,12 +1181,11 @@ void tick_nohz_idle_enter(void) local_irq_disable(); =20 ts =3D this_cpu_ptr(&tick_cpu_sched); - WARN_ON_ONCE(ts->timer_expires_base); - tick_sched_flag_set(ts, TS_FLAG_INIDLE); - kcpustat_dyntick_start(); - tick_nohz_start_idle(ts); + ts->idle_entrytime =3D ktime_get(); + kcpustat_dyntick_start(ts->idle_entrytime); + tick_nohz_clock_sleep(ts); =20 local_irq_enable(); } @@ -1319,10 +1213,13 @@ void tick_nohz_irq_exit(void) { struct tick_sched *ts =3D this_cpu_ptr(&tick_cpu_sched); =20 - if (tick_sched_flag_test(ts, TS_FLAG_INIDLE)) - tick_nohz_start_idle(ts); - else + if (tick_sched_flag_test(ts, TS_FLAG_INIDLE)) { + ts->idle_entrytime =3D ktime_get(); + kcpustat_irq_exit(ts->idle_entrytime); + tick_nohz_clock_sleep(ts); + } else { tick_nohz_full_update_tick(ts); + } } =20 /** @@ -1467,11 +1364,11 @@ void tick_nohz_idle_exit(void) now =3D ktime_get(); =20 if (idle_active) - tick_nohz_stop_idle(ts, now); + tick_nohz_clock_wakeup(ts); =20 if (tick_stopped) tick_nohz_idle_update_tick(ts, now); - kcpustat_dyntick_stop(); + kcpustat_dyntick_stop(now); =20 local_irq_enable(); } @@ -1527,9 +1424,14 @@ static inline void tick_nohz_irq_enter(void) =20 if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED | TS_FLAG_IDLE_ACTIVE)) return; + now =3D ktime_get(); - if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE)) - tick_nohz_stop_idle(ts, now); + + if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE)) { + tick_nohz_clock_wakeup(ts); + kcpustat_irq_enter(now); + } + /* * If all CPUs are idle we may need to update a stale jiffies value. * Note nohz_full is a special case: a timekeeper is guaranteed to stay --=20 2.51.1 From nobody Sat Feb 7 21:14:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1E3CC3EFD1F; Fri, 6 Feb 2026 14:24:17 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387857; cv=none; b=LJvY9WkZFvqG9elYdeRsKNPItM4SI4L+9MEyp/JfisG5EQ86jlBe/vz0XYlnsWxAexT/FfabeV6RjWDeV5x1Kw9/NmpubhvbfKYff9WRVRyvAxAKg21pVr82mg9EwjDN/FpmoO/YtJQR7XdfFEXeYbvijAQTmhnFVQTx3im9upk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387857; c=relaxed/simple; bh=+EcaS9wpjAI5nYExsBELcaZu8W8ohxG84dL93BxRlIw=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=JoO0hf3m6RmlUG1o7Bgx3qFojmj3jNEO/0sKKslU+XQoKnlF+fkOK0QwSTH/nnIKdaaEHFAUTs2it/tXfLstALmc5N+rHHidMsRSIgG3INxq7Dam/AayNB7mqtKS8w6xT1v4prP3LGiBe2NVPvX/6Mr1YjjWVkNQdWud5JM/7S4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=GGt7uhGg; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="GGt7uhGg" Received: by smtp.kernel.org (Postfix) with ESMTPSA id B39C4C19421; Fri, 6 Feb 2026 14:24:09 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1770387857; bh=+EcaS9wpjAI5nYExsBELcaZu8W8ohxG84dL93BxRlIw=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=GGt7uhGgdfn/qTXYNfVirakaELJhXF1cQom9lCy+xWNR1NRHpLbDYvvRVJxyQdfIO UTt6ynULkVtYtWROVD1Vam+xUXFqff8GWwTEgIs+OXx96Ad9LSn1eRszqA8NLDypdC /9o1J8IT+0hjlkLxFvhgut5S63r3IyCR9AwNosfJ7ZJhHSgqzghAUFrZYhN18Ntu5j +sR3iRCTUXtJmujbbURmvCNYZSoCb0/bt9UGoW3pKBYPB/S1EbCNv6Q5FrP0C/d1YU yjWz5AVtQXxK40Ps91hx2GeC0ai6KNMpnmCDnzTkKCOjb8SjoXRQZKORvKBdB4Sg5H 0DqwO4r8eBmmw== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Christophe Leroy (CS GROUP)" , "Rafael J. Wysocki" , Alexander Gordeev , Anna-Maria Behnsen , Ben Segall , Boqun Feng , Christian Borntraeger , Dietmar Eggemann , Heiko Carstens , Ingo Molnar , Jan Kiszka , Joel Fernandes , Juri Lelli , Kieran Bingham , Madhavan Srinivasan , Mel Gorman , Michael Ellerman , Neeraj Upadhyay , Nicholas Piggin , "Paul E . McKenney" , Peter Zijlstra , Steven Rostedt , Sven Schnelle , Thomas Gleixner , Uladzislau Rezki , Valentin Schneider , Vasily Gorbik , Vincent Guittot , Viresh Kumar , Xin Zhao , linux-pm@vger.kernel.org, linux-s390@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, Shrikanth Hegde Subject: [PATCH 10/15] tick/sched: Remove unused fields Date: Fri, 6 Feb 2026 15:22:40 +0100 Message-ID: <20260206142245.58987-11-frederic@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20260206142245.58987-1-frederic@kernel.org> References: <20260206142245.58987-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Remove fields after the dyntick-idle cputime migration to scheduler code. Signed-off-by: Frederic Weisbecker --- kernel/time/tick-sched.h | 12 ------------ kernel/time/timer_list.c | 6 +----- scripts/gdb/linux/timerlist.py | 4 ---- 3 files changed, 1 insertion(+), 21 deletions(-) diff --git a/kernel/time/tick-sched.h b/kernel/time/tick-sched.h index b4a7822f495d..79b9252047b1 100644 --- a/kernel/time/tick-sched.h +++ b/kernel/time/tick-sched.h @@ -44,9 +44,7 @@ struct tick_device { * to resume the tick timer operation in the timeline * when the CPU returns from nohz sleep. * @next_tick: Next tick to be fired when in dynticks mode. - * @idle_jiffies: jiffies at the entry to idle for idle time accounting * @idle_waketime: Time when the idle was interrupted - * @idle_sleeptime_seq: sequence counter for data consistency * @idle_entrytime: Time when the idle call was entered * @last_jiffies: Base jiffies snapshot when next event was last computed * @timer_expires_base: Base time clock monotonic for @timer_expires @@ -55,9 +53,6 @@ struct tick_device { * @idle_expires: Next tick in idle, for debugging purpose only * @idle_calls: Total number of idle calls * @idle_sleeps: Number of idle calls, where the sched tick was stopped - * @idle_exittime: Time when the idle state was left - * @idle_sleeptime: Sum of the time slept in idle with sched tick stopped - * @iowait_sleeptime: Sum of the time slept in idle with sched tick stoppe= d, with IO outstanding * @tick_dep_mask: Tick dependency mask - is set, if someone needs the tick * @check_clocks: Notification mechanism about clocksource changes */ @@ -73,12 +68,10 @@ struct tick_sched { struct hrtimer sched_timer; ktime_t last_tick; ktime_t next_tick; - unsigned long idle_jiffies; ktime_t idle_waketime; unsigned int got_idle_tick; =20 /* Idle entry */ - seqcount_t idle_sleeptime_seq; ktime_t idle_entrytime; =20 /* Tick stop */ @@ -90,11 +83,6 @@ struct tick_sched { unsigned long idle_calls; unsigned long idle_sleeps; =20 - /* Idle exit */ - ktime_t idle_exittime; - ktime_t idle_sleeptime; - ktime_t iowait_sleeptime; - /* Full dynticks handling */ atomic_t tick_dep_mask; =20 diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c index 488e47e96e93..e77b512e8597 100644 --- a/kernel/time/timer_list.c +++ b/kernel/time/timer_list.c @@ -154,14 +154,10 @@ static void print_cpu(struct seq_file *m, int cpu, u6= 4 now) P_flag(highres, TS_FLAG_HIGHRES); P_ns(last_tick); P_flag(tick_stopped, TS_FLAG_STOPPED); - P(idle_jiffies); P(idle_calls); P(idle_sleeps); P_ns(idle_entrytime); P_ns(idle_waketime); - P_ns(idle_exittime); - P_ns(idle_sleeptime); - P_ns(iowait_sleeptime); P(last_jiffies); P(next_timer); P_ns(idle_expires); @@ -258,7 +254,7 @@ static void timer_list_show_tickdevices_header(struct s= eq_file *m) =20 static inline void timer_list_header(struct seq_file *m, u64 now) { - SEQ_printf(m, "Timer List Version: v0.10\n"); + SEQ_printf(m, "Timer List Version: v0.11\n"); SEQ_printf(m, "HRTIMER_MAX_CLOCK_BASES: %d\n", HRTIMER_MAX_CLOCK_BASES); SEQ_printf(m, "now at %Ld nsecs\n", (unsigned long long)now); SEQ_printf(m, "\n"); diff --git a/scripts/gdb/linux/timerlist.py b/scripts/gdb/linux/timerlist.py index ccc24d30de80..c14ce55674c9 100644 --- a/scripts/gdb/linux/timerlist.py +++ b/scripts/gdb/linux/timerlist.py @@ -90,14 +90,10 @@ def print_cpu(hrtimer_bases, cpu, max_clock_bases): text +=3D f" .{'nohz':15s}: {int(bool(ts['flags'] & TS_FLAG_N= OHZ))}\n" text +=3D f" .{'last_tick':15s}: {ts['last_tick']}\n" text +=3D f" .{'tick_stopped':15s}: {int(bool(ts['flags'] & T= S_FLAG_STOPPED))}\n" - text +=3D f" .{'idle_jiffies':15s}: {ts['idle_jiffies']}\n" text +=3D f" .{'idle_calls':15s}: {ts['idle_calls']}\n" text +=3D f" .{'idle_sleeps':15s}: {ts['idle_sleeps']}\n" text +=3D f" .{'idle_entrytime':15s}: {ts['idle_entrytime']} = nsecs\n" text +=3D f" .{'idle_waketime':15s}: {ts['idle_waketime']} ns= ecs\n" - text +=3D f" .{'idle_exittime':15s}: {ts['idle_exittime']} ns= ecs\n" - text +=3D f" .{'idle_sleeptime':15s}: {ts['idle_sleeptime']} = nsecs\n" - text +=3D f" .{'iowait_sleeptime':15s}: {ts['iowait_sleeptime= ']} nsecs\n" text +=3D f" .{'last_jiffies':15s}: {ts['last_jiffies']}\n" text +=3D f" .{'next_timer':15s}: {ts['next_timer']}\n" text +=3D f" .{'idle_expires':15s}: {ts['idle_expires']} nsec= s\n" --=20 2.51.1 From nobody Sat Feb 7 21:14:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2B0AB3EFD1F; Fri, 6 Feb 2026 14:24:24 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387865; cv=none; b=DxuPm9nG5jSF/Zng/vhG8wVRngf/u+MMvHUxQ6giDydWQF3zgOZD1tLTqtIY+j2FPnEJD9ZvWknRL4ML/tXu4fEDbtgy72ixZ3A4eZetQu7WBBiYOS94aYWrkB2g5braKZRf7jk5kEgdLPAXfDmg8jpM8UGe7FyZodkORwmCVEQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387865; c=relaxed/simple; bh=YMOHPjr/Wf/QWbFo/ZB5fJoqYZ8Q3rjy/SRpdY/7S30=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=WapJsFSCVzZ+UhqSvlquFuHjAss1O7OtVgjy23G0SuLEcA4rxOXU8sULGT0yFMX6AyClIMphOPKi4stOicaoHhu59rHfn/h2at0jXdH1Pb7SxW1Emd7G6vDwdFLGWdFigzgC+TGZ9fgjWL03UxijnbukKVHpbMGELrajiZko9QI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=Eyi386IP; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="Eyi386IP" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 6A780C2BC87; Fri, 6 Feb 2026 14:24:17 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1770387864; bh=YMOHPjr/Wf/QWbFo/ZB5fJoqYZ8Q3rjy/SRpdY/7S30=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Eyi386IPJs0ew93oNS6xfd09R1UlajX6a5KrKlsJtSf9mcz+6T9YaurId5Htp1MpM WfPcsKebHrzcgsjgE0wdSaJxcL5U22ixrgJXmJOUk4kOTrUiLwZqSH2+HSa1hX39PA TZnUSqQFUpP5k0z9UYebJ5CVWrFmZFi7DO3gefzrpsC0SOp2hl646yWhU4rhIC9f9k skSWd04NnZEEVztfQm/uZZ6OFk3onL1UtKxMAt694F/qSu98hH8tX/Hpfh1UrZjWaa rCknHJn0J9KvsyBu2m3Jgv0rO4T+0JHWddOoF3tCQ1uaB32OFIQCHv73caXhwphMV0 UHfCvUQvH8WUw== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Christophe Leroy (CS GROUP)" , "Rafael J. Wysocki" , Alexander Gordeev , Anna-Maria Behnsen , Ben Segall , Boqun Feng , Christian Borntraeger , Dietmar Eggemann , Heiko Carstens , Ingo Molnar , Jan Kiszka , Joel Fernandes , Juri Lelli , Kieran Bingham , Madhavan Srinivasan , Mel Gorman , Michael Ellerman , Neeraj Upadhyay , Nicholas Piggin , "Paul E . McKenney" , Peter Zijlstra , Steven Rostedt , Sven Schnelle , Thomas Gleixner , Uladzislau Rezki , Valentin Schneider , Vasily Gorbik , Vincent Guittot , Viresh Kumar , Xin Zhao , linux-pm@vger.kernel.org, linux-s390@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, Shrikanth Hegde Subject: [PATCH 11/15] tick/sched: Account tickless idle cputime only when tick is stopped Date: Fri, 6 Feb 2026 15:22:41 +0100 Message-ID: <20260206142245.58987-12-frederic@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20260206142245.58987-1-frederic@kernel.org> References: <20260206142245.58987-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" There is no real point in switching to dyntick-idle cputime accounting mode if the tick is not actually stopped. This just adds overhead, notably fetching the GTOD, on each idle exit and each idle IRQ entry for no reason during short idle trips. Signed-off-by: Frederic Weisbecker --- kernel/time/tick-sched.c | 44 ++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index aa36c8d218e2..bceed0c4dd2c 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -1141,8 +1141,10 @@ void tick_nohz_idle_stop_tick(void) ts->idle_sleeps++; ts->idle_expires =3D expires; =20 - if (!was_stopped && tick_sched_flag_test(ts, TS_FLAG_STOPPED)) + if (!was_stopped && tick_sched_flag_test(ts, TS_FLAG_STOPPED)) { + kcpustat_dyntick_start(ts->idle_entrytime); nohz_balance_enter_idle(cpu); + } } else { tick_nohz_retain_tick(ts); } @@ -1184,7 +1186,6 @@ void tick_nohz_idle_enter(void) WARN_ON_ONCE(ts->timer_expires_base); tick_sched_flag_set(ts, TS_FLAG_INIDLE); ts->idle_entrytime =3D ktime_get(); - kcpustat_dyntick_start(ts->idle_entrytime); tick_nohz_clock_sleep(ts); =20 local_irq_enable(); @@ -1214,9 +1215,10 @@ void tick_nohz_irq_exit(void) struct tick_sched *ts =3D this_cpu_ptr(&tick_cpu_sched); =20 if (tick_sched_flag_test(ts, TS_FLAG_INIDLE)) { - ts->idle_entrytime =3D ktime_get(); - kcpustat_irq_exit(ts->idle_entrytime); tick_nohz_clock_sleep(ts); + ts->idle_entrytime =3D ktime_get(); + if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) + kcpustat_irq_exit(ts->idle_entrytime); } else { tick_nohz_full_update_tick(ts); } @@ -1317,8 +1319,11 @@ void tick_nohz_idle_restart_tick(void) { struct tick_sched *ts =3D this_cpu_ptr(&tick_cpu_sched); =20 - if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) - tick_nohz_restart_sched_tick(ts, ktime_get()); + if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) { + ktime_t now =3D ktime_get(); + kcpustat_dyntick_stop(now); + tick_nohz_restart_sched_tick(ts, now); + } } =20 static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now) @@ -1348,7 +1353,6 @@ static void tick_nohz_idle_update_tick(struct tick_sc= hed *ts, ktime_t now) void tick_nohz_idle_exit(void) { struct tick_sched *ts =3D this_cpu_ptr(&tick_cpu_sched); - bool idle_active, tick_stopped; ktime_t now; =20 local_irq_disable(); @@ -1357,18 +1361,13 @@ void tick_nohz_idle_exit(void) WARN_ON_ONCE(ts->timer_expires_base); =20 tick_sched_flag_clear(ts, TS_FLAG_INIDLE); - idle_active =3D tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE); - tick_stopped =3D tick_sched_flag_test(ts, TS_FLAG_STOPPED); + tick_nohz_clock_wakeup(ts); =20 - if (idle_active || tick_stopped) + if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) { now =3D ktime_get(); - - if (idle_active) - tick_nohz_clock_wakeup(ts); - - if (tick_stopped) + kcpustat_dyntick_stop(now); tick_nohz_idle_update_tick(ts, now); - kcpustat_dyntick_stop(now); + } =20 local_irq_enable(); } @@ -1422,15 +1421,13 @@ static inline void tick_nohz_irq_enter(void) struct tick_sched *ts =3D this_cpu_ptr(&tick_cpu_sched); ktime_t now; =20 - if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED | TS_FLAG_IDLE_ACTIVE)) + tick_nohz_clock_wakeup(ts); + + if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED)) return; =20 now =3D ktime_get(); - - if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE)) { - tick_nohz_clock_wakeup(ts); - kcpustat_irq_enter(now); - } + kcpustat_irq_enter(now); =20 /* * If all CPUs are idle we may need to update a stale jiffies value. @@ -1439,8 +1436,7 @@ static inline void tick_nohz_irq_enter(void) * rare case (typically stop machine). So we must make sure we have a * last resort. */ - if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) - tick_nohz_update_jiffies(now); + tick_nohz_update_jiffies(now); } =20 #else --=20 2.51.1 From nobody Sat Feb 7 21:14:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id BEAFA3EFD1F; Fri, 6 Feb 2026 14:24:32 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387872; cv=none; b=Hv//2LsZ+3w7jWzMoFCtF1R7c8m7dhSTRqzoBQOsn1aH9ePKT3P6j3jy+CFIXsYQyUYnPwKjGYxJK6Aj6mOzPwTZmdpQXdKAenU+CmzEe37+Fd8HxXdCV7ZBKV3BG72g7ps6/NMcwdz5GHGeb3cLP5CpiS1YZRIEn/qinQh/ryQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387872; c=relaxed/simple; bh=//EtvZKobCna72857H6n33BD/elKdPC5DUmkUub5hKc=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=D7ofJHMb1cAhF2PWKAAsWXbuBptr9VWineXFD4PPRuwuwlcbYoMCfuUroQSZD+G+xfoCQ7AaGsw2WOTS/lhAPmBsKW4AXCQLoiVS754CUcB8nm/pYgyImmzyuSmz9mHR3O7qXBQIlaRCKV1yB9xXArApYjTlwOKq+uIGpo2jr7w= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=jxn/Mj8d; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="jxn/Mj8d" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 3DEBDC116C6; Fri, 6 Feb 2026 14:24:25 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1770387872; bh=//EtvZKobCna72857H6n33BD/elKdPC5DUmkUub5hKc=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=jxn/Mj8dW/tra7FN7+Oh0XhZYfX0wbnou8S3FdxKUXiEMky/8nSQDiLRjlzM/rUUO te0/HNoMSmI13uUW/ooM9bmU9XpsxfQz4TN3tesCUnifM+Yu2KNpCdxGKaXDj6b78n a4R0Y0hhfgn8WJbHFteRCXJyW3rlzkMU5Nwu8RETbokq3a5pSagIqo0Ya/6hY4oMXy W33IgloZg45tGo/OezgaM8d0hFIpf89sG43oO711GotnRxCB9Cp260IaDW98W9duu1 /TFJ4VLUlrx7eD0Nng1scT2+/7DfVKdQF6TjjgQFr2CJvsirtt4VGEDZq5h6ZasAiM kMIq1rWI8aNJQ== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Christophe Leroy (CS GROUP)" , "Rafael J. Wysocki" , Alexander Gordeev , Anna-Maria Behnsen , Ben Segall , Boqun Feng , Christian Borntraeger , Dietmar Eggemann , Heiko Carstens , Ingo Molnar , Jan Kiszka , Joel Fernandes , Juri Lelli , Kieran Bingham , Madhavan Srinivasan , Mel Gorman , Michael Ellerman , Neeraj Upadhyay , Nicholas Piggin , "Paul E . McKenney" , Peter Zijlstra , Steven Rostedt , Sven Schnelle , Thomas Gleixner , Uladzislau Rezki , Valentin Schneider , Vasily Gorbik , Vincent Guittot , Viresh Kumar , Xin Zhao , linux-pm@vger.kernel.org, linux-s390@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, Shrikanth Hegde Subject: [PATCH 12/15] tick/sched: Consolidate idle time fetching APIs Date: Fri, 6 Feb 2026 15:22:42 +0100 Message-ID: <20260206142245.58987-13-frederic@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20260206142245.58987-1-frederic@kernel.org> References: <20260206142245.58987-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Fetching the idle cputime is available through a variety of accessors all over the place depending on the different accounting flavours and needs: - idle vtime generic accounting can be accessed by kcpustat_field(), kcpustat_cpu_fetch(), get_idle/iowait_time() and get_cpu_idle/iowait_time_us() - dynticks-idle accounting can only be accessed by get_idle/iowait_time() or get_cpu_idle/iowait_time_us() - CONFIG_NO_HZ_COMMON=3Dn idle accounting can be accessed by kcpustat_field= () kcpustat_cpu_fetch(), or get_idle/iowait_time() but not by get_cpu_idle/iowait_time_us() Moreover get_idle/iowait_time() relies on get_cpu_idle/iowait_time_us() with a non-sensical conversion to microseconds and back to nanoseconds on the way. Start consolidating the APIs with removing get_idle/iowait_time() and make kcpustat_field() and kcpustat_cpu_fetch() work for all cases. Signed-off-by: Frederic Weisbecker --- fs/proc/stat.c | 40 +++--------------------- fs/proc/uptime.c | 8 ++--- include/linux/kernel_stat.h | 34 ++++++++++++++++++--- kernel/sched/cputime.c | 61 ++++++++++++++++++++++++------------- 4 files changed, 76 insertions(+), 67 deletions(-) diff --git a/fs/proc/stat.c b/fs/proc/stat.c index 8b444e862319..c00468a83f64 100644 --- a/fs/proc/stat.c +++ b/fs/proc/stat.c @@ -22,38 +22,6 @@ #define arch_irq_stat() 0 #endif =20 -u64 get_idle_time(struct kernel_cpustat *kcs, int cpu) -{ - u64 idle, idle_usecs =3D -1ULL; - - if (cpu_online(cpu)) - idle_usecs =3D get_cpu_idle_time_us(cpu, NULL); - - if (idle_usecs =3D=3D -1ULL) - /* !NO_HZ or cpu offline so we can rely on cpustat.idle */ - idle =3D kcs->cpustat[CPUTIME_IDLE]; - else - idle =3D idle_usecs * NSEC_PER_USEC; - - return idle; -} - -static u64 get_iowait_time(struct kernel_cpustat *kcs, int cpu) -{ - u64 iowait, iowait_usecs =3D -1ULL; - - if (cpu_online(cpu)) - iowait_usecs =3D get_cpu_iowait_time_us(cpu, NULL); - - if (iowait_usecs =3D=3D -1ULL) - /* !NO_HZ or cpu offline so we can rely on cpustat.iowait */ - iowait =3D kcs->cpustat[CPUTIME_IOWAIT]; - else - iowait =3D iowait_usecs * NSEC_PER_USEC; - - return iowait; -} - static void show_irq_gap(struct seq_file *p, unsigned int gap) { static const char zeros[] =3D " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0"; @@ -105,8 +73,8 @@ static int show_stat(struct seq_file *p, void *v) user +=3D cpustat[CPUTIME_USER]; nice +=3D cpustat[CPUTIME_NICE]; system +=3D cpustat[CPUTIME_SYSTEM]; - idle +=3D get_idle_time(&kcpustat, i); - iowait +=3D get_iowait_time(&kcpustat, i); + idle +=3D cpustat[CPUTIME_IDLE]; + iowait +=3D cpustat[CPUTIME_IOWAIT]; irq +=3D cpustat[CPUTIME_IRQ]; softirq +=3D cpustat[CPUTIME_SOFTIRQ]; steal +=3D cpustat[CPUTIME_STEAL]; @@ -146,8 +114,8 @@ static int show_stat(struct seq_file *p, void *v) user =3D cpustat[CPUTIME_USER]; nice =3D cpustat[CPUTIME_NICE]; system =3D cpustat[CPUTIME_SYSTEM]; - idle =3D get_idle_time(&kcpustat, i); - iowait =3D get_iowait_time(&kcpustat, i); + idle =3D cpustat[CPUTIME_IDLE]; + iowait =3D cpustat[CPUTIME_IOWAIT]; irq =3D cpustat[CPUTIME_IRQ]; softirq =3D cpustat[CPUTIME_SOFTIRQ]; steal =3D cpustat[CPUTIME_STEAL]; diff --git a/fs/proc/uptime.c b/fs/proc/uptime.c index b5343d209381..433aa947cd57 100644 --- a/fs/proc/uptime.c +++ b/fs/proc/uptime.c @@ -18,12 +18,8 @@ static int uptime_proc_show(struct seq_file *m, void *v) int i; =20 idle_nsec =3D 0; - for_each_possible_cpu(i) { - struct kernel_cpustat kcs; - - kcpustat_cpu_fetch(&kcs, i); - idle_nsec +=3D get_idle_time(&kcs, i); - } + for_each_possible_cpu(i) + idle_nsec +=3D kcpustat_field(CPUTIME_IDLE, i); =20 ktime_get_boottime_ts64(&uptime); timens_add_boottime(&uptime); diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index 9343353ac7a3..3680519d7b2c 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -110,32 +110,59 @@ extern void kcpustat_dyntick_start(u64 now); extern void kcpustat_dyntick_stop(u64 now); extern void kcpustat_irq_enter(u64 now); extern void kcpustat_irq_exit(u64 now); +extern u64 kcpustat_field_idle(int cpu); +extern u64 kcpustat_field_iowait(int cpu); =20 static inline bool kcpustat_idle_dyntick(void) { return __this_cpu_read(kernel_cpustat.idle_dyntick); } #else +static inline u64 kcpustat_field_idle(int cpu) +{ + return kcpustat_cpu(cpu).cpustat[CPUTIME_IDLE]; +} +static inline u64 kcpustat_field_iowait(int cpu) +{ + return kcpustat_cpu(cpu).cpustat[CPUTIME_IOWAIT]; +} + static inline bool kcpustat_idle_dyntick(void) { return false; } #endif /* CONFIG_NO_HZ_COMMON */ =20 +/* Fetch cputime values when vtime is disabled on a CPU */ +static inline u64 kcpustat_field_default(enum cpu_usage_stat usage, int cp= u) +{ + if (usage =3D=3D CPUTIME_IDLE) + return kcpustat_field_idle(cpu); + if (usage =3D=3D CPUTIME_IOWAIT) + return kcpustat_field_iowait(cpu); + return kcpustat_cpu(cpu).cpustat[usage]; +} + +static inline void kcpustat_cpu_fetch_default(struct kernel_cpustat *dst, = int cpu) +{ + *dst =3D kcpustat_cpu(cpu); + dst->cpustat[CPUTIME_IDLE] =3D kcpustat_field_idle(cpu); + dst->cpustat[CPUTIME_IOWAIT] =3D kcpustat_field_iowait(cpu); +} + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN extern u64 kcpustat_field(enum cpu_usage_stat usage, int cpu); extern void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu); #else static inline u64 kcpustat_field(enum cpu_usage_stat usage, int cpu) { - return kcpustat_cpu(cpu).cpustat[usage]; + return kcpustat_field_default(usage, cpu); } =20 static inline void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu) { - *dst =3D kcpustat_cpu(cpu); + kcpustat_cpu_fetch_default(dst, cpu); } - #endif /* !CONFIG_VIRT_CPU_ACCOUNTING_GEN */ =20 extern void account_user_time(struct task_struct *, u64); @@ -145,7 +172,6 @@ extern void account_system_index_time(struct task_struc= t *, u64, enum cpu_usage_stat); extern void account_steal_time(u64); extern void account_idle_time(u64); -extern u64 get_idle_time(struct kernel_cpustat *kcs, int cpu); =20 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE static inline void account_process_tick(struct task_struct *tsk, int user) diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index d2cad4d8dc10..057fdc00dbc6 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -476,24 +476,14 @@ void kcpustat_irq_exit(u64 now) kcpustat_idle_start(kc, now); } =20 -static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, - bool compute_delta, u64 *last_update_time) +static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx, + bool compute_delta, u64 now) { struct kernel_cpustat *kc =3D &kcpustat_cpu(cpu); u64 *cpustat =3D kc->cpustat; unsigned int seq; - ktime_t now; u64 idle; =20 - now =3D ktime_get(); - if (last_update_time) - *last_update_time =3D ktime_to_us(now); - - if (vtime_generic_enabled_cpu(cpu)) { - idle =3D kcpustat_field(idx, cpu); - goto to_us; - } - do { seq =3D read_seqcount_begin(&kc->idle_sleeptime_seq); =20 @@ -503,12 +493,42 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_us= age_stat idx, idle =3D cpustat[idx]; } while (read_seqcount_retry(&kc->idle_sleeptime_seq, seq)); =20 -to_us: - do_div(idle, NSEC_PER_USEC); - return idle; } =20 +u64 kcpustat_field_idle(int cpu) +{ + return kcpustat_field_dyntick(cpu, CPUTIME_IDLE, + !nr_iowait_cpu(cpu), ktime_get()); +} +EXPORT_SYMBOL_GPL(kcpustat_field_idle); + +u64 kcpustat_field_iowait(int cpu) +{ + return kcpustat_field_dyntick(cpu, CPUTIME_IOWAIT, + nr_iowait_cpu(cpu), ktime_get()); +} +EXPORT_SYMBOL_GPL(kcpustat_field_iowait); + +static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, + bool compute_delta, u64 *last_update_time) +{ + ktime_t now =3D ktime_get(); + u64 res; + + if (vtime_generic_enabled_cpu(cpu)) + res =3D kcpustat_field(idx, cpu); + else + res =3D kcpustat_field_dyntick(cpu, idx, compute_delta, now); + + do_div(res, NSEC_PER_USEC); + + if (last_update_time) + *last_update_time =3D res; + + return res; +} + /** * get_cpu_idle_time_us - get the total idle time of a CPU * @cpu: CPU number to query @@ -556,7 +576,6 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_ti= me) nr_iowait_cpu(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); - #endif /* CONFIG_NO_HZ_COMMON */ =20 /* @@ -1110,8 +1129,8 @@ u64 kcpustat_field(enum cpu_usage_stat usage, int cpu) struct rq *rq; int err; =20 - if (!vtime_accounting_enabled_cpu(cpu)) - return val; + if (!vtime_generic_enabled_cpu(cpu)) + return kcpustat_field_default(usage, cpu); =20 rq =3D cpu_rq(cpu); =20 @@ -1206,8 +1225,8 @@ void kcpustat_cpu_fetch(struct kernel_cpustat *dst, i= nt cpu) struct rq *rq; int err; =20 - if (!vtime_accounting_enabled_cpu(cpu)) { - *dst =3D *src; + if (!vtime_generic_enabled_cpu(cpu)) { + kcpustat_cpu_fetch_default(dst, cpu); return; } =20 @@ -1220,7 +1239,7 @@ void kcpustat_cpu_fetch(struct kernel_cpustat *dst, i= nt cpu) curr =3D rcu_dereference(rq->curr); if (WARN_ON_ONCE(!curr)) { rcu_read_unlock(); - *dst =3D *src; + kcpustat_cpu_fetch_default(dst, cpu); return; } =20 --=20 2.51.1 From nobody Sat Feb 7 21:14:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4C28A212542; Fri, 6 Feb 2026 14:24:40 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387880; cv=none; b=PZjS15xYOOZt5woJ7Anp8Etbh1sxqaDPKjNCWDeNzlQA/02ie8tvDL8LSJYzDDEGS8WI4xDsfpDQYiRtNX6FVKsW/S/hEh+lKPr0Qtvnc5HJkK/vaKpk/87aLOvIlB6BKtdpF6ibSZ2obhI+E5Q6B6bEaA+Iphj4GCcSpoWMyi4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387880; c=relaxed/simple; bh=zeMY7mxj0srGl3hWZZOQh0RLV4mh+9l/FD58tASJvfM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=kFPlmvDxxeYLF5+64utMCHYTD8Rvj9I01w8JDDD3jSEbYciwxYefwj5ho26JP+wfqXuY+1KBK6iUXR4qgQHcjRYxg9i6ntrs2u33G2AwpNVCSb6G9hqCpq4WwoiuPRofoDcUVqi98kUs+UT0pb4Hsr//X/5LjsJnaKAzdq/6Kps= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=hrOFa03K; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="hrOFa03K" Received: by smtp.kernel.org (Postfix) with ESMTPSA id C76B1C19423; Fri, 6 Feb 2026 14:24:32 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1770387880; bh=zeMY7mxj0srGl3hWZZOQh0RLV4mh+9l/FD58tASJvfM=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=hrOFa03KsXBbAi+G+laA6W2FdtL1OTf8jjVdPgDEkcHnh7AfmUqJP0xbQrIB56tXs vft78NXCJP3gTkKa3uTh43RjHo59/LIQoembb692h3MiH+ZgOo7vhkrMXEQO9yt5dk lSzzGiFbyhXunMNKDP9qeMp5RYCqQqDdrwAtPsnh7cAbpfTdoMzEwbz6z3WHYXRGEl TtmTgmdtN4Xm0/M5+l/7BSf9VFGKCdTaWtN8mxNBWWFCiG+hQrJglQvSH+gbJXON+J sjjpt303nYWXGsMy1k4EOV3R8Qo/kipjjWHi9OD7gmuM9qPF0Z6p9EkD2njRkV/6ow Ip8rw4TeBIsKw== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Christophe Leroy (CS GROUP)" , "Rafael J. Wysocki" , Alexander Gordeev , Anna-Maria Behnsen , Ben Segall , Boqun Feng , Christian Borntraeger , Dietmar Eggemann , Heiko Carstens , Ingo Molnar , Jan Kiszka , Joel Fernandes , Juri Lelli , Kieran Bingham , Madhavan Srinivasan , Mel Gorman , Michael Ellerman , Neeraj Upadhyay , Nicholas Piggin , "Paul E . McKenney" , Peter Zijlstra , Steven Rostedt , Sven Schnelle , Thomas Gleixner , Uladzislau Rezki , Valentin Schneider , Vasily Gorbik , Vincent Guittot , Viresh Kumar , Xin Zhao , linux-pm@vger.kernel.org, linux-s390@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, Shrikanth Hegde Subject: [PATCH 13/15] sched/cputime: Provide get_cpu_[idle|iowait]_time_us() off-case Date: Fri, 6 Feb 2026 15:22:43 +0100 Message-ID: <20260206142245.58987-14-frederic@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20260206142245.58987-1-frederic@kernel.org> References: <20260206142245.58987-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The last reason why get_cpu_idle/iowait_time_us() may return -1 now is if the config doesn't support nohz. The ad-hoc replacement solution by cpufreq is to compute jiffies minus the whole busy cputime. Although the intention should provide a coherent low resolution estimation of the idle and iowait time, the implementation is buggy because jiffies don't start at 0. Just provide instead a real get_cpu_[idle|iowait]_time_us() offcase. Signed-off-by: Frederic Weisbecker --- drivers/cpufreq/cpufreq.c | 29 +---------------------------- include/linux/kernel_stat.h | 3 +++ include/linux/tick.h | 4 ---- kernel/sched/cputime.c | 12 +++++++++--- 4 files changed, 13 insertions(+), 35 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 4472bb1ec83c..ecb9634cd06b 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -130,38 +130,11 @@ struct kobject *get_governor_parent_kobj(struct cpufr= eq_policy *policy) } EXPORT_SYMBOL_GPL(get_governor_parent_kobj); =20 -static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) -{ - struct kernel_cpustat kcpustat; - u64 cur_wall_time; - u64 idle_time; - u64 busy_time; - - cur_wall_time =3D jiffies64_to_nsecs(get_jiffies_64()); - - kcpustat_cpu_fetch(&kcpustat, cpu); - - busy_time =3D kcpustat.cpustat[CPUTIME_USER]; - busy_time +=3D kcpustat.cpustat[CPUTIME_SYSTEM]; - busy_time +=3D kcpustat.cpustat[CPUTIME_IRQ]; - busy_time +=3D kcpustat.cpustat[CPUTIME_SOFTIRQ]; - busy_time +=3D kcpustat.cpustat[CPUTIME_STEAL]; - busy_time +=3D kcpustat.cpustat[CPUTIME_NICE]; - - idle_time =3D cur_wall_time - busy_time; - if (wall) - *wall =3D div_u64(cur_wall_time, NSEC_PER_USEC); - - return div_u64(idle_time, NSEC_PER_USEC); -} - u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) { u64 idle_time =3D get_cpu_idle_time_us(cpu, io_busy ? wall : NULL); =20 - if (idle_time =3D=3D -1ULL) - return get_cpu_idle_time_jiffy(cpu, wall); - else if (!io_busy) + if (!io_busy) idle_time +=3D get_cpu_iowait_time_us(cpu, wall); =20 return idle_time; diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index 3680519d7b2c..512104b0ff49 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -133,6 +133,9 @@ static inline bool kcpustat_idle_dyntick(void) } #endif /* CONFIG_NO_HZ_COMMON */ =20 +extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time); +extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time); + /* Fetch cputime values when vtime is disabled on a CPU */ static inline u64 kcpustat_field_default(enum cpu_usage_stat usage, int cp= u) { diff --git a/include/linux/tick.h b/include/linux/tick.h index 738007d6f577..1cf4651f09ad 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -139,8 +139,6 @@ extern bool tick_nohz_idle_got_tick(void); extern ktime_t tick_nohz_get_next_hrtimer(void); extern ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next); extern unsigned long tick_nohz_get_idle_calls_cpu(int cpu); -extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time); -extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time); #else /* !CONFIG_NO_HZ_COMMON */ #define tick_nohz_enabled (0) static inline bool tick_nohz_is_active(void) { return false; } @@ -162,8 +160,6 @@ static inline ktime_t tick_nohz_get_sleep_length(ktime_= t *delta_next) *delta_next =3D TICK_NSEC; return *delta_next; } -static inline u64 get_cpu_idle_time_us(int cpu, u64 *unused) { return -1; } -static inline u64 get_cpu_iowait_time_us(int cpu, u64 *unused) { return -1= ; } #endif /* !CONFIG_NO_HZ_COMMON */ =20 /* diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 057fdc00dbc6..d588a4a50e57 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -509,6 +509,13 @@ u64 kcpustat_field_iowait(int cpu) nr_iowait_cpu(cpu), ktime_get()); } EXPORT_SYMBOL_GPL(kcpustat_field_iowait); +#else +static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx, + bool compute_delta, ktime_t now) +{ + return kcpustat_cpu(cpu).cpustat[idx]; +} +#endif /* CONFIG_NO_HZ_COMMON */ =20 static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, bool compute_delta, u64 *last_update_time) @@ -544,7 +551,7 @@ static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usag= e_stat idx, * This time is measured via accounting rather than sampling, * and is as accurate as ktime_get() is. * - * Return: -1 if generic vtime is enabled, else total idle time of the @cpu + * Return: total idle time of the @cpu */ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) { @@ -568,7 +575,7 @@ EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); * This time is measured via accounting rather than sampling, * and is as accurate as ktime_get() is. * - * Return: -1 if generic vtime is enabled, else total iowait time of @cpu + * Return: total iowait time of @cpu */ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) { @@ -576,7 +583,6 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_ti= me) nr_iowait_cpu(cpu), last_update_time); } EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); -#endif /* CONFIG_NO_HZ_COMMON */ =20 /* * Use precise platform statistics if available: --=20 2.51.1 From nobody Sat Feb 7 21:14:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 55EF73EF0A3; Fri, 6 Feb 2026 14:24:48 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387888; cv=none; b=QWv98v+xHpa+4p14bos4FjELmtMzOq9sOEVKRhAOpj5BJf8Vxvegokib7xDwyWe441pwN6K14T0NjLCdpfMZwp00nB7OhiFnfoLTLRLURAMlHtuWS5IRFbr1kla2rqsaoM6BFIpvS99ksnDlfqsAoQCAwS9opZ5XqemAlXexJxI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387888; c=relaxed/simple; bh=1a8G23aJDuAPjK+U9mKc7z3SxXZk+XIfUuQdAEaa5yE=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=tGN1wJ1tuqSdCl5DAwQWh4QRnygM1u/D//aSrYijRdpP8B7YE+3A9qAj/6JjtYazoMflUvxzov5G5G9j/2AWjp3hYNJxzKr0eQP1NWHurH8m70t6vz0I2+jRAkhJMc3JKAaLwXzQQ/cbCSBX4WSKlImQ7OqhMR+j5bZY5iaDUvo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=pyvvjMdH; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="pyvvjMdH" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 95E3AC19424; Fri, 6 Feb 2026 14:24:40 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1770387888; bh=1a8G23aJDuAPjK+U9mKc7z3SxXZk+XIfUuQdAEaa5yE=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=pyvvjMdH3hIzpy6aY54/LbDRrNWblh4tmt5l6YkK+fTkvt2lfzilz/SMkfwmrBCTa SdLlYowtBljkJ90q0VuzNj72diFgimT3w+ipxS9PtB8dSXBlzs4yP8OfTHShrVnSgr gIf5I4KVPgt8ITyocNbbNovtS1ms8WfLW13HT+LGC2E4XC8HPszlhfOeclfv2fM5jt dXpbutriVMuVXNcdLINPem/tHoz9daDrciUC3ojdNmBJ73HAVeG1sT48y3yzyYxiIE 55GK83AOK2oYqnpMm+ZYmC8nbn9afCH/50djVJ6jux706lusveFloWpWJaZDjViUcl ImbJ0ASRdg5Qw== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Christophe Leroy (CS GROUP)" , "Rafael J. Wysocki" , Alexander Gordeev , Anna-Maria Behnsen , Ben Segall , Boqun Feng , Christian Borntraeger , Dietmar Eggemann , Heiko Carstens , Ingo Molnar , Jan Kiszka , Joel Fernandes , Juri Lelli , Kieran Bingham , Madhavan Srinivasan , Mel Gorman , Michael Ellerman , Neeraj Upadhyay , Nicholas Piggin , "Paul E . McKenney" , Peter Zijlstra , Steven Rostedt , Sven Schnelle , Thomas Gleixner , Uladzislau Rezki , Valentin Schneider , Vasily Gorbik , Vincent Guittot , Viresh Kumar , Xin Zhao , linux-pm@vger.kernel.org, linux-s390@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, Shrikanth Hegde Subject: [PATCH 14/15] sched/cputime: Handle idle irqtime gracefully Date: Fri, 6 Feb 2026 15:22:44 +0100 Message-ID: <20260206142245.58987-15-frederic@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20260206142245.58987-1-frederic@kernel.org> References: <20260206142245.58987-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The dyntick-idle cputime accounting always assumes that IRQ time accounting is enabled and consequently stops elapsing the idle time during dyntick-idle IRQs. This doesn't mix up well with disabled IRQ time accounting because then idle IRQs become a cputime blind-spot. Also this feature is disabled on most configurations and the overhead of pausing dyntick-idle accounting while in idle IRQs could then be avoided. Fix the situation with conditionally pausing dyntick-idle accounting during idle IRQs only if neither native vtime (which does IRQ time accounting) nor generic IRQ time accounting are enabled. Also make sure that the accumulated IRQ time is not accidentally substracted from later accounting. Signed-off-by: Frederic Weisbecker --- kernel/sched/cputime.c | 24 +++++++++++++++++++++--- kernel/sched/sched.h | 1 + 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index d588a4a50e57..92fa2f037b6e 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -46,7 +46,8 @@ static void irqtime_account_delta(struct irqtime *irqtime= , u64 delta, u64_stats_update_begin(&irqtime->sync); cpustat[idx] +=3D delta; irqtime->total +=3D delta; - irqtime->tick_delta +=3D delta; + if (!irqtime->idle_dyntick) + irqtime->tick_delta +=3D delta; u64_stats_update_end(&irqtime->sync); } =20 @@ -81,6 +82,16 @@ void irqtime_account_irq(struct task_struct *curr, unsig= ned int offset) irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ); } =20 +static inline void irqtime_dyntick_start(void) +{ + __this_cpu_write(cpu_irqtime.idle_dyntick, true); +} + +static inline void irqtime_dyntick_stop(void) +{ + __this_cpu_write(cpu_irqtime.idle_dyntick, false); +} + static u64 irqtime_tick_accounted(u64 maxtime) { struct irqtime *irqtime =3D this_cpu_ptr(&cpu_irqtime); @@ -94,6 +105,9 @@ static u64 irqtime_tick_accounted(u64 maxtime) =20 #else /* !CONFIG_IRQ_TIME_ACCOUNTING: */ =20 +static inline void irqtime_dyntick_start(void) { } +static inline void irqtime_dyntick_stop(void) { } + static u64 irqtime_tick_accounted(u64 dummy) { return 0; @@ -444,6 +458,7 @@ void kcpustat_dyntick_stop(u64 now) WARN_ON_ONCE(!kc->idle_dyntick); kcpustat_idle_stop(kc, now); kc->idle_dyntick =3D false; + irqtime_dyntick_stop(); vtime_dyntick_stop(); steal_account_process_time(ULONG_MAX); } @@ -455,6 +470,7 @@ void kcpustat_dyntick_start(u64 now) =20 if (!vtime_generic_enabled_this_cpu()) { vtime_dyntick_start(); + irqtime_dyntick_start(); kc->idle_dyntick =3D true; kcpustat_idle_start(kc, now); } @@ -464,7 +480,8 @@ void kcpustat_irq_enter(u64 now) { struct kernel_cpustat *kc =3D kcpustat_this_cpu; =20 - if (!vtime_generic_enabled_this_cpu()) + if (!vtime_generic_enabled_this_cpu() && + (irqtime_enabled() || vtime_accounting_enabled_this_cpu())) kcpustat_idle_stop(kc, now); } =20 @@ -472,7 +489,8 @@ void kcpustat_irq_exit(u64 now) { struct kernel_cpustat *kc =3D kcpustat_this_cpu; =20 - if (!vtime_generic_enabled_this_cpu()) + if (!vtime_generic_enabled_this_cpu() && + (irqtime_enabled() || vtime_accounting_enabled_this_cpu())) kcpustat_idle_start(kc, now); } =20 diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index d30cca6870f5..cf677ff12b10 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -3307,6 +3307,7 @@ static inline void sched_core_tick(struct rq *rq) { } #ifdef CONFIG_IRQ_TIME_ACCOUNTING =20 struct irqtime { + bool idle_dyntick; u64 total; u64 tick_delta; u64 irq_start_time; --=20 2.51.1 From nobody Sat Feb 7 21:14:53 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1AE0F3EF0A3; Fri, 6 Feb 2026 14:24:56 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387896; cv=none; b=aGH+jJKADAl89IEyQTGeazgUs0DP8aKOFyXQM2y6LWbg5icKUeDQG+VwRqDiPBSSh89Bts4WAyhIaaeZqHdcmqE/t8GFAhEdcdjk0OttOysE8MkQ4YYwNtsNZ/7TFkilnGLxxGlXBXZj3R/b/OMKVmo+gIcKddTWANRLRvzMFM4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1770387896; c=relaxed/simple; bh=axD30NXxtXM5SUHtedxFjg90A77tUvhavW17z9RBWt4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=pkfLzW+GA/kWg+7lUVYrLH83pbn9ncwl5NVrql2WQ+v3jVODoh9z++9P0IGfPzP+nyJWRwHNUY8m24tuc7sZhhlVrp1AkRTR+ZUWOObNr3mU6o3NM4fY0sI2rHrKY7U7eiG71+9XvLpuIzi7HxKPtsJz6Gc9v1dvHl+SPiHvcY8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=JnGDPW3W; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="JnGDPW3W" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 684F4C116C6; Fri, 6 Feb 2026 14:24:48 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1770387896; bh=axD30NXxtXM5SUHtedxFjg90A77tUvhavW17z9RBWt4=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=JnGDPW3WKQhZalgvDsfBR/ftmpCbHIg1kQ0jgLm0KnWWKT+zYa23crSRw7f5YRXCM btDzqBLYR0Y0L4Cokn+/iY4Hvkk1LkJUyVU9RhKCw56WiwHT/UxGr31MDM0RACUhrV YlzTAPXCA4NQtRJKJWqG1kMqJAvuoV7QdX7PlPuOIxgYZxAAilXOiHkCS39U7SyzkR 6gVTqKVllraCVZZi/Pl0sOAUW9n76CxeszJ1hzlDS8iF/uHHjHHoHKmekWABTOsAVz eX8l3T5Ax5VY6BHzJDpKsCH3geoZS801SZSmFMGC/Ebc1IKNLQCaeHmaINBKfiIT72 /v4G2qyFhFU9A== From: Frederic Weisbecker To: LKML Cc: Frederic Weisbecker , "Christophe Leroy (CS GROUP)" , "Rafael J. Wysocki" , Alexander Gordeev , Anna-Maria Behnsen , Ben Segall , Boqun Feng , Christian Borntraeger , Dietmar Eggemann , Heiko Carstens , Ingo Molnar , Jan Kiszka , Joel Fernandes , Juri Lelli , Kieran Bingham , Madhavan Srinivasan , Mel Gorman , Michael Ellerman , Neeraj Upadhyay , Nicholas Piggin , "Paul E . McKenney" , Peter Zijlstra , Steven Rostedt , Sven Schnelle , Thomas Gleixner , Uladzislau Rezki , Valentin Schneider , Vasily Gorbik , Vincent Guittot , Viresh Kumar , Xin Zhao , linux-pm@vger.kernel.org, linux-s390@vger.kernel.org, linuxppc-dev@lists.ozlabs.org, Shrikanth Hegde Subject: [PATCH 15/15] sched/cputime: Handle dyntick-idle steal time correctly Date: Fri, 6 Feb 2026 15:22:45 +0100 Message-ID: <20260206142245.58987-16-frederic@kernel.org> X-Mailer: git-send-email 2.51.1 In-Reply-To: <20260206142245.58987-1-frederic@kernel.org> References: <20260206142245.58987-1-frederic@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" The dyntick-idle steal time is currently accounted when the tick restarts but the stolen idle time is not substracted from the idle time that was already accounted. This is to avoid observing the idle time going backward as the dyntick-idle cputime accessors can't reliably know in advance the stolen idle time. In order to maintain a forward progressing idle cputime while substracting idle steal time from it, keep track of the previously accounted idle stolen time and substract it from _later_ idle cputime accounting. Signed-off-by: Frederic Weisbecker --- include/linux/kernel_stat.h | 1 + kernel/sched/cputime.c | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index 512104b0ff49..24a54a6151ba 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -39,6 +39,7 @@ struct kernel_cpustat { bool idle_elapse; seqcount_t idle_sleeptime_seq; u64 idle_entrytime; + u64 idle_stealtime; #endif u64 cpustat[NR_STATS]; }; diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 92fa2f037b6e..7e79288eb327 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -424,19 +424,25 @@ static inline void irqtime_account_process_tick(struc= t task_struct *p, int user_ static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now) { u64 *cpustat =3D kc->cpustat; - u64 delta; + u64 delta, steal, steal_delta; =20 if (!kc->idle_elapse) return; =20 delta =3D now - kc->idle_entrytime; + steal =3D steal_account_process_time(delta); =20 write_seqcount_begin(&kc->idle_sleeptime_seq); + steal_delta =3D min_t(u64, kc->idle_stealtime, delta); + delta -=3D steal_delta; + kc->idle_stealtime -=3D steal_delta; + if (nr_iowait_cpu(smp_processor_id()) > 0) cpustat[CPUTIME_IOWAIT] +=3D delta; else cpustat[CPUTIME_IDLE] +=3D delta; =20 + kc->idle_stealtime +=3D steal; kc->idle_entrytime =3D now; kc->idle_elapse =3D false; write_seqcount_end(&kc->idle_sleeptime_seq); @@ -460,7 +466,6 @@ void kcpustat_dyntick_stop(u64 now) kc->idle_dyntick =3D false; irqtime_dyntick_stop(); vtime_dyntick_stop(); - steal_account_process_time(ULONG_MAX); } } =20 @@ -505,10 +510,14 @@ static u64 kcpustat_field_dyntick(int cpu, enum cpu_u= sage_stat idx, do { seq =3D read_seqcount_begin(&kc->idle_sleeptime_seq); =20 - if (kc->idle_elapse && compute_delta) - idle =3D cpustat[idx] + (now - kc->idle_entrytime); - else - idle =3D cpustat[idx]; + idle =3D cpustat[idx]; + + if (kc->idle_elapse && compute_delta) { + u64 delta =3D now - kc->idle_entrytime; + + delta -=3D min_t(u64, kc->idle_stealtime, delta); + idle +=3D delta; + } } while (read_seqcount_retry(&kc->idle_sleeptime_seq, seq)); =20 return idle; --=20 2.51.1