include/linux/timecounter.h | 33 +++++++++++++++++++++++++++++++-- kernel/time/timecounter.c | 35 ----------------------------------- 2 files changed, 31 insertions(+), 37 deletions(-)
New network transport protocols want NIC drivers to get hwtstamps
of all incoming packets, and possibly all outgoing packets.
Swift congestion control is used by good old TCP transport and is
our primary need for timecounter_cyc2time(). This will be upstreamed soon.
This means timecounter_cyc2time() can be called more than 100 million
times per second on a busy server.
Inlining timecounter_cyc2time() brings a 12 % improvement on a
UDP receive stress test on a 100Gbit NIC.
Note that FDO, LTO, PGO are unable to magically help for this
case, presumably because NIC drivers are almost exclusively shipped
as modules.
Add an unlikely() around the cc_cyc2ns_backwards() case,
even if FDO (when used) is able to take care of this optimization.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Link: https://research.google/pubs/swift-delay-is-simple-and-effective-for-congestion-control-in-the-datacenter/
Cc: Kevin Yang <yyd@google.com>
Cc: Willem de Bruijn <willemb@google.com>
Cc: Neal Cardwell <ncardwell@google.com>
Cc: Yuchung Cheng <ycheng@google.com>
---
include/linux/timecounter.h | 33 +++++++++++++++++++++++++++++++--
kernel/time/timecounter.c | 35 -----------------------------------
2 files changed, 31 insertions(+), 37 deletions(-)
diff --git a/include/linux/timecounter.h b/include/linux/timecounter.h
index dce03a5cafb7cfe61bd83304ef49f7721735da64..de21312ebed0f41946304f95f05cc625ea7f2a68 100644
--- a/include/linux/timecounter.h
+++ b/include/linux/timecounter.h
@@ -115,6 +115,16 @@ extern void timecounter_init(struct timecounter *tc,
*/
extern u64 timecounter_read(struct timecounter *tc);
+/*
+ * This is like cyclecounter_cyc2ns(), but it is used for computing a
+ * time previous to the time stored in the cycle counter.
+ */
+static inline u64 cc_cyc2ns_backwards(const struct cyclecounter *cc,
+ u64 cycles, u64 frac)
+{
+ return ((cycles * cc->mult) - frac) >> cc->shift;
+}
+
/**
* timecounter_cyc2time - convert a cycle counter to same
* time base as values returned by
@@ -131,7 +141,26 @@ extern u64 timecounter_read(struct timecounter *tc);
*
* Returns: cycle counter converted to nanoseconds since the initial time stamp
*/
-extern u64 timecounter_cyc2time(const struct timecounter *tc,
- u64 cycle_tstamp);
+static inline u64 timecounter_cyc2time(const struct timecounter *tc,
+ u64 cycle_tstamp)
+{
+ const struct cyclecounter *cc = tc->cc;
+ u64 delta = (cycle_tstamp - tc->cycle_last) & cc->mask;
+ u64 nsec = tc->nsec, frac = tc->frac;
+
+ /*
+ * Instead of always treating cycle_tstamp as more recent
+ * than tc->cycle_last, detect when it is too far in the
+ * future and treat it as old time stamp instead.
+ */
+ if (unlikely(delta > cc->mask / 2)) {
+ delta = (tc->cycle_last - cycle_tstamp) & cc->mask;
+ nsec -= cc_cyc2ns_backwards(cc, delta, frac);
+ } else {
+ nsec += cyclecounter_cyc2ns(cc, delta, tc->mask, &frac);
+ }
+
+ return nsec;
+}
#endif
diff --git a/kernel/time/timecounter.c b/kernel/time/timecounter.c
index 3d2a354cfe1c1b205aa960a748a76f8dd94335e2..2e64dbb6302d71d8b092d92dda0b71c99cdc053d 100644
--- a/kernel/time/timecounter.c
+++ b/kernel/time/timecounter.c
@@ -62,38 +62,3 @@ u64 timecounter_read(struct timecounter *tc)
}
EXPORT_SYMBOL_GPL(timecounter_read);
-/*
- * This is like cyclecounter_cyc2ns(), but it is used for computing a
- * time previous to the time stored in the cycle counter.
- */
-static u64 cc_cyc2ns_backwards(const struct cyclecounter *cc,
- u64 cycles, u64 mask, u64 frac)
-{
- u64 ns = (u64) cycles;
-
- ns = ((ns * cc->mult) - frac) >> cc->shift;
-
- return ns;
-}
-
-u64 timecounter_cyc2time(const struct timecounter *tc,
- u64 cycle_tstamp)
-{
- u64 delta = (cycle_tstamp - tc->cycle_last) & tc->cc->mask;
- u64 nsec = tc->nsec, frac = tc->frac;
-
- /*
- * Instead of always treating cycle_tstamp as more recent
- * than tc->cycle_last, detect when it is too far in the
- * future and treat it as old time stamp instead.
- */
- if (delta > tc->cc->mask / 2) {
- delta = (tc->cycle_last - cycle_tstamp) & tc->cc->mask;
- nsec -= cc_cyc2ns_backwards(tc->cc, delta, tc->mask, frac);
- } else {
- nsec += cyclecounter_cyc2ns(tc->cc, delta, tc->mask, &frac);
- }
-
- return nsec;
-}
-EXPORT_SYMBOL_GPL(timecounter_cyc2time);
--
2.52.0.107.ga0afd4fd5b-googOn Sat, Nov 29, 2025 at 5:57 PM Eric Dumazet <edumazet@google.com> wrote: > > New network transport protocols want NIC drivers to get hwtstamps > of all incoming packets, and possibly all outgoing packets. > > Swift congestion control is used by good old TCP transport and is > our primary need for timecounter_cyc2time(). This will be upstreamed soon. > > This means timecounter_cyc2time() can be called more than 100 million > times per second on a busy server. > > Inlining timecounter_cyc2time() brings a 12 % improvement on a > UDP receive stress test on a 100Gbit NIC. > > Note that FDO, LTO, PGO are unable to magically help for this > case, presumably because NIC drivers are almost exclusively shipped > as modules. > > Add an unlikely() around the cc_cyc2ns_backwards() case, > even if FDO (when used) is able to take care of this optimization. > > Signed-off-by: Eric Dumazet <edumazet@google.com> > Link: https://research.google/pubs/swift-delay-is-simple-and-effective-for-congestion-control-in-the-datacenter/ > Cc: Kevin Yang <yyd@google.com> > Cc: Willem de Bruijn <willemb@google.com> > Cc: Neal Cardwell <ncardwell@google.com> > Cc: Yuchung Cheng <ycheng@google.com> When I'm browsing the modification related to SWIFT, I noticed this patch seems to have not been merged yet? Thanks, Jason
On Tue, Jan 20 2026 at 16:18, Jason Xing wrote: > On Sat, Nov 29, 2025 at 5:57 PM Eric Dumazet <edumazet@google.com> wrote: > > When I'm browsing the modification related to SWIFT, I noticed this > patch seems to have not been merged yet? It's in the tip tree and scheduled for the next merge window.
On Tue, Jan 20, 2026 at 10:06 AM Thomas Gleixner <tglx@kernel.org> wrote:
>
> On Tue, Jan 20 2026 at 16:18, Jason Xing wrote:
> > On Sat, Nov 29, 2025 at 5:57 PM Eric Dumazet <edumazet@google.com> wrote:
> >
> > When I'm browsing the modification related to SWIFT, I noticed this
> > patch seems to have not been merged yet?
>
> It's in the tip tree and scheduled for the next merge window.
You beat me to it, thank you
tip tree :
commit 4725344462362e2ce2645f354737a8ea4280fa57
Author: Eric Dumazet <edumazet@google.com>
AuthorDate: Sat Nov 29 09:57:40 2025 +0000
Commit: Thomas Gleixner <tglx@kernel.org>
CommitDate: Mon Dec 15 20:16:49 2025 +0100
time/timecounter: Inline timecounter_cyc2time()
On Tue, Jan 20, 2026 at 5:11 PM Eric Dumazet <edumazet@google.com> wrote: > > On Tue, Jan 20, 2026 at 10:06 AM Thomas Gleixner <tglx@kernel.org> wrote: > > > > On Tue, Jan 20 2026 at 16:18, Jason Xing wrote: > > > On Sat, Nov 29, 2025 at 5:57 PM Eric Dumazet <edumazet@google.com> wrote: > > > > > > When I'm browsing the modification related to SWIFT, I noticed this > > > patch seems to have not been merged yet? > > > > It's in the tip tree and scheduled for the next merge window. > > You beat me to it, thank you > > tip tree : > > commit 4725344462362e2ce2645f354737a8ea4280fa57 > Author: Eric Dumazet <edumazet@google.com> > AuthorDate: Sat Nov 29 09:57:40 2025 +0000 > Commit: Thomas Gleixner <tglx@kernel.org> > CommitDate: Mon Dec 15 20:16:49 2025 +0100 > > time/timecounter: Inline timecounter_cyc2time() Thanks, I see now :) Thanks, Jason
On Sat, Nov 29 2025 at 09:57, Eric Dumazet wrote: > Add an unlikely() around the cc_cyc2ns_backwards() case, > even if FDO (when used) is able to take care of this optimization. > > Signed-off-by: Eric Dumazet <edumazet@google.com> Earmarked for post rc1
Eric Dumazet wrote: > New network transport protocols want NIC drivers to get hwtstamps > of all incoming packets, and possibly all outgoing packets. > > Swift congestion control is used by good old TCP transport and is > our primary need for timecounter_cyc2time(). This will be upstreamed soon. > > This means timecounter_cyc2time() can be called more than 100 million > times per second on a busy server. > > Inlining timecounter_cyc2time() brings a 12 % improvement on a > UDP receive stress test on a 100Gbit NIC. > > Note that FDO, LTO, PGO are unable to magically help for this > case, presumably because NIC drivers are almost exclusively shipped > as modules. > > Add an unlikely() around the cc_cyc2ns_backwards() case, > even if FDO (when used) is able to take care of this optimization. > > Signed-off-by: Eric Dumazet <edumazet@google.com> > Link: https://research.google/pubs/swift-delay-is-simple-and-effective-for-congestion-control-in-the-datacenter/ > Cc: Kevin Yang <yyd@google.com> > Cc: Willem de Bruijn <willemb@google.com> > Cc: Neal Cardwell <ncardwell@google.com> > Cc: Yuchung Cheng <ycheng@google.com> Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Kevin Yang <yyd@google.com> On Sat, Nov 29, 2025 at 12:06 PM Willem de Bruijn <willemdebruijn.kernel@gmail.com> wrote: > > Eric Dumazet wrote: > > New network transport protocols want NIC drivers to get hwtstamps > > of all incoming packets, and possibly all outgoing packets. > > > > Swift congestion control is used by good old TCP transport and is > > our primary need for timecounter_cyc2time(). This will be upstreamed soon. > > > > This means timecounter_cyc2time() can be called more than 100 million > > times per second on a busy server. > > > > Inlining timecounter_cyc2time() brings a 12 % improvement on a > > UDP receive stress test on a 100Gbit NIC. > > > > Note that FDO, LTO, PGO are unable to magically help for this > > case, presumably because NIC drivers are almost exclusively shipped > > as modules. > > > > Add an unlikely() around the cc_cyc2ns_backwards() case, > > even if FDO (when used) is able to take care of this optimization. > > > > Signed-off-by: Eric Dumazet <edumazet@google.com> > > Link: https://research.google/pubs/swift-delay-is-simple-and-effective-for-congestion-control-in-the-datacenter/ > > Cc: Kevin Yang <yyd@google.com> > > Cc: Willem de Bruijn <willemb@google.com> > > Cc: Neal Cardwell <ncardwell@google.com> > > Cc: Yuchung Cheng <ycheng@google.com> > > Reviewed-by: Willem de Bruijn <willemb@google.com>
The following commit has been merged into the timers/core branch of tip:
Commit-ID: 4725344462362e2ce2645f354737a8ea4280fa57
Gitweb: https://git.kernel.org/tip/4725344462362e2ce2645f354737a8ea4280fa57
Author: Eric Dumazet <edumazet@google.com>
AuthorDate: Sat, 29 Nov 2025 09:57:40
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Mon, 15 Dec 2025 20:16:49 +01:00
time/timecounter: Inline timecounter_cyc2time()
New network transport protocols want NIC drivers to get hardware timestamps
of all incoming packets, and possibly all outgoing packets.
One example is the upcoming 'Swift congestion control' which is used by TCP
transport and is the primary need for timecounter_cyc2time(). This means
timecounter_cyc2time() can be called more than 100 million times per second
on a busy server.
Inlining timecounter_cyc2time() brings a 12% improvement on a UDP receive
stress test on a 100Gbit NIC.
Note that FDO, LTO, PGO are unable to magically help for this case,
presumably because NIC drivers are almost exclusively shipped as modules.
Add an unlikely() around the cc_cyc2ns_backwards() case, even if FDO (when
used) is able to take care of this optimization.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://research.google/pubs/swift-delay-is-simple-and-effective-for-congestion-control-in-the-datacenter/
Link: https://patch.msgid.link/20251129095740.3338476-1-edumazet@google.com
---
include/linux/timecounter.h | 31 +++++++++++++++++++++++++++++--
kernel/time/timecounter.c | 35 -----------------------------------
2 files changed, 29 insertions(+), 37 deletions(-)
diff --git a/include/linux/timecounter.h b/include/linux/timecounter.h
index dce03a5..7de6b35 100644
--- a/include/linux/timecounter.h
+++ b/include/linux/timecounter.h
@@ -115,6 +115,15 @@ extern void timecounter_init(struct timecounter *tc,
*/
extern u64 timecounter_read(struct timecounter *tc);
+/*
+ * This is like cyclecounter_cyc2ns(), but it is used for computing a
+ * time previous to the time stored in the cycle counter.
+ */
+static inline u64 cc_cyc2ns_backwards(const struct cyclecounter *cc, u64 cycles, u64 frac)
+{
+ return ((cycles * cc->mult) - frac) >> cc->shift;
+}
+
/**
* timecounter_cyc2time - convert a cycle counter to same
* time base as values returned by
@@ -131,7 +140,25 @@ extern u64 timecounter_read(struct timecounter *tc);
*
* Returns: cycle counter converted to nanoseconds since the initial time stamp
*/
-extern u64 timecounter_cyc2time(const struct timecounter *tc,
- u64 cycle_tstamp);
+static inline u64 timecounter_cyc2time(const struct timecounter *tc, u64 cycle_tstamp)
+{
+ const struct cyclecounter *cc = tc->cc;
+ u64 delta = (cycle_tstamp - tc->cycle_last) & cc->mask;
+ u64 nsec = tc->nsec, frac = tc->frac;
+
+ /*
+ * Instead of always treating cycle_tstamp as more recent than
+ * tc->cycle_last, detect when it is too far in the future and
+ * treat it as old time stamp instead.
+ */
+ if (unlikely(delta > cc->mask / 2)) {
+ delta = (tc->cycle_last - cycle_tstamp) & cc->mask;
+ nsec -= cc_cyc2ns_backwards(cc, delta, frac);
+ } else {
+ nsec += cyclecounter_cyc2ns(cc, delta, tc->mask, &frac);
+ }
+
+ return nsec;
+}
#endif
diff --git a/kernel/time/timecounter.c b/kernel/time/timecounter.c
index 3d2a354..2e64dbb 100644
--- a/kernel/time/timecounter.c
+++ b/kernel/time/timecounter.c
@@ -62,38 +62,3 @@ u64 timecounter_read(struct timecounter *tc)
}
EXPORT_SYMBOL_GPL(timecounter_read);
-/*
- * This is like cyclecounter_cyc2ns(), but it is used for computing a
- * time previous to the time stored in the cycle counter.
- */
-static u64 cc_cyc2ns_backwards(const struct cyclecounter *cc,
- u64 cycles, u64 mask, u64 frac)
-{
- u64 ns = (u64) cycles;
-
- ns = ((ns * cc->mult) - frac) >> cc->shift;
-
- return ns;
-}
-
-u64 timecounter_cyc2time(const struct timecounter *tc,
- u64 cycle_tstamp)
-{
- u64 delta = (cycle_tstamp - tc->cycle_last) & tc->cc->mask;
- u64 nsec = tc->nsec, frac = tc->frac;
-
- /*
- * Instead of always treating cycle_tstamp as more recent
- * than tc->cycle_last, detect when it is too far in the
- * future and treat it as old time stamp instead.
- */
- if (delta > tc->cc->mask / 2) {
- delta = (tc->cycle_last - cycle_tstamp) & tc->cc->mask;
- nsec -= cc_cyc2ns_backwards(tc->cc, delta, tc->mask, frac);
- } else {
- nsec += cyclecounter_cyc2ns(tc->cc, delta, tc->mask, &frac);
- }
-
- return nsec;
-}
-EXPORT_SYMBOL_GPL(timecounter_cyc2time);© 2016 - 2026 Red Hat, Inc.