tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

time/timecounter: Inline timecounter_cyc2time()

New network transport protocols want NIC drivers to get hardware timestamps
of all incoming packets, and possibly all outgoing packets.

One example is the upcoming 'Swift congestion control' which is used by TCP
transport and is the primary need for timecounter_cyc2time(). This means
timecounter_cyc2time() can be called more than 100 million times per second
on a busy server.

Inlining timecounter_cyc2time() brings a 12% improvement on a UDP receive
stress test on a 100Gbit NIC.

Note that FDO, LTO, PGO are unable to magically help for this case,
presumably because NIC drivers are almost exclusively shipped as modules.

Add an unlikely() around the cc_cyc2ns_backwards() case, even if FDO (when
used) is able to take care of this optimization.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://research.google/pubs/swift-delay-is-simple-and-effective-for-congestion-control-in-the-datacenter/
Link: https://patch.msgid.link/20251129095740.3338476-1-edumazet@google.com

authored by

Eric Dumazet and committed by
Thomas Gleixner 47253444 8f0b4cce

+29 -37
+29 -2
include/linux/timecounter.h
··· 115 115 */ 116 116 extern u64 timecounter_read(struct timecounter *tc); 117 117 118 + /* 119 + * This is like cyclecounter_cyc2ns(), but it is used for computing a 120 + * time previous to the time stored in the cycle counter. 121 + */ 122 + static inline u64 cc_cyc2ns_backwards(const struct cyclecounter *cc, u64 cycles, u64 frac) 123 + { 124 + return ((cycles * cc->mult) - frac) >> cc->shift; 125 + } 126 + 118 127 /** 119 128 * timecounter_cyc2time - convert a cycle counter to same 120 129 * time base as values returned by ··· 140 131 * 141 132 * Returns: cycle counter converted to nanoseconds since the initial time stamp 142 133 */ 143 - extern u64 timecounter_cyc2time(const struct timecounter *tc, 144 - u64 cycle_tstamp); 134 + static inline u64 timecounter_cyc2time(const struct timecounter *tc, u64 cycle_tstamp) 135 + { 136 + const struct cyclecounter *cc = tc->cc; 137 + u64 delta = (cycle_tstamp - tc->cycle_last) & cc->mask; 138 + u64 nsec = tc->nsec, frac = tc->frac; 139 + 140 + /* 141 + * Instead of always treating cycle_tstamp as more recent than 142 + * tc->cycle_last, detect when it is too far in the future and 143 + * treat it as old time stamp instead. 144 + */ 145 + if (unlikely(delta > cc->mask / 2)) { 146 + delta = (tc->cycle_last - cycle_tstamp) & cc->mask; 147 + nsec -= cc_cyc2ns_backwards(cc, delta, frac); 148 + } else { 149 + nsec += cyclecounter_cyc2ns(cc, delta, tc->mask, &frac); 150 + } 151 + 152 + return nsec; 153 + } 145 154 146 155 #endif
-35
kernel/time/timecounter.c
··· 62 62 } 63 63 EXPORT_SYMBOL_GPL(timecounter_read); 64 64 65 - /* 66 - * This is like cyclecounter_cyc2ns(), but it is used for computing a 67 - * time previous to the time stored in the cycle counter. 68 - */ 69 - static u64 cc_cyc2ns_backwards(const struct cyclecounter *cc, 70 - u64 cycles, u64 mask, u64 frac) 71 - { 72 - u64 ns = (u64) cycles; 73 - 74 - ns = ((ns * cc->mult) - frac) >> cc->shift; 75 - 76 - return ns; 77 - } 78 - 79 - u64 timecounter_cyc2time(const struct timecounter *tc, 80 - u64 cycle_tstamp) 81 - { 82 - u64 delta = (cycle_tstamp - tc->cycle_last) & tc->cc->mask; 83 - u64 nsec = tc->nsec, frac = tc->frac; 84 - 85 - /* 86 - * Instead of always treating cycle_tstamp as more recent 87 - * than tc->cycle_last, detect when it is too far in the 88 - * future and treat it as old time stamp instead. 89 - */ 90 - if (delta > tc->cc->mask / 2) { 91 - delta = (tc->cycle_last - cycle_tstamp) & tc->cc->mask; 92 - nsec -= cc_cyc2ns_backwards(tc->cc, delta, tc->mask, frac); 93 - } else { 94 - nsec += cyclecounter_cyc2ns(tc->cc, delta, tc->mask, &frac); 95 - } 96 - 97 - return nsec; 98 - } 99 - EXPORT_SYMBOL_GPL(timecounter_cyc2time);