Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
vdso: Consolidate nanoseconds calculation
Consolidate nanoseconds calculation to simplify and reduce code
duplication.
Suggested-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Adrian Hunter <adrian.hunter@intel.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/r/20240325064023.2997-3-adrian.hunter@intel.com
authored by
Adrian Hunter
and committed by
Thomas Gleixner
5b26ef66
c8e3a8b6
+28
-34
+8
-9
arch/x86/include/asm/vdso/gettimeofday.h
+8
-9
arch/x86/include/asm/vdso/gettimeofday.h
···
300
300
#define vdso_cycles_ok arch_vdso_cycles_ok
301
301
302
302
/*
303
-
* x86 specific delta calculation.
303
+
* x86 specific calculation of nanoseconds for the current cycle count
304
304
*
305
305
* The regular implementation assumes that clocksource reads are globally
306
306
* monotonic. The TSC can be slightly off across sockets which can cause
···
308
308
* jump.
309
309
*
310
310
* Therefore it needs to be verified that @cycles are greater than
311
-
* @last. If not then use @last, which is the base time of the current
312
-
* conversion period.
311
+
* @vd->cycles_last. If not then use @vd->cycles_last, which is the base
312
+
* time of the current conversion period.
313
313
*
314
314
* This variant also uses a custom mask because while the clocksource mask of
315
315
* all the VDSO capable clocksources on x86 is U64_MAX, the above code uses
···
317
317
* declares everything with the MSB/Sign-bit set as invalid. Therefore the
318
318
* effective mask is S64_MAX.
319
319
*/
320
-
static __always_inline
321
-
u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
320
+
static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base)
322
321
{
323
322
/*
324
323
* Due to the MSB/Sign-bit being used as invalid marker (see
325
324
* arch_vdso_cycles_valid() above), the effective mask is S64_MAX.
326
325
*/
327
-
u64 delta = (cycles - last) & S64_MAX;
326
+
u64 delta = (cycles - vd->cycle_last) & S64_MAX;
328
327
329
328
/*
330
329
* Due to the above mentioned TSC wobbles, filter out negative motion.
331
330
* Per the above masking, the effective sign bit is now bit 62.
332
331
*/
333
332
if (unlikely(delta & (1ULL << 62)))
334
-
return 0;
333
+
return base >> vd->shift;
335
334
336
-
return delta * mult;
335
+
return ((delta * vd->mult) + base) >> vd->shift;
337
336
}
338
-
#define vdso_calc_delta vdso_calc_delta
337
+
#define vdso_calc_ns vdso_calc_ns
339
338
340
339
#endif /* !__ASSEMBLY__ */
341
340
+20
-25
lib/vdso/gettimeofday.c
+20
-25
lib/vdso/gettimeofday.c
···
5
5
#include <vdso/datapage.h>
6
6
#include <vdso/helpers.h>
7
7
8
-
#ifndef vdso_calc_delta
8
+
#ifndef vdso_calc_ns
9
9
10
10
#ifdef VDSO_DELTA_NOMASK
11
-
# define VDSO_DELTA_MASK(mask) U64_MAX
11
+
# define VDSO_DELTA_MASK(vd) U64_MAX
12
12
#else
13
-
# define VDSO_DELTA_MASK(mask) (mask)
14
-
#endif
15
-
16
-
/*
17
-
* Default implementation which works for all sane clocksources. That
18
-
* obviously excludes x86/TSC.
19
-
*/
20
-
static __always_inline
21
-
u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
22
-
{
23
-
return ((cycles - last) & VDSO_DELTA_MASK(mask)) * mult;
24
-
}
13
+
# define VDSO_DELTA_MASK(vd) (vd->mask)
25
14
#endif
26
15
27
16
#ifndef vdso_shift_ns
···
19
30
return ns >> shift;
20
31
}
21
32
#endif
33
+
34
+
/*
35
+
* Default implementation which works for all sane clocksources. That
36
+
* obviously excludes x86/TSC.
37
+
*/
38
+
static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base)
39
+
{
40
+
u64 delta = (cycles - vd->cycle_last) & VDSO_DELTA_MASK(vd);
41
+
42
+
return vdso_shift_ns((delta * vd->mult) + base, vd->shift);
43
+
}
44
+
#endif /* vdso_calc_ns */
22
45
23
46
#ifndef __arch_vdso_hres_capable
24
47
static inline bool __arch_vdso_hres_capable(void)
···
57
56
static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
58
57
struct __kernel_timespec *ts)
59
58
{
60
-
const struct vdso_data *vd;
61
59
const struct timens_offset *offs = &vdns->offset[clk];
62
60
const struct vdso_timestamp *vdso_ts;
63
-
u64 cycles, last, ns;
61
+
const struct vdso_data *vd;
62
+
u64 cycles, ns;
64
63
u32 seq;
65
64
s64 sec;
66
65
···
81
80
cycles = __arch_get_hw_counter(vd->clock_mode, vd);
82
81
if (unlikely(!vdso_cycles_ok(cycles)))
83
82
return -1;
84
-
ns = vdso_ts->nsec;
85
-
last = vd->cycle_last;
86
-
ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
87
-
ns = vdso_shift_ns(ns, vd->shift);
83
+
ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec);
88
84
sec = vdso_ts->sec;
89
85
} while (unlikely(vdso_read_retry(vd, seq)));
90
86
···
116
118
struct __kernel_timespec *ts)
117
119
{
118
120
const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
119
-
u64 cycles, last, sec, ns;
121
+
u64 cycles, sec, ns;
120
122
u32 seq;
121
123
122
124
/* Allows to compile the high resolution parts out */
···
149
151
cycles = __arch_get_hw_counter(vd->clock_mode, vd);
150
152
if (unlikely(!vdso_cycles_ok(cycles)))
151
153
return -1;
152
-
ns = vdso_ts->nsec;
153
-
last = vd->cycle_last;
154
-
ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
155
-
ns = vdso_shift_ns(ns, vd->shift);
154
+
ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec);
156
155
sec = vdso_ts->sec;
157
156
} while (unlikely(vdso_read_retry(vd, seq)));
158
157