Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
vdso: Remove remnants of architecture-specific time storage
All users of the time releated parts of the vDSO are now using the generic
storage implementation. Remove the therefore unnecessary compatibility
accessor functions and symbols.
Co-developed-by: Nam Cao <namcao@linutronix.de>
Signed-off-by: Nam Cao <namcao@linutronix.de>
Signed-off-by: Thomas Weißschuh <thomas.weissschuh@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/all/20250204-vdso-store-rng-v3-18-13a4669dfc8c@linutronix.de
authored by
Thomas Weißschuh
and committed by
Thomas Gleixner
ac1a42f4
998a8a26
+53
-89
+4
-16
include/asm-generic/vdso/vsyscall.h
+4
-16
include/asm-generic/vdso/vsyscall.h
···
20
20
}
21
21
#endif
22
22
23
-
#else /* !CONFIG_GENERIC_VDSO_DATA_STORE */
24
-
25
-
#ifndef __arch_get_k_vdso_data
26
-
static __always_inline struct vdso_data *__arch_get_k_vdso_data(void)
27
-
{
28
-
return NULL;
29
-
}
30
-
#endif /* __arch_get_k_vdso_data */
31
-
#define vdso_k_time_data __arch_get_k_vdso_data()
32
-
33
-
#define __arch_get_vdso_u_time_data __arch_get_vdso_data
34
-
35
23
#endif /* CONFIG_GENERIC_VDSO_DATA_STORE */
36
24
37
25
#ifndef __arch_update_vsyscall
38
-
static __always_inline void __arch_update_vsyscall(struct vdso_data *vdata)
26
+
static __always_inline void __arch_update_vsyscall(struct vdso_time_data *vdata)
39
27
{
40
28
}
41
29
#endif /* __arch_update_vsyscall */
42
30
43
-
#ifndef __arch_sync_vdso_data
44
-
static __always_inline void __arch_sync_vdso_data(struct vdso_data *vdata)
31
+
#ifndef __arch_sync_vdso_time_data
32
+
static __always_inline void __arch_sync_vdso_time_data(struct vdso_time_data *vdata)
45
33
{
46
34
}
47
-
#endif /* __arch_sync_vdso_data */
35
+
#endif /* __arch_sync_vdso_time_data */
48
36
49
37
#endif /* !__ASSEMBLY__ */
50
38
-3
include/linux/time_namespace.h
-3
include/linux/time_namespace.h
···
8
8
#include <linux/ns_common.h>
9
9
#include <linux/err.h>
10
10
#include <linux/time64.h>
11
-
#include <vdso/datapage.h>
12
11
13
12
struct user_namespace;
14
13
extern struct user_namespace init_user_ns;
···
164
165
return tim;
165
166
}
166
167
#endif
167
-
168
-
struct vdso_data *arch_get_vdso_data(void *vvar_page);
169
168
170
169
#endif /* _LINUX_TIMENS_H */
+1
-18
include/vdso/datapage.h
+1
-18
include/vdso/datapage.h
···
128
128
struct arch_vdso_time_data arch_data;
129
129
};
130
130
131
-
#define vdso_data vdso_time_data
132
-
133
131
/**
134
132
* struct vdso_rng_data - vdso RNG state information
135
133
* @generation: counter representing the number of RNG reseeds
···
147
149
* With the hidden visibility, the compiler simply generates a PC-relative
148
150
* relocation, and this is what we need.
149
151
*/
150
-
#ifndef CONFIG_GENERIC_VDSO_DATA_STORE
151
-
extern struct vdso_time_data _vdso_data[CS_BASES] __attribute__((visibility("hidden")));
152
-
extern struct vdso_time_data _timens_data[CS_BASES] __attribute__((visibility("hidden")));
153
-
#else
152
+
#ifdef CONFIG_GENERIC_VDSO_DATA_STORE
154
153
extern struct vdso_time_data vdso_u_time_data[CS_BASES] __attribute__((visibility("hidden")));
155
154
extern struct vdso_rng_data vdso_u_rng_data __attribute__((visibility("hidden")));
156
155
extern struct vdso_arch_data vdso_u_arch_data __attribute__((visibility("hidden")));
···
155
160
extern struct vdso_time_data *vdso_k_time_data;
156
161
extern struct vdso_rng_data *vdso_k_rng_data;
157
162
extern struct vdso_arch_data *vdso_k_arch_data;
158
-
#endif
159
-
160
-
/**
161
-
* union vdso_data_store - Generic vDSO data page
162
-
*/
163
-
union vdso_data_store {
164
-
struct vdso_time_data data[CS_BASES];
165
-
u8 page[1U << CONFIG_PAGE_SHIFT];
166
-
};
167
-
168
-
#ifdef CONFIG_GENERIC_VDSO_DATA_STORE
169
163
170
164
#define VDSO_ARCH_DATA_SIZE ALIGN(sizeof(struct vdso_arch_data), PAGE_SIZE)
171
165
#define VDSO_ARCH_DATA_PAGES (VDSO_ARCH_DATA_SIZE >> PAGE_SHIFT)
···
173
189
/*
174
190
* The generic vDSO implementation requires that gettimeofday.h
175
191
* provides:
176
-
* - __arch_get_vdso_data(): to get the vdso datapage.
177
192
* - __arch_get_hw_counter(): to get the hw counter based on the
178
193
* clock_mode.
179
194
* - gettimeofday_fallback(): fallback for gettimeofday.
+4
-4
include/vdso/helpers.h
+4
-4
include/vdso/helpers.h
···
7
7
#include <asm/barrier.h>
8
8
#include <vdso/datapage.h>
9
9
10
-
static __always_inline u32 vdso_read_begin(const struct vdso_data *vd)
10
+
static __always_inline u32 vdso_read_begin(const struct vdso_time_data *vd)
11
11
{
12
12
u32 seq;
13
13
···
18
18
return seq;
19
19
}
20
20
21
-
static __always_inline u32 vdso_read_retry(const struct vdso_data *vd,
21
+
static __always_inline u32 vdso_read_retry(const struct vdso_time_data *vd,
22
22
u32 start)
23
23
{
24
24
u32 seq;
···
28
28
return seq != start;
29
29
}
30
30
31
-
static __always_inline void vdso_write_begin(struct vdso_data *vd)
31
+
static __always_inline void vdso_write_begin(struct vdso_time_data *vd)
32
32
{
33
33
/*
34
34
* WRITE_ONCE() is required otherwise the compiler can validly tear
···
40
40
smp_wmb();
41
41
}
42
42
43
-
static __always_inline void vdso_write_end(struct vdso_data *vd)
43
+
static __always_inline void vdso_write_end(struct vdso_time_data *vd)
44
44
{
45
45
smp_wmb();
46
46
/*
+6
-6
kernel/time/namespace.c
+6
-6
kernel/time/namespace.c
···
165
165
* HVCLOCK
166
166
* VVAR
167
167
*
168
-
* The check for vdso_data->clock_mode is in the unlikely path of
168
+
* The check for vdso_time_data->clock_mode is in the unlikely path of
169
169
* the seq begin magic. So for the non-timens case most of the time
170
170
* 'seq' is even, so the branch is not taken.
171
171
*
172
172
* If 'seq' is odd, i.e. a concurrent update is in progress, the extra check
173
-
* for vdso_data->clock_mode is a non-issue. The task is spin waiting for the
173
+
* for vdso_time_data->clock_mode is a non-issue. The task is spin waiting for the
174
174
* update to finish and for 'seq' to become even anyway.
175
175
*
176
-
* Timens page has vdso_data->clock_mode set to VDSO_CLOCKMODE_TIMENS which
176
+
* Timens page has vdso_time_data->clock_mode set to VDSO_CLOCKMODE_TIMENS which
177
177
* enforces the time namespace handling path.
178
178
*/
179
-
static void timens_setup_vdso_data(struct vdso_data *vdata,
179
+
static void timens_setup_vdso_data(struct vdso_time_data *vdata,
180
180
struct time_namespace *ns)
181
181
{
182
182
struct timens_offset *offset = vdata->offset;
···
219
219
static void timens_set_vvar_page(struct task_struct *task,
220
220
struct time_namespace *ns)
221
221
{
222
-
struct vdso_data *vdata;
222
+
struct vdso_time_data *vdata;
223
223
unsigned int i;
224
224
225
225
if (ns == &init_time_ns)
···
235
235
goto out;
236
236
237
237
ns->frozen_offsets = true;
238
-
vdata = arch_get_vdso_data(page_address(ns->vvar_page));
238
+
vdata = page_address(ns->vvar_page);
239
239
240
240
for (i = 0; i < CS_BASES; i++)
241
241
timens_setup_vdso_data(&vdata[i], ns);
+9
-10
kernel/time/vsyscall.c
+9
-10
kernel/time/vsyscall.c
···
15
15
16
16
#include "timekeeping_internal.h"
17
17
18
-
static inline void update_vdso_data(struct vdso_data *vdata,
19
-
struct timekeeper *tk)
18
+
static inline void update_vdso_time_data(struct vdso_time_data *vdata, struct timekeeper *tk)
20
19
{
21
20
struct vdso_timestamp *vdso_ts;
22
21
u64 nsec, sec;
···
76
77
77
78
void update_vsyscall(struct timekeeper *tk)
78
79
{
79
-
struct vdso_data *vdata = vdso_k_time_data;
80
+
struct vdso_time_data *vdata = vdso_k_time_data;
80
81
struct vdso_timestamp *vdso_ts;
81
82
s32 clock_mode;
82
83
u64 nsec;
···
116
117
* update of the high resolution parts.
117
118
*/
118
119
if (clock_mode != VDSO_CLOCKMODE_NONE)
119
-
update_vdso_data(vdata, tk);
120
+
update_vdso_time_data(vdata, tk);
120
121
121
122
__arch_update_vsyscall(vdata);
122
123
123
124
vdso_write_end(vdata);
124
125
125
-
__arch_sync_vdso_data(vdata);
126
+
__arch_sync_vdso_time_data(vdata);
126
127
}
127
128
128
129
void update_vsyscall_tz(void)
129
130
{
130
-
struct vdso_data *vdata = vdso_k_time_data;
131
+
struct vdso_time_data *vdata = vdso_k_time_data;
131
132
132
133
vdata[CS_HRES_COARSE].tz_minuteswest = sys_tz.tz_minuteswest;
133
134
vdata[CS_HRES_COARSE].tz_dsttime = sys_tz.tz_dsttime;
134
135
135
-
__arch_sync_vdso_data(vdata);
136
+
__arch_sync_vdso_time_data(vdata);
136
137
}
137
138
138
139
/**
···
149
150
*/
150
151
unsigned long vdso_update_begin(void)
151
152
{
152
-
struct vdso_data *vdata = vdso_k_time_data;
153
+
struct vdso_time_data *vdata = vdso_k_time_data;
153
154
unsigned long flags = timekeeper_lock_irqsave();
154
155
155
156
vdso_write_begin(vdata);
···
166
167
*/
167
168
void vdso_update_end(unsigned long flags)
168
169
{
169
-
struct vdso_data *vdata = vdso_k_time_data;
170
+
struct vdso_time_data *vdata = vdso_k_time_data;
170
171
171
172
vdso_write_end(vdata);
172
-
__arch_sync_vdso_data(vdata);
173
+
__arch_sync_vdso_time_data(vdata);
173
174
timekeeper_unlock_irqrestore(flags);
174
175
}
+4
-6
lib/vdso/datastore.c
+4
-6
lib/vdso/datastore.c
···
12
12
* The vDSO data page.
13
13
*/
14
14
#ifdef CONFIG_HAVE_GENERIC_VDSO
15
-
static union vdso_data_store vdso_time_data_store __page_aligned_data;
15
+
static union {
16
+
struct vdso_time_data data[CS_BASES];
17
+
u8 page[PAGE_SIZE];
18
+
} vdso_time_data_store __page_aligned_data;
16
19
struct vdso_time_data *vdso_k_time_data = vdso_time_data_store.data;
17
20
static_assert(sizeof(vdso_time_data_store) == PAGE_SIZE);
18
21
#endif /* CONFIG_HAVE_GENERIC_VDSO */
···
125
122
mmap_read_unlock(mm);
126
123
127
124
return 0;
128
-
}
129
-
130
-
struct vdso_time_data *arch_get_vdso_data(void *vvar_page)
131
-
{
132
-
return (struct vdso_time_data *)vvar_page;
133
125
}
134
126
#endif
+25
-26
lib/vdso/gettimeofday.c
+25
-26
lib/vdso/gettimeofday.c
···
17
17
#endif
18
18
19
19
#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
20
-
static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta)
20
+
static __always_inline bool vdso_delta_ok(const struct vdso_time_data *vd, u64 delta)
21
21
{
22
22
return delta < vd->max_cycles;
23
23
}
24
24
#else
25
-
static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta)
25
+
static __always_inline bool vdso_delta_ok(const struct vdso_time_data *vd, u64 delta)
26
26
{
27
27
return true;
28
28
}
···
39
39
* Default implementation which works for all sane clocksources. That
40
40
* obviously excludes x86/TSC.
41
41
*/
42
-
static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base)
42
+
static __always_inline u64 vdso_calc_ns(const struct vdso_time_data *vd, u64 cycles, u64 base)
43
43
{
44
44
u64 delta = (cycles - vd->cycle_last) & VDSO_DELTA_MASK(vd);
45
45
···
58
58
#endif
59
59
60
60
#ifndef vdso_clocksource_ok
61
-
static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
61
+
static inline bool vdso_clocksource_ok(const struct vdso_time_data *vd)
62
62
{
63
63
return vd->clock_mode != VDSO_CLOCKMODE_NONE;
64
64
}
···
79
79
{
80
80
return (void *)vd + PAGE_SIZE;
81
81
}
82
-
#define __arch_get_timens_vdso_data(vd) __arch_get_vdso_u_timens_data(vd)
83
82
#endif /* CONFIG_GENERIC_VDSO_DATA_STORE */
84
83
85
-
static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
84
+
static __always_inline int do_hres_timens(const struct vdso_time_data *vdns, clockid_t clk,
86
85
struct __kernel_timespec *ts)
87
86
{
88
87
const struct timens_offset *offs = &vdns->offset[clk];
89
88
const struct vdso_timestamp *vdso_ts;
90
-
const struct vdso_data *vd;
89
+
const struct vdso_time_data *vd;
91
90
u64 cycles, ns;
92
91
u32 seq;
93
92
s64 sec;
94
93
95
94
vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE);
96
-
vd = __arch_get_timens_vdso_data(vd);
95
+
vd = __arch_get_vdso_u_timens_data(vd);
97
96
if (clk != CLOCK_MONOTONIC_RAW)
98
97
vd = &vd[CS_HRES_COARSE];
99
98
else
···
127
128
}
128
129
#else
129
130
static __always_inline
130
-
const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
131
+
const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd)
131
132
{
132
133
return NULL;
133
134
}
134
135
135
-
static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
136
+
static __always_inline int do_hres_timens(const struct vdso_time_data *vdns, clockid_t clk,
136
137
struct __kernel_timespec *ts)
137
138
{
138
139
return -EINVAL;
139
140
}
140
141
#endif
141
142
142
-
static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
143
+
static __always_inline int do_hres(const struct vdso_time_data *vd, clockid_t clk,
143
144
struct __kernel_timespec *ts)
144
145
{
145
146
const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
···
191
192
}
192
193
193
194
#ifdef CONFIG_TIME_NS
194
-
static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
195
+
static __always_inline int do_coarse_timens(const struct vdso_time_data *vdns, clockid_t clk,
195
196
struct __kernel_timespec *ts)
196
197
{
197
-
const struct vdso_data *vd = __arch_get_timens_vdso_data(vdns);
198
+
const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns);
198
199
const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
199
200
const struct timens_offset *offs = &vdns->offset[clk];
200
201
u64 nsec;
···
220
221
return 0;
221
222
}
222
223
#else
223
-
static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
224
+
static __always_inline int do_coarse_timens(const struct vdso_time_data *vdns, clockid_t clk,
224
225
struct __kernel_timespec *ts)
225
226
{
226
227
return -1;
227
228
}
228
229
#endif
229
230
230
-
static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,
231
+
static __always_inline int do_coarse(const struct vdso_time_data *vd, clockid_t clk,
231
232
struct __kernel_timespec *ts)
232
233
{
233
234
const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
···
254
255
}
255
256
256
257
static __always_inline int
257
-
__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock,
258
+
__cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock,
258
259
struct __kernel_timespec *ts)
259
260
{
260
261
u32 msk;
···
281
282
}
282
283
283
284
static __maybe_unused int
284
-
__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock,
285
+
__cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock,
285
286
struct __kernel_timespec *ts)
286
287
{
287
288
int ret = __cvdso_clock_gettime_common(vd, clock, ts);
···
299
300
300
301
#ifdef BUILD_VDSO32
301
302
static __maybe_unused int
302
-
__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock,
303
+
__cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock,
303
304
struct old_timespec32 *res)
304
305
{
305
306
struct __kernel_timespec ts;
···
325
326
#endif /* BUILD_VDSO32 */
326
327
327
328
static __maybe_unused int
328
-
__cvdso_gettimeofday_data(const struct vdso_data *vd,
329
+
__cvdso_gettimeofday_data(const struct vdso_time_data *vd,
329
330
struct __kernel_old_timeval *tv, struct timezone *tz)
330
331
{
331
332
···
342
343
if (unlikely(tz != NULL)) {
343
344
if (IS_ENABLED(CONFIG_TIME_NS) &&
344
345
vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
345
-
vd = __arch_get_timens_vdso_data(vd);
346
+
vd = __arch_get_vdso_u_timens_data(vd);
346
347
347
348
tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
348
349
tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
···
359
360
360
361
#ifdef VDSO_HAS_TIME
361
362
static __maybe_unused __kernel_old_time_t
362
-
__cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time)
363
+
__cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time)
363
364
{
364
365
__kernel_old_time_t t;
365
366
366
367
if (IS_ENABLED(CONFIG_TIME_NS) &&
367
368
vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
368
-
vd = __arch_get_timens_vdso_data(vd);
369
+
vd = __arch_get_vdso_u_timens_data(vd);
369
370
370
371
t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
371
372
···
383
384
384
385
#ifdef VDSO_HAS_CLOCK_GETRES
385
386
static __maybe_unused
386
-
int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock,
387
+
int __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock,
387
388
struct __kernel_timespec *res)
388
389
{
389
390
u32 msk;
···
395
396
396
397
if (IS_ENABLED(CONFIG_TIME_NS) &&
397
398
vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
398
-
vd = __arch_get_timens_vdso_data(vd);
399
+
vd = __arch_get_vdso_u_timens_data(vd);
399
400
400
401
/*
401
402
* Convert the clockid to a bitmask and use it to check which
···
424
425
}
425
426
426
427
static __maybe_unused
427
-
int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock,
428
+
int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock,
428
429
struct __kernel_timespec *res)
429
430
{
430
431
int ret = __cvdso_clock_getres_common(vd, clock, res);
···
442
443
443
444
#ifdef BUILD_VDSO32
444
445
static __maybe_unused int
445
-
__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock,
446
+
__cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock,
446
447
struct old_timespec32 *res)
447
448
{
448
449
struct __kernel_timespec ts;