Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
vdso: Add generic time data storage
Historically each architecture defined their own way to store the vDSO
data page. Add a generic mechanism to provide storage for that page.
Furthermore this generic storage will be extended to also provide
uniform storage for *non*-time-related data, like the random state or
architecture-specific data. These will have their own pages and data
structures, so rename 'vdso_data' into 'vdso_time_data' to make that
split clear from the name.
Also introduce a new consistent naming scheme for the symbols related to
the vDSO, which makes it clear if the symbol is accessible from
userspace or kernel space and the type of data behind the symbol.
The generic fault handler contains an optimization to prefault the vvar
page when the timens page is accessed. This was lifted from s390 and x86.
Co-developed-by: Nam Cao <namcao@linutronix.de>
Signed-off-by: Nam Cao <namcao@linutronix.de>
Signed-off-by: Thomas Weißschuh <thomas.weissschuh@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/all/20250204-vdso-store-rng-v3-5-13a4669dfc8c@linutronix.de
authored by
Thomas Weißschuh
and committed by
Thomas Gleixner
df7fcbef
127b0e05
+195
-19
+16
include/asm-generic/vdso/vsyscall.h
+16
include/asm-generic/vdso/vsyscall.h
···
4
4
5
5
#ifndef __ASSEMBLY__
6
6
7
+
#ifdef CONFIG_GENERIC_VDSO_DATA_STORE
8
+
9
+
#ifndef __arch_get_vdso_u_time_data
10
+
static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void)
11
+
{
12
+
return vdso_u_time_data;
13
+
}
14
+
#endif
15
+
16
+
#else /* !CONFIG_GENERIC_VDSO_DATA_STORE */
17
+
7
18
#ifndef __arch_get_k_vdso_data
8
19
static __always_inline struct vdso_data *__arch_get_k_vdso_data(void)
9
20
{
10
21
return NULL;
11
22
}
12
23
#endif /* __arch_get_k_vdso_data */
24
+
#define vdso_k_time_data __arch_get_k_vdso_data()
25
+
26
+
#define __arch_get_vdso_u_time_data __arch_get_vdso_data
27
+
28
+
#endif /* CONFIG_GENERIC_VDSO_DATA_STORE */
13
29
14
30
#ifndef __arch_update_vsyscall
15
31
static __always_inline void __arch_update_vsyscall(struct vdso_data *vdata)
+1
include/linux/time_namespace.h
+1
include/linux/time_namespace.h
+10
include/linux/vdso_datastore.h
+10
include/linux/vdso_datastore.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
#ifndef _LINUX_VDSO_DATASTORE_H
3
+
#define _LINUX_VDSO_DATASTORE_H
4
+
5
+
#include <linux/mm_types.h>
6
+
7
+
extern const struct vm_special_mapping vdso_vvar_mapping;
8
+
struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr);
9
+
10
+
#endif /* _LINUX_VDSO_DATASTORE_H */
+33
-8
include/vdso/datapage.h
+33
-8
include/vdso/datapage.h
···
45
45
*
46
46
* There is one vdso_timestamp object in vvar for each vDSO-accelerated
47
47
* clock_id. For high-resolution clocks, this encodes the time
48
-
* corresponding to vdso_data.cycle_last. For coarse clocks this encodes
48
+
* corresponding to vdso_time_data.cycle_last. For coarse clocks this encodes
49
49
* the actual time.
50
50
*
51
51
* To be noticed that for highres clocks nsec is left-shifted by
52
-
* vdso_data.cs[x].shift.
52
+
* vdso_time_data[x].shift.
53
53
*/
54
54
struct vdso_timestamp {
55
55
u64 sec;
···
57
57
};
58
58
59
59
/**
60
-
* struct vdso_data - vdso datapage representation
60
+
* struct vdso_time_data - vdso datapage representation
61
61
* @seq: timebase sequence counter
62
62
* @clock_mode: clock mode
63
63
* @cycle_last: timebase at clocksource init
···
74
74
* @arch_data: architecture specific data (optional, defaults
75
75
* to an empty struct)
76
76
*
77
-
* vdso_data will be accessed by 64 bit and compat code at the same time
77
+
* vdso_time_data will be accessed by 64 bit and compat code at the same time
78
78
* so we should be careful before modifying this structure.
79
79
*
80
80
* The ordering of the struct members is optimized to have fast access to the
···
92
92
* For clocks which are not affected by time namespace adjustment the
93
93
* offset must be zero.
94
94
*/
95
-
struct vdso_data {
95
+
struct vdso_time_data {
96
96
u32 seq;
97
97
98
98
s32 clock_mode;
···
117
117
struct arch_vdso_time_data arch_data;
118
118
};
119
119
120
+
#define vdso_data vdso_time_data
121
+
120
122
/**
121
123
* struct vdso_rng_data - vdso RNG state information
122
124
* @generation: counter representing the number of RNG reseeds
···
138
136
* With the hidden visibility, the compiler simply generates a PC-relative
139
137
* relocation, and this is what we need.
140
138
*/
141
-
extern struct vdso_data _vdso_data[CS_BASES] __attribute__((visibility("hidden")));
142
-
extern struct vdso_data _timens_data[CS_BASES] __attribute__((visibility("hidden")));
139
+
#ifndef CONFIG_GENERIC_VDSO_DATA_STORE
140
+
extern struct vdso_time_data _vdso_data[CS_BASES] __attribute__((visibility("hidden")));
141
+
extern struct vdso_time_data _timens_data[CS_BASES] __attribute__((visibility("hidden")));
143
142
extern struct vdso_rng_data _vdso_rng_data __attribute__((visibility("hidden")));
143
+
#else
144
+
extern struct vdso_time_data vdso_u_time_data[CS_BASES] __attribute__((visibility("hidden")));
145
+
146
+
extern struct vdso_time_data *vdso_k_time_data;
147
+
#endif
144
148
145
149
/**
146
150
* union vdso_data_store - Generic vDSO data page
147
151
*/
148
152
union vdso_data_store {
149
-
struct vdso_data data[CS_BASES];
153
+
struct vdso_time_data data[CS_BASES];
150
154
u8 page[1U << CONFIG_PAGE_SHIFT];
151
155
};
156
+
157
+
#ifdef CONFIG_GENERIC_VDSO_DATA_STORE
158
+
159
+
enum vdso_pages {
160
+
VDSO_TIME_PAGE_OFFSET,
161
+
VDSO_TIMENS_PAGE_OFFSET,
162
+
VDSO_NR_PAGES
163
+
};
164
+
165
+
#endif /* CONFIG_GENERIC_VDSO_DATA_STORE */
152
166
153
167
/*
154
168
* The generic vDSO implementation requires that gettimeofday.h
···
181
163
#else
182
164
#include <asm/vdso/gettimeofday.h>
183
165
#endif /* ENABLE_COMPAT_VDSO */
166
+
167
+
#else /* !__ASSEMBLY__ */
168
+
169
+
#define VDSO_VVAR_SYMS \
170
+
PROVIDE(vdso_u_data = . - __VDSO_PAGES * PAGE_SIZE); \
171
+
PROVIDE(vdso_u_time_data = vdso_u_data); \
172
+
184
173
185
174
#endif /* !__ASSEMBLY__ */
186
175
+4
-4
kernel/time/vsyscall.c
+4
-4
kernel/time/vsyscall.c
···
77
77
78
78
void update_vsyscall(struct timekeeper *tk)
79
79
{
80
-
struct vdso_data *vdata = __arch_get_k_vdso_data();
80
+
struct vdso_data *vdata = vdso_k_time_data;
81
81
struct vdso_timestamp *vdso_ts;
82
82
s32 clock_mode;
83
83
u64 nsec;
···
128
128
129
129
void update_vsyscall_tz(void)
130
130
{
131
-
struct vdso_data *vdata = __arch_get_k_vdso_data();
131
+
struct vdso_data *vdata = vdso_k_time_data;
132
132
133
133
vdata[CS_HRES_COARSE].tz_minuteswest = sys_tz.tz_minuteswest;
134
134
vdata[CS_HRES_COARSE].tz_dsttime = sys_tz.tz_dsttime;
···
150
150
*/
151
151
unsigned long vdso_update_begin(void)
152
152
{
153
-
struct vdso_data *vdata = __arch_get_k_vdso_data();
153
+
struct vdso_data *vdata = vdso_k_time_data;
154
154
unsigned long flags = timekeeper_lock_irqsave();
155
155
156
156
vdso_write_begin(vdata);
···
167
167
*/
168
168
void vdso_update_end(unsigned long flags)
169
169
{
170
-
struct vdso_data *vdata = __arch_get_k_vdso_data();
170
+
struct vdso_data *vdata = vdso_k_time_data;
171
171
172
172
vdso_write_end(vdata);
173
173
__arch_sync_vdso_data(vdata);
+1
-1
lib/Makefile
+1
-1
lib/Makefile
···
132
132
obj-$(CONFIG_DEBUG_INFO_REDUCED) += debug_info.o
133
133
CFLAGS_debug_info.o += $(call cc-option, -femit-struct-debug-detailed=any)
134
134
135
-
obj-y += math/ crypto/
135
+
obj-y += math/ crypto/ vdso/
136
136
137
137
obj-$(CONFIG_GENERIC_IOMAP) += iomap.o
138
138
obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o
+5
lib/vdso/Kconfig
+5
lib/vdso/Kconfig
+3
lib/vdso/Makefile
+3
lib/vdso/Makefile
+103
lib/vdso/datastore.c
+103
lib/vdso/datastore.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
3
+
#include <linux/linkage.h>
4
+
#include <linux/mmap_lock.h>
5
+
#include <linux/mm.h>
6
+
#include <linux/time_namespace.h>
7
+
#include <linux/types.h>
8
+
#include <linux/vdso_datastore.h>
9
+
#include <vdso/datapage.h>
10
+
11
+
/*
12
+
* The vDSO data page.
13
+
*/
14
+
#ifdef CONFIG_HAVE_GENERIC_VDSO
15
+
static union vdso_data_store vdso_time_data_store __page_aligned_data;
16
+
struct vdso_time_data *vdso_k_time_data = vdso_time_data_store.data;
17
+
static_assert(sizeof(vdso_time_data_store) == PAGE_SIZE);
18
+
#endif /* CONFIG_HAVE_GENERIC_VDSO */
19
+
20
+
static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
21
+
struct vm_area_struct *vma, struct vm_fault *vmf)
22
+
{
23
+
struct page *timens_page = find_timens_vvar_page(vma);
24
+
unsigned long addr, pfn;
25
+
vm_fault_t err;
26
+
27
+
switch (vmf->pgoff) {
28
+
case VDSO_TIME_PAGE_OFFSET:
29
+
if (!IS_ENABLED(CONFIG_HAVE_GENERIC_VDSO))
30
+
return VM_FAULT_SIGBUS;
31
+
pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data));
32
+
if (timens_page) {
33
+
/*
34
+
* Fault in VVAR page too, since it will be accessed
35
+
* to get clock data anyway.
36
+
*/
37
+
addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE;
38
+
err = vmf_insert_pfn(vma, addr, pfn);
39
+
if (unlikely(err & VM_FAULT_ERROR))
40
+
return err;
41
+
pfn = page_to_pfn(timens_page);
42
+
}
43
+
break;
44
+
case VDSO_TIMENS_PAGE_OFFSET:
45
+
/*
46
+
* If a task belongs to a time namespace then a namespace
47
+
* specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
48
+
* the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
49
+
* offset.
50
+
* See also the comment near timens_setup_vdso_data().
51
+
*/
52
+
if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page)
53
+
return VM_FAULT_SIGBUS;
54
+
pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data));
55
+
break;
56
+
default:
57
+
return VM_FAULT_SIGBUS;
58
+
}
59
+
60
+
return vmf_insert_pfn(vma, vmf->address, pfn);
61
+
}
62
+
63
+
const struct vm_special_mapping vdso_vvar_mapping = {
64
+
.name = "[vvar]",
65
+
.fault = vvar_fault,
66
+
};
67
+
68
+
struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr)
69
+
{
70
+
return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE,
71
+
VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP | VM_PFNMAP,
72
+
&vdso_vvar_mapping);
73
+
}
74
+
75
+
#ifdef CONFIG_TIME_NS
76
+
/*
77
+
* The vvar page layout depends on whether a task belongs to the root or
78
+
* non-root time namespace. Whenever a task changes its namespace, the VVAR
79
+
* page tables are cleared and then they will be re-faulted with a
80
+
* corresponding layout.
81
+
* See also the comment near timens_setup_vdso_data() for details.
82
+
*/
83
+
int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
84
+
{
85
+
struct mm_struct *mm = task->mm;
86
+
struct vm_area_struct *vma;
87
+
VMA_ITERATOR(vmi, mm, 0);
88
+
89
+
mmap_read_lock(mm);
90
+
for_each_vma(vmi, vma) {
91
+
if (vma_is_special_mapping(vma, &vdso_vvar_mapping))
92
+
zap_vma_pages(vma);
93
+
}
94
+
mmap_read_unlock(mm);
95
+
96
+
return 0;
97
+
}
98
+
99
+
struct vdso_time_data *arch_get_vdso_data(void *vvar_page)
100
+
{
101
+
return (struct vdso_time_data *)vvar_page;
102
+
}
103
+
#endif
+19
-6
lib/vdso/gettimeofday.c
+19
-6
lib/vdso/gettimeofday.c
···
5
5
#include <vdso/datapage.h>
6
6
#include <vdso/helpers.h>
7
7
8
+
/* Bring in default accessors */
9
+
#include <vdso/vsyscall.h>
10
+
8
11
#ifndef vdso_calc_ns
9
12
10
13
#ifdef VDSO_DELTA_NOMASK
···
72
69
#endif
73
70
74
71
#ifdef CONFIG_TIME_NS
72
+
73
+
#ifdef CONFIG_GENERIC_VDSO_DATA_STORE
74
+
static __always_inline
75
+
const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd)
76
+
{
77
+
return (void *)vd + PAGE_SIZE;
78
+
}
79
+
#define __arch_get_timens_vdso_data(vd) __arch_get_vdso_u_timens_data(vd)
80
+
#endif /* CONFIG_GENERIC_VDSO_DATA_STORE */
81
+
75
82
static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
76
83
struct __kernel_timespec *ts)
77
84
{
···
295
282
static __maybe_unused int
296
283
__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
297
284
{
298
-
return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts);
285
+
return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts);
299
286
}
300
287
301
288
#ifdef BUILD_VDSO32
···
321
308
static __maybe_unused int
322
309
__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
323
310
{
324
-
return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res);
311
+
return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res);
325
312
}
326
313
#endif /* BUILD_VDSO32 */
327
314
···
355
342
static __maybe_unused int
356
343
__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
357
344
{
358
-
return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz);
345
+
return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz);
359
346
}
360
347
361
348
#ifdef VDSO_HAS_TIME
···
378
365
379
366
static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
380
367
{
381
-
return __cvdso_time_data(__arch_get_vdso_data(), time);
368
+
return __cvdso_time_data(__arch_get_vdso_u_time_data(), time);
382
369
}
383
370
#endif /* VDSO_HAS_TIME */
384
371
···
438
425
static __maybe_unused
439
426
int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
440
427
{
441
-
return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res);
428
+
return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res);
442
429
}
443
430
444
431
#ifdef BUILD_VDSO32
···
464
451
static __maybe_unused int
465
452
__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
466
453
{
467
-
return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(),
454
+
return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(),
468
455
clock, res);
469
456
}
470
457
#endif /* BUILD_VDSO32 */