Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'timers-vdso-2025-03-23' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull VDSO infrastructure updates from Thomas Gleixner:
- Consolidate the VDSO storage
The VDSO data storage and data layout has been largely architecture
specific for historical reasons. That increases the maintenance
effort and causes inconsistencies over and over.
There is no real technical reason for architecture specific layouts
and implementations. The architecture specific details can easily be
integrated into a generic layout, which also reduces the amount of
duplicated code for managing the mappings.
Convert all architectures over to a unified layout and common mapping
infrastructure. This splits the VDSO data layout into subsystem
specific blocks, timekeeping, random and architecture parts, which
provides a better structure and allows to improve and update the
functionalities without conflict and interaction.
- Rework the timekeeping data storage
The current implementation is designed for exposing system
timekeeping accessors, which was good enough at the time when it was
designed.
PTP and Time Sensitive Networking (TSN) change that as there are
requirements to expose independent PTP clocks, which are not related
to system timekeeping.
Replace the monolithic data storage by a structured layout, which
allows to add support for independent PTP clocks on top while reusing
both the data structures and the time accessor implementations.
* tag 'timers-vdso-2025-03-23' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (55 commits)
sparc/vdso: Always reject undefined references during linking
x86/vdso: Always reject undefined references during linking
vdso: Rework struct vdso_time_data and introduce struct vdso_clock
vdso: Move architecture related data before basetime data
powerpc/vdso: Prepare introduction of struct vdso_clock
arm64/vdso: Prepare introduction of struct vdso_clock
x86/vdso: Prepare introduction of struct vdso_clock
time/namespace: Prepare introduction of struct vdso_clock
vdso/namespace: Rename timens_setup_vdso_data() to reflect new vdso_clock struct
vdso/vsyscall: Prepare introduction of struct vdso_clock
vdso/gettimeofday: Prepare helper functions for introduction of struct vdso_clock
vdso/gettimeofday: Prepare do_coarse_timens() for introduction of struct vdso_clock
vdso/gettimeofday: Prepare do_coarse() for introduction of struct vdso_clock
vdso/gettimeofday: Prepare do_hres_timens() for introduction of struct vdso_clock
vdso/gettimeofday: Prepare do_hres() for introduction of struct vdso_clock
vdso/gettimeofday: Prepare introduction of struct vdso_clock
vdso/helpers: Prepare introduction of struct vdso_clock
vdso/datapage: Define vdso_clock to prepare for multiple PTP clocks
vdso: Make vdso_time_data cacheline aligned
arm64: Make asm/cache.h compatible with vDSO
...
+1408
-1584
+1
MAINTAINERS
+1
MAINTAINERS
+4
arch/Kconfig
+4
arch/Kconfig
···
1584
1584
entries at 4000, 5000 and 6000 locations. This option turns on syscall
1585
1585
related optimizations for a given architecture.
1586
1586
1587
+
config ARCH_HAS_VDSO_ARCH_DATA
1588
+
depends on GENERIC_VDSO_DATA_STORE
1589
+
bool
1590
+
1587
1591
config ARCH_HAS_VDSO_TIME_DATA
1588
1592
bool
1589
1593
+2
arch/arm/include/asm/vdso.h
+2
arch/arm/include/asm/vdso.h
+1
-6
arch/arm/include/asm/vdso/gettimeofday.h
+1
-6
arch/arm/include/asm/vdso/gettimeofday.h
···
112
112
#define __arch_vdso_hres_capable arm_vdso_hres_capable
113
113
114
114
static __always_inline u64 __arch_get_hw_counter(int clock_mode,
115
-
const struct vdso_data *vd)
115
+
const struct vdso_time_data *vd)
116
116
{
117
117
#ifdef CONFIG_ARM_ARCH_TIMER
118
118
u64 cycle_now;
···
133
133
/* Make GCC happy. This is compiled out anyway */
134
134
return 0;
135
135
#endif
136
-
}
137
-
138
-
static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
139
-
{
140
-
return _vdso_data;
141
136
}
142
137
143
138
#endif /* !__ASSEMBLY__ */
+2
-10
arch/arm/include/asm/vdso/vsyscall.h
+2
-10
arch/arm/include/asm/vdso/vsyscall.h
···
7
7
#include <vdso/datapage.h>
8
8
#include <asm/cacheflush.h>
9
9
10
-
extern struct vdso_data *vdso_data;
11
10
extern bool cntvct_ok;
12
11
13
12
static __always_inline
14
-
struct vdso_data *__arm_get_k_vdso_data(void)
15
-
{
16
-
return vdso_data;
17
-
}
18
-
#define __arch_get_k_vdso_data __arm_get_k_vdso_data
19
-
20
-
static __always_inline
21
-
void __arm_sync_vdso_data(struct vdso_data *vdata)
13
+
void __arch_sync_vdso_time_data(struct vdso_time_data *vdata)
22
14
{
23
15
flush_dcache_page(virt_to_page(vdata));
24
16
}
25
-
#define __arch_sync_vdso_data __arm_sync_vdso_data
17
+
#define __arch_sync_vdso_time_data __arch_sync_vdso_time_data
26
18
27
19
/* The asm-generic header needs to be included after the definitions above */
28
20
#include <asm-generic/vdso/vsyscall.h>
-4
arch/arm/kernel/asm-offsets.c
-4
arch/arm/kernel/asm-offsets.c
···
153
153
DEFINE(CACHE_WRITEBACK_ORDER, __CACHE_WRITEBACK_ORDER);
154
154
DEFINE(CACHE_WRITEBACK_GRANULE, __CACHE_WRITEBACK_GRANULE);
155
155
BLANK();
156
-
#ifdef CONFIG_VDSO
157
-
DEFINE(VDSO_DATA_SIZE, sizeof(union vdso_data_store));
158
-
#endif
159
-
BLANK();
160
156
#ifdef CONFIG_ARM_MPU
161
157
DEFINE(MPU_RNG_INFO_RNGS, offsetof(struct mpu_rgn_info, rgns));
162
158
DEFINE(MPU_RNG_INFO_USED, offsetof(struct mpu_rgn_info, used));
+7
-27
arch/arm/kernel/vdso.c
+7
-27
arch/arm/kernel/vdso.c
···
7
7
*/
8
8
9
9
#include <linux/cache.h>
10
+
#include <linux/vdso_datastore.h>
10
11
#include <linux/elf.h>
11
12
#include <linux/err.h>
12
13
#include <linux/kernel.h>
···
33
32
34
33
/* Total number of pages needed for the data and text portions of the VDSO. */
35
34
unsigned int vdso_total_pages __ro_after_init;
36
-
37
-
static union vdso_data_store vdso_data_store __page_aligned_data;
38
-
struct vdso_data *vdso_data = vdso_data_store.data;
39
-
40
-
static struct page *vdso_data_page __ro_after_init;
41
-
static const struct vm_special_mapping vdso_data_mapping = {
42
-
.name = "[vvar]",
43
-
.pages = &vdso_data_page,
44
-
};
45
35
46
36
static int vdso_mremap(const struct vm_special_mapping *sm,
47
37
struct vm_area_struct *new_vma)
···
184
192
if (vdso_text_pagelist == NULL)
185
193
return -ENOMEM;
186
194
187
-
/* Grab the VDSO data page. */
188
-
vdso_data_page = virt_to_page(vdso_data);
189
-
190
195
/* Grab the VDSO text pages. */
191
196
for (i = 0; i < text_pages; i++) {
192
197
struct page *page;
···
194
205
195
206
vdso_text_mapping.pages = vdso_text_pagelist;
196
207
197
-
vdso_total_pages = 1; /* for the data/vvar page */
208
+
vdso_total_pages = VDSO_NR_PAGES; /* for the data/vvar pages */
198
209
vdso_total_pages += text_pages;
199
210
200
211
cntvct_ok = cntvct_functional();
···
205
216
}
206
217
arch_initcall(vdso_init);
207
218
208
-
static int install_vvar(struct mm_struct *mm, unsigned long addr)
209
-
{
210
-
struct vm_area_struct *vma;
211
-
212
-
vma = _install_special_mapping(mm, addr, PAGE_SIZE,
213
-
VM_READ | VM_MAYREAD,
214
-
&vdso_data_mapping);
215
-
216
-
return PTR_ERR_OR_ZERO(vma);
217
-
}
219
+
static_assert(__VDSO_PAGES == VDSO_NR_PAGES);
218
220
219
221
/* assumes mmap_lock is write-locked */
220
222
void arm_install_vdso(struct mm_struct *mm, unsigned long addr)
···
218
238
if (vdso_text_pagelist == NULL)
219
239
return;
220
240
221
-
if (install_vvar(mm, addr))
241
+
if (IS_ERR(vdso_install_vvar_mapping(mm, addr)))
222
242
return;
223
243
224
-
/* Account for vvar page. */
225
-
addr += PAGE_SIZE;
226
-
len = (vdso_total_pages - 1) << PAGE_SHIFT;
244
+
/* Account for vvar pages. */
245
+
addr += VDSO_NR_PAGES * PAGE_SIZE;
246
+
len = (vdso_total_pages - VDSO_NR_PAGES) << PAGE_SHIFT;
227
247
228
248
vma = _install_special_mapping(mm, addr, len,
229
249
VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC,
+1
arch/arm/mm/Kconfig
+1
arch/arm/mm/Kconfig
+1
-1
arch/arm/vdso/Makefile
+1
-1
arch/arm/vdso/Makefile
+2
-2
arch/arm/vdso/vdso.lds.S
+2
-2
arch/arm/vdso/vdso.lds.S
···
11
11
*/
12
12
13
13
#include <linux/const.h>
14
-
#include <asm/asm-offsets.h>
15
14
#include <asm/page.h>
16
15
#include <asm/vdso.h>
16
+
#include <vdso/datapage.h>
17
17
18
18
OUTPUT_FORMAT("elf32-littlearm", "elf32-bigarm", "elf32-littlearm")
19
19
OUTPUT_ARCH(arm)
20
20
21
21
SECTIONS
22
22
{
23
-
PROVIDE(_vdso_data = . - VDSO_DATA_SIZE);
23
+
VDSO_VVAR_SYMS
24
24
25
25
. = SIZEOF_HEADERS;
26
26
+1
arch/arm64/Kconfig
+1
arch/arm64/Kconfig
+2
-2
arch/arm64/include/asm/cache.h
+2
-2
arch/arm64/include/asm/cache.h
···
35
35
#define ARCH_DMA_MINALIGN (128)
36
36
#define ARCH_KMALLOC_MINALIGN (8)
37
37
38
-
#ifndef __ASSEMBLY__
38
+
#if !defined(__ASSEMBLY__) && !defined(BUILD_VDSO)
39
39
40
40
#include <linux/bitops.h>
41
41
#include <linux/kasan-enabled.h>
···
118
118
return ctr;
119
119
}
120
120
121
-
#endif /* __ASSEMBLY__ */
121
+
#endif /* !defined(__ASSEMBLY__) && !defined(BUILD_VDSO) */
122
122
123
123
#endif
+1
-1
arch/arm64/include/asm/vdso.h
+1
-1
arch/arm64/include/asm/vdso.h
+13
-25
arch/arm64/include/asm/vdso/compat_gettimeofday.h
+13
-25
arch/arm64/include/asm/vdso/compat_gettimeofday.h
···
104
104
}
105
105
106
106
static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
107
-
const struct vdso_data *vd)
107
+
const struct vdso_time_data *vd)
108
108
{
109
109
u64 res;
110
110
···
131
131
return res;
132
132
}
133
133
134
-
static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
134
+
static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void)
135
135
{
136
-
const struct vdso_data *ret;
136
+
const struct vdso_time_data *ret;
137
137
138
138
/*
139
-
* This simply puts &_vdso_data into ret. The reason why we don't use
140
-
* `ret = _vdso_data` is that the compiler tends to optimise this in a
141
-
* very suboptimal way: instead of keeping &_vdso_data in a register,
142
-
* it goes through a relocation almost every time _vdso_data must be
139
+
* This simply puts &_vdso_time_data into ret. The reason why we don't use
140
+
* `ret = _vdso_time_data` is that the compiler tends to optimise this in a
141
+
* very suboptimal way: instead of keeping &_vdso_time_data in a register,
142
+
* it goes through a relocation almost every time _vdso_time_data must be
143
143
* accessed (even in subfunctions). This is both time and space
144
144
* consuming: each relocation uses a word in the code section, and it
145
145
* has to be loaded at runtime.
146
146
*
147
147
* This trick hides the assignment from the compiler. Since it cannot
148
148
* track where the pointer comes from, it will only use one relocation
149
-
* where __arch_get_vdso_data() is called, and then keep the result in
150
-
* a register.
149
+
* where __aarch64_get_vdso_u_time_data() is called, and then keep the
150
+
* result in a register.
151
151
*/
152
-
asm volatile("mov %0, %1" : "=r"(ret) : "r"(_vdso_data));
152
+
asm volatile("mov %0, %1" : "=r"(ret) : "r"(&vdso_u_time_data));
153
153
154
154
return ret;
155
155
}
156
+
#define __arch_get_vdso_u_time_data __arch_get_vdso_u_time_data
156
157
157
-
#ifdef CONFIG_TIME_NS
158
-
static __always_inline
159
-
const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
158
+
static inline bool vdso_clocksource_ok(const struct vdso_clock *vc)
160
159
{
161
-
const struct vdso_data *ret;
162
-
163
-
/* See __arch_get_vdso_data(). */
164
-
asm volatile("mov %0, %1" : "=r"(ret) : "r"(_timens_data));
165
-
166
-
return ret;
167
-
}
168
-
#endif
169
-
170
-
static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
171
-
{
172
-
return vd->clock_mode == VDSO_CLOCKMODE_ARCHTIMER;
160
+
return vc->clock_mode == VDSO_CLOCKMODE_ARCHTIMER;
173
161
}
174
162
#define vdso_clocksource_ok vdso_clocksource_ok
175
163
-12
arch/arm64/include/asm/vdso/getrandom.h
-12
arch/arm64/include/asm/vdso/getrandom.h
···
33
33
return ret;
34
34
}
35
35
36
-
static __always_inline const struct vdso_rng_data *__arch_get_vdso_rng_data(void)
37
-
{
38
-
/*
39
-
* The RNG data is in the real VVAR data page, but if a task belongs to a time namespace
40
-
* then VVAR_DATA_PAGE_OFFSET points to the namespace-specific VVAR page and VVAR_TIMENS_
41
-
* PAGE_OFFSET points to the real VVAR page.
42
-
*/
43
-
if (IS_ENABLED(CONFIG_TIME_NS) && _vdso_data->clock_mode == VDSO_CLOCKMODE_TIMENS)
44
-
return (void *)&_vdso_rng_data + VVAR_TIMENS_PAGE_OFFSET * (1UL << CONFIG_PAGE_SHIFT);
45
-
return &_vdso_rng_data;
46
-
}
47
-
48
36
#endif /* !__ASSEMBLY__ */
49
37
50
38
#endif /* __ASM_VDSO_GETRANDOM_H */
+1
-15
arch/arm64/include/asm/vdso/gettimeofday.h
+1
-15
arch/arm64/include/asm/vdso/gettimeofday.h
···
67
67
}
68
68
69
69
static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
70
-
const struct vdso_data *vd)
70
+
const struct vdso_time_data *vd)
71
71
{
72
72
u64 res;
73
73
···
98
98
99
99
return res;
100
100
}
101
-
102
-
static __always_inline
103
-
const struct vdso_data *__arch_get_vdso_data(void)
104
-
{
105
-
return _vdso_data;
106
-
}
107
-
108
-
#ifdef CONFIG_TIME_NS
109
-
static __always_inline
110
-
const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
111
-
{
112
-
return _timens_data;
113
-
}
114
-
#endif
115
101
116
102
#endif /* !__ASSEMBLY__ */
117
103
+3
-26
arch/arm64/include/asm/vdso/vsyscall.h
+3
-26
arch/arm64/include/asm/vdso/vsyscall.h
···
2
2
#ifndef __ASM_VDSO_VSYSCALL_H
3
3
#define __ASM_VDSO_VSYSCALL_H
4
4
5
-
#define __VDSO_RND_DATA_OFFSET 480
6
-
7
5
#ifndef __ASSEMBLY__
8
6
9
7
#include <vdso/datapage.h>
10
8
11
-
enum vvar_pages {
12
-
VVAR_DATA_PAGE_OFFSET,
13
-
VVAR_TIMENS_PAGE_OFFSET,
14
-
VVAR_NR_PAGES,
15
-
};
16
-
17
9
#define VDSO_PRECISION_MASK ~(0xFF00ULL<<48)
18
10
19
-
extern struct vdso_data *vdso_data;
20
11
21
12
/*
22
13
* Update the vDSO data page to keep in sync with kernel timekeeping.
23
14
*/
24
15
static __always_inline
25
-
struct vdso_data *__arm64_get_k_vdso_data(void)
16
+
void __arm64_update_vsyscall(struct vdso_time_data *vdata)
26
17
{
27
-
return vdso_data;
28
-
}
29
-
#define __arch_get_k_vdso_data __arm64_get_k_vdso_data
30
-
31
-
static __always_inline
32
-
struct vdso_rng_data *__arm64_get_k_vdso_rnd_data(void)
33
-
{
34
-
return (void *)vdso_data + __VDSO_RND_DATA_OFFSET;
35
-
}
36
-
#define __arch_get_k_vdso_rng_data __arm64_get_k_vdso_rnd_data
37
-
38
-
static __always_inline
39
-
void __arm64_update_vsyscall(struct vdso_data *vdata)
40
-
{
41
-
vdata[CS_HRES_COARSE].mask = VDSO_PRECISION_MASK;
42
-
vdata[CS_RAW].mask = VDSO_PRECISION_MASK;
18
+
vdata->clock_data[CS_HRES_COARSE].mask = VDSO_PRECISION_MASK;
19
+
vdata->clock_data[CS_RAW].mask = VDSO_PRECISION_MASK;
43
20
}
44
21
#define __arch_update_vsyscall __arm64_update_vsyscall
45
22
+5
-85
arch/arm64/kernel/vdso.c
+5
-85
arch/arm64/kernel/vdso.c
···
18
18
#include <linux/sched.h>
19
19
#include <linux/signal.h>
20
20
#include <linux/slab.h>
21
-
#include <linux/time_namespace.h>
21
+
#include <linux/vdso_datastore.h>
22
22
#include <linux/vmalloc.h>
23
23
#include <vdso/datapage.h>
24
24
#include <vdso/helpers.h>
···
56
56
},
57
57
#endif /* CONFIG_COMPAT_VDSO */
58
58
};
59
-
60
-
/*
61
-
* The vDSO data page.
62
-
*/
63
-
static union vdso_data_store vdso_data_store __page_aligned_data;
64
-
struct vdso_data *vdso_data = vdso_data_store.data;
65
59
66
60
static int vdso_mremap(const struct vm_special_mapping *sm,
67
61
struct vm_area_struct *new_vma)
···
98
104
return 0;
99
105
}
100
106
101
-
#ifdef CONFIG_TIME_NS
102
-
struct vdso_data *arch_get_vdso_data(void *vvar_page)
103
-
{
104
-
return (struct vdso_data *)(vvar_page);
105
-
}
106
-
107
-
static const struct vm_special_mapping vvar_map;
108
-
109
-
/*
110
-
* The vvar mapping contains data for a specific time namespace, so when a task
111
-
* changes namespace we must unmap its vvar data for the old namespace.
112
-
* Subsequent faults will map in data for the new namespace.
113
-
*
114
-
* For more details see timens_setup_vdso_data().
115
-
*/
116
-
int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
117
-
{
118
-
struct mm_struct *mm = task->mm;
119
-
struct vm_area_struct *vma;
120
-
VMA_ITERATOR(vmi, mm, 0);
121
-
122
-
mmap_read_lock(mm);
123
-
124
-
for_each_vma(vmi, vma) {
125
-
if (vma_is_special_mapping(vma, &vvar_map))
126
-
zap_vma_pages(vma);
127
-
}
128
-
129
-
mmap_read_unlock(mm);
130
-
return 0;
131
-
}
132
-
#endif
133
-
134
-
static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
135
-
struct vm_area_struct *vma, struct vm_fault *vmf)
136
-
{
137
-
struct page *timens_page = find_timens_vvar_page(vma);
138
-
unsigned long pfn;
139
-
140
-
switch (vmf->pgoff) {
141
-
case VVAR_DATA_PAGE_OFFSET:
142
-
if (timens_page)
143
-
pfn = page_to_pfn(timens_page);
144
-
else
145
-
pfn = sym_to_pfn(vdso_data);
146
-
break;
147
-
#ifdef CONFIG_TIME_NS
148
-
case VVAR_TIMENS_PAGE_OFFSET:
149
-
/*
150
-
* If a task belongs to a time namespace then a namespace
151
-
* specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
152
-
* the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
153
-
* offset.
154
-
* See also the comment near timens_setup_vdso_data().
155
-
*/
156
-
if (!timens_page)
157
-
return VM_FAULT_SIGBUS;
158
-
pfn = sym_to_pfn(vdso_data);
159
-
break;
160
-
#endif /* CONFIG_TIME_NS */
161
-
default:
162
-
return VM_FAULT_SIGBUS;
163
-
}
164
-
165
-
return vmf_insert_pfn(vma, vmf->address, pfn);
166
-
}
167
-
168
-
static const struct vm_special_mapping vvar_map = {
169
-
.name = "[vvar]",
170
-
.fault = vvar_fault,
171
-
};
172
-
173
107
static int __setup_additional_pages(enum vdso_abi abi,
174
108
struct mm_struct *mm,
175
109
struct linux_binprm *bprm,
···
107
185
unsigned long gp_flags = 0;
108
186
void *ret;
109
187
110
-
BUILD_BUG_ON(VVAR_NR_PAGES != __VVAR_PAGES);
188
+
BUILD_BUG_ON(VDSO_NR_PAGES != __VDSO_PAGES);
111
189
112
190
vdso_text_len = vdso_info[abi].vdso_pages << PAGE_SHIFT;
113
191
/* Be sure to map the data page */
114
-
vdso_mapping_len = vdso_text_len + VVAR_NR_PAGES * PAGE_SIZE;
192
+
vdso_mapping_len = vdso_text_len + VDSO_NR_PAGES * PAGE_SIZE;
115
193
116
194
vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);
117
195
if (IS_ERR_VALUE(vdso_base)) {
···
119
197
goto up_fail;
120
198
}
121
199
122
-
ret = _install_special_mapping(mm, vdso_base, VVAR_NR_PAGES * PAGE_SIZE,
123
-
VM_READ|VM_MAYREAD|VM_PFNMAP,
124
-
&vvar_map);
200
+
ret = vdso_install_vvar_mapping(mm, vdso_base);
125
201
if (IS_ERR(ret))
126
202
goto up_fail;
127
203
128
204
if (system_supports_bti_kernel())
129
205
gp_flags = VM_ARM64_BTI;
130
206
131
-
vdso_base += VVAR_NR_PAGES * PAGE_SIZE;
207
+
vdso_base += VDSO_NR_PAGES * PAGE_SIZE;
132
208
mm->context.vdso = (void *)vdso_base;
133
209
ret = _install_special_mapping(mm, vdso_base, vdso_text_len,
134
210
VM_READ|VM_EXEC|gp_flags|
+1
-1
arch/arm64/kernel/vdso/Makefile
+1
-1
arch/arm64/kernel/vdso/Makefile
+2
-5
arch/arm64/kernel/vdso/vdso.lds.S
+2
-5
arch/arm64/kernel/vdso/vdso.lds.S
···
20
20
21
21
SECTIONS
22
22
{
23
-
PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
24
-
PROVIDE(_vdso_rng_data = _vdso_data + __VDSO_RND_DATA_OFFSET);
25
-
#ifdef CONFIG_TIME_NS
26
-
PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
27
-
#endif
23
+
VDSO_VVAR_SYMS
24
+
28
25
. = SIZEOF_HEADERS;
29
26
30
27
.hash : { *(.hash) } :text
+1
-1
arch/arm64/kernel/vdso32/Makefile
+1
-1
arch/arm64/kernel/vdso32/Makefile
+3
-4
arch/arm64/kernel/vdso32/vdso.lds.S
+3
-4
arch/arm64/kernel/vdso32/vdso.lds.S
···
12
12
#include <asm/page.h>
13
13
#include <asm/vdso.h>
14
14
#include <asm-generic/vmlinux.lds.h>
15
+
#include <vdso/datapage.h>
15
16
16
17
OUTPUT_FORMAT("elf32-littlearm", "elf32-bigarm", "elf32-littlearm")
17
18
OUTPUT_ARCH(arm)
18
19
19
20
SECTIONS
20
21
{
21
-
PROVIDE_HIDDEN(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
22
-
#ifdef CONFIG_TIME_NS
23
-
PROVIDE_HIDDEN(_timens_data = _vdso_data + PAGE_SIZE);
24
-
#endif
22
+
VDSO_VVAR_SYMS
23
+
25
24
. = SIZEOF_HEADERS;
26
25
27
26
.hash : { *(.hash) } :text
+1
-1
arch/csky/kernel/vdso/Makefile
+1
-1
arch/csky/kernel/vdso/Makefile
+2
arch/loongarch/Kconfig
+2
arch/loongarch/Kconfig
···
30
30
select ARCH_HAS_SET_MEMORY
31
31
select ARCH_HAS_SET_DIRECT_MAP
32
32
select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
33
+
select ARCH_HAS_VDSO_ARCH_DATA
33
34
select ARCH_INLINE_READ_LOCK if !PREEMPTION
34
35
select ARCH_INLINE_READ_LOCK_BH if !PREEMPTION
35
36
select ARCH_INLINE_READ_LOCK_IRQ if !PREEMPTION
···
107
106
select GENERIC_SCHED_CLOCK
108
107
select GENERIC_SMP_IDLE_THREAD
109
108
select GENERIC_TIME_VSYSCALL
109
+
select GENERIC_VDSO_DATA_STORE
110
110
select GENERIC_VDSO_TIME_NS
111
111
select GPIOLIB
112
112
select HAS_IOPORT
-1
arch/loongarch/include/asm/vdso.h
-1
arch/loongarch/include/asm/vdso.h
+25
arch/loongarch/include/asm/vdso/arch_data.h
+25
arch/loongarch/include/asm/vdso/arch_data.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-or-later */
2
+
/*
3
+
* Author: Huacai Chen <chenhuacai@loongson.cn>
4
+
* Copyright (C) 2020-2022 Loongson Technology Corporation Limited
5
+
*/
6
+
7
+
#ifndef _VDSO_ARCH_DATA_H
8
+
#define _VDSO_ARCH_DATA_H
9
+
10
+
#ifndef __ASSEMBLY__
11
+
12
+
#include <asm/asm.h>
13
+
#include <asm/vdso.h>
14
+
15
+
struct vdso_pcpu_data {
16
+
u32 node;
17
+
} ____cacheline_aligned_in_smp;
18
+
19
+
struct vdso_arch_data {
20
+
struct vdso_pcpu_data pdata[NR_CPUS];
21
+
};
22
+
23
+
#endif /* __ASSEMBLY__ */
24
+
25
+
#endif
-5
arch/loongarch/include/asm/vdso/getrandom.h
-5
arch/loongarch/include/asm/vdso/getrandom.h
+1
-13
arch/loongarch/include/asm/vdso/gettimeofday.h
+1
-13
arch/loongarch/include/asm/vdso/gettimeofday.h
···
72
72
}
73
73
74
74
static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
75
-
const struct vdso_data *vd)
75
+
const struct vdso_time_data *vd)
76
76
{
77
77
uint64_t count;
78
78
···
89
89
}
90
90
#define __arch_vdso_hres_capable loongarch_vdso_hres_capable
91
91
92
-
static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
93
-
{
94
-
return _vdso_data;
95
-
}
96
-
97
-
#ifdef CONFIG_TIME_NS
98
-
static __always_inline
99
-
const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
100
-
{
101
-
return _timens_data;
102
-
}
103
-
#endif
104
92
#endif /* !__ASSEMBLY__ */
105
93
106
94
#endif /* __ASM_VDSO_GETTIMEOFDAY_H */
+2
-36
arch/loongarch/include/asm/vdso/vdso.h
+2
-36
arch/loongarch/include/asm/vdso/vdso.h
···
12
12
#include <asm/asm.h>
13
13
#include <asm/page.h>
14
14
#include <asm/vdso.h>
15
+
#include <vdso/datapage.h>
15
16
16
-
struct vdso_pcpu_data {
17
-
u32 node;
18
-
} ____cacheline_aligned_in_smp;
19
-
20
-
struct loongarch_vdso_data {
21
-
struct vdso_pcpu_data pdata[NR_CPUS];
22
-
struct vdso_rng_data rng_data;
23
-
};
24
-
25
-
/*
26
-
* The layout of vvar:
27
-
*
28
-
* high
29
-
* +---------------------+--------------------------+
30
-
* | loongarch vdso data | LOONGARCH_VDSO_DATA_SIZE |
31
-
* +---------------------+--------------------------+
32
-
* | time-ns vdso data | PAGE_SIZE |
33
-
* +---------------------+--------------------------+
34
-
* | generic vdso data | PAGE_SIZE |
35
-
* +---------------------+--------------------------+
36
-
* low
37
-
*/
38
-
#define LOONGARCH_VDSO_DATA_SIZE PAGE_ALIGN(sizeof(struct loongarch_vdso_data))
39
-
#define LOONGARCH_VDSO_DATA_PAGES (LOONGARCH_VDSO_DATA_SIZE >> PAGE_SHIFT)
40
-
41
-
enum vvar_pages {
42
-
VVAR_GENERIC_PAGE_OFFSET,
43
-
VVAR_TIMENS_PAGE_OFFSET,
44
-
VVAR_LOONGARCH_PAGES_START,
45
-
VVAR_LOONGARCH_PAGES_END = VVAR_LOONGARCH_PAGES_START + LOONGARCH_VDSO_DATA_PAGES - 1,
46
-
VVAR_NR_PAGES,
47
-
};
48
-
49
-
#define VVAR_SIZE (VVAR_NR_PAGES << PAGE_SHIFT)
50
-
51
-
extern struct loongarch_vdso_data _loongarch_data __attribute__((visibility("hidden")));
17
+
#define VVAR_SIZE (VDSO_NR_PAGES << PAGE_SHIFT)
52
18
53
19
#endif /* __ASSEMBLY__ */
54
20
-17
arch/loongarch/include/asm/vdso/vsyscall.h
-17
arch/loongarch/include/asm/vdso/vsyscall.h
···
6
6
7
7
#include <vdso/datapage.h>
8
8
9
-
extern struct vdso_data *vdso_data;
10
-
extern struct vdso_rng_data *vdso_rng_data;
11
-
12
-
static __always_inline
13
-
struct vdso_data *__loongarch_get_k_vdso_data(void)
14
-
{
15
-
return vdso_data;
16
-
}
17
-
#define __arch_get_k_vdso_data __loongarch_get_k_vdso_data
18
-
19
-
static __always_inline
20
-
struct vdso_rng_data *__loongarch_get_k_vdso_rng_data(void)
21
-
{
22
-
return vdso_rng_data;
23
-
}
24
-
#define __arch_get_k_vdso_rng_data __loongarch_get_k_vdso_rng_data
25
-
26
9
/* The asm-generic header needs to be included after the definitions above */
27
10
#include <asm-generic/vdso/vsyscall.h>
28
11
+1
-1
arch/loongarch/kernel/asm-offsets.c
+1
-1
arch/loongarch/kernel/asm-offsets.c
+3
-89
arch/loongarch/kernel/vdso.c
+3
-89
arch/loongarch/kernel/vdso.c
···
14
14
#include <linux/random.h>
15
15
#include <linux/sched.h>
16
16
#include <linux/slab.h>
17
-
#include <linux/time_namespace.h>
17
+
#include <linux/vdso_datastore.h>
18
18
19
19
#include <asm/page.h>
20
20
#include <asm/vdso.h>
···
25
25
26
26
extern char vdso_start[], vdso_end[];
27
27
28
-
/* Kernel-provided data used by the VDSO. */
29
-
static union vdso_data_store generic_vdso_data __page_aligned_data;
30
-
31
-
static union {
32
-
u8 page[LOONGARCH_VDSO_DATA_SIZE];
33
-
struct loongarch_vdso_data vdata;
34
-
} loongarch_vdso_data __page_aligned_data;
35
-
36
-
struct vdso_data *vdso_data = generic_vdso_data.data;
37
-
struct vdso_pcpu_data *vdso_pdata = loongarch_vdso_data.vdata.pdata;
38
-
struct vdso_rng_data *vdso_rng_data = &loongarch_vdso_data.vdata.rng_data;
39
-
40
28
static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma)
41
29
{
42
30
current->mm->context.vdso = (void *)(new_vma->vm_start);
···
32
44
return 0;
33
45
}
34
46
35
-
static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
36
-
struct vm_area_struct *vma, struct vm_fault *vmf)
37
-
{
38
-
unsigned long pfn;
39
-
struct page *timens_page = find_timens_vvar_page(vma);
40
-
41
-
switch (vmf->pgoff) {
42
-
case VVAR_GENERIC_PAGE_OFFSET:
43
-
if (!timens_page)
44
-
pfn = sym_to_pfn(vdso_data);
45
-
else
46
-
pfn = page_to_pfn(timens_page);
47
-
break;
48
-
#ifdef CONFIG_TIME_NS
49
-
case VVAR_TIMENS_PAGE_OFFSET:
50
-
/*
51
-
* If a task belongs to a time namespace then a namespace specific
52
-
* VVAR is mapped with the VVAR_GENERIC_PAGE_OFFSET and the real
53
-
* VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET offset.
54
-
* See also the comment near timens_setup_vdso_data().
55
-
*/
56
-
if (!timens_page)
57
-
return VM_FAULT_SIGBUS;
58
-
else
59
-
pfn = sym_to_pfn(vdso_data);
60
-
break;
61
-
#endif /* CONFIG_TIME_NS */
62
-
case VVAR_LOONGARCH_PAGES_START ... VVAR_LOONGARCH_PAGES_END:
63
-
pfn = sym_to_pfn(&loongarch_vdso_data) + vmf->pgoff - VVAR_LOONGARCH_PAGES_START;
64
-
break;
65
-
default:
66
-
return VM_FAULT_SIGBUS;
67
-
}
68
-
69
-
return vmf_insert_pfn(vma, vmf->address, pfn);
70
-
}
71
-
72
47
struct loongarch_vdso_info vdso_info = {
73
48
.vdso = vdso_start,
74
49
.code_mapping = {
75
50
.name = "[vdso]",
76
51
.mremap = vdso_mremap,
77
-
},
78
-
.data_mapping = {
79
-
.name = "[vvar]",
80
-
.fault = vvar_fault,
81
52
},
82
53
.offset_sigreturn = vdso_offset_sigreturn,
83
54
};
···
48
101
BUG_ON(!PAGE_ALIGNED(vdso_info.vdso));
49
102
50
103
for_each_possible_cpu(cpu)
51
-
vdso_pdata[cpu].node = cpu_to_node(cpu);
104
+
vdso_k_arch_data->pdata[cpu].node = cpu_to_node(cpu);
52
105
53
106
vdso_info.size = PAGE_ALIGN(vdso_end - vdso_start);
54
107
vdso_info.code_mapping.pages =
···
61
114
return 0;
62
115
}
63
116
subsys_initcall(init_vdso);
64
-
65
-
#ifdef CONFIG_TIME_NS
66
-
struct vdso_data *arch_get_vdso_data(void *vvar_page)
67
-
{
68
-
return (struct vdso_data *)(vvar_page);
69
-
}
70
-
71
-
/*
72
-
* The vvar mapping contains data for a specific time namespace, so when a
73
-
* task changes namespace we must unmap its vvar data for the old namespace.
74
-
* Subsequent faults will map in data for the new namespace.
75
-
*
76
-
* For more details see timens_setup_vdso_data().
77
-
*/
78
-
int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
79
-
{
80
-
struct mm_struct *mm = task->mm;
81
-
struct vm_area_struct *vma;
82
-
83
-
VMA_ITERATOR(vmi, mm, 0);
84
-
85
-
mmap_read_lock(mm);
86
-
for_each_vma(vmi, vma) {
87
-
if (vma_is_special_mapping(vma, &vdso_info.data_mapping))
88
-
zap_vma_pages(vma);
89
-
}
90
-
mmap_read_unlock(mm);
91
-
92
-
return 0;
93
-
}
94
-
#endif
95
117
96
118
static unsigned long vdso_base(void)
97
119
{
···
97
181
goto out;
98
182
}
99
183
100
-
vma = _install_special_mapping(mm, data_addr, VVAR_SIZE,
101
-
VM_READ | VM_MAYREAD | VM_PFNMAP,
102
-
&info->data_mapping);
184
+
vma = vdso_install_vvar_mapping(mm, data_addr);
103
185
if (IS_ERR(vma)) {
104
186
ret = PTR_ERR(vma);
105
187
goto out;
+1
-1
arch/loongarch/vdso/Makefile
+1
-1
arch/loongarch/vdso/Makefile
+3
-5
arch/loongarch/vdso/vdso.lds.S
+3
-5
arch/loongarch/vdso/vdso.lds.S
···
5
5
*/
6
6
#include <asm/page.h>
7
7
#include <generated/asm-offsets.h>
8
+
#include <vdso/datapage.h>
8
9
9
10
OUTPUT_FORMAT("elf64-loongarch", "elf64-loongarch", "elf64-loongarch")
10
11
···
13
12
14
13
SECTIONS
15
14
{
16
-
PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
17
-
#ifdef CONFIG_TIME_NS
18
-
PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
19
-
#endif
20
-
PROVIDE(_loongarch_data = _vdso_data + 2 * PAGE_SIZE);
15
+
VDSO_VVAR_SYMS
16
+
21
17
. = SIZEOF_HEADERS;
22
18
23
19
.hash : { *(.hash) } :text
+2
-10
arch/loongarch/vdso/vgetcpu.c
+2
-10
arch/loongarch/vdso/vgetcpu.c
···
19
19
return cpu_id;
20
20
}
21
21
22
-
static __always_inline const struct vdso_pcpu_data *get_pcpu_data(void)
23
-
{
24
-
return _loongarch_data.pdata;
25
-
}
26
-
27
22
extern
28
23
int __vdso_getcpu(unsigned int *cpu, unsigned int *node, struct getcpu_cache *unused);
29
24
int __vdso_getcpu(unsigned int *cpu, unsigned int *node, struct getcpu_cache *unused)
30
25
{
31
26
int cpu_id;
32
-
const struct vdso_pcpu_data *data;
33
27
34
28
cpu_id = read_cpu_id();
35
29
36
30
if (cpu)
37
31
*cpu = cpu_id;
38
32
39
-
if (node) {
40
-
data = get_pcpu_data();
41
-
*node = data[cpu_id].node;
42
-
}
33
+
if (node)
34
+
*node = vdso_u_arch_data.pdata[cpu_id].node;
43
35
44
36
return 0;
45
37
}
+1
arch/mips/Kconfig
+1
arch/mips/Kconfig
···
51
51
select GENERIC_SMP_IDLE_THREAD
52
52
select GENERIC_IDLE_POLL_SETUP
53
53
select GENERIC_TIME_VSYSCALL
54
+
select GENERIC_VDSO_DATA_STORE
54
55
select GUP_GET_PXX_LOW_HIGH if CPU_MIPS32 && PHYS_ADDR_T_64BIT
55
56
select HAS_IOPORT if !NO_IOPORT_MAP || ISA
56
57
select HAVE_ARCH_COMPILER_H
+5
-4
arch/mips/include/asm/vdso/gettimeofday.h
+5
-4
arch/mips/include/asm/vdso/gettimeofday.h
···
167
167
168
168
#ifdef CONFIG_CLKSRC_MIPS_GIC
169
169
170
-
static __always_inline u64 read_gic_count(const struct vdso_data *data)
170
+
static __always_inline u64 read_gic_count(const struct vdso_time_data *data)
171
171
{
172
172
void __iomem *gic = get_gic(data);
173
173
u32 hi, hi2, lo;
···
184
184
#endif
185
185
186
186
static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
187
-
const struct vdso_data *vd)
187
+
const struct vdso_time_data *vd)
188
188
{
189
189
#ifdef CONFIG_CSRC_R4K
190
190
if (clock_mode == VDSO_CLOCKMODE_R4K)
···
209
209
}
210
210
#define __arch_vdso_hres_capable mips_vdso_hres_capable
211
211
212
-
static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
212
+
static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void)
213
213
{
214
-
return get_vdso_data();
214
+
return get_vdso_time_data();
215
215
}
216
+
#define __arch_get_vdso_u_time_data __arch_get_vdso_u_time_data
216
217
217
218
#endif /* !__ASSEMBLY__ */
218
219
+8
-11
arch/mips/include/asm/vdso/vdso.h
+8
-11
arch/mips/include/asm/vdso/vdso.h
···
5
5
*/
6
6
7
7
#include <asm/sgidefs.h>
8
+
#include <vdso/page.h>
9
+
10
+
#define __VDSO_PAGES 4
8
11
9
12
#ifndef __ASSEMBLY__
10
13
11
14
#include <asm/asm.h>
12
-
#include <asm/page.h>
13
15
#include <asm/vdso.h>
14
16
15
-
static inline unsigned long get_vdso_base(void)
17
+
static inline const struct vdso_time_data *get_vdso_time_data(void)
16
18
{
17
-
unsigned long addr;
19
+
const struct vdso_time_data *addr;
18
20
19
21
/*
20
22
* We can't use cpu_has_mips_r6 since it needs the cpu_data[]
···
29
27
* We can't use addiupc because there is no label-label
30
28
* support for the addiupc reloc
31
29
*/
32
-
__asm__("lapc %0, _start \n"
30
+
__asm__("lapc %0, vdso_u_time_data \n"
33
31
: "=r" (addr) : :);
34
32
#else
35
33
/*
···
48
46
" .set noreorder \n"
49
47
" bal 1f \n"
50
48
" nop \n"
51
-
" .word _start - . \n"
49
+
" .word vdso_u_time_data - . \n"
52
50
"1: lw %0, 0($31) \n"
53
51
" " STR(PTR_ADDU) " %0, $31, %0 \n"
54
52
" .set pop \n"
···
60
58
return addr;
61
59
}
62
60
63
-
static inline const struct vdso_data *get_vdso_data(void)
64
-
{
65
-
return (const struct vdso_data *)(get_vdso_base() - PAGE_SIZE);
66
-
}
67
-
68
61
#ifdef CONFIG_CLKSRC_MIPS_GIC
69
62
70
-
static inline void __iomem *get_gic(const struct vdso_data *data)
63
+
static inline void __iomem *get_gic(const struct vdso_time_data *data)
71
64
{
72
65
return (void __iomem *)((unsigned long)data & PAGE_MASK) - PAGE_SIZE;
73
66
}
+2
-12
arch/mips/include/asm/vdso/vsyscall.h
+2
-12
arch/mips/include/asm/vdso/vsyscall.h
···
2
2
#ifndef __ASM_VDSO_VSYSCALL_H
3
3
#define __ASM_VDSO_VSYSCALL_H
4
4
5
+
#include <asm/page.h>
6
+
5
7
#ifndef __ASSEMBLY__
6
8
7
9
#include <vdso/datapage.h>
8
-
9
-
extern struct vdso_data *vdso_data;
10
-
11
-
/*
12
-
* Update the vDSO data page to keep in sync with kernel timekeeping.
13
-
*/
14
-
static __always_inline
15
-
struct vdso_data *__mips_get_k_vdso_data(void)
16
-
{
17
-
return vdso_data;
18
-
}
19
-
#define __arch_get_k_vdso_data __mips_get_k_vdso_data
20
10
21
11
/* The asm-generic header needs to be included after the definitions above */
22
12
#include <asm-generic/vdso/vsyscall.h>
+18
-29
arch/mips/kernel/vdso.c
+18
-29
arch/mips/kernel/vdso.c
···
15
15
#include <linux/random.h>
16
16
#include <linux/sched.h>
17
17
#include <linux/slab.h>
18
+
#include <linux/vdso_datastore.h>
18
19
19
20
#include <asm/abi.h>
20
21
#include <asm/mips-cps.h>
···
24
23
#include <vdso/helpers.h>
25
24
#include <vdso/vsyscall.h>
26
25
27
-
/* Kernel-provided data used by the VDSO. */
28
-
static union vdso_data_store mips_vdso_data __page_aligned_data;
29
-
struct vdso_data *vdso_data = mips_vdso_data.data;
30
-
31
-
/*
32
-
* Mapping for the VDSO data/GIC pages. The real pages are mapped manually, as
33
-
* what we map and where within the area they are mapped is determined at
34
-
* runtime.
35
-
*/
36
-
static struct page *no_pages[] = { NULL };
37
-
static struct vm_special_mapping vdso_vvar_mapping = {
38
-
.name = "[vvar]",
39
-
.pages = no_pages,
40
-
};
26
+
static_assert(VDSO_NR_PAGES == __VDSO_PAGES);
41
27
42
28
static void __init init_vdso_image(struct mips_vdso_image *image)
43
29
{
···
78
90
{
79
91
struct mips_vdso_image *image = current->thread.abi->vdso;
80
92
struct mm_struct *mm = current->mm;
81
-
unsigned long gic_size, vvar_size, size, base, data_addr, vdso_addr, gic_pfn, gic_base;
93
+
unsigned long gic_size, size, base, data_addr, vdso_addr, gic_pfn, gic_base;
82
94
struct vm_area_struct *vma;
83
95
int ret;
84
96
···
107
119
* the counter registers at the start.
108
120
*/
109
121
gic_size = mips_gic_present() ? PAGE_SIZE : 0;
110
-
vvar_size = gic_size + PAGE_SIZE;
111
-
size = vvar_size + image->size;
122
+
size = gic_size + VDSO_NR_PAGES * PAGE_SIZE + image->size;
112
123
113
124
/*
114
125
* Find a region that's large enough for us to perform the
···
130
143
*/
131
144
if (cpu_has_dc_aliases) {
132
145
base = __ALIGN_MASK(base, shm_align_mask);
133
-
base += ((unsigned long)vdso_data - gic_size) & shm_align_mask;
146
+
base += ((unsigned long)vdso_k_time_data - gic_size) & shm_align_mask;
134
147
}
135
148
136
149
data_addr = base + gic_size;
137
-
vdso_addr = data_addr + PAGE_SIZE;
150
+
vdso_addr = data_addr + VDSO_NR_PAGES * PAGE_SIZE;
138
151
139
-
vma = _install_special_mapping(mm, base, vvar_size,
140
-
VM_READ | VM_MAYREAD,
141
-
&vdso_vvar_mapping);
152
+
vma = vdso_install_vvar_mapping(mm, data_addr);
142
153
if (IS_ERR(vma)) {
143
154
ret = PTR_ERR(vma);
144
155
goto out;
···
146
161
if (gic_size) {
147
162
gic_base = (unsigned long)mips_gic_base + MIPS_GIC_USER_OFS;
148
163
gic_pfn = PFN_DOWN(__pa(gic_base));
164
+
static const struct vm_special_mapping gic_mapping = {
165
+
.name = "[gic]",
166
+
.pages = (struct page **) { NULL },
167
+
};
168
+
169
+
vma = _install_special_mapping(mm, base, gic_size, VM_READ | VM_MAYREAD,
170
+
&gic_mapping);
171
+
if (IS_ERR(vma)) {
172
+
ret = PTR_ERR(vma);
173
+
goto out;
174
+
}
149
175
150
176
ret = io_remap_pfn_range(vma, base, gic_pfn, gic_size,
151
177
pgprot_noncached(vma->vm_page_prot));
152
178
if (ret)
153
179
goto out;
154
180
}
155
-
156
-
/* Map data page. */
157
-
ret = remap_pfn_range(vma, data_addr,
158
-
virt_to_phys(vdso_data) >> PAGE_SHIFT,
159
-
PAGE_SIZE, vma->vm_page_prot);
160
-
if (ret)
161
-
goto out;
162
181
163
182
/* Map VDSO image. */
164
183
vma = _install_special_mapping(mm, vdso_addr, image->size,
+1
-1
arch/mips/vdso/Makefile
+1
-1
arch/mips/vdso/Makefile
+4
-1
arch/mips/vdso/vdso.lds.S
+4
-1
arch/mips/vdso/vdso.lds.S
···
5
5
*/
6
6
7
7
#include <asm/sgidefs.h>
8
+
#include <asm/vdso/vdso.h>
9
+
#include <vdso/datapage.h>
8
10
9
11
#if _MIPS_SIM == _MIPS_SIM_ABI64
10
12
OUTPUT_FORMAT("elf64-tradlittlemips", "elf64-tradbigmips", "elf64-tradlittlemips")
···
20
18
21
19
SECTIONS
22
20
{
23
-
PROVIDE(_start = .);
21
+
VDSO_VVAR_SYMS
22
+
24
23
. = SIZEOF_HEADERS;
25
24
26
25
/*
-2
arch/parisc/include/asm/vdso.h
-2
arch/parisc/include/asm/vdso.h
···
12
12
#define VDSO64_SYMBOL(tsk, name) ((tsk)->mm->context.vdso_base + (vdso64_offset_##name))
13
13
#define VDSO32_SYMBOL(tsk, name) ((tsk)->mm->context.vdso_base + (vdso32_offset_##name))
14
14
15
-
extern struct vdso_data *vdso_data;
16
-
17
15
#endif /* __ASSEMBLY __ */
18
16
19
17
/* Default link addresses for the vDSOs */
+1
-1
arch/parisc/kernel/vdso32/Makefile
+1
-1
arch/parisc/kernel/vdso32/Makefile
+1
-1
arch/parisc/kernel/vdso64/Makefile
+1
-1
arch/parisc/kernel/vdso64/Makefile
+2
arch/powerpc/Kconfig
+2
arch/powerpc/Kconfig
···
159
159
select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
160
160
select ARCH_HAS_UACCESS_FLUSHCACHE
161
161
select ARCH_HAS_UBSAN
162
+
select ARCH_HAS_VDSO_ARCH_DATA
162
163
select ARCH_HAVE_NMI_SAFE_CMPXCHG
163
164
select ARCH_HAVE_EXTRA_ELF_NOTES if SPU_BASE
164
165
select ARCH_KEEP_MEMBLOCK
···
210
209
select GENERIC_PTDUMP
211
210
select GENERIC_SMP_IDLE_THREAD
212
211
select GENERIC_TIME_VSYSCALL
212
+
select GENERIC_VDSO_DATA_STORE
213
213
select GENERIC_VDSO_TIME_NS
214
214
select HAS_IOPORT if PCI
215
215
select HAVE_ARCH_AUDITSYSCALL
+1
arch/powerpc/include/asm/vdso.h
+1
arch/powerpc/include/asm/vdso.h
+37
arch/powerpc/include/asm/vdso/arch_data.h
+37
arch/powerpc/include/asm/vdso/arch_data.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/*
3
+
* Copyright (C) 2002 Peter Bergner <bergner@vnet.ibm.com>, IBM
4
+
* Copyright (C) 2005 Benjamin Herrenschmidy <benh@kernel.crashing.org>,
5
+
* IBM Corp.
6
+
*/
7
+
#ifndef _ASM_POWERPC_VDSO_ARCH_DATA_H
8
+
#define _ASM_POWERPC_VDSO_ARCH_DATA_H
9
+
10
+
#include <linux/unistd.h>
11
+
#include <linux/types.h>
12
+
13
+
#define SYSCALL_MAP_SIZE ((NR_syscalls + 31) / 32)
14
+
15
+
#ifdef CONFIG_PPC64
16
+
17
+
struct vdso_arch_data {
18
+
__u64 tb_ticks_per_sec; /* Timebase tics / sec */
19
+
__u32 dcache_block_size; /* L1 d-cache block size */
20
+
__u32 icache_block_size; /* L1 i-cache block size */
21
+
__u32 dcache_log_block_size; /* L1 d-cache log block size */
22
+
__u32 icache_log_block_size; /* L1 i-cache log block size */
23
+
__u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */
24
+
__u32 compat_syscall_map[SYSCALL_MAP_SIZE]; /* Map of compat syscalls */
25
+
};
26
+
27
+
#else /* CONFIG_PPC64 */
28
+
29
+
struct vdso_arch_data {
30
+
__u64 tb_ticks_per_sec; /* Timebase tics / sec */
31
+
__u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */
32
+
__u32 compat_syscall_map[0]; /* No compat syscalls on PPC32 */
33
+
};
34
+
35
+
#endif /* CONFIG_PPC64 */
36
+
37
+
#endif /* _ASM_POWERPC_VDSO_ARCH_DATA_H */
+6
-5
arch/powerpc/include/asm/vdso/getrandom.h
+6
-5
arch/powerpc/include/asm/vdso/getrandom.h
···
43
43
(unsigned long)len, (unsigned long)flags);
44
44
}
45
45
46
-
static __always_inline struct vdso_rng_data *__arch_get_vdso_rng_data(void)
46
+
static __always_inline const struct vdso_rng_data *__arch_get_vdso_u_rng_data(void)
47
47
{
48
-
struct vdso_arch_data *data;
48
+
struct vdso_rng_data *data;
49
49
50
50
asm (
51
51
" bcl 20, 31, .+4 ;"
52
52
"0: mflr %0 ;"
53
-
" addis %0, %0, (_vdso_datapage - 0b)@ha ;"
54
-
" addi %0, %0, (_vdso_datapage - 0b)@l ;"
53
+
" addis %0, %0, (vdso_u_rng_data - 0b)@ha ;"
54
+
" addi %0, %0, (vdso_u_rng_data - 0b)@l ;"
55
55
: "=r" (data) : : "lr"
56
56
);
57
57
58
-
return &data->rng_data;
58
+
return data;
59
59
}
60
+
#define __arch_get_vdso_u_rng_data __arch_get_vdso_u_rng_data
60
61
61
62
ssize_t __c_kernel_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state,
62
63
size_t opaque_len);
+10
-19
arch/powerpc/include/asm/vdso/gettimeofday.h
+10
-19
arch/powerpc/include/asm/vdso/gettimeofday.h
···
94
94
#endif
95
95
96
96
static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
97
-
const struct vdso_data *vd)
97
+
const struct vdso_time_data *vd)
98
98
{
99
99
return get_tb();
100
100
}
101
101
102
-
const struct vdso_data *__arch_get_vdso_data(void);
103
-
104
-
#ifdef CONFIG_TIME_NS
105
-
static __always_inline
106
-
const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
107
-
{
108
-
return (void *)vd + (1U << CONFIG_PAGE_SHIFT);
109
-
}
110
-
#endif
111
-
112
-
static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
102
+
static inline bool vdso_clocksource_ok(const struct vdso_clock *vc)
113
103
{
114
104
return true;
115
105
}
···
125
135
126
136
#ifdef __powerpc64__
127
137
int __c_kernel_clock_gettime(clockid_t clock, struct __kernel_timespec *ts,
128
-
const struct vdso_data *vd);
138
+
const struct vdso_time_data *vd);
129
139
int __c_kernel_clock_getres(clockid_t clock_id, struct __kernel_timespec *res,
130
-
const struct vdso_data *vd);
140
+
const struct vdso_time_data *vd);
131
141
#else
132
142
int __c_kernel_clock_gettime(clockid_t clock, struct old_timespec32 *ts,
133
-
const struct vdso_data *vd);
143
+
const struct vdso_time_data *vd);
134
144
int __c_kernel_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts,
135
-
const struct vdso_data *vd);
145
+
const struct vdso_time_data *vd);
136
146
int __c_kernel_clock_getres(clockid_t clock_id, struct old_timespec32 *res,
137
-
const struct vdso_data *vd);
147
+
const struct vdso_time_data *vd);
138
148
#endif
139
149
int __c_kernel_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz,
140
-
const struct vdso_data *vd);
150
+
const struct vdso_time_data *vd);
141
151
__kernel_old_time_t __c_kernel_time(__kernel_old_time_t *time,
142
-
const struct vdso_data *vd);
152
+
const struct vdso_time_data *vd);
153
+
143
154
#endif /* __ASSEMBLY__ */
144
155
145
156
#endif /* _ASM_POWERPC_VDSO_GETTIMEOFDAY_H */
-13
arch/powerpc/include/asm/vdso/vsyscall.h
-13
arch/powerpc/include/asm/vdso/vsyscall.h
···
6
6
7
7
#include <asm/vdso_datapage.h>
8
8
9
-
static __always_inline
10
-
struct vdso_data *__arch_get_k_vdso_data(void)
11
-
{
12
-
return vdso_data->data;
13
-
}
14
-
#define __arch_get_k_vdso_data __arch_get_k_vdso_data
15
-
16
-
static __always_inline
17
-
struct vdso_rng_data *__arch_get_k_vdso_rng_data(void)
18
-
{
19
-
return &vdso_data->rng_data;
20
-
}
21
-
22
9
/* The asm-generic header needs to be included after the definitions above */
23
10
#include <asm-generic/vdso/vsyscall.h>
24
11
+3
-41
arch/powerpc/include/asm/vdso_datapage.h
+3
-41
arch/powerpc/include/asm/vdso_datapage.h
···
11
11
12
12
#ifndef __ASSEMBLY__
13
13
14
-
#include <linux/unistd.h>
15
-
#include <linux/time.h>
16
14
#include <vdso/datapage.h>
17
-
18
-
#define SYSCALL_MAP_SIZE ((NR_syscalls + 31) / 32)
19
-
20
-
#ifdef CONFIG_PPC64
21
-
22
-
struct vdso_arch_data {
23
-
__u64 tb_ticks_per_sec; /* Timebase tics / sec */
24
-
__u32 dcache_block_size; /* L1 d-cache block size */
25
-
__u32 icache_block_size; /* L1 i-cache block size */
26
-
__u32 dcache_log_block_size; /* L1 d-cache log block size */
27
-
__u32 icache_log_block_size; /* L1 i-cache log block size */
28
-
__u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */
29
-
__u32 compat_syscall_map[SYSCALL_MAP_SIZE]; /* Map of compat syscalls */
30
-
31
-
struct vdso_rng_data rng_data;
32
-
33
-
struct vdso_data data[CS_BASES] __aligned(1 << CONFIG_PAGE_SHIFT);
34
-
};
35
-
36
-
#else /* CONFIG_PPC64 */
37
-
38
-
struct vdso_arch_data {
39
-
__u64 tb_ticks_per_sec; /* Timebase tics / sec */
40
-
__u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */
41
-
__u32 compat_syscall_map[0]; /* No compat syscalls on PPC32 */
42
-
struct vdso_rng_data rng_data;
43
-
44
-
struct vdso_data data[CS_BASES] __aligned(1 << CONFIG_PAGE_SHIFT);
45
-
};
46
-
47
-
#endif /* CONFIG_PPC64 */
48
-
49
-
extern struct vdso_arch_data *vdso_data;
50
15
51
16
#else /* __ASSEMBLY__ */
52
17
53
-
.macro get_datapage ptr offset=0
18
+
.macro get_datapage ptr symbol
54
19
bcl 20, 31, .+4
55
20
999:
56
21
mflr \ptr
57
-
addis \ptr, \ptr, (_vdso_datapage - 999b + \offset)@ha
58
-
addi \ptr, \ptr, (_vdso_datapage - 999b + \offset)@l
22
+
addis \ptr, \ptr, (\symbol - 999b)@ha
23
+
addi \ptr, \ptr, (\symbol - 999b)@l
59
24
.endm
60
-
61
-
#include <asm/asm-offsets.h>
62
-
#include <asm/page.h>
63
25
64
26
#endif /* __ASSEMBLY__ */
65
27
-1
arch/powerpc/kernel/asm-offsets.c
-1
arch/powerpc/kernel/asm-offsets.c
···
334
334
#endif /* ! CONFIG_PPC64 */
335
335
336
336
/* datapage offsets for use by vdso */
337
-
OFFSET(VDSO_DATA_OFFSET, vdso_arch_data, data);
338
337
OFFSET(CFG_TB_TICKS_PER_SEC, vdso_arch_data, tb_ticks_per_sec);
339
338
#ifdef CONFIG_PPC64
340
339
OFFSET(CFG_ICACHE_BLOCKSZ, vdso_arch_data, icache_block_size);
+1
-1
arch/powerpc/kernel/time.c
+1
-1
arch/powerpc/kernel/time.c
+11
-104
arch/powerpc/kernel/vdso.c
+11
-104
arch/powerpc/kernel/vdso.c
···
17
17
#include <linux/elf.h>
18
18
#include <linux/security.h>
19
19
#include <linux/syscalls.h>
20
-
#include <linux/time_namespace.h>
20
+
#include <linux/vdso_datastore.h>
21
21
#include <vdso/datapage.h>
22
22
23
23
#include <asm/syscall.h>
···
32
32
#include <asm/vdso_datapage.h>
33
33
#include <asm/setup.h>
34
34
35
+
static_assert(__VDSO_PAGES == VDSO_NR_PAGES);
36
+
35
37
/* The alignment of the vDSO */
36
38
#define VDSO_ALIGNMENT (1 << 16)
37
39
···
41
39
extern char vdso64_start, vdso64_end;
42
40
43
41
long sys_ni_syscall(void);
44
-
45
-
/*
46
-
* The vdso data page (aka. systemcfg for old ppc64 fans) is here.
47
-
* Once the early boot kernel code no longer needs to muck around
48
-
* with it, it will become dynamically allocated
49
-
*/
50
-
static union {
51
-
struct vdso_arch_data data;
52
-
u8 page[2 * PAGE_SIZE];
53
-
} vdso_data_store __page_aligned_data;
54
-
struct vdso_arch_data *vdso_data = &vdso_data_store.data;
55
-
56
-
enum vvar_pages {
57
-
VVAR_BASE_PAGE_OFFSET,
58
-
VVAR_TIME_PAGE_OFFSET,
59
-
VVAR_TIMENS_PAGE_OFFSET,
60
-
VVAR_NR_PAGES,
61
-
};
62
42
63
43
static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma,
64
44
unsigned long text_size)
···
80
96
mm->context.vdso = NULL;
81
97
}
82
98
83
-
static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
84
-
struct vm_area_struct *vma, struct vm_fault *vmf);
85
-
86
-
static struct vm_special_mapping vvar_spec __ro_after_init = {
87
-
.name = "[vvar]",
88
-
.fault = vvar_fault,
89
-
};
90
-
91
99
static struct vm_special_mapping vdso32_spec __ro_after_init = {
92
100
.name = "[vdso]",
93
101
.mremap = vdso32_mremap,
···
92
116
.close = vdso_close,
93
117
};
94
118
95
-
#ifdef CONFIG_TIME_NS
96
-
struct vdso_data *arch_get_vdso_data(void *vvar_page)
97
-
{
98
-
return vvar_page;
99
-
}
100
-
101
-
/*
102
-
* The vvar mapping contains data for a specific time namespace, so when a task
103
-
* changes namespace we must unmap its vvar data for the old namespace.
104
-
* Subsequent faults will map in data for the new namespace.
105
-
*
106
-
* For more details see timens_setup_vdso_data().
107
-
*/
108
-
int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
109
-
{
110
-
struct mm_struct *mm = task->mm;
111
-
VMA_ITERATOR(vmi, mm, 0);
112
-
struct vm_area_struct *vma;
113
-
114
-
mmap_read_lock(mm);
115
-
for_each_vma(vmi, vma) {
116
-
if (vma_is_special_mapping(vma, &vvar_spec))
117
-
zap_vma_pages(vma);
118
-
}
119
-
mmap_read_unlock(mm);
120
-
121
-
return 0;
122
-
}
123
-
#endif
124
-
125
-
static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
126
-
struct vm_area_struct *vma, struct vm_fault *vmf)
127
-
{
128
-
struct page *timens_page = find_timens_vvar_page(vma);
129
-
unsigned long pfn;
130
-
131
-
switch (vmf->pgoff) {
132
-
case VVAR_BASE_PAGE_OFFSET:
133
-
pfn = virt_to_pfn(vdso_data);
134
-
break;
135
-
case VVAR_TIME_PAGE_OFFSET:
136
-
if (timens_page)
137
-
pfn = page_to_pfn(timens_page);
138
-
else
139
-
pfn = virt_to_pfn(vdso_data->data);
140
-
break;
141
-
#ifdef CONFIG_TIME_NS
142
-
case VVAR_TIMENS_PAGE_OFFSET:
143
-
/*
144
-
* If a task belongs to a time namespace then a namespace
145
-
* specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
146
-
* the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
147
-
* offset.
148
-
* See also the comment near timens_setup_vdso_data().
149
-
*/
150
-
if (!timens_page)
151
-
return VM_FAULT_SIGBUS;
152
-
pfn = virt_to_pfn(vdso_data->data);
153
-
break;
154
-
#endif /* CONFIG_TIME_NS */
155
-
default:
156
-
return VM_FAULT_SIGBUS;
157
-
}
158
-
159
-
return vmf_insert_pfn(vma, vmf->address, pfn);
160
-
}
161
-
162
119
/*
163
120
* This is called from binfmt_elf, we create the special vma for the
164
121
* vDSO and insert it into the mm struct tree
···
100
191
{
101
192
unsigned long vdso_size, vdso_base, mappings_size;
102
193
struct vm_special_mapping *vdso_spec;
103
-
unsigned long vvar_size = VVAR_NR_PAGES * PAGE_SIZE;
194
+
unsigned long vvar_size = VDSO_NR_PAGES * PAGE_SIZE;
104
195
struct mm_struct *mm = current->mm;
105
196
struct vm_area_struct *vma;
106
197
···
126
217
/* Add required alignment. */
127
218
vdso_base = ALIGN(vdso_base, VDSO_ALIGNMENT);
128
219
129
-
vma = _install_special_mapping(mm, vdso_base, vvar_size,
130
-
VM_READ | VM_MAYREAD | VM_IO |
131
-
VM_DONTDUMP | VM_PFNMAP, &vvar_spec);
220
+
vma = vdso_install_vvar_mapping(mm, vdso_base);
132
221
if (IS_ERR(vma))
133
222
return PTR_ERR(vma);
134
223
···
206
299
207
300
for (i = 0; i < NR_syscalls; i++) {
208
301
if (sys_call_table[i] != (void *)&sys_ni_syscall)
209
-
vdso_data->syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f);
302
+
vdso_k_arch_data->syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f);
210
303
if (IS_ENABLED(CONFIG_COMPAT) &&
211
304
compat_sys_call_table[i] != (void *)&sys_ni_syscall)
212
-
vdso_data->compat_syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f);
305
+
vdso_k_arch_data->compat_syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f);
213
306
}
214
307
}
215
308
···
259
352
static int __init vdso_init(void)
260
353
{
261
354
#ifdef CONFIG_PPC64
262
-
vdso_data->dcache_block_size = ppc64_caches.l1d.block_size;
263
-
vdso_data->icache_block_size = ppc64_caches.l1i.block_size;
264
-
vdso_data->dcache_log_block_size = ppc64_caches.l1d.log_block_size;
265
-
vdso_data->icache_log_block_size = ppc64_caches.l1i.log_block_size;
355
+
vdso_k_arch_data->dcache_block_size = ppc64_caches.l1d.block_size;
356
+
vdso_k_arch_data->icache_block_size = ppc64_caches.l1i.block_size;
357
+
vdso_k_arch_data->dcache_log_block_size = ppc64_caches.l1d.log_block_size;
358
+
vdso_k_arch_data->icache_log_block_size = ppc64_caches.l1i.log_block_size;
266
359
#endif /* CONFIG_PPC64 */
267
360
268
361
vdso_setup_syscall_map();
+1
-1
arch/powerpc/kernel/vdso/Makefile
+1
-1
arch/powerpc/kernel/vdso/Makefile
···
3
3
# List of files in the vdso, has to be asm only for now
4
4
5
5
# Include the generic Makefile to check the built vdso.
6
-
include $(srctree)/lib/vdso/Makefile
6
+
include $(srctree)/lib/vdso/Makefile.include
7
7
8
8
obj-vdso32 = sigtramp32-32.o gettimeofday-32.o datapage-32.o cacheflush-32.o note-32.o getcpu-32.o
9
9
obj-vdso64 = sigtramp64-64.o gettimeofday-64.o datapage-64.o cacheflush-64.o note-64.o getcpu-64.o
+1
-1
arch/powerpc/kernel/vdso/cacheflush.S
+1
-1
arch/powerpc/kernel/vdso/cacheflush.S
+2
-2
arch/powerpc/kernel/vdso/datapage.S
+2
-2
arch/powerpc/kernel/vdso/datapage.S
···
28
28
mflr r12
29
29
.cfi_register lr,r12
30
30
mr. r4,r3
31
-
get_datapage r3
31
+
get_datapage r3 vdso_u_arch_data
32
32
mtlr r12
33
33
#ifdef __powerpc64__
34
34
addi r3,r3,CFG_SYSCALL_MAP64
···
52
52
.cfi_startproc
53
53
mflr r12
54
54
.cfi_register lr,r12
55
-
get_datapage r3
55
+
get_datapage r3 vdso_u_arch_data
56
56
#ifndef __powerpc64__
57
57
lwz r4,(CFG_TB_TICKS_PER_SEC + 4)(r3)
58
58
#endif
+2
-2
arch/powerpc/kernel/vdso/gettimeofday.S
+2
-2
arch/powerpc/kernel/vdso/gettimeofday.S
···
33
33
.cfi_rel_offset r2, PPC_MIN_STKFRM + STK_GOT
34
34
#endif
35
35
.ifeq \call_time
36
-
get_datapage r5 VDSO_DATA_OFFSET
36
+
get_datapage r5 vdso_u_time_data
37
37
.else
38
-
get_datapage r4 VDSO_DATA_OFFSET
38
+
get_datapage r4 vdso_u_time_data
39
39
.endif
40
40
bl CFUNC(DOTSYM(\funct))
41
41
PPC_LL r0, PPC_MIN_STKFRM + PPC_LR_STKOFF(r1)
+3
-1
arch/powerpc/kernel/vdso/vdso32.lds.S
+3
-1
arch/powerpc/kernel/vdso/vdso32.lds.S
···
6
6
#include <asm/vdso.h>
7
7
#include <asm/page.h>
8
8
#include <asm-generic/vmlinux.lds.h>
9
+
#include <vdso/datapage.h>
9
10
10
11
#ifdef __LITTLE_ENDIAN__
11
12
OUTPUT_FORMAT("elf32-powerpcle", "elf32-powerpcle", "elf32-powerpcle")
···
17
16
18
17
SECTIONS
19
18
{
20
-
PROVIDE(_vdso_datapage = . - 3 * PAGE_SIZE);
19
+
VDSO_VVAR_SYMS
20
+
21
21
. = SIZEOF_HEADERS;
22
22
23
23
.hash : { *(.hash) } :text
+3
-1
arch/powerpc/kernel/vdso/vdso64.lds.S
+3
-1
arch/powerpc/kernel/vdso/vdso64.lds.S
···
6
6
#include <asm/vdso.h>
7
7
#include <asm/page.h>
8
8
#include <asm-generic/vmlinux.lds.h>
9
+
#include <vdso/datapage.h>
9
10
10
11
#ifdef __LITTLE_ENDIAN__
11
12
OUTPUT_FORMAT("elf64-powerpcle", "elf64-powerpcle", "elf64-powerpcle")
···
17
16
18
17
SECTIONS
19
18
{
20
-
PROVIDE(_vdso_datapage = . - 3 * PAGE_SIZE);
19
+
VDSO_VVAR_SYMS
20
+
21
21
. = SIZEOF_HEADERS;
22
22
23
23
.hash : { *(.hash) } :text
+7
-7
arch/powerpc/kernel/vdso/vgettimeofday.c
+7
-7
arch/powerpc/kernel/vdso/vgettimeofday.c
···
7
7
8
8
#ifdef __powerpc64__
9
9
int __c_kernel_clock_gettime(clockid_t clock, struct __kernel_timespec *ts,
10
-
const struct vdso_data *vd)
10
+
const struct vdso_time_data *vd)
11
11
{
12
12
return __cvdso_clock_gettime_data(vd, clock, ts);
13
13
}
14
14
15
15
int __c_kernel_clock_getres(clockid_t clock_id, struct __kernel_timespec *res,
16
-
const struct vdso_data *vd)
16
+
const struct vdso_time_data *vd)
17
17
{
18
18
return __cvdso_clock_getres_data(vd, clock_id, res);
19
19
}
20
20
#else
21
21
int __c_kernel_clock_gettime(clockid_t clock, struct old_timespec32 *ts,
22
-
const struct vdso_data *vd)
22
+
const struct vdso_time_data *vd)
23
23
{
24
24
return __cvdso_clock_gettime32_data(vd, clock, ts);
25
25
}
26
26
27
27
int __c_kernel_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts,
28
-
const struct vdso_data *vd)
28
+
const struct vdso_time_data *vd)
29
29
{
30
30
return __cvdso_clock_gettime_data(vd, clock, ts);
31
31
}
32
32
33
33
int __c_kernel_clock_getres(clockid_t clock_id, struct old_timespec32 *res,
34
-
const struct vdso_data *vd)
34
+
const struct vdso_time_data *vd)
35
35
{
36
36
return __cvdso_clock_getres_time32_data(vd, clock_id, res);
37
37
}
38
38
#endif
39
39
40
40
int __c_kernel_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz,
41
-
const struct vdso_data *vd)
41
+
const struct vdso_time_data *vd)
42
42
{
43
43
return __cvdso_gettimeofday_data(vd, tv, tz);
44
44
}
45
45
46
-
__kernel_old_time_t __c_kernel_time(__kernel_old_time_t *time, const struct vdso_data *vd)
46
+
__kernel_old_time_t __c_kernel_time(__kernel_old_time_t *time, const struct vdso_time_data *vd)
47
47
{
48
48
return __cvdso_time_data(vd, time);
49
49
}
+2
-1
arch/riscv/Kconfig
+2
-1
arch/riscv/Kconfig
···
53
53
select ARCH_HAS_SYSCALL_WRAPPER
54
54
select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
55
55
select ARCH_HAS_UBSAN
56
-
select ARCH_HAS_VDSO_TIME_DATA
56
+
select ARCH_HAS_VDSO_ARCH_DATA if GENERIC_VDSO_DATA_STORE
57
57
select ARCH_KEEP_MEMBLOCK if ACPI
58
58
select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE if 64BIT && MMU
59
59
select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX
···
117
117
select GENERIC_SCHED_CLOCK
118
118
select GENERIC_SMP_IDLE_THREAD
119
119
select GENERIC_TIME_VSYSCALL if MMU && 64BIT
120
+
select GENERIC_VDSO_DATA_STORE if MMU
120
121
select GENERIC_VDSO_TIME_NS if HAVE_GENERIC_VDSO
121
122
select HARDIRQS_SW_RESEND
122
123
select HAS_IOPORT if MMU
+1
-1
arch/riscv/include/asm/vdso.h
+1
-1
arch/riscv/include/asm/vdso.h
+1
-13
arch/riscv/include/asm/vdso/gettimeofday.h
+1
-13
arch/riscv/include/asm/vdso/gettimeofday.h
···
69
69
#endif /* CONFIG_GENERIC_TIME_VSYSCALL */
70
70
71
71
static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
72
-
const struct vdso_data *vd)
72
+
const struct vdso_time_data *vd)
73
73
{
74
74
/*
75
75
* The purpose of csr_read(CSR_TIME) is to trap the system into
···
79
79
return csr_read(CSR_TIME);
80
80
}
81
81
82
-
static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
83
-
{
84
-
return _vdso_data;
85
-
}
86
-
87
-
#ifdef CONFIG_TIME_NS
88
-
static __always_inline
89
-
const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
90
-
{
91
-
return _timens_data;
92
-
}
93
-
#endif
94
82
#endif /* !__ASSEMBLY__ */
95
83
96
84
#endif /* __ASM_VDSO_GETTIMEOFDAY_H */
+4
-4
arch/riscv/include/asm/vdso/time_data.h
arch/riscv/include/asm/vdso/arch_data.h
+4
-4
arch/riscv/include/asm/vdso/time_data.h
arch/riscv/include/asm/vdso/arch_data.h
···
1
1
/* SPDX-License-Identifier: GPL-2.0 */
2
-
#ifndef __RISCV_ASM_VDSO_TIME_DATA_H
3
-
#define __RISCV_ASM_VDSO_TIME_DATA_H
2
+
#ifndef __RISCV_ASM_VDSO_ARCH_DATA_H
3
+
#define __RISCV_ASM_VDSO_ARCH_DATA_H
4
4
5
5
#include <linux/types.h>
6
6
#include <vdso/datapage.h>
7
7
#include <asm/hwprobe.h>
8
8
9
-
struct arch_vdso_time_data {
9
+
struct vdso_arch_data {
10
10
/* Stash static answers to the hwprobe queries when all CPUs are selected. */
11
11
__u64 all_cpu_hwprobe_values[RISCV_HWPROBE_MAX_KEY + 1];
12
12
···
14
14
__u8 homogeneous_cpus;
15
15
};
16
16
17
-
#endif /* __RISCV_ASM_VDSO_TIME_DATA_H */
17
+
#endif /* __RISCV_ASM_VDSO_ARCH_DATA_H */
-9
arch/riscv/include/asm/vdso/vsyscall.h
-9
arch/riscv/include/asm/vdso/vsyscall.h
···
6
6
7
7
#include <vdso/datapage.h>
8
8
9
-
extern struct vdso_data *vdso_data;
10
-
11
-
static __always_inline struct vdso_data *__riscv_get_k_vdso_data(void)
12
-
{
13
-
return vdso_data;
14
-
}
15
-
16
-
#define __arch_get_k_vdso_data __riscv_get_k_vdso_data
17
-
18
9
/* The asm-generic header needs to be included after the definitions above */
19
10
#include <asm-generic/vdso/vsyscall.h>
20
11
+1
-2
arch/riscv/kernel/sys_hwprobe.c
+1
-2
arch/riscv/kernel/sys_hwprobe.c
···
450
450
451
451
static int __init init_hwprobe_vdso_data(void)
452
452
{
453
-
struct vdso_data *vd = __arch_get_k_vdso_data();
454
-
struct arch_vdso_time_data *avd = &vd->arch_data;
453
+
struct vdso_arch_data *avd = vdso_k_arch_data;
455
454
u64 id_bitsmash = 0;
456
455
struct riscv_hwprobe pair;
457
456
int key;
+4
-86
arch/riscv/kernel/vdso.c
+4
-86
arch/riscv/kernel/vdso.c
···
13
13
#include <linux/err.h>
14
14
#include <asm/page.h>
15
15
#include <asm/vdso.h>
16
-
#include <linux/time_namespace.h>
16
+
#include <linux/vdso_datastore.h>
17
17
#include <vdso/datapage.h>
18
18
#include <vdso/vsyscall.h>
19
19
20
-
enum vvar_pages {
21
-
VVAR_DATA_PAGE_OFFSET,
22
-
VVAR_TIMENS_PAGE_OFFSET,
23
-
VVAR_NR_PAGES,
24
-
};
25
-
26
-
#define VVAR_SIZE (VVAR_NR_PAGES << PAGE_SHIFT)
27
-
28
-
static union vdso_data_store vdso_data_store __page_aligned_data;
29
-
struct vdso_data *vdso_data = vdso_data_store.data;
20
+
#define VVAR_SIZE (VDSO_NR_PAGES << PAGE_SHIFT)
30
21
31
22
struct __vdso_info {
32
23
const char *name;
···
70
79
vdso_info->cm->pages = vdso_pagelist;
71
80
}
72
81
73
-
#ifdef CONFIG_TIME_NS
74
-
struct vdso_data *arch_get_vdso_data(void *vvar_page)
75
-
{
76
-
return (struct vdso_data *)(vvar_page);
77
-
}
78
-
79
-
static const struct vm_special_mapping rv_vvar_map;
80
-
81
-
/*
82
-
* The vvar mapping contains data for a specific time namespace, so when a task
83
-
* changes namespace we must unmap its vvar data for the old namespace.
84
-
* Subsequent faults will map in data for the new namespace.
85
-
*
86
-
* For more details see timens_setup_vdso_data().
87
-
*/
88
-
int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
89
-
{
90
-
struct mm_struct *mm = task->mm;
91
-
struct vm_area_struct *vma;
92
-
VMA_ITERATOR(vmi, mm, 0);
93
-
94
-
mmap_read_lock(mm);
95
-
96
-
for_each_vma(vmi, vma) {
97
-
if (vma_is_special_mapping(vma, &rv_vvar_map))
98
-
zap_vma_pages(vma);
99
-
}
100
-
101
-
mmap_read_unlock(mm);
102
-
return 0;
103
-
}
104
-
#endif
105
-
106
-
static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
107
-
struct vm_area_struct *vma, struct vm_fault *vmf)
108
-
{
109
-
struct page *timens_page = find_timens_vvar_page(vma);
110
-
unsigned long pfn;
111
-
112
-
switch (vmf->pgoff) {
113
-
case VVAR_DATA_PAGE_OFFSET:
114
-
if (timens_page)
115
-
pfn = page_to_pfn(timens_page);
116
-
else
117
-
pfn = sym_to_pfn(vdso_data);
118
-
break;
119
-
#ifdef CONFIG_TIME_NS
120
-
case VVAR_TIMENS_PAGE_OFFSET:
121
-
/*
122
-
* If a task belongs to a time namespace then a namespace
123
-
* specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
124
-
* the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
125
-
* offset.
126
-
* See also the comment near timens_setup_vdso_data().
127
-
*/
128
-
if (!timens_page)
129
-
return VM_FAULT_SIGBUS;
130
-
pfn = sym_to_pfn(vdso_data);
131
-
break;
132
-
#endif /* CONFIG_TIME_NS */
133
-
default:
134
-
return VM_FAULT_SIGBUS;
135
-
}
136
-
137
-
return vmf_insert_pfn(vma, vmf->address, pfn);
138
-
}
139
-
140
-
static const struct vm_special_mapping rv_vvar_map = {
141
-
.name = "[vvar]",
142
-
.fault = vvar_fault,
143
-
};
144
-
145
82
static struct vm_special_mapping rv_vdso_map __ro_after_init = {
146
83
.name = "[vdso]",
147
84
.mremap = vdso_mremap,
···
115
196
unsigned long vdso_base, vdso_text_len, vdso_mapping_len;
116
197
void *ret;
117
198
118
-
BUILD_BUG_ON(VVAR_NR_PAGES != __VVAR_PAGES);
199
+
BUILD_BUG_ON(VDSO_NR_PAGES != __VDSO_PAGES);
119
200
120
201
vdso_text_len = vdso_info->vdso_pages << PAGE_SHIFT;
121
202
/* Be sure to map the data page */
···
127
208
goto up_fail;
128
209
}
129
210
130
-
ret = _install_special_mapping(mm, vdso_base, VVAR_SIZE,
131
-
(VM_READ | VM_MAYREAD | VM_PFNMAP), &rv_vvar_map);
211
+
ret = vdso_install_vvar_mapping(mm, vdso_base);
132
212
if (IS_ERR(ret))
133
213
goto up_fail;
134
214
+1
-1
arch/riscv/kernel/vdso/Makefile
+1
-1
arch/riscv/kernel/vdso/Makefile
+2
-4
arch/riscv/kernel/vdso/hwprobe.c
+2
-4
arch/riscv/kernel/vdso/hwprobe.c
···
16
16
size_t cpusetsize, unsigned long *cpus,
17
17
unsigned int flags)
18
18
{
19
-
const struct vdso_data *vd = __arch_get_vdso_data();
20
-
const struct arch_vdso_time_data *avd = &vd->arch_data;
19
+
const struct vdso_arch_data *avd = &vdso_u_arch_data;
21
20
bool all_cpus = !cpusetsize && !cpus;
22
21
struct riscv_hwprobe *p = pairs;
23
22
struct riscv_hwprobe *end = pairs + pair_count;
···
50
51
size_t cpusetsize, unsigned long *cpus,
51
52
unsigned int flags)
52
53
{
53
-
const struct vdso_data *vd = __arch_get_vdso_data();
54
-
const struct arch_vdso_time_data *avd = &vd->arch_data;
54
+
const struct vdso_arch_data *avd = &vdso_u_arch_data;
55
55
struct riscv_hwprobe *p = pairs;
56
56
struct riscv_hwprobe *end = pairs + pair_count;
57
57
unsigned char *c = (unsigned char *)cpus;
+3
-4
arch/riscv/kernel/vdso/vdso.lds.S
+3
-4
arch/riscv/kernel/vdso/vdso.lds.S
···
4
4
*/
5
5
#include <asm/page.h>
6
6
#include <asm/vdso.h>
7
+
#include <vdso/datapage.h>
7
8
8
9
OUTPUT_ARCH(riscv)
9
10
10
11
SECTIONS
11
12
{
12
-
PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
13
-
#ifdef CONFIG_TIME_NS
14
-
PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
15
-
#endif
13
+
VDSO_VVAR_SYMS
14
+
16
15
. = SIZEOF_HEADERS;
17
16
18
17
.hash : { *(.hash) } :text
+1
arch/s390/Kconfig
+1
arch/s390/Kconfig
+1
-3
arch/s390/include/asm/vdso.h
+1
-3
arch/s390/include/asm/vdso.h
-12
arch/s390/include/asm/vdso/getrandom.h
-12
arch/s390/include/asm/vdso/getrandom.h
···
23
23
return syscall3(__NR_getrandom, (long)buffer, (long)len, (long)flags);
24
24
}
25
25
26
-
static __always_inline const struct vdso_rng_data *__arch_get_vdso_rng_data(void)
27
-
{
28
-
/*
29
-
* The RNG data is in the real VVAR data page, but if a task belongs to a time namespace
30
-
* then VVAR_DATA_PAGE_OFFSET points to the namespace-specific VVAR page and VVAR_TIMENS_
31
-
* PAGE_OFFSET points to the real VVAR page.
32
-
*/
33
-
if (IS_ENABLED(CONFIG_TIME_NS) && _vdso_data->clock_mode == VDSO_CLOCKMODE_TIMENS)
34
-
return (void *)&_vdso_rng_data + VVAR_TIMENS_PAGE_OFFSET * PAGE_SIZE;
35
-
return &_vdso_rng_data;
36
-
}
37
-
38
26
#endif /* !__ASSEMBLY__ */
39
27
40
28
#endif /* __ASM_VDSO_GETRANDOM_H */
+1
-14
arch/s390/include/asm/vdso/gettimeofday.h
+1
-14
arch/s390/include/asm/vdso/gettimeofday.h
···
14
14
#include <linux/compiler.h>
15
15
16
16
17
-
static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
18
-
{
19
-
return _vdso_data;
20
-
}
21
-
22
-
static inline u64 __arch_get_hw_counter(s32 clock_mode, const struct vdso_data *vd)
17
+
static inline u64 __arch_get_hw_counter(s32 clock_mode, const struct vdso_time_data *vd)
23
18
{
24
19
u64 adj, now;
25
20
···
43
48
{
44
49
return syscall2(__NR_clock_getres, (long)clkid, (long)ts);
45
50
}
46
-
47
-
#ifdef CONFIG_TIME_NS
48
-
static __always_inline
49
-
const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
50
-
{
51
-
return _timens_data;
52
-
}
53
-
#endif
54
51
55
52
#endif
-20
arch/s390/include/asm/vdso/vsyscall.h
-20
arch/s390/include/asm/vdso/vsyscall.h
···
2
2
#ifndef __ASM_VDSO_VSYSCALL_H
3
3
#define __ASM_VDSO_VSYSCALL_H
4
4
5
-
#define __VDSO_RND_DATA_OFFSET 768
6
-
7
5
#ifndef __ASSEMBLY__
8
6
9
7
#include <linux/hrtimer.h>
10
8
#include <vdso/datapage.h>
11
9
#include <asm/vdso.h>
12
-
13
-
enum vvar_pages {
14
-
VVAR_DATA_PAGE_OFFSET,
15
-
VVAR_TIMENS_PAGE_OFFSET,
16
-
VVAR_NR_PAGES
17
-
};
18
-
19
-
static __always_inline struct vdso_data *__s390_get_k_vdso_data(void)
20
-
{
21
-
return vdso_data;
22
-
}
23
-
#define __arch_get_k_vdso_data __s390_get_k_vdso_data
24
-
25
-
static __always_inline struct vdso_rng_data *__s390_get_k_vdso_rnd_data(void)
26
-
{
27
-
return (void *)vdso_data + __VDSO_RND_DATA_OFFSET;
28
-
}
29
-
#define __arch_get_k_vdso_rng_data __s390_get_k_vdso_rnd_data
30
10
31
11
/* The asm-generic header needs to be included after the definitions above */
32
12
#include <asm-generic/vdso/vsyscall.h>
+3
-8
arch/s390/kernel/time.c
+3
-8
arch/s390/kernel/time.c
···
79
79
{
80
80
struct ptff_qto qto;
81
81
struct ptff_qui qui;
82
-
int cs;
83
82
84
83
/* Initialize TOD steering parameters */
85
84
tod_steering_end = tod_clock_base.tod;
86
-
for (cs = 0; cs < CS_BASES; cs++)
87
-
vdso_data[cs].arch_data.tod_steering_end = tod_steering_end;
85
+
vdso_k_time_data->arch_data.tod_steering_end = tod_steering_end;
88
86
89
87
if (!test_facility(28))
90
88
return;
···
371
373
{
372
374
unsigned long now, adj;
373
375
struct ptff_qto qto;
374
-
int cs;
375
376
376
377
/* Fixup the monotonic sched clock. */
377
378
tod_clock_base.eitod += delta;
···
386
389
panic("TOD clock sync offset %li is too large to drift\n",
387
390
tod_steering_delta);
388
391
tod_steering_end = now + (abs(tod_steering_delta) << 15);
389
-
for (cs = 0; cs < CS_BASES; cs++) {
390
-
vdso_data[cs].arch_data.tod_steering_end = tod_steering_end;
391
-
vdso_data[cs].arch_data.tod_steering_delta = tod_steering_delta;
392
-
}
392
+
vdso_k_time_data->arch_data.tod_steering_end = tod_steering_end;
393
+
vdso_k_time_data->arch_data.tod_steering_delta = tod_steering_delta;
393
394
394
395
/* Update LPAR offset. */
395
396
if (ptff_query(PTFF_QTO) && ptff(&qto, sizeof(qto), PTFF_QTO) == 0)
+5
-92
arch/s390/kernel/vdso.c
+5
-92
arch/s390/kernel/vdso.c
···
16
16
#include <linux/mm.h>
17
17
#include <linux/slab.h>
18
18
#include <linux/smp.h>
19
-
#include <linux/time_namespace.h>
20
19
#include <linux/random.h>
20
+
#include <linux/vdso_datastore.h>
21
21
#include <vdso/datapage.h>
22
22
#include <asm/vdso/vsyscall.h>
23
23
#include <asm/alternative.h>
···
26
26
extern char vdso64_start[], vdso64_end[];
27
27
extern char vdso32_start[], vdso32_end[];
28
28
29
-
static struct vm_special_mapping vvar_mapping;
30
-
31
-
static union vdso_data_store vdso_data_store __page_aligned_data;
32
-
33
-
struct vdso_data *vdso_data = vdso_data_store.data;
34
-
35
-
#ifdef CONFIG_TIME_NS
36
-
struct vdso_data *arch_get_vdso_data(void *vvar_page)
37
-
{
38
-
return (struct vdso_data *)(vvar_page);
39
-
}
40
-
41
-
/*
42
-
* The VVAR page layout depends on whether a task belongs to the root or
43
-
* non-root time namespace. Whenever a task changes its namespace, the VVAR
44
-
* page tables are cleared and then they will be re-faulted with a
45
-
* corresponding layout.
46
-
* See also the comment near timens_setup_vdso_data() for details.
47
-
*/
48
-
int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
49
-
{
50
-
struct mm_struct *mm = task->mm;
51
-
VMA_ITERATOR(vmi, mm, 0);
52
-
struct vm_area_struct *vma;
53
-
54
-
mmap_read_lock(mm);
55
-
for_each_vma(vmi, vma) {
56
-
if (!vma_is_special_mapping(vma, &vvar_mapping))
57
-
continue;
58
-
zap_vma_pages(vma);
59
-
break;
60
-
}
61
-
mmap_read_unlock(mm);
62
-
return 0;
63
-
}
64
-
#endif
65
-
66
-
static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
67
-
struct vm_area_struct *vma, struct vm_fault *vmf)
68
-
{
69
-
struct page *timens_page = find_timens_vvar_page(vma);
70
-
unsigned long addr, pfn;
71
-
vm_fault_t err;
72
-
73
-
switch (vmf->pgoff) {
74
-
case VVAR_DATA_PAGE_OFFSET:
75
-
pfn = virt_to_pfn(vdso_data);
76
-
if (timens_page) {
77
-
/*
78
-
* Fault in VVAR page too, since it will be accessed
79
-
* to get clock data anyway.
80
-
*/
81
-
addr = vmf->address + VVAR_TIMENS_PAGE_OFFSET * PAGE_SIZE;
82
-
err = vmf_insert_pfn(vma, addr, pfn);
83
-
if (unlikely(err & VM_FAULT_ERROR))
84
-
return err;
85
-
pfn = page_to_pfn(timens_page);
86
-
}
87
-
break;
88
-
#ifdef CONFIG_TIME_NS
89
-
case VVAR_TIMENS_PAGE_OFFSET:
90
-
/*
91
-
* If a task belongs to a time namespace then a namespace
92
-
* specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
93
-
* the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
94
-
* offset.
95
-
* See also the comment near timens_setup_vdso_data().
96
-
*/
97
-
if (!timens_page)
98
-
return VM_FAULT_SIGBUS;
99
-
pfn = virt_to_pfn(vdso_data);
100
-
break;
101
-
#endif /* CONFIG_TIME_NS */
102
-
default:
103
-
return VM_FAULT_SIGBUS;
104
-
}
105
-
return vmf_insert_pfn(vma, vmf->address, pfn);
106
-
}
107
-
108
29
static int vdso_mremap(const struct vm_special_mapping *sm,
109
30
struct vm_area_struct *vma)
110
31
{
111
32
current->mm->context.vdso_base = vma->vm_start;
112
33
return 0;
113
34
}
114
-
115
-
static struct vm_special_mapping vvar_mapping = {
116
-
.name = "[vvar]",
117
-
.fault = vvar_fault,
118
-
};
119
35
120
36
static struct vm_special_mapping vdso64_mapping = {
121
37
.name = "[vdso]",
···
58
142
struct vm_area_struct *vma;
59
143
int rc;
60
144
61
-
BUILD_BUG_ON(VVAR_NR_PAGES != __VVAR_PAGES);
145
+
BUILD_BUG_ON(VDSO_NR_PAGES != __VDSO_PAGES);
62
146
if (mmap_write_lock_killable(mm))
63
147
return -EINTR;
64
148
···
73
157
rc = vvar_start;
74
158
if (IS_ERR_VALUE(vvar_start))
75
159
goto out;
76
-
vma = _install_special_mapping(mm, vvar_start, VVAR_NR_PAGES*PAGE_SIZE,
77
-
VM_READ|VM_MAYREAD|VM_IO|VM_DONTDUMP|
78
-
VM_PFNMAP,
79
-
&vvar_mapping);
160
+
vma = vdso_install_vvar_mapping(mm, vvar_start);
80
161
rc = PTR_ERR(vma);
81
162
if (IS_ERR(vma))
82
163
goto out;
83
-
vdso_text_start = vvar_start + VVAR_NR_PAGES * PAGE_SIZE;
164
+
vdso_text_start = vvar_start + VDSO_NR_PAGES * PAGE_SIZE;
84
165
/* VM_MAYWRITE for COW so gdb can set breakpoints */
85
166
vma = _install_special_mapping(mm, vdso_text_start, vdso_text_len,
86
167
VM_READ|VM_EXEC|
···
133
220
134
221
unsigned long vdso_size(void)
135
222
{
136
-
return vdso_text_size() + VVAR_NR_PAGES * PAGE_SIZE;
223
+
return vdso_text_size() + VDSO_NR_PAGES * PAGE_SIZE;
137
224
}
138
225
139
226
int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
+1
-1
arch/s390/kernel/vdso32/Makefile
+1
-1
arch/s390/kernel/vdso32/Makefile
+3
-4
arch/s390/kernel/vdso32/vdso32.lds.S
+3
-4
arch/s390/kernel/vdso32/vdso32.lds.S
···
6
6
7
7
#include <asm/page.h>
8
8
#include <asm/vdso.h>
9
+
#include <vdso/datapage.h>
9
10
10
11
OUTPUT_FORMAT("elf32-s390", "elf32-s390", "elf32-s390")
11
12
OUTPUT_ARCH(s390:31-bit)
12
13
13
14
SECTIONS
14
15
{
15
-
PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
16
-
#ifdef CONFIG_TIME_NS
17
-
PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
18
-
#endif
16
+
VDSO_VVAR_SYMS
17
+
19
18
. = SIZEOF_HEADERS;
20
19
21
20
.hash : { *(.hash) } :text
+1
-1
arch/s390/kernel/vdso64/Makefile
+1
-1
arch/s390/kernel/vdso64/Makefile
···
2
2
# List of files in the vdso
3
3
4
4
# Include the generic Makefile to check the built vdso.
5
-
include $(srctree)/lib/vdso/Makefile
5
+
include $(srctree)/lib/vdso/Makefile.include
6
6
obj-vdso64 = vdso_user_wrapper.o note.o vgetrandom-chacha.o
7
7
obj-cvdso64 = vdso64_generic.o getcpu.o vgetrandom.o
8
8
VDSO_CFLAGS_REMOVE := -pg $(CC_FLAGS_FTRACE) $(CC_FLAGS_EXPOLINE)
+3
-5
arch/s390/kernel/vdso64/vdso64.lds.S
+3
-5
arch/s390/kernel/vdso64/vdso64.lds.S
···
7
7
#include <asm/vdso/vsyscall.h>
8
8
#include <asm/page.h>
9
9
#include <asm/vdso.h>
10
+
#include <vdso/datapage.h>
10
11
11
12
OUTPUT_FORMAT("elf64-s390", "elf64-s390", "elf64-s390")
12
13
OUTPUT_ARCH(s390:64-bit)
13
14
14
15
SECTIONS
15
16
{
16
-
PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
17
-
PROVIDE(_vdso_rng_data = _vdso_data + __VDSO_RND_DATA_OFFSET);
18
-
#ifdef CONFIG_TIME_NS
19
-
PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
20
-
#endif
17
+
VDSO_VVAR_SYMS
18
+
21
19
. = SIZEOF_HEADERS;
22
20
23
21
.hash : { *(.hash) } :text
+3
-4
arch/sparc/vdso/Makefile
+3
-4
arch/sparc/vdso/Makefile
···
22
22
23
23
CPPFLAGS_vdso.lds += -P -C
24
24
25
-
VDSO_LDFLAGS_vdso.lds = -m elf64_sparc -soname linux-vdso.so.1 --no-undefined \
25
+
VDSO_LDFLAGS_vdso.lds = -m elf64_sparc -soname linux-vdso.so.1 \
26
26
-z max-page-size=8192
27
27
28
28
$(obj)/vdso64.so.dbg: $(obj)/vdso.lds $(vobjs) FORCE
···
101
101
quiet_cmd_vdso = VDSO $@
102
102
cmd_vdso = $(LD) -nostdlib -o $@ \
103
103
$(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \
104
-
-T $(filter %.lds,$^) $(filter %.o,$^) && \
105
-
sh $(src)/checkundef.sh '$(OBJDUMP)' '$@'
104
+
-T $(filter %.lds,$^) $(filter %.o,$^)
106
105
107
-
VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 -Bsymbolic
106
+
VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 -Bsymbolic --no-undefined
-10
arch/sparc/vdso/checkundef.sh
-10
arch/sparc/vdso/checkundef.sh
+1
arch/x86/Kconfig
+1
arch/x86/Kconfig
+4
-5
arch/x86/entry/vdso/Makefile
+4
-5
arch/x86/entry/vdso/Makefile
···
4
4
#
5
5
6
6
# Include the generic Makefile to check the built vDSO:
7
-
include $(srctree)/lib/vdso/Makefile
7
+
include $(srctree)/lib/vdso/Makefile.include
8
8
9
9
# Files to link into the vDSO:
10
10
vobjs-y := vdso-note.o vclock_gettime.o vgetcpu.o vgetrandom.o vgetrandom-chacha.o
···
32
32
33
33
CPPFLAGS_vdso.lds += -P -C
34
34
35
-
VDSO_LDFLAGS_vdso.lds = -m elf_x86_64 -soname linux-vdso.so.1 --no-undefined \
35
+
VDSO_LDFLAGS_vdso.lds = -m elf_x86_64 -soname linux-vdso.so.1 \
36
36
-z max-page-size=4096
37
37
38
38
$(obj)/vdso64.so.dbg: $(obj)/vdso.lds $(vobjs) FORCE
···
152
152
quiet_cmd_vdso = VDSO $@
153
153
cmd_vdso = $(LD) -o $@ \
154
154
$(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \
155
-
-T $(filter %.lds,$^) $(filter %.o,$^) && \
156
-
sh $(src)/checkundef.sh '$(NM)' '$@'
155
+
-T $(filter %.lds,$^) $(filter %.o,$^)
157
156
158
-
VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 \
157
+
VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 --no-undefined \
159
158
$(call ld-option, --eh-frame-hdr) -Bsymbolic -z noexecstack
160
159
161
160
quiet_cmd_vdso_and_check = VDSO $@
-10
arch/x86/entry/vdso/checkundef.sh
-10
arch/x86/entry/vdso/checkundef.sh
+3
-7
arch/x86/entry/vdso/vdso-layout.lds.S
+3
-7
arch/x86/entry/vdso/vdso-layout.lds.S
···
1
1
/* SPDX-License-Identifier: GPL-2.0 */
2
2
#include <asm/vdso.h>
3
3
#include <asm/vdso/vsyscall.h>
4
+
#include <vdso/datapage.h>
4
5
5
6
/*
6
7
* Linker script for vDSO. This is an ELF shared object prelinked to
···
18
17
* segment.
19
18
*/
20
19
21
-
vvar_start = . - __VVAR_PAGES * PAGE_SIZE;
22
-
vvar_page = vvar_start;
20
+
VDSO_VVAR_SYMS
23
21
24
-
vdso_rng_data = vvar_page + __VDSO_RND_DATA_OFFSET;
25
-
26
-
timens_page = vvar_start + PAGE_SIZE;
27
-
28
-
vclock_pages = vvar_start + VDSO_NR_VCLOCK_PAGES * PAGE_SIZE;
22
+
vclock_pages = VDSO_VCLOCK_PAGES_START(vdso_u_data);
29
23
pvclock_page = vclock_pages + VDSO_PAGE_PVCLOCK_OFFSET * PAGE_SIZE;
30
24
hvclock_page = vclock_pages + VDSO_PAGE_HVCLOCK_OFFSET * PAGE_SIZE;
31
25
-21
arch/x86/entry/vdso/vdso2c.c
-21
arch/x86/entry/vdso/vdso2c.c
···
69
69
70
70
const char *outfilename;
71
71
72
-
/* Symbols that we need in vdso2c. */
73
-
enum {
74
-
sym_vvar_start,
75
-
sym_vvar_page,
76
-
sym_pvclock_page,
77
-
sym_hvclock_page,
78
-
sym_timens_page,
79
-
};
80
-
81
-
const int special_pages[] = {
82
-
sym_vvar_page,
83
-
sym_pvclock_page,
84
-
sym_hvclock_page,
85
-
sym_timens_page,
86
-
};
87
-
88
72
struct vdso_sym {
89
73
const char *name;
90
74
bool export;
91
75
};
92
76
93
77
struct vdso_sym required_syms[] = {
94
-
[sym_vvar_start] = {"vvar_start", true},
95
-
[sym_vvar_page] = {"vvar_page", true},
96
-
[sym_pvclock_page] = {"pvclock_page", true},
97
-
[sym_hvclock_page] = {"hvclock_page", true},
98
-
[sym_timens_page] = {"timens_page", true},
99
78
{"VDSO32_NOTE_MASK", true},
100
79
{"__kernel_vsyscall", true},
101
80
{"__kernel_sigreturn", true},
-20
arch/x86/entry/vdso/vdso2c.h
-20
arch/x86/entry/vdso/vdso2c.h
···
150
150
}
151
151
}
152
152
153
-
/* Validate mapping addresses. */
154
-
for (i = 0; i < sizeof(special_pages) / sizeof(special_pages[0]); i++) {
155
-
INT_BITS symval = syms[special_pages[i]];
156
-
157
-
if (!symval)
158
-
continue; /* The mapping isn't used; ignore it. */
159
-
160
-
if (symval % 4096)
161
-
fail("%s must be a multiple of 4096\n",
162
-
required_syms[i].name);
163
-
if (symval + 4096 < syms[sym_vvar_start])
164
-
fail("%s underruns vvar_start\n",
165
-
required_syms[i].name);
166
-
if (symval + 4096 > 0)
167
-
fail("%s is on the wrong side of the vdso text\n",
168
-
required_syms[i].name);
169
-
}
170
-
if (syms[sym_vvar_start] % 4096)
171
-
fail("vvar_begin must be a multiple of 4096\n");
172
-
173
153
if (!image_name) {
174
154
fwrite(stripped_addr, stripped_len, 1, outfile);
175
155
return;
+7
-118
arch/x86/entry/vdso/vma.c
+7
-118
arch/x86/entry/vdso/vma.c
···
14
14
#include <linux/elf.h>
15
15
#include <linux/cpu.h>
16
16
#include <linux/ptrace.h>
17
-
#include <linux/time_namespace.h>
17
+
#include <linux/vdso_datastore.h>
18
18
19
19
#include <asm/pvclock.h>
20
20
#include <asm/vgtod.h>
···
27
27
#include <asm/vdso/vsyscall.h>
28
28
#include <clocksource/hyperv_timer.h>
29
29
30
-
struct vdso_data *arch_get_vdso_data(void *vvar_page)
31
-
{
32
-
return (struct vdso_data *)vvar_page;
33
-
}
34
-
35
-
static union vdso_data_store vdso_data_store __page_aligned_data;
36
-
struct vdso_data *vdso_data = vdso_data_store.data;
30
+
static_assert(VDSO_NR_PAGES + VDSO_NR_VCLOCK_PAGES == __VDSO_PAGES);
37
31
38
32
unsigned int vclocks_used __read_mostly;
39
33
···
47
53
return 0;
48
54
}
49
55
50
-
static const struct vm_special_mapping vvar_mapping;
51
56
struct linux_binprm;
52
57
53
58
static vm_fault_t vdso_fault(const struct vm_special_mapping *sm,
···
90
97
return 0;
91
98
}
92
99
93
-
#ifdef CONFIG_TIME_NS
94
-
/*
95
-
* The vvar page layout depends on whether a task belongs to the root or
96
-
* non-root time namespace. Whenever a task changes its namespace, the VVAR
97
-
* page tables are cleared and then they will re-faulted with a
98
-
* corresponding layout.
99
-
* See also the comment near timens_setup_vdso_data() for details.
100
-
*/
101
-
int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
102
-
{
103
-
struct mm_struct *mm = task->mm;
104
-
struct vm_area_struct *vma;
105
-
VMA_ITERATOR(vmi, mm, 0);
106
-
107
-
mmap_read_lock(mm);
108
-
for_each_vma(vmi, vma) {
109
-
if (vma_is_special_mapping(vma, &vvar_mapping))
110
-
zap_vma_pages(vma);
111
-
}
112
-
mmap_read_unlock(mm);
113
-
114
-
return 0;
115
-
}
116
-
#endif
117
-
118
-
static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
119
-
struct vm_area_struct *vma, struct vm_fault *vmf)
120
-
{
121
-
const struct vdso_image *image = vma->vm_mm->context.vdso_image;
122
-
unsigned long pfn;
123
-
long sym_offset;
124
-
125
-
if (!image)
126
-
return VM_FAULT_SIGBUS;
127
-
128
-
sym_offset = (long)(vmf->pgoff << PAGE_SHIFT) +
129
-
image->sym_vvar_start;
130
-
131
-
/*
132
-
* Sanity check: a symbol offset of zero means that the page
133
-
* does not exist for this vdso image, not that the page is at
134
-
* offset zero relative to the text mapping. This should be
135
-
* impossible here, because sym_offset should only be zero for
136
-
* the page past the end of the vvar mapping.
137
-
*/
138
-
if (sym_offset == 0)
139
-
return VM_FAULT_SIGBUS;
140
-
141
-
if (sym_offset == image->sym_vvar_page) {
142
-
struct page *timens_page = find_timens_vvar_page(vma);
143
-
144
-
pfn = __pa_symbol(vdso_data) >> PAGE_SHIFT;
145
-
146
-
/*
147
-
* If a task belongs to a time namespace then a namespace
148
-
* specific VVAR is mapped with the sym_vvar_page offset and
149
-
* the real VVAR page is mapped with the sym_timens_page
150
-
* offset.
151
-
* See also the comment near timens_setup_vdso_data().
152
-
*/
153
-
if (timens_page) {
154
-
unsigned long addr;
155
-
vm_fault_t err;
156
-
157
-
/*
158
-
* Optimization: inside time namespace pre-fault
159
-
* VVAR page too. As on timens page there are only
160
-
* offsets for clocks on VVAR, it'll be faulted
161
-
* shortly by VDSO code.
162
-
*/
163
-
addr = vmf->address + (image->sym_timens_page - sym_offset);
164
-
err = vmf_insert_pfn(vma, addr, pfn);
165
-
if (unlikely(err & VM_FAULT_ERROR))
166
-
return err;
167
-
168
-
pfn = page_to_pfn(timens_page);
169
-
}
170
-
171
-
return vmf_insert_pfn(vma, vmf->address, pfn);
172
-
173
-
} else if (sym_offset == image->sym_timens_page) {
174
-
struct page *timens_page = find_timens_vvar_page(vma);
175
-
176
-
if (!timens_page)
177
-
return VM_FAULT_SIGBUS;
178
-
179
-
pfn = __pa_symbol(vdso_data) >> PAGE_SHIFT;
180
-
return vmf_insert_pfn(vma, vmf->address, pfn);
181
-
}
182
-
183
-
return VM_FAULT_SIGBUS;
184
-
}
185
-
186
100
static vm_fault_t vvar_vclock_fault(const struct vm_special_mapping *sm,
187
101
struct vm_area_struct *vma, struct vm_fault *vmf)
188
102
{
···
111
211
case VDSO_PAGE_HVCLOCK_OFFSET:
112
212
{
113
213
unsigned long pfn = hv_get_tsc_pfn();
114
-
115
214
if (pfn && vclock_was_used(VDSO_CLOCKMODE_HVCLOCK))
116
215
return vmf_insert_pfn(vma, vmf->address, pfn);
117
216
break;
···
125
226
.name = "[vdso]",
126
227
.fault = vdso_fault,
127
228
.mremap = vdso_mremap,
128
-
};
129
-
static const struct vm_special_mapping vvar_mapping = {
130
-
.name = "[vvar]",
131
-
.fault = vvar_fault,
132
229
};
133
230
static const struct vm_special_mapping vvar_vclock_mapping = {
134
231
.name = "[vvar_vclock]",
···
147
252
return -EINTR;
148
253
149
254
addr = get_unmapped_area(NULL, addr,
150
-
image->size - image->sym_vvar_start, 0, 0);
255
+
image->size + __VDSO_PAGES * PAGE_SIZE, 0, 0);
151
256
if (IS_ERR_VALUE(addr)) {
152
257
ret = addr;
153
258
goto up_fail;
154
259
}
155
260
156
-
text_start = addr - image->sym_vvar_start;
261
+
text_start = addr + __VDSO_PAGES * PAGE_SIZE;
157
262
158
263
/*
159
264
* MAYWRITE to allow gdb to COW and set breakpoints
···
170
275
goto up_fail;
171
276
}
172
277
173
-
vma = _install_special_mapping(mm,
174
-
addr,
175
-
(__VVAR_PAGES - VDSO_NR_VCLOCK_PAGES) * PAGE_SIZE,
176
-
VM_READ|VM_MAYREAD|VM_IO|VM_DONTDUMP|
177
-
VM_PFNMAP,
178
-
&vvar_mapping);
179
-
278
+
vma = vdso_install_vvar_mapping(mm, addr);
180
279
if (IS_ERR(vma)) {
181
280
ret = PTR_ERR(vma);
182
281
do_munmap(mm, text_start, image->size, NULL);
···
178
289
}
179
290
180
291
vma = _install_special_mapping(mm,
181
-
addr + (__VVAR_PAGES - VDSO_NR_VCLOCK_PAGES) * PAGE_SIZE,
292
+
VDSO_VCLOCK_PAGES_START(addr),
182
293
VDSO_NR_VCLOCK_PAGES * PAGE_SIZE,
183
294
VM_READ|VM_MAYREAD|VM_IO|VM_DONTDUMP|
184
295
VM_PFNMAP,
···
215
326
*/
216
327
for_each_vma(vmi, vma) {
217
328
if (vma_is_special_mapping(vma, &vdso_mapping) ||
218
-
vma_is_special_mapping(vma, &vvar_mapping) ||
329
+
vma_is_special_mapping(vma, &vdso_vvar_mapping) ||
219
330
vma_is_special_mapping(vma, &vvar_vclock_mapping)) {
220
331
mmap_write_unlock(mm);
221
332
return -EEXIST;
-6
arch/x86/include/asm/vdso.h
-6
arch/x86/include/asm/vdso.h
···
18
18
unsigned long extable_base, extable_len;
19
19
const void *extable;
20
20
21
-
long sym_vvar_start; /* Negative offset to the vvar area */
22
-
23
-
long sym_vvar_page;
24
-
long sym_pvclock_page;
25
-
long sym_hvclock_page;
26
-
long sym_timens_page;
27
21
long sym_VDSO32_NOTE_MASK;
28
22
long sym___kernel_sigreturn;
29
23
long sym___kernel_rt_sigreturn;
-10
arch/x86/include/asm/vdso/getrandom.h
-10
arch/x86/include/asm/vdso/getrandom.h
···
27
27
return ret;
28
28
}
29
29
30
-
extern struct vdso_rng_data vdso_rng_data
31
-
__attribute__((visibility("hidden")));
32
-
33
-
static __always_inline const struct vdso_rng_data *__arch_get_vdso_rng_data(void)
34
-
{
35
-
if (IS_ENABLED(CONFIG_TIME_NS) && __arch_get_vdso_data()->clock_mode == VDSO_CLOCKMODE_TIMENS)
36
-
return (void *)&vdso_rng_data + ((void *)&timens_page - (void *)__arch_get_vdso_data());
37
-
return &vdso_rng_data;
38
-
}
39
-
40
30
#endif /* !__ASSEMBLER__ */
41
31
42
32
#endif /* __ASM_VDSO_GETRANDOM_H */
+9
-28
arch/x86/include/asm/vdso/gettimeofday.h
+9
-28
arch/x86/include/asm/vdso/gettimeofday.h
···
19
19
#include <asm/pvclock.h>
20
20
#include <clocksource/hyperv_timer.h>
21
21
22
-
extern struct vdso_data vvar_page
23
-
__attribute__((visibility("hidden")));
24
-
25
-
extern struct vdso_data timens_page
26
-
__attribute__((visibility("hidden")));
27
-
28
22
#define VDSO_HAS_TIME 1
29
23
30
24
#define VDSO_HAS_CLOCK_GETRES 1
···
51
57
#ifdef CONFIG_HYPERV_TIMER
52
58
extern struct ms_hyperv_tsc_page hvclock_page
53
59
__attribute__((visibility("hidden")));
54
-
#endif
55
-
56
-
#ifdef CONFIG_TIME_NS
57
-
static __always_inline
58
-
const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
59
-
{
60
-
return &timens_page;
61
-
}
62
60
#endif
63
61
64
62
#ifndef BUILD_VDSO32
···
236
250
#endif
237
251
238
252
static inline u64 __arch_get_hw_counter(s32 clock_mode,
239
-
const struct vdso_data *vd)
253
+
const struct vdso_time_data *vd)
240
254
{
241
255
if (likely(clock_mode == VDSO_CLOCKMODE_TSC))
242
256
return (u64)rdtsc_ordered() & S64_MAX;
···
261
275
return U64_MAX;
262
276
}
263
277
264
-
static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
265
-
{
266
-
return &vvar_page;
267
-
}
268
-
269
-
static inline bool arch_vdso_clocksource_ok(const struct vdso_data *vd)
278
+
static inline bool arch_vdso_clocksource_ok(const struct vdso_clock *vc)
270
279
{
271
280
return true;
272
281
}
···
300
319
* declares everything with the MSB/Sign-bit set as invalid. Therefore the
301
320
* effective mask is S64_MAX.
302
321
*/
303
-
static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base)
322
+
static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base)
304
323
{
305
-
u64 delta = cycles - vd->cycle_last;
324
+
u64 delta = cycles - vc->cycle_last;
306
325
307
326
/*
308
327
* Negative motion and deltas which can cause multiplication
309
328
* overflow require special treatment. This check covers both as
310
-
* negative motion is guaranteed to be greater than @vd::max_cycles
329
+
* negative motion is guaranteed to be greater than @vc::max_cycles
311
330
* due to unsigned comparison.
312
331
*
313
332
* Due to the MSB/Sign-bit being used as invalid marker (see
314
333
* arch_vdso_cycles_ok() above), the effective mask is S64_MAX, but that
315
334
* case is also unlikely and will also take the unlikely path here.
316
335
*/
317
-
if (unlikely(delta > vd->max_cycles)) {
336
+
if (unlikely(delta > vc->max_cycles)) {
318
337
/*
319
338
* Due to the above mentioned TSC wobbles, filter out
320
339
* negative motion. Per the above masking, the effective
321
340
* sign bit is now bit 62.
322
341
*/
323
342
if (delta & (1ULL << 62))
324
-
return base >> vd->shift;
343
+
return base >> vc->shift;
325
344
326
345
/* Handle multiplication overflow gracefully */
327
-
return mul_u64_u32_add_u64_shr(delta & S64_MAX, vd->mult, base, vd->shift);
346
+
return mul_u64_u32_add_u64_shr(delta & S64_MAX, vc->mult, base, vc->shift);
328
347
}
329
348
330
-
return ((delta * vd->mult) + base) >> vd->shift;
349
+
return ((delta * vc->mult) + base) >> vc->shift;
331
350
}
332
351
#define vdso_calc_ns vdso_calc_ns
333
352
+2
-21
arch/x86/include/asm/vdso/vsyscall.h
+2
-21
arch/x86/include/asm/vdso/vsyscall.h
···
2
2
#ifndef __ASM_VDSO_VSYSCALL_H
3
3
#define __ASM_VDSO_VSYSCALL_H
4
4
5
-
#define __VDSO_RND_DATA_OFFSET 640
6
-
#define __VVAR_PAGES 4
5
+
#define __VDSO_PAGES 6
7
6
8
7
#define VDSO_NR_VCLOCK_PAGES 2
8
+
#define VDSO_VCLOCK_PAGES_START(_b) ((_b) + (__VDSO_PAGES - VDSO_NR_VCLOCK_PAGES) * PAGE_SIZE)
9
9
#define VDSO_PAGE_PVCLOCK_OFFSET 0
10
10
#define VDSO_PAGE_HVCLOCK_OFFSET 1
11
11
···
13
13
14
14
#include <vdso/datapage.h>
15
15
#include <asm/vgtod.h>
16
-
17
-
extern struct vdso_data *vdso_data;
18
-
19
-
/*
20
-
* Update the vDSO data page to keep in sync with kernel timekeeping.
21
-
*/
22
-
static __always_inline
23
-
struct vdso_data *__x86_get_k_vdso_data(void)
24
-
{
25
-
return vdso_data;
26
-
}
27
-
#define __arch_get_k_vdso_data __x86_get_k_vdso_data
28
-
29
-
static __always_inline
30
-
struct vdso_rng_data *__x86_get_k_vdso_rng_data(void)
31
-
{
32
-
return (void *)vdso_data + __VDSO_RND_DATA_OFFSET;
33
-
}
34
-
#define __arch_get_k_vdso_rng_data __x86_get_k_vdso_rng_data
35
16
36
17
/* The asm-generic header needs to be included after the definitions above */
37
18
#include <asm-generic/vdso/vsyscall.h>
+3
-3
drivers/char/random.c
+3
-3
drivers/char/random.c
···
278
278
WRITE_ONCE(base_crng.generation, next_gen);
279
279
#ifdef CONFIG_VDSO_GETRANDOM
280
280
/* base_crng.generation's invalid value is ULONG_MAX, while
281
-
* _vdso_rng_data.generation's invalid value is 0, so add one to the
281
+
* vdso_k_rng_data->generation's invalid value is 0, so add one to the
282
282
* former to arrive at the latter. Use smp_store_release so that this
283
283
* is ordered with the write above to base_crng.generation. Pairs with
284
284
* the smp_rmb() before the syscall in the vDSO code.
···
290
290
* because the vDSO side only checks whether the value changed, without
291
291
* actually using or interpreting the value.
292
292
*/
293
-
smp_store_release((unsigned long *)&__arch_get_k_vdso_rng_data()->generation, next_gen + 1);
293
+
smp_store_release((unsigned long *)&vdso_k_rng_data->generation, next_gen + 1);
294
294
#endif
295
295
if (!static_branch_likely(&crng_is_ready))
296
296
crng_init = CRNG_READY;
···
743
743
queue_work(system_unbound_wq, &set_ready);
744
744
atomic_notifier_call_chain(&random_ready_notifier, 0, NULL);
745
745
#ifdef CONFIG_VDSO_GETRANDOM
746
-
WRITE_ONCE(__arch_get_k_vdso_rng_data()->is_ready, true);
746
+
WRITE_ONCE(vdso_k_rng_data->is_ready, true);
747
747
#endif
748
748
wake_up_interruptible(&crng_init_wait);
749
749
kill_fasync(&fasync, SIGIO, POLL_IN);
+19
-8
include/asm-generic/vdso/vsyscall.h
+19
-8
include/asm-generic/vdso/vsyscall.h
···
4
4
5
5
#ifndef __ASSEMBLY__
6
6
7
-
#ifndef __arch_get_k_vdso_data
8
-
static __always_inline struct vdso_data *__arch_get_k_vdso_data(void)
7
+
#ifdef CONFIG_GENERIC_VDSO_DATA_STORE
8
+
9
+
#ifndef __arch_get_vdso_u_time_data
10
+
static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void)
9
11
{
10
-
return NULL;
12
+
return &vdso_u_time_data;
11
13
}
12
-
#endif /* __arch_get_k_vdso_data */
14
+
#endif
15
+
16
+
#ifndef __arch_get_vdso_u_rng_data
17
+
static __always_inline const struct vdso_rng_data *__arch_get_vdso_u_rng_data(void)
18
+
{
19
+
return &vdso_u_rng_data;
20
+
}
21
+
#endif
22
+
23
+
#endif /* CONFIG_GENERIC_VDSO_DATA_STORE */
13
24
14
25
#ifndef __arch_update_vsyscall
15
-
static __always_inline void __arch_update_vsyscall(struct vdso_data *vdata)
26
+
static __always_inline void __arch_update_vsyscall(struct vdso_time_data *vdata)
16
27
{
17
28
}
18
29
#endif /* __arch_update_vsyscall */
19
30
20
-
#ifndef __arch_sync_vdso_data
21
-
static __always_inline void __arch_sync_vdso_data(struct vdso_data *vdata)
31
+
#ifndef __arch_sync_vdso_time_data
32
+
static __always_inline void __arch_sync_vdso_time_data(struct vdso_time_data *vdata)
22
33
{
23
34
}
24
-
#endif /* __arch_sync_vdso_data */
35
+
#endif /* __arch_sync_vdso_time_data */
25
36
26
37
#endif /* !__ASSEMBLY__ */
27
38
+1
-9
include/linux/align.h
+1
-9
include/linux/align.h
···
2
2
#ifndef _LINUX_ALIGN_H
3
3
#define _LINUX_ALIGN_H
4
4
5
-
#include <linux/const.h>
6
-
7
-
/* @a is a power of 2 value */
8
-
#define ALIGN(x, a) __ALIGN_KERNEL((x), (a))
9
-
#define ALIGN_DOWN(x, a) __ALIGN_KERNEL((x) - ((a) - 1), (a))
10
-
#define __ALIGN_MASK(x, mask) __ALIGN_KERNEL_MASK((x), (mask))
11
-
#define PTR_ALIGN(p, a) ((typeof(p))ALIGN((unsigned long)(p), (a)))
12
-
#define PTR_ALIGN_DOWN(p, a) ((typeof(p))ALIGN_DOWN((unsigned long)(p), (a)))
13
-
#define IS_ALIGNED(x, a) (((x) & ((typeof(x))(a) - 1)) == 0)
5
+
#include <vdso/align.h>
14
6
15
7
#endif /* _LINUX_ALIGN_H */
+1
-8
include/linux/cache.h
+1
-8
include/linux/cache.h
···
3
3
#define __LINUX_CACHE_H
4
4
5
5
#include <uapi/linux/kernel.h>
6
+
#include <vdso/cache.h>
6
7
#include <asm/cache.h>
7
8
8
9
#ifndef L1_CACHE_ALIGN
9
10
#define L1_CACHE_ALIGN(x) __ALIGN_KERNEL(x, L1_CACHE_BYTES)
10
-
#endif
11
-
12
-
#ifndef SMP_CACHE_BYTES
13
-
#define SMP_CACHE_BYTES L1_CACHE_BYTES
14
11
#endif
15
12
16
13
/**
···
58
61
*/
59
62
#ifndef __ro_after_init
60
63
#define __ro_after_init __section(".data..ro_after_init")
61
-
#endif
62
-
63
-
#ifndef ____cacheline_aligned
64
-
#define ____cacheline_aligned __attribute__((__aligned__(SMP_CACHE_BYTES)))
65
64
#endif
66
65
67
66
#ifndef ____cacheline_aligned_in_smp
-2
include/linux/time_namespace.h
-2
include/linux/time_namespace.h
+10
include/linux/vdso_datastore.h
+10
include/linux/vdso_datastore.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
#ifndef _LINUX_VDSO_DATASTORE_H
3
+
#define _LINUX_VDSO_DATASTORE_H
4
+
5
+
#include <linux/mm_types.h>
6
+
7
+
extern const struct vm_special_mapping vdso_vvar_mapping;
8
+
struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr);
9
+
10
+
#endif /* _LINUX_VDSO_DATASTORE_H */
+38
include/uapi/linux/elf.h
+38
include/uapi/linux/elf.h
···
11
11
typedef __u32 Elf32_Off;
12
12
typedef __s32 Elf32_Sword;
13
13
typedef __u32 Elf32_Word;
14
+
typedef __u16 Elf32_Versym;
14
15
15
16
/* 64-bit ELF base types. */
16
17
typedef __u64 Elf64_Addr;
···
22
21
typedef __u32 Elf64_Word;
23
22
typedef __u64 Elf64_Xword;
24
23
typedef __s64 Elf64_Sxword;
24
+
typedef __u16 Elf64_Versym;
25
25
26
26
/* These constants are for the segment types stored in the image headers */
27
27
#define PT_NULL 0
···
109
107
#define DT_VALRNGLO 0x6ffffd00
110
108
#define DT_VALRNGHI 0x6ffffdff
111
109
#define DT_ADDRRNGLO 0x6ffffe00
110
+
#define DT_GNU_HASH 0x6ffffef5
112
111
#define DT_ADDRRNGHI 0x6ffffeff
113
112
#define DT_VERSYM 0x6ffffff0
114
113
#define DT_RELACOUNT 0x6ffffff9
···
128
125
#define STB_GLOBAL 1
129
126
#define STB_WEAK 2
130
127
128
+
#define STN_UNDEF 0
129
+
131
130
#define STT_NOTYPE 0
132
131
#define STT_OBJECT 1
133
132
#define STT_FUNC 2
···
137
132
#define STT_FILE 4
138
133
#define STT_COMMON 5
139
134
#define STT_TLS 6
135
+
136
+
#define VER_FLG_BASE 0x1
137
+
#define VER_FLG_WEAK 0x2
140
138
141
139
#define ELF_ST_BIND(x) ((x) >> 4)
142
140
#define ELF_ST_TYPE(x) ((x) & 0xf)
···
579
571
580
572
/* Bits for GNU_PROPERTY_AARCH64_FEATURE_1_BTI */
581
573
#define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0)
574
+
575
+
typedef struct {
576
+
Elf32_Half vd_version;
577
+
Elf32_Half vd_flags;
578
+
Elf32_Half vd_ndx;
579
+
Elf32_Half vd_cnt;
580
+
Elf32_Word vd_hash;
581
+
Elf32_Word vd_aux;
582
+
Elf32_Word vd_next;
583
+
} Elf32_Verdef;
584
+
585
+
typedef struct {
586
+
Elf64_Half vd_version;
587
+
Elf64_Half vd_flags;
588
+
Elf64_Half vd_ndx;
589
+
Elf64_Half vd_cnt;
590
+
Elf64_Word vd_hash;
591
+
Elf64_Word vd_aux;
592
+
Elf64_Word vd_next;
593
+
} Elf64_Verdef;
594
+
595
+
typedef struct {
596
+
Elf32_Word vda_name;
597
+
Elf32_Word vda_next;
598
+
} Elf32_Verdaux;
599
+
600
+
typedef struct {
601
+
Elf64_Word vda_name;
602
+
Elf64_Word vda_next;
603
+
} Elf64_Verdaux;
582
604
583
605
#endif /* _UAPI_LINUX_ELF_H */
+15
include/vdso/align.h
+15
include/vdso/align.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
#ifndef __VDSO_ALIGN_H
3
+
#define __VDSO_ALIGN_H
4
+
5
+
#include <vdso/const.h>
6
+
7
+
/* @a is a power of 2 value */
8
+
#define ALIGN(x, a) __ALIGN_KERNEL((x), (a))
9
+
#define ALIGN_DOWN(x, a) __ALIGN_KERNEL((x) - ((a) - 1), (a))
10
+
#define __ALIGN_MASK(x, mask) __ALIGN_KERNEL_MASK((x), (mask))
11
+
#define PTR_ALIGN(p, a) ((typeof(p))ALIGN((unsigned long)(p), (a)))
12
+
#define PTR_ALIGN_DOWN(p, a) ((typeof(p))ALIGN_DOWN((unsigned long)(p), (a)))
13
+
#define IS_ALIGNED(x, a) (((x) & ((typeof(x))(a) - 1)) == 0)
14
+
15
+
#endif /* __VDSO_ALIGN_H */
+15
include/vdso/cache.h
+15
include/vdso/cache.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
#ifndef __VDSO_CACHE_H
3
+
#define __VDSO_CACHE_H
4
+
5
+
#include <asm/cache.h>
6
+
7
+
#ifndef SMP_CACHE_BYTES
8
+
#define SMP_CACHE_BYTES L1_CACHE_BYTES
9
+
#endif
10
+
11
+
#ifndef ____cacheline_aligned
12
+
#define ____cacheline_aligned __attribute__((__aligned__(SMP_CACHE_BYTES)))
13
+
#endif
14
+
15
+
#endif /* __VDSO_ALIGN_H */
+87
-33
include/vdso/datapage.h
+87
-33
include/vdso/datapage.h
···
9
9
#include <uapi/linux/types.h>
10
10
#include <uapi/asm-generic/errno-base.h>
11
11
12
+
#include <vdso/align.h>
12
13
#include <vdso/bits.h>
14
+
#include <vdso/cache.h>
13
15
#include <vdso/clocksource.h>
14
16
#include <vdso/ktime.h>
15
17
#include <vdso/limits.h>
16
18
#include <vdso/math64.h>
19
+
#include <vdso/page.h>
17
20
#include <vdso/processor.h>
18
21
#include <vdso/time.h>
19
22
#include <vdso/time32.h>
···
26
23
#include <asm/vdso/time_data.h>
27
24
#else
28
25
struct arch_vdso_time_data {};
26
+
#endif
27
+
28
+
#if defined(CONFIG_ARCH_HAS_VDSO_ARCH_DATA)
29
+
#include <asm/vdso/arch_data.h>
30
+
#elif defined(CONFIG_GENERIC_VDSO_DATA_STORE)
31
+
struct vdso_arch_data {
32
+
/* Needed for the generic code, never actually used at runtime */
33
+
char __unused;
34
+
};
29
35
#endif
30
36
31
37
#define VDSO_BASES (CLOCK_TAI + 1)
···
57
45
*
58
46
* There is one vdso_timestamp object in vvar for each vDSO-accelerated
59
47
* clock_id. For high-resolution clocks, this encodes the time
60
-
* corresponding to vdso_data.cycle_last. For coarse clocks this encodes
48
+
* corresponding to vdso_time_data.cycle_last. For coarse clocks this encodes
61
49
* the actual time.
62
50
*
63
51
* To be noticed that for highres clocks nsec is left-shifted by
64
-
* vdso_data.cs[x].shift.
52
+
* vdso_time_data[x].shift.
65
53
*/
66
54
struct vdso_timestamp {
67
55
u64 sec;
···
69
57
};
70
58
71
59
/**
72
-
* struct vdso_data - vdso datapage representation
60
+
* struct vdso_clock - vdso per clocksource datapage representation
73
61
* @seq: timebase sequence counter
74
62
* @clock_mode: clock mode
75
63
* @cycle_last: timebase at clocksource init
···
79
67
* @shift: clocksource shift
80
68
* @basetime[clock_id]: basetime per clock_id
81
69
* @offset[clock_id]: time namespace offset per clock_id
82
-
* @tz_minuteswest: minutes west of Greenwich
83
-
* @tz_dsttime: type of DST correction
84
-
* @hrtimer_res: hrtimer resolution
85
-
* @__unused: unused
86
-
* @arch_data: architecture specific data (optional, defaults
87
-
* to an empty struct)
88
70
*
89
-
* vdso_data will be accessed by 64 bit and compat code at the same time
90
-
* so we should be careful before modifying this structure.
91
-
*
92
-
* The ordering of the struct members is optimized to have fast access to the
93
-
* often required struct members which are related to CLOCK_REALTIME and
94
-
* CLOCK_MONOTONIC. This information is stored in the first cache lines.
71
+
* See also struct vdso_time_data for basic access and ordering information as
72
+
* struct vdso_clock is used there.
95
73
*
96
74
* @basetime is used to store the base time for the system wide time getter
97
75
* VVAR page.
···
94
92
* For clocks which are not affected by time namespace adjustment the
95
93
* offset must be zero.
96
94
*/
97
-
struct vdso_data {
95
+
struct vdso_clock {
98
96
u32 seq;
99
97
100
98
s32 clock_mode;
···
110
108
struct vdso_timestamp basetime[VDSO_BASES];
111
109
struct timens_offset offset[VDSO_BASES];
112
110
};
113
-
114
-
s32 tz_minuteswest;
115
-
s32 tz_dsttime;
116
-
u32 hrtimer_res;
117
-
u32 __unused;
118
-
119
-
struct arch_vdso_time_data arch_data;
120
111
};
112
+
113
+
/**
114
+
* struct vdso_time_data - vdso datapage representation
115
+
* @arch_data: architecture specific data (optional, defaults
116
+
* to an empty struct)
117
+
* @clock_data: clocksource related data (array)
118
+
* @tz_minuteswest: minutes west of Greenwich
119
+
* @tz_dsttime: type of DST correction
120
+
* @hrtimer_res: hrtimer resolution
121
+
* @__unused: unused
122
+
*
123
+
* vdso_time_data will be accessed by 64 bit and compat code at the same time
124
+
* so we should be careful before modifying this structure.
125
+
*
126
+
* The ordering of the struct members is optimized to have fast acces to the
127
+
* often required struct members which are related to CLOCK_REALTIME and
128
+
* CLOCK_MONOTONIC. This information is stored in the first cache lines.
129
+
*/
130
+
struct vdso_time_data {
131
+
struct arch_vdso_time_data arch_data;
132
+
133
+
struct vdso_clock clock_data[CS_BASES];
134
+
135
+
s32 tz_minuteswest;
136
+
s32 tz_dsttime;
137
+
u32 hrtimer_res;
138
+
u32 __unused;
139
+
} ____cacheline_aligned;
121
140
122
141
/**
123
142
* struct vdso_rng_data - vdso RNG state information
···
159
136
* With the hidden visibility, the compiler simply generates a PC-relative
160
137
* relocation, and this is what we need.
161
138
*/
162
-
extern struct vdso_data _vdso_data[CS_BASES] __attribute__((visibility("hidden")));
163
-
extern struct vdso_data _timens_data[CS_BASES] __attribute__((visibility("hidden")));
164
-
extern struct vdso_rng_data _vdso_rng_data __attribute__((visibility("hidden")));
139
+
#ifdef CONFIG_GENERIC_VDSO_DATA_STORE
140
+
extern struct vdso_time_data vdso_u_time_data __attribute__((visibility("hidden")));
141
+
extern struct vdso_rng_data vdso_u_rng_data __attribute__((visibility("hidden")));
142
+
extern struct vdso_arch_data vdso_u_arch_data __attribute__((visibility("hidden")));
165
143
166
-
/**
167
-
* union vdso_data_store - Generic vDSO data page
168
-
*/
169
-
union vdso_data_store {
170
-
struct vdso_data data[CS_BASES];
171
-
u8 page[1U << CONFIG_PAGE_SHIFT];
144
+
extern struct vdso_time_data *vdso_k_time_data;
145
+
extern struct vdso_rng_data *vdso_k_rng_data;
146
+
extern struct vdso_arch_data *vdso_k_arch_data;
147
+
148
+
#define VDSO_ARCH_DATA_SIZE ALIGN(sizeof(struct vdso_arch_data), PAGE_SIZE)
149
+
#define VDSO_ARCH_DATA_PAGES (VDSO_ARCH_DATA_SIZE >> PAGE_SHIFT)
150
+
151
+
enum vdso_pages {
152
+
VDSO_TIME_PAGE_OFFSET,
153
+
VDSO_TIMENS_PAGE_OFFSET,
154
+
VDSO_RNG_PAGE_OFFSET,
155
+
VDSO_ARCH_PAGES_START,
156
+
VDSO_ARCH_PAGES_END = VDSO_ARCH_PAGES_START + VDSO_ARCH_DATA_PAGES - 1,
157
+
VDSO_NR_PAGES
172
158
};
159
+
160
+
#endif /* CONFIG_GENERIC_VDSO_DATA_STORE */
173
161
174
162
/*
175
163
* The generic vDSO implementation requires that gettimeofday.h
176
164
* provides:
177
-
* - __arch_get_vdso_data(): to get the vdso datapage.
178
165
* - __arch_get_hw_counter(): to get the hw counter based on the
179
166
* clock_mode.
180
167
* - gettimeofday_fallback(): fallback for gettimeofday.
···
196
163
#else
197
164
#include <asm/vdso/gettimeofday.h>
198
165
#endif /* ENABLE_COMPAT_VDSO */
166
+
167
+
#else /* !__ASSEMBLY__ */
168
+
169
+
#ifdef CONFIG_VDSO_GETRANDOM
170
+
#define __vdso_u_rng_data PROVIDE(vdso_u_rng_data = vdso_u_data + 2 * PAGE_SIZE);
171
+
#else
172
+
#define __vdso_u_rng_data
173
+
#endif
174
+
175
+
#ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA
176
+
#define __vdso_u_arch_data PROVIDE(vdso_u_arch_data = vdso_u_data + 3 * PAGE_SIZE);
177
+
#else
178
+
#define __vdso_u_arch_data
179
+
#endif
180
+
181
+
#define VDSO_VVAR_SYMS \
182
+
PROVIDE(vdso_u_data = . - __VDSO_PAGES * PAGE_SIZE); \
183
+
PROVIDE(vdso_u_time_data = vdso_u_data); \
184
+
__vdso_u_rng_data \
185
+
__vdso_u_arch_data \
186
+
199
187
200
188
#endif /* !__ASSEMBLY__ */
201
189
+14
-10
include/vdso/helpers.h
+14
-10
include/vdso/helpers.h
···
7
7
#include <asm/barrier.h>
8
8
#include <vdso/datapage.h>
9
9
10
-
static __always_inline u32 vdso_read_begin(const struct vdso_data *vd)
10
+
static __always_inline u32 vdso_read_begin(const struct vdso_clock *vc)
11
11
{
12
12
u32 seq;
13
13
14
-
while (unlikely((seq = READ_ONCE(vd->seq)) & 1))
14
+
while (unlikely((seq = READ_ONCE(vc->seq)) & 1))
15
15
cpu_relax();
16
16
17
17
smp_rmb();
18
18
return seq;
19
19
}
20
20
21
-
static __always_inline u32 vdso_read_retry(const struct vdso_data *vd,
21
+
static __always_inline u32 vdso_read_retry(const struct vdso_clock *vc,
22
22
u32 start)
23
23
{
24
24
u32 seq;
25
25
26
26
smp_rmb();
27
-
seq = READ_ONCE(vd->seq);
27
+
seq = READ_ONCE(vc->seq);
28
28
return seq != start;
29
29
}
30
30
31
-
static __always_inline void vdso_write_begin(struct vdso_data *vd)
31
+
static __always_inline void vdso_write_begin(struct vdso_time_data *vd)
32
32
{
33
+
struct vdso_clock *vc = vd->clock_data;
34
+
33
35
/*
34
36
* WRITE_ONCE() is required otherwise the compiler can validly tear
35
37
* updates to vd[x].seq and it is possible that the value seen by the
36
38
* reader is inconsistent.
37
39
*/
38
-
WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
39
-
WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
40
+
WRITE_ONCE(vc[CS_HRES_COARSE].seq, vc[CS_HRES_COARSE].seq + 1);
41
+
WRITE_ONCE(vc[CS_RAW].seq, vc[CS_RAW].seq + 1);
40
42
smp_wmb();
41
43
}
42
44
43
-
static __always_inline void vdso_write_end(struct vdso_data *vd)
45
+
static __always_inline void vdso_write_end(struct vdso_time_data *vd)
44
46
{
47
+
struct vdso_clock *vc = vd->clock_data;
48
+
45
49
smp_wmb();
46
50
/*
47
51
* WRITE_ONCE() is required otherwise the compiler can validly tear
48
52
* updates to vd[x].seq and it is possible that the value seen by the
49
53
* reader is inconsistent.
50
54
*/
51
-
WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
52
-
WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
55
+
WRITE_ONCE(vc[CS_HRES_COARSE].seq, vc[CS_HRES_COARSE].seq + 1);
56
+
WRITE_ONCE(vc[CS_RAW].seq, vc[CS_RAW].seq + 1);
53
57
}
54
58
55
59
#endif /* !__ASSEMBLY__ */
+13
-11
kernel/time/namespace.c
+13
-11
kernel/time/namespace.c
···
165
165
* HVCLOCK
166
166
* VVAR
167
167
*
168
-
* The check for vdso_data->clock_mode is in the unlikely path of
168
+
* The check for vdso_clock->clock_mode is in the unlikely path of
169
169
* the seq begin magic. So for the non-timens case most of the time
170
170
* 'seq' is even, so the branch is not taken.
171
171
*
172
172
* If 'seq' is odd, i.e. a concurrent update is in progress, the extra check
173
-
* for vdso_data->clock_mode is a non-issue. The task is spin waiting for the
173
+
* for vdso_clock->clock_mode is a non-issue. The task is spin waiting for the
174
174
* update to finish and for 'seq' to become even anyway.
175
175
*
176
-
* Timens page has vdso_data->clock_mode set to VDSO_CLOCKMODE_TIMENS which
176
+
* Timens page has vdso_clock->clock_mode set to VDSO_CLOCKMODE_TIMENS which
177
177
* enforces the time namespace handling path.
178
178
*/
179
-
static void timens_setup_vdso_data(struct vdso_data *vdata,
180
-
struct time_namespace *ns)
179
+
static void timens_setup_vdso_clock_data(struct vdso_clock *vc,
180
+
struct time_namespace *ns)
181
181
{
182
-
struct timens_offset *offset = vdata->offset;
182
+
struct timens_offset *offset = vc->offset;
183
183
struct timens_offset monotonic = offset_from_ts(ns->offsets.monotonic);
184
184
struct timens_offset boottime = offset_from_ts(ns->offsets.boottime);
185
185
186
-
vdata->seq = 1;
187
-
vdata->clock_mode = VDSO_CLOCKMODE_TIMENS;
186
+
vc->seq = 1;
187
+
vc->clock_mode = VDSO_CLOCKMODE_TIMENS;
188
188
offset[CLOCK_MONOTONIC] = monotonic;
189
189
offset[CLOCK_MONOTONIC_RAW] = monotonic;
190
190
offset[CLOCK_MONOTONIC_COARSE] = monotonic;
···
219
219
static void timens_set_vvar_page(struct task_struct *task,
220
220
struct time_namespace *ns)
221
221
{
222
-
struct vdso_data *vdata;
222
+
struct vdso_time_data *vdata;
223
+
struct vdso_clock *vc;
223
224
unsigned int i;
224
225
225
226
if (ns == &init_time_ns)
···
236
235
goto out;
237
236
238
237
ns->frozen_offsets = true;
239
-
vdata = arch_get_vdso_data(page_address(ns->vvar_page));
238
+
vdata = page_address(ns->vvar_page);
239
+
vc = vdata->clock_data;
240
240
241
241
for (i = 0; i < CS_BASES; i++)
242
-
timens_setup_vdso_data(&vdata[i], ns);
242
+
timens_setup_vdso_clock_data(&vc[i], ns);
243
243
244
244
out:
245
245
mutex_unlock(&offset_lock);
+33
-33
kernel/time/vsyscall.c
+33
-33
kernel/time/vsyscall.c
···
15
15
16
16
#include "timekeeping_internal.h"
17
17
18
-
static inline void update_vdso_data(struct vdso_data *vdata,
19
-
struct timekeeper *tk)
18
+
static inline void update_vdso_time_data(struct vdso_time_data *vdata, struct timekeeper *tk)
20
19
{
20
+
struct vdso_clock *vc = vdata->clock_data;
21
21
struct vdso_timestamp *vdso_ts;
22
22
u64 nsec, sec;
23
23
24
-
vdata[CS_HRES_COARSE].cycle_last = tk->tkr_mono.cycle_last;
24
+
vc[CS_HRES_COARSE].cycle_last = tk->tkr_mono.cycle_last;
25
25
#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
26
-
vdata[CS_HRES_COARSE].max_cycles = tk->tkr_mono.clock->max_cycles;
26
+
vc[CS_HRES_COARSE].max_cycles = tk->tkr_mono.clock->max_cycles;
27
27
#endif
28
-
vdata[CS_HRES_COARSE].mask = tk->tkr_mono.mask;
29
-
vdata[CS_HRES_COARSE].mult = tk->tkr_mono.mult;
30
-
vdata[CS_HRES_COARSE].shift = tk->tkr_mono.shift;
31
-
vdata[CS_RAW].cycle_last = tk->tkr_raw.cycle_last;
28
+
vc[CS_HRES_COARSE].mask = tk->tkr_mono.mask;
29
+
vc[CS_HRES_COARSE].mult = tk->tkr_mono.mult;
30
+
vc[CS_HRES_COARSE].shift = tk->tkr_mono.shift;
31
+
vc[CS_RAW].cycle_last = tk->tkr_raw.cycle_last;
32
32
#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
33
-
vdata[CS_RAW].max_cycles = tk->tkr_raw.clock->max_cycles;
33
+
vc[CS_RAW].max_cycles = tk->tkr_raw.clock->max_cycles;
34
34
#endif
35
-
vdata[CS_RAW].mask = tk->tkr_raw.mask;
36
-
vdata[CS_RAW].mult = tk->tkr_raw.mult;
37
-
vdata[CS_RAW].shift = tk->tkr_raw.shift;
35
+
vc[CS_RAW].mask = tk->tkr_raw.mask;
36
+
vc[CS_RAW].mult = tk->tkr_raw.mult;
37
+
vc[CS_RAW].shift = tk->tkr_raw.shift;
38
38
39
39
/* CLOCK_MONOTONIC */
40
-
vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_MONOTONIC];
40
+
vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_MONOTONIC];
41
41
vdso_ts->sec = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
42
42
43
43
nsec = tk->tkr_mono.xtime_nsec;
···
55
55
nsec += (u64)tk->monotonic_to_boot.tv_nsec << tk->tkr_mono.shift;
56
56
57
57
/* CLOCK_BOOTTIME */
58
-
vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_BOOTTIME];
58
+
vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_BOOTTIME];
59
59
vdso_ts->sec = sec;
60
60
61
61
while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) {
···
65
65
vdso_ts->nsec = nsec;
66
66
67
67
/* CLOCK_MONOTONIC_RAW */
68
-
vdso_ts = &vdata[CS_RAW].basetime[CLOCK_MONOTONIC_RAW];
68
+
vdso_ts = &vc[CS_RAW].basetime[CLOCK_MONOTONIC_RAW];
69
69
vdso_ts->sec = tk->raw_sec;
70
70
vdso_ts->nsec = tk->tkr_raw.xtime_nsec;
71
71
72
72
/* CLOCK_TAI */
73
-
vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_TAI];
73
+
vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_TAI];
74
74
vdso_ts->sec = tk->xtime_sec + (s64)tk->tai_offset;
75
75
vdso_ts->nsec = tk->tkr_mono.xtime_nsec;
76
76
}
77
77
78
78
void update_vsyscall(struct timekeeper *tk)
79
79
{
80
-
struct vdso_data *vdata = __arch_get_k_vdso_data();
80
+
struct vdso_time_data *vdata = vdso_k_time_data;
81
+
struct vdso_clock *vc = vdata->clock_data;
81
82
struct vdso_timestamp *vdso_ts;
82
83
s32 clock_mode;
83
84
u64 nsec;
···
87
86
vdso_write_begin(vdata);
88
87
89
88
clock_mode = tk->tkr_mono.clock->vdso_clock_mode;
90
-
vdata[CS_HRES_COARSE].clock_mode = clock_mode;
91
-
vdata[CS_RAW].clock_mode = clock_mode;
89
+
vc[CS_HRES_COARSE].clock_mode = clock_mode;
90
+
vc[CS_RAW].clock_mode = clock_mode;
92
91
93
92
/* CLOCK_REALTIME also required for time() */
94
-
vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_REALTIME];
93
+
vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME];
95
94
vdso_ts->sec = tk->xtime_sec;
96
95
vdso_ts->nsec = tk->tkr_mono.xtime_nsec;
97
96
98
97
/* CLOCK_REALTIME_COARSE */
99
-
vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_REALTIME_COARSE];
98
+
vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME_COARSE];
100
99
vdso_ts->sec = tk->xtime_sec;
101
100
vdso_ts->nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
102
101
103
102
/* CLOCK_MONOTONIC_COARSE */
104
-
vdso_ts = &vdata[CS_HRES_COARSE].basetime[CLOCK_MONOTONIC_COARSE];
103
+
vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_MONOTONIC_COARSE];
105
104
vdso_ts->sec = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
106
105
nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
107
106
nsec = nsec + tk->wall_to_monotonic.tv_nsec;
···
109
108
110
109
/*
111
110
* Read without the seqlock held by clock_getres().
112
-
* Note: No need to have a second copy.
113
111
*/
114
-
WRITE_ONCE(vdata[CS_HRES_COARSE].hrtimer_res, hrtimer_resolution);
112
+
WRITE_ONCE(vdata->hrtimer_res, hrtimer_resolution);
115
113
116
114
/*
117
115
* If the current clocksource is not VDSO capable, then spare the
118
116
* update of the high resolution parts.
119
117
*/
120
118
if (clock_mode != VDSO_CLOCKMODE_NONE)
121
-
update_vdso_data(vdata, tk);
119
+
update_vdso_time_data(vdata, tk);
122
120
123
121
__arch_update_vsyscall(vdata);
124
122
125
123
vdso_write_end(vdata);
126
124
127
-
__arch_sync_vdso_data(vdata);
125
+
__arch_sync_vdso_time_data(vdata);
128
126
}
129
127
130
128
void update_vsyscall_tz(void)
131
129
{
132
-
struct vdso_data *vdata = __arch_get_k_vdso_data();
130
+
struct vdso_time_data *vdata = vdso_k_time_data;
133
131
134
-
vdata[CS_HRES_COARSE].tz_minuteswest = sys_tz.tz_minuteswest;
135
-
vdata[CS_HRES_COARSE].tz_dsttime = sys_tz.tz_dsttime;
132
+
vdata->tz_minuteswest = sys_tz.tz_minuteswest;
133
+
vdata->tz_dsttime = sys_tz.tz_dsttime;
136
134
137
-
__arch_sync_vdso_data(vdata);
135
+
__arch_sync_vdso_time_data(vdata);
138
136
}
139
137
140
138
/**
···
150
150
*/
151
151
unsigned long vdso_update_begin(void)
152
152
{
153
-
struct vdso_data *vdata = __arch_get_k_vdso_data();
153
+
struct vdso_time_data *vdata = vdso_k_time_data;
154
154
unsigned long flags = timekeeper_lock_irqsave();
155
155
156
156
vdso_write_begin(vdata);
···
167
167
*/
168
168
void vdso_update_end(unsigned long flags)
169
169
{
170
-
struct vdso_data *vdata = __arch_get_k_vdso_data();
170
+
struct vdso_time_data *vdata = vdso_k_time_data;
171
171
172
172
vdso_write_end(vdata);
173
-
__arch_sync_vdso_data(vdata);
173
+
__arch_sync_vdso_time_data(vdata);
174
174
timekeeper_unlock_irqrestore(flags);
175
175
}
+1
-1
lib/Makefile
+1
-1
lib/Makefile
···
124
124
obj-$(CONFIG_DEBUG_INFO_REDUCED) += debug_info.o
125
125
CFLAGS_debug_info.o += $(call cc-option, -femit-struct-debug-detailed=any)
126
126
127
-
obj-y += math/ crypto/ tests/
127
+
obj-y += math/ crypto/ tests/ vdso/
128
128
129
129
obj-$(CONFIG_GENERIC_IOMAP) += iomap.o
130
130
obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o
+5
lib/vdso/Kconfig
+5
lib/vdso/Kconfig
+2
-17
lib/vdso/Makefile
+2
-17
lib/vdso/Makefile
···
1
-
# SPDX-License-Identifier: GPL-2.0
1
+
# SPDX-License-Identifier: GPL-2.0-only
2
2
3
-
GENERIC_VDSO_MK_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
4
-
GENERIC_VDSO_DIR := $(dir $(GENERIC_VDSO_MK_PATH))
5
-
6
-
c-gettimeofday-$(CONFIG_GENERIC_GETTIMEOFDAY) := $(addprefix $(GENERIC_VDSO_DIR), gettimeofday.c)
7
-
c-getrandom-$(CONFIG_VDSO_GETRANDOM) := $(addprefix $(GENERIC_VDSO_DIR), getrandom.c)
8
-
9
-
# This cmd checks that the vdso library does not contain dynamic relocations.
10
-
# It has to be called after the linking of the vdso library and requires it
11
-
# as a parameter.
12
-
#
13
-
# As a workaround for some GNU ld ports which produce unneeded R_*_NONE
14
-
# dynamic relocations, ignore R_*_NONE.
15
-
quiet_cmd_vdso_check = VDSOCHK $@
16
-
cmd_vdso_check = if $(READELF) -rW $@ | grep -v _NONE | grep -q " R_\w*_"; \
17
-
then (echo >&2 "$@: dynamic relocations are not supported"; \
18
-
rm -f $@; /bin/false); fi
3
+
obj-$(CONFIG_GENERIC_VDSO_DATA_STORE) += datastore.o
+18
lib/vdso/Makefile.include
+18
lib/vdso/Makefile.include
···
1
+
# SPDX-License-Identifier: GPL-2.0
2
+
3
+
GENERIC_VDSO_MK_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
4
+
GENERIC_VDSO_DIR := $(dir $(GENERIC_VDSO_MK_PATH))
5
+
6
+
c-gettimeofday-$(CONFIG_GENERIC_GETTIMEOFDAY) := $(addprefix $(GENERIC_VDSO_DIR), gettimeofday.c)
7
+
c-getrandom-$(CONFIG_VDSO_GETRANDOM) := $(addprefix $(GENERIC_VDSO_DIR), getrandom.c)
8
+
9
+
# This cmd checks that the vdso library does not contain dynamic relocations.
10
+
# It has to be called after the linking of the vdso library and requires it
11
+
# as a parameter.
12
+
#
13
+
# As a workaround for some GNU ld ports which produce unneeded R_*_NONE
14
+
# dynamic relocations, ignore R_*_NONE.
15
+
quiet_cmd_vdso_check = VDSOCHK $@
16
+
cmd_vdso_check = if $(READELF) -rW $@ | grep -v _NONE | grep -q " R_\w*_"; \
17
+
then (echo >&2 "$@: dynamic relocations are not supported"; \
18
+
rm -f $@; /bin/false); fi
+129
lib/vdso/datastore.c
+129
lib/vdso/datastore.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
3
+
#include <linux/linkage.h>
4
+
#include <linux/mmap_lock.h>
5
+
#include <linux/mm.h>
6
+
#include <linux/time_namespace.h>
7
+
#include <linux/types.h>
8
+
#include <linux/vdso_datastore.h>
9
+
#include <vdso/datapage.h>
10
+
11
+
/*
12
+
* The vDSO data page.
13
+
*/
14
+
#ifdef CONFIG_HAVE_GENERIC_VDSO
15
+
static union {
16
+
struct vdso_time_data data;
17
+
u8 page[PAGE_SIZE];
18
+
} vdso_time_data_store __page_aligned_data;
19
+
struct vdso_time_data *vdso_k_time_data = &vdso_time_data_store.data;
20
+
static_assert(sizeof(vdso_time_data_store) == PAGE_SIZE);
21
+
#endif /* CONFIG_HAVE_GENERIC_VDSO */
22
+
23
+
#ifdef CONFIG_VDSO_GETRANDOM
24
+
static union {
25
+
struct vdso_rng_data data;
26
+
u8 page[PAGE_SIZE];
27
+
} vdso_rng_data_store __page_aligned_data;
28
+
struct vdso_rng_data *vdso_k_rng_data = &vdso_rng_data_store.data;
29
+
static_assert(sizeof(vdso_rng_data_store) == PAGE_SIZE);
30
+
#endif /* CONFIG_VDSO_GETRANDOM */
31
+
32
+
#ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA
33
+
static union {
34
+
struct vdso_arch_data data;
35
+
u8 page[VDSO_ARCH_DATA_SIZE];
36
+
} vdso_arch_data_store __page_aligned_data;
37
+
struct vdso_arch_data *vdso_k_arch_data = &vdso_arch_data_store.data;
38
+
#endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */
39
+
40
+
static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
41
+
struct vm_area_struct *vma, struct vm_fault *vmf)
42
+
{
43
+
struct page *timens_page = find_timens_vvar_page(vma);
44
+
unsigned long addr, pfn;
45
+
vm_fault_t err;
46
+
47
+
switch (vmf->pgoff) {
48
+
case VDSO_TIME_PAGE_OFFSET:
49
+
if (!IS_ENABLED(CONFIG_HAVE_GENERIC_VDSO))
50
+
return VM_FAULT_SIGBUS;
51
+
pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data));
52
+
if (timens_page) {
53
+
/*
54
+
* Fault in VVAR page too, since it will be accessed
55
+
* to get clock data anyway.
56
+
*/
57
+
addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE;
58
+
err = vmf_insert_pfn(vma, addr, pfn);
59
+
if (unlikely(err & VM_FAULT_ERROR))
60
+
return err;
61
+
pfn = page_to_pfn(timens_page);
62
+
}
63
+
break;
64
+
case VDSO_TIMENS_PAGE_OFFSET:
65
+
/*
66
+
* If a task belongs to a time namespace then a namespace
67
+
* specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
68
+
* the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
69
+
* offset.
70
+
* See also the comment near timens_setup_vdso_data().
71
+
*/
72
+
if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page)
73
+
return VM_FAULT_SIGBUS;
74
+
pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data));
75
+
break;
76
+
case VDSO_RNG_PAGE_OFFSET:
77
+
if (!IS_ENABLED(CONFIG_VDSO_GETRANDOM))
78
+
return VM_FAULT_SIGBUS;
79
+
pfn = __phys_to_pfn(__pa_symbol(vdso_k_rng_data));
80
+
break;
81
+
case VDSO_ARCH_PAGES_START ... VDSO_ARCH_PAGES_END:
82
+
if (!IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
83
+
return VM_FAULT_SIGBUS;
84
+
pfn = __phys_to_pfn(__pa_symbol(vdso_k_arch_data)) +
85
+
vmf->pgoff - VDSO_ARCH_PAGES_START;
86
+
break;
87
+
default:
88
+
return VM_FAULT_SIGBUS;
89
+
}
90
+
91
+
return vmf_insert_pfn(vma, vmf->address, pfn);
92
+
}
93
+
94
+
const struct vm_special_mapping vdso_vvar_mapping = {
95
+
.name = "[vvar]",
96
+
.fault = vvar_fault,
97
+
};
98
+
99
+
struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr)
100
+
{
101
+
return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE,
102
+
VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP | VM_PFNMAP,
103
+
&vdso_vvar_mapping);
104
+
}
105
+
106
+
#ifdef CONFIG_TIME_NS
107
+
/*
108
+
* The vvar page layout depends on whether a task belongs to the root or
109
+
* non-root time namespace. Whenever a task changes its namespace, the VVAR
110
+
* page tables are cleared and then they will be re-faulted with a
111
+
* corresponding layout.
112
+
* See also the comment near timens_setup_vdso_clock_data() for details.
113
+
*/
114
+
int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
115
+
{
116
+
struct mm_struct *mm = task->mm;
117
+
struct vm_area_struct *vma;
118
+
VMA_ITERATOR(vmi, mm, 0);
119
+
120
+
mmap_read_lock(mm);
121
+
for_each_vma(vmi, vma) {
122
+
if (vma_is_special_mapping(vma, &vdso_vvar_mapping))
123
+
zap_vma_pages(vma);
124
+
}
125
+
mmap_read_unlock(mm);
126
+
127
+
return 0;
128
+
}
129
+
#endif
+6
-2
lib/vdso/getrandom.c
+6
-2
lib/vdso/getrandom.c
···
12
12
#include <uapi/linux/mman.h>
13
13
#include <uapi/linux/random.h>
14
14
15
+
/* Bring in default accessors */
16
+
#include <vdso/vsyscall.h>
17
+
15
18
#undef PAGE_SIZE
16
19
#undef PAGE_MASK
17
20
#define PAGE_SIZE (1UL << CONFIG_PAGE_SHIFT)
···
155
152
156
153
/*
157
154
* Prevent the syscall from being reordered wrt current_generation. Pairs with the
158
-
* smp_store_release(&_vdso_rng_data.generation) in random.c.
155
+
* smp_store_release(&vdso_k_rng_data->generation) in random.c.
159
156
*/
160
157
smp_rmb();
161
158
···
259
256
static __always_inline ssize_t
260
257
__cvdso_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len)
261
258
{
262
-
return __cvdso_getrandom_data(__arch_get_vdso_rng_data(), buffer, len, flags, opaque_state, opaque_len);
259
+
return __cvdso_getrandom_data(__arch_get_vdso_u_rng_data(), buffer, len, flags,
260
+
opaque_state, opaque_len);
263
261
}
+111
-85
lib/vdso/gettimeofday.c
+111
-85
lib/vdso/gettimeofday.c
···
5
5
#include <vdso/datapage.h>
6
6
#include <vdso/helpers.h>
7
7
8
+
/* Bring in default accessors */
9
+
#include <vdso/vsyscall.h>
10
+
8
11
#ifndef vdso_calc_ns
9
12
10
13
#ifdef VDSO_DELTA_NOMASK
···
17
14
#endif
18
15
19
16
#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
20
-
static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta)
17
+
static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta)
21
18
{
22
-
return delta < vd->max_cycles;
19
+
return delta < vc->max_cycles;
23
20
}
24
21
#else
25
-
static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta)
22
+
static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta)
26
23
{
27
24
return true;
28
25
}
···
39
36
* Default implementation which works for all sane clocksources. That
40
37
* obviously excludes x86/TSC.
41
38
*/
42
-
static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base)
39
+
static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base)
43
40
{
44
-
u64 delta = (cycles - vd->cycle_last) & VDSO_DELTA_MASK(vd);
41
+
u64 delta = (cycles - vc->cycle_last) & VDSO_DELTA_MASK(vc);
45
42
46
-
if (likely(vdso_delta_ok(vd, delta)))
47
-
return vdso_shift_ns((delta * vd->mult) + base, vd->shift);
43
+
if (likely(vdso_delta_ok(vc, delta)))
44
+
return vdso_shift_ns((delta * vc->mult) + base, vc->shift);
48
45
49
-
return mul_u64_u32_add_u64_shr(delta, vd->mult, base, vd->shift);
46
+
return mul_u64_u32_add_u64_shr(delta, vc->mult, base, vc->shift);
50
47
}
51
48
#endif /* vdso_calc_ns */
52
49
···
58
55
#endif
59
56
60
57
#ifndef vdso_clocksource_ok
61
-
static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
58
+
static inline bool vdso_clocksource_ok(const struct vdso_clock *vc)
62
59
{
63
-
return vd->clock_mode != VDSO_CLOCKMODE_NONE;
60
+
return vc->clock_mode != VDSO_CLOCKMODE_NONE;
64
61
}
65
62
#endif
66
63
···
72
69
#endif
73
70
74
71
#ifdef CONFIG_TIME_NS
75
-
static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
76
-
struct __kernel_timespec *ts)
72
+
73
+
#ifdef CONFIG_GENERIC_VDSO_DATA_STORE
74
+
static __always_inline
75
+
const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd)
77
76
{
78
-
const struct timens_offset *offs = &vdns->offset[clk];
77
+
return (void *)vd + PAGE_SIZE;
78
+
}
79
+
#endif /* CONFIG_GENERIC_VDSO_DATA_STORE */
80
+
81
+
static __always_inline
82
+
int do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
83
+
clockid_t clk, struct __kernel_timespec *ts)
84
+
{
85
+
const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns);
86
+
const struct timens_offset *offs = &vcns->offset[clk];
87
+
const struct vdso_clock *vc = vd->clock_data;
79
88
const struct vdso_timestamp *vdso_ts;
80
-
const struct vdso_data *vd;
81
89
u64 cycles, ns;
82
90
u32 seq;
83
91
s64 sec;
84
92
85
-
vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE);
86
-
vd = __arch_get_timens_vdso_data(vd);
87
93
if (clk != CLOCK_MONOTONIC_RAW)
88
-
vd = &vd[CS_HRES_COARSE];
94
+
vc = &vc[CS_HRES_COARSE];
89
95
else
90
-
vd = &vd[CS_RAW];
91
-
vdso_ts = &vd->basetime[clk];
96
+
vc = &vc[CS_RAW];
97
+
vdso_ts = &vc->basetime[clk];
92
98
93
99
do {
94
-
seq = vdso_read_begin(vd);
100
+
seq = vdso_read_begin(vc);
95
101
96
-
if (unlikely(!vdso_clocksource_ok(vd)))
102
+
if (unlikely(!vdso_clocksource_ok(vc)))
97
103
return -1;
98
104
99
-
cycles = __arch_get_hw_counter(vd->clock_mode, vd);
105
+
cycles = __arch_get_hw_counter(vc->clock_mode, vd);
100
106
if (unlikely(!vdso_cycles_ok(cycles)))
101
107
return -1;
102
-
ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec);
108
+
ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec);
103
109
sec = vdso_ts->sec;
104
-
} while (unlikely(vdso_read_retry(vd, seq)));
110
+
} while (unlikely(vdso_read_retry(vc, seq)));
105
111
106
112
/* Add the namespace offset */
107
113
sec += offs->sec;
···
127
115
}
128
116
#else
129
117
static __always_inline
130
-
const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
118
+
const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd)
131
119
{
132
120
return NULL;
133
121
}
134
122
135
-
static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
136
-
struct __kernel_timespec *ts)
123
+
static __always_inline
124
+
int do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
125
+
clockid_t clk, struct __kernel_timespec *ts)
137
126
{
138
127
return -EINVAL;
139
128
}
140
129
#endif
141
130
142
-
static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
143
-
struct __kernel_timespec *ts)
131
+
static __always_inline
132
+
int do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc,
133
+
clockid_t clk, struct __kernel_timespec *ts)
144
134
{
145
-
const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
135
+
const struct vdso_timestamp *vdso_ts = &vc->basetime[clk];
146
136
u64 cycles, sec, ns;
147
137
u32 seq;
148
138
···
156
142
/*
157
143
* Open coded function vdso_read_begin() to handle
158
144
* VDSO_CLOCKMODE_TIMENS. Time namespace enabled tasks have a
159
-
* special VVAR page installed which has vd->seq set to 1 and
160
-
* vd->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non time
145
+
* special VVAR page installed which has vc->seq set to 1 and
146
+
* vc->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non time
161
147
* namespace affected tasks this does not affect performance
162
-
* because if vd->seq is odd, i.e. a concurrent update is in
163
-
* progress the extra check for vd->clock_mode is just a few
164
-
* extra instructions while spin waiting for vd->seq to become
148
+
* because if vc->seq is odd, i.e. a concurrent update is in
149
+
* progress the extra check for vc->clock_mode is just a few
150
+
* extra instructions while spin waiting for vc->seq to become
165
151
* even again.
166
152
*/
167
-
while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) {
153
+
while (unlikely((seq = READ_ONCE(vc->seq)) & 1)) {
168
154
if (IS_ENABLED(CONFIG_TIME_NS) &&
169
-
vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
170
-
return do_hres_timens(vd, clk, ts);
155
+
vc->clock_mode == VDSO_CLOCKMODE_TIMENS)
156
+
return do_hres_timens(vd, vc, clk, ts);
171
157
cpu_relax();
172
158
}
173
159
smp_rmb();
174
160
175
-
if (unlikely(!vdso_clocksource_ok(vd)))
161
+
if (unlikely(!vdso_clocksource_ok(vc)))
176
162
return -1;
177
163
178
-
cycles = __arch_get_hw_counter(vd->clock_mode, vd);
164
+
cycles = __arch_get_hw_counter(vc->clock_mode, vd);
179
165
if (unlikely(!vdso_cycles_ok(cycles)))
180
166
return -1;
181
-
ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec);
167
+
ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec);
182
168
sec = vdso_ts->sec;
183
-
} while (unlikely(vdso_read_retry(vd, seq)));
169
+
} while (unlikely(vdso_read_retry(vc, seq)));
184
170
185
171
/*
186
172
* Do this outside the loop: a race inside the loop could result
···
193
179
}
194
180
195
181
#ifdef CONFIG_TIME_NS
196
-
static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
197
-
struct __kernel_timespec *ts)
182
+
static __always_inline
183
+
int do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
184
+
clockid_t clk, struct __kernel_timespec *ts)
198
185
{
199
-
const struct vdso_data *vd = __arch_get_timens_vdso_data(vdns);
200
-
const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
201
-
const struct timens_offset *offs = &vdns->offset[clk];
186
+
const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns);
187
+
const struct timens_offset *offs = &vcns->offset[clk];
188
+
const struct vdso_clock *vc = vd->clock_data;
189
+
const struct vdso_timestamp *vdso_ts;
202
190
u64 nsec;
203
191
s64 sec;
204
192
s32 seq;
205
193
194
+
vdso_ts = &vc->basetime[clk];
195
+
206
196
do {
207
-
seq = vdso_read_begin(vd);
197
+
seq = vdso_read_begin(vc);
208
198
sec = vdso_ts->sec;
209
199
nsec = vdso_ts->nsec;
210
-
} while (unlikely(vdso_read_retry(vd, seq)));
200
+
} while (unlikely(vdso_read_retry(vc, seq)));
211
201
212
202
/* Add the namespace offset */
213
203
sec += offs->sec;
···
226
208
return 0;
227
209
}
228
210
#else
229
-
static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
230
-
struct __kernel_timespec *ts)
211
+
static __always_inline
212
+
int do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
213
+
clockid_t clk, struct __kernel_timespec *ts)
231
214
{
232
215
return -1;
233
216
}
234
217
#endif
235
218
236
-
static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,
237
-
struct __kernel_timespec *ts)
219
+
static __always_inline
220
+
int do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc,
221
+
clockid_t clk, struct __kernel_timespec *ts)
238
222
{
239
-
const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
223
+
const struct vdso_timestamp *vdso_ts = &vc->basetime[clk];
240
224
u32 seq;
241
225
242
226
do {
···
246
226
* Open coded function vdso_read_begin() to handle
247
227
* VDSO_CLOCK_TIMENS. See comment in do_hres().
248
228
*/
249
-
while ((seq = READ_ONCE(vd->seq)) & 1) {
229
+
while ((seq = READ_ONCE(vc->seq)) & 1) {
250
230
if (IS_ENABLED(CONFIG_TIME_NS) &&
251
-
vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
252
-
return do_coarse_timens(vd, clk, ts);
231
+
vc->clock_mode == VDSO_CLOCKMODE_TIMENS)
232
+
return do_coarse_timens(vd, vc, clk, ts);
253
233
cpu_relax();
254
234
}
255
235
smp_rmb();
256
236
257
237
ts->tv_sec = vdso_ts->sec;
258
238
ts->tv_nsec = vdso_ts->nsec;
259
-
} while (unlikely(vdso_read_retry(vd, seq)));
239
+
} while (unlikely(vdso_read_retry(vc, seq)));
260
240
261
241
return 0;
262
242
}
263
243
264
244
static __always_inline int
265
-
__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock,
245
+
__cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock,
266
246
struct __kernel_timespec *ts)
267
247
{
248
+
const struct vdso_clock *vc = vd->clock_data;
268
249
u32 msk;
269
250
270
251
/* Check for negative values or invalid clocks */
···
278
257
*/
279
258
msk = 1U << clock;
280
259
if (likely(msk & VDSO_HRES))
281
-
vd = &vd[CS_HRES_COARSE];
260
+
vc = &vc[CS_HRES_COARSE];
282
261
else if (msk & VDSO_COARSE)
283
-
return do_coarse(&vd[CS_HRES_COARSE], clock, ts);
262
+
return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts);
284
263
else if (msk & VDSO_RAW)
285
-
vd = &vd[CS_RAW];
264
+
vc = &vc[CS_RAW];
286
265
else
287
266
return -1;
288
267
289
-
return do_hres(vd, clock, ts);
268
+
return do_hres(vd, vc, clock, ts);
290
269
}
291
270
292
271
static __maybe_unused int
293
-
__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock,
272
+
__cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock,
294
273
struct __kernel_timespec *ts)
295
274
{
296
275
int ret = __cvdso_clock_gettime_common(vd, clock, ts);
···
303
282
static __maybe_unused int
304
283
__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
305
284
{
306
-
return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts);
285
+
return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts);
307
286
}
308
287
309
288
#ifdef BUILD_VDSO32
310
289
static __maybe_unused int
311
-
__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock,
290
+
__cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock,
312
291
struct old_timespec32 *res)
313
292
{
314
293
struct __kernel_timespec ts;
···
329
308
static __maybe_unused int
330
309
__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
331
310
{
332
-
return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res);
311
+
return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res);
333
312
}
334
313
#endif /* BUILD_VDSO32 */
335
314
336
315
static __maybe_unused int
337
-
__cvdso_gettimeofday_data(const struct vdso_data *vd,
316
+
__cvdso_gettimeofday_data(const struct vdso_time_data *vd,
338
317
struct __kernel_old_timeval *tv, struct timezone *tz)
339
318
{
319
+
const struct vdso_clock *vc = vd->clock_data;
340
320
341
321
if (likely(tv != NULL)) {
342
322
struct __kernel_timespec ts;
343
323
344
-
if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
324
+
if (do_hres(vd, &vc[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
345
325
return gettimeofday_fallback(tv, tz);
346
326
347
327
tv->tv_sec = ts.tv_sec;
···
351
329
352
330
if (unlikely(tz != NULL)) {
353
331
if (IS_ENABLED(CONFIG_TIME_NS) &&
354
-
vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
355
-
vd = __arch_get_timens_vdso_data(vd);
332
+
vc->clock_mode == VDSO_CLOCKMODE_TIMENS)
333
+
vd = __arch_get_vdso_u_timens_data(vd);
356
334
357
335
tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
358
336
tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
···
364
342
static __maybe_unused int
365
343
__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
366
344
{
367
-
return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz);
345
+
return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz);
368
346
}
369
347
370
348
#ifdef VDSO_HAS_TIME
371
349
static __maybe_unused __kernel_old_time_t
372
-
__cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time)
350
+
__cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time)
373
351
{
352
+
const struct vdso_clock *vc = vd->clock_data;
374
353
__kernel_old_time_t t;
375
354
376
355
if (IS_ENABLED(CONFIG_TIME_NS) &&
377
-
vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
378
-
vd = __arch_get_timens_vdso_data(vd);
356
+
vc->clock_mode == VDSO_CLOCKMODE_TIMENS) {
357
+
vd = __arch_get_vdso_u_timens_data(vd);
358
+
vc = vd->clock_data;
359
+
}
379
360
380
-
t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
361
+
t = READ_ONCE(vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
381
362
382
363
if (time)
383
364
*time = t;
···
390
365
391
366
static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
392
367
{
393
-
return __cvdso_time_data(__arch_get_vdso_data(), time);
368
+
return __cvdso_time_data(__arch_get_vdso_u_time_data(), time);
394
369
}
395
370
#endif /* VDSO_HAS_TIME */
396
371
397
372
#ifdef VDSO_HAS_CLOCK_GETRES
398
373
static __maybe_unused
399
-
int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock,
374
+
int __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock,
400
375
struct __kernel_timespec *res)
401
376
{
377
+
const struct vdso_clock *vc = vd->clock_data;
402
378
u32 msk;
403
379
u64 ns;
404
380
···
408
382
return -1;
409
383
410
384
if (IS_ENABLED(CONFIG_TIME_NS) &&
411
-
vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
412
-
vd = __arch_get_timens_vdso_data(vd);
385
+
vc->clock_mode == VDSO_CLOCKMODE_TIMENS)
386
+
vd = __arch_get_vdso_u_timens_data(vd);
413
387
414
388
/*
415
389
* Convert the clockid to a bitmask and use it to check which
···
420
394
/*
421
395
* Preserves the behaviour of posix_get_hrtimer_res().
422
396
*/
423
-
ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
397
+
ns = READ_ONCE(vd->hrtimer_res);
424
398
} else if (msk & VDSO_COARSE) {
425
399
/*
426
400
* Preserves the behaviour of posix_get_coarse_res().
···
438
412
}
439
413
440
414
static __maybe_unused
441
-
int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock,
415
+
int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock,
442
416
struct __kernel_timespec *res)
443
417
{
444
418
int ret = __cvdso_clock_getres_common(vd, clock, res);
···
451
425
static __maybe_unused
452
426
int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
453
427
{
454
-
return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res);
428
+
return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res);
455
429
}
456
430
457
431
#ifdef BUILD_VDSO32
458
432
static __maybe_unused int
459
-
__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock,
433
+
__cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock,
460
434
struct old_timespec32 *res)
461
435
{
462
436
struct __kernel_timespec ts;
···
477
451
static __maybe_unused int
478
452
__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
479
453
{
480
-
return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(),
454
+
return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(),
481
455
clock, res);
482
456
}
483
457
#endif /* BUILD_VDSO32 */
+1
tools/include/nolibc/Makefile
+1
tools/include/nolibc/Makefile
+7
tools/include/nolibc/limits.h
+7
tools/include/nolibc/limits.h
+524
tools/include/uapi/linux/elf.h
+524
tools/include/uapi/linux/elf.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
2
+
#ifndef _LINUX_ELF_H
3
+
#define _LINUX_ELF_H
4
+
5
+
#include <linux/types.h>
6
+
#include <linux/elf-em.h>
7
+
8
+
/* 32-bit ELF base types. */
9
+
typedef __u32 Elf32_Addr;
10
+
typedef __u16 Elf32_Half;
11
+
typedef __u32 Elf32_Off;
12
+
typedef __s32 Elf32_Sword;
13
+
typedef __u32 Elf32_Word;
14
+
typedef __u16 Elf32_Versym;
15
+
16
+
/* 64-bit ELF base types. */
17
+
typedef __u64 Elf64_Addr;
18
+
typedef __u16 Elf64_Half;
19
+
typedef __s16 Elf64_SHalf;
20
+
typedef __u64 Elf64_Off;
21
+
typedef __s32 Elf64_Sword;
22
+
typedef __u32 Elf64_Word;
23
+
typedef __u64 Elf64_Xword;
24
+
typedef __s64 Elf64_Sxword;
25
+
typedef __u16 Elf64_Versym;
26
+
27
+
/* These constants are for the segment types stored in the image headers */
28
+
#define PT_NULL 0
29
+
#define PT_LOAD 1
30
+
#define PT_DYNAMIC 2
31
+
#define PT_INTERP 3
32
+
#define PT_NOTE 4
33
+
#define PT_SHLIB 5
34
+
#define PT_PHDR 6
35
+
#define PT_TLS 7 /* Thread local storage segment */
36
+
#define PT_LOOS 0x60000000 /* OS-specific */
37
+
#define PT_HIOS 0x6fffffff /* OS-specific */
38
+
#define PT_LOPROC 0x70000000
39
+
#define PT_HIPROC 0x7fffffff
40
+
#define PT_GNU_EH_FRAME (PT_LOOS + 0x474e550)
41
+
#define PT_GNU_STACK (PT_LOOS + 0x474e551)
42
+
#define PT_GNU_RELRO (PT_LOOS + 0x474e552)
43
+
#define PT_GNU_PROPERTY (PT_LOOS + 0x474e553)
44
+
45
+
46
+
/* ARM MTE memory tag segment type */
47
+
#define PT_AARCH64_MEMTAG_MTE (PT_LOPROC + 0x2)
48
+
49
+
/*
50
+
* Extended Numbering
51
+
*
52
+
* If the real number of program header table entries is larger than
53
+
* or equal to PN_XNUM(0xffff), it is set to sh_info field of the
54
+
* section header at index 0, and PN_XNUM is set to e_phnum
55
+
* field. Otherwise, the section header at index 0 is zero
56
+
* initialized, if it exists.
57
+
*
58
+
* Specifications are available in:
59
+
*
60
+
* - Oracle: Linker and Libraries.
61
+
* Part No: 817–1984–19, August 2011.
62
+
* https://docs.oracle.com/cd/E18752_01/pdf/817-1984.pdf
63
+
*
64
+
* - System V ABI AMD64 Architecture Processor Supplement
65
+
* Draft Version 0.99.4,
66
+
* January 13, 2010.
67
+
* http://www.cs.washington.edu/education/courses/cse351/12wi/supp-docs/abi.pdf
68
+
*/
69
+
#define PN_XNUM 0xffff
70
+
71
+
/* These constants define the different elf file types */
72
+
#define ET_NONE 0
73
+
#define ET_REL 1
74
+
#define ET_EXEC 2
75
+
#define ET_DYN 3
76
+
#define ET_CORE 4
77
+
#define ET_LOPROC 0xff00
78
+
#define ET_HIPROC 0xffff
79
+
80
+
/* This is the info that is needed to parse the dynamic section of the file */
81
+
#define DT_NULL 0
82
+
#define DT_NEEDED 1
83
+
#define DT_PLTRELSZ 2
84
+
#define DT_PLTGOT 3
85
+
#define DT_HASH 4
86
+
#define DT_STRTAB 5
87
+
#define DT_SYMTAB 6
88
+
#define DT_RELA 7
89
+
#define DT_RELASZ 8
90
+
#define DT_RELAENT 9
91
+
#define DT_STRSZ 10
92
+
#define DT_SYMENT 11
93
+
#define DT_INIT 12
94
+
#define DT_FINI 13
95
+
#define DT_SONAME 14
96
+
#define DT_RPATH 15
97
+
#define DT_SYMBOLIC 16
98
+
#define DT_REL 17
99
+
#define DT_RELSZ 18
100
+
#define DT_RELENT 19
101
+
#define DT_PLTREL 20
102
+
#define DT_DEBUG 21
103
+
#define DT_TEXTREL 22
104
+
#define DT_JMPREL 23
105
+
#define DT_ENCODING 32
106
+
#define OLD_DT_LOOS 0x60000000
107
+
#define DT_LOOS 0x6000000d
108
+
#define DT_HIOS 0x6ffff000
109
+
#define DT_VALRNGLO 0x6ffffd00
110
+
#define DT_VALRNGHI 0x6ffffdff
111
+
#define DT_ADDRRNGLO 0x6ffffe00
112
+
#define DT_GNU_HASH 0x6ffffef5
113
+
#define DT_ADDRRNGHI 0x6ffffeff
114
+
#define DT_VERSYM 0x6ffffff0
115
+
#define DT_RELACOUNT 0x6ffffff9
116
+
#define DT_RELCOUNT 0x6ffffffa
117
+
#define DT_FLAGS_1 0x6ffffffb
118
+
#define DT_VERDEF 0x6ffffffc
119
+
#define DT_VERDEFNUM 0x6ffffffd
120
+
#define DT_VERNEED 0x6ffffffe
121
+
#define DT_VERNEEDNUM 0x6fffffff
122
+
#define OLD_DT_HIOS 0x6fffffff
123
+
#define DT_LOPROC 0x70000000
124
+
#define DT_HIPROC 0x7fffffff
125
+
126
+
/* This info is needed when parsing the symbol table */
127
+
#define STB_LOCAL 0
128
+
#define STB_GLOBAL 1
129
+
#define STB_WEAK 2
130
+
131
+
#define STN_UNDEF 0
132
+
133
+
#define STT_NOTYPE 0
134
+
#define STT_OBJECT 1
135
+
#define STT_FUNC 2
136
+
#define STT_SECTION 3
137
+
#define STT_FILE 4
138
+
#define STT_COMMON 5
139
+
#define STT_TLS 6
140
+
141
+
#define VER_FLG_BASE 0x1
142
+
#define VER_FLG_WEAK 0x2
143
+
144
+
#define ELF_ST_BIND(x) ((x) >> 4)
145
+
#define ELF_ST_TYPE(x) ((x) & 0xf)
146
+
#define ELF32_ST_BIND(x) ELF_ST_BIND(x)
147
+
#define ELF32_ST_TYPE(x) ELF_ST_TYPE(x)
148
+
#define ELF64_ST_BIND(x) ELF_ST_BIND(x)
149
+
#define ELF64_ST_TYPE(x) ELF_ST_TYPE(x)
150
+
151
+
typedef struct {
152
+
Elf32_Sword d_tag;
153
+
union {
154
+
Elf32_Sword d_val;
155
+
Elf32_Addr d_ptr;
156
+
} d_un;
157
+
} Elf32_Dyn;
158
+
159
+
typedef struct {
160
+
Elf64_Sxword d_tag; /* entry tag value */
161
+
union {
162
+
Elf64_Xword d_val;
163
+
Elf64_Addr d_ptr;
164
+
} d_un;
165
+
} Elf64_Dyn;
166
+
167
+
/* The following are used with relocations */
168
+
#define ELF32_R_SYM(x) ((x) >> 8)
169
+
#define ELF32_R_TYPE(x) ((x) & 0xff)
170
+
171
+
#define ELF64_R_SYM(i) ((i) >> 32)
172
+
#define ELF64_R_TYPE(i) ((i) & 0xffffffff)
173
+
174
+
typedef struct elf32_rel {
175
+
Elf32_Addr r_offset;
176
+
Elf32_Word r_info;
177
+
} Elf32_Rel;
178
+
179
+
typedef struct elf64_rel {
180
+
Elf64_Addr r_offset; /* Location at which to apply the action */
181
+
Elf64_Xword r_info; /* index and type of relocation */
182
+
} Elf64_Rel;
183
+
184
+
typedef struct elf32_rela {
185
+
Elf32_Addr r_offset;
186
+
Elf32_Word r_info;
187
+
Elf32_Sword r_addend;
188
+
} Elf32_Rela;
189
+
190
+
typedef struct elf64_rela {
191
+
Elf64_Addr r_offset; /* Location at which to apply the action */
192
+
Elf64_Xword r_info; /* index and type of relocation */
193
+
Elf64_Sxword r_addend; /* Constant addend used to compute value */
194
+
} Elf64_Rela;
195
+
196
+
typedef struct elf32_sym {
197
+
Elf32_Word st_name;
198
+
Elf32_Addr st_value;
199
+
Elf32_Word st_size;
200
+
unsigned char st_info;
201
+
unsigned char st_other;
202
+
Elf32_Half st_shndx;
203
+
} Elf32_Sym;
204
+
205
+
typedef struct elf64_sym {
206
+
Elf64_Word st_name; /* Symbol name, index in string tbl */
207
+
unsigned char st_info; /* Type and binding attributes */
208
+
unsigned char st_other; /* No defined meaning, 0 */
209
+
Elf64_Half st_shndx; /* Associated section index */
210
+
Elf64_Addr st_value; /* Value of the symbol */
211
+
Elf64_Xword st_size; /* Associated symbol size */
212
+
} Elf64_Sym;
213
+
214
+
215
+
#define EI_NIDENT 16
216
+
217
+
typedef struct elf32_hdr {
218
+
unsigned char e_ident[EI_NIDENT];
219
+
Elf32_Half e_type;
220
+
Elf32_Half e_machine;
221
+
Elf32_Word e_version;
222
+
Elf32_Addr e_entry; /* Entry point */
223
+
Elf32_Off e_phoff;
224
+
Elf32_Off e_shoff;
225
+
Elf32_Word e_flags;
226
+
Elf32_Half e_ehsize;
227
+
Elf32_Half e_phentsize;
228
+
Elf32_Half e_phnum;
229
+
Elf32_Half e_shentsize;
230
+
Elf32_Half e_shnum;
231
+
Elf32_Half e_shstrndx;
232
+
} Elf32_Ehdr;
233
+
234
+
typedef struct elf64_hdr {
235
+
unsigned char e_ident[EI_NIDENT]; /* ELF "magic number" */
236
+
Elf64_Half e_type;
237
+
Elf64_Half e_machine;
238
+
Elf64_Word e_version;
239
+
Elf64_Addr e_entry; /* Entry point virtual address */
240
+
Elf64_Off e_phoff; /* Program header table file offset */
241
+
Elf64_Off e_shoff; /* Section header table file offset */
242
+
Elf64_Word e_flags;
243
+
Elf64_Half e_ehsize;
244
+
Elf64_Half e_phentsize;
245
+
Elf64_Half e_phnum;
246
+
Elf64_Half e_shentsize;
247
+
Elf64_Half e_shnum;
248
+
Elf64_Half e_shstrndx;
249
+
} Elf64_Ehdr;
250
+
251
+
/* These constants define the permissions on sections in the program
252
+
header, p_flags. */
253
+
#define PF_R 0x4
254
+
#define PF_W 0x2
255
+
#define PF_X 0x1
256
+
257
+
typedef struct elf32_phdr {
258
+
Elf32_Word p_type;
259
+
Elf32_Off p_offset;
260
+
Elf32_Addr p_vaddr;
261
+
Elf32_Addr p_paddr;
262
+
Elf32_Word p_filesz;
263
+
Elf32_Word p_memsz;
264
+
Elf32_Word p_flags;
265
+
Elf32_Word p_align;
266
+
} Elf32_Phdr;
267
+
268
+
typedef struct elf64_phdr {
269
+
Elf64_Word p_type;
270
+
Elf64_Word p_flags;
271
+
Elf64_Off p_offset; /* Segment file offset */
272
+
Elf64_Addr p_vaddr; /* Segment virtual address */
273
+
Elf64_Addr p_paddr; /* Segment physical address */
274
+
Elf64_Xword p_filesz; /* Segment size in file */
275
+
Elf64_Xword p_memsz; /* Segment size in memory */
276
+
Elf64_Xword p_align; /* Segment alignment, file & memory */
277
+
} Elf64_Phdr;
278
+
279
+
/* sh_type */
280
+
#define SHT_NULL 0
281
+
#define SHT_PROGBITS 1
282
+
#define SHT_SYMTAB 2
283
+
#define SHT_STRTAB 3
284
+
#define SHT_RELA 4
285
+
#define SHT_HASH 5
286
+
#define SHT_DYNAMIC 6
287
+
#define SHT_NOTE 7
288
+
#define SHT_NOBITS 8
289
+
#define SHT_REL 9
290
+
#define SHT_SHLIB 10
291
+
#define SHT_DYNSYM 11
292
+
#define SHT_NUM 12
293
+
#define SHT_LOPROC 0x70000000
294
+
#define SHT_HIPROC 0x7fffffff
295
+
#define SHT_LOUSER 0x80000000
296
+
#define SHT_HIUSER 0xffffffff
297
+
298
+
/* sh_flags */
299
+
#define SHF_WRITE 0x1
300
+
#define SHF_ALLOC 0x2
301
+
#define SHF_EXECINSTR 0x4
302
+
#define SHF_RELA_LIVEPATCH 0x00100000
303
+
#define SHF_RO_AFTER_INIT 0x00200000
304
+
#define SHF_MASKPROC 0xf0000000
305
+
306
+
/* special section indexes */
307
+
#define SHN_UNDEF 0
308
+
#define SHN_LORESERVE 0xff00
309
+
#define SHN_LOPROC 0xff00
310
+
#define SHN_HIPROC 0xff1f
311
+
#define SHN_LIVEPATCH 0xff20
312
+
#define SHN_ABS 0xfff1
313
+
#define SHN_COMMON 0xfff2
314
+
#define SHN_HIRESERVE 0xffff
315
+
316
+
typedef struct elf32_shdr {
317
+
Elf32_Word sh_name;
318
+
Elf32_Word sh_type;
319
+
Elf32_Word sh_flags;
320
+
Elf32_Addr sh_addr;
321
+
Elf32_Off sh_offset;
322
+
Elf32_Word sh_size;
323
+
Elf32_Word sh_link;
324
+
Elf32_Word sh_info;
325
+
Elf32_Word sh_addralign;
326
+
Elf32_Word sh_entsize;
327
+
} Elf32_Shdr;
328
+
329
+
typedef struct elf64_shdr {
330
+
Elf64_Word sh_name; /* Section name, index in string tbl */
331
+
Elf64_Word sh_type; /* Type of section */
332
+
Elf64_Xword sh_flags; /* Miscellaneous section attributes */
333
+
Elf64_Addr sh_addr; /* Section virtual addr at execution */
334
+
Elf64_Off sh_offset; /* Section file offset */
335
+
Elf64_Xword sh_size; /* Size of section in bytes */
336
+
Elf64_Word sh_link; /* Index of another section */
337
+
Elf64_Word sh_info; /* Additional section information */
338
+
Elf64_Xword sh_addralign; /* Section alignment */
339
+
Elf64_Xword sh_entsize; /* Entry size if section holds table */
340
+
} Elf64_Shdr;
341
+
342
+
#define EI_MAG0 0 /* e_ident[] indexes */
343
+
#define EI_MAG1 1
344
+
#define EI_MAG2 2
345
+
#define EI_MAG3 3
346
+
#define EI_CLASS 4
347
+
#define EI_DATA 5
348
+
#define EI_VERSION 6
349
+
#define EI_OSABI 7
350
+
#define EI_PAD 8
351
+
352
+
#define ELFMAG0 0x7f /* EI_MAG */
353
+
#define ELFMAG1 'E'
354
+
#define ELFMAG2 'L'
355
+
#define ELFMAG3 'F'
356
+
#define ELFMAG "\177ELF"
357
+
#define SELFMAG 4
358
+
359
+
#define ELFCLASSNONE 0 /* EI_CLASS */
360
+
#define ELFCLASS32 1
361
+
#define ELFCLASS64 2
362
+
#define ELFCLASSNUM 3
363
+
364
+
#define ELFDATANONE 0 /* e_ident[EI_DATA] */
365
+
#define ELFDATA2LSB 1
366
+
#define ELFDATA2MSB 2
367
+
368
+
#define EV_NONE 0 /* e_version, EI_VERSION */
369
+
#define EV_CURRENT 1
370
+
#define EV_NUM 2
371
+
372
+
#define ELFOSABI_NONE 0
373
+
#define ELFOSABI_LINUX 3
374
+
375
+
#ifndef ELF_OSABI
376
+
#define ELF_OSABI ELFOSABI_NONE
377
+
#endif
378
+
379
+
/*
380
+
* Notes used in ET_CORE. Architectures export some of the arch register sets
381
+
* using the corresponding note types via the PTRACE_GETREGSET and
382
+
* PTRACE_SETREGSET requests.
383
+
* The note name for these types is "LINUX", except NT_PRFPREG that is named
384
+
* "CORE".
385
+
*/
386
+
#define NT_PRSTATUS 1
387
+
#define NT_PRFPREG 2
388
+
#define NT_PRPSINFO 3
389
+
#define NT_TASKSTRUCT 4
390
+
#define NT_AUXV 6
391
+
/*
392
+
* Note to userspace developers: size of NT_SIGINFO note may increase
393
+
* in the future to accomodate more fields, don't assume it is fixed!
394
+
*/
395
+
#define NT_SIGINFO 0x53494749
396
+
#define NT_FILE 0x46494c45
397
+
#define NT_PRXFPREG 0x46e62b7f /* copied from gdb5.1/include/elf/common.h */
398
+
#define NT_PPC_VMX 0x100 /* PowerPC Altivec/VMX registers */
399
+
#define NT_PPC_SPE 0x101 /* PowerPC SPE/EVR registers */
400
+
#define NT_PPC_VSX 0x102 /* PowerPC VSX registers */
401
+
#define NT_PPC_TAR 0x103 /* Target Address Register */
402
+
#define NT_PPC_PPR 0x104 /* Program Priority Register */
403
+
#define NT_PPC_DSCR 0x105 /* Data Stream Control Register */
404
+
#define NT_PPC_EBB 0x106 /* Event Based Branch Registers */
405
+
#define NT_PPC_PMU 0x107 /* Performance Monitor Registers */
406
+
#define NT_PPC_TM_CGPR 0x108 /* TM checkpointed GPR Registers */
407
+
#define NT_PPC_TM_CFPR 0x109 /* TM checkpointed FPR Registers */
408
+
#define NT_PPC_TM_CVMX 0x10a /* TM checkpointed VMX Registers */
409
+
#define NT_PPC_TM_CVSX 0x10b /* TM checkpointed VSX Registers */
410
+
#define NT_PPC_TM_SPR 0x10c /* TM Special Purpose Registers */
411
+
#define NT_PPC_TM_CTAR 0x10d /* TM checkpointed Target Address Register */
412
+
#define NT_PPC_TM_CPPR 0x10e /* TM checkpointed Program Priority Register */
413
+
#define NT_PPC_TM_CDSCR 0x10f /* TM checkpointed Data Stream Control Register */
414
+
#define NT_PPC_PKEY 0x110 /* Memory Protection Keys registers */
415
+
#define NT_PPC_DEXCR 0x111 /* PowerPC DEXCR registers */
416
+
#define NT_PPC_HASHKEYR 0x112 /* PowerPC HASHKEYR register */
417
+
#define NT_386_TLS 0x200 /* i386 TLS slots (struct user_desc) */
418
+
#define NT_386_IOPERM 0x201 /* x86 io permission bitmap (1=deny) */
419
+
#define NT_X86_XSTATE 0x202 /* x86 extended state using xsave */
420
+
/* Old binutils treats 0x203 as a CET state */
421
+
#define NT_X86_SHSTK 0x204 /* x86 SHSTK state */
422
+
#define NT_X86_XSAVE_LAYOUT 0x205 /* XSAVE layout description */
423
+
#define NT_S390_HIGH_GPRS 0x300 /* s390 upper register halves */
424
+
#define NT_S390_TIMER 0x301 /* s390 timer register */
425
+
#define NT_S390_TODCMP 0x302 /* s390 TOD clock comparator register */
426
+
#define NT_S390_TODPREG 0x303 /* s390 TOD programmable register */
427
+
#define NT_S390_CTRS 0x304 /* s390 control registers */
428
+
#define NT_S390_PREFIX 0x305 /* s390 prefix register */
429
+
#define NT_S390_LAST_BREAK 0x306 /* s390 breaking event address */
430
+
#define NT_S390_SYSTEM_CALL 0x307 /* s390 system call restart data */
431
+
#define NT_S390_TDB 0x308 /* s390 transaction diagnostic block */
432
+
#define NT_S390_VXRS_LOW 0x309 /* s390 vector registers 0-15 upper half */
433
+
#define NT_S390_VXRS_HIGH 0x30a /* s390 vector registers 16-31 */
434
+
#define NT_S390_GS_CB 0x30b /* s390 guarded storage registers */
435
+
#define NT_S390_GS_BC 0x30c /* s390 guarded storage broadcast control block */
436
+
#define NT_S390_RI_CB 0x30d /* s390 runtime instrumentation */
437
+
#define NT_S390_PV_CPU_DATA 0x30e /* s390 protvirt cpu dump data */
438
+
#define NT_ARM_VFP 0x400 /* ARM VFP/NEON registers */
439
+
#define NT_ARM_TLS 0x401 /* ARM TLS register */
440
+
#define NT_ARM_HW_BREAK 0x402 /* ARM hardware breakpoint registers */
441
+
#define NT_ARM_HW_WATCH 0x403 /* ARM hardware watchpoint registers */
442
+
#define NT_ARM_SYSTEM_CALL 0x404 /* ARM system call number */
443
+
#define NT_ARM_SVE 0x405 /* ARM Scalable Vector Extension registers */
444
+
#define NT_ARM_PAC_MASK 0x406 /* ARM pointer authentication code masks */
445
+
#define NT_ARM_PACA_KEYS 0x407 /* ARM pointer authentication address keys */
446
+
#define NT_ARM_PACG_KEYS 0x408 /* ARM pointer authentication generic key */
447
+
#define NT_ARM_TAGGED_ADDR_CTRL 0x409 /* arm64 tagged address control (prctl()) */
448
+
#define NT_ARM_PAC_ENABLED_KEYS 0x40a /* arm64 ptr auth enabled keys (prctl()) */
449
+
#define NT_ARM_SSVE 0x40b /* ARM Streaming SVE registers */
450
+
#define NT_ARM_ZA 0x40c /* ARM SME ZA registers */
451
+
#define NT_ARM_ZT 0x40d /* ARM SME ZT registers */
452
+
#define NT_ARM_FPMR 0x40e /* ARM floating point mode register */
453
+
#define NT_ARM_POE 0x40f /* ARM POE registers */
454
+
#define NT_ARM_GCS 0x410 /* ARM GCS state */
455
+
#define NT_ARC_V2 0x600 /* ARCv2 accumulator/extra registers */
456
+
#define NT_VMCOREDD 0x700 /* Vmcore Device Dump Note */
457
+
#define NT_MIPS_DSP 0x800 /* MIPS DSP ASE registers */
458
+
#define NT_MIPS_FP_MODE 0x801 /* MIPS floating-point mode */
459
+
#define NT_MIPS_MSA 0x802 /* MIPS SIMD registers */
460
+
#define NT_RISCV_CSR 0x900 /* RISC-V Control and Status Registers */
461
+
#define NT_RISCV_VECTOR 0x901 /* RISC-V vector registers */
462
+
#define NT_RISCV_TAGGED_ADDR_CTRL 0x902 /* RISC-V tagged address control (prctl()) */
463
+
#define NT_LOONGARCH_CPUCFG 0xa00 /* LoongArch CPU config registers */
464
+
#define NT_LOONGARCH_CSR 0xa01 /* LoongArch control and status registers */
465
+
#define NT_LOONGARCH_LSX 0xa02 /* LoongArch Loongson SIMD Extension registers */
466
+
#define NT_LOONGARCH_LASX 0xa03 /* LoongArch Loongson Advanced SIMD Extension registers */
467
+
#define NT_LOONGARCH_LBT 0xa04 /* LoongArch Loongson Binary Translation registers */
468
+
#define NT_LOONGARCH_HW_BREAK 0xa05 /* LoongArch hardware breakpoint registers */
469
+
#define NT_LOONGARCH_HW_WATCH 0xa06 /* LoongArch hardware watchpoint registers */
470
+
471
+
/* Note types with note name "GNU" */
472
+
#define NT_GNU_PROPERTY_TYPE_0 5
473
+
474
+
/* Note header in a PT_NOTE section */
475
+
typedef struct elf32_note {
476
+
Elf32_Word n_namesz; /* Name size */
477
+
Elf32_Word n_descsz; /* Content size */
478
+
Elf32_Word n_type; /* Content type */
479
+
} Elf32_Nhdr;
480
+
481
+
/* Note header in a PT_NOTE section */
482
+
typedef struct elf64_note {
483
+
Elf64_Word n_namesz; /* Name size */
484
+
Elf64_Word n_descsz; /* Content size */
485
+
Elf64_Word n_type; /* Content type */
486
+
} Elf64_Nhdr;
487
+
488
+
/* .note.gnu.property types for EM_AARCH64: */
489
+
#define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
490
+
491
+
/* Bits for GNU_PROPERTY_AARCH64_FEATURE_1_BTI */
492
+
#define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0)
493
+
494
+
typedef struct {
495
+
Elf32_Half vd_version;
496
+
Elf32_Half vd_flags;
497
+
Elf32_Half vd_ndx;
498
+
Elf32_Half vd_cnt;
499
+
Elf32_Word vd_hash;
500
+
Elf32_Word vd_aux;
501
+
Elf32_Word vd_next;
502
+
} Elf32_Verdef;
503
+
504
+
typedef struct {
505
+
Elf64_Half vd_version;
506
+
Elf64_Half vd_flags;
507
+
Elf64_Half vd_ndx;
508
+
Elf64_Half vd_cnt;
509
+
Elf64_Word vd_hash;
510
+
Elf64_Word vd_aux;
511
+
Elf64_Word vd_next;
512
+
} Elf64_Verdef;
513
+
514
+
typedef struct {
515
+
Elf32_Word vda_name;
516
+
Elf32_Word vda_next;
517
+
} Elf32_Verdaux;
518
+
519
+
typedef struct {
520
+
Elf64_Word vda_name;
521
+
Elf64_Word vda_next;
522
+
} Elf64_Verdaux;
523
+
524
+
#endif /* _LINUX_ELF_H */
+4
-1
tools/testing/selftests/lib.mk
+4
-1
tools/testing/selftests/lib.mk
+9
-2
tools/testing/selftests/vDSO/Makefile
+9
-2
tools/testing/selftests/vDSO/Makefile
···
19
19
endif
20
20
21
21
include ../lib.mk
22
+
23
+
CFLAGS += $(TOOLS_INCLUDES)
24
+
25
+
CFLAGS_NOLIBC := -nostdlib -nostdinc -ffreestanding -fno-asynchronous-unwind-tables \
26
+
-fno-stack-protector -include $(top_srcdir)/tools/include/nolibc/nolibc.h \
27
+
-I$(top_srcdir)/tools/include/nolibc/ $(KHDR_INCLUDES)
28
+
22
29
$(OUTPUT)/vdso_test_gettimeofday: parse_vdso.c vdso_test_gettimeofday.c
23
30
$(OUTPUT)/vdso_test_getcpu: parse_vdso.c vdso_test_getcpu.c
24
31
$(OUTPUT)/vdso_test_abi: parse_vdso.c vdso_test_abi.c
25
32
$(OUTPUT)/vdso_test_clock_getres: vdso_test_clock_getres.c
26
33
27
-
$(OUTPUT)/vdso_standalone_test_x86: vdso_standalone_test_x86.c parse_vdso.c
28
-
$(OUTPUT)/vdso_standalone_test_x86: CFLAGS +=-nostdlib -fno-asynchronous-unwind-tables -fno-stack-protector
34
+
$(OUTPUT)/vdso_standalone_test_x86: vdso_standalone_test_x86.c parse_vdso.c | headers
35
+
$(OUTPUT)/vdso_standalone_test_x86: CFLAGS:=$(CFLAGS_NOLIBC) $(CFLAGS)
29
36
30
37
$(OUTPUT)/vdso_test_correctness: vdso_test_correctness.c
31
38
$(OUTPUT)/vdso_test_correctness: LDFLAGS += -ldl
+3
-16
tools/testing/selftests/vDSO/parse_vdso.c
+3
-16
tools/testing/selftests/vDSO/parse_vdso.c
···
19
19
#include <stdint.h>
20
20
#include <string.h>
21
21
#include <limits.h>
22
-
#include <elf.h>
22
+
#include <linux/auxvec.h>
23
+
#include <linux/elf.h>
23
24
24
25
#include "parse_vdso.h"
25
26
26
27
/* And here's the code. */
27
28
#ifndef ELF_BITS
28
-
# if ULONG_MAX > 0xffffffffUL
29
+
# if __SIZEOF_LONG__ >= 8
29
30
# define ELF_BITS 64
30
31
# else
31
32
# define ELF_BITS 32
···
297
296
}
298
297
299
298
return 0;
300
-
}
301
-
302
-
void vdso_init_from_auxv(void *auxv)
303
-
{
304
-
ELF(auxv_t) *elf_auxv = auxv;
305
-
for (int i = 0; elf_auxv[i].a_type != AT_NULL; i++)
306
-
{
307
-
if (elf_auxv[i].a_type == AT_SYSINFO_EHDR) {
308
-
vdso_init_from_sysinfo_ehdr(elf_auxv[i].a_un.a_val);
309
-
return;
310
-
}
311
-
}
312
-
313
-
vdso_info.valid = false;
314
299
}
-1
tools/testing/selftests/vDSO/parse_vdso.h
-1
tools/testing/selftests/vDSO/parse_vdso.h
+44
-128
tools/testing/selftests/vDSO/vdso_standalone_test_x86.c
+44
-128
tools/testing/selftests/vDSO/vdso_standalone_test_x86.c
···
1
1
// SPDX-License-Identifier: GPL-2.0-only
2
2
/*
3
-
* vdso_test.c: Sample code to test parse_vdso.c on x86
4
-
* Copyright (c) 2011-2014 Andy Lutomirski
3
+
* vdso_test_gettimeofday.c: Sample code to test parse_vdso.c and
4
+
* vDSO gettimeofday()
5
+
* Copyright (c) 2014 Andy Lutomirski
5
6
*
6
-
* You can amuse yourself by compiling with:
7
-
* gcc -std=gnu99 -nostdlib
8
-
* -Os -fno-asynchronous-unwind-tables -flto -lgcc_s
9
-
* vdso_standalone_test_x86.c parse_vdso.c
10
-
* to generate a small binary. On x86_64, you can omit -lgcc_s
11
-
* if you want the binary to be completely standalone.
7
+
* Compile with:
8
+
* gcc -std=gnu99 vdso_test_gettimeofday.c parse_vdso_gettimeofday.c
9
+
*
10
+
* Tested on x86, 32-bit and 64-bit. It may work on other architectures, too.
12
11
*/
13
12
14
-
#include <sys/syscall.h>
13
+
#include <stdio.h>
14
+
#ifndef NOLIBC
15
+
#include <sys/auxv.h>
15
16
#include <sys/time.h>
16
-
#include <unistd.h>
17
-
#include <stdint.h>
17
+
#endif
18
18
19
+
#include "../kselftest.h"
19
20
#include "parse_vdso.h"
21
+
#include "vdso_config.h"
22
+
#include "vdso_call.h"
20
23
21
-
/* We need some libc functions... */
22
-
int strcmp(const char *a, const char *b)
24
+
int main(int argc, char **argv)
23
25
{
24
-
/* This implementation is buggy: it never returns -1. */
25
-
while (*a || *b) {
26
-
if (*a != *b)
27
-
return 1;
28
-
if (*a == 0 || *b == 0)
29
-
return 1;
30
-
a++;
31
-
b++;
26
+
const char *version = versions[VDSO_VERSION];
27
+
const char **name = (const char **)&names[VDSO_NAMES];
28
+
29
+
unsigned long sysinfo_ehdr = getauxval(AT_SYSINFO_EHDR);
30
+
if (!sysinfo_ehdr) {
31
+
printf("AT_SYSINFO_EHDR is not present!\n");
32
+
return KSFT_SKIP;
33
+
}
34
+
35
+
vdso_init_from_sysinfo_ehdr(getauxval(AT_SYSINFO_EHDR));
36
+
37
+
/* Find gettimeofday. */
38
+
typedef long (*gtod_t)(struct timeval *tv, struct timezone *tz);
39
+
gtod_t gtod = (gtod_t)vdso_sym(version, name[0]);
40
+
41
+
if (!gtod) {
42
+
printf("Could not find %s\n", name[0]);
43
+
return KSFT_SKIP;
44
+
}
45
+
46
+
struct timeval tv;
47
+
long ret = VDSO_CALL(gtod, 2, &tv, 0);
48
+
49
+
if (ret == 0) {
50
+
printf("The time is %lld.%06lld\n",
51
+
(long long)tv.tv_sec, (long long)tv.tv_usec);
52
+
} else {
53
+
printf("%s failed\n", name[0]);
54
+
return KSFT_FAIL;
32
55
}
33
56
34
57
return 0;
35
58
}
36
-
37
-
/*
38
-
* The clang build needs this, although gcc does not.
39
-
* Stolen from lib/string.c.
40
-
*/
41
-
void *memcpy(void *dest, const void *src, size_t count)
42
-
{
43
-
char *tmp = dest;
44
-
const char *s = src;
45
-
46
-
while (count--)
47
-
*tmp++ = *s++;
48
-
return dest;
49
-
}
50
-
51
-
/* ...and two syscalls. This is x86-specific. */
52
-
static inline long x86_syscall3(long nr, long a0, long a1, long a2)
53
-
{
54
-
long ret;
55
-
#ifdef __x86_64__
56
-
asm volatile ("syscall" : "=a" (ret) : "a" (nr),
57
-
"D" (a0), "S" (a1), "d" (a2) :
58
-
"cc", "memory", "rcx",
59
-
"r8", "r9", "r10", "r11" );
60
-
#else
61
-
asm volatile ("int $0x80" : "=a" (ret) : "a" (nr),
62
-
"b" (a0), "c" (a1), "d" (a2) :
63
-
"cc", "memory" );
64
-
#endif
65
-
return ret;
66
-
}
67
-
68
-
static inline long linux_write(int fd, const void *data, size_t len)
69
-
{
70
-
return x86_syscall3(__NR_write, fd, (long)data, (long)len);
71
-
}
72
-
73
-
static inline void linux_exit(int code)
74
-
{
75
-
x86_syscall3(__NR_exit, code, 0, 0);
76
-
}
77
-
78
-
void to_base10(char *lastdig, time_t n)
79
-
{
80
-
while (n) {
81
-
*lastdig = (n % 10) + '0';
82
-
n /= 10;
83
-
lastdig--;
84
-
}
85
-
}
86
-
87
-
void c_main(void **stack)
88
-
{
89
-
/* Parse the stack */
90
-
long argc = (long)*stack;
91
-
stack += argc + 2;
92
-
93
-
/* Now we're pointing at the environment. Skip it. */
94
-
while(*stack)
95
-
stack++;
96
-
stack++;
97
-
98
-
/* Now we're pointing at auxv. Initialize the vDSO parser. */
99
-
vdso_init_from_auxv((void *)stack);
100
-
101
-
/* Find gettimeofday. */
102
-
typedef long (*gtod_t)(struct timeval *tv, struct timezone *tz);
103
-
gtod_t gtod = (gtod_t)vdso_sym("LINUX_2.6", "__vdso_gettimeofday");
104
-
105
-
if (!gtod)
106
-
linux_exit(1);
107
-
108
-
struct timeval tv;
109
-
long ret = gtod(&tv, 0);
110
-
111
-
if (ret == 0) {
112
-
char buf[] = "The time is .000000\n";
113
-
to_base10(buf + 31, tv.tv_sec);
114
-
to_base10(buf + 38, tv.tv_usec);
115
-
linux_write(1, buf, sizeof(buf) - 1);
116
-
} else {
117
-
linux_exit(ret);
118
-
}
119
-
120
-
linux_exit(0);
121
-
}
122
-
123
-
/*
124
-
* This is the real entry point. It passes the initial stack into
125
-
* the C entry point.
126
-
*/
127
-
asm (
128
-
".text\n"
129
-
".global _start\n"
130
-
".type _start,@function\n"
131
-
"_start:\n\t"
132
-
#ifdef __x86_64__
133
-
"mov %rsp,%rdi\n\t"
134
-
"and $-16,%rsp\n\t"
135
-
"sub $8,%rsp\n\t"
136
-
"jmp c_main"
137
-
#else
138
-
"push %esp\n\t"
139
-
"call c_main\n\t"
140
-
"int $3"
141
-
#endif
142
-
);
+2
-2
tools/testing/selftests/vDSO/vdso_test_gettimeofday.c
+2
-2
tools/testing/selftests/vDSO/vdso_test_gettimeofday.c
···
10
10
* Tested on x86, 32-bit and 64-bit. It may work on other architectures, too.
11
11
*/
12
12
13
-
#include <stdint.h>
14
-
#include <elf.h>
15
13
#include <stdio.h>
14
+
#ifndef NOLIBC
16
15
#include <sys/auxv.h>
17
16
#include <sys/time.h>
17
+
#endif
18
18
19
19
#include "../kselftest.h"
20
20
#include "parse_vdso.h"