Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'random-6.12-rc1-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/crng/random
Pull random number generator updates from Jason Donenfeld:
"Originally I'd planned on sending each of the vDSO getrandom()
architecture ports to their respective arch trees. But as we started
to work on this, we found lots of interesting issues in the shared
code and infrastructure, the fixes for which the various archs needed
to base their work.
So in the end, this turned into a nice collaborative effort fixing up
issues and porting to 5 new architectures -- arm64, powerpc64,
powerpc32, s390x, and loongarch64 -- with everybody pitching in and
commenting on each other's code. It was a fun development cycle.
This contains:
- Numerous fixups to the vDSO selftest infrastructure, getting it
running successfully on more platforms, and fixing bugs in it.
- Additions to the vDSO getrandom & chacha selftests. Basically every
time manual review unearthed a bug in a revision of an arch patch,
or an ambiguity, the tests were augmented.
By the time the last arch was submitted for review, s390x, v1 of
the series was essentially fine right out of the gate.
- Fixes to the the generic C implementation of vDSO getrandom, to
build and run successfully on all archs, decoupling it from
assumptions we had (unintentionally) made on x86_64 that didn't
carry through to the other architectures.
- Port of vDSO getrandom to LoongArch64, from Xi Ruoyao and acked by
Huacai Chen.
- Port of vDSO getrandom to ARM64, from Adhemerval Zanella and acked
by Will Deacon.
- Port of vDSO getrandom to PowerPC, in both 32-bit and 64-bit
varieties, from Christophe Leroy and acked by Michael Ellerman.
- Port of vDSO getrandom to S390X from Heiko Carstens, the arch
maintainer.
While it'd be natural for there to be things to fix up over the course
of the development cycle, these patches got a decent amount of review
from a fairly diverse crew of folks on the mailing lists, and, for the
most part, they've been cooking in linux-next, which has been helpful
for ironing out build issues.
In terms of architectures, I think that mostly takes care of the
important 64-bit archs with hardware still being produced and running
production loads in settings where vDSO getrandom is likely to help.
Arguably there's still RISC-V left, and we'll see for 6.13 whether
they find it useful and submit a port"
* tag 'random-6.12-rc1-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/crng/random: (47 commits)
selftests: vDSO: check cpu caps before running chacha test
s390/vdso: Wire up getrandom() vdso implementation
s390/vdso: Move vdso symbol handling to separate header file
s390/vdso: Allow alternatives in vdso code
s390/module: Provide find_section() helper
s390/facility: Let test_facility() generate static branch if possible
s390/alternatives: Remove ALT_FACILITY_EARLY
s390/facility: Disable compile time optimization for decompressor code
selftests: vDSO: fix vdso_config for s390
selftests: vDSO: fix ELF hash table entry size for s390x
powerpc/vdso: Wire up getrandom() vDSO implementation on VDSO64
powerpc/vdso: Wire up getrandom() vDSO implementation on VDSO32
powerpc/vdso: Refactor CFLAGS for CVDSO build
powerpc/vdso32: Add crtsavres
mm: Define VM_DROPPABLE for powerpc/32
powerpc/vdso: Fix VDSO data access when running in a non-root time namespace
selftests: vDSO: don't include generated headers for chacha test
arm64: vDSO: Wire up getrandom() vDSO implementation
arm64: alternative: make alternative_has_cap_likely() VDSO compatible
selftests: vDSO: also test counter in vdso_test_chacha
...
+1944
-239
+5
-3
Documentation/ABI/stable/vdso
+5
-3
Documentation/ABI/stable/vdso
···
9
9
the vDSO and it often contains useful and highly-optimized alternatives
10
10
to real syscalls.
11
11
12
-
These functions are called just like ordinary C function according to
13
-
your platform's ABI. Call them from a sensible context. (For example,
14
-
if you set CS on x86 to something strange, the vDSO functions are
12
+
These functions are called according to your platform's ABI. On many
13
+
platforms they are called just like ordinary C function. On other platforms
14
+
(ex: powerpc) they are called with the same convention as system calls which
15
+
is different from ordinary C functions. Call them from a sensible context.
16
+
(For example, if you set CS on x86 to something strange, the vDSO functions are
15
17
within their rights to crash.) In addition, if you pass a bad
16
18
pointer to a vDSO function, you might get SIGSEGV instead of -EFAULT.
17
19
+1
arch/arm64/Kconfig
+1
arch/arm64/Kconfig
+4
arch/arm64/include/asm/alternative-macros.h
+4
arch/arm64/include/asm/alternative-macros.h
+5
-1
arch/arm64/include/asm/mman.h
+5
-1
arch/arm64/include/asm/mman.h
···
2
2
#ifndef __ASM_MMAN_H__
3
3
#define __ASM_MMAN_H__
4
4
5
+
#include <uapi/asm/mman.h>
6
+
7
+
#ifndef BUILD_VDSO
5
8
#include <linux/compiler.h>
6
9
#include <linux/types.h>
7
-
#include <uapi/asm/mman.h>
8
10
9
11
static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot,
10
12
unsigned long pkey)
···
69
67
return !(vm_flags & VM_MTE) || (vm_flags & VM_MTE_ALLOWED);
70
68
}
71
69
#define arch_validate_flags(vm_flags) arch_validate_flags(vm_flags)
70
+
71
+
#endif /* !BUILD_VDSO */
72
72
73
73
#endif /* ! __ASM_MMAN_H__ */
+50
arch/arm64/include/asm/vdso/getrandom.h
+50
arch/arm64/include/asm/vdso/getrandom.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
3
+
#ifndef __ASM_VDSO_GETRANDOM_H
4
+
#define __ASM_VDSO_GETRANDOM_H
5
+
6
+
#ifndef __ASSEMBLY__
7
+
8
+
#include <asm/unistd.h>
9
+
#include <asm/vdso/vsyscall.h>
10
+
#include <vdso/datapage.h>
11
+
12
+
/**
13
+
* getrandom_syscall - Invoke the getrandom() syscall.
14
+
* @buffer: Destination buffer to fill with random bytes.
15
+
* @len: Size of @buffer in bytes.
16
+
* @flags: Zero or more GRND_* flags.
17
+
* Returns: The number of random bytes written to @buffer, or a negative value indicating an error.
18
+
*/
19
+
static __always_inline ssize_t getrandom_syscall(void *_buffer, size_t _len, unsigned int _flags)
20
+
{
21
+
register void *buffer asm ("x0") = _buffer;
22
+
register size_t len asm ("x1") = _len;
23
+
register unsigned int flags asm ("x2") = _flags;
24
+
register long ret asm ("x0");
25
+
register long nr asm ("x8") = __NR_getrandom;
26
+
27
+
asm volatile(
28
+
" svc #0\n"
29
+
: "=r" (ret)
30
+
: "r" (buffer), "r" (len), "r" (flags), "r" (nr)
31
+
: "memory");
32
+
33
+
return ret;
34
+
}
35
+
36
+
static __always_inline const struct vdso_rng_data *__arch_get_vdso_rng_data(void)
37
+
{
38
+
/*
39
+
* The RNG data is in the real VVAR data page, but if a task belongs to a time namespace
40
+
* then VVAR_DATA_PAGE_OFFSET points to the namespace-specific VVAR page and VVAR_TIMENS_
41
+
* PAGE_OFFSET points to the real VVAR page.
42
+
*/
43
+
if (IS_ENABLED(CONFIG_TIME_NS) && _vdso_data->clock_mode == VDSO_CLOCKMODE_TIMENS)
44
+
return (void *)&_vdso_rng_data + VVAR_TIMENS_PAGE_OFFSET * (1UL << CONFIG_PAGE_SHIFT);
45
+
return &_vdso_rng_data;
46
+
}
47
+
48
+
#endif /* !__ASSEMBLY__ */
49
+
50
+
#endif /* __ASM_VDSO_GETRANDOM_H */
+15
arch/arm64/include/asm/vdso/vsyscall.h
+15
arch/arm64/include/asm/vdso/vsyscall.h
···
2
2
#ifndef __ASM_VDSO_VSYSCALL_H
3
3
#define __ASM_VDSO_VSYSCALL_H
4
4
5
+
#define __VDSO_RND_DATA_OFFSET 480
6
+
5
7
#ifndef __ASSEMBLY__
6
8
7
9
#include <linux/timekeeper_internal.h>
8
10
#include <vdso/datapage.h>
11
+
12
+
enum vvar_pages {
13
+
VVAR_DATA_PAGE_OFFSET,
14
+
VVAR_TIMENS_PAGE_OFFSET,
15
+
VVAR_NR_PAGES,
16
+
};
9
17
10
18
#define VDSO_PRECISION_MASK ~(0xFF00ULL<<48)
11
19
···
28
20
return vdso_data;
29
21
}
30
22
#define __arch_get_k_vdso_data __arm64_get_k_vdso_data
23
+
24
+
static __always_inline
25
+
struct vdso_rng_data *__arm64_get_k_vdso_rnd_data(void)
26
+
{
27
+
return (void *)vdso_data + __VDSO_RND_DATA_OFFSET;
28
+
}
29
+
#define __arch_get_k_vdso_rng_data __arm64_get_k_vdso_rnd_data
31
30
32
31
static __always_inline
33
32
void __arm64_update_vsyscall(struct vdso_data *vdata, struct timekeeper *tk)
-6
arch/arm64/kernel/vdso.c
-6
arch/arm64/kernel/vdso.c
+17
-8
arch/arm64/kernel/vdso/Makefile
+17
-8
arch/arm64/kernel/vdso/Makefile
···
9
9
# Include the generic Makefile to check the built vdso.
10
10
include $(srctree)/lib/vdso/Makefile
11
11
12
-
obj-vdso := vgettimeofday.o note.o sigreturn.o
12
+
obj-vdso := vgettimeofday.o note.o sigreturn.o vgetrandom.o vgetrandom-chacha.o
13
13
14
14
# Build rules
15
15
targets := $(obj-vdso) vdso.so vdso.so.dbg
···
34
34
ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO
35
35
36
36
# -Wmissing-prototypes and -Wmissing-declarations are removed from
37
-
# the CFLAGS of vgettimeofday.c to make possible to build the
38
-
# kernel with CONFIG_WERROR enabled.
39
-
CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) \
40
-
$(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) \
41
-
$(CC_FLAGS_LTO) $(CC_FLAGS_CFI) \
42
-
-Wmissing-prototypes -Wmissing-declarations
37
+
# the CFLAGS to make possible to build the kernel with CONFIG_WERROR enabled.
38
+
CC_FLAGS_REMOVE_VDSO := $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) \
39
+
$(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) \
40
+
$(CC_FLAGS_LTO) $(CC_FLAGS_CFI) \
41
+
-Wmissing-prototypes -Wmissing-declarations
43
42
44
-
CFLAGS_vgettimeofday.o = -O2 -mcmodel=tiny -fasynchronous-unwind-tables
43
+
CC_FLAGS_ADD_VDSO := -O2 -mcmodel=tiny -fasynchronous-unwind-tables
44
+
45
+
CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_REMOVE_VDSO)
46
+
CFLAGS_REMOVE_vgetrandom.o = $(CC_FLAGS_REMOVE_VDSO)
47
+
48
+
CFLAGS_vgettimeofday.o = $(CC_FLAGS_ADD_VDSO)
49
+
CFLAGS_vgetrandom.o = $(CC_FLAGS_ADD_VDSO)
45
50
46
51
ifneq ($(c-gettimeofday-y),)
47
52
CFLAGS_vgettimeofday.o += -include $(c-gettimeofday-y)
53
+
endif
54
+
55
+
ifneq ($(c-getrandom-y),)
56
+
CFLAGS_vgetrandom.o += -include $(c-getrandom-y)
48
57
endif
49
58
50
59
targets += vdso.lds
+4
arch/arm64/kernel/vdso/vdso.lds.S
+4
arch/arm64/kernel/vdso/vdso.lds.S
···
11
11
#include <linux/const.h>
12
12
#include <asm/page.h>
13
13
#include <asm/vdso.h>
14
+
#include <asm/vdso/vsyscall.h>
14
15
#include <asm-generic/vmlinux.lds.h>
16
+
#include <vdso/datapage.h>
15
17
16
18
OUTPUT_FORMAT("elf64-littleaarch64", "elf64-bigaarch64", "elf64-littleaarch64")
17
19
OUTPUT_ARCH(aarch64)
···
21
19
SECTIONS
22
20
{
23
21
PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
22
+
PROVIDE(_vdso_rng_data = _vdso_data + __VDSO_RND_DATA_OFFSET);
24
23
#ifdef CONFIG_TIME_NS
25
24
PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
26
25
#endif
···
105
102
__kernel_gettimeofday;
106
103
__kernel_clock_gettime;
107
104
__kernel_clock_getres;
105
+
__kernel_getrandom;
108
106
local: *;
109
107
};
110
108
}
+172
arch/arm64/kernel/vdso/vgetrandom-chacha.S
+172
arch/arm64/kernel/vdso/vgetrandom-chacha.S
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
3
+
#include <linux/linkage.h>
4
+
#include <asm/cache.h>
5
+
#include <asm/assembler.h>
6
+
7
+
.text
8
+
9
+
#define state0 v0
10
+
#define state1 v1
11
+
#define state2 v2
12
+
#define state3 v3
13
+
#define copy0 v4
14
+
#define copy0_q q4
15
+
#define copy1 v5
16
+
#define copy2 v6
17
+
#define copy3 v7
18
+
#define copy3_d d7
19
+
#define one_d d16
20
+
#define one_q q16
21
+
#define one_v v16
22
+
#define tmp v17
23
+
#define rot8 v18
24
+
25
+
/*
26
+
* ARM64 ChaCha20 implementation meant for vDSO. Produces a given positive
27
+
* number of blocks of output with nonce 0, taking an input key and 8-bytes
28
+
* counter. Importantly does not spill to the stack.
29
+
*
30
+
* This implementation avoids d8-d15 because they are callee-save in user
31
+
* space.
32
+
*
33
+
* void __arch_chacha20_blocks_nostack(uint8_t *dst_bytes,
34
+
* const uint8_t *key,
35
+
* uint32_t *counter,
36
+
* size_t nblocks)
37
+
*
38
+
* x0: output bytes
39
+
* x1: 32-byte key input
40
+
* x2: 8-byte counter input/output
41
+
* x3: number of 64-byte block to write to output
42
+
*/
43
+
SYM_FUNC_START(__arch_chacha20_blocks_nostack)
44
+
45
+
/* copy0 = "expand 32-byte k" */
46
+
mov_q x8, 0x3320646e61707865
47
+
mov_q x9, 0x6b20657479622d32
48
+
mov copy0.d[0], x8
49
+
mov copy0.d[1], x9
50
+
51
+
/* copy1,copy2 = key */
52
+
ld1 { copy1.4s, copy2.4s }, [x1]
53
+
/* copy3 = counter || zero nonce */
54
+
ld1 { copy3.2s }, [x2]
55
+
56
+
movi one_v.2s, #1
57
+
uzp1 one_v.4s, one_v.4s, one_v.4s
58
+
59
+
.Lblock:
60
+
/* copy state to auxiliary vectors for the final add after the permute. */
61
+
mov state0.16b, copy0.16b
62
+
mov state1.16b, copy1.16b
63
+
mov state2.16b, copy2.16b
64
+
mov state3.16b, copy3.16b
65
+
66
+
mov w4, 20
67
+
.Lpermute:
68
+
/*
69
+
* Permute one 64-byte block where the state matrix is stored in the four NEON
70
+
* registers state0-state3. It performs matrix operations on four words in parallel,
71
+
* but requires shuffling to rearrange the words after each round.
72
+
*/
73
+
74
+
.Ldoubleround:
75
+
/* state0 += state1, state3 = rotl32(state3 ^ state0, 16) */
76
+
add state0.4s, state0.4s, state1.4s
77
+
eor state3.16b, state3.16b, state0.16b
78
+
rev32 state3.8h, state3.8h
79
+
80
+
/* state2 += state3, state1 = rotl32(state1 ^ state2, 12) */
81
+
add state2.4s, state2.4s, state3.4s
82
+
eor tmp.16b, state1.16b, state2.16b
83
+
shl state1.4s, tmp.4s, #12
84
+
sri state1.4s, tmp.4s, #20
85
+
86
+
/* state0 += state1, state3 = rotl32(state3 ^ state0, 8) */
87
+
add state0.4s, state0.4s, state1.4s
88
+
eor tmp.16b, state3.16b, state0.16b
89
+
shl state3.4s, tmp.4s, #8
90
+
sri state3.4s, tmp.4s, #24
91
+
92
+
/* state2 += state3, state1 = rotl32(state1 ^ state2, 7) */
93
+
add state2.4s, state2.4s, state3.4s
94
+
eor tmp.16b, state1.16b, state2.16b
95
+
shl state1.4s, tmp.4s, #7
96
+
sri state1.4s, tmp.4s, #25
97
+
98
+
/* state1[0,1,2,3] = state1[1,2,3,0] */
99
+
ext state1.16b, state1.16b, state1.16b, #4
100
+
/* state2[0,1,2,3] = state2[2,3,0,1] */
101
+
ext state2.16b, state2.16b, state2.16b, #8
102
+
/* state3[0,1,2,3] = state3[1,2,3,0] */
103
+
ext state3.16b, state3.16b, state3.16b, #12
104
+
105
+
/* state0 += state1, state3 = rotl32(state3 ^ state0, 16) */
106
+
add state0.4s, state0.4s, state1.4s
107
+
eor state3.16b, state3.16b, state0.16b
108
+
rev32 state3.8h, state3.8h
109
+
110
+
/* state2 += state3, state1 = rotl32(state1 ^ state2, 12) */
111
+
add state2.4s, state2.4s, state3.4s
112
+
eor tmp.16b, state1.16b, state2.16b
113
+
shl state1.4s, tmp.4s, #12
114
+
sri state1.4s, tmp.4s, #20
115
+
116
+
/* state0 += state1, state3 = rotl32(state3 ^ state0, 8) */
117
+
add state0.4s, state0.4s, state1.4s
118
+
eor tmp.16b, state3.16b, state0.16b
119
+
shl state3.4s, tmp.4s, #8
120
+
sri state3.4s, tmp.4s, #24
121
+
122
+
/* state2 += state3, state1 = rotl32(state1 ^ state2, 7) */
123
+
add state2.4s, state2.4s, state3.4s
124
+
eor tmp.16b, state1.16b, state2.16b
125
+
shl state1.4s, tmp.4s, #7
126
+
sri state1.4s, tmp.4s, #25
127
+
128
+
/* state1[0,1,2,3] = state1[3,0,1,2] */
129
+
ext state1.16b, state1.16b, state1.16b, #12
130
+
/* state2[0,1,2,3] = state2[2,3,0,1] */
131
+
ext state2.16b, state2.16b, state2.16b, #8
132
+
/* state3[0,1,2,3] = state3[1,2,3,0] */
133
+
ext state3.16b, state3.16b, state3.16b, #4
134
+
135
+
subs w4, w4, #2
136
+
b.ne .Ldoubleround
137
+
138
+
/* output0 = state0 + state0 */
139
+
add state0.4s, state0.4s, copy0.4s
140
+
/* output1 = state1 + state1 */
141
+
add state1.4s, state1.4s, copy1.4s
142
+
/* output2 = state2 + state2 */
143
+
add state2.4s, state2.4s, copy2.4s
144
+
/* output2 = state3 + state3 */
145
+
add state3.4s, state3.4s, copy3.4s
146
+
st1 { state0.16b - state3.16b }, [x0]
147
+
148
+
/*
149
+
* ++copy3.counter, the 'add' clears the upper half of the SIMD register
150
+
* which is the expected behaviour here.
151
+
*/
152
+
add copy3_d, copy3_d, one_d
153
+
154
+
/* output += 64, --nblocks */
155
+
add x0, x0, 64
156
+
subs x3, x3, #1
157
+
b.ne .Lblock
158
+
159
+
/* counter = copy3.counter */
160
+
st1 { copy3.2s }, [x2]
161
+
162
+
/* Zero out the potentially sensitive regs, in case nothing uses these again. */
163
+
movi state0.16b, #0
164
+
movi state1.16b, #0
165
+
movi state2.16b, #0
166
+
movi state3.16b, #0
167
+
movi copy1.16b, #0
168
+
movi copy2.16b, #0
169
+
ret
170
+
SYM_FUNC_END(__arch_chacha20_blocks_nostack)
171
+
172
+
emit_aarch64_feature_1_and
+15
arch/arm64/kernel/vdso/vgetrandom.c
+15
arch/arm64/kernel/vdso/vgetrandom.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
3
+
#include <uapi/asm-generic/errno.h>
4
+
5
+
typeof(__cvdso_getrandom) __kernel_getrandom;
6
+
7
+
ssize_t __kernel_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len)
8
+
{
9
+
if (alternative_has_cap_likely(ARM64_HAS_FPSIMD))
10
+
return __cvdso_getrandom(buffer, len, flags, opaque_state, opaque_len);
11
+
12
+
if (unlikely(opaque_len == ~0UL && !buffer && !len && !flags))
13
+
return -ENOSYS;
14
+
return getrandom_syscall(buffer, len, flags);
15
+
}
+1
arch/loongarch/Kconfig
+1
arch/loongarch/Kconfig
+39
arch/loongarch/include/asm/vdso/getrandom.h
+39
arch/loongarch/include/asm/vdso/getrandom.h
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/*
3
+
* Copyright (C) 2024 Xi Ruoyao <xry111@xry111.site>. All Rights Reserved.
4
+
*/
5
+
#ifndef __ASM_VDSO_GETRANDOM_H
6
+
#define __ASM_VDSO_GETRANDOM_H
7
+
8
+
#ifndef __ASSEMBLY__
9
+
10
+
#include <asm/unistd.h>
11
+
#include <asm/vdso/vdso.h>
12
+
13
+
static __always_inline ssize_t getrandom_syscall(void *_buffer, size_t _len, unsigned int _flags)
14
+
{
15
+
register long ret asm("a0");
16
+
register long nr asm("a7") = __NR_getrandom;
17
+
register void *buffer asm("a0") = _buffer;
18
+
register size_t len asm("a1") = _len;
19
+
register unsigned int flags asm("a2") = _flags;
20
+
21
+
asm volatile(
22
+
" syscall 0\n"
23
+
: "+r" (ret)
24
+
: "r" (nr), "r" (buffer), "r" (len), "r" (flags)
25
+
: "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", "$t8",
26
+
"memory");
27
+
28
+
return ret;
29
+
}
30
+
31
+
static __always_inline const struct vdso_rng_data *__arch_get_vdso_rng_data(void)
32
+
{
33
+
return (const struct vdso_rng_data *)(get_vdso_data() + VVAR_LOONGARCH_PAGES_START *
34
+
PAGE_SIZE + offsetof(struct loongarch_vdso_data, rng_data));
35
+
}
36
+
37
+
#endif /* !__ASSEMBLY__ */
38
+
39
+
#endif /* __ASM_VDSO_GETRANDOM_H */
+6
arch/loongarch/include/asm/vdso/vdso.h
+6
arch/loongarch/include/asm/vdso/vdso.h
···
4
4
* Copyright (C) 2020-2022 Loongson Technology Corporation Limited
5
5
*/
6
6
7
+
#ifndef _ASM_VDSO_VDSO_H
8
+
#define _ASM_VDSO_VDSO_H
9
+
7
10
#ifndef __ASSEMBLY__
8
11
9
12
#include <asm/asm.h>
···
19
16
20
17
struct loongarch_vdso_data {
21
18
struct vdso_pcpu_data pdata[NR_CPUS];
19
+
struct vdso_rng_data rng_data;
22
20
};
23
21
24
22
/*
···
67
63
}
68
64
69
65
#endif /* __ASSEMBLY__ */
66
+
67
+
#endif
+8
arch/loongarch/include/asm/vdso/vsyscall.h
+8
arch/loongarch/include/asm/vdso/vsyscall.h
···
8
8
#include <vdso/datapage.h>
9
9
10
10
extern struct vdso_data *vdso_data;
11
+
extern struct vdso_rng_data *vdso_rng_data;
11
12
12
13
/*
13
14
* Update the vDSO data page to keep in sync with kernel timekeeping.
···
19
18
return vdso_data;
20
19
}
21
20
#define __arch_get_k_vdso_data __loongarch_get_k_vdso_data
21
+
22
+
static __always_inline
23
+
struct vdso_rng_data *__loongarch_get_k_vdso_rng_data(void)
24
+
{
25
+
return vdso_rng_data;
26
+
}
27
+
#define __arch_get_k_vdso_rng_data __loongarch_get_k_vdso_rng_data
22
28
23
29
/* The asm-generic header needs to be included after the definitions above */
24
30
#include <asm-generic/vdso/vsyscall.h>
+1
arch/loongarch/kernel/vdso.c
+1
arch/loongarch/kernel/vdso.c
···
37
37
static struct page *vdso_pages[] = { NULL };
38
38
struct vdso_data *vdso_data = generic_vdso_data.data;
39
39
struct vdso_pcpu_data *vdso_pdata = loongarch_vdso_data.vdata.pdata;
40
+
struct vdso_rng_data *vdso_rng_data = &loongarch_vdso_data.vdata.rng_data;
40
41
41
42
static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma)
42
43
{
+6
-1
arch/loongarch/vdso/Makefile
+6
-1
arch/loongarch/vdso/Makefile
···
4
4
# Include the generic Makefile to check the built vdso.
5
5
include $(srctree)/lib/vdso/Makefile
6
6
7
-
obj-vdso-y := elf.o vgetcpu.o vgettimeofday.o sigreturn.o
7
+
obj-vdso-y := elf.o vgetcpu.o vgettimeofday.o vgetrandom.o \
8
+
vgetrandom-chacha.o sigreturn.o
8
9
9
10
# Common compiler flags between ABIs.
10
11
ccflags-vdso := \
···
28
27
29
28
ifneq ($(c-gettimeofday-y),)
30
29
CFLAGS_vgettimeofday.o += -include $(c-gettimeofday-y)
30
+
endif
31
+
32
+
ifneq ($(c-getrandom-y),)
33
+
CFLAGS_vgetrandom.o += -include $(c-getrandom-y)
31
34
endif
32
35
33
36
# VDSO linker flags.
+1
arch/loongarch/vdso/vdso.lds.S
+1
arch/loongarch/vdso/vdso.lds.S
+242
arch/loongarch/vdso/vgetrandom-chacha.S
+242
arch/loongarch/vdso/vgetrandom-chacha.S
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/*
3
+
* Copyright (C) 2024 Xi Ruoyao <xry111@xry111.site>. All Rights Reserved.
4
+
*/
5
+
6
+
#include <asm/asm.h>
7
+
#include <asm/regdef.h>
8
+
#include <linux/linkage.h>
9
+
10
+
.text
11
+
12
+
/* Salsa20 quarter-round */
13
+
.macro QR a b c d
14
+
add.w \a, \a, \b
15
+
xor \d, \d, \a
16
+
rotri.w \d, \d, 16
17
+
18
+
add.w \c, \c, \d
19
+
xor \b, \b, \c
20
+
rotri.w \b, \b, 20
21
+
22
+
add.w \a, \a, \b
23
+
xor \d, \d, \a
24
+
rotri.w \d, \d, 24
25
+
26
+
add.w \c, \c, \d
27
+
xor \b, \b, \c
28
+
rotri.w \b, \b, 25
29
+
.endm
30
+
31
+
/*
32
+
* Very basic LoongArch implementation of ChaCha20. Produces a given positive
33
+
* number of blocks of output with a nonce of 0, taking an input key and
34
+
* 8-byte counter. Importantly does not spill to the stack. Its arguments
35
+
* are:
36
+
*
37
+
* a0: output bytes
38
+
* a1: 32-byte key input
39
+
* a2: 8-byte counter input/output
40
+
* a3: number of 64-byte blocks to write to output
41
+
*/
42
+
SYM_FUNC_START(__arch_chacha20_blocks_nostack)
43
+
44
+
/* We don't need a frame pointer */
45
+
#define s9 fp
46
+
47
+
#define output a0
48
+
#define key a1
49
+
#define counter a2
50
+
#define nblocks a3
51
+
#define i a4
52
+
#define state0 s0
53
+
#define state1 s1
54
+
#define state2 s2
55
+
#define state3 s3
56
+
#define state4 s4
57
+
#define state5 s5
58
+
#define state6 s6
59
+
#define state7 s7
60
+
#define state8 s8
61
+
#define state9 s9
62
+
#define state10 a5
63
+
#define state11 a6
64
+
#define state12 a7
65
+
#define state13 t0
66
+
#define state14 t1
67
+
#define state15 t2
68
+
#define cnt_lo t3
69
+
#define cnt_hi t4
70
+
#define copy0 t5
71
+
#define copy1 t6
72
+
#define copy2 t7
73
+
74
+
/* Reuse i as copy3 */
75
+
#define copy3 i
76
+
77
+
/*
78
+
* The ABI requires s0-s9 saved, and sp aligned to 16-byte.
79
+
* This does not violate the stack-less requirement: no sensitive data
80
+
* is spilled onto the stack.
81
+
*/
82
+
PTR_ADDI sp, sp, (-SZREG * 10) & STACK_ALIGN
83
+
REG_S s0, sp, 0
84
+
REG_S s1, sp, SZREG
85
+
REG_S s2, sp, SZREG * 2
86
+
REG_S s3, sp, SZREG * 3
87
+
REG_S s4, sp, SZREG * 4
88
+
REG_S s5, sp, SZREG * 5
89
+
REG_S s6, sp, SZREG * 6
90
+
REG_S s7, sp, SZREG * 7
91
+
REG_S s8, sp, SZREG * 8
92
+
REG_S s9, sp, SZREG * 9
93
+
94
+
li.w copy0, 0x61707865
95
+
li.w copy1, 0x3320646e
96
+
li.w copy2, 0x79622d32
97
+
98
+
ld.w cnt_lo, counter, 0
99
+
ld.w cnt_hi, counter, 4
100
+
101
+
.Lblock:
102
+
/* state[0,1,2,3] = "expand 32-byte k" */
103
+
move state0, copy0
104
+
move state1, copy1
105
+
move state2, copy2
106
+
li.w state3, 0x6b206574
107
+
108
+
/* state[4,5,..,11] = key */
109
+
ld.w state4, key, 0
110
+
ld.w state5, key, 4
111
+
ld.w state6, key, 8
112
+
ld.w state7, key, 12
113
+
ld.w state8, key, 16
114
+
ld.w state9, key, 20
115
+
ld.w state10, key, 24
116
+
ld.w state11, key, 28
117
+
118
+
/* state[12,13] = counter */
119
+
move state12, cnt_lo
120
+
move state13, cnt_hi
121
+
122
+
/* state[14,15] = 0 */
123
+
move state14, zero
124
+
move state15, zero
125
+
126
+
li.w i, 10
127
+
.Lpermute:
128
+
/* odd round */
129
+
QR state0, state4, state8, state12
130
+
QR state1, state5, state9, state13
131
+
QR state2, state6, state10, state14
132
+
QR state3, state7, state11, state15
133
+
134
+
/* even round */
135
+
QR state0, state5, state10, state15
136
+
QR state1, state6, state11, state12
137
+
QR state2, state7, state8, state13
138
+
QR state3, state4, state9, state14
139
+
140
+
addi.w i, i, -1
141
+
bnez i, .Lpermute
142
+
143
+
/*
144
+
* copy[3] = "expa", materialize it here because copy[3] shares the
145
+
* same register with i which just became dead.
146
+
*/
147
+
li.w copy3, 0x6b206574
148
+
149
+
/* output[0,1,2,3] = copy[0,1,2,3] + state[0,1,2,3] */
150
+
add.w state0, state0, copy0
151
+
add.w state1, state1, copy1
152
+
add.w state2, state2, copy2
153
+
add.w state3, state3, copy3
154
+
st.w state0, output, 0
155
+
st.w state1, output, 4
156
+
st.w state2, output, 8
157
+
st.w state3, output, 12
158
+
159
+
/* from now on state[0,1,2,3] are scratch registers */
160
+
161
+
/* state[0,1,2,3] = lo32(key) */
162
+
ld.w state0, key, 0
163
+
ld.w state1, key, 4
164
+
ld.w state2, key, 8
165
+
ld.w state3, key, 12
166
+
167
+
/* output[4,5,6,7] = state[0,1,2,3] + state[4,5,6,7] */
168
+
add.w state4, state4, state0
169
+
add.w state5, state5, state1
170
+
add.w state6, state6, state2
171
+
add.w state7, state7, state3
172
+
st.w state4, output, 16
173
+
st.w state5, output, 20
174
+
st.w state6, output, 24
175
+
st.w state7, output, 28
176
+
177
+
/* state[0,1,2,3] = hi32(key) */
178
+
ld.w state0, key, 16
179
+
ld.w state1, key, 20
180
+
ld.w state2, key, 24
181
+
ld.w state3, key, 28
182
+
183
+
/* output[8,9,10,11] = state[0,1,2,3] + state[8,9,10,11] */
184
+
add.w state8, state8, state0
185
+
add.w state9, state9, state1
186
+
add.w state10, state10, state2
187
+
add.w state11, state11, state3
188
+
st.w state8, output, 32
189
+
st.w state9, output, 36
190
+
st.w state10, output, 40
191
+
st.w state11, output, 44
192
+
193
+
/* output[12,13,14,15] = state[12,13,14,15] + [cnt_lo, cnt_hi, 0, 0] */
194
+
add.w state12, state12, cnt_lo
195
+
add.w state13, state13, cnt_hi
196
+
st.w state12, output, 48
197
+
st.w state13, output, 52
198
+
st.w state14, output, 56
199
+
st.w state15, output, 60
200
+
201
+
/* ++counter */
202
+
addi.w cnt_lo, cnt_lo, 1
203
+
sltui state0, cnt_lo, 1
204
+
add.w cnt_hi, cnt_hi, state0
205
+
206
+
/* output += 64 */
207
+
PTR_ADDI output, output, 64
208
+
/* --nblocks */
209
+
PTR_ADDI nblocks, nblocks, -1
210
+
bnez nblocks, .Lblock
211
+
212
+
/* counter = [cnt_lo, cnt_hi] */
213
+
st.w cnt_lo, counter, 0
214
+
st.w cnt_hi, counter, 4
215
+
216
+
/*
217
+
* Zero out the potentially sensitive regs, in case nothing uses these
218
+
* again. As at now copy[0,1,2,3] just contains "expand 32-byte k" and
219
+
* state[0,...,9] are s0-s9 those we'll restore in the epilogue, so we
220
+
* only need to zero state[11,...,15].
221
+
*/
222
+
move state10, zero
223
+
move state11, zero
224
+
move state12, zero
225
+
move state13, zero
226
+
move state14, zero
227
+
move state15, zero
228
+
229
+
REG_L s0, sp, 0
230
+
REG_L s1, sp, SZREG
231
+
REG_L s2, sp, SZREG * 2
232
+
REG_L s3, sp, SZREG * 3
233
+
REG_L s4, sp, SZREG * 4
234
+
REG_L s5, sp, SZREG * 5
235
+
REG_L s6, sp, SZREG * 6
236
+
REG_L s7, sp, SZREG * 7
237
+
REG_L s8, sp, SZREG * 8
238
+
REG_L s9, sp, SZREG * 9
239
+
PTR_ADDI sp, sp, -((-SZREG * 10) & STACK_ALIGN)
240
+
241
+
jr ra
242
+
SYM_FUNC_END(__arch_chacha20_blocks_nostack)
+10
arch/loongarch/vdso/vgetrandom.c
+10
arch/loongarch/vdso/vgetrandom.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/*
3
+
* Copyright (C) 2024 Xi Ruoyao <xry111@xry111.site>. All Rights Reserved.
4
+
*/
5
+
#include <linux/types.h>
6
+
7
+
ssize_t __vdso_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len)
8
+
{
9
+
return __cvdso_getrandom(buffer, len, flags, opaque_state, opaque_len);
10
+
}
+1
arch/powerpc/Kconfig
+1
arch/powerpc/Kconfig
+1
-1
arch/powerpc/include/asm/mman.h
+1
-1
arch/powerpc/include/asm/mman.h
+54
arch/powerpc/include/asm/vdso/getrandom.h
+54
arch/powerpc/include/asm/vdso/getrandom.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/*
3
+
* Copyright (C) 2024 Christophe Leroy <christophe.leroy@csgroup.eu>, CS GROUP France
4
+
*/
5
+
#ifndef _ASM_POWERPC_VDSO_GETRANDOM_H
6
+
#define _ASM_POWERPC_VDSO_GETRANDOM_H
7
+
8
+
#ifndef __ASSEMBLY__
9
+
10
+
static __always_inline int do_syscall_3(const unsigned long _r0, const unsigned long _r3,
11
+
const unsigned long _r4, const unsigned long _r5)
12
+
{
13
+
register long r0 asm("r0") = _r0;
14
+
register unsigned long r3 asm("r3") = _r3;
15
+
register unsigned long r4 asm("r4") = _r4;
16
+
register unsigned long r5 asm("r5") = _r5;
17
+
register int ret asm ("r3");
18
+
19
+
asm volatile(
20
+
" sc\n"
21
+
" bns+ 1f\n"
22
+
" neg %0, %0\n"
23
+
"1:\n"
24
+
: "=r" (ret), "+r" (r4), "+r" (r5), "+r" (r0)
25
+
: "r" (r3)
26
+
: "memory", "r6", "r7", "r8", "r9", "r10", "r11", "r12", "cr0", "ctr");
27
+
28
+
return ret;
29
+
}
30
+
31
+
/**
32
+
* getrandom_syscall - Invoke the getrandom() syscall.
33
+
* @buffer: Destination buffer to fill with random bytes.
34
+
* @len: Size of @buffer in bytes.
35
+
* @flags: Zero or more GRND_* flags.
36
+
* Returns: The number of bytes written to @buffer, or a negative value indicating an error.
37
+
*/
38
+
static __always_inline ssize_t getrandom_syscall(void *buffer, size_t len, unsigned int flags)
39
+
{
40
+
return do_syscall_3(__NR_getrandom, (unsigned long)buffer,
41
+
(unsigned long)len, (unsigned long)flags);
42
+
}
43
+
44
+
static __always_inline struct vdso_rng_data *__arch_get_vdso_rng_data(void)
45
+
{
46
+
return NULL;
47
+
}
48
+
49
+
ssize_t __c_kernel_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state,
50
+
size_t opaque_len, const struct vdso_rng_data *vd);
51
+
52
+
#endif /* !__ASSEMBLY__ */
53
+
54
+
#endif /* _ASM_POWERPC_VDSO_GETRANDOM_H */
+6
arch/powerpc/include/asm/vdso/vsyscall.h
+6
arch/powerpc/include/asm/vdso/vsyscall.h
···
17
17
}
18
18
#define __arch_get_k_vdso_data __arch_get_k_vdso_data
19
19
20
+
static __always_inline
21
+
struct vdso_rng_data *__arch_get_k_vdso_rng_data(void)
22
+
{
23
+
return &vdso_data->rng_data;
24
+
}
25
+
20
26
/* The asm-generic header needs to be included after the definitions above */
21
27
#include <asm-generic/vdso/vsyscall.h>
22
28
+17
arch/powerpc/include/asm/vdso_datapage.h
+17
arch/powerpc/include/asm/vdso_datapage.h
···
83
83
__u32 compat_syscall_map[SYSCALL_MAP_SIZE]; /* Map of compat syscalls */
84
84
85
85
struct vdso_data data[CS_BASES];
86
+
struct vdso_rng_data rng_data;
86
87
};
87
88
88
89
#else /* CONFIG_PPC64 */
···
96
95
__u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */
97
96
__u32 compat_syscall_map[0]; /* No compat syscalls on PPC32 */
98
97
struct vdso_data data[CS_BASES];
98
+
struct vdso_rng_data rng_data;
99
99
};
100
100
101
101
#endif /* CONFIG_PPC64 */
···
111
109
mflr \ptr
112
110
addis \ptr, \ptr, (_vdso_datapage - 999b)@ha
113
111
addi \ptr, \ptr, (_vdso_datapage - 999b)@l
112
+
.endm
113
+
114
+
#include <asm/asm-offsets.h>
115
+
#include <asm/page.h>
116
+
117
+
.macro get_realdatapage ptr scratch
118
+
get_datapage \ptr
119
+
#ifdef CONFIG_TIME_NS
120
+
lwz \scratch, VDSO_CLOCKMODE_OFFSET(\ptr)
121
+
xoris \scratch, \scratch, VDSO_CLOCKMODE_TIMENS@h
122
+
xori \scratch, \scratch, VDSO_CLOCKMODE_TIMENS@l
123
+
cntlzw \scratch, \scratch
124
+
rlwinm \scratch, \scratch, PAGE_SHIFT - 5, 1 << PAGE_SHIFT
125
+
add \ptr, \ptr, \scratch
126
+
#endif
114
127
.endm
115
128
116
129
#endif /* __ASSEMBLY__ */
+3
arch/powerpc/kernel/asm-offsets.c
+3
arch/powerpc/kernel/asm-offsets.c
···
335
335
336
336
/* datapage offsets for use by vdso */
337
337
OFFSET(VDSO_DATA_OFFSET, vdso_arch_data, data);
338
+
OFFSET(VDSO_RNG_DATA_OFFSET, vdso_arch_data, rng_data);
338
339
OFFSET(CFG_TB_TICKS_PER_SEC, vdso_arch_data, tb_ticks_per_sec);
339
340
#ifdef CONFIG_PPC64
340
341
OFFSET(CFG_ICACHE_BLOCKSZ, vdso_arch_data, icache_block_size);
···
347
346
#else
348
347
OFFSET(CFG_SYSCALL_MAP32, vdso_arch_data, syscall_map);
349
348
#endif
349
+
OFFSET(VDSO_CLOCKMODE_OFFSET, vdso_arch_data, data[0].clock_mode);
350
+
DEFINE(VDSO_CLOCKMODE_TIMENS, VDSO_CLOCKMODE_TIMENS);
350
351
351
352
#ifdef CONFIG_BUG
352
353
DEFINE(BUG_ENTRY_SIZE, sizeof(struct bug_entry));
+33
-24
arch/powerpc/kernel/vdso/Makefile
+33
-24
arch/powerpc/kernel/vdso/Makefile
···
8
8
obj-vdso32 = sigtramp32-32.o gettimeofday-32.o datapage-32.o cacheflush-32.o note-32.o getcpu-32.o
9
9
obj-vdso64 = sigtramp64-64.o gettimeofday-64.o datapage-64.o cacheflush-64.o note-64.o getcpu-64.o
10
10
11
+
obj-vdso32 += getrandom-32.o vgetrandom-chacha-32.o
12
+
obj-vdso64 += getrandom-64.o vgetrandom-chacha-64.o
13
+
11
14
ifneq ($(c-gettimeofday-y),)
12
15
CFLAGS_vgettimeofday-32.o += -include $(c-gettimeofday-y)
13
-
CFLAGS_vgettimeofday-32.o += $(DISABLE_LATENT_ENTROPY_PLUGIN)
14
-
CFLAGS_vgettimeofday-32.o += $(call cc-option, -fno-stack-protector)
15
-
CFLAGS_vgettimeofday-32.o += -DDISABLE_BRANCH_PROFILING
16
-
CFLAGS_vgettimeofday-32.o += -ffreestanding -fasynchronous-unwind-tables
17
-
CFLAGS_REMOVE_vgettimeofday-32.o = $(CC_FLAGS_FTRACE)
18
-
CFLAGS_REMOVE_vgettimeofday-32.o += -mcmodel=medium -mabi=elfv1 -mabi=elfv2 -mcall-aixdesc
19
-
# This flag is supported by clang for 64-bit but not 32-bit so it will cause
20
-
# an unused command line flag warning for this file.
21
-
ifdef CONFIG_CC_IS_CLANG
22
-
CFLAGS_REMOVE_vgettimeofday-32.o += -fno-stack-clash-protection
23
-
endif
24
-
CFLAGS_vgettimeofday-64.o += -include $(c-gettimeofday-y)
25
-
CFLAGS_vgettimeofday-64.o += $(DISABLE_LATENT_ENTROPY_PLUGIN)
26
-
CFLAGS_vgettimeofday-64.o += $(call cc-option, -fno-stack-protector)
27
-
CFLAGS_vgettimeofday-64.o += -DDISABLE_BRANCH_PROFILING
28
-
CFLAGS_vgettimeofday-64.o += -ffreestanding -fasynchronous-unwind-tables
29
-
CFLAGS_REMOVE_vgettimeofday-64.o = $(CC_FLAGS_FTRACE)
30
16
# Go prior to 1.16.x assumes r30 is not clobbered by any VDSO code. That used to be true
31
17
# by accident when the VDSO was hand-written asm code, but may not be now that the VDSO is
32
18
# compiler generated. To avoid breaking Go tell GCC not to use r30. Impact on code
33
19
# generation is minimal, it will just use r29 instead.
34
-
CFLAGS_vgettimeofday-64.o += $(call cc-option, -ffixed-r30)
20
+
CFLAGS_vgettimeofday-64.o += -include $(c-gettimeofday-y) $(call cc-option, -ffixed-r30)
21
+
endif
22
+
23
+
ifneq ($(c-getrandom-y),)
24
+
CFLAGS_vgetrandom-32.o += -include $(c-getrandom-y)
25
+
CFLAGS_vgetrandom-64.o += -include $(c-getrandom-y) $(call cc-option, -ffixed-r30)
35
26
endif
36
27
37
28
# Build rules
···
33
42
VDSOCC := $(CC)
34
43
endif
35
44
36
-
targets := $(obj-vdso32) vdso32.so.dbg vgettimeofday-32.o
45
+
targets := $(obj-vdso32) vdso32.so.dbg vgettimeofday-32.o vgetrandom-32.o
46
+
targets += crtsavres-32.o
37
47
obj-vdso32 := $(addprefix $(obj)/, $(obj-vdso32))
38
-
targets += $(obj-vdso64) vdso64.so.dbg vgettimeofday-64.o
48
+
targets += $(obj-vdso64) vdso64.so.dbg vgettimeofday-64.o vgetrandom-64.o
39
49
obj-vdso64 := $(addprefix $(obj)/, $(obj-vdso64))
40
50
41
-
ccflags-y := -fno-common -fno-builtin
51
+
ccflags-y := -fno-common -fno-builtin -DBUILD_VDSO
52
+
ccflags-y += $(DISABLE_LATENT_ENTROPY_PLUGIN)
53
+
ccflags-y += $(call cc-option, -fno-stack-protector)
54
+
ccflags-y += -DDISABLE_BRANCH_PROFILING
55
+
ccflags-y += -ffreestanding -fasynchronous-unwind-tables
56
+
ccflags-remove-y := $(CC_FLAGS_FTRACE)
42
57
ldflags-y := -Wl,--hash-style=both -nostdlib -shared -z noexecstack $(CLANG_FLAGS)
43
58
ldflags-$(CONFIG_LD_IS_LLD) += $(call cc-option,--ld-path=$(LD),-fuse-ld=lld)
44
59
ldflags-$(CONFIG_LD_ORPHAN_WARN) += -Wl,--orphan-handling=$(CONFIG_LD_ORPHAN_WARN_LEVEL)
···
53
56
ldflags-y += $(filter-out $(CC_AUTO_VAR_INIT_ZERO_ENABLER) $(CC_FLAGS_FTRACE) -Wa$(comma)%, $(KBUILD_CFLAGS))
54
57
55
58
CC32FLAGS := -m32
59
+
CC32FLAGSREMOVE := -mcmodel=medium -mabi=elfv1 -mabi=elfv2 -mcall-aixdesc
60
+
# This flag is supported by clang for 64-bit but not 32-bit so it will cause
61
+
# an unused command line flag warning for this file.
62
+
ifdef CONFIG_CC_IS_CLANG
63
+
CC32FLAGSREMOVE += -fno-stack-clash-protection
64
+
endif
56
65
LD32FLAGS := -Wl,-soname=linux-vdso32.so.1
57
66
AS32FLAGS := -D__VDSO32__
58
67
···
71
68
CPPFLAGS_vdso64.lds += -P -C
72
69
73
70
# link rule for the .so file, .lds has to be first
74
-
$(obj)/vdso32.so.dbg: $(obj)/vdso32.lds $(obj-vdso32) $(obj)/vgettimeofday-32.o FORCE
71
+
$(obj)/vdso32.so.dbg: $(obj)/vdso32.lds $(obj-vdso32) $(obj)/vgettimeofday-32.o $(obj)/vgetrandom-32.o $(obj)/crtsavres-32.o FORCE
75
72
$(call if_changed,vdso32ld_and_check)
76
-
$(obj)/vdso64.so.dbg: $(obj)/vdso64.lds $(obj-vdso64) $(obj)/vgettimeofday-64.o FORCE
73
+
$(obj)/vdso64.so.dbg: $(obj)/vdso64.lds $(obj-vdso64) $(obj)/vgettimeofday-64.o $(obj)/vgetrandom-64.o FORCE
77
74
$(call if_changed,vdso64ld_and_check)
78
75
79
76
# assembly rules for the .S files
80
77
$(obj-vdso32): %-32.o: %.S FORCE
81
78
$(call if_changed_dep,vdso32as)
79
+
$(obj)/crtsavres-32.o: %-32.o: $(srctree)/arch/powerpc/lib/crtsavres.S FORCE
80
+
$(call if_changed_dep,vdso32as)
82
81
$(obj)/vgettimeofday-32.o: %-32.o: %.c FORCE
82
+
$(call if_changed_dep,vdso32cc)
83
+
$(obj)/vgetrandom-32.o: %-32.o: %.c FORCE
83
84
$(call if_changed_dep,vdso32cc)
84
85
$(obj-vdso64): %-64.o: %.S FORCE
85
86
$(call if_changed_dep,vdso64as)
86
87
$(obj)/vgettimeofday-64.o: %-64.o: %.c FORCE
88
+
$(call if_changed_dep,cc_o_c)
89
+
$(obj)/vgetrandom-64.o: %-64.o: %.c FORCE
87
90
$(call if_changed_dep,cc_o_c)
88
91
89
92
# Generate VDSO offsets using helper script
···
111
102
quiet_cmd_vdso32as = VDSO32A $@
112
103
cmd_vdso32as = $(VDSOCC) $(a_flags) $(CC32FLAGS) $(AS32FLAGS) -c -o $@ $<
113
104
quiet_cmd_vdso32cc = VDSO32C $@
114
-
cmd_vdso32cc = $(VDSOCC) $(c_flags) $(CC32FLAGS) -c -o $@ $<
105
+
cmd_vdso32cc = $(VDSOCC) $(filter-out $(CC32FLAGSREMOVE), $(c_flags)) $(CC32FLAGS) -c -o $@ $<
115
106
116
107
quiet_cmd_vdso64ld_and_check = VDSO64L $@
117
108
cmd_vdso64ld_and_check = $(VDSOCC) $(ldflags-y) $(LD64FLAGS) -o $@ -Wl,-T$(filter %.lds,$^) $(filter %.o,$^); $(cmd_vdso_check)
+1
-1
arch/powerpc/kernel/vdso/cacheflush.S
+1
-1
arch/powerpc/kernel/vdso/cacheflush.S
+2
-2
arch/powerpc/kernel/vdso/datapage.S
+2
-2
arch/powerpc/kernel/vdso/datapage.S
···
28
28
mflr r12
29
29
.cfi_register lr,r12
30
30
mr. r4,r3
31
-
get_datapage r3
31
+
get_realdatapage r3, r11
32
32
mtlr r12
33
33
#ifdef __powerpc64__
34
34
addi r3,r3,CFG_SYSCALL_MAP64
···
52
52
.cfi_startproc
53
53
mflr r12
54
54
.cfi_register lr,r12
55
-
get_datapage r3
55
+
get_realdatapage r3, r11
56
56
#ifndef __powerpc64__
57
57
lwz r4,(CFG_TB_TICKS_PER_SEC + 4)(r3)
58
58
#endif
+58
arch/powerpc/kernel/vdso/getrandom.S
+58
arch/powerpc/kernel/vdso/getrandom.S
···
1
+
/* SPDX-License-Identifier: GPL-2.0-or-later */
2
+
/*
3
+
* Userland implementation of getrandom() for processes
4
+
* for use in the vDSO
5
+
*
6
+
* Copyright (C) 2024 Christophe Leroy <christophe.leroy@csgroup.eu>, CS GROUP France
7
+
*/
8
+
#include <asm/processor.h>
9
+
#include <asm/ppc_asm.h>
10
+
#include <asm/vdso.h>
11
+
#include <asm/vdso_datapage.h>
12
+
#include <asm/asm-offsets.h>
13
+
#include <asm/unistd.h>
14
+
15
+
/*
16
+
* The macro sets two stack frames, one for the caller and one for the callee
17
+
* because there are no requirement for the caller to set a stack frame when
18
+
* calling VDSO so it may have omitted to set one, especially on PPC64
19
+
*/
20
+
21
+
.macro cvdso_call funct
22
+
.cfi_startproc
23
+
PPC_STLU r1, -PPC_MIN_STKFRM(r1)
24
+
.cfi_adjust_cfa_offset PPC_MIN_STKFRM
25
+
mflr r0
26
+
PPC_STLU r1, -PPC_MIN_STKFRM(r1)
27
+
.cfi_adjust_cfa_offset PPC_MIN_STKFRM
28
+
PPC_STL r0, PPC_MIN_STKFRM + PPC_LR_STKOFF(r1)
29
+
.cfi_rel_offset lr, PPC_MIN_STKFRM + PPC_LR_STKOFF
30
+
#ifdef __powerpc64__
31
+
PPC_STL r2, PPC_MIN_STKFRM + STK_GOT(r1)
32
+
.cfi_rel_offset r2, PPC_MIN_STKFRM + STK_GOT
33
+
#endif
34
+
get_realdatapage r8, r11
35
+
addi r8, r8, VDSO_RNG_DATA_OFFSET
36
+
bl CFUNC(DOTSYM(\funct))
37
+
PPC_LL r0, PPC_MIN_STKFRM + PPC_LR_STKOFF(r1)
38
+
#ifdef __powerpc64__
39
+
PPC_LL r2, PPC_MIN_STKFRM + STK_GOT(r1)
40
+
.cfi_restore r2
41
+
#endif
42
+
cmpwi r3, 0
43
+
mtlr r0
44
+
addi r1, r1, 2 * PPC_MIN_STKFRM
45
+
.cfi_restore lr
46
+
.cfi_def_cfa_offset 0
47
+
crclr so
48
+
bgelr+
49
+
crset so
50
+
neg r3, r3
51
+
blr
52
+
.cfi_endproc
53
+
.endm
54
+
55
+
.text
56
+
V_FUNCTION_BEGIN(__kernel_getrandom)
57
+
cvdso_call __c_kernel_getrandom
58
+
V_FUNCTION_END(__kernel_getrandom)
-13
arch/powerpc/kernel/vdso/gettimeofday.S
-13
arch/powerpc/kernel/vdso/gettimeofday.S
···
118
118
V_FUNCTION_BEGIN(__kernel_time)
119
119
cvdso_call __c_kernel_time call_time=1
120
120
V_FUNCTION_END(__kernel_time)
121
-
122
-
/* Routines for restoring integer registers, called by the compiler. */
123
-
/* Called with r11 pointing to the stack header word of the caller of the */
124
-
/* function, just beyond the end of the integer restore area. */
125
-
#ifndef __powerpc64__
126
-
_GLOBAL(_restgpr_31_x)
127
-
_GLOBAL(_rest32gpr_31_x)
128
-
lwz r0,4(r11)
129
-
lwz r31,-4(r11)
130
-
mtlr r0
131
-
mr r1,r11
132
-
blr
133
-
#endif
+1
arch/powerpc/kernel/vdso/vdso32.lds.S
+1
arch/powerpc/kernel/vdso/vdso32.lds.S
+1
arch/powerpc/kernel/vdso/vdso64.lds.S
+1
arch/powerpc/kernel/vdso/vdso64.lds.S
+365
arch/powerpc/kernel/vdso/vgetrandom-chacha.S
+365
arch/powerpc/kernel/vdso/vgetrandom-chacha.S
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/*
3
+
* Copyright (C) 2024 Christophe Leroy <christophe.leroy@csgroup.eu>, CS GROUP France
4
+
*/
5
+
6
+
#include <linux/linkage.h>
7
+
8
+
#include <asm/ppc_asm.h>
9
+
10
+
#define dst_bytes r3
11
+
#define key r4
12
+
#define counter r5
13
+
#define nblocks r6
14
+
15
+
#define idx_r0 r0
16
+
#define val4 r4
17
+
18
+
#define const0 0x61707865
19
+
#define const1 0x3320646e
20
+
#define const2 0x79622d32
21
+
#define const3 0x6b206574
22
+
23
+
#define key0 r5
24
+
#define key1 r6
25
+
#define key2 r7
26
+
#define key3 r8
27
+
#define key4 r9
28
+
#define key5 r10
29
+
#define key6 r11
30
+
#define key7 r12
31
+
32
+
#define counter0 r14
33
+
#define counter1 r15
34
+
35
+
#define state0 r16
36
+
#define state1 r17
37
+
#define state2 r18
38
+
#define state3 r19
39
+
#define state4 r20
40
+
#define state5 r21
41
+
#define state6 r22
42
+
#define state7 r23
43
+
#define state8 r24
44
+
#define state9 r25
45
+
#define state10 r26
46
+
#define state11 r27
47
+
#define state12 r28
48
+
#define state13 r29
49
+
#define state14 r30
50
+
#define state15 r31
51
+
52
+
.macro quarterround4 a1 b1 c1 d1 a2 b2 c2 d2 a3 b3 c3 d3 a4 b4 c4 d4
53
+
add \a1, \a1, \b1
54
+
add \a2, \a2, \b2
55
+
add \a3, \a3, \b3
56
+
add \a4, \a4, \b4
57
+
xor \d1, \d1, \a1
58
+
xor \d2, \d2, \a2
59
+
xor \d3, \d3, \a3
60
+
xor \d4, \d4, \a4
61
+
rotlwi \d1, \d1, 16
62
+
rotlwi \d2, \d2, 16
63
+
rotlwi \d3, \d3, 16
64
+
rotlwi \d4, \d4, 16
65
+
add \c1, \c1, \d1
66
+
add \c2, \c2, \d2
67
+
add \c3, \c3, \d3
68
+
add \c4, \c4, \d4
69
+
xor \b1, \b1, \c1
70
+
xor \b2, \b2, \c2
71
+
xor \b3, \b3, \c3
72
+
xor \b4, \b4, \c4
73
+
rotlwi \b1, \b1, 12
74
+
rotlwi \b2, \b2, 12
75
+
rotlwi \b3, \b3, 12
76
+
rotlwi \b4, \b4, 12
77
+
add \a1, \a1, \b1
78
+
add \a2, \a2, \b2
79
+
add \a3, \a3, \b3
80
+
add \a4, \a4, \b4
81
+
xor \d1, \d1, \a1
82
+
xor \d2, \d2, \a2
83
+
xor \d3, \d3, \a3
84
+
xor \d4, \d4, \a4
85
+
rotlwi \d1, \d1, 8
86
+
rotlwi \d2, \d2, 8
87
+
rotlwi \d3, \d3, 8
88
+
rotlwi \d4, \d4, 8
89
+
add \c1, \c1, \d1
90
+
add \c2, \c2, \d2
91
+
add \c3, \c3, \d3
92
+
add \c4, \c4, \d4
93
+
xor \b1, \b1, \c1
94
+
xor \b2, \b2, \c2
95
+
xor \b3, \b3, \c3
96
+
xor \b4, \b4, \c4
97
+
rotlwi \b1, \b1, 7
98
+
rotlwi \b2, \b2, 7
99
+
rotlwi \b3, \b3, 7
100
+
rotlwi \b4, \b4, 7
101
+
.endm
102
+
103
+
#define QUARTERROUND4(a1,b1,c1,d1,a2,b2,c2,d2,a3,b3,c3,d3,a4,b4,c4,d4) \
104
+
quarterround4 state##a1 state##b1 state##c1 state##d1 \
105
+
state##a2 state##b2 state##c2 state##d2 \
106
+
state##a3 state##b3 state##c3 state##d3 \
107
+
state##a4 state##b4 state##c4 state##d4
108
+
109
+
/*
110
+
* Very basic 32 bits implementation of ChaCha20. Produces a given positive number
111
+
* of blocks of output with a nonce of 0, taking an input key and 8-byte
112
+
* counter. Importantly does not spill to the stack. Its arguments are:
113
+
*
114
+
* r3: output bytes
115
+
* r4: 32-byte key input
116
+
* r5: 8-byte counter input/output (saved on stack)
117
+
* r6: number of 64-byte blocks to write to output
118
+
*
119
+
* r0: counter of blocks (initialised with r6)
120
+
* r4: Value '4' after key has been read.
121
+
* r5-r12: key
122
+
* r14-r15: counter
123
+
* r16-r31: state
124
+
*/
125
+
SYM_FUNC_START(__arch_chacha20_blocks_nostack)
126
+
#ifdef __powerpc64__
127
+
std counter, -216(r1)
128
+
129
+
std r14, -144(r1)
130
+
std r15, -136(r1)
131
+
std r16, -128(r1)
132
+
std r17, -120(r1)
133
+
std r18, -112(r1)
134
+
std r19, -104(r1)
135
+
std r20, -96(r1)
136
+
std r21, -88(r1)
137
+
std r22, -80(r1)
138
+
std r23, -72(r1)
139
+
std r24, -64(r1)
140
+
std r25, -56(r1)
141
+
std r26, -48(r1)
142
+
std r27, -40(r1)
143
+
std r28, -32(r1)
144
+
std r29, -24(r1)
145
+
std r30, -16(r1)
146
+
std r31, -8(r1)
147
+
#else
148
+
stwu r1, -96(r1)
149
+
stw counter, 20(r1)
150
+
#ifdef __BIG_ENDIAN__
151
+
stmw r14, 24(r1)
152
+
#else
153
+
stw r14, 24(r1)
154
+
stw r15, 28(r1)
155
+
stw r16, 32(r1)
156
+
stw r17, 36(r1)
157
+
stw r18, 40(r1)
158
+
stw r19, 44(r1)
159
+
stw r20, 48(r1)
160
+
stw r21, 52(r1)
161
+
stw r22, 56(r1)
162
+
stw r23, 60(r1)
163
+
stw r24, 64(r1)
164
+
stw r25, 68(r1)
165
+
stw r26, 72(r1)
166
+
stw r27, 76(r1)
167
+
stw r28, 80(r1)
168
+
stw r29, 84(r1)
169
+
stw r30, 88(r1)
170
+
stw r31, 92(r1)
171
+
#endif
172
+
#endif /* __powerpc64__ */
173
+
174
+
lwz counter0, 0(counter)
175
+
lwz counter1, 4(counter)
176
+
#ifdef __powerpc64__
177
+
rldimi counter0, counter1, 32, 0
178
+
#endif
179
+
mr idx_r0, nblocks
180
+
subi dst_bytes, dst_bytes, 4
181
+
182
+
lwz key0, 0(key)
183
+
lwz key1, 4(key)
184
+
lwz key2, 8(key)
185
+
lwz key3, 12(key)
186
+
lwz key4, 16(key)
187
+
lwz key5, 20(key)
188
+
lwz key6, 24(key)
189
+
lwz key7, 28(key)
190
+
191
+
li val4, 4
192
+
.Lblock:
193
+
li r31, 10
194
+
195
+
lis state0, const0@ha
196
+
lis state1, const1@ha
197
+
lis state2, const2@ha
198
+
lis state3, const3@ha
199
+
addi state0, state0, const0@l
200
+
addi state1, state1, const1@l
201
+
addi state2, state2, const2@l
202
+
addi state3, state3, const3@l
203
+
204
+
mtctr r31
205
+
206
+
mr state4, key0
207
+
mr state5, key1
208
+
mr state6, key2
209
+
mr state7, key3
210
+
mr state8, key4
211
+
mr state9, key5
212
+
mr state10, key6
213
+
mr state11, key7
214
+
215
+
mr state12, counter0
216
+
mr state13, counter1
217
+
218
+
li state14, 0
219
+
li state15, 0
220
+
221
+
.Lpermute:
222
+
QUARTERROUND4( 0, 4, 8,12, 1, 5, 9,13, 2, 6,10,14, 3, 7,11,15)
223
+
QUARTERROUND4( 0, 5,10,15, 1, 6,11,12, 2, 7, 8,13, 3, 4, 9,14)
224
+
225
+
bdnz .Lpermute
226
+
227
+
addis state0, state0, const0@ha
228
+
addis state1, state1, const1@ha
229
+
addis state2, state2, const2@ha
230
+
addis state3, state3, const3@ha
231
+
addi state0, state0, const0@l
232
+
addi state1, state1, const1@l
233
+
addi state2, state2, const2@l
234
+
addi state3, state3, const3@l
235
+
236
+
add state4, state4, key0
237
+
add state5, state5, key1
238
+
add state6, state6, key2
239
+
add state7, state7, key3
240
+
add state8, state8, key4
241
+
add state9, state9, key5
242
+
add state10, state10, key6
243
+
add state11, state11, key7
244
+
245
+
add state12, state12, counter0
246
+
add state13, state13, counter1
247
+
248
+
#ifdef __BIG_ENDIAN__
249
+
stwbrx state0, val4, dst_bytes
250
+
addi dst_bytes, dst_bytes, 8
251
+
stwbrx state1, 0, dst_bytes
252
+
stwbrx state2, val4, dst_bytes
253
+
addi dst_bytes, dst_bytes, 8
254
+
stwbrx state3, 0, dst_bytes
255
+
stwbrx state4, val4, dst_bytes
256
+
addi dst_bytes, dst_bytes, 8
257
+
stwbrx state5, 0, dst_bytes
258
+
stwbrx state6, val4, dst_bytes
259
+
addi dst_bytes, dst_bytes, 8
260
+
stwbrx state7, 0, dst_bytes
261
+
stwbrx state8, val4, dst_bytes
262
+
addi dst_bytes, dst_bytes, 8
263
+
stwbrx state9, 0, dst_bytes
264
+
stwbrx state10, val4, dst_bytes
265
+
addi dst_bytes, dst_bytes, 8
266
+
stwbrx state11, 0, dst_bytes
267
+
stwbrx state12, val4, dst_bytes
268
+
addi dst_bytes, dst_bytes, 8
269
+
stwbrx state13, 0, dst_bytes
270
+
stwbrx state14, val4, dst_bytes
271
+
addi dst_bytes, dst_bytes, 8
272
+
stwbrx state15, 0, dst_bytes
273
+
#else
274
+
stw state0, 4(dst_bytes)
275
+
stw state1, 8(dst_bytes)
276
+
stw state2, 12(dst_bytes)
277
+
stw state3, 16(dst_bytes)
278
+
stw state4, 20(dst_bytes)
279
+
stw state5, 24(dst_bytes)
280
+
stw state6, 28(dst_bytes)
281
+
stw state7, 32(dst_bytes)
282
+
stw state8, 36(dst_bytes)
283
+
stw state9, 40(dst_bytes)
284
+
stw state10, 44(dst_bytes)
285
+
stw state11, 48(dst_bytes)
286
+
stw state12, 52(dst_bytes)
287
+
stw state13, 56(dst_bytes)
288
+
stw state14, 60(dst_bytes)
289
+
stwu state15, 64(dst_bytes)
290
+
#endif
291
+
292
+
subic. idx_r0, idx_r0, 1 /* subi. can't use r0 as source */
293
+
294
+
#ifdef __powerpc64__
295
+
addi counter0, counter0, 1
296
+
srdi counter1, counter0, 32
297
+
#else
298
+
addic counter0, counter0, 1
299
+
addze counter1, counter1
300
+
#endif
301
+
302
+
bne .Lblock
303
+
304
+
#ifdef __powerpc64__
305
+
ld counter, -216(r1)
306
+
#else
307
+
lwz counter, 20(r1)
308
+
#endif
309
+
stw counter0, 0(counter)
310
+
stw counter1, 4(counter)
311
+
312
+
li r6, 0
313
+
li r7, 0
314
+
li r8, 0
315
+
li r9, 0
316
+
li r10, 0
317
+
li r11, 0
318
+
li r12, 0
319
+
320
+
#ifdef __powerpc64__
321
+
ld r14, -144(r1)
322
+
ld r15, -136(r1)
323
+
ld r16, -128(r1)
324
+
ld r17, -120(r1)
325
+
ld r18, -112(r1)
326
+
ld r19, -104(r1)
327
+
ld r20, -96(r1)
328
+
ld r21, -88(r1)
329
+
ld r22, -80(r1)
330
+
ld r23, -72(r1)
331
+
ld r24, -64(r1)
332
+
ld r25, -56(r1)
333
+
ld r26, -48(r1)
334
+
ld r27, -40(r1)
335
+
ld r28, -32(r1)
336
+
ld r29, -24(r1)
337
+
ld r30, -16(r1)
338
+
ld r31, -8(r1)
339
+
#else
340
+
#ifdef __BIG_ENDIAN__
341
+
lmw r14, 24(r1)
342
+
#else
343
+
lwz r14, 24(r1)
344
+
lwz r15, 28(r1)
345
+
lwz r16, 32(r1)
346
+
lwz r17, 36(r1)
347
+
lwz r18, 40(r1)
348
+
lwz r19, 44(r1)
349
+
lwz r20, 48(r1)
350
+
lwz r21, 52(r1)
351
+
lwz r22, 56(r1)
352
+
lwz r23, 60(r1)
353
+
lwz r24, 64(r1)
354
+
lwz r25, 68(r1)
355
+
lwz r26, 72(r1)
356
+
lwz r27, 76(r1)
357
+
lwz r28, 80(r1)
358
+
lwz r29, 84(r1)
359
+
lwz r30, 88(r1)
360
+
lwz r31, 92(r1)
361
+
#endif
362
+
addi r1, r1, 96
363
+
#endif /* __powerpc64__ */
364
+
blr
365
+
SYM_FUNC_END(__arch_chacha20_blocks_nostack)
+14
arch/powerpc/kernel/vdso/vgetrandom.c
+14
arch/powerpc/kernel/vdso/vgetrandom.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/*
3
+
* Powerpc userspace implementation of getrandom()
4
+
*
5
+
* Copyright (C) 2024 Christophe Leroy <christophe.leroy@csgroup.eu>, CS GROUP France
6
+
*/
7
+
#include <linux/time.h>
8
+
#include <linux/types.h>
9
+
10
+
ssize_t __c_kernel_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state,
11
+
size_t opaque_len, const struct vdso_rng_data *vd)
12
+
{
13
+
return __cvdso_getrandom_data(vd, buffer, len, flags, opaque_state, opaque_len);
14
+
}
+1
arch/s390/Kconfig
+1
arch/s390/Kconfig
+1
-5
arch/s390/include/asm/alternative.h
+1
-5
arch/s390/include/asm/alternative.h
···
39
39
#define ALT_TYPE_SHIFT 20
40
40
#define ALT_CTX_SHIFT 28
41
41
42
-
#define ALT_FACILITY_EARLY(facility) (ALT_CTX_EARLY << ALT_CTX_SHIFT | \
43
-
ALT_TYPE_FACILITY << ALT_TYPE_SHIFT | \
44
-
(facility) << ALT_DATA_SHIFT)
45
-
46
-
#define ALT_FACILITY(facility) (ALT_CTX_LATE << ALT_CTX_SHIFT | \
42
+
#define ALT_FACILITY(facility) (ALT_CTX_EARLY << ALT_CTX_SHIFT | \
47
43
ALT_TYPE_FACILITY << ALT_TYPE_SHIFT | \
48
44
(facility) << ALT_DATA_SHIFT)
49
45
+30
-7
arch/s390/include/asm/facility.h
+30
-7
arch/s390/include/asm/facility.h
···
14
14
#include <linux/string.h>
15
15
#include <linux/types.h>
16
16
#include <linux/preempt.h>
17
-
17
+
#include <asm/alternative.h>
18
18
#include <asm/lowcore.h>
19
19
20
20
#define MAX_FACILITY_BIT (sizeof(stfle_fac_list) * 8)
···
39
39
ptr[nr >> 3] &= ~(0x80 >> (nr & 7));
40
40
}
41
41
42
-
static inline int __test_facility(unsigned long nr, void *facilities)
42
+
static __always_inline bool __test_facility(unsigned long nr, void *facilities)
43
43
{
44
44
unsigned char *ptr;
45
45
46
46
if (nr >= MAX_FACILITY_BIT)
47
-
return 0;
47
+
return false;
48
48
ptr = (unsigned char *) facilities + (nr >> 3);
49
49
return (*ptr & (0x80 >> (nr & 7))) != 0;
50
+
}
51
+
52
+
/*
53
+
* __test_facility_constant() generates a single instruction branch. If the
54
+
* tested facility is available (likely) the branch is patched into a nop.
55
+
*
56
+
* Do not use this function unless you know what you are doing. All users are
57
+
* supposed to use test_facility() which will do the right thing.
58
+
*/
59
+
static __always_inline bool __test_facility_constant(unsigned long nr)
60
+
{
61
+
asm goto(
62
+
ALTERNATIVE("brcl 15,%l[l_no]", "brcl 0,0", ALT_FACILITY(%[nr]))
63
+
:
64
+
: [nr] "i" (nr)
65
+
:
66
+
: l_no);
67
+
return true;
68
+
l_no:
69
+
return false;
50
70
}
51
71
52
72
/*
···
74
54
* That makes it easier to query facility bits with the bit number as
75
55
* documented in the Principles of Operation.
76
56
*/
77
-
static inline int test_facility(unsigned long nr)
57
+
static __always_inline bool test_facility(unsigned long nr)
78
58
{
79
59
unsigned long facilities_als[] = { FACILITIES_ALS };
80
60
81
-
if (__builtin_constant_p(nr) && nr < sizeof(facilities_als) * 8) {
82
-
if (__test_facility(nr, &facilities_als))
83
-
return 1;
61
+
if (!__is_defined(__DECOMPRESSOR) && __builtin_constant_p(nr)) {
62
+
if (nr < sizeof(facilities_als) * 8) {
63
+
if (__test_facility(nr, &facilities_als))
64
+
return true;
65
+
}
66
+
return __test_facility_constant(nr);
84
67
}
85
68
return __test_facility(nr, &stfle_fac_list);
86
69
}
+22
arch/s390/include/asm/fpu-insn-asm.h
+22
arch/s390/include/asm/fpu-insn-asm.h
···
407
407
MRXBOPC 0, 0x0E, v1
408
408
.endm
409
409
410
+
/* VECTOR STORE BYTE REVERSED ELEMENTS */
411
+
.macro VSTBR vr1, disp, index="%r0", base, m
412
+
VX_NUM v1, \vr1
413
+
GR_NUM x2, \index
414
+
GR_NUM b2, \base
415
+
.word 0xE600 | ((v1&15) << 4) | (x2&15)
416
+
.word (b2 << 12) | (\disp)
417
+
MRXBOPC \m, 0x0E, v1
418
+
.endm
419
+
.macro VSTBRH vr1, disp, index="%r0", base
420
+
VSTBR \vr1, \disp, \index, \base, 1
421
+
.endm
422
+
.macro VSTBRF vr1, disp, index="%r0", base
423
+
VSTBR \vr1, \disp, \index, \base, 2
424
+
.endm
425
+
.macro VSTBRG vr1, disp, index="%r0", base
426
+
VSTBR \vr1, \disp, \index, \base, 3
427
+
.endm
428
+
.macro VSTBRQ vr1, disp, index="%r0", base
429
+
VSTBR \vr1, \disp, \index, \base, 4
430
+
.endm
431
+
410
432
/* VECTOR STORE MULTIPLE */
411
433
.macro VSTM vfrom, vto, disp, base, hint=3
412
434
VX_NUM v1, \vfrom
+14
arch/s390/include/asm/module.h
+14
arch/s390/include/asm/module.h
···
38
38
#endif /* CONFIG_FUNCTION_TRACER */
39
39
};
40
40
41
+
static inline const Elf_Shdr *find_section(const Elf_Ehdr *hdr,
42
+
const Elf_Shdr *sechdrs,
43
+
const char *name)
44
+
{
45
+
const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
46
+
const Elf_Shdr *s, *se;
47
+
48
+
for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) {
49
+
if (strcmp(name, secstrs + s->sh_name) == 0)
50
+
return s;
51
+
}
52
+
return NULL;
53
+
}
54
+
41
55
#endif /* _ASM_S390_MODULE_H */
+17
arch/s390/include/asm/vdso-symbols.h
+17
arch/s390/include/asm/vdso-symbols.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
#ifndef __S390_VDSO_SYMBOLS_H__
3
+
#define __S390_VDSO_SYMBOLS_H__
4
+
5
+
#include <generated/vdso64-offsets.h>
6
+
#ifdef CONFIG_COMPAT
7
+
#include <generated/vdso32-offsets.h>
8
+
#endif
9
+
10
+
#define VDSO64_SYMBOL(tsk, name) ((tsk)->mm->context.vdso_base + (vdso64_offset_##name))
11
+
#ifdef CONFIG_COMPAT
12
+
#define VDSO32_SYMBOL(tsk, name) ((tsk)->mm->context.vdso_base + (vdso32_offset_##name))
13
+
#else
14
+
#define VDSO32_SYMBOL(tsk, name) (-1UL)
15
+
#endif
16
+
17
+
#endif /* __S390_VDSO_SYMBOLS_H__ */
-12
arch/s390/include/asm/vdso.h
-12
arch/s390/include/asm/vdso.h
···
6
6
7
7
#ifndef __ASSEMBLY__
8
8
9
-
#include <generated/vdso64-offsets.h>
10
-
#ifdef CONFIG_COMPAT
11
-
#include <generated/vdso32-offsets.h>
12
-
#endif
13
-
14
-
#define VDSO64_SYMBOL(tsk, name) ((tsk)->mm->context.vdso_base + (vdso64_offset_##name))
15
-
#ifdef CONFIG_COMPAT
16
-
#define VDSO32_SYMBOL(tsk, name) ((tsk)->mm->context.vdso_base + (vdso32_offset_##name))
17
-
#else
18
-
#define VDSO32_SYMBOL(tsk, name) (-1UL)
19
-
#endif
20
-
21
9
extern struct vdso_data *vdso_data;
22
10
23
11
int vdso_getcpu_init(void);
+40
arch/s390/include/asm/vdso/getrandom.h
+40
arch/s390/include/asm/vdso/getrandom.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
3
+
#ifndef __ASM_VDSO_GETRANDOM_H
4
+
#define __ASM_VDSO_GETRANDOM_H
5
+
6
+
#ifndef __ASSEMBLY__
7
+
8
+
#include <vdso/datapage.h>
9
+
#include <asm/vdso/vsyscall.h>
10
+
#include <asm/syscall.h>
11
+
#include <asm/unistd.h>
12
+
#include <asm/page.h>
13
+
14
+
/**
15
+
* getrandom_syscall - Invoke the getrandom() syscall.
16
+
* @buffer: Destination buffer to fill with random bytes.
17
+
* @len: Size of @buffer in bytes.
18
+
* @flags: Zero or more GRND_* flags.
19
+
* Returns: The number of random bytes written to @buffer, or a negative value indicating an error.
20
+
*/
21
+
static __always_inline ssize_t getrandom_syscall(void *buffer, size_t len, unsigned int flags)
22
+
{
23
+
return syscall3(__NR_getrandom, (long)buffer, (long)len, (long)flags);
24
+
}
25
+
26
+
static __always_inline const struct vdso_rng_data *__arch_get_vdso_rng_data(void)
27
+
{
28
+
/*
29
+
* The RNG data is in the real VVAR data page, but if a task belongs to a time namespace
30
+
* then VVAR_DATA_PAGE_OFFSET points to the namespace-specific VVAR page and VVAR_TIMENS_
31
+
* PAGE_OFFSET points to the real VVAR page.
32
+
*/
33
+
if (IS_ENABLED(CONFIG_TIME_NS) && _vdso_data->clock_mode == VDSO_CLOCKMODE_TIMENS)
34
+
return (void *)&_vdso_rng_data + VVAR_TIMENS_PAGE_OFFSET * PAGE_SIZE;
35
+
return &_vdso_rng_data;
36
+
}
37
+
38
+
#endif /* !__ASSEMBLY__ */
39
+
40
+
#endif /* __ASM_VDSO_GETRANDOM_H */
+15
arch/s390/include/asm/vdso/vsyscall.h
+15
arch/s390/include/asm/vdso/vsyscall.h
···
2
2
#ifndef __ASM_VDSO_VSYSCALL_H
3
3
#define __ASM_VDSO_VSYSCALL_H
4
4
5
+
#define __VDSO_RND_DATA_OFFSET 768
6
+
5
7
#ifndef __ASSEMBLY__
6
8
7
9
#include <linux/hrtimer.h>
8
10
#include <linux/timekeeper_internal.h>
9
11
#include <vdso/datapage.h>
10
12
#include <asm/vdso.h>
13
+
14
+
enum vvar_pages {
15
+
VVAR_DATA_PAGE_OFFSET,
16
+
VVAR_TIMENS_PAGE_OFFSET,
17
+
VVAR_NR_PAGES
18
+
};
19
+
11
20
/*
12
21
* Update the vDSO data page to keep in sync with kernel timekeeping.
13
22
*/
···
26
17
return vdso_data;
27
18
}
28
19
#define __arch_get_k_vdso_data __s390_get_k_vdso_data
20
+
21
+
static __always_inline struct vdso_rng_data *__s390_get_k_vdso_rnd_data(void)
22
+
{
23
+
return (void *)vdso_data + __VDSO_RND_DATA_OFFSET;
24
+
}
25
+
#define __arch_get_k_vdso_rng_data __s390_get_k_vdso_rnd_data
29
26
30
27
/* The asm-generic header needs to be included after the definitions above */
31
28
#include <asm-generic/vdso/vsyscall.h>
+1
-1
arch/s390/kernel/compat_signal.c
+1
-1
arch/s390/kernel/compat_signal.c
···
24
24
#include <linux/tty.h>
25
25
#include <linux/personality.h>
26
26
#include <linux/binfmts.h>
27
+
#include <asm/vdso-symbols.h>
27
28
#include <asm/access-regs.h>
28
29
#include <asm/ucontext.h>
29
30
#include <linux/uaccess.h>
30
31
#include <asm/lowcore.h>
31
-
#include <asm/vdso.h>
32
32
#include <asm/fpu.h>
33
33
#include "compat_linux.h"
34
34
#include "compat_ptrace.h"
+1
-1
arch/s390/kernel/entry.S
+1
-1
arch/s390/kernel/entry.S
···
42
42
43
43
.macro LPSWEY address, lpswe
44
44
ALTERNATIVE_2 "b \lpswe;nopr", \
45
-
".insn siy,0xeb0000000071,\address,0", ALT_FACILITY_EARLY(193), \
45
+
".insn siy,0xeb0000000071,\address,0", ALT_FACILITY(193), \
46
46
__stringify(.insn siy,0xeb0000000071,LOWCORE_ALT_ADDRESS+\address,0), \
47
47
ALT_LOWCORE
48
48
.endm
+1
-1
arch/s390/kernel/signal.c
+1
-1
arch/s390/kernel/signal.c
+20
-6
arch/s390/kernel/vdso.c
+20
-6
arch/s390/kernel/vdso.c
···
12
12
#include <linux/errno.h>
13
13
#include <linux/init.h>
14
14
#include <linux/kernel.h>
15
+
#include <linux/module.h>
15
16
#include <linux/mm.h>
16
17
#include <linux/slab.h>
17
18
#include <linux/smp.h>
18
19
#include <linux/time_namespace.h>
19
20
#include <linux/random.h>
20
21
#include <vdso/datapage.h>
22
+
#include <asm/vdso/vsyscall.h>
23
+
#include <asm/alternative.h>
21
24
#include <asm/vdso.h>
22
25
23
26
extern char vdso64_start[], vdso64_end[];
···
31
28
static union vdso_data_store vdso_data_store __page_aligned_data;
32
29
33
30
struct vdso_data *vdso_data = vdso_data_store.data;
34
-
35
-
enum vvar_pages {
36
-
VVAR_DATA_PAGE_OFFSET,
37
-
VVAR_TIMENS_PAGE_OFFSET,
38
-
VVAR_NR_PAGES,
39
-
};
40
31
41
32
#ifdef CONFIG_TIME_NS
42
33
struct vdso_data *arch_get_vdso_data(void *vvar_page)
···
247
250
return pagelist;
248
251
}
249
252
253
+
static void vdso_apply_alternatives(void)
254
+
{
255
+
const struct elf64_shdr *alt, *shdr;
256
+
struct alt_instr *start, *end;
257
+
const struct elf64_hdr *hdr;
258
+
259
+
hdr = (struct elf64_hdr *)vdso64_start;
260
+
shdr = (void *)hdr + hdr->e_shoff;
261
+
alt = find_section(hdr, shdr, ".altinstructions");
262
+
if (!alt)
263
+
return;
264
+
start = (void *)hdr + alt->sh_offset;
265
+
end = (void *)hdr + alt->sh_offset + alt->sh_size;
266
+
apply_alternatives(start, end);
267
+
}
268
+
250
269
static int __init vdso_init(void)
251
270
{
271
+
vdso_apply_alternatives();
252
272
vdso64_mapping.pages = vdso_setup_pages(vdso64_start, vdso64_end);
253
273
if (IS_ENABLED(CONFIG_COMPAT))
254
274
vdso32_mapping.pages = vdso_setup_pages(vdso32_start, vdso32_end);
+7
-2
arch/s390/kernel/vdso64/Makefile
+7
-2
arch/s390/kernel/vdso64/Makefile
···
3
3
4
4
# Include the generic Makefile to check the built vdso.
5
5
include $(srctree)/lib/vdso/Makefile
6
-
obj-vdso64 = vdso_user_wrapper.o note.o
7
-
obj-cvdso64 = vdso64_generic.o getcpu.o
6
+
obj-vdso64 = vdso_user_wrapper.o note.o vgetrandom-chacha.o
7
+
obj-cvdso64 = vdso64_generic.o getcpu.o vgetrandom.o
8
8
VDSO_CFLAGS_REMOVE := -pg $(CC_FLAGS_FTRACE) $(CC_FLAGS_EXPOLINE) $(CC_FLAGS_CHECK_STACK)
9
9
CFLAGS_REMOVE_getcpu.o = $(VDSO_CFLAGS_REMOVE)
10
+
CFLAGS_REMOVE_vgetrandom.o = $(VDSO_CFLAGS_REMOVE)
10
11
CFLAGS_REMOVE_vdso64_generic.o = $(VDSO_CFLAGS_REMOVE)
12
+
13
+
ifneq ($(c-getrandom-y),)
14
+
CFLAGS_vgetrandom.o += -include $(c-getrandom-y)
15
+
endif
11
16
12
17
# Build rules
13
18
+1
arch/s390/kernel/vdso64/vdso.h
+1
arch/s390/kernel/vdso64/vdso.h
···
10
10
int __s390_vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz);
11
11
int __s390_vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts);
12
12
int __s390_vdso_clock_getres(clockid_t clock, struct __kernel_timespec *ts);
13
+
ssize_t __kernel_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len);
13
14
14
15
#endif /* __ARCH_S390_KERNEL_VDSO64_VDSO_H */
+7
arch/s390/kernel/vdso64/vdso64.lds.S
+7
arch/s390/kernel/vdso64/vdso64.lds.S
···
4
4
* library
5
5
*/
6
6
7
+
#include <asm/vdso/vsyscall.h>
7
8
#include <asm/page.h>
8
9
#include <asm/vdso.h>
9
10
···
14
13
SECTIONS
15
14
{
16
15
PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
16
+
PROVIDE(_vdso_rng_data = _vdso_data + __VDSO_RND_DATA_OFFSET);
17
17
#ifdef CONFIG_TIME_NS
18
18
PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
19
19
#endif
···
43
41
*/
44
42
.rodata : { *(.rodata .rodata.* .gnu.linkonce.r.*) }
45
43
.rodata1 : { *(.rodata1) }
44
+
45
+
. = ALIGN(8);
46
+
.altinstructions : { *(.altinstructions) }
47
+
.altinstr_replacement : { *(.altinstr_replacement) }
46
48
47
49
.dynamic : { *(.dynamic) } :text :dynamic
48
50
···
146
140
__kernel_restart_syscall;
147
141
__kernel_rt_sigreturn;
148
142
__kernel_sigreturn;
143
+
__kernel_getrandom;
149
144
local: *;
150
145
};
151
146
}
+185
arch/s390/kernel/vdso64/vgetrandom-chacha.S
+185
arch/s390/kernel/vdso64/vgetrandom-chacha.S
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
3
+
#include <linux/linkage.h>
4
+
#include <asm/alternative.h>
5
+
#include <asm/fpu-insn.h>
6
+
7
+
#define STATE0 %v0
8
+
#define STATE1 %v1
9
+
#define STATE2 %v2
10
+
#define STATE3 %v3
11
+
#define COPY0 %v4
12
+
#define COPY1 %v5
13
+
#define COPY2 %v6
14
+
#define COPY3 %v7
15
+
#define PERM4 %v16
16
+
#define PERM8 %v17
17
+
#define PERM12 %v18
18
+
#define BEPERM %v19
19
+
#define TMP0 %v20
20
+
#define TMP1 %v21
21
+
#define TMP2 %v22
22
+
#define TMP3 %v23
23
+
24
+
.section .rodata
25
+
26
+
.balign 128
27
+
.Lconstants:
28
+
.long 0x61707865,0x3320646e,0x79622d32,0x6b206574 # endian-neutral
29
+
.long 0x04050607,0x08090a0b,0x0c0d0e0f,0x00010203 # rotl 4 bytes
30
+
.long 0x08090a0b,0x0c0d0e0f,0x00010203,0x04050607 # rotl 8 bytes
31
+
.long 0x0c0d0e0f,0x00010203,0x04050607,0x08090a0b # rotl 12 bytes
32
+
.long 0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c # byte swap
33
+
34
+
.text
35
+
/*
36
+
* s390 ChaCha20 implementation meant for vDSO. Produces a given positive
37
+
* number of blocks of output with nonce 0, taking an input key and 8-bytes
38
+
* counter. Does not spill to the stack.
39
+
*
40
+
* void __arch_chacha20_blocks_nostack(uint8_t *dst_bytes,
41
+
* const uint8_t *key,
42
+
* uint32_t *counter,
43
+
* size_t nblocks)
44
+
*/
45
+
SYM_FUNC_START(__arch_chacha20_blocks_nostack)
46
+
larl %r1,.Lconstants
47
+
48
+
/* COPY0 = "expand 32-byte k" */
49
+
VL COPY0,0,,%r1
50
+
51
+
/* PERM4-PERM12,BEPERM = byte selectors for VPERM */
52
+
VLM PERM4,BEPERM,16,%r1
53
+
54
+
/* COPY1,COPY2 = key */
55
+
VLM COPY1,COPY2,0,%r3
56
+
57
+
/* COPY3 = counter || zero nonce */
58
+
lg %r3,0(%r4)
59
+
VZERO COPY3
60
+
VLVGG COPY3,%r3,0
61
+
62
+
lghi %r1,0
63
+
.Lblock:
64
+
VLR STATE0,COPY0
65
+
VLR STATE1,COPY1
66
+
VLR STATE2,COPY2
67
+
VLR STATE3,COPY3
68
+
69
+
lghi %r0,10
70
+
.Ldoubleround:
71
+
/* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */
72
+
VAF STATE0,STATE0,STATE1
73
+
VX STATE3,STATE3,STATE0
74
+
VERLLF STATE3,STATE3,16
75
+
76
+
/* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */
77
+
VAF STATE2,STATE2,STATE3
78
+
VX STATE1,STATE1,STATE2
79
+
VERLLF STATE1,STATE1,12
80
+
81
+
/* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */
82
+
VAF STATE0,STATE0,STATE1
83
+
VX STATE3,STATE3,STATE0
84
+
VERLLF STATE3,STATE3,8
85
+
86
+
/* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */
87
+
VAF STATE2,STATE2,STATE3
88
+
VX STATE1,STATE1,STATE2
89
+
VERLLF STATE1,STATE1,7
90
+
91
+
/* STATE1[0,1,2,3] = STATE1[1,2,3,0] */
92
+
VPERM STATE1,STATE1,STATE1,PERM4
93
+
/* STATE2[0,1,2,3] = STATE2[2,3,0,1] */
94
+
VPERM STATE2,STATE2,STATE2,PERM8
95
+
/* STATE3[0,1,2,3] = STATE3[3,0,1,2] */
96
+
VPERM STATE3,STATE3,STATE3,PERM12
97
+
98
+
/* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */
99
+
VAF STATE0,STATE0,STATE1
100
+
VX STATE3,STATE3,STATE0
101
+
VERLLF STATE3,STATE3,16
102
+
103
+
/* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */
104
+
VAF STATE2,STATE2,STATE3
105
+
VX STATE1,STATE1,STATE2
106
+
VERLLF STATE1,STATE1,12
107
+
108
+
/* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */
109
+
VAF STATE0,STATE0,STATE1
110
+
VX STATE3,STATE3,STATE0
111
+
VERLLF STATE3,STATE3,8
112
+
113
+
/* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */
114
+
VAF STATE2,STATE2,STATE3
115
+
VX STATE1,STATE1,STATE2
116
+
VERLLF STATE1,STATE1,7
117
+
118
+
/* STATE1[0,1,2,3] = STATE1[3,0,1,2] */
119
+
VPERM STATE1,STATE1,STATE1,PERM12
120
+
/* STATE2[0,1,2,3] = STATE2[2,3,0,1] */
121
+
VPERM STATE2,STATE2,STATE2,PERM8
122
+
/* STATE3[0,1,2,3] = STATE3[1,2,3,0] */
123
+
VPERM STATE3,STATE3,STATE3,PERM4
124
+
brctg %r0,.Ldoubleround
125
+
126
+
/* OUTPUT0 = STATE0 + STATE0 */
127
+
VAF STATE0,STATE0,COPY0
128
+
/* OUTPUT1 = STATE1 + STATE1 */
129
+
VAF STATE1,STATE1,COPY1
130
+
/* OUTPUT2 = STATE2 + STATE2 */
131
+
VAF STATE2,STATE2,COPY2
132
+
/* OUTPUT2 = STATE3 + STATE3 */
133
+
VAF STATE3,STATE3,COPY3
134
+
135
+
/*
136
+
* 32 bit wise little endian store to OUTPUT. If the vector
137
+
* enhancement facility 2 is not installed use the slow path.
138
+
*/
139
+
ALTERNATIVE "brc 0xf,.Lstoreslow", "nop", ALT_FACILITY(148)
140
+
VSTBRF STATE0,0,,%r2
141
+
VSTBRF STATE1,16,,%r2
142
+
VSTBRF STATE2,32,,%r2
143
+
VSTBRF STATE3,48,,%r2
144
+
.Lstoredone:
145
+
146
+
/* ++COPY3.COUNTER */
147
+
/* alsih %r3,1 */
148
+
.insn rilu,0xcc0a00000000,%r3,1
149
+
alcr %r3,%r1
150
+
VLVGG COPY3,%r3,0
151
+
152
+
/* OUTPUT += 64, --NBLOCKS */
153
+
aghi %r2,64
154
+
brctg %r5,.Lblock
155
+
156
+
/* COUNTER = COPY3.COUNTER */
157
+
stg %r3,0(%r4)
158
+
159
+
/* Zero out potentially sensitive regs */
160
+
VZERO STATE0
161
+
VZERO STATE1
162
+
VZERO STATE2
163
+
VZERO STATE3
164
+
VZERO COPY1
165
+
VZERO COPY2
166
+
167
+
/* Early exit if TMP0-TMP3 have not been used */
168
+
ALTERNATIVE "nopr", "br %r14", ALT_FACILITY(148)
169
+
170
+
VZERO TMP0
171
+
VZERO TMP1
172
+
VZERO TMP2
173
+
VZERO TMP3
174
+
175
+
br %r14
176
+
177
+
.Lstoreslow:
178
+
/* Convert STATE to little endian format and store to OUTPUT */
179
+
VPERM TMP0,STATE0,STATE0,BEPERM
180
+
VPERM TMP1,STATE1,STATE1,BEPERM
181
+
VPERM TMP2,STATE2,STATE2,BEPERM
182
+
VPERM TMP3,STATE3,STATE3,BEPERM
183
+
VSTM TMP0,TMP3,0,%r2
184
+
j .Lstoredone
185
+
SYM_FUNC_END(__arch_chacha20_blocks_nostack)
+14
arch/s390/kernel/vdso64/vgetrandom.c
+14
arch/s390/kernel/vdso64/vgetrandom.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
3
+
#include <asm/facility.h>
4
+
#include <uapi/asm-generic/errno.h>
5
+
#include "vdso.h"
6
+
7
+
ssize_t __kernel_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len)
8
+
{
9
+
if (test_facility(129))
10
+
return __cvdso_getrandom(buffer, len, flags, opaque_state, opaque_len);
11
+
if (unlikely(opaque_len == ~0UL && !buffer && !len && !flags))
12
+
return -ENOSYS;
13
+
return getrandom_syscall(buffer, len, flags);
14
+
}
-2
arch/x86/entry/vdso/vgetrandom.c
-2
arch/x86/entry/vdso/vgetrandom.c
···
6
6
7
7
#include "../../../../lib/vdso/getrandom.c"
8
8
9
-
ssize_t __vdso_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len);
10
-
11
9
ssize_t __vdso_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len)
12
10
{
13
11
return __cvdso_getrandom(buffer, len, flags, opaque_state, opaque_len);
+3
arch/x86/entry/vdso/vma.c
+3
arch/x86/entry/vdso/vma.c
+1
arch/x86/include/asm/pvclock.h
+1
arch/x86/include/asm/pvclock.h
···
5
5
#include <asm/clocksource.h>
6
6
#include <asm/pvclock-abi.h>
7
7
8
+
struct timespec64;
8
9
/* some helper functions for xen and kvm pv clock sources */
9
10
u64 pvclock_clocksource_read(struct pvclock_vcpu_time_info *src);
10
11
u64 pvclock_clocksource_read_nowd(struct pvclock_vcpu_time_info *src);
-13
arch/x86/include/asm/vdso/getrandom.h
-13
arch/x86/include/asm/vdso/getrandom.h
···
37
37
return &__vdso_rng_data;
38
38
}
39
39
40
-
/**
41
-
* __arch_chacha20_blocks_nostack - Generate ChaCha20 stream without using the stack.
42
-
* @dst_bytes: Destination buffer to hold @nblocks * 64 bytes of output.
43
-
* @key: 32-byte input key.
44
-
* @counter: 8-byte counter, read on input and updated on return.
45
-
* @nblocks: Number of blocks to generate.
46
-
*
47
-
* Generates a given positive number of blocks of ChaCha20 output with nonce=0, and does not write
48
-
* to any stack or memory outside of the parameters passed to it, in order to mitigate stack data
49
-
* leaking into forked child processes.
50
-
*/
51
-
extern void __arch_chacha20_blocks_nostack(u8 *dst_bytes, const u32 *key, u32 *counter, size_t nblocks);
52
-
53
40
#endif /* !__ASSEMBLY__ */
54
41
55
42
#endif /* __ASM_VDSO_GETRANDOM_H */
+7
-3
arch/x86/include/asm/vdso/vsyscall.h
+7
-3
arch/x86/include/asm/vdso/vsyscall.h
···
9
9
#include <asm/vgtod.h>
10
10
#include <asm/vvar.h>
11
11
12
-
DEFINE_VVAR(struct vdso_data, _vdso_data);
13
-
DEFINE_VVAR_SINGLE(struct vdso_rng_data, _vdso_rng_data);
14
-
15
12
/*
16
13
* Update the vDSO data page to keep in sync with kernel timekeeping.
17
14
*/
···
18
21
return _vdso_data;
19
22
}
20
23
#define __arch_get_k_vdso_data __x86_get_k_vdso_data
24
+
25
+
static __always_inline
26
+
struct vdso_rng_data *__x86_get_k_vdso_rng_data(void)
27
+
{
28
+
return &_vdso_rng_data;
29
+
}
30
+
#define __arch_get_k_vdso_rng_data __x86_get_k_vdso_rng_data
21
31
22
32
/* The asm-generic header needs to be included after the definitions above */
23
33
#include <asm-generic/vdso/vsyscall.h>
+10
-2
drivers/char/random.c
+10
-2
drivers/char/random.c
···
59
59
#ifdef CONFIG_VDSO_GETRANDOM
60
60
#include <vdso/getrandom.h>
61
61
#include <vdso/datapage.h>
62
+
#include <vdso/vsyscall.h>
62
63
#endif
63
64
#include <asm/archrandom.h>
64
65
#include <asm/processor.h>
···
282
281
* former to arrive at the latter. Use smp_store_release so that this
283
282
* is ordered with the write above to base_crng.generation. Pairs with
284
283
* the smp_rmb() before the syscall in the vDSO code.
284
+
*
285
+
* Cast to unsigned long for 32-bit architectures, since atomic 64-bit
286
+
* operations are not supported on those architectures. This is safe
287
+
* because base_crng.generation is a 32-bit value. On big-endian
288
+
* architectures it will be stored in the upper 32 bits, but that's okay
289
+
* because the vDSO side only checks whether the value changed, without
290
+
* actually using or interpreting the value.
285
291
*/
286
-
smp_store_release(&_vdso_rng_data.generation, next_gen + 1);
292
+
smp_store_release((unsigned long *)&__arch_get_k_vdso_rng_data()->generation, next_gen + 1);
287
293
#endif
288
294
if (!static_branch_likely(&crng_is_ready))
289
295
crng_init = CRNG_READY;
···
743
735
queue_work(system_unbound_wq, &set_ready);
744
736
atomic_notifier_call_chain(&random_ready_notifier, 0, NULL);
745
737
#ifdef CONFIG_VDSO_GETRANDOM
746
-
WRITE_ONCE(_vdso_rng_data.is_ready, true);
738
+
WRITE_ONCE(__arch_get_k_vdso_rng_data()->is_ready, true);
747
739
#endif
748
740
wake_up_interruptible(&crng_init_wait);
749
741
kill_fasync(&fasync, SIGIO, POLL_IN);
+3
-1
fs/proc/task_mmu.c
+3
-1
fs/proc/task_mmu.c
···
989
989
#ifdef CONFIG_X86_USER_SHADOW_STACK
990
990
[ilog2(VM_SHADOW_STACK)] = "ss",
991
991
#endif
992
-
#ifdef CONFIG_64BIT
992
+
#if defined(CONFIG_64BIT) || defined(CONFIG_PPC32)
993
993
[ilog2(VM_DROPPABLE)] = "dp",
994
+
#endif
995
+
#ifdef CONFIG_64BIT
994
996
[ilog2(VM_SEALED)] = "sl",
995
997
#endif
996
998
};
+1
-10
include/asm-generic/unaligned.h
+1
-10
include/asm-generic/unaligned.h
···
8
8
*/
9
9
#include <linux/unaligned/packed_struct.h>
10
10
#include <asm/byteorder.h>
11
-
12
-
#define __get_unaligned_t(type, ptr) ({ \
13
-
const struct { type x; } __packed *__pptr = (typeof(__pptr))(ptr); \
14
-
__pptr->x; \
15
-
})
16
-
17
-
#define __put_unaligned_t(type, val, ptr) do { \
18
-
struct { type x; } __packed *__pptr = (typeof(__pptr))(ptr); \
19
-
__pptr->x = (val); \
20
-
} while (0)
11
+
#include <vdso/unaligned.h>
21
12
22
13
#define get_unaligned(ptr) __get_unaligned_t(typeof(*(ptr)), (ptr))
23
14
#define put_unaligned(val, ptr) __put_unaligned_t(typeof(*(ptr)), (val), (ptr))
+3
-1
include/linux/mm.h
+3
-1
include/linux/mm.h
···
367
367
368
368
#if defined(CONFIG_X86)
369
369
# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
370
-
#elif defined(CONFIG_PPC)
370
+
#elif defined(CONFIG_PPC64)
371
371
# define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */
372
372
#elif defined(CONFIG_PARISC)
373
373
# define VM_GROWSUP VM_ARCH_1
···
417
417
#ifdef CONFIG_64BIT
418
418
#define VM_DROPPABLE_BIT 40
419
419
#define VM_DROPPABLE BIT(VM_DROPPABLE_BIT)
420
+
#elif defined(CONFIG_PPC32)
421
+
#define VM_DROPPABLE VM_ARCH_1
420
422
#else
421
423
#define VM_DROPPABLE VM_NONE
422
424
#endif
+2
-2
include/trace/events/mmflags.h
+2
-2
include/trace/events/mmflags.h
···
143
143
144
144
#if defined(CONFIG_X86)
145
145
#define __VM_ARCH_SPECIFIC_1 {VM_PAT, "pat" }
146
-
#elif defined(CONFIG_PPC)
146
+
#elif defined(CONFIG_PPC64)
147
147
#define __VM_ARCH_SPECIFIC_1 {VM_SAO, "sao" }
148
148
#elif defined(CONFIG_PARISC)
149
149
#define __VM_ARCH_SPECIFIC_1 {VM_GROWSUP, "growsup" }
···
165
165
# define IF_HAVE_UFFD_MINOR(flag, name)
166
166
#endif
167
167
168
-
#ifdef CONFIG_64BIT
168
+
#if defined(CONFIG_64BIT) || defined(CONFIG_PPC32)
169
169
# define IF_HAVE_VM_DROPPABLE(flag, name) {flag, name},
170
170
#else
171
171
# define IF_HAVE_VM_DROPPABLE(flag, name)
+28
include/vdso/getrandom.h
+28
include/vdso/getrandom.h
···
43
43
bool in_use;
44
44
};
45
45
46
+
/**
47
+
* __arch_chacha20_blocks_nostack - Generate ChaCha20 stream without using the stack.
48
+
* @dst_bytes: Destination buffer to hold @nblocks * 64 bytes of output.
49
+
* @key: 32-byte input key.
50
+
* @counter: 8-byte counter, read on input and updated on return.
51
+
* @nblocks: Number of blocks to generate.
52
+
*
53
+
* Generates a given positive number of blocks of ChaCha20 output with nonce=0, and does not write
54
+
* to any stack or memory outside of the parameters passed to it, in order to mitigate stack data
55
+
* leaking into forked child processes.
56
+
*/
57
+
extern void __arch_chacha20_blocks_nostack(u8 *dst_bytes, const u32 *key, u32 *counter, size_t nblocks);
58
+
59
+
/**
60
+
* __vdso_getrandom - Architecture-specific vDSO implementation of getrandom() syscall.
61
+
* @buffer: Passed to __cvdso_getrandom().
62
+
* @len: Passed to __cvdso_getrandom().
63
+
* @flags: Passed to __cvdso_getrandom().
64
+
* @opaque_state: Passed to __cvdso_getrandom().
65
+
* @opaque_len: Passed to __cvdso_getrandom();
66
+
*
67
+
* This function is implemented by making a single call to to __cvdso_getrandom(), whose
68
+
* documentation may be consulted for more information.
69
+
*
70
+
* Returns: The return value of __cvdso_getrandom().
71
+
*/
72
+
extern ssize_t __vdso_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len);
73
+
46
74
#endif /* _VDSO_GETRANDOM_H */
+1
include/vdso/helpers.h
+1
include/vdso/helpers.h
+15
include/vdso/unaligned.h
+15
include/vdso/unaligned.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
#ifndef __VDSO_UNALIGNED_H
3
+
#define __VDSO_UNALIGNED_H
4
+
5
+
#define __get_unaligned_t(type, ptr) ({ \
6
+
const struct { type x; } __packed *__pptr = (typeof(__pptr))(ptr); \
7
+
__pptr->x; \
8
+
})
9
+
10
+
#define __put_unaligned_t(type, val, ptr) do { \
11
+
struct { type x; } __packed *__pptr = (typeof(__pptr))(ptr); \
12
+
__pptr->x = (val); \
13
+
} while (0)
14
+
15
+
#endif /* __VDSO_UNALIGNED_H */
+1
lib/vdso/Makefile
+1
lib/vdso/Makefile
···
4
4
GENERIC_VDSO_DIR := $(dir $(GENERIC_VDSO_MK_PATH))
5
5
6
6
c-gettimeofday-$(CONFIG_GENERIC_GETTIMEOFDAY) := $(addprefix $(GENERIC_VDSO_DIR), gettimeofday.c)
7
+
c-getrandom-$(CONFIG_VDSO_GETRANDOM) := $(addprefix $(GENERIC_VDSO_DIR), getrandom.c)
7
8
8
9
# This cmd checks that the vdso library does not contain dynamic relocations.
9
10
# It has to be called after the linking of the vdso library and requires it
+16
-11
lib/vdso/getrandom.c
+16
-11
lib/vdso/getrandom.c
···
3
3
* Copyright (C) 2022-2024 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4
4
*/
5
5
6
-
#include <linux/cache.h>
7
-
#include <linux/kernel.h>
8
-
#include <linux/time64.h>
6
+
#include <linux/array_size.h>
7
+
#include <linux/minmax.h>
9
8
#include <vdso/datapage.h>
10
9
#include <vdso/getrandom.h>
10
+
#include <vdso/unaligned.h>
11
11
#include <asm/vdso/getrandom.h>
12
-
#include <asm/vdso/vsyscall.h>
13
-
#include <asm/unaligned.h>
14
12
#include <uapi/linux/mman.h>
13
+
#include <uapi/linux/random.h>
14
+
15
+
#undef PAGE_SIZE
16
+
#undef PAGE_MASK
17
+
#define PAGE_SIZE (1UL << CONFIG_PAGE_SHIFT)
18
+
#define PAGE_MASK (~(PAGE_SIZE - 1))
15
19
16
20
#define MEMCPY_AND_ZERO_SRC(type, dst, src, len) do { \
17
21
while (len >= sizeof(type)) { \
···
72
68
struct vgetrandom_state *state = opaque_state;
73
69
size_t batch_len, nblocks, orig_len = len;
74
70
bool in_use, have_retried = false;
75
-
unsigned long current_generation;
76
71
void *orig_buffer = buffer;
72
+
u64 current_generation;
77
73
u32 counter[2] = { 0 };
78
74
79
75
if (unlikely(opaque_len == ~0UL && !buffer && !len && !flags)) {
80
-
*(struct vgetrandom_opaque_params *)opaque_state = (struct vgetrandom_opaque_params) {
81
-
.size_of_opaque_state = sizeof(*state),
82
-
.mmap_prot = PROT_READ | PROT_WRITE,
83
-
.mmap_flags = MAP_DROPPABLE | MAP_ANONYMOUS
84
-
};
76
+
struct vgetrandom_opaque_params *params = opaque_state;
77
+
params->size_of_opaque_state = sizeof(*state);
78
+
params->mmap_prot = PROT_READ | PROT_WRITE;
79
+
params->mmap_flags = MAP_DROPPABLE | MAP_ANONYMOUS;
80
+
for (size_t i = 0; i < ARRAY_SIZE(params->reserved); ++i)
81
+
params->reserved[i] = 0;
85
82
return 0;
86
83
}
87
84
+10
tools/include/asm/alternative.h
+10
tools/include/asm/alternative.h
···
2
2
#ifndef _TOOLS_ASM_ALTERNATIVE_ASM_H
3
3
#define _TOOLS_ASM_ALTERNATIVE_ASM_H
4
4
5
+
#if defined(__s390x__)
6
+
#ifdef __ASSEMBLY__
7
+
.macro ALTERNATIVE oldinstr, newinstr, feature
8
+
\oldinstr
9
+
.endm
10
+
#endif
11
+
#else
12
+
5
13
/* Just disable it so we can build arch/x86/lib/memcpy_64.S for perf bench: */
6
14
7
15
#define ALTERNATIVE #
16
+
17
+
#endif
8
18
9
19
#endif
tools/include/generated/asm-offsets.h
tools/include/generated/asm-offsets.h
tools/include/generated/asm/cpucap-defs.h
tools/include/generated/asm/cpucap-defs.h
tools/include/generated/asm/sysreg-defs.h
tools/include/generated/asm/sysreg-defs.h
+4
tools/include/linux/compiler.h
+4
tools/include/linux/compiler.h
···
2
2
#ifndef _TOOLS_LINUX_COMPILER_H_
3
3
#define _TOOLS_LINUX_COMPILER_H_
4
4
5
+
#ifndef __ASSEMBLY__
6
+
5
7
#include <linux/compiler_types.h>
6
8
7
9
#ifndef __compiletime_error
···
225
223
#define OPTIMIZER_HIDE_VAR(var) \
226
224
__asm__ ("" : "=r" (var) : "0" (var))
227
225
#endif
226
+
227
+
#endif /* __ASSEMBLY__ */
228
228
229
229
#endif /* _TOOLS_LINUX_COMPILER_H */
+4
tools/include/linux/linkage.h
+4
tools/include/linux/linkage.h
+9
-12
tools/testing/selftests/vDSO/Makefile
+9
-12
tools/testing/selftests/vDSO/Makefile
···
1
1
# SPDX-License-Identifier: GPL-2.0
2
-
uname_M := $(shell uname -m 2>/dev/null || echo not)
3
-
ARCH ?= $(shell echo $(uname_M) | sed -e s/i.86/x86/ -e s/x86_64/x86/)
4
-
SODIUM := $(shell pkg-config --libs libsodium 2>/dev/null)
2
+
include ../../../scripts/Makefile.arch
5
3
6
4
TEST_GEN_PROGS := vdso_test_gettimeofday
7
5
TEST_GEN_PROGS += vdso_test_getcpu
···
9
11
TEST_GEN_PROGS += vdso_standalone_test_x86
10
12
endif
11
13
TEST_GEN_PROGS += vdso_test_correctness
12
-
ifeq ($(uname_M),x86_64)
14
+
ifeq ($(ARCH)$(CONFIG_X86_32),$(filter $(ARCH)$(CONFIG_X86_32),x86 x86_64 loongarch arm64 powerpc s390))
13
15
TEST_GEN_PROGS += vdso_test_getrandom
14
-
ifneq ($(SODIUM),)
15
16
TEST_GEN_PROGS += vdso_test_chacha
16
17
endif
17
-
endif
18
18
19
-
CFLAGS := -std=gnu99
19
+
CFLAGS := -std=gnu99 -O2
20
20
21
21
ifeq ($(CONFIG_X86_32),y)
22
22
LDLIBS += -lgcc_s
···
34
38
35
39
$(OUTPUT)/vdso_test_getrandom: parse_vdso.c
36
40
$(OUTPUT)/vdso_test_getrandom: CFLAGS += -isystem $(top_srcdir)/tools/include \
41
+
$(KHDR_INCLUDES) \
37
42
-isystem $(top_srcdir)/include/uapi
38
43
39
-
$(OUTPUT)/vdso_test_chacha: $(top_srcdir)/arch/$(ARCH)/entry/vdso/vgetrandom-chacha.S
44
+
$(OUTPUT)/vdso_test_chacha: $(top_srcdir)/tools/arch/$(SRCARCH)/vdso/vgetrandom-chacha.S
40
45
$(OUTPUT)/vdso_test_chacha: CFLAGS += -idirafter $(top_srcdir)/tools/include \
41
-
-isystem $(top_srcdir)/arch/$(ARCH)/include \
42
-
-isystem $(top_srcdir)/include \
43
-
-D__ASSEMBLY__ -DBULID_VDSO -DCONFIG_FUNCTION_ALIGNMENT=0 \
44
-
-Wa,--noexecstack $(SODIUM)
46
+
-idirafter $(top_srcdir)/tools/include/generated \
47
+
-idirafter $(top_srcdir)/arch/$(SRCARCH)/include \
48
+
-idirafter $(top_srcdir)/include \
49
+
-D__ASSEMBLY__ -Wa,--noexecstack
+12
-5
tools/testing/selftests/vDSO/parse_vdso.c
+12
-5
tools/testing/selftests/vDSO/parse_vdso.c
···
36
36
#define ELF_BITS_XFORM(bits, x) ELF_BITS_XFORM2(bits, x)
37
37
#define ELF(x) ELF_BITS_XFORM(ELF_BITS, x)
38
38
39
+
#ifdef __s390x__
40
+
#define ELF_HASH_ENTRY ELF(Xword)
41
+
#else
42
+
#define ELF_HASH_ENTRY ELF(Word)
43
+
#endif
44
+
39
45
static struct vdso_info
40
46
{
41
47
bool valid;
···
53
47
/* Symbol table */
54
48
ELF(Sym) *symtab;
55
49
const char *symstrings;
56
-
ELF(Word) *bucket, *chain;
57
-
ELF(Word) nbucket, nchain;
50
+
ELF_HASH_ENTRY *bucket, *chain;
51
+
ELF_HASH_ENTRY nbucket, nchain;
58
52
59
53
/* Version table */
60
54
ELF(Versym) *versym;
···
121
115
/*
122
116
* Fish out the useful bits of the dynamic table.
123
117
*/
124
-
ELF(Word) *hash = 0;
118
+
ELF_HASH_ENTRY *hash = 0;
125
119
vdso_info.symstrings = 0;
126
120
vdso_info.symtab = 0;
127
121
vdso_info.versym = 0;
···
139
133
+ vdso_info.load_offset);
140
134
break;
141
135
case DT_HASH:
142
-
hash = (ELF(Word) *)
136
+
hash = (ELF_HASH_ENTRY *)
143
137
((uintptr_t)dyn[i].d_un.d_ptr
144
138
+ vdso_info.load_offset);
145
139
break;
···
222
216
ELF(Sym) *sym = &vdso_info.symtab[chain];
223
217
224
218
/* Check for a defined global or weak function w/ right name. */
225
-
if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
219
+
if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC &&
220
+
ELF64_ST_TYPE(sym->st_info) != STT_NOTYPE)
226
221
continue;
227
222
if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
228
223
ELF64_ST_BIND(sym->st_info) != STB_WEAK)
+70
tools/testing/selftests/vDSO/vdso_call.h
+70
tools/testing/selftests/vDSO/vdso_call.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/*
3
+
* Macro to call vDSO functions
4
+
*
5
+
* Copyright (C) 2024 Christophe Leroy <christophe.leroy@csgroup.eu>, CS GROUP France
6
+
*/
7
+
#ifndef __VDSO_CALL_H__
8
+
#define __VDSO_CALL_H__
9
+
10
+
#ifdef __powerpc__
11
+
12
+
#define LOADARGS_1(fn, __arg1) do { \
13
+
_r0 = fn; \
14
+
_r3 = (long)__arg1; \
15
+
} while (0)
16
+
17
+
#define LOADARGS_2(fn, __arg1, __arg2) do { \
18
+
_r0 = fn; \
19
+
_r3 = (long)__arg1; \
20
+
_r4 = (long)__arg2; \
21
+
} while (0)
22
+
23
+
#define LOADARGS_3(fn, __arg1, __arg2, __arg3) do { \
24
+
_r0 = fn; \
25
+
_r3 = (long)__arg1; \
26
+
_r4 = (long)__arg2; \
27
+
_r5 = (long)__arg3; \
28
+
} while (0)
29
+
30
+
#define LOADARGS_5(fn, __arg1, __arg2, __arg3, __arg4, __arg5) do { \
31
+
_r0 = fn; \
32
+
_r3 = (long)__arg1; \
33
+
_r4 = (long)__arg2; \
34
+
_r5 = (long)__arg3; \
35
+
_r6 = (long)__arg4; \
36
+
_r7 = (long)__arg5; \
37
+
} while (0)
38
+
39
+
#define VDSO_CALL(fn, nr, args...) ({ \
40
+
register void *_r0 asm ("r0"); \
41
+
register long _r3 asm ("r3"); \
42
+
register long _r4 asm ("r4"); \
43
+
register long _r5 asm ("r5"); \
44
+
register long _r6 asm ("r6"); \
45
+
register long _r7 asm ("r7"); \
46
+
register long _r8 asm ("r8"); \
47
+
register long _rval asm ("r3"); \
48
+
\
49
+
LOADARGS_##nr(fn, args); \
50
+
\
51
+
asm volatile( \
52
+
" mtctr %0\n" \
53
+
" bctrl\n" \
54
+
" bns+ 1f\n" \
55
+
" neg 3, 3\n" \
56
+
"1:" \
57
+
: "+r" (_r0), "=r" (_r3), "+r" (_r4), "+r" (_r5), \
58
+
"+r" (_r6), "+r" (_r7), "+r" (_r8) \
59
+
: "r" (_rval) \
60
+
: "r9", "r10", "r11", "r12", "cr0", "cr1", "cr5", \
61
+
"cr6", "cr7", "xer", "lr", "ctr", "memory" \
62
+
); \
63
+
_rval; \
64
+
})
65
+
66
+
#else
67
+
#define VDSO_CALL(fn, nr, args...) fn(args)
68
+
#endif
69
+
70
+
#endif
+8
-10
tools/testing/selftests/vDSO/vdso_config.h
+8
-10
tools/testing/selftests/vDSO/vdso_config.h
···
18
18
#elif defined(__aarch64__)
19
19
#define VDSO_VERSION 3
20
20
#define VDSO_NAMES 0
21
+
#elif defined(__powerpc64__)
22
+
#define VDSO_VERSION 1
23
+
#define VDSO_NAMES 0
21
24
#elif defined(__powerpc__)
22
25
#define VDSO_VERSION 1
23
26
#define VDSO_NAMES 0
24
27
#define VDSO_32BIT 1
25
-
#elif defined(__powerpc64__)
26
-
#define VDSO_VERSION 1
27
-
#define VDSO_NAMES 0
28
-
#elif defined (__s390__)
28
+
#elif defined (__s390__) && !defined(__s390x__)
29
29
#define VDSO_VERSION 2
30
30
#define VDSO_NAMES 0
31
31
#define VDSO_32BIT 1
32
-
#elif defined (__s390X__)
32
+
#elif defined (__s390x__)
33
33
#define VDSO_VERSION 2
34
34
#define VDSO_NAMES 0
35
35
#elif defined(__mips__)
···
68
68
"LINUX_5.10"
69
69
};
70
70
71
-
static const char *names[2][6] = {
71
+
static const char *names[2][7] = {
72
72
{
73
73
"__kernel_gettimeofday",
74
74
"__kernel_clock_gettime",
75
75
"__kernel_time",
76
76
"__kernel_clock_getres",
77
77
"__kernel_getcpu",
78
-
#if defined(VDSO_32BIT)
79
78
"__kernel_clock_gettime64",
80
-
#endif
79
+
"__kernel_getrandom",
81
80
},
82
81
{
83
82
"__vdso_gettimeofday",
···
84
85
"__vdso_time",
85
86
"__vdso_clock_getres",
86
87
"__vdso_getcpu",
87
-
#if defined(VDSO_32BIT)
88
88
"__vdso_clock_gettime64",
89
-
#endif
89
+
"__vdso_getrandom",
90
90
},
91
91
};
92
92
+6
-8
tools/testing/selftests/vDSO/vdso_test_abi.c
+6
-8
tools/testing/selftests/vDSO/vdso_test_abi.c
···
20
20
21
21
#include "../kselftest.h"
22
22
#include "vdso_config.h"
23
-
24
-
extern void *vdso_sym(const char *version, const char *name);
25
-
extern void vdso_init_from_sysinfo_ehdr(uintptr_t base);
26
-
extern void vdso_init_from_auxv(void *auxv);
23
+
#include "vdso_call.h"
24
+
#include "parse_vdso.h"
27
25
28
26
static const char *version;
29
27
static const char **name;
···
59
61
}
60
62
61
63
struct timeval tv;
62
-
long ret = vdso_gettimeofday(&tv, 0);
64
+
long ret = VDSO_CALL(vdso_gettimeofday, 2, &tv, 0);
63
65
64
66
if (ret == 0) {
65
67
ksft_print_msg("The time is %lld.%06lld\n",
···
84
86
}
85
87
86
88
struct timespec ts;
87
-
long ret = vdso_clock_gettime(clk_id, &ts);
89
+
long ret = VDSO_CALL(vdso_clock_gettime, 2, clk_id, &ts);
88
90
89
91
if (ret == 0) {
90
92
ksft_print_msg("The time is %lld.%06lld\n",
···
109
111
return;
110
112
}
111
113
112
-
long ret = vdso_time(NULL);
114
+
long ret = VDSO_CALL(vdso_time, 1, NULL);
113
115
114
116
if (ret > 0) {
115
117
ksft_print_msg("The time in hours since January 1, 1970 is %lld\n",
···
136
138
}
137
139
138
140
struct timespec ts, sys_ts;
139
-
long ret = vdso_clock_getres(clk_id, &ts);
141
+
long ret = VDSO_CALL(vdso_clock_getres, 2, clk_id, &ts);
140
142
141
143
if (ret == 0) {
142
144
ksft_print_msg("The vdso resolution is %lld %lld\n",
+93
-10
tools/testing/selftests/vDSO/vdso_test_chacha.c
+93
-10
tools/testing/selftests/vDSO/vdso_test_chacha.c
···
3
3
* Copyright (C) 2022-2024 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4
4
*/
5
5
6
-
#include <sodium/crypto_stream_chacha20.h>
6
+
#include <tools/le_byteshift.h>
7
7
#include <sys/random.h>
8
+
#include <sys/auxv.h>
8
9
#include <string.h>
9
10
#include <stdint.h>
11
+
#include <stdbool.h>
10
12
#include "../kselftest.h"
11
13
12
-
extern void __arch_chacha20_blocks_nostack(uint8_t *dst_bytes, const uint8_t *key, uint32_t *counter, size_t nblocks);
14
+
#if defined(__aarch64__)
15
+
static bool cpu_has_capabilities(void)
16
+
{
17
+
return getauxval(AT_HWCAP) & HWCAP_ASIMD;
18
+
}
19
+
#elif defined(__s390x__)
20
+
static bool cpu_has_capabilities(void)
21
+
{
22
+
return getauxval(AT_HWCAP) & HWCAP_S390_VXRS;
23
+
}
24
+
#else
25
+
static bool cpu_has_capabilities(void)
26
+
{
27
+
return true;
28
+
}
29
+
#endif
30
+
31
+
static uint32_t rol32(uint32_t word, unsigned int shift)
32
+
{
33
+
return (word << (shift & 31)) | (word >> ((-shift) & 31));
34
+
}
35
+
36
+
static void reference_chacha20_blocks(uint8_t *dst_bytes, const uint32_t *key, uint32_t *counter, size_t nblocks)
37
+
{
38
+
uint32_t s[16] = {
39
+
0x61707865U, 0x3320646eU, 0x79622d32U, 0x6b206574U,
40
+
key[0], key[1], key[2], key[3], key[4], key[5], key[6], key[7],
41
+
counter[0], counter[1], 0, 0
42
+
};
43
+
44
+
while (nblocks--) {
45
+
uint32_t x[16];
46
+
memcpy(x, s, sizeof(x));
47
+
for (unsigned int r = 0; r < 20; r += 2) {
48
+
#define QR(a, b, c, d) ( \
49
+
x[a] += x[b], \
50
+
x[d] = rol32(x[d] ^ x[a], 16), \
51
+
x[c] += x[d], \
52
+
x[b] = rol32(x[b] ^ x[c], 12), \
53
+
x[a] += x[b], \
54
+
x[d] = rol32(x[d] ^ x[a], 8), \
55
+
x[c] += x[d], \
56
+
x[b] = rol32(x[b] ^ x[c], 7))
57
+
58
+
QR(0, 4, 8, 12);
59
+
QR(1, 5, 9, 13);
60
+
QR(2, 6, 10, 14);
61
+
QR(3, 7, 11, 15);
62
+
QR(0, 5, 10, 15);
63
+
QR(1, 6, 11, 12);
64
+
QR(2, 7, 8, 13);
65
+
QR(3, 4, 9, 14);
66
+
}
67
+
for (unsigned int i = 0; i < 16; ++i, dst_bytes += sizeof(uint32_t))
68
+
put_unaligned_le32(x[i] + s[i], dst_bytes);
69
+
if (!++s[12])
70
+
++s[13];
71
+
}
72
+
counter[0] = s[12];
73
+
counter[1] = s[13];
74
+
}
75
+
76
+
typedef uint8_t u8;
77
+
typedef uint32_t u32;
78
+
typedef uint64_t u64;
79
+
#include <vdso/getrandom.h>
13
80
14
81
int main(int argc, char *argv[])
15
82
{
16
83
enum { TRIALS = 1000, BLOCKS = 128, BLOCK_SIZE = 64 };
17
-
static const uint8_t nonce[8] = { 0 };
18
-
uint32_t counter[2];
19
-
uint8_t key[32];
84
+
uint32_t key[8], counter1[2], counter2[2];
20
85
uint8_t output1[BLOCK_SIZE * BLOCKS], output2[BLOCK_SIZE * BLOCKS];
21
86
22
87
ksft_print_header();
88
+
if (!cpu_has_capabilities())
89
+
ksft_exit_skip("Required CPU capabilities missing\n");
23
90
ksft_set_plan(1);
24
91
25
92
for (unsigned int trial = 0; trial < TRIALS; ++trial) {
···
94
27
printf("getrandom() failed!\n");
95
28
return KSFT_SKIP;
96
29
}
97
-
crypto_stream_chacha20(output1, sizeof(output1), nonce, key);
30
+
memset(counter1, 0, sizeof(counter1));
31
+
reference_chacha20_blocks(output1, key, counter1, BLOCKS);
98
32
for (unsigned int split = 0; split < BLOCKS; ++split) {
99
33
memset(output2, 'X', sizeof(output2));
100
-
memset(counter, 0, sizeof(counter));
34
+
memset(counter2, 0, sizeof(counter2));
101
35
if (split)
102
-
__arch_chacha20_blocks_nostack(output2, key, counter, split);
103
-
__arch_chacha20_blocks_nostack(output2 + split * BLOCK_SIZE, key, counter, BLOCKS - split);
104
-
if (memcmp(output1, output2, sizeof(output1)))
36
+
__arch_chacha20_blocks_nostack(output2, key, counter2, split);
37
+
__arch_chacha20_blocks_nostack(output2 + split * BLOCK_SIZE, key, counter2, BLOCKS - split);
38
+
if (memcmp(output1, output2, sizeof(output1)) || memcmp(counter1, counter2, sizeof(counter1)))
105
39
return KSFT_FAIL;
106
40
}
107
41
}
42
+
memset(counter1, 0, sizeof(counter1));
43
+
counter1[0] = (uint32_t)-BLOCKS + 2;
44
+
memset(counter2, 0, sizeof(counter2));
45
+
counter2[0] = (uint32_t)-BLOCKS + 2;
46
+
47
+
reference_chacha20_blocks(output1, key, counter1, BLOCKS);
48
+
__arch_chacha20_blocks_nostack(output2, key, counter2, BLOCKS);
49
+
if (memcmp(output1, output2, sizeof(output1)) || memcmp(counter1, counter2, sizeof(counter1)))
50
+
return KSFT_FAIL;
51
+
52
+
reference_chacha20_blocks(output1, key, counter1, BLOCKS);
53
+
__arch_chacha20_blocks_nostack(output2, key, counter2, BLOCKS);
54
+
if (memcmp(output1, output2, sizeof(output1)) || memcmp(counter1, counter2, sizeof(counter1)))
55
+
return KSFT_FAIL;
56
+
108
57
ksft_test_result_pass("chacha: PASS\n");
109
58
return KSFT_PASS;
110
59
}
+14
-7
tools/testing/selftests/vDSO/vdso_test_correctness.c
+14
-7
tools/testing/selftests/vDSO/vdso_test_correctness.c
···
20
20
#include <limits.h>
21
21
22
22
#include "vdso_config.h"
23
+
#include "vdso_call.h"
23
24
#include "../kselftest.h"
24
25
25
26
static const char **name;
···
115
114
if (!vdso)
116
115
vdso = dlopen("linux-gate.so.1",
117
116
RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);
117
+
if (!vdso)
118
+
vdso = dlopen("linux-vdso32.so.1",
119
+
RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);
120
+
if (!vdso)
121
+
vdso = dlopen("linux-vdso64.so.1",
122
+
RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);
118
123
if (!vdso) {
119
124
printf("[WARN]\tfailed to find vDSO\n");
120
125
return;
···
187
180
188
181
ret_sys = sys_getcpu(&cpu_sys, &node_sys, 0);
189
182
if (vdso_getcpu)
190
-
ret_vdso = vdso_getcpu(&cpu_vdso, &node_vdso, 0);
183
+
ret_vdso = VDSO_CALL(vdso_getcpu, 3, &cpu_vdso, &node_vdso, 0);
191
184
if (vgetcpu)
192
185
ret_vsys = vgetcpu(&cpu_vsys, &node_vsys, 0);
193
186
···
270
263
271
264
if (sys_clock_gettime(clock, &start) < 0) {
272
265
if (errno == EINVAL) {
273
-
vdso_ret = vdso_clock_gettime(clock, &vdso);
266
+
vdso_ret = VDSO_CALL(vdso_clock_gettime, 2, clock, &vdso);
274
267
if (vdso_ret == -EINVAL) {
275
268
printf("[OK]\tNo such clock.\n");
276
269
} else {
···
283
276
return;
284
277
}
285
278
286
-
vdso_ret = vdso_clock_gettime(clock, &vdso);
279
+
vdso_ret = VDSO_CALL(vdso_clock_gettime, 2, clock, &vdso);
287
280
end_ret = sys_clock_gettime(clock, &end);
288
281
289
282
if (vdso_ret != 0 || end_ret != 0) {
···
332
325
333
326
if (sys_clock_gettime64(clock, &start) < 0) {
334
327
if (errno == EINVAL) {
335
-
vdso_ret = vdso_clock_gettime64(clock, &vdso);
328
+
vdso_ret = VDSO_CALL(vdso_clock_gettime64, 2, clock, &vdso);
336
329
if (vdso_ret == -EINVAL) {
337
330
printf("[OK]\tNo such clock.\n");
338
331
} else {
···
345
338
return;
346
339
}
347
340
348
-
vdso_ret = vdso_clock_gettime64(clock, &vdso);
341
+
vdso_ret = VDSO_CALL(vdso_clock_gettime64, 2, clock, &vdso);
349
342
end_ret = sys_clock_gettime64(clock, &end);
350
343
351
344
if (vdso_ret != 0 || end_ret != 0) {
···
402
395
return;
403
396
}
404
397
405
-
vdso_ret = vdso_gettimeofday(&vdso, &vdso_tz);
398
+
vdso_ret = VDSO_CALL(vdso_gettimeofday, 2, &vdso, &vdso_tz);
406
399
end_ret = sys_gettimeofday(&end, NULL);
407
400
408
401
if (vdso_ret != 0 || end_ret != 0) {
···
432
425
}
433
426
434
427
/* And make sure that passing NULL for tz doesn't crash. */
435
-
vdso_gettimeofday(&vdso, NULL);
428
+
VDSO_CALL(vdso_gettimeofday, 2, &vdso, NULL);
436
429
}
437
430
438
431
int main(int argc, char **argv)
+2
-1
tools/testing/selftests/vDSO/vdso_test_getcpu.c
+2
-1
tools/testing/selftests/vDSO/vdso_test_getcpu.c
···
14
14
#include "../kselftest.h"
15
15
#include "parse_vdso.h"
16
16
#include "vdso_config.h"
17
+
#include "vdso_call.h"
17
18
18
19
struct getcpu_cache;
19
20
typedef long (*getcpu_t)(unsigned int *, unsigned int *,
···
43
42
return KSFT_SKIP;
44
43
}
45
44
46
-
ret = get_cpu(&cpu, &node, 0);
45
+
ret = VDSO_CALL(get_cpu, 3, &cpu, &node, 0);
47
46
if (ret == 0) {
48
47
printf("Running on CPU %u node %u\n", cpu, node);
49
48
} else {
+83
-45
tools/testing/selftests/vDSO/vdso_test_getrandom.c
+83
-45
tools/testing/selftests/vDSO/vdso_test_getrandom.c
···
16
16
#include <sys/mman.h>
17
17
#include <sys/random.h>
18
18
#include <sys/syscall.h>
19
+
#include <sys/ptrace.h>
20
+
#include <sys/wait.h>
19
21
#include <sys/types.h>
20
22
#include <linux/random.h>
23
+
#include <linux/compiler.h>
24
+
#include <linux/ptrace.h>
21
25
22
26
#include "../kselftest.h"
23
27
#include "parse_vdso.h"
28
+
#include "vdso_config.h"
29
+
#include "vdso_call.h"
24
30
25
31
#ifndef timespecsub
26
32
#define timespecsub(tsp, usp, vsp) \
···
44
38
pthread_mutex_t lock;
45
39
void **states;
46
40
size_t len, cap;
47
-
} grnd_allocator = {
48
-
.lock = PTHREAD_MUTEX_INITIALIZER
49
-
};
50
-
51
-
static struct {
52
41
ssize_t(*fn)(void *, size_t, unsigned long, void *, size_t);
53
-
pthread_key_t key;
54
-
pthread_once_t initialized;
55
42
struct vgetrandom_opaque_params params;
56
-
} grnd_ctx = {
57
-
.initialized = PTHREAD_ONCE_INIT
43
+
} vgrnd = {
44
+
.lock = PTHREAD_MUTEX_INITIALIZER
58
45
};
59
46
60
47
static void *vgetrandom_get_state(void)
61
48
{
62
49
void *state = NULL;
63
50
64
-
pthread_mutex_lock(&grnd_allocator.lock);
65
-
if (!grnd_allocator.len) {
51
+
pthread_mutex_lock(&vgrnd.lock);
52
+
if (!vgrnd.len) {
66
53
size_t page_size = getpagesize();
67
54
size_t new_cap;
68
55
size_t alloc_size, num = sysconf(_SC_NPROCESSORS_ONLN); /* Just a decent heuristic. */
69
56
void *new_block, *new_states;
70
57
71
-
alloc_size = (num * grnd_ctx.params.size_of_opaque_state + page_size - 1) & (~(page_size - 1));
72
-
num = (page_size / grnd_ctx.params.size_of_opaque_state) * (alloc_size / page_size);
73
-
new_block = mmap(0, alloc_size, grnd_ctx.params.mmap_prot, grnd_ctx.params.mmap_flags, -1, 0);
58
+
alloc_size = (num * vgrnd.params.size_of_opaque_state + page_size - 1) & (~(page_size - 1));
59
+
num = (page_size / vgrnd.params.size_of_opaque_state) * (alloc_size / page_size);
60
+
new_block = mmap(0, alloc_size, vgrnd.params.mmap_prot, vgrnd.params.mmap_flags, -1, 0);
74
61
if (new_block == MAP_FAILED)
75
62
goto out;
76
63
77
-
new_cap = grnd_allocator.cap + num;
78
-
new_states = reallocarray(grnd_allocator.states, new_cap, sizeof(*grnd_allocator.states));
64
+
new_cap = vgrnd.cap + num;
65
+
new_states = reallocarray(vgrnd.states, new_cap, sizeof(*vgrnd.states));
79
66
if (!new_states)
80
67
goto unmap;
81
-
grnd_allocator.cap = new_cap;
82
-
grnd_allocator.states = new_states;
68
+
vgrnd.cap = new_cap;
69
+
vgrnd.states = new_states;
83
70
84
71
for (size_t i = 0; i < num; ++i) {
85
-
if (((uintptr_t)new_block & (page_size - 1)) + grnd_ctx.params.size_of_opaque_state > page_size)
72
+
if (((uintptr_t)new_block & (page_size - 1)) + vgrnd.params.size_of_opaque_state > page_size)
86
73
new_block = (void *)(((uintptr_t)new_block + page_size - 1) & (~(page_size - 1)));
87
-
grnd_allocator.states[i] = new_block;
88
-
new_block += grnd_ctx.params.size_of_opaque_state;
74
+
vgrnd.states[i] = new_block;
75
+
new_block += vgrnd.params.size_of_opaque_state;
89
76
}
90
-
grnd_allocator.len = num;
77
+
vgrnd.len = num;
91
78
goto success;
92
79
93
80
unmap:
···
88
89
goto out;
89
90
}
90
91
success:
91
-
state = grnd_allocator.states[--grnd_allocator.len];
92
+
state = vgrnd.states[--vgrnd.len];
92
93
93
94
out:
94
-
pthread_mutex_unlock(&grnd_allocator.lock);
95
+
pthread_mutex_unlock(&vgrnd.lock);
95
96
return state;
96
97
}
97
98
···
99
100
{
100
101
if (!state)
101
102
return;
102
-
pthread_mutex_lock(&grnd_allocator.lock);
103
-
grnd_allocator.states[grnd_allocator.len++] = state;
104
-
pthread_mutex_unlock(&grnd_allocator.lock);
103
+
pthread_mutex_lock(&vgrnd.lock);
104
+
vgrnd.states[vgrnd.len++] = state;
105
+
pthread_mutex_unlock(&vgrnd.lock);
105
106
}
106
107
107
108
static void vgetrandom_init(void)
108
109
{
109
-
if (pthread_key_create(&grnd_ctx.key, vgetrandom_put_state) != 0)
110
-
return;
110
+
const char *version = versions[VDSO_VERSION];
111
+
const char *name = names[VDSO_NAMES][6];
111
112
unsigned long sysinfo_ehdr = getauxval(AT_SYSINFO_EHDR);
113
+
size_t ret;
114
+
112
115
if (!sysinfo_ehdr) {
113
116
printf("AT_SYSINFO_EHDR is not present!\n");
114
117
exit(KSFT_SKIP);
115
118
}
116
119
vdso_init_from_sysinfo_ehdr(sysinfo_ehdr);
117
-
grnd_ctx.fn = (__typeof__(grnd_ctx.fn))vdso_sym("LINUX_2.6", "__vdso_getrandom");
118
-
if (!grnd_ctx.fn) {
119
-
printf("__vdso_getrandom is missing!\n");
120
+
vgrnd.fn = (__typeof__(vgrnd.fn))vdso_sym(version, name);
121
+
if (!vgrnd.fn) {
122
+
printf("%s is missing!\n", name);
120
123
exit(KSFT_FAIL);
121
124
}
122
-
if (grnd_ctx.fn(NULL, 0, 0, &grnd_ctx.params, ~0UL) != 0) {
125
+
ret = VDSO_CALL(vgrnd.fn, 5, NULL, 0, 0, &vgrnd.params, ~0UL);
126
+
if (ret == -ENOSYS) {
127
+
printf("unsupported architecture\n");
128
+
exit(KSFT_SKIP);
129
+
} else if (ret) {
123
130
printf("failed to fetch vgetrandom params!\n");
124
131
exit(KSFT_FAIL);
125
132
}
···
133
128
134
129
static ssize_t vgetrandom(void *buf, size_t len, unsigned long flags)
135
130
{
136
-
void *state;
131
+
static __thread void *state;
137
132
138
-
pthread_once(&grnd_ctx.initialized, vgetrandom_init);
139
-
state = pthread_getspecific(grnd_ctx.key);
140
133
if (!state) {
141
134
state = vgetrandom_get_state();
142
-
if (pthread_setspecific(grnd_ctx.key, state) != 0) {
143
-
vgetrandom_put_state(state);
144
-
state = NULL;
145
-
}
146
135
if (!state) {
147
136
printf("vgetrandom_get_state failed!\n");
148
137
exit(KSFT_FAIL);
149
138
}
150
139
}
151
-
return grnd_ctx.fn(buf, len, flags, state, grnd_ctx.params.size_of_opaque_state);
140
+
return VDSO_CALL(vgrnd.fn, 5, buf, len, flags, state, vgrnd.params.size_of_opaque_state);
152
141
}
153
142
154
143
enum { TRIALS = 25000000, THREADS = 256 };
155
144
156
-
static void *test_vdso_getrandom(void *)
145
+
static void *test_vdso_getrandom(void *ctx)
157
146
{
158
147
for (size_t i = 0; i < TRIALS; ++i) {
159
148
unsigned int val;
···
157
158
return NULL;
158
159
}
159
160
160
-
static void *test_libc_getrandom(void *)
161
+
static void *test_libc_getrandom(void *ctx)
161
162
{
162
163
for (size_t i = 0; i < TRIALS; ++i) {
163
164
unsigned int val;
···
167
168
return NULL;
168
169
}
169
170
170
-
static void *test_syscall_getrandom(void *)
171
+
static void *test_syscall_getrandom(void *ctx)
171
172
{
172
173
for (size_t i = 0; i < TRIALS; ++i) {
173
174
unsigned int val;
···
243
244
static void kselftest(void)
244
245
{
245
246
uint8_t weird_size[1263];
247
+
pid_t child;
246
248
247
249
ksft_print_header();
248
-
ksft_set_plan(1);
250
+
ksft_set_plan(2);
249
251
250
252
for (size_t i = 0; i < 1000; ++i) {
251
253
ssize_t ret = vgetrandom(weird_size, sizeof(weird_size), 0);
···
255
255
}
256
256
257
257
ksft_test_result_pass("getrandom: PASS\n");
258
+
259
+
unshare(CLONE_NEWUSER);
260
+
assert(unshare(CLONE_NEWTIME) == 0);
261
+
child = fork();
262
+
assert(child >= 0);
263
+
if (!child) {
264
+
vgetrandom_init();
265
+
child = getpid();
266
+
assert(ptrace(PTRACE_TRACEME, 0, NULL, NULL) == 0);
267
+
assert(kill(child, SIGSTOP) == 0);
268
+
assert(vgetrandom(weird_size, sizeof(weird_size), 0) == sizeof(weird_size));
269
+
_exit(0);
270
+
}
271
+
for (;;) {
272
+
struct ptrace_syscall_info info = { 0 };
273
+
int status, ret;
274
+
assert(waitpid(child, &status, 0) >= 0);
275
+
if (WIFEXITED(status)) {
276
+
if (WEXITSTATUS(status) != 0)
277
+
exit(KSFT_FAIL);
278
+
break;
279
+
}
280
+
assert(WIFSTOPPED(status));
281
+
if (WSTOPSIG(status) == SIGSTOP)
282
+
assert(ptrace(PTRACE_SETOPTIONS, child, 0, PTRACE_O_TRACESYSGOOD) == 0);
283
+
else if (WSTOPSIG(status) == (SIGTRAP | 0x80)) {
284
+
assert(ptrace(PTRACE_GET_SYSCALL_INFO, child, sizeof(info), &info) > 0);
285
+
if (info.op == PTRACE_SYSCALL_INFO_ENTRY && info.entry.nr == __NR_getrandom &&
286
+
info.entry.args[0] == (uintptr_t)weird_size && info.entry.args[1] == sizeof(weird_size))
287
+
exit(KSFT_FAIL);
288
+
}
289
+
assert(ptrace(PTRACE_SYSCALL, child, 0, 0) == 0);
290
+
}
291
+
292
+
ksft_test_result_pass("getrandom timens: PASS\n");
293
+
258
294
exit(KSFT_PASS);
259
295
}
260
296
···
301
265
302
266
int main(int argc, char *argv[])
303
267
{
268
+
vgetrandom_init();
269
+
304
270
if (argc == 1) {
305
271
kselftest();
306
272
return 0;
+2
-1
tools/testing/selftests/vDSO/vdso_test_gettimeofday.c
+2
-1
tools/testing/selftests/vDSO/vdso_test_gettimeofday.c
···
19
19
#include "../kselftest.h"
20
20
#include "parse_vdso.h"
21
21
#include "vdso_config.h"
22
+
#include "vdso_call.h"
22
23
23
24
int main(int argc, char **argv)
24
25
{
···
44
43
}
45
44
46
45
struct timeval tv;
47
-
long ret = gtod(&tv, 0);
46
+
long ret = VDSO_CALL(gtod, 2, &tv, 0);
48
47
49
48
if (ret == 0) {
50
49
printf("The time is %lld.%06lld\n",