Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
powerpc/vdso: Wire up getrandom() vDSO implementation on VDSO64
Extend getrandom() vDSO implementation to VDSO64.
Tested on QEMU on both ppc64_defconfig and ppc64le_defconfig.
Results from a Power9 (PowerNV):
~ # ./vdso_test_getrandom bench-single
vdso: 25000000 times in 0.787943615 seconds
libc: 25000000 times in 14.101887252 seconds
syscall: 25000000 times in 14.047475082 seconds
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Tested-by: Madhavan Srinivasan <maddy@linux.ibm.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
authored by
Christophe Leroy
and committed by
Jason A. Donenfeld
8072b39c
53cee505
+69
-3
+1
-1
arch/powerpc/Kconfig
+1
-1
arch/powerpc/Kconfig
+6
-2
arch/powerpc/kernel/vdso/Makefile
+6
-2
arch/powerpc/kernel/vdso/Makefile
···
9
9
obj-vdso64 = sigtramp64-64.o gettimeofday-64.o datapage-64.o cacheflush-64.o note-64.o getcpu-64.o
10
10
11
11
obj-vdso32 += getrandom-32.o vgetrandom-chacha-32.o
12
+
obj-vdso64 += getrandom-64.o vgetrandom-chacha-64.o
12
13
13
14
ifneq ($(c-gettimeofday-y),)
14
15
CFLAGS_vgettimeofday-32.o += -include $(c-gettimeofday-y)
···
22
21
23
22
ifneq ($(c-getrandom-y),)
24
23
CFLAGS_vgetrandom-32.o += -include $(c-getrandom-y)
24
+
CFLAGS_vgetrandom-64.o += -include $(c-getrandom-y) $(call cc-option, -ffixed-r30)
25
25
endif
26
26
27
27
# Build rules
···
36
34
targets := $(obj-vdso32) vdso32.so.dbg vgettimeofday-32.o vgetrandom-32.o
37
35
targets += crtsavres-32.o
38
36
obj-vdso32 := $(addprefix $(obj)/, $(obj-vdso32))
39
-
targets += $(obj-vdso64) vdso64.so.dbg vgettimeofday-64.o
37
+
targets += $(obj-vdso64) vdso64.so.dbg vgettimeofday-64.o vgetrandom-64.o
40
38
obj-vdso64 := $(addprefix $(obj)/, $(obj-vdso64))
41
39
42
40
ccflags-y := -fno-common -fno-builtin -DBUILD_VDSO
···
73
71
# link rule for the .so file, .lds has to be first
74
72
$(obj)/vdso32.so.dbg: $(obj)/vdso32.lds $(obj-vdso32) $(obj)/vgettimeofday-32.o $(obj)/vgetrandom-32.o $(obj)/crtsavres-32.o FORCE
75
73
$(call if_changed,vdso32ld_and_check)
76
-
$(obj)/vdso64.so.dbg: $(obj)/vdso64.lds $(obj-vdso64) $(obj)/vgettimeofday-64.o FORCE
74
+
$(obj)/vdso64.so.dbg: $(obj)/vdso64.lds $(obj-vdso64) $(obj)/vgettimeofday-64.o $(obj)/vgetrandom-64.o FORCE
77
75
$(call if_changed,vdso64ld_and_check)
78
76
79
77
# assembly rules for the .S files
···
88
86
$(obj-vdso64): %-64.o: %.S FORCE
89
87
$(call if_changed_dep,vdso64as)
90
88
$(obj)/vgettimeofday-64.o: %-64.o: %.c FORCE
89
+
$(call if_changed_dep,cc_o_c)
90
+
$(obj)/vgetrandom-64.o: %-64.o: %.c FORCE
91
91
$(call if_changed_dep,cc_o_c)
92
92
93
93
# Generate VDSO offsets using helper script
+8
arch/powerpc/kernel/vdso/getrandom.S
+8
arch/powerpc/kernel/vdso/getrandom.S
···
27
27
.cfi_adjust_cfa_offset PPC_MIN_STKFRM
28
28
PPC_STL r0, PPC_MIN_STKFRM + PPC_LR_STKOFF(r1)
29
29
.cfi_rel_offset lr, PPC_MIN_STKFRM + PPC_LR_STKOFF
30
+
#ifdef __powerpc64__
31
+
PPC_STL r2, PPC_MIN_STKFRM + STK_GOT(r1)
32
+
.cfi_rel_offset r2, PPC_MIN_STKFRM + STK_GOT
33
+
#endif
30
34
get_realdatapage r8, r11
31
35
addi r8, r8, VDSO_RNG_DATA_OFFSET
32
36
bl CFUNC(DOTSYM(\funct))
33
37
PPC_LL r0, PPC_MIN_STKFRM + PPC_LR_STKOFF(r1)
38
+
#ifdef __powerpc64__
39
+
PPC_LL r2, PPC_MIN_STKFRM + STK_GOT(r1)
40
+
.cfi_restore r2
41
+
#endif
34
42
cmpwi r3, 0
35
43
mtlr r0
36
44
addi r1, r1, 2 * PPC_MIN_STKFRM
+1
arch/powerpc/kernel/vdso/vdso64.lds.S
+1
arch/powerpc/kernel/vdso/vdso64.lds.S
+53
arch/powerpc/kernel/vdso/vgetrandom-chacha.S
+53
arch/powerpc/kernel/vdso/vgetrandom-chacha.S
···
124
124
*/
125
125
SYM_FUNC_START(__arch_chacha20_blocks_nostack)
126
126
#ifdef __powerpc64__
127
+
std counter, -216(r1)
128
+
129
+
std r14, -144(r1)
130
+
std r15, -136(r1)
131
+
std r16, -128(r1)
132
+
std r17, -120(r1)
133
+
std r18, -112(r1)
134
+
std r19, -104(r1)
135
+
std r20, -96(r1)
136
+
std r21, -88(r1)
137
+
std r22, -80(r1)
138
+
std r23, -72(r1)
139
+
std r24, -64(r1)
140
+
std r25, -56(r1)
141
+
std r26, -48(r1)
142
+
std r27, -40(r1)
143
+
std r28, -32(r1)
144
+
std r29, -24(r1)
145
+
std r30, -16(r1)
146
+
std r31, -8(r1)
127
147
#else
128
148
stwu r1, -96(r1)
129
149
stw counter, 20(r1)
···
169
149
stw r30, 88(r1)
170
150
stw r31, 92(r1)
171
151
#endif
152
+
#endif /* __powerpc64__ */
172
153
173
154
lwz counter0, 0(counter)
174
155
lwz counter1, 4(counter)
156
+
#ifdef __powerpc64__
157
+
rldimi counter0, counter1, 32, 0
158
+
#endif
175
159
mr idx_r0, nblocks
176
160
subi dst_bytes, dst_bytes, 4
177
161
···
291
267
292
268
subic. idx_r0, idx_r0, 1 /* subi. can't use r0 as source */
293
269
270
+
#ifdef __powerpc64__
271
+
addi counter0, counter0, 1
272
+
srdi counter1, counter0, 32
273
+
#else
294
274
addic counter0, counter0, 1
295
275
addze counter1, counter1
276
+
#endif
296
277
297
278
bne .Lblock
298
279
280
+
#ifdef __powerpc64__
281
+
ld counter, -216(r1)
282
+
#else
299
283
lwz counter, 20(r1)
284
+
#endif
300
285
stw counter0, 0(counter)
301
286
stw counter1, 4(counter)
302
287
···
317
284
li r11, 0
318
285
li r12, 0
319
286
287
+
#ifdef __powerpc64__
288
+
ld r14, -144(r1)
289
+
ld r15, -136(r1)
290
+
ld r16, -128(r1)
291
+
ld r17, -120(r1)
292
+
ld r18, -112(r1)
293
+
ld r19, -104(r1)
294
+
ld r20, -96(r1)
295
+
ld r21, -88(r1)
296
+
ld r22, -80(r1)
297
+
ld r23, -72(r1)
298
+
ld r24, -64(r1)
299
+
ld r25, -56(r1)
300
+
ld r26, -48(r1)
301
+
ld r27, -40(r1)
302
+
ld r28, -32(r1)
303
+
ld r29, -24(r1)
304
+
ld r30, -16(r1)
305
+
ld r31, -8(r1)
306
+
#else
320
307
#ifdef __BIG_ENDIAN__
321
308
lmw r14, 24(r1)
322
309
#else