Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
x86/uaccess: Don't jump between functions
For unwinding sanity, a function shouldn't jump to the middle of another
function. Move the short string user copy code out to a separate
non-function code snippet.
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/9519e4853148b765e047967708f2b61e56c93186.1649718562.git.jpoimboe@redhat.com
authored by
Josh Poimboeuf
and committed by
Peter Zijlstra
02041b32
b2d229d4
+52
-35
+52
-35
arch/x86/lib/copy_user_64.S
+52
-35
arch/x86/lib/copy_user_64.S
···
53
53
SYM_FUNC_START(copy_user_generic_unrolled)
54
54
ASM_STAC
55
55
cmpl $8,%edx
56
-
jb 20f /* less then 8 bytes, go to byte copy loop */
56
+
jb .Lcopy_user_short_string_bytes
57
57
ALIGN_DESTINATION
58
58
movl %edx,%ecx
59
59
andl $63,%edx
60
60
shrl $6,%ecx
61
-
jz .L_copy_short_string
61
+
jz copy_user_short_string
62
62
1: movq (%rsi),%r8
63
63
2: movq 1*8(%rsi),%r9
64
64
3: movq 2*8(%rsi),%r10
···
79
79
leaq 64(%rdi),%rdi
80
80
decl %ecx
81
81
jnz 1b
82
-
.L_copy_short_string:
83
-
movl %edx,%ecx
84
-
andl $7,%edx
85
-
shrl $3,%ecx
86
-
jz 20f
87
-
18: movq (%rsi),%r8
88
-
19: movq %r8,(%rdi)
89
-
leaq 8(%rsi),%rsi
90
-
leaq 8(%rdi),%rdi
91
-
decl %ecx
92
-
jnz 18b
93
-
20: andl %edx,%edx
94
-
jz 23f
95
-
movl %edx,%ecx
96
-
21: movb (%rsi),%al
97
-
22: movb %al,(%rdi)
98
-
incq %rsi
99
-
incq %rdi
100
-
decl %ecx
101
-
jnz 21b
102
-
23: xor %eax,%eax
103
-
ASM_CLAC
104
-
RET
82
+
jmp copy_user_short_string
105
83
106
84
30: shll $6,%ecx
107
85
addl %ecx,%edx
108
-
jmp 60f
109
-
40: leal (%rdx,%rcx,8),%edx
110
-
jmp 60f
111
-
50: movl %ecx,%edx
112
-
60: jmp .Lcopy_user_handle_tail /* ecx is zerorest also */
86
+
jmp .Lcopy_user_handle_tail
113
87
114
88
_ASM_EXTABLE_CPY(1b, 30b)
115
89
_ASM_EXTABLE_CPY(2b, 30b)
···
101
127
_ASM_EXTABLE_CPY(14b, 30b)
102
128
_ASM_EXTABLE_CPY(15b, 30b)
103
129
_ASM_EXTABLE_CPY(16b, 30b)
104
-
_ASM_EXTABLE_CPY(18b, 40b)
105
-
_ASM_EXTABLE_CPY(19b, 40b)
106
-
_ASM_EXTABLE_CPY(21b, 50b)
107
-
_ASM_EXTABLE_CPY(22b, 50b)
108
130
SYM_FUNC_END(copy_user_generic_unrolled)
109
131
EXPORT_SYMBOL(copy_user_generic_unrolled)
110
132
···
161
191
SYM_FUNC_START(copy_user_enhanced_fast_string)
162
192
ASM_STAC
163
193
/* CPUs without FSRM should avoid rep movsb for short copies */
164
-
ALTERNATIVE "cmpl $64, %edx; jb .L_copy_short_string", "", X86_FEATURE_FSRM
194
+
ALTERNATIVE "cmpl $64, %edx; jb copy_user_short_string", "", X86_FEATURE_FSRM
165
195
movl %edx,%ecx
166
196
1: rep movsb
167
197
xorl %eax,%eax
···
212
242
jmp .Lcopy_user_handle_tail
213
243
214
244
SYM_CODE_END(.Lcopy_user_handle_tail)
245
+
246
+
/*
247
+
* Finish memcpy of less than 64 bytes. #AC should already be set.
248
+
*
249
+
* Input:
250
+
* rdi destination
251
+
* rsi source
252
+
* rdx count (< 64)
253
+
*
254
+
* Output:
255
+
* eax uncopied bytes or 0 if successful.
256
+
*/
257
+
SYM_CODE_START_LOCAL(copy_user_short_string)
258
+
movl %edx,%ecx
259
+
andl $7,%edx
260
+
shrl $3,%ecx
261
+
jz .Lcopy_user_short_string_bytes
262
+
18: movq (%rsi),%r8
263
+
19: movq %r8,(%rdi)
264
+
leaq 8(%rsi),%rsi
265
+
leaq 8(%rdi),%rdi
266
+
decl %ecx
267
+
jnz 18b
268
+
.Lcopy_user_short_string_bytes:
269
+
andl %edx,%edx
270
+
jz 23f
271
+
movl %edx,%ecx
272
+
21: movb (%rsi),%al
273
+
22: movb %al,(%rdi)
274
+
incq %rsi
275
+
incq %rdi
276
+
decl %ecx
277
+
jnz 21b
278
+
23: xor %eax,%eax
279
+
ASM_CLAC
280
+
RET
281
+
282
+
40: leal (%rdx,%rcx,8),%edx
283
+
jmp 60f
284
+
50: movl %ecx,%edx /* ecx is zerorest also */
285
+
60: jmp .Lcopy_user_handle_tail
286
+
287
+
_ASM_EXTABLE_CPY(18b, 40b)
288
+
_ASM_EXTABLE_CPY(19b, 40b)
289
+
_ASM_EXTABLE_CPY(21b, 50b)
290
+
_ASM_EXTABLE_CPY(22b, 50b)
291
+
SYM_CODE_END(copy_user_short_string)
215
292
216
293
/*
217
294
* copy_user_nocache - Uncached memory copy with exception handling