Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
arm64: futex: Support futex with FEAT_LSUI
Current futex atomic operations are implemented using LL/SC instructions
while temporarily clearing PSTATE.PAN and setting PSTATE.TCO (if
KASAN_HW_TAGS is enabled). With Armv9.6, FEAT_LSUI provides atomic
instructions for user memory access in the kernel without the need for
PSTATE bits toggling.
Use the FEAT_LSUI instructions to implement the futex atomic operations.
Note that some futex operations do not have a matching LSUI instruction,
(eor or word-sized cmpxchg). For such cases, use cas{al}t to implement
the operation.
Signed-off-by: Yeoreum Yun <yeoreum.yun@arm.com>
[catalin.marinas@arm.com: add comment on -EAGAIN in __lsui_futex_cmpxchg()]
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
authored by
Yeoreum Yun
and committed by
Catalin Marinas
44adf2bf
eaa3babc
+185
-2
+158
-2
arch/arm64/include/asm/futex.h
+158
-2
arch/arm64/include/asm/futex.h
···
9
9
#include <linux/uaccess.h>
10
10
11
11
#include <asm/errno.h>
12
+
#include <asm/lsui.h>
12
13
13
14
#define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of? */
14
15
···
90
89
return ret;
91
90
}
92
91
92
+
#ifdef CONFIG_ARM64_LSUI
93
+
94
+
/*
95
+
* Wrap LSUI instructions with uaccess_ttbr0_enable()/disable(), as
96
+
* PAN toggling is not required.
97
+
*/
98
+
99
+
#define LSUI_FUTEX_ATOMIC_OP(op, asm_op) \
100
+
static __always_inline int \
101
+
__lsui_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \
102
+
{ \
103
+
int ret = 0; \
104
+
int oldval; \
105
+
\
106
+
uaccess_ttbr0_enable(); \
107
+
\
108
+
asm volatile("// __lsui_futex_atomic_" #op "\n" \
109
+
__LSUI_PREAMBLE \
110
+
"1: " #asm_op "al %w[oparg], %w[oldval], %[uaddr]\n" \
111
+
"2:\n" \
112
+
_ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret]) \
113
+
: [ret] "+r" (ret), [uaddr] "+Q" (*uaddr), \
114
+
[oldval] "=r" (oldval) \
115
+
: [oparg] "r" (oparg) \
116
+
: "memory"); \
117
+
\
118
+
uaccess_ttbr0_disable(); \
119
+
\
120
+
if (!ret) \
121
+
*oval = oldval; \
122
+
return ret; \
123
+
}
124
+
125
+
LSUI_FUTEX_ATOMIC_OP(add, ldtadd)
126
+
LSUI_FUTEX_ATOMIC_OP(or, ldtset)
127
+
LSUI_FUTEX_ATOMIC_OP(andnot, ldtclr)
128
+
LSUI_FUTEX_ATOMIC_OP(set, swpt)
129
+
130
+
static __always_inline int
131
+
__lsui_cmpxchg64(u64 __user *uaddr, u64 *oldval, u64 newval)
132
+
{
133
+
int ret = 0;
134
+
135
+
uaccess_ttbr0_enable();
136
+
137
+
asm volatile("// __lsui_cmpxchg64\n"
138
+
__LSUI_PREAMBLE
139
+
"1: casalt %[oldval], %[newval], %[uaddr]\n"
140
+
"2:\n"
141
+
_ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret])
142
+
: [ret] "+r" (ret), [uaddr] "+Q" (*uaddr),
143
+
[oldval] "+r" (*oldval)
144
+
: [newval] "r" (newval)
145
+
: "memory");
146
+
147
+
uaccess_ttbr0_disable();
148
+
149
+
return ret;
150
+
}
151
+
152
+
static __always_inline int
153
+
__lsui_cmpxchg32(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
154
+
{
155
+
u64 __user *uaddr64;
156
+
bool futex_pos, other_pos;
157
+
u32 other, orig_other;
158
+
union {
159
+
u32 futex[2];
160
+
u64 raw;
161
+
} oval64, orig64, nval64;
162
+
163
+
uaddr64 = (u64 __user *)PTR_ALIGN_DOWN(uaddr, sizeof(u64));
164
+
futex_pos = !IS_ALIGNED((unsigned long)uaddr, sizeof(u64));
165
+
other_pos = !futex_pos;
166
+
167
+
oval64.futex[futex_pos] = oldval;
168
+
if (get_user(oval64.futex[other_pos], (u32 __user *)uaddr64 + other_pos))
169
+
return -EFAULT;
170
+
171
+
orig64.raw = oval64.raw;
172
+
173
+
nval64.futex[futex_pos] = newval;
174
+
nval64.futex[other_pos] = oval64.futex[other_pos];
175
+
176
+
if (__lsui_cmpxchg64(uaddr64, &oval64.raw, nval64.raw))
177
+
return -EFAULT;
178
+
179
+
oldval = oval64.futex[futex_pos];
180
+
other = oval64.futex[other_pos];
181
+
orig_other = orig64.futex[other_pos];
182
+
183
+
if (other != orig_other)
184
+
return -EAGAIN;
185
+
186
+
*oval = oldval;
187
+
188
+
return 0;
189
+
}
190
+
191
+
static __always_inline int
192
+
__lsui_futex_atomic_and(int oparg, u32 __user *uaddr, int *oval)
193
+
{
194
+
/*
195
+
* Undo the bitwise negation applied to the oparg passed from
196
+
* arch_futex_atomic_op_inuser() with FUTEX_OP_ANDN.
197
+
*/
198
+
return __lsui_futex_atomic_andnot(~oparg, uaddr, oval);
199
+
}
200
+
201
+
static __always_inline int
202
+
__lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval)
203
+
{
204
+
u32 oldval, newval, val;
205
+
int ret, i;
206
+
207
+
if (get_user(oldval, uaddr))
208
+
return -EFAULT;
209
+
210
+
/*
211
+
* there are no ldteor/stteor instructions...
212
+
*/
213
+
for (i = 0; i < FUTEX_MAX_LOOPS; i++) {
214
+
newval = oldval ^ oparg;
215
+
216
+
ret = __lsui_cmpxchg32(uaddr, oldval, newval, &val);
217
+
switch (ret) {
218
+
case -EFAULT:
219
+
return ret;
220
+
case -EAGAIN:
221
+
continue;
222
+
}
223
+
224
+
if (val == oldval) {
225
+
*oval = val;
226
+
return 0;
227
+
}
228
+
229
+
oldval = val;
230
+
}
231
+
232
+
return -EAGAIN;
233
+
}
234
+
235
+
static __always_inline int
236
+
__lsui_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
237
+
{
238
+
/*
239
+
* Callers of futex_atomic_cmpxchg_inatomic() already retry on
240
+
* -EAGAIN, no need for another loop of max retries.
241
+
*/
242
+
return __lsui_cmpxchg32(uaddr, oldval, newval, oval);
243
+
}
244
+
#endif /* CONFIG_ARM64_LSUI */
245
+
246
+
93
247
#define FUTEX_ATOMIC_OP(op) \
94
248
static __always_inline int \
95
249
__futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \
96
250
{ \
97
-
return __llsc_futex_atomic_##op(oparg, uaddr, oval); \
251
+
return __lsui_llsc_body(futex_atomic_##op, oparg, uaddr, oval); \
98
252
}
99
253
100
254
FUTEX_ATOMIC_OP(add)
···
261
105
static __always_inline int
262
106
__futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
263
107
{
264
-
return __llsc_futex_cmpxchg(uaddr, oldval, newval, oval);
108
+
return __lsui_llsc_body(futex_cmpxchg, uaddr, oldval, newval, oval);
265
109
}
266
110
267
111
static inline int
+27
arch/arm64/include/asm/lsui.h
+27
arch/arm64/include/asm/lsui.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
#ifndef __ASM_LSUI_H
3
+
#define __ASM_LSUI_H
4
+
5
+
#include <linux/compiler_types.h>
6
+
#include <linux/stringify.h>
7
+
#include <asm/alternative.h>
8
+
#include <asm/alternative-macros.h>
9
+
#include <asm/cpucaps.h>
10
+
11
+
#define __LSUI_PREAMBLE ".arch_extension lsui\n"
12
+
13
+
#ifdef CONFIG_ARM64_LSUI
14
+
15
+
#define __lsui_llsc_body(op, ...) \
16
+
({ \
17
+
alternative_has_cap_unlikely(ARM64_HAS_LSUI) ? \
18
+
__lsui_##op(__VA_ARGS__) : __llsc_##op(__VA_ARGS__); \
19
+
})
20
+
21
+
#else /* CONFIG_ARM64_LSUI */
22
+
23
+
#define __lsui_llsc_body(op, ...) __llsc_##op(__VA_ARGS__)
24
+
25
+
#endif /* CONFIG_ARM64_LSUI */
26
+
27
+
#endif /* __ASM_LSUI_H */