Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
percpu: Add {raw,this}_cpu_try_cmpxchg()
Add the try_cmpxchg() form to the per-cpu ops.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Mark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/20230531132323.587480729@infradead.org
+128
-4
+109
-4
include/asm-generic/percpu.h
+109
-4
include/asm-generic/percpu.h
···
89
89
__ret; \
90
90
})
91
91
92
-
#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
92
+
#define __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, _cmpxchg) \
93
+
({ \
94
+
typeof(pcp) __val, __old = *(ovalp); \
95
+
__val = _cmpxchg(pcp, __old, nval); \
96
+
if (__val != __old) \
97
+
*(ovalp) = __val; \
98
+
__val == __old; \
99
+
})
100
+
101
+
#define raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \
93
102
({ \
94
103
typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \
95
-
typeof(pcp) __ret; \
96
-
__ret = *__p; \
97
-
if (__ret == (oval)) \
104
+
typeof(pcp) __val = *__p, __old = *(ovalp); \
105
+
bool __ret; \
106
+
if (__val == __old) { \
98
107
*__p = nval; \
108
+
__ret = true; \
109
+
} else { \
110
+
*(ovalp) = __val; \
111
+
__ret = false; \
112
+
} \
99
113
__ret; \
114
+
})
115
+
116
+
#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
117
+
({ \
118
+
typeof(pcp) __old = (oval); \
119
+
raw_cpu_generic_try_cmpxchg(pcp, &__old, nval); \
120
+
__old; \
100
121
})
101
122
102
123
#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
···
187
166
unsigned long __flags; \
188
167
raw_local_irq_save(__flags); \
189
168
__ret = raw_cpu_generic_xchg(pcp, nval); \
169
+
raw_local_irq_restore(__flags); \
170
+
__ret; \
171
+
})
172
+
173
+
#define this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \
174
+
({ \
175
+
bool __ret; \
176
+
unsigned long __flags; \
177
+
raw_local_irq_save(__flags); \
178
+
__ret = raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval); \
190
179
raw_local_irq_restore(__flags); \
191
180
__ret; \
192
181
})
···
311
280
#endif
312
281
#ifndef raw_cpu_xchg_8
313
282
#define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
283
+
#endif
284
+
285
+
#ifndef raw_cpu_try_cmpxchg_1
286
+
#ifdef raw_cpu_cmpxchg_1
287
+
#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
288
+
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_1)
289
+
#else
290
+
#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
291
+
raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
292
+
#endif
293
+
#endif
294
+
#ifndef raw_cpu_try_cmpxchg_2
295
+
#ifdef raw_cpu_cmpxchg_2
296
+
#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
297
+
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_2)
298
+
#else
299
+
#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
300
+
raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
301
+
#endif
302
+
#endif
303
+
#ifndef raw_cpu_try_cmpxchg_4
304
+
#ifdef raw_cpu_cmpxchg_4
305
+
#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
306
+
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_4)
307
+
#else
308
+
#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
309
+
raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
310
+
#endif
311
+
#endif
312
+
#ifndef raw_cpu_try_cmpxchg_8
313
+
#ifdef raw_cpu_cmpxchg_8
314
+
#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
315
+
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_8)
316
+
#else
317
+
#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
318
+
raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
319
+
#endif
314
320
#endif
315
321
316
322
#ifndef raw_cpu_cmpxchg_1
···
473
405
#endif
474
406
#ifndef this_cpu_xchg_8
475
407
#define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval)
408
+
#endif
409
+
410
+
#ifndef this_cpu_try_cmpxchg_1
411
+
#ifdef this_cpu_cmpxchg_1
412
+
#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
413
+
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_1)
414
+
#else
415
+
#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
416
+
this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
417
+
#endif
418
+
#endif
419
+
#ifndef this_cpu_try_cmpxchg_2
420
+
#ifdef this_cpu_cmpxchg_2
421
+
#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
422
+
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_2)
423
+
#else
424
+
#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
425
+
this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
426
+
#endif
427
+
#endif
428
+
#ifndef this_cpu_try_cmpxchg_4
429
+
#ifdef this_cpu_cmpxchg_4
430
+
#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
431
+
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_4)
432
+
#else
433
+
#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
434
+
this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
435
+
#endif
436
+
#endif
437
+
#ifndef this_cpu_try_cmpxchg_8
438
+
#ifdef this_cpu_cmpxchg_8
439
+
#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
440
+
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_8)
441
+
#else
442
+
#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
443
+
this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
444
+
#endif
476
445
#endif
477
446
478
447
#ifndef this_cpu_cmpxchg_1
+19
include/linux/percpu-defs.h
+19
include/linux/percpu-defs.h
···
343
343
pscr2_ret__; \
344
344
})
345
345
346
+
#define __pcpu_size_call_return2bool(stem, variable, ...) \
347
+
({ \
348
+
bool pscr2_ret__; \
349
+
__verify_pcpu_ptr(&(variable)); \
350
+
switch(sizeof(variable)) { \
351
+
case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
352
+
case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
353
+
case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
354
+
case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
355
+
default: \
356
+
__bad_size_call_parameter(); break; \
357
+
} \
358
+
pscr2_ret__; \
359
+
})
360
+
346
361
/*
347
362
* Special handling for cmpxchg_double. cmpxchg_double is passed two
348
363
* percpu variables. The first has to be aligned to a double word
···
441
426
#define raw_cpu_xchg(pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)
442
427
#define raw_cpu_cmpxchg(pcp, oval, nval) \
443
428
__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
429
+
#define raw_cpu_try_cmpxchg(pcp, ovalp, nval) \
430
+
__pcpu_size_call_return2bool(raw_cpu_try_cmpxchg_, pcp, ovalp, nval)
444
431
#define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
445
432
__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
446
433
···
530
513
#define this_cpu_xchg(pcp, nval) __pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)
531
514
#define this_cpu_cmpxchg(pcp, oval, nval) \
532
515
__pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
516
+
#define this_cpu_try_cmpxchg(pcp, ovalp, nval) \
517
+
__pcpu_size_call_return2bool(this_cpu_try_cmpxchg_, pcp, ovalp, nval)
533
518
#define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
534
519
__pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
535
520