Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'x86-percpu-2024-07-17' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 percpu updates from Ingo Molnar:
- Enable the named address spaces based percpu accessors optimization
on all GCC versions that contain this feature, detected through a
build-time testcase.
This effectively enables the feature on GCC 6, GCC 7 and GCC 8
versions.
- Fix operand constraint modifier in __raw_cpu_write()
- Reorganize the per-CPU headers for better readability
- Misc cleanups and fixes
* tag 'x86-percpu-2024-07-17' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
x86/percpu: Enable named address spaces for all capable GCC versions
x86/percpu: Clean up <asm/percpu.h> vertical alignment details
x86/percpu: Clean up <asm/percpu.h> a bit
x86/percpu: Move some percpu accessors around to reduce ifdeffery
x86/percpu: Rename percpu_stable_op() to __raw_cpu_read_stable()
x86/percpu: Fix operand constraint modifier in __raw_cpu_write()
x86/percpu: Introduce the __raw_cpu_read_const() macro
x86/percpu: Unify percpu read-write accessors
x86/percpu: Move some percpu macros around for readability
x86/percpu: Introduce the pcpu_binary_op() macro
+281
-257
+2
-1
arch/x86/Kconfig
+2
-1
arch/x86/Kconfig
···
2428
2428
endmenu
2429
2429
2430
2430
config CC_HAS_NAMED_AS
2431
-
def_bool CC_IS_GCC && GCC_VERSION >= 90100
2431
+
def_bool $(success,echo 'int __seg_fs fs; int __seg_gs gs;' | $(CC) -x c - -S -o /dev/null)
2432
+
depends on CC_IS_GCC
2432
2433
2433
2434
config CC_HAS_NAMED_AS_FIXED_SANITIZERS
2434
2435
def_bool CC_IS_GCC && GCC_VERSION >= 130300
+279
-256
arch/x86/include/asm/percpu.h
+279
-256
arch/x86/include/asm/percpu.h
···
3
3
#define _ASM_X86_PERCPU_H
4
4
5
5
#ifdef CONFIG_X86_64
6
-
#define __percpu_seg gs
7
-
#define __percpu_rel (%rip)
6
+
# define __percpu_seg gs
7
+
# define __percpu_rel (%rip)
8
8
#else
9
-
#define __percpu_seg fs
10
-
#define __percpu_rel
9
+
# define __percpu_seg fs
10
+
# define __percpu_rel
11
11
#endif
12
12
13
13
#ifdef __ASSEMBLY__
14
14
15
15
#ifdef CONFIG_SMP
16
-
#define __percpu %__percpu_seg:
16
+
# define __percpu %__percpu_seg:
17
17
#else
18
-
#define __percpu
18
+
# define __percpu
19
19
#endif
20
20
21
21
#define PER_CPU_VAR(var) __percpu(var)__percpu_rel
22
22
23
23
#ifdef CONFIG_X86_64_SMP
24
-
#define INIT_PER_CPU_VAR(var) init_per_cpu__##var
24
+
# define INIT_PER_CPU_VAR(var) init_per_cpu__##var
25
25
#else
26
-
#define INIT_PER_CPU_VAR(var) var
26
+
# define INIT_PER_CPU_VAR(var) var
27
27
#endif
28
28
29
-
#else /* ...!ASSEMBLY */
29
+
#else /* !__ASSEMBLY__: */
30
30
31
31
#include <linux/build_bug.h>
32
32
#include <linux/stringify.h>
···
37
37
#ifdef CONFIG_CC_HAS_NAMED_AS
38
38
39
39
#ifdef __CHECKER__
40
-
#define __seg_gs __attribute__((address_space(__seg_gs)))
41
-
#define __seg_fs __attribute__((address_space(__seg_fs)))
40
+
# define __seg_gs __attribute__((address_space(__seg_gs)))
41
+
# define __seg_fs __attribute__((address_space(__seg_fs)))
42
42
#endif
43
43
44
44
#ifdef CONFIG_X86_64
45
-
#define __percpu_seg_override __seg_gs
45
+
# define __percpu_seg_override __seg_gs
46
46
#else
47
-
#define __percpu_seg_override __seg_fs
47
+
# define __percpu_seg_override __seg_fs
48
48
#endif
49
49
50
50
#define __percpu_prefix ""
51
51
52
-
#else /* CONFIG_CC_HAS_NAMED_AS */
52
+
#else /* !CONFIG_CC_HAS_NAMED_AS: */
53
53
54
54
#define __percpu_seg_override
55
55
#define __percpu_prefix "%%"__stringify(__percpu_seg)":"
···
68
68
* sizeof(this_cpu_off) becames 4.
69
69
*/
70
70
#ifndef BUILD_VDSO32_64
71
-
#define arch_raw_cpu_ptr(_ptr) \
72
-
({ \
73
-
unsigned long tcp_ptr__ = raw_cpu_read_long(this_cpu_off); \
74
-
tcp_ptr__ += (__force unsigned long)(_ptr); \
75
-
(typeof(*(_ptr)) __kernel __force *)tcp_ptr__; \
71
+
#define arch_raw_cpu_ptr(_ptr) \
72
+
({ \
73
+
unsigned long tcp_ptr__ = raw_cpu_read_long(this_cpu_off); \
74
+
\
75
+
tcp_ptr__ += (__force unsigned long)(_ptr); \
76
+
(typeof(*(_ptr)) __kernel __force *)tcp_ptr__; \
76
77
})
77
78
#else
78
79
#define arch_raw_cpu_ptr(_ptr) ({ BUILD_BUG(); (typeof(_ptr))0; })
···
81
80
82
81
#define PER_CPU_VAR(var) %__percpu_seg:(var)__percpu_rel
83
82
84
-
#else /* CONFIG_SMP */
83
+
#else /* !CONFIG_SMP: */
84
+
85
85
#define __percpu_seg_override
86
86
#define __percpu_prefix ""
87
87
#define __force_percpu_prefix ""
···
98
96
#define __force_percpu_arg(x) __force_percpu_prefix "%" #x
99
97
100
98
/*
101
-
* Initialized pointers to per-cpu variables needed for the boot
99
+
* Initialized pointers to per-CPU variables needed for the boot
102
100
* processor need to use these macros to get the proper address
103
101
* offset from __per_cpu_load on SMP.
104
102
*
···
108
106
extern typeof(var) init_per_cpu_var(var)
109
107
110
108
#ifdef CONFIG_X86_64_SMP
111
-
#define init_per_cpu_var(var) init_per_cpu__##var
109
+
# define init_per_cpu_var(var) init_per_cpu__##var
112
110
#else
113
-
#define init_per_cpu_var(var) var
111
+
# define init_per_cpu_var(var) var
114
112
#endif
115
113
116
-
/* For arch-specific code, we can use direct single-insn ops (they
117
-
* don't give an lvalue though). */
114
+
/*
115
+
* For arch-specific code, we can use direct single-insn ops (they
116
+
* don't give an lvalue though).
117
+
*/
118
118
119
-
#define __pcpu_type_1 u8
120
-
#define __pcpu_type_2 u16
121
-
#define __pcpu_type_4 u32
122
-
#define __pcpu_type_8 u64
119
+
#define __pcpu_type_1 u8
120
+
#define __pcpu_type_2 u16
121
+
#define __pcpu_type_4 u32
122
+
#define __pcpu_type_8 u64
123
123
124
-
#define __pcpu_cast_1(val) ((u8)(((unsigned long) val) & 0xff))
125
-
#define __pcpu_cast_2(val) ((u16)(((unsigned long) val) & 0xffff))
126
-
#define __pcpu_cast_4(val) ((u32)(((unsigned long) val) & 0xffffffff))
127
-
#define __pcpu_cast_8(val) ((u64)(val))
124
+
#define __pcpu_cast_1(val) ((u8)(((unsigned long) val) & 0xff))
125
+
#define __pcpu_cast_2(val) ((u16)(((unsigned long) val) & 0xffff))
126
+
#define __pcpu_cast_4(val) ((u32)(((unsigned long) val) & 0xffffffff))
127
+
#define __pcpu_cast_8(val) ((u64)(val))
128
128
129
-
#define __pcpu_op1_1(op, dst) op "b " dst
130
-
#define __pcpu_op1_2(op, dst) op "w " dst
131
-
#define __pcpu_op1_4(op, dst) op "l " dst
132
-
#define __pcpu_op1_8(op, dst) op "q " dst
129
+
#define __pcpu_op1_1(op, dst) op "b " dst
130
+
#define __pcpu_op1_2(op, dst) op "w " dst
131
+
#define __pcpu_op1_4(op, dst) op "l " dst
132
+
#define __pcpu_op1_8(op, dst) op "q " dst
133
133
134
134
#define __pcpu_op2_1(op, src, dst) op "b " src ", " dst
135
135
#define __pcpu_op2_2(op, src, dst) op "w " src ", " dst
136
136
#define __pcpu_op2_4(op, src, dst) op "l " src ", " dst
137
137
#define __pcpu_op2_8(op, src, dst) op "q " src ", " dst
138
138
139
-
#define __pcpu_reg_1(mod, x) mod "q" (x)
140
-
#define __pcpu_reg_2(mod, x) mod "r" (x)
141
-
#define __pcpu_reg_4(mod, x) mod "r" (x)
142
-
#define __pcpu_reg_8(mod, x) mod "r" (x)
139
+
#define __pcpu_reg_1(mod, x) mod "q" (x)
140
+
#define __pcpu_reg_2(mod, x) mod "r" (x)
141
+
#define __pcpu_reg_4(mod, x) mod "r" (x)
142
+
#define __pcpu_reg_8(mod, x) mod "r" (x)
143
143
144
-
#define __pcpu_reg_imm_1(x) "qi" (x)
145
-
#define __pcpu_reg_imm_2(x) "ri" (x)
146
-
#define __pcpu_reg_imm_4(x) "ri" (x)
147
-
#define __pcpu_reg_imm_8(x) "re" (x)
144
+
#define __pcpu_reg_imm_1(x) "qi" (x)
145
+
#define __pcpu_reg_imm_2(x) "ri" (x)
146
+
#define __pcpu_reg_imm_4(x) "ri" (x)
147
+
#define __pcpu_reg_imm_8(x) "re" (x)
148
148
149
-
#define percpu_to_op(size, qual, op, _var, _val) \
149
+
#ifdef CONFIG_USE_X86_SEG_SUPPORT
150
+
151
+
#define __raw_cpu_read(size, qual, pcp) \
152
+
({ \
153
+
*(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \
154
+
})
155
+
156
+
#define __raw_cpu_write(size, qual, pcp, val) \
157
+
do { \
158
+
*(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \
159
+
} while (0)
160
+
161
+
#define __raw_cpu_read_const(pcp) __raw_cpu_read(, , pcp)
162
+
163
+
#else /* !CONFIG_USE_X86_SEG_SUPPORT: */
164
+
165
+
#define __raw_cpu_read(size, qual, _var) \
166
+
({ \
167
+
__pcpu_type_##size pfo_val__; \
168
+
\
169
+
asm qual (__pcpu_op2_##size("mov", __percpu_arg([var]), "%[val]") \
170
+
: [val] __pcpu_reg_##size("=", pfo_val__) \
171
+
: [var] "m" (__my_cpu_var(_var))); \
172
+
\
173
+
(typeof(_var))(unsigned long) pfo_val__; \
174
+
})
175
+
176
+
#define __raw_cpu_write(size, qual, _var, _val) \
150
177
do { \
151
178
__pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \
179
+
\
180
+
if (0) { \
181
+
typeof(_var) pto_tmp__; \
182
+
pto_tmp__ = (_val); \
183
+
(void)pto_tmp__; \
184
+
} \
185
+
asm qual(__pcpu_op2_##size("mov", "%[val]", __percpu_arg([var])) \
186
+
: [var] "=m" (__my_cpu_var(_var)) \
187
+
: [val] __pcpu_reg_imm_##size(pto_val__)); \
188
+
} while (0)
189
+
190
+
/*
191
+
* The generic per-CPU infrastrucutre is not suitable for
192
+
* reading const-qualified variables.
193
+
*/
194
+
#define __raw_cpu_read_const(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
195
+
196
+
#endif /* CONFIG_USE_X86_SEG_SUPPORT */
197
+
198
+
#define __raw_cpu_read_stable(size, _var) \
199
+
({ \
200
+
__pcpu_type_##size pfo_val__; \
201
+
\
202
+
asm(__pcpu_op2_##size("mov", __force_percpu_arg(a[var]), "%[val]") \
203
+
: [val] __pcpu_reg_##size("=", pfo_val__) \
204
+
: [var] "i" (&(_var))); \
205
+
\
206
+
(typeof(_var))(unsigned long) pfo_val__; \
207
+
})
208
+
209
+
#define percpu_unary_op(size, qual, op, _var) \
210
+
({ \
211
+
asm qual (__pcpu_op1_##size(op, __percpu_arg([var])) \
212
+
: [var] "+m" (__my_cpu_var(_var))); \
213
+
})
214
+
215
+
#define percpu_binary_op(size, qual, op, _var, _val) \
216
+
do { \
217
+
__pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \
218
+
\
152
219
if (0) { \
153
220
typeof(_var) pto_tmp__; \
154
221
pto_tmp__ = (_val); \
···
228
157
: [val] __pcpu_reg_imm_##size(pto_val__)); \
229
158
} while (0)
230
159
231
-
#define percpu_unary_op(size, qual, op, _var) \
232
-
({ \
233
-
asm qual (__pcpu_op1_##size(op, __percpu_arg([var])) \
234
-
: [var] "+m" (__my_cpu_var(_var))); \
235
-
})
236
-
237
160
/*
238
-
* Generate a percpu add to memory instruction and optimize code
161
+
* Generate a per-CPU add to memory instruction and optimize code
239
162
* if one is added or subtracted.
240
163
*/
241
164
#define percpu_add_op(size, qual, var, val) \
···
237
172
const int pao_ID__ = (__builtin_constant_p(val) && \
238
173
((val) == 1 || (val) == -1)) ? \
239
174
(int)(val) : 0; \
175
+
\
240
176
if (0) { \
241
177
typeof(var) pao_tmp__; \
242
178
pao_tmp__ = (val); \
···
248
182
else if (pao_ID__ == -1) \
249
183
percpu_unary_op(size, qual, "dec", var); \
250
184
else \
251
-
percpu_to_op(size, qual, "add", var, val); \
185
+
percpu_binary_op(size, qual, "add", var, val); \
252
186
} while (0)
253
-
254
-
#define percpu_from_op(size, qual, op, _var) \
255
-
({ \
256
-
__pcpu_type_##size pfo_val__; \
257
-
asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \
258
-
: [val] __pcpu_reg_##size("=", pfo_val__) \
259
-
: [var] "m" (__my_cpu_var(_var))); \
260
-
(typeof(_var))(unsigned long) pfo_val__; \
261
-
})
262
-
263
-
#define percpu_stable_op(size, op, _var) \
264
-
({ \
265
-
__pcpu_type_##size pfo_val__; \
266
-
asm(__pcpu_op2_##size(op, __force_percpu_arg(a[var]), "%[val]") \
267
-
: [val] __pcpu_reg_##size("=", pfo_val__) \
268
-
: [var] "i" (&(_var))); \
269
-
(typeof(_var))(unsigned long) pfo_val__; \
270
-
})
271
187
272
188
/*
273
189
* Add return operation
···
257
209
#define percpu_add_return_op(size, qual, _var, _val) \
258
210
({ \
259
211
__pcpu_type_##size paro_tmp__ = __pcpu_cast_##size(_val); \
212
+
\
260
213
asm qual (__pcpu_op2_##size("xadd", "%[tmp]", \
261
214
__percpu_arg([var])) \
262
215
: [tmp] __pcpu_reg_##size("+", paro_tmp__), \
···
273
224
#define raw_percpu_xchg_op(_var, _nval) \
274
225
({ \
275
226
typeof(_var) pxo_old__ = raw_cpu_read(_var); \
227
+
\
276
228
raw_cpu_write(_var, _nval); \
229
+
\
277
230
pxo_old__; \
278
231
})
279
232
280
233
/*
281
-
* this_cpu_xchg() is implemented using cmpxchg without a lock prefix.
282
-
* xchg is expensive due to the implied lock prefix. The processor
283
-
* cannot prefetch cachelines if xchg is used.
234
+
* this_cpu_xchg() is implemented using CMPXCHG without a LOCK prefix.
235
+
* XCHG is expensive due to the implied LOCK prefix. The processor
236
+
* cannot prefetch cachelines if XCHG is used.
284
237
*/
285
238
#define this_percpu_xchg_op(_var, _nval) \
286
239
({ \
287
240
typeof(_var) pxo_old__ = this_cpu_read(_var); \
241
+
\
288
242
do { } while (!this_cpu_try_cmpxchg(_var, &pxo_old__, _nval)); \
243
+
\
289
244
pxo_old__; \
290
245
})
291
246
292
247
/*
293
-
* cmpxchg has no such implied lock semantics as a result it is much
294
-
* more efficient for cpu local operations.
248
+
* CMPXCHG has no such implied lock semantics as a result it is much
249
+
* more efficient for CPU-local operations.
295
250
*/
296
251
#define percpu_cmpxchg_op(size, qual, _var, _oval, _nval) \
297
252
({ \
298
253
__pcpu_type_##size pco_old__ = __pcpu_cast_##size(_oval); \
299
254
__pcpu_type_##size pco_new__ = __pcpu_cast_##size(_nval); \
255
+
\
300
256
asm qual (__pcpu_op2_##size("cmpxchg", "%[nval]", \
301
257
__percpu_arg([var])) \
302
258
: [oval] "+a" (pco_old__), \
303
259
[var] "+m" (__my_cpu_var(_var)) \
304
260
: [nval] __pcpu_reg_##size(, pco_new__) \
305
261
: "memory"); \
262
+
\
306
263
(typeof(_var))(unsigned long) pco_old__; \
307
264
})
308
265
···
318
263
__pcpu_type_##size *pco_oval__ = (__pcpu_type_##size *)(_ovalp); \
319
264
__pcpu_type_##size pco_old__ = *pco_oval__; \
320
265
__pcpu_type_##size pco_new__ = __pcpu_cast_##size(_nval); \
266
+
\
321
267
asm qual (__pcpu_op2_##size("cmpxchg", "%[nval]", \
322
268
__percpu_arg([var])) \
323
269
CC_SET(z) \
···
329
273
: "memory"); \
330
274
if (unlikely(!success)) \
331
275
*pco_oval__ = pco_old__; \
276
+
\
332
277
likely(success); \
333
278
})
334
279
335
280
#if defined(CONFIG_X86_32) && !defined(CONFIG_UML)
281
+
336
282
#define percpu_cmpxchg64_op(size, qual, _var, _oval, _nval) \
337
283
({ \
338
284
union { \
···
360
302
old__.var; \
361
303
})
362
304
363
-
#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, , pcp, oval, nval)
364
-
#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, volatile, pcp, oval, nval)
305
+
#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, , pcp, oval, nval)
306
+
#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, volatile, pcp, oval, nval)
365
307
366
308
#define percpu_try_cmpxchg64_op(size, qual, _var, _ovalp, _nval) \
367
309
({ \
···
390
332
: "memory"); \
391
333
if (unlikely(!success)) \
392
334
*_oval = old__.var; \
335
+
\
393
336
likely(success); \
394
337
})
395
338
396
339
#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg64_op(8, , pcp, ovalp, nval)
397
340
#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg64_op(8, volatile, pcp, ovalp, nval)
398
-
#endif
341
+
342
+
#endif /* defined(CONFIG_X86_32) && !defined(CONFIG_UML) */
399
343
400
344
#ifdef CONFIG_X86_64
401
-
#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval);
402
-
#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval);
345
+
#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval);
346
+
#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval);
403
347
404
348
#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, , pcp, ovalp, nval);
405
349
#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, volatile, pcp, ovalp, nval);
···
431
371
old__.var; \
432
372
})
433
373
434
-
#define raw_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, , pcp, oval, nval)
435
-
#define this_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, volatile, pcp, oval, nval)
374
+
#define raw_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, , pcp, oval, nval)
375
+
#define this_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, volatile, pcp, oval, nval)
436
376
437
377
#define percpu_try_cmpxchg128_op(size, qual, _var, _ovalp, _nval) \
438
378
({ \
···
466
406
467
407
#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) percpu_try_cmpxchg128_op(16, , pcp, ovalp, nval)
468
408
#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) percpu_try_cmpxchg128_op(16, volatile, pcp, ovalp, nval)
469
-
#endif
470
409
471
-
/*
472
-
* this_cpu_read() makes gcc load the percpu variable every time it is
473
-
* accessed while this_cpu_read_stable() allows the value to be cached.
474
-
* this_cpu_read_stable() is more efficient and can be used if its value
475
-
* is guaranteed to be valid across cpus. The current users include
476
-
* pcpu_hot.current_task and pcpu_hot.top_of_stack, both of which are
477
-
* actually per-thread variables implemented as per-CPU variables and
478
-
* thus stable for the duration of the respective task.
479
-
*/
480
-
#define this_cpu_read_stable(pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp)
410
+
#endif /* CONFIG_X86_64 */
481
411
482
-
#ifdef CONFIG_USE_X86_SEG_SUPPORT
412
+
#define raw_cpu_read_1(pcp) __raw_cpu_read(1, , pcp)
413
+
#define raw_cpu_read_2(pcp) __raw_cpu_read(2, , pcp)
414
+
#define raw_cpu_read_4(pcp) __raw_cpu_read(4, , pcp)
415
+
#define raw_cpu_write_1(pcp, val) __raw_cpu_write(1, , pcp, val)
416
+
#define raw_cpu_write_2(pcp, val) __raw_cpu_write(2, , pcp, val)
417
+
#define raw_cpu_write_4(pcp, val) __raw_cpu_write(4, , pcp, val)
483
418
484
-
#define __raw_cpu_read(qual, pcp) \
485
-
({ \
486
-
*(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \
487
-
})
419
+
#define this_cpu_read_1(pcp) __raw_cpu_read(1, volatile, pcp)
420
+
#define this_cpu_read_2(pcp) __raw_cpu_read(2, volatile, pcp)
421
+
#define this_cpu_read_4(pcp) __raw_cpu_read(4, volatile, pcp)
422
+
#define this_cpu_write_1(pcp, val) __raw_cpu_write(1, volatile, pcp, val)
423
+
#define this_cpu_write_2(pcp, val) __raw_cpu_write(2, volatile, pcp, val)
424
+
#define this_cpu_write_4(pcp, val) __raw_cpu_write(4, volatile, pcp, val)
488
425
489
-
#define __raw_cpu_write(qual, pcp, val) \
490
-
do { \
491
-
*(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \
492
-
} while (0)
426
+
#define this_cpu_read_stable_1(pcp) __raw_cpu_read_stable(1, pcp)
427
+
#define this_cpu_read_stable_2(pcp) __raw_cpu_read_stable(2, pcp)
428
+
#define this_cpu_read_stable_4(pcp) __raw_cpu_read_stable(4, pcp)
493
429
494
-
#define raw_cpu_read_1(pcp) __raw_cpu_read(, pcp)
495
-
#define raw_cpu_read_2(pcp) __raw_cpu_read(, pcp)
496
-
#define raw_cpu_read_4(pcp) __raw_cpu_read(, pcp)
497
-
#define raw_cpu_write_1(pcp, val) __raw_cpu_write(, pcp, val)
498
-
#define raw_cpu_write_2(pcp, val) __raw_cpu_write(, pcp, val)
499
-
#define raw_cpu_write_4(pcp, val) __raw_cpu_write(, pcp, val)
430
+
#define raw_cpu_add_1(pcp, val) percpu_add_op(1, , (pcp), val)
431
+
#define raw_cpu_add_2(pcp, val) percpu_add_op(2, , (pcp), val)
432
+
#define raw_cpu_add_4(pcp, val) percpu_add_op(4, , (pcp), val)
433
+
#define raw_cpu_and_1(pcp, val) percpu_binary_op(1, , "and", (pcp), val)
434
+
#define raw_cpu_and_2(pcp, val) percpu_binary_op(2, , "and", (pcp), val)
435
+
#define raw_cpu_and_4(pcp, val) percpu_binary_op(4, , "and", (pcp), val)
436
+
#define raw_cpu_or_1(pcp, val) percpu_binary_op(1, , "or", (pcp), val)
437
+
#define raw_cpu_or_2(pcp, val) percpu_binary_op(2, , "or", (pcp), val)
438
+
#define raw_cpu_or_4(pcp, val) percpu_binary_op(4, , "or", (pcp), val)
439
+
#define raw_cpu_xchg_1(pcp, val) raw_percpu_xchg_op(pcp, val)
440
+
#define raw_cpu_xchg_2(pcp, val) raw_percpu_xchg_op(pcp, val)
441
+
#define raw_cpu_xchg_4(pcp, val) raw_percpu_xchg_op(pcp, val)
500
442
501
-
#define this_cpu_read_1(pcp) __raw_cpu_read(volatile, pcp)
502
-
#define this_cpu_read_2(pcp) __raw_cpu_read(volatile, pcp)
503
-
#define this_cpu_read_4(pcp) __raw_cpu_read(volatile, pcp)
504
-
#define this_cpu_write_1(pcp, val) __raw_cpu_write(volatile, pcp, val)
505
-
#define this_cpu_write_2(pcp, val) __raw_cpu_write(volatile, pcp, val)
506
-
#define this_cpu_write_4(pcp, val) __raw_cpu_write(volatile, pcp, val)
443
+
#define this_cpu_add_1(pcp, val) percpu_add_op(1, volatile, (pcp), val)
444
+
#define this_cpu_add_2(pcp, val) percpu_add_op(2, volatile, (pcp), val)
445
+
#define this_cpu_add_4(pcp, val) percpu_add_op(4, volatile, (pcp), val)
446
+
#define this_cpu_and_1(pcp, val) percpu_binary_op(1, volatile, "and", (pcp), val)
447
+
#define this_cpu_and_2(pcp, val) percpu_binary_op(2, volatile, "and", (pcp), val)
448
+
#define this_cpu_and_4(pcp, val) percpu_binary_op(4, volatile, "and", (pcp), val)
449
+
#define this_cpu_or_1(pcp, val) percpu_binary_op(1, volatile, "or", (pcp), val)
450
+
#define this_cpu_or_2(pcp, val) percpu_binary_op(2, volatile, "or", (pcp), val)
451
+
#define this_cpu_or_4(pcp, val) percpu_binary_op(4, volatile, "or", (pcp), val)
452
+
#define this_cpu_xchg_1(pcp, nval) this_percpu_xchg_op(pcp, nval)
453
+
#define this_cpu_xchg_2(pcp, nval) this_percpu_xchg_op(pcp, nval)
454
+
#define this_cpu_xchg_4(pcp, nval) this_percpu_xchg_op(pcp, nval)
507
455
508
-
#ifdef CONFIG_X86_64
509
-
#define raw_cpu_read_8(pcp) __raw_cpu_read(, pcp)
510
-
#define raw_cpu_write_8(pcp, val) __raw_cpu_write(, pcp, val)
456
+
#define raw_cpu_add_return_1(pcp, val) percpu_add_return_op(1, , pcp, val)
457
+
#define raw_cpu_add_return_2(pcp, val) percpu_add_return_op(2, , pcp, val)
458
+
#define raw_cpu_add_return_4(pcp, val) percpu_add_return_op(4, , pcp, val)
459
+
#define raw_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(1, , pcp, oval, nval)
460
+
#define raw_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, , pcp, oval, nval)
461
+
#define raw_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, , pcp, oval, nval)
462
+
#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) percpu_try_cmpxchg_op(1, , pcp, ovalp, nval)
463
+
#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) percpu_try_cmpxchg_op(2, , pcp, ovalp, nval)
464
+
#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) percpu_try_cmpxchg_op(4, , pcp, ovalp, nval)
511
465
512
-
#define this_cpu_read_8(pcp) __raw_cpu_read(volatile, pcp)
513
-
#define this_cpu_write_8(pcp, val) __raw_cpu_write(volatile, pcp, val)
514
-
#endif
515
-
516
-
#define this_cpu_read_const(pcp) __raw_cpu_read(, pcp)
517
-
#else /* CONFIG_USE_X86_SEG_SUPPORT */
518
-
519
-
#define raw_cpu_read_1(pcp) percpu_from_op(1, , "mov", pcp)
520
-
#define raw_cpu_read_2(pcp) percpu_from_op(2, , "mov", pcp)
521
-
#define raw_cpu_read_4(pcp) percpu_from_op(4, , "mov", pcp)
522
-
#define raw_cpu_write_1(pcp, val) percpu_to_op(1, , "mov", (pcp), val)
523
-
#define raw_cpu_write_2(pcp, val) percpu_to_op(2, , "mov", (pcp), val)
524
-
#define raw_cpu_write_4(pcp, val) percpu_to_op(4, , "mov", (pcp), val)
525
-
526
-
#define this_cpu_read_1(pcp) percpu_from_op(1, volatile, "mov", pcp)
527
-
#define this_cpu_read_2(pcp) percpu_from_op(2, volatile, "mov", pcp)
528
-
#define this_cpu_read_4(pcp) percpu_from_op(4, volatile, "mov", pcp)
529
-
#define this_cpu_write_1(pcp, val) percpu_to_op(1, volatile, "mov", (pcp), val)
530
-
#define this_cpu_write_2(pcp, val) percpu_to_op(2, volatile, "mov", (pcp), val)
531
-
#define this_cpu_write_4(pcp, val) percpu_to_op(4, volatile, "mov", (pcp), val)
532
-
533
-
#ifdef CONFIG_X86_64
534
-
#define raw_cpu_read_8(pcp) percpu_from_op(8, , "mov", pcp)
535
-
#define raw_cpu_write_8(pcp, val) percpu_to_op(8, , "mov", (pcp), val)
536
-
537
-
#define this_cpu_read_8(pcp) percpu_from_op(8, volatile, "mov", pcp)
538
-
#define this_cpu_write_8(pcp, val) percpu_to_op(8, volatile, "mov", (pcp), val)
539
-
#endif
540
-
541
-
/*
542
-
* The generic per-cpu infrastrucutre is not suitable for
543
-
* reading const-qualified variables.
544
-
*/
545
-
#define this_cpu_read_const(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
546
-
#endif /* CONFIG_USE_X86_SEG_SUPPORT */
547
-
548
-
#define this_cpu_read_stable_1(pcp) percpu_stable_op(1, "mov", pcp)
549
-
#define this_cpu_read_stable_2(pcp) percpu_stable_op(2, "mov", pcp)
550
-
#define this_cpu_read_stable_4(pcp) percpu_stable_op(4, "mov", pcp)
551
-
552
-
#define raw_cpu_add_1(pcp, val) percpu_add_op(1, , (pcp), val)
553
-
#define raw_cpu_add_2(pcp, val) percpu_add_op(2, , (pcp), val)
554
-
#define raw_cpu_add_4(pcp, val) percpu_add_op(4, , (pcp), val)
555
-
#define raw_cpu_and_1(pcp, val) percpu_to_op(1, , "and", (pcp), val)
556
-
#define raw_cpu_and_2(pcp, val) percpu_to_op(2, , "and", (pcp), val)
557
-
#define raw_cpu_and_4(pcp, val) percpu_to_op(4, , "and", (pcp), val)
558
-
#define raw_cpu_or_1(pcp, val) percpu_to_op(1, , "or", (pcp), val)
559
-
#define raw_cpu_or_2(pcp, val) percpu_to_op(2, , "or", (pcp), val)
560
-
#define raw_cpu_or_4(pcp, val) percpu_to_op(4, , "or", (pcp), val)
561
-
#define raw_cpu_xchg_1(pcp, val) raw_percpu_xchg_op(pcp, val)
562
-
#define raw_cpu_xchg_2(pcp, val) raw_percpu_xchg_op(pcp, val)
563
-
#define raw_cpu_xchg_4(pcp, val) raw_percpu_xchg_op(pcp, val)
564
-
565
-
#define this_cpu_add_1(pcp, val) percpu_add_op(1, volatile, (pcp), val)
566
-
#define this_cpu_add_2(pcp, val) percpu_add_op(2, volatile, (pcp), val)
567
-
#define this_cpu_add_4(pcp, val) percpu_add_op(4, volatile, (pcp), val)
568
-
#define this_cpu_and_1(pcp, val) percpu_to_op(1, volatile, "and", (pcp), val)
569
-
#define this_cpu_and_2(pcp, val) percpu_to_op(2, volatile, "and", (pcp), val)
570
-
#define this_cpu_and_4(pcp, val) percpu_to_op(4, volatile, "and", (pcp), val)
571
-
#define this_cpu_or_1(pcp, val) percpu_to_op(1, volatile, "or", (pcp), val)
572
-
#define this_cpu_or_2(pcp, val) percpu_to_op(2, volatile, "or", (pcp), val)
573
-
#define this_cpu_or_4(pcp, val) percpu_to_op(4, volatile, "or", (pcp), val)
574
-
#define this_cpu_xchg_1(pcp, nval) this_percpu_xchg_op(pcp, nval)
575
-
#define this_cpu_xchg_2(pcp, nval) this_percpu_xchg_op(pcp, nval)
576
-
#define this_cpu_xchg_4(pcp, nval) this_percpu_xchg_op(pcp, nval)
577
-
578
-
#define raw_cpu_add_return_1(pcp, val) percpu_add_return_op(1, , pcp, val)
579
-
#define raw_cpu_add_return_2(pcp, val) percpu_add_return_op(2, , pcp, val)
580
-
#define raw_cpu_add_return_4(pcp, val) percpu_add_return_op(4, , pcp, val)
581
-
#define raw_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(1, , pcp, oval, nval)
582
-
#define raw_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, , pcp, oval, nval)
583
-
#define raw_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, , pcp, oval, nval)
584
-
#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) percpu_try_cmpxchg_op(1, , pcp, ovalp, nval)
585
-
#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) percpu_try_cmpxchg_op(2, , pcp, ovalp, nval)
586
-
#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) percpu_try_cmpxchg_op(4, , pcp, ovalp, nval)
587
-
588
-
#define this_cpu_add_return_1(pcp, val) percpu_add_return_op(1, volatile, pcp, val)
589
-
#define this_cpu_add_return_2(pcp, val) percpu_add_return_op(2, volatile, pcp, val)
590
-
#define this_cpu_add_return_4(pcp, val) percpu_add_return_op(4, volatile, pcp, val)
591
-
#define this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(1, volatile, pcp, oval, nval)
592
-
#define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, volatile, pcp, oval, nval)
593
-
#define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, volatile, pcp, oval, nval)
466
+
#define this_cpu_add_return_1(pcp, val) percpu_add_return_op(1, volatile, pcp, val)
467
+
#define this_cpu_add_return_2(pcp, val) percpu_add_return_op(2, volatile, pcp, val)
468
+
#define this_cpu_add_return_4(pcp, val) percpu_add_return_op(4, volatile, pcp, val)
469
+
#define this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(1, volatile, pcp, oval, nval)
470
+
#define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, volatile, pcp, oval, nval)
471
+
#define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, volatile, pcp, oval, nval)
594
472
#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) percpu_try_cmpxchg_op(1, volatile, pcp, ovalp, nval)
595
473
#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) percpu_try_cmpxchg_op(2, volatile, pcp, ovalp, nval)
596
474
#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) percpu_try_cmpxchg_op(4, volatile, pcp, ovalp, nval)
597
475
598
476
/*
599
-
* Per cpu atomic 64 bit operations are only available under 64 bit.
600
-
* 32 bit must fall back to generic operations.
477
+
* Per-CPU atomic 64-bit operations are only available under 64-bit kernels.
478
+
* 32-bit kernels must fall back to generic operations.
601
479
*/
602
480
#ifdef CONFIG_X86_64
603
-
#define this_cpu_read_stable_8(pcp) percpu_stable_op(8, "mov", pcp)
604
481
605
-
#define raw_cpu_add_8(pcp, val) percpu_add_op(8, , (pcp), val)
606
-
#define raw_cpu_and_8(pcp, val) percpu_to_op(8, , "and", (pcp), val)
607
-
#define raw_cpu_or_8(pcp, val) percpu_to_op(8, , "or", (pcp), val)
608
-
#define raw_cpu_add_return_8(pcp, val) percpu_add_return_op(8, , pcp, val)
609
-
#define raw_cpu_xchg_8(pcp, nval) raw_percpu_xchg_op(pcp, nval)
610
-
#define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval)
611
-
#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, , pcp, ovalp, nval)
482
+
#define raw_cpu_read_8(pcp) __raw_cpu_read(8, , pcp)
483
+
#define raw_cpu_write_8(pcp, val) __raw_cpu_write(8, , pcp, val)
612
484
613
-
#define this_cpu_add_8(pcp, val) percpu_add_op(8, volatile, (pcp), val)
614
-
#define this_cpu_and_8(pcp, val) percpu_to_op(8, volatile, "and", (pcp), val)
615
-
#define this_cpu_or_8(pcp, val) percpu_to_op(8, volatile, "or", (pcp), val)
616
-
#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val)
617
-
#define this_cpu_xchg_8(pcp, nval) this_percpu_xchg_op(pcp, nval)
618
-
#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
485
+
#define this_cpu_read_8(pcp) __raw_cpu_read(8, volatile, pcp)
486
+
#define this_cpu_write_8(pcp, val) __raw_cpu_write(8, volatile, pcp, val)
487
+
488
+
#define this_cpu_read_stable_8(pcp) __raw_cpu_read_stable(8, pcp)
489
+
490
+
#define raw_cpu_add_8(pcp, val) percpu_add_op(8, , (pcp), val)
491
+
#define raw_cpu_and_8(pcp, val) percpu_binary_op(8, , "and", (pcp), val)
492
+
#define raw_cpu_or_8(pcp, val) percpu_binary_op(8, , "or", (pcp), val)
493
+
#define raw_cpu_add_return_8(pcp, val) percpu_add_return_op(8, , pcp, val)
494
+
#define raw_cpu_xchg_8(pcp, nval) raw_percpu_xchg_op(pcp, nval)
495
+
#define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval)
496
+
#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, , pcp, ovalp, nval)
497
+
498
+
#define this_cpu_add_8(pcp, val) percpu_add_op(8, volatile, (pcp), val)
499
+
#define this_cpu_and_8(pcp, val) percpu_binary_op(8, volatile, "and", (pcp), val)
500
+
#define this_cpu_or_8(pcp, val) percpu_binary_op(8, volatile, "or", (pcp), val)
501
+
#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val)
502
+
#define this_cpu_xchg_8(pcp, nval) this_percpu_xchg_op(pcp, nval)
503
+
#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
619
504
#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, volatile, pcp, ovalp, nval)
620
505
621
-
#define raw_cpu_read_long(pcp) raw_cpu_read_8(pcp)
622
-
#else
623
-
/* There is no generic 64 bit read stable operation for 32 bit targets. */
624
-
#define this_cpu_read_stable_8(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
506
+
#define raw_cpu_read_long(pcp) raw_cpu_read_8(pcp)
625
507
626
-
#define raw_cpu_read_long(pcp) raw_cpu_read_4(pcp)
627
-
#endif
508
+
#else /* !CONFIG_X86_64: */
509
+
510
+
/* There is no generic 64-bit read stable operation for 32-bit targets. */
511
+
#define this_cpu_read_stable_8(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
512
+
513
+
#define raw_cpu_read_long(pcp) raw_cpu_read_4(pcp)
514
+
515
+
#endif /* CONFIG_X86_64 */
516
+
517
+
#define this_cpu_read_const(pcp) __raw_cpu_read_const(pcp)
518
+
519
+
/*
520
+
* this_cpu_read() makes the compiler load the per-CPU variable every time
521
+
* it is accessed while this_cpu_read_stable() allows the value to be cached.
522
+
* this_cpu_read_stable() is more efficient and can be used if its value
523
+
* is guaranteed to be valid across CPUs. The current users include
524
+
* pcpu_hot.current_task and pcpu_hot.top_of_stack, both of which are
525
+
* actually per-thread variables implemented as per-CPU variables and
526
+
* thus stable for the duration of the respective task.
527
+
*/
528
+
#define this_cpu_read_stable(pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp)
628
529
629
530
#define x86_this_cpu_constant_test_bit(_nr, _var) \
630
531
({ \
631
532
unsigned long __percpu *addr__ = \
632
533
(unsigned long __percpu *)&(_var) + ((_nr) / BITS_PER_LONG); \
534
+
\
633
535
!!((1UL << ((_nr) % BITS_PER_LONG)) & raw_cpu_read(*addr__)); \
634
536
})
635
537
636
-
#define x86_this_cpu_variable_test_bit(_nr, _var) \
637
-
({ \
638
-
bool oldbit; \
639
-
\
640
-
asm volatile("btl %[nr], " __percpu_arg([var]) \
641
-
CC_SET(c) \
642
-
: CC_OUT(c) (oldbit) \
643
-
: [var] "m" (__my_cpu_var(_var)), \
644
-
[nr] "rI" (_nr)); \
645
-
oldbit; \
538
+
#define x86_this_cpu_variable_test_bit(_nr, _var) \
539
+
({ \
540
+
bool oldbit; \
541
+
\
542
+
asm volatile("btl %[nr], " __percpu_arg([var]) \
543
+
CC_SET(c) \
544
+
: CC_OUT(c) (oldbit) \
545
+
: [var] "m" (__my_cpu_var(_var)), \
546
+
[nr] "rI" (_nr)); \
547
+
oldbit; \
646
548
})
647
549
648
-
#define x86_this_cpu_test_bit(_nr, _var) \
649
-
(__builtin_constant_p(_nr) \
650
-
? x86_this_cpu_constant_test_bit(_nr, _var) \
550
+
#define x86_this_cpu_test_bit(_nr, _var) \
551
+
(__builtin_constant_p(_nr) \
552
+
? x86_this_cpu_constant_test_bit(_nr, _var) \
651
553
: x86_this_cpu_variable_test_bit(_nr, _var))
652
554
653
555
···
640
618
{ [0 ... NR_CPUS-1] = _initvalue }; \
641
619
__typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map
642
620
643
-
#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
621
+
#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
644
622
EXPORT_PER_CPU_SYMBOL(_name)
645
623
646
-
#define DECLARE_EARLY_PER_CPU(_type, _name) \
647
-
DECLARE_PER_CPU(_type, _name); \
648
-
extern __typeof__(_type) *_name##_early_ptr; \
624
+
#define DECLARE_EARLY_PER_CPU(_type, _name) \
625
+
DECLARE_PER_CPU(_type, _name); \
626
+
extern __typeof__(_type) *_name##_early_ptr; \
649
627
extern __typeof__(_type) _name##_early_map[]
650
628
651
-
#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \
652
-
DECLARE_PER_CPU_READ_MOSTLY(_type, _name); \
653
-
extern __typeof__(_type) *_name##_early_ptr; \
629
+
#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \
630
+
DECLARE_PER_CPU_READ_MOSTLY(_type, _name); \
631
+
extern __typeof__(_type) *_name##_early_ptr; \
654
632
extern __typeof__(_type) _name##_early_map[]
655
633
656
-
#define early_per_cpu_ptr(_name) (_name##_early_ptr)
657
-
#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
658
-
#define early_per_cpu(_name, _cpu) \
659
-
*(early_per_cpu_ptr(_name) ? \
660
-
&early_per_cpu_ptr(_name)[_cpu] : \
634
+
#define early_per_cpu_ptr(_name) (_name##_early_ptr)
635
+
#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
636
+
637
+
#define early_per_cpu(_name, _cpu) \
638
+
*(early_per_cpu_ptr(_name) ? \
639
+
&early_per_cpu_ptr(_name)[_cpu] : \
661
640
&per_cpu(_name, _cpu))
662
641
663
-
#else /* !CONFIG_SMP */
664
-
#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \
642
+
#else /* !CONFIG_SMP: */
643
+
#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \
665
644
DEFINE_PER_CPU(_type, _name) = _initvalue
666
645
667
646
#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \
668
647
DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue
669
648
670
-
#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
649
+
#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
671
650
EXPORT_PER_CPU_SYMBOL(_name)
672
651
673
-
#define DECLARE_EARLY_PER_CPU(_type, _name) \
652
+
#define DECLARE_EARLY_PER_CPU(_type, _name) \
674
653
DECLARE_PER_CPU(_type, _name)
675
654
676
-
#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \
655
+
#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \
677
656
DECLARE_PER_CPU_READ_MOSTLY(_type, _name)
678
657
679
-
#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu)
680
-
#define early_per_cpu_ptr(_name) NULL
658
+
#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu)
659
+
#define early_per_cpu_ptr(_name) NULL
681
660
/* no early_per_cpu_map() */
682
661
683
-
#endif /* !CONFIG_SMP */
662
+
#endif /* !CONFIG_SMP */
684
663
685
664
#endif /* _ASM_X86_PERCPU_H */