Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_GENERIC_PERCPU_H_
3#define _ASM_GENERIC_PERCPU_H_
4
5#include <linux/compiler.h>
6#include <linux/threads.h>
7#include <linux/percpu-defs.h>
8
9#ifdef CONFIG_SMP
10
11/*
12 * per_cpu_offset() is the offset that has to be added to a
13 * percpu variable to get to the instance for a certain processor.
14 *
15 * Most arches use the __per_cpu_offset array for those offsets but
16 * some arches have their own ways of determining the offset (x86_64, s390).
17 */
18#ifndef __per_cpu_offset
19extern unsigned long __per_cpu_offset[NR_CPUS];
20
21#define per_cpu_offset(x) (__per_cpu_offset[x])
22#endif
23
24/*
25 * Determine the offset for the currently active processor.
26 * An arch may define __my_cpu_offset to provide a more effective
27 * means of obtaining the offset to the per cpu variables of the
28 * current processor.
29 */
30#ifndef __my_cpu_offset
31#define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
32#endif
33#ifdef CONFIG_DEBUG_PREEMPT
34#define my_cpu_offset per_cpu_offset(smp_processor_id())
35#else
36#define my_cpu_offset __my_cpu_offset
37#endif
38
39/*
40 * Arch may define arch_raw_cpu_ptr() to provide more efficient address
41 * translations for raw_cpu_ptr().
42 */
43#ifndef arch_raw_cpu_ptr
44#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
45#endif
46
47#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
48extern void setup_per_cpu_areas(void);
49#endif
50
51#endif /* SMP */
52
53#ifndef PER_CPU_BASE_SECTION
54#ifdef CONFIG_SMP
55#define PER_CPU_BASE_SECTION ".data..percpu"
56#else
57#define PER_CPU_BASE_SECTION ".data"
58#endif
59#endif
60
61#ifndef PER_CPU_ATTRIBUTES
62#define PER_CPU_ATTRIBUTES
63#endif
64
65#define raw_cpu_generic_read(pcp) \
66({ \
67 *raw_cpu_ptr(&(pcp)); \
68})
69
70#define raw_cpu_generic_to_op(pcp, val, op) \
71do { \
72 *raw_cpu_ptr(&(pcp)) op val; \
73} while (0)
74
75#define raw_cpu_generic_add_return(pcp, val) \
76({ \
77 typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \
78 \
79 *__p += val; \
80 *__p; \
81})
82
83#define raw_cpu_generic_xchg(pcp, nval) \
84({ \
85 typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \
86 typeof(pcp) __ret; \
87 __ret = *__p; \
88 *__p = nval; \
89 __ret; \
90})
91
92#define __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, _cmpxchg) \
93({ \
94 typeof(pcp) __val, __old = *(ovalp); \
95 __val = _cmpxchg(pcp, __old, nval); \
96 if (__val != __old) \
97 *(ovalp) = __val; \
98 __val == __old; \
99})
100
101#define raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \
102({ \
103 typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \
104 typeof(pcp) __val = *__p, ___old = *(ovalp); \
105 bool __ret; \
106 if (__val == ___old) { \
107 *__p = nval; \
108 __ret = true; \
109 } else { \
110 *(ovalp) = __val; \
111 __ret = false; \
112 } \
113 __ret; \
114})
115
116#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
117({ \
118 typeof(pcp) __old = (oval); \
119 raw_cpu_generic_try_cmpxchg(pcp, &__old, nval); \
120 __old; \
121})
122
123#define __this_cpu_generic_read_nopreempt(pcp) \
124({ \
125 typeof(pcp) ___ret; \
126 preempt_disable_notrace(); \
127 ___ret = READ_ONCE(*raw_cpu_ptr(&(pcp))); \
128 preempt_enable_notrace(); \
129 ___ret; \
130})
131
132#define __this_cpu_generic_read_noirq(pcp) \
133({ \
134 typeof(pcp) ___ret; \
135 unsigned long ___flags; \
136 raw_local_irq_save(___flags); \
137 ___ret = raw_cpu_generic_read(pcp); \
138 raw_local_irq_restore(___flags); \
139 ___ret; \
140})
141
142#define this_cpu_generic_read(pcp) \
143({ \
144 typeof(pcp) __ret; \
145 if (__native_word(pcp)) \
146 __ret = __this_cpu_generic_read_nopreempt(pcp); \
147 else \
148 __ret = __this_cpu_generic_read_noirq(pcp); \
149 __ret; \
150})
151
152#define this_cpu_generic_to_op(pcp, val, op) \
153do { \
154 unsigned long __flags; \
155 raw_local_irq_save(__flags); \
156 raw_cpu_generic_to_op(pcp, val, op); \
157 raw_local_irq_restore(__flags); \
158} while (0)
159
160
161#define this_cpu_generic_add_return(pcp, val) \
162({ \
163 typeof(pcp) __ret; \
164 unsigned long __flags; \
165 raw_local_irq_save(__flags); \
166 __ret = raw_cpu_generic_add_return(pcp, val); \
167 raw_local_irq_restore(__flags); \
168 __ret; \
169})
170
171#define this_cpu_generic_xchg(pcp, nval) \
172({ \
173 typeof(pcp) __ret; \
174 unsigned long __flags; \
175 raw_local_irq_save(__flags); \
176 __ret = raw_cpu_generic_xchg(pcp, nval); \
177 raw_local_irq_restore(__flags); \
178 __ret; \
179})
180
181#define this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \
182({ \
183 bool __ret; \
184 unsigned long __flags; \
185 raw_local_irq_save(__flags); \
186 __ret = raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval); \
187 raw_local_irq_restore(__flags); \
188 __ret; \
189})
190
191#define this_cpu_generic_cmpxchg(pcp, oval, nval) \
192({ \
193 typeof(pcp) __ret; \
194 unsigned long __flags; \
195 raw_local_irq_save(__flags); \
196 __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval); \
197 raw_local_irq_restore(__flags); \
198 __ret; \
199})
200
201#ifndef raw_cpu_read_1
202#define raw_cpu_read_1(pcp) raw_cpu_generic_read(pcp)
203#endif
204#ifndef raw_cpu_read_2
205#define raw_cpu_read_2(pcp) raw_cpu_generic_read(pcp)
206#endif
207#ifndef raw_cpu_read_4
208#define raw_cpu_read_4(pcp) raw_cpu_generic_read(pcp)
209#endif
210#ifndef raw_cpu_read_8
211#define raw_cpu_read_8(pcp) raw_cpu_generic_read(pcp)
212#endif
213
214#ifndef raw_cpu_write_1
215#define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
216#endif
217#ifndef raw_cpu_write_2
218#define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
219#endif
220#ifndef raw_cpu_write_4
221#define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
222#endif
223#ifndef raw_cpu_write_8
224#define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
225#endif
226
227#ifndef raw_cpu_add_1
228#define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
229#endif
230#ifndef raw_cpu_add_2
231#define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
232#endif
233#ifndef raw_cpu_add_4
234#define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
235#endif
236#ifndef raw_cpu_add_8
237#define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
238#endif
239
240#ifndef raw_cpu_and_1
241#define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
242#endif
243#ifndef raw_cpu_and_2
244#define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
245#endif
246#ifndef raw_cpu_and_4
247#define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
248#endif
249#ifndef raw_cpu_and_8
250#define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
251#endif
252
253#ifndef raw_cpu_or_1
254#define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
255#endif
256#ifndef raw_cpu_or_2
257#define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
258#endif
259#ifndef raw_cpu_or_4
260#define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
261#endif
262#ifndef raw_cpu_or_8
263#define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
264#endif
265
266#ifndef raw_cpu_add_return_1
267#define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val)
268#endif
269#ifndef raw_cpu_add_return_2
270#define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val)
271#endif
272#ifndef raw_cpu_add_return_4
273#define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val)
274#endif
275#ifndef raw_cpu_add_return_8
276#define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val)
277#endif
278
279#ifndef raw_cpu_xchg_1
280#define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
281#endif
282#ifndef raw_cpu_xchg_2
283#define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
284#endif
285#ifndef raw_cpu_xchg_4
286#define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
287#endif
288#ifndef raw_cpu_xchg_8
289#define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
290#endif
291
292#ifndef raw_cpu_try_cmpxchg_1
293#ifdef raw_cpu_cmpxchg_1
294#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
295 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_1)
296#else
297#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
298 raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
299#endif
300#endif
301#ifndef raw_cpu_try_cmpxchg_2
302#ifdef raw_cpu_cmpxchg_2
303#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
304 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_2)
305#else
306#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
307 raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
308#endif
309#endif
310#ifndef raw_cpu_try_cmpxchg_4
311#ifdef raw_cpu_cmpxchg_4
312#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
313 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_4)
314#else
315#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
316 raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
317#endif
318#endif
319#ifndef raw_cpu_try_cmpxchg_8
320#ifdef raw_cpu_cmpxchg_8
321#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
322 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_8)
323#else
324#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
325 raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
326#endif
327#endif
328
329#ifndef raw_cpu_try_cmpxchg64
330#ifdef raw_cpu_cmpxchg64
331#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \
332 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg64)
333#else
334#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \
335 raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
336#endif
337#endif
338#ifndef raw_cpu_try_cmpxchg128
339#ifdef raw_cpu_cmpxchg128
340#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \
341 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg128)
342#else
343#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \
344 raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
345#endif
346#endif
347
348#ifndef raw_cpu_cmpxchg_1
349#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
350 raw_cpu_generic_cmpxchg(pcp, oval, nval)
351#endif
352#ifndef raw_cpu_cmpxchg_2
353#define raw_cpu_cmpxchg_2(pcp, oval, nval) \
354 raw_cpu_generic_cmpxchg(pcp, oval, nval)
355#endif
356#ifndef raw_cpu_cmpxchg_4
357#define raw_cpu_cmpxchg_4(pcp, oval, nval) \
358 raw_cpu_generic_cmpxchg(pcp, oval, nval)
359#endif
360#ifndef raw_cpu_cmpxchg_8
361#define raw_cpu_cmpxchg_8(pcp, oval, nval) \
362 raw_cpu_generic_cmpxchg(pcp, oval, nval)
363#endif
364
365#ifndef raw_cpu_cmpxchg64
366#define raw_cpu_cmpxchg64(pcp, oval, nval) \
367 raw_cpu_generic_cmpxchg(pcp, oval, nval)
368#endif
369#ifndef raw_cpu_cmpxchg128
370#define raw_cpu_cmpxchg128(pcp, oval, nval) \
371 raw_cpu_generic_cmpxchg(pcp, oval, nval)
372#endif
373
374#ifndef this_cpu_read_1
375#define this_cpu_read_1(pcp) this_cpu_generic_read(pcp)
376#endif
377#ifndef this_cpu_read_2
378#define this_cpu_read_2(pcp) this_cpu_generic_read(pcp)
379#endif
380#ifndef this_cpu_read_4
381#define this_cpu_read_4(pcp) this_cpu_generic_read(pcp)
382#endif
383#ifndef this_cpu_read_8
384#define this_cpu_read_8(pcp) this_cpu_generic_read(pcp)
385#endif
386
387#ifndef this_cpu_write_1
388#define this_cpu_write_1(pcp, val) this_cpu_generic_to_op(pcp, val, =)
389#endif
390#ifndef this_cpu_write_2
391#define this_cpu_write_2(pcp, val) this_cpu_generic_to_op(pcp, val, =)
392#endif
393#ifndef this_cpu_write_4
394#define this_cpu_write_4(pcp, val) this_cpu_generic_to_op(pcp, val, =)
395#endif
396#ifndef this_cpu_write_8
397#define this_cpu_write_8(pcp, val) this_cpu_generic_to_op(pcp, val, =)
398#endif
399
400#ifndef this_cpu_add_1
401#define this_cpu_add_1(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
402#endif
403#ifndef this_cpu_add_2
404#define this_cpu_add_2(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
405#endif
406#ifndef this_cpu_add_4
407#define this_cpu_add_4(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
408#endif
409#ifndef this_cpu_add_8
410#define this_cpu_add_8(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
411#endif
412
413#ifndef this_cpu_and_1
414#define this_cpu_and_1(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
415#endif
416#ifndef this_cpu_and_2
417#define this_cpu_and_2(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
418#endif
419#ifndef this_cpu_and_4
420#define this_cpu_and_4(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
421#endif
422#ifndef this_cpu_and_8
423#define this_cpu_and_8(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
424#endif
425
426#ifndef this_cpu_or_1
427#define this_cpu_or_1(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
428#endif
429#ifndef this_cpu_or_2
430#define this_cpu_or_2(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
431#endif
432#ifndef this_cpu_or_4
433#define this_cpu_or_4(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
434#endif
435#ifndef this_cpu_or_8
436#define this_cpu_or_8(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
437#endif
438
439#ifndef this_cpu_add_return_1
440#define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val)
441#endif
442#ifndef this_cpu_add_return_2
443#define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val)
444#endif
445#ifndef this_cpu_add_return_4
446#define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val)
447#endif
448#ifndef this_cpu_add_return_8
449#define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val)
450#endif
451
452#ifndef this_cpu_xchg_1
453#define this_cpu_xchg_1(pcp, nval) this_cpu_generic_xchg(pcp, nval)
454#endif
455#ifndef this_cpu_xchg_2
456#define this_cpu_xchg_2(pcp, nval) this_cpu_generic_xchg(pcp, nval)
457#endif
458#ifndef this_cpu_xchg_4
459#define this_cpu_xchg_4(pcp, nval) this_cpu_generic_xchg(pcp, nval)
460#endif
461#ifndef this_cpu_xchg_8
462#define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval)
463#endif
464
465#ifndef this_cpu_try_cmpxchg_1
466#ifdef this_cpu_cmpxchg_1
467#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
468 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_1)
469#else
470#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
471 this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
472#endif
473#endif
474#ifndef this_cpu_try_cmpxchg_2
475#ifdef this_cpu_cmpxchg_2
476#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
477 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_2)
478#else
479#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
480 this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
481#endif
482#endif
483#ifndef this_cpu_try_cmpxchg_4
484#ifdef this_cpu_cmpxchg_4
485#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
486 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_4)
487#else
488#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
489 this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
490#endif
491#endif
492#ifndef this_cpu_try_cmpxchg_8
493#ifdef this_cpu_cmpxchg_8
494#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
495 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_8)
496#else
497#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
498 this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
499#endif
500#endif
501
502#ifndef this_cpu_try_cmpxchg64
503#ifdef this_cpu_cmpxchg64
504#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \
505 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg64)
506#else
507#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \
508 this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
509#endif
510#endif
511#ifndef this_cpu_try_cmpxchg128
512#ifdef this_cpu_cmpxchg128
513#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \
514 __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg128)
515#else
516#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \
517 this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
518#endif
519#endif
520
521#ifndef this_cpu_cmpxchg_1
522#define this_cpu_cmpxchg_1(pcp, oval, nval) \
523 this_cpu_generic_cmpxchg(pcp, oval, nval)
524#endif
525#ifndef this_cpu_cmpxchg_2
526#define this_cpu_cmpxchg_2(pcp, oval, nval) \
527 this_cpu_generic_cmpxchg(pcp, oval, nval)
528#endif
529#ifndef this_cpu_cmpxchg_4
530#define this_cpu_cmpxchg_4(pcp, oval, nval) \
531 this_cpu_generic_cmpxchg(pcp, oval, nval)
532#endif
533#ifndef this_cpu_cmpxchg_8
534#define this_cpu_cmpxchg_8(pcp, oval, nval) \
535 this_cpu_generic_cmpxchg(pcp, oval, nval)
536#endif
537
538#ifndef this_cpu_cmpxchg64
539#define this_cpu_cmpxchg64(pcp, oval, nval) \
540 this_cpu_generic_cmpxchg(pcp, oval, nval)
541#endif
542#ifndef this_cpu_cmpxchg128
543#define this_cpu_cmpxchg128(pcp, oval, nval) \
544 this_cpu_generic_cmpxchg(pcp, oval, nval)
545#endif
546
547#endif /* _ASM_GENERIC_PERCPU_H_ */