Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __ARCH_S390_PERCPU__
3#define __ARCH_S390_PERCPU__
4
5#include <linux/preempt.h>
6#include <asm/cmpxchg.h>
7
8/*
9 * s390 uses its own implementation for per cpu data, the offset of
10 * the cpu local data area is cached in the cpu's lowcore memory.
11 */
12#define __my_cpu_offset S390_lowcore.percpu_offset
13
14/*
15 * For 64 bit module code, the module may be more than 4G above the
16 * per cpu area, use weak definitions to force the compiler to
17 * generate external references.
18 */
19#if defined(MODULE)
20#define ARCH_NEEDS_WEAK_PER_CPU
21#endif
22
23/*
24 * We use a compare-and-swap loop since that uses less cpu cycles than
25 * disabling and enabling interrupts like the generic variant would do.
26 */
27#define arch_this_cpu_to_op_simple(pcp, val, op) \
28({ \
29 typedef typeof(pcp) pcp_op_T__; \
30 pcp_op_T__ old__, new__, prev__; \
31 pcp_op_T__ *ptr__; \
32 preempt_disable_notrace(); \
33 ptr__ = raw_cpu_ptr(&(pcp)); \
34 prev__ = READ_ONCE(*ptr__); \
35 do { \
36 old__ = prev__; \
37 new__ = old__ op (val); \
38 prev__ = cmpxchg(ptr__, old__, new__); \
39 } while (prev__ != old__); \
40 preempt_enable_notrace(); \
41 new__; \
42})
43
44#define this_cpu_add_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
45#define this_cpu_add_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
46#define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
47#define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
48#define this_cpu_and_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
49#define this_cpu_and_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
50#define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
51#define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
52
53#ifndef CONFIG_HAVE_MARCH_Z196_FEATURES
54
55#define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
56#define this_cpu_add_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
57#define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
58#define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
59#define this_cpu_and_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
60#define this_cpu_and_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
61#define this_cpu_or_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
62#define this_cpu_or_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
63
64#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
65
66#define arch_this_cpu_add(pcp, val, op1, op2, szcast) \
67{ \
68 typedef typeof(pcp) pcp_op_T__; \
69 pcp_op_T__ val__ = (val); \
70 pcp_op_T__ old__, *ptr__; \
71 preempt_disable_notrace(); \
72 ptr__ = raw_cpu_ptr(&(pcp)); \
73 if (__builtin_constant_p(val__) && \
74 ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \
75 asm volatile( \
76 op2 " %[ptr__],%[val__]\n" \
77 : [ptr__] "+Q" (*ptr__) \
78 : [val__] "i" ((szcast)val__) \
79 : "cc"); \
80 } else { \
81 asm volatile( \
82 op1 " %[old__],%[val__],%[ptr__]\n" \
83 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \
84 : [val__] "d" (val__) \
85 : "cc"); \
86 } \
87 preempt_enable_notrace(); \
88}
89
90#define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int)
91#define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long)
92
93#define arch_this_cpu_add_return(pcp, val, op) \
94({ \
95 typedef typeof(pcp) pcp_op_T__; \
96 pcp_op_T__ val__ = (val); \
97 pcp_op_T__ old__, *ptr__; \
98 preempt_disable_notrace(); \
99 ptr__ = raw_cpu_ptr(&(pcp)); \
100 asm volatile( \
101 op " %[old__],%[val__],%[ptr__]\n" \
102 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \
103 : [val__] "d" (val__) \
104 : "cc"); \
105 preempt_enable_notrace(); \
106 old__ + val__; \
107})
108
109#define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa")
110#define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag")
111
112#define arch_this_cpu_to_op(pcp, val, op) \
113{ \
114 typedef typeof(pcp) pcp_op_T__; \
115 pcp_op_T__ val__ = (val); \
116 pcp_op_T__ old__, *ptr__; \
117 preempt_disable_notrace(); \
118 ptr__ = raw_cpu_ptr(&(pcp)); \
119 asm volatile( \
120 op " %[old__],%[val__],%[ptr__]\n" \
121 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \
122 : [val__] "d" (val__) \
123 : "cc"); \
124 preempt_enable_notrace(); \
125}
126
127#define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan")
128#define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang")
129#define this_cpu_or_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lao")
130#define this_cpu_or_8(pcp, val) arch_this_cpu_to_op(pcp, val, "laog")
131
132#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
133
134#define arch_this_cpu_cmpxchg(pcp, oval, nval) \
135({ \
136 typedef typeof(pcp) pcp_op_T__; \
137 pcp_op_T__ ret__; \
138 pcp_op_T__ *ptr__; \
139 preempt_disable_notrace(); \
140 ptr__ = raw_cpu_ptr(&(pcp)); \
141 ret__ = cmpxchg(ptr__, oval, nval); \
142 preempt_enable_notrace(); \
143 ret__; \
144})
145
146#define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
147#define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
148#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
149#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
150
151#define arch_this_cpu_xchg(pcp, nval) \
152({ \
153 typeof(pcp) *ptr__; \
154 typeof(pcp) ret__; \
155 preempt_disable_notrace(); \
156 ptr__ = raw_cpu_ptr(&(pcp)); \
157 ret__ = xchg(ptr__, nval); \
158 preempt_enable_notrace(); \
159 ret__; \
160})
161
162#define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval)
163#define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval)
164#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)
165#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)
166
167#define arch_this_cpu_cmpxchg_double(pcp1, pcp2, o1, o2, n1, n2) \
168({ \
169 typeof(pcp1) *p1__; \
170 typeof(pcp2) *p2__; \
171 int ret__; \
172 \
173 preempt_disable_notrace(); \
174 p1__ = raw_cpu_ptr(&(pcp1)); \
175 p2__ = raw_cpu_ptr(&(pcp2)); \
176 ret__ = __cmpxchg_double((unsigned long)p1__, (unsigned long)p2__, \
177 (unsigned long)(o1), (unsigned long)(o2), \
178 (unsigned long)(n1), (unsigned long)(n2)); \
179 preempt_enable_notrace(); \
180 ret__; \
181})
182
183#define this_cpu_cmpxchg_double_8 arch_this_cpu_cmpxchg_double
184
185#include <asm-generic/percpu.h>
186
187#endif /* __ARCH_S390_PERCPU__ */
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __ARCH_S390_PERCPU__
3#define __ARCH_S390_PERCPU__
4
5#include <linux/preempt.h>
6#include <asm/cmpxchg.h>
7#include <asm/march.h>
8
9/*
10 * s390 uses its own implementation for per cpu data, the offset of
11 * the cpu local data area is cached in the cpu's lowcore memory.
12 */
13#define __my_cpu_offset get_lowcore()->percpu_offset
14
15/*
16 * For 64 bit module code, the module may be more than 4G above the
17 * per cpu area, use weak definitions to force the compiler to
18 * generate external references.
19 */
20#if defined(MODULE)
21#define ARCH_NEEDS_WEAK_PER_CPU
22#endif
23
24/*
25 * We use a compare-and-swap loop since that uses less cpu cycles than
26 * disabling and enabling interrupts like the generic variant would do.
27 */
28#define arch_this_cpu_to_op_simple(pcp, val, op) \
29({ \
30 typedef typeof(pcp) pcp_op_T__; \
31 pcp_op_T__ old__, new__, prev__; \
32 pcp_op_T__ *ptr__; \
33 preempt_disable_notrace(); \
34 ptr__ = raw_cpu_ptr(&(pcp)); \
35 prev__ = READ_ONCE(*ptr__); \
36 do { \
37 old__ = prev__; \
38 new__ = old__ op (val); \
39 prev__ = cmpxchg(ptr__, old__, new__); \
40 } while (prev__ != old__); \
41 preempt_enable_notrace(); \
42 new__; \
43})
44
45#define this_cpu_add_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
46#define this_cpu_add_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
47#define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
48#define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
49#define this_cpu_and_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
50#define this_cpu_and_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
51#define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
52#define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
53
54#ifndef MARCH_HAS_Z196_FEATURES
55
56#define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
57#define this_cpu_add_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
58#define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
59#define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
60#define this_cpu_and_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
61#define this_cpu_and_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
62#define this_cpu_or_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
63#define this_cpu_or_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
64
65#else /* MARCH_HAS_Z196_FEATURES */
66
67#define arch_this_cpu_add(pcp, val, op1, op2, szcast) \
68{ \
69 typedef typeof(pcp) pcp_op_T__; \
70 pcp_op_T__ val__ = (val); \
71 pcp_op_T__ old__, *ptr__; \
72 preempt_disable_notrace(); \
73 ptr__ = raw_cpu_ptr(&(pcp)); \
74 if (__builtin_constant_p(val__) && \
75 ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \
76 asm volatile( \
77 op2 " %[ptr__],%[val__]\n" \
78 : [ptr__] "+Q" (*ptr__) \
79 : [val__] "i" ((szcast)val__) \
80 : "cc"); \
81 } else { \
82 asm volatile( \
83 op1 " %[old__],%[val__],%[ptr__]\n" \
84 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \
85 : [val__] "d" (val__) \
86 : "cc"); \
87 } \
88 preempt_enable_notrace(); \
89}
90
91#define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int)
92#define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long)
93
94#define arch_this_cpu_add_return(pcp, val, op) \
95({ \
96 typedef typeof(pcp) pcp_op_T__; \
97 pcp_op_T__ val__ = (val); \
98 pcp_op_T__ old__, *ptr__; \
99 preempt_disable_notrace(); \
100 ptr__ = raw_cpu_ptr(&(pcp)); \
101 asm volatile( \
102 op " %[old__],%[val__],%[ptr__]\n" \
103 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \
104 : [val__] "d" (val__) \
105 : "cc"); \
106 preempt_enable_notrace(); \
107 old__ + val__; \
108})
109
110#define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa")
111#define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag")
112
113#define arch_this_cpu_to_op(pcp, val, op) \
114{ \
115 typedef typeof(pcp) pcp_op_T__; \
116 pcp_op_T__ val__ = (val); \
117 pcp_op_T__ old__, *ptr__; \
118 preempt_disable_notrace(); \
119 ptr__ = raw_cpu_ptr(&(pcp)); \
120 asm volatile( \
121 op " %[old__],%[val__],%[ptr__]\n" \
122 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \
123 : [val__] "d" (val__) \
124 : "cc"); \
125 preempt_enable_notrace(); \
126}
127
128#define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan")
129#define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang")
130#define this_cpu_or_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lao")
131#define this_cpu_or_8(pcp, val) arch_this_cpu_to_op(pcp, val, "laog")
132
133#endif /* MARCH_HAS_Z196_FEATURES */
134
135#define arch_this_cpu_cmpxchg(pcp, oval, nval) \
136({ \
137 typedef typeof(pcp) pcp_op_T__; \
138 pcp_op_T__ ret__; \
139 pcp_op_T__ *ptr__; \
140 preempt_disable_notrace(); \
141 ptr__ = raw_cpu_ptr(&(pcp)); \
142 ret__ = cmpxchg(ptr__, oval, nval); \
143 preempt_enable_notrace(); \
144 ret__; \
145})
146
147#define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
148#define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
149#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
150#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
151
152#define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n)
153
154#define this_cpu_cmpxchg128(pcp, oval, nval) \
155({ \
156 typedef typeof(pcp) pcp_op_T__; \
157 u128 old__, new__, ret__; \
158 pcp_op_T__ *ptr__; \
159 old__ = oval; \
160 new__ = nval; \
161 preempt_disable_notrace(); \
162 ptr__ = raw_cpu_ptr(&(pcp)); \
163 ret__ = cmpxchg128((void *)ptr__, old__, new__); \
164 preempt_enable_notrace(); \
165 ret__; \
166})
167
168#define arch_this_cpu_xchg(pcp, nval) \
169({ \
170 typeof(pcp) *ptr__; \
171 typeof(pcp) ret__; \
172 preempt_disable_notrace(); \
173 ptr__ = raw_cpu_ptr(&(pcp)); \
174 ret__ = xchg(ptr__, nval); \
175 preempt_enable_notrace(); \
176 ret__; \
177})
178
179#define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval)
180#define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval)
181#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)
182#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)
183
184#include <asm-generic/percpu.h>
185
186#endif /* __ARCH_S390_PERCPU__ */