Loading...
1#ifndef __ALPHA_SYSTEM_H
2#error Do not include xchg.h directly!
3#else
4/*
5 * xchg/xchg_local and cmpxchg/cmpxchg_local share the same code
6 * except that local version do not have the expensive memory barrier.
7 * So this file is included twice from asm/system.h.
8 */
9
10/*
11 * Atomic exchange.
12 * Since it can be used to implement critical sections
13 * it must clobber "memory" (also for interrupts in UP).
14 */
15
16static inline unsigned long
17____xchg(_u8, volatile char *m, unsigned long val)
18{
19 unsigned long ret, tmp, addr64;
20
21 __asm__ __volatile__(
22 " andnot %4,7,%3\n"
23 " insbl %1,%4,%1\n"
24 "1: ldq_l %2,0(%3)\n"
25 " extbl %2,%4,%0\n"
26 " mskbl %2,%4,%2\n"
27 " or %1,%2,%2\n"
28 " stq_c %2,0(%3)\n"
29 " beq %2,2f\n"
30 __ASM__MB
31 ".subsection 2\n"
32 "2: br 1b\n"
33 ".previous"
34 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
35 : "r" ((long)m), "1" (val) : "memory");
36
37 return ret;
38}
39
40static inline unsigned long
41____xchg(_u16, volatile short *m, unsigned long val)
42{
43 unsigned long ret, tmp, addr64;
44
45 __asm__ __volatile__(
46 " andnot %4,7,%3\n"
47 " inswl %1,%4,%1\n"
48 "1: ldq_l %2,0(%3)\n"
49 " extwl %2,%4,%0\n"
50 " mskwl %2,%4,%2\n"
51 " or %1,%2,%2\n"
52 " stq_c %2,0(%3)\n"
53 " beq %2,2f\n"
54 __ASM__MB
55 ".subsection 2\n"
56 "2: br 1b\n"
57 ".previous"
58 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
59 : "r" ((long)m), "1" (val) : "memory");
60
61 return ret;
62}
63
64static inline unsigned long
65____xchg(_u32, volatile int *m, unsigned long val)
66{
67 unsigned long dummy;
68
69 __asm__ __volatile__(
70 "1: ldl_l %0,%4\n"
71 " bis $31,%3,%1\n"
72 " stl_c %1,%2\n"
73 " beq %1,2f\n"
74 __ASM__MB
75 ".subsection 2\n"
76 "2: br 1b\n"
77 ".previous"
78 : "=&r" (val), "=&r" (dummy), "=m" (*m)
79 : "rI" (val), "m" (*m) : "memory");
80
81 return val;
82}
83
84static inline unsigned long
85____xchg(_u64, volatile long *m, unsigned long val)
86{
87 unsigned long dummy;
88
89 __asm__ __volatile__(
90 "1: ldq_l %0,%4\n"
91 " bis $31,%3,%1\n"
92 " stq_c %1,%2\n"
93 " beq %1,2f\n"
94 __ASM__MB
95 ".subsection 2\n"
96 "2: br 1b\n"
97 ".previous"
98 : "=&r" (val), "=&r" (dummy), "=m" (*m)
99 : "rI" (val), "m" (*m) : "memory");
100
101 return val;
102}
103
104/* This function doesn't exist, so you'll get a linker error
105 if something tries to do an invalid xchg(). */
106extern void __xchg_called_with_bad_pointer(void);
107
108static __always_inline unsigned long
109____xchg(, volatile void *ptr, unsigned long x, int size)
110{
111 switch (size) {
112 case 1:
113 return ____xchg(_u8, ptr, x);
114 case 2:
115 return ____xchg(_u16, ptr, x);
116 case 4:
117 return ____xchg(_u32, ptr, x);
118 case 8:
119 return ____xchg(_u64, ptr, x);
120 }
121 __xchg_called_with_bad_pointer();
122 return x;
123}
124
125/*
126 * Atomic compare and exchange. Compare OLD with MEM, if identical,
127 * store NEW in MEM. Return the initial value in MEM. Success is
128 * indicated by comparing RETURN with OLD.
129 *
130 * The memory barrier should be placed in SMP only when we actually
131 * make the change. If we don't change anything (so if the returned
132 * prev is equal to old) then we aren't acquiring anything new and
133 * we don't need any memory barrier as far I can tell.
134 */
135
136static inline unsigned long
137____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new)
138{
139 unsigned long prev, tmp, cmp, addr64;
140
141 __asm__ __volatile__(
142 " andnot %5,7,%4\n"
143 " insbl %1,%5,%1\n"
144 "1: ldq_l %2,0(%4)\n"
145 " extbl %2,%5,%0\n"
146 " cmpeq %0,%6,%3\n"
147 " beq %3,2f\n"
148 " mskbl %2,%5,%2\n"
149 " or %1,%2,%2\n"
150 " stq_c %2,0(%4)\n"
151 " beq %2,3f\n"
152 __ASM__MB
153 "2:\n"
154 ".subsection 2\n"
155 "3: br 1b\n"
156 ".previous"
157 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
158 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
159
160 return prev;
161}
162
163static inline unsigned long
164____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new)
165{
166 unsigned long prev, tmp, cmp, addr64;
167
168 __asm__ __volatile__(
169 " andnot %5,7,%4\n"
170 " inswl %1,%5,%1\n"
171 "1: ldq_l %2,0(%4)\n"
172 " extwl %2,%5,%0\n"
173 " cmpeq %0,%6,%3\n"
174 " beq %3,2f\n"
175 " mskwl %2,%5,%2\n"
176 " or %1,%2,%2\n"
177 " stq_c %2,0(%4)\n"
178 " beq %2,3f\n"
179 __ASM__MB
180 "2:\n"
181 ".subsection 2\n"
182 "3: br 1b\n"
183 ".previous"
184 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
185 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
186
187 return prev;
188}
189
190static inline unsigned long
191____cmpxchg(_u32, volatile int *m, int old, int new)
192{
193 unsigned long prev, cmp;
194
195 __asm__ __volatile__(
196 "1: ldl_l %0,%5\n"
197 " cmpeq %0,%3,%1\n"
198 " beq %1,2f\n"
199 " mov %4,%1\n"
200 " stl_c %1,%2\n"
201 " beq %1,3f\n"
202 __ASM__MB
203 "2:\n"
204 ".subsection 2\n"
205 "3: br 1b\n"
206 ".previous"
207 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
208 : "r"((long) old), "r"(new), "m"(*m) : "memory");
209
210 return prev;
211}
212
213static inline unsigned long
214____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new)
215{
216 unsigned long prev, cmp;
217
218 __asm__ __volatile__(
219 "1: ldq_l %0,%5\n"
220 " cmpeq %0,%3,%1\n"
221 " beq %1,2f\n"
222 " mov %4,%1\n"
223 " stq_c %1,%2\n"
224 " beq %1,3f\n"
225 __ASM__MB
226 "2:\n"
227 ".subsection 2\n"
228 "3: br 1b\n"
229 ".previous"
230 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
231 : "r"((long) old), "r"(new), "m"(*m) : "memory");
232
233 return prev;
234}
235
236/* This function doesn't exist, so you'll get a linker error
237 if something tries to do an invalid cmpxchg(). */
238extern void __cmpxchg_called_with_bad_pointer(void);
239
240static __always_inline unsigned long
241____cmpxchg(, volatile void *ptr, unsigned long old, unsigned long new,
242 int size)
243{
244 switch (size) {
245 case 1:
246 return ____cmpxchg(_u8, ptr, old, new);
247 case 2:
248 return ____cmpxchg(_u16, ptr, old, new);
249 case 4:
250 return ____cmpxchg(_u32, ptr, old, new);
251 case 8:
252 return ____cmpxchg(_u64, ptr, old, new);
253 }
254 __cmpxchg_called_with_bad_pointer();
255 return old;
256}
257
258#endif
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ALPHA_CMPXCHG_H
3#error Do not include xchg.h directly!
4#else
5/*
6 * xchg/xchg_local and cmpxchg/cmpxchg_local share the same code
7 * except that local version do not have the expensive memory barrier.
8 * So this file is included twice from asm/cmpxchg.h.
9 */
10
11/*
12 * Atomic exchange.
13 * Since it can be used to implement critical sections
14 * it must clobber "memory" (also for interrupts in UP).
15 */
16
17static inline unsigned long
18____xchg(_u8, volatile char *m, unsigned long val)
19{
20 unsigned long ret, tmp, addr64;
21
22 __asm__ __volatile__(
23 " andnot %4,7,%3\n"
24 " insbl %1,%4,%1\n"
25 "1: ldq_l %2,0(%3)\n"
26 " extbl %2,%4,%0\n"
27 " mskbl %2,%4,%2\n"
28 " or %1,%2,%2\n"
29 " stq_c %2,0(%3)\n"
30 " beq %2,2f\n"
31 ".subsection 2\n"
32 "2: br 1b\n"
33 ".previous"
34 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
35 : "r" ((long)m), "1" (val) : "memory");
36
37 return ret;
38}
39
40static inline unsigned long
41____xchg(_u16, volatile short *m, unsigned long val)
42{
43 unsigned long ret, tmp, addr64;
44
45 __asm__ __volatile__(
46 " andnot %4,7,%3\n"
47 " inswl %1,%4,%1\n"
48 "1: ldq_l %2,0(%3)\n"
49 " extwl %2,%4,%0\n"
50 " mskwl %2,%4,%2\n"
51 " or %1,%2,%2\n"
52 " stq_c %2,0(%3)\n"
53 " beq %2,2f\n"
54 ".subsection 2\n"
55 "2: br 1b\n"
56 ".previous"
57 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
58 : "r" ((long)m), "1" (val) : "memory");
59
60 return ret;
61}
62
63static inline unsigned long
64____xchg(_u32, volatile int *m, unsigned long val)
65{
66 unsigned long dummy;
67
68 __asm__ __volatile__(
69 "1: ldl_l %0,%4\n"
70 " bis $31,%3,%1\n"
71 " stl_c %1,%2\n"
72 " beq %1,2f\n"
73 ".subsection 2\n"
74 "2: br 1b\n"
75 ".previous"
76 : "=&r" (val), "=&r" (dummy), "=m" (*m)
77 : "rI" (val), "m" (*m) : "memory");
78
79 return val;
80}
81
82static inline unsigned long
83____xchg(_u64, volatile long *m, unsigned long val)
84{
85 unsigned long dummy;
86
87 __asm__ __volatile__(
88 "1: ldq_l %0,%4\n"
89 " bis $31,%3,%1\n"
90 " stq_c %1,%2\n"
91 " beq %1,2f\n"
92 ".subsection 2\n"
93 "2: br 1b\n"
94 ".previous"
95 : "=&r" (val), "=&r" (dummy), "=m" (*m)
96 : "rI" (val), "m" (*m) : "memory");
97
98 return val;
99}
100
101/* This function doesn't exist, so you'll get a linker error
102 if something tries to do an invalid xchg(). */
103extern void __xchg_called_with_bad_pointer(void);
104
105static __always_inline unsigned long
106____xchg(, volatile void *ptr, unsigned long x, int size)
107{
108 switch (size) {
109 case 1:
110 return ____xchg(_u8, ptr, x);
111 case 2:
112 return ____xchg(_u16, ptr, x);
113 case 4:
114 return ____xchg(_u32, ptr, x);
115 case 8:
116 return ____xchg(_u64, ptr, x);
117 }
118 __xchg_called_with_bad_pointer();
119 return x;
120}
121
122/*
123 * Atomic compare and exchange. Compare OLD with MEM, if identical,
124 * store NEW in MEM. Return the initial value in MEM. Success is
125 * indicated by comparing RETURN with OLD.
126 */
127
128static inline unsigned long
129____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new)
130{
131 unsigned long prev, tmp, cmp, addr64;
132
133 __asm__ __volatile__(
134 " andnot %5,7,%4\n"
135 " insbl %1,%5,%1\n"
136 "1: ldq_l %2,0(%4)\n"
137 " extbl %2,%5,%0\n"
138 " cmpeq %0,%6,%3\n"
139 " beq %3,2f\n"
140 " mskbl %2,%5,%2\n"
141 " or %1,%2,%2\n"
142 " stq_c %2,0(%4)\n"
143 " beq %2,3f\n"
144 "2:\n"
145 ".subsection 2\n"
146 "3: br 1b\n"
147 ".previous"
148 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
149 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
150
151 return prev;
152}
153
154static inline unsigned long
155____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new)
156{
157 unsigned long prev, tmp, cmp, addr64;
158
159 __asm__ __volatile__(
160 " andnot %5,7,%4\n"
161 " inswl %1,%5,%1\n"
162 "1: ldq_l %2,0(%4)\n"
163 " extwl %2,%5,%0\n"
164 " cmpeq %0,%6,%3\n"
165 " beq %3,2f\n"
166 " mskwl %2,%5,%2\n"
167 " or %1,%2,%2\n"
168 " stq_c %2,0(%4)\n"
169 " beq %2,3f\n"
170 "2:\n"
171 ".subsection 2\n"
172 "3: br 1b\n"
173 ".previous"
174 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
175 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
176
177 return prev;
178}
179
180static inline unsigned long
181____cmpxchg(_u32, volatile int *m, int old, int new)
182{
183 unsigned long prev, cmp;
184
185 __asm__ __volatile__(
186 "1: ldl_l %0,%5\n"
187 " cmpeq %0,%3,%1\n"
188 " beq %1,2f\n"
189 " mov %4,%1\n"
190 " stl_c %1,%2\n"
191 " beq %1,3f\n"
192 "2:\n"
193 ".subsection 2\n"
194 "3: br 1b\n"
195 ".previous"
196 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
197 : "r"((long) old), "r"(new), "m"(*m) : "memory");
198
199 return prev;
200}
201
202static inline unsigned long
203____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new)
204{
205 unsigned long prev, cmp;
206
207 __asm__ __volatile__(
208 "1: ldq_l %0,%5\n"
209 " cmpeq %0,%3,%1\n"
210 " beq %1,2f\n"
211 " mov %4,%1\n"
212 " stq_c %1,%2\n"
213 " beq %1,3f\n"
214 "2:\n"
215 ".subsection 2\n"
216 "3: br 1b\n"
217 ".previous"
218 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
219 : "r"((long) old), "r"(new), "m"(*m) : "memory");
220
221 return prev;
222}
223
224/* This function doesn't exist, so you'll get a linker error
225 if something tries to do an invalid cmpxchg(). */
226extern void __cmpxchg_called_with_bad_pointer(void);
227
228static __always_inline unsigned long
229____cmpxchg(, volatile void *ptr, unsigned long old, unsigned long new,
230 int size)
231{
232 switch (size) {
233 case 1:
234 return ____cmpxchg(_u8, ptr, old, new);
235 case 2:
236 return ____cmpxchg(_u16, ptr, old, new);
237 case 4:
238 return ____cmpxchg(_u32, ptr, old, new);
239 case 8:
240 return ____cmpxchg(_u64, ptr, old, new);
241 }
242 __cmpxchg_called_with_bad_pointer();
243 return old;
244}
245
246#endif