Loading...
1#ifndef __ALPHA_SYSTEM_H
2#error Do not include xchg.h directly!
3#else
4/*
5 * xchg/xchg_local and cmpxchg/cmpxchg_local share the same code
6 * except that local version do not have the expensive memory barrier.
7 * So this file is included twice from asm/system.h.
8 */
9
10/*
11 * Atomic exchange.
12 * Since it can be used to implement critical sections
13 * it must clobber "memory" (also for interrupts in UP).
14 */
15
16static inline unsigned long
17____xchg(_u8, volatile char *m, unsigned long val)
18{
19 unsigned long ret, tmp, addr64;
20
21 __asm__ __volatile__(
22 " andnot %4,7,%3\n"
23 " insbl %1,%4,%1\n"
24 "1: ldq_l %2,0(%3)\n"
25 " extbl %2,%4,%0\n"
26 " mskbl %2,%4,%2\n"
27 " or %1,%2,%2\n"
28 " stq_c %2,0(%3)\n"
29 " beq %2,2f\n"
30 __ASM__MB
31 ".subsection 2\n"
32 "2: br 1b\n"
33 ".previous"
34 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
35 : "r" ((long)m), "1" (val) : "memory");
36
37 return ret;
38}
39
40static inline unsigned long
41____xchg(_u16, volatile short *m, unsigned long val)
42{
43 unsigned long ret, tmp, addr64;
44
45 __asm__ __volatile__(
46 " andnot %4,7,%3\n"
47 " inswl %1,%4,%1\n"
48 "1: ldq_l %2,0(%3)\n"
49 " extwl %2,%4,%0\n"
50 " mskwl %2,%4,%2\n"
51 " or %1,%2,%2\n"
52 " stq_c %2,0(%3)\n"
53 " beq %2,2f\n"
54 __ASM__MB
55 ".subsection 2\n"
56 "2: br 1b\n"
57 ".previous"
58 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
59 : "r" ((long)m), "1" (val) : "memory");
60
61 return ret;
62}
63
64static inline unsigned long
65____xchg(_u32, volatile int *m, unsigned long val)
66{
67 unsigned long dummy;
68
69 __asm__ __volatile__(
70 "1: ldl_l %0,%4\n"
71 " bis $31,%3,%1\n"
72 " stl_c %1,%2\n"
73 " beq %1,2f\n"
74 __ASM__MB
75 ".subsection 2\n"
76 "2: br 1b\n"
77 ".previous"
78 : "=&r" (val), "=&r" (dummy), "=m" (*m)
79 : "rI" (val), "m" (*m) : "memory");
80
81 return val;
82}
83
84static inline unsigned long
85____xchg(_u64, volatile long *m, unsigned long val)
86{
87 unsigned long dummy;
88
89 __asm__ __volatile__(
90 "1: ldq_l %0,%4\n"
91 " bis $31,%3,%1\n"
92 " stq_c %1,%2\n"
93 " beq %1,2f\n"
94 __ASM__MB
95 ".subsection 2\n"
96 "2: br 1b\n"
97 ".previous"
98 : "=&r" (val), "=&r" (dummy), "=m" (*m)
99 : "rI" (val), "m" (*m) : "memory");
100
101 return val;
102}
103
104/* This function doesn't exist, so you'll get a linker error
105 if something tries to do an invalid xchg(). */
106extern void __xchg_called_with_bad_pointer(void);
107
108static __always_inline unsigned long
109____xchg(, volatile void *ptr, unsigned long x, int size)
110{
111 switch (size) {
112 case 1:
113 return ____xchg(_u8, ptr, x);
114 case 2:
115 return ____xchg(_u16, ptr, x);
116 case 4:
117 return ____xchg(_u32, ptr, x);
118 case 8:
119 return ____xchg(_u64, ptr, x);
120 }
121 __xchg_called_with_bad_pointer();
122 return x;
123}
124
125/*
126 * Atomic compare and exchange. Compare OLD with MEM, if identical,
127 * store NEW in MEM. Return the initial value in MEM. Success is
128 * indicated by comparing RETURN with OLD.
129 *
130 * The memory barrier should be placed in SMP only when we actually
131 * make the change. If we don't change anything (so if the returned
132 * prev is equal to old) then we aren't acquiring anything new and
133 * we don't need any memory barrier as far I can tell.
134 */
135
136static inline unsigned long
137____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new)
138{
139 unsigned long prev, tmp, cmp, addr64;
140
141 __asm__ __volatile__(
142 " andnot %5,7,%4\n"
143 " insbl %1,%5,%1\n"
144 "1: ldq_l %2,0(%4)\n"
145 " extbl %2,%5,%0\n"
146 " cmpeq %0,%6,%3\n"
147 " beq %3,2f\n"
148 " mskbl %2,%5,%2\n"
149 " or %1,%2,%2\n"
150 " stq_c %2,0(%4)\n"
151 " beq %2,3f\n"
152 __ASM__MB
153 "2:\n"
154 ".subsection 2\n"
155 "3: br 1b\n"
156 ".previous"
157 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
158 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
159
160 return prev;
161}
162
163static inline unsigned long
164____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new)
165{
166 unsigned long prev, tmp, cmp, addr64;
167
168 __asm__ __volatile__(
169 " andnot %5,7,%4\n"
170 " inswl %1,%5,%1\n"
171 "1: ldq_l %2,0(%4)\n"
172 " extwl %2,%5,%0\n"
173 " cmpeq %0,%6,%3\n"
174 " beq %3,2f\n"
175 " mskwl %2,%5,%2\n"
176 " or %1,%2,%2\n"
177 " stq_c %2,0(%4)\n"
178 " beq %2,3f\n"
179 __ASM__MB
180 "2:\n"
181 ".subsection 2\n"
182 "3: br 1b\n"
183 ".previous"
184 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
185 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
186
187 return prev;
188}
189
190static inline unsigned long
191____cmpxchg(_u32, volatile int *m, int old, int new)
192{
193 unsigned long prev, cmp;
194
195 __asm__ __volatile__(
196 "1: ldl_l %0,%5\n"
197 " cmpeq %0,%3,%1\n"
198 " beq %1,2f\n"
199 " mov %4,%1\n"
200 " stl_c %1,%2\n"
201 " beq %1,3f\n"
202 __ASM__MB
203 "2:\n"
204 ".subsection 2\n"
205 "3: br 1b\n"
206 ".previous"
207 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
208 : "r"((long) old), "r"(new), "m"(*m) : "memory");
209
210 return prev;
211}
212
213static inline unsigned long
214____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new)
215{
216 unsigned long prev, cmp;
217
218 __asm__ __volatile__(
219 "1: ldq_l %0,%5\n"
220 " cmpeq %0,%3,%1\n"
221 " beq %1,2f\n"
222 " mov %4,%1\n"
223 " stq_c %1,%2\n"
224 " beq %1,3f\n"
225 __ASM__MB
226 "2:\n"
227 ".subsection 2\n"
228 "3: br 1b\n"
229 ".previous"
230 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
231 : "r"((long) old), "r"(new), "m"(*m) : "memory");
232
233 return prev;
234}
235
236/* This function doesn't exist, so you'll get a linker error
237 if something tries to do an invalid cmpxchg(). */
238extern void __cmpxchg_called_with_bad_pointer(void);
239
240static __always_inline unsigned long
241____cmpxchg(, volatile void *ptr, unsigned long old, unsigned long new,
242 int size)
243{
244 switch (size) {
245 case 1:
246 return ____cmpxchg(_u8, ptr, old, new);
247 case 2:
248 return ____cmpxchg(_u16, ptr, old, new);
249 case 4:
250 return ____cmpxchg(_u32, ptr, old, new);
251 case 8:
252 return ____cmpxchg(_u64, ptr, old, new);
253 }
254 __cmpxchg_called_with_bad_pointer();
255 return old;
256}
257
258#endif
1#ifndef _ALPHA_CMPXCHG_H
2#error Do not include xchg.h directly!
3#else
4/*
5 * xchg/xchg_local and cmpxchg/cmpxchg_local share the same code
6 * except that local version do not have the expensive memory barrier.
7 * So this file is included twice from asm/cmpxchg.h.
8 */
9
10/*
11 * Atomic exchange.
12 * Since it can be used to implement critical sections
13 * it must clobber "memory" (also for interrupts in UP).
14 */
15
16static inline unsigned long
17____xchg(_u8, volatile char *m, unsigned long val)
18{
19 unsigned long ret, tmp, addr64;
20
21 __asm__ __volatile__(
22 " andnot %4,7,%3\n"
23 " insbl %1,%4,%1\n"
24 "1: ldq_l %2,0(%3)\n"
25 " extbl %2,%4,%0\n"
26 " mskbl %2,%4,%2\n"
27 " or %1,%2,%2\n"
28 " stq_c %2,0(%3)\n"
29 " beq %2,2f\n"
30 __ASM__MB
31 ".subsection 2\n"
32 "2: br 1b\n"
33 ".previous"
34 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
35 : "r" ((long)m), "1" (val) : "memory");
36
37 return ret;
38}
39
40static inline unsigned long
41____xchg(_u16, volatile short *m, unsigned long val)
42{
43 unsigned long ret, tmp, addr64;
44
45 __asm__ __volatile__(
46 " andnot %4,7,%3\n"
47 " inswl %1,%4,%1\n"
48 "1: ldq_l %2,0(%3)\n"
49 " extwl %2,%4,%0\n"
50 " mskwl %2,%4,%2\n"
51 " or %1,%2,%2\n"
52 " stq_c %2,0(%3)\n"
53 " beq %2,2f\n"
54 __ASM__MB
55 ".subsection 2\n"
56 "2: br 1b\n"
57 ".previous"
58 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
59 : "r" ((long)m), "1" (val) : "memory");
60
61 return ret;
62}
63
64static inline unsigned long
65____xchg(_u32, volatile int *m, unsigned long val)
66{
67 unsigned long dummy;
68
69 __asm__ __volatile__(
70 "1: ldl_l %0,%4\n"
71 " bis $31,%3,%1\n"
72 " stl_c %1,%2\n"
73 " beq %1,2f\n"
74 __ASM__MB
75 ".subsection 2\n"
76 "2: br 1b\n"
77 ".previous"
78 : "=&r" (val), "=&r" (dummy), "=m" (*m)
79 : "rI" (val), "m" (*m) : "memory");
80
81 return val;
82}
83
84static inline unsigned long
85____xchg(_u64, volatile long *m, unsigned long val)
86{
87 unsigned long dummy;
88
89 __asm__ __volatile__(
90 "1: ldq_l %0,%4\n"
91 " bis $31,%3,%1\n"
92 " stq_c %1,%2\n"
93 " beq %1,2f\n"
94 __ASM__MB
95 ".subsection 2\n"
96 "2: br 1b\n"
97 ".previous"
98 : "=&r" (val), "=&r" (dummy), "=m" (*m)
99 : "rI" (val), "m" (*m) : "memory");
100
101 return val;
102}
103
104/* This function doesn't exist, so you'll get a linker error
105 if something tries to do an invalid xchg(). */
106extern void __xchg_called_with_bad_pointer(void);
107
108static __always_inline unsigned long
109____xchg(, volatile void *ptr, unsigned long x, int size)
110{
111 switch (size) {
112 case 1:
113 return ____xchg(_u8, ptr, x);
114 case 2:
115 return ____xchg(_u16, ptr, x);
116 case 4:
117 return ____xchg(_u32, ptr, x);
118 case 8:
119 return ____xchg(_u64, ptr, x);
120 }
121 __xchg_called_with_bad_pointer();
122 return x;
123}
124
125/*
126 * Atomic compare and exchange. Compare OLD with MEM, if identical,
127 * store NEW in MEM. Return the initial value in MEM. Success is
128 * indicated by comparing RETURN with OLD.
129 *
130 * The memory barrier should be placed in SMP only when we actually
131 * make the change. If we don't change anything (so if the returned
132 * prev is equal to old) then we aren't acquiring anything new and
133 * we don't need any memory barrier as far I can tell.
134 */
135
136static inline unsigned long
137____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new)
138{
139 unsigned long prev, tmp, cmp, addr64;
140
141 __asm__ __volatile__(
142 " andnot %5,7,%4\n"
143 " insbl %1,%5,%1\n"
144 "1: ldq_l %2,0(%4)\n"
145 " extbl %2,%5,%0\n"
146 " cmpeq %0,%6,%3\n"
147 " beq %3,2f\n"
148 " mskbl %2,%5,%2\n"
149 " or %1,%2,%2\n"
150 " stq_c %2,0(%4)\n"
151 " beq %2,3f\n"
152 __ASM__MB
153 "2:\n"
154 ".subsection 2\n"
155 "3: br 1b\n"
156 ".previous"
157 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
158 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
159
160 return prev;
161}
162
163static inline unsigned long
164____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new)
165{
166 unsigned long prev, tmp, cmp, addr64;
167
168 __asm__ __volatile__(
169 " andnot %5,7,%4\n"
170 " inswl %1,%5,%1\n"
171 "1: ldq_l %2,0(%4)\n"
172 " extwl %2,%5,%0\n"
173 " cmpeq %0,%6,%3\n"
174 " beq %3,2f\n"
175 " mskwl %2,%5,%2\n"
176 " or %1,%2,%2\n"
177 " stq_c %2,0(%4)\n"
178 " beq %2,3f\n"
179 __ASM__MB
180 "2:\n"
181 ".subsection 2\n"
182 "3: br 1b\n"
183 ".previous"
184 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
185 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
186
187 return prev;
188}
189
190static inline unsigned long
191____cmpxchg(_u32, volatile int *m, int old, int new)
192{
193 unsigned long prev, cmp;
194
195 __asm__ __volatile__(
196 "1: ldl_l %0,%5\n"
197 " cmpeq %0,%3,%1\n"
198 " beq %1,2f\n"
199 " mov %4,%1\n"
200 " stl_c %1,%2\n"
201 " beq %1,3f\n"
202 __ASM__MB
203 "2:\n"
204 ".subsection 2\n"
205 "3: br 1b\n"
206 ".previous"
207 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
208 : "r"((long) old), "r"(new), "m"(*m) : "memory");
209
210 return prev;
211}
212
213static inline unsigned long
214____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new)
215{
216 unsigned long prev, cmp;
217
218 __asm__ __volatile__(
219 "1: ldq_l %0,%5\n"
220 " cmpeq %0,%3,%1\n"
221 " beq %1,2f\n"
222 " mov %4,%1\n"
223 " stq_c %1,%2\n"
224 " beq %1,3f\n"
225 __ASM__MB
226 "2:\n"
227 ".subsection 2\n"
228 "3: br 1b\n"
229 ".previous"
230 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
231 : "r"((long) old), "r"(new), "m"(*m) : "memory");
232
233 return prev;
234}
235
236/* This function doesn't exist, so you'll get a linker error
237 if something tries to do an invalid cmpxchg(). */
238extern void __cmpxchg_called_with_bad_pointer(void);
239
240static __always_inline unsigned long
241____cmpxchg(, volatile void *ptr, unsigned long old, unsigned long new,
242 int size)
243{
244 switch (size) {
245 case 1:
246 return ____cmpxchg(_u8, ptr, old, new);
247 case 2:
248 return ____cmpxchg(_u16, ptr, old, new);
249 case 4:
250 return ____cmpxchg(_u32, ptr, old, new);
251 case 8:
252 return ____cmpxchg(_u64, ptr, old, new);
253 }
254 __cmpxchg_called_with_bad_pointer();
255 return old;
256}
257
258#endif