Loading...
1#ifndef _ASM_POWERPC_ATOMIC_H_
2#define _ASM_POWERPC_ATOMIC_H_
3
4/*
5 * PowerPC atomic operations
6 */
7
8#ifdef __KERNEL__
9#include <linux/types.h>
10#include <asm/cmpxchg.h>
11#include <asm/barrier.h>
12
13#define ATOMIC_INIT(i) { (i) }
14
15/*
16 * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with
17 * a "bne-" instruction at the end, so an isync is enough as a acquire barrier
18 * on the platform without lwsync.
19 */
20#define __atomic_op_acquire(op, args...) \
21({ \
22 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \
23 __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory"); \
24 __ret; \
25})
26
27#define __atomic_op_release(op, args...) \
28({ \
29 __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory"); \
30 op##_relaxed(args); \
31})
32
33static __inline__ int atomic_read(const atomic_t *v)
34{
35 int t;
36
37 __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter));
38
39 return t;
40}
41
42static __inline__ void atomic_set(atomic_t *v, int i)
43{
44 __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
45}
46
47#define ATOMIC_OP(op, asm_op) \
48static __inline__ void atomic_##op(int a, atomic_t *v) \
49{ \
50 int t; \
51 \
52 __asm__ __volatile__( \
53"1: lwarx %0,0,%3 # atomic_" #op "\n" \
54 #asm_op " %0,%2,%0\n" \
55 PPC405_ERR77(0,%3) \
56" stwcx. %0,0,%3 \n" \
57" bne- 1b\n" \
58 : "=&r" (t), "+m" (v->counter) \
59 : "r" (a), "r" (&v->counter) \
60 : "cc"); \
61} \
62
63#define ATOMIC_OP_RETURN_RELAXED(op, asm_op) \
64static inline int atomic_##op##_return_relaxed(int a, atomic_t *v) \
65{ \
66 int t; \
67 \
68 __asm__ __volatile__( \
69"1: lwarx %0,0,%3 # atomic_" #op "_return_relaxed\n" \
70 #asm_op " %0,%2,%0\n" \
71 PPC405_ERR77(0, %3) \
72" stwcx. %0,0,%3\n" \
73" bne- 1b\n" \
74 : "=&r" (t), "+m" (v->counter) \
75 : "r" (a), "r" (&v->counter) \
76 : "cc"); \
77 \
78 return t; \
79}
80
81#define ATOMIC_OPS(op, asm_op) \
82 ATOMIC_OP(op, asm_op) \
83 ATOMIC_OP_RETURN_RELAXED(op, asm_op)
84
85ATOMIC_OPS(add, add)
86ATOMIC_OPS(sub, subf)
87
88ATOMIC_OP(and, and)
89ATOMIC_OP(or, or)
90ATOMIC_OP(xor, xor)
91
92#define atomic_add_return_relaxed atomic_add_return_relaxed
93#define atomic_sub_return_relaxed atomic_sub_return_relaxed
94
95#undef ATOMIC_OPS
96#undef ATOMIC_OP_RETURN_RELAXED
97#undef ATOMIC_OP
98
99#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0)
100
101static __inline__ void atomic_inc(atomic_t *v)
102{
103 int t;
104
105 __asm__ __volatile__(
106"1: lwarx %0,0,%2 # atomic_inc\n\
107 addic %0,%0,1\n"
108 PPC405_ERR77(0,%2)
109" stwcx. %0,0,%2 \n\
110 bne- 1b"
111 : "=&r" (t), "+m" (v->counter)
112 : "r" (&v->counter)
113 : "cc", "xer");
114}
115
116static __inline__ int atomic_inc_return_relaxed(atomic_t *v)
117{
118 int t;
119
120 __asm__ __volatile__(
121"1: lwarx %0,0,%2 # atomic_inc_return_relaxed\n"
122" addic %0,%0,1\n"
123 PPC405_ERR77(0, %2)
124" stwcx. %0,0,%2\n"
125" bne- 1b"
126 : "=&r" (t), "+m" (v->counter)
127 : "r" (&v->counter)
128 : "cc", "xer");
129
130 return t;
131}
132
133/*
134 * atomic_inc_and_test - increment and test
135 * @v: pointer of type atomic_t
136 *
137 * Atomically increments @v by 1
138 * and returns true if the result is zero, or false for all
139 * other cases.
140 */
141#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
142
143static __inline__ void atomic_dec(atomic_t *v)
144{
145 int t;
146
147 __asm__ __volatile__(
148"1: lwarx %0,0,%2 # atomic_dec\n\
149 addic %0,%0,-1\n"
150 PPC405_ERR77(0,%2)\
151" stwcx. %0,0,%2\n\
152 bne- 1b"
153 : "=&r" (t), "+m" (v->counter)
154 : "r" (&v->counter)
155 : "cc", "xer");
156}
157
158static __inline__ int atomic_dec_return_relaxed(atomic_t *v)
159{
160 int t;
161
162 __asm__ __volatile__(
163"1: lwarx %0,0,%2 # atomic_dec_return_relaxed\n"
164" addic %0,%0,-1\n"
165 PPC405_ERR77(0, %2)
166" stwcx. %0,0,%2\n"
167" bne- 1b"
168 : "=&r" (t), "+m" (v->counter)
169 : "r" (&v->counter)
170 : "cc", "xer");
171
172 return t;
173}
174
175#define atomic_inc_return_relaxed atomic_inc_return_relaxed
176#define atomic_dec_return_relaxed atomic_dec_return_relaxed
177
178#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
179#define atomic_cmpxchg_relaxed(v, o, n) \
180 cmpxchg_relaxed(&((v)->counter), (o), (n))
181#define atomic_cmpxchg_acquire(v, o, n) \
182 cmpxchg_acquire(&((v)->counter), (o), (n))
183
184#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
185#define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
186
187/**
188 * __atomic_add_unless - add unless the number is a given value
189 * @v: pointer of type atomic_t
190 * @a: the amount to add to v...
191 * @u: ...unless v is equal to u.
192 *
193 * Atomically adds @a to @v, so long as it was not @u.
194 * Returns the old value of @v.
195 */
196static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u)
197{
198 int t;
199
200 __asm__ __volatile__ (
201 PPC_ATOMIC_ENTRY_BARRIER
202"1: lwarx %0,0,%1 # __atomic_add_unless\n\
203 cmpw 0,%0,%3 \n\
204 beq- 2f \n\
205 add %0,%2,%0 \n"
206 PPC405_ERR77(0,%2)
207" stwcx. %0,0,%1 \n\
208 bne- 1b \n"
209 PPC_ATOMIC_EXIT_BARRIER
210" subf %0,%2,%0 \n\
2112:"
212 : "=&r" (t)
213 : "r" (&v->counter), "r" (a), "r" (u)
214 : "cc", "memory");
215
216 return t;
217}
218
219/**
220 * atomic_inc_not_zero - increment unless the number is zero
221 * @v: pointer of type atomic_t
222 *
223 * Atomically increments @v by 1, so long as @v is non-zero.
224 * Returns non-zero if @v was non-zero, and zero otherwise.
225 */
226static __inline__ int atomic_inc_not_zero(atomic_t *v)
227{
228 int t1, t2;
229
230 __asm__ __volatile__ (
231 PPC_ATOMIC_ENTRY_BARRIER
232"1: lwarx %0,0,%2 # atomic_inc_not_zero\n\
233 cmpwi 0,%0,0\n\
234 beq- 2f\n\
235 addic %1,%0,1\n"
236 PPC405_ERR77(0,%2)
237" stwcx. %1,0,%2\n\
238 bne- 1b\n"
239 PPC_ATOMIC_EXIT_BARRIER
240 "\n\
2412:"
242 : "=&r" (t1), "=&r" (t2)
243 : "r" (&v->counter)
244 : "cc", "xer", "memory");
245
246 return t1;
247}
248#define atomic_inc_not_zero(v) atomic_inc_not_zero((v))
249
250#define atomic_sub_and_test(a, v) (atomic_sub_return((a), (v)) == 0)
251#define atomic_dec_and_test(v) (atomic_dec_return((v)) == 0)
252
253/*
254 * Atomically test *v and decrement if it is greater than 0.
255 * The function returns the old value of *v minus 1, even if
256 * the atomic variable, v, was not decremented.
257 */
258static __inline__ int atomic_dec_if_positive(atomic_t *v)
259{
260 int t;
261
262 __asm__ __volatile__(
263 PPC_ATOMIC_ENTRY_BARRIER
264"1: lwarx %0,0,%1 # atomic_dec_if_positive\n\
265 cmpwi %0,1\n\
266 addi %0,%0,-1\n\
267 blt- 2f\n"
268 PPC405_ERR77(0,%1)
269" stwcx. %0,0,%1\n\
270 bne- 1b"
271 PPC_ATOMIC_EXIT_BARRIER
272 "\n\
2732:" : "=&b" (t)
274 : "r" (&v->counter)
275 : "cc", "memory");
276
277 return t;
278}
279#define atomic_dec_if_positive atomic_dec_if_positive
280
281#ifdef __powerpc64__
282
283#define ATOMIC64_INIT(i) { (i) }
284
285static __inline__ long atomic64_read(const atomic64_t *v)
286{
287 long t;
288
289 __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter));
290
291 return t;
292}
293
294static __inline__ void atomic64_set(atomic64_t *v, long i)
295{
296 __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
297}
298
299#define ATOMIC64_OP(op, asm_op) \
300static __inline__ void atomic64_##op(long a, atomic64_t *v) \
301{ \
302 long t; \
303 \
304 __asm__ __volatile__( \
305"1: ldarx %0,0,%3 # atomic64_" #op "\n" \
306 #asm_op " %0,%2,%0\n" \
307" stdcx. %0,0,%3 \n" \
308" bne- 1b\n" \
309 : "=&r" (t), "+m" (v->counter) \
310 : "r" (a), "r" (&v->counter) \
311 : "cc"); \
312}
313
314#define ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \
315static inline long \
316atomic64_##op##_return_relaxed(long a, atomic64_t *v) \
317{ \
318 long t; \
319 \
320 __asm__ __volatile__( \
321"1: ldarx %0,0,%3 # atomic64_" #op "_return_relaxed\n" \
322 #asm_op " %0,%2,%0\n" \
323" stdcx. %0,0,%3\n" \
324" bne- 1b\n" \
325 : "=&r" (t), "+m" (v->counter) \
326 : "r" (a), "r" (&v->counter) \
327 : "cc"); \
328 \
329 return t; \
330}
331
332#define ATOMIC64_OPS(op, asm_op) \
333 ATOMIC64_OP(op, asm_op) \
334 ATOMIC64_OP_RETURN_RELAXED(op, asm_op)
335
336ATOMIC64_OPS(add, add)
337ATOMIC64_OPS(sub, subf)
338ATOMIC64_OP(and, and)
339ATOMIC64_OP(or, or)
340ATOMIC64_OP(xor, xor)
341
342#define atomic64_add_return_relaxed atomic64_add_return_relaxed
343#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
344
345#undef ATOPIC64_OPS
346#undef ATOMIC64_OP_RETURN_RELAXED
347#undef ATOMIC64_OP
348
349#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0)
350
351static __inline__ void atomic64_inc(atomic64_t *v)
352{
353 long t;
354
355 __asm__ __volatile__(
356"1: ldarx %0,0,%2 # atomic64_inc\n\
357 addic %0,%0,1\n\
358 stdcx. %0,0,%2 \n\
359 bne- 1b"
360 : "=&r" (t), "+m" (v->counter)
361 : "r" (&v->counter)
362 : "cc", "xer");
363}
364
365static __inline__ long atomic64_inc_return_relaxed(atomic64_t *v)
366{
367 long t;
368
369 __asm__ __volatile__(
370"1: ldarx %0,0,%2 # atomic64_inc_return_relaxed\n"
371" addic %0,%0,1\n"
372" stdcx. %0,0,%2\n"
373" bne- 1b"
374 : "=&r" (t), "+m" (v->counter)
375 : "r" (&v->counter)
376 : "cc", "xer");
377
378 return t;
379}
380
381/*
382 * atomic64_inc_and_test - increment and test
383 * @v: pointer of type atomic64_t
384 *
385 * Atomically increments @v by 1
386 * and returns true if the result is zero, or false for all
387 * other cases.
388 */
389#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
390
391static __inline__ void atomic64_dec(atomic64_t *v)
392{
393 long t;
394
395 __asm__ __volatile__(
396"1: ldarx %0,0,%2 # atomic64_dec\n\
397 addic %0,%0,-1\n\
398 stdcx. %0,0,%2\n\
399 bne- 1b"
400 : "=&r" (t), "+m" (v->counter)
401 : "r" (&v->counter)
402 : "cc", "xer");
403}
404
405static __inline__ long atomic64_dec_return_relaxed(atomic64_t *v)
406{
407 long t;
408
409 __asm__ __volatile__(
410"1: ldarx %0,0,%2 # atomic64_dec_return_relaxed\n"
411" addic %0,%0,-1\n"
412" stdcx. %0,0,%2\n"
413" bne- 1b"
414 : "=&r" (t), "+m" (v->counter)
415 : "r" (&v->counter)
416 : "cc", "xer");
417
418 return t;
419}
420
421#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed
422#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed
423
424#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0)
425#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)
426
427/*
428 * Atomically test *v and decrement if it is greater than 0.
429 * The function returns the old value of *v minus 1.
430 */
431static __inline__ long atomic64_dec_if_positive(atomic64_t *v)
432{
433 long t;
434
435 __asm__ __volatile__(
436 PPC_ATOMIC_ENTRY_BARRIER
437"1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\
438 addic. %0,%0,-1\n\
439 blt- 2f\n\
440 stdcx. %0,0,%1\n\
441 bne- 1b"
442 PPC_ATOMIC_EXIT_BARRIER
443 "\n\
4442:" : "=&r" (t)
445 : "r" (&v->counter)
446 : "cc", "xer", "memory");
447
448 return t;
449}
450
451#define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
452#define atomic64_cmpxchg_relaxed(v, o, n) \
453 cmpxchg_relaxed(&((v)->counter), (o), (n))
454#define atomic64_cmpxchg_acquire(v, o, n) \
455 cmpxchg_acquire(&((v)->counter), (o), (n))
456
457#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
458#define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
459
460/**
461 * atomic64_add_unless - add unless the number is a given value
462 * @v: pointer of type atomic64_t
463 * @a: the amount to add to v...
464 * @u: ...unless v is equal to u.
465 *
466 * Atomically adds @a to @v, so long as it was not @u.
467 * Returns the old value of @v.
468 */
469static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u)
470{
471 long t;
472
473 __asm__ __volatile__ (
474 PPC_ATOMIC_ENTRY_BARRIER
475"1: ldarx %0,0,%1 # __atomic_add_unless\n\
476 cmpd 0,%0,%3 \n\
477 beq- 2f \n\
478 add %0,%2,%0 \n"
479" stdcx. %0,0,%1 \n\
480 bne- 1b \n"
481 PPC_ATOMIC_EXIT_BARRIER
482" subf %0,%2,%0 \n\
4832:"
484 : "=&r" (t)
485 : "r" (&v->counter), "r" (a), "r" (u)
486 : "cc", "memory");
487
488 return t != u;
489}
490
491/**
492 * atomic_inc64_not_zero - increment unless the number is zero
493 * @v: pointer of type atomic64_t
494 *
495 * Atomically increments @v by 1, so long as @v is non-zero.
496 * Returns non-zero if @v was non-zero, and zero otherwise.
497 */
498static __inline__ long atomic64_inc_not_zero(atomic64_t *v)
499{
500 long t1, t2;
501
502 __asm__ __volatile__ (
503 PPC_ATOMIC_ENTRY_BARRIER
504"1: ldarx %0,0,%2 # atomic64_inc_not_zero\n\
505 cmpdi 0,%0,0\n\
506 beq- 2f\n\
507 addic %1,%0,1\n\
508 stdcx. %1,0,%2\n\
509 bne- 1b\n"
510 PPC_ATOMIC_EXIT_BARRIER
511 "\n\
5122:"
513 : "=&r" (t1), "=&r" (t2)
514 : "r" (&v->counter)
515 : "cc", "xer", "memory");
516
517 return t1;
518}
519
520#endif /* __powerpc64__ */
521
522#endif /* __KERNEL__ */
523#endif /* _ASM_POWERPC_ATOMIC_H_ */
1#ifndef _ASM_POWERPC_ATOMIC_H_
2#define _ASM_POWERPC_ATOMIC_H_
3
4/*
5 * PowerPC atomic operations
6 */
7
8#include <linux/types.h>
9
10#ifdef __KERNEL__
11#include <linux/compiler.h>
12#include <asm/synch.h>
13#include <asm/asm-compat.h>
14#include <asm/system.h>
15
16#define ATOMIC_INIT(i) { (i) }
17
18static __inline__ int atomic_read(const atomic_t *v)
19{
20 int t;
21
22 __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter));
23
24 return t;
25}
26
27static __inline__ void atomic_set(atomic_t *v, int i)
28{
29 __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
30}
31
32static __inline__ void atomic_add(int a, atomic_t *v)
33{
34 int t;
35
36 __asm__ __volatile__(
37"1: lwarx %0,0,%3 # atomic_add\n\
38 add %0,%2,%0\n"
39 PPC405_ERR77(0,%3)
40" stwcx. %0,0,%3 \n\
41 bne- 1b"
42 : "=&r" (t), "+m" (v->counter)
43 : "r" (a), "r" (&v->counter)
44 : "cc");
45}
46
47static __inline__ int atomic_add_return(int a, atomic_t *v)
48{
49 int t;
50
51 __asm__ __volatile__(
52 PPC_RELEASE_BARRIER
53"1: lwarx %0,0,%2 # atomic_add_return\n\
54 add %0,%1,%0\n"
55 PPC405_ERR77(0,%2)
56" stwcx. %0,0,%2 \n\
57 bne- 1b"
58 PPC_ACQUIRE_BARRIER
59 : "=&r" (t)
60 : "r" (a), "r" (&v->counter)
61 : "cc", "memory");
62
63 return t;
64}
65
66#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0)
67
68static __inline__ void atomic_sub(int a, atomic_t *v)
69{
70 int t;
71
72 __asm__ __volatile__(
73"1: lwarx %0,0,%3 # atomic_sub\n\
74 subf %0,%2,%0\n"
75 PPC405_ERR77(0,%3)
76" stwcx. %0,0,%3 \n\
77 bne- 1b"
78 : "=&r" (t), "+m" (v->counter)
79 : "r" (a), "r" (&v->counter)
80 : "cc");
81}
82
83static __inline__ int atomic_sub_return(int a, atomic_t *v)
84{
85 int t;
86
87 __asm__ __volatile__(
88 PPC_RELEASE_BARRIER
89"1: lwarx %0,0,%2 # atomic_sub_return\n\
90 subf %0,%1,%0\n"
91 PPC405_ERR77(0,%2)
92" stwcx. %0,0,%2 \n\
93 bne- 1b"
94 PPC_ACQUIRE_BARRIER
95 : "=&r" (t)
96 : "r" (a), "r" (&v->counter)
97 : "cc", "memory");
98
99 return t;
100}
101
102static __inline__ void atomic_inc(atomic_t *v)
103{
104 int t;
105
106 __asm__ __volatile__(
107"1: lwarx %0,0,%2 # atomic_inc\n\
108 addic %0,%0,1\n"
109 PPC405_ERR77(0,%2)
110" stwcx. %0,0,%2 \n\
111 bne- 1b"
112 : "=&r" (t), "+m" (v->counter)
113 : "r" (&v->counter)
114 : "cc", "xer");
115}
116
117static __inline__ int atomic_inc_return(atomic_t *v)
118{
119 int t;
120
121 __asm__ __volatile__(
122 PPC_RELEASE_BARRIER
123"1: lwarx %0,0,%1 # atomic_inc_return\n\
124 addic %0,%0,1\n"
125 PPC405_ERR77(0,%1)
126" stwcx. %0,0,%1 \n\
127 bne- 1b"
128 PPC_ACQUIRE_BARRIER
129 : "=&r" (t)
130 : "r" (&v->counter)
131 : "cc", "xer", "memory");
132
133 return t;
134}
135
136/*
137 * atomic_inc_and_test - increment and test
138 * @v: pointer of type atomic_t
139 *
140 * Atomically increments @v by 1
141 * and returns true if the result is zero, or false for all
142 * other cases.
143 */
144#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
145
146static __inline__ void atomic_dec(atomic_t *v)
147{
148 int t;
149
150 __asm__ __volatile__(
151"1: lwarx %0,0,%2 # atomic_dec\n\
152 addic %0,%0,-1\n"
153 PPC405_ERR77(0,%2)\
154" stwcx. %0,0,%2\n\
155 bne- 1b"
156 : "=&r" (t), "+m" (v->counter)
157 : "r" (&v->counter)
158 : "cc", "xer");
159}
160
161static __inline__ int atomic_dec_return(atomic_t *v)
162{
163 int t;
164
165 __asm__ __volatile__(
166 PPC_RELEASE_BARRIER
167"1: lwarx %0,0,%1 # atomic_dec_return\n\
168 addic %0,%0,-1\n"
169 PPC405_ERR77(0,%1)
170" stwcx. %0,0,%1\n\
171 bne- 1b"
172 PPC_ACQUIRE_BARRIER
173 : "=&r" (t)
174 : "r" (&v->counter)
175 : "cc", "xer", "memory");
176
177 return t;
178}
179
180#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
181#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
182
183/**
184 * __atomic_add_unless - add unless the number is a given value
185 * @v: pointer of type atomic_t
186 * @a: the amount to add to v...
187 * @u: ...unless v is equal to u.
188 *
189 * Atomically adds @a to @v, so long as it was not @u.
190 * Returns the old value of @v.
191 */
192static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u)
193{
194 int t;
195
196 __asm__ __volatile__ (
197 PPC_RELEASE_BARRIER
198"1: lwarx %0,0,%1 # __atomic_add_unless\n\
199 cmpw 0,%0,%3 \n\
200 beq- 2f \n\
201 add %0,%2,%0 \n"
202 PPC405_ERR77(0,%2)
203" stwcx. %0,0,%1 \n\
204 bne- 1b \n"
205 PPC_ACQUIRE_BARRIER
206" subf %0,%2,%0 \n\
2072:"
208 : "=&r" (t)
209 : "r" (&v->counter), "r" (a), "r" (u)
210 : "cc", "memory");
211
212 return t;
213}
214
215
216#define atomic_sub_and_test(a, v) (atomic_sub_return((a), (v)) == 0)
217#define atomic_dec_and_test(v) (atomic_dec_return((v)) == 0)
218
219/*
220 * Atomically test *v and decrement if it is greater than 0.
221 * The function returns the old value of *v minus 1, even if
222 * the atomic variable, v, was not decremented.
223 */
224static __inline__ int atomic_dec_if_positive(atomic_t *v)
225{
226 int t;
227
228 __asm__ __volatile__(
229 PPC_RELEASE_BARRIER
230"1: lwarx %0,0,%1 # atomic_dec_if_positive\n\
231 cmpwi %0,1\n\
232 addi %0,%0,-1\n\
233 blt- 2f\n"
234 PPC405_ERR77(0,%1)
235" stwcx. %0,0,%1\n\
236 bne- 1b"
237 PPC_ACQUIRE_BARRIER
238 "\n\
2392:" : "=&b" (t)
240 : "r" (&v->counter)
241 : "cc", "memory");
242
243 return t;
244}
245
246#define smp_mb__before_atomic_dec() smp_mb()
247#define smp_mb__after_atomic_dec() smp_mb()
248#define smp_mb__before_atomic_inc() smp_mb()
249#define smp_mb__after_atomic_inc() smp_mb()
250
251#ifdef __powerpc64__
252
253#define ATOMIC64_INIT(i) { (i) }
254
255static __inline__ long atomic64_read(const atomic64_t *v)
256{
257 long t;
258
259 __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter));
260
261 return t;
262}
263
264static __inline__ void atomic64_set(atomic64_t *v, long i)
265{
266 __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
267}
268
269static __inline__ void atomic64_add(long a, atomic64_t *v)
270{
271 long t;
272
273 __asm__ __volatile__(
274"1: ldarx %0,0,%3 # atomic64_add\n\
275 add %0,%2,%0\n\
276 stdcx. %0,0,%3 \n\
277 bne- 1b"
278 : "=&r" (t), "+m" (v->counter)
279 : "r" (a), "r" (&v->counter)
280 : "cc");
281}
282
283static __inline__ long atomic64_add_return(long a, atomic64_t *v)
284{
285 long t;
286
287 __asm__ __volatile__(
288 PPC_RELEASE_BARRIER
289"1: ldarx %0,0,%2 # atomic64_add_return\n\
290 add %0,%1,%0\n\
291 stdcx. %0,0,%2 \n\
292 bne- 1b"
293 PPC_ACQUIRE_BARRIER
294 : "=&r" (t)
295 : "r" (a), "r" (&v->counter)
296 : "cc", "memory");
297
298 return t;
299}
300
301#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0)
302
303static __inline__ void atomic64_sub(long a, atomic64_t *v)
304{
305 long t;
306
307 __asm__ __volatile__(
308"1: ldarx %0,0,%3 # atomic64_sub\n\
309 subf %0,%2,%0\n\
310 stdcx. %0,0,%3 \n\
311 bne- 1b"
312 : "=&r" (t), "+m" (v->counter)
313 : "r" (a), "r" (&v->counter)
314 : "cc");
315}
316
317static __inline__ long atomic64_sub_return(long a, atomic64_t *v)
318{
319 long t;
320
321 __asm__ __volatile__(
322 PPC_RELEASE_BARRIER
323"1: ldarx %0,0,%2 # atomic64_sub_return\n\
324 subf %0,%1,%0\n\
325 stdcx. %0,0,%2 \n\
326 bne- 1b"
327 PPC_ACQUIRE_BARRIER
328 : "=&r" (t)
329 : "r" (a), "r" (&v->counter)
330 : "cc", "memory");
331
332 return t;
333}
334
335static __inline__ void atomic64_inc(atomic64_t *v)
336{
337 long t;
338
339 __asm__ __volatile__(
340"1: ldarx %0,0,%2 # atomic64_inc\n\
341 addic %0,%0,1\n\
342 stdcx. %0,0,%2 \n\
343 bne- 1b"
344 : "=&r" (t), "+m" (v->counter)
345 : "r" (&v->counter)
346 : "cc", "xer");
347}
348
349static __inline__ long atomic64_inc_return(atomic64_t *v)
350{
351 long t;
352
353 __asm__ __volatile__(
354 PPC_RELEASE_BARRIER
355"1: ldarx %0,0,%1 # atomic64_inc_return\n\
356 addic %0,%0,1\n\
357 stdcx. %0,0,%1 \n\
358 bne- 1b"
359 PPC_ACQUIRE_BARRIER
360 : "=&r" (t)
361 : "r" (&v->counter)
362 : "cc", "xer", "memory");
363
364 return t;
365}
366
367/*
368 * atomic64_inc_and_test - increment and test
369 * @v: pointer of type atomic64_t
370 *
371 * Atomically increments @v by 1
372 * and returns true if the result is zero, or false for all
373 * other cases.
374 */
375#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
376
377static __inline__ void atomic64_dec(atomic64_t *v)
378{
379 long t;
380
381 __asm__ __volatile__(
382"1: ldarx %0,0,%2 # atomic64_dec\n\
383 addic %0,%0,-1\n\
384 stdcx. %0,0,%2\n\
385 bne- 1b"
386 : "=&r" (t), "+m" (v->counter)
387 : "r" (&v->counter)
388 : "cc", "xer");
389}
390
391static __inline__ long atomic64_dec_return(atomic64_t *v)
392{
393 long t;
394
395 __asm__ __volatile__(
396 PPC_RELEASE_BARRIER
397"1: ldarx %0,0,%1 # atomic64_dec_return\n\
398 addic %0,%0,-1\n\
399 stdcx. %0,0,%1\n\
400 bne- 1b"
401 PPC_ACQUIRE_BARRIER
402 : "=&r" (t)
403 : "r" (&v->counter)
404 : "cc", "xer", "memory");
405
406 return t;
407}
408
409#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0)
410#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)
411
412/*
413 * Atomically test *v and decrement if it is greater than 0.
414 * The function returns the old value of *v minus 1.
415 */
416static __inline__ long atomic64_dec_if_positive(atomic64_t *v)
417{
418 long t;
419
420 __asm__ __volatile__(
421 PPC_RELEASE_BARRIER
422"1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\
423 addic. %0,%0,-1\n\
424 blt- 2f\n\
425 stdcx. %0,0,%1\n\
426 bne- 1b"
427 PPC_ACQUIRE_BARRIER
428 "\n\
4292:" : "=&r" (t)
430 : "r" (&v->counter)
431 : "cc", "xer", "memory");
432
433 return t;
434}
435
436#define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
437#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
438
439/**
440 * atomic64_add_unless - add unless the number is a given value
441 * @v: pointer of type atomic64_t
442 * @a: the amount to add to v...
443 * @u: ...unless v is equal to u.
444 *
445 * Atomically adds @a to @v, so long as it was not @u.
446 * Returns the old value of @v.
447 */
448static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u)
449{
450 long t;
451
452 __asm__ __volatile__ (
453 PPC_RELEASE_BARRIER
454"1: ldarx %0,0,%1 # __atomic_add_unless\n\
455 cmpd 0,%0,%3 \n\
456 beq- 2f \n\
457 add %0,%2,%0 \n"
458" stdcx. %0,0,%1 \n\
459 bne- 1b \n"
460 PPC_ACQUIRE_BARRIER
461" subf %0,%2,%0 \n\
4622:"
463 : "=&r" (t)
464 : "r" (&v->counter), "r" (a), "r" (u)
465 : "cc", "memory");
466
467 return t != u;
468}
469
470#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
471
472#endif /* __powerpc64__ */
473
474#endif /* __KERNEL__ */
475#endif /* _ASM_POWERPC_ATOMIC_H_ */