Loading...
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
4 */
5
6#ifndef _ASM_ARC_ATOMIC_H
7#define _ASM_ARC_ATOMIC_H
8
9#ifndef __ASSEMBLY__
10
11#include <linux/types.h>
12#include <linux/compiler.h>
13#include <asm/cmpxchg.h>
14#include <asm/barrier.h>
15#include <asm/smp.h>
16
17#define arch_atomic_read(v) READ_ONCE((v)->counter)
18
19#ifdef CONFIG_ARC_HAS_LLSC
20#include <asm/atomic-llsc.h>
21#else
22#include <asm/atomic-spinlock.h>
23#endif
24
25/*
26 * 64-bit atomics
27 */
28#ifdef CONFIG_GENERIC_ATOMIC64
29#include <asm-generic/atomic64.h>
30#else
31#include <asm/atomic64-arcv2.h>
32#endif
33
34#endif /* !__ASSEMBLY__ */
35
36#endif
1/*
2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9#ifndef _ASM_ARC_ATOMIC_H
10#define _ASM_ARC_ATOMIC_H
11
12#ifndef __ASSEMBLY__
13
14#include <linux/types.h>
15#include <linux/compiler.h>
16#include <asm/cmpxchg.h>
17#include <asm/barrier.h>
18#include <asm/smp.h>
19
20#define ATOMIC_INIT(i) { (i) }
21
22#ifndef CONFIG_ARC_PLAT_EZNPS
23
24#define atomic_read(v) READ_ONCE((v)->counter)
25
26#ifdef CONFIG_ARC_HAS_LLSC
27
28#define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
29
30#define ATOMIC_OP(op, c_op, asm_op) \
31static inline void atomic_##op(int i, atomic_t *v) \
32{ \
33 unsigned int val; \
34 \
35 __asm__ __volatile__( \
36 "1: llock %[val], [%[ctr]] \n" \
37 " " #asm_op " %[val], %[val], %[i] \n" \
38 " scond %[val], [%[ctr]] \n" \
39 " bnz 1b \n" \
40 : [val] "=&r" (val) /* Early clobber to prevent reg reuse */ \
41 : [ctr] "r" (&v->counter), /* Not "m": llock only supports reg direct addr mode */ \
42 [i] "ir" (i) \
43 : "cc"); \
44} \
45
46#define ATOMIC_OP_RETURN(op, c_op, asm_op) \
47static inline int atomic_##op##_return(int i, atomic_t *v) \
48{ \
49 unsigned int val; \
50 \
51 /* \
52 * Explicit full memory barrier needed before/after as \
53 * LLOCK/SCOND thmeselves don't provide any such semantics \
54 */ \
55 smp_mb(); \
56 \
57 __asm__ __volatile__( \
58 "1: llock %[val], [%[ctr]] \n" \
59 " " #asm_op " %[val], %[val], %[i] \n" \
60 " scond %[val], [%[ctr]] \n" \
61 " bnz 1b \n" \
62 : [val] "=&r" (val) \
63 : [ctr] "r" (&v->counter), \
64 [i] "ir" (i) \
65 : "cc"); \
66 \
67 smp_mb(); \
68 \
69 return val; \
70}
71
72#define ATOMIC_FETCH_OP(op, c_op, asm_op) \
73static inline int atomic_fetch_##op(int i, atomic_t *v) \
74{ \
75 unsigned int val, orig; \
76 \
77 /* \
78 * Explicit full memory barrier needed before/after as \
79 * LLOCK/SCOND thmeselves don't provide any such semantics \
80 */ \
81 smp_mb(); \
82 \
83 __asm__ __volatile__( \
84 "1: llock %[orig], [%[ctr]] \n" \
85 " " #asm_op " %[val], %[orig], %[i] \n" \
86 " scond %[val], [%[ctr]] \n" \
87 " \n" \
88 : [val] "=&r" (val), \
89 [orig] "=&r" (orig) \
90 : [ctr] "r" (&v->counter), \
91 [i] "ir" (i) \
92 : "cc"); \
93 \
94 smp_mb(); \
95 \
96 return orig; \
97}
98
99#else /* !CONFIG_ARC_HAS_LLSC */
100
101#ifndef CONFIG_SMP
102
103 /* violating atomic_xxx API locking protocol in UP for optimization sake */
104#define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
105
106#else
107
108static inline void atomic_set(atomic_t *v, int i)
109{
110 /*
111 * Independent of hardware support, all of the atomic_xxx() APIs need
112 * to follow the same locking rules to make sure that a "hardware"
113 * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn
114 * sequence
115 *
116 * Thus atomic_set() despite being 1 insn (and seemingly atomic)
117 * requires the locking.
118 */
119 unsigned long flags;
120
121 atomic_ops_lock(flags);
122 WRITE_ONCE(v->counter, i);
123 atomic_ops_unlock(flags);
124}
125
126#define atomic_set_release(v, i) atomic_set((v), (i))
127
128#endif
129
130/*
131 * Non hardware assisted Atomic-R-M-W
132 * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
133 */
134
135#define ATOMIC_OP(op, c_op, asm_op) \
136static inline void atomic_##op(int i, atomic_t *v) \
137{ \
138 unsigned long flags; \
139 \
140 atomic_ops_lock(flags); \
141 v->counter c_op i; \
142 atomic_ops_unlock(flags); \
143}
144
145#define ATOMIC_OP_RETURN(op, c_op, asm_op) \
146static inline int atomic_##op##_return(int i, atomic_t *v) \
147{ \
148 unsigned long flags; \
149 unsigned long temp; \
150 \
151 /* \
152 * spin lock/unlock provides the needed smp_mb() before/after \
153 */ \
154 atomic_ops_lock(flags); \
155 temp = v->counter; \
156 temp c_op i; \
157 v->counter = temp; \
158 atomic_ops_unlock(flags); \
159 \
160 return temp; \
161}
162
163#define ATOMIC_FETCH_OP(op, c_op, asm_op) \
164static inline int atomic_fetch_##op(int i, atomic_t *v) \
165{ \
166 unsigned long flags; \
167 unsigned long orig; \
168 \
169 /* \
170 * spin lock/unlock provides the needed smp_mb() before/after \
171 */ \
172 atomic_ops_lock(flags); \
173 orig = v->counter; \
174 v->counter c_op i; \
175 atomic_ops_unlock(flags); \
176 \
177 return orig; \
178}
179
180#endif /* !CONFIG_ARC_HAS_LLSC */
181
182#define ATOMIC_OPS(op, c_op, asm_op) \
183 ATOMIC_OP(op, c_op, asm_op) \
184 ATOMIC_OP_RETURN(op, c_op, asm_op) \
185 ATOMIC_FETCH_OP(op, c_op, asm_op)
186
187ATOMIC_OPS(add, +=, add)
188ATOMIC_OPS(sub, -=, sub)
189
190#define atomic_andnot atomic_andnot
191
192#undef ATOMIC_OPS
193#define ATOMIC_OPS(op, c_op, asm_op) \
194 ATOMIC_OP(op, c_op, asm_op) \
195 ATOMIC_FETCH_OP(op, c_op, asm_op)
196
197ATOMIC_OPS(and, &=, and)
198ATOMIC_OPS(andnot, &= ~, bic)
199ATOMIC_OPS(or, |=, or)
200ATOMIC_OPS(xor, ^=, xor)
201
202#else /* CONFIG_ARC_PLAT_EZNPS */
203
204static inline int atomic_read(const atomic_t *v)
205{
206 int temp;
207
208 __asm__ __volatile__(
209 " ld.di %0, [%1]"
210 : "=r"(temp)
211 : "r"(&v->counter)
212 : "memory");
213 return temp;
214}
215
216static inline void atomic_set(atomic_t *v, int i)
217{
218 __asm__ __volatile__(
219 " st.di %0,[%1]"
220 :
221 : "r"(i), "r"(&v->counter)
222 : "memory");
223}
224
225#define ATOMIC_OP(op, c_op, asm_op) \
226static inline void atomic_##op(int i, atomic_t *v) \
227{ \
228 __asm__ __volatile__( \
229 " mov r2, %0\n" \
230 " mov r3, %1\n" \
231 " .word %2\n" \
232 : \
233 : "r"(i), "r"(&v->counter), "i"(asm_op) \
234 : "r2", "r3", "memory"); \
235} \
236
237#define ATOMIC_OP_RETURN(op, c_op, asm_op) \
238static inline int atomic_##op##_return(int i, atomic_t *v) \
239{ \
240 unsigned int temp = i; \
241 \
242 /* Explicit full memory barrier needed before/after */ \
243 smp_mb(); \
244 \
245 __asm__ __volatile__( \
246 " mov r2, %0\n" \
247 " mov r3, %1\n" \
248 " .word %2\n" \
249 " mov %0, r2" \
250 : "+r"(temp) \
251 : "r"(&v->counter), "i"(asm_op) \
252 : "r2", "r3", "memory"); \
253 \
254 smp_mb(); \
255 \
256 temp c_op i; \
257 \
258 return temp; \
259}
260
261#define ATOMIC_FETCH_OP(op, c_op, asm_op) \
262static inline int atomic_fetch_##op(int i, atomic_t *v) \
263{ \
264 unsigned int temp = i; \
265 \
266 /* Explicit full memory barrier needed before/after */ \
267 smp_mb(); \
268 \
269 __asm__ __volatile__( \
270 " mov r2, %0\n" \
271 " mov r3, %1\n" \
272 " .word %2\n" \
273 " mov %0, r2" \
274 : "+r"(temp) \
275 : "r"(&v->counter), "i"(asm_op) \
276 : "r2", "r3", "memory"); \
277 \
278 smp_mb(); \
279 \
280 return temp; \
281}
282
283#define ATOMIC_OPS(op, c_op, asm_op) \
284 ATOMIC_OP(op, c_op, asm_op) \
285 ATOMIC_OP_RETURN(op, c_op, asm_op) \
286 ATOMIC_FETCH_OP(op, c_op, asm_op)
287
288ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3)
289#define atomic_sub(i, v) atomic_add(-(i), (v))
290#define atomic_sub_return(i, v) atomic_add_return(-(i), (v))
291#define atomic_fetch_sub(i, v) atomic_fetch_add(-(i), (v))
292
293#undef ATOMIC_OPS
294#define ATOMIC_OPS(op, c_op, asm_op) \
295 ATOMIC_OP(op, c_op, asm_op) \
296 ATOMIC_FETCH_OP(op, c_op, asm_op)
297
298ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3)
299#define atomic_andnot(mask, v) atomic_and(~(mask), (v))
300#define atomic_fetch_andnot(mask, v) atomic_fetch_and(~(mask), (v))
301ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3)
302ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)
303
304#endif /* CONFIG_ARC_PLAT_EZNPS */
305
306#undef ATOMIC_OPS
307#undef ATOMIC_FETCH_OP
308#undef ATOMIC_OP_RETURN
309#undef ATOMIC_OP
310
311/**
312 * __atomic_add_unless - add unless the number is a given value
313 * @v: pointer of type atomic_t
314 * @a: the amount to add to v...
315 * @u: ...unless v is equal to u.
316 *
317 * Atomically adds @a to @v, so long as it was not @u.
318 * Returns the old value of @v
319 */
320#define __atomic_add_unless(v, a, u) \
321({ \
322 int c, old; \
323 \
324 /* \
325 * Explicit full memory barrier needed before/after as \
326 * LLOCK/SCOND thmeselves don't provide any such semantics \
327 */ \
328 smp_mb(); \
329 \
330 c = atomic_read(v); \
331 while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c)\
332 c = old; \
333 \
334 smp_mb(); \
335 \
336 c; \
337})
338
339#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
340
341#define atomic_inc(v) atomic_add(1, v)
342#define atomic_dec(v) atomic_sub(1, v)
343
344#define atomic_inc_and_test(v) (atomic_add_return(1, v) == 0)
345#define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0)
346#define atomic_inc_return(v) atomic_add_return(1, (v))
347#define atomic_dec_return(v) atomic_sub_return(1, (v))
348#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0)
349
350#define atomic_add_negative(i, v) (atomic_add_return(i, v) < 0)
351
352
353#ifdef CONFIG_GENERIC_ATOMIC64
354
355#include <asm-generic/atomic64.h>
356
357#else /* Kconfig ensures this is only enabled with needed h/w assist */
358
359/*
360 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)
361 * - The address HAS to be 64-bit aligned
362 * - There are 2 semantics involved here:
363 * = exclusive implies no interim update between load/store to same addr
364 * = both words are observed/updated together: this is guaranteed even
365 * for regular 64-bit load (LDD) / store (STD). Thus atomic64_set()
366 * is NOT required to use LLOCKD+SCONDD, STD suffices
367 */
368
369typedef struct {
370 aligned_u64 counter;
371} atomic64_t;
372
373#define ATOMIC64_INIT(a) { (a) }
374
375static inline long long atomic64_read(const atomic64_t *v)
376{
377 unsigned long long val;
378
379 __asm__ __volatile__(
380 " ldd %0, [%1] \n"
381 : "=r"(val)
382 : "r"(&v->counter));
383
384 return val;
385}
386
387static inline void atomic64_set(atomic64_t *v, long long a)
388{
389 /*
390 * This could have been a simple assignment in "C" but would need
391 * explicit volatile. Otherwise gcc optimizers could elide the store
392 * which borked atomic64 self-test
393 * In the inline asm version, memory clobber needed for exact same
394 * reason, to tell gcc about the store.
395 *
396 * This however is not needed for sibling atomic64_add() etc since both
397 * load/store are explicitly done in inline asm. As long as API is used
398 * for each access, gcc has no way to optimize away any load/store
399 */
400 __asm__ __volatile__(
401 " std %0, [%1] \n"
402 :
403 : "r"(a), "r"(&v->counter)
404 : "memory");
405}
406
407#define ATOMIC64_OP(op, op1, op2) \
408static inline void atomic64_##op(long long a, atomic64_t *v) \
409{ \
410 unsigned long long val; \
411 \
412 __asm__ __volatile__( \
413 "1: \n" \
414 " llockd %0, [%1] \n" \
415 " " #op1 " %L0, %L0, %L2 \n" \
416 " " #op2 " %H0, %H0, %H2 \n" \
417 " scondd %0, [%1] \n" \
418 " bnz 1b \n" \
419 : "=&r"(val) \
420 : "r"(&v->counter), "ir"(a) \
421 : "cc"); \
422} \
423
424#define ATOMIC64_OP_RETURN(op, op1, op2) \
425static inline long long atomic64_##op##_return(long long a, atomic64_t *v) \
426{ \
427 unsigned long long val; \
428 \
429 smp_mb(); \
430 \
431 __asm__ __volatile__( \
432 "1: \n" \
433 " llockd %0, [%1] \n" \
434 " " #op1 " %L0, %L0, %L2 \n" \
435 " " #op2 " %H0, %H0, %H2 \n" \
436 " scondd %0, [%1] \n" \
437 " bnz 1b \n" \
438 : [val] "=&r"(val) \
439 : "r"(&v->counter), "ir"(a) \
440 : "cc"); /* memory clobber comes from smp_mb() */ \
441 \
442 smp_mb(); \
443 \
444 return val; \
445}
446
447#define ATOMIC64_FETCH_OP(op, op1, op2) \
448static inline long long atomic64_fetch_##op(long long a, atomic64_t *v) \
449{ \
450 unsigned long long val, orig; \
451 \
452 smp_mb(); \
453 \
454 __asm__ __volatile__( \
455 "1: \n" \
456 " llockd %0, [%2] \n" \
457 " " #op1 " %L1, %L0, %L3 \n" \
458 " " #op2 " %H1, %H0, %H3 \n" \
459 " scondd %1, [%2] \n" \
460 " bnz 1b \n" \
461 : "=&r"(orig), "=&r"(val) \
462 : "r"(&v->counter), "ir"(a) \
463 : "cc"); /* memory clobber comes from smp_mb() */ \
464 \
465 smp_mb(); \
466 \
467 return orig; \
468}
469
470#define ATOMIC64_OPS(op, op1, op2) \
471 ATOMIC64_OP(op, op1, op2) \
472 ATOMIC64_OP_RETURN(op, op1, op2) \
473 ATOMIC64_FETCH_OP(op, op1, op2)
474
475#define atomic64_andnot atomic64_andnot
476
477ATOMIC64_OPS(add, add.f, adc)
478ATOMIC64_OPS(sub, sub.f, sbc)
479ATOMIC64_OPS(and, and, and)
480ATOMIC64_OPS(andnot, bic, bic)
481ATOMIC64_OPS(or, or, or)
482ATOMIC64_OPS(xor, xor, xor)
483
484#undef ATOMIC64_OPS
485#undef ATOMIC64_FETCH_OP
486#undef ATOMIC64_OP_RETURN
487#undef ATOMIC64_OP
488
489static inline long long
490atomic64_cmpxchg(atomic64_t *ptr, long long expected, long long new)
491{
492 long long prev;
493
494 smp_mb();
495
496 __asm__ __volatile__(
497 "1: llockd %0, [%1] \n"
498 " brne %L0, %L2, 2f \n"
499 " brne %H0, %H2, 2f \n"
500 " scondd %3, [%1] \n"
501 " bnz 1b \n"
502 "2: \n"
503 : "=&r"(prev)
504 : "r"(ptr), "ir"(expected), "r"(new)
505 : "cc"); /* memory clobber comes from smp_mb() */
506
507 smp_mb();
508
509 return prev;
510}
511
512static inline long long atomic64_xchg(atomic64_t *ptr, long long new)
513{
514 long long prev;
515
516 smp_mb();
517
518 __asm__ __volatile__(
519 "1: llockd %0, [%1] \n"
520 " scondd %2, [%1] \n"
521 " bnz 1b \n"
522 "2: \n"
523 : "=&r"(prev)
524 : "r"(ptr), "r"(new)
525 : "cc"); /* memory clobber comes from smp_mb() */
526
527 smp_mb();
528
529 return prev;
530}
531
532/**
533 * atomic64_dec_if_positive - decrement by 1 if old value positive
534 * @v: pointer of type atomic64_t
535 *
536 * The function returns the old value of *v minus 1, even if
537 * the atomic variable, v, was not decremented.
538 */
539
540static inline long long atomic64_dec_if_positive(atomic64_t *v)
541{
542 long long val;
543
544 smp_mb();
545
546 __asm__ __volatile__(
547 "1: llockd %0, [%1] \n"
548 " sub.f %L0, %L0, 1 # w0 - 1, set C on borrow\n"
549 " sub.c %H0, %H0, 1 # if C set, w1 - 1\n"
550 " brlt %H0, 0, 2f \n"
551 " scondd %0, [%1] \n"
552 " bnz 1b \n"
553 "2: \n"
554 : "=&r"(val)
555 : "r"(&v->counter)
556 : "cc"); /* memory clobber comes from smp_mb() */
557
558 smp_mb();
559
560 return val;
561}
562
563/**
564 * atomic64_add_unless - add unless the number is a given value
565 * @v: pointer of type atomic64_t
566 * @a: the amount to add to v...
567 * @u: ...unless v is equal to u.
568 *
569 * if (v != u) { v += a; ret = 1} else {ret = 0}
570 * Returns 1 iff @v was not @u (i.e. if add actually happened)
571 */
572static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u)
573{
574 long long val;
575 int op_done;
576
577 smp_mb();
578
579 __asm__ __volatile__(
580 "1: llockd %0, [%2] \n"
581 " mov %1, 1 \n"
582 " brne %L0, %L4, 2f # continue to add since v != u \n"
583 " breq.d %H0, %H4, 3f # return since v == u \n"
584 " mov %1, 0 \n"
585 "2: \n"
586 " add.f %L0, %L0, %L3 \n"
587 " adc %H0, %H0, %H3 \n"
588 " scondd %0, [%2] \n"
589 " bnz 1b \n"
590 "3: \n"
591 : "=&r"(val), "=&r" (op_done)
592 : "r"(&v->counter), "r"(a), "r"(u)
593 : "cc"); /* memory clobber comes from smp_mb() */
594
595 smp_mb();
596
597 return op_done;
598}
599
600#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0)
601#define atomic64_inc(v) atomic64_add(1LL, (v))
602#define atomic64_inc_return(v) atomic64_add_return(1LL, (v))
603#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
604#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0)
605#define atomic64_dec(v) atomic64_sub(1LL, (v))
606#define atomic64_dec_return(v) atomic64_sub_return(1LL, (v))
607#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)
608#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL)
609
610#endif /* !CONFIG_GENERIC_ATOMIC64 */
611
612#endif /* !__ASSEMBLY__ */
613
614#endif