Loading...
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
4 */
5
6#ifndef _ASM_ARC_ATOMIC_H
7#define _ASM_ARC_ATOMIC_H
8
9#ifndef __ASSEMBLY__
10
11#include <linux/types.h>
12#include <linux/compiler.h>
13#include <asm/cmpxchg.h>
14#include <asm/barrier.h>
15#include <asm/smp.h>
16
17#define arch_atomic_read(v) READ_ONCE((v)->counter)
18
19#ifdef CONFIG_ARC_HAS_LLSC
20#include <asm/atomic-llsc.h>
21#else
22#include <asm/atomic-spinlock.h>
23#endif
24
25/*
26 * 64-bit atomics
27 */
28#ifdef CONFIG_GENERIC_ATOMIC64
29#include <asm-generic/atomic64.h>
30#else
31#include <asm/atomic64-arcv2.h>
32#endif
33
34#endif /* !__ASSEMBLY__ */
35
36#endif
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
4 */
5
6#ifndef _ASM_ARC_ATOMIC_H
7#define _ASM_ARC_ATOMIC_H
8
9#ifndef __ASSEMBLY__
10
11#include <linux/types.h>
12#include <linux/compiler.h>
13#include <asm/cmpxchg.h>
14#include <asm/barrier.h>
15#include <asm/smp.h>
16
17#ifndef CONFIG_ARC_PLAT_EZNPS
18
19#define atomic_read(v) READ_ONCE((v)->counter)
20
21#ifdef CONFIG_ARC_HAS_LLSC
22
23#define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
24
25#define ATOMIC_OP(op, c_op, asm_op) \
26static inline void atomic_##op(int i, atomic_t *v) \
27{ \
28 unsigned int val; \
29 \
30 __asm__ __volatile__( \
31 "1: llock %[val], [%[ctr]] \n" \
32 " " #asm_op " %[val], %[val], %[i] \n" \
33 " scond %[val], [%[ctr]] \n" \
34 " bnz 1b \n" \
35 : [val] "=&r" (val) /* Early clobber to prevent reg reuse */ \
36 : [ctr] "r" (&v->counter), /* Not "m": llock only supports reg direct addr mode */ \
37 [i] "ir" (i) \
38 : "cc"); \
39} \
40
41#define ATOMIC_OP_RETURN(op, c_op, asm_op) \
42static inline int atomic_##op##_return(int i, atomic_t *v) \
43{ \
44 unsigned int val; \
45 \
46 /* \
47 * Explicit full memory barrier needed before/after as \
48 * LLOCK/SCOND thmeselves don't provide any such semantics \
49 */ \
50 smp_mb(); \
51 \
52 __asm__ __volatile__( \
53 "1: llock %[val], [%[ctr]] \n" \
54 " " #asm_op " %[val], %[val], %[i] \n" \
55 " scond %[val], [%[ctr]] \n" \
56 " bnz 1b \n" \
57 : [val] "=&r" (val) \
58 : [ctr] "r" (&v->counter), \
59 [i] "ir" (i) \
60 : "cc"); \
61 \
62 smp_mb(); \
63 \
64 return val; \
65}
66
67#define ATOMIC_FETCH_OP(op, c_op, asm_op) \
68static inline int atomic_fetch_##op(int i, atomic_t *v) \
69{ \
70 unsigned int val, orig; \
71 \
72 /* \
73 * Explicit full memory barrier needed before/after as \
74 * LLOCK/SCOND thmeselves don't provide any such semantics \
75 */ \
76 smp_mb(); \
77 \
78 __asm__ __volatile__( \
79 "1: llock %[orig], [%[ctr]] \n" \
80 " " #asm_op " %[val], %[orig], %[i] \n" \
81 " scond %[val], [%[ctr]] \n" \
82 " bnz 1b \n" \
83 : [val] "=&r" (val), \
84 [orig] "=&r" (orig) \
85 : [ctr] "r" (&v->counter), \
86 [i] "ir" (i) \
87 : "cc"); \
88 \
89 smp_mb(); \
90 \
91 return orig; \
92}
93
94#else /* !CONFIG_ARC_HAS_LLSC */
95
96#ifndef CONFIG_SMP
97
98 /* violating atomic_xxx API locking protocol in UP for optimization sake */
99#define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
100
101#else
102
103static inline void atomic_set(atomic_t *v, int i)
104{
105 /*
106 * Independent of hardware support, all of the atomic_xxx() APIs need
107 * to follow the same locking rules to make sure that a "hardware"
108 * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn
109 * sequence
110 *
111 * Thus atomic_set() despite being 1 insn (and seemingly atomic)
112 * requires the locking.
113 */
114 unsigned long flags;
115
116 atomic_ops_lock(flags);
117 WRITE_ONCE(v->counter, i);
118 atomic_ops_unlock(flags);
119}
120
121#define atomic_set_release(v, i) atomic_set((v), (i))
122
123#endif
124
125/*
126 * Non hardware assisted Atomic-R-M-W
127 * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
128 */
129
130#define ATOMIC_OP(op, c_op, asm_op) \
131static inline void atomic_##op(int i, atomic_t *v) \
132{ \
133 unsigned long flags; \
134 \
135 atomic_ops_lock(flags); \
136 v->counter c_op i; \
137 atomic_ops_unlock(flags); \
138}
139
140#define ATOMIC_OP_RETURN(op, c_op, asm_op) \
141static inline int atomic_##op##_return(int i, atomic_t *v) \
142{ \
143 unsigned long flags; \
144 unsigned long temp; \
145 \
146 /* \
147 * spin lock/unlock provides the needed smp_mb() before/after \
148 */ \
149 atomic_ops_lock(flags); \
150 temp = v->counter; \
151 temp c_op i; \
152 v->counter = temp; \
153 atomic_ops_unlock(flags); \
154 \
155 return temp; \
156}
157
158#define ATOMIC_FETCH_OP(op, c_op, asm_op) \
159static inline int atomic_fetch_##op(int i, atomic_t *v) \
160{ \
161 unsigned long flags; \
162 unsigned long orig; \
163 \
164 /* \
165 * spin lock/unlock provides the needed smp_mb() before/after \
166 */ \
167 atomic_ops_lock(flags); \
168 orig = v->counter; \
169 v->counter c_op i; \
170 atomic_ops_unlock(flags); \
171 \
172 return orig; \
173}
174
175#endif /* !CONFIG_ARC_HAS_LLSC */
176
177#define ATOMIC_OPS(op, c_op, asm_op) \
178 ATOMIC_OP(op, c_op, asm_op) \
179 ATOMIC_OP_RETURN(op, c_op, asm_op) \
180 ATOMIC_FETCH_OP(op, c_op, asm_op)
181
182ATOMIC_OPS(add, +=, add)
183ATOMIC_OPS(sub, -=, sub)
184
185#define atomic_andnot atomic_andnot
186#define atomic_fetch_andnot atomic_fetch_andnot
187
188#undef ATOMIC_OPS
189#define ATOMIC_OPS(op, c_op, asm_op) \
190 ATOMIC_OP(op, c_op, asm_op) \
191 ATOMIC_FETCH_OP(op, c_op, asm_op)
192
193ATOMIC_OPS(and, &=, and)
194ATOMIC_OPS(andnot, &= ~, bic)
195ATOMIC_OPS(or, |=, or)
196ATOMIC_OPS(xor, ^=, xor)
197
198#else /* CONFIG_ARC_PLAT_EZNPS */
199
200static inline int atomic_read(const atomic_t *v)
201{
202 int temp;
203
204 __asm__ __volatile__(
205 " ld.di %0, [%1]"
206 : "=r"(temp)
207 : "r"(&v->counter)
208 : "memory");
209 return temp;
210}
211
212static inline void atomic_set(atomic_t *v, int i)
213{
214 __asm__ __volatile__(
215 " st.di %0,[%1]"
216 :
217 : "r"(i), "r"(&v->counter)
218 : "memory");
219}
220
221#define ATOMIC_OP(op, c_op, asm_op) \
222static inline void atomic_##op(int i, atomic_t *v) \
223{ \
224 __asm__ __volatile__( \
225 " mov r2, %0\n" \
226 " mov r3, %1\n" \
227 " .word %2\n" \
228 : \
229 : "r"(i), "r"(&v->counter), "i"(asm_op) \
230 : "r2", "r3", "memory"); \
231} \
232
233#define ATOMIC_OP_RETURN(op, c_op, asm_op) \
234static inline int atomic_##op##_return(int i, atomic_t *v) \
235{ \
236 unsigned int temp = i; \
237 \
238 /* Explicit full memory barrier needed before/after */ \
239 smp_mb(); \
240 \
241 __asm__ __volatile__( \
242 " mov r2, %0\n" \
243 " mov r3, %1\n" \
244 " .word %2\n" \
245 " mov %0, r2" \
246 : "+r"(temp) \
247 : "r"(&v->counter), "i"(asm_op) \
248 : "r2", "r3", "memory"); \
249 \
250 smp_mb(); \
251 \
252 temp c_op i; \
253 \
254 return temp; \
255}
256
257#define ATOMIC_FETCH_OP(op, c_op, asm_op) \
258static inline int atomic_fetch_##op(int i, atomic_t *v) \
259{ \
260 unsigned int temp = i; \
261 \
262 /* Explicit full memory barrier needed before/after */ \
263 smp_mb(); \
264 \
265 __asm__ __volatile__( \
266 " mov r2, %0\n" \
267 " mov r3, %1\n" \
268 " .word %2\n" \
269 " mov %0, r2" \
270 : "+r"(temp) \
271 : "r"(&v->counter), "i"(asm_op) \
272 : "r2", "r3", "memory"); \
273 \
274 smp_mb(); \
275 \
276 return temp; \
277}
278
279#define ATOMIC_OPS(op, c_op, asm_op) \
280 ATOMIC_OP(op, c_op, asm_op) \
281 ATOMIC_OP_RETURN(op, c_op, asm_op) \
282 ATOMIC_FETCH_OP(op, c_op, asm_op)
283
284ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3)
285#define atomic_sub(i, v) atomic_add(-(i), (v))
286#define atomic_sub_return(i, v) atomic_add_return(-(i), (v))
287#define atomic_fetch_sub(i, v) atomic_fetch_add(-(i), (v))
288
289#undef ATOMIC_OPS
290#define ATOMIC_OPS(op, c_op, asm_op) \
291 ATOMIC_OP(op, c_op, asm_op) \
292 ATOMIC_FETCH_OP(op, c_op, asm_op)
293
294ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3)
295ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3)
296ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)
297
298#endif /* CONFIG_ARC_PLAT_EZNPS */
299
300#undef ATOMIC_OPS
301#undef ATOMIC_FETCH_OP
302#undef ATOMIC_OP_RETURN
303#undef ATOMIC_OP
304
305#ifdef CONFIG_GENERIC_ATOMIC64
306
307#include <asm-generic/atomic64.h>
308
309#else /* Kconfig ensures this is only enabled with needed h/w assist */
310
311/*
312 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)
313 * - The address HAS to be 64-bit aligned
314 * - There are 2 semantics involved here:
315 * = exclusive implies no interim update between load/store to same addr
316 * = both words are observed/updated together: this is guaranteed even
317 * for regular 64-bit load (LDD) / store (STD). Thus atomic64_set()
318 * is NOT required to use LLOCKD+SCONDD, STD suffices
319 */
320
321typedef struct {
322 s64 __aligned(8) counter;
323} atomic64_t;
324
325#define ATOMIC64_INIT(a) { (a) }
326
327static inline s64 atomic64_read(const atomic64_t *v)
328{
329 s64 val;
330
331 __asm__ __volatile__(
332 " ldd %0, [%1] \n"
333 : "=r"(val)
334 : "r"(&v->counter));
335
336 return val;
337}
338
339static inline void atomic64_set(atomic64_t *v, s64 a)
340{
341 /*
342 * This could have been a simple assignment in "C" but would need
343 * explicit volatile. Otherwise gcc optimizers could elide the store
344 * which borked atomic64 self-test
345 * In the inline asm version, memory clobber needed for exact same
346 * reason, to tell gcc about the store.
347 *
348 * This however is not needed for sibling atomic64_add() etc since both
349 * load/store are explicitly done in inline asm. As long as API is used
350 * for each access, gcc has no way to optimize away any load/store
351 */
352 __asm__ __volatile__(
353 " std %0, [%1] \n"
354 :
355 : "r"(a), "r"(&v->counter)
356 : "memory");
357}
358
359#define ATOMIC64_OP(op, op1, op2) \
360static inline void atomic64_##op(s64 a, atomic64_t *v) \
361{ \
362 s64 val; \
363 \
364 __asm__ __volatile__( \
365 "1: \n" \
366 " llockd %0, [%1] \n" \
367 " " #op1 " %L0, %L0, %L2 \n" \
368 " " #op2 " %H0, %H0, %H2 \n" \
369 " scondd %0, [%1] \n" \
370 " bnz 1b \n" \
371 : "=&r"(val) \
372 : "r"(&v->counter), "ir"(a) \
373 : "cc"); \
374} \
375
376#define ATOMIC64_OP_RETURN(op, op1, op2) \
377static inline s64 atomic64_##op##_return(s64 a, atomic64_t *v) \
378{ \
379 s64 val; \
380 \
381 smp_mb(); \
382 \
383 __asm__ __volatile__( \
384 "1: \n" \
385 " llockd %0, [%1] \n" \
386 " " #op1 " %L0, %L0, %L2 \n" \
387 " " #op2 " %H0, %H0, %H2 \n" \
388 " scondd %0, [%1] \n" \
389 " bnz 1b \n" \
390 : [val] "=&r"(val) \
391 : "r"(&v->counter), "ir"(a) \
392 : "cc"); /* memory clobber comes from smp_mb() */ \
393 \
394 smp_mb(); \
395 \
396 return val; \
397}
398
399#define ATOMIC64_FETCH_OP(op, op1, op2) \
400static inline s64 atomic64_fetch_##op(s64 a, atomic64_t *v) \
401{ \
402 s64 val, orig; \
403 \
404 smp_mb(); \
405 \
406 __asm__ __volatile__( \
407 "1: \n" \
408 " llockd %0, [%2] \n" \
409 " " #op1 " %L1, %L0, %L3 \n" \
410 " " #op2 " %H1, %H0, %H3 \n" \
411 " scondd %1, [%2] \n" \
412 " bnz 1b \n" \
413 : "=&r"(orig), "=&r"(val) \
414 : "r"(&v->counter), "ir"(a) \
415 : "cc"); /* memory clobber comes from smp_mb() */ \
416 \
417 smp_mb(); \
418 \
419 return orig; \
420}
421
422#define ATOMIC64_OPS(op, op1, op2) \
423 ATOMIC64_OP(op, op1, op2) \
424 ATOMIC64_OP_RETURN(op, op1, op2) \
425 ATOMIC64_FETCH_OP(op, op1, op2)
426
427#define atomic64_andnot atomic64_andnot
428#define atomic64_fetch_andnot atomic64_fetch_andnot
429
430ATOMIC64_OPS(add, add.f, adc)
431ATOMIC64_OPS(sub, sub.f, sbc)
432ATOMIC64_OPS(and, and, and)
433ATOMIC64_OPS(andnot, bic, bic)
434ATOMIC64_OPS(or, or, or)
435ATOMIC64_OPS(xor, xor, xor)
436
437#undef ATOMIC64_OPS
438#undef ATOMIC64_FETCH_OP
439#undef ATOMIC64_OP_RETURN
440#undef ATOMIC64_OP
441
442static inline s64
443atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new)
444{
445 s64 prev;
446
447 smp_mb();
448
449 __asm__ __volatile__(
450 "1: llockd %0, [%1] \n"
451 " brne %L0, %L2, 2f \n"
452 " brne %H0, %H2, 2f \n"
453 " scondd %3, [%1] \n"
454 " bnz 1b \n"
455 "2: \n"
456 : "=&r"(prev)
457 : "r"(ptr), "ir"(expected), "r"(new)
458 : "cc"); /* memory clobber comes from smp_mb() */
459
460 smp_mb();
461
462 return prev;
463}
464
465static inline s64 atomic64_xchg(atomic64_t *ptr, s64 new)
466{
467 s64 prev;
468
469 smp_mb();
470
471 __asm__ __volatile__(
472 "1: llockd %0, [%1] \n"
473 " scondd %2, [%1] \n"
474 " bnz 1b \n"
475 "2: \n"
476 : "=&r"(prev)
477 : "r"(ptr), "r"(new)
478 : "cc"); /* memory clobber comes from smp_mb() */
479
480 smp_mb();
481
482 return prev;
483}
484
485/**
486 * atomic64_dec_if_positive - decrement by 1 if old value positive
487 * @v: pointer of type atomic64_t
488 *
489 * The function returns the old value of *v minus 1, even if
490 * the atomic variable, v, was not decremented.
491 */
492
493static inline s64 atomic64_dec_if_positive(atomic64_t *v)
494{
495 s64 val;
496
497 smp_mb();
498
499 __asm__ __volatile__(
500 "1: llockd %0, [%1] \n"
501 " sub.f %L0, %L0, 1 # w0 - 1, set C on borrow\n"
502 " sub.c %H0, %H0, 1 # if C set, w1 - 1\n"
503 " brlt %H0, 0, 2f \n"
504 " scondd %0, [%1] \n"
505 " bnz 1b \n"
506 "2: \n"
507 : "=&r"(val)
508 : "r"(&v->counter)
509 : "cc"); /* memory clobber comes from smp_mb() */
510
511 smp_mb();
512
513 return val;
514}
515#define atomic64_dec_if_positive atomic64_dec_if_positive
516
517/**
518 * atomic64_fetch_add_unless - add unless the number is a given value
519 * @v: pointer of type atomic64_t
520 * @a: the amount to add to v...
521 * @u: ...unless v is equal to u.
522 *
523 * Atomically adds @a to @v, if it was not @u.
524 * Returns the old value of @v
525 */
526static inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
527{
528 s64 old, temp;
529
530 smp_mb();
531
532 __asm__ __volatile__(
533 "1: llockd %0, [%2] \n"
534 " brne %L0, %L4, 2f # continue to add since v != u \n"
535 " breq.d %H0, %H4, 3f # return since v == u \n"
536 "2: \n"
537 " add.f %L1, %L0, %L3 \n"
538 " adc %H1, %H0, %H3 \n"
539 " scondd %1, [%2] \n"
540 " bnz 1b \n"
541 "3: \n"
542 : "=&r"(old), "=&r" (temp)
543 : "r"(&v->counter), "r"(a), "r"(u)
544 : "cc"); /* memory clobber comes from smp_mb() */
545
546 smp_mb();
547
548 return old;
549}
550#define atomic64_fetch_add_unless atomic64_fetch_add_unless
551
552#endif /* !CONFIG_GENERIC_ATOMIC64 */
553
554#endif /* !__ASSEMBLY__ */
555
556#endif