Loading...
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
4 * Copyright (C) 2012 Regents of the University of California
5 * Copyright (C) 2017 SiFive
6 */
7
8#ifndef _ASM_RISCV_ATOMIC_H
9#define _ASM_RISCV_ATOMIC_H
10
11#ifdef CONFIG_GENERIC_ATOMIC64
12# include <asm-generic/atomic64.h>
13#else
14# if (__riscv_xlen < 64)
15# error "64-bit atomics require XLEN to be at least 64"
16# endif
17#endif
18
19#include <asm/cmpxchg.h>
20
21#define __atomic_acquire_fence() \
22 __asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory")
23
24#define __atomic_release_fence() \
25 __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory");
26
27static __always_inline int arch_atomic_read(const atomic_t *v)
28{
29 return READ_ONCE(v->counter);
30}
31static __always_inline void arch_atomic_set(atomic_t *v, int i)
32{
33 WRITE_ONCE(v->counter, i);
34}
35
36#ifndef CONFIG_GENERIC_ATOMIC64
37#define ATOMIC64_INIT(i) { (i) }
38static __always_inline s64 arch_atomic64_read(const atomic64_t *v)
39{
40 return READ_ONCE(v->counter);
41}
42static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
43{
44 WRITE_ONCE(v->counter, i);
45}
46#endif
47
48/*
49 * First, the atomic ops that have no ordering constraints and therefor don't
50 * have the AQ or RL bits set. These don't return anything, so there's only
51 * one version to worry about.
52 */
53#define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \
54static __always_inline \
55void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
56{ \
57 __asm__ __volatile__ ( \
58 " amo" #asm_op "." #asm_type " zero, %1, %0" \
59 : "+A" (v->counter) \
60 : "r" (I) \
61 : "memory"); \
62} \
63
64#ifdef CONFIG_GENERIC_ATOMIC64
65#define ATOMIC_OPS(op, asm_op, I) \
66 ATOMIC_OP (op, asm_op, I, w, int, )
67#else
68#define ATOMIC_OPS(op, asm_op, I) \
69 ATOMIC_OP (op, asm_op, I, w, int, ) \
70 ATOMIC_OP (op, asm_op, I, d, s64, 64)
71#endif
72
73ATOMIC_OPS(add, add, i)
74ATOMIC_OPS(sub, add, -i)
75ATOMIC_OPS(and, and, i)
76ATOMIC_OPS( or, or, i)
77ATOMIC_OPS(xor, xor, i)
78
79#undef ATOMIC_OP
80#undef ATOMIC_OPS
81
82/*
83 * Atomic ops that have ordered, relaxed, acquire, and release variants.
84 * There's two flavors of these: the arithmatic ops have both fetch and return
85 * versions, while the logical ops only have fetch versions.
86 */
87#define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \
88static __always_inline \
89c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i, \
90 atomic##prefix##_t *v) \
91{ \
92 register c_type ret; \
93 __asm__ __volatile__ ( \
94 " amo" #asm_op "." #asm_type " %1, %2, %0" \
95 : "+A" (v->counter), "=r" (ret) \
96 : "r" (I) \
97 : "memory"); \
98 return ret; \
99} \
100static __always_inline \
101c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \
102{ \
103 register c_type ret; \
104 __asm__ __volatile__ ( \
105 " amo" #asm_op "." #asm_type ".aqrl %1, %2, %0" \
106 : "+A" (v->counter), "=r" (ret) \
107 : "r" (I) \
108 : "memory"); \
109 return ret; \
110}
111
112#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix) \
113static __always_inline \
114c_type arch_atomic##prefix##_##op##_return_relaxed(c_type i, \
115 atomic##prefix##_t *v) \
116{ \
117 return arch_atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \
118} \
119static __always_inline \
120c_type arch_atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \
121{ \
122 return arch_atomic##prefix##_fetch_##op(i, v) c_op I; \
123}
124
125#ifdef CONFIG_GENERIC_ATOMIC64
126#define ATOMIC_OPS(op, asm_op, c_op, I) \
127 ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \
128 ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, )
129#else
130#define ATOMIC_OPS(op, asm_op, c_op, I) \
131 ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \
132 ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, ) \
133 ATOMIC_FETCH_OP( op, asm_op, I, d, s64, 64) \
134 ATOMIC_OP_RETURN(op, asm_op, c_op, I, d, s64, 64)
135#endif
136
137ATOMIC_OPS(add, add, +, i)
138ATOMIC_OPS(sub, add, +, -i)
139
140#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
141#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
142#define arch_atomic_add_return arch_atomic_add_return
143#define arch_atomic_sub_return arch_atomic_sub_return
144
145#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
146#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
147#define arch_atomic_fetch_add arch_atomic_fetch_add
148#define arch_atomic_fetch_sub arch_atomic_fetch_sub
149
150#ifndef CONFIG_GENERIC_ATOMIC64
151#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed
152#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed
153#define arch_atomic64_add_return arch_atomic64_add_return
154#define arch_atomic64_sub_return arch_atomic64_sub_return
155
156#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed
157#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed
158#define arch_atomic64_fetch_add arch_atomic64_fetch_add
159#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
160#endif
161
162#undef ATOMIC_OPS
163
164#ifdef CONFIG_GENERIC_ATOMIC64
165#define ATOMIC_OPS(op, asm_op, I) \
166 ATOMIC_FETCH_OP(op, asm_op, I, w, int, )
167#else
168#define ATOMIC_OPS(op, asm_op, I) \
169 ATOMIC_FETCH_OP(op, asm_op, I, w, int, ) \
170 ATOMIC_FETCH_OP(op, asm_op, I, d, s64, 64)
171#endif
172
173ATOMIC_OPS(and, and, i)
174ATOMIC_OPS( or, or, i)
175ATOMIC_OPS(xor, xor, i)
176
177#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
178#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
179#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
180#define arch_atomic_fetch_and arch_atomic_fetch_and
181#define arch_atomic_fetch_or arch_atomic_fetch_or
182#define arch_atomic_fetch_xor arch_atomic_fetch_xor
183
184#ifndef CONFIG_GENERIC_ATOMIC64
185#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
186#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed
187#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
188#define arch_atomic64_fetch_and arch_atomic64_fetch_and
189#define arch_atomic64_fetch_or arch_atomic64_fetch_or
190#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
191#endif
192
193#undef ATOMIC_OPS
194
195#undef ATOMIC_FETCH_OP
196#undef ATOMIC_OP_RETURN
197
198/* This is required to provide a full barrier on success. */
199static __always_inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
200{
201 int prev, rc;
202
203 __asm__ __volatile__ (
204 "0: lr.w %[p], %[c]\n"
205 " beq %[p], %[u], 1f\n"
206 " add %[rc], %[p], %[a]\n"
207 " sc.w.rl %[rc], %[rc], %[c]\n"
208 " bnez %[rc], 0b\n"
209 RISCV_FULL_BARRIER
210 "1:\n"
211 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
212 : [a]"r" (a), [u]"r" (u)
213 : "memory");
214 return prev;
215}
216#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
217
218#ifndef CONFIG_GENERIC_ATOMIC64
219static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
220{
221 s64 prev;
222 long rc;
223
224 __asm__ __volatile__ (
225 "0: lr.d %[p], %[c]\n"
226 " beq %[p], %[u], 1f\n"
227 " add %[rc], %[p], %[a]\n"
228 " sc.d.rl %[rc], %[rc], %[c]\n"
229 " bnez %[rc], 0b\n"
230 RISCV_FULL_BARRIER
231 "1:\n"
232 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
233 : [a]"r" (a), [u]"r" (u)
234 : "memory");
235 return prev;
236}
237#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
238#endif
239
240static __always_inline bool arch_atomic_inc_unless_negative(atomic_t *v)
241{
242 int prev, rc;
243
244 __asm__ __volatile__ (
245 "0: lr.w %[p], %[c]\n"
246 " bltz %[p], 1f\n"
247 " addi %[rc], %[p], 1\n"
248 " sc.w.rl %[rc], %[rc], %[c]\n"
249 " bnez %[rc], 0b\n"
250 RISCV_FULL_BARRIER
251 "1:\n"
252 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
253 :
254 : "memory");
255 return !(prev < 0);
256}
257
258#define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
259
260static __always_inline bool arch_atomic_dec_unless_positive(atomic_t *v)
261{
262 int prev, rc;
263
264 __asm__ __volatile__ (
265 "0: lr.w %[p], %[c]\n"
266 " bgtz %[p], 1f\n"
267 " addi %[rc], %[p], -1\n"
268 " sc.w.rl %[rc], %[rc], %[c]\n"
269 " bnez %[rc], 0b\n"
270 RISCV_FULL_BARRIER
271 "1:\n"
272 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
273 :
274 : "memory");
275 return !(prev > 0);
276}
277
278#define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
279
280static __always_inline int arch_atomic_dec_if_positive(atomic_t *v)
281{
282 int prev, rc;
283
284 __asm__ __volatile__ (
285 "0: lr.w %[p], %[c]\n"
286 " addi %[rc], %[p], -1\n"
287 " bltz %[rc], 1f\n"
288 " sc.w.rl %[rc], %[rc], %[c]\n"
289 " bnez %[rc], 0b\n"
290 RISCV_FULL_BARRIER
291 "1:\n"
292 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
293 :
294 : "memory");
295 return prev - 1;
296}
297
298#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
299
300#ifndef CONFIG_GENERIC_ATOMIC64
301static __always_inline bool arch_atomic64_inc_unless_negative(atomic64_t *v)
302{
303 s64 prev;
304 long rc;
305
306 __asm__ __volatile__ (
307 "0: lr.d %[p], %[c]\n"
308 " bltz %[p], 1f\n"
309 " addi %[rc], %[p], 1\n"
310 " sc.d.rl %[rc], %[rc], %[c]\n"
311 " bnez %[rc], 0b\n"
312 RISCV_FULL_BARRIER
313 "1:\n"
314 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
315 :
316 : "memory");
317 return !(prev < 0);
318}
319
320#define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
321
322static __always_inline bool arch_atomic64_dec_unless_positive(atomic64_t *v)
323{
324 s64 prev;
325 long rc;
326
327 __asm__ __volatile__ (
328 "0: lr.d %[p], %[c]\n"
329 " bgtz %[p], 1f\n"
330 " addi %[rc], %[p], -1\n"
331 " sc.d.rl %[rc], %[rc], %[c]\n"
332 " bnez %[rc], 0b\n"
333 RISCV_FULL_BARRIER
334 "1:\n"
335 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
336 :
337 : "memory");
338 return !(prev > 0);
339}
340
341#define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
342
343static __always_inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
344{
345 s64 prev;
346 long rc;
347
348 __asm__ __volatile__ (
349 "0: lr.d %[p], %[c]\n"
350 " addi %[rc], %[p], -1\n"
351 " bltz %[rc], 1f\n"
352 " sc.d.rl %[rc], %[rc], %[c]\n"
353 " bnez %[rc], 0b\n"
354 RISCV_FULL_BARRIER
355 "1:\n"
356 : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
357 :
358 : "memory");
359 return prev - 1;
360}
361
362#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
363#endif
364
365#endif /* _ASM_RISCV_ATOMIC_H */