Loading...
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Generic barrier definitions.
4 *
5 * It should be possible to use these on really simple architectures,
6 * but it serves more as a starting point for new ports.
7 *
8 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
9 * Written by David Howells (dhowells@redhat.com)
10 */
11#ifndef __ASM_GENERIC_BARRIER_H
12#define __ASM_GENERIC_BARRIER_H
13
14#ifndef __ASSEMBLY__
15
16#include <linux/compiler.h>
17
18#ifndef nop
19#define nop() asm volatile ("nop")
20#endif
21
22/*
23 * Force strict CPU ordering. And yes, this is required on UP too when we're
24 * talking to devices.
25 *
26 * Fall back to compiler barriers if nothing better is provided.
27 */
28
29#ifndef mb
30#define mb() barrier()
31#endif
32
33#ifndef rmb
34#define rmb() mb()
35#endif
36
37#ifndef wmb
38#define wmb() mb()
39#endif
40
41#ifndef dma_rmb
42#define dma_rmb() rmb()
43#endif
44
45#ifndef dma_wmb
46#define dma_wmb() wmb()
47#endif
48
49#ifndef read_barrier_depends
50#define read_barrier_depends() do { } while (0)
51#endif
52
53#ifndef __smp_mb
54#define __smp_mb() mb()
55#endif
56
57#ifndef __smp_rmb
58#define __smp_rmb() rmb()
59#endif
60
61#ifndef __smp_wmb
62#define __smp_wmb() wmb()
63#endif
64
65#ifndef __smp_read_barrier_depends
66#define __smp_read_barrier_depends() read_barrier_depends()
67#endif
68
69#ifdef CONFIG_SMP
70
71#ifndef smp_mb
72#define smp_mb() __smp_mb()
73#endif
74
75#ifndef smp_rmb
76#define smp_rmb() __smp_rmb()
77#endif
78
79#ifndef smp_wmb
80#define smp_wmb() __smp_wmb()
81#endif
82
83#ifndef smp_read_barrier_depends
84#define smp_read_barrier_depends() __smp_read_barrier_depends()
85#endif
86
87#else /* !CONFIG_SMP */
88
89#ifndef smp_mb
90#define smp_mb() barrier()
91#endif
92
93#ifndef smp_rmb
94#define smp_rmb() barrier()
95#endif
96
97#ifndef smp_wmb
98#define smp_wmb() barrier()
99#endif
100
101#ifndef smp_read_barrier_depends
102#define smp_read_barrier_depends() do { } while (0)
103#endif
104
105#endif /* CONFIG_SMP */
106
107#ifndef __smp_store_mb
108#define __smp_store_mb(var, value) do { WRITE_ONCE(var, value); __smp_mb(); } while (0)
109#endif
110
111#ifndef __smp_mb__before_atomic
112#define __smp_mb__before_atomic() __smp_mb()
113#endif
114
115#ifndef __smp_mb__after_atomic
116#define __smp_mb__after_atomic() __smp_mb()
117#endif
118
119#ifndef __smp_store_release
120#define __smp_store_release(p, v) \
121do { \
122 compiletime_assert_atomic_type(*p); \
123 __smp_mb(); \
124 WRITE_ONCE(*p, v); \
125} while (0)
126#endif
127
128#ifndef __smp_load_acquire
129#define __smp_load_acquire(p) \
130({ \
131 typeof(*p) ___p1 = READ_ONCE(*p); \
132 compiletime_assert_atomic_type(*p); \
133 __smp_mb(); \
134 ___p1; \
135})
136#endif
137
138#ifdef CONFIG_SMP
139
140#ifndef smp_store_mb
141#define smp_store_mb(var, value) __smp_store_mb(var, value)
142#endif
143
144#ifndef smp_mb__before_atomic
145#define smp_mb__before_atomic() __smp_mb__before_atomic()
146#endif
147
148#ifndef smp_mb__after_atomic
149#define smp_mb__after_atomic() __smp_mb__after_atomic()
150#endif
151
152#ifndef smp_store_release
153#define smp_store_release(p, v) __smp_store_release(p, v)
154#endif
155
156#ifndef smp_load_acquire
157#define smp_load_acquire(p) __smp_load_acquire(p)
158#endif
159
160#else /* !CONFIG_SMP */
161
162#ifndef smp_store_mb
163#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)
164#endif
165
166#ifndef smp_mb__before_atomic
167#define smp_mb__before_atomic() barrier()
168#endif
169
170#ifndef smp_mb__after_atomic
171#define smp_mb__after_atomic() barrier()
172#endif
173
174#ifndef smp_store_release
175#define smp_store_release(p, v) \
176do { \
177 compiletime_assert_atomic_type(*p); \
178 barrier(); \
179 WRITE_ONCE(*p, v); \
180} while (0)
181#endif
182
183#ifndef smp_load_acquire
184#define smp_load_acquire(p) \
185({ \
186 typeof(*p) ___p1 = READ_ONCE(*p); \
187 compiletime_assert_atomic_type(*p); \
188 barrier(); \
189 ___p1; \
190})
191#endif
192
193#endif /* CONFIG_SMP */
194
195/* Barriers for virtual machine guests when talking to an SMP host */
196#define virt_mb() __smp_mb()
197#define virt_rmb() __smp_rmb()
198#define virt_wmb() __smp_wmb()
199#define virt_read_barrier_depends() __smp_read_barrier_depends()
200#define virt_store_mb(var, value) __smp_store_mb(var, value)
201#define virt_mb__before_atomic() __smp_mb__before_atomic()
202#define virt_mb__after_atomic() __smp_mb__after_atomic()
203#define virt_store_release(p, v) __smp_store_release(p, v)
204#define virt_load_acquire(p) __smp_load_acquire(p)
205
206/**
207 * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency
208 *
209 * A control dependency provides a LOAD->STORE order, the additional RMB
210 * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order,
211 * aka. (load)-ACQUIRE.
212 *
213 * Architectures that do not do load speculation can have this be barrier().
214 */
215#ifndef smp_acquire__after_ctrl_dep
216#define smp_acquire__after_ctrl_dep() smp_rmb()
217#endif
218
219/**
220 * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees
221 * @ptr: pointer to the variable to wait on
222 * @cond: boolean expression to wait for
223 *
224 * Equivalent to using READ_ONCE() on the condition variable.
225 *
226 * Due to C lacking lambda expressions we load the value of *ptr into a
227 * pre-named variable @VAL to be used in @cond.
228 */
229#ifndef smp_cond_load_relaxed
230#define smp_cond_load_relaxed(ptr, cond_expr) ({ \
231 typeof(ptr) __PTR = (ptr); \
232 typeof(*ptr) VAL; \
233 for (;;) { \
234 VAL = READ_ONCE(*__PTR); \
235 if (cond_expr) \
236 break; \
237 cpu_relax(); \
238 } \
239 VAL; \
240})
241#endif
242
243/**
244 * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering
245 * @ptr: pointer to the variable to wait on
246 * @cond: boolean expression to wait for
247 *
248 * Equivalent to using smp_load_acquire() on the condition variable but employs
249 * the control dependency of the wait to reduce the barrier on many platforms.
250 */
251#ifndef smp_cond_load_acquire
252#define smp_cond_load_acquire(ptr, cond_expr) ({ \
253 typeof(*ptr) _val; \
254 _val = smp_cond_load_relaxed(ptr, cond_expr); \
255 smp_acquire__after_ctrl_dep(); \
256 _val; \
257})
258#endif
259
260#endif /* !__ASSEMBLY__ */
261#endif /* __ASM_GENERIC_BARRIER_H */
1/* Generic barrier definitions, based on MN10300 definitions.
2 *
3 * It should be possible to use these on really simple architectures,
4 * but it serves more as a starting point for new ports.
5 *
6 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
7 * Written by David Howells (dhowells@redhat.com)
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public Licence
11 * as published by the Free Software Foundation; either version
12 * 2 of the Licence, or (at your option) any later version.
13 */
14#ifndef __ASM_GENERIC_BARRIER_H
15#define __ASM_GENERIC_BARRIER_H
16
17#ifndef __ASSEMBLY__
18
19#define nop() asm volatile ("nop")
20
21/*
22 * Force strict CPU ordering.
23 * And yes, this is required on UP too when we're talking
24 * to devices.
25 *
26 * This implementation only contains a compiler barrier.
27 */
28
29#define mb() asm volatile ("": : :"memory")
30#define rmb() mb()
31#define wmb() asm volatile ("": : :"memory")
32
33#ifdef CONFIG_SMP
34#define smp_mb() mb()
35#define smp_rmb() rmb()
36#define smp_wmb() wmb()
37#else
38#define smp_mb() barrier()
39#define smp_rmb() barrier()
40#define smp_wmb() barrier()
41#endif
42
43#define set_mb(var, value) do { var = value; mb(); } while (0)
44#define set_wmb(var, value) do { var = value; wmb(); } while (0)
45
46#define read_barrier_depends() do {} while (0)
47#define smp_read_barrier_depends() do {} while (0)
48
49#endif /* !__ASSEMBLY__ */
50#endif /* __ASM_GENERIC_BARRIER_H */