Loading...
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Generic barrier definitions.
4 *
5 * It should be possible to use these on really simple architectures,
6 * but it serves more as a starting point for new ports.
7 *
8 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
9 * Written by David Howells (dhowells@redhat.com)
10 */
11#ifndef __ASM_GENERIC_BARRIER_H
12#define __ASM_GENERIC_BARRIER_H
13
14#ifndef __ASSEMBLY__
15
16#include <linux/compiler.h>
17#include <linux/kcsan-checks.h>
18#include <asm/rwonce.h>
19
20#ifndef nop
21#define nop() asm volatile ("nop")
22#endif
23
24/*
25 * Architectures that want generic instrumentation can define __ prefixed
26 * variants of all barriers.
27 */
28
29#ifdef __mb
30#define mb() do { kcsan_mb(); __mb(); } while (0)
31#endif
32
33#ifdef __rmb
34#define rmb() do { kcsan_rmb(); __rmb(); } while (0)
35#endif
36
37#ifdef __wmb
38#define wmb() do { kcsan_wmb(); __wmb(); } while (0)
39#endif
40
41#ifdef __dma_mb
42#define dma_mb() do { kcsan_mb(); __dma_mb(); } while (0)
43#endif
44
45#ifdef __dma_rmb
46#define dma_rmb() do { kcsan_rmb(); __dma_rmb(); } while (0)
47#endif
48
49#ifdef __dma_wmb
50#define dma_wmb() do { kcsan_wmb(); __dma_wmb(); } while (0)
51#endif
52
53/*
54 * Force strict CPU ordering. And yes, this is required on UP too when we're
55 * talking to devices.
56 *
57 * Fall back to compiler barriers if nothing better is provided.
58 */
59
60#ifndef mb
61#define mb() barrier()
62#endif
63
64#ifndef rmb
65#define rmb() mb()
66#endif
67
68#ifndef wmb
69#define wmb() mb()
70#endif
71
72#ifndef dma_mb
73#define dma_mb() mb()
74#endif
75
76#ifndef dma_rmb
77#define dma_rmb() rmb()
78#endif
79
80#ifndef dma_wmb
81#define dma_wmb() wmb()
82#endif
83
84#ifndef __smp_mb
85#define __smp_mb() mb()
86#endif
87
88#ifndef __smp_rmb
89#define __smp_rmb() rmb()
90#endif
91
92#ifndef __smp_wmb
93#define __smp_wmb() wmb()
94#endif
95
96#ifdef CONFIG_SMP
97
98#ifndef smp_mb
99#define smp_mb() do { kcsan_mb(); __smp_mb(); } while (0)
100#endif
101
102#ifndef smp_rmb
103#define smp_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0)
104#endif
105
106#ifndef smp_wmb
107#define smp_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0)
108#endif
109
110#else /* !CONFIG_SMP */
111
112#ifndef smp_mb
113#define smp_mb() barrier()
114#endif
115
116#ifndef smp_rmb
117#define smp_rmb() barrier()
118#endif
119
120#ifndef smp_wmb
121#define smp_wmb() barrier()
122#endif
123
124#endif /* CONFIG_SMP */
125
126#ifndef __smp_store_mb
127#define __smp_store_mb(var, value) do { WRITE_ONCE(var, value); __smp_mb(); } while (0)
128#endif
129
130#ifndef __smp_mb__before_atomic
131#define __smp_mb__before_atomic() __smp_mb()
132#endif
133
134#ifndef __smp_mb__after_atomic
135#define __smp_mb__after_atomic() __smp_mb()
136#endif
137
138#ifndef __smp_store_release
139#define __smp_store_release(p, v) \
140do { \
141 compiletime_assert_atomic_type(*p); \
142 __smp_mb(); \
143 WRITE_ONCE(*p, v); \
144} while (0)
145#endif
146
147#ifndef __smp_load_acquire
148#define __smp_load_acquire(p) \
149({ \
150 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \
151 compiletime_assert_atomic_type(*p); \
152 __smp_mb(); \
153 (typeof(*p))___p1; \
154})
155#endif
156
157#ifdef CONFIG_SMP
158
159#ifndef smp_store_mb
160#define smp_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0)
161#endif
162
163#ifndef smp_mb__before_atomic
164#define smp_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0)
165#endif
166
167#ifndef smp_mb__after_atomic
168#define smp_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0)
169#endif
170
171#ifndef smp_store_release
172#define smp_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0)
173#endif
174
175#ifndef smp_load_acquire
176#define smp_load_acquire(p) __smp_load_acquire(p)
177#endif
178
179#else /* !CONFIG_SMP */
180
181#ifndef smp_store_mb
182#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)
183#endif
184
185#ifndef smp_mb__before_atomic
186#define smp_mb__before_atomic() barrier()
187#endif
188
189#ifndef smp_mb__after_atomic
190#define smp_mb__after_atomic() barrier()
191#endif
192
193#ifndef smp_store_release
194#define smp_store_release(p, v) \
195do { \
196 compiletime_assert_atomic_type(*p); \
197 barrier(); \
198 WRITE_ONCE(*p, v); \
199} while (0)
200#endif
201
202#ifndef smp_load_acquire
203#define smp_load_acquire(p) \
204({ \
205 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \
206 compiletime_assert_atomic_type(*p); \
207 barrier(); \
208 (typeof(*p))___p1; \
209})
210#endif
211
212#endif /* CONFIG_SMP */
213
214/* Barriers for virtual machine guests when talking to an SMP host */
215#define virt_mb() do { kcsan_mb(); __smp_mb(); } while (0)
216#define virt_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0)
217#define virt_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0)
218#define virt_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0)
219#define virt_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0)
220#define virt_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0)
221#define virt_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while (0)
222#define virt_load_acquire(p) __smp_load_acquire(p)
223
224/**
225 * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency
226 *
227 * A control dependency provides a LOAD->STORE order, the additional RMB
228 * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order,
229 * aka. (load)-ACQUIRE.
230 *
231 * Architectures that do not do load speculation can have this be barrier().
232 */
233#ifndef smp_acquire__after_ctrl_dep
234#define smp_acquire__after_ctrl_dep() smp_rmb()
235#endif
236
237/**
238 * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees
239 * @ptr: pointer to the variable to wait on
240 * @cond: boolean expression to wait for
241 *
242 * Equivalent to using READ_ONCE() on the condition variable.
243 *
244 * Due to C lacking lambda expressions we load the value of *ptr into a
245 * pre-named variable @VAL to be used in @cond.
246 */
247#ifndef smp_cond_load_relaxed
248#define smp_cond_load_relaxed(ptr, cond_expr) ({ \
249 typeof(ptr) __PTR = (ptr); \
250 __unqual_scalar_typeof(*ptr) VAL; \
251 for (;;) { \
252 VAL = READ_ONCE(*__PTR); \
253 if (cond_expr) \
254 break; \
255 cpu_relax(); \
256 } \
257 (typeof(*ptr))VAL; \
258})
259#endif
260
261/**
262 * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering
263 * @ptr: pointer to the variable to wait on
264 * @cond: boolean expression to wait for
265 *
266 * Equivalent to using smp_load_acquire() on the condition variable but employs
267 * the control dependency of the wait to reduce the barrier on many platforms.
268 */
269#ifndef smp_cond_load_acquire
270#define smp_cond_load_acquire(ptr, cond_expr) ({ \
271 __unqual_scalar_typeof(*ptr) _val; \
272 _val = smp_cond_load_relaxed(ptr, cond_expr); \
273 smp_acquire__after_ctrl_dep(); \
274 (typeof(*ptr))_val; \
275})
276#endif
277
278/*
279 * pmem_wmb() ensures that all stores for which the modification
280 * are written to persistent storage by preceding instructions have
281 * updated persistent storage before any data access or data transfer
282 * caused by subsequent instructions is initiated.
283 */
284#ifndef pmem_wmb
285#define pmem_wmb() wmb()
286#endif
287
288/*
289 * ioremap_wc() maps I/O memory as memory with write-combining attributes. For
290 * this kind of memory accesses, the CPU may wait for prior accesses to be
291 * merged with subsequent ones. In some situation, such wait is bad for the
292 * performance. io_stop_wc() can be used to prevent the merging of
293 * write-combining memory accesses before this macro with those after it.
294 */
295#ifndef io_stop_wc
296#define io_stop_wc() do { } while (0)
297#endif
298
299#endif /* !__ASSEMBLY__ */
300#endif /* __ASM_GENERIC_BARRIER_H */
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Generic barrier definitions.
4 *
5 * It should be possible to use these on really simple architectures,
6 * but it serves more as a starting point for new ports.
7 *
8 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
9 * Written by David Howells (dhowells@redhat.com)
10 */
11#ifndef __ASM_GENERIC_BARRIER_H
12#define __ASM_GENERIC_BARRIER_H
13
14#ifndef __ASSEMBLY__
15
16#include <asm/rwonce.h>
17
18#ifndef nop
19#define nop() asm volatile ("nop")
20#endif
21
22/*
23 * Force strict CPU ordering. And yes, this is required on UP too when we're
24 * talking to devices.
25 *
26 * Fall back to compiler barriers if nothing better is provided.
27 */
28
29#ifndef mb
30#define mb() barrier()
31#endif
32
33#ifndef rmb
34#define rmb() mb()
35#endif
36
37#ifndef wmb
38#define wmb() mb()
39#endif
40
41#ifndef dma_rmb
42#define dma_rmb() rmb()
43#endif
44
45#ifndef dma_wmb
46#define dma_wmb() wmb()
47#endif
48
49#ifndef __smp_mb
50#define __smp_mb() mb()
51#endif
52
53#ifndef __smp_rmb
54#define __smp_rmb() rmb()
55#endif
56
57#ifndef __smp_wmb
58#define __smp_wmb() wmb()
59#endif
60
61#ifdef CONFIG_SMP
62
63#ifndef smp_mb
64#define smp_mb() __smp_mb()
65#endif
66
67#ifndef smp_rmb
68#define smp_rmb() __smp_rmb()
69#endif
70
71#ifndef smp_wmb
72#define smp_wmb() __smp_wmb()
73#endif
74
75#else /* !CONFIG_SMP */
76
77#ifndef smp_mb
78#define smp_mb() barrier()
79#endif
80
81#ifndef smp_rmb
82#define smp_rmb() barrier()
83#endif
84
85#ifndef smp_wmb
86#define smp_wmb() barrier()
87#endif
88
89#endif /* CONFIG_SMP */
90
91#ifndef __smp_store_mb
92#define __smp_store_mb(var, value) do { WRITE_ONCE(var, value); __smp_mb(); } while (0)
93#endif
94
95#ifndef __smp_mb__before_atomic
96#define __smp_mb__before_atomic() __smp_mb()
97#endif
98
99#ifndef __smp_mb__after_atomic
100#define __smp_mb__after_atomic() __smp_mb()
101#endif
102
103#ifndef __smp_store_release
104#define __smp_store_release(p, v) \
105do { \
106 compiletime_assert_atomic_type(*p); \
107 __smp_mb(); \
108 WRITE_ONCE(*p, v); \
109} while (0)
110#endif
111
112#ifndef __smp_load_acquire
113#define __smp_load_acquire(p) \
114({ \
115 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \
116 compiletime_assert_atomic_type(*p); \
117 __smp_mb(); \
118 (typeof(*p))___p1; \
119})
120#endif
121
122#ifdef CONFIG_SMP
123
124#ifndef smp_store_mb
125#define smp_store_mb(var, value) __smp_store_mb(var, value)
126#endif
127
128#ifndef smp_mb__before_atomic
129#define smp_mb__before_atomic() __smp_mb__before_atomic()
130#endif
131
132#ifndef smp_mb__after_atomic
133#define smp_mb__after_atomic() __smp_mb__after_atomic()
134#endif
135
136#ifndef smp_store_release
137#define smp_store_release(p, v) __smp_store_release(p, v)
138#endif
139
140#ifndef smp_load_acquire
141#define smp_load_acquire(p) __smp_load_acquire(p)
142#endif
143
144#else /* !CONFIG_SMP */
145
146#ifndef smp_store_mb
147#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)
148#endif
149
150#ifndef smp_mb__before_atomic
151#define smp_mb__before_atomic() barrier()
152#endif
153
154#ifndef smp_mb__after_atomic
155#define smp_mb__after_atomic() barrier()
156#endif
157
158#ifndef smp_store_release
159#define smp_store_release(p, v) \
160do { \
161 compiletime_assert_atomic_type(*p); \
162 barrier(); \
163 WRITE_ONCE(*p, v); \
164} while (0)
165#endif
166
167#ifndef smp_load_acquire
168#define smp_load_acquire(p) \
169({ \
170 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \
171 compiletime_assert_atomic_type(*p); \
172 barrier(); \
173 (typeof(*p))___p1; \
174})
175#endif
176
177#endif /* CONFIG_SMP */
178
179/* Barriers for virtual machine guests when talking to an SMP host */
180#define virt_mb() __smp_mb()
181#define virt_rmb() __smp_rmb()
182#define virt_wmb() __smp_wmb()
183#define virt_store_mb(var, value) __smp_store_mb(var, value)
184#define virt_mb__before_atomic() __smp_mb__before_atomic()
185#define virt_mb__after_atomic() __smp_mb__after_atomic()
186#define virt_store_release(p, v) __smp_store_release(p, v)
187#define virt_load_acquire(p) __smp_load_acquire(p)
188
189/**
190 * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency
191 *
192 * A control dependency provides a LOAD->STORE order, the additional RMB
193 * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order,
194 * aka. (load)-ACQUIRE.
195 *
196 * Architectures that do not do load speculation can have this be barrier().
197 */
198#ifndef smp_acquire__after_ctrl_dep
199#define smp_acquire__after_ctrl_dep() smp_rmb()
200#endif
201
202/**
203 * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees
204 * @ptr: pointer to the variable to wait on
205 * @cond: boolean expression to wait for
206 *
207 * Equivalent to using READ_ONCE() on the condition variable.
208 *
209 * Due to C lacking lambda expressions we load the value of *ptr into a
210 * pre-named variable @VAL to be used in @cond.
211 */
212#ifndef smp_cond_load_relaxed
213#define smp_cond_load_relaxed(ptr, cond_expr) ({ \
214 typeof(ptr) __PTR = (ptr); \
215 __unqual_scalar_typeof(*ptr) VAL; \
216 for (;;) { \
217 VAL = READ_ONCE(*__PTR); \
218 if (cond_expr) \
219 break; \
220 cpu_relax(); \
221 } \
222 (typeof(*ptr))VAL; \
223})
224#endif
225
226/**
227 * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering
228 * @ptr: pointer to the variable to wait on
229 * @cond: boolean expression to wait for
230 *
231 * Equivalent to using smp_load_acquire() on the condition variable but employs
232 * the control dependency of the wait to reduce the barrier on many platforms.
233 */
234#ifndef smp_cond_load_acquire
235#define smp_cond_load_acquire(ptr, cond_expr) ({ \
236 __unqual_scalar_typeof(*ptr) _val; \
237 _val = smp_cond_load_relaxed(ptr, cond_expr); \
238 smp_acquire__after_ctrl_dep(); \
239 (typeof(*ptr))_val; \
240})
241#endif
242
243/*
244 * pmem_wmb() ensures that all stores for which the modification
245 * are written to persistent storage by preceding instructions have
246 * updated persistent storage before any data access or data transfer
247 * caused by subsequent instructions is initiated.
248 */
249#ifndef pmem_wmb
250#define pmem_wmb() wmb()
251#endif
252
253#endif /* !__ASSEMBLY__ */
254#endif /* __ASM_GENERIC_BARRIER_H */