Loading...
1/*
2 * Generic barrier definitions, originally based on MN10300 definitions.
3 *
4 * It should be possible to use these on really simple architectures,
5 * but it serves more as a starting point for new ports.
6 *
7 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
8 * Written by David Howells (dhowells@redhat.com)
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public Licence
12 * as published by the Free Software Foundation; either version
13 * 2 of the Licence, or (at your option) any later version.
14 */
15#ifndef __ASM_GENERIC_BARRIER_H
16#define __ASM_GENERIC_BARRIER_H
17
18#ifndef __ASSEMBLY__
19
20#include <linux/compiler.h>
21
22#ifndef nop
23#define nop() asm volatile ("nop")
24#endif
25
26/*
27 * Force strict CPU ordering. And yes, this is required on UP too when we're
28 * talking to devices.
29 *
30 * Fall back to compiler barriers if nothing better is provided.
31 */
32
33#ifndef mb
34#define mb() barrier()
35#endif
36
37#ifndef rmb
38#define rmb() mb()
39#endif
40
41#ifndef wmb
42#define wmb() mb()
43#endif
44
45#ifndef dma_rmb
46#define dma_rmb() rmb()
47#endif
48
49#ifndef dma_wmb
50#define dma_wmb() wmb()
51#endif
52
53#ifndef read_barrier_depends
54#define read_barrier_depends() do { } while (0)
55#endif
56
57#ifndef __smp_mb
58#define __smp_mb() mb()
59#endif
60
61#ifndef __smp_rmb
62#define __smp_rmb() rmb()
63#endif
64
65#ifndef __smp_wmb
66#define __smp_wmb() wmb()
67#endif
68
69#ifndef __smp_read_barrier_depends
70#define __smp_read_barrier_depends() read_barrier_depends()
71#endif
72
73#ifdef CONFIG_SMP
74
75#ifndef smp_mb
76#define smp_mb() __smp_mb()
77#endif
78
79#ifndef smp_rmb
80#define smp_rmb() __smp_rmb()
81#endif
82
83#ifndef smp_wmb
84#define smp_wmb() __smp_wmb()
85#endif
86
87#ifndef smp_read_barrier_depends
88#define smp_read_barrier_depends() __smp_read_barrier_depends()
89#endif
90
91#else /* !CONFIG_SMP */
92
93#ifndef smp_mb
94#define smp_mb() barrier()
95#endif
96
97#ifndef smp_rmb
98#define smp_rmb() barrier()
99#endif
100
101#ifndef smp_wmb
102#define smp_wmb() barrier()
103#endif
104
105#ifndef smp_read_barrier_depends
106#define smp_read_barrier_depends() do { } while (0)
107#endif
108
109#endif /* CONFIG_SMP */
110
111#ifndef __smp_store_mb
112#define __smp_store_mb(var, value) do { WRITE_ONCE(var, value); __smp_mb(); } while (0)
113#endif
114
115#ifndef __smp_mb__before_atomic
116#define __smp_mb__before_atomic() __smp_mb()
117#endif
118
119#ifndef __smp_mb__after_atomic
120#define __smp_mb__after_atomic() __smp_mb()
121#endif
122
123#ifndef __smp_store_release
124#define __smp_store_release(p, v) \
125do { \
126 compiletime_assert_atomic_type(*p); \
127 __smp_mb(); \
128 WRITE_ONCE(*p, v); \
129} while (0)
130#endif
131
132#ifndef __smp_load_acquire
133#define __smp_load_acquire(p) \
134({ \
135 typeof(*p) ___p1 = READ_ONCE(*p); \
136 compiletime_assert_atomic_type(*p); \
137 __smp_mb(); \
138 ___p1; \
139})
140#endif
141
142#ifdef CONFIG_SMP
143
144#ifndef smp_store_mb
145#define smp_store_mb(var, value) __smp_store_mb(var, value)
146#endif
147
148#ifndef smp_mb__before_atomic
149#define smp_mb__before_atomic() __smp_mb__before_atomic()
150#endif
151
152#ifndef smp_mb__after_atomic
153#define smp_mb__after_atomic() __smp_mb__after_atomic()
154#endif
155
156#ifndef smp_store_release
157#define smp_store_release(p, v) __smp_store_release(p, v)
158#endif
159
160#ifndef smp_load_acquire
161#define smp_load_acquire(p) __smp_load_acquire(p)
162#endif
163
164#else /* !CONFIG_SMP */
165
166#ifndef smp_store_mb
167#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)
168#endif
169
170#ifndef smp_mb__before_atomic
171#define smp_mb__before_atomic() barrier()
172#endif
173
174#ifndef smp_mb__after_atomic
175#define smp_mb__after_atomic() barrier()
176#endif
177
178#ifndef smp_store_release
179#define smp_store_release(p, v) \
180do { \
181 compiletime_assert_atomic_type(*p); \
182 barrier(); \
183 WRITE_ONCE(*p, v); \
184} while (0)
185#endif
186
187#ifndef smp_load_acquire
188#define smp_load_acquire(p) \
189({ \
190 typeof(*p) ___p1 = READ_ONCE(*p); \
191 compiletime_assert_atomic_type(*p); \
192 barrier(); \
193 ___p1; \
194})
195#endif
196
197#endif /* CONFIG_SMP */
198
199/* Barriers for virtual machine guests when talking to an SMP host */
200#define virt_mb() __smp_mb()
201#define virt_rmb() __smp_rmb()
202#define virt_wmb() __smp_wmb()
203#define virt_read_barrier_depends() __smp_read_barrier_depends()
204#define virt_store_mb(var, value) __smp_store_mb(var, value)
205#define virt_mb__before_atomic() __smp_mb__before_atomic()
206#define virt_mb__after_atomic() __smp_mb__after_atomic()
207#define virt_store_release(p, v) __smp_store_release(p, v)
208#define virt_load_acquire(p) __smp_load_acquire(p)
209
210/**
211 * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency
212 *
213 * A control dependency provides a LOAD->STORE order, the additional RMB
214 * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order,
215 * aka. (load)-ACQUIRE.
216 *
217 * Architectures that do not do load speculation can have this be barrier().
218 */
219#ifndef smp_acquire__after_ctrl_dep
220#define smp_acquire__after_ctrl_dep() smp_rmb()
221#endif
222
223/**
224 * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering
225 * @ptr: pointer to the variable to wait on
226 * @cond: boolean expression to wait for
227 *
228 * Equivalent to using smp_load_acquire() on the condition variable but employs
229 * the control dependency of the wait to reduce the barrier on many platforms.
230 *
231 * Due to C lacking lambda expressions we load the value of *ptr into a
232 * pre-named variable @VAL to be used in @cond.
233 */
234#ifndef smp_cond_load_acquire
235#define smp_cond_load_acquire(ptr, cond_expr) ({ \
236 typeof(ptr) __PTR = (ptr); \
237 typeof(*ptr) VAL; \
238 for (;;) { \
239 VAL = READ_ONCE(*__PTR); \
240 if (cond_expr) \
241 break; \
242 cpu_relax(); \
243 } \
244 smp_acquire__after_ctrl_dep(); \
245 VAL; \
246})
247#endif
248
249#endif /* !__ASSEMBLY__ */
250#endif /* __ASM_GENERIC_BARRIER_H */
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Generic barrier definitions.
4 *
5 * It should be possible to use these on really simple architectures,
6 * but it serves more as a starting point for new ports.
7 *
8 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
9 * Written by David Howells (dhowells@redhat.com)
10 */
11#ifndef __ASM_GENERIC_BARRIER_H
12#define __ASM_GENERIC_BARRIER_H
13
14#ifndef __ASSEMBLY__
15
16#include <asm/rwonce.h>
17
18#ifndef nop
19#define nop() asm volatile ("nop")
20#endif
21
22/*
23 * Force strict CPU ordering. And yes, this is required on UP too when we're
24 * talking to devices.
25 *
26 * Fall back to compiler barriers if nothing better is provided.
27 */
28
29#ifndef mb
30#define mb() barrier()
31#endif
32
33#ifndef rmb
34#define rmb() mb()
35#endif
36
37#ifndef wmb
38#define wmb() mb()
39#endif
40
41#ifndef dma_rmb
42#define dma_rmb() rmb()
43#endif
44
45#ifndef dma_wmb
46#define dma_wmb() wmb()
47#endif
48
49#ifndef __smp_mb
50#define __smp_mb() mb()
51#endif
52
53#ifndef __smp_rmb
54#define __smp_rmb() rmb()
55#endif
56
57#ifndef __smp_wmb
58#define __smp_wmb() wmb()
59#endif
60
61#ifdef CONFIG_SMP
62
63#ifndef smp_mb
64#define smp_mb() __smp_mb()
65#endif
66
67#ifndef smp_rmb
68#define smp_rmb() __smp_rmb()
69#endif
70
71#ifndef smp_wmb
72#define smp_wmb() __smp_wmb()
73#endif
74
75#else /* !CONFIG_SMP */
76
77#ifndef smp_mb
78#define smp_mb() barrier()
79#endif
80
81#ifndef smp_rmb
82#define smp_rmb() barrier()
83#endif
84
85#ifndef smp_wmb
86#define smp_wmb() barrier()
87#endif
88
89#endif /* CONFIG_SMP */
90
91#ifndef __smp_store_mb
92#define __smp_store_mb(var, value) do { WRITE_ONCE(var, value); __smp_mb(); } while (0)
93#endif
94
95#ifndef __smp_mb__before_atomic
96#define __smp_mb__before_atomic() __smp_mb()
97#endif
98
99#ifndef __smp_mb__after_atomic
100#define __smp_mb__after_atomic() __smp_mb()
101#endif
102
103#ifndef __smp_store_release
104#define __smp_store_release(p, v) \
105do { \
106 compiletime_assert_atomic_type(*p); \
107 __smp_mb(); \
108 WRITE_ONCE(*p, v); \
109} while (0)
110#endif
111
112#ifndef __smp_load_acquire
113#define __smp_load_acquire(p) \
114({ \
115 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \
116 compiletime_assert_atomic_type(*p); \
117 __smp_mb(); \
118 (typeof(*p))___p1; \
119})
120#endif
121
122#ifdef CONFIG_SMP
123
124#ifndef smp_store_mb
125#define smp_store_mb(var, value) __smp_store_mb(var, value)
126#endif
127
128#ifndef smp_mb__before_atomic
129#define smp_mb__before_atomic() __smp_mb__before_atomic()
130#endif
131
132#ifndef smp_mb__after_atomic
133#define smp_mb__after_atomic() __smp_mb__after_atomic()
134#endif
135
136#ifndef smp_store_release
137#define smp_store_release(p, v) __smp_store_release(p, v)
138#endif
139
140#ifndef smp_load_acquire
141#define smp_load_acquire(p) __smp_load_acquire(p)
142#endif
143
144#else /* !CONFIG_SMP */
145
146#ifndef smp_store_mb
147#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)
148#endif
149
150#ifndef smp_mb__before_atomic
151#define smp_mb__before_atomic() barrier()
152#endif
153
154#ifndef smp_mb__after_atomic
155#define smp_mb__after_atomic() barrier()
156#endif
157
158#ifndef smp_store_release
159#define smp_store_release(p, v) \
160do { \
161 compiletime_assert_atomic_type(*p); \
162 barrier(); \
163 WRITE_ONCE(*p, v); \
164} while (0)
165#endif
166
167#ifndef smp_load_acquire
168#define smp_load_acquire(p) \
169({ \
170 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \
171 compiletime_assert_atomic_type(*p); \
172 barrier(); \
173 (typeof(*p))___p1; \
174})
175#endif
176
177#endif /* CONFIG_SMP */
178
179/* Barriers for virtual machine guests when talking to an SMP host */
180#define virt_mb() __smp_mb()
181#define virt_rmb() __smp_rmb()
182#define virt_wmb() __smp_wmb()
183#define virt_store_mb(var, value) __smp_store_mb(var, value)
184#define virt_mb__before_atomic() __smp_mb__before_atomic()
185#define virt_mb__after_atomic() __smp_mb__after_atomic()
186#define virt_store_release(p, v) __smp_store_release(p, v)
187#define virt_load_acquire(p) __smp_load_acquire(p)
188
189/**
190 * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency
191 *
192 * A control dependency provides a LOAD->STORE order, the additional RMB
193 * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order,
194 * aka. (load)-ACQUIRE.
195 *
196 * Architectures that do not do load speculation can have this be barrier().
197 */
198#ifndef smp_acquire__after_ctrl_dep
199#define smp_acquire__after_ctrl_dep() smp_rmb()
200#endif
201
202/**
203 * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees
204 * @ptr: pointer to the variable to wait on
205 * @cond: boolean expression to wait for
206 *
207 * Equivalent to using READ_ONCE() on the condition variable.
208 *
209 * Due to C lacking lambda expressions we load the value of *ptr into a
210 * pre-named variable @VAL to be used in @cond.
211 */
212#ifndef smp_cond_load_relaxed
213#define smp_cond_load_relaxed(ptr, cond_expr) ({ \
214 typeof(ptr) __PTR = (ptr); \
215 __unqual_scalar_typeof(*ptr) VAL; \
216 for (;;) { \
217 VAL = READ_ONCE(*__PTR); \
218 if (cond_expr) \
219 break; \
220 cpu_relax(); \
221 } \
222 (typeof(*ptr))VAL; \
223})
224#endif
225
226/**
227 * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering
228 * @ptr: pointer to the variable to wait on
229 * @cond: boolean expression to wait for
230 *
231 * Equivalent to using smp_load_acquire() on the condition variable but employs
232 * the control dependency of the wait to reduce the barrier on many platforms.
233 */
234#ifndef smp_cond_load_acquire
235#define smp_cond_load_acquire(ptr, cond_expr) ({ \
236 __unqual_scalar_typeof(*ptr) _val; \
237 _val = smp_cond_load_relaxed(ptr, cond_expr); \
238 smp_acquire__after_ctrl_dep(); \
239 (typeof(*ptr))_val; \
240})
241#endif
242
243/*
244 * pmem_wmb() ensures that all stores for which the modification
245 * are written to persistent storage by preceding instructions have
246 * updated persistent storage before any data access or data transfer
247 * caused by subsequent instructions is initiated.
248 */
249#ifndef pmem_wmb
250#define pmem_wmb() wmb()
251#endif
252
253#endif /* !__ASSEMBLY__ */
254#endif /* __ASM_GENERIC_BARRIER_H */