Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __ASM_ARM_CMPXCHG_H
3#define __ASM_ARM_CMPXCHG_H
4
5#include <linux/irqflags.h>
6#include <linux/prefetch.h>
7#include <asm/barrier.h>
8
9#if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110)
10/*
11 * On the StrongARM, "swp" is terminally broken since it bypasses the
12 * cache totally. This means that the cache becomes inconsistent, and,
13 * since we use normal loads/stores as well, this is really bad.
14 * Typically, this causes oopsen in filp_close, but could have other,
15 * more disastrous effects. There are two work-arounds:
16 * 1. Disable interrupts and emulate the atomic swap
17 * 2. Clean the cache, perform atomic swap, flush the cache
18 *
19 * We choose (1) since its the "easiest" to achieve here and is not
20 * dependent on the processor type.
21 *
22 * NOTE that this solution won't work on an SMP system, so explcitly
23 * forbid it here.
24 */
25#define swp_is_buggy
26#endif
27
28static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
29{
30 extern void __bad_xchg(volatile void *, int);
31 unsigned long ret;
32#ifdef swp_is_buggy
33 unsigned long flags;
34#endif
35#if __LINUX_ARM_ARCH__ >= 6
36 unsigned int tmp;
37#endif
38
39 prefetchw((const void *)ptr);
40
41 switch (size) {
42#if __LINUX_ARM_ARCH__ >= 6
43#ifndef CONFIG_CPU_V6 /* MIN ARCH >= V6K */
44 case 1:
45 asm volatile("@ __xchg1\n"
46 "1: ldrexb %0, [%3]\n"
47 " strexb %1, %2, [%3]\n"
48 " teq %1, #0\n"
49 " bne 1b"
50 : "=&r" (ret), "=&r" (tmp)
51 : "r" (x), "r" (ptr)
52 : "memory", "cc");
53 break;
54 case 2:
55 asm volatile("@ __xchg2\n"
56 "1: ldrexh %0, [%3]\n"
57 " strexh %1, %2, [%3]\n"
58 " teq %1, #0\n"
59 " bne 1b"
60 : "=&r" (ret), "=&r" (tmp)
61 : "r" (x), "r" (ptr)
62 : "memory", "cc");
63 break;
64#endif
65 case 4:
66 asm volatile("@ __xchg4\n"
67 "1: ldrex %0, [%3]\n"
68 " strex %1, %2, [%3]\n"
69 " teq %1, #0\n"
70 " bne 1b"
71 : "=&r" (ret), "=&r" (tmp)
72 : "r" (x), "r" (ptr)
73 : "memory", "cc");
74 break;
75#elif defined(swp_is_buggy)
76#ifdef CONFIG_SMP
77#error SMP is not supported on this platform
78#endif
79 case 1:
80 raw_local_irq_save(flags);
81 ret = *(volatile unsigned char *)ptr;
82 *(volatile unsigned char *)ptr = x;
83 raw_local_irq_restore(flags);
84 break;
85
86 case 4:
87 raw_local_irq_save(flags);
88 ret = *(volatile unsigned long *)ptr;
89 *(volatile unsigned long *)ptr = x;
90 raw_local_irq_restore(flags);
91 break;
92#else
93 case 1:
94 asm volatile("@ __xchg1\n"
95 " swpb %0, %1, [%2]"
96 : "=&r" (ret)
97 : "r" (x), "r" (ptr)
98 : "memory", "cc");
99 break;
100 case 4:
101 asm volatile("@ __xchg4\n"
102 " swp %0, %1, [%2]"
103 : "=&r" (ret)
104 : "r" (x), "r" (ptr)
105 : "memory", "cc");
106 break;
107#endif
108 default:
109 /* Cause a link-time error, the xchg() size is not supported */
110 __bad_xchg(ptr, size), ret = 0;
111 break;
112 }
113
114 return ret;
115}
116
117#define arch_xchg_relaxed(ptr, x) ({ \
118 (__typeof__(*(ptr)))__xchg((unsigned long)(x), (ptr), \
119 sizeof(*(ptr))); \
120})
121
122#include <asm-generic/cmpxchg-local.h>
123
124#if __LINUX_ARM_ARCH__ < 6
125/* min ARCH < ARMv6 */
126
127#ifdef CONFIG_SMP
128#error "SMP is not supported on this platform"
129#endif
130
131#define arch_xchg arch_xchg_relaxed
132
133/*
134 * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
135 * them available.
136 */
137#define arch_cmpxchg_local(ptr, o, n) ({ \
138 (__typeof(*ptr))__generic_cmpxchg_local((ptr), \
139 (unsigned long)(o), \
140 (unsigned long)(n), \
141 sizeof(*(ptr))); \
142})
143
144#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
145
146#include <asm-generic/cmpxchg.h>
147
148#else /* min ARCH >= ARMv6 */
149
150extern void __bad_cmpxchg(volatile void *ptr, int size);
151
152/*
153 * cmpxchg only support 32-bits operands on ARMv6.
154 */
155
156static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
157 unsigned long new, int size)
158{
159 unsigned long oldval, res;
160
161 prefetchw((const void *)ptr);
162
163 switch (size) {
164#ifndef CONFIG_CPU_V6 /* min ARCH >= ARMv6K */
165 case 1:
166 do {
167 asm volatile("@ __cmpxchg1\n"
168 " ldrexb %1, [%2]\n"
169 " mov %0, #0\n"
170 " teq %1, %3\n"
171 " strexbeq %0, %4, [%2]\n"
172 : "=&r" (res), "=&r" (oldval)
173 : "r" (ptr), "Ir" (old), "r" (new)
174 : "memory", "cc");
175 } while (res);
176 break;
177 case 2:
178 do {
179 asm volatile("@ __cmpxchg1\n"
180 " ldrexh %1, [%2]\n"
181 " mov %0, #0\n"
182 " teq %1, %3\n"
183 " strexheq %0, %4, [%2]\n"
184 : "=&r" (res), "=&r" (oldval)
185 : "r" (ptr), "Ir" (old), "r" (new)
186 : "memory", "cc");
187 } while (res);
188 break;
189#endif
190 case 4:
191 do {
192 asm volatile("@ __cmpxchg4\n"
193 " ldrex %1, [%2]\n"
194 " mov %0, #0\n"
195 " teq %1, %3\n"
196 " strexeq %0, %4, [%2]\n"
197 : "=&r" (res), "=&r" (oldval)
198 : "r" (ptr), "Ir" (old), "r" (new)
199 : "memory", "cc");
200 } while (res);
201 break;
202 default:
203 __bad_cmpxchg(ptr, size);
204 oldval = 0;
205 }
206
207 return oldval;
208}
209
210#define arch_cmpxchg_relaxed(ptr,o,n) ({ \
211 (__typeof__(*(ptr)))__cmpxchg((ptr), \
212 (unsigned long)(o), \
213 (unsigned long)(n), \
214 sizeof(*(ptr))); \
215})
216
217static inline unsigned long __cmpxchg_local(volatile void *ptr,
218 unsigned long old,
219 unsigned long new, int size)
220{
221 unsigned long ret;
222
223 switch (size) {
224#ifdef CONFIG_CPU_V6 /* min ARCH == ARMv6 */
225 case 1:
226 case 2:
227 ret = __generic_cmpxchg_local(ptr, old, new, size);
228 break;
229#endif
230 default:
231 ret = __cmpxchg(ptr, old, new, size);
232 }
233
234 return ret;
235}
236
237#define arch_cmpxchg_local(ptr, o, n) ({ \
238 (__typeof(*ptr))__cmpxchg_local((ptr), \
239 (unsigned long)(o), \
240 (unsigned long)(n), \
241 sizeof(*(ptr))); \
242})
243
244static inline unsigned long long __cmpxchg64(unsigned long long *ptr,
245 unsigned long long old,
246 unsigned long long new)
247{
248 unsigned long long oldval;
249 unsigned long res;
250
251 prefetchw(ptr);
252
253 __asm__ __volatile__(
254"1: ldrexd %1, %H1, [%3]\n"
255" teq %1, %4\n"
256" teqeq %H1, %H4\n"
257" bne 2f\n"
258" strexd %0, %5, %H5, [%3]\n"
259" teq %0, #0\n"
260" bne 1b\n"
261"2:"
262 : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr)
263 : "r" (ptr), "r" (old), "r" (new)
264 : "cc");
265
266 return oldval;
267}
268
269#define arch_cmpxchg64_relaxed(ptr, o, n) ({ \
270 (__typeof__(*(ptr)))__cmpxchg64((ptr), \
271 (unsigned long long)(o), \
272 (unsigned long long)(n)); \
273})
274
275#define arch_cmpxchg64_local(ptr, o, n) arch_cmpxchg64_relaxed((ptr), (o), (n))
276
277#endif /* __LINUX_ARM_ARCH__ >= 6 */
278
279#endif /* __ASM_ARM_CMPXCHG_H */
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __ASM_ARM_CMPXCHG_H
3#define __ASM_ARM_CMPXCHG_H
4
5#include <linux/irqflags.h>
6#include <linux/prefetch.h>
7#include <asm/barrier.h>
8
9#if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110)
10/*
11 * On the StrongARM, "swp" is terminally broken since it bypasses the
12 * cache totally. This means that the cache becomes inconsistent, and,
13 * since we use normal loads/stores as well, this is really bad.
14 * Typically, this causes oopsen in filp_close, but could have other,
15 * more disastrous effects. There are two work-arounds:
16 * 1. Disable interrupts and emulate the atomic swap
17 * 2. Clean the cache, perform atomic swap, flush the cache
18 *
19 * We choose (1) since its the "easiest" to achieve here and is not
20 * dependent on the processor type.
21 *
22 * NOTE that this solution won't work on an SMP system, so explcitly
23 * forbid it here.
24 */
25#define swp_is_buggy
26#endif
27
28static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
29{
30 extern void __bad_xchg(volatile void *, int);
31 unsigned long ret;
32#ifdef swp_is_buggy
33 unsigned long flags;
34#endif
35#if __LINUX_ARM_ARCH__ >= 6
36 unsigned int tmp;
37#endif
38
39 prefetchw((const void *)ptr);
40
41 switch (size) {
42#if __LINUX_ARM_ARCH__ >= 6
43#ifndef CONFIG_CPU_V6 /* MIN ARCH >= V6K */
44 case 1:
45 asm volatile("@ __xchg1\n"
46 "1: ldrexb %0, [%3]\n"
47 " strexb %1, %2, [%3]\n"
48 " teq %1, #0\n"
49 " bne 1b"
50 : "=&r" (ret), "=&r" (tmp)
51 : "r" (x), "r" (ptr)
52 : "memory", "cc");
53 break;
54 case 2:
55 asm volatile("@ __xchg2\n"
56 "1: ldrexh %0, [%3]\n"
57 " strexh %1, %2, [%3]\n"
58 " teq %1, #0\n"
59 " bne 1b"
60 : "=&r" (ret), "=&r" (tmp)
61 : "r" (x), "r" (ptr)
62 : "memory", "cc");
63 break;
64#endif
65 case 4:
66 asm volatile("@ __xchg4\n"
67 "1: ldrex %0, [%3]\n"
68 " strex %1, %2, [%3]\n"
69 " teq %1, #0\n"
70 " bne 1b"
71 : "=&r" (ret), "=&r" (tmp)
72 : "r" (x), "r" (ptr)
73 : "memory", "cc");
74 break;
75#elif defined(swp_is_buggy)
76#ifdef CONFIG_SMP
77#error SMP is not supported on this platform
78#endif
79 case 1:
80 raw_local_irq_save(flags);
81 ret = *(volatile unsigned char *)ptr;
82 *(volatile unsigned char *)ptr = x;
83 raw_local_irq_restore(flags);
84 break;
85
86 case 4:
87 raw_local_irq_save(flags);
88 ret = *(volatile unsigned long *)ptr;
89 *(volatile unsigned long *)ptr = x;
90 raw_local_irq_restore(flags);
91 break;
92#else
93 case 1:
94 asm volatile("@ __xchg1\n"
95 " swpb %0, %1, [%2]"
96 : "=&r" (ret)
97 : "r" (x), "r" (ptr)
98 : "memory", "cc");
99 break;
100 case 4:
101 asm volatile("@ __xchg4\n"
102 " swp %0, %1, [%2]"
103 : "=&r" (ret)
104 : "r" (x), "r" (ptr)
105 : "memory", "cc");
106 break;
107#endif
108 default:
109 /* Cause a link-time error, the xchg() size is not supported */
110 __bad_xchg(ptr, size), ret = 0;
111 break;
112 }
113
114 return ret;
115}
116
117#define arch_xchg_relaxed(ptr, x) ({ \
118 (__typeof__(*(ptr)))__xchg((unsigned long)(x), (ptr), \
119 sizeof(*(ptr))); \
120})
121
122#include <asm-generic/cmpxchg-local.h>
123
124#if __LINUX_ARM_ARCH__ < 6
125/* min ARCH < ARMv6 */
126
127#ifdef CONFIG_SMP
128#error "SMP is not supported on this platform"
129#endif
130
131#define arch_xchg arch_xchg_relaxed
132
133/*
134 * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
135 * them available.
136 */
137#define arch_cmpxchg_local(ptr, o, n) ({ \
138 (__typeof(*ptr))__generic_cmpxchg_local((ptr), \
139 (unsigned long)(o), \
140 (unsigned long)(n), \
141 sizeof(*(ptr))); \
142})
143
144#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
145
146#include <asm-generic/cmpxchg.h>
147
148#else /* min ARCH >= ARMv6 */
149
150extern void __bad_cmpxchg(volatile void *ptr, int size);
151
152/*
153 * cmpxchg only support 32-bits operands on ARMv6.
154 */
155
156static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
157 unsigned long new, int size)
158{
159 unsigned long oldval, res;
160
161 prefetchw((const void *)ptr);
162
163 switch (size) {
164#ifndef CONFIG_CPU_V6 /* min ARCH >= ARMv6K */
165 case 1:
166 do {
167 asm volatile("@ __cmpxchg1\n"
168 " ldrexb %1, [%2]\n"
169 " mov %0, #0\n"
170 " teq %1, %3\n"
171 " strexbeq %0, %4, [%2]\n"
172 : "=&r" (res), "=&r" (oldval)
173 : "r" (ptr), "Ir" (old), "r" (new)
174 : "memory", "cc");
175 } while (res);
176 break;
177 case 2:
178 do {
179 asm volatile("@ __cmpxchg1\n"
180 " ldrexh %1, [%2]\n"
181 " mov %0, #0\n"
182 " teq %1, %3\n"
183 " strexheq %0, %4, [%2]\n"
184 : "=&r" (res), "=&r" (oldval)
185 : "r" (ptr), "Ir" (old), "r" (new)
186 : "memory", "cc");
187 } while (res);
188 break;
189#endif
190 case 4:
191 do {
192 asm volatile("@ __cmpxchg4\n"
193 " ldrex %1, [%2]\n"
194 " mov %0, #0\n"
195 " teq %1, %3\n"
196 " strexeq %0, %4, [%2]\n"
197 : "=&r" (res), "=&r" (oldval)
198 : "r" (ptr), "Ir" (old), "r" (new)
199 : "memory", "cc");
200 } while (res);
201 break;
202 default:
203 __bad_cmpxchg(ptr, size);
204 oldval = 0;
205 }
206
207 return oldval;
208}
209
210#define arch_cmpxchg_relaxed(ptr,o,n) ({ \
211 (__typeof__(*(ptr)))__cmpxchg((ptr), \
212 (unsigned long)(o), \
213 (unsigned long)(n), \
214 sizeof(*(ptr))); \
215})
216
217static inline unsigned long __cmpxchg_local(volatile void *ptr,
218 unsigned long old,
219 unsigned long new, int size)
220{
221 unsigned long ret;
222
223 switch (size) {
224#ifdef CONFIG_CPU_V6 /* min ARCH == ARMv6 */
225 case 1:
226 case 2:
227 ret = __generic_cmpxchg_local(ptr, old, new, size);
228 break;
229#endif
230 default:
231 ret = __cmpxchg(ptr, old, new, size);
232 }
233
234 return ret;
235}
236
237#define arch_cmpxchg_local(ptr, o, n) ({ \
238 (__typeof(*ptr))__cmpxchg_local((ptr), \
239 (unsigned long)(o), \
240 (unsigned long)(n), \
241 sizeof(*(ptr))); \
242})
243
244static inline unsigned long long __cmpxchg64(unsigned long long *ptr,
245 unsigned long long old,
246 unsigned long long new)
247{
248 unsigned long long oldval;
249 unsigned long res;
250
251 prefetchw(ptr);
252
253 __asm__ __volatile__(
254"1: ldrexd %1, %H1, [%3]\n"
255" teq %1, %4\n"
256" teqeq %H1, %H4\n"
257" bne 2f\n"
258" strexd %0, %5, %H5, [%3]\n"
259" teq %0, #0\n"
260" bne 1b\n"
261"2:"
262 : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr)
263 : "r" (ptr), "r" (old), "r" (new)
264 : "cc");
265
266 return oldval;
267}
268
269#define arch_cmpxchg64_relaxed(ptr, o, n) ({ \
270 (__typeof__(*(ptr)))__cmpxchg64((ptr), \
271 (unsigned long long)(o), \
272 (unsigned long long)(n)); \
273})
274
275#define arch_cmpxchg64_local(ptr, o, n) arch_cmpxchg64_relaxed((ptr), (o), (n))
276
277#endif /* __LINUX_ARM_ARCH__ >= 6 */
278
279#endif /* __ASM_ARM_CMPXCHG_H */