Loading...
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_POWERPC_CMPXCHG_H_
3#define _ASM_POWERPC_CMPXCHG_H_
4
5#ifdef __KERNEL__
6#include <linux/compiler.h>
7#include <asm/synch.h>
8#include <linux/bug.h>
9
10#ifdef __BIG_ENDIAN
11#define BITOFF_CAL(size, off) ((sizeof(u32) - size - off) * BITS_PER_BYTE)
12#else
13#define BITOFF_CAL(size, off) (off * BITS_PER_BYTE)
14#endif
15
16#define XCHG_GEN(type, sfx, cl) \
17static inline u32 __xchg_##type##sfx(volatile void *p, u32 val) \
18{ \
19 unsigned int prev, prev_mask, tmp, bitoff, off; \
20 \
21 off = (unsigned long)p % sizeof(u32); \
22 bitoff = BITOFF_CAL(sizeof(type), off); \
23 p -= off; \
24 val <<= bitoff; \
25 prev_mask = (u32)(type)-1 << bitoff; \
26 \
27 __asm__ __volatile__( \
28"1: lwarx %0,0,%3\n" \
29" andc %1,%0,%5\n" \
30" or %1,%1,%4\n" \
31" stwcx. %1,0,%3\n" \
32" bne- 1b\n" \
33 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \
34 : "r" (p), "r" (val), "r" (prev_mask) \
35 : "cc", cl); \
36 \
37 return prev >> bitoff; \
38}
39
40#define CMPXCHG_GEN(type, sfx, br, br2, cl) \
41static inline \
42u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new) \
43{ \
44 unsigned int prev, prev_mask, tmp, bitoff, off; \
45 \
46 off = (unsigned long)p % sizeof(u32); \
47 bitoff = BITOFF_CAL(sizeof(type), off); \
48 p -= off; \
49 old <<= bitoff; \
50 new <<= bitoff; \
51 prev_mask = (u32)(type)-1 << bitoff; \
52 \
53 __asm__ __volatile__( \
54 br \
55"1: lwarx %0,0,%3\n" \
56" and %1,%0,%6\n" \
57" cmpw 0,%1,%4\n" \
58" bne- 2f\n" \
59" andc %1,%0,%6\n" \
60" or %1,%1,%5\n" \
61" stwcx. %1,0,%3\n" \
62" bne- 1b\n" \
63 br2 \
64 "\n" \
65"2:" \
66 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \
67 : "r" (p), "r" (old), "r" (new), "r" (prev_mask) \
68 : "cc", cl); \
69 \
70 return prev >> bitoff; \
71}
72
73/*
74 * Atomic exchange
75 *
76 * Changes the memory location '*p' to be val and returns
77 * the previous value stored there.
78 */
79
80#ifndef CONFIG_PPC_HAS_LBARX_LHARX
81XCHG_GEN(u8, _local, "memory");
82XCHG_GEN(u8, _relaxed, "cc");
83XCHG_GEN(u16, _local, "memory");
84XCHG_GEN(u16, _relaxed, "cc");
85#else
86static __always_inline unsigned long
87__xchg_u8_local(volatile void *p, unsigned long val)
88{
89 unsigned long prev;
90
91 __asm__ __volatile__(
92"1: lbarx %0,0,%2 # __xchg_u8_local\n"
93" stbcx. %3,0,%2 \n"
94" bne- 1b"
95 : "=&r" (prev), "+m" (*(volatile unsigned char *)p)
96 : "r" (p), "r" (val)
97 : "cc", "memory");
98
99 return prev;
100}
101
102static __always_inline unsigned long
103__xchg_u8_relaxed(u8 *p, unsigned long val)
104{
105 unsigned long prev;
106
107 __asm__ __volatile__(
108"1: lbarx %0,0,%2 # __xchg_u8_relaxed\n"
109" stbcx. %3,0,%2\n"
110" bne- 1b"
111 : "=&r" (prev), "+m" (*p)
112 : "r" (p), "r" (val)
113 : "cc");
114
115 return prev;
116}
117
118static __always_inline unsigned long
119__xchg_u16_local(volatile void *p, unsigned long val)
120{
121 unsigned long prev;
122
123 __asm__ __volatile__(
124"1: lharx %0,0,%2 # __xchg_u16_local\n"
125" sthcx. %3,0,%2\n"
126" bne- 1b"
127 : "=&r" (prev), "+m" (*(volatile unsigned short *)p)
128 : "r" (p), "r" (val)
129 : "cc", "memory");
130
131 return prev;
132}
133
134static __always_inline unsigned long
135__xchg_u16_relaxed(u16 *p, unsigned long val)
136{
137 unsigned long prev;
138
139 __asm__ __volatile__(
140"1: lharx %0,0,%2 # __xchg_u16_relaxed\n"
141" sthcx. %3,0,%2\n"
142" bne- 1b"
143 : "=&r" (prev), "+m" (*p)
144 : "r" (p), "r" (val)
145 : "cc");
146
147 return prev;
148}
149#endif
150
151static __always_inline unsigned long
152__xchg_u32_local(volatile void *p, unsigned long val)
153{
154 unsigned long prev;
155
156 __asm__ __volatile__(
157"1: lwarx %0,0,%2 \n"
158" stwcx. %3,0,%2 \n\
159 bne- 1b"
160 : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
161 : "r" (p), "r" (val)
162 : "cc", "memory");
163
164 return prev;
165}
166
167static __always_inline unsigned long
168__xchg_u32_relaxed(u32 *p, unsigned long val)
169{
170 unsigned long prev;
171
172 __asm__ __volatile__(
173"1: lwarx %0,0,%2\n"
174" stwcx. %3,0,%2\n"
175" bne- 1b"
176 : "=&r" (prev), "+m" (*p)
177 : "r" (p), "r" (val)
178 : "cc");
179
180 return prev;
181}
182
183#ifdef CONFIG_PPC64
184static __always_inline unsigned long
185__xchg_u64_local(volatile void *p, unsigned long val)
186{
187 unsigned long prev;
188
189 __asm__ __volatile__(
190"1: ldarx %0,0,%2 \n"
191" stdcx. %3,0,%2 \n\
192 bne- 1b"
193 : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
194 : "r" (p), "r" (val)
195 : "cc", "memory");
196
197 return prev;
198}
199
200static __always_inline unsigned long
201__xchg_u64_relaxed(u64 *p, unsigned long val)
202{
203 unsigned long prev;
204
205 __asm__ __volatile__(
206"1: ldarx %0,0,%2\n"
207" stdcx. %3,0,%2\n"
208" bne- 1b"
209 : "=&r" (prev), "+m" (*p)
210 : "r" (p), "r" (val)
211 : "cc");
212
213 return prev;
214}
215#endif
216
217static __always_inline unsigned long
218__xchg_local(void *ptr, unsigned long x, unsigned int size)
219{
220 switch (size) {
221 case 1:
222 return __xchg_u8_local(ptr, x);
223 case 2:
224 return __xchg_u16_local(ptr, x);
225 case 4:
226 return __xchg_u32_local(ptr, x);
227#ifdef CONFIG_PPC64
228 case 8:
229 return __xchg_u64_local(ptr, x);
230#endif
231 }
232 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg");
233 return x;
234}
235
236static __always_inline unsigned long
237__xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
238{
239 switch (size) {
240 case 1:
241 return __xchg_u8_relaxed(ptr, x);
242 case 2:
243 return __xchg_u16_relaxed(ptr, x);
244 case 4:
245 return __xchg_u32_relaxed(ptr, x);
246#ifdef CONFIG_PPC64
247 case 8:
248 return __xchg_u64_relaxed(ptr, x);
249#endif
250 }
251 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
252 return x;
253}
254#define arch_xchg_local(ptr,x) \
255 ({ \
256 __typeof__(*(ptr)) _x_ = (x); \
257 (__typeof__(*(ptr))) __xchg_local((ptr), \
258 (unsigned long)_x_, sizeof(*(ptr))); \
259 })
260
261#define arch_xchg_relaxed(ptr, x) \
262({ \
263 __typeof__(*(ptr)) _x_ = (x); \
264 (__typeof__(*(ptr))) __xchg_relaxed((ptr), \
265 (unsigned long)_x_, sizeof(*(ptr))); \
266})
267
268/*
269 * Compare and exchange - if *p == old, set it to new,
270 * and return the old value of *p.
271 */
272#ifndef CONFIG_PPC_HAS_LBARX_LHARX
273CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
274CMPXCHG_GEN(u8, _local, , , "memory");
275CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
276CMPXCHG_GEN(u8, _relaxed, , , "cc");
277CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
278CMPXCHG_GEN(u16, _local, , , "memory");
279CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
280CMPXCHG_GEN(u16, _relaxed, , , "cc");
281#else
282static __always_inline unsigned long
283__cmpxchg_u8(volatile unsigned char *p, unsigned long old, unsigned long new)
284{
285 unsigned int prev;
286
287 __asm__ __volatile__ (
288 PPC_ATOMIC_ENTRY_BARRIER
289"1: lbarx %0,0,%2 # __cmpxchg_u8\n"
290" cmpw 0,%0,%3\n"
291" bne- 2f\n"
292" stbcx. %4,0,%2\n"
293" bne- 1b"
294 PPC_ATOMIC_EXIT_BARRIER
295 "\n\
2962:"
297 : "=&r" (prev), "+m" (*p)
298 : "r" (p), "r" (old), "r" (new)
299 : "cc", "memory");
300
301 return prev;
302}
303
304static __always_inline unsigned long
305__cmpxchg_u8_local(volatile unsigned char *p, unsigned long old,
306 unsigned long new)
307{
308 unsigned int prev;
309
310 __asm__ __volatile__ (
311"1: lbarx %0,0,%2 # __cmpxchg_u8_local\n"
312" cmpw 0,%0,%3\n"
313" bne- 2f\n"
314" stbcx. %4,0,%2\n"
315" bne- 1b\n"
316"2:"
317 : "=&r" (prev), "+m" (*p)
318 : "r" (p), "r" (old), "r" (new)
319 : "cc", "memory");
320
321 return prev;
322}
323
324static __always_inline unsigned long
325__cmpxchg_u8_relaxed(u8 *p, unsigned long old, unsigned long new)
326{
327 unsigned long prev;
328
329 __asm__ __volatile__ (
330"1: lbarx %0,0,%2 # __cmpxchg_u8_relaxed\n"
331" cmpw 0,%0,%3\n"
332" bne- 2f\n"
333" stbcx. %4,0,%2\n"
334" bne- 1b\n"
335"2:"
336 : "=&r" (prev), "+m" (*p)
337 : "r" (p), "r" (old), "r" (new)
338 : "cc");
339
340 return prev;
341}
342
343static __always_inline unsigned long
344__cmpxchg_u8_acquire(u8 *p, unsigned long old, unsigned long new)
345{
346 unsigned long prev;
347
348 __asm__ __volatile__ (
349"1: lbarx %0,0,%2 # __cmpxchg_u8_acquire\n"
350" cmpw 0,%0,%3\n"
351" bne- 2f\n"
352" stbcx. %4,0,%2\n"
353" bne- 1b\n"
354 PPC_ACQUIRE_BARRIER
355"2:"
356 : "=&r" (prev), "+m" (*p)
357 : "r" (p), "r" (old), "r" (new)
358 : "cc", "memory");
359
360 return prev;
361}
362
363static __always_inline unsigned long
364__cmpxchg_u16(volatile unsigned short *p, unsigned long old, unsigned long new)
365{
366 unsigned int prev;
367
368 __asm__ __volatile__ (
369 PPC_ATOMIC_ENTRY_BARRIER
370"1: lharx %0,0,%2 # __cmpxchg_u16\n"
371" cmpw 0,%0,%3\n"
372" bne- 2f\n"
373" sthcx. %4,0,%2\n"
374" bne- 1b\n"
375 PPC_ATOMIC_EXIT_BARRIER
376"2:"
377 : "=&r" (prev), "+m" (*p)
378 : "r" (p), "r" (old), "r" (new)
379 : "cc", "memory");
380
381 return prev;
382}
383
384static __always_inline unsigned long
385__cmpxchg_u16_local(volatile unsigned short *p, unsigned long old,
386 unsigned long new)
387{
388 unsigned int prev;
389
390 __asm__ __volatile__ (
391"1: lharx %0,0,%2 # __cmpxchg_u16_local\n"
392" cmpw 0,%0,%3\n"
393" bne- 2f\n"
394" sthcx. %4,0,%2\n"
395" bne- 1b"
396"2:"
397 : "=&r" (prev), "+m" (*p)
398 : "r" (p), "r" (old), "r" (new)
399 : "cc", "memory");
400
401 return prev;
402}
403
404static __always_inline unsigned long
405__cmpxchg_u16_relaxed(u16 *p, unsigned long old, unsigned long new)
406{
407 unsigned long prev;
408
409 __asm__ __volatile__ (
410"1: lharx %0,0,%2 # __cmpxchg_u16_relaxed\n"
411" cmpw 0,%0,%3\n"
412" bne- 2f\n"
413" sthcx. %4,0,%2\n"
414" bne- 1b\n"
415"2:"
416 : "=&r" (prev), "+m" (*p)
417 : "r" (p), "r" (old), "r" (new)
418 : "cc");
419
420 return prev;
421}
422
423static __always_inline unsigned long
424__cmpxchg_u16_acquire(u16 *p, unsigned long old, unsigned long new)
425{
426 unsigned long prev;
427
428 __asm__ __volatile__ (
429"1: lharx %0,0,%2 # __cmpxchg_u16_acquire\n"
430" cmpw 0,%0,%3\n"
431" bne- 2f\n"
432" sthcx. %4,0,%2\n"
433" bne- 1b\n"
434 PPC_ACQUIRE_BARRIER
435"2:"
436 : "=&r" (prev), "+m" (*p)
437 : "r" (p), "r" (old), "r" (new)
438 : "cc", "memory");
439
440 return prev;
441}
442#endif
443
444static __always_inline unsigned long
445__cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
446{
447 unsigned int prev;
448
449 __asm__ __volatile__ (
450 PPC_ATOMIC_ENTRY_BARRIER
451"1: lwarx %0,0,%2 # __cmpxchg_u32\n\
452 cmpw 0,%0,%3\n\
453 bne- 2f\n"
454" stwcx. %4,0,%2\n\
455 bne- 1b"
456 PPC_ATOMIC_EXIT_BARRIER
457 "\n\
4582:"
459 : "=&r" (prev), "+m" (*p)
460 : "r" (p), "r" (old), "r" (new)
461 : "cc", "memory");
462
463 return prev;
464}
465
466static __always_inline unsigned long
467__cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
468 unsigned long new)
469{
470 unsigned int prev;
471
472 __asm__ __volatile__ (
473"1: lwarx %0,0,%2 # __cmpxchg_u32\n\
474 cmpw 0,%0,%3\n\
475 bne- 2f\n"
476" stwcx. %4,0,%2\n\
477 bne- 1b"
478 "\n\
4792:"
480 : "=&r" (prev), "+m" (*p)
481 : "r" (p), "r" (old), "r" (new)
482 : "cc", "memory");
483
484 return prev;
485}
486
487static __always_inline unsigned long
488__cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new)
489{
490 unsigned long prev;
491
492 __asm__ __volatile__ (
493"1: lwarx %0,0,%2 # __cmpxchg_u32_relaxed\n"
494" cmpw 0,%0,%3\n"
495" bne- 2f\n"
496" stwcx. %4,0,%2\n"
497" bne- 1b\n"
498"2:"
499 : "=&r" (prev), "+m" (*p)
500 : "r" (p), "r" (old), "r" (new)
501 : "cc");
502
503 return prev;
504}
505
506/*
507 * cmpxchg family don't have order guarantee if cmp part fails, therefore we
508 * can avoid superfluous barriers if we use assembly code to implement
509 * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for
510 * cmpxchg_release() because that will result in putting a barrier in the
511 * middle of a ll/sc loop, which is probably a bad idea. For example, this
512 * might cause the conditional store more likely to fail.
513 */
514static __always_inline unsigned long
515__cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new)
516{
517 unsigned long prev;
518
519 __asm__ __volatile__ (
520"1: lwarx %0,0,%2 # __cmpxchg_u32_acquire\n"
521" cmpw 0,%0,%3\n"
522" bne- 2f\n"
523" stwcx. %4,0,%2\n"
524" bne- 1b\n"
525 PPC_ACQUIRE_BARRIER
526 "\n"
527"2:"
528 : "=&r" (prev), "+m" (*p)
529 : "r" (p), "r" (old), "r" (new)
530 : "cc", "memory");
531
532 return prev;
533}
534
535#ifdef CONFIG_PPC64
536static __always_inline unsigned long
537__cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
538{
539 unsigned long prev;
540
541 __asm__ __volatile__ (
542 PPC_ATOMIC_ENTRY_BARRIER
543"1: ldarx %0,0,%2 # __cmpxchg_u64\n\
544 cmpd 0,%0,%3\n\
545 bne- 2f\n\
546 stdcx. %4,0,%2\n\
547 bne- 1b"
548 PPC_ATOMIC_EXIT_BARRIER
549 "\n\
5502:"
551 : "=&r" (prev), "+m" (*p)
552 : "r" (p), "r" (old), "r" (new)
553 : "cc", "memory");
554
555 return prev;
556}
557
558static __always_inline unsigned long
559__cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
560 unsigned long new)
561{
562 unsigned long prev;
563
564 __asm__ __volatile__ (
565"1: ldarx %0,0,%2 # __cmpxchg_u64\n\
566 cmpd 0,%0,%3\n\
567 bne- 2f\n\
568 stdcx. %4,0,%2\n\
569 bne- 1b"
570 "\n\
5712:"
572 : "=&r" (prev), "+m" (*p)
573 : "r" (p), "r" (old), "r" (new)
574 : "cc", "memory");
575
576 return prev;
577}
578
579static __always_inline unsigned long
580__cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new)
581{
582 unsigned long prev;
583
584 __asm__ __volatile__ (
585"1: ldarx %0,0,%2 # __cmpxchg_u64_relaxed\n"
586" cmpd 0,%0,%3\n"
587" bne- 2f\n"
588" stdcx. %4,0,%2\n"
589" bne- 1b\n"
590"2:"
591 : "=&r" (prev), "+m" (*p)
592 : "r" (p), "r" (old), "r" (new)
593 : "cc");
594
595 return prev;
596}
597
598static __always_inline unsigned long
599__cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new)
600{
601 unsigned long prev;
602
603 __asm__ __volatile__ (
604"1: ldarx %0,0,%2 # __cmpxchg_u64_acquire\n"
605" cmpd 0,%0,%3\n"
606" bne- 2f\n"
607" stdcx. %4,0,%2\n"
608" bne- 1b\n"
609 PPC_ACQUIRE_BARRIER
610 "\n"
611"2:"
612 : "=&r" (prev), "+m" (*p)
613 : "r" (p), "r" (old), "r" (new)
614 : "cc", "memory");
615
616 return prev;
617}
618#endif
619
620static __always_inline unsigned long
621__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
622 unsigned int size)
623{
624 switch (size) {
625 case 1:
626 return __cmpxchg_u8(ptr, old, new);
627 case 2:
628 return __cmpxchg_u16(ptr, old, new);
629 case 4:
630 return __cmpxchg_u32(ptr, old, new);
631#ifdef CONFIG_PPC64
632 case 8:
633 return __cmpxchg_u64(ptr, old, new);
634#endif
635 }
636 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg");
637 return old;
638}
639
640static __always_inline unsigned long
641__cmpxchg_local(void *ptr, unsigned long old, unsigned long new,
642 unsigned int size)
643{
644 switch (size) {
645 case 1:
646 return __cmpxchg_u8_local(ptr, old, new);
647 case 2:
648 return __cmpxchg_u16_local(ptr, old, new);
649 case 4:
650 return __cmpxchg_u32_local(ptr, old, new);
651#ifdef CONFIG_PPC64
652 case 8:
653 return __cmpxchg_u64_local(ptr, old, new);
654#endif
655 }
656 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local");
657 return old;
658}
659
660static __always_inline unsigned long
661__cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new,
662 unsigned int size)
663{
664 switch (size) {
665 case 1:
666 return __cmpxchg_u8_relaxed(ptr, old, new);
667 case 2:
668 return __cmpxchg_u16_relaxed(ptr, old, new);
669 case 4:
670 return __cmpxchg_u32_relaxed(ptr, old, new);
671#ifdef CONFIG_PPC64
672 case 8:
673 return __cmpxchg_u64_relaxed(ptr, old, new);
674#endif
675 }
676 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed");
677 return old;
678}
679
680static __always_inline unsigned long
681__cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new,
682 unsigned int size)
683{
684 switch (size) {
685 case 1:
686 return __cmpxchg_u8_acquire(ptr, old, new);
687 case 2:
688 return __cmpxchg_u16_acquire(ptr, old, new);
689 case 4:
690 return __cmpxchg_u32_acquire(ptr, old, new);
691#ifdef CONFIG_PPC64
692 case 8:
693 return __cmpxchg_u64_acquire(ptr, old, new);
694#endif
695 }
696 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire");
697 return old;
698}
699#define arch_cmpxchg(ptr, o, n) \
700 ({ \
701 __typeof__(*(ptr)) _o_ = (o); \
702 __typeof__(*(ptr)) _n_ = (n); \
703 (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \
704 (unsigned long)_n_, sizeof(*(ptr))); \
705 })
706
707
708#define arch_cmpxchg_local(ptr, o, n) \
709 ({ \
710 __typeof__(*(ptr)) _o_ = (o); \
711 __typeof__(*(ptr)) _n_ = (n); \
712 (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \
713 (unsigned long)_n_, sizeof(*(ptr))); \
714 })
715
716#define arch_cmpxchg_relaxed(ptr, o, n) \
717({ \
718 __typeof__(*(ptr)) _o_ = (o); \
719 __typeof__(*(ptr)) _n_ = (n); \
720 (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \
721 (unsigned long)_o_, (unsigned long)_n_, \
722 sizeof(*(ptr))); \
723})
724
725#define arch_cmpxchg_acquire(ptr, o, n) \
726({ \
727 __typeof__(*(ptr)) _o_ = (o); \
728 __typeof__(*(ptr)) _n_ = (n); \
729 (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \
730 (unsigned long)_o_, (unsigned long)_n_, \
731 sizeof(*(ptr))); \
732})
733#ifdef CONFIG_PPC64
734#define arch_cmpxchg64(ptr, o, n) \
735 ({ \
736 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
737 arch_cmpxchg((ptr), (o), (n)); \
738 })
739#define arch_cmpxchg64_local(ptr, o, n) \
740 ({ \
741 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
742 arch_cmpxchg_local((ptr), (o), (n)); \
743 })
744#define arch_cmpxchg64_relaxed(ptr, o, n) \
745({ \
746 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
747 arch_cmpxchg_relaxed((ptr), (o), (n)); \
748})
749#define arch_cmpxchg64_acquire(ptr, o, n) \
750({ \
751 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
752 arch_cmpxchg_acquire((ptr), (o), (n)); \
753})
754#else
755#include <asm-generic/cmpxchg-local.h>
756#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))
757#endif
758
759#endif /* __KERNEL__ */
760#endif /* _ASM_POWERPC_CMPXCHG_H_ */
1#ifndef _ASM_POWERPC_CMPXCHG_H_
2#define _ASM_POWERPC_CMPXCHG_H_
3
4#ifdef __KERNEL__
5#include <linux/compiler.h>
6#include <asm/synch.h>
7#include <asm/asm-compat.h>
8#include <linux/bug.h>
9
10#ifdef __BIG_ENDIAN
11#define BITOFF_CAL(size, off) ((sizeof(u32) - size - off) * BITS_PER_BYTE)
12#else
13#define BITOFF_CAL(size, off) (off * BITS_PER_BYTE)
14#endif
15
16#define XCHG_GEN(type, sfx, cl) \
17static inline u32 __xchg_##type##sfx(volatile void *p, u32 val) \
18{ \
19 unsigned int prev, prev_mask, tmp, bitoff, off; \
20 \
21 off = (unsigned long)p % sizeof(u32); \
22 bitoff = BITOFF_CAL(sizeof(type), off); \
23 p -= off; \
24 val <<= bitoff; \
25 prev_mask = (u32)(type)-1 << bitoff; \
26 \
27 __asm__ __volatile__( \
28"1: lwarx %0,0,%3\n" \
29" andc %1,%0,%5\n" \
30" or %1,%1,%4\n" \
31 PPC405_ERR77(0,%3) \
32" stwcx. %1,0,%3\n" \
33" bne- 1b\n" \
34 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \
35 : "r" (p), "r" (val), "r" (prev_mask) \
36 : "cc", cl); \
37 \
38 return prev >> bitoff; \
39}
40
41#define CMPXCHG_GEN(type, sfx, br, br2, cl) \
42static inline \
43u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new) \
44{ \
45 unsigned int prev, prev_mask, tmp, bitoff, off; \
46 \
47 off = (unsigned long)p % sizeof(u32); \
48 bitoff = BITOFF_CAL(sizeof(type), off); \
49 p -= off; \
50 old <<= bitoff; \
51 new <<= bitoff; \
52 prev_mask = (u32)(type)-1 << bitoff; \
53 \
54 __asm__ __volatile__( \
55 br \
56"1: lwarx %0,0,%3\n" \
57" and %1,%0,%6\n" \
58" cmpw 0,%1,%4\n" \
59" bne- 2f\n" \
60" andc %1,%0,%6\n" \
61" or %1,%1,%5\n" \
62 PPC405_ERR77(0,%3) \
63" stwcx. %1,0,%3\n" \
64" bne- 1b\n" \
65 br2 \
66 "\n" \
67"2:" \
68 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \
69 : "r" (p), "r" (old), "r" (new), "r" (prev_mask) \
70 : "cc", cl); \
71 \
72 return prev >> bitoff; \
73}
74
75/*
76 * Atomic exchange
77 *
78 * Changes the memory location '*p' to be val and returns
79 * the previous value stored there.
80 */
81
82XCHG_GEN(u8, _local, "memory");
83XCHG_GEN(u8, _relaxed, "cc");
84XCHG_GEN(u16, _local, "memory");
85XCHG_GEN(u16, _relaxed, "cc");
86
87static __always_inline unsigned long
88__xchg_u32_local(volatile void *p, unsigned long val)
89{
90 unsigned long prev;
91
92 __asm__ __volatile__(
93"1: lwarx %0,0,%2 \n"
94 PPC405_ERR77(0,%2)
95" stwcx. %3,0,%2 \n\
96 bne- 1b"
97 : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
98 : "r" (p), "r" (val)
99 : "cc", "memory");
100
101 return prev;
102}
103
104static __always_inline unsigned long
105__xchg_u32_relaxed(u32 *p, unsigned long val)
106{
107 unsigned long prev;
108
109 __asm__ __volatile__(
110"1: lwarx %0,0,%2\n"
111 PPC405_ERR77(0, %2)
112" stwcx. %3,0,%2\n"
113" bne- 1b"
114 : "=&r" (prev), "+m" (*p)
115 : "r" (p), "r" (val)
116 : "cc");
117
118 return prev;
119}
120
121#ifdef CONFIG_PPC64
122static __always_inline unsigned long
123__xchg_u64_local(volatile void *p, unsigned long val)
124{
125 unsigned long prev;
126
127 __asm__ __volatile__(
128"1: ldarx %0,0,%2 \n"
129 PPC405_ERR77(0,%2)
130" stdcx. %3,0,%2 \n\
131 bne- 1b"
132 : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
133 : "r" (p), "r" (val)
134 : "cc", "memory");
135
136 return prev;
137}
138
139static __always_inline unsigned long
140__xchg_u64_relaxed(u64 *p, unsigned long val)
141{
142 unsigned long prev;
143
144 __asm__ __volatile__(
145"1: ldarx %0,0,%2\n"
146 PPC405_ERR77(0, %2)
147" stdcx. %3,0,%2\n"
148" bne- 1b"
149 : "=&r" (prev), "+m" (*p)
150 : "r" (p), "r" (val)
151 : "cc");
152
153 return prev;
154}
155#endif
156
157static __always_inline unsigned long
158__xchg_local(void *ptr, unsigned long x, unsigned int size)
159{
160 switch (size) {
161 case 1:
162 return __xchg_u8_local(ptr, x);
163 case 2:
164 return __xchg_u16_local(ptr, x);
165 case 4:
166 return __xchg_u32_local(ptr, x);
167#ifdef CONFIG_PPC64
168 case 8:
169 return __xchg_u64_local(ptr, x);
170#endif
171 }
172 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg");
173 return x;
174}
175
176static __always_inline unsigned long
177__xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
178{
179 switch (size) {
180 case 1:
181 return __xchg_u8_relaxed(ptr, x);
182 case 2:
183 return __xchg_u16_relaxed(ptr, x);
184 case 4:
185 return __xchg_u32_relaxed(ptr, x);
186#ifdef CONFIG_PPC64
187 case 8:
188 return __xchg_u64_relaxed(ptr, x);
189#endif
190 }
191 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
192 return x;
193}
194#define xchg_local(ptr,x) \
195 ({ \
196 __typeof__(*(ptr)) _x_ = (x); \
197 (__typeof__(*(ptr))) __xchg_local((ptr), \
198 (unsigned long)_x_, sizeof(*(ptr))); \
199 })
200
201#define xchg_relaxed(ptr, x) \
202({ \
203 __typeof__(*(ptr)) _x_ = (x); \
204 (__typeof__(*(ptr))) __xchg_relaxed((ptr), \
205 (unsigned long)_x_, sizeof(*(ptr))); \
206})
207/*
208 * Compare and exchange - if *p == old, set it to new,
209 * and return the old value of *p.
210 */
211
212CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
213CMPXCHG_GEN(u8, _local, , , "memory");
214CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
215CMPXCHG_GEN(u8, _relaxed, , , "cc");
216CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
217CMPXCHG_GEN(u16, _local, , , "memory");
218CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
219CMPXCHG_GEN(u16, _relaxed, , , "cc");
220
221static __always_inline unsigned long
222__cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
223{
224 unsigned int prev;
225
226 __asm__ __volatile__ (
227 PPC_ATOMIC_ENTRY_BARRIER
228"1: lwarx %0,0,%2 # __cmpxchg_u32\n\
229 cmpw 0,%0,%3\n\
230 bne- 2f\n"
231 PPC405_ERR77(0,%2)
232" stwcx. %4,0,%2\n\
233 bne- 1b"
234 PPC_ATOMIC_EXIT_BARRIER
235 "\n\
2362:"
237 : "=&r" (prev), "+m" (*p)
238 : "r" (p), "r" (old), "r" (new)
239 : "cc", "memory");
240
241 return prev;
242}
243
244static __always_inline unsigned long
245__cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
246 unsigned long new)
247{
248 unsigned int prev;
249
250 __asm__ __volatile__ (
251"1: lwarx %0,0,%2 # __cmpxchg_u32\n\
252 cmpw 0,%0,%3\n\
253 bne- 2f\n"
254 PPC405_ERR77(0,%2)
255" stwcx. %4,0,%2\n\
256 bne- 1b"
257 "\n\
2582:"
259 : "=&r" (prev), "+m" (*p)
260 : "r" (p), "r" (old), "r" (new)
261 : "cc", "memory");
262
263 return prev;
264}
265
266static __always_inline unsigned long
267__cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new)
268{
269 unsigned long prev;
270
271 __asm__ __volatile__ (
272"1: lwarx %0,0,%2 # __cmpxchg_u32_relaxed\n"
273" cmpw 0,%0,%3\n"
274" bne- 2f\n"
275 PPC405_ERR77(0, %2)
276" stwcx. %4,0,%2\n"
277" bne- 1b\n"
278"2:"
279 : "=&r" (prev), "+m" (*p)
280 : "r" (p), "r" (old), "r" (new)
281 : "cc");
282
283 return prev;
284}
285
286/*
287 * cmpxchg family don't have order guarantee if cmp part fails, therefore we
288 * can avoid superfluous barriers if we use assembly code to implement
289 * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for
290 * cmpxchg_release() because that will result in putting a barrier in the
291 * middle of a ll/sc loop, which is probably a bad idea. For example, this
292 * might cause the conditional store more likely to fail.
293 */
294static __always_inline unsigned long
295__cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new)
296{
297 unsigned long prev;
298
299 __asm__ __volatile__ (
300"1: lwarx %0,0,%2 # __cmpxchg_u32_acquire\n"
301" cmpw 0,%0,%3\n"
302" bne- 2f\n"
303 PPC405_ERR77(0, %2)
304" stwcx. %4,0,%2\n"
305" bne- 1b\n"
306 PPC_ACQUIRE_BARRIER
307 "\n"
308"2:"
309 : "=&r" (prev), "+m" (*p)
310 : "r" (p), "r" (old), "r" (new)
311 : "cc", "memory");
312
313 return prev;
314}
315
316#ifdef CONFIG_PPC64
317static __always_inline unsigned long
318__cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
319{
320 unsigned long prev;
321
322 __asm__ __volatile__ (
323 PPC_ATOMIC_ENTRY_BARRIER
324"1: ldarx %0,0,%2 # __cmpxchg_u64\n\
325 cmpd 0,%0,%3\n\
326 bne- 2f\n\
327 stdcx. %4,0,%2\n\
328 bne- 1b"
329 PPC_ATOMIC_EXIT_BARRIER
330 "\n\
3312:"
332 : "=&r" (prev), "+m" (*p)
333 : "r" (p), "r" (old), "r" (new)
334 : "cc", "memory");
335
336 return prev;
337}
338
339static __always_inline unsigned long
340__cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
341 unsigned long new)
342{
343 unsigned long prev;
344
345 __asm__ __volatile__ (
346"1: ldarx %0,0,%2 # __cmpxchg_u64\n\
347 cmpd 0,%0,%3\n\
348 bne- 2f\n\
349 stdcx. %4,0,%2\n\
350 bne- 1b"
351 "\n\
3522:"
353 : "=&r" (prev), "+m" (*p)
354 : "r" (p), "r" (old), "r" (new)
355 : "cc", "memory");
356
357 return prev;
358}
359
360static __always_inline unsigned long
361__cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new)
362{
363 unsigned long prev;
364
365 __asm__ __volatile__ (
366"1: ldarx %0,0,%2 # __cmpxchg_u64_relaxed\n"
367" cmpd 0,%0,%3\n"
368" bne- 2f\n"
369" stdcx. %4,0,%2\n"
370" bne- 1b\n"
371"2:"
372 : "=&r" (prev), "+m" (*p)
373 : "r" (p), "r" (old), "r" (new)
374 : "cc");
375
376 return prev;
377}
378
379static __always_inline unsigned long
380__cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new)
381{
382 unsigned long prev;
383
384 __asm__ __volatile__ (
385"1: ldarx %0,0,%2 # __cmpxchg_u64_acquire\n"
386" cmpd 0,%0,%3\n"
387" bne- 2f\n"
388" stdcx. %4,0,%2\n"
389" bne- 1b\n"
390 PPC_ACQUIRE_BARRIER
391 "\n"
392"2:"
393 : "=&r" (prev), "+m" (*p)
394 : "r" (p), "r" (old), "r" (new)
395 : "cc", "memory");
396
397 return prev;
398}
399#endif
400
401static __always_inline unsigned long
402__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
403 unsigned int size)
404{
405 switch (size) {
406 case 1:
407 return __cmpxchg_u8(ptr, old, new);
408 case 2:
409 return __cmpxchg_u16(ptr, old, new);
410 case 4:
411 return __cmpxchg_u32(ptr, old, new);
412#ifdef CONFIG_PPC64
413 case 8:
414 return __cmpxchg_u64(ptr, old, new);
415#endif
416 }
417 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg");
418 return old;
419}
420
421static __always_inline unsigned long
422__cmpxchg_local(void *ptr, unsigned long old, unsigned long new,
423 unsigned int size)
424{
425 switch (size) {
426 case 1:
427 return __cmpxchg_u8_local(ptr, old, new);
428 case 2:
429 return __cmpxchg_u16_local(ptr, old, new);
430 case 4:
431 return __cmpxchg_u32_local(ptr, old, new);
432#ifdef CONFIG_PPC64
433 case 8:
434 return __cmpxchg_u64_local(ptr, old, new);
435#endif
436 }
437 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local");
438 return old;
439}
440
441static __always_inline unsigned long
442__cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new,
443 unsigned int size)
444{
445 switch (size) {
446 case 1:
447 return __cmpxchg_u8_relaxed(ptr, old, new);
448 case 2:
449 return __cmpxchg_u16_relaxed(ptr, old, new);
450 case 4:
451 return __cmpxchg_u32_relaxed(ptr, old, new);
452#ifdef CONFIG_PPC64
453 case 8:
454 return __cmpxchg_u64_relaxed(ptr, old, new);
455#endif
456 }
457 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed");
458 return old;
459}
460
461static __always_inline unsigned long
462__cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new,
463 unsigned int size)
464{
465 switch (size) {
466 case 1:
467 return __cmpxchg_u8_acquire(ptr, old, new);
468 case 2:
469 return __cmpxchg_u16_acquire(ptr, old, new);
470 case 4:
471 return __cmpxchg_u32_acquire(ptr, old, new);
472#ifdef CONFIG_PPC64
473 case 8:
474 return __cmpxchg_u64_acquire(ptr, old, new);
475#endif
476 }
477 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire");
478 return old;
479}
480#define cmpxchg(ptr, o, n) \
481 ({ \
482 __typeof__(*(ptr)) _o_ = (o); \
483 __typeof__(*(ptr)) _n_ = (n); \
484 (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \
485 (unsigned long)_n_, sizeof(*(ptr))); \
486 })
487
488
489#define cmpxchg_local(ptr, o, n) \
490 ({ \
491 __typeof__(*(ptr)) _o_ = (o); \
492 __typeof__(*(ptr)) _n_ = (n); \
493 (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \
494 (unsigned long)_n_, sizeof(*(ptr))); \
495 })
496
497#define cmpxchg_relaxed(ptr, o, n) \
498({ \
499 __typeof__(*(ptr)) _o_ = (o); \
500 __typeof__(*(ptr)) _n_ = (n); \
501 (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \
502 (unsigned long)_o_, (unsigned long)_n_, \
503 sizeof(*(ptr))); \
504})
505
506#define cmpxchg_acquire(ptr, o, n) \
507({ \
508 __typeof__(*(ptr)) _o_ = (o); \
509 __typeof__(*(ptr)) _n_ = (n); \
510 (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \
511 (unsigned long)_o_, (unsigned long)_n_, \
512 sizeof(*(ptr))); \
513})
514#ifdef CONFIG_PPC64
515#define cmpxchg64(ptr, o, n) \
516 ({ \
517 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
518 cmpxchg((ptr), (o), (n)); \
519 })
520#define cmpxchg64_local(ptr, o, n) \
521 ({ \
522 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
523 cmpxchg_local((ptr), (o), (n)); \
524 })
525#define cmpxchg64_relaxed(ptr, o, n) \
526({ \
527 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
528 cmpxchg_relaxed((ptr), (o), (n)); \
529})
530#define cmpxchg64_acquire(ptr, o, n) \
531({ \
532 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
533 cmpxchg_acquire((ptr), (o), (n)); \
534})
535#else
536#include <asm-generic/cmpxchg-local.h>
537#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
538#endif
539
540#endif /* __KERNEL__ */
541#endif /* _ASM_POWERPC_CMPXCHG_H_ */