Loading...
1#ifndef _ASM_X86_CMPXCHG_32_H
2#define _ASM_X86_CMPXCHG_32_H
3
4/*
5 * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
6 * you need to test for the feature in boot_cpu_data.
7 */
8
9/*
10 * CMPXCHG8B only writes to the target if we had the previous
11 * value in registers, otherwise it acts as a read and gives us the
12 * "new previous" value. That is why there is a loop. Preloading
13 * EDX:EAX is a performance optimization: in the common case it means
14 * we need only one locked operation.
15 *
16 * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very
17 * least an FPU save and/or %cr0.ts manipulation.
18 *
19 * cmpxchg8b must be used with the lock prefix here to allow the
20 * instruction to be executed atomically. We need to have the reader
21 * side to see the coherent 64bit value.
22 */
23static inline void set_64bit(volatile u64 *ptr, u64 value)
24{
25 u32 low = value;
26 u32 high = value >> 32;
27 u64 prev = *ptr;
28
29 asm volatile("\n1:\t"
30 LOCK_PREFIX "cmpxchg8b %0\n\t"
31 "jnz 1b"
32 : "=m" (*ptr), "+A" (prev)
33 : "b" (low), "c" (high)
34 : "memory");
35}
36
37#ifdef CONFIG_X86_CMPXCHG64
38#define cmpxchg64(ptr, o, n) \
39 ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
40 (unsigned long long)(n)))
41#define cmpxchg64_local(ptr, o, n) \
42 ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
43 (unsigned long long)(n)))
44#endif
45
46static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
47{
48 u64 prev;
49 asm volatile(LOCK_PREFIX "cmpxchg8b %1"
50 : "=A" (prev),
51 "+m" (*ptr)
52 : "b" ((u32)new),
53 "c" ((u32)(new >> 32)),
54 "0" (old)
55 : "memory");
56 return prev;
57}
58
59static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
60{
61 u64 prev;
62 asm volatile("cmpxchg8b %1"
63 : "=A" (prev),
64 "+m" (*ptr)
65 : "b" ((u32)new),
66 "c" ((u32)(new >> 32)),
67 "0" (old)
68 : "memory");
69 return prev;
70}
71
72#ifndef CONFIG_X86_CMPXCHG64
73/*
74 * Building a kernel capable running on 80386 and 80486. It may be necessary
75 * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
76 */
77
78#define cmpxchg64(ptr, o, n) \
79({ \
80 __typeof__(*(ptr)) __ret; \
81 __typeof__(*(ptr)) __old = (o); \
82 __typeof__(*(ptr)) __new = (n); \
83 alternative_io(LOCK_PREFIX_HERE \
84 "call cmpxchg8b_emu", \
85 "lock; cmpxchg8b (%%esi)" , \
86 X86_FEATURE_CX8, \
87 "=A" (__ret), \
88 "S" ((ptr)), "0" (__old), \
89 "b" ((unsigned int)__new), \
90 "c" ((unsigned int)(__new>>32)) \
91 : "memory"); \
92 __ret; })
93
94
95#define cmpxchg64_local(ptr, o, n) \
96({ \
97 __typeof__(*(ptr)) __ret; \
98 __typeof__(*(ptr)) __old = (o); \
99 __typeof__(*(ptr)) __new = (n); \
100 alternative_io("call cmpxchg8b_emu", \
101 "cmpxchg8b (%%esi)" , \
102 X86_FEATURE_CX8, \
103 "=A" (__ret), \
104 "S" ((ptr)), "0" (__old), \
105 "b" ((unsigned int)__new), \
106 "c" ((unsigned int)(__new>>32)) \
107 : "memory"); \
108 __ret; })
109
110#endif
111
112#define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX8)
113
114#endif /* _ASM_X86_CMPXCHG_32_H */
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_X86_CMPXCHG_32_H
3#define _ASM_X86_CMPXCHG_32_H
4
5/*
6 * Note: if you use set64_bit(), __cmpxchg64(), or their variants,
7 * you need to test for the feature in boot_cpu_data.
8 */
9
10#ifdef CONFIG_X86_CMPXCHG64
11#define arch_cmpxchg64(ptr, o, n) \
12 ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
13 (unsigned long long)(n)))
14#define arch_cmpxchg64_local(ptr, o, n) \
15 ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
16 (unsigned long long)(n)))
17#define arch_try_cmpxchg64(ptr, po, n) \
18 __try_cmpxchg64((ptr), (unsigned long long *)(po), \
19 (unsigned long long)(n))
20#endif
21
22static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
23{
24 u64 prev;
25 asm volatile(LOCK_PREFIX "cmpxchg8b %1"
26 : "=A" (prev),
27 "+m" (*ptr)
28 : "b" ((u32)new),
29 "c" ((u32)(new >> 32)),
30 "0" (old)
31 : "memory");
32 return prev;
33}
34
35static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
36{
37 u64 prev;
38 asm volatile("cmpxchg8b %1"
39 : "=A" (prev),
40 "+m" (*ptr)
41 : "b" ((u32)new),
42 "c" ((u32)(new >> 32)),
43 "0" (old)
44 : "memory");
45 return prev;
46}
47
48static inline bool __try_cmpxchg64(volatile u64 *ptr, u64 *pold, u64 new)
49{
50 bool success;
51 u64 old = *pold;
52 asm volatile(LOCK_PREFIX "cmpxchg8b %[ptr]"
53 CC_SET(z)
54 : CC_OUT(z) (success),
55 [ptr] "+m" (*ptr),
56 "+A" (old)
57 : "b" ((u32)new),
58 "c" ((u32)(new >> 32))
59 : "memory");
60
61 if (unlikely(!success))
62 *pold = old;
63 return success;
64}
65
66#ifndef CONFIG_X86_CMPXCHG64
67/*
68 * Building a kernel capable running on 80386 and 80486. It may be necessary
69 * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
70 */
71
72#define arch_cmpxchg64(ptr, o, n) \
73({ \
74 __typeof__(*(ptr)) __ret; \
75 __typeof__(*(ptr)) __old = (o); \
76 __typeof__(*(ptr)) __new = (n); \
77 alternative_io(LOCK_PREFIX_HERE \
78 "call cmpxchg8b_emu", \
79 "lock; cmpxchg8b (%%esi)" , \
80 X86_FEATURE_CX8, \
81 "=A" (__ret), \
82 "S" ((ptr)), "0" (__old), \
83 "b" ((unsigned int)__new), \
84 "c" ((unsigned int)(__new>>32)) \
85 : "memory"); \
86 __ret; })
87
88
89#define arch_cmpxchg64_local(ptr, o, n) \
90({ \
91 __typeof__(*(ptr)) __ret; \
92 __typeof__(*(ptr)) __old = (o); \
93 __typeof__(*(ptr)) __new = (n); \
94 alternative_io("call cmpxchg8b_emu", \
95 "cmpxchg8b (%%esi)" , \
96 X86_FEATURE_CX8, \
97 "=A" (__ret), \
98 "S" ((ptr)), "0" (__old), \
99 "b" ((unsigned int)__new), \
100 "c" ((unsigned int)(__new>>32)) \
101 : "memory"); \
102 __ret; })
103
104#endif
105
106#define system_has_cmpxchg64() boot_cpu_has(X86_FEATURE_CX8)
107
108#endif /* _ASM_X86_CMPXCHG_32_H */