1#ifndef __ASM_SPINLOCK_H
  2#define __ASM_SPINLOCK_H
  3
  4#if __LINUX_ARM_ARCH__ < 6
  5#error SMP not supported on pre-ARMv6 CPUs
  6#endif
  7
 
 
  8#include <asm/processor.h>
  9
 10/*
 11 * sev and wfe are ARMv6K extensions.  Uniprocessor ARMv6 may not have the K
 12 * extensions, so when running on UP, we have to patch these instructions away.
 13 */
 14#define ALT_SMP(smp, up)					\
 15	"9998:	" smp "\n"					\
 16	"	.pushsection \".alt.smp.init\", \"a\"\n"	\
 17	"	.long	9998b\n"				\
 18	"	" up "\n"					\
 19	"	.popsection\n"
 20
 21#ifdef CONFIG_THUMB2_KERNEL
 22#define SEV		ALT_SMP("sev.w", "nop.w")
 23/*
 24 * For Thumb-2, special care is needed to ensure that the conditional WFE
 25 * instruction really does assemble to exactly 4 bytes (as required by
 26 * the SMP_ON_UP fixup code).   By itself "wfene" might cause the
 27 * assembler to insert a extra (16-bit) IT instruction, depending on the
 28 * presence or absence of neighbouring conditional instructions.
 29 *
 30 * To avoid this unpredictableness, an approprite IT is inserted explicitly:
 31 * the assembler won't change IT instructions which are explicitly present
 32 * in the input.
 33 */
 34#define WFE(cond)	ALT_SMP(		\
 35	"it " cond "\n\t"			\
 36	"wfe" cond ".n",			\
 37						\
 38	"nop.w"					\
 39)
 40#else
 41#define SEV		ALT_SMP("sev", "nop")
 42#define WFE(cond)	ALT_SMP("wfe" cond, "nop")
 43#endif
 44
 
 
 45static inline void dsb_sev(void)
 46{
 47#if __LINUX_ARM_ARCH__ >= 7
 48	__asm__ __volatile__ (
 49		"dsb\n"
 50		SEV
 51	);
 52#else
 53	__asm__ __volatile__ (
 54		"mcr p15, 0, %0, c7, c10, 4\n"
 55		SEV
 56		: : "r" (0)
 57	);
 58#endif
 59}
 60
 61/*
 62 * ARMv6 Spin-locking.
 63 *
 64 * We exclusively read the old value.  If it is zero, we may have
 65 * won the lock, so we try exclusively storing it.  A memory barrier
 66 * is required after we get a lock, and before we release it, because
 67 * V6 CPUs are assumed to have weakly ordered memory.
 68 *
 69 * Unlocked value: 0
 70 * Locked value: 1
 
 71 */
 72
 73#define arch_spin_is_locked(x)		((x)->lock != 0)
 74#define arch_spin_unlock_wait(lock) \
 75	do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0)
 
 
 
 
 
 
 
 
 
 
 
 
 76
 77#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
 78
 79static inline void arch_spin_lock(arch_spinlock_t *lock)
 80{
 81	unsigned long tmp;
 
 
 82
 
 83	__asm__ __volatile__(
 84"1:	ldrex	%0, [%1]\n"
 85"	teq	%0, #0\n"
 86	WFE("ne")
 87"	strexeq	%0, %2, [%1]\n"
 88"	teqeq	%0, #0\n"
 89"	bne	1b"
 90	: "=&r" (tmp)
 91	: "r" (&lock->lock), "r" (1)
 92	: "cc");
 93
 
 
 
 
 
 94	smp_mb();
 95}
 96
 97static inline int arch_spin_trylock(arch_spinlock_t *lock)
 98{
 99	unsigned long tmp;
 
100
101	__asm__ __volatile__(
102"	ldrex	%0, [%1]\n"
103"	teq	%0, #0\n"
104"	strexeq	%0, %2, [%1]"
105	: "=&r" (tmp)
106	: "r" (&lock->lock), "r" (1)
107	: "cc");
 
 
 
 
 
108
109	if (tmp == 0) {
110		smp_mb();
111		return 1;
112	} else {
113		return 0;
114	}
115}
116
117static inline void arch_spin_unlock(arch_spinlock_t *lock)
118{
119	smp_mb();
 
 
 
120
121	__asm__ __volatile__(
122"	str	%1, [%0]\n"
123	:
124	: "r" (&lock->lock), "r" (0)
125	: "cc");
126
127	dsb_sev();
 
 
 
 
 
 
 
 
128}
 
129
130/*
131 * RWLOCKS
132 *
133 *
134 * Write locks are easy - we just set bit 31.  When unlocking, we can
135 * just write zero since the lock is exclusively held.
136 */
137
138static inline void arch_write_lock(arch_rwlock_t *rw)
139{
140	unsigned long tmp;
141
 
142	__asm__ __volatile__(
143"1:	ldrex	%0, [%1]\n"
144"	teq	%0, #0\n"
145	WFE("ne")
146"	strexeq	%0, %2, [%1]\n"
147"	teq	%0, #0\n"
148"	bne	1b"
149	: "=&r" (tmp)
150	: "r" (&rw->lock), "r" (0x80000000)
151	: "cc");
152
153	smp_mb();
154}
155
156static inline int arch_write_trylock(arch_rwlock_t *rw)
157{
158	unsigned long tmp;
159
160	__asm__ __volatile__(
161"1:	ldrex	%0, [%1]\n"
162"	teq	%0, #0\n"
163"	strexeq	%0, %2, [%1]"
164	: "=&r" (tmp)
165	: "r" (&rw->lock), "r" (0x80000000)
166	: "cc");
 
 
 
 
167
168	if (tmp == 0) {
169		smp_mb();
170		return 1;
171	} else {
172		return 0;
173	}
174}
175
176static inline void arch_write_unlock(arch_rwlock_t *rw)
177{
178	smp_mb();
179
180	__asm__ __volatile__(
181	"str	%1, [%0]\n"
182	:
183	: "r" (&rw->lock), "r" (0)
184	: "cc");
185
186	dsb_sev();
187}
188
189/* write_can_lock - would write_trylock() succeed? */
190#define arch_write_can_lock(x)		((x)->lock == 0)
191
192/*
193 * Read locks are a bit more hairy:
194 *  - Exclusively load the lock value.
195 *  - Increment it.
196 *  - Store new lock value if positive, and we still own this location.
197 *    If the value is negative, we've already failed.
198 *  - If we failed to store the value, we want a negative result.
199 *  - If we failed, try again.
200 * Unlocking is similarly hairy.  We may have multiple read locks
201 * currently active.  However, we know we won't have any write
202 * locks.
203 */
204static inline void arch_read_lock(arch_rwlock_t *rw)
205{
206	unsigned long tmp, tmp2;
207
 
208	__asm__ __volatile__(
209"1:	ldrex	%0, [%2]\n"
210"	adds	%0, %0, #1\n"
211"	strexpl	%1, %0, [%2]\n"
212	WFE("mi")
213"	rsbpls	%0, %1, #0\n"
214"	bmi	1b"
215	: "=&r" (tmp), "=&r" (tmp2)
216	: "r" (&rw->lock)
217	: "cc");
218
219	smp_mb();
220}
221
222static inline void arch_read_unlock(arch_rwlock_t *rw)
223{
224	unsigned long tmp, tmp2;
225
226	smp_mb();
227
 
228	__asm__ __volatile__(
229"1:	ldrex	%0, [%2]\n"
230"	sub	%0, %0, #1\n"
231"	strex	%1, %0, [%2]\n"
232"	teq	%1, #0\n"
233"	bne	1b"
234	: "=&r" (tmp), "=&r" (tmp2)
235	: "r" (&rw->lock)
236	: "cc");
237
238	if (tmp == 0)
239		dsb_sev();
240}
241
242static inline int arch_read_trylock(arch_rwlock_t *rw)
243{
244	unsigned long tmp, tmp2 = 1;
245
246	__asm__ __volatile__(
247"1:	ldrex	%0, [%2]\n"
248"	adds	%0, %0, #1\n"
249"	strexpl	%1, %0, [%2]\n"
250	: "=&r" (tmp), "+r" (tmp2)
251	: "r" (&rw->lock)
252	: "cc");
 
 
 
 
253
254	smp_mb();
255	return tmp2 == 0;
 
 
 
 
 
256}
257
258/* read_can_lock - would read_trylock() succeed? */
259#define arch_read_can_lock(x)		((x)->lock < 0x80000000)
260
261#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
262#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
263
264#define arch_spin_relax(lock)	cpu_relax()
265#define arch_read_relax(lock)	cpu_relax()
266#define arch_write_relax(lock)	cpu_relax()
267
268#endif /* __ASM_SPINLOCK_H */

  1#ifndef __ASM_SPINLOCK_H
  2#define __ASM_SPINLOCK_H
  3
  4#if __LINUX_ARM_ARCH__ < 6
  5#error SMP not supported on pre-ARMv6 CPUs
  6#endif
  7
  8#include <linux/prefetch.h>
  9#include <asm/barrier.h>
 10#include <asm/processor.h>
 11
 12/*
 13 * sev and wfe are ARMv6K extensions.  Uniprocessor ARMv6 may not have the K
 14 * extensions, so when running on UP, we have to patch these instructions away.
 15 */
 
 
 
 
 
 
 
 16#ifdef CONFIG_THUMB2_KERNEL
 
 17/*
 18 * For Thumb-2, special care is needed to ensure that the conditional WFE
 19 * instruction really does assemble to exactly 4 bytes (as required by
 20 * the SMP_ON_UP fixup code).   By itself "wfene" might cause the
 21 * assembler to insert a extra (16-bit) IT instruction, depending on the
 22 * presence or absence of neighbouring conditional instructions.
 23 *
 24 * To avoid this unpredictableness, an approprite IT is inserted explicitly:
 25 * the assembler won't change IT instructions which are explicitly present
 26 * in the input.
 27 */
 28#define WFE(cond)	__ALT_SMP_ASM(		\
 29	"it " cond "\n\t"			\
 30	"wfe" cond ".n",			\
 31						\
 32	"nop.w"					\
 33)
 34#else
 35#define WFE(cond)	__ALT_SMP_ASM("wfe" cond, "nop")
 
 36#endif
 37
 38#define SEV		__ALT_SMP_ASM(WASM(sev), WASM(nop))
 39
 40static inline void dsb_sev(void)
 41{
 42
 43	dsb(ishst);
 44	__asm__(SEV);
 
 
 
 
 
 
 
 
 
 45}
 46
 47/*
 48 * ARMv6 ticket-based spin-locking.
 
 
 
 
 
 49 *
 50 * A memory barrier is required after we get a lock, and before we
 51 * release it, because V6 CPUs are assumed to have weakly ordered
 52 * memory.
 53 */
 54
 55static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
 56{
 57	u16 owner = READ_ONCE(lock->tickets.owner);
 58
 59	for (;;) {
 60		arch_spinlock_t tmp = READ_ONCE(*lock);
 61
 62		if (tmp.tickets.owner == tmp.tickets.next ||
 63		    tmp.tickets.owner != owner)
 64			break;
 65
 66		wfe();
 67	}
 68	smp_acquire__after_ctrl_dep();
 69}
 70
 71#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
 72
 73static inline void arch_spin_lock(arch_spinlock_t *lock)
 74{
 75	unsigned long tmp;
 76	u32 newval;
 77	arch_spinlock_t lockval;
 78
 79	prefetchw(&lock->slock);
 80	__asm__ __volatile__(
 81"1:	ldrex	%0, [%3]\n"
 82"	add	%1, %0, %4\n"
 83"	strex	%2, %1, [%3]\n"
 84"	teq	%2, #0\n"
 
 85"	bne	1b"
 86	: "=&r" (lockval), "=&r" (newval), "=&r" (tmp)
 87	: "r" (&lock->slock), "I" (1 << TICKET_SHIFT)
 88	: "cc");
 89
 90	while (lockval.tickets.next != lockval.tickets.owner) {
 91		wfe();
 92		lockval.tickets.owner = ACCESS_ONCE(lock->tickets.owner);
 93	}
 94
 95	smp_mb();
 96}
 97
 98static inline int arch_spin_trylock(arch_spinlock_t *lock)
 99{
100	unsigned long contended, res;
101	u32 slock;
102
103	prefetchw(&lock->slock);
104	do {
105		__asm__ __volatile__(
106		"	ldrex	%0, [%3]\n"
107		"	mov	%2, #0\n"
108		"	subs	%1, %0, %0, ror #16\n"
109		"	addeq	%0, %0, %4\n"
110		"	strexeq	%2, %0, [%3]"
111		: "=&r" (slock), "=&r" (contended), "=&r" (res)
112		: "r" (&lock->slock), "I" (1 << TICKET_SHIFT)
113		: "cc");
114	} while (res);
115
116	if (!contended) {
117		smp_mb();
118		return 1;
119	} else {
120		return 0;
121	}
122}
123
124static inline void arch_spin_unlock(arch_spinlock_t *lock)
125{
126	smp_mb();
127	lock->tickets.owner++;
128	dsb_sev();
129}
130
131static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
132{
133	return lock.tickets.owner == lock.tickets.next;
134}
 
135
136static inline int arch_spin_is_locked(arch_spinlock_t *lock)
137{
138	return !arch_spin_value_unlocked(READ_ONCE(*lock));
139}
140
141static inline int arch_spin_is_contended(arch_spinlock_t *lock)
142{
143	struct __raw_tickets tickets = READ_ONCE(lock->tickets);
144	return (tickets.next - tickets.owner) > 1;
145}
146#define arch_spin_is_contended	arch_spin_is_contended
147
148/*
149 * RWLOCKS
150 *
151 *
152 * Write locks are easy - we just set bit 31.  When unlocking, we can
153 * just write zero since the lock is exclusively held.
154 */
155
156static inline void arch_write_lock(arch_rwlock_t *rw)
157{
158	unsigned long tmp;
159
160	prefetchw(&rw->lock);
161	__asm__ __volatile__(
162"1:	ldrex	%0, [%1]\n"
163"	teq	%0, #0\n"
164	WFE("ne")
165"	strexeq	%0, %2, [%1]\n"
166"	teq	%0, #0\n"
167"	bne	1b"
168	: "=&r" (tmp)
169	: "r" (&rw->lock), "r" (0x80000000)
170	: "cc");
171
172	smp_mb();
173}
174
175static inline int arch_write_trylock(arch_rwlock_t *rw)
176{
177	unsigned long contended, res;
178
179	prefetchw(&rw->lock);
180	do {
181		__asm__ __volatile__(
182		"	ldrex	%0, [%2]\n"
183		"	mov	%1, #0\n"
184		"	teq	%0, #0\n"
185		"	strexeq	%1, %3, [%2]"
186		: "=&r" (contended), "=&r" (res)
187		: "r" (&rw->lock), "r" (0x80000000)
188		: "cc");
189	} while (res);
190
191	if (!contended) {
192		smp_mb();
193		return 1;
194	} else {
195		return 0;
196	}
197}
198
199static inline void arch_write_unlock(arch_rwlock_t *rw)
200{
201	smp_mb();
202
203	__asm__ __volatile__(
204	"str	%1, [%0]\n"
205	:
206	: "r" (&rw->lock), "r" (0)
207	: "cc");
208
209	dsb_sev();
210}
211
212/* write_can_lock - would write_trylock() succeed? */
213#define arch_write_can_lock(x)		(ACCESS_ONCE((x)->lock) == 0)
214
215/*
216 * Read locks are a bit more hairy:
217 *  - Exclusively load the lock value.
218 *  - Increment it.
219 *  - Store new lock value if positive, and we still own this location.
220 *    If the value is negative, we've already failed.
221 *  - If we failed to store the value, we want a negative result.
222 *  - If we failed, try again.
223 * Unlocking is similarly hairy.  We may have multiple read locks
224 * currently active.  However, we know we won't have any write
225 * locks.
226 */
227static inline void arch_read_lock(arch_rwlock_t *rw)
228{
229	unsigned long tmp, tmp2;
230
231	prefetchw(&rw->lock);
232	__asm__ __volatile__(
233"1:	ldrex	%0, [%2]\n"
234"	adds	%0, %0, #1\n"
235"	strexpl	%1, %0, [%2]\n"
236	WFE("mi")
237"	rsbpls	%0, %1, #0\n"
238"	bmi	1b"
239	: "=&r" (tmp), "=&r" (tmp2)
240	: "r" (&rw->lock)
241	: "cc");
242
243	smp_mb();
244}
245
246static inline void arch_read_unlock(arch_rwlock_t *rw)
247{
248	unsigned long tmp, tmp2;
249
250	smp_mb();
251
252	prefetchw(&rw->lock);
253	__asm__ __volatile__(
254"1:	ldrex	%0, [%2]\n"
255"	sub	%0, %0, #1\n"
256"	strex	%1, %0, [%2]\n"
257"	teq	%1, #0\n"
258"	bne	1b"
259	: "=&r" (tmp), "=&r" (tmp2)
260	: "r" (&rw->lock)
261	: "cc");
262
263	if (tmp == 0)
264		dsb_sev();
265}
266
267static inline int arch_read_trylock(arch_rwlock_t *rw)
268{
269	unsigned long contended, res;
270
271	prefetchw(&rw->lock);
272	do {
273		__asm__ __volatile__(
274		"	ldrex	%0, [%2]\n"
275		"	mov	%1, #0\n"
276		"	adds	%0, %0, #1\n"
277		"	strexpl	%1, %0, [%2]"
278		: "=&r" (contended), "=&r" (res)
279		: "r" (&rw->lock)
280		: "cc");
281	} while (res);
282
283	/* If the lock is negative, then it is already held for write. */
284	if (contended < 0x80000000) {
285		smp_mb();
286		return 1;
287	} else {
288		return 0;
289	}
290}
291
292/* read_can_lock - would read_trylock() succeed? */
293#define arch_read_can_lock(x)		(ACCESS_ONCE((x)->lock) < 0x80000000)
294
295#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
296#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
297
298#define arch_spin_relax(lock)	cpu_relax()
299#define arch_read_relax(lock)	cpu_relax()
300#define arch_write_relax(lock)	cpu_relax()
301
302#endif /* __ASM_SPINLOCK_H */