Loading...
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8 */
9#ifndef _ASM_SPINLOCK_H
10#define _ASM_SPINLOCK_H
11
12#include <linux/compiler.h>
13
14#include <asm/barrier.h>
15#include <asm/processor.h>
16#include <asm/compiler.h>
17#include <asm/war.h>
18
19/*
20 * Your basic SMP spinlocks, allowing only a single CPU anywhere
21 *
22 * Simple spin lock operations. There are two variants, one clears IRQ's
23 * on the local processor, one does not.
24 *
25 * These are fair FIFO ticket locks
26 *
27 * (the type definitions are in asm/spinlock_types.h)
28 */
29
30
31/*
32 * Ticket locks are conceptually two parts, one indicating the current head of
33 * the queue, and the other indicating the current tail. The lock is acquired
34 * by atomically noting the tail and incrementing it by one (thus adding
35 * ourself to the queue and noting our position), then waiting until the head
36 * becomes equal to the the initial value of the tail.
37 */
38
39static inline int arch_spin_is_locked(arch_spinlock_t *lock)
40{
41 u32 counters = ACCESS_ONCE(lock->lock);
42
43 return ((counters >> 16) ^ counters) & 0xffff;
44}
45
46static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
47{
48 return lock.h.serving_now == lock.h.ticket;
49}
50
51#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
52
53static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
54{
55 u16 owner = READ_ONCE(lock->h.serving_now);
56 smp_rmb();
57 for (;;) {
58 arch_spinlock_t tmp = READ_ONCE(*lock);
59
60 if (tmp.h.serving_now == tmp.h.ticket ||
61 tmp.h.serving_now != owner)
62 break;
63
64 cpu_relax();
65 }
66 smp_acquire__after_ctrl_dep();
67}
68
69static inline int arch_spin_is_contended(arch_spinlock_t *lock)
70{
71 u32 counters = ACCESS_ONCE(lock->lock);
72
73 return (((counters >> 16) - counters) & 0xffff) > 1;
74}
75#define arch_spin_is_contended arch_spin_is_contended
76
77static inline void arch_spin_lock(arch_spinlock_t *lock)
78{
79 int my_ticket;
80 int tmp;
81 int inc = 0x10000;
82
83 if (R10000_LLSC_WAR) {
84 __asm__ __volatile__ (
85 " .set push # arch_spin_lock \n"
86 " .set noreorder \n"
87 " \n"
88 "1: ll %[ticket], %[ticket_ptr] \n"
89 " addu %[my_ticket], %[ticket], %[inc] \n"
90 " sc %[my_ticket], %[ticket_ptr] \n"
91 " beqzl %[my_ticket], 1b \n"
92 " nop \n"
93 " srl %[my_ticket], %[ticket], 16 \n"
94 " andi %[ticket], %[ticket], 0xffff \n"
95 " bne %[ticket], %[my_ticket], 4f \n"
96 " subu %[ticket], %[my_ticket], %[ticket] \n"
97 "2: \n"
98 " .subsection 2 \n"
99 "4: andi %[ticket], %[ticket], 0xffff \n"
100 " sll %[ticket], 5 \n"
101 " \n"
102 "6: bnez %[ticket], 6b \n"
103 " subu %[ticket], 1 \n"
104 " \n"
105 " lhu %[ticket], %[serving_now_ptr] \n"
106 " beq %[ticket], %[my_ticket], 2b \n"
107 " subu %[ticket], %[my_ticket], %[ticket] \n"
108 " b 4b \n"
109 " subu %[ticket], %[ticket], 1 \n"
110 " .previous \n"
111 " .set pop \n"
112 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
113 [serving_now_ptr] "+m" (lock->h.serving_now),
114 [ticket] "=&r" (tmp),
115 [my_ticket] "=&r" (my_ticket)
116 : [inc] "r" (inc));
117 } else {
118 __asm__ __volatile__ (
119 " .set push # arch_spin_lock \n"
120 " .set noreorder \n"
121 " \n"
122 "1: ll %[ticket], %[ticket_ptr] \n"
123 " addu %[my_ticket], %[ticket], %[inc] \n"
124 " sc %[my_ticket], %[ticket_ptr] \n"
125 " beqz %[my_ticket], 1b \n"
126 " srl %[my_ticket], %[ticket], 16 \n"
127 " andi %[ticket], %[ticket], 0xffff \n"
128 " bne %[ticket], %[my_ticket], 4f \n"
129 " subu %[ticket], %[my_ticket], %[ticket] \n"
130 "2: .insn \n"
131 " .subsection 2 \n"
132 "4: andi %[ticket], %[ticket], 0xffff \n"
133 " sll %[ticket], 5 \n"
134 " \n"
135 "6: bnez %[ticket], 6b \n"
136 " subu %[ticket], 1 \n"
137 " \n"
138 " lhu %[ticket], %[serving_now_ptr] \n"
139 " beq %[ticket], %[my_ticket], 2b \n"
140 " subu %[ticket], %[my_ticket], %[ticket] \n"
141 " b 4b \n"
142 " subu %[ticket], %[ticket], 1 \n"
143 " .previous \n"
144 " .set pop \n"
145 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
146 [serving_now_ptr] "+m" (lock->h.serving_now),
147 [ticket] "=&r" (tmp),
148 [my_ticket] "=&r" (my_ticket)
149 : [inc] "r" (inc));
150 }
151
152 smp_llsc_mb();
153}
154
155static inline void arch_spin_unlock(arch_spinlock_t *lock)
156{
157 unsigned int serving_now = lock->h.serving_now + 1;
158 wmb();
159 lock->h.serving_now = (u16)serving_now;
160 nudge_writes();
161}
162
163static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
164{
165 int tmp, tmp2, tmp3;
166 int inc = 0x10000;
167
168 if (R10000_LLSC_WAR) {
169 __asm__ __volatile__ (
170 " .set push # arch_spin_trylock \n"
171 " .set noreorder \n"
172 " \n"
173 "1: ll %[ticket], %[ticket_ptr] \n"
174 " srl %[my_ticket], %[ticket], 16 \n"
175 " andi %[now_serving], %[ticket], 0xffff \n"
176 " bne %[my_ticket], %[now_serving], 3f \n"
177 " addu %[ticket], %[ticket], %[inc] \n"
178 " sc %[ticket], %[ticket_ptr] \n"
179 " beqzl %[ticket], 1b \n"
180 " li %[ticket], 1 \n"
181 "2: \n"
182 " .subsection 2 \n"
183 "3: b 2b \n"
184 " li %[ticket], 0 \n"
185 " .previous \n"
186 " .set pop \n"
187 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
188 [ticket] "=&r" (tmp),
189 [my_ticket] "=&r" (tmp2),
190 [now_serving] "=&r" (tmp3)
191 : [inc] "r" (inc));
192 } else {
193 __asm__ __volatile__ (
194 " .set push # arch_spin_trylock \n"
195 " .set noreorder \n"
196 " \n"
197 "1: ll %[ticket], %[ticket_ptr] \n"
198 " srl %[my_ticket], %[ticket], 16 \n"
199 " andi %[now_serving], %[ticket], 0xffff \n"
200 " bne %[my_ticket], %[now_serving], 3f \n"
201 " addu %[ticket], %[ticket], %[inc] \n"
202 " sc %[ticket], %[ticket_ptr] \n"
203 " beqz %[ticket], 1b \n"
204 " li %[ticket], 1 \n"
205 "2: .insn \n"
206 " .subsection 2 \n"
207 "3: b 2b \n"
208 " li %[ticket], 0 \n"
209 " .previous \n"
210 " .set pop \n"
211 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
212 [ticket] "=&r" (tmp),
213 [my_ticket] "=&r" (tmp2),
214 [now_serving] "=&r" (tmp3)
215 : [inc] "r" (inc));
216 }
217
218 smp_llsc_mb();
219
220 return tmp;
221}
222
223/*
224 * Read-write spinlocks, allowing multiple readers but only one writer.
225 *
226 * NOTE! it is quite common to have readers in interrupts but no interrupt
227 * writers. For those circumstances we can "mix" irq-safe locks - any writer
228 * needs to get a irq-safe write-lock, but readers can get non-irqsafe
229 * read-locks.
230 */
231
232/*
233 * read_can_lock - would read_trylock() succeed?
234 * @lock: the rwlock in question.
235 */
236#define arch_read_can_lock(rw) ((rw)->lock >= 0)
237
238/*
239 * write_can_lock - would write_trylock() succeed?
240 * @lock: the rwlock in question.
241 */
242#define arch_write_can_lock(rw) (!(rw)->lock)
243
244static inline void arch_read_lock(arch_rwlock_t *rw)
245{
246 unsigned int tmp;
247
248 if (R10000_LLSC_WAR) {
249 __asm__ __volatile__(
250 " .set noreorder # arch_read_lock \n"
251 "1: ll %1, %2 \n"
252 " bltz %1, 1b \n"
253 " addu %1, 1 \n"
254 " sc %1, %0 \n"
255 " beqzl %1, 1b \n"
256 " nop \n"
257 " .set reorder \n"
258 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
259 : GCC_OFF_SMALL_ASM() (rw->lock)
260 : "memory");
261 } else {
262 do {
263 __asm__ __volatile__(
264 "1: ll %1, %2 # arch_read_lock \n"
265 " bltz %1, 1b \n"
266 " addu %1, 1 \n"
267 "2: sc %1, %0 \n"
268 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
269 : GCC_OFF_SMALL_ASM() (rw->lock)
270 : "memory");
271 } while (unlikely(!tmp));
272 }
273
274 smp_llsc_mb();
275}
276
277static inline void arch_read_unlock(arch_rwlock_t *rw)
278{
279 unsigned int tmp;
280
281 smp_mb__before_llsc();
282
283 if (R10000_LLSC_WAR) {
284 __asm__ __volatile__(
285 "1: ll %1, %2 # arch_read_unlock \n"
286 " addiu %1, -1 \n"
287 " sc %1, %0 \n"
288 " beqzl %1, 1b \n"
289 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
290 : GCC_OFF_SMALL_ASM() (rw->lock)
291 : "memory");
292 } else {
293 do {
294 __asm__ __volatile__(
295 "1: ll %1, %2 # arch_read_unlock \n"
296 " addiu %1, -1 \n"
297 " sc %1, %0 \n"
298 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
299 : GCC_OFF_SMALL_ASM() (rw->lock)
300 : "memory");
301 } while (unlikely(!tmp));
302 }
303}
304
305static inline void arch_write_lock(arch_rwlock_t *rw)
306{
307 unsigned int tmp;
308
309 if (R10000_LLSC_WAR) {
310 __asm__ __volatile__(
311 " .set noreorder # arch_write_lock \n"
312 "1: ll %1, %2 \n"
313 " bnez %1, 1b \n"
314 " lui %1, 0x8000 \n"
315 " sc %1, %0 \n"
316 " beqzl %1, 1b \n"
317 " nop \n"
318 " .set reorder \n"
319 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
320 : GCC_OFF_SMALL_ASM() (rw->lock)
321 : "memory");
322 } else {
323 do {
324 __asm__ __volatile__(
325 "1: ll %1, %2 # arch_write_lock \n"
326 " bnez %1, 1b \n"
327 " lui %1, 0x8000 \n"
328 "2: sc %1, %0 \n"
329 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
330 : GCC_OFF_SMALL_ASM() (rw->lock)
331 : "memory");
332 } while (unlikely(!tmp));
333 }
334
335 smp_llsc_mb();
336}
337
338static inline void arch_write_unlock(arch_rwlock_t *rw)
339{
340 smp_mb__before_llsc();
341
342 __asm__ __volatile__(
343 " # arch_write_unlock \n"
344 " sw $0, %0 \n"
345 : "=m" (rw->lock)
346 : "m" (rw->lock)
347 : "memory");
348}
349
350static inline int arch_read_trylock(arch_rwlock_t *rw)
351{
352 unsigned int tmp;
353 int ret;
354
355 if (R10000_LLSC_WAR) {
356 __asm__ __volatile__(
357 " .set noreorder # arch_read_trylock \n"
358 " li %2, 0 \n"
359 "1: ll %1, %3 \n"
360 " bltz %1, 2f \n"
361 " addu %1, 1 \n"
362 " sc %1, %0 \n"
363 " .set reorder \n"
364 " beqzl %1, 1b \n"
365 " nop \n"
366 __WEAK_LLSC_MB
367 " li %2, 1 \n"
368 "2: \n"
369 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
370 : GCC_OFF_SMALL_ASM() (rw->lock)
371 : "memory");
372 } else {
373 __asm__ __volatile__(
374 " .set noreorder # arch_read_trylock \n"
375 " li %2, 0 \n"
376 "1: ll %1, %3 \n"
377 " bltz %1, 2f \n"
378 " addu %1, 1 \n"
379 " sc %1, %0 \n"
380 " beqz %1, 1b \n"
381 " nop \n"
382 " .set reorder \n"
383 __WEAK_LLSC_MB
384 " li %2, 1 \n"
385 "2: .insn \n"
386 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
387 : GCC_OFF_SMALL_ASM() (rw->lock)
388 : "memory");
389 }
390
391 return ret;
392}
393
394static inline int arch_write_trylock(arch_rwlock_t *rw)
395{
396 unsigned int tmp;
397 int ret;
398
399 if (R10000_LLSC_WAR) {
400 __asm__ __volatile__(
401 " .set noreorder # arch_write_trylock \n"
402 " li %2, 0 \n"
403 "1: ll %1, %3 \n"
404 " bnez %1, 2f \n"
405 " lui %1, 0x8000 \n"
406 " sc %1, %0 \n"
407 " beqzl %1, 1b \n"
408 " nop \n"
409 __WEAK_LLSC_MB
410 " li %2, 1 \n"
411 " .set reorder \n"
412 "2: \n"
413 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
414 : GCC_OFF_SMALL_ASM() (rw->lock)
415 : "memory");
416 } else {
417 do {
418 __asm__ __volatile__(
419 " ll %1, %3 # arch_write_trylock \n"
420 " li %2, 0 \n"
421 " bnez %1, 2f \n"
422 " lui %1, 0x8000 \n"
423 " sc %1, %0 \n"
424 " li %2, 1 \n"
425 "2: .insn \n"
426 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp),
427 "=&r" (ret)
428 : GCC_OFF_SMALL_ASM() (rw->lock)
429 : "memory");
430 } while (unlikely(!tmp));
431
432 smp_llsc_mb();
433 }
434
435 return ret;
436}
437
438#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
439#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
440
441#define arch_spin_relax(lock) cpu_relax()
442#define arch_read_relax(lock) cpu_relax()
443#define arch_write_relax(lock) cpu_relax()
444
445#endif /* _ASM_SPINLOCK_H */
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8 */
9#ifndef _ASM_SPINLOCK_H
10#define _ASM_SPINLOCK_H
11
12#include <linux/compiler.h>
13
14#include <asm/barrier.h>
15#include <asm/compiler.h>
16#include <asm/war.h>
17
18/*
19 * Your basic SMP spinlocks, allowing only a single CPU anywhere
20 *
21 * Simple spin lock operations. There are two variants, one clears IRQ's
22 * on the local processor, one does not.
23 *
24 * These are fair FIFO ticket locks
25 *
26 * (the type definitions are in asm/spinlock_types.h)
27 */
28
29
30/*
31 * Ticket locks are conceptually two parts, one indicating the current head of
32 * the queue, and the other indicating the current tail. The lock is acquired
33 * by atomically noting the tail and incrementing it by one (thus adding
34 * ourself to the queue and noting our position), then waiting until the head
35 * becomes equal to the the initial value of the tail.
36 */
37
38static inline int arch_spin_is_locked(arch_spinlock_t *lock)
39{
40 u32 counters = ACCESS_ONCE(lock->lock);
41
42 return ((counters >> 16) ^ counters) & 0xffff;
43}
44
45static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
46{
47 return lock.h.serving_now == lock.h.ticket;
48}
49
50#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
51#define arch_spin_unlock_wait(x) \
52 while (arch_spin_is_locked(x)) { cpu_relax(); }
53
54static inline int arch_spin_is_contended(arch_spinlock_t *lock)
55{
56 u32 counters = ACCESS_ONCE(lock->lock);
57
58 return (((counters >> 16) - counters) & 0xffff) > 1;
59}
60#define arch_spin_is_contended arch_spin_is_contended
61
62static inline void arch_spin_lock(arch_spinlock_t *lock)
63{
64 int my_ticket;
65 int tmp;
66 int inc = 0x10000;
67
68 if (R10000_LLSC_WAR) {
69 __asm__ __volatile__ (
70 " .set push # arch_spin_lock \n"
71 " .set noreorder \n"
72 " \n"
73 "1: ll %[ticket], %[ticket_ptr] \n"
74 " addu %[my_ticket], %[ticket], %[inc] \n"
75 " sc %[my_ticket], %[ticket_ptr] \n"
76 " beqzl %[my_ticket], 1b \n"
77 " nop \n"
78 " srl %[my_ticket], %[ticket], 16 \n"
79 " andi %[ticket], %[ticket], 0xffff \n"
80 " bne %[ticket], %[my_ticket], 4f \n"
81 " subu %[ticket], %[my_ticket], %[ticket] \n"
82 "2: \n"
83 " .subsection 2 \n"
84 "4: andi %[ticket], %[ticket], 0xffff \n"
85 " sll %[ticket], 5 \n"
86 " \n"
87 "6: bnez %[ticket], 6b \n"
88 " subu %[ticket], 1 \n"
89 " \n"
90 " lhu %[ticket], %[serving_now_ptr] \n"
91 " beq %[ticket], %[my_ticket], 2b \n"
92 " subu %[ticket], %[my_ticket], %[ticket] \n"
93 " b 4b \n"
94 " subu %[ticket], %[ticket], 1 \n"
95 " .previous \n"
96 " .set pop \n"
97 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
98 [serving_now_ptr] "+m" (lock->h.serving_now),
99 [ticket] "=&r" (tmp),
100 [my_ticket] "=&r" (my_ticket)
101 : [inc] "r" (inc));
102 } else {
103 __asm__ __volatile__ (
104 " .set push # arch_spin_lock \n"
105 " .set noreorder \n"
106 " \n"
107 "1: ll %[ticket], %[ticket_ptr] \n"
108 " addu %[my_ticket], %[ticket], %[inc] \n"
109 " sc %[my_ticket], %[ticket_ptr] \n"
110 " beqz %[my_ticket], 1b \n"
111 " srl %[my_ticket], %[ticket], 16 \n"
112 " andi %[ticket], %[ticket], 0xffff \n"
113 " bne %[ticket], %[my_ticket], 4f \n"
114 " subu %[ticket], %[my_ticket], %[ticket] \n"
115 "2: \n"
116 " .subsection 2 \n"
117 "4: andi %[ticket], %[ticket], 0xffff \n"
118 " sll %[ticket], 5 \n"
119 " \n"
120 "6: bnez %[ticket], 6b \n"
121 " subu %[ticket], 1 \n"
122 " \n"
123 " lhu %[ticket], %[serving_now_ptr] \n"
124 " beq %[ticket], %[my_ticket], 2b \n"
125 " subu %[ticket], %[my_ticket], %[ticket] \n"
126 " b 4b \n"
127 " subu %[ticket], %[ticket], 1 \n"
128 " .previous \n"
129 " .set pop \n"
130 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
131 [serving_now_ptr] "+m" (lock->h.serving_now),
132 [ticket] "=&r" (tmp),
133 [my_ticket] "=&r" (my_ticket)
134 : [inc] "r" (inc));
135 }
136
137 smp_llsc_mb();
138}
139
140static inline void arch_spin_unlock(arch_spinlock_t *lock)
141{
142 unsigned int serving_now = lock->h.serving_now + 1;
143 wmb();
144 lock->h.serving_now = (u16)serving_now;
145 nudge_writes();
146}
147
148static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
149{
150 int tmp, tmp2, tmp3;
151 int inc = 0x10000;
152
153 if (R10000_LLSC_WAR) {
154 __asm__ __volatile__ (
155 " .set push # arch_spin_trylock \n"
156 " .set noreorder \n"
157 " \n"
158 "1: ll %[ticket], %[ticket_ptr] \n"
159 " srl %[my_ticket], %[ticket], 16 \n"
160 " andi %[now_serving], %[ticket], 0xffff \n"
161 " bne %[my_ticket], %[now_serving], 3f \n"
162 " addu %[ticket], %[ticket], %[inc] \n"
163 " sc %[ticket], %[ticket_ptr] \n"
164 " beqzl %[ticket], 1b \n"
165 " li %[ticket], 1 \n"
166 "2: \n"
167 " .subsection 2 \n"
168 "3: b 2b \n"
169 " li %[ticket], 0 \n"
170 " .previous \n"
171 " .set pop \n"
172 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
173 [ticket] "=&r" (tmp),
174 [my_ticket] "=&r" (tmp2),
175 [now_serving] "=&r" (tmp3)
176 : [inc] "r" (inc));
177 } else {
178 __asm__ __volatile__ (
179 " .set push # arch_spin_trylock \n"
180 " .set noreorder \n"
181 " \n"
182 "1: ll %[ticket], %[ticket_ptr] \n"
183 " srl %[my_ticket], %[ticket], 16 \n"
184 " andi %[now_serving], %[ticket], 0xffff \n"
185 " bne %[my_ticket], %[now_serving], 3f \n"
186 " addu %[ticket], %[ticket], %[inc] \n"
187 " sc %[ticket], %[ticket_ptr] \n"
188 " beqz %[ticket], 1b \n"
189 " li %[ticket], 1 \n"
190 "2: \n"
191 " .subsection 2 \n"
192 "3: b 2b \n"
193 " li %[ticket], 0 \n"
194 " .previous \n"
195 " .set pop \n"
196 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
197 [ticket] "=&r" (tmp),
198 [my_ticket] "=&r" (tmp2),
199 [now_serving] "=&r" (tmp3)
200 : [inc] "r" (inc));
201 }
202
203 smp_llsc_mb();
204
205 return tmp;
206}
207
208/*
209 * Read-write spinlocks, allowing multiple readers but only one writer.
210 *
211 * NOTE! it is quite common to have readers in interrupts but no interrupt
212 * writers. For those circumstances we can "mix" irq-safe locks - any writer
213 * needs to get a irq-safe write-lock, but readers can get non-irqsafe
214 * read-locks.
215 */
216
217/*
218 * read_can_lock - would read_trylock() succeed?
219 * @lock: the rwlock in question.
220 */
221#define arch_read_can_lock(rw) ((rw)->lock >= 0)
222
223/*
224 * write_can_lock - would write_trylock() succeed?
225 * @lock: the rwlock in question.
226 */
227#define arch_write_can_lock(rw) (!(rw)->lock)
228
229static inline void arch_read_lock(arch_rwlock_t *rw)
230{
231 unsigned int tmp;
232
233 if (R10000_LLSC_WAR) {
234 __asm__ __volatile__(
235 " .set noreorder # arch_read_lock \n"
236 "1: ll %1, %2 \n"
237 " bltz %1, 1b \n"
238 " addu %1, 1 \n"
239 " sc %1, %0 \n"
240 " beqzl %1, 1b \n"
241 " nop \n"
242 " .set reorder \n"
243 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
244 : GCC_OFF_SMALL_ASM() (rw->lock)
245 : "memory");
246 } else {
247 do {
248 __asm__ __volatile__(
249 "1: ll %1, %2 # arch_read_lock \n"
250 " bltz %1, 1b \n"
251 " addu %1, 1 \n"
252 "2: sc %1, %0 \n"
253 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
254 : GCC_OFF_SMALL_ASM() (rw->lock)
255 : "memory");
256 } while (unlikely(!tmp));
257 }
258
259 smp_llsc_mb();
260}
261
262static inline void arch_read_unlock(arch_rwlock_t *rw)
263{
264 unsigned int tmp;
265
266 smp_mb__before_llsc();
267
268 if (R10000_LLSC_WAR) {
269 __asm__ __volatile__(
270 "1: ll %1, %2 # arch_read_unlock \n"
271 " addiu %1, -1 \n"
272 " sc %1, %0 \n"
273 " beqzl %1, 1b \n"
274 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
275 : GCC_OFF_SMALL_ASM() (rw->lock)
276 : "memory");
277 } else {
278 do {
279 __asm__ __volatile__(
280 "1: ll %1, %2 # arch_read_unlock \n"
281 " addiu %1, -1 \n"
282 " sc %1, %0 \n"
283 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
284 : GCC_OFF_SMALL_ASM() (rw->lock)
285 : "memory");
286 } while (unlikely(!tmp));
287 }
288}
289
290static inline void arch_write_lock(arch_rwlock_t *rw)
291{
292 unsigned int tmp;
293
294 if (R10000_LLSC_WAR) {
295 __asm__ __volatile__(
296 " .set noreorder # arch_write_lock \n"
297 "1: ll %1, %2 \n"
298 " bnez %1, 1b \n"
299 " lui %1, 0x8000 \n"
300 " sc %1, %0 \n"
301 " beqzl %1, 1b \n"
302 " nop \n"
303 " .set reorder \n"
304 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
305 : GCC_OFF_SMALL_ASM() (rw->lock)
306 : "memory");
307 } else {
308 do {
309 __asm__ __volatile__(
310 "1: ll %1, %2 # arch_write_lock \n"
311 " bnez %1, 1b \n"
312 " lui %1, 0x8000 \n"
313 "2: sc %1, %0 \n"
314 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
315 : GCC_OFF_SMALL_ASM() (rw->lock)
316 : "memory");
317 } while (unlikely(!tmp));
318 }
319
320 smp_llsc_mb();
321}
322
323static inline void arch_write_unlock(arch_rwlock_t *rw)
324{
325 smp_mb__before_llsc();
326
327 __asm__ __volatile__(
328 " # arch_write_unlock \n"
329 " sw $0, %0 \n"
330 : "=m" (rw->lock)
331 : "m" (rw->lock)
332 : "memory");
333}
334
335static inline int arch_read_trylock(arch_rwlock_t *rw)
336{
337 unsigned int tmp;
338 int ret;
339
340 if (R10000_LLSC_WAR) {
341 __asm__ __volatile__(
342 " .set noreorder # arch_read_trylock \n"
343 " li %2, 0 \n"
344 "1: ll %1, %3 \n"
345 " bltz %1, 2f \n"
346 " addu %1, 1 \n"
347 " sc %1, %0 \n"
348 " .set reorder \n"
349 " beqzl %1, 1b \n"
350 " nop \n"
351 __WEAK_LLSC_MB
352 " li %2, 1 \n"
353 "2: \n"
354 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
355 : GCC_OFF_SMALL_ASM() (rw->lock)
356 : "memory");
357 } else {
358 __asm__ __volatile__(
359 " .set noreorder # arch_read_trylock \n"
360 " li %2, 0 \n"
361 "1: ll %1, %3 \n"
362 " bltz %1, 2f \n"
363 " addu %1, 1 \n"
364 " sc %1, %0 \n"
365 " beqz %1, 1b \n"
366 " nop \n"
367 " .set reorder \n"
368 __WEAK_LLSC_MB
369 " li %2, 1 \n"
370 "2: \n"
371 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
372 : GCC_OFF_SMALL_ASM() (rw->lock)
373 : "memory");
374 }
375
376 return ret;
377}
378
379static inline int arch_write_trylock(arch_rwlock_t *rw)
380{
381 unsigned int tmp;
382 int ret;
383
384 if (R10000_LLSC_WAR) {
385 __asm__ __volatile__(
386 " .set noreorder # arch_write_trylock \n"
387 " li %2, 0 \n"
388 "1: ll %1, %3 \n"
389 " bnez %1, 2f \n"
390 " lui %1, 0x8000 \n"
391 " sc %1, %0 \n"
392 " beqzl %1, 1b \n"
393 " nop \n"
394 __WEAK_LLSC_MB
395 " li %2, 1 \n"
396 " .set reorder \n"
397 "2: \n"
398 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
399 : GCC_OFF_SMALL_ASM() (rw->lock)
400 : "memory");
401 } else {
402 do {
403 __asm__ __volatile__(
404 " ll %1, %3 # arch_write_trylock \n"
405 " li %2, 0 \n"
406 " bnez %1, 2f \n"
407 " lui %1, 0x8000 \n"
408 " sc %1, %0 \n"
409 " li %2, 1 \n"
410 "2: \n"
411 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp),
412 "=&r" (ret)
413 : GCC_OFF_SMALL_ASM() (rw->lock)
414 : "memory");
415 } while (unlikely(!tmp));
416
417 smp_llsc_mb();
418 }
419
420 return ret;
421}
422
423#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
424#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
425
426#define arch_spin_relax(lock) cpu_relax()
427#define arch_read_relax(lock) cpu_relax()
428#define arch_write_relax(lock) cpu_relax()
429
430#endif /* _ASM_SPINLOCK_H */