Loading...
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8 */
9#ifndef _ASM_SPINLOCK_H
10#define _ASM_SPINLOCK_H
11
12#include <linux/compiler.h>
13
14#include <asm/barrier.h>
15#include <asm/war.h>
16
17/*
18 * Your basic SMP spinlocks, allowing only a single CPU anywhere
19 *
20 * Simple spin lock operations. There are two variants, one clears IRQ's
21 * on the local processor, one does not.
22 *
23 * These are fair FIFO ticket locks
24 *
25 * (the type definitions are in asm/spinlock_types.h)
26 */
27
28
29/*
30 * Ticket locks are conceptually two parts, one indicating the current head of
31 * the queue, and the other indicating the current tail. The lock is acquired
32 * by atomically noting the tail and incrementing it by one (thus adding
33 * ourself to the queue and noting our position), then waiting until the head
34 * becomes equal to the the initial value of the tail.
35 */
36
37static inline int arch_spin_is_locked(arch_spinlock_t *lock)
38{
39 u32 counters = ACCESS_ONCE(lock->lock);
40
41 return ((counters >> 16) ^ counters) & 0xffff;
42}
43
44#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
45#define arch_spin_unlock_wait(x) \
46 while (arch_spin_is_locked(x)) { cpu_relax(); }
47
48static inline int arch_spin_is_contended(arch_spinlock_t *lock)
49{
50 u32 counters = ACCESS_ONCE(lock->lock);
51
52 return (((counters >> 16) - counters) & 0xffff) > 1;
53}
54#define arch_spin_is_contended arch_spin_is_contended
55
56static inline void arch_spin_lock(arch_spinlock_t *lock)
57{
58 int my_ticket;
59 int tmp;
60 int inc = 0x10000;
61
62 if (R10000_LLSC_WAR) {
63 __asm__ __volatile__ (
64 " .set push # arch_spin_lock \n"
65 " .set noreorder \n"
66 " \n"
67 "1: ll %[ticket], %[ticket_ptr] \n"
68 " addu %[my_ticket], %[ticket], %[inc] \n"
69 " sc %[my_ticket], %[ticket_ptr] \n"
70 " beqzl %[my_ticket], 1b \n"
71 " nop \n"
72 " srl %[my_ticket], %[ticket], 16 \n"
73 " andi %[ticket], %[ticket], 0xffff \n"
74 " bne %[ticket], %[my_ticket], 4f \n"
75 " subu %[ticket], %[my_ticket], %[ticket] \n"
76 "2: \n"
77 " .subsection 2 \n"
78 "4: andi %[ticket], %[ticket], 0xffff \n"
79 " sll %[ticket], 5 \n"
80 " \n"
81 "6: bnez %[ticket], 6b \n"
82 " subu %[ticket], 1 \n"
83 " \n"
84 " lhu %[ticket], %[serving_now_ptr] \n"
85 " beq %[ticket], %[my_ticket], 2b \n"
86 " subu %[ticket], %[my_ticket], %[ticket] \n"
87 " b 4b \n"
88 " subu %[ticket], %[ticket], 1 \n"
89 " .previous \n"
90 " .set pop \n"
91 : [ticket_ptr] "+m" (lock->lock),
92 [serving_now_ptr] "+m" (lock->h.serving_now),
93 [ticket] "=&r" (tmp),
94 [my_ticket] "=&r" (my_ticket)
95 : [inc] "r" (inc));
96 } else {
97 __asm__ __volatile__ (
98 " .set push # arch_spin_lock \n"
99 " .set noreorder \n"
100 " \n"
101 "1: ll %[ticket], %[ticket_ptr] \n"
102 " addu %[my_ticket], %[ticket], %[inc] \n"
103 " sc %[my_ticket], %[ticket_ptr] \n"
104 " beqz %[my_ticket], 1b \n"
105 " srl %[my_ticket], %[ticket], 16 \n"
106 " andi %[ticket], %[ticket], 0xffff \n"
107 " bne %[ticket], %[my_ticket], 4f \n"
108 " subu %[ticket], %[my_ticket], %[ticket] \n"
109 "2: \n"
110 " .subsection 2 \n"
111 "4: andi %[ticket], %[ticket], 0x1fff \n"
112 " sll %[ticket], 5 \n"
113 " \n"
114 "6: bnez %[ticket], 6b \n"
115 " subu %[ticket], 1 \n"
116 " \n"
117 " lhu %[ticket], %[serving_now_ptr] \n"
118 " beq %[ticket], %[my_ticket], 2b \n"
119 " subu %[ticket], %[my_ticket], %[ticket] \n"
120 " b 4b \n"
121 " subu %[ticket], %[ticket], 1 \n"
122 " .previous \n"
123 " .set pop \n"
124 : [ticket_ptr] "+m" (lock->lock),
125 [serving_now_ptr] "+m" (lock->h.serving_now),
126 [ticket] "=&r" (tmp),
127 [my_ticket] "=&r" (my_ticket)
128 : [inc] "r" (inc));
129 }
130
131 smp_llsc_mb();
132}
133
134static inline void arch_spin_unlock(arch_spinlock_t *lock)
135{
136 unsigned int serving_now = lock->h.serving_now + 1;
137 wmb();
138 lock->h.serving_now = (u16)serving_now;
139 nudge_writes();
140}
141
142static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
143{
144 int tmp, tmp2, tmp3;
145 int inc = 0x10000;
146
147 if (R10000_LLSC_WAR) {
148 __asm__ __volatile__ (
149 " .set push # arch_spin_trylock \n"
150 " .set noreorder \n"
151 " \n"
152 "1: ll %[ticket], %[ticket_ptr] \n"
153 " srl %[my_ticket], %[ticket], 16 \n"
154 " andi %[now_serving], %[ticket], 0xffff \n"
155 " bne %[my_ticket], %[now_serving], 3f \n"
156 " addu %[ticket], %[ticket], %[inc] \n"
157 " sc %[ticket], %[ticket_ptr] \n"
158 " beqzl %[ticket], 1b \n"
159 " li %[ticket], 1 \n"
160 "2: \n"
161 " .subsection 2 \n"
162 "3: b 2b \n"
163 " li %[ticket], 0 \n"
164 " .previous \n"
165 " .set pop \n"
166 : [ticket_ptr] "+m" (lock->lock),
167 [ticket] "=&r" (tmp),
168 [my_ticket] "=&r" (tmp2),
169 [now_serving] "=&r" (tmp3)
170 : [inc] "r" (inc));
171 } else {
172 __asm__ __volatile__ (
173 " .set push # arch_spin_trylock \n"
174 " .set noreorder \n"
175 " \n"
176 "1: ll %[ticket], %[ticket_ptr] \n"
177 " srl %[my_ticket], %[ticket], 16 \n"
178 " andi %[now_serving], %[ticket], 0xffff \n"
179 " bne %[my_ticket], %[now_serving], 3f \n"
180 " addu %[ticket], %[ticket], %[inc] \n"
181 " sc %[ticket], %[ticket_ptr] \n"
182 " beqz %[ticket], 1b \n"
183 " li %[ticket], 1 \n"
184 "2: \n"
185 " .subsection 2 \n"
186 "3: b 2b \n"
187 " li %[ticket], 0 \n"
188 " .previous \n"
189 " .set pop \n"
190 : [ticket_ptr] "+m" (lock->lock),
191 [ticket] "=&r" (tmp),
192 [my_ticket] "=&r" (tmp2),
193 [now_serving] "=&r" (tmp3)
194 : [inc] "r" (inc));
195 }
196
197 smp_llsc_mb();
198
199 return tmp;
200}
201
202/*
203 * Read-write spinlocks, allowing multiple readers but only one writer.
204 *
205 * NOTE! it is quite common to have readers in interrupts but no interrupt
206 * writers. For those circumstances we can "mix" irq-safe locks - any writer
207 * needs to get a irq-safe write-lock, but readers can get non-irqsafe
208 * read-locks.
209 */
210
211/*
212 * read_can_lock - would read_trylock() succeed?
213 * @lock: the rwlock in question.
214 */
215#define arch_read_can_lock(rw) ((rw)->lock >= 0)
216
217/*
218 * write_can_lock - would write_trylock() succeed?
219 * @lock: the rwlock in question.
220 */
221#define arch_write_can_lock(rw) (!(rw)->lock)
222
223static inline void arch_read_lock(arch_rwlock_t *rw)
224{
225 unsigned int tmp;
226
227 if (R10000_LLSC_WAR) {
228 __asm__ __volatile__(
229 " .set noreorder # arch_read_lock \n"
230 "1: ll %1, %2 \n"
231 " bltz %1, 1b \n"
232 " addu %1, 1 \n"
233 " sc %1, %0 \n"
234 " beqzl %1, 1b \n"
235 " nop \n"
236 " .set reorder \n"
237 : "=m" (rw->lock), "=&r" (tmp)
238 : "m" (rw->lock)
239 : "memory");
240 } else {
241 do {
242 __asm__ __volatile__(
243 "1: ll %1, %2 # arch_read_lock \n"
244 " bltz %1, 1b \n"
245 " addu %1, 1 \n"
246 "2: sc %1, %0 \n"
247 : "=m" (rw->lock), "=&r" (tmp)
248 : "m" (rw->lock)
249 : "memory");
250 } while (unlikely(!tmp));
251 }
252
253 smp_llsc_mb();
254}
255
256/* Note the use of sub, not subu which will make the kernel die with an
257 overflow exception if we ever try to unlock an rwlock that is already
258 unlocked or is being held by a writer. */
259static inline void arch_read_unlock(arch_rwlock_t *rw)
260{
261 unsigned int tmp;
262
263 smp_mb__before_llsc();
264
265 if (R10000_LLSC_WAR) {
266 __asm__ __volatile__(
267 "1: ll %1, %2 # arch_read_unlock \n"
268 " sub %1, 1 \n"
269 " sc %1, %0 \n"
270 " beqzl %1, 1b \n"
271 : "=m" (rw->lock), "=&r" (tmp)
272 : "m" (rw->lock)
273 : "memory");
274 } else {
275 do {
276 __asm__ __volatile__(
277 "1: ll %1, %2 # arch_read_unlock \n"
278 " sub %1, 1 \n"
279 " sc %1, %0 \n"
280 : "=m" (rw->lock), "=&r" (tmp)
281 : "m" (rw->lock)
282 : "memory");
283 } while (unlikely(!tmp));
284 }
285}
286
287static inline void arch_write_lock(arch_rwlock_t *rw)
288{
289 unsigned int tmp;
290
291 if (R10000_LLSC_WAR) {
292 __asm__ __volatile__(
293 " .set noreorder # arch_write_lock \n"
294 "1: ll %1, %2 \n"
295 " bnez %1, 1b \n"
296 " lui %1, 0x8000 \n"
297 " sc %1, %0 \n"
298 " beqzl %1, 1b \n"
299 " nop \n"
300 " .set reorder \n"
301 : "=m" (rw->lock), "=&r" (tmp)
302 : "m" (rw->lock)
303 : "memory");
304 } else {
305 do {
306 __asm__ __volatile__(
307 "1: ll %1, %2 # arch_write_lock \n"
308 " bnez %1, 1b \n"
309 " lui %1, 0x8000 \n"
310 "2: sc %1, %0 \n"
311 : "=m" (rw->lock), "=&r" (tmp)
312 : "m" (rw->lock)
313 : "memory");
314 } while (unlikely(!tmp));
315 }
316
317 smp_llsc_mb();
318}
319
320static inline void arch_write_unlock(arch_rwlock_t *rw)
321{
322 smp_mb();
323
324 __asm__ __volatile__(
325 " # arch_write_unlock \n"
326 " sw $0, %0 \n"
327 : "=m" (rw->lock)
328 : "m" (rw->lock)
329 : "memory");
330}
331
332static inline int arch_read_trylock(arch_rwlock_t *rw)
333{
334 unsigned int tmp;
335 int ret;
336
337 if (R10000_LLSC_WAR) {
338 __asm__ __volatile__(
339 " .set noreorder # arch_read_trylock \n"
340 " li %2, 0 \n"
341 "1: ll %1, %3 \n"
342 " bltz %1, 2f \n"
343 " addu %1, 1 \n"
344 " sc %1, %0 \n"
345 " .set reorder \n"
346 " beqzl %1, 1b \n"
347 " nop \n"
348 __WEAK_LLSC_MB
349 " li %2, 1 \n"
350 "2: \n"
351 : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
352 : "m" (rw->lock)
353 : "memory");
354 } else {
355 __asm__ __volatile__(
356 " .set noreorder # arch_read_trylock \n"
357 " li %2, 0 \n"
358 "1: ll %1, %3 \n"
359 " bltz %1, 2f \n"
360 " addu %1, 1 \n"
361 " sc %1, %0 \n"
362 " beqz %1, 1b \n"
363 " nop \n"
364 " .set reorder \n"
365 __WEAK_LLSC_MB
366 " li %2, 1 \n"
367 "2: \n"
368 : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
369 : "m" (rw->lock)
370 : "memory");
371 }
372
373 return ret;
374}
375
376static inline int arch_write_trylock(arch_rwlock_t *rw)
377{
378 unsigned int tmp;
379 int ret;
380
381 if (R10000_LLSC_WAR) {
382 __asm__ __volatile__(
383 " .set noreorder # arch_write_trylock \n"
384 " li %2, 0 \n"
385 "1: ll %1, %3 \n"
386 " bnez %1, 2f \n"
387 " lui %1, 0x8000 \n"
388 " sc %1, %0 \n"
389 " beqzl %1, 1b \n"
390 " nop \n"
391 __WEAK_LLSC_MB
392 " li %2, 1 \n"
393 " .set reorder \n"
394 "2: \n"
395 : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
396 : "m" (rw->lock)
397 : "memory");
398 } else {
399 do {
400 __asm__ __volatile__(
401 " ll %1, %3 # arch_write_trylock \n"
402 " li %2, 0 \n"
403 " bnez %1, 2f \n"
404 " lui %1, 0x8000 \n"
405 " sc %1, %0 \n"
406 " li %2, 1 \n"
407 "2: \n"
408 : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
409 : "m" (rw->lock)
410 : "memory");
411 } while (unlikely(!tmp));
412
413 smp_llsc_mb();
414 }
415
416 return ret;
417}
418
419#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
420#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
421
422#define arch_spin_relax(lock) cpu_relax()
423#define arch_read_relax(lock) cpu_relax()
424#define arch_write_relax(lock) cpu_relax()
425
426#endif /* _ASM_SPINLOCK_H */
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8 */
9#ifndef _ASM_SPINLOCK_H
10#define _ASM_SPINLOCK_H
11
12#include <linux/compiler.h>
13
14#include <asm/barrier.h>
15#include <asm/compiler.h>
16#include <asm/war.h>
17
18/*
19 * Your basic SMP spinlocks, allowing only a single CPU anywhere
20 *
21 * Simple spin lock operations. There are two variants, one clears IRQ's
22 * on the local processor, one does not.
23 *
24 * These are fair FIFO ticket locks
25 *
26 * (the type definitions are in asm/spinlock_types.h)
27 */
28
29
30/*
31 * Ticket locks are conceptually two parts, one indicating the current head of
32 * the queue, and the other indicating the current tail. The lock is acquired
33 * by atomically noting the tail and incrementing it by one (thus adding
34 * ourself to the queue and noting our position), then waiting until the head
35 * becomes equal to the the initial value of the tail.
36 */
37
38static inline int arch_spin_is_locked(arch_spinlock_t *lock)
39{
40 u32 counters = ACCESS_ONCE(lock->lock);
41
42 return ((counters >> 16) ^ counters) & 0xffff;
43}
44
45static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
46{
47 return lock.h.serving_now == lock.h.ticket;
48}
49
50#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
51#define arch_spin_unlock_wait(x) \
52 while (arch_spin_is_locked(x)) { cpu_relax(); }
53
54static inline int arch_spin_is_contended(arch_spinlock_t *lock)
55{
56 u32 counters = ACCESS_ONCE(lock->lock);
57
58 return (((counters >> 16) - counters) & 0xffff) > 1;
59}
60#define arch_spin_is_contended arch_spin_is_contended
61
62static inline void arch_spin_lock(arch_spinlock_t *lock)
63{
64 int my_ticket;
65 int tmp;
66 int inc = 0x10000;
67
68 if (R10000_LLSC_WAR) {
69 __asm__ __volatile__ (
70 " .set push # arch_spin_lock \n"
71 " .set noreorder \n"
72 " \n"
73 "1: ll %[ticket], %[ticket_ptr] \n"
74 " addu %[my_ticket], %[ticket], %[inc] \n"
75 " sc %[my_ticket], %[ticket_ptr] \n"
76 " beqzl %[my_ticket], 1b \n"
77 " nop \n"
78 " srl %[my_ticket], %[ticket], 16 \n"
79 " andi %[ticket], %[ticket], 0xffff \n"
80 " bne %[ticket], %[my_ticket], 4f \n"
81 " subu %[ticket], %[my_ticket], %[ticket] \n"
82 "2: \n"
83 " .subsection 2 \n"
84 "4: andi %[ticket], %[ticket], 0xffff \n"
85 " sll %[ticket], 5 \n"
86 " \n"
87 "6: bnez %[ticket], 6b \n"
88 " subu %[ticket], 1 \n"
89 " \n"
90 " lhu %[ticket], %[serving_now_ptr] \n"
91 " beq %[ticket], %[my_ticket], 2b \n"
92 " subu %[ticket], %[my_ticket], %[ticket] \n"
93 " b 4b \n"
94 " subu %[ticket], %[ticket], 1 \n"
95 " .previous \n"
96 " .set pop \n"
97 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
98 [serving_now_ptr] "+m" (lock->h.serving_now),
99 [ticket] "=&r" (tmp),
100 [my_ticket] "=&r" (my_ticket)
101 : [inc] "r" (inc));
102 } else {
103 __asm__ __volatile__ (
104 " .set push # arch_spin_lock \n"
105 " .set noreorder \n"
106 " \n"
107 "1: ll %[ticket], %[ticket_ptr] \n"
108 " addu %[my_ticket], %[ticket], %[inc] \n"
109 " sc %[my_ticket], %[ticket_ptr] \n"
110 " beqz %[my_ticket], 1b \n"
111 " srl %[my_ticket], %[ticket], 16 \n"
112 " andi %[ticket], %[ticket], 0xffff \n"
113 " bne %[ticket], %[my_ticket], 4f \n"
114 " subu %[ticket], %[my_ticket], %[ticket] \n"
115 "2: \n"
116 " .subsection 2 \n"
117 "4: andi %[ticket], %[ticket], 0xffff \n"
118 " sll %[ticket], 5 \n"
119 " \n"
120 "6: bnez %[ticket], 6b \n"
121 " subu %[ticket], 1 \n"
122 " \n"
123 " lhu %[ticket], %[serving_now_ptr] \n"
124 " beq %[ticket], %[my_ticket], 2b \n"
125 " subu %[ticket], %[my_ticket], %[ticket] \n"
126 " b 4b \n"
127 " subu %[ticket], %[ticket], 1 \n"
128 " .previous \n"
129 " .set pop \n"
130 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
131 [serving_now_ptr] "+m" (lock->h.serving_now),
132 [ticket] "=&r" (tmp),
133 [my_ticket] "=&r" (my_ticket)
134 : [inc] "r" (inc));
135 }
136
137 smp_llsc_mb();
138}
139
140static inline void arch_spin_unlock(arch_spinlock_t *lock)
141{
142 unsigned int serving_now = lock->h.serving_now + 1;
143 wmb();
144 lock->h.serving_now = (u16)serving_now;
145 nudge_writes();
146}
147
148static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
149{
150 int tmp, tmp2, tmp3;
151 int inc = 0x10000;
152
153 if (R10000_LLSC_WAR) {
154 __asm__ __volatile__ (
155 " .set push # arch_spin_trylock \n"
156 " .set noreorder \n"
157 " \n"
158 "1: ll %[ticket], %[ticket_ptr] \n"
159 " srl %[my_ticket], %[ticket], 16 \n"
160 " andi %[now_serving], %[ticket], 0xffff \n"
161 " bne %[my_ticket], %[now_serving], 3f \n"
162 " addu %[ticket], %[ticket], %[inc] \n"
163 " sc %[ticket], %[ticket_ptr] \n"
164 " beqzl %[ticket], 1b \n"
165 " li %[ticket], 1 \n"
166 "2: \n"
167 " .subsection 2 \n"
168 "3: b 2b \n"
169 " li %[ticket], 0 \n"
170 " .previous \n"
171 " .set pop \n"
172 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
173 [ticket] "=&r" (tmp),
174 [my_ticket] "=&r" (tmp2),
175 [now_serving] "=&r" (tmp3)
176 : [inc] "r" (inc));
177 } else {
178 __asm__ __volatile__ (
179 " .set push # arch_spin_trylock \n"
180 " .set noreorder \n"
181 " \n"
182 "1: ll %[ticket], %[ticket_ptr] \n"
183 " srl %[my_ticket], %[ticket], 16 \n"
184 " andi %[now_serving], %[ticket], 0xffff \n"
185 " bne %[my_ticket], %[now_serving], 3f \n"
186 " addu %[ticket], %[ticket], %[inc] \n"
187 " sc %[ticket], %[ticket_ptr] \n"
188 " beqz %[ticket], 1b \n"
189 " li %[ticket], 1 \n"
190 "2: \n"
191 " .subsection 2 \n"
192 "3: b 2b \n"
193 " li %[ticket], 0 \n"
194 " .previous \n"
195 " .set pop \n"
196 : [ticket_ptr] "+" GCC_OFF_SMALL_ASM() (lock->lock),
197 [ticket] "=&r" (tmp),
198 [my_ticket] "=&r" (tmp2),
199 [now_serving] "=&r" (tmp3)
200 : [inc] "r" (inc));
201 }
202
203 smp_llsc_mb();
204
205 return tmp;
206}
207
208/*
209 * Read-write spinlocks, allowing multiple readers but only one writer.
210 *
211 * NOTE! it is quite common to have readers in interrupts but no interrupt
212 * writers. For those circumstances we can "mix" irq-safe locks - any writer
213 * needs to get a irq-safe write-lock, but readers can get non-irqsafe
214 * read-locks.
215 */
216
217/*
218 * read_can_lock - would read_trylock() succeed?
219 * @lock: the rwlock in question.
220 */
221#define arch_read_can_lock(rw) ((rw)->lock >= 0)
222
223/*
224 * write_can_lock - would write_trylock() succeed?
225 * @lock: the rwlock in question.
226 */
227#define arch_write_can_lock(rw) (!(rw)->lock)
228
229static inline void arch_read_lock(arch_rwlock_t *rw)
230{
231 unsigned int tmp;
232
233 if (R10000_LLSC_WAR) {
234 __asm__ __volatile__(
235 " .set noreorder # arch_read_lock \n"
236 "1: ll %1, %2 \n"
237 " bltz %1, 1b \n"
238 " addu %1, 1 \n"
239 " sc %1, %0 \n"
240 " beqzl %1, 1b \n"
241 " nop \n"
242 " .set reorder \n"
243 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
244 : GCC_OFF_SMALL_ASM() (rw->lock)
245 : "memory");
246 } else {
247 do {
248 __asm__ __volatile__(
249 "1: ll %1, %2 # arch_read_lock \n"
250 " bltz %1, 1b \n"
251 " addu %1, 1 \n"
252 "2: sc %1, %0 \n"
253 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
254 : GCC_OFF_SMALL_ASM() (rw->lock)
255 : "memory");
256 } while (unlikely(!tmp));
257 }
258
259 smp_llsc_mb();
260}
261
262static inline void arch_read_unlock(arch_rwlock_t *rw)
263{
264 unsigned int tmp;
265
266 smp_mb__before_llsc();
267
268 if (R10000_LLSC_WAR) {
269 __asm__ __volatile__(
270 "1: ll %1, %2 # arch_read_unlock \n"
271 " addiu %1, -1 \n"
272 " sc %1, %0 \n"
273 " beqzl %1, 1b \n"
274 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
275 : GCC_OFF_SMALL_ASM() (rw->lock)
276 : "memory");
277 } else {
278 do {
279 __asm__ __volatile__(
280 "1: ll %1, %2 # arch_read_unlock \n"
281 " addiu %1, -1 \n"
282 " sc %1, %0 \n"
283 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
284 : GCC_OFF_SMALL_ASM() (rw->lock)
285 : "memory");
286 } while (unlikely(!tmp));
287 }
288}
289
290static inline void arch_write_lock(arch_rwlock_t *rw)
291{
292 unsigned int tmp;
293
294 if (R10000_LLSC_WAR) {
295 __asm__ __volatile__(
296 " .set noreorder # arch_write_lock \n"
297 "1: ll %1, %2 \n"
298 " bnez %1, 1b \n"
299 " lui %1, 0x8000 \n"
300 " sc %1, %0 \n"
301 " beqzl %1, 1b \n"
302 " nop \n"
303 " .set reorder \n"
304 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
305 : GCC_OFF_SMALL_ASM() (rw->lock)
306 : "memory");
307 } else {
308 do {
309 __asm__ __volatile__(
310 "1: ll %1, %2 # arch_write_lock \n"
311 " bnez %1, 1b \n"
312 " lui %1, 0x8000 \n"
313 "2: sc %1, %0 \n"
314 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp)
315 : GCC_OFF_SMALL_ASM() (rw->lock)
316 : "memory");
317 } while (unlikely(!tmp));
318 }
319
320 smp_llsc_mb();
321}
322
323static inline void arch_write_unlock(arch_rwlock_t *rw)
324{
325 smp_mb__before_llsc();
326
327 __asm__ __volatile__(
328 " # arch_write_unlock \n"
329 " sw $0, %0 \n"
330 : "=m" (rw->lock)
331 : "m" (rw->lock)
332 : "memory");
333}
334
335static inline int arch_read_trylock(arch_rwlock_t *rw)
336{
337 unsigned int tmp;
338 int ret;
339
340 if (R10000_LLSC_WAR) {
341 __asm__ __volatile__(
342 " .set noreorder # arch_read_trylock \n"
343 " li %2, 0 \n"
344 "1: ll %1, %3 \n"
345 " bltz %1, 2f \n"
346 " addu %1, 1 \n"
347 " sc %1, %0 \n"
348 " .set reorder \n"
349 " beqzl %1, 1b \n"
350 " nop \n"
351 __WEAK_LLSC_MB
352 " li %2, 1 \n"
353 "2: \n"
354 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
355 : GCC_OFF_SMALL_ASM() (rw->lock)
356 : "memory");
357 } else {
358 __asm__ __volatile__(
359 " .set noreorder # arch_read_trylock \n"
360 " li %2, 0 \n"
361 "1: ll %1, %3 \n"
362 " bltz %1, 2f \n"
363 " addu %1, 1 \n"
364 " sc %1, %0 \n"
365 " beqz %1, 1b \n"
366 " nop \n"
367 " .set reorder \n"
368 __WEAK_LLSC_MB
369 " li %2, 1 \n"
370 "2: \n"
371 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
372 : GCC_OFF_SMALL_ASM() (rw->lock)
373 : "memory");
374 }
375
376 return ret;
377}
378
379static inline int arch_write_trylock(arch_rwlock_t *rw)
380{
381 unsigned int tmp;
382 int ret;
383
384 if (R10000_LLSC_WAR) {
385 __asm__ __volatile__(
386 " .set noreorder # arch_write_trylock \n"
387 " li %2, 0 \n"
388 "1: ll %1, %3 \n"
389 " bnez %1, 2f \n"
390 " lui %1, 0x8000 \n"
391 " sc %1, %0 \n"
392 " beqzl %1, 1b \n"
393 " nop \n"
394 __WEAK_LLSC_MB
395 " li %2, 1 \n"
396 " .set reorder \n"
397 "2: \n"
398 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp), "=&r" (ret)
399 : GCC_OFF_SMALL_ASM() (rw->lock)
400 : "memory");
401 } else {
402 do {
403 __asm__ __volatile__(
404 " ll %1, %3 # arch_write_trylock \n"
405 " li %2, 0 \n"
406 " bnez %1, 2f \n"
407 " lui %1, 0x8000 \n"
408 " sc %1, %0 \n"
409 " li %2, 1 \n"
410 "2: \n"
411 : "=" GCC_OFF_SMALL_ASM() (rw->lock), "=&r" (tmp),
412 "=&r" (ret)
413 : GCC_OFF_SMALL_ASM() (rw->lock)
414 : "memory");
415 } while (unlikely(!tmp));
416
417 smp_llsc_mb();
418 }
419
420 return ret;
421}
422
423#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
424#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
425
426#define arch_spin_relax(lock) cpu_relax()
427#define arch_read_relax(lock) cpu_relax()
428#define arch_write_relax(lock) cpu_relax()
429
430#endif /* _ASM_SPINLOCK_H */