Loading...
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (c) 1999, 2000 Silicon Graphics, Inc.
8 */
9#ifndef _ASM_BITOPS_H
10#define _ASM_BITOPS_H
11
12#ifndef _LINUX_BITOPS_H
13#error only <linux/bitops.h> can be included directly
14#endif
15
16#include <linux/compiler.h>
17#include <linux/types.h>
18#include <asm/barrier.h>
19#include <asm/byteorder.h> /* sigh ... */
20#include <asm/compiler.h>
21#include <asm/cpu-features.h>
22#include <asm/llsc.h>
23#include <asm/sgidefs.h>
24#include <asm/war.h>
25
26/*
27 * These are the "slower" versions of the functions and are in bitops.c.
28 * These functions call raw_local_irq_{save,restore}().
29 */
30void __mips_set_bit(unsigned long nr, volatile unsigned long *addr);
31void __mips_clear_bit(unsigned long nr, volatile unsigned long *addr);
32void __mips_change_bit(unsigned long nr, volatile unsigned long *addr);
33int __mips_test_and_set_bit(unsigned long nr,
34 volatile unsigned long *addr);
35int __mips_test_and_set_bit_lock(unsigned long nr,
36 volatile unsigned long *addr);
37int __mips_test_and_clear_bit(unsigned long nr,
38 volatile unsigned long *addr);
39int __mips_test_and_change_bit(unsigned long nr,
40 volatile unsigned long *addr);
41
42
43/*
44 * set_bit - Atomically set a bit in memory
45 * @nr: the bit to set
46 * @addr: the address to start counting from
47 *
48 * This function is atomic and may not be reordered. See __set_bit()
49 * if you do not require the atomic guarantees.
50 * Note that @nr may be almost arbitrarily large; this function is not
51 * restricted to acting on a single-word quantity.
52 */
53static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
54{
55 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
56 int bit = nr & SZLONG_MASK;
57 unsigned long temp;
58
59 if (kernel_uses_llsc && R10000_LLSC_WAR) {
60 __asm__ __volatile__(
61 " .set arch=r4000 \n"
62 "1: " __LL "%0, %1 # set_bit \n"
63 " or %0, %2 \n"
64 " " __SC "%0, %1 \n"
65 " beqzl %0, 1b \n"
66 " .set mips0 \n"
67 : "=&r" (temp), "=" GCC_OFF_SMALL_ASM() (*m)
68 : "ir" (1UL << bit), GCC_OFF_SMALL_ASM() (*m));
69#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
70 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
71 do {
72 __asm__ __volatile__(
73 " " __LL "%0, %1 # set_bit \n"
74 " " __INS "%0, %3, %2, 1 \n"
75 " " __SC "%0, %1 \n"
76 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
77 : "ir" (bit), "r" (~0));
78 } while (unlikely(!temp));
79#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
80 } else if (kernel_uses_llsc) {
81 do {
82 __asm__ __volatile__(
83 " .set "MIPS_ISA_ARCH_LEVEL" \n"
84 " " __LL "%0, %1 # set_bit \n"
85 " or %0, %2 \n"
86 " " __SC "%0, %1 \n"
87 " .set mips0 \n"
88 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
89 : "ir" (1UL << bit));
90 } while (unlikely(!temp));
91 } else
92 __mips_set_bit(nr, addr);
93}
94
95/*
96 * clear_bit - Clears a bit in memory
97 * @nr: Bit to clear
98 * @addr: Address to start counting from
99 *
100 * clear_bit() is atomic and may not be reordered. However, it does
101 * not contain a memory barrier, so if it is used for locking purposes,
102 * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic()
103 * in order to ensure changes are visible on other processors.
104 */
105static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
106{
107 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
108 int bit = nr & SZLONG_MASK;
109 unsigned long temp;
110
111 if (kernel_uses_llsc && R10000_LLSC_WAR) {
112 __asm__ __volatile__(
113 " .set arch=r4000 \n"
114 "1: " __LL "%0, %1 # clear_bit \n"
115 " and %0, %2 \n"
116 " " __SC "%0, %1 \n"
117 " beqzl %0, 1b \n"
118 " .set mips0 \n"
119 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
120 : "ir" (~(1UL << bit)));
121#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
122 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
123 do {
124 __asm__ __volatile__(
125 " " __LL "%0, %1 # clear_bit \n"
126 " " __INS "%0, $0, %2, 1 \n"
127 " " __SC "%0, %1 \n"
128 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
129 : "ir" (bit));
130 } while (unlikely(!temp));
131#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
132 } else if (kernel_uses_llsc) {
133 do {
134 __asm__ __volatile__(
135 " .set "MIPS_ISA_ARCH_LEVEL" \n"
136 " " __LL "%0, %1 # clear_bit \n"
137 " and %0, %2 \n"
138 " " __SC "%0, %1 \n"
139 " .set mips0 \n"
140 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
141 : "ir" (~(1UL << bit)));
142 } while (unlikely(!temp));
143 } else
144 __mips_clear_bit(nr, addr);
145}
146
147/*
148 * clear_bit_unlock - Clears a bit in memory
149 * @nr: Bit to clear
150 * @addr: Address to start counting from
151 *
152 * clear_bit() is atomic and implies release semantics before the memory
153 * operation. It can be used for an unlock.
154 */
155static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
156{
157 smp_mb__before_atomic();
158 clear_bit(nr, addr);
159}
160
161/*
162 * change_bit - Toggle a bit in memory
163 * @nr: Bit to change
164 * @addr: Address to start counting from
165 *
166 * change_bit() is atomic and may not be reordered.
167 * Note that @nr may be almost arbitrarily large; this function is not
168 * restricted to acting on a single-word quantity.
169 */
170static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
171{
172 int bit = nr & SZLONG_MASK;
173
174 if (kernel_uses_llsc && R10000_LLSC_WAR) {
175 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
176 unsigned long temp;
177
178 __asm__ __volatile__(
179 " .set arch=r4000 \n"
180 "1: " __LL "%0, %1 # change_bit \n"
181 " xor %0, %2 \n"
182 " " __SC "%0, %1 \n"
183 " beqzl %0, 1b \n"
184 " .set mips0 \n"
185 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
186 : "ir" (1UL << bit));
187 } else if (kernel_uses_llsc) {
188 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
189 unsigned long temp;
190
191 do {
192 __asm__ __volatile__(
193 " .set "MIPS_ISA_ARCH_LEVEL" \n"
194 " " __LL "%0, %1 # change_bit \n"
195 " xor %0, %2 \n"
196 " " __SC "%0, %1 \n"
197 " .set mips0 \n"
198 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
199 : "ir" (1UL << bit));
200 } while (unlikely(!temp));
201 } else
202 __mips_change_bit(nr, addr);
203}
204
205/*
206 * test_and_set_bit - Set a bit and return its old value
207 * @nr: Bit to set
208 * @addr: Address to count from
209 *
210 * This operation is atomic and cannot be reordered.
211 * It also implies a memory barrier.
212 */
213static inline int test_and_set_bit(unsigned long nr,
214 volatile unsigned long *addr)
215{
216 int bit = nr & SZLONG_MASK;
217 unsigned long res;
218
219 smp_mb__before_llsc();
220
221 if (kernel_uses_llsc && R10000_LLSC_WAR) {
222 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
223 unsigned long temp;
224
225 __asm__ __volatile__(
226 " .set arch=r4000 \n"
227 "1: " __LL "%0, %1 # test_and_set_bit \n"
228 " or %2, %0, %3 \n"
229 " " __SC "%2, %1 \n"
230 " beqzl %2, 1b \n"
231 " and %2, %0, %3 \n"
232 " .set mips0 \n"
233 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
234 : "r" (1UL << bit)
235 : "memory");
236 } else if (kernel_uses_llsc) {
237 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
238 unsigned long temp;
239
240 do {
241 __asm__ __volatile__(
242 " .set "MIPS_ISA_ARCH_LEVEL" \n"
243 " " __LL "%0, %1 # test_and_set_bit \n"
244 " or %2, %0, %3 \n"
245 " " __SC "%2, %1 \n"
246 " .set mips0 \n"
247 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
248 : "r" (1UL << bit)
249 : "memory");
250 } while (unlikely(!res));
251
252 res = temp & (1UL << bit);
253 } else
254 res = __mips_test_and_set_bit(nr, addr);
255
256 smp_llsc_mb();
257
258 return res != 0;
259}
260
261/*
262 * test_and_set_bit_lock - Set a bit and return its old value
263 * @nr: Bit to set
264 * @addr: Address to count from
265 *
266 * This operation is atomic and implies acquire ordering semantics
267 * after the memory operation.
268 */
269static inline int test_and_set_bit_lock(unsigned long nr,
270 volatile unsigned long *addr)
271{
272 int bit = nr & SZLONG_MASK;
273 unsigned long res;
274
275 if (kernel_uses_llsc && R10000_LLSC_WAR) {
276 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
277 unsigned long temp;
278
279 __asm__ __volatile__(
280 " .set arch=r4000 \n"
281 "1: " __LL "%0, %1 # test_and_set_bit \n"
282 " or %2, %0, %3 \n"
283 " " __SC "%2, %1 \n"
284 " beqzl %2, 1b \n"
285 " and %2, %0, %3 \n"
286 " .set mips0 \n"
287 : "=&r" (temp), "+m" (*m), "=&r" (res)
288 : "r" (1UL << bit)
289 : "memory");
290 } else if (kernel_uses_llsc) {
291 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
292 unsigned long temp;
293
294 do {
295 __asm__ __volatile__(
296 " .set "MIPS_ISA_ARCH_LEVEL" \n"
297 " " __LL "%0, %1 # test_and_set_bit \n"
298 " or %2, %0, %3 \n"
299 " " __SC "%2, %1 \n"
300 " .set mips0 \n"
301 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
302 : "r" (1UL << bit)
303 : "memory");
304 } while (unlikely(!res));
305
306 res = temp & (1UL << bit);
307 } else
308 res = __mips_test_and_set_bit_lock(nr, addr);
309
310 smp_llsc_mb();
311
312 return res != 0;
313}
314/*
315 * test_and_clear_bit - Clear a bit and return its old value
316 * @nr: Bit to clear
317 * @addr: Address to count from
318 *
319 * This operation is atomic and cannot be reordered.
320 * It also implies a memory barrier.
321 */
322static inline int test_and_clear_bit(unsigned long nr,
323 volatile unsigned long *addr)
324{
325 int bit = nr & SZLONG_MASK;
326 unsigned long res;
327
328 smp_mb__before_llsc();
329
330 if (kernel_uses_llsc && R10000_LLSC_WAR) {
331 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
332 unsigned long temp;
333
334 __asm__ __volatile__(
335 " .set arch=r4000 \n"
336 "1: " __LL "%0, %1 # test_and_clear_bit \n"
337 " or %2, %0, %3 \n"
338 " xor %2, %3 \n"
339 " " __SC "%2, %1 \n"
340 " beqzl %2, 1b \n"
341 " and %2, %0, %3 \n"
342 " .set mips0 \n"
343 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
344 : "r" (1UL << bit)
345 : "memory");
346#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
347 } else if (kernel_uses_llsc && __builtin_constant_p(nr)) {
348 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
349 unsigned long temp;
350
351 do {
352 __asm__ __volatile__(
353 " " __LL "%0, %1 # test_and_clear_bit \n"
354 " " __EXT "%2, %0, %3, 1 \n"
355 " " __INS "%0, $0, %3, 1 \n"
356 " " __SC "%0, %1 \n"
357 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
358 : "ir" (bit)
359 : "memory");
360 } while (unlikely(!temp));
361#endif
362 } else if (kernel_uses_llsc) {
363 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
364 unsigned long temp;
365
366 do {
367 __asm__ __volatile__(
368 " .set "MIPS_ISA_ARCH_LEVEL" \n"
369 " " __LL "%0, %1 # test_and_clear_bit \n"
370 " or %2, %0, %3 \n"
371 " xor %2, %3 \n"
372 " " __SC "%2, %1 \n"
373 " .set mips0 \n"
374 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
375 : "r" (1UL << bit)
376 : "memory");
377 } while (unlikely(!res));
378
379 res = temp & (1UL << bit);
380 } else
381 res = __mips_test_and_clear_bit(nr, addr);
382
383 smp_llsc_mb();
384
385 return res != 0;
386}
387
388/*
389 * test_and_change_bit - Change a bit and return its old value
390 * @nr: Bit to change
391 * @addr: Address to count from
392 *
393 * This operation is atomic and cannot be reordered.
394 * It also implies a memory barrier.
395 */
396static inline int test_and_change_bit(unsigned long nr,
397 volatile unsigned long *addr)
398{
399 int bit = nr & SZLONG_MASK;
400 unsigned long res;
401
402 smp_mb__before_llsc();
403
404 if (kernel_uses_llsc && R10000_LLSC_WAR) {
405 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
406 unsigned long temp;
407
408 __asm__ __volatile__(
409 " .set arch=r4000 \n"
410 "1: " __LL "%0, %1 # test_and_change_bit \n"
411 " xor %2, %0, %3 \n"
412 " " __SC "%2, %1 \n"
413 " beqzl %2, 1b \n"
414 " and %2, %0, %3 \n"
415 " .set mips0 \n"
416 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
417 : "r" (1UL << bit)
418 : "memory");
419 } else if (kernel_uses_llsc) {
420 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
421 unsigned long temp;
422
423 do {
424 __asm__ __volatile__(
425 " .set "MIPS_ISA_ARCH_LEVEL" \n"
426 " " __LL "%0, %1 # test_and_change_bit \n"
427 " xor %2, %0, %3 \n"
428 " " __SC "\t%2, %1 \n"
429 " .set mips0 \n"
430 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
431 : "r" (1UL << bit)
432 : "memory");
433 } while (unlikely(!res));
434
435 res = temp & (1UL << bit);
436 } else
437 res = __mips_test_and_change_bit(nr, addr);
438
439 smp_llsc_mb();
440
441 return res != 0;
442}
443
444#include <asm-generic/bitops/non-atomic.h>
445
446/*
447 * __clear_bit_unlock - Clears a bit in memory
448 * @nr: Bit to clear
449 * @addr: Address to start counting from
450 *
451 * __clear_bit() is non-atomic and implies release semantics before the memory
452 * operation. It can be used for an unlock if no other CPUs can concurrently
453 * modify other bits in the word.
454 */
455static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
456{
457 smp_mb__before_llsc();
458 __clear_bit(nr, addr);
459 nudge_writes();
460}
461
462/*
463 * Return the bit position (0..63) of the most significant 1 bit in a word
464 * Returns -1 if no 1 bit exists
465 */
466static inline unsigned long __fls(unsigned long word)
467{
468 int num;
469
470 if (BITS_PER_LONG == 32 && !__builtin_constant_p(word) &&
471 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
472 __asm__(
473 " .set push \n"
474 " .set "MIPS_ISA_LEVEL" \n"
475 " clz %0, %1 \n"
476 " .set pop \n"
477 : "=r" (num)
478 : "r" (word));
479
480 return 31 - num;
481 }
482
483 if (BITS_PER_LONG == 64 && !__builtin_constant_p(word) &&
484 __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) {
485 __asm__(
486 " .set push \n"
487 " .set "MIPS_ISA_LEVEL" \n"
488 " dclz %0, %1 \n"
489 " .set pop \n"
490 : "=r" (num)
491 : "r" (word));
492
493 return 63 - num;
494 }
495
496 num = BITS_PER_LONG - 1;
497
498#if BITS_PER_LONG == 64
499 if (!(word & (~0ul << 32))) {
500 num -= 32;
501 word <<= 32;
502 }
503#endif
504 if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
505 num -= 16;
506 word <<= 16;
507 }
508 if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
509 num -= 8;
510 word <<= 8;
511 }
512 if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
513 num -= 4;
514 word <<= 4;
515 }
516 if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
517 num -= 2;
518 word <<= 2;
519 }
520 if (!(word & (~0ul << (BITS_PER_LONG-1))))
521 num -= 1;
522 return num;
523}
524
525/*
526 * __ffs - find first bit in word.
527 * @word: The word to search
528 *
529 * Returns 0..SZLONG-1
530 * Undefined if no bit exists, so code should check against 0 first.
531 */
532static inline unsigned long __ffs(unsigned long word)
533{
534 return __fls(word & -word);
535}
536
537/*
538 * fls - find last bit set.
539 * @word: The word to search
540 *
541 * This is defined the same way as ffs.
542 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
543 */
544static inline int fls(int x)
545{
546 int r;
547
548 if (!__builtin_constant_p(x) &&
549 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
550 __asm__(
551 " .set push \n"
552 " .set "MIPS_ISA_LEVEL" \n"
553 " clz %0, %1 \n"
554 " .set pop \n"
555 : "=r" (x)
556 : "r" (x));
557
558 return 32 - x;
559 }
560
561 r = 32;
562 if (!x)
563 return 0;
564 if (!(x & 0xffff0000u)) {
565 x <<= 16;
566 r -= 16;
567 }
568 if (!(x & 0xff000000u)) {
569 x <<= 8;
570 r -= 8;
571 }
572 if (!(x & 0xf0000000u)) {
573 x <<= 4;
574 r -= 4;
575 }
576 if (!(x & 0xc0000000u)) {
577 x <<= 2;
578 r -= 2;
579 }
580 if (!(x & 0x80000000u)) {
581 x <<= 1;
582 r -= 1;
583 }
584 return r;
585}
586
587#include <asm-generic/bitops/fls64.h>
588
589/*
590 * ffs - find first bit set.
591 * @word: The word to search
592 *
593 * This is defined the same way as
594 * the libc and compiler builtin ffs routines, therefore
595 * differs in spirit from the above ffz (man ffs).
596 */
597static inline int ffs(int word)
598{
599 if (!word)
600 return 0;
601
602 return fls(word & -word);
603}
604
605#include <asm-generic/bitops/ffz.h>
606#include <asm-generic/bitops/find.h>
607
608#ifdef __KERNEL__
609
610#include <asm-generic/bitops/sched.h>
611
612#include <asm/arch_hweight.h>
613#include <asm-generic/bitops/const_hweight.h>
614
615#include <asm-generic/bitops/le.h>
616#include <asm-generic/bitops/ext2-atomic.h>
617
618#endif /* __KERNEL__ */
619
620#endif /* _ASM_BITOPS_H */
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (c) 1999, 2000 Silicon Graphics, Inc.
8 */
9#ifndef _ASM_BITOPS_H
10#define _ASM_BITOPS_H
11
12#ifndef _LINUX_BITOPS_H
13#error only <linux/bitops.h> can be included directly
14#endif
15
16#include <linux/bits.h>
17#include <linux/compiler.h>
18#include <linux/types.h>
19#include <asm/asm.h>
20#include <asm/barrier.h>
21#include <asm/byteorder.h> /* sigh ... */
22#include <asm/compiler.h>
23#include <asm/cpu-features.h>
24#include <asm/sgidefs.h>
25
26#define __bit_op(mem, insn, inputs...) do { \
27 unsigned long __temp; \
28 \
29 asm volatile( \
30 " .set push \n" \
31 " .set " MIPS_ISA_LEVEL " \n" \
32 " " __SYNC(full, loongson3_war) " \n" \
33 "1: " __stringify(LONG_LL) " %0, %1 \n" \
34 " " insn " \n" \
35 " " __stringify(LONG_SC) " %0, %1 \n" \
36 " " __stringify(SC_BEQZ) " %0, 1b \n" \
37 " .set pop \n" \
38 : "=&r"(__temp), "+" GCC_OFF_SMALL_ASM()(mem) \
39 : inputs \
40 : __LLSC_CLOBBER); \
41} while (0)
42
43#define __test_bit_op(mem, ll_dst, insn, inputs...) ({ \
44 unsigned long __orig, __temp; \
45 \
46 asm volatile( \
47 " .set push \n" \
48 " .set " MIPS_ISA_LEVEL " \n" \
49 " " __SYNC(full, loongson3_war) " \n" \
50 "1: " __stringify(LONG_LL) " " ll_dst ", %2\n" \
51 " " insn " \n" \
52 " " __stringify(LONG_SC) " %1, %2 \n" \
53 " " __stringify(SC_BEQZ) " %1, 1b \n" \
54 " .set pop \n" \
55 : "=&r"(__orig), "=&r"(__temp), \
56 "+" GCC_OFF_SMALL_ASM()(mem) \
57 : inputs \
58 : __LLSC_CLOBBER); \
59 \
60 __orig; \
61})
62
63/*
64 * These are the "slower" versions of the functions and are in bitops.c.
65 * These functions call raw_local_irq_{save,restore}().
66 */
67void __mips_set_bit(unsigned long nr, volatile unsigned long *addr);
68void __mips_clear_bit(unsigned long nr, volatile unsigned long *addr);
69void __mips_change_bit(unsigned long nr, volatile unsigned long *addr);
70int __mips_test_and_set_bit_lock(unsigned long nr,
71 volatile unsigned long *addr);
72int __mips_test_and_clear_bit(unsigned long nr,
73 volatile unsigned long *addr);
74int __mips_test_and_change_bit(unsigned long nr,
75 volatile unsigned long *addr);
76
77
78/*
79 * set_bit - Atomically set a bit in memory
80 * @nr: the bit to set
81 * @addr: the address to start counting from
82 *
83 * This function is atomic and may not be reordered. See __set_bit()
84 * if you do not require the atomic guarantees.
85 * Note that @nr may be almost arbitrarily large; this function is not
86 * restricted to acting on a single-word quantity.
87 */
88static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
89{
90 volatile unsigned long *m = &addr[BIT_WORD(nr)];
91 int bit = nr % BITS_PER_LONG;
92
93 if (!kernel_uses_llsc) {
94 __mips_set_bit(nr, addr);
95 return;
96 }
97
98 if ((MIPS_ISA_REV >= 2) && __builtin_constant_p(bit) && (bit >= 16)) {
99 __bit_op(*m, __stringify(LONG_INS) " %0, %3, %2, 1", "i"(bit), "r"(~0));
100 return;
101 }
102
103 __bit_op(*m, "or\t%0, %2", "ir"(BIT(bit)));
104}
105
106/*
107 * clear_bit - Clears a bit in memory
108 * @nr: Bit to clear
109 * @addr: Address to start counting from
110 *
111 * clear_bit() is atomic and may not be reordered. However, it does
112 * not contain a memory barrier, so if it is used for locking purposes,
113 * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic()
114 * in order to ensure changes are visible on other processors.
115 */
116static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
117{
118 volatile unsigned long *m = &addr[BIT_WORD(nr)];
119 int bit = nr % BITS_PER_LONG;
120
121 if (!kernel_uses_llsc) {
122 __mips_clear_bit(nr, addr);
123 return;
124 }
125
126 if ((MIPS_ISA_REV >= 2) && __builtin_constant_p(bit)) {
127 __bit_op(*m, __stringify(LONG_INS) " %0, $0, %2, 1", "i"(bit));
128 return;
129 }
130
131 __bit_op(*m, "and\t%0, %2", "ir"(~BIT(bit)));
132}
133
134/*
135 * clear_bit_unlock - Clears a bit in memory
136 * @nr: Bit to clear
137 * @addr: Address to start counting from
138 *
139 * clear_bit() is atomic and implies release semantics before the memory
140 * operation. It can be used for an unlock.
141 */
142static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
143{
144 smp_mb__before_atomic();
145 clear_bit(nr, addr);
146}
147
148/*
149 * change_bit - Toggle a bit in memory
150 * @nr: Bit to change
151 * @addr: Address to start counting from
152 *
153 * change_bit() is atomic and may not be reordered.
154 * Note that @nr may be almost arbitrarily large; this function is not
155 * restricted to acting on a single-word quantity.
156 */
157static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
158{
159 volatile unsigned long *m = &addr[BIT_WORD(nr)];
160 int bit = nr % BITS_PER_LONG;
161
162 if (!kernel_uses_llsc) {
163 __mips_change_bit(nr, addr);
164 return;
165 }
166
167 __bit_op(*m, "xor\t%0, %2", "ir"(BIT(bit)));
168}
169
170/*
171 * test_and_set_bit_lock - Set a bit and return its old value
172 * @nr: Bit to set
173 * @addr: Address to count from
174 *
175 * This operation is atomic and implies acquire ordering semantics
176 * after the memory operation.
177 */
178static inline int test_and_set_bit_lock(unsigned long nr,
179 volatile unsigned long *addr)
180{
181 volatile unsigned long *m = &addr[BIT_WORD(nr)];
182 int bit = nr % BITS_PER_LONG;
183 unsigned long res, orig;
184
185 if (!kernel_uses_llsc) {
186 res = __mips_test_and_set_bit_lock(nr, addr);
187 } else {
188 orig = __test_bit_op(*m, "%0",
189 "or\t%1, %0, %3",
190 "ir"(BIT(bit)));
191 res = (orig & BIT(bit)) != 0;
192 }
193
194 smp_llsc_mb();
195
196 return res;
197}
198
199/*
200 * test_and_set_bit - Set a bit and return its old value
201 * @nr: Bit to set
202 * @addr: Address to count from
203 *
204 * This operation is atomic and cannot be reordered.
205 * It also implies a memory barrier.
206 */
207static inline int test_and_set_bit(unsigned long nr,
208 volatile unsigned long *addr)
209{
210 smp_mb__before_atomic();
211 return test_and_set_bit_lock(nr, addr);
212}
213
214/*
215 * test_and_clear_bit - Clear a bit and return its old value
216 * @nr: Bit to clear
217 * @addr: Address to count from
218 *
219 * This operation is atomic and cannot be reordered.
220 * It also implies a memory barrier.
221 */
222static inline int test_and_clear_bit(unsigned long nr,
223 volatile unsigned long *addr)
224{
225 volatile unsigned long *m = &addr[BIT_WORD(nr)];
226 int bit = nr % BITS_PER_LONG;
227 unsigned long res, orig;
228
229 smp_mb__before_atomic();
230
231 if (!kernel_uses_llsc) {
232 res = __mips_test_and_clear_bit(nr, addr);
233 } else if ((MIPS_ISA_REV >= 2) && __builtin_constant_p(nr)) {
234 res = __test_bit_op(*m, "%1",
235 __stringify(LONG_EXT) " %0, %1, %3, 1;"
236 __stringify(LONG_INS) " %1, $0, %3, 1",
237 "i"(bit));
238 } else {
239 orig = __test_bit_op(*m, "%0",
240 "or\t%1, %0, %3;"
241 "xor\t%1, %1, %3",
242 "ir"(BIT(bit)));
243 res = (orig & BIT(bit)) != 0;
244 }
245
246 smp_llsc_mb();
247
248 return res;
249}
250
251/*
252 * test_and_change_bit - Change a bit and return its old value
253 * @nr: Bit to change
254 * @addr: Address to count from
255 *
256 * This operation is atomic and cannot be reordered.
257 * It also implies a memory barrier.
258 */
259static inline int test_and_change_bit(unsigned long nr,
260 volatile unsigned long *addr)
261{
262 volatile unsigned long *m = &addr[BIT_WORD(nr)];
263 int bit = nr % BITS_PER_LONG;
264 unsigned long res, orig;
265
266 smp_mb__before_atomic();
267
268 if (!kernel_uses_llsc) {
269 res = __mips_test_and_change_bit(nr, addr);
270 } else {
271 orig = __test_bit_op(*m, "%0",
272 "xor\t%1, %0, %3",
273 "ir"(BIT(bit)));
274 res = (orig & BIT(bit)) != 0;
275 }
276
277 smp_llsc_mb();
278
279 return res;
280}
281
282#undef __bit_op
283#undef __test_bit_op
284
285#include <asm-generic/bitops/non-atomic.h>
286
287/*
288 * __clear_bit_unlock - Clears a bit in memory
289 * @nr: Bit to clear
290 * @addr: Address to start counting from
291 *
292 * __clear_bit() is non-atomic and implies release semantics before the memory
293 * operation. It can be used for an unlock if no other CPUs can concurrently
294 * modify other bits in the word.
295 */
296static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
297{
298 smp_mb__before_llsc();
299 __clear_bit(nr, addr);
300 nudge_writes();
301}
302
303/*
304 * Return the bit position (0..63) of the most significant 1 bit in a word
305 * Returns -1 if no 1 bit exists
306 */
307static __always_inline unsigned long __fls(unsigned long word)
308{
309 int num;
310
311 if (BITS_PER_LONG == 32 && !__builtin_constant_p(word) &&
312 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
313 __asm__(
314 " .set push \n"
315 " .set "MIPS_ISA_LEVEL" \n"
316 " clz %0, %1 \n"
317 " .set pop \n"
318 : "=r" (num)
319 : "r" (word));
320
321 return 31 - num;
322 }
323
324 if (BITS_PER_LONG == 64 && !__builtin_constant_p(word) &&
325 __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) {
326 __asm__(
327 " .set push \n"
328 " .set "MIPS_ISA_LEVEL" \n"
329 " dclz %0, %1 \n"
330 " .set pop \n"
331 : "=r" (num)
332 : "r" (word));
333
334 return 63 - num;
335 }
336
337 num = BITS_PER_LONG - 1;
338
339#if BITS_PER_LONG == 64
340 if (!(word & (~0ul << 32))) {
341 num -= 32;
342 word <<= 32;
343 }
344#endif
345 if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
346 num -= 16;
347 word <<= 16;
348 }
349 if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
350 num -= 8;
351 word <<= 8;
352 }
353 if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
354 num -= 4;
355 word <<= 4;
356 }
357 if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
358 num -= 2;
359 word <<= 2;
360 }
361 if (!(word & (~0ul << (BITS_PER_LONG-1))))
362 num -= 1;
363 return num;
364}
365
366/*
367 * __ffs - find first bit in word.
368 * @word: The word to search
369 *
370 * Returns 0..SZLONG-1
371 * Undefined if no bit exists, so code should check against 0 first.
372 */
373static __always_inline unsigned long __ffs(unsigned long word)
374{
375 return __fls(word & -word);
376}
377
378/*
379 * fls - find last bit set.
380 * @word: The word to search
381 *
382 * This is defined the same way as ffs.
383 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
384 */
385static inline int fls(unsigned int x)
386{
387 int r;
388
389 if (!__builtin_constant_p(x) &&
390 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
391 __asm__(
392 " .set push \n"
393 " .set "MIPS_ISA_LEVEL" \n"
394 " clz %0, %1 \n"
395 " .set pop \n"
396 : "=r" (x)
397 : "r" (x));
398
399 return 32 - x;
400 }
401
402 r = 32;
403 if (!x)
404 return 0;
405 if (!(x & 0xffff0000u)) {
406 x <<= 16;
407 r -= 16;
408 }
409 if (!(x & 0xff000000u)) {
410 x <<= 8;
411 r -= 8;
412 }
413 if (!(x & 0xf0000000u)) {
414 x <<= 4;
415 r -= 4;
416 }
417 if (!(x & 0xc0000000u)) {
418 x <<= 2;
419 r -= 2;
420 }
421 if (!(x & 0x80000000u)) {
422 x <<= 1;
423 r -= 1;
424 }
425 return r;
426}
427
428#include <asm-generic/bitops/fls64.h>
429
430/*
431 * ffs - find first bit set.
432 * @word: The word to search
433 *
434 * This is defined the same way as
435 * the libc and compiler builtin ffs routines, therefore
436 * differs in spirit from the below ffz (man ffs).
437 */
438static inline int ffs(int word)
439{
440 if (!word)
441 return 0;
442
443 return fls(word & -word);
444}
445
446#include <asm-generic/bitops/ffz.h>
447
448#ifdef __KERNEL__
449
450#include <asm-generic/bitops/sched.h>
451
452#include <asm/arch_hweight.h>
453#include <asm-generic/bitops/const_hweight.h>
454
455#include <asm-generic/bitops/le.h>
456#include <asm-generic/bitops/ext2-atomic.h>
457
458#endif /* __KERNEL__ */
459
460#endif /* _ASM_BITOPS_H */