Loading...
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (c) 1999, 2000 Silicon Graphics, Inc.
8 */
9#ifndef _ASM_BITOPS_H
10#define _ASM_BITOPS_H
11
12#ifndef _LINUX_BITOPS_H
13#error only <linux/bitops.h> can be included directly
14#endif
15
16#include <linux/compiler.h>
17#include <linux/types.h>
18#include <asm/barrier.h>
19#include <asm/byteorder.h> /* sigh ... */
20#include <asm/cpu-features.h>
21#include <asm/sgidefs.h>
22#include <asm/war.h>
23
24#if _MIPS_SZLONG == 32
25#define SZLONG_LOG 5
26#define SZLONG_MASK 31UL
27#define __LL "ll "
28#define __SC "sc "
29#define __INS "ins "
30#define __EXT "ext "
31#elif _MIPS_SZLONG == 64
32#define SZLONG_LOG 6
33#define SZLONG_MASK 63UL
34#define __LL "lld "
35#define __SC "scd "
36#define __INS "dins "
37#define __EXT "dext "
38#endif
39
40/*
41 * clear_bit() doesn't provide any barrier for the compiler.
42 */
43#define smp_mb__before_clear_bit() smp_mb__before_llsc()
44#define smp_mb__after_clear_bit() smp_llsc_mb()
45
46
47/*
48 * These are the "slower" versions of the functions and are in bitops.c.
49 * These functions call raw_local_irq_{save,restore}().
50 */
51void __mips_set_bit(unsigned long nr, volatile unsigned long *addr);
52void __mips_clear_bit(unsigned long nr, volatile unsigned long *addr);
53void __mips_change_bit(unsigned long nr, volatile unsigned long *addr);
54int __mips_test_and_set_bit(unsigned long nr,
55 volatile unsigned long *addr);
56int __mips_test_and_set_bit_lock(unsigned long nr,
57 volatile unsigned long *addr);
58int __mips_test_and_clear_bit(unsigned long nr,
59 volatile unsigned long *addr);
60int __mips_test_and_change_bit(unsigned long nr,
61 volatile unsigned long *addr);
62
63
64/*
65 * set_bit - Atomically set a bit in memory
66 * @nr: the bit to set
67 * @addr: the address to start counting from
68 *
69 * This function is atomic and may not be reordered. See __set_bit()
70 * if you do not require the atomic guarantees.
71 * Note that @nr may be almost arbitrarily large; this function is not
72 * restricted to acting on a single-word quantity.
73 */
74static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
75{
76 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
77 int bit = nr & SZLONG_MASK;
78 unsigned long temp;
79
80 if (kernel_uses_llsc && R10000_LLSC_WAR) {
81 __asm__ __volatile__(
82 " .set arch=r4000 \n"
83 "1: " __LL "%0, %1 # set_bit \n"
84 " or %0, %2 \n"
85 " " __SC "%0, %1 \n"
86 " beqzl %0, 1b \n"
87 " .set mips0 \n"
88 : "=&r" (temp), "=m" (*m)
89 : "ir" (1UL << bit), "m" (*m));
90#ifdef CONFIG_CPU_MIPSR2
91 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
92 do {
93 __asm__ __volatile__(
94 " " __LL "%0, %1 # set_bit \n"
95 " " __INS "%0, %3, %2, 1 \n"
96 " " __SC "%0, %1 \n"
97 : "=&r" (temp), "+m" (*m)
98 : "ir" (bit), "r" (~0));
99 } while (unlikely(!temp));
100#endif /* CONFIG_CPU_MIPSR2 */
101 } else if (kernel_uses_llsc) {
102 do {
103 __asm__ __volatile__(
104 " .set arch=r4000 \n"
105 " " __LL "%0, %1 # set_bit \n"
106 " or %0, %2 \n"
107 " " __SC "%0, %1 \n"
108 " .set mips0 \n"
109 : "=&r" (temp), "+m" (*m)
110 : "ir" (1UL << bit));
111 } while (unlikely(!temp));
112 } else
113 __mips_set_bit(nr, addr);
114}
115
116/*
117 * clear_bit - Clears a bit in memory
118 * @nr: Bit to clear
119 * @addr: Address to start counting from
120 *
121 * clear_bit() is atomic and may not be reordered. However, it does
122 * not contain a memory barrier, so if it is used for locking purposes,
123 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
124 * in order to ensure changes are visible on other processors.
125 */
126static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
127{
128 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
129 int bit = nr & SZLONG_MASK;
130 unsigned long temp;
131
132 if (kernel_uses_llsc && R10000_LLSC_WAR) {
133 __asm__ __volatile__(
134 " .set arch=r4000 \n"
135 "1: " __LL "%0, %1 # clear_bit \n"
136 " and %0, %2 \n"
137 " " __SC "%0, %1 \n"
138 " beqzl %0, 1b \n"
139 " .set mips0 \n"
140 : "=&r" (temp), "+m" (*m)
141 : "ir" (~(1UL << bit)));
142#ifdef CONFIG_CPU_MIPSR2
143 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
144 do {
145 __asm__ __volatile__(
146 " " __LL "%0, %1 # clear_bit \n"
147 " " __INS "%0, $0, %2, 1 \n"
148 " " __SC "%0, %1 \n"
149 : "=&r" (temp), "+m" (*m)
150 : "ir" (bit));
151 } while (unlikely(!temp));
152#endif /* CONFIG_CPU_MIPSR2 */
153 } else if (kernel_uses_llsc) {
154 do {
155 __asm__ __volatile__(
156 " .set arch=r4000 \n"
157 " " __LL "%0, %1 # clear_bit \n"
158 " and %0, %2 \n"
159 " " __SC "%0, %1 \n"
160 " .set mips0 \n"
161 : "=&r" (temp), "+m" (*m)
162 : "ir" (~(1UL << bit)));
163 } while (unlikely(!temp));
164 } else
165 __mips_clear_bit(nr, addr);
166}
167
168/*
169 * clear_bit_unlock - Clears a bit in memory
170 * @nr: Bit to clear
171 * @addr: Address to start counting from
172 *
173 * clear_bit() is atomic and implies release semantics before the memory
174 * operation. It can be used for an unlock.
175 */
176static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
177{
178 smp_mb__before_clear_bit();
179 clear_bit(nr, addr);
180}
181
182/*
183 * change_bit - Toggle a bit in memory
184 * @nr: Bit to change
185 * @addr: Address to start counting from
186 *
187 * change_bit() is atomic and may not be reordered.
188 * Note that @nr may be almost arbitrarily large; this function is not
189 * restricted to acting on a single-word quantity.
190 */
191static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
192{
193 int bit = nr & SZLONG_MASK;
194
195 if (kernel_uses_llsc && R10000_LLSC_WAR) {
196 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
197 unsigned long temp;
198
199 __asm__ __volatile__(
200 " .set arch=r4000 \n"
201 "1: " __LL "%0, %1 # change_bit \n"
202 " xor %0, %2 \n"
203 " " __SC "%0, %1 \n"
204 " beqzl %0, 1b \n"
205 " .set mips0 \n"
206 : "=&r" (temp), "+m" (*m)
207 : "ir" (1UL << bit));
208 } else if (kernel_uses_llsc) {
209 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
210 unsigned long temp;
211
212 do {
213 __asm__ __volatile__(
214 " .set arch=r4000 \n"
215 " " __LL "%0, %1 # change_bit \n"
216 " xor %0, %2 \n"
217 " " __SC "%0, %1 \n"
218 " .set mips0 \n"
219 : "=&r" (temp), "+m" (*m)
220 : "ir" (1UL << bit));
221 } while (unlikely(!temp));
222 } else
223 __mips_change_bit(nr, addr);
224}
225
226/*
227 * test_and_set_bit - Set a bit and return its old value
228 * @nr: Bit to set
229 * @addr: Address to count from
230 *
231 * This operation is atomic and cannot be reordered.
232 * It also implies a memory barrier.
233 */
234static inline int test_and_set_bit(unsigned long nr,
235 volatile unsigned long *addr)
236{
237 int bit = nr & SZLONG_MASK;
238 unsigned long res;
239
240 smp_mb__before_llsc();
241
242 if (kernel_uses_llsc && R10000_LLSC_WAR) {
243 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
244 unsigned long temp;
245
246 __asm__ __volatile__(
247 " .set arch=r4000 \n"
248 "1: " __LL "%0, %1 # test_and_set_bit \n"
249 " or %2, %0, %3 \n"
250 " " __SC "%2, %1 \n"
251 " beqzl %2, 1b \n"
252 " and %2, %0, %3 \n"
253 " .set mips0 \n"
254 : "=&r" (temp), "+m" (*m), "=&r" (res)
255 : "r" (1UL << bit)
256 : "memory");
257 } else if (kernel_uses_llsc) {
258 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
259 unsigned long temp;
260
261 do {
262 __asm__ __volatile__(
263 " .set arch=r4000 \n"
264 " " __LL "%0, %1 # test_and_set_bit \n"
265 " or %2, %0, %3 \n"
266 " " __SC "%2, %1 \n"
267 " .set mips0 \n"
268 : "=&r" (temp), "+m" (*m), "=&r" (res)
269 : "r" (1UL << bit)
270 : "memory");
271 } while (unlikely(!res));
272
273 res = temp & (1UL << bit);
274 } else
275 res = __mips_test_and_set_bit(nr, addr);
276
277 smp_llsc_mb();
278
279 return res != 0;
280}
281
282/*
283 * test_and_set_bit_lock - Set a bit and return its old value
284 * @nr: Bit to set
285 * @addr: Address to count from
286 *
287 * This operation is atomic and implies acquire ordering semantics
288 * after the memory operation.
289 */
290static inline int test_and_set_bit_lock(unsigned long nr,
291 volatile unsigned long *addr)
292{
293 int bit = nr & SZLONG_MASK;
294 unsigned long res;
295
296 if (kernel_uses_llsc && R10000_LLSC_WAR) {
297 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
298 unsigned long temp;
299
300 __asm__ __volatile__(
301 " .set arch=r4000 \n"
302 "1: " __LL "%0, %1 # test_and_set_bit \n"
303 " or %2, %0, %3 \n"
304 " " __SC "%2, %1 \n"
305 " beqzl %2, 1b \n"
306 " and %2, %0, %3 \n"
307 " .set mips0 \n"
308 : "=&r" (temp), "+m" (*m), "=&r" (res)
309 : "r" (1UL << bit)
310 : "memory");
311 } else if (kernel_uses_llsc) {
312 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
313 unsigned long temp;
314
315 do {
316 __asm__ __volatile__(
317 " .set arch=r4000 \n"
318 " " __LL "%0, %1 # test_and_set_bit \n"
319 " or %2, %0, %3 \n"
320 " " __SC "%2, %1 \n"
321 " .set mips0 \n"
322 : "=&r" (temp), "+m" (*m), "=&r" (res)
323 : "r" (1UL << bit)
324 : "memory");
325 } while (unlikely(!res));
326
327 res = temp & (1UL << bit);
328 } else
329 res = __mips_test_and_set_bit_lock(nr, addr);
330
331 smp_llsc_mb();
332
333 return res != 0;
334}
335/*
336 * test_and_clear_bit - Clear a bit and return its old value
337 * @nr: Bit to clear
338 * @addr: Address to count from
339 *
340 * This operation is atomic and cannot be reordered.
341 * It also implies a memory barrier.
342 */
343static inline int test_and_clear_bit(unsigned long nr,
344 volatile unsigned long *addr)
345{
346 int bit = nr & SZLONG_MASK;
347 unsigned long res;
348
349 smp_mb__before_llsc();
350
351 if (kernel_uses_llsc && R10000_LLSC_WAR) {
352 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
353 unsigned long temp;
354
355 __asm__ __volatile__(
356 " .set arch=r4000 \n"
357 "1: " __LL "%0, %1 # test_and_clear_bit \n"
358 " or %2, %0, %3 \n"
359 " xor %2, %3 \n"
360 " " __SC "%2, %1 \n"
361 " beqzl %2, 1b \n"
362 " and %2, %0, %3 \n"
363 " .set mips0 \n"
364 : "=&r" (temp), "+m" (*m), "=&r" (res)
365 : "r" (1UL << bit)
366 : "memory");
367#ifdef CONFIG_CPU_MIPSR2
368 } else if (kernel_uses_llsc && __builtin_constant_p(nr)) {
369 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
370 unsigned long temp;
371
372 do {
373 __asm__ __volatile__(
374 " " __LL "%0, %1 # test_and_clear_bit \n"
375 " " __EXT "%2, %0, %3, 1 \n"
376 " " __INS "%0, $0, %3, 1 \n"
377 " " __SC "%0, %1 \n"
378 : "=&r" (temp), "+m" (*m), "=&r" (res)
379 : "ir" (bit)
380 : "memory");
381 } while (unlikely(!temp));
382#endif
383 } else if (kernel_uses_llsc) {
384 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
385 unsigned long temp;
386
387 do {
388 __asm__ __volatile__(
389 " .set arch=r4000 \n"
390 " " __LL "%0, %1 # test_and_clear_bit \n"
391 " or %2, %0, %3 \n"
392 " xor %2, %3 \n"
393 " " __SC "%2, %1 \n"
394 " .set mips0 \n"
395 : "=&r" (temp), "+m" (*m), "=&r" (res)
396 : "r" (1UL << bit)
397 : "memory");
398 } while (unlikely(!res));
399
400 res = temp & (1UL << bit);
401 } else
402 res = __mips_test_and_clear_bit(nr, addr);
403
404 smp_llsc_mb();
405
406 return res != 0;
407}
408
409/*
410 * test_and_change_bit - Change a bit and return its old value
411 * @nr: Bit to change
412 * @addr: Address to count from
413 *
414 * This operation is atomic and cannot be reordered.
415 * It also implies a memory barrier.
416 */
417static inline int test_and_change_bit(unsigned long nr,
418 volatile unsigned long *addr)
419{
420 int bit = nr & SZLONG_MASK;
421 unsigned long res;
422
423 smp_mb__before_llsc();
424
425 if (kernel_uses_llsc && R10000_LLSC_WAR) {
426 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
427 unsigned long temp;
428
429 __asm__ __volatile__(
430 " .set arch=r4000 \n"
431 "1: " __LL "%0, %1 # test_and_change_bit \n"
432 " xor %2, %0, %3 \n"
433 " " __SC "%2, %1 \n"
434 " beqzl %2, 1b \n"
435 " and %2, %0, %3 \n"
436 " .set mips0 \n"
437 : "=&r" (temp), "+m" (*m), "=&r" (res)
438 : "r" (1UL << bit)
439 : "memory");
440 } else if (kernel_uses_llsc) {
441 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
442 unsigned long temp;
443
444 do {
445 __asm__ __volatile__(
446 " .set arch=r4000 \n"
447 " " __LL "%0, %1 # test_and_change_bit \n"
448 " xor %2, %0, %3 \n"
449 " " __SC "\t%2, %1 \n"
450 " .set mips0 \n"
451 : "=&r" (temp), "+m" (*m), "=&r" (res)
452 : "r" (1UL << bit)
453 : "memory");
454 } while (unlikely(!res));
455
456 res = temp & (1UL << bit);
457 } else
458 res = __mips_test_and_change_bit(nr, addr);
459
460 smp_llsc_mb();
461
462 return res != 0;
463}
464
465#include <asm-generic/bitops/non-atomic.h>
466
467/*
468 * __clear_bit_unlock - Clears a bit in memory
469 * @nr: Bit to clear
470 * @addr: Address to start counting from
471 *
472 * __clear_bit() is non-atomic and implies release semantics before the memory
473 * operation. It can be used for an unlock if no other CPUs can concurrently
474 * modify other bits in the word.
475 */
476static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
477{
478 smp_mb();
479 __clear_bit(nr, addr);
480}
481
482/*
483 * Return the bit position (0..63) of the most significant 1 bit in a word
484 * Returns -1 if no 1 bit exists
485 */
486static inline unsigned long __fls(unsigned long word)
487{
488 int num;
489
490 if (BITS_PER_LONG == 32 &&
491 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
492 __asm__(
493 " .set push \n"
494 " .set mips32 \n"
495 " clz %0, %1 \n"
496 " .set pop \n"
497 : "=r" (num)
498 : "r" (word));
499
500 return 31 - num;
501 }
502
503 if (BITS_PER_LONG == 64 &&
504 __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) {
505 __asm__(
506 " .set push \n"
507 " .set mips64 \n"
508 " dclz %0, %1 \n"
509 " .set pop \n"
510 : "=r" (num)
511 : "r" (word));
512
513 return 63 - num;
514 }
515
516 num = BITS_PER_LONG - 1;
517
518#if BITS_PER_LONG == 64
519 if (!(word & (~0ul << 32))) {
520 num -= 32;
521 word <<= 32;
522 }
523#endif
524 if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
525 num -= 16;
526 word <<= 16;
527 }
528 if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
529 num -= 8;
530 word <<= 8;
531 }
532 if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
533 num -= 4;
534 word <<= 4;
535 }
536 if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
537 num -= 2;
538 word <<= 2;
539 }
540 if (!(word & (~0ul << (BITS_PER_LONG-1))))
541 num -= 1;
542 return num;
543}
544
545/*
546 * __ffs - find first bit in word.
547 * @word: The word to search
548 *
549 * Returns 0..SZLONG-1
550 * Undefined if no bit exists, so code should check against 0 first.
551 */
552static inline unsigned long __ffs(unsigned long word)
553{
554 return __fls(word & -word);
555}
556
557/*
558 * fls - find last bit set.
559 * @word: The word to search
560 *
561 * This is defined the same way as ffs.
562 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
563 */
564static inline int fls(int x)
565{
566 int r;
567
568 if (__builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
569 __asm__("clz %0, %1" : "=r" (x) : "r" (x));
570
571 return 32 - x;
572 }
573
574 r = 32;
575 if (!x)
576 return 0;
577 if (!(x & 0xffff0000u)) {
578 x <<= 16;
579 r -= 16;
580 }
581 if (!(x & 0xff000000u)) {
582 x <<= 8;
583 r -= 8;
584 }
585 if (!(x & 0xf0000000u)) {
586 x <<= 4;
587 r -= 4;
588 }
589 if (!(x & 0xc0000000u)) {
590 x <<= 2;
591 r -= 2;
592 }
593 if (!(x & 0x80000000u)) {
594 x <<= 1;
595 r -= 1;
596 }
597 return r;
598}
599
600#include <asm-generic/bitops/fls64.h>
601
602/*
603 * ffs - find first bit set.
604 * @word: The word to search
605 *
606 * This is defined the same way as
607 * the libc and compiler builtin ffs routines, therefore
608 * differs in spirit from the above ffz (man ffs).
609 */
610static inline int ffs(int word)
611{
612 if (!word)
613 return 0;
614
615 return fls(word & -word);
616}
617
618#include <asm-generic/bitops/ffz.h>
619#include <asm-generic/bitops/find.h>
620
621#ifdef __KERNEL__
622
623#include <asm-generic/bitops/sched.h>
624
625#include <asm/arch_hweight.h>
626#include <asm-generic/bitops/const_hweight.h>
627
628#include <asm-generic/bitops/le.h>
629#include <asm-generic/bitops/ext2-atomic.h>
630
631#endif /* __KERNEL__ */
632
633#endif /* _ASM_BITOPS_H */
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (c) 1999, 2000 Silicon Graphics, Inc.
8 */
9#ifndef _ASM_BITOPS_H
10#define _ASM_BITOPS_H
11
12#ifndef _LINUX_BITOPS_H
13#error only <linux/bitops.h> can be included directly
14#endif
15
16#include <linux/compiler.h>
17#include <linux/irqflags.h>
18#include <linux/types.h>
19#include <asm/barrier.h>
20#include <asm/byteorder.h> /* sigh ... */
21#include <asm/cpu-features.h>
22#include <asm/sgidefs.h>
23#include <asm/war.h>
24
25#if _MIPS_SZLONG == 32
26#define SZLONG_LOG 5
27#define SZLONG_MASK 31UL
28#define __LL "ll "
29#define __SC "sc "
30#define __INS "ins "
31#define __EXT "ext "
32#elif _MIPS_SZLONG == 64
33#define SZLONG_LOG 6
34#define SZLONG_MASK 63UL
35#define __LL "lld "
36#define __SC "scd "
37#define __INS "dins "
38#define __EXT "dext "
39#endif
40
41/*
42 * clear_bit() doesn't provide any barrier for the compiler.
43 */
44#define smp_mb__before_clear_bit() smp_mb__before_llsc()
45#define smp_mb__after_clear_bit() smp_llsc_mb()
46
47/*
48 * set_bit - Atomically set a bit in memory
49 * @nr: the bit to set
50 * @addr: the address to start counting from
51 *
52 * This function is atomic and may not be reordered. See __set_bit()
53 * if you do not require the atomic guarantees.
54 * Note that @nr may be almost arbitrarily large; this function is not
55 * restricted to acting on a single-word quantity.
56 */
57static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
58{
59 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
60 unsigned short bit = nr & SZLONG_MASK;
61 unsigned long temp;
62
63 if (kernel_uses_llsc && R10000_LLSC_WAR) {
64 __asm__ __volatile__(
65 " .set mips3 \n"
66 "1: " __LL "%0, %1 # set_bit \n"
67 " or %0, %2 \n"
68 " " __SC "%0, %1 \n"
69 " beqzl %0, 1b \n"
70 " .set mips0 \n"
71 : "=&r" (temp), "=m" (*m)
72 : "ir" (1UL << bit), "m" (*m));
73#ifdef CONFIG_CPU_MIPSR2
74 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
75 do {
76 __asm__ __volatile__(
77 " " __LL "%0, %1 # set_bit \n"
78 " " __INS "%0, %3, %2, 1 \n"
79 " " __SC "%0, %1 \n"
80 : "=&r" (temp), "+m" (*m)
81 : "ir" (bit), "r" (~0));
82 } while (unlikely(!temp));
83#endif /* CONFIG_CPU_MIPSR2 */
84 } else if (kernel_uses_llsc) {
85 do {
86 __asm__ __volatile__(
87 " .set mips3 \n"
88 " " __LL "%0, %1 # set_bit \n"
89 " or %0, %2 \n"
90 " " __SC "%0, %1 \n"
91 " .set mips0 \n"
92 : "=&r" (temp), "+m" (*m)
93 : "ir" (1UL << bit));
94 } while (unlikely(!temp));
95 } else {
96 volatile unsigned long *a = addr;
97 unsigned long mask;
98 unsigned long flags;
99
100 a += nr >> SZLONG_LOG;
101 mask = 1UL << bit;
102 raw_local_irq_save(flags);
103 *a |= mask;
104 raw_local_irq_restore(flags);
105 }
106}
107
108/*
109 * clear_bit - Clears a bit in memory
110 * @nr: Bit to clear
111 * @addr: Address to start counting from
112 *
113 * clear_bit() is atomic and may not be reordered. However, it does
114 * not contain a memory barrier, so if it is used for locking purposes,
115 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
116 * in order to ensure changes are visible on other processors.
117 */
118static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
119{
120 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
121 unsigned short bit = nr & SZLONG_MASK;
122 unsigned long temp;
123
124 if (kernel_uses_llsc && R10000_LLSC_WAR) {
125 __asm__ __volatile__(
126 " .set mips3 \n"
127 "1: " __LL "%0, %1 # clear_bit \n"
128 " and %0, %2 \n"
129 " " __SC "%0, %1 \n"
130 " beqzl %0, 1b \n"
131 " .set mips0 \n"
132 : "=&r" (temp), "+m" (*m)
133 : "ir" (~(1UL << bit)));
134#ifdef CONFIG_CPU_MIPSR2
135 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
136 do {
137 __asm__ __volatile__(
138 " " __LL "%0, %1 # clear_bit \n"
139 " " __INS "%0, $0, %2, 1 \n"
140 " " __SC "%0, %1 \n"
141 : "=&r" (temp), "+m" (*m)
142 : "ir" (bit));
143 } while (unlikely(!temp));
144#endif /* CONFIG_CPU_MIPSR2 */
145 } else if (kernel_uses_llsc) {
146 do {
147 __asm__ __volatile__(
148 " .set mips3 \n"
149 " " __LL "%0, %1 # clear_bit \n"
150 " and %0, %2 \n"
151 " " __SC "%0, %1 \n"
152 " .set mips0 \n"
153 : "=&r" (temp), "+m" (*m)
154 : "ir" (~(1UL << bit)));
155 } while (unlikely(!temp));
156 } else {
157 volatile unsigned long *a = addr;
158 unsigned long mask;
159 unsigned long flags;
160
161 a += nr >> SZLONG_LOG;
162 mask = 1UL << bit;
163 raw_local_irq_save(flags);
164 *a &= ~mask;
165 raw_local_irq_restore(flags);
166 }
167}
168
169/*
170 * clear_bit_unlock - Clears a bit in memory
171 * @nr: Bit to clear
172 * @addr: Address to start counting from
173 *
174 * clear_bit() is atomic and implies release semantics before the memory
175 * operation. It can be used for an unlock.
176 */
177static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
178{
179 smp_mb__before_clear_bit();
180 clear_bit(nr, addr);
181}
182
183/*
184 * change_bit - Toggle a bit in memory
185 * @nr: Bit to change
186 * @addr: Address to start counting from
187 *
188 * change_bit() is atomic and may not be reordered.
189 * Note that @nr may be almost arbitrarily large; this function is not
190 * restricted to acting on a single-word quantity.
191 */
192static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
193{
194 unsigned short bit = nr & SZLONG_MASK;
195
196 if (kernel_uses_llsc && R10000_LLSC_WAR) {
197 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
198 unsigned long temp;
199
200 __asm__ __volatile__(
201 " .set mips3 \n"
202 "1: " __LL "%0, %1 # change_bit \n"
203 " xor %0, %2 \n"
204 " " __SC "%0, %1 \n"
205 " beqzl %0, 1b \n"
206 " .set mips0 \n"
207 : "=&r" (temp), "+m" (*m)
208 : "ir" (1UL << bit));
209 } else if (kernel_uses_llsc) {
210 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
211 unsigned long temp;
212
213 do {
214 __asm__ __volatile__(
215 " .set mips3 \n"
216 " " __LL "%0, %1 # change_bit \n"
217 " xor %0, %2 \n"
218 " " __SC "%0, %1 \n"
219 " .set mips0 \n"
220 : "=&r" (temp), "+m" (*m)
221 : "ir" (1UL << bit));
222 } while (unlikely(!temp));
223 } else {
224 volatile unsigned long *a = addr;
225 unsigned long mask;
226 unsigned long flags;
227
228 a += nr >> SZLONG_LOG;
229 mask = 1UL << bit;
230 raw_local_irq_save(flags);
231 *a ^= mask;
232 raw_local_irq_restore(flags);
233 }
234}
235
236/*
237 * test_and_set_bit - Set a bit and return its old value
238 * @nr: Bit to set
239 * @addr: Address to count from
240 *
241 * This operation is atomic and cannot be reordered.
242 * It also implies a memory barrier.
243 */
244static inline int test_and_set_bit(unsigned long nr,
245 volatile unsigned long *addr)
246{
247 unsigned short bit = nr & SZLONG_MASK;
248 unsigned long res;
249
250 smp_mb__before_llsc();
251
252 if (kernel_uses_llsc && R10000_LLSC_WAR) {
253 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
254 unsigned long temp;
255
256 __asm__ __volatile__(
257 " .set mips3 \n"
258 "1: " __LL "%0, %1 # test_and_set_bit \n"
259 " or %2, %0, %3 \n"
260 " " __SC "%2, %1 \n"
261 " beqzl %2, 1b \n"
262 " and %2, %0, %3 \n"
263 " .set mips0 \n"
264 : "=&r" (temp), "+m" (*m), "=&r" (res)
265 : "r" (1UL << bit)
266 : "memory");
267 } else if (kernel_uses_llsc) {
268 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
269 unsigned long temp;
270
271 do {
272 __asm__ __volatile__(
273 " .set mips3 \n"
274 " " __LL "%0, %1 # test_and_set_bit \n"
275 " or %2, %0, %3 \n"
276 " " __SC "%2, %1 \n"
277 " .set mips0 \n"
278 : "=&r" (temp), "+m" (*m), "=&r" (res)
279 : "r" (1UL << bit)
280 : "memory");
281 } while (unlikely(!res));
282
283 res = temp & (1UL << bit);
284 } else {
285 volatile unsigned long *a = addr;
286 unsigned long mask;
287 unsigned long flags;
288
289 a += nr >> SZLONG_LOG;
290 mask = 1UL << bit;
291 raw_local_irq_save(flags);
292 res = (mask & *a);
293 *a |= mask;
294 raw_local_irq_restore(flags);
295 }
296
297 smp_llsc_mb();
298
299 return res != 0;
300}
301
302/*
303 * test_and_set_bit_lock - Set a bit and return its old value
304 * @nr: Bit to set
305 * @addr: Address to count from
306 *
307 * This operation is atomic and implies acquire ordering semantics
308 * after the memory operation.
309 */
310static inline int test_and_set_bit_lock(unsigned long nr,
311 volatile unsigned long *addr)
312{
313 unsigned short bit = nr & SZLONG_MASK;
314 unsigned long res;
315
316 if (kernel_uses_llsc && R10000_LLSC_WAR) {
317 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
318 unsigned long temp;
319
320 __asm__ __volatile__(
321 " .set mips3 \n"
322 "1: " __LL "%0, %1 # test_and_set_bit \n"
323 " or %2, %0, %3 \n"
324 " " __SC "%2, %1 \n"
325 " beqzl %2, 1b \n"
326 " and %2, %0, %3 \n"
327 " .set mips0 \n"
328 : "=&r" (temp), "+m" (*m), "=&r" (res)
329 : "r" (1UL << bit)
330 : "memory");
331 } else if (kernel_uses_llsc) {
332 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
333 unsigned long temp;
334
335 do {
336 __asm__ __volatile__(
337 " .set mips3 \n"
338 " " __LL "%0, %1 # test_and_set_bit \n"
339 " or %2, %0, %3 \n"
340 " " __SC "%2, %1 \n"
341 " .set mips0 \n"
342 : "=&r" (temp), "+m" (*m), "=&r" (res)
343 : "r" (1UL << bit)
344 : "memory");
345 } while (unlikely(!res));
346
347 res = temp & (1UL << bit);
348 } else {
349 volatile unsigned long *a = addr;
350 unsigned long mask;
351 unsigned long flags;
352
353 a += nr >> SZLONG_LOG;
354 mask = 1UL << bit;
355 raw_local_irq_save(flags);
356 res = (mask & *a);
357 *a |= mask;
358 raw_local_irq_restore(flags);
359 }
360
361 smp_llsc_mb();
362
363 return res != 0;
364}
365/*
366 * test_and_clear_bit - Clear a bit and return its old value
367 * @nr: Bit to clear
368 * @addr: Address to count from
369 *
370 * This operation is atomic and cannot be reordered.
371 * It also implies a memory barrier.
372 */
373static inline int test_and_clear_bit(unsigned long nr,
374 volatile unsigned long *addr)
375{
376 unsigned short bit = nr & SZLONG_MASK;
377 unsigned long res;
378
379 smp_mb__before_llsc();
380
381 if (kernel_uses_llsc && R10000_LLSC_WAR) {
382 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
383 unsigned long temp;
384
385 __asm__ __volatile__(
386 " .set mips3 \n"
387 "1: " __LL "%0, %1 # test_and_clear_bit \n"
388 " or %2, %0, %3 \n"
389 " xor %2, %3 \n"
390 " " __SC "%2, %1 \n"
391 " beqzl %2, 1b \n"
392 " and %2, %0, %3 \n"
393 " .set mips0 \n"
394 : "=&r" (temp), "+m" (*m), "=&r" (res)
395 : "r" (1UL << bit)
396 : "memory");
397#ifdef CONFIG_CPU_MIPSR2
398 } else if (kernel_uses_llsc && __builtin_constant_p(nr)) {
399 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
400 unsigned long temp;
401
402 do {
403 __asm__ __volatile__(
404 " " __LL "%0, %1 # test_and_clear_bit \n"
405 " " __EXT "%2, %0, %3, 1 \n"
406 " " __INS "%0, $0, %3, 1 \n"
407 " " __SC "%0, %1 \n"
408 : "=&r" (temp), "+m" (*m), "=&r" (res)
409 : "ir" (bit)
410 : "memory");
411 } while (unlikely(!temp));
412#endif
413 } else if (kernel_uses_llsc) {
414 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
415 unsigned long temp;
416
417 do {
418 __asm__ __volatile__(
419 " .set mips3 \n"
420 " " __LL "%0, %1 # test_and_clear_bit \n"
421 " or %2, %0, %3 \n"
422 " xor %2, %3 \n"
423 " " __SC "%2, %1 \n"
424 " .set mips0 \n"
425 : "=&r" (temp), "+m" (*m), "=&r" (res)
426 : "r" (1UL << bit)
427 : "memory");
428 } while (unlikely(!res));
429
430 res = temp & (1UL << bit);
431 } else {
432 volatile unsigned long *a = addr;
433 unsigned long mask;
434 unsigned long flags;
435
436 a += nr >> SZLONG_LOG;
437 mask = 1UL << bit;
438 raw_local_irq_save(flags);
439 res = (mask & *a);
440 *a &= ~mask;
441 raw_local_irq_restore(flags);
442 }
443
444 smp_llsc_mb();
445
446 return res != 0;
447}
448
449/*
450 * test_and_change_bit - Change a bit and return its old value
451 * @nr: Bit to change
452 * @addr: Address to count from
453 *
454 * This operation is atomic and cannot be reordered.
455 * It also implies a memory barrier.
456 */
457static inline int test_and_change_bit(unsigned long nr,
458 volatile unsigned long *addr)
459{
460 unsigned short bit = nr & SZLONG_MASK;
461 unsigned long res;
462
463 smp_mb__before_llsc();
464
465 if (kernel_uses_llsc && R10000_LLSC_WAR) {
466 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
467 unsigned long temp;
468
469 __asm__ __volatile__(
470 " .set mips3 \n"
471 "1: " __LL "%0, %1 # test_and_change_bit \n"
472 " xor %2, %0, %3 \n"
473 " " __SC "%2, %1 \n"
474 " beqzl %2, 1b \n"
475 " and %2, %0, %3 \n"
476 " .set mips0 \n"
477 : "=&r" (temp), "+m" (*m), "=&r" (res)
478 : "r" (1UL << bit)
479 : "memory");
480 } else if (kernel_uses_llsc) {
481 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
482 unsigned long temp;
483
484 do {
485 __asm__ __volatile__(
486 " .set mips3 \n"
487 " " __LL "%0, %1 # test_and_change_bit \n"
488 " xor %2, %0, %3 \n"
489 " " __SC "\t%2, %1 \n"
490 " .set mips0 \n"
491 : "=&r" (temp), "+m" (*m), "=&r" (res)
492 : "r" (1UL << bit)
493 : "memory");
494 } while (unlikely(!res));
495
496 res = temp & (1UL << bit);
497 } else {
498 volatile unsigned long *a = addr;
499 unsigned long mask;
500 unsigned long flags;
501
502 a += nr >> SZLONG_LOG;
503 mask = 1UL << bit;
504 raw_local_irq_save(flags);
505 res = (mask & *a);
506 *a ^= mask;
507 raw_local_irq_restore(flags);
508 }
509
510 smp_llsc_mb();
511
512 return res != 0;
513}
514
515#include <asm-generic/bitops/non-atomic.h>
516
517/*
518 * __clear_bit_unlock - Clears a bit in memory
519 * @nr: Bit to clear
520 * @addr: Address to start counting from
521 *
522 * __clear_bit() is non-atomic and implies release semantics before the memory
523 * operation. It can be used for an unlock if no other CPUs can concurrently
524 * modify other bits in the word.
525 */
526static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
527{
528 smp_mb();
529 __clear_bit(nr, addr);
530}
531
532/*
533 * Return the bit position (0..63) of the most significant 1 bit in a word
534 * Returns -1 if no 1 bit exists
535 */
536static inline unsigned long __fls(unsigned long word)
537{
538 int num;
539
540 if (BITS_PER_LONG == 32 &&
541 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
542 __asm__(
543 " .set push \n"
544 " .set mips32 \n"
545 " clz %0, %1 \n"
546 " .set pop \n"
547 : "=r" (num)
548 : "r" (word));
549
550 return 31 - num;
551 }
552
553 if (BITS_PER_LONG == 64 &&
554 __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) {
555 __asm__(
556 " .set push \n"
557 " .set mips64 \n"
558 " dclz %0, %1 \n"
559 " .set pop \n"
560 : "=r" (num)
561 : "r" (word));
562
563 return 63 - num;
564 }
565
566 num = BITS_PER_LONG - 1;
567
568#if BITS_PER_LONG == 64
569 if (!(word & (~0ul << 32))) {
570 num -= 32;
571 word <<= 32;
572 }
573#endif
574 if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
575 num -= 16;
576 word <<= 16;
577 }
578 if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
579 num -= 8;
580 word <<= 8;
581 }
582 if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
583 num -= 4;
584 word <<= 4;
585 }
586 if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
587 num -= 2;
588 word <<= 2;
589 }
590 if (!(word & (~0ul << (BITS_PER_LONG-1))))
591 num -= 1;
592 return num;
593}
594
595/*
596 * __ffs - find first bit in word.
597 * @word: The word to search
598 *
599 * Returns 0..SZLONG-1
600 * Undefined if no bit exists, so code should check against 0 first.
601 */
602static inline unsigned long __ffs(unsigned long word)
603{
604 return __fls(word & -word);
605}
606
607/*
608 * fls - find last bit set.
609 * @word: The word to search
610 *
611 * This is defined the same way as ffs.
612 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
613 */
614static inline int fls(int x)
615{
616 int r;
617
618 if (__builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
619 __asm__("clz %0, %1" : "=r" (x) : "r" (x));
620
621 return 32 - x;
622 }
623
624 r = 32;
625 if (!x)
626 return 0;
627 if (!(x & 0xffff0000u)) {
628 x <<= 16;
629 r -= 16;
630 }
631 if (!(x & 0xff000000u)) {
632 x <<= 8;
633 r -= 8;
634 }
635 if (!(x & 0xf0000000u)) {
636 x <<= 4;
637 r -= 4;
638 }
639 if (!(x & 0xc0000000u)) {
640 x <<= 2;
641 r -= 2;
642 }
643 if (!(x & 0x80000000u)) {
644 x <<= 1;
645 r -= 1;
646 }
647 return r;
648}
649
650#include <asm-generic/bitops/fls64.h>
651
652/*
653 * ffs - find first bit set.
654 * @word: The word to search
655 *
656 * This is defined the same way as
657 * the libc and compiler builtin ffs routines, therefore
658 * differs in spirit from the above ffz (man ffs).
659 */
660static inline int ffs(int word)
661{
662 if (!word)
663 return 0;
664
665 return fls(word & -word);
666}
667
668#include <asm-generic/bitops/ffz.h>
669#include <asm-generic/bitops/find.h>
670
671#ifdef __KERNEL__
672
673#include <asm-generic/bitops/sched.h>
674
675#include <asm/arch_hweight.h>
676#include <asm-generic/bitops/const_hweight.h>
677
678#include <asm-generic/bitops/le.h>
679#include <asm-generic/bitops/ext2-atomic.h>
680
681#endif /* __KERNEL__ */
682
683#endif /* _ASM_BITOPS_H */