Loading...
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (c) 1999, 2000 Silicon Graphics, Inc.
8 */
9#ifndef _ASM_BITOPS_H
10#define _ASM_BITOPS_H
11
12#ifndef _LINUX_BITOPS_H
13#error only <linux/bitops.h> can be included directly
14#endif
15
16#include <linux/bits.h>
17#include <linux/compiler.h>
18#include <linux/types.h>
19#include <asm/barrier.h>
20#include <asm/byteorder.h> /* sigh ... */
21#include <asm/compiler.h>
22#include <asm/cpu-features.h>
23#include <asm/isa-rev.h>
24#include <asm/llsc.h>
25#include <asm/sgidefs.h>
26#include <asm/war.h>
27
28#define __bit_op(mem, insn, inputs...) do { \
29 unsigned long temp; \
30 \
31 asm volatile( \
32 " .set push \n" \
33 " .set " MIPS_ISA_LEVEL " \n" \
34 " " __SYNC(full, loongson3_war) " \n" \
35 "1: " __LL "%0, %1 \n" \
36 " " insn " \n" \
37 " " __SC "%0, %1 \n" \
38 " " __SC_BEQZ "%0, 1b \n" \
39 " .set pop \n" \
40 : "=&r"(temp), "+" GCC_OFF_SMALL_ASM()(mem) \
41 : inputs \
42 : __LLSC_CLOBBER); \
43} while (0)
44
45#define __test_bit_op(mem, ll_dst, insn, inputs...) ({ \
46 unsigned long orig, temp; \
47 \
48 asm volatile( \
49 " .set push \n" \
50 " .set " MIPS_ISA_LEVEL " \n" \
51 " " __SYNC(full, loongson3_war) " \n" \
52 "1: " __LL ll_dst ", %2 \n" \
53 " " insn " \n" \
54 " " __SC "%1, %2 \n" \
55 " " __SC_BEQZ "%1, 1b \n" \
56 " .set pop \n" \
57 : "=&r"(orig), "=&r"(temp), \
58 "+" GCC_OFF_SMALL_ASM()(mem) \
59 : inputs \
60 : __LLSC_CLOBBER); \
61 \
62 orig; \
63})
64
65/*
66 * These are the "slower" versions of the functions and are in bitops.c.
67 * These functions call raw_local_irq_{save,restore}().
68 */
69void __mips_set_bit(unsigned long nr, volatile unsigned long *addr);
70void __mips_clear_bit(unsigned long nr, volatile unsigned long *addr);
71void __mips_change_bit(unsigned long nr, volatile unsigned long *addr);
72int __mips_test_and_set_bit_lock(unsigned long nr,
73 volatile unsigned long *addr);
74int __mips_test_and_clear_bit(unsigned long nr,
75 volatile unsigned long *addr);
76int __mips_test_and_change_bit(unsigned long nr,
77 volatile unsigned long *addr);
78
79
80/*
81 * set_bit - Atomically set a bit in memory
82 * @nr: the bit to set
83 * @addr: the address to start counting from
84 *
85 * This function is atomic and may not be reordered. See __set_bit()
86 * if you do not require the atomic guarantees.
87 * Note that @nr may be almost arbitrarily large; this function is not
88 * restricted to acting on a single-word quantity.
89 */
90static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
91{
92 volatile unsigned long *m = &addr[BIT_WORD(nr)];
93 int bit = nr % BITS_PER_LONG;
94
95 if (!kernel_uses_llsc) {
96 __mips_set_bit(nr, addr);
97 return;
98 }
99
100 if ((MIPS_ISA_REV >= 2) && __builtin_constant_p(bit) && (bit >= 16)) {
101 __bit_op(*m, __INS "%0, %3, %2, 1", "i"(bit), "r"(~0));
102 return;
103 }
104
105 __bit_op(*m, "or\t%0, %2", "ir"(BIT(bit)));
106}
107
108/*
109 * clear_bit - Clears a bit in memory
110 * @nr: Bit to clear
111 * @addr: Address to start counting from
112 *
113 * clear_bit() is atomic and may not be reordered. However, it does
114 * not contain a memory barrier, so if it is used for locking purposes,
115 * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic()
116 * in order to ensure changes are visible on other processors.
117 */
118static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
119{
120 volatile unsigned long *m = &addr[BIT_WORD(nr)];
121 int bit = nr % BITS_PER_LONG;
122
123 if (!kernel_uses_llsc) {
124 __mips_clear_bit(nr, addr);
125 return;
126 }
127
128 if ((MIPS_ISA_REV >= 2) && __builtin_constant_p(bit)) {
129 __bit_op(*m, __INS "%0, $0, %2, 1", "i"(bit));
130 return;
131 }
132
133 __bit_op(*m, "and\t%0, %2", "ir"(~BIT(bit)));
134}
135
136/*
137 * clear_bit_unlock - Clears a bit in memory
138 * @nr: Bit to clear
139 * @addr: Address to start counting from
140 *
141 * clear_bit() is atomic and implies release semantics before the memory
142 * operation. It can be used for an unlock.
143 */
144static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
145{
146 smp_mb__before_atomic();
147 clear_bit(nr, addr);
148}
149
150/*
151 * change_bit - Toggle a bit in memory
152 * @nr: Bit to change
153 * @addr: Address to start counting from
154 *
155 * change_bit() is atomic and may not be reordered.
156 * Note that @nr may be almost arbitrarily large; this function is not
157 * restricted to acting on a single-word quantity.
158 */
159static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
160{
161 volatile unsigned long *m = &addr[BIT_WORD(nr)];
162 int bit = nr % BITS_PER_LONG;
163
164 if (!kernel_uses_llsc) {
165 __mips_change_bit(nr, addr);
166 return;
167 }
168
169 __bit_op(*m, "xor\t%0, %2", "ir"(BIT(bit)));
170}
171
172/*
173 * test_and_set_bit_lock - Set a bit and return its old value
174 * @nr: Bit to set
175 * @addr: Address to count from
176 *
177 * This operation is atomic and implies acquire ordering semantics
178 * after the memory operation.
179 */
180static inline int test_and_set_bit_lock(unsigned long nr,
181 volatile unsigned long *addr)
182{
183 volatile unsigned long *m = &addr[BIT_WORD(nr)];
184 int bit = nr % BITS_PER_LONG;
185 unsigned long res, orig;
186
187 if (!kernel_uses_llsc) {
188 res = __mips_test_and_set_bit_lock(nr, addr);
189 } else {
190 orig = __test_bit_op(*m, "%0",
191 "or\t%1, %0, %3",
192 "ir"(BIT(bit)));
193 res = (orig & BIT(bit)) != 0;
194 }
195
196 smp_llsc_mb();
197
198 return res;
199}
200
201/*
202 * test_and_set_bit - Set a bit and return its old value
203 * @nr: Bit to set
204 * @addr: Address to count from
205 *
206 * This operation is atomic and cannot be reordered.
207 * It also implies a memory barrier.
208 */
209static inline int test_and_set_bit(unsigned long nr,
210 volatile unsigned long *addr)
211{
212 smp_mb__before_atomic();
213 return test_and_set_bit_lock(nr, addr);
214}
215
216/*
217 * test_and_clear_bit - Clear a bit and return its old value
218 * @nr: Bit to clear
219 * @addr: Address to count from
220 *
221 * This operation is atomic and cannot be reordered.
222 * It also implies a memory barrier.
223 */
224static inline int test_and_clear_bit(unsigned long nr,
225 volatile unsigned long *addr)
226{
227 volatile unsigned long *m = &addr[BIT_WORD(nr)];
228 int bit = nr % BITS_PER_LONG;
229 unsigned long res, orig;
230
231 smp_mb__before_atomic();
232
233 if (!kernel_uses_llsc) {
234 res = __mips_test_and_clear_bit(nr, addr);
235 } else if ((MIPS_ISA_REV >= 2) && __builtin_constant_p(nr)) {
236 res = __test_bit_op(*m, "%1",
237 __EXT "%0, %1, %3, 1;"
238 __INS "%1, $0, %3, 1",
239 "i"(bit));
240 } else {
241 orig = __test_bit_op(*m, "%0",
242 "or\t%1, %0, %3;"
243 "xor\t%1, %1, %3",
244 "ir"(BIT(bit)));
245 res = (orig & BIT(bit)) != 0;
246 }
247
248 smp_llsc_mb();
249
250 return res;
251}
252
253/*
254 * test_and_change_bit - Change a bit and return its old value
255 * @nr: Bit to change
256 * @addr: Address to count from
257 *
258 * This operation is atomic and cannot be reordered.
259 * It also implies a memory barrier.
260 */
261static inline int test_and_change_bit(unsigned long nr,
262 volatile unsigned long *addr)
263{
264 volatile unsigned long *m = &addr[BIT_WORD(nr)];
265 int bit = nr % BITS_PER_LONG;
266 unsigned long res, orig;
267
268 smp_mb__before_atomic();
269
270 if (!kernel_uses_llsc) {
271 res = __mips_test_and_change_bit(nr, addr);
272 } else {
273 orig = __test_bit_op(*m, "%0",
274 "xor\t%1, %0, %3",
275 "ir"(BIT(bit)));
276 res = (orig & BIT(bit)) != 0;
277 }
278
279 smp_llsc_mb();
280
281 return res;
282}
283
284#undef __bit_op
285#undef __test_bit_op
286
287#include <asm-generic/bitops/non-atomic.h>
288
289/*
290 * __clear_bit_unlock - Clears a bit in memory
291 * @nr: Bit to clear
292 * @addr: Address to start counting from
293 *
294 * __clear_bit() is non-atomic and implies release semantics before the memory
295 * operation. It can be used for an unlock if no other CPUs can concurrently
296 * modify other bits in the word.
297 */
298static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
299{
300 smp_mb__before_llsc();
301 __clear_bit(nr, addr);
302 nudge_writes();
303}
304
305/*
306 * Return the bit position (0..63) of the most significant 1 bit in a word
307 * Returns -1 if no 1 bit exists
308 */
309static __always_inline unsigned long __fls(unsigned long word)
310{
311 int num;
312
313 if (BITS_PER_LONG == 32 && !__builtin_constant_p(word) &&
314 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
315 __asm__(
316 " .set push \n"
317 " .set "MIPS_ISA_LEVEL" \n"
318 " clz %0, %1 \n"
319 " .set pop \n"
320 : "=r" (num)
321 : "r" (word));
322
323 return 31 - num;
324 }
325
326 if (BITS_PER_LONG == 64 && !__builtin_constant_p(word) &&
327 __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) {
328 __asm__(
329 " .set push \n"
330 " .set "MIPS_ISA_LEVEL" \n"
331 " dclz %0, %1 \n"
332 " .set pop \n"
333 : "=r" (num)
334 : "r" (word));
335
336 return 63 - num;
337 }
338
339 num = BITS_PER_LONG - 1;
340
341#if BITS_PER_LONG == 64
342 if (!(word & (~0ul << 32))) {
343 num -= 32;
344 word <<= 32;
345 }
346#endif
347 if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
348 num -= 16;
349 word <<= 16;
350 }
351 if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
352 num -= 8;
353 word <<= 8;
354 }
355 if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
356 num -= 4;
357 word <<= 4;
358 }
359 if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
360 num -= 2;
361 word <<= 2;
362 }
363 if (!(word & (~0ul << (BITS_PER_LONG-1))))
364 num -= 1;
365 return num;
366}
367
368/*
369 * __ffs - find first bit in word.
370 * @word: The word to search
371 *
372 * Returns 0..SZLONG-1
373 * Undefined if no bit exists, so code should check against 0 first.
374 */
375static __always_inline unsigned long __ffs(unsigned long word)
376{
377 return __fls(word & -word);
378}
379
380/*
381 * fls - find last bit set.
382 * @word: The word to search
383 *
384 * This is defined the same way as ffs.
385 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
386 */
387static inline int fls(unsigned int x)
388{
389 int r;
390
391 if (!__builtin_constant_p(x) &&
392 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
393 __asm__(
394 " .set push \n"
395 " .set "MIPS_ISA_LEVEL" \n"
396 " clz %0, %1 \n"
397 " .set pop \n"
398 : "=r" (x)
399 : "r" (x));
400
401 return 32 - x;
402 }
403
404 r = 32;
405 if (!x)
406 return 0;
407 if (!(x & 0xffff0000u)) {
408 x <<= 16;
409 r -= 16;
410 }
411 if (!(x & 0xff000000u)) {
412 x <<= 8;
413 r -= 8;
414 }
415 if (!(x & 0xf0000000u)) {
416 x <<= 4;
417 r -= 4;
418 }
419 if (!(x & 0xc0000000u)) {
420 x <<= 2;
421 r -= 2;
422 }
423 if (!(x & 0x80000000u)) {
424 x <<= 1;
425 r -= 1;
426 }
427 return r;
428}
429
430#include <asm-generic/bitops/fls64.h>
431
432/*
433 * ffs - find first bit set.
434 * @word: The word to search
435 *
436 * This is defined the same way as
437 * the libc and compiler builtin ffs routines, therefore
438 * differs in spirit from the above ffz (man ffs).
439 */
440static inline int ffs(int word)
441{
442 if (!word)
443 return 0;
444
445 return fls(word & -word);
446}
447
448#include <asm-generic/bitops/ffz.h>
449#include <asm-generic/bitops/find.h>
450
451#ifdef __KERNEL__
452
453#include <asm-generic/bitops/sched.h>
454
455#include <asm/arch_hweight.h>
456#include <asm-generic/bitops/const_hweight.h>
457
458#include <asm-generic/bitops/le.h>
459#include <asm-generic/bitops/ext2-atomic.h>
460
461#endif /* __KERNEL__ */
462
463#endif /* _ASM_BITOPS_H */
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (c) 1999, 2000 Silicon Graphics, Inc.
8 */
9#ifndef _ASM_BITOPS_H
10#define _ASM_BITOPS_H
11
12#ifndef _LINUX_BITOPS_H
13#error only <linux/bitops.h> can be included directly
14#endif
15
16#include <linux/compiler.h>
17#include <linux/irqflags.h>
18#include <linux/types.h>
19#include <asm/barrier.h>
20#include <asm/bug.h>
21#include <asm/byteorder.h> /* sigh ... */
22#include <asm/cpu-features.h>
23#include <asm/sgidefs.h>
24#include <asm/war.h>
25
26#if _MIPS_SZLONG == 32
27#define SZLONG_LOG 5
28#define SZLONG_MASK 31UL
29#define __LL "ll "
30#define __SC "sc "
31#define __INS "ins "
32#define __EXT "ext "
33#elif _MIPS_SZLONG == 64
34#define SZLONG_LOG 6
35#define SZLONG_MASK 63UL
36#define __LL "lld "
37#define __SC "scd "
38#define __INS "dins "
39#define __EXT "dext "
40#endif
41
42/*
43 * clear_bit() doesn't provide any barrier for the compiler.
44 */
45#define smp_mb__before_clear_bit() smp_mb__before_llsc()
46#define smp_mb__after_clear_bit() smp_llsc_mb()
47
48/*
49 * set_bit - Atomically set a bit in memory
50 * @nr: the bit to set
51 * @addr: the address to start counting from
52 *
53 * This function is atomic and may not be reordered. See __set_bit()
54 * if you do not require the atomic guarantees.
55 * Note that @nr may be almost arbitrarily large; this function is not
56 * restricted to acting on a single-word quantity.
57 */
58static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
59{
60 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
61 unsigned short bit = nr & SZLONG_MASK;
62 unsigned long temp;
63
64 if (kernel_uses_llsc && R10000_LLSC_WAR) {
65 __asm__ __volatile__(
66 " .set mips3 \n"
67 "1: " __LL "%0, %1 # set_bit \n"
68 " or %0, %2 \n"
69 " " __SC "%0, %1 \n"
70 " beqzl %0, 1b \n"
71 " .set mips0 \n"
72 : "=&r" (temp), "=m" (*m)
73 : "ir" (1UL << bit), "m" (*m));
74#ifdef CONFIG_CPU_MIPSR2
75 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
76 do {
77 __asm__ __volatile__(
78 " " __LL "%0, %1 # set_bit \n"
79 " " __INS "%0, %3, %2, 1 \n"
80 " " __SC "%0, %1 \n"
81 : "=&r" (temp), "+m" (*m)
82 : "ir" (bit), "r" (~0));
83 } while (unlikely(!temp));
84#endif /* CONFIG_CPU_MIPSR2 */
85 } else if (kernel_uses_llsc) {
86 do {
87 __asm__ __volatile__(
88 " .set mips3 \n"
89 " " __LL "%0, %1 # set_bit \n"
90 " or %0, %2 \n"
91 " " __SC "%0, %1 \n"
92 " .set mips0 \n"
93 : "=&r" (temp), "+m" (*m)
94 : "ir" (1UL << bit));
95 } while (unlikely(!temp));
96 } else {
97 volatile unsigned long *a = addr;
98 unsigned long mask;
99 unsigned long flags;
100
101 a += nr >> SZLONG_LOG;
102 mask = 1UL << bit;
103 raw_local_irq_save(flags);
104 *a |= mask;
105 raw_local_irq_restore(flags);
106 }
107}
108
109/*
110 * clear_bit - Clears a bit in memory
111 * @nr: Bit to clear
112 * @addr: Address to start counting from
113 *
114 * clear_bit() is atomic and may not be reordered. However, it does
115 * not contain a memory barrier, so if it is used for locking purposes,
116 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
117 * in order to ensure changes are visible on other processors.
118 */
119static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
120{
121 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
122 unsigned short bit = nr & SZLONG_MASK;
123 unsigned long temp;
124
125 if (kernel_uses_llsc && R10000_LLSC_WAR) {
126 __asm__ __volatile__(
127 " .set mips3 \n"
128 "1: " __LL "%0, %1 # clear_bit \n"
129 " and %0, %2 \n"
130 " " __SC "%0, %1 \n"
131 " beqzl %0, 1b \n"
132 " .set mips0 \n"
133 : "=&r" (temp), "+m" (*m)
134 : "ir" (~(1UL << bit)));
135#ifdef CONFIG_CPU_MIPSR2
136 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
137 do {
138 __asm__ __volatile__(
139 " " __LL "%0, %1 # clear_bit \n"
140 " " __INS "%0, $0, %2, 1 \n"
141 " " __SC "%0, %1 \n"
142 : "=&r" (temp), "+m" (*m)
143 : "ir" (bit));
144 } while (unlikely(!temp));
145#endif /* CONFIG_CPU_MIPSR2 */
146 } else if (kernel_uses_llsc) {
147 do {
148 __asm__ __volatile__(
149 " .set mips3 \n"
150 " " __LL "%0, %1 # clear_bit \n"
151 " and %0, %2 \n"
152 " " __SC "%0, %1 \n"
153 " .set mips0 \n"
154 : "=&r" (temp), "+m" (*m)
155 : "ir" (~(1UL << bit)));
156 } while (unlikely(!temp));
157 } else {
158 volatile unsigned long *a = addr;
159 unsigned long mask;
160 unsigned long flags;
161
162 a += nr >> SZLONG_LOG;
163 mask = 1UL << bit;
164 raw_local_irq_save(flags);
165 *a &= ~mask;
166 raw_local_irq_restore(flags);
167 }
168}
169
170/*
171 * clear_bit_unlock - Clears a bit in memory
172 * @nr: Bit to clear
173 * @addr: Address to start counting from
174 *
175 * clear_bit() is atomic and implies release semantics before the memory
176 * operation. It can be used for an unlock.
177 */
178static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
179{
180 smp_mb__before_clear_bit();
181 clear_bit(nr, addr);
182}
183
184/*
185 * change_bit - Toggle a bit in memory
186 * @nr: Bit to change
187 * @addr: Address to start counting from
188 *
189 * change_bit() is atomic and may not be reordered.
190 * Note that @nr may be almost arbitrarily large; this function is not
191 * restricted to acting on a single-word quantity.
192 */
193static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
194{
195 unsigned short bit = nr & SZLONG_MASK;
196
197 if (kernel_uses_llsc && R10000_LLSC_WAR) {
198 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
199 unsigned long temp;
200
201 __asm__ __volatile__(
202 " .set mips3 \n"
203 "1: " __LL "%0, %1 # change_bit \n"
204 " xor %0, %2 \n"
205 " " __SC "%0, %1 \n"
206 " beqzl %0, 1b \n"
207 " .set mips0 \n"
208 : "=&r" (temp), "+m" (*m)
209 : "ir" (1UL << bit));
210 } else if (kernel_uses_llsc) {
211 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
212 unsigned long temp;
213
214 do {
215 __asm__ __volatile__(
216 " .set mips3 \n"
217 " " __LL "%0, %1 # change_bit \n"
218 " xor %0, %2 \n"
219 " " __SC "%0, %1 \n"
220 " .set mips0 \n"
221 : "=&r" (temp), "+m" (*m)
222 : "ir" (1UL << bit));
223 } while (unlikely(!temp));
224 } else {
225 volatile unsigned long *a = addr;
226 unsigned long mask;
227 unsigned long flags;
228
229 a += nr >> SZLONG_LOG;
230 mask = 1UL << bit;
231 raw_local_irq_save(flags);
232 *a ^= mask;
233 raw_local_irq_restore(flags);
234 }
235}
236
237/*
238 * test_and_set_bit - Set a bit and return its old value
239 * @nr: Bit to set
240 * @addr: Address to count from
241 *
242 * This operation is atomic and cannot be reordered.
243 * It also implies a memory barrier.
244 */
245static inline int test_and_set_bit(unsigned long nr,
246 volatile unsigned long *addr)
247{
248 unsigned short bit = nr & SZLONG_MASK;
249 unsigned long res;
250
251 smp_mb__before_llsc();
252
253 if (kernel_uses_llsc && R10000_LLSC_WAR) {
254 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
255 unsigned long temp;
256
257 __asm__ __volatile__(
258 " .set mips3 \n"
259 "1: " __LL "%0, %1 # test_and_set_bit \n"
260 " or %2, %0, %3 \n"
261 " " __SC "%2, %1 \n"
262 " beqzl %2, 1b \n"
263 " and %2, %0, %3 \n"
264 " .set mips0 \n"
265 : "=&r" (temp), "+m" (*m), "=&r" (res)
266 : "r" (1UL << bit)
267 : "memory");
268 } else if (kernel_uses_llsc) {
269 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
270 unsigned long temp;
271
272 do {
273 __asm__ __volatile__(
274 " .set mips3 \n"
275 " " __LL "%0, %1 # test_and_set_bit \n"
276 " or %2, %0, %3 \n"
277 " " __SC "%2, %1 \n"
278 " .set mips0 \n"
279 : "=&r" (temp), "+m" (*m), "=&r" (res)
280 : "r" (1UL << bit)
281 : "memory");
282 } while (unlikely(!res));
283
284 res = temp & (1UL << bit);
285 } else {
286 volatile unsigned long *a = addr;
287 unsigned long mask;
288 unsigned long flags;
289
290 a += nr >> SZLONG_LOG;
291 mask = 1UL << bit;
292 raw_local_irq_save(flags);
293 res = (mask & *a);
294 *a |= mask;
295 raw_local_irq_restore(flags);
296 }
297
298 smp_llsc_mb();
299
300 return res != 0;
301}
302
303/*
304 * test_and_set_bit_lock - Set a bit and return its old value
305 * @nr: Bit to set
306 * @addr: Address to count from
307 *
308 * This operation is atomic and implies acquire ordering semantics
309 * after the memory operation.
310 */
311static inline int test_and_set_bit_lock(unsigned long nr,
312 volatile unsigned long *addr)
313{
314 unsigned short bit = nr & SZLONG_MASK;
315 unsigned long res;
316
317 if (kernel_uses_llsc && R10000_LLSC_WAR) {
318 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
319 unsigned long temp;
320
321 __asm__ __volatile__(
322 " .set mips3 \n"
323 "1: " __LL "%0, %1 # test_and_set_bit \n"
324 " or %2, %0, %3 \n"
325 " " __SC "%2, %1 \n"
326 " beqzl %2, 1b \n"
327 " and %2, %0, %3 \n"
328 " .set mips0 \n"
329 : "=&r" (temp), "+m" (*m), "=&r" (res)
330 : "r" (1UL << bit)
331 : "memory");
332 } else if (kernel_uses_llsc) {
333 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
334 unsigned long temp;
335
336 do {
337 __asm__ __volatile__(
338 " .set mips3 \n"
339 " " __LL "%0, %1 # test_and_set_bit \n"
340 " or %2, %0, %3 \n"
341 " " __SC "%2, %1 \n"
342 " .set mips0 \n"
343 : "=&r" (temp), "+m" (*m), "=&r" (res)
344 : "r" (1UL << bit)
345 : "memory");
346 } while (unlikely(!res));
347
348 res = temp & (1UL << bit);
349 } else {
350 volatile unsigned long *a = addr;
351 unsigned long mask;
352 unsigned long flags;
353
354 a += nr >> SZLONG_LOG;
355 mask = 1UL << bit;
356 raw_local_irq_save(flags);
357 res = (mask & *a);
358 *a |= mask;
359 raw_local_irq_restore(flags);
360 }
361
362 smp_llsc_mb();
363
364 return res != 0;
365}
366/*
367 * test_and_clear_bit - Clear a bit and return its old value
368 * @nr: Bit to clear
369 * @addr: Address to count from
370 *
371 * This operation is atomic and cannot be reordered.
372 * It also implies a memory barrier.
373 */
374static inline int test_and_clear_bit(unsigned long nr,
375 volatile unsigned long *addr)
376{
377 unsigned short bit = nr & SZLONG_MASK;
378 unsigned long res;
379
380 smp_mb__before_llsc();
381
382 if (kernel_uses_llsc && R10000_LLSC_WAR) {
383 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
384 unsigned long temp;
385
386 __asm__ __volatile__(
387 " .set mips3 \n"
388 "1: " __LL "%0, %1 # test_and_clear_bit \n"
389 " or %2, %0, %3 \n"
390 " xor %2, %3 \n"
391 " " __SC "%2, %1 \n"
392 " beqzl %2, 1b \n"
393 " and %2, %0, %3 \n"
394 " .set mips0 \n"
395 : "=&r" (temp), "+m" (*m), "=&r" (res)
396 : "r" (1UL << bit)
397 : "memory");
398#ifdef CONFIG_CPU_MIPSR2
399 } else if (kernel_uses_llsc && __builtin_constant_p(nr)) {
400 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
401 unsigned long temp;
402
403 do {
404 __asm__ __volatile__(
405 " " __LL "%0, %1 # test_and_clear_bit \n"
406 " " __EXT "%2, %0, %3, 1 \n"
407 " " __INS "%0, $0, %3, 1 \n"
408 " " __SC "%0, %1 \n"
409 : "=&r" (temp), "+m" (*m), "=&r" (res)
410 : "ir" (bit)
411 : "memory");
412 } while (unlikely(!temp));
413#endif
414 } else if (kernel_uses_llsc) {
415 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
416 unsigned long temp;
417
418 do {
419 __asm__ __volatile__(
420 " .set mips3 \n"
421 " " __LL "%0, %1 # test_and_clear_bit \n"
422 " or %2, %0, %3 \n"
423 " xor %2, %3 \n"
424 " " __SC "%2, %1 \n"
425 " .set mips0 \n"
426 : "=&r" (temp), "+m" (*m), "=&r" (res)
427 : "r" (1UL << bit)
428 : "memory");
429 } while (unlikely(!res));
430
431 res = temp & (1UL << bit);
432 } else {
433 volatile unsigned long *a = addr;
434 unsigned long mask;
435 unsigned long flags;
436
437 a += nr >> SZLONG_LOG;
438 mask = 1UL << bit;
439 raw_local_irq_save(flags);
440 res = (mask & *a);
441 *a &= ~mask;
442 raw_local_irq_restore(flags);
443 }
444
445 smp_llsc_mb();
446
447 return res != 0;
448}
449
450/*
451 * test_and_change_bit - Change a bit and return its old value
452 * @nr: Bit to change
453 * @addr: Address to count from
454 *
455 * This operation is atomic and cannot be reordered.
456 * It also implies a memory barrier.
457 */
458static inline int test_and_change_bit(unsigned long nr,
459 volatile unsigned long *addr)
460{
461 unsigned short bit = nr & SZLONG_MASK;
462 unsigned long res;
463
464 smp_mb__before_llsc();
465
466 if (kernel_uses_llsc && R10000_LLSC_WAR) {
467 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
468 unsigned long temp;
469
470 __asm__ __volatile__(
471 " .set mips3 \n"
472 "1: " __LL "%0, %1 # test_and_change_bit \n"
473 " xor %2, %0, %3 \n"
474 " " __SC "%2, %1 \n"
475 " beqzl %2, 1b \n"
476 " and %2, %0, %3 \n"
477 " .set mips0 \n"
478 : "=&r" (temp), "+m" (*m), "=&r" (res)
479 : "r" (1UL << bit)
480 : "memory");
481 } else if (kernel_uses_llsc) {
482 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
483 unsigned long temp;
484
485 do {
486 __asm__ __volatile__(
487 " .set mips3 \n"
488 " " __LL "%0, %1 # test_and_change_bit \n"
489 " xor %2, %0, %3 \n"
490 " " __SC "\t%2, %1 \n"
491 " .set mips0 \n"
492 : "=&r" (temp), "+m" (*m), "=&r" (res)
493 : "r" (1UL << bit)
494 : "memory");
495 } while (unlikely(!res));
496
497 res = temp & (1UL << bit);
498 } else {
499 volatile unsigned long *a = addr;
500 unsigned long mask;
501 unsigned long flags;
502
503 a += nr >> SZLONG_LOG;
504 mask = 1UL << bit;
505 raw_local_irq_save(flags);
506 res = (mask & *a);
507 *a ^= mask;
508 raw_local_irq_restore(flags);
509 }
510
511 smp_llsc_mb();
512
513 return res != 0;
514}
515
516#include <asm-generic/bitops/non-atomic.h>
517
518/*
519 * __clear_bit_unlock - Clears a bit in memory
520 * @nr: Bit to clear
521 * @addr: Address to start counting from
522 *
523 * __clear_bit() is non-atomic and implies release semantics before the memory
524 * operation. It can be used for an unlock if no other CPUs can concurrently
525 * modify other bits in the word.
526 */
527static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
528{
529 smp_mb();
530 __clear_bit(nr, addr);
531}
532
533/*
534 * Return the bit position (0..63) of the most significant 1 bit in a word
535 * Returns -1 if no 1 bit exists
536 */
537static inline unsigned long __fls(unsigned long word)
538{
539 int num;
540
541 if (BITS_PER_LONG == 32 &&
542 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
543 __asm__(
544 " .set push \n"
545 " .set mips32 \n"
546 " clz %0, %1 \n"
547 " .set pop \n"
548 : "=r" (num)
549 : "r" (word));
550
551 return 31 - num;
552 }
553
554 if (BITS_PER_LONG == 64 &&
555 __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) {
556 __asm__(
557 " .set push \n"
558 " .set mips64 \n"
559 " dclz %0, %1 \n"
560 " .set pop \n"
561 : "=r" (num)
562 : "r" (word));
563
564 return 63 - num;
565 }
566
567 num = BITS_PER_LONG - 1;
568
569#if BITS_PER_LONG == 64
570 if (!(word & (~0ul << 32))) {
571 num -= 32;
572 word <<= 32;
573 }
574#endif
575 if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
576 num -= 16;
577 word <<= 16;
578 }
579 if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
580 num -= 8;
581 word <<= 8;
582 }
583 if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
584 num -= 4;
585 word <<= 4;
586 }
587 if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
588 num -= 2;
589 word <<= 2;
590 }
591 if (!(word & (~0ul << (BITS_PER_LONG-1))))
592 num -= 1;
593 return num;
594}
595
596/*
597 * __ffs - find first bit in word.
598 * @word: The word to search
599 *
600 * Returns 0..SZLONG-1
601 * Undefined if no bit exists, so code should check against 0 first.
602 */
603static inline unsigned long __ffs(unsigned long word)
604{
605 return __fls(word & -word);
606}
607
608/*
609 * fls - find last bit set.
610 * @word: The word to search
611 *
612 * This is defined the same way as ffs.
613 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
614 */
615static inline int fls(int x)
616{
617 int r;
618
619 if (__builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
620 __asm__("clz %0, %1" : "=r" (x) : "r" (x));
621
622 return 32 - x;
623 }
624
625 r = 32;
626 if (!x)
627 return 0;
628 if (!(x & 0xffff0000u)) {
629 x <<= 16;
630 r -= 16;
631 }
632 if (!(x & 0xff000000u)) {
633 x <<= 8;
634 r -= 8;
635 }
636 if (!(x & 0xf0000000u)) {
637 x <<= 4;
638 r -= 4;
639 }
640 if (!(x & 0xc0000000u)) {
641 x <<= 2;
642 r -= 2;
643 }
644 if (!(x & 0x80000000u)) {
645 x <<= 1;
646 r -= 1;
647 }
648 return r;
649}
650
651#include <asm-generic/bitops/fls64.h>
652
653/*
654 * ffs - find first bit set.
655 * @word: The word to search
656 *
657 * This is defined the same way as
658 * the libc and compiler builtin ffs routines, therefore
659 * differs in spirit from the above ffz (man ffs).
660 */
661static inline int ffs(int word)
662{
663 if (!word)
664 return 0;
665
666 return fls(word & -word);
667}
668
669#include <asm-generic/bitops/ffz.h>
670#include <asm-generic/bitops/find.h>
671
672#ifdef __KERNEL__
673
674#include <asm-generic/bitops/sched.h>
675
676#include <asm/arch_hweight.h>
677#include <asm-generic/bitops/const_hweight.h>
678
679#include <asm-generic/bitops/le.h>
680#include <asm-generic/bitops/ext2-atomic.h>
681
682#endif /* __KERNEL__ */
683
684#endif /* _ASM_BITOPS_H */