Loading...
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (c) 1999, 2000 Silicon Graphics, Inc.
8 */
9#ifndef _ASM_BITOPS_H
10#define _ASM_BITOPS_H
11
12#ifndef _LINUX_BITOPS_H
13#error only <linux/bitops.h> can be included directly
14#endif
15
16#include <linux/bits.h>
17#include <linux/compiler.h>
18#include <linux/types.h>
19#include <asm/barrier.h>
20#include <asm/byteorder.h> /* sigh ... */
21#include <asm/compiler.h>
22#include <asm/cpu-features.h>
23#include <asm/isa-rev.h>
24#include <asm/llsc.h>
25#include <asm/sgidefs.h>
26#include <asm/war.h>
27
28#define __bit_op(mem, insn, inputs...) do { \
29 unsigned long temp; \
30 \
31 asm volatile( \
32 " .set push \n" \
33 " .set " MIPS_ISA_LEVEL " \n" \
34 " " __SYNC(full, loongson3_war) " \n" \
35 "1: " __LL "%0, %1 \n" \
36 " " insn " \n" \
37 " " __SC "%0, %1 \n" \
38 " " __SC_BEQZ "%0, 1b \n" \
39 " .set pop \n" \
40 : "=&r"(temp), "+" GCC_OFF_SMALL_ASM()(mem) \
41 : inputs \
42 : __LLSC_CLOBBER); \
43} while (0)
44
45#define __test_bit_op(mem, ll_dst, insn, inputs...) ({ \
46 unsigned long orig, temp; \
47 \
48 asm volatile( \
49 " .set push \n" \
50 " .set " MIPS_ISA_LEVEL " \n" \
51 " " __SYNC(full, loongson3_war) " \n" \
52 "1: " __LL ll_dst ", %2 \n" \
53 " " insn " \n" \
54 " " __SC "%1, %2 \n" \
55 " " __SC_BEQZ "%1, 1b \n" \
56 " .set pop \n" \
57 : "=&r"(orig), "=&r"(temp), \
58 "+" GCC_OFF_SMALL_ASM()(mem) \
59 : inputs \
60 : __LLSC_CLOBBER); \
61 \
62 orig; \
63})
64
65/*
66 * These are the "slower" versions of the functions and are in bitops.c.
67 * These functions call raw_local_irq_{save,restore}().
68 */
69void __mips_set_bit(unsigned long nr, volatile unsigned long *addr);
70void __mips_clear_bit(unsigned long nr, volatile unsigned long *addr);
71void __mips_change_bit(unsigned long nr, volatile unsigned long *addr);
72int __mips_test_and_set_bit_lock(unsigned long nr,
73 volatile unsigned long *addr);
74int __mips_test_and_clear_bit(unsigned long nr,
75 volatile unsigned long *addr);
76int __mips_test_and_change_bit(unsigned long nr,
77 volatile unsigned long *addr);
78
79
80/*
81 * set_bit - Atomically set a bit in memory
82 * @nr: the bit to set
83 * @addr: the address to start counting from
84 *
85 * This function is atomic and may not be reordered. See __set_bit()
86 * if you do not require the atomic guarantees.
87 * Note that @nr may be almost arbitrarily large; this function is not
88 * restricted to acting on a single-word quantity.
89 */
90static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
91{
92 volatile unsigned long *m = &addr[BIT_WORD(nr)];
93 int bit = nr % BITS_PER_LONG;
94
95 if (!kernel_uses_llsc) {
96 __mips_set_bit(nr, addr);
97 return;
98 }
99
100 if ((MIPS_ISA_REV >= 2) && __builtin_constant_p(bit) && (bit >= 16)) {
101 __bit_op(*m, __INS "%0, %3, %2, 1", "i"(bit), "r"(~0));
102 return;
103 }
104
105 __bit_op(*m, "or\t%0, %2", "ir"(BIT(bit)));
106}
107
108/*
109 * clear_bit - Clears a bit in memory
110 * @nr: Bit to clear
111 * @addr: Address to start counting from
112 *
113 * clear_bit() is atomic and may not be reordered. However, it does
114 * not contain a memory barrier, so if it is used for locking purposes,
115 * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic()
116 * in order to ensure changes are visible on other processors.
117 */
118static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
119{
120 volatile unsigned long *m = &addr[BIT_WORD(nr)];
121 int bit = nr % BITS_PER_LONG;
122
123 if (!kernel_uses_llsc) {
124 __mips_clear_bit(nr, addr);
125 return;
126 }
127
128 if ((MIPS_ISA_REV >= 2) && __builtin_constant_p(bit)) {
129 __bit_op(*m, __INS "%0, $0, %2, 1", "i"(bit));
130 return;
131 }
132
133 __bit_op(*m, "and\t%0, %2", "ir"(~BIT(bit)));
134}
135
136/*
137 * clear_bit_unlock - Clears a bit in memory
138 * @nr: Bit to clear
139 * @addr: Address to start counting from
140 *
141 * clear_bit() is atomic and implies release semantics before the memory
142 * operation. It can be used for an unlock.
143 */
144static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
145{
146 smp_mb__before_atomic();
147 clear_bit(nr, addr);
148}
149
150/*
151 * change_bit - Toggle a bit in memory
152 * @nr: Bit to change
153 * @addr: Address to start counting from
154 *
155 * change_bit() is atomic and may not be reordered.
156 * Note that @nr may be almost arbitrarily large; this function is not
157 * restricted to acting on a single-word quantity.
158 */
159static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
160{
161 volatile unsigned long *m = &addr[BIT_WORD(nr)];
162 int bit = nr % BITS_PER_LONG;
163
164 if (!kernel_uses_llsc) {
165 __mips_change_bit(nr, addr);
166 return;
167 }
168
169 __bit_op(*m, "xor\t%0, %2", "ir"(BIT(bit)));
170}
171
172/*
173 * test_and_set_bit_lock - Set a bit and return its old value
174 * @nr: Bit to set
175 * @addr: Address to count from
176 *
177 * This operation is atomic and implies acquire ordering semantics
178 * after the memory operation.
179 */
180static inline int test_and_set_bit_lock(unsigned long nr,
181 volatile unsigned long *addr)
182{
183 volatile unsigned long *m = &addr[BIT_WORD(nr)];
184 int bit = nr % BITS_PER_LONG;
185 unsigned long res, orig;
186
187 if (!kernel_uses_llsc) {
188 res = __mips_test_and_set_bit_lock(nr, addr);
189 } else {
190 orig = __test_bit_op(*m, "%0",
191 "or\t%1, %0, %3",
192 "ir"(BIT(bit)));
193 res = (orig & BIT(bit)) != 0;
194 }
195
196 smp_llsc_mb();
197
198 return res;
199}
200
201/*
202 * test_and_set_bit - Set a bit and return its old value
203 * @nr: Bit to set
204 * @addr: Address to count from
205 *
206 * This operation is atomic and cannot be reordered.
207 * It also implies a memory barrier.
208 */
209static inline int test_and_set_bit(unsigned long nr,
210 volatile unsigned long *addr)
211{
212 smp_mb__before_atomic();
213 return test_and_set_bit_lock(nr, addr);
214}
215
216/*
217 * test_and_clear_bit - Clear a bit and return its old value
218 * @nr: Bit to clear
219 * @addr: Address to count from
220 *
221 * This operation is atomic and cannot be reordered.
222 * It also implies a memory barrier.
223 */
224static inline int test_and_clear_bit(unsigned long nr,
225 volatile unsigned long *addr)
226{
227 volatile unsigned long *m = &addr[BIT_WORD(nr)];
228 int bit = nr % BITS_PER_LONG;
229 unsigned long res, orig;
230
231 smp_mb__before_atomic();
232
233 if (!kernel_uses_llsc) {
234 res = __mips_test_and_clear_bit(nr, addr);
235 } else if ((MIPS_ISA_REV >= 2) && __builtin_constant_p(nr)) {
236 res = __test_bit_op(*m, "%1",
237 __EXT "%0, %1, %3, 1;"
238 __INS "%1, $0, %3, 1",
239 "i"(bit));
240 } else {
241 orig = __test_bit_op(*m, "%0",
242 "or\t%1, %0, %3;"
243 "xor\t%1, %1, %3",
244 "ir"(BIT(bit)));
245 res = (orig & BIT(bit)) != 0;
246 }
247
248 smp_llsc_mb();
249
250 return res;
251}
252
253/*
254 * test_and_change_bit - Change a bit and return its old value
255 * @nr: Bit to change
256 * @addr: Address to count from
257 *
258 * This operation is atomic and cannot be reordered.
259 * It also implies a memory barrier.
260 */
261static inline int test_and_change_bit(unsigned long nr,
262 volatile unsigned long *addr)
263{
264 volatile unsigned long *m = &addr[BIT_WORD(nr)];
265 int bit = nr % BITS_PER_LONG;
266 unsigned long res, orig;
267
268 smp_mb__before_atomic();
269
270 if (!kernel_uses_llsc) {
271 res = __mips_test_and_change_bit(nr, addr);
272 } else {
273 orig = __test_bit_op(*m, "%0",
274 "xor\t%1, %0, %3",
275 "ir"(BIT(bit)));
276 res = (orig & BIT(bit)) != 0;
277 }
278
279 smp_llsc_mb();
280
281 return res;
282}
283
284#undef __bit_op
285#undef __test_bit_op
286
287#include <asm-generic/bitops/non-atomic.h>
288
289/*
290 * __clear_bit_unlock - Clears a bit in memory
291 * @nr: Bit to clear
292 * @addr: Address to start counting from
293 *
294 * __clear_bit() is non-atomic and implies release semantics before the memory
295 * operation. It can be used for an unlock if no other CPUs can concurrently
296 * modify other bits in the word.
297 */
298static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
299{
300 smp_mb__before_llsc();
301 __clear_bit(nr, addr);
302 nudge_writes();
303}
304
305/*
306 * Return the bit position (0..63) of the most significant 1 bit in a word
307 * Returns -1 if no 1 bit exists
308 */
309static __always_inline unsigned long __fls(unsigned long word)
310{
311 int num;
312
313 if (BITS_PER_LONG == 32 && !__builtin_constant_p(word) &&
314 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
315 __asm__(
316 " .set push \n"
317 " .set "MIPS_ISA_LEVEL" \n"
318 " clz %0, %1 \n"
319 " .set pop \n"
320 : "=r" (num)
321 : "r" (word));
322
323 return 31 - num;
324 }
325
326 if (BITS_PER_LONG == 64 && !__builtin_constant_p(word) &&
327 __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) {
328 __asm__(
329 " .set push \n"
330 " .set "MIPS_ISA_LEVEL" \n"
331 " dclz %0, %1 \n"
332 " .set pop \n"
333 : "=r" (num)
334 : "r" (word));
335
336 return 63 - num;
337 }
338
339 num = BITS_PER_LONG - 1;
340
341#if BITS_PER_LONG == 64
342 if (!(word & (~0ul << 32))) {
343 num -= 32;
344 word <<= 32;
345 }
346#endif
347 if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
348 num -= 16;
349 word <<= 16;
350 }
351 if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
352 num -= 8;
353 word <<= 8;
354 }
355 if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
356 num -= 4;
357 word <<= 4;
358 }
359 if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
360 num -= 2;
361 word <<= 2;
362 }
363 if (!(word & (~0ul << (BITS_PER_LONG-1))))
364 num -= 1;
365 return num;
366}
367
368/*
369 * __ffs - find first bit in word.
370 * @word: The word to search
371 *
372 * Returns 0..SZLONG-1
373 * Undefined if no bit exists, so code should check against 0 first.
374 */
375static __always_inline unsigned long __ffs(unsigned long word)
376{
377 return __fls(word & -word);
378}
379
380/*
381 * fls - find last bit set.
382 * @word: The word to search
383 *
384 * This is defined the same way as ffs.
385 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
386 */
387static inline int fls(unsigned int x)
388{
389 int r;
390
391 if (!__builtin_constant_p(x) &&
392 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
393 __asm__(
394 " .set push \n"
395 " .set "MIPS_ISA_LEVEL" \n"
396 " clz %0, %1 \n"
397 " .set pop \n"
398 : "=r" (x)
399 : "r" (x));
400
401 return 32 - x;
402 }
403
404 r = 32;
405 if (!x)
406 return 0;
407 if (!(x & 0xffff0000u)) {
408 x <<= 16;
409 r -= 16;
410 }
411 if (!(x & 0xff000000u)) {
412 x <<= 8;
413 r -= 8;
414 }
415 if (!(x & 0xf0000000u)) {
416 x <<= 4;
417 r -= 4;
418 }
419 if (!(x & 0xc0000000u)) {
420 x <<= 2;
421 r -= 2;
422 }
423 if (!(x & 0x80000000u)) {
424 x <<= 1;
425 r -= 1;
426 }
427 return r;
428}
429
430#include <asm-generic/bitops/fls64.h>
431
432/*
433 * ffs - find first bit set.
434 * @word: The word to search
435 *
436 * This is defined the same way as
437 * the libc and compiler builtin ffs routines, therefore
438 * differs in spirit from the above ffz (man ffs).
439 */
440static inline int ffs(int word)
441{
442 if (!word)
443 return 0;
444
445 return fls(word & -word);
446}
447
448#include <asm-generic/bitops/ffz.h>
449#include <asm-generic/bitops/find.h>
450
451#ifdef __KERNEL__
452
453#include <asm-generic/bitops/sched.h>
454
455#include <asm/arch_hweight.h>
456#include <asm-generic/bitops/const_hweight.h>
457
458#include <asm-generic/bitops/le.h>
459#include <asm-generic/bitops/ext2-atomic.h>
460
461#endif /* __KERNEL__ */
462
463#endif /* _ASM_BITOPS_H */
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (c) 1999, 2000 Silicon Graphics, Inc.
8 */
9#ifndef _ASM_BITOPS_H
10#define _ASM_BITOPS_H
11
12#ifndef _LINUX_BITOPS_H
13#error only <linux/bitops.h> can be included directly
14#endif
15
16#include <linux/compiler.h>
17#include <linux/types.h>
18#include <asm/barrier.h>
19#include <asm/byteorder.h> /* sigh ... */
20#include <asm/compiler.h>
21#include <asm/cpu-features.h>
22#include <asm/llsc.h>
23#include <asm/sgidefs.h>
24#include <asm/war.h>
25
26/*
27 * These are the "slower" versions of the functions and are in bitops.c.
28 * These functions call raw_local_irq_{save,restore}().
29 */
30void __mips_set_bit(unsigned long nr, volatile unsigned long *addr);
31void __mips_clear_bit(unsigned long nr, volatile unsigned long *addr);
32void __mips_change_bit(unsigned long nr, volatile unsigned long *addr);
33int __mips_test_and_set_bit(unsigned long nr,
34 volatile unsigned long *addr);
35int __mips_test_and_set_bit_lock(unsigned long nr,
36 volatile unsigned long *addr);
37int __mips_test_and_clear_bit(unsigned long nr,
38 volatile unsigned long *addr);
39int __mips_test_and_change_bit(unsigned long nr,
40 volatile unsigned long *addr);
41
42
43/*
44 * set_bit - Atomically set a bit in memory
45 * @nr: the bit to set
46 * @addr: the address to start counting from
47 *
48 * This function is atomic and may not be reordered. See __set_bit()
49 * if you do not require the atomic guarantees.
50 * Note that @nr may be almost arbitrarily large; this function is not
51 * restricted to acting on a single-word quantity.
52 */
53static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
54{
55 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
56 int bit = nr & SZLONG_MASK;
57 unsigned long temp;
58
59 if (kernel_uses_llsc && R10000_LLSC_WAR) {
60 __asm__ __volatile__(
61 " .set push \n"
62 " .set arch=r4000 \n"
63 "1: " __LL "%0, %1 # set_bit \n"
64 " or %0, %2 \n"
65 " " __SC "%0, %1 \n"
66 " beqzl %0, 1b \n"
67 " .set pop \n"
68 : "=&r" (temp), "=" GCC_OFF_SMALL_ASM() (*m)
69 : "ir" (1UL << bit), GCC_OFF_SMALL_ASM() (*m)
70 : __LLSC_CLOBBER);
71#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
72 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
73 loongson_llsc_mb();
74 do {
75 __asm__ __volatile__(
76 " " __LL "%0, %1 # set_bit \n"
77 " " __INS "%0, %3, %2, 1 \n"
78 " " __SC "%0, %1 \n"
79 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
80 : "ir" (bit), "r" (~0)
81 : __LLSC_CLOBBER);
82 } while (unlikely(!temp));
83#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
84 } else if (kernel_uses_llsc) {
85 loongson_llsc_mb();
86 do {
87 __asm__ __volatile__(
88 " .set push \n"
89 " .set "MIPS_ISA_ARCH_LEVEL" \n"
90 " " __LL "%0, %1 # set_bit \n"
91 " or %0, %2 \n"
92 " " __SC "%0, %1 \n"
93 " .set pop \n"
94 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
95 : "ir" (1UL << bit)
96 : __LLSC_CLOBBER);
97 } while (unlikely(!temp));
98 } else
99 __mips_set_bit(nr, addr);
100}
101
102/*
103 * clear_bit - Clears a bit in memory
104 * @nr: Bit to clear
105 * @addr: Address to start counting from
106 *
107 * clear_bit() is atomic and may not be reordered. However, it does
108 * not contain a memory barrier, so if it is used for locking purposes,
109 * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic()
110 * in order to ensure changes are visible on other processors.
111 */
112static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
113{
114 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
115 int bit = nr & SZLONG_MASK;
116 unsigned long temp;
117
118 if (kernel_uses_llsc && R10000_LLSC_WAR) {
119 __asm__ __volatile__(
120 " .set push \n"
121 " .set arch=r4000 \n"
122 "1: " __LL "%0, %1 # clear_bit \n"
123 " and %0, %2 \n"
124 " " __SC "%0, %1 \n"
125 " beqzl %0, 1b \n"
126 " .set pop \n"
127 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
128 : "ir" (~(1UL << bit))
129 : __LLSC_CLOBBER);
130#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
131 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
132 loongson_llsc_mb();
133 do {
134 __asm__ __volatile__(
135 " " __LL "%0, %1 # clear_bit \n"
136 " " __INS "%0, $0, %2, 1 \n"
137 " " __SC "%0, %1 \n"
138 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
139 : "ir" (bit)
140 : __LLSC_CLOBBER);
141 } while (unlikely(!temp));
142#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
143 } else if (kernel_uses_llsc) {
144 loongson_llsc_mb();
145 do {
146 __asm__ __volatile__(
147 " .set push \n"
148 " .set "MIPS_ISA_ARCH_LEVEL" \n"
149 " " __LL "%0, %1 # clear_bit \n"
150 " and %0, %2 \n"
151 " " __SC "%0, %1 \n"
152 " .set pop \n"
153 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
154 : "ir" (~(1UL << bit))
155 : __LLSC_CLOBBER);
156 } while (unlikely(!temp));
157 } else
158 __mips_clear_bit(nr, addr);
159}
160
161/*
162 * clear_bit_unlock - Clears a bit in memory
163 * @nr: Bit to clear
164 * @addr: Address to start counting from
165 *
166 * clear_bit() is atomic and implies release semantics before the memory
167 * operation. It can be used for an unlock.
168 */
169static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
170{
171 smp_mb__before_atomic();
172 clear_bit(nr, addr);
173}
174
175/*
176 * change_bit - Toggle a bit in memory
177 * @nr: Bit to change
178 * @addr: Address to start counting from
179 *
180 * change_bit() is atomic and may not be reordered.
181 * Note that @nr may be almost arbitrarily large; this function is not
182 * restricted to acting on a single-word quantity.
183 */
184static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
185{
186 int bit = nr & SZLONG_MASK;
187
188 if (kernel_uses_llsc && R10000_LLSC_WAR) {
189 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
190 unsigned long temp;
191
192 __asm__ __volatile__(
193 " .set push \n"
194 " .set arch=r4000 \n"
195 "1: " __LL "%0, %1 # change_bit \n"
196 " xor %0, %2 \n"
197 " " __SC "%0, %1 \n"
198 " beqzl %0, 1b \n"
199 " .set pop \n"
200 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
201 : "ir" (1UL << bit)
202 : __LLSC_CLOBBER);
203 } else if (kernel_uses_llsc) {
204 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
205 unsigned long temp;
206
207 loongson_llsc_mb();
208 do {
209 __asm__ __volatile__(
210 " .set push \n"
211 " .set "MIPS_ISA_ARCH_LEVEL" \n"
212 " " __LL "%0, %1 # change_bit \n"
213 " xor %0, %2 \n"
214 " " __SC "%0, %1 \n"
215 " .set pop \n"
216 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
217 : "ir" (1UL << bit)
218 : __LLSC_CLOBBER);
219 } while (unlikely(!temp));
220 } else
221 __mips_change_bit(nr, addr);
222}
223
224/*
225 * test_and_set_bit - Set a bit and return its old value
226 * @nr: Bit to set
227 * @addr: Address to count from
228 *
229 * This operation is atomic and cannot be reordered.
230 * It also implies a memory barrier.
231 */
232static inline int test_and_set_bit(unsigned long nr,
233 volatile unsigned long *addr)
234{
235 int bit = nr & SZLONG_MASK;
236 unsigned long res;
237
238 smp_mb__before_llsc();
239
240 if (kernel_uses_llsc && R10000_LLSC_WAR) {
241 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
242 unsigned long temp;
243
244 __asm__ __volatile__(
245 " .set push \n"
246 " .set arch=r4000 \n"
247 "1: " __LL "%0, %1 # test_and_set_bit \n"
248 " or %2, %0, %3 \n"
249 " " __SC "%2, %1 \n"
250 " beqzl %2, 1b \n"
251 " and %2, %0, %3 \n"
252 " .set pop \n"
253 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
254 : "r" (1UL << bit)
255 : __LLSC_CLOBBER);
256 } else if (kernel_uses_llsc) {
257 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
258 unsigned long temp;
259
260 loongson_llsc_mb();
261 do {
262 __asm__ __volatile__(
263 " .set push \n"
264 " .set "MIPS_ISA_ARCH_LEVEL" \n"
265 " " __LL "%0, %1 # test_and_set_bit \n"
266 " or %2, %0, %3 \n"
267 " " __SC "%2, %1 \n"
268 " .set pop \n"
269 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
270 : "r" (1UL << bit)
271 : __LLSC_CLOBBER);
272 } while (unlikely(!res));
273
274 res = temp & (1UL << bit);
275 } else
276 res = __mips_test_and_set_bit(nr, addr);
277
278 smp_llsc_mb();
279
280 return res != 0;
281}
282
283/*
284 * test_and_set_bit_lock - Set a bit and return its old value
285 * @nr: Bit to set
286 * @addr: Address to count from
287 *
288 * This operation is atomic and implies acquire ordering semantics
289 * after the memory operation.
290 */
291static inline int test_and_set_bit_lock(unsigned long nr,
292 volatile unsigned long *addr)
293{
294 int bit = nr & SZLONG_MASK;
295 unsigned long res;
296
297 if (kernel_uses_llsc && R10000_LLSC_WAR) {
298 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
299 unsigned long temp;
300
301 __asm__ __volatile__(
302 " .set push \n"
303 " .set arch=r4000 \n"
304 "1: " __LL "%0, %1 # test_and_set_bit \n"
305 " or %2, %0, %3 \n"
306 " " __SC "%2, %1 \n"
307 " beqzl %2, 1b \n"
308 " and %2, %0, %3 \n"
309 " .set pop \n"
310 : "=&r" (temp), "+m" (*m), "=&r" (res)
311 : "r" (1UL << bit)
312 : __LLSC_CLOBBER);
313 } else if (kernel_uses_llsc) {
314 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
315 unsigned long temp;
316
317 loongson_llsc_mb();
318 do {
319 __asm__ __volatile__(
320 " .set push \n"
321 " .set "MIPS_ISA_ARCH_LEVEL" \n"
322 " " __LL "%0, %1 # test_and_set_bit \n"
323 " or %2, %0, %3 \n"
324 " " __SC "%2, %1 \n"
325 " .set pop \n"
326 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
327 : "r" (1UL << bit)
328 : __LLSC_CLOBBER);
329 } while (unlikely(!res));
330
331 res = temp & (1UL << bit);
332 } else
333 res = __mips_test_and_set_bit_lock(nr, addr);
334
335 smp_llsc_mb();
336
337 return res != 0;
338}
339/*
340 * test_and_clear_bit - Clear a bit and return its old value
341 * @nr: Bit to clear
342 * @addr: Address to count from
343 *
344 * This operation is atomic and cannot be reordered.
345 * It also implies a memory barrier.
346 */
347static inline int test_and_clear_bit(unsigned long nr,
348 volatile unsigned long *addr)
349{
350 int bit = nr & SZLONG_MASK;
351 unsigned long res;
352
353 smp_mb__before_llsc();
354
355 if (kernel_uses_llsc && R10000_LLSC_WAR) {
356 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
357 unsigned long temp;
358
359 __asm__ __volatile__(
360 " .set push \n"
361 " .set arch=r4000 \n"
362 "1: " __LL "%0, %1 # test_and_clear_bit \n"
363 " or %2, %0, %3 \n"
364 " xor %2, %3 \n"
365 " " __SC "%2, %1 \n"
366 " beqzl %2, 1b \n"
367 " and %2, %0, %3 \n"
368 " .set pop \n"
369 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
370 : "r" (1UL << bit)
371 : __LLSC_CLOBBER);
372#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
373 } else if (kernel_uses_llsc && __builtin_constant_p(nr)) {
374 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
375 unsigned long temp;
376
377 loongson_llsc_mb();
378 do {
379 __asm__ __volatile__(
380 " " __LL "%0, %1 # test_and_clear_bit \n"
381 " " __EXT "%2, %0, %3, 1 \n"
382 " " __INS "%0, $0, %3, 1 \n"
383 " " __SC "%0, %1 \n"
384 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
385 : "ir" (bit)
386 : __LLSC_CLOBBER);
387 } while (unlikely(!temp));
388#endif
389 } else if (kernel_uses_llsc) {
390 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
391 unsigned long temp;
392
393 loongson_llsc_mb();
394 do {
395 __asm__ __volatile__(
396 " .set push \n"
397 " .set "MIPS_ISA_ARCH_LEVEL" \n"
398 " " __LL "%0, %1 # test_and_clear_bit \n"
399 " or %2, %0, %3 \n"
400 " xor %2, %3 \n"
401 " " __SC "%2, %1 \n"
402 " .set pop \n"
403 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
404 : "r" (1UL << bit)
405 : __LLSC_CLOBBER);
406 } while (unlikely(!res));
407
408 res = temp & (1UL << bit);
409 } else
410 res = __mips_test_and_clear_bit(nr, addr);
411
412 smp_llsc_mb();
413
414 return res != 0;
415}
416
417/*
418 * test_and_change_bit - Change a bit and return its old value
419 * @nr: Bit to change
420 * @addr: Address to count from
421 *
422 * This operation is atomic and cannot be reordered.
423 * It also implies a memory barrier.
424 */
425static inline int test_and_change_bit(unsigned long nr,
426 volatile unsigned long *addr)
427{
428 int bit = nr & SZLONG_MASK;
429 unsigned long res;
430
431 smp_mb__before_llsc();
432
433 if (kernel_uses_llsc && R10000_LLSC_WAR) {
434 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
435 unsigned long temp;
436
437 __asm__ __volatile__(
438 " .set push \n"
439 " .set arch=r4000 \n"
440 "1: " __LL "%0, %1 # test_and_change_bit \n"
441 " xor %2, %0, %3 \n"
442 " " __SC "%2, %1 \n"
443 " beqzl %2, 1b \n"
444 " and %2, %0, %3 \n"
445 " .set pop \n"
446 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
447 : "r" (1UL << bit)
448 : __LLSC_CLOBBER);
449 } else if (kernel_uses_llsc) {
450 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
451 unsigned long temp;
452
453 loongson_llsc_mb();
454 do {
455 __asm__ __volatile__(
456 " .set push \n"
457 " .set "MIPS_ISA_ARCH_LEVEL" \n"
458 " " __LL "%0, %1 # test_and_change_bit \n"
459 " xor %2, %0, %3 \n"
460 " " __SC "\t%2, %1 \n"
461 " .set pop \n"
462 : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
463 : "r" (1UL << bit)
464 : __LLSC_CLOBBER);
465 } while (unlikely(!res));
466
467 res = temp & (1UL << bit);
468 } else
469 res = __mips_test_and_change_bit(nr, addr);
470
471 smp_llsc_mb();
472
473 return res != 0;
474}
475
476#include <asm-generic/bitops/non-atomic.h>
477
478/*
479 * __clear_bit_unlock - Clears a bit in memory
480 * @nr: Bit to clear
481 * @addr: Address to start counting from
482 *
483 * __clear_bit() is non-atomic and implies release semantics before the memory
484 * operation. It can be used for an unlock if no other CPUs can concurrently
485 * modify other bits in the word.
486 */
487static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
488{
489 smp_mb__before_llsc();
490 __clear_bit(nr, addr);
491 nudge_writes();
492}
493
494/*
495 * Return the bit position (0..63) of the most significant 1 bit in a word
496 * Returns -1 if no 1 bit exists
497 */
498static __always_inline unsigned long __fls(unsigned long word)
499{
500 int num;
501
502 if (BITS_PER_LONG == 32 && !__builtin_constant_p(word) &&
503 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
504 __asm__(
505 " .set push \n"
506 " .set "MIPS_ISA_LEVEL" \n"
507 " clz %0, %1 \n"
508 " .set pop \n"
509 : "=r" (num)
510 : "r" (word));
511
512 return 31 - num;
513 }
514
515 if (BITS_PER_LONG == 64 && !__builtin_constant_p(word) &&
516 __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) {
517 __asm__(
518 " .set push \n"
519 " .set "MIPS_ISA_LEVEL" \n"
520 " dclz %0, %1 \n"
521 " .set pop \n"
522 : "=r" (num)
523 : "r" (word));
524
525 return 63 - num;
526 }
527
528 num = BITS_PER_LONG - 1;
529
530#if BITS_PER_LONG == 64
531 if (!(word & (~0ul << 32))) {
532 num -= 32;
533 word <<= 32;
534 }
535#endif
536 if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
537 num -= 16;
538 word <<= 16;
539 }
540 if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
541 num -= 8;
542 word <<= 8;
543 }
544 if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
545 num -= 4;
546 word <<= 4;
547 }
548 if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
549 num -= 2;
550 word <<= 2;
551 }
552 if (!(word & (~0ul << (BITS_PER_LONG-1))))
553 num -= 1;
554 return num;
555}
556
557/*
558 * __ffs - find first bit in word.
559 * @word: The word to search
560 *
561 * Returns 0..SZLONG-1
562 * Undefined if no bit exists, so code should check against 0 first.
563 */
564static __always_inline unsigned long __ffs(unsigned long word)
565{
566 return __fls(word & -word);
567}
568
569/*
570 * fls - find last bit set.
571 * @word: The word to search
572 *
573 * This is defined the same way as ffs.
574 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
575 */
576static inline int fls(unsigned int x)
577{
578 int r;
579
580 if (!__builtin_constant_p(x) &&
581 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
582 __asm__(
583 " .set push \n"
584 " .set "MIPS_ISA_LEVEL" \n"
585 " clz %0, %1 \n"
586 " .set pop \n"
587 : "=r" (x)
588 : "r" (x));
589
590 return 32 - x;
591 }
592
593 r = 32;
594 if (!x)
595 return 0;
596 if (!(x & 0xffff0000u)) {
597 x <<= 16;
598 r -= 16;
599 }
600 if (!(x & 0xff000000u)) {
601 x <<= 8;
602 r -= 8;
603 }
604 if (!(x & 0xf0000000u)) {
605 x <<= 4;
606 r -= 4;
607 }
608 if (!(x & 0xc0000000u)) {
609 x <<= 2;
610 r -= 2;
611 }
612 if (!(x & 0x80000000u)) {
613 x <<= 1;
614 r -= 1;
615 }
616 return r;
617}
618
619#include <asm-generic/bitops/fls64.h>
620
621/*
622 * ffs - find first bit set.
623 * @word: The word to search
624 *
625 * This is defined the same way as
626 * the libc and compiler builtin ffs routines, therefore
627 * differs in spirit from the above ffz (man ffs).
628 */
629static inline int ffs(int word)
630{
631 if (!word)
632 return 0;
633
634 return fls(word & -word);
635}
636
637#include <asm-generic/bitops/ffz.h>
638#include <asm-generic/bitops/find.h>
639
640#ifdef __KERNEL__
641
642#include <asm-generic/bitops/sched.h>
643
644#include <asm/arch_hweight.h>
645#include <asm-generic/bitops/const_hweight.h>
646
647#include <asm-generic/bitops/le.h>
648#include <asm-generic/bitops/ext2-atomic.h>
649
650#endif /* __KERNEL__ */
651
652#endif /* _ASM_BITOPS_H */