Loading...
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012 Regents of the University of California
4 */
5
6#ifndef _ASM_RISCV_BITOPS_H
7#define _ASM_RISCV_BITOPS_H
8
9#ifndef _LINUX_BITOPS_H
10#error "Only <linux/bitops.h> can be included directly"
11#endif /* _LINUX_BITOPS_H */
12
13#include <linux/compiler.h>
14#include <linux/irqflags.h>
15#include <asm/barrier.h>
16#include <asm/bitsperlong.h>
17
18#if !defined(CONFIG_RISCV_ISA_ZBB) || defined(NO_ALTERNATIVE)
19#include <asm-generic/bitops/__ffs.h>
20#include <asm-generic/bitops/__fls.h>
21#include <asm-generic/bitops/ffs.h>
22#include <asm-generic/bitops/fls.h>
23
24#else
25#define __HAVE_ARCH___FFS
26#define __HAVE_ARCH___FLS
27#define __HAVE_ARCH_FFS
28#define __HAVE_ARCH_FLS
29
30#include <asm-generic/bitops/__ffs.h>
31#include <asm-generic/bitops/__fls.h>
32#include <asm-generic/bitops/ffs.h>
33#include <asm-generic/bitops/fls.h>
34
35#include <asm/alternative-macros.h>
36#include <asm/hwcap.h>
37
38#if (BITS_PER_LONG == 64)
39#define CTZW "ctzw "
40#define CLZW "clzw "
41#elif (BITS_PER_LONG == 32)
42#define CTZW "ctz "
43#define CLZW "clz "
44#else
45#error "Unexpected BITS_PER_LONG"
46#endif
47
48static __always_inline unsigned long variable__ffs(unsigned long word)
49{
50 asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0,
51 RISCV_ISA_EXT_ZBB, 1)
52 : : : : legacy);
53
54 asm volatile (".option push\n"
55 ".option arch,+zbb\n"
56 "ctz %0, %1\n"
57 ".option pop\n"
58 : "=r" (word) : "r" (word) :);
59
60 return word;
61
62legacy:
63 return generic___ffs(word);
64}
65
66/**
67 * __ffs - find first set bit in a long word
68 * @word: The word to search
69 *
70 * Undefined if no set bit exists, so code should check against 0 first.
71 */
72#define __ffs(word) \
73 (__builtin_constant_p(word) ? \
74 (unsigned long)__builtin_ctzl(word) : \
75 variable__ffs(word))
76
77static __always_inline unsigned long variable__fls(unsigned long word)
78{
79 asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0,
80 RISCV_ISA_EXT_ZBB, 1)
81 : : : : legacy);
82
83 asm volatile (".option push\n"
84 ".option arch,+zbb\n"
85 "clz %0, %1\n"
86 ".option pop\n"
87 : "=r" (word) : "r" (word) :);
88
89 return BITS_PER_LONG - 1 - word;
90
91legacy:
92 return generic___fls(word);
93}
94
95/**
96 * __fls - find last set bit in a long word
97 * @word: the word to search
98 *
99 * Undefined if no set bit exists, so code should check against 0 first.
100 */
101#define __fls(word) \
102 (__builtin_constant_p(word) ? \
103 (unsigned long)(BITS_PER_LONG - 1 - __builtin_clzl(word)) : \
104 variable__fls(word))
105
106static __always_inline int variable_ffs(int x)
107{
108 asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0,
109 RISCV_ISA_EXT_ZBB, 1)
110 : : : : legacy);
111
112 if (!x)
113 return 0;
114
115 asm volatile (".option push\n"
116 ".option arch,+zbb\n"
117 CTZW "%0, %1\n"
118 ".option pop\n"
119 : "=r" (x) : "r" (x) :);
120
121 return x + 1;
122
123legacy:
124 return generic_ffs(x);
125}
126
127/**
128 * ffs - find first set bit in a word
129 * @x: the word to search
130 *
131 * This is defined the same way as the libc and compiler builtin ffs routines.
132 *
133 * ffs(value) returns 0 if value is 0 or the position of the first set bit if
134 * value is nonzero. The first (least significant) bit is at position 1.
135 */
136#define ffs(x) (__builtin_constant_p(x) ? __builtin_ffs(x) : variable_ffs(x))
137
138static __always_inline int variable_fls(unsigned int x)
139{
140 asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0,
141 RISCV_ISA_EXT_ZBB, 1)
142 : : : : legacy);
143
144 if (!x)
145 return 0;
146
147 asm volatile (".option push\n"
148 ".option arch,+zbb\n"
149 CLZW "%0, %1\n"
150 ".option pop\n"
151 : "=r" (x) : "r" (x) :);
152
153 return 32 - x;
154
155legacy:
156 return generic_fls(x);
157}
158
159/**
160 * fls - find last set bit in a word
161 * @x: the word to search
162 *
163 * This is defined in a similar way as ffs, but returns the position of the most
164 * significant set bit.
165 *
166 * fls(value) returns 0 if value is 0 or the position of the last set bit if
167 * value is nonzero. The last (most significant) bit is at position 32.
168 */
169#define fls(x) \
170({ \
171 typeof(x) x_ = (x); \
172 __builtin_constant_p(x_) ? \
173 (int)((x_ != 0) ? (32 - __builtin_clz(x_)) : 0) \
174 : \
175 variable_fls(x_); \
176})
177
178#endif /* !defined(CONFIG_RISCV_ISA_ZBB) || defined(NO_ALTERNATIVE) */
179
180#include <asm-generic/bitops/ffz.h>
181#include <asm-generic/bitops/fls64.h>
182#include <asm-generic/bitops/sched.h>
183
184#include <asm/arch_hweight.h>
185
186#include <asm-generic/bitops/const_hweight.h>
187
188#if (BITS_PER_LONG == 64)
189#define __AMO(op) "amo" #op ".d"
190#elif (BITS_PER_LONG == 32)
191#define __AMO(op) "amo" #op ".w"
192#else
193#error "Unexpected BITS_PER_LONG"
194#endif
195
196#define __test_and_op_bit_ord(op, mod, nr, addr, ord) \
197({ \
198 unsigned long __res, __mask; \
199 __mask = BIT_MASK(nr); \
200 __asm__ __volatile__ ( \
201 __AMO(op) #ord " %0, %2, %1" \
202 : "=r" (__res), "+A" (addr[BIT_WORD(nr)]) \
203 : "r" (mod(__mask)) \
204 : "memory"); \
205 ((__res & __mask) != 0); \
206})
207
208#define __op_bit_ord(op, mod, nr, addr, ord) \
209 __asm__ __volatile__ ( \
210 __AMO(op) #ord " zero, %1, %0" \
211 : "+A" (addr[BIT_WORD(nr)]) \
212 : "r" (mod(BIT_MASK(nr))) \
213 : "memory");
214
215#define __test_and_op_bit(op, mod, nr, addr) \
216 __test_and_op_bit_ord(op, mod, nr, addr, .aqrl)
217#define __op_bit(op, mod, nr, addr) \
218 __op_bit_ord(op, mod, nr, addr, )
219
220/* Bitmask modifiers */
221#define __NOP(x) (x)
222#define __NOT(x) (~(x))
223
224/**
225 * test_and_set_bit - Set a bit and return its old value
226 * @nr: Bit to set
227 * @addr: Address to count from
228 *
229 * This operation may be reordered on other architectures than x86.
230 */
231static inline int test_and_set_bit(int nr, volatile unsigned long *addr)
232{
233 return __test_and_op_bit(or, __NOP, nr, addr);
234}
235
236/**
237 * test_and_clear_bit - Clear a bit and return its old value
238 * @nr: Bit to clear
239 * @addr: Address to count from
240 *
241 * This operation can be reordered on other architectures other than x86.
242 */
243static inline int test_and_clear_bit(int nr, volatile unsigned long *addr)
244{
245 return __test_and_op_bit(and, __NOT, nr, addr);
246}
247
248/**
249 * test_and_change_bit - Change a bit and return its old value
250 * @nr: Bit to change
251 * @addr: Address to count from
252 *
253 * This operation is atomic and cannot be reordered.
254 * It also implies a memory barrier.
255 */
256static inline int test_and_change_bit(int nr, volatile unsigned long *addr)
257{
258 return __test_and_op_bit(xor, __NOP, nr, addr);
259}
260
261/**
262 * set_bit - Atomically set a bit in memory
263 * @nr: the bit to set
264 * @addr: the address to start counting from
265 *
266 * Note: there are no guarantees that this function will not be reordered
267 * on non x86 architectures, so if you are writing portable code,
268 * make sure not to rely on its reordering guarantees.
269 *
270 * Note that @nr may be almost arbitrarily large; this function is not
271 * restricted to acting on a single-word quantity.
272 */
273static inline void set_bit(int nr, volatile unsigned long *addr)
274{
275 __op_bit(or, __NOP, nr, addr);
276}
277
278/**
279 * clear_bit - Clears a bit in memory
280 * @nr: Bit to clear
281 * @addr: Address to start counting from
282 *
283 * Note: there are no guarantees that this function will not be reordered
284 * on non x86 architectures, so if you are writing portable code,
285 * make sure not to rely on its reordering guarantees.
286 */
287static inline void clear_bit(int nr, volatile unsigned long *addr)
288{
289 __op_bit(and, __NOT, nr, addr);
290}
291
292/**
293 * change_bit - Toggle a bit in memory
294 * @nr: Bit to change
295 * @addr: Address to start counting from
296 *
297 * change_bit() may be reordered on other architectures than x86.
298 * Note that @nr may be almost arbitrarily large; this function is not
299 * restricted to acting on a single-word quantity.
300 */
301static inline void change_bit(int nr, volatile unsigned long *addr)
302{
303 __op_bit(xor, __NOP, nr, addr);
304}
305
306/**
307 * test_and_set_bit_lock - Set a bit and return its old value, for lock
308 * @nr: Bit to set
309 * @addr: Address to count from
310 *
311 * This operation is atomic and provides acquire barrier semantics.
312 * It can be used to implement bit locks.
313 */
314static inline int test_and_set_bit_lock(
315 unsigned long nr, volatile unsigned long *addr)
316{
317 return __test_and_op_bit_ord(or, __NOP, nr, addr, .aq);
318}
319
320/**
321 * clear_bit_unlock - Clear a bit in memory, for unlock
322 * @nr: the bit to set
323 * @addr: the address to start counting from
324 *
325 * This operation is atomic and provides release barrier semantics.
326 */
327static inline void clear_bit_unlock(
328 unsigned long nr, volatile unsigned long *addr)
329{
330 __op_bit_ord(and, __NOT, nr, addr, .rl);
331}
332
333/**
334 * __clear_bit_unlock - Clear a bit in memory, for unlock
335 * @nr: the bit to set
336 * @addr: the address to start counting from
337 *
338 * This operation is like clear_bit_unlock, however it is not atomic.
339 * It does provide release barrier semantics so it can be used to unlock
340 * a bit lock, however it would only be used if no other CPU can modify
341 * any bits in the memory until the lock is released (a good example is
342 * if the bit lock itself protects access to the other bits in the word).
343 *
344 * On RISC-V systems there seems to be no benefit to taking advantage of the
345 * non-atomic property here: it's a lot more instructions and we still have to
346 * provide release semantics anyway.
347 */
348static inline void __clear_bit_unlock(
349 unsigned long nr, volatile unsigned long *addr)
350{
351 clear_bit_unlock(nr, addr);
352}
353
354static inline bool xor_unlock_is_negative_byte(unsigned long mask,
355 volatile unsigned long *addr)
356{
357 unsigned long res;
358 __asm__ __volatile__ (
359 __AMO(xor) ".rl %0, %2, %1"
360 : "=r" (res), "+A" (*addr)
361 : "r" (__NOP(mask))
362 : "memory");
363 return (res & BIT(7)) != 0;
364}
365
366#undef __test_and_op_bit
367#undef __op_bit
368#undef __NOP
369#undef __NOT
370#undef __AMO
371
372#include <asm-generic/bitops/non-atomic.h>
373#include <asm-generic/bitops/le.h>
374#include <asm-generic/bitops/ext2-atomic.h>
375
376#endif /* _ASM_RISCV_BITOPS_H */