1 /* SPDX-License-Identifier: GPL-2.0 */
32  * These have to be done with inline assembly: that way the bit-setting
48 #define CONST_MASK_ADDR(nr, addr)	WBYTE_ADDR((void *)(addr) + ((nr)>>3))  argument
49 #define CONST_MASK(nr)			(1 << ((nr) & 7))  argument
52 arch_set_bit(long nr, volatile unsigned long *addr)  in arch_set_bit()  argument
54 	if (__builtin_constant_p(nr)) {  in arch_set_bit()
56 			: CONST_MASK_ADDR(nr, addr)  in arch_set_bit()
57 			: "iq" (CONST_MASK(nr))  in arch_set_bit()
61 			: : RLONG_ADDR(addr), "Ir" (nr) : "memory");  in arch_set_bit()
66 arch___set_bit(unsigned long nr, volatile unsigned long *addr)  in arch___set_bit()  argument
68 	asm volatile(__ASM_SIZE(bts) " %1,%0" : : ADDR, "Ir" (nr) : "memory");  in arch___set_bit()
72 arch_clear_bit(long nr, volatile unsigned long *addr)  in arch_clear_bit()  argument
74 	if (__builtin_constant_p(nr)) {  in arch_clear_bit()
76 			: CONST_MASK_ADDR(nr, addr)  in arch_clear_bit()
77 			: "iq" (~CONST_MASK(nr)));  in arch_clear_bit()
80 			: : RLONG_ADDR(addr), "Ir" (nr) : "memory");  in arch_clear_bit()
85 arch_clear_bit_unlock(long nr, volatile unsigned long *addr)  in arch_clear_bit_unlock()  argument
88 	arch_clear_bit(nr, addr);  in arch_clear_bit_unlock()
92 arch___clear_bit(unsigned long nr, volatile unsigned long *addr)  in arch___clear_bit()  argument
94 	asm volatile(__ASM_SIZE(btr) " %1,%0" : : ADDR, "Ir" (nr) : "memory");  in arch___clear_bit()
98 arch_clear_bit_unlock_is_negative_byte(long nr, volatile unsigned long *addr)  in arch_clear_bit_unlock_is_negative_byte()  argument
104 		: "ir" ((char) ~(1 << nr)) : "memory");  in arch_clear_bit_unlock_is_negative_byte()
111 arch___clear_bit_unlock(long nr, volatile unsigned long *addr)  in arch___clear_bit_unlock()  argument
113 	arch___clear_bit(nr, addr);  in arch___clear_bit_unlock()
117 arch___change_bit(unsigned long nr, volatile unsigned long *addr)  in arch___change_bit()  argument
119 	asm volatile(__ASM_SIZE(btc) " %1,%0" : : ADDR, "Ir" (nr) : "memory");  in arch___change_bit()
123 arch_change_bit(long nr, volatile unsigned long *addr)  in arch_change_bit()  argument
125 	if (__builtin_constant_p(nr)) {  in arch_change_bit()
127 			: CONST_MASK_ADDR(nr, addr)  in arch_change_bit()
128 			: "iq" (CONST_MASK(nr)));  in arch_change_bit()
131 			: : RLONG_ADDR(addr), "Ir" (nr) : "memory");  in arch_change_bit()
136 arch_test_and_set_bit(long nr, volatile unsigned long *addr)  in arch_test_and_set_bit()  argument
138 	return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(bts), *addr, c, "Ir", nr);  in arch_test_and_set_bit()
142 arch_test_and_set_bit_lock(long nr, volatile unsigned long *addr)  in arch_test_and_set_bit_lock()  argument
144 	return arch_test_and_set_bit(nr, addr);  in arch_test_and_set_bit_lock()
148 arch___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)  in arch___test_and_set_bit()  argument
155 	    : ADDR, "Ir" (nr) : "memory");  in arch___test_and_set_bit()
160 arch_test_and_clear_bit(long nr, volatile unsigned long *addr)  in arch_test_and_clear_bit()  argument
162 	return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btr), *addr, c, "Ir", nr);  in arch_test_and_clear_bit()
167  * the local CPU, but not other CPUs. Portable code should not
170  * accessed from a hypervisor on the same CPU if running in a VM: don't change
174 arch___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)  in arch___test_and_clear_bit()  argument
181 		     : ADDR, "Ir" (nr) : "memory");  in arch___test_and_clear_bit()
186 arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)  in arch___test_and_change_bit()  argument
193 		     : ADDR, "Ir" (nr) : "memory");  in arch___test_and_change_bit()
199 arch_test_and_change_bit(long nr, volatile unsigned long *addr)  in arch_test_and_change_bit()  argument
201 	return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btc), *addr, c, "Ir", nr);  in arch_test_and_change_bit()
204 static __always_inline bool constant_test_bit(long nr, const volatile unsigned long *addr)  in constant_test_bit()  argument
206 	return ((1UL << (nr & (BITS_PER_LONG-1))) &  in constant_test_bit()
207 		(addr[nr >> _BITOPS_LONG_SHIFT])) != 0;  in constant_test_bit()
210 static __always_inline bool constant_test_bit_acquire(long nr, const volatile unsigned long *addr)  in constant_test_bit_acquire()  argument
217 		     : "m" (((unsigned char *)addr)[nr >> 3]),  in constant_test_bit_acquire()
218 		       "i" (1 << (nr & 7))  in constant_test_bit_acquire()
224 static __always_inline bool variable_test_bit(long nr, volatile const unsigned long *addr)  in variable_test_bit()  argument
231 		     : "m" (*(unsigned long *)addr), "Ir" (nr) : "memory");  in variable_test_bit()
237 arch_test_bit(unsigned long nr, const volatile unsigned long *addr)  in arch_test_bit()  argument
239 	return __builtin_constant_p(nr) ? constant_test_bit(nr, addr) :  in arch_test_bit()
240 					  variable_test_bit(nr, addr);  in arch_test_bit()
244 arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)  in arch_test_bit_acquire()  argument
246 	return __builtin_constant_p(nr) ? constant_test_bit_acquire(nr, addr) :  in arch_test_bit_acquire()
247 					  variable_test_bit(nr, addr);  in arch_test_bit_acquire()
259  * __ffs - find first set bit in word
278  * ffz - find first zero bit in word
312 	 * dest reg is undefined if x==0, but their CPU architect says its  in variable_ffs()
321 	    : "rm" (x), "0" (-1));  in variable_ffs()
325 	    : "=&r" (r) : "rm" (x), "r" (-1));  in variable_ffs()
329 	    "movl $-1,%0\n"  in variable_ffs()
336  * ffs - find first set bit in word
349  * fls - find last set bit in word
366 	 * dest reg is undefined if x==0, but their CPU architect says its  in fls()
375 	    : "rm" (x), "0" (-1));  in fls()
379 	    : "=&r" (r) : "rm" (x), "rm" (-1));  in fls()
383 	    "movl $-1,%0\n"  in fls()
390  * fls64 - find last set bit in a 64-bit word
403 	int bitpos = -1;  in fls64()
406 	 * dest reg is undefined if x==0, but their CPU architect says its  in fls64()
415 #include <asm-generic/bitops/fls64.h>
418 #include <asm-generic/bitops/sched.h>
422 #include <asm-generic/bitops/const_hweight.h>
424 #include <asm-generic/bitops/instrumented-atomic.h>
425 #include <asm-generic/bitops/instrumented-non-atomic.h>
426 #include <asm-generic/bitops/instrumented-lock.h>
428 #include <asm-generic/bitops/le.h>
430 #include <asm-generic/bitops/ext2-atomic-setbit.h>