1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 4 */ 5 6 #ifndef _ASM_BITOPS_H 7 #define _ASM_BITOPS_H 8 9 #ifndef _LINUX_BITOPS_H 10 #error only <linux/bitops.h> can be included directly 11 #endif 12 13 #ifndef __ASSEMBLY__ 14 15 #include <linux/types.h> 16 #include <linux/compiler.h> 17 #include <asm/barrier.h> 18 #ifndef CONFIG_ARC_HAS_LLSC 19 #include <asm/smp.h> 20 #endif 21 22 #ifdef CONFIG_ARC_HAS_LLSC 23 24 /* 25 * Hardware assisted Atomic-R-M-W 26 */ 27 28 #define BIT_OP(op, c_op, asm_op) \ 29 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\ 30 { \ 31 unsigned int temp; \ 32 \ 33 m += nr >> 5; \ 34 \ 35 nr &= 0x1f; \ 36 \ 37 __asm__ __volatile__( \ 38 "1: llock %0, [%1] \n" \ 39 " " #asm_op " %0, %0, %2 \n" \ 40 " scond %0, [%1] \n" \ 41 " bnz 1b \n" \ 42 : "=&r"(temp) /* Early clobber, to prevent reg reuse */ \ 43 : "r"(m), /* Not "m": llock only supports reg direct addr mode */ \ 44 "ir"(nr) \ 45 : "cc"); \ 46 } 47 48 /* 49 * Semantically: 50 * Test the bit 51 * if clear 52 * set it and return 0 (old value) 53 * else 54 * return 1 (old value). 55 * 56 * Since ARC lacks a equivalent h/w primitive, the bit is set unconditionally 57 * and the old value of bit is returned 58 */ 59 #define TEST_N_BIT_OP(op, c_op, asm_op) \ 60 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\ 61 { \ 62 unsigned long old, temp; \ 63 \ 64 m += nr >> 5; \ 65 \ 66 nr &= 0x1f; \ 67 \ 68 /* \ 69 * Explicit full memory barrier needed before/after as \ 70 * LLOCK/SCOND themselves don't provide any such smenatic \ 71 */ \ 72 smp_mb(); \ 73 \ 74 __asm__ __volatile__( \ 75 "1: llock %0, [%2] \n" \ 76 " " #asm_op " %1, %0, %3 \n" \ 77 " scond %1, [%2] \n" \ 78 " bnz 1b \n" \ 79 : "=&r"(old), "=&r"(temp) \ 80 : "r"(m), "ir"(nr) \ 81 : "cc"); \ 82 \ 83 smp_mb(); \ 84 \ 85 return (old & (1 << nr)) != 0; \ 86 } 87 88 #elif !defined(CONFIG_ARC_PLAT_EZNPS) 89 90 /* 91 * Non hardware assisted Atomic-R-M-W 92 * Locking would change to irq-disabling only (UP) and spinlocks (SMP) 93 * 94 * There's "significant" micro-optimization in writing our own variants of 95 * bitops (over generic variants) 96 * 97 * (1) The generic APIs have "signed" @nr while we have it "unsigned" 98 * This avoids extra code to be generated for pointer arithmatic, since 99 * is "not sure" that index is NOT -ve 100 * (2) Utilize the fact that ARCompact bit fidding insn (BSET/BCLR/ASL) etc 101 * only consider bottom 5 bits of @nr, so NO need to mask them off. 102 * (GCC Quirk: however for constant @nr we still need to do the masking 103 * at compile time) 104 */ 105 106 #define BIT_OP(op, c_op, asm_op) \ 107 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\ 108 { \ 109 unsigned long temp, flags; \ 110 m += nr >> 5; \ 111 \ 112 /* \ 113 * spin lock/unlock provide the needed smp_mb() before/after \ 114 */ \ 115 bitops_lock(flags); \ 116 \ 117 temp = *m; \ 118 *m = temp c_op (1UL << (nr & 0x1f)); \ 119 \ 120 bitops_unlock(flags); \ 121 } 122 123 #define TEST_N_BIT_OP(op, c_op, asm_op) \ 124 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\ 125 { \ 126 unsigned long old, flags; \ 127 m += nr >> 5; \ 128 \ 129 bitops_lock(flags); \ 130 \ 131 old = *m; \ 132 *m = old c_op (1UL << (nr & 0x1f)); \ 133 \ 134 bitops_unlock(flags); \ 135 \ 136 return (old & (1UL << (nr & 0x1f))) != 0; \ 137 } 138 139 #else /* CONFIG_ARC_PLAT_EZNPS */ 140 141 #define BIT_OP(op, c_op, asm_op) \ 142 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\ 143 { \ 144 m += nr >> 5; \ 145 \ 146 nr = (1UL << (nr & 0x1f)); \ 147 if (asm_op == CTOP_INST_AAND_DI_R2_R2_R3) \ 148 nr = ~nr; \ 149 \ 150 __asm__ __volatile__( \ 151 " mov r2, %0\n" \ 152 " mov r3, %1\n" \ 153 " .word %2\n" \ 154 : \ 155 : "r"(nr), "r"(m), "i"(asm_op) \ 156 : "r2", "r3", "memory"); \ 157 } 158 159 #define TEST_N_BIT_OP(op, c_op, asm_op) \ 160 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\ 161 { \ 162 unsigned long old; \ 163 \ 164 m += nr >> 5; \ 165 \ 166 nr = old = (1UL << (nr & 0x1f)); \ 167 if (asm_op == CTOP_INST_AAND_DI_R2_R2_R3) \ 168 old = ~old; \ 169 \ 170 /* Explicit full memory barrier needed before/after */ \ 171 smp_mb(); \ 172 \ 173 __asm__ __volatile__( \ 174 " mov r2, %0\n" \ 175 " mov r3, %1\n" \ 176 " .word %2\n" \ 177 " mov %0, r2" \ 178 : "+r"(old) \ 179 : "r"(m), "i"(asm_op) \ 180 : "r2", "r3", "memory"); \ 181 \ 182 smp_mb(); \ 183 \ 184 return (old & nr) != 0; \ 185 } 186 187 #endif /* CONFIG_ARC_PLAT_EZNPS */ 188 189 /*************************************** 190 * Non atomic variants 191 **************************************/ 192 193 #define __BIT_OP(op, c_op, asm_op) \ 194 static inline void __##op##_bit(unsigned long nr, volatile unsigned long *m) \ 195 { \ 196 unsigned long temp; \ 197 m += nr >> 5; \ 198 \ 199 temp = *m; \ 200 *m = temp c_op (1UL << (nr & 0x1f)); \ 201 } 202 203 #define __TEST_N_BIT_OP(op, c_op, asm_op) \ 204 static inline int __test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\ 205 { \ 206 unsigned long old; \ 207 m += nr >> 5; \ 208 \ 209 old = *m; \ 210 *m = old c_op (1UL << (nr & 0x1f)); \ 211 \ 212 return (old & (1UL << (nr & 0x1f))) != 0; \ 213 } 214 215 #define BIT_OPS(op, c_op, asm_op) \ 216 \ 217 /* set_bit(), clear_bit(), change_bit() */ \ 218 BIT_OP(op, c_op, asm_op) \ 219 \ 220 /* test_and_set_bit(), test_and_clear_bit(), test_and_change_bit() */\ 221 TEST_N_BIT_OP(op, c_op, asm_op) \ 222 \ 223 /* __set_bit(), __clear_bit(), __change_bit() */ \ 224 __BIT_OP(op, c_op, asm_op) \ 225 \ 226 /* __test_and_set_bit(), __test_and_clear_bit(), __test_and_change_bit() */\ 227 __TEST_N_BIT_OP(op, c_op, asm_op) 228 229 #ifndef CONFIG_ARC_PLAT_EZNPS 230 BIT_OPS(set, |, bset) 231 BIT_OPS(clear, & ~, bclr) 232 BIT_OPS(change, ^, bxor) 233 #else 234 BIT_OPS(set, |, CTOP_INST_AOR_DI_R2_R2_R3) 235 BIT_OPS(clear, & ~, CTOP_INST_AAND_DI_R2_R2_R3) 236 BIT_OPS(change, ^, CTOP_INST_AXOR_DI_R2_R2_R3) 237 #endif 238 239 /* 240 * This routine doesn't need to be atomic. 241 */ 242 static inline int 243 test_bit(unsigned int nr, const volatile unsigned long *addr) 244 { 245 unsigned long mask; 246 247 addr += nr >> 5; 248 249 mask = 1UL << (nr & 0x1f); 250 251 return ((mask & *addr) != 0); 252 } 253 254 #ifdef CONFIG_ISA_ARCOMPACT 255 256 /* 257 * Count the number of zeros, starting from MSB 258 * Helper for fls( ) friends 259 * This is a pure count, so (1-32) or (0-31) doesn't apply 260 * It could be 0 to 32, based on num of 0's in there 261 * clz(0x8000_0000) = 0, clz(0xFFFF_FFFF)=0, clz(0) = 32, clz(1) = 31 262 */ 263 static inline __attribute__ ((const)) int clz(unsigned int x) 264 { 265 unsigned int res; 266 267 __asm__ __volatile__( 268 " norm.f %0, %1 \n" 269 " mov.n %0, 0 \n" 270 " add.p %0, %0, 1 \n" 271 : "=r"(res) 272 : "r"(x) 273 : "cc"); 274 275 return res; 276 } 277 278 static inline int constant_fls(unsigned int x) 279 { 280 int r = 32; 281 282 if (!x) 283 return 0; 284 if (!(x & 0xffff0000u)) { 285 x <<= 16; 286 r -= 16; 287 } 288 if (!(x & 0xff000000u)) { 289 x <<= 8; 290 r -= 8; 291 } 292 if (!(x & 0xf0000000u)) { 293 x <<= 4; 294 r -= 4; 295 } 296 if (!(x & 0xc0000000u)) { 297 x <<= 2; 298 r -= 2; 299 } 300 if (!(x & 0x80000000u)) { 301 x <<= 1; 302 r -= 1; 303 } 304 return r; 305 } 306 307 /* 308 * fls = Find Last Set in word 309 * @result: [1-32] 310 * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0 311 */ 312 static inline __attribute__ ((const)) int fls(unsigned int x) 313 { 314 if (__builtin_constant_p(x)) 315 return constant_fls(x); 316 317 return 32 - clz(x); 318 } 319 320 /* 321 * __fls: Similar to fls, but zero based (0-31) 322 */ 323 static inline __attribute__ ((const)) int __fls(unsigned long x) 324 { 325 if (!x) 326 return 0; 327 else 328 return fls(x) - 1; 329 } 330 331 /* 332 * ffs = Find First Set in word (LSB to MSB) 333 * @result: [1-32], 0 if all 0's 334 */ 335 #define ffs(x) ({ unsigned long __t = (x); fls(__t & -__t); }) 336 337 /* 338 * __ffs: Similar to ffs, but zero based (0-31) 339 */ 340 static inline __attribute__ ((const)) unsigned long __ffs(unsigned long word) 341 { 342 if (!word) 343 return word; 344 345 return ffs(word) - 1; 346 } 347 348 #else /* CONFIG_ISA_ARCV2 */ 349 350 /* 351 * fls = Find Last Set in word 352 * @result: [1-32] 353 * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0 354 */ 355 static inline __attribute__ ((const)) int fls(unsigned long x) 356 { 357 int n; 358 359 asm volatile( 360 " fls.f %0, %1 \n" /* 0:31; 0(Z) if src 0 */ 361 " add.nz %0, %0, 1 \n" /* 0:31 -> 1:32 */ 362 : "=r"(n) /* Early clobber not needed */ 363 : "r"(x) 364 : "cc"); 365 366 return n; 367 } 368 369 /* 370 * __fls: Similar to fls, but zero based (0-31). Also 0 if no bit set 371 */ 372 static inline __attribute__ ((const)) int __fls(unsigned long x) 373 { 374 /* FLS insn has exactly same semantics as the API */ 375 return __builtin_arc_fls(x); 376 } 377 378 /* 379 * ffs = Find First Set in word (LSB to MSB) 380 * @result: [1-32], 0 if all 0's 381 */ 382 static inline __attribute__ ((const)) int ffs(unsigned long x) 383 { 384 int n; 385 386 asm volatile( 387 " ffs.f %0, %1 \n" /* 0:31; 31(Z) if src 0 */ 388 " add.nz %0, %0, 1 \n" /* 0:31 -> 1:32 */ 389 " mov.z %0, 0 \n" /* 31(Z)-> 0 */ 390 : "=r"(n) /* Early clobber not needed */ 391 : "r"(x) 392 : "cc"); 393 394 return n; 395 } 396 397 /* 398 * __ffs: Similar to ffs, but zero based (0-31) 399 */ 400 static inline __attribute__ ((const)) unsigned long __ffs(unsigned long x) 401 { 402 unsigned long n; 403 404 asm volatile( 405 " ffs.f %0, %1 \n" /* 0:31; 31(Z) if src 0 */ 406 " mov.z %0, 0 \n" /* 31(Z)-> 0 */ 407 : "=r"(n) 408 : "r"(x) 409 : "cc"); 410 411 return n; 412 413 } 414 415 #endif /* CONFIG_ISA_ARCOMPACT */ 416 417 /* 418 * ffz = Find First Zero in word. 419 * @return:[0-31], 32 if all 1's 420 */ 421 #define ffz(x) __ffs(~(x)) 422 423 #include <asm-generic/bitops/hweight.h> 424 #include <asm-generic/bitops/fls64.h> 425 #include <asm-generic/bitops/sched.h> 426 #include <asm-generic/bitops/lock.h> 427 428 #include <asm-generic/bitops/find.h> 429 #include <asm-generic/bitops/le.h> 430 #include <asm-generic/bitops/ext2-atomic-setbit.h> 431 432 #endif /* !__ASSEMBLY__ */ 433 434 #endif 435