xref: /openbmc/linux/arch/arc/include/asm/bitops.h (revision 9fb29c73)
1 /*
2  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  */
8 
9 #ifndef _ASM_BITOPS_H
10 #define _ASM_BITOPS_H
11 
12 #ifndef _LINUX_BITOPS_H
13 #error only <linux/bitops.h> can be included directly
14 #endif
15 
16 #ifndef __ASSEMBLY__
17 
18 #include <linux/types.h>
19 #include <linux/compiler.h>
20 #include <asm/barrier.h>
21 #ifndef CONFIG_ARC_HAS_LLSC
22 #include <asm/smp.h>
23 #endif
24 
25 #ifdef CONFIG_ARC_HAS_LLSC
26 
27 /*
28  * Hardware assisted Atomic-R-M-W
29  */
30 
31 #define BIT_OP(op, c_op, asm_op)					\
32 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
33 {									\
34 	unsigned int temp;						\
35 									\
36 	m += nr >> 5;							\
37 									\
38 	nr &= 0x1f;							\
39 									\
40 	__asm__ __volatile__(						\
41 	"1:	llock       %0, [%1]		\n"			\
42 	"	" #asm_op " %0, %0, %2	\n"				\
43 	"	scond       %0, [%1]		\n"			\
44 	"	bnz         1b			\n"			\
45 	: "=&r"(temp)	/* Early clobber, to prevent reg reuse */	\
46 	: "r"(m),	/* Not "m": llock only supports reg direct addr mode */	\
47 	  "ir"(nr)							\
48 	: "cc");							\
49 }
50 
51 /*
52  * Semantically:
53  *    Test the bit
54  *    if clear
55  *        set it and return 0 (old value)
56  *    else
57  *        return 1 (old value).
58  *
59  * Since ARC lacks a equivalent h/w primitive, the bit is set unconditionally
60  * and the old value of bit is returned
61  */
62 #define TEST_N_BIT_OP(op, c_op, asm_op)					\
63 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
64 {									\
65 	unsigned long old, temp;					\
66 									\
67 	m += nr >> 5;							\
68 									\
69 	nr &= 0x1f;							\
70 									\
71 	/*								\
72 	 * Explicit full memory barrier needed before/after as		\
73 	 * LLOCK/SCOND themselves don't provide any such smenatic	\
74 	 */								\
75 	smp_mb();							\
76 									\
77 	__asm__ __volatile__(						\
78 	"1:	llock       %0, [%2]	\n"				\
79 	"	" #asm_op " %1, %0, %3	\n"				\
80 	"	scond       %1, [%2]	\n"				\
81 	"	bnz         1b		\n"				\
82 	: "=&r"(old), "=&r"(temp)					\
83 	: "r"(m), "ir"(nr)						\
84 	: "cc");							\
85 									\
86 	smp_mb();							\
87 									\
88 	return (old & (1 << nr)) != 0;					\
89 }
90 
91 #elif !defined(CONFIG_ARC_PLAT_EZNPS)
92 
93 /*
94  * Non hardware assisted Atomic-R-M-W
95  * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
96  *
97  * There's "significant" micro-optimization in writing our own variants of
98  * bitops (over generic variants)
99  *
100  * (1) The generic APIs have "signed" @nr while we have it "unsigned"
101  *     This avoids extra code to be generated for pointer arithmatic, since
102  *     is "not sure" that index is NOT -ve
103  * (2) Utilize the fact that ARCompact bit fidding insn (BSET/BCLR/ASL) etc
104  *     only consider bottom 5 bits of @nr, so NO need to mask them off.
105  *     (GCC Quirk: however for constant @nr we still need to do the masking
106  *             at compile time)
107  */
108 
109 #define BIT_OP(op, c_op, asm_op)					\
110 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
111 {									\
112 	unsigned long temp, flags;					\
113 	m += nr >> 5;							\
114 									\
115 	/*								\
116 	 * spin lock/unlock provide the needed smp_mb() before/after	\
117 	 */								\
118 	bitops_lock(flags);						\
119 									\
120 	temp = *m;							\
121 	*m = temp c_op (1UL << (nr & 0x1f));					\
122 									\
123 	bitops_unlock(flags);						\
124 }
125 
126 #define TEST_N_BIT_OP(op, c_op, asm_op)					\
127 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
128 {									\
129 	unsigned long old, flags;					\
130 	m += nr >> 5;							\
131 									\
132 	bitops_lock(flags);						\
133 									\
134 	old = *m;							\
135 	*m = old c_op (1UL << (nr & 0x1f));				\
136 									\
137 	bitops_unlock(flags);						\
138 									\
139 	return (old & (1UL << (nr & 0x1f))) != 0;			\
140 }
141 
142 #else /* CONFIG_ARC_PLAT_EZNPS */
143 
144 #define BIT_OP(op, c_op, asm_op)					\
145 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
146 {									\
147 	m += nr >> 5;							\
148 									\
149 	nr = (1UL << (nr & 0x1f));					\
150 	if (asm_op == CTOP_INST_AAND_DI_R2_R2_R3)			\
151 		nr = ~nr;						\
152 									\
153 	__asm__ __volatile__(						\
154 	"	mov r2, %0\n"						\
155 	"	mov r3, %1\n"						\
156 	"	.word %2\n"						\
157 	:								\
158 	: "r"(nr), "r"(m), "i"(asm_op)					\
159 	: "r2", "r3", "memory");					\
160 }
161 
162 #define TEST_N_BIT_OP(op, c_op, asm_op)					\
163 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
164 {									\
165 	unsigned long old;						\
166 									\
167 	m += nr >> 5;							\
168 									\
169 	nr = old = (1UL << (nr & 0x1f));				\
170 	if (asm_op == CTOP_INST_AAND_DI_R2_R2_R3)			\
171 		old = ~old;						\
172 									\
173 	/* Explicit full memory barrier needed before/after */		\
174 	smp_mb();							\
175 									\
176 	__asm__ __volatile__(						\
177 	"	mov r2, %0\n"						\
178 	"	mov r3, %1\n"						\
179 	"       .word %2\n"						\
180 	"	mov %0, r2"						\
181 	: "+r"(old)							\
182 	: "r"(m), "i"(asm_op)						\
183 	: "r2", "r3", "memory");					\
184 									\
185 	smp_mb();							\
186 									\
187 	return (old & nr) != 0;					\
188 }
189 
190 #endif /* CONFIG_ARC_PLAT_EZNPS */
191 
192 /***************************************
193  * Non atomic variants
194  **************************************/
195 
196 #define __BIT_OP(op, c_op, asm_op)					\
197 static inline void __##op##_bit(unsigned long nr, volatile unsigned long *m)	\
198 {									\
199 	unsigned long temp;						\
200 	m += nr >> 5;							\
201 									\
202 	temp = *m;							\
203 	*m = temp c_op (1UL << (nr & 0x1f));				\
204 }
205 
206 #define __TEST_N_BIT_OP(op, c_op, asm_op)				\
207 static inline int __test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
208 {									\
209 	unsigned long old;						\
210 	m += nr >> 5;							\
211 									\
212 	old = *m;							\
213 	*m = old c_op (1UL << (nr & 0x1f));				\
214 									\
215 	return (old & (1UL << (nr & 0x1f))) != 0;			\
216 }
217 
218 #define BIT_OPS(op, c_op, asm_op)					\
219 									\
220 	/* set_bit(), clear_bit(), change_bit() */			\
221 	BIT_OP(op, c_op, asm_op)					\
222 									\
223 	/* test_and_set_bit(), test_and_clear_bit(), test_and_change_bit() */\
224 	TEST_N_BIT_OP(op, c_op, asm_op)					\
225 									\
226 	/* __set_bit(), __clear_bit(), __change_bit() */		\
227 	__BIT_OP(op, c_op, asm_op)					\
228 									\
229 	/* __test_and_set_bit(), __test_and_clear_bit(), __test_and_change_bit() */\
230 	__TEST_N_BIT_OP(op, c_op, asm_op)
231 
232 #ifndef CONFIG_ARC_PLAT_EZNPS
233 BIT_OPS(set, |, bset)
234 BIT_OPS(clear, & ~, bclr)
235 BIT_OPS(change, ^, bxor)
236 #else
237 BIT_OPS(set, |, CTOP_INST_AOR_DI_R2_R2_R3)
238 BIT_OPS(clear, & ~, CTOP_INST_AAND_DI_R2_R2_R3)
239 BIT_OPS(change, ^, CTOP_INST_AXOR_DI_R2_R2_R3)
240 #endif
241 
242 /*
243  * This routine doesn't need to be atomic.
244  */
245 static inline int
246 test_bit(unsigned int nr, const volatile unsigned long *addr)
247 {
248 	unsigned long mask;
249 
250 	addr += nr >> 5;
251 
252 	mask = 1UL << (nr & 0x1f);
253 
254 	return ((mask & *addr) != 0);
255 }
256 
257 #ifdef CONFIG_ISA_ARCOMPACT
258 
259 /*
260  * Count the number of zeros, starting from MSB
261  * Helper for fls( ) friends
262  * This is a pure count, so (1-32) or (0-31) doesn't apply
263  * It could be 0 to 32, based on num of 0's in there
264  * clz(0x8000_0000) = 0, clz(0xFFFF_FFFF)=0, clz(0) = 32, clz(1) = 31
265  */
266 static inline __attribute__ ((const)) int clz(unsigned int x)
267 {
268 	unsigned int res;
269 
270 	__asm__ __volatile__(
271 	"	norm.f  %0, %1		\n"
272 	"	mov.n   %0, 0		\n"
273 	"	add.p   %0, %0, 1	\n"
274 	: "=r"(res)
275 	: "r"(x)
276 	: "cc");
277 
278 	return res;
279 }
280 
281 static inline int constant_fls(unsigned int x)
282 {
283 	int r = 32;
284 
285 	if (!x)
286 		return 0;
287 	if (!(x & 0xffff0000u)) {
288 		x <<= 16;
289 		r -= 16;
290 	}
291 	if (!(x & 0xff000000u)) {
292 		x <<= 8;
293 		r -= 8;
294 	}
295 	if (!(x & 0xf0000000u)) {
296 		x <<= 4;
297 		r -= 4;
298 	}
299 	if (!(x & 0xc0000000u)) {
300 		x <<= 2;
301 		r -= 2;
302 	}
303 	if (!(x & 0x80000000u)) {
304 		x <<= 1;
305 		r -= 1;
306 	}
307 	return r;
308 }
309 
310 /*
311  * fls = Find Last Set in word
312  * @result: [1-32]
313  * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0
314  */
315 static inline __attribute__ ((const)) int fls(unsigned int x)
316 {
317 	if (__builtin_constant_p(x))
318 	       return constant_fls(x);
319 
320 	return 32 - clz(x);
321 }
322 
323 /*
324  * __fls: Similar to fls, but zero based (0-31)
325  */
326 static inline __attribute__ ((const)) int __fls(unsigned long x)
327 {
328 	if (!x)
329 		return 0;
330 	else
331 		return fls(x) - 1;
332 }
333 
334 /*
335  * ffs = Find First Set in word (LSB to MSB)
336  * @result: [1-32], 0 if all 0's
337  */
338 #define ffs(x)	({ unsigned long __t = (x); fls(__t & -__t); })
339 
340 /*
341  * __ffs: Similar to ffs, but zero based (0-31)
342  */
343 static inline __attribute__ ((const)) int __ffs(unsigned long word)
344 {
345 	if (!word)
346 		return word;
347 
348 	return ffs(word) - 1;
349 }
350 
351 #else	/* CONFIG_ISA_ARCV2 */
352 
353 /*
354  * fls = Find Last Set in word
355  * @result: [1-32]
356  * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0
357  */
358 static inline __attribute__ ((const)) int fls(unsigned long x)
359 {
360 	int n;
361 
362 	asm volatile(
363 	"	fls.f	%0, %1		\n"  /* 0:31; 0(Z) if src 0 */
364 	"	add.nz	%0, %0, 1	\n"  /* 0:31 -> 1:32 */
365 	: "=r"(n)	/* Early clobber not needed */
366 	: "r"(x)
367 	: "cc");
368 
369 	return n;
370 }
371 
372 /*
373  * __fls: Similar to fls, but zero based (0-31). Also 0 if no bit set
374  */
375 static inline __attribute__ ((const)) int __fls(unsigned long x)
376 {
377 	/* FLS insn has exactly same semantics as the API */
378 	return	__builtin_arc_fls(x);
379 }
380 
381 /*
382  * ffs = Find First Set in word (LSB to MSB)
383  * @result: [1-32], 0 if all 0's
384  */
385 static inline __attribute__ ((const)) int ffs(unsigned long x)
386 {
387 	int n;
388 
389 	asm volatile(
390 	"	ffs.f	%0, %1		\n"  /* 0:31; 31(Z) if src 0 */
391 	"	add.nz	%0, %0, 1	\n"  /* 0:31 -> 1:32 */
392 	"	mov.z	%0, 0		\n"  /* 31(Z)-> 0 */
393 	: "=r"(n)	/* Early clobber not needed */
394 	: "r"(x)
395 	: "cc");
396 
397 	return n;
398 }
399 
400 /*
401  * __ffs: Similar to ffs, but zero based (0-31)
402  */
403 static inline __attribute__ ((const)) int __ffs(unsigned long x)
404 {
405 	int n;
406 
407 	asm volatile(
408 	"	ffs.f	%0, %1		\n"  /* 0:31; 31(Z) if src 0 */
409 	"	mov.z	%0, 0		\n"  /* 31(Z)-> 0 */
410 	: "=r"(n)
411 	: "r"(x)
412 	: "cc");
413 
414 	return n;
415 
416 }
417 
418 #endif	/* CONFIG_ISA_ARCOMPACT */
419 
420 /*
421  * ffz = Find First Zero in word.
422  * @return:[0-31], 32 if all 1's
423  */
424 #define ffz(x)	__ffs(~(x))
425 
426 #include <asm-generic/bitops/hweight.h>
427 #include <asm-generic/bitops/fls64.h>
428 #include <asm-generic/bitops/sched.h>
429 #include <asm-generic/bitops/lock.h>
430 
431 #include <asm-generic/bitops/find.h>
432 #include <asm-generic/bitops/le.h>
433 #include <asm-generic/bitops/ext2-atomic-setbit.h>
434 
435 #endif /* !__ASSEMBLY__ */
436 
437 #endif
438