xref: /openbmc/linux/arch/mips/include/asm/atomic.h (revision e8e0929d) 
1 /*
2  * Atomic operations that C can't guarantee us.  Useful for
3  * resource counting etc..
4  *
5  * But use these as seldom as possible since they are much more slower
6  * than regular operations.
7  *
8  * This file is subject to the terms and conditions of the GNU General Public
9  * License.  See the file "COPYING" in the main directory of this archive
10  * for more details.
11  *
12  * Copyright (C) 1996, 97, 99, 2000, 03, 04, 06 by Ralf Baechle
13  */
14 #ifndef _ASM_ATOMIC_H
15 #define _ASM_ATOMIC_H
16 
17 #include <linux/irqflags.h>
18 #include <linux/types.h>
19 #include <asm/barrier.h>
20 #include <asm/cpu-features.h>
21 #include <asm/war.h>
22 #include <asm/system.h>
23 
24 #define ATOMIC_INIT(i)    { (i) }
25 
26 /*
27  * atomic_read - read atomic variable
28  * @v: pointer of type atomic_t
29  *
30  * Atomically reads the value of @v.
31  */
32 #define atomic_read(v)		((v)->counter)
33 
34 /*
35  * atomic_set - set atomic variable
36  * @v: pointer of type atomic_t
37  * @i: required value
38  *
39  * Atomically sets the value of @v to @i.
40  */
41 #define atomic_set(v, i)		((v)->counter = (i))
42 
43 /*
44  * atomic_add - add integer to atomic variable
45  * @i: integer value to add
46  * @v: pointer of type atomic_t
47  *
48  * Atomically adds @i to @v.
49  */
50 static __inline__ void atomic_add(int i, atomic_t * v)
51 {
52 	if (kernel_uses_llsc && R10000_LLSC_WAR) {
53 		int temp;
54 
55 		__asm__ __volatile__(
56 		"	.set	mips3					\n"
57 		"1:	ll	%0, %1		# atomic_add		\n"
58 		"	addu	%0, %2					\n"
59 		"	sc	%0, %1					\n"
60 		"	beqzl	%0, 1b					\n"
61 		"	.set	mips0					\n"
62 		: "=&r" (temp), "=m" (v->counter)
63 		: "Ir" (i), "m" (v->counter));
64 	} else if (kernel_uses_llsc) {
65 		int temp;
66 
67 		__asm__ __volatile__(
68 		"	.set	mips3					\n"
69 		"1:	ll	%0, %1		# atomic_add		\n"
70 		"	addu	%0, %2					\n"
71 		"	sc	%0, %1					\n"
72 		"	beqz	%0, 2f					\n"
73 		"	.subsection 2					\n"
74 		"2:	b	1b					\n"
75 		"	.previous					\n"
76 		"	.set	mips0					\n"
77 		: "=&r" (temp), "=m" (v->counter)
78 		: "Ir" (i), "m" (v->counter));
79 	} else {
80 		unsigned long flags;
81 
82 		raw_local_irq_save(flags);
83 		v->counter += i;
84 		raw_local_irq_restore(flags);
85 	}
86 }
87 
88 /*
89  * atomic_sub - subtract the atomic variable
90  * @i: integer value to subtract
91  * @v: pointer of type atomic_t
92  *
93  * Atomically subtracts @i from @v.
94  */
95 static __inline__ void atomic_sub(int i, atomic_t * v)
96 {
97 	if (kernel_uses_llsc && R10000_LLSC_WAR) {
98 		int temp;
99 
100 		__asm__ __volatile__(
101 		"	.set	mips3					\n"
102 		"1:	ll	%0, %1		# atomic_sub		\n"
103 		"	subu	%0, %2					\n"
104 		"	sc	%0, %1					\n"
105 		"	beqzl	%0, 1b					\n"
106 		"	.set	mips0					\n"
107 		: "=&r" (temp), "=m" (v->counter)
108 		: "Ir" (i), "m" (v->counter));
109 	} else if (kernel_uses_llsc) {
110 		int temp;
111 
112 		__asm__ __volatile__(
113 		"	.set	mips3					\n"
114 		"1:	ll	%0, %1		# atomic_sub		\n"
115 		"	subu	%0, %2					\n"
116 		"	sc	%0, %1					\n"
117 		"	beqz	%0, 2f					\n"
118 		"	.subsection 2					\n"
119 		"2:	b	1b					\n"
120 		"	.previous					\n"
121 		"	.set	mips0					\n"
122 		: "=&r" (temp), "=m" (v->counter)
123 		: "Ir" (i), "m" (v->counter));
124 	} else {
125 		unsigned long flags;
126 
127 		raw_local_irq_save(flags);
128 		v->counter -= i;
129 		raw_local_irq_restore(flags);
130 	}
131 }
132 
133 /*
134  * Same as above, but return the result value
135  */
136 static __inline__ int atomic_add_return(int i, atomic_t * v)
137 {
138 	int result;
139 
140 	smp_llsc_mb();
141 
142 	if (kernel_uses_llsc && R10000_LLSC_WAR) {
143 		int temp;
144 
145 		__asm__ __volatile__(
146 		"	.set	mips3					\n"
147 		"1:	ll	%1, %2		# atomic_add_return	\n"
148 		"	addu	%0, %1, %3				\n"
149 		"	sc	%0, %2					\n"
150 		"	beqzl	%0, 1b					\n"
151 		"	addu	%0, %1, %3				\n"
152 		"	.set	mips0					\n"
153 		: "=&r" (result), "=&r" (temp), "=m" (v->counter)
154 		: "Ir" (i), "m" (v->counter)
155 		: "memory");
156 	} else if (kernel_uses_llsc) {
157 		int temp;
158 
159 		__asm__ __volatile__(
160 		"	.set	mips3					\n"
161 		"1:	ll	%1, %2		# atomic_add_return	\n"
162 		"	addu	%0, %1, %3				\n"
163 		"	sc	%0, %2					\n"
164 		"	beqz	%0, 2f					\n"
165 		"	addu	%0, %1, %3				\n"
166 		"	.subsection 2					\n"
167 		"2:	b	1b					\n"
168 		"	.previous					\n"
169 		"	.set	mips0					\n"
170 		: "=&r" (result), "=&r" (temp), "=m" (v->counter)
171 		: "Ir" (i), "m" (v->counter)
172 		: "memory");
173 	} else {
174 		unsigned long flags;
175 
176 		raw_local_irq_save(flags);
177 		result = v->counter;
178 		result += i;
179 		v->counter = result;
180 		raw_local_irq_restore(flags);
181 	}
182 
183 	smp_llsc_mb();
184 
185 	return result;
186 }
187 
188 static __inline__ int atomic_sub_return(int i, atomic_t * v)
189 {
190 	int result;
191 
192 	smp_llsc_mb();
193 
194 	if (kernel_uses_llsc && R10000_LLSC_WAR) {
195 		int temp;
196 
197 		__asm__ __volatile__(
198 		"	.set	mips3					\n"
199 		"1:	ll	%1, %2		# atomic_sub_return	\n"
200 		"	subu	%0, %1, %3				\n"
201 		"	sc	%0, %2					\n"
202 		"	beqzl	%0, 1b					\n"
203 		"	subu	%0, %1, %3				\n"
204 		"	.set	mips0					\n"
205 		: "=&r" (result), "=&r" (temp), "=m" (v->counter)
206 		: "Ir" (i), "m" (v->counter)
207 		: "memory");
208 	} else if (kernel_uses_llsc) {
209 		int temp;
210 
211 		__asm__ __volatile__(
212 		"	.set	mips3					\n"
213 		"1:	ll	%1, %2		# atomic_sub_return	\n"
214 		"	subu	%0, %1, %3				\n"
215 		"	sc	%0, %2					\n"
216 		"	beqz	%0, 2f					\n"
217 		"	subu	%0, %1, %3				\n"
218 		"	.subsection 2					\n"
219 		"2:	b	1b					\n"
220 		"	.previous					\n"
221 		"	.set	mips0					\n"
222 		: "=&r" (result), "=&r" (temp), "=m" (v->counter)
223 		: "Ir" (i), "m" (v->counter)
224 		: "memory");
225 	} else {
226 		unsigned long flags;
227 
228 		raw_local_irq_save(flags);
229 		result = v->counter;
230 		result -= i;
231 		v->counter = result;
232 		raw_local_irq_restore(flags);
233 	}
234 
235 	smp_llsc_mb();
236 
237 	return result;
238 }
239 
240 /*
241  * atomic_sub_if_positive - conditionally subtract integer from atomic variable
242  * @i: integer value to subtract
243  * @v: pointer of type atomic_t
244  *
245  * Atomically test @v and subtract @i if @v is greater or equal than @i.
246  * The function returns the old value of @v minus @i.
247  */
248 static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
249 {
250 	int result;
251 
252 	smp_llsc_mb();
253 
254 	if (kernel_uses_llsc && R10000_LLSC_WAR) {
255 		int temp;
256 
257 		__asm__ __volatile__(
258 		"	.set	mips3					\n"
259 		"1:	ll	%1, %2		# atomic_sub_if_positive\n"
260 		"	subu	%0, %1, %3				\n"
261 		"	bltz	%0, 1f					\n"
262 		"	sc	%0, %2					\n"
263 		"	.set	noreorder				\n"
264 		"	beqzl	%0, 1b					\n"
265 		"	 subu	%0, %1, %3				\n"
266 		"	.set	reorder					\n"
267 		"1:							\n"
268 		"	.set	mips0					\n"
269 		: "=&r" (result), "=&r" (temp), "=m" (v->counter)
270 		: "Ir" (i), "m" (v->counter)
271 		: "memory");
272 	} else if (kernel_uses_llsc) {
273 		int temp;
274 
275 		__asm__ __volatile__(
276 		"	.set	mips3					\n"
277 		"1:	ll	%1, %2		# atomic_sub_if_positive\n"
278 		"	subu	%0, %1, %3				\n"
279 		"	bltz	%0, 1f					\n"
280 		"	sc	%0, %2					\n"
281 		"	.set	noreorder				\n"
282 		"	beqz	%0, 2f					\n"
283 		"	 subu	%0, %1, %3				\n"
284 		"	.set	reorder					\n"
285 		"	.subsection 2					\n"
286 		"2:	b	1b					\n"
287 		"	.previous					\n"
288 		"1:							\n"
289 		"	.set	mips0					\n"
290 		: "=&r" (result), "=&r" (temp), "=m" (v->counter)
291 		: "Ir" (i), "m" (v->counter)
292 		: "memory");
293 	} else {
294 		unsigned long flags;
295 
296 		raw_local_irq_save(flags);
297 		result = v->counter;
298 		result -= i;
299 		if (result >= 0)
300 			v->counter = result;
301 		raw_local_irq_restore(flags);
302 	}
303 
304 	smp_llsc_mb();
305 
306 	return result;
307 }
308 
309 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
310 #define atomic_xchg(v, new) (xchg(&((v)->counter), (new)))
311 
312 /**
313  * atomic_add_unless - add unless the number is a given value
314  * @v: pointer of type atomic_t
315  * @a: the amount to add to v...
316  * @u: ...unless v is equal to u.
317  *
318  * Atomically adds @a to @v, so long as it was not @u.
319  * Returns non-zero if @v was not @u, and zero otherwise.
320  */
321 static __inline__ int atomic_add_unless(atomic_t *v, int a, int u)
322 {
323 	int c, old;
324 	c = atomic_read(v);
325 	for (;;) {
326 		if (unlikely(c == (u)))
327 			break;
328 		old = atomic_cmpxchg((v), c, c + (a));
329 		if (likely(old == c))
330 			break;
331 		c = old;
332 	}
333 	return c != (u);
334 }
335 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
336 
337 #define atomic_dec_return(v) atomic_sub_return(1, (v))
338 #define atomic_inc_return(v) atomic_add_return(1, (v))
339 
340 /*
341  * atomic_sub_and_test - subtract value from variable and test result
342  * @i: integer value to subtract
343  * @v: pointer of type atomic_t
344  *
345  * Atomically subtracts @i from @v and returns
346  * true if the result is zero, or false for all
347  * other cases.
348  */
349 #define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0)
350 
351 /*
352  * atomic_inc_and_test - increment and test
353  * @v: pointer of type atomic_t
354  *
355  * Atomically increments @v by 1
356  * and returns true if the result is zero, or false for all
357  * other cases.
358  */
359 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
360 
361 /*
362  * atomic_dec_and_test - decrement by 1 and test
363  * @v: pointer of type atomic_t
364  *
365  * Atomically decrements @v by 1 and
366  * returns true if the result is 0, or false for all other
367  * cases.
368  */
369 #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0)
370 
371 /*
372  * atomic_dec_if_positive - decrement by 1 if old value positive
373  * @v: pointer of type atomic_t
374  */
375 #define atomic_dec_if_positive(v)	atomic_sub_if_positive(1, v)
376 
377 /*
378  * atomic_inc - increment atomic variable
379  * @v: pointer of type atomic_t
380  *
381  * Atomically increments @v by 1.
382  */
383 #define atomic_inc(v) atomic_add(1, (v))
384 
385 /*
386  * atomic_dec - decrement and test
387  * @v: pointer of type atomic_t
388  *
389  * Atomically decrements @v by 1.
390  */
391 #define atomic_dec(v) atomic_sub(1, (v))
392 
393 /*
394  * atomic_add_negative - add and test if negative
395  * @v: pointer of type atomic_t
396  * @i: integer value to add
397  *
398  * Atomically adds @i to @v and returns true
399  * if the result is negative, or false when
400  * result is greater than or equal to zero.
401  */
402 #define atomic_add_negative(i, v) (atomic_add_return(i, (v)) < 0)
403 
404 #ifdef CONFIG_64BIT
405 
406 #define ATOMIC64_INIT(i)    { (i) }
407 
408 /*
409  * atomic64_read - read atomic variable
410  * @v: pointer of type atomic64_t
411  *
412  */
413 #define atomic64_read(v)	((v)->counter)
414 
415 /*
416  * atomic64_set - set atomic variable
417  * @v: pointer of type atomic64_t
418  * @i: required value
419  */
420 #define atomic64_set(v, i)	((v)->counter = (i))
421 
422 /*
423  * atomic64_add - add integer to atomic variable
424  * @i: integer value to add
425  * @v: pointer of type atomic64_t
426  *
427  * Atomically adds @i to @v.
428  */
429 static __inline__ void atomic64_add(long i, atomic64_t * v)
430 {
431 	if (kernel_uses_llsc && R10000_LLSC_WAR) {
432 		long temp;
433 
434 		__asm__ __volatile__(
435 		"	.set	mips3					\n"
436 		"1:	lld	%0, %1		# atomic64_add		\n"
437 		"	addu	%0, %2					\n"
438 		"	scd	%0, %1					\n"
439 		"	beqzl	%0, 1b					\n"
440 		"	.set	mips0					\n"
441 		: "=&r" (temp), "=m" (v->counter)
442 		: "Ir" (i), "m" (v->counter));
443 	} else if (kernel_uses_llsc) {
444 		long temp;
445 
446 		__asm__ __volatile__(
447 		"	.set	mips3					\n"
448 		"1:	lld	%0, %1		# atomic64_add		\n"
449 		"	addu	%0, %2					\n"
450 		"	scd	%0, %1					\n"
451 		"	beqz	%0, 2f					\n"
452 		"	.subsection 2					\n"
453 		"2:	b	1b					\n"
454 		"	.previous					\n"
455 		"	.set	mips0					\n"
456 		: "=&r" (temp), "=m" (v->counter)
457 		: "Ir" (i), "m" (v->counter));
458 	} else {
459 		unsigned long flags;
460 
461 		raw_local_irq_save(flags);
462 		v->counter += i;
463 		raw_local_irq_restore(flags);
464 	}
465 }
466 
467 /*
468  * atomic64_sub - subtract the atomic variable
469  * @i: integer value to subtract
470  * @v: pointer of type atomic64_t
471  *
472  * Atomically subtracts @i from @v.
473  */
474 static __inline__ void atomic64_sub(long i, atomic64_t * v)
475 {
476 	if (kernel_uses_llsc && R10000_LLSC_WAR) {
477 		long temp;
478 
479 		__asm__ __volatile__(
480 		"	.set	mips3					\n"
481 		"1:	lld	%0, %1		# atomic64_sub		\n"
482 		"	subu	%0, %2					\n"
483 		"	scd	%0, %1					\n"
484 		"	beqzl	%0, 1b					\n"
485 		"	.set	mips0					\n"
486 		: "=&r" (temp), "=m" (v->counter)
487 		: "Ir" (i), "m" (v->counter));
488 	} else if (kernel_uses_llsc) {
489 		long temp;
490 
491 		__asm__ __volatile__(
492 		"	.set	mips3					\n"
493 		"1:	lld	%0, %1		# atomic64_sub		\n"
494 		"	subu	%0, %2					\n"
495 		"	scd	%0, %1					\n"
496 		"	beqz	%0, 2f					\n"
497 		"	.subsection 2					\n"
498 		"2:	b	1b					\n"
499 		"	.previous					\n"
500 		"	.set	mips0					\n"
501 		: "=&r" (temp), "=m" (v->counter)
502 		: "Ir" (i), "m" (v->counter));
503 	} else {
504 		unsigned long flags;
505 
506 		raw_local_irq_save(flags);
507 		v->counter -= i;
508 		raw_local_irq_restore(flags);
509 	}
510 }
511 
512 /*
513  * Same as above, but return the result value
514  */
515 static __inline__ long atomic64_add_return(long i, atomic64_t * v)
516 {
517 	long result;
518 
519 	smp_llsc_mb();
520 
521 	if (kernel_uses_llsc && R10000_LLSC_WAR) {
522 		long temp;
523 
524 		__asm__ __volatile__(
525 		"	.set	mips3					\n"
526 		"1:	lld	%1, %2		# atomic64_add_return	\n"
527 		"	addu	%0, %1, %3				\n"
528 		"	scd	%0, %2					\n"
529 		"	beqzl	%0, 1b					\n"
530 		"	addu	%0, %1, %3				\n"
531 		"	.set	mips0					\n"
532 		: "=&r" (result), "=&r" (temp), "=m" (v->counter)
533 		: "Ir" (i), "m" (v->counter)
534 		: "memory");
535 	} else if (kernel_uses_llsc) {
536 		long temp;
537 
538 		__asm__ __volatile__(
539 		"	.set	mips3					\n"
540 		"1:	lld	%1, %2		# atomic64_add_return	\n"
541 		"	addu	%0, %1, %3				\n"
542 		"	scd	%0, %2					\n"
543 		"	beqz	%0, 2f					\n"
544 		"	addu	%0, %1, %3				\n"
545 		"	.subsection 2					\n"
546 		"2:	b	1b					\n"
547 		"	.previous					\n"
548 		"	.set	mips0					\n"
549 		: "=&r" (result), "=&r" (temp), "=m" (v->counter)
550 		: "Ir" (i), "m" (v->counter)
551 		: "memory");
552 	} else {
553 		unsigned long flags;
554 
555 		raw_local_irq_save(flags);
556 		result = v->counter;
557 		result += i;
558 		v->counter = result;
559 		raw_local_irq_restore(flags);
560 	}
561 
562 	smp_llsc_mb();
563 
564 	return result;
565 }
566 
567 static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
568 {
569 	long result;
570 
571 	smp_llsc_mb();
572 
573 	if (kernel_uses_llsc && R10000_LLSC_WAR) {
574 		long temp;
575 
576 		__asm__ __volatile__(
577 		"	.set	mips3					\n"
578 		"1:	lld	%1, %2		# atomic64_sub_return	\n"
579 		"	subu	%0, %1, %3				\n"
580 		"	scd	%0, %2					\n"
581 		"	beqzl	%0, 1b					\n"
582 		"	subu	%0, %1, %3				\n"
583 		"	.set	mips0					\n"
584 		: "=&r" (result), "=&r" (temp), "=m" (v->counter)
585 		: "Ir" (i), "m" (v->counter)
586 		: "memory");
587 	} else if (kernel_uses_llsc) {
588 		long temp;
589 
590 		__asm__ __volatile__(
591 		"	.set	mips3					\n"
592 		"1:	lld	%1, %2		# atomic64_sub_return	\n"
593 		"	subu	%0, %1, %3				\n"
594 		"	scd	%0, %2					\n"
595 		"	beqz	%0, 2f					\n"
596 		"	subu	%0, %1, %3				\n"
597 		"	.subsection 2					\n"
598 		"2:	b	1b					\n"
599 		"	.previous					\n"
600 		"	.set	mips0					\n"
601 		: "=&r" (result), "=&r" (temp), "=m" (v->counter)
602 		: "Ir" (i), "m" (v->counter)
603 		: "memory");
604 	} else {
605 		unsigned long flags;
606 
607 		raw_local_irq_save(flags);
608 		result = v->counter;
609 		result -= i;
610 		v->counter = result;
611 		raw_local_irq_restore(flags);
612 	}
613 
614 	smp_llsc_mb();
615 
616 	return result;
617 }
618 
619 /*
620  * atomic64_sub_if_positive - conditionally subtract integer from atomic variable
621  * @i: integer value to subtract
622  * @v: pointer of type atomic64_t
623  *
624  * Atomically test @v and subtract @i if @v is greater or equal than @i.
625  * The function returns the old value of @v minus @i.
626  */
627 static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
628 {
629 	long result;
630 
631 	smp_llsc_mb();
632 
633 	if (kernel_uses_llsc && R10000_LLSC_WAR) {
634 		long temp;
635 
636 		__asm__ __volatile__(
637 		"	.set	mips3					\n"
638 		"1:	lld	%1, %2		# atomic64_sub_if_positive\n"
639 		"	dsubu	%0, %1, %3				\n"
640 		"	bltz	%0, 1f					\n"
641 		"	scd	%0, %2					\n"
642 		"	.set	noreorder				\n"
643 		"	beqzl	%0, 1b					\n"
644 		"	 dsubu	%0, %1, %3				\n"
645 		"	.set	reorder					\n"
646 		"1:							\n"
647 		"	.set	mips0					\n"
648 		: "=&r" (result), "=&r" (temp), "=m" (v->counter)
649 		: "Ir" (i), "m" (v->counter)
650 		: "memory");
651 	} else if (kernel_uses_llsc) {
652 		long temp;
653 
654 		__asm__ __volatile__(
655 		"	.set	mips3					\n"
656 		"1:	lld	%1, %2		# atomic64_sub_if_positive\n"
657 		"	dsubu	%0, %1, %3				\n"
658 		"	bltz	%0, 1f					\n"
659 		"	scd	%0, %2					\n"
660 		"	.set	noreorder				\n"
661 		"	beqz	%0, 2f					\n"
662 		"	 dsubu	%0, %1, %3				\n"
663 		"	.set	reorder					\n"
664 		"	.subsection 2					\n"
665 		"2:	b	1b					\n"
666 		"	.previous					\n"
667 		"1:							\n"
668 		"	.set	mips0					\n"
669 		: "=&r" (result), "=&r" (temp), "=m" (v->counter)
670 		: "Ir" (i), "m" (v->counter)
671 		: "memory");
672 	} else {
673 		unsigned long flags;
674 
675 		raw_local_irq_save(flags);
676 		result = v->counter;
677 		result -= i;
678 		if (result >= 0)
679 			v->counter = result;
680 		raw_local_irq_restore(flags);
681 	}
682 
683 	smp_llsc_mb();
684 
685 	return result;
686 }
687 
688 #define atomic64_cmpxchg(v, o, n) \
689 	((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))
690 #define atomic64_xchg(v, new) (xchg(&((v)->counter), (new)))
691 
692 /**
693  * atomic64_add_unless - add unless the number is a given value
694  * @v: pointer of type atomic64_t
695  * @a: the amount to add to v...
696  * @u: ...unless v is equal to u.
697  *
698  * Atomically adds @a to @v, so long as it was not @u.
699  * Returns non-zero if @v was not @u, and zero otherwise.
700  */
701 static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u)
702 {
703 	long c, old;
704 	c = atomic64_read(v);
705 	for (;;) {
706 		if (unlikely(c == (u)))
707 			break;
708 		old = atomic64_cmpxchg((v), c, c + (a));
709 		if (likely(old == c))
710 			break;
711 		c = old;
712 	}
713 	return c != (u);
714 }
715 
716 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
717 
718 #define atomic64_dec_return(v) atomic64_sub_return(1, (v))
719 #define atomic64_inc_return(v) atomic64_add_return(1, (v))
720 
721 /*
722  * atomic64_sub_and_test - subtract value from variable and test result
723  * @i: integer value to subtract
724  * @v: pointer of type atomic64_t
725  *
726  * Atomically subtracts @i from @v and returns
727  * true if the result is zero, or false for all
728  * other cases.
729  */
730 #define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0)
731 
732 /*
733  * atomic64_inc_and_test - increment and test
734  * @v: pointer of type atomic64_t
735  *
736  * Atomically increments @v by 1
737  * and returns true if the result is zero, or false for all
738  * other cases.
739  */
740 #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
741 
742 /*
743  * atomic64_dec_and_test - decrement by 1 and test
744  * @v: pointer of type atomic64_t
745  *
746  * Atomically decrements @v by 1 and
747  * returns true if the result is 0, or false for all other
748  * cases.
749  */
750 #define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0)
751 
752 /*
753  * atomic64_dec_if_positive - decrement by 1 if old value positive
754  * @v: pointer of type atomic64_t
755  */
756 #define atomic64_dec_if_positive(v)	atomic64_sub_if_positive(1, v)
757 
758 /*
759  * atomic64_inc - increment atomic variable
760  * @v: pointer of type atomic64_t
761  *
762  * Atomically increments @v by 1.
763  */
764 #define atomic64_inc(v) atomic64_add(1, (v))
765 
766 /*
767  * atomic64_dec - decrement and test
768  * @v: pointer of type atomic64_t
769  *
770  * Atomically decrements @v by 1.
771  */
772 #define atomic64_dec(v) atomic64_sub(1, (v))
773 
774 /*
775  * atomic64_add_negative - add and test if negative
776  * @v: pointer of type atomic64_t
777  * @i: integer value to add
778  *
779  * Atomically adds @i to @v and returns true
780  * if the result is negative, or false when
781  * result is greater than or equal to zero.
782  */
783 #define atomic64_add_negative(i, v) (atomic64_add_return(i, (v)) < 0)
784 
785 #endif /* CONFIG_64BIT */
786 
787 /*
788  * atomic*_return operations are serializing but not the non-*_return
789  * versions.
790  */
791 #define smp_mb__before_atomic_dec()	smp_llsc_mb()
792 #define smp_mb__after_atomic_dec()	smp_llsc_mb()
793 #define smp_mb__before_atomic_inc()	smp_llsc_mb()
794 #define smp_mb__after_atomic_inc()	smp_llsc_mb()
795 
796 #include <asm-generic/atomic-long.h>
797 
798 #endif /* _ASM_ATOMIC_H */
799