1 // SPDX-License-Identifier: GPL-2.0
2 
3 // Generated by scripts/atomic/gen-atomic-instrumented.sh
4 // DO NOT MODIFY THIS FILE DIRECTLY
5 
6 /*
7  * This file provides wrappers with KASAN instrumentation for atomic operations.
8  * To use this functionality an arch's atomic.h file needs to define all
9  * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
10  * this file at the end. This file provides atomic_read() that forwards to
11  * arch_atomic_read() for actual atomic operation.
12  * Note: if an arch atomic operation is implemented by means of other atomic
13  * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
14  * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
15  * double instrumentation.
16  */
17 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
18 #define _LINUX_ATOMIC_INSTRUMENTED_H
19 
20 #include <linux/build_bug.h>
21 #include <linux/compiler.h>
22 #include <linux/instrumented.h>
23 
24 static __always_inline int
25 atomic_read(const atomic_t *v)
26 {
27 	instrument_atomic_read(v, sizeof(*v));
28 	return arch_atomic_read(v);
29 }
30 
31 static __always_inline int
32 atomic_read_acquire(const atomic_t *v)
33 {
34 	instrument_atomic_read(v, sizeof(*v));
35 	return arch_atomic_read_acquire(v);
36 }
37 
38 static __always_inline void
39 atomic_set(atomic_t *v, int i)
40 {
41 	instrument_atomic_write(v, sizeof(*v));
42 	arch_atomic_set(v, i);
43 }
44 
45 static __always_inline void
46 atomic_set_release(atomic_t *v, int i)
47 {
48 	kcsan_release();
49 	instrument_atomic_write(v, sizeof(*v));
50 	arch_atomic_set_release(v, i);
51 }
52 
53 static __always_inline void
54 atomic_add(int i, atomic_t *v)
55 {
56 	instrument_atomic_read_write(v, sizeof(*v));
57 	arch_atomic_add(i, v);
58 }
59 
60 static __always_inline int
61 atomic_add_return(int i, atomic_t *v)
62 {
63 	kcsan_mb();
64 	instrument_atomic_read_write(v, sizeof(*v));
65 	return arch_atomic_add_return(i, v);
66 }
67 
68 static __always_inline int
69 atomic_add_return_acquire(int i, atomic_t *v)
70 {
71 	instrument_atomic_read_write(v, sizeof(*v));
72 	return arch_atomic_add_return_acquire(i, v);
73 }
74 
75 static __always_inline int
76 atomic_add_return_release(int i, atomic_t *v)
77 {
78 	kcsan_release();
79 	instrument_atomic_read_write(v, sizeof(*v));
80 	return arch_atomic_add_return_release(i, v);
81 }
82 
83 static __always_inline int
84 atomic_add_return_relaxed(int i, atomic_t *v)
85 {
86 	instrument_atomic_read_write(v, sizeof(*v));
87 	return arch_atomic_add_return_relaxed(i, v);
88 }
89 
90 static __always_inline int
91 atomic_fetch_add(int i, atomic_t *v)
92 {
93 	kcsan_mb();
94 	instrument_atomic_read_write(v, sizeof(*v));
95 	return arch_atomic_fetch_add(i, v);
96 }
97 
98 static __always_inline int
99 atomic_fetch_add_acquire(int i, atomic_t *v)
100 {
101 	instrument_atomic_read_write(v, sizeof(*v));
102 	return arch_atomic_fetch_add_acquire(i, v);
103 }
104 
105 static __always_inline int
106 atomic_fetch_add_release(int i, atomic_t *v)
107 {
108 	kcsan_release();
109 	instrument_atomic_read_write(v, sizeof(*v));
110 	return arch_atomic_fetch_add_release(i, v);
111 }
112 
113 static __always_inline int
114 atomic_fetch_add_relaxed(int i, atomic_t *v)
115 {
116 	instrument_atomic_read_write(v, sizeof(*v));
117 	return arch_atomic_fetch_add_relaxed(i, v);
118 }
119 
120 static __always_inline void
121 atomic_sub(int i, atomic_t *v)
122 {
123 	instrument_atomic_read_write(v, sizeof(*v));
124 	arch_atomic_sub(i, v);
125 }
126 
127 static __always_inline int
128 atomic_sub_return(int i, atomic_t *v)
129 {
130 	kcsan_mb();
131 	instrument_atomic_read_write(v, sizeof(*v));
132 	return arch_atomic_sub_return(i, v);
133 }
134 
135 static __always_inline int
136 atomic_sub_return_acquire(int i, atomic_t *v)
137 {
138 	instrument_atomic_read_write(v, sizeof(*v));
139 	return arch_atomic_sub_return_acquire(i, v);
140 }
141 
142 static __always_inline int
143 atomic_sub_return_release(int i, atomic_t *v)
144 {
145 	kcsan_release();
146 	instrument_atomic_read_write(v, sizeof(*v));
147 	return arch_atomic_sub_return_release(i, v);
148 }
149 
150 static __always_inline int
151 atomic_sub_return_relaxed(int i, atomic_t *v)
152 {
153 	instrument_atomic_read_write(v, sizeof(*v));
154 	return arch_atomic_sub_return_relaxed(i, v);
155 }
156 
157 static __always_inline int
158 atomic_fetch_sub(int i, atomic_t *v)
159 {
160 	kcsan_mb();
161 	instrument_atomic_read_write(v, sizeof(*v));
162 	return arch_atomic_fetch_sub(i, v);
163 }
164 
165 static __always_inline int
166 atomic_fetch_sub_acquire(int i, atomic_t *v)
167 {
168 	instrument_atomic_read_write(v, sizeof(*v));
169 	return arch_atomic_fetch_sub_acquire(i, v);
170 }
171 
172 static __always_inline int
173 atomic_fetch_sub_release(int i, atomic_t *v)
174 {
175 	kcsan_release();
176 	instrument_atomic_read_write(v, sizeof(*v));
177 	return arch_atomic_fetch_sub_release(i, v);
178 }
179 
180 static __always_inline int
181 atomic_fetch_sub_relaxed(int i, atomic_t *v)
182 {
183 	instrument_atomic_read_write(v, sizeof(*v));
184 	return arch_atomic_fetch_sub_relaxed(i, v);
185 }
186 
187 static __always_inline void
188 atomic_inc(atomic_t *v)
189 {
190 	instrument_atomic_read_write(v, sizeof(*v));
191 	arch_atomic_inc(v);
192 }
193 
194 static __always_inline int
195 atomic_inc_return(atomic_t *v)
196 {
197 	kcsan_mb();
198 	instrument_atomic_read_write(v, sizeof(*v));
199 	return arch_atomic_inc_return(v);
200 }
201 
202 static __always_inline int
203 atomic_inc_return_acquire(atomic_t *v)
204 {
205 	instrument_atomic_read_write(v, sizeof(*v));
206 	return arch_atomic_inc_return_acquire(v);
207 }
208 
209 static __always_inline int
210 atomic_inc_return_release(atomic_t *v)
211 {
212 	kcsan_release();
213 	instrument_atomic_read_write(v, sizeof(*v));
214 	return arch_atomic_inc_return_release(v);
215 }
216 
217 static __always_inline int
218 atomic_inc_return_relaxed(atomic_t *v)
219 {
220 	instrument_atomic_read_write(v, sizeof(*v));
221 	return arch_atomic_inc_return_relaxed(v);
222 }
223 
224 static __always_inline int
225 atomic_fetch_inc(atomic_t *v)
226 {
227 	kcsan_mb();
228 	instrument_atomic_read_write(v, sizeof(*v));
229 	return arch_atomic_fetch_inc(v);
230 }
231 
232 static __always_inline int
233 atomic_fetch_inc_acquire(atomic_t *v)
234 {
235 	instrument_atomic_read_write(v, sizeof(*v));
236 	return arch_atomic_fetch_inc_acquire(v);
237 }
238 
239 static __always_inline int
240 atomic_fetch_inc_release(atomic_t *v)
241 {
242 	kcsan_release();
243 	instrument_atomic_read_write(v, sizeof(*v));
244 	return arch_atomic_fetch_inc_release(v);
245 }
246 
247 static __always_inline int
248 atomic_fetch_inc_relaxed(atomic_t *v)
249 {
250 	instrument_atomic_read_write(v, sizeof(*v));
251 	return arch_atomic_fetch_inc_relaxed(v);
252 }
253 
254 static __always_inline void
255 atomic_dec(atomic_t *v)
256 {
257 	instrument_atomic_read_write(v, sizeof(*v));
258 	arch_atomic_dec(v);
259 }
260 
261 static __always_inline int
262 atomic_dec_return(atomic_t *v)
263 {
264 	kcsan_mb();
265 	instrument_atomic_read_write(v, sizeof(*v));
266 	return arch_atomic_dec_return(v);
267 }
268 
269 static __always_inline int
270 atomic_dec_return_acquire(atomic_t *v)
271 {
272 	instrument_atomic_read_write(v, sizeof(*v));
273 	return arch_atomic_dec_return_acquire(v);
274 }
275 
276 static __always_inline int
277 atomic_dec_return_release(atomic_t *v)
278 {
279 	kcsan_release();
280 	instrument_atomic_read_write(v, sizeof(*v));
281 	return arch_atomic_dec_return_release(v);
282 }
283 
284 static __always_inline int
285 atomic_dec_return_relaxed(atomic_t *v)
286 {
287 	instrument_atomic_read_write(v, sizeof(*v));
288 	return arch_atomic_dec_return_relaxed(v);
289 }
290 
291 static __always_inline int
292 atomic_fetch_dec(atomic_t *v)
293 {
294 	kcsan_mb();
295 	instrument_atomic_read_write(v, sizeof(*v));
296 	return arch_atomic_fetch_dec(v);
297 }
298 
299 static __always_inline int
300 atomic_fetch_dec_acquire(atomic_t *v)
301 {
302 	instrument_atomic_read_write(v, sizeof(*v));
303 	return arch_atomic_fetch_dec_acquire(v);
304 }
305 
306 static __always_inline int
307 atomic_fetch_dec_release(atomic_t *v)
308 {
309 	kcsan_release();
310 	instrument_atomic_read_write(v, sizeof(*v));
311 	return arch_atomic_fetch_dec_release(v);
312 }
313 
314 static __always_inline int
315 atomic_fetch_dec_relaxed(atomic_t *v)
316 {
317 	instrument_atomic_read_write(v, sizeof(*v));
318 	return arch_atomic_fetch_dec_relaxed(v);
319 }
320 
321 static __always_inline void
322 atomic_and(int i, atomic_t *v)
323 {
324 	instrument_atomic_read_write(v, sizeof(*v));
325 	arch_atomic_and(i, v);
326 }
327 
328 static __always_inline int
329 atomic_fetch_and(int i, atomic_t *v)
330 {
331 	kcsan_mb();
332 	instrument_atomic_read_write(v, sizeof(*v));
333 	return arch_atomic_fetch_and(i, v);
334 }
335 
336 static __always_inline int
337 atomic_fetch_and_acquire(int i, atomic_t *v)
338 {
339 	instrument_atomic_read_write(v, sizeof(*v));
340 	return arch_atomic_fetch_and_acquire(i, v);
341 }
342 
343 static __always_inline int
344 atomic_fetch_and_release(int i, atomic_t *v)
345 {
346 	kcsan_release();
347 	instrument_atomic_read_write(v, sizeof(*v));
348 	return arch_atomic_fetch_and_release(i, v);
349 }
350 
351 static __always_inline int
352 atomic_fetch_and_relaxed(int i, atomic_t *v)
353 {
354 	instrument_atomic_read_write(v, sizeof(*v));
355 	return arch_atomic_fetch_and_relaxed(i, v);
356 }
357 
358 static __always_inline void
359 atomic_andnot(int i, atomic_t *v)
360 {
361 	instrument_atomic_read_write(v, sizeof(*v));
362 	arch_atomic_andnot(i, v);
363 }
364 
365 static __always_inline int
366 atomic_fetch_andnot(int i, atomic_t *v)
367 {
368 	kcsan_mb();
369 	instrument_atomic_read_write(v, sizeof(*v));
370 	return arch_atomic_fetch_andnot(i, v);
371 }
372 
373 static __always_inline int
374 atomic_fetch_andnot_acquire(int i, atomic_t *v)
375 {
376 	instrument_atomic_read_write(v, sizeof(*v));
377 	return arch_atomic_fetch_andnot_acquire(i, v);
378 }
379 
380 static __always_inline int
381 atomic_fetch_andnot_release(int i, atomic_t *v)
382 {
383 	kcsan_release();
384 	instrument_atomic_read_write(v, sizeof(*v));
385 	return arch_atomic_fetch_andnot_release(i, v);
386 }
387 
388 static __always_inline int
389 atomic_fetch_andnot_relaxed(int i, atomic_t *v)
390 {
391 	instrument_atomic_read_write(v, sizeof(*v));
392 	return arch_atomic_fetch_andnot_relaxed(i, v);
393 }
394 
395 static __always_inline void
396 atomic_or(int i, atomic_t *v)
397 {
398 	instrument_atomic_read_write(v, sizeof(*v));
399 	arch_atomic_or(i, v);
400 }
401 
402 static __always_inline int
403 atomic_fetch_or(int i, atomic_t *v)
404 {
405 	kcsan_mb();
406 	instrument_atomic_read_write(v, sizeof(*v));
407 	return arch_atomic_fetch_or(i, v);
408 }
409 
410 static __always_inline int
411 atomic_fetch_or_acquire(int i, atomic_t *v)
412 {
413 	instrument_atomic_read_write(v, sizeof(*v));
414 	return arch_atomic_fetch_or_acquire(i, v);
415 }
416 
417 static __always_inline int
418 atomic_fetch_or_release(int i, atomic_t *v)
419 {
420 	kcsan_release();
421 	instrument_atomic_read_write(v, sizeof(*v));
422 	return arch_atomic_fetch_or_release(i, v);
423 }
424 
425 static __always_inline int
426 atomic_fetch_or_relaxed(int i, atomic_t *v)
427 {
428 	instrument_atomic_read_write(v, sizeof(*v));
429 	return arch_atomic_fetch_or_relaxed(i, v);
430 }
431 
432 static __always_inline void
433 atomic_xor(int i, atomic_t *v)
434 {
435 	instrument_atomic_read_write(v, sizeof(*v));
436 	arch_atomic_xor(i, v);
437 }
438 
439 static __always_inline int
440 atomic_fetch_xor(int i, atomic_t *v)
441 {
442 	kcsan_mb();
443 	instrument_atomic_read_write(v, sizeof(*v));
444 	return arch_atomic_fetch_xor(i, v);
445 }
446 
447 static __always_inline int
448 atomic_fetch_xor_acquire(int i, atomic_t *v)
449 {
450 	instrument_atomic_read_write(v, sizeof(*v));
451 	return arch_atomic_fetch_xor_acquire(i, v);
452 }
453 
454 static __always_inline int
455 atomic_fetch_xor_release(int i, atomic_t *v)
456 {
457 	kcsan_release();
458 	instrument_atomic_read_write(v, sizeof(*v));
459 	return arch_atomic_fetch_xor_release(i, v);
460 }
461 
462 static __always_inline int
463 atomic_fetch_xor_relaxed(int i, atomic_t *v)
464 {
465 	instrument_atomic_read_write(v, sizeof(*v));
466 	return arch_atomic_fetch_xor_relaxed(i, v);
467 }
468 
469 static __always_inline int
470 atomic_xchg(atomic_t *v, int i)
471 {
472 	kcsan_mb();
473 	instrument_atomic_read_write(v, sizeof(*v));
474 	return arch_atomic_xchg(v, i);
475 }
476 
477 static __always_inline int
478 atomic_xchg_acquire(atomic_t *v, int i)
479 {
480 	instrument_atomic_read_write(v, sizeof(*v));
481 	return arch_atomic_xchg_acquire(v, i);
482 }
483 
484 static __always_inline int
485 atomic_xchg_release(atomic_t *v, int i)
486 {
487 	kcsan_release();
488 	instrument_atomic_read_write(v, sizeof(*v));
489 	return arch_atomic_xchg_release(v, i);
490 }
491 
492 static __always_inline int
493 atomic_xchg_relaxed(atomic_t *v, int i)
494 {
495 	instrument_atomic_read_write(v, sizeof(*v));
496 	return arch_atomic_xchg_relaxed(v, i);
497 }
498 
499 static __always_inline int
500 atomic_cmpxchg(atomic_t *v, int old, int new)
501 {
502 	kcsan_mb();
503 	instrument_atomic_read_write(v, sizeof(*v));
504 	return arch_atomic_cmpxchg(v, old, new);
505 }
506 
507 static __always_inline int
508 atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
509 {
510 	instrument_atomic_read_write(v, sizeof(*v));
511 	return arch_atomic_cmpxchg_acquire(v, old, new);
512 }
513 
514 static __always_inline int
515 atomic_cmpxchg_release(atomic_t *v, int old, int new)
516 {
517 	kcsan_release();
518 	instrument_atomic_read_write(v, sizeof(*v));
519 	return arch_atomic_cmpxchg_release(v, old, new);
520 }
521 
522 static __always_inline int
523 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
524 {
525 	instrument_atomic_read_write(v, sizeof(*v));
526 	return arch_atomic_cmpxchg_relaxed(v, old, new);
527 }
528 
529 static __always_inline bool
530 atomic_try_cmpxchg(atomic_t *v, int *old, int new)
531 {
532 	kcsan_mb();
533 	instrument_atomic_read_write(v, sizeof(*v));
534 	instrument_atomic_read_write(old, sizeof(*old));
535 	return arch_atomic_try_cmpxchg(v, old, new);
536 }
537 
538 static __always_inline bool
539 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
540 {
541 	instrument_atomic_read_write(v, sizeof(*v));
542 	instrument_atomic_read_write(old, sizeof(*old));
543 	return arch_atomic_try_cmpxchg_acquire(v, old, new);
544 }
545 
546 static __always_inline bool
547 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
548 {
549 	kcsan_release();
550 	instrument_atomic_read_write(v, sizeof(*v));
551 	instrument_atomic_read_write(old, sizeof(*old));
552 	return arch_atomic_try_cmpxchg_release(v, old, new);
553 }
554 
555 static __always_inline bool
556 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
557 {
558 	instrument_atomic_read_write(v, sizeof(*v));
559 	instrument_atomic_read_write(old, sizeof(*old));
560 	return arch_atomic_try_cmpxchg_relaxed(v, old, new);
561 }
562 
563 static __always_inline bool
564 atomic_sub_and_test(int i, atomic_t *v)
565 {
566 	kcsan_mb();
567 	instrument_atomic_read_write(v, sizeof(*v));
568 	return arch_atomic_sub_and_test(i, v);
569 }
570 
571 static __always_inline bool
572 atomic_dec_and_test(atomic_t *v)
573 {
574 	kcsan_mb();
575 	instrument_atomic_read_write(v, sizeof(*v));
576 	return arch_atomic_dec_and_test(v);
577 }
578 
579 static __always_inline bool
580 atomic_inc_and_test(atomic_t *v)
581 {
582 	kcsan_mb();
583 	instrument_atomic_read_write(v, sizeof(*v));
584 	return arch_atomic_inc_and_test(v);
585 }
586 
587 static __always_inline bool
588 atomic_add_negative(int i, atomic_t *v)
589 {
590 	kcsan_mb();
591 	instrument_atomic_read_write(v, sizeof(*v));
592 	return arch_atomic_add_negative(i, v);
593 }
594 
595 static __always_inline bool
596 atomic_add_negative_acquire(int i, atomic_t *v)
597 {
598 	instrument_atomic_read_write(v, sizeof(*v));
599 	return arch_atomic_add_negative_acquire(i, v);
600 }
601 
602 static __always_inline bool
603 atomic_add_negative_release(int i, atomic_t *v)
604 {
605 	kcsan_release();
606 	instrument_atomic_read_write(v, sizeof(*v));
607 	return arch_atomic_add_negative_release(i, v);
608 }
609 
610 static __always_inline bool
611 atomic_add_negative_relaxed(int i, atomic_t *v)
612 {
613 	instrument_atomic_read_write(v, sizeof(*v));
614 	return arch_atomic_add_negative_relaxed(i, v);
615 }
616 
617 static __always_inline int
618 atomic_fetch_add_unless(atomic_t *v, int a, int u)
619 {
620 	kcsan_mb();
621 	instrument_atomic_read_write(v, sizeof(*v));
622 	return arch_atomic_fetch_add_unless(v, a, u);
623 }
624 
625 static __always_inline bool
626 atomic_add_unless(atomic_t *v, int a, int u)
627 {
628 	kcsan_mb();
629 	instrument_atomic_read_write(v, sizeof(*v));
630 	return arch_atomic_add_unless(v, a, u);
631 }
632 
633 static __always_inline bool
634 atomic_inc_not_zero(atomic_t *v)
635 {
636 	kcsan_mb();
637 	instrument_atomic_read_write(v, sizeof(*v));
638 	return arch_atomic_inc_not_zero(v);
639 }
640 
641 static __always_inline bool
642 atomic_inc_unless_negative(atomic_t *v)
643 {
644 	kcsan_mb();
645 	instrument_atomic_read_write(v, sizeof(*v));
646 	return arch_atomic_inc_unless_negative(v);
647 }
648 
649 static __always_inline bool
650 atomic_dec_unless_positive(atomic_t *v)
651 {
652 	kcsan_mb();
653 	instrument_atomic_read_write(v, sizeof(*v));
654 	return arch_atomic_dec_unless_positive(v);
655 }
656 
657 static __always_inline int
658 atomic_dec_if_positive(atomic_t *v)
659 {
660 	kcsan_mb();
661 	instrument_atomic_read_write(v, sizeof(*v));
662 	return arch_atomic_dec_if_positive(v);
663 }
664 
665 static __always_inline s64
666 atomic64_read(const atomic64_t *v)
667 {
668 	instrument_atomic_read(v, sizeof(*v));
669 	return arch_atomic64_read(v);
670 }
671 
672 static __always_inline s64
673 atomic64_read_acquire(const atomic64_t *v)
674 {
675 	instrument_atomic_read(v, sizeof(*v));
676 	return arch_atomic64_read_acquire(v);
677 }
678 
679 static __always_inline void
680 atomic64_set(atomic64_t *v, s64 i)
681 {
682 	instrument_atomic_write(v, sizeof(*v));
683 	arch_atomic64_set(v, i);
684 }
685 
686 static __always_inline void
687 atomic64_set_release(atomic64_t *v, s64 i)
688 {
689 	kcsan_release();
690 	instrument_atomic_write(v, sizeof(*v));
691 	arch_atomic64_set_release(v, i);
692 }
693 
694 static __always_inline void
695 atomic64_add(s64 i, atomic64_t *v)
696 {
697 	instrument_atomic_read_write(v, sizeof(*v));
698 	arch_atomic64_add(i, v);
699 }
700 
701 static __always_inline s64
702 atomic64_add_return(s64 i, atomic64_t *v)
703 {
704 	kcsan_mb();
705 	instrument_atomic_read_write(v, sizeof(*v));
706 	return arch_atomic64_add_return(i, v);
707 }
708 
709 static __always_inline s64
710 atomic64_add_return_acquire(s64 i, atomic64_t *v)
711 {
712 	instrument_atomic_read_write(v, sizeof(*v));
713 	return arch_atomic64_add_return_acquire(i, v);
714 }
715 
716 static __always_inline s64
717 atomic64_add_return_release(s64 i, atomic64_t *v)
718 {
719 	kcsan_release();
720 	instrument_atomic_read_write(v, sizeof(*v));
721 	return arch_atomic64_add_return_release(i, v);
722 }
723 
724 static __always_inline s64
725 atomic64_add_return_relaxed(s64 i, atomic64_t *v)
726 {
727 	instrument_atomic_read_write(v, sizeof(*v));
728 	return arch_atomic64_add_return_relaxed(i, v);
729 }
730 
731 static __always_inline s64
732 atomic64_fetch_add(s64 i, atomic64_t *v)
733 {
734 	kcsan_mb();
735 	instrument_atomic_read_write(v, sizeof(*v));
736 	return arch_atomic64_fetch_add(i, v);
737 }
738 
739 static __always_inline s64
740 atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
741 {
742 	instrument_atomic_read_write(v, sizeof(*v));
743 	return arch_atomic64_fetch_add_acquire(i, v);
744 }
745 
746 static __always_inline s64
747 atomic64_fetch_add_release(s64 i, atomic64_t *v)
748 {
749 	kcsan_release();
750 	instrument_atomic_read_write(v, sizeof(*v));
751 	return arch_atomic64_fetch_add_release(i, v);
752 }
753 
754 static __always_inline s64
755 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
756 {
757 	instrument_atomic_read_write(v, sizeof(*v));
758 	return arch_atomic64_fetch_add_relaxed(i, v);
759 }
760 
761 static __always_inline void
762 atomic64_sub(s64 i, atomic64_t *v)
763 {
764 	instrument_atomic_read_write(v, sizeof(*v));
765 	arch_atomic64_sub(i, v);
766 }
767 
768 static __always_inline s64
769 atomic64_sub_return(s64 i, atomic64_t *v)
770 {
771 	kcsan_mb();
772 	instrument_atomic_read_write(v, sizeof(*v));
773 	return arch_atomic64_sub_return(i, v);
774 }
775 
776 static __always_inline s64
777 atomic64_sub_return_acquire(s64 i, atomic64_t *v)
778 {
779 	instrument_atomic_read_write(v, sizeof(*v));
780 	return arch_atomic64_sub_return_acquire(i, v);
781 }
782 
783 static __always_inline s64
784 atomic64_sub_return_release(s64 i, atomic64_t *v)
785 {
786 	kcsan_release();
787 	instrument_atomic_read_write(v, sizeof(*v));
788 	return arch_atomic64_sub_return_release(i, v);
789 }
790 
791 static __always_inline s64
792 atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
793 {
794 	instrument_atomic_read_write(v, sizeof(*v));
795 	return arch_atomic64_sub_return_relaxed(i, v);
796 }
797 
798 static __always_inline s64
799 atomic64_fetch_sub(s64 i, atomic64_t *v)
800 {
801 	kcsan_mb();
802 	instrument_atomic_read_write(v, sizeof(*v));
803 	return arch_atomic64_fetch_sub(i, v);
804 }
805 
806 static __always_inline s64
807 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
808 {
809 	instrument_atomic_read_write(v, sizeof(*v));
810 	return arch_atomic64_fetch_sub_acquire(i, v);
811 }
812 
813 static __always_inline s64
814 atomic64_fetch_sub_release(s64 i, atomic64_t *v)
815 {
816 	kcsan_release();
817 	instrument_atomic_read_write(v, sizeof(*v));
818 	return arch_atomic64_fetch_sub_release(i, v);
819 }
820 
821 static __always_inline s64
822 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
823 {
824 	instrument_atomic_read_write(v, sizeof(*v));
825 	return arch_atomic64_fetch_sub_relaxed(i, v);
826 }
827 
828 static __always_inline void
829 atomic64_inc(atomic64_t *v)
830 {
831 	instrument_atomic_read_write(v, sizeof(*v));
832 	arch_atomic64_inc(v);
833 }
834 
835 static __always_inline s64
836 atomic64_inc_return(atomic64_t *v)
837 {
838 	kcsan_mb();
839 	instrument_atomic_read_write(v, sizeof(*v));
840 	return arch_atomic64_inc_return(v);
841 }
842 
843 static __always_inline s64
844 atomic64_inc_return_acquire(atomic64_t *v)
845 {
846 	instrument_atomic_read_write(v, sizeof(*v));
847 	return arch_atomic64_inc_return_acquire(v);
848 }
849 
850 static __always_inline s64
851 atomic64_inc_return_release(atomic64_t *v)
852 {
853 	kcsan_release();
854 	instrument_atomic_read_write(v, sizeof(*v));
855 	return arch_atomic64_inc_return_release(v);
856 }
857 
858 static __always_inline s64
859 atomic64_inc_return_relaxed(atomic64_t *v)
860 {
861 	instrument_atomic_read_write(v, sizeof(*v));
862 	return arch_atomic64_inc_return_relaxed(v);
863 }
864 
865 static __always_inline s64
866 atomic64_fetch_inc(atomic64_t *v)
867 {
868 	kcsan_mb();
869 	instrument_atomic_read_write(v, sizeof(*v));
870 	return arch_atomic64_fetch_inc(v);
871 }
872 
873 static __always_inline s64
874 atomic64_fetch_inc_acquire(atomic64_t *v)
875 {
876 	instrument_atomic_read_write(v, sizeof(*v));
877 	return arch_atomic64_fetch_inc_acquire(v);
878 }
879 
880 static __always_inline s64
881 atomic64_fetch_inc_release(atomic64_t *v)
882 {
883 	kcsan_release();
884 	instrument_atomic_read_write(v, sizeof(*v));
885 	return arch_atomic64_fetch_inc_release(v);
886 }
887 
888 static __always_inline s64
889 atomic64_fetch_inc_relaxed(atomic64_t *v)
890 {
891 	instrument_atomic_read_write(v, sizeof(*v));
892 	return arch_atomic64_fetch_inc_relaxed(v);
893 }
894 
895 static __always_inline void
896 atomic64_dec(atomic64_t *v)
897 {
898 	instrument_atomic_read_write(v, sizeof(*v));
899 	arch_atomic64_dec(v);
900 }
901 
902 static __always_inline s64
903 atomic64_dec_return(atomic64_t *v)
904 {
905 	kcsan_mb();
906 	instrument_atomic_read_write(v, sizeof(*v));
907 	return arch_atomic64_dec_return(v);
908 }
909 
910 static __always_inline s64
911 atomic64_dec_return_acquire(atomic64_t *v)
912 {
913 	instrument_atomic_read_write(v, sizeof(*v));
914 	return arch_atomic64_dec_return_acquire(v);
915 }
916 
917 static __always_inline s64
918 atomic64_dec_return_release(atomic64_t *v)
919 {
920 	kcsan_release();
921 	instrument_atomic_read_write(v, sizeof(*v));
922 	return arch_atomic64_dec_return_release(v);
923 }
924 
925 static __always_inline s64
926 atomic64_dec_return_relaxed(atomic64_t *v)
927 {
928 	instrument_atomic_read_write(v, sizeof(*v));
929 	return arch_atomic64_dec_return_relaxed(v);
930 }
931 
932 static __always_inline s64
933 atomic64_fetch_dec(atomic64_t *v)
934 {
935 	kcsan_mb();
936 	instrument_atomic_read_write(v, sizeof(*v));
937 	return arch_atomic64_fetch_dec(v);
938 }
939 
940 static __always_inline s64
941 atomic64_fetch_dec_acquire(atomic64_t *v)
942 {
943 	instrument_atomic_read_write(v, sizeof(*v));
944 	return arch_atomic64_fetch_dec_acquire(v);
945 }
946 
947 static __always_inline s64
948 atomic64_fetch_dec_release(atomic64_t *v)
949 {
950 	kcsan_release();
951 	instrument_atomic_read_write(v, sizeof(*v));
952 	return arch_atomic64_fetch_dec_release(v);
953 }
954 
955 static __always_inline s64
956 atomic64_fetch_dec_relaxed(atomic64_t *v)
957 {
958 	instrument_atomic_read_write(v, sizeof(*v));
959 	return arch_atomic64_fetch_dec_relaxed(v);
960 }
961 
962 static __always_inline void
963 atomic64_and(s64 i, atomic64_t *v)
964 {
965 	instrument_atomic_read_write(v, sizeof(*v));
966 	arch_atomic64_and(i, v);
967 }
968 
969 static __always_inline s64
970 atomic64_fetch_and(s64 i, atomic64_t *v)
971 {
972 	kcsan_mb();
973 	instrument_atomic_read_write(v, sizeof(*v));
974 	return arch_atomic64_fetch_and(i, v);
975 }
976 
977 static __always_inline s64
978 atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
979 {
980 	instrument_atomic_read_write(v, sizeof(*v));
981 	return arch_atomic64_fetch_and_acquire(i, v);
982 }
983 
984 static __always_inline s64
985 atomic64_fetch_and_release(s64 i, atomic64_t *v)
986 {
987 	kcsan_release();
988 	instrument_atomic_read_write(v, sizeof(*v));
989 	return arch_atomic64_fetch_and_release(i, v);
990 }
991 
992 static __always_inline s64
993 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
994 {
995 	instrument_atomic_read_write(v, sizeof(*v));
996 	return arch_atomic64_fetch_and_relaxed(i, v);
997 }
998 
999 static __always_inline void
1000 atomic64_andnot(s64 i, atomic64_t *v)
1001 {
1002 	instrument_atomic_read_write(v, sizeof(*v));
1003 	arch_atomic64_andnot(i, v);
1004 }
1005 
1006 static __always_inline s64
1007 atomic64_fetch_andnot(s64 i, atomic64_t *v)
1008 {
1009 	kcsan_mb();
1010 	instrument_atomic_read_write(v, sizeof(*v));
1011 	return arch_atomic64_fetch_andnot(i, v);
1012 }
1013 
1014 static __always_inline s64
1015 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1016 {
1017 	instrument_atomic_read_write(v, sizeof(*v));
1018 	return arch_atomic64_fetch_andnot_acquire(i, v);
1019 }
1020 
1021 static __always_inline s64
1022 atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1023 {
1024 	kcsan_release();
1025 	instrument_atomic_read_write(v, sizeof(*v));
1026 	return arch_atomic64_fetch_andnot_release(i, v);
1027 }
1028 
1029 static __always_inline s64
1030 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1031 {
1032 	instrument_atomic_read_write(v, sizeof(*v));
1033 	return arch_atomic64_fetch_andnot_relaxed(i, v);
1034 }
1035 
1036 static __always_inline void
1037 atomic64_or(s64 i, atomic64_t *v)
1038 {
1039 	instrument_atomic_read_write(v, sizeof(*v));
1040 	arch_atomic64_or(i, v);
1041 }
1042 
1043 static __always_inline s64
1044 atomic64_fetch_or(s64 i, atomic64_t *v)
1045 {
1046 	kcsan_mb();
1047 	instrument_atomic_read_write(v, sizeof(*v));
1048 	return arch_atomic64_fetch_or(i, v);
1049 }
1050 
1051 static __always_inline s64
1052 atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1053 {
1054 	instrument_atomic_read_write(v, sizeof(*v));
1055 	return arch_atomic64_fetch_or_acquire(i, v);
1056 }
1057 
1058 static __always_inline s64
1059 atomic64_fetch_or_release(s64 i, atomic64_t *v)
1060 {
1061 	kcsan_release();
1062 	instrument_atomic_read_write(v, sizeof(*v));
1063 	return arch_atomic64_fetch_or_release(i, v);
1064 }
1065 
1066 static __always_inline s64
1067 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1068 {
1069 	instrument_atomic_read_write(v, sizeof(*v));
1070 	return arch_atomic64_fetch_or_relaxed(i, v);
1071 }
1072 
1073 static __always_inline void
1074 atomic64_xor(s64 i, atomic64_t *v)
1075 {
1076 	instrument_atomic_read_write(v, sizeof(*v));
1077 	arch_atomic64_xor(i, v);
1078 }
1079 
1080 static __always_inline s64
1081 atomic64_fetch_xor(s64 i, atomic64_t *v)
1082 {
1083 	kcsan_mb();
1084 	instrument_atomic_read_write(v, sizeof(*v));
1085 	return arch_atomic64_fetch_xor(i, v);
1086 }
1087 
1088 static __always_inline s64
1089 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1090 {
1091 	instrument_atomic_read_write(v, sizeof(*v));
1092 	return arch_atomic64_fetch_xor_acquire(i, v);
1093 }
1094 
1095 static __always_inline s64
1096 atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1097 {
1098 	kcsan_release();
1099 	instrument_atomic_read_write(v, sizeof(*v));
1100 	return arch_atomic64_fetch_xor_release(i, v);
1101 }
1102 
1103 static __always_inline s64
1104 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1105 {
1106 	instrument_atomic_read_write(v, sizeof(*v));
1107 	return arch_atomic64_fetch_xor_relaxed(i, v);
1108 }
1109 
1110 static __always_inline s64
1111 atomic64_xchg(atomic64_t *v, s64 i)
1112 {
1113 	kcsan_mb();
1114 	instrument_atomic_read_write(v, sizeof(*v));
1115 	return arch_atomic64_xchg(v, i);
1116 }
1117 
1118 static __always_inline s64
1119 atomic64_xchg_acquire(atomic64_t *v, s64 i)
1120 {
1121 	instrument_atomic_read_write(v, sizeof(*v));
1122 	return arch_atomic64_xchg_acquire(v, i);
1123 }
1124 
1125 static __always_inline s64
1126 atomic64_xchg_release(atomic64_t *v, s64 i)
1127 {
1128 	kcsan_release();
1129 	instrument_atomic_read_write(v, sizeof(*v));
1130 	return arch_atomic64_xchg_release(v, i);
1131 }
1132 
1133 static __always_inline s64
1134 atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1135 {
1136 	instrument_atomic_read_write(v, sizeof(*v));
1137 	return arch_atomic64_xchg_relaxed(v, i);
1138 }
1139 
1140 static __always_inline s64
1141 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1142 {
1143 	kcsan_mb();
1144 	instrument_atomic_read_write(v, sizeof(*v));
1145 	return arch_atomic64_cmpxchg(v, old, new);
1146 }
1147 
1148 static __always_inline s64
1149 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1150 {
1151 	instrument_atomic_read_write(v, sizeof(*v));
1152 	return arch_atomic64_cmpxchg_acquire(v, old, new);
1153 }
1154 
1155 static __always_inline s64
1156 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1157 {
1158 	kcsan_release();
1159 	instrument_atomic_read_write(v, sizeof(*v));
1160 	return arch_atomic64_cmpxchg_release(v, old, new);
1161 }
1162 
1163 static __always_inline s64
1164 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1165 {
1166 	instrument_atomic_read_write(v, sizeof(*v));
1167 	return arch_atomic64_cmpxchg_relaxed(v, old, new);
1168 }
1169 
1170 static __always_inline bool
1171 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1172 {
1173 	kcsan_mb();
1174 	instrument_atomic_read_write(v, sizeof(*v));
1175 	instrument_atomic_read_write(old, sizeof(*old));
1176 	return arch_atomic64_try_cmpxchg(v, old, new);
1177 }
1178 
1179 static __always_inline bool
1180 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1181 {
1182 	instrument_atomic_read_write(v, sizeof(*v));
1183 	instrument_atomic_read_write(old, sizeof(*old));
1184 	return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1185 }
1186 
1187 static __always_inline bool
1188 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1189 {
1190 	kcsan_release();
1191 	instrument_atomic_read_write(v, sizeof(*v));
1192 	instrument_atomic_read_write(old, sizeof(*old));
1193 	return arch_atomic64_try_cmpxchg_release(v, old, new);
1194 }
1195 
1196 static __always_inline bool
1197 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1198 {
1199 	instrument_atomic_read_write(v, sizeof(*v));
1200 	instrument_atomic_read_write(old, sizeof(*old));
1201 	return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1202 }
1203 
1204 static __always_inline bool
1205 atomic64_sub_and_test(s64 i, atomic64_t *v)
1206 {
1207 	kcsan_mb();
1208 	instrument_atomic_read_write(v, sizeof(*v));
1209 	return arch_atomic64_sub_and_test(i, v);
1210 }
1211 
1212 static __always_inline bool
1213 atomic64_dec_and_test(atomic64_t *v)
1214 {
1215 	kcsan_mb();
1216 	instrument_atomic_read_write(v, sizeof(*v));
1217 	return arch_atomic64_dec_and_test(v);
1218 }
1219 
1220 static __always_inline bool
1221 atomic64_inc_and_test(atomic64_t *v)
1222 {
1223 	kcsan_mb();
1224 	instrument_atomic_read_write(v, sizeof(*v));
1225 	return arch_atomic64_inc_and_test(v);
1226 }
1227 
1228 static __always_inline bool
1229 atomic64_add_negative(s64 i, atomic64_t *v)
1230 {
1231 	kcsan_mb();
1232 	instrument_atomic_read_write(v, sizeof(*v));
1233 	return arch_atomic64_add_negative(i, v);
1234 }
1235 
1236 static __always_inline bool
1237 atomic64_add_negative_acquire(s64 i, atomic64_t *v)
1238 {
1239 	instrument_atomic_read_write(v, sizeof(*v));
1240 	return arch_atomic64_add_negative_acquire(i, v);
1241 }
1242 
1243 static __always_inline bool
1244 atomic64_add_negative_release(s64 i, atomic64_t *v)
1245 {
1246 	kcsan_release();
1247 	instrument_atomic_read_write(v, sizeof(*v));
1248 	return arch_atomic64_add_negative_release(i, v);
1249 }
1250 
1251 static __always_inline bool
1252 atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
1253 {
1254 	instrument_atomic_read_write(v, sizeof(*v));
1255 	return arch_atomic64_add_negative_relaxed(i, v);
1256 }
1257 
1258 static __always_inline s64
1259 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1260 {
1261 	kcsan_mb();
1262 	instrument_atomic_read_write(v, sizeof(*v));
1263 	return arch_atomic64_fetch_add_unless(v, a, u);
1264 }
1265 
1266 static __always_inline bool
1267 atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1268 {
1269 	kcsan_mb();
1270 	instrument_atomic_read_write(v, sizeof(*v));
1271 	return arch_atomic64_add_unless(v, a, u);
1272 }
1273 
1274 static __always_inline bool
1275 atomic64_inc_not_zero(atomic64_t *v)
1276 {
1277 	kcsan_mb();
1278 	instrument_atomic_read_write(v, sizeof(*v));
1279 	return arch_atomic64_inc_not_zero(v);
1280 }
1281 
1282 static __always_inline bool
1283 atomic64_inc_unless_negative(atomic64_t *v)
1284 {
1285 	kcsan_mb();
1286 	instrument_atomic_read_write(v, sizeof(*v));
1287 	return arch_atomic64_inc_unless_negative(v);
1288 }
1289 
1290 static __always_inline bool
1291 atomic64_dec_unless_positive(atomic64_t *v)
1292 {
1293 	kcsan_mb();
1294 	instrument_atomic_read_write(v, sizeof(*v));
1295 	return arch_atomic64_dec_unless_positive(v);
1296 }
1297 
1298 static __always_inline s64
1299 atomic64_dec_if_positive(atomic64_t *v)
1300 {
1301 	kcsan_mb();
1302 	instrument_atomic_read_write(v, sizeof(*v));
1303 	return arch_atomic64_dec_if_positive(v);
1304 }
1305 
1306 static __always_inline long
1307 atomic_long_read(const atomic_long_t *v)
1308 {
1309 	instrument_atomic_read(v, sizeof(*v));
1310 	return arch_atomic_long_read(v);
1311 }
1312 
1313 static __always_inline long
1314 atomic_long_read_acquire(const atomic_long_t *v)
1315 {
1316 	instrument_atomic_read(v, sizeof(*v));
1317 	return arch_atomic_long_read_acquire(v);
1318 }
1319 
1320 static __always_inline void
1321 atomic_long_set(atomic_long_t *v, long i)
1322 {
1323 	instrument_atomic_write(v, sizeof(*v));
1324 	arch_atomic_long_set(v, i);
1325 }
1326 
1327 static __always_inline void
1328 atomic_long_set_release(atomic_long_t *v, long i)
1329 {
1330 	kcsan_release();
1331 	instrument_atomic_write(v, sizeof(*v));
1332 	arch_atomic_long_set_release(v, i);
1333 }
1334 
1335 static __always_inline void
1336 atomic_long_add(long i, atomic_long_t *v)
1337 {
1338 	instrument_atomic_read_write(v, sizeof(*v));
1339 	arch_atomic_long_add(i, v);
1340 }
1341 
1342 static __always_inline long
1343 atomic_long_add_return(long i, atomic_long_t *v)
1344 {
1345 	kcsan_mb();
1346 	instrument_atomic_read_write(v, sizeof(*v));
1347 	return arch_atomic_long_add_return(i, v);
1348 }
1349 
1350 static __always_inline long
1351 atomic_long_add_return_acquire(long i, atomic_long_t *v)
1352 {
1353 	instrument_atomic_read_write(v, sizeof(*v));
1354 	return arch_atomic_long_add_return_acquire(i, v);
1355 }
1356 
1357 static __always_inline long
1358 atomic_long_add_return_release(long i, atomic_long_t *v)
1359 {
1360 	kcsan_release();
1361 	instrument_atomic_read_write(v, sizeof(*v));
1362 	return arch_atomic_long_add_return_release(i, v);
1363 }
1364 
1365 static __always_inline long
1366 atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1367 {
1368 	instrument_atomic_read_write(v, sizeof(*v));
1369 	return arch_atomic_long_add_return_relaxed(i, v);
1370 }
1371 
1372 static __always_inline long
1373 atomic_long_fetch_add(long i, atomic_long_t *v)
1374 {
1375 	kcsan_mb();
1376 	instrument_atomic_read_write(v, sizeof(*v));
1377 	return arch_atomic_long_fetch_add(i, v);
1378 }
1379 
1380 static __always_inline long
1381 atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1382 {
1383 	instrument_atomic_read_write(v, sizeof(*v));
1384 	return arch_atomic_long_fetch_add_acquire(i, v);
1385 }
1386 
1387 static __always_inline long
1388 atomic_long_fetch_add_release(long i, atomic_long_t *v)
1389 {
1390 	kcsan_release();
1391 	instrument_atomic_read_write(v, sizeof(*v));
1392 	return arch_atomic_long_fetch_add_release(i, v);
1393 }
1394 
1395 static __always_inline long
1396 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1397 {
1398 	instrument_atomic_read_write(v, sizeof(*v));
1399 	return arch_atomic_long_fetch_add_relaxed(i, v);
1400 }
1401 
1402 static __always_inline void
1403 atomic_long_sub(long i, atomic_long_t *v)
1404 {
1405 	instrument_atomic_read_write(v, sizeof(*v));
1406 	arch_atomic_long_sub(i, v);
1407 }
1408 
1409 static __always_inline long
1410 atomic_long_sub_return(long i, atomic_long_t *v)
1411 {
1412 	kcsan_mb();
1413 	instrument_atomic_read_write(v, sizeof(*v));
1414 	return arch_atomic_long_sub_return(i, v);
1415 }
1416 
1417 static __always_inline long
1418 atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1419 {
1420 	instrument_atomic_read_write(v, sizeof(*v));
1421 	return arch_atomic_long_sub_return_acquire(i, v);
1422 }
1423 
1424 static __always_inline long
1425 atomic_long_sub_return_release(long i, atomic_long_t *v)
1426 {
1427 	kcsan_release();
1428 	instrument_atomic_read_write(v, sizeof(*v));
1429 	return arch_atomic_long_sub_return_release(i, v);
1430 }
1431 
1432 static __always_inline long
1433 atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1434 {
1435 	instrument_atomic_read_write(v, sizeof(*v));
1436 	return arch_atomic_long_sub_return_relaxed(i, v);
1437 }
1438 
1439 static __always_inline long
1440 atomic_long_fetch_sub(long i, atomic_long_t *v)
1441 {
1442 	kcsan_mb();
1443 	instrument_atomic_read_write(v, sizeof(*v));
1444 	return arch_atomic_long_fetch_sub(i, v);
1445 }
1446 
1447 static __always_inline long
1448 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1449 {
1450 	instrument_atomic_read_write(v, sizeof(*v));
1451 	return arch_atomic_long_fetch_sub_acquire(i, v);
1452 }
1453 
1454 static __always_inline long
1455 atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1456 {
1457 	kcsan_release();
1458 	instrument_atomic_read_write(v, sizeof(*v));
1459 	return arch_atomic_long_fetch_sub_release(i, v);
1460 }
1461 
1462 static __always_inline long
1463 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1464 {
1465 	instrument_atomic_read_write(v, sizeof(*v));
1466 	return arch_atomic_long_fetch_sub_relaxed(i, v);
1467 }
1468 
1469 static __always_inline void
1470 atomic_long_inc(atomic_long_t *v)
1471 {
1472 	instrument_atomic_read_write(v, sizeof(*v));
1473 	arch_atomic_long_inc(v);
1474 }
1475 
1476 static __always_inline long
1477 atomic_long_inc_return(atomic_long_t *v)
1478 {
1479 	kcsan_mb();
1480 	instrument_atomic_read_write(v, sizeof(*v));
1481 	return arch_atomic_long_inc_return(v);
1482 }
1483 
1484 static __always_inline long
1485 atomic_long_inc_return_acquire(atomic_long_t *v)
1486 {
1487 	instrument_atomic_read_write(v, sizeof(*v));
1488 	return arch_atomic_long_inc_return_acquire(v);
1489 }
1490 
1491 static __always_inline long
1492 atomic_long_inc_return_release(atomic_long_t *v)
1493 {
1494 	kcsan_release();
1495 	instrument_atomic_read_write(v, sizeof(*v));
1496 	return arch_atomic_long_inc_return_release(v);
1497 }
1498 
1499 static __always_inline long
1500 atomic_long_inc_return_relaxed(atomic_long_t *v)
1501 {
1502 	instrument_atomic_read_write(v, sizeof(*v));
1503 	return arch_atomic_long_inc_return_relaxed(v);
1504 }
1505 
1506 static __always_inline long
1507 atomic_long_fetch_inc(atomic_long_t *v)
1508 {
1509 	kcsan_mb();
1510 	instrument_atomic_read_write(v, sizeof(*v));
1511 	return arch_atomic_long_fetch_inc(v);
1512 }
1513 
1514 static __always_inline long
1515 atomic_long_fetch_inc_acquire(atomic_long_t *v)
1516 {
1517 	instrument_atomic_read_write(v, sizeof(*v));
1518 	return arch_atomic_long_fetch_inc_acquire(v);
1519 }
1520 
1521 static __always_inline long
1522 atomic_long_fetch_inc_release(atomic_long_t *v)
1523 {
1524 	kcsan_release();
1525 	instrument_atomic_read_write(v, sizeof(*v));
1526 	return arch_atomic_long_fetch_inc_release(v);
1527 }
1528 
1529 static __always_inline long
1530 atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1531 {
1532 	instrument_atomic_read_write(v, sizeof(*v));
1533 	return arch_atomic_long_fetch_inc_relaxed(v);
1534 }
1535 
1536 static __always_inline void
1537 atomic_long_dec(atomic_long_t *v)
1538 {
1539 	instrument_atomic_read_write(v, sizeof(*v));
1540 	arch_atomic_long_dec(v);
1541 }
1542 
1543 static __always_inline long
1544 atomic_long_dec_return(atomic_long_t *v)
1545 {
1546 	kcsan_mb();
1547 	instrument_atomic_read_write(v, sizeof(*v));
1548 	return arch_atomic_long_dec_return(v);
1549 }
1550 
1551 static __always_inline long
1552 atomic_long_dec_return_acquire(atomic_long_t *v)
1553 {
1554 	instrument_atomic_read_write(v, sizeof(*v));
1555 	return arch_atomic_long_dec_return_acquire(v);
1556 }
1557 
1558 static __always_inline long
1559 atomic_long_dec_return_release(atomic_long_t *v)
1560 {
1561 	kcsan_release();
1562 	instrument_atomic_read_write(v, sizeof(*v));
1563 	return arch_atomic_long_dec_return_release(v);
1564 }
1565 
1566 static __always_inline long
1567 atomic_long_dec_return_relaxed(atomic_long_t *v)
1568 {
1569 	instrument_atomic_read_write(v, sizeof(*v));
1570 	return arch_atomic_long_dec_return_relaxed(v);
1571 }
1572 
1573 static __always_inline long
1574 atomic_long_fetch_dec(atomic_long_t *v)
1575 {
1576 	kcsan_mb();
1577 	instrument_atomic_read_write(v, sizeof(*v));
1578 	return arch_atomic_long_fetch_dec(v);
1579 }
1580 
1581 static __always_inline long
1582 atomic_long_fetch_dec_acquire(atomic_long_t *v)
1583 {
1584 	instrument_atomic_read_write(v, sizeof(*v));
1585 	return arch_atomic_long_fetch_dec_acquire(v);
1586 }
1587 
1588 static __always_inline long
1589 atomic_long_fetch_dec_release(atomic_long_t *v)
1590 {
1591 	kcsan_release();
1592 	instrument_atomic_read_write(v, sizeof(*v));
1593 	return arch_atomic_long_fetch_dec_release(v);
1594 }
1595 
1596 static __always_inline long
1597 atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1598 {
1599 	instrument_atomic_read_write(v, sizeof(*v));
1600 	return arch_atomic_long_fetch_dec_relaxed(v);
1601 }
1602 
1603 static __always_inline void
1604 atomic_long_and(long i, atomic_long_t *v)
1605 {
1606 	instrument_atomic_read_write(v, sizeof(*v));
1607 	arch_atomic_long_and(i, v);
1608 }
1609 
1610 static __always_inline long
1611 atomic_long_fetch_and(long i, atomic_long_t *v)
1612 {
1613 	kcsan_mb();
1614 	instrument_atomic_read_write(v, sizeof(*v));
1615 	return arch_atomic_long_fetch_and(i, v);
1616 }
1617 
1618 static __always_inline long
1619 atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1620 {
1621 	instrument_atomic_read_write(v, sizeof(*v));
1622 	return arch_atomic_long_fetch_and_acquire(i, v);
1623 }
1624 
1625 static __always_inline long
1626 atomic_long_fetch_and_release(long i, atomic_long_t *v)
1627 {
1628 	kcsan_release();
1629 	instrument_atomic_read_write(v, sizeof(*v));
1630 	return arch_atomic_long_fetch_and_release(i, v);
1631 }
1632 
1633 static __always_inline long
1634 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1635 {
1636 	instrument_atomic_read_write(v, sizeof(*v));
1637 	return arch_atomic_long_fetch_and_relaxed(i, v);
1638 }
1639 
1640 static __always_inline void
1641 atomic_long_andnot(long i, atomic_long_t *v)
1642 {
1643 	instrument_atomic_read_write(v, sizeof(*v));
1644 	arch_atomic_long_andnot(i, v);
1645 }
1646 
1647 static __always_inline long
1648 atomic_long_fetch_andnot(long i, atomic_long_t *v)
1649 {
1650 	kcsan_mb();
1651 	instrument_atomic_read_write(v, sizeof(*v));
1652 	return arch_atomic_long_fetch_andnot(i, v);
1653 }
1654 
1655 static __always_inline long
1656 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1657 {
1658 	instrument_atomic_read_write(v, sizeof(*v));
1659 	return arch_atomic_long_fetch_andnot_acquire(i, v);
1660 }
1661 
1662 static __always_inline long
1663 atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1664 {
1665 	kcsan_release();
1666 	instrument_atomic_read_write(v, sizeof(*v));
1667 	return arch_atomic_long_fetch_andnot_release(i, v);
1668 }
1669 
1670 static __always_inline long
1671 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1672 {
1673 	instrument_atomic_read_write(v, sizeof(*v));
1674 	return arch_atomic_long_fetch_andnot_relaxed(i, v);
1675 }
1676 
1677 static __always_inline void
1678 atomic_long_or(long i, atomic_long_t *v)
1679 {
1680 	instrument_atomic_read_write(v, sizeof(*v));
1681 	arch_atomic_long_or(i, v);
1682 }
1683 
1684 static __always_inline long
1685 atomic_long_fetch_or(long i, atomic_long_t *v)
1686 {
1687 	kcsan_mb();
1688 	instrument_atomic_read_write(v, sizeof(*v));
1689 	return arch_atomic_long_fetch_or(i, v);
1690 }
1691 
1692 static __always_inline long
1693 atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1694 {
1695 	instrument_atomic_read_write(v, sizeof(*v));
1696 	return arch_atomic_long_fetch_or_acquire(i, v);
1697 }
1698 
1699 static __always_inline long
1700 atomic_long_fetch_or_release(long i, atomic_long_t *v)
1701 {
1702 	kcsan_release();
1703 	instrument_atomic_read_write(v, sizeof(*v));
1704 	return arch_atomic_long_fetch_or_release(i, v);
1705 }
1706 
1707 static __always_inline long
1708 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1709 {
1710 	instrument_atomic_read_write(v, sizeof(*v));
1711 	return arch_atomic_long_fetch_or_relaxed(i, v);
1712 }
1713 
1714 static __always_inline void
1715 atomic_long_xor(long i, atomic_long_t *v)
1716 {
1717 	instrument_atomic_read_write(v, sizeof(*v));
1718 	arch_atomic_long_xor(i, v);
1719 }
1720 
1721 static __always_inline long
1722 atomic_long_fetch_xor(long i, atomic_long_t *v)
1723 {
1724 	kcsan_mb();
1725 	instrument_atomic_read_write(v, sizeof(*v));
1726 	return arch_atomic_long_fetch_xor(i, v);
1727 }
1728 
1729 static __always_inline long
1730 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1731 {
1732 	instrument_atomic_read_write(v, sizeof(*v));
1733 	return arch_atomic_long_fetch_xor_acquire(i, v);
1734 }
1735 
1736 static __always_inline long
1737 atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1738 {
1739 	kcsan_release();
1740 	instrument_atomic_read_write(v, sizeof(*v));
1741 	return arch_atomic_long_fetch_xor_release(i, v);
1742 }
1743 
1744 static __always_inline long
1745 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1746 {
1747 	instrument_atomic_read_write(v, sizeof(*v));
1748 	return arch_atomic_long_fetch_xor_relaxed(i, v);
1749 }
1750 
1751 static __always_inline long
1752 atomic_long_xchg(atomic_long_t *v, long i)
1753 {
1754 	kcsan_mb();
1755 	instrument_atomic_read_write(v, sizeof(*v));
1756 	return arch_atomic_long_xchg(v, i);
1757 }
1758 
1759 static __always_inline long
1760 atomic_long_xchg_acquire(atomic_long_t *v, long i)
1761 {
1762 	instrument_atomic_read_write(v, sizeof(*v));
1763 	return arch_atomic_long_xchg_acquire(v, i);
1764 }
1765 
1766 static __always_inline long
1767 atomic_long_xchg_release(atomic_long_t *v, long i)
1768 {
1769 	kcsan_release();
1770 	instrument_atomic_read_write(v, sizeof(*v));
1771 	return arch_atomic_long_xchg_release(v, i);
1772 }
1773 
1774 static __always_inline long
1775 atomic_long_xchg_relaxed(atomic_long_t *v, long i)
1776 {
1777 	instrument_atomic_read_write(v, sizeof(*v));
1778 	return arch_atomic_long_xchg_relaxed(v, i);
1779 }
1780 
1781 static __always_inline long
1782 atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1783 {
1784 	kcsan_mb();
1785 	instrument_atomic_read_write(v, sizeof(*v));
1786 	return arch_atomic_long_cmpxchg(v, old, new);
1787 }
1788 
1789 static __always_inline long
1790 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1791 {
1792 	instrument_atomic_read_write(v, sizeof(*v));
1793 	return arch_atomic_long_cmpxchg_acquire(v, old, new);
1794 }
1795 
1796 static __always_inline long
1797 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1798 {
1799 	kcsan_release();
1800 	instrument_atomic_read_write(v, sizeof(*v));
1801 	return arch_atomic_long_cmpxchg_release(v, old, new);
1802 }
1803 
1804 static __always_inline long
1805 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1806 {
1807 	instrument_atomic_read_write(v, sizeof(*v));
1808 	return arch_atomic_long_cmpxchg_relaxed(v, old, new);
1809 }
1810 
1811 static __always_inline bool
1812 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
1813 {
1814 	kcsan_mb();
1815 	instrument_atomic_read_write(v, sizeof(*v));
1816 	instrument_atomic_read_write(old, sizeof(*old));
1817 	return arch_atomic_long_try_cmpxchg(v, old, new);
1818 }
1819 
1820 static __always_inline bool
1821 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
1822 {
1823 	instrument_atomic_read_write(v, sizeof(*v));
1824 	instrument_atomic_read_write(old, sizeof(*old));
1825 	return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
1826 }
1827 
1828 static __always_inline bool
1829 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
1830 {
1831 	kcsan_release();
1832 	instrument_atomic_read_write(v, sizeof(*v));
1833 	instrument_atomic_read_write(old, sizeof(*old));
1834 	return arch_atomic_long_try_cmpxchg_release(v, old, new);
1835 }
1836 
1837 static __always_inline bool
1838 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
1839 {
1840 	instrument_atomic_read_write(v, sizeof(*v));
1841 	instrument_atomic_read_write(old, sizeof(*old));
1842 	return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
1843 }
1844 
1845 static __always_inline bool
1846 atomic_long_sub_and_test(long i, atomic_long_t *v)
1847 {
1848 	kcsan_mb();
1849 	instrument_atomic_read_write(v, sizeof(*v));
1850 	return arch_atomic_long_sub_and_test(i, v);
1851 }
1852 
1853 static __always_inline bool
1854 atomic_long_dec_and_test(atomic_long_t *v)
1855 {
1856 	kcsan_mb();
1857 	instrument_atomic_read_write(v, sizeof(*v));
1858 	return arch_atomic_long_dec_and_test(v);
1859 }
1860 
1861 static __always_inline bool
1862 atomic_long_inc_and_test(atomic_long_t *v)
1863 {
1864 	kcsan_mb();
1865 	instrument_atomic_read_write(v, sizeof(*v));
1866 	return arch_atomic_long_inc_and_test(v);
1867 }
1868 
1869 static __always_inline bool
1870 atomic_long_add_negative(long i, atomic_long_t *v)
1871 {
1872 	kcsan_mb();
1873 	instrument_atomic_read_write(v, sizeof(*v));
1874 	return arch_atomic_long_add_negative(i, v);
1875 }
1876 
1877 static __always_inline bool
1878 atomic_long_add_negative_acquire(long i, atomic_long_t *v)
1879 {
1880 	instrument_atomic_read_write(v, sizeof(*v));
1881 	return arch_atomic_long_add_negative_acquire(i, v);
1882 }
1883 
1884 static __always_inline bool
1885 atomic_long_add_negative_release(long i, atomic_long_t *v)
1886 {
1887 	kcsan_release();
1888 	instrument_atomic_read_write(v, sizeof(*v));
1889 	return arch_atomic_long_add_negative_release(i, v);
1890 }
1891 
1892 static __always_inline bool
1893 atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
1894 {
1895 	instrument_atomic_read_write(v, sizeof(*v));
1896 	return arch_atomic_long_add_negative_relaxed(i, v);
1897 }
1898 
1899 static __always_inline long
1900 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1901 {
1902 	kcsan_mb();
1903 	instrument_atomic_read_write(v, sizeof(*v));
1904 	return arch_atomic_long_fetch_add_unless(v, a, u);
1905 }
1906 
1907 static __always_inline bool
1908 atomic_long_add_unless(atomic_long_t *v, long a, long u)
1909 {
1910 	kcsan_mb();
1911 	instrument_atomic_read_write(v, sizeof(*v));
1912 	return arch_atomic_long_add_unless(v, a, u);
1913 }
1914 
1915 static __always_inline bool
1916 atomic_long_inc_not_zero(atomic_long_t *v)
1917 {
1918 	kcsan_mb();
1919 	instrument_atomic_read_write(v, sizeof(*v));
1920 	return arch_atomic_long_inc_not_zero(v);
1921 }
1922 
1923 static __always_inline bool
1924 atomic_long_inc_unless_negative(atomic_long_t *v)
1925 {
1926 	kcsan_mb();
1927 	instrument_atomic_read_write(v, sizeof(*v));
1928 	return arch_atomic_long_inc_unless_negative(v);
1929 }
1930 
1931 static __always_inline bool
1932 atomic_long_dec_unless_positive(atomic_long_t *v)
1933 {
1934 	kcsan_mb();
1935 	instrument_atomic_read_write(v, sizeof(*v));
1936 	return arch_atomic_long_dec_unless_positive(v);
1937 }
1938 
1939 static __always_inline long
1940 atomic_long_dec_if_positive(atomic_long_t *v)
1941 {
1942 	kcsan_mb();
1943 	instrument_atomic_read_write(v, sizeof(*v));
1944 	return arch_atomic_long_dec_if_positive(v);
1945 }
1946 
1947 #define xchg(ptr, ...) \
1948 ({ \
1949 	typeof(ptr) __ai_ptr = (ptr); \
1950 	kcsan_mb(); \
1951 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1952 	arch_xchg(__ai_ptr, __VA_ARGS__); \
1953 })
1954 
1955 #define xchg_acquire(ptr, ...) \
1956 ({ \
1957 	typeof(ptr) __ai_ptr = (ptr); \
1958 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1959 	arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1960 })
1961 
1962 #define xchg_release(ptr, ...) \
1963 ({ \
1964 	typeof(ptr) __ai_ptr = (ptr); \
1965 	kcsan_release(); \
1966 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1967 	arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1968 })
1969 
1970 #define xchg_relaxed(ptr, ...) \
1971 ({ \
1972 	typeof(ptr) __ai_ptr = (ptr); \
1973 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1974 	arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1975 })
1976 
1977 #define cmpxchg(ptr, ...) \
1978 ({ \
1979 	typeof(ptr) __ai_ptr = (ptr); \
1980 	kcsan_mb(); \
1981 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1982 	arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1983 })
1984 
1985 #define cmpxchg_acquire(ptr, ...) \
1986 ({ \
1987 	typeof(ptr) __ai_ptr = (ptr); \
1988 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1989 	arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1990 })
1991 
1992 #define cmpxchg_release(ptr, ...) \
1993 ({ \
1994 	typeof(ptr) __ai_ptr = (ptr); \
1995 	kcsan_release(); \
1996 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1997 	arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1998 })
1999 
2000 #define cmpxchg_relaxed(ptr, ...) \
2001 ({ \
2002 	typeof(ptr) __ai_ptr = (ptr); \
2003 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2004 	arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
2005 })
2006 
2007 #define cmpxchg64(ptr, ...) \
2008 ({ \
2009 	typeof(ptr) __ai_ptr = (ptr); \
2010 	kcsan_mb(); \
2011 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2012 	arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
2013 })
2014 
2015 #define cmpxchg64_acquire(ptr, ...) \
2016 ({ \
2017 	typeof(ptr) __ai_ptr = (ptr); \
2018 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2019 	arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
2020 })
2021 
2022 #define cmpxchg64_release(ptr, ...) \
2023 ({ \
2024 	typeof(ptr) __ai_ptr = (ptr); \
2025 	kcsan_release(); \
2026 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2027 	arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
2028 })
2029 
2030 #define cmpxchg64_relaxed(ptr, ...) \
2031 ({ \
2032 	typeof(ptr) __ai_ptr = (ptr); \
2033 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2034 	arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
2035 })
2036 
2037 #define try_cmpxchg(ptr, oldp, ...) \
2038 ({ \
2039 	typeof(ptr) __ai_ptr = (ptr); \
2040 	typeof(oldp) __ai_oldp = (oldp); \
2041 	kcsan_mb(); \
2042 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043 	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2044 	arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2045 })
2046 
2047 #define try_cmpxchg_acquire(ptr, oldp, ...) \
2048 ({ \
2049 	typeof(ptr) __ai_ptr = (ptr); \
2050 	typeof(oldp) __ai_oldp = (oldp); \
2051 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2052 	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2053 	arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2054 })
2055 
2056 #define try_cmpxchg_release(ptr, oldp, ...) \
2057 ({ \
2058 	typeof(ptr) __ai_ptr = (ptr); \
2059 	typeof(oldp) __ai_oldp = (oldp); \
2060 	kcsan_release(); \
2061 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2062 	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2063 	arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2064 })
2065 
2066 #define try_cmpxchg_relaxed(ptr, oldp, ...) \
2067 ({ \
2068 	typeof(ptr) __ai_ptr = (ptr); \
2069 	typeof(oldp) __ai_oldp = (oldp); \
2070 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2071 	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2072 	arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2073 })
2074 
2075 #define try_cmpxchg64(ptr, oldp, ...) \
2076 ({ \
2077 	typeof(ptr) __ai_ptr = (ptr); \
2078 	typeof(oldp) __ai_oldp = (oldp); \
2079 	kcsan_mb(); \
2080 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2081 	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2082 	arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2083 })
2084 
2085 #define try_cmpxchg64_acquire(ptr, oldp, ...) \
2086 ({ \
2087 	typeof(ptr) __ai_ptr = (ptr); \
2088 	typeof(oldp) __ai_oldp = (oldp); \
2089 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2090 	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2091 	arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2092 })
2093 
2094 #define try_cmpxchg64_release(ptr, oldp, ...) \
2095 ({ \
2096 	typeof(ptr) __ai_ptr = (ptr); \
2097 	typeof(oldp) __ai_oldp = (oldp); \
2098 	kcsan_release(); \
2099 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2100 	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2101 	arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2102 })
2103 
2104 #define try_cmpxchg64_relaxed(ptr, oldp, ...) \
2105 ({ \
2106 	typeof(ptr) __ai_ptr = (ptr); \
2107 	typeof(oldp) __ai_oldp = (oldp); \
2108 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2109 	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2110 	arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2111 })
2112 
2113 #define cmpxchg_local(ptr, ...) \
2114 ({ \
2115 	typeof(ptr) __ai_ptr = (ptr); \
2116 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2117 	arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
2118 })
2119 
2120 #define cmpxchg64_local(ptr, ...) \
2121 ({ \
2122 	typeof(ptr) __ai_ptr = (ptr); \
2123 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2124 	arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
2125 })
2126 
2127 #define sync_cmpxchg(ptr, ...) \
2128 ({ \
2129 	typeof(ptr) __ai_ptr = (ptr); \
2130 	kcsan_mb(); \
2131 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2132 	arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
2133 })
2134 
2135 #define try_cmpxchg_local(ptr, oldp, ...) \
2136 ({ \
2137 	typeof(ptr) __ai_ptr = (ptr); \
2138 	typeof(oldp) __ai_oldp = (oldp); \
2139 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2140 	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2141 	arch_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2142 })
2143 
2144 #define try_cmpxchg64_local(ptr, oldp, ...) \
2145 ({ \
2146 	typeof(ptr) __ai_ptr = (ptr); \
2147 	typeof(oldp) __ai_oldp = (oldp); \
2148 	instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2149 	instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2150 	arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2151 })
2152 
2153 #define cmpxchg_double(ptr, ...) \
2154 ({ \
2155 	typeof(ptr) __ai_ptr = (ptr); \
2156 	kcsan_mb(); \
2157 	instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2158 	arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
2159 })
2160 
2161 
2162 #define cmpxchg_double_local(ptr, ...) \
2163 ({ \
2164 	typeof(ptr) __ai_ptr = (ptr); \
2165 	instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2166 	arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
2167 })
2168 
2169 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2170 // 6b513a42e1a1b5962532a019b7fc91eaa044ad5e
2171