1 // SPDX-License-Identifier: GPL-2.0 2 3 // Generated by scripts/atomic/gen-atomic-instrumented.sh 4 // DO NOT MODIFY THIS FILE DIRECTLY 5 6 /* 7 * This file provides wrappers with KASAN instrumentation for atomic operations. 8 * To use this functionality an arch's atomic.h file needs to define all 9 * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include 10 * this file at the end. This file provides atomic_read() that forwards to 11 * arch_atomic_read() for actual atomic operation. 12 * Note: if an arch atomic operation is implemented by means of other atomic 13 * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use 14 * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid 15 * double instrumentation. 16 */ 17 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H 18 #define _LINUX_ATOMIC_INSTRUMENTED_H 19 20 #include <linux/build_bug.h> 21 #include <linux/compiler.h> 22 #include <linux/instrumented.h> 23 24 static __always_inline int 25 atomic_read(const atomic_t *v) 26 { 27 instrument_atomic_read(v, sizeof(*v)); 28 return arch_atomic_read(v); 29 } 30 31 static __always_inline int 32 atomic_read_acquire(const atomic_t *v) 33 { 34 instrument_atomic_read(v, sizeof(*v)); 35 return arch_atomic_read_acquire(v); 36 } 37 38 static __always_inline void 39 atomic_set(atomic_t *v, int i) 40 { 41 instrument_atomic_write(v, sizeof(*v)); 42 arch_atomic_set(v, i); 43 } 44 45 static __always_inline void 46 atomic_set_release(atomic_t *v, int i) 47 { 48 kcsan_release(); 49 instrument_atomic_write(v, sizeof(*v)); 50 arch_atomic_set_release(v, i); 51 } 52 53 static __always_inline void 54 atomic_add(int i, atomic_t *v) 55 { 56 instrument_atomic_read_write(v, sizeof(*v)); 57 arch_atomic_add(i, v); 58 } 59 60 static __always_inline int 61 atomic_add_return(int i, atomic_t *v) 62 { 63 kcsan_mb(); 64 instrument_atomic_read_write(v, sizeof(*v)); 65 return arch_atomic_add_return(i, v); 66 } 67 68 static __always_inline int 69 atomic_add_return_acquire(int i, atomic_t *v) 70 { 71 instrument_atomic_read_write(v, sizeof(*v)); 72 return arch_atomic_add_return_acquire(i, v); 73 } 74 75 static __always_inline int 76 atomic_add_return_release(int i, atomic_t *v) 77 { 78 kcsan_release(); 79 instrument_atomic_read_write(v, sizeof(*v)); 80 return arch_atomic_add_return_release(i, v); 81 } 82 83 static __always_inline int 84 atomic_add_return_relaxed(int i, atomic_t *v) 85 { 86 instrument_atomic_read_write(v, sizeof(*v)); 87 return arch_atomic_add_return_relaxed(i, v); 88 } 89 90 static __always_inline int 91 atomic_fetch_add(int i, atomic_t *v) 92 { 93 kcsan_mb(); 94 instrument_atomic_read_write(v, sizeof(*v)); 95 return arch_atomic_fetch_add(i, v); 96 } 97 98 static __always_inline int 99 atomic_fetch_add_acquire(int i, atomic_t *v) 100 { 101 instrument_atomic_read_write(v, sizeof(*v)); 102 return arch_atomic_fetch_add_acquire(i, v); 103 } 104 105 static __always_inline int 106 atomic_fetch_add_release(int i, atomic_t *v) 107 { 108 kcsan_release(); 109 instrument_atomic_read_write(v, sizeof(*v)); 110 return arch_atomic_fetch_add_release(i, v); 111 } 112 113 static __always_inline int 114 atomic_fetch_add_relaxed(int i, atomic_t *v) 115 { 116 instrument_atomic_read_write(v, sizeof(*v)); 117 return arch_atomic_fetch_add_relaxed(i, v); 118 } 119 120 static __always_inline void 121 atomic_sub(int i, atomic_t *v) 122 { 123 instrument_atomic_read_write(v, sizeof(*v)); 124 arch_atomic_sub(i, v); 125 } 126 127 static __always_inline int 128 atomic_sub_return(int i, atomic_t *v) 129 { 130 kcsan_mb(); 131 instrument_atomic_read_write(v, sizeof(*v)); 132 return arch_atomic_sub_return(i, v); 133 } 134 135 static __always_inline int 136 atomic_sub_return_acquire(int i, atomic_t *v) 137 { 138 instrument_atomic_read_write(v, sizeof(*v)); 139 return arch_atomic_sub_return_acquire(i, v); 140 } 141 142 static __always_inline int 143 atomic_sub_return_release(int i, atomic_t *v) 144 { 145 kcsan_release(); 146 instrument_atomic_read_write(v, sizeof(*v)); 147 return arch_atomic_sub_return_release(i, v); 148 } 149 150 static __always_inline int 151 atomic_sub_return_relaxed(int i, atomic_t *v) 152 { 153 instrument_atomic_read_write(v, sizeof(*v)); 154 return arch_atomic_sub_return_relaxed(i, v); 155 } 156 157 static __always_inline int 158 atomic_fetch_sub(int i, atomic_t *v) 159 { 160 kcsan_mb(); 161 instrument_atomic_read_write(v, sizeof(*v)); 162 return arch_atomic_fetch_sub(i, v); 163 } 164 165 static __always_inline int 166 atomic_fetch_sub_acquire(int i, atomic_t *v) 167 { 168 instrument_atomic_read_write(v, sizeof(*v)); 169 return arch_atomic_fetch_sub_acquire(i, v); 170 } 171 172 static __always_inline int 173 atomic_fetch_sub_release(int i, atomic_t *v) 174 { 175 kcsan_release(); 176 instrument_atomic_read_write(v, sizeof(*v)); 177 return arch_atomic_fetch_sub_release(i, v); 178 } 179 180 static __always_inline int 181 atomic_fetch_sub_relaxed(int i, atomic_t *v) 182 { 183 instrument_atomic_read_write(v, sizeof(*v)); 184 return arch_atomic_fetch_sub_relaxed(i, v); 185 } 186 187 static __always_inline void 188 atomic_inc(atomic_t *v) 189 { 190 instrument_atomic_read_write(v, sizeof(*v)); 191 arch_atomic_inc(v); 192 } 193 194 static __always_inline int 195 atomic_inc_return(atomic_t *v) 196 { 197 kcsan_mb(); 198 instrument_atomic_read_write(v, sizeof(*v)); 199 return arch_atomic_inc_return(v); 200 } 201 202 static __always_inline int 203 atomic_inc_return_acquire(atomic_t *v) 204 { 205 instrument_atomic_read_write(v, sizeof(*v)); 206 return arch_atomic_inc_return_acquire(v); 207 } 208 209 static __always_inline int 210 atomic_inc_return_release(atomic_t *v) 211 { 212 kcsan_release(); 213 instrument_atomic_read_write(v, sizeof(*v)); 214 return arch_atomic_inc_return_release(v); 215 } 216 217 static __always_inline int 218 atomic_inc_return_relaxed(atomic_t *v) 219 { 220 instrument_atomic_read_write(v, sizeof(*v)); 221 return arch_atomic_inc_return_relaxed(v); 222 } 223 224 static __always_inline int 225 atomic_fetch_inc(atomic_t *v) 226 { 227 kcsan_mb(); 228 instrument_atomic_read_write(v, sizeof(*v)); 229 return arch_atomic_fetch_inc(v); 230 } 231 232 static __always_inline int 233 atomic_fetch_inc_acquire(atomic_t *v) 234 { 235 instrument_atomic_read_write(v, sizeof(*v)); 236 return arch_atomic_fetch_inc_acquire(v); 237 } 238 239 static __always_inline int 240 atomic_fetch_inc_release(atomic_t *v) 241 { 242 kcsan_release(); 243 instrument_atomic_read_write(v, sizeof(*v)); 244 return arch_atomic_fetch_inc_release(v); 245 } 246 247 static __always_inline int 248 atomic_fetch_inc_relaxed(atomic_t *v) 249 { 250 instrument_atomic_read_write(v, sizeof(*v)); 251 return arch_atomic_fetch_inc_relaxed(v); 252 } 253 254 static __always_inline void 255 atomic_dec(atomic_t *v) 256 { 257 instrument_atomic_read_write(v, sizeof(*v)); 258 arch_atomic_dec(v); 259 } 260 261 static __always_inline int 262 atomic_dec_return(atomic_t *v) 263 { 264 kcsan_mb(); 265 instrument_atomic_read_write(v, sizeof(*v)); 266 return arch_atomic_dec_return(v); 267 } 268 269 static __always_inline int 270 atomic_dec_return_acquire(atomic_t *v) 271 { 272 instrument_atomic_read_write(v, sizeof(*v)); 273 return arch_atomic_dec_return_acquire(v); 274 } 275 276 static __always_inline int 277 atomic_dec_return_release(atomic_t *v) 278 { 279 kcsan_release(); 280 instrument_atomic_read_write(v, sizeof(*v)); 281 return arch_atomic_dec_return_release(v); 282 } 283 284 static __always_inline int 285 atomic_dec_return_relaxed(atomic_t *v) 286 { 287 instrument_atomic_read_write(v, sizeof(*v)); 288 return arch_atomic_dec_return_relaxed(v); 289 } 290 291 static __always_inline int 292 atomic_fetch_dec(atomic_t *v) 293 { 294 kcsan_mb(); 295 instrument_atomic_read_write(v, sizeof(*v)); 296 return arch_atomic_fetch_dec(v); 297 } 298 299 static __always_inline int 300 atomic_fetch_dec_acquire(atomic_t *v) 301 { 302 instrument_atomic_read_write(v, sizeof(*v)); 303 return arch_atomic_fetch_dec_acquire(v); 304 } 305 306 static __always_inline int 307 atomic_fetch_dec_release(atomic_t *v) 308 { 309 kcsan_release(); 310 instrument_atomic_read_write(v, sizeof(*v)); 311 return arch_atomic_fetch_dec_release(v); 312 } 313 314 static __always_inline int 315 atomic_fetch_dec_relaxed(atomic_t *v) 316 { 317 instrument_atomic_read_write(v, sizeof(*v)); 318 return arch_atomic_fetch_dec_relaxed(v); 319 } 320 321 static __always_inline void 322 atomic_and(int i, atomic_t *v) 323 { 324 instrument_atomic_read_write(v, sizeof(*v)); 325 arch_atomic_and(i, v); 326 } 327 328 static __always_inline int 329 atomic_fetch_and(int i, atomic_t *v) 330 { 331 kcsan_mb(); 332 instrument_atomic_read_write(v, sizeof(*v)); 333 return arch_atomic_fetch_and(i, v); 334 } 335 336 static __always_inline int 337 atomic_fetch_and_acquire(int i, atomic_t *v) 338 { 339 instrument_atomic_read_write(v, sizeof(*v)); 340 return arch_atomic_fetch_and_acquire(i, v); 341 } 342 343 static __always_inline int 344 atomic_fetch_and_release(int i, atomic_t *v) 345 { 346 kcsan_release(); 347 instrument_atomic_read_write(v, sizeof(*v)); 348 return arch_atomic_fetch_and_release(i, v); 349 } 350 351 static __always_inline int 352 atomic_fetch_and_relaxed(int i, atomic_t *v) 353 { 354 instrument_atomic_read_write(v, sizeof(*v)); 355 return arch_atomic_fetch_and_relaxed(i, v); 356 } 357 358 static __always_inline void 359 atomic_andnot(int i, atomic_t *v) 360 { 361 instrument_atomic_read_write(v, sizeof(*v)); 362 arch_atomic_andnot(i, v); 363 } 364 365 static __always_inline int 366 atomic_fetch_andnot(int i, atomic_t *v) 367 { 368 kcsan_mb(); 369 instrument_atomic_read_write(v, sizeof(*v)); 370 return arch_atomic_fetch_andnot(i, v); 371 } 372 373 static __always_inline int 374 atomic_fetch_andnot_acquire(int i, atomic_t *v) 375 { 376 instrument_atomic_read_write(v, sizeof(*v)); 377 return arch_atomic_fetch_andnot_acquire(i, v); 378 } 379 380 static __always_inline int 381 atomic_fetch_andnot_release(int i, atomic_t *v) 382 { 383 kcsan_release(); 384 instrument_atomic_read_write(v, sizeof(*v)); 385 return arch_atomic_fetch_andnot_release(i, v); 386 } 387 388 static __always_inline int 389 atomic_fetch_andnot_relaxed(int i, atomic_t *v) 390 { 391 instrument_atomic_read_write(v, sizeof(*v)); 392 return arch_atomic_fetch_andnot_relaxed(i, v); 393 } 394 395 static __always_inline void 396 atomic_or(int i, atomic_t *v) 397 { 398 instrument_atomic_read_write(v, sizeof(*v)); 399 arch_atomic_or(i, v); 400 } 401 402 static __always_inline int 403 atomic_fetch_or(int i, atomic_t *v) 404 { 405 kcsan_mb(); 406 instrument_atomic_read_write(v, sizeof(*v)); 407 return arch_atomic_fetch_or(i, v); 408 } 409 410 static __always_inline int 411 atomic_fetch_or_acquire(int i, atomic_t *v) 412 { 413 instrument_atomic_read_write(v, sizeof(*v)); 414 return arch_atomic_fetch_or_acquire(i, v); 415 } 416 417 static __always_inline int 418 atomic_fetch_or_release(int i, atomic_t *v) 419 { 420 kcsan_release(); 421 instrument_atomic_read_write(v, sizeof(*v)); 422 return arch_atomic_fetch_or_release(i, v); 423 } 424 425 static __always_inline int 426 atomic_fetch_or_relaxed(int i, atomic_t *v) 427 { 428 instrument_atomic_read_write(v, sizeof(*v)); 429 return arch_atomic_fetch_or_relaxed(i, v); 430 } 431 432 static __always_inline void 433 atomic_xor(int i, atomic_t *v) 434 { 435 instrument_atomic_read_write(v, sizeof(*v)); 436 arch_atomic_xor(i, v); 437 } 438 439 static __always_inline int 440 atomic_fetch_xor(int i, atomic_t *v) 441 { 442 kcsan_mb(); 443 instrument_atomic_read_write(v, sizeof(*v)); 444 return arch_atomic_fetch_xor(i, v); 445 } 446 447 static __always_inline int 448 atomic_fetch_xor_acquire(int i, atomic_t *v) 449 { 450 instrument_atomic_read_write(v, sizeof(*v)); 451 return arch_atomic_fetch_xor_acquire(i, v); 452 } 453 454 static __always_inline int 455 atomic_fetch_xor_release(int i, atomic_t *v) 456 { 457 kcsan_release(); 458 instrument_atomic_read_write(v, sizeof(*v)); 459 return arch_atomic_fetch_xor_release(i, v); 460 } 461 462 static __always_inline int 463 atomic_fetch_xor_relaxed(int i, atomic_t *v) 464 { 465 instrument_atomic_read_write(v, sizeof(*v)); 466 return arch_atomic_fetch_xor_relaxed(i, v); 467 } 468 469 static __always_inline int 470 atomic_xchg(atomic_t *v, int i) 471 { 472 kcsan_mb(); 473 instrument_atomic_read_write(v, sizeof(*v)); 474 return arch_atomic_xchg(v, i); 475 } 476 477 static __always_inline int 478 atomic_xchg_acquire(atomic_t *v, int i) 479 { 480 instrument_atomic_read_write(v, sizeof(*v)); 481 return arch_atomic_xchg_acquire(v, i); 482 } 483 484 static __always_inline int 485 atomic_xchg_release(atomic_t *v, int i) 486 { 487 kcsan_release(); 488 instrument_atomic_read_write(v, sizeof(*v)); 489 return arch_atomic_xchg_release(v, i); 490 } 491 492 static __always_inline int 493 atomic_xchg_relaxed(atomic_t *v, int i) 494 { 495 instrument_atomic_read_write(v, sizeof(*v)); 496 return arch_atomic_xchg_relaxed(v, i); 497 } 498 499 static __always_inline int 500 atomic_cmpxchg(atomic_t *v, int old, int new) 501 { 502 kcsan_mb(); 503 instrument_atomic_read_write(v, sizeof(*v)); 504 return arch_atomic_cmpxchg(v, old, new); 505 } 506 507 static __always_inline int 508 atomic_cmpxchg_acquire(atomic_t *v, int old, int new) 509 { 510 instrument_atomic_read_write(v, sizeof(*v)); 511 return arch_atomic_cmpxchg_acquire(v, old, new); 512 } 513 514 static __always_inline int 515 atomic_cmpxchg_release(atomic_t *v, int old, int new) 516 { 517 kcsan_release(); 518 instrument_atomic_read_write(v, sizeof(*v)); 519 return arch_atomic_cmpxchg_release(v, old, new); 520 } 521 522 static __always_inline int 523 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) 524 { 525 instrument_atomic_read_write(v, sizeof(*v)); 526 return arch_atomic_cmpxchg_relaxed(v, old, new); 527 } 528 529 static __always_inline bool 530 atomic_try_cmpxchg(atomic_t *v, int *old, int new) 531 { 532 kcsan_mb(); 533 instrument_atomic_read_write(v, sizeof(*v)); 534 instrument_atomic_read_write(old, sizeof(*old)); 535 return arch_atomic_try_cmpxchg(v, old, new); 536 } 537 538 static __always_inline bool 539 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 540 { 541 instrument_atomic_read_write(v, sizeof(*v)); 542 instrument_atomic_read_write(old, sizeof(*old)); 543 return arch_atomic_try_cmpxchg_acquire(v, old, new); 544 } 545 546 static __always_inline bool 547 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 548 { 549 kcsan_release(); 550 instrument_atomic_read_write(v, sizeof(*v)); 551 instrument_atomic_read_write(old, sizeof(*old)); 552 return arch_atomic_try_cmpxchg_release(v, old, new); 553 } 554 555 static __always_inline bool 556 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) 557 { 558 instrument_atomic_read_write(v, sizeof(*v)); 559 instrument_atomic_read_write(old, sizeof(*old)); 560 return arch_atomic_try_cmpxchg_relaxed(v, old, new); 561 } 562 563 static __always_inline bool 564 atomic_sub_and_test(int i, atomic_t *v) 565 { 566 kcsan_mb(); 567 instrument_atomic_read_write(v, sizeof(*v)); 568 return arch_atomic_sub_and_test(i, v); 569 } 570 571 static __always_inline bool 572 atomic_dec_and_test(atomic_t *v) 573 { 574 kcsan_mb(); 575 instrument_atomic_read_write(v, sizeof(*v)); 576 return arch_atomic_dec_and_test(v); 577 } 578 579 static __always_inline bool 580 atomic_inc_and_test(atomic_t *v) 581 { 582 kcsan_mb(); 583 instrument_atomic_read_write(v, sizeof(*v)); 584 return arch_atomic_inc_and_test(v); 585 } 586 587 static __always_inline bool 588 atomic_add_negative(int i, atomic_t *v) 589 { 590 kcsan_mb(); 591 instrument_atomic_read_write(v, sizeof(*v)); 592 return arch_atomic_add_negative(i, v); 593 } 594 595 static __always_inline bool 596 atomic_add_negative_acquire(int i, atomic_t *v) 597 { 598 instrument_atomic_read_write(v, sizeof(*v)); 599 return arch_atomic_add_negative_acquire(i, v); 600 } 601 602 static __always_inline bool 603 atomic_add_negative_release(int i, atomic_t *v) 604 { 605 kcsan_release(); 606 instrument_atomic_read_write(v, sizeof(*v)); 607 return arch_atomic_add_negative_release(i, v); 608 } 609 610 static __always_inline bool 611 atomic_add_negative_relaxed(int i, atomic_t *v) 612 { 613 instrument_atomic_read_write(v, sizeof(*v)); 614 return arch_atomic_add_negative_relaxed(i, v); 615 } 616 617 static __always_inline int 618 atomic_fetch_add_unless(atomic_t *v, int a, int u) 619 { 620 kcsan_mb(); 621 instrument_atomic_read_write(v, sizeof(*v)); 622 return arch_atomic_fetch_add_unless(v, a, u); 623 } 624 625 static __always_inline bool 626 atomic_add_unless(atomic_t *v, int a, int u) 627 { 628 kcsan_mb(); 629 instrument_atomic_read_write(v, sizeof(*v)); 630 return arch_atomic_add_unless(v, a, u); 631 } 632 633 static __always_inline bool 634 atomic_inc_not_zero(atomic_t *v) 635 { 636 kcsan_mb(); 637 instrument_atomic_read_write(v, sizeof(*v)); 638 return arch_atomic_inc_not_zero(v); 639 } 640 641 static __always_inline bool 642 atomic_inc_unless_negative(atomic_t *v) 643 { 644 kcsan_mb(); 645 instrument_atomic_read_write(v, sizeof(*v)); 646 return arch_atomic_inc_unless_negative(v); 647 } 648 649 static __always_inline bool 650 atomic_dec_unless_positive(atomic_t *v) 651 { 652 kcsan_mb(); 653 instrument_atomic_read_write(v, sizeof(*v)); 654 return arch_atomic_dec_unless_positive(v); 655 } 656 657 static __always_inline int 658 atomic_dec_if_positive(atomic_t *v) 659 { 660 kcsan_mb(); 661 instrument_atomic_read_write(v, sizeof(*v)); 662 return arch_atomic_dec_if_positive(v); 663 } 664 665 static __always_inline s64 666 atomic64_read(const atomic64_t *v) 667 { 668 instrument_atomic_read(v, sizeof(*v)); 669 return arch_atomic64_read(v); 670 } 671 672 static __always_inline s64 673 atomic64_read_acquire(const atomic64_t *v) 674 { 675 instrument_atomic_read(v, sizeof(*v)); 676 return arch_atomic64_read_acquire(v); 677 } 678 679 static __always_inline void 680 atomic64_set(atomic64_t *v, s64 i) 681 { 682 instrument_atomic_write(v, sizeof(*v)); 683 arch_atomic64_set(v, i); 684 } 685 686 static __always_inline void 687 atomic64_set_release(atomic64_t *v, s64 i) 688 { 689 kcsan_release(); 690 instrument_atomic_write(v, sizeof(*v)); 691 arch_atomic64_set_release(v, i); 692 } 693 694 static __always_inline void 695 atomic64_add(s64 i, atomic64_t *v) 696 { 697 instrument_atomic_read_write(v, sizeof(*v)); 698 arch_atomic64_add(i, v); 699 } 700 701 static __always_inline s64 702 atomic64_add_return(s64 i, atomic64_t *v) 703 { 704 kcsan_mb(); 705 instrument_atomic_read_write(v, sizeof(*v)); 706 return arch_atomic64_add_return(i, v); 707 } 708 709 static __always_inline s64 710 atomic64_add_return_acquire(s64 i, atomic64_t *v) 711 { 712 instrument_atomic_read_write(v, sizeof(*v)); 713 return arch_atomic64_add_return_acquire(i, v); 714 } 715 716 static __always_inline s64 717 atomic64_add_return_release(s64 i, atomic64_t *v) 718 { 719 kcsan_release(); 720 instrument_atomic_read_write(v, sizeof(*v)); 721 return arch_atomic64_add_return_release(i, v); 722 } 723 724 static __always_inline s64 725 atomic64_add_return_relaxed(s64 i, atomic64_t *v) 726 { 727 instrument_atomic_read_write(v, sizeof(*v)); 728 return arch_atomic64_add_return_relaxed(i, v); 729 } 730 731 static __always_inline s64 732 atomic64_fetch_add(s64 i, atomic64_t *v) 733 { 734 kcsan_mb(); 735 instrument_atomic_read_write(v, sizeof(*v)); 736 return arch_atomic64_fetch_add(i, v); 737 } 738 739 static __always_inline s64 740 atomic64_fetch_add_acquire(s64 i, atomic64_t *v) 741 { 742 instrument_atomic_read_write(v, sizeof(*v)); 743 return arch_atomic64_fetch_add_acquire(i, v); 744 } 745 746 static __always_inline s64 747 atomic64_fetch_add_release(s64 i, atomic64_t *v) 748 { 749 kcsan_release(); 750 instrument_atomic_read_write(v, sizeof(*v)); 751 return arch_atomic64_fetch_add_release(i, v); 752 } 753 754 static __always_inline s64 755 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) 756 { 757 instrument_atomic_read_write(v, sizeof(*v)); 758 return arch_atomic64_fetch_add_relaxed(i, v); 759 } 760 761 static __always_inline void 762 atomic64_sub(s64 i, atomic64_t *v) 763 { 764 instrument_atomic_read_write(v, sizeof(*v)); 765 arch_atomic64_sub(i, v); 766 } 767 768 static __always_inline s64 769 atomic64_sub_return(s64 i, atomic64_t *v) 770 { 771 kcsan_mb(); 772 instrument_atomic_read_write(v, sizeof(*v)); 773 return arch_atomic64_sub_return(i, v); 774 } 775 776 static __always_inline s64 777 atomic64_sub_return_acquire(s64 i, atomic64_t *v) 778 { 779 instrument_atomic_read_write(v, sizeof(*v)); 780 return arch_atomic64_sub_return_acquire(i, v); 781 } 782 783 static __always_inline s64 784 atomic64_sub_return_release(s64 i, atomic64_t *v) 785 { 786 kcsan_release(); 787 instrument_atomic_read_write(v, sizeof(*v)); 788 return arch_atomic64_sub_return_release(i, v); 789 } 790 791 static __always_inline s64 792 atomic64_sub_return_relaxed(s64 i, atomic64_t *v) 793 { 794 instrument_atomic_read_write(v, sizeof(*v)); 795 return arch_atomic64_sub_return_relaxed(i, v); 796 } 797 798 static __always_inline s64 799 atomic64_fetch_sub(s64 i, atomic64_t *v) 800 { 801 kcsan_mb(); 802 instrument_atomic_read_write(v, sizeof(*v)); 803 return arch_atomic64_fetch_sub(i, v); 804 } 805 806 static __always_inline s64 807 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) 808 { 809 instrument_atomic_read_write(v, sizeof(*v)); 810 return arch_atomic64_fetch_sub_acquire(i, v); 811 } 812 813 static __always_inline s64 814 atomic64_fetch_sub_release(s64 i, atomic64_t *v) 815 { 816 kcsan_release(); 817 instrument_atomic_read_write(v, sizeof(*v)); 818 return arch_atomic64_fetch_sub_release(i, v); 819 } 820 821 static __always_inline s64 822 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) 823 { 824 instrument_atomic_read_write(v, sizeof(*v)); 825 return arch_atomic64_fetch_sub_relaxed(i, v); 826 } 827 828 static __always_inline void 829 atomic64_inc(atomic64_t *v) 830 { 831 instrument_atomic_read_write(v, sizeof(*v)); 832 arch_atomic64_inc(v); 833 } 834 835 static __always_inline s64 836 atomic64_inc_return(atomic64_t *v) 837 { 838 kcsan_mb(); 839 instrument_atomic_read_write(v, sizeof(*v)); 840 return arch_atomic64_inc_return(v); 841 } 842 843 static __always_inline s64 844 atomic64_inc_return_acquire(atomic64_t *v) 845 { 846 instrument_atomic_read_write(v, sizeof(*v)); 847 return arch_atomic64_inc_return_acquire(v); 848 } 849 850 static __always_inline s64 851 atomic64_inc_return_release(atomic64_t *v) 852 { 853 kcsan_release(); 854 instrument_atomic_read_write(v, sizeof(*v)); 855 return arch_atomic64_inc_return_release(v); 856 } 857 858 static __always_inline s64 859 atomic64_inc_return_relaxed(atomic64_t *v) 860 { 861 instrument_atomic_read_write(v, sizeof(*v)); 862 return arch_atomic64_inc_return_relaxed(v); 863 } 864 865 static __always_inline s64 866 atomic64_fetch_inc(atomic64_t *v) 867 { 868 kcsan_mb(); 869 instrument_atomic_read_write(v, sizeof(*v)); 870 return arch_atomic64_fetch_inc(v); 871 } 872 873 static __always_inline s64 874 atomic64_fetch_inc_acquire(atomic64_t *v) 875 { 876 instrument_atomic_read_write(v, sizeof(*v)); 877 return arch_atomic64_fetch_inc_acquire(v); 878 } 879 880 static __always_inline s64 881 atomic64_fetch_inc_release(atomic64_t *v) 882 { 883 kcsan_release(); 884 instrument_atomic_read_write(v, sizeof(*v)); 885 return arch_atomic64_fetch_inc_release(v); 886 } 887 888 static __always_inline s64 889 atomic64_fetch_inc_relaxed(atomic64_t *v) 890 { 891 instrument_atomic_read_write(v, sizeof(*v)); 892 return arch_atomic64_fetch_inc_relaxed(v); 893 } 894 895 static __always_inline void 896 atomic64_dec(atomic64_t *v) 897 { 898 instrument_atomic_read_write(v, sizeof(*v)); 899 arch_atomic64_dec(v); 900 } 901 902 static __always_inline s64 903 atomic64_dec_return(atomic64_t *v) 904 { 905 kcsan_mb(); 906 instrument_atomic_read_write(v, sizeof(*v)); 907 return arch_atomic64_dec_return(v); 908 } 909 910 static __always_inline s64 911 atomic64_dec_return_acquire(atomic64_t *v) 912 { 913 instrument_atomic_read_write(v, sizeof(*v)); 914 return arch_atomic64_dec_return_acquire(v); 915 } 916 917 static __always_inline s64 918 atomic64_dec_return_release(atomic64_t *v) 919 { 920 kcsan_release(); 921 instrument_atomic_read_write(v, sizeof(*v)); 922 return arch_atomic64_dec_return_release(v); 923 } 924 925 static __always_inline s64 926 atomic64_dec_return_relaxed(atomic64_t *v) 927 { 928 instrument_atomic_read_write(v, sizeof(*v)); 929 return arch_atomic64_dec_return_relaxed(v); 930 } 931 932 static __always_inline s64 933 atomic64_fetch_dec(atomic64_t *v) 934 { 935 kcsan_mb(); 936 instrument_atomic_read_write(v, sizeof(*v)); 937 return arch_atomic64_fetch_dec(v); 938 } 939 940 static __always_inline s64 941 atomic64_fetch_dec_acquire(atomic64_t *v) 942 { 943 instrument_atomic_read_write(v, sizeof(*v)); 944 return arch_atomic64_fetch_dec_acquire(v); 945 } 946 947 static __always_inline s64 948 atomic64_fetch_dec_release(atomic64_t *v) 949 { 950 kcsan_release(); 951 instrument_atomic_read_write(v, sizeof(*v)); 952 return arch_atomic64_fetch_dec_release(v); 953 } 954 955 static __always_inline s64 956 atomic64_fetch_dec_relaxed(atomic64_t *v) 957 { 958 instrument_atomic_read_write(v, sizeof(*v)); 959 return arch_atomic64_fetch_dec_relaxed(v); 960 } 961 962 static __always_inline void 963 atomic64_and(s64 i, atomic64_t *v) 964 { 965 instrument_atomic_read_write(v, sizeof(*v)); 966 arch_atomic64_and(i, v); 967 } 968 969 static __always_inline s64 970 atomic64_fetch_and(s64 i, atomic64_t *v) 971 { 972 kcsan_mb(); 973 instrument_atomic_read_write(v, sizeof(*v)); 974 return arch_atomic64_fetch_and(i, v); 975 } 976 977 static __always_inline s64 978 atomic64_fetch_and_acquire(s64 i, atomic64_t *v) 979 { 980 instrument_atomic_read_write(v, sizeof(*v)); 981 return arch_atomic64_fetch_and_acquire(i, v); 982 } 983 984 static __always_inline s64 985 atomic64_fetch_and_release(s64 i, atomic64_t *v) 986 { 987 kcsan_release(); 988 instrument_atomic_read_write(v, sizeof(*v)); 989 return arch_atomic64_fetch_and_release(i, v); 990 } 991 992 static __always_inline s64 993 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) 994 { 995 instrument_atomic_read_write(v, sizeof(*v)); 996 return arch_atomic64_fetch_and_relaxed(i, v); 997 } 998 999 static __always_inline void 1000 atomic64_andnot(s64 i, atomic64_t *v) 1001 { 1002 instrument_atomic_read_write(v, sizeof(*v)); 1003 arch_atomic64_andnot(i, v); 1004 } 1005 1006 static __always_inline s64 1007 atomic64_fetch_andnot(s64 i, atomic64_t *v) 1008 { 1009 kcsan_mb(); 1010 instrument_atomic_read_write(v, sizeof(*v)); 1011 return arch_atomic64_fetch_andnot(i, v); 1012 } 1013 1014 static __always_inline s64 1015 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 1016 { 1017 instrument_atomic_read_write(v, sizeof(*v)); 1018 return arch_atomic64_fetch_andnot_acquire(i, v); 1019 } 1020 1021 static __always_inline s64 1022 atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 1023 { 1024 kcsan_release(); 1025 instrument_atomic_read_write(v, sizeof(*v)); 1026 return arch_atomic64_fetch_andnot_release(i, v); 1027 } 1028 1029 static __always_inline s64 1030 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) 1031 { 1032 instrument_atomic_read_write(v, sizeof(*v)); 1033 return arch_atomic64_fetch_andnot_relaxed(i, v); 1034 } 1035 1036 static __always_inline void 1037 atomic64_or(s64 i, atomic64_t *v) 1038 { 1039 instrument_atomic_read_write(v, sizeof(*v)); 1040 arch_atomic64_or(i, v); 1041 } 1042 1043 static __always_inline s64 1044 atomic64_fetch_or(s64 i, atomic64_t *v) 1045 { 1046 kcsan_mb(); 1047 instrument_atomic_read_write(v, sizeof(*v)); 1048 return arch_atomic64_fetch_or(i, v); 1049 } 1050 1051 static __always_inline s64 1052 atomic64_fetch_or_acquire(s64 i, atomic64_t *v) 1053 { 1054 instrument_atomic_read_write(v, sizeof(*v)); 1055 return arch_atomic64_fetch_or_acquire(i, v); 1056 } 1057 1058 static __always_inline s64 1059 atomic64_fetch_or_release(s64 i, atomic64_t *v) 1060 { 1061 kcsan_release(); 1062 instrument_atomic_read_write(v, sizeof(*v)); 1063 return arch_atomic64_fetch_or_release(i, v); 1064 } 1065 1066 static __always_inline s64 1067 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) 1068 { 1069 instrument_atomic_read_write(v, sizeof(*v)); 1070 return arch_atomic64_fetch_or_relaxed(i, v); 1071 } 1072 1073 static __always_inline void 1074 atomic64_xor(s64 i, atomic64_t *v) 1075 { 1076 instrument_atomic_read_write(v, sizeof(*v)); 1077 arch_atomic64_xor(i, v); 1078 } 1079 1080 static __always_inline s64 1081 atomic64_fetch_xor(s64 i, atomic64_t *v) 1082 { 1083 kcsan_mb(); 1084 instrument_atomic_read_write(v, sizeof(*v)); 1085 return arch_atomic64_fetch_xor(i, v); 1086 } 1087 1088 static __always_inline s64 1089 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) 1090 { 1091 instrument_atomic_read_write(v, sizeof(*v)); 1092 return arch_atomic64_fetch_xor_acquire(i, v); 1093 } 1094 1095 static __always_inline s64 1096 atomic64_fetch_xor_release(s64 i, atomic64_t *v) 1097 { 1098 kcsan_release(); 1099 instrument_atomic_read_write(v, sizeof(*v)); 1100 return arch_atomic64_fetch_xor_release(i, v); 1101 } 1102 1103 static __always_inline s64 1104 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) 1105 { 1106 instrument_atomic_read_write(v, sizeof(*v)); 1107 return arch_atomic64_fetch_xor_relaxed(i, v); 1108 } 1109 1110 static __always_inline s64 1111 atomic64_xchg(atomic64_t *v, s64 i) 1112 { 1113 kcsan_mb(); 1114 instrument_atomic_read_write(v, sizeof(*v)); 1115 return arch_atomic64_xchg(v, i); 1116 } 1117 1118 static __always_inline s64 1119 atomic64_xchg_acquire(atomic64_t *v, s64 i) 1120 { 1121 instrument_atomic_read_write(v, sizeof(*v)); 1122 return arch_atomic64_xchg_acquire(v, i); 1123 } 1124 1125 static __always_inline s64 1126 atomic64_xchg_release(atomic64_t *v, s64 i) 1127 { 1128 kcsan_release(); 1129 instrument_atomic_read_write(v, sizeof(*v)); 1130 return arch_atomic64_xchg_release(v, i); 1131 } 1132 1133 static __always_inline s64 1134 atomic64_xchg_relaxed(atomic64_t *v, s64 i) 1135 { 1136 instrument_atomic_read_write(v, sizeof(*v)); 1137 return arch_atomic64_xchg_relaxed(v, i); 1138 } 1139 1140 static __always_inline s64 1141 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 1142 { 1143 kcsan_mb(); 1144 instrument_atomic_read_write(v, sizeof(*v)); 1145 return arch_atomic64_cmpxchg(v, old, new); 1146 } 1147 1148 static __always_inline s64 1149 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) 1150 { 1151 instrument_atomic_read_write(v, sizeof(*v)); 1152 return arch_atomic64_cmpxchg_acquire(v, old, new); 1153 } 1154 1155 static __always_inline s64 1156 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) 1157 { 1158 kcsan_release(); 1159 instrument_atomic_read_write(v, sizeof(*v)); 1160 return arch_atomic64_cmpxchg_release(v, old, new); 1161 } 1162 1163 static __always_inline s64 1164 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) 1165 { 1166 instrument_atomic_read_write(v, sizeof(*v)); 1167 return arch_atomic64_cmpxchg_relaxed(v, old, new); 1168 } 1169 1170 static __always_inline bool 1171 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 1172 { 1173 kcsan_mb(); 1174 instrument_atomic_read_write(v, sizeof(*v)); 1175 instrument_atomic_read_write(old, sizeof(*old)); 1176 return arch_atomic64_try_cmpxchg(v, old, new); 1177 } 1178 1179 static __always_inline bool 1180 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 1181 { 1182 instrument_atomic_read_write(v, sizeof(*v)); 1183 instrument_atomic_read_write(old, sizeof(*old)); 1184 return arch_atomic64_try_cmpxchg_acquire(v, old, new); 1185 } 1186 1187 static __always_inline bool 1188 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 1189 { 1190 kcsan_release(); 1191 instrument_atomic_read_write(v, sizeof(*v)); 1192 instrument_atomic_read_write(old, sizeof(*old)); 1193 return arch_atomic64_try_cmpxchg_release(v, old, new); 1194 } 1195 1196 static __always_inline bool 1197 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) 1198 { 1199 instrument_atomic_read_write(v, sizeof(*v)); 1200 instrument_atomic_read_write(old, sizeof(*old)); 1201 return arch_atomic64_try_cmpxchg_relaxed(v, old, new); 1202 } 1203 1204 static __always_inline bool 1205 atomic64_sub_and_test(s64 i, atomic64_t *v) 1206 { 1207 kcsan_mb(); 1208 instrument_atomic_read_write(v, sizeof(*v)); 1209 return arch_atomic64_sub_and_test(i, v); 1210 } 1211 1212 static __always_inline bool 1213 atomic64_dec_and_test(atomic64_t *v) 1214 { 1215 kcsan_mb(); 1216 instrument_atomic_read_write(v, sizeof(*v)); 1217 return arch_atomic64_dec_and_test(v); 1218 } 1219 1220 static __always_inline bool 1221 atomic64_inc_and_test(atomic64_t *v) 1222 { 1223 kcsan_mb(); 1224 instrument_atomic_read_write(v, sizeof(*v)); 1225 return arch_atomic64_inc_and_test(v); 1226 } 1227 1228 static __always_inline bool 1229 atomic64_add_negative(s64 i, atomic64_t *v) 1230 { 1231 kcsan_mb(); 1232 instrument_atomic_read_write(v, sizeof(*v)); 1233 return arch_atomic64_add_negative(i, v); 1234 } 1235 1236 static __always_inline bool 1237 atomic64_add_negative_acquire(s64 i, atomic64_t *v) 1238 { 1239 instrument_atomic_read_write(v, sizeof(*v)); 1240 return arch_atomic64_add_negative_acquire(i, v); 1241 } 1242 1243 static __always_inline bool 1244 atomic64_add_negative_release(s64 i, atomic64_t *v) 1245 { 1246 kcsan_release(); 1247 instrument_atomic_read_write(v, sizeof(*v)); 1248 return arch_atomic64_add_negative_release(i, v); 1249 } 1250 1251 static __always_inline bool 1252 atomic64_add_negative_relaxed(s64 i, atomic64_t *v) 1253 { 1254 instrument_atomic_read_write(v, sizeof(*v)); 1255 return arch_atomic64_add_negative_relaxed(i, v); 1256 } 1257 1258 static __always_inline s64 1259 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 1260 { 1261 kcsan_mb(); 1262 instrument_atomic_read_write(v, sizeof(*v)); 1263 return arch_atomic64_fetch_add_unless(v, a, u); 1264 } 1265 1266 static __always_inline bool 1267 atomic64_add_unless(atomic64_t *v, s64 a, s64 u) 1268 { 1269 kcsan_mb(); 1270 instrument_atomic_read_write(v, sizeof(*v)); 1271 return arch_atomic64_add_unless(v, a, u); 1272 } 1273 1274 static __always_inline bool 1275 atomic64_inc_not_zero(atomic64_t *v) 1276 { 1277 kcsan_mb(); 1278 instrument_atomic_read_write(v, sizeof(*v)); 1279 return arch_atomic64_inc_not_zero(v); 1280 } 1281 1282 static __always_inline bool 1283 atomic64_inc_unless_negative(atomic64_t *v) 1284 { 1285 kcsan_mb(); 1286 instrument_atomic_read_write(v, sizeof(*v)); 1287 return arch_atomic64_inc_unless_negative(v); 1288 } 1289 1290 static __always_inline bool 1291 atomic64_dec_unless_positive(atomic64_t *v) 1292 { 1293 kcsan_mb(); 1294 instrument_atomic_read_write(v, sizeof(*v)); 1295 return arch_atomic64_dec_unless_positive(v); 1296 } 1297 1298 static __always_inline s64 1299 atomic64_dec_if_positive(atomic64_t *v) 1300 { 1301 kcsan_mb(); 1302 instrument_atomic_read_write(v, sizeof(*v)); 1303 return arch_atomic64_dec_if_positive(v); 1304 } 1305 1306 static __always_inline long 1307 atomic_long_read(const atomic_long_t *v) 1308 { 1309 instrument_atomic_read(v, sizeof(*v)); 1310 return arch_atomic_long_read(v); 1311 } 1312 1313 static __always_inline long 1314 atomic_long_read_acquire(const atomic_long_t *v) 1315 { 1316 instrument_atomic_read(v, sizeof(*v)); 1317 return arch_atomic_long_read_acquire(v); 1318 } 1319 1320 static __always_inline void 1321 atomic_long_set(atomic_long_t *v, long i) 1322 { 1323 instrument_atomic_write(v, sizeof(*v)); 1324 arch_atomic_long_set(v, i); 1325 } 1326 1327 static __always_inline void 1328 atomic_long_set_release(atomic_long_t *v, long i) 1329 { 1330 kcsan_release(); 1331 instrument_atomic_write(v, sizeof(*v)); 1332 arch_atomic_long_set_release(v, i); 1333 } 1334 1335 static __always_inline void 1336 atomic_long_add(long i, atomic_long_t *v) 1337 { 1338 instrument_atomic_read_write(v, sizeof(*v)); 1339 arch_atomic_long_add(i, v); 1340 } 1341 1342 static __always_inline long 1343 atomic_long_add_return(long i, atomic_long_t *v) 1344 { 1345 kcsan_mb(); 1346 instrument_atomic_read_write(v, sizeof(*v)); 1347 return arch_atomic_long_add_return(i, v); 1348 } 1349 1350 static __always_inline long 1351 atomic_long_add_return_acquire(long i, atomic_long_t *v) 1352 { 1353 instrument_atomic_read_write(v, sizeof(*v)); 1354 return arch_atomic_long_add_return_acquire(i, v); 1355 } 1356 1357 static __always_inline long 1358 atomic_long_add_return_release(long i, atomic_long_t *v) 1359 { 1360 kcsan_release(); 1361 instrument_atomic_read_write(v, sizeof(*v)); 1362 return arch_atomic_long_add_return_release(i, v); 1363 } 1364 1365 static __always_inline long 1366 atomic_long_add_return_relaxed(long i, atomic_long_t *v) 1367 { 1368 instrument_atomic_read_write(v, sizeof(*v)); 1369 return arch_atomic_long_add_return_relaxed(i, v); 1370 } 1371 1372 static __always_inline long 1373 atomic_long_fetch_add(long i, atomic_long_t *v) 1374 { 1375 kcsan_mb(); 1376 instrument_atomic_read_write(v, sizeof(*v)); 1377 return arch_atomic_long_fetch_add(i, v); 1378 } 1379 1380 static __always_inline long 1381 atomic_long_fetch_add_acquire(long i, atomic_long_t *v) 1382 { 1383 instrument_atomic_read_write(v, sizeof(*v)); 1384 return arch_atomic_long_fetch_add_acquire(i, v); 1385 } 1386 1387 static __always_inline long 1388 atomic_long_fetch_add_release(long i, atomic_long_t *v) 1389 { 1390 kcsan_release(); 1391 instrument_atomic_read_write(v, sizeof(*v)); 1392 return arch_atomic_long_fetch_add_release(i, v); 1393 } 1394 1395 static __always_inline long 1396 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) 1397 { 1398 instrument_atomic_read_write(v, sizeof(*v)); 1399 return arch_atomic_long_fetch_add_relaxed(i, v); 1400 } 1401 1402 static __always_inline void 1403 atomic_long_sub(long i, atomic_long_t *v) 1404 { 1405 instrument_atomic_read_write(v, sizeof(*v)); 1406 arch_atomic_long_sub(i, v); 1407 } 1408 1409 static __always_inline long 1410 atomic_long_sub_return(long i, atomic_long_t *v) 1411 { 1412 kcsan_mb(); 1413 instrument_atomic_read_write(v, sizeof(*v)); 1414 return arch_atomic_long_sub_return(i, v); 1415 } 1416 1417 static __always_inline long 1418 atomic_long_sub_return_acquire(long i, atomic_long_t *v) 1419 { 1420 instrument_atomic_read_write(v, sizeof(*v)); 1421 return arch_atomic_long_sub_return_acquire(i, v); 1422 } 1423 1424 static __always_inline long 1425 atomic_long_sub_return_release(long i, atomic_long_t *v) 1426 { 1427 kcsan_release(); 1428 instrument_atomic_read_write(v, sizeof(*v)); 1429 return arch_atomic_long_sub_return_release(i, v); 1430 } 1431 1432 static __always_inline long 1433 atomic_long_sub_return_relaxed(long i, atomic_long_t *v) 1434 { 1435 instrument_atomic_read_write(v, sizeof(*v)); 1436 return arch_atomic_long_sub_return_relaxed(i, v); 1437 } 1438 1439 static __always_inline long 1440 atomic_long_fetch_sub(long i, atomic_long_t *v) 1441 { 1442 kcsan_mb(); 1443 instrument_atomic_read_write(v, sizeof(*v)); 1444 return arch_atomic_long_fetch_sub(i, v); 1445 } 1446 1447 static __always_inline long 1448 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) 1449 { 1450 instrument_atomic_read_write(v, sizeof(*v)); 1451 return arch_atomic_long_fetch_sub_acquire(i, v); 1452 } 1453 1454 static __always_inline long 1455 atomic_long_fetch_sub_release(long i, atomic_long_t *v) 1456 { 1457 kcsan_release(); 1458 instrument_atomic_read_write(v, sizeof(*v)); 1459 return arch_atomic_long_fetch_sub_release(i, v); 1460 } 1461 1462 static __always_inline long 1463 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) 1464 { 1465 instrument_atomic_read_write(v, sizeof(*v)); 1466 return arch_atomic_long_fetch_sub_relaxed(i, v); 1467 } 1468 1469 static __always_inline void 1470 atomic_long_inc(atomic_long_t *v) 1471 { 1472 instrument_atomic_read_write(v, sizeof(*v)); 1473 arch_atomic_long_inc(v); 1474 } 1475 1476 static __always_inline long 1477 atomic_long_inc_return(atomic_long_t *v) 1478 { 1479 kcsan_mb(); 1480 instrument_atomic_read_write(v, sizeof(*v)); 1481 return arch_atomic_long_inc_return(v); 1482 } 1483 1484 static __always_inline long 1485 atomic_long_inc_return_acquire(atomic_long_t *v) 1486 { 1487 instrument_atomic_read_write(v, sizeof(*v)); 1488 return arch_atomic_long_inc_return_acquire(v); 1489 } 1490 1491 static __always_inline long 1492 atomic_long_inc_return_release(atomic_long_t *v) 1493 { 1494 kcsan_release(); 1495 instrument_atomic_read_write(v, sizeof(*v)); 1496 return arch_atomic_long_inc_return_release(v); 1497 } 1498 1499 static __always_inline long 1500 atomic_long_inc_return_relaxed(atomic_long_t *v) 1501 { 1502 instrument_atomic_read_write(v, sizeof(*v)); 1503 return arch_atomic_long_inc_return_relaxed(v); 1504 } 1505 1506 static __always_inline long 1507 atomic_long_fetch_inc(atomic_long_t *v) 1508 { 1509 kcsan_mb(); 1510 instrument_atomic_read_write(v, sizeof(*v)); 1511 return arch_atomic_long_fetch_inc(v); 1512 } 1513 1514 static __always_inline long 1515 atomic_long_fetch_inc_acquire(atomic_long_t *v) 1516 { 1517 instrument_atomic_read_write(v, sizeof(*v)); 1518 return arch_atomic_long_fetch_inc_acquire(v); 1519 } 1520 1521 static __always_inline long 1522 atomic_long_fetch_inc_release(atomic_long_t *v) 1523 { 1524 kcsan_release(); 1525 instrument_atomic_read_write(v, sizeof(*v)); 1526 return arch_atomic_long_fetch_inc_release(v); 1527 } 1528 1529 static __always_inline long 1530 atomic_long_fetch_inc_relaxed(atomic_long_t *v) 1531 { 1532 instrument_atomic_read_write(v, sizeof(*v)); 1533 return arch_atomic_long_fetch_inc_relaxed(v); 1534 } 1535 1536 static __always_inline void 1537 atomic_long_dec(atomic_long_t *v) 1538 { 1539 instrument_atomic_read_write(v, sizeof(*v)); 1540 arch_atomic_long_dec(v); 1541 } 1542 1543 static __always_inline long 1544 atomic_long_dec_return(atomic_long_t *v) 1545 { 1546 kcsan_mb(); 1547 instrument_atomic_read_write(v, sizeof(*v)); 1548 return arch_atomic_long_dec_return(v); 1549 } 1550 1551 static __always_inline long 1552 atomic_long_dec_return_acquire(atomic_long_t *v) 1553 { 1554 instrument_atomic_read_write(v, sizeof(*v)); 1555 return arch_atomic_long_dec_return_acquire(v); 1556 } 1557 1558 static __always_inline long 1559 atomic_long_dec_return_release(atomic_long_t *v) 1560 { 1561 kcsan_release(); 1562 instrument_atomic_read_write(v, sizeof(*v)); 1563 return arch_atomic_long_dec_return_release(v); 1564 } 1565 1566 static __always_inline long 1567 atomic_long_dec_return_relaxed(atomic_long_t *v) 1568 { 1569 instrument_atomic_read_write(v, sizeof(*v)); 1570 return arch_atomic_long_dec_return_relaxed(v); 1571 } 1572 1573 static __always_inline long 1574 atomic_long_fetch_dec(atomic_long_t *v) 1575 { 1576 kcsan_mb(); 1577 instrument_atomic_read_write(v, sizeof(*v)); 1578 return arch_atomic_long_fetch_dec(v); 1579 } 1580 1581 static __always_inline long 1582 atomic_long_fetch_dec_acquire(atomic_long_t *v) 1583 { 1584 instrument_atomic_read_write(v, sizeof(*v)); 1585 return arch_atomic_long_fetch_dec_acquire(v); 1586 } 1587 1588 static __always_inline long 1589 atomic_long_fetch_dec_release(atomic_long_t *v) 1590 { 1591 kcsan_release(); 1592 instrument_atomic_read_write(v, sizeof(*v)); 1593 return arch_atomic_long_fetch_dec_release(v); 1594 } 1595 1596 static __always_inline long 1597 atomic_long_fetch_dec_relaxed(atomic_long_t *v) 1598 { 1599 instrument_atomic_read_write(v, sizeof(*v)); 1600 return arch_atomic_long_fetch_dec_relaxed(v); 1601 } 1602 1603 static __always_inline void 1604 atomic_long_and(long i, atomic_long_t *v) 1605 { 1606 instrument_atomic_read_write(v, sizeof(*v)); 1607 arch_atomic_long_and(i, v); 1608 } 1609 1610 static __always_inline long 1611 atomic_long_fetch_and(long i, atomic_long_t *v) 1612 { 1613 kcsan_mb(); 1614 instrument_atomic_read_write(v, sizeof(*v)); 1615 return arch_atomic_long_fetch_and(i, v); 1616 } 1617 1618 static __always_inline long 1619 atomic_long_fetch_and_acquire(long i, atomic_long_t *v) 1620 { 1621 instrument_atomic_read_write(v, sizeof(*v)); 1622 return arch_atomic_long_fetch_and_acquire(i, v); 1623 } 1624 1625 static __always_inline long 1626 atomic_long_fetch_and_release(long i, atomic_long_t *v) 1627 { 1628 kcsan_release(); 1629 instrument_atomic_read_write(v, sizeof(*v)); 1630 return arch_atomic_long_fetch_and_release(i, v); 1631 } 1632 1633 static __always_inline long 1634 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) 1635 { 1636 instrument_atomic_read_write(v, sizeof(*v)); 1637 return arch_atomic_long_fetch_and_relaxed(i, v); 1638 } 1639 1640 static __always_inline void 1641 atomic_long_andnot(long i, atomic_long_t *v) 1642 { 1643 instrument_atomic_read_write(v, sizeof(*v)); 1644 arch_atomic_long_andnot(i, v); 1645 } 1646 1647 static __always_inline long 1648 atomic_long_fetch_andnot(long i, atomic_long_t *v) 1649 { 1650 kcsan_mb(); 1651 instrument_atomic_read_write(v, sizeof(*v)); 1652 return arch_atomic_long_fetch_andnot(i, v); 1653 } 1654 1655 static __always_inline long 1656 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) 1657 { 1658 instrument_atomic_read_write(v, sizeof(*v)); 1659 return arch_atomic_long_fetch_andnot_acquire(i, v); 1660 } 1661 1662 static __always_inline long 1663 atomic_long_fetch_andnot_release(long i, atomic_long_t *v) 1664 { 1665 kcsan_release(); 1666 instrument_atomic_read_write(v, sizeof(*v)); 1667 return arch_atomic_long_fetch_andnot_release(i, v); 1668 } 1669 1670 static __always_inline long 1671 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) 1672 { 1673 instrument_atomic_read_write(v, sizeof(*v)); 1674 return arch_atomic_long_fetch_andnot_relaxed(i, v); 1675 } 1676 1677 static __always_inline void 1678 atomic_long_or(long i, atomic_long_t *v) 1679 { 1680 instrument_atomic_read_write(v, sizeof(*v)); 1681 arch_atomic_long_or(i, v); 1682 } 1683 1684 static __always_inline long 1685 atomic_long_fetch_or(long i, atomic_long_t *v) 1686 { 1687 kcsan_mb(); 1688 instrument_atomic_read_write(v, sizeof(*v)); 1689 return arch_atomic_long_fetch_or(i, v); 1690 } 1691 1692 static __always_inline long 1693 atomic_long_fetch_or_acquire(long i, atomic_long_t *v) 1694 { 1695 instrument_atomic_read_write(v, sizeof(*v)); 1696 return arch_atomic_long_fetch_or_acquire(i, v); 1697 } 1698 1699 static __always_inline long 1700 atomic_long_fetch_or_release(long i, atomic_long_t *v) 1701 { 1702 kcsan_release(); 1703 instrument_atomic_read_write(v, sizeof(*v)); 1704 return arch_atomic_long_fetch_or_release(i, v); 1705 } 1706 1707 static __always_inline long 1708 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) 1709 { 1710 instrument_atomic_read_write(v, sizeof(*v)); 1711 return arch_atomic_long_fetch_or_relaxed(i, v); 1712 } 1713 1714 static __always_inline void 1715 atomic_long_xor(long i, atomic_long_t *v) 1716 { 1717 instrument_atomic_read_write(v, sizeof(*v)); 1718 arch_atomic_long_xor(i, v); 1719 } 1720 1721 static __always_inline long 1722 atomic_long_fetch_xor(long i, atomic_long_t *v) 1723 { 1724 kcsan_mb(); 1725 instrument_atomic_read_write(v, sizeof(*v)); 1726 return arch_atomic_long_fetch_xor(i, v); 1727 } 1728 1729 static __always_inline long 1730 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) 1731 { 1732 instrument_atomic_read_write(v, sizeof(*v)); 1733 return arch_atomic_long_fetch_xor_acquire(i, v); 1734 } 1735 1736 static __always_inline long 1737 atomic_long_fetch_xor_release(long i, atomic_long_t *v) 1738 { 1739 kcsan_release(); 1740 instrument_atomic_read_write(v, sizeof(*v)); 1741 return arch_atomic_long_fetch_xor_release(i, v); 1742 } 1743 1744 static __always_inline long 1745 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) 1746 { 1747 instrument_atomic_read_write(v, sizeof(*v)); 1748 return arch_atomic_long_fetch_xor_relaxed(i, v); 1749 } 1750 1751 static __always_inline long 1752 atomic_long_xchg(atomic_long_t *v, long i) 1753 { 1754 kcsan_mb(); 1755 instrument_atomic_read_write(v, sizeof(*v)); 1756 return arch_atomic_long_xchg(v, i); 1757 } 1758 1759 static __always_inline long 1760 atomic_long_xchg_acquire(atomic_long_t *v, long i) 1761 { 1762 instrument_atomic_read_write(v, sizeof(*v)); 1763 return arch_atomic_long_xchg_acquire(v, i); 1764 } 1765 1766 static __always_inline long 1767 atomic_long_xchg_release(atomic_long_t *v, long i) 1768 { 1769 kcsan_release(); 1770 instrument_atomic_read_write(v, sizeof(*v)); 1771 return arch_atomic_long_xchg_release(v, i); 1772 } 1773 1774 static __always_inline long 1775 atomic_long_xchg_relaxed(atomic_long_t *v, long i) 1776 { 1777 instrument_atomic_read_write(v, sizeof(*v)); 1778 return arch_atomic_long_xchg_relaxed(v, i); 1779 } 1780 1781 static __always_inline long 1782 atomic_long_cmpxchg(atomic_long_t *v, long old, long new) 1783 { 1784 kcsan_mb(); 1785 instrument_atomic_read_write(v, sizeof(*v)); 1786 return arch_atomic_long_cmpxchg(v, old, new); 1787 } 1788 1789 static __always_inline long 1790 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) 1791 { 1792 instrument_atomic_read_write(v, sizeof(*v)); 1793 return arch_atomic_long_cmpxchg_acquire(v, old, new); 1794 } 1795 1796 static __always_inline long 1797 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) 1798 { 1799 kcsan_release(); 1800 instrument_atomic_read_write(v, sizeof(*v)); 1801 return arch_atomic_long_cmpxchg_release(v, old, new); 1802 } 1803 1804 static __always_inline long 1805 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) 1806 { 1807 instrument_atomic_read_write(v, sizeof(*v)); 1808 return arch_atomic_long_cmpxchg_relaxed(v, old, new); 1809 } 1810 1811 static __always_inline bool 1812 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) 1813 { 1814 kcsan_mb(); 1815 instrument_atomic_read_write(v, sizeof(*v)); 1816 instrument_atomic_read_write(old, sizeof(*old)); 1817 return arch_atomic_long_try_cmpxchg(v, old, new); 1818 } 1819 1820 static __always_inline bool 1821 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) 1822 { 1823 instrument_atomic_read_write(v, sizeof(*v)); 1824 instrument_atomic_read_write(old, sizeof(*old)); 1825 return arch_atomic_long_try_cmpxchg_acquire(v, old, new); 1826 } 1827 1828 static __always_inline bool 1829 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) 1830 { 1831 kcsan_release(); 1832 instrument_atomic_read_write(v, sizeof(*v)); 1833 instrument_atomic_read_write(old, sizeof(*old)); 1834 return arch_atomic_long_try_cmpxchg_release(v, old, new); 1835 } 1836 1837 static __always_inline bool 1838 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) 1839 { 1840 instrument_atomic_read_write(v, sizeof(*v)); 1841 instrument_atomic_read_write(old, sizeof(*old)); 1842 return arch_atomic_long_try_cmpxchg_relaxed(v, old, new); 1843 } 1844 1845 static __always_inline bool 1846 atomic_long_sub_and_test(long i, atomic_long_t *v) 1847 { 1848 kcsan_mb(); 1849 instrument_atomic_read_write(v, sizeof(*v)); 1850 return arch_atomic_long_sub_and_test(i, v); 1851 } 1852 1853 static __always_inline bool 1854 atomic_long_dec_and_test(atomic_long_t *v) 1855 { 1856 kcsan_mb(); 1857 instrument_atomic_read_write(v, sizeof(*v)); 1858 return arch_atomic_long_dec_and_test(v); 1859 } 1860 1861 static __always_inline bool 1862 atomic_long_inc_and_test(atomic_long_t *v) 1863 { 1864 kcsan_mb(); 1865 instrument_atomic_read_write(v, sizeof(*v)); 1866 return arch_atomic_long_inc_and_test(v); 1867 } 1868 1869 static __always_inline bool 1870 atomic_long_add_negative(long i, atomic_long_t *v) 1871 { 1872 kcsan_mb(); 1873 instrument_atomic_read_write(v, sizeof(*v)); 1874 return arch_atomic_long_add_negative(i, v); 1875 } 1876 1877 static __always_inline bool 1878 atomic_long_add_negative_acquire(long i, atomic_long_t *v) 1879 { 1880 instrument_atomic_read_write(v, sizeof(*v)); 1881 return arch_atomic_long_add_negative_acquire(i, v); 1882 } 1883 1884 static __always_inline bool 1885 atomic_long_add_negative_release(long i, atomic_long_t *v) 1886 { 1887 kcsan_release(); 1888 instrument_atomic_read_write(v, sizeof(*v)); 1889 return arch_atomic_long_add_negative_release(i, v); 1890 } 1891 1892 static __always_inline bool 1893 atomic_long_add_negative_relaxed(long i, atomic_long_t *v) 1894 { 1895 instrument_atomic_read_write(v, sizeof(*v)); 1896 return arch_atomic_long_add_negative_relaxed(i, v); 1897 } 1898 1899 static __always_inline long 1900 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) 1901 { 1902 kcsan_mb(); 1903 instrument_atomic_read_write(v, sizeof(*v)); 1904 return arch_atomic_long_fetch_add_unless(v, a, u); 1905 } 1906 1907 static __always_inline bool 1908 atomic_long_add_unless(atomic_long_t *v, long a, long u) 1909 { 1910 kcsan_mb(); 1911 instrument_atomic_read_write(v, sizeof(*v)); 1912 return arch_atomic_long_add_unless(v, a, u); 1913 } 1914 1915 static __always_inline bool 1916 atomic_long_inc_not_zero(atomic_long_t *v) 1917 { 1918 kcsan_mb(); 1919 instrument_atomic_read_write(v, sizeof(*v)); 1920 return arch_atomic_long_inc_not_zero(v); 1921 } 1922 1923 static __always_inline bool 1924 atomic_long_inc_unless_negative(atomic_long_t *v) 1925 { 1926 kcsan_mb(); 1927 instrument_atomic_read_write(v, sizeof(*v)); 1928 return arch_atomic_long_inc_unless_negative(v); 1929 } 1930 1931 static __always_inline bool 1932 atomic_long_dec_unless_positive(atomic_long_t *v) 1933 { 1934 kcsan_mb(); 1935 instrument_atomic_read_write(v, sizeof(*v)); 1936 return arch_atomic_long_dec_unless_positive(v); 1937 } 1938 1939 static __always_inline long 1940 atomic_long_dec_if_positive(atomic_long_t *v) 1941 { 1942 kcsan_mb(); 1943 instrument_atomic_read_write(v, sizeof(*v)); 1944 return arch_atomic_long_dec_if_positive(v); 1945 } 1946 1947 #define xchg(ptr, ...) \ 1948 ({ \ 1949 typeof(ptr) __ai_ptr = (ptr); \ 1950 kcsan_mb(); \ 1951 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1952 arch_xchg(__ai_ptr, __VA_ARGS__); \ 1953 }) 1954 1955 #define xchg_acquire(ptr, ...) \ 1956 ({ \ 1957 typeof(ptr) __ai_ptr = (ptr); \ 1958 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1959 arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ 1960 }) 1961 1962 #define xchg_release(ptr, ...) \ 1963 ({ \ 1964 typeof(ptr) __ai_ptr = (ptr); \ 1965 kcsan_release(); \ 1966 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1967 arch_xchg_release(__ai_ptr, __VA_ARGS__); \ 1968 }) 1969 1970 #define xchg_relaxed(ptr, ...) \ 1971 ({ \ 1972 typeof(ptr) __ai_ptr = (ptr); \ 1973 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1974 arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ 1975 }) 1976 1977 #define cmpxchg(ptr, ...) \ 1978 ({ \ 1979 typeof(ptr) __ai_ptr = (ptr); \ 1980 kcsan_mb(); \ 1981 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1982 arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ 1983 }) 1984 1985 #define cmpxchg_acquire(ptr, ...) \ 1986 ({ \ 1987 typeof(ptr) __ai_ptr = (ptr); \ 1988 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1989 arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ 1990 }) 1991 1992 #define cmpxchg_release(ptr, ...) \ 1993 ({ \ 1994 typeof(ptr) __ai_ptr = (ptr); \ 1995 kcsan_release(); \ 1996 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1997 arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ 1998 }) 1999 2000 #define cmpxchg_relaxed(ptr, ...) \ 2001 ({ \ 2002 typeof(ptr) __ai_ptr = (ptr); \ 2003 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2004 arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ 2005 }) 2006 2007 #define cmpxchg64(ptr, ...) \ 2008 ({ \ 2009 typeof(ptr) __ai_ptr = (ptr); \ 2010 kcsan_mb(); \ 2011 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2012 arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \ 2013 }) 2014 2015 #define cmpxchg64_acquire(ptr, ...) \ 2016 ({ \ 2017 typeof(ptr) __ai_ptr = (ptr); \ 2018 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2019 arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ 2020 }) 2021 2022 #define cmpxchg64_release(ptr, ...) \ 2023 ({ \ 2024 typeof(ptr) __ai_ptr = (ptr); \ 2025 kcsan_release(); \ 2026 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2027 arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ 2028 }) 2029 2030 #define cmpxchg64_relaxed(ptr, ...) \ 2031 ({ \ 2032 typeof(ptr) __ai_ptr = (ptr); \ 2033 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2034 arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ 2035 }) 2036 2037 #define try_cmpxchg(ptr, oldp, ...) \ 2038 ({ \ 2039 typeof(ptr) __ai_ptr = (ptr); \ 2040 typeof(oldp) __ai_oldp = (oldp); \ 2041 kcsan_mb(); \ 2042 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2043 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 2044 arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 2045 }) 2046 2047 #define try_cmpxchg_acquire(ptr, oldp, ...) \ 2048 ({ \ 2049 typeof(ptr) __ai_ptr = (ptr); \ 2050 typeof(oldp) __ai_oldp = (oldp); \ 2051 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2052 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 2053 arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 2054 }) 2055 2056 #define try_cmpxchg_release(ptr, oldp, ...) \ 2057 ({ \ 2058 typeof(ptr) __ai_ptr = (ptr); \ 2059 typeof(oldp) __ai_oldp = (oldp); \ 2060 kcsan_release(); \ 2061 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2062 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 2063 arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 2064 }) 2065 2066 #define try_cmpxchg_relaxed(ptr, oldp, ...) \ 2067 ({ \ 2068 typeof(ptr) __ai_ptr = (ptr); \ 2069 typeof(oldp) __ai_oldp = (oldp); \ 2070 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2071 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 2072 arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 2073 }) 2074 2075 #define try_cmpxchg64(ptr, oldp, ...) \ 2076 ({ \ 2077 typeof(ptr) __ai_ptr = (ptr); \ 2078 typeof(oldp) __ai_oldp = (oldp); \ 2079 kcsan_mb(); \ 2080 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2081 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 2082 arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 2083 }) 2084 2085 #define try_cmpxchg64_acquire(ptr, oldp, ...) \ 2086 ({ \ 2087 typeof(ptr) __ai_ptr = (ptr); \ 2088 typeof(oldp) __ai_oldp = (oldp); \ 2089 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2090 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 2091 arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 2092 }) 2093 2094 #define try_cmpxchg64_release(ptr, oldp, ...) \ 2095 ({ \ 2096 typeof(ptr) __ai_ptr = (ptr); \ 2097 typeof(oldp) __ai_oldp = (oldp); \ 2098 kcsan_release(); \ 2099 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2100 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 2101 arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 2102 }) 2103 2104 #define try_cmpxchg64_relaxed(ptr, oldp, ...) \ 2105 ({ \ 2106 typeof(ptr) __ai_ptr = (ptr); \ 2107 typeof(oldp) __ai_oldp = (oldp); \ 2108 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2109 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 2110 arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 2111 }) 2112 2113 #define cmpxchg_local(ptr, ...) \ 2114 ({ \ 2115 typeof(ptr) __ai_ptr = (ptr); \ 2116 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2117 arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ 2118 }) 2119 2120 #define cmpxchg64_local(ptr, ...) \ 2121 ({ \ 2122 typeof(ptr) __ai_ptr = (ptr); \ 2123 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2124 arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ 2125 }) 2126 2127 #define sync_cmpxchg(ptr, ...) \ 2128 ({ \ 2129 typeof(ptr) __ai_ptr = (ptr); \ 2130 kcsan_mb(); \ 2131 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2132 arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ 2133 }) 2134 2135 #define try_cmpxchg_local(ptr, oldp, ...) \ 2136 ({ \ 2137 typeof(ptr) __ai_ptr = (ptr); \ 2138 typeof(oldp) __ai_oldp = (oldp); \ 2139 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2140 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 2141 arch_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 2142 }) 2143 2144 #define try_cmpxchg64_local(ptr, oldp, ...) \ 2145 ({ \ 2146 typeof(ptr) __ai_ptr = (ptr); \ 2147 typeof(oldp) __ai_oldp = (oldp); \ 2148 instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2149 instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ 2150 arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 2151 }) 2152 2153 #define cmpxchg_double(ptr, ...) \ 2154 ({ \ 2155 typeof(ptr) __ai_ptr = (ptr); \ 2156 kcsan_mb(); \ 2157 instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ 2158 arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \ 2159 }) 2160 2161 2162 #define cmpxchg_double_local(ptr, ...) \ 2163 ({ \ 2164 typeof(ptr) __ai_ptr = (ptr); \ 2165 instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ 2166 arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ 2167 }) 2168 2169 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ 2170 // 6b513a42e1a1b5962532a019b7fc91eaa044ad5e 2171