1 // SPDX-License-Identifier: GPL-2.0 2 3 // Generated by scripts/atomic/gen-atomic-instrumented.sh 4 // DO NOT MODIFY THIS FILE DIRECTLY 5 6 /* 7 * This file provides wrappers with KASAN instrumentation for atomic operations. 8 * To use this functionality an arch's atomic.h file needs to define all 9 * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include 10 * this file at the end. This file provides atomic_read() that forwards to 11 * arch_atomic_read() for actual atomic operation. 12 * Note: if an arch atomic operation is implemented by means of other atomic 13 * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use 14 * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid 15 * double instrumentation. 16 */ 17 #ifndef _LINUX_ATOMIC_INSTRUMENTED_H 18 #define _LINUX_ATOMIC_INSTRUMENTED_H 19 20 #include <linux/build_bug.h> 21 #include <linux/compiler.h> 22 #include <linux/instrumented.h> 23 24 static __always_inline int 25 atomic_read(const atomic_t *v) 26 { 27 instrument_atomic_read(v, sizeof(*v)); 28 return arch_atomic_read(v); 29 } 30 31 static __always_inline int 32 atomic_read_acquire(const atomic_t *v) 33 { 34 instrument_atomic_read(v, sizeof(*v)); 35 return arch_atomic_read_acquire(v); 36 } 37 38 static __always_inline void 39 atomic_set(atomic_t *v, int i) 40 { 41 instrument_atomic_write(v, sizeof(*v)); 42 arch_atomic_set(v, i); 43 } 44 45 static __always_inline void 46 atomic_set_release(atomic_t *v, int i) 47 { 48 kcsan_release(); 49 instrument_atomic_write(v, sizeof(*v)); 50 arch_atomic_set_release(v, i); 51 } 52 53 static __always_inline void 54 atomic_add(int i, atomic_t *v) 55 { 56 instrument_atomic_read_write(v, sizeof(*v)); 57 arch_atomic_add(i, v); 58 } 59 60 static __always_inline int 61 atomic_add_return(int i, atomic_t *v) 62 { 63 kcsan_mb(); 64 instrument_atomic_read_write(v, sizeof(*v)); 65 return arch_atomic_add_return(i, v); 66 } 67 68 static __always_inline int 69 atomic_add_return_acquire(int i, atomic_t *v) 70 { 71 instrument_atomic_read_write(v, sizeof(*v)); 72 return arch_atomic_add_return_acquire(i, v); 73 } 74 75 static __always_inline int 76 atomic_add_return_release(int i, atomic_t *v) 77 { 78 kcsan_release(); 79 instrument_atomic_read_write(v, sizeof(*v)); 80 return arch_atomic_add_return_release(i, v); 81 } 82 83 static __always_inline int 84 atomic_add_return_relaxed(int i, atomic_t *v) 85 { 86 instrument_atomic_read_write(v, sizeof(*v)); 87 return arch_atomic_add_return_relaxed(i, v); 88 } 89 90 static __always_inline int 91 atomic_fetch_add(int i, atomic_t *v) 92 { 93 kcsan_mb(); 94 instrument_atomic_read_write(v, sizeof(*v)); 95 return arch_atomic_fetch_add(i, v); 96 } 97 98 static __always_inline int 99 atomic_fetch_add_acquire(int i, atomic_t *v) 100 { 101 instrument_atomic_read_write(v, sizeof(*v)); 102 return arch_atomic_fetch_add_acquire(i, v); 103 } 104 105 static __always_inline int 106 atomic_fetch_add_release(int i, atomic_t *v) 107 { 108 kcsan_release(); 109 instrument_atomic_read_write(v, sizeof(*v)); 110 return arch_atomic_fetch_add_release(i, v); 111 } 112 113 static __always_inline int 114 atomic_fetch_add_relaxed(int i, atomic_t *v) 115 { 116 instrument_atomic_read_write(v, sizeof(*v)); 117 return arch_atomic_fetch_add_relaxed(i, v); 118 } 119 120 static __always_inline void 121 atomic_sub(int i, atomic_t *v) 122 { 123 instrument_atomic_read_write(v, sizeof(*v)); 124 arch_atomic_sub(i, v); 125 } 126 127 static __always_inline int 128 atomic_sub_return(int i, atomic_t *v) 129 { 130 kcsan_mb(); 131 instrument_atomic_read_write(v, sizeof(*v)); 132 return arch_atomic_sub_return(i, v); 133 } 134 135 static __always_inline int 136 atomic_sub_return_acquire(int i, atomic_t *v) 137 { 138 instrument_atomic_read_write(v, sizeof(*v)); 139 return arch_atomic_sub_return_acquire(i, v); 140 } 141 142 static __always_inline int 143 atomic_sub_return_release(int i, atomic_t *v) 144 { 145 kcsan_release(); 146 instrument_atomic_read_write(v, sizeof(*v)); 147 return arch_atomic_sub_return_release(i, v); 148 } 149 150 static __always_inline int 151 atomic_sub_return_relaxed(int i, atomic_t *v) 152 { 153 instrument_atomic_read_write(v, sizeof(*v)); 154 return arch_atomic_sub_return_relaxed(i, v); 155 } 156 157 static __always_inline int 158 atomic_fetch_sub(int i, atomic_t *v) 159 { 160 kcsan_mb(); 161 instrument_atomic_read_write(v, sizeof(*v)); 162 return arch_atomic_fetch_sub(i, v); 163 } 164 165 static __always_inline int 166 atomic_fetch_sub_acquire(int i, atomic_t *v) 167 { 168 instrument_atomic_read_write(v, sizeof(*v)); 169 return arch_atomic_fetch_sub_acquire(i, v); 170 } 171 172 static __always_inline int 173 atomic_fetch_sub_release(int i, atomic_t *v) 174 { 175 kcsan_release(); 176 instrument_atomic_read_write(v, sizeof(*v)); 177 return arch_atomic_fetch_sub_release(i, v); 178 } 179 180 static __always_inline int 181 atomic_fetch_sub_relaxed(int i, atomic_t *v) 182 { 183 instrument_atomic_read_write(v, sizeof(*v)); 184 return arch_atomic_fetch_sub_relaxed(i, v); 185 } 186 187 static __always_inline void 188 atomic_inc(atomic_t *v) 189 { 190 instrument_atomic_read_write(v, sizeof(*v)); 191 arch_atomic_inc(v); 192 } 193 194 static __always_inline int 195 atomic_inc_return(atomic_t *v) 196 { 197 kcsan_mb(); 198 instrument_atomic_read_write(v, sizeof(*v)); 199 return arch_atomic_inc_return(v); 200 } 201 202 static __always_inline int 203 atomic_inc_return_acquire(atomic_t *v) 204 { 205 instrument_atomic_read_write(v, sizeof(*v)); 206 return arch_atomic_inc_return_acquire(v); 207 } 208 209 static __always_inline int 210 atomic_inc_return_release(atomic_t *v) 211 { 212 kcsan_release(); 213 instrument_atomic_read_write(v, sizeof(*v)); 214 return arch_atomic_inc_return_release(v); 215 } 216 217 static __always_inline int 218 atomic_inc_return_relaxed(atomic_t *v) 219 { 220 instrument_atomic_read_write(v, sizeof(*v)); 221 return arch_atomic_inc_return_relaxed(v); 222 } 223 224 static __always_inline int 225 atomic_fetch_inc(atomic_t *v) 226 { 227 kcsan_mb(); 228 instrument_atomic_read_write(v, sizeof(*v)); 229 return arch_atomic_fetch_inc(v); 230 } 231 232 static __always_inline int 233 atomic_fetch_inc_acquire(atomic_t *v) 234 { 235 instrument_atomic_read_write(v, sizeof(*v)); 236 return arch_atomic_fetch_inc_acquire(v); 237 } 238 239 static __always_inline int 240 atomic_fetch_inc_release(atomic_t *v) 241 { 242 kcsan_release(); 243 instrument_atomic_read_write(v, sizeof(*v)); 244 return arch_atomic_fetch_inc_release(v); 245 } 246 247 static __always_inline int 248 atomic_fetch_inc_relaxed(atomic_t *v) 249 { 250 instrument_atomic_read_write(v, sizeof(*v)); 251 return arch_atomic_fetch_inc_relaxed(v); 252 } 253 254 static __always_inline void 255 atomic_dec(atomic_t *v) 256 { 257 instrument_atomic_read_write(v, sizeof(*v)); 258 arch_atomic_dec(v); 259 } 260 261 static __always_inline int 262 atomic_dec_return(atomic_t *v) 263 { 264 kcsan_mb(); 265 instrument_atomic_read_write(v, sizeof(*v)); 266 return arch_atomic_dec_return(v); 267 } 268 269 static __always_inline int 270 atomic_dec_return_acquire(atomic_t *v) 271 { 272 instrument_atomic_read_write(v, sizeof(*v)); 273 return arch_atomic_dec_return_acquire(v); 274 } 275 276 static __always_inline int 277 atomic_dec_return_release(atomic_t *v) 278 { 279 kcsan_release(); 280 instrument_atomic_read_write(v, sizeof(*v)); 281 return arch_atomic_dec_return_release(v); 282 } 283 284 static __always_inline int 285 atomic_dec_return_relaxed(atomic_t *v) 286 { 287 instrument_atomic_read_write(v, sizeof(*v)); 288 return arch_atomic_dec_return_relaxed(v); 289 } 290 291 static __always_inline int 292 atomic_fetch_dec(atomic_t *v) 293 { 294 kcsan_mb(); 295 instrument_atomic_read_write(v, sizeof(*v)); 296 return arch_atomic_fetch_dec(v); 297 } 298 299 static __always_inline int 300 atomic_fetch_dec_acquire(atomic_t *v) 301 { 302 instrument_atomic_read_write(v, sizeof(*v)); 303 return arch_atomic_fetch_dec_acquire(v); 304 } 305 306 static __always_inline int 307 atomic_fetch_dec_release(atomic_t *v) 308 { 309 kcsan_release(); 310 instrument_atomic_read_write(v, sizeof(*v)); 311 return arch_atomic_fetch_dec_release(v); 312 } 313 314 static __always_inline int 315 atomic_fetch_dec_relaxed(atomic_t *v) 316 { 317 instrument_atomic_read_write(v, sizeof(*v)); 318 return arch_atomic_fetch_dec_relaxed(v); 319 } 320 321 static __always_inline void 322 atomic_and(int i, atomic_t *v) 323 { 324 instrument_atomic_read_write(v, sizeof(*v)); 325 arch_atomic_and(i, v); 326 } 327 328 static __always_inline int 329 atomic_fetch_and(int i, atomic_t *v) 330 { 331 kcsan_mb(); 332 instrument_atomic_read_write(v, sizeof(*v)); 333 return arch_atomic_fetch_and(i, v); 334 } 335 336 static __always_inline int 337 atomic_fetch_and_acquire(int i, atomic_t *v) 338 { 339 instrument_atomic_read_write(v, sizeof(*v)); 340 return arch_atomic_fetch_and_acquire(i, v); 341 } 342 343 static __always_inline int 344 atomic_fetch_and_release(int i, atomic_t *v) 345 { 346 kcsan_release(); 347 instrument_atomic_read_write(v, sizeof(*v)); 348 return arch_atomic_fetch_and_release(i, v); 349 } 350 351 static __always_inline int 352 atomic_fetch_and_relaxed(int i, atomic_t *v) 353 { 354 instrument_atomic_read_write(v, sizeof(*v)); 355 return arch_atomic_fetch_and_relaxed(i, v); 356 } 357 358 static __always_inline void 359 atomic_andnot(int i, atomic_t *v) 360 { 361 instrument_atomic_read_write(v, sizeof(*v)); 362 arch_atomic_andnot(i, v); 363 } 364 365 static __always_inline int 366 atomic_fetch_andnot(int i, atomic_t *v) 367 { 368 kcsan_mb(); 369 instrument_atomic_read_write(v, sizeof(*v)); 370 return arch_atomic_fetch_andnot(i, v); 371 } 372 373 static __always_inline int 374 atomic_fetch_andnot_acquire(int i, atomic_t *v) 375 { 376 instrument_atomic_read_write(v, sizeof(*v)); 377 return arch_atomic_fetch_andnot_acquire(i, v); 378 } 379 380 static __always_inline int 381 atomic_fetch_andnot_release(int i, atomic_t *v) 382 { 383 kcsan_release(); 384 instrument_atomic_read_write(v, sizeof(*v)); 385 return arch_atomic_fetch_andnot_release(i, v); 386 } 387 388 static __always_inline int 389 atomic_fetch_andnot_relaxed(int i, atomic_t *v) 390 { 391 instrument_atomic_read_write(v, sizeof(*v)); 392 return arch_atomic_fetch_andnot_relaxed(i, v); 393 } 394 395 static __always_inline void 396 atomic_or(int i, atomic_t *v) 397 { 398 instrument_atomic_read_write(v, sizeof(*v)); 399 arch_atomic_or(i, v); 400 } 401 402 static __always_inline int 403 atomic_fetch_or(int i, atomic_t *v) 404 { 405 kcsan_mb(); 406 instrument_atomic_read_write(v, sizeof(*v)); 407 return arch_atomic_fetch_or(i, v); 408 } 409 410 static __always_inline int 411 atomic_fetch_or_acquire(int i, atomic_t *v) 412 { 413 instrument_atomic_read_write(v, sizeof(*v)); 414 return arch_atomic_fetch_or_acquire(i, v); 415 } 416 417 static __always_inline int 418 atomic_fetch_or_release(int i, atomic_t *v) 419 { 420 kcsan_release(); 421 instrument_atomic_read_write(v, sizeof(*v)); 422 return arch_atomic_fetch_or_release(i, v); 423 } 424 425 static __always_inline int 426 atomic_fetch_or_relaxed(int i, atomic_t *v) 427 { 428 instrument_atomic_read_write(v, sizeof(*v)); 429 return arch_atomic_fetch_or_relaxed(i, v); 430 } 431 432 static __always_inline void 433 atomic_xor(int i, atomic_t *v) 434 { 435 instrument_atomic_read_write(v, sizeof(*v)); 436 arch_atomic_xor(i, v); 437 } 438 439 static __always_inline int 440 atomic_fetch_xor(int i, atomic_t *v) 441 { 442 kcsan_mb(); 443 instrument_atomic_read_write(v, sizeof(*v)); 444 return arch_atomic_fetch_xor(i, v); 445 } 446 447 static __always_inline int 448 atomic_fetch_xor_acquire(int i, atomic_t *v) 449 { 450 instrument_atomic_read_write(v, sizeof(*v)); 451 return arch_atomic_fetch_xor_acquire(i, v); 452 } 453 454 static __always_inline int 455 atomic_fetch_xor_release(int i, atomic_t *v) 456 { 457 kcsan_release(); 458 instrument_atomic_read_write(v, sizeof(*v)); 459 return arch_atomic_fetch_xor_release(i, v); 460 } 461 462 static __always_inline int 463 atomic_fetch_xor_relaxed(int i, atomic_t *v) 464 { 465 instrument_atomic_read_write(v, sizeof(*v)); 466 return arch_atomic_fetch_xor_relaxed(i, v); 467 } 468 469 static __always_inline int 470 atomic_xchg(atomic_t *v, int i) 471 { 472 kcsan_mb(); 473 instrument_atomic_read_write(v, sizeof(*v)); 474 return arch_atomic_xchg(v, i); 475 } 476 477 static __always_inline int 478 atomic_xchg_acquire(atomic_t *v, int i) 479 { 480 instrument_atomic_read_write(v, sizeof(*v)); 481 return arch_atomic_xchg_acquire(v, i); 482 } 483 484 static __always_inline int 485 atomic_xchg_release(atomic_t *v, int i) 486 { 487 kcsan_release(); 488 instrument_atomic_read_write(v, sizeof(*v)); 489 return arch_atomic_xchg_release(v, i); 490 } 491 492 static __always_inline int 493 atomic_xchg_relaxed(atomic_t *v, int i) 494 { 495 instrument_atomic_read_write(v, sizeof(*v)); 496 return arch_atomic_xchg_relaxed(v, i); 497 } 498 499 static __always_inline int 500 atomic_cmpxchg(atomic_t *v, int old, int new) 501 { 502 kcsan_mb(); 503 instrument_atomic_read_write(v, sizeof(*v)); 504 return arch_atomic_cmpxchg(v, old, new); 505 } 506 507 static __always_inline int 508 atomic_cmpxchg_acquire(atomic_t *v, int old, int new) 509 { 510 instrument_atomic_read_write(v, sizeof(*v)); 511 return arch_atomic_cmpxchg_acquire(v, old, new); 512 } 513 514 static __always_inline int 515 atomic_cmpxchg_release(atomic_t *v, int old, int new) 516 { 517 kcsan_release(); 518 instrument_atomic_read_write(v, sizeof(*v)); 519 return arch_atomic_cmpxchg_release(v, old, new); 520 } 521 522 static __always_inline int 523 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) 524 { 525 instrument_atomic_read_write(v, sizeof(*v)); 526 return arch_atomic_cmpxchg_relaxed(v, old, new); 527 } 528 529 static __always_inline bool 530 atomic_try_cmpxchg(atomic_t *v, int *old, int new) 531 { 532 kcsan_mb(); 533 instrument_atomic_read_write(v, sizeof(*v)); 534 instrument_atomic_read_write(old, sizeof(*old)); 535 return arch_atomic_try_cmpxchg(v, old, new); 536 } 537 538 static __always_inline bool 539 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 540 { 541 instrument_atomic_read_write(v, sizeof(*v)); 542 instrument_atomic_read_write(old, sizeof(*old)); 543 return arch_atomic_try_cmpxchg_acquire(v, old, new); 544 } 545 546 static __always_inline bool 547 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 548 { 549 kcsan_release(); 550 instrument_atomic_read_write(v, sizeof(*v)); 551 instrument_atomic_read_write(old, sizeof(*old)); 552 return arch_atomic_try_cmpxchg_release(v, old, new); 553 } 554 555 static __always_inline bool 556 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) 557 { 558 instrument_atomic_read_write(v, sizeof(*v)); 559 instrument_atomic_read_write(old, sizeof(*old)); 560 return arch_atomic_try_cmpxchg_relaxed(v, old, new); 561 } 562 563 static __always_inline bool 564 atomic_sub_and_test(int i, atomic_t *v) 565 { 566 kcsan_mb(); 567 instrument_atomic_read_write(v, sizeof(*v)); 568 return arch_atomic_sub_and_test(i, v); 569 } 570 571 static __always_inline bool 572 atomic_dec_and_test(atomic_t *v) 573 { 574 kcsan_mb(); 575 instrument_atomic_read_write(v, sizeof(*v)); 576 return arch_atomic_dec_and_test(v); 577 } 578 579 static __always_inline bool 580 atomic_inc_and_test(atomic_t *v) 581 { 582 kcsan_mb(); 583 instrument_atomic_read_write(v, sizeof(*v)); 584 return arch_atomic_inc_and_test(v); 585 } 586 587 static __always_inline bool 588 atomic_add_negative(int i, atomic_t *v) 589 { 590 kcsan_mb(); 591 instrument_atomic_read_write(v, sizeof(*v)); 592 return arch_atomic_add_negative(i, v); 593 } 594 595 static __always_inline int 596 atomic_fetch_add_unless(atomic_t *v, int a, int u) 597 { 598 kcsan_mb(); 599 instrument_atomic_read_write(v, sizeof(*v)); 600 return arch_atomic_fetch_add_unless(v, a, u); 601 } 602 603 static __always_inline bool 604 atomic_add_unless(atomic_t *v, int a, int u) 605 { 606 kcsan_mb(); 607 instrument_atomic_read_write(v, sizeof(*v)); 608 return arch_atomic_add_unless(v, a, u); 609 } 610 611 static __always_inline bool 612 atomic_inc_not_zero(atomic_t *v) 613 { 614 kcsan_mb(); 615 instrument_atomic_read_write(v, sizeof(*v)); 616 return arch_atomic_inc_not_zero(v); 617 } 618 619 static __always_inline bool 620 atomic_inc_unless_negative(atomic_t *v) 621 { 622 kcsan_mb(); 623 instrument_atomic_read_write(v, sizeof(*v)); 624 return arch_atomic_inc_unless_negative(v); 625 } 626 627 static __always_inline bool 628 atomic_dec_unless_positive(atomic_t *v) 629 { 630 kcsan_mb(); 631 instrument_atomic_read_write(v, sizeof(*v)); 632 return arch_atomic_dec_unless_positive(v); 633 } 634 635 static __always_inline int 636 atomic_dec_if_positive(atomic_t *v) 637 { 638 kcsan_mb(); 639 instrument_atomic_read_write(v, sizeof(*v)); 640 return arch_atomic_dec_if_positive(v); 641 } 642 643 static __always_inline s64 644 atomic64_read(const atomic64_t *v) 645 { 646 instrument_atomic_read(v, sizeof(*v)); 647 return arch_atomic64_read(v); 648 } 649 650 static __always_inline s64 651 atomic64_read_acquire(const atomic64_t *v) 652 { 653 instrument_atomic_read(v, sizeof(*v)); 654 return arch_atomic64_read_acquire(v); 655 } 656 657 static __always_inline void 658 atomic64_set(atomic64_t *v, s64 i) 659 { 660 instrument_atomic_write(v, sizeof(*v)); 661 arch_atomic64_set(v, i); 662 } 663 664 static __always_inline void 665 atomic64_set_release(atomic64_t *v, s64 i) 666 { 667 kcsan_release(); 668 instrument_atomic_write(v, sizeof(*v)); 669 arch_atomic64_set_release(v, i); 670 } 671 672 static __always_inline void 673 atomic64_add(s64 i, atomic64_t *v) 674 { 675 instrument_atomic_read_write(v, sizeof(*v)); 676 arch_atomic64_add(i, v); 677 } 678 679 static __always_inline s64 680 atomic64_add_return(s64 i, atomic64_t *v) 681 { 682 kcsan_mb(); 683 instrument_atomic_read_write(v, sizeof(*v)); 684 return arch_atomic64_add_return(i, v); 685 } 686 687 static __always_inline s64 688 atomic64_add_return_acquire(s64 i, atomic64_t *v) 689 { 690 instrument_atomic_read_write(v, sizeof(*v)); 691 return arch_atomic64_add_return_acquire(i, v); 692 } 693 694 static __always_inline s64 695 atomic64_add_return_release(s64 i, atomic64_t *v) 696 { 697 kcsan_release(); 698 instrument_atomic_read_write(v, sizeof(*v)); 699 return arch_atomic64_add_return_release(i, v); 700 } 701 702 static __always_inline s64 703 atomic64_add_return_relaxed(s64 i, atomic64_t *v) 704 { 705 instrument_atomic_read_write(v, sizeof(*v)); 706 return arch_atomic64_add_return_relaxed(i, v); 707 } 708 709 static __always_inline s64 710 atomic64_fetch_add(s64 i, atomic64_t *v) 711 { 712 kcsan_mb(); 713 instrument_atomic_read_write(v, sizeof(*v)); 714 return arch_atomic64_fetch_add(i, v); 715 } 716 717 static __always_inline s64 718 atomic64_fetch_add_acquire(s64 i, atomic64_t *v) 719 { 720 instrument_atomic_read_write(v, sizeof(*v)); 721 return arch_atomic64_fetch_add_acquire(i, v); 722 } 723 724 static __always_inline s64 725 atomic64_fetch_add_release(s64 i, atomic64_t *v) 726 { 727 kcsan_release(); 728 instrument_atomic_read_write(v, sizeof(*v)); 729 return arch_atomic64_fetch_add_release(i, v); 730 } 731 732 static __always_inline s64 733 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) 734 { 735 instrument_atomic_read_write(v, sizeof(*v)); 736 return arch_atomic64_fetch_add_relaxed(i, v); 737 } 738 739 static __always_inline void 740 atomic64_sub(s64 i, atomic64_t *v) 741 { 742 instrument_atomic_read_write(v, sizeof(*v)); 743 arch_atomic64_sub(i, v); 744 } 745 746 static __always_inline s64 747 atomic64_sub_return(s64 i, atomic64_t *v) 748 { 749 kcsan_mb(); 750 instrument_atomic_read_write(v, sizeof(*v)); 751 return arch_atomic64_sub_return(i, v); 752 } 753 754 static __always_inline s64 755 atomic64_sub_return_acquire(s64 i, atomic64_t *v) 756 { 757 instrument_atomic_read_write(v, sizeof(*v)); 758 return arch_atomic64_sub_return_acquire(i, v); 759 } 760 761 static __always_inline s64 762 atomic64_sub_return_release(s64 i, atomic64_t *v) 763 { 764 kcsan_release(); 765 instrument_atomic_read_write(v, sizeof(*v)); 766 return arch_atomic64_sub_return_release(i, v); 767 } 768 769 static __always_inline s64 770 atomic64_sub_return_relaxed(s64 i, atomic64_t *v) 771 { 772 instrument_atomic_read_write(v, sizeof(*v)); 773 return arch_atomic64_sub_return_relaxed(i, v); 774 } 775 776 static __always_inline s64 777 atomic64_fetch_sub(s64 i, atomic64_t *v) 778 { 779 kcsan_mb(); 780 instrument_atomic_read_write(v, sizeof(*v)); 781 return arch_atomic64_fetch_sub(i, v); 782 } 783 784 static __always_inline s64 785 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) 786 { 787 instrument_atomic_read_write(v, sizeof(*v)); 788 return arch_atomic64_fetch_sub_acquire(i, v); 789 } 790 791 static __always_inline s64 792 atomic64_fetch_sub_release(s64 i, atomic64_t *v) 793 { 794 kcsan_release(); 795 instrument_atomic_read_write(v, sizeof(*v)); 796 return arch_atomic64_fetch_sub_release(i, v); 797 } 798 799 static __always_inline s64 800 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) 801 { 802 instrument_atomic_read_write(v, sizeof(*v)); 803 return arch_atomic64_fetch_sub_relaxed(i, v); 804 } 805 806 static __always_inline void 807 atomic64_inc(atomic64_t *v) 808 { 809 instrument_atomic_read_write(v, sizeof(*v)); 810 arch_atomic64_inc(v); 811 } 812 813 static __always_inline s64 814 atomic64_inc_return(atomic64_t *v) 815 { 816 kcsan_mb(); 817 instrument_atomic_read_write(v, sizeof(*v)); 818 return arch_atomic64_inc_return(v); 819 } 820 821 static __always_inline s64 822 atomic64_inc_return_acquire(atomic64_t *v) 823 { 824 instrument_atomic_read_write(v, sizeof(*v)); 825 return arch_atomic64_inc_return_acquire(v); 826 } 827 828 static __always_inline s64 829 atomic64_inc_return_release(atomic64_t *v) 830 { 831 kcsan_release(); 832 instrument_atomic_read_write(v, sizeof(*v)); 833 return arch_atomic64_inc_return_release(v); 834 } 835 836 static __always_inline s64 837 atomic64_inc_return_relaxed(atomic64_t *v) 838 { 839 instrument_atomic_read_write(v, sizeof(*v)); 840 return arch_atomic64_inc_return_relaxed(v); 841 } 842 843 static __always_inline s64 844 atomic64_fetch_inc(atomic64_t *v) 845 { 846 kcsan_mb(); 847 instrument_atomic_read_write(v, sizeof(*v)); 848 return arch_atomic64_fetch_inc(v); 849 } 850 851 static __always_inline s64 852 atomic64_fetch_inc_acquire(atomic64_t *v) 853 { 854 instrument_atomic_read_write(v, sizeof(*v)); 855 return arch_atomic64_fetch_inc_acquire(v); 856 } 857 858 static __always_inline s64 859 atomic64_fetch_inc_release(atomic64_t *v) 860 { 861 kcsan_release(); 862 instrument_atomic_read_write(v, sizeof(*v)); 863 return arch_atomic64_fetch_inc_release(v); 864 } 865 866 static __always_inline s64 867 atomic64_fetch_inc_relaxed(atomic64_t *v) 868 { 869 instrument_atomic_read_write(v, sizeof(*v)); 870 return arch_atomic64_fetch_inc_relaxed(v); 871 } 872 873 static __always_inline void 874 atomic64_dec(atomic64_t *v) 875 { 876 instrument_atomic_read_write(v, sizeof(*v)); 877 arch_atomic64_dec(v); 878 } 879 880 static __always_inline s64 881 atomic64_dec_return(atomic64_t *v) 882 { 883 kcsan_mb(); 884 instrument_atomic_read_write(v, sizeof(*v)); 885 return arch_atomic64_dec_return(v); 886 } 887 888 static __always_inline s64 889 atomic64_dec_return_acquire(atomic64_t *v) 890 { 891 instrument_atomic_read_write(v, sizeof(*v)); 892 return arch_atomic64_dec_return_acquire(v); 893 } 894 895 static __always_inline s64 896 atomic64_dec_return_release(atomic64_t *v) 897 { 898 kcsan_release(); 899 instrument_atomic_read_write(v, sizeof(*v)); 900 return arch_atomic64_dec_return_release(v); 901 } 902 903 static __always_inline s64 904 atomic64_dec_return_relaxed(atomic64_t *v) 905 { 906 instrument_atomic_read_write(v, sizeof(*v)); 907 return arch_atomic64_dec_return_relaxed(v); 908 } 909 910 static __always_inline s64 911 atomic64_fetch_dec(atomic64_t *v) 912 { 913 kcsan_mb(); 914 instrument_atomic_read_write(v, sizeof(*v)); 915 return arch_atomic64_fetch_dec(v); 916 } 917 918 static __always_inline s64 919 atomic64_fetch_dec_acquire(atomic64_t *v) 920 { 921 instrument_atomic_read_write(v, sizeof(*v)); 922 return arch_atomic64_fetch_dec_acquire(v); 923 } 924 925 static __always_inline s64 926 atomic64_fetch_dec_release(atomic64_t *v) 927 { 928 kcsan_release(); 929 instrument_atomic_read_write(v, sizeof(*v)); 930 return arch_atomic64_fetch_dec_release(v); 931 } 932 933 static __always_inline s64 934 atomic64_fetch_dec_relaxed(atomic64_t *v) 935 { 936 instrument_atomic_read_write(v, sizeof(*v)); 937 return arch_atomic64_fetch_dec_relaxed(v); 938 } 939 940 static __always_inline void 941 atomic64_and(s64 i, atomic64_t *v) 942 { 943 instrument_atomic_read_write(v, sizeof(*v)); 944 arch_atomic64_and(i, v); 945 } 946 947 static __always_inline s64 948 atomic64_fetch_and(s64 i, atomic64_t *v) 949 { 950 kcsan_mb(); 951 instrument_atomic_read_write(v, sizeof(*v)); 952 return arch_atomic64_fetch_and(i, v); 953 } 954 955 static __always_inline s64 956 atomic64_fetch_and_acquire(s64 i, atomic64_t *v) 957 { 958 instrument_atomic_read_write(v, sizeof(*v)); 959 return arch_atomic64_fetch_and_acquire(i, v); 960 } 961 962 static __always_inline s64 963 atomic64_fetch_and_release(s64 i, atomic64_t *v) 964 { 965 kcsan_release(); 966 instrument_atomic_read_write(v, sizeof(*v)); 967 return arch_atomic64_fetch_and_release(i, v); 968 } 969 970 static __always_inline s64 971 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) 972 { 973 instrument_atomic_read_write(v, sizeof(*v)); 974 return arch_atomic64_fetch_and_relaxed(i, v); 975 } 976 977 static __always_inline void 978 atomic64_andnot(s64 i, atomic64_t *v) 979 { 980 instrument_atomic_read_write(v, sizeof(*v)); 981 arch_atomic64_andnot(i, v); 982 } 983 984 static __always_inline s64 985 atomic64_fetch_andnot(s64 i, atomic64_t *v) 986 { 987 kcsan_mb(); 988 instrument_atomic_read_write(v, sizeof(*v)); 989 return arch_atomic64_fetch_andnot(i, v); 990 } 991 992 static __always_inline s64 993 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 994 { 995 instrument_atomic_read_write(v, sizeof(*v)); 996 return arch_atomic64_fetch_andnot_acquire(i, v); 997 } 998 999 static __always_inline s64 1000 atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 1001 { 1002 kcsan_release(); 1003 instrument_atomic_read_write(v, sizeof(*v)); 1004 return arch_atomic64_fetch_andnot_release(i, v); 1005 } 1006 1007 static __always_inline s64 1008 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) 1009 { 1010 instrument_atomic_read_write(v, sizeof(*v)); 1011 return arch_atomic64_fetch_andnot_relaxed(i, v); 1012 } 1013 1014 static __always_inline void 1015 atomic64_or(s64 i, atomic64_t *v) 1016 { 1017 instrument_atomic_read_write(v, sizeof(*v)); 1018 arch_atomic64_or(i, v); 1019 } 1020 1021 static __always_inline s64 1022 atomic64_fetch_or(s64 i, atomic64_t *v) 1023 { 1024 kcsan_mb(); 1025 instrument_atomic_read_write(v, sizeof(*v)); 1026 return arch_atomic64_fetch_or(i, v); 1027 } 1028 1029 static __always_inline s64 1030 atomic64_fetch_or_acquire(s64 i, atomic64_t *v) 1031 { 1032 instrument_atomic_read_write(v, sizeof(*v)); 1033 return arch_atomic64_fetch_or_acquire(i, v); 1034 } 1035 1036 static __always_inline s64 1037 atomic64_fetch_or_release(s64 i, atomic64_t *v) 1038 { 1039 kcsan_release(); 1040 instrument_atomic_read_write(v, sizeof(*v)); 1041 return arch_atomic64_fetch_or_release(i, v); 1042 } 1043 1044 static __always_inline s64 1045 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) 1046 { 1047 instrument_atomic_read_write(v, sizeof(*v)); 1048 return arch_atomic64_fetch_or_relaxed(i, v); 1049 } 1050 1051 static __always_inline void 1052 atomic64_xor(s64 i, atomic64_t *v) 1053 { 1054 instrument_atomic_read_write(v, sizeof(*v)); 1055 arch_atomic64_xor(i, v); 1056 } 1057 1058 static __always_inline s64 1059 atomic64_fetch_xor(s64 i, atomic64_t *v) 1060 { 1061 kcsan_mb(); 1062 instrument_atomic_read_write(v, sizeof(*v)); 1063 return arch_atomic64_fetch_xor(i, v); 1064 } 1065 1066 static __always_inline s64 1067 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) 1068 { 1069 instrument_atomic_read_write(v, sizeof(*v)); 1070 return arch_atomic64_fetch_xor_acquire(i, v); 1071 } 1072 1073 static __always_inline s64 1074 atomic64_fetch_xor_release(s64 i, atomic64_t *v) 1075 { 1076 kcsan_release(); 1077 instrument_atomic_read_write(v, sizeof(*v)); 1078 return arch_atomic64_fetch_xor_release(i, v); 1079 } 1080 1081 static __always_inline s64 1082 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) 1083 { 1084 instrument_atomic_read_write(v, sizeof(*v)); 1085 return arch_atomic64_fetch_xor_relaxed(i, v); 1086 } 1087 1088 static __always_inline s64 1089 atomic64_xchg(atomic64_t *v, s64 i) 1090 { 1091 kcsan_mb(); 1092 instrument_atomic_read_write(v, sizeof(*v)); 1093 return arch_atomic64_xchg(v, i); 1094 } 1095 1096 static __always_inline s64 1097 atomic64_xchg_acquire(atomic64_t *v, s64 i) 1098 { 1099 instrument_atomic_read_write(v, sizeof(*v)); 1100 return arch_atomic64_xchg_acquire(v, i); 1101 } 1102 1103 static __always_inline s64 1104 atomic64_xchg_release(atomic64_t *v, s64 i) 1105 { 1106 kcsan_release(); 1107 instrument_atomic_read_write(v, sizeof(*v)); 1108 return arch_atomic64_xchg_release(v, i); 1109 } 1110 1111 static __always_inline s64 1112 atomic64_xchg_relaxed(atomic64_t *v, s64 i) 1113 { 1114 instrument_atomic_read_write(v, sizeof(*v)); 1115 return arch_atomic64_xchg_relaxed(v, i); 1116 } 1117 1118 static __always_inline s64 1119 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 1120 { 1121 kcsan_mb(); 1122 instrument_atomic_read_write(v, sizeof(*v)); 1123 return arch_atomic64_cmpxchg(v, old, new); 1124 } 1125 1126 static __always_inline s64 1127 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) 1128 { 1129 instrument_atomic_read_write(v, sizeof(*v)); 1130 return arch_atomic64_cmpxchg_acquire(v, old, new); 1131 } 1132 1133 static __always_inline s64 1134 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) 1135 { 1136 kcsan_release(); 1137 instrument_atomic_read_write(v, sizeof(*v)); 1138 return arch_atomic64_cmpxchg_release(v, old, new); 1139 } 1140 1141 static __always_inline s64 1142 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) 1143 { 1144 instrument_atomic_read_write(v, sizeof(*v)); 1145 return arch_atomic64_cmpxchg_relaxed(v, old, new); 1146 } 1147 1148 static __always_inline bool 1149 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 1150 { 1151 kcsan_mb(); 1152 instrument_atomic_read_write(v, sizeof(*v)); 1153 instrument_atomic_read_write(old, sizeof(*old)); 1154 return arch_atomic64_try_cmpxchg(v, old, new); 1155 } 1156 1157 static __always_inline bool 1158 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 1159 { 1160 instrument_atomic_read_write(v, sizeof(*v)); 1161 instrument_atomic_read_write(old, sizeof(*old)); 1162 return arch_atomic64_try_cmpxchg_acquire(v, old, new); 1163 } 1164 1165 static __always_inline bool 1166 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 1167 { 1168 kcsan_release(); 1169 instrument_atomic_read_write(v, sizeof(*v)); 1170 instrument_atomic_read_write(old, sizeof(*old)); 1171 return arch_atomic64_try_cmpxchg_release(v, old, new); 1172 } 1173 1174 static __always_inline bool 1175 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) 1176 { 1177 instrument_atomic_read_write(v, sizeof(*v)); 1178 instrument_atomic_read_write(old, sizeof(*old)); 1179 return arch_atomic64_try_cmpxchg_relaxed(v, old, new); 1180 } 1181 1182 static __always_inline bool 1183 atomic64_sub_and_test(s64 i, atomic64_t *v) 1184 { 1185 kcsan_mb(); 1186 instrument_atomic_read_write(v, sizeof(*v)); 1187 return arch_atomic64_sub_and_test(i, v); 1188 } 1189 1190 static __always_inline bool 1191 atomic64_dec_and_test(atomic64_t *v) 1192 { 1193 kcsan_mb(); 1194 instrument_atomic_read_write(v, sizeof(*v)); 1195 return arch_atomic64_dec_and_test(v); 1196 } 1197 1198 static __always_inline bool 1199 atomic64_inc_and_test(atomic64_t *v) 1200 { 1201 kcsan_mb(); 1202 instrument_atomic_read_write(v, sizeof(*v)); 1203 return arch_atomic64_inc_and_test(v); 1204 } 1205 1206 static __always_inline bool 1207 atomic64_add_negative(s64 i, atomic64_t *v) 1208 { 1209 kcsan_mb(); 1210 instrument_atomic_read_write(v, sizeof(*v)); 1211 return arch_atomic64_add_negative(i, v); 1212 } 1213 1214 static __always_inline s64 1215 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 1216 { 1217 kcsan_mb(); 1218 instrument_atomic_read_write(v, sizeof(*v)); 1219 return arch_atomic64_fetch_add_unless(v, a, u); 1220 } 1221 1222 static __always_inline bool 1223 atomic64_add_unless(atomic64_t *v, s64 a, s64 u) 1224 { 1225 kcsan_mb(); 1226 instrument_atomic_read_write(v, sizeof(*v)); 1227 return arch_atomic64_add_unless(v, a, u); 1228 } 1229 1230 static __always_inline bool 1231 atomic64_inc_not_zero(atomic64_t *v) 1232 { 1233 kcsan_mb(); 1234 instrument_atomic_read_write(v, sizeof(*v)); 1235 return arch_atomic64_inc_not_zero(v); 1236 } 1237 1238 static __always_inline bool 1239 atomic64_inc_unless_negative(atomic64_t *v) 1240 { 1241 kcsan_mb(); 1242 instrument_atomic_read_write(v, sizeof(*v)); 1243 return arch_atomic64_inc_unless_negative(v); 1244 } 1245 1246 static __always_inline bool 1247 atomic64_dec_unless_positive(atomic64_t *v) 1248 { 1249 kcsan_mb(); 1250 instrument_atomic_read_write(v, sizeof(*v)); 1251 return arch_atomic64_dec_unless_positive(v); 1252 } 1253 1254 static __always_inline s64 1255 atomic64_dec_if_positive(atomic64_t *v) 1256 { 1257 kcsan_mb(); 1258 instrument_atomic_read_write(v, sizeof(*v)); 1259 return arch_atomic64_dec_if_positive(v); 1260 } 1261 1262 static __always_inline long 1263 atomic_long_read(const atomic_long_t *v) 1264 { 1265 instrument_atomic_read(v, sizeof(*v)); 1266 return arch_atomic_long_read(v); 1267 } 1268 1269 static __always_inline long 1270 atomic_long_read_acquire(const atomic_long_t *v) 1271 { 1272 instrument_atomic_read(v, sizeof(*v)); 1273 return arch_atomic_long_read_acquire(v); 1274 } 1275 1276 static __always_inline void 1277 atomic_long_set(atomic_long_t *v, long i) 1278 { 1279 instrument_atomic_write(v, sizeof(*v)); 1280 arch_atomic_long_set(v, i); 1281 } 1282 1283 static __always_inline void 1284 atomic_long_set_release(atomic_long_t *v, long i) 1285 { 1286 kcsan_release(); 1287 instrument_atomic_write(v, sizeof(*v)); 1288 arch_atomic_long_set_release(v, i); 1289 } 1290 1291 static __always_inline void 1292 atomic_long_add(long i, atomic_long_t *v) 1293 { 1294 instrument_atomic_read_write(v, sizeof(*v)); 1295 arch_atomic_long_add(i, v); 1296 } 1297 1298 static __always_inline long 1299 atomic_long_add_return(long i, atomic_long_t *v) 1300 { 1301 kcsan_mb(); 1302 instrument_atomic_read_write(v, sizeof(*v)); 1303 return arch_atomic_long_add_return(i, v); 1304 } 1305 1306 static __always_inline long 1307 atomic_long_add_return_acquire(long i, atomic_long_t *v) 1308 { 1309 instrument_atomic_read_write(v, sizeof(*v)); 1310 return arch_atomic_long_add_return_acquire(i, v); 1311 } 1312 1313 static __always_inline long 1314 atomic_long_add_return_release(long i, atomic_long_t *v) 1315 { 1316 kcsan_release(); 1317 instrument_atomic_read_write(v, sizeof(*v)); 1318 return arch_atomic_long_add_return_release(i, v); 1319 } 1320 1321 static __always_inline long 1322 atomic_long_add_return_relaxed(long i, atomic_long_t *v) 1323 { 1324 instrument_atomic_read_write(v, sizeof(*v)); 1325 return arch_atomic_long_add_return_relaxed(i, v); 1326 } 1327 1328 static __always_inline long 1329 atomic_long_fetch_add(long i, atomic_long_t *v) 1330 { 1331 kcsan_mb(); 1332 instrument_atomic_read_write(v, sizeof(*v)); 1333 return arch_atomic_long_fetch_add(i, v); 1334 } 1335 1336 static __always_inline long 1337 atomic_long_fetch_add_acquire(long i, atomic_long_t *v) 1338 { 1339 instrument_atomic_read_write(v, sizeof(*v)); 1340 return arch_atomic_long_fetch_add_acquire(i, v); 1341 } 1342 1343 static __always_inline long 1344 atomic_long_fetch_add_release(long i, atomic_long_t *v) 1345 { 1346 kcsan_release(); 1347 instrument_atomic_read_write(v, sizeof(*v)); 1348 return arch_atomic_long_fetch_add_release(i, v); 1349 } 1350 1351 static __always_inline long 1352 atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) 1353 { 1354 instrument_atomic_read_write(v, sizeof(*v)); 1355 return arch_atomic_long_fetch_add_relaxed(i, v); 1356 } 1357 1358 static __always_inline void 1359 atomic_long_sub(long i, atomic_long_t *v) 1360 { 1361 instrument_atomic_read_write(v, sizeof(*v)); 1362 arch_atomic_long_sub(i, v); 1363 } 1364 1365 static __always_inline long 1366 atomic_long_sub_return(long i, atomic_long_t *v) 1367 { 1368 kcsan_mb(); 1369 instrument_atomic_read_write(v, sizeof(*v)); 1370 return arch_atomic_long_sub_return(i, v); 1371 } 1372 1373 static __always_inline long 1374 atomic_long_sub_return_acquire(long i, atomic_long_t *v) 1375 { 1376 instrument_atomic_read_write(v, sizeof(*v)); 1377 return arch_atomic_long_sub_return_acquire(i, v); 1378 } 1379 1380 static __always_inline long 1381 atomic_long_sub_return_release(long i, atomic_long_t *v) 1382 { 1383 kcsan_release(); 1384 instrument_atomic_read_write(v, sizeof(*v)); 1385 return arch_atomic_long_sub_return_release(i, v); 1386 } 1387 1388 static __always_inline long 1389 atomic_long_sub_return_relaxed(long i, atomic_long_t *v) 1390 { 1391 instrument_atomic_read_write(v, sizeof(*v)); 1392 return arch_atomic_long_sub_return_relaxed(i, v); 1393 } 1394 1395 static __always_inline long 1396 atomic_long_fetch_sub(long i, atomic_long_t *v) 1397 { 1398 kcsan_mb(); 1399 instrument_atomic_read_write(v, sizeof(*v)); 1400 return arch_atomic_long_fetch_sub(i, v); 1401 } 1402 1403 static __always_inline long 1404 atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) 1405 { 1406 instrument_atomic_read_write(v, sizeof(*v)); 1407 return arch_atomic_long_fetch_sub_acquire(i, v); 1408 } 1409 1410 static __always_inline long 1411 atomic_long_fetch_sub_release(long i, atomic_long_t *v) 1412 { 1413 kcsan_release(); 1414 instrument_atomic_read_write(v, sizeof(*v)); 1415 return arch_atomic_long_fetch_sub_release(i, v); 1416 } 1417 1418 static __always_inline long 1419 atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) 1420 { 1421 instrument_atomic_read_write(v, sizeof(*v)); 1422 return arch_atomic_long_fetch_sub_relaxed(i, v); 1423 } 1424 1425 static __always_inline void 1426 atomic_long_inc(atomic_long_t *v) 1427 { 1428 instrument_atomic_read_write(v, sizeof(*v)); 1429 arch_atomic_long_inc(v); 1430 } 1431 1432 static __always_inline long 1433 atomic_long_inc_return(atomic_long_t *v) 1434 { 1435 kcsan_mb(); 1436 instrument_atomic_read_write(v, sizeof(*v)); 1437 return arch_atomic_long_inc_return(v); 1438 } 1439 1440 static __always_inline long 1441 atomic_long_inc_return_acquire(atomic_long_t *v) 1442 { 1443 instrument_atomic_read_write(v, sizeof(*v)); 1444 return arch_atomic_long_inc_return_acquire(v); 1445 } 1446 1447 static __always_inline long 1448 atomic_long_inc_return_release(atomic_long_t *v) 1449 { 1450 kcsan_release(); 1451 instrument_atomic_read_write(v, sizeof(*v)); 1452 return arch_atomic_long_inc_return_release(v); 1453 } 1454 1455 static __always_inline long 1456 atomic_long_inc_return_relaxed(atomic_long_t *v) 1457 { 1458 instrument_atomic_read_write(v, sizeof(*v)); 1459 return arch_atomic_long_inc_return_relaxed(v); 1460 } 1461 1462 static __always_inline long 1463 atomic_long_fetch_inc(atomic_long_t *v) 1464 { 1465 kcsan_mb(); 1466 instrument_atomic_read_write(v, sizeof(*v)); 1467 return arch_atomic_long_fetch_inc(v); 1468 } 1469 1470 static __always_inline long 1471 atomic_long_fetch_inc_acquire(atomic_long_t *v) 1472 { 1473 instrument_atomic_read_write(v, sizeof(*v)); 1474 return arch_atomic_long_fetch_inc_acquire(v); 1475 } 1476 1477 static __always_inline long 1478 atomic_long_fetch_inc_release(atomic_long_t *v) 1479 { 1480 kcsan_release(); 1481 instrument_atomic_read_write(v, sizeof(*v)); 1482 return arch_atomic_long_fetch_inc_release(v); 1483 } 1484 1485 static __always_inline long 1486 atomic_long_fetch_inc_relaxed(atomic_long_t *v) 1487 { 1488 instrument_atomic_read_write(v, sizeof(*v)); 1489 return arch_atomic_long_fetch_inc_relaxed(v); 1490 } 1491 1492 static __always_inline void 1493 atomic_long_dec(atomic_long_t *v) 1494 { 1495 instrument_atomic_read_write(v, sizeof(*v)); 1496 arch_atomic_long_dec(v); 1497 } 1498 1499 static __always_inline long 1500 atomic_long_dec_return(atomic_long_t *v) 1501 { 1502 kcsan_mb(); 1503 instrument_atomic_read_write(v, sizeof(*v)); 1504 return arch_atomic_long_dec_return(v); 1505 } 1506 1507 static __always_inline long 1508 atomic_long_dec_return_acquire(atomic_long_t *v) 1509 { 1510 instrument_atomic_read_write(v, sizeof(*v)); 1511 return arch_atomic_long_dec_return_acquire(v); 1512 } 1513 1514 static __always_inline long 1515 atomic_long_dec_return_release(atomic_long_t *v) 1516 { 1517 kcsan_release(); 1518 instrument_atomic_read_write(v, sizeof(*v)); 1519 return arch_atomic_long_dec_return_release(v); 1520 } 1521 1522 static __always_inline long 1523 atomic_long_dec_return_relaxed(atomic_long_t *v) 1524 { 1525 instrument_atomic_read_write(v, sizeof(*v)); 1526 return arch_atomic_long_dec_return_relaxed(v); 1527 } 1528 1529 static __always_inline long 1530 atomic_long_fetch_dec(atomic_long_t *v) 1531 { 1532 kcsan_mb(); 1533 instrument_atomic_read_write(v, sizeof(*v)); 1534 return arch_atomic_long_fetch_dec(v); 1535 } 1536 1537 static __always_inline long 1538 atomic_long_fetch_dec_acquire(atomic_long_t *v) 1539 { 1540 instrument_atomic_read_write(v, sizeof(*v)); 1541 return arch_atomic_long_fetch_dec_acquire(v); 1542 } 1543 1544 static __always_inline long 1545 atomic_long_fetch_dec_release(atomic_long_t *v) 1546 { 1547 kcsan_release(); 1548 instrument_atomic_read_write(v, sizeof(*v)); 1549 return arch_atomic_long_fetch_dec_release(v); 1550 } 1551 1552 static __always_inline long 1553 atomic_long_fetch_dec_relaxed(atomic_long_t *v) 1554 { 1555 instrument_atomic_read_write(v, sizeof(*v)); 1556 return arch_atomic_long_fetch_dec_relaxed(v); 1557 } 1558 1559 static __always_inline void 1560 atomic_long_and(long i, atomic_long_t *v) 1561 { 1562 instrument_atomic_read_write(v, sizeof(*v)); 1563 arch_atomic_long_and(i, v); 1564 } 1565 1566 static __always_inline long 1567 atomic_long_fetch_and(long i, atomic_long_t *v) 1568 { 1569 kcsan_mb(); 1570 instrument_atomic_read_write(v, sizeof(*v)); 1571 return arch_atomic_long_fetch_and(i, v); 1572 } 1573 1574 static __always_inline long 1575 atomic_long_fetch_and_acquire(long i, atomic_long_t *v) 1576 { 1577 instrument_atomic_read_write(v, sizeof(*v)); 1578 return arch_atomic_long_fetch_and_acquire(i, v); 1579 } 1580 1581 static __always_inline long 1582 atomic_long_fetch_and_release(long i, atomic_long_t *v) 1583 { 1584 kcsan_release(); 1585 instrument_atomic_read_write(v, sizeof(*v)); 1586 return arch_atomic_long_fetch_and_release(i, v); 1587 } 1588 1589 static __always_inline long 1590 atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) 1591 { 1592 instrument_atomic_read_write(v, sizeof(*v)); 1593 return arch_atomic_long_fetch_and_relaxed(i, v); 1594 } 1595 1596 static __always_inline void 1597 atomic_long_andnot(long i, atomic_long_t *v) 1598 { 1599 instrument_atomic_read_write(v, sizeof(*v)); 1600 arch_atomic_long_andnot(i, v); 1601 } 1602 1603 static __always_inline long 1604 atomic_long_fetch_andnot(long i, atomic_long_t *v) 1605 { 1606 kcsan_mb(); 1607 instrument_atomic_read_write(v, sizeof(*v)); 1608 return arch_atomic_long_fetch_andnot(i, v); 1609 } 1610 1611 static __always_inline long 1612 atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) 1613 { 1614 instrument_atomic_read_write(v, sizeof(*v)); 1615 return arch_atomic_long_fetch_andnot_acquire(i, v); 1616 } 1617 1618 static __always_inline long 1619 atomic_long_fetch_andnot_release(long i, atomic_long_t *v) 1620 { 1621 kcsan_release(); 1622 instrument_atomic_read_write(v, sizeof(*v)); 1623 return arch_atomic_long_fetch_andnot_release(i, v); 1624 } 1625 1626 static __always_inline long 1627 atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) 1628 { 1629 instrument_atomic_read_write(v, sizeof(*v)); 1630 return arch_atomic_long_fetch_andnot_relaxed(i, v); 1631 } 1632 1633 static __always_inline void 1634 atomic_long_or(long i, atomic_long_t *v) 1635 { 1636 instrument_atomic_read_write(v, sizeof(*v)); 1637 arch_atomic_long_or(i, v); 1638 } 1639 1640 static __always_inline long 1641 atomic_long_fetch_or(long i, atomic_long_t *v) 1642 { 1643 kcsan_mb(); 1644 instrument_atomic_read_write(v, sizeof(*v)); 1645 return arch_atomic_long_fetch_or(i, v); 1646 } 1647 1648 static __always_inline long 1649 atomic_long_fetch_or_acquire(long i, atomic_long_t *v) 1650 { 1651 instrument_atomic_read_write(v, sizeof(*v)); 1652 return arch_atomic_long_fetch_or_acquire(i, v); 1653 } 1654 1655 static __always_inline long 1656 atomic_long_fetch_or_release(long i, atomic_long_t *v) 1657 { 1658 kcsan_release(); 1659 instrument_atomic_read_write(v, sizeof(*v)); 1660 return arch_atomic_long_fetch_or_release(i, v); 1661 } 1662 1663 static __always_inline long 1664 atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) 1665 { 1666 instrument_atomic_read_write(v, sizeof(*v)); 1667 return arch_atomic_long_fetch_or_relaxed(i, v); 1668 } 1669 1670 static __always_inline void 1671 atomic_long_xor(long i, atomic_long_t *v) 1672 { 1673 instrument_atomic_read_write(v, sizeof(*v)); 1674 arch_atomic_long_xor(i, v); 1675 } 1676 1677 static __always_inline long 1678 atomic_long_fetch_xor(long i, atomic_long_t *v) 1679 { 1680 kcsan_mb(); 1681 instrument_atomic_read_write(v, sizeof(*v)); 1682 return arch_atomic_long_fetch_xor(i, v); 1683 } 1684 1685 static __always_inline long 1686 atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) 1687 { 1688 instrument_atomic_read_write(v, sizeof(*v)); 1689 return arch_atomic_long_fetch_xor_acquire(i, v); 1690 } 1691 1692 static __always_inline long 1693 atomic_long_fetch_xor_release(long i, atomic_long_t *v) 1694 { 1695 kcsan_release(); 1696 instrument_atomic_read_write(v, sizeof(*v)); 1697 return arch_atomic_long_fetch_xor_release(i, v); 1698 } 1699 1700 static __always_inline long 1701 atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) 1702 { 1703 instrument_atomic_read_write(v, sizeof(*v)); 1704 return arch_atomic_long_fetch_xor_relaxed(i, v); 1705 } 1706 1707 static __always_inline long 1708 atomic_long_xchg(atomic_long_t *v, long i) 1709 { 1710 kcsan_mb(); 1711 instrument_atomic_read_write(v, sizeof(*v)); 1712 return arch_atomic_long_xchg(v, i); 1713 } 1714 1715 static __always_inline long 1716 atomic_long_xchg_acquire(atomic_long_t *v, long i) 1717 { 1718 instrument_atomic_read_write(v, sizeof(*v)); 1719 return arch_atomic_long_xchg_acquire(v, i); 1720 } 1721 1722 static __always_inline long 1723 atomic_long_xchg_release(atomic_long_t *v, long i) 1724 { 1725 kcsan_release(); 1726 instrument_atomic_read_write(v, sizeof(*v)); 1727 return arch_atomic_long_xchg_release(v, i); 1728 } 1729 1730 static __always_inline long 1731 atomic_long_xchg_relaxed(atomic_long_t *v, long i) 1732 { 1733 instrument_atomic_read_write(v, sizeof(*v)); 1734 return arch_atomic_long_xchg_relaxed(v, i); 1735 } 1736 1737 static __always_inline long 1738 atomic_long_cmpxchg(atomic_long_t *v, long old, long new) 1739 { 1740 kcsan_mb(); 1741 instrument_atomic_read_write(v, sizeof(*v)); 1742 return arch_atomic_long_cmpxchg(v, old, new); 1743 } 1744 1745 static __always_inline long 1746 atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) 1747 { 1748 instrument_atomic_read_write(v, sizeof(*v)); 1749 return arch_atomic_long_cmpxchg_acquire(v, old, new); 1750 } 1751 1752 static __always_inline long 1753 atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) 1754 { 1755 kcsan_release(); 1756 instrument_atomic_read_write(v, sizeof(*v)); 1757 return arch_atomic_long_cmpxchg_release(v, old, new); 1758 } 1759 1760 static __always_inline long 1761 atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) 1762 { 1763 instrument_atomic_read_write(v, sizeof(*v)); 1764 return arch_atomic_long_cmpxchg_relaxed(v, old, new); 1765 } 1766 1767 static __always_inline bool 1768 atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) 1769 { 1770 kcsan_mb(); 1771 instrument_atomic_read_write(v, sizeof(*v)); 1772 instrument_atomic_read_write(old, sizeof(*old)); 1773 return arch_atomic_long_try_cmpxchg(v, old, new); 1774 } 1775 1776 static __always_inline bool 1777 atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) 1778 { 1779 instrument_atomic_read_write(v, sizeof(*v)); 1780 instrument_atomic_read_write(old, sizeof(*old)); 1781 return arch_atomic_long_try_cmpxchg_acquire(v, old, new); 1782 } 1783 1784 static __always_inline bool 1785 atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) 1786 { 1787 kcsan_release(); 1788 instrument_atomic_read_write(v, sizeof(*v)); 1789 instrument_atomic_read_write(old, sizeof(*old)); 1790 return arch_atomic_long_try_cmpxchg_release(v, old, new); 1791 } 1792 1793 static __always_inline bool 1794 atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) 1795 { 1796 instrument_atomic_read_write(v, sizeof(*v)); 1797 instrument_atomic_read_write(old, sizeof(*old)); 1798 return arch_atomic_long_try_cmpxchg_relaxed(v, old, new); 1799 } 1800 1801 static __always_inline bool 1802 atomic_long_sub_and_test(long i, atomic_long_t *v) 1803 { 1804 kcsan_mb(); 1805 instrument_atomic_read_write(v, sizeof(*v)); 1806 return arch_atomic_long_sub_and_test(i, v); 1807 } 1808 1809 static __always_inline bool 1810 atomic_long_dec_and_test(atomic_long_t *v) 1811 { 1812 kcsan_mb(); 1813 instrument_atomic_read_write(v, sizeof(*v)); 1814 return arch_atomic_long_dec_and_test(v); 1815 } 1816 1817 static __always_inline bool 1818 atomic_long_inc_and_test(atomic_long_t *v) 1819 { 1820 kcsan_mb(); 1821 instrument_atomic_read_write(v, sizeof(*v)); 1822 return arch_atomic_long_inc_and_test(v); 1823 } 1824 1825 static __always_inline bool 1826 atomic_long_add_negative(long i, atomic_long_t *v) 1827 { 1828 kcsan_mb(); 1829 instrument_atomic_read_write(v, sizeof(*v)); 1830 return arch_atomic_long_add_negative(i, v); 1831 } 1832 1833 static __always_inline long 1834 atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) 1835 { 1836 kcsan_mb(); 1837 instrument_atomic_read_write(v, sizeof(*v)); 1838 return arch_atomic_long_fetch_add_unless(v, a, u); 1839 } 1840 1841 static __always_inline bool 1842 atomic_long_add_unless(atomic_long_t *v, long a, long u) 1843 { 1844 kcsan_mb(); 1845 instrument_atomic_read_write(v, sizeof(*v)); 1846 return arch_atomic_long_add_unless(v, a, u); 1847 } 1848 1849 static __always_inline bool 1850 atomic_long_inc_not_zero(atomic_long_t *v) 1851 { 1852 kcsan_mb(); 1853 instrument_atomic_read_write(v, sizeof(*v)); 1854 return arch_atomic_long_inc_not_zero(v); 1855 } 1856 1857 static __always_inline bool 1858 atomic_long_inc_unless_negative(atomic_long_t *v) 1859 { 1860 kcsan_mb(); 1861 instrument_atomic_read_write(v, sizeof(*v)); 1862 return arch_atomic_long_inc_unless_negative(v); 1863 } 1864 1865 static __always_inline bool 1866 atomic_long_dec_unless_positive(atomic_long_t *v) 1867 { 1868 kcsan_mb(); 1869 instrument_atomic_read_write(v, sizeof(*v)); 1870 return arch_atomic_long_dec_unless_positive(v); 1871 } 1872 1873 static __always_inline long 1874 atomic_long_dec_if_positive(atomic_long_t *v) 1875 { 1876 kcsan_mb(); 1877 instrument_atomic_read_write(v, sizeof(*v)); 1878 return arch_atomic_long_dec_if_positive(v); 1879 } 1880 1881 #define xchg(ptr, ...) \ 1882 ({ \ 1883 typeof(ptr) __ai_ptr = (ptr); \ 1884 kcsan_mb(); \ 1885 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1886 arch_xchg(__ai_ptr, __VA_ARGS__); \ 1887 }) 1888 1889 #define xchg_acquire(ptr, ...) \ 1890 ({ \ 1891 typeof(ptr) __ai_ptr = (ptr); \ 1892 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1893 arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ 1894 }) 1895 1896 #define xchg_release(ptr, ...) \ 1897 ({ \ 1898 typeof(ptr) __ai_ptr = (ptr); \ 1899 kcsan_release(); \ 1900 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1901 arch_xchg_release(__ai_ptr, __VA_ARGS__); \ 1902 }) 1903 1904 #define xchg_relaxed(ptr, ...) \ 1905 ({ \ 1906 typeof(ptr) __ai_ptr = (ptr); \ 1907 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1908 arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ 1909 }) 1910 1911 #define cmpxchg(ptr, ...) \ 1912 ({ \ 1913 typeof(ptr) __ai_ptr = (ptr); \ 1914 kcsan_mb(); \ 1915 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1916 arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ 1917 }) 1918 1919 #define cmpxchg_acquire(ptr, ...) \ 1920 ({ \ 1921 typeof(ptr) __ai_ptr = (ptr); \ 1922 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1923 arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ 1924 }) 1925 1926 #define cmpxchg_release(ptr, ...) \ 1927 ({ \ 1928 typeof(ptr) __ai_ptr = (ptr); \ 1929 kcsan_release(); \ 1930 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1931 arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ 1932 }) 1933 1934 #define cmpxchg_relaxed(ptr, ...) \ 1935 ({ \ 1936 typeof(ptr) __ai_ptr = (ptr); \ 1937 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1938 arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ 1939 }) 1940 1941 #define cmpxchg64(ptr, ...) \ 1942 ({ \ 1943 typeof(ptr) __ai_ptr = (ptr); \ 1944 kcsan_mb(); \ 1945 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1946 arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \ 1947 }) 1948 1949 #define cmpxchg64_acquire(ptr, ...) \ 1950 ({ \ 1951 typeof(ptr) __ai_ptr = (ptr); \ 1952 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1953 arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ 1954 }) 1955 1956 #define cmpxchg64_release(ptr, ...) \ 1957 ({ \ 1958 typeof(ptr) __ai_ptr = (ptr); \ 1959 kcsan_release(); \ 1960 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1961 arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ 1962 }) 1963 1964 #define cmpxchg64_relaxed(ptr, ...) \ 1965 ({ \ 1966 typeof(ptr) __ai_ptr = (ptr); \ 1967 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1968 arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ 1969 }) 1970 1971 #define try_cmpxchg(ptr, oldp, ...) \ 1972 ({ \ 1973 typeof(ptr) __ai_ptr = (ptr); \ 1974 typeof(oldp) __ai_oldp = (oldp); \ 1975 kcsan_mb(); \ 1976 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1977 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ 1978 arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 1979 }) 1980 1981 #define try_cmpxchg_acquire(ptr, oldp, ...) \ 1982 ({ \ 1983 typeof(ptr) __ai_ptr = (ptr); \ 1984 typeof(oldp) __ai_oldp = (oldp); \ 1985 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1986 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ 1987 arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 1988 }) 1989 1990 #define try_cmpxchg_release(ptr, oldp, ...) \ 1991 ({ \ 1992 typeof(ptr) __ai_ptr = (ptr); \ 1993 typeof(oldp) __ai_oldp = (oldp); \ 1994 kcsan_release(); \ 1995 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1996 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ 1997 arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 1998 }) 1999 2000 #define try_cmpxchg_relaxed(ptr, oldp, ...) \ 2001 ({ \ 2002 typeof(ptr) __ai_ptr = (ptr); \ 2003 typeof(oldp) __ai_oldp = (oldp); \ 2004 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2005 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ 2006 arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 2007 }) 2008 2009 #define try_cmpxchg64(ptr, oldp, ...) \ 2010 ({ \ 2011 typeof(ptr) __ai_ptr = (ptr); \ 2012 typeof(oldp) __ai_oldp = (oldp); \ 2013 kcsan_mb(); \ 2014 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2015 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ 2016 arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 2017 }) 2018 2019 #define try_cmpxchg64_acquire(ptr, oldp, ...) \ 2020 ({ \ 2021 typeof(ptr) __ai_ptr = (ptr); \ 2022 typeof(oldp) __ai_oldp = (oldp); \ 2023 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2024 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ 2025 arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 2026 }) 2027 2028 #define try_cmpxchg64_release(ptr, oldp, ...) \ 2029 ({ \ 2030 typeof(ptr) __ai_ptr = (ptr); \ 2031 typeof(oldp) __ai_oldp = (oldp); \ 2032 kcsan_release(); \ 2033 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2034 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ 2035 arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 2036 }) 2037 2038 #define try_cmpxchg64_relaxed(ptr, oldp, ...) \ 2039 ({ \ 2040 typeof(ptr) __ai_ptr = (ptr); \ 2041 typeof(oldp) __ai_oldp = (oldp); \ 2042 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2043 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ 2044 arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 2045 }) 2046 2047 #define cmpxchg_local(ptr, ...) \ 2048 ({ \ 2049 typeof(ptr) __ai_ptr = (ptr); \ 2050 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2051 arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ 2052 }) 2053 2054 #define cmpxchg64_local(ptr, ...) \ 2055 ({ \ 2056 typeof(ptr) __ai_ptr = (ptr); \ 2057 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2058 arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ 2059 }) 2060 2061 #define sync_cmpxchg(ptr, ...) \ 2062 ({ \ 2063 typeof(ptr) __ai_ptr = (ptr); \ 2064 kcsan_mb(); \ 2065 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 2066 arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ 2067 }) 2068 2069 #define cmpxchg_double(ptr, ...) \ 2070 ({ \ 2071 typeof(ptr) __ai_ptr = (ptr); \ 2072 kcsan_mb(); \ 2073 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ 2074 arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \ 2075 }) 2076 2077 2078 #define cmpxchg_double_local(ptr, ...) \ 2079 ({ \ 2080 typeof(ptr) __ai_ptr = (ptr); \ 2081 instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ 2082 arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ 2083 }) 2084 2085 #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ 2086 // 764f741eb77a7ad565dc8d99ce2837d5542e8aee 2087