1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * KCSAN test with various race scenarious to test runtime behaviour. Since the 4 * interface with which KCSAN's reports are obtained is via the console, this is 5 * the output we should verify. For each test case checks the presence (or 6 * absence) of generated reports. Relies on 'console' tracepoint to capture 7 * reports as they appear in the kernel log. 8 * 9 * Makes use of KUnit for test organization, and the Torture framework for test 10 * thread control. 11 * 12 * Copyright (C) 2020, Google LLC. 13 * Author: Marco Elver <elver@google.com> 14 */ 15 16 #define pr_fmt(fmt) "kcsan_test: " fmt 17 18 #include <kunit/test.h> 19 #include <linux/atomic.h> 20 #include <linux/bitops.h> 21 #include <linux/jiffies.h> 22 #include <linux/kcsan-checks.h> 23 #include <linux/kernel.h> 24 #include <linux/mutex.h> 25 #include <linux/sched.h> 26 #include <linux/seqlock.h> 27 #include <linux/spinlock.h> 28 #include <linux/string.h> 29 #include <linux/timer.h> 30 #include <linux/torture.h> 31 #include <linux/tracepoint.h> 32 #include <linux/types.h> 33 #include <trace/events/printk.h> 34 35 #define KCSAN_TEST_REQUIRES(test, cond) do { \ 36 if (!(cond)) \ 37 kunit_skip((test), "Test requires: " #cond); \ 38 } while (0) 39 40 #ifdef CONFIG_CC_HAS_TSAN_COMPOUND_READ_BEFORE_WRITE 41 #define __KCSAN_ACCESS_RW(alt) (KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE) 42 #else 43 #define __KCSAN_ACCESS_RW(alt) (alt) 44 #endif 45 46 /* Points to current test-case memory access "kernels". */ 47 static void (*access_kernels[2])(void); 48 49 static struct task_struct **threads; /* Lists of threads. */ 50 static unsigned long end_time; /* End time of test. */ 51 52 /* Report as observed from console. */ 53 static struct { 54 spinlock_t lock; 55 int nlines; 56 char lines[3][512]; 57 } observed = { 58 .lock = __SPIN_LOCK_UNLOCKED(observed.lock), 59 }; 60 61 /* Setup test checking loop. */ 62 static __no_kcsan inline void 63 begin_test_checks(void (*func1)(void), void (*func2)(void)) 64 { 65 kcsan_disable_current(); 66 67 /* 68 * Require at least as long as KCSAN_REPORT_ONCE_IN_MS, to ensure at 69 * least one race is reported. 70 */ 71 end_time = jiffies + msecs_to_jiffies(CONFIG_KCSAN_REPORT_ONCE_IN_MS + 500); 72 73 /* Signal start; release potential initialization of shared data. */ 74 smp_store_release(&access_kernels[0], func1); 75 smp_store_release(&access_kernels[1], func2); 76 } 77 78 /* End test checking loop. */ 79 static __no_kcsan inline bool 80 end_test_checks(bool stop) 81 { 82 if (!stop && time_before(jiffies, end_time)) { 83 /* Continue checking */ 84 might_sleep(); 85 return false; 86 } 87 88 kcsan_enable_current(); 89 return true; 90 } 91 92 /* 93 * Probe for console output: checks if a race was reported, and obtains observed 94 * lines of interest. 95 */ 96 __no_kcsan 97 static void probe_console(void *ignore, const char *buf, size_t len) 98 { 99 unsigned long flags; 100 int nlines; 101 102 /* 103 * Note that KCSAN reports under a global lock, so we do not risk the 104 * possibility of having multiple reports interleaved. If that were the 105 * case, we'd expect tests to fail. 106 */ 107 108 spin_lock_irqsave(&observed.lock, flags); 109 nlines = observed.nlines; 110 111 if (strnstr(buf, "BUG: KCSAN: ", len) && strnstr(buf, "test_", len)) { 112 /* 113 * KCSAN report and related to the test. 114 * 115 * The provided @buf is not NUL-terminated; copy no more than 116 * @len bytes and let strscpy() add the missing NUL-terminator. 117 */ 118 strscpy(observed.lines[0], buf, min(len + 1, sizeof(observed.lines[0]))); 119 nlines = 1; 120 } else if ((nlines == 1 || nlines == 2) && strnstr(buf, "bytes by", len)) { 121 strscpy(observed.lines[nlines++], buf, min(len + 1, sizeof(observed.lines[0]))); 122 123 if (strnstr(buf, "race at unknown origin", len)) { 124 if (WARN_ON(nlines != 2)) 125 goto out; 126 127 /* No second line of interest. */ 128 strcpy(observed.lines[nlines++], "<none>"); 129 } 130 } 131 132 out: 133 WRITE_ONCE(observed.nlines, nlines); /* Publish new nlines. */ 134 spin_unlock_irqrestore(&observed.lock, flags); 135 } 136 137 /* Check if a report related to the test exists. */ 138 __no_kcsan 139 static bool report_available(void) 140 { 141 return READ_ONCE(observed.nlines) == ARRAY_SIZE(observed.lines); 142 } 143 144 /* Report information we expect in a report. */ 145 struct expect_report { 146 /* Access information of both accesses. */ 147 struct { 148 void *fn; /* Function pointer to expected function of top frame. */ 149 void *addr; /* Address of access; unchecked if NULL. */ 150 size_t size; /* Size of access; unchecked if @addr is NULL. */ 151 int type; /* Access type, see KCSAN_ACCESS definitions. */ 152 } access[2]; 153 }; 154 155 /* Check observed report matches information in @r. */ 156 __no_kcsan 157 static bool __report_matches(const struct expect_report *r) 158 { 159 const bool is_assert = (r->access[0].type | r->access[1].type) & KCSAN_ACCESS_ASSERT; 160 bool ret = false; 161 unsigned long flags; 162 typeof(observed.lines) expect; 163 const char *end; 164 char *cur; 165 int i; 166 167 /* Doubled-checked locking. */ 168 if (!report_available()) 169 return false; 170 171 /* Generate expected report contents. */ 172 173 /* Title */ 174 cur = expect[0]; 175 end = &expect[0][sizeof(expect[0]) - 1]; 176 cur += scnprintf(cur, end - cur, "BUG: KCSAN: %s in ", 177 is_assert ? "assert: race" : "data-race"); 178 if (r->access[1].fn) { 179 char tmp[2][64]; 180 int cmp; 181 182 /* Expect lexographically sorted function names in title. */ 183 scnprintf(tmp[0], sizeof(tmp[0]), "%pS", r->access[0].fn); 184 scnprintf(tmp[1], sizeof(tmp[1]), "%pS", r->access[1].fn); 185 cmp = strcmp(tmp[0], tmp[1]); 186 cur += scnprintf(cur, end - cur, "%ps / %ps", 187 cmp < 0 ? r->access[0].fn : r->access[1].fn, 188 cmp < 0 ? r->access[1].fn : r->access[0].fn); 189 } else { 190 scnprintf(cur, end - cur, "%pS", r->access[0].fn); 191 /* The exact offset won't match, remove it. */ 192 cur = strchr(expect[0], '+'); 193 if (cur) 194 *cur = '\0'; 195 } 196 197 /* Access 1 */ 198 cur = expect[1]; 199 end = &expect[1][sizeof(expect[1]) - 1]; 200 if (!r->access[1].fn) 201 cur += scnprintf(cur, end - cur, "race at unknown origin, with "); 202 203 /* Access 1 & 2 */ 204 for (i = 0; i < 2; ++i) { 205 const int ty = r->access[i].type; 206 const char *const access_type = 207 (ty & KCSAN_ACCESS_ASSERT) ? 208 ((ty & KCSAN_ACCESS_WRITE) ? 209 "assert no accesses" : 210 "assert no writes") : 211 ((ty & KCSAN_ACCESS_WRITE) ? 212 ((ty & KCSAN_ACCESS_COMPOUND) ? 213 "read-write" : 214 "write") : 215 "read"); 216 const bool is_atomic = (ty & KCSAN_ACCESS_ATOMIC); 217 const bool is_scoped = (ty & KCSAN_ACCESS_SCOPED); 218 const char *const access_type_aux = 219 (is_atomic && is_scoped) ? " (marked, reordered)" 220 : (is_atomic ? " (marked)" 221 : (is_scoped ? " (reordered)" : "")); 222 223 if (i == 1) { 224 /* Access 2 */ 225 cur = expect[2]; 226 end = &expect[2][sizeof(expect[2]) - 1]; 227 228 if (!r->access[1].fn) { 229 /* Dummy string if no second access is available. */ 230 strcpy(cur, "<none>"); 231 break; 232 } 233 } 234 235 cur += scnprintf(cur, end - cur, "%s%s to ", access_type, 236 access_type_aux); 237 238 if (r->access[i].addr) /* Address is optional. */ 239 cur += scnprintf(cur, end - cur, "0x%px of %zu bytes", 240 r->access[i].addr, r->access[i].size); 241 } 242 243 spin_lock_irqsave(&observed.lock, flags); 244 if (!report_available()) 245 goto out; /* A new report is being captured. */ 246 247 /* Finally match expected output to what we actually observed. */ 248 ret = strstr(observed.lines[0], expect[0]) && 249 /* Access info may appear in any order. */ 250 ((strstr(observed.lines[1], expect[1]) && 251 strstr(observed.lines[2], expect[2])) || 252 (strstr(observed.lines[1], expect[2]) && 253 strstr(observed.lines[2], expect[1]))); 254 out: 255 spin_unlock_irqrestore(&observed.lock, flags); 256 return ret; 257 } 258 259 static __always_inline const struct expect_report * 260 __report_set_scoped(struct expect_report *r, int accesses) 261 { 262 BUILD_BUG_ON(accesses > 3); 263 264 if (accesses & 1) 265 r->access[0].type |= KCSAN_ACCESS_SCOPED; 266 else 267 r->access[0].type &= ~KCSAN_ACCESS_SCOPED; 268 269 if (accesses & 2) 270 r->access[1].type |= KCSAN_ACCESS_SCOPED; 271 else 272 r->access[1].type &= ~KCSAN_ACCESS_SCOPED; 273 274 return r; 275 } 276 277 __no_kcsan 278 static bool report_matches_any_reordered(struct expect_report *r) 279 { 280 return __report_matches(__report_set_scoped(r, 0)) || 281 __report_matches(__report_set_scoped(r, 1)) || 282 __report_matches(__report_set_scoped(r, 2)) || 283 __report_matches(__report_set_scoped(r, 3)); 284 } 285 286 #ifdef CONFIG_KCSAN_WEAK_MEMORY 287 /* Due to reordering accesses, any access may appear as "(reordered)". */ 288 #define report_matches report_matches_any_reordered 289 #else 290 #define report_matches __report_matches 291 #endif 292 293 /* ===== Test kernels ===== */ 294 295 static long test_sink; 296 static long test_var; 297 /* @test_array should be large enough to fall into multiple watchpoint slots. */ 298 static long test_array[3 * PAGE_SIZE / sizeof(long)]; 299 static struct { 300 long val[8]; 301 } test_struct; 302 static DEFINE_SEQLOCK(test_seqlock); 303 static DEFINE_SPINLOCK(test_spinlock); 304 static DEFINE_MUTEX(test_mutex); 305 306 /* 307 * Helper to avoid compiler optimizing out reads, and to generate source values 308 * for writes. 309 */ 310 __no_kcsan 311 static noinline void sink_value(long v) { WRITE_ONCE(test_sink, v); } 312 313 /* 314 * Generates a delay and some accesses that enter the runtime but do not produce 315 * data races. 316 */ 317 static noinline void test_delay(int iter) 318 { 319 while (iter--) 320 sink_value(READ_ONCE(test_sink)); 321 } 322 323 static noinline void test_kernel_read(void) { sink_value(test_var); } 324 325 static noinline void test_kernel_write(void) 326 { 327 test_var = READ_ONCE_NOCHECK(test_sink) + 1; 328 } 329 330 static noinline void test_kernel_write_nochange(void) { test_var = 42; } 331 332 /* Suffixed by value-change exception filter. */ 333 static noinline void test_kernel_write_nochange_rcu(void) { test_var = 42; } 334 335 static noinline void test_kernel_read_atomic(void) 336 { 337 sink_value(READ_ONCE(test_var)); 338 } 339 340 static noinline void test_kernel_write_atomic(void) 341 { 342 WRITE_ONCE(test_var, READ_ONCE_NOCHECK(test_sink) + 1); 343 } 344 345 static noinline void test_kernel_atomic_rmw(void) 346 { 347 /* Use builtin, so we can set up the "bad" atomic/non-atomic scenario. */ 348 __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED); 349 } 350 351 __no_kcsan 352 static noinline void test_kernel_write_uninstrumented(void) { test_var++; } 353 354 static noinline void test_kernel_data_race(void) { data_race(test_var++); } 355 356 static noinline void test_kernel_assert_writer(void) 357 { 358 ASSERT_EXCLUSIVE_WRITER(test_var); 359 } 360 361 static noinline void test_kernel_assert_access(void) 362 { 363 ASSERT_EXCLUSIVE_ACCESS(test_var); 364 } 365 366 #define TEST_CHANGE_BITS 0xff00ff00 367 368 static noinline void test_kernel_change_bits(void) 369 { 370 if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { 371 /* 372 * Avoid race of unknown origin for this test, just pretend they 373 * are atomic. 374 */ 375 kcsan_nestable_atomic_begin(); 376 test_var ^= TEST_CHANGE_BITS; 377 kcsan_nestable_atomic_end(); 378 } else 379 WRITE_ONCE(test_var, READ_ONCE(test_var) ^ TEST_CHANGE_BITS); 380 } 381 382 static noinline void test_kernel_assert_bits_change(void) 383 { 384 ASSERT_EXCLUSIVE_BITS(test_var, TEST_CHANGE_BITS); 385 } 386 387 static noinline void test_kernel_assert_bits_nochange(void) 388 { 389 ASSERT_EXCLUSIVE_BITS(test_var, ~TEST_CHANGE_BITS); 390 } 391 392 /* 393 * Scoped assertions do trigger anywhere in scope. However, the report should 394 * still only point at the start of the scope. 395 */ 396 static noinline void test_enter_scope(void) 397 { 398 int x = 0; 399 400 /* Unrelated accesses to scoped assert. */ 401 READ_ONCE(test_sink); 402 kcsan_check_read(&x, sizeof(x)); 403 } 404 405 static noinline void test_kernel_assert_writer_scoped(void) 406 { 407 ASSERT_EXCLUSIVE_WRITER_SCOPED(test_var); 408 test_enter_scope(); 409 } 410 411 static noinline void test_kernel_assert_access_scoped(void) 412 { 413 ASSERT_EXCLUSIVE_ACCESS_SCOPED(test_var); 414 test_enter_scope(); 415 } 416 417 static noinline void test_kernel_rmw_array(void) 418 { 419 int i; 420 421 for (i = 0; i < ARRAY_SIZE(test_array); ++i) 422 test_array[i]++; 423 } 424 425 static noinline void test_kernel_write_struct(void) 426 { 427 kcsan_check_write(&test_struct, sizeof(test_struct)); 428 kcsan_disable_current(); 429 test_struct.val[3]++; /* induce value change */ 430 kcsan_enable_current(); 431 } 432 433 static noinline void test_kernel_write_struct_part(void) 434 { 435 test_struct.val[3] = 42; 436 } 437 438 static noinline void test_kernel_read_struct_zero_size(void) 439 { 440 kcsan_check_read(&test_struct.val[3], 0); 441 } 442 443 static noinline void test_kernel_jiffies_reader(void) 444 { 445 sink_value((long)jiffies); 446 } 447 448 static noinline void test_kernel_seqlock_reader(void) 449 { 450 unsigned int seq; 451 452 do { 453 seq = read_seqbegin(&test_seqlock); 454 sink_value(test_var); 455 } while (read_seqretry(&test_seqlock, seq)); 456 } 457 458 static noinline void test_kernel_seqlock_writer(void) 459 { 460 unsigned long flags; 461 462 write_seqlock_irqsave(&test_seqlock, flags); 463 test_var++; 464 write_sequnlock_irqrestore(&test_seqlock, flags); 465 } 466 467 static noinline void test_kernel_atomic_builtins(void) 468 { 469 /* 470 * Generate concurrent accesses, expecting no reports, ensuring KCSAN 471 * treats builtin atomics as actually atomic. 472 */ 473 __atomic_load_n(&test_var, __ATOMIC_RELAXED); 474 } 475 476 static noinline void test_kernel_xor_1bit(void) 477 { 478 /* Do not report data races between the read-writes. */ 479 kcsan_nestable_atomic_begin(); 480 test_var ^= 0x10000; 481 kcsan_nestable_atomic_end(); 482 } 483 484 #define TEST_KERNEL_LOCKED(name, acquire, release) \ 485 static noinline void test_kernel_##name(void) \ 486 { \ 487 long *flag = &test_struct.val[0]; \ 488 long v = 0; \ 489 if (!(acquire)) \ 490 return; \ 491 while (v++ < 100) { \ 492 test_var++; \ 493 barrier(); \ 494 } \ 495 release; \ 496 test_delay(10); \ 497 } 498 499 TEST_KERNEL_LOCKED(with_memorder, 500 cmpxchg_acquire(flag, 0, 1) == 0, 501 smp_store_release(flag, 0)); 502 TEST_KERNEL_LOCKED(wrong_memorder, 503 cmpxchg_relaxed(flag, 0, 1) == 0, 504 WRITE_ONCE(*flag, 0)); 505 TEST_KERNEL_LOCKED(atomic_builtin_with_memorder, 506 __atomic_compare_exchange_n(flag, &v, 1, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED), 507 __atomic_store_n(flag, 0, __ATOMIC_RELEASE)); 508 TEST_KERNEL_LOCKED(atomic_builtin_wrong_memorder, 509 __atomic_compare_exchange_n(flag, &v, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED), 510 __atomic_store_n(flag, 0, __ATOMIC_RELAXED)); 511 512 /* ===== Test cases ===== */ 513 514 /* 515 * Tests that various barriers have the expected effect on internal state. Not 516 * exhaustive on atomic_t operations. Unlike the selftest, also checks for 517 * too-strict barrier instrumentation; these can be tolerated, because it does 518 * not cause false positives, but at least we should be aware of such cases. 519 */ 520 static void test_barrier_nothreads(struct kunit *test) 521 { 522 #ifdef CONFIG_KCSAN_WEAK_MEMORY 523 struct kcsan_scoped_access *reorder_access = ¤t->kcsan_ctx.reorder_access; 524 #else 525 struct kcsan_scoped_access *reorder_access = NULL; 526 #endif 527 arch_spinlock_t arch_spinlock = __ARCH_SPIN_LOCK_UNLOCKED; 528 atomic_t dummy; 529 530 KCSAN_TEST_REQUIRES(test, reorder_access != NULL); 531 KCSAN_TEST_REQUIRES(test, IS_ENABLED(CONFIG_SMP)); 532 533 #define __KCSAN_EXPECT_BARRIER(access_type, barrier, order_before, name) \ 534 do { \ 535 reorder_access->type = (access_type) | KCSAN_ACCESS_SCOPED; \ 536 reorder_access->size = sizeof(test_var); \ 537 barrier; \ 538 KUNIT_EXPECT_EQ_MSG(test, reorder_access->size, \ 539 order_before ? 0 : sizeof(test_var), \ 540 "improperly instrumented type=(" #access_type "): " name); \ 541 } while (0) 542 #define KCSAN_EXPECT_READ_BARRIER(b, o) __KCSAN_EXPECT_BARRIER(0, b, o, #b) 543 #define KCSAN_EXPECT_WRITE_BARRIER(b, o) __KCSAN_EXPECT_BARRIER(KCSAN_ACCESS_WRITE, b, o, #b) 544 #define KCSAN_EXPECT_RW_BARRIER(b, o) __KCSAN_EXPECT_BARRIER(KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE, b, o, #b) 545 546 /* 547 * Lockdep initialization can strengthen certain locking operations due 548 * to calling into instrumented files; "warm up" our locks. 549 */ 550 spin_lock(&test_spinlock); 551 spin_unlock(&test_spinlock); 552 mutex_lock(&test_mutex); 553 mutex_unlock(&test_mutex); 554 555 /* Force creating a valid entry in reorder_access first. */ 556 test_var = 0; 557 while (test_var++ < 1000000 && reorder_access->size != sizeof(test_var)) 558 __kcsan_check_read(&test_var, sizeof(test_var)); 559 KUNIT_ASSERT_EQ(test, reorder_access->size, sizeof(test_var)); 560 561 kcsan_nestable_atomic_begin(); /* No watchpoints in called functions. */ 562 563 KCSAN_EXPECT_READ_BARRIER(mb(), true); 564 KCSAN_EXPECT_READ_BARRIER(wmb(), false); 565 KCSAN_EXPECT_READ_BARRIER(rmb(), true); 566 KCSAN_EXPECT_READ_BARRIER(smp_mb(), true); 567 KCSAN_EXPECT_READ_BARRIER(smp_wmb(), false); 568 KCSAN_EXPECT_READ_BARRIER(smp_rmb(), true); 569 KCSAN_EXPECT_READ_BARRIER(dma_wmb(), false); 570 KCSAN_EXPECT_READ_BARRIER(dma_rmb(), true); 571 KCSAN_EXPECT_READ_BARRIER(smp_mb__before_atomic(), true); 572 KCSAN_EXPECT_READ_BARRIER(smp_mb__after_atomic(), true); 573 KCSAN_EXPECT_READ_BARRIER(smp_mb__after_spinlock(), true); 574 KCSAN_EXPECT_READ_BARRIER(smp_store_mb(test_var, 0), true); 575 KCSAN_EXPECT_READ_BARRIER(smp_load_acquire(&test_var), false); 576 KCSAN_EXPECT_READ_BARRIER(smp_store_release(&test_var, 0), true); 577 KCSAN_EXPECT_READ_BARRIER(xchg(&test_var, 0), true); 578 KCSAN_EXPECT_READ_BARRIER(xchg_release(&test_var, 0), true); 579 KCSAN_EXPECT_READ_BARRIER(xchg_relaxed(&test_var, 0), false); 580 KCSAN_EXPECT_READ_BARRIER(cmpxchg(&test_var, 0, 0), true); 581 KCSAN_EXPECT_READ_BARRIER(cmpxchg_release(&test_var, 0, 0), true); 582 KCSAN_EXPECT_READ_BARRIER(cmpxchg_relaxed(&test_var, 0, 0), false); 583 KCSAN_EXPECT_READ_BARRIER(atomic_read(&dummy), false); 584 KCSAN_EXPECT_READ_BARRIER(atomic_read_acquire(&dummy), false); 585 KCSAN_EXPECT_READ_BARRIER(atomic_set(&dummy, 0), false); 586 KCSAN_EXPECT_READ_BARRIER(atomic_set_release(&dummy, 0), true); 587 KCSAN_EXPECT_READ_BARRIER(atomic_add(1, &dummy), false); 588 KCSAN_EXPECT_READ_BARRIER(atomic_add_return(1, &dummy), true); 589 KCSAN_EXPECT_READ_BARRIER(atomic_add_return_acquire(1, &dummy), false); 590 KCSAN_EXPECT_READ_BARRIER(atomic_add_return_release(1, &dummy), true); 591 KCSAN_EXPECT_READ_BARRIER(atomic_add_return_relaxed(1, &dummy), false); 592 KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add(1, &dummy), true); 593 KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_acquire(1, &dummy), false); 594 KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_release(1, &dummy), true); 595 KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false); 596 KCSAN_EXPECT_READ_BARRIER(test_and_set_bit(0, &test_var), true); 597 KCSAN_EXPECT_READ_BARRIER(test_and_clear_bit(0, &test_var), true); 598 KCSAN_EXPECT_READ_BARRIER(test_and_change_bit(0, &test_var), true); 599 KCSAN_EXPECT_READ_BARRIER(clear_bit_unlock(0, &test_var), true); 600 KCSAN_EXPECT_READ_BARRIER(__clear_bit_unlock(0, &test_var), true); 601 KCSAN_EXPECT_READ_BARRIER(arch_spin_lock(&arch_spinlock), false); 602 KCSAN_EXPECT_READ_BARRIER(arch_spin_unlock(&arch_spinlock), true); 603 KCSAN_EXPECT_READ_BARRIER(spin_lock(&test_spinlock), false); 604 KCSAN_EXPECT_READ_BARRIER(spin_unlock(&test_spinlock), true); 605 KCSAN_EXPECT_READ_BARRIER(mutex_lock(&test_mutex), false); 606 KCSAN_EXPECT_READ_BARRIER(mutex_unlock(&test_mutex), true); 607 608 KCSAN_EXPECT_WRITE_BARRIER(mb(), true); 609 KCSAN_EXPECT_WRITE_BARRIER(wmb(), true); 610 KCSAN_EXPECT_WRITE_BARRIER(rmb(), false); 611 KCSAN_EXPECT_WRITE_BARRIER(smp_mb(), true); 612 KCSAN_EXPECT_WRITE_BARRIER(smp_wmb(), true); 613 KCSAN_EXPECT_WRITE_BARRIER(smp_rmb(), false); 614 KCSAN_EXPECT_WRITE_BARRIER(dma_wmb(), true); 615 KCSAN_EXPECT_WRITE_BARRIER(dma_rmb(), false); 616 KCSAN_EXPECT_WRITE_BARRIER(smp_mb__before_atomic(), true); 617 KCSAN_EXPECT_WRITE_BARRIER(smp_mb__after_atomic(), true); 618 KCSAN_EXPECT_WRITE_BARRIER(smp_mb__after_spinlock(), true); 619 KCSAN_EXPECT_WRITE_BARRIER(smp_store_mb(test_var, 0), true); 620 KCSAN_EXPECT_WRITE_BARRIER(smp_load_acquire(&test_var), false); 621 KCSAN_EXPECT_WRITE_BARRIER(smp_store_release(&test_var, 0), true); 622 KCSAN_EXPECT_WRITE_BARRIER(xchg(&test_var, 0), true); 623 KCSAN_EXPECT_WRITE_BARRIER(xchg_release(&test_var, 0), true); 624 KCSAN_EXPECT_WRITE_BARRIER(xchg_relaxed(&test_var, 0), false); 625 KCSAN_EXPECT_WRITE_BARRIER(cmpxchg(&test_var, 0, 0), true); 626 KCSAN_EXPECT_WRITE_BARRIER(cmpxchg_release(&test_var, 0, 0), true); 627 KCSAN_EXPECT_WRITE_BARRIER(cmpxchg_relaxed(&test_var, 0, 0), false); 628 KCSAN_EXPECT_WRITE_BARRIER(atomic_read(&dummy), false); 629 KCSAN_EXPECT_WRITE_BARRIER(atomic_read_acquire(&dummy), false); 630 KCSAN_EXPECT_WRITE_BARRIER(atomic_set(&dummy, 0), false); 631 KCSAN_EXPECT_WRITE_BARRIER(atomic_set_release(&dummy, 0), true); 632 KCSAN_EXPECT_WRITE_BARRIER(atomic_add(1, &dummy), false); 633 KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return(1, &dummy), true); 634 KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_acquire(1, &dummy), false); 635 KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_release(1, &dummy), true); 636 KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_relaxed(1, &dummy), false); 637 KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add(1, &dummy), true); 638 KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_acquire(1, &dummy), false); 639 KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_release(1, &dummy), true); 640 KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false); 641 KCSAN_EXPECT_WRITE_BARRIER(test_and_set_bit(0, &test_var), true); 642 KCSAN_EXPECT_WRITE_BARRIER(test_and_clear_bit(0, &test_var), true); 643 KCSAN_EXPECT_WRITE_BARRIER(test_and_change_bit(0, &test_var), true); 644 KCSAN_EXPECT_WRITE_BARRIER(clear_bit_unlock(0, &test_var), true); 645 KCSAN_EXPECT_WRITE_BARRIER(__clear_bit_unlock(0, &test_var), true); 646 KCSAN_EXPECT_WRITE_BARRIER(arch_spin_lock(&arch_spinlock), false); 647 KCSAN_EXPECT_WRITE_BARRIER(arch_spin_unlock(&arch_spinlock), true); 648 KCSAN_EXPECT_WRITE_BARRIER(spin_lock(&test_spinlock), false); 649 KCSAN_EXPECT_WRITE_BARRIER(spin_unlock(&test_spinlock), true); 650 KCSAN_EXPECT_WRITE_BARRIER(mutex_lock(&test_mutex), false); 651 KCSAN_EXPECT_WRITE_BARRIER(mutex_unlock(&test_mutex), true); 652 653 KCSAN_EXPECT_RW_BARRIER(mb(), true); 654 KCSAN_EXPECT_RW_BARRIER(wmb(), true); 655 KCSAN_EXPECT_RW_BARRIER(rmb(), true); 656 KCSAN_EXPECT_RW_BARRIER(smp_mb(), true); 657 KCSAN_EXPECT_RW_BARRIER(smp_wmb(), true); 658 KCSAN_EXPECT_RW_BARRIER(smp_rmb(), true); 659 KCSAN_EXPECT_RW_BARRIER(dma_wmb(), true); 660 KCSAN_EXPECT_RW_BARRIER(dma_rmb(), true); 661 KCSAN_EXPECT_RW_BARRIER(smp_mb__before_atomic(), true); 662 KCSAN_EXPECT_RW_BARRIER(smp_mb__after_atomic(), true); 663 KCSAN_EXPECT_RW_BARRIER(smp_mb__after_spinlock(), true); 664 KCSAN_EXPECT_RW_BARRIER(smp_store_mb(test_var, 0), true); 665 KCSAN_EXPECT_RW_BARRIER(smp_load_acquire(&test_var), false); 666 KCSAN_EXPECT_RW_BARRIER(smp_store_release(&test_var, 0), true); 667 KCSAN_EXPECT_RW_BARRIER(xchg(&test_var, 0), true); 668 KCSAN_EXPECT_RW_BARRIER(xchg_release(&test_var, 0), true); 669 KCSAN_EXPECT_RW_BARRIER(xchg_relaxed(&test_var, 0), false); 670 KCSAN_EXPECT_RW_BARRIER(cmpxchg(&test_var, 0, 0), true); 671 KCSAN_EXPECT_RW_BARRIER(cmpxchg_release(&test_var, 0, 0), true); 672 KCSAN_EXPECT_RW_BARRIER(cmpxchg_relaxed(&test_var, 0, 0), false); 673 KCSAN_EXPECT_RW_BARRIER(atomic_read(&dummy), false); 674 KCSAN_EXPECT_RW_BARRIER(atomic_read_acquire(&dummy), false); 675 KCSAN_EXPECT_RW_BARRIER(atomic_set(&dummy, 0), false); 676 KCSAN_EXPECT_RW_BARRIER(atomic_set_release(&dummy, 0), true); 677 KCSAN_EXPECT_RW_BARRIER(atomic_add(1, &dummy), false); 678 KCSAN_EXPECT_RW_BARRIER(atomic_add_return(1, &dummy), true); 679 KCSAN_EXPECT_RW_BARRIER(atomic_add_return_acquire(1, &dummy), false); 680 KCSAN_EXPECT_RW_BARRIER(atomic_add_return_release(1, &dummy), true); 681 KCSAN_EXPECT_RW_BARRIER(atomic_add_return_relaxed(1, &dummy), false); 682 KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add(1, &dummy), true); 683 KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_acquire(1, &dummy), false); 684 KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_release(1, &dummy), true); 685 KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false); 686 KCSAN_EXPECT_RW_BARRIER(test_and_set_bit(0, &test_var), true); 687 KCSAN_EXPECT_RW_BARRIER(test_and_clear_bit(0, &test_var), true); 688 KCSAN_EXPECT_RW_BARRIER(test_and_change_bit(0, &test_var), true); 689 KCSAN_EXPECT_RW_BARRIER(clear_bit_unlock(0, &test_var), true); 690 KCSAN_EXPECT_RW_BARRIER(__clear_bit_unlock(0, &test_var), true); 691 KCSAN_EXPECT_RW_BARRIER(arch_spin_lock(&arch_spinlock), false); 692 KCSAN_EXPECT_RW_BARRIER(arch_spin_unlock(&arch_spinlock), true); 693 KCSAN_EXPECT_RW_BARRIER(spin_lock(&test_spinlock), false); 694 KCSAN_EXPECT_RW_BARRIER(spin_unlock(&test_spinlock), true); 695 KCSAN_EXPECT_RW_BARRIER(mutex_lock(&test_mutex), false); 696 KCSAN_EXPECT_RW_BARRIER(mutex_unlock(&test_mutex), true); 697 698 #ifdef clear_bit_unlock_is_negative_byte 699 KCSAN_EXPECT_READ_BARRIER(clear_bit_unlock_is_negative_byte(0, &test_var), true); 700 KCSAN_EXPECT_WRITE_BARRIER(clear_bit_unlock_is_negative_byte(0, &test_var), true); 701 KCSAN_EXPECT_RW_BARRIER(clear_bit_unlock_is_negative_byte(0, &test_var), true); 702 #endif 703 kcsan_nestable_atomic_end(); 704 } 705 706 /* Simple test with normal data race. */ 707 __no_kcsan 708 static void test_basic(struct kunit *test) 709 { 710 struct expect_report expect = { 711 .access = { 712 { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 713 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 714 }, 715 }; 716 struct expect_report never = { 717 .access = { 718 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 719 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 720 }, 721 }; 722 bool match_expect = false; 723 bool match_never = false; 724 725 begin_test_checks(test_kernel_write, test_kernel_read); 726 do { 727 match_expect |= report_matches(&expect); 728 match_never = report_matches(&never); 729 } while (!end_test_checks(match_never)); 730 KUNIT_EXPECT_TRUE(test, match_expect); 731 KUNIT_EXPECT_FALSE(test, match_never); 732 } 733 734 /* 735 * Stress KCSAN with lots of concurrent races on different addresses until 736 * timeout. 737 */ 738 __no_kcsan 739 static void test_concurrent_races(struct kunit *test) 740 { 741 struct expect_report expect = { 742 .access = { 743 /* NULL will match any address. */ 744 { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, 745 { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(0) }, 746 }, 747 }; 748 struct expect_report never = { 749 .access = { 750 { test_kernel_rmw_array, NULL, 0, 0 }, 751 { test_kernel_rmw_array, NULL, 0, 0 }, 752 }, 753 }; 754 bool match_expect = false; 755 bool match_never = false; 756 757 begin_test_checks(test_kernel_rmw_array, test_kernel_rmw_array); 758 do { 759 match_expect |= report_matches(&expect); 760 match_never |= report_matches(&never); 761 } while (!end_test_checks(false)); 762 KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check matches exist. */ 763 KUNIT_EXPECT_FALSE(test, match_never); 764 } 765 766 /* Test the KCSAN_REPORT_VALUE_CHANGE_ONLY option. */ 767 __no_kcsan 768 static void test_novalue_change(struct kunit *test) 769 { 770 struct expect_report expect_rw = { 771 .access = { 772 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 773 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 774 }, 775 }; 776 struct expect_report expect_ww = { 777 .access = { 778 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 779 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 780 }, 781 }; 782 bool match_expect = false; 783 784 test_kernel_write_nochange(); /* Reset value. */ 785 begin_test_checks(test_kernel_write_nochange, test_kernel_read); 786 do { 787 match_expect = report_matches(&expect_rw) || report_matches(&expect_ww); 788 } while (!end_test_checks(match_expect)); 789 if (IS_ENABLED(CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY)) 790 KUNIT_EXPECT_FALSE(test, match_expect); 791 else 792 KUNIT_EXPECT_TRUE(test, match_expect); 793 } 794 795 /* 796 * Test that the rules where the KCSAN_REPORT_VALUE_CHANGE_ONLY option should 797 * never apply work. 798 */ 799 __no_kcsan 800 static void test_novalue_change_exception(struct kunit *test) 801 { 802 struct expect_report expect_rw = { 803 .access = { 804 { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 805 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 806 }, 807 }; 808 struct expect_report expect_ww = { 809 .access = { 810 { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 811 { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 812 }, 813 }; 814 bool match_expect = false; 815 816 test_kernel_write_nochange_rcu(); /* Reset value. */ 817 begin_test_checks(test_kernel_write_nochange_rcu, test_kernel_read); 818 do { 819 match_expect = report_matches(&expect_rw) || report_matches(&expect_ww); 820 } while (!end_test_checks(match_expect)); 821 KUNIT_EXPECT_TRUE(test, match_expect); 822 } 823 824 /* Test that data races of unknown origin are reported. */ 825 __no_kcsan 826 static void test_unknown_origin(struct kunit *test) 827 { 828 struct expect_report expect = { 829 .access = { 830 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 831 { NULL }, 832 }, 833 }; 834 bool match_expect = false; 835 836 begin_test_checks(test_kernel_write_uninstrumented, test_kernel_read); 837 do { 838 match_expect = report_matches(&expect); 839 } while (!end_test_checks(match_expect)); 840 if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN)) 841 KUNIT_EXPECT_TRUE(test, match_expect); 842 else 843 KUNIT_EXPECT_FALSE(test, match_expect); 844 } 845 846 /* Test KCSAN_ASSUME_PLAIN_WRITES_ATOMIC if it is selected. */ 847 __no_kcsan 848 static void test_write_write_assume_atomic(struct kunit *test) 849 { 850 struct expect_report expect = { 851 .access = { 852 { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 853 { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 854 }, 855 }; 856 bool match_expect = false; 857 858 begin_test_checks(test_kernel_write, test_kernel_write); 859 do { 860 sink_value(READ_ONCE(test_var)); /* induce value-change */ 861 match_expect = report_matches(&expect); 862 } while (!end_test_checks(match_expect)); 863 if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) 864 KUNIT_EXPECT_FALSE(test, match_expect); 865 else 866 KUNIT_EXPECT_TRUE(test, match_expect); 867 } 868 869 /* 870 * Test that data races with writes larger than word-size are always reported, 871 * even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected. 872 */ 873 __no_kcsan 874 static void test_write_write_struct(struct kunit *test) 875 { 876 struct expect_report expect = { 877 .access = { 878 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, 879 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, 880 }, 881 }; 882 bool match_expect = false; 883 884 begin_test_checks(test_kernel_write_struct, test_kernel_write_struct); 885 do { 886 match_expect = report_matches(&expect); 887 } while (!end_test_checks(match_expect)); 888 KUNIT_EXPECT_TRUE(test, match_expect); 889 } 890 891 /* 892 * Test that data races where only one write is larger than word-size are always 893 * reported, even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected. 894 */ 895 __no_kcsan 896 static void test_write_write_struct_part(struct kunit *test) 897 { 898 struct expect_report expect = { 899 .access = { 900 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, 901 { test_kernel_write_struct_part, &test_struct.val[3], sizeof(test_struct.val[3]), KCSAN_ACCESS_WRITE }, 902 }, 903 }; 904 bool match_expect = false; 905 906 begin_test_checks(test_kernel_write_struct, test_kernel_write_struct_part); 907 do { 908 match_expect = report_matches(&expect); 909 } while (!end_test_checks(match_expect)); 910 KUNIT_EXPECT_TRUE(test, match_expect); 911 } 912 913 /* Test that races with atomic accesses never result in reports. */ 914 __no_kcsan 915 static void test_read_atomic_write_atomic(struct kunit *test) 916 { 917 bool match_never = false; 918 919 begin_test_checks(test_kernel_read_atomic, test_kernel_write_atomic); 920 do { 921 match_never = report_available(); 922 } while (!end_test_checks(match_never)); 923 KUNIT_EXPECT_FALSE(test, match_never); 924 } 925 926 /* Test that a race with an atomic and plain access result in reports. */ 927 __no_kcsan 928 static void test_read_plain_atomic_write(struct kunit *test) 929 { 930 struct expect_report expect = { 931 .access = { 932 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 933 { test_kernel_write_atomic, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC }, 934 }, 935 }; 936 bool match_expect = false; 937 938 KCSAN_TEST_REQUIRES(test, !IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)); 939 940 begin_test_checks(test_kernel_read, test_kernel_write_atomic); 941 do { 942 match_expect = report_matches(&expect); 943 } while (!end_test_checks(match_expect)); 944 KUNIT_EXPECT_TRUE(test, match_expect); 945 } 946 947 /* Test that atomic RMWs generate correct report. */ 948 __no_kcsan 949 static void test_read_plain_atomic_rmw(struct kunit *test) 950 { 951 struct expect_report expect = { 952 .access = { 953 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 954 { test_kernel_atomic_rmw, &test_var, sizeof(test_var), 955 KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC }, 956 }, 957 }; 958 bool match_expect = false; 959 960 KCSAN_TEST_REQUIRES(test, !IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)); 961 962 begin_test_checks(test_kernel_read, test_kernel_atomic_rmw); 963 do { 964 match_expect = report_matches(&expect); 965 } while (!end_test_checks(match_expect)); 966 KUNIT_EXPECT_TRUE(test, match_expect); 967 } 968 969 /* Zero-sized accesses should never cause data race reports. */ 970 __no_kcsan 971 static void test_zero_size_access(struct kunit *test) 972 { 973 struct expect_report expect = { 974 .access = { 975 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, 976 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, 977 }, 978 }; 979 struct expect_report never = { 980 .access = { 981 { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, 982 { test_kernel_read_struct_zero_size, &test_struct.val[3], 0, 0 }, 983 }, 984 }; 985 bool match_expect = false; 986 bool match_never = false; 987 988 begin_test_checks(test_kernel_write_struct, test_kernel_read_struct_zero_size); 989 do { 990 match_expect |= report_matches(&expect); 991 match_never = report_matches(&never); 992 } while (!end_test_checks(match_never)); 993 KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check. */ 994 KUNIT_EXPECT_FALSE(test, match_never); 995 } 996 997 /* Test the data_race() macro. */ 998 __no_kcsan 999 static void test_data_race(struct kunit *test) 1000 { 1001 bool match_never = false; 1002 1003 begin_test_checks(test_kernel_data_race, test_kernel_data_race); 1004 do { 1005 match_never = report_available(); 1006 } while (!end_test_checks(match_never)); 1007 KUNIT_EXPECT_FALSE(test, match_never); 1008 } 1009 1010 __no_kcsan 1011 static void test_assert_exclusive_writer(struct kunit *test) 1012 { 1013 struct expect_report expect = { 1014 .access = { 1015 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, 1016 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 1017 }, 1018 }; 1019 bool match_expect = false; 1020 1021 begin_test_checks(test_kernel_assert_writer, test_kernel_write_nochange); 1022 do { 1023 match_expect = report_matches(&expect); 1024 } while (!end_test_checks(match_expect)); 1025 KUNIT_EXPECT_TRUE(test, match_expect); 1026 } 1027 1028 __no_kcsan 1029 static void test_assert_exclusive_access(struct kunit *test) 1030 { 1031 struct expect_report expect = { 1032 .access = { 1033 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, 1034 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 1035 }, 1036 }; 1037 bool match_expect = false; 1038 1039 begin_test_checks(test_kernel_assert_access, test_kernel_read); 1040 do { 1041 match_expect = report_matches(&expect); 1042 } while (!end_test_checks(match_expect)); 1043 KUNIT_EXPECT_TRUE(test, match_expect); 1044 } 1045 1046 __no_kcsan 1047 static void test_assert_exclusive_access_writer(struct kunit *test) 1048 { 1049 struct expect_report expect_access_writer = { 1050 .access = { 1051 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, 1052 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, 1053 }, 1054 }; 1055 struct expect_report expect_access_access = { 1056 .access = { 1057 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, 1058 { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, 1059 }, 1060 }; 1061 struct expect_report never = { 1062 .access = { 1063 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, 1064 { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, 1065 }, 1066 }; 1067 bool match_expect_access_writer = false; 1068 bool match_expect_access_access = false; 1069 bool match_never = false; 1070 1071 begin_test_checks(test_kernel_assert_access, test_kernel_assert_writer); 1072 do { 1073 match_expect_access_writer |= report_matches(&expect_access_writer); 1074 match_expect_access_access |= report_matches(&expect_access_access); 1075 match_never |= report_matches(&never); 1076 } while (!end_test_checks(match_never)); 1077 KUNIT_EXPECT_TRUE(test, match_expect_access_writer); 1078 KUNIT_EXPECT_TRUE(test, match_expect_access_access); 1079 KUNIT_EXPECT_FALSE(test, match_never); 1080 } 1081 1082 __no_kcsan 1083 static void test_assert_exclusive_bits_change(struct kunit *test) 1084 { 1085 struct expect_report expect = { 1086 .access = { 1087 { test_kernel_assert_bits_change, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, 1088 { test_kernel_change_bits, &test_var, sizeof(test_var), 1089 KCSAN_ACCESS_WRITE | (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) ? 0 : KCSAN_ACCESS_ATOMIC) }, 1090 }, 1091 }; 1092 bool match_expect = false; 1093 1094 begin_test_checks(test_kernel_assert_bits_change, test_kernel_change_bits); 1095 do { 1096 match_expect = report_matches(&expect); 1097 } while (!end_test_checks(match_expect)); 1098 KUNIT_EXPECT_TRUE(test, match_expect); 1099 } 1100 1101 __no_kcsan 1102 static void test_assert_exclusive_bits_nochange(struct kunit *test) 1103 { 1104 bool match_never = false; 1105 1106 begin_test_checks(test_kernel_assert_bits_nochange, test_kernel_change_bits); 1107 do { 1108 match_never = report_available(); 1109 } while (!end_test_checks(match_never)); 1110 KUNIT_EXPECT_FALSE(test, match_never); 1111 } 1112 1113 __no_kcsan 1114 static void test_assert_exclusive_writer_scoped(struct kunit *test) 1115 { 1116 struct expect_report expect_start = { 1117 .access = { 1118 { test_kernel_assert_writer_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED }, 1119 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 1120 }, 1121 }; 1122 struct expect_report expect_inscope = { 1123 .access = { 1124 { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED }, 1125 { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, 1126 }, 1127 }; 1128 bool match_expect_start = false; 1129 bool match_expect_inscope = false; 1130 1131 begin_test_checks(test_kernel_assert_writer_scoped, test_kernel_write_nochange); 1132 do { 1133 match_expect_start |= report_matches(&expect_start); 1134 match_expect_inscope |= report_matches(&expect_inscope); 1135 } while (!end_test_checks(match_expect_inscope)); 1136 KUNIT_EXPECT_TRUE(test, match_expect_start); 1137 KUNIT_EXPECT_FALSE(test, match_expect_inscope); 1138 } 1139 1140 __no_kcsan 1141 static void test_assert_exclusive_access_scoped(struct kunit *test) 1142 { 1143 struct expect_report expect_start1 = { 1144 .access = { 1145 { test_kernel_assert_access_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED }, 1146 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 1147 }, 1148 }; 1149 struct expect_report expect_start2 = { 1150 .access = { expect_start1.access[0], expect_start1.access[0] }, 1151 }; 1152 struct expect_report expect_inscope = { 1153 .access = { 1154 { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED }, 1155 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 1156 }, 1157 }; 1158 bool match_expect_start = false; 1159 bool match_expect_inscope = false; 1160 1161 begin_test_checks(test_kernel_assert_access_scoped, test_kernel_read); 1162 end_time += msecs_to_jiffies(1000); /* This test requires a bit more time. */ 1163 do { 1164 match_expect_start |= report_matches(&expect_start1) || report_matches(&expect_start2); 1165 match_expect_inscope |= report_matches(&expect_inscope); 1166 } while (!end_test_checks(match_expect_inscope)); 1167 KUNIT_EXPECT_TRUE(test, match_expect_start); 1168 KUNIT_EXPECT_FALSE(test, match_expect_inscope); 1169 } 1170 1171 /* 1172 * jiffies is special (declared to be volatile) and its accesses are typically 1173 * not marked; this test ensures that the compiler nor KCSAN gets confused about 1174 * jiffies's declaration on different architectures. 1175 */ 1176 __no_kcsan 1177 static void test_jiffies_noreport(struct kunit *test) 1178 { 1179 bool match_never = false; 1180 1181 begin_test_checks(test_kernel_jiffies_reader, test_kernel_jiffies_reader); 1182 do { 1183 match_never = report_available(); 1184 } while (!end_test_checks(match_never)); 1185 KUNIT_EXPECT_FALSE(test, match_never); 1186 } 1187 1188 /* Test that racing accesses in seqlock critical sections are not reported. */ 1189 __no_kcsan 1190 static void test_seqlock_noreport(struct kunit *test) 1191 { 1192 bool match_never = false; 1193 1194 begin_test_checks(test_kernel_seqlock_reader, test_kernel_seqlock_writer); 1195 do { 1196 match_never = report_available(); 1197 } while (!end_test_checks(match_never)); 1198 KUNIT_EXPECT_FALSE(test, match_never); 1199 } 1200 1201 /* 1202 * Test atomic builtins work and required instrumentation functions exist. We 1203 * also test that KCSAN understands they're atomic by racing with them via 1204 * test_kernel_atomic_builtins(), and expect no reports. 1205 * 1206 * The atomic builtins _SHOULD NOT_ be used in normal kernel code! 1207 */ 1208 static void test_atomic_builtins(struct kunit *test) 1209 { 1210 bool match_never = false; 1211 1212 begin_test_checks(test_kernel_atomic_builtins, test_kernel_atomic_builtins); 1213 do { 1214 long tmp; 1215 1216 kcsan_enable_current(); 1217 1218 __atomic_store_n(&test_var, 42L, __ATOMIC_RELAXED); 1219 KUNIT_EXPECT_EQ(test, 42L, __atomic_load_n(&test_var, __ATOMIC_RELAXED)); 1220 1221 KUNIT_EXPECT_EQ(test, 42L, __atomic_exchange_n(&test_var, 20, __ATOMIC_RELAXED)); 1222 KUNIT_EXPECT_EQ(test, 20L, test_var); 1223 1224 tmp = 20L; 1225 KUNIT_EXPECT_TRUE(test, __atomic_compare_exchange_n(&test_var, &tmp, 30L, 1226 0, __ATOMIC_RELAXED, 1227 __ATOMIC_RELAXED)); 1228 KUNIT_EXPECT_EQ(test, tmp, 20L); 1229 KUNIT_EXPECT_EQ(test, test_var, 30L); 1230 KUNIT_EXPECT_FALSE(test, __atomic_compare_exchange_n(&test_var, &tmp, 40L, 1231 1, __ATOMIC_RELAXED, 1232 __ATOMIC_RELAXED)); 1233 KUNIT_EXPECT_EQ(test, tmp, 30L); 1234 KUNIT_EXPECT_EQ(test, test_var, 30L); 1235 1236 KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED)); 1237 KUNIT_EXPECT_EQ(test, 31L, __atomic_fetch_sub(&test_var, 1, __ATOMIC_RELAXED)); 1238 KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_and(&test_var, 0xf, __ATOMIC_RELAXED)); 1239 KUNIT_EXPECT_EQ(test, 14L, __atomic_fetch_xor(&test_var, 0xf, __ATOMIC_RELAXED)); 1240 KUNIT_EXPECT_EQ(test, 1L, __atomic_fetch_or(&test_var, 0xf0, __ATOMIC_RELAXED)); 1241 KUNIT_EXPECT_EQ(test, 241L, __atomic_fetch_nand(&test_var, 0xf, __ATOMIC_RELAXED)); 1242 KUNIT_EXPECT_EQ(test, -2L, test_var); 1243 1244 __atomic_thread_fence(__ATOMIC_SEQ_CST); 1245 __atomic_signal_fence(__ATOMIC_SEQ_CST); 1246 1247 kcsan_disable_current(); 1248 1249 match_never = report_available(); 1250 } while (!end_test_checks(match_never)); 1251 KUNIT_EXPECT_FALSE(test, match_never); 1252 } 1253 1254 __no_kcsan 1255 static void test_1bit_value_change(struct kunit *test) 1256 { 1257 struct expect_report expect = { 1258 .access = { 1259 { test_kernel_read, &test_var, sizeof(test_var), 0 }, 1260 { test_kernel_xor_1bit, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, 1261 }, 1262 }; 1263 bool match = false; 1264 1265 begin_test_checks(test_kernel_read, test_kernel_xor_1bit); 1266 do { 1267 match = IS_ENABLED(CONFIG_KCSAN_PERMISSIVE) 1268 ? report_available() 1269 : report_matches(&expect); 1270 } while (!end_test_checks(match)); 1271 if (IS_ENABLED(CONFIG_KCSAN_PERMISSIVE)) 1272 KUNIT_EXPECT_FALSE(test, match); 1273 else 1274 KUNIT_EXPECT_TRUE(test, match); 1275 } 1276 1277 __no_kcsan 1278 static void test_correct_barrier(struct kunit *test) 1279 { 1280 struct expect_report expect = { 1281 .access = { 1282 { test_kernel_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, 1283 { test_kernel_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, 1284 }, 1285 }; 1286 bool match_expect = false; 1287 1288 test_struct.val[0] = 0; /* init unlocked */ 1289 begin_test_checks(test_kernel_with_memorder, test_kernel_with_memorder); 1290 do { 1291 match_expect = report_matches_any_reordered(&expect); 1292 } while (!end_test_checks(match_expect)); 1293 KUNIT_EXPECT_FALSE(test, match_expect); 1294 } 1295 1296 __no_kcsan 1297 static void test_missing_barrier(struct kunit *test) 1298 { 1299 struct expect_report expect = { 1300 .access = { 1301 { test_kernel_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, 1302 { test_kernel_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, 1303 }, 1304 }; 1305 bool match_expect = false; 1306 1307 test_struct.val[0] = 0; /* init unlocked */ 1308 begin_test_checks(test_kernel_wrong_memorder, test_kernel_wrong_memorder); 1309 do { 1310 match_expect = report_matches_any_reordered(&expect); 1311 } while (!end_test_checks(match_expect)); 1312 if (IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY)) 1313 KUNIT_EXPECT_TRUE(test, match_expect); 1314 else 1315 KUNIT_EXPECT_FALSE(test, match_expect); 1316 } 1317 1318 __no_kcsan 1319 static void test_atomic_builtins_correct_barrier(struct kunit *test) 1320 { 1321 struct expect_report expect = { 1322 .access = { 1323 { test_kernel_atomic_builtin_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, 1324 { test_kernel_atomic_builtin_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, 1325 }, 1326 }; 1327 bool match_expect = false; 1328 1329 test_struct.val[0] = 0; /* init unlocked */ 1330 begin_test_checks(test_kernel_atomic_builtin_with_memorder, 1331 test_kernel_atomic_builtin_with_memorder); 1332 do { 1333 match_expect = report_matches_any_reordered(&expect); 1334 } while (!end_test_checks(match_expect)); 1335 KUNIT_EXPECT_FALSE(test, match_expect); 1336 } 1337 1338 __no_kcsan 1339 static void test_atomic_builtins_missing_barrier(struct kunit *test) 1340 { 1341 struct expect_report expect = { 1342 .access = { 1343 { test_kernel_atomic_builtin_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, 1344 { test_kernel_atomic_builtin_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, 1345 }, 1346 }; 1347 bool match_expect = false; 1348 1349 test_struct.val[0] = 0; /* init unlocked */ 1350 begin_test_checks(test_kernel_atomic_builtin_wrong_memorder, 1351 test_kernel_atomic_builtin_wrong_memorder); 1352 do { 1353 match_expect = report_matches_any_reordered(&expect); 1354 } while (!end_test_checks(match_expect)); 1355 if (IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY)) 1356 KUNIT_EXPECT_TRUE(test, match_expect); 1357 else 1358 KUNIT_EXPECT_FALSE(test, match_expect); 1359 } 1360 1361 /* 1362 * Generate thread counts for all test cases. Values generated are in interval 1363 * [2, 5] followed by exponentially increasing thread counts from 8 to 32. 1364 * 1365 * The thread counts are chosen to cover potentially interesting boundaries and 1366 * corner cases (2 to 5), and then stress the system with larger counts. 1367 */ 1368 static const void *nthreads_gen_params(const void *prev, char *desc) 1369 { 1370 long nthreads = (long)prev; 1371 1372 if (nthreads < 0 || nthreads >= 32) 1373 nthreads = 0; /* stop */ 1374 else if (!nthreads) 1375 nthreads = 2; /* initial value */ 1376 else if (nthreads < 5) 1377 nthreads++; 1378 else if (nthreads == 5) 1379 nthreads = 8; 1380 else 1381 nthreads *= 2; 1382 1383 if (!preempt_model_preemptible() || 1384 !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) { 1385 /* 1386 * Without any preemption, keep 2 CPUs free for other tasks, one 1387 * of which is the main test case function checking for 1388 * completion or failure. 1389 */ 1390 const long min_unused_cpus = preempt_model_none() ? 2 : 0; 1391 const long min_required_cpus = 2 + min_unused_cpus; 1392 1393 if (num_online_cpus() < min_required_cpus) { 1394 pr_err_once("Too few online CPUs (%u < %ld) for test\n", 1395 num_online_cpus(), min_required_cpus); 1396 nthreads = 0; 1397 } else if (nthreads >= num_online_cpus() - min_unused_cpus) { 1398 /* Use negative value to indicate last param. */ 1399 nthreads = -(num_online_cpus() - min_unused_cpus); 1400 pr_warn_once("Limiting number of threads to %ld (only %d online CPUs)\n", 1401 -nthreads, num_online_cpus()); 1402 } 1403 } 1404 1405 snprintf(desc, KUNIT_PARAM_DESC_SIZE, "threads=%ld", abs(nthreads)); 1406 return (void *)nthreads; 1407 } 1408 1409 #define KCSAN_KUNIT_CASE(test_name) KUNIT_CASE_PARAM(test_name, nthreads_gen_params) 1410 static struct kunit_case kcsan_test_cases[] = { 1411 KUNIT_CASE(test_barrier_nothreads), 1412 KCSAN_KUNIT_CASE(test_basic), 1413 KCSAN_KUNIT_CASE(test_concurrent_races), 1414 KCSAN_KUNIT_CASE(test_novalue_change), 1415 KCSAN_KUNIT_CASE(test_novalue_change_exception), 1416 KCSAN_KUNIT_CASE(test_unknown_origin), 1417 KCSAN_KUNIT_CASE(test_write_write_assume_atomic), 1418 KCSAN_KUNIT_CASE(test_write_write_struct), 1419 KCSAN_KUNIT_CASE(test_write_write_struct_part), 1420 KCSAN_KUNIT_CASE(test_read_atomic_write_atomic), 1421 KCSAN_KUNIT_CASE(test_read_plain_atomic_write), 1422 KCSAN_KUNIT_CASE(test_read_plain_atomic_rmw), 1423 KCSAN_KUNIT_CASE(test_zero_size_access), 1424 KCSAN_KUNIT_CASE(test_data_race), 1425 KCSAN_KUNIT_CASE(test_assert_exclusive_writer), 1426 KCSAN_KUNIT_CASE(test_assert_exclusive_access), 1427 KCSAN_KUNIT_CASE(test_assert_exclusive_access_writer), 1428 KCSAN_KUNIT_CASE(test_assert_exclusive_bits_change), 1429 KCSAN_KUNIT_CASE(test_assert_exclusive_bits_nochange), 1430 KCSAN_KUNIT_CASE(test_assert_exclusive_writer_scoped), 1431 KCSAN_KUNIT_CASE(test_assert_exclusive_access_scoped), 1432 KCSAN_KUNIT_CASE(test_jiffies_noreport), 1433 KCSAN_KUNIT_CASE(test_seqlock_noreport), 1434 KCSAN_KUNIT_CASE(test_atomic_builtins), 1435 KCSAN_KUNIT_CASE(test_1bit_value_change), 1436 KCSAN_KUNIT_CASE(test_correct_barrier), 1437 KCSAN_KUNIT_CASE(test_missing_barrier), 1438 KCSAN_KUNIT_CASE(test_atomic_builtins_correct_barrier), 1439 KCSAN_KUNIT_CASE(test_atomic_builtins_missing_barrier), 1440 {}, 1441 }; 1442 1443 /* ===== End test cases ===== */ 1444 1445 /* Concurrent accesses from interrupts. */ 1446 __no_kcsan 1447 static void access_thread_timer(struct timer_list *timer) 1448 { 1449 static atomic_t cnt = ATOMIC_INIT(0); 1450 unsigned int idx; 1451 void (*func)(void); 1452 1453 idx = (unsigned int)atomic_inc_return(&cnt) % ARRAY_SIZE(access_kernels); 1454 /* Acquire potential initialization. */ 1455 func = smp_load_acquire(&access_kernels[idx]); 1456 if (func) 1457 func(); 1458 } 1459 1460 /* The main loop for each thread. */ 1461 __no_kcsan 1462 static int access_thread(void *arg) 1463 { 1464 struct timer_list timer; 1465 unsigned int cnt = 0; 1466 unsigned int idx; 1467 void (*func)(void); 1468 1469 timer_setup_on_stack(&timer, access_thread_timer, 0); 1470 do { 1471 might_sleep(); 1472 1473 if (!timer_pending(&timer)) 1474 mod_timer(&timer, jiffies + 1); 1475 else { 1476 /* Iterate through all kernels. */ 1477 idx = cnt++ % ARRAY_SIZE(access_kernels); 1478 /* Acquire potential initialization. */ 1479 func = smp_load_acquire(&access_kernels[idx]); 1480 if (func) 1481 func(); 1482 } 1483 } while (!torture_must_stop()); 1484 del_timer_sync(&timer); 1485 destroy_timer_on_stack(&timer); 1486 1487 torture_kthread_stopping("access_thread"); 1488 return 0; 1489 } 1490 1491 __no_kcsan 1492 static int test_init(struct kunit *test) 1493 { 1494 unsigned long flags; 1495 int nthreads; 1496 int i; 1497 1498 spin_lock_irqsave(&observed.lock, flags); 1499 for (i = 0; i < ARRAY_SIZE(observed.lines); ++i) 1500 observed.lines[i][0] = '\0'; 1501 observed.nlines = 0; 1502 spin_unlock_irqrestore(&observed.lock, flags); 1503 1504 if (strstr(test->name, "nothreads")) 1505 return 0; 1506 1507 if (!torture_init_begin((char *)test->name, 1)) 1508 return -EBUSY; 1509 1510 if (WARN_ON(threads)) 1511 goto err; 1512 1513 for (i = 0; i < ARRAY_SIZE(access_kernels); ++i) { 1514 if (WARN_ON(access_kernels[i])) 1515 goto err; 1516 } 1517 1518 nthreads = abs((long)test->param_value); 1519 if (WARN_ON(!nthreads)) 1520 goto err; 1521 1522 threads = kcalloc(nthreads + 1, sizeof(struct task_struct *), GFP_KERNEL); 1523 if (WARN_ON(!threads)) 1524 goto err; 1525 1526 threads[nthreads] = NULL; 1527 for (i = 0; i < nthreads; ++i) { 1528 if (torture_create_kthread(access_thread, NULL, threads[i])) 1529 goto err; 1530 } 1531 1532 torture_init_end(); 1533 1534 return 0; 1535 1536 err: 1537 kfree(threads); 1538 threads = NULL; 1539 torture_init_end(); 1540 return -EINVAL; 1541 } 1542 1543 __no_kcsan 1544 static void test_exit(struct kunit *test) 1545 { 1546 struct task_struct **stop_thread; 1547 int i; 1548 1549 if (strstr(test->name, "nothreads")) 1550 return; 1551 1552 if (torture_cleanup_begin()) 1553 return; 1554 1555 for (i = 0; i < ARRAY_SIZE(access_kernels); ++i) 1556 WRITE_ONCE(access_kernels[i], NULL); 1557 1558 if (threads) { 1559 for (stop_thread = threads; *stop_thread; stop_thread++) 1560 torture_stop_kthread(reader_thread, *stop_thread); 1561 1562 kfree(threads); 1563 threads = NULL; 1564 } 1565 1566 torture_cleanup_end(); 1567 } 1568 1569 __no_kcsan 1570 static void register_tracepoints(struct tracepoint *tp, void *ignore) 1571 { 1572 check_trace_callback_type_console(probe_console); 1573 if (!strcmp(tp->name, "console")) 1574 WARN_ON(tracepoint_probe_register(tp, probe_console, NULL)); 1575 } 1576 1577 __no_kcsan 1578 static void unregister_tracepoints(struct tracepoint *tp, void *ignore) 1579 { 1580 if (!strcmp(tp->name, "console")) 1581 tracepoint_probe_unregister(tp, probe_console, NULL); 1582 } 1583 1584 static int kcsan_suite_init(struct kunit_suite *suite) 1585 { 1586 /* 1587 * Because we want to be able to build the test as a module, we need to 1588 * iterate through all known tracepoints, since the static registration 1589 * won't work here. 1590 */ 1591 for_each_kernel_tracepoint(register_tracepoints, NULL); 1592 return 0; 1593 } 1594 1595 static void kcsan_suite_exit(struct kunit_suite *suite) 1596 { 1597 for_each_kernel_tracepoint(unregister_tracepoints, NULL); 1598 tracepoint_synchronize_unregister(); 1599 } 1600 1601 static struct kunit_suite kcsan_test_suite = { 1602 .name = "kcsan", 1603 .test_cases = kcsan_test_cases, 1604 .init = test_init, 1605 .exit = test_exit, 1606 .suite_init = kcsan_suite_init, 1607 .suite_exit = kcsan_suite_exit, 1608 }; 1609 1610 kunit_test_suites(&kcsan_test_suite); 1611 1612 MODULE_LICENSE("GPL v2"); 1613 MODULE_AUTHOR("Marco Elver <elver@google.com>"); 1614