xref: /openbmc/linux/kernel/kcsan/kcsan_test.c (revision 48ca54e3)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * KCSAN test with various race scenarious to test runtime behaviour. Since the
4  * interface with which KCSAN's reports are obtained is via the console, this is
5  * the output we should verify. For each test case checks the presence (or
6  * absence) of generated reports. Relies on 'console' tracepoint to capture
7  * reports as they appear in the kernel log.
8  *
9  * Makes use of KUnit for test organization, and the Torture framework for test
10  * thread control.
11  *
12  * Copyright (C) 2020, Google LLC.
13  * Author: Marco Elver <elver@google.com>
14  */
15 
16 #define pr_fmt(fmt) "kcsan_test: " fmt
17 
18 #include <kunit/test.h>
19 #include <linux/atomic.h>
20 #include <linux/bitops.h>
21 #include <linux/jiffies.h>
22 #include <linux/kcsan-checks.h>
23 #include <linux/kernel.h>
24 #include <linux/mutex.h>
25 #include <linux/sched.h>
26 #include <linux/seqlock.h>
27 #include <linux/spinlock.h>
28 #include <linux/string.h>
29 #include <linux/timer.h>
30 #include <linux/torture.h>
31 #include <linux/tracepoint.h>
32 #include <linux/types.h>
33 #include <trace/events/printk.h>
34 
35 #define KCSAN_TEST_REQUIRES(test, cond) do {			\
36 	if (!(cond))						\
37 		kunit_skip((test), "Test requires: " #cond);	\
38 } while (0)
39 
40 #ifdef CONFIG_CC_HAS_TSAN_COMPOUND_READ_BEFORE_WRITE
41 #define __KCSAN_ACCESS_RW(alt) (KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE)
42 #else
43 #define __KCSAN_ACCESS_RW(alt) (alt)
44 #endif
45 
46 /* Points to current test-case memory access "kernels". */
47 static void (*access_kernels[2])(void);
48 
49 static struct task_struct **threads; /* Lists of threads. */
50 static unsigned long end_time;       /* End time of test. */
51 
52 /* Report as observed from console. */
53 static struct {
54 	spinlock_t lock;
55 	int nlines;
56 	char lines[3][512];
57 } observed = {
58 	.lock = __SPIN_LOCK_UNLOCKED(observed.lock),
59 };
60 
61 /* Setup test checking loop. */
62 static __no_kcsan inline void
63 begin_test_checks(void (*func1)(void), void (*func2)(void))
64 {
65 	kcsan_disable_current();
66 
67 	/*
68 	 * Require at least as long as KCSAN_REPORT_ONCE_IN_MS, to ensure at
69 	 * least one race is reported.
70 	 */
71 	end_time = jiffies + msecs_to_jiffies(CONFIG_KCSAN_REPORT_ONCE_IN_MS + 500);
72 
73 	/* Signal start; release potential initialization of shared data. */
74 	smp_store_release(&access_kernels[0], func1);
75 	smp_store_release(&access_kernels[1], func2);
76 }
77 
78 /* End test checking loop. */
79 static __no_kcsan inline bool
80 end_test_checks(bool stop)
81 {
82 	if (!stop && time_before(jiffies, end_time)) {
83 		/* Continue checking */
84 		might_sleep();
85 		return false;
86 	}
87 
88 	kcsan_enable_current();
89 	return true;
90 }
91 
92 /*
93  * Probe for console output: checks if a race was reported, and obtains observed
94  * lines of interest.
95  */
96 __no_kcsan
97 static void probe_console(void *ignore, const char *buf, size_t len)
98 {
99 	unsigned long flags;
100 	int nlines;
101 
102 	/*
103 	 * Note that KCSAN reports under a global lock, so we do not risk the
104 	 * possibility of having multiple reports interleaved. If that were the
105 	 * case, we'd expect tests to fail.
106 	 */
107 
108 	spin_lock_irqsave(&observed.lock, flags);
109 	nlines = observed.nlines;
110 
111 	if (strnstr(buf, "BUG: KCSAN: ", len) && strnstr(buf, "test_", len)) {
112 		/*
113 		 * KCSAN report and related to the test.
114 		 *
115 		 * The provided @buf is not NUL-terminated; copy no more than
116 		 * @len bytes and let strscpy() add the missing NUL-terminator.
117 		 */
118 		strscpy(observed.lines[0], buf, min(len + 1, sizeof(observed.lines[0])));
119 		nlines = 1;
120 	} else if ((nlines == 1 || nlines == 2) && strnstr(buf, "bytes by", len)) {
121 		strscpy(observed.lines[nlines++], buf, min(len + 1, sizeof(observed.lines[0])));
122 
123 		if (strnstr(buf, "race at unknown origin", len)) {
124 			if (WARN_ON(nlines != 2))
125 				goto out;
126 
127 			/* No second line of interest. */
128 			strcpy(observed.lines[nlines++], "<none>");
129 		}
130 	}
131 
132 out:
133 	WRITE_ONCE(observed.nlines, nlines); /* Publish new nlines. */
134 	spin_unlock_irqrestore(&observed.lock, flags);
135 }
136 
137 /* Check if a report related to the test exists. */
138 __no_kcsan
139 static bool report_available(void)
140 {
141 	return READ_ONCE(observed.nlines) == ARRAY_SIZE(observed.lines);
142 }
143 
144 /* Report information we expect in a report. */
145 struct expect_report {
146 	/* Access information of both accesses. */
147 	struct {
148 		void *fn;    /* Function pointer to expected function of top frame. */
149 		void *addr;  /* Address of access; unchecked if NULL. */
150 		size_t size; /* Size of access; unchecked if @addr is NULL. */
151 		int type;    /* Access type, see KCSAN_ACCESS definitions. */
152 	} access[2];
153 };
154 
155 /* Check observed report matches information in @r. */
156 __no_kcsan
157 static bool __report_matches(const struct expect_report *r)
158 {
159 	const bool is_assert = (r->access[0].type | r->access[1].type) & KCSAN_ACCESS_ASSERT;
160 	bool ret = false;
161 	unsigned long flags;
162 	typeof(observed.lines) expect;
163 	const char *end;
164 	char *cur;
165 	int i;
166 
167 	/* Doubled-checked locking. */
168 	if (!report_available())
169 		return false;
170 
171 	/* Generate expected report contents. */
172 
173 	/* Title */
174 	cur = expect[0];
175 	end = &expect[0][sizeof(expect[0]) - 1];
176 	cur += scnprintf(cur, end - cur, "BUG: KCSAN: %s in ",
177 			 is_assert ? "assert: race" : "data-race");
178 	if (r->access[1].fn) {
179 		char tmp[2][64];
180 		int cmp;
181 
182 		/* Expect lexographically sorted function names in title. */
183 		scnprintf(tmp[0], sizeof(tmp[0]), "%pS", r->access[0].fn);
184 		scnprintf(tmp[1], sizeof(tmp[1]), "%pS", r->access[1].fn);
185 		cmp = strcmp(tmp[0], tmp[1]);
186 		cur += scnprintf(cur, end - cur, "%ps / %ps",
187 				 cmp < 0 ? r->access[0].fn : r->access[1].fn,
188 				 cmp < 0 ? r->access[1].fn : r->access[0].fn);
189 	} else {
190 		scnprintf(cur, end - cur, "%pS", r->access[0].fn);
191 		/* The exact offset won't match, remove it. */
192 		cur = strchr(expect[0], '+');
193 		if (cur)
194 			*cur = '\0';
195 	}
196 
197 	/* Access 1 */
198 	cur = expect[1];
199 	end = &expect[1][sizeof(expect[1]) - 1];
200 	if (!r->access[1].fn)
201 		cur += scnprintf(cur, end - cur, "race at unknown origin, with ");
202 
203 	/* Access 1 & 2 */
204 	for (i = 0; i < 2; ++i) {
205 		const int ty = r->access[i].type;
206 		const char *const access_type =
207 			(ty & KCSAN_ACCESS_ASSERT) ?
208 				      ((ty & KCSAN_ACCESS_WRITE) ?
209 					       "assert no accesses" :
210 					       "assert no writes") :
211 				      ((ty & KCSAN_ACCESS_WRITE) ?
212 					       ((ty & KCSAN_ACCESS_COMPOUND) ?
213 							"read-write" :
214 							"write") :
215 					       "read");
216 		const bool is_atomic = (ty & KCSAN_ACCESS_ATOMIC);
217 		const bool is_scoped = (ty & KCSAN_ACCESS_SCOPED);
218 		const char *const access_type_aux =
219 				(is_atomic && is_scoped)	? " (marked, reordered)"
220 				: (is_atomic			? " (marked)"
221 				   : (is_scoped			? " (reordered)" : ""));
222 
223 		if (i == 1) {
224 			/* Access 2 */
225 			cur = expect[2];
226 			end = &expect[2][sizeof(expect[2]) - 1];
227 
228 			if (!r->access[1].fn) {
229 				/* Dummy string if no second access is available. */
230 				strcpy(cur, "<none>");
231 				break;
232 			}
233 		}
234 
235 		cur += scnprintf(cur, end - cur, "%s%s to ", access_type,
236 				 access_type_aux);
237 
238 		if (r->access[i].addr) /* Address is optional. */
239 			cur += scnprintf(cur, end - cur, "0x%px of %zu bytes",
240 					 r->access[i].addr, r->access[i].size);
241 	}
242 
243 	spin_lock_irqsave(&observed.lock, flags);
244 	if (!report_available())
245 		goto out; /* A new report is being captured. */
246 
247 	/* Finally match expected output to what we actually observed. */
248 	ret = strstr(observed.lines[0], expect[0]) &&
249 	      /* Access info may appear in any order. */
250 	      ((strstr(observed.lines[1], expect[1]) &&
251 		strstr(observed.lines[2], expect[2])) ||
252 	       (strstr(observed.lines[1], expect[2]) &&
253 		strstr(observed.lines[2], expect[1])));
254 out:
255 	spin_unlock_irqrestore(&observed.lock, flags);
256 	return ret;
257 }
258 
259 static __always_inline const struct expect_report *
260 __report_set_scoped(struct expect_report *r, int accesses)
261 {
262 	BUILD_BUG_ON(accesses > 3);
263 
264 	if (accesses & 1)
265 		r->access[0].type |= KCSAN_ACCESS_SCOPED;
266 	else
267 		r->access[0].type &= ~KCSAN_ACCESS_SCOPED;
268 
269 	if (accesses & 2)
270 		r->access[1].type |= KCSAN_ACCESS_SCOPED;
271 	else
272 		r->access[1].type &= ~KCSAN_ACCESS_SCOPED;
273 
274 	return r;
275 }
276 
277 __no_kcsan
278 static bool report_matches_any_reordered(struct expect_report *r)
279 {
280 	return __report_matches(__report_set_scoped(r, 0)) ||
281 	       __report_matches(__report_set_scoped(r, 1)) ||
282 	       __report_matches(__report_set_scoped(r, 2)) ||
283 	       __report_matches(__report_set_scoped(r, 3));
284 }
285 
286 #ifdef CONFIG_KCSAN_WEAK_MEMORY
287 /* Due to reordering accesses, any access may appear as "(reordered)". */
288 #define report_matches report_matches_any_reordered
289 #else
290 #define report_matches __report_matches
291 #endif
292 
293 /* ===== Test kernels ===== */
294 
295 static long test_sink;
296 static long test_var;
297 /* @test_array should be large enough to fall into multiple watchpoint slots. */
298 static long test_array[3 * PAGE_SIZE / sizeof(long)];
299 static struct {
300 	long val[8];
301 } test_struct;
302 static DEFINE_SEQLOCK(test_seqlock);
303 static DEFINE_SPINLOCK(test_spinlock);
304 static DEFINE_MUTEX(test_mutex);
305 
306 /*
307  * Helper to avoid compiler optimizing out reads, and to generate source values
308  * for writes.
309  */
310 __no_kcsan
311 static noinline void sink_value(long v) { WRITE_ONCE(test_sink, v); }
312 
313 /*
314  * Generates a delay and some accesses that enter the runtime but do not produce
315  * data races.
316  */
317 static noinline void test_delay(int iter)
318 {
319 	while (iter--)
320 		sink_value(READ_ONCE(test_sink));
321 }
322 
323 static noinline void test_kernel_read(void) { sink_value(test_var); }
324 
325 static noinline void test_kernel_write(void)
326 {
327 	test_var = READ_ONCE_NOCHECK(test_sink) + 1;
328 }
329 
330 static noinline void test_kernel_write_nochange(void) { test_var = 42; }
331 
332 /* Suffixed by value-change exception filter. */
333 static noinline void test_kernel_write_nochange_rcu(void) { test_var = 42; }
334 
335 static noinline void test_kernel_read_atomic(void)
336 {
337 	sink_value(READ_ONCE(test_var));
338 }
339 
340 static noinline void test_kernel_write_atomic(void)
341 {
342 	WRITE_ONCE(test_var, READ_ONCE_NOCHECK(test_sink) + 1);
343 }
344 
345 static noinline void test_kernel_atomic_rmw(void)
346 {
347 	/* Use builtin, so we can set up the "bad" atomic/non-atomic scenario. */
348 	__atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED);
349 }
350 
351 __no_kcsan
352 static noinline void test_kernel_write_uninstrumented(void) { test_var++; }
353 
354 static noinline void test_kernel_data_race(void) { data_race(test_var++); }
355 
356 static noinline void test_kernel_assert_writer(void)
357 {
358 	ASSERT_EXCLUSIVE_WRITER(test_var);
359 }
360 
361 static noinline void test_kernel_assert_access(void)
362 {
363 	ASSERT_EXCLUSIVE_ACCESS(test_var);
364 }
365 
366 #define TEST_CHANGE_BITS 0xff00ff00
367 
368 static noinline void test_kernel_change_bits(void)
369 {
370 	if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) {
371 		/*
372 		 * Avoid race of unknown origin for this test, just pretend they
373 		 * are atomic.
374 		 */
375 		kcsan_nestable_atomic_begin();
376 		test_var ^= TEST_CHANGE_BITS;
377 		kcsan_nestable_atomic_end();
378 	} else
379 		WRITE_ONCE(test_var, READ_ONCE(test_var) ^ TEST_CHANGE_BITS);
380 }
381 
382 static noinline void test_kernel_assert_bits_change(void)
383 {
384 	ASSERT_EXCLUSIVE_BITS(test_var, TEST_CHANGE_BITS);
385 }
386 
387 static noinline void test_kernel_assert_bits_nochange(void)
388 {
389 	ASSERT_EXCLUSIVE_BITS(test_var, ~TEST_CHANGE_BITS);
390 }
391 
392 /*
393  * Scoped assertions do trigger anywhere in scope. However, the report should
394  * still only point at the start of the scope.
395  */
396 static noinline void test_enter_scope(void)
397 {
398 	int x = 0;
399 
400 	/* Unrelated accesses to scoped assert. */
401 	READ_ONCE(test_sink);
402 	kcsan_check_read(&x, sizeof(x));
403 }
404 
405 static noinline void test_kernel_assert_writer_scoped(void)
406 {
407 	ASSERT_EXCLUSIVE_WRITER_SCOPED(test_var);
408 	test_enter_scope();
409 }
410 
411 static noinline void test_kernel_assert_access_scoped(void)
412 {
413 	ASSERT_EXCLUSIVE_ACCESS_SCOPED(test_var);
414 	test_enter_scope();
415 }
416 
417 static noinline void test_kernel_rmw_array(void)
418 {
419 	int i;
420 
421 	for (i = 0; i < ARRAY_SIZE(test_array); ++i)
422 		test_array[i]++;
423 }
424 
425 static noinline void test_kernel_write_struct(void)
426 {
427 	kcsan_check_write(&test_struct, sizeof(test_struct));
428 	kcsan_disable_current();
429 	test_struct.val[3]++; /* induce value change */
430 	kcsan_enable_current();
431 }
432 
433 static noinline void test_kernel_write_struct_part(void)
434 {
435 	test_struct.val[3] = 42;
436 }
437 
438 static noinline void test_kernel_read_struct_zero_size(void)
439 {
440 	kcsan_check_read(&test_struct.val[3], 0);
441 }
442 
443 static noinline void test_kernel_jiffies_reader(void)
444 {
445 	sink_value((long)jiffies);
446 }
447 
448 static noinline void test_kernel_seqlock_reader(void)
449 {
450 	unsigned int seq;
451 
452 	do {
453 		seq = read_seqbegin(&test_seqlock);
454 		sink_value(test_var);
455 	} while (read_seqretry(&test_seqlock, seq));
456 }
457 
458 static noinline void test_kernel_seqlock_writer(void)
459 {
460 	unsigned long flags;
461 
462 	write_seqlock_irqsave(&test_seqlock, flags);
463 	test_var++;
464 	write_sequnlock_irqrestore(&test_seqlock, flags);
465 }
466 
467 static noinline void test_kernel_atomic_builtins(void)
468 {
469 	/*
470 	 * Generate concurrent accesses, expecting no reports, ensuring KCSAN
471 	 * treats builtin atomics as actually atomic.
472 	 */
473 	__atomic_load_n(&test_var, __ATOMIC_RELAXED);
474 }
475 
476 static noinline void test_kernel_xor_1bit(void)
477 {
478 	/* Do not report data races between the read-writes. */
479 	kcsan_nestable_atomic_begin();
480 	test_var ^= 0x10000;
481 	kcsan_nestable_atomic_end();
482 }
483 
484 #define TEST_KERNEL_LOCKED(name, acquire, release)		\
485 	static noinline void test_kernel_##name(void)		\
486 	{							\
487 		long *flag = &test_struct.val[0];		\
488 		long v = 0;					\
489 		if (!(acquire))					\
490 			return;					\
491 		while (v++ < 100) {				\
492 			test_var++;				\
493 			barrier();				\
494 		}						\
495 		release;					\
496 		test_delay(10);					\
497 	}
498 
499 TEST_KERNEL_LOCKED(with_memorder,
500 		   cmpxchg_acquire(flag, 0, 1) == 0,
501 		   smp_store_release(flag, 0));
502 TEST_KERNEL_LOCKED(wrong_memorder,
503 		   cmpxchg_relaxed(flag, 0, 1) == 0,
504 		   WRITE_ONCE(*flag, 0));
505 TEST_KERNEL_LOCKED(atomic_builtin_with_memorder,
506 		   __atomic_compare_exchange_n(flag, &v, 1, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED),
507 		   __atomic_store_n(flag, 0, __ATOMIC_RELEASE));
508 TEST_KERNEL_LOCKED(atomic_builtin_wrong_memorder,
509 		   __atomic_compare_exchange_n(flag, &v, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED),
510 		   __atomic_store_n(flag, 0, __ATOMIC_RELAXED));
511 
512 /* ===== Test cases ===== */
513 
514 /*
515  * Tests that various barriers have the expected effect on internal state. Not
516  * exhaustive on atomic_t operations. Unlike the selftest, also checks for
517  * too-strict barrier instrumentation; these can be tolerated, because it does
518  * not cause false positives, but at least we should be aware of such cases.
519  */
520 static void test_barrier_nothreads(struct kunit *test)
521 {
522 #ifdef CONFIG_KCSAN_WEAK_MEMORY
523 	struct kcsan_scoped_access *reorder_access = &current->kcsan_ctx.reorder_access;
524 #else
525 	struct kcsan_scoped_access *reorder_access = NULL;
526 #endif
527 	arch_spinlock_t arch_spinlock = __ARCH_SPIN_LOCK_UNLOCKED;
528 	atomic_t dummy;
529 
530 	KCSAN_TEST_REQUIRES(test, reorder_access != NULL);
531 	KCSAN_TEST_REQUIRES(test, IS_ENABLED(CONFIG_SMP));
532 
533 #define __KCSAN_EXPECT_BARRIER(access_type, barrier, order_before, name)			\
534 	do {											\
535 		reorder_access->type = (access_type) | KCSAN_ACCESS_SCOPED;			\
536 		reorder_access->size = sizeof(test_var);					\
537 		barrier;									\
538 		KUNIT_EXPECT_EQ_MSG(test, reorder_access->size,					\
539 				    order_before ? 0 : sizeof(test_var),			\
540 				    "improperly instrumented type=(" #access_type "): " name);	\
541 	} while (0)
542 #define KCSAN_EXPECT_READ_BARRIER(b, o)  __KCSAN_EXPECT_BARRIER(0, b, o, #b)
543 #define KCSAN_EXPECT_WRITE_BARRIER(b, o) __KCSAN_EXPECT_BARRIER(KCSAN_ACCESS_WRITE, b, o, #b)
544 #define KCSAN_EXPECT_RW_BARRIER(b, o)    __KCSAN_EXPECT_BARRIER(KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE, b, o, #b)
545 
546 	/*
547 	 * Lockdep initialization can strengthen certain locking operations due
548 	 * to calling into instrumented files; "warm up" our locks.
549 	 */
550 	spin_lock(&test_spinlock);
551 	spin_unlock(&test_spinlock);
552 	mutex_lock(&test_mutex);
553 	mutex_unlock(&test_mutex);
554 
555 	/* Force creating a valid entry in reorder_access first. */
556 	test_var = 0;
557 	while (test_var++ < 1000000 && reorder_access->size != sizeof(test_var))
558 		__kcsan_check_read(&test_var, sizeof(test_var));
559 	KUNIT_ASSERT_EQ(test, reorder_access->size, sizeof(test_var));
560 
561 	kcsan_nestable_atomic_begin(); /* No watchpoints in called functions. */
562 
563 	KCSAN_EXPECT_READ_BARRIER(mb(), true);
564 	KCSAN_EXPECT_READ_BARRIER(wmb(), false);
565 	KCSAN_EXPECT_READ_BARRIER(rmb(), true);
566 	KCSAN_EXPECT_READ_BARRIER(smp_mb(), true);
567 	KCSAN_EXPECT_READ_BARRIER(smp_wmb(), false);
568 	KCSAN_EXPECT_READ_BARRIER(smp_rmb(), true);
569 	KCSAN_EXPECT_READ_BARRIER(dma_wmb(), false);
570 	KCSAN_EXPECT_READ_BARRIER(dma_rmb(), true);
571 	KCSAN_EXPECT_READ_BARRIER(smp_mb__before_atomic(), true);
572 	KCSAN_EXPECT_READ_BARRIER(smp_mb__after_atomic(), true);
573 	KCSAN_EXPECT_READ_BARRIER(smp_mb__after_spinlock(), true);
574 	KCSAN_EXPECT_READ_BARRIER(smp_store_mb(test_var, 0), true);
575 	KCSAN_EXPECT_READ_BARRIER(smp_load_acquire(&test_var), false);
576 	KCSAN_EXPECT_READ_BARRIER(smp_store_release(&test_var, 0), true);
577 	KCSAN_EXPECT_READ_BARRIER(xchg(&test_var, 0), true);
578 	KCSAN_EXPECT_READ_BARRIER(xchg_release(&test_var, 0), true);
579 	KCSAN_EXPECT_READ_BARRIER(xchg_relaxed(&test_var, 0), false);
580 	KCSAN_EXPECT_READ_BARRIER(cmpxchg(&test_var, 0,  0), true);
581 	KCSAN_EXPECT_READ_BARRIER(cmpxchg_release(&test_var, 0,  0), true);
582 	KCSAN_EXPECT_READ_BARRIER(cmpxchg_relaxed(&test_var, 0,  0), false);
583 	KCSAN_EXPECT_READ_BARRIER(atomic_read(&dummy), false);
584 	KCSAN_EXPECT_READ_BARRIER(atomic_read_acquire(&dummy), false);
585 	KCSAN_EXPECT_READ_BARRIER(atomic_set(&dummy, 0), false);
586 	KCSAN_EXPECT_READ_BARRIER(atomic_set_release(&dummy, 0), true);
587 	KCSAN_EXPECT_READ_BARRIER(atomic_add(1, &dummy), false);
588 	KCSAN_EXPECT_READ_BARRIER(atomic_add_return(1, &dummy), true);
589 	KCSAN_EXPECT_READ_BARRIER(atomic_add_return_acquire(1, &dummy), false);
590 	KCSAN_EXPECT_READ_BARRIER(atomic_add_return_release(1, &dummy), true);
591 	KCSAN_EXPECT_READ_BARRIER(atomic_add_return_relaxed(1, &dummy), false);
592 	KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add(1, &dummy), true);
593 	KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_acquire(1, &dummy), false);
594 	KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_release(1, &dummy), true);
595 	KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false);
596 	KCSAN_EXPECT_READ_BARRIER(test_and_set_bit(0, &test_var), true);
597 	KCSAN_EXPECT_READ_BARRIER(test_and_clear_bit(0, &test_var), true);
598 	KCSAN_EXPECT_READ_BARRIER(test_and_change_bit(0, &test_var), true);
599 	KCSAN_EXPECT_READ_BARRIER(clear_bit_unlock(0, &test_var), true);
600 	KCSAN_EXPECT_READ_BARRIER(__clear_bit_unlock(0, &test_var), true);
601 	KCSAN_EXPECT_READ_BARRIER(arch_spin_lock(&arch_spinlock), false);
602 	KCSAN_EXPECT_READ_BARRIER(arch_spin_unlock(&arch_spinlock), true);
603 	KCSAN_EXPECT_READ_BARRIER(spin_lock(&test_spinlock), false);
604 	KCSAN_EXPECT_READ_BARRIER(spin_unlock(&test_spinlock), true);
605 	KCSAN_EXPECT_READ_BARRIER(mutex_lock(&test_mutex), false);
606 	KCSAN_EXPECT_READ_BARRIER(mutex_unlock(&test_mutex), true);
607 
608 	KCSAN_EXPECT_WRITE_BARRIER(mb(), true);
609 	KCSAN_EXPECT_WRITE_BARRIER(wmb(), true);
610 	KCSAN_EXPECT_WRITE_BARRIER(rmb(), false);
611 	KCSAN_EXPECT_WRITE_BARRIER(smp_mb(), true);
612 	KCSAN_EXPECT_WRITE_BARRIER(smp_wmb(), true);
613 	KCSAN_EXPECT_WRITE_BARRIER(smp_rmb(), false);
614 	KCSAN_EXPECT_WRITE_BARRIER(dma_wmb(), true);
615 	KCSAN_EXPECT_WRITE_BARRIER(dma_rmb(), false);
616 	KCSAN_EXPECT_WRITE_BARRIER(smp_mb__before_atomic(), true);
617 	KCSAN_EXPECT_WRITE_BARRIER(smp_mb__after_atomic(), true);
618 	KCSAN_EXPECT_WRITE_BARRIER(smp_mb__after_spinlock(), true);
619 	KCSAN_EXPECT_WRITE_BARRIER(smp_store_mb(test_var, 0), true);
620 	KCSAN_EXPECT_WRITE_BARRIER(smp_load_acquire(&test_var), false);
621 	KCSAN_EXPECT_WRITE_BARRIER(smp_store_release(&test_var, 0), true);
622 	KCSAN_EXPECT_WRITE_BARRIER(xchg(&test_var, 0), true);
623 	KCSAN_EXPECT_WRITE_BARRIER(xchg_release(&test_var, 0), true);
624 	KCSAN_EXPECT_WRITE_BARRIER(xchg_relaxed(&test_var, 0), false);
625 	KCSAN_EXPECT_WRITE_BARRIER(cmpxchg(&test_var, 0,  0), true);
626 	KCSAN_EXPECT_WRITE_BARRIER(cmpxchg_release(&test_var, 0,  0), true);
627 	KCSAN_EXPECT_WRITE_BARRIER(cmpxchg_relaxed(&test_var, 0,  0), false);
628 	KCSAN_EXPECT_WRITE_BARRIER(atomic_read(&dummy), false);
629 	KCSAN_EXPECT_WRITE_BARRIER(atomic_read_acquire(&dummy), false);
630 	KCSAN_EXPECT_WRITE_BARRIER(atomic_set(&dummy, 0), false);
631 	KCSAN_EXPECT_WRITE_BARRIER(atomic_set_release(&dummy, 0), true);
632 	KCSAN_EXPECT_WRITE_BARRIER(atomic_add(1, &dummy), false);
633 	KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return(1, &dummy), true);
634 	KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_acquire(1, &dummy), false);
635 	KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_release(1, &dummy), true);
636 	KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_relaxed(1, &dummy), false);
637 	KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add(1, &dummy), true);
638 	KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_acquire(1, &dummy), false);
639 	KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_release(1, &dummy), true);
640 	KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false);
641 	KCSAN_EXPECT_WRITE_BARRIER(test_and_set_bit(0, &test_var), true);
642 	KCSAN_EXPECT_WRITE_BARRIER(test_and_clear_bit(0, &test_var), true);
643 	KCSAN_EXPECT_WRITE_BARRIER(test_and_change_bit(0, &test_var), true);
644 	KCSAN_EXPECT_WRITE_BARRIER(clear_bit_unlock(0, &test_var), true);
645 	KCSAN_EXPECT_WRITE_BARRIER(__clear_bit_unlock(0, &test_var), true);
646 	KCSAN_EXPECT_WRITE_BARRIER(arch_spin_lock(&arch_spinlock), false);
647 	KCSAN_EXPECT_WRITE_BARRIER(arch_spin_unlock(&arch_spinlock), true);
648 	KCSAN_EXPECT_WRITE_BARRIER(spin_lock(&test_spinlock), false);
649 	KCSAN_EXPECT_WRITE_BARRIER(spin_unlock(&test_spinlock), true);
650 	KCSAN_EXPECT_WRITE_BARRIER(mutex_lock(&test_mutex), false);
651 	KCSAN_EXPECT_WRITE_BARRIER(mutex_unlock(&test_mutex), true);
652 
653 	KCSAN_EXPECT_RW_BARRIER(mb(), true);
654 	KCSAN_EXPECT_RW_BARRIER(wmb(), true);
655 	KCSAN_EXPECT_RW_BARRIER(rmb(), true);
656 	KCSAN_EXPECT_RW_BARRIER(smp_mb(), true);
657 	KCSAN_EXPECT_RW_BARRIER(smp_wmb(), true);
658 	KCSAN_EXPECT_RW_BARRIER(smp_rmb(), true);
659 	KCSAN_EXPECT_RW_BARRIER(dma_wmb(), true);
660 	KCSAN_EXPECT_RW_BARRIER(dma_rmb(), true);
661 	KCSAN_EXPECT_RW_BARRIER(smp_mb__before_atomic(), true);
662 	KCSAN_EXPECT_RW_BARRIER(smp_mb__after_atomic(), true);
663 	KCSAN_EXPECT_RW_BARRIER(smp_mb__after_spinlock(), true);
664 	KCSAN_EXPECT_RW_BARRIER(smp_store_mb(test_var, 0), true);
665 	KCSAN_EXPECT_RW_BARRIER(smp_load_acquire(&test_var), false);
666 	KCSAN_EXPECT_RW_BARRIER(smp_store_release(&test_var, 0), true);
667 	KCSAN_EXPECT_RW_BARRIER(xchg(&test_var, 0), true);
668 	KCSAN_EXPECT_RW_BARRIER(xchg_release(&test_var, 0), true);
669 	KCSAN_EXPECT_RW_BARRIER(xchg_relaxed(&test_var, 0), false);
670 	KCSAN_EXPECT_RW_BARRIER(cmpxchg(&test_var, 0,  0), true);
671 	KCSAN_EXPECT_RW_BARRIER(cmpxchg_release(&test_var, 0,  0), true);
672 	KCSAN_EXPECT_RW_BARRIER(cmpxchg_relaxed(&test_var, 0,  0), false);
673 	KCSAN_EXPECT_RW_BARRIER(atomic_read(&dummy), false);
674 	KCSAN_EXPECT_RW_BARRIER(atomic_read_acquire(&dummy), false);
675 	KCSAN_EXPECT_RW_BARRIER(atomic_set(&dummy, 0), false);
676 	KCSAN_EXPECT_RW_BARRIER(atomic_set_release(&dummy, 0), true);
677 	KCSAN_EXPECT_RW_BARRIER(atomic_add(1, &dummy), false);
678 	KCSAN_EXPECT_RW_BARRIER(atomic_add_return(1, &dummy), true);
679 	KCSAN_EXPECT_RW_BARRIER(atomic_add_return_acquire(1, &dummy), false);
680 	KCSAN_EXPECT_RW_BARRIER(atomic_add_return_release(1, &dummy), true);
681 	KCSAN_EXPECT_RW_BARRIER(atomic_add_return_relaxed(1, &dummy), false);
682 	KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add(1, &dummy), true);
683 	KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_acquire(1, &dummy), false);
684 	KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_release(1, &dummy), true);
685 	KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false);
686 	KCSAN_EXPECT_RW_BARRIER(test_and_set_bit(0, &test_var), true);
687 	KCSAN_EXPECT_RW_BARRIER(test_and_clear_bit(0, &test_var), true);
688 	KCSAN_EXPECT_RW_BARRIER(test_and_change_bit(0, &test_var), true);
689 	KCSAN_EXPECT_RW_BARRIER(clear_bit_unlock(0, &test_var), true);
690 	KCSAN_EXPECT_RW_BARRIER(__clear_bit_unlock(0, &test_var), true);
691 	KCSAN_EXPECT_RW_BARRIER(arch_spin_lock(&arch_spinlock), false);
692 	KCSAN_EXPECT_RW_BARRIER(arch_spin_unlock(&arch_spinlock), true);
693 	KCSAN_EXPECT_RW_BARRIER(spin_lock(&test_spinlock), false);
694 	KCSAN_EXPECT_RW_BARRIER(spin_unlock(&test_spinlock), true);
695 	KCSAN_EXPECT_RW_BARRIER(mutex_lock(&test_mutex), false);
696 	KCSAN_EXPECT_RW_BARRIER(mutex_unlock(&test_mutex), true);
697 
698 #ifdef clear_bit_unlock_is_negative_byte
699 	KCSAN_EXPECT_READ_BARRIER(clear_bit_unlock_is_negative_byte(0, &test_var), true);
700 	KCSAN_EXPECT_WRITE_BARRIER(clear_bit_unlock_is_negative_byte(0, &test_var), true);
701 	KCSAN_EXPECT_RW_BARRIER(clear_bit_unlock_is_negative_byte(0, &test_var), true);
702 #endif
703 	kcsan_nestable_atomic_end();
704 }
705 
706 /* Simple test with normal data race. */
707 __no_kcsan
708 static void test_basic(struct kunit *test)
709 {
710 	struct expect_report expect = {
711 		.access = {
712 			{ test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
713 			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
714 		},
715 	};
716 	struct expect_report never = {
717 		.access = {
718 			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
719 			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
720 		},
721 	};
722 	bool match_expect = false;
723 	bool match_never = false;
724 
725 	begin_test_checks(test_kernel_write, test_kernel_read);
726 	do {
727 		match_expect |= report_matches(&expect);
728 		match_never = report_matches(&never);
729 	} while (!end_test_checks(match_never));
730 	KUNIT_EXPECT_TRUE(test, match_expect);
731 	KUNIT_EXPECT_FALSE(test, match_never);
732 }
733 
734 /*
735  * Stress KCSAN with lots of concurrent races on different addresses until
736  * timeout.
737  */
738 __no_kcsan
739 static void test_concurrent_races(struct kunit *test)
740 {
741 	struct expect_report expect = {
742 		.access = {
743 			/* NULL will match any address. */
744 			{ test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
745 			{ test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(0) },
746 		},
747 	};
748 	struct expect_report never = {
749 		.access = {
750 			{ test_kernel_rmw_array, NULL, 0, 0 },
751 			{ test_kernel_rmw_array, NULL, 0, 0 },
752 		},
753 	};
754 	bool match_expect = false;
755 	bool match_never = false;
756 
757 	begin_test_checks(test_kernel_rmw_array, test_kernel_rmw_array);
758 	do {
759 		match_expect |= report_matches(&expect);
760 		match_never |= report_matches(&never);
761 	} while (!end_test_checks(false));
762 	KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check matches exist. */
763 	KUNIT_EXPECT_FALSE(test, match_never);
764 }
765 
766 /* Test the KCSAN_REPORT_VALUE_CHANGE_ONLY option. */
767 __no_kcsan
768 static void test_novalue_change(struct kunit *test)
769 {
770 	struct expect_report expect_rw = {
771 		.access = {
772 			{ test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
773 			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
774 		},
775 	};
776 	struct expect_report expect_ww = {
777 		.access = {
778 			{ test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
779 			{ test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
780 		},
781 	};
782 	bool match_expect = false;
783 
784 	test_kernel_write_nochange(); /* Reset value. */
785 	begin_test_checks(test_kernel_write_nochange, test_kernel_read);
786 	do {
787 		match_expect = report_matches(&expect_rw) || report_matches(&expect_ww);
788 	} while (!end_test_checks(match_expect));
789 	if (IS_ENABLED(CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY))
790 		KUNIT_EXPECT_FALSE(test, match_expect);
791 	else
792 		KUNIT_EXPECT_TRUE(test, match_expect);
793 }
794 
795 /*
796  * Test that the rules where the KCSAN_REPORT_VALUE_CHANGE_ONLY option should
797  * never apply work.
798  */
799 __no_kcsan
800 static void test_novalue_change_exception(struct kunit *test)
801 {
802 	struct expect_report expect_rw = {
803 		.access = {
804 			{ test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
805 			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
806 		},
807 	};
808 	struct expect_report expect_ww = {
809 		.access = {
810 			{ test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
811 			{ test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
812 		},
813 	};
814 	bool match_expect = false;
815 
816 	test_kernel_write_nochange_rcu(); /* Reset value. */
817 	begin_test_checks(test_kernel_write_nochange_rcu, test_kernel_read);
818 	do {
819 		match_expect = report_matches(&expect_rw) || report_matches(&expect_ww);
820 	} while (!end_test_checks(match_expect));
821 	KUNIT_EXPECT_TRUE(test, match_expect);
822 }
823 
824 /* Test that data races of unknown origin are reported. */
825 __no_kcsan
826 static void test_unknown_origin(struct kunit *test)
827 {
828 	struct expect_report expect = {
829 		.access = {
830 			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
831 			{ NULL },
832 		},
833 	};
834 	bool match_expect = false;
835 
836 	begin_test_checks(test_kernel_write_uninstrumented, test_kernel_read);
837 	do {
838 		match_expect = report_matches(&expect);
839 	} while (!end_test_checks(match_expect));
840 	if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN))
841 		KUNIT_EXPECT_TRUE(test, match_expect);
842 	else
843 		KUNIT_EXPECT_FALSE(test, match_expect);
844 }
845 
846 /* Test KCSAN_ASSUME_PLAIN_WRITES_ATOMIC if it is selected. */
847 __no_kcsan
848 static void test_write_write_assume_atomic(struct kunit *test)
849 {
850 	struct expect_report expect = {
851 		.access = {
852 			{ test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
853 			{ test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
854 		},
855 	};
856 	bool match_expect = false;
857 
858 	begin_test_checks(test_kernel_write, test_kernel_write);
859 	do {
860 		sink_value(READ_ONCE(test_var)); /* induce value-change */
861 		match_expect = report_matches(&expect);
862 	} while (!end_test_checks(match_expect));
863 	if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC))
864 		KUNIT_EXPECT_FALSE(test, match_expect);
865 	else
866 		KUNIT_EXPECT_TRUE(test, match_expect);
867 }
868 
869 /*
870  * Test that data races with writes larger than word-size are always reported,
871  * even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected.
872  */
873 __no_kcsan
874 static void test_write_write_struct(struct kunit *test)
875 {
876 	struct expect_report expect = {
877 		.access = {
878 			{ test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
879 			{ test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
880 		},
881 	};
882 	bool match_expect = false;
883 
884 	begin_test_checks(test_kernel_write_struct, test_kernel_write_struct);
885 	do {
886 		match_expect = report_matches(&expect);
887 	} while (!end_test_checks(match_expect));
888 	KUNIT_EXPECT_TRUE(test, match_expect);
889 }
890 
891 /*
892  * Test that data races where only one write is larger than word-size are always
893  * reported, even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected.
894  */
895 __no_kcsan
896 static void test_write_write_struct_part(struct kunit *test)
897 {
898 	struct expect_report expect = {
899 		.access = {
900 			{ test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
901 			{ test_kernel_write_struct_part, &test_struct.val[3], sizeof(test_struct.val[3]), KCSAN_ACCESS_WRITE },
902 		},
903 	};
904 	bool match_expect = false;
905 
906 	begin_test_checks(test_kernel_write_struct, test_kernel_write_struct_part);
907 	do {
908 		match_expect = report_matches(&expect);
909 	} while (!end_test_checks(match_expect));
910 	KUNIT_EXPECT_TRUE(test, match_expect);
911 }
912 
913 /* Test that races with atomic accesses never result in reports. */
914 __no_kcsan
915 static void test_read_atomic_write_atomic(struct kunit *test)
916 {
917 	bool match_never = false;
918 
919 	begin_test_checks(test_kernel_read_atomic, test_kernel_write_atomic);
920 	do {
921 		match_never = report_available();
922 	} while (!end_test_checks(match_never));
923 	KUNIT_EXPECT_FALSE(test, match_never);
924 }
925 
926 /* Test that a race with an atomic and plain access result in reports. */
927 __no_kcsan
928 static void test_read_plain_atomic_write(struct kunit *test)
929 {
930 	struct expect_report expect = {
931 		.access = {
932 			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
933 			{ test_kernel_write_atomic, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC },
934 		},
935 	};
936 	bool match_expect = false;
937 
938 	KCSAN_TEST_REQUIRES(test, !IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS));
939 
940 	begin_test_checks(test_kernel_read, test_kernel_write_atomic);
941 	do {
942 		match_expect = report_matches(&expect);
943 	} while (!end_test_checks(match_expect));
944 	KUNIT_EXPECT_TRUE(test, match_expect);
945 }
946 
947 /* Test that atomic RMWs generate correct report. */
948 __no_kcsan
949 static void test_read_plain_atomic_rmw(struct kunit *test)
950 {
951 	struct expect_report expect = {
952 		.access = {
953 			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
954 			{ test_kernel_atomic_rmw, &test_var, sizeof(test_var),
955 				KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC },
956 		},
957 	};
958 	bool match_expect = false;
959 
960 	KCSAN_TEST_REQUIRES(test, !IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS));
961 
962 	begin_test_checks(test_kernel_read, test_kernel_atomic_rmw);
963 	do {
964 		match_expect = report_matches(&expect);
965 	} while (!end_test_checks(match_expect));
966 	KUNIT_EXPECT_TRUE(test, match_expect);
967 }
968 
969 /* Zero-sized accesses should never cause data race reports. */
970 __no_kcsan
971 static void test_zero_size_access(struct kunit *test)
972 {
973 	struct expect_report expect = {
974 		.access = {
975 			{ test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
976 			{ test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
977 		},
978 	};
979 	struct expect_report never = {
980 		.access = {
981 			{ test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
982 			{ test_kernel_read_struct_zero_size, &test_struct.val[3], 0, 0 },
983 		},
984 	};
985 	bool match_expect = false;
986 	bool match_never = false;
987 
988 	begin_test_checks(test_kernel_write_struct, test_kernel_read_struct_zero_size);
989 	do {
990 		match_expect |= report_matches(&expect);
991 		match_never = report_matches(&never);
992 	} while (!end_test_checks(match_never));
993 	KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check. */
994 	KUNIT_EXPECT_FALSE(test, match_never);
995 }
996 
997 /* Test the data_race() macro. */
998 __no_kcsan
999 static void test_data_race(struct kunit *test)
1000 {
1001 	bool match_never = false;
1002 
1003 	begin_test_checks(test_kernel_data_race, test_kernel_data_race);
1004 	do {
1005 		match_never = report_available();
1006 	} while (!end_test_checks(match_never));
1007 	KUNIT_EXPECT_FALSE(test, match_never);
1008 }
1009 
1010 __no_kcsan
1011 static void test_assert_exclusive_writer(struct kunit *test)
1012 {
1013 	struct expect_report expect = {
1014 		.access = {
1015 			{ test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
1016 			{ test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
1017 		},
1018 	};
1019 	bool match_expect = false;
1020 
1021 	begin_test_checks(test_kernel_assert_writer, test_kernel_write_nochange);
1022 	do {
1023 		match_expect = report_matches(&expect);
1024 	} while (!end_test_checks(match_expect));
1025 	KUNIT_EXPECT_TRUE(test, match_expect);
1026 }
1027 
1028 __no_kcsan
1029 static void test_assert_exclusive_access(struct kunit *test)
1030 {
1031 	struct expect_report expect = {
1032 		.access = {
1033 			{ test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
1034 			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
1035 		},
1036 	};
1037 	bool match_expect = false;
1038 
1039 	begin_test_checks(test_kernel_assert_access, test_kernel_read);
1040 	do {
1041 		match_expect = report_matches(&expect);
1042 	} while (!end_test_checks(match_expect));
1043 	KUNIT_EXPECT_TRUE(test, match_expect);
1044 }
1045 
1046 __no_kcsan
1047 static void test_assert_exclusive_access_writer(struct kunit *test)
1048 {
1049 	struct expect_report expect_access_writer = {
1050 		.access = {
1051 			{ test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
1052 			{ test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
1053 		},
1054 	};
1055 	struct expect_report expect_access_access = {
1056 		.access = {
1057 			{ test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
1058 			{ test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
1059 		},
1060 	};
1061 	struct expect_report never = {
1062 		.access = {
1063 			{ test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
1064 			{ test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
1065 		},
1066 	};
1067 	bool match_expect_access_writer = false;
1068 	bool match_expect_access_access = false;
1069 	bool match_never = false;
1070 
1071 	begin_test_checks(test_kernel_assert_access, test_kernel_assert_writer);
1072 	do {
1073 		match_expect_access_writer |= report_matches(&expect_access_writer);
1074 		match_expect_access_access |= report_matches(&expect_access_access);
1075 		match_never |= report_matches(&never);
1076 	} while (!end_test_checks(match_never));
1077 	KUNIT_EXPECT_TRUE(test, match_expect_access_writer);
1078 	KUNIT_EXPECT_TRUE(test, match_expect_access_access);
1079 	KUNIT_EXPECT_FALSE(test, match_never);
1080 }
1081 
1082 __no_kcsan
1083 static void test_assert_exclusive_bits_change(struct kunit *test)
1084 {
1085 	struct expect_report expect = {
1086 		.access = {
1087 			{ test_kernel_assert_bits_change, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
1088 			{ test_kernel_change_bits, &test_var, sizeof(test_var),
1089 				KCSAN_ACCESS_WRITE | (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) ? 0 : KCSAN_ACCESS_ATOMIC) },
1090 		},
1091 	};
1092 	bool match_expect = false;
1093 
1094 	begin_test_checks(test_kernel_assert_bits_change, test_kernel_change_bits);
1095 	do {
1096 		match_expect = report_matches(&expect);
1097 	} while (!end_test_checks(match_expect));
1098 	KUNIT_EXPECT_TRUE(test, match_expect);
1099 }
1100 
1101 __no_kcsan
1102 static void test_assert_exclusive_bits_nochange(struct kunit *test)
1103 {
1104 	bool match_never = false;
1105 
1106 	begin_test_checks(test_kernel_assert_bits_nochange, test_kernel_change_bits);
1107 	do {
1108 		match_never = report_available();
1109 	} while (!end_test_checks(match_never));
1110 	KUNIT_EXPECT_FALSE(test, match_never);
1111 }
1112 
1113 __no_kcsan
1114 static void test_assert_exclusive_writer_scoped(struct kunit *test)
1115 {
1116 	struct expect_report expect_start = {
1117 		.access = {
1118 			{ test_kernel_assert_writer_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED },
1119 			{ test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
1120 		},
1121 	};
1122 	struct expect_report expect_inscope = {
1123 		.access = {
1124 			{ test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED },
1125 			{ test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
1126 		},
1127 	};
1128 	bool match_expect_start = false;
1129 	bool match_expect_inscope = false;
1130 
1131 	begin_test_checks(test_kernel_assert_writer_scoped, test_kernel_write_nochange);
1132 	do {
1133 		match_expect_start |= report_matches(&expect_start);
1134 		match_expect_inscope |= report_matches(&expect_inscope);
1135 	} while (!end_test_checks(match_expect_inscope));
1136 	KUNIT_EXPECT_TRUE(test, match_expect_start);
1137 	KUNIT_EXPECT_FALSE(test, match_expect_inscope);
1138 }
1139 
1140 __no_kcsan
1141 static void test_assert_exclusive_access_scoped(struct kunit *test)
1142 {
1143 	struct expect_report expect_start1 = {
1144 		.access = {
1145 			{ test_kernel_assert_access_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED },
1146 			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
1147 		},
1148 	};
1149 	struct expect_report expect_start2 = {
1150 		.access = { expect_start1.access[0], expect_start1.access[0] },
1151 	};
1152 	struct expect_report expect_inscope = {
1153 		.access = {
1154 			{ test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED },
1155 			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
1156 		},
1157 	};
1158 	bool match_expect_start = false;
1159 	bool match_expect_inscope = false;
1160 
1161 	begin_test_checks(test_kernel_assert_access_scoped, test_kernel_read);
1162 	end_time += msecs_to_jiffies(1000); /* This test requires a bit more time. */
1163 	do {
1164 		match_expect_start |= report_matches(&expect_start1) || report_matches(&expect_start2);
1165 		match_expect_inscope |= report_matches(&expect_inscope);
1166 	} while (!end_test_checks(match_expect_inscope));
1167 	KUNIT_EXPECT_TRUE(test, match_expect_start);
1168 	KUNIT_EXPECT_FALSE(test, match_expect_inscope);
1169 }
1170 
1171 /*
1172  * jiffies is special (declared to be volatile) and its accesses are typically
1173  * not marked; this test ensures that the compiler nor KCSAN gets confused about
1174  * jiffies's declaration on different architectures.
1175  */
1176 __no_kcsan
1177 static void test_jiffies_noreport(struct kunit *test)
1178 {
1179 	bool match_never = false;
1180 
1181 	begin_test_checks(test_kernel_jiffies_reader, test_kernel_jiffies_reader);
1182 	do {
1183 		match_never = report_available();
1184 	} while (!end_test_checks(match_never));
1185 	KUNIT_EXPECT_FALSE(test, match_never);
1186 }
1187 
1188 /* Test that racing accesses in seqlock critical sections are not reported. */
1189 __no_kcsan
1190 static void test_seqlock_noreport(struct kunit *test)
1191 {
1192 	bool match_never = false;
1193 
1194 	begin_test_checks(test_kernel_seqlock_reader, test_kernel_seqlock_writer);
1195 	do {
1196 		match_never = report_available();
1197 	} while (!end_test_checks(match_never));
1198 	KUNIT_EXPECT_FALSE(test, match_never);
1199 }
1200 
1201 /*
1202  * Test atomic builtins work and required instrumentation functions exist. We
1203  * also test that KCSAN understands they're atomic by racing with them via
1204  * test_kernel_atomic_builtins(), and expect no reports.
1205  *
1206  * The atomic builtins _SHOULD NOT_ be used in normal kernel code!
1207  */
1208 static void test_atomic_builtins(struct kunit *test)
1209 {
1210 	bool match_never = false;
1211 
1212 	begin_test_checks(test_kernel_atomic_builtins, test_kernel_atomic_builtins);
1213 	do {
1214 		long tmp;
1215 
1216 		kcsan_enable_current();
1217 
1218 		__atomic_store_n(&test_var, 42L, __ATOMIC_RELAXED);
1219 		KUNIT_EXPECT_EQ(test, 42L, __atomic_load_n(&test_var, __ATOMIC_RELAXED));
1220 
1221 		KUNIT_EXPECT_EQ(test, 42L, __atomic_exchange_n(&test_var, 20, __ATOMIC_RELAXED));
1222 		KUNIT_EXPECT_EQ(test, 20L, test_var);
1223 
1224 		tmp = 20L;
1225 		KUNIT_EXPECT_TRUE(test, __atomic_compare_exchange_n(&test_var, &tmp, 30L,
1226 								    0, __ATOMIC_RELAXED,
1227 								    __ATOMIC_RELAXED));
1228 		KUNIT_EXPECT_EQ(test, tmp, 20L);
1229 		KUNIT_EXPECT_EQ(test, test_var, 30L);
1230 		KUNIT_EXPECT_FALSE(test, __atomic_compare_exchange_n(&test_var, &tmp, 40L,
1231 								     1, __ATOMIC_RELAXED,
1232 								     __ATOMIC_RELAXED));
1233 		KUNIT_EXPECT_EQ(test, tmp, 30L);
1234 		KUNIT_EXPECT_EQ(test, test_var, 30L);
1235 
1236 		KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED));
1237 		KUNIT_EXPECT_EQ(test, 31L, __atomic_fetch_sub(&test_var, 1, __ATOMIC_RELAXED));
1238 		KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_and(&test_var, 0xf, __ATOMIC_RELAXED));
1239 		KUNIT_EXPECT_EQ(test, 14L, __atomic_fetch_xor(&test_var, 0xf, __ATOMIC_RELAXED));
1240 		KUNIT_EXPECT_EQ(test, 1L, __atomic_fetch_or(&test_var, 0xf0, __ATOMIC_RELAXED));
1241 		KUNIT_EXPECT_EQ(test, 241L, __atomic_fetch_nand(&test_var, 0xf, __ATOMIC_RELAXED));
1242 		KUNIT_EXPECT_EQ(test, -2L, test_var);
1243 
1244 		__atomic_thread_fence(__ATOMIC_SEQ_CST);
1245 		__atomic_signal_fence(__ATOMIC_SEQ_CST);
1246 
1247 		kcsan_disable_current();
1248 
1249 		match_never = report_available();
1250 	} while (!end_test_checks(match_never));
1251 	KUNIT_EXPECT_FALSE(test, match_never);
1252 }
1253 
1254 __no_kcsan
1255 static void test_1bit_value_change(struct kunit *test)
1256 {
1257 	struct expect_report expect = {
1258 		.access = {
1259 			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
1260 			{ test_kernel_xor_1bit, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
1261 		},
1262 	};
1263 	bool match = false;
1264 
1265 	begin_test_checks(test_kernel_read, test_kernel_xor_1bit);
1266 	do {
1267 		match = IS_ENABLED(CONFIG_KCSAN_PERMISSIVE)
1268 				? report_available()
1269 				: report_matches(&expect);
1270 	} while (!end_test_checks(match));
1271 	if (IS_ENABLED(CONFIG_KCSAN_PERMISSIVE))
1272 		KUNIT_EXPECT_FALSE(test, match);
1273 	else
1274 		KUNIT_EXPECT_TRUE(test, match);
1275 }
1276 
1277 __no_kcsan
1278 static void test_correct_barrier(struct kunit *test)
1279 {
1280 	struct expect_report expect = {
1281 		.access = {
1282 			{ test_kernel_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
1283 			{ test_kernel_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) },
1284 		},
1285 	};
1286 	bool match_expect = false;
1287 
1288 	test_struct.val[0] = 0; /* init unlocked */
1289 	begin_test_checks(test_kernel_with_memorder, test_kernel_with_memorder);
1290 	do {
1291 		match_expect = report_matches_any_reordered(&expect);
1292 	} while (!end_test_checks(match_expect));
1293 	KUNIT_EXPECT_FALSE(test, match_expect);
1294 }
1295 
1296 __no_kcsan
1297 static void test_missing_barrier(struct kunit *test)
1298 {
1299 	struct expect_report expect = {
1300 		.access = {
1301 			{ test_kernel_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
1302 			{ test_kernel_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) },
1303 		},
1304 	};
1305 	bool match_expect = false;
1306 
1307 	test_struct.val[0] = 0; /* init unlocked */
1308 	begin_test_checks(test_kernel_wrong_memorder, test_kernel_wrong_memorder);
1309 	do {
1310 		match_expect = report_matches_any_reordered(&expect);
1311 	} while (!end_test_checks(match_expect));
1312 	if (IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY))
1313 		KUNIT_EXPECT_TRUE(test, match_expect);
1314 	else
1315 		KUNIT_EXPECT_FALSE(test, match_expect);
1316 }
1317 
1318 __no_kcsan
1319 static void test_atomic_builtins_correct_barrier(struct kunit *test)
1320 {
1321 	struct expect_report expect = {
1322 		.access = {
1323 			{ test_kernel_atomic_builtin_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
1324 			{ test_kernel_atomic_builtin_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) },
1325 		},
1326 	};
1327 	bool match_expect = false;
1328 
1329 	test_struct.val[0] = 0; /* init unlocked */
1330 	begin_test_checks(test_kernel_atomic_builtin_with_memorder,
1331 			  test_kernel_atomic_builtin_with_memorder);
1332 	do {
1333 		match_expect = report_matches_any_reordered(&expect);
1334 	} while (!end_test_checks(match_expect));
1335 	KUNIT_EXPECT_FALSE(test, match_expect);
1336 }
1337 
1338 __no_kcsan
1339 static void test_atomic_builtins_missing_barrier(struct kunit *test)
1340 {
1341 	struct expect_report expect = {
1342 		.access = {
1343 			{ test_kernel_atomic_builtin_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
1344 			{ test_kernel_atomic_builtin_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) },
1345 		},
1346 	};
1347 	bool match_expect = false;
1348 
1349 	test_struct.val[0] = 0; /* init unlocked */
1350 	begin_test_checks(test_kernel_atomic_builtin_wrong_memorder,
1351 			  test_kernel_atomic_builtin_wrong_memorder);
1352 	do {
1353 		match_expect = report_matches_any_reordered(&expect);
1354 	} while (!end_test_checks(match_expect));
1355 	if (IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY))
1356 		KUNIT_EXPECT_TRUE(test, match_expect);
1357 	else
1358 		KUNIT_EXPECT_FALSE(test, match_expect);
1359 }
1360 
1361 /*
1362  * Generate thread counts for all test cases. Values generated are in interval
1363  * [2, 5] followed by exponentially increasing thread counts from 8 to 32.
1364  *
1365  * The thread counts are chosen to cover potentially interesting boundaries and
1366  * corner cases (2 to 5), and then stress the system with larger counts.
1367  */
1368 static const void *nthreads_gen_params(const void *prev, char *desc)
1369 {
1370 	long nthreads = (long)prev;
1371 
1372 	if (nthreads < 0 || nthreads >= 32)
1373 		nthreads = 0; /* stop */
1374 	else if (!nthreads)
1375 		nthreads = 2; /* initial value */
1376 	else if (nthreads < 5)
1377 		nthreads++;
1378 	else if (nthreads == 5)
1379 		nthreads = 8;
1380 	else
1381 		nthreads *= 2;
1382 
1383 	if (!preempt_model_preemptible() ||
1384 	    !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) {
1385 		/*
1386 		 * Without any preemption, keep 2 CPUs free for other tasks, one
1387 		 * of which is the main test case function checking for
1388 		 * completion or failure.
1389 		 */
1390 		const long min_unused_cpus = preempt_model_none() ? 2 : 0;
1391 		const long min_required_cpus = 2 + min_unused_cpus;
1392 
1393 		if (num_online_cpus() < min_required_cpus) {
1394 			pr_err_once("Too few online CPUs (%u < %ld) for test\n",
1395 				    num_online_cpus(), min_required_cpus);
1396 			nthreads = 0;
1397 		} else if (nthreads >= num_online_cpus() - min_unused_cpus) {
1398 			/* Use negative value to indicate last param. */
1399 			nthreads = -(num_online_cpus() - min_unused_cpus);
1400 			pr_warn_once("Limiting number of threads to %ld (only %d online CPUs)\n",
1401 				     -nthreads, num_online_cpus());
1402 		}
1403 	}
1404 
1405 	snprintf(desc, KUNIT_PARAM_DESC_SIZE, "threads=%ld", abs(nthreads));
1406 	return (void *)nthreads;
1407 }
1408 
1409 #define KCSAN_KUNIT_CASE(test_name) KUNIT_CASE_PARAM(test_name, nthreads_gen_params)
1410 static struct kunit_case kcsan_test_cases[] = {
1411 	KUNIT_CASE(test_barrier_nothreads),
1412 	KCSAN_KUNIT_CASE(test_basic),
1413 	KCSAN_KUNIT_CASE(test_concurrent_races),
1414 	KCSAN_KUNIT_CASE(test_novalue_change),
1415 	KCSAN_KUNIT_CASE(test_novalue_change_exception),
1416 	KCSAN_KUNIT_CASE(test_unknown_origin),
1417 	KCSAN_KUNIT_CASE(test_write_write_assume_atomic),
1418 	KCSAN_KUNIT_CASE(test_write_write_struct),
1419 	KCSAN_KUNIT_CASE(test_write_write_struct_part),
1420 	KCSAN_KUNIT_CASE(test_read_atomic_write_atomic),
1421 	KCSAN_KUNIT_CASE(test_read_plain_atomic_write),
1422 	KCSAN_KUNIT_CASE(test_read_plain_atomic_rmw),
1423 	KCSAN_KUNIT_CASE(test_zero_size_access),
1424 	KCSAN_KUNIT_CASE(test_data_race),
1425 	KCSAN_KUNIT_CASE(test_assert_exclusive_writer),
1426 	KCSAN_KUNIT_CASE(test_assert_exclusive_access),
1427 	KCSAN_KUNIT_CASE(test_assert_exclusive_access_writer),
1428 	KCSAN_KUNIT_CASE(test_assert_exclusive_bits_change),
1429 	KCSAN_KUNIT_CASE(test_assert_exclusive_bits_nochange),
1430 	KCSAN_KUNIT_CASE(test_assert_exclusive_writer_scoped),
1431 	KCSAN_KUNIT_CASE(test_assert_exclusive_access_scoped),
1432 	KCSAN_KUNIT_CASE(test_jiffies_noreport),
1433 	KCSAN_KUNIT_CASE(test_seqlock_noreport),
1434 	KCSAN_KUNIT_CASE(test_atomic_builtins),
1435 	KCSAN_KUNIT_CASE(test_1bit_value_change),
1436 	KCSAN_KUNIT_CASE(test_correct_barrier),
1437 	KCSAN_KUNIT_CASE(test_missing_barrier),
1438 	KCSAN_KUNIT_CASE(test_atomic_builtins_correct_barrier),
1439 	KCSAN_KUNIT_CASE(test_atomic_builtins_missing_barrier),
1440 	{},
1441 };
1442 
1443 /* ===== End test cases ===== */
1444 
1445 /* Concurrent accesses from interrupts. */
1446 __no_kcsan
1447 static void access_thread_timer(struct timer_list *timer)
1448 {
1449 	static atomic_t cnt = ATOMIC_INIT(0);
1450 	unsigned int idx;
1451 	void (*func)(void);
1452 
1453 	idx = (unsigned int)atomic_inc_return(&cnt) % ARRAY_SIZE(access_kernels);
1454 	/* Acquire potential initialization. */
1455 	func = smp_load_acquire(&access_kernels[idx]);
1456 	if (func)
1457 		func();
1458 }
1459 
1460 /* The main loop for each thread. */
1461 __no_kcsan
1462 static int access_thread(void *arg)
1463 {
1464 	struct timer_list timer;
1465 	unsigned int cnt = 0;
1466 	unsigned int idx;
1467 	void (*func)(void);
1468 
1469 	timer_setup_on_stack(&timer, access_thread_timer, 0);
1470 	do {
1471 		might_sleep();
1472 
1473 		if (!timer_pending(&timer))
1474 			mod_timer(&timer, jiffies + 1);
1475 		else {
1476 			/* Iterate through all kernels. */
1477 			idx = cnt++ % ARRAY_SIZE(access_kernels);
1478 			/* Acquire potential initialization. */
1479 			func = smp_load_acquire(&access_kernels[idx]);
1480 			if (func)
1481 				func();
1482 		}
1483 	} while (!torture_must_stop());
1484 	del_timer_sync(&timer);
1485 	destroy_timer_on_stack(&timer);
1486 
1487 	torture_kthread_stopping("access_thread");
1488 	return 0;
1489 }
1490 
1491 __no_kcsan
1492 static int test_init(struct kunit *test)
1493 {
1494 	unsigned long flags;
1495 	int nthreads;
1496 	int i;
1497 
1498 	spin_lock_irqsave(&observed.lock, flags);
1499 	for (i = 0; i < ARRAY_SIZE(observed.lines); ++i)
1500 		observed.lines[i][0] = '\0';
1501 	observed.nlines = 0;
1502 	spin_unlock_irqrestore(&observed.lock, flags);
1503 
1504 	if (strstr(test->name, "nothreads"))
1505 		return 0;
1506 
1507 	if (!torture_init_begin((char *)test->name, 1))
1508 		return -EBUSY;
1509 
1510 	if (WARN_ON(threads))
1511 		goto err;
1512 
1513 	for (i = 0; i < ARRAY_SIZE(access_kernels); ++i) {
1514 		if (WARN_ON(access_kernels[i]))
1515 			goto err;
1516 	}
1517 
1518 	nthreads = abs((long)test->param_value);
1519 	if (WARN_ON(!nthreads))
1520 		goto err;
1521 
1522 	threads = kcalloc(nthreads + 1, sizeof(struct task_struct *), GFP_KERNEL);
1523 	if (WARN_ON(!threads))
1524 		goto err;
1525 
1526 	threads[nthreads] = NULL;
1527 	for (i = 0; i < nthreads; ++i) {
1528 		if (torture_create_kthread(access_thread, NULL, threads[i]))
1529 			goto err;
1530 	}
1531 
1532 	torture_init_end();
1533 
1534 	return 0;
1535 
1536 err:
1537 	kfree(threads);
1538 	threads = NULL;
1539 	torture_init_end();
1540 	return -EINVAL;
1541 }
1542 
1543 __no_kcsan
1544 static void test_exit(struct kunit *test)
1545 {
1546 	struct task_struct **stop_thread;
1547 	int i;
1548 
1549 	if (strstr(test->name, "nothreads"))
1550 		return;
1551 
1552 	if (torture_cleanup_begin())
1553 		return;
1554 
1555 	for (i = 0; i < ARRAY_SIZE(access_kernels); ++i)
1556 		WRITE_ONCE(access_kernels[i], NULL);
1557 
1558 	if (threads) {
1559 		for (stop_thread = threads; *stop_thread; stop_thread++)
1560 			torture_stop_kthread(reader_thread, *stop_thread);
1561 
1562 		kfree(threads);
1563 		threads = NULL;
1564 	}
1565 
1566 	torture_cleanup_end();
1567 }
1568 
1569 __no_kcsan
1570 static void register_tracepoints(struct tracepoint *tp, void *ignore)
1571 {
1572 	check_trace_callback_type_console(probe_console);
1573 	if (!strcmp(tp->name, "console"))
1574 		WARN_ON(tracepoint_probe_register(tp, probe_console, NULL));
1575 }
1576 
1577 __no_kcsan
1578 static void unregister_tracepoints(struct tracepoint *tp, void *ignore)
1579 {
1580 	if (!strcmp(tp->name, "console"))
1581 		tracepoint_probe_unregister(tp, probe_console, NULL);
1582 }
1583 
1584 static int kcsan_suite_init(struct kunit_suite *suite)
1585 {
1586 	/*
1587 	 * Because we want to be able to build the test as a module, we need to
1588 	 * iterate through all known tracepoints, since the static registration
1589 	 * won't work here.
1590 	 */
1591 	for_each_kernel_tracepoint(register_tracepoints, NULL);
1592 	return 0;
1593 }
1594 
1595 static void kcsan_suite_exit(struct kunit_suite *suite)
1596 {
1597 	for_each_kernel_tracepoint(unregister_tracepoints, NULL);
1598 	tracepoint_synchronize_unregister();
1599 }
1600 
1601 static struct kunit_suite kcsan_test_suite = {
1602 	.name = "kcsan",
1603 	.test_cases = kcsan_test_cases,
1604 	.init = test_init,
1605 	.exit = test_exit,
1606 	.suite_init = kcsan_suite_init,
1607 	.suite_exit = kcsan_suite_exit,
1608 };
1609 
1610 kunit_test_suites(&kcsan_test_suite);
1611 
1612 MODULE_LICENSE("GPL v2");
1613 MODULE_AUTHOR("Marco Elver <elver@google.com>");
1614