1 /* SPDX-License-Identifier: GPL-2.0 */
2 
3 /*
4  * This file provides wrappers with sanitizer instrumentation for non-atomic
5  * bit operations.
6  *
7  * To use this functionality, an arch's bitops.h file needs to define each of
8  * the below bit operations with an arch_ prefix (e.g. arch_set_bit(),
9  * arch___set_bit(), etc.).
10  */
11 #ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
12 #define _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
13 
14 #include <linux/instrumented.h>
15 
16 /**
17  * ___set_bit - Set a bit in memory
18  * @nr: the bit to set
19  * @addr: the address to start counting from
20  *
21  * Unlike set_bit(), this function is non-atomic. If it is called on the same
22  * region of memory concurrently, the effect may be that only one operation
23  * succeeds.
24  */
25 static __always_inline void
26 ___set_bit(unsigned long nr, volatile unsigned long *addr)
27 {
28 	instrument_write(addr + BIT_WORD(nr), sizeof(long));
29 	arch___set_bit(nr, addr);
30 }
31 
32 /**
33  * ___clear_bit - Clears a bit in memory
34  * @nr: the bit to clear
35  * @addr: the address to start counting from
36  *
37  * Unlike clear_bit(), this function is non-atomic. If it is called on the same
38  * region of memory concurrently, the effect may be that only one operation
39  * succeeds.
40  */
41 static __always_inline void
42 ___clear_bit(unsigned long nr, volatile unsigned long *addr)
43 {
44 	instrument_write(addr + BIT_WORD(nr), sizeof(long));
45 	arch___clear_bit(nr, addr);
46 }
47 
48 /**
49  * ___change_bit - Toggle a bit in memory
50  * @nr: the bit to change
51  * @addr: the address to start counting from
52  *
53  * Unlike change_bit(), this function is non-atomic. If it is called on the same
54  * region of memory concurrently, the effect may be that only one operation
55  * succeeds.
56  */
57 static __always_inline void
58 ___change_bit(unsigned long nr, volatile unsigned long *addr)
59 {
60 	instrument_write(addr + BIT_WORD(nr), sizeof(long));
61 	arch___change_bit(nr, addr);
62 }
63 
64 static __always_inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
65 {
66 	if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) {
67 		/*
68 		 * We treat non-atomic read-write bitops a little more special.
69 		 * Given the operations here only modify a single bit, assuming
70 		 * non-atomicity of the writer is sufficient may be reasonable
71 		 * for certain usage (and follows the permissible nature of the
72 		 * assume-plain-writes-atomic rule):
73 		 * 1. report read-modify-write races -> check read;
74 		 * 2. do not report races with marked readers, but do report
75 		 *    races with unmarked readers -> check "atomic" write.
76 		 */
77 		kcsan_check_read(addr + BIT_WORD(nr), sizeof(long));
78 		/*
79 		 * Use generic write instrumentation, in case other sanitizers
80 		 * or tools are enabled alongside KCSAN.
81 		 */
82 		instrument_write(addr + BIT_WORD(nr), sizeof(long));
83 	} else {
84 		instrument_read_write(addr + BIT_WORD(nr), sizeof(long));
85 	}
86 }
87 
88 /**
89  * ___test_and_set_bit - Set a bit and return its old value
90  * @nr: Bit to set
91  * @addr: Address to count from
92  *
93  * This operation is non-atomic. If two instances of this operation race, one
94  * can appear to succeed but actually fail.
95  */
96 static __always_inline bool
97 ___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
98 {
99 	__instrument_read_write_bitop(nr, addr);
100 	return arch___test_and_set_bit(nr, addr);
101 }
102 
103 /**
104  * ___test_and_clear_bit - Clear a bit and return its old value
105  * @nr: Bit to clear
106  * @addr: Address to count from
107  *
108  * This operation is non-atomic. If two instances of this operation race, one
109  * can appear to succeed but actually fail.
110  */
111 static __always_inline bool
112 ___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
113 {
114 	__instrument_read_write_bitop(nr, addr);
115 	return arch___test_and_clear_bit(nr, addr);
116 }
117 
118 /**
119  * ___test_and_change_bit - Change a bit and return its old value
120  * @nr: Bit to change
121  * @addr: Address to count from
122  *
123  * This operation is non-atomic. If two instances of this operation race, one
124  * can appear to succeed but actually fail.
125  */
126 static __always_inline bool
127 ___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
128 {
129 	__instrument_read_write_bitop(nr, addr);
130 	return arch___test_and_change_bit(nr, addr);
131 }
132 
133 /**
134  * _test_bit - Determine whether a bit is set
135  * @nr: bit number to test
136  * @addr: Address to start counting from
137  */
138 static __always_inline bool
139 _test_bit(unsigned long nr, const volatile unsigned long *addr)
140 {
141 	instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
142 	return arch_test_bit(nr, addr);
143 }
144 
145 /**
146  * _test_bit_acquire - Determine, with acquire semantics, whether a bit is set
147  * @nr: bit number to test
148  * @addr: Address to start counting from
149  */
150 static __always_inline bool
151 _test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
152 {
153 	instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
154 	return arch_test_bit_acquire(nr, addr);
155 }
156 
157 #endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */
158