1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_GENERIC_BITOPS_ATOMIC_H_ 3 #define _ASM_GENERIC_BITOPS_ATOMIC_H_ 4 5 #include <asm/types.h> 6 #include <linux/irqflags.h> 7 8 #ifdef CONFIG_SMP 9 #include <asm/spinlock.h> 10 #include <asm/cache.h> /* we use L1_CACHE_BYTES */ 11 12 /* Use an array of spinlocks for our atomic_ts. 13 * Hash function to index into a different SPINLOCK. 14 * Since "a" is usually an address, use one spinlock per cacheline. 15 */ 16 # define ATOMIC_HASH_SIZE 4 17 # define ATOMIC_HASH(a) (&(__atomic_hash[ (((unsigned long) a)/L1_CACHE_BYTES) & (ATOMIC_HASH_SIZE-1) ])) 18 19 extern arch_spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] __lock_aligned; 20 21 /* Can't use raw_spin_lock_irq because of #include problems, so 22 * this is the substitute */ 23 #define _atomic_spin_lock_irqsave(l,f) do { \ 24 arch_spinlock_t *s = ATOMIC_HASH(l); \ 25 local_irq_save(f); \ 26 arch_spin_lock(s); \ 27 } while(0) 28 29 #define _atomic_spin_unlock_irqrestore(l,f) do { \ 30 arch_spinlock_t *s = ATOMIC_HASH(l); \ 31 arch_spin_unlock(s); \ 32 local_irq_restore(f); \ 33 } while(0) 34 35 36 #else 37 # define _atomic_spin_lock_irqsave(l,f) do { local_irq_save(f); } while (0) 38 # define _atomic_spin_unlock_irqrestore(l,f) do { local_irq_restore(f); } while (0) 39 #endif 40 41 /* 42 * NMI events can occur at any time, including when interrupts have been 43 * disabled by *_irqsave(). So you can get NMI events occurring while a 44 * *_bit function is holding a spin lock. If the NMI handler also wants 45 * to do bit manipulation (and they do) then you can get a deadlock 46 * between the original caller of *_bit() and the NMI handler. 47 * 48 * by Keith Owens 49 */ 50 51 /** 52 * set_bit - Atomically set a bit in memory 53 * @nr: the bit to set 54 * @addr: the address to start counting from 55 * 56 * This function is atomic and may not be reordered. See __set_bit() 57 * if you do not require the atomic guarantees. 58 * 59 * Note: there are no guarantees that this function will not be reordered 60 * on non x86 architectures, so if you are writing portable code, 61 * make sure not to rely on its reordering guarantees. 62 * 63 * Note that @nr may be almost arbitrarily large; this function is not 64 * restricted to acting on a single-word quantity. 65 */ 66 static inline void set_bit(int nr, volatile unsigned long *addr) 67 { 68 unsigned long mask = BIT_MASK(nr); 69 unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 70 unsigned long flags; 71 72 _atomic_spin_lock_irqsave(p, flags); 73 *p |= mask; 74 _atomic_spin_unlock_irqrestore(p, flags); 75 } 76 77 /** 78 * clear_bit - Clears a bit in memory 79 * @nr: Bit to clear 80 * @addr: Address to start counting from 81 * 82 * clear_bit() is atomic and may not be reordered. However, it does 83 * not contain a memory barrier, so if it is used for locking purposes, 84 * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic() 85 * in order to ensure changes are visible on other processors. 86 */ 87 static inline void clear_bit(int nr, volatile unsigned long *addr) 88 { 89 unsigned long mask = BIT_MASK(nr); 90 unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 91 unsigned long flags; 92 93 _atomic_spin_lock_irqsave(p, flags); 94 *p &= ~mask; 95 _atomic_spin_unlock_irqrestore(p, flags); 96 } 97 98 /** 99 * change_bit - Toggle a bit in memory 100 * @nr: Bit to change 101 * @addr: Address to start counting from 102 * 103 * change_bit() is atomic and may not be reordered. It may be 104 * reordered on other architectures than x86. 105 * Note that @nr may be almost arbitrarily large; this function is not 106 * restricted to acting on a single-word quantity. 107 */ 108 static inline void change_bit(int nr, volatile unsigned long *addr) 109 { 110 unsigned long mask = BIT_MASK(nr); 111 unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 112 unsigned long flags; 113 114 _atomic_spin_lock_irqsave(p, flags); 115 *p ^= mask; 116 _atomic_spin_unlock_irqrestore(p, flags); 117 } 118 119 /** 120 * test_and_set_bit - Set a bit and return its old value 121 * @nr: Bit to set 122 * @addr: Address to count from 123 * 124 * This operation is atomic and cannot be reordered. 125 * It may be reordered on other architectures than x86. 126 * It also implies a memory barrier. 127 */ 128 static inline int test_and_set_bit(int nr, volatile unsigned long *addr) 129 { 130 unsigned long mask = BIT_MASK(nr); 131 unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 132 unsigned long old; 133 unsigned long flags; 134 135 _atomic_spin_lock_irqsave(p, flags); 136 old = *p; 137 *p = old | mask; 138 _atomic_spin_unlock_irqrestore(p, flags); 139 140 return (old & mask) != 0; 141 } 142 143 /** 144 * test_and_clear_bit - Clear a bit and return its old value 145 * @nr: Bit to clear 146 * @addr: Address to count from 147 * 148 * This operation is atomic and cannot be reordered. 149 * It can be reorderdered on other architectures other than x86. 150 * It also implies a memory barrier. 151 */ 152 static inline int test_and_clear_bit(int nr, volatile unsigned long *addr) 153 { 154 unsigned long mask = BIT_MASK(nr); 155 unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 156 unsigned long old; 157 unsigned long flags; 158 159 _atomic_spin_lock_irqsave(p, flags); 160 old = *p; 161 *p = old & ~mask; 162 _atomic_spin_unlock_irqrestore(p, flags); 163 164 return (old & mask) != 0; 165 } 166 167 /** 168 * test_and_change_bit - Change a bit and return its old value 169 * @nr: Bit to change 170 * @addr: Address to count from 171 * 172 * This operation is atomic and cannot be reordered. 173 * It also implies a memory barrier. 174 */ 175 static inline int test_and_change_bit(int nr, volatile unsigned long *addr) 176 { 177 unsigned long mask = BIT_MASK(nr); 178 unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); 179 unsigned long old; 180 unsigned long flags; 181 182 _atomic_spin_lock_irqsave(p, flags); 183 old = *p; 184 *p = old ^ mask; 185 _atomic_spin_unlock_irqrestore(p, flags); 186 187 return (old & mask) != 0; 188 } 189 190 #endif /* _ASM_GENERIC_BITOPS_ATOMIC_H */ 191