1 /* 2 * Generic implementation of 64-bit atomics using spinlocks, 3 * useful on processors that don't have 64-bit atomic instructions. 4 * 5 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License 9 * as published by the Free Software Foundation; either version 10 * 2 of the License, or (at your option) any later version. 11 */ 12 #include <linux/types.h> 13 #include <linux/cache.h> 14 #include <linux/spinlock.h> 15 #include <linux/init.h> 16 #include <linux/export.h> 17 #include <linux/atomic.h> 18 19 /* 20 * We use a hashed array of spinlocks to provide exclusive access 21 * to each atomic64_t variable. Since this is expected to used on 22 * systems with small numbers of CPUs (<= 4 or so), we use a 23 * relatively small array of 16 spinlocks to avoid wasting too much 24 * memory on the spinlock array. 25 */ 26 #define NR_LOCKS 16 27 28 /* 29 * Ensure each lock is in a separate cacheline. 30 */ 31 static union { 32 raw_spinlock_t lock; 33 char pad[L1_CACHE_BYTES]; 34 } atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = { 35 [0 ... (NR_LOCKS - 1)] = { 36 .lock = __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock), 37 }, 38 }; 39 40 static inline raw_spinlock_t *lock_addr(const atomic64_t *v) 41 { 42 unsigned long addr = (unsigned long) v; 43 44 addr >>= L1_CACHE_SHIFT; 45 addr ^= (addr >> 8) ^ (addr >> 16); 46 return &atomic64_lock[addr & (NR_LOCKS - 1)].lock; 47 } 48 49 long long atomic64_read(const atomic64_t *v) 50 { 51 unsigned long flags; 52 raw_spinlock_t *lock = lock_addr(v); 53 long long val; 54 55 raw_spin_lock_irqsave(lock, flags); 56 val = v->counter; 57 raw_spin_unlock_irqrestore(lock, flags); 58 return val; 59 } 60 EXPORT_SYMBOL(atomic64_read); 61 62 void atomic64_set(atomic64_t *v, long long i) 63 { 64 unsigned long flags; 65 raw_spinlock_t *lock = lock_addr(v); 66 67 raw_spin_lock_irqsave(lock, flags); 68 v->counter = i; 69 raw_spin_unlock_irqrestore(lock, flags); 70 } 71 EXPORT_SYMBOL(atomic64_set); 72 73 void atomic64_add(long long a, atomic64_t *v) 74 { 75 unsigned long flags; 76 raw_spinlock_t *lock = lock_addr(v); 77 78 raw_spin_lock_irqsave(lock, flags); 79 v->counter += a; 80 raw_spin_unlock_irqrestore(lock, flags); 81 } 82 EXPORT_SYMBOL(atomic64_add); 83 84 long long atomic64_add_return(long long a, atomic64_t *v) 85 { 86 unsigned long flags; 87 raw_spinlock_t *lock = lock_addr(v); 88 long long val; 89 90 raw_spin_lock_irqsave(lock, flags); 91 val = v->counter += a; 92 raw_spin_unlock_irqrestore(lock, flags); 93 return val; 94 } 95 EXPORT_SYMBOL(atomic64_add_return); 96 97 void atomic64_sub(long long a, atomic64_t *v) 98 { 99 unsigned long flags; 100 raw_spinlock_t *lock = lock_addr(v); 101 102 raw_spin_lock_irqsave(lock, flags); 103 v->counter -= a; 104 raw_spin_unlock_irqrestore(lock, flags); 105 } 106 EXPORT_SYMBOL(atomic64_sub); 107 108 long long atomic64_sub_return(long long a, atomic64_t *v) 109 { 110 unsigned long flags; 111 raw_spinlock_t *lock = lock_addr(v); 112 long long val; 113 114 raw_spin_lock_irqsave(lock, flags); 115 val = v->counter -= a; 116 raw_spin_unlock_irqrestore(lock, flags); 117 return val; 118 } 119 EXPORT_SYMBOL(atomic64_sub_return); 120 121 long long atomic64_dec_if_positive(atomic64_t *v) 122 { 123 unsigned long flags; 124 raw_spinlock_t *lock = lock_addr(v); 125 long long val; 126 127 raw_spin_lock_irqsave(lock, flags); 128 val = v->counter - 1; 129 if (val >= 0) 130 v->counter = val; 131 raw_spin_unlock_irqrestore(lock, flags); 132 return val; 133 } 134 EXPORT_SYMBOL(atomic64_dec_if_positive); 135 136 long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n) 137 { 138 unsigned long flags; 139 raw_spinlock_t *lock = lock_addr(v); 140 long long val; 141 142 raw_spin_lock_irqsave(lock, flags); 143 val = v->counter; 144 if (val == o) 145 v->counter = n; 146 raw_spin_unlock_irqrestore(lock, flags); 147 return val; 148 } 149 EXPORT_SYMBOL(atomic64_cmpxchg); 150 151 long long atomic64_xchg(atomic64_t *v, long long new) 152 { 153 unsigned long flags; 154 raw_spinlock_t *lock = lock_addr(v); 155 long long val; 156 157 raw_spin_lock_irqsave(lock, flags); 158 val = v->counter; 159 v->counter = new; 160 raw_spin_unlock_irqrestore(lock, flags); 161 return val; 162 } 163 EXPORT_SYMBOL(atomic64_xchg); 164 165 int atomic64_add_unless(atomic64_t *v, long long a, long long u) 166 { 167 unsigned long flags; 168 raw_spinlock_t *lock = lock_addr(v); 169 int ret = 0; 170 171 raw_spin_lock_irqsave(lock, flags); 172 if (v->counter != u) { 173 v->counter += a; 174 ret = 1; 175 } 176 raw_spin_unlock_irqrestore(lock, flags); 177 return ret; 178 } 179 EXPORT_SYMBOL(atomic64_add_unless); 180