1 /* 2 * Generic implementation of 64-bit atomics using spinlocks, 3 * useful on processors that don't have 64-bit atomic instructions. 4 * 5 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License 9 * as published by the Free Software Foundation; either version 10 * 2 of the License, or (at your option) any later version. 11 */ 12 #include <linux/types.h> 13 #include <linux/cache.h> 14 #include <linux/spinlock.h> 15 #include <linux/init.h> 16 #include <linux/export.h> 17 #include <linux/atomic.h> 18 19 /* 20 * We use a hashed array of spinlocks to provide exclusive access 21 * to each atomic64_t variable. Since this is expected to used on 22 * systems with small numbers of CPUs (<= 4 or so), we use a 23 * relatively small array of 16 spinlocks to avoid wasting too much 24 * memory on the spinlock array. 25 */ 26 #define NR_LOCKS 16 27 28 /* 29 * Ensure each lock is in a separate cacheline. 30 */ 31 static union { 32 raw_spinlock_t lock; 33 char pad[L1_CACHE_BYTES]; 34 } atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp; 35 36 static inline raw_spinlock_t *lock_addr(const atomic64_t *v) 37 { 38 unsigned long addr = (unsigned long) v; 39 40 addr >>= L1_CACHE_SHIFT; 41 addr ^= (addr >> 8) ^ (addr >> 16); 42 return &atomic64_lock[addr & (NR_LOCKS - 1)].lock; 43 } 44 45 long long atomic64_read(const atomic64_t *v) 46 { 47 unsigned long flags; 48 raw_spinlock_t *lock = lock_addr(v); 49 long long val; 50 51 raw_spin_lock_irqsave(lock, flags); 52 val = v->counter; 53 raw_spin_unlock_irqrestore(lock, flags); 54 return val; 55 } 56 EXPORT_SYMBOL(atomic64_read); 57 58 void atomic64_set(atomic64_t *v, long long i) 59 { 60 unsigned long flags; 61 raw_spinlock_t *lock = lock_addr(v); 62 63 raw_spin_lock_irqsave(lock, flags); 64 v->counter = i; 65 raw_spin_unlock_irqrestore(lock, flags); 66 } 67 EXPORT_SYMBOL(atomic64_set); 68 69 void atomic64_add(long long a, atomic64_t *v) 70 { 71 unsigned long flags; 72 raw_spinlock_t *lock = lock_addr(v); 73 74 raw_spin_lock_irqsave(lock, flags); 75 v->counter += a; 76 raw_spin_unlock_irqrestore(lock, flags); 77 } 78 EXPORT_SYMBOL(atomic64_add); 79 80 long long atomic64_add_return(long long a, atomic64_t *v) 81 { 82 unsigned long flags; 83 raw_spinlock_t *lock = lock_addr(v); 84 long long val; 85 86 raw_spin_lock_irqsave(lock, flags); 87 val = v->counter += a; 88 raw_spin_unlock_irqrestore(lock, flags); 89 return val; 90 } 91 EXPORT_SYMBOL(atomic64_add_return); 92 93 void atomic64_sub(long long a, atomic64_t *v) 94 { 95 unsigned long flags; 96 raw_spinlock_t *lock = lock_addr(v); 97 98 raw_spin_lock_irqsave(lock, flags); 99 v->counter -= a; 100 raw_spin_unlock_irqrestore(lock, flags); 101 } 102 EXPORT_SYMBOL(atomic64_sub); 103 104 long long atomic64_sub_return(long long a, atomic64_t *v) 105 { 106 unsigned long flags; 107 raw_spinlock_t *lock = lock_addr(v); 108 long long val; 109 110 raw_spin_lock_irqsave(lock, flags); 111 val = v->counter -= a; 112 raw_spin_unlock_irqrestore(lock, flags); 113 return val; 114 } 115 EXPORT_SYMBOL(atomic64_sub_return); 116 117 long long atomic64_dec_if_positive(atomic64_t *v) 118 { 119 unsigned long flags; 120 raw_spinlock_t *lock = lock_addr(v); 121 long long val; 122 123 raw_spin_lock_irqsave(lock, flags); 124 val = v->counter - 1; 125 if (val >= 0) 126 v->counter = val; 127 raw_spin_unlock_irqrestore(lock, flags); 128 return val; 129 } 130 EXPORT_SYMBOL(atomic64_dec_if_positive); 131 132 long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n) 133 { 134 unsigned long flags; 135 raw_spinlock_t *lock = lock_addr(v); 136 long long val; 137 138 raw_spin_lock_irqsave(lock, flags); 139 val = v->counter; 140 if (val == o) 141 v->counter = n; 142 raw_spin_unlock_irqrestore(lock, flags); 143 return val; 144 } 145 EXPORT_SYMBOL(atomic64_cmpxchg); 146 147 long long atomic64_xchg(atomic64_t *v, long long new) 148 { 149 unsigned long flags; 150 raw_spinlock_t *lock = lock_addr(v); 151 long long val; 152 153 raw_spin_lock_irqsave(lock, flags); 154 val = v->counter; 155 v->counter = new; 156 raw_spin_unlock_irqrestore(lock, flags); 157 return val; 158 } 159 EXPORT_SYMBOL(atomic64_xchg); 160 161 int atomic64_add_unless(atomic64_t *v, long long a, long long u) 162 { 163 unsigned long flags; 164 raw_spinlock_t *lock = lock_addr(v); 165 int ret = 0; 166 167 raw_spin_lock_irqsave(lock, flags); 168 if (v->counter != u) { 169 v->counter += a; 170 ret = 1; 171 } 172 raw_spin_unlock_irqrestore(lock, flags); 173 return ret; 174 } 175 EXPORT_SYMBOL(atomic64_add_unless); 176 177 static int init_atomic64_lock(void) 178 { 179 int i; 180 181 for (i = 0; i < NR_LOCKS; ++i) 182 raw_spin_lock_init(&atomic64_lock[i].lock); 183 return 0; 184 } 185 186 pure_initcall(init_atomic64_lock); 187