1 /* 2 * Fast batching percpu counters. 3 */ 4 5 #include <linux/percpu_counter.h> 6 #include <linux/notifier.h> 7 #include <linux/mutex.h> 8 #include <linux/init.h> 9 #include <linux/cpu.h> 10 #include <linux/module.h> 11 12 #ifdef CONFIG_HOTPLUG_CPU 13 static LIST_HEAD(percpu_counters); 14 static DEFINE_MUTEX(percpu_counters_lock); 15 #endif 16 17 void percpu_counter_set(struct percpu_counter *fbc, s64 amount) 18 { 19 int cpu; 20 21 spin_lock(&fbc->lock); 22 for_each_possible_cpu(cpu) { 23 s32 *pcount = per_cpu_ptr(fbc->counters, cpu); 24 *pcount = 0; 25 } 26 fbc->count = amount; 27 spin_unlock(&fbc->lock); 28 } 29 EXPORT_SYMBOL(percpu_counter_set); 30 31 void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch) 32 { 33 s64 count; 34 s32 *pcount; 35 int cpu = get_cpu(); 36 37 pcount = per_cpu_ptr(fbc->counters, cpu); 38 count = *pcount + amount; 39 if (count >= batch || count <= -batch) { 40 spin_lock(&fbc->lock); 41 fbc->count += count; 42 *pcount = 0; 43 spin_unlock(&fbc->lock); 44 } else { 45 *pcount = count; 46 } 47 put_cpu(); 48 } 49 EXPORT_SYMBOL(__percpu_counter_add); 50 51 /* 52 * Add up all the per-cpu counts, return the result. This is a more accurate 53 * but much slower version of percpu_counter_read_positive() 54 */ 55 s64 __percpu_counter_sum(struct percpu_counter *fbc) 56 { 57 s64 ret; 58 int cpu; 59 60 spin_lock(&fbc->lock); 61 ret = fbc->count; 62 for_each_online_cpu(cpu) { 63 s32 *pcount = per_cpu_ptr(fbc->counters, cpu); 64 ret += *pcount; 65 } 66 spin_unlock(&fbc->lock); 67 return ret; 68 } 69 EXPORT_SYMBOL(__percpu_counter_sum); 70 71 static struct lock_class_key percpu_counter_irqsafe; 72 73 int percpu_counter_init(struct percpu_counter *fbc, s64 amount) 74 { 75 spin_lock_init(&fbc->lock); 76 fbc->count = amount; 77 fbc->counters = alloc_percpu(s32); 78 if (!fbc->counters) 79 return -ENOMEM; 80 #ifdef CONFIG_HOTPLUG_CPU 81 mutex_lock(&percpu_counters_lock); 82 list_add(&fbc->list, &percpu_counters); 83 mutex_unlock(&percpu_counters_lock); 84 #endif 85 return 0; 86 } 87 EXPORT_SYMBOL(percpu_counter_init); 88 89 int percpu_counter_init_irq(struct percpu_counter *fbc, s64 amount) 90 { 91 int err; 92 93 err = percpu_counter_init(fbc, amount); 94 if (!err) 95 lockdep_set_class(&fbc->lock, &percpu_counter_irqsafe); 96 return err; 97 } 98 99 void percpu_counter_destroy(struct percpu_counter *fbc) 100 { 101 if (!fbc->counters) 102 return; 103 104 free_percpu(fbc->counters); 105 #ifdef CONFIG_HOTPLUG_CPU 106 mutex_lock(&percpu_counters_lock); 107 list_del(&fbc->list); 108 mutex_unlock(&percpu_counters_lock); 109 #endif 110 } 111 EXPORT_SYMBOL(percpu_counter_destroy); 112 113 #ifdef CONFIG_HOTPLUG_CPU 114 static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb, 115 unsigned long action, void *hcpu) 116 { 117 unsigned int cpu; 118 struct percpu_counter *fbc; 119 120 if (action != CPU_DEAD) 121 return NOTIFY_OK; 122 123 cpu = (unsigned long)hcpu; 124 mutex_lock(&percpu_counters_lock); 125 list_for_each_entry(fbc, &percpu_counters, list) { 126 s32 *pcount; 127 unsigned long flags; 128 129 spin_lock_irqsave(&fbc->lock, flags); 130 pcount = per_cpu_ptr(fbc->counters, cpu); 131 fbc->count += *pcount; 132 *pcount = 0; 133 spin_unlock_irqrestore(&fbc->lock, flags); 134 } 135 mutex_unlock(&percpu_counters_lock); 136 return NOTIFY_OK; 137 } 138 139 static int __init percpu_counter_startup(void) 140 { 141 hotcpu_notifier(percpu_counter_hotcpu_callback, 0); 142 return 0; 143 } 144 module_init(percpu_counter_startup); 145 #endif 146