1 /* 2 * Performance events callchain code, extracted from core.c: 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra 7 * Copyright � 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 8 * 9 * For licensing details see kernel-base/COPYING 10 */ 11 12 #include <linux/perf_event.h> 13 #include <linux/slab.h> 14 #include "internal.h" 15 16 struct callchain_cpus_entries { 17 struct rcu_head rcu_head; 18 struct perf_callchain_entry *cpu_entries[0]; 19 }; 20 21 static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]); 22 static atomic_t nr_callchain_events; 23 static DEFINE_MUTEX(callchain_mutex); 24 static struct callchain_cpus_entries *callchain_cpus_entries; 25 26 27 __weak void perf_callchain_kernel(struct perf_callchain_entry *entry, 28 struct pt_regs *regs) 29 { 30 } 31 32 __weak void perf_callchain_user(struct perf_callchain_entry *entry, 33 struct pt_regs *regs) 34 { 35 } 36 37 static void release_callchain_buffers_rcu(struct rcu_head *head) 38 { 39 struct callchain_cpus_entries *entries; 40 int cpu; 41 42 entries = container_of(head, struct callchain_cpus_entries, rcu_head); 43 44 for_each_possible_cpu(cpu) 45 kfree(entries->cpu_entries[cpu]); 46 47 kfree(entries); 48 } 49 50 static void release_callchain_buffers(void) 51 { 52 struct callchain_cpus_entries *entries; 53 54 entries = callchain_cpus_entries; 55 RCU_INIT_POINTER(callchain_cpus_entries, NULL); 56 call_rcu(&entries->rcu_head, release_callchain_buffers_rcu); 57 } 58 59 static int alloc_callchain_buffers(void) 60 { 61 int cpu; 62 int size; 63 struct callchain_cpus_entries *entries; 64 65 /* 66 * We can't use the percpu allocation API for data that can be 67 * accessed from NMI. Use a temporary manual per cpu allocation 68 * until that gets sorted out. 69 */ 70 size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]); 71 72 entries = kzalloc(size, GFP_KERNEL); 73 if (!entries) 74 return -ENOMEM; 75 76 size = sizeof(struct perf_callchain_entry) * PERF_NR_CONTEXTS; 77 78 for_each_possible_cpu(cpu) { 79 entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL, 80 cpu_to_node(cpu)); 81 if (!entries->cpu_entries[cpu]) 82 goto fail; 83 } 84 85 rcu_assign_pointer(callchain_cpus_entries, entries); 86 87 return 0; 88 89 fail: 90 for_each_possible_cpu(cpu) 91 kfree(entries->cpu_entries[cpu]); 92 kfree(entries); 93 94 return -ENOMEM; 95 } 96 97 int get_callchain_buffers(void) 98 { 99 int err = 0; 100 int count; 101 102 mutex_lock(&callchain_mutex); 103 104 count = atomic_inc_return(&nr_callchain_events); 105 if (WARN_ON_ONCE(count < 1)) { 106 err = -EINVAL; 107 goto exit; 108 } 109 110 if (count > 1) { 111 /* If the allocation failed, give up */ 112 if (!callchain_cpus_entries) 113 err = -ENOMEM; 114 goto exit; 115 } 116 117 err = alloc_callchain_buffers(); 118 exit: 119 if (err) 120 atomic_dec(&nr_callchain_events); 121 122 mutex_unlock(&callchain_mutex); 123 124 return err; 125 } 126 127 void put_callchain_buffers(void) 128 { 129 if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) { 130 release_callchain_buffers(); 131 mutex_unlock(&callchain_mutex); 132 } 133 } 134 135 static struct perf_callchain_entry *get_callchain_entry(int *rctx) 136 { 137 int cpu; 138 struct callchain_cpus_entries *entries; 139 140 *rctx = get_recursion_context(this_cpu_ptr(callchain_recursion)); 141 if (*rctx == -1) 142 return NULL; 143 144 entries = rcu_dereference(callchain_cpus_entries); 145 if (!entries) 146 return NULL; 147 148 cpu = smp_processor_id(); 149 150 return &entries->cpu_entries[cpu][*rctx]; 151 } 152 153 static void 154 put_callchain_entry(int rctx) 155 { 156 put_recursion_context(this_cpu_ptr(callchain_recursion), rctx); 157 } 158 159 struct perf_callchain_entry * 160 perf_callchain(struct perf_event *event, struct pt_regs *regs) 161 { 162 bool kernel = !event->attr.exclude_callchain_kernel; 163 bool user = !event->attr.exclude_callchain_user; 164 /* Disallow cross-task user callchains. */ 165 bool crosstask = event->ctx->task && event->ctx->task != current; 166 167 if (!kernel && !user) 168 return NULL; 169 170 return get_perf_callchain(regs, 0, kernel, user, crosstask, true); 171 } 172 173 struct perf_callchain_entry * 174 get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user, 175 bool crosstask, bool add_mark) 176 { 177 struct perf_callchain_entry *entry; 178 int rctx; 179 180 entry = get_callchain_entry(&rctx); 181 if (rctx == -1) 182 return NULL; 183 184 if (!entry) 185 goto exit_put; 186 187 entry->nr = init_nr; 188 189 if (kernel && !user_mode(regs)) { 190 if (add_mark) 191 perf_callchain_store(entry, PERF_CONTEXT_KERNEL); 192 perf_callchain_kernel(entry, regs); 193 } 194 195 if (user) { 196 if (!user_mode(regs)) { 197 if (current->mm) 198 regs = task_pt_regs(current); 199 else 200 regs = NULL; 201 } 202 203 if (regs) { 204 if (crosstask) 205 goto exit_put; 206 207 if (add_mark) 208 perf_callchain_store(entry, PERF_CONTEXT_USER); 209 perf_callchain_user(entry, regs); 210 } 211 } 212 213 exit_put: 214 put_callchain_entry(rctx); 215 216 return entry; 217 } 218