xref: /openbmc/linux/kernel/profile.c (revision e868d61272caa648214046a096e5a6bfc068dc8c)
1 /*
2  *  linux/kernel/profile.c
3  *  Simple profiling. Manages a direct-mapped profile hit count buffer,
4  *  with configurable resolution, support for restricting the cpus on
5  *  which profiling is done, and switching between cpu time and
6  *  schedule() calls via kernel command line parameters passed at boot.
7  *
8  *  Scheduler profiling support, Arjan van de Ven and Ingo Molnar,
9  *	Red Hat, July 2004
10  *  Consolidation of architecture support code for profiling,
11  *	William Irwin, Oracle, July 2004
12  *  Amortized hit count accounting via per-cpu open-addressed hashtables
13  *	to resolve timer interrupt livelocks, William Irwin, Oracle, 2004
14  */
15 
16 #include <linux/module.h>
17 #include <linux/profile.h>
18 #include <linux/bootmem.h>
19 #include <linux/notifier.h>
20 #include <linux/mm.h>
21 #include <linux/cpumask.h>
22 #include <linux/cpu.h>
23 #include <linux/profile.h>
24 #include <linux/highmem.h>
25 #include <linux/mutex.h>
26 #include <asm/sections.h>
27 #include <asm/semaphore.h>
28 #include <asm/irq_regs.h>
29 
30 struct profile_hit {
31 	u32 pc, hits;
32 };
33 #define PROFILE_GRPSHIFT	3
34 #define PROFILE_GRPSZ		(1 << PROFILE_GRPSHIFT)
35 #define NR_PROFILE_HIT		(PAGE_SIZE/sizeof(struct profile_hit))
36 #define NR_PROFILE_GRP		(NR_PROFILE_HIT/PROFILE_GRPSZ)
37 
38 /* Oprofile timer tick hook */
39 int (*timer_hook)(struct pt_regs *) __read_mostly;
40 
41 static atomic_t *prof_buffer;
42 static unsigned long prof_len, prof_shift;
43 
44 int prof_on __read_mostly;
45 EXPORT_SYMBOL_GPL(prof_on);
46 
47 static cpumask_t prof_cpu_mask = CPU_MASK_ALL;
48 #ifdef CONFIG_SMP
49 static DEFINE_PER_CPU(struct profile_hit *[2], cpu_profile_hits);
50 static DEFINE_PER_CPU(int, cpu_profile_flip);
51 static DEFINE_MUTEX(profile_flip_mutex);
52 #endif /* CONFIG_SMP */
53 
54 static int __init profile_setup(char * str)
55 {
56 	static char __initdata schedstr[] = "schedule";
57 	static char __initdata sleepstr[] = "sleep";
58 	static char __initdata kvmstr[] = "kvm";
59 	int par;
60 
61 	if (!strncmp(str, sleepstr, strlen(sleepstr))) {
62 		prof_on = SLEEP_PROFILING;
63 		if (str[strlen(sleepstr)] == ',')
64 			str += strlen(sleepstr) + 1;
65 		if (get_option(&str, &par))
66 			prof_shift = par;
67 		printk(KERN_INFO
68 			"kernel sleep profiling enabled (shift: %ld)\n",
69 			prof_shift);
70 	} else if (!strncmp(str, schedstr, strlen(schedstr))) {
71 		prof_on = SCHED_PROFILING;
72 		if (str[strlen(schedstr)] == ',')
73 			str += strlen(schedstr) + 1;
74 		if (get_option(&str, &par))
75 			prof_shift = par;
76 		printk(KERN_INFO
77 			"kernel schedule profiling enabled (shift: %ld)\n",
78 			prof_shift);
79 	} else if (!strncmp(str, kvmstr, strlen(kvmstr))) {
80 		prof_on = KVM_PROFILING;
81 		if (str[strlen(kvmstr)] == ',')
82 			str += strlen(kvmstr) + 1;
83 		if (get_option(&str, &par))
84 			prof_shift = par;
85 		printk(KERN_INFO
86 			"kernel KVM profiling enabled (shift: %ld)\n",
87 			prof_shift);
88 	} else if (get_option(&str, &par)) {
89 		prof_shift = par;
90 		prof_on = CPU_PROFILING;
91 		printk(KERN_INFO "kernel profiling enabled (shift: %ld)\n",
92 			prof_shift);
93 	}
94 	return 1;
95 }
96 __setup("profile=", profile_setup);
97 
98 
99 void __init profile_init(void)
100 {
101 	if (!prof_on)
102 		return;
103 
104 	/* only text is profiled */
105 	prof_len = (_etext - _stext) >> prof_shift;
106 	prof_buffer = alloc_bootmem(prof_len*sizeof(atomic_t));
107 }
108 
109 /* Profile event notifications */
110 
111 #ifdef CONFIG_PROFILING
112 
113 static BLOCKING_NOTIFIER_HEAD(task_exit_notifier);
114 static ATOMIC_NOTIFIER_HEAD(task_free_notifier);
115 static BLOCKING_NOTIFIER_HEAD(munmap_notifier);
116 
117 void profile_task_exit(struct task_struct * task)
118 {
119 	blocking_notifier_call_chain(&task_exit_notifier, 0, task);
120 }
121 
122 int profile_handoff_task(struct task_struct * task)
123 {
124 	int ret;
125 	ret = atomic_notifier_call_chain(&task_free_notifier, 0, task);
126 	return (ret == NOTIFY_OK) ? 1 : 0;
127 }
128 
129 void profile_munmap(unsigned long addr)
130 {
131 	blocking_notifier_call_chain(&munmap_notifier, 0, (void *)addr);
132 }
133 
134 int task_handoff_register(struct notifier_block * n)
135 {
136 	return atomic_notifier_chain_register(&task_free_notifier, n);
137 }
138 
139 int task_handoff_unregister(struct notifier_block * n)
140 {
141 	return atomic_notifier_chain_unregister(&task_free_notifier, n);
142 }
143 
144 int profile_event_register(enum profile_type type, struct notifier_block * n)
145 {
146 	int err = -EINVAL;
147 
148 	switch (type) {
149 		case PROFILE_TASK_EXIT:
150 			err = blocking_notifier_chain_register(
151 					&task_exit_notifier, n);
152 			break;
153 		case PROFILE_MUNMAP:
154 			err = blocking_notifier_chain_register(
155 					&munmap_notifier, n);
156 			break;
157 	}
158 
159 	return err;
160 }
161 
162 
163 int profile_event_unregister(enum profile_type type, struct notifier_block * n)
164 {
165 	int err = -EINVAL;
166 
167 	switch (type) {
168 		case PROFILE_TASK_EXIT:
169 			err = blocking_notifier_chain_unregister(
170 					&task_exit_notifier, n);
171 			break;
172 		case PROFILE_MUNMAP:
173 			err = blocking_notifier_chain_unregister(
174 					&munmap_notifier, n);
175 			break;
176 	}
177 
178 	return err;
179 }
180 
181 int register_timer_hook(int (*hook)(struct pt_regs *))
182 {
183 	if (timer_hook)
184 		return -EBUSY;
185 	timer_hook = hook;
186 	return 0;
187 }
188 
189 void unregister_timer_hook(int (*hook)(struct pt_regs *))
190 {
191 	WARN_ON(hook != timer_hook);
192 	timer_hook = NULL;
193 	/* make sure all CPUs see the NULL hook */
194 	synchronize_sched();  /* Allow ongoing interrupts to complete. */
195 }
196 
197 EXPORT_SYMBOL_GPL(register_timer_hook);
198 EXPORT_SYMBOL_GPL(unregister_timer_hook);
199 EXPORT_SYMBOL_GPL(task_handoff_register);
200 EXPORT_SYMBOL_GPL(task_handoff_unregister);
201 
202 #endif /* CONFIG_PROFILING */
203 
204 EXPORT_SYMBOL_GPL(profile_event_register);
205 EXPORT_SYMBOL_GPL(profile_event_unregister);
206 
207 #ifdef CONFIG_SMP
208 /*
209  * Each cpu has a pair of open-addressed hashtables for pending
210  * profile hits. read_profile() IPI's all cpus to request them
211  * to flip buffers and flushes their contents to prof_buffer itself.
212  * Flip requests are serialized by the profile_flip_mutex. The sole
213  * use of having a second hashtable is for avoiding cacheline
214  * contention that would otherwise happen during flushes of pending
215  * profile hits required for the accuracy of reported profile hits
216  * and so resurrect the interrupt livelock issue.
217  *
218  * The open-addressed hashtables are indexed by profile buffer slot
219  * and hold the number of pending hits to that profile buffer slot on
220  * a cpu in an entry. When the hashtable overflows, all pending hits
221  * are accounted to their corresponding profile buffer slots with
222  * atomic_add() and the hashtable emptied. As numerous pending hits
223  * may be accounted to a profile buffer slot in a hashtable entry,
224  * this amortizes a number of atomic profile buffer increments likely
225  * to be far larger than the number of entries in the hashtable,
226  * particularly given that the number of distinct profile buffer
227  * positions to which hits are accounted during short intervals (e.g.
228  * several seconds) is usually very small. Exclusion from buffer
229  * flipping is provided by interrupt disablement (note that for
230  * SCHED_PROFILING or SLEEP_PROFILING profile_hit() may be called from
231  * process context).
232  * The hash function is meant to be lightweight as opposed to strong,
233  * and was vaguely inspired by ppc64 firmware-supported inverted
234  * pagetable hash functions, but uses a full hashtable full of finite
235  * collision chains, not just pairs of them.
236  *
237  * -- wli
238  */
239 static void __profile_flip_buffers(void *unused)
240 {
241 	int cpu = smp_processor_id();
242 
243 	per_cpu(cpu_profile_flip, cpu) = !per_cpu(cpu_profile_flip, cpu);
244 }
245 
246 static void profile_flip_buffers(void)
247 {
248 	int i, j, cpu;
249 
250 	mutex_lock(&profile_flip_mutex);
251 	j = per_cpu(cpu_profile_flip, get_cpu());
252 	put_cpu();
253 	on_each_cpu(__profile_flip_buffers, NULL, 0, 1);
254 	for_each_online_cpu(cpu) {
255 		struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[j];
256 		for (i = 0; i < NR_PROFILE_HIT; ++i) {
257 			if (!hits[i].hits) {
258 				if (hits[i].pc)
259 					hits[i].pc = 0;
260 				continue;
261 			}
262 			atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
263 			hits[i].hits = hits[i].pc = 0;
264 		}
265 	}
266 	mutex_unlock(&profile_flip_mutex);
267 }
268 
269 static void profile_discard_flip_buffers(void)
270 {
271 	int i, cpu;
272 
273 	mutex_lock(&profile_flip_mutex);
274 	i = per_cpu(cpu_profile_flip, get_cpu());
275 	put_cpu();
276 	on_each_cpu(__profile_flip_buffers, NULL, 0, 1);
277 	for_each_online_cpu(cpu) {
278 		struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[i];
279 		memset(hits, 0, NR_PROFILE_HIT*sizeof(struct profile_hit));
280 	}
281 	mutex_unlock(&profile_flip_mutex);
282 }
283 
284 void profile_hits(int type, void *__pc, unsigned int nr_hits)
285 {
286 	unsigned long primary, secondary, flags, pc = (unsigned long)__pc;
287 	int i, j, cpu;
288 	struct profile_hit *hits;
289 
290 	if (prof_on != type || !prof_buffer)
291 		return;
292 	pc = min((pc - (unsigned long)_stext) >> prof_shift, prof_len - 1);
293 	i = primary = (pc & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
294 	secondary = (~(pc << 1) & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
295 	cpu = get_cpu();
296 	hits = per_cpu(cpu_profile_hits, cpu)[per_cpu(cpu_profile_flip, cpu)];
297 	if (!hits) {
298 		put_cpu();
299 		return;
300 	}
301 	/*
302 	 * We buffer the global profiler buffer into a per-CPU
303 	 * queue and thus reduce the number of global (and possibly
304 	 * NUMA-alien) accesses. The write-queue is self-coalescing:
305 	 */
306 	local_irq_save(flags);
307 	do {
308 		for (j = 0; j < PROFILE_GRPSZ; ++j) {
309 			if (hits[i + j].pc == pc) {
310 				hits[i + j].hits += nr_hits;
311 				goto out;
312 			} else if (!hits[i + j].hits) {
313 				hits[i + j].pc = pc;
314 				hits[i + j].hits = nr_hits;
315 				goto out;
316 			}
317 		}
318 		i = (i + secondary) & (NR_PROFILE_HIT - 1);
319 	} while (i != primary);
320 
321 	/*
322 	 * Add the current hit(s) and flush the write-queue out
323 	 * to the global buffer:
324 	 */
325 	atomic_add(nr_hits, &prof_buffer[pc]);
326 	for (i = 0; i < NR_PROFILE_HIT; ++i) {
327 		atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
328 		hits[i].pc = hits[i].hits = 0;
329 	}
330 out:
331 	local_irq_restore(flags);
332 	put_cpu();
333 }
334 
335 static int __devinit profile_cpu_callback(struct notifier_block *info,
336 					unsigned long action, void *__cpu)
337 {
338 	int node, cpu = (unsigned long)__cpu;
339 	struct page *page;
340 
341 	switch (action) {
342 	case CPU_UP_PREPARE:
343 	case CPU_UP_PREPARE_FROZEN:
344 		node = cpu_to_node(cpu);
345 		per_cpu(cpu_profile_flip, cpu) = 0;
346 		if (!per_cpu(cpu_profile_hits, cpu)[1]) {
347 			page = alloc_pages_node(node,
348 					GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
349 					0);
350 			if (!page)
351 				return NOTIFY_BAD;
352 			per_cpu(cpu_profile_hits, cpu)[1] = page_address(page);
353 		}
354 		if (!per_cpu(cpu_profile_hits, cpu)[0]) {
355 			page = alloc_pages_node(node,
356 					GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
357 					0);
358 			if (!page)
359 				goto out_free;
360 			per_cpu(cpu_profile_hits, cpu)[0] = page_address(page);
361 		}
362 		break;
363 	out_free:
364 		page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[1]);
365 		per_cpu(cpu_profile_hits, cpu)[1] = NULL;
366 		__free_page(page);
367 		return NOTIFY_BAD;
368 	case CPU_ONLINE:
369 	case CPU_ONLINE_FROZEN:
370 		cpu_set(cpu, prof_cpu_mask);
371 		break;
372 	case CPU_UP_CANCELED:
373 	case CPU_UP_CANCELED_FROZEN:
374 	case CPU_DEAD:
375 	case CPU_DEAD_FROZEN:
376 		cpu_clear(cpu, prof_cpu_mask);
377 		if (per_cpu(cpu_profile_hits, cpu)[0]) {
378 			page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[0]);
379 			per_cpu(cpu_profile_hits, cpu)[0] = NULL;
380 			__free_page(page);
381 		}
382 		if (per_cpu(cpu_profile_hits, cpu)[1]) {
383 			page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[1]);
384 			per_cpu(cpu_profile_hits, cpu)[1] = NULL;
385 			__free_page(page);
386 		}
387 		break;
388 	}
389 	return NOTIFY_OK;
390 }
391 #else /* !CONFIG_SMP */
392 #define profile_flip_buffers()		do { } while (0)
393 #define profile_discard_flip_buffers()	do { } while (0)
394 #define profile_cpu_callback		NULL
395 
396 void profile_hits(int type, void *__pc, unsigned int nr_hits)
397 {
398 	unsigned long pc;
399 
400 	if (prof_on != type || !prof_buffer)
401 		return;
402 	pc = ((unsigned long)__pc - (unsigned long)_stext) >> prof_shift;
403 	atomic_add(nr_hits, &prof_buffer[min(pc, prof_len - 1)]);
404 }
405 #endif /* !CONFIG_SMP */
406 
407 EXPORT_SYMBOL_GPL(profile_hits);
408 
409 void profile_tick(int type)
410 {
411 	struct pt_regs *regs = get_irq_regs();
412 
413 	if (type == CPU_PROFILING && timer_hook)
414 		timer_hook(regs);
415 	if (!user_mode(regs) && cpu_isset(smp_processor_id(), prof_cpu_mask))
416 		profile_hit(type, (void *)profile_pc(regs));
417 }
418 
419 #ifdef CONFIG_PROC_FS
420 #include <linux/proc_fs.h>
421 #include <asm/uaccess.h>
422 #include <asm/ptrace.h>
423 
424 static int prof_cpu_mask_read_proc (char *page, char **start, off_t off,
425 			int count, int *eof, void *data)
426 {
427 	int len = cpumask_scnprintf(page, count, *(cpumask_t *)data);
428 	if (count - len < 2)
429 		return -EINVAL;
430 	len += sprintf(page + len, "\n");
431 	return len;
432 }
433 
434 static int prof_cpu_mask_write_proc (struct file *file, const char __user *buffer,
435 					unsigned long count, void *data)
436 {
437 	cpumask_t *mask = (cpumask_t *)data;
438 	unsigned long full_count = count, err;
439 	cpumask_t new_value;
440 
441 	err = cpumask_parse_user(buffer, count, new_value);
442 	if (err)
443 		return err;
444 
445 	*mask = new_value;
446 	return full_count;
447 }
448 
449 void create_prof_cpu_mask(struct proc_dir_entry *root_irq_dir)
450 {
451 	struct proc_dir_entry *entry;
452 
453 	/* create /proc/irq/prof_cpu_mask */
454 	if (!(entry = create_proc_entry("prof_cpu_mask", 0600, root_irq_dir)))
455 		return;
456 	entry->data = (void *)&prof_cpu_mask;
457 	entry->read_proc = prof_cpu_mask_read_proc;
458 	entry->write_proc = prof_cpu_mask_write_proc;
459 }
460 
461 /*
462  * This function accesses profiling information. The returned data is
463  * binary: the sampling step and the actual contents of the profile
464  * buffer. Use of the program readprofile is recommended in order to
465  * get meaningful info out of these data.
466  */
467 static ssize_t
468 read_profile(struct file *file, char __user *buf, size_t count, loff_t *ppos)
469 {
470 	unsigned long p = *ppos;
471 	ssize_t read;
472 	char * pnt;
473 	unsigned int sample_step = 1 << prof_shift;
474 
475 	profile_flip_buffers();
476 	if (p >= (prof_len+1)*sizeof(unsigned int))
477 		return 0;
478 	if (count > (prof_len+1)*sizeof(unsigned int) - p)
479 		count = (prof_len+1)*sizeof(unsigned int) - p;
480 	read = 0;
481 
482 	while (p < sizeof(unsigned int) && count > 0) {
483 		if (put_user(*((char *)(&sample_step)+p),buf))
484 			return -EFAULT;
485 		buf++; p++; count--; read++;
486 	}
487 	pnt = (char *)prof_buffer + p - sizeof(atomic_t);
488 	if (copy_to_user(buf,(void *)pnt,count))
489 		return -EFAULT;
490 	read += count;
491 	*ppos += read;
492 	return read;
493 }
494 
495 /*
496  * Writing to /proc/profile resets the counters
497  *
498  * Writing a 'profiling multiplier' value into it also re-sets the profiling
499  * interrupt frequency, on architectures that support this.
500  */
501 static ssize_t write_profile(struct file *file, const char __user *buf,
502 			     size_t count, loff_t *ppos)
503 {
504 #ifdef CONFIG_SMP
505 	extern int setup_profiling_timer (unsigned int multiplier);
506 
507 	if (count == sizeof(int)) {
508 		unsigned int multiplier;
509 
510 		if (copy_from_user(&multiplier, buf, sizeof(int)))
511 			return -EFAULT;
512 
513 		if (setup_profiling_timer(multiplier))
514 			return -EINVAL;
515 	}
516 #endif
517 	profile_discard_flip_buffers();
518 	memset(prof_buffer, 0, prof_len * sizeof(atomic_t));
519 	return count;
520 }
521 
522 static const struct file_operations proc_profile_operations = {
523 	.read		= read_profile,
524 	.write		= write_profile,
525 };
526 
527 #ifdef CONFIG_SMP
528 static void __init profile_nop(void *unused)
529 {
530 }
531 
532 static int __init create_hash_tables(void)
533 {
534 	int cpu;
535 
536 	for_each_online_cpu(cpu) {
537 		int node = cpu_to_node(cpu);
538 		struct page *page;
539 
540 		page = alloc_pages_node(node,
541 				GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
542 				0);
543 		if (!page)
544 			goto out_cleanup;
545 		per_cpu(cpu_profile_hits, cpu)[1]
546 				= (struct profile_hit *)page_address(page);
547 		page = alloc_pages_node(node,
548 				GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
549 				0);
550 		if (!page)
551 			goto out_cleanup;
552 		per_cpu(cpu_profile_hits, cpu)[0]
553 				= (struct profile_hit *)page_address(page);
554 	}
555 	return 0;
556 out_cleanup:
557 	prof_on = 0;
558 	smp_mb();
559 	on_each_cpu(profile_nop, NULL, 0, 1);
560 	for_each_online_cpu(cpu) {
561 		struct page *page;
562 
563 		if (per_cpu(cpu_profile_hits, cpu)[0]) {
564 			page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[0]);
565 			per_cpu(cpu_profile_hits, cpu)[0] = NULL;
566 			__free_page(page);
567 		}
568 		if (per_cpu(cpu_profile_hits, cpu)[1]) {
569 			page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[1]);
570 			per_cpu(cpu_profile_hits, cpu)[1] = NULL;
571 			__free_page(page);
572 		}
573 	}
574 	return -1;
575 }
576 #else
577 #define create_hash_tables()			({ 0; })
578 #endif
579 
580 static int __init create_proc_profile(void)
581 {
582 	struct proc_dir_entry *entry;
583 
584 	if (!prof_on)
585 		return 0;
586 	if (create_hash_tables())
587 		return -1;
588 	if (!(entry = create_proc_entry("profile", S_IWUSR | S_IRUGO, NULL)))
589 		return 0;
590 	entry->proc_fops = &proc_profile_operations;
591 	entry->size = (1+prof_len) * sizeof(atomic_t);
592 	hotcpu_notifier(profile_cpu_callback, 0);
593 	return 0;
594 }
595 module_init(create_proc_profile);
596 #endif /* CONFIG_PROC_FS */
597