1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2019 Facebook
3 #include "vmlinux.h"
4 #include <bpf/bpf_helpers.h>
5 #include "runqslower.h"
6 
7 #define TASK_RUNNING 0
8 
9 const volatile __u64 min_us = 0;
10 const volatile pid_t targ_pid = 0;
11 
12 struct {
13 	__uint(type, BPF_MAP_TYPE_HASH);
14 	__uint(max_entries, 10240);
15 	__type(key, u32);
16 	__type(value, u64);
17 } start SEC(".maps");
18 
19 struct {
20 	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
21 	__uint(key_size, sizeof(u32));
22 	__uint(value_size, sizeof(u32));
23 } events SEC(".maps");
24 
25 /* record enqueue timestamp */
26 __always_inline
27 static int trace_enqueue(u32 tgid, u32 pid)
28 {
29 	u64 ts;
30 
31 	if (!pid || (targ_pid && targ_pid != pid))
32 		return 0;
33 
34 	ts = bpf_ktime_get_ns();
35 	bpf_map_update_elem(&start, &pid, &ts, 0);
36 	return 0;
37 }
38 
39 SEC("tp_btf/sched_wakeup")
40 int handle__sched_wakeup(u64 *ctx)
41 {
42 	/* TP_PROTO(struct task_struct *p) */
43 	struct task_struct *p = (void *)ctx[0];
44 
45 	return trace_enqueue(p->tgid, p->pid);
46 }
47 
48 SEC("tp_btf/sched_wakeup_new")
49 int handle__sched_wakeup_new(u64 *ctx)
50 {
51 	/* TP_PROTO(struct task_struct *p) */
52 	struct task_struct *p = (void *)ctx[0];
53 
54 	return trace_enqueue(p->tgid, p->pid);
55 }
56 
57 SEC("tp_btf/sched_switch")
58 int handle__sched_switch(u64 *ctx)
59 {
60 	/* TP_PROTO(bool preempt, struct task_struct *prev,
61 	 *	    struct task_struct *next)
62 	 */
63 	struct task_struct *prev = (struct task_struct *)ctx[1];
64 	struct task_struct *next = (struct task_struct *)ctx[2];
65 	struct event event = {};
66 	u64 *tsp, delta_us;
67 	long state;
68 	u32 pid;
69 
70 	/* ivcsw: treat like an enqueue event and store timestamp */
71 	if (prev->state == TASK_RUNNING)
72 		trace_enqueue(prev->tgid, prev->pid);
73 
74 	pid = next->pid;
75 
76 	/* fetch timestamp and calculate delta */
77 	tsp = bpf_map_lookup_elem(&start, &pid);
78 	if (!tsp)
79 		return 0;   /* missed enqueue */
80 
81 	delta_us = (bpf_ktime_get_ns() - *tsp) / 1000;
82 	if (min_us && delta_us <= min_us)
83 		return 0;
84 
85 	event.pid = pid;
86 	event.delta_us = delta_us;
87 	bpf_get_current_comm(&event.task, sizeof(event.task));
88 
89 	/* output */
90 	bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU,
91 			      &event, sizeof(event));
92 
93 	bpf_map_delete_elem(&start, &pid);
94 	return 0;
95 }
96 
97 char LICENSE[] SEC("license") = "GPL";
98