1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2019 Facebook
3 #include "vmlinux.h"
4 #include <bpf/bpf_helpers.h>
5 #include "runqslower.h"
6 
7 #define TASK_RUNNING 0
8 
9 #define BPF_F_INDEX_MASK		0xffffffffULL
10 #define BPF_F_CURRENT_CPU		BPF_F_INDEX_MASK
11 
12 const volatile __u64 min_us = 0;
13 const volatile pid_t targ_pid = 0;
14 
15 struct {
16 	__uint(type, BPF_MAP_TYPE_HASH);
17 	__uint(max_entries, 10240);
18 	__type(key, u32);
19 	__type(value, u64);
20 } start SEC(".maps");
21 
22 struct {
23 	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
24 	__uint(key_size, sizeof(u32));
25 	__uint(value_size, sizeof(u32));
26 } events SEC(".maps");
27 
28 /* record enqueue timestamp */
29 __always_inline
30 static int trace_enqueue(u32 tgid, u32 pid)
31 {
32 	u64 ts;
33 
34 	if (!pid || (targ_pid && targ_pid != pid))
35 		return 0;
36 
37 	ts = bpf_ktime_get_ns();
38 	bpf_map_update_elem(&start, &pid, &ts, 0);
39 	return 0;
40 }
41 
42 SEC("tp_btf/sched_wakeup")
43 int handle__sched_wakeup(u64 *ctx)
44 {
45 	/* TP_PROTO(struct task_struct *p) */
46 	struct task_struct *p = (void *)ctx[0];
47 
48 	return trace_enqueue(p->tgid, p->pid);
49 }
50 
51 SEC("tp_btf/sched_wakeup_new")
52 int handle__sched_wakeup_new(u64 *ctx)
53 {
54 	/* TP_PROTO(struct task_struct *p) */
55 	struct task_struct *p = (void *)ctx[0];
56 
57 	return trace_enqueue(p->tgid, p->pid);
58 }
59 
60 SEC("tp_btf/sched_switch")
61 int handle__sched_switch(u64 *ctx)
62 {
63 	/* TP_PROTO(bool preempt, struct task_struct *prev,
64 	 *	    struct task_struct *next)
65 	 */
66 	struct task_struct *prev = (struct task_struct *)ctx[1];
67 	struct task_struct *next = (struct task_struct *)ctx[2];
68 	struct event event = {};
69 	u64 *tsp, delta_us;
70 	long state;
71 	u32 pid;
72 
73 	/* ivcsw: treat like an enqueue event and store timestamp */
74 	if (prev->state == TASK_RUNNING)
75 		trace_enqueue(prev->tgid, prev->pid);
76 
77 	pid = next->pid;
78 
79 	/* fetch timestamp and calculate delta */
80 	tsp = bpf_map_lookup_elem(&start, &pid);
81 	if (!tsp)
82 		return 0;   /* missed enqueue */
83 
84 	delta_us = (bpf_ktime_get_ns() - *tsp) / 1000;
85 	if (min_us && delta_us <= min_us)
86 		return 0;
87 
88 	event.pid = pid;
89 	event.delta_us = delta_us;
90 	bpf_get_current_comm(&event.task, sizeof(event.task));
91 
92 	/* output */
93 	bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU,
94 			      &event, sizeof(event));
95 
96 	bpf_map_delete_elem(&start, &pid);
97 	return 0;
98 }
99 
100 char LICENSE[] SEC("license") = "GPL";
101