1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2019 Facebook 3 #include "vmlinux.h" 4 #include <bpf/bpf_helpers.h> 5 #include "runqslower.h" 6 7 #define TASK_RUNNING 0 8 9 const volatile __u64 min_us = 0; 10 const volatile pid_t targ_pid = 0; 11 12 struct { 13 __uint(type, BPF_MAP_TYPE_HASH); 14 __uint(max_entries, 10240); 15 __type(key, u32); 16 __type(value, u64); 17 } start SEC(".maps"); 18 19 struct { 20 __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); 21 __uint(key_size, sizeof(u32)); 22 __uint(value_size, sizeof(u32)); 23 } events SEC(".maps"); 24 25 /* record enqueue timestamp */ 26 __always_inline 27 static int trace_enqueue(u32 tgid, u32 pid) 28 { 29 u64 ts; 30 31 if (!pid || (targ_pid && targ_pid != pid)) 32 return 0; 33 34 ts = bpf_ktime_get_ns(); 35 bpf_map_update_elem(&start, &pid, &ts, 0); 36 return 0; 37 } 38 39 SEC("tp_btf/sched_wakeup") 40 int handle__sched_wakeup(u64 *ctx) 41 { 42 /* TP_PROTO(struct task_struct *p) */ 43 struct task_struct *p = (void *)ctx[0]; 44 45 return trace_enqueue(p->tgid, p->pid); 46 } 47 48 SEC("tp_btf/sched_wakeup_new") 49 int handle__sched_wakeup_new(u64 *ctx) 50 { 51 /* TP_PROTO(struct task_struct *p) */ 52 struct task_struct *p = (void *)ctx[0]; 53 54 return trace_enqueue(p->tgid, p->pid); 55 } 56 57 SEC("tp_btf/sched_switch") 58 int handle__sched_switch(u64 *ctx) 59 { 60 /* TP_PROTO(bool preempt, struct task_struct *prev, 61 * struct task_struct *next) 62 */ 63 struct task_struct *prev = (struct task_struct *)ctx[1]; 64 struct task_struct *next = (struct task_struct *)ctx[2]; 65 struct event event = {}; 66 u64 *tsp, delta_us; 67 long state; 68 u32 pid; 69 70 /* ivcsw: treat like an enqueue event and store timestamp */ 71 if (prev->state == TASK_RUNNING) 72 trace_enqueue(prev->tgid, prev->pid); 73 74 pid = next->pid; 75 76 /* fetch timestamp and calculate delta */ 77 tsp = bpf_map_lookup_elem(&start, &pid); 78 if (!tsp) 79 return 0; /* missed enqueue */ 80 81 delta_us = (bpf_ktime_get_ns() - *tsp) / 1000; 82 if (min_us && delta_us <= min_us) 83 return 0; 84 85 event.pid = pid; 86 event.delta_us = delta_us; 87 bpf_get_current_comm(&event.task, sizeof(event.task)); 88 89 /* output */ 90 bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, 91 &event, sizeof(event)); 92 93 bpf_map_delete_elem(&start, &pid); 94 return 0; 95 } 96 97 char LICENSE[] SEC("license") = "GPL"; 98