1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2019 Facebook 3 #include "vmlinux.h" 4 #include <bpf/bpf_helpers.h> 5 #include "runqslower.h" 6 7 #define TASK_RUNNING 0 8 #define BPF_F_CURRENT_CPU 0xffffffffULL 9 10 const volatile __u64 min_us = 0; 11 const volatile pid_t targ_pid = 0; 12 13 struct { 14 __uint(type, BPF_MAP_TYPE_HASH); 15 __uint(max_entries, 10240); 16 __type(key, u32); 17 __type(value, u64); 18 } start SEC(".maps"); 19 20 struct { 21 __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); 22 __uint(key_size, sizeof(u32)); 23 __uint(value_size, sizeof(u32)); 24 } events SEC(".maps"); 25 26 /* record enqueue timestamp */ 27 __always_inline 28 static int trace_enqueue(u32 tgid, u32 pid) 29 { 30 u64 ts; 31 32 if (!pid || (targ_pid && targ_pid != pid)) 33 return 0; 34 35 ts = bpf_ktime_get_ns(); 36 bpf_map_update_elem(&start, &pid, &ts, 0); 37 return 0; 38 } 39 40 SEC("tp_btf/sched_wakeup") 41 int handle__sched_wakeup(u64 *ctx) 42 { 43 /* TP_PROTO(struct task_struct *p) */ 44 struct task_struct *p = (void *)ctx[0]; 45 46 return trace_enqueue(p->tgid, p->pid); 47 } 48 49 SEC("tp_btf/sched_wakeup_new") 50 int handle__sched_wakeup_new(u64 *ctx) 51 { 52 /* TP_PROTO(struct task_struct *p) */ 53 struct task_struct *p = (void *)ctx[0]; 54 55 return trace_enqueue(p->tgid, p->pid); 56 } 57 58 SEC("tp_btf/sched_switch") 59 int handle__sched_switch(u64 *ctx) 60 { 61 /* TP_PROTO(bool preempt, struct task_struct *prev, 62 * struct task_struct *next) 63 */ 64 struct task_struct *prev = (struct task_struct *)ctx[1]; 65 struct task_struct *next = (struct task_struct *)ctx[2]; 66 struct event event = {}; 67 u64 *tsp, delta_us; 68 long state; 69 u32 pid; 70 71 /* ivcsw: treat like an enqueue event and store timestamp */ 72 if (prev->state == TASK_RUNNING) 73 trace_enqueue(prev->tgid, prev->pid); 74 75 pid = next->pid; 76 77 /* fetch timestamp and calculate delta */ 78 tsp = bpf_map_lookup_elem(&start, &pid); 79 if (!tsp) 80 return 0; /* missed enqueue */ 81 82 delta_us = (bpf_ktime_get_ns() - *tsp) / 1000; 83 if (min_us && delta_us <= min_us) 84 return 0; 85 86 event.pid = pid; 87 event.delta_us = delta_us; 88 bpf_get_current_comm(&event.task, sizeof(event.task)); 89 90 /* output */ 91 bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, 92 &event, sizeof(event)); 93 94 bpf_map_delete_elem(&start, &pid); 95 return 0; 96 } 97 98 char LICENSE[] SEC("license") = "GPL"; 99