1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2019 Facebook 3 #include "vmlinux.h" 4 #include <bpf_helpers.h> 5 #include "runqslower.h" 6 7 #define TASK_RUNNING 0 8 9 #define BPF_F_INDEX_MASK 0xffffffffULL 10 #define BPF_F_CURRENT_CPU BPF_F_INDEX_MASK 11 12 const volatile __u64 min_us = 0; 13 const volatile pid_t targ_pid = 0; 14 15 struct { 16 __uint(type, BPF_MAP_TYPE_HASH); 17 __uint(max_entries, 10240); 18 __type(key, u32); 19 __type(value, u64); 20 } start SEC(".maps"); 21 22 struct { 23 __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); 24 __uint(key_size, sizeof(u32)); 25 __uint(value_size, sizeof(u32)); 26 } events SEC(".maps"); 27 28 /* record enqueue timestamp */ 29 __always_inline 30 static int trace_enqueue(u32 tgid, u32 pid) 31 { 32 u64 ts; 33 34 if (!pid || (targ_pid && targ_pid != pid)) 35 return 0; 36 37 ts = bpf_ktime_get_ns(); 38 bpf_map_update_elem(&start, &pid, &ts, 0); 39 return 0; 40 } 41 42 SEC("tp_btf/sched_wakeup") 43 int handle__sched_wakeup(u64 *ctx) 44 { 45 /* TP_PROTO(struct task_struct *p) */ 46 struct task_struct *p = (void *)ctx[0]; 47 48 return trace_enqueue(p->tgid, p->pid); 49 } 50 51 SEC("tp_btf/sched_wakeup_new") 52 int handle__sched_wakeup_new(u64 *ctx) 53 { 54 /* TP_PROTO(struct task_struct *p) */ 55 struct task_struct *p = (void *)ctx[0]; 56 57 return trace_enqueue(p->tgid, p->pid); 58 } 59 60 SEC("tp_btf/sched_switch") 61 int handle__sched_switch(u64 *ctx) 62 { 63 /* TP_PROTO(bool preempt, struct task_struct *prev, 64 * struct task_struct *next) 65 */ 66 struct task_struct *prev = (struct task_struct *)ctx[1]; 67 struct task_struct *next = (struct task_struct *)ctx[2]; 68 struct event event = {}; 69 u64 *tsp, delta_us; 70 long state; 71 u32 pid; 72 73 /* ivcsw: treat like an enqueue event and store timestamp */ 74 if (prev->state == TASK_RUNNING) 75 trace_enqueue(prev->tgid, prev->pid); 76 77 pid = next->pid; 78 79 /* fetch timestamp and calculate delta */ 80 tsp = bpf_map_lookup_elem(&start, &pid); 81 if (!tsp) 82 return 0; /* missed enqueue */ 83 84 delta_us = (bpf_ktime_get_ns() - *tsp) / 1000; 85 if (min_us && delta_us <= min_us) 86 return 0; 87 88 event.pid = pid; 89 event.delta_us = delta_us; 90 bpf_get_current_comm(&event.task, sizeof(event.task)); 91 92 /* output */ 93 bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, 94 &event, sizeof(event)); 95 96 bpf_map_delete_elem(&start, &pid); 97 return 0; 98 } 99 100 char LICENSE[] SEC("license") = "GPL"; 101