1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 // Copyright (c) 2022 Google 3 #include "vmlinux.h" 4 #include <bpf/bpf_helpers.h> 5 #include <bpf/bpf_tracing.h> 6 #include <bpf/bpf_core_read.h> 7 8 /* maximum stack trace depth */ 9 #define MAX_STACKS 8 10 11 /* default buffer size */ 12 #define MAX_ENTRIES 10240 13 14 struct contention_key { 15 __s32 stack_id; 16 }; 17 18 struct contention_data { 19 __u64 total_time; 20 __u64 min_time; 21 __u64 max_time; 22 __u32 count; 23 __u32 flags; 24 }; 25 26 struct tstamp_data { 27 __u64 timestamp; 28 __u64 lock; 29 __u32 flags; 30 __s32 stack_id; 31 }; 32 33 /* callstack storage */ 34 struct { 35 __uint(type, BPF_MAP_TYPE_STACK_TRACE); 36 __uint(key_size, sizeof(__u32)); 37 __uint(value_size, MAX_STACKS * sizeof(__u64)); 38 __uint(max_entries, MAX_ENTRIES); 39 } stacks SEC(".maps"); 40 41 /* maintain timestamp at the beginning of contention */ 42 struct { 43 __uint(type, BPF_MAP_TYPE_HASH); 44 __type(key, int); 45 __type(value, struct tstamp_data); 46 __uint(max_entries, MAX_ENTRIES); 47 } tstamp SEC(".maps"); 48 49 /* actual lock contention statistics */ 50 struct { 51 __uint(type, BPF_MAP_TYPE_HASH); 52 __uint(key_size, sizeof(struct contention_key)); 53 __uint(value_size, sizeof(struct contention_data)); 54 __uint(max_entries, MAX_ENTRIES); 55 } lock_stat SEC(".maps"); 56 57 struct { 58 __uint(type, BPF_MAP_TYPE_HASH); 59 __uint(key_size, sizeof(__u32)); 60 __uint(value_size, sizeof(__u8)); 61 __uint(max_entries, 1); 62 } cpu_filter SEC(".maps"); 63 64 struct { 65 __uint(type, BPF_MAP_TYPE_HASH); 66 __uint(key_size, sizeof(__u32)); 67 __uint(value_size, sizeof(__u8)); 68 __uint(max_entries, 1); 69 } task_filter SEC(".maps"); 70 71 /* control flags */ 72 int enabled; 73 int has_cpu; 74 int has_task; 75 int stack_skip; 76 77 /* error stat */ 78 int lost; 79 80 static inline int can_record(void) 81 { 82 if (has_cpu) { 83 __u32 cpu = bpf_get_smp_processor_id(); 84 __u8 *ok; 85 86 ok = bpf_map_lookup_elem(&cpu_filter, &cpu); 87 if (!ok) 88 return 0; 89 } 90 91 if (has_task) { 92 __u8 *ok; 93 __u32 pid = bpf_get_current_pid_tgid(); 94 95 ok = bpf_map_lookup_elem(&task_filter, &pid); 96 if (!ok) 97 return 0; 98 } 99 100 return 1; 101 } 102 103 SEC("tp_btf/contention_begin") 104 int contention_begin(u64 *ctx) 105 { 106 __u32 pid; 107 struct tstamp_data *pelem; 108 109 if (!enabled || !can_record()) 110 return 0; 111 112 pid = bpf_get_current_pid_tgid(); 113 pelem = bpf_map_lookup_elem(&tstamp, &pid); 114 if (pelem && pelem->lock) 115 return 0; 116 117 if (pelem == NULL) { 118 struct tstamp_data zero = {}; 119 120 bpf_map_update_elem(&tstamp, &pid, &zero, BPF_ANY); 121 pelem = bpf_map_lookup_elem(&tstamp, &pid); 122 if (pelem == NULL) { 123 lost++; 124 return 0; 125 } 126 } 127 128 pelem->timestamp = bpf_ktime_get_ns(); 129 pelem->lock = (__u64)ctx[0]; 130 pelem->flags = (__u32)ctx[1]; 131 pelem->stack_id = bpf_get_stackid(ctx, &stacks, BPF_F_FAST_STACK_CMP | stack_skip); 132 133 if (pelem->stack_id < 0) 134 lost++; 135 return 0; 136 } 137 138 SEC("tp_btf/contention_end") 139 int contention_end(u64 *ctx) 140 { 141 __u32 pid; 142 struct tstamp_data *pelem; 143 struct contention_key key; 144 struct contention_data *data; 145 __u64 duration; 146 147 if (!enabled) 148 return 0; 149 150 pid = bpf_get_current_pid_tgid(); 151 pelem = bpf_map_lookup_elem(&tstamp, &pid); 152 if (!pelem || pelem->lock != ctx[0]) 153 return 0; 154 155 duration = bpf_ktime_get_ns() - pelem->timestamp; 156 157 key.stack_id = pelem->stack_id; 158 data = bpf_map_lookup_elem(&lock_stat, &key); 159 if (!data) { 160 struct contention_data first = { 161 .total_time = duration, 162 .max_time = duration, 163 .min_time = duration, 164 .count = 1, 165 .flags = pelem->flags, 166 }; 167 168 bpf_map_update_elem(&lock_stat, &key, &first, BPF_NOEXIST); 169 bpf_map_delete_elem(&tstamp, &pid); 170 return 0; 171 } 172 173 __sync_fetch_and_add(&data->total_time, duration); 174 __sync_fetch_and_add(&data->count, 1); 175 176 /* FIXME: need atomic operations */ 177 if (data->max_time < duration) 178 data->max_time = duration; 179 if (data->min_time > duration) 180 data->min_time = duration; 181 182 bpf_map_delete_elem(&tstamp, &pid); 183 return 0; 184 } 185 186 char LICENSE[] SEC("license") = "Dual BSD/GPL"; 187