1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 // Copyright (c) 2022 Google 3 #include "vmlinux.h" 4 #include <bpf/bpf_helpers.h> 5 #include <bpf/bpf_tracing.h> 6 #include <bpf/bpf_core_read.h> 7 8 /* maximum stack trace depth */ 9 #define MAX_STACKS 8 10 11 /* default buffer size */ 12 #define MAX_ENTRIES 10240 13 14 struct contention_key { 15 __s32 stack_id; 16 }; 17 18 struct contention_data { 19 __u64 total_time; 20 __u64 min_time; 21 __u64 max_time; 22 __u32 count; 23 __u32 flags; 24 }; 25 26 struct tstamp_data { 27 __u64 timestamp; 28 __u64 lock; 29 __u32 flags; 30 __s32 stack_id; 31 }; 32 33 /* callstack storage */ 34 struct { 35 __uint(type, BPF_MAP_TYPE_STACK_TRACE); 36 __uint(key_size, sizeof(__u32)); 37 __uint(value_size, MAX_STACKS * sizeof(__u64)); 38 __uint(max_entries, MAX_ENTRIES); 39 } stacks SEC(".maps"); 40 41 /* maintain timestamp at the beginning of contention */ 42 struct { 43 __uint(type, BPF_MAP_TYPE_TASK_STORAGE); 44 __uint(map_flags, BPF_F_NO_PREALLOC); 45 __type(key, int); 46 __type(value, struct tstamp_data); 47 } tstamp SEC(".maps"); 48 49 /* actual lock contention statistics */ 50 struct { 51 __uint(type, BPF_MAP_TYPE_HASH); 52 __uint(key_size, sizeof(struct contention_key)); 53 __uint(value_size, sizeof(struct contention_data)); 54 __uint(max_entries, MAX_ENTRIES); 55 } lock_stat SEC(".maps"); 56 57 struct { 58 __uint(type, BPF_MAP_TYPE_HASH); 59 __uint(key_size, sizeof(__u32)); 60 __uint(value_size, sizeof(__u8)); 61 __uint(max_entries, 1); 62 } cpu_filter SEC(".maps"); 63 64 struct { 65 __uint(type, BPF_MAP_TYPE_HASH); 66 __uint(key_size, sizeof(__u32)); 67 __uint(value_size, sizeof(__u8)); 68 __uint(max_entries, 1); 69 } task_filter SEC(".maps"); 70 71 /* control flags */ 72 int enabled; 73 int has_cpu; 74 int has_task; 75 76 /* error stat */ 77 unsigned long lost; 78 79 static inline int can_record(void) 80 { 81 if (has_cpu) { 82 __u32 cpu = bpf_get_smp_processor_id(); 83 __u8 *ok; 84 85 ok = bpf_map_lookup_elem(&cpu_filter, &cpu); 86 if (!ok) 87 return 0; 88 } 89 90 if (has_task) { 91 __u8 *ok; 92 __u32 pid = bpf_get_current_pid_tgid(); 93 94 ok = bpf_map_lookup_elem(&task_filter, &pid); 95 if (!ok) 96 return 0; 97 } 98 99 return 1; 100 } 101 102 SEC("tp_btf/contention_begin") 103 int contention_begin(u64 *ctx) 104 { 105 struct task_struct *curr; 106 struct tstamp_data *pelem; 107 108 if (!enabled || !can_record()) 109 return 0; 110 111 curr = bpf_get_current_task_btf(); 112 pelem = bpf_task_storage_get(&tstamp, curr, NULL, 113 BPF_LOCAL_STORAGE_GET_F_CREATE); 114 if (!pelem || pelem->lock) 115 return 0; 116 117 pelem->timestamp = bpf_ktime_get_ns(); 118 pelem->lock = (__u64)ctx[0]; 119 pelem->flags = (__u32)ctx[1]; 120 pelem->stack_id = bpf_get_stackid(ctx, &stacks, BPF_F_FAST_STACK_CMP); 121 122 if (pelem->stack_id < 0) 123 lost++; 124 return 0; 125 } 126 127 SEC("tp_btf/contention_end") 128 int contention_end(u64 *ctx) 129 { 130 struct task_struct *curr; 131 struct tstamp_data *pelem; 132 struct contention_key key; 133 struct contention_data *data; 134 __u64 duration; 135 136 if (!enabled) 137 return 0; 138 139 curr = bpf_get_current_task_btf(); 140 pelem = bpf_task_storage_get(&tstamp, curr, NULL, 0); 141 if (!pelem || pelem->lock != ctx[0]) 142 return 0; 143 144 duration = bpf_ktime_get_ns() - pelem->timestamp; 145 146 key.stack_id = pelem->stack_id; 147 data = bpf_map_lookup_elem(&lock_stat, &key); 148 if (!data) { 149 struct contention_data first = { 150 .total_time = duration, 151 .max_time = duration, 152 .min_time = duration, 153 .count = 1, 154 .flags = pelem->flags, 155 }; 156 157 bpf_map_update_elem(&lock_stat, &key, &first, BPF_NOEXIST); 158 pelem->lock = 0; 159 return 0; 160 } 161 162 __sync_fetch_and_add(&data->total_time, duration); 163 __sync_fetch_and_add(&data->count, 1); 164 165 /* FIXME: need atomic operations */ 166 if (data->max_time < duration) 167 data->max_time = duration; 168 if (data->min_time > duration) 169 data->min_time = duration; 170 171 pelem->lock = 0; 172 return 0; 173 } 174 175 char LICENSE[] SEC("license") = "Dual BSD/GPL"; 176