1 // SPDX-License-Identifier: GPL-2.0 2 3 /* Copyright (c) 2019 Facebook */ 4 5 #include <assert.h> 6 #include <limits.h> 7 #include <unistd.h> 8 #include <sys/time.h> 9 #include <sys/resource.h> 10 #include <linux/err.h> 11 #include <linux/zalloc.h> 12 #include <bpf/bpf.h> 13 #include <bpf/btf.h> 14 #include <bpf/libbpf.h> 15 16 #include "bpf_counter.h" 17 #include "counts.h" 18 #include "debug.h" 19 #include "evsel.h" 20 #include "target.h" 21 22 #include "bpf_skel/bpf_prog_profiler.skel.h" 23 24 static inline void *u64_to_ptr(__u64 ptr) 25 { 26 return (void *)(unsigned long)ptr; 27 } 28 29 static void set_max_rlimit(void) 30 { 31 struct rlimit rinf = { RLIM_INFINITY, RLIM_INFINITY }; 32 33 setrlimit(RLIMIT_MEMLOCK, &rinf); 34 } 35 36 static struct bpf_counter *bpf_counter_alloc(void) 37 { 38 struct bpf_counter *counter; 39 40 counter = zalloc(sizeof(*counter)); 41 if (counter) 42 INIT_LIST_HEAD(&counter->list); 43 return counter; 44 } 45 46 static int bpf_program_profiler__destroy(struct evsel *evsel) 47 { 48 struct bpf_counter *counter, *tmp; 49 50 list_for_each_entry_safe(counter, tmp, 51 &evsel->bpf_counter_list, list) { 52 list_del_init(&counter->list); 53 bpf_prog_profiler_bpf__destroy(counter->skel); 54 free(counter); 55 } 56 assert(list_empty(&evsel->bpf_counter_list)); 57 58 return 0; 59 } 60 61 static char *bpf_target_prog_name(int tgt_fd) 62 { 63 struct bpf_prog_info_linear *info_linear; 64 struct bpf_func_info *func_info; 65 const struct btf_type *t; 66 char *name = NULL; 67 struct btf *btf; 68 69 info_linear = bpf_program__get_prog_info_linear( 70 tgt_fd, 1UL << BPF_PROG_INFO_FUNC_INFO); 71 if (IS_ERR_OR_NULL(info_linear)) { 72 pr_debug("failed to get info_linear for prog FD %d\n", tgt_fd); 73 return NULL; 74 } 75 76 if (info_linear->info.btf_id == 0 || 77 btf__get_from_id(info_linear->info.btf_id, &btf)) { 78 pr_debug("prog FD %d doesn't have valid btf\n", tgt_fd); 79 goto out; 80 } 81 82 func_info = u64_to_ptr(info_linear->info.func_info); 83 t = btf__type_by_id(btf, func_info[0].type_id); 84 if (!t) { 85 pr_debug("btf %d doesn't have type %d\n", 86 info_linear->info.btf_id, func_info[0].type_id); 87 goto out; 88 } 89 name = strdup(btf__name_by_offset(btf, t->name_off)); 90 out: 91 free(info_linear); 92 return name; 93 } 94 95 static int bpf_program_profiler_load_one(struct evsel *evsel, u32 prog_id) 96 { 97 struct bpf_prog_profiler_bpf *skel; 98 struct bpf_counter *counter; 99 struct bpf_program *prog; 100 char *prog_name; 101 int prog_fd; 102 int err; 103 104 prog_fd = bpf_prog_get_fd_by_id(prog_id); 105 if (prog_fd < 0) { 106 pr_err("Failed to open fd for bpf prog %u\n", prog_id); 107 return -1; 108 } 109 counter = bpf_counter_alloc(); 110 if (!counter) { 111 close(prog_fd); 112 return -1; 113 } 114 115 skel = bpf_prog_profiler_bpf__open(); 116 if (!skel) { 117 pr_err("Failed to open bpf skeleton\n"); 118 goto err_out; 119 } 120 121 skel->rodata->num_cpu = evsel__nr_cpus(evsel); 122 123 bpf_map__resize(skel->maps.events, evsel__nr_cpus(evsel)); 124 bpf_map__resize(skel->maps.fentry_readings, 1); 125 bpf_map__resize(skel->maps.accum_readings, 1); 126 127 prog_name = bpf_target_prog_name(prog_fd); 128 if (!prog_name) { 129 pr_err("Failed to get program name for bpf prog %u. Does it have BTF?\n", prog_id); 130 goto err_out; 131 } 132 133 bpf_object__for_each_program(prog, skel->obj) { 134 err = bpf_program__set_attach_target(prog, prog_fd, prog_name); 135 if (err) { 136 pr_err("bpf_program__set_attach_target failed.\n" 137 "Does bpf prog %u have BTF?\n", prog_id); 138 goto err_out; 139 } 140 } 141 set_max_rlimit(); 142 err = bpf_prog_profiler_bpf__load(skel); 143 if (err) { 144 pr_err("bpf_prog_profiler_bpf__load failed\n"); 145 goto err_out; 146 } 147 148 assert(skel != NULL); 149 counter->skel = skel; 150 list_add(&counter->list, &evsel->bpf_counter_list); 151 close(prog_fd); 152 return 0; 153 err_out: 154 bpf_prog_profiler_bpf__destroy(skel); 155 free(counter); 156 close(prog_fd); 157 return -1; 158 } 159 160 static int bpf_program_profiler__load(struct evsel *evsel, struct target *target) 161 { 162 char *bpf_str, *bpf_str_, *tok, *saveptr = NULL, *p; 163 u32 prog_id; 164 int ret; 165 166 bpf_str_ = bpf_str = strdup(target->bpf_str); 167 if (!bpf_str) 168 return -1; 169 170 while ((tok = strtok_r(bpf_str, ",", &saveptr)) != NULL) { 171 prog_id = strtoul(tok, &p, 10); 172 if (prog_id == 0 || prog_id == UINT_MAX || 173 (*p != '\0' && *p != ',')) { 174 pr_err("Failed to parse bpf prog ids %s\n", 175 target->bpf_str); 176 return -1; 177 } 178 179 ret = bpf_program_profiler_load_one(evsel, prog_id); 180 if (ret) { 181 bpf_program_profiler__destroy(evsel); 182 free(bpf_str_); 183 return -1; 184 } 185 bpf_str = NULL; 186 } 187 free(bpf_str_); 188 return 0; 189 } 190 191 static int bpf_program_profiler__enable(struct evsel *evsel) 192 { 193 struct bpf_counter *counter; 194 int ret; 195 196 list_for_each_entry(counter, &evsel->bpf_counter_list, list) { 197 assert(counter->skel != NULL); 198 ret = bpf_prog_profiler_bpf__attach(counter->skel); 199 if (ret) { 200 bpf_program_profiler__destroy(evsel); 201 return ret; 202 } 203 } 204 return 0; 205 } 206 207 static int bpf_program_profiler__read(struct evsel *evsel) 208 { 209 // perf_cpu_map uses /sys/devices/system/cpu/online 210 int num_cpu = evsel__nr_cpus(evsel); 211 // BPF_MAP_TYPE_PERCPU_ARRAY uses /sys/devices/system/cpu/possible 212 // Sometimes possible > online, like on a Ryzen 3900X that has 24 213 // threads but its possible showed 0-31 -acme 214 int num_cpu_bpf = libbpf_num_possible_cpus(); 215 struct bpf_perf_event_value values[num_cpu_bpf]; 216 struct bpf_counter *counter; 217 int reading_map_fd; 218 __u32 key = 0; 219 int err, cpu; 220 221 if (list_empty(&evsel->bpf_counter_list)) 222 return -EAGAIN; 223 224 for (cpu = 0; cpu < num_cpu; cpu++) { 225 perf_counts(evsel->counts, cpu, 0)->val = 0; 226 perf_counts(evsel->counts, cpu, 0)->ena = 0; 227 perf_counts(evsel->counts, cpu, 0)->run = 0; 228 } 229 list_for_each_entry(counter, &evsel->bpf_counter_list, list) { 230 struct bpf_prog_profiler_bpf *skel = counter->skel; 231 232 assert(skel != NULL); 233 reading_map_fd = bpf_map__fd(skel->maps.accum_readings); 234 235 err = bpf_map_lookup_elem(reading_map_fd, &key, values); 236 if (err) { 237 pr_err("failed to read value\n"); 238 return err; 239 } 240 241 for (cpu = 0; cpu < num_cpu; cpu++) { 242 perf_counts(evsel->counts, cpu, 0)->val += values[cpu].counter; 243 perf_counts(evsel->counts, cpu, 0)->ena += values[cpu].enabled; 244 perf_counts(evsel->counts, cpu, 0)->run += values[cpu].running; 245 } 246 } 247 return 0; 248 } 249 250 static int bpf_program_profiler__install_pe(struct evsel *evsel, int cpu, 251 int fd) 252 { 253 struct bpf_prog_profiler_bpf *skel; 254 struct bpf_counter *counter; 255 int ret; 256 257 list_for_each_entry(counter, &evsel->bpf_counter_list, list) { 258 skel = counter->skel; 259 assert(skel != NULL); 260 261 ret = bpf_map_update_elem(bpf_map__fd(skel->maps.events), 262 &cpu, &fd, BPF_ANY); 263 if (ret) 264 return ret; 265 } 266 return 0; 267 } 268 269 struct bpf_counter_ops bpf_program_profiler_ops = { 270 .load = bpf_program_profiler__load, 271 .enable = bpf_program_profiler__enable, 272 .read = bpf_program_profiler__read, 273 .destroy = bpf_program_profiler__destroy, 274 .install_pe = bpf_program_profiler__install_pe, 275 }; 276 277 int bpf_counter__install_pe(struct evsel *evsel, int cpu, int fd) 278 { 279 if (list_empty(&evsel->bpf_counter_list)) 280 return 0; 281 return evsel->bpf_counter_ops->install_pe(evsel, cpu, fd); 282 } 283 284 int bpf_counter__load(struct evsel *evsel, struct target *target) 285 { 286 if (target__has_bpf(target)) 287 evsel->bpf_counter_ops = &bpf_program_profiler_ops; 288 289 if (evsel->bpf_counter_ops) 290 return evsel->bpf_counter_ops->load(evsel, target); 291 return 0; 292 } 293 294 int bpf_counter__enable(struct evsel *evsel) 295 { 296 if (list_empty(&evsel->bpf_counter_list)) 297 return 0; 298 return evsel->bpf_counter_ops->enable(evsel); 299 } 300 301 int bpf_counter__read(struct evsel *evsel) 302 { 303 if (list_empty(&evsel->bpf_counter_list)) 304 return -EAGAIN; 305 return evsel->bpf_counter_ops->read(evsel); 306 } 307 308 void bpf_counter__destroy(struct evsel *evsel) 309 { 310 if (list_empty(&evsel->bpf_counter_list)) 311 return; 312 evsel->bpf_counter_ops->destroy(evsel); 313 evsel->bpf_counter_ops = NULL; 314 } 315