1 // SPDX-License-Identifier: GPL-2.0 2 #include <perf/evlist.h> 3 #include <perf/evsel.h> 4 #include <linux/bitops.h> 5 #include <linux/list.h> 6 #include <linux/hash.h> 7 #include <sys/ioctl.h> 8 #include <internal/evlist.h> 9 #include <internal/evsel.h> 10 #include <internal/xyarray.h> 11 #include <internal/mmap.h> 12 #include <internal/cpumap.h> 13 #include <internal/threadmap.h> 14 #include <internal/lib.h> 15 #include <linux/zalloc.h> 16 #include <stdlib.h> 17 #include <errno.h> 18 #include <unistd.h> 19 #include <fcntl.h> 20 #include <signal.h> 21 #include <poll.h> 22 #include <sys/mman.h> 23 #include <perf/cpumap.h> 24 #include <perf/threadmap.h> 25 #include <api/fd/array.h> 26 27 void perf_evlist__init(struct perf_evlist *evlist) 28 { 29 INIT_LIST_HEAD(&evlist->entries); 30 evlist->nr_entries = 0; 31 fdarray__init(&evlist->pollfd, 64); 32 perf_evlist__reset_id_hash(evlist); 33 } 34 35 static void __perf_evlist__propagate_maps(struct perf_evlist *evlist, 36 struct perf_evsel *evsel) 37 { 38 /* 39 * We already have cpus for evsel (via PMU sysfs) so 40 * keep it, if there's no target cpu list defined. 41 */ 42 if (!evsel->own_cpus || evlist->has_user_cpus) { 43 perf_cpu_map__put(evsel->cpus); 44 evsel->cpus = perf_cpu_map__get(evlist->cpus); 45 } else if (!evsel->system_wide && perf_cpu_map__empty(evlist->cpus)) { 46 perf_cpu_map__put(evsel->cpus); 47 evsel->cpus = perf_cpu_map__get(evlist->cpus); 48 } else if (evsel->cpus != evsel->own_cpus) { 49 perf_cpu_map__put(evsel->cpus); 50 evsel->cpus = perf_cpu_map__get(evsel->own_cpus); 51 } 52 53 perf_thread_map__put(evsel->threads); 54 evsel->threads = perf_thread_map__get(evlist->threads); 55 evlist->all_cpus = perf_cpu_map__merge(evlist->all_cpus, evsel->cpus); 56 } 57 58 static void perf_evlist__propagate_maps(struct perf_evlist *evlist) 59 { 60 struct perf_evsel *evsel; 61 62 perf_evlist__for_each_evsel(evlist, evsel) 63 __perf_evlist__propagate_maps(evlist, evsel); 64 } 65 66 void perf_evlist__add(struct perf_evlist *evlist, 67 struct perf_evsel *evsel) 68 { 69 list_add_tail(&evsel->node, &evlist->entries); 70 evlist->nr_entries += 1; 71 __perf_evlist__propagate_maps(evlist, evsel); 72 } 73 74 void perf_evlist__remove(struct perf_evlist *evlist, 75 struct perf_evsel *evsel) 76 { 77 list_del_init(&evsel->node); 78 evlist->nr_entries -= 1; 79 } 80 81 struct perf_evlist *perf_evlist__new(void) 82 { 83 struct perf_evlist *evlist = zalloc(sizeof(*evlist)); 84 85 if (evlist != NULL) 86 perf_evlist__init(evlist); 87 88 return evlist; 89 } 90 91 struct perf_evsel * 92 perf_evlist__next(struct perf_evlist *evlist, struct perf_evsel *prev) 93 { 94 struct perf_evsel *next; 95 96 if (!prev) { 97 next = list_first_entry(&evlist->entries, 98 struct perf_evsel, 99 node); 100 } else { 101 next = list_next_entry(prev, node); 102 } 103 104 /* Empty list is noticed here so don't need checking on entry. */ 105 if (&next->node == &evlist->entries) 106 return NULL; 107 108 return next; 109 } 110 111 static void perf_evlist__purge(struct perf_evlist *evlist) 112 { 113 struct perf_evsel *pos, *n; 114 115 perf_evlist__for_each_entry_safe(evlist, n, pos) { 116 list_del_init(&pos->node); 117 perf_evsel__delete(pos); 118 } 119 120 evlist->nr_entries = 0; 121 } 122 123 void perf_evlist__exit(struct perf_evlist *evlist) 124 { 125 perf_cpu_map__put(evlist->cpus); 126 perf_cpu_map__put(evlist->all_cpus); 127 perf_thread_map__put(evlist->threads); 128 evlist->cpus = NULL; 129 evlist->all_cpus = NULL; 130 evlist->threads = NULL; 131 fdarray__exit(&evlist->pollfd); 132 } 133 134 void perf_evlist__delete(struct perf_evlist *evlist) 135 { 136 if (evlist == NULL) 137 return; 138 139 perf_evlist__munmap(evlist); 140 perf_evlist__close(evlist); 141 perf_evlist__purge(evlist); 142 perf_evlist__exit(evlist); 143 free(evlist); 144 } 145 146 void perf_evlist__set_maps(struct perf_evlist *evlist, 147 struct perf_cpu_map *cpus, 148 struct perf_thread_map *threads) 149 { 150 /* 151 * Allow for the possibility that one or another of the maps isn't being 152 * changed i.e. don't put it. Note we are assuming the maps that are 153 * being applied are brand new and evlist is taking ownership of the 154 * original reference count of 1. If that is not the case it is up to 155 * the caller to increase the reference count. 156 */ 157 if (cpus != evlist->cpus) { 158 perf_cpu_map__put(evlist->cpus); 159 evlist->cpus = perf_cpu_map__get(cpus); 160 } 161 162 if (threads != evlist->threads) { 163 perf_thread_map__put(evlist->threads); 164 evlist->threads = perf_thread_map__get(threads); 165 } 166 167 if (!evlist->all_cpus && cpus) 168 evlist->all_cpus = perf_cpu_map__get(cpus); 169 170 perf_evlist__propagate_maps(evlist); 171 } 172 173 int perf_evlist__open(struct perf_evlist *evlist) 174 { 175 struct perf_evsel *evsel; 176 int err; 177 178 perf_evlist__for_each_entry(evlist, evsel) { 179 err = perf_evsel__open(evsel, evsel->cpus, evsel->threads); 180 if (err < 0) 181 goto out_err; 182 } 183 184 return 0; 185 186 out_err: 187 perf_evlist__close(evlist); 188 return err; 189 } 190 191 void perf_evlist__close(struct perf_evlist *evlist) 192 { 193 struct perf_evsel *evsel; 194 195 perf_evlist__for_each_entry_reverse(evlist, evsel) 196 perf_evsel__close(evsel); 197 } 198 199 void perf_evlist__enable(struct perf_evlist *evlist) 200 { 201 struct perf_evsel *evsel; 202 203 perf_evlist__for_each_entry(evlist, evsel) 204 perf_evsel__enable(evsel); 205 } 206 207 void perf_evlist__disable(struct perf_evlist *evlist) 208 { 209 struct perf_evsel *evsel; 210 211 perf_evlist__for_each_entry(evlist, evsel) 212 perf_evsel__disable(evsel); 213 } 214 215 u64 perf_evlist__read_format(struct perf_evlist *evlist) 216 { 217 struct perf_evsel *first = perf_evlist__first(evlist); 218 219 return first->attr.read_format; 220 } 221 222 #define SID(e, x, y) xyarray__entry(e->sample_id, x, y) 223 224 static void perf_evlist__id_hash(struct perf_evlist *evlist, 225 struct perf_evsel *evsel, 226 int cpu, int thread, u64 id) 227 { 228 int hash; 229 struct perf_sample_id *sid = SID(evsel, cpu, thread); 230 231 sid->id = id; 232 sid->evsel = evsel; 233 hash = hash_64(sid->id, PERF_EVLIST__HLIST_BITS); 234 hlist_add_head(&sid->node, &evlist->heads[hash]); 235 } 236 237 void perf_evlist__reset_id_hash(struct perf_evlist *evlist) 238 { 239 int i; 240 241 for (i = 0; i < PERF_EVLIST__HLIST_SIZE; ++i) 242 INIT_HLIST_HEAD(&evlist->heads[i]); 243 } 244 245 void perf_evlist__id_add(struct perf_evlist *evlist, 246 struct perf_evsel *evsel, 247 int cpu, int thread, u64 id) 248 { 249 perf_evlist__id_hash(evlist, evsel, cpu, thread, id); 250 evsel->id[evsel->ids++] = id; 251 } 252 253 int perf_evlist__id_add_fd(struct perf_evlist *evlist, 254 struct perf_evsel *evsel, 255 int cpu, int thread, int fd) 256 { 257 u64 read_data[4] = { 0, }; 258 int id_idx = 1; /* The first entry is the counter value */ 259 u64 id; 260 int ret; 261 262 ret = ioctl(fd, PERF_EVENT_IOC_ID, &id); 263 if (!ret) 264 goto add; 265 266 if (errno != ENOTTY) 267 return -1; 268 269 /* Legacy way to get event id.. All hail to old kernels! */ 270 271 /* 272 * This way does not work with group format read, so bail 273 * out in that case. 274 */ 275 if (perf_evlist__read_format(evlist) & PERF_FORMAT_GROUP) 276 return -1; 277 278 if (!(evsel->attr.read_format & PERF_FORMAT_ID) || 279 read(fd, &read_data, sizeof(read_data)) == -1) 280 return -1; 281 282 if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 283 ++id_idx; 284 if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 285 ++id_idx; 286 287 id = read_data[id_idx]; 288 289 add: 290 perf_evlist__id_add(evlist, evsel, cpu, thread, id); 291 return 0; 292 } 293 294 int perf_evlist__alloc_pollfd(struct perf_evlist *evlist) 295 { 296 int nr_cpus = perf_cpu_map__nr(evlist->cpus); 297 int nr_threads = perf_thread_map__nr(evlist->threads); 298 int nfds = 0; 299 struct perf_evsel *evsel; 300 301 perf_evlist__for_each_entry(evlist, evsel) { 302 if (evsel->system_wide) 303 nfds += nr_cpus; 304 else 305 nfds += nr_cpus * nr_threads; 306 } 307 308 if (fdarray__available_entries(&evlist->pollfd) < nfds && 309 fdarray__grow(&evlist->pollfd, nfds) < 0) 310 return -ENOMEM; 311 312 return 0; 313 } 314 315 int perf_evlist__add_pollfd(struct perf_evlist *evlist, int fd, 316 void *ptr, short revent, enum fdarray_flags flags) 317 { 318 int pos = fdarray__add(&evlist->pollfd, fd, revent | POLLERR | POLLHUP, flags); 319 320 if (pos >= 0) { 321 evlist->pollfd.priv[pos].ptr = ptr; 322 fcntl(fd, F_SETFL, O_NONBLOCK); 323 } 324 325 return pos; 326 } 327 328 static void perf_evlist__munmap_filtered(struct fdarray *fda, int fd, 329 void *arg __maybe_unused) 330 { 331 struct perf_mmap *map = fda->priv[fd].ptr; 332 333 if (map) 334 perf_mmap__put(map); 335 } 336 337 int perf_evlist__filter_pollfd(struct perf_evlist *evlist, short revents_and_mask) 338 { 339 return fdarray__filter(&evlist->pollfd, revents_and_mask, 340 perf_evlist__munmap_filtered, NULL); 341 } 342 343 int perf_evlist__poll(struct perf_evlist *evlist, int timeout) 344 { 345 return fdarray__poll(&evlist->pollfd, timeout); 346 } 347 348 static struct perf_mmap* perf_evlist__alloc_mmap(struct perf_evlist *evlist, bool overwrite) 349 { 350 int i; 351 struct perf_mmap *map; 352 353 map = zalloc(evlist->nr_mmaps * sizeof(struct perf_mmap)); 354 if (!map) 355 return NULL; 356 357 for (i = 0; i < evlist->nr_mmaps; i++) { 358 struct perf_mmap *prev = i ? &map[i - 1] : NULL; 359 360 /* 361 * When the perf_mmap() call is made we grab one refcount, plus 362 * one extra to let perf_mmap__consume() get the last 363 * events after all real references (perf_mmap__get()) are 364 * dropped. 365 * 366 * Each PERF_EVENT_IOC_SET_OUTPUT points to this mmap and 367 * thus does perf_mmap__get() on it. 368 */ 369 perf_mmap__init(&map[i], prev, overwrite, NULL); 370 } 371 372 return map; 373 } 374 375 static void perf_evsel__set_sid_idx(struct perf_evsel *evsel, int idx, int cpu, int thread) 376 { 377 struct perf_sample_id *sid = SID(evsel, cpu, thread); 378 379 sid->idx = idx; 380 sid->cpu = perf_cpu_map__cpu(evsel->cpus, cpu); 381 sid->tid = perf_thread_map__pid(evsel->threads, thread); 382 } 383 384 static struct perf_mmap* 385 perf_evlist__mmap_cb_get(struct perf_evlist *evlist, bool overwrite, int idx) 386 { 387 struct perf_mmap *maps; 388 389 maps = overwrite ? evlist->mmap_ovw : evlist->mmap; 390 391 if (!maps) { 392 maps = perf_evlist__alloc_mmap(evlist, overwrite); 393 if (!maps) 394 return NULL; 395 396 if (overwrite) 397 evlist->mmap_ovw = maps; 398 else 399 evlist->mmap = maps; 400 } 401 402 return &maps[idx]; 403 } 404 405 #define FD(e, x, y) (*(int *) xyarray__entry(e->fd, x, y)) 406 407 static int 408 perf_evlist__mmap_cb_mmap(struct perf_mmap *map, struct perf_mmap_param *mp, 409 int output, int cpu) 410 { 411 return perf_mmap__mmap(map, mp, output, cpu); 412 } 413 414 static void perf_evlist__set_mmap_first(struct perf_evlist *evlist, struct perf_mmap *map, 415 bool overwrite) 416 { 417 if (overwrite) 418 evlist->mmap_ovw_first = map; 419 else 420 evlist->mmap_first = map; 421 } 422 423 static int 424 mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, 425 int idx, struct perf_mmap_param *mp, int cpu_idx, 426 int thread, int *_output, int *_output_overwrite) 427 { 428 int evlist_cpu = perf_cpu_map__cpu(evlist->cpus, cpu_idx); 429 struct perf_evsel *evsel; 430 int revent; 431 432 perf_evlist__for_each_entry(evlist, evsel) { 433 bool overwrite = evsel->attr.write_backward; 434 struct perf_mmap *map; 435 int *output, fd, cpu; 436 437 if (evsel->system_wide && thread) 438 continue; 439 440 cpu = perf_cpu_map__idx(evsel->cpus, evlist_cpu); 441 if (cpu == -1) 442 continue; 443 444 map = ops->get(evlist, overwrite, idx); 445 if (map == NULL) 446 return -ENOMEM; 447 448 if (overwrite) { 449 mp->prot = PROT_READ; 450 output = _output_overwrite; 451 } else { 452 mp->prot = PROT_READ | PROT_WRITE; 453 output = _output; 454 } 455 456 fd = FD(evsel, cpu, thread); 457 458 if (*output == -1) { 459 *output = fd; 460 461 /* 462 * The last one will be done at perf_mmap__consume(), so that we 463 * make sure we don't prevent tools from consuming every last event in 464 * the ring buffer. 465 * 466 * I.e. we can get the POLLHUP meaning that the fd doesn't exist 467 * anymore, but the last events for it are still in the ring buffer, 468 * waiting to be consumed. 469 * 470 * Tools can chose to ignore this at their own discretion, but the 471 * evlist layer can't just drop it when filtering events in 472 * perf_evlist__filter_pollfd(). 473 */ 474 refcount_set(&map->refcnt, 2); 475 476 if (ops->mmap(map, mp, *output, evlist_cpu) < 0) 477 return -1; 478 479 if (!idx) 480 perf_evlist__set_mmap_first(evlist, map, overwrite); 481 } else { 482 if (ioctl(fd, PERF_EVENT_IOC_SET_OUTPUT, *output) != 0) 483 return -1; 484 485 perf_mmap__get(map); 486 } 487 488 revent = !overwrite ? POLLIN : 0; 489 490 if (!evsel->system_wide && 491 perf_evlist__add_pollfd(evlist, fd, map, revent, fdarray_flag__default) < 0) { 492 perf_mmap__put(map); 493 return -1; 494 } 495 496 if (evsel->attr.read_format & PERF_FORMAT_ID) { 497 if (perf_evlist__id_add_fd(evlist, evsel, cpu, thread, 498 fd) < 0) 499 return -1; 500 perf_evsel__set_sid_idx(evsel, idx, cpu, thread); 501 } 502 } 503 504 return 0; 505 } 506 507 static int 508 mmap_per_thread(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, 509 struct perf_mmap_param *mp) 510 { 511 int thread; 512 int nr_threads = perf_thread_map__nr(evlist->threads); 513 514 for (thread = 0; thread < nr_threads; thread++) { 515 int output = -1; 516 int output_overwrite = -1; 517 518 if (ops->idx) 519 ops->idx(evlist, mp, thread, false); 520 521 if (mmap_per_evsel(evlist, ops, thread, mp, 0, thread, 522 &output, &output_overwrite)) 523 goto out_unmap; 524 } 525 526 return 0; 527 528 out_unmap: 529 perf_evlist__munmap(evlist); 530 return -1; 531 } 532 533 static int 534 mmap_per_cpu(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, 535 struct perf_mmap_param *mp) 536 { 537 int nr_threads = perf_thread_map__nr(evlist->threads); 538 int nr_cpus = perf_cpu_map__nr(evlist->cpus); 539 int cpu, thread; 540 541 for (cpu = 0; cpu < nr_cpus; cpu++) { 542 int output = -1; 543 int output_overwrite = -1; 544 545 if (ops->idx) 546 ops->idx(evlist, mp, cpu, true); 547 548 for (thread = 0; thread < nr_threads; thread++) { 549 if (mmap_per_evsel(evlist, ops, cpu, mp, cpu, 550 thread, &output, &output_overwrite)) 551 goto out_unmap; 552 } 553 } 554 555 return 0; 556 557 out_unmap: 558 perf_evlist__munmap(evlist); 559 return -1; 560 } 561 562 static int perf_evlist__nr_mmaps(struct perf_evlist *evlist) 563 { 564 int nr_mmaps; 565 566 nr_mmaps = perf_cpu_map__nr(evlist->cpus); 567 if (perf_cpu_map__empty(evlist->cpus)) 568 nr_mmaps = perf_thread_map__nr(evlist->threads); 569 570 return nr_mmaps; 571 } 572 573 int perf_evlist__mmap_ops(struct perf_evlist *evlist, 574 struct perf_evlist_mmap_ops *ops, 575 struct perf_mmap_param *mp) 576 { 577 struct perf_evsel *evsel; 578 const struct perf_cpu_map *cpus = evlist->cpus; 579 const struct perf_thread_map *threads = evlist->threads; 580 581 if (!ops || !ops->get || !ops->mmap) 582 return -EINVAL; 583 584 mp->mask = evlist->mmap_len - page_size - 1; 585 586 evlist->nr_mmaps = perf_evlist__nr_mmaps(evlist); 587 588 perf_evlist__for_each_entry(evlist, evsel) { 589 if ((evsel->attr.read_format & PERF_FORMAT_ID) && 590 evsel->sample_id == NULL && 591 perf_evsel__alloc_id(evsel, perf_cpu_map__nr(cpus), threads->nr) < 0) 592 return -ENOMEM; 593 } 594 595 if (evlist->pollfd.entries == NULL && perf_evlist__alloc_pollfd(evlist) < 0) 596 return -ENOMEM; 597 598 if (perf_cpu_map__empty(cpus)) 599 return mmap_per_thread(evlist, ops, mp); 600 601 return mmap_per_cpu(evlist, ops, mp); 602 } 603 604 int perf_evlist__mmap(struct perf_evlist *evlist, int pages) 605 { 606 struct perf_mmap_param mp; 607 struct perf_evlist_mmap_ops ops = { 608 .get = perf_evlist__mmap_cb_get, 609 .mmap = perf_evlist__mmap_cb_mmap, 610 }; 611 612 evlist->mmap_len = (pages + 1) * page_size; 613 614 return perf_evlist__mmap_ops(evlist, &ops, &mp); 615 } 616 617 void perf_evlist__munmap(struct perf_evlist *evlist) 618 { 619 int i; 620 621 if (evlist->mmap) { 622 for (i = 0; i < evlist->nr_mmaps; i++) 623 perf_mmap__munmap(&evlist->mmap[i]); 624 } 625 626 if (evlist->mmap_ovw) { 627 for (i = 0; i < evlist->nr_mmaps; i++) 628 perf_mmap__munmap(&evlist->mmap_ovw[i]); 629 } 630 631 zfree(&evlist->mmap); 632 zfree(&evlist->mmap_ovw); 633 } 634 635 struct perf_mmap* 636 perf_evlist__next_mmap(struct perf_evlist *evlist, struct perf_mmap *map, 637 bool overwrite) 638 { 639 if (map) 640 return map->next; 641 642 return overwrite ? evlist->mmap_ovw_first : evlist->mmap_first; 643 } 644