1 // SPDX-License-Identifier: GPL-2.0 2 #include <perf/evlist.h> 3 #include <perf/evsel.h> 4 #include <linux/bitops.h> 5 #include <linux/list.h> 6 #include <linux/hash.h> 7 #include <sys/ioctl.h> 8 #include <internal/evlist.h> 9 #include <internal/evsel.h> 10 #include <internal/xyarray.h> 11 #include <internal/mmap.h> 12 #include <internal/cpumap.h> 13 #include <internal/threadmap.h> 14 #include <internal/lib.h> 15 #include <linux/zalloc.h> 16 #include <stdlib.h> 17 #include <errno.h> 18 #include <unistd.h> 19 #include <fcntl.h> 20 #include <signal.h> 21 #include <poll.h> 22 #include <sys/mman.h> 23 #include <perf/cpumap.h> 24 #include <perf/threadmap.h> 25 #include <api/fd/array.h> 26 27 void perf_evlist__init(struct perf_evlist *evlist) 28 { 29 INIT_LIST_HEAD(&evlist->entries); 30 evlist->nr_entries = 0; 31 fdarray__init(&evlist->pollfd, 64); 32 perf_evlist__reset_id_hash(evlist); 33 } 34 35 static void __perf_evlist__propagate_maps(struct perf_evlist *evlist, 36 struct perf_evsel *evsel) 37 { 38 /* 39 * We already have cpus for evsel (via PMU sysfs) so 40 * keep it, if there's no target cpu list defined. 41 */ 42 if (!evsel->own_cpus || evlist->has_user_cpus) { 43 perf_cpu_map__put(evsel->cpus); 44 evsel->cpus = perf_cpu_map__get(evlist->cpus); 45 } else if (!evsel->system_wide && perf_cpu_map__empty(evlist->cpus)) { 46 perf_cpu_map__put(evsel->cpus); 47 evsel->cpus = perf_cpu_map__get(evlist->cpus); 48 } else if (evsel->cpus != evsel->own_cpus) { 49 perf_cpu_map__put(evsel->cpus); 50 evsel->cpus = perf_cpu_map__get(evsel->own_cpus); 51 } 52 53 perf_thread_map__put(evsel->threads); 54 evsel->threads = perf_thread_map__get(evlist->threads); 55 evlist->all_cpus = perf_cpu_map__merge(evlist->all_cpus, evsel->cpus); 56 } 57 58 static void perf_evlist__propagate_maps(struct perf_evlist *evlist) 59 { 60 struct perf_evsel *evsel; 61 62 perf_evlist__for_each_evsel(evlist, evsel) 63 __perf_evlist__propagate_maps(evlist, evsel); 64 } 65 66 void perf_evlist__add(struct perf_evlist *evlist, 67 struct perf_evsel *evsel) 68 { 69 evsel->idx = evlist->nr_entries; 70 list_add_tail(&evsel->node, &evlist->entries); 71 evlist->nr_entries += 1; 72 __perf_evlist__propagate_maps(evlist, evsel); 73 } 74 75 void perf_evlist__remove(struct perf_evlist *evlist, 76 struct perf_evsel *evsel) 77 { 78 list_del_init(&evsel->node); 79 evlist->nr_entries -= 1; 80 } 81 82 struct perf_evlist *perf_evlist__new(void) 83 { 84 struct perf_evlist *evlist = zalloc(sizeof(*evlist)); 85 86 if (evlist != NULL) 87 perf_evlist__init(evlist); 88 89 return evlist; 90 } 91 92 struct perf_evsel * 93 perf_evlist__next(struct perf_evlist *evlist, struct perf_evsel *prev) 94 { 95 struct perf_evsel *next; 96 97 if (!prev) { 98 next = list_first_entry(&evlist->entries, 99 struct perf_evsel, 100 node); 101 } else { 102 next = list_next_entry(prev, node); 103 } 104 105 /* Empty list is noticed here so don't need checking on entry. */ 106 if (&next->node == &evlist->entries) 107 return NULL; 108 109 return next; 110 } 111 112 static void perf_evlist__purge(struct perf_evlist *evlist) 113 { 114 struct perf_evsel *pos, *n; 115 116 perf_evlist__for_each_entry_safe(evlist, n, pos) { 117 list_del_init(&pos->node); 118 perf_evsel__delete(pos); 119 } 120 121 evlist->nr_entries = 0; 122 } 123 124 void perf_evlist__exit(struct perf_evlist *evlist) 125 { 126 perf_cpu_map__put(evlist->cpus); 127 perf_cpu_map__put(evlist->all_cpus); 128 perf_thread_map__put(evlist->threads); 129 evlist->cpus = NULL; 130 evlist->all_cpus = NULL; 131 evlist->threads = NULL; 132 fdarray__exit(&evlist->pollfd); 133 } 134 135 void perf_evlist__delete(struct perf_evlist *evlist) 136 { 137 if (evlist == NULL) 138 return; 139 140 perf_evlist__munmap(evlist); 141 perf_evlist__close(evlist); 142 perf_evlist__purge(evlist); 143 perf_evlist__exit(evlist); 144 free(evlist); 145 } 146 147 void perf_evlist__set_maps(struct perf_evlist *evlist, 148 struct perf_cpu_map *cpus, 149 struct perf_thread_map *threads) 150 { 151 /* 152 * Allow for the possibility that one or another of the maps isn't being 153 * changed i.e. don't put it. Note we are assuming the maps that are 154 * being applied are brand new and evlist is taking ownership of the 155 * original reference count of 1. If that is not the case it is up to 156 * the caller to increase the reference count. 157 */ 158 if (cpus != evlist->cpus) { 159 perf_cpu_map__put(evlist->cpus); 160 evlist->cpus = perf_cpu_map__get(cpus); 161 } 162 163 if (threads != evlist->threads) { 164 perf_thread_map__put(evlist->threads); 165 evlist->threads = perf_thread_map__get(threads); 166 } 167 168 if (!evlist->all_cpus && cpus) 169 evlist->all_cpus = perf_cpu_map__get(cpus); 170 171 perf_evlist__propagate_maps(evlist); 172 } 173 174 int perf_evlist__open(struct perf_evlist *evlist) 175 { 176 struct perf_evsel *evsel; 177 int err; 178 179 perf_evlist__for_each_entry(evlist, evsel) { 180 err = perf_evsel__open(evsel, evsel->cpus, evsel->threads); 181 if (err < 0) 182 goto out_err; 183 } 184 185 return 0; 186 187 out_err: 188 perf_evlist__close(evlist); 189 return err; 190 } 191 192 void perf_evlist__close(struct perf_evlist *evlist) 193 { 194 struct perf_evsel *evsel; 195 196 perf_evlist__for_each_entry_reverse(evlist, evsel) 197 perf_evsel__close(evsel); 198 } 199 200 void perf_evlist__enable(struct perf_evlist *evlist) 201 { 202 struct perf_evsel *evsel; 203 204 perf_evlist__for_each_entry(evlist, evsel) 205 perf_evsel__enable(evsel); 206 } 207 208 void perf_evlist__disable(struct perf_evlist *evlist) 209 { 210 struct perf_evsel *evsel; 211 212 perf_evlist__for_each_entry(evlist, evsel) 213 perf_evsel__disable(evsel); 214 } 215 216 u64 perf_evlist__read_format(struct perf_evlist *evlist) 217 { 218 struct perf_evsel *first = perf_evlist__first(evlist); 219 220 return first->attr.read_format; 221 } 222 223 #define SID(e, x, y) xyarray__entry(e->sample_id, x, y) 224 225 static void perf_evlist__id_hash(struct perf_evlist *evlist, 226 struct perf_evsel *evsel, 227 int cpu, int thread, u64 id) 228 { 229 int hash; 230 struct perf_sample_id *sid = SID(evsel, cpu, thread); 231 232 sid->id = id; 233 sid->evsel = evsel; 234 hash = hash_64(sid->id, PERF_EVLIST__HLIST_BITS); 235 hlist_add_head(&sid->node, &evlist->heads[hash]); 236 } 237 238 void perf_evlist__reset_id_hash(struct perf_evlist *evlist) 239 { 240 int i; 241 242 for (i = 0; i < PERF_EVLIST__HLIST_SIZE; ++i) 243 INIT_HLIST_HEAD(&evlist->heads[i]); 244 } 245 246 void perf_evlist__id_add(struct perf_evlist *evlist, 247 struct perf_evsel *evsel, 248 int cpu, int thread, u64 id) 249 { 250 perf_evlist__id_hash(evlist, evsel, cpu, thread, id); 251 evsel->id[evsel->ids++] = id; 252 } 253 254 int perf_evlist__id_add_fd(struct perf_evlist *evlist, 255 struct perf_evsel *evsel, 256 int cpu, int thread, int fd) 257 { 258 u64 read_data[4] = { 0, }; 259 int id_idx = 1; /* The first entry is the counter value */ 260 u64 id; 261 int ret; 262 263 ret = ioctl(fd, PERF_EVENT_IOC_ID, &id); 264 if (!ret) 265 goto add; 266 267 if (errno != ENOTTY) 268 return -1; 269 270 /* Legacy way to get event id.. All hail to old kernels! */ 271 272 /* 273 * This way does not work with group format read, so bail 274 * out in that case. 275 */ 276 if (perf_evlist__read_format(evlist) & PERF_FORMAT_GROUP) 277 return -1; 278 279 if (!(evsel->attr.read_format & PERF_FORMAT_ID) || 280 read(fd, &read_data, sizeof(read_data)) == -1) 281 return -1; 282 283 if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 284 ++id_idx; 285 if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 286 ++id_idx; 287 288 id = read_data[id_idx]; 289 290 add: 291 perf_evlist__id_add(evlist, evsel, cpu, thread, id); 292 return 0; 293 } 294 295 int perf_evlist__alloc_pollfd(struct perf_evlist *evlist) 296 { 297 int nr_cpus = perf_cpu_map__nr(evlist->cpus); 298 int nr_threads = perf_thread_map__nr(evlist->threads); 299 int nfds = 0; 300 struct perf_evsel *evsel; 301 302 perf_evlist__for_each_entry(evlist, evsel) { 303 if (evsel->system_wide) 304 nfds += nr_cpus; 305 else 306 nfds += nr_cpus * nr_threads; 307 } 308 309 if (fdarray__available_entries(&evlist->pollfd) < nfds && 310 fdarray__grow(&evlist->pollfd, nfds) < 0) 311 return -ENOMEM; 312 313 return 0; 314 } 315 316 int perf_evlist__add_pollfd(struct perf_evlist *evlist, int fd, 317 void *ptr, short revent, enum fdarray_flags flags) 318 { 319 int pos = fdarray__add(&evlist->pollfd, fd, revent | POLLERR | POLLHUP, flags); 320 321 if (pos >= 0) { 322 evlist->pollfd.priv[pos].ptr = ptr; 323 fcntl(fd, F_SETFL, O_NONBLOCK); 324 } 325 326 return pos; 327 } 328 329 static void perf_evlist__munmap_filtered(struct fdarray *fda, int fd, 330 void *arg __maybe_unused) 331 { 332 struct perf_mmap *map = fda->priv[fd].ptr; 333 334 if (map) 335 perf_mmap__put(map); 336 } 337 338 int perf_evlist__filter_pollfd(struct perf_evlist *evlist, short revents_and_mask) 339 { 340 return fdarray__filter(&evlist->pollfd, revents_and_mask, 341 perf_evlist__munmap_filtered, NULL); 342 } 343 344 int perf_evlist__poll(struct perf_evlist *evlist, int timeout) 345 { 346 return fdarray__poll(&evlist->pollfd, timeout); 347 } 348 349 static struct perf_mmap* perf_evlist__alloc_mmap(struct perf_evlist *evlist, bool overwrite) 350 { 351 int i; 352 struct perf_mmap *map; 353 354 map = zalloc(evlist->nr_mmaps * sizeof(struct perf_mmap)); 355 if (!map) 356 return NULL; 357 358 for (i = 0; i < evlist->nr_mmaps; i++) { 359 struct perf_mmap *prev = i ? &map[i - 1] : NULL; 360 361 /* 362 * When the perf_mmap() call is made we grab one refcount, plus 363 * one extra to let perf_mmap__consume() get the last 364 * events after all real references (perf_mmap__get()) are 365 * dropped. 366 * 367 * Each PERF_EVENT_IOC_SET_OUTPUT points to this mmap and 368 * thus does perf_mmap__get() on it. 369 */ 370 perf_mmap__init(&map[i], prev, overwrite, NULL); 371 } 372 373 return map; 374 } 375 376 static void perf_evsel__set_sid_idx(struct perf_evsel *evsel, int idx, int cpu, int thread) 377 { 378 struct perf_sample_id *sid = SID(evsel, cpu, thread); 379 380 sid->idx = idx; 381 sid->cpu = perf_cpu_map__cpu(evsel->cpus, cpu); 382 sid->tid = perf_thread_map__pid(evsel->threads, thread); 383 } 384 385 static struct perf_mmap* 386 perf_evlist__mmap_cb_get(struct perf_evlist *evlist, bool overwrite, int idx) 387 { 388 struct perf_mmap *maps; 389 390 maps = overwrite ? evlist->mmap_ovw : evlist->mmap; 391 392 if (!maps) { 393 maps = perf_evlist__alloc_mmap(evlist, overwrite); 394 if (!maps) 395 return NULL; 396 397 if (overwrite) 398 evlist->mmap_ovw = maps; 399 else 400 evlist->mmap = maps; 401 } 402 403 return &maps[idx]; 404 } 405 406 #define FD(e, x, y) (*(int *) xyarray__entry(e->fd, x, y)) 407 408 static int 409 perf_evlist__mmap_cb_mmap(struct perf_mmap *map, struct perf_mmap_param *mp, 410 int output, int cpu) 411 { 412 return perf_mmap__mmap(map, mp, output, cpu); 413 } 414 415 static void perf_evlist__set_mmap_first(struct perf_evlist *evlist, struct perf_mmap *map, 416 bool overwrite) 417 { 418 if (overwrite) 419 evlist->mmap_ovw_first = map; 420 else 421 evlist->mmap_first = map; 422 } 423 424 static int 425 mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, 426 int idx, struct perf_mmap_param *mp, int cpu_idx, 427 int thread, int *_output, int *_output_overwrite) 428 { 429 int evlist_cpu = perf_cpu_map__cpu(evlist->cpus, cpu_idx); 430 struct perf_evsel *evsel; 431 int revent; 432 433 perf_evlist__for_each_entry(evlist, evsel) { 434 bool overwrite = evsel->attr.write_backward; 435 struct perf_mmap *map; 436 int *output, fd, cpu; 437 438 if (evsel->system_wide && thread) 439 continue; 440 441 cpu = perf_cpu_map__idx(evsel->cpus, evlist_cpu); 442 if (cpu == -1) 443 continue; 444 445 map = ops->get(evlist, overwrite, idx); 446 if (map == NULL) 447 return -ENOMEM; 448 449 if (overwrite) { 450 mp->prot = PROT_READ; 451 output = _output_overwrite; 452 } else { 453 mp->prot = PROT_READ | PROT_WRITE; 454 output = _output; 455 } 456 457 fd = FD(evsel, cpu, thread); 458 459 if (*output == -1) { 460 *output = fd; 461 462 /* 463 * The last one will be done at perf_mmap__consume(), so that we 464 * make sure we don't prevent tools from consuming every last event in 465 * the ring buffer. 466 * 467 * I.e. we can get the POLLHUP meaning that the fd doesn't exist 468 * anymore, but the last events for it are still in the ring buffer, 469 * waiting to be consumed. 470 * 471 * Tools can chose to ignore this at their own discretion, but the 472 * evlist layer can't just drop it when filtering events in 473 * perf_evlist__filter_pollfd(). 474 */ 475 refcount_set(&map->refcnt, 2); 476 477 if (ops->mmap(map, mp, *output, evlist_cpu) < 0) 478 return -1; 479 480 if (!idx) 481 perf_evlist__set_mmap_first(evlist, map, overwrite); 482 } else { 483 if (ioctl(fd, PERF_EVENT_IOC_SET_OUTPUT, *output) != 0) 484 return -1; 485 486 perf_mmap__get(map); 487 } 488 489 revent = !overwrite ? POLLIN : 0; 490 491 if (!evsel->system_wide && 492 perf_evlist__add_pollfd(evlist, fd, map, revent, fdarray_flag__default) < 0) { 493 perf_mmap__put(map); 494 return -1; 495 } 496 497 if (evsel->attr.read_format & PERF_FORMAT_ID) { 498 if (perf_evlist__id_add_fd(evlist, evsel, cpu, thread, 499 fd) < 0) 500 return -1; 501 perf_evsel__set_sid_idx(evsel, idx, cpu, thread); 502 } 503 } 504 505 return 0; 506 } 507 508 static int 509 mmap_per_thread(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, 510 struct perf_mmap_param *mp) 511 { 512 int thread; 513 int nr_threads = perf_thread_map__nr(evlist->threads); 514 515 for (thread = 0; thread < nr_threads; thread++) { 516 int output = -1; 517 int output_overwrite = -1; 518 519 if (ops->idx) 520 ops->idx(evlist, mp, thread, false); 521 522 if (mmap_per_evsel(evlist, ops, thread, mp, 0, thread, 523 &output, &output_overwrite)) 524 goto out_unmap; 525 } 526 527 return 0; 528 529 out_unmap: 530 perf_evlist__munmap(evlist); 531 return -1; 532 } 533 534 static int 535 mmap_per_cpu(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, 536 struct perf_mmap_param *mp) 537 { 538 int nr_threads = perf_thread_map__nr(evlist->threads); 539 int nr_cpus = perf_cpu_map__nr(evlist->cpus); 540 int cpu, thread; 541 542 for (cpu = 0; cpu < nr_cpus; cpu++) { 543 int output = -1; 544 int output_overwrite = -1; 545 546 if (ops->idx) 547 ops->idx(evlist, mp, cpu, true); 548 549 for (thread = 0; thread < nr_threads; thread++) { 550 if (mmap_per_evsel(evlist, ops, cpu, mp, cpu, 551 thread, &output, &output_overwrite)) 552 goto out_unmap; 553 } 554 } 555 556 return 0; 557 558 out_unmap: 559 perf_evlist__munmap(evlist); 560 return -1; 561 } 562 563 static int perf_evlist__nr_mmaps(struct perf_evlist *evlist) 564 { 565 int nr_mmaps; 566 567 nr_mmaps = perf_cpu_map__nr(evlist->cpus); 568 if (perf_cpu_map__empty(evlist->cpus)) 569 nr_mmaps = perf_thread_map__nr(evlist->threads); 570 571 return nr_mmaps; 572 } 573 574 int perf_evlist__mmap_ops(struct perf_evlist *evlist, 575 struct perf_evlist_mmap_ops *ops, 576 struct perf_mmap_param *mp) 577 { 578 struct perf_evsel *evsel; 579 const struct perf_cpu_map *cpus = evlist->cpus; 580 const struct perf_thread_map *threads = evlist->threads; 581 582 if (!ops || !ops->get || !ops->mmap) 583 return -EINVAL; 584 585 mp->mask = evlist->mmap_len - page_size - 1; 586 587 evlist->nr_mmaps = perf_evlist__nr_mmaps(evlist); 588 589 perf_evlist__for_each_entry(evlist, evsel) { 590 if ((evsel->attr.read_format & PERF_FORMAT_ID) && 591 evsel->sample_id == NULL && 592 perf_evsel__alloc_id(evsel, perf_cpu_map__nr(cpus), threads->nr) < 0) 593 return -ENOMEM; 594 } 595 596 if (evlist->pollfd.entries == NULL && perf_evlist__alloc_pollfd(evlist) < 0) 597 return -ENOMEM; 598 599 if (perf_cpu_map__empty(cpus)) 600 return mmap_per_thread(evlist, ops, mp); 601 602 return mmap_per_cpu(evlist, ops, mp); 603 } 604 605 int perf_evlist__mmap(struct perf_evlist *evlist, int pages) 606 { 607 struct perf_mmap_param mp; 608 struct perf_evlist_mmap_ops ops = { 609 .get = perf_evlist__mmap_cb_get, 610 .mmap = perf_evlist__mmap_cb_mmap, 611 }; 612 613 evlist->mmap_len = (pages + 1) * page_size; 614 615 return perf_evlist__mmap_ops(evlist, &ops, &mp); 616 } 617 618 void perf_evlist__munmap(struct perf_evlist *evlist) 619 { 620 int i; 621 622 if (evlist->mmap) { 623 for (i = 0; i < evlist->nr_mmaps; i++) 624 perf_mmap__munmap(&evlist->mmap[i]); 625 } 626 627 if (evlist->mmap_ovw) { 628 for (i = 0; i < evlist->nr_mmaps; i++) 629 perf_mmap__munmap(&evlist->mmap_ovw[i]); 630 } 631 632 zfree(&evlist->mmap); 633 zfree(&evlist->mmap_ovw); 634 } 635 636 struct perf_mmap* 637 perf_evlist__next_mmap(struct perf_evlist *evlist, struct perf_mmap *map, 638 bool overwrite) 639 { 640 if (map) 641 return map->next; 642 643 return overwrite ? evlist->mmap_ovw_first : evlist->mmap_first; 644 } 645 646 void __perf_evlist__set_leader(struct list_head *list) 647 { 648 struct perf_evsel *evsel, *leader; 649 650 leader = list_entry(list->next, struct perf_evsel, node); 651 evsel = list_entry(list->prev, struct perf_evsel, node); 652 653 leader->nr_members = evsel->idx - leader->idx + 1; 654 655 __perf_evlist__for_each_entry(list, evsel) 656 evsel->leader = leader; 657 } 658 659 void perf_evlist__set_leader(struct perf_evlist *evlist) 660 { 661 if (evlist->nr_entries) { 662 evlist->nr_groups = evlist->nr_entries > 1 ? 1 : 0; 663 __perf_evlist__set_leader(&evlist->entries); 664 } 665 } 666