1 // SPDX-License-Identifier: GPL-2.0 2 #include <api/fs/fs.h> 3 #include "cpumap.h" 4 #include "debug.h" 5 #include "event.h" 6 #include <assert.h> 7 #include <dirent.h> 8 #include <stdio.h> 9 #include <stdlib.h> 10 #include <linux/bitmap.h> 11 #include "asm/bug.h" 12 13 #include <linux/ctype.h> 14 #include <linux/zalloc.h> 15 16 static struct perf_cpu max_cpu_num; 17 static struct perf_cpu max_present_cpu_num; 18 static int max_node_num; 19 /** 20 * The numa node X as read from /sys/devices/system/node/nodeX indexed by the 21 * CPU number. 22 */ 23 static int *cpunode_map; 24 25 bool perf_record_cpu_map_data__test_bit(int i, 26 const struct perf_record_cpu_map_data *data) 27 { 28 int bit_word32 = i / 32; 29 __u32 bit_mask32 = 1U << (i & 31); 30 int bit_word64 = i / 64; 31 __u64 bit_mask64 = ((__u64)1) << (i & 63); 32 33 return (data->mask32_data.long_size == 4) 34 ? (bit_word32 < data->mask32_data.nr) && 35 (data->mask32_data.mask[bit_word32] & bit_mask32) != 0 36 : (bit_word64 < data->mask64_data.nr) && 37 (data->mask64_data.mask[bit_word64] & bit_mask64) != 0; 38 } 39 40 /* Read ith mask value from data into the given 64-bit sized bitmap */ 41 static void perf_record_cpu_map_data__read_one_mask(const struct perf_record_cpu_map_data *data, 42 int i, unsigned long *bitmap) 43 { 44 #if __SIZEOF_LONG__ == 8 45 if (data->mask32_data.long_size == 4) 46 bitmap[0] = data->mask32_data.mask[i]; 47 else 48 bitmap[0] = data->mask64_data.mask[i]; 49 #else 50 if (data->mask32_data.long_size == 4) { 51 bitmap[0] = data->mask32_data.mask[i]; 52 bitmap[1] = 0; 53 } else { 54 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 55 bitmap[0] = (unsigned long)(data->mask64_data.mask[i] >> 32); 56 bitmap[1] = (unsigned long)data->mask64_data.mask[i]; 57 #else 58 bitmap[0] = (unsigned long)data->mask64_data.mask[i]; 59 bitmap[1] = (unsigned long)(data->mask64_data.mask[i] >> 32); 60 #endif 61 } 62 #endif 63 } 64 static struct perf_cpu_map *cpu_map__from_entries(const struct perf_record_cpu_map_data *data) 65 { 66 struct perf_cpu_map *map; 67 68 map = perf_cpu_map__empty_new(data->cpus_data.nr); 69 if (map) { 70 unsigned i; 71 72 for (i = 0; i < data->cpus_data.nr; i++) { 73 /* 74 * Special treatment for -1, which is not real cpu number, 75 * and we need to use (int) -1 to initialize map[i], 76 * otherwise it would become 65535. 77 */ 78 if (data->cpus_data.cpu[i] == (u16) -1) 79 map->map[i].cpu = -1; 80 else 81 map->map[i].cpu = (int) data->cpus_data.cpu[i]; 82 } 83 } 84 85 return map; 86 } 87 88 static struct perf_cpu_map *cpu_map__from_mask(const struct perf_record_cpu_map_data *data) 89 { 90 DECLARE_BITMAP(local_copy, 64); 91 int weight = 0, mask_nr = data->mask32_data.nr; 92 struct perf_cpu_map *map; 93 94 for (int i = 0; i < mask_nr; i++) { 95 perf_record_cpu_map_data__read_one_mask(data, i, local_copy); 96 weight += bitmap_weight(local_copy, 64); 97 } 98 99 map = perf_cpu_map__empty_new(weight); 100 if (!map) 101 return NULL; 102 103 for (int i = 0, j = 0; i < mask_nr; i++) { 104 int cpus_per_i = (i * data->mask32_data.long_size * BITS_PER_BYTE); 105 int cpu; 106 107 perf_record_cpu_map_data__read_one_mask(data, i, local_copy); 108 for_each_set_bit(cpu, local_copy, 64) 109 map->map[j++].cpu = cpu + cpus_per_i; 110 } 111 return map; 112 113 } 114 115 static struct perf_cpu_map *cpu_map__from_range(const struct perf_record_cpu_map_data *data) 116 { 117 struct perf_cpu_map *map; 118 unsigned int i = 0; 119 120 map = perf_cpu_map__empty_new(data->range_cpu_data.end_cpu - 121 data->range_cpu_data.start_cpu + 1 + data->range_cpu_data.any_cpu); 122 if (!map) 123 return NULL; 124 125 if (data->range_cpu_data.any_cpu) 126 map->map[i++].cpu = -1; 127 128 for (int cpu = data->range_cpu_data.start_cpu; cpu <= data->range_cpu_data.end_cpu; 129 i++, cpu++) 130 map->map[i].cpu = cpu; 131 132 return map; 133 } 134 135 struct perf_cpu_map *cpu_map__new_data(const struct perf_record_cpu_map_data *data) 136 { 137 switch (data->type) { 138 case PERF_CPU_MAP__CPUS: 139 return cpu_map__from_entries(data); 140 case PERF_CPU_MAP__MASK: 141 return cpu_map__from_mask(data); 142 case PERF_CPU_MAP__RANGE_CPUS: 143 return cpu_map__from_range(data); 144 default: 145 pr_err("cpu_map__new_data unknown type %d\n", data->type); 146 return NULL; 147 } 148 } 149 150 size_t cpu_map__fprintf(struct perf_cpu_map *map, FILE *fp) 151 { 152 #define BUFSIZE 1024 153 char buf[BUFSIZE]; 154 155 cpu_map__snprint(map, buf, sizeof(buf)); 156 return fprintf(fp, "%s\n", buf); 157 #undef BUFSIZE 158 } 159 160 struct perf_cpu_map *perf_cpu_map__empty_new(int nr) 161 { 162 struct perf_cpu_map *cpus = malloc(sizeof(*cpus) + sizeof(int) * nr); 163 164 if (cpus != NULL) { 165 int i; 166 167 cpus->nr = nr; 168 for (i = 0; i < nr; i++) 169 cpus->map[i].cpu = -1; 170 171 refcount_set(&cpus->refcnt, 1); 172 } 173 174 return cpus; 175 } 176 177 struct cpu_aggr_map *cpu_aggr_map__empty_new(int nr) 178 { 179 struct cpu_aggr_map *cpus = malloc(sizeof(*cpus) + sizeof(struct aggr_cpu_id) * nr); 180 181 if (cpus != NULL) { 182 int i; 183 184 cpus->nr = nr; 185 for (i = 0; i < nr; i++) 186 cpus->map[i] = aggr_cpu_id__empty(); 187 188 refcount_set(&cpus->refcnt, 1); 189 } 190 191 return cpus; 192 } 193 194 static int cpu__get_topology_int(int cpu, const char *name, int *value) 195 { 196 char path[PATH_MAX]; 197 198 snprintf(path, PATH_MAX, 199 "devices/system/cpu/cpu%d/topology/%s", cpu, name); 200 201 return sysfs__read_int(path, value); 202 } 203 204 int cpu__get_socket_id(struct perf_cpu cpu) 205 { 206 int value, ret = cpu__get_topology_int(cpu.cpu, "physical_package_id", &value); 207 return ret ?: value; 208 } 209 210 struct aggr_cpu_id aggr_cpu_id__socket(struct perf_cpu cpu, void *data __maybe_unused) 211 { 212 struct aggr_cpu_id id = aggr_cpu_id__empty(); 213 214 id.socket = cpu__get_socket_id(cpu); 215 return id; 216 } 217 218 static int aggr_cpu_id__cmp(const void *a_pointer, const void *b_pointer) 219 { 220 struct aggr_cpu_id *a = (struct aggr_cpu_id *)a_pointer; 221 struct aggr_cpu_id *b = (struct aggr_cpu_id *)b_pointer; 222 223 if (a->node != b->node) 224 return a->node - b->node; 225 else if (a->socket != b->socket) 226 return a->socket - b->socket; 227 else if (a->die != b->die) 228 return a->die - b->die; 229 else if (a->core != b->core) 230 return a->core - b->core; 231 else 232 return a->thread_idx - b->thread_idx; 233 } 234 235 struct cpu_aggr_map *cpu_aggr_map__new(const struct perf_cpu_map *cpus, 236 aggr_cpu_id_get_t get_id, 237 void *data) 238 { 239 int idx; 240 struct perf_cpu cpu; 241 struct cpu_aggr_map *c = cpu_aggr_map__empty_new(cpus->nr); 242 243 if (!c) 244 return NULL; 245 246 /* Reset size as it may only be partially filled */ 247 c->nr = 0; 248 249 perf_cpu_map__for_each_cpu(cpu, idx, cpus) { 250 bool duplicate = false; 251 struct aggr_cpu_id cpu_id = get_id(cpu, data); 252 253 for (int j = 0; j < c->nr; j++) { 254 if (aggr_cpu_id__equal(&cpu_id, &c->map[j])) { 255 duplicate = true; 256 break; 257 } 258 } 259 if (!duplicate) { 260 c->map[c->nr] = cpu_id; 261 c->nr++; 262 } 263 } 264 /* Trim. */ 265 if (c->nr != cpus->nr) { 266 struct cpu_aggr_map *trimmed_c = 267 realloc(c, 268 sizeof(struct cpu_aggr_map) + sizeof(struct aggr_cpu_id) * c->nr); 269 270 if (trimmed_c) 271 c = trimmed_c; 272 } 273 /* ensure we process id in increasing order */ 274 qsort(c->map, c->nr, sizeof(struct aggr_cpu_id), aggr_cpu_id__cmp); 275 276 return c; 277 278 } 279 280 int cpu__get_die_id(struct perf_cpu cpu) 281 { 282 int value, ret = cpu__get_topology_int(cpu.cpu, "die_id", &value); 283 284 return ret ?: value; 285 } 286 287 struct aggr_cpu_id aggr_cpu_id__die(struct perf_cpu cpu, void *data) 288 { 289 struct aggr_cpu_id id; 290 int die; 291 292 die = cpu__get_die_id(cpu); 293 /* There is no die_id on legacy system. */ 294 if (die == -1) 295 die = 0; 296 297 /* 298 * die_id is relative to socket, so start 299 * with the socket ID and then add die to 300 * make a unique ID. 301 */ 302 id = aggr_cpu_id__socket(cpu, data); 303 if (aggr_cpu_id__is_empty(&id)) 304 return id; 305 306 id.die = die; 307 return id; 308 } 309 310 int cpu__get_core_id(struct perf_cpu cpu) 311 { 312 int value, ret = cpu__get_topology_int(cpu.cpu, "core_id", &value); 313 return ret ?: value; 314 } 315 316 struct aggr_cpu_id aggr_cpu_id__core(struct perf_cpu cpu, void *data) 317 { 318 struct aggr_cpu_id id; 319 int core = cpu__get_core_id(cpu); 320 321 /* aggr_cpu_id__die returns a struct with socket and die set. */ 322 id = aggr_cpu_id__die(cpu, data); 323 if (aggr_cpu_id__is_empty(&id)) 324 return id; 325 326 /* 327 * core_id is relative to socket and die, we need a global id. 328 * So we combine the result from cpu_map__get_die with the core id 329 */ 330 id.core = core; 331 return id; 332 333 } 334 335 struct aggr_cpu_id aggr_cpu_id__cpu(struct perf_cpu cpu, void *data) 336 { 337 struct aggr_cpu_id id; 338 339 /* aggr_cpu_id__core returns a struct with socket, die and core set. */ 340 id = aggr_cpu_id__core(cpu, data); 341 if (aggr_cpu_id__is_empty(&id)) 342 return id; 343 344 id.cpu = cpu; 345 return id; 346 347 } 348 349 struct aggr_cpu_id aggr_cpu_id__node(struct perf_cpu cpu, void *data __maybe_unused) 350 { 351 struct aggr_cpu_id id = aggr_cpu_id__empty(); 352 353 id.node = cpu__get_node(cpu); 354 return id; 355 } 356 357 /* setup simple routines to easily access node numbers given a cpu number */ 358 static int get_max_num(char *path, int *max) 359 { 360 size_t num; 361 char *buf; 362 int err = 0; 363 364 if (filename__read_str(path, &buf, &num)) 365 return -1; 366 367 buf[num] = '\0'; 368 369 /* start on the right, to find highest node num */ 370 while (--num) { 371 if ((buf[num] == ',') || (buf[num] == '-')) { 372 num++; 373 break; 374 } 375 } 376 if (sscanf(&buf[num], "%d", max) < 1) { 377 err = -1; 378 goto out; 379 } 380 381 /* convert from 0-based to 1-based */ 382 (*max)++; 383 384 out: 385 free(buf); 386 return err; 387 } 388 389 /* Determine highest possible cpu in the system for sparse allocation */ 390 static void set_max_cpu_num(void) 391 { 392 const char *mnt; 393 char path[PATH_MAX]; 394 int ret = -1; 395 396 /* set up default */ 397 max_cpu_num.cpu = 4096; 398 max_present_cpu_num.cpu = 4096; 399 400 mnt = sysfs__mountpoint(); 401 if (!mnt) 402 goto out; 403 404 /* get the highest possible cpu number for a sparse allocation */ 405 ret = snprintf(path, PATH_MAX, "%s/devices/system/cpu/possible", mnt); 406 if (ret >= PATH_MAX) { 407 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 408 goto out; 409 } 410 411 ret = get_max_num(path, &max_cpu_num.cpu); 412 if (ret) 413 goto out; 414 415 /* get the highest present cpu number for a sparse allocation */ 416 ret = snprintf(path, PATH_MAX, "%s/devices/system/cpu/present", mnt); 417 if (ret >= PATH_MAX) { 418 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 419 goto out; 420 } 421 422 ret = get_max_num(path, &max_present_cpu_num.cpu); 423 424 out: 425 if (ret) 426 pr_err("Failed to read max cpus, using default of %d\n", max_cpu_num.cpu); 427 } 428 429 /* Determine highest possible node in the system for sparse allocation */ 430 static void set_max_node_num(void) 431 { 432 const char *mnt; 433 char path[PATH_MAX]; 434 int ret = -1; 435 436 /* set up default */ 437 max_node_num = 8; 438 439 mnt = sysfs__mountpoint(); 440 if (!mnt) 441 goto out; 442 443 /* get the highest possible cpu number for a sparse allocation */ 444 ret = snprintf(path, PATH_MAX, "%s/devices/system/node/possible", mnt); 445 if (ret >= PATH_MAX) { 446 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 447 goto out; 448 } 449 450 ret = get_max_num(path, &max_node_num); 451 452 out: 453 if (ret) 454 pr_err("Failed to read max nodes, using default of %d\n", max_node_num); 455 } 456 457 int cpu__max_node(void) 458 { 459 if (unlikely(!max_node_num)) 460 set_max_node_num(); 461 462 return max_node_num; 463 } 464 465 struct perf_cpu cpu__max_cpu(void) 466 { 467 if (unlikely(!max_cpu_num.cpu)) 468 set_max_cpu_num(); 469 470 return max_cpu_num; 471 } 472 473 struct perf_cpu cpu__max_present_cpu(void) 474 { 475 if (unlikely(!max_present_cpu_num.cpu)) 476 set_max_cpu_num(); 477 478 return max_present_cpu_num; 479 } 480 481 482 int cpu__get_node(struct perf_cpu cpu) 483 { 484 if (unlikely(cpunode_map == NULL)) { 485 pr_debug("cpu_map not initialized\n"); 486 return -1; 487 } 488 489 return cpunode_map[cpu.cpu]; 490 } 491 492 static int init_cpunode_map(void) 493 { 494 int i; 495 496 set_max_cpu_num(); 497 set_max_node_num(); 498 499 cpunode_map = calloc(max_cpu_num.cpu, sizeof(int)); 500 if (!cpunode_map) { 501 pr_err("%s: calloc failed\n", __func__); 502 return -1; 503 } 504 505 for (i = 0; i < max_cpu_num.cpu; i++) 506 cpunode_map[i] = -1; 507 508 return 0; 509 } 510 511 int cpu__setup_cpunode_map(void) 512 { 513 struct dirent *dent1, *dent2; 514 DIR *dir1, *dir2; 515 unsigned int cpu, mem; 516 char buf[PATH_MAX]; 517 char path[PATH_MAX]; 518 const char *mnt; 519 int n; 520 521 /* initialize globals */ 522 if (init_cpunode_map()) 523 return -1; 524 525 mnt = sysfs__mountpoint(); 526 if (!mnt) 527 return 0; 528 529 n = snprintf(path, PATH_MAX, "%s/devices/system/node", mnt); 530 if (n >= PATH_MAX) { 531 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 532 return -1; 533 } 534 535 dir1 = opendir(path); 536 if (!dir1) 537 return 0; 538 539 /* walk tree and setup map */ 540 while ((dent1 = readdir(dir1)) != NULL) { 541 if (dent1->d_type != DT_DIR || sscanf(dent1->d_name, "node%u", &mem) < 1) 542 continue; 543 544 n = snprintf(buf, PATH_MAX, "%s/%s", path, dent1->d_name); 545 if (n >= PATH_MAX) { 546 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 547 continue; 548 } 549 550 dir2 = opendir(buf); 551 if (!dir2) 552 continue; 553 while ((dent2 = readdir(dir2)) != NULL) { 554 if (dent2->d_type != DT_LNK || sscanf(dent2->d_name, "cpu%u", &cpu) < 1) 555 continue; 556 cpunode_map[cpu] = mem; 557 } 558 closedir(dir2); 559 } 560 closedir(dir1); 561 return 0; 562 } 563 564 size_t cpu_map__snprint(struct perf_cpu_map *map, char *buf, size_t size) 565 { 566 int i, start = -1; 567 bool first = true; 568 size_t ret = 0; 569 570 #define COMMA first ? "" : "," 571 572 for (i = 0; i < map->nr + 1; i++) { 573 struct perf_cpu cpu = { .cpu = INT_MAX }; 574 bool last = i == map->nr; 575 576 if (!last) 577 cpu = map->map[i]; 578 579 if (start == -1) { 580 start = i; 581 if (last) { 582 ret += snprintf(buf + ret, size - ret, 583 "%s%d", COMMA, 584 map->map[i].cpu); 585 } 586 } else if (((i - start) != (cpu.cpu - map->map[start].cpu)) || last) { 587 int end = i - 1; 588 589 if (start == end) { 590 ret += snprintf(buf + ret, size - ret, 591 "%s%d", COMMA, 592 map->map[start].cpu); 593 } else { 594 ret += snprintf(buf + ret, size - ret, 595 "%s%d-%d", COMMA, 596 map->map[start].cpu, map->map[end].cpu); 597 } 598 first = false; 599 start = i; 600 } 601 } 602 603 #undef COMMA 604 605 pr_debug2("cpumask list: %s\n", buf); 606 return ret; 607 } 608 609 static char hex_char(unsigned char val) 610 { 611 if (val < 10) 612 return val + '0'; 613 if (val < 16) 614 return val - 10 + 'a'; 615 return '?'; 616 } 617 618 size_t cpu_map__snprint_mask(struct perf_cpu_map *map, char *buf, size_t size) 619 { 620 int i, cpu; 621 char *ptr = buf; 622 unsigned char *bitmap; 623 struct perf_cpu last_cpu = perf_cpu_map__cpu(map, map->nr - 1); 624 625 if (buf == NULL) 626 return 0; 627 628 bitmap = zalloc(last_cpu.cpu / 8 + 1); 629 if (bitmap == NULL) { 630 buf[0] = '\0'; 631 return 0; 632 } 633 634 for (i = 0; i < map->nr; i++) { 635 cpu = perf_cpu_map__cpu(map, i).cpu; 636 bitmap[cpu / 8] |= 1 << (cpu % 8); 637 } 638 639 for (cpu = last_cpu.cpu / 4 * 4; cpu >= 0; cpu -= 4) { 640 unsigned char bits = bitmap[cpu / 8]; 641 642 if (cpu % 8) 643 bits >>= 4; 644 else 645 bits &= 0xf; 646 647 *ptr++ = hex_char(bits); 648 if ((cpu % 32) == 0 && cpu > 0) 649 *ptr++ = ','; 650 } 651 *ptr = '\0'; 652 free(bitmap); 653 654 buf[size - 1] = '\0'; 655 return ptr - buf; 656 } 657 658 const struct perf_cpu_map *cpu_map__online(void) /* thread unsafe */ 659 { 660 static const struct perf_cpu_map *online = NULL; 661 662 if (!online) 663 online = perf_cpu_map__new(NULL); /* from /sys/devices/system/cpu/online */ 664 665 return online; 666 } 667 668 bool aggr_cpu_id__equal(const struct aggr_cpu_id *a, const struct aggr_cpu_id *b) 669 { 670 return a->thread_idx == b->thread_idx && 671 a->node == b->node && 672 a->socket == b->socket && 673 a->die == b->die && 674 a->core == b->core && 675 a->cpu.cpu == b->cpu.cpu; 676 } 677 678 bool aggr_cpu_id__is_empty(const struct aggr_cpu_id *a) 679 { 680 return a->thread_idx == -1 && 681 a->node == -1 && 682 a->socket == -1 && 683 a->die == -1 && 684 a->core == -1 && 685 a->cpu.cpu == -1; 686 } 687 688 struct aggr_cpu_id aggr_cpu_id__empty(void) 689 { 690 struct aggr_cpu_id ret = { 691 .thread_idx = -1, 692 .node = -1, 693 .socket = -1, 694 .die = -1, 695 .core = -1, 696 .cpu = (struct perf_cpu){ .cpu = -1 }, 697 }; 698 return ret; 699 } 700