1 // SPDX-License-Identifier: GPL-2.0 2 #include <stddef.h> 3 #include <stdlib.h> 4 #include <string.h> 5 #include <errno.h> 6 #include <sys/types.h> 7 #include <sys/stat.h> 8 #include <unistd.h> 9 #include <api/fs/fs.h> 10 #include <linux/kernel.h> 11 #include "map_symbol.h" 12 #include "mem-events.h" 13 #include "debug.h" 14 #include "symbol.h" 15 #include "pmu.h" 16 #include "pmu-hybrid.h" 17 18 unsigned int perf_mem_events__loads_ldlat = 30; 19 20 #define E(t, n, s) { .tag = t, .name = n, .sysfs_name = s } 21 22 static struct perf_mem_event perf_mem_events[PERF_MEM_EVENTS__MAX] = { 23 E("ldlat-loads", "cpu/mem-loads,ldlat=%u/P", "cpu/events/mem-loads"), 24 E("ldlat-stores", "cpu/mem-stores/P", "cpu/events/mem-stores"), 25 E(NULL, NULL, NULL), 26 }; 27 #undef E 28 29 #undef E 30 31 static char mem_loads_name[100]; 32 static bool mem_loads_name__init; 33 34 struct perf_mem_event * __weak perf_mem_events__ptr(int i) 35 { 36 if (i >= PERF_MEM_EVENTS__MAX) 37 return NULL; 38 39 return &perf_mem_events[i]; 40 } 41 42 char * __weak perf_mem_events__name(int i, char *pmu_name __maybe_unused) 43 { 44 struct perf_mem_event *e = perf_mem_events__ptr(i); 45 46 if (!e) 47 return NULL; 48 49 if (i == PERF_MEM_EVENTS__LOAD) { 50 if (!mem_loads_name__init) { 51 mem_loads_name__init = true; 52 scnprintf(mem_loads_name, sizeof(mem_loads_name), 53 e->name, perf_mem_events__loads_ldlat); 54 } 55 return mem_loads_name; 56 } 57 58 return (char *)e->name; 59 } 60 61 __weak bool is_mem_loads_aux_event(struct evsel *leader __maybe_unused) 62 { 63 return false; 64 } 65 66 int perf_mem_events__parse(const char *str) 67 { 68 char *tok, *saveptr = NULL; 69 bool found = false; 70 char *buf; 71 int j; 72 73 /* We need buffer that we know we can write to. */ 74 buf = malloc(strlen(str) + 1); 75 if (!buf) 76 return -ENOMEM; 77 78 strcpy(buf, str); 79 80 tok = strtok_r((char *)buf, ",", &saveptr); 81 82 while (tok) { 83 for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) { 84 struct perf_mem_event *e = perf_mem_events__ptr(j); 85 86 if (!e->tag) 87 continue; 88 89 if (strstr(e->tag, tok)) 90 e->record = found = true; 91 } 92 93 tok = strtok_r(NULL, ",", &saveptr); 94 } 95 96 free(buf); 97 98 if (found) 99 return 0; 100 101 pr_err("failed: event '%s' not found, use '-e list' to get list of available events\n", str); 102 return -1; 103 } 104 105 static bool perf_mem_event__supported(const char *mnt, char *sysfs_name) 106 { 107 char path[PATH_MAX]; 108 struct stat st; 109 110 scnprintf(path, PATH_MAX, "%s/devices/%s", mnt, sysfs_name); 111 return !stat(path, &st); 112 } 113 114 int perf_mem_events__init(void) 115 { 116 const char *mnt = sysfs__mount(); 117 bool found = false; 118 int j; 119 120 if (!mnt) 121 return -ENOENT; 122 123 for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) { 124 struct perf_mem_event *e = perf_mem_events__ptr(j); 125 struct perf_pmu *pmu; 126 char sysfs_name[100]; 127 128 /* 129 * If the event entry isn't valid, skip initialization 130 * and "e->supported" will keep false. 131 */ 132 if (!e->tag) 133 continue; 134 135 if (!perf_pmu__has_hybrid()) { 136 scnprintf(sysfs_name, sizeof(sysfs_name), 137 e->sysfs_name, "cpu"); 138 e->supported = perf_mem_event__supported(mnt, sysfs_name); 139 } else { 140 perf_pmu__for_each_hybrid_pmu(pmu) { 141 scnprintf(sysfs_name, sizeof(sysfs_name), 142 e->sysfs_name, pmu->name); 143 e->supported |= perf_mem_event__supported(mnt, sysfs_name); 144 } 145 } 146 147 if (e->supported) 148 found = true; 149 } 150 151 return found ? 0 : -ENOENT; 152 } 153 154 void perf_mem_events__list(void) 155 { 156 int j; 157 158 for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) { 159 struct perf_mem_event *e = perf_mem_events__ptr(j); 160 161 fprintf(stderr, "%-13s%-*s%s\n", 162 e->tag ?: "", 163 verbose > 0 ? 25 : 0, 164 verbose > 0 ? perf_mem_events__name(j, NULL) : "", 165 e->supported ? ": available" : ""); 166 } 167 } 168 169 static const char * const tlb_access[] = { 170 "N/A", 171 "HIT", 172 "MISS", 173 "L1", 174 "L2", 175 "Walker", 176 "Fault", 177 }; 178 179 int perf_mem__tlb_scnprintf(char *out, size_t sz, struct mem_info *mem_info) 180 { 181 size_t l = 0, i; 182 u64 m = PERF_MEM_TLB_NA; 183 u64 hit, miss; 184 185 sz -= 1; /* -1 for null termination */ 186 out[0] = '\0'; 187 188 if (mem_info) 189 m = mem_info->data_src.mem_dtlb; 190 191 hit = m & PERF_MEM_TLB_HIT; 192 miss = m & PERF_MEM_TLB_MISS; 193 194 /* already taken care of */ 195 m &= ~(PERF_MEM_TLB_HIT|PERF_MEM_TLB_MISS); 196 197 for (i = 0; m && i < ARRAY_SIZE(tlb_access); i++, m >>= 1) { 198 if (!(m & 0x1)) 199 continue; 200 if (l) { 201 strcat(out, " or "); 202 l += 4; 203 } 204 l += scnprintf(out + l, sz - l, tlb_access[i]); 205 } 206 if (*out == '\0') 207 l += scnprintf(out, sz - l, "N/A"); 208 if (hit) 209 l += scnprintf(out + l, sz - l, " hit"); 210 if (miss) 211 l += scnprintf(out + l, sz - l, " miss"); 212 213 return l; 214 } 215 216 static const char * const mem_lvl[] = { 217 "N/A", 218 "HIT", 219 "MISS", 220 "L1", 221 "LFB", 222 "L2", 223 "L3", 224 "Local RAM", 225 "Remote RAM (1 hop)", 226 "Remote RAM (2 hops)", 227 "Remote Cache (1 hop)", 228 "Remote Cache (2 hops)", 229 "I/O", 230 "Uncached", 231 }; 232 233 static const char * const mem_lvlnum[] = { 234 [PERF_MEM_LVLNUM_ANY_CACHE] = "Any cache", 235 [PERF_MEM_LVLNUM_LFB] = "LFB", 236 [PERF_MEM_LVLNUM_RAM] = "RAM", 237 [PERF_MEM_LVLNUM_PMEM] = "PMEM", 238 [PERF_MEM_LVLNUM_NA] = "N/A", 239 }; 240 241 int perf_mem__lvl_scnprintf(char *out, size_t sz, struct mem_info *mem_info) 242 { 243 size_t i, l = 0; 244 u64 m = PERF_MEM_LVL_NA; 245 u64 hit, miss; 246 int printed; 247 248 if (mem_info) 249 m = mem_info->data_src.mem_lvl; 250 251 sz -= 1; /* -1 for null termination */ 252 out[0] = '\0'; 253 254 hit = m & PERF_MEM_LVL_HIT; 255 miss = m & PERF_MEM_LVL_MISS; 256 257 /* already taken care of */ 258 m &= ~(PERF_MEM_LVL_HIT|PERF_MEM_LVL_MISS); 259 260 261 if (mem_info && mem_info->data_src.mem_remote) { 262 strcat(out, "Remote "); 263 l += 7; 264 } 265 266 printed = 0; 267 for (i = 0; m && i < ARRAY_SIZE(mem_lvl); i++, m >>= 1) { 268 if (!(m & 0x1)) 269 continue; 270 if (printed++) { 271 strcat(out, " or "); 272 l += 4; 273 } 274 l += scnprintf(out + l, sz - l, mem_lvl[i]); 275 } 276 277 if (mem_info && mem_info->data_src.mem_lvl_num) { 278 int lvl = mem_info->data_src.mem_lvl_num; 279 if (printed++) { 280 strcat(out, " or "); 281 l += 4; 282 } 283 if (mem_lvlnum[lvl]) 284 l += scnprintf(out + l, sz - l, mem_lvlnum[lvl]); 285 else 286 l += scnprintf(out + l, sz - l, "L%d", lvl); 287 } 288 289 if (l == 0) 290 l += scnprintf(out + l, sz - l, "N/A"); 291 if (hit) 292 l += scnprintf(out + l, sz - l, " hit"); 293 if (miss) 294 l += scnprintf(out + l, sz - l, " miss"); 295 296 return l; 297 } 298 299 static const char * const snoop_access[] = { 300 "N/A", 301 "None", 302 "Hit", 303 "Miss", 304 "HitM", 305 }; 306 307 int perf_mem__snp_scnprintf(char *out, size_t sz, struct mem_info *mem_info) 308 { 309 size_t i, l = 0; 310 u64 m = PERF_MEM_SNOOP_NA; 311 312 sz -= 1; /* -1 for null termination */ 313 out[0] = '\0'; 314 315 if (mem_info) 316 m = mem_info->data_src.mem_snoop; 317 318 for (i = 0; m && i < ARRAY_SIZE(snoop_access); i++, m >>= 1) { 319 if (!(m & 0x1)) 320 continue; 321 if (l) { 322 strcat(out, " or "); 323 l += 4; 324 } 325 l += scnprintf(out + l, sz - l, snoop_access[i]); 326 } 327 if (mem_info && 328 (mem_info->data_src.mem_snoopx & PERF_MEM_SNOOPX_FWD)) { 329 if (l) { 330 strcat(out, " or "); 331 l += 4; 332 } 333 l += scnprintf(out + l, sz - l, "Fwd"); 334 } 335 336 if (*out == '\0') 337 l += scnprintf(out, sz - l, "N/A"); 338 339 return l; 340 } 341 342 int perf_mem__lck_scnprintf(char *out, size_t sz, struct mem_info *mem_info) 343 { 344 u64 mask = PERF_MEM_LOCK_NA; 345 int l; 346 347 if (mem_info) 348 mask = mem_info->data_src.mem_lock; 349 350 if (mask & PERF_MEM_LOCK_NA) 351 l = scnprintf(out, sz, "N/A"); 352 else if (mask & PERF_MEM_LOCK_LOCKED) 353 l = scnprintf(out, sz, "Yes"); 354 else 355 l = scnprintf(out, sz, "No"); 356 357 return l; 358 } 359 360 int perf_mem__blk_scnprintf(char *out, size_t sz, struct mem_info *mem_info) 361 { 362 size_t l = 0; 363 u64 mask = PERF_MEM_BLK_NA; 364 365 sz -= 1; /* -1 for null termination */ 366 out[0] = '\0'; 367 368 if (mem_info) 369 mask = mem_info->data_src.mem_blk; 370 371 if (!mask || (mask & PERF_MEM_BLK_NA)) { 372 l += scnprintf(out + l, sz - l, " N/A"); 373 return l; 374 } 375 if (mask & PERF_MEM_BLK_DATA) 376 l += scnprintf(out + l, sz - l, " Data"); 377 if (mask & PERF_MEM_BLK_ADDR) 378 l += scnprintf(out + l, sz - l, " Addr"); 379 380 return l; 381 } 382 383 int perf_script__meminfo_scnprintf(char *out, size_t sz, struct mem_info *mem_info) 384 { 385 int i = 0; 386 387 i += perf_mem__lvl_scnprintf(out, sz, mem_info); 388 i += scnprintf(out + i, sz - i, "|SNP "); 389 i += perf_mem__snp_scnprintf(out + i, sz - i, mem_info); 390 i += scnprintf(out + i, sz - i, "|TLB "); 391 i += perf_mem__tlb_scnprintf(out + i, sz - i, mem_info); 392 i += scnprintf(out + i, sz - i, "|LCK "); 393 i += perf_mem__lck_scnprintf(out + i, sz - i, mem_info); 394 i += scnprintf(out + i, sz - i, "|BLK "); 395 i += perf_mem__blk_scnprintf(out + i, sz - i, mem_info); 396 397 return i; 398 } 399 400 int c2c_decode_stats(struct c2c_stats *stats, struct mem_info *mi) 401 { 402 union perf_mem_data_src *data_src = &mi->data_src; 403 u64 daddr = mi->daddr.addr; 404 u64 op = data_src->mem_op; 405 u64 lvl = data_src->mem_lvl; 406 u64 snoop = data_src->mem_snoop; 407 u64 lock = data_src->mem_lock; 408 u64 blk = data_src->mem_blk; 409 /* 410 * Skylake might report unknown remote level via this 411 * bit, consider it when evaluating remote HITMs. 412 */ 413 bool mrem = data_src->mem_remote; 414 int err = 0; 415 416 #define HITM_INC(__f) \ 417 do { \ 418 stats->__f++; \ 419 stats->tot_hitm++; \ 420 } while (0) 421 422 #define P(a, b) PERF_MEM_##a##_##b 423 424 stats->nr_entries++; 425 426 if (lock & P(LOCK, LOCKED)) stats->locks++; 427 428 if (blk & P(BLK, DATA)) stats->blk_data++; 429 if (blk & P(BLK, ADDR)) stats->blk_addr++; 430 431 if (op & P(OP, LOAD)) { 432 /* load */ 433 stats->load++; 434 435 if (!daddr) { 436 stats->ld_noadrs++; 437 return -1; 438 } 439 440 if (lvl & P(LVL, HIT)) { 441 if (lvl & P(LVL, UNC)) stats->ld_uncache++; 442 if (lvl & P(LVL, IO)) stats->ld_io++; 443 if (lvl & P(LVL, LFB)) stats->ld_fbhit++; 444 if (lvl & P(LVL, L1 )) stats->ld_l1hit++; 445 if (lvl & P(LVL, L2 )) stats->ld_l2hit++; 446 if (lvl & P(LVL, L3 )) { 447 if (snoop & P(SNOOP, HITM)) 448 HITM_INC(lcl_hitm); 449 else 450 stats->ld_llchit++; 451 } 452 453 if (lvl & P(LVL, LOC_RAM)) { 454 stats->lcl_dram++; 455 if (snoop & P(SNOOP, HIT)) 456 stats->ld_shared++; 457 else 458 stats->ld_excl++; 459 } 460 461 if ((lvl & P(LVL, REM_RAM1)) || 462 (lvl & P(LVL, REM_RAM2)) || 463 mrem) { 464 stats->rmt_dram++; 465 if (snoop & P(SNOOP, HIT)) 466 stats->ld_shared++; 467 else 468 stats->ld_excl++; 469 } 470 } 471 472 if ((lvl & P(LVL, REM_CCE1)) || 473 (lvl & P(LVL, REM_CCE2)) || 474 mrem) { 475 if (snoop & P(SNOOP, HIT)) 476 stats->rmt_hit++; 477 else if (snoop & P(SNOOP, HITM)) 478 HITM_INC(rmt_hitm); 479 } 480 481 if ((lvl & P(LVL, MISS))) 482 stats->ld_miss++; 483 484 } else if (op & P(OP, STORE)) { 485 /* store */ 486 stats->store++; 487 488 if (!daddr) { 489 stats->st_noadrs++; 490 return -1; 491 } 492 493 if (lvl & P(LVL, HIT)) { 494 if (lvl & P(LVL, UNC)) stats->st_uncache++; 495 if (lvl & P(LVL, L1 )) stats->st_l1hit++; 496 } 497 if (lvl & P(LVL, MISS)) 498 if (lvl & P(LVL, L1)) stats->st_l1miss++; 499 } else { 500 /* unparsable data_src? */ 501 stats->noparse++; 502 return -1; 503 } 504 505 if (!mi->daddr.ms.map || !mi->iaddr.ms.map) { 506 stats->nomap++; 507 return -1; 508 } 509 510 #undef P 511 #undef HITM_INC 512 return err; 513 } 514 515 void c2c_add_stats(struct c2c_stats *stats, struct c2c_stats *add) 516 { 517 stats->nr_entries += add->nr_entries; 518 519 stats->locks += add->locks; 520 stats->store += add->store; 521 stats->st_uncache += add->st_uncache; 522 stats->st_noadrs += add->st_noadrs; 523 stats->st_l1hit += add->st_l1hit; 524 stats->st_l1miss += add->st_l1miss; 525 stats->load += add->load; 526 stats->ld_excl += add->ld_excl; 527 stats->ld_shared += add->ld_shared; 528 stats->ld_uncache += add->ld_uncache; 529 stats->ld_io += add->ld_io; 530 stats->ld_miss += add->ld_miss; 531 stats->ld_noadrs += add->ld_noadrs; 532 stats->ld_fbhit += add->ld_fbhit; 533 stats->ld_l1hit += add->ld_l1hit; 534 stats->ld_l2hit += add->ld_l2hit; 535 stats->ld_llchit += add->ld_llchit; 536 stats->lcl_hitm += add->lcl_hitm; 537 stats->rmt_hitm += add->rmt_hitm; 538 stats->tot_hitm += add->tot_hitm; 539 stats->rmt_hit += add->rmt_hit; 540 stats->lcl_dram += add->lcl_dram; 541 stats->rmt_dram += add->rmt_dram; 542 stats->blk_data += add->blk_data; 543 stats->blk_addr += add->blk_addr; 544 stats->nomap += add->nomap; 545 stats->noparse += add->noparse; 546 } 547