1 // SPDX-License-Identifier: GPL-2.0 2 #include <stddef.h> 3 #include <stdlib.h> 4 #include <string.h> 5 #include <errno.h> 6 #include <sys/types.h> 7 #include <sys/stat.h> 8 #include <unistd.h> 9 #include <api/fs/fs.h> 10 #include <linux/kernel.h> 11 #include "map_symbol.h" 12 #include "mem-events.h" 13 #include "debug.h" 14 #include "symbol.h" 15 16 unsigned int perf_mem_events__loads_ldlat = 30; 17 18 #define E(t, n, s) { .tag = t, .name = n, .sysfs_name = s } 19 20 static struct perf_mem_event perf_mem_events[PERF_MEM_EVENTS__MAX] = { 21 E("ldlat-loads", "cpu/mem-loads,ldlat=%u/P", "cpu/events/mem-loads"), 22 E("ldlat-stores", "cpu/mem-stores/P", "cpu/events/mem-stores"), 23 E(NULL, NULL, NULL), 24 }; 25 #undef E 26 27 #undef E 28 29 static char mem_loads_name[100]; 30 static bool mem_loads_name__init; 31 32 struct perf_mem_event * __weak perf_mem_events__ptr(int i) 33 { 34 if (i >= PERF_MEM_EVENTS__MAX) 35 return NULL; 36 37 return &perf_mem_events[i]; 38 } 39 40 char * __weak perf_mem_events__name(int i) 41 { 42 struct perf_mem_event *e = perf_mem_events__ptr(i); 43 44 if (!e) 45 return NULL; 46 47 if (i == PERF_MEM_EVENTS__LOAD) { 48 if (!mem_loads_name__init) { 49 mem_loads_name__init = true; 50 scnprintf(mem_loads_name, sizeof(mem_loads_name), 51 e->name, perf_mem_events__loads_ldlat); 52 } 53 return mem_loads_name; 54 } 55 56 return (char *)e->name; 57 } 58 59 __weak bool is_mem_loads_aux_event(struct evsel *leader __maybe_unused) 60 { 61 return false; 62 } 63 64 int perf_mem_events__parse(const char *str) 65 { 66 char *tok, *saveptr = NULL; 67 bool found = false; 68 char *buf; 69 int j; 70 71 /* We need buffer that we know we can write to. */ 72 buf = malloc(strlen(str) + 1); 73 if (!buf) 74 return -ENOMEM; 75 76 strcpy(buf, str); 77 78 tok = strtok_r((char *)buf, ",", &saveptr); 79 80 while (tok) { 81 for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) { 82 struct perf_mem_event *e = perf_mem_events__ptr(j); 83 84 if (!e->tag) 85 continue; 86 87 if (strstr(e->tag, tok)) 88 e->record = found = true; 89 } 90 91 tok = strtok_r(NULL, ",", &saveptr); 92 } 93 94 free(buf); 95 96 if (found) 97 return 0; 98 99 pr_err("failed: event '%s' not found, use '-e list' to get list of available events\n", str); 100 return -1; 101 } 102 103 int perf_mem_events__init(void) 104 { 105 const char *mnt = sysfs__mount(); 106 bool found = false; 107 int j; 108 109 if (!mnt) 110 return -ENOENT; 111 112 for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) { 113 char path[PATH_MAX]; 114 struct perf_mem_event *e = perf_mem_events__ptr(j); 115 struct stat st; 116 117 /* 118 * If the event entry isn't valid, skip initialization 119 * and "e->supported" will keep false. 120 */ 121 if (!e->tag) 122 continue; 123 124 scnprintf(path, PATH_MAX, "%s/devices/%s", 125 mnt, e->sysfs_name); 126 127 if (!stat(path, &st)) 128 e->supported = found = true; 129 } 130 131 return found ? 0 : -ENOENT; 132 } 133 134 void perf_mem_events__list(void) 135 { 136 int j; 137 138 for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) { 139 struct perf_mem_event *e = perf_mem_events__ptr(j); 140 141 fprintf(stderr, "%-13s%-*s%s\n", 142 e->tag ?: "", 143 verbose > 0 ? 25 : 0, 144 verbose > 0 ? perf_mem_events__name(j) : "", 145 e->supported ? ": available" : ""); 146 } 147 } 148 149 static const char * const tlb_access[] = { 150 "N/A", 151 "HIT", 152 "MISS", 153 "L1", 154 "L2", 155 "Walker", 156 "Fault", 157 }; 158 159 int perf_mem__tlb_scnprintf(char *out, size_t sz, struct mem_info *mem_info) 160 { 161 size_t l = 0, i; 162 u64 m = PERF_MEM_TLB_NA; 163 u64 hit, miss; 164 165 sz -= 1; /* -1 for null termination */ 166 out[0] = '\0'; 167 168 if (mem_info) 169 m = mem_info->data_src.mem_dtlb; 170 171 hit = m & PERF_MEM_TLB_HIT; 172 miss = m & PERF_MEM_TLB_MISS; 173 174 /* already taken care of */ 175 m &= ~(PERF_MEM_TLB_HIT|PERF_MEM_TLB_MISS); 176 177 for (i = 0; m && i < ARRAY_SIZE(tlb_access); i++, m >>= 1) { 178 if (!(m & 0x1)) 179 continue; 180 if (l) { 181 strcat(out, " or "); 182 l += 4; 183 } 184 l += scnprintf(out + l, sz - l, tlb_access[i]); 185 } 186 if (*out == '\0') 187 l += scnprintf(out, sz - l, "N/A"); 188 if (hit) 189 l += scnprintf(out + l, sz - l, " hit"); 190 if (miss) 191 l += scnprintf(out + l, sz - l, " miss"); 192 193 return l; 194 } 195 196 static const char * const mem_lvl[] = { 197 "N/A", 198 "HIT", 199 "MISS", 200 "L1", 201 "LFB", 202 "L2", 203 "L3", 204 "Local RAM", 205 "Remote RAM (1 hop)", 206 "Remote RAM (2 hops)", 207 "Remote Cache (1 hop)", 208 "Remote Cache (2 hops)", 209 "I/O", 210 "Uncached", 211 }; 212 213 static const char * const mem_lvlnum[] = { 214 [PERF_MEM_LVLNUM_ANY_CACHE] = "Any cache", 215 [PERF_MEM_LVLNUM_LFB] = "LFB", 216 [PERF_MEM_LVLNUM_RAM] = "RAM", 217 [PERF_MEM_LVLNUM_PMEM] = "PMEM", 218 [PERF_MEM_LVLNUM_NA] = "N/A", 219 }; 220 221 int perf_mem__lvl_scnprintf(char *out, size_t sz, struct mem_info *mem_info) 222 { 223 size_t i, l = 0; 224 u64 m = PERF_MEM_LVL_NA; 225 u64 hit, miss; 226 int printed; 227 228 if (mem_info) 229 m = mem_info->data_src.mem_lvl; 230 231 sz -= 1; /* -1 for null termination */ 232 out[0] = '\0'; 233 234 hit = m & PERF_MEM_LVL_HIT; 235 miss = m & PERF_MEM_LVL_MISS; 236 237 /* already taken care of */ 238 m &= ~(PERF_MEM_LVL_HIT|PERF_MEM_LVL_MISS); 239 240 241 if (mem_info && mem_info->data_src.mem_remote) { 242 strcat(out, "Remote "); 243 l += 7; 244 } 245 246 printed = 0; 247 for (i = 0; m && i < ARRAY_SIZE(mem_lvl); i++, m >>= 1) { 248 if (!(m & 0x1)) 249 continue; 250 if (printed++) { 251 strcat(out, " or "); 252 l += 4; 253 } 254 l += scnprintf(out + l, sz - l, mem_lvl[i]); 255 } 256 257 if (mem_info && mem_info->data_src.mem_lvl_num) { 258 int lvl = mem_info->data_src.mem_lvl_num; 259 if (printed++) { 260 strcat(out, " or "); 261 l += 4; 262 } 263 if (mem_lvlnum[lvl]) 264 l += scnprintf(out + l, sz - l, mem_lvlnum[lvl]); 265 else 266 l += scnprintf(out + l, sz - l, "L%d", lvl); 267 } 268 269 if (l == 0) 270 l += scnprintf(out + l, sz - l, "N/A"); 271 if (hit) 272 l += scnprintf(out + l, sz - l, " hit"); 273 if (miss) 274 l += scnprintf(out + l, sz - l, " miss"); 275 276 return l; 277 } 278 279 static const char * const snoop_access[] = { 280 "N/A", 281 "None", 282 "Hit", 283 "Miss", 284 "HitM", 285 }; 286 287 int perf_mem__snp_scnprintf(char *out, size_t sz, struct mem_info *mem_info) 288 { 289 size_t i, l = 0; 290 u64 m = PERF_MEM_SNOOP_NA; 291 292 sz -= 1; /* -1 for null termination */ 293 out[0] = '\0'; 294 295 if (mem_info) 296 m = mem_info->data_src.mem_snoop; 297 298 for (i = 0; m && i < ARRAY_SIZE(snoop_access); i++, m >>= 1) { 299 if (!(m & 0x1)) 300 continue; 301 if (l) { 302 strcat(out, " or "); 303 l += 4; 304 } 305 l += scnprintf(out + l, sz - l, snoop_access[i]); 306 } 307 if (mem_info && 308 (mem_info->data_src.mem_snoopx & PERF_MEM_SNOOPX_FWD)) { 309 if (l) { 310 strcat(out, " or "); 311 l += 4; 312 } 313 l += scnprintf(out + l, sz - l, "Fwd"); 314 } 315 316 if (*out == '\0') 317 l += scnprintf(out, sz - l, "N/A"); 318 319 return l; 320 } 321 322 int perf_mem__lck_scnprintf(char *out, size_t sz, struct mem_info *mem_info) 323 { 324 u64 mask = PERF_MEM_LOCK_NA; 325 int l; 326 327 if (mem_info) 328 mask = mem_info->data_src.mem_lock; 329 330 if (mask & PERF_MEM_LOCK_NA) 331 l = scnprintf(out, sz, "N/A"); 332 else if (mask & PERF_MEM_LOCK_LOCKED) 333 l = scnprintf(out, sz, "Yes"); 334 else 335 l = scnprintf(out, sz, "No"); 336 337 return l; 338 } 339 340 int perf_mem__blk_scnprintf(char *out, size_t sz, struct mem_info *mem_info) 341 { 342 size_t l = 0; 343 u64 mask = PERF_MEM_BLK_NA; 344 345 sz -= 1; /* -1 for null termination */ 346 out[0] = '\0'; 347 348 if (mem_info) 349 mask = mem_info->data_src.mem_blk; 350 351 if (!mask || (mask & PERF_MEM_BLK_NA)) { 352 l += scnprintf(out + l, sz - l, " N/A"); 353 return l; 354 } 355 if (mask & PERF_MEM_BLK_DATA) 356 l += scnprintf(out + l, sz - l, " Data"); 357 if (mask & PERF_MEM_BLK_ADDR) 358 l += scnprintf(out + l, sz - l, " Addr"); 359 360 return l; 361 } 362 363 int perf_script__meminfo_scnprintf(char *out, size_t sz, struct mem_info *mem_info) 364 { 365 int i = 0; 366 367 i += perf_mem__lvl_scnprintf(out, sz, mem_info); 368 i += scnprintf(out + i, sz - i, "|SNP "); 369 i += perf_mem__snp_scnprintf(out + i, sz - i, mem_info); 370 i += scnprintf(out + i, sz - i, "|TLB "); 371 i += perf_mem__tlb_scnprintf(out + i, sz - i, mem_info); 372 i += scnprintf(out + i, sz - i, "|LCK "); 373 i += perf_mem__lck_scnprintf(out + i, sz - i, mem_info); 374 i += scnprintf(out + i, sz - i, "|BLK "); 375 i += perf_mem__blk_scnprintf(out + i, sz - i, mem_info); 376 377 return i; 378 } 379 380 int c2c_decode_stats(struct c2c_stats *stats, struct mem_info *mi) 381 { 382 union perf_mem_data_src *data_src = &mi->data_src; 383 u64 daddr = mi->daddr.addr; 384 u64 op = data_src->mem_op; 385 u64 lvl = data_src->mem_lvl; 386 u64 snoop = data_src->mem_snoop; 387 u64 lock = data_src->mem_lock; 388 u64 blk = data_src->mem_blk; 389 /* 390 * Skylake might report unknown remote level via this 391 * bit, consider it when evaluating remote HITMs. 392 */ 393 bool mrem = data_src->mem_remote; 394 int err = 0; 395 396 #define HITM_INC(__f) \ 397 do { \ 398 stats->__f++; \ 399 stats->tot_hitm++; \ 400 } while (0) 401 402 #define P(a, b) PERF_MEM_##a##_##b 403 404 stats->nr_entries++; 405 406 if (lock & P(LOCK, LOCKED)) stats->locks++; 407 408 if (blk & P(BLK, DATA)) stats->blk_data++; 409 if (blk & P(BLK, ADDR)) stats->blk_addr++; 410 411 if (op & P(OP, LOAD)) { 412 /* load */ 413 stats->load++; 414 415 if (!daddr) { 416 stats->ld_noadrs++; 417 return -1; 418 } 419 420 if (lvl & P(LVL, HIT)) { 421 if (lvl & P(LVL, UNC)) stats->ld_uncache++; 422 if (lvl & P(LVL, IO)) stats->ld_io++; 423 if (lvl & P(LVL, LFB)) stats->ld_fbhit++; 424 if (lvl & P(LVL, L1 )) stats->ld_l1hit++; 425 if (lvl & P(LVL, L2 )) stats->ld_l2hit++; 426 if (lvl & P(LVL, L3 )) { 427 if (snoop & P(SNOOP, HITM)) 428 HITM_INC(lcl_hitm); 429 else 430 stats->ld_llchit++; 431 } 432 433 if (lvl & P(LVL, LOC_RAM)) { 434 stats->lcl_dram++; 435 if (snoop & P(SNOOP, HIT)) 436 stats->ld_shared++; 437 else 438 stats->ld_excl++; 439 } 440 441 if ((lvl & P(LVL, REM_RAM1)) || 442 (lvl & P(LVL, REM_RAM2)) || 443 mrem) { 444 stats->rmt_dram++; 445 if (snoop & P(SNOOP, HIT)) 446 stats->ld_shared++; 447 else 448 stats->ld_excl++; 449 } 450 } 451 452 if ((lvl & P(LVL, REM_CCE1)) || 453 (lvl & P(LVL, REM_CCE2)) || 454 mrem) { 455 if (snoop & P(SNOOP, HIT)) 456 stats->rmt_hit++; 457 else if (snoop & P(SNOOP, HITM)) 458 HITM_INC(rmt_hitm); 459 } 460 461 if ((lvl & P(LVL, MISS))) 462 stats->ld_miss++; 463 464 } else if (op & P(OP, STORE)) { 465 /* store */ 466 stats->store++; 467 468 if (!daddr) { 469 stats->st_noadrs++; 470 return -1; 471 } 472 473 if (lvl & P(LVL, HIT)) { 474 if (lvl & P(LVL, UNC)) stats->st_uncache++; 475 if (lvl & P(LVL, L1 )) stats->st_l1hit++; 476 } 477 if (lvl & P(LVL, MISS)) 478 if (lvl & P(LVL, L1)) stats->st_l1miss++; 479 } else { 480 /* unparsable data_src? */ 481 stats->noparse++; 482 return -1; 483 } 484 485 if (!mi->daddr.ms.map || !mi->iaddr.ms.map) { 486 stats->nomap++; 487 return -1; 488 } 489 490 #undef P 491 #undef HITM_INC 492 return err; 493 } 494 495 void c2c_add_stats(struct c2c_stats *stats, struct c2c_stats *add) 496 { 497 stats->nr_entries += add->nr_entries; 498 499 stats->locks += add->locks; 500 stats->store += add->store; 501 stats->st_uncache += add->st_uncache; 502 stats->st_noadrs += add->st_noadrs; 503 stats->st_l1hit += add->st_l1hit; 504 stats->st_l1miss += add->st_l1miss; 505 stats->load += add->load; 506 stats->ld_excl += add->ld_excl; 507 stats->ld_shared += add->ld_shared; 508 stats->ld_uncache += add->ld_uncache; 509 stats->ld_io += add->ld_io; 510 stats->ld_miss += add->ld_miss; 511 stats->ld_noadrs += add->ld_noadrs; 512 stats->ld_fbhit += add->ld_fbhit; 513 stats->ld_l1hit += add->ld_l1hit; 514 stats->ld_l2hit += add->ld_l2hit; 515 stats->ld_llchit += add->ld_llchit; 516 stats->lcl_hitm += add->lcl_hitm; 517 stats->rmt_hitm += add->rmt_hitm; 518 stats->tot_hitm += add->tot_hitm; 519 stats->rmt_hit += add->rmt_hit; 520 stats->lcl_dram += add->lcl_dram; 521 stats->rmt_dram += add->rmt_dram; 522 stats->blk_data += add->blk_data; 523 stats->blk_addr += add->blk_addr; 524 stats->nomap += add->nomap; 525 stats->noparse += add->noparse; 526 } 527