1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/perf_event.h> 3 #include <linux/types.h> 4 5 #include <asm/perf_event.h> 6 #include <asm/msr.h> 7 #include <asm/insn.h> 8 9 #include "../perf_event.h" 10 11 /* 12 * Intel LBR_SELECT bits 13 * Intel Vol3a, April 2011, Section 16.7 Table 16-10 14 * 15 * Hardware branch filter (not available on all CPUs) 16 */ 17 #define LBR_KERNEL_BIT 0 /* do not capture at ring0 */ 18 #define LBR_USER_BIT 1 /* do not capture at ring > 0 */ 19 #define LBR_JCC_BIT 2 /* do not capture conditional branches */ 20 #define LBR_REL_CALL_BIT 3 /* do not capture relative calls */ 21 #define LBR_IND_CALL_BIT 4 /* do not capture indirect calls */ 22 #define LBR_RETURN_BIT 5 /* do not capture near returns */ 23 #define LBR_IND_JMP_BIT 6 /* do not capture indirect jumps */ 24 #define LBR_REL_JMP_BIT 7 /* do not capture relative jumps */ 25 #define LBR_FAR_BIT 8 /* do not capture far branches */ 26 #define LBR_CALL_STACK_BIT 9 /* enable call stack */ 27 28 /* 29 * Following bit only exists in Linux; we mask it out before writing it to 30 * the actual MSR. But it helps the constraint perf code to understand 31 * that this is a separate configuration. 32 */ 33 #define LBR_NO_INFO_BIT 63 /* don't read LBR_INFO. */ 34 35 #define LBR_KERNEL (1 << LBR_KERNEL_BIT) 36 #define LBR_USER (1 << LBR_USER_BIT) 37 #define LBR_JCC (1 << LBR_JCC_BIT) 38 #define LBR_REL_CALL (1 << LBR_REL_CALL_BIT) 39 #define LBR_IND_CALL (1 << LBR_IND_CALL_BIT) 40 #define LBR_RETURN (1 << LBR_RETURN_BIT) 41 #define LBR_REL_JMP (1 << LBR_REL_JMP_BIT) 42 #define LBR_IND_JMP (1 << LBR_IND_JMP_BIT) 43 #define LBR_FAR (1 << LBR_FAR_BIT) 44 #define LBR_CALL_STACK (1 << LBR_CALL_STACK_BIT) 45 #define LBR_NO_INFO (1ULL << LBR_NO_INFO_BIT) 46 47 #define LBR_PLM (LBR_KERNEL | LBR_USER) 48 49 #define LBR_SEL_MASK 0x3ff /* valid bits in LBR_SELECT */ 50 #define LBR_NOT_SUPP -1 /* LBR filter not supported */ 51 #define LBR_IGN 0 /* ignored */ 52 53 #define LBR_ANY \ 54 (LBR_JCC |\ 55 LBR_REL_CALL |\ 56 LBR_IND_CALL |\ 57 LBR_RETURN |\ 58 LBR_REL_JMP |\ 59 LBR_IND_JMP |\ 60 LBR_FAR) 61 62 #define LBR_FROM_FLAG_MISPRED BIT_ULL(63) 63 #define LBR_FROM_FLAG_IN_TX BIT_ULL(62) 64 #define LBR_FROM_FLAG_ABORT BIT_ULL(61) 65 66 #define LBR_FROM_SIGNEXT_2MSB (BIT_ULL(60) | BIT_ULL(59)) 67 68 /* 69 * x86control flow change classification 70 * x86control flow changes include branches, interrupts, traps, faults 71 */ 72 enum { 73 X86_BR_NONE = 0, /* unknown */ 74 75 X86_BR_USER = 1 << 0, /* branch target is user */ 76 X86_BR_KERNEL = 1 << 1, /* branch target is kernel */ 77 78 X86_BR_CALL = 1 << 2, /* call */ 79 X86_BR_RET = 1 << 3, /* return */ 80 X86_BR_SYSCALL = 1 << 4, /* syscall */ 81 X86_BR_SYSRET = 1 << 5, /* syscall return */ 82 X86_BR_INT = 1 << 6, /* sw interrupt */ 83 X86_BR_IRET = 1 << 7, /* return from interrupt */ 84 X86_BR_JCC = 1 << 8, /* conditional */ 85 X86_BR_JMP = 1 << 9, /* jump */ 86 X86_BR_IRQ = 1 << 10,/* hw interrupt or trap or fault */ 87 X86_BR_IND_CALL = 1 << 11,/* indirect calls */ 88 X86_BR_ABORT = 1 << 12,/* transaction abort */ 89 X86_BR_IN_TX = 1 << 13,/* in transaction */ 90 X86_BR_NO_TX = 1 << 14,/* not in transaction */ 91 X86_BR_ZERO_CALL = 1 << 15,/* zero length call */ 92 X86_BR_CALL_STACK = 1 << 16,/* call stack */ 93 X86_BR_IND_JMP = 1 << 17,/* indirect jump */ 94 95 X86_BR_TYPE_SAVE = 1 << 18,/* indicate to save branch type */ 96 97 }; 98 99 #define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL) 100 #define X86_BR_ANYTX (X86_BR_NO_TX | X86_BR_IN_TX) 101 102 #define X86_BR_ANY \ 103 (X86_BR_CALL |\ 104 X86_BR_RET |\ 105 X86_BR_SYSCALL |\ 106 X86_BR_SYSRET |\ 107 X86_BR_INT |\ 108 X86_BR_IRET |\ 109 X86_BR_JCC |\ 110 X86_BR_JMP |\ 111 X86_BR_IRQ |\ 112 X86_BR_ABORT |\ 113 X86_BR_IND_CALL |\ 114 X86_BR_IND_JMP |\ 115 X86_BR_ZERO_CALL) 116 117 #define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY) 118 119 #define X86_BR_ANY_CALL \ 120 (X86_BR_CALL |\ 121 X86_BR_IND_CALL |\ 122 X86_BR_ZERO_CALL |\ 123 X86_BR_SYSCALL |\ 124 X86_BR_IRQ |\ 125 X86_BR_INT) 126 127 /* 128 * Intel LBR_CTL bits 129 * 130 * Hardware branch filter for Arch LBR 131 */ 132 #define ARCH_LBR_KERNEL_BIT 1 /* capture at ring0 */ 133 #define ARCH_LBR_USER_BIT 2 /* capture at ring > 0 */ 134 #define ARCH_LBR_CALL_STACK_BIT 3 /* enable call stack */ 135 #define ARCH_LBR_JCC_BIT 16 /* capture conditional branches */ 136 #define ARCH_LBR_REL_JMP_BIT 17 /* capture relative jumps */ 137 #define ARCH_LBR_IND_JMP_BIT 18 /* capture indirect jumps */ 138 #define ARCH_LBR_REL_CALL_BIT 19 /* capture relative calls */ 139 #define ARCH_LBR_IND_CALL_BIT 20 /* capture indirect calls */ 140 #define ARCH_LBR_RETURN_BIT 21 /* capture near returns */ 141 #define ARCH_LBR_OTHER_BRANCH_BIT 22 /* capture other branches */ 142 143 #define ARCH_LBR_KERNEL (1ULL << ARCH_LBR_KERNEL_BIT) 144 #define ARCH_LBR_USER (1ULL << ARCH_LBR_USER_BIT) 145 #define ARCH_LBR_CALL_STACK (1ULL << ARCH_LBR_CALL_STACK_BIT) 146 #define ARCH_LBR_JCC (1ULL << ARCH_LBR_JCC_BIT) 147 #define ARCH_LBR_REL_JMP (1ULL << ARCH_LBR_REL_JMP_BIT) 148 #define ARCH_LBR_IND_JMP (1ULL << ARCH_LBR_IND_JMP_BIT) 149 #define ARCH_LBR_REL_CALL (1ULL << ARCH_LBR_REL_CALL_BIT) 150 #define ARCH_LBR_IND_CALL (1ULL << ARCH_LBR_IND_CALL_BIT) 151 #define ARCH_LBR_RETURN (1ULL << ARCH_LBR_RETURN_BIT) 152 #define ARCH_LBR_OTHER_BRANCH (1ULL << ARCH_LBR_OTHER_BRANCH_BIT) 153 154 #define ARCH_LBR_ANY \ 155 (ARCH_LBR_JCC |\ 156 ARCH_LBR_REL_JMP |\ 157 ARCH_LBR_IND_JMP |\ 158 ARCH_LBR_REL_CALL |\ 159 ARCH_LBR_IND_CALL |\ 160 ARCH_LBR_RETURN |\ 161 ARCH_LBR_OTHER_BRANCH) 162 163 #define ARCH_LBR_CTL_MASK 0x7f000e 164 165 static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc); 166 167 static __always_inline bool is_lbr_call_stack_bit_set(u64 config) 168 { 169 if (static_cpu_has(X86_FEATURE_ARCH_LBR)) 170 return !!(config & ARCH_LBR_CALL_STACK); 171 172 return !!(config & LBR_CALL_STACK); 173 } 174 175 /* 176 * We only support LBR implementations that have FREEZE_LBRS_ON_PMI 177 * otherwise it becomes near impossible to get a reliable stack. 178 */ 179 180 static void __intel_pmu_lbr_enable(bool pmi) 181 { 182 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 183 u64 debugctl, lbr_select = 0, orig_debugctl; 184 185 /* 186 * No need to unfreeze manually, as v4 can do that as part 187 * of the GLOBAL_STATUS ack. 188 */ 189 if (pmi && x86_pmu.version >= 4) 190 return; 191 192 /* 193 * No need to reprogram LBR_SELECT in a PMI, as it 194 * did not change. 195 */ 196 if (cpuc->lbr_sel) 197 lbr_select = cpuc->lbr_sel->config & x86_pmu.lbr_sel_mask; 198 if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && !pmi && cpuc->lbr_sel) 199 wrmsrl(MSR_LBR_SELECT, lbr_select); 200 201 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 202 orig_debugctl = debugctl; 203 204 if (!static_cpu_has(X86_FEATURE_ARCH_LBR)) 205 debugctl |= DEBUGCTLMSR_LBR; 206 /* 207 * LBR callstack does not work well with FREEZE_LBRS_ON_PMI. 208 * If FREEZE_LBRS_ON_PMI is set, PMI near call/return instructions 209 * may cause superfluous increase/decrease of LBR_TOS. 210 */ 211 if (is_lbr_call_stack_bit_set(lbr_select)) 212 debugctl &= ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; 213 else 214 debugctl |= DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; 215 216 if (orig_debugctl != debugctl) 217 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 218 219 if (static_cpu_has(X86_FEATURE_ARCH_LBR)) 220 wrmsrl(MSR_ARCH_LBR_CTL, lbr_select | ARCH_LBR_CTL_LBREN); 221 } 222 223 void intel_pmu_lbr_reset_32(void) 224 { 225 int i; 226 227 for (i = 0; i < x86_pmu.lbr_nr; i++) 228 wrmsrl(x86_pmu.lbr_from + i, 0); 229 } 230 231 void intel_pmu_lbr_reset_64(void) 232 { 233 int i; 234 235 for (i = 0; i < x86_pmu.lbr_nr; i++) { 236 wrmsrl(x86_pmu.lbr_from + i, 0); 237 wrmsrl(x86_pmu.lbr_to + i, 0); 238 if (x86_pmu.lbr_has_info) 239 wrmsrl(x86_pmu.lbr_info + i, 0); 240 } 241 } 242 243 static void intel_pmu_arch_lbr_reset(void) 244 { 245 /* Write to ARCH_LBR_DEPTH MSR, all LBR entries are reset to 0 */ 246 wrmsrl(MSR_ARCH_LBR_DEPTH, x86_pmu.lbr_nr); 247 } 248 249 void intel_pmu_lbr_reset(void) 250 { 251 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 252 253 if (!x86_pmu.lbr_nr) 254 return; 255 256 x86_pmu.lbr_reset(); 257 258 cpuc->last_task_ctx = NULL; 259 cpuc->last_log_id = 0; 260 if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && cpuc->lbr_select) 261 wrmsrl(MSR_LBR_SELECT, 0); 262 } 263 264 /* 265 * TOS = most recently recorded branch 266 */ 267 static inline u64 intel_pmu_lbr_tos(void) 268 { 269 u64 tos; 270 271 rdmsrl(x86_pmu.lbr_tos, tos); 272 return tos; 273 } 274 275 enum { 276 LBR_NONE, 277 LBR_VALID, 278 }; 279 280 /* 281 * For format LBR_FORMAT_EIP_FLAGS2, bits 61:62 in MSR_LAST_BRANCH_FROM_x 282 * are the TSX flags when TSX is supported, but when TSX is not supported 283 * they have no consistent behavior: 284 * 285 * - For wrmsr(), bits 61:62 are considered part of the sign extension. 286 * - For HW updates (branch captures) bits 61:62 are always OFF and are not 287 * part of the sign extension. 288 * 289 * Therefore, if: 290 * 291 * 1) LBR format LBR_FORMAT_EIP_FLAGS2 292 * 2) CPU has no TSX support enabled 293 * 294 * ... then any value passed to wrmsr() must be sign extended to 63 bits and any 295 * value from rdmsr() must be converted to have a 61 bits sign extension, 296 * ignoring the TSX flags. 297 */ 298 static inline bool lbr_from_signext_quirk_needed(void) 299 { 300 bool tsx_support = boot_cpu_has(X86_FEATURE_HLE) || 301 boot_cpu_has(X86_FEATURE_RTM); 302 303 return !tsx_support; 304 } 305 306 static DEFINE_STATIC_KEY_FALSE(lbr_from_quirk_key); 307 308 /* If quirk is enabled, ensure sign extension is 63 bits: */ 309 inline u64 lbr_from_signext_quirk_wr(u64 val) 310 { 311 if (static_branch_unlikely(&lbr_from_quirk_key)) { 312 /* 313 * Sign extend into bits 61:62 while preserving bit 63. 314 * 315 * Quirk is enabled when TSX is disabled. Therefore TSX bits 316 * in val are always OFF and must be changed to be sign 317 * extension bits. Since bits 59:60 are guaranteed to be 318 * part of the sign extension bits, we can just copy them 319 * to 61:62. 320 */ 321 val |= (LBR_FROM_SIGNEXT_2MSB & val) << 2; 322 } 323 return val; 324 } 325 326 /* 327 * If quirk is needed, ensure sign extension is 61 bits: 328 */ 329 static u64 lbr_from_signext_quirk_rd(u64 val) 330 { 331 if (static_branch_unlikely(&lbr_from_quirk_key)) { 332 /* 333 * Quirk is on when TSX is not enabled. Therefore TSX 334 * flags must be read as OFF. 335 */ 336 val &= ~(LBR_FROM_FLAG_IN_TX | LBR_FROM_FLAG_ABORT); 337 } 338 return val; 339 } 340 341 static __always_inline void wrlbr_from(unsigned int idx, u64 val) 342 { 343 val = lbr_from_signext_quirk_wr(val); 344 wrmsrl(x86_pmu.lbr_from + idx, val); 345 } 346 347 static __always_inline void wrlbr_to(unsigned int idx, u64 val) 348 { 349 wrmsrl(x86_pmu.lbr_to + idx, val); 350 } 351 352 static __always_inline void wrlbr_info(unsigned int idx, u64 val) 353 { 354 wrmsrl(x86_pmu.lbr_info + idx, val); 355 } 356 357 static __always_inline u64 rdlbr_from(unsigned int idx, struct lbr_entry *lbr) 358 { 359 u64 val; 360 361 if (lbr) 362 return lbr->from; 363 364 rdmsrl(x86_pmu.lbr_from + idx, val); 365 366 return lbr_from_signext_quirk_rd(val); 367 } 368 369 static __always_inline u64 rdlbr_to(unsigned int idx, struct lbr_entry *lbr) 370 { 371 u64 val; 372 373 if (lbr) 374 return lbr->to; 375 376 rdmsrl(x86_pmu.lbr_to + idx, val); 377 378 return val; 379 } 380 381 static __always_inline u64 rdlbr_info(unsigned int idx, struct lbr_entry *lbr) 382 { 383 u64 val; 384 385 if (lbr) 386 return lbr->info; 387 388 rdmsrl(x86_pmu.lbr_info + idx, val); 389 390 return val; 391 } 392 393 static inline void 394 wrlbr_all(struct lbr_entry *lbr, unsigned int idx, bool need_info) 395 { 396 wrlbr_from(idx, lbr->from); 397 wrlbr_to(idx, lbr->to); 398 if (need_info) 399 wrlbr_info(idx, lbr->info); 400 } 401 402 static inline bool 403 rdlbr_all(struct lbr_entry *lbr, unsigned int idx, bool need_info) 404 { 405 u64 from = rdlbr_from(idx, NULL); 406 407 /* Don't read invalid entry */ 408 if (!from) 409 return false; 410 411 lbr->from = from; 412 lbr->to = rdlbr_to(idx, NULL); 413 if (need_info) 414 lbr->info = rdlbr_info(idx, NULL); 415 416 return true; 417 } 418 419 void intel_pmu_lbr_restore(void *ctx) 420 { 421 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 422 struct x86_perf_task_context *task_ctx = ctx; 423 bool need_info = x86_pmu.lbr_has_info; 424 u64 tos = task_ctx->tos; 425 unsigned lbr_idx, mask; 426 int i; 427 428 mask = x86_pmu.lbr_nr - 1; 429 for (i = 0; i < task_ctx->valid_lbrs; i++) { 430 lbr_idx = (tos - i) & mask; 431 wrlbr_all(&task_ctx->lbr[i], lbr_idx, need_info); 432 } 433 434 for (; i < x86_pmu.lbr_nr; i++) { 435 lbr_idx = (tos - i) & mask; 436 wrlbr_from(lbr_idx, 0); 437 wrlbr_to(lbr_idx, 0); 438 if (need_info) 439 wrlbr_info(lbr_idx, 0); 440 } 441 442 wrmsrl(x86_pmu.lbr_tos, tos); 443 444 if (cpuc->lbr_select) 445 wrmsrl(MSR_LBR_SELECT, task_ctx->lbr_sel); 446 } 447 448 static void intel_pmu_arch_lbr_restore(void *ctx) 449 { 450 struct x86_perf_task_context_arch_lbr *task_ctx = ctx; 451 struct lbr_entry *entries = task_ctx->entries; 452 int i; 453 454 /* Fast reset the LBRs before restore if the call stack is not full. */ 455 if (!entries[x86_pmu.lbr_nr - 1].from) 456 intel_pmu_arch_lbr_reset(); 457 458 for (i = 0; i < x86_pmu.lbr_nr; i++) { 459 if (!entries[i].from) 460 break; 461 wrlbr_all(&entries[i], i, true); 462 } 463 } 464 465 /* 466 * Restore the Architecture LBR state from the xsave area in the perf 467 * context data for the task via the XRSTORS instruction. 468 */ 469 static void intel_pmu_arch_lbr_xrstors(void *ctx) 470 { 471 struct x86_perf_task_context_arch_lbr_xsave *task_ctx = ctx; 472 473 xrstors(&task_ctx->xsave, XFEATURE_MASK_LBR); 474 } 475 476 static __always_inline bool lbr_is_reset_in_cstate(void *ctx) 477 { 478 if (static_cpu_has(X86_FEATURE_ARCH_LBR)) 479 return x86_pmu.lbr_deep_c_reset && !rdlbr_from(0, NULL); 480 481 return !rdlbr_from(((struct x86_perf_task_context *)ctx)->tos, NULL); 482 } 483 484 static void __intel_pmu_lbr_restore(void *ctx) 485 { 486 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 487 488 if (task_context_opt(ctx)->lbr_callstack_users == 0 || 489 task_context_opt(ctx)->lbr_stack_state == LBR_NONE) { 490 intel_pmu_lbr_reset(); 491 return; 492 } 493 494 /* 495 * Does not restore the LBR registers, if 496 * - No one else touched them, and 497 * - Was not cleared in Cstate 498 */ 499 if ((ctx == cpuc->last_task_ctx) && 500 (task_context_opt(ctx)->log_id == cpuc->last_log_id) && 501 !lbr_is_reset_in_cstate(ctx)) { 502 task_context_opt(ctx)->lbr_stack_state = LBR_NONE; 503 return; 504 } 505 506 x86_pmu.lbr_restore(ctx); 507 508 task_context_opt(ctx)->lbr_stack_state = LBR_NONE; 509 } 510 511 void intel_pmu_lbr_save(void *ctx) 512 { 513 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 514 struct x86_perf_task_context *task_ctx = ctx; 515 bool need_info = x86_pmu.lbr_has_info; 516 unsigned lbr_idx, mask; 517 u64 tos; 518 int i; 519 520 mask = x86_pmu.lbr_nr - 1; 521 tos = intel_pmu_lbr_tos(); 522 for (i = 0; i < x86_pmu.lbr_nr; i++) { 523 lbr_idx = (tos - i) & mask; 524 if (!rdlbr_all(&task_ctx->lbr[i], lbr_idx, need_info)) 525 break; 526 } 527 task_ctx->valid_lbrs = i; 528 task_ctx->tos = tos; 529 530 if (cpuc->lbr_select) 531 rdmsrl(MSR_LBR_SELECT, task_ctx->lbr_sel); 532 } 533 534 static void intel_pmu_arch_lbr_save(void *ctx) 535 { 536 struct x86_perf_task_context_arch_lbr *task_ctx = ctx; 537 struct lbr_entry *entries = task_ctx->entries; 538 int i; 539 540 for (i = 0; i < x86_pmu.lbr_nr; i++) { 541 if (!rdlbr_all(&entries[i], i, true)) 542 break; 543 } 544 545 /* LBR call stack is not full. Reset is required in restore. */ 546 if (i < x86_pmu.lbr_nr) 547 entries[x86_pmu.lbr_nr - 1].from = 0; 548 } 549 550 /* 551 * Save the Architecture LBR state to the xsave area in the perf 552 * context data for the task via the XSAVES instruction. 553 */ 554 static void intel_pmu_arch_lbr_xsaves(void *ctx) 555 { 556 struct x86_perf_task_context_arch_lbr_xsave *task_ctx = ctx; 557 558 xsaves(&task_ctx->xsave, XFEATURE_MASK_LBR); 559 } 560 561 static void __intel_pmu_lbr_save(void *ctx) 562 { 563 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 564 565 if (task_context_opt(ctx)->lbr_callstack_users == 0) { 566 task_context_opt(ctx)->lbr_stack_state = LBR_NONE; 567 return; 568 } 569 570 x86_pmu.lbr_save(ctx); 571 572 task_context_opt(ctx)->lbr_stack_state = LBR_VALID; 573 574 cpuc->last_task_ctx = ctx; 575 cpuc->last_log_id = ++task_context_opt(ctx)->log_id; 576 } 577 578 void intel_pmu_lbr_swap_task_ctx(struct perf_event_context *prev, 579 struct perf_event_context *next) 580 { 581 void *prev_ctx_data, *next_ctx_data; 582 583 swap(prev->task_ctx_data, next->task_ctx_data); 584 585 /* 586 * Architecture specific synchronization makes sense in 587 * case both prev->task_ctx_data and next->task_ctx_data 588 * pointers are allocated. 589 */ 590 591 prev_ctx_data = next->task_ctx_data; 592 next_ctx_data = prev->task_ctx_data; 593 594 if (!prev_ctx_data || !next_ctx_data) 595 return; 596 597 swap(task_context_opt(prev_ctx_data)->lbr_callstack_users, 598 task_context_opt(next_ctx_data)->lbr_callstack_users); 599 } 600 601 void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in) 602 { 603 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 604 void *task_ctx; 605 606 if (!cpuc->lbr_users) 607 return; 608 609 /* 610 * If LBR callstack feature is enabled and the stack was saved when 611 * the task was scheduled out, restore the stack. Otherwise flush 612 * the LBR stack. 613 */ 614 task_ctx = ctx ? ctx->task_ctx_data : NULL; 615 if (task_ctx) { 616 if (sched_in) 617 __intel_pmu_lbr_restore(task_ctx); 618 else 619 __intel_pmu_lbr_save(task_ctx); 620 return; 621 } 622 623 /* 624 * Since a context switch can flip the address space and LBR entries 625 * are not tagged with an identifier, we need to wipe the LBR, even for 626 * per-cpu events. You simply cannot resolve the branches from the old 627 * address space. 628 */ 629 if (sched_in) 630 intel_pmu_lbr_reset(); 631 } 632 633 static inline bool branch_user_callstack(unsigned br_sel) 634 { 635 return (br_sel & X86_BR_USER) && (br_sel & X86_BR_CALL_STACK); 636 } 637 638 void intel_pmu_lbr_add(struct perf_event *event) 639 { 640 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 641 642 if (!x86_pmu.lbr_nr) 643 return; 644 645 if (event->hw.flags & PERF_X86_EVENT_LBR_SELECT) 646 cpuc->lbr_select = 1; 647 648 cpuc->br_sel = event->hw.branch_reg.reg; 649 650 if (branch_user_callstack(cpuc->br_sel) && event->ctx->task_ctx_data) 651 task_context_opt(event->ctx->task_ctx_data)->lbr_callstack_users++; 652 653 /* 654 * Request pmu::sched_task() callback, which will fire inside the 655 * regular perf event scheduling, so that call will: 656 * 657 * - restore or wipe; when LBR-callstack, 658 * - wipe; otherwise, 659 * 660 * when this is from __perf_event_task_sched_in(). 661 * 662 * However, if this is from perf_install_in_context(), no such callback 663 * will follow and we'll need to reset the LBR here if this is the 664 * first LBR event. 665 * 666 * The problem is, we cannot tell these cases apart... but we can 667 * exclude the biggest chunk of cases by looking at 668 * event->total_time_running. An event that has accrued runtime cannot 669 * be 'new'. Conversely, a new event can get installed through the 670 * context switch path for the first time. 671 */ 672 if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip > 0) 673 cpuc->lbr_pebs_users++; 674 perf_sched_cb_inc(event->ctx->pmu); 675 if (!cpuc->lbr_users++ && !event->total_time_running) 676 intel_pmu_lbr_reset(); 677 } 678 679 void release_lbr_buffers(void) 680 { 681 struct kmem_cache *kmem_cache; 682 struct cpu_hw_events *cpuc; 683 int cpu; 684 685 if (!static_cpu_has(X86_FEATURE_ARCH_LBR)) 686 return; 687 688 for_each_possible_cpu(cpu) { 689 cpuc = per_cpu_ptr(&cpu_hw_events, cpu); 690 kmem_cache = x86_get_pmu(cpu)->task_ctx_cache; 691 if (kmem_cache && cpuc->lbr_xsave) { 692 kmem_cache_free(kmem_cache, cpuc->lbr_xsave); 693 cpuc->lbr_xsave = NULL; 694 } 695 } 696 } 697 698 void reserve_lbr_buffers(void) 699 { 700 struct kmem_cache *kmem_cache; 701 struct cpu_hw_events *cpuc; 702 int cpu; 703 704 if (!static_cpu_has(X86_FEATURE_ARCH_LBR)) 705 return; 706 707 for_each_possible_cpu(cpu) { 708 cpuc = per_cpu_ptr(&cpu_hw_events, cpu); 709 kmem_cache = x86_get_pmu(cpu)->task_ctx_cache; 710 if (!kmem_cache || cpuc->lbr_xsave) 711 continue; 712 713 cpuc->lbr_xsave = kmem_cache_alloc_node(kmem_cache, 714 GFP_KERNEL | __GFP_ZERO, 715 cpu_to_node(cpu)); 716 } 717 } 718 719 void intel_pmu_lbr_del(struct perf_event *event) 720 { 721 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 722 723 if (!x86_pmu.lbr_nr) 724 return; 725 726 if (branch_user_callstack(cpuc->br_sel) && 727 event->ctx->task_ctx_data) 728 task_context_opt(event->ctx->task_ctx_data)->lbr_callstack_users--; 729 730 if (event->hw.flags & PERF_X86_EVENT_LBR_SELECT) 731 cpuc->lbr_select = 0; 732 733 if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip > 0) 734 cpuc->lbr_pebs_users--; 735 cpuc->lbr_users--; 736 WARN_ON_ONCE(cpuc->lbr_users < 0); 737 WARN_ON_ONCE(cpuc->lbr_pebs_users < 0); 738 perf_sched_cb_dec(event->ctx->pmu); 739 } 740 741 static inline bool vlbr_exclude_host(void) 742 { 743 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 744 745 return test_bit(INTEL_PMC_IDX_FIXED_VLBR, 746 (unsigned long *)&cpuc->intel_ctrl_guest_mask); 747 } 748 749 void intel_pmu_lbr_enable_all(bool pmi) 750 { 751 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 752 753 if (cpuc->lbr_users && !vlbr_exclude_host()) 754 __intel_pmu_lbr_enable(pmi); 755 } 756 757 void intel_pmu_lbr_disable_all(void) 758 { 759 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 760 761 if (cpuc->lbr_users && !vlbr_exclude_host()) { 762 if (static_cpu_has(X86_FEATURE_ARCH_LBR)) 763 return __intel_pmu_arch_lbr_disable(); 764 765 __intel_pmu_lbr_disable(); 766 } 767 } 768 769 void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) 770 { 771 unsigned long mask = x86_pmu.lbr_nr - 1; 772 struct perf_branch_entry *br = cpuc->lbr_entries; 773 u64 tos = intel_pmu_lbr_tos(); 774 int i; 775 776 for (i = 0; i < x86_pmu.lbr_nr; i++) { 777 unsigned long lbr_idx = (tos - i) & mask; 778 union { 779 struct { 780 u32 from; 781 u32 to; 782 }; 783 u64 lbr; 784 } msr_lastbranch; 785 786 rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr); 787 788 perf_clear_branch_entry_bitfields(br); 789 790 br->from = msr_lastbranch.from; 791 br->to = msr_lastbranch.to; 792 br++; 793 } 794 cpuc->lbr_stack.nr = i; 795 cpuc->lbr_stack.hw_idx = tos; 796 } 797 798 /* 799 * Due to lack of segmentation in Linux the effective address (offset) 800 * is the same as the linear address, allowing us to merge the LIP and EIP 801 * LBR formats. 802 */ 803 void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) 804 { 805 bool need_info = false, call_stack = false; 806 unsigned long mask = x86_pmu.lbr_nr - 1; 807 struct perf_branch_entry *br = cpuc->lbr_entries; 808 u64 tos = intel_pmu_lbr_tos(); 809 int i; 810 int out = 0; 811 int num = x86_pmu.lbr_nr; 812 813 if (cpuc->lbr_sel) { 814 need_info = !(cpuc->lbr_sel->config & LBR_NO_INFO); 815 if (cpuc->lbr_sel->config & LBR_CALL_STACK) 816 call_stack = true; 817 } 818 819 for (i = 0; i < num; i++) { 820 unsigned long lbr_idx = (tos - i) & mask; 821 u64 from, to, mis = 0, pred = 0, in_tx = 0, abort = 0; 822 u16 cycles = 0; 823 824 from = rdlbr_from(lbr_idx, NULL); 825 to = rdlbr_to(lbr_idx, NULL); 826 827 /* 828 * Read LBR call stack entries 829 * until invalid entry (0s) is detected. 830 */ 831 if (call_stack && !from) 832 break; 833 834 if (x86_pmu.lbr_has_info) { 835 if (need_info) { 836 u64 info; 837 838 info = rdlbr_info(lbr_idx, NULL); 839 mis = !!(info & LBR_INFO_MISPRED); 840 pred = !mis; 841 cycles = (info & LBR_INFO_CYCLES); 842 if (x86_pmu.lbr_has_tsx) { 843 in_tx = !!(info & LBR_INFO_IN_TX); 844 abort = !!(info & LBR_INFO_ABORT); 845 } 846 } 847 } else { 848 int skip = 0; 849 850 if (x86_pmu.lbr_from_flags) { 851 mis = !!(from & LBR_FROM_FLAG_MISPRED); 852 pred = !mis; 853 skip = 1; 854 } 855 if (x86_pmu.lbr_has_tsx) { 856 in_tx = !!(from & LBR_FROM_FLAG_IN_TX); 857 abort = !!(from & LBR_FROM_FLAG_ABORT); 858 skip = 3; 859 } 860 from = (u64)((((s64)from) << skip) >> skip); 861 862 if (x86_pmu.lbr_to_cycles) { 863 cycles = ((to >> 48) & LBR_INFO_CYCLES); 864 to = (u64)((((s64)to) << 16) >> 16); 865 } 866 } 867 868 /* 869 * Some CPUs report duplicated abort records, 870 * with the second entry not having an abort bit set. 871 * Skip them here. This loop runs backwards, 872 * so we need to undo the previous record. 873 * If the abort just happened outside the window 874 * the extra entry cannot be removed. 875 */ 876 if (abort && x86_pmu.lbr_double_abort && out > 0) 877 out--; 878 879 perf_clear_branch_entry_bitfields(br+out); 880 br[out].from = from; 881 br[out].to = to; 882 br[out].mispred = mis; 883 br[out].predicted = pred; 884 br[out].in_tx = in_tx; 885 br[out].abort = abort; 886 br[out].cycles = cycles; 887 out++; 888 } 889 cpuc->lbr_stack.nr = out; 890 cpuc->lbr_stack.hw_idx = tos; 891 } 892 893 static DEFINE_STATIC_KEY_FALSE(x86_lbr_mispred); 894 static DEFINE_STATIC_KEY_FALSE(x86_lbr_cycles); 895 static DEFINE_STATIC_KEY_FALSE(x86_lbr_type); 896 897 static __always_inline int get_lbr_br_type(u64 info) 898 { 899 int type = 0; 900 901 if (static_branch_likely(&x86_lbr_type)) 902 type = (info & LBR_INFO_BR_TYPE) >> LBR_INFO_BR_TYPE_OFFSET; 903 904 return type; 905 } 906 907 static __always_inline bool get_lbr_mispred(u64 info) 908 { 909 bool mispred = 0; 910 911 if (static_branch_likely(&x86_lbr_mispred)) 912 mispred = !!(info & LBR_INFO_MISPRED); 913 914 return mispred; 915 } 916 917 static __always_inline u16 get_lbr_cycles(u64 info) 918 { 919 u16 cycles = info & LBR_INFO_CYCLES; 920 921 if (static_cpu_has(X86_FEATURE_ARCH_LBR) && 922 (!static_branch_likely(&x86_lbr_cycles) || 923 !(info & LBR_INFO_CYC_CNT_VALID))) 924 cycles = 0; 925 926 return cycles; 927 } 928 929 static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc, 930 struct lbr_entry *entries) 931 { 932 struct perf_branch_entry *e; 933 struct lbr_entry *lbr; 934 u64 from, to, info; 935 int i; 936 937 for (i = 0; i < x86_pmu.lbr_nr; i++) { 938 lbr = entries ? &entries[i] : NULL; 939 e = &cpuc->lbr_entries[i]; 940 941 from = rdlbr_from(i, lbr); 942 /* 943 * Read LBR entries until invalid entry (0s) is detected. 944 */ 945 if (!from) 946 break; 947 948 to = rdlbr_to(i, lbr); 949 info = rdlbr_info(i, lbr); 950 951 perf_clear_branch_entry_bitfields(e); 952 953 e->from = from; 954 e->to = to; 955 e->mispred = get_lbr_mispred(info); 956 e->predicted = !e->mispred; 957 e->in_tx = !!(info & LBR_INFO_IN_TX); 958 e->abort = !!(info & LBR_INFO_ABORT); 959 e->cycles = get_lbr_cycles(info); 960 e->type = get_lbr_br_type(info); 961 } 962 963 cpuc->lbr_stack.nr = i; 964 } 965 966 static void intel_pmu_arch_lbr_read(struct cpu_hw_events *cpuc) 967 { 968 intel_pmu_store_lbr(cpuc, NULL); 969 } 970 971 static void intel_pmu_arch_lbr_read_xsave(struct cpu_hw_events *cpuc) 972 { 973 struct x86_perf_task_context_arch_lbr_xsave *xsave = cpuc->lbr_xsave; 974 975 if (!xsave) { 976 intel_pmu_store_lbr(cpuc, NULL); 977 return; 978 } 979 xsaves(&xsave->xsave, XFEATURE_MASK_LBR); 980 981 intel_pmu_store_lbr(cpuc, xsave->lbr.entries); 982 } 983 984 void intel_pmu_lbr_read(void) 985 { 986 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 987 988 /* 989 * Don't read when all LBRs users are using adaptive PEBS. 990 * 991 * This could be smarter and actually check the event, 992 * but this simple approach seems to work for now. 993 */ 994 if (!cpuc->lbr_users || vlbr_exclude_host() || 995 cpuc->lbr_users == cpuc->lbr_pebs_users) 996 return; 997 998 x86_pmu.lbr_read(cpuc); 999 1000 intel_pmu_lbr_filter(cpuc); 1001 } 1002 1003 /* 1004 * SW filter is used: 1005 * - in case there is no HW filter 1006 * - in case the HW filter has errata or limitations 1007 */ 1008 static int intel_pmu_setup_sw_lbr_filter(struct perf_event *event) 1009 { 1010 u64 br_type = event->attr.branch_sample_type; 1011 int mask = 0; 1012 1013 if (br_type & PERF_SAMPLE_BRANCH_USER) 1014 mask |= X86_BR_USER; 1015 1016 if (br_type & PERF_SAMPLE_BRANCH_KERNEL) 1017 mask |= X86_BR_KERNEL; 1018 1019 /* we ignore BRANCH_HV here */ 1020 1021 if (br_type & PERF_SAMPLE_BRANCH_ANY) 1022 mask |= X86_BR_ANY; 1023 1024 if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL) 1025 mask |= X86_BR_ANY_CALL; 1026 1027 if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN) 1028 mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET; 1029 1030 if (br_type & PERF_SAMPLE_BRANCH_IND_CALL) 1031 mask |= X86_BR_IND_CALL; 1032 1033 if (br_type & PERF_SAMPLE_BRANCH_ABORT_TX) 1034 mask |= X86_BR_ABORT; 1035 1036 if (br_type & PERF_SAMPLE_BRANCH_IN_TX) 1037 mask |= X86_BR_IN_TX; 1038 1039 if (br_type & PERF_SAMPLE_BRANCH_NO_TX) 1040 mask |= X86_BR_NO_TX; 1041 1042 if (br_type & PERF_SAMPLE_BRANCH_COND) 1043 mask |= X86_BR_JCC; 1044 1045 if (br_type & PERF_SAMPLE_BRANCH_CALL_STACK) { 1046 if (!x86_pmu_has_lbr_callstack()) 1047 return -EOPNOTSUPP; 1048 if (mask & ~(X86_BR_USER | X86_BR_KERNEL)) 1049 return -EINVAL; 1050 mask |= X86_BR_CALL | X86_BR_IND_CALL | X86_BR_RET | 1051 X86_BR_CALL_STACK; 1052 } 1053 1054 if (br_type & PERF_SAMPLE_BRANCH_IND_JUMP) 1055 mask |= X86_BR_IND_JMP; 1056 1057 if (br_type & PERF_SAMPLE_BRANCH_CALL) 1058 mask |= X86_BR_CALL | X86_BR_ZERO_CALL; 1059 1060 if (br_type & PERF_SAMPLE_BRANCH_TYPE_SAVE) 1061 mask |= X86_BR_TYPE_SAVE; 1062 1063 /* 1064 * stash actual user request into reg, it may 1065 * be used by fixup code for some CPU 1066 */ 1067 event->hw.branch_reg.reg = mask; 1068 return 0; 1069 } 1070 1071 /* 1072 * setup the HW LBR filter 1073 * Used only when available, may not be enough to disambiguate 1074 * all branches, may need the help of the SW filter 1075 */ 1076 static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event) 1077 { 1078 struct hw_perf_event_extra *reg; 1079 u64 br_type = event->attr.branch_sample_type; 1080 u64 mask = 0, v; 1081 int i; 1082 1083 for (i = 0; i < PERF_SAMPLE_BRANCH_MAX_SHIFT; i++) { 1084 if (!(br_type & (1ULL << i))) 1085 continue; 1086 1087 v = x86_pmu.lbr_sel_map[i]; 1088 if (v == LBR_NOT_SUPP) 1089 return -EOPNOTSUPP; 1090 1091 if (v != LBR_IGN) 1092 mask |= v; 1093 } 1094 1095 reg = &event->hw.branch_reg; 1096 reg->idx = EXTRA_REG_LBR; 1097 1098 if (static_cpu_has(X86_FEATURE_ARCH_LBR)) { 1099 reg->config = mask; 1100 1101 /* 1102 * The Arch LBR HW can retrieve the common branch types 1103 * from the LBR_INFO. It doesn't require the high overhead 1104 * SW disassemble. 1105 * Enable the branch type by default for the Arch LBR. 1106 */ 1107 reg->reg |= X86_BR_TYPE_SAVE; 1108 return 0; 1109 } 1110 1111 /* 1112 * The first 9 bits (LBR_SEL_MASK) in LBR_SELECT operate 1113 * in suppress mode. So LBR_SELECT should be set to 1114 * (~mask & LBR_SEL_MASK) | (mask & ~LBR_SEL_MASK) 1115 * But the 10th bit LBR_CALL_STACK does not operate 1116 * in suppress mode. 1117 */ 1118 reg->config = mask ^ (x86_pmu.lbr_sel_mask & ~LBR_CALL_STACK); 1119 1120 if ((br_type & PERF_SAMPLE_BRANCH_NO_CYCLES) && 1121 (br_type & PERF_SAMPLE_BRANCH_NO_FLAGS) && 1122 x86_pmu.lbr_has_info) 1123 reg->config |= LBR_NO_INFO; 1124 1125 return 0; 1126 } 1127 1128 int intel_pmu_setup_lbr_filter(struct perf_event *event) 1129 { 1130 int ret = 0; 1131 1132 /* 1133 * no LBR on this PMU 1134 */ 1135 if (!x86_pmu.lbr_nr) 1136 return -EOPNOTSUPP; 1137 1138 /* 1139 * setup SW LBR filter 1140 */ 1141 ret = intel_pmu_setup_sw_lbr_filter(event); 1142 if (ret) 1143 return ret; 1144 1145 /* 1146 * setup HW LBR filter, if any 1147 */ 1148 if (x86_pmu.lbr_sel_map) 1149 ret = intel_pmu_setup_hw_lbr_filter(event); 1150 1151 return ret; 1152 } 1153 1154 /* 1155 * return the type of control flow change at address "from" 1156 * instruction is not necessarily a branch (in case of interrupt). 1157 * 1158 * The branch type returned also includes the priv level of the 1159 * target of the control flow change (X86_BR_USER, X86_BR_KERNEL). 1160 * 1161 * If a branch type is unknown OR the instruction cannot be 1162 * decoded (e.g., text page not present), then X86_BR_NONE is 1163 * returned. 1164 */ 1165 static int branch_type(unsigned long from, unsigned long to, int abort) 1166 { 1167 struct insn insn; 1168 void *addr; 1169 int bytes_read, bytes_left; 1170 int ret = X86_BR_NONE; 1171 int ext, to_plm, from_plm; 1172 u8 buf[MAX_INSN_SIZE]; 1173 int is64 = 0; 1174 1175 to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER; 1176 from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER; 1177 1178 /* 1179 * maybe zero if lbr did not fill up after a reset by the time 1180 * we get a PMU interrupt 1181 */ 1182 if (from == 0 || to == 0) 1183 return X86_BR_NONE; 1184 1185 if (abort) 1186 return X86_BR_ABORT | to_plm; 1187 1188 if (from_plm == X86_BR_USER) { 1189 /* 1190 * can happen if measuring at the user level only 1191 * and we interrupt in a kernel thread, e.g., idle. 1192 */ 1193 if (!current->mm) 1194 return X86_BR_NONE; 1195 1196 /* may fail if text not present */ 1197 bytes_left = copy_from_user_nmi(buf, (void __user *)from, 1198 MAX_INSN_SIZE); 1199 bytes_read = MAX_INSN_SIZE - bytes_left; 1200 if (!bytes_read) 1201 return X86_BR_NONE; 1202 1203 addr = buf; 1204 } else { 1205 /* 1206 * The LBR logs any address in the IP, even if the IP just 1207 * faulted. This means userspace can control the from address. 1208 * Ensure we don't blindly read any address by validating it is 1209 * a known text address. 1210 */ 1211 if (kernel_text_address(from)) { 1212 addr = (void *)from; 1213 /* 1214 * Assume we can get the maximum possible size 1215 * when grabbing kernel data. This is not 1216 * _strictly_ true since we could possibly be 1217 * executing up next to a memory hole, but 1218 * it is very unlikely to be a problem. 1219 */ 1220 bytes_read = MAX_INSN_SIZE; 1221 } else { 1222 return X86_BR_NONE; 1223 } 1224 } 1225 1226 /* 1227 * decoder needs to know the ABI especially 1228 * on 64-bit systems running 32-bit apps 1229 */ 1230 #ifdef CONFIG_X86_64 1231 is64 = kernel_ip((unsigned long)addr) || any_64bit_mode(current_pt_regs()); 1232 #endif 1233 insn_init(&insn, addr, bytes_read, is64); 1234 if (insn_get_opcode(&insn)) 1235 return X86_BR_ABORT; 1236 1237 switch (insn.opcode.bytes[0]) { 1238 case 0xf: 1239 switch (insn.opcode.bytes[1]) { 1240 case 0x05: /* syscall */ 1241 case 0x34: /* sysenter */ 1242 ret = X86_BR_SYSCALL; 1243 break; 1244 case 0x07: /* sysret */ 1245 case 0x35: /* sysexit */ 1246 ret = X86_BR_SYSRET; 1247 break; 1248 case 0x80 ... 0x8f: /* conditional */ 1249 ret = X86_BR_JCC; 1250 break; 1251 default: 1252 ret = X86_BR_NONE; 1253 } 1254 break; 1255 case 0x70 ... 0x7f: /* conditional */ 1256 ret = X86_BR_JCC; 1257 break; 1258 case 0xc2: /* near ret */ 1259 case 0xc3: /* near ret */ 1260 case 0xca: /* far ret */ 1261 case 0xcb: /* far ret */ 1262 ret = X86_BR_RET; 1263 break; 1264 case 0xcf: /* iret */ 1265 ret = X86_BR_IRET; 1266 break; 1267 case 0xcc ... 0xce: /* int */ 1268 ret = X86_BR_INT; 1269 break; 1270 case 0xe8: /* call near rel */ 1271 if (insn_get_immediate(&insn) || insn.immediate1.value == 0) { 1272 /* zero length call */ 1273 ret = X86_BR_ZERO_CALL; 1274 break; 1275 } 1276 fallthrough; 1277 case 0x9a: /* call far absolute */ 1278 ret = X86_BR_CALL; 1279 break; 1280 case 0xe0 ... 0xe3: /* loop jmp */ 1281 ret = X86_BR_JCC; 1282 break; 1283 case 0xe9 ... 0xeb: /* jmp */ 1284 ret = X86_BR_JMP; 1285 break; 1286 case 0xff: /* call near absolute, call far absolute ind */ 1287 if (insn_get_modrm(&insn)) 1288 return X86_BR_ABORT; 1289 1290 ext = (insn.modrm.bytes[0] >> 3) & 0x7; 1291 switch (ext) { 1292 case 2: /* near ind call */ 1293 case 3: /* far ind call */ 1294 ret = X86_BR_IND_CALL; 1295 break; 1296 case 4: 1297 case 5: 1298 ret = X86_BR_IND_JMP; 1299 break; 1300 } 1301 break; 1302 default: 1303 ret = X86_BR_NONE; 1304 } 1305 /* 1306 * interrupts, traps, faults (and thus ring transition) may 1307 * occur on any instructions. Thus, to classify them correctly, 1308 * we need to first look at the from and to priv levels. If they 1309 * are different and to is in the kernel, then it indicates 1310 * a ring transition. If the from instruction is not a ring 1311 * transition instr (syscall, systenter, int), then it means 1312 * it was a irq, trap or fault. 1313 * 1314 * we have no way of detecting kernel to kernel faults. 1315 */ 1316 if (from_plm == X86_BR_USER && to_plm == X86_BR_KERNEL 1317 && ret != X86_BR_SYSCALL && ret != X86_BR_INT) 1318 ret = X86_BR_IRQ; 1319 1320 /* 1321 * branch priv level determined by target as 1322 * is done by HW when LBR_SELECT is implemented 1323 */ 1324 if (ret != X86_BR_NONE) 1325 ret |= to_plm; 1326 1327 return ret; 1328 } 1329 1330 #define X86_BR_TYPE_MAP_MAX 16 1331 1332 static int branch_map[X86_BR_TYPE_MAP_MAX] = { 1333 PERF_BR_CALL, /* X86_BR_CALL */ 1334 PERF_BR_RET, /* X86_BR_RET */ 1335 PERF_BR_SYSCALL, /* X86_BR_SYSCALL */ 1336 PERF_BR_SYSRET, /* X86_BR_SYSRET */ 1337 PERF_BR_UNKNOWN, /* X86_BR_INT */ 1338 PERF_BR_ERET, /* X86_BR_IRET */ 1339 PERF_BR_COND, /* X86_BR_JCC */ 1340 PERF_BR_UNCOND, /* X86_BR_JMP */ 1341 PERF_BR_IRQ, /* X86_BR_IRQ */ 1342 PERF_BR_IND_CALL, /* X86_BR_IND_CALL */ 1343 PERF_BR_UNKNOWN, /* X86_BR_ABORT */ 1344 PERF_BR_UNKNOWN, /* X86_BR_IN_TX */ 1345 PERF_BR_UNKNOWN, /* X86_BR_NO_TX */ 1346 PERF_BR_CALL, /* X86_BR_ZERO_CALL */ 1347 PERF_BR_UNKNOWN, /* X86_BR_CALL_STACK */ 1348 PERF_BR_IND, /* X86_BR_IND_JMP */ 1349 }; 1350 1351 static int 1352 common_branch_type(int type) 1353 { 1354 int i; 1355 1356 type >>= 2; /* skip X86_BR_USER and X86_BR_KERNEL */ 1357 1358 if (type) { 1359 i = __ffs(type); 1360 if (i < X86_BR_TYPE_MAP_MAX) 1361 return branch_map[i]; 1362 } 1363 1364 return PERF_BR_UNKNOWN; 1365 } 1366 1367 enum { 1368 ARCH_LBR_BR_TYPE_JCC = 0, 1369 ARCH_LBR_BR_TYPE_NEAR_IND_JMP = 1, 1370 ARCH_LBR_BR_TYPE_NEAR_REL_JMP = 2, 1371 ARCH_LBR_BR_TYPE_NEAR_IND_CALL = 3, 1372 ARCH_LBR_BR_TYPE_NEAR_REL_CALL = 4, 1373 ARCH_LBR_BR_TYPE_NEAR_RET = 5, 1374 ARCH_LBR_BR_TYPE_KNOWN_MAX = ARCH_LBR_BR_TYPE_NEAR_RET, 1375 1376 ARCH_LBR_BR_TYPE_MAP_MAX = 16, 1377 }; 1378 1379 static const int arch_lbr_br_type_map[ARCH_LBR_BR_TYPE_MAP_MAX] = { 1380 [ARCH_LBR_BR_TYPE_JCC] = X86_BR_JCC, 1381 [ARCH_LBR_BR_TYPE_NEAR_IND_JMP] = X86_BR_IND_JMP, 1382 [ARCH_LBR_BR_TYPE_NEAR_REL_JMP] = X86_BR_JMP, 1383 [ARCH_LBR_BR_TYPE_NEAR_IND_CALL] = X86_BR_IND_CALL, 1384 [ARCH_LBR_BR_TYPE_NEAR_REL_CALL] = X86_BR_CALL, 1385 [ARCH_LBR_BR_TYPE_NEAR_RET] = X86_BR_RET, 1386 }; 1387 1388 /* 1389 * implement actual branch filter based on user demand. 1390 * Hardware may not exactly satisfy that request, thus 1391 * we need to inspect opcodes. Mismatched branches are 1392 * discarded. Therefore, the number of branches returned 1393 * in PERF_SAMPLE_BRANCH_STACK sample may vary. 1394 */ 1395 static void 1396 intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) 1397 { 1398 u64 from, to; 1399 int br_sel = cpuc->br_sel; 1400 int i, j, type, to_plm; 1401 bool compress = false; 1402 1403 /* if sampling all branches, then nothing to filter */ 1404 if (((br_sel & X86_BR_ALL) == X86_BR_ALL) && 1405 ((br_sel & X86_BR_TYPE_SAVE) != X86_BR_TYPE_SAVE)) 1406 return; 1407 1408 for (i = 0; i < cpuc->lbr_stack.nr; i++) { 1409 1410 from = cpuc->lbr_entries[i].from; 1411 to = cpuc->lbr_entries[i].to; 1412 type = cpuc->lbr_entries[i].type; 1413 1414 /* 1415 * Parse the branch type recorded in LBR_x_INFO MSR. 1416 * Doesn't support OTHER_BRANCH decoding for now. 1417 * OTHER_BRANCH branch type still rely on software decoding. 1418 */ 1419 if (static_cpu_has(X86_FEATURE_ARCH_LBR) && 1420 type <= ARCH_LBR_BR_TYPE_KNOWN_MAX) { 1421 to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER; 1422 type = arch_lbr_br_type_map[type] | to_plm; 1423 } else 1424 type = branch_type(from, to, cpuc->lbr_entries[i].abort); 1425 if (type != X86_BR_NONE && (br_sel & X86_BR_ANYTX)) { 1426 if (cpuc->lbr_entries[i].in_tx) 1427 type |= X86_BR_IN_TX; 1428 else 1429 type |= X86_BR_NO_TX; 1430 } 1431 1432 /* if type does not correspond, then discard */ 1433 if (type == X86_BR_NONE || (br_sel & type) != type) { 1434 cpuc->lbr_entries[i].from = 0; 1435 compress = true; 1436 } 1437 1438 if ((br_sel & X86_BR_TYPE_SAVE) == X86_BR_TYPE_SAVE) 1439 cpuc->lbr_entries[i].type = common_branch_type(type); 1440 } 1441 1442 if (!compress) 1443 return; 1444 1445 /* remove all entries with from=0 */ 1446 for (i = 0; i < cpuc->lbr_stack.nr; ) { 1447 if (!cpuc->lbr_entries[i].from) { 1448 j = i; 1449 while (++j < cpuc->lbr_stack.nr) 1450 cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j]; 1451 cpuc->lbr_stack.nr--; 1452 if (!cpuc->lbr_entries[i].from) 1453 continue; 1454 } 1455 i++; 1456 } 1457 } 1458 1459 void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr) 1460 { 1461 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1462 1463 /* Cannot get TOS for large PEBS and Arch LBR */ 1464 if (static_cpu_has(X86_FEATURE_ARCH_LBR) || 1465 (cpuc->n_pebs == cpuc->n_large_pebs)) 1466 cpuc->lbr_stack.hw_idx = -1ULL; 1467 else 1468 cpuc->lbr_stack.hw_idx = intel_pmu_lbr_tos(); 1469 1470 intel_pmu_store_lbr(cpuc, lbr); 1471 intel_pmu_lbr_filter(cpuc); 1472 } 1473 1474 /* 1475 * Map interface branch filters onto LBR filters 1476 */ 1477 static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { 1478 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, 1479 [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, 1480 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, 1481 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, 1482 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_REL_JMP 1483 | LBR_IND_JMP | LBR_FAR, 1484 /* 1485 * NHM/WSM erratum: must include REL_JMP+IND_JMP to get CALL branches 1486 */ 1487 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = 1488 LBR_REL_CALL | LBR_IND_CALL | LBR_REL_JMP | LBR_IND_JMP | LBR_FAR, 1489 /* 1490 * NHM/WSM erratum: must include IND_JMP to capture IND_CALL 1491 */ 1492 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL | LBR_IND_JMP, 1493 [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, 1494 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, 1495 }; 1496 1497 static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { 1498 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, 1499 [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, 1500 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, 1501 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, 1502 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR, 1503 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL 1504 | LBR_FAR, 1505 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL, 1506 [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, 1507 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, 1508 [PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_REL_CALL, 1509 }; 1510 1511 static const int hsw_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { 1512 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, 1513 [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, 1514 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, 1515 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, 1516 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR, 1517 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL 1518 | LBR_FAR, 1519 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL, 1520 [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, 1521 [PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = LBR_REL_CALL | LBR_IND_CALL 1522 | LBR_RETURN | LBR_CALL_STACK, 1523 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, 1524 [PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_REL_CALL, 1525 }; 1526 1527 static int arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { 1528 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = ARCH_LBR_ANY, 1529 [PERF_SAMPLE_BRANCH_USER_SHIFT] = ARCH_LBR_USER, 1530 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = ARCH_LBR_KERNEL, 1531 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, 1532 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = ARCH_LBR_RETURN | 1533 ARCH_LBR_OTHER_BRANCH, 1534 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = ARCH_LBR_REL_CALL | 1535 ARCH_LBR_IND_CALL | 1536 ARCH_LBR_OTHER_BRANCH, 1537 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = ARCH_LBR_IND_CALL, 1538 [PERF_SAMPLE_BRANCH_COND_SHIFT] = ARCH_LBR_JCC, 1539 [PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = ARCH_LBR_REL_CALL | 1540 ARCH_LBR_IND_CALL | 1541 ARCH_LBR_RETURN | 1542 ARCH_LBR_CALL_STACK, 1543 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = ARCH_LBR_IND_JMP, 1544 [PERF_SAMPLE_BRANCH_CALL_SHIFT] = ARCH_LBR_REL_CALL, 1545 }; 1546 1547 /* core */ 1548 void __init intel_pmu_lbr_init_core(void) 1549 { 1550 x86_pmu.lbr_nr = 4; 1551 x86_pmu.lbr_tos = MSR_LBR_TOS; 1552 x86_pmu.lbr_from = MSR_LBR_CORE_FROM; 1553 x86_pmu.lbr_to = MSR_LBR_CORE_TO; 1554 1555 /* 1556 * SW branch filter usage: 1557 * - compensate for lack of HW filter 1558 */ 1559 } 1560 1561 /* nehalem/westmere */ 1562 void __init intel_pmu_lbr_init_nhm(void) 1563 { 1564 x86_pmu.lbr_nr = 16; 1565 x86_pmu.lbr_tos = MSR_LBR_TOS; 1566 x86_pmu.lbr_from = MSR_LBR_NHM_FROM; 1567 x86_pmu.lbr_to = MSR_LBR_NHM_TO; 1568 1569 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1570 x86_pmu.lbr_sel_map = nhm_lbr_sel_map; 1571 1572 /* 1573 * SW branch filter usage: 1574 * - workaround LBR_SEL errata (see above) 1575 * - support syscall, sysret capture. 1576 * That requires LBR_FAR but that means far 1577 * jmp need to be filtered out 1578 */ 1579 } 1580 1581 /* sandy bridge */ 1582 void __init intel_pmu_lbr_init_snb(void) 1583 { 1584 x86_pmu.lbr_nr = 16; 1585 x86_pmu.lbr_tos = MSR_LBR_TOS; 1586 x86_pmu.lbr_from = MSR_LBR_NHM_FROM; 1587 x86_pmu.lbr_to = MSR_LBR_NHM_TO; 1588 1589 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1590 x86_pmu.lbr_sel_map = snb_lbr_sel_map; 1591 1592 /* 1593 * SW branch filter usage: 1594 * - support syscall, sysret capture. 1595 * That requires LBR_FAR but that means far 1596 * jmp need to be filtered out 1597 */ 1598 } 1599 1600 static inline struct kmem_cache * 1601 create_lbr_kmem_cache(size_t size, size_t align) 1602 { 1603 return kmem_cache_create("x86_lbr", size, align, 0, NULL); 1604 } 1605 1606 /* haswell */ 1607 void intel_pmu_lbr_init_hsw(void) 1608 { 1609 size_t size = sizeof(struct x86_perf_task_context); 1610 1611 x86_pmu.lbr_nr = 16; 1612 x86_pmu.lbr_tos = MSR_LBR_TOS; 1613 x86_pmu.lbr_from = MSR_LBR_NHM_FROM; 1614 x86_pmu.lbr_to = MSR_LBR_NHM_TO; 1615 1616 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1617 x86_pmu.lbr_sel_map = hsw_lbr_sel_map; 1618 1619 x86_get_pmu(smp_processor_id())->task_ctx_cache = create_lbr_kmem_cache(size, 0); 1620 } 1621 1622 /* skylake */ 1623 __init void intel_pmu_lbr_init_skl(void) 1624 { 1625 size_t size = sizeof(struct x86_perf_task_context); 1626 1627 x86_pmu.lbr_nr = 32; 1628 x86_pmu.lbr_tos = MSR_LBR_TOS; 1629 x86_pmu.lbr_from = MSR_LBR_NHM_FROM; 1630 x86_pmu.lbr_to = MSR_LBR_NHM_TO; 1631 x86_pmu.lbr_info = MSR_LBR_INFO_0; 1632 1633 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1634 x86_pmu.lbr_sel_map = hsw_lbr_sel_map; 1635 1636 x86_get_pmu(smp_processor_id())->task_ctx_cache = create_lbr_kmem_cache(size, 0); 1637 1638 /* 1639 * SW branch filter usage: 1640 * - support syscall, sysret capture. 1641 * That requires LBR_FAR but that means far 1642 * jmp need to be filtered out 1643 */ 1644 } 1645 1646 /* atom */ 1647 void __init intel_pmu_lbr_init_atom(void) 1648 { 1649 /* 1650 * only models starting at stepping 10 seems 1651 * to have an operational LBR which can freeze 1652 * on PMU interrupt 1653 */ 1654 if (boot_cpu_data.x86_model == 28 1655 && boot_cpu_data.x86_stepping < 10) { 1656 pr_cont("LBR disabled due to erratum"); 1657 return; 1658 } 1659 1660 x86_pmu.lbr_nr = 8; 1661 x86_pmu.lbr_tos = MSR_LBR_TOS; 1662 x86_pmu.lbr_from = MSR_LBR_CORE_FROM; 1663 x86_pmu.lbr_to = MSR_LBR_CORE_TO; 1664 1665 /* 1666 * SW branch filter usage: 1667 * - compensate for lack of HW filter 1668 */ 1669 } 1670 1671 /* slm */ 1672 void __init intel_pmu_lbr_init_slm(void) 1673 { 1674 x86_pmu.lbr_nr = 8; 1675 x86_pmu.lbr_tos = MSR_LBR_TOS; 1676 x86_pmu.lbr_from = MSR_LBR_CORE_FROM; 1677 x86_pmu.lbr_to = MSR_LBR_CORE_TO; 1678 1679 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1680 x86_pmu.lbr_sel_map = nhm_lbr_sel_map; 1681 1682 /* 1683 * SW branch filter usage: 1684 * - compensate for lack of HW filter 1685 */ 1686 pr_cont("8-deep LBR, "); 1687 } 1688 1689 /* Knights Landing */ 1690 void intel_pmu_lbr_init_knl(void) 1691 { 1692 x86_pmu.lbr_nr = 8; 1693 x86_pmu.lbr_tos = MSR_LBR_TOS; 1694 x86_pmu.lbr_from = MSR_LBR_NHM_FROM; 1695 x86_pmu.lbr_to = MSR_LBR_NHM_TO; 1696 1697 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1698 x86_pmu.lbr_sel_map = snb_lbr_sel_map; 1699 1700 /* Knights Landing does have MISPREDICT bit */ 1701 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_LIP) 1702 x86_pmu.intel_cap.lbr_format = LBR_FORMAT_EIP_FLAGS; 1703 } 1704 1705 void intel_pmu_lbr_init(void) 1706 { 1707 switch (x86_pmu.intel_cap.lbr_format) { 1708 case LBR_FORMAT_EIP_FLAGS2: 1709 x86_pmu.lbr_has_tsx = 1; 1710 x86_pmu.lbr_from_flags = 1; 1711 if (lbr_from_signext_quirk_needed()) 1712 static_branch_enable(&lbr_from_quirk_key); 1713 break; 1714 1715 case LBR_FORMAT_EIP_FLAGS: 1716 x86_pmu.lbr_from_flags = 1; 1717 break; 1718 1719 case LBR_FORMAT_INFO: 1720 x86_pmu.lbr_has_tsx = 1; 1721 fallthrough; 1722 case LBR_FORMAT_INFO2: 1723 x86_pmu.lbr_has_info = 1; 1724 break; 1725 1726 case LBR_FORMAT_TIME: 1727 x86_pmu.lbr_from_flags = 1; 1728 x86_pmu.lbr_to_cycles = 1; 1729 break; 1730 } 1731 1732 if (x86_pmu.lbr_has_info) { 1733 /* 1734 * Only used in combination with baseline pebs. 1735 */ 1736 static_branch_enable(&x86_lbr_mispred); 1737 static_branch_enable(&x86_lbr_cycles); 1738 } 1739 } 1740 1741 /* 1742 * LBR state size is variable based on the max number of registers. 1743 * This calculates the expected state size, which should match 1744 * what the hardware enumerates for the size of XFEATURE_LBR. 1745 */ 1746 static inline unsigned int get_lbr_state_size(void) 1747 { 1748 return sizeof(struct arch_lbr_state) + 1749 x86_pmu.lbr_nr * sizeof(struct lbr_entry); 1750 } 1751 1752 static bool is_arch_lbr_xsave_available(void) 1753 { 1754 if (!boot_cpu_has(X86_FEATURE_XSAVES)) 1755 return false; 1756 1757 /* 1758 * Check the LBR state with the corresponding software structure. 1759 * Disable LBR XSAVES support if the size doesn't match. 1760 */ 1761 if (xfeature_size(XFEATURE_LBR) == 0) 1762 return false; 1763 1764 if (WARN_ON(xfeature_size(XFEATURE_LBR) != get_lbr_state_size())) 1765 return false; 1766 1767 return true; 1768 } 1769 1770 void __init intel_pmu_arch_lbr_init(void) 1771 { 1772 struct pmu *pmu = x86_get_pmu(smp_processor_id()); 1773 union cpuid28_eax eax; 1774 union cpuid28_ebx ebx; 1775 union cpuid28_ecx ecx; 1776 unsigned int unused_edx; 1777 bool arch_lbr_xsave; 1778 size_t size; 1779 u64 lbr_nr; 1780 1781 /* Arch LBR Capabilities */ 1782 cpuid(28, &eax.full, &ebx.full, &ecx.full, &unused_edx); 1783 1784 lbr_nr = fls(eax.split.lbr_depth_mask) * 8; 1785 if (!lbr_nr) 1786 goto clear_arch_lbr; 1787 1788 /* Apply the max depth of Arch LBR */ 1789 if (wrmsrl_safe(MSR_ARCH_LBR_DEPTH, lbr_nr)) 1790 goto clear_arch_lbr; 1791 1792 x86_pmu.lbr_depth_mask = eax.split.lbr_depth_mask; 1793 x86_pmu.lbr_deep_c_reset = eax.split.lbr_deep_c_reset; 1794 x86_pmu.lbr_lip = eax.split.lbr_lip; 1795 x86_pmu.lbr_cpl = ebx.split.lbr_cpl; 1796 x86_pmu.lbr_filter = ebx.split.lbr_filter; 1797 x86_pmu.lbr_call_stack = ebx.split.lbr_call_stack; 1798 x86_pmu.lbr_mispred = ecx.split.lbr_mispred; 1799 x86_pmu.lbr_timed_lbr = ecx.split.lbr_timed_lbr; 1800 x86_pmu.lbr_br_type = ecx.split.lbr_br_type; 1801 x86_pmu.lbr_nr = lbr_nr; 1802 1803 if (x86_pmu.lbr_mispred) 1804 static_branch_enable(&x86_lbr_mispred); 1805 if (x86_pmu.lbr_timed_lbr) 1806 static_branch_enable(&x86_lbr_cycles); 1807 if (x86_pmu.lbr_br_type) 1808 static_branch_enable(&x86_lbr_type); 1809 1810 arch_lbr_xsave = is_arch_lbr_xsave_available(); 1811 if (arch_lbr_xsave) { 1812 size = sizeof(struct x86_perf_task_context_arch_lbr_xsave) + 1813 get_lbr_state_size(); 1814 pmu->task_ctx_cache = create_lbr_kmem_cache(size, 1815 XSAVE_ALIGNMENT); 1816 } 1817 1818 if (!pmu->task_ctx_cache) { 1819 arch_lbr_xsave = false; 1820 1821 size = sizeof(struct x86_perf_task_context_arch_lbr) + 1822 lbr_nr * sizeof(struct lbr_entry); 1823 pmu->task_ctx_cache = create_lbr_kmem_cache(size, 0); 1824 } 1825 1826 x86_pmu.lbr_from = MSR_ARCH_LBR_FROM_0; 1827 x86_pmu.lbr_to = MSR_ARCH_LBR_TO_0; 1828 x86_pmu.lbr_info = MSR_ARCH_LBR_INFO_0; 1829 1830 /* LBR callstack requires both CPL and Branch Filtering support */ 1831 if (!x86_pmu.lbr_cpl || 1832 !x86_pmu.lbr_filter || 1833 !x86_pmu.lbr_call_stack) 1834 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = LBR_NOT_SUPP; 1835 1836 if (!x86_pmu.lbr_cpl) { 1837 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_NOT_SUPP; 1838 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_NOT_SUPP; 1839 } else if (!x86_pmu.lbr_filter) { 1840 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_NOT_SUPP; 1841 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_NOT_SUPP; 1842 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_NOT_SUPP; 1843 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_NOT_SUPP; 1844 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_NOT_SUPP; 1845 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_NOT_SUPP; 1846 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_NOT_SUPP; 1847 } 1848 1849 x86_pmu.lbr_ctl_mask = ARCH_LBR_CTL_MASK; 1850 x86_pmu.lbr_ctl_map = arch_lbr_ctl_map; 1851 1852 if (!x86_pmu.lbr_cpl && !x86_pmu.lbr_filter) 1853 x86_pmu.lbr_ctl_map = NULL; 1854 1855 x86_pmu.lbr_reset = intel_pmu_arch_lbr_reset; 1856 if (arch_lbr_xsave) { 1857 x86_pmu.lbr_save = intel_pmu_arch_lbr_xsaves; 1858 x86_pmu.lbr_restore = intel_pmu_arch_lbr_xrstors; 1859 x86_pmu.lbr_read = intel_pmu_arch_lbr_read_xsave; 1860 pr_cont("XSAVE "); 1861 } else { 1862 x86_pmu.lbr_save = intel_pmu_arch_lbr_save; 1863 x86_pmu.lbr_restore = intel_pmu_arch_lbr_restore; 1864 x86_pmu.lbr_read = intel_pmu_arch_lbr_read; 1865 } 1866 1867 pr_cont("Architectural LBR, "); 1868 1869 return; 1870 1871 clear_arch_lbr: 1872 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_ARCH_LBR); 1873 } 1874 1875 /** 1876 * x86_perf_get_lbr - get the LBR records information 1877 * 1878 * @lbr: the caller's memory to store the LBR records information 1879 * 1880 * Returns: 0 indicates the LBR info has been successfully obtained 1881 */ 1882 int x86_perf_get_lbr(struct x86_pmu_lbr *lbr) 1883 { 1884 int lbr_fmt = x86_pmu.intel_cap.lbr_format; 1885 1886 lbr->nr = x86_pmu.lbr_nr; 1887 lbr->from = x86_pmu.lbr_from; 1888 lbr->to = x86_pmu.lbr_to; 1889 lbr->info = (lbr_fmt == LBR_FORMAT_INFO) ? x86_pmu.lbr_info : 0; 1890 1891 return 0; 1892 } 1893 EXPORT_SYMBOL_GPL(x86_perf_get_lbr); 1894 1895 struct event_constraint vlbr_constraint = 1896 __EVENT_CONSTRAINT(INTEL_FIXED_VLBR_EVENT, (1ULL << INTEL_PMC_IDX_FIXED_VLBR), 1897 FIXED_EVENT_FLAGS, 1, 0, PERF_X86_EVENT_LBR_SELECT); 1898