1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * KVM PMU support for Intel CPUs 4 * 5 * Copyright 2011 Red Hat, Inc. and/or its affiliates. 6 * 7 * Authors: 8 * Avi Kivity <avi@redhat.com> 9 * Gleb Natapov <gleb@redhat.com> 10 */ 11 #include <linux/types.h> 12 #include <linux/kvm_host.h> 13 #include <linux/perf_event.h> 14 #include <asm/perf_event.h> 15 #include "x86.h" 16 #include "cpuid.h" 17 #include "lapic.h" 18 #include "nested.h" 19 #include "pmu.h" 20 21 #define MSR_PMC_FULL_WIDTH_BIT (MSR_IA32_PMC0 - MSR_IA32_PERFCTR0) 22 23 static struct kvm_event_hw_type_mapping intel_arch_events[] = { 24 /* Index must match CPUID 0x0A.EBX bit vector */ 25 [0] = { 0x3c, 0x00, PERF_COUNT_HW_CPU_CYCLES }, 26 [1] = { 0xc0, 0x00, PERF_COUNT_HW_INSTRUCTIONS }, 27 [2] = { 0x3c, 0x01, PERF_COUNT_HW_BUS_CYCLES }, 28 [3] = { 0x2e, 0x4f, PERF_COUNT_HW_CACHE_REFERENCES }, 29 [4] = { 0x2e, 0x41, PERF_COUNT_HW_CACHE_MISSES }, 30 [5] = { 0xc4, 0x00, PERF_COUNT_HW_BRANCH_INSTRUCTIONS }, 31 [6] = { 0xc5, 0x00, PERF_COUNT_HW_BRANCH_MISSES }, 32 [7] = { 0x00, 0x03, PERF_COUNT_HW_REF_CPU_CYCLES }, 33 }; 34 35 /* mapping between fixed pmc index and intel_arch_events array */ 36 static int fixed_pmc_events[] = {1, 0, 7}; 37 38 static void reprogram_fixed_counters(struct kvm_pmu *pmu, u64 data) 39 { 40 int i; 41 42 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { 43 u8 new_ctrl = fixed_ctrl_field(data, i); 44 u8 old_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl, i); 45 struct kvm_pmc *pmc; 46 47 pmc = get_fixed_pmc(pmu, MSR_CORE_PERF_FIXED_CTR0 + i); 48 49 if (old_ctrl == new_ctrl) 50 continue; 51 52 __set_bit(INTEL_PMC_IDX_FIXED + i, pmu->pmc_in_use); 53 reprogram_fixed_counter(pmc, new_ctrl, i); 54 } 55 56 pmu->fixed_ctr_ctrl = data; 57 } 58 59 /* function is called when global control register has been updated. */ 60 static void global_ctrl_changed(struct kvm_pmu *pmu, u64 data) 61 { 62 int bit; 63 u64 diff = pmu->global_ctrl ^ data; 64 65 pmu->global_ctrl = data; 66 67 for_each_set_bit(bit, (unsigned long *)&diff, X86_PMC_IDX_MAX) 68 reprogram_counter(pmu, bit); 69 } 70 71 static unsigned intel_find_arch_event(struct kvm_pmu *pmu, 72 u8 event_select, 73 u8 unit_mask) 74 { 75 int i; 76 77 for (i = 0; i < ARRAY_SIZE(intel_arch_events); i++) 78 if (intel_arch_events[i].eventsel == event_select 79 && intel_arch_events[i].unit_mask == unit_mask 80 && (pmu->available_event_types & (1 << i))) 81 break; 82 83 if (i == ARRAY_SIZE(intel_arch_events)) 84 return PERF_COUNT_HW_MAX; 85 86 return intel_arch_events[i].event_type; 87 } 88 89 static unsigned intel_find_fixed_event(int idx) 90 { 91 u32 event; 92 size_t size = ARRAY_SIZE(fixed_pmc_events); 93 94 if (idx >= size) 95 return PERF_COUNT_HW_MAX; 96 97 event = fixed_pmc_events[array_index_nospec(idx, size)]; 98 return intel_arch_events[event].event_type; 99 } 100 101 /* check if a PMC is enabled by comparing it with globl_ctrl bits. */ 102 static bool intel_pmc_is_enabled(struct kvm_pmc *pmc) 103 { 104 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 105 106 return test_bit(pmc->idx, (unsigned long *)&pmu->global_ctrl); 107 } 108 109 static struct kvm_pmc *intel_pmc_idx_to_pmc(struct kvm_pmu *pmu, int pmc_idx) 110 { 111 if (pmc_idx < INTEL_PMC_IDX_FIXED) 112 return get_gp_pmc(pmu, MSR_P6_EVNTSEL0 + pmc_idx, 113 MSR_P6_EVNTSEL0); 114 else { 115 u32 idx = pmc_idx - INTEL_PMC_IDX_FIXED; 116 117 return get_fixed_pmc(pmu, idx + MSR_CORE_PERF_FIXED_CTR0); 118 } 119 } 120 121 /* returns 0 if idx's corresponding MSR exists; otherwise returns 1. */ 122 static int intel_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) 123 { 124 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 125 bool fixed = idx & (1u << 30); 126 127 idx &= ~(3u << 30); 128 129 return (!fixed && idx >= pmu->nr_arch_gp_counters) || 130 (fixed && idx >= pmu->nr_arch_fixed_counters); 131 } 132 133 static struct kvm_pmc *intel_rdpmc_ecx_to_pmc(struct kvm_vcpu *vcpu, 134 unsigned int idx, u64 *mask) 135 { 136 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 137 bool fixed = idx & (1u << 30); 138 struct kvm_pmc *counters; 139 unsigned int num_counters; 140 141 idx &= ~(3u << 30); 142 if (fixed) { 143 counters = pmu->fixed_counters; 144 num_counters = pmu->nr_arch_fixed_counters; 145 } else { 146 counters = pmu->gp_counters; 147 num_counters = pmu->nr_arch_gp_counters; 148 } 149 if (idx >= num_counters) 150 return NULL; 151 *mask &= pmu->counter_bitmask[fixed ? KVM_PMC_FIXED : KVM_PMC_GP]; 152 return &counters[array_index_nospec(idx, num_counters)]; 153 } 154 155 static inline u64 vcpu_get_perf_capabilities(struct kvm_vcpu *vcpu) 156 { 157 if (!guest_cpuid_has(vcpu, X86_FEATURE_PDCM)) 158 return 0; 159 160 return vcpu->arch.perf_capabilities; 161 } 162 163 static inline bool fw_writes_is_enabled(struct kvm_vcpu *vcpu) 164 { 165 return (vcpu_get_perf_capabilities(vcpu) & PMU_CAP_FW_WRITES) != 0; 166 } 167 168 static inline struct kvm_pmc *get_fw_gp_pmc(struct kvm_pmu *pmu, u32 msr) 169 { 170 if (!fw_writes_is_enabled(pmu_to_vcpu(pmu))) 171 return NULL; 172 173 return get_gp_pmc(pmu, msr, MSR_IA32_PMC0); 174 } 175 176 bool intel_pmu_lbr_is_compatible(struct kvm_vcpu *vcpu) 177 { 178 /* 179 * As a first step, a guest could only enable LBR feature if its 180 * cpu model is the same as the host because the LBR registers 181 * would be pass-through to the guest and they're model specific. 182 */ 183 return boot_cpu_data.x86_model == guest_cpuid_model(vcpu); 184 } 185 186 bool intel_pmu_lbr_is_enabled(struct kvm_vcpu *vcpu) 187 { 188 struct x86_pmu_lbr *lbr = vcpu_to_lbr_records(vcpu); 189 190 return lbr->nr && (vcpu_get_perf_capabilities(vcpu) & PMU_CAP_LBR_FMT); 191 } 192 193 static bool intel_pmu_is_valid_lbr_msr(struct kvm_vcpu *vcpu, u32 index) 194 { 195 struct x86_pmu_lbr *records = vcpu_to_lbr_records(vcpu); 196 bool ret = false; 197 198 if (!intel_pmu_lbr_is_enabled(vcpu)) 199 return ret; 200 201 ret = (index == MSR_LBR_SELECT) || (index == MSR_LBR_TOS) || 202 (index >= records->from && index < records->from + records->nr) || 203 (index >= records->to && index < records->to + records->nr); 204 205 if (!ret && records->info) 206 ret = (index >= records->info && index < records->info + records->nr); 207 208 return ret; 209 } 210 211 static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) 212 { 213 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 214 int ret; 215 216 switch (msr) { 217 case MSR_CORE_PERF_FIXED_CTR_CTRL: 218 case MSR_CORE_PERF_GLOBAL_STATUS: 219 case MSR_CORE_PERF_GLOBAL_CTRL: 220 case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 221 ret = pmu->version > 1; 222 break; 223 default: 224 ret = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0) || 225 get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0) || 226 get_fixed_pmc(pmu, msr) || get_fw_gp_pmc(pmu, msr) || 227 intel_pmu_is_valid_lbr_msr(vcpu, msr); 228 break; 229 } 230 231 return ret; 232 } 233 234 static struct kvm_pmc *intel_msr_idx_to_pmc(struct kvm_vcpu *vcpu, u32 msr) 235 { 236 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 237 struct kvm_pmc *pmc; 238 239 pmc = get_fixed_pmc(pmu, msr); 240 pmc = pmc ? pmc : get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0); 241 pmc = pmc ? pmc : get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0); 242 243 return pmc; 244 } 245 246 static inline void intel_pmu_release_guest_lbr_event(struct kvm_vcpu *vcpu) 247 { 248 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 249 250 if (lbr_desc->event) { 251 perf_event_release_kernel(lbr_desc->event); 252 lbr_desc->event = NULL; 253 vcpu_to_pmu(vcpu)->event_count--; 254 } 255 } 256 257 int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu) 258 { 259 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 260 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 261 struct perf_event *event; 262 263 /* 264 * The perf_event_attr is constructed in the minimum efficient way: 265 * - set 'pinned = true' to make it task pinned so that if another 266 * cpu pinned event reclaims LBR, the event->oncpu will be set to -1; 267 * - set '.exclude_host = true' to record guest branches behavior; 268 * 269 * - set '.config = INTEL_FIXED_VLBR_EVENT' to indicates host perf 270 * schedule the event without a real HW counter but a fake one; 271 * check is_guest_lbr_event() and __intel_get_event_constraints(); 272 * 273 * - set 'sample_type = PERF_SAMPLE_BRANCH_STACK' and 274 * 'branch_sample_type = PERF_SAMPLE_BRANCH_CALL_STACK | 275 * PERF_SAMPLE_BRANCH_USER' to configure it as a LBR callstack 276 * event, which helps KVM to save/restore guest LBR records 277 * during host context switches and reduces quite a lot overhead, 278 * check branch_user_callstack() and intel_pmu_lbr_sched_task(); 279 */ 280 struct perf_event_attr attr = { 281 .type = PERF_TYPE_RAW, 282 .size = sizeof(attr), 283 .config = INTEL_FIXED_VLBR_EVENT, 284 .sample_type = PERF_SAMPLE_BRANCH_STACK, 285 .pinned = true, 286 .exclude_host = true, 287 .branch_sample_type = PERF_SAMPLE_BRANCH_CALL_STACK | 288 PERF_SAMPLE_BRANCH_USER, 289 }; 290 291 if (unlikely(lbr_desc->event)) { 292 __set_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use); 293 return 0; 294 } 295 296 event = perf_event_create_kernel_counter(&attr, -1, 297 current, NULL, NULL); 298 if (IS_ERR(event)) { 299 pr_debug_ratelimited("%s: failed %ld\n", 300 __func__, PTR_ERR(event)); 301 return PTR_ERR(event); 302 } 303 lbr_desc->event = event; 304 pmu->event_count++; 305 __set_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use); 306 return 0; 307 } 308 309 /* 310 * It's safe to access LBR msrs from guest when they have not 311 * been passthrough since the host would help restore or reset 312 * the LBR msrs records when the guest LBR event is scheduled in. 313 */ 314 static bool intel_pmu_handle_lbr_msrs_access(struct kvm_vcpu *vcpu, 315 struct msr_data *msr_info, bool read) 316 { 317 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 318 u32 index = msr_info->index; 319 320 if (!intel_pmu_is_valid_lbr_msr(vcpu, index)) 321 return false; 322 323 if (!lbr_desc->event && intel_pmu_create_guest_lbr_event(vcpu) < 0) 324 goto dummy; 325 326 /* 327 * Disable irq to ensure the LBR feature doesn't get reclaimed by the 328 * host at the time the value is read from the msr, and this avoids the 329 * host LBR value to be leaked to the guest. If LBR has been reclaimed, 330 * return 0 on guest reads. 331 */ 332 local_irq_disable(); 333 if (lbr_desc->event->state == PERF_EVENT_STATE_ACTIVE) { 334 if (read) 335 rdmsrl(index, msr_info->data); 336 else 337 wrmsrl(index, msr_info->data); 338 __set_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use); 339 local_irq_enable(); 340 return true; 341 } 342 clear_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use); 343 local_irq_enable(); 344 345 dummy: 346 if (read) 347 msr_info->data = 0; 348 return true; 349 } 350 351 static int intel_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 352 { 353 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 354 struct kvm_pmc *pmc; 355 u32 msr = msr_info->index; 356 357 switch (msr) { 358 case MSR_CORE_PERF_FIXED_CTR_CTRL: 359 msr_info->data = pmu->fixed_ctr_ctrl; 360 return 0; 361 case MSR_CORE_PERF_GLOBAL_STATUS: 362 msr_info->data = pmu->global_status; 363 return 0; 364 case MSR_CORE_PERF_GLOBAL_CTRL: 365 msr_info->data = pmu->global_ctrl; 366 return 0; 367 case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 368 msr_info->data = 0; 369 return 0; 370 default: 371 if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)) || 372 (pmc = get_gp_pmc(pmu, msr, MSR_IA32_PMC0))) { 373 u64 val = pmc_read_counter(pmc); 374 msr_info->data = 375 val & pmu->counter_bitmask[KVM_PMC_GP]; 376 return 0; 377 } else if ((pmc = get_fixed_pmc(pmu, msr))) { 378 u64 val = pmc_read_counter(pmc); 379 msr_info->data = 380 val & pmu->counter_bitmask[KVM_PMC_FIXED]; 381 return 0; 382 } else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) { 383 msr_info->data = pmc->eventsel; 384 return 0; 385 } else if (intel_pmu_handle_lbr_msrs_access(vcpu, msr_info, true)) 386 return 0; 387 } 388 389 return 1; 390 } 391 392 static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 393 { 394 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 395 struct kvm_pmc *pmc; 396 u32 msr = msr_info->index; 397 u64 data = msr_info->data; 398 399 switch (msr) { 400 case MSR_CORE_PERF_FIXED_CTR_CTRL: 401 if (pmu->fixed_ctr_ctrl == data) 402 return 0; 403 if (!(data & 0xfffffffffffff444ull)) { 404 reprogram_fixed_counters(pmu, data); 405 return 0; 406 } 407 break; 408 case MSR_CORE_PERF_GLOBAL_STATUS: 409 if (msr_info->host_initiated) { 410 pmu->global_status = data; 411 return 0; 412 } 413 break; /* RO MSR */ 414 case MSR_CORE_PERF_GLOBAL_CTRL: 415 if (pmu->global_ctrl == data) 416 return 0; 417 if (kvm_valid_perf_global_ctrl(pmu, data)) { 418 global_ctrl_changed(pmu, data); 419 return 0; 420 } 421 break; 422 case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 423 if (!(data & pmu->global_ovf_ctrl_mask)) { 424 if (!msr_info->host_initiated) 425 pmu->global_status &= ~data; 426 return 0; 427 } 428 break; 429 default: 430 if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)) || 431 (pmc = get_gp_pmc(pmu, msr, MSR_IA32_PMC0))) { 432 if ((msr & MSR_PMC_FULL_WIDTH_BIT) && 433 (data & ~pmu->counter_bitmask[KVM_PMC_GP])) 434 return 1; 435 if (!msr_info->host_initiated && 436 !(msr & MSR_PMC_FULL_WIDTH_BIT)) 437 data = (s64)(s32)data; 438 pmc->counter += data - pmc_read_counter(pmc); 439 if (pmc->perf_event && !pmc->is_paused) 440 perf_event_period(pmc->perf_event, 441 get_sample_period(pmc, data)); 442 return 0; 443 } else if ((pmc = get_fixed_pmc(pmu, msr))) { 444 pmc->counter += data - pmc_read_counter(pmc); 445 if (pmc->perf_event && !pmc->is_paused) 446 perf_event_period(pmc->perf_event, 447 get_sample_period(pmc, data)); 448 return 0; 449 } else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) { 450 if (data == pmc->eventsel) 451 return 0; 452 if (!(data & pmu->reserved_bits)) { 453 reprogram_gp_counter(pmc, data); 454 return 0; 455 } 456 } else if (intel_pmu_handle_lbr_msrs_access(vcpu, msr_info, false)) 457 return 0; 458 } 459 460 return 1; 461 } 462 463 static void intel_pmu_refresh(struct kvm_vcpu *vcpu) 464 { 465 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 466 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 467 468 struct x86_pmu_capability x86_pmu; 469 struct kvm_cpuid_entry2 *entry; 470 union cpuid10_eax eax; 471 union cpuid10_edx edx; 472 473 pmu->nr_arch_gp_counters = 0; 474 pmu->nr_arch_fixed_counters = 0; 475 pmu->counter_bitmask[KVM_PMC_GP] = 0; 476 pmu->counter_bitmask[KVM_PMC_FIXED] = 0; 477 pmu->version = 0; 478 pmu->reserved_bits = 0xffffffff00200000ull; 479 480 entry = kvm_find_cpuid_entry(vcpu, 0xa, 0); 481 if (!entry) 482 return; 483 eax.full = entry->eax; 484 edx.full = entry->edx; 485 486 pmu->version = eax.split.version_id; 487 if (!pmu->version) 488 return; 489 490 perf_get_x86_pmu_capability(&x86_pmu); 491 492 pmu->nr_arch_gp_counters = min_t(int, eax.split.num_counters, 493 x86_pmu.num_counters_gp); 494 eax.split.bit_width = min_t(int, eax.split.bit_width, x86_pmu.bit_width_gp); 495 pmu->counter_bitmask[KVM_PMC_GP] = ((u64)1 << eax.split.bit_width) - 1; 496 eax.split.mask_length = min_t(int, eax.split.mask_length, x86_pmu.events_mask_len); 497 pmu->available_event_types = ~entry->ebx & 498 ((1ull << eax.split.mask_length) - 1); 499 500 if (pmu->version == 1) { 501 pmu->nr_arch_fixed_counters = 0; 502 } else { 503 pmu->nr_arch_fixed_counters = 504 min_t(int, edx.split.num_counters_fixed, 505 x86_pmu.num_counters_fixed); 506 edx.split.bit_width_fixed = min_t(int, 507 edx.split.bit_width_fixed, x86_pmu.bit_width_fixed); 508 pmu->counter_bitmask[KVM_PMC_FIXED] = 509 ((u64)1 << edx.split.bit_width_fixed) - 1; 510 } 511 512 pmu->global_ctrl = ((1ull << pmu->nr_arch_gp_counters) - 1) | 513 (((1ull << pmu->nr_arch_fixed_counters) - 1) << INTEL_PMC_IDX_FIXED); 514 pmu->global_ctrl_mask = ~pmu->global_ctrl; 515 pmu->global_ovf_ctrl_mask = pmu->global_ctrl_mask 516 & ~(MSR_CORE_PERF_GLOBAL_OVF_CTRL_OVF_BUF | 517 MSR_CORE_PERF_GLOBAL_OVF_CTRL_COND_CHGD); 518 if (vmx_pt_mode_is_host_guest()) 519 pmu->global_ovf_ctrl_mask &= 520 ~MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI; 521 522 entry = kvm_find_cpuid_entry(vcpu, 7, 0); 523 if (entry && 524 (boot_cpu_has(X86_FEATURE_HLE) || boot_cpu_has(X86_FEATURE_RTM)) && 525 (entry->ebx & (X86_FEATURE_HLE|X86_FEATURE_RTM))) 526 pmu->reserved_bits ^= HSW_IN_TX|HSW_IN_TX_CHECKPOINTED; 527 528 bitmap_set(pmu->all_valid_pmc_idx, 529 0, pmu->nr_arch_gp_counters); 530 bitmap_set(pmu->all_valid_pmc_idx, 531 INTEL_PMC_MAX_GENERIC, pmu->nr_arch_fixed_counters); 532 533 nested_vmx_pmu_entry_exit_ctls_update(vcpu); 534 535 if (intel_pmu_lbr_is_compatible(vcpu)) 536 x86_perf_get_lbr(&lbr_desc->records); 537 else 538 lbr_desc->records.nr = 0; 539 540 if (lbr_desc->records.nr) 541 bitmap_set(pmu->all_valid_pmc_idx, INTEL_PMC_IDX_FIXED_VLBR, 1); 542 } 543 544 static void intel_pmu_init(struct kvm_vcpu *vcpu) 545 { 546 int i; 547 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 548 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 549 550 for (i = 0; i < INTEL_PMC_MAX_GENERIC; i++) { 551 pmu->gp_counters[i].type = KVM_PMC_GP; 552 pmu->gp_counters[i].vcpu = vcpu; 553 pmu->gp_counters[i].idx = i; 554 pmu->gp_counters[i].current_config = 0; 555 } 556 557 for (i = 0; i < INTEL_PMC_MAX_FIXED; i++) { 558 pmu->fixed_counters[i].type = KVM_PMC_FIXED; 559 pmu->fixed_counters[i].vcpu = vcpu; 560 pmu->fixed_counters[i].idx = i + INTEL_PMC_IDX_FIXED; 561 pmu->fixed_counters[i].current_config = 0; 562 } 563 564 vcpu->arch.perf_capabilities = vmx_get_perf_capabilities(); 565 lbr_desc->records.nr = 0; 566 lbr_desc->event = NULL; 567 lbr_desc->msr_passthrough = false; 568 } 569 570 static void intel_pmu_reset(struct kvm_vcpu *vcpu) 571 { 572 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 573 struct kvm_pmc *pmc = NULL; 574 int i; 575 576 for (i = 0; i < INTEL_PMC_MAX_GENERIC; i++) { 577 pmc = &pmu->gp_counters[i]; 578 579 pmc_stop_counter(pmc); 580 pmc->counter = pmc->eventsel = 0; 581 } 582 583 for (i = 0; i < INTEL_PMC_MAX_FIXED; i++) { 584 pmc = &pmu->fixed_counters[i]; 585 586 pmc_stop_counter(pmc); 587 pmc->counter = 0; 588 } 589 590 pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status = 0; 591 592 intel_pmu_release_guest_lbr_event(vcpu); 593 } 594 595 /* 596 * Emulate LBR_On_PMI behavior for 1 < pmu.version < 4. 597 * 598 * If Freeze_LBR_On_PMI = 1, the LBR is frozen on PMI and 599 * the KVM emulates to clear the LBR bit (bit 0) in IA32_DEBUGCTL. 600 * 601 * Guest needs to re-enable LBR to resume branches recording. 602 */ 603 static void intel_pmu_legacy_freezing_lbrs_on_pmi(struct kvm_vcpu *vcpu) 604 { 605 u64 data = vmcs_read64(GUEST_IA32_DEBUGCTL); 606 607 if (data & DEBUGCTLMSR_FREEZE_LBRS_ON_PMI) { 608 data &= ~DEBUGCTLMSR_LBR; 609 vmcs_write64(GUEST_IA32_DEBUGCTL, data); 610 } 611 } 612 613 static void intel_pmu_deliver_pmi(struct kvm_vcpu *vcpu) 614 { 615 u8 version = vcpu_to_pmu(vcpu)->version; 616 617 if (!intel_pmu_lbr_is_enabled(vcpu)) 618 return; 619 620 if (version > 1 && version < 4) 621 intel_pmu_legacy_freezing_lbrs_on_pmi(vcpu); 622 } 623 624 static void vmx_update_intercept_for_lbr_msrs(struct kvm_vcpu *vcpu, bool set) 625 { 626 struct x86_pmu_lbr *lbr = vcpu_to_lbr_records(vcpu); 627 int i; 628 629 for (i = 0; i < lbr->nr; i++) { 630 vmx_set_intercept_for_msr(vcpu, lbr->from + i, MSR_TYPE_RW, set); 631 vmx_set_intercept_for_msr(vcpu, lbr->to + i, MSR_TYPE_RW, set); 632 if (lbr->info) 633 vmx_set_intercept_for_msr(vcpu, lbr->info + i, MSR_TYPE_RW, set); 634 } 635 636 vmx_set_intercept_for_msr(vcpu, MSR_LBR_SELECT, MSR_TYPE_RW, set); 637 vmx_set_intercept_for_msr(vcpu, MSR_LBR_TOS, MSR_TYPE_RW, set); 638 } 639 640 static inline void vmx_disable_lbr_msrs_passthrough(struct kvm_vcpu *vcpu) 641 { 642 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 643 644 if (!lbr_desc->msr_passthrough) 645 return; 646 647 vmx_update_intercept_for_lbr_msrs(vcpu, true); 648 lbr_desc->msr_passthrough = false; 649 } 650 651 static inline void vmx_enable_lbr_msrs_passthrough(struct kvm_vcpu *vcpu) 652 { 653 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 654 655 if (lbr_desc->msr_passthrough) 656 return; 657 658 vmx_update_intercept_for_lbr_msrs(vcpu, false); 659 lbr_desc->msr_passthrough = true; 660 } 661 662 /* 663 * Higher priority host perf events (e.g. cpu pinned) could reclaim the 664 * pmu resources (e.g. LBR) that were assigned to the guest. This is 665 * usually done via ipi calls (more details in perf_install_in_context). 666 * 667 * Before entering the non-root mode (with irq disabled here), double 668 * confirm that the pmu features enabled to the guest are not reclaimed 669 * by higher priority host events. Otherwise, disallow vcpu's access to 670 * the reclaimed features. 671 */ 672 void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu) 673 { 674 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 675 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); 676 677 if (!lbr_desc->event) { 678 vmx_disable_lbr_msrs_passthrough(vcpu); 679 if (vmcs_read64(GUEST_IA32_DEBUGCTL) & DEBUGCTLMSR_LBR) 680 goto warn; 681 if (test_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use)) 682 goto warn; 683 return; 684 } 685 686 if (lbr_desc->event->state < PERF_EVENT_STATE_ACTIVE) { 687 vmx_disable_lbr_msrs_passthrough(vcpu); 688 __clear_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use); 689 goto warn; 690 } else 691 vmx_enable_lbr_msrs_passthrough(vcpu); 692 693 return; 694 695 warn: 696 pr_warn_ratelimited("kvm: vcpu-%d: fail to passthrough LBR.\n", 697 vcpu->vcpu_id); 698 } 699 700 static void intel_pmu_cleanup(struct kvm_vcpu *vcpu) 701 { 702 if (!(vmcs_read64(GUEST_IA32_DEBUGCTL) & DEBUGCTLMSR_LBR)) 703 intel_pmu_release_guest_lbr_event(vcpu); 704 } 705 706 struct kvm_pmu_ops intel_pmu_ops = { 707 .find_arch_event = intel_find_arch_event, 708 .find_fixed_event = intel_find_fixed_event, 709 .pmc_is_enabled = intel_pmc_is_enabled, 710 .pmc_idx_to_pmc = intel_pmc_idx_to_pmc, 711 .rdpmc_ecx_to_pmc = intel_rdpmc_ecx_to_pmc, 712 .msr_idx_to_pmc = intel_msr_idx_to_pmc, 713 .is_valid_rdpmc_ecx = intel_is_valid_rdpmc_ecx, 714 .is_valid_msr = intel_is_valid_msr, 715 .get_msr = intel_pmu_get_msr, 716 .set_msr = intel_pmu_set_msr, 717 .refresh = intel_pmu_refresh, 718 .init = intel_pmu_init, 719 .reset = intel_pmu_reset, 720 .deliver_pmi = intel_pmu_deliver_pmi, 721 .cleanup = intel_pmu_cleanup, 722 }; 723