1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine -- Performance Monitoring Unit support 4 * 5 * Copyright 2015 Red Hat, Inc. and/or its affiliates. 6 * 7 * Authors: 8 * Avi Kivity <avi@redhat.com> 9 * Gleb Natapov <gleb@redhat.com> 10 * Wei Huang <wei@redhat.com> 11 */ 12 13 #include <linux/types.h> 14 #include <linux/kvm_host.h> 15 #include <linux/perf_event.h> 16 #include <linux/bsearch.h> 17 #include <linux/sort.h> 18 #include <asm/perf_event.h> 19 #include "x86.h" 20 #include "cpuid.h" 21 #include "lapic.h" 22 #include "pmu.h" 23 24 /* This is enough to filter the vast majority of currently defined events. */ 25 #define KVM_PMU_EVENT_FILTER_MAX_EVENTS 300 26 27 /* NOTE: 28 * - Each perf counter is defined as "struct kvm_pmc"; 29 * - There are two types of perf counters: general purpose (gp) and fixed. 30 * gp counters are stored in gp_counters[] and fixed counters are stored 31 * in fixed_counters[] respectively. Both of them are part of "struct 32 * kvm_pmu"; 33 * - pmu.c understands the difference between gp counters and fixed counters. 34 * However AMD doesn't support fixed-counters; 35 * - There are three types of index to access perf counters (PMC): 36 * 1. MSR (named msr): For example Intel has MSR_IA32_PERFCTRn and AMD 37 * has MSR_K7_PERFCTRn. 38 * 2. MSR Index (named idx): This normally is used by RDPMC instruction. 39 * For instance AMD RDPMC instruction uses 0000_0003h in ECX to access 40 * C001_0007h (MSR_K7_PERCTR3). Intel has a similar mechanism, except 41 * that it also supports fixed counters. idx can be used to as index to 42 * gp and fixed counters. 43 * 3. Global PMC Index (named pmc): pmc is an index specific to PMU 44 * code. Each pmc, stored in kvm_pmc.idx field, is unique across 45 * all perf counters (both gp and fixed). The mapping relationship 46 * between pmc and perf counters is as the following: 47 * * Intel: [0 .. INTEL_PMC_MAX_GENERIC-1] <=> gp counters 48 * [INTEL_PMC_IDX_FIXED .. INTEL_PMC_IDX_FIXED + 2] <=> fixed 49 * * AMD: [0 .. AMD64_NUM_COUNTERS-1] <=> gp counters 50 */ 51 52 static struct kvm_pmu_ops kvm_pmu_ops __read_mostly; 53 54 #define KVM_X86_PMU_OP(func) \ 55 DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \ 56 *(((struct kvm_pmu_ops *)0)->func)); 57 #define KVM_X86_PMU_OP_OPTIONAL KVM_X86_PMU_OP 58 #include <asm/kvm-x86-pmu-ops.h> 59 60 void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops) 61 { 62 memcpy(&kvm_pmu_ops, pmu_ops, sizeof(kvm_pmu_ops)); 63 64 #define __KVM_X86_PMU_OP(func) \ 65 static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func); 66 #define KVM_X86_PMU_OP(func) \ 67 WARN_ON(!kvm_pmu_ops.func); __KVM_X86_PMU_OP(func) 68 #define KVM_X86_PMU_OP_OPTIONAL __KVM_X86_PMU_OP 69 #include <asm/kvm-x86-pmu-ops.h> 70 #undef __KVM_X86_PMU_OP 71 } 72 73 static inline bool pmc_is_enabled(struct kvm_pmc *pmc) 74 { 75 return static_call(kvm_x86_pmu_pmc_is_enabled)(pmc); 76 } 77 78 static void kvm_pmi_trigger_fn(struct irq_work *irq_work) 79 { 80 struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work); 81 struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu); 82 83 kvm_pmu_deliver_pmi(vcpu); 84 } 85 86 static inline void __kvm_perf_overflow(struct kvm_pmc *pmc, bool in_pmi) 87 { 88 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 89 90 /* Ignore counters that have been reprogrammed already. */ 91 if (test_and_set_bit(pmc->idx, pmu->reprogram_pmi)) 92 return; 93 94 __set_bit(pmc->idx, (unsigned long *)&pmu->global_status); 95 kvm_make_request(KVM_REQ_PMU, pmc->vcpu); 96 97 if (!pmc->intr) 98 return; 99 100 /* 101 * Inject PMI. If vcpu was in a guest mode during NMI PMI 102 * can be ejected on a guest mode re-entry. Otherwise we can't 103 * be sure that vcpu wasn't executing hlt instruction at the 104 * time of vmexit and is not going to re-enter guest mode until 105 * woken up. So we should wake it, but this is impossible from 106 * NMI context. Do it from irq work instead. 107 */ 108 if (in_pmi && !kvm_handling_nmi_from_guest(pmc->vcpu)) 109 irq_work_queue(&pmc_to_pmu(pmc)->irq_work); 110 else 111 kvm_make_request(KVM_REQ_PMI, pmc->vcpu); 112 } 113 114 static void kvm_perf_overflow(struct perf_event *perf_event, 115 struct perf_sample_data *data, 116 struct pt_regs *regs) 117 { 118 struct kvm_pmc *pmc = perf_event->overflow_handler_context; 119 120 __kvm_perf_overflow(pmc, true); 121 } 122 123 static void pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, 124 u64 config, bool exclude_user, 125 bool exclude_kernel, bool intr) 126 { 127 struct perf_event *event; 128 struct perf_event_attr attr = { 129 .type = type, 130 .size = sizeof(attr), 131 .pinned = true, 132 .exclude_idle = true, 133 .exclude_host = 1, 134 .exclude_user = exclude_user, 135 .exclude_kernel = exclude_kernel, 136 .config = config, 137 }; 138 139 if (type == PERF_TYPE_HARDWARE && config >= PERF_COUNT_HW_MAX) 140 return; 141 142 attr.sample_period = get_sample_period(pmc, pmc->counter); 143 144 if ((attr.config & HSW_IN_TX_CHECKPOINTED) && 145 guest_cpuid_is_intel(pmc->vcpu)) { 146 /* 147 * HSW_IN_TX_CHECKPOINTED is not supported with nonzero 148 * period. Just clear the sample period so at least 149 * allocating the counter doesn't fail. 150 */ 151 attr.sample_period = 0; 152 } 153 154 event = perf_event_create_kernel_counter(&attr, -1, current, 155 kvm_perf_overflow, pmc); 156 if (IS_ERR(event)) { 157 pr_debug_ratelimited("kvm_pmu: event creation failed %ld for pmc->idx = %d\n", 158 PTR_ERR(event), pmc->idx); 159 return; 160 } 161 162 pmc->perf_event = event; 163 pmc_to_pmu(pmc)->event_count++; 164 clear_bit(pmc->idx, pmc_to_pmu(pmc)->reprogram_pmi); 165 pmc->is_paused = false; 166 pmc->intr = intr; 167 } 168 169 static void pmc_pause_counter(struct kvm_pmc *pmc) 170 { 171 u64 counter = pmc->counter; 172 173 if (!pmc->perf_event || pmc->is_paused) 174 return; 175 176 /* update counter, reset event value to avoid redundant accumulation */ 177 counter += perf_event_pause(pmc->perf_event, true); 178 pmc->counter = counter & pmc_bitmask(pmc); 179 pmc->is_paused = true; 180 } 181 182 static bool pmc_resume_counter(struct kvm_pmc *pmc) 183 { 184 if (!pmc->perf_event) 185 return false; 186 187 /* recalibrate sample period and check if it's accepted by perf core */ 188 if (perf_event_period(pmc->perf_event, 189 get_sample_period(pmc, pmc->counter))) 190 return false; 191 192 /* reuse perf_event to serve as pmc_reprogram_counter() does*/ 193 perf_event_enable(pmc->perf_event); 194 pmc->is_paused = false; 195 196 clear_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->reprogram_pmi); 197 return true; 198 } 199 200 static int cmp_u64(const void *pa, const void *pb) 201 { 202 u64 a = *(u64 *)pa; 203 u64 b = *(u64 *)pb; 204 205 return (a > b) - (a < b); 206 } 207 208 void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) 209 { 210 u64 config; 211 u32 type = PERF_TYPE_RAW; 212 struct kvm *kvm = pmc->vcpu->kvm; 213 struct kvm_pmu_event_filter *filter; 214 struct kvm_pmu *pmu = vcpu_to_pmu(pmc->vcpu); 215 bool allow_event = true; 216 217 if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL) 218 printk_once("kvm pmu: pin control bit is ignored\n"); 219 220 pmc->eventsel = eventsel; 221 222 pmc_pause_counter(pmc); 223 224 if (!(eventsel & ARCH_PERFMON_EVENTSEL_ENABLE) || !pmc_is_enabled(pmc)) 225 return; 226 227 filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu); 228 if (filter) { 229 __u64 key = eventsel & AMD64_RAW_EVENT_MASK_NB; 230 231 if (bsearch(&key, filter->events, filter->nevents, 232 sizeof(__u64), cmp_u64)) 233 allow_event = filter->action == KVM_PMU_EVENT_ALLOW; 234 else 235 allow_event = filter->action == KVM_PMU_EVENT_DENY; 236 } 237 if (!allow_event) 238 return; 239 240 if (!(eventsel & (ARCH_PERFMON_EVENTSEL_EDGE | 241 ARCH_PERFMON_EVENTSEL_INV | 242 ARCH_PERFMON_EVENTSEL_CMASK | 243 HSW_IN_TX | 244 HSW_IN_TX_CHECKPOINTED))) { 245 config = static_call(kvm_x86_pmu_pmc_perf_hw_id)(pmc); 246 if (config != PERF_COUNT_HW_MAX) 247 type = PERF_TYPE_HARDWARE; 248 } 249 250 if (type == PERF_TYPE_RAW) 251 config = eventsel & pmu->raw_event_mask; 252 253 if (pmc->current_config == eventsel && pmc_resume_counter(pmc)) 254 return; 255 256 pmc_release_perf_event(pmc); 257 258 pmc->current_config = eventsel; 259 pmc_reprogram_counter(pmc, type, config, 260 !(eventsel & ARCH_PERFMON_EVENTSEL_USR), 261 !(eventsel & ARCH_PERFMON_EVENTSEL_OS), 262 eventsel & ARCH_PERFMON_EVENTSEL_INT); 263 } 264 EXPORT_SYMBOL_GPL(reprogram_gp_counter); 265 266 void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx) 267 { 268 unsigned en_field = ctrl & 0x3; 269 bool pmi = ctrl & 0x8; 270 struct kvm_pmu_event_filter *filter; 271 struct kvm *kvm = pmc->vcpu->kvm; 272 273 pmc_pause_counter(pmc); 274 275 if (!en_field || !pmc_is_enabled(pmc)) 276 return; 277 278 filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu); 279 if (filter) { 280 if (filter->action == KVM_PMU_EVENT_DENY && 281 test_bit(idx, (ulong *)&filter->fixed_counter_bitmap)) 282 return; 283 if (filter->action == KVM_PMU_EVENT_ALLOW && 284 !test_bit(idx, (ulong *)&filter->fixed_counter_bitmap)) 285 return; 286 } 287 288 if (pmc->current_config == (u64)ctrl && pmc_resume_counter(pmc)) 289 return; 290 291 pmc_release_perf_event(pmc); 292 293 pmc->current_config = (u64)ctrl; 294 pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE, 295 static_call(kvm_x86_pmu_pmc_perf_hw_id)(pmc), 296 !(en_field & 0x2), /* exclude user */ 297 !(en_field & 0x1), /* exclude kernel */ 298 pmi); 299 } 300 EXPORT_SYMBOL_GPL(reprogram_fixed_counter); 301 302 void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx) 303 { 304 struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, pmc_idx); 305 306 if (!pmc) 307 return; 308 309 if (pmc_is_gp(pmc)) 310 reprogram_gp_counter(pmc, pmc->eventsel); 311 else { 312 int idx = pmc_idx - INTEL_PMC_IDX_FIXED; 313 u8 ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl, idx); 314 315 reprogram_fixed_counter(pmc, ctrl, idx); 316 } 317 } 318 EXPORT_SYMBOL_GPL(reprogram_counter); 319 320 void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) 321 { 322 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 323 int bit; 324 325 for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) { 326 struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit); 327 328 if (unlikely(!pmc || !pmc->perf_event)) { 329 clear_bit(bit, pmu->reprogram_pmi); 330 continue; 331 } 332 333 reprogram_counter(pmu, bit); 334 } 335 336 /* 337 * Unused perf_events are only released if the corresponding MSRs 338 * weren't accessed during the last vCPU time slice. kvm_arch_sched_in 339 * triggers KVM_REQ_PMU if cleanup is needed. 340 */ 341 if (unlikely(pmu->need_cleanup)) 342 kvm_pmu_cleanup(vcpu); 343 } 344 345 /* check if idx is a valid index to access PMU */ 346 bool kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) 347 { 348 return static_call(kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx); 349 } 350 351 bool is_vmware_backdoor_pmc(u32 pmc_idx) 352 { 353 switch (pmc_idx) { 354 case VMWARE_BACKDOOR_PMC_HOST_TSC: 355 case VMWARE_BACKDOOR_PMC_REAL_TIME: 356 case VMWARE_BACKDOOR_PMC_APPARENT_TIME: 357 return true; 358 } 359 return false; 360 } 361 362 static int kvm_pmu_rdpmc_vmware(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) 363 { 364 u64 ctr_val; 365 366 switch (idx) { 367 case VMWARE_BACKDOOR_PMC_HOST_TSC: 368 ctr_val = rdtsc(); 369 break; 370 case VMWARE_BACKDOOR_PMC_REAL_TIME: 371 ctr_val = ktime_get_boottime_ns(); 372 break; 373 case VMWARE_BACKDOOR_PMC_APPARENT_TIME: 374 ctr_val = ktime_get_boottime_ns() + 375 vcpu->kvm->arch.kvmclock_offset; 376 break; 377 default: 378 return 1; 379 } 380 381 *data = ctr_val; 382 return 0; 383 } 384 385 int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) 386 { 387 bool fast_mode = idx & (1u << 31); 388 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 389 struct kvm_pmc *pmc; 390 u64 mask = fast_mode ? ~0u : ~0ull; 391 392 if (!pmu->version) 393 return 1; 394 395 if (is_vmware_backdoor_pmc(idx)) 396 return kvm_pmu_rdpmc_vmware(vcpu, idx, data); 397 398 pmc = static_call(kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask); 399 if (!pmc) 400 return 1; 401 402 if (!(kvm_read_cr4(vcpu) & X86_CR4_PCE) && 403 (static_call(kvm_x86_get_cpl)(vcpu) != 0) && 404 (kvm_read_cr0(vcpu) & X86_CR0_PE)) 405 return 1; 406 407 *data = pmc_read_counter(pmc) & mask; 408 return 0; 409 } 410 411 void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) 412 { 413 if (lapic_in_kernel(vcpu)) { 414 static_call_cond(kvm_x86_pmu_deliver_pmi)(vcpu); 415 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC); 416 } 417 } 418 419 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) 420 { 421 return static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) || 422 static_call(kvm_x86_pmu_is_valid_msr)(vcpu, msr); 423 } 424 425 static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr) 426 { 427 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 428 struct kvm_pmc *pmc = static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr); 429 430 if (pmc) 431 __set_bit(pmc->idx, pmu->pmc_in_use); 432 } 433 434 int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 435 { 436 return static_call(kvm_x86_pmu_get_msr)(vcpu, msr_info); 437 } 438 439 int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 440 { 441 kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index); 442 return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info); 443 } 444 445 /* refresh PMU settings. This function generally is called when underlying 446 * settings are changed (such as changes of PMU CPUID by guest VMs), which 447 * should rarely happen. 448 */ 449 void kvm_pmu_refresh(struct kvm_vcpu *vcpu) 450 { 451 static_call(kvm_x86_pmu_refresh)(vcpu); 452 } 453 454 void kvm_pmu_reset(struct kvm_vcpu *vcpu) 455 { 456 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 457 458 irq_work_sync(&pmu->irq_work); 459 static_call(kvm_x86_pmu_reset)(vcpu); 460 } 461 462 void kvm_pmu_init(struct kvm_vcpu *vcpu) 463 { 464 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 465 466 memset(pmu, 0, sizeof(*pmu)); 467 static_call(kvm_x86_pmu_init)(vcpu); 468 init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn); 469 pmu->event_count = 0; 470 pmu->need_cleanup = false; 471 kvm_pmu_refresh(vcpu); 472 } 473 474 static inline bool pmc_speculative_in_use(struct kvm_pmc *pmc) 475 { 476 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 477 478 if (pmc_is_fixed(pmc)) 479 return fixed_ctrl_field(pmu->fixed_ctr_ctrl, 480 pmc->idx - INTEL_PMC_IDX_FIXED) & 0x3; 481 482 return pmc->eventsel & ARCH_PERFMON_EVENTSEL_ENABLE; 483 } 484 485 /* Release perf_events for vPMCs that have been unused for a full time slice. */ 486 void kvm_pmu_cleanup(struct kvm_vcpu *vcpu) 487 { 488 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 489 struct kvm_pmc *pmc = NULL; 490 DECLARE_BITMAP(bitmask, X86_PMC_IDX_MAX); 491 int i; 492 493 pmu->need_cleanup = false; 494 495 bitmap_andnot(bitmask, pmu->all_valid_pmc_idx, 496 pmu->pmc_in_use, X86_PMC_IDX_MAX); 497 498 for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) { 499 pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i); 500 501 if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc)) 502 pmc_stop_counter(pmc); 503 } 504 505 static_call_cond(kvm_x86_pmu_cleanup)(vcpu); 506 507 bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX); 508 } 509 510 void kvm_pmu_destroy(struct kvm_vcpu *vcpu) 511 { 512 kvm_pmu_reset(vcpu); 513 } 514 515 static void kvm_pmu_incr_counter(struct kvm_pmc *pmc) 516 { 517 struct kvm_pmu *pmu = pmc_to_pmu(pmc); 518 u64 prev_count; 519 520 prev_count = pmc->counter; 521 pmc->counter = (pmc->counter + 1) & pmc_bitmask(pmc); 522 523 reprogram_counter(pmu, pmc->idx); 524 if (pmc->counter < prev_count) 525 __kvm_perf_overflow(pmc, false); 526 } 527 528 static inline bool eventsel_match_perf_hw_id(struct kvm_pmc *pmc, 529 unsigned int perf_hw_id) 530 { 531 u64 old_eventsel = pmc->eventsel; 532 unsigned int config; 533 534 pmc->eventsel &= (ARCH_PERFMON_EVENTSEL_EVENT | ARCH_PERFMON_EVENTSEL_UMASK); 535 config = static_call(kvm_x86_pmu_pmc_perf_hw_id)(pmc); 536 pmc->eventsel = old_eventsel; 537 return config == perf_hw_id; 538 } 539 540 static inline bool cpl_is_matched(struct kvm_pmc *pmc) 541 { 542 bool select_os, select_user; 543 u64 config = pmc->current_config; 544 545 if (pmc_is_gp(pmc)) { 546 select_os = config & ARCH_PERFMON_EVENTSEL_OS; 547 select_user = config & ARCH_PERFMON_EVENTSEL_USR; 548 } else { 549 select_os = config & 0x1; 550 select_user = config & 0x2; 551 } 552 553 return (static_call(kvm_x86_get_cpl)(pmc->vcpu) == 0) ? select_os : select_user; 554 } 555 556 void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, u64 perf_hw_id) 557 { 558 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 559 struct kvm_pmc *pmc; 560 int i; 561 562 for_each_set_bit(i, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX) { 563 pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i); 564 565 if (!pmc || !pmc_is_enabled(pmc) || !pmc_speculative_in_use(pmc)) 566 continue; 567 568 /* Ignore checks for edge detect, pin control, invert and CMASK bits */ 569 if (eventsel_match_perf_hw_id(pmc, perf_hw_id) && cpl_is_matched(pmc)) 570 kvm_pmu_incr_counter(pmc); 571 } 572 } 573 EXPORT_SYMBOL_GPL(kvm_pmu_trigger_event); 574 575 int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp) 576 { 577 struct kvm_pmu_event_filter tmp, *filter; 578 size_t size; 579 int r; 580 581 if (copy_from_user(&tmp, argp, sizeof(tmp))) 582 return -EFAULT; 583 584 if (tmp.action != KVM_PMU_EVENT_ALLOW && 585 tmp.action != KVM_PMU_EVENT_DENY) 586 return -EINVAL; 587 588 if (tmp.flags != 0) 589 return -EINVAL; 590 591 if (tmp.nevents > KVM_PMU_EVENT_FILTER_MAX_EVENTS) 592 return -E2BIG; 593 594 size = struct_size(filter, events, tmp.nevents); 595 filter = kmalloc(size, GFP_KERNEL_ACCOUNT); 596 if (!filter) 597 return -ENOMEM; 598 599 r = -EFAULT; 600 if (copy_from_user(filter, argp, size)) 601 goto cleanup; 602 603 /* Ensure nevents can't be changed between the user copies. */ 604 *filter = tmp; 605 606 /* 607 * Sort the in-kernel list so that we can search it with bsearch. 608 */ 609 sort(&filter->events, filter->nevents, sizeof(__u64), cmp_u64, NULL); 610 611 mutex_lock(&kvm->lock); 612 filter = rcu_replace_pointer(kvm->arch.pmu_event_filter, filter, 613 mutex_is_locked(&kvm->lock)); 614 mutex_unlock(&kvm->lock); 615 616 synchronize_srcu_expedited(&kvm->srcu); 617 r = 0; 618 cleanup: 619 kfree(filter); 620 return r; 621 } 622