120c8ccb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2f5132b01SGleb Natapov /*
3c7a7062fSGuo Chao * Kernel-based Virtual Machine -- Performance Monitoring Unit support
4f5132b01SGleb Natapov *
525462f7fSWei Huang * Copyright 2015 Red Hat, Inc. and/or its affiliates.
6f5132b01SGleb Natapov *
7f5132b01SGleb Natapov * Authors:
8f5132b01SGleb Natapov * Avi Kivity <avi@redhat.com>
9f5132b01SGleb Natapov * Gleb Natapov <gleb@redhat.com>
1025462f7fSWei Huang * Wei Huang <wei@redhat.com>
11f5132b01SGleb Natapov */
128d20bd63SSean Christopherson #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
13f5132b01SGleb Natapov
14f5132b01SGleb Natapov #include <linux/types.h>
15f5132b01SGleb Natapov #include <linux/kvm_host.h>
16f5132b01SGleb Natapov #include <linux/perf_event.h>
177ff775acSJim Mattson #include <linux/bsearch.h>
187ff775acSJim Mattson #include <linux/sort.h>
19d27aa7f1SNadav Amit #include <asm/perf_event.h>
2043d62d10SLike Xu #include <asm/cpu_device_id.h>
21f5132b01SGleb Natapov #include "x86.h"
22f5132b01SGleb Natapov #include "cpuid.h"
23f5132b01SGleb Natapov #include "lapic.h"
24474a5bb9SWei Huang #include "pmu.h"
25f5132b01SGleb Natapov
2630cd8604SEric Hankland /* This is enough to filter the vast majority of currently defined events. */
2730cd8604SEric Hankland #define KVM_PMU_EVENT_FILTER_MAX_EVENTS 300
2866bb8a06SEric Hankland
29968635abSLike Xu struct x86_pmu_capability __read_mostly kvm_pmu_cap;
30968635abSLike Xu EXPORT_SYMBOL_GPL(kvm_pmu_cap);
31968635abSLike Xu
32974850beSLike Xu /* Precise Distribution of Instructions Retired (PDIR) */
33974850beSLike Xu static const struct x86_cpu_id vmx_pebs_pdir_cpu[] = {
3443d62d10SLike Xu X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, NULL),
3543d62d10SLike Xu X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, NULL),
36974850beSLike Xu /* Instruction-Accurate PDIR (PDIR++) */
37974850beSLike Xu X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, NULL),
38974850beSLike Xu {}
39974850beSLike Xu };
40974850beSLike Xu
41974850beSLike Xu /* Precise Distribution (PDist) */
42974850beSLike Xu static const struct x86_cpu_id vmx_pebs_pdist_cpu[] = {
43974850beSLike Xu X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, NULL),
4443d62d10SLike Xu {}
4543d62d10SLike Xu };
4643d62d10SLike Xu
4725462f7fSWei Huang /* NOTE:
4825462f7fSWei Huang * - Each perf counter is defined as "struct kvm_pmc";
4925462f7fSWei Huang * - There are two types of perf counters: general purpose (gp) and fixed.
5025462f7fSWei Huang * gp counters are stored in gp_counters[] and fixed counters are stored
5125462f7fSWei Huang * in fixed_counters[] respectively. Both of them are part of "struct
5225462f7fSWei Huang * kvm_pmu";
5325462f7fSWei Huang * - pmu.c understands the difference between gp counters and fixed counters.
5425462f7fSWei Huang * However AMD doesn't support fixed-counters;
5525462f7fSWei Huang * - There are three types of index to access perf counters (PMC):
5625462f7fSWei Huang * 1. MSR (named msr): For example Intel has MSR_IA32_PERFCTRn and AMD
57a33095f4SLike Xu * has MSR_K7_PERFCTRn and, for families 15H and later,
58a33095f4SLike Xu * MSR_F15H_PERF_CTRn, where MSR_F15H_PERF_CTR[0-3] are
59a33095f4SLike Xu * aliased to MSR_K7_PERFCTRn.
6025462f7fSWei Huang * 2. MSR Index (named idx): This normally is used by RDPMC instruction.
6125462f7fSWei Huang * For instance AMD RDPMC instruction uses 0000_0003h in ECX to access
6225462f7fSWei Huang * C001_0007h (MSR_K7_PERCTR3). Intel has a similar mechanism, except
6325462f7fSWei Huang * that it also supports fixed counters. idx can be used to as index to
6425462f7fSWei Huang * gp and fixed counters.
6525462f7fSWei Huang * 3. Global PMC Index (named pmc): pmc is an index specific to PMU
6625462f7fSWei Huang * code. Each pmc, stored in kvm_pmc.idx field, is unique across
6725462f7fSWei Huang * all perf counters (both gp and fixed). The mapping relationship
6825462f7fSWei Huang * between pmc and perf counters is as the following:
694f1fa2a1SLike Xu * * Intel: [0 .. KVM_INTEL_PMC_MAX_GENERIC-1] <=> gp counters
7025462f7fSWei Huang * [INTEL_PMC_IDX_FIXED .. INTEL_PMC_IDX_FIXED + 2] <=> fixed
71a33095f4SLike Xu * * AMD: [0 .. AMD64_NUM_COUNTERS-1] and, for families 15H
72a33095f4SLike Xu * and later, [0 .. AMD64_NUM_COUNTERS_CORE-1] <=> gp counters
7325462f7fSWei Huang */
74f5132b01SGleb Natapov
758f969c0cSLike Xu static struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
768f969c0cSLike Xu
771921f3aaSLike Xu #define KVM_X86_PMU_OP(func) \
781921f3aaSLike Xu DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \
791921f3aaSLike Xu *(((struct kvm_pmu_ops *)0)->func));
801921f3aaSLike Xu #define KVM_X86_PMU_OP_OPTIONAL KVM_X86_PMU_OP
811921f3aaSLike Xu #include <asm/kvm-x86-pmu-ops.h>
821921f3aaSLike Xu
kvm_pmu_ops_update(const struct kvm_pmu_ops * pmu_ops)838f969c0cSLike Xu void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops)
848f969c0cSLike Xu {
858f969c0cSLike Xu memcpy(&kvm_pmu_ops, pmu_ops, sizeof(kvm_pmu_ops));
861921f3aaSLike Xu
871921f3aaSLike Xu #define __KVM_X86_PMU_OP(func) \
881921f3aaSLike Xu static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func);
891921f3aaSLike Xu #define KVM_X86_PMU_OP(func) \
901921f3aaSLike Xu WARN_ON(!kvm_pmu_ops.func); __KVM_X86_PMU_OP(func)
911921f3aaSLike Xu #define KVM_X86_PMU_OP_OPTIONAL __KVM_X86_PMU_OP
921921f3aaSLike Xu #include <asm/kvm-x86-pmu-ops.h>
931921f3aaSLike Xu #undef __KVM_X86_PMU_OP
948f969c0cSLike Xu }
958f969c0cSLike Xu
__kvm_perf_overflow(struct kvm_pmc * pmc,bool in_pmi)9640ccb96dSLike Xu static inline void __kvm_perf_overflow(struct kvm_pmc *pmc, bool in_pmi)
97f5132b01SGleb Natapov {
98212dba12SWei Huang struct kvm_pmu *pmu = pmc_to_pmu(pmc);
9979f3e3b5SLike Xu bool skip_pmi = false;
100e84cfe4cSWei Huang
10179f3e3b5SLike Xu if (pmc->perf_event && pmc->perf_event->attr.precise_ip) {
102f331601cSLike Xu if (!in_pmi) {
103f331601cSLike Xu /*
104f331601cSLike Xu * TODO: KVM is currently _choosing_ to not generate records
105f331601cSLike Xu * for emulated instructions, avoiding BUFFER_OVF PMI when
106f331601cSLike Xu * there are no records. Strictly speaking, it should be done
107f331601cSLike Xu * as well in the right context to improve sampling accuracy.
108f331601cSLike Xu */
109f331601cSLike Xu skip_pmi = true;
110f331601cSLike Xu } else {
11179f3e3b5SLike Xu /* Indicate PEBS overflow PMI to guest. */
11279f3e3b5SLike Xu skip_pmi = __test_and_set_bit(GLOBAL_STATUS_BUFFER_OVF_BIT,
11379f3e3b5SLike Xu (unsigned long *)&pmu->global_status);
114f331601cSLike Xu }
11579f3e3b5SLike Xu } else {
116f5132b01SGleb Natapov __set_bit(pmc->idx, (unsigned long *)&pmu->global_status);
11779f3e3b5SLike Xu }
118f5132b01SGleb Natapov
11973554b29SJim Mattson if (pmc->intr && !skip_pmi)
120f5132b01SGleb Natapov kvm_make_request(KVM_REQ_PMI, pmc->vcpu);
121f5132b01SGleb Natapov }
12240ccb96dSLike Xu
kvm_perf_overflow(struct perf_event * perf_event,struct perf_sample_data * data,struct pt_regs * regs)12340ccb96dSLike Xu static void kvm_perf_overflow(struct perf_event *perf_event,
12440ccb96dSLike Xu struct perf_sample_data *data,
12540ccb96dSLike Xu struct pt_regs *regs)
12640ccb96dSLike Xu {
12740ccb96dSLike Xu struct kvm_pmc *pmc = perf_event->overflow_handler_context;
12840ccb96dSLike Xu
129de0f6195SLike Xu /*
130de0f6195SLike Xu * Ignore overflow events for counters that are scheduled to be
131de0f6195SLike Xu * reprogrammed, e.g. if a PMI for the previous event races with KVM's
132de0f6195SLike Xu * handling of a related guest WRMSR.
133de0f6195SLike Xu */
134de0f6195SLike Xu if (test_and_set_bit(pmc->idx, pmc_to_pmu(pmc)->reprogram_pmi))
135de0f6195SLike Xu return;
136de0f6195SLike Xu
13740ccb96dSLike Xu __kvm_perf_overflow(pmc, true);
138de0f6195SLike Xu
139de0f6195SLike Xu kvm_make_request(KVM_REQ_PMU, pmc->vcpu);
140f5132b01SGleb Natapov }
141f5132b01SGleb Natapov
pmc_get_pebs_precise_level(struct kvm_pmc * pmc)142974850beSLike Xu static u64 pmc_get_pebs_precise_level(struct kvm_pmc *pmc)
143974850beSLike Xu {
144974850beSLike Xu /*
145974850beSLike Xu * For some model specific pebs counters with special capabilities
146974850beSLike Xu * (PDIR, PDIR++, PDIST), KVM needs to raise the event precise
147974850beSLike Xu * level to the maximum value (currently 3, backwards compatible)
148974850beSLike Xu * so that the perf subsystem would assign specific hardware counter
149974850beSLike Xu * with that capability for vPMC.
150974850beSLike Xu */
151974850beSLike Xu if ((pmc->idx == 0 && x86_match_cpu(vmx_pebs_pdist_cpu)) ||
152974850beSLike Xu (pmc->idx == 32 && x86_match_cpu(vmx_pebs_pdir_cpu)))
153974850beSLike Xu return 3;
154974850beSLike Xu
155974850beSLike Xu /*
156974850beSLike Xu * The non-zero precision level of guest event makes the ordinary
157974850beSLike Xu * guest event becomes a guest PEBS event and triggers the host
158974850beSLike Xu * PEBS PMI handler to determine whether the PEBS overflow PMI
159974850beSLike Xu * comes from the host counters or the guest.
160974850beSLike Xu */
161974850beSLike Xu return 1;
162974850beSLike Xu }
163974850beSLike Xu
pmc_reprogram_counter(struct kvm_pmc * pmc,u32 type,u64 config,bool exclude_user,bool exclude_kernel,bool intr)164dcbb816aSSean Christopherson static int pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, u64 config,
165dcbb816aSSean Christopherson bool exclude_user, bool exclude_kernel,
166dcbb816aSSean Christopherson bool intr)
167f5132b01SGleb Natapov {
16879f3e3b5SLike Xu struct kvm_pmu *pmu = pmc_to_pmu(pmc);
169f5132b01SGleb Natapov struct perf_event *event;
170f5132b01SGleb Natapov struct perf_event_attr attr = {
171f5132b01SGleb Natapov .type = type,
172f5132b01SGleb Natapov .size = sizeof(attr),
173f5132b01SGleb Natapov .pinned = true,
174f5132b01SGleb Natapov .exclude_idle = true,
175f5132b01SGleb Natapov .exclude_host = 1,
176f5132b01SGleb Natapov .exclude_user = exclude_user,
177f5132b01SGleb Natapov .exclude_kernel = exclude_kernel,
178f5132b01SGleb Natapov .config = config,
179f5132b01SGleb Natapov };
18079f3e3b5SLike Xu bool pebs = test_bit(pmc->idx, (unsigned long *)&pmu->pebs_enable);
181e84cfe4cSWei Huang
182168d918fSEric Hankland attr.sample_period = get_sample_period(pmc, pmc->counter);
183bba82fd7SRobert O'Callahan
184e644896fSLike Xu if ((attr.config & HSW_IN_TX_CHECKPOINTED) &&
185e644896fSLike Xu guest_cpuid_is_intel(pmc->vcpu)) {
186bba82fd7SRobert O'Callahan /*
187bba82fd7SRobert O'Callahan * HSW_IN_TX_CHECKPOINTED is not supported with nonzero
188bba82fd7SRobert O'Callahan * period. Just clear the sample period so at least
189bba82fd7SRobert O'Callahan * allocating the counter doesn't fail.
190bba82fd7SRobert O'Callahan */
191bba82fd7SRobert O'Callahan attr.sample_period = 0;
192bba82fd7SRobert O'Callahan }
19379f3e3b5SLike Xu if (pebs) {
19479f3e3b5SLike Xu /*
19579f3e3b5SLike Xu * For most PEBS hardware events, the difference in the software
19679f3e3b5SLike Xu * precision levels of guest and host PEBS events will not affect
19779f3e3b5SLike Xu * the accuracy of the PEBS profiling result, because the "event IP"
19879f3e3b5SLike Xu * in the PEBS record is calibrated on the guest side.
19979f3e3b5SLike Xu */
200974850beSLike Xu attr.precise_ip = pmc_get_pebs_precise_level(pmc);
20179f3e3b5SLike Xu }
202f5132b01SGleb Natapov
203f5132b01SGleb Natapov event = perf_event_create_kernel_counter(&attr, -1, current,
204f5132b01SGleb Natapov kvm_perf_overflow, pmc);
205f5132b01SGleb Natapov if (IS_ERR(event)) {
2066fc3977cSLike Xu pr_debug_ratelimited("kvm_pmu: event creation failed %ld for pmc->idx = %d\n",
2076fc3977cSLike Xu PTR_ERR(event), pmc->idx);
208dcbb816aSSean Christopherson return PTR_ERR(event);
209f5132b01SGleb Natapov }
210f5132b01SGleb Natapov
211f5132b01SGleb Natapov pmc->perf_event = event;
212b35e5548SLike Xu pmc_to_pmu(pmc)->event_count++;
213e79f49c3SLike Xu pmc->is_paused = false;
21479f3e3b5SLike Xu pmc->intr = intr || pebs;
215dcbb816aSSean Christopherson return 0;
216f5132b01SGleb Natapov }
217f5132b01SGleb Natapov
pmc_pause_counter(struct kvm_pmc * pmc)218a6da0d77SLike Xu static void pmc_pause_counter(struct kvm_pmc *pmc)
219a6da0d77SLike Xu {
220a6da0d77SLike Xu u64 counter = pmc->counter;
221a6da0d77SLike Xu
222e79f49c3SLike Xu if (!pmc->perf_event || pmc->is_paused)
223a6da0d77SLike Xu return;
224a6da0d77SLike Xu
225a6da0d77SLike Xu /* update counter, reset event value to avoid redundant accumulation */
226a6da0d77SLike Xu counter += perf_event_pause(pmc->perf_event, true);
227a6da0d77SLike Xu pmc->counter = counter & pmc_bitmask(pmc);
228e79f49c3SLike Xu pmc->is_paused = true;
229a6da0d77SLike Xu }
230a6da0d77SLike Xu
pmc_resume_counter(struct kvm_pmc * pmc)231a6da0d77SLike Xu static bool pmc_resume_counter(struct kvm_pmc *pmc)
232a6da0d77SLike Xu {
233a6da0d77SLike Xu if (!pmc->perf_event)
234a6da0d77SLike Xu return false;
235a6da0d77SLike Xu
236a6da0d77SLike Xu /* recalibrate sample period and check if it's accepted by perf core */
23755c590adSLike Xu if (is_sampling_event(pmc->perf_event) &&
23855c590adSLike Xu perf_event_period(pmc->perf_event,
239168d918fSEric Hankland get_sample_period(pmc, pmc->counter)))
240a6da0d77SLike Xu return false;
241a6da0d77SLike Xu
242cf52de61SLike Xu if (test_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->pebs_enable) !=
243cf52de61SLike Xu (!!pmc->perf_event->attr.precise_ip))
24479f3e3b5SLike Xu return false;
24579f3e3b5SLike Xu
246a6da0d77SLike Xu /* reuse perf_event to serve as pmc_reprogram_counter() does*/
247a6da0d77SLike Xu perf_event_enable(pmc->perf_event);
248e79f49c3SLike Xu pmc->is_paused = false;
249a6da0d77SLike Xu
250a6da0d77SLike Xu return true;
251a6da0d77SLike Xu }
252a6da0d77SLike Xu
pmc_release_perf_event(struct kvm_pmc * pmc)25357bbd59aSSean Christopherson static void pmc_release_perf_event(struct kvm_pmc *pmc)
25457bbd59aSSean Christopherson {
25557bbd59aSSean Christopherson if (pmc->perf_event) {
25657bbd59aSSean Christopherson perf_event_release_kernel(pmc->perf_event);
25757bbd59aSSean Christopherson pmc->perf_event = NULL;
25857bbd59aSSean Christopherson pmc->current_config = 0;
25957bbd59aSSean Christopherson pmc_to_pmu(pmc)->event_count--;
26057bbd59aSSean Christopherson }
26157bbd59aSSean Christopherson }
26257bbd59aSSean Christopherson
pmc_stop_counter(struct kvm_pmc * pmc)26357bbd59aSSean Christopherson static void pmc_stop_counter(struct kvm_pmc *pmc)
26457bbd59aSSean Christopherson {
26557bbd59aSSean Christopherson if (pmc->perf_event) {
26657bbd59aSSean Christopherson pmc->counter = pmc_read_counter(pmc);
26757bbd59aSSean Christopherson pmc_release_perf_event(pmc);
26857bbd59aSSean Christopherson }
26957bbd59aSSean Christopherson }
27057bbd59aSSean Christopherson
filter_cmp(const void * pa,const void * pb,u64 mask)27114329b82SAaron Lewis static int filter_cmp(const void *pa, const void *pb, u64 mask)
2727ff775acSJim Mattson {
27314329b82SAaron Lewis u64 a = *(u64 *)pa & mask;
27414329b82SAaron Lewis u64 b = *(u64 *)pb & mask;
2754ac19eadSAaron Lewis
2764ac19eadSAaron Lewis return (a > b) - (a < b);
2777ff775acSJim Mattson }
2787ff775acSJim Mattson
27914329b82SAaron Lewis
filter_sort_cmp(const void * pa,const void * pb)28014329b82SAaron Lewis static int filter_sort_cmp(const void *pa, const void *pb)
281c5a287faSAaron Lewis {
28214329b82SAaron Lewis return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT |
28314329b82SAaron Lewis KVM_PMU_MASKED_ENTRY_EXCLUDE));
284c5a287faSAaron Lewis }
285c5a287faSAaron Lewis
28614329b82SAaron Lewis /*
28714329b82SAaron Lewis * For the event filter, searching is done on the 'includes' list and
28814329b82SAaron Lewis * 'excludes' list separately rather than on the 'events' list (which
28914329b82SAaron Lewis * has both). As a result the exclude bit can be ignored.
29014329b82SAaron Lewis */
filter_event_cmp(const void * pa,const void * pb)29114329b82SAaron Lewis static int filter_event_cmp(const void *pa, const void *pb)
292c5a287faSAaron Lewis {
29314329b82SAaron Lewis return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT));
294c5a287faSAaron Lewis }
295c5a287faSAaron Lewis
find_filter_index(u64 * events,u64 nevents,u64 key)29614329b82SAaron Lewis static int find_filter_index(u64 *events, u64 nevents, u64 key)
29714329b82SAaron Lewis {
29814329b82SAaron Lewis u64 *fe = bsearch(&key, events, nevents, sizeof(events[0]),
29914329b82SAaron Lewis filter_event_cmp);
30014329b82SAaron Lewis
30114329b82SAaron Lewis if (!fe)
30214329b82SAaron Lewis return -1;
30314329b82SAaron Lewis
30414329b82SAaron Lewis return fe - events;
30514329b82SAaron Lewis }
30614329b82SAaron Lewis
is_filter_entry_match(u64 filter_event,u64 umask)30714329b82SAaron Lewis static bool is_filter_entry_match(u64 filter_event, u64 umask)
30814329b82SAaron Lewis {
30914329b82SAaron Lewis u64 mask = filter_event >> (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8);
31014329b82SAaron Lewis u64 match = filter_event & KVM_PMU_MASKED_ENTRY_UMASK_MATCH;
31114329b82SAaron Lewis
31214329b82SAaron Lewis BUILD_BUG_ON((KVM_PMU_ENCODE_MASKED_ENTRY(0, 0xff, 0, false) >>
31314329b82SAaron Lewis (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8)) !=
31414329b82SAaron Lewis ARCH_PERFMON_EVENTSEL_UMASK);
31514329b82SAaron Lewis
31614329b82SAaron Lewis return (umask & mask) == match;
31714329b82SAaron Lewis }
31814329b82SAaron Lewis
filter_contains_match(u64 * events,u64 nevents,u64 eventsel)31914329b82SAaron Lewis static bool filter_contains_match(u64 *events, u64 nevents, u64 eventsel)
32014329b82SAaron Lewis {
32114329b82SAaron Lewis u64 event_select = eventsel & kvm_pmu_ops.EVENTSEL_EVENT;
32214329b82SAaron Lewis u64 umask = eventsel & ARCH_PERFMON_EVENTSEL_UMASK;
32314329b82SAaron Lewis int i, index;
32414329b82SAaron Lewis
32514329b82SAaron Lewis index = find_filter_index(events, nevents, event_select);
32614329b82SAaron Lewis if (index < 0)
32714329b82SAaron Lewis return false;
32814329b82SAaron Lewis
32914329b82SAaron Lewis /*
33014329b82SAaron Lewis * Entries are sorted by the event select. Walk the list in both
33114329b82SAaron Lewis * directions to process all entries with the targeted event select.
33214329b82SAaron Lewis */
33314329b82SAaron Lewis for (i = index; i < nevents; i++) {
33414329b82SAaron Lewis if (filter_event_cmp(&events[i], &event_select))
33514329b82SAaron Lewis break;
33614329b82SAaron Lewis
33714329b82SAaron Lewis if (is_filter_entry_match(events[i], umask))
33814329b82SAaron Lewis return true;
33914329b82SAaron Lewis }
34014329b82SAaron Lewis
34114329b82SAaron Lewis for (i = index - 1; i >= 0; i--) {
34214329b82SAaron Lewis if (filter_event_cmp(&events[i], &event_select))
34314329b82SAaron Lewis break;
34414329b82SAaron Lewis
34514329b82SAaron Lewis if (is_filter_entry_match(events[i], umask))
34614329b82SAaron Lewis return true;
34714329b82SAaron Lewis }
34814329b82SAaron Lewis
34914329b82SAaron Lewis return false;
35014329b82SAaron Lewis }
35114329b82SAaron Lewis
is_gp_event_allowed(struct kvm_x86_pmu_event_filter * f,u64 eventsel)35214329b82SAaron Lewis static bool is_gp_event_allowed(struct kvm_x86_pmu_event_filter *f,
35314329b82SAaron Lewis u64 eventsel)
35414329b82SAaron Lewis {
35514329b82SAaron Lewis if (filter_contains_match(f->includes, f->nr_includes, eventsel) &&
35614329b82SAaron Lewis !filter_contains_match(f->excludes, f->nr_excludes, eventsel))
35714329b82SAaron Lewis return f->action == KVM_PMU_EVENT_ALLOW;
35814329b82SAaron Lewis
35914329b82SAaron Lewis return f->action == KVM_PMU_EVENT_DENY;
36014329b82SAaron Lewis }
36114329b82SAaron Lewis
is_fixed_event_allowed(struct kvm_x86_pmu_event_filter * filter,int idx)36214329b82SAaron Lewis static bool is_fixed_event_allowed(struct kvm_x86_pmu_event_filter *filter,
36314329b82SAaron Lewis int idx)
364c5a287faSAaron Lewis {
365c5a287faSAaron Lewis int fixed_idx = idx - INTEL_PMC_IDX_FIXED;
366c5a287faSAaron Lewis
367c5a287faSAaron Lewis if (filter->action == KVM_PMU_EVENT_DENY &&
368c5a287faSAaron Lewis test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap))
369c5a287faSAaron Lewis return false;
370c5a287faSAaron Lewis if (filter->action == KVM_PMU_EVENT_ALLOW &&
371c5a287faSAaron Lewis !test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap))
372c5a287faSAaron Lewis return false;
373c5a287faSAaron Lewis
374c5a287faSAaron Lewis return true;
375c5a287faSAaron Lewis }
376c5a287faSAaron Lewis
check_pmu_event_filter(struct kvm_pmc * pmc)37789cb454eSLike Xu static bool check_pmu_event_filter(struct kvm_pmc *pmc)
37889cb454eSLike Xu {
37914329b82SAaron Lewis struct kvm_x86_pmu_event_filter *filter;
38089cb454eSLike Xu struct kvm *kvm = pmc->vcpu->kvm;
38189cb454eSLike Xu
38289cb454eSLike Xu filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu);
38389cb454eSLike Xu if (!filter)
384c5a287faSAaron Lewis return true;
38589cb454eSLike Xu
386c5a287faSAaron Lewis if (pmc_is_gp(pmc))
387c5a287faSAaron Lewis return is_gp_event_allowed(filter, pmc->eventsel);
38889cb454eSLike Xu
389c5a287faSAaron Lewis return is_fixed_event_allowed(filter, pmc->idx);
39089cb454eSLike Xu }
39189cb454eSLike Xu
pmc_event_is_allowed(struct kvm_pmc * pmc)392dfdeda67SAaron Lewis static bool pmc_event_is_allowed(struct kvm_pmc *pmc)
393dfdeda67SAaron Lewis {
394dfdeda67SAaron Lewis return pmc_is_globally_enabled(pmc) && pmc_speculative_in_use(pmc) &&
3956de2ccc1SSean Christopherson static_call(kvm_x86_pmu_hw_event_available)(pmc) &&
396dfdeda67SAaron Lewis check_pmu_event_filter(pmc);
397dfdeda67SAaron Lewis }
398dfdeda67SAaron Lewis
reprogram_counter(struct kvm_pmc * pmc)39968fb4757SLike Xu static void reprogram_counter(struct kvm_pmc *pmc)
400f5132b01SGleb Natapov {
40189cb454eSLike Xu struct kvm_pmu *pmu = pmc_to_pmu(pmc);
402fb121aafSLike Xu u64 eventsel = pmc->eventsel;
40302791a5cSLike Xu u64 new_config = eventsel;
40402791a5cSLike Xu u8 fixed_ctr_ctrl;
40502791a5cSLike Xu
40602791a5cSLike Xu pmc_pause_counter(pmc);
40702791a5cSLike Xu
408dfdeda67SAaron Lewis if (!pmc_event_is_allowed(pmc))
409dcbb816aSSean Christopherson goto reprogram_complete;
410f5132b01SGleb Natapov
411de0f6195SLike Xu if (pmc->counter < pmc->prev_counter)
412de0f6195SLike Xu __kvm_perf_overflow(pmc, false);
413de0f6195SLike Xu
414a7b9d2ccSGleb Natapov if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL)
415a7b9d2ccSGleb Natapov printk_once("kvm pmu: pin control bit is ignored\n");
416a7b9d2ccSGleb Natapov
41702791a5cSLike Xu if (pmc_is_fixed(pmc)) {
41802791a5cSLike Xu fixed_ctr_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl,
41902791a5cSLike Xu pmc->idx - INTEL_PMC_IDX_FIXED);
42002791a5cSLike Xu if (fixed_ctr_ctrl & 0x1)
42102791a5cSLike Xu eventsel |= ARCH_PERFMON_EVENTSEL_OS;
42202791a5cSLike Xu if (fixed_ctr_ctrl & 0x2)
42302791a5cSLike Xu eventsel |= ARCH_PERFMON_EVENTSEL_USR;
42402791a5cSLike Xu if (fixed_ctr_ctrl & 0x8)
42502791a5cSLike Xu eventsel |= ARCH_PERFMON_EVENTSEL_INT;
42602791a5cSLike Xu new_config = (u64)fixed_ctr_ctrl;
427f5132b01SGleb Natapov }
428f5132b01SGleb Natapov
42902791a5cSLike Xu if (pmc->current_config == new_config && pmc_resume_counter(pmc))
430dcbb816aSSean Christopherson goto reprogram_complete;
431a6da0d77SLike Xu
432a6da0d77SLike Xu pmc_release_perf_event(pmc);
433a6da0d77SLike Xu
43402791a5cSLike Xu pmc->current_config = new_config;
435dcbb816aSSean Christopherson
436dcbb816aSSean Christopherson /*
437dcbb816aSSean Christopherson * If reprogramming fails, e.g. due to contention, leave the counter's
438dcbb816aSSean Christopherson * regprogram bit set, i.e. opportunistically try again on the next PMU
439dcbb816aSSean Christopherson * refresh. Don't make a new request as doing so can stall the guest
440dcbb816aSSean Christopherson * if reprogramming repeatedly fails.
441dcbb816aSSean Christopherson */
442dcbb816aSSean Christopherson if (pmc_reprogram_counter(pmc, PERF_TYPE_RAW,
44302791a5cSLike Xu (eventsel & pmu->raw_event_mask),
444f5132b01SGleb Natapov !(eventsel & ARCH_PERFMON_EVENTSEL_USR),
445f5132b01SGleb Natapov !(eventsel & ARCH_PERFMON_EVENTSEL_OS),
446dcbb816aSSean Christopherson eventsel & ARCH_PERFMON_EVENTSEL_INT))
447dcbb816aSSean Christopherson return;
448dcbb816aSSean Christopherson
449dcbb816aSSean Christopherson reprogram_complete:
450dcbb816aSSean Christopherson clear_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->reprogram_pmi);
451de0f6195SLike Xu pmc->prev_counter = 0;
452f5132b01SGleb Natapov }
453f5132b01SGleb Natapov
kvm_pmu_handle_event(struct kvm_vcpu * vcpu)454e5af058aSWei Huang void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
455e5af058aSWei Huang {
456e5af058aSWei Huang struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
457e5af058aSWei Huang int bit;
458e5af058aSWei Huang
4594be94672SLike Xu for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) {
4601921f3aaSLike Xu struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit);
461e5af058aSWei Huang
46268fb4757SLike Xu if (unlikely(!pmc)) {
4634be94672SLike Xu clear_bit(bit, pmu->reprogram_pmi);
464e5af058aSWei Huang continue;
465e5af058aSWei Huang }
46668fb4757SLike Xu
467a40239b4SLike Xu reprogram_counter(pmc);
468e5af058aSWei Huang }
469b35e5548SLike Xu
470b35e5548SLike Xu /*
471b35e5548SLike Xu * Unused perf_events are only released if the corresponding MSRs
472b35e5548SLike Xu * weren't accessed during the last vCPU time slice. kvm_arch_sched_in
473b35e5548SLike Xu * triggers KVM_REQ_PMU if cleanup is needed.
474b35e5548SLike Xu */
475b35e5548SLike Xu if (unlikely(pmu->need_cleanup))
476b35e5548SLike Xu kvm_pmu_cleanup(vcpu);
477e5af058aSWei Huang }
478e5af058aSWei Huang
479e5af058aSWei Huang /* check if idx is a valid index to access PMU */
kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu * vcpu,unsigned int idx)480e6cd31f1SJim Mattson bool kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx)
481e5af058aSWei Huang {
4821921f3aaSLike Xu return static_call(kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx);
48341aac14aSWei Huang }
48441aac14aSWei Huang
is_vmware_backdoor_pmc(u32 pmc_idx)4852d7921c4SArbel Moshe bool is_vmware_backdoor_pmc(u32 pmc_idx)
4862d7921c4SArbel Moshe {
4872d7921c4SArbel Moshe switch (pmc_idx) {
4882d7921c4SArbel Moshe case VMWARE_BACKDOOR_PMC_HOST_TSC:
4892d7921c4SArbel Moshe case VMWARE_BACKDOOR_PMC_REAL_TIME:
4902d7921c4SArbel Moshe case VMWARE_BACKDOOR_PMC_APPARENT_TIME:
4912d7921c4SArbel Moshe return true;
4922d7921c4SArbel Moshe }
4932d7921c4SArbel Moshe return false;
4942d7921c4SArbel Moshe }
4952d7921c4SArbel Moshe
kvm_pmu_rdpmc_vmware(struct kvm_vcpu * vcpu,unsigned idx,u64 * data)4962d7921c4SArbel Moshe static int kvm_pmu_rdpmc_vmware(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
4972d7921c4SArbel Moshe {
4982d7921c4SArbel Moshe u64 ctr_val;
4992d7921c4SArbel Moshe
5002d7921c4SArbel Moshe switch (idx) {
5012d7921c4SArbel Moshe case VMWARE_BACKDOOR_PMC_HOST_TSC:
5022d7921c4SArbel Moshe ctr_val = rdtsc();
5032d7921c4SArbel Moshe break;
5042d7921c4SArbel Moshe case VMWARE_BACKDOOR_PMC_REAL_TIME:
5059285ec4cSJason A. Donenfeld ctr_val = ktime_get_boottime_ns();
5062d7921c4SArbel Moshe break;
5072d7921c4SArbel Moshe case VMWARE_BACKDOOR_PMC_APPARENT_TIME:
5089285ec4cSJason A. Donenfeld ctr_val = ktime_get_boottime_ns() +
5092d7921c4SArbel Moshe vcpu->kvm->arch.kvmclock_offset;
5102d7921c4SArbel Moshe break;
5112d7921c4SArbel Moshe default:
5122d7921c4SArbel Moshe return 1;
5132d7921c4SArbel Moshe }
5142d7921c4SArbel Moshe
5152d7921c4SArbel Moshe *data = ctr_val;
5162d7921c4SArbel Moshe return 0;
5172d7921c4SArbel Moshe }
5182d7921c4SArbel Moshe
kvm_pmu_rdpmc(struct kvm_vcpu * vcpu,unsigned idx,u64 * data)51941aac14aSWei Huang int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
52041aac14aSWei Huang {
52141aac14aSWei Huang bool fast_mode = idx & (1u << 31);
522672ff6cfSLiran Alon struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
52341aac14aSWei Huang struct kvm_pmc *pmc;
5240e6f467eSPaolo Bonzini u64 mask = fast_mode ? ~0u : ~0ull;
52541aac14aSWei Huang
526672ff6cfSLiran Alon if (!pmu->version)
527672ff6cfSLiran Alon return 1;
528672ff6cfSLiran Alon
5292d7921c4SArbel Moshe if (is_vmware_backdoor_pmc(idx))
5302d7921c4SArbel Moshe return kvm_pmu_rdpmc_vmware(vcpu, idx, data);
5312d7921c4SArbel Moshe
5321921f3aaSLike Xu pmc = static_call(kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask);
53341aac14aSWei Huang if (!pmc)
53441aac14aSWei Huang return 1;
53541aac14aSWei Huang
536607475cfSBinbin Wu if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCE) &&
537b3646477SJason Baron (static_call(kvm_x86_get_cpl)(vcpu) != 0) &&
538607475cfSBinbin Wu kvm_is_cr0_bit_set(vcpu, X86_CR0_PE))
539632a4cf5SLike Xu return 1;
540632a4cf5SLike Xu
5410e6f467eSPaolo Bonzini *data = pmc_read_counter(pmc) & mask;
542e5af058aSWei Huang return 0;
543e5af058aSWei Huang }
544e5af058aSWei Huang
kvm_pmu_deliver_pmi(struct kvm_vcpu * vcpu)545e5af058aSWei Huang void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
546e5af058aSWei Huang {
547e6209a3bSLike Xu if (lapic_in_kernel(vcpu)) {
5481921f3aaSLike Xu static_call_cond(kvm_x86_pmu_deliver_pmi)(vcpu);
549e5af058aSWei Huang kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
550e5af058aSWei Huang }
551e6209a3bSLike Xu }
552e5af058aSWei Huang
kvm_pmu_is_valid_msr(struct kvm_vcpu * vcpu,u32 msr)553545feb96SSean Christopherson bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
554f5132b01SGleb Natapov {
555c85cdc1cSLike Xu switch (msr) {
556c85cdc1cSLike Xu case MSR_CORE_PERF_GLOBAL_STATUS:
557c85cdc1cSLike Xu case MSR_CORE_PERF_GLOBAL_CTRL:
558c85cdc1cSLike Xu case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
559c85cdc1cSLike Xu return kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu));
560c85cdc1cSLike Xu default:
561c85cdc1cSLike Xu break;
562c85cdc1cSLike Xu }
5631921f3aaSLike Xu return static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) ||
564545feb96SSean Christopherson static_call(kvm_x86_pmu_is_valid_msr)(vcpu, msr);
565f5132b01SGleb Natapov }
566f5132b01SGleb Natapov
kvm_pmu_mark_pmc_in_use(struct kvm_vcpu * vcpu,u32 msr)567b35e5548SLike Xu static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
568b35e5548SLike Xu {
569b35e5548SLike Xu struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
5701921f3aaSLike Xu struct kvm_pmc *pmc = static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr);
571b35e5548SLike Xu
572b35e5548SLike Xu if (pmc)
573b35e5548SLike Xu __set_bit(pmc->idx, pmu->pmc_in_use);
574b35e5548SLike Xu }
575b35e5548SLike Xu
kvm_pmu_get_msr(struct kvm_vcpu * vcpu,struct msr_data * msr_info)576cbd71758SWei Wang int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
577f5132b01SGleb Natapov {
578c85cdc1cSLike Xu struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
579c85cdc1cSLike Xu u32 msr = msr_info->index;
580c85cdc1cSLike Xu
581c85cdc1cSLike Xu switch (msr) {
582c85cdc1cSLike Xu case MSR_CORE_PERF_GLOBAL_STATUS:
5834a277189SLike Xu case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS:
584c85cdc1cSLike Xu msr_info->data = pmu->global_status;
585c85cdc1cSLike Xu break;
5864a277189SLike Xu case MSR_AMD64_PERF_CNTR_GLOBAL_CTL:
587c85cdc1cSLike Xu case MSR_CORE_PERF_GLOBAL_CTRL:
588c85cdc1cSLike Xu msr_info->data = pmu->global_ctrl;
589c85cdc1cSLike Xu break;
5904a277189SLike Xu case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR:
591c85cdc1cSLike Xu case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
592c85cdc1cSLike Xu msr_info->data = 0;
593c85cdc1cSLike Xu break;
594c85cdc1cSLike Xu default:
5951921f3aaSLike Xu return static_call(kvm_x86_pmu_get_msr)(vcpu, msr_info);
596f5132b01SGleb Natapov }
597f5132b01SGleb Natapov
598c85cdc1cSLike Xu return 0;
599c85cdc1cSLike Xu }
600c85cdc1cSLike Xu
kvm_pmu_set_msr(struct kvm_vcpu * vcpu,struct msr_data * msr_info)601afd80d85SPaolo Bonzini int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
602f5132b01SGleb Natapov {
603c85cdc1cSLike Xu struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
604c85cdc1cSLike Xu u32 msr = msr_info->index;
605c85cdc1cSLike Xu u64 data = msr_info->data;
606c85cdc1cSLike Xu u64 diff;
607c85cdc1cSLike Xu
6084a277189SLike Xu /*
6094a277189SLike Xu * Note, AMD ignores writes to reserved bits and read-only PMU MSRs,
6104a277189SLike Xu * whereas Intel generates #GP on attempts to write reserved/RO MSRs.
6114a277189SLike Xu */
612c85cdc1cSLike Xu switch (msr) {
613c85cdc1cSLike Xu case MSR_CORE_PERF_GLOBAL_STATUS:
614c85cdc1cSLike Xu if (!msr_info->host_initiated)
615c85cdc1cSLike Xu return 1; /* RO MSR */
6164a277189SLike Xu fallthrough;
6174a277189SLike Xu case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS:
6184a277189SLike Xu /* Per PPR, Read-only MSR. Writes are ignored. */
6194a277189SLike Xu if (!msr_info->host_initiated)
6204a277189SLike Xu break;
621c85cdc1cSLike Xu
622c85cdc1cSLike Xu if (data & pmu->global_status_mask)
623c85cdc1cSLike Xu return 1;
624c85cdc1cSLike Xu
625c85cdc1cSLike Xu pmu->global_status = data;
626c85cdc1cSLike Xu break;
6274a277189SLike Xu case MSR_AMD64_PERF_CNTR_GLOBAL_CTL:
6284a277189SLike Xu data &= ~pmu->global_ctrl_mask;
6294a277189SLike Xu fallthrough;
630c85cdc1cSLike Xu case MSR_CORE_PERF_GLOBAL_CTRL:
631c85cdc1cSLike Xu if (!kvm_valid_perf_global_ctrl(pmu, data))
632c85cdc1cSLike Xu return 1;
633c85cdc1cSLike Xu
634c85cdc1cSLike Xu if (pmu->global_ctrl != data) {
635c85cdc1cSLike Xu diff = pmu->global_ctrl ^ data;
636c85cdc1cSLike Xu pmu->global_ctrl = data;
637c85cdc1cSLike Xu reprogram_counters(pmu, diff);
638c85cdc1cSLike Xu }
639c85cdc1cSLike Xu break;
640c85cdc1cSLike Xu case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
641c85cdc1cSLike Xu /*
642c85cdc1cSLike Xu * GLOBAL_OVF_CTRL, a.k.a. GLOBAL STATUS_RESET, clears bits in
643c85cdc1cSLike Xu * GLOBAL_STATUS, and so the set of reserved bits is the same.
644c85cdc1cSLike Xu */
645c85cdc1cSLike Xu if (data & pmu->global_status_mask)
646c85cdc1cSLike Xu return 1;
6474a277189SLike Xu fallthrough;
6484a277189SLike Xu case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR:
649c85cdc1cSLike Xu if (!msr_info->host_initiated)
650c85cdc1cSLike Xu pmu->global_status &= ~data;
651c85cdc1cSLike Xu break;
652c85cdc1cSLike Xu default:
653b35e5548SLike Xu kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index);
6541921f3aaSLike Xu return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info);
655f5132b01SGleb Natapov }
656f5132b01SGleb Natapov
657c85cdc1cSLike Xu return 0;
658c85cdc1cSLike Xu }
659c85cdc1cSLike Xu
kvm_pmu_reset(struct kvm_vcpu * vcpu)660e5af058aSWei Huang void kvm_pmu_reset(struct kvm_vcpu *vcpu)
661e5af058aSWei Huang {
66257bbd59aSSean Christopherson struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
66357bbd59aSSean Christopherson struct kvm_pmc *pmc;
66457bbd59aSSean Christopherson int i;
66557bbd59aSSean Christopherson
6667c7ddf45SSean Christopherson pmu->need_cleanup = false;
6677c7ddf45SSean Christopherson
66857bbd59aSSean Christopherson bitmap_zero(pmu->reprogram_pmi, X86_PMC_IDX_MAX);
66957bbd59aSSean Christopherson
67057bbd59aSSean Christopherson for_each_set_bit(i, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX) {
67157bbd59aSSean Christopherson pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
67257bbd59aSSean Christopherson if (!pmc)
67357bbd59aSSean Christopherson continue;
67457bbd59aSSean Christopherson
67557bbd59aSSean Christopherson pmc_stop_counter(pmc);
67657bbd59aSSean Christopherson pmc->counter = 0;
67757bbd59aSSean Christopherson
67857bbd59aSSean Christopherson if (pmc_is_gp(pmc))
67957bbd59aSSean Christopherson pmc->eventsel = 0;
68057bbd59aSSean Christopherson }
68157bbd59aSSean Christopherson
68257bbd59aSSean Christopherson pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status = 0;
68357bbd59aSSean Christopherson
68457bbd59aSSean Christopherson static_call_cond(kvm_x86_pmu_reset)(vcpu);
685e5af058aSWei Huang }
686e5af058aSWei Huang
6877c7ddf45SSean Christopherson
6887c7ddf45SSean Christopherson /*
6897c7ddf45SSean Christopherson * Refresh the PMU configuration for the vCPU, e.g. if userspace changes CPUID
6907c7ddf45SSean Christopherson * and/or PERF_CAPABILITIES.
6917c7ddf45SSean Christopherson */
kvm_pmu_refresh(struct kvm_vcpu * vcpu)6927c7ddf45SSean Christopherson void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
6937c7ddf45SSean Christopherson {
6946393087dSSean Christopherson struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
6956393087dSSean Christopherson
6967c7ddf45SSean Christopherson if (KVM_BUG_ON(kvm_vcpu_has_run(vcpu), vcpu->kvm))
6977c7ddf45SSean Christopherson return;
6987c7ddf45SSean Christopherson
6997c7ddf45SSean Christopherson /*
7007c7ddf45SSean Christopherson * Stop/release all existing counters/events before realizing the new
7017c7ddf45SSean Christopherson * vPMU model.
7027c7ddf45SSean Christopherson */
7037c7ddf45SSean Christopherson kvm_pmu_reset(vcpu);
7047c7ddf45SSean Christopherson
7056393087dSSean Christopherson pmu->version = 0;
7066393087dSSean Christopherson pmu->nr_arch_gp_counters = 0;
7076393087dSSean Christopherson pmu->nr_arch_fixed_counters = 0;
7086393087dSSean Christopherson pmu->counter_bitmask[KVM_PMC_GP] = 0;
7096393087dSSean Christopherson pmu->counter_bitmask[KVM_PMC_FIXED] = 0;
7106393087dSSean Christopherson pmu->reserved_bits = 0xffffffff00200000ull;
7116393087dSSean Christopherson pmu->raw_event_mask = X86_RAW_EVENT_MASK;
7126393087dSSean Christopherson pmu->global_ctrl_mask = ~0ull;
7136393087dSSean Christopherson pmu->global_status_mask = ~0ull;
7146393087dSSean Christopherson pmu->fixed_ctr_ctrl_mask = ~0ull;
7156393087dSSean Christopherson pmu->pebs_enable_mask = ~0ull;
7166393087dSSean Christopherson pmu->pebs_data_cfg_mask = ~0ull;
7176393087dSSean Christopherson bitmap_zero(pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX);
7186393087dSSean Christopherson
71967d2212bSSean Christopherson if (!vcpu->kvm->arch.enable_pmu)
72067d2212bSSean Christopherson return;
72167d2212bSSean Christopherson
7227c7ddf45SSean Christopherson static_call(kvm_x86_pmu_refresh)(vcpu);
72367d2212bSSean Christopherson
72467d2212bSSean Christopherson /*
72567d2212bSSean Christopherson * At RESET, both Intel and AMD CPUs set all enable bits for general
72667d2212bSSean Christopherson * purpose counters in IA32_PERF_GLOBAL_CTRL (so that software that
72767d2212bSSean Christopherson * was written for v1 PMUs don't unknowingly leave GP counters disabled
72867d2212bSSean Christopherson * in the global controls). Emulate that behavior when refreshing the
72967d2212bSSean Christopherson * PMU so that userspace doesn't need to manually set PERF_GLOBAL_CTRL.
73067d2212bSSean Christopherson */
73167d2212bSSean Christopherson if (kvm_pmu_has_perf_global_ctrl(pmu) && pmu->nr_arch_gp_counters)
73267d2212bSSean Christopherson pmu->global_ctrl = GENMASK_ULL(pmu->nr_arch_gp_counters - 1, 0);
7337c7ddf45SSean Christopherson }
7347c7ddf45SSean Christopherson
kvm_pmu_init(struct kvm_vcpu * vcpu)735f5132b01SGleb Natapov void kvm_pmu_init(struct kvm_vcpu *vcpu)
736f5132b01SGleb Natapov {
737212dba12SWei Huang struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
738f5132b01SGleb Natapov
739f5132b01SGleb Natapov memset(pmu, 0, sizeof(*pmu));
7401921f3aaSLike Xu static_call(kvm_x86_pmu_init)(vcpu);
741b35e5548SLike Xu pmu->event_count = 0;
742b35e5548SLike Xu pmu->need_cleanup = false;
743c6702c9dSWei Huang kvm_pmu_refresh(vcpu);
744f5132b01SGleb Natapov }
745f5132b01SGleb Natapov
746b35e5548SLike Xu /* Release perf_events for vPMCs that have been unused for a full time slice. */
kvm_pmu_cleanup(struct kvm_vcpu * vcpu)747b35e5548SLike Xu void kvm_pmu_cleanup(struct kvm_vcpu *vcpu)
748b35e5548SLike Xu {
749b35e5548SLike Xu struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
750b35e5548SLike Xu struct kvm_pmc *pmc = NULL;
751b35e5548SLike Xu DECLARE_BITMAP(bitmask, X86_PMC_IDX_MAX);
752b35e5548SLike Xu int i;
753b35e5548SLike Xu
754b35e5548SLike Xu pmu->need_cleanup = false;
755b35e5548SLike Xu
756b35e5548SLike Xu bitmap_andnot(bitmask, pmu->all_valid_pmc_idx,
757b35e5548SLike Xu pmu->pmc_in_use, X86_PMC_IDX_MAX);
758b35e5548SLike Xu
759b35e5548SLike Xu for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) {
7601921f3aaSLike Xu pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
761b35e5548SLike Xu
762b35e5548SLike Xu if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc))
763b35e5548SLike Xu pmc_stop_counter(pmc);
764b35e5548SLike Xu }
765b35e5548SLike Xu
7661921f3aaSLike Xu static_call_cond(kvm_x86_pmu_cleanup)(vcpu);
7679aa4f622SLike Xu
768b35e5548SLike Xu bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
769b35e5548SLike Xu }
770b35e5548SLike Xu
kvm_pmu_destroy(struct kvm_vcpu * vcpu)771f5132b01SGleb Natapov void kvm_pmu_destroy(struct kvm_vcpu *vcpu)
772f5132b01SGleb Natapov {
773f5132b01SGleb Natapov kvm_pmu_reset(vcpu);
774f5132b01SGleb Natapov }
77566bb8a06SEric Hankland
kvm_pmu_incr_counter(struct kvm_pmc * pmc)7769cd803d4SEric Hankland static void kvm_pmu_incr_counter(struct kvm_pmc *pmc)
7779cd803d4SEric Hankland {
778de0f6195SLike Xu pmc->prev_counter = pmc->counter;
7799cd803d4SEric Hankland pmc->counter = (pmc->counter + 1) & pmc_bitmask(pmc);
7804fa5843dSLike Xu kvm_pmu_request_counter_reprogram(pmc);
7819cd803d4SEric Hankland }
7829cd803d4SEric Hankland
eventsel_match_perf_hw_id(struct kvm_pmc * pmc,unsigned int perf_hw_id)7839cd803d4SEric Hankland static inline bool eventsel_match_perf_hw_id(struct kvm_pmc *pmc,
7849cd803d4SEric Hankland unsigned int perf_hw_id)
7859cd803d4SEric Hankland {
78608dca7a8SLike Xu return !((pmc->eventsel ^ perf_get_hw_event_config(perf_hw_id)) &
78708dca7a8SLike Xu AMD64_RAW_EVENT_MASK_NB);
7889cd803d4SEric Hankland }
7899cd803d4SEric Hankland
cpl_is_matched(struct kvm_pmc * pmc)7909cd803d4SEric Hankland static inline bool cpl_is_matched(struct kvm_pmc *pmc)
7919cd803d4SEric Hankland {
7929cd803d4SEric Hankland bool select_os, select_user;
79368fb4757SLike Xu u64 config;
7949cd803d4SEric Hankland
7959cd803d4SEric Hankland if (pmc_is_gp(pmc)) {
79668fb4757SLike Xu config = pmc->eventsel;
7979cd803d4SEric Hankland select_os = config & ARCH_PERFMON_EVENTSEL_OS;
7989cd803d4SEric Hankland select_user = config & ARCH_PERFMON_EVENTSEL_USR;
7999cd803d4SEric Hankland } else {
80068fb4757SLike Xu config = fixed_ctrl_field(pmc_to_pmu(pmc)->fixed_ctr_ctrl,
80168fb4757SLike Xu pmc->idx - INTEL_PMC_IDX_FIXED);
8029cd803d4SEric Hankland select_os = config & 0x1;
8039cd803d4SEric Hankland select_user = config & 0x2;
8049cd803d4SEric Hankland }
8059cd803d4SEric Hankland
8069cd803d4SEric Hankland return (static_call(kvm_x86_get_cpl)(pmc->vcpu) == 0) ? select_os : select_user;
8079cd803d4SEric Hankland }
8089cd803d4SEric Hankland
kvm_pmu_trigger_event(struct kvm_vcpu * vcpu,u64 perf_hw_id)8099cd803d4SEric Hankland void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, u64 perf_hw_id)
8109cd803d4SEric Hankland {
8119cd803d4SEric Hankland struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
8129cd803d4SEric Hankland struct kvm_pmc *pmc;
8139cd803d4SEric Hankland int i;
8149cd803d4SEric Hankland
8159cd803d4SEric Hankland for_each_set_bit(i, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX) {
8161921f3aaSLike Xu pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
8179cd803d4SEric Hankland
818dfdeda67SAaron Lewis if (!pmc || !pmc_event_is_allowed(pmc))
8199cd803d4SEric Hankland continue;
8209cd803d4SEric Hankland
8219cd803d4SEric Hankland /* Ignore checks for edge detect, pin control, invert and CMASK bits */
8229cd803d4SEric Hankland if (eventsel_match_perf_hw_id(pmc, perf_hw_id) && cpl_is_matched(pmc))
8239cd803d4SEric Hankland kvm_pmu_incr_counter(pmc);
8249cd803d4SEric Hankland }
8259cd803d4SEric Hankland }
8269cd803d4SEric Hankland EXPORT_SYMBOL_GPL(kvm_pmu_trigger_event);
8279cd803d4SEric Hankland
is_masked_filter_valid(const struct kvm_x86_pmu_event_filter * filter)82814329b82SAaron Lewis static bool is_masked_filter_valid(const struct kvm_x86_pmu_event_filter *filter)
82914329b82SAaron Lewis {
83014329b82SAaron Lewis u64 mask = kvm_pmu_ops.EVENTSEL_EVENT |
83114329b82SAaron Lewis KVM_PMU_MASKED_ENTRY_UMASK_MASK |
83214329b82SAaron Lewis KVM_PMU_MASKED_ENTRY_UMASK_MATCH |
83314329b82SAaron Lewis KVM_PMU_MASKED_ENTRY_EXCLUDE;
83414329b82SAaron Lewis int i;
83514329b82SAaron Lewis
83614329b82SAaron Lewis for (i = 0; i < filter->nevents; i++) {
83714329b82SAaron Lewis if (filter->events[i] & ~mask)
83814329b82SAaron Lewis return false;
83914329b82SAaron Lewis }
84014329b82SAaron Lewis
84114329b82SAaron Lewis return true;
84214329b82SAaron Lewis }
84314329b82SAaron Lewis
convert_to_masked_filter(struct kvm_x86_pmu_event_filter * filter)84414329b82SAaron Lewis static void convert_to_masked_filter(struct kvm_x86_pmu_event_filter *filter)
8458589827fSAaron Lewis {
8468589827fSAaron Lewis int i, j;
8478589827fSAaron Lewis
8488589827fSAaron Lewis for (i = 0, j = 0; i < filter->nevents; i++) {
84914329b82SAaron Lewis /*
85014329b82SAaron Lewis * Skip events that are impossible to match against a guest
85114329b82SAaron Lewis * event. When filtering, only the event select + unit mask
85214329b82SAaron Lewis * of the guest event is used. To maintain backwards
85314329b82SAaron Lewis * compatibility, impossible filters can't be rejected :-(
85414329b82SAaron Lewis */
8558589827fSAaron Lewis if (filter->events[i] & ~(kvm_pmu_ops.EVENTSEL_EVENT |
8568589827fSAaron Lewis ARCH_PERFMON_EVENTSEL_UMASK))
8578589827fSAaron Lewis continue;
85814329b82SAaron Lewis /*
85914329b82SAaron Lewis * Convert userspace events to a common in-kernel event so
86014329b82SAaron Lewis * only one code path is needed to support both events. For
86114329b82SAaron Lewis * the in-kernel events use masked events because they are
86214329b82SAaron Lewis * flexible enough to handle both cases. To convert to masked
86314329b82SAaron Lewis * events all that's needed is to add an "all ones" umask_mask,
86414329b82SAaron Lewis * (unmasked filter events don't support EXCLUDE).
86514329b82SAaron Lewis */
86614329b82SAaron Lewis filter->events[j++] = filter->events[i] |
86714329b82SAaron Lewis (0xFFULL << KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT);
8688589827fSAaron Lewis }
8698589827fSAaron Lewis
8708589827fSAaron Lewis filter->nevents = j;
8718589827fSAaron Lewis }
8728589827fSAaron Lewis
prepare_filter_lists(struct kvm_x86_pmu_event_filter * filter)87314329b82SAaron Lewis static int prepare_filter_lists(struct kvm_x86_pmu_event_filter *filter)
87414329b82SAaron Lewis {
87514329b82SAaron Lewis int i;
87614329b82SAaron Lewis
87714329b82SAaron Lewis if (!(filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS))
87814329b82SAaron Lewis convert_to_masked_filter(filter);
87914329b82SAaron Lewis else if (!is_masked_filter_valid(filter))
88014329b82SAaron Lewis return -EINVAL;
88114329b82SAaron Lewis
88214329b82SAaron Lewis /*
88314329b82SAaron Lewis * Sort entries by event select and includes vs. excludes so that all
88414329b82SAaron Lewis * entries for a given event select can be processed efficiently during
88514329b82SAaron Lewis * filtering. The EXCLUDE flag uses a more significant bit than the
88614329b82SAaron Lewis * event select, and so the sorted list is also effectively split into
88714329b82SAaron Lewis * includes and excludes sub-lists.
88814329b82SAaron Lewis */
88914329b82SAaron Lewis sort(&filter->events, filter->nevents, sizeof(filter->events[0]),
89014329b82SAaron Lewis filter_sort_cmp, NULL);
89114329b82SAaron Lewis
89214329b82SAaron Lewis i = filter->nevents;
89314329b82SAaron Lewis /* Find the first EXCLUDE event (only supported for masked events). */
89414329b82SAaron Lewis if (filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS) {
89514329b82SAaron Lewis for (i = 0; i < filter->nevents; i++) {
89614329b82SAaron Lewis if (filter->events[i] & KVM_PMU_MASKED_ENTRY_EXCLUDE)
89714329b82SAaron Lewis break;
89814329b82SAaron Lewis }
89914329b82SAaron Lewis }
90014329b82SAaron Lewis
90114329b82SAaron Lewis filter->nr_includes = i;
90214329b82SAaron Lewis filter->nr_excludes = filter->nevents - filter->nr_includes;
90314329b82SAaron Lewis filter->includes = filter->events;
90414329b82SAaron Lewis filter->excludes = filter->events + filter->nr_includes;
90514329b82SAaron Lewis
90614329b82SAaron Lewis return 0;
90714329b82SAaron Lewis }
90814329b82SAaron Lewis
kvm_vm_ioctl_set_pmu_event_filter(struct kvm * kvm,void __user * argp)90966bb8a06SEric Hankland int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp)
91066bb8a06SEric Hankland {
91114329b82SAaron Lewis struct kvm_pmu_event_filter __user *user_filter = argp;
91214329b82SAaron Lewis struct kvm_x86_pmu_event_filter *filter;
91314329b82SAaron Lewis struct kvm_pmu_event_filter tmp;
914f1c5651fSSean Christopherson struct kvm_vcpu *vcpu;
915f1c5651fSSean Christopherson unsigned long i;
91666bb8a06SEric Hankland size_t size;
91766bb8a06SEric Hankland int r;
91866bb8a06SEric Hankland
91914329b82SAaron Lewis if (copy_from_user(&tmp, user_filter, sizeof(tmp)))
92066bb8a06SEric Hankland return -EFAULT;
92166bb8a06SEric Hankland
92266bb8a06SEric Hankland if (tmp.action != KVM_PMU_EVENT_ALLOW &&
92366bb8a06SEric Hankland tmp.action != KVM_PMU_EVENT_DENY)
92466bb8a06SEric Hankland return -EINVAL;
92566bb8a06SEric Hankland
92614329b82SAaron Lewis if (tmp.flags & ~KVM_PMU_EVENT_FLAGS_VALID_MASK)
92730cd8604SEric Hankland return -EINVAL;
92830cd8604SEric Hankland
92966bb8a06SEric Hankland if (tmp.nevents > KVM_PMU_EVENT_FILTER_MAX_EVENTS)
93066bb8a06SEric Hankland return -E2BIG;
93166bb8a06SEric Hankland
93266bb8a06SEric Hankland size = struct_size(filter, events, tmp.nevents);
93314329b82SAaron Lewis filter = kzalloc(size, GFP_KERNEL_ACCOUNT);
93466bb8a06SEric Hankland if (!filter)
93566bb8a06SEric Hankland return -ENOMEM;
93666bb8a06SEric Hankland
93714329b82SAaron Lewis filter->action = tmp.action;
93814329b82SAaron Lewis filter->nevents = tmp.nevents;
93914329b82SAaron Lewis filter->fixed_counter_bitmap = tmp.fixed_counter_bitmap;
94014329b82SAaron Lewis filter->flags = tmp.flags;
94114329b82SAaron Lewis
94266bb8a06SEric Hankland r = -EFAULT;
94314329b82SAaron Lewis if (copy_from_user(filter->events, user_filter->events,
94414329b82SAaron Lewis sizeof(filter->events[0]) * filter->nevents))
94566bb8a06SEric Hankland goto cleanup;
94666bb8a06SEric Hankland
94714329b82SAaron Lewis r = prepare_filter_lists(filter);
94814329b82SAaron Lewis if (r)
94914329b82SAaron Lewis goto cleanup;
9507ff775acSJim Mattson
95166bb8a06SEric Hankland mutex_lock(&kvm->lock);
95212e78e69SPaul E. McKenney filter = rcu_replace_pointer(kvm->arch.pmu_event_filter, filter,
95366bb8a06SEric Hankland mutex_is_locked(&kvm->lock));
95495744a90SMichal Luczaj mutex_unlock(&kvm->lock);
955f1c5651fSSean Christopherson synchronize_srcu_expedited(&kvm->srcu);
956f1c5651fSSean Christopherson
957f1c5651fSSean Christopherson BUILD_BUG_ON(sizeof(((struct kvm_pmu *)0)->reprogram_pmi) >
958f1c5651fSSean Christopherson sizeof(((struct kvm_pmu *)0)->__reprogram_pmi));
959f1c5651fSSean Christopherson
960f1c5651fSSean Christopherson kvm_for_each_vcpu(i, vcpu, kvm)
961f1c5651fSSean Christopherson atomic64_set(&vcpu_to_pmu(vcpu)->__reprogram_pmi, -1ull);
962f1c5651fSSean Christopherson
963f1c5651fSSean Christopherson kvm_make_all_cpus_request(kvm, KVM_REQ_PMU);
964f1c5651fSSean Christopherson
96566bb8a06SEric Hankland r = 0;
96666bb8a06SEric Hankland cleanup:
96766bb8a06SEric Hankland kfree(filter);
96866bb8a06SEric Hankland return r;
96966bb8a06SEric Hankland }
970