1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2019 Arm Limited 4 * Author: Andrew Murray <Andrew.Murray@arm.com> 5 */ 6 #include <linux/kvm_host.h> 7 #include <linux/perf_event.h> 8 9 static DEFINE_PER_CPU(struct kvm_pmu_events, kvm_pmu_events); 10 11 /* 12 * Given the perf event attributes and system type, determine 13 * if we are going to need to switch counters at guest entry/exit. 14 */ 15 static bool kvm_pmu_switch_needed(struct perf_event_attr *attr) 16 { 17 /** 18 * With VHE the guest kernel runs at EL1 and the host at EL2, 19 * where user (EL0) is excluded then we have no reason to switch 20 * counters. 21 */ 22 if (has_vhe() && attr->exclude_user) 23 return false; 24 25 /* Only switch if attributes are different */ 26 return (attr->exclude_host != attr->exclude_guest); 27 } 28 29 struct kvm_pmu_events *kvm_get_pmu_events(void) 30 { 31 return this_cpu_ptr(&kvm_pmu_events); 32 } 33 34 /* 35 * Add events to track that we may want to switch at guest entry/exit 36 * time. 37 */ 38 void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr) 39 { 40 struct kvm_pmu_events *pmu = kvm_get_pmu_events(); 41 42 if (!kvm_arm_support_pmu_v3() || !pmu || !kvm_pmu_switch_needed(attr)) 43 return; 44 45 if (!attr->exclude_host) 46 pmu->events_host |= set; 47 if (!attr->exclude_guest) 48 pmu->events_guest |= set; 49 } 50 51 /* 52 * Stop tracking events 53 */ 54 void kvm_clr_pmu_events(u32 clr) 55 { 56 struct kvm_pmu_events *pmu = kvm_get_pmu_events(); 57 58 if (!kvm_arm_support_pmu_v3() || !pmu) 59 return; 60 61 pmu->events_host &= ~clr; 62 pmu->events_guest &= ~clr; 63 } 64 65 #define PMEVTYPER_READ_CASE(idx) \ 66 case idx: \ 67 return read_sysreg(pmevtyper##idx##_el0) 68 69 #define PMEVTYPER_WRITE_CASE(idx) \ 70 case idx: \ 71 write_sysreg(val, pmevtyper##idx##_el0); \ 72 break 73 74 #define PMEVTYPER_CASES(readwrite) \ 75 PMEVTYPER_##readwrite##_CASE(0); \ 76 PMEVTYPER_##readwrite##_CASE(1); \ 77 PMEVTYPER_##readwrite##_CASE(2); \ 78 PMEVTYPER_##readwrite##_CASE(3); \ 79 PMEVTYPER_##readwrite##_CASE(4); \ 80 PMEVTYPER_##readwrite##_CASE(5); \ 81 PMEVTYPER_##readwrite##_CASE(6); \ 82 PMEVTYPER_##readwrite##_CASE(7); \ 83 PMEVTYPER_##readwrite##_CASE(8); \ 84 PMEVTYPER_##readwrite##_CASE(9); \ 85 PMEVTYPER_##readwrite##_CASE(10); \ 86 PMEVTYPER_##readwrite##_CASE(11); \ 87 PMEVTYPER_##readwrite##_CASE(12); \ 88 PMEVTYPER_##readwrite##_CASE(13); \ 89 PMEVTYPER_##readwrite##_CASE(14); \ 90 PMEVTYPER_##readwrite##_CASE(15); \ 91 PMEVTYPER_##readwrite##_CASE(16); \ 92 PMEVTYPER_##readwrite##_CASE(17); \ 93 PMEVTYPER_##readwrite##_CASE(18); \ 94 PMEVTYPER_##readwrite##_CASE(19); \ 95 PMEVTYPER_##readwrite##_CASE(20); \ 96 PMEVTYPER_##readwrite##_CASE(21); \ 97 PMEVTYPER_##readwrite##_CASE(22); \ 98 PMEVTYPER_##readwrite##_CASE(23); \ 99 PMEVTYPER_##readwrite##_CASE(24); \ 100 PMEVTYPER_##readwrite##_CASE(25); \ 101 PMEVTYPER_##readwrite##_CASE(26); \ 102 PMEVTYPER_##readwrite##_CASE(27); \ 103 PMEVTYPER_##readwrite##_CASE(28); \ 104 PMEVTYPER_##readwrite##_CASE(29); \ 105 PMEVTYPER_##readwrite##_CASE(30) 106 107 /* 108 * Read a value direct from PMEVTYPER<idx> where idx is 0-30 109 * or PMCCFILTR_EL0 where idx is ARMV8_PMU_CYCLE_IDX (31). 110 */ 111 static u64 kvm_vcpu_pmu_read_evtype_direct(int idx) 112 { 113 switch (idx) { 114 PMEVTYPER_CASES(READ); 115 case ARMV8_PMU_CYCLE_IDX: 116 return read_sysreg(pmccfiltr_el0); 117 default: 118 WARN_ON(1); 119 } 120 121 return 0; 122 } 123 124 /* 125 * Write a value direct to PMEVTYPER<idx> where idx is 0-30 126 * or PMCCFILTR_EL0 where idx is ARMV8_PMU_CYCLE_IDX (31). 127 */ 128 static void kvm_vcpu_pmu_write_evtype_direct(int idx, u32 val) 129 { 130 switch (idx) { 131 PMEVTYPER_CASES(WRITE); 132 case ARMV8_PMU_CYCLE_IDX: 133 write_sysreg(val, pmccfiltr_el0); 134 break; 135 default: 136 WARN_ON(1); 137 } 138 } 139 140 /* 141 * Modify ARMv8 PMU events to include EL0 counting 142 */ 143 static void kvm_vcpu_pmu_enable_el0(unsigned long events) 144 { 145 u64 typer; 146 u32 counter; 147 148 for_each_set_bit(counter, &events, 32) { 149 typer = kvm_vcpu_pmu_read_evtype_direct(counter); 150 typer &= ~ARMV8_PMU_EXCLUDE_EL0; 151 kvm_vcpu_pmu_write_evtype_direct(counter, typer); 152 } 153 } 154 155 /* 156 * Modify ARMv8 PMU events to exclude EL0 counting 157 */ 158 static void kvm_vcpu_pmu_disable_el0(unsigned long events) 159 { 160 u64 typer; 161 u32 counter; 162 163 for_each_set_bit(counter, &events, 32) { 164 typer = kvm_vcpu_pmu_read_evtype_direct(counter); 165 typer |= ARMV8_PMU_EXCLUDE_EL0; 166 kvm_vcpu_pmu_write_evtype_direct(counter, typer); 167 } 168 } 169 170 /* 171 * On VHE ensure that only guest events have EL0 counting enabled. 172 * This is called from both vcpu_{load,put} and the sysreg handling. 173 * Since the latter is preemptible, special care must be taken to 174 * disable preemption. 175 */ 176 void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) 177 { 178 struct kvm_pmu_events *pmu; 179 u32 events_guest, events_host; 180 181 if (!kvm_arm_support_pmu_v3() || !has_vhe()) 182 return; 183 184 preempt_disable(); 185 pmu = kvm_get_pmu_events(); 186 events_guest = pmu->events_guest; 187 events_host = pmu->events_host; 188 189 kvm_vcpu_pmu_enable_el0(events_guest); 190 kvm_vcpu_pmu_disable_el0(events_host); 191 preempt_enable(); 192 } 193 194 /* 195 * On VHE ensure that only host events have EL0 counting enabled 196 */ 197 void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) 198 { 199 struct kvm_pmu_events *pmu; 200 u32 events_guest, events_host; 201 202 if (!kvm_arm_support_pmu_v3() || !has_vhe()) 203 return; 204 205 pmu = kvm_get_pmu_events(); 206 events_guest = pmu->events_guest; 207 events_host = pmu->events_host; 208 209 kvm_vcpu_pmu_enable_el0(events_host); 210 kvm_vcpu_pmu_disable_el0(events_guest); 211 } 212 213 /* 214 * With VHE, keep track of the PMUSERENR_EL0 value for the host EL0 on the pCPU 215 * where PMUSERENR_EL0 for the guest is loaded, since PMUSERENR_EL0 is switched 216 * to the value for the guest on vcpu_load(). The value for the host EL0 217 * will be restored on vcpu_put(), before returning to userspace. 218 * This isn't necessary for nVHE, as the register is context switched for 219 * every guest enter/exit. 220 * 221 * Return true if KVM takes care of the register. Otherwise return false. 222 */ 223 bool kvm_set_pmuserenr(u64 val) 224 { 225 struct kvm_cpu_context *hctxt; 226 struct kvm_vcpu *vcpu; 227 228 if (!kvm_arm_support_pmu_v3() || !has_vhe()) 229 return false; 230 231 vcpu = kvm_get_running_vcpu(); 232 if (!vcpu || !vcpu_get_flag(vcpu, PMUSERENR_ON_CPU)) 233 return false; 234 235 hctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; 236 ctxt_sys_reg(hctxt, PMUSERENR_EL0) = val; 237 return true; 238 } 239 240 /* 241 * If we interrupted the guest to update the host PMU context, make 242 * sure we re-apply the guest EL0 state. 243 */ 244 void kvm_vcpu_pmu_resync_el0(void) 245 { 246 struct kvm_vcpu *vcpu; 247 248 if (!has_vhe() || !in_interrupt()) 249 return; 250 251 vcpu = kvm_get_running_vcpu(); 252 if (!vcpu) 253 return; 254 255 kvm_make_request(KVM_REQ_RESYNC_PMU_EL0, vcpu); 256 } 257