1 /* 2 * Copyright (C) 2012,2013 - ARM Ltd 3 * Author: Marc Zyngier <marc.zyngier@arm.com> 4 * 5 * Derived from arch/arm/kvm/coproc.c: 6 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 7 * Authors: Rusty Russell <rusty@rustcorp.com.au> 8 * Christoffer Dall <c.dall@virtualopensystems.com> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License, version 2, as 12 * published by the Free Software Foundation. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this program. If not, see <http://www.gnu.org/licenses/>. 21 */ 22 23 #include <linux/mm.h> 24 #include <linux/kvm_host.h> 25 #include <linux/uaccess.h> 26 #include <asm/kvm_arm.h> 27 #include <asm/kvm_host.h> 28 #include <asm/kvm_emulate.h> 29 #include <asm/kvm_coproc.h> 30 #include <asm/kvm_mmu.h> 31 #include <asm/cacheflush.h> 32 #include <asm/cputype.h> 33 #include <asm/debug-monitors.h> 34 #include <trace/events/kvm.h> 35 36 #include "sys_regs.h" 37 38 /* 39 * All of this file is extremly similar to the ARM coproc.c, but the 40 * types are different. My gut feeling is that it should be pretty 41 * easy to merge, but that would be an ABI breakage -- again. VFP 42 * would also need to be abstracted. 43 * 44 * For AArch32, we only take care of what is being trapped. Anything 45 * that has to do with init and userspace access has to go via the 46 * 64bit interface. 47 */ 48 49 /* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */ 50 static u32 cache_levels; 51 52 /* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */ 53 #define CSSELR_MAX 12 54 55 /* Which cache CCSIDR represents depends on CSSELR value. */ 56 static u32 get_ccsidr(u32 csselr) 57 { 58 u32 ccsidr; 59 60 /* Make sure noone else changes CSSELR during this! */ 61 local_irq_disable(); 62 /* Put value into CSSELR */ 63 asm volatile("msr csselr_el1, %x0" : : "r" (csselr)); 64 isb(); 65 /* Read result out of CCSIDR */ 66 asm volatile("mrs %0, ccsidr_el1" : "=r" (ccsidr)); 67 local_irq_enable(); 68 69 return ccsidr; 70 } 71 72 static void do_dc_cisw(u32 val) 73 { 74 asm volatile("dc cisw, %x0" : : "r" (val)); 75 dsb(ish); 76 } 77 78 static void do_dc_csw(u32 val) 79 { 80 asm volatile("dc csw, %x0" : : "r" (val)); 81 dsb(ish); 82 } 83 84 /* See note at ARM ARM B1.14.4 */ 85 static bool access_dcsw(struct kvm_vcpu *vcpu, 86 const struct sys_reg_params *p, 87 const struct sys_reg_desc *r) 88 { 89 unsigned long val; 90 int cpu; 91 92 if (!p->is_write) 93 return read_from_write_only(vcpu, p); 94 95 cpu = get_cpu(); 96 97 cpumask_setall(&vcpu->arch.require_dcache_flush); 98 cpumask_clear_cpu(cpu, &vcpu->arch.require_dcache_flush); 99 100 /* If we were already preempted, take the long way around */ 101 if (cpu != vcpu->arch.last_pcpu) { 102 flush_cache_all(); 103 goto done; 104 } 105 106 val = *vcpu_reg(vcpu, p->Rt); 107 108 switch (p->CRm) { 109 case 6: /* Upgrade DCISW to DCCISW, as per HCR.SWIO */ 110 case 14: /* DCCISW */ 111 do_dc_cisw(val); 112 break; 113 114 case 10: /* DCCSW */ 115 do_dc_csw(val); 116 break; 117 } 118 119 done: 120 put_cpu(); 121 122 return true; 123 } 124 125 /* 126 * Generic accessor for VM registers. Only called as long as HCR_TVM 127 * is set. 128 */ 129 static bool access_vm_reg(struct kvm_vcpu *vcpu, 130 const struct sys_reg_params *p, 131 const struct sys_reg_desc *r) 132 { 133 unsigned long val; 134 135 BUG_ON(!p->is_write); 136 137 val = *vcpu_reg(vcpu, p->Rt); 138 if (!p->is_aarch32) { 139 vcpu_sys_reg(vcpu, r->reg) = val; 140 } else { 141 if (!p->is_32bit) 142 vcpu_cp15_64_high(vcpu, r->reg) = val >> 32; 143 vcpu_cp15_64_low(vcpu, r->reg) = val & 0xffffffffUL; 144 } 145 146 return true; 147 } 148 149 /* 150 * SCTLR_EL1 accessor. Only called as long as HCR_TVM is set. If the 151 * guest enables the MMU, we stop trapping the VM sys_regs and leave 152 * it in complete control of the caches. 153 */ 154 static bool access_sctlr(struct kvm_vcpu *vcpu, 155 const struct sys_reg_params *p, 156 const struct sys_reg_desc *r) 157 { 158 access_vm_reg(vcpu, p, r); 159 160 if (vcpu_has_cache_enabled(vcpu)) { /* MMU+Caches enabled? */ 161 vcpu->arch.hcr_el2 &= ~HCR_TVM; 162 stage2_flush_vm(vcpu->kvm); 163 } 164 165 return true; 166 } 167 168 static bool trap_raz_wi(struct kvm_vcpu *vcpu, 169 const struct sys_reg_params *p, 170 const struct sys_reg_desc *r) 171 { 172 if (p->is_write) 173 return ignore_write(vcpu, p); 174 else 175 return read_zero(vcpu, p); 176 } 177 178 static bool trap_oslsr_el1(struct kvm_vcpu *vcpu, 179 const struct sys_reg_params *p, 180 const struct sys_reg_desc *r) 181 { 182 if (p->is_write) { 183 return ignore_write(vcpu, p); 184 } else { 185 *vcpu_reg(vcpu, p->Rt) = (1 << 3); 186 return true; 187 } 188 } 189 190 static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu, 191 const struct sys_reg_params *p, 192 const struct sys_reg_desc *r) 193 { 194 if (p->is_write) { 195 return ignore_write(vcpu, p); 196 } else { 197 u32 val; 198 asm volatile("mrs %0, dbgauthstatus_el1" : "=r" (val)); 199 *vcpu_reg(vcpu, p->Rt) = val; 200 return true; 201 } 202 } 203 204 /* 205 * We want to avoid world-switching all the DBG registers all the 206 * time: 207 * 208 * - If we've touched any debug register, it is likely that we're 209 * going to touch more of them. It then makes sense to disable the 210 * traps and start doing the save/restore dance 211 * - If debug is active (DBG_MDSCR_KDE or DBG_MDSCR_MDE set), it is 212 * then mandatory to save/restore the registers, as the guest 213 * depends on them. 214 * 215 * For this, we use a DIRTY bit, indicating the guest has modified the 216 * debug registers, used as follow: 217 * 218 * On guest entry: 219 * - If the dirty bit is set (because we're coming back from trapping), 220 * disable the traps, save host registers, restore guest registers. 221 * - If debug is actively in use (DBG_MDSCR_KDE or DBG_MDSCR_MDE set), 222 * set the dirty bit, disable the traps, save host registers, 223 * restore guest registers. 224 * - Otherwise, enable the traps 225 * 226 * On guest exit: 227 * - If the dirty bit is set, save guest registers, restore host 228 * registers and clear the dirty bit. This ensure that the host can 229 * now use the debug registers. 230 */ 231 static bool trap_debug_regs(struct kvm_vcpu *vcpu, 232 const struct sys_reg_params *p, 233 const struct sys_reg_desc *r) 234 { 235 if (p->is_write) { 236 vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt); 237 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY; 238 } else { 239 *vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg); 240 } 241 242 return true; 243 } 244 245 static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) 246 { 247 u64 amair; 248 249 asm volatile("mrs %0, amair_el1\n" : "=r" (amair)); 250 vcpu_sys_reg(vcpu, AMAIR_EL1) = amair; 251 } 252 253 static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) 254 { 255 /* 256 * Simply map the vcpu_id into the Aff0 field of the MPIDR. 257 */ 258 vcpu_sys_reg(vcpu, MPIDR_EL1) = (1UL << 31) | (vcpu->vcpu_id & 0xff); 259 } 260 261 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */ 262 #define DBG_BCR_BVR_WCR_WVR_EL1(n) \ 263 /* DBGBVRn_EL1 */ \ 264 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b100), \ 265 trap_debug_regs, reset_val, (DBGBVR0_EL1 + (n)), 0 }, \ 266 /* DBGBCRn_EL1 */ \ 267 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b101), \ 268 trap_debug_regs, reset_val, (DBGBCR0_EL1 + (n)), 0 }, \ 269 /* DBGWVRn_EL1 */ \ 270 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b110), \ 271 trap_debug_regs, reset_val, (DBGWVR0_EL1 + (n)), 0 }, \ 272 /* DBGWCRn_EL1 */ \ 273 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b111), \ 274 trap_debug_regs, reset_val, (DBGWCR0_EL1 + (n)), 0 } 275 276 /* 277 * Architected system registers. 278 * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2 279 * 280 * We could trap ID_DFR0 and tell the guest we don't support performance 281 * monitoring. Unfortunately the patch to make the kernel check ID_DFR0 was 282 * NAKed, so it will read the PMCR anyway. 283 * 284 * Therefore we tell the guest we have 0 counters. Unfortunately, we 285 * must always support PMCCNTR (the cycle counter): we just RAZ/WI for 286 * all PM registers, which doesn't crash the guest kernel at least. 287 * 288 * Debug handling: We do trap most, if not all debug related system 289 * registers. The implementation is good enough to ensure that a guest 290 * can use these with minimal performance degradation. The drawback is 291 * that we don't implement any of the external debug, none of the 292 * OSlock protocol. This should be revisited if we ever encounter a 293 * more demanding guest... 294 */ 295 static const struct sys_reg_desc sys_reg_descs[] = { 296 /* DC ISW */ 297 { Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b0110), Op2(0b010), 298 access_dcsw }, 299 /* DC CSW */ 300 { Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b1010), Op2(0b010), 301 access_dcsw }, 302 /* DC CISW */ 303 { Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b1110), Op2(0b010), 304 access_dcsw }, 305 306 DBG_BCR_BVR_WCR_WVR_EL1(0), 307 DBG_BCR_BVR_WCR_WVR_EL1(1), 308 /* MDCCINT_EL1 */ 309 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b000), 310 trap_debug_regs, reset_val, MDCCINT_EL1, 0 }, 311 /* MDSCR_EL1 */ 312 { Op0(0b10), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b010), 313 trap_debug_regs, reset_val, MDSCR_EL1, 0 }, 314 DBG_BCR_BVR_WCR_WVR_EL1(2), 315 DBG_BCR_BVR_WCR_WVR_EL1(3), 316 DBG_BCR_BVR_WCR_WVR_EL1(4), 317 DBG_BCR_BVR_WCR_WVR_EL1(5), 318 DBG_BCR_BVR_WCR_WVR_EL1(6), 319 DBG_BCR_BVR_WCR_WVR_EL1(7), 320 DBG_BCR_BVR_WCR_WVR_EL1(8), 321 DBG_BCR_BVR_WCR_WVR_EL1(9), 322 DBG_BCR_BVR_WCR_WVR_EL1(10), 323 DBG_BCR_BVR_WCR_WVR_EL1(11), 324 DBG_BCR_BVR_WCR_WVR_EL1(12), 325 DBG_BCR_BVR_WCR_WVR_EL1(13), 326 DBG_BCR_BVR_WCR_WVR_EL1(14), 327 DBG_BCR_BVR_WCR_WVR_EL1(15), 328 329 /* MDRAR_EL1 */ 330 { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000), 331 trap_raz_wi }, 332 /* OSLAR_EL1 */ 333 { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b100), 334 trap_raz_wi }, 335 /* OSLSR_EL1 */ 336 { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0001), Op2(0b100), 337 trap_oslsr_el1 }, 338 /* OSDLR_EL1 */ 339 { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0011), Op2(0b100), 340 trap_raz_wi }, 341 /* DBGPRCR_EL1 */ 342 { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0100), Op2(0b100), 343 trap_raz_wi }, 344 /* DBGCLAIMSET_EL1 */ 345 { Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1000), Op2(0b110), 346 trap_raz_wi }, 347 /* DBGCLAIMCLR_EL1 */ 348 { Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1001), Op2(0b110), 349 trap_raz_wi }, 350 /* DBGAUTHSTATUS_EL1 */ 351 { Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1110), Op2(0b110), 352 trap_dbgauthstatus_el1 }, 353 354 /* TEECR32_EL1 */ 355 { Op0(0b10), Op1(0b010), CRn(0b0000), CRm(0b0000), Op2(0b000), 356 NULL, reset_val, TEECR32_EL1, 0 }, 357 /* TEEHBR32_EL1 */ 358 { Op0(0b10), Op1(0b010), CRn(0b0001), CRm(0b0000), Op2(0b000), 359 NULL, reset_val, TEEHBR32_EL1, 0 }, 360 361 /* MDCCSR_EL1 */ 362 { Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0001), Op2(0b000), 363 trap_raz_wi }, 364 /* DBGDTR_EL0 */ 365 { Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0100), Op2(0b000), 366 trap_raz_wi }, 367 /* DBGDTR[TR]X_EL0 */ 368 { Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0101), Op2(0b000), 369 trap_raz_wi }, 370 371 /* DBGVCR32_EL2 */ 372 { Op0(0b10), Op1(0b100), CRn(0b0000), CRm(0b0111), Op2(0b000), 373 NULL, reset_val, DBGVCR32_EL2, 0 }, 374 375 /* MPIDR_EL1 */ 376 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b101), 377 NULL, reset_mpidr, MPIDR_EL1 }, 378 /* SCTLR_EL1 */ 379 { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000), 380 access_sctlr, reset_val, SCTLR_EL1, 0x00C50078 }, 381 /* CPACR_EL1 */ 382 { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b010), 383 NULL, reset_val, CPACR_EL1, 0 }, 384 /* TTBR0_EL1 */ 385 { Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b000), 386 access_vm_reg, reset_unknown, TTBR0_EL1 }, 387 /* TTBR1_EL1 */ 388 { Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b001), 389 access_vm_reg, reset_unknown, TTBR1_EL1 }, 390 /* TCR_EL1 */ 391 { Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b010), 392 access_vm_reg, reset_val, TCR_EL1, 0 }, 393 394 /* AFSR0_EL1 */ 395 { Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0001), Op2(0b000), 396 access_vm_reg, reset_unknown, AFSR0_EL1 }, 397 /* AFSR1_EL1 */ 398 { Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0001), Op2(0b001), 399 access_vm_reg, reset_unknown, AFSR1_EL1 }, 400 /* ESR_EL1 */ 401 { Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0010), Op2(0b000), 402 access_vm_reg, reset_unknown, ESR_EL1 }, 403 /* FAR_EL1 */ 404 { Op0(0b11), Op1(0b000), CRn(0b0110), CRm(0b0000), Op2(0b000), 405 access_vm_reg, reset_unknown, FAR_EL1 }, 406 /* PAR_EL1 */ 407 { Op0(0b11), Op1(0b000), CRn(0b0111), CRm(0b0100), Op2(0b000), 408 NULL, reset_unknown, PAR_EL1 }, 409 410 /* PMINTENSET_EL1 */ 411 { Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b001), 412 trap_raz_wi }, 413 /* PMINTENCLR_EL1 */ 414 { Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b010), 415 trap_raz_wi }, 416 417 /* MAIR_EL1 */ 418 { Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0010), Op2(0b000), 419 access_vm_reg, reset_unknown, MAIR_EL1 }, 420 /* AMAIR_EL1 */ 421 { Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0011), Op2(0b000), 422 access_vm_reg, reset_amair_el1, AMAIR_EL1 }, 423 424 /* VBAR_EL1 */ 425 { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b0000), Op2(0b000), 426 NULL, reset_val, VBAR_EL1, 0 }, 427 /* CONTEXTIDR_EL1 */ 428 { Op0(0b11), Op1(0b000), CRn(0b1101), CRm(0b0000), Op2(0b001), 429 access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 }, 430 /* TPIDR_EL1 */ 431 { Op0(0b11), Op1(0b000), CRn(0b1101), CRm(0b0000), Op2(0b100), 432 NULL, reset_unknown, TPIDR_EL1 }, 433 434 /* CNTKCTL_EL1 */ 435 { Op0(0b11), Op1(0b000), CRn(0b1110), CRm(0b0001), Op2(0b000), 436 NULL, reset_val, CNTKCTL_EL1, 0}, 437 438 /* CSSELR_EL1 */ 439 { Op0(0b11), Op1(0b010), CRn(0b0000), CRm(0b0000), Op2(0b000), 440 NULL, reset_unknown, CSSELR_EL1 }, 441 442 /* PMCR_EL0 */ 443 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b000), 444 trap_raz_wi }, 445 /* PMCNTENSET_EL0 */ 446 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b001), 447 trap_raz_wi }, 448 /* PMCNTENCLR_EL0 */ 449 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b010), 450 trap_raz_wi }, 451 /* PMOVSCLR_EL0 */ 452 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b011), 453 trap_raz_wi }, 454 /* PMSWINC_EL0 */ 455 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b100), 456 trap_raz_wi }, 457 /* PMSELR_EL0 */ 458 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b101), 459 trap_raz_wi }, 460 /* PMCEID0_EL0 */ 461 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b110), 462 trap_raz_wi }, 463 /* PMCEID1_EL0 */ 464 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b111), 465 trap_raz_wi }, 466 /* PMCCNTR_EL0 */ 467 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b000), 468 trap_raz_wi }, 469 /* PMXEVTYPER_EL0 */ 470 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b001), 471 trap_raz_wi }, 472 /* PMXEVCNTR_EL0 */ 473 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b010), 474 trap_raz_wi }, 475 /* PMUSERENR_EL0 */ 476 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b000), 477 trap_raz_wi }, 478 /* PMOVSSET_EL0 */ 479 { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b011), 480 trap_raz_wi }, 481 482 /* TPIDR_EL0 */ 483 { Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b010), 484 NULL, reset_unknown, TPIDR_EL0 }, 485 /* TPIDRRO_EL0 */ 486 { Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b011), 487 NULL, reset_unknown, TPIDRRO_EL0 }, 488 489 /* DACR32_EL2 */ 490 { Op0(0b11), Op1(0b100), CRn(0b0011), CRm(0b0000), Op2(0b000), 491 NULL, reset_unknown, DACR32_EL2 }, 492 /* IFSR32_EL2 */ 493 { Op0(0b11), Op1(0b100), CRn(0b0101), CRm(0b0000), Op2(0b001), 494 NULL, reset_unknown, IFSR32_EL2 }, 495 /* FPEXC32_EL2 */ 496 { Op0(0b11), Op1(0b100), CRn(0b0101), CRm(0b0011), Op2(0b000), 497 NULL, reset_val, FPEXC32_EL2, 0x70 }, 498 }; 499 500 static bool trap_dbgidr(struct kvm_vcpu *vcpu, 501 const struct sys_reg_params *p, 502 const struct sys_reg_desc *r) 503 { 504 if (p->is_write) { 505 return ignore_write(vcpu, p); 506 } else { 507 u64 dfr = read_cpuid(ID_AA64DFR0_EL1); 508 u64 pfr = read_cpuid(ID_AA64PFR0_EL1); 509 u32 el3 = !!((pfr >> 12) & 0xf); 510 511 *vcpu_reg(vcpu, p->Rt) = ((((dfr >> 20) & 0xf) << 28) | 512 (((dfr >> 12) & 0xf) << 24) | 513 (((dfr >> 28) & 0xf) << 20) | 514 (6 << 16) | (el3 << 14) | (el3 << 12)); 515 return true; 516 } 517 } 518 519 static bool trap_debug32(struct kvm_vcpu *vcpu, 520 const struct sys_reg_params *p, 521 const struct sys_reg_desc *r) 522 { 523 if (p->is_write) { 524 vcpu_cp14(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt); 525 vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY; 526 } else { 527 *vcpu_reg(vcpu, p->Rt) = vcpu_cp14(vcpu, r->reg); 528 } 529 530 return true; 531 } 532 533 #define DBG_BCR_BVR_WCR_WVR(n) \ 534 /* DBGBVRn */ \ 535 { Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_debug32, \ 536 NULL, (cp14_DBGBVR0 + (n) * 2) }, \ 537 /* DBGBCRn */ \ 538 { Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_debug32, \ 539 NULL, (cp14_DBGBCR0 + (n) * 2) }, \ 540 /* DBGWVRn */ \ 541 { Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_debug32, \ 542 NULL, (cp14_DBGWVR0 + (n) * 2) }, \ 543 /* DBGWCRn */ \ 544 { Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_debug32, \ 545 NULL, (cp14_DBGWCR0 + (n) * 2) } 546 547 #define DBGBXVR(n) \ 548 { Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_debug32, \ 549 NULL, cp14_DBGBXVR0 + n * 2 } 550 551 /* 552 * Trapped cp14 registers. We generally ignore most of the external 553 * debug, on the principle that they don't really make sense to a 554 * guest. Revisit this one day, whould this principle change. 555 */ 556 static const struct sys_reg_desc cp14_regs[] = { 557 /* DBGIDR */ 558 { Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgidr }, 559 /* DBGDTRRXext */ 560 { Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi }, 561 562 DBG_BCR_BVR_WCR_WVR(0), 563 /* DBGDSCRint */ 564 { Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi }, 565 DBG_BCR_BVR_WCR_WVR(1), 566 /* DBGDCCINT */ 567 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug32 }, 568 /* DBGDSCRext */ 569 { Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug32 }, 570 DBG_BCR_BVR_WCR_WVR(2), 571 /* DBGDTR[RT]Xint */ 572 { Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi }, 573 /* DBGDTR[RT]Xext */ 574 { Op1( 0), CRn( 0), CRm( 3), Op2( 2), trap_raz_wi }, 575 DBG_BCR_BVR_WCR_WVR(3), 576 DBG_BCR_BVR_WCR_WVR(4), 577 DBG_BCR_BVR_WCR_WVR(5), 578 /* DBGWFAR */ 579 { Op1( 0), CRn( 0), CRm( 6), Op2( 0), trap_raz_wi }, 580 /* DBGOSECCR */ 581 { Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi }, 582 DBG_BCR_BVR_WCR_WVR(6), 583 /* DBGVCR */ 584 { Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug32 }, 585 DBG_BCR_BVR_WCR_WVR(7), 586 DBG_BCR_BVR_WCR_WVR(8), 587 DBG_BCR_BVR_WCR_WVR(9), 588 DBG_BCR_BVR_WCR_WVR(10), 589 DBG_BCR_BVR_WCR_WVR(11), 590 DBG_BCR_BVR_WCR_WVR(12), 591 DBG_BCR_BVR_WCR_WVR(13), 592 DBG_BCR_BVR_WCR_WVR(14), 593 DBG_BCR_BVR_WCR_WVR(15), 594 595 /* DBGDRAR (32bit) */ 596 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), trap_raz_wi }, 597 598 DBGBXVR(0), 599 /* DBGOSLAR */ 600 { Op1( 0), CRn( 1), CRm( 0), Op2( 4), trap_raz_wi }, 601 DBGBXVR(1), 602 /* DBGOSLSR */ 603 { Op1( 0), CRn( 1), CRm( 1), Op2( 4), trap_oslsr_el1 }, 604 DBGBXVR(2), 605 DBGBXVR(3), 606 /* DBGOSDLR */ 607 { Op1( 0), CRn( 1), CRm( 3), Op2( 4), trap_raz_wi }, 608 DBGBXVR(4), 609 /* DBGPRCR */ 610 { Op1( 0), CRn( 1), CRm( 4), Op2( 4), trap_raz_wi }, 611 DBGBXVR(5), 612 DBGBXVR(6), 613 DBGBXVR(7), 614 DBGBXVR(8), 615 DBGBXVR(9), 616 DBGBXVR(10), 617 DBGBXVR(11), 618 DBGBXVR(12), 619 DBGBXVR(13), 620 DBGBXVR(14), 621 DBGBXVR(15), 622 623 /* DBGDSAR (32bit) */ 624 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), trap_raz_wi }, 625 626 /* DBGDEVID2 */ 627 { Op1( 0), CRn( 7), CRm( 0), Op2( 7), trap_raz_wi }, 628 /* DBGDEVID1 */ 629 { Op1( 0), CRn( 7), CRm( 1), Op2( 7), trap_raz_wi }, 630 /* DBGDEVID */ 631 { Op1( 0), CRn( 7), CRm( 2), Op2( 7), trap_raz_wi }, 632 /* DBGCLAIMSET */ 633 { Op1( 0), CRn( 7), CRm( 8), Op2( 6), trap_raz_wi }, 634 /* DBGCLAIMCLR */ 635 { Op1( 0), CRn( 7), CRm( 9), Op2( 6), trap_raz_wi }, 636 /* DBGAUTHSTATUS */ 637 { Op1( 0), CRn( 7), CRm(14), Op2( 6), trap_dbgauthstatus_el1 }, 638 }; 639 640 /* Trapped cp14 64bit registers */ 641 static const struct sys_reg_desc cp14_64_regs[] = { 642 /* DBGDRAR (64bit) */ 643 { Op1( 0), CRm( 1), .access = trap_raz_wi }, 644 645 /* DBGDSAR (64bit) */ 646 { Op1( 0), CRm( 2), .access = trap_raz_wi }, 647 }; 648 649 /* 650 * Trapped cp15 registers. TTBR0/TTBR1 get a double encoding, 651 * depending on the way they are accessed (as a 32bit or a 64bit 652 * register). 653 */ 654 static const struct sys_reg_desc cp15_regs[] = { 655 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_sctlr, NULL, c1_SCTLR }, 656 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 }, 657 { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 }, 658 { Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, c2_TTBCR }, 659 { Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, c3_DACR }, 660 { Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, c5_DFSR }, 661 { Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, c5_IFSR }, 662 { Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, c5_ADFSR }, 663 { Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, c5_AIFSR }, 664 { Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, c6_DFAR }, 665 { Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, c6_IFAR }, 666 667 /* 668 * DC{C,I,CI}SW operations: 669 */ 670 { Op1( 0), CRn( 7), CRm( 6), Op2( 2), access_dcsw }, 671 { Op1( 0), CRn( 7), CRm(10), Op2( 2), access_dcsw }, 672 { Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw }, 673 674 /* PMU */ 675 { Op1( 0), CRn( 9), CRm(12), Op2( 0), trap_raz_wi }, 676 { Op1( 0), CRn( 9), CRm(12), Op2( 1), trap_raz_wi }, 677 { Op1( 0), CRn( 9), CRm(12), Op2( 2), trap_raz_wi }, 678 { Op1( 0), CRn( 9), CRm(12), Op2( 3), trap_raz_wi }, 679 { Op1( 0), CRn( 9), CRm(12), Op2( 5), trap_raz_wi }, 680 { Op1( 0), CRn( 9), CRm(12), Op2( 6), trap_raz_wi }, 681 { Op1( 0), CRn( 9), CRm(12), Op2( 7), trap_raz_wi }, 682 { Op1( 0), CRn( 9), CRm(13), Op2( 0), trap_raz_wi }, 683 { Op1( 0), CRn( 9), CRm(13), Op2( 1), trap_raz_wi }, 684 { Op1( 0), CRn( 9), CRm(13), Op2( 2), trap_raz_wi }, 685 { Op1( 0), CRn( 9), CRm(14), Op2( 0), trap_raz_wi }, 686 { Op1( 0), CRn( 9), CRm(14), Op2( 1), trap_raz_wi }, 687 { Op1( 0), CRn( 9), CRm(14), Op2( 2), trap_raz_wi }, 688 689 { Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, c10_PRRR }, 690 { Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, c10_NMRR }, 691 { Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, c10_AMAIR0 }, 692 { Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, c10_AMAIR1 }, 693 { Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, c13_CID }, 694 }; 695 696 static const struct sys_reg_desc cp15_64_regs[] = { 697 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR0 }, 698 { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR1 }, 699 }; 700 701 /* Target specific emulation tables */ 702 static struct kvm_sys_reg_target_table *target_tables[KVM_ARM_NUM_TARGETS]; 703 704 void kvm_register_target_sys_reg_table(unsigned int target, 705 struct kvm_sys_reg_target_table *table) 706 { 707 target_tables[target] = table; 708 } 709 710 /* Get specific register table for this target. */ 711 static const struct sys_reg_desc *get_target_table(unsigned target, 712 bool mode_is_64, 713 size_t *num) 714 { 715 struct kvm_sys_reg_target_table *table; 716 717 table = target_tables[target]; 718 if (mode_is_64) { 719 *num = table->table64.num; 720 return table->table64.table; 721 } else { 722 *num = table->table32.num; 723 return table->table32.table; 724 } 725 } 726 727 static const struct sys_reg_desc *find_reg(const struct sys_reg_params *params, 728 const struct sys_reg_desc table[], 729 unsigned int num) 730 { 731 unsigned int i; 732 733 for (i = 0; i < num; i++) { 734 const struct sys_reg_desc *r = &table[i]; 735 736 if (params->Op0 != r->Op0) 737 continue; 738 if (params->Op1 != r->Op1) 739 continue; 740 if (params->CRn != r->CRn) 741 continue; 742 if (params->CRm != r->CRm) 743 continue; 744 if (params->Op2 != r->Op2) 745 continue; 746 747 return r; 748 } 749 return NULL; 750 } 751 752 int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run) 753 { 754 kvm_inject_undefined(vcpu); 755 return 1; 756 } 757 758 /* 759 * emulate_cp -- tries to match a sys_reg access in a handling table, and 760 * call the corresponding trap handler. 761 * 762 * @params: pointer to the descriptor of the access 763 * @table: array of trap descriptors 764 * @num: size of the trap descriptor array 765 * 766 * Return 0 if the access has been handled, and -1 if not. 767 */ 768 static int emulate_cp(struct kvm_vcpu *vcpu, 769 const struct sys_reg_params *params, 770 const struct sys_reg_desc *table, 771 size_t num) 772 { 773 const struct sys_reg_desc *r; 774 775 if (!table) 776 return -1; /* Not handled */ 777 778 r = find_reg(params, table, num); 779 780 if (r) { 781 /* 782 * Not having an accessor means that we have 783 * configured a trap that we don't know how to 784 * handle. This certainly qualifies as a gross bug 785 * that should be fixed right away. 786 */ 787 BUG_ON(!r->access); 788 789 if (likely(r->access(vcpu, params, r))) { 790 /* Skip instruction, since it was emulated */ 791 kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); 792 } 793 794 /* Handled */ 795 return 0; 796 } 797 798 /* Not handled */ 799 return -1; 800 } 801 802 static void unhandled_cp_access(struct kvm_vcpu *vcpu, 803 struct sys_reg_params *params) 804 { 805 u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu); 806 int cp; 807 808 switch(hsr_ec) { 809 case ESR_EL2_EC_CP15_32: 810 case ESR_EL2_EC_CP15_64: 811 cp = 15; 812 break; 813 case ESR_EL2_EC_CP14_MR: 814 case ESR_EL2_EC_CP14_64: 815 cp = 14; 816 break; 817 default: 818 WARN_ON((cp = -1)); 819 } 820 821 kvm_err("Unsupported guest CP%d access at: %08lx\n", 822 cp, *vcpu_pc(vcpu)); 823 print_sys_reg_instr(params); 824 kvm_inject_undefined(vcpu); 825 } 826 827 /** 828 * kvm_handle_cp_64 -- handles a mrrc/mcrr trap on a guest CP15 access 829 * @vcpu: The VCPU pointer 830 * @run: The kvm_run struct 831 */ 832 static int kvm_handle_cp_64(struct kvm_vcpu *vcpu, 833 const struct sys_reg_desc *global, 834 size_t nr_global, 835 const struct sys_reg_desc *target_specific, 836 size_t nr_specific) 837 { 838 struct sys_reg_params params; 839 u32 hsr = kvm_vcpu_get_hsr(vcpu); 840 int Rt2 = (hsr >> 10) & 0xf; 841 842 params.is_aarch32 = true; 843 params.is_32bit = false; 844 params.CRm = (hsr >> 1) & 0xf; 845 params.Rt = (hsr >> 5) & 0xf; 846 params.is_write = ((hsr & 1) == 0); 847 848 params.Op0 = 0; 849 params.Op1 = (hsr >> 16) & 0xf; 850 params.Op2 = 0; 851 params.CRn = 0; 852 853 /* 854 * Massive hack here. Store Rt2 in the top 32bits so we only 855 * have one register to deal with. As we use the same trap 856 * backends between AArch32 and AArch64, we get away with it. 857 */ 858 if (params.is_write) { 859 u64 val = *vcpu_reg(vcpu, params.Rt); 860 val &= 0xffffffff; 861 val |= *vcpu_reg(vcpu, Rt2) << 32; 862 *vcpu_reg(vcpu, params.Rt) = val; 863 } 864 865 if (!emulate_cp(vcpu, ¶ms, target_specific, nr_specific)) 866 goto out; 867 if (!emulate_cp(vcpu, ¶ms, global, nr_global)) 868 goto out; 869 870 unhandled_cp_access(vcpu, ¶ms); 871 872 out: 873 /* Do the opposite hack for the read side */ 874 if (!params.is_write) { 875 u64 val = *vcpu_reg(vcpu, params.Rt); 876 val >>= 32; 877 *vcpu_reg(vcpu, Rt2) = val; 878 } 879 880 return 1; 881 } 882 883 /** 884 * kvm_handle_cp15_32 -- handles a mrc/mcr trap on a guest CP15 access 885 * @vcpu: The VCPU pointer 886 * @run: The kvm_run struct 887 */ 888 static int kvm_handle_cp_32(struct kvm_vcpu *vcpu, 889 const struct sys_reg_desc *global, 890 size_t nr_global, 891 const struct sys_reg_desc *target_specific, 892 size_t nr_specific) 893 { 894 struct sys_reg_params params; 895 u32 hsr = kvm_vcpu_get_hsr(vcpu); 896 897 params.is_aarch32 = true; 898 params.is_32bit = true; 899 params.CRm = (hsr >> 1) & 0xf; 900 params.Rt = (hsr >> 5) & 0xf; 901 params.is_write = ((hsr & 1) == 0); 902 params.CRn = (hsr >> 10) & 0xf; 903 params.Op0 = 0; 904 params.Op1 = (hsr >> 14) & 0x7; 905 params.Op2 = (hsr >> 17) & 0x7; 906 907 if (!emulate_cp(vcpu, ¶ms, target_specific, nr_specific)) 908 return 1; 909 if (!emulate_cp(vcpu, ¶ms, global, nr_global)) 910 return 1; 911 912 unhandled_cp_access(vcpu, ¶ms); 913 return 1; 914 } 915 916 int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run) 917 { 918 const struct sys_reg_desc *target_specific; 919 size_t num; 920 921 target_specific = get_target_table(vcpu->arch.target, false, &num); 922 return kvm_handle_cp_64(vcpu, 923 cp15_64_regs, ARRAY_SIZE(cp15_64_regs), 924 target_specific, num); 925 } 926 927 int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run) 928 { 929 const struct sys_reg_desc *target_specific; 930 size_t num; 931 932 target_specific = get_target_table(vcpu->arch.target, false, &num); 933 return kvm_handle_cp_32(vcpu, 934 cp15_regs, ARRAY_SIZE(cp15_regs), 935 target_specific, num); 936 } 937 938 int kvm_handle_cp14_64(struct kvm_vcpu *vcpu, struct kvm_run *run) 939 { 940 return kvm_handle_cp_64(vcpu, 941 cp14_64_regs, ARRAY_SIZE(cp14_64_regs), 942 NULL, 0); 943 } 944 945 int kvm_handle_cp14_32(struct kvm_vcpu *vcpu, struct kvm_run *run) 946 { 947 return kvm_handle_cp_32(vcpu, 948 cp14_regs, ARRAY_SIZE(cp14_regs), 949 NULL, 0); 950 } 951 952 static int emulate_sys_reg(struct kvm_vcpu *vcpu, 953 const struct sys_reg_params *params) 954 { 955 size_t num; 956 const struct sys_reg_desc *table, *r; 957 958 table = get_target_table(vcpu->arch.target, true, &num); 959 960 /* Search target-specific then generic table. */ 961 r = find_reg(params, table, num); 962 if (!r) 963 r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs)); 964 965 if (likely(r)) { 966 /* 967 * Not having an accessor means that we have 968 * configured a trap that we don't know how to 969 * handle. This certainly qualifies as a gross bug 970 * that should be fixed right away. 971 */ 972 BUG_ON(!r->access); 973 974 if (likely(r->access(vcpu, params, r))) { 975 /* Skip instruction, since it was emulated */ 976 kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); 977 return 1; 978 } 979 /* If access function fails, it should complain. */ 980 } else { 981 kvm_err("Unsupported guest sys_reg access at: %lx\n", 982 *vcpu_pc(vcpu)); 983 print_sys_reg_instr(params); 984 } 985 kvm_inject_undefined(vcpu); 986 return 1; 987 } 988 989 static void reset_sys_reg_descs(struct kvm_vcpu *vcpu, 990 const struct sys_reg_desc *table, size_t num) 991 { 992 unsigned long i; 993 994 for (i = 0; i < num; i++) 995 if (table[i].reset) 996 table[i].reset(vcpu, &table[i]); 997 } 998 999 /** 1000 * kvm_handle_sys_reg -- handles a mrs/msr trap on a guest sys_reg access 1001 * @vcpu: The VCPU pointer 1002 * @run: The kvm_run struct 1003 */ 1004 int kvm_handle_sys_reg(struct kvm_vcpu *vcpu, struct kvm_run *run) 1005 { 1006 struct sys_reg_params params; 1007 unsigned long esr = kvm_vcpu_get_hsr(vcpu); 1008 1009 params.is_aarch32 = false; 1010 params.is_32bit = false; 1011 params.Op0 = (esr >> 20) & 3; 1012 params.Op1 = (esr >> 14) & 0x7; 1013 params.CRn = (esr >> 10) & 0xf; 1014 params.CRm = (esr >> 1) & 0xf; 1015 params.Op2 = (esr >> 17) & 0x7; 1016 params.Rt = (esr >> 5) & 0x1f; 1017 params.is_write = !(esr & 1); 1018 1019 return emulate_sys_reg(vcpu, ¶ms); 1020 } 1021 1022 /****************************************************************************** 1023 * Userspace API 1024 *****************************************************************************/ 1025 1026 static bool index_to_params(u64 id, struct sys_reg_params *params) 1027 { 1028 switch (id & KVM_REG_SIZE_MASK) { 1029 case KVM_REG_SIZE_U64: 1030 /* Any unused index bits means it's not valid. */ 1031 if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK 1032 | KVM_REG_ARM_COPROC_MASK 1033 | KVM_REG_ARM64_SYSREG_OP0_MASK 1034 | KVM_REG_ARM64_SYSREG_OP1_MASK 1035 | KVM_REG_ARM64_SYSREG_CRN_MASK 1036 | KVM_REG_ARM64_SYSREG_CRM_MASK 1037 | KVM_REG_ARM64_SYSREG_OP2_MASK)) 1038 return false; 1039 params->Op0 = ((id & KVM_REG_ARM64_SYSREG_OP0_MASK) 1040 >> KVM_REG_ARM64_SYSREG_OP0_SHIFT); 1041 params->Op1 = ((id & KVM_REG_ARM64_SYSREG_OP1_MASK) 1042 >> KVM_REG_ARM64_SYSREG_OP1_SHIFT); 1043 params->CRn = ((id & KVM_REG_ARM64_SYSREG_CRN_MASK) 1044 >> KVM_REG_ARM64_SYSREG_CRN_SHIFT); 1045 params->CRm = ((id & KVM_REG_ARM64_SYSREG_CRM_MASK) 1046 >> KVM_REG_ARM64_SYSREG_CRM_SHIFT); 1047 params->Op2 = ((id & KVM_REG_ARM64_SYSREG_OP2_MASK) 1048 >> KVM_REG_ARM64_SYSREG_OP2_SHIFT); 1049 return true; 1050 default: 1051 return false; 1052 } 1053 } 1054 1055 /* Decode an index value, and find the sys_reg_desc entry. */ 1056 static const struct sys_reg_desc *index_to_sys_reg_desc(struct kvm_vcpu *vcpu, 1057 u64 id) 1058 { 1059 size_t num; 1060 const struct sys_reg_desc *table, *r; 1061 struct sys_reg_params params; 1062 1063 /* We only do sys_reg for now. */ 1064 if ((id & KVM_REG_ARM_COPROC_MASK) != KVM_REG_ARM64_SYSREG) 1065 return NULL; 1066 1067 if (!index_to_params(id, ¶ms)) 1068 return NULL; 1069 1070 table = get_target_table(vcpu->arch.target, true, &num); 1071 r = find_reg(¶ms, table, num); 1072 if (!r) 1073 r = find_reg(¶ms, sys_reg_descs, ARRAY_SIZE(sys_reg_descs)); 1074 1075 /* Not saved in the sys_reg array? */ 1076 if (r && !r->reg) 1077 r = NULL; 1078 1079 return r; 1080 } 1081 1082 /* 1083 * These are the invariant sys_reg registers: we let the guest see the 1084 * host versions of these, so they're part of the guest state. 1085 * 1086 * A future CPU may provide a mechanism to present different values to 1087 * the guest, or a future kvm may trap them. 1088 */ 1089 1090 #define FUNCTION_INVARIANT(reg) \ 1091 static void get_##reg(struct kvm_vcpu *v, \ 1092 const struct sys_reg_desc *r) \ 1093 { \ 1094 u64 val; \ 1095 \ 1096 asm volatile("mrs %0, " __stringify(reg) "\n" \ 1097 : "=r" (val)); \ 1098 ((struct sys_reg_desc *)r)->val = val; \ 1099 } 1100 1101 FUNCTION_INVARIANT(midr_el1) 1102 FUNCTION_INVARIANT(ctr_el0) 1103 FUNCTION_INVARIANT(revidr_el1) 1104 FUNCTION_INVARIANT(id_pfr0_el1) 1105 FUNCTION_INVARIANT(id_pfr1_el1) 1106 FUNCTION_INVARIANT(id_dfr0_el1) 1107 FUNCTION_INVARIANT(id_afr0_el1) 1108 FUNCTION_INVARIANT(id_mmfr0_el1) 1109 FUNCTION_INVARIANT(id_mmfr1_el1) 1110 FUNCTION_INVARIANT(id_mmfr2_el1) 1111 FUNCTION_INVARIANT(id_mmfr3_el1) 1112 FUNCTION_INVARIANT(id_isar0_el1) 1113 FUNCTION_INVARIANT(id_isar1_el1) 1114 FUNCTION_INVARIANT(id_isar2_el1) 1115 FUNCTION_INVARIANT(id_isar3_el1) 1116 FUNCTION_INVARIANT(id_isar4_el1) 1117 FUNCTION_INVARIANT(id_isar5_el1) 1118 FUNCTION_INVARIANT(clidr_el1) 1119 FUNCTION_INVARIANT(aidr_el1) 1120 1121 /* ->val is filled in by kvm_sys_reg_table_init() */ 1122 static struct sys_reg_desc invariant_sys_regs[] = { 1123 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b000), 1124 NULL, get_midr_el1 }, 1125 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b110), 1126 NULL, get_revidr_el1 }, 1127 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b000), 1128 NULL, get_id_pfr0_el1 }, 1129 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b001), 1130 NULL, get_id_pfr1_el1 }, 1131 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b010), 1132 NULL, get_id_dfr0_el1 }, 1133 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b011), 1134 NULL, get_id_afr0_el1 }, 1135 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b100), 1136 NULL, get_id_mmfr0_el1 }, 1137 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b101), 1138 NULL, get_id_mmfr1_el1 }, 1139 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b110), 1140 NULL, get_id_mmfr2_el1 }, 1141 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b111), 1142 NULL, get_id_mmfr3_el1 }, 1143 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b000), 1144 NULL, get_id_isar0_el1 }, 1145 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b001), 1146 NULL, get_id_isar1_el1 }, 1147 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b010), 1148 NULL, get_id_isar2_el1 }, 1149 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b011), 1150 NULL, get_id_isar3_el1 }, 1151 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b100), 1152 NULL, get_id_isar4_el1 }, 1153 { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b101), 1154 NULL, get_id_isar5_el1 }, 1155 { Op0(0b11), Op1(0b001), CRn(0b0000), CRm(0b0000), Op2(0b001), 1156 NULL, get_clidr_el1 }, 1157 { Op0(0b11), Op1(0b001), CRn(0b0000), CRm(0b0000), Op2(0b111), 1158 NULL, get_aidr_el1 }, 1159 { Op0(0b11), Op1(0b011), CRn(0b0000), CRm(0b0000), Op2(0b001), 1160 NULL, get_ctr_el0 }, 1161 }; 1162 1163 static int reg_from_user(u64 *val, const void __user *uaddr, u64 id) 1164 { 1165 if (copy_from_user(val, uaddr, KVM_REG_SIZE(id)) != 0) 1166 return -EFAULT; 1167 return 0; 1168 } 1169 1170 static int reg_to_user(void __user *uaddr, const u64 *val, u64 id) 1171 { 1172 if (copy_to_user(uaddr, val, KVM_REG_SIZE(id)) != 0) 1173 return -EFAULT; 1174 return 0; 1175 } 1176 1177 static int get_invariant_sys_reg(u64 id, void __user *uaddr) 1178 { 1179 struct sys_reg_params params; 1180 const struct sys_reg_desc *r; 1181 1182 if (!index_to_params(id, ¶ms)) 1183 return -ENOENT; 1184 1185 r = find_reg(¶ms, invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs)); 1186 if (!r) 1187 return -ENOENT; 1188 1189 return reg_to_user(uaddr, &r->val, id); 1190 } 1191 1192 static int set_invariant_sys_reg(u64 id, void __user *uaddr) 1193 { 1194 struct sys_reg_params params; 1195 const struct sys_reg_desc *r; 1196 int err; 1197 u64 val = 0; /* Make sure high bits are 0 for 32-bit regs */ 1198 1199 if (!index_to_params(id, ¶ms)) 1200 return -ENOENT; 1201 r = find_reg(¶ms, invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs)); 1202 if (!r) 1203 return -ENOENT; 1204 1205 err = reg_from_user(&val, uaddr, id); 1206 if (err) 1207 return err; 1208 1209 /* This is what we mean by invariant: you can't change it. */ 1210 if (r->val != val) 1211 return -EINVAL; 1212 1213 return 0; 1214 } 1215 1216 static bool is_valid_cache(u32 val) 1217 { 1218 u32 level, ctype; 1219 1220 if (val >= CSSELR_MAX) 1221 return -ENOENT; 1222 1223 /* Bottom bit is Instruction or Data bit. Next 3 bits are level. */ 1224 level = (val >> 1); 1225 ctype = (cache_levels >> (level * 3)) & 7; 1226 1227 switch (ctype) { 1228 case 0: /* No cache */ 1229 return false; 1230 case 1: /* Instruction cache only */ 1231 return (val & 1); 1232 case 2: /* Data cache only */ 1233 case 4: /* Unified cache */ 1234 return !(val & 1); 1235 case 3: /* Separate instruction and data caches */ 1236 return true; 1237 default: /* Reserved: we can't know instruction or data. */ 1238 return false; 1239 } 1240 } 1241 1242 static int demux_c15_get(u64 id, void __user *uaddr) 1243 { 1244 u32 val; 1245 u32 __user *uval = uaddr; 1246 1247 /* Fail if we have unknown bits set. */ 1248 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK 1249 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1))) 1250 return -ENOENT; 1251 1252 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) { 1253 case KVM_REG_ARM_DEMUX_ID_CCSIDR: 1254 if (KVM_REG_SIZE(id) != 4) 1255 return -ENOENT; 1256 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK) 1257 >> KVM_REG_ARM_DEMUX_VAL_SHIFT; 1258 if (!is_valid_cache(val)) 1259 return -ENOENT; 1260 1261 return put_user(get_ccsidr(val), uval); 1262 default: 1263 return -ENOENT; 1264 } 1265 } 1266 1267 static int demux_c15_set(u64 id, void __user *uaddr) 1268 { 1269 u32 val, newval; 1270 u32 __user *uval = uaddr; 1271 1272 /* Fail if we have unknown bits set. */ 1273 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK 1274 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1))) 1275 return -ENOENT; 1276 1277 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) { 1278 case KVM_REG_ARM_DEMUX_ID_CCSIDR: 1279 if (KVM_REG_SIZE(id) != 4) 1280 return -ENOENT; 1281 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK) 1282 >> KVM_REG_ARM_DEMUX_VAL_SHIFT; 1283 if (!is_valid_cache(val)) 1284 return -ENOENT; 1285 1286 if (get_user(newval, uval)) 1287 return -EFAULT; 1288 1289 /* This is also invariant: you can't change it. */ 1290 if (newval != get_ccsidr(val)) 1291 return -EINVAL; 1292 return 0; 1293 default: 1294 return -ENOENT; 1295 } 1296 } 1297 1298 int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 1299 { 1300 const struct sys_reg_desc *r; 1301 void __user *uaddr = (void __user *)(unsigned long)reg->addr; 1302 1303 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX) 1304 return demux_c15_get(reg->id, uaddr); 1305 1306 if (KVM_REG_SIZE(reg->id) != sizeof(__u64)) 1307 return -ENOENT; 1308 1309 r = index_to_sys_reg_desc(vcpu, reg->id); 1310 if (!r) 1311 return get_invariant_sys_reg(reg->id, uaddr); 1312 1313 return reg_to_user(uaddr, &vcpu_sys_reg(vcpu, r->reg), reg->id); 1314 } 1315 1316 int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 1317 { 1318 const struct sys_reg_desc *r; 1319 void __user *uaddr = (void __user *)(unsigned long)reg->addr; 1320 1321 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX) 1322 return demux_c15_set(reg->id, uaddr); 1323 1324 if (KVM_REG_SIZE(reg->id) != sizeof(__u64)) 1325 return -ENOENT; 1326 1327 r = index_to_sys_reg_desc(vcpu, reg->id); 1328 if (!r) 1329 return set_invariant_sys_reg(reg->id, uaddr); 1330 1331 return reg_from_user(&vcpu_sys_reg(vcpu, r->reg), uaddr, reg->id); 1332 } 1333 1334 static unsigned int num_demux_regs(void) 1335 { 1336 unsigned int i, count = 0; 1337 1338 for (i = 0; i < CSSELR_MAX; i++) 1339 if (is_valid_cache(i)) 1340 count++; 1341 1342 return count; 1343 } 1344 1345 static int write_demux_regids(u64 __user *uindices) 1346 { 1347 u64 val = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX; 1348 unsigned int i; 1349 1350 val |= KVM_REG_ARM_DEMUX_ID_CCSIDR; 1351 for (i = 0; i < CSSELR_MAX; i++) { 1352 if (!is_valid_cache(i)) 1353 continue; 1354 if (put_user(val | i, uindices)) 1355 return -EFAULT; 1356 uindices++; 1357 } 1358 return 0; 1359 } 1360 1361 static u64 sys_reg_to_index(const struct sys_reg_desc *reg) 1362 { 1363 return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | 1364 KVM_REG_ARM64_SYSREG | 1365 (reg->Op0 << KVM_REG_ARM64_SYSREG_OP0_SHIFT) | 1366 (reg->Op1 << KVM_REG_ARM64_SYSREG_OP1_SHIFT) | 1367 (reg->CRn << KVM_REG_ARM64_SYSREG_CRN_SHIFT) | 1368 (reg->CRm << KVM_REG_ARM64_SYSREG_CRM_SHIFT) | 1369 (reg->Op2 << KVM_REG_ARM64_SYSREG_OP2_SHIFT)); 1370 } 1371 1372 static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind) 1373 { 1374 if (!*uind) 1375 return true; 1376 1377 if (put_user(sys_reg_to_index(reg), *uind)) 1378 return false; 1379 1380 (*uind)++; 1381 return true; 1382 } 1383 1384 /* Assumed ordered tables, see kvm_sys_reg_table_init. */ 1385 static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind) 1386 { 1387 const struct sys_reg_desc *i1, *i2, *end1, *end2; 1388 unsigned int total = 0; 1389 size_t num; 1390 1391 /* We check for duplicates here, to allow arch-specific overrides. */ 1392 i1 = get_target_table(vcpu->arch.target, true, &num); 1393 end1 = i1 + num; 1394 i2 = sys_reg_descs; 1395 end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs); 1396 1397 BUG_ON(i1 == end1 || i2 == end2); 1398 1399 /* Walk carefully, as both tables may refer to the same register. */ 1400 while (i1 || i2) { 1401 int cmp = cmp_sys_reg(i1, i2); 1402 /* target-specific overrides generic entry. */ 1403 if (cmp <= 0) { 1404 /* Ignore registers we trap but don't save. */ 1405 if (i1->reg) { 1406 if (!copy_reg_to_user(i1, &uind)) 1407 return -EFAULT; 1408 total++; 1409 } 1410 } else { 1411 /* Ignore registers we trap but don't save. */ 1412 if (i2->reg) { 1413 if (!copy_reg_to_user(i2, &uind)) 1414 return -EFAULT; 1415 total++; 1416 } 1417 } 1418 1419 if (cmp <= 0 && ++i1 == end1) 1420 i1 = NULL; 1421 if (cmp >= 0 && ++i2 == end2) 1422 i2 = NULL; 1423 } 1424 return total; 1425 } 1426 1427 unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu) 1428 { 1429 return ARRAY_SIZE(invariant_sys_regs) 1430 + num_demux_regs() 1431 + walk_sys_regs(vcpu, (u64 __user *)NULL); 1432 } 1433 1434 int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) 1435 { 1436 unsigned int i; 1437 int err; 1438 1439 /* Then give them all the invariant registers' indices. */ 1440 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) { 1441 if (put_user(sys_reg_to_index(&invariant_sys_regs[i]), uindices)) 1442 return -EFAULT; 1443 uindices++; 1444 } 1445 1446 err = walk_sys_regs(vcpu, uindices); 1447 if (err < 0) 1448 return err; 1449 uindices += err; 1450 1451 return write_demux_regids(uindices); 1452 } 1453 1454 static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n) 1455 { 1456 unsigned int i; 1457 1458 for (i = 1; i < n; i++) { 1459 if (cmp_sys_reg(&table[i-1], &table[i]) >= 0) { 1460 kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1); 1461 return 1; 1462 } 1463 } 1464 1465 return 0; 1466 } 1467 1468 void kvm_sys_reg_table_init(void) 1469 { 1470 unsigned int i; 1471 struct sys_reg_desc clidr; 1472 1473 /* Make sure tables are unique and in order. */ 1474 BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs))); 1475 BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs))); 1476 BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs))); 1477 BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs))); 1478 BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs))); 1479 BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs))); 1480 1481 /* We abuse the reset function to overwrite the table itself. */ 1482 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) 1483 invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]); 1484 1485 /* 1486 * CLIDR format is awkward, so clean it up. See ARM B4.1.20: 1487 * 1488 * If software reads the Cache Type fields from Ctype1 1489 * upwards, once it has seen a value of 0b000, no caches 1490 * exist at further-out levels of the hierarchy. So, for 1491 * example, if Ctype3 is the first Cache Type field with a 1492 * value of 0b000, the values of Ctype4 to Ctype7 must be 1493 * ignored. 1494 */ 1495 get_clidr_el1(NULL, &clidr); /* Ugly... */ 1496 cache_levels = clidr.val; 1497 for (i = 0; i < 7; i++) 1498 if (((cache_levels >> (i*3)) & 7) == 0) 1499 break; 1500 /* Clear all higher bits. */ 1501 cache_levels &= (1 << (i*3))-1; 1502 } 1503 1504 /** 1505 * kvm_reset_sys_regs - sets system registers to reset value 1506 * @vcpu: The VCPU pointer 1507 * 1508 * This function finds the right table above and sets the registers on the 1509 * virtual CPU struct to their architecturally defined reset values. 1510 */ 1511 void kvm_reset_sys_regs(struct kvm_vcpu *vcpu) 1512 { 1513 size_t num; 1514 const struct sys_reg_desc *table; 1515 1516 /* Catch someone adding a register without putting in reset entry. */ 1517 memset(&vcpu->arch.ctxt.sys_regs, 0x42, sizeof(vcpu->arch.ctxt.sys_regs)); 1518 1519 /* Generic chip reset first (so target could override). */ 1520 reset_sys_reg_descs(vcpu, sys_reg_descs, ARRAY_SIZE(sys_reg_descs)); 1521 1522 table = get_target_table(vcpu->arch.target, true, &num); 1523 reset_sys_reg_descs(vcpu, table, num); 1524 1525 for (num = 1; num < NR_SYS_REGS; num++) 1526 if (vcpu_sys_reg(vcpu, num) == 0x4242424242424242) 1527 panic("Didn't reset vcpu_sys_reg(%zi)", num); 1528 } 1529