1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * KVM/MIPS: Support for hardware virtualization extensions 7 * 8 * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. 9 * Authors: Yann Le Du <ledu@kymasys.com> 10 */ 11 12 #include <linux/errno.h> 13 #include <linux/err.h> 14 #include <linux/module.h> 15 #include <linux/preempt.h> 16 #include <linux/vmalloc.h> 17 #include <asm/cacheflush.h> 18 #include <asm/cacheops.h> 19 #include <asm/cmpxchg.h> 20 #include <asm/fpu.h> 21 #include <asm/hazards.h> 22 #include <asm/inst.h> 23 #include <asm/mmu_context.h> 24 #include <asm/r4kcache.h> 25 #include <asm/time.h> 26 #include <asm/tlb.h> 27 #include <asm/tlbex.h> 28 29 #include <linux/kvm_host.h> 30 31 #include "interrupt.h" 32 33 #include "trace.h" 34 35 /* Pointers to last VCPU loaded on each physical CPU */ 36 static struct kvm_vcpu *last_vcpu[NR_CPUS]; 37 /* Pointers to last VCPU executed on each physical CPU */ 38 static struct kvm_vcpu *last_exec_vcpu[NR_CPUS]; 39 40 /* 41 * Number of guest VTLB entries to use, so we can catch inconsistency between 42 * CPUs. 43 */ 44 static unsigned int kvm_vz_guest_vtlb_size; 45 46 static inline long kvm_vz_read_gc0_ebase(void) 47 { 48 if (sizeof(long) == 8 && cpu_has_ebase_wg) 49 return read_gc0_ebase_64(); 50 else 51 return read_gc0_ebase(); 52 } 53 54 static inline void kvm_vz_write_gc0_ebase(long v) 55 { 56 /* 57 * First write with WG=1 to write upper bits, then write again in case 58 * WG should be left at 0. 59 * write_gc0_ebase_64() is no longer UNDEFINED since R6. 60 */ 61 if (sizeof(long) == 8 && 62 (cpu_has_mips64r6 || cpu_has_ebase_wg)) { 63 write_gc0_ebase_64(v | MIPS_EBASE_WG); 64 write_gc0_ebase_64(v); 65 } else { 66 write_gc0_ebase(v | MIPS_EBASE_WG); 67 write_gc0_ebase(v); 68 } 69 } 70 71 /* 72 * These Config bits may be writable by the guest: 73 * Config: [K23, KU] (!TLB), K0 74 * Config1: (none) 75 * Config2: [TU, SU] (impl) 76 * Config3: ISAOnExc 77 * Config4: FTLBPageSize 78 * Config5: K, CV, MSAEn, UFE, FRE, SBRI, UFR 79 */ 80 81 static inline unsigned int kvm_vz_config_guest_wrmask(struct kvm_vcpu *vcpu) 82 { 83 return CONF_CM_CMASK; 84 } 85 86 static inline unsigned int kvm_vz_config1_guest_wrmask(struct kvm_vcpu *vcpu) 87 { 88 return 0; 89 } 90 91 static inline unsigned int kvm_vz_config2_guest_wrmask(struct kvm_vcpu *vcpu) 92 { 93 return 0; 94 } 95 96 static inline unsigned int kvm_vz_config3_guest_wrmask(struct kvm_vcpu *vcpu) 97 { 98 return MIPS_CONF3_ISA_OE; 99 } 100 101 static inline unsigned int kvm_vz_config4_guest_wrmask(struct kvm_vcpu *vcpu) 102 { 103 /* no need to be exact */ 104 return MIPS_CONF4_VFTLBPAGESIZE; 105 } 106 107 static inline unsigned int kvm_vz_config5_guest_wrmask(struct kvm_vcpu *vcpu) 108 { 109 unsigned int mask = MIPS_CONF5_K | MIPS_CONF5_CV | MIPS_CONF5_SBRI; 110 111 /* Permit MSAEn changes if MSA supported and enabled */ 112 if (kvm_mips_guest_has_msa(&vcpu->arch)) 113 mask |= MIPS_CONF5_MSAEN; 114 115 /* 116 * Permit guest FPU mode changes if FPU is enabled and the relevant 117 * feature exists according to FIR register. 118 */ 119 if (kvm_mips_guest_has_fpu(&vcpu->arch)) { 120 if (cpu_has_ufr) 121 mask |= MIPS_CONF5_UFR; 122 if (cpu_has_fre) 123 mask |= MIPS_CONF5_FRE | MIPS_CONF5_UFE; 124 } 125 126 return mask; 127 } 128 129 /* 130 * VZ optionally allows these additional Config bits to be written by root: 131 * Config: M, [MT] 132 * Config1: M, [MMUSize-1, C2, MD, PC, WR, CA], FP 133 * Config2: M 134 * Config3: M, MSAP, [BPG], ULRI, [DSP2P, DSPP], CTXTC, [ITL, LPA, VEIC, 135 * VInt, SP, CDMM, MT, SM, TL] 136 * Config4: M, [VTLBSizeExt, MMUSizeExt] 137 * Config5: [MRP] 138 */ 139 140 static inline unsigned int kvm_vz_config_user_wrmask(struct kvm_vcpu *vcpu) 141 { 142 return kvm_vz_config_guest_wrmask(vcpu) | MIPS_CONF_M; 143 } 144 145 static inline unsigned int kvm_vz_config1_user_wrmask(struct kvm_vcpu *vcpu) 146 { 147 unsigned int mask = kvm_vz_config1_guest_wrmask(vcpu) | MIPS_CONF_M; 148 149 /* Permit FPU to be present if FPU is supported */ 150 if (kvm_mips_guest_can_have_fpu(&vcpu->arch)) 151 mask |= MIPS_CONF1_FP; 152 153 return mask; 154 } 155 156 static inline unsigned int kvm_vz_config2_user_wrmask(struct kvm_vcpu *vcpu) 157 { 158 return kvm_vz_config2_guest_wrmask(vcpu) | MIPS_CONF_M; 159 } 160 161 static inline unsigned int kvm_vz_config3_user_wrmask(struct kvm_vcpu *vcpu) 162 { 163 unsigned int mask = kvm_vz_config3_guest_wrmask(vcpu) | MIPS_CONF_M | 164 MIPS_CONF3_ULRI | MIPS_CONF3_CTXTC; 165 166 /* Permit MSA to be present if MSA is supported */ 167 if (kvm_mips_guest_can_have_msa(&vcpu->arch)) 168 mask |= MIPS_CONF3_MSA; 169 170 return mask; 171 } 172 173 static inline unsigned int kvm_vz_config4_user_wrmask(struct kvm_vcpu *vcpu) 174 { 175 return kvm_vz_config4_guest_wrmask(vcpu) | MIPS_CONF_M; 176 } 177 178 static inline unsigned int kvm_vz_config5_user_wrmask(struct kvm_vcpu *vcpu) 179 { 180 return kvm_vz_config5_guest_wrmask(vcpu); 181 } 182 183 static gpa_t kvm_vz_gva_to_gpa_cb(gva_t gva) 184 { 185 /* VZ guest has already converted gva to gpa */ 186 return gva; 187 } 188 189 static void kvm_vz_queue_irq(struct kvm_vcpu *vcpu, unsigned int priority) 190 { 191 set_bit(priority, &vcpu->arch.pending_exceptions); 192 clear_bit(priority, &vcpu->arch.pending_exceptions_clr); 193 } 194 195 static void kvm_vz_dequeue_irq(struct kvm_vcpu *vcpu, unsigned int priority) 196 { 197 clear_bit(priority, &vcpu->arch.pending_exceptions); 198 set_bit(priority, &vcpu->arch.pending_exceptions_clr); 199 } 200 201 static void kvm_vz_queue_timer_int_cb(struct kvm_vcpu *vcpu) 202 { 203 /* 204 * timer expiry is asynchronous to vcpu execution therefore defer guest 205 * cp0 accesses 206 */ 207 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER); 208 } 209 210 static void kvm_vz_dequeue_timer_int_cb(struct kvm_vcpu *vcpu) 211 { 212 /* 213 * timer expiry is asynchronous to vcpu execution therefore defer guest 214 * cp0 accesses 215 */ 216 kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_TIMER); 217 } 218 219 static void kvm_vz_queue_io_int_cb(struct kvm_vcpu *vcpu, 220 struct kvm_mips_interrupt *irq) 221 { 222 int intr = (int)irq->irq; 223 224 /* 225 * interrupts are asynchronous to vcpu execution therefore defer guest 226 * cp0 accesses 227 */ 228 switch (intr) { 229 case 2: 230 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_IO); 231 break; 232 233 case 3: 234 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_IPI_1); 235 break; 236 237 case 4: 238 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_IPI_2); 239 break; 240 241 default: 242 break; 243 } 244 245 } 246 247 static void kvm_vz_dequeue_io_int_cb(struct kvm_vcpu *vcpu, 248 struct kvm_mips_interrupt *irq) 249 { 250 int intr = (int)irq->irq; 251 252 /* 253 * interrupts are asynchronous to vcpu execution therefore defer guest 254 * cp0 accesses 255 */ 256 switch (intr) { 257 case -2: 258 kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_IO); 259 break; 260 261 case -3: 262 kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_IPI_1); 263 break; 264 265 case -4: 266 kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_IPI_2); 267 break; 268 269 default: 270 break; 271 } 272 273 } 274 275 static u32 kvm_vz_priority_to_irq[MIPS_EXC_MAX] = { 276 [MIPS_EXC_INT_TIMER] = C_IRQ5, 277 [MIPS_EXC_INT_IO] = C_IRQ0, 278 [MIPS_EXC_INT_IPI_1] = C_IRQ1, 279 [MIPS_EXC_INT_IPI_2] = C_IRQ2, 280 }; 281 282 static int kvm_vz_irq_deliver_cb(struct kvm_vcpu *vcpu, unsigned int priority, 283 u32 cause) 284 { 285 u32 irq = (priority < MIPS_EXC_MAX) ? 286 kvm_vz_priority_to_irq[priority] : 0; 287 288 switch (priority) { 289 case MIPS_EXC_INT_TIMER: 290 set_gc0_cause(C_TI); 291 break; 292 293 case MIPS_EXC_INT_IO: 294 case MIPS_EXC_INT_IPI_1: 295 case MIPS_EXC_INT_IPI_2: 296 if (cpu_has_guestctl2) 297 set_c0_guestctl2(irq); 298 else 299 set_gc0_cause(irq); 300 break; 301 302 default: 303 break; 304 } 305 306 clear_bit(priority, &vcpu->arch.pending_exceptions); 307 return 1; 308 } 309 310 static int kvm_vz_irq_clear_cb(struct kvm_vcpu *vcpu, unsigned int priority, 311 u32 cause) 312 { 313 u32 irq = (priority < MIPS_EXC_MAX) ? 314 kvm_vz_priority_to_irq[priority] : 0; 315 316 switch (priority) { 317 case MIPS_EXC_INT_TIMER: 318 /* 319 * Call to kvm_write_c0_guest_compare() clears Cause.TI in 320 * kvm_mips_emulate_CP0(). Explicitly clear irq associated with 321 * Cause.IP[IPTI] if GuestCtl2 virtual interrupt register not 322 * supported or if not using GuestCtl2 Hardware Clear. 323 */ 324 if (cpu_has_guestctl2) { 325 if (!(read_c0_guestctl2() & (irq << 14))) 326 clear_c0_guestctl2(irq); 327 } else { 328 clear_gc0_cause(irq); 329 } 330 break; 331 332 case MIPS_EXC_INT_IO: 333 case MIPS_EXC_INT_IPI_1: 334 case MIPS_EXC_INT_IPI_2: 335 /* Clear GuestCtl2.VIP irq if not using Hardware Clear */ 336 if (cpu_has_guestctl2) { 337 if (!(read_c0_guestctl2() & (irq << 14))) 338 clear_c0_guestctl2(irq); 339 } else { 340 clear_gc0_cause(irq); 341 } 342 break; 343 344 default: 345 break; 346 } 347 348 clear_bit(priority, &vcpu->arch.pending_exceptions_clr); 349 return 1; 350 } 351 352 /* 353 * VZ guest timer handling. 354 */ 355 356 /** 357 * _kvm_vz_restore_stimer() - Restore soft timer state. 358 * @vcpu: Virtual CPU. 359 * @compare: CP0_Compare register value, restored by caller. 360 * @cause: CP0_Cause register to restore. 361 * 362 * Restore VZ state relating to the soft timer. 363 */ 364 static void _kvm_vz_restore_stimer(struct kvm_vcpu *vcpu, u32 compare, 365 u32 cause) 366 { 367 /* 368 * Avoid spurious counter interrupts by setting Guest CP0_Count to just 369 * after Guest CP0_Compare. 370 */ 371 write_c0_gtoffset(compare - read_c0_count()); 372 373 back_to_back_c0_hazard(); 374 write_gc0_cause(cause); 375 } 376 377 /** 378 * kvm_vz_restore_timer() - Restore guest timer state. 379 * @vcpu: Virtual CPU. 380 * 381 * Restore soft timer state from saved context. 382 */ 383 static void kvm_vz_restore_timer(struct kvm_vcpu *vcpu) 384 { 385 struct mips_coproc *cop0 = vcpu->arch.cop0; 386 u32 cause, compare; 387 388 compare = kvm_read_sw_gc0_compare(cop0); 389 cause = kvm_read_sw_gc0_cause(cop0); 390 391 write_gc0_compare(compare); 392 _kvm_vz_restore_stimer(vcpu, compare, cause); 393 } 394 395 /** 396 * kvm_vz_save_timer() - Save guest timer state. 397 * @vcpu: Virtual CPU. 398 * 399 * Save VZ guest timer state. 400 */ 401 static void kvm_vz_save_timer(struct kvm_vcpu *vcpu) 402 { 403 struct mips_coproc *cop0 = vcpu->arch.cop0; 404 u32 compare, cause; 405 406 compare = read_gc0_compare(); 407 cause = read_gc0_cause(); 408 409 /* save timer-related state to VCPU context */ 410 kvm_write_sw_gc0_cause(cop0, cause); 411 kvm_write_sw_gc0_compare(cop0, compare); 412 } 413 414 /** 415 * is_eva_access() - Find whether an instruction is an EVA memory accessor. 416 * @inst: 32-bit instruction encoding. 417 * 418 * Finds whether @inst encodes an EVA memory access instruction, which would 419 * indicate that emulation of it should access the user mode address space 420 * instead of the kernel mode address space. This matters for MUSUK segments 421 * which are TLB mapped for user mode but unmapped for kernel mode. 422 * 423 * Returns: Whether @inst encodes an EVA accessor instruction. 424 */ 425 static bool is_eva_access(union mips_instruction inst) 426 { 427 if (inst.spec3_format.opcode != spec3_op) 428 return false; 429 430 switch (inst.spec3_format.func) { 431 case lwle_op: 432 case lwre_op: 433 case cachee_op: 434 case sbe_op: 435 case she_op: 436 case sce_op: 437 case swe_op: 438 case swle_op: 439 case swre_op: 440 case prefe_op: 441 case lbue_op: 442 case lhue_op: 443 case lbe_op: 444 case lhe_op: 445 case lle_op: 446 case lwe_op: 447 return true; 448 default: 449 return false; 450 } 451 } 452 453 /** 454 * is_eva_am_mapped() - Find whether an access mode is mapped. 455 * @vcpu: KVM VCPU state. 456 * @am: 3-bit encoded access mode. 457 * @eu: Segment becomes unmapped and uncached when Status.ERL=1. 458 * 459 * Decode @am to find whether it encodes a mapped segment for the current VCPU 460 * state. Where necessary @eu and the actual instruction causing the fault are 461 * taken into account to make the decision. 462 * 463 * Returns: Whether the VCPU faulted on a TLB mapped address. 464 */ 465 static bool is_eva_am_mapped(struct kvm_vcpu *vcpu, unsigned int am, bool eu) 466 { 467 u32 am_lookup; 468 int err; 469 470 /* 471 * Interpret access control mode. We assume address errors will already 472 * have been caught by the guest, leaving us with: 473 * AM UM SM KM 31..24 23..16 474 * UK 0 000 Unm 0 0 475 * MK 1 001 TLB 1 476 * MSK 2 010 TLB TLB 1 477 * MUSK 3 011 TLB TLB TLB 1 478 * MUSUK 4 100 TLB TLB Unm 0 1 479 * USK 5 101 Unm Unm 0 0 480 * - 6 110 0 0 481 * UUSK 7 111 Unm Unm Unm 0 0 482 * 483 * We shift a magic value by AM across the sign bit to find if always 484 * TLB mapped, and if not shift by 8 again to find if it depends on KM. 485 */ 486 am_lookup = 0x70080000 << am; 487 if ((s32)am_lookup < 0) { 488 /* 489 * MK, MSK, MUSK 490 * Always TLB mapped, unless SegCtl.EU && ERL 491 */ 492 if (!eu || !(read_gc0_status() & ST0_ERL)) 493 return true; 494 } else { 495 am_lookup <<= 8; 496 if ((s32)am_lookup < 0) { 497 union mips_instruction inst; 498 unsigned int status; 499 u32 *opc; 500 501 /* 502 * MUSUK 503 * TLB mapped if not in kernel mode 504 */ 505 status = read_gc0_status(); 506 if (!(status & (ST0_EXL | ST0_ERL)) && 507 (status & ST0_KSU)) 508 return true; 509 /* 510 * EVA access instructions in kernel 511 * mode access user address space. 512 */ 513 opc = (u32 *)vcpu->arch.pc; 514 if (vcpu->arch.host_cp0_cause & CAUSEF_BD) 515 opc += 1; 516 err = kvm_get_badinstr(opc, vcpu, &inst.word); 517 if (!err && is_eva_access(inst)) 518 return true; 519 } 520 } 521 522 return false; 523 } 524 525 /** 526 * kvm_vz_gva_to_gpa() - Convert valid GVA to GPA. 527 * @vcpu: KVM VCPU state. 528 * @gva: Guest virtual address to convert. 529 * @gpa: Output guest physical address. 530 * 531 * Convert a guest virtual address (GVA) which is valid according to the guest 532 * context, to a guest physical address (GPA). 533 * 534 * Returns: 0 on success. 535 * -errno on failure. 536 */ 537 static int kvm_vz_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, 538 unsigned long *gpa) 539 { 540 u32 gva32 = gva; 541 unsigned long segctl; 542 543 if ((long)gva == (s32)gva32) { 544 /* Handle canonical 32-bit virtual address */ 545 if (cpu_guest_has_segments) { 546 unsigned long mask, pa; 547 548 switch (gva32 >> 29) { 549 case 0: 550 case 1: /* CFG5 (1GB) */ 551 segctl = read_gc0_segctl2() >> 16; 552 mask = (unsigned long)0xfc0000000ull; 553 break; 554 case 2: 555 case 3: /* CFG4 (1GB) */ 556 segctl = read_gc0_segctl2(); 557 mask = (unsigned long)0xfc0000000ull; 558 break; 559 case 4: /* CFG3 (512MB) */ 560 segctl = read_gc0_segctl1() >> 16; 561 mask = (unsigned long)0xfe0000000ull; 562 break; 563 case 5: /* CFG2 (512MB) */ 564 segctl = read_gc0_segctl1(); 565 mask = (unsigned long)0xfe0000000ull; 566 break; 567 case 6: /* CFG1 (512MB) */ 568 segctl = read_gc0_segctl0() >> 16; 569 mask = (unsigned long)0xfe0000000ull; 570 break; 571 case 7: /* CFG0 (512MB) */ 572 segctl = read_gc0_segctl0(); 573 mask = (unsigned long)0xfe0000000ull; 574 break; 575 default: 576 /* 577 * GCC 4.9 isn't smart enough to figure out that 578 * segctl and mask are always initialised. 579 */ 580 unreachable(); 581 } 582 583 if (is_eva_am_mapped(vcpu, (segctl >> 4) & 0x7, 584 segctl & 0x0008)) 585 goto tlb_mapped; 586 587 /* Unmapped, find guest physical address */ 588 pa = (segctl << 20) & mask; 589 pa |= gva32 & ~mask; 590 *gpa = pa; 591 return 0; 592 } else if ((s32)gva32 < (s32)0xc0000000) { 593 /* legacy unmapped KSeg0 or KSeg1 */ 594 *gpa = gva32 & 0x1fffffff; 595 return 0; 596 } 597 #ifdef CONFIG_64BIT 598 } else if ((gva & 0xc000000000000000) == 0x8000000000000000) { 599 /* XKPHYS */ 600 if (cpu_guest_has_segments) { 601 /* 602 * Each of the 8 regions can be overridden by SegCtl2.XR 603 * to use SegCtl1.XAM. 604 */ 605 segctl = read_gc0_segctl2(); 606 if (segctl & (1ull << (56 + ((gva >> 59) & 0x7)))) { 607 segctl = read_gc0_segctl1(); 608 if (is_eva_am_mapped(vcpu, (segctl >> 59) & 0x7, 609 0)) 610 goto tlb_mapped; 611 } 612 613 } 614 /* 615 * Traditionally fully unmapped. 616 * Bits 61:59 specify the CCA, which we can just mask off here. 617 * Bits 58:PABITS should be zero, but we shouldn't have got here 618 * if it wasn't. 619 */ 620 *gpa = gva & 0x07ffffffffffffff; 621 return 0; 622 #endif 623 } 624 625 tlb_mapped: 626 return kvm_vz_guest_tlb_lookup(vcpu, gva, gpa); 627 } 628 629 /** 630 * kvm_vz_badvaddr_to_gpa() - Convert GVA BadVAddr from root exception to GPA. 631 * @vcpu: KVM VCPU state. 632 * @badvaddr: Root BadVAddr. 633 * @gpa: Output guest physical address. 634 * 635 * VZ implementations are permitted to report guest virtual addresses (GVA) in 636 * BadVAddr on a root exception during guest execution, instead of the more 637 * convenient guest physical addresses (GPA). When we get a GVA, this function 638 * converts it to a GPA, taking into account guest segmentation and guest TLB 639 * state. 640 * 641 * Returns: 0 on success. 642 * -errno on failure. 643 */ 644 static int kvm_vz_badvaddr_to_gpa(struct kvm_vcpu *vcpu, unsigned long badvaddr, 645 unsigned long *gpa) 646 { 647 unsigned int gexccode = (vcpu->arch.host_cp0_guestctl0 & 648 MIPS_GCTL0_GEXC) >> MIPS_GCTL0_GEXC_SHIFT; 649 650 /* If BadVAddr is GPA, then all is well in the world */ 651 if (likely(gexccode == MIPS_GCTL0_GEXC_GPA)) { 652 *gpa = badvaddr; 653 return 0; 654 } 655 656 /* Otherwise we'd expect it to be GVA ... */ 657 if (WARN(gexccode != MIPS_GCTL0_GEXC_GVA, 658 "Unexpected gexccode %#x\n", gexccode)) 659 return -EINVAL; 660 661 /* ... and we need to perform the GVA->GPA translation in software */ 662 return kvm_vz_gva_to_gpa(vcpu, badvaddr, gpa); 663 } 664 665 static int kvm_trap_vz_no_handler(struct kvm_vcpu *vcpu) 666 { 667 u32 *opc = (u32 *) vcpu->arch.pc; 668 u32 cause = vcpu->arch.host_cp0_cause; 669 u32 exccode = (cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE; 670 unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr; 671 u32 inst = 0; 672 673 /* 674 * Fetch the instruction. 675 */ 676 if (cause & CAUSEF_BD) 677 opc += 1; 678 kvm_get_badinstr(opc, vcpu, &inst); 679 680 kvm_err("Exception Code: %d not handled @ PC: %p, inst: 0x%08x BadVaddr: %#lx Status: %#x\n", 681 exccode, opc, inst, badvaddr, 682 read_gc0_status()); 683 kvm_arch_vcpu_dump_regs(vcpu); 684 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 685 return RESUME_HOST; 686 } 687 688 static enum emulation_result kvm_vz_gpsi_cop0(union mips_instruction inst, 689 u32 *opc, u32 cause, 690 struct kvm_run *run, 691 struct kvm_vcpu *vcpu) 692 { 693 struct mips_coproc *cop0 = vcpu->arch.cop0; 694 enum emulation_result er = EMULATE_DONE; 695 u32 rt, rd, sel; 696 unsigned long curr_pc; 697 unsigned long val; 698 699 /* 700 * Update PC and hold onto current PC in case there is 701 * an error and we want to rollback the PC 702 */ 703 curr_pc = vcpu->arch.pc; 704 er = update_pc(vcpu, cause); 705 if (er == EMULATE_FAIL) 706 return er; 707 708 if (inst.co_format.co) { 709 switch (inst.co_format.func) { 710 case wait_op: 711 er = kvm_mips_emul_wait(vcpu); 712 break; 713 default: 714 er = EMULATE_FAIL; 715 } 716 } else { 717 rt = inst.c0r_format.rt; 718 rd = inst.c0r_format.rd; 719 sel = inst.c0r_format.sel; 720 721 switch (inst.c0r_format.rs) { 722 case dmfc_op: 723 case mfc_op: 724 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS 725 cop0->stat[rd][sel]++; 726 #endif 727 if (rd == MIPS_CP0_COUNT && 728 sel == 0) { /* Count */ 729 val = kvm_mips_read_count(vcpu); 730 } else if (rd == MIPS_CP0_COMPARE && 731 sel == 0) { /* Compare */ 732 val = read_gc0_compare(); 733 } else if (rd == MIPS_CP0_LLADDR && 734 sel == 0) { /* LLAddr */ 735 if (cpu_guest_has_rw_llb) 736 val = read_gc0_lladdr() & 737 MIPS_LLADDR_LLB; 738 else 739 val = 0; 740 } else if ((rd == MIPS_CP0_PRID && 741 (sel == 0 || /* PRid */ 742 sel == 2 || /* CDMMBase */ 743 sel == 3)) || /* CMGCRBase */ 744 (rd == MIPS_CP0_STATUS && 745 (sel == 2 || /* SRSCtl */ 746 sel == 3)) || /* SRSMap */ 747 (rd == MIPS_CP0_CONFIG && 748 (sel == 7)) || /* Config7 */ 749 (rd == MIPS_CP0_ERRCTL && 750 (sel == 0))) { /* ErrCtl */ 751 val = cop0->reg[rd][sel]; 752 } else { 753 val = 0; 754 er = EMULATE_FAIL; 755 } 756 757 if (er != EMULATE_FAIL) { 758 /* Sign extend */ 759 if (inst.c0r_format.rs == mfc_op) 760 val = (int)val; 761 vcpu->arch.gprs[rt] = val; 762 } 763 764 trace_kvm_hwr(vcpu, (inst.c0r_format.rs == mfc_op) ? 765 KVM_TRACE_MFC0 : KVM_TRACE_DMFC0, 766 KVM_TRACE_COP0(rd, sel), val); 767 break; 768 769 case dmtc_op: 770 case mtc_op: 771 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS 772 cop0->stat[rd][sel]++; 773 #endif 774 val = vcpu->arch.gprs[rt]; 775 trace_kvm_hwr(vcpu, (inst.c0r_format.rs == mtc_op) ? 776 KVM_TRACE_MTC0 : KVM_TRACE_DMTC0, 777 KVM_TRACE_COP0(rd, sel), val); 778 779 if (rd == MIPS_CP0_COUNT && 780 sel == 0) { /* Count */ 781 kvm_mips_write_count(vcpu, vcpu->arch.gprs[rt]); 782 } else if (rd == MIPS_CP0_COMPARE && 783 sel == 0) { /* Compare */ 784 kvm_mips_write_compare(vcpu, 785 vcpu->arch.gprs[rt], 786 true); 787 } else if (rd == MIPS_CP0_LLADDR && 788 sel == 0) { /* LLAddr */ 789 /* 790 * P5600 generates GPSI on guest MTC0 LLAddr. 791 * Only allow the guest to clear LLB. 792 */ 793 if (cpu_guest_has_rw_llb && 794 !(val & MIPS_LLADDR_LLB)) 795 write_gc0_lladdr(0); 796 } else if (rd == MIPS_CP0_ERRCTL && 797 (sel == 0)) { /* ErrCtl */ 798 /* ignore the written value */ 799 } else { 800 er = EMULATE_FAIL; 801 } 802 break; 803 804 default: 805 er = EMULATE_FAIL; 806 break; 807 } 808 } 809 /* Rollback PC only if emulation was unsuccessful */ 810 if (er == EMULATE_FAIL) { 811 kvm_err("[%#lx]%s: unsupported cop0 instruction 0x%08x\n", 812 curr_pc, __func__, inst.word); 813 814 vcpu->arch.pc = curr_pc; 815 } 816 817 return er; 818 } 819 820 static enum emulation_result kvm_vz_gpsi_cache(union mips_instruction inst, 821 u32 *opc, u32 cause, 822 struct kvm_run *run, 823 struct kvm_vcpu *vcpu) 824 { 825 enum emulation_result er = EMULATE_DONE; 826 u32 cache, op_inst, op, base; 827 s16 offset; 828 struct kvm_vcpu_arch *arch = &vcpu->arch; 829 unsigned long va, curr_pc; 830 831 /* 832 * Update PC and hold onto current PC in case there is 833 * an error and we want to rollback the PC 834 */ 835 curr_pc = vcpu->arch.pc; 836 er = update_pc(vcpu, cause); 837 if (er == EMULATE_FAIL) 838 return er; 839 840 base = inst.i_format.rs; 841 op_inst = inst.i_format.rt; 842 if (cpu_has_mips_r6) 843 offset = inst.spec3_format.simmediate; 844 else 845 offset = inst.i_format.simmediate; 846 cache = op_inst & CacheOp_Cache; 847 op = op_inst & CacheOp_Op; 848 849 va = arch->gprs[base] + offset; 850 851 kvm_debug("CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n", 852 cache, op, base, arch->gprs[base], offset); 853 854 /* Secondary or tirtiary cache ops ignored */ 855 if (cache != Cache_I && cache != Cache_D) 856 return EMULATE_DONE; 857 858 switch (op_inst) { 859 case Index_Invalidate_I: 860 flush_icache_line_indexed(va); 861 return EMULATE_DONE; 862 case Index_Writeback_Inv_D: 863 flush_dcache_line_indexed(va); 864 return EMULATE_DONE; 865 default: 866 break; 867 }; 868 869 kvm_err("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n", 870 curr_pc, vcpu->arch.gprs[31], cache, op, base, arch->gprs[base], 871 offset); 872 /* Rollback PC */ 873 vcpu->arch.pc = curr_pc; 874 875 return EMULATE_FAIL; 876 } 877 878 static enum emulation_result kvm_trap_vz_handle_gpsi(u32 cause, u32 *opc, 879 struct kvm_vcpu *vcpu) 880 { 881 enum emulation_result er = EMULATE_DONE; 882 struct kvm_vcpu_arch *arch = &vcpu->arch; 883 struct kvm_run *run = vcpu->run; 884 union mips_instruction inst; 885 int rd, rt, sel; 886 int err; 887 888 /* 889 * Fetch the instruction. 890 */ 891 if (cause & CAUSEF_BD) 892 opc += 1; 893 err = kvm_get_badinstr(opc, vcpu, &inst.word); 894 if (err) 895 return EMULATE_FAIL; 896 897 switch (inst.r_format.opcode) { 898 case cop0_op: 899 er = kvm_vz_gpsi_cop0(inst, opc, cause, run, vcpu); 900 break; 901 #ifndef CONFIG_CPU_MIPSR6 902 case cache_op: 903 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE); 904 er = kvm_vz_gpsi_cache(inst, opc, cause, run, vcpu); 905 break; 906 #endif 907 case spec3_op: 908 switch (inst.spec3_format.func) { 909 #ifdef CONFIG_CPU_MIPSR6 910 case cache6_op: 911 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE); 912 er = kvm_vz_gpsi_cache(inst, opc, cause, run, vcpu); 913 break; 914 #endif 915 case rdhwr_op: 916 if (inst.r_format.rs || (inst.r_format.re >> 3)) 917 goto unknown; 918 919 rd = inst.r_format.rd; 920 rt = inst.r_format.rt; 921 sel = inst.r_format.re & 0x7; 922 923 switch (rd) { 924 case MIPS_HWR_CC: /* Read count register */ 925 arch->gprs[rt] = 926 (long)(int)kvm_mips_read_count(vcpu); 927 break; 928 default: 929 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, 930 KVM_TRACE_HWR(rd, sel), 0); 931 goto unknown; 932 }; 933 934 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, 935 KVM_TRACE_HWR(rd, sel), arch->gprs[rt]); 936 937 er = update_pc(vcpu, cause); 938 break; 939 default: 940 goto unknown; 941 }; 942 break; 943 unknown: 944 945 default: 946 kvm_err("GPSI exception not supported (%p/%#x)\n", 947 opc, inst.word); 948 kvm_arch_vcpu_dump_regs(vcpu); 949 er = EMULATE_FAIL; 950 break; 951 } 952 953 return er; 954 } 955 956 static enum emulation_result kvm_trap_vz_handle_gsfc(u32 cause, u32 *opc, 957 struct kvm_vcpu *vcpu) 958 { 959 enum emulation_result er = EMULATE_DONE; 960 struct kvm_vcpu_arch *arch = &vcpu->arch; 961 union mips_instruction inst; 962 int err; 963 964 /* 965 * Fetch the instruction. 966 */ 967 if (cause & CAUSEF_BD) 968 opc += 1; 969 err = kvm_get_badinstr(opc, vcpu, &inst.word); 970 if (err) 971 return EMULATE_FAIL; 972 973 /* complete MTC0 on behalf of guest and advance EPC */ 974 if (inst.c0r_format.opcode == cop0_op && 975 inst.c0r_format.rs == mtc_op && 976 inst.c0r_format.z == 0) { 977 int rt = inst.c0r_format.rt; 978 int rd = inst.c0r_format.rd; 979 int sel = inst.c0r_format.sel; 980 unsigned int val = arch->gprs[rt]; 981 unsigned int old_val, change; 982 983 trace_kvm_hwr(vcpu, KVM_TRACE_MTC0, KVM_TRACE_COP0(rd, sel), 984 val); 985 986 if ((rd == MIPS_CP0_STATUS) && (sel == 0)) { 987 /* FR bit should read as zero if no FPU */ 988 if (!kvm_mips_guest_has_fpu(&vcpu->arch)) 989 val &= ~(ST0_CU1 | ST0_FR); 990 991 /* 992 * Also don't allow FR to be set if host doesn't support 993 * it. 994 */ 995 if (!(boot_cpu_data.fpu_id & MIPS_FPIR_F64)) 996 val &= ~ST0_FR; 997 998 old_val = read_gc0_status(); 999 change = val ^ old_val; 1000 1001 if (change & ST0_FR) { 1002 /* 1003 * FPU and Vector register state is made 1004 * UNPREDICTABLE by a change of FR, so don't 1005 * even bother saving it. 1006 */ 1007 kvm_drop_fpu(vcpu); 1008 } 1009 1010 /* 1011 * If MSA state is already live, it is undefined how it 1012 * interacts with FR=0 FPU state, and we don't want to 1013 * hit reserved instruction exceptions trying to save 1014 * the MSA state later when CU=1 && FR=1, so play it 1015 * safe and save it first. 1016 */ 1017 if (change & ST0_CU1 && !(val & ST0_FR) && 1018 vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) 1019 kvm_lose_fpu(vcpu); 1020 1021 write_gc0_status(val); 1022 } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) { 1023 u32 old_cause = read_gc0_cause(); 1024 u32 change = old_cause ^ val; 1025 1026 /* DC bit enabling/disabling timer? */ 1027 if (change & CAUSEF_DC) { 1028 if (val & CAUSEF_DC) 1029 kvm_mips_count_disable_cause(vcpu); 1030 else 1031 kvm_mips_count_enable_cause(vcpu); 1032 } 1033 1034 /* Only certain bits are RW to the guest */ 1035 change &= (CAUSEF_DC | CAUSEF_IV | CAUSEF_WP | 1036 CAUSEF_IP0 | CAUSEF_IP1); 1037 1038 /* WP can only be cleared */ 1039 change &= ~CAUSEF_WP | old_cause; 1040 1041 write_gc0_cause(old_cause ^ change); 1042 } else if ((rd == MIPS_CP0_STATUS) && (sel == 1)) { /* IntCtl */ 1043 write_gc0_intctl(val); 1044 } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) { 1045 old_val = read_gc0_config5(); 1046 change = val ^ old_val; 1047 /* Handle changes in FPU/MSA modes */ 1048 preempt_disable(); 1049 1050 /* 1051 * Propagate FRE changes immediately if the FPU 1052 * context is already loaded. 1053 */ 1054 if (change & MIPS_CONF5_FRE && 1055 vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU) 1056 change_c0_config5(MIPS_CONF5_FRE, val); 1057 1058 preempt_enable(); 1059 1060 val = old_val ^ 1061 (change & kvm_vz_config5_guest_wrmask(vcpu)); 1062 write_gc0_config5(val); 1063 } else { 1064 kvm_err("Handle GSFC, unsupported field change @ %p: %#x\n", 1065 opc, inst.word); 1066 er = EMULATE_FAIL; 1067 } 1068 1069 if (er != EMULATE_FAIL) 1070 er = update_pc(vcpu, cause); 1071 } else { 1072 kvm_err("Handle GSFC, unrecognized instruction @ %p: %#x\n", 1073 opc, inst.word); 1074 er = EMULATE_FAIL; 1075 } 1076 1077 return er; 1078 } 1079 1080 static enum emulation_result kvm_trap_vz_handle_hc(u32 cause, u32 *opc, 1081 struct kvm_vcpu *vcpu) 1082 { 1083 enum emulation_result er; 1084 union mips_instruction inst; 1085 unsigned long curr_pc; 1086 int err; 1087 1088 if (cause & CAUSEF_BD) 1089 opc += 1; 1090 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1091 if (err) 1092 return EMULATE_FAIL; 1093 1094 /* 1095 * Update PC and hold onto current PC in case there is 1096 * an error and we want to rollback the PC 1097 */ 1098 curr_pc = vcpu->arch.pc; 1099 er = update_pc(vcpu, cause); 1100 if (er == EMULATE_FAIL) 1101 return er; 1102 1103 er = kvm_mips_emul_hypcall(vcpu, inst); 1104 if (er == EMULATE_FAIL) 1105 vcpu->arch.pc = curr_pc; 1106 1107 return er; 1108 } 1109 1110 static enum emulation_result kvm_trap_vz_no_handler_guest_exit(u32 gexccode, 1111 u32 cause, 1112 u32 *opc, 1113 struct kvm_vcpu *vcpu) 1114 { 1115 u32 inst; 1116 1117 /* 1118 * Fetch the instruction. 1119 */ 1120 if (cause & CAUSEF_BD) 1121 opc += 1; 1122 kvm_get_badinstr(opc, vcpu, &inst); 1123 1124 kvm_err("Guest Exception Code: %d not yet handled @ PC: %p, inst: 0x%08x Status: %#x\n", 1125 gexccode, opc, inst, read_gc0_status()); 1126 1127 return EMULATE_FAIL; 1128 } 1129 1130 static int kvm_trap_vz_handle_guest_exit(struct kvm_vcpu *vcpu) 1131 { 1132 u32 *opc = (u32 *) vcpu->arch.pc; 1133 u32 cause = vcpu->arch.host_cp0_cause; 1134 enum emulation_result er = EMULATE_DONE; 1135 u32 gexccode = (vcpu->arch.host_cp0_guestctl0 & 1136 MIPS_GCTL0_GEXC) >> MIPS_GCTL0_GEXC_SHIFT; 1137 int ret = RESUME_GUEST; 1138 1139 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_GEXCCODE_BASE + gexccode); 1140 switch (gexccode) { 1141 case MIPS_GCTL0_GEXC_GPSI: 1142 ++vcpu->stat.vz_gpsi_exits; 1143 er = kvm_trap_vz_handle_gpsi(cause, opc, vcpu); 1144 break; 1145 case MIPS_GCTL0_GEXC_GSFC: 1146 ++vcpu->stat.vz_gsfc_exits; 1147 er = kvm_trap_vz_handle_gsfc(cause, opc, vcpu); 1148 break; 1149 case MIPS_GCTL0_GEXC_HC: 1150 ++vcpu->stat.vz_hc_exits; 1151 er = kvm_trap_vz_handle_hc(cause, opc, vcpu); 1152 break; 1153 case MIPS_GCTL0_GEXC_GRR: 1154 ++vcpu->stat.vz_grr_exits; 1155 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1156 vcpu); 1157 break; 1158 case MIPS_GCTL0_GEXC_GVA: 1159 ++vcpu->stat.vz_gva_exits; 1160 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1161 vcpu); 1162 break; 1163 case MIPS_GCTL0_GEXC_GHFC: 1164 ++vcpu->stat.vz_ghfc_exits; 1165 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1166 vcpu); 1167 break; 1168 case MIPS_GCTL0_GEXC_GPA: 1169 ++vcpu->stat.vz_gpa_exits; 1170 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1171 vcpu); 1172 break; 1173 default: 1174 ++vcpu->stat.vz_resvd_exits; 1175 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1176 vcpu); 1177 break; 1178 1179 } 1180 1181 if (er == EMULATE_DONE) { 1182 ret = RESUME_GUEST; 1183 } else if (er == EMULATE_HYPERCALL) { 1184 ret = kvm_mips_handle_hypcall(vcpu); 1185 } else { 1186 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1187 ret = RESUME_HOST; 1188 } 1189 return ret; 1190 } 1191 1192 /** 1193 * kvm_trap_vz_handle_cop_unusuable() - Guest used unusable coprocessor. 1194 * @vcpu: Virtual CPU context. 1195 * 1196 * Handle when the guest attempts to use a coprocessor which hasn't been allowed 1197 * by the root context. 1198 */ 1199 static int kvm_trap_vz_handle_cop_unusable(struct kvm_vcpu *vcpu) 1200 { 1201 struct kvm_run *run = vcpu->run; 1202 u32 cause = vcpu->arch.host_cp0_cause; 1203 enum emulation_result er = EMULATE_FAIL; 1204 int ret = RESUME_GUEST; 1205 1206 if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1) { 1207 /* 1208 * If guest FPU not present, the FPU operation should have been 1209 * treated as a reserved instruction! 1210 * If FPU already in use, we shouldn't get this at all. 1211 */ 1212 if (WARN_ON(!kvm_mips_guest_has_fpu(&vcpu->arch) || 1213 vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)) { 1214 preempt_enable(); 1215 return EMULATE_FAIL; 1216 } 1217 1218 kvm_own_fpu(vcpu); 1219 er = EMULATE_DONE; 1220 } 1221 /* other coprocessors not handled */ 1222 1223 switch (er) { 1224 case EMULATE_DONE: 1225 ret = RESUME_GUEST; 1226 break; 1227 1228 case EMULATE_FAIL: 1229 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1230 ret = RESUME_HOST; 1231 break; 1232 1233 default: 1234 BUG(); 1235 } 1236 return ret; 1237 } 1238 1239 /** 1240 * kvm_trap_vz_handle_msa_disabled() - Guest used MSA while disabled in root. 1241 * @vcpu: Virtual CPU context. 1242 * 1243 * Handle when the guest attempts to use MSA when it is disabled in the root 1244 * context. 1245 */ 1246 static int kvm_trap_vz_handle_msa_disabled(struct kvm_vcpu *vcpu) 1247 { 1248 struct kvm_run *run = vcpu->run; 1249 1250 /* 1251 * If MSA not present or not exposed to guest or FR=0, the MSA operation 1252 * should have been treated as a reserved instruction! 1253 * Same if CU1=1, FR=0. 1254 * If MSA already in use, we shouldn't get this at all. 1255 */ 1256 if (!kvm_mips_guest_has_msa(&vcpu->arch) || 1257 (read_gc0_status() & (ST0_CU1 | ST0_FR)) == ST0_CU1 || 1258 !(read_gc0_config5() & MIPS_CONF5_MSAEN) || 1259 vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) { 1260 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1261 return RESUME_HOST; 1262 } 1263 1264 kvm_own_msa(vcpu); 1265 1266 return RESUME_GUEST; 1267 } 1268 1269 static int kvm_trap_vz_handle_tlb_ld_miss(struct kvm_vcpu *vcpu) 1270 { 1271 struct kvm_run *run = vcpu->run; 1272 u32 *opc = (u32 *) vcpu->arch.pc; 1273 u32 cause = vcpu->arch.host_cp0_cause; 1274 ulong badvaddr = vcpu->arch.host_cp0_badvaddr; 1275 union mips_instruction inst; 1276 enum emulation_result er = EMULATE_DONE; 1277 int err, ret = RESUME_GUEST; 1278 1279 if (kvm_mips_handle_vz_root_tlb_fault(badvaddr, vcpu, false)) { 1280 /* A code fetch fault doesn't count as an MMIO */ 1281 if (kvm_is_ifetch_fault(&vcpu->arch)) { 1282 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1283 return RESUME_HOST; 1284 } 1285 1286 /* Fetch the instruction */ 1287 if (cause & CAUSEF_BD) 1288 opc += 1; 1289 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1290 if (err) { 1291 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1292 return RESUME_HOST; 1293 } 1294 1295 /* Treat as MMIO */ 1296 er = kvm_mips_emulate_load(inst, cause, run, vcpu); 1297 if (er == EMULATE_FAIL) { 1298 kvm_err("Guest Emulate Load from MMIO space failed: PC: %p, BadVaddr: %#lx\n", 1299 opc, badvaddr); 1300 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1301 } 1302 } 1303 1304 if (er == EMULATE_DONE) { 1305 ret = RESUME_GUEST; 1306 } else if (er == EMULATE_DO_MMIO) { 1307 run->exit_reason = KVM_EXIT_MMIO; 1308 ret = RESUME_HOST; 1309 } else { 1310 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1311 ret = RESUME_HOST; 1312 } 1313 return ret; 1314 } 1315 1316 static int kvm_trap_vz_handle_tlb_st_miss(struct kvm_vcpu *vcpu) 1317 { 1318 struct kvm_run *run = vcpu->run; 1319 u32 *opc = (u32 *) vcpu->arch.pc; 1320 u32 cause = vcpu->arch.host_cp0_cause; 1321 ulong badvaddr = vcpu->arch.host_cp0_badvaddr; 1322 union mips_instruction inst; 1323 enum emulation_result er = EMULATE_DONE; 1324 int err; 1325 int ret = RESUME_GUEST; 1326 1327 /* Just try the access again if we couldn't do the translation */ 1328 if (kvm_vz_badvaddr_to_gpa(vcpu, badvaddr, &badvaddr)) 1329 return RESUME_GUEST; 1330 vcpu->arch.host_cp0_badvaddr = badvaddr; 1331 1332 if (kvm_mips_handle_vz_root_tlb_fault(badvaddr, vcpu, true)) { 1333 /* Fetch the instruction */ 1334 if (cause & CAUSEF_BD) 1335 opc += 1; 1336 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1337 if (err) { 1338 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1339 return RESUME_HOST; 1340 } 1341 1342 /* Treat as MMIO */ 1343 er = kvm_mips_emulate_store(inst, cause, run, vcpu); 1344 if (er == EMULATE_FAIL) { 1345 kvm_err("Guest Emulate Store to MMIO space failed: PC: %p, BadVaddr: %#lx\n", 1346 opc, badvaddr); 1347 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1348 } 1349 } 1350 1351 if (er == EMULATE_DONE) { 1352 ret = RESUME_GUEST; 1353 } else if (er == EMULATE_DO_MMIO) { 1354 run->exit_reason = KVM_EXIT_MMIO; 1355 ret = RESUME_HOST; 1356 } else { 1357 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1358 ret = RESUME_HOST; 1359 } 1360 return ret; 1361 } 1362 1363 static u64 kvm_vz_get_one_regs[] = { 1364 KVM_REG_MIPS_CP0_INDEX, 1365 KVM_REG_MIPS_CP0_ENTRYLO0, 1366 KVM_REG_MIPS_CP0_ENTRYLO1, 1367 KVM_REG_MIPS_CP0_CONTEXT, 1368 KVM_REG_MIPS_CP0_PAGEMASK, 1369 KVM_REG_MIPS_CP0_PAGEGRAIN, 1370 KVM_REG_MIPS_CP0_WIRED, 1371 KVM_REG_MIPS_CP0_HWRENA, 1372 KVM_REG_MIPS_CP0_BADVADDR, 1373 KVM_REG_MIPS_CP0_COUNT, 1374 KVM_REG_MIPS_CP0_ENTRYHI, 1375 KVM_REG_MIPS_CP0_COMPARE, 1376 KVM_REG_MIPS_CP0_STATUS, 1377 KVM_REG_MIPS_CP0_INTCTL, 1378 KVM_REG_MIPS_CP0_CAUSE, 1379 KVM_REG_MIPS_CP0_EPC, 1380 KVM_REG_MIPS_CP0_PRID, 1381 KVM_REG_MIPS_CP0_EBASE, 1382 KVM_REG_MIPS_CP0_CONFIG, 1383 KVM_REG_MIPS_CP0_CONFIG1, 1384 KVM_REG_MIPS_CP0_CONFIG2, 1385 KVM_REG_MIPS_CP0_CONFIG3, 1386 KVM_REG_MIPS_CP0_CONFIG4, 1387 KVM_REG_MIPS_CP0_CONFIG5, 1388 #ifdef CONFIG_64BIT 1389 KVM_REG_MIPS_CP0_XCONTEXT, 1390 #endif 1391 KVM_REG_MIPS_CP0_ERROREPC, 1392 1393 KVM_REG_MIPS_COUNT_CTL, 1394 KVM_REG_MIPS_COUNT_RESUME, 1395 KVM_REG_MIPS_COUNT_HZ, 1396 }; 1397 1398 static u64 kvm_vz_get_one_regs_contextconfig[] = { 1399 KVM_REG_MIPS_CP0_CONTEXTCONFIG, 1400 #ifdef CONFIG_64BIT 1401 KVM_REG_MIPS_CP0_XCONTEXTCONFIG, 1402 #endif 1403 }; 1404 1405 static u64 kvm_vz_get_one_regs_segments[] = { 1406 KVM_REG_MIPS_CP0_SEGCTL0, 1407 KVM_REG_MIPS_CP0_SEGCTL1, 1408 KVM_REG_MIPS_CP0_SEGCTL2, 1409 }; 1410 1411 static u64 kvm_vz_get_one_regs_htw[] = { 1412 KVM_REG_MIPS_CP0_PWBASE, 1413 KVM_REG_MIPS_CP0_PWFIELD, 1414 KVM_REG_MIPS_CP0_PWSIZE, 1415 KVM_REG_MIPS_CP0_PWCTL, 1416 }; 1417 1418 static u64 kvm_vz_get_one_regs_kscratch[] = { 1419 KVM_REG_MIPS_CP0_KSCRATCH1, 1420 KVM_REG_MIPS_CP0_KSCRATCH2, 1421 KVM_REG_MIPS_CP0_KSCRATCH3, 1422 KVM_REG_MIPS_CP0_KSCRATCH4, 1423 KVM_REG_MIPS_CP0_KSCRATCH5, 1424 KVM_REG_MIPS_CP0_KSCRATCH6, 1425 }; 1426 1427 static unsigned long kvm_vz_num_regs(struct kvm_vcpu *vcpu) 1428 { 1429 unsigned long ret; 1430 1431 ret = ARRAY_SIZE(kvm_vz_get_one_regs); 1432 if (cpu_guest_has_userlocal) 1433 ++ret; 1434 if (cpu_guest_has_badinstr) 1435 ++ret; 1436 if (cpu_guest_has_badinstrp) 1437 ++ret; 1438 if (cpu_guest_has_contextconfig) 1439 ret += ARRAY_SIZE(kvm_vz_get_one_regs_contextconfig); 1440 if (cpu_guest_has_segments) 1441 ret += ARRAY_SIZE(kvm_vz_get_one_regs_segments); 1442 if (cpu_guest_has_htw) 1443 ret += ARRAY_SIZE(kvm_vz_get_one_regs_htw); 1444 ret += __arch_hweight8(cpu_data[0].guest.kscratch_mask); 1445 1446 return ret; 1447 } 1448 1449 static int kvm_vz_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices) 1450 { 1451 u64 index; 1452 unsigned int i; 1453 1454 if (copy_to_user(indices, kvm_vz_get_one_regs, 1455 sizeof(kvm_vz_get_one_regs))) 1456 return -EFAULT; 1457 indices += ARRAY_SIZE(kvm_vz_get_one_regs); 1458 1459 if (cpu_guest_has_userlocal) { 1460 index = KVM_REG_MIPS_CP0_USERLOCAL; 1461 if (copy_to_user(indices, &index, sizeof(index))) 1462 return -EFAULT; 1463 ++indices; 1464 } 1465 if (cpu_guest_has_badinstr) { 1466 index = KVM_REG_MIPS_CP0_BADINSTR; 1467 if (copy_to_user(indices, &index, sizeof(index))) 1468 return -EFAULT; 1469 ++indices; 1470 } 1471 if (cpu_guest_has_badinstrp) { 1472 index = KVM_REG_MIPS_CP0_BADINSTRP; 1473 if (copy_to_user(indices, &index, sizeof(index))) 1474 return -EFAULT; 1475 ++indices; 1476 } 1477 if (cpu_guest_has_contextconfig) { 1478 if (copy_to_user(indices, kvm_vz_get_one_regs_contextconfig, 1479 sizeof(kvm_vz_get_one_regs_contextconfig))) 1480 return -EFAULT; 1481 indices += ARRAY_SIZE(kvm_vz_get_one_regs_contextconfig); 1482 } 1483 if (cpu_guest_has_segments) { 1484 if (copy_to_user(indices, kvm_vz_get_one_regs_segments, 1485 sizeof(kvm_vz_get_one_regs_segments))) 1486 return -EFAULT; 1487 indices += ARRAY_SIZE(kvm_vz_get_one_regs_segments); 1488 } 1489 if (cpu_guest_has_htw) { 1490 if (copy_to_user(indices, kvm_vz_get_one_regs_htw, 1491 sizeof(kvm_vz_get_one_regs_htw))) 1492 return -EFAULT; 1493 indices += ARRAY_SIZE(kvm_vz_get_one_regs_htw); 1494 } 1495 for (i = 0; i < 6; ++i) { 1496 if (!cpu_guest_has_kscr(i + 2)) 1497 continue; 1498 1499 if (copy_to_user(indices, &kvm_vz_get_one_regs_kscratch[i], 1500 sizeof(kvm_vz_get_one_regs_kscratch[i]))) 1501 return -EFAULT; 1502 ++indices; 1503 } 1504 1505 return 0; 1506 } 1507 1508 static inline s64 entrylo_kvm_to_user(unsigned long v) 1509 { 1510 s64 mask, ret = v; 1511 1512 if (BITS_PER_LONG == 32) { 1513 /* 1514 * KVM API exposes 64-bit version of the register, so move the 1515 * RI/XI bits up into place. 1516 */ 1517 mask = MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI; 1518 ret &= ~mask; 1519 ret |= ((s64)v & mask) << 32; 1520 } 1521 return ret; 1522 } 1523 1524 static inline unsigned long entrylo_user_to_kvm(s64 v) 1525 { 1526 unsigned long mask, ret = v; 1527 1528 if (BITS_PER_LONG == 32) { 1529 /* 1530 * KVM API exposes 64-bit versiono of the register, so move the 1531 * RI/XI bits down into place. 1532 */ 1533 mask = MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI; 1534 ret &= ~mask; 1535 ret |= (v >> 32) & mask; 1536 } 1537 return ret; 1538 } 1539 1540 static int kvm_vz_get_one_reg(struct kvm_vcpu *vcpu, 1541 const struct kvm_one_reg *reg, 1542 s64 *v) 1543 { 1544 struct mips_coproc *cop0 = vcpu->arch.cop0; 1545 unsigned int idx; 1546 1547 switch (reg->id) { 1548 case KVM_REG_MIPS_CP0_INDEX: 1549 *v = (long)read_gc0_index(); 1550 break; 1551 case KVM_REG_MIPS_CP0_ENTRYLO0: 1552 *v = entrylo_kvm_to_user(read_gc0_entrylo0()); 1553 break; 1554 case KVM_REG_MIPS_CP0_ENTRYLO1: 1555 *v = entrylo_kvm_to_user(read_gc0_entrylo1()); 1556 break; 1557 case KVM_REG_MIPS_CP0_CONTEXT: 1558 *v = (long)read_gc0_context(); 1559 break; 1560 case KVM_REG_MIPS_CP0_CONTEXTCONFIG: 1561 if (!cpu_guest_has_contextconfig) 1562 return -EINVAL; 1563 *v = read_gc0_contextconfig(); 1564 break; 1565 case KVM_REG_MIPS_CP0_USERLOCAL: 1566 if (!cpu_guest_has_userlocal) 1567 return -EINVAL; 1568 *v = read_gc0_userlocal(); 1569 break; 1570 #ifdef CONFIG_64BIT 1571 case KVM_REG_MIPS_CP0_XCONTEXTCONFIG: 1572 if (!cpu_guest_has_contextconfig) 1573 return -EINVAL; 1574 *v = read_gc0_xcontextconfig(); 1575 break; 1576 #endif 1577 case KVM_REG_MIPS_CP0_PAGEMASK: 1578 *v = (long)read_gc0_pagemask(); 1579 break; 1580 case KVM_REG_MIPS_CP0_PAGEGRAIN: 1581 *v = (long)read_gc0_pagegrain(); 1582 break; 1583 case KVM_REG_MIPS_CP0_SEGCTL0: 1584 if (!cpu_guest_has_segments) 1585 return -EINVAL; 1586 *v = read_gc0_segctl0(); 1587 break; 1588 case KVM_REG_MIPS_CP0_SEGCTL1: 1589 if (!cpu_guest_has_segments) 1590 return -EINVAL; 1591 *v = read_gc0_segctl1(); 1592 break; 1593 case KVM_REG_MIPS_CP0_SEGCTL2: 1594 if (!cpu_guest_has_segments) 1595 return -EINVAL; 1596 *v = read_gc0_segctl2(); 1597 break; 1598 case KVM_REG_MIPS_CP0_PWBASE: 1599 if (!cpu_guest_has_htw) 1600 return -EINVAL; 1601 *v = read_gc0_pwbase(); 1602 break; 1603 case KVM_REG_MIPS_CP0_PWFIELD: 1604 if (!cpu_guest_has_htw) 1605 return -EINVAL; 1606 *v = read_gc0_pwfield(); 1607 break; 1608 case KVM_REG_MIPS_CP0_PWSIZE: 1609 if (!cpu_guest_has_htw) 1610 return -EINVAL; 1611 *v = read_gc0_pwsize(); 1612 break; 1613 case KVM_REG_MIPS_CP0_WIRED: 1614 *v = (long)read_gc0_wired(); 1615 break; 1616 case KVM_REG_MIPS_CP0_PWCTL: 1617 if (!cpu_guest_has_htw) 1618 return -EINVAL; 1619 *v = read_gc0_pwctl(); 1620 break; 1621 case KVM_REG_MIPS_CP0_HWRENA: 1622 *v = (long)read_gc0_hwrena(); 1623 break; 1624 case KVM_REG_MIPS_CP0_BADVADDR: 1625 *v = (long)read_gc0_badvaddr(); 1626 break; 1627 case KVM_REG_MIPS_CP0_BADINSTR: 1628 if (!cpu_guest_has_badinstr) 1629 return -EINVAL; 1630 *v = read_gc0_badinstr(); 1631 break; 1632 case KVM_REG_MIPS_CP0_BADINSTRP: 1633 if (!cpu_guest_has_badinstrp) 1634 return -EINVAL; 1635 *v = read_gc0_badinstrp(); 1636 break; 1637 case KVM_REG_MIPS_CP0_COUNT: 1638 *v = kvm_mips_read_count(vcpu); 1639 break; 1640 case KVM_REG_MIPS_CP0_ENTRYHI: 1641 *v = (long)read_gc0_entryhi(); 1642 break; 1643 case KVM_REG_MIPS_CP0_COMPARE: 1644 *v = (long)read_gc0_compare(); 1645 break; 1646 case KVM_REG_MIPS_CP0_STATUS: 1647 *v = (long)read_gc0_status(); 1648 break; 1649 case KVM_REG_MIPS_CP0_INTCTL: 1650 *v = read_gc0_intctl(); 1651 break; 1652 case KVM_REG_MIPS_CP0_CAUSE: 1653 *v = (long)read_gc0_cause(); 1654 break; 1655 case KVM_REG_MIPS_CP0_EPC: 1656 *v = (long)read_gc0_epc(); 1657 break; 1658 case KVM_REG_MIPS_CP0_PRID: 1659 *v = (long)kvm_read_c0_guest_prid(cop0); 1660 break; 1661 case KVM_REG_MIPS_CP0_EBASE: 1662 *v = kvm_vz_read_gc0_ebase(); 1663 break; 1664 case KVM_REG_MIPS_CP0_CONFIG: 1665 *v = read_gc0_config(); 1666 break; 1667 case KVM_REG_MIPS_CP0_CONFIG1: 1668 if (!cpu_guest_has_conf1) 1669 return -EINVAL; 1670 *v = read_gc0_config1(); 1671 break; 1672 case KVM_REG_MIPS_CP0_CONFIG2: 1673 if (!cpu_guest_has_conf2) 1674 return -EINVAL; 1675 *v = read_gc0_config2(); 1676 break; 1677 case KVM_REG_MIPS_CP0_CONFIG3: 1678 if (!cpu_guest_has_conf3) 1679 return -EINVAL; 1680 *v = read_gc0_config3(); 1681 break; 1682 case KVM_REG_MIPS_CP0_CONFIG4: 1683 if (!cpu_guest_has_conf4) 1684 return -EINVAL; 1685 *v = read_gc0_config4(); 1686 break; 1687 case KVM_REG_MIPS_CP0_CONFIG5: 1688 if (!cpu_guest_has_conf5) 1689 return -EINVAL; 1690 *v = read_gc0_config5(); 1691 break; 1692 #ifdef CONFIG_64BIT 1693 case KVM_REG_MIPS_CP0_XCONTEXT: 1694 *v = read_gc0_xcontext(); 1695 break; 1696 #endif 1697 case KVM_REG_MIPS_CP0_ERROREPC: 1698 *v = (long)read_gc0_errorepc(); 1699 break; 1700 case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6: 1701 idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2; 1702 if (!cpu_guest_has_kscr(idx)) 1703 return -EINVAL; 1704 switch (idx) { 1705 case 2: 1706 *v = (long)read_gc0_kscratch1(); 1707 break; 1708 case 3: 1709 *v = (long)read_gc0_kscratch2(); 1710 break; 1711 case 4: 1712 *v = (long)read_gc0_kscratch3(); 1713 break; 1714 case 5: 1715 *v = (long)read_gc0_kscratch4(); 1716 break; 1717 case 6: 1718 *v = (long)read_gc0_kscratch5(); 1719 break; 1720 case 7: 1721 *v = (long)read_gc0_kscratch6(); 1722 break; 1723 } 1724 break; 1725 case KVM_REG_MIPS_COUNT_CTL: 1726 *v = vcpu->arch.count_ctl; 1727 break; 1728 case KVM_REG_MIPS_COUNT_RESUME: 1729 *v = ktime_to_ns(vcpu->arch.count_resume); 1730 break; 1731 case KVM_REG_MIPS_COUNT_HZ: 1732 *v = vcpu->arch.count_hz; 1733 break; 1734 default: 1735 return -EINVAL; 1736 } 1737 return 0; 1738 } 1739 1740 static int kvm_vz_set_one_reg(struct kvm_vcpu *vcpu, 1741 const struct kvm_one_reg *reg, 1742 s64 v) 1743 { 1744 struct mips_coproc *cop0 = vcpu->arch.cop0; 1745 unsigned int idx; 1746 int ret = 0; 1747 unsigned int cur, change; 1748 1749 switch (reg->id) { 1750 case KVM_REG_MIPS_CP0_INDEX: 1751 write_gc0_index(v); 1752 break; 1753 case KVM_REG_MIPS_CP0_ENTRYLO0: 1754 write_gc0_entrylo0(entrylo_user_to_kvm(v)); 1755 break; 1756 case KVM_REG_MIPS_CP0_ENTRYLO1: 1757 write_gc0_entrylo1(entrylo_user_to_kvm(v)); 1758 break; 1759 case KVM_REG_MIPS_CP0_CONTEXT: 1760 write_gc0_context(v); 1761 break; 1762 case KVM_REG_MIPS_CP0_CONTEXTCONFIG: 1763 if (!cpu_guest_has_contextconfig) 1764 return -EINVAL; 1765 write_gc0_contextconfig(v); 1766 break; 1767 case KVM_REG_MIPS_CP0_USERLOCAL: 1768 if (!cpu_guest_has_userlocal) 1769 return -EINVAL; 1770 write_gc0_userlocal(v); 1771 break; 1772 #ifdef CONFIG_64BIT 1773 case KVM_REG_MIPS_CP0_XCONTEXTCONFIG: 1774 if (!cpu_guest_has_contextconfig) 1775 return -EINVAL; 1776 write_gc0_xcontextconfig(v); 1777 break; 1778 #endif 1779 case KVM_REG_MIPS_CP0_PAGEMASK: 1780 write_gc0_pagemask(v); 1781 break; 1782 case KVM_REG_MIPS_CP0_PAGEGRAIN: 1783 write_gc0_pagegrain(v); 1784 break; 1785 case KVM_REG_MIPS_CP0_SEGCTL0: 1786 if (!cpu_guest_has_segments) 1787 return -EINVAL; 1788 write_gc0_segctl0(v); 1789 break; 1790 case KVM_REG_MIPS_CP0_SEGCTL1: 1791 if (!cpu_guest_has_segments) 1792 return -EINVAL; 1793 write_gc0_segctl1(v); 1794 break; 1795 case KVM_REG_MIPS_CP0_SEGCTL2: 1796 if (!cpu_guest_has_segments) 1797 return -EINVAL; 1798 write_gc0_segctl2(v); 1799 break; 1800 case KVM_REG_MIPS_CP0_PWBASE: 1801 if (!cpu_guest_has_htw) 1802 return -EINVAL; 1803 write_gc0_pwbase(v); 1804 break; 1805 case KVM_REG_MIPS_CP0_PWFIELD: 1806 if (!cpu_guest_has_htw) 1807 return -EINVAL; 1808 write_gc0_pwfield(v); 1809 break; 1810 case KVM_REG_MIPS_CP0_PWSIZE: 1811 if (!cpu_guest_has_htw) 1812 return -EINVAL; 1813 write_gc0_pwsize(v); 1814 break; 1815 case KVM_REG_MIPS_CP0_WIRED: 1816 change_gc0_wired(MIPSR6_WIRED_WIRED, v); 1817 break; 1818 case KVM_REG_MIPS_CP0_PWCTL: 1819 if (!cpu_guest_has_htw) 1820 return -EINVAL; 1821 write_gc0_pwctl(v); 1822 break; 1823 case KVM_REG_MIPS_CP0_HWRENA: 1824 write_gc0_hwrena(v); 1825 break; 1826 case KVM_REG_MIPS_CP0_BADVADDR: 1827 write_gc0_badvaddr(v); 1828 break; 1829 case KVM_REG_MIPS_CP0_BADINSTR: 1830 if (!cpu_guest_has_badinstr) 1831 return -EINVAL; 1832 write_gc0_badinstr(v); 1833 break; 1834 case KVM_REG_MIPS_CP0_BADINSTRP: 1835 if (!cpu_guest_has_badinstrp) 1836 return -EINVAL; 1837 write_gc0_badinstrp(v); 1838 break; 1839 case KVM_REG_MIPS_CP0_COUNT: 1840 kvm_mips_write_count(vcpu, v); 1841 break; 1842 case KVM_REG_MIPS_CP0_ENTRYHI: 1843 write_gc0_entryhi(v); 1844 break; 1845 case KVM_REG_MIPS_CP0_COMPARE: 1846 kvm_mips_write_compare(vcpu, v, false); 1847 break; 1848 case KVM_REG_MIPS_CP0_STATUS: 1849 write_gc0_status(v); 1850 break; 1851 case KVM_REG_MIPS_CP0_INTCTL: 1852 write_gc0_intctl(v); 1853 break; 1854 case KVM_REG_MIPS_CP0_CAUSE: 1855 /* 1856 * If the timer is stopped or started (DC bit) it must look 1857 * atomic with changes to the timer interrupt pending bit (TI). 1858 * A timer interrupt should not happen in between. 1859 */ 1860 if ((read_gc0_cause() ^ v) & CAUSEF_DC) { 1861 if (v & CAUSEF_DC) { 1862 /* disable timer first */ 1863 kvm_mips_count_disable_cause(vcpu); 1864 change_gc0_cause((u32)~CAUSEF_DC, v); 1865 } else { 1866 /* enable timer last */ 1867 change_gc0_cause((u32)~CAUSEF_DC, v); 1868 kvm_mips_count_enable_cause(vcpu); 1869 } 1870 } else { 1871 write_gc0_cause(v); 1872 } 1873 break; 1874 case KVM_REG_MIPS_CP0_EPC: 1875 write_gc0_epc(v); 1876 break; 1877 case KVM_REG_MIPS_CP0_PRID: 1878 kvm_write_c0_guest_prid(cop0, v); 1879 break; 1880 case KVM_REG_MIPS_CP0_EBASE: 1881 kvm_vz_write_gc0_ebase(v); 1882 break; 1883 case KVM_REG_MIPS_CP0_CONFIG: 1884 cur = read_gc0_config(); 1885 change = (cur ^ v) & kvm_vz_config_user_wrmask(vcpu); 1886 if (change) { 1887 v = cur ^ change; 1888 write_gc0_config(v); 1889 } 1890 break; 1891 case KVM_REG_MIPS_CP0_CONFIG1: 1892 if (!cpu_guest_has_conf1) 1893 break; 1894 cur = read_gc0_config1(); 1895 change = (cur ^ v) & kvm_vz_config1_user_wrmask(vcpu); 1896 if (change) { 1897 v = cur ^ change; 1898 write_gc0_config1(v); 1899 } 1900 break; 1901 case KVM_REG_MIPS_CP0_CONFIG2: 1902 if (!cpu_guest_has_conf2) 1903 break; 1904 cur = read_gc0_config2(); 1905 change = (cur ^ v) & kvm_vz_config2_user_wrmask(vcpu); 1906 if (change) { 1907 v = cur ^ change; 1908 write_gc0_config2(v); 1909 } 1910 break; 1911 case KVM_REG_MIPS_CP0_CONFIG3: 1912 if (!cpu_guest_has_conf3) 1913 break; 1914 cur = read_gc0_config3(); 1915 change = (cur ^ v) & kvm_vz_config3_user_wrmask(vcpu); 1916 if (change) { 1917 v = cur ^ change; 1918 write_gc0_config3(v); 1919 } 1920 break; 1921 case KVM_REG_MIPS_CP0_CONFIG4: 1922 if (!cpu_guest_has_conf4) 1923 break; 1924 cur = read_gc0_config4(); 1925 change = (cur ^ v) & kvm_vz_config4_user_wrmask(vcpu); 1926 if (change) { 1927 v = cur ^ change; 1928 write_gc0_config4(v); 1929 } 1930 break; 1931 case KVM_REG_MIPS_CP0_CONFIG5: 1932 if (!cpu_guest_has_conf5) 1933 break; 1934 cur = read_gc0_config5(); 1935 change = (cur ^ v) & kvm_vz_config5_user_wrmask(vcpu); 1936 if (change) { 1937 v = cur ^ change; 1938 write_gc0_config5(v); 1939 } 1940 break; 1941 #ifdef CONFIG_64BIT 1942 case KVM_REG_MIPS_CP0_XCONTEXT: 1943 write_gc0_xcontext(v); 1944 break; 1945 #endif 1946 case KVM_REG_MIPS_CP0_ERROREPC: 1947 write_gc0_errorepc(v); 1948 break; 1949 case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6: 1950 idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2; 1951 if (!cpu_guest_has_kscr(idx)) 1952 return -EINVAL; 1953 switch (idx) { 1954 case 2: 1955 write_gc0_kscratch1(v); 1956 break; 1957 case 3: 1958 write_gc0_kscratch2(v); 1959 break; 1960 case 4: 1961 write_gc0_kscratch3(v); 1962 break; 1963 case 5: 1964 write_gc0_kscratch4(v); 1965 break; 1966 case 6: 1967 write_gc0_kscratch5(v); 1968 break; 1969 case 7: 1970 write_gc0_kscratch6(v); 1971 break; 1972 } 1973 break; 1974 case KVM_REG_MIPS_COUNT_CTL: 1975 ret = kvm_mips_set_count_ctl(vcpu, v); 1976 break; 1977 case KVM_REG_MIPS_COUNT_RESUME: 1978 ret = kvm_mips_set_count_resume(vcpu, v); 1979 break; 1980 case KVM_REG_MIPS_COUNT_HZ: 1981 ret = kvm_mips_set_count_hz(vcpu, v); 1982 break; 1983 default: 1984 return -EINVAL; 1985 } 1986 return ret; 1987 } 1988 1989 #define guestid_cache(cpu) (cpu_data[cpu].guestid_cache) 1990 static void kvm_vz_get_new_guestid(unsigned long cpu, struct kvm_vcpu *vcpu) 1991 { 1992 unsigned long guestid = guestid_cache(cpu); 1993 1994 if (!(++guestid & GUESTID_MASK)) { 1995 if (cpu_has_vtag_icache) 1996 flush_icache_all(); 1997 1998 if (!guestid) /* fix version if needed */ 1999 guestid = GUESTID_FIRST_VERSION; 2000 2001 ++guestid; /* guestid 0 reserved for root */ 2002 2003 /* start new guestid cycle */ 2004 kvm_vz_local_flush_roottlb_all_guests(); 2005 kvm_vz_local_flush_guesttlb_all(); 2006 } 2007 2008 guestid_cache(cpu) = guestid; 2009 } 2010 2011 /* Returns 1 if the guest TLB may be clobbered */ 2012 static int kvm_vz_check_requests(struct kvm_vcpu *vcpu, int cpu) 2013 { 2014 int ret = 0; 2015 int i; 2016 2017 if (!vcpu->requests) 2018 return 0; 2019 2020 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { 2021 if (cpu_has_guestid) { 2022 /* Drop all GuestIDs for this VCPU */ 2023 for_each_possible_cpu(i) 2024 vcpu->arch.vzguestid[i] = 0; 2025 /* This will clobber guest TLB contents too */ 2026 ret = 1; 2027 } 2028 /* 2029 * For Root ASID Dealias (RAD) we don't do anything here, but we 2030 * still need the request to ensure we recheck asid_flush_mask. 2031 * We can still return 0 as only the root TLB will be affected 2032 * by a root ASID flush. 2033 */ 2034 } 2035 2036 return ret; 2037 } 2038 2039 static void kvm_vz_vcpu_save_wired(struct kvm_vcpu *vcpu) 2040 { 2041 unsigned int wired = read_gc0_wired(); 2042 struct kvm_mips_tlb *tlbs; 2043 int i; 2044 2045 /* Expand the wired TLB array if necessary */ 2046 wired &= MIPSR6_WIRED_WIRED; 2047 if (wired > vcpu->arch.wired_tlb_limit) { 2048 tlbs = krealloc(vcpu->arch.wired_tlb, wired * 2049 sizeof(*vcpu->arch.wired_tlb), GFP_ATOMIC); 2050 if (WARN_ON(!tlbs)) { 2051 /* Save whatever we can */ 2052 wired = vcpu->arch.wired_tlb_limit; 2053 } else { 2054 vcpu->arch.wired_tlb = tlbs; 2055 vcpu->arch.wired_tlb_limit = wired; 2056 } 2057 } 2058 2059 if (wired) 2060 /* Save wired entries from the guest TLB */ 2061 kvm_vz_save_guesttlb(vcpu->arch.wired_tlb, 0, wired); 2062 /* Invalidate any dropped entries since last time */ 2063 for (i = wired; i < vcpu->arch.wired_tlb_used; ++i) { 2064 vcpu->arch.wired_tlb[i].tlb_hi = UNIQUE_GUEST_ENTRYHI(i); 2065 vcpu->arch.wired_tlb[i].tlb_lo[0] = 0; 2066 vcpu->arch.wired_tlb[i].tlb_lo[1] = 0; 2067 vcpu->arch.wired_tlb[i].tlb_mask = 0; 2068 } 2069 vcpu->arch.wired_tlb_used = wired; 2070 } 2071 2072 static void kvm_vz_vcpu_load_wired(struct kvm_vcpu *vcpu) 2073 { 2074 /* Load wired entries into the guest TLB */ 2075 if (vcpu->arch.wired_tlb) 2076 kvm_vz_load_guesttlb(vcpu->arch.wired_tlb, 0, 2077 vcpu->arch.wired_tlb_used); 2078 } 2079 2080 static void kvm_vz_vcpu_load_tlb(struct kvm_vcpu *vcpu, int cpu) 2081 { 2082 struct kvm *kvm = vcpu->kvm; 2083 struct mm_struct *gpa_mm = &kvm->arch.gpa_mm; 2084 bool migrated; 2085 2086 /* 2087 * Are we entering guest context on a different CPU to last time? 2088 * If so, the VCPU's guest TLB state on this CPU may be stale. 2089 */ 2090 migrated = (vcpu->arch.last_exec_cpu != cpu); 2091 vcpu->arch.last_exec_cpu = cpu; 2092 2093 /* 2094 * A vcpu's GuestID is set in GuestCtl1.ID when the vcpu is loaded and 2095 * remains set until another vcpu is loaded in. As a rule GuestRID 2096 * remains zeroed when in root context unless the kernel is busy 2097 * manipulating guest tlb entries. 2098 */ 2099 if (cpu_has_guestid) { 2100 /* 2101 * Check if our GuestID is of an older version and thus invalid. 2102 * 2103 * We also discard the stored GuestID if we've executed on 2104 * another CPU, as the guest mappings may have changed without 2105 * hypervisor knowledge. 2106 */ 2107 if (migrated || 2108 (vcpu->arch.vzguestid[cpu] ^ guestid_cache(cpu)) & 2109 GUESTID_VERSION_MASK) { 2110 kvm_vz_get_new_guestid(cpu, vcpu); 2111 vcpu->arch.vzguestid[cpu] = guestid_cache(cpu); 2112 trace_kvm_guestid_change(vcpu, 2113 vcpu->arch.vzguestid[cpu]); 2114 } 2115 2116 /* Restore GuestID */ 2117 change_c0_guestctl1(GUESTID_MASK, vcpu->arch.vzguestid[cpu]); 2118 } else { 2119 /* 2120 * The Guest TLB only stores a single guest's TLB state, so 2121 * flush it if another VCPU has executed on this CPU. 2122 * 2123 * We also flush if we've executed on another CPU, as the guest 2124 * mappings may have changed without hypervisor knowledge. 2125 */ 2126 if (migrated || last_exec_vcpu[cpu] != vcpu) 2127 kvm_vz_local_flush_guesttlb_all(); 2128 last_exec_vcpu[cpu] = vcpu; 2129 2130 /* 2131 * Root ASID dealiases guest GPA mappings in the root TLB. 2132 * Allocate new root ASID if needed. 2133 */ 2134 if (cpumask_test_and_clear_cpu(cpu, &kvm->arch.asid_flush_mask) 2135 || (cpu_context(cpu, gpa_mm) ^ asid_cache(cpu)) & 2136 asid_version_mask(cpu)) 2137 get_new_mmu_context(gpa_mm, cpu); 2138 } 2139 } 2140 2141 static int kvm_vz_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 2142 { 2143 struct mips_coproc *cop0 = vcpu->arch.cop0; 2144 bool migrated, all; 2145 2146 /* 2147 * Have we migrated to a different CPU? 2148 * If so, any old guest TLB state may be stale. 2149 */ 2150 migrated = (vcpu->arch.last_sched_cpu != cpu); 2151 2152 /* 2153 * Was this the last VCPU to run on this CPU? 2154 * If not, any old guest state from this VCPU will have been clobbered. 2155 */ 2156 all = migrated || (last_vcpu[cpu] != vcpu); 2157 last_vcpu[cpu] = vcpu; 2158 2159 /* 2160 * Restore CP0_Wired unconditionally as we clear it after use, and 2161 * restore wired guest TLB entries (while in guest context). 2162 */ 2163 kvm_restore_gc0_wired(cop0); 2164 if (current->flags & PF_VCPU) { 2165 tlbw_use_hazard(); 2166 kvm_vz_vcpu_load_tlb(vcpu, cpu); 2167 kvm_vz_vcpu_load_wired(vcpu); 2168 } 2169 2170 /* 2171 * Restore timer state regardless, as e.g. Cause.TI can change over time 2172 * if left unmaintained. 2173 */ 2174 kvm_vz_restore_timer(vcpu); 2175 2176 /* Don't bother restoring registers multiple times unless necessary */ 2177 if (!all) 2178 return 0; 2179 2180 /* 2181 * Restore config registers first, as some implementations restrict 2182 * writes to other registers when the corresponding feature bits aren't 2183 * set. For example Status.CU1 cannot be set unless Config1.FP is set. 2184 */ 2185 kvm_restore_gc0_config(cop0); 2186 if (cpu_guest_has_conf1) 2187 kvm_restore_gc0_config1(cop0); 2188 if (cpu_guest_has_conf2) 2189 kvm_restore_gc0_config2(cop0); 2190 if (cpu_guest_has_conf3) 2191 kvm_restore_gc0_config3(cop0); 2192 if (cpu_guest_has_conf4) 2193 kvm_restore_gc0_config4(cop0); 2194 if (cpu_guest_has_conf5) 2195 kvm_restore_gc0_config5(cop0); 2196 if (cpu_guest_has_conf6) 2197 kvm_restore_gc0_config6(cop0); 2198 if (cpu_guest_has_conf7) 2199 kvm_restore_gc0_config7(cop0); 2200 2201 kvm_restore_gc0_index(cop0); 2202 kvm_restore_gc0_entrylo0(cop0); 2203 kvm_restore_gc0_entrylo1(cop0); 2204 kvm_restore_gc0_context(cop0); 2205 if (cpu_guest_has_contextconfig) 2206 kvm_restore_gc0_contextconfig(cop0); 2207 #ifdef CONFIG_64BIT 2208 kvm_restore_gc0_xcontext(cop0); 2209 if (cpu_guest_has_contextconfig) 2210 kvm_restore_gc0_xcontextconfig(cop0); 2211 #endif 2212 kvm_restore_gc0_pagemask(cop0); 2213 kvm_restore_gc0_pagegrain(cop0); 2214 kvm_restore_gc0_hwrena(cop0); 2215 kvm_restore_gc0_badvaddr(cop0); 2216 kvm_restore_gc0_entryhi(cop0); 2217 kvm_restore_gc0_status(cop0); 2218 kvm_restore_gc0_intctl(cop0); 2219 kvm_restore_gc0_epc(cop0); 2220 kvm_vz_write_gc0_ebase(kvm_read_sw_gc0_ebase(cop0)); 2221 if (cpu_guest_has_userlocal) 2222 kvm_restore_gc0_userlocal(cop0); 2223 2224 kvm_restore_gc0_errorepc(cop0); 2225 2226 /* restore KScratch registers if enabled in guest */ 2227 if (cpu_guest_has_conf4) { 2228 if (cpu_guest_has_kscr(2)) 2229 kvm_restore_gc0_kscratch1(cop0); 2230 if (cpu_guest_has_kscr(3)) 2231 kvm_restore_gc0_kscratch2(cop0); 2232 if (cpu_guest_has_kscr(4)) 2233 kvm_restore_gc0_kscratch3(cop0); 2234 if (cpu_guest_has_kscr(5)) 2235 kvm_restore_gc0_kscratch4(cop0); 2236 if (cpu_guest_has_kscr(6)) 2237 kvm_restore_gc0_kscratch5(cop0); 2238 if (cpu_guest_has_kscr(7)) 2239 kvm_restore_gc0_kscratch6(cop0); 2240 } 2241 2242 if (cpu_guest_has_badinstr) 2243 kvm_restore_gc0_badinstr(cop0); 2244 if (cpu_guest_has_badinstrp) 2245 kvm_restore_gc0_badinstrp(cop0); 2246 2247 if (cpu_guest_has_segments) { 2248 kvm_restore_gc0_segctl0(cop0); 2249 kvm_restore_gc0_segctl1(cop0); 2250 kvm_restore_gc0_segctl2(cop0); 2251 } 2252 2253 /* restore HTW registers */ 2254 if (cpu_guest_has_htw) { 2255 kvm_restore_gc0_pwbase(cop0); 2256 kvm_restore_gc0_pwfield(cop0); 2257 kvm_restore_gc0_pwsize(cop0); 2258 kvm_restore_gc0_pwctl(cop0); 2259 } 2260 2261 /* restore Root.GuestCtl2 from unused Guest guestctl2 register */ 2262 if (cpu_has_guestctl2) 2263 write_c0_guestctl2( 2264 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL]); 2265 2266 /* 2267 * We should clear linked load bit to break interrupted atomics. This 2268 * prevents a SC on the next VCPU from succeeding by matching a LL on 2269 * the previous VCPU. 2270 */ 2271 if (cpu_guest_has_rw_llb) 2272 write_gc0_lladdr(0); 2273 2274 return 0; 2275 } 2276 2277 static int kvm_vz_vcpu_put(struct kvm_vcpu *vcpu, int cpu) 2278 { 2279 struct mips_coproc *cop0 = vcpu->arch.cop0; 2280 2281 if (current->flags & PF_VCPU) 2282 kvm_vz_vcpu_save_wired(vcpu); 2283 2284 kvm_lose_fpu(vcpu); 2285 2286 kvm_save_gc0_index(cop0); 2287 kvm_save_gc0_entrylo0(cop0); 2288 kvm_save_gc0_entrylo1(cop0); 2289 kvm_save_gc0_context(cop0); 2290 if (cpu_guest_has_contextconfig) 2291 kvm_save_gc0_contextconfig(cop0); 2292 #ifdef CONFIG_64BIT 2293 kvm_save_gc0_xcontext(cop0); 2294 if (cpu_guest_has_contextconfig) 2295 kvm_save_gc0_xcontextconfig(cop0); 2296 #endif 2297 kvm_save_gc0_pagemask(cop0); 2298 kvm_save_gc0_pagegrain(cop0); 2299 kvm_save_gc0_wired(cop0); 2300 /* allow wired TLB entries to be overwritten */ 2301 clear_gc0_wired(MIPSR6_WIRED_WIRED); 2302 kvm_save_gc0_hwrena(cop0); 2303 kvm_save_gc0_badvaddr(cop0); 2304 kvm_save_gc0_entryhi(cop0); 2305 kvm_save_gc0_status(cop0); 2306 kvm_save_gc0_intctl(cop0); 2307 kvm_save_gc0_epc(cop0); 2308 kvm_write_sw_gc0_ebase(cop0, kvm_vz_read_gc0_ebase()); 2309 if (cpu_guest_has_userlocal) 2310 kvm_save_gc0_userlocal(cop0); 2311 2312 /* only save implemented config registers */ 2313 kvm_save_gc0_config(cop0); 2314 if (cpu_guest_has_conf1) 2315 kvm_save_gc0_config1(cop0); 2316 if (cpu_guest_has_conf2) 2317 kvm_save_gc0_config2(cop0); 2318 if (cpu_guest_has_conf3) 2319 kvm_save_gc0_config3(cop0); 2320 if (cpu_guest_has_conf4) 2321 kvm_save_gc0_config4(cop0); 2322 if (cpu_guest_has_conf5) 2323 kvm_save_gc0_config5(cop0); 2324 if (cpu_guest_has_conf6) 2325 kvm_save_gc0_config6(cop0); 2326 if (cpu_guest_has_conf7) 2327 kvm_save_gc0_config7(cop0); 2328 2329 kvm_save_gc0_errorepc(cop0); 2330 2331 /* save KScratch registers if enabled in guest */ 2332 if (cpu_guest_has_conf4) { 2333 if (cpu_guest_has_kscr(2)) 2334 kvm_save_gc0_kscratch1(cop0); 2335 if (cpu_guest_has_kscr(3)) 2336 kvm_save_gc0_kscratch2(cop0); 2337 if (cpu_guest_has_kscr(4)) 2338 kvm_save_gc0_kscratch3(cop0); 2339 if (cpu_guest_has_kscr(5)) 2340 kvm_save_gc0_kscratch4(cop0); 2341 if (cpu_guest_has_kscr(6)) 2342 kvm_save_gc0_kscratch5(cop0); 2343 if (cpu_guest_has_kscr(7)) 2344 kvm_save_gc0_kscratch6(cop0); 2345 } 2346 2347 if (cpu_guest_has_badinstr) 2348 kvm_save_gc0_badinstr(cop0); 2349 if (cpu_guest_has_badinstrp) 2350 kvm_save_gc0_badinstrp(cop0); 2351 2352 if (cpu_guest_has_segments) { 2353 kvm_save_gc0_segctl0(cop0); 2354 kvm_save_gc0_segctl1(cop0); 2355 kvm_save_gc0_segctl2(cop0); 2356 } 2357 2358 /* save HTW registers if enabled in guest */ 2359 if (cpu_guest_has_htw && 2360 kvm_read_sw_gc0_config3(cop0) & MIPS_CONF3_PW) { 2361 kvm_save_gc0_pwbase(cop0); 2362 kvm_save_gc0_pwfield(cop0); 2363 kvm_save_gc0_pwsize(cop0); 2364 kvm_save_gc0_pwctl(cop0); 2365 } 2366 2367 kvm_vz_save_timer(vcpu); 2368 2369 /* save Root.GuestCtl2 in unused Guest guestctl2 register */ 2370 if (cpu_has_guestctl2) 2371 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL] = 2372 read_c0_guestctl2(); 2373 2374 return 0; 2375 } 2376 2377 /** 2378 * kvm_vz_resize_guest_vtlb() - Attempt to resize guest VTLB. 2379 * @size: Number of guest VTLB entries (0 < @size <= root VTLB entries). 2380 * 2381 * Attempt to resize the guest VTLB by writing guest Config registers. This is 2382 * necessary for cores with a shared root/guest TLB to avoid overlap with wired 2383 * entries in the root VTLB. 2384 * 2385 * Returns: The resulting guest VTLB size. 2386 */ 2387 static unsigned int kvm_vz_resize_guest_vtlb(unsigned int size) 2388 { 2389 unsigned int config4 = 0, ret = 0, limit; 2390 2391 /* Write MMUSize - 1 into guest Config registers */ 2392 if (cpu_guest_has_conf1) 2393 change_gc0_config1(MIPS_CONF1_TLBS, 2394 (size - 1) << MIPS_CONF1_TLBS_SHIFT); 2395 if (cpu_guest_has_conf4) { 2396 config4 = read_gc0_config4(); 2397 if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) == 2398 MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT) { 2399 config4 &= ~MIPS_CONF4_VTLBSIZEEXT; 2400 config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) << 2401 MIPS_CONF4_VTLBSIZEEXT_SHIFT; 2402 } else if ((config4 & MIPS_CONF4_MMUEXTDEF) == 2403 MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT) { 2404 config4 &= ~MIPS_CONF4_MMUSIZEEXT; 2405 config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) << 2406 MIPS_CONF4_MMUSIZEEXT_SHIFT; 2407 } 2408 write_gc0_config4(config4); 2409 } 2410 2411 /* 2412 * Set Guest.Wired.Limit = 0 (no limit up to Guest.MMUSize-1), unless it 2413 * would exceed Root.Wired.Limit (clearing Guest.Wired.Wired so write 2414 * not dropped) 2415 */ 2416 if (cpu_has_mips_r6) { 2417 limit = (read_c0_wired() & MIPSR6_WIRED_LIMIT) >> 2418 MIPSR6_WIRED_LIMIT_SHIFT; 2419 if (size - 1 <= limit) 2420 limit = 0; 2421 write_gc0_wired(limit << MIPSR6_WIRED_LIMIT_SHIFT); 2422 } 2423 2424 /* Read back MMUSize - 1 */ 2425 back_to_back_c0_hazard(); 2426 if (cpu_guest_has_conf1) 2427 ret = (read_gc0_config1() & MIPS_CONF1_TLBS) >> 2428 MIPS_CONF1_TLBS_SHIFT; 2429 if (config4) { 2430 if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) == 2431 MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT) 2432 ret |= ((config4 & MIPS_CONF4_VTLBSIZEEXT) >> 2433 MIPS_CONF4_VTLBSIZEEXT_SHIFT) << 2434 MIPS_CONF1_TLBS_SIZE; 2435 else if ((config4 & MIPS_CONF4_MMUEXTDEF) == 2436 MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT) 2437 ret |= ((config4 & MIPS_CONF4_MMUSIZEEXT) >> 2438 MIPS_CONF4_MMUSIZEEXT_SHIFT) << 2439 MIPS_CONF1_TLBS_SIZE; 2440 } 2441 return ret + 1; 2442 } 2443 2444 static int kvm_vz_hardware_enable(void) 2445 { 2446 unsigned int mmu_size, guest_mmu_size, ftlb_size; 2447 2448 /* 2449 * ImgTec cores tend to use a shared root/guest TLB. To avoid overlap of 2450 * root wired and guest entries, the guest TLB may need resizing. 2451 */ 2452 mmu_size = current_cpu_data.tlbsizevtlb; 2453 ftlb_size = current_cpu_data.tlbsize - mmu_size; 2454 2455 /* Try switching to maximum guest VTLB size for flush */ 2456 guest_mmu_size = kvm_vz_resize_guest_vtlb(mmu_size); 2457 current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size; 2458 kvm_vz_local_flush_guesttlb_all(); 2459 2460 /* 2461 * Reduce to make space for root wired entries and at least 2 root 2462 * non-wired entries. This does assume that long-term wired entries 2463 * won't be added later. 2464 */ 2465 guest_mmu_size = mmu_size - num_wired_entries() - 2; 2466 guest_mmu_size = kvm_vz_resize_guest_vtlb(guest_mmu_size); 2467 current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size; 2468 2469 /* 2470 * Write the VTLB size, but if another CPU has already written, check it 2471 * matches or we won't provide a consistent view to the guest. If this 2472 * ever happens it suggests an asymmetric number of wired entries. 2473 */ 2474 if (cmpxchg(&kvm_vz_guest_vtlb_size, 0, guest_mmu_size) && 2475 WARN(guest_mmu_size != kvm_vz_guest_vtlb_size, 2476 "Available guest VTLB size mismatch")) 2477 return -EINVAL; 2478 2479 /* 2480 * Enable virtualization features granting guest direct control of 2481 * certain features: 2482 * CP0=1: Guest coprocessor 0 context. 2483 * AT=Guest: Guest MMU. 2484 * CG=1: Hit (virtual address) CACHE operations (optional). 2485 * CF=1: Guest Config registers. 2486 * CGI=1: Indexed flush CACHE operations (optional). 2487 */ 2488 write_c0_guestctl0(MIPS_GCTL0_CP0 | 2489 (MIPS_GCTL0_AT_GUEST << MIPS_GCTL0_AT_SHIFT) | 2490 MIPS_GCTL0_CG | MIPS_GCTL0_CF); 2491 if (cpu_has_guestctl0ext) 2492 set_c0_guestctl0ext(MIPS_GCTL0EXT_CGI); 2493 2494 if (cpu_has_guestid) { 2495 write_c0_guestctl1(0); 2496 kvm_vz_local_flush_roottlb_all_guests(); 2497 2498 GUESTID_MASK = current_cpu_data.guestid_mask; 2499 GUESTID_FIRST_VERSION = GUESTID_MASK + 1; 2500 GUESTID_VERSION_MASK = ~GUESTID_MASK; 2501 2502 current_cpu_data.guestid_cache = GUESTID_FIRST_VERSION; 2503 } 2504 2505 /* clear any pending injected virtual guest interrupts */ 2506 if (cpu_has_guestctl2) 2507 clear_c0_guestctl2(0x3f << 10); 2508 2509 return 0; 2510 } 2511 2512 static void kvm_vz_hardware_disable(void) 2513 { 2514 kvm_vz_local_flush_guesttlb_all(); 2515 2516 if (cpu_has_guestid) { 2517 write_c0_guestctl1(0); 2518 kvm_vz_local_flush_roottlb_all_guests(); 2519 } 2520 } 2521 2522 static int kvm_vz_check_extension(struct kvm *kvm, long ext) 2523 { 2524 int r; 2525 2526 switch (ext) { 2527 case KVM_CAP_MIPS_VZ: 2528 /* we wouldn't be here unless cpu_has_vz */ 2529 r = 1; 2530 break; 2531 #ifdef CONFIG_64BIT 2532 case KVM_CAP_MIPS_64BIT: 2533 /* We support 64-bit registers/operations and addresses */ 2534 r = 2; 2535 break; 2536 #endif 2537 default: 2538 r = 0; 2539 break; 2540 } 2541 2542 return r; 2543 } 2544 2545 static int kvm_vz_vcpu_init(struct kvm_vcpu *vcpu) 2546 { 2547 int i; 2548 2549 for_each_possible_cpu(i) 2550 vcpu->arch.vzguestid[i] = 0; 2551 2552 return 0; 2553 } 2554 2555 static void kvm_vz_vcpu_uninit(struct kvm_vcpu *vcpu) 2556 { 2557 int cpu; 2558 2559 /* 2560 * If the VCPU is freed and reused as another VCPU, we don't want the 2561 * matching pointer wrongly hanging around in last_vcpu[] or 2562 * last_exec_vcpu[]. 2563 */ 2564 for_each_possible_cpu(cpu) { 2565 if (last_vcpu[cpu] == vcpu) 2566 last_vcpu[cpu] = NULL; 2567 if (last_exec_vcpu[cpu] == vcpu) 2568 last_exec_vcpu[cpu] = NULL; 2569 } 2570 } 2571 2572 static int kvm_vz_vcpu_setup(struct kvm_vcpu *vcpu) 2573 { 2574 struct mips_coproc *cop0 = vcpu->arch.cop0; 2575 unsigned long count_hz = 100*1000*1000; /* default to 100 MHz */ 2576 2577 /* 2578 * Start off the timer at the same frequency as the host timer, but the 2579 * soft timer doesn't handle frequencies greater than 1GHz yet. 2580 */ 2581 if (mips_hpt_frequency && mips_hpt_frequency <= NSEC_PER_SEC) 2582 count_hz = mips_hpt_frequency; 2583 kvm_mips_init_count(vcpu, count_hz); 2584 2585 /* 2586 * Initialize guest register state to valid architectural reset state. 2587 */ 2588 2589 /* PageGrain */ 2590 if (cpu_has_mips_r6) 2591 kvm_write_sw_gc0_pagegrain(cop0, PG_RIE | PG_XIE | PG_IEC); 2592 /* Wired */ 2593 if (cpu_has_mips_r6) 2594 kvm_write_sw_gc0_wired(cop0, 2595 read_gc0_wired() & MIPSR6_WIRED_LIMIT); 2596 /* Status */ 2597 kvm_write_sw_gc0_status(cop0, ST0_BEV | ST0_ERL); 2598 if (cpu_has_mips_r6) 2599 kvm_change_sw_gc0_status(cop0, ST0_FR, read_gc0_status()); 2600 /* IntCtl */ 2601 kvm_write_sw_gc0_intctl(cop0, read_gc0_intctl() & 2602 (INTCTLF_IPFDC | INTCTLF_IPPCI | INTCTLF_IPTI)); 2603 /* PRId */ 2604 kvm_write_sw_gc0_prid(cop0, boot_cpu_data.processor_id); 2605 /* EBase */ 2606 kvm_write_sw_gc0_ebase(cop0, (s32)0x80000000 | vcpu->vcpu_id); 2607 /* Config */ 2608 kvm_save_gc0_config(cop0); 2609 /* architecturally writable (e.g. from guest) */ 2610 kvm_change_sw_gc0_config(cop0, CONF_CM_CMASK, 2611 _page_cachable_default >> _CACHE_SHIFT); 2612 /* architecturally read only, but maybe writable from root */ 2613 kvm_change_sw_gc0_config(cop0, MIPS_CONF_MT, read_c0_config()); 2614 if (cpu_guest_has_conf1) { 2615 kvm_set_sw_gc0_config(cop0, MIPS_CONF_M); 2616 /* Config1 */ 2617 kvm_save_gc0_config1(cop0); 2618 /* architecturally read only, but maybe writable from root */ 2619 kvm_clear_sw_gc0_config1(cop0, MIPS_CONF1_C2 | 2620 MIPS_CONF1_MD | 2621 MIPS_CONF1_PC | 2622 MIPS_CONF1_WR | 2623 MIPS_CONF1_CA | 2624 MIPS_CONF1_FP); 2625 } 2626 if (cpu_guest_has_conf2) { 2627 kvm_set_sw_gc0_config1(cop0, MIPS_CONF_M); 2628 /* Config2 */ 2629 kvm_save_gc0_config2(cop0); 2630 } 2631 if (cpu_guest_has_conf3) { 2632 kvm_set_sw_gc0_config2(cop0, MIPS_CONF_M); 2633 /* Config3 */ 2634 kvm_save_gc0_config3(cop0); 2635 /* architecturally writable (e.g. from guest) */ 2636 kvm_clear_sw_gc0_config3(cop0, MIPS_CONF3_ISA_OE); 2637 /* architecturally read only, but maybe writable from root */ 2638 kvm_clear_sw_gc0_config3(cop0, MIPS_CONF3_MSA | 2639 MIPS_CONF3_BPG | 2640 MIPS_CONF3_ULRI | 2641 MIPS_CONF3_DSP | 2642 MIPS_CONF3_CTXTC | 2643 MIPS_CONF3_ITL | 2644 MIPS_CONF3_LPA | 2645 MIPS_CONF3_VEIC | 2646 MIPS_CONF3_VINT | 2647 MIPS_CONF3_SP | 2648 MIPS_CONF3_CDMM | 2649 MIPS_CONF3_MT | 2650 MIPS_CONF3_SM | 2651 MIPS_CONF3_TL); 2652 } 2653 if (cpu_guest_has_conf4) { 2654 kvm_set_sw_gc0_config3(cop0, MIPS_CONF_M); 2655 /* Config4 */ 2656 kvm_save_gc0_config4(cop0); 2657 } 2658 if (cpu_guest_has_conf5) { 2659 kvm_set_sw_gc0_config4(cop0, MIPS_CONF_M); 2660 /* Config5 */ 2661 kvm_save_gc0_config5(cop0); 2662 /* architecturally writable (e.g. from guest) */ 2663 kvm_clear_sw_gc0_config5(cop0, MIPS_CONF5_K | 2664 MIPS_CONF5_CV | 2665 MIPS_CONF5_MSAEN | 2666 MIPS_CONF5_UFE | 2667 MIPS_CONF5_FRE | 2668 MIPS_CONF5_SBRI | 2669 MIPS_CONF5_UFR); 2670 /* architecturally read only, but maybe writable from root */ 2671 kvm_clear_sw_gc0_config5(cop0, MIPS_CONF5_MRP); 2672 } 2673 2674 if (cpu_guest_has_contextconfig) { 2675 /* ContextConfig */ 2676 kvm_write_sw_gc0_contextconfig(cop0, 0x007ffff0); 2677 #ifdef CONFIG_64BIT 2678 /* XContextConfig */ 2679 /* bits SEGBITS-13+3:4 set */ 2680 kvm_write_sw_gc0_xcontextconfig(cop0, 2681 ((1ull << (cpu_vmbits - 13)) - 1) << 4); 2682 #endif 2683 } 2684 2685 /* Implementation dependent, use the legacy layout */ 2686 if (cpu_guest_has_segments) { 2687 /* SegCtl0, SegCtl1, SegCtl2 */ 2688 kvm_write_sw_gc0_segctl0(cop0, 0x00200010); 2689 kvm_write_sw_gc0_segctl1(cop0, 0x00000002 | 2690 (_page_cachable_default >> _CACHE_SHIFT) << 2691 (16 + MIPS_SEGCFG_C_SHIFT)); 2692 kvm_write_sw_gc0_segctl2(cop0, 0x00380438); 2693 } 2694 2695 /* reset HTW registers */ 2696 if (cpu_guest_has_htw && cpu_has_mips_r6) { 2697 /* PWField */ 2698 kvm_write_sw_gc0_pwfield(cop0, 0x0c30c302); 2699 /* PWSize */ 2700 kvm_write_sw_gc0_pwsize(cop0, 1 << MIPS_PWSIZE_PTW_SHIFT); 2701 } 2702 2703 /* start with no pending virtual guest interrupts */ 2704 if (cpu_has_guestctl2) 2705 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL] = 0; 2706 2707 /* Put PC at reset vector */ 2708 vcpu->arch.pc = CKSEG1ADDR(0x1fc00000); 2709 2710 return 0; 2711 } 2712 2713 static void kvm_vz_flush_shadow_all(struct kvm *kvm) 2714 { 2715 if (cpu_has_guestid) { 2716 /* Flush GuestID for each VCPU individually */ 2717 kvm_flush_remote_tlbs(kvm); 2718 } else { 2719 /* 2720 * For each CPU there is a single GPA ASID used by all VCPUs in 2721 * the VM, so it doesn't make sense for the VCPUs to handle 2722 * invalidation of these ASIDs individually. 2723 * 2724 * Instead mark all CPUs as needing ASID invalidation in 2725 * asid_flush_mask, and just use kvm_flush_remote_tlbs(kvm) to 2726 * kick any running VCPUs so they check asid_flush_mask. 2727 */ 2728 cpumask_setall(&kvm->arch.asid_flush_mask); 2729 kvm_flush_remote_tlbs(kvm); 2730 } 2731 } 2732 2733 static void kvm_vz_flush_shadow_memslot(struct kvm *kvm, 2734 const struct kvm_memory_slot *slot) 2735 { 2736 kvm_vz_flush_shadow_all(kvm); 2737 } 2738 2739 static void kvm_vz_vcpu_reenter(struct kvm_run *run, struct kvm_vcpu *vcpu) 2740 { 2741 int cpu = smp_processor_id(); 2742 int preserve_guest_tlb; 2743 2744 preserve_guest_tlb = kvm_vz_check_requests(vcpu, cpu); 2745 2746 if (preserve_guest_tlb) 2747 kvm_vz_vcpu_save_wired(vcpu); 2748 2749 kvm_vz_vcpu_load_tlb(vcpu, cpu); 2750 2751 if (preserve_guest_tlb) 2752 kvm_vz_vcpu_load_wired(vcpu); 2753 } 2754 2755 static int kvm_vz_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu) 2756 { 2757 int cpu = smp_processor_id(); 2758 int r; 2759 2760 /* Check if we have any exceptions/interrupts pending */ 2761 kvm_mips_deliver_interrupts(vcpu, read_gc0_cause()); 2762 2763 kvm_vz_check_requests(vcpu, cpu); 2764 kvm_vz_vcpu_load_tlb(vcpu, cpu); 2765 kvm_vz_vcpu_load_wired(vcpu); 2766 2767 r = vcpu->arch.vcpu_run(run, vcpu); 2768 2769 kvm_vz_vcpu_save_wired(vcpu); 2770 2771 return r; 2772 } 2773 2774 static struct kvm_mips_callbacks kvm_vz_callbacks = { 2775 .handle_cop_unusable = kvm_trap_vz_handle_cop_unusable, 2776 .handle_tlb_mod = kvm_trap_vz_handle_tlb_st_miss, 2777 .handle_tlb_ld_miss = kvm_trap_vz_handle_tlb_ld_miss, 2778 .handle_tlb_st_miss = kvm_trap_vz_handle_tlb_st_miss, 2779 .handle_addr_err_st = kvm_trap_vz_no_handler, 2780 .handle_addr_err_ld = kvm_trap_vz_no_handler, 2781 .handle_syscall = kvm_trap_vz_no_handler, 2782 .handle_res_inst = kvm_trap_vz_no_handler, 2783 .handle_break = kvm_trap_vz_no_handler, 2784 .handle_msa_disabled = kvm_trap_vz_handle_msa_disabled, 2785 .handle_guest_exit = kvm_trap_vz_handle_guest_exit, 2786 2787 .hardware_enable = kvm_vz_hardware_enable, 2788 .hardware_disable = kvm_vz_hardware_disable, 2789 .check_extension = kvm_vz_check_extension, 2790 .vcpu_init = kvm_vz_vcpu_init, 2791 .vcpu_uninit = kvm_vz_vcpu_uninit, 2792 .vcpu_setup = kvm_vz_vcpu_setup, 2793 .flush_shadow_all = kvm_vz_flush_shadow_all, 2794 .flush_shadow_memslot = kvm_vz_flush_shadow_memslot, 2795 .gva_to_gpa = kvm_vz_gva_to_gpa_cb, 2796 .queue_timer_int = kvm_vz_queue_timer_int_cb, 2797 .dequeue_timer_int = kvm_vz_dequeue_timer_int_cb, 2798 .queue_io_int = kvm_vz_queue_io_int_cb, 2799 .dequeue_io_int = kvm_vz_dequeue_io_int_cb, 2800 .irq_deliver = kvm_vz_irq_deliver_cb, 2801 .irq_clear = kvm_vz_irq_clear_cb, 2802 .num_regs = kvm_vz_num_regs, 2803 .copy_reg_indices = kvm_vz_copy_reg_indices, 2804 .get_one_reg = kvm_vz_get_one_reg, 2805 .set_one_reg = kvm_vz_set_one_reg, 2806 .vcpu_load = kvm_vz_vcpu_load, 2807 .vcpu_put = kvm_vz_vcpu_put, 2808 .vcpu_run = kvm_vz_vcpu_run, 2809 .vcpu_reenter = kvm_vz_vcpu_reenter, 2810 }; 2811 2812 int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks) 2813 { 2814 if (!cpu_has_vz) 2815 return -ENODEV; 2816 2817 /* 2818 * VZ requires at least 2 KScratch registers, so it should have been 2819 * possible to allocate pgd_reg. 2820 */ 2821 if (WARN(pgd_reg == -1, 2822 "pgd_reg not allocated even though cpu_has_vz\n")) 2823 return -ENODEV; 2824 2825 pr_info("Starting KVM with MIPS VZ extensions\n"); 2826 2827 *install_callbacks = &kvm_vz_callbacks; 2828 return 0; 2829 } 2830