1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * KVM/MIPS: Support for hardware virtualization extensions 7 * 8 * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. 9 * Authors: Yann Le Du <ledu@kymasys.com> 10 */ 11 12 #include <linux/errno.h> 13 #include <linux/err.h> 14 #include <linux/module.h> 15 #include <linux/preempt.h> 16 #include <linux/vmalloc.h> 17 #include <asm/cacheflush.h> 18 #include <asm/cacheops.h> 19 #include <asm/cmpxchg.h> 20 #include <asm/fpu.h> 21 #include <asm/hazards.h> 22 #include <asm/inst.h> 23 #include <asm/mmu_context.h> 24 #include <asm/r4kcache.h> 25 #include <asm/time.h> 26 #include <asm/tlb.h> 27 #include <asm/tlbex.h> 28 29 #include <linux/kvm_host.h> 30 31 #include "interrupt.h" 32 #ifdef CONFIG_CPU_LOONGSON64 33 #include "loongson_regs.h" 34 #endif 35 36 #include "trace.h" 37 38 /* Pointers to last VCPU loaded on each physical CPU */ 39 static struct kvm_vcpu *last_vcpu[NR_CPUS]; 40 /* Pointers to last VCPU executed on each physical CPU */ 41 static struct kvm_vcpu *last_exec_vcpu[NR_CPUS]; 42 43 /* 44 * Number of guest VTLB entries to use, so we can catch inconsistency between 45 * CPUs. 46 */ 47 static unsigned int kvm_vz_guest_vtlb_size; 48 49 static inline long kvm_vz_read_gc0_ebase(void) 50 { 51 if (sizeof(long) == 8 && cpu_has_ebase_wg) 52 return read_gc0_ebase_64(); 53 else 54 return read_gc0_ebase(); 55 } 56 57 static inline void kvm_vz_write_gc0_ebase(long v) 58 { 59 /* 60 * First write with WG=1 to write upper bits, then write again in case 61 * WG should be left at 0. 62 * write_gc0_ebase_64() is no longer UNDEFINED since R6. 63 */ 64 if (sizeof(long) == 8 && 65 (cpu_has_mips64r6 || cpu_has_ebase_wg)) { 66 write_gc0_ebase_64(v | MIPS_EBASE_WG); 67 write_gc0_ebase_64(v); 68 } else { 69 write_gc0_ebase(v | MIPS_EBASE_WG); 70 write_gc0_ebase(v); 71 } 72 } 73 74 /* 75 * These Config bits may be writable by the guest: 76 * Config: [K23, KU] (!TLB), K0 77 * Config1: (none) 78 * Config2: [TU, SU] (impl) 79 * Config3: ISAOnExc 80 * Config4: FTLBPageSize 81 * Config5: K, CV, MSAEn, UFE, FRE, SBRI, UFR 82 */ 83 84 static inline unsigned int kvm_vz_config_guest_wrmask(struct kvm_vcpu *vcpu) 85 { 86 return CONF_CM_CMASK; 87 } 88 89 static inline unsigned int kvm_vz_config1_guest_wrmask(struct kvm_vcpu *vcpu) 90 { 91 return 0; 92 } 93 94 static inline unsigned int kvm_vz_config2_guest_wrmask(struct kvm_vcpu *vcpu) 95 { 96 return 0; 97 } 98 99 static inline unsigned int kvm_vz_config3_guest_wrmask(struct kvm_vcpu *vcpu) 100 { 101 return MIPS_CONF3_ISA_OE; 102 } 103 104 static inline unsigned int kvm_vz_config4_guest_wrmask(struct kvm_vcpu *vcpu) 105 { 106 /* no need to be exact */ 107 return MIPS_CONF4_VFTLBPAGESIZE; 108 } 109 110 static inline unsigned int kvm_vz_config5_guest_wrmask(struct kvm_vcpu *vcpu) 111 { 112 unsigned int mask = MIPS_CONF5_K | MIPS_CONF5_CV | MIPS_CONF5_SBRI; 113 114 /* Permit MSAEn changes if MSA supported and enabled */ 115 if (kvm_mips_guest_has_msa(&vcpu->arch)) 116 mask |= MIPS_CONF5_MSAEN; 117 118 /* 119 * Permit guest FPU mode changes if FPU is enabled and the relevant 120 * feature exists according to FIR register. 121 */ 122 if (kvm_mips_guest_has_fpu(&vcpu->arch)) { 123 if (cpu_has_ufr) 124 mask |= MIPS_CONF5_UFR; 125 if (cpu_has_fre) 126 mask |= MIPS_CONF5_FRE | MIPS_CONF5_UFE; 127 } 128 129 return mask; 130 } 131 132 static inline unsigned int kvm_vz_config6_guest_wrmask(struct kvm_vcpu *vcpu) 133 { 134 return LOONGSON_CONF6_INTIMER | LOONGSON_CONF6_EXTIMER; 135 } 136 137 /* 138 * VZ optionally allows these additional Config bits to be written by root: 139 * Config: M, [MT] 140 * Config1: M, [MMUSize-1, C2, MD, PC, WR, CA], FP 141 * Config2: M 142 * Config3: M, MSAP, [BPG], ULRI, [DSP2P, DSPP], CTXTC, [ITL, LPA, VEIC, 143 * VInt, SP, CDMM, MT, SM, TL] 144 * Config4: M, [VTLBSizeExt, MMUSizeExt] 145 * Config5: MRP 146 */ 147 148 static inline unsigned int kvm_vz_config_user_wrmask(struct kvm_vcpu *vcpu) 149 { 150 return kvm_vz_config_guest_wrmask(vcpu) | MIPS_CONF_M; 151 } 152 153 static inline unsigned int kvm_vz_config1_user_wrmask(struct kvm_vcpu *vcpu) 154 { 155 unsigned int mask = kvm_vz_config1_guest_wrmask(vcpu) | MIPS_CONF_M; 156 157 /* Permit FPU to be present if FPU is supported */ 158 if (kvm_mips_guest_can_have_fpu(&vcpu->arch)) 159 mask |= MIPS_CONF1_FP; 160 161 return mask; 162 } 163 164 static inline unsigned int kvm_vz_config2_user_wrmask(struct kvm_vcpu *vcpu) 165 { 166 return kvm_vz_config2_guest_wrmask(vcpu) | MIPS_CONF_M; 167 } 168 169 static inline unsigned int kvm_vz_config3_user_wrmask(struct kvm_vcpu *vcpu) 170 { 171 unsigned int mask = kvm_vz_config3_guest_wrmask(vcpu) | MIPS_CONF_M | 172 MIPS_CONF3_ULRI | MIPS_CONF3_CTXTC; 173 174 /* Permit MSA to be present if MSA is supported */ 175 if (kvm_mips_guest_can_have_msa(&vcpu->arch)) 176 mask |= MIPS_CONF3_MSA; 177 178 return mask; 179 } 180 181 static inline unsigned int kvm_vz_config4_user_wrmask(struct kvm_vcpu *vcpu) 182 { 183 return kvm_vz_config4_guest_wrmask(vcpu) | MIPS_CONF_M; 184 } 185 186 static inline unsigned int kvm_vz_config5_user_wrmask(struct kvm_vcpu *vcpu) 187 { 188 return kvm_vz_config5_guest_wrmask(vcpu) | MIPS_CONF5_MRP; 189 } 190 191 static inline unsigned int kvm_vz_config6_user_wrmask(struct kvm_vcpu *vcpu) 192 { 193 return kvm_vz_config6_guest_wrmask(vcpu) | 194 LOONGSON_CONF6_SFBEN | LOONGSON_CONF6_FTLBDIS; 195 } 196 197 static gpa_t kvm_vz_gva_to_gpa_cb(gva_t gva) 198 { 199 /* VZ guest has already converted gva to gpa */ 200 return gva; 201 } 202 203 static void kvm_vz_queue_irq(struct kvm_vcpu *vcpu, unsigned int priority) 204 { 205 set_bit(priority, &vcpu->arch.pending_exceptions); 206 clear_bit(priority, &vcpu->arch.pending_exceptions_clr); 207 } 208 209 static void kvm_vz_dequeue_irq(struct kvm_vcpu *vcpu, unsigned int priority) 210 { 211 clear_bit(priority, &vcpu->arch.pending_exceptions); 212 set_bit(priority, &vcpu->arch.pending_exceptions_clr); 213 } 214 215 static void kvm_vz_queue_timer_int_cb(struct kvm_vcpu *vcpu) 216 { 217 /* 218 * timer expiry is asynchronous to vcpu execution therefore defer guest 219 * cp0 accesses 220 */ 221 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER); 222 } 223 224 static void kvm_vz_dequeue_timer_int_cb(struct kvm_vcpu *vcpu) 225 { 226 /* 227 * timer expiry is asynchronous to vcpu execution therefore defer guest 228 * cp0 accesses 229 */ 230 kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_TIMER); 231 } 232 233 static void kvm_vz_queue_io_int_cb(struct kvm_vcpu *vcpu, 234 struct kvm_mips_interrupt *irq) 235 { 236 int intr = (int)irq->irq; 237 238 /* 239 * interrupts are asynchronous to vcpu execution therefore defer guest 240 * cp0 accesses 241 */ 242 kvm_vz_queue_irq(vcpu, kvm_irq_to_priority(intr)); 243 } 244 245 static void kvm_vz_dequeue_io_int_cb(struct kvm_vcpu *vcpu, 246 struct kvm_mips_interrupt *irq) 247 { 248 int intr = (int)irq->irq; 249 250 /* 251 * interrupts are asynchronous to vcpu execution therefore defer guest 252 * cp0 accesses 253 */ 254 kvm_vz_dequeue_irq(vcpu, kvm_irq_to_priority(-intr)); 255 } 256 257 static int kvm_vz_irq_deliver_cb(struct kvm_vcpu *vcpu, unsigned int priority, 258 u32 cause) 259 { 260 u32 irq = (priority < MIPS_EXC_MAX) ? 261 kvm_priority_to_irq[priority] : 0; 262 263 switch (priority) { 264 case MIPS_EXC_INT_TIMER: 265 set_gc0_cause(C_TI); 266 break; 267 268 case MIPS_EXC_INT_IO_1: 269 case MIPS_EXC_INT_IO_2: 270 case MIPS_EXC_INT_IPI_1: 271 case MIPS_EXC_INT_IPI_2: 272 if (cpu_has_guestctl2) 273 set_c0_guestctl2(irq); 274 else 275 set_gc0_cause(irq); 276 break; 277 278 default: 279 break; 280 } 281 282 clear_bit(priority, &vcpu->arch.pending_exceptions); 283 return 1; 284 } 285 286 static int kvm_vz_irq_clear_cb(struct kvm_vcpu *vcpu, unsigned int priority, 287 u32 cause) 288 { 289 u32 irq = (priority < MIPS_EXC_MAX) ? 290 kvm_priority_to_irq[priority] : 0; 291 292 switch (priority) { 293 case MIPS_EXC_INT_TIMER: 294 /* 295 * Explicitly clear irq associated with Cause.IP[IPTI] 296 * if GuestCtl2 virtual interrupt register not 297 * supported or if not using GuestCtl2 Hardware Clear. 298 */ 299 if (cpu_has_guestctl2) { 300 if (!(read_c0_guestctl2() & (irq << 14))) 301 clear_c0_guestctl2(irq); 302 } else { 303 clear_gc0_cause(irq); 304 } 305 break; 306 307 case MIPS_EXC_INT_IO_1: 308 case MIPS_EXC_INT_IO_2: 309 case MIPS_EXC_INT_IPI_1: 310 case MIPS_EXC_INT_IPI_2: 311 /* Clear GuestCtl2.VIP irq if not using Hardware Clear */ 312 if (cpu_has_guestctl2) { 313 if (!(read_c0_guestctl2() & (irq << 14))) 314 clear_c0_guestctl2(irq); 315 } else { 316 clear_gc0_cause(irq); 317 } 318 break; 319 320 default: 321 break; 322 } 323 324 clear_bit(priority, &vcpu->arch.pending_exceptions_clr); 325 return 1; 326 } 327 328 /* 329 * VZ guest timer handling. 330 */ 331 332 /** 333 * kvm_vz_should_use_htimer() - Find whether to use the VZ hard guest timer. 334 * @vcpu: Virtual CPU. 335 * 336 * Returns: true if the VZ GTOffset & real guest CP0_Count should be used 337 * instead of software emulation of guest timer. 338 * false otherwise. 339 */ 340 static bool kvm_vz_should_use_htimer(struct kvm_vcpu *vcpu) 341 { 342 if (kvm_mips_count_disabled(vcpu)) 343 return false; 344 345 /* Chosen frequency must match real frequency */ 346 if (mips_hpt_frequency != vcpu->arch.count_hz) 347 return false; 348 349 /* We don't support a CP0_GTOffset with fewer bits than CP0_Count */ 350 if (current_cpu_data.gtoffset_mask != 0xffffffff) 351 return false; 352 353 return true; 354 } 355 356 /** 357 * _kvm_vz_restore_stimer() - Restore soft timer state. 358 * @vcpu: Virtual CPU. 359 * @compare: CP0_Compare register value, restored by caller. 360 * @cause: CP0_Cause register to restore. 361 * 362 * Restore VZ state relating to the soft timer. The hard timer can be enabled 363 * later. 364 */ 365 static void _kvm_vz_restore_stimer(struct kvm_vcpu *vcpu, u32 compare, 366 u32 cause) 367 { 368 /* 369 * Avoid spurious counter interrupts by setting Guest CP0_Count to just 370 * after Guest CP0_Compare. 371 */ 372 write_c0_gtoffset(compare - read_c0_count()); 373 374 back_to_back_c0_hazard(); 375 write_gc0_cause(cause); 376 } 377 378 /** 379 * _kvm_vz_restore_htimer() - Restore hard timer state. 380 * @vcpu: Virtual CPU. 381 * @compare: CP0_Compare register value, restored by caller. 382 * @cause: CP0_Cause register to restore. 383 * 384 * Restore hard timer Guest.Count & Guest.Cause taking care to preserve the 385 * value of Guest.CP0_Cause.TI while restoring Guest.CP0_Cause. 386 */ 387 static void _kvm_vz_restore_htimer(struct kvm_vcpu *vcpu, 388 u32 compare, u32 cause) 389 { 390 u32 start_count, after_count; 391 unsigned long flags; 392 393 /* 394 * Freeze the soft-timer and sync the guest CP0_Count with it. We do 395 * this with interrupts disabled to avoid latency. 396 */ 397 local_irq_save(flags); 398 kvm_mips_freeze_hrtimer(vcpu, &start_count); 399 write_c0_gtoffset(start_count - read_c0_count()); 400 local_irq_restore(flags); 401 402 /* restore guest CP0_Cause, as TI may already be set */ 403 back_to_back_c0_hazard(); 404 write_gc0_cause(cause); 405 406 /* 407 * The above sequence isn't atomic and would result in lost timer 408 * interrupts if we're not careful. Detect if a timer interrupt is due 409 * and assert it. 410 */ 411 back_to_back_c0_hazard(); 412 after_count = read_gc0_count(); 413 if (after_count - start_count > compare - start_count - 1) 414 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER); 415 } 416 417 /** 418 * kvm_vz_restore_timer() - Restore timer state. 419 * @vcpu: Virtual CPU. 420 * 421 * Restore soft timer state from saved context. 422 */ 423 static void kvm_vz_restore_timer(struct kvm_vcpu *vcpu) 424 { 425 struct mips_coproc *cop0 = &vcpu->arch.cop0; 426 u32 cause, compare; 427 428 compare = kvm_read_sw_gc0_compare(cop0); 429 cause = kvm_read_sw_gc0_cause(cop0); 430 431 write_gc0_compare(compare); 432 _kvm_vz_restore_stimer(vcpu, compare, cause); 433 } 434 435 /** 436 * kvm_vz_acquire_htimer() - Switch to hard timer state. 437 * @vcpu: Virtual CPU. 438 * 439 * Restore hard timer state on top of existing soft timer state if possible. 440 * 441 * Since hard timer won't remain active over preemption, preemption should be 442 * disabled by the caller. 443 */ 444 void kvm_vz_acquire_htimer(struct kvm_vcpu *vcpu) 445 { 446 u32 gctl0; 447 448 gctl0 = read_c0_guestctl0(); 449 if (!(gctl0 & MIPS_GCTL0_GT) && kvm_vz_should_use_htimer(vcpu)) { 450 /* enable guest access to hard timer */ 451 write_c0_guestctl0(gctl0 | MIPS_GCTL0_GT); 452 453 _kvm_vz_restore_htimer(vcpu, read_gc0_compare(), 454 read_gc0_cause()); 455 } 456 } 457 458 /** 459 * _kvm_vz_save_htimer() - Switch to software emulation of guest timer. 460 * @vcpu: Virtual CPU. 461 * @out_compare: Pointer to write compare value to. 462 * @out_cause: Pointer to write cause value to. 463 * 464 * Save VZ guest timer state and switch to software emulation of guest CP0 465 * timer. The hard timer must already be in use, so preemption should be 466 * disabled. 467 */ 468 static void _kvm_vz_save_htimer(struct kvm_vcpu *vcpu, 469 u32 *out_compare, u32 *out_cause) 470 { 471 u32 cause, compare, before_count, end_count; 472 ktime_t before_time; 473 474 compare = read_gc0_compare(); 475 *out_compare = compare; 476 477 before_time = ktime_get(); 478 479 /* 480 * Record the CP0_Count *prior* to saving CP0_Cause, so we have a time 481 * at which no pending timer interrupt is missing. 482 */ 483 before_count = read_gc0_count(); 484 back_to_back_c0_hazard(); 485 cause = read_gc0_cause(); 486 *out_cause = cause; 487 488 /* 489 * Record a final CP0_Count which we will transfer to the soft-timer. 490 * This is recorded *after* saving CP0_Cause, so we don't get any timer 491 * interrupts from just after the final CP0_Count point. 492 */ 493 back_to_back_c0_hazard(); 494 end_count = read_gc0_count(); 495 496 /* 497 * The above sequence isn't atomic, so we could miss a timer interrupt 498 * between reading CP0_Cause and end_count. Detect and record any timer 499 * interrupt due between before_count and end_count. 500 */ 501 if (end_count - before_count > compare - before_count - 1) 502 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER); 503 504 /* 505 * Restore soft-timer, ignoring a small amount of negative drift due to 506 * delay between freeze_hrtimer and setting CP0_GTOffset. 507 */ 508 kvm_mips_restore_hrtimer(vcpu, before_time, end_count, -0x10000); 509 } 510 511 /** 512 * kvm_vz_save_timer() - Save guest timer state. 513 * @vcpu: Virtual CPU. 514 * 515 * Save VZ guest timer state and switch to soft guest timer if hard timer was in 516 * use. 517 */ 518 static void kvm_vz_save_timer(struct kvm_vcpu *vcpu) 519 { 520 struct mips_coproc *cop0 = &vcpu->arch.cop0; 521 u32 gctl0, compare, cause; 522 523 gctl0 = read_c0_guestctl0(); 524 if (gctl0 & MIPS_GCTL0_GT) { 525 /* disable guest use of hard timer */ 526 write_c0_guestctl0(gctl0 & ~MIPS_GCTL0_GT); 527 528 /* save hard timer state */ 529 _kvm_vz_save_htimer(vcpu, &compare, &cause); 530 } else { 531 compare = read_gc0_compare(); 532 cause = read_gc0_cause(); 533 } 534 535 /* save timer-related state to VCPU context */ 536 kvm_write_sw_gc0_cause(cop0, cause); 537 kvm_write_sw_gc0_compare(cop0, compare); 538 } 539 540 /** 541 * kvm_vz_lose_htimer() - Ensure hard guest timer is not in use. 542 * @vcpu: Virtual CPU. 543 * 544 * Transfers the state of the hard guest timer to the soft guest timer, leaving 545 * guest state intact so it can continue to be used with the soft timer. 546 */ 547 void kvm_vz_lose_htimer(struct kvm_vcpu *vcpu) 548 { 549 u32 gctl0, compare, cause; 550 551 preempt_disable(); 552 gctl0 = read_c0_guestctl0(); 553 if (gctl0 & MIPS_GCTL0_GT) { 554 /* disable guest use of timer */ 555 write_c0_guestctl0(gctl0 & ~MIPS_GCTL0_GT); 556 557 /* switch to soft timer */ 558 _kvm_vz_save_htimer(vcpu, &compare, &cause); 559 560 /* leave soft timer in usable state */ 561 _kvm_vz_restore_stimer(vcpu, compare, cause); 562 } 563 preempt_enable(); 564 } 565 566 /** 567 * is_eva_access() - Find whether an instruction is an EVA memory accessor. 568 * @inst: 32-bit instruction encoding. 569 * 570 * Finds whether @inst encodes an EVA memory access instruction, which would 571 * indicate that emulation of it should access the user mode address space 572 * instead of the kernel mode address space. This matters for MUSUK segments 573 * which are TLB mapped for user mode but unmapped for kernel mode. 574 * 575 * Returns: Whether @inst encodes an EVA accessor instruction. 576 */ 577 static bool is_eva_access(union mips_instruction inst) 578 { 579 if (inst.spec3_format.opcode != spec3_op) 580 return false; 581 582 switch (inst.spec3_format.func) { 583 case lwle_op: 584 case lwre_op: 585 case cachee_op: 586 case sbe_op: 587 case she_op: 588 case sce_op: 589 case swe_op: 590 case swle_op: 591 case swre_op: 592 case prefe_op: 593 case lbue_op: 594 case lhue_op: 595 case lbe_op: 596 case lhe_op: 597 case lle_op: 598 case lwe_op: 599 return true; 600 default: 601 return false; 602 } 603 } 604 605 /** 606 * is_eva_am_mapped() - Find whether an access mode is mapped. 607 * @vcpu: KVM VCPU state. 608 * @am: 3-bit encoded access mode. 609 * @eu: Segment becomes unmapped and uncached when Status.ERL=1. 610 * 611 * Decode @am to find whether it encodes a mapped segment for the current VCPU 612 * state. Where necessary @eu and the actual instruction causing the fault are 613 * taken into account to make the decision. 614 * 615 * Returns: Whether the VCPU faulted on a TLB mapped address. 616 */ 617 static bool is_eva_am_mapped(struct kvm_vcpu *vcpu, unsigned int am, bool eu) 618 { 619 u32 am_lookup; 620 int err; 621 622 /* 623 * Interpret access control mode. We assume address errors will already 624 * have been caught by the guest, leaving us with: 625 * AM UM SM KM 31..24 23..16 626 * UK 0 000 Unm 0 0 627 * MK 1 001 TLB 1 628 * MSK 2 010 TLB TLB 1 629 * MUSK 3 011 TLB TLB TLB 1 630 * MUSUK 4 100 TLB TLB Unm 0 1 631 * USK 5 101 Unm Unm 0 0 632 * - 6 110 0 0 633 * UUSK 7 111 Unm Unm Unm 0 0 634 * 635 * We shift a magic value by AM across the sign bit to find if always 636 * TLB mapped, and if not shift by 8 again to find if it depends on KM. 637 */ 638 am_lookup = 0x70080000 << am; 639 if ((s32)am_lookup < 0) { 640 /* 641 * MK, MSK, MUSK 642 * Always TLB mapped, unless SegCtl.EU && ERL 643 */ 644 if (!eu || !(read_gc0_status() & ST0_ERL)) 645 return true; 646 } else { 647 am_lookup <<= 8; 648 if ((s32)am_lookup < 0) { 649 union mips_instruction inst; 650 unsigned int status; 651 u32 *opc; 652 653 /* 654 * MUSUK 655 * TLB mapped if not in kernel mode 656 */ 657 status = read_gc0_status(); 658 if (!(status & (ST0_EXL | ST0_ERL)) && 659 (status & ST0_KSU)) 660 return true; 661 /* 662 * EVA access instructions in kernel 663 * mode access user address space. 664 */ 665 opc = (u32 *)vcpu->arch.pc; 666 if (vcpu->arch.host_cp0_cause & CAUSEF_BD) 667 opc += 1; 668 err = kvm_get_badinstr(opc, vcpu, &inst.word); 669 if (!err && is_eva_access(inst)) 670 return true; 671 } 672 } 673 674 return false; 675 } 676 677 /** 678 * kvm_vz_gva_to_gpa() - Convert valid GVA to GPA. 679 * @vcpu: KVM VCPU state. 680 * @gva: Guest virtual address to convert. 681 * @gpa: Output guest physical address. 682 * 683 * Convert a guest virtual address (GVA) which is valid according to the guest 684 * context, to a guest physical address (GPA). 685 * 686 * Returns: 0 on success. 687 * -errno on failure. 688 */ 689 static int kvm_vz_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, 690 unsigned long *gpa) 691 { 692 u32 gva32 = gva; 693 unsigned long segctl; 694 695 if ((long)gva == (s32)gva32) { 696 /* Handle canonical 32-bit virtual address */ 697 if (cpu_guest_has_segments) { 698 unsigned long mask, pa; 699 700 switch (gva32 >> 29) { 701 case 0: 702 case 1: /* CFG5 (1GB) */ 703 segctl = read_gc0_segctl2() >> 16; 704 mask = (unsigned long)0xfc0000000ull; 705 break; 706 case 2: 707 case 3: /* CFG4 (1GB) */ 708 segctl = read_gc0_segctl2(); 709 mask = (unsigned long)0xfc0000000ull; 710 break; 711 case 4: /* CFG3 (512MB) */ 712 segctl = read_gc0_segctl1() >> 16; 713 mask = (unsigned long)0xfe0000000ull; 714 break; 715 case 5: /* CFG2 (512MB) */ 716 segctl = read_gc0_segctl1(); 717 mask = (unsigned long)0xfe0000000ull; 718 break; 719 case 6: /* CFG1 (512MB) */ 720 segctl = read_gc0_segctl0() >> 16; 721 mask = (unsigned long)0xfe0000000ull; 722 break; 723 case 7: /* CFG0 (512MB) */ 724 segctl = read_gc0_segctl0(); 725 mask = (unsigned long)0xfe0000000ull; 726 break; 727 default: 728 /* 729 * GCC 4.9 isn't smart enough to figure out that 730 * segctl and mask are always initialised. 731 */ 732 unreachable(); 733 } 734 735 if (is_eva_am_mapped(vcpu, (segctl >> 4) & 0x7, 736 segctl & 0x0008)) 737 goto tlb_mapped; 738 739 /* Unmapped, find guest physical address */ 740 pa = (segctl << 20) & mask; 741 pa |= gva32 & ~mask; 742 *gpa = pa; 743 return 0; 744 } else if ((s32)gva32 < (s32)0xc0000000) { 745 /* legacy unmapped KSeg0 or KSeg1 */ 746 *gpa = gva32 & 0x1fffffff; 747 return 0; 748 } 749 #ifdef CONFIG_64BIT 750 } else if ((gva & 0xc000000000000000) == 0x8000000000000000) { 751 /* XKPHYS */ 752 if (cpu_guest_has_segments) { 753 /* 754 * Each of the 8 regions can be overridden by SegCtl2.XR 755 * to use SegCtl1.XAM. 756 */ 757 segctl = read_gc0_segctl2(); 758 if (segctl & (1ull << (56 + ((gva >> 59) & 0x7)))) { 759 segctl = read_gc0_segctl1(); 760 if (is_eva_am_mapped(vcpu, (segctl >> 59) & 0x7, 761 0)) 762 goto tlb_mapped; 763 } 764 765 } 766 /* 767 * Traditionally fully unmapped. 768 * Bits 61:59 specify the CCA, which we can just mask off here. 769 * Bits 58:PABITS should be zero, but we shouldn't have got here 770 * if it wasn't. 771 */ 772 *gpa = gva & 0x07ffffffffffffff; 773 return 0; 774 #endif 775 } 776 777 tlb_mapped: 778 return kvm_vz_guest_tlb_lookup(vcpu, gva, gpa); 779 } 780 781 /** 782 * kvm_vz_badvaddr_to_gpa() - Convert GVA BadVAddr from root exception to GPA. 783 * @vcpu: KVM VCPU state. 784 * @badvaddr: Root BadVAddr. 785 * @gpa: Output guest physical address. 786 * 787 * VZ implementations are permitted to report guest virtual addresses (GVA) in 788 * BadVAddr on a root exception during guest execution, instead of the more 789 * convenient guest physical addresses (GPA). When we get a GVA, this function 790 * converts it to a GPA, taking into account guest segmentation and guest TLB 791 * state. 792 * 793 * Returns: 0 on success. 794 * -errno on failure. 795 */ 796 static int kvm_vz_badvaddr_to_gpa(struct kvm_vcpu *vcpu, unsigned long badvaddr, 797 unsigned long *gpa) 798 { 799 unsigned int gexccode = (vcpu->arch.host_cp0_guestctl0 & 800 MIPS_GCTL0_GEXC) >> MIPS_GCTL0_GEXC_SHIFT; 801 802 /* If BadVAddr is GPA, then all is well in the world */ 803 if (likely(gexccode == MIPS_GCTL0_GEXC_GPA)) { 804 *gpa = badvaddr; 805 return 0; 806 } 807 808 /* Otherwise we'd expect it to be GVA ... */ 809 if (WARN(gexccode != MIPS_GCTL0_GEXC_GVA, 810 "Unexpected gexccode %#x\n", gexccode)) 811 return -EINVAL; 812 813 /* ... and we need to perform the GVA->GPA translation in software */ 814 return kvm_vz_gva_to_gpa(vcpu, badvaddr, gpa); 815 } 816 817 static int kvm_trap_vz_no_handler(struct kvm_vcpu *vcpu) 818 { 819 u32 *opc = (u32 *) vcpu->arch.pc; 820 u32 cause = vcpu->arch.host_cp0_cause; 821 u32 exccode = (cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE; 822 unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr; 823 u32 inst = 0; 824 825 /* 826 * Fetch the instruction. 827 */ 828 if (cause & CAUSEF_BD) 829 opc += 1; 830 kvm_get_badinstr(opc, vcpu, &inst); 831 832 kvm_err("Exception Code: %d not handled @ PC: %p, inst: 0x%08x BadVaddr: %#lx Status: %#x\n", 833 exccode, opc, inst, badvaddr, 834 read_gc0_status()); 835 kvm_arch_vcpu_dump_regs(vcpu); 836 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 837 return RESUME_HOST; 838 } 839 840 static unsigned long mips_process_maar(unsigned int op, unsigned long val) 841 { 842 /* Mask off unused bits */ 843 unsigned long mask = 0xfffff000 | MIPS_MAAR_S | MIPS_MAAR_VL; 844 845 if (read_gc0_pagegrain() & PG_ELPA) 846 mask |= 0x00ffffff00000000ull; 847 if (cpu_guest_has_mvh) 848 mask |= MIPS_MAAR_VH; 849 850 /* Set or clear VH */ 851 if (op == mtc_op) { 852 /* clear VH */ 853 val &= ~MIPS_MAAR_VH; 854 } else if (op == dmtc_op) { 855 /* set VH to match VL */ 856 val &= ~MIPS_MAAR_VH; 857 if (val & MIPS_MAAR_VL) 858 val |= MIPS_MAAR_VH; 859 } 860 861 return val & mask; 862 } 863 864 static void kvm_write_maari(struct kvm_vcpu *vcpu, unsigned long val) 865 { 866 struct mips_coproc *cop0 = &vcpu->arch.cop0; 867 868 val &= MIPS_MAARI_INDEX; 869 if (val == MIPS_MAARI_INDEX) 870 kvm_write_sw_gc0_maari(cop0, ARRAY_SIZE(vcpu->arch.maar) - 1); 871 else if (val < ARRAY_SIZE(vcpu->arch.maar)) 872 kvm_write_sw_gc0_maari(cop0, val); 873 } 874 875 static enum emulation_result kvm_vz_gpsi_cop0(union mips_instruction inst, 876 u32 *opc, u32 cause, 877 struct kvm_vcpu *vcpu) 878 { 879 struct mips_coproc *cop0 = &vcpu->arch.cop0; 880 enum emulation_result er = EMULATE_DONE; 881 u32 rt, rd, sel; 882 unsigned long curr_pc; 883 unsigned long val; 884 885 /* 886 * Update PC and hold onto current PC in case there is 887 * an error and we want to rollback the PC 888 */ 889 curr_pc = vcpu->arch.pc; 890 er = update_pc(vcpu, cause); 891 if (er == EMULATE_FAIL) 892 return er; 893 894 if (inst.co_format.co) { 895 switch (inst.co_format.func) { 896 case wait_op: 897 er = kvm_mips_emul_wait(vcpu); 898 break; 899 default: 900 er = EMULATE_FAIL; 901 } 902 } else { 903 rt = inst.c0r_format.rt; 904 rd = inst.c0r_format.rd; 905 sel = inst.c0r_format.sel; 906 907 switch (inst.c0r_format.rs) { 908 case dmfc_op: 909 case mfc_op: 910 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS 911 cop0->stat[rd][sel]++; 912 #endif 913 if (rd == MIPS_CP0_COUNT && 914 sel == 0) { /* Count */ 915 val = kvm_mips_read_count(vcpu); 916 } else if (rd == MIPS_CP0_COMPARE && 917 sel == 0) { /* Compare */ 918 val = read_gc0_compare(); 919 } else if (rd == MIPS_CP0_LLADDR && 920 sel == 0) { /* LLAddr */ 921 if (cpu_guest_has_rw_llb) 922 val = read_gc0_lladdr() & 923 MIPS_LLADDR_LLB; 924 else 925 val = 0; 926 } else if (rd == MIPS_CP0_LLADDR && 927 sel == 1 && /* MAAR */ 928 cpu_guest_has_maar && 929 !cpu_guest_has_dyn_maar) { 930 /* MAARI must be in range */ 931 BUG_ON(kvm_read_sw_gc0_maari(cop0) >= 932 ARRAY_SIZE(vcpu->arch.maar)); 933 val = vcpu->arch.maar[ 934 kvm_read_sw_gc0_maari(cop0)]; 935 } else if ((rd == MIPS_CP0_PRID && 936 (sel == 0 || /* PRid */ 937 sel == 2 || /* CDMMBase */ 938 sel == 3)) || /* CMGCRBase */ 939 (rd == MIPS_CP0_STATUS && 940 (sel == 2 || /* SRSCtl */ 941 sel == 3)) || /* SRSMap */ 942 (rd == MIPS_CP0_CONFIG && 943 (sel == 6 || /* Config6 */ 944 sel == 7)) || /* Config7 */ 945 (rd == MIPS_CP0_LLADDR && 946 (sel == 2) && /* MAARI */ 947 cpu_guest_has_maar && 948 !cpu_guest_has_dyn_maar) || 949 (rd == MIPS_CP0_ERRCTL && 950 (sel == 0))) { /* ErrCtl */ 951 val = cop0->reg[rd][sel]; 952 #ifdef CONFIG_CPU_LOONGSON64 953 } else if (rd == MIPS_CP0_DIAG && 954 (sel == 0)) { /* Diag */ 955 val = cop0->reg[rd][sel]; 956 #endif 957 } else { 958 val = 0; 959 er = EMULATE_FAIL; 960 } 961 962 if (er != EMULATE_FAIL) { 963 /* Sign extend */ 964 if (inst.c0r_format.rs == mfc_op) 965 val = (int)val; 966 vcpu->arch.gprs[rt] = val; 967 } 968 969 trace_kvm_hwr(vcpu, (inst.c0r_format.rs == mfc_op) ? 970 KVM_TRACE_MFC0 : KVM_TRACE_DMFC0, 971 KVM_TRACE_COP0(rd, sel), val); 972 break; 973 974 case dmtc_op: 975 case mtc_op: 976 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS 977 cop0->stat[rd][sel]++; 978 #endif 979 val = vcpu->arch.gprs[rt]; 980 trace_kvm_hwr(vcpu, (inst.c0r_format.rs == mtc_op) ? 981 KVM_TRACE_MTC0 : KVM_TRACE_DMTC0, 982 KVM_TRACE_COP0(rd, sel), val); 983 984 if (rd == MIPS_CP0_COUNT && 985 sel == 0) { /* Count */ 986 kvm_vz_lose_htimer(vcpu); 987 kvm_mips_write_count(vcpu, vcpu->arch.gprs[rt]); 988 } else if (rd == MIPS_CP0_COMPARE && 989 sel == 0) { /* Compare */ 990 kvm_mips_write_compare(vcpu, 991 vcpu->arch.gprs[rt], 992 true); 993 } else if (rd == MIPS_CP0_LLADDR && 994 sel == 0) { /* LLAddr */ 995 /* 996 * P5600 generates GPSI on guest MTC0 LLAddr. 997 * Only allow the guest to clear LLB. 998 */ 999 if (cpu_guest_has_rw_llb && 1000 !(val & MIPS_LLADDR_LLB)) 1001 write_gc0_lladdr(0); 1002 } else if (rd == MIPS_CP0_LLADDR && 1003 sel == 1 && /* MAAR */ 1004 cpu_guest_has_maar && 1005 !cpu_guest_has_dyn_maar) { 1006 val = mips_process_maar(inst.c0r_format.rs, 1007 val); 1008 1009 /* MAARI must be in range */ 1010 BUG_ON(kvm_read_sw_gc0_maari(cop0) >= 1011 ARRAY_SIZE(vcpu->arch.maar)); 1012 vcpu->arch.maar[kvm_read_sw_gc0_maari(cop0)] = 1013 val; 1014 } else if (rd == MIPS_CP0_LLADDR && 1015 (sel == 2) && /* MAARI */ 1016 cpu_guest_has_maar && 1017 !cpu_guest_has_dyn_maar) { 1018 kvm_write_maari(vcpu, val); 1019 } else if (rd == MIPS_CP0_CONFIG && 1020 (sel == 6)) { 1021 cop0->reg[rd][sel] = (int)val; 1022 } else if (rd == MIPS_CP0_ERRCTL && 1023 (sel == 0)) { /* ErrCtl */ 1024 /* ignore the written value */ 1025 #ifdef CONFIG_CPU_LOONGSON64 1026 } else if (rd == MIPS_CP0_DIAG && 1027 (sel == 0)) { /* Diag */ 1028 unsigned long flags; 1029 1030 local_irq_save(flags); 1031 if (val & LOONGSON_DIAG_BTB) { 1032 /* Flush BTB */ 1033 set_c0_diag(LOONGSON_DIAG_BTB); 1034 } 1035 if (val & LOONGSON_DIAG_ITLB) { 1036 /* Flush ITLB */ 1037 set_c0_diag(LOONGSON_DIAG_ITLB); 1038 } 1039 if (val & LOONGSON_DIAG_DTLB) { 1040 /* Flush DTLB */ 1041 set_c0_diag(LOONGSON_DIAG_DTLB); 1042 } 1043 if (val & LOONGSON_DIAG_VTLB) { 1044 /* Flush VTLB */ 1045 kvm_loongson_clear_guest_vtlb(); 1046 } 1047 if (val & LOONGSON_DIAG_FTLB) { 1048 /* Flush FTLB */ 1049 kvm_loongson_clear_guest_ftlb(); 1050 } 1051 local_irq_restore(flags); 1052 #endif 1053 } else { 1054 er = EMULATE_FAIL; 1055 } 1056 break; 1057 1058 default: 1059 er = EMULATE_FAIL; 1060 break; 1061 } 1062 } 1063 /* Rollback PC only if emulation was unsuccessful */ 1064 if (er == EMULATE_FAIL) { 1065 kvm_err("[%#lx]%s: unsupported cop0 instruction 0x%08x\n", 1066 curr_pc, __func__, inst.word); 1067 1068 vcpu->arch.pc = curr_pc; 1069 } 1070 1071 return er; 1072 } 1073 1074 static enum emulation_result kvm_vz_gpsi_cache(union mips_instruction inst, 1075 u32 *opc, u32 cause, 1076 struct kvm_vcpu *vcpu) 1077 { 1078 enum emulation_result er = EMULATE_DONE; 1079 u32 cache, op_inst, op, base; 1080 s16 offset; 1081 struct kvm_vcpu_arch *arch = &vcpu->arch; 1082 unsigned long va, curr_pc; 1083 1084 /* 1085 * Update PC and hold onto current PC in case there is 1086 * an error and we want to rollback the PC 1087 */ 1088 curr_pc = vcpu->arch.pc; 1089 er = update_pc(vcpu, cause); 1090 if (er == EMULATE_FAIL) 1091 return er; 1092 1093 base = inst.i_format.rs; 1094 op_inst = inst.i_format.rt; 1095 if (cpu_has_mips_r6) 1096 offset = inst.spec3_format.simmediate; 1097 else 1098 offset = inst.i_format.simmediate; 1099 cache = op_inst & CacheOp_Cache; 1100 op = op_inst & CacheOp_Op; 1101 1102 va = arch->gprs[base] + offset; 1103 1104 kvm_debug("CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n", 1105 cache, op, base, arch->gprs[base], offset); 1106 1107 /* Secondary or tirtiary cache ops ignored */ 1108 if (cache != Cache_I && cache != Cache_D) 1109 return EMULATE_DONE; 1110 1111 switch (op_inst) { 1112 case Index_Invalidate_I: 1113 flush_icache_line_indexed(va); 1114 return EMULATE_DONE; 1115 case Index_Writeback_Inv_D: 1116 flush_dcache_line_indexed(va); 1117 return EMULATE_DONE; 1118 case Hit_Invalidate_I: 1119 case Hit_Invalidate_D: 1120 case Hit_Writeback_Inv_D: 1121 if (boot_cpu_type() == CPU_CAVIUM_OCTEON3) { 1122 /* We can just flush entire icache */ 1123 local_flush_icache_range(0, 0); 1124 return EMULATE_DONE; 1125 } 1126 1127 /* So far, other platforms support guest hit cache ops */ 1128 break; 1129 default: 1130 break; 1131 } 1132 1133 kvm_err("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n", 1134 curr_pc, vcpu->arch.gprs[31], cache, op, base, arch->gprs[base], 1135 offset); 1136 /* Rollback PC */ 1137 vcpu->arch.pc = curr_pc; 1138 1139 return EMULATE_FAIL; 1140 } 1141 1142 #ifdef CONFIG_CPU_LOONGSON64 1143 static enum emulation_result kvm_vz_gpsi_lwc2(union mips_instruction inst, 1144 u32 *opc, u32 cause, 1145 struct kvm_vcpu *vcpu) 1146 { 1147 unsigned int rs, rd; 1148 unsigned int hostcfg; 1149 unsigned long curr_pc; 1150 enum emulation_result er = EMULATE_DONE; 1151 1152 /* 1153 * Update PC and hold onto current PC in case there is 1154 * an error and we want to rollback the PC 1155 */ 1156 curr_pc = vcpu->arch.pc; 1157 er = update_pc(vcpu, cause); 1158 if (er == EMULATE_FAIL) 1159 return er; 1160 1161 rs = inst.loongson3_lscsr_format.rs; 1162 rd = inst.loongson3_lscsr_format.rd; 1163 switch (inst.loongson3_lscsr_format.fr) { 1164 case 0x8: /* Read CPUCFG */ 1165 ++vcpu->stat.vz_cpucfg_exits; 1166 hostcfg = read_cpucfg(vcpu->arch.gprs[rs]); 1167 1168 switch (vcpu->arch.gprs[rs]) { 1169 case LOONGSON_CFG0: 1170 vcpu->arch.gprs[rd] = 0x14c000; 1171 break; 1172 case LOONGSON_CFG1: 1173 hostcfg &= (LOONGSON_CFG1_FP | LOONGSON_CFG1_MMI | 1174 LOONGSON_CFG1_MSA1 | LOONGSON_CFG1_MSA2 | 1175 LOONGSON_CFG1_SFBP); 1176 vcpu->arch.gprs[rd] = hostcfg; 1177 break; 1178 case LOONGSON_CFG2: 1179 hostcfg &= (LOONGSON_CFG2_LEXT1 | LOONGSON_CFG2_LEXT2 | 1180 LOONGSON_CFG2_LEXT3 | LOONGSON_CFG2_LSPW); 1181 vcpu->arch.gprs[rd] = hostcfg; 1182 break; 1183 case LOONGSON_CFG3: 1184 vcpu->arch.gprs[rd] = hostcfg; 1185 break; 1186 default: 1187 /* Don't export any other advanced features to guest */ 1188 vcpu->arch.gprs[rd] = 0; 1189 break; 1190 } 1191 break; 1192 1193 default: 1194 kvm_err("lwc2 emulate not impl %d rs %lx @%lx\n", 1195 inst.loongson3_lscsr_format.fr, vcpu->arch.gprs[rs], curr_pc); 1196 er = EMULATE_FAIL; 1197 break; 1198 } 1199 1200 /* Rollback PC only if emulation was unsuccessful */ 1201 if (er == EMULATE_FAIL) { 1202 kvm_err("[%#lx]%s: unsupported lwc2 instruction 0x%08x 0x%08x\n", 1203 curr_pc, __func__, inst.word, inst.loongson3_lscsr_format.fr); 1204 1205 vcpu->arch.pc = curr_pc; 1206 } 1207 1208 return er; 1209 } 1210 #endif 1211 1212 static enum emulation_result kvm_trap_vz_handle_gpsi(u32 cause, u32 *opc, 1213 struct kvm_vcpu *vcpu) 1214 { 1215 enum emulation_result er = EMULATE_DONE; 1216 struct kvm_vcpu_arch *arch = &vcpu->arch; 1217 union mips_instruction inst; 1218 int rd, rt, sel; 1219 int err; 1220 1221 /* 1222 * Fetch the instruction. 1223 */ 1224 if (cause & CAUSEF_BD) 1225 opc += 1; 1226 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1227 if (err) 1228 return EMULATE_FAIL; 1229 1230 switch (inst.r_format.opcode) { 1231 case cop0_op: 1232 er = kvm_vz_gpsi_cop0(inst, opc, cause, vcpu); 1233 break; 1234 #ifndef CONFIG_CPU_MIPSR6 1235 case cache_op: 1236 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE); 1237 er = kvm_vz_gpsi_cache(inst, opc, cause, vcpu); 1238 break; 1239 #endif 1240 #ifdef CONFIG_CPU_LOONGSON64 1241 case lwc2_op: 1242 er = kvm_vz_gpsi_lwc2(inst, opc, cause, vcpu); 1243 break; 1244 #endif 1245 case spec3_op: 1246 switch (inst.spec3_format.func) { 1247 #ifdef CONFIG_CPU_MIPSR6 1248 case cache6_op: 1249 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE); 1250 er = kvm_vz_gpsi_cache(inst, opc, cause, vcpu); 1251 break; 1252 #endif 1253 case rdhwr_op: 1254 if (inst.r_format.rs || (inst.r_format.re >> 3)) 1255 goto unknown; 1256 1257 rd = inst.r_format.rd; 1258 rt = inst.r_format.rt; 1259 sel = inst.r_format.re & 0x7; 1260 1261 switch (rd) { 1262 case MIPS_HWR_CC: /* Read count register */ 1263 arch->gprs[rt] = 1264 (long)(int)kvm_mips_read_count(vcpu); 1265 break; 1266 default: 1267 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, 1268 KVM_TRACE_HWR(rd, sel), 0); 1269 goto unknown; 1270 } 1271 1272 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, 1273 KVM_TRACE_HWR(rd, sel), arch->gprs[rt]); 1274 1275 er = update_pc(vcpu, cause); 1276 break; 1277 default: 1278 goto unknown; 1279 } 1280 break; 1281 unknown: 1282 1283 default: 1284 kvm_err("GPSI exception not supported (%p/%#x)\n", 1285 opc, inst.word); 1286 kvm_arch_vcpu_dump_regs(vcpu); 1287 er = EMULATE_FAIL; 1288 break; 1289 } 1290 1291 return er; 1292 } 1293 1294 static enum emulation_result kvm_trap_vz_handle_gsfc(u32 cause, u32 *opc, 1295 struct kvm_vcpu *vcpu) 1296 { 1297 enum emulation_result er = EMULATE_DONE; 1298 struct kvm_vcpu_arch *arch = &vcpu->arch; 1299 union mips_instruction inst; 1300 int err; 1301 1302 /* 1303 * Fetch the instruction. 1304 */ 1305 if (cause & CAUSEF_BD) 1306 opc += 1; 1307 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1308 if (err) 1309 return EMULATE_FAIL; 1310 1311 /* complete MTC0 on behalf of guest and advance EPC */ 1312 if (inst.c0r_format.opcode == cop0_op && 1313 inst.c0r_format.rs == mtc_op && 1314 inst.c0r_format.z == 0) { 1315 int rt = inst.c0r_format.rt; 1316 int rd = inst.c0r_format.rd; 1317 int sel = inst.c0r_format.sel; 1318 unsigned int val = arch->gprs[rt]; 1319 unsigned int old_val, change; 1320 1321 trace_kvm_hwr(vcpu, KVM_TRACE_MTC0, KVM_TRACE_COP0(rd, sel), 1322 val); 1323 1324 if ((rd == MIPS_CP0_STATUS) && (sel == 0)) { 1325 /* FR bit should read as zero if no FPU */ 1326 if (!kvm_mips_guest_has_fpu(&vcpu->arch)) 1327 val &= ~(ST0_CU1 | ST0_FR); 1328 1329 /* 1330 * Also don't allow FR to be set if host doesn't support 1331 * it. 1332 */ 1333 if (!(boot_cpu_data.fpu_id & MIPS_FPIR_F64)) 1334 val &= ~ST0_FR; 1335 1336 old_val = read_gc0_status(); 1337 change = val ^ old_val; 1338 1339 if (change & ST0_FR) { 1340 /* 1341 * FPU and Vector register state is made 1342 * UNPREDICTABLE by a change of FR, so don't 1343 * even bother saving it. 1344 */ 1345 kvm_drop_fpu(vcpu); 1346 } 1347 1348 /* 1349 * If MSA state is already live, it is undefined how it 1350 * interacts with FR=0 FPU state, and we don't want to 1351 * hit reserved instruction exceptions trying to save 1352 * the MSA state later when CU=1 && FR=1, so play it 1353 * safe and save it first. 1354 */ 1355 if (change & ST0_CU1 && !(val & ST0_FR) && 1356 vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) 1357 kvm_lose_fpu(vcpu); 1358 1359 write_gc0_status(val); 1360 } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) { 1361 u32 old_cause = read_gc0_cause(); 1362 u32 change = old_cause ^ val; 1363 1364 /* DC bit enabling/disabling timer? */ 1365 if (change & CAUSEF_DC) { 1366 if (val & CAUSEF_DC) { 1367 kvm_vz_lose_htimer(vcpu); 1368 kvm_mips_count_disable_cause(vcpu); 1369 } else { 1370 kvm_mips_count_enable_cause(vcpu); 1371 } 1372 } 1373 1374 /* Only certain bits are RW to the guest */ 1375 change &= (CAUSEF_DC | CAUSEF_IV | CAUSEF_WP | 1376 CAUSEF_IP0 | CAUSEF_IP1); 1377 1378 /* WP can only be cleared */ 1379 change &= ~CAUSEF_WP | old_cause; 1380 1381 write_gc0_cause(old_cause ^ change); 1382 } else if ((rd == MIPS_CP0_STATUS) && (sel == 1)) { /* IntCtl */ 1383 write_gc0_intctl(val); 1384 } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) { 1385 old_val = read_gc0_config5(); 1386 change = val ^ old_val; 1387 /* Handle changes in FPU/MSA modes */ 1388 preempt_disable(); 1389 1390 /* 1391 * Propagate FRE changes immediately if the FPU 1392 * context is already loaded. 1393 */ 1394 if (change & MIPS_CONF5_FRE && 1395 vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU) 1396 change_c0_config5(MIPS_CONF5_FRE, val); 1397 1398 preempt_enable(); 1399 1400 val = old_val ^ 1401 (change & kvm_vz_config5_guest_wrmask(vcpu)); 1402 write_gc0_config5(val); 1403 } else { 1404 kvm_err("Handle GSFC, unsupported field change @ %p: %#x\n", 1405 opc, inst.word); 1406 er = EMULATE_FAIL; 1407 } 1408 1409 if (er != EMULATE_FAIL) 1410 er = update_pc(vcpu, cause); 1411 } else { 1412 kvm_err("Handle GSFC, unrecognized instruction @ %p: %#x\n", 1413 opc, inst.word); 1414 er = EMULATE_FAIL; 1415 } 1416 1417 return er; 1418 } 1419 1420 static enum emulation_result kvm_trap_vz_handle_ghfc(u32 cause, u32 *opc, 1421 struct kvm_vcpu *vcpu) 1422 { 1423 /* 1424 * Presumably this is due to MC (guest mode change), so lets trace some 1425 * relevant info. 1426 */ 1427 trace_kvm_guest_mode_change(vcpu); 1428 1429 return EMULATE_DONE; 1430 } 1431 1432 static enum emulation_result kvm_trap_vz_handle_hc(u32 cause, u32 *opc, 1433 struct kvm_vcpu *vcpu) 1434 { 1435 enum emulation_result er; 1436 union mips_instruction inst; 1437 unsigned long curr_pc; 1438 int err; 1439 1440 if (cause & CAUSEF_BD) 1441 opc += 1; 1442 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1443 if (err) 1444 return EMULATE_FAIL; 1445 1446 /* 1447 * Update PC and hold onto current PC in case there is 1448 * an error and we want to rollback the PC 1449 */ 1450 curr_pc = vcpu->arch.pc; 1451 er = update_pc(vcpu, cause); 1452 if (er == EMULATE_FAIL) 1453 return er; 1454 1455 er = kvm_mips_emul_hypcall(vcpu, inst); 1456 if (er == EMULATE_FAIL) 1457 vcpu->arch.pc = curr_pc; 1458 1459 return er; 1460 } 1461 1462 static enum emulation_result kvm_trap_vz_no_handler_guest_exit(u32 gexccode, 1463 u32 cause, 1464 u32 *opc, 1465 struct kvm_vcpu *vcpu) 1466 { 1467 u32 inst; 1468 1469 /* 1470 * Fetch the instruction. 1471 */ 1472 if (cause & CAUSEF_BD) 1473 opc += 1; 1474 kvm_get_badinstr(opc, vcpu, &inst); 1475 1476 kvm_err("Guest Exception Code: %d not yet handled @ PC: %p, inst: 0x%08x Status: %#x\n", 1477 gexccode, opc, inst, read_gc0_status()); 1478 1479 return EMULATE_FAIL; 1480 } 1481 1482 static int kvm_trap_vz_handle_guest_exit(struct kvm_vcpu *vcpu) 1483 { 1484 u32 *opc = (u32 *) vcpu->arch.pc; 1485 u32 cause = vcpu->arch.host_cp0_cause; 1486 enum emulation_result er = EMULATE_DONE; 1487 u32 gexccode = (vcpu->arch.host_cp0_guestctl0 & 1488 MIPS_GCTL0_GEXC) >> MIPS_GCTL0_GEXC_SHIFT; 1489 int ret = RESUME_GUEST; 1490 1491 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_GEXCCODE_BASE + gexccode); 1492 switch (gexccode) { 1493 case MIPS_GCTL0_GEXC_GPSI: 1494 ++vcpu->stat.vz_gpsi_exits; 1495 er = kvm_trap_vz_handle_gpsi(cause, opc, vcpu); 1496 break; 1497 case MIPS_GCTL0_GEXC_GSFC: 1498 ++vcpu->stat.vz_gsfc_exits; 1499 er = kvm_trap_vz_handle_gsfc(cause, opc, vcpu); 1500 break; 1501 case MIPS_GCTL0_GEXC_HC: 1502 ++vcpu->stat.vz_hc_exits; 1503 er = kvm_trap_vz_handle_hc(cause, opc, vcpu); 1504 break; 1505 case MIPS_GCTL0_GEXC_GRR: 1506 ++vcpu->stat.vz_grr_exits; 1507 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1508 vcpu); 1509 break; 1510 case MIPS_GCTL0_GEXC_GVA: 1511 ++vcpu->stat.vz_gva_exits; 1512 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1513 vcpu); 1514 break; 1515 case MIPS_GCTL0_GEXC_GHFC: 1516 ++vcpu->stat.vz_ghfc_exits; 1517 er = kvm_trap_vz_handle_ghfc(cause, opc, vcpu); 1518 break; 1519 case MIPS_GCTL0_GEXC_GPA: 1520 ++vcpu->stat.vz_gpa_exits; 1521 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1522 vcpu); 1523 break; 1524 default: 1525 ++vcpu->stat.vz_resvd_exits; 1526 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1527 vcpu); 1528 break; 1529 1530 } 1531 1532 if (er == EMULATE_DONE) { 1533 ret = RESUME_GUEST; 1534 } else if (er == EMULATE_HYPERCALL) { 1535 ret = kvm_mips_handle_hypcall(vcpu); 1536 } else { 1537 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1538 ret = RESUME_HOST; 1539 } 1540 return ret; 1541 } 1542 1543 /** 1544 * kvm_trap_vz_handle_cop_unusable() - Guest used unusable coprocessor. 1545 * @vcpu: Virtual CPU context. 1546 * 1547 * Handle when the guest attempts to use a coprocessor which hasn't been allowed 1548 * by the root context. 1549 * 1550 * Return: value indicating whether to resume the host or the guest 1551 * (RESUME_HOST or RESUME_GUEST) 1552 */ 1553 static int kvm_trap_vz_handle_cop_unusable(struct kvm_vcpu *vcpu) 1554 { 1555 u32 cause = vcpu->arch.host_cp0_cause; 1556 enum emulation_result er = EMULATE_FAIL; 1557 int ret = RESUME_GUEST; 1558 1559 if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1) { 1560 /* 1561 * If guest FPU not present, the FPU operation should have been 1562 * treated as a reserved instruction! 1563 * If FPU already in use, we shouldn't get this at all. 1564 */ 1565 if (WARN_ON(!kvm_mips_guest_has_fpu(&vcpu->arch) || 1566 vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)) { 1567 preempt_enable(); 1568 return EMULATE_FAIL; 1569 } 1570 1571 kvm_own_fpu(vcpu); 1572 er = EMULATE_DONE; 1573 } 1574 /* other coprocessors not handled */ 1575 1576 switch (er) { 1577 case EMULATE_DONE: 1578 ret = RESUME_GUEST; 1579 break; 1580 1581 case EMULATE_FAIL: 1582 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1583 ret = RESUME_HOST; 1584 break; 1585 1586 default: 1587 BUG(); 1588 } 1589 return ret; 1590 } 1591 1592 /** 1593 * kvm_trap_vz_handle_msa_disabled() - Guest used MSA while disabled in root. 1594 * @vcpu: Virtual CPU context. 1595 * 1596 * Handle when the guest attempts to use MSA when it is disabled in the root 1597 * context. 1598 * 1599 * Return: value indicating whether to resume the host or the guest 1600 * (RESUME_HOST or RESUME_GUEST) 1601 */ 1602 static int kvm_trap_vz_handle_msa_disabled(struct kvm_vcpu *vcpu) 1603 { 1604 /* 1605 * If MSA not present or not exposed to guest or FR=0, the MSA operation 1606 * should have been treated as a reserved instruction! 1607 * Same if CU1=1, FR=0. 1608 * If MSA already in use, we shouldn't get this at all. 1609 */ 1610 if (!kvm_mips_guest_has_msa(&vcpu->arch) || 1611 (read_gc0_status() & (ST0_CU1 | ST0_FR)) == ST0_CU1 || 1612 !(read_gc0_config5() & MIPS_CONF5_MSAEN) || 1613 vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) { 1614 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1615 return RESUME_HOST; 1616 } 1617 1618 kvm_own_msa(vcpu); 1619 1620 return RESUME_GUEST; 1621 } 1622 1623 static int kvm_trap_vz_handle_tlb_ld_miss(struct kvm_vcpu *vcpu) 1624 { 1625 struct kvm_run *run = vcpu->run; 1626 u32 *opc = (u32 *) vcpu->arch.pc; 1627 u32 cause = vcpu->arch.host_cp0_cause; 1628 ulong badvaddr = vcpu->arch.host_cp0_badvaddr; 1629 union mips_instruction inst; 1630 enum emulation_result er = EMULATE_DONE; 1631 int err, ret = RESUME_GUEST; 1632 1633 if (kvm_mips_handle_vz_root_tlb_fault(badvaddr, vcpu, false)) { 1634 /* A code fetch fault doesn't count as an MMIO */ 1635 if (kvm_is_ifetch_fault(&vcpu->arch)) { 1636 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1637 return RESUME_HOST; 1638 } 1639 1640 /* Fetch the instruction */ 1641 if (cause & CAUSEF_BD) 1642 opc += 1; 1643 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1644 if (err) { 1645 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1646 return RESUME_HOST; 1647 } 1648 1649 /* Treat as MMIO */ 1650 er = kvm_mips_emulate_load(inst, cause, vcpu); 1651 if (er == EMULATE_FAIL) { 1652 kvm_err("Guest Emulate Load from MMIO space failed: PC: %p, BadVaddr: %#lx\n", 1653 opc, badvaddr); 1654 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1655 } 1656 } 1657 1658 if (er == EMULATE_DONE) { 1659 ret = RESUME_GUEST; 1660 } else if (er == EMULATE_DO_MMIO) { 1661 run->exit_reason = KVM_EXIT_MMIO; 1662 ret = RESUME_HOST; 1663 } else { 1664 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1665 ret = RESUME_HOST; 1666 } 1667 return ret; 1668 } 1669 1670 static int kvm_trap_vz_handle_tlb_st_miss(struct kvm_vcpu *vcpu) 1671 { 1672 struct kvm_run *run = vcpu->run; 1673 u32 *opc = (u32 *) vcpu->arch.pc; 1674 u32 cause = vcpu->arch.host_cp0_cause; 1675 ulong badvaddr = vcpu->arch.host_cp0_badvaddr; 1676 union mips_instruction inst; 1677 enum emulation_result er = EMULATE_DONE; 1678 int err; 1679 int ret = RESUME_GUEST; 1680 1681 /* Just try the access again if we couldn't do the translation */ 1682 if (kvm_vz_badvaddr_to_gpa(vcpu, badvaddr, &badvaddr)) 1683 return RESUME_GUEST; 1684 vcpu->arch.host_cp0_badvaddr = badvaddr; 1685 1686 if (kvm_mips_handle_vz_root_tlb_fault(badvaddr, vcpu, true)) { 1687 /* Fetch the instruction */ 1688 if (cause & CAUSEF_BD) 1689 opc += 1; 1690 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1691 if (err) { 1692 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1693 return RESUME_HOST; 1694 } 1695 1696 /* Treat as MMIO */ 1697 er = kvm_mips_emulate_store(inst, cause, vcpu); 1698 if (er == EMULATE_FAIL) { 1699 kvm_err("Guest Emulate Store to MMIO space failed: PC: %p, BadVaddr: %#lx\n", 1700 opc, badvaddr); 1701 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1702 } 1703 } 1704 1705 if (er == EMULATE_DONE) { 1706 ret = RESUME_GUEST; 1707 } else if (er == EMULATE_DO_MMIO) { 1708 run->exit_reason = KVM_EXIT_MMIO; 1709 ret = RESUME_HOST; 1710 } else { 1711 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1712 ret = RESUME_HOST; 1713 } 1714 return ret; 1715 } 1716 1717 static u64 kvm_vz_get_one_regs[] = { 1718 KVM_REG_MIPS_CP0_INDEX, 1719 KVM_REG_MIPS_CP0_ENTRYLO0, 1720 KVM_REG_MIPS_CP0_ENTRYLO1, 1721 KVM_REG_MIPS_CP0_CONTEXT, 1722 KVM_REG_MIPS_CP0_PAGEMASK, 1723 KVM_REG_MIPS_CP0_PAGEGRAIN, 1724 KVM_REG_MIPS_CP0_WIRED, 1725 KVM_REG_MIPS_CP0_HWRENA, 1726 KVM_REG_MIPS_CP0_BADVADDR, 1727 KVM_REG_MIPS_CP0_COUNT, 1728 KVM_REG_MIPS_CP0_ENTRYHI, 1729 KVM_REG_MIPS_CP0_COMPARE, 1730 KVM_REG_MIPS_CP0_STATUS, 1731 KVM_REG_MIPS_CP0_INTCTL, 1732 KVM_REG_MIPS_CP0_CAUSE, 1733 KVM_REG_MIPS_CP0_EPC, 1734 KVM_REG_MIPS_CP0_PRID, 1735 KVM_REG_MIPS_CP0_EBASE, 1736 KVM_REG_MIPS_CP0_CONFIG, 1737 KVM_REG_MIPS_CP0_CONFIG1, 1738 KVM_REG_MIPS_CP0_CONFIG2, 1739 KVM_REG_MIPS_CP0_CONFIG3, 1740 KVM_REG_MIPS_CP0_CONFIG4, 1741 KVM_REG_MIPS_CP0_CONFIG5, 1742 KVM_REG_MIPS_CP0_CONFIG6, 1743 #ifdef CONFIG_64BIT 1744 KVM_REG_MIPS_CP0_XCONTEXT, 1745 #endif 1746 KVM_REG_MIPS_CP0_ERROREPC, 1747 1748 KVM_REG_MIPS_COUNT_CTL, 1749 KVM_REG_MIPS_COUNT_RESUME, 1750 KVM_REG_MIPS_COUNT_HZ, 1751 }; 1752 1753 static u64 kvm_vz_get_one_regs_contextconfig[] = { 1754 KVM_REG_MIPS_CP0_CONTEXTCONFIG, 1755 #ifdef CONFIG_64BIT 1756 KVM_REG_MIPS_CP0_XCONTEXTCONFIG, 1757 #endif 1758 }; 1759 1760 static u64 kvm_vz_get_one_regs_segments[] = { 1761 KVM_REG_MIPS_CP0_SEGCTL0, 1762 KVM_REG_MIPS_CP0_SEGCTL1, 1763 KVM_REG_MIPS_CP0_SEGCTL2, 1764 }; 1765 1766 static u64 kvm_vz_get_one_regs_htw[] = { 1767 KVM_REG_MIPS_CP0_PWBASE, 1768 KVM_REG_MIPS_CP0_PWFIELD, 1769 KVM_REG_MIPS_CP0_PWSIZE, 1770 KVM_REG_MIPS_CP0_PWCTL, 1771 }; 1772 1773 static u64 kvm_vz_get_one_regs_kscratch[] = { 1774 KVM_REG_MIPS_CP0_KSCRATCH1, 1775 KVM_REG_MIPS_CP0_KSCRATCH2, 1776 KVM_REG_MIPS_CP0_KSCRATCH3, 1777 KVM_REG_MIPS_CP0_KSCRATCH4, 1778 KVM_REG_MIPS_CP0_KSCRATCH5, 1779 KVM_REG_MIPS_CP0_KSCRATCH6, 1780 }; 1781 1782 static unsigned long kvm_vz_num_regs(struct kvm_vcpu *vcpu) 1783 { 1784 unsigned long ret; 1785 1786 ret = ARRAY_SIZE(kvm_vz_get_one_regs); 1787 if (cpu_guest_has_userlocal) 1788 ++ret; 1789 if (cpu_guest_has_badinstr) 1790 ++ret; 1791 if (cpu_guest_has_badinstrp) 1792 ++ret; 1793 if (cpu_guest_has_contextconfig) 1794 ret += ARRAY_SIZE(kvm_vz_get_one_regs_contextconfig); 1795 if (cpu_guest_has_segments) 1796 ret += ARRAY_SIZE(kvm_vz_get_one_regs_segments); 1797 if (cpu_guest_has_htw || cpu_guest_has_ldpte) 1798 ret += ARRAY_SIZE(kvm_vz_get_one_regs_htw); 1799 if (cpu_guest_has_maar && !cpu_guest_has_dyn_maar) 1800 ret += 1 + ARRAY_SIZE(vcpu->arch.maar); 1801 ret += __arch_hweight8(cpu_data[0].guest.kscratch_mask); 1802 1803 return ret; 1804 } 1805 1806 static int kvm_vz_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices) 1807 { 1808 u64 index; 1809 unsigned int i; 1810 1811 if (copy_to_user(indices, kvm_vz_get_one_regs, 1812 sizeof(kvm_vz_get_one_regs))) 1813 return -EFAULT; 1814 indices += ARRAY_SIZE(kvm_vz_get_one_regs); 1815 1816 if (cpu_guest_has_userlocal) { 1817 index = KVM_REG_MIPS_CP0_USERLOCAL; 1818 if (copy_to_user(indices, &index, sizeof(index))) 1819 return -EFAULT; 1820 ++indices; 1821 } 1822 if (cpu_guest_has_badinstr) { 1823 index = KVM_REG_MIPS_CP0_BADINSTR; 1824 if (copy_to_user(indices, &index, sizeof(index))) 1825 return -EFAULT; 1826 ++indices; 1827 } 1828 if (cpu_guest_has_badinstrp) { 1829 index = KVM_REG_MIPS_CP0_BADINSTRP; 1830 if (copy_to_user(indices, &index, sizeof(index))) 1831 return -EFAULT; 1832 ++indices; 1833 } 1834 if (cpu_guest_has_contextconfig) { 1835 if (copy_to_user(indices, kvm_vz_get_one_regs_contextconfig, 1836 sizeof(kvm_vz_get_one_regs_contextconfig))) 1837 return -EFAULT; 1838 indices += ARRAY_SIZE(kvm_vz_get_one_regs_contextconfig); 1839 } 1840 if (cpu_guest_has_segments) { 1841 if (copy_to_user(indices, kvm_vz_get_one_regs_segments, 1842 sizeof(kvm_vz_get_one_regs_segments))) 1843 return -EFAULT; 1844 indices += ARRAY_SIZE(kvm_vz_get_one_regs_segments); 1845 } 1846 if (cpu_guest_has_htw || cpu_guest_has_ldpte) { 1847 if (copy_to_user(indices, kvm_vz_get_one_regs_htw, 1848 sizeof(kvm_vz_get_one_regs_htw))) 1849 return -EFAULT; 1850 indices += ARRAY_SIZE(kvm_vz_get_one_regs_htw); 1851 } 1852 if (cpu_guest_has_maar && !cpu_guest_has_dyn_maar) { 1853 for (i = 0; i < ARRAY_SIZE(vcpu->arch.maar); ++i) { 1854 index = KVM_REG_MIPS_CP0_MAAR(i); 1855 if (copy_to_user(indices, &index, sizeof(index))) 1856 return -EFAULT; 1857 ++indices; 1858 } 1859 1860 index = KVM_REG_MIPS_CP0_MAARI; 1861 if (copy_to_user(indices, &index, sizeof(index))) 1862 return -EFAULT; 1863 ++indices; 1864 } 1865 for (i = 0; i < 6; ++i) { 1866 if (!cpu_guest_has_kscr(i + 2)) 1867 continue; 1868 1869 if (copy_to_user(indices, &kvm_vz_get_one_regs_kscratch[i], 1870 sizeof(kvm_vz_get_one_regs_kscratch[i]))) 1871 return -EFAULT; 1872 ++indices; 1873 } 1874 1875 return 0; 1876 } 1877 1878 static inline s64 entrylo_kvm_to_user(unsigned long v) 1879 { 1880 s64 mask, ret = v; 1881 1882 if (BITS_PER_LONG == 32) { 1883 /* 1884 * KVM API exposes 64-bit version of the register, so move the 1885 * RI/XI bits up into place. 1886 */ 1887 mask = MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI; 1888 ret &= ~mask; 1889 ret |= ((s64)v & mask) << 32; 1890 } 1891 return ret; 1892 } 1893 1894 static inline unsigned long entrylo_user_to_kvm(s64 v) 1895 { 1896 unsigned long mask, ret = v; 1897 1898 if (BITS_PER_LONG == 32) { 1899 /* 1900 * KVM API exposes 64-bit versiono of the register, so move the 1901 * RI/XI bits down into place. 1902 */ 1903 mask = MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI; 1904 ret &= ~mask; 1905 ret |= (v >> 32) & mask; 1906 } 1907 return ret; 1908 } 1909 1910 static int kvm_vz_get_one_reg(struct kvm_vcpu *vcpu, 1911 const struct kvm_one_reg *reg, 1912 s64 *v) 1913 { 1914 struct mips_coproc *cop0 = &vcpu->arch.cop0; 1915 unsigned int idx; 1916 1917 switch (reg->id) { 1918 case KVM_REG_MIPS_CP0_INDEX: 1919 *v = (long)read_gc0_index(); 1920 break; 1921 case KVM_REG_MIPS_CP0_ENTRYLO0: 1922 *v = entrylo_kvm_to_user(read_gc0_entrylo0()); 1923 break; 1924 case KVM_REG_MIPS_CP0_ENTRYLO1: 1925 *v = entrylo_kvm_to_user(read_gc0_entrylo1()); 1926 break; 1927 case KVM_REG_MIPS_CP0_CONTEXT: 1928 *v = (long)read_gc0_context(); 1929 break; 1930 case KVM_REG_MIPS_CP0_CONTEXTCONFIG: 1931 if (!cpu_guest_has_contextconfig) 1932 return -EINVAL; 1933 *v = read_gc0_contextconfig(); 1934 break; 1935 case KVM_REG_MIPS_CP0_USERLOCAL: 1936 if (!cpu_guest_has_userlocal) 1937 return -EINVAL; 1938 *v = read_gc0_userlocal(); 1939 break; 1940 #ifdef CONFIG_64BIT 1941 case KVM_REG_MIPS_CP0_XCONTEXTCONFIG: 1942 if (!cpu_guest_has_contextconfig) 1943 return -EINVAL; 1944 *v = read_gc0_xcontextconfig(); 1945 break; 1946 #endif 1947 case KVM_REG_MIPS_CP0_PAGEMASK: 1948 *v = (long)read_gc0_pagemask(); 1949 break; 1950 case KVM_REG_MIPS_CP0_PAGEGRAIN: 1951 *v = (long)read_gc0_pagegrain(); 1952 break; 1953 case KVM_REG_MIPS_CP0_SEGCTL0: 1954 if (!cpu_guest_has_segments) 1955 return -EINVAL; 1956 *v = read_gc0_segctl0(); 1957 break; 1958 case KVM_REG_MIPS_CP0_SEGCTL1: 1959 if (!cpu_guest_has_segments) 1960 return -EINVAL; 1961 *v = read_gc0_segctl1(); 1962 break; 1963 case KVM_REG_MIPS_CP0_SEGCTL2: 1964 if (!cpu_guest_has_segments) 1965 return -EINVAL; 1966 *v = read_gc0_segctl2(); 1967 break; 1968 case KVM_REG_MIPS_CP0_PWBASE: 1969 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 1970 return -EINVAL; 1971 *v = read_gc0_pwbase(); 1972 break; 1973 case KVM_REG_MIPS_CP0_PWFIELD: 1974 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 1975 return -EINVAL; 1976 *v = read_gc0_pwfield(); 1977 break; 1978 case KVM_REG_MIPS_CP0_PWSIZE: 1979 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 1980 return -EINVAL; 1981 *v = read_gc0_pwsize(); 1982 break; 1983 case KVM_REG_MIPS_CP0_WIRED: 1984 *v = (long)read_gc0_wired(); 1985 break; 1986 case KVM_REG_MIPS_CP0_PWCTL: 1987 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 1988 return -EINVAL; 1989 *v = read_gc0_pwctl(); 1990 break; 1991 case KVM_REG_MIPS_CP0_HWRENA: 1992 *v = (long)read_gc0_hwrena(); 1993 break; 1994 case KVM_REG_MIPS_CP0_BADVADDR: 1995 *v = (long)read_gc0_badvaddr(); 1996 break; 1997 case KVM_REG_MIPS_CP0_BADINSTR: 1998 if (!cpu_guest_has_badinstr) 1999 return -EINVAL; 2000 *v = read_gc0_badinstr(); 2001 break; 2002 case KVM_REG_MIPS_CP0_BADINSTRP: 2003 if (!cpu_guest_has_badinstrp) 2004 return -EINVAL; 2005 *v = read_gc0_badinstrp(); 2006 break; 2007 case KVM_REG_MIPS_CP0_COUNT: 2008 *v = kvm_mips_read_count(vcpu); 2009 break; 2010 case KVM_REG_MIPS_CP0_ENTRYHI: 2011 *v = (long)read_gc0_entryhi(); 2012 break; 2013 case KVM_REG_MIPS_CP0_COMPARE: 2014 *v = (long)read_gc0_compare(); 2015 break; 2016 case KVM_REG_MIPS_CP0_STATUS: 2017 *v = (long)read_gc0_status(); 2018 break; 2019 case KVM_REG_MIPS_CP0_INTCTL: 2020 *v = read_gc0_intctl(); 2021 break; 2022 case KVM_REG_MIPS_CP0_CAUSE: 2023 *v = (long)read_gc0_cause(); 2024 break; 2025 case KVM_REG_MIPS_CP0_EPC: 2026 *v = (long)read_gc0_epc(); 2027 break; 2028 case KVM_REG_MIPS_CP0_PRID: 2029 switch (boot_cpu_type()) { 2030 case CPU_CAVIUM_OCTEON3: 2031 /* Octeon III has a read-only guest.PRid */ 2032 *v = read_gc0_prid(); 2033 break; 2034 default: 2035 *v = (long)kvm_read_c0_guest_prid(cop0); 2036 break; 2037 } 2038 break; 2039 case KVM_REG_MIPS_CP0_EBASE: 2040 *v = kvm_vz_read_gc0_ebase(); 2041 break; 2042 case KVM_REG_MIPS_CP0_CONFIG: 2043 *v = read_gc0_config(); 2044 break; 2045 case KVM_REG_MIPS_CP0_CONFIG1: 2046 if (!cpu_guest_has_conf1) 2047 return -EINVAL; 2048 *v = read_gc0_config1(); 2049 break; 2050 case KVM_REG_MIPS_CP0_CONFIG2: 2051 if (!cpu_guest_has_conf2) 2052 return -EINVAL; 2053 *v = read_gc0_config2(); 2054 break; 2055 case KVM_REG_MIPS_CP0_CONFIG3: 2056 if (!cpu_guest_has_conf3) 2057 return -EINVAL; 2058 *v = read_gc0_config3(); 2059 break; 2060 case KVM_REG_MIPS_CP0_CONFIG4: 2061 if (!cpu_guest_has_conf4) 2062 return -EINVAL; 2063 *v = read_gc0_config4(); 2064 break; 2065 case KVM_REG_MIPS_CP0_CONFIG5: 2066 if (!cpu_guest_has_conf5) 2067 return -EINVAL; 2068 *v = read_gc0_config5(); 2069 break; 2070 case KVM_REG_MIPS_CP0_CONFIG6: 2071 *v = kvm_read_sw_gc0_config6(cop0); 2072 break; 2073 case KVM_REG_MIPS_CP0_MAAR(0) ... KVM_REG_MIPS_CP0_MAAR(0x3f): 2074 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar) 2075 return -EINVAL; 2076 idx = reg->id - KVM_REG_MIPS_CP0_MAAR(0); 2077 if (idx >= ARRAY_SIZE(vcpu->arch.maar)) 2078 return -EINVAL; 2079 *v = vcpu->arch.maar[idx]; 2080 break; 2081 case KVM_REG_MIPS_CP0_MAARI: 2082 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar) 2083 return -EINVAL; 2084 *v = kvm_read_sw_gc0_maari(&vcpu->arch.cop0); 2085 break; 2086 #ifdef CONFIG_64BIT 2087 case KVM_REG_MIPS_CP0_XCONTEXT: 2088 *v = read_gc0_xcontext(); 2089 break; 2090 #endif 2091 case KVM_REG_MIPS_CP0_ERROREPC: 2092 *v = (long)read_gc0_errorepc(); 2093 break; 2094 case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6: 2095 idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2; 2096 if (!cpu_guest_has_kscr(idx)) 2097 return -EINVAL; 2098 switch (idx) { 2099 case 2: 2100 *v = (long)read_gc0_kscratch1(); 2101 break; 2102 case 3: 2103 *v = (long)read_gc0_kscratch2(); 2104 break; 2105 case 4: 2106 *v = (long)read_gc0_kscratch3(); 2107 break; 2108 case 5: 2109 *v = (long)read_gc0_kscratch4(); 2110 break; 2111 case 6: 2112 *v = (long)read_gc0_kscratch5(); 2113 break; 2114 case 7: 2115 *v = (long)read_gc0_kscratch6(); 2116 break; 2117 } 2118 break; 2119 case KVM_REG_MIPS_COUNT_CTL: 2120 *v = vcpu->arch.count_ctl; 2121 break; 2122 case KVM_REG_MIPS_COUNT_RESUME: 2123 *v = ktime_to_ns(vcpu->arch.count_resume); 2124 break; 2125 case KVM_REG_MIPS_COUNT_HZ: 2126 *v = vcpu->arch.count_hz; 2127 break; 2128 default: 2129 return -EINVAL; 2130 } 2131 return 0; 2132 } 2133 2134 static int kvm_vz_set_one_reg(struct kvm_vcpu *vcpu, 2135 const struct kvm_one_reg *reg, 2136 s64 v) 2137 { 2138 struct mips_coproc *cop0 = &vcpu->arch.cop0; 2139 unsigned int idx; 2140 int ret = 0; 2141 unsigned int cur, change; 2142 2143 switch (reg->id) { 2144 case KVM_REG_MIPS_CP0_INDEX: 2145 write_gc0_index(v); 2146 break; 2147 case KVM_REG_MIPS_CP0_ENTRYLO0: 2148 write_gc0_entrylo0(entrylo_user_to_kvm(v)); 2149 break; 2150 case KVM_REG_MIPS_CP0_ENTRYLO1: 2151 write_gc0_entrylo1(entrylo_user_to_kvm(v)); 2152 break; 2153 case KVM_REG_MIPS_CP0_CONTEXT: 2154 write_gc0_context(v); 2155 break; 2156 case KVM_REG_MIPS_CP0_CONTEXTCONFIG: 2157 if (!cpu_guest_has_contextconfig) 2158 return -EINVAL; 2159 write_gc0_contextconfig(v); 2160 break; 2161 case KVM_REG_MIPS_CP0_USERLOCAL: 2162 if (!cpu_guest_has_userlocal) 2163 return -EINVAL; 2164 write_gc0_userlocal(v); 2165 break; 2166 #ifdef CONFIG_64BIT 2167 case KVM_REG_MIPS_CP0_XCONTEXTCONFIG: 2168 if (!cpu_guest_has_contextconfig) 2169 return -EINVAL; 2170 write_gc0_xcontextconfig(v); 2171 break; 2172 #endif 2173 case KVM_REG_MIPS_CP0_PAGEMASK: 2174 write_gc0_pagemask(v); 2175 break; 2176 case KVM_REG_MIPS_CP0_PAGEGRAIN: 2177 write_gc0_pagegrain(v); 2178 break; 2179 case KVM_REG_MIPS_CP0_SEGCTL0: 2180 if (!cpu_guest_has_segments) 2181 return -EINVAL; 2182 write_gc0_segctl0(v); 2183 break; 2184 case KVM_REG_MIPS_CP0_SEGCTL1: 2185 if (!cpu_guest_has_segments) 2186 return -EINVAL; 2187 write_gc0_segctl1(v); 2188 break; 2189 case KVM_REG_MIPS_CP0_SEGCTL2: 2190 if (!cpu_guest_has_segments) 2191 return -EINVAL; 2192 write_gc0_segctl2(v); 2193 break; 2194 case KVM_REG_MIPS_CP0_PWBASE: 2195 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 2196 return -EINVAL; 2197 write_gc0_pwbase(v); 2198 break; 2199 case KVM_REG_MIPS_CP0_PWFIELD: 2200 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 2201 return -EINVAL; 2202 write_gc0_pwfield(v); 2203 break; 2204 case KVM_REG_MIPS_CP0_PWSIZE: 2205 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 2206 return -EINVAL; 2207 write_gc0_pwsize(v); 2208 break; 2209 case KVM_REG_MIPS_CP0_WIRED: 2210 change_gc0_wired(MIPSR6_WIRED_WIRED, v); 2211 break; 2212 case KVM_REG_MIPS_CP0_PWCTL: 2213 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 2214 return -EINVAL; 2215 write_gc0_pwctl(v); 2216 break; 2217 case KVM_REG_MIPS_CP0_HWRENA: 2218 write_gc0_hwrena(v); 2219 break; 2220 case KVM_REG_MIPS_CP0_BADVADDR: 2221 write_gc0_badvaddr(v); 2222 break; 2223 case KVM_REG_MIPS_CP0_BADINSTR: 2224 if (!cpu_guest_has_badinstr) 2225 return -EINVAL; 2226 write_gc0_badinstr(v); 2227 break; 2228 case KVM_REG_MIPS_CP0_BADINSTRP: 2229 if (!cpu_guest_has_badinstrp) 2230 return -EINVAL; 2231 write_gc0_badinstrp(v); 2232 break; 2233 case KVM_REG_MIPS_CP0_COUNT: 2234 kvm_mips_write_count(vcpu, v); 2235 break; 2236 case KVM_REG_MIPS_CP0_ENTRYHI: 2237 write_gc0_entryhi(v); 2238 break; 2239 case KVM_REG_MIPS_CP0_COMPARE: 2240 kvm_mips_write_compare(vcpu, v, false); 2241 break; 2242 case KVM_REG_MIPS_CP0_STATUS: 2243 write_gc0_status(v); 2244 break; 2245 case KVM_REG_MIPS_CP0_INTCTL: 2246 write_gc0_intctl(v); 2247 break; 2248 case KVM_REG_MIPS_CP0_CAUSE: 2249 /* 2250 * If the timer is stopped or started (DC bit) it must look 2251 * atomic with changes to the timer interrupt pending bit (TI). 2252 * A timer interrupt should not happen in between. 2253 */ 2254 if ((read_gc0_cause() ^ v) & CAUSEF_DC) { 2255 if (v & CAUSEF_DC) { 2256 /* disable timer first */ 2257 kvm_mips_count_disable_cause(vcpu); 2258 change_gc0_cause((u32)~CAUSEF_DC, v); 2259 } else { 2260 /* enable timer last */ 2261 change_gc0_cause((u32)~CAUSEF_DC, v); 2262 kvm_mips_count_enable_cause(vcpu); 2263 } 2264 } else { 2265 write_gc0_cause(v); 2266 } 2267 break; 2268 case KVM_REG_MIPS_CP0_EPC: 2269 write_gc0_epc(v); 2270 break; 2271 case KVM_REG_MIPS_CP0_PRID: 2272 switch (boot_cpu_type()) { 2273 case CPU_CAVIUM_OCTEON3: 2274 /* Octeon III has a guest.PRid, but its read-only */ 2275 break; 2276 default: 2277 kvm_write_c0_guest_prid(cop0, v); 2278 break; 2279 } 2280 break; 2281 case KVM_REG_MIPS_CP0_EBASE: 2282 kvm_vz_write_gc0_ebase(v); 2283 break; 2284 case KVM_REG_MIPS_CP0_CONFIG: 2285 cur = read_gc0_config(); 2286 change = (cur ^ v) & kvm_vz_config_user_wrmask(vcpu); 2287 if (change) { 2288 v = cur ^ change; 2289 write_gc0_config(v); 2290 } 2291 break; 2292 case KVM_REG_MIPS_CP0_CONFIG1: 2293 if (!cpu_guest_has_conf1) 2294 break; 2295 cur = read_gc0_config1(); 2296 change = (cur ^ v) & kvm_vz_config1_user_wrmask(vcpu); 2297 if (change) { 2298 v = cur ^ change; 2299 write_gc0_config1(v); 2300 } 2301 break; 2302 case KVM_REG_MIPS_CP0_CONFIG2: 2303 if (!cpu_guest_has_conf2) 2304 break; 2305 cur = read_gc0_config2(); 2306 change = (cur ^ v) & kvm_vz_config2_user_wrmask(vcpu); 2307 if (change) { 2308 v = cur ^ change; 2309 write_gc0_config2(v); 2310 } 2311 break; 2312 case KVM_REG_MIPS_CP0_CONFIG3: 2313 if (!cpu_guest_has_conf3) 2314 break; 2315 cur = read_gc0_config3(); 2316 change = (cur ^ v) & kvm_vz_config3_user_wrmask(vcpu); 2317 if (change) { 2318 v = cur ^ change; 2319 write_gc0_config3(v); 2320 } 2321 break; 2322 case KVM_REG_MIPS_CP0_CONFIG4: 2323 if (!cpu_guest_has_conf4) 2324 break; 2325 cur = read_gc0_config4(); 2326 change = (cur ^ v) & kvm_vz_config4_user_wrmask(vcpu); 2327 if (change) { 2328 v = cur ^ change; 2329 write_gc0_config4(v); 2330 } 2331 break; 2332 case KVM_REG_MIPS_CP0_CONFIG5: 2333 if (!cpu_guest_has_conf5) 2334 break; 2335 cur = read_gc0_config5(); 2336 change = (cur ^ v) & kvm_vz_config5_user_wrmask(vcpu); 2337 if (change) { 2338 v = cur ^ change; 2339 write_gc0_config5(v); 2340 } 2341 break; 2342 case KVM_REG_MIPS_CP0_CONFIG6: 2343 cur = kvm_read_sw_gc0_config6(cop0); 2344 change = (cur ^ v) & kvm_vz_config6_user_wrmask(vcpu); 2345 if (change) { 2346 v = cur ^ change; 2347 kvm_write_sw_gc0_config6(cop0, (int)v); 2348 } 2349 break; 2350 case KVM_REG_MIPS_CP0_MAAR(0) ... KVM_REG_MIPS_CP0_MAAR(0x3f): 2351 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar) 2352 return -EINVAL; 2353 idx = reg->id - KVM_REG_MIPS_CP0_MAAR(0); 2354 if (idx >= ARRAY_SIZE(vcpu->arch.maar)) 2355 return -EINVAL; 2356 vcpu->arch.maar[idx] = mips_process_maar(dmtc_op, v); 2357 break; 2358 case KVM_REG_MIPS_CP0_MAARI: 2359 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar) 2360 return -EINVAL; 2361 kvm_write_maari(vcpu, v); 2362 break; 2363 #ifdef CONFIG_64BIT 2364 case KVM_REG_MIPS_CP0_XCONTEXT: 2365 write_gc0_xcontext(v); 2366 break; 2367 #endif 2368 case KVM_REG_MIPS_CP0_ERROREPC: 2369 write_gc0_errorepc(v); 2370 break; 2371 case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6: 2372 idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2; 2373 if (!cpu_guest_has_kscr(idx)) 2374 return -EINVAL; 2375 switch (idx) { 2376 case 2: 2377 write_gc0_kscratch1(v); 2378 break; 2379 case 3: 2380 write_gc0_kscratch2(v); 2381 break; 2382 case 4: 2383 write_gc0_kscratch3(v); 2384 break; 2385 case 5: 2386 write_gc0_kscratch4(v); 2387 break; 2388 case 6: 2389 write_gc0_kscratch5(v); 2390 break; 2391 case 7: 2392 write_gc0_kscratch6(v); 2393 break; 2394 } 2395 break; 2396 case KVM_REG_MIPS_COUNT_CTL: 2397 ret = kvm_mips_set_count_ctl(vcpu, v); 2398 break; 2399 case KVM_REG_MIPS_COUNT_RESUME: 2400 ret = kvm_mips_set_count_resume(vcpu, v); 2401 break; 2402 case KVM_REG_MIPS_COUNT_HZ: 2403 ret = kvm_mips_set_count_hz(vcpu, v); 2404 break; 2405 default: 2406 return -EINVAL; 2407 } 2408 return ret; 2409 } 2410 2411 #define guestid_cache(cpu) (cpu_data[cpu].guestid_cache) 2412 static void kvm_vz_get_new_guestid(unsigned long cpu, struct kvm_vcpu *vcpu) 2413 { 2414 unsigned long guestid = guestid_cache(cpu); 2415 2416 if (!(++guestid & GUESTID_MASK)) { 2417 if (cpu_has_vtag_icache) 2418 flush_icache_all(); 2419 2420 if (!guestid) /* fix version if needed */ 2421 guestid = GUESTID_FIRST_VERSION; 2422 2423 ++guestid; /* guestid 0 reserved for root */ 2424 2425 /* start new guestid cycle */ 2426 kvm_vz_local_flush_roottlb_all_guests(); 2427 kvm_vz_local_flush_guesttlb_all(); 2428 } 2429 2430 guestid_cache(cpu) = guestid; 2431 } 2432 2433 /* Returns 1 if the guest TLB may be clobbered */ 2434 static int kvm_vz_check_requests(struct kvm_vcpu *vcpu, int cpu) 2435 { 2436 int ret = 0; 2437 int i; 2438 2439 if (!kvm_request_pending(vcpu)) 2440 return 0; 2441 2442 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { 2443 if (cpu_has_guestid) { 2444 /* Drop all GuestIDs for this VCPU */ 2445 for_each_possible_cpu(i) 2446 vcpu->arch.vzguestid[i] = 0; 2447 /* This will clobber guest TLB contents too */ 2448 ret = 1; 2449 } 2450 /* 2451 * For Root ASID Dealias (RAD) we don't do anything here, but we 2452 * still need the request to ensure we recheck asid_flush_mask. 2453 * We can still return 0 as only the root TLB will be affected 2454 * by a root ASID flush. 2455 */ 2456 } 2457 2458 return ret; 2459 } 2460 2461 static void kvm_vz_vcpu_save_wired(struct kvm_vcpu *vcpu) 2462 { 2463 unsigned int wired = read_gc0_wired(); 2464 struct kvm_mips_tlb *tlbs; 2465 int i; 2466 2467 /* Expand the wired TLB array if necessary */ 2468 wired &= MIPSR6_WIRED_WIRED; 2469 if (wired > vcpu->arch.wired_tlb_limit) { 2470 tlbs = krealloc(vcpu->arch.wired_tlb, wired * 2471 sizeof(*vcpu->arch.wired_tlb), GFP_ATOMIC); 2472 if (WARN_ON(!tlbs)) { 2473 /* Save whatever we can */ 2474 wired = vcpu->arch.wired_tlb_limit; 2475 } else { 2476 vcpu->arch.wired_tlb = tlbs; 2477 vcpu->arch.wired_tlb_limit = wired; 2478 } 2479 } 2480 2481 if (wired) 2482 /* Save wired entries from the guest TLB */ 2483 kvm_vz_save_guesttlb(vcpu->arch.wired_tlb, 0, wired); 2484 /* Invalidate any dropped entries since last time */ 2485 for (i = wired; i < vcpu->arch.wired_tlb_used; ++i) { 2486 vcpu->arch.wired_tlb[i].tlb_hi = UNIQUE_GUEST_ENTRYHI(i); 2487 vcpu->arch.wired_tlb[i].tlb_lo[0] = 0; 2488 vcpu->arch.wired_tlb[i].tlb_lo[1] = 0; 2489 vcpu->arch.wired_tlb[i].tlb_mask = 0; 2490 } 2491 vcpu->arch.wired_tlb_used = wired; 2492 } 2493 2494 static void kvm_vz_vcpu_load_wired(struct kvm_vcpu *vcpu) 2495 { 2496 /* Load wired entries into the guest TLB */ 2497 if (vcpu->arch.wired_tlb) 2498 kvm_vz_load_guesttlb(vcpu->arch.wired_tlb, 0, 2499 vcpu->arch.wired_tlb_used); 2500 } 2501 2502 static void kvm_vz_vcpu_load_tlb(struct kvm_vcpu *vcpu, int cpu) 2503 { 2504 struct kvm *kvm = vcpu->kvm; 2505 struct mm_struct *gpa_mm = &kvm->arch.gpa_mm; 2506 bool migrated; 2507 2508 /* 2509 * Are we entering guest context on a different CPU to last time? 2510 * If so, the VCPU's guest TLB state on this CPU may be stale. 2511 */ 2512 migrated = (vcpu->arch.last_exec_cpu != cpu); 2513 vcpu->arch.last_exec_cpu = cpu; 2514 2515 /* 2516 * A vcpu's GuestID is set in GuestCtl1.ID when the vcpu is loaded and 2517 * remains set until another vcpu is loaded in. As a rule GuestRID 2518 * remains zeroed when in root context unless the kernel is busy 2519 * manipulating guest tlb entries. 2520 */ 2521 if (cpu_has_guestid) { 2522 /* 2523 * Check if our GuestID is of an older version and thus invalid. 2524 * 2525 * We also discard the stored GuestID if we've executed on 2526 * another CPU, as the guest mappings may have changed without 2527 * hypervisor knowledge. 2528 */ 2529 if (migrated || 2530 (vcpu->arch.vzguestid[cpu] ^ guestid_cache(cpu)) & 2531 GUESTID_VERSION_MASK) { 2532 kvm_vz_get_new_guestid(cpu, vcpu); 2533 vcpu->arch.vzguestid[cpu] = guestid_cache(cpu); 2534 trace_kvm_guestid_change(vcpu, 2535 vcpu->arch.vzguestid[cpu]); 2536 } 2537 2538 /* Restore GuestID */ 2539 change_c0_guestctl1(GUESTID_MASK, vcpu->arch.vzguestid[cpu]); 2540 } else { 2541 /* 2542 * The Guest TLB only stores a single guest's TLB state, so 2543 * flush it if another VCPU has executed on this CPU. 2544 * 2545 * We also flush if we've executed on another CPU, as the guest 2546 * mappings may have changed without hypervisor knowledge. 2547 */ 2548 if (migrated || last_exec_vcpu[cpu] != vcpu) 2549 kvm_vz_local_flush_guesttlb_all(); 2550 last_exec_vcpu[cpu] = vcpu; 2551 2552 /* 2553 * Root ASID dealiases guest GPA mappings in the root TLB. 2554 * Allocate new root ASID if needed. 2555 */ 2556 if (cpumask_test_and_clear_cpu(cpu, &kvm->arch.asid_flush_mask)) 2557 get_new_mmu_context(gpa_mm); 2558 else 2559 check_mmu_context(gpa_mm); 2560 } 2561 } 2562 2563 static int kvm_vz_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 2564 { 2565 struct mips_coproc *cop0 = &vcpu->arch.cop0; 2566 bool migrated, all; 2567 2568 /* 2569 * Have we migrated to a different CPU? 2570 * If so, any old guest TLB state may be stale. 2571 */ 2572 migrated = (vcpu->arch.last_sched_cpu != cpu); 2573 2574 /* 2575 * Was this the last VCPU to run on this CPU? 2576 * If not, any old guest state from this VCPU will have been clobbered. 2577 */ 2578 all = migrated || (last_vcpu[cpu] != vcpu); 2579 last_vcpu[cpu] = vcpu; 2580 2581 /* 2582 * Restore CP0_Wired unconditionally as we clear it after use, and 2583 * restore wired guest TLB entries (while in guest context). 2584 */ 2585 kvm_restore_gc0_wired(cop0); 2586 if (current->flags & PF_VCPU) { 2587 tlbw_use_hazard(); 2588 kvm_vz_vcpu_load_tlb(vcpu, cpu); 2589 kvm_vz_vcpu_load_wired(vcpu); 2590 } 2591 2592 /* 2593 * Restore timer state regardless, as e.g. Cause.TI can change over time 2594 * if left unmaintained. 2595 */ 2596 kvm_vz_restore_timer(vcpu); 2597 2598 /* Set MC bit if we want to trace guest mode changes */ 2599 if (kvm_trace_guest_mode_change) 2600 set_c0_guestctl0(MIPS_GCTL0_MC); 2601 else 2602 clear_c0_guestctl0(MIPS_GCTL0_MC); 2603 2604 /* Don't bother restoring registers multiple times unless necessary */ 2605 if (!all) 2606 return 0; 2607 2608 /* 2609 * Restore config registers first, as some implementations restrict 2610 * writes to other registers when the corresponding feature bits aren't 2611 * set. For example Status.CU1 cannot be set unless Config1.FP is set. 2612 */ 2613 kvm_restore_gc0_config(cop0); 2614 if (cpu_guest_has_conf1) 2615 kvm_restore_gc0_config1(cop0); 2616 if (cpu_guest_has_conf2) 2617 kvm_restore_gc0_config2(cop0); 2618 if (cpu_guest_has_conf3) 2619 kvm_restore_gc0_config3(cop0); 2620 if (cpu_guest_has_conf4) 2621 kvm_restore_gc0_config4(cop0); 2622 if (cpu_guest_has_conf5) 2623 kvm_restore_gc0_config5(cop0); 2624 if (cpu_guest_has_conf6) 2625 kvm_restore_gc0_config6(cop0); 2626 if (cpu_guest_has_conf7) 2627 kvm_restore_gc0_config7(cop0); 2628 2629 kvm_restore_gc0_index(cop0); 2630 kvm_restore_gc0_entrylo0(cop0); 2631 kvm_restore_gc0_entrylo1(cop0); 2632 kvm_restore_gc0_context(cop0); 2633 if (cpu_guest_has_contextconfig) 2634 kvm_restore_gc0_contextconfig(cop0); 2635 #ifdef CONFIG_64BIT 2636 kvm_restore_gc0_xcontext(cop0); 2637 if (cpu_guest_has_contextconfig) 2638 kvm_restore_gc0_xcontextconfig(cop0); 2639 #endif 2640 kvm_restore_gc0_pagemask(cop0); 2641 kvm_restore_gc0_pagegrain(cop0); 2642 kvm_restore_gc0_hwrena(cop0); 2643 kvm_restore_gc0_badvaddr(cop0); 2644 kvm_restore_gc0_entryhi(cop0); 2645 kvm_restore_gc0_status(cop0); 2646 kvm_restore_gc0_intctl(cop0); 2647 kvm_restore_gc0_epc(cop0); 2648 kvm_vz_write_gc0_ebase(kvm_read_sw_gc0_ebase(cop0)); 2649 if (cpu_guest_has_userlocal) 2650 kvm_restore_gc0_userlocal(cop0); 2651 2652 kvm_restore_gc0_errorepc(cop0); 2653 2654 /* restore KScratch registers if enabled in guest */ 2655 if (cpu_guest_has_conf4) { 2656 if (cpu_guest_has_kscr(2)) 2657 kvm_restore_gc0_kscratch1(cop0); 2658 if (cpu_guest_has_kscr(3)) 2659 kvm_restore_gc0_kscratch2(cop0); 2660 if (cpu_guest_has_kscr(4)) 2661 kvm_restore_gc0_kscratch3(cop0); 2662 if (cpu_guest_has_kscr(5)) 2663 kvm_restore_gc0_kscratch4(cop0); 2664 if (cpu_guest_has_kscr(6)) 2665 kvm_restore_gc0_kscratch5(cop0); 2666 if (cpu_guest_has_kscr(7)) 2667 kvm_restore_gc0_kscratch6(cop0); 2668 } 2669 2670 if (cpu_guest_has_badinstr) 2671 kvm_restore_gc0_badinstr(cop0); 2672 if (cpu_guest_has_badinstrp) 2673 kvm_restore_gc0_badinstrp(cop0); 2674 2675 if (cpu_guest_has_segments) { 2676 kvm_restore_gc0_segctl0(cop0); 2677 kvm_restore_gc0_segctl1(cop0); 2678 kvm_restore_gc0_segctl2(cop0); 2679 } 2680 2681 /* restore HTW registers */ 2682 if (cpu_guest_has_htw || cpu_guest_has_ldpte) { 2683 kvm_restore_gc0_pwbase(cop0); 2684 kvm_restore_gc0_pwfield(cop0); 2685 kvm_restore_gc0_pwsize(cop0); 2686 kvm_restore_gc0_pwctl(cop0); 2687 } 2688 2689 /* restore Root.GuestCtl2 from unused Guest guestctl2 register */ 2690 if (cpu_has_guestctl2) 2691 write_c0_guestctl2( 2692 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL]); 2693 2694 /* 2695 * We should clear linked load bit to break interrupted atomics. This 2696 * prevents a SC on the next VCPU from succeeding by matching a LL on 2697 * the previous VCPU. 2698 */ 2699 if (vcpu->kvm->created_vcpus > 1) 2700 write_gc0_lladdr(0); 2701 2702 return 0; 2703 } 2704 2705 static int kvm_vz_vcpu_put(struct kvm_vcpu *vcpu, int cpu) 2706 { 2707 struct mips_coproc *cop0 = &vcpu->arch.cop0; 2708 2709 if (current->flags & PF_VCPU) 2710 kvm_vz_vcpu_save_wired(vcpu); 2711 2712 kvm_lose_fpu(vcpu); 2713 2714 kvm_save_gc0_index(cop0); 2715 kvm_save_gc0_entrylo0(cop0); 2716 kvm_save_gc0_entrylo1(cop0); 2717 kvm_save_gc0_context(cop0); 2718 if (cpu_guest_has_contextconfig) 2719 kvm_save_gc0_contextconfig(cop0); 2720 #ifdef CONFIG_64BIT 2721 kvm_save_gc0_xcontext(cop0); 2722 if (cpu_guest_has_contextconfig) 2723 kvm_save_gc0_xcontextconfig(cop0); 2724 #endif 2725 kvm_save_gc0_pagemask(cop0); 2726 kvm_save_gc0_pagegrain(cop0); 2727 kvm_save_gc0_wired(cop0); 2728 /* allow wired TLB entries to be overwritten */ 2729 clear_gc0_wired(MIPSR6_WIRED_WIRED); 2730 kvm_save_gc0_hwrena(cop0); 2731 kvm_save_gc0_badvaddr(cop0); 2732 kvm_save_gc0_entryhi(cop0); 2733 kvm_save_gc0_status(cop0); 2734 kvm_save_gc0_intctl(cop0); 2735 kvm_save_gc0_epc(cop0); 2736 kvm_write_sw_gc0_ebase(cop0, kvm_vz_read_gc0_ebase()); 2737 if (cpu_guest_has_userlocal) 2738 kvm_save_gc0_userlocal(cop0); 2739 2740 /* only save implemented config registers */ 2741 kvm_save_gc0_config(cop0); 2742 if (cpu_guest_has_conf1) 2743 kvm_save_gc0_config1(cop0); 2744 if (cpu_guest_has_conf2) 2745 kvm_save_gc0_config2(cop0); 2746 if (cpu_guest_has_conf3) 2747 kvm_save_gc0_config3(cop0); 2748 if (cpu_guest_has_conf4) 2749 kvm_save_gc0_config4(cop0); 2750 if (cpu_guest_has_conf5) 2751 kvm_save_gc0_config5(cop0); 2752 if (cpu_guest_has_conf6) 2753 kvm_save_gc0_config6(cop0); 2754 if (cpu_guest_has_conf7) 2755 kvm_save_gc0_config7(cop0); 2756 2757 kvm_save_gc0_errorepc(cop0); 2758 2759 /* save KScratch registers if enabled in guest */ 2760 if (cpu_guest_has_conf4) { 2761 if (cpu_guest_has_kscr(2)) 2762 kvm_save_gc0_kscratch1(cop0); 2763 if (cpu_guest_has_kscr(3)) 2764 kvm_save_gc0_kscratch2(cop0); 2765 if (cpu_guest_has_kscr(4)) 2766 kvm_save_gc0_kscratch3(cop0); 2767 if (cpu_guest_has_kscr(5)) 2768 kvm_save_gc0_kscratch4(cop0); 2769 if (cpu_guest_has_kscr(6)) 2770 kvm_save_gc0_kscratch5(cop0); 2771 if (cpu_guest_has_kscr(7)) 2772 kvm_save_gc0_kscratch6(cop0); 2773 } 2774 2775 if (cpu_guest_has_badinstr) 2776 kvm_save_gc0_badinstr(cop0); 2777 if (cpu_guest_has_badinstrp) 2778 kvm_save_gc0_badinstrp(cop0); 2779 2780 if (cpu_guest_has_segments) { 2781 kvm_save_gc0_segctl0(cop0); 2782 kvm_save_gc0_segctl1(cop0); 2783 kvm_save_gc0_segctl2(cop0); 2784 } 2785 2786 /* save HTW registers if enabled in guest */ 2787 if (cpu_guest_has_ldpte || (cpu_guest_has_htw && 2788 kvm_read_sw_gc0_config3(cop0) & MIPS_CONF3_PW)) { 2789 kvm_save_gc0_pwbase(cop0); 2790 kvm_save_gc0_pwfield(cop0); 2791 kvm_save_gc0_pwsize(cop0); 2792 kvm_save_gc0_pwctl(cop0); 2793 } 2794 2795 kvm_vz_save_timer(vcpu); 2796 2797 /* save Root.GuestCtl2 in unused Guest guestctl2 register */ 2798 if (cpu_has_guestctl2) 2799 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL] = 2800 read_c0_guestctl2(); 2801 2802 return 0; 2803 } 2804 2805 /** 2806 * kvm_vz_resize_guest_vtlb() - Attempt to resize guest VTLB. 2807 * @size: Number of guest VTLB entries (0 < @size <= root VTLB entries). 2808 * 2809 * Attempt to resize the guest VTLB by writing guest Config registers. This is 2810 * necessary for cores with a shared root/guest TLB to avoid overlap with wired 2811 * entries in the root VTLB. 2812 * 2813 * Returns: The resulting guest VTLB size. 2814 */ 2815 static unsigned int kvm_vz_resize_guest_vtlb(unsigned int size) 2816 { 2817 unsigned int config4 = 0, ret = 0, limit; 2818 2819 /* Write MMUSize - 1 into guest Config registers */ 2820 if (cpu_guest_has_conf1) 2821 change_gc0_config1(MIPS_CONF1_TLBS, 2822 (size - 1) << MIPS_CONF1_TLBS_SHIFT); 2823 if (cpu_guest_has_conf4) { 2824 config4 = read_gc0_config4(); 2825 if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) == 2826 MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT) { 2827 config4 &= ~MIPS_CONF4_VTLBSIZEEXT; 2828 config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) << 2829 MIPS_CONF4_VTLBSIZEEXT_SHIFT; 2830 } else if ((config4 & MIPS_CONF4_MMUEXTDEF) == 2831 MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT) { 2832 config4 &= ~MIPS_CONF4_MMUSIZEEXT; 2833 config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) << 2834 MIPS_CONF4_MMUSIZEEXT_SHIFT; 2835 } 2836 write_gc0_config4(config4); 2837 } 2838 2839 /* 2840 * Set Guest.Wired.Limit = 0 (no limit up to Guest.MMUSize-1), unless it 2841 * would exceed Root.Wired.Limit (clearing Guest.Wired.Wired so write 2842 * not dropped) 2843 */ 2844 if (cpu_has_mips_r6) { 2845 limit = (read_c0_wired() & MIPSR6_WIRED_LIMIT) >> 2846 MIPSR6_WIRED_LIMIT_SHIFT; 2847 if (size - 1 <= limit) 2848 limit = 0; 2849 write_gc0_wired(limit << MIPSR6_WIRED_LIMIT_SHIFT); 2850 } 2851 2852 /* Read back MMUSize - 1 */ 2853 back_to_back_c0_hazard(); 2854 if (cpu_guest_has_conf1) 2855 ret = (read_gc0_config1() & MIPS_CONF1_TLBS) >> 2856 MIPS_CONF1_TLBS_SHIFT; 2857 if (config4) { 2858 if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) == 2859 MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT) 2860 ret |= ((config4 & MIPS_CONF4_VTLBSIZEEXT) >> 2861 MIPS_CONF4_VTLBSIZEEXT_SHIFT) << 2862 MIPS_CONF1_TLBS_SIZE; 2863 else if ((config4 & MIPS_CONF4_MMUEXTDEF) == 2864 MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT) 2865 ret |= ((config4 & MIPS_CONF4_MMUSIZEEXT) >> 2866 MIPS_CONF4_MMUSIZEEXT_SHIFT) << 2867 MIPS_CONF1_TLBS_SIZE; 2868 } 2869 return ret + 1; 2870 } 2871 2872 static int kvm_vz_hardware_enable(void) 2873 { 2874 unsigned int mmu_size, guest_mmu_size, ftlb_size; 2875 u64 guest_cvmctl, cvmvmconfig; 2876 2877 switch (current_cpu_type()) { 2878 case CPU_CAVIUM_OCTEON3: 2879 /* Set up guest timer/perfcount IRQ lines */ 2880 guest_cvmctl = read_gc0_cvmctl(); 2881 guest_cvmctl &= ~CVMCTL_IPTI; 2882 guest_cvmctl |= 7ull << CVMCTL_IPTI_SHIFT; 2883 guest_cvmctl &= ~CVMCTL_IPPCI; 2884 guest_cvmctl |= 6ull << CVMCTL_IPPCI_SHIFT; 2885 write_gc0_cvmctl(guest_cvmctl); 2886 2887 cvmvmconfig = read_c0_cvmvmconfig(); 2888 /* No I/O hole translation. */ 2889 cvmvmconfig |= CVMVMCONF_DGHT; 2890 /* Halve the root MMU size */ 2891 mmu_size = ((cvmvmconfig & CVMVMCONF_MMUSIZEM1) 2892 >> CVMVMCONF_MMUSIZEM1_S) + 1; 2893 guest_mmu_size = mmu_size / 2; 2894 mmu_size -= guest_mmu_size; 2895 cvmvmconfig &= ~CVMVMCONF_RMMUSIZEM1; 2896 cvmvmconfig |= mmu_size - 1; 2897 write_c0_cvmvmconfig(cvmvmconfig); 2898 2899 /* Update our records */ 2900 current_cpu_data.tlbsize = mmu_size; 2901 current_cpu_data.tlbsizevtlb = mmu_size; 2902 current_cpu_data.guest.tlbsize = guest_mmu_size; 2903 2904 /* Flush moved entries in new (guest) context */ 2905 kvm_vz_local_flush_guesttlb_all(); 2906 break; 2907 default: 2908 /* 2909 * ImgTec cores tend to use a shared root/guest TLB. To avoid 2910 * overlap of root wired and guest entries, the guest TLB may 2911 * need resizing. 2912 */ 2913 mmu_size = current_cpu_data.tlbsizevtlb; 2914 ftlb_size = current_cpu_data.tlbsize - mmu_size; 2915 2916 /* Try switching to maximum guest VTLB size for flush */ 2917 guest_mmu_size = kvm_vz_resize_guest_vtlb(mmu_size); 2918 current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size; 2919 kvm_vz_local_flush_guesttlb_all(); 2920 2921 /* 2922 * Reduce to make space for root wired entries and at least 2 2923 * root non-wired entries. This does assume that long-term wired 2924 * entries won't be added later. 2925 */ 2926 guest_mmu_size = mmu_size - num_wired_entries() - 2; 2927 guest_mmu_size = kvm_vz_resize_guest_vtlb(guest_mmu_size); 2928 current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size; 2929 2930 /* 2931 * Write the VTLB size, but if another CPU has already written, 2932 * check it matches or we won't provide a consistent view to the 2933 * guest. If this ever happens it suggests an asymmetric number 2934 * of wired entries. 2935 */ 2936 if (cmpxchg(&kvm_vz_guest_vtlb_size, 0, guest_mmu_size) && 2937 WARN(guest_mmu_size != kvm_vz_guest_vtlb_size, 2938 "Available guest VTLB size mismatch")) 2939 return -EINVAL; 2940 break; 2941 } 2942 2943 /* 2944 * Enable virtualization features granting guest direct control of 2945 * certain features: 2946 * CP0=1: Guest coprocessor 0 context. 2947 * AT=Guest: Guest MMU. 2948 * CG=1: Hit (virtual address) CACHE operations (optional). 2949 * CF=1: Guest Config registers. 2950 * CGI=1: Indexed flush CACHE operations (optional). 2951 */ 2952 write_c0_guestctl0(MIPS_GCTL0_CP0 | 2953 (MIPS_GCTL0_AT_GUEST << MIPS_GCTL0_AT_SHIFT) | 2954 MIPS_GCTL0_CG | MIPS_GCTL0_CF); 2955 if (cpu_has_guestctl0ext) { 2956 if (current_cpu_type() != CPU_LOONGSON64) 2957 set_c0_guestctl0ext(MIPS_GCTL0EXT_CGI); 2958 else 2959 clear_c0_guestctl0ext(MIPS_GCTL0EXT_CGI); 2960 } 2961 2962 if (cpu_has_guestid) { 2963 write_c0_guestctl1(0); 2964 kvm_vz_local_flush_roottlb_all_guests(); 2965 2966 GUESTID_MASK = current_cpu_data.guestid_mask; 2967 GUESTID_FIRST_VERSION = GUESTID_MASK + 1; 2968 GUESTID_VERSION_MASK = ~GUESTID_MASK; 2969 2970 current_cpu_data.guestid_cache = GUESTID_FIRST_VERSION; 2971 } 2972 2973 /* clear any pending injected virtual guest interrupts */ 2974 if (cpu_has_guestctl2) 2975 clear_c0_guestctl2(0x3f << 10); 2976 2977 #ifdef CONFIG_CPU_LOONGSON64 2978 /* Control guest CCA attribute */ 2979 if (cpu_has_csr()) 2980 csr_writel(csr_readl(0xffffffec) | 0x1, 0xffffffec); 2981 #endif 2982 2983 return 0; 2984 } 2985 2986 static void kvm_vz_hardware_disable(void) 2987 { 2988 u64 cvmvmconfig; 2989 unsigned int mmu_size; 2990 2991 /* Flush any remaining guest TLB entries */ 2992 kvm_vz_local_flush_guesttlb_all(); 2993 2994 switch (current_cpu_type()) { 2995 case CPU_CAVIUM_OCTEON3: 2996 /* 2997 * Allocate whole TLB for root. Existing guest TLB entries will 2998 * change ownership to the root TLB. We should be safe though as 2999 * they've already been flushed above while in guest TLB. 3000 */ 3001 cvmvmconfig = read_c0_cvmvmconfig(); 3002 mmu_size = ((cvmvmconfig & CVMVMCONF_MMUSIZEM1) 3003 >> CVMVMCONF_MMUSIZEM1_S) + 1; 3004 cvmvmconfig &= ~CVMVMCONF_RMMUSIZEM1; 3005 cvmvmconfig |= mmu_size - 1; 3006 write_c0_cvmvmconfig(cvmvmconfig); 3007 3008 /* Update our records */ 3009 current_cpu_data.tlbsize = mmu_size; 3010 current_cpu_data.tlbsizevtlb = mmu_size; 3011 current_cpu_data.guest.tlbsize = 0; 3012 3013 /* Flush moved entries in new (root) context */ 3014 local_flush_tlb_all(); 3015 break; 3016 } 3017 3018 if (cpu_has_guestid) { 3019 write_c0_guestctl1(0); 3020 kvm_vz_local_flush_roottlb_all_guests(); 3021 } 3022 } 3023 3024 static int kvm_vz_check_extension(struct kvm *kvm, long ext) 3025 { 3026 int r; 3027 3028 switch (ext) { 3029 case KVM_CAP_MIPS_VZ: 3030 /* we wouldn't be here unless cpu_has_vz */ 3031 r = 1; 3032 break; 3033 #ifdef CONFIG_64BIT 3034 case KVM_CAP_MIPS_64BIT: 3035 /* We support 64-bit registers/operations and addresses */ 3036 r = 2; 3037 break; 3038 #endif 3039 case KVM_CAP_IOEVENTFD: 3040 r = 1; 3041 break; 3042 default: 3043 r = 0; 3044 break; 3045 } 3046 3047 return r; 3048 } 3049 3050 static int kvm_vz_vcpu_init(struct kvm_vcpu *vcpu) 3051 { 3052 int i; 3053 3054 for_each_possible_cpu(i) 3055 vcpu->arch.vzguestid[i] = 0; 3056 3057 return 0; 3058 } 3059 3060 static void kvm_vz_vcpu_uninit(struct kvm_vcpu *vcpu) 3061 { 3062 int cpu; 3063 3064 /* 3065 * If the VCPU is freed and reused as another VCPU, we don't want the 3066 * matching pointer wrongly hanging around in last_vcpu[] or 3067 * last_exec_vcpu[]. 3068 */ 3069 for_each_possible_cpu(cpu) { 3070 if (last_vcpu[cpu] == vcpu) 3071 last_vcpu[cpu] = NULL; 3072 if (last_exec_vcpu[cpu] == vcpu) 3073 last_exec_vcpu[cpu] = NULL; 3074 } 3075 } 3076 3077 static int kvm_vz_vcpu_setup(struct kvm_vcpu *vcpu) 3078 { 3079 struct mips_coproc *cop0 = &vcpu->arch.cop0; 3080 unsigned long count_hz = 100*1000*1000; /* default to 100 MHz */ 3081 3082 /* 3083 * Start off the timer at the same frequency as the host timer, but the 3084 * soft timer doesn't handle frequencies greater than 1GHz yet. 3085 */ 3086 if (mips_hpt_frequency && mips_hpt_frequency <= NSEC_PER_SEC) 3087 count_hz = mips_hpt_frequency; 3088 kvm_mips_init_count(vcpu, count_hz); 3089 3090 /* 3091 * Initialize guest register state to valid architectural reset state. 3092 */ 3093 3094 /* PageGrain */ 3095 if (cpu_has_mips_r5 || cpu_has_mips_r6) 3096 kvm_write_sw_gc0_pagegrain(cop0, PG_RIE | PG_XIE | PG_IEC); 3097 /* Wired */ 3098 if (cpu_has_mips_r6) 3099 kvm_write_sw_gc0_wired(cop0, 3100 read_gc0_wired() & MIPSR6_WIRED_LIMIT); 3101 /* Status */ 3102 kvm_write_sw_gc0_status(cop0, ST0_BEV | ST0_ERL); 3103 if (cpu_has_mips_r5 || cpu_has_mips_r6) 3104 kvm_change_sw_gc0_status(cop0, ST0_FR, read_gc0_status()); 3105 /* IntCtl */ 3106 kvm_write_sw_gc0_intctl(cop0, read_gc0_intctl() & 3107 (INTCTLF_IPFDC | INTCTLF_IPPCI | INTCTLF_IPTI)); 3108 /* PRId */ 3109 kvm_write_sw_gc0_prid(cop0, boot_cpu_data.processor_id); 3110 /* EBase */ 3111 kvm_write_sw_gc0_ebase(cop0, (s32)0x80000000 | vcpu->vcpu_id); 3112 /* Config */ 3113 kvm_save_gc0_config(cop0); 3114 /* architecturally writable (e.g. from guest) */ 3115 kvm_change_sw_gc0_config(cop0, CONF_CM_CMASK, 3116 _page_cachable_default >> _CACHE_SHIFT); 3117 /* architecturally read only, but maybe writable from root */ 3118 kvm_change_sw_gc0_config(cop0, MIPS_CONF_MT, read_c0_config()); 3119 if (cpu_guest_has_conf1) { 3120 kvm_set_sw_gc0_config(cop0, MIPS_CONF_M); 3121 /* Config1 */ 3122 kvm_save_gc0_config1(cop0); 3123 /* architecturally read only, but maybe writable from root */ 3124 kvm_clear_sw_gc0_config1(cop0, MIPS_CONF1_C2 | 3125 MIPS_CONF1_MD | 3126 MIPS_CONF1_PC | 3127 MIPS_CONF1_WR | 3128 MIPS_CONF1_CA | 3129 MIPS_CONF1_FP); 3130 } 3131 if (cpu_guest_has_conf2) { 3132 kvm_set_sw_gc0_config1(cop0, MIPS_CONF_M); 3133 /* Config2 */ 3134 kvm_save_gc0_config2(cop0); 3135 } 3136 if (cpu_guest_has_conf3) { 3137 kvm_set_sw_gc0_config2(cop0, MIPS_CONF_M); 3138 /* Config3 */ 3139 kvm_save_gc0_config3(cop0); 3140 /* architecturally writable (e.g. from guest) */ 3141 kvm_clear_sw_gc0_config3(cop0, MIPS_CONF3_ISA_OE); 3142 /* architecturally read only, but maybe writable from root */ 3143 kvm_clear_sw_gc0_config3(cop0, MIPS_CONF3_MSA | 3144 MIPS_CONF3_BPG | 3145 MIPS_CONF3_ULRI | 3146 MIPS_CONF3_DSP | 3147 MIPS_CONF3_CTXTC | 3148 MIPS_CONF3_ITL | 3149 MIPS_CONF3_LPA | 3150 MIPS_CONF3_VEIC | 3151 MIPS_CONF3_VINT | 3152 MIPS_CONF3_SP | 3153 MIPS_CONF3_CDMM | 3154 MIPS_CONF3_MT | 3155 MIPS_CONF3_SM | 3156 MIPS_CONF3_TL); 3157 } 3158 if (cpu_guest_has_conf4) { 3159 kvm_set_sw_gc0_config3(cop0, MIPS_CONF_M); 3160 /* Config4 */ 3161 kvm_save_gc0_config4(cop0); 3162 } 3163 if (cpu_guest_has_conf5) { 3164 kvm_set_sw_gc0_config4(cop0, MIPS_CONF_M); 3165 /* Config5 */ 3166 kvm_save_gc0_config5(cop0); 3167 /* architecturally writable (e.g. from guest) */ 3168 kvm_clear_sw_gc0_config5(cop0, MIPS_CONF5_K | 3169 MIPS_CONF5_CV | 3170 MIPS_CONF5_MSAEN | 3171 MIPS_CONF5_UFE | 3172 MIPS_CONF5_FRE | 3173 MIPS_CONF5_SBRI | 3174 MIPS_CONF5_UFR); 3175 /* architecturally read only, but maybe writable from root */ 3176 kvm_clear_sw_gc0_config5(cop0, MIPS_CONF5_MRP); 3177 } 3178 3179 if (cpu_guest_has_contextconfig) { 3180 /* ContextConfig */ 3181 kvm_write_sw_gc0_contextconfig(cop0, 0x007ffff0); 3182 #ifdef CONFIG_64BIT 3183 /* XContextConfig */ 3184 /* bits SEGBITS-13+3:4 set */ 3185 kvm_write_sw_gc0_xcontextconfig(cop0, 3186 ((1ull << (cpu_vmbits - 13)) - 1) << 4); 3187 #endif 3188 } 3189 3190 /* Implementation dependent, use the legacy layout */ 3191 if (cpu_guest_has_segments) { 3192 /* SegCtl0, SegCtl1, SegCtl2 */ 3193 kvm_write_sw_gc0_segctl0(cop0, 0x00200010); 3194 kvm_write_sw_gc0_segctl1(cop0, 0x00000002 | 3195 (_page_cachable_default >> _CACHE_SHIFT) << 3196 (16 + MIPS_SEGCFG_C_SHIFT)); 3197 kvm_write_sw_gc0_segctl2(cop0, 0x00380438); 3198 } 3199 3200 /* reset HTW registers */ 3201 if (cpu_guest_has_htw && (cpu_has_mips_r5 || cpu_has_mips_r6)) { 3202 /* PWField */ 3203 kvm_write_sw_gc0_pwfield(cop0, 0x0c30c302); 3204 /* PWSize */ 3205 kvm_write_sw_gc0_pwsize(cop0, 1 << MIPS_PWSIZE_PTW_SHIFT); 3206 } 3207 3208 /* start with no pending virtual guest interrupts */ 3209 if (cpu_has_guestctl2) 3210 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL] = 0; 3211 3212 /* Put PC at reset vector */ 3213 vcpu->arch.pc = CKSEG1ADDR(0x1fc00000); 3214 3215 return 0; 3216 } 3217 3218 static void kvm_vz_prepare_flush_shadow(struct kvm *kvm) 3219 { 3220 if (!cpu_has_guestid) { 3221 /* 3222 * For each CPU there is a single GPA ASID used by all VCPUs in 3223 * the VM, so it doesn't make sense for the VCPUs to handle 3224 * invalidation of these ASIDs individually. 3225 * 3226 * Instead mark all CPUs as needing ASID invalidation in 3227 * asid_flush_mask, and kvm_flush_remote_tlbs(kvm) will 3228 * kick any running VCPUs so they check asid_flush_mask. 3229 */ 3230 cpumask_setall(&kvm->arch.asid_flush_mask); 3231 } 3232 } 3233 3234 static void kvm_vz_vcpu_reenter(struct kvm_vcpu *vcpu) 3235 { 3236 int cpu = smp_processor_id(); 3237 int preserve_guest_tlb; 3238 3239 preserve_guest_tlb = kvm_vz_check_requests(vcpu, cpu); 3240 3241 if (preserve_guest_tlb) 3242 kvm_vz_vcpu_save_wired(vcpu); 3243 3244 kvm_vz_vcpu_load_tlb(vcpu, cpu); 3245 3246 if (preserve_guest_tlb) 3247 kvm_vz_vcpu_load_wired(vcpu); 3248 } 3249 3250 static int kvm_vz_vcpu_run(struct kvm_vcpu *vcpu) 3251 { 3252 int cpu = smp_processor_id(); 3253 int r; 3254 3255 kvm_vz_acquire_htimer(vcpu); 3256 /* Check if we have any exceptions/interrupts pending */ 3257 kvm_mips_deliver_interrupts(vcpu, read_gc0_cause()); 3258 3259 kvm_vz_check_requests(vcpu, cpu); 3260 kvm_vz_vcpu_load_tlb(vcpu, cpu); 3261 kvm_vz_vcpu_load_wired(vcpu); 3262 3263 r = vcpu->arch.vcpu_run(vcpu); 3264 3265 kvm_vz_vcpu_save_wired(vcpu); 3266 3267 return r; 3268 } 3269 3270 static struct kvm_mips_callbacks kvm_vz_callbacks = { 3271 .handle_cop_unusable = kvm_trap_vz_handle_cop_unusable, 3272 .handle_tlb_mod = kvm_trap_vz_handle_tlb_st_miss, 3273 .handle_tlb_ld_miss = kvm_trap_vz_handle_tlb_ld_miss, 3274 .handle_tlb_st_miss = kvm_trap_vz_handle_tlb_st_miss, 3275 .handle_addr_err_st = kvm_trap_vz_no_handler, 3276 .handle_addr_err_ld = kvm_trap_vz_no_handler, 3277 .handle_syscall = kvm_trap_vz_no_handler, 3278 .handle_res_inst = kvm_trap_vz_no_handler, 3279 .handle_break = kvm_trap_vz_no_handler, 3280 .handle_msa_disabled = kvm_trap_vz_handle_msa_disabled, 3281 .handle_guest_exit = kvm_trap_vz_handle_guest_exit, 3282 3283 .hardware_enable = kvm_vz_hardware_enable, 3284 .hardware_disable = kvm_vz_hardware_disable, 3285 .check_extension = kvm_vz_check_extension, 3286 .vcpu_init = kvm_vz_vcpu_init, 3287 .vcpu_uninit = kvm_vz_vcpu_uninit, 3288 .vcpu_setup = kvm_vz_vcpu_setup, 3289 .prepare_flush_shadow = kvm_vz_prepare_flush_shadow, 3290 .gva_to_gpa = kvm_vz_gva_to_gpa_cb, 3291 .queue_timer_int = kvm_vz_queue_timer_int_cb, 3292 .dequeue_timer_int = kvm_vz_dequeue_timer_int_cb, 3293 .queue_io_int = kvm_vz_queue_io_int_cb, 3294 .dequeue_io_int = kvm_vz_dequeue_io_int_cb, 3295 .irq_deliver = kvm_vz_irq_deliver_cb, 3296 .irq_clear = kvm_vz_irq_clear_cb, 3297 .num_regs = kvm_vz_num_regs, 3298 .copy_reg_indices = kvm_vz_copy_reg_indices, 3299 .get_one_reg = kvm_vz_get_one_reg, 3300 .set_one_reg = kvm_vz_set_one_reg, 3301 .vcpu_load = kvm_vz_vcpu_load, 3302 .vcpu_put = kvm_vz_vcpu_put, 3303 .vcpu_run = kvm_vz_vcpu_run, 3304 .vcpu_reenter = kvm_vz_vcpu_reenter, 3305 }; 3306 3307 /* FIXME: Get rid of the callbacks now that trap-and-emulate is gone. */ 3308 const struct kvm_mips_callbacks * const kvm_mips_callbacks = &kvm_vz_callbacks; 3309 3310 int kvm_mips_emulation_init(void) 3311 { 3312 if (!cpu_has_vz) 3313 return -ENODEV; 3314 3315 /* 3316 * VZ requires at least 2 KScratch registers, so it should have been 3317 * possible to allocate pgd_reg. 3318 */ 3319 if (WARN(pgd_reg == -1, 3320 "pgd_reg not allocated even though cpu_has_vz\n")) 3321 return -ENODEV; 3322 3323 pr_info("Starting KVM with MIPS VZ extensions\n"); 3324 return 0; 3325 } 3326