1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * KVM/MIPS: Support for hardware virtualization extensions 7 * 8 * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. 9 * Authors: Yann Le Du <ledu@kymasys.com> 10 */ 11 12 #include <linux/errno.h> 13 #include <linux/err.h> 14 #include <linux/module.h> 15 #include <linux/preempt.h> 16 #include <linux/vmalloc.h> 17 #include <asm/cacheflush.h> 18 #include <asm/cacheops.h> 19 #include <asm/cmpxchg.h> 20 #include <asm/fpu.h> 21 #include <asm/hazards.h> 22 #include <asm/inst.h> 23 #include <asm/mmu_context.h> 24 #include <asm/r4kcache.h> 25 #include <asm/time.h> 26 #include <asm/tlb.h> 27 #include <asm/tlbex.h> 28 29 #include <linux/kvm_host.h> 30 31 #include "interrupt.h" 32 #include "loongson_regs.h" 33 34 #include "trace.h" 35 36 /* Pointers to last VCPU loaded on each physical CPU */ 37 static struct kvm_vcpu *last_vcpu[NR_CPUS]; 38 /* Pointers to last VCPU executed on each physical CPU */ 39 static struct kvm_vcpu *last_exec_vcpu[NR_CPUS]; 40 41 /* 42 * Number of guest VTLB entries to use, so we can catch inconsistency between 43 * CPUs. 44 */ 45 static unsigned int kvm_vz_guest_vtlb_size; 46 47 static inline long kvm_vz_read_gc0_ebase(void) 48 { 49 if (sizeof(long) == 8 && cpu_has_ebase_wg) 50 return read_gc0_ebase_64(); 51 else 52 return read_gc0_ebase(); 53 } 54 55 static inline void kvm_vz_write_gc0_ebase(long v) 56 { 57 /* 58 * First write with WG=1 to write upper bits, then write again in case 59 * WG should be left at 0. 60 * write_gc0_ebase_64() is no longer UNDEFINED since R6. 61 */ 62 if (sizeof(long) == 8 && 63 (cpu_has_mips64r6 || cpu_has_ebase_wg)) { 64 write_gc0_ebase_64(v | MIPS_EBASE_WG); 65 write_gc0_ebase_64(v); 66 } else { 67 write_gc0_ebase(v | MIPS_EBASE_WG); 68 write_gc0_ebase(v); 69 } 70 } 71 72 /* 73 * These Config bits may be writable by the guest: 74 * Config: [K23, KU] (!TLB), K0 75 * Config1: (none) 76 * Config2: [TU, SU] (impl) 77 * Config3: ISAOnExc 78 * Config4: FTLBPageSize 79 * Config5: K, CV, MSAEn, UFE, FRE, SBRI, UFR 80 */ 81 82 static inline unsigned int kvm_vz_config_guest_wrmask(struct kvm_vcpu *vcpu) 83 { 84 return CONF_CM_CMASK; 85 } 86 87 static inline unsigned int kvm_vz_config1_guest_wrmask(struct kvm_vcpu *vcpu) 88 { 89 return 0; 90 } 91 92 static inline unsigned int kvm_vz_config2_guest_wrmask(struct kvm_vcpu *vcpu) 93 { 94 return 0; 95 } 96 97 static inline unsigned int kvm_vz_config3_guest_wrmask(struct kvm_vcpu *vcpu) 98 { 99 return MIPS_CONF3_ISA_OE; 100 } 101 102 static inline unsigned int kvm_vz_config4_guest_wrmask(struct kvm_vcpu *vcpu) 103 { 104 /* no need to be exact */ 105 return MIPS_CONF4_VFTLBPAGESIZE; 106 } 107 108 static inline unsigned int kvm_vz_config5_guest_wrmask(struct kvm_vcpu *vcpu) 109 { 110 unsigned int mask = MIPS_CONF5_K | MIPS_CONF5_CV | MIPS_CONF5_SBRI; 111 112 /* Permit MSAEn changes if MSA supported and enabled */ 113 if (kvm_mips_guest_has_msa(&vcpu->arch)) 114 mask |= MIPS_CONF5_MSAEN; 115 116 /* 117 * Permit guest FPU mode changes if FPU is enabled and the relevant 118 * feature exists according to FIR register. 119 */ 120 if (kvm_mips_guest_has_fpu(&vcpu->arch)) { 121 if (cpu_has_ufr) 122 mask |= MIPS_CONF5_UFR; 123 if (cpu_has_fre) 124 mask |= MIPS_CONF5_FRE | MIPS_CONF5_UFE; 125 } 126 127 return mask; 128 } 129 130 static inline unsigned int kvm_vz_config6_guest_wrmask(struct kvm_vcpu *vcpu) 131 { 132 return MIPS_CONF6_LOONGSON_INTIMER | MIPS_CONF6_LOONGSON_EXTIMER; 133 } 134 135 /* 136 * VZ optionally allows these additional Config bits to be written by root: 137 * Config: M, [MT] 138 * Config1: M, [MMUSize-1, C2, MD, PC, WR, CA], FP 139 * Config2: M 140 * Config3: M, MSAP, [BPG], ULRI, [DSP2P, DSPP], CTXTC, [ITL, LPA, VEIC, 141 * VInt, SP, CDMM, MT, SM, TL] 142 * Config4: M, [VTLBSizeExt, MMUSizeExt] 143 * Config5: MRP 144 */ 145 146 static inline unsigned int kvm_vz_config_user_wrmask(struct kvm_vcpu *vcpu) 147 { 148 return kvm_vz_config_guest_wrmask(vcpu) | MIPS_CONF_M; 149 } 150 151 static inline unsigned int kvm_vz_config1_user_wrmask(struct kvm_vcpu *vcpu) 152 { 153 unsigned int mask = kvm_vz_config1_guest_wrmask(vcpu) | MIPS_CONF_M; 154 155 /* Permit FPU to be present if FPU is supported */ 156 if (kvm_mips_guest_can_have_fpu(&vcpu->arch)) 157 mask |= MIPS_CONF1_FP; 158 159 return mask; 160 } 161 162 static inline unsigned int kvm_vz_config2_user_wrmask(struct kvm_vcpu *vcpu) 163 { 164 return kvm_vz_config2_guest_wrmask(vcpu) | MIPS_CONF_M; 165 } 166 167 static inline unsigned int kvm_vz_config3_user_wrmask(struct kvm_vcpu *vcpu) 168 { 169 unsigned int mask = kvm_vz_config3_guest_wrmask(vcpu) | MIPS_CONF_M | 170 MIPS_CONF3_ULRI | MIPS_CONF3_CTXTC; 171 172 /* Permit MSA to be present if MSA is supported */ 173 if (kvm_mips_guest_can_have_msa(&vcpu->arch)) 174 mask |= MIPS_CONF3_MSA; 175 176 return mask; 177 } 178 179 static inline unsigned int kvm_vz_config4_user_wrmask(struct kvm_vcpu *vcpu) 180 { 181 return kvm_vz_config4_guest_wrmask(vcpu) | MIPS_CONF_M; 182 } 183 184 static inline unsigned int kvm_vz_config5_user_wrmask(struct kvm_vcpu *vcpu) 185 { 186 return kvm_vz_config5_guest_wrmask(vcpu) | MIPS_CONF5_MRP; 187 } 188 189 static inline unsigned int kvm_vz_config6_user_wrmask(struct kvm_vcpu *vcpu) 190 { 191 return kvm_vz_config6_guest_wrmask(vcpu) | 192 MIPS_CONF6_LOONGSON_SFBEN | MIPS_CONF6_LOONGSON_FTLBDIS; 193 } 194 195 static gpa_t kvm_vz_gva_to_gpa_cb(gva_t gva) 196 { 197 /* VZ guest has already converted gva to gpa */ 198 return gva; 199 } 200 201 static void kvm_vz_queue_irq(struct kvm_vcpu *vcpu, unsigned int priority) 202 { 203 set_bit(priority, &vcpu->arch.pending_exceptions); 204 clear_bit(priority, &vcpu->arch.pending_exceptions_clr); 205 } 206 207 static void kvm_vz_dequeue_irq(struct kvm_vcpu *vcpu, unsigned int priority) 208 { 209 clear_bit(priority, &vcpu->arch.pending_exceptions); 210 set_bit(priority, &vcpu->arch.pending_exceptions_clr); 211 } 212 213 static void kvm_vz_queue_timer_int_cb(struct kvm_vcpu *vcpu) 214 { 215 /* 216 * timer expiry is asynchronous to vcpu execution therefore defer guest 217 * cp0 accesses 218 */ 219 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER); 220 } 221 222 static void kvm_vz_dequeue_timer_int_cb(struct kvm_vcpu *vcpu) 223 { 224 /* 225 * timer expiry is asynchronous to vcpu execution therefore defer guest 226 * cp0 accesses 227 */ 228 kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_TIMER); 229 } 230 231 static void kvm_vz_queue_io_int_cb(struct kvm_vcpu *vcpu, 232 struct kvm_mips_interrupt *irq) 233 { 234 int intr = (int)irq->irq; 235 236 /* 237 * interrupts are asynchronous to vcpu execution therefore defer guest 238 * cp0 accesses 239 */ 240 kvm_vz_queue_irq(vcpu, kvm_irq_to_priority(intr)); 241 } 242 243 static void kvm_vz_dequeue_io_int_cb(struct kvm_vcpu *vcpu, 244 struct kvm_mips_interrupt *irq) 245 { 246 int intr = (int)irq->irq; 247 248 /* 249 * interrupts are asynchronous to vcpu execution therefore defer guest 250 * cp0 accesses 251 */ 252 kvm_vz_dequeue_irq(vcpu, kvm_irq_to_priority(-intr)); 253 } 254 255 static int kvm_vz_irq_deliver_cb(struct kvm_vcpu *vcpu, unsigned int priority, 256 u32 cause) 257 { 258 u32 irq = (priority < MIPS_EXC_MAX) ? 259 kvm_priority_to_irq[priority] : 0; 260 261 switch (priority) { 262 case MIPS_EXC_INT_TIMER: 263 set_gc0_cause(C_TI); 264 break; 265 266 case MIPS_EXC_INT_IO_1: 267 case MIPS_EXC_INT_IO_2: 268 case MIPS_EXC_INT_IPI_1: 269 case MIPS_EXC_INT_IPI_2: 270 if (cpu_has_guestctl2) 271 set_c0_guestctl2(irq); 272 else 273 set_gc0_cause(irq); 274 break; 275 276 default: 277 break; 278 } 279 280 clear_bit(priority, &vcpu->arch.pending_exceptions); 281 return 1; 282 } 283 284 static int kvm_vz_irq_clear_cb(struct kvm_vcpu *vcpu, unsigned int priority, 285 u32 cause) 286 { 287 u32 irq = (priority < MIPS_EXC_MAX) ? 288 kvm_priority_to_irq[priority] : 0; 289 290 switch (priority) { 291 case MIPS_EXC_INT_TIMER: 292 /* 293 * Call to kvm_write_c0_guest_compare() clears Cause.TI in 294 * kvm_mips_emulate_CP0(). Explicitly clear irq associated with 295 * Cause.IP[IPTI] if GuestCtl2 virtual interrupt register not 296 * supported or if not using GuestCtl2 Hardware Clear. 297 */ 298 if (cpu_has_guestctl2) { 299 if (!(read_c0_guestctl2() & (irq << 14))) 300 clear_c0_guestctl2(irq); 301 } else { 302 clear_gc0_cause(irq); 303 } 304 break; 305 306 case MIPS_EXC_INT_IO_1: 307 case MIPS_EXC_INT_IO_2: 308 case MIPS_EXC_INT_IPI_1: 309 case MIPS_EXC_INT_IPI_2: 310 /* Clear GuestCtl2.VIP irq if not using Hardware Clear */ 311 if (cpu_has_guestctl2) { 312 if (!(read_c0_guestctl2() & (irq << 14))) 313 clear_c0_guestctl2(irq); 314 } else { 315 clear_gc0_cause(irq); 316 } 317 break; 318 319 default: 320 break; 321 } 322 323 clear_bit(priority, &vcpu->arch.pending_exceptions_clr); 324 return 1; 325 } 326 327 /* 328 * VZ guest timer handling. 329 */ 330 331 /** 332 * kvm_vz_should_use_htimer() - Find whether to use the VZ hard guest timer. 333 * @vcpu: Virtual CPU. 334 * 335 * Returns: true if the VZ GTOffset & real guest CP0_Count should be used 336 * instead of software emulation of guest timer. 337 * false otherwise. 338 */ 339 static bool kvm_vz_should_use_htimer(struct kvm_vcpu *vcpu) 340 { 341 if (kvm_mips_count_disabled(vcpu)) 342 return false; 343 344 /* Chosen frequency must match real frequency */ 345 if (mips_hpt_frequency != vcpu->arch.count_hz) 346 return false; 347 348 /* We don't support a CP0_GTOffset with fewer bits than CP0_Count */ 349 if (current_cpu_data.gtoffset_mask != 0xffffffff) 350 return false; 351 352 return true; 353 } 354 355 /** 356 * _kvm_vz_restore_stimer() - Restore soft timer state. 357 * @vcpu: Virtual CPU. 358 * @compare: CP0_Compare register value, restored by caller. 359 * @cause: CP0_Cause register to restore. 360 * 361 * Restore VZ state relating to the soft timer. The hard timer can be enabled 362 * later. 363 */ 364 static void _kvm_vz_restore_stimer(struct kvm_vcpu *vcpu, u32 compare, 365 u32 cause) 366 { 367 /* 368 * Avoid spurious counter interrupts by setting Guest CP0_Count to just 369 * after Guest CP0_Compare. 370 */ 371 write_c0_gtoffset(compare - read_c0_count()); 372 373 back_to_back_c0_hazard(); 374 write_gc0_cause(cause); 375 } 376 377 /** 378 * _kvm_vz_restore_htimer() - Restore hard timer state. 379 * @vcpu: Virtual CPU. 380 * @compare: CP0_Compare register value, restored by caller. 381 * @cause: CP0_Cause register to restore. 382 * 383 * Restore hard timer Guest.Count & Guest.Cause taking care to preserve the 384 * value of Guest.CP0_Cause.TI while restoring Guest.CP0_Cause. 385 */ 386 static void _kvm_vz_restore_htimer(struct kvm_vcpu *vcpu, 387 u32 compare, u32 cause) 388 { 389 u32 start_count, after_count; 390 ktime_t freeze_time; 391 unsigned long flags; 392 393 /* 394 * Freeze the soft-timer and sync the guest CP0_Count with it. We do 395 * this with interrupts disabled to avoid latency. 396 */ 397 local_irq_save(flags); 398 freeze_time = kvm_mips_freeze_hrtimer(vcpu, &start_count); 399 write_c0_gtoffset(start_count - read_c0_count()); 400 local_irq_restore(flags); 401 402 /* restore guest CP0_Cause, as TI may already be set */ 403 back_to_back_c0_hazard(); 404 write_gc0_cause(cause); 405 406 /* 407 * The above sequence isn't atomic and would result in lost timer 408 * interrupts if we're not careful. Detect if a timer interrupt is due 409 * and assert it. 410 */ 411 back_to_back_c0_hazard(); 412 after_count = read_gc0_count(); 413 if (after_count - start_count > compare - start_count - 1) 414 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER); 415 } 416 417 /** 418 * kvm_vz_restore_timer() - Restore timer state. 419 * @vcpu: Virtual CPU. 420 * 421 * Restore soft timer state from saved context. 422 */ 423 static void kvm_vz_restore_timer(struct kvm_vcpu *vcpu) 424 { 425 struct mips_coproc *cop0 = vcpu->arch.cop0; 426 u32 cause, compare; 427 428 compare = kvm_read_sw_gc0_compare(cop0); 429 cause = kvm_read_sw_gc0_cause(cop0); 430 431 write_gc0_compare(compare); 432 _kvm_vz_restore_stimer(vcpu, compare, cause); 433 } 434 435 /** 436 * kvm_vz_acquire_htimer() - Switch to hard timer state. 437 * @vcpu: Virtual CPU. 438 * 439 * Restore hard timer state on top of existing soft timer state if possible. 440 * 441 * Since hard timer won't remain active over preemption, preemption should be 442 * disabled by the caller. 443 */ 444 void kvm_vz_acquire_htimer(struct kvm_vcpu *vcpu) 445 { 446 u32 gctl0; 447 448 gctl0 = read_c0_guestctl0(); 449 if (!(gctl0 & MIPS_GCTL0_GT) && kvm_vz_should_use_htimer(vcpu)) { 450 /* enable guest access to hard timer */ 451 write_c0_guestctl0(gctl0 | MIPS_GCTL0_GT); 452 453 _kvm_vz_restore_htimer(vcpu, read_gc0_compare(), 454 read_gc0_cause()); 455 } 456 } 457 458 /** 459 * _kvm_vz_save_htimer() - Switch to software emulation of guest timer. 460 * @vcpu: Virtual CPU. 461 * @compare: Pointer to write compare value to. 462 * @cause: Pointer to write cause value to. 463 * 464 * Save VZ guest timer state and switch to software emulation of guest CP0 465 * timer. The hard timer must already be in use, so preemption should be 466 * disabled. 467 */ 468 static void _kvm_vz_save_htimer(struct kvm_vcpu *vcpu, 469 u32 *out_compare, u32 *out_cause) 470 { 471 u32 cause, compare, before_count, end_count; 472 ktime_t before_time; 473 474 compare = read_gc0_compare(); 475 *out_compare = compare; 476 477 before_time = ktime_get(); 478 479 /* 480 * Record the CP0_Count *prior* to saving CP0_Cause, so we have a time 481 * at which no pending timer interrupt is missing. 482 */ 483 before_count = read_gc0_count(); 484 back_to_back_c0_hazard(); 485 cause = read_gc0_cause(); 486 *out_cause = cause; 487 488 /* 489 * Record a final CP0_Count which we will transfer to the soft-timer. 490 * This is recorded *after* saving CP0_Cause, so we don't get any timer 491 * interrupts from just after the final CP0_Count point. 492 */ 493 back_to_back_c0_hazard(); 494 end_count = read_gc0_count(); 495 496 /* 497 * The above sequence isn't atomic, so we could miss a timer interrupt 498 * between reading CP0_Cause and end_count. Detect and record any timer 499 * interrupt due between before_count and end_count. 500 */ 501 if (end_count - before_count > compare - before_count - 1) 502 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER); 503 504 /* 505 * Restore soft-timer, ignoring a small amount of negative drift due to 506 * delay between freeze_hrtimer and setting CP0_GTOffset. 507 */ 508 kvm_mips_restore_hrtimer(vcpu, before_time, end_count, -0x10000); 509 } 510 511 /** 512 * kvm_vz_save_timer() - Save guest timer state. 513 * @vcpu: Virtual CPU. 514 * 515 * Save VZ guest timer state and switch to soft guest timer if hard timer was in 516 * use. 517 */ 518 static void kvm_vz_save_timer(struct kvm_vcpu *vcpu) 519 { 520 struct mips_coproc *cop0 = vcpu->arch.cop0; 521 u32 gctl0, compare, cause; 522 523 gctl0 = read_c0_guestctl0(); 524 if (gctl0 & MIPS_GCTL0_GT) { 525 /* disable guest use of hard timer */ 526 write_c0_guestctl0(gctl0 & ~MIPS_GCTL0_GT); 527 528 /* save hard timer state */ 529 _kvm_vz_save_htimer(vcpu, &compare, &cause); 530 } else { 531 compare = read_gc0_compare(); 532 cause = read_gc0_cause(); 533 } 534 535 /* save timer-related state to VCPU context */ 536 kvm_write_sw_gc0_cause(cop0, cause); 537 kvm_write_sw_gc0_compare(cop0, compare); 538 } 539 540 /** 541 * kvm_vz_lose_htimer() - Ensure hard guest timer is not in use. 542 * @vcpu: Virtual CPU. 543 * 544 * Transfers the state of the hard guest timer to the soft guest timer, leaving 545 * guest state intact so it can continue to be used with the soft timer. 546 */ 547 void kvm_vz_lose_htimer(struct kvm_vcpu *vcpu) 548 { 549 u32 gctl0, compare, cause; 550 551 preempt_disable(); 552 gctl0 = read_c0_guestctl0(); 553 if (gctl0 & MIPS_GCTL0_GT) { 554 /* disable guest use of timer */ 555 write_c0_guestctl0(gctl0 & ~MIPS_GCTL0_GT); 556 557 /* switch to soft timer */ 558 _kvm_vz_save_htimer(vcpu, &compare, &cause); 559 560 /* leave soft timer in usable state */ 561 _kvm_vz_restore_stimer(vcpu, compare, cause); 562 } 563 preempt_enable(); 564 } 565 566 /** 567 * is_eva_access() - Find whether an instruction is an EVA memory accessor. 568 * @inst: 32-bit instruction encoding. 569 * 570 * Finds whether @inst encodes an EVA memory access instruction, which would 571 * indicate that emulation of it should access the user mode address space 572 * instead of the kernel mode address space. This matters for MUSUK segments 573 * which are TLB mapped for user mode but unmapped for kernel mode. 574 * 575 * Returns: Whether @inst encodes an EVA accessor instruction. 576 */ 577 static bool is_eva_access(union mips_instruction inst) 578 { 579 if (inst.spec3_format.opcode != spec3_op) 580 return false; 581 582 switch (inst.spec3_format.func) { 583 case lwle_op: 584 case lwre_op: 585 case cachee_op: 586 case sbe_op: 587 case she_op: 588 case sce_op: 589 case swe_op: 590 case swle_op: 591 case swre_op: 592 case prefe_op: 593 case lbue_op: 594 case lhue_op: 595 case lbe_op: 596 case lhe_op: 597 case lle_op: 598 case lwe_op: 599 return true; 600 default: 601 return false; 602 } 603 } 604 605 /** 606 * is_eva_am_mapped() - Find whether an access mode is mapped. 607 * @vcpu: KVM VCPU state. 608 * @am: 3-bit encoded access mode. 609 * @eu: Segment becomes unmapped and uncached when Status.ERL=1. 610 * 611 * Decode @am to find whether it encodes a mapped segment for the current VCPU 612 * state. Where necessary @eu and the actual instruction causing the fault are 613 * taken into account to make the decision. 614 * 615 * Returns: Whether the VCPU faulted on a TLB mapped address. 616 */ 617 static bool is_eva_am_mapped(struct kvm_vcpu *vcpu, unsigned int am, bool eu) 618 { 619 u32 am_lookup; 620 int err; 621 622 /* 623 * Interpret access control mode. We assume address errors will already 624 * have been caught by the guest, leaving us with: 625 * AM UM SM KM 31..24 23..16 626 * UK 0 000 Unm 0 0 627 * MK 1 001 TLB 1 628 * MSK 2 010 TLB TLB 1 629 * MUSK 3 011 TLB TLB TLB 1 630 * MUSUK 4 100 TLB TLB Unm 0 1 631 * USK 5 101 Unm Unm 0 0 632 * - 6 110 0 0 633 * UUSK 7 111 Unm Unm Unm 0 0 634 * 635 * We shift a magic value by AM across the sign bit to find if always 636 * TLB mapped, and if not shift by 8 again to find if it depends on KM. 637 */ 638 am_lookup = 0x70080000 << am; 639 if ((s32)am_lookup < 0) { 640 /* 641 * MK, MSK, MUSK 642 * Always TLB mapped, unless SegCtl.EU && ERL 643 */ 644 if (!eu || !(read_gc0_status() & ST0_ERL)) 645 return true; 646 } else { 647 am_lookup <<= 8; 648 if ((s32)am_lookup < 0) { 649 union mips_instruction inst; 650 unsigned int status; 651 u32 *opc; 652 653 /* 654 * MUSUK 655 * TLB mapped if not in kernel mode 656 */ 657 status = read_gc0_status(); 658 if (!(status & (ST0_EXL | ST0_ERL)) && 659 (status & ST0_KSU)) 660 return true; 661 /* 662 * EVA access instructions in kernel 663 * mode access user address space. 664 */ 665 opc = (u32 *)vcpu->arch.pc; 666 if (vcpu->arch.host_cp0_cause & CAUSEF_BD) 667 opc += 1; 668 err = kvm_get_badinstr(opc, vcpu, &inst.word); 669 if (!err && is_eva_access(inst)) 670 return true; 671 } 672 } 673 674 return false; 675 } 676 677 /** 678 * kvm_vz_gva_to_gpa() - Convert valid GVA to GPA. 679 * @vcpu: KVM VCPU state. 680 * @gva: Guest virtual address to convert. 681 * @gpa: Output guest physical address. 682 * 683 * Convert a guest virtual address (GVA) which is valid according to the guest 684 * context, to a guest physical address (GPA). 685 * 686 * Returns: 0 on success. 687 * -errno on failure. 688 */ 689 static int kvm_vz_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, 690 unsigned long *gpa) 691 { 692 u32 gva32 = gva; 693 unsigned long segctl; 694 695 if ((long)gva == (s32)gva32) { 696 /* Handle canonical 32-bit virtual address */ 697 if (cpu_guest_has_segments) { 698 unsigned long mask, pa; 699 700 switch (gva32 >> 29) { 701 case 0: 702 case 1: /* CFG5 (1GB) */ 703 segctl = read_gc0_segctl2() >> 16; 704 mask = (unsigned long)0xfc0000000ull; 705 break; 706 case 2: 707 case 3: /* CFG4 (1GB) */ 708 segctl = read_gc0_segctl2(); 709 mask = (unsigned long)0xfc0000000ull; 710 break; 711 case 4: /* CFG3 (512MB) */ 712 segctl = read_gc0_segctl1() >> 16; 713 mask = (unsigned long)0xfe0000000ull; 714 break; 715 case 5: /* CFG2 (512MB) */ 716 segctl = read_gc0_segctl1(); 717 mask = (unsigned long)0xfe0000000ull; 718 break; 719 case 6: /* CFG1 (512MB) */ 720 segctl = read_gc0_segctl0() >> 16; 721 mask = (unsigned long)0xfe0000000ull; 722 break; 723 case 7: /* CFG0 (512MB) */ 724 segctl = read_gc0_segctl0(); 725 mask = (unsigned long)0xfe0000000ull; 726 break; 727 default: 728 /* 729 * GCC 4.9 isn't smart enough to figure out that 730 * segctl and mask are always initialised. 731 */ 732 unreachable(); 733 } 734 735 if (is_eva_am_mapped(vcpu, (segctl >> 4) & 0x7, 736 segctl & 0x0008)) 737 goto tlb_mapped; 738 739 /* Unmapped, find guest physical address */ 740 pa = (segctl << 20) & mask; 741 pa |= gva32 & ~mask; 742 *gpa = pa; 743 return 0; 744 } else if ((s32)gva32 < (s32)0xc0000000) { 745 /* legacy unmapped KSeg0 or KSeg1 */ 746 *gpa = gva32 & 0x1fffffff; 747 return 0; 748 } 749 #ifdef CONFIG_64BIT 750 } else if ((gva & 0xc000000000000000) == 0x8000000000000000) { 751 /* XKPHYS */ 752 if (cpu_guest_has_segments) { 753 /* 754 * Each of the 8 regions can be overridden by SegCtl2.XR 755 * to use SegCtl1.XAM. 756 */ 757 segctl = read_gc0_segctl2(); 758 if (segctl & (1ull << (56 + ((gva >> 59) & 0x7)))) { 759 segctl = read_gc0_segctl1(); 760 if (is_eva_am_mapped(vcpu, (segctl >> 59) & 0x7, 761 0)) 762 goto tlb_mapped; 763 } 764 765 } 766 /* 767 * Traditionally fully unmapped. 768 * Bits 61:59 specify the CCA, which we can just mask off here. 769 * Bits 58:PABITS should be zero, but we shouldn't have got here 770 * if it wasn't. 771 */ 772 *gpa = gva & 0x07ffffffffffffff; 773 return 0; 774 #endif 775 } 776 777 tlb_mapped: 778 return kvm_vz_guest_tlb_lookup(vcpu, gva, gpa); 779 } 780 781 /** 782 * kvm_vz_badvaddr_to_gpa() - Convert GVA BadVAddr from root exception to GPA. 783 * @vcpu: KVM VCPU state. 784 * @badvaddr: Root BadVAddr. 785 * @gpa: Output guest physical address. 786 * 787 * VZ implementations are permitted to report guest virtual addresses (GVA) in 788 * BadVAddr on a root exception during guest execution, instead of the more 789 * convenient guest physical addresses (GPA). When we get a GVA, this function 790 * converts it to a GPA, taking into account guest segmentation and guest TLB 791 * state. 792 * 793 * Returns: 0 on success. 794 * -errno on failure. 795 */ 796 static int kvm_vz_badvaddr_to_gpa(struct kvm_vcpu *vcpu, unsigned long badvaddr, 797 unsigned long *gpa) 798 { 799 unsigned int gexccode = (vcpu->arch.host_cp0_guestctl0 & 800 MIPS_GCTL0_GEXC) >> MIPS_GCTL0_GEXC_SHIFT; 801 802 /* If BadVAddr is GPA, then all is well in the world */ 803 if (likely(gexccode == MIPS_GCTL0_GEXC_GPA)) { 804 *gpa = badvaddr; 805 return 0; 806 } 807 808 /* Otherwise we'd expect it to be GVA ... */ 809 if (WARN(gexccode != MIPS_GCTL0_GEXC_GVA, 810 "Unexpected gexccode %#x\n", gexccode)) 811 return -EINVAL; 812 813 /* ... and we need to perform the GVA->GPA translation in software */ 814 return kvm_vz_gva_to_gpa(vcpu, badvaddr, gpa); 815 } 816 817 static int kvm_trap_vz_no_handler(struct kvm_vcpu *vcpu) 818 { 819 u32 *opc = (u32 *) vcpu->arch.pc; 820 u32 cause = vcpu->arch.host_cp0_cause; 821 u32 exccode = (cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE; 822 unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr; 823 u32 inst = 0; 824 825 /* 826 * Fetch the instruction. 827 */ 828 if (cause & CAUSEF_BD) 829 opc += 1; 830 kvm_get_badinstr(opc, vcpu, &inst); 831 832 kvm_err("Exception Code: %d not handled @ PC: %p, inst: 0x%08x BadVaddr: %#lx Status: %#x\n", 833 exccode, opc, inst, badvaddr, 834 read_gc0_status()); 835 kvm_arch_vcpu_dump_regs(vcpu); 836 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 837 return RESUME_HOST; 838 } 839 840 static unsigned long mips_process_maar(unsigned int op, unsigned long val) 841 { 842 /* Mask off unused bits */ 843 unsigned long mask = 0xfffff000 | MIPS_MAAR_S | MIPS_MAAR_VL; 844 845 if (read_gc0_pagegrain() & PG_ELPA) 846 mask |= 0x00ffffff00000000ull; 847 if (cpu_guest_has_mvh) 848 mask |= MIPS_MAAR_VH; 849 850 /* Set or clear VH */ 851 if (op == mtc_op) { 852 /* clear VH */ 853 val &= ~MIPS_MAAR_VH; 854 } else if (op == dmtc_op) { 855 /* set VH to match VL */ 856 val &= ~MIPS_MAAR_VH; 857 if (val & MIPS_MAAR_VL) 858 val |= MIPS_MAAR_VH; 859 } 860 861 return val & mask; 862 } 863 864 static void kvm_write_maari(struct kvm_vcpu *vcpu, unsigned long val) 865 { 866 struct mips_coproc *cop0 = vcpu->arch.cop0; 867 868 val &= MIPS_MAARI_INDEX; 869 if (val == MIPS_MAARI_INDEX) 870 kvm_write_sw_gc0_maari(cop0, ARRAY_SIZE(vcpu->arch.maar) - 1); 871 else if (val < ARRAY_SIZE(vcpu->arch.maar)) 872 kvm_write_sw_gc0_maari(cop0, val); 873 } 874 875 static enum emulation_result kvm_vz_gpsi_cop0(union mips_instruction inst, 876 u32 *opc, u32 cause, 877 struct kvm_run *run, 878 struct kvm_vcpu *vcpu) 879 { 880 struct mips_coproc *cop0 = vcpu->arch.cop0; 881 enum emulation_result er = EMULATE_DONE; 882 u32 rt, rd, sel; 883 unsigned long curr_pc; 884 unsigned long val; 885 886 /* 887 * Update PC and hold onto current PC in case there is 888 * an error and we want to rollback the PC 889 */ 890 curr_pc = vcpu->arch.pc; 891 er = update_pc(vcpu, cause); 892 if (er == EMULATE_FAIL) 893 return er; 894 895 if (inst.co_format.co) { 896 switch (inst.co_format.func) { 897 case wait_op: 898 er = kvm_mips_emul_wait(vcpu); 899 break; 900 default: 901 er = EMULATE_FAIL; 902 } 903 } else { 904 rt = inst.c0r_format.rt; 905 rd = inst.c0r_format.rd; 906 sel = inst.c0r_format.sel; 907 908 switch (inst.c0r_format.rs) { 909 case dmfc_op: 910 case mfc_op: 911 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS 912 cop0->stat[rd][sel]++; 913 #endif 914 if (rd == MIPS_CP0_COUNT && 915 sel == 0) { /* Count */ 916 val = kvm_mips_read_count(vcpu); 917 } else if (rd == MIPS_CP0_COMPARE && 918 sel == 0) { /* Compare */ 919 val = read_gc0_compare(); 920 } else if (rd == MIPS_CP0_LLADDR && 921 sel == 0) { /* LLAddr */ 922 if (cpu_guest_has_rw_llb) 923 val = read_gc0_lladdr() & 924 MIPS_LLADDR_LLB; 925 else 926 val = 0; 927 } else if (rd == MIPS_CP0_LLADDR && 928 sel == 1 && /* MAAR */ 929 cpu_guest_has_maar && 930 !cpu_guest_has_dyn_maar) { 931 /* MAARI must be in range */ 932 BUG_ON(kvm_read_sw_gc0_maari(cop0) >= 933 ARRAY_SIZE(vcpu->arch.maar)); 934 val = vcpu->arch.maar[ 935 kvm_read_sw_gc0_maari(cop0)]; 936 } else if ((rd == MIPS_CP0_PRID && 937 (sel == 0 || /* PRid */ 938 sel == 2 || /* CDMMBase */ 939 sel == 3)) || /* CMGCRBase */ 940 (rd == MIPS_CP0_STATUS && 941 (sel == 2 || /* SRSCtl */ 942 sel == 3)) || /* SRSMap */ 943 (rd == MIPS_CP0_CONFIG && 944 (sel == 6 || /* Config6 */ 945 sel == 7)) || /* Config7 */ 946 (rd == MIPS_CP0_LLADDR && 947 (sel == 2) && /* MAARI */ 948 cpu_guest_has_maar && 949 !cpu_guest_has_dyn_maar) || 950 (rd == MIPS_CP0_ERRCTL && 951 (sel == 0))) { /* ErrCtl */ 952 val = cop0->reg[rd][sel]; 953 #ifdef CONFIG_CPU_LOONGSON64 954 } else if (rd == MIPS_CP0_DIAG && 955 (sel == 0)) { /* Diag */ 956 val = cop0->reg[rd][sel]; 957 #endif 958 } else { 959 val = 0; 960 er = EMULATE_FAIL; 961 } 962 963 if (er != EMULATE_FAIL) { 964 /* Sign extend */ 965 if (inst.c0r_format.rs == mfc_op) 966 val = (int)val; 967 vcpu->arch.gprs[rt] = val; 968 } 969 970 trace_kvm_hwr(vcpu, (inst.c0r_format.rs == mfc_op) ? 971 KVM_TRACE_MFC0 : KVM_TRACE_DMFC0, 972 KVM_TRACE_COP0(rd, sel), val); 973 break; 974 975 case dmtc_op: 976 case mtc_op: 977 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS 978 cop0->stat[rd][sel]++; 979 #endif 980 val = vcpu->arch.gprs[rt]; 981 trace_kvm_hwr(vcpu, (inst.c0r_format.rs == mtc_op) ? 982 KVM_TRACE_MTC0 : KVM_TRACE_DMTC0, 983 KVM_TRACE_COP0(rd, sel), val); 984 985 if (rd == MIPS_CP0_COUNT && 986 sel == 0) { /* Count */ 987 kvm_vz_lose_htimer(vcpu); 988 kvm_mips_write_count(vcpu, vcpu->arch.gprs[rt]); 989 } else if (rd == MIPS_CP0_COMPARE && 990 sel == 0) { /* Compare */ 991 kvm_mips_write_compare(vcpu, 992 vcpu->arch.gprs[rt], 993 true); 994 } else if (rd == MIPS_CP0_LLADDR && 995 sel == 0) { /* LLAddr */ 996 /* 997 * P5600 generates GPSI on guest MTC0 LLAddr. 998 * Only allow the guest to clear LLB. 999 */ 1000 if (cpu_guest_has_rw_llb && 1001 !(val & MIPS_LLADDR_LLB)) 1002 write_gc0_lladdr(0); 1003 } else if (rd == MIPS_CP0_LLADDR && 1004 sel == 1 && /* MAAR */ 1005 cpu_guest_has_maar && 1006 !cpu_guest_has_dyn_maar) { 1007 val = mips_process_maar(inst.c0r_format.rs, 1008 val); 1009 1010 /* MAARI must be in range */ 1011 BUG_ON(kvm_read_sw_gc0_maari(cop0) >= 1012 ARRAY_SIZE(vcpu->arch.maar)); 1013 vcpu->arch.maar[kvm_read_sw_gc0_maari(cop0)] = 1014 val; 1015 } else if (rd == MIPS_CP0_LLADDR && 1016 (sel == 2) && /* MAARI */ 1017 cpu_guest_has_maar && 1018 !cpu_guest_has_dyn_maar) { 1019 kvm_write_maari(vcpu, val); 1020 } else if (rd == MIPS_CP0_CONFIG && 1021 (sel == 6)) { 1022 cop0->reg[rd][sel] = (int)val; 1023 } else if (rd == MIPS_CP0_ERRCTL && 1024 (sel == 0)) { /* ErrCtl */ 1025 /* ignore the written value */ 1026 #ifdef CONFIG_CPU_LOONGSON64 1027 } else if (rd == MIPS_CP0_DIAG && 1028 (sel == 0)) { /* Diag */ 1029 unsigned long flags; 1030 1031 local_irq_save(flags); 1032 if (val & LOONGSON_DIAG_BTB) { 1033 /* Flush BTB */ 1034 set_c0_diag(LOONGSON_DIAG_BTB); 1035 } 1036 if (val & LOONGSON_DIAG_ITLB) { 1037 /* Flush ITLB */ 1038 set_c0_diag(LOONGSON_DIAG_ITLB); 1039 } 1040 if (val & LOONGSON_DIAG_DTLB) { 1041 /* Flush DTLB */ 1042 set_c0_diag(LOONGSON_DIAG_DTLB); 1043 } 1044 if (val & LOONGSON_DIAG_VTLB) { 1045 /* Flush VTLB */ 1046 kvm_loongson_clear_guest_vtlb(); 1047 } 1048 if (val & LOONGSON_DIAG_FTLB) { 1049 /* Flush FTLB */ 1050 kvm_loongson_clear_guest_ftlb(); 1051 } 1052 local_irq_restore(flags); 1053 #endif 1054 } else { 1055 er = EMULATE_FAIL; 1056 } 1057 break; 1058 1059 default: 1060 er = EMULATE_FAIL; 1061 break; 1062 } 1063 } 1064 /* Rollback PC only if emulation was unsuccessful */ 1065 if (er == EMULATE_FAIL) { 1066 kvm_err("[%#lx]%s: unsupported cop0 instruction 0x%08x\n", 1067 curr_pc, __func__, inst.word); 1068 1069 vcpu->arch.pc = curr_pc; 1070 } 1071 1072 return er; 1073 } 1074 1075 static enum emulation_result kvm_vz_gpsi_cache(union mips_instruction inst, 1076 u32 *opc, u32 cause, 1077 struct kvm_run *run, 1078 struct kvm_vcpu *vcpu) 1079 { 1080 enum emulation_result er = EMULATE_DONE; 1081 u32 cache, op_inst, op, base; 1082 s16 offset; 1083 struct kvm_vcpu_arch *arch = &vcpu->arch; 1084 unsigned long va, curr_pc; 1085 1086 /* 1087 * Update PC and hold onto current PC in case there is 1088 * an error and we want to rollback the PC 1089 */ 1090 curr_pc = vcpu->arch.pc; 1091 er = update_pc(vcpu, cause); 1092 if (er == EMULATE_FAIL) 1093 return er; 1094 1095 base = inst.i_format.rs; 1096 op_inst = inst.i_format.rt; 1097 if (cpu_has_mips_r6) 1098 offset = inst.spec3_format.simmediate; 1099 else 1100 offset = inst.i_format.simmediate; 1101 cache = op_inst & CacheOp_Cache; 1102 op = op_inst & CacheOp_Op; 1103 1104 va = arch->gprs[base] + offset; 1105 1106 kvm_debug("CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n", 1107 cache, op, base, arch->gprs[base], offset); 1108 1109 /* Secondary or tirtiary cache ops ignored */ 1110 if (cache != Cache_I && cache != Cache_D) 1111 return EMULATE_DONE; 1112 1113 switch (op_inst) { 1114 case Index_Invalidate_I: 1115 flush_icache_line_indexed(va); 1116 return EMULATE_DONE; 1117 case Index_Writeback_Inv_D: 1118 flush_dcache_line_indexed(va); 1119 return EMULATE_DONE; 1120 case Hit_Invalidate_I: 1121 case Hit_Invalidate_D: 1122 case Hit_Writeback_Inv_D: 1123 if (boot_cpu_type() == CPU_CAVIUM_OCTEON3) { 1124 /* We can just flush entire icache */ 1125 local_flush_icache_range(0, 0); 1126 return EMULATE_DONE; 1127 } 1128 1129 /* So far, other platforms support guest hit cache ops */ 1130 break; 1131 default: 1132 break; 1133 } 1134 1135 kvm_err("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n", 1136 curr_pc, vcpu->arch.gprs[31], cache, op, base, arch->gprs[base], 1137 offset); 1138 /* Rollback PC */ 1139 vcpu->arch.pc = curr_pc; 1140 1141 return EMULATE_FAIL; 1142 } 1143 1144 #ifdef CONFIG_CPU_LOONGSON64 1145 static enum emulation_result kvm_vz_gpsi_lwc2(union mips_instruction inst, 1146 u32 *opc, u32 cause, 1147 struct kvm_run *run, 1148 struct kvm_vcpu *vcpu) 1149 { 1150 unsigned int rs, rd; 1151 unsigned int hostcfg; 1152 unsigned long curr_pc; 1153 enum emulation_result er = EMULATE_DONE; 1154 1155 /* 1156 * Update PC and hold onto current PC in case there is 1157 * an error and we want to rollback the PC 1158 */ 1159 curr_pc = vcpu->arch.pc; 1160 er = update_pc(vcpu, cause); 1161 if (er == EMULATE_FAIL) 1162 return er; 1163 1164 rs = inst.loongson3_lscsr_format.rs; 1165 rd = inst.loongson3_lscsr_format.rd; 1166 switch (inst.loongson3_lscsr_format.fr) { 1167 case 0x8: /* Read CPUCFG */ 1168 ++vcpu->stat.vz_cpucfg_exits; 1169 hostcfg = read_cpucfg(vcpu->arch.gprs[rs]); 1170 1171 switch (vcpu->arch.gprs[rs]) { 1172 case LOONGSON_CFG0: 1173 vcpu->arch.gprs[rd] = 0x14c000; 1174 break; 1175 case LOONGSON_CFG1: 1176 hostcfg &= (LOONGSON_CFG1_FP | LOONGSON_CFG1_MMI | 1177 LOONGSON_CFG1_MSA1 | LOONGSON_CFG1_MSA2 | 1178 LOONGSON_CFG1_SFBP); 1179 vcpu->arch.gprs[rd] = hostcfg; 1180 break; 1181 case LOONGSON_CFG2: 1182 hostcfg &= (LOONGSON_CFG2_LEXT1 | LOONGSON_CFG2_LEXT2 | 1183 LOONGSON_CFG2_LEXT3 | LOONGSON_CFG2_LSPW); 1184 vcpu->arch.gprs[rd] = hostcfg; 1185 break; 1186 case LOONGSON_CFG3: 1187 vcpu->arch.gprs[rd] = hostcfg; 1188 break; 1189 default: 1190 /* Don't export any other advanced features to guest */ 1191 vcpu->arch.gprs[rd] = 0; 1192 break; 1193 } 1194 break; 1195 1196 default: 1197 kvm_err("lwc2 emulate not impl %d rs %lx @%lx\n", 1198 inst.loongson3_lscsr_format.fr, vcpu->arch.gprs[rs], curr_pc); 1199 er = EMULATE_FAIL; 1200 break; 1201 } 1202 1203 /* Rollback PC only if emulation was unsuccessful */ 1204 if (er == EMULATE_FAIL) { 1205 kvm_err("[%#lx]%s: unsupported lwc2 instruction 0x%08x 0x%08x\n", 1206 curr_pc, __func__, inst.word, inst.loongson3_lscsr_format.fr); 1207 1208 vcpu->arch.pc = curr_pc; 1209 } 1210 1211 return er; 1212 } 1213 #endif 1214 1215 static enum emulation_result kvm_trap_vz_handle_gpsi(u32 cause, u32 *opc, 1216 struct kvm_vcpu *vcpu) 1217 { 1218 enum emulation_result er = EMULATE_DONE; 1219 struct kvm_vcpu_arch *arch = &vcpu->arch; 1220 struct kvm_run *run = vcpu->run; 1221 union mips_instruction inst; 1222 int rd, rt, sel; 1223 int err; 1224 1225 /* 1226 * Fetch the instruction. 1227 */ 1228 if (cause & CAUSEF_BD) 1229 opc += 1; 1230 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1231 if (err) 1232 return EMULATE_FAIL; 1233 1234 switch (inst.r_format.opcode) { 1235 case cop0_op: 1236 er = kvm_vz_gpsi_cop0(inst, opc, cause, run, vcpu); 1237 break; 1238 #ifndef CONFIG_CPU_MIPSR6 1239 case cache_op: 1240 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE); 1241 er = kvm_vz_gpsi_cache(inst, opc, cause, run, vcpu); 1242 break; 1243 #endif 1244 #ifdef CONFIG_CPU_LOONGSON64 1245 case lwc2_op: 1246 er = kvm_vz_gpsi_lwc2(inst, opc, cause, run, vcpu); 1247 break; 1248 #endif 1249 case spec3_op: 1250 switch (inst.spec3_format.func) { 1251 #ifdef CONFIG_CPU_MIPSR6 1252 case cache6_op: 1253 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE); 1254 er = kvm_vz_gpsi_cache(inst, opc, cause, run, vcpu); 1255 break; 1256 #endif 1257 case rdhwr_op: 1258 if (inst.r_format.rs || (inst.r_format.re >> 3)) 1259 goto unknown; 1260 1261 rd = inst.r_format.rd; 1262 rt = inst.r_format.rt; 1263 sel = inst.r_format.re & 0x7; 1264 1265 switch (rd) { 1266 case MIPS_HWR_CC: /* Read count register */ 1267 arch->gprs[rt] = 1268 (long)(int)kvm_mips_read_count(vcpu); 1269 break; 1270 default: 1271 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, 1272 KVM_TRACE_HWR(rd, sel), 0); 1273 goto unknown; 1274 } 1275 1276 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, 1277 KVM_TRACE_HWR(rd, sel), arch->gprs[rt]); 1278 1279 er = update_pc(vcpu, cause); 1280 break; 1281 default: 1282 goto unknown; 1283 } 1284 break; 1285 unknown: 1286 1287 default: 1288 kvm_err("GPSI exception not supported (%p/%#x)\n", 1289 opc, inst.word); 1290 kvm_arch_vcpu_dump_regs(vcpu); 1291 er = EMULATE_FAIL; 1292 break; 1293 } 1294 1295 return er; 1296 } 1297 1298 static enum emulation_result kvm_trap_vz_handle_gsfc(u32 cause, u32 *opc, 1299 struct kvm_vcpu *vcpu) 1300 { 1301 enum emulation_result er = EMULATE_DONE; 1302 struct kvm_vcpu_arch *arch = &vcpu->arch; 1303 union mips_instruction inst; 1304 int err; 1305 1306 /* 1307 * Fetch the instruction. 1308 */ 1309 if (cause & CAUSEF_BD) 1310 opc += 1; 1311 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1312 if (err) 1313 return EMULATE_FAIL; 1314 1315 /* complete MTC0 on behalf of guest and advance EPC */ 1316 if (inst.c0r_format.opcode == cop0_op && 1317 inst.c0r_format.rs == mtc_op && 1318 inst.c0r_format.z == 0) { 1319 int rt = inst.c0r_format.rt; 1320 int rd = inst.c0r_format.rd; 1321 int sel = inst.c0r_format.sel; 1322 unsigned int val = arch->gprs[rt]; 1323 unsigned int old_val, change; 1324 1325 trace_kvm_hwr(vcpu, KVM_TRACE_MTC0, KVM_TRACE_COP0(rd, sel), 1326 val); 1327 1328 if ((rd == MIPS_CP0_STATUS) && (sel == 0)) { 1329 /* FR bit should read as zero if no FPU */ 1330 if (!kvm_mips_guest_has_fpu(&vcpu->arch)) 1331 val &= ~(ST0_CU1 | ST0_FR); 1332 1333 /* 1334 * Also don't allow FR to be set if host doesn't support 1335 * it. 1336 */ 1337 if (!(boot_cpu_data.fpu_id & MIPS_FPIR_F64)) 1338 val &= ~ST0_FR; 1339 1340 old_val = read_gc0_status(); 1341 change = val ^ old_val; 1342 1343 if (change & ST0_FR) { 1344 /* 1345 * FPU and Vector register state is made 1346 * UNPREDICTABLE by a change of FR, so don't 1347 * even bother saving it. 1348 */ 1349 kvm_drop_fpu(vcpu); 1350 } 1351 1352 /* 1353 * If MSA state is already live, it is undefined how it 1354 * interacts with FR=0 FPU state, and we don't want to 1355 * hit reserved instruction exceptions trying to save 1356 * the MSA state later when CU=1 && FR=1, so play it 1357 * safe and save it first. 1358 */ 1359 if (change & ST0_CU1 && !(val & ST0_FR) && 1360 vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) 1361 kvm_lose_fpu(vcpu); 1362 1363 write_gc0_status(val); 1364 } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) { 1365 u32 old_cause = read_gc0_cause(); 1366 u32 change = old_cause ^ val; 1367 1368 /* DC bit enabling/disabling timer? */ 1369 if (change & CAUSEF_DC) { 1370 if (val & CAUSEF_DC) { 1371 kvm_vz_lose_htimer(vcpu); 1372 kvm_mips_count_disable_cause(vcpu); 1373 } else { 1374 kvm_mips_count_enable_cause(vcpu); 1375 } 1376 } 1377 1378 /* Only certain bits are RW to the guest */ 1379 change &= (CAUSEF_DC | CAUSEF_IV | CAUSEF_WP | 1380 CAUSEF_IP0 | CAUSEF_IP1); 1381 1382 /* WP can only be cleared */ 1383 change &= ~CAUSEF_WP | old_cause; 1384 1385 write_gc0_cause(old_cause ^ change); 1386 } else if ((rd == MIPS_CP0_STATUS) && (sel == 1)) { /* IntCtl */ 1387 write_gc0_intctl(val); 1388 } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) { 1389 old_val = read_gc0_config5(); 1390 change = val ^ old_val; 1391 /* Handle changes in FPU/MSA modes */ 1392 preempt_disable(); 1393 1394 /* 1395 * Propagate FRE changes immediately if the FPU 1396 * context is already loaded. 1397 */ 1398 if (change & MIPS_CONF5_FRE && 1399 vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU) 1400 change_c0_config5(MIPS_CONF5_FRE, val); 1401 1402 preempt_enable(); 1403 1404 val = old_val ^ 1405 (change & kvm_vz_config5_guest_wrmask(vcpu)); 1406 write_gc0_config5(val); 1407 } else { 1408 kvm_err("Handle GSFC, unsupported field change @ %p: %#x\n", 1409 opc, inst.word); 1410 er = EMULATE_FAIL; 1411 } 1412 1413 if (er != EMULATE_FAIL) 1414 er = update_pc(vcpu, cause); 1415 } else { 1416 kvm_err("Handle GSFC, unrecognized instruction @ %p: %#x\n", 1417 opc, inst.word); 1418 er = EMULATE_FAIL; 1419 } 1420 1421 return er; 1422 } 1423 1424 static enum emulation_result kvm_trap_vz_handle_ghfc(u32 cause, u32 *opc, 1425 struct kvm_vcpu *vcpu) 1426 { 1427 /* 1428 * Presumably this is due to MC (guest mode change), so lets trace some 1429 * relevant info. 1430 */ 1431 trace_kvm_guest_mode_change(vcpu); 1432 1433 return EMULATE_DONE; 1434 } 1435 1436 static enum emulation_result kvm_trap_vz_handle_hc(u32 cause, u32 *opc, 1437 struct kvm_vcpu *vcpu) 1438 { 1439 enum emulation_result er; 1440 union mips_instruction inst; 1441 unsigned long curr_pc; 1442 int err; 1443 1444 if (cause & CAUSEF_BD) 1445 opc += 1; 1446 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1447 if (err) 1448 return EMULATE_FAIL; 1449 1450 /* 1451 * Update PC and hold onto current PC in case there is 1452 * an error and we want to rollback the PC 1453 */ 1454 curr_pc = vcpu->arch.pc; 1455 er = update_pc(vcpu, cause); 1456 if (er == EMULATE_FAIL) 1457 return er; 1458 1459 er = kvm_mips_emul_hypcall(vcpu, inst); 1460 if (er == EMULATE_FAIL) 1461 vcpu->arch.pc = curr_pc; 1462 1463 return er; 1464 } 1465 1466 static enum emulation_result kvm_trap_vz_no_handler_guest_exit(u32 gexccode, 1467 u32 cause, 1468 u32 *opc, 1469 struct kvm_vcpu *vcpu) 1470 { 1471 u32 inst; 1472 1473 /* 1474 * Fetch the instruction. 1475 */ 1476 if (cause & CAUSEF_BD) 1477 opc += 1; 1478 kvm_get_badinstr(opc, vcpu, &inst); 1479 1480 kvm_err("Guest Exception Code: %d not yet handled @ PC: %p, inst: 0x%08x Status: %#x\n", 1481 gexccode, opc, inst, read_gc0_status()); 1482 1483 return EMULATE_FAIL; 1484 } 1485 1486 static int kvm_trap_vz_handle_guest_exit(struct kvm_vcpu *vcpu) 1487 { 1488 u32 *opc = (u32 *) vcpu->arch.pc; 1489 u32 cause = vcpu->arch.host_cp0_cause; 1490 enum emulation_result er = EMULATE_DONE; 1491 u32 gexccode = (vcpu->arch.host_cp0_guestctl0 & 1492 MIPS_GCTL0_GEXC) >> MIPS_GCTL0_GEXC_SHIFT; 1493 int ret = RESUME_GUEST; 1494 1495 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_GEXCCODE_BASE + gexccode); 1496 switch (gexccode) { 1497 case MIPS_GCTL0_GEXC_GPSI: 1498 ++vcpu->stat.vz_gpsi_exits; 1499 er = kvm_trap_vz_handle_gpsi(cause, opc, vcpu); 1500 break; 1501 case MIPS_GCTL0_GEXC_GSFC: 1502 ++vcpu->stat.vz_gsfc_exits; 1503 er = kvm_trap_vz_handle_gsfc(cause, opc, vcpu); 1504 break; 1505 case MIPS_GCTL0_GEXC_HC: 1506 ++vcpu->stat.vz_hc_exits; 1507 er = kvm_trap_vz_handle_hc(cause, opc, vcpu); 1508 break; 1509 case MIPS_GCTL0_GEXC_GRR: 1510 ++vcpu->stat.vz_grr_exits; 1511 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1512 vcpu); 1513 break; 1514 case MIPS_GCTL0_GEXC_GVA: 1515 ++vcpu->stat.vz_gva_exits; 1516 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1517 vcpu); 1518 break; 1519 case MIPS_GCTL0_GEXC_GHFC: 1520 ++vcpu->stat.vz_ghfc_exits; 1521 er = kvm_trap_vz_handle_ghfc(cause, opc, vcpu); 1522 break; 1523 case MIPS_GCTL0_GEXC_GPA: 1524 ++vcpu->stat.vz_gpa_exits; 1525 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1526 vcpu); 1527 break; 1528 default: 1529 ++vcpu->stat.vz_resvd_exits; 1530 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1531 vcpu); 1532 break; 1533 1534 } 1535 1536 if (er == EMULATE_DONE) { 1537 ret = RESUME_GUEST; 1538 } else if (er == EMULATE_HYPERCALL) { 1539 ret = kvm_mips_handle_hypcall(vcpu); 1540 } else { 1541 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1542 ret = RESUME_HOST; 1543 } 1544 return ret; 1545 } 1546 1547 /** 1548 * kvm_trap_vz_handle_cop_unusuable() - Guest used unusable coprocessor. 1549 * @vcpu: Virtual CPU context. 1550 * 1551 * Handle when the guest attempts to use a coprocessor which hasn't been allowed 1552 * by the root context. 1553 */ 1554 static int kvm_trap_vz_handle_cop_unusable(struct kvm_vcpu *vcpu) 1555 { 1556 struct kvm_run *run = vcpu->run; 1557 u32 cause = vcpu->arch.host_cp0_cause; 1558 enum emulation_result er = EMULATE_FAIL; 1559 int ret = RESUME_GUEST; 1560 1561 if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1) { 1562 /* 1563 * If guest FPU not present, the FPU operation should have been 1564 * treated as a reserved instruction! 1565 * If FPU already in use, we shouldn't get this at all. 1566 */ 1567 if (WARN_ON(!kvm_mips_guest_has_fpu(&vcpu->arch) || 1568 vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)) { 1569 preempt_enable(); 1570 return EMULATE_FAIL; 1571 } 1572 1573 kvm_own_fpu(vcpu); 1574 er = EMULATE_DONE; 1575 } 1576 /* other coprocessors not handled */ 1577 1578 switch (er) { 1579 case EMULATE_DONE: 1580 ret = RESUME_GUEST; 1581 break; 1582 1583 case EMULATE_FAIL: 1584 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1585 ret = RESUME_HOST; 1586 break; 1587 1588 default: 1589 BUG(); 1590 } 1591 return ret; 1592 } 1593 1594 /** 1595 * kvm_trap_vz_handle_msa_disabled() - Guest used MSA while disabled in root. 1596 * @vcpu: Virtual CPU context. 1597 * 1598 * Handle when the guest attempts to use MSA when it is disabled in the root 1599 * context. 1600 */ 1601 static int kvm_trap_vz_handle_msa_disabled(struct kvm_vcpu *vcpu) 1602 { 1603 struct kvm_run *run = vcpu->run; 1604 1605 /* 1606 * If MSA not present or not exposed to guest or FR=0, the MSA operation 1607 * should have been treated as a reserved instruction! 1608 * Same if CU1=1, FR=0. 1609 * If MSA already in use, we shouldn't get this at all. 1610 */ 1611 if (!kvm_mips_guest_has_msa(&vcpu->arch) || 1612 (read_gc0_status() & (ST0_CU1 | ST0_FR)) == ST0_CU1 || 1613 !(read_gc0_config5() & MIPS_CONF5_MSAEN) || 1614 vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) { 1615 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1616 return RESUME_HOST; 1617 } 1618 1619 kvm_own_msa(vcpu); 1620 1621 return RESUME_GUEST; 1622 } 1623 1624 static int kvm_trap_vz_handle_tlb_ld_miss(struct kvm_vcpu *vcpu) 1625 { 1626 struct kvm_run *run = vcpu->run; 1627 u32 *opc = (u32 *) vcpu->arch.pc; 1628 u32 cause = vcpu->arch.host_cp0_cause; 1629 ulong badvaddr = vcpu->arch.host_cp0_badvaddr; 1630 union mips_instruction inst; 1631 enum emulation_result er = EMULATE_DONE; 1632 int err, ret = RESUME_GUEST; 1633 1634 if (kvm_mips_handle_vz_root_tlb_fault(badvaddr, vcpu, false)) { 1635 /* A code fetch fault doesn't count as an MMIO */ 1636 if (kvm_is_ifetch_fault(&vcpu->arch)) { 1637 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1638 return RESUME_HOST; 1639 } 1640 1641 /* Fetch the instruction */ 1642 if (cause & CAUSEF_BD) 1643 opc += 1; 1644 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1645 if (err) { 1646 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1647 return RESUME_HOST; 1648 } 1649 1650 /* Treat as MMIO */ 1651 er = kvm_mips_emulate_load(inst, cause, run, vcpu); 1652 if (er == EMULATE_FAIL) { 1653 kvm_err("Guest Emulate Load from MMIO space failed: PC: %p, BadVaddr: %#lx\n", 1654 opc, badvaddr); 1655 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1656 } 1657 } 1658 1659 if (er == EMULATE_DONE) { 1660 ret = RESUME_GUEST; 1661 } else if (er == EMULATE_DO_MMIO) { 1662 run->exit_reason = KVM_EXIT_MMIO; 1663 ret = RESUME_HOST; 1664 } else { 1665 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1666 ret = RESUME_HOST; 1667 } 1668 return ret; 1669 } 1670 1671 static int kvm_trap_vz_handle_tlb_st_miss(struct kvm_vcpu *vcpu) 1672 { 1673 struct kvm_run *run = vcpu->run; 1674 u32 *opc = (u32 *) vcpu->arch.pc; 1675 u32 cause = vcpu->arch.host_cp0_cause; 1676 ulong badvaddr = vcpu->arch.host_cp0_badvaddr; 1677 union mips_instruction inst; 1678 enum emulation_result er = EMULATE_DONE; 1679 int err; 1680 int ret = RESUME_GUEST; 1681 1682 /* Just try the access again if we couldn't do the translation */ 1683 if (kvm_vz_badvaddr_to_gpa(vcpu, badvaddr, &badvaddr)) 1684 return RESUME_GUEST; 1685 vcpu->arch.host_cp0_badvaddr = badvaddr; 1686 1687 if (kvm_mips_handle_vz_root_tlb_fault(badvaddr, vcpu, true)) { 1688 /* Fetch the instruction */ 1689 if (cause & CAUSEF_BD) 1690 opc += 1; 1691 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1692 if (err) { 1693 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1694 return RESUME_HOST; 1695 } 1696 1697 /* Treat as MMIO */ 1698 er = kvm_mips_emulate_store(inst, cause, run, vcpu); 1699 if (er == EMULATE_FAIL) { 1700 kvm_err("Guest Emulate Store to MMIO space failed: PC: %p, BadVaddr: %#lx\n", 1701 opc, badvaddr); 1702 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1703 } 1704 } 1705 1706 if (er == EMULATE_DONE) { 1707 ret = RESUME_GUEST; 1708 } else if (er == EMULATE_DO_MMIO) { 1709 run->exit_reason = KVM_EXIT_MMIO; 1710 ret = RESUME_HOST; 1711 } else { 1712 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1713 ret = RESUME_HOST; 1714 } 1715 return ret; 1716 } 1717 1718 static u64 kvm_vz_get_one_regs[] = { 1719 KVM_REG_MIPS_CP0_INDEX, 1720 KVM_REG_MIPS_CP0_ENTRYLO0, 1721 KVM_REG_MIPS_CP0_ENTRYLO1, 1722 KVM_REG_MIPS_CP0_CONTEXT, 1723 KVM_REG_MIPS_CP0_PAGEMASK, 1724 KVM_REG_MIPS_CP0_PAGEGRAIN, 1725 KVM_REG_MIPS_CP0_WIRED, 1726 KVM_REG_MIPS_CP0_HWRENA, 1727 KVM_REG_MIPS_CP0_BADVADDR, 1728 KVM_REG_MIPS_CP0_COUNT, 1729 KVM_REG_MIPS_CP0_ENTRYHI, 1730 KVM_REG_MIPS_CP0_COMPARE, 1731 KVM_REG_MIPS_CP0_STATUS, 1732 KVM_REG_MIPS_CP0_INTCTL, 1733 KVM_REG_MIPS_CP0_CAUSE, 1734 KVM_REG_MIPS_CP0_EPC, 1735 KVM_REG_MIPS_CP0_PRID, 1736 KVM_REG_MIPS_CP0_EBASE, 1737 KVM_REG_MIPS_CP0_CONFIG, 1738 KVM_REG_MIPS_CP0_CONFIG1, 1739 KVM_REG_MIPS_CP0_CONFIG2, 1740 KVM_REG_MIPS_CP0_CONFIG3, 1741 KVM_REG_MIPS_CP0_CONFIG4, 1742 KVM_REG_MIPS_CP0_CONFIG5, 1743 KVM_REG_MIPS_CP0_CONFIG6, 1744 #ifdef CONFIG_64BIT 1745 KVM_REG_MIPS_CP0_XCONTEXT, 1746 #endif 1747 KVM_REG_MIPS_CP0_ERROREPC, 1748 1749 KVM_REG_MIPS_COUNT_CTL, 1750 KVM_REG_MIPS_COUNT_RESUME, 1751 KVM_REG_MIPS_COUNT_HZ, 1752 }; 1753 1754 static u64 kvm_vz_get_one_regs_contextconfig[] = { 1755 KVM_REG_MIPS_CP0_CONTEXTCONFIG, 1756 #ifdef CONFIG_64BIT 1757 KVM_REG_MIPS_CP0_XCONTEXTCONFIG, 1758 #endif 1759 }; 1760 1761 static u64 kvm_vz_get_one_regs_segments[] = { 1762 KVM_REG_MIPS_CP0_SEGCTL0, 1763 KVM_REG_MIPS_CP0_SEGCTL1, 1764 KVM_REG_MIPS_CP0_SEGCTL2, 1765 }; 1766 1767 static u64 kvm_vz_get_one_regs_htw[] = { 1768 KVM_REG_MIPS_CP0_PWBASE, 1769 KVM_REG_MIPS_CP0_PWFIELD, 1770 KVM_REG_MIPS_CP0_PWSIZE, 1771 KVM_REG_MIPS_CP0_PWCTL, 1772 }; 1773 1774 static u64 kvm_vz_get_one_regs_kscratch[] = { 1775 KVM_REG_MIPS_CP0_KSCRATCH1, 1776 KVM_REG_MIPS_CP0_KSCRATCH2, 1777 KVM_REG_MIPS_CP0_KSCRATCH3, 1778 KVM_REG_MIPS_CP0_KSCRATCH4, 1779 KVM_REG_MIPS_CP0_KSCRATCH5, 1780 KVM_REG_MIPS_CP0_KSCRATCH6, 1781 }; 1782 1783 static unsigned long kvm_vz_num_regs(struct kvm_vcpu *vcpu) 1784 { 1785 unsigned long ret; 1786 1787 ret = ARRAY_SIZE(kvm_vz_get_one_regs); 1788 if (cpu_guest_has_userlocal) 1789 ++ret; 1790 if (cpu_guest_has_badinstr) 1791 ++ret; 1792 if (cpu_guest_has_badinstrp) 1793 ++ret; 1794 if (cpu_guest_has_contextconfig) 1795 ret += ARRAY_SIZE(kvm_vz_get_one_regs_contextconfig); 1796 if (cpu_guest_has_segments) 1797 ret += ARRAY_SIZE(kvm_vz_get_one_regs_segments); 1798 if (cpu_guest_has_htw || cpu_guest_has_ldpte) 1799 ret += ARRAY_SIZE(kvm_vz_get_one_regs_htw); 1800 if (cpu_guest_has_maar && !cpu_guest_has_dyn_maar) 1801 ret += 1 + ARRAY_SIZE(vcpu->arch.maar); 1802 ret += __arch_hweight8(cpu_data[0].guest.kscratch_mask); 1803 1804 return ret; 1805 } 1806 1807 static int kvm_vz_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices) 1808 { 1809 u64 index; 1810 unsigned int i; 1811 1812 if (copy_to_user(indices, kvm_vz_get_one_regs, 1813 sizeof(kvm_vz_get_one_regs))) 1814 return -EFAULT; 1815 indices += ARRAY_SIZE(kvm_vz_get_one_regs); 1816 1817 if (cpu_guest_has_userlocal) { 1818 index = KVM_REG_MIPS_CP0_USERLOCAL; 1819 if (copy_to_user(indices, &index, sizeof(index))) 1820 return -EFAULT; 1821 ++indices; 1822 } 1823 if (cpu_guest_has_badinstr) { 1824 index = KVM_REG_MIPS_CP0_BADINSTR; 1825 if (copy_to_user(indices, &index, sizeof(index))) 1826 return -EFAULT; 1827 ++indices; 1828 } 1829 if (cpu_guest_has_badinstrp) { 1830 index = KVM_REG_MIPS_CP0_BADINSTRP; 1831 if (copy_to_user(indices, &index, sizeof(index))) 1832 return -EFAULT; 1833 ++indices; 1834 } 1835 if (cpu_guest_has_contextconfig) { 1836 if (copy_to_user(indices, kvm_vz_get_one_regs_contextconfig, 1837 sizeof(kvm_vz_get_one_regs_contextconfig))) 1838 return -EFAULT; 1839 indices += ARRAY_SIZE(kvm_vz_get_one_regs_contextconfig); 1840 } 1841 if (cpu_guest_has_segments) { 1842 if (copy_to_user(indices, kvm_vz_get_one_regs_segments, 1843 sizeof(kvm_vz_get_one_regs_segments))) 1844 return -EFAULT; 1845 indices += ARRAY_SIZE(kvm_vz_get_one_regs_segments); 1846 } 1847 if (cpu_guest_has_htw || cpu_guest_has_ldpte) { 1848 if (copy_to_user(indices, kvm_vz_get_one_regs_htw, 1849 sizeof(kvm_vz_get_one_regs_htw))) 1850 return -EFAULT; 1851 indices += ARRAY_SIZE(kvm_vz_get_one_regs_htw); 1852 } 1853 if (cpu_guest_has_maar && !cpu_guest_has_dyn_maar) { 1854 for (i = 0; i < ARRAY_SIZE(vcpu->arch.maar); ++i) { 1855 index = KVM_REG_MIPS_CP0_MAAR(i); 1856 if (copy_to_user(indices, &index, sizeof(index))) 1857 return -EFAULT; 1858 ++indices; 1859 } 1860 1861 index = KVM_REG_MIPS_CP0_MAARI; 1862 if (copy_to_user(indices, &index, sizeof(index))) 1863 return -EFAULT; 1864 ++indices; 1865 } 1866 for (i = 0; i < 6; ++i) { 1867 if (!cpu_guest_has_kscr(i + 2)) 1868 continue; 1869 1870 if (copy_to_user(indices, &kvm_vz_get_one_regs_kscratch[i], 1871 sizeof(kvm_vz_get_one_regs_kscratch[i]))) 1872 return -EFAULT; 1873 ++indices; 1874 } 1875 1876 return 0; 1877 } 1878 1879 static inline s64 entrylo_kvm_to_user(unsigned long v) 1880 { 1881 s64 mask, ret = v; 1882 1883 if (BITS_PER_LONG == 32) { 1884 /* 1885 * KVM API exposes 64-bit version of the register, so move the 1886 * RI/XI bits up into place. 1887 */ 1888 mask = MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI; 1889 ret &= ~mask; 1890 ret |= ((s64)v & mask) << 32; 1891 } 1892 return ret; 1893 } 1894 1895 static inline unsigned long entrylo_user_to_kvm(s64 v) 1896 { 1897 unsigned long mask, ret = v; 1898 1899 if (BITS_PER_LONG == 32) { 1900 /* 1901 * KVM API exposes 64-bit versiono of the register, so move the 1902 * RI/XI bits down into place. 1903 */ 1904 mask = MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI; 1905 ret &= ~mask; 1906 ret |= (v >> 32) & mask; 1907 } 1908 return ret; 1909 } 1910 1911 static int kvm_vz_get_one_reg(struct kvm_vcpu *vcpu, 1912 const struct kvm_one_reg *reg, 1913 s64 *v) 1914 { 1915 struct mips_coproc *cop0 = vcpu->arch.cop0; 1916 unsigned int idx; 1917 1918 switch (reg->id) { 1919 case KVM_REG_MIPS_CP0_INDEX: 1920 *v = (long)read_gc0_index(); 1921 break; 1922 case KVM_REG_MIPS_CP0_ENTRYLO0: 1923 *v = entrylo_kvm_to_user(read_gc0_entrylo0()); 1924 break; 1925 case KVM_REG_MIPS_CP0_ENTRYLO1: 1926 *v = entrylo_kvm_to_user(read_gc0_entrylo1()); 1927 break; 1928 case KVM_REG_MIPS_CP0_CONTEXT: 1929 *v = (long)read_gc0_context(); 1930 break; 1931 case KVM_REG_MIPS_CP0_CONTEXTCONFIG: 1932 if (!cpu_guest_has_contextconfig) 1933 return -EINVAL; 1934 *v = read_gc0_contextconfig(); 1935 break; 1936 case KVM_REG_MIPS_CP0_USERLOCAL: 1937 if (!cpu_guest_has_userlocal) 1938 return -EINVAL; 1939 *v = read_gc0_userlocal(); 1940 break; 1941 #ifdef CONFIG_64BIT 1942 case KVM_REG_MIPS_CP0_XCONTEXTCONFIG: 1943 if (!cpu_guest_has_contextconfig) 1944 return -EINVAL; 1945 *v = read_gc0_xcontextconfig(); 1946 break; 1947 #endif 1948 case KVM_REG_MIPS_CP0_PAGEMASK: 1949 *v = (long)read_gc0_pagemask(); 1950 break; 1951 case KVM_REG_MIPS_CP0_PAGEGRAIN: 1952 *v = (long)read_gc0_pagegrain(); 1953 break; 1954 case KVM_REG_MIPS_CP0_SEGCTL0: 1955 if (!cpu_guest_has_segments) 1956 return -EINVAL; 1957 *v = read_gc0_segctl0(); 1958 break; 1959 case KVM_REG_MIPS_CP0_SEGCTL1: 1960 if (!cpu_guest_has_segments) 1961 return -EINVAL; 1962 *v = read_gc0_segctl1(); 1963 break; 1964 case KVM_REG_MIPS_CP0_SEGCTL2: 1965 if (!cpu_guest_has_segments) 1966 return -EINVAL; 1967 *v = read_gc0_segctl2(); 1968 break; 1969 case KVM_REG_MIPS_CP0_PWBASE: 1970 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 1971 return -EINVAL; 1972 *v = read_gc0_pwbase(); 1973 break; 1974 case KVM_REG_MIPS_CP0_PWFIELD: 1975 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 1976 return -EINVAL; 1977 *v = read_gc0_pwfield(); 1978 break; 1979 case KVM_REG_MIPS_CP0_PWSIZE: 1980 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 1981 return -EINVAL; 1982 *v = read_gc0_pwsize(); 1983 break; 1984 case KVM_REG_MIPS_CP0_WIRED: 1985 *v = (long)read_gc0_wired(); 1986 break; 1987 case KVM_REG_MIPS_CP0_PWCTL: 1988 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 1989 return -EINVAL; 1990 *v = read_gc0_pwctl(); 1991 break; 1992 case KVM_REG_MIPS_CP0_HWRENA: 1993 *v = (long)read_gc0_hwrena(); 1994 break; 1995 case KVM_REG_MIPS_CP0_BADVADDR: 1996 *v = (long)read_gc0_badvaddr(); 1997 break; 1998 case KVM_REG_MIPS_CP0_BADINSTR: 1999 if (!cpu_guest_has_badinstr) 2000 return -EINVAL; 2001 *v = read_gc0_badinstr(); 2002 break; 2003 case KVM_REG_MIPS_CP0_BADINSTRP: 2004 if (!cpu_guest_has_badinstrp) 2005 return -EINVAL; 2006 *v = read_gc0_badinstrp(); 2007 break; 2008 case KVM_REG_MIPS_CP0_COUNT: 2009 *v = kvm_mips_read_count(vcpu); 2010 break; 2011 case KVM_REG_MIPS_CP0_ENTRYHI: 2012 *v = (long)read_gc0_entryhi(); 2013 break; 2014 case KVM_REG_MIPS_CP0_COMPARE: 2015 *v = (long)read_gc0_compare(); 2016 break; 2017 case KVM_REG_MIPS_CP0_STATUS: 2018 *v = (long)read_gc0_status(); 2019 break; 2020 case KVM_REG_MIPS_CP0_INTCTL: 2021 *v = read_gc0_intctl(); 2022 break; 2023 case KVM_REG_MIPS_CP0_CAUSE: 2024 *v = (long)read_gc0_cause(); 2025 break; 2026 case KVM_REG_MIPS_CP0_EPC: 2027 *v = (long)read_gc0_epc(); 2028 break; 2029 case KVM_REG_MIPS_CP0_PRID: 2030 switch (boot_cpu_type()) { 2031 case CPU_CAVIUM_OCTEON3: 2032 /* Octeon III has a read-only guest.PRid */ 2033 *v = read_gc0_prid(); 2034 break; 2035 default: 2036 *v = (long)kvm_read_c0_guest_prid(cop0); 2037 break; 2038 } 2039 break; 2040 case KVM_REG_MIPS_CP0_EBASE: 2041 *v = kvm_vz_read_gc0_ebase(); 2042 break; 2043 case KVM_REG_MIPS_CP0_CONFIG: 2044 *v = read_gc0_config(); 2045 break; 2046 case KVM_REG_MIPS_CP0_CONFIG1: 2047 if (!cpu_guest_has_conf1) 2048 return -EINVAL; 2049 *v = read_gc0_config1(); 2050 break; 2051 case KVM_REG_MIPS_CP0_CONFIG2: 2052 if (!cpu_guest_has_conf2) 2053 return -EINVAL; 2054 *v = read_gc0_config2(); 2055 break; 2056 case KVM_REG_MIPS_CP0_CONFIG3: 2057 if (!cpu_guest_has_conf3) 2058 return -EINVAL; 2059 *v = read_gc0_config3(); 2060 break; 2061 case KVM_REG_MIPS_CP0_CONFIG4: 2062 if (!cpu_guest_has_conf4) 2063 return -EINVAL; 2064 *v = read_gc0_config4(); 2065 break; 2066 case KVM_REG_MIPS_CP0_CONFIG5: 2067 if (!cpu_guest_has_conf5) 2068 return -EINVAL; 2069 *v = read_gc0_config5(); 2070 break; 2071 case KVM_REG_MIPS_CP0_CONFIG6: 2072 *v = kvm_read_sw_gc0_config6(cop0); 2073 break; 2074 case KVM_REG_MIPS_CP0_MAAR(0) ... KVM_REG_MIPS_CP0_MAAR(0x3f): 2075 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar) 2076 return -EINVAL; 2077 idx = reg->id - KVM_REG_MIPS_CP0_MAAR(0); 2078 if (idx >= ARRAY_SIZE(vcpu->arch.maar)) 2079 return -EINVAL; 2080 *v = vcpu->arch.maar[idx]; 2081 break; 2082 case KVM_REG_MIPS_CP0_MAARI: 2083 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar) 2084 return -EINVAL; 2085 *v = kvm_read_sw_gc0_maari(vcpu->arch.cop0); 2086 break; 2087 #ifdef CONFIG_64BIT 2088 case KVM_REG_MIPS_CP0_XCONTEXT: 2089 *v = read_gc0_xcontext(); 2090 break; 2091 #endif 2092 case KVM_REG_MIPS_CP0_ERROREPC: 2093 *v = (long)read_gc0_errorepc(); 2094 break; 2095 case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6: 2096 idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2; 2097 if (!cpu_guest_has_kscr(idx)) 2098 return -EINVAL; 2099 switch (idx) { 2100 case 2: 2101 *v = (long)read_gc0_kscratch1(); 2102 break; 2103 case 3: 2104 *v = (long)read_gc0_kscratch2(); 2105 break; 2106 case 4: 2107 *v = (long)read_gc0_kscratch3(); 2108 break; 2109 case 5: 2110 *v = (long)read_gc0_kscratch4(); 2111 break; 2112 case 6: 2113 *v = (long)read_gc0_kscratch5(); 2114 break; 2115 case 7: 2116 *v = (long)read_gc0_kscratch6(); 2117 break; 2118 } 2119 break; 2120 case KVM_REG_MIPS_COUNT_CTL: 2121 *v = vcpu->arch.count_ctl; 2122 break; 2123 case KVM_REG_MIPS_COUNT_RESUME: 2124 *v = ktime_to_ns(vcpu->arch.count_resume); 2125 break; 2126 case KVM_REG_MIPS_COUNT_HZ: 2127 *v = vcpu->arch.count_hz; 2128 break; 2129 default: 2130 return -EINVAL; 2131 } 2132 return 0; 2133 } 2134 2135 static int kvm_vz_set_one_reg(struct kvm_vcpu *vcpu, 2136 const struct kvm_one_reg *reg, 2137 s64 v) 2138 { 2139 struct mips_coproc *cop0 = vcpu->arch.cop0; 2140 unsigned int idx; 2141 int ret = 0; 2142 unsigned int cur, change; 2143 2144 switch (reg->id) { 2145 case KVM_REG_MIPS_CP0_INDEX: 2146 write_gc0_index(v); 2147 break; 2148 case KVM_REG_MIPS_CP0_ENTRYLO0: 2149 write_gc0_entrylo0(entrylo_user_to_kvm(v)); 2150 break; 2151 case KVM_REG_MIPS_CP0_ENTRYLO1: 2152 write_gc0_entrylo1(entrylo_user_to_kvm(v)); 2153 break; 2154 case KVM_REG_MIPS_CP0_CONTEXT: 2155 write_gc0_context(v); 2156 break; 2157 case KVM_REG_MIPS_CP0_CONTEXTCONFIG: 2158 if (!cpu_guest_has_contextconfig) 2159 return -EINVAL; 2160 write_gc0_contextconfig(v); 2161 break; 2162 case KVM_REG_MIPS_CP0_USERLOCAL: 2163 if (!cpu_guest_has_userlocal) 2164 return -EINVAL; 2165 write_gc0_userlocal(v); 2166 break; 2167 #ifdef CONFIG_64BIT 2168 case KVM_REG_MIPS_CP0_XCONTEXTCONFIG: 2169 if (!cpu_guest_has_contextconfig) 2170 return -EINVAL; 2171 write_gc0_xcontextconfig(v); 2172 break; 2173 #endif 2174 case KVM_REG_MIPS_CP0_PAGEMASK: 2175 write_gc0_pagemask(v); 2176 break; 2177 case KVM_REG_MIPS_CP0_PAGEGRAIN: 2178 write_gc0_pagegrain(v); 2179 break; 2180 case KVM_REG_MIPS_CP0_SEGCTL0: 2181 if (!cpu_guest_has_segments) 2182 return -EINVAL; 2183 write_gc0_segctl0(v); 2184 break; 2185 case KVM_REG_MIPS_CP0_SEGCTL1: 2186 if (!cpu_guest_has_segments) 2187 return -EINVAL; 2188 write_gc0_segctl1(v); 2189 break; 2190 case KVM_REG_MIPS_CP0_SEGCTL2: 2191 if (!cpu_guest_has_segments) 2192 return -EINVAL; 2193 write_gc0_segctl2(v); 2194 break; 2195 case KVM_REG_MIPS_CP0_PWBASE: 2196 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 2197 return -EINVAL; 2198 write_gc0_pwbase(v); 2199 break; 2200 case KVM_REG_MIPS_CP0_PWFIELD: 2201 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 2202 return -EINVAL; 2203 write_gc0_pwfield(v); 2204 break; 2205 case KVM_REG_MIPS_CP0_PWSIZE: 2206 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 2207 return -EINVAL; 2208 write_gc0_pwsize(v); 2209 break; 2210 case KVM_REG_MIPS_CP0_WIRED: 2211 change_gc0_wired(MIPSR6_WIRED_WIRED, v); 2212 break; 2213 case KVM_REG_MIPS_CP0_PWCTL: 2214 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 2215 return -EINVAL; 2216 write_gc0_pwctl(v); 2217 break; 2218 case KVM_REG_MIPS_CP0_HWRENA: 2219 write_gc0_hwrena(v); 2220 break; 2221 case KVM_REG_MIPS_CP0_BADVADDR: 2222 write_gc0_badvaddr(v); 2223 break; 2224 case KVM_REG_MIPS_CP0_BADINSTR: 2225 if (!cpu_guest_has_badinstr) 2226 return -EINVAL; 2227 write_gc0_badinstr(v); 2228 break; 2229 case KVM_REG_MIPS_CP0_BADINSTRP: 2230 if (!cpu_guest_has_badinstrp) 2231 return -EINVAL; 2232 write_gc0_badinstrp(v); 2233 break; 2234 case KVM_REG_MIPS_CP0_COUNT: 2235 kvm_mips_write_count(vcpu, v); 2236 break; 2237 case KVM_REG_MIPS_CP0_ENTRYHI: 2238 write_gc0_entryhi(v); 2239 break; 2240 case KVM_REG_MIPS_CP0_COMPARE: 2241 kvm_mips_write_compare(vcpu, v, false); 2242 break; 2243 case KVM_REG_MIPS_CP0_STATUS: 2244 write_gc0_status(v); 2245 break; 2246 case KVM_REG_MIPS_CP0_INTCTL: 2247 write_gc0_intctl(v); 2248 break; 2249 case KVM_REG_MIPS_CP0_CAUSE: 2250 /* 2251 * If the timer is stopped or started (DC bit) it must look 2252 * atomic with changes to the timer interrupt pending bit (TI). 2253 * A timer interrupt should not happen in between. 2254 */ 2255 if ((read_gc0_cause() ^ v) & CAUSEF_DC) { 2256 if (v & CAUSEF_DC) { 2257 /* disable timer first */ 2258 kvm_mips_count_disable_cause(vcpu); 2259 change_gc0_cause((u32)~CAUSEF_DC, v); 2260 } else { 2261 /* enable timer last */ 2262 change_gc0_cause((u32)~CAUSEF_DC, v); 2263 kvm_mips_count_enable_cause(vcpu); 2264 } 2265 } else { 2266 write_gc0_cause(v); 2267 } 2268 break; 2269 case KVM_REG_MIPS_CP0_EPC: 2270 write_gc0_epc(v); 2271 break; 2272 case KVM_REG_MIPS_CP0_PRID: 2273 switch (boot_cpu_type()) { 2274 case CPU_CAVIUM_OCTEON3: 2275 /* Octeon III has a guest.PRid, but its read-only */ 2276 break; 2277 default: 2278 kvm_write_c0_guest_prid(cop0, v); 2279 break; 2280 } 2281 break; 2282 case KVM_REG_MIPS_CP0_EBASE: 2283 kvm_vz_write_gc0_ebase(v); 2284 break; 2285 case KVM_REG_MIPS_CP0_CONFIG: 2286 cur = read_gc0_config(); 2287 change = (cur ^ v) & kvm_vz_config_user_wrmask(vcpu); 2288 if (change) { 2289 v = cur ^ change; 2290 write_gc0_config(v); 2291 } 2292 break; 2293 case KVM_REG_MIPS_CP0_CONFIG1: 2294 if (!cpu_guest_has_conf1) 2295 break; 2296 cur = read_gc0_config1(); 2297 change = (cur ^ v) & kvm_vz_config1_user_wrmask(vcpu); 2298 if (change) { 2299 v = cur ^ change; 2300 write_gc0_config1(v); 2301 } 2302 break; 2303 case KVM_REG_MIPS_CP0_CONFIG2: 2304 if (!cpu_guest_has_conf2) 2305 break; 2306 cur = read_gc0_config2(); 2307 change = (cur ^ v) & kvm_vz_config2_user_wrmask(vcpu); 2308 if (change) { 2309 v = cur ^ change; 2310 write_gc0_config2(v); 2311 } 2312 break; 2313 case KVM_REG_MIPS_CP0_CONFIG3: 2314 if (!cpu_guest_has_conf3) 2315 break; 2316 cur = read_gc0_config3(); 2317 change = (cur ^ v) & kvm_vz_config3_user_wrmask(vcpu); 2318 if (change) { 2319 v = cur ^ change; 2320 write_gc0_config3(v); 2321 } 2322 break; 2323 case KVM_REG_MIPS_CP0_CONFIG4: 2324 if (!cpu_guest_has_conf4) 2325 break; 2326 cur = read_gc0_config4(); 2327 change = (cur ^ v) & kvm_vz_config4_user_wrmask(vcpu); 2328 if (change) { 2329 v = cur ^ change; 2330 write_gc0_config4(v); 2331 } 2332 break; 2333 case KVM_REG_MIPS_CP0_CONFIG5: 2334 if (!cpu_guest_has_conf5) 2335 break; 2336 cur = read_gc0_config5(); 2337 change = (cur ^ v) & kvm_vz_config5_user_wrmask(vcpu); 2338 if (change) { 2339 v = cur ^ change; 2340 write_gc0_config5(v); 2341 } 2342 break; 2343 case KVM_REG_MIPS_CP0_CONFIG6: 2344 cur = kvm_read_sw_gc0_config6(cop0); 2345 change = (cur ^ v) & kvm_vz_config6_user_wrmask(vcpu); 2346 if (change) { 2347 v = cur ^ change; 2348 kvm_write_sw_gc0_config6(cop0, (int)v); 2349 } 2350 break; 2351 case KVM_REG_MIPS_CP0_MAAR(0) ... KVM_REG_MIPS_CP0_MAAR(0x3f): 2352 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar) 2353 return -EINVAL; 2354 idx = reg->id - KVM_REG_MIPS_CP0_MAAR(0); 2355 if (idx >= ARRAY_SIZE(vcpu->arch.maar)) 2356 return -EINVAL; 2357 vcpu->arch.maar[idx] = mips_process_maar(dmtc_op, v); 2358 break; 2359 case KVM_REG_MIPS_CP0_MAARI: 2360 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar) 2361 return -EINVAL; 2362 kvm_write_maari(vcpu, v); 2363 break; 2364 #ifdef CONFIG_64BIT 2365 case KVM_REG_MIPS_CP0_XCONTEXT: 2366 write_gc0_xcontext(v); 2367 break; 2368 #endif 2369 case KVM_REG_MIPS_CP0_ERROREPC: 2370 write_gc0_errorepc(v); 2371 break; 2372 case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6: 2373 idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2; 2374 if (!cpu_guest_has_kscr(idx)) 2375 return -EINVAL; 2376 switch (idx) { 2377 case 2: 2378 write_gc0_kscratch1(v); 2379 break; 2380 case 3: 2381 write_gc0_kscratch2(v); 2382 break; 2383 case 4: 2384 write_gc0_kscratch3(v); 2385 break; 2386 case 5: 2387 write_gc0_kscratch4(v); 2388 break; 2389 case 6: 2390 write_gc0_kscratch5(v); 2391 break; 2392 case 7: 2393 write_gc0_kscratch6(v); 2394 break; 2395 } 2396 break; 2397 case KVM_REG_MIPS_COUNT_CTL: 2398 ret = kvm_mips_set_count_ctl(vcpu, v); 2399 break; 2400 case KVM_REG_MIPS_COUNT_RESUME: 2401 ret = kvm_mips_set_count_resume(vcpu, v); 2402 break; 2403 case KVM_REG_MIPS_COUNT_HZ: 2404 ret = kvm_mips_set_count_hz(vcpu, v); 2405 break; 2406 default: 2407 return -EINVAL; 2408 } 2409 return ret; 2410 } 2411 2412 #define guestid_cache(cpu) (cpu_data[cpu].guestid_cache) 2413 static void kvm_vz_get_new_guestid(unsigned long cpu, struct kvm_vcpu *vcpu) 2414 { 2415 unsigned long guestid = guestid_cache(cpu); 2416 2417 if (!(++guestid & GUESTID_MASK)) { 2418 if (cpu_has_vtag_icache) 2419 flush_icache_all(); 2420 2421 if (!guestid) /* fix version if needed */ 2422 guestid = GUESTID_FIRST_VERSION; 2423 2424 ++guestid; /* guestid 0 reserved for root */ 2425 2426 /* start new guestid cycle */ 2427 kvm_vz_local_flush_roottlb_all_guests(); 2428 kvm_vz_local_flush_guesttlb_all(); 2429 } 2430 2431 guestid_cache(cpu) = guestid; 2432 } 2433 2434 /* Returns 1 if the guest TLB may be clobbered */ 2435 static int kvm_vz_check_requests(struct kvm_vcpu *vcpu, int cpu) 2436 { 2437 int ret = 0; 2438 int i; 2439 2440 if (!kvm_request_pending(vcpu)) 2441 return 0; 2442 2443 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { 2444 if (cpu_has_guestid) { 2445 /* Drop all GuestIDs for this VCPU */ 2446 for_each_possible_cpu(i) 2447 vcpu->arch.vzguestid[i] = 0; 2448 /* This will clobber guest TLB contents too */ 2449 ret = 1; 2450 } 2451 /* 2452 * For Root ASID Dealias (RAD) we don't do anything here, but we 2453 * still need the request to ensure we recheck asid_flush_mask. 2454 * We can still return 0 as only the root TLB will be affected 2455 * by a root ASID flush. 2456 */ 2457 } 2458 2459 return ret; 2460 } 2461 2462 static void kvm_vz_vcpu_save_wired(struct kvm_vcpu *vcpu) 2463 { 2464 unsigned int wired = read_gc0_wired(); 2465 struct kvm_mips_tlb *tlbs; 2466 int i; 2467 2468 /* Expand the wired TLB array if necessary */ 2469 wired &= MIPSR6_WIRED_WIRED; 2470 if (wired > vcpu->arch.wired_tlb_limit) { 2471 tlbs = krealloc(vcpu->arch.wired_tlb, wired * 2472 sizeof(*vcpu->arch.wired_tlb), GFP_ATOMIC); 2473 if (WARN_ON(!tlbs)) { 2474 /* Save whatever we can */ 2475 wired = vcpu->arch.wired_tlb_limit; 2476 } else { 2477 vcpu->arch.wired_tlb = tlbs; 2478 vcpu->arch.wired_tlb_limit = wired; 2479 } 2480 } 2481 2482 if (wired) 2483 /* Save wired entries from the guest TLB */ 2484 kvm_vz_save_guesttlb(vcpu->arch.wired_tlb, 0, wired); 2485 /* Invalidate any dropped entries since last time */ 2486 for (i = wired; i < vcpu->arch.wired_tlb_used; ++i) { 2487 vcpu->arch.wired_tlb[i].tlb_hi = UNIQUE_GUEST_ENTRYHI(i); 2488 vcpu->arch.wired_tlb[i].tlb_lo[0] = 0; 2489 vcpu->arch.wired_tlb[i].tlb_lo[1] = 0; 2490 vcpu->arch.wired_tlb[i].tlb_mask = 0; 2491 } 2492 vcpu->arch.wired_tlb_used = wired; 2493 } 2494 2495 static void kvm_vz_vcpu_load_wired(struct kvm_vcpu *vcpu) 2496 { 2497 /* Load wired entries into the guest TLB */ 2498 if (vcpu->arch.wired_tlb) 2499 kvm_vz_load_guesttlb(vcpu->arch.wired_tlb, 0, 2500 vcpu->arch.wired_tlb_used); 2501 } 2502 2503 static void kvm_vz_vcpu_load_tlb(struct kvm_vcpu *vcpu, int cpu) 2504 { 2505 struct kvm *kvm = vcpu->kvm; 2506 struct mm_struct *gpa_mm = &kvm->arch.gpa_mm; 2507 bool migrated; 2508 2509 /* 2510 * Are we entering guest context on a different CPU to last time? 2511 * If so, the VCPU's guest TLB state on this CPU may be stale. 2512 */ 2513 migrated = (vcpu->arch.last_exec_cpu != cpu); 2514 vcpu->arch.last_exec_cpu = cpu; 2515 2516 /* 2517 * A vcpu's GuestID is set in GuestCtl1.ID when the vcpu is loaded and 2518 * remains set until another vcpu is loaded in. As a rule GuestRID 2519 * remains zeroed when in root context unless the kernel is busy 2520 * manipulating guest tlb entries. 2521 */ 2522 if (cpu_has_guestid) { 2523 /* 2524 * Check if our GuestID is of an older version and thus invalid. 2525 * 2526 * We also discard the stored GuestID if we've executed on 2527 * another CPU, as the guest mappings may have changed without 2528 * hypervisor knowledge. 2529 */ 2530 if (migrated || 2531 (vcpu->arch.vzguestid[cpu] ^ guestid_cache(cpu)) & 2532 GUESTID_VERSION_MASK) { 2533 kvm_vz_get_new_guestid(cpu, vcpu); 2534 vcpu->arch.vzguestid[cpu] = guestid_cache(cpu); 2535 trace_kvm_guestid_change(vcpu, 2536 vcpu->arch.vzguestid[cpu]); 2537 } 2538 2539 /* Restore GuestID */ 2540 change_c0_guestctl1(GUESTID_MASK, vcpu->arch.vzguestid[cpu]); 2541 } else { 2542 /* 2543 * The Guest TLB only stores a single guest's TLB state, so 2544 * flush it if another VCPU has executed on this CPU. 2545 * 2546 * We also flush if we've executed on another CPU, as the guest 2547 * mappings may have changed without hypervisor knowledge. 2548 */ 2549 if (migrated || last_exec_vcpu[cpu] != vcpu) 2550 kvm_vz_local_flush_guesttlb_all(); 2551 last_exec_vcpu[cpu] = vcpu; 2552 2553 /* 2554 * Root ASID dealiases guest GPA mappings in the root TLB. 2555 * Allocate new root ASID if needed. 2556 */ 2557 if (cpumask_test_and_clear_cpu(cpu, &kvm->arch.asid_flush_mask)) 2558 get_new_mmu_context(gpa_mm); 2559 else 2560 check_mmu_context(gpa_mm); 2561 } 2562 } 2563 2564 static int kvm_vz_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 2565 { 2566 struct mips_coproc *cop0 = vcpu->arch.cop0; 2567 bool migrated, all; 2568 2569 /* 2570 * Have we migrated to a different CPU? 2571 * If so, any old guest TLB state may be stale. 2572 */ 2573 migrated = (vcpu->arch.last_sched_cpu != cpu); 2574 2575 /* 2576 * Was this the last VCPU to run on this CPU? 2577 * If not, any old guest state from this VCPU will have been clobbered. 2578 */ 2579 all = migrated || (last_vcpu[cpu] != vcpu); 2580 last_vcpu[cpu] = vcpu; 2581 2582 /* 2583 * Restore CP0_Wired unconditionally as we clear it after use, and 2584 * restore wired guest TLB entries (while in guest context). 2585 */ 2586 kvm_restore_gc0_wired(cop0); 2587 if (current->flags & PF_VCPU) { 2588 tlbw_use_hazard(); 2589 kvm_vz_vcpu_load_tlb(vcpu, cpu); 2590 kvm_vz_vcpu_load_wired(vcpu); 2591 } 2592 2593 /* 2594 * Restore timer state regardless, as e.g. Cause.TI can change over time 2595 * if left unmaintained. 2596 */ 2597 kvm_vz_restore_timer(vcpu); 2598 2599 /* Set MC bit if we want to trace guest mode changes */ 2600 if (kvm_trace_guest_mode_change) 2601 set_c0_guestctl0(MIPS_GCTL0_MC); 2602 else 2603 clear_c0_guestctl0(MIPS_GCTL0_MC); 2604 2605 /* Don't bother restoring registers multiple times unless necessary */ 2606 if (!all) 2607 return 0; 2608 2609 /* 2610 * Restore config registers first, as some implementations restrict 2611 * writes to other registers when the corresponding feature bits aren't 2612 * set. For example Status.CU1 cannot be set unless Config1.FP is set. 2613 */ 2614 kvm_restore_gc0_config(cop0); 2615 if (cpu_guest_has_conf1) 2616 kvm_restore_gc0_config1(cop0); 2617 if (cpu_guest_has_conf2) 2618 kvm_restore_gc0_config2(cop0); 2619 if (cpu_guest_has_conf3) 2620 kvm_restore_gc0_config3(cop0); 2621 if (cpu_guest_has_conf4) 2622 kvm_restore_gc0_config4(cop0); 2623 if (cpu_guest_has_conf5) 2624 kvm_restore_gc0_config5(cop0); 2625 if (cpu_guest_has_conf6) 2626 kvm_restore_gc0_config6(cop0); 2627 if (cpu_guest_has_conf7) 2628 kvm_restore_gc0_config7(cop0); 2629 2630 kvm_restore_gc0_index(cop0); 2631 kvm_restore_gc0_entrylo0(cop0); 2632 kvm_restore_gc0_entrylo1(cop0); 2633 kvm_restore_gc0_context(cop0); 2634 if (cpu_guest_has_contextconfig) 2635 kvm_restore_gc0_contextconfig(cop0); 2636 #ifdef CONFIG_64BIT 2637 kvm_restore_gc0_xcontext(cop0); 2638 if (cpu_guest_has_contextconfig) 2639 kvm_restore_gc0_xcontextconfig(cop0); 2640 #endif 2641 kvm_restore_gc0_pagemask(cop0); 2642 kvm_restore_gc0_pagegrain(cop0); 2643 kvm_restore_gc0_hwrena(cop0); 2644 kvm_restore_gc0_badvaddr(cop0); 2645 kvm_restore_gc0_entryhi(cop0); 2646 kvm_restore_gc0_status(cop0); 2647 kvm_restore_gc0_intctl(cop0); 2648 kvm_restore_gc0_epc(cop0); 2649 kvm_vz_write_gc0_ebase(kvm_read_sw_gc0_ebase(cop0)); 2650 if (cpu_guest_has_userlocal) 2651 kvm_restore_gc0_userlocal(cop0); 2652 2653 kvm_restore_gc0_errorepc(cop0); 2654 2655 /* restore KScratch registers if enabled in guest */ 2656 if (cpu_guest_has_conf4) { 2657 if (cpu_guest_has_kscr(2)) 2658 kvm_restore_gc0_kscratch1(cop0); 2659 if (cpu_guest_has_kscr(3)) 2660 kvm_restore_gc0_kscratch2(cop0); 2661 if (cpu_guest_has_kscr(4)) 2662 kvm_restore_gc0_kscratch3(cop0); 2663 if (cpu_guest_has_kscr(5)) 2664 kvm_restore_gc0_kscratch4(cop0); 2665 if (cpu_guest_has_kscr(6)) 2666 kvm_restore_gc0_kscratch5(cop0); 2667 if (cpu_guest_has_kscr(7)) 2668 kvm_restore_gc0_kscratch6(cop0); 2669 } 2670 2671 if (cpu_guest_has_badinstr) 2672 kvm_restore_gc0_badinstr(cop0); 2673 if (cpu_guest_has_badinstrp) 2674 kvm_restore_gc0_badinstrp(cop0); 2675 2676 if (cpu_guest_has_segments) { 2677 kvm_restore_gc0_segctl0(cop0); 2678 kvm_restore_gc0_segctl1(cop0); 2679 kvm_restore_gc0_segctl2(cop0); 2680 } 2681 2682 /* restore HTW registers */ 2683 if (cpu_guest_has_htw || cpu_guest_has_ldpte) { 2684 kvm_restore_gc0_pwbase(cop0); 2685 kvm_restore_gc0_pwfield(cop0); 2686 kvm_restore_gc0_pwsize(cop0); 2687 kvm_restore_gc0_pwctl(cop0); 2688 } 2689 2690 /* restore Root.GuestCtl2 from unused Guest guestctl2 register */ 2691 if (cpu_has_guestctl2) 2692 write_c0_guestctl2( 2693 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL]); 2694 2695 /* 2696 * We should clear linked load bit to break interrupted atomics. This 2697 * prevents a SC on the next VCPU from succeeding by matching a LL on 2698 * the previous VCPU. 2699 */ 2700 if (vcpu->kvm->created_vcpus > 1) 2701 write_gc0_lladdr(0); 2702 2703 return 0; 2704 } 2705 2706 static int kvm_vz_vcpu_put(struct kvm_vcpu *vcpu, int cpu) 2707 { 2708 struct mips_coproc *cop0 = vcpu->arch.cop0; 2709 2710 if (current->flags & PF_VCPU) 2711 kvm_vz_vcpu_save_wired(vcpu); 2712 2713 kvm_lose_fpu(vcpu); 2714 2715 kvm_save_gc0_index(cop0); 2716 kvm_save_gc0_entrylo0(cop0); 2717 kvm_save_gc0_entrylo1(cop0); 2718 kvm_save_gc0_context(cop0); 2719 if (cpu_guest_has_contextconfig) 2720 kvm_save_gc0_contextconfig(cop0); 2721 #ifdef CONFIG_64BIT 2722 kvm_save_gc0_xcontext(cop0); 2723 if (cpu_guest_has_contextconfig) 2724 kvm_save_gc0_xcontextconfig(cop0); 2725 #endif 2726 kvm_save_gc0_pagemask(cop0); 2727 kvm_save_gc0_pagegrain(cop0); 2728 kvm_save_gc0_wired(cop0); 2729 /* allow wired TLB entries to be overwritten */ 2730 clear_gc0_wired(MIPSR6_WIRED_WIRED); 2731 kvm_save_gc0_hwrena(cop0); 2732 kvm_save_gc0_badvaddr(cop0); 2733 kvm_save_gc0_entryhi(cop0); 2734 kvm_save_gc0_status(cop0); 2735 kvm_save_gc0_intctl(cop0); 2736 kvm_save_gc0_epc(cop0); 2737 kvm_write_sw_gc0_ebase(cop0, kvm_vz_read_gc0_ebase()); 2738 if (cpu_guest_has_userlocal) 2739 kvm_save_gc0_userlocal(cop0); 2740 2741 /* only save implemented config registers */ 2742 kvm_save_gc0_config(cop0); 2743 if (cpu_guest_has_conf1) 2744 kvm_save_gc0_config1(cop0); 2745 if (cpu_guest_has_conf2) 2746 kvm_save_gc0_config2(cop0); 2747 if (cpu_guest_has_conf3) 2748 kvm_save_gc0_config3(cop0); 2749 if (cpu_guest_has_conf4) 2750 kvm_save_gc0_config4(cop0); 2751 if (cpu_guest_has_conf5) 2752 kvm_save_gc0_config5(cop0); 2753 if (cpu_guest_has_conf6) 2754 kvm_save_gc0_config6(cop0); 2755 if (cpu_guest_has_conf7) 2756 kvm_save_gc0_config7(cop0); 2757 2758 kvm_save_gc0_errorepc(cop0); 2759 2760 /* save KScratch registers if enabled in guest */ 2761 if (cpu_guest_has_conf4) { 2762 if (cpu_guest_has_kscr(2)) 2763 kvm_save_gc0_kscratch1(cop0); 2764 if (cpu_guest_has_kscr(3)) 2765 kvm_save_gc0_kscratch2(cop0); 2766 if (cpu_guest_has_kscr(4)) 2767 kvm_save_gc0_kscratch3(cop0); 2768 if (cpu_guest_has_kscr(5)) 2769 kvm_save_gc0_kscratch4(cop0); 2770 if (cpu_guest_has_kscr(6)) 2771 kvm_save_gc0_kscratch5(cop0); 2772 if (cpu_guest_has_kscr(7)) 2773 kvm_save_gc0_kscratch6(cop0); 2774 } 2775 2776 if (cpu_guest_has_badinstr) 2777 kvm_save_gc0_badinstr(cop0); 2778 if (cpu_guest_has_badinstrp) 2779 kvm_save_gc0_badinstrp(cop0); 2780 2781 if (cpu_guest_has_segments) { 2782 kvm_save_gc0_segctl0(cop0); 2783 kvm_save_gc0_segctl1(cop0); 2784 kvm_save_gc0_segctl2(cop0); 2785 } 2786 2787 /* save HTW registers if enabled in guest */ 2788 if (cpu_guest_has_ldpte || (cpu_guest_has_htw && 2789 kvm_read_sw_gc0_config3(cop0) & MIPS_CONF3_PW)) { 2790 kvm_save_gc0_pwbase(cop0); 2791 kvm_save_gc0_pwfield(cop0); 2792 kvm_save_gc0_pwsize(cop0); 2793 kvm_save_gc0_pwctl(cop0); 2794 } 2795 2796 kvm_vz_save_timer(vcpu); 2797 2798 /* save Root.GuestCtl2 in unused Guest guestctl2 register */ 2799 if (cpu_has_guestctl2) 2800 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL] = 2801 read_c0_guestctl2(); 2802 2803 return 0; 2804 } 2805 2806 /** 2807 * kvm_vz_resize_guest_vtlb() - Attempt to resize guest VTLB. 2808 * @size: Number of guest VTLB entries (0 < @size <= root VTLB entries). 2809 * 2810 * Attempt to resize the guest VTLB by writing guest Config registers. This is 2811 * necessary for cores with a shared root/guest TLB to avoid overlap with wired 2812 * entries in the root VTLB. 2813 * 2814 * Returns: The resulting guest VTLB size. 2815 */ 2816 static unsigned int kvm_vz_resize_guest_vtlb(unsigned int size) 2817 { 2818 unsigned int config4 = 0, ret = 0, limit; 2819 2820 /* Write MMUSize - 1 into guest Config registers */ 2821 if (cpu_guest_has_conf1) 2822 change_gc0_config1(MIPS_CONF1_TLBS, 2823 (size - 1) << MIPS_CONF1_TLBS_SHIFT); 2824 if (cpu_guest_has_conf4) { 2825 config4 = read_gc0_config4(); 2826 if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) == 2827 MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT) { 2828 config4 &= ~MIPS_CONF4_VTLBSIZEEXT; 2829 config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) << 2830 MIPS_CONF4_VTLBSIZEEXT_SHIFT; 2831 } else if ((config4 & MIPS_CONF4_MMUEXTDEF) == 2832 MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT) { 2833 config4 &= ~MIPS_CONF4_MMUSIZEEXT; 2834 config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) << 2835 MIPS_CONF4_MMUSIZEEXT_SHIFT; 2836 } 2837 write_gc0_config4(config4); 2838 } 2839 2840 /* 2841 * Set Guest.Wired.Limit = 0 (no limit up to Guest.MMUSize-1), unless it 2842 * would exceed Root.Wired.Limit (clearing Guest.Wired.Wired so write 2843 * not dropped) 2844 */ 2845 if (cpu_has_mips_r6) { 2846 limit = (read_c0_wired() & MIPSR6_WIRED_LIMIT) >> 2847 MIPSR6_WIRED_LIMIT_SHIFT; 2848 if (size - 1 <= limit) 2849 limit = 0; 2850 write_gc0_wired(limit << MIPSR6_WIRED_LIMIT_SHIFT); 2851 } 2852 2853 /* Read back MMUSize - 1 */ 2854 back_to_back_c0_hazard(); 2855 if (cpu_guest_has_conf1) 2856 ret = (read_gc0_config1() & MIPS_CONF1_TLBS) >> 2857 MIPS_CONF1_TLBS_SHIFT; 2858 if (config4) { 2859 if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) == 2860 MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT) 2861 ret |= ((config4 & MIPS_CONF4_VTLBSIZEEXT) >> 2862 MIPS_CONF4_VTLBSIZEEXT_SHIFT) << 2863 MIPS_CONF1_TLBS_SIZE; 2864 else if ((config4 & MIPS_CONF4_MMUEXTDEF) == 2865 MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT) 2866 ret |= ((config4 & MIPS_CONF4_MMUSIZEEXT) >> 2867 MIPS_CONF4_MMUSIZEEXT_SHIFT) << 2868 MIPS_CONF1_TLBS_SIZE; 2869 } 2870 return ret + 1; 2871 } 2872 2873 static int kvm_vz_hardware_enable(void) 2874 { 2875 unsigned int mmu_size, guest_mmu_size, ftlb_size; 2876 u64 guest_cvmctl, cvmvmconfig; 2877 2878 switch (current_cpu_type()) { 2879 case CPU_CAVIUM_OCTEON3: 2880 /* Set up guest timer/perfcount IRQ lines */ 2881 guest_cvmctl = read_gc0_cvmctl(); 2882 guest_cvmctl &= ~CVMCTL_IPTI; 2883 guest_cvmctl |= 7ull << CVMCTL_IPTI_SHIFT; 2884 guest_cvmctl &= ~CVMCTL_IPPCI; 2885 guest_cvmctl |= 6ull << CVMCTL_IPPCI_SHIFT; 2886 write_gc0_cvmctl(guest_cvmctl); 2887 2888 cvmvmconfig = read_c0_cvmvmconfig(); 2889 /* No I/O hole translation. */ 2890 cvmvmconfig |= CVMVMCONF_DGHT; 2891 /* Halve the root MMU size */ 2892 mmu_size = ((cvmvmconfig & CVMVMCONF_MMUSIZEM1) 2893 >> CVMVMCONF_MMUSIZEM1_S) + 1; 2894 guest_mmu_size = mmu_size / 2; 2895 mmu_size -= guest_mmu_size; 2896 cvmvmconfig &= ~CVMVMCONF_RMMUSIZEM1; 2897 cvmvmconfig |= mmu_size - 1; 2898 write_c0_cvmvmconfig(cvmvmconfig); 2899 2900 /* Update our records */ 2901 current_cpu_data.tlbsize = mmu_size; 2902 current_cpu_data.tlbsizevtlb = mmu_size; 2903 current_cpu_data.guest.tlbsize = guest_mmu_size; 2904 2905 /* Flush moved entries in new (guest) context */ 2906 kvm_vz_local_flush_guesttlb_all(); 2907 break; 2908 default: 2909 /* 2910 * ImgTec cores tend to use a shared root/guest TLB. To avoid 2911 * overlap of root wired and guest entries, the guest TLB may 2912 * need resizing. 2913 */ 2914 mmu_size = current_cpu_data.tlbsizevtlb; 2915 ftlb_size = current_cpu_data.tlbsize - mmu_size; 2916 2917 /* Try switching to maximum guest VTLB size for flush */ 2918 guest_mmu_size = kvm_vz_resize_guest_vtlb(mmu_size); 2919 current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size; 2920 kvm_vz_local_flush_guesttlb_all(); 2921 2922 /* 2923 * Reduce to make space for root wired entries and at least 2 2924 * root non-wired entries. This does assume that long-term wired 2925 * entries won't be added later. 2926 */ 2927 guest_mmu_size = mmu_size - num_wired_entries() - 2; 2928 guest_mmu_size = kvm_vz_resize_guest_vtlb(guest_mmu_size); 2929 current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size; 2930 2931 /* 2932 * Write the VTLB size, but if another CPU has already written, 2933 * check it matches or we won't provide a consistent view to the 2934 * guest. If this ever happens it suggests an asymmetric number 2935 * of wired entries. 2936 */ 2937 if (cmpxchg(&kvm_vz_guest_vtlb_size, 0, guest_mmu_size) && 2938 WARN(guest_mmu_size != kvm_vz_guest_vtlb_size, 2939 "Available guest VTLB size mismatch")) 2940 return -EINVAL; 2941 break; 2942 } 2943 2944 /* 2945 * Enable virtualization features granting guest direct control of 2946 * certain features: 2947 * CP0=1: Guest coprocessor 0 context. 2948 * AT=Guest: Guest MMU. 2949 * CG=1: Hit (virtual address) CACHE operations (optional). 2950 * CF=1: Guest Config registers. 2951 * CGI=1: Indexed flush CACHE operations (optional). 2952 */ 2953 write_c0_guestctl0(MIPS_GCTL0_CP0 | 2954 (MIPS_GCTL0_AT_GUEST << MIPS_GCTL0_AT_SHIFT) | 2955 MIPS_GCTL0_CG | MIPS_GCTL0_CF); 2956 if (cpu_has_guestctl0ext) { 2957 if (current_cpu_type() != CPU_LOONGSON64) 2958 set_c0_guestctl0ext(MIPS_GCTL0EXT_CGI); 2959 else 2960 clear_c0_guestctl0ext(MIPS_GCTL0EXT_CGI); 2961 } 2962 2963 if (cpu_has_guestid) { 2964 write_c0_guestctl1(0); 2965 kvm_vz_local_flush_roottlb_all_guests(); 2966 2967 GUESTID_MASK = current_cpu_data.guestid_mask; 2968 GUESTID_FIRST_VERSION = GUESTID_MASK + 1; 2969 GUESTID_VERSION_MASK = ~GUESTID_MASK; 2970 2971 current_cpu_data.guestid_cache = GUESTID_FIRST_VERSION; 2972 } 2973 2974 /* clear any pending injected virtual guest interrupts */ 2975 if (cpu_has_guestctl2) 2976 clear_c0_guestctl2(0x3f << 10); 2977 2978 #ifdef CONFIG_CPU_LOONGSON64 2979 /* Control guest CCA attribute */ 2980 if (cpu_has_csr()) 2981 csr_writel(csr_readl(0xffffffec) | 0x1, 0xffffffec); 2982 #endif 2983 2984 return 0; 2985 } 2986 2987 static void kvm_vz_hardware_disable(void) 2988 { 2989 u64 cvmvmconfig; 2990 unsigned int mmu_size; 2991 2992 /* Flush any remaining guest TLB entries */ 2993 kvm_vz_local_flush_guesttlb_all(); 2994 2995 switch (current_cpu_type()) { 2996 case CPU_CAVIUM_OCTEON3: 2997 /* 2998 * Allocate whole TLB for root. Existing guest TLB entries will 2999 * change ownership to the root TLB. We should be safe though as 3000 * they've already been flushed above while in guest TLB. 3001 */ 3002 cvmvmconfig = read_c0_cvmvmconfig(); 3003 mmu_size = ((cvmvmconfig & CVMVMCONF_MMUSIZEM1) 3004 >> CVMVMCONF_MMUSIZEM1_S) + 1; 3005 cvmvmconfig &= ~CVMVMCONF_RMMUSIZEM1; 3006 cvmvmconfig |= mmu_size - 1; 3007 write_c0_cvmvmconfig(cvmvmconfig); 3008 3009 /* Update our records */ 3010 current_cpu_data.tlbsize = mmu_size; 3011 current_cpu_data.tlbsizevtlb = mmu_size; 3012 current_cpu_data.guest.tlbsize = 0; 3013 3014 /* Flush moved entries in new (root) context */ 3015 local_flush_tlb_all(); 3016 break; 3017 } 3018 3019 if (cpu_has_guestid) { 3020 write_c0_guestctl1(0); 3021 kvm_vz_local_flush_roottlb_all_guests(); 3022 } 3023 } 3024 3025 static int kvm_vz_check_extension(struct kvm *kvm, long ext) 3026 { 3027 int r; 3028 3029 switch (ext) { 3030 case KVM_CAP_MIPS_VZ: 3031 /* we wouldn't be here unless cpu_has_vz */ 3032 r = 1; 3033 break; 3034 #ifdef CONFIG_64BIT 3035 case KVM_CAP_MIPS_64BIT: 3036 /* We support 64-bit registers/operations and addresses */ 3037 r = 2; 3038 break; 3039 #endif 3040 case KVM_CAP_IOEVENTFD: 3041 r = 1; 3042 break; 3043 default: 3044 r = 0; 3045 break; 3046 } 3047 3048 return r; 3049 } 3050 3051 static int kvm_vz_vcpu_init(struct kvm_vcpu *vcpu) 3052 { 3053 int i; 3054 3055 for_each_possible_cpu(i) 3056 vcpu->arch.vzguestid[i] = 0; 3057 3058 return 0; 3059 } 3060 3061 static void kvm_vz_vcpu_uninit(struct kvm_vcpu *vcpu) 3062 { 3063 int cpu; 3064 3065 /* 3066 * If the VCPU is freed and reused as another VCPU, we don't want the 3067 * matching pointer wrongly hanging around in last_vcpu[] or 3068 * last_exec_vcpu[]. 3069 */ 3070 for_each_possible_cpu(cpu) { 3071 if (last_vcpu[cpu] == vcpu) 3072 last_vcpu[cpu] = NULL; 3073 if (last_exec_vcpu[cpu] == vcpu) 3074 last_exec_vcpu[cpu] = NULL; 3075 } 3076 } 3077 3078 static int kvm_vz_vcpu_setup(struct kvm_vcpu *vcpu) 3079 { 3080 struct mips_coproc *cop0 = vcpu->arch.cop0; 3081 unsigned long count_hz = 100*1000*1000; /* default to 100 MHz */ 3082 3083 /* 3084 * Start off the timer at the same frequency as the host timer, but the 3085 * soft timer doesn't handle frequencies greater than 1GHz yet. 3086 */ 3087 if (mips_hpt_frequency && mips_hpt_frequency <= NSEC_PER_SEC) 3088 count_hz = mips_hpt_frequency; 3089 kvm_mips_init_count(vcpu, count_hz); 3090 3091 /* 3092 * Initialize guest register state to valid architectural reset state. 3093 */ 3094 3095 /* PageGrain */ 3096 if (cpu_has_mips_r5 || cpu_has_mips_r6) 3097 kvm_write_sw_gc0_pagegrain(cop0, PG_RIE | PG_XIE | PG_IEC); 3098 /* Wired */ 3099 if (cpu_has_mips_r6) 3100 kvm_write_sw_gc0_wired(cop0, 3101 read_gc0_wired() & MIPSR6_WIRED_LIMIT); 3102 /* Status */ 3103 kvm_write_sw_gc0_status(cop0, ST0_BEV | ST0_ERL); 3104 if (cpu_has_mips_r5 || cpu_has_mips_r6) 3105 kvm_change_sw_gc0_status(cop0, ST0_FR, read_gc0_status()); 3106 /* IntCtl */ 3107 kvm_write_sw_gc0_intctl(cop0, read_gc0_intctl() & 3108 (INTCTLF_IPFDC | INTCTLF_IPPCI | INTCTLF_IPTI)); 3109 /* PRId */ 3110 kvm_write_sw_gc0_prid(cop0, boot_cpu_data.processor_id); 3111 /* EBase */ 3112 kvm_write_sw_gc0_ebase(cop0, (s32)0x80000000 | vcpu->vcpu_id); 3113 /* Config */ 3114 kvm_save_gc0_config(cop0); 3115 /* architecturally writable (e.g. from guest) */ 3116 kvm_change_sw_gc0_config(cop0, CONF_CM_CMASK, 3117 _page_cachable_default >> _CACHE_SHIFT); 3118 /* architecturally read only, but maybe writable from root */ 3119 kvm_change_sw_gc0_config(cop0, MIPS_CONF_MT, read_c0_config()); 3120 if (cpu_guest_has_conf1) { 3121 kvm_set_sw_gc0_config(cop0, MIPS_CONF_M); 3122 /* Config1 */ 3123 kvm_save_gc0_config1(cop0); 3124 /* architecturally read only, but maybe writable from root */ 3125 kvm_clear_sw_gc0_config1(cop0, MIPS_CONF1_C2 | 3126 MIPS_CONF1_MD | 3127 MIPS_CONF1_PC | 3128 MIPS_CONF1_WR | 3129 MIPS_CONF1_CA | 3130 MIPS_CONF1_FP); 3131 } 3132 if (cpu_guest_has_conf2) { 3133 kvm_set_sw_gc0_config1(cop0, MIPS_CONF_M); 3134 /* Config2 */ 3135 kvm_save_gc0_config2(cop0); 3136 } 3137 if (cpu_guest_has_conf3) { 3138 kvm_set_sw_gc0_config2(cop0, MIPS_CONF_M); 3139 /* Config3 */ 3140 kvm_save_gc0_config3(cop0); 3141 /* architecturally writable (e.g. from guest) */ 3142 kvm_clear_sw_gc0_config3(cop0, MIPS_CONF3_ISA_OE); 3143 /* architecturally read only, but maybe writable from root */ 3144 kvm_clear_sw_gc0_config3(cop0, MIPS_CONF3_MSA | 3145 MIPS_CONF3_BPG | 3146 MIPS_CONF3_ULRI | 3147 MIPS_CONF3_DSP | 3148 MIPS_CONF3_CTXTC | 3149 MIPS_CONF3_ITL | 3150 MIPS_CONF3_LPA | 3151 MIPS_CONF3_VEIC | 3152 MIPS_CONF3_VINT | 3153 MIPS_CONF3_SP | 3154 MIPS_CONF3_CDMM | 3155 MIPS_CONF3_MT | 3156 MIPS_CONF3_SM | 3157 MIPS_CONF3_TL); 3158 } 3159 if (cpu_guest_has_conf4) { 3160 kvm_set_sw_gc0_config3(cop0, MIPS_CONF_M); 3161 /* Config4 */ 3162 kvm_save_gc0_config4(cop0); 3163 } 3164 if (cpu_guest_has_conf5) { 3165 kvm_set_sw_gc0_config4(cop0, MIPS_CONF_M); 3166 /* Config5 */ 3167 kvm_save_gc0_config5(cop0); 3168 /* architecturally writable (e.g. from guest) */ 3169 kvm_clear_sw_gc0_config5(cop0, MIPS_CONF5_K | 3170 MIPS_CONF5_CV | 3171 MIPS_CONF5_MSAEN | 3172 MIPS_CONF5_UFE | 3173 MIPS_CONF5_FRE | 3174 MIPS_CONF5_SBRI | 3175 MIPS_CONF5_UFR); 3176 /* architecturally read only, but maybe writable from root */ 3177 kvm_clear_sw_gc0_config5(cop0, MIPS_CONF5_MRP); 3178 } 3179 3180 if (cpu_guest_has_contextconfig) { 3181 /* ContextConfig */ 3182 kvm_write_sw_gc0_contextconfig(cop0, 0x007ffff0); 3183 #ifdef CONFIG_64BIT 3184 /* XContextConfig */ 3185 /* bits SEGBITS-13+3:4 set */ 3186 kvm_write_sw_gc0_xcontextconfig(cop0, 3187 ((1ull << (cpu_vmbits - 13)) - 1) << 4); 3188 #endif 3189 } 3190 3191 /* Implementation dependent, use the legacy layout */ 3192 if (cpu_guest_has_segments) { 3193 /* SegCtl0, SegCtl1, SegCtl2 */ 3194 kvm_write_sw_gc0_segctl0(cop0, 0x00200010); 3195 kvm_write_sw_gc0_segctl1(cop0, 0x00000002 | 3196 (_page_cachable_default >> _CACHE_SHIFT) << 3197 (16 + MIPS_SEGCFG_C_SHIFT)); 3198 kvm_write_sw_gc0_segctl2(cop0, 0x00380438); 3199 } 3200 3201 /* reset HTW registers */ 3202 if (cpu_guest_has_htw && (cpu_has_mips_r5 || cpu_has_mips_r6)) { 3203 /* PWField */ 3204 kvm_write_sw_gc0_pwfield(cop0, 0x0c30c302); 3205 /* PWSize */ 3206 kvm_write_sw_gc0_pwsize(cop0, 1 << MIPS_PWSIZE_PTW_SHIFT); 3207 } 3208 3209 /* start with no pending virtual guest interrupts */ 3210 if (cpu_has_guestctl2) 3211 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL] = 0; 3212 3213 /* Put PC at reset vector */ 3214 vcpu->arch.pc = CKSEG1ADDR(0x1fc00000); 3215 3216 return 0; 3217 } 3218 3219 static void kvm_vz_flush_shadow_all(struct kvm *kvm) 3220 { 3221 if (cpu_has_guestid) { 3222 /* Flush GuestID for each VCPU individually */ 3223 kvm_flush_remote_tlbs(kvm); 3224 } else { 3225 /* 3226 * For each CPU there is a single GPA ASID used by all VCPUs in 3227 * the VM, so it doesn't make sense for the VCPUs to handle 3228 * invalidation of these ASIDs individually. 3229 * 3230 * Instead mark all CPUs as needing ASID invalidation in 3231 * asid_flush_mask, and just use kvm_flush_remote_tlbs(kvm) to 3232 * kick any running VCPUs so they check asid_flush_mask. 3233 */ 3234 cpumask_setall(&kvm->arch.asid_flush_mask); 3235 kvm_flush_remote_tlbs(kvm); 3236 } 3237 } 3238 3239 static void kvm_vz_flush_shadow_memslot(struct kvm *kvm, 3240 const struct kvm_memory_slot *slot) 3241 { 3242 kvm_vz_flush_shadow_all(kvm); 3243 } 3244 3245 static void kvm_vz_vcpu_reenter(struct kvm_run *run, struct kvm_vcpu *vcpu) 3246 { 3247 int cpu = smp_processor_id(); 3248 int preserve_guest_tlb; 3249 3250 preserve_guest_tlb = kvm_vz_check_requests(vcpu, cpu); 3251 3252 if (preserve_guest_tlb) 3253 kvm_vz_vcpu_save_wired(vcpu); 3254 3255 kvm_vz_vcpu_load_tlb(vcpu, cpu); 3256 3257 if (preserve_guest_tlb) 3258 kvm_vz_vcpu_load_wired(vcpu); 3259 } 3260 3261 static int kvm_vz_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu) 3262 { 3263 int cpu = smp_processor_id(); 3264 int r; 3265 3266 kvm_vz_acquire_htimer(vcpu); 3267 /* Check if we have any exceptions/interrupts pending */ 3268 kvm_mips_deliver_interrupts(vcpu, read_gc0_cause()); 3269 3270 kvm_vz_check_requests(vcpu, cpu); 3271 kvm_vz_vcpu_load_tlb(vcpu, cpu); 3272 kvm_vz_vcpu_load_wired(vcpu); 3273 3274 r = vcpu->arch.vcpu_run(run, vcpu); 3275 3276 kvm_vz_vcpu_save_wired(vcpu); 3277 3278 return r; 3279 } 3280 3281 static struct kvm_mips_callbacks kvm_vz_callbacks = { 3282 .handle_cop_unusable = kvm_trap_vz_handle_cop_unusable, 3283 .handle_tlb_mod = kvm_trap_vz_handle_tlb_st_miss, 3284 .handle_tlb_ld_miss = kvm_trap_vz_handle_tlb_ld_miss, 3285 .handle_tlb_st_miss = kvm_trap_vz_handle_tlb_st_miss, 3286 .handle_addr_err_st = kvm_trap_vz_no_handler, 3287 .handle_addr_err_ld = kvm_trap_vz_no_handler, 3288 .handle_syscall = kvm_trap_vz_no_handler, 3289 .handle_res_inst = kvm_trap_vz_no_handler, 3290 .handle_break = kvm_trap_vz_no_handler, 3291 .handle_msa_disabled = kvm_trap_vz_handle_msa_disabled, 3292 .handle_guest_exit = kvm_trap_vz_handle_guest_exit, 3293 3294 .hardware_enable = kvm_vz_hardware_enable, 3295 .hardware_disable = kvm_vz_hardware_disable, 3296 .check_extension = kvm_vz_check_extension, 3297 .vcpu_init = kvm_vz_vcpu_init, 3298 .vcpu_uninit = kvm_vz_vcpu_uninit, 3299 .vcpu_setup = kvm_vz_vcpu_setup, 3300 .flush_shadow_all = kvm_vz_flush_shadow_all, 3301 .flush_shadow_memslot = kvm_vz_flush_shadow_memslot, 3302 .gva_to_gpa = kvm_vz_gva_to_gpa_cb, 3303 .queue_timer_int = kvm_vz_queue_timer_int_cb, 3304 .dequeue_timer_int = kvm_vz_dequeue_timer_int_cb, 3305 .queue_io_int = kvm_vz_queue_io_int_cb, 3306 .dequeue_io_int = kvm_vz_dequeue_io_int_cb, 3307 .irq_deliver = kvm_vz_irq_deliver_cb, 3308 .irq_clear = kvm_vz_irq_clear_cb, 3309 .num_regs = kvm_vz_num_regs, 3310 .copy_reg_indices = kvm_vz_copy_reg_indices, 3311 .get_one_reg = kvm_vz_get_one_reg, 3312 .set_one_reg = kvm_vz_set_one_reg, 3313 .vcpu_load = kvm_vz_vcpu_load, 3314 .vcpu_put = kvm_vz_vcpu_put, 3315 .vcpu_run = kvm_vz_vcpu_run, 3316 .vcpu_reenter = kvm_vz_vcpu_reenter, 3317 }; 3318 3319 int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks) 3320 { 3321 if (!cpu_has_vz) 3322 return -ENODEV; 3323 3324 /* 3325 * VZ requires at least 2 KScratch registers, so it should have been 3326 * possible to allocate pgd_reg. 3327 */ 3328 if (WARN(pgd_reg == -1, 3329 "pgd_reg not allocated even though cpu_has_vz\n")) 3330 return -ENODEV; 3331 3332 pr_info("Starting KVM with MIPS VZ extensions\n"); 3333 3334 *install_callbacks = &kvm_vz_callbacks; 3335 return 0; 3336 } 3337