1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * KVM/MIPS: Support for hardware virtualization extensions 7 * 8 * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. 9 * Authors: Yann Le Du <ledu@kymasys.com> 10 */ 11 12 #include <linux/errno.h> 13 #include <linux/err.h> 14 #include <linux/module.h> 15 #include <linux/preempt.h> 16 #include <linux/vmalloc.h> 17 #include <asm/cacheflush.h> 18 #include <asm/cacheops.h> 19 #include <asm/cmpxchg.h> 20 #include <asm/fpu.h> 21 #include <asm/hazards.h> 22 #include <asm/inst.h> 23 #include <asm/mmu_context.h> 24 #include <asm/r4kcache.h> 25 #include <asm/time.h> 26 #include <asm/tlb.h> 27 #include <asm/tlbex.h> 28 29 #include <linux/kvm_host.h> 30 31 #include "interrupt.h" 32 #ifdef CONFIG_CPU_LOONGSON64 33 #include "loongson_regs.h" 34 #endif 35 36 #include "trace.h" 37 38 /* Pointers to last VCPU loaded on each physical CPU */ 39 static struct kvm_vcpu *last_vcpu[NR_CPUS]; 40 /* Pointers to last VCPU executed on each physical CPU */ 41 static struct kvm_vcpu *last_exec_vcpu[NR_CPUS]; 42 43 /* 44 * Number of guest VTLB entries to use, so we can catch inconsistency between 45 * CPUs. 46 */ 47 static unsigned int kvm_vz_guest_vtlb_size; 48 49 static inline long kvm_vz_read_gc0_ebase(void) 50 { 51 if (sizeof(long) == 8 && cpu_has_ebase_wg) 52 return read_gc0_ebase_64(); 53 else 54 return read_gc0_ebase(); 55 } 56 57 static inline void kvm_vz_write_gc0_ebase(long v) 58 { 59 /* 60 * First write with WG=1 to write upper bits, then write again in case 61 * WG should be left at 0. 62 * write_gc0_ebase_64() is no longer UNDEFINED since R6. 63 */ 64 if (sizeof(long) == 8 && 65 (cpu_has_mips64r6 || cpu_has_ebase_wg)) { 66 write_gc0_ebase_64(v | MIPS_EBASE_WG); 67 write_gc0_ebase_64(v); 68 } else { 69 write_gc0_ebase(v | MIPS_EBASE_WG); 70 write_gc0_ebase(v); 71 } 72 } 73 74 /* 75 * These Config bits may be writable by the guest: 76 * Config: [K23, KU] (!TLB), K0 77 * Config1: (none) 78 * Config2: [TU, SU] (impl) 79 * Config3: ISAOnExc 80 * Config4: FTLBPageSize 81 * Config5: K, CV, MSAEn, UFE, FRE, SBRI, UFR 82 */ 83 84 static inline unsigned int kvm_vz_config_guest_wrmask(struct kvm_vcpu *vcpu) 85 { 86 return CONF_CM_CMASK; 87 } 88 89 static inline unsigned int kvm_vz_config1_guest_wrmask(struct kvm_vcpu *vcpu) 90 { 91 return 0; 92 } 93 94 static inline unsigned int kvm_vz_config2_guest_wrmask(struct kvm_vcpu *vcpu) 95 { 96 return 0; 97 } 98 99 static inline unsigned int kvm_vz_config3_guest_wrmask(struct kvm_vcpu *vcpu) 100 { 101 return MIPS_CONF3_ISA_OE; 102 } 103 104 static inline unsigned int kvm_vz_config4_guest_wrmask(struct kvm_vcpu *vcpu) 105 { 106 /* no need to be exact */ 107 return MIPS_CONF4_VFTLBPAGESIZE; 108 } 109 110 static inline unsigned int kvm_vz_config5_guest_wrmask(struct kvm_vcpu *vcpu) 111 { 112 unsigned int mask = MIPS_CONF5_K | MIPS_CONF5_CV | MIPS_CONF5_SBRI; 113 114 /* Permit MSAEn changes if MSA supported and enabled */ 115 if (kvm_mips_guest_has_msa(&vcpu->arch)) 116 mask |= MIPS_CONF5_MSAEN; 117 118 /* 119 * Permit guest FPU mode changes if FPU is enabled and the relevant 120 * feature exists according to FIR register. 121 */ 122 if (kvm_mips_guest_has_fpu(&vcpu->arch)) { 123 if (cpu_has_ufr) 124 mask |= MIPS_CONF5_UFR; 125 if (cpu_has_fre) 126 mask |= MIPS_CONF5_FRE | MIPS_CONF5_UFE; 127 } 128 129 return mask; 130 } 131 132 static inline unsigned int kvm_vz_config6_guest_wrmask(struct kvm_vcpu *vcpu) 133 { 134 return LOONGSON_CONF6_INTIMER | LOONGSON_CONF6_EXTIMER; 135 } 136 137 /* 138 * VZ optionally allows these additional Config bits to be written by root: 139 * Config: M, [MT] 140 * Config1: M, [MMUSize-1, C2, MD, PC, WR, CA], FP 141 * Config2: M 142 * Config3: M, MSAP, [BPG], ULRI, [DSP2P, DSPP], CTXTC, [ITL, LPA, VEIC, 143 * VInt, SP, CDMM, MT, SM, TL] 144 * Config4: M, [VTLBSizeExt, MMUSizeExt] 145 * Config5: MRP 146 */ 147 148 static inline unsigned int kvm_vz_config_user_wrmask(struct kvm_vcpu *vcpu) 149 { 150 return kvm_vz_config_guest_wrmask(vcpu) | MIPS_CONF_M; 151 } 152 153 static inline unsigned int kvm_vz_config1_user_wrmask(struct kvm_vcpu *vcpu) 154 { 155 unsigned int mask = kvm_vz_config1_guest_wrmask(vcpu) | MIPS_CONF_M; 156 157 /* Permit FPU to be present if FPU is supported */ 158 if (kvm_mips_guest_can_have_fpu(&vcpu->arch)) 159 mask |= MIPS_CONF1_FP; 160 161 return mask; 162 } 163 164 static inline unsigned int kvm_vz_config2_user_wrmask(struct kvm_vcpu *vcpu) 165 { 166 return kvm_vz_config2_guest_wrmask(vcpu) | MIPS_CONF_M; 167 } 168 169 static inline unsigned int kvm_vz_config3_user_wrmask(struct kvm_vcpu *vcpu) 170 { 171 unsigned int mask = kvm_vz_config3_guest_wrmask(vcpu) | MIPS_CONF_M | 172 MIPS_CONF3_ULRI | MIPS_CONF3_CTXTC; 173 174 /* Permit MSA to be present if MSA is supported */ 175 if (kvm_mips_guest_can_have_msa(&vcpu->arch)) 176 mask |= MIPS_CONF3_MSA; 177 178 return mask; 179 } 180 181 static inline unsigned int kvm_vz_config4_user_wrmask(struct kvm_vcpu *vcpu) 182 { 183 return kvm_vz_config4_guest_wrmask(vcpu) | MIPS_CONF_M; 184 } 185 186 static inline unsigned int kvm_vz_config5_user_wrmask(struct kvm_vcpu *vcpu) 187 { 188 return kvm_vz_config5_guest_wrmask(vcpu) | MIPS_CONF5_MRP; 189 } 190 191 static inline unsigned int kvm_vz_config6_user_wrmask(struct kvm_vcpu *vcpu) 192 { 193 return kvm_vz_config6_guest_wrmask(vcpu) | 194 LOONGSON_CONF6_SFBEN | LOONGSON_CONF6_FTLBDIS; 195 } 196 197 static gpa_t kvm_vz_gva_to_gpa_cb(gva_t gva) 198 { 199 /* VZ guest has already converted gva to gpa */ 200 return gva; 201 } 202 203 static void kvm_vz_queue_irq(struct kvm_vcpu *vcpu, unsigned int priority) 204 { 205 set_bit(priority, &vcpu->arch.pending_exceptions); 206 clear_bit(priority, &vcpu->arch.pending_exceptions_clr); 207 } 208 209 static void kvm_vz_dequeue_irq(struct kvm_vcpu *vcpu, unsigned int priority) 210 { 211 clear_bit(priority, &vcpu->arch.pending_exceptions); 212 set_bit(priority, &vcpu->arch.pending_exceptions_clr); 213 } 214 215 static void kvm_vz_queue_timer_int_cb(struct kvm_vcpu *vcpu) 216 { 217 /* 218 * timer expiry is asynchronous to vcpu execution therefore defer guest 219 * cp0 accesses 220 */ 221 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER); 222 } 223 224 static void kvm_vz_dequeue_timer_int_cb(struct kvm_vcpu *vcpu) 225 { 226 /* 227 * timer expiry is asynchronous to vcpu execution therefore defer guest 228 * cp0 accesses 229 */ 230 kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_TIMER); 231 } 232 233 static void kvm_vz_queue_io_int_cb(struct kvm_vcpu *vcpu, 234 struct kvm_mips_interrupt *irq) 235 { 236 int intr = (int)irq->irq; 237 238 /* 239 * interrupts are asynchronous to vcpu execution therefore defer guest 240 * cp0 accesses 241 */ 242 kvm_vz_queue_irq(vcpu, kvm_irq_to_priority(intr)); 243 } 244 245 static void kvm_vz_dequeue_io_int_cb(struct kvm_vcpu *vcpu, 246 struct kvm_mips_interrupt *irq) 247 { 248 int intr = (int)irq->irq; 249 250 /* 251 * interrupts are asynchronous to vcpu execution therefore defer guest 252 * cp0 accesses 253 */ 254 kvm_vz_dequeue_irq(vcpu, kvm_irq_to_priority(-intr)); 255 } 256 257 static int kvm_vz_irq_deliver_cb(struct kvm_vcpu *vcpu, unsigned int priority, 258 u32 cause) 259 { 260 u32 irq = (priority < MIPS_EXC_MAX) ? 261 kvm_priority_to_irq[priority] : 0; 262 263 switch (priority) { 264 case MIPS_EXC_INT_TIMER: 265 set_gc0_cause(C_TI); 266 break; 267 268 case MIPS_EXC_INT_IO_1: 269 case MIPS_EXC_INT_IO_2: 270 case MIPS_EXC_INT_IPI_1: 271 case MIPS_EXC_INT_IPI_2: 272 if (cpu_has_guestctl2) 273 set_c0_guestctl2(irq); 274 else 275 set_gc0_cause(irq); 276 break; 277 278 default: 279 break; 280 } 281 282 clear_bit(priority, &vcpu->arch.pending_exceptions); 283 return 1; 284 } 285 286 static int kvm_vz_irq_clear_cb(struct kvm_vcpu *vcpu, unsigned int priority, 287 u32 cause) 288 { 289 u32 irq = (priority < MIPS_EXC_MAX) ? 290 kvm_priority_to_irq[priority] : 0; 291 292 switch (priority) { 293 case MIPS_EXC_INT_TIMER: 294 /* 295 * Call to kvm_write_c0_guest_compare() clears Cause.TI in 296 * kvm_mips_emulate_CP0(). Explicitly clear irq associated with 297 * Cause.IP[IPTI] if GuestCtl2 virtual interrupt register not 298 * supported or if not using GuestCtl2 Hardware Clear. 299 */ 300 if (cpu_has_guestctl2) { 301 if (!(read_c0_guestctl2() & (irq << 14))) 302 clear_c0_guestctl2(irq); 303 } else { 304 clear_gc0_cause(irq); 305 } 306 break; 307 308 case MIPS_EXC_INT_IO_1: 309 case MIPS_EXC_INT_IO_2: 310 case MIPS_EXC_INT_IPI_1: 311 case MIPS_EXC_INT_IPI_2: 312 /* Clear GuestCtl2.VIP irq if not using Hardware Clear */ 313 if (cpu_has_guestctl2) { 314 if (!(read_c0_guestctl2() & (irq << 14))) 315 clear_c0_guestctl2(irq); 316 } else { 317 clear_gc0_cause(irq); 318 } 319 break; 320 321 default: 322 break; 323 } 324 325 clear_bit(priority, &vcpu->arch.pending_exceptions_clr); 326 return 1; 327 } 328 329 /* 330 * VZ guest timer handling. 331 */ 332 333 /** 334 * kvm_vz_should_use_htimer() - Find whether to use the VZ hard guest timer. 335 * @vcpu: Virtual CPU. 336 * 337 * Returns: true if the VZ GTOffset & real guest CP0_Count should be used 338 * instead of software emulation of guest timer. 339 * false otherwise. 340 */ 341 static bool kvm_vz_should_use_htimer(struct kvm_vcpu *vcpu) 342 { 343 if (kvm_mips_count_disabled(vcpu)) 344 return false; 345 346 /* Chosen frequency must match real frequency */ 347 if (mips_hpt_frequency != vcpu->arch.count_hz) 348 return false; 349 350 /* We don't support a CP0_GTOffset with fewer bits than CP0_Count */ 351 if (current_cpu_data.gtoffset_mask != 0xffffffff) 352 return false; 353 354 return true; 355 } 356 357 /** 358 * _kvm_vz_restore_stimer() - Restore soft timer state. 359 * @vcpu: Virtual CPU. 360 * @compare: CP0_Compare register value, restored by caller. 361 * @cause: CP0_Cause register to restore. 362 * 363 * Restore VZ state relating to the soft timer. The hard timer can be enabled 364 * later. 365 */ 366 static void _kvm_vz_restore_stimer(struct kvm_vcpu *vcpu, u32 compare, 367 u32 cause) 368 { 369 /* 370 * Avoid spurious counter interrupts by setting Guest CP0_Count to just 371 * after Guest CP0_Compare. 372 */ 373 write_c0_gtoffset(compare - read_c0_count()); 374 375 back_to_back_c0_hazard(); 376 write_gc0_cause(cause); 377 } 378 379 /** 380 * _kvm_vz_restore_htimer() - Restore hard timer state. 381 * @vcpu: Virtual CPU. 382 * @compare: CP0_Compare register value, restored by caller. 383 * @cause: CP0_Cause register to restore. 384 * 385 * Restore hard timer Guest.Count & Guest.Cause taking care to preserve the 386 * value of Guest.CP0_Cause.TI while restoring Guest.CP0_Cause. 387 */ 388 static void _kvm_vz_restore_htimer(struct kvm_vcpu *vcpu, 389 u32 compare, u32 cause) 390 { 391 u32 start_count, after_count; 392 ktime_t freeze_time; 393 unsigned long flags; 394 395 /* 396 * Freeze the soft-timer and sync the guest CP0_Count with it. We do 397 * this with interrupts disabled to avoid latency. 398 */ 399 local_irq_save(flags); 400 freeze_time = kvm_mips_freeze_hrtimer(vcpu, &start_count); 401 write_c0_gtoffset(start_count - read_c0_count()); 402 local_irq_restore(flags); 403 404 /* restore guest CP0_Cause, as TI may already be set */ 405 back_to_back_c0_hazard(); 406 write_gc0_cause(cause); 407 408 /* 409 * The above sequence isn't atomic and would result in lost timer 410 * interrupts if we're not careful. Detect if a timer interrupt is due 411 * and assert it. 412 */ 413 back_to_back_c0_hazard(); 414 after_count = read_gc0_count(); 415 if (after_count - start_count > compare - start_count - 1) 416 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER); 417 } 418 419 /** 420 * kvm_vz_restore_timer() - Restore timer state. 421 * @vcpu: Virtual CPU. 422 * 423 * Restore soft timer state from saved context. 424 */ 425 static void kvm_vz_restore_timer(struct kvm_vcpu *vcpu) 426 { 427 struct mips_coproc *cop0 = vcpu->arch.cop0; 428 u32 cause, compare; 429 430 compare = kvm_read_sw_gc0_compare(cop0); 431 cause = kvm_read_sw_gc0_cause(cop0); 432 433 write_gc0_compare(compare); 434 _kvm_vz_restore_stimer(vcpu, compare, cause); 435 } 436 437 /** 438 * kvm_vz_acquire_htimer() - Switch to hard timer state. 439 * @vcpu: Virtual CPU. 440 * 441 * Restore hard timer state on top of existing soft timer state if possible. 442 * 443 * Since hard timer won't remain active over preemption, preemption should be 444 * disabled by the caller. 445 */ 446 void kvm_vz_acquire_htimer(struct kvm_vcpu *vcpu) 447 { 448 u32 gctl0; 449 450 gctl0 = read_c0_guestctl0(); 451 if (!(gctl0 & MIPS_GCTL0_GT) && kvm_vz_should_use_htimer(vcpu)) { 452 /* enable guest access to hard timer */ 453 write_c0_guestctl0(gctl0 | MIPS_GCTL0_GT); 454 455 _kvm_vz_restore_htimer(vcpu, read_gc0_compare(), 456 read_gc0_cause()); 457 } 458 } 459 460 /** 461 * _kvm_vz_save_htimer() - Switch to software emulation of guest timer. 462 * @vcpu: Virtual CPU. 463 * @compare: Pointer to write compare value to. 464 * @cause: Pointer to write cause value to. 465 * 466 * Save VZ guest timer state and switch to software emulation of guest CP0 467 * timer. The hard timer must already be in use, so preemption should be 468 * disabled. 469 */ 470 static void _kvm_vz_save_htimer(struct kvm_vcpu *vcpu, 471 u32 *out_compare, u32 *out_cause) 472 { 473 u32 cause, compare, before_count, end_count; 474 ktime_t before_time; 475 476 compare = read_gc0_compare(); 477 *out_compare = compare; 478 479 before_time = ktime_get(); 480 481 /* 482 * Record the CP0_Count *prior* to saving CP0_Cause, so we have a time 483 * at which no pending timer interrupt is missing. 484 */ 485 before_count = read_gc0_count(); 486 back_to_back_c0_hazard(); 487 cause = read_gc0_cause(); 488 *out_cause = cause; 489 490 /* 491 * Record a final CP0_Count which we will transfer to the soft-timer. 492 * This is recorded *after* saving CP0_Cause, so we don't get any timer 493 * interrupts from just after the final CP0_Count point. 494 */ 495 back_to_back_c0_hazard(); 496 end_count = read_gc0_count(); 497 498 /* 499 * The above sequence isn't atomic, so we could miss a timer interrupt 500 * between reading CP0_Cause and end_count. Detect and record any timer 501 * interrupt due between before_count and end_count. 502 */ 503 if (end_count - before_count > compare - before_count - 1) 504 kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_TIMER); 505 506 /* 507 * Restore soft-timer, ignoring a small amount of negative drift due to 508 * delay between freeze_hrtimer and setting CP0_GTOffset. 509 */ 510 kvm_mips_restore_hrtimer(vcpu, before_time, end_count, -0x10000); 511 } 512 513 /** 514 * kvm_vz_save_timer() - Save guest timer state. 515 * @vcpu: Virtual CPU. 516 * 517 * Save VZ guest timer state and switch to soft guest timer if hard timer was in 518 * use. 519 */ 520 static void kvm_vz_save_timer(struct kvm_vcpu *vcpu) 521 { 522 struct mips_coproc *cop0 = vcpu->arch.cop0; 523 u32 gctl0, compare, cause; 524 525 gctl0 = read_c0_guestctl0(); 526 if (gctl0 & MIPS_GCTL0_GT) { 527 /* disable guest use of hard timer */ 528 write_c0_guestctl0(gctl0 & ~MIPS_GCTL0_GT); 529 530 /* save hard timer state */ 531 _kvm_vz_save_htimer(vcpu, &compare, &cause); 532 } else { 533 compare = read_gc0_compare(); 534 cause = read_gc0_cause(); 535 } 536 537 /* save timer-related state to VCPU context */ 538 kvm_write_sw_gc0_cause(cop0, cause); 539 kvm_write_sw_gc0_compare(cop0, compare); 540 } 541 542 /** 543 * kvm_vz_lose_htimer() - Ensure hard guest timer is not in use. 544 * @vcpu: Virtual CPU. 545 * 546 * Transfers the state of the hard guest timer to the soft guest timer, leaving 547 * guest state intact so it can continue to be used with the soft timer. 548 */ 549 void kvm_vz_lose_htimer(struct kvm_vcpu *vcpu) 550 { 551 u32 gctl0, compare, cause; 552 553 preempt_disable(); 554 gctl0 = read_c0_guestctl0(); 555 if (gctl0 & MIPS_GCTL0_GT) { 556 /* disable guest use of timer */ 557 write_c0_guestctl0(gctl0 & ~MIPS_GCTL0_GT); 558 559 /* switch to soft timer */ 560 _kvm_vz_save_htimer(vcpu, &compare, &cause); 561 562 /* leave soft timer in usable state */ 563 _kvm_vz_restore_stimer(vcpu, compare, cause); 564 } 565 preempt_enable(); 566 } 567 568 /** 569 * is_eva_access() - Find whether an instruction is an EVA memory accessor. 570 * @inst: 32-bit instruction encoding. 571 * 572 * Finds whether @inst encodes an EVA memory access instruction, which would 573 * indicate that emulation of it should access the user mode address space 574 * instead of the kernel mode address space. This matters for MUSUK segments 575 * which are TLB mapped for user mode but unmapped for kernel mode. 576 * 577 * Returns: Whether @inst encodes an EVA accessor instruction. 578 */ 579 static bool is_eva_access(union mips_instruction inst) 580 { 581 if (inst.spec3_format.opcode != spec3_op) 582 return false; 583 584 switch (inst.spec3_format.func) { 585 case lwle_op: 586 case lwre_op: 587 case cachee_op: 588 case sbe_op: 589 case she_op: 590 case sce_op: 591 case swe_op: 592 case swle_op: 593 case swre_op: 594 case prefe_op: 595 case lbue_op: 596 case lhue_op: 597 case lbe_op: 598 case lhe_op: 599 case lle_op: 600 case lwe_op: 601 return true; 602 default: 603 return false; 604 } 605 } 606 607 /** 608 * is_eva_am_mapped() - Find whether an access mode is mapped. 609 * @vcpu: KVM VCPU state. 610 * @am: 3-bit encoded access mode. 611 * @eu: Segment becomes unmapped and uncached when Status.ERL=1. 612 * 613 * Decode @am to find whether it encodes a mapped segment for the current VCPU 614 * state. Where necessary @eu and the actual instruction causing the fault are 615 * taken into account to make the decision. 616 * 617 * Returns: Whether the VCPU faulted on a TLB mapped address. 618 */ 619 static bool is_eva_am_mapped(struct kvm_vcpu *vcpu, unsigned int am, bool eu) 620 { 621 u32 am_lookup; 622 int err; 623 624 /* 625 * Interpret access control mode. We assume address errors will already 626 * have been caught by the guest, leaving us with: 627 * AM UM SM KM 31..24 23..16 628 * UK 0 000 Unm 0 0 629 * MK 1 001 TLB 1 630 * MSK 2 010 TLB TLB 1 631 * MUSK 3 011 TLB TLB TLB 1 632 * MUSUK 4 100 TLB TLB Unm 0 1 633 * USK 5 101 Unm Unm 0 0 634 * - 6 110 0 0 635 * UUSK 7 111 Unm Unm Unm 0 0 636 * 637 * We shift a magic value by AM across the sign bit to find if always 638 * TLB mapped, and if not shift by 8 again to find if it depends on KM. 639 */ 640 am_lookup = 0x70080000 << am; 641 if ((s32)am_lookup < 0) { 642 /* 643 * MK, MSK, MUSK 644 * Always TLB mapped, unless SegCtl.EU && ERL 645 */ 646 if (!eu || !(read_gc0_status() & ST0_ERL)) 647 return true; 648 } else { 649 am_lookup <<= 8; 650 if ((s32)am_lookup < 0) { 651 union mips_instruction inst; 652 unsigned int status; 653 u32 *opc; 654 655 /* 656 * MUSUK 657 * TLB mapped if not in kernel mode 658 */ 659 status = read_gc0_status(); 660 if (!(status & (ST0_EXL | ST0_ERL)) && 661 (status & ST0_KSU)) 662 return true; 663 /* 664 * EVA access instructions in kernel 665 * mode access user address space. 666 */ 667 opc = (u32 *)vcpu->arch.pc; 668 if (vcpu->arch.host_cp0_cause & CAUSEF_BD) 669 opc += 1; 670 err = kvm_get_badinstr(opc, vcpu, &inst.word); 671 if (!err && is_eva_access(inst)) 672 return true; 673 } 674 } 675 676 return false; 677 } 678 679 /** 680 * kvm_vz_gva_to_gpa() - Convert valid GVA to GPA. 681 * @vcpu: KVM VCPU state. 682 * @gva: Guest virtual address to convert. 683 * @gpa: Output guest physical address. 684 * 685 * Convert a guest virtual address (GVA) which is valid according to the guest 686 * context, to a guest physical address (GPA). 687 * 688 * Returns: 0 on success. 689 * -errno on failure. 690 */ 691 static int kvm_vz_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, 692 unsigned long *gpa) 693 { 694 u32 gva32 = gva; 695 unsigned long segctl; 696 697 if ((long)gva == (s32)gva32) { 698 /* Handle canonical 32-bit virtual address */ 699 if (cpu_guest_has_segments) { 700 unsigned long mask, pa; 701 702 switch (gva32 >> 29) { 703 case 0: 704 case 1: /* CFG5 (1GB) */ 705 segctl = read_gc0_segctl2() >> 16; 706 mask = (unsigned long)0xfc0000000ull; 707 break; 708 case 2: 709 case 3: /* CFG4 (1GB) */ 710 segctl = read_gc0_segctl2(); 711 mask = (unsigned long)0xfc0000000ull; 712 break; 713 case 4: /* CFG3 (512MB) */ 714 segctl = read_gc0_segctl1() >> 16; 715 mask = (unsigned long)0xfe0000000ull; 716 break; 717 case 5: /* CFG2 (512MB) */ 718 segctl = read_gc0_segctl1(); 719 mask = (unsigned long)0xfe0000000ull; 720 break; 721 case 6: /* CFG1 (512MB) */ 722 segctl = read_gc0_segctl0() >> 16; 723 mask = (unsigned long)0xfe0000000ull; 724 break; 725 case 7: /* CFG0 (512MB) */ 726 segctl = read_gc0_segctl0(); 727 mask = (unsigned long)0xfe0000000ull; 728 break; 729 default: 730 /* 731 * GCC 4.9 isn't smart enough to figure out that 732 * segctl and mask are always initialised. 733 */ 734 unreachable(); 735 } 736 737 if (is_eva_am_mapped(vcpu, (segctl >> 4) & 0x7, 738 segctl & 0x0008)) 739 goto tlb_mapped; 740 741 /* Unmapped, find guest physical address */ 742 pa = (segctl << 20) & mask; 743 pa |= gva32 & ~mask; 744 *gpa = pa; 745 return 0; 746 } else if ((s32)gva32 < (s32)0xc0000000) { 747 /* legacy unmapped KSeg0 or KSeg1 */ 748 *gpa = gva32 & 0x1fffffff; 749 return 0; 750 } 751 #ifdef CONFIG_64BIT 752 } else if ((gva & 0xc000000000000000) == 0x8000000000000000) { 753 /* XKPHYS */ 754 if (cpu_guest_has_segments) { 755 /* 756 * Each of the 8 regions can be overridden by SegCtl2.XR 757 * to use SegCtl1.XAM. 758 */ 759 segctl = read_gc0_segctl2(); 760 if (segctl & (1ull << (56 + ((gva >> 59) & 0x7)))) { 761 segctl = read_gc0_segctl1(); 762 if (is_eva_am_mapped(vcpu, (segctl >> 59) & 0x7, 763 0)) 764 goto tlb_mapped; 765 } 766 767 } 768 /* 769 * Traditionally fully unmapped. 770 * Bits 61:59 specify the CCA, which we can just mask off here. 771 * Bits 58:PABITS should be zero, but we shouldn't have got here 772 * if it wasn't. 773 */ 774 *gpa = gva & 0x07ffffffffffffff; 775 return 0; 776 #endif 777 } 778 779 tlb_mapped: 780 return kvm_vz_guest_tlb_lookup(vcpu, gva, gpa); 781 } 782 783 /** 784 * kvm_vz_badvaddr_to_gpa() - Convert GVA BadVAddr from root exception to GPA. 785 * @vcpu: KVM VCPU state. 786 * @badvaddr: Root BadVAddr. 787 * @gpa: Output guest physical address. 788 * 789 * VZ implementations are permitted to report guest virtual addresses (GVA) in 790 * BadVAddr on a root exception during guest execution, instead of the more 791 * convenient guest physical addresses (GPA). When we get a GVA, this function 792 * converts it to a GPA, taking into account guest segmentation and guest TLB 793 * state. 794 * 795 * Returns: 0 on success. 796 * -errno on failure. 797 */ 798 static int kvm_vz_badvaddr_to_gpa(struct kvm_vcpu *vcpu, unsigned long badvaddr, 799 unsigned long *gpa) 800 { 801 unsigned int gexccode = (vcpu->arch.host_cp0_guestctl0 & 802 MIPS_GCTL0_GEXC) >> MIPS_GCTL0_GEXC_SHIFT; 803 804 /* If BadVAddr is GPA, then all is well in the world */ 805 if (likely(gexccode == MIPS_GCTL0_GEXC_GPA)) { 806 *gpa = badvaddr; 807 return 0; 808 } 809 810 /* Otherwise we'd expect it to be GVA ... */ 811 if (WARN(gexccode != MIPS_GCTL0_GEXC_GVA, 812 "Unexpected gexccode %#x\n", gexccode)) 813 return -EINVAL; 814 815 /* ... and we need to perform the GVA->GPA translation in software */ 816 return kvm_vz_gva_to_gpa(vcpu, badvaddr, gpa); 817 } 818 819 static int kvm_trap_vz_no_handler(struct kvm_vcpu *vcpu) 820 { 821 u32 *opc = (u32 *) vcpu->arch.pc; 822 u32 cause = vcpu->arch.host_cp0_cause; 823 u32 exccode = (cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE; 824 unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr; 825 u32 inst = 0; 826 827 /* 828 * Fetch the instruction. 829 */ 830 if (cause & CAUSEF_BD) 831 opc += 1; 832 kvm_get_badinstr(opc, vcpu, &inst); 833 834 kvm_err("Exception Code: %d not handled @ PC: %p, inst: 0x%08x BadVaddr: %#lx Status: %#x\n", 835 exccode, opc, inst, badvaddr, 836 read_gc0_status()); 837 kvm_arch_vcpu_dump_regs(vcpu); 838 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 839 return RESUME_HOST; 840 } 841 842 static unsigned long mips_process_maar(unsigned int op, unsigned long val) 843 { 844 /* Mask off unused bits */ 845 unsigned long mask = 0xfffff000 | MIPS_MAAR_S | MIPS_MAAR_VL; 846 847 if (read_gc0_pagegrain() & PG_ELPA) 848 mask |= 0x00ffffff00000000ull; 849 if (cpu_guest_has_mvh) 850 mask |= MIPS_MAAR_VH; 851 852 /* Set or clear VH */ 853 if (op == mtc_op) { 854 /* clear VH */ 855 val &= ~MIPS_MAAR_VH; 856 } else if (op == dmtc_op) { 857 /* set VH to match VL */ 858 val &= ~MIPS_MAAR_VH; 859 if (val & MIPS_MAAR_VL) 860 val |= MIPS_MAAR_VH; 861 } 862 863 return val & mask; 864 } 865 866 static void kvm_write_maari(struct kvm_vcpu *vcpu, unsigned long val) 867 { 868 struct mips_coproc *cop0 = vcpu->arch.cop0; 869 870 val &= MIPS_MAARI_INDEX; 871 if (val == MIPS_MAARI_INDEX) 872 kvm_write_sw_gc0_maari(cop0, ARRAY_SIZE(vcpu->arch.maar) - 1); 873 else if (val < ARRAY_SIZE(vcpu->arch.maar)) 874 kvm_write_sw_gc0_maari(cop0, val); 875 } 876 877 static enum emulation_result kvm_vz_gpsi_cop0(union mips_instruction inst, 878 u32 *opc, u32 cause, 879 struct kvm_vcpu *vcpu) 880 { 881 struct mips_coproc *cop0 = vcpu->arch.cop0; 882 enum emulation_result er = EMULATE_DONE; 883 u32 rt, rd, sel; 884 unsigned long curr_pc; 885 unsigned long val; 886 887 /* 888 * Update PC and hold onto current PC in case there is 889 * an error and we want to rollback the PC 890 */ 891 curr_pc = vcpu->arch.pc; 892 er = update_pc(vcpu, cause); 893 if (er == EMULATE_FAIL) 894 return er; 895 896 if (inst.co_format.co) { 897 switch (inst.co_format.func) { 898 case wait_op: 899 er = kvm_mips_emul_wait(vcpu); 900 break; 901 default: 902 er = EMULATE_FAIL; 903 } 904 } else { 905 rt = inst.c0r_format.rt; 906 rd = inst.c0r_format.rd; 907 sel = inst.c0r_format.sel; 908 909 switch (inst.c0r_format.rs) { 910 case dmfc_op: 911 case mfc_op: 912 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS 913 cop0->stat[rd][sel]++; 914 #endif 915 if (rd == MIPS_CP0_COUNT && 916 sel == 0) { /* Count */ 917 val = kvm_mips_read_count(vcpu); 918 } else if (rd == MIPS_CP0_COMPARE && 919 sel == 0) { /* Compare */ 920 val = read_gc0_compare(); 921 } else if (rd == MIPS_CP0_LLADDR && 922 sel == 0) { /* LLAddr */ 923 if (cpu_guest_has_rw_llb) 924 val = read_gc0_lladdr() & 925 MIPS_LLADDR_LLB; 926 else 927 val = 0; 928 } else if (rd == MIPS_CP0_LLADDR && 929 sel == 1 && /* MAAR */ 930 cpu_guest_has_maar && 931 !cpu_guest_has_dyn_maar) { 932 /* MAARI must be in range */ 933 BUG_ON(kvm_read_sw_gc0_maari(cop0) >= 934 ARRAY_SIZE(vcpu->arch.maar)); 935 val = vcpu->arch.maar[ 936 kvm_read_sw_gc0_maari(cop0)]; 937 } else if ((rd == MIPS_CP0_PRID && 938 (sel == 0 || /* PRid */ 939 sel == 2 || /* CDMMBase */ 940 sel == 3)) || /* CMGCRBase */ 941 (rd == MIPS_CP0_STATUS && 942 (sel == 2 || /* SRSCtl */ 943 sel == 3)) || /* SRSMap */ 944 (rd == MIPS_CP0_CONFIG && 945 (sel == 6 || /* Config6 */ 946 sel == 7)) || /* Config7 */ 947 (rd == MIPS_CP0_LLADDR && 948 (sel == 2) && /* MAARI */ 949 cpu_guest_has_maar && 950 !cpu_guest_has_dyn_maar) || 951 (rd == MIPS_CP0_ERRCTL && 952 (sel == 0))) { /* ErrCtl */ 953 val = cop0->reg[rd][sel]; 954 #ifdef CONFIG_CPU_LOONGSON64 955 } else if (rd == MIPS_CP0_DIAG && 956 (sel == 0)) { /* Diag */ 957 val = cop0->reg[rd][sel]; 958 #endif 959 } else { 960 val = 0; 961 er = EMULATE_FAIL; 962 } 963 964 if (er != EMULATE_FAIL) { 965 /* Sign extend */ 966 if (inst.c0r_format.rs == mfc_op) 967 val = (int)val; 968 vcpu->arch.gprs[rt] = val; 969 } 970 971 trace_kvm_hwr(vcpu, (inst.c0r_format.rs == mfc_op) ? 972 KVM_TRACE_MFC0 : KVM_TRACE_DMFC0, 973 KVM_TRACE_COP0(rd, sel), val); 974 break; 975 976 case dmtc_op: 977 case mtc_op: 978 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS 979 cop0->stat[rd][sel]++; 980 #endif 981 val = vcpu->arch.gprs[rt]; 982 trace_kvm_hwr(vcpu, (inst.c0r_format.rs == mtc_op) ? 983 KVM_TRACE_MTC0 : KVM_TRACE_DMTC0, 984 KVM_TRACE_COP0(rd, sel), val); 985 986 if (rd == MIPS_CP0_COUNT && 987 sel == 0) { /* Count */ 988 kvm_vz_lose_htimer(vcpu); 989 kvm_mips_write_count(vcpu, vcpu->arch.gprs[rt]); 990 } else if (rd == MIPS_CP0_COMPARE && 991 sel == 0) { /* Compare */ 992 kvm_mips_write_compare(vcpu, 993 vcpu->arch.gprs[rt], 994 true); 995 } else if (rd == MIPS_CP0_LLADDR && 996 sel == 0) { /* LLAddr */ 997 /* 998 * P5600 generates GPSI on guest MTC0 LLAddr. 999 * Only allow the guest to clear LLB. 1000 */ 1001 if (cpu_guest_has_rw_llb && 1002 !(val & MIPS_LLADDR_LLB)) 1003 write_gc0_lladdr(0); 1004 } else if (rd == MIPS_CP0_LLADDR && 1005 sel == 1 && /* MAAR */ 1006 cpu_guest_has_maar && 1007 !cpu_guest_has_dyn_maar) { 1008 val = mips_process_maar(inst.c0r_format.rs, 1009 val); 1010 1011 /* MAARI must be in range */ 1012 BUG_ON(kvm_read_sw_gc0_maari(cop0) >= 1013 ARRAY_SIZE(vcpu->arch.maar)); 1014 vcpu->arch.maar[kvm_read_sw_gc0_maari(cop0)] = 1015 val; 1016 } else if (rd == MIPS_CP0_LLADDR && 1017 (sel == 2) && /* MAARI */ 1018 cpu_guest_has_maar && 1019 !cpu_guest_has_dyn_maar) { 1020 kvm_write_maari(vcpu, val); 1021 } else if (rd == MIPS_CP0_CONFIG && 1022 (sel == 6)) { 1023 cop0->reg[rd][sel] = (int)val; 1024 } else if (rd == MIPS_CP0_ERRCTL && 1025 (sel == 0)) { /* ErrCtl */ 1026 /* ignore the written value */ 1027 #ifdef CONFIG_CPU_LOONGSON64 1028 } else if (rd == MIPS_CP0_DIAG && 1029 (sel == 0)) { /* Diag */ 1030 unsigned long flags; 1031 1032 local_irq_save(flags); 1033 if (val & LOONGSON_DIAG_BTB) { 1034 /* Flush BTB */ 1035 set_c0_diag(LOONGSON_DIAG_BTB); 1036 } 1037 if (val & LOONGSON_DIAG_ITLB) { 1038 /* Flush ITLB */ 1039 set_c0_diag(LOONGSON_DIAG_ITLB); 1040 } 1041 if (val & LOONGSON_DIAG_DTLB) { 1042 /* Flush DTLB */ 1043 set_c0_diag(LOONGSON_DIAG_DTLB); 1044 } 1045 if (val & LOONGSON_DIAG_VTLB) { 1046 /* Flush VTLB */ 1047 kvm_loongson_clear_guest_vtlb(); 1048 } 1049 if (val & LOONGSON_DIAG_FTLB) { 1050 /* Flush FTLB */ 1051 kvm_loongson_clear_guest_ftlb(); 1052 } 1053 local_irq_restore(flags); 1054 #endif 1055 } else { 1056 er = EMULATE_FAIL; 1057 } 1058 break; 1059 1060 default: 1061 er = EMULATE_FAIL; 1062 break; 1063 } 1064 } 1065 /* Rollback PC only if emulation was unsuccessful */ 1066 if (er == EMULATE_FAIL) { 1067 kvm_err("[%#lx]%s: unsupported cop0 instruction 0x%08x\n", 1068 curr_pc, __func__, inst.word); 1069 1070 vcpu->arch.pc = curr_pc; 1071 } 1072 1073 return er; 1074 } 1075 1076 static enum emulation_result kvm_vz_gpsi_cache(union mips_instruction inst, 1077 u32 *opc, u32 cause, 1078 struct kvm_vcpu *vcpu) 1079 { 1080 enum emulation_result er = EMULATE_DONE; 1081 u32 cache, op_inst, op, base; 1082 s16 offset; 1083 struct kvm_vcpu_arch *arch = &vcpu->arch; 1084 unsigned long va, curr_pc; 1085 1086 /* 1087 * Update PC and hold onto current PC in case there is 1088 * an error and we want to rollback the PC 1089 */ 1090 curr_pc = vcpu->arch.pc; 1091 er = update_pc(vcpu, cause); 1092 if (er == EMULATE_FAIL) 1093 return er; 1094 1095 base = inst.i_format.rs; 1096 op_inst = inst.i_format.rt; 1097 if (cpu_has_mips_r6) 1098 offset = inst.spec3_format.simmediate; 1099 else 1100 offset = inst.i_format.simmediate; 1101 cache = op_inst & CacheOp_Cache; 1102 op = op_inst & CacheOp_Op; 1103 1104 va = arch->gprs[base] + offset; 1105 1106 kvm_debug("CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n", 1107 cache, op, base, arch->gprs[base], offset); 1108 1109 /* Secondary or tirtiary cache ops ignored */ 1110 if (cache != Cache_I && cache != Cache_D) 1111 return EMULATE_DONE; 1112 1113 switch (op_inst) { 1114 case Index_Invalidate_I: 1115 flush_icache_line_indexed(va); 1116 return EMULATE_DONE; 1117 case Index_Writeback_Inv_D: 1118 flush_dcache_line_indexed(va); 1119 return EMULATE_DONE; 1120 case Hit_Invalidate_I: 1121 case Hit_Invalidate_D: 1122 case Hit_Writeback_Inv_D: 1123 if (boot_cpu_type() == CPU_CAVIUM_OCTEON3) { 1124 /* We can just flush entire icache */ 1125 local_flush_icache_range(0, 0); 1126 return EMULATE_DONE; 1127 } 1128 1129 /* So far, other platforms support guest hit cache ops */ 1130 break; 1131 default: 1132 break; 1133 } 1134 1135 kvm_err("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n", 1136 curr_pc, vcpu->arch.gprs[31], cache, op, base, arch->gprs[base], 1137 offset); 1138 /* Rollback PC */ 1139 vcpu->arch.pc = curr_pc; 1140 1141 return EMULATE_FAIL; 1142 } 1143 1144 #ifdef CONFIG_CPU_LOONGSON64 1145 static enum emulation_result kvm_vz_gpsi_lwc2(union mips_instruction inst, 1146 u32 *opc, u32 cause, 1147 struct kvm_run *run, 1148 struct kvm_vcpu *vcpu) 1149 { 1150 unsigned int rs, rd; 1151 unsigned int hostcfg; 1152 unsigned long curr_pc; 1153 enum emulation_result er = EMULATE_DONE; 1154 1155 /* 1156 * Update PC and hold onto current PC in case there is 1157 * an error and we want to rollback the PC 1158 */ 1159 curr_pc = vcpu->arch.pc; 1160 er = update_pc(vcpu, cause); 1161 if (er == EMULATE_FAIL) 1162 return er; 1163 1164 rs = inst.loongson3_lscsr_format.rs; 1165 rd = inst.loongson3_lscsr_format.rd; 1166 switch (inst.loongson3_lscsr_format.fr) { 1167 case 0x8: /* Read CPUCFG */ 1168 ++vcpu->stat.vz_cpucfg_exits; 1169 hostcfg = read_cpucfg(vcpu->arch.gprs[rs]); 1170 1171 switch (vcpu->arch.gprs[rs]) { 1172 case LOONGSON_CFG0: 1173 vcpu->arch.gprs[rd] = 0x14c000; 1174 break; 1175 case LOONGSON_CFG1: 1176 hostcfg &= (LOONGSON_CFG1_FP | LOONGSON_CFG1_MMI | 1177 LOONGSON_CFG1_MSA1 | LOONGSON_CFG1_MSA2 | 1178 LOONGSON_CFG1_SFBP); 1179 vcpu->arch.gprs[rd] = hostcfg; 1180 break; 1181 case LOONGSON_CFG2: 1182 hostcfg &= (LOONGSON_CFG2_LEXT1 | LOONGSON_CFG2_LEXT2 | 1183 LOONGSON_CFG2_LEXT3 | LOONGSON_CFG2_LSPW); 1184 vcpu->arch.gprs[rd] = hostcfg; 1185 break; 1186 case LOONGSON_CFG3: 1187 vcpu->arch.gprs[rd] = hostcfg; 1188 break; 1189 default: 1190 /* Don't export any other advanced features to guest */ 1191 vcpu->arch.gprs[rd] = 0; 1192 break; 1193 } 1194 break; 1195 1196 default: 1197 kvm_err("lwc2 emulate not impl %d rs %lx @%lx\n", 1198 inst.loongson3_lscsr_format.fr, vcpu->arch.gprs[rs], curr_pc); 1199 er = EMULATE_FAIL; 1200 break; 1201 } 1202 1203 /* Rollback PC only if emulation was unsuccessful */ 1204 if (er == EMULATE_FAIL) { 1205 kvm_err("[%#lx]%s: unsupported lwc2 instruction 0x%08x 0x%08x\n", 1206 curr_pc, __func__, inst.word, inst.loongson3_lscsr_format.fr); 1207 1208 vcpu->arch.pc = curr_pc; 1209 } 1210 1211 return er; 1212 } 1213 #endif 1214 1215 static enum emulation_result kvm_trap_vz_handle_gpsi(u32 cause, u32 *opc, 1216 struct kvm_vcpu *vcpu) 1217 { 1218 enum emulation_result er = EMULATE_DONE; 1219 struct kvm_vcpu_arch *arch = &vcpu->arch; 1220 union mips_instruction inst; 1221 int rd, rt, sel; 1222 int err; 1223 1224 /* 1225 * Fetch the instruction. 1226 */ 1227 if (cause & CAUSEF_BD) 1228 opc += 1; 1229 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1230 if (err) 1231 return EMULATE_FAIL; 1232 1233 switch (inst.r_format.opcode) { 1234 case cop0_op: 1235 er = kvm_vz_gpsi_cop0(inst, opc, cause, vcpu); 1236 break; 1237 #ifndef CONFIG_CPU_MIPSR6 1238 case cache_op: 1239 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE); 1240 er = kvm_vz_gpsi_cache(inst, opc, cause, vcpu); 1241 break; 1242 #endif 1243 #ifdef CONFIG_CPU_LOONGSON64 1244 case lwc2_op: 1245 er = kvm_vz_gpsi_lwc2(inst, opc, cause, run, vcpu); 1246 break; 1247 #endif 1248 case spec3_op: 1249 switch (inst.spec3_format.func) { 1250 #ifdef CONFIG_CPU_MIPSR6 1251 case cache6_op: 1252 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE); 1253 er = kvm_vz_gpsi_cache(inst, opc, cause, vcpu); 1254 break; 1255 #endif 1256 case rdhwr_op: 1257 if (inst.r_format.rs || (inst.r_format.re >> 3)) 1258 goto unknown; 1259 1260 rd = inst.r_format.rd; 1261 rt = inst.r_format.rt; 1262 sel = inst.r_format.re & 0x7; 1263 1264 switch (rd) { 1265 case MIPS_HWR_CC: /* Read count register */ 1266 arch->gprs[rt] = 1267 (long)(int)kvm_mips_read_count(vcpu); 1268 break; 1269 default: 1270 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, 1271 KVM_TRACE_HWR(rd, sel), 0); 1272 goto unknown; 1273 } 1274 1275 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, 1276 KVM_TRACE_HWR(rd, sel), arch->gprs[rt]); 1277 1278 er = update_pc(vcpu, cause); 1279 break; 1280 default: 1281 goto unknown; 1282 } 1283 break; 1284 unknown: 1285 1286 default: 1287 kvm_err("GPSI exception not supported (%p/%#x)\n", 1288 opc, inst.word); 1289 kvm_arch_vcpu_dump_regs(vcpu); 1290 er = EMULATE_FAIL; 1291 break; 1292 } 1293 1294 return er; 1295 } 1296 1297 static enum emulation_result kvm_trap_vz_handle_gsfc(u32 cause, u32 *opc, 1298 struct kvm_vcpu *vcpu) 1299 { 1300 enum emulation_result er = EMULATE_DONE; 1301 struct kvm_vcpu_arch *arch = &vcpu->arch; 1302 union mips_instruction inst; 1303 int err; 1304 1305 /* 1306 * Fetch the instruction. 1307 */ 1308 if (cause & CAUSEF_BD) 1309 opc += 1; 1310 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1311 if (err) 1312 return EMULATE_FAIL; 1313 1314 /* complete MTC0 on behalf of guest and advance EPC */ 1315 if (inst.c0r_format.opcode == cop0_op && 1316 inst.c0r_format.rs == mtc_op && 1317 inst.c0r_format.z == 0) { 1318 int rt = inst.c0r_format.rt; 1319 int rd = inst.c0r_format.rd; 1320 int sel = inst.c0r_format.sel; 1321 unsigned int val = arch->gprs[rt]; 1322 unsigned int old_val, change; 1323 1324 trace_kvm_hwr(vcpu, KVM_TRACE_MTC0, KVM_TRACE_COP0(rd, sel), 1325 val); 1326 1327 if ((rd == MIPS_CP0_STATUS) && (sel == 0)) { 1328 /* FR bit should read as zero if no FPU */ 1329 if (!kvm_mips_guest_has_fpu(&vcpu->arch)) 1330 val &= ~(ST0_CU1 | ST0_FR); 1331 1332 /* 1333 * Also don't allow FR to be set if host doesn't support 1334 * it. 1335 */ 1336 if (!(boot_cpu_data.fpu_id & MIPS_FPIR_F64)) 1337 val &= ~ST0_FR; 1338 1339 old_val = read_gc0_status(); 1340 change = val ^ old_val; 1341 1342 if (change & ST0_FR) { 1343 /* 1344 * FPU and Vector register state is made 1345 * UNPREDICTABLE by a change of FR, so don't 1346 * even bother saving it. 1347 */ 1348 kvm_drop_fpu(vcpu); 1349 } 1350 1351 /* 1352 * If MSA state is already live, it is undefined how it 1353 * interacts with FR=0 FPU state, and we don't want to 1354 * hit reserved instruction exceptions trying to save 1355 * the MSA state later when CU=1 && FR=1, so play it 1356 * safe and save it first. 1357 */ 1358 if (change & ST0_CU1 && !(val & ST0_FR) && 1359 vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) 1360 kvm_lose_fpu(vcpu); 1361 1362 write_gc0_status(val); 1363 } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) { 1364 u32 old_cause = read_gc0_cause(); 1365 u32 change = old_cause ^ val; 1366 1367 /* DC bit enabling/disabling timer? */ 1368 if (change & CAUSEF_DC) { 1369 if (val & CAUSEF_DC) { 1370 kvm_vz_lose_htimer(vcpu); 1371 kvm_mips_count_disable_cause(vcpu); 1372 } else { 1373 kvm_mips_count_enable_cause(vcpu); 1374 } 1375 } 1376 1377 /* Only certain bits are RW to the guest */ 1378 change &= (CAUSEF_DC | CAUSEF_IV | CAUSEF_WP | 1379 CAUSEF_IP0 | CAUSEF_IP1); 1380 1381 /* WP can only be cleared */ 1382 change &= ~CAUSEF_WP | old_cause; 1383 1384 write_gc0_cause(old_cause ^ change); 1385 } else if ((rd == MIPS_CP0_STATUS) && (sel == 1)) { /* IntCtl */ 1386 write_gc0_intctl(val); 1387 } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) { 1388 old_val = read_gc0_config5(); 1389 change = val ^ old_val; 1390 /* Handle changes in FPU/MSA modes */ 1391 preempt_disable(); 1392 1393 /* 1394 * Propagate FRE changes immediately if the FPU 1395 * context is already loaded. 1396 */ 1397 if (change & MIPS_CONF5_FRE && 1398 vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU) 1399 change_c0_config5(MIPS_CONF5_FRE, val); 1400 1401 preempt_enable(); 1402 1403 val = old_val ^ 1404 (change & kvm_vz_config5_guest_wrmask(vcpu)); 1405 write_gc0_config5(val); 1406 } else { 1407 kvm_err("Handle GSFC, unsupported field change @ %p: %#x\n", 1408 opc, inst.word); 1409 er = EMULATE_FAIL; 1410 } 1411 1412 if (er != EMULATE_FAIL) 1413 er = update_pc(vcpu, cause); 1414 } else { 1415 kvm_err("Handle GSFC, unrecognized instruction @ %p: %#x\n", 1416 opc, inst.word); 1417 er = EMULATE_FAIL; 1418 } 1419 1420 return er; 1421 } 1422 1423 static enum emulation_result kvm_trap_vz_handle_ghfc(u32 cause, u32 *opc, 1424 struct kvm_vcpu *vcpu) 1425 { 1426 /* 1427 * Presumably this is due to MC (guest mode change), so lets trace some 1428 * relevant info. 1429 */ 1430 trace_kvm_guest_mode_change(vcpu); 1431 1432 return EMULATE_DONE; 1433 } 1434 1435 static enum emulation_result kvm_trap_vz_handle_hc(u32 cause, u32 *opc, 1436 struct kvm_vcpu *vcpu) 1437 { 1438 enum emulation_result er; 1439 union mips_instruction inst; 1440 unsigned long curr_pc; 1441 int err; 1442 1443 if (cause & CAUSEF_BD) 1444 opc += 1; 1445 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1446 if (err) 1447 return EMULATE_FAIL; 1448 1449 /* 1450 * Update PC and hold onto current PC in case there is 1451 * an error and we want to rollback the PC 1452 */ 1453 curr_pc = vcpu->arch.pc; 1454 er = update_pc(vcpu, cause); 1455 if (er == EMULATE_FAIL) 1456 return er; 1457 1458 er = kvm_mips_emul_hypcall(vcpu, inst); 1459 if (er == EMULATE_FAIL) 1460 vcpu->arch.pc = curr_pc; 1461 1462 return er; 1463 } 1464 1465 static enum emulation_result kvm_trap_vz_no_handler_guest_exit(u32 gexccode, 1466 u32 cause, 1467 u32 *opc, 1468 struct kvm_vcpu *vcpu) 1469 { 1470 u32 inst; 1471 1472 /* 1473 * Fetch the instruction. 1474 */ 1475 if (cause & CAUSEF_BD) 1476 opc += 1; 1477 kvm_get_badinstr(opc, vcpu, &inst); 1478 1479 kvm_err("Guest Exception Code: %d not yet handled @ PC: %p, inst: 0x%08x Status: %#x\n", 1480 gexccode, opc, inst, read_gc0_status()); 1481 1482 return EMULATE_FAIL; 1483 } 1484 1485 static int kvm_trap_vz_handle_guest_exit(struct kvm_vcpu *vcpu) 1486 { 1487 u32 *opc = (u32 *) vcpu->arch.pc; 1488 u32 cause = vcpu->arch.host_cp0_cause; 1489 enum emulation_result er = EMULATE_DONE; 1490 u32 gexccode = (vcpu->arch.host_cp0_guestctl0 & 1491 MIPS_GCTL0_GEXC) >> MIPS_GCTL0_GEXC_SHIFT; 1492 int ret = RESUME_GUEST; 1493 1494 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_GEXCCODE_BASE + gexccode); 1495 switch (gexccode) { 1496 case MIPS_GCTL0_GEXC_GPSI: 1497 ++vcpu->stat.vz_gpsi_exits; 1498 er = kvm_trap_vz_handle_gpsi(cause, opc, vcpu); 1499 break; 1500 case MIPS_GCTL0_GEXC_GSFC: 1501 ++vcpu->stat.vz_gsfc_exits; 1502 er = kvm_trap_vz_handle_gsfc(cause, opc, vcpu); 1503 break; 1504 case MIPS_GCTL0_GEXC_HC: 1505 ++vcpu->stat.vz_hc_exits; 1506 er = kvm_trap_vz_handle_hc(cause, opc, vcpu); 1507 break; 1508 case MIPS_GCTL0_GEXC_GRR: 1509 ++vcpu->stat.vz_grr_exits; 1510 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1511 vcpu); 1512 break; 1513 case MIPS_GCTL0_GEXC_GVA: 1514 ++vcpu->stat.vz_gva_exits; 1515 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1516 vcpu); 1517 break; 1518 case MIPS_GCTL0_GEXC_GHFC: 1519 ++vcpu->stat.vz_ghfc_exits; 1520 er = kvm_trap_vz_handle_ghfc(cause, opc, vcpu); 1521 break; 1522 case MIPS_GCTL0_GEXC_GPA: 1523 ++vcpu->stat.vz_gpa_exits; 1524 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1525 vcpu); 1526 break; 1527 default: 1528 ++vcpu->stat.vz_resvd_exits; 1529 er = kvm_trap_vz_no_handler_guest_exit(gexccode, cause, opc, 1530 vcpu); 1531 break; 1532 1533 } 1534 1535 if (er == EMULATE_DONE) { 1536 ret = RESUME_GUEST; 1537 } else if (er == EMULATE_HYPERCALL) { 1538 ret = kvm_mips_handle_hypcall(vcpu); 1539 } else { 1540 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1541 ret = RESUME_HOST; 1542 } 1543 return ret; 1544 } 1545 1546 /** 1547 * kvm_trap_vz_handle_cop_unusuable() - Guest used unusable coprocessor. 1548 * @vcpu: Virtual CPU context. 1549 * 1550 * Handle when the guest attempts to use a coprocessor which hasn't been allowed 1551 * by the root context. 1552 */ 1553 static int kvm_trap_vz_handle_cop_unusable(struct kvm_vcpu *vcpu) 1554 { 1555 u32 cause = vcpu->arch.host_cp0_cause; 1556 enum emulation_result er = EMULATE_FAIL; 1557 int ret = RESUME_GUEST; 1558 1559 if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1) { 1560 /* 1561 * If guest FPU not present, the FPU operation should have been 1562 * treated as a reserved instruction! 1563 * If FPU already in use, we shouldn't get this at all. 1564 */ 1565 if (WARN_ON(!kvm_mips_guest_has_fpu(&vcpu->arch) || 1566 vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)) { 1567 preempt_enable(); 1568 return EMULATE_FAIL; 1569 } 1570 1571 kvm_own_fpu(vcpu); 1572 er = EMULATE_DONE; 1573 } 1574 /* other coprocessors not handled */ 1575 1576 switch (er) { 1577 case EMULATE_DONE: 1578 ret = RESUME_GUEST; 1579 break; 1580 1581 case EMULATE_FAIL: 1582 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1583 ret = RESUME_HOST; 1584 break; 1585 1586 default: 1587 BUG(); 1588 } 1589 return ret; 1590 } 1591 1592 /** 1593 * kvm_trap_vz_handle_msa_disabled() - Guest used MSA while disabled in root. 1594 * @vcpu: Virtual CPU context. 1595 * 1596 * Handle when the guest attempts to use MSA when it is disabled in the root 1597 * context. 1598 */ 1599 static int kvm_trap_vz_handle_msa_disabled(struct kvm_vcpu *vcpu) 1600 { 1601 /* 1602 * If MSA not present or not exposed to guest or FR=0, the MSA operation 1603 * should have been treated as a reserved instruction! 1604 * Same if CU1=1, FR=0. 1605 * If MSA already in use, we shouldn't get this at all. 1606 */ 1607 if (!kvm_mips_guest_has_msa(&vcpu->arch) || 1608 (read_gc0_status() & (ST0_CU1 | ST0_FR)) == ST0_CU1 || 1609 !(read_gc0_config5() & MIPS_CONF5_MSAEN) || 1610 vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) { 1611 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1612 return RESUME_HOST; 1613 } 1614 1615 kvm_own_msa(vcpu); 1616 1617 return RESUME_GUEST; 1618 } 1619 1620 static int kvm_trap_vz_handle_tlb_ld_miss(struct kvm_vcpu *vcpu) 1621 { 1622 struct kvm_run *run = vcpu->run; 1623 u32 *opc = (u32 *) vcpu->arch.pc; 1624 u32 cause = vcpu->arch.host_cp0_cause; 1625 ulong badvaddr = vcpu->arch.host_cp0_badvaddr; 1626 union mips_instruction inst; 1627 enum emulation_result er = EMULATE_DONE; 1628 int err, ret = RESUME_GUEST; 1629 1630 if (kvm_mips_handle_vz_root_tlb_fault(badvaddr, vcpu, false)) { 1631 /* A code fetch fault doesn't count as an MMIO */ 1632 if (kvm_is_ifetch_fault(&vcpu->arch)) { 1633 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1634 return RESUME_HOST; 1635 } 1636 1637 /* Fetch the instruction */ 1638 if (cause & CAUSEF_BD) 1639 opc += 1; 1640 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1641 if (err) { 1642 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1643 return RESUME_HOST; 1644 } 1645 1646 /* Treat as MMIO */ 1647 er = kvm_mips_emulate_load(inst, cause, vcpu); 1648 if (er == EMULATE_FAIL) { 1649 kvm_err("Guest Emulate Load from MMIO space failed: PC: %p, BadVaddr: %#lx\n", 1650 opc, badvaddr); 1651 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1652 } 1653 } 1654 1655 if (er == EMULATE_DONE) { 1656 ret = RESUME_GUEST; 1657 } else if (er == EMULATE_DO_MMIO) { 1658 run->exit_reason = KVM_EXIT_MMIO; 1659 ret = RESUME_HOST; 1660 } else { 1661 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1662 ret = RESUME_HOST; 1663 } 1664 return ret; 1665 } 1666 1667 static int kvm_trap_vz_handle_tlb_st_miss(struct kvm_vcpu *vcpu) 1668 { 1669 struct kvm_run *run = vcpu->run; 1670 u32 *opc = (u32 *) vcpu->arch.pc; 1671 u32 cause = vcpu->arch.host_cp0_cause; 1672 ulong badvaddr = vcpu->arch.host_cp0_badvaddr; 1673 union mips_instruction inst; 1674 enum emulation_result er = EMULATE_DONE; 1675 int err; 1676 int ret = RESUME_GUEST; 1677 1678 /* Just try the access again if we couldn't do the translation */ 1679 if (kvm_vz_badvaddr_to_gpa(vcpu, badvaddr, &badvaddr)) 1680 return RESUME_GUEST; 1681 vcpu->arch.host_cp0_badvaddr = badvaddr; 1682 1683 if (kvm_mips_handle_vz_root_tlb_fault(badvaddr, vcpu, true)) { 1684 /* Fetch the instruction */ 1685 if (cause & CAUSEF_BD) 1686 opc += 1; 1687 err = kvm_get_badinstr(opc, vcpu, &inst.word); 1688 if (err) { 1689 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1690 return RESUME_HOST; 1691 } 1692 1693 /* Treat as MMIO */ 1694 er = kvm_mips_emulate_store(inst, cause, vcpu); 1695 if (er == EMULATE_FAIL) { 1696 kvm_err("Guest Emulate Store to MMIO space failed: PC: %p, BadVaddr: %#lx\n", 1697 opc, badvaddr); 1698 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1699 } 1700 } 1701 1702 if (er == EMULATE_DONE) { 1703 ret = RESUME_GUEST; 1704 } else if (er == EMULATE_DO_MMIO) { 1705 run->exit_reason = KVM_EXIT_MMIO; 1706 ret = RESUME_HOST; 1707 } else { 1708 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1709 ret = RESUME_HOST; 1710 } 1711 return ret; 1712 } 1713 1714 static u64 kvm_vz_get_one_regs[] = { 1715 KVM_REG_MIPS_CP0_INDEX, 1716 KVM_REG_MIPS_CP0_ENTRYLO0, 1717 KVM_REG_MIPS_CP0_ENTRYLO1, 1718 KVM_REG_MIPS_CP0_CONTEXT, 1719 KVM_REG_MIPS_CP0_PAGEMASK, 1720 KVM_REG_MIPS_CP0_PAGEGRAIN, 1721 KVM_REG_MIPS_CP0_WIRED, 1722 KVM_REG_MIPS_CP0_HWRENA, 1723 KVM_REG_MIPS_CP0_BADVADDR, 1724 KVM_REG_MIPS_CP0_COUNT, 1725 KVM_REG_MIPS_CP0_ENTRYHI, 1726 KVM_REG_MIPS_CP0_COMPARE, 1727 KVM_REG_MIPS_CP0_STATUS, 1728 KVM_REG_MIPS_CP0_INTCTL, 1729 KVM_REG_MIPS_CP0_CAUSE, 1730 KVM_REG_MIPS_CP0_EPC, 1731 KVM_REG_MIPS_CP0_PRID, 1732 KVM_REG_MIPS_CP0_EBASE, 1733 KVM_REG_MIPS_CP0_CONFIG, 1734 KVM_REG_MIPS_CP0_CONFIG1, 1735 KVM_REG_MIPS_CP0_CONFIG2, 1736 KVM_REG_MIPS_CP0_CONFIG3, 1737 KVM_REG_MIPS_CP0_CONFIG4, 1738 KVM_REG_MIPS_CP0_CONFIG5, 1739 KVM_REG_MIPS_CP0_CONFIG6, 1740 #ifdef CONFIG_64BIT 1741 KVM_REG_MIPS_CP0_XCONTEXT, 1742 #endif 1743 KVM_REG_MIPS_CP0_ERROREPC, 1744 1745 KVM_REG_MIPS_COUNT_CTL, 1746 KVM_REG_MIPS_COUNT_RESUME, 1747 KVM_REG_MIPS_COUNT_HZ, 1748 }; 1749 1750 static u64 kvm_vz_get_one_regs_contextconfig[] = { 1751 KVM_REG_MIPS_CP0_CONTEXTCONFIG, 1752 #ifdef CONFIG_64BIT 1753 KVM_REG_MIPS_CP0_XCONTEXTCONFIG, 1754 #endif 1755 }; 1756 1757 static u64 kvm_vz_get_one_regs_segments[] = { 1758 KVM_REG_MIPS_CP0_SEGCTL0, 1759 KVM_REG_MIPS_CP0_SEGCTL1, 1760 KVM_REG_MIPS_CP0_SEGCTL2, 1761 }; 1762 1763 static u64 kvm_vz_get_one_regs_htw[] = { 1764 KVM_REG_MIPS_CP0_PWBASE, 1765 KVM_REG_MIPS_CP0_PWFIELD, 1766 KVM_REG_MIPS_CP0_PWSIZE, 1767 KVM_REG_MIPS_CP0_PWCTL, 1768 }; 1769 1770 static u64 kvm_vz_get_one_regs_kscratch[] = { 1771 KVM_REG_MIPS_CP0_KSCRATCH1, 1772 KVM_REG_MIPS_CP0_KSCRATCH2, 1773 KVM_REG_MIPS_CP0_KSCRATCH3, 1774 KVM_REG_MIPS_CP0_KSCRATCH4, 1775 KVM_REG_MIPS_CP0_KSCRATCH5, 1776 KVM_REG_MIPS_CP0_KSCRATCH6, 1777 }; 1778 1779 static unsigned long kvm_vz_num_regs(struct kvm_vcpu *vcpu) 1780 { 1781 unsigned long ret; 1782 1783 ret = ARRAY_SIZE(kvm_vz_get_one_regs); 1784 if (cpu_guest_has_userlocal) 1785 ++ret; 1786 if (cpu_guest_has_badinstr) 1787 ++ret; 1788 if (cpu_guest_has_badinstrp) 1789 ++ret; 1790 if (cpu_guest_has_contextconfig) 1791 ret += ARRAY_SIZE(kvm_vz_get_one_regs_contextconfig); 1792 if (cpu_guest_has_segments) 1793 ret += ARRAY_SIZE(kvm_vz_get_one_regs_segments); 1794 if (cpu_guest_has_htw || cpu_guest_has_ldpte) 1795 ret += ARRAY_SIZE(kvm_vz_get_one_regs_htw); 1796 if (cpu_guest_has_maar && !cpu_guest_has_dyn_maar) 1797 ret += 1 + ARRAY_SIZE(vcpu->arch.maar); 1798 ret += __arch_hweight8(cpu_data[0].guest.kscratch_mask); 1799 1800 return ret; 1801 } 1802 1803 static int kvm_vz_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices) 1804 { 1805 u64 index; 1806 unsigned int i; 1807 1808 if (copy_to_user(indices, kvm_vz_get_one_regs, 1809 sizeof(kvm_vz_get_one_regs))) 1810 return -EFAULT; 1811 indices += ARRAY_SIZE(kvm_vz_get_one_regs); 1812 1813 if (cpu_guest_has_userlocal) { 1814 index = KVM_REG_MIPS_CP0_USERLOCAL; 1815 if (copy_to_user(indices, &index, sizeof(index))) 1816 return -EFAULT; 1817 ++indices; 1818 } 1819 if (cpu_guest_has_badinstr) { 1820 index = KVM_REG_MIPS_CP0_BADINSTR; 1821 if (copy_to_user(indices, &index, sizeof(index))) 1822 return -EFAULT; 1823 ++indices; 1824 } 1825 if (cpu_guest_has_badinstrp) { 1826 index = KVM_REG_MIPS_CP0_BADINSTRP; 1827 if (copy_to_user(indices, &index, sizeof(index))) 1828 return -EFAULT; 1829 ++indices; 1830 } 1831 if (cpu_guest_has_contextconfig) { 1832 if (copy_to_user(indices, kvm_vz_get_one_regs_contextconfig, 1833 sizeof(kvm_vz_get_one_regs_contextconfig))) 1834 return -EFAULT; 1835 indices += ARRAY_SIZE(kvm_vz_get_one_regs_contextconfig); 1836 } 1837 if (cpu_guest_has_segments) { 1838 if (copy_to_user(indices, kvm_vz_get_one_regs_segments, 1839 sizeof(kvm_vz_get_one_regs_segments))) 1840 return -EFAULT; 1841 indices += ARRAY_SIZE(kvm_vz_get_one_regs_segments); 1842 } 1843 if (cpu_guest_has_htw || cpu_guest_has_ldpte) { 1844 if (copy_to_user(indices, kvm_vz_get_one_regs_htw, 1845 sizeof(kvm_vz_get_one_regs_htw))) 1846 return -EFAULT; 1847 indices += ARRAY_SIZE(kvm_vz_get_one_regs_htw); 1848 } 1849 if (cpu_guest_has_maar && !cpu_guest_has_dyn_maar) { 1850 for (i = 0; i < ARRAY_SIZE(vcpu->arch.maar); ++i) { 1851 index = KVM_REG_MIPS_CP0_MAAR(i); 1852 if (copy_to_user(indices, &index, sizeof(index))) 1853 return -EFAULT; 1854 ++indices; 1855 } 1856 1857 index = KVM_REG_MIPS_CP0_MAARI; 1858 if (copy_to_user(indices, &index, sizeof(index))) 1859 return -EFAULT; 1860 ++indices; 1861 } 1862 for (i = 0; i < 6; ++i) { 1863 if (!cpu_guest_has_kscr(i + 2)) 1864 continue; 1865 1866 if (copy_to_user(indices, &kvm_vz_get_one_regs_kscratch[i], 1867 sizeof(kvm_vz_get_one_regs_kscratch[i]))) 1868 return -EFAULT; 1869 ++indices; 1870 } 1871 1872 return 0; 1873 } 1874 1875 static inline s64 entrylo_kvm_to_user(unsigned long v) 1876 { 1877 s64 mask, ret = v; 1878 1879 if (BITS_PER_LONG == 32) { 1880 /* 1881 * KVM API exposes 64-bit version of the register, so move the 1882 * RI/XI bits up into place. 1883 */ 1884 mask = MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI; 1885 ret &= ~mask; 1886 ret |= ((s64)v & mask) << 32; 1887 } 1888 return ret; 1889 } 1890 1891 static inline unsigned long entrylo_user_to_kvm(s64 v) 1892 { 1893 unsigned long mask, ret = v; 1894 1895 if (BITS_PER_LONG == 32) { 1896 /* 1897 * KVM API exposes 64-bit versiono of the register, so move the 1898 * RI/XI bits down into place. 1899 */ 1900 mask = MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI; 1901 ret &= ~mask; 1902 ret |= (v >> 32) & mask; 1903 } 1904 return ret; 1905 } 1906 1907 static int kvm_vz_get_one_reg(struct kvm_vcpu *vcpu, 1908 const struct kvm_one_reg *reg, 1909 s64 *v) 1910 { 1911 struct mips_coproc *cop0 = vcpu->arch.cop0; 1912 unsigned int idx; 1913 1914 switch (reg->id) { 1915 case KVM_REG_MIPS_CP0_INDEX: 1916 *v = (long)read_gc0_index(); 1917 break; 1918 case KVM_REG_MIPS_CP0_ENTRYLO0: 1919 *v = entrylo_kvm_to_user(read_gc0_entrylo0()); 1920 break; 1921 case KVM_REG_MIPS_CP0_ENTRYLO1: 1922 *v = entrylo_kvm_to_user(read_gc0_entrylo1()); 1923 break; 1924 case KVM_REG_MIPS_CP0_CONTEXT: 1925 *v = (long)read_gc0_context(); 1926 break; 1927 case KVM_REG_MIPS_CP0_CONTEXTCONFIG: 1928 if (!cpu_guest_has_contextconfig) 1929 return -EINVAL; 1930 *v = read_gc0_contextconfig(); 1931 break; 1932 case KVM_REG_MIPS_CP0_USERLOCAL: 1933 if (!cpu_guest_has_userlocal) 1934 return -EINVAL; 1935 *v = read_gc0_userlocal(); 1936 break; 1937 #ifdef CONFIG_64BIT 1938 case KVM_REG_MIPS_CP0_XCONTEXTCONFIG: 1939 if (!cpu_guest_has_contextconfig) 1940 return -EINVAL; 1941 *v = read_gc0_xcontextconfig(); 1942 break; 1943 #endif 1944 case KVM_REG_MIPS_CP0_PAGEMASK: 1945 *v = (long)read_gc0_pagemask(); 1946 break; 1947 case KVM_REG_MIPS_CP0_PAGEGRAIN: 1948 *v = (long)read_gc0_pagegrain(); 1949 break; 1950 case KVM_REG_MIPS_CP0_SEGCTL0: 1951 if (!cpu_guest_has_segments) 1952 return -EINVAL; 1953 *v = read_gc0_segctl0(); 1954 break; 1955 case KVM_REG_MIPS_CP0_SEGCTL1: 1956 if (!cpu_guest_has_segments) 1957 return -EINVAL; 1958 *v = read_gc0_segctl1(); 1959 break; 1960 case KVM_REG_MIPS_CP0_SEGCTL2: 1961 if (!cpu_guest_has_segments) 1962 return -EINVAL; 1963 *v = read_gc0_segctl2(); 1964 break; 1965 case KVM_REG_MIPS_CP0_PWBASE: 1966 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 1967 return -EINVAL; 1968 *v = read_gc0_pwbase(); 1969 break; 1970 case KVM_REG_MIPS_CP0_PWFIELD: 1971 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 1972 return -EINVAL; 1973 *v = read_gc0_pwfield(); 1974 break; 1975 case KVM_REG_MIPS_CP0_PWSIZE: 1976 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 1977 return -EINVAL; 1978 *v = read_gc0_pwsize(); 1979 break; 1980 case KVM_REG_MIPS_CP0_WIRED: 1981 *v = (long)read_gc0_wired(); 1982 break; 1983 case KVM_REG_MIPS_CP0_PWCTL: 1984 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 1985 return -EINVAL; 1986 *v = read_gc0_pwctl(); 1987 break; 1988 case KVM_REG_MIPS_CP0_HWRENA: 1989 *v = (long)read_gc0_hwrena(); 1990 break; 1991 case KVM_REG_MIPS_CP0_BADVADDR: 1992 *v = (long)read_gc0_badvaddr(); 1993 break; 1994 case KVM_REG_MIPS_CP0_BADINSTR: 1995 if (!cpu_guest_has_badinstr) 1996 return -EINVAL; 1997 *v = read_gc0_badinstr(); 1998 break; 1999 case KVM_REG_MIPS_CP0_BADINSTRP: 2000 if (!cpu_guest_has_badinstrp) 2001 return -EINVAL; 2002 *v = read_gc0_badinstrp(); 2003 break; 2004 case KVM_REG_MIPS_CP0_COUNT: 2005 *v = kvm_mips_read_count(vcpu); 2006 break; 2007 case KVM_REG_MIPS_CP0_ENTRYHI: 2008 *v = (long)read_gc0_entryhi(); 2009 break; 2010 case KVM_REG_MIPS_CP0_COMPARE: 2011 *v = (long)read_gc0_compare(); 2012 break; 2013 case KVM_REG_MIPS_CP0_STATUS: 2014 *v = (long)read_gc0_status(); 2015 break; 2016 case KVM_REG_MIPS_CP0_INTCTL: 2017 *v = read_gc0_intctl(); 2018 break; 2019 case KVM_REG_MIPS_CP0_CAUSE: 2020 *v = (long)read_gc0_cause(); 2021 break; 2022 case KVM_REG_MIPS_CP0_EPC: 2023 *v = (long)read_gc0_epc(); 2024 break; 2025 case KVM_REG_MIPS_CP0_PRID: 2026 switch (boot_cpu_type()) { 2027 case CPU_CAVIUM_OCTEON3: 2028 /* Octeon III has a read-only guest.PRid */ 2029 *v = read_gc0_prid(); 2030 break; 2031 default: 2032 *v = (long)kvm_read_c0_guest_prid(cop0); 2033 break; 2034 } 2035 break; 2036 case KVM_REG_MIPS_CP0_EBASE: 2037 *v = kvm_vz_read_gc0_ebase(); 2038 break; 2039 case KVM_REG_MIPS_CP0_CONFIG: 2040 *v = read_gc0_config(); 2041 break; 2042 case KVM_REG_MIPS_CP0_CONFIG1: 2043 if (!cpu_guest_has_conf1) 2044 return -EINVAL; 2045 *v = read_gc0_config1(); 2046 break; 2047 case KVM_REG_MIPS_CP0_CONFIG2: 2048 if (!cpu_guest_has_conf2) 2049 return -EINVAL; 2050 *v = read_gc0_config2(); 2051 break; 2052 case KVM_REG_MIPS_CP0_CONFIG3: 2053 if (!cpu_guest_has_conf3) 2054 return -EINVAL; 2055 *v = read_gc0_config3(); 2056 break; 2057 case KVM_REG_MIPS_CP0_CONFIG4: 2058 if (!cpu_guest_has_conf4) 2059 return -EINVAL; 2060 *v = read_gc0_config4(); 2061 break; 2062 case KVM_REG_MIPS_CP0_CONFIG5: 2063 if (!cpu_guest_has_conf5) 2064 return -EINVAL; 2065 *v = read_gc0_config5(); 2066 break; 2067 case KVM_REG_MIPS_CP0_CONFIG6: 2068 *v = kvm_read_sw_gc0_config6(cop0); 2069 break; 2070 case KVM_REG_MIPS_CP0_MAAR(0) ... KVM_REG_MIPS_CP0_MAAR(0x3f): 2071 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar) 2072 return -EINVAL; 2073 idx = reg->id - KVM_REG_MIPS_CP0_MAAR(0); 2074 if (idx >= ARRAY_SIZE(vcpu->arch.maar)) 2075 return -EINVAL; 2076 *v = vcpu->arch.maar[idx]; 2077 break; 2078 case KVM_REG_MIPS_CP0_MAARI: 2079 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar) 2080 return -EINVAL; 2081 *v = kvm_read_sw_gc0_maari(vcpu->arch.cop0); 2082 break; 2083 #ifdef CONFIG_64BIT 2084 case KVM_REG_MIPS_CP0_XCONTEXT: 2085 *v = read_gc0_xcontext(); 2086 break; 2087 #endif 2088 case KVM_REG_MIPS_CP0_ERROREPC: 2089 *v = (long)read_gc0_errorepc(); 2090 break; 2091 case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6: 2092 idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2; 2093 if (!cpu_guest_has_kscr(idx)) 2094 return -EINVAL; 2095 switch (idx) { 2096 case 2: 2097 *v = (long)read_gc0_kscratch1(); 2098 break; 2099 case 3: 2100 *v = (long)read_gc0_kscratch2(); 2101 break; 2102 case 4: 2103 *v = (long)read_gc0_kscratch3(); 2104 break; 2105 case 5: 2106 *v = (long)read_gc0_kscratch4(); 2107 break; 2108 case 6: 2109 *v = (long)read_gc0_kscratch5(); 2110 break; 2111 case 7: 2112 *v = (long)read_gc0_kscratch6(); 2113 break; 2114 } 2115 break; 2116 case KVM_REG_MIPS_COUNT_CTL: 2117 *v = vcpu->arch.count_ctl; 2118 break; 2119 case KVM_REG_MIPS_COUNT_RESUME: 2120 *v = ktime_to_ns(vcpu->arch.count_resume); 2121 break; 2122 case KVM_REG_MIPS_COUNT_HZ: 2123 *v = vcpu->arch.count_hz; 2124 break; 2125 default: 2126 return -EINVAL; 2127 } 2128 return 0; 2129 } 2130 2131 static int kvm_vz_set_one_reg(struct kvm_vcpu *vcpu, 2132 const struct kvm_one_reg *reg, 2133 s64 v) 2134 { 2135 struct mips_coproc *cop0 = vcpu->arch.cop0; 2136 unsigned int idx; 2137 int ret = 0; 2138 unsigned int cur, change; 2139 2140 switch (reg->id) { 2141 case KVM_REG_MIPS_CP0_INDEX: 2142 write_gc0_index(v); 2143 break; 2144 case KVM_REG_MIPS_CP0_ENTRYLO0: 2145 write_gc0_entrylo0(entrylo_user_to_kvm(v)); 2146 break; 2147 case KVM_REG_MIPS_CP0_ENTRYLO1: 2148 write_gc0_entrylo1(entrylo_user_to_kvm(v)); 2149 break; 2150 case KVM_REG_MIPS_CP0_CONTEXT: 2151 write_gc0_context(v); 2152 break; 2153 case KVM_REG_MIPS_CP0_CONTEXTCONFIG: 2154 if (!cpu_guest_has_contextconfig) 2155 return -EINVAL; 2156 write_gc0_contextconfig(v); 2157 break; 2158 case KVM_REG_MIPS_CP0_USERLOCAL: 2159 if (!cpu_guest_has_userlocal) 2160 return -EINVAL; 2161 write_gc0_userlocal(v); 2162 break; 2163 #ifdef CONFIG_64BIT 2164 case KVM_REG_MIPS_CP0_XCONTEXTCONFIG: 2165 if (!cpu_guest_has_contextconfig) 2166 return -EINVAL; 2167 write_gc0_xcontextconfig(v); 2168 break; 2169 #endif 2170 case KVM_REG_MIPS_CP0_PAGEMASK: 2171 write_gc0_pagemask(v); 2172 break; 2173 case KVM_REG_MIPS_CP0_PAGEGRAIN: 2174 write_gc0_pagegrain(v); 2175 break; 2176 case KVM_REG_MIPS_CP0_SEGCTL0: 2177 if (!cpu_guest_has_segments) 2178 return -EINVAL; 2179 write_gc0_segctl0(v); 2180 break; 2181 case KVM_REG_MIPS_CP0_SEGCTL1: 2182 if (!cpu_guest_has_segments) 2183 return -EINVAL; 2184 write_gc0_segctl1(v); 2185 break; 2186 case KVM_REG_MIPS_CP0_SEGCTL2: 2187 if (!cpu_guest_has_segments) 2188 return -EINVAL; 2189 write_gc0_segctl2(v); 2190 break; 2191 case KVM_REG_MIPS_CP0_PWBASE: 2192 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 2193 return -EINVAL; 2194 write_gc0_pwbase(v); 2195 break; 2196 case KVM_REG_MIPS_CP0_PWFIELD: 2197 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 2198 return -EINVAL; 2199 write_gc0_pwfield(v); 2200 break; 2201 case KVM_REG_MIPS_CP0_PWSIZE: 2202 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 2203 return -EINVAL; 2204 write_gc0_pwsize(v); 2205 break; 2206 case KVM_REG_MIPS_CP0_WIRED: 2207 change_gc0_wired(MIPSR6_WIRED_WIRED, v); 2208 break; 2209 case KVM_REG_MIPS_CP0_PWCTL: 2210 if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) 2211 return -EINVAL; 2212 write_gc0_pwctl(v); 2213 break; 2214 case KVM_REG_MIPS_CP0_HWRENA: 2215 write_gc0_hwrena(v); 2216 break; 2217 case KVM_REG_MIPS_CP0_BADVADDR: 2218 write_gc0_badvaddr(v); 2219 break; 2220 case KVM_REG_MIPS_CP0_BADINSTR: 2221 if (!cpu_guest_has_badinstr) 2222 return -EINVAL; 2223 write_gc0_badinstr(v); 2224 break; 2225 case KVM_REG_MIPS_CP0_BADINSTRP: 2226 if (!cpu_guest_has_badinstrp) 2227 return -EINVAL; 2228 write_gc0_badinstrp(v); 2229 break; 2230 case KVM_REG_MIPS_CP0_COUNT: 2231 kvm_mips_write_count(vcpu, v); 2232 break; 2233 case KVM_REG_MIPS_CP0_ENTRYHI: 2234 write_gc0_entryhi(v); 2235 break; 2236 case KVM_REG_MIPS_CP0_COMPARE: 2237 kvm_mips_write_compare(vcpu, v, false); 2238 break; 2239 case KVM_REG_MIPS_CP0_STATUS: 2240 write_gc0_status(v); 2241 break; 2242 case KVM_REG_MIPS_CP0_INTCTL: 2243 write_gc0_intctl(v); 2244 break; 2245 case KVM_REG_MIPS_CP0_CAUSE: 2246 /* 2247 * If the timer is stopped or started (DC bit) it must look 2248 * atomic with changes to the timer interrupt pending bit (TI). 2249 * A timer interrupt should not happen in between. 2250 */ 2251 if ((read_gc0_cause() ^ v) & CAUSEF_DC) { 2252 if (v & CAUSEF_DC) { 2253 /* disable timer first */ 2254 kvm_mips_count_disable_cause(vcpu); 2255 change_gc0_cause((u32)~CAUSEF_DC, v); 2256 } else { 2257 /* enable timer last */ 2258 change_gc0_cause((u32)~CAUSEF_DC, v); 2259 kvm_mips_count_enable_cause(vcpu); 2260 } 2261 } else { 2262 write_gc0_cause(v); 2263 } 2264 break; 2265 case KVM_REG_MIPS_CP0_EPC: 2266 write_gc0_epc(v); 2267 break; 2268 case KVM_REG_MIPS_CP0_PRID: 2269 switch (boot_cpu_type()) { 2270 case CPU_CAVIUM_OCTEON3: 2271 /* Octeon III has a guest.PRid, but its read-only */ 2272 break; 2273 default: 2274 kvm_write_c0_guest_prid(cop0, v); 2275 break; 2276 } 2277 break; 2278 case KVM_REG_MIPS_CP0_EBASE: 2279 kvm_vz_write_gc0_ebase(v); 2280 break; 2281 case KVM_REG_MIPS_CP0_CONFIG: 2282 cur = read_gc0_config(); 2283 change = (cur ^ v) & kvm_vz_config_user_wrmask(vcpu); 2284 if (change) { 2285 v = cur ^ change; 2286 write_gc0_config(v); 2287 } 2288 break; 2289 case KVM_REG_MIPS_CP0_CONFIG1: 2290 if (!cpu_guest_has_conf1) 2291 break; 2292 cur = read_gc0_config1(); 2293 change = (cur ^ v) & kvm_vz_config1_user_wrmask(vcpu); 2294 if (change) { 2295 v = cur ^ change; 2296 write_gc0_config1(v); 2297 } 2298 break; 2299 case KVM_REG_MIPS_CP0_CONFIG2: 2300 if (!cpu_guest_has_conf2) 2301 break; 2302 cur = read_gc0_config2(); 2303 change = (cur ^ v) & kvm_vz_config2_user_wrmask(vcpu); 2304 if (change) { 2305 v = cur ^ change; 2306 write_gc0_config2(v); 2307 } 2308 break; 2309 case KVM_REG_MIPS_CP0_CONFIG3: 2310 if (!cpu_guest_has_conf3) 2311 break; 2312 cur = read_gc0_config3(); 2313 change = (cur ^ v) & kvm_vz_config3_user_wrmask(vcpu); 2314 if (change) { 2315 v = cur ^ change; 2316 write_gc0_config3(v); 2317 } 2318 break; 2319 case KVM_REG_MIPS_CP0_CONFIG4: 2320 if (!cpu_guest_has_conf4) 2321 break; 2322 cur = read_gc0_config4(); 2323 change = (cur ^ v) & kvm_vz_config4_user_wrmask(vcpu); 2324 if (change) { 2325 v = cur ^ change; 2326 write_gc0_config4(v); 2327 } 2328 break; 2329 case KVM_REG_MIPS_CP0_CONFIG5: 2330 if (!cpu_guest_has_conf5) 2331 break; 2332 cur = read_gc0_config5(); 2333 change = (cur ^ v) & kvm_vz_config5_user_wrmask(vcpu); 2334 if (change) { 2335 v = cur ^ change; 2336 write_gc0_config5(v); 2337 } 2338 break; 2339 case KVM_REG_MIPS_CP0_CONFIG6: 2340 cur = kvm_read_sw_gc0_config6(cop0); 2341 change = (cur ^ v) & kvm_vz_config6_user_wrmask(vcpu); 2342 if (change) { 2343 v = cur ^ change; 2344 kvm_write_sw_gc0_config6(cop0, (int)v); 2345 } 2346 break; 2347 case KVM_REG_MIPS_CP0_MAAR(0) ... KVM_REG_MIPS_CP0_MAAR(0x3f): 2348 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar) 2349 return -EINVAL; 2350 idx = reg->id - KVM_REG_MIPS_CP0_MAAR(0); 2351 if (idx >= ARRAY_SIZE(vcpu->arch.maar)) 2352 return -EINVAL; 2353 vcpu->arch.maar[idx] = mips_process_maar(dmtc_op, v); 2354 break; 2355 case KVM_REG_MIPS_CP0_MAARI: 2356 if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar) 2357 return -EINVAL; 2358 kvm_write_maari(vcpu, v); 2359 break; 2360 #ifdef CONFIG_64BIT 2361 case KVM_REG_MIPS_CP0_XCONTEXT: 2362 write_gc0_xcontext(v); 2363 break; 2364 #endif 2365 case KVM_REG_MIPS_CP0_ERROREPC: 2366 write_gc0_errorepc(v); 2367 break; 2368 case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6: 2369 idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2; 2370 if (!cpu_guest_has_kscr(idx)) 2371 return -EINVAL; 2372 switch (idx) { 2373 case 2: 2374 write_gc0_kscratch1(v); 2375 break; 2376 case 3: 2377 write_gc0_kscratch2(v); 2378 break; 2379 case 4: 2380 write_gc0_kscratch3(v); 2381 break; 2382 case 5: 2383 write_gc0_kscratch4(v); 2384 break; 2385 case 6: 2386 write_gc0_kscratch5(v); 2387 break; 2388 case 7: 2389 write_gc0_kscratch6(v); 2390 break; 2391 } 2392 break; 2393 case KVM_REG_MIPS_COUNT_CTL: 2394 ret = kvm_mips_set_count_ctl(vcpu, v); 2395 break; 2396 case KVM_REG_MIPS_COUNT_RESUME: 2397 ret = kvm_mips_set_count_resume(vcpu, v); 2398 break; 2399 case KVM_REG_MIPS_COUNT_HZ: 2400 ret = kvm_mips_set_count_hz(vcpu, v); 2401 break; 2402 default: 2403 return -EINVAL; 2404 } 2405 return ret; 2406 } 2407 2408 #define guestid_cache(cpu) (cpu_data[cpu].guestid_cache) 2409 static void kvm_vz_get_new_guestid(unsigned long cpu, struct kvm_vcpu *vcpu) 2410 { 2411 unsigned long guestid = guestid_cache(cpu); 2412 2413 if (!(++guestid & GUESTID_MASK)) { 2414 if (cpu_has_vtag_icache) 2415 flush_icache_all(); 2416 2417 if (!guestid) /* fix version if needed */ 2418 guestid = GUESTID_FIRST_VERSION; 2419 2420 ++guestid; /* guestid 0 reserved for root */ 2421 2422 /* start new guestid cycle */ 2423 kvm_vz_local_flush_roottlb_all_guests(); 2424 kvm_vz_local_flush_guesttlb_all(); 2425 } 2426 2427 guestid_cache(cpu) = guestid; 2428 } 2429 2430 /* Returns 1 if the guest TLB may be clobbered */ 2431 static int kvm_vz_check_requests(struct kvm_vcpu *vcpu, int cpu) 2432 { 2433 int ret = 0; 2434 int i; 2435 2436 if (!kvm_request_pending(vcpu)) 2437 return 0; 2438 2439 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { 2440 if (cpu_has_guestid) { 2441 /* Drop all GuestIDs for this VCPU */ 2442 for_each_possible_cpu(i) 2443 vcpu->arch.vzguestid[i] = 0; 2444 /* This will clobber guest TLB contents too */ 2445 ret = 1; 2446 } 2447 /* 2448 * For Root ASID Dealias (RAD) we don't do anything here, but we 2449 * still need the request to ensure we recheck asid_flush_mask. 2450 * We can still return 0 as only the root TLB will be affected 2451 * by a root ASID flush. 2452 */ 2453 } 2454 2455 return ret; 2456 } 2457 2458 static void kvm_vz_vcpu_save_wired(struct kvm_vcpu *vcpu) 2459 { 2460 unsigned int wired = read_gc0_wired(); 2461 struct kvm_mips_tlb *tlbs; 2462 int i; 2463 2464 /* Expand the wired TLB array if necessary */ 2465 wired &= MIPSR6_WIRED_WIRED; 2466 if (wired > vcpu->arch.wired_tlb_limit) { 2467 tlbs = krealloc(vcpu->arch.wired_tlb, wired * 2468 sizeof(*vcpu->arch.wired_tlb), GFP_ATOMIC); 2469 if (WARN_ON(!tlbs)) { 2470 /* Save whatever we can */ 2471 wired = vcpu->arch.wired_tlb_limit; 2472 } else { 2473 vcpu->arch.wired_tlb = tlbs; 2474 vcpu->arch.wired_tlb_limit = wired; 2475 } 2476 } 2477 2478 if (wired) 2479 /* Save wired entries from the guest TLB */ 2480 kvm_vz_save_guesttlb(vcpu->arch.wired_tlb, 0, wired); 2481 /* Invalidate any dropped entries since last time */ 2482 for (i = wired; i < vcpu->arch.wired_tlb_used; ++i) { 2483 vcpu->arch.wired_tlb[i].tlb_hi = UNIQUE_GUEST_ENTRYHI(i); 2484 vcpu->arch.wired_tlb[i].tlb_lo[0] = 0; 2485 vcpu->arch.wired_tlb[i].tlb_lo[1] = 0; 2486 vcpu->arch.wired_tlb[i].tlb_mask = 0; 2487 } 2488 vcpu->arch.wired_tlb_used = wired; 2489 } 2490 2491 static void kvm_vz_vcpu_load_wired(struct kvm_vcpu *vcpu) 2492 { 2493 /* Load wired entries into the guest TLB */ 2494 if (vcpu->arch.wired_tlb) 2495 kvm_vz_load_guesttlb(vcpu->arch.wired_tlb, 0, 2496 vcpu->arch.wired_tlb_used); 2497 } 2498 2499 static void kvm_vz_vcpu_load_tlb(struct kvm_vcpu *vcpu, int cpu) 2500 { 2501 struct kvm *kvm = vcpu->kvm; 2502 struct mm_struct *gpa_mm = &kvm->arch.gpa_mm; 2503 bool migrated; 2504 2505 /* 2506 * Are we entering guest context on a different CPU to last time? 2507 * If so, the VCPU's guest TLB state on this CPU may be stale. 2508 */ 2509 migrated = (vcpu->arch.last_exec_cpu != cpu); 2510 vcpu->arch.last_exec_cpu = cpu; 2511 2512 /* 2513 * A vcpu's GuestID is set in GuestCtl1.ID when the vcpu is loaded and 2514 * remains set until another vcpu is loaded in. As a rule GuestRID 2515 * remains zeroed when in root context unless the kernel is busy 2516 * manipulating guest tlb entries. 2517 */ 2518 if (cpu_has_guestid) { 2519 /* 2520 * Check if our GuestID is of an older version and thus invalid. 2521 * 2522 * We also discard the stored GuestID if we've executed on 2523 * another CPU, as the guest mappings may have changed without 2524 * hypervisor knowledge. 2525 */ 2526 if (migrated || 2527 (vcpu->arch.vzguestid[cpu] ^ guestid_cache(cpu)) & 2528 GUESTID_VERSION_MASK) { 2529 kvm_vz_get_new_guestid(cpu, vcpu); 2530 vcpu->arch.vzguestid[cpu] = guestid_cache(cpu); 2531 trace_kvm_guestid_change(vcpu, 2532 vcpu->arch.vzguestid[cpu]); 2533 } 2534 2535 /* Restore GuestID */ 2536 change_c0_guestctl1(GUESTID_MASK, vcpu->arch.vzguestid[cpu]); 2537 } else { 2538 /* 2539 * The Guest TLB only stores a single guest's TLB state, so 2540 * flush it if another VCPU has executed on this CPU. 2541 * 2542 * We also flush if we've executed on another CPU, as the guest 2543 * mappings may have changed without hypervisor knowledge. 2544 */ 2545 if (migrated || last_exec_vcpu[cpu] != vcpu) 2546 kvm_vz_local_flush_guesttlb_all(); 2547 last_exec_vcpu[cpu] = vcpu; 2548 2549 /* 2550 * Root ASID dealiases guest GPA mappings in the root TLB. 2551 * Allocate new root ASID if needed. 2552 */ 2553 if (cpumask_test_and_clear_cpu(cpu, &kvm->arch.asid_flush_mask)) 2554 get_new_mmu_context(gpa_mm); 2555 else 2556 check_mmu_context(gpa_mm); 2557 } 2558 } 2559 2560 static int kvm_vz_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 2561 { 2562 struct mips_coproc *cop0 = vcpu->arch.cop0; 2563 bool migrated, all; 2564 2565 /* 2566 * Have we migrated to a different CPU? 2567 * If so, any old guest TLB state may be stale. 2568 */ 2569 migrated = (vcpu->arch.last_sched_cpu != cpu); 2570 2571 /* 2572 * Was this the last VCPU to run on this CPU? 2573 * If not, any old guest state from this VCPU will have been clobbered. 2574 */ 2575 all = migrated || (last_vcpu[cpu] != vcpu); 2576 last_vcpu[cpu] = vcpu; 2577 2578 /* 2579 * Restore CP0_Wired unconditionally as we clear it after use, and 2580 * restore wired guest TLB entries (while in guest context). 2581 */ 2582 kvm_restore_gc0_wired(cop0); 2583 if (current->flags & PF_VCPU) { 2584 tlbw_use_hazard(); 2585 kvm_vz_vcpu_load_tlb(vcpu, cpu); 2586 kvm_vz_vcpu_load_wired(vcpu); 2587 } 2588 2589 /* 2590 * Restore timer state regardless, as e.g. Cause.TI can change over time 2591 * if left unmaintained. 2592 */ 2593 kvm_vz_restore_timer(vcpu); 2594 2595 /* Set MC bit if we want to trace guest mode changes */ 2596 if (kvm_trace_guest_mode_change) 2597 set_c0_guestctl0(MIPS_GCTL0_MC); 2598 else 2599 clear_c0_guestctl0(MIPS_GCTL0_MC); 2600 2601 /* Don't bother restoring registers multiple times unless necessary */ 2602 if (!all) 2603 return 0; 2604 2605 /* 2606 * Restore config registers first, as some implementations restrict 2607 * writes to other registers when the corresponding feature bits aren't 2608 * set. For example Status.CU1 cannot be set unless Config1.FP is set. 2609 */ 2610 kvm_restore_gc0_config(cop0); 2611 if (cpu_guest_has_conf1) 2612 kvm_restore_gc0_config1(cop0); 2613 if (cpu_guest_has_conf2) 2614 kvm_restore_gc0_config2(cop0); 2615 if (cpu_guest_has_conf3) 2616 kvm_restore_gc0_config3(cop0); 2617 if (cpu_guest_has_conf4) 2618 kvm_restore_gc0_config4(cop0); 2619 if (cpu_guest_has_conf5) 2620 kvm_restore_gc0_config5(cop0); 2621 if (cpu_guest_has_conf6) 2622 kvm_restore_gc0_config6(cop0); 2623 if (cpu_guest_has_conf7) 2624 kvm_restore_gc0_config7(cop0); 2625 2626 kvm_restore_gc0_index(cop0); 2627 kvm_restore_gc0_entrylo0(cop0); 2628 kvm_restore_gc0_entrylo1(cop0); 2629 kvm_restore_gc0_context(cop0); 2630 if (cpu_guest_has_contextconfig) 2631 kvm_restore_gc0_contextconfig(cop0); 2632 #ifdef CONFIG_64BIT 2633 kvm_restore_gc0_xcontext(cop0); 2634 if (cpu_guest_has_contextconfig) 2635 kvm_restore_gc0_xcontextconfig(cop0); 2636 #endif 2637 kvm_restore_gc0_pagemask(cop0); 2638 kvm_restore_gc0_pagegrain(cop0); 2639 kvm_restore_gc0_hwrena(cop0); 2640 kvm_restore_gc0_badvaddr(cop0); 2641 kvm_restore_gc0_entryhi(cop0); 2642 kvm_restore_gc0_status(cop0); 2643 kvm_restore_gc0_intctl(cop0); 2644 kvm_restore_gc0_epc(cop0); 2645 kvm_vz_write_gc0_ebase(kvm_read_sw_gc0_ebase(cop0)); 2646 if (cpu_guest_has_userlocal) 2647 kvm_restore_gc0_userlocal(cop0); 2648 2649 kvm_restore_gc0_errorepc(cop0); 2650 2651 /* restore KScratch registers if enabled in guest */ 2652 if (cpu_guest_has_conf4) { 2653 if (cpu_guest_has_kscr(2)) 2654 kvm_restore_gc0_kscratch1(cop0); 2655 if (cpu_guest_has_kscr(3)) 2656 kvm_restore_gc0_kscratch2(cop0); 2657 if (cpu_guest_has_kscr(4)) 2658 kvm_restore_gc0_kscratch3(cop0); 2659 if (cpu_guest_has_kscr(5)) 2660 kvm_restore_gc0_kscratch4(cop0); 2661 if (cpu_guest_has_kscr(6)) 2662 kvm_restore_gc0_kscratch5(cop0); 2663 if (cpu_guest_has_kscr(7)) 2664 kvm_restore_gc0_kscratch6(cop0); 2665 } 2666 2667 if (cpu_guest_has_badinstr) 2668 kvm_restore_gc0_badinstr(cop0); 2669 if (cpu_guest_has_badinstrp) 2670 kvm_restore_gc0_badinstrp(cop0); 2671 2672 if (cpu_guest_has_segments) { 2673 kvm_restore_gc0_segctl0(cop0); 2674 kvm_restore_gc0_segctl1(cop0); 2675 kvm_restore_gc0_segctl2(cop0); 2676 } 2677 2678 /* restore HTW registers */ 2679 if (cpu_guest_has_htw || cpu_guest_has_ldpte) { 2680 kvm_restore_gc0_pwbase(cop0); 2681 kvm_restore_gc0_pwfield(cop0); 2682 kvm_restore_gc0_pwsize(cop0); 2683 kvm_restore_gc0_pwctl(cop0); 2684 } 2685 2686 /* restore Root.GuestCtl2 from unused Guest guestctl2 register */ 2687 if (cpu_has_guestctl2) 2688 write_c0_guestctl2( 2689 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL]); 2690 2691 /* 2692 * We should clear linked load bit to break interrupted atomics. This 2693 * prevents a SC on the next VCPU from succeeding by matching a LL on 2694 * the previous VCPU. 2695 */ 2696 if (vcpu->kvm->created_vcpus > 1) 2697 write_gc0_lladdr(0); 2698 2699 return 0; 2700 } 2701 2702 static int kvm_vz_vcpu_put(struct kvm_vcpu *vcpu, int cpu) 2703 { 2704 struct mips_coproc *cop0 = vcpu->arch.cop0; 2705 2706 if (current->flags & PF_VCPU) 2707 kvm_vz_vcpu_save_wired(vcpu); 2708 2709 kvm_lose_fpu(vcpu); 2710 2711 kvm_save_gc0_index(cop0); 2712 kvm_save_gc0_entrylo0(cop0); 2713 kvm_save_gc0_entrylo1(cop0); 2714 kvm_save_gc0_context(cop0); 2715 if (cpu_guest_has_contextconfig) 2716 kvm_save_gc0_contextconfig(cop0); 2717 #ifdef CONFIG_64BIT 2718 kvm_save_gc0_xcontext(cop0); 2719 if (cpu_guest_has_contextconfig) 2720 kvm_save_gc0_xcontextconfig(cop0); 2721 #endif 2722 kvm_save_gc0_pagemask(cop0); 2723 kvm_save_gc0_pagegrain(cop0); 2724 kvm_save_gc0_wired(cop0); 2725 /* allow wired TLB entries to be overwritten */ 2726 clear_gc0_wired(MIPSR6_WIRED_WIRED); 2727 kvm_save_gc0_hwrena(cop0); 2728 kvm_save_gc0_badvaddr(cop0); 2729 kvm_save_gc0_entryhi(cop0); 2730 kvm_save_gc0_status(cop0); 2731 kvm_save_gc0_intctl(cop0); 2732 kvm_save_gc0_epc(cop0); 2733 kvm_write_sw_gc0_ebase(cop0, kvm_vz_read_gc0_ebase()); 2734 if (cpu_guest_has_userlocal) 2735 kvm_save_gc0_userlocal(cop0); 2736 2737 /* only save implemented config registers */ 2738 kvm_save_gc0_config(cop0); 2739 if (cpu_guest_has_conf1) 2740 kvm_save_gc0_config1(cop0); 2741 if (cpu_guest_has_conf2) 2742 kvm_save_gc0_config2(cop0); 2743 if (cpu_guest_has_conf3) 2744 kvm_save_gc0_config3(cop0); 2745 if (cpu_guest_has_conf4) 2746 kvm_save_gc0_config4(cop0); 2747 if (cpu_guest_has_conf5) 2748 kvm_save_gc0_config5(cop0); 2749 if (cpu_guest_has_conf6) 2750 kvm_save_gc0_config6(cop0); 2751 if (cpu_guest_has_conf7) 2752 kvm_save_gc0_config7(cop0); 2753 2754 kvm_save_gc0_errorepc(cop0); 2755 2756 /* save KScratch registers if enabled in guest */ 2757 if (cpu_guest_has_conf4) { 2758 if (cpu_guest_has_kscr(2)) 2759 kvm_save_gc0_kscratch1(cop0); 2760 if (cpu_guest_has_kscr(3)) 2761 kvm_save_gc0_kscratch2(cop0); 2762 if (cpu_guest_has_kscr(4)) 2763 kvm_save_gc0_kscratch3(cop0); 2764 if (cpu_guest_has_kscr(5)) 2765 kvm_save_gc0_kscratch4(cop0); 2766 if (cpu_guest_has_kscr(6)) 2767 kvm_save_gc0_kscratch5(cop0); 2768 if (cpu_guest_has_kscr(7)) 2769 kvm_save_gc0_kscratch6(cop0); 2770 } 2771 2772 if (cpu_guest_has_badinstr) 2773 kvm_save_gc0_badinstr(cop0); 2774 if (cpu_guest_has_badinstrp) 2775 kvm_save_gc0_badinstrp(cop0); 2776 2777 if (cpu_guest_has_segments) { 2778 kvm_save_gc0_segctl0(cop0); 2779 kvm_save_gc0_segctl1(cop0); 2780 kvm_save_gc0_segctl2(cop0); 2781 } 2782 2783 /* save HTW registers if enabled in guest */ 2784 if (cpu_guest_has_ldpte || (cpu_guest_has_htw && 2785 kvm_read_sw_gc0_config3(cop0) & MIPS_CONF3_PW)) { 2786 kvm_save_gc0_pwbase(cop0); 2787 kvm_save_gc0_pwfield(cop0); 2788 kvm_save_gc0_pwsize(cop0); 2789 kvm_save_gc0_pwctl(cop0); 2790 } 2791 2792 kvm_vz_save_timer(vcpu); 2793 2794 /* save Root.GuestCtl2 in unused Guest guestctl2 register */ 2795 if (cpu_has_guestctl2) 2796 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL] = 2797 read_c0_guestctl2(); 2798 2799 return 0; 2800 } 2801 2802 /** 2803 * kvm_vz_resize_guest_vtlb() - Attempt to resize guest VTLB. 2804 * @size: Number of guest VTLB entries (0 < @size <= root VTLB entries). 2805 * 2806 * Attempt to resize the guest VTLB by writing guest Config registers. This is 2807 * necessary for cores with a shared root/guest TLB to avoid overlap with wired 2808 * entries in the root VTLB. 2809 * 2810 * Returns: The resulting guest VTLB size. 2811 */ 2812 static unsigned int kvm_vz_resize_guest_vtlb(unsigned int size) 2813 { 2814 unsigned int config4 = 0, ret = 0, limit; 2815 2816 /* Write MMUSize - 1 into guest Config registers */ 2817 if (cpu_guest_has_conf1) 2818 change_gc0_config1(MIPS_CONF1_TLBS, 2819 (size - 1) << MIPS_CONF1_TLBS_SHIFT); 2820 if (cpu_guest_has_conf4) { 2821 config4 = read_gc0_config4(); 2822 if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) == 2823 MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT) { 2824 config4 &= ~MIPS_CONF4_VTLBSIZEEXT; 2825 config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) << 2826 MIPS_CONF4_VTLBSIZEEXT_SHIFT; 2827 } else if ((config4 & MIPS_CONF4_MMUEXTDEF) == 2828 MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT) { 2829 config4 &= ~MIPS_CONF4_MMUSIZEEXT; 2830 config4 |= ((size - 1) >> MIPS_CONF1_TLBS_SIZE) << 2831 MIPS_CONF4_MMUSIZEEXT_SHIFT; 2832 } 2833 write_gc0_config4(config4); 2834 } 2835 2836 /* 2837 * Set Guest.Wired.Limit = 0 (no limit up to Guest.MMUSize-1), unless it 2838 * would exceed Root.Wired.Limit (clearing Guest.Wired.Wired so write 2839 * not dropped) 2840 */ 2841 if (cpu_has_mips_r6) { 2842 limit = (read_c0_wired() & MIPSR6_WIRED_LIMIT) >> 2843 MIPSR6_WIRED_LIMIT_SHIFT; 2844 if (size - 1 <= limit) 2845 limit = 0; 2846 write_gc0_wired(limit << MIPSR6_WIRED_LIMIT_SHIFT); 2847 } 2848 2849 /* Read back MMUSize - 1 */ 2850 back_to_back_c0_hazard(); 2851 if (cpu_guest_has_conf1) 2852 ret = (read_gc0_config1() & MIPS_CONF1_TLBS) >> 2853 MIPS_CONF1_TLBS_SHIFT; 2854 if (config4) { 2855 if (cpu_has_mips_r6 || (config4 & MIPS_CONF4_MMUEXTDEF) == 2856 MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT) 2857 ret |= ((config4 & MIPS_CONF4_VTLBSIZEEXT) >> 2858 MIPS_CONF4_VTLBSIZEEXT_SHIFT) << 2859 MIPS_CONF1_TLBS_SIZE; 2860 else if ((config4 & MIPS_CONF4_MMUEXTDEF) == 2861 MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT) 2862 ret |= ((config4 & MIPS_CONF4_MMUSIZEEXT) >> 2863 MIPS_CONF4_MMUSIZEEXT_SHIFT) << 2864 MIPS_CONF1_TLBS_SIZE; 2865 } 2866 return ret + 1; 2867 } 2868 2869 static int kvm_vz_hardware_enable(void) 2870 { 2871 unsigned int mmu_size, guest_mmu_size, ftlb_size; 2872 u64 guest_cvmctl, cvmvmconfig; 2873 2874 switch (current_cpu_type()) { 2875 case CPU_CAVIUM_OCTEON3: 2876 /* Set up guest timer/perfcount IRQ lines */ 2877 guest_cvmctl = read_gc0_cvmctl(); 2878 guest_cvmctl &= ~CVMCTL_IPTI; 2879 guest_cvmctl |= 7ull << CVMCTL_IPTI_SHIFT; 2880 guest_cvmctl &= ~CVMCTL_IPPCI; 2881 guest_cvmctl |= 6ull << CVMCTL_IPPCI_SHIFT; 2882 write_gc0_cvmctl(guest_cvmctl); 2883 2884 cvmvmconfig = read_c0_cvmvmconfig(); 2885 /* No I/O hole translation. */ 2886 cvmvmconfig |= CVMVMCONF_DGHT; 2887 /* Halve the root MMU size */ 2888 mmu_size = ((cvmvmconfig & CVMVMCONF_MMUSIZEM1) 2889 >> CVMVMCONF_MMUSIZEM1_S) + 1; 2890 guest_mmu_size = mmu_size / 2; 2891 mmu_size -= guest_mmu_size; 2892 cvmvmconfig &= ~CVMVMCONF_RMMUSIZEM1; 2893 cvmvmconfig |= mmu_size - 1; 2894 write_c0_cvmvmconfig(cvmvmconfig); 2895 2896 /* Update our records */ 2897 current_cpu_data.tlbsize = mmu_size; 2898 current_cpu_data.tlbsizevtlb = mmu_size; 2899 current_cpu_data.guest.tlbsize = guest_mmu_size; 2900 2901 /* Flush moved entries in new (guest) context */ 2902 kvm_vz_local_flush_guesttlb_all(); 2903 break; 2904 default: 2905 /* 2906 * ImgTec cores tend to use a shared root/guest TLB. To avoid 2907 * overlap of root wired and guest entries, the guest TLB may 2908 * need resizing. 2909 */ 2910 mmu_size = current_cpu_data.tlbsizevtlb; 2911 ftlb_size = current_cpu_data.tlbsize - mmu_size; 2912 2913 /* Try switching to maximum guest VTLB size for flush */ 2914 guest_mmu_size = kvm_vz_resize_guest_vtlb(mmu_size); 2915 current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size; 2916 kvm_vz_local_flush_guesttlb_all(); 2917 2918 /* 2919 * Reduce to make space for root wired entries and at least 2 2920 * root non-wired entries. This does assume that long-term wired 2921 * entries won't be added later. 2922 */ 2923 guest_mmu_size = mmu_size - num_wired_entries() - 2; 2924 guest_mmu_size = kvm_vz_resize_guest_vtlb(guest_mmu_size); 2925 current_cpu_data.guest.tlbsize = guest_mmu_size + ftlb_size; 2926 2927 /* 2928 * Write the VTLB size, but if another CPU has already written, 2929 * check it matches or we won't provide a consistent view to the 2930 * guest. If this ever happens it suggests an asymmetric number 2931 * of wired entries. 2932 */ 2933 if (cmpxchg(&kvm_vz_guest_vtlb_size, 0, guest_mmu_size) && 2934 WARN(guest_mmu_size != kvm_vz_guest_vtlb_size, 2935 "Available guest VTLB size mismatch")) 2936 return -EINVAL; 2937 break; 2938 } 2939 2940 /* 2941 * Enable virtualization features granting guest direct control of 2942 * certain features: 2943 * CP0=1: Guest coprocessor 0 context. 2944 * AT=Guest: Guest MMU. 2945 * CG=1: Hit (virtual address) CACHE operations (optional). 2946 * CF=1: Guest Config registers. 2947 * CGI=1: Indexed flush CACHE operations (optional). 2948 */ 2949 write_c0_guestctl0(MIPS_GCTL0_CP0 | 2950 (MIPS_GCTL0_AT_GUEST << MIPS_GCTL0_AT_SHIFT) | 2951 MIPS_GCTL0_CG | MIPS_GCTL0_CF); 2952 if (cpu_has_guestctl0ext) { 2953 if (current_cpu_type() != CPU_LOONGSON64) 2954 set_c0_guestctl0ext(MIPS_GCTL0EXT_CGI); 2955 else 2956 clear_c0_guestctl0ext(MIPS_GCTL0EXT_CGI); 2957 } 2958 2959 if (cpu_has_guestid) { 2960 write_c0_guestctl1(0); 2961 kvm_vz_local_flush_roottlb_all_guests(); 2962 2963 GUESTID_MASK = current_cpu_data.guestid_mask; 2964 GUESTID_FIRST_VERSION = GUESTID_MASK + 1; 2965 GUESTID_VERSION_MASK = ~GUESTID_MASK; 2966 2967 current_cpu_data.guestid_cache = GUESTID_FIRST_VERSION; 2968 } 2969 2970 /* clear any pending injected virtual guest interrupts */ 2971 if (cpu_has_guestctl2) 2972 clear_c0_guestctl2(0x3f << 10); 2973 2974 #ifdef CONFIG_CPU_LOONGSON64 2975 /* Control guest CCA attribute */ 2976 if (cpu_has_csr()) 2977 csr_writel(csr_readl(0xffffffec) | 0x1, 0xffffffec); 2978 #endif 2979 2980 return 0; 2981 } 2982 2983 static void kvm_vz_hardware_disable(void) 2984 { 2985 u64 cvmvmconfig; 2986 unsigned int mmu_size; 2987 2988 /* Flush any remaining guest TLB entries */ 2989 kvm_vz_local_flush_guesttlb_all(); 2990 2991 switch (current_cpu_type()) { 2992 case CPU_CAVIUM_OCTEON3: 2993 /* 2994 * Allocate whole TLB for root. Existing guest TLB entries will 2995 * change ownership to the root TLB. We should be safe though as 2996 * they've already been flushed above while in guest TLB. 2997 */ 2998 cvmvmconfig = read_c0_cvmvmconfig(); 2999 mmu_size = ((cvmvmconfig & CVMVMCONF_MMUSIZEM1) 3000 >> CVMVMCONF_MMUSIZEM1_S) + 1; 3001 cvmvmconfig &= ~CVMVMCONF_RMMUSIZEM1; 3002 cvmvmconfig |= mmu_size - 1; 3003 write_c0_cvmvmconfig(cvmvmconfig); 3004 3005 /* Update our records */ 3006 current_cpu_data.tlbsize = mmu_size; 3007 current_cpu_data.tlbsizevtlb = mmu_size; 3008 current_cpu_data.guest.tlbsize = 0; 3009 3010 /* Flush moved entries in new (root) context */ 3011 local_flush_tlb_all(); 3012 break; 3013 } 3014 3015 if (cpu_has_guestid) { 3016 write_c0_guestctl1(0); 3017 kvm_vz_local_flush_roottlb_all_guests(); 3018 } 3019 } 3020 3021 static int kvm_vz_check_extension(struct kvm *kvm, long ext) 3022 { 3023 int r; 3024 3025 switch (ext) { 3026 case KVM_CAP_MIPS_VZ: 3027 /* we wouldn't be here unless cpu_has_vz */ 3028 r = 1; 3029 break; 3030 #ifdef CONFIG_64BIT 3031 case KVM_CAP_MIPS_64BIT: 3032 /* We support 64-bit registers/operations and addresses */ 3033 r = 2; 3034 break; 3035 #endif 3036 case KVM_CAP_IOEVENTFD: 3037 r = 1; 3038 break; 3039 default: 3040 r = 0; 3041 break; 3042 } 3043 3044 return r; 3045 } 3046 3047 static int kvm_vz_vcpu_init(struct kvm_vcpu *vcpu) 3048 { 3049 int i; 3050 3051 for_each_possible_cpu(i) 3052 vcpu->arch.vzguestid[i] = 0; 3053 3054 return 0; 3055 } 3056 3057 static void kvm_vz_vcpu_uninit(struct kvm_vcpu *vcpu) 3058 { 3059 int cpu; 3060 3061 /* 3062 * If the VCPU is freed and reused as another VCPU, we don't want the 3063 * matching pointer wrongly hanging around in last_vcpu[] or 3064 * last_exec_vcpu[]. 3065 */ 3066 for_each_possible_cpu(cpu) { 3067 if (last_vcpu[cpu] == vcpu) 3068 last_vcpu[cpu] = NULL; 3069 if (last_exec_vcpu[cpu] == vcpu) 3070 last_exec_vcpu[cpu] = NULL; 3071 } 3072 } 3073 3074 static int kvm_vz_vcpu_setup(struct kvm_vcpu *vcpu) 3075 { 3076 struct mips_coproc *cop0 = vcpu->arch.cop0; 3077 unsigned long count_hz = 100*1000*1000; /* default to 100 MHz */ 3078 3079 /* 3080 * Start off the timer at the same frequency as the host timer, but the 3081 * soft timer doesn't handle frequencies greater than 1GHz yet. 3082 */ 3083 if (mips_hpt_frequency && mips_hpt_frequency <= NSEC_PER_SEC) 3084 count_hz = mips_hpt_frequency; 3085 kvm_mips_init_count(vcpu, count_hz); 3086 3087 /* 3088 * Initialize guest register state to valid architectural reset state. 3089 */ 3090 3091 /* PageGrain */ 3092 if (cpu_has_mips_r5 || cpu_has_mips_r6) 3093 kvm_write_sw_gc0_pagegrain(cop0, PG_RIE | PG_XIE | PG_IEC); 3094 /* Wired */ 3095 if (cpu_has_mips_r6) 3096 kvm_write_sw_gc0_wired(cop0, 3097 read_gc0_wired() & MIPSR6_WIRED_LIMIT); 3098 /* Status */ 3099 kvm_write_sw_gc0_status(cop0, ST0_BEV | ST0_ERL); 3100 if (cpu_has_mips_r5 || cpu_has_mips_r6) 3101 kvm_change_sw_gc0_status(cop0, ST0_FR, read_gc0_status()); 3102 /* IntCtl */ 3103 kvm_write_sw_gc0_intctl(cop0, read_gc0_intctl() & 3104 (INTCTLF_IPFDC | INTCTLF_IPPCI | INTCTLF_IPTI)); 3105 /* PRId */ 3106 kvm_write_sw_gc0_prid(cop0, boot_cpu_data.processor_id); 3107 /* EBase */ 3108 kvm_write_sw_gc0_ebase(cop0, (s32)0x80000000 | vcpu->vcpu_id); 3109 /* Config */ 3110 kvm_save_gc0_config(cop0); 3111 /* architecturally writable (e.g. from guest) */ 3112 kvm_change_sw_gc0_config(cop0, CONF_CM_CMASK, 3113 _page_cachable_default >> _CACHE_SHIFT); 3114 /* architecturally read only, but maybe writable from root */ 3115 kvm_change_sw_gc0_config(cop0, MIPS_CONF_MT, read_c0_config()); 3116 if (cpu_guest_has_conf1) { 3117 kvm_set_sw_gc0_config(cop0, MIPS_CONF_M); 3118 /* Config1 */ 3119 kvm_save_gc0_config1(cop0); 3120 /* architecturally read only, but maybe writable from root */ 3121 kvm_clear_sw_gc0_config1(cop0, MIPS_CONF1_C2 | 3122 MIPS_CONF1_MD | 3123 MIPS_CONF1_PC | 3124 MIPS_CONF1_WR | 3125 MIPS_CONF1_CA | 3126 MIPS_CONF1_FP); 3127 } 3128 if (cpu_guest_has_conf2) { 3129 kvm_set_sw_gc0_config1(cop0, MIPS_CONF_M); 3130 /* Config2 */ 3131 kvm_save_gc0_config2(cop0); 3132 } 3133 if (cpu_guest_has_conf3) { 3134 kvm_set_sw_gc0_config2(cop0, MIPS_CONF_M); 3135 /* Config3 */ 3136 kvm_save_gc0_config3(cop0); 3137 /* architecturally writable (e.g. from guest) */ 3138 kvm_clear_sw_gc0_config3(cop0, MIPS_CONF3_ISA_OE); 3139 /* architecturally read only, but maybe writable from root */ 3140 kvm_clear_sw_gc0_config3(cop0, MIPS_CONF3_MSA | 3141 MIPS_CONF3_BPG | 3142 MIPS_CONF3_ULRI | 3143 MIPS_CONF3_DSP | 3144 MIPS_CONF3_CTXTC | 3145 MIPS_CONF3_ITL | 3146 MIPS_CONF3_LPA | 3147 MIPS_CONF3_VEIC | 3148 MIPS_CONF3_VINT | 3149 MIPS_CONF3_SP | 3150 MIPS_CONF3_CDMM | 3151 MIPS_CONF3_MT | 3152 MIPS_CONF3_SM | 3153 MIPS_CONF3_TL); 3154 } 3155 if (cpu_guest_has_conf4) { 3156 kvm_set_sw_gc0_config3(cop0, MIPS_CONF_M); 3157 /* Config4 */ 3158 kvm_save_gc0_config4(cop0); 3159 } 3160 if (cpu_guest_has_conf5) { 3161 kvm_set_sw_gc0_config4(cop0, MIPS_CONF_M); 3162 /* Config5 */ 3163 kvm_save_gc0_config5(cop0); 3164 /* architecturally writable (e.g. from guest) */ 3165 kvm_clear_sw_gc0_config5(cop0, MIPS_CONF5_K | 3166 MIPS_CONF5_CV | 3167 MIPS_CONF5_MSAEN | 3168 MIPS_CONF5_UFE | 3169 MIPS_CONF5_FRE | 3170 MIPS_CONF5_SBRI | 3171 MIPS_CONF5_UFR); 3172 /* architecturally read only, but maybe writable from root */ 3173 kvm_clear_sw_gc0_config5(cop0, MIPS_CONF5_MRP); 3174 } 3175 3176 if (cpu_guest_has_contextconfig) { 3177 /* ContextConfig */ 3178 kvm_write_sw_gc0_contextconfig(cop0, 0x007ffff0); 3179 #ifdef CONFIG_64BIT 3180 /* XContextConfig */ 3181 /* bits SEGBITS-13+3:4 set */ 3182 kvm_write_sw_gc0_xcontextconfig(cop0, 3183 ((1ull << (cpu_vmbits - 13)) - 1) << 4); 3184 #endif 3185 } 3186 3187 /* Implementation dependent, use the legacy layout */ 3188 if (cpu_guest_has_segments) { 3189 /* SegCtl0, SegCtl1, SegCtl2 */ 3190 kvm_write_sw_gc0_segctl0(cop0, 0x00200010); 3191 kvm_write_sw_gc0_segctl1(cop0, 0x00000002 | 3192 (_page_cachable_default >> _CACHE_SHIFT) << 3193 (16 + MIPS_SEGCFG_C_SHIFT)); 3194 kvm_write_sw_gc0_segctl2(cop0, 0x00380438); 3195 } 3196 3197 /* reset HTW registers */ 3198 if (cpu_guest_has_htw && (cpu_has_mips_r5 || cpu_has_mips_r6)) { 3199 /* PWField */ 3200 kvm_write_sw_gc0_pwfield(cop0, 0x0c30c302); 3201 /* PWSize */ 3202 kvm_write_sw_gc0_pwsize(cop0, 1 << MIPS_PWSIZE_PTW_SHIFT); 3203 } 3204 3205 /* start with no pending virtual guest interrupts */ 3206 if (cpu_has_guestctl2) 3207 cop0->reg[MIPS_CP0_GUESTCTL2][MIPS_CP0_GUESTCTL2_SEL] = 0; 3208 3209 /* Put PC at reset vector */ 3210 vcpu->arch.pc = CKSEG1ADDR(0x1fc00000); 3211 3212 return 0; 3213 } 3214 3215 static void kvm_vz_flush_shadow_all(struct kvm *kvm) 3216 { 3217 if (cpu_has_guestid) { 3218 /* Flush GuestID for each VCPU individually */ 3219 kvm_flush_remote_tlbs(kvm); 3220 } else { 3221 /* 3222 * For each CPU there is a single GPA ASID used by all VCPUs in 3223 * the VM, so it doesn't make sense for the VCPUs to handle 3224 * invalidation of these ASIDs individually. 3225 * 3226 * Instead mark all CPUs as needing ASID invalidation in 3227 * asid_flush_mask, and just use kvm_flush_remote_tlbs(kvm) to 3228 * kick any running VCPUs so they check asid_flush_mask. 3229 */ 3230 cpumask_setall(&kvm->arch.asid_flush_mask); 3231 kvm_flush_remote_tlbs(kvm); 3232 } 3233 } 3234 3235 static void kvm_vz_flush_shadow_memslot(struct kvm *kvm, 3236 const struct kvm_memory_slot *slot) 3237 { 3238 kvm_vz_flush_shadow_all(kvm); 3239 } 3240 3241 static void kvm_vz_vcpu_reenter(struct kvm_vcpu *vcpu) 3242 { 3243 int cpu = smp_processor_id(); 3244 int preserve_guest_tlb; 3245 3246 preserve_guest_tlb = kvm_vz_check_requests(vcpu, cpu); 3247 3248 if (preserve_guest_tlb) 3249 kvm_vz_vcpu_save_wired(vcpu); 3250 3251 kvm_vz_vcpu_load_tlb(vcpu, cpu); 3252 3253 if (preserve_guest_tlb) 3254 kvm_vz_vcpu_load_wired(vcpu); 3255 } 3256 3257 static int kvm_vz_vcpu_run(struct kvm_vcpu *vcpu) 3258 { 3259 int cpu = smp_processor_id(); 3260 int r; 3261 3262 kvm_vz_acquire_htimer(vcpu); 3263 /* Check if we have any exceptions/interrupts pending */ 3264 kvm_mips_deliver_interrupts(vcpu, read_gc0_cause()); 3265 3266 kvm_vz_check_requests(vcpu, cpu); 3267 kvm_vz_vcpu_load_tlb(vcpu, cpu); 3268 kvm_vz_vcpu_load_wired(vcpu); 3269 3270 r = vcpu->arch.vcpu_run(vcpu->run, vcpu); 3271 3272 kvm_vz_vcpu_save_wired(vcpu); 3273 3274 return r; 3275 } 3276 3277 static struct kvm_mips_callbacks kvm_vz_callbacks = { 3278 .handle_cop_unusable = kvm_trap_vz_handle_cop_unusable, 3279 .handle_tlb_mod = kvm_trap_vz_handle_tlb_st_miss, 3280 .handle_tlb_ld_miss = kvm_trap_vz_handle_tlb_ld_miss, 3281 .handle_tlb_st_miss = kvm_trap_vz_handle_tlb_st_miss, 3282 .handle_addr_err_st = kvm_trap_vz_no_handler, 3283 .handle_addr_err_ld = kvm_trap_vz_no_handler, 3284 .handle_syscall = kvm_trap_vz_no_handler, 3285 .handle_res_inst = kvm_trap_vz_no_handler, 3286 .handle_break = kvm_trap_vz_no_handler, 3287 .handle_msa_disabled = kvm_trap_vz_handle_msa_disabled, 3288 .handle_guest_exit = kvm_trap_vz_handle_guest_exit, 3289 3290 .hardware_enable = kvm_vz_hardware_enable, 3291 .hardware_disable = kvm_vz_hardware_disable, 3292 .check_extension = kvm_vz_check_extension, 3293 .vcpu_init = kvm_vz_vcpu_init, 3294 .vcpu_uninit = kvm_vz_vcpu_uninit, 3295 .vcpu_setup = kvm_vz_vcpu_setup, 3296 .flush_shadow_all = kvm_vz_flush_shadow_all, 3297 .flush_shadow_memslot = kvm_vz_flush_shadow_memslot, 3298 .gva_to_gpa = kvm_vz_gva_to_gpa_cb, 3299 .queue_timer_int = kvm_vz_queue_timer_int_cb, 3300 .dequeue_timer_int = kvm_vz_dequeue_timer_int_cb, 3301 .queue_io_int = kvm_vz_queue_io_int_cb, 3302 .dequeue_io_int = kvm_vz_dequeue_io_int_cb, 3303 .irq_deliver = kvm_vz_irq_deliver_cb, 3304 .irq_clear = kvm_vz_irq_clear_cb, 3305 .num_regs = kvm_vz_num_regs, 3306 .copy_reg_indices = kvm_vz_copy_reg_indices, 3307 .get_one_reg = kvm_vz_get_one_reg, 3308 .set_one_reg = kvm_vz_set_one_reg, 3309 .vcpu_load = kvm_vz_vcpu_load, 3310 .vcpu_put = kvm_vz_vcpu_put, 3311 .vcpu_run = kvm_vz_vcpu_run, 3312 .vcpu_reenter = kvm_vz_vcpu_reenter, 3313 }; 3314 3315 int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks) 3316 { 3317 if (!cpu_has_vz) 3318 return -ENODEV; 3319 3320 /* 3321 * VZ requires at least 2 KScratch registers, so it should have been 3322 * possible to allocate pgd_reg. 3323 */ 3324 if (WARN(pgd_reg == -1, 3325 "pgd_reg not allocated even though cpu_has_vz\n")) 3326 return -ENODEV; 3327 3328 pr_info("Starting KVM with MIPS VZ extensions\n"); 3329 3330 *install_callbacks = &kvm_vz_callbacks; 3331 return 0; 3332 } 3333