1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * KVM/MIPS: Instruction/Exception emulation 7 * 8 * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. 9 * Authors: Sanjay Lal <sanjayl@kymasys.com> 10 */ 11 12 #include <linux/errno.h> 13 #include <linux/err.h> 14 #include <linux/ktime.h> 15 #include <linux/kvm_host.h> 16 #include <linux/module.h> 17 #include <linux/vmalloc.h> 18 #include <linux/fs.h> 19 #include <linux/bootmem.h> 20 #include <linux/random.h> 21 #include <asm/page.h> 22 #include <asm/cacheflush.h> 23 #include <asm/cpu-info.h> 24 #include <asm/mmu_context.h> 25 #include <asm/tlbflush.h> 26 #include <asm/inst.h> 27 28 #undef CONFIG_MIPS_MT 29 #include <asm/r4kcache.h> 30 #define CONFIG_MIPS_MT 31 32 #include "opcode.h" 33 #include "interrupt.h" 34 #include "commpage.h" 35 36 #include "trace.h" 37 38 /* 39 * Compute the return address and do emulate branch simulation, if required. 40 * This function should be called only in branch delay slot active. 41 */ 42 unsigned long kvm_compute_return_epc(struct kvm_vcpu *vcpu, 43 unsigned long instpc) 44 { 45 unsigned int dspcontrol; 46 union mips_instruction insn; 47 struct kvm_vcpu_arch *arch = &vcpu->arch; 48 long epc = instpc; 49 long nextpc = KVM_INVALID_INST; 50 51 if (epc & 3) 52 goto unaligned; 53 54 /* Read the instruction */ 55 insn.word = kvm_get_inst((uint32_t *) epc, vcpu); 56 57 if (insn.word == KVM_INVALID_INST) 58 return KVM_INVALID_INST; 59 60 switch (insn.i_format.opcode) { 61 /* jr and jalr are in r_format format. */ 62 case spec_op: 63 switch (insn.r_format.func) { 64 case jalr_op: 65 arch->gprs[insn.r_format.rd] = epc + 8; 66 /* Fall through */ 67 case jr_op: 68 nextpc = arch->gprs[insn.r_format.rs]; 69 break; 70 } 71 break; 72 73 /* 74 * This group contains: 75 * bltz_op, bgez_op, bltzl_op, bgezl_op, 76 * bltzal_op, bgezal_op, bltzall_op, bgezall_op. 77 */ 78 case bcond_op: 79 switch (insn.i_format.rt) { 80 case bltz_op: 81 case bltzl_op: 82 if ((long)arch->gprs[insn.i_format.rs] < 0) 83 epc = epc + 4 + (insn.i_format.simmediate << 2); 84 else 85 epc += 8; 86 nextpc = epc; 87 break; 88 89 case bgez_op: 90 case bgezl_op: 91 if ((long)arch->gprs[insn.i_format.rs] >= 0) 92 epc = epc + 4 + (insn.i_format.simmediate << 2); 93 else 94 epc += 8; 95 nextpc = epc; 96 break; 97 98 case bltzal_op: 99 case bltzall_op: 100 arch->gprs[31] = epc + 8; 101 if ((long)arch->gprs[insn.i_format.rs] < 0) 102 epc = epc + 4 + (insn.i_format.simmediate << 2); 103 else 104 epc += 8; 105 nextpc = epc; 106 break; 107 108 case bgezal_op: 109 case bgezall_op: 110 arch->gprs[31] = epc + 8; 111 if ((long)arch->gprs[insn.i_format.rs] >= 0) 112 epc = epc + 4 + (insn.i_format.simmediate << 2); 113 else 114 epc += 8; 115 nextpc = epc; 116 break; 117 case bposge32_op: 118 if (!cpu_has_dsp) 119 goto sigill; 120 121 dspcontrol = rddsp(0x01); 122 123 if (dspcontrol >= 32) 124 epc = epc + 4 + (insn.i_format.simmediate << 2); 125 else 126 epc += 8; 127 nextpc = epc; 128 break; 129 } 130 break; 131 132 /* These are unconditional and in j_format. */ 133 case jal_op: 134 arch->gprs[31] = instpc + 8; 135 case j_op: 136 epc += 4; 137 epc >>= 28; 138 epc <<= 28; 139 epc |= (insn.j_format.target << 2); 140 nextpc = epc; 141 break; 142 143 /* These are conditional and in i_format. */ 144 case beq_op: 145 case beql_op: 146 if (arch->gprs[insn.i_format.rs] == 147 arch->gprs[insn.i_format.rt]) 148 epc = epc + 4 + (insn.i_format.simmediate << 2); 149 else 150 epc += 8; 151 nextpc = epc; 152 break; 153 154 case bne_op: 155 case bnel_op: 156 if (arch->gprs[insn.i_format.rs] != 157 arch->gprs[insn.i_format.rt]) 158 epc = epc + 4 + (insn.i_format.simmediate << 2); 159 else 160 epc += 8; 161 nextpc = epc; 162 break; 163 164 case blez_op: /* not really i_format */ 165 case blezl_op: 166 /* rt field assumed to be zero */ 167 if ((long)arch->gprs[insn.i_format.rs] <= 0) 168 epc = epc + 4 + (insn.i_format.simmediate << 2); 169 else 170 epc += 8; 171 nextpc = epc; 172 break; 173 174 case bgtz_op: 175 case bgtzl_op: 176 /* rt field assumed to be zero */ 177 if ((long)arch->gprs[insn.i_format.rs] > 0) 178 epc = epc + 4 + (insn.i_format.simmediate << 2); 179 else 180 epc += 8; 181 nextpc = epc; 182 break; 183 184 /* And now the FPA/cp1 branch instructions. */ 185 case cop1_op: 186 kvm_err("%s: unsupported cop1_op\n", __func__); 187 break; 188 } 189 190 return nextpc; 191 192 unaligned: 193 kvm_err("%s: unaligned epc\n", __func__); 194 return nextpc; 195 196 sigill: 197 kvm_err("%s: DSP branch but not DSP ASE\n", __func__); 198 return nextpc; 199 } 200 201 enum emulation_result update_pc(struct kvm_vcpu *vcpu, uint32_t cause) 202 { 203 unsigned long branch_pc; 204 enum emulation_result er = EMULATE_DONE; 205 206 if (cause & CAUSEF_BD) { 207 branch_pc = kvm_compute_return_epc(vcpu, vcpu->arch.pc); 208 if (branch_pc == KVM_INVALID_INST) { 209 er = EMULATE_FAIL; 210 } else { 211 vcpu->arch.pc = branch_pc; 212 kvm_debug("BD update_pc(): New PC: %#lx\n", 213 vcpu->arch.pc); 214 } 215 } else 216 vcpu->arch.pc += 4; 217 218 kvm_debug("update_pc(): New PC: %#lx\n", vcpu->arch.pc); 219 220 return er; 221 } 222 223 /** 224 * kvm_mips_count_disabled() - Find whether the CP0_Count timer is disabled. 225 * @vcpu: Virtual CPU. 226 * 227 * Returns: 1 if the CP0_Count timer is disabled by either the guest 228 * CP0_Cause.DC bit or the count_ctl.DC bit. 229 * 0 otherwise (in which case CP0_Count timer is running). 230 */ 231 static inline int kvm_mips_count_disabled(struct kvm_vcpu *vcpu) 232 { 233 struct mips_coproc *cop0 = vcpu->arch.cop0; 234 235 return (vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) || 236 (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC); 237 } 238 239 /** 240 * kvm_mips_ktime_to_count() - Scale ktime_t to a 32-bit count. 241 * 242 * Caches the dynamic nanosecond bias in vcpu->arch.count_dyn_bias. 243 * 244 * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running). 245 */ 246 static uint32_t kvm_mips_ktime_to_count(struct kvm_vcpu *vcpu, ktime_t now) 247 { 248 s64 now_ns, periods; 249 u64 delta; 250 251 now_ns = ktime_to_ns(now); 252 delta = now_ns + vcpu->arch.count_dyn_bias; 253 254 if (delta >= vcpu->arch.count_period) { 255 /* If delta is out of safe range the bias needs adjusting */ 256 periods = div64_s64(now_ns, vcpu->arch.count_period); 257 vcpu->arch.count_dyn_bias = -periods * vcpu->arch.count_period; 258 /* Recalculate delta with new bias */ 259 delta = now_ns + vcpu->arch.count_dyn_bias; 260 } 261 262 /* 263 * We've ensured that: 264 * delta < count_period 265 * 266 * Therefore the intermediate delta*count_hz will never overflow since 267 * at the boundary condition: 268 * delta = count_period 269 * delta = NSEC_PER_SEC * 2^32 / count_hz 270 * delta * count_hz = NSEC_PER_SEC * 2^32 271 */ 272 return div_u64(delta * vcpu->arch.count_hz, NSEC_PER_SEC); 273 } 274 275 /** 276 * kvm_mips_count_time() - Get effective current time. 277 * @vcpu: Virtual CPU. 278 * 279 * Get effective monotonic ktime. This is usually a straightforward ktime_get(), 280 * except when the master disable bit is set in count_ctl, in which case it is 281 * count_resume, i.e. the time that the count was disabled. 282 * 283 * Returns: Effective monotonic ktime for CP0_Count. 284 */ 285 static inline ktime_t kvm_mips_count_time(struct kvm_vcpu *vcpu) 286 { 287 if (unlikely(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC)) 288 return vcpu->arch.count_resume; 289 290 return ktime_get(); 291 } 292 293 /** 294 * kvm_mips_read_count_running() - Read the current count value as if running. 295 * @vcpu: Virtual CPU. 296 * @now: Kernel time to read CP0_Count at. 297 * 298 * Returns the current guest CP0_Count register at time @now and handles if the 299 * timer interrupt is pending and hasn't been handled yet. 300 * 301 * Returns: The current value of the guest CP0_Count register. 302 */ 303 static uint32_t kvm_mips_read_count_running(struct kvm_vcpu *vcpu, ktime_t now) 304 { 305 ktime_t expires; 306 int running; 307 308 /* Is the hrtimer pending? */ 309 expires = hrtimer_get_expires(&vcpu->arch.comparecount_timer); 310 if (ktime_compare(now, expires) >= 0) { 311 /* 312 * Cancel it while we handle it so there's no chance of 313 * interference with the timeout handler. 314 */ 315 running = hrtimer_cancel(&vcpu->arch.comparecount_timer); 316 317 /* Nothing should be waiting on the timeout */ 318 kvm_mips_callbacks->queue_timer_int(vcpu); 319 320 /* 321 * Restart the timer if it was running based on the expiry time 322 * we read, so that we don't push it back 2 periods. 323 */ 324 if (running) { 325 expires = ktime_add_ns(expires, 326 vcpu->arch.count_period); 327 hrtimer_start(&vcpu->arch.comparecount_timer, expires, 328 HRTIMER_MODE_ABS); 329 } 330 } 331 332 /* Return the biased and scaled guest CP0_Count */ 333 return vcpu->arch.count_bias + kvm_mips_ktime_to_count(vcpu, now); 334 } 335 336 /** 337 * kvm_mips_read_count() - Read the current count value. 338 * @vcpu: Virtual CPU. 339 * 340 * Read the current guest CP0_Count value, taking into account whether the timer 341 * is stopped. 342 * 343 * Returns: The current guest CP0_Count value. 344 */ 345 uint32_t kvm_mips_read_count(struct kvm_vcpu *vcpu) 346 { 347 struct mips_coproc *cop0 = vcpu->arch.cop0; 348 349 /* If count disabled just read static copy of count */ 350 if (kvm_mips_count_disabled(vcpu)) 351 return kvm_read_c0_guest_count(cop0); 352 353 return kvm_mips_read_count_running(vcpu, ktime_get()); 354 } 355 356 /** 357 * kvm_mips_freeze_hrtimer() - Safely stop the hrtimer. 358 * @vcpu: Virtual CPU. 359 * @count: Output pointer for CP0_Count value at point of freeze. 360 * 361 * Freeze the hrtimer safely and return both the ktime and the CP0_Count value 362 * at the point it was frozen. It is guaranteed that any pending interrupts at 363 * the point it was frozen are handled, and none after that point. 364 * 365 * This is useful where the time/CP0_Count is needed in the calculation of the 366 * new parameters. 367 * 368 * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running). 369 * 370 * Returns: The ktime at the point of freeze. 371 */ 372 static ktime_t kvm_mips_freeze_hrtimer(struct kvm_vcpu *vcpu, 373 uint32_t *count) 374 { 375 ktime_t now; 376 377 /* stop hrtimer before finding time */ 378 hrtimer_cancel(&vcpu->arch.comparecount_timer); 379 now = ktime_get(); 380 381 /* find count at this point and handle pending hrtimer */ 382 *count = kvm_mips_read_count_running(vcpu, now); 383 384 return now; 385 } 386 387 /** 388 * kvm_mips_resume_hrtimer() - Resume hrtimer, updating expiry. 389 * @vcpu: Virtual CPU. 390 * @now: ktime at point of resume. 391 * @count: CP0_Count at point of resume. 392 * 393 * Resumes the timer and updates the timer expiry based on @now and @count. 394 * This can be used in conjunction with kvm_mips_freeze_timer() when timer 395 * parameters need to be changed. 396 * 397 * It is guaranteed that a timer interrupt immediately after resume will be 398 * handled, but not if CP_Compare is exactly at @count. That case is already 399 * handled by kvm_mips_freeze_timer(). 400 * 401 * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running). 402 */ 403 static void kvm_mips_resume_hrtimer(struct kvm_vcpu *vcpu, 404 ktime_t now, uint32_t count) 405 { 406 struct mips_coproc *cop0 = vcpu->arch.cop0; 407 uint32_t compare; 408 u64 delta; 409 ktime_t expire; 410 411 /* Calculate timeout (wrap 0 to 2^32) */ 412 compare = kvm_read_c0_guest_compare(cop0); 413 delta = (u64)(uint32_t)(compare - count - 1) + 1; 414 delta = div_u64(delta * NSEC_PER_SEC, vcpu->arch.count_hz); 415 expire = ktime_add_ns(now, delta); 416 417 /* Update hrtimer to use new timeout */ 418 hrtimer_cancel(&vcpu->arch.comparecount_timer); 419 hrtimer_start(&vcpu->arch.comparecount_timer, expire, HRTIMER_MODE_ABS); 420 } 421 422 /** 423 * kvm_mips_update_hrtimer() - Update next expiry time of hrtimer. 424 * @vcpu: Virtual CPU. 425 * 426 * Recalculates and updates the expiry time of the hrtimer. This can be used 427 * after timer parameters have been altered which do not depend on the time that 428 * the change occurs (in those cases kvm_mips_freeze_hrtimer() and 429 * kvm_mips_resume_hrtimer() are used directly). 430 * 431 * It is guaranteed that no timer interrupts will be lost in the process. 432 * 433 * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running). 434 */ 435 static void kvm_mips_update_hrtimer(struct kvm_vcpu *vcpu) 436 { 437 ktime_t now; 438 uint32_t count; 439 440 /* 441 * freeze_hrtimer takes care of a timer interrupts <= count, and 442 * resume_hrtimer the hrtimer takes care of a timer interrupts > count. 443 */ 444 now = kvm_mips_freeze_hrtimer(vcpu, &count); 445 kvm_mips_resume_hrtimer(vcpu, now, count); 446 } 447 448 /** 449 * kvm_mips_write_count() - Modify the count and update timer. 450 * @vcpu: Virtual CPU. 451 * @count: Guest CP0_Count value to set. 452 * 453 * Sets the CP0_Count value and updates the timer accordingly. 454 */ 455 void kvm_mips_write_count(struct kvm_vcpu *vcpu, uint32_t count) 456 { 457 struct mips_coproc *cop0 = vcpu->arch.cop0; 458 ktime_t now; 459 460 /* Calculate bias */ 461 now = kvm_mips_count_time(vcpu); 462 vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now); 463 464 if (kvm_mips_count_disabled(vcpu)) 465 /* The timer's disabled, adjust the static count */ 466 kvm_write_c0_guest_count(cop0, count); 467 else 468 /* Update timeout */ 469 kvm_mips_resume_hrtimer(vcpu, now, count); 470 } 471 472 /** 473 * kvm_mips_init_count() - Initialise timer. 474 * @vcpu: Virtual CPU. 475 * 476 * Initialise the timer to a sensible frequency, namely 100MHz, zero it, and set 477 * it going if it's enabled. 478 */ 479 void kvm_mips_init_count(struct kvm_vcpu *vcpu) 480 { 481 /* 100 MHz */ 482 vcpu->arch.count_hz = 100*1000*1000; 483 vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32, 484 vcpu->arch.count_hz); 485 vcpu->arch.count_dyn_bias = 0; 486 487 /* Starting at 0 */ 488 kvm_mips_write_count(vcpu, 0); 489 } 490 491 /** 492 * kvm_mips_set_count_hz() - Update the frequency of the timer. 493 * @vcpu: Virtual CPU. 494 * @count_hz: Frequency of CP0_Count timer in Hz. 495 * 496 * Change the frequency of the CP0_Count timer. This is done atomically so that 497 * CP0_Count is continuous and no timer interrupt is lost. 498 * 499 * Returns: -EINVAL if @count_hz is out of range. 500 * 0 on success. 501 */ 502 int kvm_mips_set_count_hz(struct kvm_vcpu *vcpu, s64 count_hz) 503 { 504 struct mips_coproc *cop0 = vcpu->arch.cop0; 505 int dc; 506 ktime_t now; 507 u32 count; 508 509 /* ensure the frequency is in a sensible range... */ 510 if (count_hz <= 0 || count_hz > NSEC_PER_SEC) 511 return -EINVAL; 512 /* ... and has actually changed */ 513 if (vcpu->arch.count_hz == count_hz) 514 return 0; 515 516 /* Safely freeze timer so we can keep it continuous */ 517 dc = kvm_mips_count_disabled(vcpu); 518 if (dc) { 519 now = kvm_mips_count_time(vcpu); 520 count = kvm_read_c0_guest_count(cop0); 521 } else { 522 now = kvm_mips_freeze_hrtimer(vcpu, &count); 523 } 524 525 /* Update the frequency */ 526 vcpu->arch.count_hz = count_hz; 527 vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32, count_hz); 528 vcpu->arch.count_dyn_bias = 0; 529 530 /* Calculate adjusted bias so dynamic count is unchanged */ 531 vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now); 532 533 /* Update and resume hrtimer */ 534 if (!dc) 535 kvm_mips_resume_hrtimer(vcpu, now, count); 536 return 0; 537 } 538 539 /** 540 * kvm_mips_write_compare() - Modify compare and update timer. 541 * @vcpu: Virtual CPU. 542 * @compare: New CP0_Compare value. 543 * 544 * Update CP0_Compare to a new value and update the timeout. 545 */ 546 void kvm_mips_write_compare(struct kvm_vcpu *vcpu, uint32_t compare) 547 { 548 struct mips_coproc *cop0 = vcpu->arch.cop0; 549 550 /* if unchanged, must just be an ack */ 551 if (kvm_read_c0_guest_compare(cop0) == compare) 552 return; 553 554 /* Update compare */ 555 kvm_write_c0_guest_compare(cop0, compare); 556 557 /* Update timeout if count enabled */ 558 if (!kvm_mips_count_disabled(vcpu)) 559 kvm_mips_update_hrtimer(vcpu); 560 } 561 562 /** 563 * kvm_mips_count_disable() - Disable count. 564 * @vcpu: Virtual CPU. 565 * 566 * Disable the CP0_Count timer. A timer interrupt on or before the final stop 567 * time will be handled but not after. 568 * 569 * Assumes CP0_Count was previously enabled but now Guest.CP0_Cause.DC or 570 * count_ctl.DC has been set (count disabled). 571 * 572 * Returns: The time that the timer was stopped. 573 */ 574 static ktime_t kvm_mips_count_disable(struct kvm_vcpu *vcpu) 575 { 576 struct mips_coproc *cop0 = vcpu->arch.cop0; 577 uint32_t count; 578 ktime_t now; 579 580 /* Stop hrtimer */ 581 hrtimer_cancel(&vcpu->arch.comparecount_timer); 582 583 /* Set the static count from the dynamic count, handling pending TI */ 584 now = ktime_get(); 585 count = kvm_mips_read_count_running(vcpu, now); 586 kvm_write_c0_guest_count(cop0, count); 587 588 return now; 589 } 590 591 /** 592 * kvm_mips_count_disable_cause() - Disable count using CP0_Cause.DC. 593 * @vcpu: Virtual CPU. 594 * 595 * Disable the CP0_Count timer and set CP0_Cause.DC. A timer interrupt on or 596 * before the final stop time will be handled if the timer isn't disabled by 597 * count_ctl.DC, but not after. 598 * 599 * Assumes CP0_Cause.DC is clear (count enabled). 600 */ 601 void kvm_mips_count_disable_cause(struct kvm_vcpu *vcpu) 602 { 603 struct mips_coproc *cop0 = vcpu->arch.cop0; 604 605 kvm_set_c0_guest_cause(cop0, CAUSEF_DC); 606 if (!(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC)) 607 kvm_mips_count_disable(vcpu); 608 } 609 610 /** 611 * kvm_mips_count_enable_cause() - Enable count using CP0_Cause.DC. 612 * @vcpu: Virtual CPU. 613 * 614 * Enable the CP0_Count timer and clear CP0_Cause.DC. A timer interrupt after 615 * the start time will be handled if the timer isn't disabled by count_ctl.DC, 616 * potentially before even returning, so the caller should be careful with 617 * ordering of CP0_Cause modifications so as not to lose it. 618 * 619 * Assumes CP0_Cause.DC is set (count disabled). 620 */ 621 void kvm_mips_count_enable_cause(struct kvm_vcpu *vcpu) 622 { 623 struct mips_coproc *cop0 = vcpu->arch.cop0; 624 uint32_t count; 625 626 kvm_clear_c0_guest_cause(cop0, CAUSEF_DC); 627 628 /* 629 * Set the dynamic count to match the static count. 630 * This starts the hrtimer if count_ctl.DC allows it. 631 * Otherwise it conveniently updates the biases. 632 */ 633 count = kvm_read_c0_guest_count(cop0); 634 kvm_mips_write_count(vcpu, count); 635 } 636 637 /** 638 * kvm_mips_set_count_ctl() - Update the count control KVM register. 639 * @vcpu: Virtual CPU. 640 * @count_ctl: Count control register new value. 641 * 642 * Set the count control KVM register. The timer is updated accordingly. 643 * 644 * Returns: -EINVAL if reserved bits are set. 645 * 0 on success. 646 */ 647 int kvm_mips_set_count_ctl(struct kvm_vcpu *vcpu, s64 count_ctl) 648 { 649 struct mips_coproc *cop0 = vcpu->arch.cop0; 650 s64 changed = count_ctl ^ vcpu->arch.count_ctl; 651 s64 delta; 652 ktime_t expire, now; 653 uint32_t count, compare; 654 655 /* Only allow defined bits to be changed */ 656 if (changed & ~(s64)(KVM_REG_MIPS_COUNT_CTL_DC)) 657 return -EINVAL; 658 659 /* Apply new value */ 660 vcpu->arch.count_ctl = count_ctl; 661 662 /* Master CP0_Count disable */ 663 if (changed & KVM_REG_MIPS_COUNT_CTL_DC) { 664 /* Is CP0_Cause.DC already disabling CP0_Count? */ 665 if (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC) { 666 if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) 667 /* Just record the current time */ 668 vcpu->arch.count_resume = ktime_get(); 669 } else if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) { 670 /* disable timer and record current time */ 671 vcpu->arch.count_resume = kvm_mips_count_disable(vcpu); 672 } else { 673 /* 674 * Calculate timeout relative to static count at resume 675 * time (wrap 0 to 2^32). 676 */ 677 count = kvm_read_c0_guest_count(cop0); 678 compare = kvm_read_c0_guest_compare(cop0); 679 delta = (u64)(uint32_t)(compare - count - 1) + 1; 680 delta = div_u64(delta * NSEC_PER_SEC, 681 vcpu->arch.count_hz); 682 expire = ktime_add_ns(vcpu->arch.count_resume, delta); 683 684 /* Handle pending interrupt */ 685 now = ktime_get(); 686 if (ktime_compare(now, expire) >= 0) 687 /* Nothing should be waiting on the timeout */ 688 kvm_mips_callbacks->queue_timer_int(vcpu); 689 690 /* Resume hrtimer without changing bias */ 691 count = kvm_mips_read_count_running(vcpu, now); 692 kvm_mips_resume_hrtimer(vcpu, now, count); 693 } 694 } 695 696 return 0; 697 } 698 699 /** 700 * kvm_mips_set_count_resume() - Update the count resume KVM register. 701 * @vcpu: Virtual CPU. 702 * @count_resume: Count resume register new value. 703 * 704 * Set the count resume KVM register. 705 * 706 * Returns: -EINVAL if out of valid range (0..now). 707 * 0 on success. 708 */ 709 int kvm_mips_set_count_resume(struct kvm_vcpu *vcpu, s64 count_resume) 710 { 711 /* 712 * It doesn't make sense for the resume time to be in the future, as it 713 * would be possible for the next interrupt to be more than a full 714 * period in the future. 715 */ 716 if (count_resume < 0 || count_resume > ktime_to_ns(ktime_get())) 717 return -EINVAL; 718 719 vcpu->arch.count_resume = ns_to_ktime(count_resume); 720 return 0; 721 } 722 723 /** 724 * kvm_mips_count_timeout() - Push timer forward on timeout. 725 * @vcpu: Virtual CPU. 726 * 727 * Handle an hrtimer event by push the hrtimer forward a period. 728 * 729 * Returns: The hrtimer_restart value to return to the hrtimer subsystem. 730 */ 731 enum hrtimer_restart kvm_mips_count_timeout(struct kvm_vcpu *vcpu) 732 { 733 /* Add the Count period to the current expiry time */ 734 hrtimer_add_expires_ns(&vcpu->arch.comparecount_timer, 735 vcpu->arch.count_period); 736 return HRTIMER_RESTART; 737 } 738 739 enum emulation_result kvm_mips_emul_eret(struct kvm_vcpu *vcpu) 740 { 741 struct mips_coproc *cop0 = vcpu->arch.cop0; 742 enum emulation_result er = EMULATE_DONE; 743 744 if (kvm_read_c0_guest_status(cop0) & ST0_EXL) { 745 kvm_debug("[%#lx] ERET to %#lx\n", vcpu->arch.pc, 746 kvm_read_c0_guest_epc(cop0)); 747 kvm_clear_c0_guest_status(cop0, ST0_EXL); 748 vcpu->arch.pc = kvm_read_c0_guest_epc(cop0); 749 750 } else if (kvm_read_c0_guest_status(cop0) & ST0_ERL) { 751 kvm_clear_c0_guest_status(cop0, ST0_ERL); 752 vcpu->arch.pc = kvm_read_c0_guest_errorepc(cop0); 753 } else { 754 kvm_err("[%#lx] ERET when MIPS_SR_EXL|MIPS_SR_ERL == 0\n", 755 vcpu->arch.pc); 756 er = EMULATE_FAIL; 757 } 758 759 return er; 760 } 761 762 enum emulation_result kvm_mips_emul_wait(struct kvm_vcpu *vcpu) 763 { 764 kvm_debug("[%#lx] !!!WAIT!!! (%#lx)\n", vcpu->arch.pc, 765 vcpu->arch.pending_exceptions); 766 767 ++vcpu->stat.wait_exits; 768 trace_kvm_exit(vcpu, WAIT_EXITS); 769 if (!vcpu->arch.pending_exceptions) { 770 vcpu->arch.wait = 1; 771 kvm_vcpu_block(vcpu); 772 773 /* 774 * We we are runnable, then definitely go off to user space to 775 * check if any I/O interrupts are pending. 776 */ 777 if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) { 778 clear_bit(KVM_REQ_UNHALT, &vcpu->requests); 779 vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; 780 } 781 } 782 783 return EMULATE_DONE; 784 } 785 786 /* 787 * XXXKYMA: Linux doesn't seem to use TLBR, return EMULATE_FAIL for now so that 788 * we can catch this, if things ever change 789 */ 790 enum emulation_result kvm_mips_emul_tlbr(struct kvm_vcpu *vcpu) 791 { 792 struct mips_coproc *cop0 = vcpu->arch.cop0; 793 uint32_t pc = vcpu->arch.pc; 794 795 kvm_err("[%#x] COP0_TLBR [%ld]\n", pc, kvm_read_c0_guest_index(cop0)); 796 return EMULATE_FAIL; 797 } 798 799 /* Write Guest TLB Entry @ Index */ 800 enum emulation_result kvm_mips_emul_tlbwi(struct kvm_vcpu *vcpu) 801 { 802 struct mips_coproc *cop0 = vcpu->arch.cop0; 803 int index = kvm_read_c0_guest_index(cop0); 804 struct kvm_mips_tlb *tlb = NULL; 805 uint32_t pc = vcpu->arch.pc; 806 807 if (index < 0 || index >= KVM_MIPS_GUEST_TLB_SIZE) { 808 kvm_debug("%s: illegal index: %d\n", __func__, index); 809 kvm_debug("[%#x] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n", 810 pc, index, kvm_read_c0_guest_entryhi(cop0), 811 kvm_read_c0_guest_entrylo0(cop0), 812 kvm_read_c0_guest_entrylo1(cop0), 813 kvm_read_c0_guest_pagemask(cop0)); 814 index = (index & ~0x80000000) % KVM_MIPS_GUEST_TLB_SIZE; 815 } 816 817 tlb = &vcpu->arch.guest_tlb[index]; 818 /* 819 * Probe the shadow host TLB for the entry being overwritten, if one 820 * matches, invalidate it 821 */ 822 kvm_mips_host_tlb_inv(vcpu, tlb->tlb_hi); 823 824 tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0); 825 tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0); 826 tlb->tlb_lo0 = kvm_read_c0_guest_entrylo0(cop0); 827 tlb->tlb_lo1 = kvm_read_c0_guest_entrylo1(cop0); 828 829 kvm_debug("[%#x] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n", 830 pc, index, kvm_read_c0_guest_entryhi(cop0), 831 kvm_read_c0_guest_entrylo0(cop0), 832 kvm_read_c0_guest_entrylo1(cop0), 833 kvm_read_c0_guest_pagemask(cop0)); 834 835 return EMULATE_DONE; 836 } 837 838 /* Write Guest TLB Entry @ Random Index */ 839 enum emulation_result kvm_mips_emul_tlbwr(struct kvm_vcpu *vcpu) 840 { 841 struct mips_coproc *cop0 = vcpu->arch.cop0; 842 struct kvm_mips_tlb *tlb = NULL; 843 uint32_t pc = vcpu->arch.pc; 844 int index; 845 846 get_random_bytes(&index, sizeof(index)); 847 index &= (KVM_MIPS_GUEST_TLB_SIZE - 1); 848 849 tlb = &vcpu->arch.guest_tlb[index]; 850 851 /* 852 * Probe the shadow host TLB for the entry being overwritten, if one 853 * matches, invalidate it 854 */ 855 kvm_mips_host_tlb_inv(vcpu, tlb->tlb_hi); 856 857 tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0); 858 tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0); 859 tlb->tlb_lo0 = kvm_read_c0_guest_entrylo0(cop0); 860 tlb->tlb_lo1 = kvm_read_c0_guest_entrylo1(cop0); 861 862 kvm_debug("[%#x] COP0_TLBWR[%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx)\n", 863 pc, index, kvm_read_c0_guest_entryhi(cop0), 864 kvm_read_c0_guest_entrylo0(cop0), 865 kvm_read_c0_guest_entrylo1(cop0)); 866 867 return EMULATE_DONE; 868 } 869 870 enum emulation_result kvm_mips_emul_tlbp(struct kvm_vcpu *vcpu) 871 { 872 struct mips_coproc *cop0 = vcpu->arch.cop0; 873 long entryhi = kvm_read_c0_guest_entryhi(cop0); 874 uint32_t pc = vcpu->arch.pc; 875 int index = -1; 876 877 index = kvm_mips_guest_tlb_lookup(vcpu, entryhi); 878 879 kvm_write_c0_guest_index(cop0, index); 880 881 kvm_debug("[%#x] COP0_TLBP (entryhi: %#lx), index: %d\n", pc, entryhi, 882 index); 883 884 return EMULATE_DONE; 885 } 886 887 enum emulation_result kvm_mips_emulate_CP0(uint32_t inst, uint32_t *opc, 888 uint32_t cause, struct kvm_run *run, 889 struct kvm_vcpu *vcpu) 890 { 891 struct mips_coproc *cop0 = vcpu->arch.cop0; 892 enum emulation_result er = EMULATE_DONE; 893 int32_t rt, rd, copz, sel, co_bit, op; 894 uint32_t pc = vcpu->arch.pc; 895 unsigned long curr_pc; 896 897 /* 898 * Update PC and hold onto current PC in case there is 899 * an error and we want to rollback the PC 900 */ 901 curr_pc = vcpu->arch.pc; 902 er = update_pc(vcpu, cause); 903 if (er == EMULATE_FAIL) 904 return er; 905 906 copz = (inst >> 21) & 0x1f; 907 rt = (inst >> 16) & 0x1f; 908 rd = (inst >> 11) & 0x1f; 909 sel = inst & 0x7; 910 co_bit = (inst >> 25) & 1; 911 912 if (co_bit) { 913 op = (inst) & 0xff; 914 915 switch (op) { 916 case tlbr_op: /* Read indexed TLB entry */ 917 er = kvm_mips_emul_tlbr(vcpu); 918 break; 919 case tlbwi_op: /* Write indexed */ 920 er = kvm_mips_emul_tlbwi(vcpu); 921 break; 922 case tlbwr_op: /* Write random */ 923 er = kvm_mips_emul_tlbwr(vcpu); 924 break; 925 case tlbp_op: /* TLB Probe */ 926 er = kvm_mips_emul_tlbp(vcpu); 927 break; 928 case rfe_op: 929 kvm_err("!!!COP0_RFE!!!\n"); 930 break; 931 case eret_op: 932 er = kvm_mips_emul_eret(vcpu); 933 goto dont_update_pc; 934 break; 935 case wait_op: 936 er = kvm_mips_emul_wait(vcpu); 937 break; 938 } 939 } else { 940 switch (copz) { 941 case mfc_op: 942 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS 943 cop0->stat[rd][sel]++; 944 #endif 945 /* Get reg */ 946 if ((rd == MIPS_CP0_COUNT) && (sel == 0)) { 947 vcpu->arch.gprs[rt] = kvm_mips_read_count(vcpu); 948 } else if ((rd == MIPS_CP0_ERRCTL) && (sel == 0)) { 949 vcpu->arch.gprs[rt] = 0x0; 950 #ifdef CONFIG_KVM_MIPS_DYN_TRANS 951 kvm_mips_trans_mfc0(inst, opc, vcpu); 952 #endif 953 } else { 954 vcpu->arch.gprs[rt] = cop0->reg[rd][sel]; 955 956 #ifdef CONFIG_KVM_MIPS_DYN_TRANS 957 kvm_mips_trans_mfc0(inst, opc, vcpu); 958 #endif 959 } 960 961 kvm_debug 962 ("[%#x] MFCz[%d][%d], vcpu->arch.gprs[%d]: %#lx\n", 963 pc, rd, sel, rt, vcpu->arch.gprs[rt]); 964 965 break; 966 967 case dmfc_op: 968 vcpu->arch.gprs[rt] = cop0->reg[rd][sel]; 969 break; 970 971 case mtc_op: 972 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS 973 cop0->stat[rd][sel]++; 974 #endif 975 if ((rd == MIPS_CP0_TLB_INDEX) 976 && (vcpu->arch.gprs[rt] >= 977 KVM_MIPS_GUEST_TLB_SIZE)) { 978 kvm_err("Invalid TLB Index: %ld", 979 vcpu->arch.gprs[rt]); 980 er = EMULATE_FAIL; 981 break; 982 } 983 #define C0_EBASE_CORE_MASK 0xff 984 if ((rd == MIPS_CP0_PRID) && (sel == 1)) { 985 /* Preserve CORE number */ 986 kvm_change_c0_guest_ebase(cop0, 987 ~(C0_EBASE_CORE_MASK), 988 vcpu->arch.gprs[rt]); 989 kvm_err("MTCz, cop0->reg[EBASE]: %#lx\n", 990 kvm_read_c0_guest_ebase(cop0)); 991 } else if (rd == MIPS_CP0_TLB_HI && sel == 0) { 992 uint32_t nasid = 993 vcpu->arch.gprs[rt] & ASID_MASK; 994 if ((KSEGX(vcpu->arch.gprs[rt]) != CKSEG0) && 995 ((kvm_read_c0_guest_entryhi(cop0) & 996 ASID_MASK) != nasid)) { 997 kvm_debug("MTCz, change ASID from %#lx to %#lx\n", 998 kvm_read_c0_guest_entryhi(cop0) 999 & ASID_MASK, 1000 vcpu->arch.gprs[rt] 1001 & ASID_MASK); 1002 1003 /* Blow away the shadow host TLBs */ 1004 kvm_mips_flush_host_tlb(1); 1005 } 1006 kvm_write_c0_guest_entryhi(cop0, 1007 vcpu->arch.gprs[rt]); 1008 } 1009 /* Are we writing to COUNT */ 1010 else if ((rd == MIPS_CP0_COUNT) && (sel == 0)) { 1011 kvm_mips_write_count(vcpu, vcpu->arch.gprs[rt]); 1012 goto done; 1013 } else if ((rd == MIPS_CP0_COMPARE) && (sel == 0)) { 1014 kvm_debug("[%#x] MTCz, COMPARE %#lx <- %#lx\n", 1015 pc, kvm_read_c0_guest_compare(cop0), 1016 vcpu->arch.gprs[rt]); 1017 1018 /* If we are writing to COMPARE */ 1019 /* Clear pending timer interrupt, if any */ 1020 kvm_mips_callbacks->dequeue_timer_int(vcpu); 1021 kvm_mips_write_compare(vcpu, 1022 vcpu->arch.gprs[rt]); 1023 } else if ((rd == MIPS_CP0_STATUS) && (sel == 0)) { 1024 kvm_write_c0_guest_status(cop0, 1025 vcpu->arch.gprs[rt]); 1026 /* 1027 * Make sure that CU1 and NMI bits are 1028 * never set 1029 */ 1030 kvm_clear_c0_guest_status(cop0, 1031 (ST0_CU1 | ST0_NMI)); 1032 1033 #ifdef CONFIG_KVM_MIPS_DYN_TRANS 1034 kvm_mips_trans_mtc0(inst, opc, vcpu); 1035 #endif 1036 } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) { 1037 uint32_t old_cause, new_cause; 1038 1039 old_cause = kvm_read_c0_guest_cause(cop0); 1040 new_cause = vcpu->arch.gprs[rt]; 1041 /* Update R/W bits */ 1042 kvm_change_c0_guest_cause(cop0, 0x08800300, 1043 new_cause); 1044 /* DC bit enabling/disabling timer? */ 1045 if ((old_cause ^ new_cause) & CAUSEF_DC) { 1046 if (new_cause & CAUSEF_DC) 1047 kvm_mips_count_disable_cause(vcpu); 1048 else 1049 kvm_mips_count_enable_cause(vcpu); 1050 } 1051 } else { 1052 cop0->reg[rd][sel] = vcpu->arch.gprs[rt]; 1053 #ifdef CONFIG_KVM_MIPS_DYN_TRANS 1054 kvm_mips_trans_mtc0(inst, opc, vcpu); 1055 #endif 1056 } 1057 1058 kvm_debug("[%#x] MTCz, cop0->reg[%d][%d]: %#lx\n", pc, 1059 rd, sel, cop0->reg[rd][sel]); 1060 break; 1061 1062 case dmtc_op: 1063 kvm_err("!!!!!!![%#lx]dmtc_op: rt: %d, rd: %d, sel: %d!!!!!!\n", 1064 vcpu->arch.pc, rt, rd, sel); 1065 er = EMULATE_FAIL; 1066 break; 1067 1068 case mfmcz_op: 1069 #ifdef KVM_MIPS_DEBUG_COP0_COUNTERS 1070 cop0->stat[MIPS_CP0_STATUS][0]++; 1071 #endif 1072 if (rt != 0) { 1073 vcpu->arch.gprs[rt] = 1074 kvm_read_c0_guest_status(cop0); 1075 } 1076 /* EI */ 1077 if (inst & 0x20) { 1078 kvm_debug("[%#lx] mfmcz_op: EI\n", 1079 vcpu->arch.pc); 1080 kvm_set_c0_guest_status(cop0, ST0_IE); 1081 } else { 1082 kvm_debug("[%#lx] mfmcz_op: DI\n", 1083 vcpu->arch.pc); 1084 kvm_clear_c0_guest_status(cop0, ST0_IE); 1085 } 1086 1087 break; 1088 1089 case wrpgpr_op: 1090 { 1091 uint32_t css = 1092 cop0->reg[MIPS_CP0_STATUS][2] & 0xf; 1093 uint32_t pss = 1094 (cop0->reg[MIPS_CP0_STATUS][2] >> 6) & 0xf; 1095 /* 1096 * We don't support any shadow register sets, so 1097 * SRSCtl[PSS] == SRSCtl[CSS] = 0 1098 */ 1099 if (css || pss) { 1100 er = EMULATE_FAIL; 1101 break; 1102 } 1103 kvm_debug("WRPGPR[%d][%d] = %#lx\n", pss, rd, 1104 vcpu->arch.gprs[rt]); 1105 vcpu->arch.gprs[rd] = vcpu->arch.gprs[rt]; 1106 } 1107 break; 1108 default: 1109 kvm_err("[%#lx]MachEmulateCP0: unsupported COP0, copz: 0x%x\n", 1110 vcpu->arch.pc, copz); 1111 er = EMULATE_FAIL; 1112 break; 1113 } 1114 } 1115 1116 done: 1117 /* Rollback PC only if emulation was unsuccessful */ 1118 if (er == EMULATE_FAIL) 1119 vcpu->arch.pc = curr_pc; 1120 1121 dont_update_pc: 1122 /* 1123 * This is for special instructions whose emulation 1124 * updates the PC, so do not overwrite the PC under 1125 * any circumstances 1126 */ 1127 1128 return er; 1129 } 1130 1131 enum emulation_result kvm_mips_emulate_store(uint32_t inst, uint32_t cause, 1132 struct kvm_run *run, 1133 struct kvm_vcpu *vcpu) 1134 { 1135 enum emulation_result er = EMULATE_DO_MMIO; 1136 int32_t op, base, rt, offset; 1137 uint32_t bytes; 1138 void *data = run->mmio.data; 1139 unsigned long curr_pc; 1140 1141 /* 1142 * Update PC and hold onto current PC in case there is 1143 * an error and we want to rollback the PC 1144 */ 1145 curr_pc = vcpu->arch.pc; 1146 er = update_pc(vcpu, cause); 1147 if (er == EMULATE_FAIL) 1148 return er; 1149 1150 rt = (inst >> 16) & 0x1f; 1151 base = (inst >> 21) & 0x1f; 1152 offset = inst & 0xffff; 1153 op = (inst >> 26) & 0x3f; 1154 1155 switch (op) { 1156 case sb_op: 1157 bytes = 1; 1158 if (bytes > sizeof(run->mmio.data)) { 1159 kvm_err("%s: bad MMIO length: %d\n", __func__, 1160 run->mmio.len); 1161 } 1162 run->mmio.phys_addr = 1163 kvm_mips_callbacks->gva_to_gpa(vcpu->arch. 1164 host_cp0_badvaddr); 1165 if (run->mmio.phys_addr == KVM_INVALID_ADDR) { 1166 er = EMULATE_FAIL; 1167 break; 1168 } 1169 run->mmio.len = bytes; 1170 run->mmio.is_write = 1; 1171 vcpu->mmio_needed = 1; 1172 vcpu->mmio_is_write = 1; 1173 *(u8 *) data = vcpu->arch.gprs[rt]; 1174 kvm_debug("OP_SB: eaddr: %#lx, gpr: %#lx, data: %#x\n", 1175 vcpu->arch.host_cp0_badvaddr, vcpu->arch.gprs[rt], 1176 *(uint8_t *) data); 1177 1178 break; 1179 1180 case sw_op: 1181 bytes = 4; 1182 if (bytes > sizeof(run->mmio.data)) { 1183 kvm_err("%s: bad MMIO length: %d\n", __func__, 1184 run->mmio.len); 1185 } 1186 run->mmio.phys_addr = 1187 kvm_mips_callbacks->gva_to_gpa(vcpu->arch. 1188 host_cp0_badvaddr); 1189 if (run->mmio.phys_addr == KVM_INVALID_ADDR) { 1190 er = EMULATE_FAIL; 1191 break; 1192 } 1193 1194 run->mmio.len = bytes; 1195 run->mmio.is_write = 1; 1196 vcpu->mmio_needed = 1; 1197 vcpu->mmio_is_write = 1; 1198 *(uint32_t *) data = vcpu->arch.gprs[rt]; 1199 1200 kvm_debug("[%#lx] OP_SW: eaddr: %#lx, gpr: %#lx, data: %#x\n", 1201 vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, 1202 vcpu->arch.gprs[rt], *(uint32_t *) data); 1203 break; 1204 1205 case sh_op: 1206 bytes = 2; 1207 if (bytes > sizeof(run->mmio.data)) { 1208 kvm_err("%s: bad MMIO length: %d\n", __func__, 1209 run->mmio.len); 1210 } 1211 run->mmio.phys_addr = 1212 kvm_mips_callbacks->gva_to_gpa(vcpu->arch. 1213 host_cp0_badvaddr); 1214 if (run->mmio.phys_addr == KVM_INVALID_ADDR) { 1215 er = EMULATE_FAIL; 1216 break; 1217 } 1218 1219 run->mmio.len = bytes; 1220 run->mmio.is_write = 1; 1221 vcpu->mmio_needed = 1; 1222 vcpu->mmio_is_write = 1; 1223 *(uint16_t *) data = vcpu->arch.gprs[rt]; 1224 1225 kvm_debug("[%#lx] OP_SH: eaddr: %#lx, gpr: %#lx, data: %#x\n", 1226 vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, 1227 vcpu->arch.gprs[rt], *(uint32_t *) data); 1228 break; 1229 1230 default: 1231 kvm_err("Store not yet supported"); 1232 er = EMULATE_FAIL; 1233 break; 1234 } 1235 1236 /* Rollback PC if emulation was unsuccessful */ 1237 if (er == EMULATE_FAIL) 1238 vcpu->arch.pc = curr_pc; 1239 1240 return er; 1241 } 1242 1243 enum emulation_result kvm_mips_emulate_load(uint32_t inst, uint32_t cause, 1244 struct kvm_run *run, 1245 struct kvm_vcpu *vcpu) 1246 { 1247 enum emulation_result er = EMULATE_DO_MMIO; 1248 int32_t op, base, rt, offset; 1249 uint32_t bytes; 1250 1251 rt = (inst >> 16) & 0x1f; 1252 base = (inst >> 21) & 0x1f; 1253 offset = inst & 0xffff; 1254 op = (inst >> 26) & 0x3f; 1255 1256 vcpu->arch.pending_load_cause = cause; 1257 vcpu->arch.io_gpr = rt; 1258 1259 switch (op) { 1260 case lw_op: 1261 bytes = 4; 1262 if (bytes > sizeof(run->mmio.data)) { 1263 kvm_err("%s: bad MMIO length: %d\n", __func__, 1264 run->mmio.len); 1265 er = EMULATE_FAIL; 1266 break; 1267 } 1268 run->mmio.phys_addr = 1269 kvm_mips_callbacks->gva_to_gpa(vcpu->arch. 1270 host_cp0_badvaddr); 1271 if (run->mmio.phys_addr == KVM_INVALID_ADDR) { 1272 er = EMULATE_FAIL; 1273 break; 1274 } 1275 1276 run->mmio.len = bytes; 1277 run->mmio.is_write = 0; 1278 vcpu->mmio_needed = 1; 1279 vcpu->mmio_is_write = 0; 1280 break; 1281 1282 case lh_op: 1283 case lhu_op: 1284 bytes = 2; 1285 if (bytes > sizeof(run->mmio.data)) { 1286 kvm_err("%s: bad MMIO length: %d\n", __func__, 1287 run->mmio.len); 1288 er = EMULATE_FAIL; 1289 break; 1290 } 1291 run->mmio.phys_addr = 1292 kvm_mips_callbacks->gva_to_gpa(vcpu->arch. 1293 host_cp0_badvaddr); 1294 if (run->mmio.phys_addr == KVM_INVALID_ADDR) { 1295 er = EMULATE_FAIL; 1296 break; 1297 } 1298 1299 run->mmio.len = bytes; 1300 run->mmio.is_write = 0; 1301 vcpu->mmio_needed = 1; 1302 vcpu->mmio_is_write = 0; 1303 1304 if (op == lh_op) 1305 vcpu->mmio_needed = 2; 1306 else 1307 vcpu->mmio_needed = 1; 1308 1309 break; 1310 1311 case lbu_op: 1312 case lb_op: 1313 bytes = 1; 1314 if (bytes > sizeof(run->mmio.data)) { 1315 kvm_err("%s: bad MMIO length: %d\n", __func__, 1316 run->mmio.len); 1317 er = EMULATE_FAIL; 1318 break; 1319 } 1320 run->mmio.phys_addr = 1321 kvm_mips_callbacks->gva_to_gpa(vcpu->arch. 1322 host_cp0_badvaddr); 1323 if (run->mmio.phys_addr == KVM_INVALID_ADDR) { 1324 er = EMULATE_FAIL; 1325 break; 1326 } 1327 1328 run->mmio.len = bytes; 1329 run->mmio.is_write = 0; 1330 vcpu->mmio_is_write = 0; 1331 1332 if (op == lb_op) 1333 vcpu->mmio_needed = 2; 1334 else 1335 vcpu->mmio_needed = 1; 1336 1337 break; 1338 1339 default: 1340 kvm_err("Load not yet supported"); 1341 er = EMULATE_FAIL; 1342 break; 1343 } 1344 1345 return er; 1346 } 1347 1348 int kvm_mips_sync_icache(unsigned long va, struct kvm_vcpu *vcpu) 1349 { 1350 unsigned long offset = (va & ~PAGE_MASK); 1351 struct kvm *kvm = vcpu->kvm; 1352 unsigned long pa; 1353 gfn_t gfn; 1354 pfn_t pfn; 1355 1356 gfn = va >> PAGE_SHIFT; 1357 1358 if (gfn >= kvm->arch.guest_pmap_npages) { 1359 kvm_err("%s: Invalid gfn: %#llx\n", __func__, gfn); 1360 kvm_mips_dump_host_tlbs(); 1361 kvm_arch_vcpu_dump_regs(vcpu); 1362 return -1; 1363 } 1364 pfn = kvm->arch.guest_pmap[gfn]; 1365 pa = (pfn << PAGE_SHIFT) | offset; 1366 1367 kvm_debug("%s: va: %#lx, unmapped: %#x\n", __func__, va, 1368 CKSEG0ADDR(pa)); 1369 1370 local_flush_icache_range(CKSEG0ADDR(pa), 32); 1371 return 0; 1372 } 1373 1374 #define MIPS_CACHE_OP_INDEX_INV 0x0 1375 #define MIPS_CACHE_OP_INDEX_LD_TAG 0x1 1376 #define MIPS_CACHE_OP_INDEX_ST_TAG 0x2 1377 #define MIPS_CACHE_OP_IMP 0x3 1378 #define MIPS_CACHE_OP_HIT_INV 0x4 1379 #define MIPS_CACHE_OP_FILL_WB_INV 0x5 1380 #define MIPS_CACHE_OP_HIT_HB 0x6 1381 #define MIPS_CACHE_OP_FETCH_LOCK 0x7 1382 1383 #define MIPS_CACHE_ICACHE 0x0 1384 #define MIPS_CACHE_DCACHE 0x1 1385 #define MIPS_CACHE_SEC 0x3 1386 1387 enum emulation_result kvm_mips_emulate_cache(uint32_t inst, uint32_t *opc, 1388 uint32_t cause, 1389 struct kvm_run *run, 1390 struct kvm_vcpu *vcpu) 1391 { 1392 struct mips_coproc *cop0 = vcpu->arch.cop0; 1393 enum emulation_result er = EMULATE_DONE; 1394 int32_t offset, cache, op_inst, op, base; 1395 struct kvm_vcpu_arch *arch = &vcpu->arch; 1396 unsigned long va; 1397 unsigned long curr_pc; 1398 1399 /* 1400 * Update PC and hold onto current PC in case there is 1401 * an error and we want to rollback the PC 1402 */ 1403 curr_pc = vcpu->arch.pc; 1404 er = update_pc(vcpu, cause); 1405 if (er == EMULATE_FAIL) 1406 return er; 1407 1408 base = (inst >> 21) & 0x1f; 1409 op_inst = (inst >> 16) & 0x1f; 1410 offset = inst & 0xffff; 1411 cache = (inst >> 16) & 0x3; 1412 op = (inst >> 18) & 0x7; 1413 1414 va = arch->gprs[base] + offset; 1415 1416 kvm_debug("CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n", 1417 cache, op, base, arch->gprs[base], offset); 1418 1419 /* 1420 * Treat INDEX_INV as a nop, basically issued by Linux on startup to 1421 * invalidate the caches entirely by stepping through all the 1422 * ways/indexes 1423 */ 1424 if (op == MIPS_CACHE_OP_INDEX_INV) { 1425 kvm_debug("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n", 1426 vcpu->arch.pc, vcpu->arch.gprs[31], cache, op, base, 1427 arch->gprs[base], offset); 1428 1429 if (cache == MIPS_CACHE_DCACHE) 1430 r4k_blast_dcache(); 1431 else if (cache == MIPS_CACHE_ICACHE) 1432 r4k_blast_icache(); 1433 else { 1434 kvm_err("%s: unsupported CACHE INDEX operation\n", 1435 __func__); 1436 return EMULATE_FAIL; 1437 } 1438 1439 #ifdef CONFIG_KVM_MIPS_DYN_TRANS 1440 kvm_mips_trans_cache_index(inst, opc, vcpu); 1441 #endif 1442 goto done; 1443 } 1444 1445 preempt_disable(); 1446 if (KVM_GUEST_KSEGX(va) == KVM_GUEST_KSEG0) { 1447 if (kvm_mips_host_tlb_lookup(vcpu, va) < 0) 1448 kvm_mips_handle_kseg0_tlb_fault(va, vcpu); 1449 } else if ((KVM_GUEST_KSEGX(va) < KVM_GUEST_KSEG0) || 1450 KVM_GUEST_KSEGX(va) == KVM_GUEST_KSEG23) { 1451 int index; 1452 1453 /* If an entry already exists then skip */ 1454 if (kvm_mips_host_tlb_lookup(vcpu, va) >= 0) 1455 goto skip_fault; 1456 1457 /* 1458 * If address not in the guest TLB, then give the guest a fault, 1459 * the resulting handler will do the right thing 1460 */ 1461 index = kvm_mips_guest_tlb_lookup(vcpu, (va & VPN2_MASK) | 1462 (kvm_read_c0_guest_entryhi 1463 (cop0) & ASID_MASK)); 1464 1465 if (index < 0) { 1466 vcpu->arch.host_cp0_entryhi = (va & VPN2_MASK); 1467 vcpu->arch.host_cp0_badvaddr = va; 1468 er = kvm_mips_emulate_tlbmiss_ld(cause, NULL, run, 1469 vcpu); 1470 preempt_enable(); 1471 goto dont_update_pc; 1472 } else { 1473 struct kvm_mips_tlb *tlb = &vcpu->arch.guest_tlb[index]; 1474 /* 1475 * Check if the entry is valid, if not then setup a TLB 1476 * invalid exception to the guest 1477 */ 1478 if (!TLB_IS_VALID(*tlb, va)) { 1479 er = kvm_mips_emulate_tlbinv_ld(cause, NULL, 1480 run, vcpu); 1481 preempt_enable(); 1482 goto dont_update_pc; 1483 } else { 1484 /* 1485 * We fault an entry from the guest tlb to the 1486 * shadow host TLB 1487 */ 1488 kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb, 1489 NULL, 1490 NULL); 1491 } 1492 } 1493 } else { 1494 kvm_err("INVALID CACHE INDEX/ADDRESS (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n", 1495 cache, op, base, arch->gprs[base], offset); 1496 er = EMULATE_FAIL; 1497 preempt_enable(); 1498 goto dont_update_pc; 1499 1500 } 1501 1502 skip_fault: 1503 /* XXXKYMA: Only a subset of cache ops are supported, used by Linux */ 1504 if (cache == MIPS_CACHE_DCACHE 1505 && (op == MIPS_CACHE_OP_FILL_WB_INV 1506 || op == MIPS_CACHE_OP_HIT_INV)) { 1507 flush_dcache_line(va); 1508 1509 #ifdef CONFIG_KVM_MIPS_DYN_TRANS 1510 /* 1511 * Replace the CACHE instruction, with a SYNCI, not the same, 1512 * but avoids a trap 1513 */ 1514 kvm_mips_trans_cache_va(inst, opc, vcpu); 1515 #endif 1516 } else if (op == MIPS_CACHE_OP_HIT_INV && cache == MIPS_CACHE_ICACHE) { 1517 flush_dcache_line(va); 1518 flush_icache_line(va); 1519 1520 #ifdef CONFIG_KVM_MIPS_DYN_TRANS 1521 /* Replace the CACHE instruction, with a SYNCI */ 1522 kvm_mips_trans_cache_va(inst, opc, vcpu); 1523 #endif 1524 } else { 1525 kvm_err("NO-OP CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n", 1526 cache, op, base, arch->gprs[base], offset); 1527 er = EMULATE_FAIL; 1528 preempt_enable(); 1529 goto dont_update_pc; 1530 } 1531 1532 preempt_enable(); 1533 1534 dont_update_pc: 1535 /* Rollback PC */ 1536 vcpu->arch.pc = curr_pc; 1537 done: 1538 return er; 1539 } 1540 1541 enum emulation_result kvm_mips_emulate_inst(unsigned long cause, uint32_t *opc, 1542 struct kvm_run *run, 1543 struct kvm_vcpu *vcpu) 1544 { 1545 enum emulation_result er = EMULATE_DONE; 1546 uint32_t inst; 1547 1548 /* Fetch the instruction. */ 1549 if (cause & CAUSEF_BD) 1550 opc += 1; 1551 1552 inst = kvm_get_inst(opc, vcpu); 1553 1554 switch (((union mips_instruction)inst).r_format.opcode) { 1555 case cop0_op: 1556 er = kvm_mips_emulate_CP0(inst, opc, cause, run, vcpu); 1557 break; 1558 case sb_op: 1559 case sh_op: 1560 case sw_op: 1561 er = kvm_mips_emulate_store(inst, cause, run, vcpu); 1562 break; 1563 case lb_op: 1564 case lbu_op: 1565 case lhu_op: 1566 case lh_op: 1567 case lw_op: 1568 er = kvm_mips_emulate_load(inst, cause, run, vcpu); 1569 break; 1570 1571 case cache_op: 1572 ++vcpu->stat.cache_exits; 1573 trace_kvm_exit(vcpu, CACHE_EXITS); 1574 er = kvm_mips_emulate_cache(inst, opc, cause, run, vcpu); 1575 break; 1576 1577 default: 1578 kvm_err("Instruction emulation not supported (%p/%#x)\n", opc, 1579 inst); 1580 kvm_arch_vcpu_dump_regs(vcpu); 1581 er = EMULATE_FAIL; 1582 break; 1583 } 1584 1585 return er; 1586 } 1587 1588 enum emulation_result kvm_mips_emulate_syscall(unsigned long cause, 1589 uint32_t *opc, 1590 struct kvm_run *run, 1591 struct kvm_vcpu *vcpu) 1592 { 1593 struct mips_coproc *cop0 = vcpu->arch.cop0; 1594 struct kvm_vcpu_arch *arch = &vcpu->arch; 1595 enum emulation_result er = EMULATE_DONE; 1596 1597 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) { 1598 /* save old pc */ 1599 kvm_write_c0_guest_epc(cop0, arch->pc); 1600 kvm_set_c0_guest_status(cop0, ST0_EXL); 1601 1602 if (cause & CAUSEF_BD) 1603 kvm_set_c0_guest_cause(cop0, CAUSEF_BD); 1604 else 1605 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD); 1606 1607 kvm_debug("Delivering SYSCALL @ pc %#lx\n", arch->pc); 1608 1609 kvm_change_c0_guest_cause(cop0, (0xff), 1610 (T_SYSCALL << CAUSEB_EXCCODE)); 1611 1612 /* Set PC to the exception entry point */ 1613 arch->pc = KVM_GUEST_KSEG0 + 0x180; 1614 1615 } else { 1616 kvm_err("Trying to deliver SYSCALL when EXL is already set\n"); 1617 er = EMULATE_FAIL; 1618 } 1619 1620 return er; 1621 } 1622 1623 enum emulation_result kvm_mips_emulate_tlbmiss_ld(unsigned long cause, 1624 uint32_t *opc, 1625 struct kvm_run *run, 1626 struct kvm_vcpu *vcpu) 1627 { 1628 struct mips_coproc *cop0 = vcpu->arch.cop0; 1629 struct kvm_vcpu_arch *arch = &vcpu->arch; 1630 unsigned long entryhi = (vcpu->arch. host_cp0_badvaddr & VPN2_MASK) | 1631 (kvm_read_c0_guest_entryhi(cop0) & ASID_MASK); 1632 1633 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) { 1634 /* save old pc */ 1635 kvm_write_c0_guest_epc(cop0, arch->pc); 1636 kvm_set_c0_guest_status(cop0, ST0_EXL); 1637 1638 if (cause & CAUSEF_BD) 1639 kvm_set_c0_guest_cause(cop0, CAUSEF_BD); 1640 else 1641 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD); 1642 1643 kvm_debug("[EXL == 0] delivering TLB MISS @ pc %#lx\n", 1644 arch->pc); 1645 1646 /* set pc to the exception entry point */ 1647 arch->pc = KVM_GUEST_KSEG0 + 0x0; 1648 1649 } else { 1650 kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n", 1651 arch->pc); 1652 1653 arch->pc = KVM_GUEST_KSEG0 + 0x180; 1654 } 1655 1656 kvm_change_c0_guest_cause(cop0, (0xff), 1657 (T_TLB_LD_MISS << CAUSEB_EXCCODE)); 1658 1659 /* setup badvaddr, context and entryhi registers for the guest */ 1660 kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr); 1661 /* XXXKYMA: is the context register used by linux??? */ 1662 kvm_write_c0_guest_entryhi(cop0, entryhi); 1663 /* Blow away the shadow host TLBs */ 1664 kvm_mips_flush_host_tlb(1); 1665 1666 return EMULATE_DONE; 1667 } 1668 1669 enum emulation_result kvm_mips_emulate_tlbinv_ld(unsigned long cause, 1670 uint32_t *opc, 1671 struct kvm_run *run, 1672 struct kvm_vcpu *vcpu) 1673 { 1674 struct mips_coproc *cop0 = vcpu->arch.cop0; 1675 struct kvm_vcpu_arch *arch = &vcpu->arch; 1676 unsigned long entryhi = 1677 (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) | 1678 (kvm_read_c0_guest_entryhi(cop0) & ASID_MASK); 1679 1680 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) { 1681 /* save old pc */ 1682 kvm_write_c0_guest_epc(cop0, arch->pc); 1683 kvm_set_c0_guest_status(cop0, ST0_EXL); 1684 1685 if (cause & CAUSEF_BD) 1686 kvm_set_c0_guest_cause(cop0, CAUSEF_BD); 1687 else 1688 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD); 1689 1690 kvm_debug("[EXL == 0] delivering TLB INV @ pc %#lx\n", 1691 arch->pc); 1692 1693 /* set pc to the exception entry point */ 1694 arch->pc = KVM_GUEST_KSEG0 + 0x180; 1695 1696 } else { 1697 kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n", 1698 arch->pc); 1699 arch->pc = KVM_GUEST_KSEG0 + 0x180; 1700 } 1701 1702 kvm_change_c0_guest_cause(cop0, (0xff), 1703 (T_TLB_LD_MISS << CAUSEB_EXCCODE)); 1704 1705 /* setup badvaddr, context and entryhi registers for the guest */ 1706 kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr); 1707 /* XXXKYMA: is the context register used by linux??? */ 1708 kvm_write_c0_guest_entryhi(cop0, entryhi); 1709 /* Blow away the shadow host TLBs */ 1710 kvm_mips_flush_host_tlb(1); 1711 1712 return EMULATE_DONE; 1713 } 1714 1715 enum emulation_result kvm_mips_emulate_tlbmiss_st(unsigned long cause, 1716 uint32_t *opc, 1717 struct kvm_run *run, 1718 struct kvm_vcpu *vcpu) 1719 { 1720 struct mips_coproc *cop0 = vcpu->arch.cop0; 1721 struct kvm_vcpu_arch *arch = &vcpu->arch; 1722 unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) | 1723 (kvm_read_c0_guest_entryhi(cop0) & ASID_MASK); 1724 1725 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) { 1726 /* save old pc */ 1727 kvm_write_c0_guest_epc(cop0, arch->pc); 1728 kvm_set_c0_guest_status(cop0, ST0_EXL); 1729 1730 if (cause & CAUSEF_BD) 1731 kvm_set_c0_guest_cause(cop0, CAUSEF_BD); 1732 else 1733 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD); 1734 1735 kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n", 1736 arch->pc); 1737 1738 /* Set PC to the exception entry point */ 1739 arch->pc = KVM_GUEST_KSEG0 + 0x0; 1740 } else { 1741 kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n", 1742 arch->pc); 1743 arch->pc = KVM_GUEST_KSEG0 + 0x180; 1744 } 1745 1746 kvm_change_c0_guest_cause(cop0, (0xff), 1747 (T_TLB_ST_MISS << CAUSEB_EXCCODE)); 1748 1749 /* setup badvaddr, context and entryhi registers for the guest */ 1750 kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr); 1751 /* XXXKYMA: is the context register used by linux??? */ 1752 kvm_write_c0_guest_entryhi(cop0, entryhi); 1753 /* Blow away the shadow host TLBs */ 1754 kvm_mips_flush_host_tlb(1); 1755 1756 return EMULATE_DONE; 1757 } 1758 1759 enum emulation_result kvm_mips_emulate_tlbinv_st(unsigned long cause, 1760 uint32_t *opc, 1761 struct kvm_run *run, 1762 struct kvm_vcpu *vcpu) 1763 { 1764 struct mips_coproc *cop0 = vcpu->arch.cop0; 1765 struct kvm_vcpu_arch *arch = &vcpu->arch; 1766 unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) | 1767 (kvm_read_c0_guest_entryhi(cop0) & ASID_MASK); 1768 1769 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) { 1770 /* save old pc */ 1771 kvm_write_c0_guest_epc(cop0, arch->pc); 1772 kvm_set_c0_guest_status(cop0, ST0_EXL); 1773 1774 if (cause & CAUSEF_BD) 1775 kvm_set_c0_guest_cause(cop0, CAUSEF_BD); 1776 else 1777 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD); 1778 1779 kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n", 1780 arch->pc); 1781 1782 /* Set PC to the exception entry point */ 1783 arch->pc = KVM_GUEST_KSEG0 + 0x180; 1784 } else { 1785 kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n", 1786 arch->pc); 1787 arch->pc = KVM_GUEST_KSEG0 + 0x180; 1788 } 1789 1790 kvm_change_c0_guest_cause(cop0, (0xff), 1791 (T_TLB_ST_MISS << CAUSEB_EXCCODE)); 1792 1793 /* setup badvaddr, context and entryhi registers for the guest */ 1794 kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr); 1795 /* XXXKYMA: is the context register used by linux??? */ 1796 kvm_write_c0_guest_entryhi(cop0, entryhi); 1797 /* Blow away the shadow host TLBs */ 1798 kvm_mips_flush_host_tlb(1); 1799 1800 return EMULATE_DONE; 1801 } 1802 1803 /* TLBMOD: store into address matching TLB with Dirty bit off */ 1804 enum emulation_result kvm_mips_handle_tlbmod(unsigned long cause, uint32_t *opc, 1805 struct kvm_run *run, 1806 struct kvm_vcpu *vcpu) 1807 { 1808 enum emulation_result er = EMULATE_DONE; 1809 #ifdef DEBUG 1810 struct mips_coproc *cop0 = vcpu->arch.cop0; 1811 unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) | 1812 (kvm_read_c0_guest_entryhi(cop0) & ASID_MASK); 1813 int index; 1814 1815 /* If address not in the guest TLB, then we are in trouble */ 1816 index = kvm_mips_guest_tlb_lookup(vcpu, entryhi); 1817 if (index < 0) { 1818 /* XXXKYMA Invalidate and retry */ 1819 kvm_mips_host_tlb_inv(vcpu, vcpu->arch.host_cp0_badvaddr); 1820 kvm_err("%s: host got TLBMOD for %#lx but entry not present in Guest TLB\n", 1821 __func__, entryhi); 1822 kvm_mips_dump_guest_tlbs(vcpu); 1823 kvm_mips_dump_host_tlbs(); 1824 return EMULATE_FAIL; 1825 } 1826 #endif 1827 1828 er = kvm_mips_emulate_tlbmod(cause, opc, run, vcpu); 1829 return er; 1830 } 1831 1832 enum emulation_result kvm_mips_emulate_tlbmod(unsigned long cause, 1833 uint32_t *opc, 1834 struct kvm_run *run, 1835 struct kvm_vcpu *vcpu) 1836 { 1837 struct mips_coproc *cop0 = vcpu->arch.cop0; 1838 unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) | 1839 (kvm_read_c0_guest_entryhi(cop0) & ASID_MASK); 1840 struct kvm_vcpu_arch *arch = &vcpu->arch; 1841 1842 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) { 1843 /* save old pc */ 1844 kvm_write_c0_guest_epc(cop0, arch->pc); 1845 kvm_set_c0_guest_status(cop0, ST0_EXL); 1846 1847 if (cause & CAUSEF_BD) 1848 kvm_set_c0_guest_cause(cop0, CAUSEF_BD); 1849 else 1850 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD); 1851 1852 kvm_debug("[EXL == 0] Delivering TLB MOD @ pc %#lx\n", 1853 arch->pc); 1854 1855 arch->pc = KVM_GUEST_KSEG0 + 0x180; 1856 } else { 1857 kvm_debug("[EXL == 1] Delivering TLB MOD @ pc %#lx\n", 1858 arch->pc); 1859 arch->pc = KVM_GUEST_KSEG0 + 0x180; 1860 } 1861 1862 kvm_change_c0_guest_cause(cop0, (0xff), (T_TLB_MOD << CAUSEB_EXCCODE)); 1863 1864 /* setup badvaddr, context and entryhi registers for the guest */ 1865 kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr); 1866 /* XXXKYMA: is the context register used by linux??? */ 1867 kvm_write_c0_guest_entryhi(cop0, entryhi); 1868 /* Blow away the shadow host TLBs */ 1869 kvm_mips_flush_host_tlb(1); 1870 1871 return EMULATE_DONE; 1872 } 1873 1874 enum emulation_result kvm_mips_emulate_fpu_exc(unsigned long cause, 1875 uint32_t *opc, 1876 struct kvm_run *run, 1877 struct kvm_vcpu *vcpu) 1878 { 1879 struct mips_coproc *cop0 = vcpu->arch.cop0; 1880 struct kvm_vcpu_arch *arch = &vcpu->arch; 1881 1882 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) { 1883 /* save old pc */ 1884 kvm_write_c0_guest_epc(cop0, arch->pc); 1885 kvm_set_c0_guest_status(cop0, ST0_EXL); 1886 1887 if (cause & CAUSEF_BD) 1888 kvm_set_c0_guest_cause(cop0, CAUSEF_BD); 1889 else 1890 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD); 1891 1892 } 1893 1894 arch->pc = KVM_GUEST_KSEG0 + 0x180; 1895 1896 kvm_change_c0_guest_cause(cop0, (0xff), 1897 (T_COP_UNUSABLE << CAUSEB_EXCCODE)); 1898 kvm_change_c0_guest_cause(cop0, (CAUSEF_CE), (0x1 << CAUSEB_CE)); 1899 1900 return EMULATE_DONE; 1901 } 1902 1903 enum emulation_result kvm_mips_emulate_ri_exc(unsigned long cause, 1904 uint32_t *opc, 1905 struct kvm_run *run, 1906 struct kvm_vcpu *vcpu) 1907 { 1908 struct mips_coproc *cop0 = vcpu->arch.cop0; 1909 struct kvm_vcpu_arch *arch = &vcpu->arch; 1910 enum emulation_result er = EMULATE_DONE; 1911 1912 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) { 1913 /* save old pc */ 1914 kvm_write_c0_guest_epc(cop0, arch->pc); 1915 kvm_set_c0_guest_status(cop0, ST0_EXL); 1916 1917 if (cause & CAUSEF_BD) 1918 kvm_set_c0_guest_cause(cop0, CAUSEF_BD); 1919 else 1920 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD); 1921 1922 kvm_debug("Delivering RI @ pc %#lx\n", arch->pc); 1923 1924 kvm_change_c0_guest_cause(cop0, (0xff), 1925 (T_RES_INST << CAUSEB_EXCCODE)); 1926 1927 /* Set PC to the exception entry point */ 1928 arch->pc = KVM_GUEST_KSEG0 + 0x180; 1929 1930 } else { 1931 kvm_err("Trying to deliver RI when EXL is already set\n"); 1932 er = EMULATE_FAIL; 1933 } 1934 1935 return er; 1936 } 1937 1938 enum emulation_result kvm_mips_emulate_bp_exc(unsigned long cause, 1939 uint32_t *opc, 1940 struct kvm_run *run, 1941 struct kvm_vcpu *vcpu) 1942 { 1943 struct mips_coproc *cop0 = vcpu->arch.cop0; 1944 struct kvm_vcpu_arch *arch = &vcpu->arch; 1945 enum emulation_result er = EMULATE_DONE; 1946 1947 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) { 1948 /* save old pc */ 1949 kvm_write_c0_guest_epc(cop0, arch->pc); 1950 kvm_set_c0_guest_status(cop0, ST0_EXL); 1951 1952 if (cause & CAUSEF_BD) 1953 kvm_set_c0_guest_cause(cop0, CAUSEF_BD); 1954 else 1955 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD); 1956 1957 kvm_debug("Delivering BP @ pc %#lx\n", arch->pc); 1958 1959 kvm_change_c0_guest_cause(cop0, (0xff), 1960 (T_BREAK << CAUSEB_EXCCODE)); 1961 1962 /* Set PC to the exception entry point */ 1963 arch->pc = KVM_GUEST_KSEG0 + 0x180; 1964 1965 } else { 1966 kvm_err("Trying to deliver BP when EXL is already set\n"); 1967 er = EMULATE_FAIL; 1968 } 1969 1970 return er; 1971 } 1972 1973 /* ll/sc, rdhwr, sync emulation */ 1974 1975 #define OPCODE 0xfc000000 1976 #define BASE 0x03e00000 1977 #define RT 0x001f0000 1978 #define OFFSET 0x0000ffff 1979 #define LL 0xc0000000 1980 #define SC 0xe0000000 1981 #define SPEC0 0x00000000 1982 #define SPEC3 0x7c000000 1983 #define RD 0x0000f800 1984 #define FUNC 0x0000003f 1985 #define SYNC 0x0000000f 1986 #define RDHWR 0x0000003b 1987 1988 enum emulation_result kvm_mips_handle_ri(unsigned long cause, uint32_t *opc, 1989 struct kvm_run *run, 1990 struct kvm_vcpu *vcpu) 1991 { 1992 struct mips_coproc *cop0 = vcpu->arch.cop0; 1993 struct kvm_vcpu_arch *arch = &vcpu->arch; 1994 enum emulation_result er = EMULATE_DONE; 1995 unsigned long curr_pc; 1996 uint32_t inst; 1997 1998 /* 1999 * Update PC and hold onto current PC in case there is 2000 * an error and we want to rollback the PC 2001 */ 2002 curr_pc = vcpu->arch.pc; 2003 er = update_pc(vcpu, cause); 2004 if (er == EMULATE_FAIL) 2005 return er; 2006 2007 /* Fetch the instruction. */ 2008 if (cause & CAUSEF_BD) 2009 opc += 1; 2010 2011 inst = kvm_get_inst(opc, vcpu); 2012 2013 if (inst == KVM_INVALID_INST) { 2014 kvm_err("%s: Cannot get inst @ %p\n", __func__, opc); 2015 return EMULATE_FAIL; 2016 } 2017 2018 if ((inst & OPCODE) == SPEC3 && (inst & FUNC) == RDHWR) { 2019 int usermode = !KVM_GUEST_KERNEL_MODE(vcpu); 2020 int rd = (inst & RD) >> 11; 2021 int rt = (inst & RT) >> 16; 2022 /* If usermode, check RDHWR rd is allowed by guest HWREna */ 2023 if (usermode && !(kvm_read_c0_guest_hwrena(cop0) & BIT(rd))) { 2024 kvm_debug("RDHWR %#x disallowed by HWREna @ %p\n", 2025 rd, opc); 2026 goto emulate_ri; 2027 } 2028 switch (rd) { 2029 case 0: /* CPU number */ 2030 arch->gprs[rt] = 0; 2031 break; 2032 case 1: /* SYNCI length */ 2033 arch->gprs[rt] = min(current_cpu_data.dcache.linesz, 2034 current_cpu_data.icache.linesz); 2035 break; 2036 case 2: /* Read count register */ 2037 arch->gprs[rt] = kvm_mips_read_count(vcpu); 2038 break; 2039 case 3: /* Count register resolution */ 2040 switch (current_cpu_data.cputype) { 2041 case CPU_20KC: 2042 case CPU_25KF: 2043 arch->gprs[rt] = 1; 2044 break; 2045 default: 2046 arch->gprs[rt] = 2; 2047 } 2048 break; 2049 case 29: 2050 arch->gprs[rt] = kvm_read_c0_guest_userlocal(cop0); 2051 break; 2052 2053 default: 2054 kvm_debug("RDHWR %#x not supported @ %p\n", rd, opc); 2055 goto emulate_ri; 2056 } 2057 } else { 2058 kvm_debug("Emulate RI not supported @ %p: %#x\n", opc, inst); 2059 goto emulate_ri; 2060 } 2061 2062 return EMULATE_DONE; 2063 2064 emulate_ri: 2065 /* 2066 * Rollback PC (if in branch delay slot then the PC already points to 2067 * branch target), and pass the RI exception to the guest OS. 2068 */ 2069 vcpu->arch.pc = curr_pc; 2070 return kvm_mips_emulate_ri_exc(cause, opc, run, vcpu); 2071 } 2072 2073 enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu, 2074 struct kvm_run *run) 2075 { 2076 unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr]; 2077 enum emulation_result er = EMULATE_DONE; 2078 unsigned long curr_pc; 2079 2080 if (run->mmio.len > sizeof(*gpr)) { 2081 kvm_err("Bad MMIO length: %d", run->mmio.len); 2082 er = EMULATE_FAIL; 2083 goto done; 2084 } 2085 2086 /* 2087 * Update PC and hold onto current PC in case there is 2088 * an error and we want to rollback the PC 2089 */ 2090 curr_pc = vcpu->arch.pc; 2091 er = update_pc(vcpu, vcpu->arch.pending_load_cause); 2092 if (er == EMULATE_FAIL) 2093 return er; 2094 2095 switch (run->mmio.len) { 2096 case 4: 2097 *gpr = *(int32_t *) run->mmio.data; 2098 break; 2099 2100 case 2: 2101 if (vcpu->mmio_needed == 2) 2102 *gpr = *(int16_t *) run->mmio.data; 2103 else 2104 *gpr = *(int16_t *) run->mmio.data; 2105 2106 break; 2107 case 1: 2108 if (vcpu->mmio_needed == 2) 2109 *gpr = *(int8_t *) run->mmio.data; 2110 else 2111 *gpr = *(u8 *) run->mmio.data; 2112 break; 2113 } 2114 2115 if (vcpu->arch.pending_load_cause & CAUSEF_BD) 2116 kvm_debug("[%#lx] Completing %d byte BD Load to gpr %d (0x%08lx) type %d\n", 2117 vcpu->arch.pc, run->mmio.len, vcpu->arch.io_gpr, *gpr, 2118 vcpu->mmio_needed); 2119 2120 done: 2121 return er; 2122 } 2123 2124 static enum emulation_result kvm_mips_emulate_exc(unsigned long cause, 2125 uint32_t *opc, 2126 struct kvm_run *run, 2127 struct kvm_vcpu *vcpu) 2128 { 2129 uint32_t exccode = (cause >> CAUSEB_EXCCODE) & 0x1f; 2130 struct mips_coproc *cop0 = vcpu->arch.cop0; 2131 struct kvm_vcpu_arch *arch = &vcpu->arch; 2132 enum emulation_result er = EMULATE_DONE; 2133 2134 if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) { 2135 /* save old pc */ 2136 kvm_write_c0_guest_epc(cop0, arch->pc); 2137 kvm_set_c0_guest_status(cop0, ST0_EXL); 2138 2139 if (cause & CAUSEF_BD) 2140 kvm_set_c0_guest_cause(cop0, CAUSEF_BD); 2141 else 2142 kvm_clear_c0_guest_cause(cop0, CAUSEF_BD); 2143 2144 kvm_change_c0_guest_cause(cop0, (0xff), 2145 (exccode << CAUSEB_EXCCODE)); 2146 2147 /* Set PC to the exception entry point */ 2148 arch->pc = KVM_GUEST_KSEG0 + 0x180; 2149 kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr); 2150 2151 kvm_debug("Delivering EXC %d @ pc %#lx, badVaddr: %#lx\n", 2152 exccode, kvm_read_c0_guest_epc(cop0), 2153 kvm_read_c0_guest_badvaddr(cop0)); 2154 } else { 2155 kvm_err("Trying to deliver EXC when EXL is already set\n"); 2156 er = EMULATE_FAIL; 2157 } 2158 2159 return er; 2160 } 2161 2162 enum emulation_result kvm_mips_check_privilege(unsigned long cause, 2163 uint32_t *opc, 2164 struct kvm_run *run, 2165 struct kvm_vcpu *vcpu) 2166 { 2167 enum emulation_result er = EMULATE_DONE; 2168 uint32_t exccode = (cause >> CAUSEB_EXCCODE) & 0x1f; 2169 unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr; 2170 2171 int usermode = !KVM_GUEST_KERNEL_MODE(vcpu); 2172 2173 if (usermode) { 2174 switch (exccode) { 2175 case T_INT: 2176 case T_SYSCALL: 2177 case T_BREAK: 2178 case T_RES_INST: 2179 break; 2180 2181 case T_COP_UNUSABLE: 2182 if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 0) 2183 er = EMULATE_PRIV_FAIL; 2184 break; 2185 2186 case T_TLB_MOD: 2187 break; 2188 2189 case T_TLB_LD_MISS: 2190 /* 2191 * We we are accessing Guest kernel space, then send an 2192 * address error exception to the guest 2193 */ 2194 if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) { 2195 kvm_debug("%s: LD MISS @ %#lx\n", __func__, 2196 badvaddr); 2197 cause &= ~0xff; 2198 cause |= (T_ADDR_ERR_LD << CAUSEB_EXCCODE); 2199 er = EMULATE_PRIV_FAIL; 2200 } 2201 break; 2202 2203 case T_TLB_ST_MISS: 2204 /* 2205 * We we are accessing Guest kernel space, then send an 2206 * address error exception to the guest 2207 */ 2208 if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) { 2209 kvm_debug("%s: ST MISS @ %#lx\n", __func__, 2210 badvaddr); 2211 cause &= ~0xff; 2212 cause |= (T_ADDR_ERR_ST << CAUSEB_EXCCODE); 2213 er = EMULATE_PRIV_FAIL; 2214 } 2215 break; 2216 2217 case T_ADDR_ERR_ST: 2218 kvm_debug("%s: address error ST @ %#lx\n", __func__, 2219 badvaddr); 2220 if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) { 2221 cause &= ~0xff; 2222 cause |= (T_TLB_ST_MISS << CAUSEB_EXCCODE); 2223 } 2224 er = EMULATE_PRIV_FAIL; 2225 break; 2226 case T_ADDR_ERR_LD: 2227 kvm_debug("%s: address error LD @ %#lx\n", __func__, 2228 badvaddr); 2229 if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) { 2230 cause &= ~0xff; 2231 cause |= (T_TLB_LD_MISS << CAUSEB_EXCCODE); 2232 } 2233 er = EMULATE_PRIV_FAIL; 2234 break; 2235 default: 2236 er = EMULATE_PRIV_FAIL; 2237 break; 2238 } 2239 } 2240 2241 if (er == EMULATE_PRIV_FAIL) 2242 kvm_mips_emulate_exc(cause, opc, run, vcpu); 2243 2244 return er; 2245 } 2246 2247 /* 2248 * User Address (UA) fault, this could happen if 2249 * (1) TLB entry not present/valid in both Guest and shadow host TLBs, in this 2250 * case we pass on the fault to the guest kernel and let it handle it. 2251 * (2) TLB entry is present in the Guest TLB but not in the shadow, in this 2252 * case we inject the TLB from the Guest TLB into the shadow host TLB 2253 */ 2254 enum emulation_result kvm_mips_handle_tlbmiss(unsigned long cause, 2255 uint32_t *opc, 2256 struct kvm_run *run, 2257 struct kvm_vcpu *vcpu) 2258 { 2259 enum emulation_result er = EMULATE_DONE; 2260 uint32_t exccode = (cause >> CAUSEB_EXCCODE) & 0x1f; 2261 unsigned long va = vcpu->arch.host_cp0_badvaddr; 2262 int index; 2263 2264 kvm_debug("kvm_mips_handle_tlbmiss: badvaddr: %#lx, entryhi: %#lx\n", 2265 vcpu->arch.host_cp0_badvaddr, vcpu->arch.host_cp0_entryhi); 2266 2267 /* 2268 * KVM would not have got the exception if this entry was valid in the 2269 * shadow host TLB. Check the Guest TLB, if the entry is not there then 2270 * send the guest an exception. The guest exc handler should then inject 2271 * an entry into the guest TLB. 2272 */ 2273 index = kvm_mips_guest_tlb_lookup(vcpu, 2274 (va & VPN2_MASK) | 2275 (kvm_read_c0_guest_entryhi 2276 (vcpu->arch.cop0) & ASID_MASK)); 2277 if (index < 0) { 2278 if (exccode == T_TLB_LD_MISS) { 2279 er = kvm_mips_emulate_tlbmiss_ld(cause, opc, run, vcpu); 2280 } else if (exccode == T_TLB_ST_MISS) { 2281 er = kvm_mips_emulate_tlbmiss_st(cause, opc, run, vcpu); 2282 } else { 2283 kvm_err("%s: invalid exc code: %d\n", __func__, 2284 exccode); 2285 er = EMULATE_FAIL; 2286 } 2287 } else { 2288 struct kvm_mips_tlb *tlb = &vcpu->arch.guest_tlb[index]; 2289 2290 /* 2291 * Check if the entry is valid, if not then setup a TLB invalid 2292 * exception to the guest 2293 */ 2294 if (!TLB_IS_VALID(*tlb, va)) { 2295 if (exccode == T_TLB_LD_MISS) { 2296 er = kvm_mips_emulate_tlbinv_ld(cause, opc, run, 2297 vcpu); 2298 } else if (exccode == T_TLB_ST_MISS) { 2299 er = kvm_mips_emulate_tlbinv_st(cause, opc, run, 2300 vcpu); 2301 } else { 2302 kvm_err("%s: invalid exc code: %d\n", __func__, 2303 exccode); 2304 er = EMULATE_FAIL; 2305 } 2306 } else { 2307 kvm_debug("Injecting hi: %#lx, lo0: %#lx, lo1: %#lx into shadow host TLB\n", 2308 tlb->tlb_hi, tlb->tlb_lo0, tlb->tlb_lo1); 2309 /* 2310 * OK we have a Guest TLB entry, now inject it into the 2311 * shadow host TLB 2312 */ 2313 kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb, NULL, 2314 NULL); 2315 } 2316 } 2317 2318 return er; 2319 } 2320