1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that 7 * TLB handlers run from KSEG0 8 * 9 * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. 10 * Authors: Sanjay Lal <sanjayl@kymasys.com> 11 */ 12 13 #include <linux/sched.h> 14 #include <linux/smp.h> 15 #include <linux/mm.h> 16 #include <linux/delay.h> 17 #include <linux/export.h> 18 #include <linux/kvm_host.h> 19 #include <linux/srcu.h> 20 21 #include <asm/cpu.h> 22 #include <asm/bootinfo.h> 23 #include <asm/mmu_context.h> 24 #include <asm/pgtable.h> 25 #include <asm/cacheflush.h> 26 #include <asm/tlb.h> 27 #include <asm/tlbdebug.h> 28 29 #undef CONFIG_MIPS_MT 30 #include <asm/r4kcache.h> 31 #define CONFIG_MIPS_MT 32 33 #define KVM_GUEST_PC_TLB 0 34 #define KVM_GUEST_SP_TLB 1 35 36 #ifdef CONFIG_KVM_MIPS_VZ 37 unsigned long GUESTID_MASK; 38 EXPORT_SYMBOL_GPL(GUESTID_MASK); 39 unsigned long GUESTID_FIRST_VERSION; 40 EXPORT_SYMBOL_GPL(GUESTID_FIRST_VERSION); 41 unsigned long GUESTID_VERSION_MASK; 42 EXPORT_SYMBOL_GPL(GUESTID_VERSION_MASK); 43 44 static u32 kvm_mips_get_root_asid(struct kvm_vcpu *vcpu) 45 { 46 struct mm_struct *gpa_mm = &vcpu->kvm->arch.gpa_mm; 47 48 if (cpu_has_guestid) 49 return 0; 50 else 51 return cpu_asid(smp_processor_id(), gpa_mm); 52 } 53 #endif 54 55 static u32 kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu) 56 { 57 struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm; 58 int cpu = smp_processor_id(); 59 60 return cpu_asid(cpu, kern_mm); 61 } 62 63 static u32 kvm_mips_get_user_asid(struct kvm_vcpu *vcpu) 64 { 65 struct mm_struct *user_mm = &vcpu->arch.guest_user_mm; 66 int cpu = smp_processor_id(); 67 68 return cpu_asid(cpu, user_mm); 69 } 70 71 /* Structure defining an tlb entry data set. */ 72 73 void kvm_mips_dump_host_tlbs(void) 74 { 75 unsigned long flags; 76 77 local_irq_save(flags); 78 79 kvm_info("HOST TLBs:\n"); 80 dump_tlb_regs(); 81 pr_info("\n"); 82 dump_tlb_all(); 83 84 local_irq_restore(flags); 85 } 86 EXPORT_SYMBOL_GPL(kvm_mips_dump_host_tlbs); 87 88 void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu) 89 { 90 struct mips_coproc *cop0 = vcpu->arch.cop0; 91 struct kvm_mips_tlb tlb; 92 int i; 93 94 kvm_info("Guest TLBs:\n"); 95 kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0)); 96 97 for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) { 98 tlb = vcpu->arch.guest_tlb[i]; 99 kvm_info("TLB%c%3d Hi 0x%08lx ", 100 (tlb.tlb_lo[0] | tlb.tlb_lo[1]) & ENTRYLO_V 101 ? ' ' : '*', 102 i, tlb.tlb_hi); 103 kvm_info("Lo0=0x%09llx %c%c attr %lx ", 104 (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[0]), 105 (tlb.tlb_lo[0] & ENTRYLO_D) ? 'D' : ' ', 106 (tlb.tlb_lo[0] & ENTRYLO_G) ? 'G' : ' ', 107 (tlb.tlb_lo[0] & ENTRYLO_C) >> ENTRYLO_C_SHIFT); 108 kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n", 109 (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[1]), 110 (tlb.tlb_lo[1] & ENTRYLO_D) ? 'D' : ' ', 111 (tlb.tlb_lo[1] & ENTRYLO_G) ? 'G' : ' ', 112 (tlb.tlb_lo[1] & ENTRYLO_C) >> ENTRYLO_C_SHIFT, 113 tlb.tlb_mask); 114 } 115 } 116 EXPORT_SYMBOL_GPL(kvm_mips_dump_guest_tlbs); 117 118 int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi) 119 { 120 int i; 121 int index = -1; 122 struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb; 123 124 for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) { 125 if (TLB_HI_VPN2_HIT(tlb[i], entryhi) && 126 TLB_HI_ASID_HIT(tlb[i], entryhi)) { 127 index = i; 128 break; 129 } 130 } 131 132 kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n", 133 __func__, entryhi, index, tlb[i].tlb_lo[0], tlb[i].tlb_lo[1]); 134 135 return index; 136 } 137 EXPORT_SYMBOL_GPL(kvm_mips_guest_tlb_lookup); 138 139 static int _kvm_mips_host_tlb_inv(unsigned long entryhi) 140 { 141 int idx; 142 143 write_c0_entryhi(entryhi); 144 mtc0_tlbw_hazard(); 145 146 tlb_probe(); 147 tlb_probe_hazard(); 148 idx = read_c0_index(); 149 150 if (idx >= current_cpu_data.tlbsize) 151 BUG(); 152 153 if (idx >= 0) { 154 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 155 write_c0_entrylo0(0); 156 write_c0_entrylo1(0); 157 mtc0_tlbw_hazard(); 158 159 tlb_write_indexed(); 160 tlbw_use_hazard(); 161 } 162 163 return idx; 164 } 165 166 int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va, 167 bool user, bool kernel) 168 { 169 int idx_user, idx_kernel; 170 unsigned long flags, old_entryhi; 171 172 local_irq_save(flags); 173 174 old_entryhi = read_c0_entryhi(); 175 176 if (user) 177 idx_user = _kvm_mips_host_tlb_inv((va & VPN2_MASK) | 178 kvm_mips_get_user_asid(vcpu)); 179 if (kernel) 180 idx_kernel = _kvm_mips_host_tlb_inv((va & VPN2_MASK) | 181 kvm_mips_get_kernel_asid(vcpu)); 182 183 write_c0_entryhi(old_entryhi); 184 mtc0_tlbw_hazard(); 185 186 local_irq_restore(flags); 187 188 if (user && idx_user >= 0) 189 kvm_debug("%s: Invalidated guest user entryhi %#lx @ idx %d\n", 190 __func__, (va & VPN2_MASK) | 191 kvm_mips_get_user_asid(vcpu), idx_user); 192 if (kernel && idx_kernel >= 0) 193 kvm_debug("%s: Invalidated guest kernel entryhi %#lx @ idx %d\n", 194 __func__, (va & VPN2_MASK) | 195 kvm_mips_get_kernel_asid(vcpu), idx_kernel); 196 197 return 0; 198 } 199 EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_inv); 200 201 #ifdef CONFIG_KVM_MIPS_VZ 202 203 /* GuestID management */ 204 205 /** 206 * clear_root_gid() - Set GuestCtl1.RID for normal root operation. 207 */ 208 static inline void clear_root_gid(void) 209 { 210 if (cpu_has_guestid) { 211 clear_c0_guestctl1(MIPS_GCTL1_RID); 212 mtc0_tlbw_hazard(); 213 } 214 } 215 216 /** 217 * set_root_gid_to_guest_gid() - Set GuestCtl1.RID to match GuestCtl1.ID. 218 * 219 * Sets the root GuestID to match the current guest GuestID, for TLB operation 220 * on the GPA->RPA mappings in the root TLB. 221 * 222 * The caller must be sure to disable HTW while the root GID is set, and 223 * possibly longer if TLB registers are modified. 224 */ 225 static inline void set_root_gid_to_guest_gid(void) 226 { 227 unsigned int guestctl1; 228 229 if (cpu_has_guestid) { 230 back_to_back_c0_hazard(); 231 guestctl1 = read_c0_guestctl1(); 232 guestctl1 = (guestctl1 & ~MIPS_GCTL1_RID) | 233 ((guestctl1 & MIPS_GCTL1_ID) >> MIPS_GCTL1_ID_SHIFT) 234 << MIPS_GCTL1_RID_SHIFT; 235 write_c0_guestctl1(guestctl1); 236 mtc0_tlbw_hazard(); 237 } 238 } 239 240 int kvm_vz_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va) 241 { 242 int idx; 243 unsigned long flags, old_entryhi; 244 245 local_irq_save(flags); 246 htw_stop(); 247 248 /* Set root GuestID for root probe and write of guest TLB entry */ 249 set_root_gid_to_guest_gid(); 250 251 old_entryhi = read_c0_entryhi(); 252 253 idx = _kvm_mips_host_tlb_inv((va & VPN2_MASK) | 254 kvm_mips_get_root_asid(vcpu)); 255 256 write_c0_entryhi(old_entryhi); 257 clear_root_gid(); 258 mtc0_tlbw_hazard(); 259 260 htw_start(); 261 local_irq_restore(flags); 262 263 if (idx > 0) 264 kvm_debug("%s: Invalidated root entryhi %#lx @ idx %d\n", 265 __func__, (va & VPN2_MASK) | 266 kvm_mips_get_root_asid(vcpu), idx); 267 268 return 0; 269 } 270 EXPORT_SYMBOL_GPL(kvm_vz_host_tlb_inv); 271 272 /** 273 * kvm_vz_guest_tlb_lookup() - Lookup a guest VZ TLB mapping. 274 * @vcpu: KVM VCPU pointer. 275 * @gpa: Guest virtual address in a TLB mapped guest segment. 276 * @gpa: Ponter to output guest physical address it maps to. 277 * 278 * Converts a guest virtual address in a guest TLB mapped segment to a guest 279 * physical address, by probing the guest TLB. 280 * 281 * Returns: 0 if guest TLB mapping exists for @gva. *@gpa will have been 282 * written. 283 * -EFAULT if no guest TLB mapping exists for @gva. *@gpa may not 284 * have been written. 285 */ 286 int kvm_vz_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long gva, 287 unsigned long *gpa) 288 { 289 unsigned long o_entryhi, o_entrylo[2], o_pagemask; 290 unsigned int o_index; 291 unsigned long entrylo[2], pagemask, pagemaskbit, pa; 292 unsigned long flags; 293 int index; 294 295 /* Probe the guest TLB for a mapping */ 296 local_irq_save(flags); 297 /* Set root GuestID for root probe of guest TLB entry */ 298 htw_stop(); 299 set_root_gid_to_guest_gid(); 300 301 o_entryhi = read_gc0_entryhi(); 302 o_index = read_gc0_index(); 303 304 write_gc0_entryhi((o_entryhi & 0x3ff) | (gva & ~0xfffl)); 305 mtc0_tlbw_hazard(); 306 guest_tlb_probe(); 307 tlb_probe_hazard(); 308 309 index = read_gc0_index(); 310 if (index < 0) { 311 /* No match, fail */ 312 write_gc0_entryhi(o_entryhi); 313 write_gc0_index(o_index); 314 315 clear_root_gid(); 316 htw_start(); 317 local_irq_restore(flags); 318 return -EFAULT; 319 } 320 321 /* Match! read the TLB entry */ 322 o_entrylo[0] = read_gc0_entrylo0(); 323 o_entrylo[1] = read_gc0_entrylo1(); 324 o_pagemask = read_gc0_pagemask(); 325 326 mtc0_tlbr_hazard(); 327 guest_tlb_read(); 328 tlb_read_hazard(); 329 330 entrylo[0] = read_gc0_entrylo0(); 331 entrylo[1] = read_gc0_entrylo1(); 332 pagemask = ~read_gc0_pagemask() & ~0x1fffl; 333 334 write_gc0_entryhi(o_entryhi); 335 write_gc0_index(o_index); 336 write_gc0_entrylo0(o_entrylo[0]); 337 write_gc0_entrylo1(o_entrylo[1]); 338 write_gc0_pagemask(o_pagemask); 339 340 clear_root_gid(); 341 htw_start(); 342 local_irq_restore(flags); 343 344 /* Select one of the EntryLo values and interpret the GPA */ 345 pagemaskbit = (pagemask ^ (pagemask & (pagemask - 1))) >> 1; 346 pa = entrylo[!!(gva & pagemaskbit)]; 347 348 /* 349 * TLB entry may have become invalid since TLB probe if physical FTLB 350 * entries are shared between threads (e.g. I6400). 351 */ 352 if (!(pa & ENTRYLO_V)) 353 return -EFAULT; 354 355 /* 356 * Note, this doesn't take guest MIPS32 XPA into account, where PFN is 357 * split with XI/RI in the middle. 358 */ 359 pa = (pa << 6) & ~0xfffl; 360 pa |= gva & ~(pagemask | pagemaskbit); 361 362 *gpa = pa; 363 return 0; 364 } 365 EXPORT_SYMBOL_GPL(kvm_vz_guest_tlb_lookup); 366 367 /** 368 * kvm_vz_local_flush_roottlb_all_guests() - Flush all root TLB entries for 369 * guests. 370 * 371 * Invalidate all entries in root tlb which are GPA mappings. 372 */ 373 void kvm_vz_local_flush_roottlb_all_guests(void) 374 { 375 unsigned long flags; 376 unsigned long old_entryhi, old_pagemask, old_guestctl1; 377 int entry; 378 379 if (WARN_ON(!cpu_has_guestid)) 380 return; 381 382 local_irq_save(flags); 383 htw_stop(); 384 385 /* TLBR may clobber EntryHi.ASID, PageMask, and GuestCtl1.RID */ 386 old_entryhi = read_c0_entryhi(); 387 old_pagemask = read_c0_pagemask(); 388 old_guestctl1 = read_c0_guestctl1(); 389 390 /* 391 * Invalidate guest entries in root TLB while leaving root entries 392 * intact when possible. 393 */ 394 for (entry = 0; entry < current_cpu_data.tlbsize; entry++) { 395 write_c0_index(entry); 396 mtc0_tlbw_hazard(); 397 tlb_read(); 398 tlb_read_hazard(); 399 400 /* Don't invalidate non-guest (RVA) mappings in the root TLB */ 401 if (!(read_c0_guestctl1() & MIPS_GCTL1_RID)) 402 continue; 403 404 /* Make sure all entries differ. */ 405 write_c0_entryhi(UNIQUE_ENTRYHI(entry)); 406 write_c0_entrylo0(0); 407 write_c0_entrylo1(0); 408 write_c0_guestctl1(0); 409 mtc0_tlbw_hazard(); 410 tlb_write_indexed(); 411 } 412 413 write_c0_entryhi(old_entryhi); 414 write_c0_pagemask(old_pagemask); 415 write_c0_guestctl1(old_guestctl1); 416 tlbw_use_hazard(); 417 418 htw_start(); 419 local_irq_restore(flags); 420 } 421 EXPORT_SYMBOL_GPL(kvm_vz_local_flush_roottlb_all_guests); 422 423 /** 424 * kvm_vz_local_flush_guesttlb_all() - Flush all guest TLB entries. 425 * 426 * Invalidate all entries in guest tlb irrespective of guestid. 427 */ 428 void kvm_vz_local_flush_guesttlb_all(void) 429 { 430 unsigned long flags; 431 unsigned long old_index; 432 unsigned long old_entryhi; 433 unsigned long old_entrylo[2]; 434 unsigned long old_pagemask; 435 int entry; 436 437 local_irq_save(flags); 438 439 /* Preserve all clobbered guest registers */ 440 old_index = read_gc0_index(); 441 old_entryhi = read_gc0_entryhi(); 442 old_entrylo[0] = read_gc0_entrylo0(); 443 old_entrylo[1] = read_gc0_entrylo1(); 444 old_pagemask = read_gc0_pagemask(); 445 446 /* Invalidate guest entries in guest TLB */ 447 write_gc0_entrylo0(0); 448 write_gc0_entrylo1(0); 449 write_gc0_pagemask(0); 450 for (entry = 0; entry < current_cpu_data.guest.tlbsize; entry++) { 451 /* Make sure all entries differ. */ 452 write_gc0_index(entry); 453 write_gc0_entryhi(UNIQUE_GUEST_ENTRYHI(entry)); 454 mtc0_tlbw_hazard(); 455 guest_tlb_write_indexed(); 456 } 457 write_gc0_index(old_index); 458 write_gc0_entryhi(old_entryhi); 459 write_gc0_entrylo0(old_entrylo[0]); 460 write_gc0_entrylo1(old_entrylo[1]); 461 write_gc0_pagemask(old_pagemask); 462 tlbw_use_hazard(); 463 464 local_irq_restore(flags); 465 } 466 EXPORT_SYMBOL_GPL(kvm_vz_local_flush_guesttlb_all); 467 468 /** 469 * kvm_vz_save_guesttlb() - Save a range of guest TLB entries. 470 * @buf: Buffer to write TLB entries into. 471 * @index: Start index. 472 * @count: Number of entries to save. 473 * 474 * Save a range of guest TLB entries. The caller must ensure interrupts are 475 * disabled. 476 */ 477 void kvm_vz_save_guesttlb(struct kvm_mips_tlb *buf, unsigned int index, 478 unsigned int count) 479 { 480 unsigned int end = index + count; 481 unsigned long old_entryhi, old_entrylo0, old_entrylo1, old_pagemask; 482 unsigned int guestctl1 = 0; 483 int old_index, i; 484 485 /* Save registers we're about to clobber */ 486 old_index = read_gc0_index(); 487 old_entryhi = read_gc0_entryhi(); 488 old_entrylo0 = read_gc0_entrylo0(); 489 old_entrylo1 = read_gc0_entrylo1(); 490 old_pagemask = read_gc0_pagemask(); 491 492 /* Set root GuestID for root probe */ 493 htw_stop(); 494 set_root_gid_to_guest_gid(); 495 if (cpu_has_guestid) 496 guestctl1 = read_c0_guestctl1(); 497 498 /* Read each entry from guest TLB */ 499 for (i = index; i < end; ++i, ++buf) { 500 write_gc0_index(i); 501 502 mtc0_tlbr_hazard(); 503 guest_tlb_read(); 504 tlb_read_hazard(); 505 506 if (cpu_has_guestid && 507 (read_c0_guestctl1() ^ guestctl1) & MIPS_GCTL1_RID) { 508 /* Entry invalid or belongs to another guest */ 509 buf->tlb_hi = UNIQUE_GUEST_ENTRYHI(i); 510 buf->tlb_lo[0] = 0; 511 buf->tlb_lo[1] = 0; 512 buf->tlb_mask = 0; 513 } else { 514 /* Entry belongs to the right guest */ 515 buf->tlb_hi = read_gc0_entryhi(); 516 buf->tlb_lo[0] = read_gc0_entrylo0(); 517 buf->tlb_lo[1] = read_gc0_entrylo1(); 518 buf->tlb_mask = read_gc0_pagemask(); 519 } 520 } 521 522 /* Clear root GuestID again */ 523 clear_root_gid(); 524 htw_start(); 525 526 /* Restore clobbered registers */ 527 write_gc0_index(old_index); 528 write_gc0_entryhi(old_entryhi); 529 write_gc0_entrylo0(old_entrylo0); 530 write_gc0_entrylo1(old_entrylo1); 531 write_gc0_pagemask(old_pagemask); 532 533 tlbw_use_hazard(); 534 } 535 EXPORT_SYMBOL_GPL(kvm_vz_save_guesttlb); 536 537 /** 538 * kvm_vz_load_guesttlb() - Save a range of guest TLB entries. 539 * @buf: Buffer to read TLB entries from. 540 * @index: Start index. 541 * @count: Number of entries to load. 542 * 543 * Load a range of guest TLB entries. The caller must ensure interrupts are 544 * disabled. 545 */ 546 void kvm_vz_load_guesttlb(const struct kvm_mips_tlb *buf, unsigned int index, 547 unsigned int count) 548 { 549 unsigned int end = index + count; 550 unsigned long old_entryhi, old_entrylo0, old_entrylo1, old_pagemask; 551 int old_index, i; 552 553 /* Save registers we're about to clobber */ 554 old_index = read_gc0_index(); 555 old_entryhi = read_gc0_entryhi(); 556 old_entrylo0 = read_gc0_entrylo0(); 557 old_entrylo1 = read_gc0_entrylo1(); 558 old_pagemask = read_gc0_pagemask(); 559 560 /* Set root GuestID for root probe */ 561 htw_stop(); 562 set_root_gid_to_guest_gid(); 563 564 /* Write each entry to guest TLB */ 565 for (i = index; i < end; ++i, ++buf) { 566 write_gc0_index(i); 567 write_gc0_entryhi(buf->tlb_hi); 568 write_gc0_entrylo0(buf->tlb_lo[0]); 569 write_gc0_entrylo1(buf->tlb_lo[1]); 570 write_gc0_pagemask(buf->tlb_mask); 571 572 mtc0_tlbw_hazard(); 573 guest_tlb_write_indexed(); 574 } 575 576 /* Clear root GuestID again */ 577 clear_root_gid(); 578 htw_start(); 579 580 /* Restore clobbered registers */ 581 write_gc0_index(old_index); 582 write_gc0_entryhi(old_entryhi); 583 write_gc0_entrylo0(old_entrylo0); 584 write_gc0_entrylo1(old_entrylo1); 585 write_gc0_pagemask(old_pagemask); 586 587 tlbw_use_hazard(); 588 } 589 EXPORT_SYMBOL_GPL(kvm_vz_load_guesttlb); 590 591 #endif 592 593 /** 594 * kvm_mips_suspend_mm() - Suspend the active mm. 595 * @cpu The CPU we're running on. 596 * 597 * Suspend the active_mm, ready for a switch to a KVM guest virtual address 598 * space. This is left active for the duration of guest context, including time 599 * with interrupts enabled, so we need to be careful not to confuse e.g. cache 600 * management IPIs. 601 * 602 * kvm_mips_resume_mm() should be called before context switching to a different 603 * process so we don't need to worry about reference counting. 604 * 605 * This needs to be in static kernel code to avoid exporting init_mm. 606 */ 607 void kvm_mips_suspend_mm(int cpu) 608 { 609 cpumask_clear_cpu(cpu, mm_cpumask(current->active_mm)); 610 current->active_mm = &init_mm; 611 } 612 EXPORT_SYMBOL_GPL(kvm_mips_suspend_mm); 613 614 /** 615 * kvm_mips_resume_mm() - Resume the current process mm. 616 * @cpu The CPU we're running on. 617 * 618 * Resume the mm of the current process, after a switch back from a KVM guest 619 * virtual address space (see kvm_mips_suspend_mm()). 620 */ 621 void kvm_mips_resume_mm(int cpu) 622 { 623 cpumask_set_cpu(cpu, mm_cpumask(current->mm)); 624 current->active_mm = current->mm; 625 } 626 EXPORT_SYMBOL_GPL(kvm_mips_resume_mm); 627