1d7d5b05fSDeng-Cheng Zhu /* 2d7d5b05fSDeng-Cheng Zhu * This file is subject to the terms and conditions of the GNU General Public 3d7d5b05fSDeng-Cheng Zhu * License. See the file "COPYING" in the main directory of this archive 4d7d5b05fSDeng-Cheng Zhu * for more details. 5d7d5b05fSDeng-Cheng Zhu * 6d7d5b05fSDeng-Cheng Zhu * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that 7d7d5b05fSDeng-Cheng Zhu * TLB handlers run from KSEG0 8d7d5b05fSDeng-Cheng Zhu * 9d7d5b05fSDeng-Cheng Zhu * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. 10d7d5b05fSDeng-Cheng Zhu * Authors: Sanjay Lal <sanjayl@kymasys.com> 11d7d5b05fSDeng-Cheng Zhu */ 12d7d5b05fSDeng-Cheng Zhu 13d7d5b05fSDeng-Cheng Zhu #include <linux/sched.h> 14d7d5b05fSDeng-Cheng Zhu #include <linux/smp.h> 15d7d5b05fSDeng-Cheng Zhu #include <linux/mm.h> 16d7d5b05fSDeng-Cheng Zhu #include <linux/delay.h> 17d7d5b05fSDeng-Cheng Zhu #include <linux/module.h> 18d7d5b05fSDeng-Cheng Zhu #include <linux/kvm_host.h> 19d7d5b05fSDeng-Cheng Zhu #include <linux/srcu.h> 20d7d5b05fSDeng-Cheng Zhu 21d7d5b05fSDeng-Cheng Zhu #include <asm/cpu.h> 22d7d5b05fSDeng-Cheng Zhu #include <asm/bootinfo.h> 23d7d5b05fSDeng-Cheng Zhu #include <asm/mmu_context.h> 24d7d5b05fSDeng-Cheng Zhu #include <asm/pgtable.h> 25d7d5b05fSDeng-Cheng Zhu #include <asm/cacheflush.h> 26d7d5b05fSDeng-Cheng Zhu #include <asm/tlb.h> 27d7d5b05fSDeng-Cheng Zhu 28d7d5b05fSDeng-Cheng Zhu #undef CONFIG_MIPS_MT 29d7d5b05fSDeng-Cheng Zhu #include <asm/r4kcache.h> 30d7d5b05fSDeng-Cheng Zhu #define CONFIG_MIPS_MT 31d7d5b05fSDeng-Cheng Zhu 32d7d5b05fSDeng-Cheng Zhu #define KVM_GUEST_PC_TLB 0 33d7d5b05fSDeng-Cheng Zhu #define KVM_GUEST_SP_TLB 1 34d7d5b05fSDeng-Cheng Zhu 35d7d5b05fSDeng-Cheng Zhu #define PRIx64 "llx" 36d7d5b05fSDeng-Cheng Zhu 37d7d5b05fSDeng-Cheng Zhu atomic_t kvm_mips_instance; 38cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_instance); 39d7d5b05fSDeng-Cheng Zhu 40d7d5b05fSDeng-Cheng Zhu /* These function pointers are initialized once the KVM module is loaded */ 41ba049e93SDan Williams kvm_pfn_t (*kvm_mips_gfn_to_pfn)(struct kvm *kvm, gfn_t gfn); 42cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_gfn_to_pfn); 43d7d5b05fSDeng-Cheng Zhu 44ba049e93SDan Williams void (*kvm_mips_release_pfn_clean)(kvm_pfn_t pfn); 45cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_release_pfn_clean); 46d7d5b05fSDeng-Cheng Zhu 47ba049e93SDan Williams bool (*kvm_mips_is_error_pfn)(kvm_pfn_t pfn); 48cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_is_error_pfn); 49d7d5b05fSDeng-Cheng Zhu 50bdb7ed86SJames Hogan u32 kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu) 51d7d5b05fSDeng-Cheng Zhu { 524edf00a4SPaul Burton int cpu = smp_processor_id(); 534edf00a4SPaul Burton 544edf00a4SPaul Burton return vcpu->arch.guest_kernel_asid[cpu] & 554edf00a4SPaul Burton cpu_asid_mask(&cpu_data[cpu]); 56d7d5b05fSDeng-Cheng Zhu } 57d7d5b05fSDeng-Cheng Zhu 58bdb7ed86SJames Hogan u32 kvm_mips_get_user_asid(struct kvm_vcpu *vcpu) 59d7d5b05fSDeng-Cheng Zhu { 604edf00a4SPaul Burton int cpu = smp_processor_id(); 614edf00a4SPaul Burton 624edf00a4SPaul Burton return vcpu->arch.guest_user_asid[cpu] & 634edf00a4SPaul Burton cpu_asid_mask(&cpu_data[cpu]); 64d7d5b05fSDeng-Cheng Zhu } 65d7d5b05fSDeng-Cheng Zhu 66bdb7ed86SJames Hogan inline u32 kvm_mips_get_commpage_asid(struct kvm_vcpu *vcpu) 67d7d5b05fSDeng-Cheng Zhu { 68d7d5b05fSDeng-Cheng Zhu return vcpu->kvm->arch.commpage_tlb; 69d7d5b05fSDeng-Cheng Zhu } 70d7d5b05fSDeng-Cheng Zhu 71d7d5b05fSDeng-Cheng Zhu /* Structure defining an tlb entry data set. */ 72d7d5b05fSDeng-Cheng Zhu 73d7d5b05fSDeng-Cheng Zhu void kvm_mips_dump_host_tlbs(void) 74d7d5b05fSDeng-Cheng Zhu { 75d7d5b05fSDeng-Cheng Zhu unsigned long old_entryhi; 76d7d5b05fSDeng-Cheng Zhu unsigned long old_pagemask; 77d7d5b05fSDeng-Cheng Zhu struct kvm_mips_tlb tlb; 78d7d5b05fSDeng-Cheng Zhu unsigned long flags; 79d7d5b05fSDeng-Cheng Zhu int i; 80d7d5b05fSDeng-Cheng Zhu 81d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 82d7d5b05fSDeng-Cheng Zhu 83d7d5b05fSDeng-Cheng Zhu old_entryhi = read_c0_entryhi(); 84d7d5b05fSDeng-Cheng Zhu old_pagemask = read_c0_pagemask(); 85d7d5b05fSDeng-Cheng Zhu 86d7d5b05fSDeng-Cheng Zhu kvm_info("HOST TLBs:\n"); 874edf00a4SPaul Burton kvm_info("ASID: %#lx\n", read_c0_entryhi() & 884edf00a4SPaul Burton cpu_asid_mask(¤t_cpu_data)); 89d7d5b05fSDeng-Cheng Zhu 90d7d5b05fSDeng-Cheng Zhu for (i = 0; i < current_cpu_data.tlbsize; i++) { 91d7d5b05fSDeng-Cheng Zhu write_c0_index(i); 92d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 93d7d5b05fSDeng-Cheng Zhu 94d7d5b05fSDeng-Cheng Zhu tlb_read(); 95d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 96d7d5b05fSDeng-Cheng Zhu 97d7d5b05fSDeng-Cheng Zhu tlb.tlb_hi = read_c0_entryhi(); 98d7d5b05fSDeng-Cheng Zhu tlb.tlb_lo0 = read_c0_entrylo0(); 99d7d5b05fSDeng-Cheng Zhu tlb.tlb_lo1 = read_c0_entrylo1(); 100d7d5b05fSDeng-Cheng Zhu tlb.tlb_mask = read_c0_pagemask(); 101d7d5b05fSDeng-Cheng Zhu 102d7d5b05fSDeng-Cheng Zhu kvm_info("TLB%c%3d Hi 0x%08lx ", 103d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 | tlb.tlb_lo1) & MIPS3_PG_V ? ' ' : '*', 104d7d5b05fSDeng-Cheng Zhu i, tlb.tlb_hi); 105d7d5b05fSDeng-Cheng Zhu kvm_info("Lo0=0x%09" PRIx64 " %c%c attr %lx ", 106d7d5b05fSDeng-Cheng Zhu (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo0), 107d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 & MIPS3_PG_D) ? 'D' : ' ', 108d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 & MIPS3_PG_G) ? 'G' : ' ', 109d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 >> 3) & 7); 110d7d5b05fSDeng-Cheng Zhu kvm_info("Lo1=0x%09" PRIx64 " %c%c attr %lx sz=%lx\n", 111d7d5b05fSDeng-Cheng Zhu (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo1), 112d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo1 & MIPS3_PG_D) ? 'D' : ' ', 113d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo1 & MIPS3_PG_G) ? 'G' : ' ', 114d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo1 >> 3) & 7, tlb.tlb_mask); 115d7d5b05fSDeng-Cheng Zhu } 116d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_entryhi); 117d7d5b05fSDeng-Cheng Zhu write_c0_pagemask(old_pagemask); 118d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 119d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 120d7d5b05fSDeng-Cheng Zhu } 121cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_dump_host_tlbs); 122d7d5b05fSDeng-Cheng Zhu 123d7d5b05fSDeng-Cheng Zhu void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu) 124d7d5b05fSDeng-Cheng Zhu { 125d7d5b05fSDeng-Cheng Zhu struct mips_coproc *cop0 = vcpu->arch.cop0; 126d7d5b05fSDeng-Cheng Zhu struct kvm_mips_tlb tlb; 127d7d5b05fSDeng-Cheng Zhu int i; 128d7d5b05fSDeng-Cheng Zhu 129d7d5b05fSDeng-Cheng Zhu kvm_info("Guest TLBs:\n"); 130d7d5b05fSDeng-Cheng Zhu kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0)); 131d7d5b05fSDeng-Cheng Zhu 132d7d5b05fSDeng-Cheng Zhu for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) { 133d7d5b05fSDeng-Cheng Zhu tlb = vcpu->arch.guest_tlb[i]; 134d7d5b05fSDeng-Cheng Zhu kvm_info("TLB%c%3d Hi 0x%08lx ", 135d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 | tlb.tlb_lo1) & MIPS3_PG_V ? ' ' : '*', 136d7d5b05fSDeng-Cheng Zhu i, tlb.tlb_hi); 137d7d5b05fSDeng-Cheng Zhu kvm_info("Lo0=0x%09" PRIx64 " %c%c attr %lx ", 138d7d5b05fSDeng-Cheng Zhu (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo0), 139d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 & MIPS3_PG_D) ? 'D' : ' ', 140d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 & MIPS3_PG_G) ? 'G' : ' ', 141d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 >> 3) & 7); 142d7d5b05fSDeng-Cheng Zhu kvm_info("Lo1=0x%09" PRIx64 " %c%c attr %lx sz=%lx\n", 143d7d5b05fSDeng-Cheng Zhu (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo1), 144d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo1 & MIPS3_PG_D) ? 'D' : ' ', 145d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo1 & MIPS3_PG_G) ? 'G' : ' ', 146d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo1 >> 3) & 7, tlb.tlb_mask); 147d7d5b05fSDeng-Cheng Zhu } 148d7d5b05fSDeng-Cheng Zhu } 149cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_dump_guest_tlbs); 150d7d5b05fSDeng-Cheng Zhu 151d7d5b05fSDeng-Cheng Zhu static int kvm_mips_map_page(struct kvm *kvm, gfn_t gfn) 152d7d5b05fSDeng-Cheng Zhu { 153d7d5b05fSDeng-Cheng Zhu int srcu_idx, err = 0; 154ba049e93SDan Williams kvm_pfn_t pfn; 155d7d5b05fSDeng-Cheng Zhu 156d7d5b05fSDeng-Cheng Zhu if (kvm->arch.guest_pmap[gfn] != KVM_INVALID_PAGE) 157d7d5b05fSDeng-Cheng Zhu return 0; 158d7d5b05fSDeng-Cheng Zhu 159d7d5b05fSDeng-Cheng Zhu srcu_idx = srcu_read_lock(&kvm->srcu); 160d7d5b05fSDeng-Cheng Zhu pfn = kvm_mips_gfn_to_pfn(kvm, gfn); 161d7d5b05fSDeng-Cheng Zhu 162d7d5b05fSDeng-Cheng Zhu if (kvm_mips_is_error_pfn(pfn)) { 163d7d5b05fSDeng-Cheng Zhu kvm_err("Couldn't get pfn for gfn %#" PRIx64 "!\n", gfn); 164d7d5b05fSDeng-Cheng Zhu err = -EFAULT; 165d7d5b05fSDeng-Cheng Zhu goto out; 166d7d5b05fSDeng-Cheng Zhu } 167d7d5b05fSDeng-Cheng Zhu 168d7d5b05fSDeng-Cheng Zhu kvm->arch.guest_pmap[gfn] = pfn; 169d7d5b05fSDeng-Cheng Zhu out: 170d7d5b05fSDeng-Cheng Zhu srcu_read_unlock(&kvm->srcu, srcu_idx); 171d7d5b05fSDeng-Cheng Zhu return err; 172d7d5b05fSDeng-Cheng Zhu } 173d7d5b05fSDeng-Cheng Zhu 174d7d5b05fSDeng-Cheng Zhu /* Translate guest KSEG0 addresses to Host PA */ 175d7d5b05fSDeng-Cheng Zhu unsigned long kvm_mips_translate_guest_kseg0_to_hpa(struct kvm_vcpu *vcpu, 176d7d5b05fSDeng-Cheng Zhu unsigned long gva) 177d7d5b05fSDeng-Cheng Zhu { 178d7d5b05fSDeng-Cheng Zhu gfn_t gfn; 179d7d5b05fSDeng-Cheng Zhu uint32_t offset = gva & ~PAGE_MASK; 180d7d5b05fSDeng-Cheng Zhu struct kvm *kvm = vcpu->kvm; 181d7d5b05fSDeng-Cheng Zhu 182d7d5b05fSDeng-Cheng Zhu if (KVM_GUEST_KSEGX(gva) != KVM_GUEST_KSEG0) { 183d7d5b05fSDeng-Cheng Zhu kvm_err("%s/%p: Invalid gva: %#lx\n", __func__, 184d7d5b05fSDeng-Cheng Zhu __builtin_return_address(0), gva); 185d7d5b05fSDeng-Cheng Zhu return KVM_INVALID_PAGE; 186d7d5b05fSDeng-Cheng Zhu } 187d7d5b05fSDeng-Cheng Zhu 188d7d5b05fSDeng-Cheng Zhu gfn = (KVM_GUEST_CPHYSADDR(gva) >> PAGE_SHIFT); 189d7d5b05fSDeng-Cheng Zhu 190d7d5b05fSDeng-Cheng Zhu if (gfn >= kvm->arch.guest_pmap_npages) { 191d7d5b05fSDeng-Cheng Zhu kvm_err("%s: Invalid gfn: %#llx, GVA: %#lx\n", __func__, gfn, 192d7d5b05fSDeng-Cheng Zhu gva); 193d7d5b05fSDeng-Cheng Zhu return KVM_INVALID_PAGE; 194d7d5b05fSDeng-Cheng Zhu } 195d7d5b05fSDeng-Cheng Zhu 196d7d5b05fSDeng-Cheng Zhu if (kvm_mips_map_page(vcpu->kvm, gfn) < 0) 197d7d5b05fSDeng-Cheng Zhu return KVM_INVALID_ADDR; 198d7d5b05fSDeng-Cheng Zhu 199d7d5b05fSDeng-Cheng Zhu return (kvm->arch.guest_pmap[gfn] << PAGE_SHIFT) + offset; 200d7d5b05fSDeng-Cheng Zhu } 201cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_translate_guest_kseg0_to_hpa); 202d7d5b05fSDeng-Cheng Zhu 203d7d5b05fSDeng-Cheng Zhu /* XXXKYMA: Must be called with interrupts disabled */ 204d7d5b05fSDeng-Cheng Zhu /* set flush_dcache_mask == 0 if no dcache flush required */ 205d7d5b05fSDeng-Cheng Zhu int kvm_mips_host_tlb_write(struct kvm_vcpu *vcpu, unsigned long entryhi, 206d7d5b05fSDeng-Cheng Zhu unsigned long entrylo0, unsigned long entrylo1, 207d7d5b05fSDeng-Cheng Zhu int flush_dcache_mask) 208d7d5b05fSDeng-Cheng Zhu { 209d7d5b05fSDeng-Cheng Zhu unsigned long flags; 210d7d5b05fSDeng-Cheng Zhu unsigned long old_entryhi; 211d7d5b05fSDeng-Cheng Zhu int idx; 212d7d5b05fSDeng-Cheng Zhu 213d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 214d7d5b05fSDeng-Cheng Zhu 215d7d5b05fSDeng-Cheng Zhu old_entryhi = read_c0_entryhi(); 216d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(entryhi); 217d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 218d7d5b05fSDeng-Cheng Zhu 219d7d5b05fSDeng-Cheng Zhu tlb_probe(); 220d7d5b05fSDeng-Cheng Zhu tlb_probe_hazard(); 221d7d5b05fSDeng-Cheng Zhu idx = read_c0_index(); 222d7d5b05fSDeng-Cheng Zhu 223d7d5b05fSDeng-Cheng Zhu if (idx > current_cpu_data.tlbsize) { 224d7d5b05fSDeng-Cheng Zhu kvm_err("%s: Invalid Index: %d\n", __func__, idx); 225d7d5b05fSDeng-Cheng Zhu kvm_mips_dump_host_tlbs(); 226cfec0e75STapasweni Pathak local_irq_restore(flags); 227d7d5b05fSDeng-Cheng Zhu return -1; 228d7d5b05fSDeng-Cheng Zhu } 229d7d5b05fSDeng-Cheng Zhu 230d7d5b05fSDeng-Cheng Zhu write_c0_entrylo0(entrylo0); 231d7d5b05fSDeng-Cheng Zhu write_c0_entrylo1(entrylo1); 232d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 233d7d5b05fSDeng-Cheng Zhu 234d7d5b05fSDeng-Cheng Zhu if (idx < 0) 235d7d5b05fSDeng-Cheng Zhu tlb_write_random(); 236d7d5b05fSDeng-Cheng Zhu else 237d7d5b05fSDeng-Cheng Zhu tlb_write_indexed(); 238d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 239d7d5b05fSDeng-Cheng Zhu 240d7d5b05fSDeng-Cheng Zhu kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0(R): 0x%08lx, entrylo1(R): 0x%08lx\n", 241d7d5b05fSDeng-Cheng Zhu vcpu->arch.pc, idx, read_c0_entryhi(), 242d7d5b05fSDeng-Cheng Zhu read_c0_entrylo0(), read_c0_entrylo1()); 243d7d5b05fSDeng-Cheng Zhu 244d7d5b05fSDeng-Cheng Zhu /* Flush D-cache */ 245d7d5b05fSDeng-Cheng Zhu if (flush_dcache_mask) { 246d7d5b05fSDeng-Cheng Zhu if (entrylo0 & MIPS3_PG_V) { 247d7d5b05fSDeng-Cheng Zhu ++vcpu->stat.flush_dcache_exits; 248d7d5b05fSDeng-Cheng Zhu flush_data_cache_page((entryhi & VPN2_MASK) & 249d7d5b05fSDeng-Cheng Zhu ~flush_dcache_mask); 250d7d5b05fSDeng-Cheng Zhu } 251d7d5b05fSDeng-Cheng Zhu if (entrylo1 & MIPS3_PG_V) { 252d7d5b05fSDeng-Cheng Zhu ++vcpu->stat.flush_dcache_exits; 253d7d5b05fSDeng-Cheng Zhu flush_data_cache_page(((entryhi & VPN2_MASK) & 254d7d5b05fSDeng-Cheng Zhu ~flush_dcache_mask) | 255d7d5b05fSDeng-Cheng Zhu (0x1 << PAGE_SHIFT)); 256d7d5b05fSDeng-Cheng Zhu } 257d7d5b05fSDeng-Cheng Zhu } 258d7d5b05fSDeng-Cheng Zhu 259d7d5b05fSDeng-Cheng Zhu /* Restore old ASID */ 260d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_entryhi); 261d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 262d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 263d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 264d7d5b05fSDeng-Cheng Zhu return 0; 265d7d5b05fSDeng-Cheng Zhu } 266d7d5b05fSDeng-Cheng Zhu 267d7d5b05fSDeng-Cheng Zhu /* XXXKYMA: Must be called with interrupts disabled */ 268d7d5b05fSDeng-Cheng Zhu int kvm_mips_handle_kseg0_tlb_fault(unsigned long badvaddr, 269d7d5b05fSDeng-Cheng Zhu struct kvm_vcpu *vcpu) 270d7d5b05fSDeng-Cheng Zhu { 271d7d5b05fSDeng-Cheng Zhu gfn_t gfn; 272ba049e93SDan Williams kvm_pfn_t pfn0, pfn1; 273d7d5b05fSDeng-Cheng Zhu unsigned long vaddr = 0; 274d7d5b05fSDeng-Cheng Zhu unsigned long entryhi = 0, entrylo0 = 0, entrylo1 = 0; 275d7d5b05fSDeng-Cheng Zhu int even; 276d7d5b05fSDeng-Cheng Zhu struct kvm *kvm = vcpu->kvm; 277d7d5b05fSDeng-Cheng Zhu const int flush_dcache_mask = 0; 278f049729cSJames Hogan int ret; 279d7d5b05fSDeng-Cheng Zhu 280d7d5b05fSDeng-Cheng Zhu if (KVM_GUEST_KSEGX(badvaddr) != KVM_GUEST_KSEG0) { 281d7d5b05fSDeng-Cheng Zhu kvm_err("%s: Invalid BadVaddr: %#lx\n", __func__, badvaddr); 282d7d5b05fSDeng-Cheng Zhu kvm_mips_dump_host_tlbs(); 283d7d5b05fSDeng-Cheng Zhu return -1; 284d7d5b05fSDeng-Cheng Zhu } 285d7d5b05fSDeng-Cheng Zhu 286d7d5b05fSDeng-Cheng Zhu gfn = (KVM_GUEST_CPHYSADDR(badvaddr) >> PAGE_SHIFT); 287d7d5b05fSDeng-Cheng Zhu if (gfn >= kvm->arch.guest_pmap_npages) { 288d7d5b05fSDeng-Cheng Zhu kvm_err("%s: Invalid gfn: %#llx, BadVaddr: %#lx\n", __func__, 289d7d5b05fSDeng-Cheng Zhu gfn, badvaddr); 290d7d5b05fSDeng-Cheng Zhu kvm_mips_dump_host_tlbs(); 291d7d5b05fSDeng-Cheng Zhu return -1; 292d7d5b05fSDeng-Cheng Zhu } 293d7d5b05fSDeng-Cheng Zhu even = !(gfn & 0x1); 294d7d5b05fSDeng-Cheng Zhu vaddr = badvaddr & (PAGE_MASK << 1); 295d7d5b05fSDeng-Cheng Zhu 296d7d5b05fSDeng-Cheng Zhu if (kvm_mips_map_page(vcpu->kvm, gfn) < 0) 297d7d5b05fSDeng-Cheng Zhu return -1; 298d7d5b05fSDeng-Cheng Zhu 299d7d5b05fSDeng-Cheng Zhu if (kvm_mips_map_page(vcpu->kvm, gfn ^ 0x1) < 0) 300d7d5b05fSDeng-Cheng Zhu return -1; 301d7d5b05fSDeng-Cheng Zhu 302d7d5b05fSDeng-Cheng Zhu if (even) { 303d7d5b05fSDeng-Cheng Zhu pfn0 = kvm->arch.guest_pmap[gfn]; 304d7d5b05fSDeng-Cheng Zhu pfn1 = kvm->arch.guest_pmap[gfn ^ 0x1]; 305d7d5b05fSDeng-Cheng Zhu } else { 306d7d5b05fSDeng-Cheng Zhu pfn0 = kvm->arch.guest_pmap[gfn ^ 0x1]; 307d7d5b05fSDeng-Cheng Zhu pfn1 = kvm->arch.guest_pmap[gfn]; 308d7d5b05fSDeng-Cheng Zhu } 309d7d5b05fSDeng-Cheng Zhu 310d7d5b05fSDeng-Cheng Zhu entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) | 311d7d5b05fSDeng-Cheng Zhu (1 << 2) | (0x1 << 1); 312d7d5b05fSDeng-Cheng Zhu entrylo1 = mips3_paddr_to_tlbpfn(pfn1 << PAGE_SHIFT) | (0x3 << 3) | 313d7d5b05fSDeng-Cheng Zhu (1 << 2) | (0x1 << 1); 314d7d5b05fSDeng-Cheng Zhu 315f049729cSJames Hogan preempt_disable(); 316f049729cSJames Hogan entryhi = (vaddr | kvm_mips_get_kernel_asid(vcpu)); 317f049729cSJames Hogan ret = kvm_mips_host_tlb_write(vcpu, entryhi, entrylo0, entrylo1, 318d7d5b05fSDeng-Cheng Zhu flush_dcache_mask); 319f049729cSJames Hogan preempt_enable(); 320f049729cSJames Hogan 321f049729cSJames Hogan return ret; 322d7d5b05fSDeng-Cheng Zhu } 323cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_handle_kseg0_tlb_fault); 324d7d5b05fSDeng-Cheng Zhu 325d7d5b05fSDeng-Cheng Zhu int kvm_mips_handle_commpage_tlb_fault(unsigned long badvaddr, 326d7d5b05fSDeng-Cheng Zhu struct kvm_vcpu *vcpu) 327d7d5b05fSDeng-Cheng Zhu { 328ba049e93SDan Williams kvm_pfn_t pfn0, pfn1; 329d7d5b05fSDeng-Cheng Zhu unsigned long flags, old_entryhi = 0, vaddr = 0; 330d7d5b05fSDeng-Cheng Zhu unsigned long entrylo0 = 0, entrylo1 = 0; 331d7d5b05fSDeng-Cheng Zhu 332d7d5b05fSDeng-Cheng Zhu pfn0 = CPHYSADDR(vcpu->arch.kseg0_commpage) >> PAGE_SHIFT; 333d7d5b05fSDeng-Cheng Zhu pfn1 = 0; 334d7d5b05fSDeng-Cheng Zhu entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) | 335d7d5b05fSDeng-Cheng Zhu (1 << 2) | (0x1 << 1); 336d7d5b05fSDeng-Cheng Zhu entrylo1 = 0; 337d7d5b05fSDeng-Cheng Zhu 338d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 339d7d5b05fSDeng-Cheng Zhu 340d7d5b05fSDeng-Cheng Zhu old_entryhi = read_c0_entryhi(); 341d7d5b05fSDeng-Cheng Zhu vaddr = badvaddr & (PAGE_MASK << 1); 342d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(vaddr | kvm_mips_get_kernel_asid(vcpu)); 343d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 344d7d5b05fSDeng-Cheng Zhu write_c0_entrylo0(entrylo0); 345d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 346d7d5b05fSDeng-Cheng Zhu write_c0_entrylo1(entrylo1); 347d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 348d7d5b05fSDeng-Cheng Zhu write_c0_index(kvm_mips_get_commpage_asid(vcpu)); 349d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 350d7d5b05fSDeng-Cheng Zhu tlb_write_indexed(); 351d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 352d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 353d7d5b05fSDeng-Cheng Zhu 354d7d5b05fSDeng-Cheng Zhu kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0 (R): 0x%08lx, entrylo1(R): 0x%08lx\n", 355d7d5b05fSDeng-Cheng Zhu vcpu->arch.pc, read_c0_index(), read_c0_entryhi(), 356d7d5b05fSDeng-Cheng Zhu read_c0_entrylo0(), read_c0_entrylo1()); 357d7d5b05fSDeng-Cheng Zhu 358d7d5b05fSDeng-Cheng Zhu /* Restore old ASID */ 359d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_entryhi); 360d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 361d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 362d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 363d7d5b05fSDeng-Cheng Zhu 364d7d5b05fSDeng-Cheng Zhu return 0; 365d7d5b05fSDeng-Cheng Zhu } 366cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_handle_commpage_tlb_fault); 367d7d5b05fSDeng-Cheng Zhu 368d7d5b05fSDeng-Cheng Zhu int kvm_mips_handle_mapped_seg_tlb_fault(struct kvm_vcpu *vcpu, 369d7d5b05fSDeng-Cheng Zhu struct kvm_mips_tlb *tlb, 370d7d5b05fSDeng-Cheng Zhu unsigned long *hpa0, 371d7d5b05fSDeng-Cheng Zhu unsigned long *hpa1) 372d7d5b05fSDeng-Cheng Zhu { 373d7d5b05fSDeng-Cheng Zhu unsigned long entryhi = 0, entrylo0 = 0, entrylo1 = 0; 374d7d5b05fSDeng-Cheng Zhu struct kvm *kvm = vcpu->kvm; 375ba049e93SDan Williams kvm_pfn_t pfn0, pfn1; 376f049729cSJames Hogan int ret; 377d7d5b05fSDeng-Cheng Zhu 378d7d5b05fSDeng-Cheng Zhu if ((tlb->tlb_hi & VPN2_MASK) == 0) { 379d7d5b05fSDeng-Cheng Zhu pfn0 = 0; 380d7d5b05fSDeng-Cheng Zhu pfn1 = 0; 381d7d5b05fSDeng-Cheng Zhu } else { 382d7d5b05fSDeng-Cheng Zhu if (kvm_mips_map_page(kvm, mips3_tlbpfn_to_paddr(tlb->tlb_lo0) 383d7d5b05fSDeng-Cheng Zhu >> PAGE_SHIFT) < 0) 384d7d5b05fSDeng-Cheng Zhu return -1; 385d7d5b05fSDeng-Cheng Zhu 386d7d5b05fSDeng-Cheng Zhu if (kvm_mips_map_page(kvm, mips3_tlbpfn_to_paddr(tlb->tlb_lo1) 387d7d5b05fSDeng-Cheng Zhu >> PAGE_SHIFT) < 0) 388d7d5b05fSDeng-Cheng Zhu return -1; 389d7d5b05fSDeng-Cheng Zhu 390d7d5b05fSDeng-Cheng Zhu pfn0 = kvm->arch.guest_pmap[mips3_tlbpfn_to_paddr(tlb->tlb_lo0) 391d7d5b05fSDeng-Cheng Zhu >> PAGE_SHIFT]; 392d7d5b05fSDeng-Cheng Zhu pfn1 = kvm->arch.guest_pmap[mips3_tlbpfn_to_paddr(tlb->tlb_lo1) 393d7d5b05fSDeng-Cheng Zhu >> PAGE_SHIFT]; 394d7d5b05fSDeng-Cheng Zhu } 395d7d5b05fSDeng-Cheng Zhu 396d7d5b05fSDeng-Cheng Zhu if (hpa0) 397d7d5b05fSDeng-Cheng Zhu *hpa0 = pfn0 << PAGE_SHIFT; 398d7d5b05fSDeng-Cheng Zhu 399d7d5b05fSDeng-Cheng Zhu if (hpa1) 400d7d5b05fSDeng-Cheng Zhu *hpa1 = pfn1 << PAGE_SHIFT; 401d7d5b05fSDeng-Cheng Zhu 402d7d5b05fSDeng-Cheng Zhu /* Get attributes from the Guest TLB */ 403d7d5b05fSDeng-Cheng Zhu entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) | 404d7d5b05fSDeng-Cheng Zhu (tlb->tlb_lo0 & MIPS3_PG_D) | (tlb->tlb_lo0 & MIPS3_PG_V); 405d7d5b05fSDeng-Cheng Zhu entrylo1 = mips3_paddr_to_tlbpfn(pfn1 << PAGE_SHIFT) | (0x3 << 3) | 406d7d5b05fSDeng-Cheng Zhu (tlb->tlb_lo1 & MIPS3_PG_D) | (tlb->tlb_lo1 & MIPS3_PG_V); 407d7d5b05fSDeng-Cheng Zhu 408d7d5b05fSDeng-Cheng Zhu kvm_debug("@ %#lx tlb_lo0: 0x%08lx tlb_lo1: 0x%08lx\n", vcpu->arch.pc, 409d7d5b05fSDeng-Cheng Zhu tlb->tlb_lo0, tlb->tlb_lo1); 410d7d5b05fSDeng-Cheng Zhu 411f049729cSJames Hogan preempt_disable(); 412f049729cSJames Hogan entryhi = (tlb->tlb_hi & VPN2_MASK) | (KVM_GUEST_KERNEL_MODE(vcpu) ? 413f049729cSJames Hogan kvm_mips_get_kernel_asid(vcpu) : 414f049729cSJames Hogan kvm_mips_get_user_asid(vcpu)); 415f049729cSJames Hogan ret = kvm_mips_host_tlb_write(vcpu, entryhi, entrylo0, entrylo1, 416d7d5b05fSDeng-Cheng Zhu tlb->tlb_mask); 417f049729cSJames Hogan preempt_enable(); 418f049729cSJames Hogan 419f049729cSJames Hogan return ret; 420d7d5b05fSDeng-Cheng Zhu } 421cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_handle_mapped_seg_tlb_fault); 422d7d5b05fSDeng-Cheng Zhu 423d7d5b05fSDeng-Cheng Zhu int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi) 424d7d5b05fSDeng-Cheng Zhu { 425d7d5b05fSDeng-Cheng Zhu int i; 426d7d5b05fSDeng-Cheng Zhu int index = -1; 427d7d5b05fSDeng-Cheng Zhu struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb; 428d7d5b05fSDeng-Cheng Zhu 429d7d5b05fSDeng-Cheng Zhu for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) { 430d7d5b05fSDeng-Cheng Zhu if (TLB_HI_VPN2_HIT(tlb[i], entryhi) && 431d7d5b05fSDeng-Cheng Zhu TLB_HI_ASID_HIT(tlb[i], entryhi)) { 432d7d5b05fSDeng-Cheng Zhu index = i; 433d7d5b05fSDeng-Cheng Zhu break; 434d7d5b05fSDeng-Cheng Zhu } 435d7d5b05fSDeng-Cheng Zhu } 436d7d5b05fSDeng-Cheng Zhu 437d7d5b05fSDeng-Cheng Zhu kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n", 438d7d5b05fSDeng-Cheng Zhu __func__, entryhi, index, tlb[i].tlb_lo0, tlb[i].tlb_lo1); 439d7d5b05fSDeng-Cheng Zhu 440d7d5b05fSDeng-Cheng Zhu return index; 441d7d5b05fSDeng-Cheng Zhu } 442cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_guest_tlb_lookup); 443d7d5b05fSDeng-Cheng Zhu 444d7d5b05fSDeng-Cheng Zhu int kvm_mips_host_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long vaddr) 445d7d5b05fSDeng-Cheng Zhu { 446d7d5b05fSDeng-Cheng Zhu unsigned long old_entryhi, flags; 447d7d5b05fSDeng-Cheng Zhu int idx; 448d7d5b05fSDeng-Cheng Zhu 449d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 450d7d5b05fSDeng-Cheng Zhu 451d7d5b05fSDeng-Cheng Zhu old_entryhi = read_c0_entryhi(); 452d7d5b05fSDeng-Cheng Zhu 453d7d5b05fSDeng-Cheng Zhu if (KVM_GUEST_KERNEL_MODE(vcpu)) 454d7d5b05fSDeng-Cheng Zhu write_c0_entryhi((vaddr & VPN2_MASK) | 455d7d5b05fSDeng-Cheng Zhu kvm_mips_get_kernel_asid(vcpu)); 456d7d5b05fSDeng-Cheng Zhu else { 457d7d5b05fSDeng-Cheng Zhu write_c0_entryhi((vaddr & VPN2_MASK) | 458d7d5b05fSDeng-Cheng Zhu kvm_mips_get_user_asid(vcpu)); 459d7d5b05fSDeng-Cheng Zhu } 460d7d5b05fSDeng-Cheng Zhu 461d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 462d7d5b05fSDeng-Cheng Zhu 463d7d5b05fSDeng-Cheng Zhu tlb_probe(); 464d7d5b05fSDeng-Cheng Zhu tlb_probe_hazard(); 465d7d5b05fSDeng-Cheng Zhu idx = read_c0_index(); 466d7d5b05fSDeng-Cheng Zhu 467d7d5b05fSDeng-Cheng Zhu /* Restore old ASID */ 468d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_entryhi); 469d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 470d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 471d7d5b05fSDeng-Cheng Zhu 472d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 473d7d5b05fSDeng-Cheng Zhu 474d7d5b05fSDeng-Cheng Zhu kvm_debug("Host TLB lookup, %#lx, idx: %2d\n", vaddr, idx); 475d7d5b05fSDeng-Cheng Zhu 476d7d5b05fSDeng-Cheng Zhu return idx; 477d7d5b05fSDeng-Cheng Zhu } 478cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_lookup); 479d7d5b05fSDeng-Cheng Zhu 480d7d5b05fSDeng-Cheng Zhu int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va) 481d7d5b05fSDeng-Cheng Zhu { 482d7d5b05fSDeng-Cheng Zhu int idx; 483d7d5b05fSDeng-Cheng Zhu unsigned long flags, old_entryhi; 484d7d5b05fSDeng-Cheng Zhu 485d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 486d7d5b05fSDeng-Cheng Zhu 487d7d5b05fSDeng-Cheng Zhu old_entryhi = read_c0_entryhi(); 488d7d5b05fSDeng-Cheng Zhu 489d7d5b05fSDeng-Cheng Zhu write_c0_entryhi((va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu)); 490d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 491d7d5b05fSDeng-Cheng Zhu 492d7d5b05fSDeng-Cheng Zhu tlb_probe(); 493d7d5b05fSDeng-Cheng Zhu tlb_probe_hazard(); 494d7d5b05fSDeng-Cheng Zhu idx = read_c0_index(); 495d7d5b05fSDeng-Cheng Zhu 496d7d5b05fSDeng-Cheng Zhu if (idx >= current_cpu_data.tlbsize) 497d7d5b05fSDeng-Cheng Zhu BUG(); 498d7d5b05fSDeng-Cheng Zhu 499d7d5b05fSDeng-Cheng Zhu if (idx > 0) { 500d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 501d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 502d7d5b05fSDeng-Cheng Zhu 503d7d5b05fSDeng-Cheng Zhu write_c0_entrylo0(0); 504d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 505d7d5b05fSDeng-Cheng Zhu 506d7d5b05fSDeng-Cheng Zhu write_c0_entrylo1(0); 507d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 508d7d5b05fSDeng-Cheng Zhu 509d7d5b05fSDeng-Cheng Zhu tlb_write_indexed(); 510d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 511d7d5b05fSDeng-Cheng Zhu } 512d7d5b05fSDeng-Cheng Zhu 513d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_entryhi); 514d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 515d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 516d7d5b05fSDeng-Cheng Zhu 517d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 518d7d5b05fSDeng-Cheng Zhu 519d7d5b05fSDeng-Cheng Zhu if (idx > 0) 520d7d5b05fSDeng-Cheng Zhu kvm_debug("%s: Invalidated entryhi %#lx @ idx %d\n", __func__, 521d7d5b05fSDeng-Cheng Zhu (va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu), idx); 522d7d5b05fSDeng-Cheng Zhu 523d7d5b05fSDeng-Cheng Zhu return 0; 524d7d5b05fSDeng-Cheng Zhu } 525cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_inv); 526d7d5b05fSDeng-Cheng Zhu 527d7d5b05fSDeng-Cheng Zhu void kvm_mips_flush_host_tlb(int skip_kseg0) 528d7d5b05fSDeng-Cheng Zhu { 529d7d5b05fSDeng-Cheng Zhu unsigned long flags; 530d7d5b05fSDeng-Cheng Zhu unsigned long old_entryhi, entryhi; 531d7d5b05fSDeng-Cheng Zhu unsigned long old_pagemask; 532d7d5b05fSDeng-Cheng Zhu int entry = 0; 533d7d5b05fSDeng-Cheng Zhu int maxentry = current_cpu_data.tlbsize; 534d7d5b05fSDeng-Cheng Zhu 535d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 536d7d5b05fSDeng-Cheng Zhu 537d7d5b05fSDeng-Cheng Zhu old_entryhi = read_c0_entryhi(); 538d7d5b05fSDeng-Cheng Zhu old_pagemask = read_c0_pagemask(); 539d7d5b05fSDeng-Cheng Zhu 540d7d5b05fSDeng-Cheng Zhu /* Blast 'em all away. */ 541d7d5b05fSDeng-Cheng Zhu for (entry = 0; entry < maxentry; entry++) { 542d7d5b05fSDeng-Cheng Zhu write_c0_index(entry); 543d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 544d7d5b05fSDeng-Cheng Zhu 545d7d5b05fSDeng-Cheng Zhu if (skip_kseg0) { 546d7d5b05fSDeng-Cheng Zhu tlb_read(); 547d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 548d7d5b05fSDeng-Cheng Zhu 549d7d5b05fSDeng-Cheng Zhu entryhi = read_c0_entryhi(); 550d7d5b05fSDeng-Cheng Zhu 551d7d5b05fSDeng-Cheng Zhu /* Don't blow away guest kernel entries */ 552d7d5b05fSDeng-Cheng Zhu if (KVM_GUEST_KSEGX(entryhi) == KVM_GUEST_KSEG0) 553d7d5b05fSDeng-Cheng Zhu continue; 554d7d5b05fSDeng-Cheng Zhu } 555d7d5b05fSDeng-Cheng Zhu 556d7d5b05fSDeng-Cheng Zhu /* Make sure all entries differ. */ 557d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(UNIQUE_ENTRYHI(entry)); 558d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 559d7d5b05fSDeng-Cheng Zhu write_c0_entrylo0(0); 560d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 561d7d5b05fSDeng-Cheng Zhu write_c0_entrylo1(0); 562d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 563d7d5b05fSDeng-Cheng Zhu 564d7d5b05fSDeng-Cheng Zhu tlb_write_indexed(); 565d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 566d7d5b05fSDeng-Cheng Zhu } 567d7d5b05fSDeng-Cheng Zhu 568d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 569d7d5b05fSDeng-Cheng Zhu 570d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_entryhi); 571d7d5b05fSDeng-Cheng Zhu write_c0_pagemask(old_pagemask); 572d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 573d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 574d7d5b05fSDeng-Cheng Zhu 575d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 576d7d5b05fSDeng-Cheng Zhu } 577cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_flush_host_tlb); 578d7d5b05fSDeng-Cheng Zhu 579d7d5b05fSDeng-Cheng Zhu void kvm_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu, 580d7d5b05fSDeng-Cheng Zhu struct kvm_vcpu *vcpu) 581d7d5b05fSDeng-Cheng Zhu { 582d7d5b05fSDeng-Cheng Zhu unsigned long asid = asid_cache(cpu); 583d7d5b05fSDeng-Cheng Zhu 5844edf00a4SPaul Burton asid += cpu_asid_inc(); 5854edf00a4SPaul Burton if (!(asid & cpu_asid_mask(&cpu_data[cpu]))) { 586d7d5b05fSDeng-Cheng Zhu if (cpu_has_vtag_icache) 587d7d5b05fSDeng-Cheng Zhu flush_icache_all(); 588d7d5b05fSDeng-Cheng Zhu 589d7d5b05fSDeng-Cheng Zhu kvm_local_flush_tlb_all(); /* start new asid cycle */ 590d7d5b05fSDeng-Cheng Zhu 591d7d5b05fSDeng-Cheng Zhu if (!asid) /* fix version if needed */ 5924edf00a4SPaul Burton asid = asid_first_version(cpu); 593d7d5b05fSDeng-Cheng Zhu } 594d7d5b05fSDeng-Cheng Zhu 595d7d5b05fSDeng-Cheng Zhu cpu_context(cpu, mm) = asid_cache(cpu) = asid; 596d7d5b05fSDeng-Cheng Zhu } 597d7d5b05fSDeng-Cheng Zhu 598d7d5b05fSDeng-Cheng Zhu void kvm_local_flush_tlb_all(void) 599d7d5b05fSDeng-Cheng Zhu { 600d7d5b05fSDeng-Cheng Zhu unsigned long flags; 601d7d5b05fSDeng-Cheng Zhu unsigned long old_ctx; 602d7d5b05fSDeng-Cheng Zhu int entry = 0; 603d7d5b05fSDeng-Cheng Zhu 604d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 605d7d5b05fSDeng-Cheng Zhu /* Save old context and create impossible VPN2 value */ 606d7d5b05fSDeng-Cheng Zhu old_ctx = read_c0_entryhi(); 607d7d5b05fSDeng-Cheng Zhu write_c0_entrylo0(0); 608d7d5b05fSDeng-Cheng Zhu write_c0_entrylo1(0); 609d7d5b05fSDeng-Cheng Zhu 610d7d5b05fSDeng-Cheng Zhu /* Blast 'em all away. */ 611d7d5b05fSDeng-Cheng Zhu while (entry < current_cpu_data.tlbsize) { 612d7d5b05fSDeng-Cheng Zhu /* Make sure all entries differ. */ 613d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(UNIQUE_ENTRYHI(entry)); 614d7d5b05fSDeng-Cheng Zhu write_c0_index(entry); 615d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 616d7d5b05fSDeng-Cheng Zhu tlb_write_indexed(); 617d7d5b05fSDeng-Cheng Zhu entry++; 618d7d5b05fSDeng-Cheng Zhu } 619d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 620d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_ctx); 621d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 622d7d5b05fSDeng-Cheng Zhu 623d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 624d7d5b05fSDeng-Cheng Zhu } 625cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_local_flush_tlb_all); 626d7d5b05fSDeng-Cheng Zhu 627d7d5b05fSDeng-Cheng Zhu /** 628d7d5b05fSDeng-Cheng Zhu * kvm_mips_migrate_count() - Migrate timer. 629d7d5b05fSDeng-Cheng Zhu * @vcpu: Virtual CPU. 630d7d5b05fSDeng-Cheng Zhu * 631d7d5b05fSDeng-Cheng Zhu * Migrate CP0_Count hrtimer to the current CPU by cancelling and restarting it 632d7d5b05fSDeng-Cheng Zhu * if it was running prior to being cancelled. 633d7d5b05fSDeng-Cheng Zhu * 634d7d5b05fSDeng-Cheng Zhu * Must be called when the VCPU is migrated to a different CPU to ensure that 635d7d5b05fSDeng-Cheng Zhu * timer expiry during guest execution interrupts the guest and causes the 636d7d5b05fSDeng-Cheng Zhu * interrupt to be delivered in a timely manner. 637d7d5b05fSDeng-Cheng Zhu */ 638d7d5b05fSDeng-Cheng Zhu static void kvm_mips_migrate_count(struct kvm_vcpu *vcpu) 639d7d5b05fSDeng-Cheng Zhu { 640d7d5b05fSDeng-Cheng Zhu if (hrtimer_cancel(&vcpu->arch.comparecount_timer)) 641d7d5b05fSDeng-Cheng Zhu hrtimer_restart(&vcpu->arch.comparecount_timer); 642d7d5b05fSDeng-Cheng Zhu } 643d7d5b05fSDeng-Cheng Zhu 644d7d5b05fSDeng-Cheng Zhu /* Restore ASID once we are scheduled back after preemption */ 645d7d5b05fSDeng-Cheng Zhu void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 646d7d5b05fSDeng-Cheng Zhu { 6474edf00a4SPaul Burton unsigned long asid_mask = cpu_asid_mask(&cpu_data[cpu]); 648d7d5b05fSDeng-Cheng Zhu unsigned long flags; 649d7d5b05fSDeng-Cheng Zhu int newasid = 0; 650d7d5b05fSDeng-Cheng Zhu 651d7d5b05fSDeng-Cheng Zhu kvm_debug("%s: vcpu %p, cpu: %d\n", __func__, vcpu, cpu); 652d7d5b05fSDeng-Cheng Zhu 65392a76f6dSAdam Buchbinder /* Allocate new kernel and user ASIDs if needed */ 654d7d5b05fSDeng-Cheng Zhu 655d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 656d7d5b05fSDeng-Cheng Zhu 657caa1faa7SJames Hogan if ((vcpu->arch.guest_kernel_asid[cpu] ^ asid_cache(cpu)) & 6584edf00a4SPaul Burton asid_version_mask(cpu)) { 659d7d5b05fSDeng-Cheng Zhu kvm_get_new_mmu_context(&vcpu->arch.guest_kernel_mm, cpu, vcpu); 660d7d5b05fSDeng-Cheng Zhu vcpu->arch.guest_kernel_asid[cpu] = 661d7d5b05fSDeng-Cheng Zhu vcpu->arch.guest_kernel_mm.context.asid[cpu]; 662d7d5b05fSDeng-Cheng Zhu kvm_get_new_mmu_context(&vcpu->arch.guest_user_mm, cpu, vcpu); 663d7d5b05fSDeng-Cheng Zhu vcpu->arch.guest_user_asid[cpu] = 664d7d5b05fSDeng-Cheng Zhu vcpu->arch.guest_user_mm.context.asid[cpu]; 665d7d5b05fSDeng-Cheng Zhu newasid++; 666d7d5b05fSDeng-Cheng Zhu 667d7d5b05fSDeng-Cheng Zhu kvm_debug("[%d]: cpu_context: %#lx\n", cpu, 668d7d5b05fSDeng-Cheng Zhu cpu_context(cpu, current->mm)); 669d7d5b05fSDeng-Cheng Zhu kvm_debug("[%d]: Allocated new ASID for Guest Kernel: %#x\n", 670d7d5b05fSDeng-Cheng Zhu cpu, vcpu->arch.guest_kernel_asid[cpu]); 671d7d5b05fSDeng-Cheng Zhu kvm_debug("[%d]: Allocated new ASID for Guest User: %#x\n", cpu, 672d7d5b05fSDeng-Cheng Zhu vcpu->arch.guest_user_asid[cpu]); 673d7d5b05fSDeng-Cheng Zhu } 674d7d5b05fSDeng-Cheng Zhu 675d7d5b05fSDeng-Cheng Zhu if (vcpu->arch.last_sched_cpu != cpu) { 676d7d5b05fSDeng-Cheng Zhu kvm_debug("[%d->%d]KVM VCPU[%d] switch\n", 677d7d5b05fSDeng-Cheng Zhu vcpu->arch.last_sched_cpu, cpu, vcpu->vcpu_id); 678d7d5b05fSDeng-Cheng Zhu /* 679d7d5b05fSDeng-Cheng Zhu * Migrate the timer interrupt to the current CPU so that it 680d7d5b05fSDeng-Cheng Zhu * always interrupts the guest and synchronously triggers a 681d7d5b05fSDeng-Cheng Zhu * guest timer interrupt. 682d7d5b05fSDeng-Cheng Zhu */ 683d7d5b05fSDeng-Cheng Zhu kvm_mips_migrate_count(vcpu); 684d7d5b05fSDeng-Cheng Zhu } 685d7d5b05fSDeng-Cheng Zhu 686d7d5b05fSDeng-Cheng Zhu if (!newasid) { 687d7d5b05fSDeng-Cheng Zhu /* 688d7d5b05fSDeng-Cheng Zhu * If we preempted while the guest was executing, then reload 689d7d5b05fSDeng-Cheng Zhu * the pre-empted ASID 690d7d5b05fSDeng-Cheng Zhu */ 691d7d5b05fSDeng-Cheng Zhu if (current->flags & PF_VCPU) { 692d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(vcpu->arch. 6934edf00a4SPaul Burton preempt_entryhi & asid_mask); 694d7d5b05fSDeng-Cheng Zhu ehb(); 695d7d5b05fSDeng-Cheng Zhu } 696d7d5b05fSDeng-Cheng Zhu } else { 697d7d5b05fSDeng-Cheng Zhu /* New ASIDs were allocated for the VM */ 698d7d5b05fSDeng-Cheng Zhu 699d7d5b05fSDeng-Cheng Zhu /* 700d7d5b05fSDeng-Cheng Zhu * Were we in guest context? If so then the pre-empted ASID is 701d7d5b05fSDeng-Cheng Zhu * no longer valid, we need to set it to what it should be based 702d7d5b05fSDeng-Cheng Zhu * on the mode of the Guest (Kernel/User) 703d7d5b05fSDeng-Cheng Zhu */ 704d7d5b05fSDeng-Cheng Zhu if (current->flags & PF_VCPU) { 705d7d5b05fSDeng-Cheng Zhu if (KVM_GUEST_KERNEL_MODE(vcpu)) 706d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(vcpu->arch. 707d7d5b05fSDeng-Cheng Zhu guest_kernel_asid[cpu] & 7084edf00a4SPaul Burton asid_mask); 709d7d5b05fSDeng-Cheng Zhu else 710d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(vcpu->arch. 711d7d5b05fSDeng-Cheng Zhu guest_user_asid[cpu] & 7124edf00a4SPaul Burton asid_mask); 713d7d5b05fSDeng-Cheng Zhu ehb(); 714d7d5b05fSDeng-Cheng Zhu } 715d7d5b05fSDeng-Cheng Zhu } 716d7d5b05fSDeng-Cheng Zhu 717b86ecb37SJames Hogan /* restore guest state to registers */ 718b86ecb37SJames Hogan kvm_mips_callbacks->vcpu_set_regs(vcpu); 719b86ecb37SJames Hogan 720d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 721d7d5b05fSDeng-Cheng Zhu 722d7d5b05fSDeng-Cheng Zhu } 723cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_arch_vcpu_load); 724d7d5b05fSDeng-Cheng Zhu 725d7d5b05fSDeng-Cheng Zhu /* ASID can change if another task is scheduled during preemption */ 726d7d5b05fSDeng-Cheng Zhu void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 727d7d5b05fSDeng-Cheng Zhu { 728d7d5b05fSDeng-Cheng Zhu unsigned long flags; 729d7d5b05fSDeng-Cheng Zhu uint32_t cpu; 730d7d5b05fSDeng-Cheng Zhu 731d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 732d7d5b05fSDeng-Cheng Zhu 733d7d5b05fSDeng-Cheng Zhu cpu = smp_processor_id(); 734d7d5b05fSDeng-Cheng Zhu 735d7d5b05fSDeng-Cheng Zhu vcpu->arch.preempt_entryhi = read_c0_entryhi(); 736d7d5b05fSDeng-Cheng Zhu vcpu->arch.last_sched_cpu = cpu; 737d7d5b05fSDeng-Cheng Zhu 738b86ecb37SJames Hogan /* save guest state in registers */ 739b86ecb37SJames Hogan kvm_mips_callbacks->vcpu_get_regs(vcpu); 740b86ecb37SJames Hogan 741d7d5b05fSDeng-Cheng Zhu if (((cpu_context(cpu, current->mm) ^ asid_cache(cpu)) & 7424edf00a4SPaul Burton asid_version_mask(cpu))) { 743d7d5b05fSDeng-Cheng Zhu kvm_debug("%s: Dropping MMU Context: %#lx\n", __func__, 744d7d5b05fSDeng-Cheng Zhu cpu_context(cpu, current->mm)); 745d7d5b05fSDeng-Cheng Zhu drop_mmu_context(current->mm, cpu); 746d7d5b05fSDeng-Cheng Zhu } 747d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(cpu_asid(cpu, current->mm)); 748d7d5b05fSDeng-Cheng Zhu ehb(); 749d7d5b05fSDeng-Cheng Zhu 750d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 751d7d5b05fSDeng-Cheng Zhu } 752cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_arch_vcpu_put); 753d7d5b05fSDeng-Cheng Zhu 754bdb7ed86SJames Hogan u32 kvm_get_inst(u32 *opc, struct kvm_vcpu *vcpu) 755d7d5b05fSDeng-Cheng Zhu { 756d7d5b05fSDeng-Cheng Zhu struct mips_coproc *cop0 = vcpu->arch.cop0; 757d7d5b05fSDeng-Cheng Zhu unsigned long paddr, flags, vpn2, asid; 758d7d5b05fSDeng-Cheng Zhu uint32_t inst; 759d7d5b05fSDeng-Cheng Zhu int index; 760d7d5b05fSDeng-Cheng Zhu 761d7d5b05fSDeng-Cheng Zhu if (KVM_GUEST_KSEGX((unsigned long) opc) < KVM_GUEST_KSEG0 || 762d7d5b05fSDeng-Cheng Zhu KVM_GUEST_KSEGX((unsigned long) opc) == KVM_GUEST_KSEG23) { 763d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 764d7d5b05fSDeng-Cheng Zhu index = kvm_mips_host_tlb_lookup(vcpu, (unsigned long) opc); 765d7d5b05fSDeng-Cheng Zhu if (index >= 0) { 766d7d5b05fSDeng-Cheng Zhu inst = *(opc); 767d7d5b05fSDeng-Cheng Zhu } else { 768d7d5b05fSDeng-Cheng Zhu vpn2 = (unsigned long) opc & VPN2_MASK; 769ca64c2beSPaul Burton asid = kvm_read_c0_guest_entryhi(cop0) & 770ca64c2beSPaul Burton KVM_ENTRYHI_ASID; 771d7d5b05fSDeng-Cheng Zhu index = kvm_mips_guest_tlb_lookup(vcpu, vpn2 | asid); 772d7d5b05fSDeng-Cheng Zhu if (index < 0) { 773d7d5b05fSDeng-Cheng Zhu kvm_err("%s: get_user_failed for %p, vcpu: %p, ASID: %#lx\n", 774d7d5b05fSDeng-Cheng Zhu __func__, opc, vcpu, read_c0_entryhi()); 775d7d5b05fSDeng-Cheng Zhu kvm_mips_dump_host_tlbs(); 776d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 777d7d5b05fSDeng-Cheng Zhu return KVM_INVALID_INST; 778d7d5b05fSDeng-Cheng Zhu } 779d7d5b05fSDeng-Cheng Zhu kvm_mips_handle_mapped_seg_tlb_fault(vcpu, 780d7d5b05fSDeng-Cheng Zhu &vcpu->arch. 781d7d5b05fSDeng-Cheng Zhu guest_tlb[index], 782d7d5b05fSDeng-Cheng Zhu NULL, NULL); 783d7d5b05fSDeng-Cheng Zhu inst = *(opc); 784d7d5b05fSDeng-Cheng Zhu } 785d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 786d7d5b05fSDeng-Cheng Zhu } else if (KVM_GUEST_KSEGX(opc) == KVM_GUEST_KSEG0) { 787d7d5b05fSDeng-Cheng Zhu paddr = 788d7d5b05fSDeng-Cheng Zhu kvm_mips_translate_guest_kseg0_to_hpa(vcpu, 789d7d5b05fSDeng-Cheng Zhu (unsigned long) opc); 790d7d5b05fSDeng-Cheng Zhu inst = *(uint32_t *) CKSEG0ADDR(paddr); 791d7d5b05fSDeng-Cheng Zhu } else { 792d7d5b05fSDeng-Cheng Zhu kvm_err("%s: illegal address: %p\n", __func__, opc); 793d7d5b05fSDeng-Cheng Zhu return KVM_INVALID_INST; 794d7d5b05fSDeng-Cheng Zhu } 795d7d5b05fSDeng-Cheng Zhu 796d7d5b05fSDeng-Cheng Zhu return inst; 797d7d5b05fSDeng-Cheng Zhu } 798cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_get_inst); 799