1d7d5b05fSDeng-Cheng Zhu /* 2d7d5b05fSDeng-Cheng Zhu * This file is subject to the terms and conditions of the GNU General Public 3d7d5b05fSDeng-Cheng Zhu * License. See the file "COPYING" in the main directory of this archive 4d7d5b05fSDeng-Cheng Zhu * for more details. 5d7d5b05fSDeng-Cheng Zhu * 6d7d5b05fSDeng-Cheng Zhu * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that 7d7d5b05fSDeng-Cheng Zhu * TLB handlers run from KSEG0 8d7d5b05fSDeng-Cheng Zhu * 9d7d5b05fSDeng-Cheng Zhu * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. 10d7d5b05fSDeng-Cheng Zhu * Authors: Sanjay Lal <sanjayl@kymasys.com> 11d7d5b05fSDeng-Cheng Zhu */ 12d7d5b05fSDeng-Cheng Zhu 13d7d5b05fSDeng-Cheng Zhu #include <linux/sched.h> 14d7d5b05fSDeng-Cheng Zhu #include <linux/smp.h> 15d7d5b05fSDeng-Cheng Zhu #include <linux/mm.h> 16d7d5b05fSDeng-Cheng Zhu #include <linux/delay.h> 17d7d5b05fSDeng-Cheng Zhu #include <linux/module.h> 18d7d5b05fSDeng-Cheng Zhu #include <linux/kvm_host.h> 19d7d5b05fSDeng-Cheng Zhu #include <linux/srcu.h> 20d7d5b05fSDeng-Cheng Zhu 21d7d5b05fSDeng-Cheng Zhu #include <asm/cpu.h> 22d7d5b05fSDeng-Cheng Zhu #include <asm/bootinfo.h> 23d7d5b05fSDeng-Cheng Zhu #include <asm/mmu_context.h> 24d7d5b05fSDeng-Cheng Zhu #include <asm/pgtable.h> 25d7d5b05fSDeng-Cheng Zhu #include <asm/cacheflush.h> 26d7d5b05fSDeng-Cheng Zhu #include <asm/tlb.h> 27d7d5b05fSDeng-Cheng Zhu 28d7d5b05fSDeng-Cheng Zhu #undef CONFIG_MIPS_MT 29d7d5b05fSDeng-Cheng Zhu #include <asm/r4kcache.h> 30d7d5b05fSDeng-Cheng Zhu #define CONFIG_MIPS_MT 31d7d5b05fSDeng-Cheng Zhu 32d7d5b05fSDeng-Cheng Zhu #define KVM_GUEST_PC_TLB 0 33d7d5b05fSDeng-Cheng Zhu #define KVM_GUEST_SP_TLB 1 34d7d5b05fSDeng-Cheng Zhu 35d7d5b05fSDeng-Cheng Zhu #define PRIx64 "llx" 36d7d5b05fSDeng-Cheng Zhu 37d7d5b05fSDeng-Cheng Zhu atomic_t kvm_mips_instance; 38d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_mips_instance); 39d7d5b05fSDeng-Cheng Zhu 40d7d5b05fSDeng-Cheng Zhu /* These function pointers are initialized once the KVM module is loaded */ 41d7d5b05fSDeng-Cheng Zhu pfn_t (*kvm_mips_gfn_to_pfn)(struct kvm *kvm, gfn_t gfn); 42d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_mips_gfn_to_pfn); 43d7d5b05fSDeng-Cheng Zhu 44d7d5b05fSDeng-Cheng Zhu void (*kvm_mips_release_pfn_clean)(pfn_t pfn); 45d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_mips_release_pfn_clean); 46d7d5b05fSDeng-Cheng Zhu 47d7d5b05fSDeng-Cheng Zhu bool (*kvm_mips_is_error_pfn)(pfn_t pfn); 48d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_mips_is_error_pfn); 49d7d5b05fSDeng-Cheng Zhu 50d7d5b05fSDeng-Cheng Zhu uint32_t kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu) 51d7d5b05fSDeng-Cheng Zhu { 52d7d5b05fSDeng-Cheng Zhu return vcpu->arch.guest_kernel_asid[smp_processor_id()] & ASID_MASK; 53d7d5b05fSDeng-Cheng Zhu } 54d7d5b05fSDeng-Cheng Zhu 55d7d5b05fSDeng-Cheng Zhu uint32_t kvm_mips_get_user_asid(struct kvm_vcpu *vcpu) 56d7d5b05fSDeng-Cheng Zhu { 57d7d5b05fSDeng-Cheng Zhu return vcpu->arch.guest_user_asid[smp_processor_id()] & ASID_MASK; 58d7d5b05fSDeng-Cheng Zhu } 59d7d5b05fSDeng-Cheng Zhu 60d7d5b05fSDeng-Cheng Zhu inline uint32_t kvm_mips_get_commpage_asid(struct kvm_vcpu *vcpu) 61d7d5b05fSDeng-Cheng Zhu { 62d7d5b05fSDeng-Cheng Zhu return vcpu->kvm->arch.commpage_tlb; 63d7d5b05fSDeng-Cheng Zhu } 64d7d5b05fSDeng-Cheng Zhu 65d7d5b05fSDeng-Cheng Zhu /* Structure defining an tlb entry data set. */ 66d7d5b05fSDeng-Cheng Zhu 67d7d5b05fSDeng-Cheng Zhu void kvm_mips_dump_host_tlbs(void) 68d7d5b05fSDeng-Cheng Zhu { 69d7d5b05fSDeng-Cheng Zhu unsigned long old_entryhi; 70d7d5b05fSDeng-Cheng Zhu unsigned long old_pagemask; 71d7d5b05fSDeng-Cheng Zhu struct kvm_mips_tlb tlb; 72d7d5b05fSDeng-Cheng Zhu unsigned long flags; 73d7d5b05fSDeng-Cheng Zhu int i; 74d7d5b05fSDeng-Cheng Zhu 75d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 76d7d5b05fSDeng-Cheng Zhu 77d7d5b05fSDeng-Cheng Zhu old_entryhi = read_c0_entryhi(); 78d7d5b05fSDeng-Cheng Zhu old_pagemask = read_c0_pagemask(); 79d7d5b05fSDeng-Cheng Zhu 80d7d5b05fSDeng-Cheng Zhu kvm_info("HOST TLBs:\n"); 81d7d5b05fSDeng-Cheng Zhu kvm_info("ASID: %#lx\n", read_c0_entryhi() & ASID_MASK); 82d7d5b05fSDeng-Cheng Zhu 83d7d5b05fSDeng-Cheng Zhu for (i = 0; i < current_cpu_data.tlbsize; i++) { 84d7d5b05fSDeng-Cheng Zhu write_c0_index(i); 85d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 86d7d5b05fSDeng-Cheng Zhu 87d7d5b05fSDeng-Cheng Zhu tlb_read(); 88d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 89d7d5b05fSDeng-Cheng Zhu 90d7d5b05fSDeng-Cheng Zhu tlb.tlb_hi = read_c0_entryhi(); 91d7d5b05fSDeng-Cheng Zhu tlb.tlb_lo0 = read_c0_entrylo0(); 92d7d5b05fSDeng-Cheng Zhu tlb.tlb_lo1 = read_c0_entrylo1(); 93d7d5b05fSDeng-Cheng Zhu tlb.tlb_mask = read_c0_pagemask(); 94d7d5b05fSDeng-Cheng Zhu 95d7d5b05fSDeng-Cheng Zhu kvm_info("TLB%c%3d Hi 0x%08lx ", 96d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 | tlb.tlb_lo1) & MIPS3_PG_V ? ' ' : '*', 97d7d5b05fSDeng-Cheng Zhu i, tlb.tlb_hi); 98d7d5b05fSDeng-Cheng Zhu kvm_info("Lo0=0x%09" PRIx64 " %c%c attr %lx ", 99d7d5b05fSDeng-Cheng Zhu (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo0), 100d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 & MIPS3_PG_D) ? 'D' : ' ', 101d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 & MIPS3_PG_G) ? 'G' : ' ', 102d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 >> 3) & 7); 103d7d5b05fSDeng-Cheng Zhu kvm_info("Lo1=0x%09" PRIx64 " %c%c attr %lx sz=%lx\n", 104d7d5b05fSDeng-Cheng Zhu (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo1), 105d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo1 & MIPS3_PG_D) ? 'D' : ' ', 106d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo1 & MIPS3_PG_G) ? 'G' : ' ', 107d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo1 >> 3) & 7, tlb.tlb_mask); 108d7d5b05fSDeng-Cheng Zhu } 109d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_entryhi); 110d7d5b05fSDeng-Cheng Zhu write_c0_pagemask(old_pagemask); 111d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 112d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 113d7d5b05fSDeng-Cheng Zhu } 114d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_mips_dump_host_tlbs); 115d7d5b05fSDeng-Cheng Zhu 116d7d5b05fSDeng-Cheng Zhu void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu) 117d7d5b05fSDeng-Cheng Zhu { 118d7d5b05fSDeng-Cheng Zhu struct mips_coproc *cop0 = vcpu->arch.cop0; 119d7d5b05fSDeng-Cheng Zhu struct kvm_mips_tlb tlb; 120d7d5b05fSDeng-Cheng Zhu int i; 121d7d5b05fSDeng-Cheng Zhu 122d7d5b05fSDeng-Cheng Zhu kvm_info("Guest TLBs:\n"); 123d7d5b05fSDeng-Cheng Zhu kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0)); 124d7d5b05fSDeng-Cheng Zhu 125d7d5b05fSDeng-Cheng Zhu for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) { 126d7d5b05fSDeng-Cheng Zhu tlb = vcpu->arch.guest_tlb[i]; 127d7d5b05fSDeng-Cheng Zhu kvm_info("TLB%c%3d Hi 0x%08lx ", 128d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 | tlb.tlb_lo1) & MIPS3_PG_V ? ' ' : '*', 129d7d5b05fSDeng-Cheng Zhu i, tlb.tlb_hi); 130d7d5b05fSDeng-Cheng Zhu kvm_info("Lo0=0x%09" PRIx64 " %c%c attr %lx ", 131d7d5b05fSDeng-Cheng Zhu (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo0), 132d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 & MIPS3_PG_D) ? 'D' : ' ', 133d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 & MIPS3_PG_G) ? 'G' : ' ', 134d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo0 >> 3) & 7); 135d7d5b05fSDeng-Cheng Zhu kvm_info("Lo1=0x%09" PRIx64 " %c%c attr %lx sz=%lx\n", 136d7d5b05fSDeng-Cheng Zhu (uint64_t) mips3_tlbpfn_to_paddr(tlb.tlb_lo1), 137d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo1 & MIPS3_PG_D) ? 'D' : ' ', 138d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo1 & MIPS3_PG_G) ? 'G' : ' ', 139d7d5b05fSDeng-Cheng Zhu (tlb.tlb_lo1 >> 3) & 7, tlb.tlb_mask); 140d7d5b05fSDeng-Cheng Zhu } 141d7d5b05fSDeng-Cheng Zhu } 142d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_mips_dump_guest_tlbs); 143d7d5b05fSDeng-Cheng Zhu 144d7d5b05fSDeng-Cheng Zhu static int kvm_mips_map_page(struct kvm *kvm, gfn_t gfn) 145d7d5b05fSDeng-Cheng Zhu { 146d7d5b05fSDeng-Cheng Zhu int srcu_idx, err = 0; 147d7d5b05fSDeng-Cheng Zhu pfn_t pfn; 148d7d5b05fSDeng-Cheng Zhu 149d7d5b05fSDeng-Cheng Zhu if (kvm->arch.guest_pmap[gfn] != KVM_INVALID_PAGE) 150d7d5b05fSDeng-Cheng Zhu return 0; 151d7d5b05fSDeng-Cheng Zhu 152d7d5b05fSDeng-Cheng Zhu srcu_idx = srcu_read_lock(&kvm->srcu); 153d7d5b05fSDeng-Cheng Zhu pfn = kvm_mips_gfn_to_pfn(kvm, gfn); 154d7d5b05fSDeng-Cheng Zhu 155d7d5b05fSDeng-Cheng Zhu if (kvm_mips_is_error_pfn(pfn)) { 156d7d5b05fSDeng-Cheng Zhu kvm_err("Couldn't get pfn for gfn %#" PRIx64 "!\n", gfn); 157d7d5b05fSDeng-Cheng Zhu err = -EFAULT; 158d7d5b05fSDeng-Cheng Zhu goto out; 159d7d5b05fSDeng-Cheng Zhu } 160d7d5b05fSDeng-Cheng Zhu 161d7d5b05fSDeng-Cheng Zhu kvm->arch.guest_pmap[gfn] = pfn; 162d7d5b05fSDeng-Cheng Zhu out: 163d7d5b05fSDeng-Cheng Zhu srcu_read_unlock(&kvm->srcu, srcu_idx); 164d7d5b05fSDeng-Cheng Zhu return err; 165d7d5b05fSDeng-Cheng Zhu } 166d7d5b05fSDeng-Cheng Zhu 167d7d5b05fSDeng-Cheng Zhu /* Translate guest KSEG0 addresses to Host PA */ 168d7d5b05fSDeng-Cheng Zhu unsigned long kvm_mips_translate_guest_kseg0_to_hpa(struct kvm_vcpu *vcpu, 169d7d5b05fSDeng-Cheng Zhu unsigned long gva) 170d7d5b05fSDeng-Cheng Zhu { 171d7d5b05fSDeng-Cheng Zhu gfn_t gfn; 172d7d5b05fSDeng-Cheng Zhu uint32_t offset = gva & ~PAGE_MASK; 173d7d5b05fSDeng-Cheng Zhu struct kvm *kvm = vcpu->kvm; 174d7d5b05fSDeng-Cheng Zhu 175d7d5b05fSDeng-Cheng Zhu if (KVM_GUEST_KSEGX(gva) != KVM_GUEST_KSEG0) { 176d7d5b05fSDeng-Cheng Zhu kvm_err("%s/%p: Invalid gva: %#lx\n", __func__, 177d7d5b05fSDeng-Cheng Zhu __builtin_return_address(0), gva); 178d7d5b05fSDeng-Cheng Zhu return KVM_INVALID_PAGE; 179d7d5b05fSDeng-Cheng Zhu } 180d7d5b05fSDeng-Cheng Zhu 181d7d5b05fSDeng-Cheng Zhu gfn = (KVM_GUEST_CPHYSADDR(gva) >> PAGE_SHIFT); 182d7d5b05fSDeng-Cheng Zhu 183d7d5b05fSDeng-Cheng Zhu if (gfn >= kvm->arch.guest_pmap_npages) { 184d7d5b05fSDeng-Cheng Zhu kvm_err("%s: Invalid gfn: %#llx, GVA: %#lx\n", __func__, gfn, 185d7d5b05fSDeng-Cheng Zhu gva); 186d7d5b05fSDeng-Cheng Zhu return KVM_INVALID_PAGE; 187d7d5b05fSDeng-Cheng Zhu } 188d7d5b05fSDeng-Cheng Zhu 189d7d5b05fSDeng-Cheng Zhu if (kvm_mips_map_page(vcpu->kvm, gfn) < 0) 190d7d5b05fSDeng-Cheng Zhu return KVM_INVALID_ADDR; 191d7d5b05fSDeng-Cheng Zhu 192d7d5b05fSDeng-Cheng Zhu return (kvm->arch.guest_pmap[gfn] << PAGE_SHIFT) + offset; 193d7d5b05fSDeng-Cheng Zhu } 194d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_mips_translate_guest_kseg0_to_hpa); 195d7d5b05fSDeng-Cheng Zhu 196d7d5b05fSDeng-Cheng Zhu /* XXXKYMA: Must be called with interrupts disabled */ 197d7d5b05fSDeng-Cheng Zhu /* set flush_dcache_mask == 0 if no dcache flush required */ 198d7d5b05fSDeng-Cheng Zhu int kvm_mips_host_tlb_write(struct kvm_vcpu *vcpu, unsigned long entryhi, 199d7d5b05fSDeng-Cheng Zhu unsigned long entrylo0, unsigned long entrylo1, 200d7d5b05fSDeng-Cheng Zhu int flush_dcache_mask) 201d7d5b05fSDeng-Cheng Zhu { 202d7d5b05fSDeng-Cheng Zhu unsigned long flags; 203d7d5b05fSDeng-Cheng Zhu unsigned long old_entryhi; 204d7d5b05fSDeng-Cheng Zhu int idx; 205d7d5b05fSDeng-Cheng Zhu 206d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 207d7d5b05fSDeng-Cheng Zhu 208d7d5b05fSDeng-Cheng Zhu old_entryhi = read_c0_entryhi(); 209d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(entryhi); 210d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 211d7d5b05fSDeng-Cheng Zhu 212d7d5b05fSDeng-Cheng Zhu tlb_probe(); 213d7d5b05fSDeng-Cheng Zhu tlb_probe_hazard(); 214d7d5b05fSDeng-Cheng Zhu idx = read_c0_index(); 215d7d5b05fSDeng-Cheng Zhu 216d7d5b05fSDeng-Cheng Zhu if (idx > current_cpu_data.tlbsize) { 217d7d5b05fSDeng-Cheng Zhu kvm_err("%s: Invalid Index: %d\n", __func__, idx); 218d7d5b05fSDeng-Cheng Zhu kvm_mips_dump_host_tlbs(); 219cfec0e75STapasweni Pathak local_irq_restore(flags); 220d7d5b05fSDeng-Cheng Zhu return -1; 221d7d5b05fSDeng-Cheng Zhu } 222d7d5b05fSDeng-Cheng Zhu 223d7d5b05fSDeng-Cheng Zhu write_c0_entrylo0(entrylo0); 224d7d5b05fSDeng-Cheng Zhu write_c0_entrylo1(entrylo1); 225d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 226d7d5b05fSDeng-Cheng Zhu 227d7d5b05fSDeng-Cheng Zhu if (idx < 0) 228d7d5b05fSDeng-Cheng Zhu tlb_write_random(); 229d7d5b05fSDeng-Cheng Zhu else 230d7d5b05fSDeng-Cheng Zhu tlb_write_indexed(); 231d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 232d7d5b05fSDeng-Cheng Zhu 233d7d5b05fSDeng-Cheng Zhu kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0(R): 0x%08lx, entrylo1(R): 0x%08lx\n", 234d7d5b05fSDeng-Cheng Zhu vcpu->arch.pc, idx, read_c0_entryhi(), 235d7d5b05fSDeng-Cheng Zhu read_c0_entrylo0(), read_c0_entrylo1()); 236d7d5b05fSDeng-Cheng Zhu 237d7d5b05fSDeng-Cheng Zhu /* Flush D-cache */ 238d7d5b05fSDeng-Cheng Zhu if (flush_dcache_mask) { 239d7d5b05fSDeng-Cheng Zhu if (entrylo0 & MIPS3_PG_V) { 240d7d5b05fSDeng-Cheng Zhu ++vcpu->stat.flush_dcache_exits; 241d7d5b05fSDeng-Cheng Zhu flush_data_cache_page((entryhi & VPN2_MASK) & 242d7d5b05fSDeng-Cheng Zhu ~flush_dcache_mask); 243d7d5b05fSDeng-Cheng Zhu } 244d7d5b05fSDeng-Cheng Zhu if (entrylo1 & MIPS3_PG_V) { 245d7d5b05fSDeng-Cheng Zhu ++vcpu->stat.flush_dcache_exits; 246d7d5b05fSDeng-Cheng Zhu flush_data_cache_page(((entryhi & VPN2_MASK) & 247d7d5b05fSDeng-Cheng Zhu ~flush_dcache_mask) | 248d7d5b05fSDeng-Cheng Zhu (0x1 << PAGE_SHIFT)); 249d7d5b05fSDeng-Cheng Zhu } 250d7d5b05fSDeng-Cheng Zhu } 251d7d5b05fSDeng-Cheng Zhu 252d7d5b05fSDeng-Cheng Zhu /* Restore old ASID */ 253d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_entryhi); 254d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 255d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 256d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 257d7d5b05fSDeng-Cheng Zhu return 0; 258d7d5b05fSDeng-Cheng Zhu } 259d7d5b05fSDeng-Cheng Zhu 260d7d5b05fSDeng-Cheng Zhu /* XXXKYMA: Must be called with interrupts disabled */ 261d7d5b05fSDeng-Cheng Zhu int kvm_mips_handle_kseg0_tlb_fault(unsigned long badvaddr, 262d7d5b05fSDeng-Cheng Zhu struct kvm_vcpu *vcpu) 263d7d5b05fSDeng-Cheng Zhu { 264d7d5b05fSDeng-Cheng Zhu gfn_t gfn; 265d7d5b05fSDeng-Cheng Zhu pfn_t pfn0, pfn1; 266d7d5b05fSDeng-Cheng Zhu unsigned long vaddr = 0; 267d7d5b05fSDeng-Cheng Zhu unsigned long entryhi = 0, entrylo0 = 0, entrylo1 = 0; 268d7d5b05fSDeng-Cheng Zhu int even; 269d7d5b05fSDeng-Cheng Zhu struct kvm *kvm = vcpu->kvm; 270d7d5b05fSDeng-Cheng Zhu const int flush_dcache_mask = 0; 271d7d5b05fSDeng-Cheng Zhu 272d7d5b05fSDeng-Cheng Zhu if (KVM_GUEST_KSEGX(badvaddr) != KVM_GUEST_KSEG0) { 273d7d5b05fSDeng-Cheng Zhu kvm_err("%s: Invalid BadVaddr: %#lx\n", __func__, badvaddr); 274d7d5b05fSDeng-Cheng Zhu kvm_mips_dump_host_tlbs(); 275d7d5b05fSDeng-Cheng Zhu return -1; 276d7d5b05fSDeng-Cheng Zhu } 277d7d5b05fSDeng-Cheng Zhu 278d7d5b05fSDeng-Cheng Zhu gfn = (KVM_GUEST_CPHYSADDR(badvaddr) >> PAGE_SHIFT); 279d7d5b05fSDeng-Cheng Zhu if (gfn >= kvm->arch.guest_pmap_npages) { 280d7d5b05fSDeng-Cheng Zhu kvm_err("%s: Invalid gfn: %#llx, BadVaddr: %#lx\n", __func__, 281d7d5b05fSDeng-Cheng Zhu gfn, badvaddr); 282d7d5b05fSDeng-Cheng Zhu kvm_mips_dump_host_tlbs(); 283d7d5b05fSDeng-Cheng Zhu return -1; 284d7d5b05fSDeng-Cheng Zhu } 285d7d5b05fSDeng-Cheng Zhu even = !(gfn & 0x1); 286d7d5b05fSDeng-Cheng Zhu vaddr = badvaddr & (PAGE_MASK << 1); 287d7d5b05fSDeng-Cheng Zhu 288d7d5b05fSDeng-Cheng Zhu if (kvm_mips_map_page(vcpu->kvm, gfn) < 0) 289d7d5b05fSDeng-Cheng Zhu return -1; 290d7d5b05fSDeng-Cheng Zhu 291d7d5b05fSDeng-Cheng Zhu if (kvm_mips_map_page(vcpu->kvm, gfn ^ 0x1) < 0) 292d7d5b05fSDeng-Cheng Zhu return -1; 293d7d5b05fSDeng-Cheng Zhu 294d7d5b05fSDeng-Cheng Zhu if (even) { 295d7d5b05fSDeng-Cheng Zhu pfn0 = kvm->arch.guest_pmap[gfn]; 296d7d5b05fSDeng-Cheng Zhu pfn1 = kvm->arch.guest_pmap[gfn ^ 0x1]; 297d7d5b05fSDeng-Cheng Zhu } else { 298d7d5b05fSDeng-Cheng Zhu pfn0 = kvm->arch.guest_pmap[gfn ^ 0x1]; 299d7d5b05fSDeng-Cheng Zhu pfn1 = kvm->arch.guest_pmap[gfn]; 300d7d5b05fSDeng-Cheng Zhu } 301d7d5b05fSDeng-Cheng Zhu 302d7d5b05fSDeng-Cheng Zhu entryhi = (vaddr | kvm_mips_get_kernel_asid(vcpu)); 303d7d5b05fSDeng-Cheng Zhu entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) | 304d7d5b05fSDeng-Cheng Zhu (1 << 2) | (0x1 << 1); 305d7d5b05fSDeng-Cheng Zhu entrylo1 = mips3_paddr_to_tlbpfn(pfn1 << PAGE_SHIFT) | (0x3 << 3) | 306d7d5b05fSDeng-Cheng Zhu (1 << 2) | (0x1 << 1); 307d7d5b05fSDeng-Cheng Zhu 308d7d5b05fSDeng-Cheng Zhu return kvm_mips_host_tlb_write(vcpu, entryhi, entrylo0, entrylo1, 309d7d5b05fSDeng-Cheng Zhu flush_dcache_mask); 310d7d5b05fSDeng-Cheng Zhu } 311d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_mips_handle_kseg0_tlb_fault); 312d7d5b05fSDeng-Cheng Zhu 313d7d5b05fSDeng-Cheng Zhu int kvm_mips_handle_commpage_tlb_fault(unsigned long badvaddr, 314d7d5b05fSDeng-Cheng Zhu struct kvm_vcpu *vcpu) 315d7d5b05fSDeng-Cheng Zhu { 316d7d5b05fSDeng-Cheng Zhu pfn_t pfn0, pfn1; 317d7d5b05fSDeng-Cheng Zhu unsigned long flags, old_entryhi = 0, vaddr = 0; 318d7d5b05fSDeng-Cheng Zhu unsigned long entrylo0 = 0, entrylo1 = 0; 319d7d5b05fSDeng-Cheng Zhu 320d7d5b05fSDeng-Cheng Zhu pfn0 = CPHYSADDR(vcpu->arch.kseg0_commpage) >> PAGE_SHIFT; 321d7d5b05fSDeng-Cheng Zhu pfn1 = 0; 322d7d5b05fSDeng-Cheng Zhu entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) | 323d7d5b05fSDeng-Cheng Zhu (1 << 2) | (0x1 << 1); 324d7d5b05fSDeng-Cheng Zhu entrylo1 = 0; 325d7d5b05fSDeng-Cheng Zhu 326d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 327d7d5b05fSDeng-Cheng Zhu 328d7d5b05fSDeng-Cheng Zhu old_entryhi = read_c0_entryhi(); 329d7d5b05fSDeng-Cheng Zhu vaddr = badvaddr & (PAGE_MASK << 1); 330d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(vaddr | kvm_mips_get_kernel_asid(vcpu)); 331d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 332d7d5b05fSDeng-Cheng Zhu write_c0_entrylo0(entrylo0); 333d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 334d7d5b05fSDeng-Cheng Zhu write_c0_entrylo1(entrylo1); 335d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 336d7d5b05fSDeng-Cheng Zhu write_c0_index(kvm_mips_get_commpage_asid(vcpu)); 337d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 338d7d5b05fSDeng-Cheng Zhu tlb_write_indexed(); 339d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 340d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 341d7d5b05fSDeng-Cheng Zhu 342d7d5b05fSDeng-Cheng Zhu kvm_debug("@ %#lx idx: %2d [entryhi(R): %#lx] entrylo0 (R): 0x%08lx, entrylo1(R): 0x%08lx\n", 343d7d5b05fSDeng-Cheng Zhu vcpu->arch.pc, read_c0_index(), read_c0_entryhi(), 344d7d5b05fSDeng-Cheng Zhu read_c0_entrylo0(), read_c0_entrylo1()); 345d7d5b05fSDeng-Cheng Zhu 346d7d5b05fSDeng-Cheng Zhu /* Restore old ASID */ 347d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_entryhi); 348d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 349d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 350d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 351d7d5b05fSDeng-Cheng Zhu 352d7d5b05fSDeng-Cheng Zhu return 0; 353d7d5b05fSDeng-Cheng Zhu } 354d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_mips_handle_commpage_tlb_fault); 355d7d5b05fSDeng-Cheng Zhu 356d7d5b05fSDeng-Cheng Zhu int kvm_mips_handle_mapped_seg_tlb_fault(struct kvm_vcpu *vcpu, 357d7d5b05fSDeng-Cheng Zhu struct kvm_mips_tlb *tlb, 358d7d5b05fSDeng-Cheng Zhu unsigned long *hpa0, 359d7d5b05fSDeng-Cheng Zhu unsigned long *hpa1) 360d7d5b05fSDeng-Cheng Zhu { 361d7d5b05fSDeng-Cheng Zhu unsigned long entryhi = 0, entrylo0 = 0, entrylo1 = 0; 362d7d5b05fSDeng-Cheng Zhu struct kvm *kvm = vcpu->kvm; 363d7d5b05fSDeng-Cheng Zhu pfn_t pfn0, pfn1; 364d7d5b05fSDeng-Cheng Zhu 365d7d5b05fSDeng-Cheng Zhu if ((tlb->tlb_hi & VPN2_MASK) == 0) { 366d7d5b05fSDeng-Cheng Zhu pfn0 = 0; 367d7d5b05fSDeng-Cheng Zhu pfn1 = 0; 368d7d5b05fSDeng-Cheng Zhu } else { 369d7d5b05fSDeng-Cheng Zhu if (kvm_mips_map_page(kvm, mips3_tlbpfn_to_paddr(tlb->tlb_lo0) 370d7d5b05fSDeng-Cheng Zhu >> PAGE_SHIFT) < 0) 371d7d5b05fSDeng-Cheng Zhu return -1; 372d7d5b05fSDeng-Cheng Zhu 373d7d5b05fSDeng-Cheng Zhu if (kvm_mips_map_page(kvm, mips3_tlbpfn_to_paddr(tlb->tlb_lo1) 374d7d5b05fSDeng-Cheng Zhu >> PAGE_SHIFT) < 0) 375d7d5b05fSDeng-Cheng Zhu return -1; 376d7d5b05fSDeng-Cheng Zhu 377d7d5b05fSDeng-Cheng Zhu pfn0 = kvm->arch.guest_pmap[mips3_tlbpfn_to_paddr(tlb->tlb_lo0) 378d7d5b05fSDeng-Cheng Zhu >> PAGE_SHIFT]; 379d7d5b05fSDeng-Cheng Zhu pfn1 = kvm->arch.guest_pmap[mips3_tlbpfn_to_paddr(tlb->tlb_lo1) 380d7d5b05fSDeng-Cheng Zhu >> PAGE_SHIFT]; 381d7d5b05fSDeng-Cheng Zhu } 382d7d5b05fSDeng-Cheng Zhu 383d7d5b05fSDeng-Cheng Zhu if (hpa0) 384d7d5b05fSDeng-Cheng Zhu *hpa0 = pfn0 << PAGE_SHIFT; 385d7d5b05fSDeng-Cheng Zhu 386d7d5b05fSDeng-Cheng Zhu if (hpa1) 387d7d5b05fSDeng-Cheng Zhu *hpa1 = pfn1 << PAGE_SHIFT; 388d7d5b05fSDeng-Cheng Zhu 389d7d5b05fSDeng-Cheng Zhu /* Get attributes from the Guest TLB */ 390d7d5b05fSDeng-Cheng Zhu entryhi = (tlb->tlb_hi & VPN2_MASK) | (KVM_GUEST_KERNEL_MODE(vcpu) ? 391d7d5b05fSDeng-Cheng Zhu kvm_mips_get_kernel_asid(vcpu) : 392d7d5b05fSDeng-Cheng Zhu kvm_mips_get_user_asid(vcpu)); 393d7d5b05fSDeng-Cheng Zhu entrylo0 = mips3_paddr_to_tlbpfn(pfn0 << PAGE_SHIFT) | (0x3 << 3) | 394d7d5b05fSDeng-Cheng Zhu (tlb->tlb_lo0 & MIPS3_PG_D) | (tlb->tlb_lo0 & MIPS3_PG_V); 395d7d5b05fSDeng-Cheng Zhu entrylo1 = mips3_paddr_to_tlbpfn(pfn1 << PAGE_SHIFT) | (0x3 << 3) | 396d7d5b05fSDeng-Cheng Zhu (tlb->tlb_lo1 & MIPS3_PG_D) | (tlb->tlb_lo1 & MIPS3_PG_V); 397d7d5b05fSDeng-Cheng Zhu 398d7d5b05fSDeng-Cheng Zhu kvm_debug("@ %#lx tlb_lo0: 0x%08lx tlb_lo1: 0x%08lx\n", vcpu->arch.pc, 399d7d5b05fSDeng-Cheng Zhu tlb->tlb_lo0, tlb->tlb_lo1); 400d7d5b05fSDeng-Cheng Zhu 401d7d5b05fSDeng-Cheng Zhu return kvm_mips_host_tlb_write(vcpu, entryhi, entrylo0, entrylo1, 402d7d5b05fSDeng-Cheng Zhu tlb->tlb_mask); 403d7d5b05fSDeng-Cheng Zhu } 404d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_mips_handle_mapped_seg_tlb_fault); 405d7d5b05fSDeng-Cheng Zhu 406d7d5b05fSDeng-Cheng Zhu int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi) 407d7d5b05fSDeng-Cheng Zhu { 408d7d5b05fSDeng-Cheng Zhu int i; 409d7d5b05fSDeng-Cheng Zhu int index = -1; 410d7d5b05fSDeng-Cheng Zhu struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb; 411d7d5b05fSDeng-Cheng Zhu 412d7d5b05fSDeng-Cheng Zhu for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) { 413d7d5b05fSDeng-Cheng Zhu if (TLB_HI_VPN2_HIT(tlb[i], entryhi) && 414d7d5b05fSDeng-Cheng Zhu TLB_HI_ASID_HIT(tlb[i], entryhi)) { 415d7d5b05fSDeng-Cheng Zhu index = i; 416d7d5b05fSDeng-Cheng Zhu break; 417d7d5b05fSDeng-Cheng Zhu } 418d7d5b05fSDeng-Cheng Zhu } 419d7d5b05fSDeng-Cheng Zhu 420d7d5b05fSDeng-Cheng Zhu kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n", 421d7d5b05fSDeng-Cheng Zhu __func__, entryhi, index, tlb[i].tlb_lo0, tlb[i].tlb_lo1); 422d7d5b05fSDeng-Cheng Zhu 423d7d5b05fSDeng-Cheng Zhu return index; 424d7d5b05fSDeng-Cheng Zhu } 425d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_mips_guest_tlb_lookup); 426d7d5b05fSDeng-Cheng Zhu 427d7d5b05fSDeng-Cheng Zhu int kvm_mips_host_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long vaddr) 428d7d5b05fSDeng-Cheng Zhu { 429d7d5b05fSDeng-Cheng Zhu unsigned long old_entryhi, flags; 430d7d5b05fSDeng-Cheng Zhu int idx; 431d7d5b05fSDeng-Cheng Zhu 432d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 433d7d5b05fSDeng-Cheng Zhu 434d7d5b05fSDeng-Cheng Zhu old_entryhi = read_c0_entryhi(); 435d7d5b05fSDeng-Cheng Zhu 436d7d5b05fSDeng-Cheng Zhu if (KVM_GUEST_KERNEL_MODE(vcpu)) 437d7d5b05fSDeng-Cheng Zhu write_c0_entryhi((vaddr & VPN2_MASK) | 438d7d5b05fSDeng-Cheng Zhu kvm_mips_get_kernel_asid(vcpu)); 439d7d5b05fSDeng-Cheng Zhu else { 440d7d5b05fSDeng-Cheng Zhu write_c0_entryhi((vaddr & VPN2_MASK) | 441d7d5b05fSDeng-Cheng Zhu kvm_mips_get_user_asid(vcpu)); 442d7d5b05fSDeng-Cheng Zhu } 443d7d5b05fSDeng-Cheng Zhu 444d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 445d7d5b05fSDeng-Cheng Zhu 446d7d5b05fSDeng-Cheng Zhu tlb_probe(); 447d7d5b05fSDeng-Cheng Zhu tlb_probe_hazard(); 448d7d5b05fSDeng-Cheng Zhu idx = read_c0_index(); 449d7d5b05fSDeng-Cheng Zhu 450d7d5b05fSDeng-Cheng Zhu /* Restore old ASID */ 451d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_entryhi); 452d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 453d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 454d7d5b05fSDeng-Cheng Zhu 455d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 456d7d5b05fSDeng-Cheng Zhu 457d7d5b05fSDeng-Cheng Zhu kvm_debug("Host TLB lookup, %#lx, idx: %2d\n", vaddr, idx); 458d7d5b05fSDeng-Cheng Zhu 459d7d5b05fSDeng-Cheng Zhu return idx; 460d7d5b05fSDeng-Cheng Zhu } 461d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_mips_host_tlb_lookup); 462d7d5b05fSDeng-Cheng Zhu 463d7d5b05fSDeng-Cheng Zhu int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va) 464d7d5b05fSDeng-Cheng Zhu { 465d7d5b05fSDeng-Cheng Zhu int idx; 466d7d5b05fSDeng-Cheng Zhu unsigned long flags, old_entryhi; 467d7d5b05fSDeng-Cheng Zhu 468d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 469d7d5b05fSDeng-Cheng Zhu 470d7d5b05fSDeng-Cheng Zhu old_entryhi = read_c0_entryhi(); 471d7d5b05fSDeng-Cheng Zhu 472d7d5b05fSDeng-Cheng Zhu write_c0_entryhi((va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu)); 473d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 474d7d5b05fSDeng-Cheng Zhu 475d7d5b05fSDeng-Cheng Zhu tlb_probe(); 476d7d5b05fSDeng-Cheng Zhu tlb_probe_hazard(); 477d7d5b05fSDeng-Cheng Zhu idx = read_c0_index(); 478d7d5b05fSDeng-Cheng Zhu 479d7d5b05fSDeng-Cheng Zhu if (idx >= current_cpu_data.tlbsize) 480d7d5b05fSDeng-Cheng Zhu BUG(); 481d7d5b05fSDeng-Cheng Zhu 482d7d5b05fSDeng-Cheng Zhu if (idx > 0) { 483d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 484d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 485d7d5b05fSDeng-Cheng Zhu 486d7d5b05fSDeng-Cheng Zhu write_c0_entrylo0(0); 487d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 488d7d5b05fSDeng-Cheng Zhu 489d7d5b05fSDeng-Cheng Zhu write_c0_entrylo1(0); 490d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 491d7d5b05fSDeng-Cheng Zhu 492d7d5b05fSDeng-Cheng Zhu tlb_write_indexed(); 493d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 494d7d5b05fSDeng-Cheng Zhu } 495d7d5b05fSDeng-Cheng Zhu 496d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_entryhi); 497d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 498d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 499d7d5b05fSDeng-Cheng Zhu 500d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 501d7d5b05fSDeng-Cheng Zhu 502d7d5b05fSDeng-Cheng Zhu if (idx > 0) 503d7d5b05fSDeng-Cheng Zhu kvm_debug("%s: Invalidated entryhi %#lx @ idx %d\n", __func__, 504d7d5b05fSDeng-Cheng Zhu (va & VPN2_MASK) | kvm_mips_get_user_asid(vcpu), idx); 505d7d5b05fSDeng-Cheng Zhu 506d7d5b05fSDeng-Cheng Zhu return 0; 507d7d5b05fSDeng-Cheng Zhu } 508d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_mips_host_tlb_inv); 509d7d5b05fSDeng-Cheng Zhu 510d7d5b05fSDeng-Cheng Zhu /* XXXKYMA: Fix Guest USER/KERNEL no longer share the same ASID */ 511d7d5b05fSDeng-Cheng Zhu int kvm_mips_host_tlb_inv_index(struct kvm_vcpu *vcpu, int index) 512d7d5b05fSDeng-Cheng Zhu { 513d7d5b05fSDeng-Cheng Zhu unsigned long flags, old_entryhi; 514d7d5b05fSDeng-Cheng Zhu 515d7d5b05fSDeng-Cheng Zhu if (index >= current_cpu_data.tlbsize) 516d7d5b05fSDeng-Cheng Zhu BUG(); 517d7d5b05fSDeng-Cheng Zhu 518d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 519d7d5b05fSDeng-Cheng Zhu 520d7d5b05fSDeng-Cheng Zhu old_entryhi = read_c0_entryhi(); 521d7d5b05fSDeng-Cheng Zhu 522d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(UNIQUE_ENTRYHI(index)); 523d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 524d7d5b05fSDeng-Cheng Zhu 525d7d5b05fSDeng-Cheng Zhu write_c0_index(index); 526d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 527d7d5b05fSDeng-Cheng Zhu 528d7d5b05fSDeng-Cheng Zhu write_c0_entrylo0(0); 529d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 530d7d5b05fSDeng-Cheng Zhu 531d7d5b05fSDeng-Cheng Zhu write_c0_entrylo1(0); 532d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 533d7d5b05fSDeng-Cheng Zhu 534d7d5b05fSDeng-Cheng Zhu tlb_write_indexed(); 535d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 536d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 537d7d5b05fSDeng-Cheng Zhu 538d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_entryhi); 539d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 540d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 541d7d5b05fSDeng-Cheng Zhu 542d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 543d7d5b05fSDeng-Cheng Zhu 544d7d5b05fSDeng-Cheng Zhu return 0; 545d7d5b05fSDeng-Cheng Zhu } 546d7d5b05fSDeng-Cheng Zhu 547d7d5b05fSDeng-Cheng Zhu void kvm_mips_flush_host_tlb(int skip_kseg0) 548d7d5b05fSDeng-Cheng Zhu { 549d7d5b05fSDeng-Cheng Zhu unsigned long flags; 550d7d5b05fSDeng-Cheng Zhu unsigned long old_entryhi, entryhi; 551d7d5b05fSDeng-Cheng Zhu unsigned long old_pagemask; 552d7d5b05fSDeng-Cheng Zhu int entry = 0; 553d7d5b05fSDeng-Cheng Zhu int maxentry = current_cpu_data.tlbsize; 554d7d5b05fSDeng-Cheng Zhu 555d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 556d7d5b05fSDeng-Cheng Zhu 557d7d5b05fSDeng-Cheng Zhu old_entryhi = read_c0_entryhi(); 558d7d5b05fSDeng-Cheng Zhu old_pagemask = read_c0_pagemask(); 559d7d5b05fSDeng-Cheng Zhu 560d7d5b05fSDeng-Cheng Zhu /* Blast 'em all away. */ 561d7d5b05fSDeng-Cheng Zhu for (entry = 0; entry < maxentry; entry++) { 562d7d5b05fSDeng-Cheng Zhu write_c0_index(entry); 563d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 564d7d5b05fSDeng-Cheng Zhu 565d7d5b05fSDeng-Cheng Zhu if (skip_kseg0) { 566d7d5b05fSDeng-Cheng Zhu tlb_read(); 567d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 568d7d5b05fSDeng-Cheng Zhu 569d7d5b05fSDeng-Cheng Zhu entryhi = read_c0_entryhi(); 570d7d5b05fSDeng-Cheng Zhu 571d7d5b05fSDeng-Cheng Zhu /* Don't blow away guest kernel entries */ 572d7d5b05fSDeng-Cheng Zhu if (KVM_GUEST_KSEGX(entryhi) == KVM_GUEST_KSEG0) 573d7d5b05fSDeng-Cheng Zhu continue; 574d7d5b05fSDeng-Cheng Zhu } 575d7d5b05fSDeng-Cheng Zhu 576d7d5b05fSDeng-Cheng Zhu /* Make sure all entries differ. */ 577d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(UNIQUE_ENTRYHI(entry)); 578d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 579d7d5b05fSDeng-Cheng Zhu write_c0_entrylo0(0); 580d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 581d7d5b05fSDeng-Cheng Zhu write_c0_entrylo1(0); 582d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 583d7d5b05fSDeng-Cheng Zhu 584d7d5b05fSDeng-Cheng Zhu tlb_write_indexed(); 585d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 586d7d5b05fSDeng-Cheng Zhu } 587d7d5b05fSDeng-Cheng Zhu 588d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 589d7d5b05fSDeng-Cheng Zhu 590d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_entryhi); 591d7d5b05fSDeng-Cheng Zhu write_c0_pagemask(old_pagemask); 592d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 593d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 594d7d5b05fSDeng-Cheng Zhu 595d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 596d7d5b05fSDeng-Cheng Zhu } 597d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_mips_flush_host_tlb); 598d7d5b05fSDeng-Cheng Zhu 599d7d5b05fSDeng-Cheng Zhu void kvm_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu, 600d7d5b05fSDeng-Cheng Zhu struct kvm_vcpu *vcpu) 601d7d5b05fSDeng-Cheng Zhu { 602d7d5b05fSDeng-Cheng Zhu unsigned long asid = asid_cache(cpu); 603d7d5b05fSDeng-Cheng Zhu 604d7d5b05fSDeng-Cheng Zhu asid += ASID_INC; 605d7d5b05fSDeng-Cheng Zhu if (!(asid & ASID_MASK)) { 606d7d5b05fSDeng-Cheng Zhu if (cpu_has_vtag_icache) 607d7d5b05fSDeng-Cheng Zhu flush_icache_all(); 608d7d5b05fSDeng-Cheng Zhu 609d7d5b05fSDeng-Cheng Zhu kvm_local_flush_tlb_all(); /* start new asid cycle */ 610d7d5b05fSDeng-Cheng Zhu 611d7d5b05fSDeng-Cheng Zhu if (!asid) /* fix version if needed */ 612d7d5b05fSDeng-Cheng Zhu asid = ASID_FIRST_VERSION; 613d7d5b05fSDeng-Cheng Zhu } 614d7d5b05fSDeng-Cheng Zhu 615d7d5b05fSDeng-Cheng Zhu cpu_context(cpu, mm) = asid_cache(cpu) = asid; 616d7d5b05fSDeng-Cheng Zhu } 617d7d5b05fSDeng-Cheng Zhu 618d7d5b05fSDeng-Cheng Zhu void kvm_local_flush_tlb_all(void) 619d7d5b05fSDeng-Cheng Zhu { 620d7d5b05fSDeng-Cheng Zhu unsigned long flags; 621d7d5b05fSDeng-Cheng Zhu unsigned long old_ctx; 622d7d5b05fSDeng-Cheng Zhu int entry = 0; 623d7d5b05fSDeng-Cheng Zhu 624d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 625d7d5b05fSDeng-Cheng Zhu /* Save old context and create impossible VPN2 value */ 626d7d5b05fSDeng-Cheng Zhu old_ctx = read_c0_entryhi(); 627d7d5b05fSDeng-Cheng Zhu write_c0_entrylo0(0); 628d7d5b05fSDeng-Cheng Zhu write_c0_entrylo1(0); 629d7d5b05fSDeng-Cheng Zhu 630d7d5b05fSDeng-Cheng Zhu /* Blast 'em all away. */ 631d7d5b05fSDeng-Cheng Zhu while (entry < current_cpu_data.tlbsize) { 632d7d5b05fSDeng-Cheng Zhu /* Make sure all entries differ. */ 633d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(UNIQUE_ENTRYHI(entry)); 634d7d5b05fSDeng-Cheng Zhu write_c0_index(entry); 635d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 636d7d5b05fSDeng-Cheng Zhu tlb_write_indexed(); 637d7d5b05fSDeng-Cheng Zhu entry++; 638d7d5b05fSDeng-Cheng Zhu } 639d7d5b05fSDeng-Cheng Zhu tlbw_use_hazard(); 640d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_ctx); 641d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 642d7d5b05fSDeng-Cheng Zhu 643d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 644d7d5b05fSDeng-Cheng Zhu } 645d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_local_flush_tlb_all); 646d7d5b05fSDeng-Cheng Zhu 647d7d5b05fSDeng-Cheng Zhu /** 648d7d5b05fSDeng-Cheng Zhu * kvm_mips_migrate_count() - Migrate timer. 649d7d5b05fSDeng-Cheng Zhu * @vcpu: Virtual CPU. 650d7d5b05fSDeng-Cheng Zhu * 651d7d5b05fSDeng-Cheng Zhu * Migrate CP0_Count hrtimer to the current CPU by cancelling and restarting it 652d7d5b05fSDeng-Cheng Zhu * if it was running prior to being cancelled. 653d7d5b05fSDeng-Cheng Zhu * 654d7d5b05fSDeng-Cheng Zhu * Must be called when the VCPU is migrated to a different CPU to ensure that 655d7d5b05fSDeng-Cheng Zhu * timer expiry during guest execution interrupts the guest and causes the 656d7d5b05fSDeng-Cheng Zhu * interrupt to be delivered in a timely manner. 657d7d5b05fSDeng-Cheng Zhu */ 658d7d5b05fSDeng-Cheng Zhu static void kvm_mips_migrate_count(struct kvm_vcpu *vcpu) 659d7d5b05fSDeng-Cheng Zhu { 660d7d5b05fSDeng-Cheng Zhu if (hrtimer_cancel(&vcpu->arch.comparecount_timer)) 661d7d5b05fSDeng-Cheng Zhu hrtimer_restart(&vcpu->arch.comparecount_timer); 662d7d5b05fSDeng-Cheng Zhu } 663d7d5b05fSDeng-Cheng Zhu 664d7d5b05fSDeng-Cheng Zhu /* Restore ASID once we are scheduled back after preemption */ 665d7d5b05fSDeng-Cheng Zhu void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 666d7d5b05fSDeng-Cheng Zhu { 667d7d5b05fSDeng-Cheng Zhu unsigned long flags; 668d7d5b05fSDeng-Cheng Zhu int newasid = 0; 669d7d5b05fSDeng-Cheng Zhu 670d7d5b05fSDeng-Cheng Zhu kvm_debug("%s: vcpu %p, cpu: %d\n", __func__, vcpu, cpu); 671d7d5b05fSDeng-Cheng Zhu 672d7d5b05fSDeng-Cheng Zhu /* Alocate new kernel and user ASIDs if needed */ 673d7d5b05fSDeng-Cheng Zhu 674d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 675d7d5b05fSDeng-Cheng Zhu 676d7d5b05fSDeng-Cheng Zhu if (((vcpu->arch. 677d7d5b05fSDeng-Cheng Zhu guest_kernel_asid[cpu] ^ asid_cache(cpu)) & ASID_VERSION_MASK)) { 678d7d5b05fSDeng-Cheng Zhu kvm_get_new_mmu_context(&vcpu->arch.guest_kernel_mm, cpu, vcpu); 679d7d5b05fSDeng-Cheng Zhu vcpu->arch.guest_kernel_asid[cpu] = 680d7d5b05fSDeng-Cheng Zhu vcpu->arch.guest_kernel_mm.context.asid[cpu]; 681d7d5b05fSDeng-Cheng Zhu kvm_get_new_mmu_context(&vcpu->arch.guest_user_mm, cpu, vcpu); 682d7d5b05fSDeng-Cheng Zhu vcpu->arch.guest_user_asid[cpu] = 683d7d5b05fSDeng-Cheng Zhu vcpu->arch.guest_user_mm.context.asid[cpu]; 684d7d5b05fSDeng-Cheng Zhu newasid++; 685d7d5b05fSDeng-Cheng Zhu 686d7d5b05fSDeng-Cheng Zhu kvm_debug("[%d]: cpu_context: %#lx\n", cpu, 687d7d5b05fSDeng-Cheng Zhu cpu_context(cpu, current->mm)); 688d7d5b05fSDeng-Cheng Zhu kvm_debug("[%d]: Allocated new ASID for Guest Kernel: %#x\n", 689d7d5b05fSDeng-Cheng Zhu cpu, vcpu->arch.guest_kernel_asid[cpu]); 690d7d5b05fSDeng-Cheng Zhu kvm_debug("[%d]: Allocated new ASID for Guest User: %#x\n", cpu, 691d7d5b05fSDeng-Cheng Zhu vcpu->arch.guest_user_asid[cpu]); 692d7d5b05fSDeng-Cheng Zhu } 693d7d5b05fSDeng-Cheng Zhu 694d7d5b05fSDeng-Cheng Zhu if (vcpu->arch.last_sched_cpu != cpu) { 695d7d5b05fSDeng-Cheng Zhu kvm_debug("[%d->%d]KVM VCPU[%d] switch\n", 696d7d5b05fSDeng-Cheng Zhu vcpu->arch.last_sched_cpu, cpu, vcpu->vcpu_id); 697d7d5b05fSDeng-Cheng Zhu /* 698d7d5b05fSDeng-Cheng Zhu * Migrate the timer interrupt to the current CPU so that it 699d7d5b05fSDeng-Cheng Zhu * always interrupts the guest and synchronously triggers a 700d7d5b05fSDeng-Cheng Zhu * guest timer interrupt. 701d7d5b05fSDeng-Cheng Zhu */ 702d7d5b05fSDeng-Cheng Zhu kvm_mips_migrate_count(vcpu); 703d7d5b05fSDeng-Cheng Zhu } 704d7d5b05fSDeng-Cheng Zhu 705d7d5b05fSDeng-Cheng Zhu if (!newasid) { 706d7d5b05fSDeng-Cheng Zhu /* 707d7d5b05fSDeng-Cheng Zhu * If we preempted while the guest was executing, then reload 708d7d5b05fSDeng-Cheng Zhu * the pre-empted ASID 709d7d5b05fSDeng-Cheng Zhu */ 710d7d5b05fSDeng-Cheng Zhu if (current->flags & PF_VCPU) { 711d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(vcpu->arch. 712d7d5b05fSDeng-Cheng Zhu preempt_entryhi & ASID_MASK); 713d7d5b05fSDeng-Cheng Zhu ehb(); 714d7d5b05fSDeng-Cheng Zhu } 715d7d5b05fSDeng-Cheng Zhu } else { 716d7d5b05fSDeng-Cheng Zhu /* New ASIDs were allocated for the VM */ 717d7d5b05fSDeng-Cheng Zhu 718d7d5b05fSDeng-Cheng Zhu /* 719d7d5b05fSDeng-Cheng Zhu * Were we in guest context? If so then the pre-empted ASID is 720d7d5b05fSDeng-Cheng Zhu * no longer valid, we need to set it to what it should be based 721d7d5b05fSDeng-Cheng Zhu * on the mode of the Guest (Kernel/User) 722d7d5b05fSDeng-Cheng Zhu */ 723d7d5b05fSDeng-Cheng Zhu if (current->flags & PF_VCPU) { 724d7d5b05fSDeng-Cheng Zhu if (KVM_GUEST_KERNEL_MODE(vcpu)) 725d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(vcpu->arch. 726d7d5b05fSDeng-Cheng Zhu guest_kernel_asid[cpu] & 727d7d5b05fSDeng-Cheng Zhu ASID_MASK); 728d7d5b05fSDeng-Cheng Zhu else 729d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(vcpu->arch. 730d7d5b05fSDeng-Cheng Zhu guest_user_asid[cpu] & 731d7d5b05fSDeng-Cheng Zhu ASID_MASK); 732d7d5b05fSDeng-Cheng Zhu ehb(); 733d7d5b05fSDeng-Cheng Zhu } 734d7d5b05fSDeng-Cheng Zhu } 735d7d5b05fSDeng-Cheng Zhu 736b86ecb37SJames Hogan /* restore guest state to registers */ 737b86ecb37SJames Hogan kvm_mips_callbacks->vcpu_set_regs(vcpu); 738b86ecb37SJames Hogan 739d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 740d7d5b05fSDeng-Cheng Zhu 741d7d5b05fSDeng-Cheng Zhu } 742d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_arch_vcpu_load); 743d7d5b05fSDeng-Cheng Zhu 744d7d5b05fSDeng-Cheng Zhu /* ASID can change if another task is scheduled during preemption */ 745d7d5b05fSDeng-Cheng Zhu void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 746d7d5b05fSDeng-Cheng Zhu { 747d7d5b05fSDeng-Cheng Zhu unsigned long flags; 748d7d5b05fSDeng-Cheng Zhu uint32_t cpu; 749d7d5b05fSDeng-Cheng Zhu 750d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 751d7d5b05fSDeng-Cheng Zhu 752d7d5b05fSDeng-Cheng Zhu cpu = smp_processor_id(); 753d7d5b05fSDeng-Cheng Zhu 754d7d5b05fSDeng-Cheng Zhu vcpu->arch.preempt_entryhi = read_c0_entryhi(); 755d7d5b05fSDeng-Cheng Zhu vcpu->arch.last_sched_cpu = cpu; 756d7d5b05fSDeng-Cheng Zhu 757b86ecb37SJames Hogan /* save guest state in registers */ 758b86ecb37SJames Hogan kvm_mips_callbacks->vcpu_get_regs(vcpu); 759b86ecb37SJames Hogan 760d7d5b05fSDeng-Cheng Zhu if (((cpu_context(cpu, current->mm) ^ asid_cache(cpu)) & 761d7d5b05fSDeng-Cheng Zhu ASID_VERSION_MASK)) { 762d7d5b05fSDeng-Cheng Zhu kvm_debug("%s: Dropping MMU Context: %#lx\n", __func__, 763d7d5b05fSDeng-Cheng Zhu cpu_context(cpu, current->mm)); 764d7d5b05fSDeng-Cheng Zhu drop_mmu_context(current->mm, cpu); 765d7d5b05fSDeng-Cheng Zhu } 766d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(cpu_asid(cpu, current->mm)); 767d7d5b05fSDeng-Cheng Zhu ehb(); 768d7d5b05fSDeng-Cheng Zhu 769d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 770d7d5b05fSDeng-Cheng Zhu } 771d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_arch_vcpu_put); 772d7d5b05fSDeng-Cheng Zhu 773d7d5b05fSDeng-Cheng Zhu uint32_t kvm_get_inst(uint32_t *opc, struct kvm_vcpu *vcpu) 774d7d5b05fSDeng-Cheng Zhu { 775d7d5b05fSDeng-Cheng Zhu struct mips_coproc *cop0 = vcpu->arch.cop0; 776d7d5b05fSDeng-Cheng Zhu unsigned long paddr, flags, vpn2, asid; 777d7d5b05fSDeng-Cheng Zhu uint32_t inst; 778d7d5b05fSDeng-Cheng Zhu int index; 779d7d5b05fSDeng-Cheng Zhu 780d7d5b05fSDeng-Cheng Zhu if (KVM_GUEST_KSEGX((unsigned long) opc) < KVM_GUEST_KSEG0 || 781d7d5b05fSDeng-Cheng Zhu KVM_GUEST_KSEGX((unsigned long) opc) == KVM_GUEST_KSEG23) { 782d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 783d7d5b05fSDeng-Cheng Zhu index = kvm_mips_host_tlb_lookup(vcpu, (unsigned long) opc); 784d7d5b05fSDeng-Cheng Zhu if (index >= 0) { 785d7d5b05fSDeng-Cheng Zhu inst = *(opc); 786d7d5b05fSDeng-Cheng Zhu } else { 787d7d5b05fSDeng-Cheng Zhu vpn2 = (unsigned long) opc & VPN2_MASK; 788d7d5b05fSDeng-Cheng Zhu asid = kvm_read_c0_guest_entryhi(cop0) & ASID_MASK; 789d7d5b05fSDeng-Cheng Zhu index = kvm_mips_guest_tlb_lookup(vcpu, vpn2 | asid); 790d7d5b05fSDeng-Cheng Zhu if (index < 0) { 791d7d5b05fSDeng-Cheng Zhu kvm_err("%s: get_user_failed for %p, vcpu: %p, ASID: %#lx\n", 792d7d5b05fSDeng-Cheng Zhu __func__, opc, vcpu, read_c0_entryhi()); 793d7d5b05fSDeng-Cheng Zhu kvm_mips_dump_host_tlbs(); 794d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 795d7d5b05fSDeng-Cheng Zhu return KVM_INVALID_INST; 796d7d5b05fSDeng-Cheng Zhu } 797d7d5b05fSDeng-Cheng Zhu kvm_mips_handle_mapped_seg_tlb_fault(vcpu, 798d7d5b05fSDeng-Cheng Zhu &vcpu->arch. 799d7d5b05fSDeng-Cheng Zhu guest_tlb[index], 800d7d5b05fSDeng-Cheng Zhu NULL, NULL); 801d7d5b05fSDeng-Cheng Zhu inst = *(opc); 802d7d5b05fSDeng-Cheng Zhu } 803d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 804d7d5b05fSDeng-Cheng Zhu } else if (KVM_GUEST_KSEGX(opc) == KVM_GUEST_KSEG0) { 805d7d5b05fSDeng-Cheng Zhu paddr = 806d7d5b05fSDeng-Cheng Zhu kvm_mips_translate_guest_kseg0_to_hpa(vcpu, 807d7d5b05fSDeng-Cheng Zhu (unsigned long) opc); 808d7d5b05fSDeng-Cheng Zhu inst = *(uint32_t *) CKSEG0ADDR(paddr); 809d7d5b05fSDeng-Cheng Zhu } else { 810d7d5b05fSDeng-Cheng Zhu kvm_err("%s: illegal address: %p\n", __func__, opc); 811d7d5b05fSDeng-Cheng Zhu return KVM_INVALID_INST; 812d7d5b05fSDeng-Cheng Zhu } 813d7d5b05fSDeng-Cheng Zhu 814d7d5b05fSDeng-Cheng Zhu return inst; 815d7d5b05fSDeng-Cheng Zhu } 816d7d5b05fSDeng-Cheng Zhu EXPORT_SYMBOL(kvm_get_inst); 817