1 /* 2 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved. 3 * 4 * Authors: 5 * Alexander Graf <agraf@suse.de> 6 * Kevin Wolf <mail@kevin-wolf.de> 7 * Paul Mackerras <paulus@samba.org> 8 * 9 * Description: 10 * Functions relating to running KVM on Book 3S processors where 11 * we don't have access to hypervisor mode, and we run the guest 12 * in problem state (user mode). 13 * 14 * This file is derived from arch/powerpc/kvm/44x.c, 15 * by Hollis Blanchard <hollisb@us.ibm.com>. 16 * 17 * This program is free software; you can redistribute it and/or modify 18 * it under the terms of the GNU General Public License, version 2, as 19 * published by the Free Software Foundation. 20 */ 21 22 #include <linux/kvm_host.h> 23 #include <linux/export.h> 24 #include <linux/err.h> 25 #include <linux/slab.h> 26 27 #include <asm/reg.h> 28 #include <asm/cputable.h> 29 #include <asm/cacheflush.h> 30 #include <asm/tlbflush.h> 31 #include <asm/uaccess.h> 32 #include <asm/io.h> 33 #include <asm/kvm_ppc.h> 34 #include <asm/kvm_book3s.h> 35 #include <asm/mmu_context.h> 36 #include <asm/switch_to.h> 37 #include <asm/firmware.h> 38 #include <asm/hvcall.h> 39 #include <linux/gfp.h> 40 #include <linux/sched.h> 41 #include <linux/vmalloc.h> 42 #include <linux/highmem.h> 43 #include <linux/module.h> 44 45 #include "book3s.h" 46 47 #define CREATE_TRACE_POINTS 48 #include "trace_pr.h" 49 50 /* #define EXIT_DEBUG */ 51 /* #define DEBUG_EXT */ 52 53 static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr, 54 ulong msr); 55 56 /* Some compatibility defines */ 57 #ifdef CONFIG_PPC_BOOK3S_32 58 #define MSR_USER32 MSR_USER 59 #define MSR_USER64 MSR_USER 60 #define HW_PAGE_SIZE PAGE_SIZE 61 #endif 62 63 static void kvmppc_core_vcpu_load_pr(struct kvm_vcpu *vcpu, int cpu) 64 { 65 #ifdef CONFIG_PPC_BOOK3S_64 66 struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); 67 memcpy(svcpu->slb, to_book3s(vcpu)->slb_shadow, sizeof(svcpu->slb)); 68 svcpu->slb_max = to_book3s(vcpu)->slb_shadow_max; 69 svcpu_put(svcpu); 70 #endif 71 vcpu->cpu = smp_processor_id(); 72 #ifdef CONFIG_PPC_BOOK3S_32 73 current->thread.kvm_shadow_vcpu = vcpu->arch.shadow_vcpu; 74 #endif 75 } 76 77 static void kvmppc_core_vcpu_put_pr(struct kvm_vcpu *vcpu) 78 { 79 #ifdef CONFIG_PPC_BOOK3S_64 80 struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); 81 memcpy(to_book3s(vcpu)->slb_shadow, svcpu->slb, sizeof(svcpu->slb)); 82 to_book3s(vcpu)->slb_shadow_max = svcpu->slb_max; 83 svcpu_put(svcpu); 84 #endif 85 86 kvmppc_giveup_ext(vcpu, MSR_FP | MSR_VEC | MSR_VSX); 87 vcpu->cpu = -1; 88 } 89 90 /* Copy data needed by real-mode code from vcpu to shadow vcpu */ 91 void kvmppc_copy_to_svcpu(struct kvmppc_book3s_shadow_vcpu *svcpu, 92 struct kvm_vcpu *vcpu) 93 { 94 svcpu->gpr[0] = vcpu->arch.gpr[0]; 95 svcpu->gpr[1] = vcpu->arch.gpr[1]; 96 svcpu->gpr[2] = vcpu->arch.gpr[2]; 97 svcpu->gpr[3] = vcpu->arch.gpr[3]; 98 svcpu->gpr[4] = vcpu->arch.gpr[4]; 99 svcpu->gpr[5] = vcpu->arch.gpr[5]; 100 svcpu->gpr[6] = vcpu->arch.gpr[6]; 101 svcpu->gpr[7] = vcpu->arch.gpr[7]; 102 svcpu->gpr[8] = vcpu->arch.gpr[8]; 103 svcpu->gpr[9] = vcpu->arch.gpr[9]; 104 svcpu->gpr[10] = vcpu->arch.gpr[10]; 105 svcpu->gpr[11] = vcpu->arch.gpr[11]; 106 svcpu->gpr[12] = vcpu->arch.gpr[12]; 107 svcpu->gpr[13] = vcpu->arch.gpr[13]; 108 svcpu->cr = vcpu->arch.cr; 109 svcpu->xer = vcpu->arch.xer; 110 svcpu->ctr = vcpu->arch.ctr; 111 svcpu->lr = vcpu->arch.lr; 112 svcpu->pc = vcpu->arch.pc; 113 } 114 115 /* Copy data touched by real-mode code from shadow vcpu back to vcpu */ 116 void kvmppc_copy_from_svcpu(struct kvm_vcpu *vcpu, 117 struct kvmppc_book3s_shadow_vcpu *svcpu) 118 { 119 vcpu->arch.gpr[0] = svcpu->gpr[0]; 120 vcpu->arch.gpr[1] = svcpu->gpr[1]; 121 vcpu->arch.gpr[2] = svcpu->gpr[2]; 122 vcpu->arch.gpr[3] = svcpu->gpr[3]; 123 vcpu->arch.gpr[4] = svcpu->gpr[4]; 124 vcpu->arch.gpr[5] = svcpu->gpr[5]; 125 vcpu->arch.gpr[6] = svcpu->gpr[6]; 126 vcpu->arch.gpr[7] = svcpu->gpr[7]; 127 vcpu->arch.gpr[8] = svcpu->gpr[8]; 128 vcpu->arch.gpr[9] = svcpu->gpr[9]; 129 vcpu->arch.gpr[10] = svcpu->gpr[10]; 130 vcpu->arch.gpr[11] = svcpu->gpr[11]; 131 vcpu->arch.gpr[12] = svcpu->gpr[12]; 132 vcpu->arch.gpr[13] = svcpu->gpr[13]; 133 vcpu->arch.cr = svcpu->cr; 134 vcpu->arch.xer = svcpu->xer; 135 vcpu->arch.ctr = svcpu->ctr; 136 vcpu->arch.lr = svcpu->lr; 137 vcpu->arch.pc = svcpu->pc; 138 vcpu->arch.shadow_srr1 = svcpu->shadow_srr1; 139 vcpu->arch.fault_dar = svcpu->fault_dar; 140 vcpu->arch.fault_dsisr = svcpu->fault_dsisr; 141 vcpu->arch.last_inst = svcpu->last_inst; 142 } 143 144 static int kvmppc_core_check_requests_pr(struct kvm_vcpu *vcpu) 145 { 146 int r = 1; /* Indicate we want to get back into the guest */ 147 148 /* We misuse TLB_FLUSH to indicate that we want to clear 149 all shadow cache entries */ 150 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) 151 kvmppc_mmu_pte_flush(vcpu, 0, 0); 152 153 return r; 154 } 155 156 /************* MMU Notifiers *************/ 157 static void do_kvm_unmap_hva(struct kvm *kvm, unsigned long start, 158 unsigned long end) 159 { 160 long i; 161 struct kvm_vcpu *vcpu; 162 struct kvm_memslots *slots; 163 struct kvm_memory_slot *memslot; 164 165 slots = kvm_memslots(kvm); 166 kvm_for_each_memslot(memslot, slots) { 167 unsigned long hva_start, hva_end; 168 gfn_t gfn, gfn_end; 169 170 hva_start = max(start, memslot->userspace_addr); 171 hva_end = min(end, memslot->userspace_addr + 172 (memslot->npages << PAGE_SHIFT)); 173 if (hva_start >= hva_end) 174 continue; 175 /* 176 * {gfn(page) | page intersects with [hva_start, hva_end)} = 177 * {gfn, gfn+1, ..., gfn_end-1}. 178 */ 179 gfn = hva_to_gfn_memslot(hva_start, memslot); 180 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot); 181 kvm_for_each_vcpu(i, vcpu, kvm) 182 kvmppc_mmu_pte_pflush(vcpu, gfn << PAGE_SHIFT, 183 gfn_end << PAGE_SHIFT); 184 } 185 } 186 187 static int kvm_unmap_hva_pr(struct kvm *kvm, unsigned long hva) 188 { 189 trace_kvm_unmap_hva(hva); 190 191 do_kvm_unmap_hva(kvm, hva, hva + PAGE_SIZE); 192 193 return 0; 194 } 195 196 static int kvm_unmap_hva_range_pr(struct kvm *kvm, unsigned long start, 197 unsigned long end) 198 { 199 do_kvm_unmap_hva(kvm, start, end); 200 201 return 0; 202 } 203 204 static int kvm_age_hva_pr(struct kvm *kvm, unsigned long hva) 205 { 206 /* XXX could be more clever ;) */ 207 return 0; 208 } 209 210 static int kvm_test_age_hva_pr(struct kvm *kvm, unsigned long hva) 211 { 212 /* XXX could be more clever ;) */ 213 return 0; 214 } 215 216 static void kvm_set_spte_hva_pr(struct kvm *kvm, unsigned long hva, pte_t pte) 217 { 218 /* The page will get remapped properly on its next fault */ 219 do_kvm_unmap_hva(kvm, hva, hva + PAGE_SIZE); 220 } 221 222 /*****************************************/ 223 224 static void kvmppc_recalc_shadow_msr(struct kvm_vcpu *vcpu) 225 { 226 ulong smsr = vcpu->arch.shared->msr; 227 228 /* Guest MSR values */ 229 smsr &= MSR_FE0 | MSR_FE1 | MSR_SF | MSR_SE | MSR_BE; 230 /* Process MSR values */ 231 smsr |= MSR_ME | MSR_RI | MSR_IR | MSR_DR | MSR_PR | MSR_EE; 232 /* External providers the guest reserved */ 233 smsr |= (vcpu->arch.shared->msr & vcpu->arch.guest_owned_ext); 234 /* 64-bit Process MSR values */ 235 #ifdef CONFIG_PPC_BOOK3S_64 236 smsr |= MSR_ISF | MSR_HV; 237 #endif 238 vcpu->arch.shadow_msr = smsr; 239 } 240 241 static void kvmppc_set_msr_pr(struct kvm_vcpu *vcpu, u64 msr) 242 { 243 ulong old_msr = vcpu->arch.shared->msr; 244 245 #ifdef EXIT_DEBUG 246 printk(KERN_INFO "KVM: Set MSR to 0x%llx\n", msr); 247 #endif 248 249 msr &= to_book3s(vcpu)->msr_mask; 250 vcpu->arch.shared->msr = msr; 251 kvmppc_recalc_shadow_msr(vcpu); 252 253 if (msr & MSR_POW) { 254 if (!vcpu->arch.pending_exceptions) { 255 kvm_vcpu_block(vcpu); 256 clear_bit(KVM_REQ_UNHALT, &vcpu->requests); 257 vcpu->stat.halt_wakeup++; 258 259 /* Unset POW bit after we woke up */ 260 msr &= ~MSR_POW; 261 vcpu->arch.shared->msr = msr; 262 } 263 } 264 265 if ((vcpu->arch.shared->msr & (MSR_PR|MSR_IR|MSR_DR)) != 266 (old_msr & (MSR_PR|MSR_IR|MSR_DR))) { 267 kvmppc_mmu_flush_segments(vcpu); 268 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)); 269 270 /* Preload magic page segment when in kernel mode */ 271 if (!(msr & MSR_PR) && vcpu->arch.magic_page_pa) { 272 struct kvm_vcpu_arch *a = &vcpu->arch; 273 274 if (msr & MSR_DR) 275 kvmppc_mmu_map_segment(vcpu, a->magic_page_ea); 276 else 277 kvmppc_mmu_map_segment(vcpu, a->magic_page_pa); 278 } 279 } 280 281 /* 282 * When switching from 32 to 64-bit, we may have a stale 32-bit 283 * magic page around, we need to flush it. Typically 32-bit magic 284 * page will be instanciated when calling into RTAS. Note: We 285 * assume that such transition only happens while in kernel mode, 286 * ie, we never transition from user 32-bit to kernel 64-bit with 287 * a 32-bit magic page around. 288 */ 289 if (vcpu->arch.magic_page_pa && 290 !(old_msr & MSR_PR) && !(old_msr & MSR_SF) && (msr & MSR_SF)) { 291 /* going from RTAS to normal kernel code */ 292 kvmppc_mmu_pte_flush(vcpu, (uint32_t)vcpu->arch.magic_page_pa, 293 ~0xFFFUL); 294 } 295 296 /* Preload FPU if it's enabled */ 297 if (vcpu->arch.shared->msr & MSR_FP) 298 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP); 299 } 300 301 void kvmppc_set_pvr_pr(struct kvm_vcpu *vcpu, u32 pvr) 302 { 303 u32 host_pvr; 304 305 vcpu->arch.hflags &= ~BOOK3S_HFLAG_SLB; 306 vcpu->arch.pvr = pvr; 307 #ifdef CONFIG_PPC_BOOK3S_64 308 if ((pvr >= 0x330000) && (pvr < 0x70330000)) { 309 kvmppc_mmu_book3s_64_init(vcpu); 310 if (!to_book3s(vcpu)->hior_explicit) 311 to_book3s(vcpu)->hior = 0xfff00000; 312 to_book3s(vcpu)->msr_mask = 0xffffffffffffffffULL; 313 vcpu->arch.cpu_type = KVM_CPU_3S_64; 314 } else 315 #endif 316 { 317 kvmppc_mmu_book3s_32_init(vcpu); 318 if (!to_book3s(vcpu)->hior_explicit) 319 to_book3s(vcpu)->hior = 0; 320 to_book3s(vcpu)->msr_mask = 0xffffffffULL; 321 vcpu->arch.cpu_type = KVM_CPU_3S_32; 322 } 323 324 kvmppc_sanity_check(vcpu); 325 326 /* If we are in hypervisor level on 970, we can tell the CPU to 327 * treat DCBZ as 32 bytes store */ 328 vcpu->arch.hflags &= ~BOOK3S_HFLAG_DCBZ32; 329 if (vcpu->arch.mmu.is_dcbz32(vcpu) && (mfmsr() & MSR_HV) && 330 !strcmp(cur_cpu_spec->platform, "ppc970")) 331 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32; 332 333 /* Cell performs badly if MSR_FEx are set. So let's hope nobody 334 really needs them in a VM on Cell and force disable them. */ 335 if (!strcmp(cur_cpu_spec->platform, "ppc-cell-be")) 336 to_book3s(vcpu)->msr_mask &= ~(MSR_FE0 | MSR_FE1); 337 338 /* 339 * If they're asking for POWER6 or later, set the flag 340 * indicating that we can do multiple large page sizes 341 * and 1TB segments. 342 * Also set the flag that indicates that tlbie has the large 343 * page bit in the RB operand instead of the instruction. 344 */ 345 switch (PVR_VER(pvr)) { 346 case PVR_POWER6: 347 case PVR_POWER7: 348 case PVR_POWER7p: 349 case PVR_POWER8: 350 vcpu->arch.hflags |= BOOK3S_HFLAG_MULTI_PGSIZE | 351 BOOK3S_HFLAG_NEW_TLBIE; 352 break; 353 } 354 355 #ifdef CONFIG_PPC_BOOK3S_32 356 /* 32 bit Book3S always has 32 byte dcbz */ 357 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32; 358 #endif 359 360 /* On some CPUs we can execute paired single operations natively */ 361 asm ( "mfpvr %0" : "=r"(host_pvr)); 362 switch (host_pvr) { 363 case 0x00080200: /* lonestar 2.0 */ 364 case 0x00088202: /* lonestar 2.2 */ 365 case 0x70000100: /* gekko 1.0 */ 366 case 0x00080100: /* gekko 2.0 */ 367 case 0x00083203: /* gekko 2.3a */ 368 case 0x00083213: /* gekko 2.3b */ 369 case 0x00083204: /* gekko 2.4 */ 370 case 0x00083214: /* gekko 2.4e (8SE) - retail HW2 */ 371 case 0x00087200: /* broadway */ 372 vcpu->arch.hflags |= BOOK3S_HFLAG_NATIVE_PS; 373 /* Enable HID2.PSE - in case we need it later */ 374 mtspr(SPRN_HID2_GEKKO, mfspr(SPRN_HID2_GEKKO) | (1 << 29)); 375 } 376 } 377 378 /* Book3s_32 CPUs always have 32 bytes cache line size, which Linux assumes. To 379 * make Book3s_32 Linux work on Book3s_64, we have to make sure we trap dcbz to 380 * emulate 32 bytes dcbz length. 381 * 382 * The Book3s_64 inventors also realized this case and implemented a special bit 383 * in the HID5 register, which is a hypervisor ressource. Thus we can't use it. 384 * 385 * My approach here is to patch the dcbz instruction on executing pages. 386 */ 387 static void kvmppc_patch_dcbz(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte) 388 { 389 struct page *hpage; 390 u64 hpage_offset; 391 u32 *page; 392 int i; 393 394 hpage = gfn_to_page(vcpu->kvm, pte->raddr >> PAGE_SHIFT); 395 if (is_error_page(hpage)) 396 return; 397 398 hpage_offset = pte->raddr & ~PAGE_MASK; 399 hpage_offset &= ~0xFFFULL; 400 hpage_offset /= 4; 401 402 get_page(hpage); 403 page = kmap_atomic(hpage); 404 405 /* patch dcbz into reserved instruction, so we trap */ 406 for (i=hpage_offset; i < hpage_offset + (HW_PAGE_SIZE / 4); i++) 407 if ((page[i] & 0xff0007ff) == INS_DCBZ) 408 page[i] &= 0xfffffff7; 409 410 kunmap_atomic(page); 411 put_page(hpage); 412 } 413 414 static int kvmppc_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 415 { 416 ulong mp_pa = vcpu->arch.magic_page_pa; 417 418 if (!(vcpu->arch.shared->msr & MSR_SF)) 419 mp_pa = (uint32_t)mp_pa; 420 421 if (unlikely(mp_pa) && 422 unlikely((mp_pa & KVM_PAM) >> PAGE_SHIFT == gfn)) { 423 return 1; 424 } 425 426 return kvm_is_visible_gfn(vcpu->kvm, gfn); 427 } 428 429 int kvmppc_handle_pagefault(struct kvm_run *run, struct kvm_vcpu *vcpu, 430 ulong eaddr, int vec) 431 { 432 bool data = (vec == BOOK3S_INTERRUPT_DATA_STORAGE); 433 bool iswrite = false; 434 int r = RESUME_GUEST; 435 int relocated; 436 int page_found = 0; 437 struct kvmppc_pte pte; 438 bool is_mmio = false; 439 bool dr = (vcpu->arch.shared->msr & MSR_DR) ? true : false; 440 bool ir = (vcpu->arch.shared->msr & MSR_IR) ? true : false; 441 u64 vsid; 442 443 relocated = data ? dr : ir; 444 if (data && (vcpu->arch.fault_dsisr & DSISR_ISSTORE)) 445 iswrite = true; 446 447 /* Resolve real address if translation turned on */ 448 if (relocated) { 449 page_found = vcpu->arch.mmu.xlate(vcpu, eaddr, &pte, data, iswrite); 450 } else { 451 pte.may_execute = true; 452 pte.may_read = true; 453 pte.may_write = true; 454 pte.raddr = eaddr & KVM_PAM; 455 pte.eaddr = eaddr; 456 pte.vpage = eaddr >> 12; 457 pte.page_size = MMU_PAGE_64K; 458 } 459 460 switch (vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) { 461 case 0: 462 pte.vpage |= ((u64)VSID_REAL << (SID_SHIFT - 12)); 463 break; 464 case MSR_DR: 465 case MSR_IR: 466 vcpu->arch.mmu.esid_to_vsid(vcpu, eaddr >> SID_SHIFT, &vsid); 467 468 if ((vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) == MSR_DR) 469 pte.vpage |= ((u64)VSID_REAL_DR << (SID_SHIFT - 12)); 470 else 471 pte.vpage |= ((u64)VSID_REAL_IR << (SID_SHIFT - 12)); 472 pte.vpage |= vsid; 473 474 if (vsid == -1) 475 page_found = -EINVAL; 476 break; 477 } 478 479 if (vcpu->arch.mmu.is_dcbz32(vcpu) && 480 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) { 481 /* 482 * If we do the dcbz hack, we have to NX on every execution, 483 * so we can patch the executing code. This renders our guest 484 * NX-less. 485 */ 486 pte.may_execute = !data; 487 } 488 489 if (page_found == -ENOENT) { 490 /* Page not found in guest PTE entries */ 491 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu); 492 vcpu->arch.shared->dsisr = vcpu->arch.fault_dsisr; 493 vcpu->arch.shared->msr |= 494 vcpu->arch.shadow_srr1 & 0x00000000f8000000ULL; 495 kvmppc_book3s_queue_irqprio(vcpu, vec); 496 } else if (page_found == -EPERM) { 497 /* Storage protection */ 498 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu); 499 vcpu->arch.shared->dsisr = vcpu->arch.fault_dsisr & ~DSISR_NOHPTE; 500 vcpu->arch.shared->dsisr |= DSISR_PROTFAULT; 501 vcpu->arch.shared->msr |= 502 vcpu->arch.shadow_srr1 & 0x00000000f8000000ULL; 503 kvmppc_book3s_queue_irqprio(vcpu, vec); 504 } else if (page_found == -EINVAL) { 505 /* Page not found in guest SLB */ 506 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu); 507 kvmppc_book3s_queue_irqprio(vcpu, vec + 0x80); 508 } else if (!is_mmio && 509 kvmppc_visible_gfn(vcpu, pte.raddr >> PAGE_SHIFT)) { 510 if (data && !(vcpu->arch.fault_dsisr & DSISR_NOHPTE)) { 511 /* 512 * There is already a host HPTE there, presumably 513 * a read-only one for a page the guest thinks 514 * is writable, so get rid of it first. 515 */ 516 kvmppc_mmu_unmap_page(vcpu, &pte); 517 } 518 /* The guest's PTE is not mapped yet. Map on the host */ 519 kvmppc_mmu_map_page(vcpu, &pte, iswrite); 520 if (data) 521 vcpu->stat.sp_storage++; 522 else if (vcpu->arch.mmu.is_dcbz32(vcpu) && 523 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) 524 kvmppc_patch_dcbz(vcpu, &pte); 525 } else { 526 /* MMIO */ 527 vcpu->stat.mmio_exits++; 528 vcpu->arch.paddr_accessed = pte.raddr; 529 vcpu->arch.vaddr_accessed = pte.eaddr; 530 r = kvmppc_emulate_mmio(run, vcpu); 531 if ( r == RESUME_HOST_NV ) 532 r = RESUME_HOST; 533 } 534 535 return r; 536 } 537 538 static inline int get_fpr_index(int i) 539 { 540 return i * TS_FPRWIDTH; 541 } 542 543 /* Give up external provider (FPU, Altivec, VSX) */ 544 void kvmppc_giveup_ext(struct kvm_vcpu *vcpu, ulong msr) 545 { 546 struct thread_struct *t = ¤t->thread; 547 u64 *vcpu_fpr = vcpu->arch.fpr; 548 #ifdef CONFIG_VSX 549 u64 *vcpu_vsx = vcpu->arch.vsr; 550 #endif 551 u64 *thread_fpr = &t->fp_state.fpr[0][0]; 552 int i; 553 554 /* 555 * VSX instructions can access FP and vector registers, so if 556 * we are giving up VSX, make sure we give up FP and VMX as well. 557 */ 558 if (msr & MSR_VSX) 559 msr |= MSR_FP | MSR_VEC; 560 561 msr &= vcpu->arch.guest_owned_ext; 562 if (!msr) 563 return; 564 565 #ifdef DEBUG_EXT 566 printk(KERN_INFO "Giving up ext 0x%lx\n", msr); 567 #endif 568 569 if (msr & MSR_FP) { 570 /* 571 * Note that on CPUs with VSX, giveup_fpu stores 572 * both the traditional FP registers and the added VSX 573 * registers into thread.fp_state.fpr[]. 574 */ 575 if (current->thread.regs->msr & MSR_FP) 576 giveup_fpu(current); 577 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++) 578 vcpu_fpr[i] = thread_fpr[get_fpr_index(i)]; 579 580 vcpu->arch.fpscr = t->fp_state.fpscr; 581 582 #ifdef CONFIG_VSX 583 if (cpu_has_feature(CPU_FTR_VSX)) 584 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr) / 2; i++) 585 vcpu_vsx[i] = thread_fpr[get_fpr_index(i) + 1]; 586 #endif 587 } 588 589 #ifdef CONFIG_ALTIVEC 590 if (msr & MSR_VEC) { 591 if (current->thread.regs->msr & MSR_VEC) 592 giveup_altivec(current); 593 memcpy(vcpu->arch.vr, t->vr_state.vr, sizeof(vcpu->arch.vr)); 594 vcpu->arch.vscr = t->vr_state.vscr; 595 } 596 #endif 597 598 vcpu->arch.guest_owned_ext &= ~(msr | MSR_VSX); 599 kvmppc_recalc_shadow_msr(vcpu); 600 } 601 602 static int kvmppc_read_inst(struct kvm_vcpu *vcpu) 603 { 604 ulong srr0 = kvmppc_get_pc(vcpu); 605 u32 last_inst = kvmppc_get_last_inst(vcpu); 606 int ret; 607 608 ret = kvmppc_ld(vcpu, &srr0, sizeof(u32), &last_inst, false); 609 if (ret == -ENOENT) { 610 ulong msr = vcpu->arch.shared->msr; 611 612 msr = kvmppc_set_field(msr, 33, 33, 1); 613 msr = kvmppc_set_field(msr, 34, 36, 0); 614 vcpu->arch.shared->msr = kvmppc_set_field(msr, 42, 47, 0); 615 kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_INST_STORAGE); 616 return EMULATE_AGAIN; 617 } 618 619 return EMULATE_DONE; 620 } 621 622 static int kvmppc_check_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr) 623 { 624 625 /* Need to do paired single emulation? */ 626 if (!(vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE)) 627 return EMULATE_DONE; 628 629 /* Read out the instruction */ 630 if (kvmppc_read_inst(vcpu) == EMULATE_DONE) 631 /* Need to emulate */ 632 return EMULATE_FAIL; 633 634 return EMULATE_AGAIN; 635 } 636 637 /* Handle external providers (FPU, Altivec, VSX) */ 638 static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr, 639 ulong msr) 640 { 641 struct thread_struct *t = ¤t->thread; 642 u64 *vcpu_fpr = vcpu->arch.fpr; 643 #ifdef CONFIG_VSX 644 u64 *vcpu_vsx = vcpu->arch.vsr; 645 #endif 646 u64 *thread_fpr = &t->fp_state.fpr[0][0]; 647 int i; 648 649 /* When we have paired singles, we emulate in software */ 650 if (vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE) 651 return RESUME_GUEST; 652 653 if (!(vcpu->arch.shared->msr & msr)) { 654 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 655 return RESUME_GUEST; 656 } 657 658 if (msr == MSR_VSX) { 659 /* No VSX? Give an illegal instruction interrupt */ 660 #ifdef CONFIG_VSX 661 if (!cpu_has_feature(CPU_FTR_VSX)) 662 #endif 663 { 664 kvmppc_core_queue_program(vcpu, SRR1_PROGILL); 665 return RESUME_GUEST; 666 } 667 668 /* 669 * We have to load up all the FP and VMX registers before 670 * we can let the guest use VSX instructions. 671 */ 672 msr = MSR_FP | MSR_VEC | MSR_VSX; 673 } 674 675 /* See if we already own all the ext(s) needed */ 676 msr &= ~vcpu->arch.guest_owned_ext; 677 if (!msr) 678 return RESUME_GUEST; 679 680 #ifdef DEBUG_EXT 681 printk(KERN_INFO "Loading up ext 0x%lx\n", msr); 682 #endif 683 684 if (msr & MSR_FP) { 685 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++) 686 thread_fpr[get_fpr_index(i)] = vcpu_fpr[i]; 687 #ifdef CONFIG_VSX 688 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr) / 2; i++) 689 thread_fpr[get_fpr_index(i) + 1] = vcpu_vsx[i]; 690 #endif 691 t->fp_state.fpscr = vcpu->arch.fpscr; 692 t->fpexc_mode = 0; 693 kvmppc_load_up_fpu(); 694 } 695 696 if (msr & MSR_VEC) { 697 #ifdef CONFIG_ALTIVEC 698 memcpy(t->vr_state.vr, vcpu->arch.vr, sizeof(vcpu->arch.vr)); 699 t->vr_state.vscr = vcpu->arch.vscr; 700 t->vrsave = -1; 701 kvmppc_load_up_altivec(); 702 #endif 703 } 704 705 current->thread.regs->msr |= msr; 706 vcpu->arch.guest_owned_ext |= msr; 707 kvmppc_recalc_shadow_msr(vcpu); 708 709 return RESUME_GUEST; 710 } 711 712 /* 713 * Kernel code using FP or VMX could have flushed guest state to 714 * the thread_struct; if so, get it back now. 715 */ 716 static void kvmppc_handle_lost_ext(struct kvm_vcpu *vcpu) 717 { 718 unsigned long lost_ext; 719 720 lost_ext = vcpu->arch.guest_owned_ext & ~current->thread.regs->msr; 721 if (!lost_ext) 722 return; 723 724 if (lost_ext & MSR_FP) 725 kvmppc_load_up_fpu(); 726 #ifdef CONFIG_ALTIVEC 727 if (lost_ext & MSR_VEC) 728 kvmppc_load_up_altivec(); 729 #endif 730 current->thread.regs->msr |= lost_ext; 731 } 732 733 int kvmppc_handle_exit_pr(struct kvm_run *run, struct kvm_vcpu *vcpu, 734 unsigned int exit_nr) 735 { 736 int r = RESUME_HOST; 737 int s; 738 739 vcpu->stat.sum_exits++; 740 741 run->exit_reason = KVM_EXIT_UNKNOWN; 742 run->ready_for_interrupt_injection = 1; 743 744 /* We get here with MSR.EE=1 */ 745 746 trace_kvm_exit(exit_nr, vcpu); 747 kvm_guest_exit(); 748 749 switch (exit_nr) { 750 case BOOK3S_INTERRUPT_INST_STORAGE: 751 { 752 ulong shadow_srr1 = vcpu->arch.shadow_srr1; 753 vcpu->stat.pf_instruc++; 754 755 #ifdef CONFIG_PPC_BOOK3S_32 756 /* We set segments as unused segments when invalidating them. So 757 * treat the respective fault as segment fault. */ 758 { 759 struct kvmppc_book3s_shadow_vcpu *svcpu; 760 u32 sr; 761 762 svcpu = svcpu_get(vcpu); 763 sr = svcpu->sr[kvmppc_get_pc(vcpu) >> SID_SHIFT]; 764 svcpu_put(svcpu); 765 if (sr == SR_INVALID) { 766 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)); 767 r = RESUME_GUEST; 768 break; 769 } 770 } 771 #endif 772 773 /* only care about PTEG not found errors, but leave NX alone */ 774 if (shadow_srr1 & 0x40000000) { 775 int idx = srcu_read_lock(&vcpu->kvm->srcu); 776 r = kvmppc_handle_pagefault(run, vcpu, kvmppc_get_pc(vcpu), exit_nr); 777 srcu_read_unlock(&vcpu->kvm->srcu, idx); 778 vcpu->stat.sp_instruc++; 779 } else if (vcpu->arch.mmu.is_dcbz32(vcpu) && 780 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) { 781 /* 782 * XXX If we do the dcbz hack we use the NX bit to flush&patch the page, 783 * so we can't use the NX bit inside the guest. Let's cross our fingers, 784 * that no guest that needs the dcbz hack does NX. 785 */ 786 kvmppc_mmu_pte_flush(vcpu, kvmppc_get_pc(vcpu), ~0xFFFUL); 787 r = RESUME_GUEST; 788 } else { 789 vcpu->arch.shared->msr |= shadow_srr1 & 0x58000000; 790 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 791 r = RESUME_GUEST; 792 } 793 break; 794 } 795 case BOOK3S_INTERRUPT_DATA_STORAGE: 796 { 797 ulong dar = kvmppc_get_fault_dar(vcpu); 798 u32 fault_dsisr = vcpu->arch.fault_dsisr; 799 vcpu->stat.pf_storage++; 800 801 #ifdef CONFIG_PPC_BOOK3S_32 802 /* We set segments as unused segments when invalidating them. So 803 * treat the respective fault as segment fault. */ 804 { 805 struct kvmppc_book3s_shadow_vcpu *svcpu; 806 u32 sr; 807 808 svcpu = svcpu_get(vcpu); 809 sr = svcpu->sr[dar >> SID_SHIFT]; 810 svcpu_put(svcpu); 811 if (sr == SR_INVALID) { 812 kvmppc_mmu_map_segment(vcpu, dar); 813 r = RESUME_GUEST; 814 break; 815 } 816 } 817 #endif 818 819 /* 820 * We need to handle missing shadow PTEs, and 821 * protection faults due to us mapping a page read-only 822 * when the guest thinks it is writable. 823 */ 824 if (fault_dsisr & (DSISR_NOHPTE | DSISR_PROTFAULT)) { 825 int idx = srcu_read_lock(&vcpu->kvm->srcu); 826 r = kvmppc_handle_pagefault(run, vcpu, dar, exit_nr); 827 srcu_read_unlock(&vcpu->kvm->srcu, idx); 828 } else { 829 vcpu->arch.shared->dar = dar; 830 vcpu->arch.shared->dsisr = fault_dsisr; 831 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 832 r = RESUME_GUEST; 833 } 834 break; 835 } 836 case BOOK3S_INTERRUPT_DATA_SEGMENT: 837 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_fault_dar(vcpu)) < 0) { 838 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu); 839 kvmppc_book3s_queue_irqprio(vcpu, 840 BOOK3S_INTERRUPT_DATA_SEGMENT); 841 } 842 r = RESUME_GUEST; 843 break; 844 case BOOK3S_INTERRUPT_INST_SEGMENT: 845 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)) < 0) { 846 kvmppc_book3s_queue_irqprio(vcpu, 847 BOOK3S_INTERRUPT_INST_SEGMENT); 848 } 849 r = RESUME_GUEST; 850 break; 851 /* We're good on these - the host merely wanted to get our attention */ 852 case BOOK3S_INTERRUPT_DECREMENTER: 853 case BOOK3S_INTERRUPT_HV_DECREMENTER: 854 vcpu->stat.dec_exits++; 855 r = RESUME_GUEST; 856 break; 857 case BOOK3S_INTERRUPT_EXTERNAL: 858 case BOOK3S_INTERRUPT_EXTERNAL_LEVEL: 859 case BOOK3S_INTERRUPT_EXTERNAL_HV: 860 vcpu->stat.ext_intr_exits++; 861 r = RESUME_GUEST; 862 break; 863 case BOOK3S_INTERRUPT_PERFMON: 864 r = RESUME_GUEST; 865 break; 866 case BOOK3S_INTERRUPT_PROGRAM: 867 case BOOK3S_INTERRUPT_H_EMUL_ASSIST: 868 { 869 enum emulation_result er; 870 ulong flags; 871 872 program_interrupt: 873 flags = vcpu->arch.shadow_srr1 & 0x1f0000ull; 874 875 if (vcpu->arch.shared->msr & MSR_PR) { 876 #ifdef EXIT_DEBUG 877 printk(KERN_INFO "Userspace triggered 0x700 exception at 0x%lx (0x%x)\n", kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu)); 878 #endif 879 if ((kvmppc_get_last_inst(vcpu) & 0xff0007ff) != 880 (INS_DCBZ & 0xfffffff7)) { 881 kvmppc_core_queue_program(vcpu, flags); 882 r = RESUME_GUEST; 883 break; 884 } 885 } 886 887 vcpu->stat.emulated_inst_exits++; 888 er = kvmppc_emulate_instruction(run, vcpu); 889 switch (er) { 890 case EMULATE_DONE: 891 r = RESUME_GUEST_NV; 892 break; 893 case EMULATE_AGAIN: 894 r = RESUME_GUEST; 895 break; 896 case EMULATE_FAIL: 897 printk(KERN_CRIT "%s: emulation at %lx failed (%08x)\n", 898 __func__, kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu)); 899 kvmppc_core_queue_program(vcpu, flags); 900 r = RESUME_GUEST; 901 break; 902 case EMULATE_DO_MMIO: 903 run->exit_reason = KVM_EXIT_MMIO; 904 r = RESUME_HOST_NV; 905 break; 906 case EMULATE_EXIT_USER: 907 r = RESUME_HOST_NV; 908 break; 909 default: 910 BUG(); 911 } 912 break; 913 } 914 case BOOK3S_INTERRUPT_SYSCALL: 915 if (vcpu->arch.papr_enabled && 916 (kvmppc_get_last_sc(vcpu) == 0x44000022) && 917 !(vcpu->arch.shared->msr & MSR_PR)) { 918 /* SC 1 papr hypercalls */ 919 ulong cmd = kvmppc_get_gpr(vcpu, 3); 920 int i; 921 922 #ifdef CONFIG_PPC_BOOK3S_64 923 if (kvmppc_h_pr(vcpu, cmd) == EMULATE_DONE) { 924 r = RESUME_GUEST; 925 break; 926 } 927 #endif 928 929 run->papr_hcall.nr = cmd; 930 for (i = 0; i < 9; ++i) { 931 ulong gpr = kvmppc_get_gpr(vcpu, 4 + i); 932 run->papr_hcall.args[i] = gpr; 933 } 934 run->exit_reason = KVM_EXIT_PAPR_HCALL; 935 vcpu->arch.hcall_needed = 1; 936 r = RESUME_HOST; 937 } else if (vcpu->arch.osi_enabled && 938 (((u32)kvmppc_get_gpr(vcpu, 3)) == OSI_SC_MAGIC_R3) && 939 (((u32)kvmppc_get_gpr(vcpu, 4)) == OSI_SC_MAGIC_R4)) { 940 /* MOL hypercalls */ 941 u64 *gprs = run->osi.gprs; 942 int i; 943 944 run->exit_reason = KVM_EXIT_OSI; 945 for (i = 0; i < 32; i++) 946 gprs[i] = kvmppc_get_gpr(vcpu, i); 947 vcpu->arch.osi_needed = 1; 948 r = RESUME_HOST_NV; 949 } else if (!(vcpu->arch.shared->msr & MSR_PR) && 950 (((u32)kvmppc_get_gpr(vcpu, 0)) == KVM_SC_MAGIC_R0)) { 951 /* KVM PV hypercalls */ 952 kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu)); 953 r = RESUME_GUEST; 954 } else { 955 /* Guest syscalls */ 956 vcpu->stat.syscall_exits++; 957 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 958 r = RESUME_GUEST; 959 } 960 break; 961 case BOOK3S_INTERRUPT_FP_UNAVAIL: 962 case BOOK3S_INTERRUPT_ALTIVEC: 963 case BOOK3S_INTERRUPT_VSX: 964 { 965 int ext_msr = 0; 966 967 switch (exit_nr) { 968 case BOOK3S_INTERRUPT_FP_UNAVAIL: ext_msr = MSR_FP; break; 969 case BOOK3S_INTERRUPT_ALTIVEC: ext_msr = MSR_VEC; break; 970 case BOOK3S_INTERRUPT_VSX: ext_msr = MSR_VSX; break; 971 } 972 973 switch (kvmppc_check_ext(vcpu, exit_nr)) { 974 case EMULATE_DONE: 975 /* everything ok - let's enable the ext */ 976 r = kvmppc_handle_ext(vcpu, exit_nr, ext_msr); 977 break; 978 case EMULATE_FAIL: 979 /* we need to emulate this instruction */ 980 goto program_interrupt; 981 break; 982 default: 983 /* nothing to worry about - go again */ 984 break; 985 } 986 break; 987 } 988 case BOOK3S_INTERRUPT_ALIGNMENT: 989 if (kvmppc_read_inst(vcpu) == EMULATE_DONE) { 990 vcpu->arch.shared->dsisr = kvmppc_alignment_dsisr(vcpu, 991 kvmppc_get_last_inst(vcpu)); 992 vcpu->arch.shared->dar = kvmppc_alignment_dar(vcpu, 993 kvmppc_get_last_inst(vcpu)); 994 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 995 } 996 r = RESUME_GUEST; 997 break; 998 case BOOK3S_INTERRUPT_MACHINE_CHECK: 999 case BOOK3S_INTERRUPT_TRACE: 1000 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 1001 r = RESUME_GUEST; 1002 break; 1003 default: 1004 { 1005 ulong shadow_srr1 = vcpu->arch.shadow_srr1; 1006 /* Ugh - bork here! What did we get? */ 1007 printk(KERN_EMERG "exit_nr=0x%x | pc=0x%lx | msr=0x%lx\n", 1008 exit_nr, kvmppc_get_pc(vcpu), shadow_srr1); 1009 r = RESUME_HOST; 1010 BUG(); 1011 break; 1012 } 1013 } 1014 1015 if (!(r & RESUME_HOST)) { 1016 /* To avoid clobbering exit_reason, only check for signals if 1017 * we aren't already exiting to userspace for some other 1018 * reason. */ 1019 1020 /* 1021 * Interrupts could be timers for the guest which we have to 1022 * inject again, so let's postpone them until we're in the guest 1023 * and if we really did time things so badly, then we just exit 1024 * again due to a host external interrupt. 1025 */ 1026 local_irq_disable(); 1027 s = kvmppc_prepare_to_enter(vcpu); 1028 if (s <= 0) { 1029 local_irq_enable(); 1030 r = s; 1031 } else { 1032 kvmppc_fix_ee_before_entry(); 1033 } 1034 kvmppc_handle_lost_ext(vcpu); 1035 } 1036 1037 trace_kvm_book3s_reenter(r, vcpu); 1038 1039 return r; 1040 } 1041 1042 static int kvm_arch_vcpu_ioctl_get_sregs_pr(struct kvm_vcpu *vcpu, 1043 struct kvm_sregs *sregs) 1044 { 1045 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu); 1046 int i; 1047 1048 sregs->pvr = vcpu->arch.pvr; 1049 1050 sregs->u.s.sdr1 = to_book3s(vcpu)->sdr1; 1051 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) { 1052 for (i = 0; i < 64; i++) { 1053 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige | i; 1054 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv; 1055 } 1056 } else { 1057 for (i = 0; i < 16; i++) 1058 sregs->u.s.ppc32.sr[i] = vcpu->arch.shared->sr[i]; 1059 1060 for (i = 0; i < 8; i++) { 1061 sregs->u.s.ppc32.ibat[i] = vcpu3s->ibat[i].raw; 1062 sregs->u.s.ppc32.dbat[i] = vcpu3s->dbat[i].raw; 1063 } 1064 } 1065 1066 return 0; 1067 } 1068 1069 static int kvm_arch_vcpu_ioctl_set_sregs_pr(struct kvm_vcpu *vcpu, 1070 struct kvm_sregs *sregs) 1071 { 1072 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu); 1073 int i; 1074 1075 kvmppc_set_pvr_pr(vcpu, sregs->pvr); 1076 1077 vcpu3s->sdr1 = sregs->u.s.sdr1; 1078 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) { 1079 for (i = 0; i < 64; i++) { 1080 vcpu->arch.mmu.slbmte(vcpu, sregs->u.s.ppc64.slb[i].slbv, 1081 sregs->u.s.ppc64.slb[i].slbe); 1082 } 1083 } else { 1084 for (i = 0; i < 16; i++) { 1085 vcpu->arch.mmu.mtsrin(vcpu, i, sregs->u.s.ppc32.sr[i]); 1086 } 1087 for (i = 0; i < 8; i++) { 1088 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), false, 1089 (u32)sregs->u.s.ppc32.ibat[i]); 1090 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), true, 1091 (u32)(sregs->u.s.ppc32.ibat[i] >> 32)); 1092 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), false, 1093 (u32)sregs->u.s.ppc32.dbat[i]); 1094 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), true, 1095 (u32)(sregs->u.s.ppc32.dbat[i] >> 32)); 1096 } 1097 } 1098 1099 /* Flush the MMU after messing with the segments */ 1100 kvmppc_mmu_pte_flush(vcpu, 0, 0); 1101 1102 return 0; 1103 } 1104 1105 static int kvmppc_get_one_reg_pr(struct kvm_vcpu *vcpu, u64 id, 1106 union kvmppc_one_reg *val) 1107 { 1108 int r = 0; 1109 1110 switch (id) { 1111 case KVM_REG_PPC_HIOR: 1112 *val = get_reg_val(id, to_book3s(vcpu)->hior); 1113 break; 1114 #ifdef CONFIG_VSX 1115 case KVM_REG_PPC_VSR0 ... KVM_REG_PPC_VSR31: { 1116 long int i = id - KVM_REG_PPC_VSR0; 1117 1118 if (!cpu_has_feature(CPU_FTR_VSX)) { 1119 r = -ENXIO; 1120 break; 1121 } 1122 val->vsxval[0] = vcpu->arch.fpr[i]; 1123 val->vsxval[1] = vcpu->arch.vsr[i]; 1124 break; 1125 } 1126 #endif /* CONFIG_VSX */ 1127 default: 1128 r = -EINVAL; 1129 break; 1130 } 1131 1132 return r; 1133 } 1134 1135 static int kvmppc_set_one_reg_pr(struct kvm_vcpu *vcpu, u64 id, 1136 union kvmppc_one_reg *val) 1137 { 1138 int r = 0; 1139 1140 switch (id) { 1141 case KVM_REG_PPC_HIOR: 1142 to_book3s(vcpu)->hior = set_reg_val(id, *val); 1143 to_book3s(vcpu)->hior_explicit = true; 1144 break; 1145 #ifdef CONFIG_VSX 1146 case KVM_REG_PPC_VSR0 ... KVM_REG_PPC_VSR31: { 1147 long int i = id - KVM_REG_PPC_VSR0; 1148 1149 if (!cpu_has_feature(CPU_FTR_VSX)) { 1150 r = -ENXIO; 1151 break; 1152 } 1153 vcpu->arch.fpr[i] = val->vsxval[0]; 1154 vcpu->arch.vsr[i] = val->vsxval[1]; 1155 break; 1156 } 1157 #endif /* CONFIG_VSX */ 1158 default: 1159 r = -EINVAL; 1160 break; 1161 } 1162 1163 return r; 1164 } 1165 1166 static struct kvm_vcpu *kvmppc_core_vcpu_create_pr(struct kvm *kvm, 1167 unsigned int id) 1168 { 1169 struct kvmppc_vcpu_book3s *vcpu_book3s; 1170 struct kvm_vcpu *vcpu; 1171 int err = -ENOMEM; 1172 unsigned long p; 1173 1174 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); 1175 if (!vcpu) 1176 goto out; 1177 1178 vcpu_book3s = vzalloc(sizeof(struct kvmppc_vcpu_book3s)); 1179 if (!vcpu_book3s) 1180 goto free_vcpu; 1181 vcpu->arch.book3s = vcpu_book3s; 1182 1183 #ifdef CONFIG_KVM_BOOK3S_32 1184 vcpu->arch.shadow_vcpu = 1185 kzalloc(sizeof(*vcpu->arch.shadow_vcpu), GFP_KERNEL); 1186 if (!vcpu->arch.shadow_vcpu) 1187 goto free_vcpu3s; 1188 #endif 1189 1190 err = kvm_vcpu_init(vcpu, kvm, id); 1191 if (err) 1192 goto free_shadow_vcpu; 1193 1194 err = -ENOMEM; 1195 p = __get_free_page(GFP_KERNEL|__GFP_ZERO); 1196 if (!p) 1197 goto uninit_vcpu; 1198 /* the real shared page fills the last 4k of our page */ 1199 vcpu->arch.shared = (void *)(p + PAGE_SIZE - 4096); 1200 1201 #ifdef CONFIG_PPC_BOOK3S_64 1202 /* 1203 * Default to the same as the host if we're on sufficiently 1204 * recent machine that we have 1TB segments; 1205 * otherwise default to PPC970FX. 1206 */ 1207 vcpu->arch.pvr = 0x3C0301; 1208 if (mmu_has_feature(MMU_FTR_1T_SEGMENT)) 1209 vcpu->arch.pvr = mfspr(SPRN_PVR); 1210 #else 1211 /* default to book3s_32 (750) */ 1212 vcpu->arch.pvr = 0x84202; 1213 #endif 1214 kvmppc_set_pvr_pr(vcpu, vcpu->arch.pvr); 1215 vcpu->arch.slb_nr = 64; 1216 1217 vcpu->arch.shadow_msr = MSR_USER64; 1218 1219 err = kvmppc_mmu_init(vcpu); 1220 if (err < 0) 1221 goto uninit_vcpu; 1222 1223 return vcpu; 1224 1225 uninit_vcpu: 1226 kvm_vcpu_uninit(vcpu); 1227 free_shadow_vcpu: 1228 #ifdef CONFIG_KVM_BOOK3S_32 1229 kfree(vcpu->arch.shadow_vcpu); 1230 free_vcpu3s: 1231 #endif 1232 vfree(vcpu_book3s); 1233 free_vcpu: 1234 kmem_cache_free(kvm_vcpu_cache, vcpu); 1235 out: 1236 return ERR_PTR(err); 1237 } 1238 1239 static void kvmppc_core_vcpu_free_pr(struct kvm_vcpu *vcpu) 1240 { 1241 struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu); 1242 1243 free_page((unsigned long)vcpu->arch.shared & PAGE_MASK); 1244 kvm_vcpu_uninit(vcpu); 1245 #ifdef CONFIG_KVM_BOOK3S_32 1246 kfree(vcpu->arch.shadow_vcpu); 1247 #endif 1248 vfree(vcpu_book3s); 1249 kmem_cache_free(kvm_vcpu_cache, vcpu); 1250 } 1251 1252 static int kvmppc_vcpu_run_pr(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) 1253 { 1254 int ret; 1255 struct thread_fp_state fp; 1256 int fpexc_mode; 1257 #ifdef CONFIG_ALTIVEC 1258 struct thread_vr_state vr; 1259 unsigned long uninitialized_var(vrsave); 1260 int used_vr; 1261 #endif 1262 #ifdef CONFIG_VSX 1263 int used_vsr; 1264 #endif 1265 ulong ext_msr; 1266 1267 /* Check if we can run the vcpu at all */ 1268 if (!vcpu->arch.sane) { 1269 kvm_run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1270 ret = -EINVAL; 1271 goto out; 1272 } 1273 1274 /* 1275 * Interrupts could be timers for the guest which we have to inject 1276 * again, so let's postpone them until we're in the guest and if we 1277 * really did time things so badly, then we just exit again due to 1278 * a host external interrupt. 1279 */ 1280 local_irq_disable(); 1281 ret = kvmppc_prepare_to_enter(vcpu); 1282 if (ret <= 0) { 1283 local_irq_enable(); 1284 goto out; 1285 } 1286 1287 /* Save FPU state in stack */ 1288 if (current->thread.regs->msr & MSR_FP) 1289 giveup_fpu(current); 1290 fp = current->thread.fp_state; 1291 fpexc_mode = current->thread.fpexc_mode; 1292 1293 #ifdef CONFIG_ALTIVEC 1294 /* Save Altivec state in stack */ 1295 used_vr = current->thread.used_vr; 1296 if (used_vr) { 1297 if (current->thread.regs->msr & MSR_VEC) 1298 giveup_altivec(current); 1299 vr = current->thread.vr_state; 1300 vrsave = current->thread.vrsave; 1301 } 1302 #endif 1303 1304 #ifdef CONFIG_VSX 1305 /* Save VSX state in stack */ 1306 used_vsr = current->thread.used_vsr; 1307 if (used_vsr && (current->thread.regs->msr & MSR_VSX)) 1308 __giveup_vsx(current); 1309 #endif 1310 1311 /* Remember the MSR with disabled extensions */ 1312 ext_msr = current->thread.regs->msr; 1313 1314 /* Preload FPU if it's enabled */ 1315 if (vcpu->arch.shared->msr & MSR_FP) 1316 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP); 1317 1318 kvmppc_fix_ee_before_entry(); 1319 1320 ret = __kvmppc_vcpu_run(kvm_run, vcpu); 1321 1322 /* No need for kvm_guest_exit. It's done in handle_exit. 1323 We also get here with interrupts enabled. */ 1324 1325 /* Make sure we save the guest FPU/Altivec/VSX state */ 1326 kvmppc_giveup_ext(vcpu, MSR_FP | MSR_VEC | MSR_VSX); 1327 1328 current->thread.regs->msr = ext_msr; 1329 1330 /* Restore FPU/VSX state from stack */ 1331 current->thread.fp_state = fp; 1332 current->thread.fpexc_mode = fpexc_mode; 1333 1334 #ifdef CONFIG_ALTIVEC 1335 /* Restore Altivec state from stack */ 1336 if (used_vr && current->thread.used_vr) { 1337 current->thread.vr_state = vr; 1338 current->thread.vrsave = vrsave; 1339 } 1340 current->thread.used_vr = used_vr; 1341 #endif 1342 1343 #ifdef CONFIG_VSX 1344 current->thread.used_vsr = used_vsr; 1345 #endif 1346 1347 out: 1348 vcpu->mode = OUTSIDE_GUEST_MODE; 1349 return ret; 1350 } 1351 1352 /* 1353 * Get (and clear) the dirty memory log for a memory slot. 1354 */ 1355 static int kvm_vm_ioctl_get_dirty_log_pr(struct kvm *kvm, 1356 struct kvm_dirty_log *log) 1357 { 1358 struct kvm_memory_slot *memslot; 1359 struct kvm_vcpu *vcpu; 1360 ulong ga, ga_end; 1361 int is_dirty = 0; 1362 int r; 1363 unsigned long n; 1364 1365 mutex_lock(&kvm->slots_lock); 1366 1367 r = kvm_get_dirty_log(kvm, log, &is_dirty); 1368 if (r) 1369 goto out; 1370 1371 /* If nothing is dirty, don't bother messing with page tables. */ 1372 if (is_dirty) { 1373 memslot = id_to_memslot(kvm->memslots, log->slot); 1374 1375 ga = memslot->base_gfn << PAGE_SHIFT; 1376 ga_end = ga + (memslot->npages << PAGE_SHIFT); 1377 1378 kvm_for_each_vcpu(n, vcpu, kvm) 1379 kvmppc_mmu_pte_pflush(vcpu, ga, ga_end); 1380 1381 n = kvm_dirty_bitmap_bytes(memslot); 1382 memset(memslot->dirty_bitmap, 0, n); 1383 } 1384 1385 r = 0; 1386 out: 1387 mutex_unlock(&kvm->slots_lock); 1388 return r; 1389 } 1390 1391 static void kvmppc_core_flush_memslot_pr(struct kvm *kvm, 1392 struct kvm_memory_slot *memslot) 1393 { 1394 return; 1395 } 1396 1397 static int kvmppc_core_prepare_memory_region_pr(struct kvm *kvm, 1398 struct kvm_memory_slot *memslot, 1399 struct kvm_userspace_memory_region *mem) 1400 { 1401 return 0; 1402 } 1403 1404 static void kvmppc_core_commit_memory_region_pr(struct kvm *kvm, 1405 struct kvm_userspace_memory_region *mem, 1406 const struct kvm_memory_slot *old) 1407 { 1408 return; 1409 } 1410 1411 static void kvmppc_core_free_memslot_pr(struct kvm_memory_slot *free, 1412 struct kvm_memory_slot *dont) 1413 { 1414 return; 1415 } 1416 1417 static int kvmppc_core_create_memslot_pr(struct kvm_memory_slot *slot, 1418 unsigned long npages) 1419 { 1420 return 0; 1421 } 1422 1423 1424 #ifdef CONFIG_PPC64 1425 static int kvm_vm_ioctl_get_smmu_info_pr(struct kvm *kvm, 1426 struct kvm_ppc_smmu_info *info) 1427 { 1428 long int i; 1429 struct kvm_vcpu *vcpu; 1430 1431 info->flags = 0; 1432 1433 /* SLB is always 64 entries */ 1434 info->slb_size = 64; 1435 1436 /* Standard 4k base page size segment */ 1437 info->sps[0].page_shift = 12; 1438 info->sps[0].slb_enc = 0; 1439 info->sps[0].enc[0].page_shift = 12; 1440 info->sps[0].enc[0].pte_enc = 0; 1441 1442 /* 1443 * 64k large page size. 1444 * We only want to put this in if the CPUs we're emulating 1445 * support it, but unfortunately we don't have a vcpu easily 1446 * to hand here to test. Just pick the first vcpu, and if 1447 * that doesn't exist yet, report the minimum capability, 1448 * i.e., no 64k pages. 1449 * 1T segment support goes along with 64k pages. 1450 */ 1451 i = 1; 1452 vcpu = kvm_get_vcpu(kvm, 0); 1453 if (vcpu && (vcpu->arch.hflags & BOOK3S_HFLAG_MULTI_PGSIZE)) { 1454 info->flags = KVM_PPC_1T_SEGMENTS; 1455 info->sps[i].page_shift = 16; 1456 info->sps[i].slb_enc = SLB_VSID_L | SLB_VSID_LP_01; 1457 info->sps[i].enc[0].page_shift = 16; 1458 info->sps[i].enc[0].pte_enc = 1; 1459 ++i; 1460 } 1461 1462 /* Standard 16M large page size segment */ 1463 info->sps[i].page_shift = 24; 1464 info->sps[i].slb_enc = SLB_VSID_L; 1465 info->sps[i].enc[0].page_shift = 24; 1466 info->sps[i].enc[0].pte_enc = 0; 1467 1468 return 0; 1469 } 1470 #else 1471 static int kvm_vm_ioctl_get_smmu_info_pr(struct kvm *kvm, 1472 struct kvm_ppc_smmu_info *info) 1473 { 1474 /* We should not get called */ 1475 BUG(); 1476 } 1477 #endif /* CONFIG_PPC64 */ 1478 1479 static unsigned int kvm_global_user_count = 0; 1480 static DEFINE_SPINLOCK(kvm_global_user_count_lock); 1481 1482 static int kvmppc_core_init_vm_pr(struct kvm *kvm) 1483 { 1484 mutex_init(&kvm->arch.hpt_mutex); 1485 1486 if (firmware_has_feature(FW_FEATURE_SET_MODE)) { 1487 spin_lock(&kvm_global_user_count_lock); 1488 if (++kvm_global_user_count == 1) 1489 pSeries_disable_reloc_on_exc(); 1490 spin_unlock(&kvm_global_user_count_lock); 1491 } 1492 return 0; 1493 } 1494 1495 static void kvmppc_core_destroy_vm_pr(struct kvm *kvm) 1496 { 1497 #ifdef CONFIG_PPC64 1498 WARN_ON(!list_empty(&kvm->arch.spapr_tce_tables)); 1499 #endif 1500 1501 if (firmware_has_feature(FW_FEATURE_SET_MODE)) { 1502 spin_lock(&kvm_global_user_count_lock); 1503 BUG_ON(kvm_global_user_count == 0); 1504 if (--kvm_global_user_count == 0) 1505 pSeries_enable_reloc_on_exc(); 1506 spin_unlock(&kvm_global_user_count_lock); 1507 } 1508 } 1509 1510 static int kvmppc_core_check_processor_compat_pr(void) 1511 { 1512 /* we are always compatible */ 1513 return 0; 1514 } 1515 1516 static long kvm_arch_vm_ioctl_pr(struct file *filp, 1517 unsigned int ioctl, unsigned long arg) 1518 { 1519 return -ENOTTY; 1520 } 1521 1522 static struct kvmppc_ops kvm_ops_pr = { 1523 .get_sregs = kvm_arch_vcpu_ioctl_get_sregs_pr, 1524 .set_sregs = kvm_arch_vcpu_ioctl_set_sregs_pr, 1525 .get_one_reg = kvmppc_get_one_reg_pr, 1526 .set_one_reg = kvmppc_set_one_reg_pr, 1527 .vcpu_load = kvmppc_core_vcpu_load_pr, 1528 .vcpu_put = kvmppc_core_vcpu_put_pr, 1529 .set_msr = kvmppc_set_msr_pr, 1530 .vcpu_run = kvmppc_vcpu_run_pr, 1531 .vcpu_create = kvmppc_core_vcpu_create_pr, 1532 .vcpu_free = kvmppc_core_vcpu_free_pr, 1533 .check_requests = kvmppc_core_check_requests_pr, 1534 .get_dirty_log = kvm_vm_ioctl_get_dirty_log_pr, 1535 .flush_memslot = kvmppc_core_flush_memslot_pr, 1536 .prepare_memory_region = kvmppc_core_prepare_memory_region_pr, 1537 .commit_memory_region = kvmppc_core_commit_memory_region_pr, 1538 .unmap_hva = kvm_unmap_hva_pr, 1539 .unmap_hva_range = kvm_unmap_hva_range_pr, 1540 .age_hva = kvm_age_hva_pr, 1541 .test_age_hva = kvm_test_age_hva_pr, 1542 .set_spte_hva = kvm_set_spte_hva_pr, 1543 .mmu_destroy = kvmppc_mmu_destroy_pr, 1544 .free_memslot = kvmppc_core_free_memslot_pr, 1545 .create_memslot = kvmppc_core_create_memslot_pr, 1546 .init_vm = kvmppc_core_init_vm_pr, 1547 .destroy_vm = kvmppc_core_destroy_vm_pr, 1548 .get_smmu_info = kvm_vm_ioctl_get_smmu_info_pr, 1549 .emulate_op = kvmppc_core_emulate_op_pr, 1550 .emulate_mtspr = kvmppc_core_emulate_mtspr_pr, 1551 .emulate_mfspr = kvmppc_core_emulate_mfspr_pr, 1552 .fast_vcpu_kick = kvm_vcpu_kick, 1553 .arch_vm_ioctl = kvm_arch_vm_ioctl_pr, 1554 }; 1555 1556 1557 int kvmppc_book3s_init_pr(void) 1558 { 1559 int r; 1560 1561 r = kvmppc_core_check_processor_compat_pr(); 1562 if (r < 0) 1563 return r; 1564 1565 kvm_ops_pr.owner = THIS_MODULE; 1566 kvmppc_pr_ops = &kvm_ops_pr; 1567 1568 r = kvmppc_mmu_hpte_sysinit(); 1569 return r; 1570 } 1571 1572 void kvmppc_book3s_exit_pr(void) 1573 { 1574 kvmppc_pr_ops = NULL; 1575 kvmppc_mmu_hpte_sysexit(); 1576 } 1577 1578 /* 1579 * We only support separate modules for book3s 64 1580 */ 1581 #ifdef CONFIG_PPC_BOOK3S_64 1582 1583 module_init(kvmppc_book3s_init_pr); 1584 module_exit(kvmppc_book3s_exit_pr); 1585 1586 MODULE_LICENSE("GPL"); 1587 #endif 1588