1 /* 2 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved. 3 * 4 * Authors: 5 * Alexander Graf <agraf@suse.de> 6 * Kevin Wolf <mail@kevin-wolf.de> 7 * Paul Mackerras <paulus@samba.org> 8 * 9 * Description: 10 * Functions relating to running KVM on Book 3S processors where 11 * we don't have access to hypervisor mode, and we run the guest 12 * in problem state (user mode). 13 * 14 * This file is derived from arch/powerpc/kvm/44x.c, 15 * by Hollis Blanchard <hollisb@us.ibm.com>. 16 * 17 * This program is free software; you can redistribute it and/or modify 18 * it under the terms of the GNU General Public License, version 2, as 19 * published by the Free Software Foundation. 20 */ 21 22 #include <linux/kvm_host.h> 23 #include <linux/err.h> 24 #include <linux/slab.h> 25 26 #include <asm/reg.h> 27 #include <asm/cputable.h> 28 #include <asm/cacheflush.h> 29 #include <asm/tlbflush.h> 30 #include <asm/uaccess.h> 31 #include <asm/io.h> 32 #include <asm/kvm_ppc.h> 33 #include <asm/kvm_book3s.h> 34 #include <asm/mmu_context.h> 35 #include <linux/gfp.h> 36 #include <linux/sched.h> 37 #include <linux/vmalloc.h> 38 #include <linux/highmem.h> 39 40 #include "trace.h" 41 42 /* #define EXIT_DEBUG */ 43 /* #define DEBUG_EXT */ 44 45 static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr, 46 ulong msr); 47 48 /* Some compatibility defines */ 49 #ifdef CONFIG_PPC_BOOK3S_32 50 #define MSR_USER32 MSR_USER 51 #define MSR_USER64 MSR_USER 52 #define HW_PAGE_SIZE PAGE_SIZE 53 #endif 54 55 void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 56 { 57 #ifdef CONFIG_PPC_BOOK3S_64 58 memcpy(to_svcpu(vcpu)->slb, to_book3s(vcpu)->slb_shadow, sizeof(to_svcpu(vcpu)->slb)); 59 memcpy(&get_paca()->shadow_vcpu, to_book3s(vcpu)->shadow_vcpu, 60 sizeof(get_paca()->shadow_vcpu)); 61 to_svcpu(vcpu)->slb_max = to_book3s(vcpu)->slb_shadow_max; 62 #endif 63 64 #ifdef CONFIG_PPC_BOOK3S_32 65 current->thread.kvm_shadow_vcpu = to_book3s(vcpu)->shadow_vcpu; 66 #endif 67 } 68 69 void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu) 70 { 71 #ifdef CONFIG_PPC_BOOK3S_64 72 memcpy(to_book3s(vcpu)->slb_shadow, to_svcpu(vcpu)->slb, sizeof(to_svcpu(vcpu)->slb)); 73 memcpy(to_book3s(vcpu)->shadow_vcpu, &get_paca()->shadow_vcpu, 74 sizeof(get_paca()->shadow_vcpu)); 75 to_book3s(vcpu)->slb_shadow_max = to_svcpu(vcpu)->slb_max; 76 #endif 77 78 kvmppc_giveup_ext(vcpu, MSR_FP); 79 kvmppc_giveup_ext(vcpu, MSR_VEC); 80 kvmppc_giveup_ext(vcpu, MSR_VSX); 81 } 82 83 static void kvmppc_recalc_shadow_msr(struct kvm_vcpu *vcpu) 84 { 85 ulong smsr = vcpu->arch.shared->msr; 86 87 /* Guest MSR values */ 88 smsr &= MSR_FE0 | MSR_FE1 | MSR_SF | MSR_SE | MSR_BE | MSR_DE; 89 /* Process MSR values */ 90 smsr |= MSR_ME | MSR_RI | MSR_IR | MSR_DR | MSR_PR | MSR_EE; 91 /* External providers the guest reserved */ 92 smsr |= (vcpu->arch.shared->msr & vcpu->arch.guest_owned_ext); 93 /* 64-bit Process MSR values */ 94 #ifdef CONFIG_PPC_BOOK3S_64 95 smsr |= MSR_ISF | MSR_HV; 96 #endif 97 vcpu->arch.shadow_msr = smsr; 98 } 99 100 void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 msr) 101 { 102 ulong old_msr = vcpu->arch.shared->msr; 103 104 #ifdef EXIT_DEBUG 105 printk(KERN_INFO "KVM: Set MSR to 0x%llx\n", msr); 106 #endif 107 108 msr &= to_book3s(vcpu)->msr_mask; 109 vcpu->arch.shared->msr = msr; 110 kvmppc_recalc_shadow_msr(vcpu); 111 112 if (msr & MSR_POW) { 113 if (!vcpu->arch.pending_exceptions) { 114 kvm_vcpu_block(vcpu); 115 vcpu->stat.halt_wakeup++; 116 117 /* Unset POW bit after we woke up */ 118 msr &= ~MSR_POW; 119 vcpu->arch.shared->msr = msr; 120 } 121 } 122 123 if ((vcpu->arch.shared->msr & (MSR_PR|MSR_IR|MSR_DR)) != 124 (old_msr & (MSR_PR|MSR_IR|MSR_DR))) { 125 kvmppc_mmu_flush_segments(vcpu); 126 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)); 127 128 /* Preload magic page segment when in kernel mode */ 129 if (!(msr & MSR_PR) && vcpu->arch.magic_page_pa) { 130 struct kvm_vcpu_arch *a = &vcpu->arch; 131 132 if (msr & MSR_DR) 133 kvmppc_mmu_map_segment(vcpu, a->magic_page_ea); 134 else 135 kvmppc_mmu_map_segment(vcpu, a->magic_page_pa); 136 } 137 } 138 139 /* Preload FPU if it's enabled */ 140 if (vcpu->arch.shared->msr & MSR_FP) 141 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP); 142 } 143 144 void kvmppc_set_pvr(struct kvm_vcpu *vcpu, u32 pvr) 145 { 146 u32 host_pvr; 147 148 vcpu->arch.hflags &= ~BOOK3S_HFLAG_SLB; 149 vcpu->arch.pvr = pvr; 150 #ifdef CONFIG_PPC_BOOK3S_64 151 if ((pvr >= 0x330000) && (pvr < 0x70330000)) { 152 kvmppc_mmu_book3s_64_init(vcpu); 153 to_book3s(vcpu)->hior = 0xfff00000; 154 to_book3s(vcpu)->msr_mask = 0xffffffffffffffffULL; 155 } else 156 #endif 157 { 158 kvmppc_mmu_book3s_32_init(vcpu); 159 to_book3s(vcpu)->hior = 0; 160 to_book3s(vcpu)->msr_mask = 0xffffffffULL; 161 } 162 163 /* If we are in hypervisor level on 970, we can tell the CPU to 164 * treat DCBZ as 32 bytes store */ 165 vcpu->arch.hflags &= ~BOOK3S_HFLAG_DCBZ32; 166 if (vcpu->arch.mmu.is_dcbz32(vcpu) && (mfmsr() & MSR_HV) && 167 !strcmp(cur_cpu_spec->platform, "ppc970")) 168 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32; 169 170 /* Cell performs badly if MSR_FEx are set. So let's hope nobody 171 really needs them in a VM on Cell and force disable them. */ 172 if (!strcmp(cur_cpu_spec->platform, "ppc-cell-be")) 173 to_book3s(vcpu)->msr_mask &= ~(MSR_FE0 | MSR_FE1); 174 175 #ifdef CONFIG_PPC_BOOK3S_32 176 /* 32 bit Book3S always has 32 byte dcbz */ 177 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32; 178 #endif 179 180 /* On some CPUs we can execute paired single operations natively */ 181 asm ( "mfpvr %0" : "=r"(host_pvr)); 182 switch (host_pvr) { 183 case 0x00080200: /* lonestar 2.0 */ 184 case 0x00088202: /* lonestar 2.2 */ 185 case 0x70000100: /* gekko 1.0 */ 186 case 0x00080100: /* gekko 2.0 */ 187 case 0x00083203: /* gekko 2.3a */ 188 case 0x00083213: /* gekko 2.3b */ 189 case 0x00083204: /* gekko 2.4 */ 190 case 0x00083214: /* gekko 2.4e (8SE) - retail HW2 */ 191 case 0x00087200: /* broadway */ 192 vcpu->arch.hflags |= BOOK3S_HFLAG_NATIVE_PS; 193 /* Enable HID2.PSE - in case we need it later */ 194 mtspr(SPRN_HID2_GEKKO, mfspr(SPRN_HID2_GEKKO) | (1 << 29)); 195 } 196 } 197 198 /* Book3s_32 CPUs always have 32 bytes cache line size, which Linux assumes. To 199 * make Book3s_32 Linux work on Book3s_64, we have to make sure we trap dcbz to 200 * emulate 32 bytes dcbz length. 201 * 202 * The Book3s_64 inventors also realized this case and implemented a special bit 203 * in the HID5 register, which is a hypervisor ressource. Thus we can't use it. 204 * 205 * My approach here is to patch the dcbz instruction on executing pages. 206 */ 207 static void kvmppc_patch_dcbz(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte) 208 { 209 struct page *hpage; 210 u64 hpage_offset; 211 u32 *page; 212 int i; 213 214 hpage = gfn_to_page(vcpu->kvm, pte->raddr >> PAGE_SHIFT); 215 if (is_error_page(hpage)) { 216 kvm_release_page_clean(hpage); 217 return; 218 } 219 220 hpage_offset = pte->raddr & ~PAGE_MASK; 221 hpage_offset &= ~0xFFFULL; 222 hpage_offset /= 4; 223 224 get_page(hpage); 225 page = kmap_atomic(hpage, KM_USER0); 226 227 /* patch dcbz into reserved instruction, so we trap */ 228 for (i=hpage_offset; i < hpage_offset + (HW_PAGE_SIZE / 4); i++) 229 if ((page[i] & 0xff0007ff) == INS_DCBZ) 230 page[i] &= 0xfffffff7; 231 232 kunmap_atomic(page, KM_USER0); 233 put_page(hpage); 234 } 235 236 static int kvmppc_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 237 { 238 ulong mp_pa = vcpu->arch.magic_page_pa; 239 240 if (unlikely(mp_pa) && 241 unlikely((mp_pa & KVM_PAM) >> PAGE_SHIFT == gfn)) { 242 return 1; 243 } 244 245 return kvm_is_visible_gfn(vcpu->kvm, gfn); 246 } 247 248 int kvmppc_handle_pagefault(struct kvm_run *run, struct kvm_vcpu *vcpu, 249 ulong eaddr, int vec) 250 { 251 bool data = (vec == BOOK3S_INTERRUPT_DATA_STORAGE); 252 int r = RESUME_GUEST; 253 int relocated; 254 int page_found = 0; 255 struct kvmppc_pte pte; 256 bool is_mmio = false; 257 bool dr = (vcpu->arch.shared->msr & MSR_DR) ? true : false; 258 bool ir = (vcpu->arch.shared->msr & MSR_IR) ? true : false; 259 u64 vsid; 260 261 relocated = data ? dr : ir; 262 263 /* Resolve real address if translation turned on */ 264 if (relocated) { 265 page_found = vcpu->arch.mmu.xlate(vcpu, eaddr, &pte, data); 266 } else { 267 pte.may_execute = true; 268 pte.may_read = true; 269 pte.may_write = true; 270 pte.raddr = eaddr & KVM_PAM; 271 pte.eaddr = eaddr; 272 pte.vpage = eaddr >> 12; 273 } 274 275 switch (vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) { 276 case 0: 277 pte.vpage |= ((u64)VSID_REAL << (SID_SHIFT - 12)); 278 break; 279 case MSR_DR: 280 case MSR_IR: 281 vcpu->arch.mmu.esid_to_vsid(vcpu, eaddr >> SID_SHIFT, &vsid); 282 283 if ((vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) == MSR_DR) 284 pte.vpage |= ((u64)VSID_REAL_DR << (SID_SHIFT - 12)); 285 else 286 pte.vpage |= ((u64)VSID_REAL_IR << (SID_SHIFT - 12)); 287 pte.vpage |= vsid; 288 289 if (vsid == -1) 290 page_found = -EINVAL; 291 break; 292 } 293 294 if (vcpu->arch.mmu.is_dcbz32(vcpu) && 295 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) { 296 /* 297 * If we do the dcbz hack, we have to NX on every execution, 298 * so we can patch the executing code. This renders our guest 299 * NX-less. 300 */ 301 pte.may_execute = !data; 302 } 303 304 if (page_found == -ENOENT) { 305 /* Page not found in guest PTE entries */ 306 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu); 307 vcpu->arch.shared->dsisr = to_svcpu(vcpu)->fault_dsisr; 308 vcpu->arch.shared->msr |= 309 (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL); 310 kvmppc_book3s_queue_irqprio(vcpu, vec); 311 } else if (page_found == -EPERM) { 312 /* Storage protection */ 313 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu); 314 vcpu->arch.shared->dsisr = 315 to_svcpu(vcpu)->fault_dsisr & ~DSISR_NOHPTE; 316 vcpu->arch.shared->dsisr |= DSISR_PROTFAULT; 317 vcpu->arch.shared->msr |= 318 (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL); 319 kvmppc_book3s_queue_irqprio(vcpu, vec); 320 } else if (page_found == -EINVAL) { 321 /* Page not found in guest SLB */ 322 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu); 323 kvmppc_book3s_queue_irqprio(vcpu, vec + 0x80); 324 } else if (!is_mmio && 325 kvmppc_visible_gfn(vcpu, pte.raddr >> PAGE_SHIFT)) { 326 /* The guest's PTE is not mapped yet. Map on the host */ 327 kvmppc_mmu_map_page(vcpu, &pte); 328 if (data) 329 vcpu->stat.sp_storage++; 330 else if (vcpu->arch.mmu.is_dcbz32(vcpu) && 331 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) 332 kvmppc_patch_dcbz(vcpu, &pte); 333 } else { 334 /* MMIO */ 335 vcpu->stat.mmio_exits++; 336 vcpu->arch.paddr_accessed = pte.raddr; 337 r = kvmppc_emulate_mmio(run, vcpu); 338 if ( r == RESUME_HOST_NV ) 339 r = RESUME_HOST; 340 } 341 342 return r; 343 } 344 345 static inline int get_fpr_index(int i) 346 { 347 #ifdef CONFIG_VSX 348 i *= 2; 349 #endif 350 return i; 351 } 352 353 /* Give up external provider (FPU, Altivec, VSX) */ 354 void kvmppc_giveup_ext(struct kvm_vcpu *vcpu, ulong msr) 355 { 356 struct thread_struct *t = ¤t->thread; 357 u64 *vcpu_fpr = vcpu->arch.fpr; 358 #ifdef CONFIG_VSX 359 u64 *vcpu_vsx = vcpu->arch.vsr; 360 #endif 361 u64 *thread_fpr = (u64*)t->fpr; 362 int i; 363 364 if (!(vcpu->arch.guest_owned_ext & msr)) 365 return; 366 367 #ifdef DEBUG_EXT 368 printk(KERN_INFO "Giving up ext 0x%lx\n", msr); 369 #endif 370 371 switch (msr) { 372 case MSR_FP: 373 giveup_fpu(current); 374 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++) 375 vcpu_fpr[i] = thread_fpr[get_fpr_index(i)]; 376 377 vcpu->arch.fpscr = t->fpscr.val; 378 break; 379 case MSR_VEC: 380 #ifdef CONFIG_ALTIVEC 381 giveup_altivec(current); 382 memcpy(vcpu->arch.vr, t->vr, sizeof(vcpu->arch.vr)); 383 vcpu->arch.vscr = t->vscr; 384 #endif 385 break; 386 case MSR_VSX: 387 #ifdef CONFIG_VSX 388 __giveup_vsx(current); 389 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++) 390 vcpu_vsx[i] = thread_fpr[get_fpr_index(i) + 1]; 391 #endif 392 break; 393 default: 394 BUG(); 395 } 396 397 vcpu->arch.guest_owned_ext &= ~msr; 398 current->thread.regs->msr &= ~msr; 399 kvmppc_recalc_shadow_msr(vcpu); 400 } 401 402 static int kvmppc_read_inst(struct kvm_vcpu *vcpu) 403 { 404 ulong srr0 = kvmppc_get_pc(vcpu); 405 u32 last_inst = kvmppc_get_last_inst(vcpu); 406 int ret; 407 408 ret = kvmppc_ld(vcpu, &srr0, sizeof(u32), &last_inst, false); 409 if (ret == -ENOENT) { 410 ulong msr = vcpu->arch.shared->msr; 411 412 msr = kvmppc_set_field(msr, 33, 33, 1); 413 msr = kvmppc_set_field(msr, 34, 36, 0); 414 vcpu->arch.shared->msr = kvmppc_set_field(msr, 42, 47, 0); 415 kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_INST_STORAGE); 416 return EMULATE_AGAIN; 417 } 418 419 return EMULATE_DONE; 420 } 421 422 static int kvmppc_check_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr) 423 { 424 425 /* Need to do paired single emulation? */ 426 if (!(vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE)) 427 return EMULATE_DONE; 428 429 /* Read out the instruction */ 430 if (kvmppc_read_inst(vcpu) == EMULATE_DONE) 431 /* Need to emulate */ 432 return EMULATE_FAIL; 433 434 return EMULATE_AGAIN; 435 } 436 437 /* Handle external providers (FPU, Altivec, VSX) */ 438 static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr, 439 ulong msr) 440 { 441 struct thread_struct *t = ¤t->thread; 442 u64 *vcpu_fpr = vcpu->arch.fpr; 443 #ifdef CONFIG_VSX 444 u64 *vcpu_vsx = vcpu->arch.vsr; 445 #endif 446 u64 *thread_fpr = (u64*)t->fpr; 447 int i; 448 449 /* When we have paired singles, we emulate in software */ 450 if (vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE) 451 return RESUME_GUEST; 452 453 if (!(vcpu->arch.shared->msr & msr)) { 454 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 455 return RESUME_GUEST; 456 } 457 458 /* We already own the ext */ 459 if (vcpu->arch.guest_owned_ext & msr) { 460 return RESUME_GUEST; 461 } 462 463 #ifdef DEBUG_EXT 464 printk(KERN_INFO "Loading up ext 0x%lx\n", msr); 465 #endif 466 467 current->thread.regs->msr |= msr; 468 469 switch (msr) { 470 case MSR_FP: 471 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++) 472 thread_fpr[get_fpr_index(i)] = vcpu_fpr[i]; 473 474 t->fpscr.val = vcpu->arch.fpscr; 475 t->fpexc_mode = 0; 476 kvmppc_load_up_fpu(); 477 break; 478 case MSR_VEC: 479 #ifdef CONFIG_ALTIVEC 480 memcpy(t->vr, vcpu->arch.vr, sizeof(vcpu->arch.vr)); 481 t->vscr = vcpu->arch.vscr; 482 t->vrsave = -1; 483 kvmppc_load_up_altivec(); 484 #endif 485 break; 486 case MSR_VSX: 487 #ifdef CONFIG_VSX 488 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++) 489 thread_fpr[get_fpr_index(i) + 1] = vcpu_vsx[i]; 490 kvmppc_load_up_vsx(); 491 #endif 492 break; 493 default: 494 BUG(); 495 } 496 497 vcpu->arch.guest_owned_ext |= msr; 498 499 kvmppc_recalc_shadow_msr(vcpu); 500 501 return RESUME_GUEST; 502 } 503 504 int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu, 505 unsigned int exit_nr) 506 { 507 int r = RESUME_HOST; 508 509 vcpu->stat.sum_exits++; 510 511 run->exit_reason = KVM_EXIT_UNKNOWN; 512 run->ready_for_interrupt_injection = 1; 513 514 trace_kvm_book3s_exit(exit_nr, vcpu); 515 kvm_resched(vcpu); 516 switch (exit_nr) { 517 case BOOK3S_INTERRUPT_INST_STORAGE: 518 vcpu->stat.pf_instruc++; 519 520 #ifdef CONFIG_PPC_BOOK3S_32 521 /* We set segments as unused segments when invalidating them. So 522 * treat the respective fault as segment fault. */ 523 if (to_svcpu(vcpu)->sr[kvmppc_get_pc(vcpu) >> SID_SHIFT] 524 == SR_INVALID) { 525 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)); 526 r = RESUME_GUEST; 527 break; 528 } 529 #endif 530 531 /* only care about PTEG not found errors, but leave NX alone */ 532 if (to_svcpu(vcpu)->shadow_srr1 & 0x40000000) { 533 r = kvmppc_handle_pagefault(run, vcpu, kvmppc_get_pc(vcpu), exit_nr); 534 vcpu->stat.sp_instruc++; 535 } else if (vcpu->arch.mmu.is_dcbz32(vcpu) && 536 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) { 537 /* 538 * XXX If we do the dcbz hack we use the NX bit to flush&patch the page, 539 * so we can't use the NX bit inside the guest. Let's cross our fingers, 540 * that no guest that needs the dcbz hack does NX. 541 */ 542 kvmppc_mmu_pte_flush(vcpu, kvmppc_get_pc(vcpu), ~0xFFFUL); 543 r = RESUME_GUEST; 544 } else { 545 vcpu->arch.shared->msr |= 546 to_svcpu(vcpu)->shadow_srr1 & 0x58000000; 547 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 548 r = RESUME_GUEST; 549 } 550 break; 551 case BOOK3S_INTERRUPT_DATA_STORAGE: 552 { 553 ulong dar = kvmppc_get_fault_dar(vcpu); 554 vcpu->stat.pf_storage++; 555 556 #ifdef CONFIG_PPC_BOOK3S_32 557 /* We set segments as unused segments when invalidating them. So 558 * treat the respective fault as segment fault. */ 559 if ((to_svcpu(vcpu)->sr[dar >> SID_SHIFT]) == SR_INVALID) { 560 kvmppc_mmu_map_segment(vcpu, dar); 561 r = RESUME_GUEST; 562 break; 563 } 564 #endif 565 566 /* The only case we need to handle is missing shadow PTEs */ 567 if (to_svcpu(vcpu)->fault_dsisr & DSISR_NOHPTE) { 568 r = kvmppc_handle_pagefault(run, vcpu, dar, exit_nr); 569 } else { 570 vcpu->arch.shared->dar = dar; 571 vcpu->arch.shared->dsisr = to_svcpu(vcpu)->fault_dsisr; 572 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 573 r = RESUME_GUEST; 574 } 575 break; 576 } 577 case BOOK3S_INTERRUPT_DATA_SEGMENT: 578 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_fault_dar(vcpu)) < 0) { 579 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu); 580 kvmppc_book3s_queue_irqprio(vcpu, 581 BOOK3S_INTERRUPT_DATA_SEGMENT); 582 } 583 r = RESUME_GUEST; 584 break; 585 case BOOK3S_INTERRUPT_INST_SEGMENT: 586 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)) < 0) { 587 kvmppc_book3s_queue_irqprio(vcpu, 588 BOOK3S_INTERRUPT_INST_SEGMENT); 589 } 590 r = RESUME_GUEST; 591 break; 592 /* We're good on these - the host merely wanted to get our attention */ 593 case BOOK3S_INTERRUPT_DECREMENTER: 594 vcpu->stat.dec_exits++; 595 r = RESUME_GUEST; 596 break; 597 case BOOK3S_INTERRUPT_EXTERNAL: 598 vcpu->stat.ext_intr_exits++; 599 r = RESUME_GUEST; 600 break; 601 case BOOK3S_INTERRUPT_PERFMON: 602 r = RESUME_GUEST; 603 break; 604 case BOOK3S_INTERRUPT_PROGRAM: 605 { 606 enum emulation_result er; 607 ulong flags; 608 609 program_interrupt: 610 flags = to_svcpu(vcpu)->shadow_srr1 & 0x1f0000ull; 611 612 if (vcpu->arch.shared->msr & MSR_PR) { 613 #ifdef EXIT_DEBUG 614 printk(KERN_INFO "Userspace triggered 0x700 exception at 0x%lx (0x%x)\n", kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu)); 615 #endif 616 if ((kvmppc_get_last_inst(vcpu) & 0xff0007ff) != 617 (INS_DCBZ & 0xfffffff7)) { 618 kvmppc_core_queue_program(vcpu, flags); 619 r = RESUME_GUEST; 620 break; 621 } 622 } 623 624 vcpu->stat.emulated_inst_exits++; 625 er = kvmppc_emulate_instruction(run, vcpu); 626 switch (er) { 627 case EMULATE_DONE: 628 r = RESUME_GUEST_NV; 629 break; 630 case EMULATE_AGAIN: 631 r = RESUME_GUEST; 632 break; 633 case EMULATE_FAIL: 634 printk(KERN_CRIT "%s: emulation at %lx failed (%08x)\n", 635 __func__, kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu)); 636 kvmppc_core_queue_program(vcpu, flags); 637 r = RESUME_GUEST; 638 break; 639 case EMULATE_DO_MMIO: 640 run->exit_reason = KVM_EXIT_MMIO; 641 r = RESUME_HOST_NV; 642 break; 643 default: 644 BUG(); 645 } 646 break; 647 } 648 case BOOK3S_INTERRUPT_SYSCALL: 649 if (vcpu->arch.osi_enabled && 650 (((u32)kvmppc_get_gpr(vcpu, 3)) == OSI_SC_MAGIC_R3) && 651 (((u32)kvmppc_get_gpr(vcpu, 4)) == OSI_SC_MAGIC_R4)) { 652 /* MOL hypercalls */ 653 u64 *gprs = run->osi.gprs; 654 int i; 655 656 run->exit_reason = KVM_EXIT_OSI; 657 for (i = 0; i < 32; i++) 658 gprs[i] = kvmppc_get_gpr(vcpu, i); 659 vcpu->arch.osi_needed = 1; 660 r = RESUME_HOST_NV; 661 } else if (!(vcpu->arch.shared->msr & MSR_PR) && 662 (((u32)kvmppc_get_gpr(vcpu, 0)) == KVM_SC_MAGIC_R0)) { 663 /* KVM PV hypercalls */ 664 kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu)); 665 r = RESUME_GUEST; 666 } else { 667 /* Guest syscalls */ 668 vcpu->stat.syscall_exits++; 669 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 670 r = RESUME_GUEST; 671 } 672 break; 673 case BOOK3S_INTERRUPT_FP_UNAVAIL: 674 case BOOK3S_INTERRUPT_ALTIVEC: 675 case BOOK3S_INTERRUPT_VSX: 676 { 677 int ext_msr = 0; 678 679 switch (exit_nr) { 680 case BOOK3S_INTERRUPT_FP_UNAVAIL: ext_msr = MSR_FP; break; 681 case BOOK3S_INTERRUPT_ALTIVEC: ext_msr = MSR_VEC; break; 682 case BOOK3S_INTERRUPT_VSX: ext_msr = MSR_VSX; break; 683 } 684 685 switch (kvmppc_check_ext(vcpu, exit_nr)) { 686 case EMULATE_DONE: 687 /* everything ok - let's enable the ext */ 688 r = kvmppc_handle_ext(vcpu, exit_nr, ext_msr); 689 break; 690 case EMULATE_FAIL: 691 /* we need to emulate this instruction */ 692 goto program_interrupt; 693 break; 694 default: 695 /* nothing to worry about - go again */ 696 break; 697 } 698 break; 699 } 700 case BOOK3S_INTERRUPT_ALIGNMENT: 701 if (kvmppc_read_inst(vcpu) == EMULATE_DONE) { 702 vcpu->arch.shared->dsisr = kvmppc_alignment_dsisr(vcpu, 703 kvmppc_get_last_inst(vcpu)); 704 vcpu->arch.shared->dar = kvmppc_alignment_dar(vcpu, 705 kvmppc_get_last_inst(vcpu)); 706 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 707 } 708 r = RESUME_GUEST; 709 break; 710 case BOOK3S_INTERRUPT_MACHINE_CHECK: 711 case BOOK3S_INTERRUPT_TRACE: 712 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 713 r = RESUME_GUEST; 714 break; 715 default: 716 /* Ugh - bork here! What did we get? */ 717 printk(KERN_EMERG "exit_nr=0x%x | pc=0x%lx | msr=0x%lx\n", 718 exit_nr, kvmppc_get_pc(vcpu), to_svcpu(vcpu)->shadow_srr1); 719 r = RESUME_HOST; 720 BUG(); 721 break; 722 } 723 724 725 if (!(r & RESUME_HOST)) { 726 /* To avoid clobbering exit_reason, only check for signals if 727 * we aren't already exiting to userspace for some other 728 * reason. */ 729 if (signal_pending(current)) { 730 #ifdef EXIT_DEBUG 731 printk(KERN_EMERG "KVM: Going back to host\n"); 732 #endif 733 vcpu->stat.signal_exits++; 734 run->exit_reason = KVM_EXIT_INTR; 735 r = -EINTR; 736 } else { 737 /* In case an interrupt came in that was triggered 738 * from userspace (like DEC), we need to check what 739 * to inject now! */ 740 kvmppc_core_deliver_interrupts(vcpu); 741 } 742 } 743 744 trace_kvm_book3s_reenter(r, vcpu); 745 746 return r; 747 } 748 749 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 750 struct kvm_sregs *sregs) 751 { 752 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu); 753 int i; 754 755 sregs->pvr = vcpu->arch.pvr; 756 757 sregs->u.s.sdr1 = to_book3s(vcpu)->sdr1; 758 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) { 759 for (i = 0; i < 64; i++) { 760 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige | i; 761 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv; 762 } 763 } else { 764 for (i = 0; i < 16; i++) 765 sregs->u.s.ppc32.sr[i] = vcpu->arch.shared->sr[i]; 766 767 for (i = 0; i < 8; i++) { 768 sregs->u.s.ppc32.ibat[i] = vcpu3s->ibat[i].raw; 769 sregs->u.s.ppc32.dbat[i] = vcpu3s->dbat[i].raw; 770 } 771 } 772 773 return 0; 774 } 775 776 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 777 struct kvm_sregs *sregs) 778 { 779 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu); 780 int i; 781 782 kvmppc_set_pvr(vcpu, sregs->pvr); 783 784 vcpu3s->sdr1 = sregs->u.s.sdr1; 785 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) { 786 for (i = 0; i < 64; i++) { 787 vcpu->arch.mmu.slbmte(vcpu, sregs->u.s.ppc64.slb[i].slbv, 788 sregs->u.s.ppc64.slb[i].slbe); 789 } 790 } else { 791 for (i = 0; i < 16; i++) { 792 vcpu->arch.mmu.mtsrin(vcpu, i, sregs->u.s.ppc32.sr[i]); 793 } 794 for (i = 0; i < 8; i++) { 795 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), false, 796 (u32)sregs->u.s.ppc32.ibat[i]); 797 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), true, 798 (u32)(sregs->u.s.ppc32.ibat[i] >> 32)); 799 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), false, 800 (u32)sregs->u.s.ppc32.dbat[i]); 801 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), true, 802 (u32)(sregs->u.s.ppc32.dbat[i] >> 32)); 803 } 804 } 805 806 /* Flush the MMU after messing with the segments */ 807 kvmppc_mmu_pte_flush(vcpu, 0, 0); 808 809 return 0; 810 } 811 812 int kvmppc_core_check_processor_compat(void) 813 { 814 return 0; 815 } 816 817 struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id) 818 { 819 struct kvmppc_vcpu_book3s *vcpu_book3s; 820 struct kvm_vcpu *vcpu; 821 int err = -ENOMEM; 822 unsigned long p; 823 824 vcpu_book3s = vzalloc(sizeof(struct kvmppc_vcpu_book3s)); 825 if (!vcpu_book3s) 826 goto out; 827 828 vcpu_book3s->shadow_vcpu = (struct kvmppc_book3s_shadow_vcpu *) 829 kzalloc(sizeof(*vcpu_book3s->shadow_vcpu), GFP_KERNEL); 830 if (!vcpu_book3s->shadow_vcpu) 831 goto free_vcpu; 832 833 vcpu = &vcpu_book3s->vcpu; 834 err = kvm_vcpu_init(vcpu, kvm, id); 835 if (err) 836 goto free_shadow_vcpu; 837 838 p = __get_free_page(GFP_KERNEL|__GFP_ZERO); 839 /* the real shared page fills the last 4k of our page */ 840 vcpu->arch.shared = (void*)(p + PAGE_SIZE - 4096); 841 if (!p) 842 goto uninit_vcpu; 843 844 vcpu->arch.host_retip = kvm_return_point; 845 vcpu->arch.host_msr = mfmsr(); 846 #ifdef CONFIG_PPC_BOOK3S_64 847 /* default to book3s_64 (970fx) */ 848 vcpu->arch.pvr = 0x3C0301; 849 #else 850 /* default to book3s_32 (750) */ 851 vcpu->arch.pvr = 0x84202; 852 #endif 853 kvmppc_set_pvr(vcpu, vcpu->arch.pvr); 854 vcpu->arch.slb_nr = 64; 855 856 /* remember where some real-mode handlers are */ 857 vcpu->arch.trampoline_lowmem = __pa(kvmppc_handler_lowmem_trampoline); 858 vcpu->arch.trampoline_enter = __pa(kvmppc_handler_trampoline_enter); 859 vcpu->arch.highmem_handler = (ulong)kvmppc_handler_highmem; 860 #ifdef CONFIG_PPC_BOOK3S_64 861 vcpu->arch.rmcall = *(ulong*)kvmppc_rmcall; 862 #else 863 vcpu->arch.rmcall = (ulong)kvmppc_rmcall; 864 #endif 865 866 vcpu->arch.shadow_msr = MSR_USER64; 867 868 err = kvmppc_mmu_init(vcpu); 869 if (err < 0) 870 goto uninit_vcpu; 871 872 return vcpu; 873 874 uninit_vcpu: 875 kvm_vcpu_uninit(vcpu); 876 free_shadow_vcpu: 877 kfree(vcpu_book3s->shadow_vcpu); 878 free_vcpu: 879 vfree(vcpu_book3s); 880 out: 881 return ERR_PTR(err); 882 } 883 884 void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu) 885 { 886 struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu); 887 888 free_page((unsigned long)vcpu->arch.shared & PAGE_MASK); 889 kvm_vcpu_uninit(vcpu); 890 kfree(vcpu_book3s->shadow_vcpu); 891 vfree(vcpu_book3s); 892 } 893 894 int kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) 895 { 896 int ret; 897 double fpr[32][TS_FPRWIDTH]; 898 unsigned int fpscr; 899 int fpexc_mode; 900 #ifdef CONFIG_ALTIVEC 901 vector128 vr[32]; 902 vector128 vscr; 903 unsigned long uninitialized_var(vrsave); 904 int used_vr; 905 #endif 906 #ifdef CONFIG_VSX 907 int used_vsr; 908 #endif 909 ulong ext_msr; 910 911 /* No need to go into the guest when all we do is going out */ 912 if (signal_pending(current)) { 913 kvm_run->exit_reason = KVM_EXIT_INTR; 914 return -EINTR; 915 } 916 917 /* Save FPU state in stack */ 918 if (current->thread.regs->msr & MSR_FP) 919 giveup_fpu(current); 920 memcpy(fpr, current->thread.fpr, sizeof(current->thread.fpr)); 921 fpscr = current->thread.fpscr.val; 922 fpexc_mode = current->thread.fpexc_mode; 923 924 #ifdef CONFIG_ALTIVEC 925 /* Save Altivec state in stack */ 926 used_vr = current->thread.used_vr; 927 if (used_vr) { 928 if (current->thread.regs->msr & MSR_VEC) 929 giveup_altivec(current); 930 memcpy(vr, current->thread.vr, sizeof(current->thread.vr)); 931 vscr = current->thread.vscr; 932 vrsave = current->thread.vrsave; 933 } 934 #endif 935 936 #ifdef CONFIG_VSX 937 /* Save VSX state in stack */ 938 used_vsr = current->thread.used_vsr; 939 if (used_vsr && (current->thread.regs->msr & MSR_VSX)) 940 __giveup_vsx(current); 941 #endif 942 943 /* Remember the MSR with disabled extensions */ 944 ext_msr = current->thread.regs->msr; 945 946 /* Preload FPU if it's enabled */ 947 if (vcpu->arch.shared->msr & MSR_FP) 948 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP); 949 950 kvm_guest_enter(); 951 952 ret = __kvmppc_vcpu_run(kvm_run, vcpu); 953 954 kvm_guest_exit(); 955 956 local_irq_disable(); 957 958 current->thread.regs->msr = ext_msr; 959 960 /* Make sure we save the guest FPU/Altivec/VSX state */ 961 kvmppc_giveup_ext(vcpu, MSR_FP); 962 kvmppc_giveup_ext(vcpu, MSR_VEC); 963 kvmppc_giveup_ext(vcpu, MSR_VSX); 964 965 /* Restore FPU state from stack */ 966 memcpy(current->thread.fpr, fpr, sizeof(current->thread.fpr)); 967 current->thread.fpscr.val = fpscr; 968 current->thread.fpexc_mode = fpexc_mode; 969 970 #ifdef CONFIG_ALTIVEC 971 /* Restore Altivec state from stack */ 972 if (used_vr && current->thread.used_vr) { 973 memcpy(current->thread.vr, vr, sizeof(current->thread.vr)); 974 current->thread.vscr = vscr; 975 current->thread.vrsave = vrsave; 976 } 977 current->thread.used_vr = used_vr; 978 #endif 979 980 #ifdef CONFIG_VSX 981 current->thread.used_vsr = used_vsr; 982 #endif 983 984 return ret; 985 } 986 987 int kvmppc_core_prepare_memory_region(struct kvm *kvm, 988 struct kvm_userspace_memory_region *mem) 989 { 990 return 0; 991 } 992 993 void kvmppc_core_commit_memory_region(struct kvm *kvm, 994 struct kvm_userspace_memory_region *mem) 995 { 996 } 997 998 int kvmppc_core_init_vm(struct kvm *kvm) 999 { 1000 return 0; 1001 } 1002 1003 void kvmppc_core_destroy_vm(struct kvm *kvm) 1004 { 1005 } 1006 1007 static int kvmppc_book3s_init(void) 1008 { 1009 int r; 1010 1011 r = kvm_init(NULL, sizeof(struct kvmppc_vcpu_book3s), 0, 1012 THIS_MODULE); 1013 1014 if (r) 1015 return r; 1016 1017 r = kvmppc_mmu_hpte_sysinit(); 1018 1019 return r; 1020 } 1021 1022 static void kvmppc_book3s_exit(void) 1023 { 1024 kvmppc_mmu_hpte_sysexit(); 1025 kvm_exit(); 1026 } 1027 1028 module_init(kvmppc_book3s_init); 1029 module_exit(kvmppc_book3s_exit); 1030