1 /* 2 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved. 3 * 4 * Authors: 5 * Alexander Graf <agraf@suse.de> 6 * Kevin Wolf <mail@kevin-wolf.de> 7 * 8 * Description: 9 * This file is derived from arch/powerpc/kvm/44x.c, 10 * by Hollis Blanchard <hollisb@us.ibm.com>. 11 * 12 * This program is free software; you can redistribute it and/or modify 13 * it under the terms of the GNU General Public License, version 2, as 14 * published by the Free Software Foundation. 15 */ 16 17 #include <linux/kvm_host.h> 18 #include <linux/err.h> 19 #include <linux/slab.h> 20 #include "trace.h" 21 22 #include <asm/reg.h> 23 #include <asm/cputable.h> 24 #include <asm/cacheflush.h> 25 #include <asm/tlbflush.h> 26 #include <asm/uaccess.h> 27 #include <asm/io.h> 28 #include <asm/kvm_ppc.h> 29 #include <asm/kvm_book3s.h> 30 #include <asm/mmu_context.h> 31 #include <linux/gfp.h> 32 #include <linux/sched.h> 33 #include <linux/vmalloc.h> 34 #include <linux/highmem.h> 35 36 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU 37 38 /* #define EXIT_DEBUG */ 39 /* #define DEBUG_EXT */ 40 41 static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr, 42 ulong msr); 43 44 /* Some compatibility defines */ 45 #ifdef CONFIG_PPC_BOOK3S_32 46 #define MSR_USER32 MSR_USER 47 #define MSR_USER64 MSR_USER 48 #define HW_PAGE_SIZE PAGE_SIZE 49 #endif 50 51 struct kvm_stats_debugfs_item debugfs_entries[] = { 52 { "exits", VCPU_STAT(sum_exits) }, 53 { "mmio", VCPU_STAT(mmio_exits) }, 54 { "sig", VCPU_STAT(signal_exits) }, 55 { "sysc", VCPU_STAT(syscall_exits) }, 56 { "inst_emu", VCPU_STAT(emulated_inst_exits) }, 57 { "dec", VCPU_STAT(dec_exits) }, 58 { "ext_intr", VCPU_STAT(ext_intr_exits) }, 59 { "queue_intr", VCPU_STAT(queue_intr) }, 60 { "halt_wakeup", VCPU_STAT(halt_wakeup) }, 61 { "pf_storage", VCPU_STAT(pf_storage) }, 62 { "sp_storage", VCPU_STAT(sp_storage) }, 63 { "pf_instruc", VCPU_STAT(pf_instruc) }, 64 { "sp_instruc", VCPU_STAT(sp_instruc) }, 65 { "ld", VCPU_STAT(ld) }, 66 { "ld_slow", VCPU_STAT(ld_slow) }, 67 { "st", VCPU_STAT(st) }, 68 { "st_slow", VCPU_STAT(st_slow) }, 69 { NULL } 70 }; 71 72 void kvmppc_core_load_host_debugstate(struct kvm_vcpu *vcpu) 73 { 74 } 75 76 void kvmppc_core_load_guest_debugstate(struct kvm_vcpu *vcpu) 77 { 78 } 79 80 void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 81 { 82 #ifdef CONFIG_PPC_BOOK3S_64 83 memcpy(to_svcpu(vcpu)->slb, to_book3s(vcpu)->slb_shadow, sizeof(to_svcpu(vcpu)->slb)); 84 memcpy(&get_paca()->shadow_vcpu, to_book3s(vcpu)->shadow_vcpu, 85 sizeof(get_paca()->shadow_vcpu)); 86 to_svcpu(vcpu)->slb_max = to_book3s(vcpu)->slb_shadow_max; 87 #endif 88 89 #ifdef CONFIG_PPC_BOOK3S_32 90 current->thread.kvm_shadow_vcpu = to_book3s(vcpu)->shadow_vcpu; 91 #endif 92 } 93 94 void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu) 95 { 96 #ifdef CONFIG_PPC_BOOK3S_64 97 memcpy(to_book3s(vcpu)->slb_shadow, to_svcpu(vcpu)->slb, sizeof(to_svcpu(vcpu)->slb)); 98 memcpy(to_book3s(vcpu)->shadow_vcpu, &get_paca()->shadow_vcpu, 99 sizeof(get_paca()->shadow_vcpu)); 100 to_book3s(vcpu)->slb_shadow_max = to_svcpu(vcpu)->slb_max; 101 #endif 102 103 kvmppc_giveup_ext(vcpu, MSR_FP); 104 kvmppc_giveup_ext(vcpu, MSR_VEC); 105 kvmppc_giveup_ext(vcpu, MSR_VSX); 106 } 107 108 static void kvmppc_recalc_shadow_msr(struct kvm_vcpu *vcpu) 109 { 110 ulong smsr = vcpu->arch.shared->msr; 111 112 /* Guest MSR values */ 113 smsr &= MSR_FE0 | MSR_FE1 | MSR_SF | MSR_SE | MSR_BE | MSR_DE; 114 /* Process MSR values */ 115 smsr |= MSR_ME | MSR_RI | MSR_IR | MSR_DR | MSR_PR | MSR_EE; 116 /* External providers the guest reserved */ 117 smsr |= (vcpu->arch.shared->msr & vcpu->arch.guest_owned_ext); 118 /* 64-bit Process MSR values */ 119 #ifdef CONFIG_PPC_BOOK3S_64 120 smsr |= MSR_ISF | MSR_HV; 121 #endif 122 vcpu->arch.shadow_msr = smsr; 123 } 124 125 void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 msr) 126 { 127 ulong old_msr = vcpu->arch.shared->msr; 128 129 #ifdef EXIT_DEBUG 130 printk(KERN_INFO "KVM: Set MSR to 0x%llx\n", msr); 131 #endif 132 133 msr &= to_book3s(vcpu)->msr_mask; 134 vcpu->arch.shared->msr = msr; 135 kvmppc_recalc_shadow_msr(vcpu); 136 137 if (msr & MSR_POW) { 138 if (!vcpu->arch.pending_exceptions) { 139 kvm_vcpu_block(vcpu); 140 vcpu->stat.halt_wakeup++; 141 142 /* Unset POW bit after we woke up */ 143 msr &= ~MSR_POW; 144 vcpu->arch.shared->msr = msr; 145 } 146 } 147 148 if ((vcpu->arch.shared->msr & (MSR_PR|MSR_IR|MSR_DR)) != 149 (old_msr & (MSR_PR|MSR_IR|MSR_DR))) { 150 kvmppc_mmu_flush_segments(vcpu); 151 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)); 152 153 /* Preload magic page segment when in kernel mode */ 154 if (!(msr & MSR_PR) && vcpu->arch.magic_page_pa) { 155 struct kvm_vcpu_arch *a = &vcpu->arch; 156 157 if (msr & MSR_DR) 158 kvmppc_mmu_map_segment(vcpu, a->magic_page_ea); 159 else 160 kvmppc_mmu_map_segment(vcpu, a->magic_page_pa); 161 } 162 } 163 164 /* Preload FPU if it's enabled */ 165 if (vcpu->arch.shared->msr & MSR_FP) 166 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP); 167 } 168 169 void kvmppc_inject_interrupt(struct kvm_vcpu *vcpu, int vec, u64 flags) 170 { 171 vcpu->arch.shared->srr0 = kvmppc_get_pc(vcpu); 172 vcpu->arch.shared->srr1 = vcpu->arch.shared->msr | flags; 173 kvmppc_set_pc(vcpu, to_book3s(vcpu)->hior + vec); 174 vcpu->arch.mmu.reset_msr(vcpu); 175 } 176 177 static int kvmppc_book3s_vec2irqprio(unsigned int vec) 178 { 179 unsigned int prio; 180 181 switch (vec) { 182 case 0x100: prio = BOOK3S_IRQPRIO_SYSTEM_RESET; break; 183 case 0x200: prio = BOOK3S_IRQPRIO_MACHINE_CHECK; break; 184 case 0x300: prio = BOOK3S_IRQPRIO_DATA_STORAGE; break; 185 case 0x380: prio = BOOK3S_IRQPRIO_DATA_SEGMENT; break; 186 case 0x400: prio = BOOK3S_IRQPRIO_INST_STORAGE; break; 187 case 0x480: prio = BOOK3S_IRQPRIO_INST_SEGMENT; break; 188 case 0x500: prio = BOOK3S_IRQPRIO_EXTERNAL; break; 189 case 0x501: prio = BOOK3S_IRQPRIO_EXTERNAL_LEVEL; break; 190 case 0x600: prio = BOOK3S_IRQPRIO_ALIGNMENT; break; 191 case 0x700: prio = BOOK3S_IRQPRIO_PROGRAM; break; 192 case 0x800: prio = BOOK3S_IRQPRIO_FP_UNAVAIL; break; 193 case 0x900: prio = BOOK3S_IRQPRIO_DECREMENTER; break; 194 case 0xc00: prio = BOOK3S_IRQPRIO_SYSCALL; break; 195 case 0xd00: prio = BOOK3S_IRQPRIO_DEBUG; break; 196 case 0xf20: prio = BOOK3S_IRQPRIO_ALTIVEC; break; 197 case 0xf40: prio = BOOK3S_IRQPRIO_VSX; break; 198 default: prio = BOOK3S_IRQPRIO_MAX; break; 199 } 200 201 return prio; 202 } 203 204 static void kvmppc_book3s_dequeue_irqprio(struct kvm_vcpu *vcpu, 205 unsigned int vec) 206 { 207 clear_bit(kvmppc_book3s_vec2irqprio(vec), 208 &vcpu->arch.pending_exceptions); 209 210 if (!vcpu->arch.pending_exceptions) 211 vcpu->arch.shared->int_pending = 0; 212 } 213 214 void kvmppc_book3s_queue_irqprio(struct kvm_vcpu *vcpu, unsigned int vec) 215 { 216 vcpu->stat.queue_intr++; 217 218 set_bit(kvmppc_book3s_vec2irqprio(vec), 219 &vcpu->arch.pending_exceptions); 220 #ifdef EXIT_DEBUG 221 printk(KERN_INFO "Queueing interrupt %x\n", vec); 222 #endif 223 } 224 225 226 void kvmppc_core_queue_program(struct kvm_vcpu *vcpu, ulong flags) 227 { 228 to_book3s(vcpu)->prog_flags = flags; 229 kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_PROGRAM); 230 } 231 232 void kvmppc_core_queue_dec(struct kvm_vcpu *vcpu) 233 { 234 kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_DECREMENTER); 235 } 236 237 int kvmppc_core_pending_dec(struct kvm_vcpu *vcpu) 238 { 239 return test_bit(BOOK3S_INTERRUPT_DECREMENTER >> 7, &vcpu->arch.pending_exceptions); 240 } 241 242 void kvmppc_core_dequeue_dec(struct kvm_vcpu *vcpu) 243 { 244 kvmppc_book3s_dequeue_irqprio(vcpu, BOOK3S_INTERRUPT_DECREMENTER); 245 } 246 247 void kvmppc_core_queue_external(struct kvm_vcpu *vcpu, 248 struct kvm_interrupt *irq) 249 { 250 unsigned int vec = BOOK3S_INTERRUPT_EXTERNAL; 251 252 if (irq->irq == KVM_INTERRUPT_SET_LEVEL) 253 vec = BOOK3S_INTERRUPT_EXTERNAL_LEVEL; 254 255 kvmppc_book3s_queue_irqprio(vcpu, vec); 256 } 257 258 void kvmppc_core_dequeue_external(struct kvm_vcpu *vcpu, 259 struct kvm_interrupt *irq) 260 { 261 kvmppc_book3s_dequeue_irqprio(vcpu, BOOK3S_INTERRUPT_EXTERNAL); 262 kvmppc_book3s_dequeue_irqprio(vcpu, BOOK3S_INTERRUPT_EXTERNAL_LEVEL); 263 } 264 265 int kvmppc_book3s_irqprio_deliver(struct kvm_vcpu *vcpu, unsigned int priority) 266 { 267 int deliver = 1; 268 int vec = 0; 269 ulong flags = 0ULL; 270 ulong crit_raw = vcpu->arch.shared->critical; 271 ulong crit_r1 = kvmppc_get_gpr(vcpu, 1); 272 bool crit; 273 274 /* Truncate crit indicators in 32 bit mode */ 275 if (!(vcpu->arch.shared->msr & MSR_SF)) { 276 crit_raw &= 0xffffffff; 277 crit_r1 &= 0xffffffff; 278 } 279 280 /* Critical section when crit == r1 */ 281 crit = (crit_raw == crit_r1); 282 /* ... and we're in supervisor mode */ 283 crit = crit && !(vcpu->arch.shared->msr & MSR_PR); 284 285 switch (priority) { 286 case BOOK3S_IRQPRIO_DECREMENTER: 287 deliver = (vcpu->arch.shared->msr & MSR_EE) && !crit; 288 vec = BOOK3S_INTERRUPT_DECREMENTER; 289 break; 290 case BOOK3S_IRQPRIO_EXTERNAL: 291 case BOOK3S_IRQPRIO_EXTERNAL_LEVEL: 292 deliver = (vcpu->arch.shared->msr & MSR_EE) && !crit; 293 vec = BOOK3S_INTERRUPT_EXTERNAL; 294 break; 295 case BOOK3S_IRQPRIO_SYSTEM_RESET: 296 vec = BOOK3S_INTERRUPT_SYSTEM_RESET; 297 break; 298 case BOOK3S_IRQPRIO_MACHINE_CHECK: 299 vec = BOOK3S_INTERRUPT_MACHINE_CHECK; 300 break; 301 case BOOK3S_IRQPRIO_DATA_STORAGE: 302 vec = BOOK3S_INTERRUPT_DATA_STORAGE; 303 break; 304 case BOOK3S_IRQPRIO_INST_STORAGE: 305 vec = BOOK3S_INTERRUPT_INST_STORAGE; 306 break; 307 case BOOK3S_IRQPRIO_DATA_SEGMENT: 308 vec = BOOK3S_INTERRUPT_DATA_SEGMENT; 309 break; 310 case BOOK3S_IRQPRIO_INST_SEGMENT: 311 vec = BOOK3S_INTERRUPT_INST_SEGMENT; 312 break; 313 case BOOK3S_IRQPRIO_ALIGNMENT: 314 vec = BOOK3S_INTERRUPT_ALIGNMENT; 315 break; 316 case BOOK3S_IRQPRIO_PROGRAM: 317 vec = BOOK3S_INTERRUPT_PROGRAM; 318 flags = to_book3s(vcpu)->prog_flags; 319 break; 320 case BOOK3S_IRQPRIO_VSX: 321 vec = BOOK3S_INTERRUPT_VSX; 322 break; 323 case BOOK3S_IRQPRIO_ALTIVEC: 324 vec = BOOK3S_INTERRUPT_ALTIVEC; 325 break; 326 case BOOK3S_IRQPRIO_FP_UNAVAIL: 327 vec = BOOK3S_INTERRUPT_FP_UNAVAIL; 328 break; 329 case BOOK3S_IRQPRIO_SYSCALL: 330 vec = BOOK3S_INTERRUPT_SYSCALL; 331 break; 332 case BOOK3S_IRQPRIO_DEBUG: 333 vec = BOOK3S_INTERRUPT_TRACE; 334 break; 335 case BOOK3S_IRQPRIO_PERFORMANCE_MONITOR: 336 vec = BOOK3S_INTERRUPT_PERFMON; 337 break; 338 default: 339 deliver = 0; 340 printk(KERN_ERR "KVM: Unknown interrupt: 0x%x\n", priority); 341 break; 342 } 343 344 #if 0 345 printk(KERN_INFO "Deliver interrupt 0x%x? %x\n", vec, deliver); 346 #endif 347 348 if (deliver) 349 kvmppc_inject_interrupt(vcpu, vec, flags); 350 351 return deliver; 352 } 353 354 /* 355 * This function determines if an irqprio should be cleared once issued. 356 */ 357 static bool clear_irqprio(struct kvm_vcpu *vcpu, unsigned int priority) 358 { 359 switch (priority) { 360 case BOOK3S_IRQPRIO_DECREMENTER: 361 /* DEC interrupts get cleared by mtdec */ 362 return false; 363 case BOOK3S_IRQPRIO_EXTERNAL_LEVEL: 364 /* External interrupts get cleared by userspace */ 365 return false; 366 } 367 368 return true; 369 } 370 371 void kvmppc_core_deliver_interrupts(struct kvm_vcpu *vcpu) 372 { 373 unsigned long *pending = &vcpu->arch.pending_exceptions; 374 unsigned long old_pending = vcpu->arch.pending_exceptions; 375 unsigned int priority; 376 377 #ifdef EXIT_DEBUG 378 if (vcpu->arch.pending_exceptions) 379 printk(KERN_EMERG "KVM: Check pending: %lx\n", vcpu->arch.pending_exceptions); 380 #endif 381 priority = __ffs(*pending); 382 while (priority < BOOK3S_IRQPRIO_MAX) { 383 if (kvmppc_book3s_irqprio_deliver(vcpu, priority) && 384 clear_irqprio(vcpu, priority)) { 385 clear_bit(priority, &vcpu->arch.pending_exceptions); 386 break; 387 } 388 389 priority = find_next_bit(pending, 390 BITS_PER_BYTE * sizeof(*pending), 391 priority + 1); 392 } 393 394 /* Tell the guest about our interrupt status */ 395 if (*pending) 396 vcpu->arch.shared->int_pending = 1; 397 else if (old_pending) 398 vcpu->arch.shared->int_pending = 0; 399 } 400 401 void kvmppc_set_pvr(struct kvm_vcpu *vcpu, u32 pvr) 402 { 403 u32 host_pvr; 404 405 vcpu->arch.hflags &= ~BOOK3S_HFLAG_SLB; 406 vcpu->arch.pvr = pvr; 407 #ifdef CONFIG_PPC_BOOK3S_64 408 if ((pvr >= 0x330000) && (pvr < 0x70330000)) { 409 kvmppc_mmu_book3s_64_init(vcpu); 410 to_book3s(vcpu)->hior = 0xfff00000; 411 to_book3s(vcpu)->msr_mask = 0xffffffffffffffffULL; 412 } else 413 #endif 414 { 415 kvmppc_mmu_book3s_32_init(vcpu); 416 to_book3s(vcpu)->hior = 0; 417 to_book3s(vcpu)->msr_mask = 0xffffffffULL; 418 } 419 420 /* If we are in hypervisor level on 970, we can tell the CPU to 421 * treat DCBZ as 32 bytes store */ 422 vcpu->arch.hflags &= ~BOOK3S_HFLAG_DCBZ32; 423 if (vcpu->arch.mmu.is_dcbz32(vcpu) && (mfmsr() & MSR_HV) && 424 !strcmp(cur_cpu_spec->platform, "ppc970")) 425 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32; 426 427 /* Cell performs badly if MSR_FEx are set. So let's hope nobody 428 really needs them in a VM on Cell and force disable them. */ 429 if (!strcmp(cur_cpu_spec->platform, "ppc-cell-be")) 430 to_book3s(vcpu)->msr_mask &= ~(MSR_FE0 | MSR_FE1); 431 432 #ifdef CONFIG_PPC_BOOK3S_32 433 /* 32 bit Book3S always has 32 byte dcbz */ 434 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32; 435 #endif 436 437 /* On some CPUs we can execute paired single operations natively */ 438 asm ( "mfpvr %0" : "=r"(host_pvr)); 439 switch (host_pvr) { 440 case 0x00080200: /* lonestar 2.0 */ 441 case 0x00088202: /* lonestar 2.2 */ 442 case 0x70000100: /* gekko 1.0 */ 443 case 0x00080100: /* gekko 2.0 */ 444 case 0x00083203: /* gekko 2.3a */ 445 case 0x00083213: /* gekko 2.3b */ 446 case 0x00083204: /* gekko 2.4 */ 447 case 0x00083214: /* gekko 2.4e (8SE) - retail HW2 */ 448 case 0x00087200: /* broadway */ 449 vcpu->arch.hflags |= BOOK3S_HFLAG_NATIVE_PS; 450 /* Enable HID2.PSE - in case we need it later */ 451 mtspr(SPRN_HID2_GEKKO, mfspr(SPRN_HID2_GEKKO) | (1 << 29)); 452 } 453 } 454 455 pfn_t kvmppc_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn) 456 { 457 ulong mp_pa = vcpu->arch.magic_page_pa; 458 459 /* Magic page override */ 460 if (unlikely(mp_pa) && 461 unlikely(((gfn << PAGE_SHIFT) & KVM_PAM) == 462 ((mp_pa & PAGE_MASK) & KVM_PAM))) { 463 ulong shared_page = ((ulong)vcpu->arch.shared) & PAGE_MASK; 464 pfn_t pfn; 465 466 pfn = (pfn_t)virt_to_phys((void*)shared_page) >> PAGE_SHIFT; 467 get_page(pfn_to_page(pfn)); 468 return pfn; 469 } 470 471 return gfn_to_pfn(vcpu->kvm, gfn); 472 } 473 474 /* Book3s_32 CPUs always have 32 bytes cache line size, which Linux assumes. To 475 * make Book3s_32 Linux work on Book3s_64, we have to make sure we trap dcbz to 476 * emulate 32 bytes dcbz length. 477 * 478 * The Book3s_64 inventors also realized this case and implemented a special bit 479 * in the HID5 register, which is a hypervisor ressource. Thus we can't use it. 480 * 481 * My approach here is to patch the dcbz instruction on executing pages. 482 */ 483 static void kvmppc_patch_dcbz(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte) 484 { 485 struct page *hpage; 486 u64 hpage_offset; 487 u32 *page; 488 int i; 489 490 hpage = gfn_to_page(vcpu->kvm, pte->raddr >> PAGE_SHIFT); 491 if (is_error_page(hpage)) { 492 kvm_release_page_clean(hpage); 493 return; 494 } 495 496 hpage_offset = pte->raddr & ~PAGE_MASK; 497 hpage_offset &= ~0xFFFULL; 498 hpage_offset /= 4; 499 500 get_page(hpage); 501 page = kmap_atomic(hpage, KM_USER0); 502 503 /* patch dcbz into reserved instruction, so we trap */ 504 for (i=hpage_offset; i < hpage_offset + (HW_PAGE_SIZE / 4); i++) 505 if ((page[i] & 0xff0007ff) == INS_DCBZ) 506 page[i] &= 0xfffffff7; 507 508 kunmap_atomic(page, KM_USER0); 509 put_page(hpage); 510 } 511 512 static int kvmppc_xlate(struct kvm_vcpu *vcpu, ulong eaddr, bool data, 513 struct kvmppc_pte *pte) 514 { 515 int relocated = (vcpu->arch.shared->msr & (data ? MSR_DR : MSR_IR)); 516 int r; 517 518 if (relocated) { 519 r = vcpu->arch.mmu.xlate(vcpu, eaddr, pte, data); 520 } else { 521 pte->eaddr = eaddr; 522 pte->raddr = eaddr & KVM_PAM; 523 pte->vpage = VSID_REAL | eaddr >> 12; 524 pte->may_read = true; 525 pte->may_write = true; 526 pte->may_execute = true; 527 r = 0; 528 } 529 530 return r; 531 } 532 533 static hva_t kvmppc_bad_hva(void) 534 { 535 return PAGE_OFFSET; 536 } 537 538 static hva_t kvmppc_pte_to_hva(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte, 539 bool read) 540 { 541 hva_t hpage; 542 543 if (read && !pte->may_read) 544 goto err; 545 546 if (!read && !pte->may_write) 547 goto err; 548 549 hpage = gfn_to_hva(vcpu->kvm, pte->raddr >> PAGE_SHIFT); 550 if (kvm_is_error_hva(hpage)) 551 goto err; 552 553 return hpage | (pte->raddr & ~PAGE_MASK); 554 err: 555 return kvmppc_bad_hva(); 556 } 557 558 int kvmppc_st(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, 559 bool data) 560 { 561 struct kvmppc_pte pte; 562 563 vcpu->stat.st++; 564 565 if (kvmppc_xlate(vcpu, *eaddr, data, &pte)) 566 return -ENOENT; 567 568 *eaddr = pte.raddr; 569 570 if (!pte.may_write) 571 return -EPERM; 572 573 if (kvm_write_guest(vcpu->kvm, pte.raddr, ptr, size)) 574 return EMULATE_DO_MMIO; 575 576 return EMULATE_DONE; 577 } 578 579 int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, 580 bool data) 581 { 582 struct kvmppc_pte pte; 583 hva_t hva = *eaddr; 584 585 vcpu->stat.ld++; 586 587 if (kvmppc_xlate(vcpu, *eaddr, data, &pte)) 588 goto nopte; 589 590 *eaddr = pte.raddr; 591 592 hva = kvmppc_pte_to_hva(vcpu, &pte, true); 593 if (kvm_is_error_hva(hva)) 594 goto mmio; 595 596 if (copy_from_user(ptr, (void __user *)hva, size)) { 597 printk(KERN_INFO "kvmppc_ld at 0x%lx failed\n", hva); 598 goto mmio; 599 } 600 601 return EMULATE_DONE; 602 603 nopte: 604 return -ENOENT; 605 mmio: 606 return EMULATE_DO_MMIO; 607 } 608 609 static int kvmppc_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 610 { 611 ulong mp_pa = vcpu->arch.magic_page_pa; 612 613 if (unlikely(mp_pa) && 614 unlikely((mp_pa & KVM_PAM) >> PAGE_SHIFT == gfn)) { 615 return 1; 616 } 617 618 return kvm_is_visible_gfn(vcpu->kvm, gfn); 619 } 620 621 int kvmppc_handle_pagefault(struct kvm_run *run, struct kvm_vcpu *vcpu, 622 ulong eaddr, int vec) 623 { 624 bool data = (vec == BOOK3S_INTERRUPT_DATA_STORAGE); 625 int r = RESUME_GUEST; 626 int relocated; 627 int page_found = 0; 628 struct kvmppc_pte pte; 629 bool is_mmio = false; 630 bool dr = (vcpu->arch.shared->msr & MSR_DR) ? true : false; 631 bool ir = (vcpu->arch.shared->msr & MSR_IR) ? true : false; 632 u64 vsid; 633 634 relocated = data ? dr : ir; 635 636 /* Resolve real address if translation turned on */ 637 if (relocated) { 638 page_found = vcpu->arch.mmu.xlate(vcpu, eaddr, &pte, data); 639 } else { 640 pte.may_execute = true; 641 pte.may_read = true; 642 pte.may_write = true; 643 pte.raddr = eaddr & KVM_PAM; 644 pte.eaddr = eaddr; 645 pte.vpage = eaddr >> 12; 646 } 647 648 switch (vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) { 649 case 0: 650 pte.vpage |= ((u64)VSID_REAL << (SID_SHIFT - 12)); 651 break; 652 case MSR_DR: 653 case MSR_IR: 654 vcpu->arch.mmu.esid_to_vsid(vcpu, eaddr >> SID_SHIFT, &vsid); 655 656 if ((vcpu->arch.shared->msr & (MSR_DR|MSR_IR)) == MSR_DR) 657 pte.vpage |= ((u64)VSID_REAL_DR << (SID_SHIFT - 12)); 658 else 659 pte.vpage |= ((u64)VSID_REAL_IR << (SID_SHIFT - 12)); 660 pte.vpage |= vsid; 661 662 if (vsid == -1) 663 page_found = -EINVAL; 664 break; 665 } 666 667 if (vcpu->arch.mmu.is_dcbz32(vcpu) && 668 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) { 669 /* 670 * If we do the dcbz hack, we have to NX on every execution, 671 * so we can patch the executing code. This renders our guest 672 * NX-less. 673 */ 674 pte.may_execute = !data; 675 } 676 677 if (page_found == -ENOENT) { 678 /* Page not found in guest PTE entries */ 679 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu); 680 vcpu->arch.shared->dsisr = to_svcpu(vcpu)->fault_dsisr; 681 vcpu->arch.shared->msr |= 682 (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL); 683 kvmppc_book3s_queue_irqprio(vcpu, vec); 684 } else if (page_found == -EPERM) { 685 /* Storage protection */ 686 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu); 687 vcpu->arch.shared->dsisr = 688 to_svcpu(vcpu)->fault_dsisr & ~DSISR_NOHPTE; 689 vcpu->arch.shared->dsisr |= DSISR_PROTFAULT; 690 vcpu->arch.shared->msr |= 691 (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL); 692 kvmppc_book3s_queue_irqprio(vcpu, vec); 693 } else if (page_found == -EINVAL) { 694 /* Page not found in guest SLB */ 695 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu); 696 kvmppc_book3s_queue_irqprio(vcpu, vec + 0x80); 697 } else if (!is_mmio && 698 kvmppc_visible_gfn(vcpu, pte.raddr >> PAGE_SHIFT)) { 699 /* The guest's PTE is not mapped yet. Map on the host */ 700 kvmppc_mmu_map_page(vcpu, &pte); 701 if (data) 702 vcpu->stat.sp_storage++; 703 else if (vcpu->arch.mmu.is_dcbz32(vcpu) && 704 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) 705 kvmppc_patch_dcbz(vcpu, &pte); 706 } else { 707 /* MMIO */ 708 vcpu->stat.mmio_exits++; 709 vcpu->arch.paddr_accessed = pte.raddr; 710 r = kvmppc_emulate_mmio(run, vcpu); 711 if ( r == RESUME_HOST_NV ) 712 r = RESUME_HOST; 713 } 714 715 return r; 716 } 717 718 static inline int get_fpr_index(int i) 719 { 720 #ifdef CONFIG_VSX 721 i *= 2; 722 #endif 723 return i; 724 } 725 726 /* Give up external provider (FPU, Altivec, VSX) */ 727 void kvmppc_giveup_ext(struct kvm_vcpu *vcpu, ulong msr) 728 { 729 struct thread_struct *t = ¤t->thread; 730 u64 *vcpu_fpr = vcpu->arch.fpr; 731 #ifdef CONFIG_VSX 732 u64 *vcpu_vsx = vcpu->arch.vsr; 733 #endif 734 u64 *thread_fpr = (u64*)t->fpr; 735 int i; 736 737 if (!(vcpu->arch.guest_owned_ext & msr)) 738 return; 739 740 #ifdef DEBUG_EXT 741 printk(KERN_INFO "Giving up ext 0x%lx\n", msr); 742 #endif 743 744 switch (msr) { 745 case MSR_FP: 746 giveup_fpu(current); 747 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++) 748 vcpu_fpr[i] = thread_fpr[get_fpr_index(i)]; 749 750 vcpu->arch.fpscr = t->fpscr.val; 751 break; 752 case MSR_VEC: 753 #ifdef CONFIG_ALTIVEC 754 giveup_altivec(current); 755 memcpy(vcpu->arch.vr, t->vr, sizeof(vcpu->arch.vr)); 756 vcpu->arch.vscr = t->vscr; 757 #endif 758 break; 759 case MSR_VSX: 760 #ifdef CONFIG_VSX 761 __giveup_vsx(current); 762 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++) 763 vcpu_vsx[i] = thread_fpr[get_fpr_index(i) + 1]; 764 #endif 765 break; 766 default: 767 BUG(); 768 } 769 770 vcpu->arch.guest_owned_ext &= ~msr; 771 current->thread.regs->msr &= ~msr; 772 kvmppc_recalc_shadow_msr(vcpu); 773 } 774 775 static int kvmppc_read_inst(struct kvm_vcpu *vcpu) 776 { 777 ulong srr0 = kvmppc_get_pc(vcpu); 778 u32 last_inst = kvmppc_get_last_inst(vcpu); 779 int ret; 780 781 ret = kvmppc_ld(vcpu, &srr0, sizeof(u32), &last_inst, false); 782 if (ret == -ENOENT) { 783 ulong msr = vcpu->arch.shared->msr; 784 785 msr = kvmppc_set_field(msr, 33, 33, 1); 786 msr = kvmppc_set_field(msr, 34, 36, 0); 787 vcpu->arch.shared->msr = kvmppc_set_field(msr, 42, 47, 0); 788 kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_INST_STORAGE); 789 return EMULATE_AGAIN; 790 } 791 792 return EMULATE_DONE; 793 } 794 795 static int kvmppc_check_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr) 796 { 797 798 /* Need to do paired single emulation? */ 799 if (!(vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE)) 800 return EMULATE_DONE; 801 802 /* Read out the instruction */ 803 if (kvmppc_read_inst(vcpu) == EMULATE_DONE) 804 /* Need to emulate */ 805 return EMULATE_FAIL; 806 807 return EMULATE_AGAIN; 808 } 809 810 /* Handle external providers (FPU, Altivec, VSX) */ 811 static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr, 812 ulong msr) 813 { 814 struct thread_struct *t = ¤t->thread; 815 u64 *vcpu_fpr = vcpu->arch.fpr; 816 #ifdef CONFIG_VSX 817 u64 *vcpu_vsx = vcpu->arch.vsr; 818 #endif 819 u64 *thread_fpr = (u64*)t->fpr; 820 int i; 821 822 /* When we have paired singles, we emulate in software */ 823 if (vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE) 824 return RESUME_GUEST; 825 826 if (!(vcpu->arch.shared->msr & msr)) { 827 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 828 return RESUME_GUEST; 829 } 830 831 /* We already own the ext */ 832 if (vcpu->arch.guest_owned_ext & msr) { 833 return RESUME_GUEST; 834 } 835 836 #ifdef DEBUG_EXT 837 printk(KERN_INFO "Loading up ext 0x%lx\n", msr); 838 #endif 839 840 current->thread.regs->msr |= msr; 841 842 switch (msr) { 843 case MSR_FP: 844 for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++) 845 thread_fpr[get_fpr_index(i)] = vcpu_fpr[i]; 846 847 t->fpscr.val = vcpu->arch.fpscr; 848 t->fpexc_mode = 0; 849 kvmppc_load_up_fpu(); 850 break; 851 case MSR_VEC: 852 #ifdef CONFIG_ALTIVEC 853 memcpy(t->vr, vcpu->arch.vr, sizeof(vcpu->arch.vr)); 854 t->vscr = vcpu->arch.vscr; 855 t->vrsave = -1; 856 kvmppc_load_up_altivec(); 857 #endif 858 break; 859 case MSR_VSX: 860 #ifdef CONFIG_VSX 861 for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++) 862 thread_fpr[get_fpr_index(i) + 1] = vcpu_vsx[i]; 863 kvmppc_load_up_vsx(); 864 #endif 865 break; 866 default: 867 BUG(); 868 } 869 870 vcpu->arch.guest_owned_ext |= msr; 871 872 kvmppc_recalc_shadow_msr(vcpu); 873 874 return RESUME_GUEST; 875 } 876 877 int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu, 878 unsigned int exit_nr) 879 { 880 int r = RESUME_HOST; 881 882 vcpu->stat.sum_exits++; 883 884 run->exit_reason = KVM_EXIT_UNKNOWN; 885 run->ready_for_interrupt_injection = 1; 886 887 trace_kvm_book3s_exit(exit_nr, vcpu); 888 kvm_resched(vcpu); 889 switch (exit_nr) { 890 case BOOK3S_INTERRUPT_INST_STORAGE: 891 vcpu->stat.pf_instruc++; 892 893 #ifdef CONFIG_PPC_BOOK3S_32 894 /* We set segments as unused segments when invalidating them. So 895 * treat the respective fault as segment fault. */ 896 if (to_svcpu(vcpu)->sr[kvmppc_get_pc(vcpu) >> SID_SHIFT] 897 == SR_INVALID) { 898 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)); 899 r = RESUME_GUEST; 900 break; 901 } 902 #endif 903 904 /* only care about PTEG not found errors, but leave NX alone */ 905 if (to_svcpu(vcpu)->shadow_srr1 & 0x40000000) { 906 r = kvmppc_handle_pagefault(run, vcpu, kvmppc_get_pc(vcpu), exit_nr); 907 vcpu->stat.sp_instruc++; 908 } else if (vcpu->arch.mmu.is_dcbz32(vcpu) && 909 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) { 910 /* 911 * XXX If we do the dcbz hack we use the NX bit to flush&patch the page, 912 * so we can't use the NX bit inside the guest. Let's cross our fingers, 913 * that no guest that needs the dcbz hack does NX. 914 */ 915 kvmppc_mmu_pte_flush(vcpu, kvmppc_get_pc(vcpu), ~0xFFFUL); 916 r = RESUME_GUEST; 917 } else { 918 vcpu->arch.shared->msr |= 919 to_svcpu(vcpu)->shadow_srr1 & 0x58000000; 920 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 921 r = RESUME_GUEST; 922 } 923 break; 924 case BOOK3S_INTERRUPT_DATA_STORAGE: 925 { 926 ulong dar = kvmppc_get_fault_dar(vcpu); 927 vcpu->stat.pf_storage++; 928 929 #ifdef CONFIG_PPC_BOOK3S_32 930 /* We set segments as unused segments when invalidating them. So 931 * treat the respective fault as segment fault. */ 932 if ((to_svcpu(vcpu)->sr[dar >> SID_SHIFT]) == SR_INVALID) { 933 kvmppc_mmu_map_segment(vcpu, dar); 934 r = RESUME_GUEST; 935 break; 936 } 937 #endif 938 939 /* The only case we need to handle is missing shadow PTEs */ 940 if (to_svcpu(vcpu)->fault_dsisr & DSISR_NOHPTE) { 941 r = kvmppc_handle_pagefault(run, vcpu, dar, exit_nr); 942 } else { 943 vcpu->arch.shared->dar = dar; 944 vcpu->arch.shared->dsisr = to_svcpu(vcpu)->fault_dsisr; 945 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 946 r = RESUME_GUEST; 947 } 948 break; 949 } 950 case BOOK3S_INTERRUPT_DATA_SEGMENT: 951 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_fault_dar(vcpu)) < 0) { 952 vcpu->arch.shared->dar = kvmppc_get_fault_dar(vcpu); 953 kvmppc_book3s_queue_irqprio(vcpu, 954 BOOK3S_INTERRUPT_DATA_SEGMENT); 955 } 956 r = RESUME_GUEST; 957 break; 958 case BOOK3S_INTERRUPT_INST_SEGMENT: 959 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)) < 0) { 960 kvmppc_book3s_queue_irqprio(vcpu, 961 BOOK3S_INTERRUPT_INST_SEGMENT); 962 } 963 r = RESUME_GUEST; 964 break; 965 /* We're good on these - the host merely wanted to get our attention */ 966 case BOOK3S_INTERRUPT_DECREMENTER: 967 vcpu->stat.dec_exits++; 968 r = RESUME_GUEST; 969 break; 970 case BOOK3S_INTERRUPT_EXTERNAL: 971 vcpu->stat.ext_intr_exits++; 972 r = RESUME_GUEST; 973 break; 974 case BOOK3S_INTERRUPT_PERFMON: 975 r = RESUME_GUEST; 976 break; 977 case BOOK3S_INTERRUPT_PROGRAM: 978 { 979 enum emulation_result er; 980 ulong flags; 981 982 program_interrupt: 983 flags = to_svcpu(vcpu)->shadow_srr1 & 0x1f0000ull; 984 985 if (vcpu->arch.shared->msr & MSR_PR) { 986 #ifdef EXIT_DEBUG 987 printk(KERN_INFO "Userspace triggered 0x700 exception at 0x%lx (0x%x)\n", kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu)); 988 #endif 989 if ((kvmppc_get_last_inst(vcpu) & 0xff0007ff) != 990 (INS_DCBZ & 0xfffffff7)) { 991 kvmppc_core_queue_program(vcpu, flags); 992 r = RESUME_GUEST; 993 break; 994 } 995 } 996 997 vcpu->stat.emulated_inst_exits++; 998 er = kvmppc_emulate_instruction(run, vcpu); 999 switch (er) { 1000 case EMULATE_DONE: 1001 r = RESUME_GUEST_NV; 1002 break; 1003 case EMULATE_AGAIN: 1004 r = RESUME_GUEST; 1005 break; 1006 case EMULATE_FAIL: 1007 printk(KERN_CRIT "%s: emulation at %lx failed (%08x)\n", 1008 __func__, kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu)); 1009 kvmppc_core_queue_program(vcpu, flags); 1010 r = RESUME_GUEST; 1011 break; 1012 case EMULATE_DO_MMIO: 1013 run->exit_reason = KVM_EXIT_MMIO; 1014 r = RESUME_HOST_NV; 1015 break; 1016 default: 1017 BUG(); 1018 } 1019 break; 1020 } 1021 case BOOK3S_INTERRUPT_SYSCALL: 1022 if (vcpu->arch.osi_enabled && 1023 (((u32)kvmppc_get_gpr(vcpu, 3)) == OSI_SC_MAGIC_R3) && 1024 (((u32)kvmppc_get_gpr(vcpu, 4)) == OSI_SC_MAGIC_R4)) { 1025 /* MOL hypercalls */ 1026 u64 *gprs = run->osi.gprs; 1027 int i; 1028 1029 run->exit_reason = KVM_EXIT_OSI; 1030 for (i = 0; i < 32; i++) 1031 gprs[i] = kvmppc_get_gpr(vcpu, i); 1032 vcpu->arch.osi_needed = 1; 1033 r = RESUME_HOST_NV; 1034 } else if (!(vcpu->arch.shared->msr & MSR_PR) && 1035 (((u32)kvmppc_get_gpr(vcpu, 0)) == KVM_SC_MAGIC_R0)) { 1036 /* KVM PV hypercalls */ 1037 kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu)); 1038 r = RESUME_GUEST; 1039 } else { 1040 /* Guest syscalls */ 1041 vcpu->stat.syscall_exits++; 1042 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 1043 r = RESUME_GUEST; 1044 } 1045 break; 1046 case BOOK3S_INTERRUPT_FP_UNAVAIL: 1047 case BOOK3S_INTERRUPT_ALTIVEC: 1048 case BOOK3S_INTERRUPT_VSX: 1049 { 1050 int ext_msr = 0; 1051 1052 switch (exit_nr) { 1053 case BOOK3S_INTERRUPT_FP_UNAVAIL: ext_msr = MSR_FP; break; 1054 case BOOK3S_INTERRUPT_ALTIVEC: ext_msr = MSR_VEC; break; 1055 case BOOK3S_INTERRUPT_VSX: ext_msr = MSR_VSX; break; 1056 } 1057 1058 switch (kvmppc_check_ext(vcpu, exit_nr)) { 1059 case EMULATE_DONE: 1060 /* everything ok - let's enable the ext */ 1061 r = kvmppc_handle_ext(vcpu, exit_nr, ext_msr); 1062 break; 1063 case EMULATE_FAIL: 1064 /* we need to emulate this instruction */ 1065 goto program_interrupt; 1066 break; 1067 default: 1068 /* nothing to worry about - go again */ 1069 break; 1070 } 1071 break; 1072 } 1073 case BOOK3S_INTERRUPT_ALIGNMENT: 1074 if (kvmppc_read_inst(vcpu) == EMULATE_DONE) { 1075 vcpu->arch.shared->dsisr = kvmppc_alignment_dsisr(vcpu, 1076 kvmppc_get_last_inst(vcpu)); 1077 vcpu->arch.shared->dar = kvmppc_alignment_dar(vcpu, 1078 kvmppc_get_last_inst(vcpu)); 1079 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 1080 } 1081 r = RESUME_GUEST; 1082 break; 1083 case BOOK3S_INTERRUPT_MACHINE_CHECK: 1084 case BOOK3S_INTERRUPT_TRACE: 1085 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 1086 r = RESUME_GUEST; 1087 break; 1088 default: 1089 /* Ugh - bork here! What did we get? */ 1090 printk(KERN_EMERG "exit_nr=0x%x | pc=0x%lx | msr=0x%lx\n", 1091 exit_nr, kvmppc_get_pc(vcpu), to_svcpu(vcpu)->shadow_srr1); 1092 r = RESUME_HOST; 1093 BUG(); 1094 break; 1095 } 1096 1097 1098 if (!(r & RESUME_HOST)) { 1099 /* To avoid clobbering exit_reason, only check for signals if 1100 * we aren't already exiting to userspace for some other 1101 * reason. */ 1102 if (signal_pending(current)) { 1103 #ifdef EXIT_DEBUG 1104 printk(KERN_EMERG "KVM: Going back to host\n"); 1105 #endif 1106 vcpu->stat.signal_exits++; 1107 run->exit_reason = KVM_EXIT_INTR; 1108 r = -EINTR; 1109 } else { 1110 /* In case an interrupt came in that was triggered 1111 * from userspace (like DEC), we need to check what 1112 * to inject now! */ 1113 kvmppc_core_deliver_interrupts(vcpu); 1114 } 1115 } 1116 1117 trace_kvm_book3s_reenter(r, vcpu); 1118 1119 return r; 1120 } 1121 1122 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) 1123 { 1124 return 0; 1125 } 1126 1127 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 1128 { 1129 int i; 1130 1131 regs->pc = kvmppc_get_pc(vcpu); 1132 regs->cr = kvmppc_get_cr(vcpu); 1133 regs->ctr = kvmppc_get_ctr(vcpu); 1134 regs->lr = kvmppc_get_lr(vcpu); 1135 regs->xer = kvmppc_get_xer(vcpu); 1136 regs->msr = vcpu->arch.shared->msr; 1137 regs->srr0 = vcpu->arch.shared->srr0; 1138 regs->srr1 = vcpu->arch.shared->srr1; 1139 regs->pid = vcpu->arch.pid; 1140 regs->sprg0 = vcpu->arch.shared->sprg0; 1141 regs->sprg1 = vcpu->arch.shared->sprg1; 1142 regs->sprg2 = vcpu->arch.shared->sprg2; 1143 regs->sprg3 = vcpu->arch.shared->sprg3; 1144 regs->sprg5 = vcpu->arch.sprg4; 1145 regs->sprg6 = vcpu->arch.sprg5; 1146 regs->sprg7 = vcpu->arch.sprg6; 1147 1148 for (i = 0; i < ARRAY_SIZE(regs->gpr); i++) 1149 regs->gpr[i] = kvmppc_get_gpr(vcpu, i); 1150 1151 return 0; 1152 } 1153 1154 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 1155 { 1156 int i; 1157 1158 kvmppc_set_pc(vcpu, regs->pc); 1159 kvmppc_set_cr(vcpu, regs->cr); 1160 kvmppc_set_ctr(vcpu, regs->ctr); 1161 kvmppc_set_lr(vcpu, regs->lr); 1162 kvmppc_set_xer(vcpu, regs->xer); 1163 kvmppc_set_msr(vcpu, regs->msr); 1164 vcpu->arch.shared->srr0 = regs->srr0; 1165 vcpu->arch.shared->srr1 = regs->srr1; 1166 vcpu->arch.shared->sprg0 = regs->sprg0; 1167 vcpu->arch.shared->sprg1 = regs->sprg1; 1168 vcpu->arch.shared->sprg2 = regs->sprg2; 1169 vcpu->arch.shared->sprg3 = regs->sprg3; 1170 vcpu->arch.sprg5 = regs->sprg4; 1171 vcpu->arch.sprg6 = regs->sprg5; 1172 vcpu->arch.sprg7 = regs->sprg6; 1173 1174 for (i = 0; i < ARRAY_SIZE(regs->gpr); i++) 1175 kvmppc_set_gpr(vcpu, i, regs->gpr[i]); 1176 1177 return 0; 1178 } 1179 1180 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 1181 struct kvm_sregs *sregs) 1182 { 1183 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu); 1184 int i; 1185 1186 sregs->pvr = vcpu->arch.pvr; 1187 1188 sregs->u.s.sdr1 = to_book3s(vcpu)->sdr1; 1189 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) { 1190 for (i = 0; i < 64; i++) { 1191 sregs->u.s.ppc64.slb[i].slbe = vcpu3s->slb[i].orige | i; 1192 sregs->u.s.ppc64.slb[i].slbv = vcpu3s->slb[i].origv; 1193 } 1194 } else { 1195 for (i = 0; i < 16; i++) 1196 sregs->u.s.ppc32.sr[i] = vcpu->arch.shared->sr[i]; 1197 1198 for (i = 0; i < 8; i++) { 1199 sregs->u.s.ppc32.ibat[i] = vcpu3s->ibat[i].raw; 1200 sregs->u.s.ppc32.dbat[i] = vcpu3s->dbat[i].raw; 1201 } 1202 } 1203 1204 return 0; 1205 } 1206 1207 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 1208 struct kvm_sregs *sregs) 1209 { 1210 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu); 1211 int i; 1212 1213 kvmppc_set_pvr(vcpu, sregs->pvr); 1214 1215 vcpu3s->sdr1 = sregs->u.s.sdr1; 1216 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) { 1217 for (i = 0; i < 64; i++) { 1218 vcpu->arch.mmu.slbmte(vcpu, sregs->u.s.ppc64.slb[i].slbv, 1219 sregs->u.s.ppc64.slb[i].slbe); 1220 } 1221 } else { 1222 for (i = 0; i < 16; i++) { 1223 vcpu->arch.mmu.mtsrin(vcpu, i, sregs->u.s.ppc32.sr[i]); 1224 } 1225 for (i = 0; i < 8; i++) { 1226 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), false, 1227 (u32)sregs->u.s.ppc32.ibat[i]); 1228 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), true, 1229 (u32)(sregs->u.s.ppc32.ibat[i] >> 32)); 1230 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), false, 1231 (u32)sregs->u.s.ppc32.dbat[i]); 1232 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), true, 1233 (u32)(sregs->u.s.ppc32.dbat[i] >> 32)); 1234 } 1235 } 1236 1237 /* Flush the MMU after messing with the segments */ 1238 kvmppc_mmu_pte_flush(vcpu, 0, 0); 1239 1240 return 0; 1241 } 1242 1243 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 1244 { 1245 return -ENOTSUPP; 1246 } 1247 1248 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 1249 { 1250 return -ENOTSUPP; 1251 } 1252 1253 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 1254 struct kvm_translation *tr) 1255 { 1256 return 0; 1257 } 1258 1259 /* 1260 * Get (and clear) the dirty memory log for a memory slot. 1261 */ 1262 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, 1263 struct kvm_dirty_log *log) 1264 { 1265 struct kvm_memory_slot *memslot; 1266 struct kvm_vcpu *vcpu; 1267 ulong ga, ga_end; 1268 int is_dirty = 0; 1269 int r; 1270 unsigned long n; 1271 1272 mutex_lock(&kvm->slots_lock); 1273 1274 r = kvm_get_dirty_log(kvm, log, &is_dirty); 1275 if (r) 1276 goto out; 1277 1278 /* If nothing is dirty, don't bother messing with page tables. */ 1279 if (is_dirty) { 1280 memslot = &kvm->memslots->memslots[log->slot]; 1281 1282 ga = memslot->base_gfn << PAGE_SHIFT; 1283 ga_end = ga + (memslot->npages << PAGE_SHIFT); 1284 1285 kvm_for_each_vcpu(n, vcpu, kvm) 1286 kvmppc_mmu_pte_pflush(vcpu, ga, ga_end); 1287 1288 n = kvm_dirty_bitmap_bytes(memslot); 1289 memset(memslot->dirty_bitmap, 0, n); 1290 } 1291 1292 r = 0; 1293 out: 1294 mutex_unlock(&kvm->slots_lock); 1295 return r; 1296 } 1297 1298 int kvmppc_core_check_processor_compat(void) 1299 { 1300 return 0; 1301 } 1302 1303 struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id) 1304 { 1305 struct kvmppc_vcpu_book3s *vcpu_book3s; 1306 struct kvm_vcpu *vcpu; 1307 int err = -ENOMEM; 1308 unsigned long p; 1309 1310 vcpu_book3s = vzalloc(sizeof(struct kvmppc_vcpu_book3s)); 1311 if (!vcpu_book3s) 1312 goto out; 1313 1314 vcpu_book3s->shadow_vcpu = (struct kvmppc_book3s_shadow_vcpu *) 1315 kzalloc(sizeof(*vcpu_book3s->shadow_vcpu), GFP_KERNEL); 1316 if (!vcpu_book3s->shadow_vcpu) 1317 goto free_vcpu; 1318 1319 vcpu = &vcpu_book3s->vcpu; 1320 err = kvm_vcpu_init(vcpu, kvm, id); 1321 if (err) 1322 goto free_shadow_vcpu; 1323 1324 p = __get_free_page(GFP_KERNEL|__GFP_ZERO); 1325 /* the real shared page fills the last 4k of our page */ 1326 vcpu->arch.shared = (void*)(p + PAGE_SIZE - 4096); 1327 if (!p) 1328 goto uninit_vcpu; 1329 1330 vcpu->arch.host_retip = kvm_return_point; 1331 vcpu->arch.host_msr = mfmsr(); 1332 #ifdef CONFIG_PPC_BOOK3S_64 1333 /* default to book3s_64 (970fx) */ 1334 vcpu->arch.pvr = 0x3C0301; 1335 #else 1336 /* default to book3s_32 (750) */ 1337 vcpu->arch.pvr = 0x84202; 1338 #endif 1339 kvmppc_set_pvr(vcpu, vcpu->arch.pvr); 1340 vcpu_book3s->slb_nr = 64; 1341 1342 /* remember where some real-mode handlers are */ 1343 vcpu->arch.trampoline_lowmem = kvmppc_trampoline_lowmem; 1344 vcpu->arch.trampoline_enter = kvmppc_trampoline_enter; 1345 vcpu->arch.highmem_handler = (ulong)kvmppc_handler_highmem; 1346 #ifdef CONFIG_PPC_BOOK3S_64 1347 vcpu->arch.rmcall = *(ulong*)kvmppc_rmcall; 1348 #else 1349 vcpu->arch.rmcall = (ulong)kvmppc_rmcall; 1350 #endif 1351 1352 vcpu->arch.shadow_msr = MSR_USER64; 1353 1354 err = kvmppc_mmu_init(vcpu); 1355 if (err < 0) 1356 goto uninit_vcpu; 1357 1358 return vcpu; 1359 1360 uninit_vcpu: 1361 kvm_vcpu_uninit(vcpu); 1362 free_shadow_vcpu: 1363 kfree(vcpu_book3s->shadow_vcpu); 1364 free_vcpu: 1365 vfree(vcpu_book3s); 1366 out: 1367 return ERR_PTR(err); 1368 } 1369 1370 void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu) 1371 { 1372 struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu); 1373 1374 free_page((unsigned long)vcpu->arch.shared & PAGE_MASK); 1375 kvm_vcpu_uninit(vcpu); 1376 kfree(vcpu_book3s->shadow_vcpu); 1377 vfree(vcpu_book3s); 1378 } 1379 1380 extern int __kvmppc_vcpu_entry(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu); 1381 int __kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) 1382 { 1383 int ret; 1384 double fpr[32][TS_FPRWIDTH]; 1385 unsigned int fpscr; 1386 int fpexc_mode; 1387 #ifdef CONFIG_ALTIVEC 1388 vector128 vr[32]; 1389 vector128 vscr; 1390 unsigned long uninitialized_var(vrsave); 1391 int used_vr; 1392 #endif 1393 #ifdef CONFIG_VSX 1394 int used_vsr; 1395 #endif 1396 ulong ext_msr; 1397 1398 /* No need to go into the guest when all we do is going out */ 1399 if (signal_pending(current)) { 1400 kvm_run->exit_reason = KVM_EXIT_INTR; 1401 return -EINTR; 1402 } 1403 1404 /* Save FPU state in stack */ 1405 if (current->thread.regs->msr & MSR_FP) 1406 giveup_fpu(current); 1407 memcpy(fpr, current->thread.fpr, sizeof(current->thread.fpr)); 1408 fpscr = current->thread.fpscr.val; 1409 fpexc_mode = current->thread.fpexc_mode; 1410 1411 #ifdef CONFIG_ALTIVEC 1412 /* Save Altivec state in stack */ 1413 used_vr = current->thread.used_vr; 1414 if (used_vr) { 1415 if (current->thread.regs->msr & MSR_VEC) 1416 giveup_altivec(current); 1417 memcpy(vr, current->thread.vr, sizeof(current->thread.vr)); 1418 vscr = current->thread.vscr; 1419 vrsave = current->thread.vrsave; 1420 } 1421 #endif 1422 1423 #ifdef CONFIG_VSX 1424 /* Save VSX state in stack */ 1425 used_vsr = current->thread.used_vsr; 1426 if (used_vsr && (current->thread.regs->msr & MSR_VSX)) 1427 __giveup_vsx(current); 1428 #endif 1429 1430 /* Remember the MSR with disabled extensions */ 1431 ext_msr = current->thread.regs->msr; 1432 1433 /* XXX we get called with irq disabled - change that! */ 1434 local_irq_enable(); 1435 1436 /* Preload FPU if it's enabled */ 1437 if (vcpu->arch.shared->msr & MSR_FP) 1438 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP); 1439 1440 ret = __kvmppc_vcpu_entry(kvm_run, vcpu); 1441 1442 local_irq_disable(); 1443 1444 current->thread.regs->msr = ext_msr; 1445 1446 /* Make sure we save the guest FPU/Altivec/VSX state */ 1447 kvmppc_giveup_ext(vcpu, MSR_FP); 1448 kvmppc_giveup_ext(vcpu, MSR_VEC); 1449 kvmppc_giveup_ext(vcpu, MSR_VSX); 1450 1451 /* Restore FPU state from stack */ 1452 memcpy(current->thread.fpr, fpr, sizeof(current->thread.fpr)); 1453 current->thread.fpscr.val = fpscr; 1454 current->thread.fpexc_mode = fpexc_mode; 1455 1456 #ifdef CONFIG_ALTIVEC 1457 /* Restore Altivec state from stack */ 1458 if (used_vr && current->thread.used_vr) { 1459 memcpy(current->thread.vr, vr, sizeof(current->thread.vr)); 1460 current->thread.vscr = vscr; 1461 current->thread.vrsave = vrsave; 1462 } 1463 current->thread.used_vr = used_vr; 1464 #endif 1465 1466 #ifdef CONFIG_VSX 1467 current->thread.used_vsr = used_vsr; 1468 #endif 1469 1470 return ret; 1471 } 1472 1473 static int kvmppc_book3s_init(void) 1474 { 1475 int r; 1476 1477 r = kvm_init(NULL, sizeof(struct kvmppc_vcpu_book3s), 0, 1478 THIS_MODULE); 1479 1480 if (r) 1481 return r; 1482 1483 r = kvmppc_mmu_hpte_sysinit(); 1484 1485 return r; 1486 } 1487 1488 static void kvmppc_book3s_exit(void) 1489 { 1490 kvmppc_mmu_hpte_sysexit(); 1491 kvm_exit(); 1492 } 1493 1494 module_init(kvmppc_book3s_init); 1495 module_exit(kvmppc_book3s_exit); 1496