1 /* 2 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 3 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved. 4 * 5 * Authors: 6 * Paul Mackerras <paulus@au1.ibm.com> 7 * Alexander Graf <agraf@suse.de> 8 * Kevin Wolf <mail@kevin-wolf.de> 9 * 10 * Description: KVM functions specific to running on Book 3S 11 * processors in hypervisor mode (specifically POWER7 and later). 12 * 13 * This file is derived from arch/powerpc/kvm/book3s.c, 14 * by Alexander Graf <agraf@suse.de>. 15 * 16 * This program is free software; you can redistribute it and/or modify 17 * it under the terms of the GNU General Public License, version 2, as 18 * published by the Free Software Foundation. 19 */ 20 21 #include <linux/kvm_host.h> 22 #include <linux/err.h> 23 #include <linux/slab.h> 24 #include <linux/preempt.h> 25 #include <linux/sched.h> 26 #include <linux/delay.h> 27 #include <linux/export.h> 28 #include <linux/fs.h> 29 #include <linux/anon_inodes.h> 30 #include <linux/cpumask.h> 31 #include <linux/spinlock.h> 32 #include <linux/page-flags.h> 33 #include <linux/srcu.h> 34 #include <linux/miscdevice.h> 35 #include <linux/debugfs.h> 36 37 #include <asm/reg.h> 38 #include <asm/cputable.h> 39 #include <asm/cacheflush.h> 40 #include <asm/tlbflush.h> 41 #include <asm/uaccess.h> 42 #include <asm/io.h> 43 #include <asm/kvm_ppc.h> 44 #include <asm/kvm_book3s.h> 45 #include <asm/mmu_context.h> 46 #include <asm/lppaca.h> 47 #include <asm/processor.h> 48 #include <asm/cputhreads.h> 49 #include <asm/page.h> 50 #include <asm/hvcall.h> 51 #include <asm/switch_to.h> 52 #include <asm/smp.h> 53 #include <asm/dbell.h> 54 #include <linux/gfp.h> 55 #include <linux/vmalloc.h> 56 #include <linux/highmem.h> 57 #include <linux/hugetlb.h> 58 #include <linux/module.h> 59 60 #include "book3s.h" 61 62 #define CREATE_TRACE_POINTS 63 #include "trace_hv.h" 64 65 /* #define EXIT_DEBUG */ 66 /* #define EXIT_DEBUG_SIMPLE */ 67 /* #define EXIT_DEBUG_INT */ 68 69 /* Used to indicate that a guest page fault needs to be handled */ 70 #define RESUME_PAGE_FAULT (RESUME_GUEST | RESUME_FLAG_ARCH1) 71 72 /* Used as a "null" value for timebase values */ 73 #define TB_NIL (~(u64)0) 74 75 static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1); 76 77 static int dynamic_mt_modes = 6; 78 module_param(dynamic_mt_modes, int, S_IRUGO | S_IWUSR); 79 MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)"); 80 static int target_smt_mode; 81 module_param(target_smt_mode, int, S_IRUGO | S_IWUSR); 82 MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)"); 83 84 static void kvmppc_end_cede(struct kvm_vcpu *vcpu); 85 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu); 86 87 static bool kvmppc_ipi_thread(int cpu) 88 { 89 /* On POWER8 for IPIs to threads in the same core, use msgsnd */ 90 if (cpu_has_feature(CPU_FTR_ARCH_207S)) { 91 preempt_disable(); 92 if (cpu_first_thread_sibling(cpu) == 93 cpu_first_thread_sibling(smp_processor_id())) { 94 unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER); 95 msg |= cpu_thread_in_core(cpu); 96 smp_mb(); 97 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg)); 98 preempt_enable(); 99 return true; 100 } 101 preempt_enable(); 102 } 103 104 #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP) 105 if (cpu >= 0 && cpu < nr_cpu_ids && paca[cpu].kvm_hstate.xics_phys) { 106 xics_wake_cpu(cpu); 107 return true; 108 } 109 #endif 110 111 return false; 112 } 113 114 static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) 115 { 116 int cpu; 117 wait_queue_head_t *wqp; 118 119 wqp = kvm_arch_vcpu_wq(vcpu); 120 if (waitqueue_active(wqp)) { 121 wake_up_interruptible(wqp); 122 ++vcpu->stat.halt_wakeup; 123 } 124 125 if (kvmppc_ipi_thread(vcpu->arch.thread_cpu)) 126 return; 127 128 /* CPU points to the first thread of the core */ 129 cpu = vcpu->cpu; 130 if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu)) 131 smp_send_reschedule(cpu); 132 } 133 134 /* 135 * We use the vcpu_load/put functions to measure stolen time. 136 * Stolen time is counted as time when either the vcpu is able to 137 * run as part of a virtual core, but the task running the vcore 138 * is preempted or sleeping, or when the vcpu needs something done 139 * in the kernel by the task running the vcpu, but that task is 140 * preempted or sleeping. Those two things have to be counted 141 * separately, since one of the vcpu tasks will take on the job 142 * of running the core, and the other vcpu tasks in the vcore will 143 * sleep waiting for it to do that, but that sleep shouldn't count 144 * as stolen time. 145 * 146 * Hence we accumulate stolen time when the vcpu can run as part of 147 * a vcore using vc->stolen_tb, and the stolen time when the vcpu 148 * needs its task to do other things in the kernel (for example, 149 * service a page fault) in busy_stolen. We don't accumulate 150 * stolen time for a vcore when it is inactive, or for a vcpu 151 * when it is in state RUNNING or NOTREADY. NOTREADY is a bit of 152 * a misnomer; it means that the vcpu task is not executing in 153 * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in 154 * the kernel. We don't have any way of dividing up that time 155 * between time that the vcpu is genuinely stopped, time that 156 * the task is actively working on behalf of the vcpu, and time 157 * that the task is preempted, so we don't count any of it as 158 * stolen. 159 * 160 * Updates to busy_stolen are protected by arch.tbacct_lock; 161 * updates to vc->stolen_tb are protected by the vcore->stoltb_lock 162 * lock. The stolen times are measured in units of timebase ticks. 163 * (Note that the != TB_NIL checks below are purely defensive; 164 * they should never fail.) 165 */ 166 167 static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc) 168 { 169 unsigned long flags; 170 171 spin_lock_irqsave(&vc->stoltb_lock, flags); 172 vc->preempt_tb = mftb(); 173 spin_unlock_irqrestore(&vc->stoltb_lock, flags); 174 } 175 176 static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc) 177 { 178 unsigned long flags; 179 180 spin_lock_irqsave(&vc->stoltb_lock, flags); 181 if (vc->preempt_tb != TB_NIL) { 182 vc->stolen_tb += mftb() - vc->preempt_tb; 183 vc->preempt_tb = TB_NIL; 184 } 185 spin_unlock_irqrestore(&vc->stoltb_lock, flags); 186 } 187 188 static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu) 189 { 190 struct kvmppc_vcore *vc = vcpu->arch.vcore; 191 unsigned long flags; 192 193 /* 194 * We can test vc->runner without taking the vcore lock, 195 * because only this task ever sets vc->runner to this 196 * vcpu, and once it is set to this vcpu, only this task 197 * ever sets it to NULL. 198 */ 199 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING) 200 kvmppc_core_end_stolen(vc); 201 202 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); 203 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST && 204 vcpu->arch.busy_preempt != TB_NIL) { 205 vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt; 206 vcpu->arch.busy_preempt = TB_NIL; 207 } 208 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); 209 } 210 211 static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu) 212 { 213 struct kvmppc_vcore *vc = vcpu->arch.vcore; 214 unsigned long flags; 215 216 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING) 217 kvmppc_core_start_stolen(vc); 218 219 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); 220 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST) 221 vcpu->arch.busy_preempt = mftb(); 222 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); 223 } 224 225 static void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr) 226 { 227 /* 228 * Check for illegal transactional state bit combination 229 * and if we find it, force the TS field to a safe state. 230 */ 231 if ((msr & MSR_TS_MASK) == MSR_TS_MASK) 232 msr &= ~MSR_TS_MASK; 233 vcpu->arch.shregs.msr = msr; 234 kvmppc_end_cede(vcpu); 235 } 236 237 static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr) 238 { 239 vcpu->arch.pvr = pvr; 240 } 241 242 static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) 243 { 244 unsigned long pcr = 0; 245 struct kvmppc_vcore *vc = vcpu->arch.vcore; 246 247 if (arch_compat) { 248 switch (arch_compat) { 249 case PVR_ARCH_205: 250 /* 251 * If an arch bit is set in PCR, all the defined 252 * higher-order arch bits also have to be set. 253 */ 254 pcr = PCR_ARCH_206 | PCR_ARCH_205; 255 break; 256 case PVR_ARCH_206: 257 case PVR_ARCH_206p: 258 pcr = PCR_ARCH_206; 259 break; 260 case PVR_ARCH_207: 261 break; 262 default: 263 return -EINVAL; 264 } 265 266 if (!cpu_has_feature(CPU_FTR_ARCH_207S)) { 267 /* POWER7 can't emulate POWER8 */ 268 if (!(pcr & PCR_ARCH_206)) 269 return -EINVAL; 270 pcr &= ~PCR_ARCH_206; 271 } 272 } 273 274 spin_lock(&vc->lock); 275 vc->arch_compat = arch_compat; 276 vc->pcr = pcr; 277 spin_unlock(&vc->lock); 278 279 return 0; 280 } 281 282 static void kvmppc_dump_regs(struct kvm_vcpu *vcpu) 283 { 284 int r; 285 286 pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id); 287 pr_err("pc = %.16lx msr = %.16llx trap = %x\n", 288 vcpu->arch.pc, vcpu->arch.shregs.msr, vcpu->arch.trap); 289 for (r = 0; r < 16; ++r) 290 pr_err("r%2d = %.16lx r%d = %.16lx\n", 291 r, kvmppc_get_gpr(vcpu, r), 292 r+16, kvmppc_get_gpr(vcpu, r+16)); 293 pr_err("ctr = %.16lx lr = %.16lx\n", 294 vcpu->arch.ctr, vcpu->arch.lr); 295 pr_err("srr0 = %.16llx srr1 = %.16llx\n", 296 vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1); 297 pr_err("sprg0 = %.16llx sprg1 = %.16llx\n", 298 vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1); 299 pr_err("sprg2 = %.16llx sprg3 = %.16llx\n", 300 vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3); 301 pr_err("cr = %.8x xer = %.16lx dsisr = %.8x\n", 302 vcpu->arch.cr, vcpu->arch.xer, vcpu->arch.shregs.dsisr); 303 pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar); 304 pr_err("fault dar = %.16lx dsisr = %.8x\n", 305 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr); 306 pr_err("SLB (%d entries):\n", vcpu->arch.slb_max); 307 for (r = 0; r < vcpu->arch.slb_max; ++r) 308 pr_err(" ESID = %.16llx VSID = %.16llx\n", 309 vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv); 310 pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n", 311 vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1, 312 vcpu->arch.last_inst); 313 } 314 315 static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id) 316 { 317 int r; 318 struct kvm_vcpu *v, *ret = NULL; 319 320 mutex_lock(&kvm->lock); 321 kvm_for_each_vcpu(r, v, kvm) { 322 if (v->vcpu_id == id) { 323 ret = v; 324 break; 325 } 326 } 327 mutex_unlock(&kvm->lock); 328 return ret; 329 } 330 331 static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa) 332 { 333 vpa->__old_status |= LPPACA_OLD_SHARED_PROC; 334 vpa->yield_count = cpu_to_be32(1); 335 } 336 337 static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v, 338 unsigned long addr, unsigned long len) 339 { 340 /* check address is cacheline aligned */ 341 if (addr & (L1_CACHE_BYTES - 1)) 342 return -EINVAL; 343 spin_lock(&vcpu->arch.vpa_update_lock); 344 if (v->next_gpa != addr || v->len != len) { 345 v->next_gpa = addr; 346 v->len = addr ? len : 0; 347 v->update_pending = 1; 348 } 349 spin_unlock(&vcpu->arch.vpa_update_lock); 350 return 0; 351 } 352 353 /* Length for a per-processor buffer is passed in at offset 4 in the buffer */ 354 struct reg_vpa { 355 u32 dummy; 356 union { 357 __be16 hword; 358 __be32 word; 359 } length; 360 }; 361 362 static int vpa_is_registered(struct kvmppc_vpa *vpap) 363 { 364 if (vpap->update_pending) 365 return vpap->next_gpa != 0; 366 return vpap->pinned_addr != NULL; 367 } 368 369 static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu, 370 unsigned long flags, 371 unsigned long vcpuid, unsigned long vpa) 372 { 373 struct kvm *kvm = vcpu->kvm; 374 unsigned long len, nb; 375 void *va; 376 struct kvm_vcpu *tvcpu; 377 int err; 378 int subfunc; 379 struct kvmppc_vpa *vpap; 380 381 tvcpu = kvmppc_find_vcpu(kvm, vcpuid); 382 if (!tvcpu) 383 return H_PARAMETER; 384 385 subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK; 386 if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL || 387 subfunc == H_VPA_REG_SLB) { 388 /* Registering new area - address must be cache-line aligned */ 389 if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa) 390 return H_PARAMETER; 391 392 /* convert logical addr to kernel addr and read length */ 393 va = kvmppc_pin_guest_page(kvm, vpa, &nb); 394 if (va == NULL) 395 return H_PARAMETER; 396 if (subfunc == H_VPA_REG_VPA) 397 len = be16_to_cpu(((struct reg_vpa *)va)->length.hword); 398 else 399 len = be32_to_cpu(((struct reg_vpa *)va)->length.word); 400 kvmppc_unpin_guest_page(kvm, va, vpa, false); 401 402 /* Check length */ 403 if (len > nb || len < sizeof(struct reg_vpa)) 404 return H_PARAMETER; 405 } else { 406 vpa = 0; 407 len = 0; 408 } 409 410 err = H_PARAMETER; 411 vpap = NULL; 412 spin_lock(&tvcpu->arch.vpa_update_lock); 413 414 switch (subfunc) { 415 case H_VPA_REG_VPA: /* register VPA */ 416 if (len < sizeof(struct lppaca)) 417 break; 418 vpap = &tvcpu->arch.vpa; 419 err = 0; 420 break; 421 422 case H_VPA_REG_DTL: /* register DTL */ 423 if (len < sizeof(struct dtl_entry)) 424 break; 425 len -= len % sizeof(struct dtl_entry); 426 427 /* Check that they have previously registered a VPA */ 428 err = H_RESOURCE; 429 if (!vpa_is_registered(&tvcpu->arch.vpa)) 430 break; 431 432 vpap = &tvcpu->arch.dtl; 433 err = 0; 434 break; 435 436 case H_VPA_REG_SLB: /* register SLB shadow buffer */ 437 /* Check that they have previously registered a VPA */ 438 err = H_RESOURCE; 439 if (!vpa_is_registered(&tvcpu->arch.vpa)) 440 break; 441 442 vpap = &tvcpu->arch.slb_shadow; 443 err = 0; 444 break; 445 446 case H_VPA_DEREG_VPA: /* deregister VPA */ 447 /* Check they don't still have a DTL or SLB buf registered */ 448 err = H_RESOURCE; 449 if (vpa_is_registered(&tvcpu->arch.dtl) || 450 vpa_is_registered(&tvcpu->arch.slb_shadow)) 451 break; 452 453 vpap = &tvcpu->arch.vpa; 454 err = 0; 455 break; 456 457 case H_VPA_DEREG_DTL: /* deregister DTL */ 458 vpap = &tvcpu->arch.dtl; 459 err = 0; 460 break; 461 462 case H_VPA_DEREG_SLB: /* deregister SLB shadow buffer */ 463 vpap = &tvcpu->arch.slb_shadow; 464 err = 0; 465 break; 466 } 467 468 if (vpap) { 469 vpap->next_gpa = vpa; 470 vpap->len = len; 471 vpap->update_pending = 1; 472 } 473 474 spin_unlock(&tvcpu->arch.vpa_update_lock); 475 476 return err; 477 } 478 479 static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap) 480 { 481 struct kvm *kvm = vcpu->kvm; 482 void *va; 483 unsigned long nb; 484 unsigned long gpa; 485 486 /* 487 * We need to pin the page pointed to by vpap->next_gpa, 488 * but we can't call kvmppc_pin_guest_page under the lock 489 * as it does get_user_pages() and down_read(). So we 490 * have to drop the lock, pin the page, then get the lock 491 * again and check that a new area didn't get registered 492 * in the meantime. 493 */ 494 for (;;) { 495 gpa = vpap->next_gpa; 496 spin_unlock(&vcpu->arch.vpa_update_lock); 497 va = NULL; 498 nb = 0; 499 if (gpa) 500 va = kvmppc_pin_guest_page(kvm, gpa, &nb); 501 spin_lock(&vcpu->arch.vpa_update_lock); 502 if (gpa == vpap->next_gpa) 503 break; 504 /* sigh... unpin that one and try again */ 505 if (va) 506 kvmppc_unpin_guest_page(kvm, va, gpa, false); 507 } 508 509 vpap->update_pending = 0; 510 if (va && nb < vpap->len) { 511 /* 512 * If it's now too short, it must be that userspace 513 * has changed the mappings underlying guest memory, 514 * so unregister the region. 515 */ 516 kvmppc_unpin_guest_page(kvm, va, gpa, false); 517 va = NULL; 518 } 519 if (vpap->pinned_addr) 520 kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa, 521 vpap->dirty); 522 vpap->gpa = gpa; 523 vpap->pinned_addr = va; 524 vpap->dirty = false; 525 if (va) 526 vpap->pinned_end = va + vpap->len; 527 } 528 529 static void kvmppc_update_vpas(struct kvm_vcpu *vcpu) 530 { 531 if (!(vcpu->arch.vpa.update_pending || 532 vcpu->arch.slb_shadow.update_pending || 533 vcpu->arch.dtl.update_pending)) 534 return; 535 536 spin_lock(&vcpu->arch.vpa_update_lock); 537 if (vcpu->arch.vpa.update_pending) { 538 kvmppc_update_vpa(vcpu, &vcpu->arch.vpa); 539 if (vcpu->arch.vpa.pinned_addr) 540 init_vpa(vcpu, vcpu->arch.vpa.pinned_addr); 541 } 542 if (vcpu->arch.dtl.update_pending) { 543 kvmppc_update_vpa(vcpu, &vcpu->arch.dtl); 544 vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr; 545 vcpu->arch.dtl_index = 0; 546 } 547 if (vcpu->arch.slb_shadow.update_pending) 548 kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow); 549 spin_unlock(&vcpu->arch.vpa_update_lock); 550 } 551 552 /* 553 * Return the accumulated stolen time for the vcore up until `now'. 554 * The caller should hold the vcore lock. 555 */ 556 static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now) 557 { 558 u64 p; 559 unsigned long flags; 560 561 spin_lock_irqsave(&vc->stoltb_lock, flags); 562 p = vc->stolen_tb; 563 if (vc->vcore_state != VCORE_INACTIVE && 564 vc->preempt_tb != TB_NIL) 565 p += now - vc->preempt_tb; 566 spin_unlock_irqrestore(&vc->stoltb_lock, flags); 567 return p; 568 } 569 570 static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, 571 struct kvmppc_vcore *vc) 572 { 573 struct dtl_entry *dt; 574 struct lppaca *vpa; 575 unsigned long stolen; 576 unsigned long core_stolen; 577 u64 now; 578 579 dt = vcpu->arch.dtl_ptr; 580 vpa = vcpu->arch.vpa.pinned_addr; 581 now = mftb(); 582 core_stolen = vcore_stolen_time(vc, now); 583 stolen = core_stolen - vcpu->arch.stolen_logged; 584 vcpu->arch.stolen_logged = core_stolen; 585 spin_lock_irq(&vcpu->arch.tbacct_lock); 586 stolen += vcpu->arch.busy_stolen; 587 vcpu->arch.busy_stolen = 0; 588 spin_unlock_irq(&vcpu->arch.tbacct_lock); 589 if (!dt || !vpa) 590 return; 591 memset(dt, 0, sizeof(struct dtl_entry)); 592 dt->dispatch_reason = 7; 593 dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid); 594 dt->timebase = cpu_to_be64(now + vc->tb_offset); 595 dt->enqueue_to_dispatch_time = cpu_to_be32(stolen); 596 dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu)); 597 dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr); 598 ++dt; 599 if (dt == vcpu->arch.dtl.pinned_end) 600 dt = vcpu->arch.dtl.pinned_addr; 601 vcpu->arch.dtl_ptr = dt; 602 /* order writing *dt vs. writing vpa->dtl_idx */ 603 smp_wmb(); 604 vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index); 605 vcpu->arch.dtl.dirty = true; 606 } 607 608 static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu) 609 { 610 if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207) 611 return true; 612 if ((!vcpu->arch.vcore->arch_compat) && 613 cpu_has_feature(CPU_FTR_ARCH_207S)) 614 return true; 615 return false; 616 } 617 618 static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags, 619 unsigned long resource, unsigned long value1, 620 unsigned long value2) 621 { 622 switch (resource) { 623 case H_SET_MODE_RESOURCE_SET_CIABR: 624 if (!kvmppc_power8_compatible(vcpu)) 625 return H_P2; 626 if (value2) 627 return H_P4; 628 if (mflags) 629 return H_UNSUPPORTED_FLAG_START; 630 /* Guests can't breakpoint the hypervisor */ 631 if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER) 632 return H_P3; 633 vcpu->arch.ciabr = value1; 634 return H_SUCCESS; 635 case H_SET_MODE_RESOURCE_SET_DAWR: 636 if (!kvmppc_power8_compatible(vcpu)) 637 return H_P2; 638 if (mflags) 639 return H_UNSUPPORTED_FLAG_START; 640 if (value2 & DABRX_HYP) 641 return H_P4; 642 vcpu->arch.dawr = value1; 643 vcpu->arch.dawrx = value2; 644 return H_SUCCESS; 645 default: 646 return H_TOO_HARD; 647 } 648 } 649 650 static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target) 651 { 652 struct kvmppc_vcore *vcore = target->arch.vcore; 653 654 /* 655 * We expect to have been called by the real mode handler 656 * (kvmppc_rm_h_confer()) which would have directly returned 657 * H_SUCCESS if the source vcore wasn't idle (e.g. if it may 658 * have useful work to do and should not confer) so we don't 659 * recheck that here. 660 */ 661 662 spin_lock(&vcore->lock); 663 if (target->arch.state == KVMPPC_VCPU_RUNNABLE && 664 vcore->vcore_state != VCORE_INACTIVE && 665 vcore->runner) 666 target = vcore->runner; 667 spin_unlock(&vcore->lock); 668 669 return kvm_vcpu_yield_to(target); 670 } 671 672 static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu) 673 { 674 int yield_count = 0; 675 struct lppaca *lppaca; 676 677 spin_lock(&vcpu->arch.vpa_update_lock); 678 lppaca = (struct lppaca *)vcpu->arch.vpa.pinned_addr; 679 if (lppaca) 680 yield_count = be32_to_cpu(lppaca->yield_count); 681 spin_unlock(&vcpu->arch.vpa_update_lock); 682 return yield_count; 683 } 684 685 int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) 686 { 687 unsigned long req = kvmppc_get_gpr(vcpu, 3); 688 unsigned long target, ret = H_SUCCESS; 689 int yield_count; 690 struct kvm_vcpu *tvcpu; 691 int idx, rc; 692 693 if (req <= MAX_HCALL_OPCODE && 694 !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls)) 695 return RESUME_HOST; 696 697 switch (req) { 698 case H_CEDE: 699 break; 700 case H_PROD: 701 target = kvmppc_get_gpr(vcpu, 4); 702 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target); 703 if (!tvcpu) { 704 ret = H_PARAMETER; 705 break; 706 } 707 tvcpu->arch.prodded = 1; 708 smp_mb(); 709 if (vcpu->arch.ceded) { 710 if (waitqueue_active(&vcpu->wq)) { 711 wake_up_interruptible(&vcpu->wq); 712 vcpu->stat.halt_wakeup++; 713 } 714 } 715 break; 716 case H_CONFER: 717 target = kvmppc_get_gpr(vcpu, 4); 718 if (target == -1) 719 break; 720 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target); 721 if (!tvcpu) { 722 ret = H_PARAMETER; 723 break; 724 } 725 yield_count = kvmppc_get_gpr(vcpu, 5); 726 if (kvmppc_get_yield_count(tvcpu) != yield_count) 727 break; 728 kvm_arch_vcpu_yield_to(tvcpu); 729 break; 730 case H_REGISTER_VPA: 731 ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4), 732 kvmppc_get_gpr(vcpu, 5), 733 kvmppc_get_gpr(vcpu, 6)); 734 break; 735 case H_RTAS: 736 if (list_empty(&vcpu->kvm->arch.rtas_tokens)) 737 return RESUME_HOST; 738 739 idx = srcu_read_lock(&vcpu->kvm->srcu); 740 rc = kvmppc_rtas_hcall(vcpu); 741 srcu_read_unlock(&vcpu->kvm->srcu, idx); 742 743 if (rc == -ENOENT) 744 return RESUME_HOST; 745 else if (rc == 0) 746 break; 747 748 /* Send the error out to userspace via KVM_RUN */ 749 return rc; 750 case H_LOGICAL_CI_LOAD: 751 ret = kvmppc_h_logical_ci_load(vcpu); 752 if (ret == H_TOO_HARD) 753 return RESUME_HOST; 754 break; 755 case H_LOGICAL_CI_STORE: 756 ret = kvmppc_h_logical_ci_store(vcpu); 757 if (ret == H_TOO_HARD) 758 return RESUME_HOST; 759 break; 760 case H_SET_MODE: 761 ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4), 762 kvmppc_get_gpr(vcpu, 5), 763 kvmppc_get_gpr(vcpu, 6), 764 kvmppc_get_gpr(vcpu, 7)); 765 if (ret == H_TOO_HARD) 766 return RESUME_HOST; 767 break; 768 case H_XIRR: 769 case H_CPPR: 770 case H_EOI: 771 case H_IPI: 772 case H_IPOLL: 773 case H_XIRR_X: 774 if (kvmppc_xics_enabled(vcpu)) { 775 ret = kvmppc_xics_hcall(vcpu, req); 776 break; 777 } /* fallthrough */ 778 default: 779 return RESUME_HOST; 780 } 781 kvmppc_set_gpr(vcpu, 3, ret); 782 vcpu->arch.hcall_needed = 0; 783 return RESUME_GUEST; 784 } 785 786 static int kvmppc_hcall_impl_hv(unsigned long cmd) 787 { 788 switch (cmd) { 789 case H_CEDE: 790 case H_PROD: 791 case H_CONFER: 792 case H_REGISTER_VPA: 793 case H_SET_MODE: 794 case H_LOGICAL_CI_LOAD: 795 case H_LOGICAL_CI_STORE: 796 #ifdef CONFIG_KVM_XICS 797 case H_XIRR: 798 case H_CPPR: 799 case H_EOI: 800 case H_IPI: 801 case H_IPOLL: 802 case H_XIRR_X: 803 #endif 804 return 1; 805 } 806 807 /* See if it's in the real-mode table */ 808 return kvmppc_hcall_impl_hv_realmode(cmd); 809 } 810 811 static int kvmppc_emulate_debug_inst(struct kvm_run *run, 812 struct kvm_vcpu *vcpu) 813 { 814 u32 last_inst; 815 816 if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) != 817 EMULATE_DONE) { 818 /* 819 * Fetch failed, so return to guest and 820 * try executing it again. 821 */ 822 return RESUME_GUEST; 823 } 824 825 if (last_inst == KVMPPC_INST_SW_BREAKPOINT) { 826 run->exit_reason = KVM_EXIT_DEBUG; 827 run->debug.arch.address = kvmppc_get_pc(vcpu); 828 return RESUME_HOST; 829 } else { 830 kvmppc_core_queue_program(vcpu, SRR1_PROGILL); 831 return RESUME_GUEST; 832 } 833 } 834 835 static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu, 836 struct task_struct *tsk) 837 { 838 int r = RESUME_HOST; 839 840 vcpu->stat.sum_exits++; 841 842 run->exit_reason = KVM_EXIT_UNKNOWN; 843 run->ready_for_interrupt_injection = 1; 844 switch (vcpu->arch.trap) { 845 /* We're good on these - the host merely wanted to get our attention */ 846 case BOOK3S_INTERRUPT_HV_DECREMENTER: 847 vcpu->stat.dec_exits++; 848 r = RESUME_GUEST; 849 break; 850 case BOOK3S_INTERRUPT_EXTERNAL: 851 case BOOK3S_INTERRUPT_H_DOORBELL: 852 vcpu->stat.ext_intr_exits++; 853 r = RESUME_GUEST; 854 break; 855 /* HMI is hypervisor interrupt and host has handled it. Resume guest.*/ 856 case BOOK3S_INTERRUPT_HMI: 857 case BOOK3S_INTERRUPT_PERFMON: 858 r = RESUME_GUEST; 859 break; 860 case BOOK3S_INTERRUPT_MACHINE_CHECK: 861 /* 862 * Deliver a machine check interrupt to the guest. 863 * We have to do this, even if the host has handled the 864 * machine check, because machine checks use SRR0/1 and 865 * the interrupt might have trashed guest state in them. 866 */ 867 kvmppc_book3s_queue_irqprio(vcpu, 868 BOOK3S_INTERRUPT_MACHINE_CHECK); 869 r = RESUME_GUEST; 870 break; 871 case BOOK3S_INTERRUPT_PROGRAM: 872 { 873 ulong flags; 874 /* 875 * Normally program interrupts are delivered directly 876 * to the guest by the hardware, but we can get here 877 * as a result of a hypervisor emulation interrupt 878 * (e40) getting turned into a 700 by BML RTAS. 879 */ 880 flags = vcpu->arch.shregs.msr & 0x1f0000ull; 881 kvmppc_core_queue_program(vcpu, flags); 882 r = RESUME_GUEST; 883 break; 884 } 885 case BOOK3S_INTERRUPT_SYSCALL: 886 { 887 /* hcall - punt to userspace */ 888 int i; 889 890 /* hypercall with MSR_PR has already been handled in rmode, 891 * and never reaches here. 892 */ 893 894 run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3); 895 for (i = 0; i < 9; ++i) 896 run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i); 897 run->exit_reason = KVM_EXIT_PAPR_HCALL; 898 vcpu->arch.hcall_needed = 1; 899 r = RESUME_HOST; 900 break; 901 } 902 /* 903 * We get these next two if the guest accesses a page which it thinks 904 * it has mapped but which is not actually present, either because 905 * it is for an emulated I/O device or because the corresonding 906 * host page has been paged out. Any other HDSI/HISI interrupts 907 * have been handled already. 908 */ 909 case BOOK3S_INTERRUPT_H_DATA_STORAGE: 910 r = RESUME_PAGE_FAULT; 911 break; 912 case BOOK3S_INTERRUPT_H_INST_STORAGE: 913 vcpu->arch.fault_dar = kvmppc_get_pc(vcpu); 914 vcpu->arch.fault_dsisr = 0; 915 r = RESUME_PAGE_FAULT; 916 break; 917 /* 918 * This occurs if the guest executes an illegal instruction. 919 * If the guest debug is disabled, generate a program interrupt 920 * to the guest. If guest debug is enabled, we need to check 921 * whether the instruction is a software breakpoint instruction. 922 * Accordingly return to Guest or Host. 923 */ 924 case BOOK3S_INTERRUPT_H_EMUL_ASSIST: 925 if (vcpu->arch.emul_inst != KVM_INST_FETCH_FAILED) 926 vcpu->arch.last_inst = kvmppc_need_byteswap(vcpu) ? 927 swab32(vcpu->arch.emul_inst) : 928 vcpu->arch.emul_inst; 929 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) { 930 r = kvmppc_emulate_debug_inst(run, vcpu); 931 } else { 932 kvmppc_core_queue_program(vcpu, SRR1_PROGILL); 933 r = RESUME_GUEST; 934 } 935 break; 936 /* 937 * This occurs if the guest (kernel or userspace), does something that 938 * is prohibited by HFSCR. We just generate a program interrupt to 939 * the guest. 940 */ 941 case BOOK3S_INTERRUPT_H_FAC_UNAVAIL: 942 kvmppc_core_queue_program(vcpu, SRR1_PROGILL); 943 r = RESUME_GUEST; 944 break; 945 default: 946 kvmppc_dump_regs(vcpu); 947 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n", 948 vcpu->arch.trap, kvmppc_get_pc(vcpu), 949 vcpu->arch.shregs.msr); 950 run->hw.hardware_exit_reason = vcpu->arch.trap; 951 r = RESUME_HOST; 952 break; 953 } 954 955 return r; 956 } 957 958 static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu, 959 struct kvm_sregs *sregs) 960 { 961 int i; 962 963 memset(sregs, 0, sizeof(struct kvm_sregs)); 964 sregs->pvr = vcpu->arch.pvr; 965 for (i = 0; i < vcpu->arch.slb_max; i++) { 966 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige; 967 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv; 968 } 969 970 return 0; 971 } 972 973 static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu, 974 struct kvm_sregs *sregs) 975 { 976 int i, j; 977 978 /* Only accept the same PVR as the host's, since we can't spoof it */ 979 if (sregs->pvr != vcpu->arch.pvr) 980 return -EINVAL; 981 982 j = 0; 983 for (i = 0; i < vcpu->arch.slb_nr; i++) { 984 if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) { 985 vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe; 986 vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv; 987 ++j; 988 } 989 } 990 vcpu->arch.slb_max = j; 991 992 return 0; 993 } 994 995 static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr, 996 bool preserve_top32) 997 { 998 struct kvm *kvm = vcpu->kvm; 999 struct kvmppc_vcore *vc = vcpu->arch.vcore; 1000 u64 mask; 1001 1002 mutex_lock(&kvm->lock); 1003 spin_lock(&vc->lock); 1004 /* 1005 * If ILE (interrupt little-endian) has changed, update the 1006 * MSR_LE bit in the intr_msr for each vcpu in this vcore. 1007 */ 1008 if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) { 1009 struct kvm_vcpu *vcpu; 1010 int i; 1011 1012 kvm_for_each_vcpu(i, vcpu, kvm) { 1013 if (vcpu->arch.vcore != vc) 1014 continue; 1015 if (new_lpcr & LPCR_ILE) 1016 vcpu->arch.intr_msr |= MSR_LE; 1017 else 1018 vcpu->arch.intr_msr &= ~MSR_LE; 1019 } 1020 } 1021 1022 /* 1023 * Userspace can only modify DPFD (default prefetch depth), 1024 * ILE (interrupt little-endian) and TC (translation control). 1025 * On POWER8 userspace can also modify AIL (alt. interrupt loc.) 1026 */ 1027 mask = LPCR_DPFD | LPCR_ILE | LPCR_TC; 1028 if (cpu_has_feature(CPU_FTR_ARCH_207S)) 1029 mask |= LPCR_AIL; 1030 1031 /* Broken 32-bit version of LPCR must not clear top bits */ 1032 if (preserve_top32) 1033 mask &= 0xFFFFFFFF; 1034 vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask); 1035 spin_unlock(&vc->lock); 1036 mutex_unlock(&kvm->lock); 1037 } 1038 1039 static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, 1040 union kvmppc_one_reg *val) 1041 { 1042 int r = 0; 1043 long int i; 1044 1045 switch (id) { 1046 case KVM_REG_PPC_DEBUG_INST: 1047 *val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT); 1048 break; 1049 case KVM_REG_PPC_HIOR: 1050 *val = get_reg_val(id, 0); 1051 break; 1052 case KVM_REG_PPC_DABR: 1053 *val = get_reg_val(id, vcpu->arch.dabr); 1054 break; 1055 case KVM_REG_PPC_DABRX: 1056 *val = get_reg_val(id, vcpu->arch.dabrx); 1057 break; 1058 case KVM_REG_PPC_DSCR: 1059 *val = get_reg_val(id, vcpu->arch.dscr); 1060 break; 1061 case KVM_REG_PPC_PURR: 1062 *val = get_reg_val(id, vcpu->arch.purr); 1063 break; 1064 case KVM_REG_PPC_SPURR: 1065 *val = get_reg_val(id, vcpu->arch.spurr); 1066 break; 1067 case KVM_REG_PPC_AMR: 1068 *val = get_reg_val(id, vcpu->arch.amr); 1069 break; 1070 case KVM_REG_PPC_UAMOR: 1071 *val = get_reg_val(id, vcpu->arch.uamor); 1072 break; 1073 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS: 1074 i = id - KVM_REG_PPC_MMCR0; 1075 *val = get_reg_val(id, vcpu->arch.mmcr[i]); 1076 break; 1077 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8: 1078 i = id - KVM_REG_PPC_PMC1; 1079 *val = get_reg_val(id, vcpu->arch.pmc[i]); 1080 break; 1081 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2: 1082 i = id - KVM_REG_PPC_SPMC1; 1083 *val = get_reg_val(id, vcpu->arch.spmc[i]); 1084 break; 1085 case KVM_REG_PPC_SIAR: 1086 *val = get_reg_val(id, vcpu->arch.siar); 1087 break; 1088 case KVM_REG_PPC_SDAR: 1089 *val = get_reg_val(id, vcpu->arch.sdar); 1090 break; 1091 case KVM_REG_PPC_SIER: 1092 *val = get_reg_val(id, vcpu->arch.sier); 1093 break; 1094 case KVM_REG_PPC_IAMR: 1095 *val = get_reg_val(id, vcpu->arch.iamr); 1096 break; 1097 case KVM_REG_PPC_PSPB: 1098 *val = get_reg_val(id, vcpu->arch.pspb); 1099 break; 1100 case KVM_REG_PPC_DPDES: 1101 *val = get_reg_val(id, vcpu->arch.vcore->dpdes); 1102 break; 1103 case KVM_REG_PPC_DAWR: 1104 *val = get_reg_val(id, vcpu->arch.dawr); 1105 break; 1106 case KVM_REG_PPC_DAWRX: 1107 *val = get_reg_val(id, vcpu->arch.dawrx); 1108 break; 1109 case KVM_REG_PPC_CIABR: 1110 *val = get_reg_val(id, vcpu->arch.ciabr); 1111 break; 1112 case KVM_REG_PPC_CSIGR: 1113 *val = get_reg_val(id, vcpu->arch.csigr); 1114 break; 1115 case KVM_REG_PPC_TACR: 1116 *val = get_reg_val(id, vcpu->arch.tacr); 1117 break; 1118 case KVM_REG_PPC_TCSCR: 1119 *val = get_reg_val(id, vcpu->arch.tcscr); 1120 break; 1121 case KVM_REG_PPC_PID: 1122 *val = get_reg_val(id, vcpu->arch.pid); 1123 break; 1124 case KVM_REG_PPC_ACOP: 1125 *val = get_reg_val(id, vcpu->arch.acop); 1126 break; 1127 case KVM_REG_PPC_WORT: 1128 *val = get_reg_val(id, vcpu->arch.wort); 1129 break; 1130 case KVM_REG_PPC_VPA_ADDR: 1131 spin_lock(&vcpu->arch.vpa_update_lock); 1132 *val = get_reg_val(id, vcpu->arch.vpa.next_gpa); 1133 spin_unlock(&vcpu->arch.vpa_update_lock); 1134 break; 1135 case KVM_REG_PPC_VPA_SLB: 1136 spin_lock(&vcpu->arch.vpa_update_lock); 1137 val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa; 1138 val->vpaval.length = vcpu->arch.slb_shadow.len; 1139 spin_unlock(&vcpu->arch.vpa_update_lock); 1140 break; 1141 case KVM_REG_PPC_VPA_DTL: 1142 spin_lock(&vcpu->arch.vpa_update_lock); 1143 val->vpaval.addr = vcpu->arch.dtl.next_gpa; 1144 val->vpaval.length = vcpu->arch.dtl.len; 1145 spin_unlock(&vcpu->arch.vpa_update_lock); 1146 break; 1147 case KVM_REG_PPC_TB_OFFSET: 1148 *val = get_reg_val(id, vcpu->arch.vcore->tb_offset); 1149 break; 1150 case KVM_REG_PPC_LPCR: 1151 case KVM_REG_PPC_LPCR_64: 1152 *val = get_reg_val(id, vcpu->arch.vcore->lpcr); 1153 break; 1154 case KVM_REG_PPC_PPR: 1155 *val = get_reg_val(id, vcpu->arch.ppr); 1156 break; 1157 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1158 case KVM_REG_PPC_TFHAR: 1159 *val = get_reg_val(id, vcpu->arch.tfhar); 1160 break; 1161 case KVM_REG_PPC_TFIAR: 1162 *val = get_reg_val(id, vcpu->arch.tfiar); 1163 break; 1164 case KVM_REG_PPC_TEXASR: 1165 *val = get_reg_val(id, vcpu->arch.texasr); 1166 break; 1167 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31: 1168 i = id - KVM_REG_PPC_TM_GPR0; 1169 *val = get_reg_val(id, vcpu->arch.gpr_tm[i]); 1170 break; 1171 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63: 1172 { 1173 int j; 1174 i = id - KVM_REG_PPC_TM_VSR0; 1175 if (i < 32) 1176 for (j = 0; j < TS_FPRWIDTH; j++) 1177 val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j]; 1178 else { 1179 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1180 val->vval = vcpu->arch.vr_tm.vr[i-32]; 1181 else 1182 r = -ENXIO; 1183 } 1184 break; 1185 } 1186 case KVM_REG_PPC_TM_CR: 1187 *val = get_reg_val(id, vcpu->arch.cr_tm); 1188 break; 1189 case KVM_REG_PPC_TM_LR: 1190 *val = get_reg_val(id, vcpu->arch.lr_tm); 1191 break; 1192 case KVM_REG_PPC_TM_CTR: 1193 *val = get_reg_val(id, vcpu->arch.ctr_tm); 1194 break; 1195 case KVM_REG_PPC_TM_FPSCR: 1196 *val = get_reg_val(id, vcpu->arch.fp_tm.fpscr); 1197 break; 1198 case KVM_REG_PPC_TM_AMR: 1199 *val = get_reg_val(id, vcpu->arch.amr_tm); 1200 break; 1201 case KVM_REG_PPC_TM_PPR: 1202 *val = get_reg_val(id, vcpu->arch.ppr_tm); 1203 break; 1204 case KVM_REG_PPC_TM_VRSAVE: 1205 *val = get_reg_val(id, vcpu->arch.vrsave_tm); 1206 break; 1207 case KVM_REG_PPC_TM_VSCR: 1208 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1209 *val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]); 1210 else 1211 r = -ENXIO; 1212 break; 1213 case KVM_REG_PPC_TM_DSCR: 1214 *val = get_reg_val(id, vcpu->arch.dscr_tm); 1215 break; 1216 case KVM_REG_PPC_TM_TAR: 1217 *val = get_reg_val(id, vcpu->arch.tar_tm); 1218 break; 1219 #endif 1220 case KVM_REG_PPC_ARCH_COMPAT: 1221 *val = get_reg_val(id, vcpu->arch.vcore->arch_compat); 1222 break; 1223 default: 1224 r = -EINVAL; 1225 break; 1226 } 1227 1228 return r; 1229 } 1230 1231 static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, 1232 union kvmppc_one_reg *val) 1233 { 1234 int r = 0; 1235 long int i; 1236 unsigned long addr, len; 1237 1238 switch (id) { 1239 case KVM_REG_PPC_HIOR: 1240 /* Only allow this to be set to zero */ 1241 if (set_reg_val(id, *val)) 1242 r = -EINVAL; 1243 break; 1244 case KVM_REG_PPC_DABR: 1245 vcpu->arch.dabr = set_reg_val(id, *val); 1246 break; 1247 case KVM_REG_PPC_DABRX: 1248 vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP; 1249 break; 1250 case KVM_REG_PPC_DSCR: 1251 vcpu->arch.dscr = set_reg_val(id, *val); 1252 break; 1253 case KVM_REG_PPC_PURR: 1254 vcpu->arch.purr = set_reg_val(id, *val); 1255 break; 1256 case KVM_REG_PPC_SPURR: 1257 vcpu->arch.spurr = set_reg_val(id, *val); 1258 break; 1259 case KVM_REG_PPC_AMR: 1260 vcpu->arch.amr = set_reg_val(id, *val); 1261 break; 1262 case KVM_REG_PPC_UAMOR: 1263 vcpu->arch.uamor = set_reg_val(id, *val); 1264 break; 1265 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS: 1266 i = id - KVM_REG_PPC_MMCR0; 1267 vcpu->arch.mmcr[i] = set_reg_val(id, *val); 1268 break; 1269 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8: 1270 i = id - KVM_REG_PPC_PMC1; 1271 vcpu->arch.pmc[i] = set_reg_val(id, *val); 1272 break; 1273 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2: 1274 i = id - KVM_REG_PPC_SPMC1; 1275 vcpu->arch.spmc[i] = set_reg_val(id, *val); 1276 break; 1277 case KVM_REG_PPC_SIAR: 1278 vcpu->arch.siar = set_reg_val(id, *val); 1279 break; 1280 case KVM_REG_PPC_SDAR: 1281 vcpu->arch.sdar = set_reg_val(id, *val); 1282 break; 1283 case KVM_REG_PPC_SIER: 1284 vcpu->arch.sier = set_reg_val(id, *val); 1285 break; 1286 case KVM_REG_PPC_IAMR: 1287 vcpu->arch.iamr = set_reg_val(id, *val); 1288 break; 1289 case KVM_REG_PPC_PSPB: 1290 vcpu->arch.pspb = set_reg_val(id, *val); 1291 break; 1292 case KVM_REG_PPC_DPDES: 1293 vcpu->arch.vcore->dpdes = set_reg_val(id, *val); 1294 break; 1295 case KVM_REG_PPC_DAWR: 1296 vcpu->arch.dawr = set_reg_val(id, *val); 1297 break; 1298 case KVM_REG_PPC_DAWRX: 1299 vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP; 1300 break; 1301 case KVM_REG_PPC_CIABR: 1302 vcpu->arch.ciabr = set_reg_val(id, *val); 1303 /* Don't allow setting breakpoints in hypervisor code */ 1304 if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER) 1305 vcpu->arch.ciabr &= ~CIABR_PRIV; /* disable */ 1306 break; 1307 case KVM_REG_PPC_CSIGR: 1308 vcpu->arch.csigr = set_reg_val(id, *val); 1309 break; 1310 case KVM_REG_PPC_TACR: 1311 vcpu->arch.tacr = set_reg_val(id, *val); 1312 break; 1313 case KVM_REG_PPC_TCSCR: 1314 vcpu->arch.tcscr = set_reg_val(id, *val); 1315 break; 1316 case KVM_REG_PPC_PID: 1317 vcpu->arch.pid = set_reg_val(id, *val); 1318 break; 1319 case KVM_REG_PPC_ACOP: 1320 vcpu->arch.acop = set_reg_val(id, *val); 1321 break; 1322 case KVM_REG_PPC_WORT: 1323 vcpu->arch.wort = set_reg_val(id, *val); 1324 break; 1325 case KVM_REG_PPC_VPA_ADDR: 1326 addr = set_reg_val(id, *val); 1327 r = -EINVAL; 1328 if (!addr && (vcpu->arch.slb_shadow.next_gpa || 1329 vcpu->arch.dtl.next_gpa)) 1330 break; 1331 r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca)); 1332 break; 1333 case KVM_REG_PPC_VPA_SLB: 1334 addr = val->vpaval.addr; 1335 len = val->vpaval.length; 1336 r = -EINVAL; 1337 if (addr && !vcpu->arch.vpa.next_gpa) 1338 break; 1339 r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len); 1340 break; 1341 case KVM_REG_PPC_VPA_DTL: 1342 addr = val->vpaval.addr; 1343 len = val->vpaval.length; 1344 r = -EINVAL; 1345 if (addr && (len < sizeof(struct dtl_entry) || 1346 !vcpu->arch.vpa.next_gpa)) 1347 break; 1348 len -= len % sizeof(struct dtl_entry); 1349 r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len); 1350 break; 1351 case KVM_REG_PPC_TB_OFFSET: 1352 /* round up to multiple of 2^24 */ 1353 vcpu->arch.vcore->tb_offset = 1354 ALIGN(set_reg_val(id, *val), 1UL << 24); 1355 break; 1356 case KVM_REG_PPC_LPCR: 1357 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true); 1358 break; 1359 case KVM_REG_PPC_LPCR_64: 1360 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false); 1361 break; 1362 case KVM_REG_PPC_PPR: 1363 vcpu->arch.ppr = set_reg_val(id, *val); 1364 break; 1365 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1366 case KVM_REG_PPC_TFHAR: 1367 vcpu->arch.tfhar = set_reg_val(id, *val); 1368 break; 1369 case KVM_REG_PPC_TFIAR: 1370 vcpu->arch.tfiar = set_reg_val(id, *val); 1371 break; 1372 case KVM_REG_PPC_TEXASR: 1373 vcpu->arch.texasr = set_reg_val(id, *val); 1374 break; 1375 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31: 1376 i = id - KVM_REG_PPC_TM_GPR0; 1377 vcpu->arch.gpr_tm[i] = set_reg_val(id, *val); 1378 break; 1379 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63: 1380 { 1381 int j; 1382 i = id - KVM_REG_PPC_TM_VSR0; 1383 if (i < 32) 1384 for (j = 0; j < TS_FPRWIDTH; j++) 1385 vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j]; 1386 else 1387 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1388 vcpu->arch.vr_tm.vr[i-32] = val->vval; 1389 else 1390 r = -ENXIO; 1391 break; 1392 } 1393 case KVM_REG_PPC_TM_CR: 1394 vcpu->arch.cr_tm = set_reg_val(id, *val); 1395 break; 1396 case KVM_REG_PPC_TM_LR: 1397 vcpu->arch.lr_tm = set_reg_val(id, *val); 1398 break; 1399 case KVM_REG_PPC_TM_CTR: 1400 vcpu->arch.ctr_tm = set_reg_val(id, *val); 1401 break; 1402 case KVM_REG_PPC_TM_FPSCR: 1403 vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val); 1404 break; 1405 case KVM_REG_PPC_TM_AMR: 1406 vcpu->arch.amr_tm = set_reg_val(id, *val); 1407 break; 1408 case KVM_REG_PPC_TM_PPR: 1409 vcpu->arch.ppr_tm = set_reg_val(id, *val); 1410 break; 1411 case KVM_REG_PPC_TM_VRSAVE: 1412 vcpu->arch.vrsave_tm = set_reg_val(id, *val); 1413 break; 1414 case KVM_REG_PPC_TM_VSCR: 1415 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1416 vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val); 1417 else 1418 r = - ENXIO; 1419 break; 1420 case KVM_REG_PPC_TM_DSCR: 1421 vcpu->arch.dscr_tm = set_reg_val(id, *val); 1422 break; 1423 case KVM_REG_PPC_TM_TAR: 1424 vcpu->arch.tar_tm = set_reg_val(id, *val); 1425 break; 1426 #endif 1427 case KVM_REG_PPC_ARCH_COMPAT: 1428 r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val)); 1429 break; 1430 default: 1431 r = -EINVAL; 1432 break; 1433 } 1434 1435 return r; 1436 } 1437 1438 static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core) 1439 { 1440 struct kvmppc_vcore *vcore; 1441 1442 vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL); 1443 1444 if (vcore == NULL) 1445 return NULL; 1446 1447 INIT_LIST_HEAD(&vcore->runnable_threads); 1448 spin_lock_init(&vcore->lock); 1449 spin_lock_init(&vcore->stoltb_lock); 1450 init_waitqueue_head(&vcore->wq); 1451 vcore->preempt_tb = TB_NIL; 1452 vcore->lpcr = kvm->arch.lpcr; 1453 vcore->first_vcpuid = core * threads_per_subcore; 1454 vcore->kvm = kvm; 1455 INIT_LIST_HEAD(&vcore->preempt_list); 1456 1457 return vcore; 1458 } 1459 1460 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1461 static struct debugfs_timings_element { 1462 const char *name; 1463 size_t offset; 1464 } timings[] = { 1465 {"rm_entry", offsetof(struct kvm_vcpu, arch.rm_entry)}, 1466 {"rm_intr", offsetof(struct kvm_vcpu, arch.rm_intr)}, 1467 {"rm_exit", offsetof(struct kvm_vcpu, arch.rm_exit)}, 1468 {"guest", offsetof(struct kvm_vcpu, arch.guest_time)}, 1469 {"cede", offsetof(struct kvm_vcpu, arch.cede_time)}, 1470 }; 1471 1472 #define N_TIMINGS (sizeof(timings) / sizeof(timings[0])) 1473 1474 struct debugfs_timings_state { 1475 struct kvm_vcpu *vcpu; 1476 unsigned int buflen; 1477 char buf[N_TIMINGS * 100]; 1478 }; 1479 1480 static int debugfs_timings_open(struct inode *inode, struct file *file) 1481 { 1482 struct kvm_vcpu *vcpu = inode->i_private; 1483 struct debugfs_timings_state *p; 1484 1485 p = kzalloc(sizeof(*p), GFP_KERNEL); 1486 if (!p) 1487 return -ENOMEM; 1488 1489 kvm_get_kvm(vcpu->kvm); 1490 p->vcpu = vcpu; 1491 file->private_data = p; 1492 1493 return nonseekable_open(inode, file); 1494 } 1495 1496 static int debugfs_timings_release(struct inode *inode, struct file *file) 1497 { 1498 struct debugfs_timings_state *p = file->private_data; 1499 1500 kvm_put_kvm(p->vcpu->kvm); 1501 kfree(p); 1502 return 0; 1503 } 1504 1505 static ssize_t debugfs_timings_read(struct file *file, char __user *buf, 1506 size_t len, loff_t *ppos) 1507 { 1508 struct debugfs_timings_state *p = file->private_data; 1509 struct kvm_vcpu *vcpu = p->vcpu; 1510 char *s, *buf_end; 1511 struct kvmhv_tb_accumulator tb; 1512 u64 count; 1513 loff_t pos; 1514 ssize_t n; 1515 int i, loops; 1516 bool ok; 1517 1518 if (!p->buflen) { 1519 s = p->buf; 1520 buf_end = s + sizeof(p->buf); 1521 for (i = 0; i < N_TIMINGS; ++i) { 1522 struct kvmhv_tb_accumulator *acc; 1523 1524 acc = (struct kvmhv_tb_accumulator *) 1525 ((unsigned long)vcpu + timings[i].offset); 1526 ok = false; 1527 for (loops = 0; loops < 1000; ++loops) { 1528 count = acc->seqcount; 1529 if (!(count & 1)) { 1530 smp_rmb(); 1531 tb = *acc; 1532 smp_rmb(); 1533 if (count == acc->seqcount) { 1534 ok = true; 1535 break; 1536 } 1537 } 1538 udelay(1); 1539 } 1540 if (!ok) 1541 snprintf(s, buf_end - s, "%s: stuck\n", 1542 timings[i].name); 1543 else 1544 snprintf(s, buf_end - s, 1545 "%s: %llu %llu %llu %llu\n", 1546 timings[i].name, count / 2, 1547 tb_to_ns(tb.tb_total), 1548 tb_to_ns(tb.tb_min), 1549 tb_to_ns(tb.tb_max)); 1550 s += strlen(s); 1551 } 1552 p->buflen = s - p->buf; 1553 } 1554 1555 pos = *ppos; 1556 if (pos >= p->buflen) 1557 return 0; 1558 if (len > p->buflen - pos) 1559 len = p->buflen - pos; 1560 n = copy_to_user(buf, p->buf + pos, len); 1561 if (n) { 1562 if (n == len) 1563 return -EFAULT; 1564 len -= n; 1565 } 1566 *ppos = pos + len; 1567 return len; 1568 } 1569 1570 static ssize_t debugfs_timings_write(struct file *file, const char __user *buf, 1571 size_t len, loff_t *ppos) 1572 { 1573 return -EACCES; 1574 } 1575 1576 static const struct file_operations debugfs_timings_ops = { 1577 .owner = THIS_MODULE, 1578 .open = debugfs_timings_open, 1579 .release = debugfs_timings_release, 1580 .read = debugfs_timings_read, 1581 .write = debugfs_timings_write, 1582 .llseek = generic_file_llseek, 1583 }; 1584 1585 /* Create a debugfs directory for the vcpu */ 1586 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id) 1587 { 1588 char buf[16]; 1589 struct kvm *kvm = vcpu->kvm; 1590 1591 snprintf(buf, sizeof(buf), "vcpu%u", id); 1592 if (IS_ERR_OR_NULL(kvm->arch.debugfs_dir)) 1593 return; 1594 vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir); 1595 if (IS_ERR_OR_NULL(vcpu->arch.debugfs_dir)) 1596 return; 1597 vcpu->arch.debugfs_timings = 1598 debugfs_create_file("timings", 0444, vcpu->arch.debugfs_dir, 1599 vcpu, &debugfs_timings_ops); 1600 } 1601 1602 #else /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */ 1603 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id) 1604 { 1605 } 1606 #endif /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */ 1607 1608 static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm, 1609 unsigned int id) 1610 { 1611 struct kvm_vcpu *vcpu; 1612 int err = -EINVAL; 1613 int core; 1614 struct kvmppc_vcore *vcore; 1615 1616 core = id / threads_per_subcore; 1617 if (core >= KVM_MAX_VCORES) 1618 goto out; 1619 1620 err = -ENOMEM; 1621 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); 1622 if (!vcpu) 1623 goto out; 1624 1625 err = kvm_vcpu_init(vcpu, kvm, id); 1626 if (err) 1627 goto free_vcpu; 1628 1629 vcpu->arch.shared = &vcpu->arch.shregs; 1630 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1631 /* 1632 * The shared struct is never shared on HV, 1633 * so we can always use host endianness 1634 */ 1635 #ifdef __BIG_ENDIAN__ 1636 vcpu->arch.shared_big_endian = true; 1637 #else 1638 vcpu->arch.shared_big_endian = false; 1639 #endif 1640 #endif 1641 vcpu->arch.mmcr[0] = MMCR0_FC; 1642 vcpu->arch.ctrl = CTRL_RUNLATCH; 1643 /* default to host PVR, since we can't spoof it */ 1644 kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR)); 1645 spin_lock_init(&vcpu->arch.vpa_update_lock); 1646 spin_lock_init(&vcpu->arch.tbacct_lock); 1647 vcpu->arch.busy_preempt = TB_NIL; 1648 vcpu->arch.intr_msr = MSR_SF | MSR_ME; 1649 1650 kvmppc_mmu_book3s_hv_init(vcpu); 1651 1652 vcpu->arch.state = KVMPPC_VCPU_NOTREADY; 1653 1654 init_waitqueue_head(&vcpu->arch.cpu_run); 1655 1656 mutex_lock(&kvm->lock); 1657 vcore = kvm->arch.vcores[core]; 1658 if (!vcore) { 1659 vcore = kvmppc_vcore_create(kvm, core); 1660 kvm->arch.vcores[core] = vcore; 1661 kvm->arch.online_vcores++; 1662 } 1663 mutex_unlock(&kvm->lock); 1664 1665 if (!vcore) 1666 goto free_vcpu; 1667 1668 spin_lock(&vcore->lock); 1669 ++vcore->num_threads; 1670 spin_unlock(&vcore->lock); 1671 vcpu->arch.vcore = vcore; 1672 vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid; 1673 vcpu->arch.thread_cpu = -1; 1674 1675 vcpu->arch.cpu_type = KVM_CPU_3S_64; 1676 kvmppc_sanity_check(vcpu); 1677 1678 debugfs_vcpu_init(vcpu, id); 1679 1680 return vcpu; 1681 1682 free_vcpu: 1683 kmem_cache_free(kvm_vcpu_cache, vcpu); 1684 out: 1685 return ERR_PTR(err); 1686 } 1687 1688 static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa) 1689 { 1690 if (vpa->pinned_addr) 1691 kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa, 1692 vpa->dirty); 1693 } 1694 1695 static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu) 1696 { 1697 spin_lock(&vcpu->arch.vpa_update_lock); 1698 unpin_vpa(vcpu->kvm, &vcpu->arch.dtl); 1699 unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow); 1700 unpin_vpa(vcpu->kvm, &vcpu->arch.vpa); 1701 spin_unlock(&vcpu->arch.vpa_update_lock); 1702 kvm_vcpu_uninit(vcpu); 1703 kmem_cache_free(kvm_vcpu_cache, vcpu); 1704 } 1705 1706 static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu) 1707 { 1708 /* Indicate we want to get back into the guest */ 1709 return 1; 1710 } 1711 1712 static void kvmppc_set_timer(struct kvm_vcpu *vcpu) 1713 { 1714 unsigned long dec_nsec, now; 1715 1716 now = get_tb(); 1717 if (now > vcpu->arch.dec_expires) { 1718 /* decrementer has already gone negative */ 1719 kvmppc_core_queue_dec(vcpu); 1720 kvmppc_core_prepare_to_enter(vcpu); 1721 return; 1722 } 1723 dec_nsec = (vcpu->arch.dec_expires - now) * NSEC_PER_SEC 1724 / tb_ticks_per_sec; 1725 hrtimer_start(&vcpu->arch.dec_timer, ktime_set(0, dec_nsec), 1726 HRTIMER_MODE_REL); 1727 vcpu->arch.timer_running = 1; 1728 } 1729 1730 static void kvmppc_end_cede(struct kvm_vcpu *vcpu) 1731 { 1732 vcpu->arch.ceded = 0; 1733 if (vcpu->arch.timer_running) { 1734 hrtimer_try_to_cancel(&vcpu->arch.dec_timer); 1735 vcpu->arch.timer_running = 0; 1736 } 1737 } 1738 1739 extern void __kvmppc_vcore_entry(void); 1740 1741 static void kvmppc_remove_runnable(struct kvmppc_vcore *vc, 1742 struct kvm_vcpu *vcpu) 1743 { 1744 u64 now; 1745 1746 if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE) 1747 return; 1748 spin_lock_irq(&vcpu->arch.tbacct_lock); 1749 now = mftb(); 1750 vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) - 1751 vcpu->arch.stolen_logged; 1752 vcpu->arch.busy_preempt = now; 1753 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST; 1754 spin_unlock_irq(&vcpu->arch.tbacct_lock); 1755 --vc->n_runnable; 1756 list_del(&vcpu->arch.run_list); 1757 } 1758 1759 static int kvmppc_grab_hwthread(int cpu) 1760 { 1761 struct paca_struct *tpaca; 1762 long timeout = 10000; 1763 1764 tpaca = &paca[cpu]; 1765 1766 /* Ensure the thread won't go into the kernel if it wakes */ 1767 tpaca->kvm_hstate.kvm_vcpu = NULL; 1768 tpaca->kvm_hstate.kvm_vcore = NULL; 1769 tpaca->kvm_hstate.napping = 0; 1770 smp_wmb(); 1771 tpaca->kvm_hstate.hwthread_req = 1; 1772 1773 /* 1774 * If the thread is already executing in the kernel (e.g. handling 1775 * a stray interrupt), wait for it to get back to nap mode. 1776 * The smp_mb() is to ensure that our setting of hwthread_req 1777 * is visible before we look at hwthread_state, so if this 1778 * races with the code at system_reset_pSeries and the thread 1779 * misses our setting of hwthread_req, we are sure to see its 1780 * setting of hwthread_state, and vice versa. 1781 */ 1782 smp_mb(); 1783 while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) { 1784 if (--timeout <= 0) { 1785 pr_err("KVM: couldn't grab cpu %d\n", cpu); 1786 return -EBUSY; 1787 } 1788 udelay(1); 1789 } 1790 return 0; 1791 } 1792 1793 static void kvmppc_release_hwthread(int cpu) 1794 { 1795 struct paca_struct *tpaca; 1796 1797 tpaca = &paca[cpu]; 1798 tpaca->kvm_hstate.hwthread_req = 0; 1799 tpaca->kvm_hstate.kvm_vcpu = NULL; 1800 tpaca->kvm_hstate.kvm_vcore = NULL; 1801 tpaca->kvm_hstate.kvm_split_mode = NULL; 1802 } 1803 1804 static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc) 1805 { 1806 int cpu; 1807 struct paca_struct *tpaca; 1808 struct kvmppc_vcore *mvc = vc->master_vcore; 1809 1810 cpu = vc->pcpu; 1811 if (vcpu) { 1812 if (vcpu->arch.timer_running) { 1813 hrtimer_try_to_cancel(&vcpu->arch.dec_timer); 1814 vcpu->arch.timer_running = 0; 1815 } 1816 cpu += vcpu->arch.ptid; 1817 vcpu->cpu = mvc->pcpu; 1818 vcpu->arch.thread_cpu = cpu; 1819 } 1820 tpaca = &paca[cpu]; 1821 tpaca->kvm_hstate.kvm_vcpu = vcpu; 1822 tpaca->kvm_hstate.ptid = cpu - mvc->pcpu; 1823 /* Order stores to hstate.kvm_vcpu etc. before store to kvm_vcore */ 1824 smp_wmb(); 1825 tpaca->kvm_hstate.kvm_vcore = mvc; 1826 if (cpu != smp_processor_id()) 1827 kvmppc_ipi_thread(cpu); 1828 } 1829 1830 static void kvmppc_wait_for_nap(void) 1831 { 1832 int cpu = smp_processor_id(); 1833 int i, loops; 1834 1835 for (loops = 0; loops < 1000000; ++loops) { 1836 /* 1837 * Check if all threads are finished. 1838 * We set the vcore pointer when starting a thread 1839 * and the thread clears it when finished, so we look 1840 * for any threads that still have a non-NULL vcore ptr. 1841 */ 1842 for (i = 1; i < threads_per_subcore; ++i) 1843 if (paca[cpu + i].kvm_hstate.kvm_vcore) 1844 break; 1845 if (i == threads_per_subcore) { 1846 HMT_medium(); 1847 return; 1848 } 1849 HMT_low(); 1850 } 1851 HMT_medium(); 1852 for (i = 1; i < threads_per_subcore; ++i) 1853 if (paca[cpu + i].kvm_hstate.kvm_vcore) 1854 pr_err("KVM: CPU %d seems to be stuck\n", cpu + i); 1855 } 1856 1857 /* 1858 * Check that we are on thread 0 and that any other threads in 1859 * this core are off-line. Then grab the threads so they can't 1860 * enter the kernel. 1861 */ 1862 static int on_primary_thread(void) 1863 { 1864 int cpu = smp_processor_id(); 1865 int thr; 1866 1867 /* Are we on a primary subcore? */ 1868 if (cpu_thread_in_subcore(cpu)) 1869 return 0; 1870 1871 thr = 0; 1872 while (++thr < threads_per_subcore) 1873 if (cpu_online(cpu + thr)) 1874 return 0; 1875 1876 /* Grab all hw threads so they can't go into the kernel */ 1877 for (thr = 1; thr < threads_per_subcore; ++thr) { 1878 if (kvmppc_grab_hwthread(cpu + thr)) { 1879 /* Couldn't grab one; let the others go */ 1880 do { 1881 kvmppc_release_hwthread(cpu + thr); 1882 } while (--thr > 0); 1883 return 0; 1884 } 1885 } 1886 return 1; 1887 } 1888 1889 /* 1890 * A list of virtual cores for each physical CPU. 1891 * These are vcores that could run but their runner VCPU tasks are 1892 * (or may be) preempted. 1893 */ 1894 struct preempted_vcore_list { 1895 struct list_head list; 1896 spinlock_t lock; 1897 }; 1898 1899 static DEFINE_PER_CPU(struct preempted_vcore_list, preempted_vcores); 1900 1901 static void init_vcore_lists(void) 1902 { 1903 int cpu; 1904 1905 for_each_possible_cpu(cpu) { 1906 struct preempted_vcore_list *lp = &per_cpu(preempted_vcores, cpu); 1907 spin_lock_init(&lp->lock); 1908 INIT_LIST_HEAD(&lp->list); 1909 } 1910 } 1911 1912 static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc) 1913 { 1914 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores); 1915 1916 vc->vcore_state = VCORE_PREEMPT; 1917 vc->pcpu = smp_processor_id(); 1918 if (vc->num_threads < threads_per_subcore) { 1919 spin_lock(&lp->lock); 1920 list_add_tail(&vc->preempt_list, &lp->list); 1921 spin_unlock(&lp->lock); 1922 } 1923 1924 /* Start accumulating stolen time */ 1925 kvmppc_core_start_stolen(vc); 1926 } 1927 1928 static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc) 1929 { 1930 struct preempted_vcore_list *lp; 1931 1932 kvmppc_core_end_stolen(vc); 1933 if (!list_empty(&vc->preempt_list)) { 1934 lp = &per_cpu(preempted_vcores, vc->pcpu); 1935 spin_lock(&lp->lock); 1936 list_del_init(&vc->preempt_list); 1937 spin_unlock(&lp->lock); 1938 } 1939 vc->vcore_state = VCORE_INACTIVE; 1940 } 1941 1942 /* 1943 * This stores information about the virtual cores currently 1944 * assigned to a physical core. 1945 */ 1946 struct core_info { 1947 int n_subcores; 1948 int max_subcore_threads; 1949 int total_threads; 1950 int subcore_threads[MAX_SUBCORES]; 1951 struct kvm *subcore_vm[MAX_SUBCORES]; 1952 struct list_head vcs[MAX_SUBCORES]; 1953 }; 1954 1955 /* 1956 * This mapping means subcores 0 and 1 can use threads 0-3 and 4-7 1957 * respectively in 2-way micro-threading (split-core) mode. 1958 */ 1959 static int subcore_thread_map[MAX_SUBCORES] = { 0, 4, 2, 6 }; 1960 1961 static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc) 1962 { 1963 int sub; 1964 1965 memset(cip, 0, sizeof(*cip)); 1966 cip->n_subcores = 1; 1967 cip->max_subcore_threads = vc->num_threads; 1968 cip->total_threads = vc->num_threads; 1969 cip->subcore_threads[0] = vc->num_threads; 1970 cip->subcore_vm[0] = vc->kvm; 1971 for (sub = 0; sub < MAX_SUBCORES; ++sub) 1972 INIT_LIST_HEAD(&cip->vcs[sub]); 1973 list_add_tail(&vc->preempt_list, &cip->vcs[0]); 1974 } 1975 1976 static bool subcore_config_ok(int n_subcores, int n_threads) 1977 { 1978 /* Can only dynamically split if unsplit to begin with */ 1979 if (n_subcores > 1 && threads_per_subcore < MAX_SMT_THREADS) 1980 return false; 1981 if (n_subcores > MAX_SUBCORES) 1982 return false; 1983 if (n_subcores > 1) { 1984 if (!(dynamic_mt_modes & 2)) 1985 n_subcores = 4; 1986 if (n_subcores > 2 && !(dynamic_mt_modes & 4)) 1987 return false; 1988 } 1989 1990 return n_subcores * roundup_pow_of_two(n_threads) <= MAX_SMT_THREADS; 1991 } 1992 1993 static void init_master_vcore(struct kvmppc_vcore *vc) 1994 { 1995 vc->master_vcore = vc; 1996 vc->entry_exit_map = 0; 1997 vc->in_guest = 0; 1998 vc->napping_threads = 0; 1999 vc->conferring_threads = 0; 2000 } 2001 2002 /* 2003 * See if the existing subcores can be split into 3 (or fewer) subcores 2004 * of at most two threads each, so we can fit in another vcore. This 2005 * assumes there are at most two subcores and at most 6 threads in total. 2006 */ 2007 static bool can_split_piggybacked_subcores(struct core_info *cip) 2008 { 2009 int sub, new_sub; 2010 int large_sub = -1; 2011 int thr; 2012 int n_subcores = cip->n_subcores; 2013 struct kvmppc_vcore *vc, *vcnext; 2014 struct kvmppc_vcore *master_vc = NULL; 2015 2016 for (sub = 0; sub < cip->n_subcores; ++sub) { 2017 if (cip->subcore_threads[sub] <= 2) 2018 continue; 2019 if (large_sub >= 0) 2020 return false; 2021 large_sub = sub; 2022 vc = list_first_entry(&cip->vcs[sub], struct kvmppc_vcore, 2023 preempt_list); 2024 if (vc->num_threads > 2) 2025 return false; 2026 n_subcores += (cip->subcore_threads[sub] - 1) >> 1; 2027 } 2028 if (large_sub < 0 || !subcore_config_ok(n_subcores + 1, 2)) 2029 return false; 2030 2031 /* 2032 * Seems feasible, so go through and move vcores to new subcores. 2033 * Note that when we have two or more vcores in one subcore, 2034 * all those vcores must have only one thread each. 2035 */ 2036 new_sub = cip->n_subcores; 2037 thr = 0; 2038 sub = large_sub; 2039 list_for_each_entry_safe(vc, vcnext, &cip->vcs[sub], preempt_list) { 2040 if (thr >= 2) { 2041 list_del(&vc->preempt_list); 2042 list_add_tail(&vc->preempt_list, &cip->vcs[new_sub]); 2043 /* vc->num_threads must be 1 */ 2044 if (++cip->subcore_threads[new_sub] == 1) { 2045 cip->subcore_vm[new_sub] = vc->kvm; 2046 init_master_vcore(vc); 2047 master_vc = vc; 2048 ++cip->n_subcores; 2049 } else { 2050 vc->master_vcore = master_vc; 2051 ++new_sub; 2052 } 2053 } 2054 thr += vc->num_threads; 2055 } 2056 cip->subcore_threads[large_sub] = 2; 2057 cip->max_subcore_threads = 2; 2058 2059 return true; 2060 } 2061 2062 static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip) 2063 { 2064 int n_threads = vc->num_threads; 2065 int sub; 2066 2067 if (!cpu_has_feature(CPU_FTR_ARCH_207S)) 2068 return false; 2069 2070 if (n_threads < cip->max_subcore_threads) 2071 n_threads = cip->max_subcore_threads; 2072 if (subcore_config_ok(cip->n_subcores + 1, n_threads)) { 2073 cip->max_subcore_threads = n_threads; 2074 } else if (cip->n_subcores <= 2 && cip->total_threads <= 6 && 2075 vc->num_threads <= 2) { 2076 /* 2077 * We may be able to fit another subcore in by 2078 * splitting an existing subcore with 3 or 4 2079 * threads into two 2-thread subcores, or one 2080 * with 5 or 6 threads into three subcores. 2081 * We can only do this if those subcores have 2082 * piggybacked virtual cores. 2083 */ 2084 if (!can_split_piggybacked_subcores(cip)) 2085 return false; 2086 } else { 2087 return false; 2088 } 2089 2090 sub = cip->n_subcores; 2091 ++cip->n_subcores; 2092 cip->total_threads += vc->num_threads; 2093 cip->subcore_threads[sub] = vc->num_threads; 2094 cip->subcore_vm[sub] = vc->kvm; 2095 init_master_vcore(vc); 2096 list_del(&vc->preempt_list); 2097 list_add_tail(&vc->preempt_list, &cip->vcs[sub]); 2098 2099 return true; 2100 } 2101 2102 static bool can_piggyback_subcore(struct kvmppc_vcore *pvc, 2103 struct core_info *cip, int sub) 2104 { 2105 struct kvmppc_vcore *vc; 2106 int n_thr; 2107 2108 vc = list_first_entry(&cip->vcs[sub], struct kvmppc_vcore, 2109 preempt_list); 2110 2111 /* require same VM and same per-core reg values */ 2112 if (pvc->kvm != vc->kvm || 2113 pvc->tb_offset != vc->tb_offset || 2114 pvc->pcr != vc->pcr || 2115 pvc->lpcr != vc->lpcr) 2116 return false; 2117 2118 /* P8 guest with > 1 thread per core would see wrong TIR value */ 2119 if (cpu_has_feature(CPU_FTR_ARCH_207S) && 2120 (vc->num_threads > 1 || pvc->num_threads > 1)) 2121 return false; 2122 2123 n_thr = cip->subcore_threads[sub] + pvc->num_threads; 2124 if (n_thr > cip->max_subcore_threads) { 2125 if (!subcore_config_ok(cip->n_subcores, n_thr)) 2126 return false; 2127 cip->max_subcore_threads = n_thr; 2128 } 2129 2130 cip->total_threads += pvc->num_threads; 2131 cip->subcore_threads[sub] = n_thr; 2132 pvc->master_vcore = vc; 2133 list_del(&pvc->preempt_list); 2134 list_add_tail(&pvc->preempt_list, &cip->vcs[sub]); 2135 2136 return true; 2137 } 2138 2139 /* 2140 * Work out whether it is possible to piggyback the execution of 2141 * vcore *pvc onto the execution of the other vcores described in *cip. 2142 */ 2143 static bool can_piggyback(struct kvmppc_vcore *pvc, struct core_info *cip, 2144 int target_threads) 2145 { 2146 int sub; 2147 2148 if (cip->total_threads + pvc->num_threads > target_threads) 2149 return false; 2150 for (sub = 0; sub < cip->n_subcores; ++sub) 2151 if (cip->subcore_threads[sub] && 2152 can_piggyback_subcore(pvc, cip, sub)) 2153 return true; 2154 2155 if (can_dynamic_split(pvc, cip)) 2156 return true; 2157 2158 return false; 2159 } 2160 2161 static void prepare_threads(struct kvmppc_vcore *vc) 2162 { 2163 struct kvm_vcpu *vcpu, *vnext; 2164 2165 list_for_each_entry_safe(vcpu, vnext, &vc->runnable_threads, 2166 arch.run_list) { 2167 if (signal_pending(vcpu->arch.run_task)) 2168 vcpu->arch.ret = -EINTR; 2169 else if (vcpu->arch.vpa.update_pending || 2170 vcpu->arch.slb_shadow.update_pending || 2171 vcpu->arch.dtl.update_pending) 2172 vcpu->arch.ret = RESUME_GUEST; 2173 else 2174 continue; 2175 kvmppc_remove_runnable(vc, vcpu); 2176 wake_up(&vcpu->arch.cpu_run); 2177 } 2178 } 2179 2180 static void collect_piggybacks(struct core_info *cip, int target_threads) 2181 { 2182 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores); 2183 struct kvmppc_vcore *pvc, *vcnext; 2184 2185 spin_lock(&lp->lock); 2186 list_for_each_entry_safe(pvc, vcnext, &lp->list, preempt_list) { 2187 if (!spin_trylock(&pvc->lock)) 2188 continue; 2189 prepare_threads(pvc); 2190 if (!pvc->n_runnable) { 2191 list_del_init(&pvc->preempt_list); 2192 if (pvc->runner == NULL) { 2193 pvc->vcore_state = VCORE_INACTIVE; 2194 kvmppc_core_end_stolen(pvc); 2195 } 2196 spin_unlock(&pvc->lock); 2197 continue; 2198 } 2199 if (!can_piggyback(pvc, cip, target_threads)) { 2200 spin_unlock(&pvc->lock); 2201 continue; 2202 } 2203 kvmppc_core_end_stolen(pvc); 2204 pvc->vcore_state = VCORE_PIGGYBACK; 2205 if (cip->total_threads >= target_threads) 2206 break; 2207 } 2208 spin_unlock(&lp->lock); 2209 } 2210 2211 static void post_guest_process(struct kvmppc_vcore *vc, bool is_master) 2212 { 2213 int still_running = 0; 2214 u64 now; 2215 long ret; 2216 struct kvm_vcpu *vcpu, *vnext; 2217 2218 spin_lock(&vc->lock); 2219 now = get_tb(); 2220 list_for_each_entry_safe(vcpu, vnext, &vc->runnable_threads, 2221 arch.run_list) { 2222 /* cancel pending dec exception if dec is positive */ 2223 if (now < vcpu->arch.dec_expires && 2224 kvmppc_core_pending_dec(vcpu)) 2225 kvmppc_core_dequeue_dec(vcpu); 2226 2227 trace_kvm_guest_exit(vcpu); 2228 2229 ret = RESUME_GUEST; 2230 if (vcpu->arch.trap) 2231 ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu, 2232 vcpu->arch.run_task); 2233 2234 vcpu->arch.ret = ret; 2235 vcpu->arch.trap = 0; 2236 2237 if (is_kvmppc_resume_guest(vcpu->arch.ret)) { 2238 if (vcpu->arch.pending_exceptions) 2239 kvmppc_core_prepare_to_enter(vcpu); 2240 if (vcpu->arch.ceded) 2241 kvmppc_set_timer(vcpu); 2242 else 2243 ++still_running; 2244 } else { 2245 kvmppc_remove_runnable(vc, vcpu); 2246 wake_up(&vcpu->arch.cpu_run); 2247 } 2248 } 2249 list_del_init(&vc->preempt_list); 2250 if (!is_master) { 2251 if (still_running > 0) { 2252 kvmppc_vcore_preempt(vc); 2253 } else if (vc->runner) { 2254 vc->vcore_state = VCORE_PREEMPT; 2255 kvmppc_core_start_stolen(vc); 2256 } else { 2257 vc->vcore_state = VCORE_INACTIVE; 2258 } 2259 if (vc->n_runnable > 0 && vc->runner == NULL) { 2260 /* make sure there's a candidate runner awake */ 2261 vcpu = list_first_entry(&vc->runnable_threads, 2262 struct kvm_vcpu, arch.run_list); 2263 wake_up(&vcpu->arch.cpu_run); 2264 } 2265 } 2266 spin_unlock(&vc->lock); 2267 } 2268 2269 /* 2270 * Run a set of guest threads on a physical core. 2271 * Called with vc->lock held. 2272 */ 2273 static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) 2274 { 2275 struct kvm_vcpu *vcpu, *vnext; 2276 int i; 2277 int srcu_idx; 2278 struct core_info core_info; 2279 struct kvmppc_vcore *pvc, *vcnext; 2280 struct kvm_split_mode split_info, *sip; 2281 int split, subcore_size, active; 2282 int sub; 2283 bool thr0_done; 2284 unsigned long cmd_bit, stat_bit; 2285 int pcpu, thr; 2286 int target_threads; 2287 2288 /* 2289 * Remove from the list any threads that have a signal pending 2290 * or need a VPA update done 2291 */ 2292 prepare_threads(vc); 2293 2294 /* if the runner is no longer runnable, let the caller pick a new one */ 2295 if (vc->runner->arch.state != KVMPPC_VCPU_RUNNABLE) 2296 return; 2297 2298 /* 2299 * Initialize *vc. 2300 */ 2301 init_master_vcore(vc); 2302 vc->preempt_tb = TB_NIL; 2303 2304 /* 2305 * Make sure we are running on primary threads, and that secondary 2306 * threads are offline. Also check if the number of threads in this 2307 * guest are greater than the current system threads per guest. 2308 */ 2309 if ((threads_per_core > 1) && 2310 ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) { 2311 list_for_each_entry_safe(vcpu, vnext, &vc->runnable_threads, 2312 arch.run_list) { 2313 vcpu->arch.ret = -EBUSY; 2314 kvmppc_remove_runnable(vc, vcpu); 2315 wake_up(&vcpu->arch.cpu_run); 2316 } 2317 goto out; 2318 } 2319 2320 /* 2321 * See if we could run any other vcores on the physical core 2322 * along with this one. 2323 */ 2324 init_core_info(&core_info, vc); 2325 pcpu = smp_processor_id(); 2326 target_threads = threads_per_subcore; 2327 if (target_smt_mode && target_smt_mode < target_threads) 2328 target_threads = target_smt_mode; 2329 if (vc->num_threads < target_threads) 2330 collect_piggybacks(&core_info, target_threads); 2331 2332 /* Decide on micro-threading (split-core) mode */ 2333 subcore_size = threads_per_subcore; 2334 cmd_bit = stat_bit = 0; 2335 split = core_info.n_subcores; 2336 sip = NULL; 2337 if (split > 1) { 2338 /* threads_per_subcore must be MAX_SMT_THREADS (8) here */ 2339 if (split == 2 && (dynamic_mt_modes & 2)) { 2340 cmd_bit = HID0_POWER8_1TO2LPAR; 2341 stat_bit = HID0_POWER8_2LPARMODE; 2342 } else { 2343 split = 4; 2344 cmd_bit = HID0_POWER8_1TO4LPAR; 2345 stat_bit = HID0_POWER8_4LPARMODE; 2346 } 2347 subcore_size = MAX_SMT_THREADS / split; 2348 sip = &split_info; 2349 memset(&split_info, 0, sizeof(split_info)); 2350 split_info.rpr = mfspr(SPRN_RPR); 2351 split_info.pmmar = mfspr(SPRN_PMMAR); 2352 split_info.ldbar = mfspr(SPRN_LDBAR); 2353 split_info.subcore_size = subcore_size; 2354 for (sub = 0; sub < core_info.n_subcores; ++sub) 2355 split_info.master_vcs[sub] = 2356 list_first_entry(&core_info.vcs[sub], 2357 struct kvmppc_vcore, preempt_list); 2358 /* order writes to split_info before kvm_split_mode pointer */ 2359 smp_wmb(); 2360 } 2361 pcpu = smp_processor_id(); 2362 for (thr = 0; thr < threads_per_subcore; ++thr) 2363 paca[pcpu + thr].kvm_hstate.kvm_split_mode = sip; 2364 2365 /* Initiate micro-threading (split-core) if required */ 2366 if (cmd_bit) { 2367 unsigned long hid0 = mfspr(SPRN_HID0); 2368 2369 hid0 |= cmd_bit | HID0_POWER8_DYNLPARDIS; 2370 mb(); 2371 mtspr(SPRN_HID0, hid0); 2372 isync(); 2373 for (;;) { 2374 hid0 = mfspr(SPRN_HID0); 2375 if (hid0 & stat_bit) 2376 break; 2377 cpu_relax(); 2378 } 2379 } 2380 2381 /* Start all the threads */ 2382 active = 0; 2383 for (sub = 0; sub < core_info.n_subcores; ++sub) { 2384 thr = subcore_thread_map[sub]; 2385 thr0_done = false; 2386 active |= 1 << thr; 2387 list_for_each_entry(pvc, &core_info.vcs[sub], preempt_list) { 2388 pvc->pcpu = pcpu + thr; 2389 list_for_each_entry(vcpu, &pvc->runnable_threads, 2390 arch.run_list) { 2391 kvmppc_start_thread(vcpu, pvc); 2392 kvmppc_create_dtl_entry(vcpu, pvc); 2393 trace_kvm_guest_enter(vcpu); 2394 if (!vcpu->arch.ptid) 2395 thr0_done = true; 2396 active |= 1 << (thr + vcpu->arch.ptid); 2397 } 2398 /* 2399 * We need to start the first thread of each subcore 2400 * even if it doesn't have a vcpu. 2401 */ 2402 if (pvc->master_vcore == pvc && !thr0_done) 2403 kvmppc_start_thread(NULL, pvc); 2404 thr += pvc->num_threads; 2405 } 2406 } 2407 2408 /* 2409 * Ensure that split_info.do_nap is set after setting 2410 * the vcore pointer in the PACA of the secondaries. 2411 */ 2412 smp_mb(); 2413 if (cmd_bit) 2414 split_info.do_nap = 1; /* ask secondaries to nap when done */ 2415 2416 /* 2417 * When doing micro-threading, poke the inactive threads as well. 2418 * This gets them to the nap instruction after kvm_do_nap, 2419 * which reduces the time taken to unsplit later. 2420 */ 2421 if (split > 1) 2422 for (thr = 1; thr < threads_per_subcore; ++thr) 2423 if (!(active & (1 << thr))) 2424 kvmppc_ipi_thread(pcpu + thr); 2425 2426 vc->vcore_state = VCORE_RUNNING; 2427 preempt_disable(); 2428 2429 trace_kvmppc_run_core(vc, 0); 2430 2431 for (sub = 0; sub < core_info.n_subcores; ++sub) 2432 list_for_each_entry(pvc, &core_info.vcs[sub], preempt_list) 2433 spin_unlock(&pvc->lock); 2434 2435 kvm_guest_enter(); 2436 2437 srcu_idx = srcu_read_lock(&vc->kvm->srcu); 2438 2439 __kvmppc_vcore_entry(); 2440 2441 srcu_read_unlock(&vc->kvm->srcu, srcu_idx); 2442 2443 spin_lock(&vc->lock); 2444 /* prevent other vcpu threads from doing kvmppc_start_thread() now */ 2445 vc->vcore_state = VCORE_EXITING; 2446 2447 /* wait for secondary threads to finish writing their state to memory */ 2448 kvmppc_wait_for_nap(); 2449 2450 /* Return to whole-core mode if we split the core earlier */ 2451 if (split > 1) { 2452 unsigned long hid0 = mfspr(SPRN_HID0); 2453 unsigned long loops = 0; 2454 2455 hid0 &= ~HID0_POWER8_DYNLPARDIS; 2456 stat_bit = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE; 2457 mb(); 2458 mtspr(SPRN_HID0, hid0); 2459 isync(); 2460 for (;;) { 2461 hid0 = mfspr(SPRN_HID0); 2462 if (!(hid0 & stat_bit)) 2463 break; 2464 cpu_relax(); 2465 ++loops; 2466 } 2467 split_info.do_nap = 0; 2468 } 2469 2470 /* Let secondaries go back to the offline loop */ 2471 for (i = 0; i < threads_per_subcore; ++i) { 2472 kvmppc_release_hwthread(pcpu + i); 2473 if (sip && sip->napped[i]) 2474 kvmppc_ipi_thread(pcpu + i); 2475 } 2476 2477 spin_unlock(&vc->lock); 2478 2479 /* make sure updates to secondary vcpu structs are visible now */ 2480 smp_mb(); 2481 kvm_guest_exit(); 2482 2483 for (sub = 0; sub < core_info.n_subcores; ++sub) 2484 list_for_each_entry_safe(pvc, vcnext, &core_info.vcs[sub], 2485 preempt_list) 2486 post_guest_process(pvc, pvc == vc); 2487 2488 spin_lock(&vc->lock); 2489 preempt_enable(); 2490 2491 out: 2492 vc->vcore_state = VCORE_INACTIVE; 2493 trace_kvmppc_run_core(vc, 1); 2494 } 2495 2496 /* 2497 * Wait for some other vcpu thread to execute us, and 2498 * wake us up when we need to handle something in the host. 2499 */ 2500 static void kvmppc_wait_for_exec(struct kvmppc_vcore *vc, 2501 struct kvm_vcpu *vcpu, int wait_state) 2502 { 2503 DEFINE_WAIT(wait); 2504 2505 prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state); 2506 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) { 2507 spin_unlock(&vc->lock); 2508 schedule(); 2509 spin_lock(&vc->lock); 2510 } 2511 finish_wait(&vcpu->arch.cpu_run, &wait); 2512 } 2513 2514 /* 2515 * All the vcpus in this vcore are idle, so wait for a decrementer 2516 * or external interrupt to one of the vcpus. vc->lock is held. 2517 */ 2518 static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) 2519 { 2520 struct kvm_vcpu *vcpu; 2521 int do_sleep = 1; 2522 2523 DEFINE_WAIT(wait); 2524 2525 prepare_to_wait(&vc->wq, &wait, TASK_INTERRUPTIBLE); 2526 2527 /* 2528 * Check one last time for pending exceptions and ceded state after 2529 * we put ourselves on the wait queue 2530 */ 2531 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) { 2532 if (vcpu->arch.pending_exceptions || !vcpu->arch.ceded) { 2533 do_sleep = 0; 2534 break; 2535 } 2536 } 2537 2538 if (!do_sleep) { 2539 finish_wait(&vc->wq, &wait); 2540 return; 2541 } 2542 2543 vc->vcore_state = VCORE_SLEEPING; 2544 trace_kvmppc_vcore_blocked(vc, 0); 2545 spin_unlock(&vc->lock); 2546 schedule(); 2547 finish_wait(&vc->wq, &wait); 2548 spin_lock(&vc->lock); 2549 vc->vcore_state = VCORE_INACTIVE; 2550 trace_kvmppc_vcore_blocked(vc, 1); 2551 } 2552 2553 static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) 2554 { 2555 int n_ceded; 2556 struct kvmppc_vcore *vc; 2557 struct kvm_vcpu *v, *vn; 2558 2559 trace_kvmppc_run_vcpu_enter(vcpu); 2560 2561 kvm_run->exit_reason = 0; 2562 vcpu->arch.ret = RESUME_GUEST; 2563 vcpu->arch.trap = 0; 2564 kvmppc_update_vpas(vcpu); 2565 2566 /* 2567 * Synchronize with other threads in this virtual core 2568 */ 2569 vc = vcpu->arch.vcore; 2570 spin_lock(&vc->lock); 2571 vcpu->arch.ceded = 0; 2572 vcpu->arch.run_task = current; 2573 vcpu->arch.kvm_run = kvm_run; 2574 vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb()); 2575 vcpu->arch.state = KVMPPC_VCPU_RUNNABLE; 2576 vcpu->arch.busy_preempt = TB_NIL; 2577 list_add_tail(&vcpu->arch.run_list, &vc->runnable_threads); 2578 ++vc->n_runnable; 2579 2580 /* 2581 * This happens the first time this is called for a vcpu. 2582 * If the vcore is already running, we may be able to start 2583 * this thread straight away and have it join in. 2584 */ 2585 if (!signal_pending(current)) { 2586 if (vc->vcore_state == VCORE_PIGGYBACK) { 2587 struct kvmppc_vcore *mvc = vc->master_vcore; 2588 if (spin_trylock(&mvc->lock)) { 2589 if (mvc->vcore_state == VCORE_RUNNING && 2590 !VCORE_IS_EXITING(mvc)) { 2591 kvmppc_create_dtl_entry(vcpu, vc); 2592 kvmppc_start_thread(vcpu, vc); 2593 trace_kvm_guest_enter(vcpu); 2594 } 2595 spin_unlock(&mvc->lock); 2596 } 2597 } else if (vc->vcore_state == VCORE_RUNNING && 2598 !VCORE_IS_EXITING(vc)) { 2599 kvmppc_create_dtl_entry(vcpu, vc); 2600 kvmppc_start_thread(vcpu, vc); 2601 trace_kvm_guest_enter(vcpu); 2602 } else if (vc->vcore_state == VCORE_SLEEPING) { 2603 wake_up(&vc->wq); 2604 } 2605 2606 } 2607 2608 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE && 2609 !signal_pending(current)) { 2610 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL) 2611 kvmppc_vcore_end_preempt(vc); 2612 2613 if (vc->vcore_state != VCORE_INACTIVE) { 2614 kvmppc_wait_for_exec(vc, vcpu, TASK_INTERRUPTIBLE); 2615 continue; 2616 } 2617 list_for_each_entry_safe(v, vn, &vc->runnable_threads, 2618 arch.run_list) { 2619 kvmppc_core_prepare_to_enter(v); 2620 if (signal_pending(v->arch.run_task)) { 2621 kvmppc_remove_runnable(vc, v); 2622 v->stat.signal_exits++; 2623 v->arch.kvm_run->exit_reason = KVM_EXIT_INTR; 2624 v->arch.ret = -EINTR; 2625 wake_up(&v->arch.cpu_run); 2626 } 2627 } 2628 if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE) 2629 break; 2630 n_ceded = 0; 2631 list_for_each_entry(v, &vc->runnable_threads, arch.run_list) { 2632 if (!v->arch.pending_exceptions) 2633 n_ceded += v->arch.ceded; 2634 else 2635 v->arch.ceded = 0; 2636 } 2637 vc->runner = vcpu; 2638 if (n_ceded == vc->n_runnable) { 2639 kvmppc_vcore_blocked(vc); 2640 } else if (need_resched()) { 2641 kvmppc_vcore_preempt(vc); 2642 /* Let something else run */ 2643 cond_resched_lock(&vc->lock); 2644 if (vc->vcore_state == VCORE_PREEMPT) 2645 kvmppc_vcore_end_preempt(vc); 2646 } else { 2647 kvmppc_run_core(vc); 2648 } 2649 vc->runner = NULL; 2650 } 2651 2652 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE && 2653 (vc->vcore_state == VCORE_RUNNING || 2654 vc->vcore_state == VCORE_EXITING || 2655 vc->vcore_state == VCORE_PIGGYBACK)) 2656 kvmppc_wait_for_exec(vc, vcpu, TASK_UNINTERRUPTIBLE); 2657 2658 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL) 2659 kvmppc_vcore_end_preempt(vc); 2660 2661 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) { 2662 kvmppc_remove_runnable(vc, vcpu); 2663 vcpu->stat.signal_exits++; 2664 kvm_run->exit_reason = KVM_EXIT_INTR; 2665 vcpu->arch.ret = -EINTR; 2666 } 2667 2668 if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) { 2669 /* Wake up some vcpu to run the core */ 2670 v = list_first_entry(&vc->runnable_threads, 2671 struct kvm_vcpu, arch.run_list); 2672 wake_up(&v->arch.cpu_run); 2673 } 2674 2675 trace_kvmppc_run_vcpu_exit(vcpu, kvm_run); 2676 spin_unlock(&vc->lock); 2677 return vcpu->arch.ret; 2678 } 2679 2680 static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu) 2681 { 2682 int r; 2683 int srcu_idx; 2684 2685 if (!vcpu->arch.sane) { 2686 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 2687 return -EINVAL; 2688 } 2689 2690 kvmppc_core_prepare_to_enter(vcpu); 2691 2692 /* No need to go into the guest when all we'll do is come back out */ 2693 if (signal_pending(current)) { 2694 run->exit_reason = KVM_EXIT_INTR; 2695 return -EINTR; 2696 } 2697 2698 atomic_inc(&vcpu->kvm->arch.vcpus_running); 2699 /* Order vcpus_running vs. hpte_setup_done, see kvmppc_alloc_reset_hpt */ 2700 smp_mb(); 2701 2702 /* On the first time here, set up HTAB and VRMA */ 2703 if (!vcpu->kvm->arch.hpte_setup_done) { 2704 r = kvmppc_hv_setup_htab_rma(vcpu); 2705 if (r) 2706 goto out; 2707 } 2708 2709 flush_fp_to_thread(current); 2710 flush_altivec_to_thread(current); 2711 flush_vsx_to_thread(current); 2712 vcpu->arch.wqp = &vcpu->arch.vcore->wq; 2713 vcpu->arch.pgdir = current->mm->pgd; 2714 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST; 2715 2716 do { 2717 r = kvmppc_run_vcpu(run, vcpu); 2718 2719 if (run->exit_reason == KVM_EXIT_PAPR_HCALL && 2720 !(vcpu->arch.shregs.msr & MSR_PR)) { 2721 trace_kvm_hcall_enter(vcpu); 2722 r = kvmppc_pseries_do_hcall(vcpu); 2723 trace_kvm_hcall_exit(vcpu, r); 2724 kvmppc_core_prepare_to_enter(vcpu); 2725 } else if (r == RESUME_PAGE_FAULT) { 2726 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 2727 r = kvmppc_book3s_hv_page_fault(run, vcpu, 2728 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr); 2729 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); 2730 } 2731 } while (is_kvmppc_resume_guest(r)); 2732 2733 out: 2734 vcpu->arch.state = KVMPPC_VCPU_NOTREADY; 2735 atomic_dec(&vcpu->kvm->arch.vcpus_running); 2736 return r; 2737 } 2738 2739 static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps, 2740 int linux_psize) 2741 { 2742 struct mmu_psize_def *def = &mmu_psize_defs[linux_psize]; 2743 2744 if (!def->shift) 2745 return; 2746 (*sps)->page_shift = def->shift; 2747 (*sps)->slb_enc = def->sllp; 2748 (*sps)->enc[0].page_shift = def->shift; 2749 (*sps)->enc[0].pte_enc = def->penc[linux_psize]; 2750 /* 2751 * Add 16MB MPSS support if host supports it 2752 */ 2753 if (linux_psize != MMU_PAGE_16M && def->penc[MMU_PAGE_16M] != -1) { 2754 (*sps)->enc[1].page_shift = 24; 2755 (*sps)->enc[1].pte_enc = def->penc[MMU_PAGE_16M]; 2756 } 2757 (*sps)++; 2758 } 2759 2760 static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm, 2761 struct kvm_ppc_smmu_info *info) 2762 { 2763 struct kvm_ppc_one_seg_page_size *sps; 2764 2765 info->flags = KVM_PPC_PAGE_SIZES_REAL; 2766 if (mmu_has_feature(MMU_FTR_1T_SEGMENT)) 2767 info->flags |= KVM_PPC_1T_SEGMENTS; 2768 info->slb_size = mmu_slb_size; 2769 2770 /* We only support these sizes for now, and no muti-size segments */ 2771 sps = &info->sps[0]; 2772 kvmppc_add_seg_page_size(&sps, MMU_PAGE_4K); 2773 kvmppc_add_seg_page_size(&sps, MMU_PAGE_64K); 2774 kvmppc_add_seg_page_size(&sps, MMU_PAGE_16M); 2775 2776 return 0; 2777 } 2778 2779 /* 2780 * Get (and clear) the dirty memory log for a memory slot. 2781 */ 2782 static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm, 2783 struct kvm_dirty_log *log) 2784 { 2785 struct kvm_memslots *slots; 2786 struct kvm_memory_slot *memslot; 2787 int r; 2788 unsigned long n; 2789 2790 mutex_lock(&kvm->slots_lock); 2791 2792 r = -EINVAL; 2793 if (log->slot >= KVM_USER_MEM_SLOTS) 2794 goto out; 2795 2796 slots = kvm_memslots(kvm); 2797 memslot = id_to_memslot(slots, log->slot); 2798 r = -ENOENT; 2799 if (!memslot->dirty_bitmap) 2800 goto out; 2801 2802 n = kvm_dirty_bitmap_bytes(memslot); 2803 memset(memslot->dirty_bitmap, 0, n); 2804 2805 r = kvmppc_hv_get_dirty_log(kvm, memslot, memslot->dirty_bitmap); 2806 if (r) 2807 goto out; 2808 2809 r = -EFAULT; 2810 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 2811 goto out; 2812 2813 r = 0; 2814 out: 2815 mutex_unlock(&kvm->slots_lock); 2816 return r; 2817 } 2818 2819 static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *free, 2820 struct kvm_memory_slot *dont) 2821 { 2822 if (!dont || free->arch.rmap != dont->arch.rmap) { 2823 vfree(free->arch.rmap); 2824 free->arch.rmap = NULL; 2825 } 2826 } 2827 2828 static int kvmppc_core_create_memslot_hv(struct kvm_memory_slot *slot, 2829 unsigned long npages) 2830 { 2831 slot->arch.rmap = vzalloc(npages * sizeof(*slot->arch.rmap)); 2832 if (!slot->arch.rmap) 2833 return -ENOMEM; 2834 2835 return 0; 2836 } 2837 2838 static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm, 2839 struct kvm_memory_slot *memslot, 2840 const struct kvm_userspace_memory_region *mem) 2841 { 2842 return 0; 2843 } 2844 2845 static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm, 2846 const struct kvm_userspace_memory_region *mem, 2847 const struct kvm_memory_slot *old, 2848 const struct kvm_memory_slot *new) 2849 { 2850 unsigned long npages = mem->memory_size >> PAGE_SHIFT; 2851 struct kvm_memslots *slots; 2852 struct kvm_memory_slot *memslot; 2853 2854 if (npages && old->npages) { 2855 /* 2856 * If modifying a memslot, reset all the rmap dirty bits. 2857 * If this is a new memslot, we don't need to do anything 2858 * since the rmap array starts out as all zeroes, 2859 * i.e. no pages are dirty. 2860 */ 2861 slots = kvm_memslots(kvm); 2862 memslot = id_to_memslot(slots, mem->slot); 2863 kvmppc_hv_get_dirty_log(kvm, memslot, NULL); 2864 } 2865 } 2866 2867 /* 2868 * Update LPCR values in kvm->arch and in vcores. 2869 * Caller must hold kvm->lock. 2870 */ 2871 void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask) 2872 { 2873 long int i; 2874 u32 cores_done = 0; 2875 2876 if ((kvm->arch.lpcr & mask) == lpcr) 2877 return; 2878 2879 kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr; 2880 2881 for (i = 0; i < KVM_MAX_VCORES; ++i) { 2882 struct kvmppc_vcore *vc = kvm->arch.vcores[i]; 2883 if (!vc) 2884 continue; 2885 spin_lock(&vc->lock); 2886 vc->lpcr = (vc->lpcr & ~mask) | lpcr; 2887 spin_unlock(&vc->lock); 2888 if (++cores_done >= kvm->arch.online_vcores) 2889 break; 2890 } 2891 } 2892 2893 static void kvmppc_mmu_destroy_hv(struct kvm_vcpu *vcpu) 2894 { 2895 return; 2896 } 2897 2898 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu) 2899 { 2900 int err = 0; 2901 struct kvm *kvm = vcpu->kvm; 2902 unsigned long hva; 2903 struct kvm_memory_slot *memslot; 2904 struct vm_area_struct *vma; 2905 unsigned long lpcr = 0, senc; 2906 unsigned long psize, porder; 2907 int srcu_idx; 2908 2909 mutex_lock(&kvm->lock); 2910 if (kvm->arch.hpte_setup_done) 2911 goto out; /* another vcpu beat us to it */ 2912 2913 /* Allocate hashed page table (if not done already) and reset it */ 2914 if (!kvm->arch.hpt_virt) { 2915 err = kvmppc_alloc_hpt(kvm, NULL); 2916 if (err) { 2917 pr_err("KVM: Couldn't alloc HPT\n"); 2918 goto out; 2919 } 2920 } 2921 2922 /* Look up the memslot for guest physical address 0 */ 2923 srcu_idx = srcu_read_lock(&kvm->srcu); 2924 memslot = gfn_to_memslot(kvm, 0); 2925 2926 /* We must have some memory at 0 by now */ 2927 err = -EINVAL; 2928 if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) 2929 goto out_srcu; 2930 2931 /* Look up the VMA for the start of this memory slot */ 2932 hva = memslot->userspace_addr; 2933 down_read(¤t->mm->mmap_sem); 2934 vma = find_vma(current->mm, hva); 2935 if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO)) 2936 goto up_out; 2937 2938 psize = vma_kernel_pagesize(vma); 2939 porder = __ilog2(psize); 2940 2941 up_read(¤t->mm->mmap_sem); 2942 2943 /* We can handle 4k, 64k or 16M pages in the VRMA */ 2944 err = -EINVAL; 2945 if (!(psize == 0x1000 || psize == 0x10000 || 2946 psize == 0x1000000)) 2947 goto out_srcu; 2948 2949 /* Update VRMASD field in the LPCR */ 2950 senc = slb_pgsize_encoding(psize); 2951 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T | 2952 (VRMA_VSID << SLB_VSID_SHIFT_1T); 2953 /* the -4 is to account for senc values starting at 0x10 */ 2954 lpcr = senc << (LPCR_VRMASD_SH - 4); 2955 2956 /* Create HPTEs in the hash page table for the VRMA */ 2957 kvmppc_map_vrma(vcpu, memslot, porder); 2958 2959 kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD); 2960 2961 /* Order updates to kvm->arch.lpcr etc. vs. hpte_setup_done */ 2962 smp_wmb(); 2963 kvm->arch.hpte_setup_done = 1; 2964 err = 0; 2965 out_srcu: 2966 srcu_read_unlock(&kvm->srcu, srcu_idx); 2967 out: 2968 mutex_unlock(&kvm->lock); 2969 return err; 2970 2971 up_out: 2972 up_read(¤t->mm->mmap_sem); 2973 goto out_srcu; 2974 } 2975 2976 static int kvmppc_core_init_vm_hv(struct kvm *kvm) 2977 { 2978 unsigned long lpcr, lpid; 2979 char buf[32]; 2980 2981 /* Allocate the guest's logical partition ID */ 2982 2983 lpid = kvmppc_alloc_lpid(); 2984 if ((long)lpid < 0) 2985 return -ENOMEM; 2986 kvm->arch.lpid = lpid; 2987 2988 /* 2989 * Since we don't flush the TLB when tearing down a VM, 2990 * and this lpid might have previously been used, 2991 * make sure we flush on each core before running the new VM. 2992 */ 2993 cpumask_setall(&kvm->arch.need_tlb_flush); 2994 2995 /* Start out with the default set of hcalls enabled */ 2996 memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls, 2997 sizeof(kvm->arch.enabled_hcalls)); 2998 2999 kvm->arch.host_sdr1 = mfspr(SPRN_SDR1); 3000 3001 /* Init LPCR for virtual RMA mode */ 3002 kvm->arch.host_lpid = mfspr(SPRN_LPID); 3003 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR); 3004 lpcr &= LPCR_PECE | LPCR_LPES; 3005 lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE | 3006 LPCR_VPM0 | LPCR_VPM1; 3007 kvm->arch.vrma_slb_v = SLB_VSID_B_1T | 3008 (VRMA_VSID << SLB_VSID_SHIFT_1T); 3009 /* On POWER8 turn on online bit to enable PURR/SPURR */ 3010 if (cpu_has_feature(CPU_FTR_ARCH_207S)) 3011 lpcr |= LPCR_ONL; 3012 kvm->arch.lpcr = lpcr; 3013 3014 /* 3015 * Track that we now have a HV mode VM active. This blocks secondary 3016 * CPU threads from coming online. 3017 */ 3018 kvm_hv_vm_activated(); 3019 3020 /* 3021 * Create a debugfs directory for the VM 3022 */ 3023 snprintf(buf, sizeof(buf), "vm%d", current->pid); 3024 kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir); 3025 if (!IS_ERR_OR_NULL(kvm->arch.debugfs_dir)) 3026 kvmppc_mmu_debugfs_init(kvm); 3027 3028 return 0; 3029 } 3030 3031 static void kvmppc_free_vcores(struct kvm *kvm) 3032 { 3033 long int i; 3034 3035 for (i = 0; i < KVM_MAX_VCORES; ++i) 3036 kfree(kvm->arch.vcores[i]); 3037 kvm->arch.online_vcores = 0; 3038 } 3039 3040 static void kvmppc_core_destroy_vm_hv(struct kvm *kvm) 3041 { 3042 debugfs_remove_recursive(kvm->arch.debugfs_dir); 3043 3044 kvm_hv_vm_deactivated(); 3045 3046 kvmppc_free_vcores(kvm); 3047 3048 kvmppc_free_hpt(kvm); 3049 } 3050 3051 /* We don't need to emulate any privileged instructions or dcbz */ 3052 static int kvmppc_core_emulate_op_hv(struct kvm_run *run, struct kvm_vcpu *vcpu, 3053 unsigned int inst, int *advance) 3054 { 3055 return EMULATE_FAIL; 3056 } 3057 3058 static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn, 3059 ulong spr_val) 3060 { 3061 return EMULATE_FAIL; 3062 } 3063 3064 static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn, 3065 ulong *spr_val) 3066 { 3067 return EMULATE_FAIL; 3068 } 3069 3070 static int kvmppc_core_check_processor_compat_hv(void) 3071 { 3072 if (!cpu_has_feature(CPU_FTR_HVMODE) || 3073 !cpu_has_feature(CPU_FTR_ARCH_206)) 3074 return -EIO; 3075 return 0; 3076 } 3077 3078 static long kvm_arch_vm_ioctl_hv(struct file *filp, 3079 unsigned int ioctl, unsigned long arg) 3080 { 3081 struct kvm *kvm __maybe_unused = filp->private_data; 3082 void __user *argp = (void __user *)arg; 3083 long r; 3084 3085 switch (ioctl) { 3086 3087 case KVM_PPC_ALLOCATE_HTAB: { 3088 u32 htab_order; 3089 3090 r = -EFAULT; 3091 if (get_user(htab_order, (u32 __user *)argp)) 3092 break; 3093 r = kvmppc_alloc_reset_hpt(kvm, &htab_order); 3094 if (r) 3095 break; 3096 r = -EFAULT; 3097 if (put_user(htab_order, (u32 __user *)argp)) 3098 break; 3099 r = 0; 3100 break; 3101 } 3102 3103 case KVM_PPC_GET_HTAB_FD: { 3104 struct kvm_get_htab_fd ghf; 3105 3106 r = -EFAULT; 3107 if (copy_from_user(&ghf, argp, sizeof(ghf))) 3108 break; 3109 r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf); 3110 break; 3111 } 3112 3113 default: 3114 r = -ENOTTY; 3115 } 3116 3117 return r; 3118 } 3119 3120 /* 3121 * List of hcall numbers to enable by default. 3122 * For compatibility with old userspace, we enable by default 3123 * all hcalls that were implemented before the hcall-enabling 3124 * facility was added. Note this list should not include H_RTAS. 3125 */ 3126 static unsigned int default_hcall_list[] = { 3127 H_REMOVE, 3128 H_ENTER, 3129 H_READ, 3130 H_PROTECT, 3131 H_BULK_REMOVE, 3132 H_GET_TCE, 3133 H_PUT_TCE, 3134 H_SET_DABR, 3135 H_SET_XDABR, 3136 H_CEDE, 3137 H_PROD, 3138 H_CONFER, 3139 H_REGISTER_VPA, 3140 #ifdef CONFIG_KVM_XICS 3141 H_EOI, 3142 H_CPPR, 3143 H_IPI, 3144 H_IPOLL, 3145 H_XIRR, 3146 H_XIRR_X, 3147 #endif 3148 0 3149 }; 3150 3151 static void init_default_hcalls(void) 3152 { 3153 int i; 3154 unsigned int hcall; 3155 3156 for (i = 0; default_hcall_list[i]; ++i) { 3157 hcall = default_hcall_list[i]; 3158 WARN_ON(!kvmppc_hcall_impl_hv(hcall)); 3159 __set_bit(hcall / 4, default_enabled_hcalls); 3160 } 3161 } 3162 3163 static struct kvmppc_ops kvm_ops_hv = { 3164 .get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv, 3165 .set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv, 3166 .get_one_reg = kvmppc_get_one_reg_hv, 3167 .set_one_reg = kvmppc_set_one_reg_hv, 3168 .vcpu_load = kvmppc_core_vcpu_load_hv, 3169 .vcpu_put = kvmppc_core_vcpu_put_hv, 3170 .set_msr = kvmppc_set_msr_hv, 3171 .vcpu_run = kvmppc_vcpu_run_hv, 3172 .vcpu_create = kvmppc_core_vcpu_create_hv, 3173 .vcpu_free = kvmppc_core_vcpu_free_hv, 3174 .check_requests = kvmppc_core_check_requests_hv, 3175 .get_dirty_log = kvm_vm_ioctl_get_dirty_log_hv, 3176 .flush_memslot = kvmppc_core_flush_memslot_hv, 3177 .prepare_memory_region = kvmppc_core_prepare_memory_region_hv, 3178 .commit_memory_region = kvmppc_core_commit_memory_region_hv, 3179 .unmap_hva = kvm_unmap_hva_hv, 3180 .unmap_hva_range = kvm_unmap_hva_range_hv, 3181 .age_hva = kvm_age_hva_hv, 3182 .test_age_hva = kvm_test_age_hva_hv, 3183 .set_spte_hva = kvm_set_spte_hva_hv, 3184 .mmu_destroy = kvmppc_mmu_destroy_hv, 3185 .free_memslot = kvmppc_core_free_memslot_hv, 3186 .create_memslot = kvmppc_core_create_memslot_hv, 3187 .init_vm = kvmppc_core_init_vm_hv, 3188 .destroy_vm = kvmppc_core_destroy_vm_hv, 3189 .get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv, 3190 .emulate_op = kvmppc_core_emulate_op_hv, 3191 .emulate_mtspr = kvmppc_core_emulate_mtspr_hv, 3192 .emulate_mfspr = kvmppc_core_emulate_mfspr_hv, 3193 .fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv, 3194 .arch_vm_ioctl = kvm_arch_vm_ioctl_hv, 3195 .hcall_implemented = kvmppc_hcall_impl_hv, 3196 }; 3197 3198 static int kvmppc_book3s_init_hv(void) 3199 { 3200 int r; 3201 /* 3202 * FIXME!! Do we need to check on all cpus ? 3203 */ 3204 r = kvmppc_core_check_processor_compat_hv(); 3205 if (r < 0) 3206 return -ENODEV; 3207 3208 kvm_ops_hv.owner = THIS_MODULE; 3209 kvmppc_hv_ops = &kvm_ops_hv; 3210 3211 init_default_hcalls(); 3212 3213 init_vcore_lists(); 3214 3215 r = kvmppc_mmu_hv_init(); 3216 return r; 3217 } 3218 3219 static void kvmppc_book3s_exit_hv(void) 3220 { 3221 kvmppc_hv_ops = NULL; 3222 } 3223 3224 module_init(kvmppc_book3s_init_hv); 3225 module_exit(kvmppc_book3s_exit_hv); 3226 MODULE_LICENSE("GPL"); 3227 MODULE_ALIAS_MISCDEV(KVM_MINOR); 3228 MODULE_ALIAS("devname:kvm"); 3229