1 /* 2 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 3 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved. 4 * 5 * Authors: 6 * Paul Mackerras <paulus@au1.ibm.com> 7 * Alexander Graf <agraf@suse.de> 8 * Kevin Wolf <mail@kevin-wolf.de> 9 * 10 * Description: KVM functions specific to running on Book 3S 11 * processors in hypervisor mode (specifically POWER7 and later). 12 * 13 * This file is derived from arch/powerpc/kvm/book3s.c, 14 * by Alexander Graf <agraf@suse.de>. 15 * 16 * This program is free software; you can redistribute it and/or modify 17 * it under the terms of the GNU General Public License, version 2, as 18 * published by the Free Software Foundation. 19 */ 20 21 #include <linux/kvm_host.h> 22 #include <linux/err.h> 23 #include <linux/slab.h> 24 #include <linux/preempt.h> 25 #include <linux/sched.h> 26 #include <linux/delay.h> 27 #include <linux/export.h> 28 #include <linux/fs.h> 29 #include <linux/anon_inodes.h> 30 #include <linux/cpu.h> 31 #include <linux/cpumask.h> 32 #include <linux/spinlock.h> 33 #include <linux/page-flags.h> 34 #include <linux/srcu.h> 35 #include <linux/miscdevice.h> 36 #include <linux/debugfs.h> 37 38 #include <asm/reg.h> 39 #include <asm/cputable.h> 40 #include <asm/cacheflush.h> 41 #include <asm/tlbflush.h> 42 #include <linux/uaccess.h> 43 #include <asm/io.h> 44 #include <asm/kvm_ppc.h> 45 #include <asm/kvm_book3s.h> 46 #include <asm/mmu_context.h> 47 #include <asm/lppaca.h> 48 #include <asm/processor.h> 49 #include <asm/cputhreads.h> 50 #include <asm/page.h> 51 #include <asm/hvcall.h> 52 #include <asm/switch_to.h> 53 #include <asm/smp.h> 54 #include <asm/dbell.h> 55 #include <asm/hmi.h> 56 #include <asm/pnv-pci.h> 57 #include <asm/mmu.h> 58 #include <asm/opal.h> 59 #include <asm/xics.h> 60 #include <linux/gfp.h> 61 #include <linux/vmalloc.h> 62 #include <linux/highmem.h> 63 #include <linux/hugetlb.h> 64 #include <linux/kvm_irqfd.h> 65 #include <linux/irqbypass.h> 66 #include <linux/module.h> 67 #include <linux/compiler.h> 68 #include <linux/of.h> 69 70 #include "book3s.h" 71 72 #define CREATE_TRACE_POINTS 73 #include "trace_hv.h" 74 75 /* #define EXIT_DEBUG */ 76 /* #define EXIT_DEBUG_SIMPLE */ 77 /* #define EXIT_DEBUG_INT */ 78 79 /* Used to indicate that a guest page fault needs to be handled */ 80 #define RESUME_PAGE_FAULT (RESUME_GUEST | RESUME_FLAG_ARCH1) 81 /* Used to indicate that a guest passthrough interrupt needs to be handled */ 82 #define RESUME_PASSTHROUGH (RESUME_GUEST | RESUME_FLAG_ARCH2) 83 84 /* Used as a "null" value for timebase values */ 85 #define TB_NIL (~(u64)0) 86 87 static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1); 88 89 static int dynamic_mt_modes = 6; 90 module_param(dynamic_mt_modes, int, S_IRUGO | S_IWUSR); 91 MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)"); 92 static int target_smt_mode; 93 module_param(target_smt_mode, int, S_IRUGO | S_IWUSR); 94 MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)"); 95 96 #ifdef CONFIG_KVM_XICS 97 static struct kernel_param_ops module_param_ops = { 98 .set = param_set_int, 99 .get = param_get_int, 100 }; 101 102 module_param_cb(kvm_irq_bypass, &module_param_ops, &kvm_irq_bypass, 103 S_IRUGO | S_IWUSR); 104 MODULE_PARM_DESC(kvm_irq_bypass, "Bypass passthrough interrupt optimization"); 105 106 module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect, 107 S_IRUGO | S_IWUSR); 108 MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core"); 109 #endif 110 111 static void kvmppc_end_cede(struct kvm_vcpu *vcpu); 112 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu); 113 114 static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc, 115 int *ip) 116 { 117 int i = *ip; 118 struct kvm_vcpu *vcpu; 119 120 while (++i < MAX_SMT_THREADS) { 121 vcpu = READ_ONCE(vc->runnable_threads[i]); 122 if (vcpu) { 123 *ip = i; 124 return vcpu; 125 } 126 } 127 return NULL; 128 } 129 130 /* Used to traverse the list of runnable threads for a given vcore */ 131 #define for_each_runnable_thread(i, vcpu, vc) \ 132 for (i = -1; (vcpu = next_runnable_thread(vc, &i)); ) 133 134 static bool kvmppc_ipi_thread(int cpu) 135 { 136 unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER); 137 138 /* On POWER9 we can use msgsnd to IPI any cpu */ 139 if (cpu_has_feature(CPU_FTR_ARCH_300)) { 140 msg |= get_hard_smp_processor_id(cpu); 141 smp_mb(); 142 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg)); 143 return true; 144 } 145 146 /* On POWER8 for IPIs to threads in the same core, use msgsnd */ 147 if (cpu_has_feature(CPU_FTR_ARCH_207S)) { 148 preempt_disable(); 149 if (cpu_first_thread_sibling(cpu) == 150 cpu_first_thread_sibling(smp_processor_id())) { 151 msg |= cpu_thread_in_core(cpu); 152 smp_mb(); 153 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg)); 154 preempt_enable(); 155 return true; 156 } 157 preempt_enable(); 158 } 159 160 #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP) 161 if (cpu >= 0 && cpu < nr_cpu_ids) { 162 if (paca[cpu].kvm_hstate.xics_phys) { 163 xics_wake_cpu(cpu); 164 return true; 165 } 166 opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY); 167 return true; 168 } 169 #endif 170 171 return false; 172 } 173 174 static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) 175 { 176 int cpu; 177 struct swait_queue_head *wqp; 178 179 wqp = kvm_arch_vcpu_wq(vcpu); 180 if (swait_active(wqp)) { 181 swake_up(wqp); 182 ++vcpu->stat.halt_wakeup; 183 } 184 185 if (kvmppc_ipi_thread(vcpu->arch.thread_cpu)) 186 return; 187 188 /* CPU points to the first thread of the core */ 189 cpu = vcpu->cpu; 190 if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu)) 191 smp_send_reschedule(cpu); 192 } 193 194 /* 195 * We use the vcpu_load/put functions to measure stolen time. 196 * Stolen time is counted as time when either the vcpu is able to 197 * run as part of a virtual core, but the task running the vcore 198 * is preempted or sleeping, or when the vcpu needs something done 199 * in the kernel by the task running the vcpu, but that task is 200 * preempted or sleeping. Those two things have to be counted 201 * separately, since one of the vcpu tasks will take on the job 202 * of running the core, and the other vcpu tasks in the vcore will 203 * sleep waiting for it to do that, but that sleep shouldn't count 204 * as stolen time. 205 * 206 * Hence we accumulate stolen time when the vcpu can run as part of 207 * a vcore using vc->stolen_tb, and the stolen time when the vcpu 208 * needs its task to do other things in the kernel (for example, 209 * service a page fault) in busy_stolen. We don't accumulate 210 * stolen time for a vcore when it is inactive, or for a vcpu 211 * when it is in state RUNNING or NOTREADY. NOTREADY is a bit of 212 * a misnomer; it means that the vcpu task is not executing in 213 * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in 214 * the kernel. We don't have any way of dividing up that time 215 * between time that the vcpu is genuinely stopped, time that 216 * the task is actively working on behalf of the vcpu, and time 217 * that the task is preempted, so we don't count any of it as 218 * stolen. 219 * 220 * Updates to busy_stolen are protected by arch.tbacct_lock; 221 * updates to vc->stolen_tb are protected by the vcore->stoltb_lock 222 * lock. The stolen times are measured in units of timebase ticks. 223 * (Note that the != TB_NIL checks below are purely defensive; 224 * they should never fail.) 225 */ 226 227 static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc) 228 { 229 unsigned long flags; 230 231 spin_lock_irqsave(&vc->stoltb_lock, flags); 232 vc->preempt_tb = mftb(); 233 spin_unlock_irqrestore(&vc->stoltb_lock, flags); 234 } 235 236 static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc) 237 { 238 unsigned long flags; 239 240 spin_lock_irqsave(&vc->stoltb_lock, flags); 241 if (vc->preempt_tb != TB_NIL) { 242 vc->stolen_tb += mftb() - vc->preempt_tb; 243 vc->preempt_tb = TB_NIL; 244 } 245 spin_unlock_irqrestore(&vc->stoltb_lock, flags); 246 } 247 248 static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu) 249 { 250 struct kvmppc_vcore *vc = vcpu->arch.vcore; 251 unsigned long flags; 252 253 /* 254 * We can test vc->runner without taking the vcore lock, 255 * because only this task ever sets vc->runner to this 256 * vcpu, and once it is set to this vcpu, only this task 257 * ever sets it to NULL. 258 */ 259 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING) 260 kvmppc_core_end_stolen(vc); 261 262 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); 263 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST && 264 vcpu->arch.busy_preempt != TB_NIL) { 265 vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt; 266 vcpu->arch.busy_preempt = TB_NIL; 267 } 268 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); 269 } 270 271 static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu) 272 { 273 struct kvmppc_vcore *vc = vcpu->arch.vcore; 274 unsigned long flags; 275 276 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING) 277 kvmppc_core_start_stolen(vc); 278 279 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); 280 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST) 281 vcpu->arch.busy_preempt = mftb(); 282 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); 283 } 284 285 static void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr) 286 { 287 /* 288 * Check for illegal transactional state bit combination 289 * and if we find it, force the TS field to a safe state. 290 */ 291 if ((msr & MSR_TS_MASK) == MSR_TS_MASK) 292 msr &= ~MSR_TS_MASK; 293 vcpu->arch.shregs.msr = msr; 294 kvmppc_end_cede(vcpu); 295 } 296 297 static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr) 298 { 299 vcpu->arch.pvr = pvr; 300 } 301 302 /* Dummy value used in computing PCR value below */ 303 #define PCR_ARCH_300 (PCR_ARCH_207 << 1) 304 305 static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) 306 { 307 unsigned long host_pcr_bit = 0, guest_pcr_bit = 0; 308 struct kvmppc_vcore *vc = vcpu->arch.vcore; 309 310 /* We can (emulate) our own architecture version and anything older */ 311 if (cpu_has_feature(CPU_FTR_ARCH_300)) 312 host_pcr_bit = PCR_ARCH_300; 313 else if (cpu_has_feature(CPU_FTR_ARCH_207S)) 314 host_pcr_bit = PCR_ARCH_207; 315 else if (cpu_has_feature(CPU_FTR_ARCH_206)) 316 host_pcr_bit = PCR_ARCH_206; 317 else 318 host_pcr_bit = PCR_ARCH_205; 319 320 /* Determine lowest PCR bit needed to run guest in given PVR level */ 321 guest_pcr_bit = host_pcr_bit; 322 if (arch_compat) { 323 switch (arch_compat) { 324 case PVR_ARCH_205: 325 guest_pcr_bit = PCR_ARCH_205; 326 break; 327 case PVR_ARCH_206: 328 case PVR_ARCH_206p: 329 guest_pcr_bit = PCR_ARCH_206; 330 break; 331 case PVR_ARCH_207: 332 guest_pcr_bit = PCR_ARCH_207; 333 break; 334 case PVR_ARCH_300: 335 guest_pcr_bit = PCR_ARCH_300; 336 break; 337 default: 338 return -EINVAL; 339 } 340 } 341 342 /* Check requested PCR bits don't exceed our capabilities */ 343 if (guest_pcr_bit > host_pcr_bit) 344 return -EINVAL; 345 346 spin_lock(&vc->lock); 347 vc->arch_compat = arch_compat; 348 /* Set all PCR bits for which guest_pcr_bit <= bit < host_pcr_bit */ 349 vc->pcr = host_pcr_bit - guest_pcr_bit; 350 spin_unlock(&vc->lock); 351 352 return 0; 353 } 354 355 static void kvmppc_dump_regs(struct kvm_vcpu *vcpu) 356 { 357 int r; 358 359 pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id); 360 pr_err("pc = %.16lx msr = %.16llx trap = %x\n", 361 vcpu->arch.pc, vcpu->arch.shregs.msr, vcpu->arch.trap); 362 for (r = 0; r < 16; ++r) 363 pr_err("r%2d = %.16lx r%d = %.16lx\n", 364 r, kvmppc_get_gpr(vcpu, r), 365 r+16, kvmppc_get_gpr(vcpu, r+16)); 366 pr_err("ctr = %.16lx lr = %.16lx\n", 367 vcpu->arch.ctr, vcpu->arch.lr); 368 pr_err("srr0 = %.16llx srr1 = %.16llx\n", 369 vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1); 370 pr_err("sprg0 = %.16llx sprg1 = %.16llx\n", 371 vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1); 372 pr_err("sprg2 = %.16llx sprg3 = %.16llx\n", 373 vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3); 374 pr_err("cr = %.8x xer = %.16lx dsisr = %.8x\n", 375 vcpu->arch.cr, vcpu->arch.xer, vcpu->arch.shregs.dsisr); 376 pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar); 377 pr_err("fault dar = %.16lx dsisr = %.8x\n", 378 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr); 379 pr_err("SLB (%d entries):\n", vcpu->arch.slb_max); 380 for (r = 0; r < vcpu->arch.slb_max; ++r) 381 pr_err(" ESID = %.16llx VSID = %.16llx\n", 382 vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv); 383 pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n", 384 vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1, 385 vcpu->arch.last_inst); 386 } 387 388 static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id) 389 { 390 struct kvm_vcpu *ret; 391 392 mutex_lock(&kvm->lock); 393 ret = kvm_get_vcpu_by_id(kvm, id); 394 mutex_unlock(&kvm->lock); 395 return ret; 396 } 397 398 static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa) 399 { 400 vpa->__old_status |= LPPACA_OLD_SHARED_PROC; 401 vpa->yield_count = cpu_to_be32(1); 402 } 403 404 static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v, 405 unsigned long addr, unsigned long len) 406 { 407 /* check address is cacheline aligned */ 408 if (addr & (L1_CACHE_BYTES - 1)) 409 return -EINVAL; 410 spin_lock(&vcpu->arch.vpa_update_lock); 411 if (v->next_gpa != addr || v->len != len) { 412 v->next_gpa = addr; 413 v->len = addr ? len : 0; 414 v->update_pending = 1; 415 } 416 spin_unlock(&vcpu->arch.vpa_update_lock); 417 return 0; 418 } 419 420 /* Length for a per-processor buffer is passed in at offset 4 in the buffer */ 421 struct reg_vpa { 422 u32 dummy; 423 union { 424 __be16 hword; 425 __be32 word; 426 } length; 427 }; 428 429 static int vpa_is_registered(struct kvmppc_vpa *vpap) 430 { 431 if (vpap->update_pending) 432 return vpap->next_gpa != 0; 433 return vpap->pinned_addr != NULL; 434 } 435 436 static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu, 437 unsigned long flags, 438 unsigned long vcpuid, unsigned long vpa) 439 { 440 struct kvm *kvm = vcpu->kvm; 441 unsigned long len, nb; 442 void *va; 443 struct kvm_vcpu *tvcpu; 444 int err; 445 int subfunc; 446 struct kvmppc_vpa *vpap; 447 448 tvcpu = kvmppc_find_vcpu(kvm, vcpuid); 449 if (!tvcpu) 450 return H_PARAMETER; 451 452 subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK; 453 if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL || 454 subfunc == H_VPA_REG_SLB) { 455 /* Registering new area - address must be cache-line aligned */ 456 if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa) 457 return H_PARAMETER; 458 459 /* convert logical addr to kernel addr and read length */ 460 va = kvmppc_pin_guest_page(kvm, vpa, &nb); 461 if (va == NULL) 462 return H_PARAMETER; 463 if (subfunc == H_VPA_REG_VPA) 464 len = be16_to_cpu(((struct reg_vpa *)va)->length.hword); 465 else 466 len = be32_to_cpu(((struct reg_vpa *)va)->length.word); 467 kvmppc_unpin_guest_page(kvm, va, vpa, false); 468 469 /* Check length */ 470 if (len > nb || len < sizeof(struct reg_vpa)) 471 return H_PARAMETER; 472 } else { 473 vpa = 0; 474 len = 0; 475 } 476 477 err = H_PARAMETER; 478 vpap = NULL; 479 spin_lock(&tvcpu->arch.vpa_update_lock); 480 481 switch (subfunc) { 482 case H_VPA_REG_VPA: /* register VPA */ 483 if (len < sizeof(struct lppaca)) 484 break; 485 vpap = &tvcpu->arch.vpa; 486 err = 0; 487 break; 488 489 case H_VPA_REG_DTL: /* register DTL */ 490 if (len < sizeof(struct dtl_entry)) 491 break; 492 len -= len % sizeof(struct dtl_entry); 493 494 /* Check that they have previously registered a VPA */ 495 err = H_RESOURCE; 496 if (!vpa_is_registered(&tvcpu->arch.vpa)) 497 break; 498 499 vpap = &tvcpu->arch.dtl; 500 err = 0; 501 break; 502 503 case H_VPA_REG_SLB: /* register SLB shadow buffer */ 504 /* Check that they have previously registered a VPA */ 505 err = H_RESOURCE; 506 if (!vpa_is_registered(&tvcpu->arch.vpa)) 507 break; 508 509 vpap = &tvcpu->arch.slb_shadow; 510 err = 0; 511 break; 512 513 case H_VPA_DEREG_VPA: /* deregister VPA */ 514 /* Check they don't still have a DTL or SLB buf registered */ 515 err = H_RESOURCE; 516 if (vpa_is_registered(&tvcpu->arch.dtl) || 517 vpa_is_registered(&tvcpu->arch.slb_shadow)) 518 break; 519 520 vpap = &tvcpu->arch.vpa; 521 err = 0; 522 break; 523 524 case H_VPA_DEREG_DTL: /* deregister DTL */ 525 vpap = &tvcpu->arch.dtl; 526 err = 0; 527 break; 528 529 case H_VPA_DEREG_SLB: /* deregister SLB shadow buffer */ 530 vpap = &tvcpu->arch.slb_shadow; 531 err = 0; 532 break; 533 } 534 535 if (vpap) { 536 vpap->next_gpa = vpa; 537 vpap->len = len; 538 vpap->update_pending = 1; 539 } 540 541 spin_unlock(&tvcpu->arch.vpa_update_lock); 542 543 return err; 544 } 545 546 static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap) 547 { 548 struct kvm *kvm = vcpu->kvm; 549 void *va; 550 unsigned long nb; 551 unsigned long gpa; 552 553 /* 554 * We need to pin the page pointed to by vpap->next_gpa, 555 * but we can't call kvmppc_pin_guest_page under the lock 556 * as it does get_user_pages() and down_read(). So we 557 * have to drop the lock, pin the page, then get the lock 558 * again and check that a new area didn't get registered 559 * in the meantime. 560 */ 561 for (;;) { 562 gpa = vpap->next_gpa; 563 spin_unlock(&vcpu->arch.vpa_update_lock); 564 va = NULL; 565 nb = 0; 566 if (gpa) 567 va = kvmppc_pin_guest_page(kvm, gpa, &nb); 568 spin_lock(&vcpu->arch.vpa_update_lock); 569 if (gpa == vpap->next_gpa) 570 break; 571 /* sigh... unpin that one and try again */ 572 if (va) 573 kvmppc_unpin_guest_page(kvm, va, gpa, false); 574 } 575 576 vpap->update_pending = 0; 577 if (va && nb < vpap->len) { 578 /* 579 * If it's now too short, it must be that userspace 580 * has changed the mappings underlying guest memory, 581 * so unregister the region. 582 */ 583 kvmppc_unpin_guest_page(kvm, va, gpa, false); 584 va = NULL; 585 } 586 if (vpap->pinned_addr) 587 kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa, 588 vpap->dirty); 589 vpap->gpa = gpa; 590 vpap->pinned_addr = va; 591 vpap->dirty = false; 592 if (va) 593 vpap->pinned_end = va + vpap->len; 594 } 595 596 static void kvmppc_update_vpas(struct kvm_vcpu *vcpu) 597 { 598 if (!(vcpu->arch.vpa.update_pending || 599 vcpu->arch.slb_shadow.update_pending || 600 vcpu->arch.dtl.update_pending)) 601 return; 602 603 spin_lock(&vcpu->arch.vpa_update_lock); 604 if (vcpu->arch.vpa.update_pending) { 605 kvmppc_update_vpa(vcpu, &vcpu->arch.vpa); 606 if (vcpu->arch.vpa.pinned_addr) 607 init_vpa(vcpu, vcpu->arch.vpa.pinned_addr); 608 } 609 if (vcpu->arch.dtl.update_pending) { 610 kvmppc_update_vpa(vcpu, &vcpu->arch.dtl); 611 vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr; 612 vcpu->arch.dtl_index = 0; 613 } 614 if (vcpu->arch.slb_shadow.update_pending) 615 kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow); 616 spin_unlock(&vcpu->arch.vpa_update_lock); 617 } 618 619 /* 620 * Return the accumulated stolen time for the vcore up until `now'. 621 * The caller should hold the vcore lock. 622 */ 623 static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now) 624 { 625 u64 p; 626 unsigned long flags; 627 628 spin_lock_irqsave(&vc->stoltb_lock, flags); 629 p = vc->stolen_tb; 630 if (vc->vcore_state != VCORE_INACTIVE && 631 vc->preempt_tb != TB_NIL) 632 p += now - vc->preempt_tb; 633 spin_unlock_irqrestore(&vc->stoltb_lock, flags); 634 return p; 635 } 636 637 static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, 638 struct kvmppc_vcore *vc) 639 { 640 struct dtl_entry *dt; 641 struct lppaca *vpa; 642 unsigned long stolen; 643 unsigned long core_stolen; 644 u64 now; 645 646 dt = vcpu->arch.dtl_ptr; 647 vpa = vcpu->arch.vpa.pinned_addr; 648 now = mftb(); 649 core_stolen = vcore_stolen_time(vc, now); 650 stolen = core_stolen - vcpu->arch.stolen_logged; 651 vcpu->arch.stolen_logged = core_stolen; 652 spin_lock_irq(&vcpu->arch.tbacct_lock); 653 stolen += vcpu->arch.busy_stolen; 654 vcpu->arch.busy_stolen = 0; 655 spin_unlock_irq(&vcpu->arch.tbacct_lock); 656 if (!dt || !vpa) 657 return; 658 memset(dt, 0, sizeof(struct dtl_entry)); 659 dt->dispatch_reason = 7; 660 dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid); 661 dt->timebase = cpu_to_be64(now + vc->tb_offset); 662 dt->enqueue_to_dispatch_time = cpu_to_be32(stolen); 663 dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu)); 664 dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr); 665 ++dt; 666 if (dt == vcpu->arch.dtl.pinned_end) 667 dt = vcpu->arch.dtl.pinned_addr; 668 vcpu->arch.dtl_ptr = dt; 669 /* order writing *dt vs. writing vpa->dtl_idx */ 670 smp_wmb(); 671 vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index); 672 vcpu->arch.dtl.dirty = true; 673 } 674 675 static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu) 676 { 677 if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207) 678 return true; 679 if ((!vcpu->arch.vcore->arch_compat) && 680 cpu_has_feature(CPU_FTR_ARCH_207S)) 681 return true; 682 return false; 683 } 684 685 static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags, 686 unsigned long resource, unsigned long value1, 687 unsigned long value2) 688 { 689 switch (resource) { 690 case H_SET_MODE_RESOURCE_SET_CIABR: 691 if (!kvmppc_power8_compatible(vcpu)) 692 return H_P2; 693 if (value2) 694 return H_P4; 695 if (mflags) 696 return H_UNSUPPORTED_FLAG_START; 697 /* Guests can't breakpoint the hypervisor */ 698 if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER) 699 return H_P3; 700 vcpu->arch.ciabr = value1; 701 return H_SUCCESS; 702 case H_SET_MODE_RESOURCE_SET_DAWR: 703 if (!kvmppc_power8_compatible(vcpu)) 704 return H_P2; 705 if (mflags) 706 return H_UNSUPPORTED_FLAG_START; 707 if (value2 & DABRX_HYP) 708 return H_P4; 709 vcpu->arch.dawr = value1; 710 vcpu->arch.dawrx = value2; 711 return H_SUCCESS; 712 default: 713 return H_TOO_HARD; 714 } 715 } 716 717 static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target) 718 { 719 struct kvmppc_vcore *vcore = target->arch.vcore; 720 721 /* 722 * We expect to have been called by the real mode handler 723 * (kvmppc_rm_h_confer()) which would have directly returned 724 * H_SUCCESS if the source vcore wasn't idle (e.g. if it may 725 * have useful work to do and should not confer) so we don't 726 * recheck that here. 727 */ 728 729 spin_lock(&vcore->lock); 730 if (target->arch.state == KVMPPC_VCPU_RUNNABLE && 731 vcore->vcore_state != VCORE_INACTIVE && 732 vcore->runner) 733 target = vcore->runner; 734 spin_unlock(&vcore->lock); 735 736 return kvm_vcpu_yield_to(target); 737 } 738 739 static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu) 740 { 741 int yield_count = 0; 742 struct lppaca *lppaca; 743 744 spin_lock(&vcpu->arch.vpa_update_lock); 745 lppaca = (struct lppaca *)vcpu->arch.vpa.pinned_addr; 746 if (lppaca) 747 yield_count = be32_to_cpu(lppaca->yield_count); 748 spin_unlock(&vcpu->arch.vpa_update_lock); 749 return yield_count; 750 } 751 752 int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) 753 { 754 unsigned long req = kvmppc_get_gpr(vcpu, 3); 755 unsigned long target, ret = H_SUCCESS; 756 int yield_count; 757 struct kvm_vcpu *tvcpu; 758 int idx, rc; 759 760 if (req <= MAX_HCALL_OPCODE && 761 !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls)) 762 return RESUME_HOST; 763 764 switch (req) { 765 case H_CEDE: 766 break; 767 case H_PROD: 768 target = kvmppc_get_gpr(vcpu, 4); 769 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target); 770 if (!tvcpu) { 771 ret = H_PARAMETER; 772 break; 773 } 774 tvcpu->arch.prodded = 1; 775 smp_mb(); 776 if (vcpu->arch.ceded) { 777 if (swait_active(&vcpu->wq)) { 778 swake_up(&vcpu->wq); 779 vcpu->stat.halt_wakeup++; 780 } 781 } 782 break; 783 case H_CONFER: 784 target = kvmppc_get_gpr(vcpu, 4); 785 if (target == -1) 786 break; 787 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target); 788 if (!tvcpu) { 789 ret = H_PARAMETER; 790 break; 791 } 792 yield_count = kvmppc_get_gpr(vcpu, 5); 793 if (kvmppc_get_yield_count(tvcpu) != yield_count) 794 break; 795 kvm_arch_vcpu_yield_to(tvcpu); 796 break; 797 case H_REGISTER_VPA: 798 ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4), 799 kvmppc_get_gpr(vcpu, 5), 800 kvmppc_get_gpr(vcpu, 6)); 801 break; 802 case H_RTAS: 803 if (list_empty(&vcpu->kvm->arch.rtas_tokens)) 804 return RESUME_HOST; 805 806 idx = srcu_read_lock(&vcpu->kvm->srcu); 807 rc = kvmppc_rtas_hcall(vcpu); 808 srcu_read_unlock(&vcpu->kvm->srcu, idx); 809 810 if (rc == -ENOENT) 811 return RESUME_HOST; 812 else if (rc == 0) 813 break; 814 815 /* Send the error out to userspace via KVM_RUN */ 816 return rc; 817 case H_LOGICAL_CI_LOAD: 818 ret = kvmppc_h_logical_ci_load(vcpu); 819 if (ret == H_TOO_HARD) 820 return RESUME_HOST; 821 break; 822 case H_LOGICAL_CI_STORE: 823 ret = kvmppc_h_logical_ci_store(vcpu); 824 if (ret == H_TOO_HARD) 825 return RESUME_HOST; 826 break; 827 case H_SET_MODE: 828 ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4), 829 kvmppc_get_gpr(vcpu, 5), 830 kvmppc_get_gpr(vcpu, 6), 831 kvmppc_get_gpr(vcpu, 7)); 832 if (ret == H_TOO_HARD) 833 return RESUME_HOST; 834 break; 835 case H_XIRR: 836 case H_CPPR: 837 case H_EOI: 838 case H_IPI: 839 case H_IPOLL: 840 case H_XIRR_X: 841 if (kvmppc_xics_enabled(vcpu)) { 842 ret = kvmppc_xics_hcall(vcpu, req); 843 break; 844 } 845 return RESUME_HOST; 846 case H_PUT_TCE: 847 ret = kvmppc_h_put_tce(vcpu, kvmppc_get_gpr(vcpu, 4), 848 kvmppc_get_gpr(vcpu, 5), 849 kvmppc_get_gpr(vcpu, 6)); 850 if (ret == H_TOO_HARD) 851 return RESUME_HOST; 852 break; 853 case H_PUT_TCE_INDIRECT: 854 ret = kvmppc_h_put_tce_indirect(vcpu, kvmppc_get_gpr(vcpu, 4), 855 kvmppc_get_gpr(vcpu, 5), 856 kvmppc_get_gpr(vcpu, 6), 857 kvmppc_get_gpr(vcpu, 7)); 858 if (ret == H_TOO_HARD) 859 return RESUME_HOST; 860 break; 861 case H_STUFF_TCE: 862 ret = kvmppc_h_stuff_tce(vcpu, kvmppc_get_gpr(vcpu, 4), 863 kvmppc_get_gpr(vcpu, 5), 864 kvmppc_get_gpr(vcpu, 6), 865 kvmppc_get_gpr(vcpu, 7)); 866 if (ret == H_TOO_HARD) 867 return RESUME_HOST; 868 break; 869 default: 870 return RESUME_HOST; 871 } 872 kvmppc_set_gpr(vcpu, 3, ret); 873 vcpu->arch.hcall_needed = 0; 874 return RESUME_GUEST; 875 } 876 877 static int kvmppc_hcall_impl_hv(unsigned long cmd) 878 { 879 switch (cmd) { 880 case H_CEDE: 881 case H_PROD: 882 case H_CONFER: 883 case H_REGISTER_VPA: 884 case H_SET_MODE: 885 case H_LOGICAL_CI_LOAD: 886 case H_LOGICAL_CI_STORE: 887 #ifdef CONFIG_KVM_XICS 888 case H_XIRR: 889 case H_CPPR: 890 case H_EOI: 891 case H_IPI: 892 case H_IPOLL: 893 case H_XIRR_X: 894 #endif 895 return 1; 896 } 897 898 /* See if it's in the real-mode table */ 899 return kvmppc_hcall_impl_hv_realmode(cmd); 900 } 901 902 static int kvmppc_emulate_debug_inst(struct kvm_run *run, 903 struct kvm_vcpu *vcpu) 904 { 905 u32 last_inst; 906 907 if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) != 908 EMULATE_DONE) { 909 /* 910 * Fetch failed, so return to guest and 911 * try executing it again. 912 */ 913 return RESUME_GUEST; 914 } 915 916 if (last_inst == KVMPPC_INST_SW_BREAKPOINT) { 917 run->exit_reason = KVM_EXIT_DEBUG; 918 run->debug.arch.address = kvmppc_get_pc(vcpu); 919 return RESUME_HOST; 920 } else { 921 kvmppc_core_queue_program(vcpu, SRR1_PROGILL); 922 return RESUME_GUEST; 923 } 924 } 925 926 static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu, 927 struct task_struct *tsk) 928 { 929 int r = RESUME_HOST; 930 931 vcpu->stat.sum_exits++; 932 933 /* 934 * This can happen if an interrupt occurs in the last stages 935 * of guest entry or the first stages of guest exit (i.e. after 936 * setting paca->kvm_hstate.in_guest to KVM_GUEST_MODE_GUEST_HV 937 * and before setting it to KVM_GUEST_MODE_HOST_HV). 938 * That can happen due to a bug, or due to a machine check 939 * occurring at just the wrong time. 940 */ 941 if (vcpu->arch.shregs.msr & MSR_HV) { 942 printk(KERN_EMERG "KVM trap in HV mode!\n"); 943 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n", 944 vcpu->arch.trap, kvmppc_get_pc(vcpu), 945 vcpu->arch.shregs.msr); 946 kvmppc_dump_regs(vcpu); 947 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 948 run->hw.hardware_exit_reason = vcpu->arch.trap; 949 return RESUME_HOST; 950 } 951 run->exit_reason = KVM_EXIT_UNKNOWN; 952 run->ready_for_interrupt_injection = 1; 953 switch (vcpu->arch.trap) { 954 /* We're good on these - the host merely wanted to get our attention */ 955 case BOOK3S_INTERRUPT_HV_DECREMENTER: 956 vcpu->stat.dec_exits++; 957 r = RESUME_GUEST; 958 break; 959 case BOOK3S_INTERRUPT_EXTERNAL: 960 case BOOK3S_INTERRUPT_H_DOORBELL: 961 case BOOK3S_INTERRUPT_H_VIRT: 962 vcpu->stat.ext_intr_exits++; 963 r = RESUME_GUEST; 964 break; 965 /* HMI is hypervisor interrupt and host has handled it. Resume guest.*/ 966 case BOOK3S_INTERRUPT_HMI: 967 case BOOK3S_INTERRUPT_PERFMON: 968 r = RESUME_GUEST; 969 break; 970 case BOOK3S_INTERRUPT_MACHINE_CHECK: 971 /* 972 * Deliver a machine check interrupt to the guest. 973 * We have to do this, even if the host has handled the 974 * machine check, because machine checks use SRR0/1 and 975 * the interrupt might have trashed guest state in them. 976 */ 977 kvmppc_book3s_queue_irqprio(vcpu, 978 BOOK3S_INTERRUPT_MACHINE_CHECK); 979 r = RESUME_GUEST; 980 break; 981 case BOOK3S_INTERRUPT_PROGRAM: 982 { 983 ulong flags; 984 /* 985 * Normally program interrupts are delivered directly 986 * to the guest by the hardware, but we can get here 987 * as a result of a hypervisor emulation interrupt 988 * (e40) getting turned into a 700 by BML RTAS. 989 */ 990 flags = vcpu->arch.shregs.msr & 0x1f0000ull; 991 kvmppc_core_queue_program(vcpu, flags); 992 r = RESUME_GUEST; 993 break; 994 } 995 case BOOK3S_INTERRUPT_SYSCALL: 996 { 997 /* hcall - punt to userspace */ 998 int i; 999 1000 /* hypercall with MSR_PR has already been handled in rmode, 1001 * and never reaches here. 1002 */ 1003 1004 run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3); 1005 for (i = 0; i < 9; ++i) 1006 run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i); 1007 run->exit_reason = KVM_EXIT_PAPR_HCALL; 1008 vcpu->arch.hcall_needed = 1; 1009 r = RESUME_HOST; 1010 break; 1011 } 1012 /* 1013 * We get these next two if the guest accesses a page which it thinks 1014 * it has mapped but which is not actually present, either because 1015 * it is for an emulated I/O device or because the corresonding 1016 * host page has been paged out. Any other HDSI/HISI interrupts 1017 * have been handled already. 1018 */ 1019 case BOOK3S_INTERRUPT_H_DATA_STORAGE: 1020 r = RESUME_PAGE_FAULT; 1021 break; 1022 case BOOK3S_INTERRUPT_H_INST_STORAGE: 1023 vcpu->arch.fault_dar = kvmppc_get_pc(vcpu); 1024 vcpu->arch.fault_dsisr = 0; 1025 r = RESUME_PAGE_FAULT; 1026 break; 1027 /* 1028 * This occurs if the guest executes an illegal instruction. 1029 * If the guest debug is disabled, generate a program interrupt 1030 * to the guest. If guest debug is enabled, we need to check 1031 * whether the instruction is a software breakpoint instruction. 1032 * Accordingly return to Guest or Host. 1033 */ 1034 case BOOK3S_INTERRUPT_H_EMUL_ASSIST: 1035 if (vcpu->arch.emul_inst != KVM_INST_FETCH_FAILED) 1036 vcpu->arch.last_inst = kvmppc_need_byteswap(vcpu) ? 1037 swab32(vcpu->arch.emul_inst) : 1038 vcpu->arch.emul_inst; 1039 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) { 1040 r = kvmppc_emulate_debug_inst(run, vcpu); 1041 } else { 1042 kvmppc_core_queue_program(vcpu, SRR1_PROGILL); 1043 r = RESUME_GUEST; 1044 } 1045 break; 1046 /* 1047 * This occurs if the guest (kernel or userspace), does something that 1048 * is prohibited by HFSCR. We just generate a program interrupt to 1049 * the guest. 1050 */ 1051 case BOOK3S_INTERRUPT_H_FAC_UNAVAIL: 1052 kvmppc_core_queue_program(vcpu, SRR1_PROGILL); 1053 r = RESUME_GUEST; 1054 break; 1055 case BOOK3S_INTERRUPT_HV_RM_HARD: 1056 r = RESUME_PASSTHROUGH; 1057 break; 1058 default: 1059 kvmppc_dump_regs(vcpu); 1060 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n", 1061 vcpu->arch.trap, kvmppc_get_pc(vcpu), 1062 vcpu->arch.shregs.msr); 1063 run->hw.hardware_exit_reason = vcpu->arch.trap; 1064 r = RESUME_HOST; 1065 break; 1066 } 1067 1068 return r; 1069 } 1070 1071 static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu, 1072 struct kvm_sregs *sregs) 1073 { 1074 int i; 1075 1076 memset(sregs, 0, sizeof(struct kvm_sregs)); 1077 sregs->pvr = vcpu->arch.pvr; 1078 for (i = 0; i < vcpu->arch.slb_max; i++) { 1079 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige; 1080 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv; 1081 } 1082 1083 return 0; 1084 } 1085 1086 static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu, 1087 struct kvm_sregs *sregs) 1088 { 1089 int i, j; 1090 1091 /* Only accept the same PVR as the host's, since we can't spoof it */ 1092 if (sregs->pvr != vcpu->arch.pvr) 1093 return -EINVAL; 1094 1095 j = 0; 1096 for (i = 0; i < vcpu->arch.slb_nr; i++) { 1097 if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) { 1098 vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe; 1099 vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv; 1100 ++j; 1101 } 1102 } 1103 vcpu->arch.slb_max = j; 1104 1105 return 0; 1106 } 1107 1108 static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr, 1109 bool preserve_top32) 1110 { 1111 struct kvm *kvm = vcpu->kvm; 1112 struct kvmppc_vcore *vc = vcpu->arch.vcore; 1113 u64 mask; 1114 1115 mutex_lock(&kvm->lock); 1116 spin_lock(&vc->lock); 1117 /* 1118 * If ILE (interrupt little-endian) has changed, update the 1119 * MSR_LE bit in the intr_msr for each vcpu in this vcore. 1120 */ 1121 if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) { 1122 struct kvm_vcpu *vcpu; 1123 int i; 1124 1125 kvm_for_each_vcpu(i, vcpu, kvm) { 1126 if (vcpu->arch.vcore != vc) 1127 continue; 1128 if (new_lpcr & LPCR_ILE) 1129 vcpu->arch.intr_msr |= MSR_LE; 1130 else 1131 vcpu->arch.intr_msr &= ~MSR_LE; 1132 } 1133 } 1134 1135 /* 1136 * Userspace can only modify DPFD (default prefetch depth), 1137 * ILE (interrupt little-endian) and TC (translation control). 1138 * On POWER8 userspace can also modify AIL (alt. interrupt loc.) 1139 */ 1140 mask = LPCR_DPFD | LPCR_ILE | LPCR_TC; 1141 if (cpu_has_feature(CPU_FTR_ARCH_207S)) 1142 mask |= LPCR_AIL; 1143 1144 /* Broken 32-bit version of LPCR must not clear top bits */ 1145 if (preserve_top32) 1146 mask &= 0xFFFFFFFF; 1147 vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask); 1148 spin_unlock(&vc->lock); 1149 mutex_unlock(&kvm->lock); 1150 } 1151 1152 static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, 1153 union kvmppc_one_reg *val) 1154 { 1155 int r = 0; 1156 long int i; 1157 1158 switch (id) { 1159 case KVM_REG_PPC_DEBUG_INST: 1160 *val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT); 1161 break; 1162 case KVM_REG_PPC_HIOR: 1163 *val = get_reg_val(id, 0); 1164 break; 1165 case KVM_REG_PPC_DABR: 1166 *val = get_reg_val(id, vcpu->arch.dabr); 1167 break; 1168 case KVM_REG_PPC_DABRX: 1169 *val = get_reg_val(id, vcpu->arch.dabrx); 1170 break; 1171 case KVM_REG_PPC_DSCR: 1172 *val = get_reg_val(id, vcpu->arch.dscr); 1173 break; 1174 case KVM_REG_PPC_PURR: 1175 *val = get_reg_val(id, vcpu->arch.purr); 1176 break; 1177 case KVM_REG_PPC_SPURR: 1178 *val = get_reg_val(id, vcpu->arch.spurr); 1179 break; 1180 case KVM_REG_PPC_AMR: 1181 *val = get_reg_val(id, vcpu->arch.amr); 1182 break; 1183 case KVM_REG_PPC_UAMOR: 1184 *val = get_reg_val(id, vcpu->arch.uamor); 1185 break; 1186 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS: 1187 i = id - KVM_REG_PPC_MMCR0; 1188 *val = get_reg_val(id, vcpu->arch.mmcr[i]); 1189 break; 1190 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8: 1191 i = id - KVM_REG_PPC_PMC1; 1192 *val = get_reg_val(id, vcpu->arch.pmc[i]); 1193 break; 1194 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2: 1195 i = id - KVM_REG_PPC_SPMC1; 1196 *val = get_reg_val(id, vcpu->arch.spmc[i]); 1197 break; 1198 case KVM_REG_PPC_SIAR: 1199 *val = get_reg_val(id, vcpu->arch.siar); 1200 break; 1201 case KVM_REG_PPC_SDAR: 1202 *val = get_reg_val(id, vcpu->arch.sdar); 1203 break; 1204 case KVM_REG_PPC_SIER: 1205 *val = get_reg_val(id, vcpu->arch.sier); 1206 break; 1207 case KVM_REG_PPC_IAMR: 1208 *val = get_reg_val(id, vcpu->arch.iamr); 1209 break; 1210 case KVM_REG_PPC_PSPB: 1211 *val = get_reg_val(id, vcpu->arch.pspb); 1212 break; 1213 case KVM_REG_PPC_DPDES: 1214 *val = get_reg_val(id, vcpu->arch.vcore->dpdes); 1215 break; 1216 case KVM_REG_PPC_VTB: 1217 *val = get_reg_val(id, vcpu->arch.vcore->vtb); 1218 break; 1219 case KVM_REG_PPC_DAWR: 1220 *val = get_reg_val(id, vcpu->arch.dawr); 1221 break; 1222 case KVM_REG_PPC_DAWRX: 1223 *val = get_reg_val(id, vcpu->arch.dawrx); 1224 break; 1225 case KVM_REG_PPC_CIABR: 1226 *val = get_reg_val(id, vcpu->arch.ciabr); 1227 break; 1228 case KVM_REG_PPC_CSIGR: 1229 *val = get_reg_val(id, vcpu->arch.csigr); 1230 break; 1231 case KVM_REG_PPC_TACR: 1232 *val = get_reg_val(id, vcpu->arch.tacr); 1233 break; 1234 case KVM_REG_PPC_TCSCR: 1235 *val = get_reg_val(id, vcpu->arch.tcscr); 1236 break; 1237 case KVM_REG_PPC_PID: 1238 *val = get_reg_val(id, vcpu->arch.pid); 1239 break; 1240 case KVM_REG_PPC_ACOP: 1241 *val = get_reg_val(id, vcpu->arch.acop); 1242 break; 1243 case KVM_REG_PPC_WORT: 1244 *val = get_reg_val(id, vcpu->arch.wort); 1245 break; 1246 case KVM_REG_PPC_TIDR: 1247 *val = get_reg_val(id, vcpu->arch.tid); 1248 break; 1249 case KVM_REG_PPC_PSSCR: 1250 *val = get_reg_val(id, vcpu->arch.psscr); 1251 break; 1252 case KVM_REG_PPC_VPA_ADDR: 1253 spin_lock(&vcpu->arch.vpa_update_lock); 1254 *val = get_reg_val(id, vcpu->arch.vpa.next_gpa); 1255 spin_unlock(&vcpu->arch.vpa_update_lock); 1256 break; 1257 case KVM_REG_PPC_VPA_SLB: 1258 spin_lock(&vcpu->arch.vpa_update_lock); 1259 val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa; 1260 val->vpaval.length = vcpu->arch.slb_shadow.len; 1261 spin_unlock(&vcpu->arch.vpa_update_lock); 1262 break; 1263 case KVM_REG_PPC_VPA_DTL: 1264 spin_lock(&vcpu->arch.vpa_update_lock); 1265 val->vpaval.addr = vcpu->arch.dtl.next_gpa; 1266 val->vpaval.length = vcpu->arch.dtl.len; 1267 spin_unlock(&vcpu->arch.vpa_update_lock); 1268 break; 1269 case KVM_REG_PPC_TB_OFFSET: 1270 *val = get_reg_val(id, vcpu->arch.vcore->tb_offset); 1271 break; 1272 case KVM_REG_PPC_LPCR: 1273 case KVM_REG_PPC_LPCR_64: 1274 *val = get_reg_val(id, vcpu->arch.vcore->lpcr); 1275 break; 1276 case KVM_REG_PPC_PPR: 1277 *val = get_reg_val(id, vcpu->arch.ppr); 1278 break; 1279 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1280 case KVM_REG_PPC_TFHAR: 1281 *val = get_reg_val(id, vcpu->arch.tfhar); 1282 break; 1283 case KVM_REG_PPC_TFIAR: 1284 *val = get_reg_val(id, vcpu->arch.tfiar); 1285 break; 1286 case KVM_REG_PPC_TEXASR: 1287 *val = get_reg_val(id, vcpu->arch.texasr); 1288 break; 1289 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31: 1290 i = id - KVM_REG_PPC_TM_GPR0; 1291 *val = get_reg_val(id, vcpu->arch.gpr_tm[i]); 1292 break; 1293 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63: 1294 { 1295 int j; 1296 i = id - KVM_REG_PPC_TM_VSR0; 1297 if (i < 32) 1298 for (j = 0; j < TS_FPRWIDTH; j++) 1299 val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j]; 1300 else { 1301 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1302 val->vval = vcpu->arch.vr_tm.vr[i-32]; 1303 else 1304 r = -ENXIO; 1305 } 1306 break; 1307 } 1308 case KVM_REG_PPC_TM_CR: 1309 *val = get_reg_val(id, vcpu->arch.cr_tm); 1310 break; 1311 case KVM_REG_PPC_TM_XER: 1312 *val = get_reg_val(id, vcpu->arch.xer_tm); 1313 break; 1314 case KVM_REG_PPC_TM_LR: 1315 *val = get_reg_val(id, vcpu->arch.lr_tm); 1316 break; 1317 case KVM_REG_PPC_TM_CTR: 1318 *val = get_reg_val(id, vcpu->arch.ctr_tm); 1319 break; 1320 case KVM_REG_PPC_TM_FPSCR: 1321 *val = get_reg_val(id, vcpu->arch.fp_tm.fpscr); 1322 break; 1323 case KVM_REG_PPC_TM_AMR: 1324 *val = get_reg_val(id, vcpu->arch.amr_tm); 1325 break; 1326 case KVM_REG_PPC_TM_PPR: 1327 *val = get_reg_val(id, vcpu->arch.ppr_tm); 1328 break; 1329 case KVM_REG_PPC_TM_VRSAVE: 1330 *val = get_reg_val(id, vcpu->arch.vrsave_tm); 1331 break; 1332 case KVM_REG_PPC_TM_VSCR: 1333 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1334 *val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]); 1335 else 1336 r = -ENXIO; 1337 break; 1338 case KVM_REG_PPC_TM_DSCR: 1339 *val = get_reg_val(id, vcpu->arch.dscr_tm); 1340 break; 1341 case KVM_REG_PPC_TM_TAR: 1342 *val = get_reg_val(id, vcpu->arch.tar_tm); 1343 break; 1344 #endif 1345 case KVM_REG_PPC_ARCH_COMPAT: 1346 *val = get_reg_val(id, vcpu->arch.vcore->arch_compat); 1347 break; 1348 default: 1349 r = -EINVAL; 1350 break; 1351 } 1352 1353 return r; 1354 } 1355 1356 static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, 1357 union kvmppc_one_reg *val) 1358 { 1359 int r = 0; 1360 long int i; 1361 unsigned long addr, len; 1362 1363 switch (id) { 1364 case KVM_REG_PPC_HIOR: 1365 /* Only allow this to be set to zero */ 1366 if (set_reg_val(id, *val)) 1367 r = -EINVAL; 1368 break; 1369 case KVM_REG_PPC_DABR: 1370 vcpu->arch.dabr = set_reg_val(id, *val); 1371 break; 1372 case KVM_REG_PPC_DABRX: 1373 vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP; 1374 break; 1375 case KVM_REG_PPC_DSCR: 1376 vcpu->arch.dscr = set_reg_val(id, *val); 1377 break; 1378 case KVM_REG_PPC_PURR: 1379 vcpu->arch.purr = set_reg_val(id, *val); 1380 break; 1381 case KVM_REG_PPC_SPURR: 1382 vcpu->arch.spurr = set_reg_val(id, *val); 1383 break; 1384 case KVM_REG_PPC_AMR: 1385 vcpu->arch.amr = set_reg_val(id, *val); 1386 break; 1387 case KVM_REG_PPC_UAMOR: 1388 vcpu->arch.uamor = set_reg_val(id, *val); 1389 break; 1390 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS: 1391 i = id - KVM_REG_PPC_MMCR0; 1392 vcpu->arch.mmcr[i] = set_reg_val(id, *val); 1393 break; 1394 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8: 1395 i = id - KVM_REG_PPC_PMC1; 1396 vcpu->arch.pmc[i] = set_reg_val(id, *val); 1397 break; 1398 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2: 1399 i = id - KVM_REG_PPC_SPMC1; 1400 vcpu->arch.spmc[i] = set_reg_val(id, *val); 1401 break; 1402 case KVM_REG_PPC_SIAR: 1403 vcpu->arch.siar = set_reg_val(id, *val); 1404 break; 1405 case KVM_REG_PPC_SDAR: 1406 vcpu->arch.sdar = set_reg_val(id, *val); 1407 break; 1408 case KVM_REG_PPC_SIER: 1409 vcpu->arch.sier = set_reg_val(id, *val); 1410 break; 1411 case KVM_REG_PPC_IAMR: 1412 vcpu->arch.iamr = set_reg_val(id, *val); 1413 break; 1414 case KVM_REG_PPC_PSPB: 1415 vcpu->arch.pspb = set_reg_val(id, *val); 1416 break; 1417 case KVM_REG_PPC_DPDES: 1418 vcpu->arch.vcore->dpdes = set_reg_val(id, *val); 1419 break; 1420 case KVM_REG_PPC_VTB: 1421 vcpu->arch.vcore->vtb = set_reg_val(id, *val); 1422 break; 1423 case KVM_REG_PPC_DAWR: 1424 vcpu->arch.dawr = set_reg_val(id, *val); 1425 break; 1426 case KVM_REG_PPC_DAWRX: 1427 vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP; 1428 break; 1429 case KVM_REG_PPC_CIABR: 1430 vcpu->arch.ciabr = set_reg_val(id, *val); 1431 /* Don't allow setting breakpoints in hypervisor code */ 1432 if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER) 1433 vcpu->arch.ciabr &= ~CIABR_PRIV; /* disable */ 1434 break; 1435 case KVM_REG_PPC_CSIGR: 1436 vcpu->arch.csigr = set_reg_val(id, *val); 1437 break; 1438 case KVM_REG_PPC_TACR: 1439 vcpu->arch.tacr = set_reg_val(id, *val); 1440 break; 1441 case KVM_REG_PPC_TCSCR: 1442 vcpu->arch.tcscr = set_reg_val(id, *val); 1443 break; 1444 case KVM_REG_PPC_PID: 1445 vcpu->arch.pid = set_reg_val(id, *val); 1446 break; 1447 case KVM_REG_PPC_ACOP: 1448 vcpu->arch.acop = set_reg_val(id, *val); 1449 break; 1450 case KVM_REG_PPC_WORT: 1451 vcpu->arch.wort = set_reg_val(id, *val); 1452 break; 1453 case KVM_REG_PPC_TIDR: 1454 vcpu->arch.tid = set_reg_val(id, *val); 1455 break; 1456 case KVM_REG_PPC_PSSCR: 1457 vcpu->arch.psscr = set_reg_val(id, *val) & PSSCR_GUEST_VIS; 1458 break; 1459 case KVM_REG_PPC_VPA_ADDR: 1460 addr = set_reg_val(id, *val); 1461 r = -EINVAL; 1462 if (!addr && (vcpu->arch.slb_shadow.next_gpa || 1463 vcpu->arch.dtl.next_gpa)) 1464 break; 1465 r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca)); 1466 break; 1467 case KVM_REG_PPC_VPA_SLB: 1468 addr = val->vpaval.addr; 1469 len = val->vpaval.length; 1470 r = -EINVAL; 1471 if (addr && !vcpu->arch.vpa.next_gpa) 1472 break; 1473 r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len); 1474 break; 1475 case KVM_REG_PPC_VPA_DTL: 1476 addr = val->vpaval.addr; 1477 len = val->vpaval.length; 1478 r = -EINVAL; 1479 if (addr && (len < sizeof(struct dtl_entry) || 1480 !vcpu->arch.vpa.next_gpa)) 1481 break; 1482 len -= len % sizeof(struct dtl_entry); 1483 r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len); 1484 break; 1485 case KVM_REG_PPC_TB_OFFSET: 1486 /* round up to multiple of 2^24 */ 1487 vcpu->arch.vcore->tb_offset = 1488 ALIGN(set_reg_val(id, *val), 1UL << 24); 1489 break; 1490 case KVM_REG_PPC_LPCR: 1491 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true); 1492 break; 1493 case KVM_REG_PPC_LPCR_64: 1494 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false); 1495 break; 1496 case KVM_REG_PPC_PPR: 1497 vcpu->arch.ppr = set_reg_val(id, *val); 1498 break; 1499 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1500 case KVM_REG_PPC_TFHAR: 1501 vcpu->arch.tfhar = set_reg_val(id, *val); 1502 break; 1503 case KVM_REG_PPC_TFIAR: 1504 vcpu->arch.tfiar = set_reg_val(id, *val); 1505 break; 1506 case KVM_REG_PPC_TEXASR: 1507 vcpu->arch.texasr = set_reg_val(id, *val); 1508 break; 1509 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31: 1510 i = id - KVM_REG_PPC_TM_GPR0; 1511 vcpu->arch.gpr_tm[i] = set_reg_val(id, *val); 1512 break; 1513 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63: 1514 { 1515 int j; 1516 i = id - KVM_REG_PPC_TM_VSR0; 1517 if (i < 32) 1518 for (j = 0; j < TS_FPRWIDTH; j++) 1519 vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j]; 1520 else 1521 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1522 vcpu->arch.vr_tm.vr[i-32] = val->vval; 1523 else 1524 r = -ENXIO; 1525 break; 1526 } 1527 case KVM_REG_PPC_TM_CR: 1528 vcpu->arch.cr_tm = set_reg_val(id, *val); 1529 break; 1530 case KVM_REG_PPC_TM_XER: 1531 vcpu->arch.xer_tm = set_reg_val(id, *val); 1532 break; 1533 case KVM_REG_PPC_TM_LR: 1534 vcpu->arch.lr_tm = set_reg_val(id, *val); 1535 break; 1536 case KVM_REG_PPC_TM_CTR: 1537 vcpu->arch.ctr_tm = set_reg_val(id, *val); 1538 break; 1539 case KVM_REG_PPC_TM_FPSCR: 1540 vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val); 1541 break; 1542 case KVM_REG_PPC_TM_AMR: 1543 vcpu->arch.amr_tm = set_reg_val(id, *val); 1544 break; 1545 case KVM_REG_PPC_TM_PPR: 1546 vcpu->arch.ppr_tm = set_reg_val(id, *val); 1547 break; 1548 case KVM_REG_PPC_TM_VRSAVE: 1549 vcpu->arch.vrsave_tm = set_reg_val(id, *val); 1550 break; 1551 case KVM_REG_PPC_TM_VSCR: 1552 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1553 vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val); 1554 else 1555 r = - ENXIO; 1556 break; 1557 case KVM_REG_PPC_TM_DSCR: 1558 vcpu->arch.dscr_tm = set_reg_val(id, *val); 1559 break; 1560 case KVM_REG_PPC_TM_TAR: 1561 vcpu->arch.tar_tm = set_reg_val(id, *val); 1562 break; 1563 #endif 1564 case KVM_REG_PPC_ARCH_COMPAT: 1565 r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val)); 1566 break; 1567 default: 1568 r = -EINVAL; 1569 break; 1570 } 1571 1572 return r; 1573 } 1574 1575 /* 1576 * On POWER9, threads are independent and can be in different partitions. 1577 * Therefore we consider each thread to be a subcore. 1578 * There is a restriction that all threads have to be in the same 1579 * MMU mode (radix or HPT), unfortunately, but since we only support 1580 * HPT guests on a HPT host so far, that isn't an impediment yet. 1581 */ 1582 static int threads_per_vcore(void) 1583 { 1584 if (cpu_has_feature(CPU_FTR_ARCH_300)) 1585 return 1; 1586 return threads_per_subcore; 1587 } 1588 1589 static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core) 1590 { 1591 struct kvmppc_vcore *vcore; 1592 1593 vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL); 1594 1595 if (vcore == NULL) 1596 return NULL; 1597 1598 spin_lock_init(&vcore->lock); 1599 spin_lock_init(&vcore->stoltb_lock); 1600 init_swait_queue_head(&vcore->wq); 1601 vcore->preempt_tb = TB_NIL; 1602 vcore->lpcr = kvm->arch.lpcr; 1603 vcore->first_vcpuid = core * threads_per_vcore(); 1604 vcore->kvm = kvm; 1605 INIT_LIST_HEAD(&vcore->preempt_list); 1606 1607 return vcore; 1608 } 1609 1610 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1611 static struct debugfs_timings_element { 1612 const char *name; 1613 size_t offset; 1614 } timings[] = { 1615 {"rm_entry", offsetof(struct kvm_vcpu, arch.rm_entry)}, 1616 {"rm_intr", offsetof(struct kvm_vcpu, arch.rm_intr)}, 1617 {"rm_exit", offsetof(struct kvm_vcpu, arch.rm_exit)}, 1618 {"guest", offsetof(struct kvm_vcpu, arch.guest_time)}, 1619 {"cede", offsetof(struct kvm_vcpu, arch.cede_time)}, 1620 }; 1621 1622 #define N_TIMINGS (sizeof(timings) / sizeof(timings[0])) 1623 1624 struct debugfs_timings_state { 1625 struct kvm_vcpu *vcpu; 1626 unsigned int buflen; 1627 char buf[N_TIMINGS * 100]; 1628 }; 1629 1630 static int debugfs_timings_open(struct inode *inode, struct file *file) 1631 { 1632 struct kvm_vcpu *vcpu = inode->i_private; 1633 struct debugfs_timings_state *p; 1634 1635 p = kzalloc(sizeof(*p), GFP_KERNEL); 1636 if (!p) 1637 return -ENOMEM; 1638 1639 kvm_get_kvm(vcpu->kvm); 1640 p->vcpu = vcpu; 1641 file->private_data = p; 1642 1643 return nonseekable_open(inode, file); 1644 } 1645 1646 static int debugfs_timings_release(struct inode *inode, struct file *file) 1647 { 1648 struct debugfs_timings_state *p = file->private_data; 1649 1650 kvm_put_kvm(p->vcpu->kvm); 1651 kfree(p); 1652 return 0; 1653 } 1654 1655 static ssize_t debugfs_timings_read(struct file *file, char __user *buf, 1656 size_t len, loff_t *ppos) 1657 { 1658 struct debugfs_timings_state *p = file->private_data; 1659 struct kvm_vcpu *vcpu = p->vcpu; 1660 char *s, *buf_end; 1661 struct kvmhv_tb_accumulator tb; 1662 u64 count; 1663 loff_t pos; 1664 ssize_t n; 1665 int i, loops; 1666 bool ok; 1667 1668 if (!p->buflen) { 1669 s = p->buf; 1670 buf_end = s + sizeof(p->buf); 1671 for (i = 0; i < N_TIMINGS; ++i) { 1672 struct kvmhv_tb_accumulator *acc; 1673 1674 acc = (struct kvmhv_tb_accumulator *) 1675 ((unsigned long)vcpu + timings[i].offset); 1676 ok = false; 1677 for (loops = 0; loops < 1000; ++loops) { 1678 count = acc->seqcount; 1679 if (!(count & 1)) { 1680 smp_rmb(); 1681 tb = *acc; 1682 smp_rmb(); 1683 if (count == acc->seqcount) { 1684 ok = true; 1685 break; 1686 } 1687 } 1688 udelay(1); 1689 } 1690 if (!ok) 1691 snprintf(s, buf_end - s, "%s: stuck\n", 1692 timings[i].name); 1693 else 1694 snprintf(s, buf_end - s, 1695 "%s: %llu %llu %llu %llu\n", 1696 timings[i].name, count / 2, 1697 tb_to_ns(tb.tb_total), 1698 tb_to_ns(tb.tb_min), 1699 tb_to_ns(tb.tb_max)); 1700 s += strlen(s); 1701 } 1702 p->buflen = s - p->buf; 1703 } 1704 1705 pos = *ppos; 1706 if (pos >= p->buflen) 1707 return 0; 1708 if (len > p->buflen - pos) 1709 len = p->buflen - pos; 1710 n = copy_to_user(buf, p->buf + pos, len); 1711 if (n) { 1712 if (n == len) 1713 return -EFAULT; 1714 len -= n; 1715 } 1716 *ppos = pos + len; 1717 return len; 1718 } 1719 1720 static ssize_t debugfs_timings_write(struct file *file, const char __user *buf, 1721 size_t len, loff_t *ppos) 1722 { 1723 return -EACCES; 1724 } 1725 1726 static const struct file_operations debugfs_timings_ops = { 1727 .owner = THIS_MODULE, 1728 .open = debugfs_timings_open, 1729 .release = debugfs_timings_release, 1730 .read = debugfs_timings_read, 1731 .write = debugfs_timings_write, 1732 .llseek = generic_file_llseek, 1733 }; 1734 1735 /* Create a debugfs directory for the vcpu */ 1736 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id) 1737 { 1738 char buf[16]; 1739 struct kvm *kvm = vcpu->kvm; 1740 1741 snprintf(buf, sizeof(buf), "vcpu%u", id); 1742 if (IS_ERR_OR_NULL(kvm->arch.debugfs_dir)) 1743 return; 1744 vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir); 1745 if (IS_ERR_OR_NULL(vcpu->arch.debugfs_dir)) 1746 return; 1747 vcpu->arch.debugfs_timings = 1748 debugfs_create_file("timings", 0444, vcpu->arch.debugfs_dir, 1749 vcpu, &debugfs_timings_ops); 1750 } 1751 1752 #else /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */ 1753 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id) 1754 { 1755 } 1756 #endif /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */ 1757 1758 static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm, 1759 unsigned int id) 1760 { 1761 struct kvm_vcpu *vcpu; 1762 int err = -EINVAL; 1763 int core; 1764 struct kvmppc_vcore *vcore; 1765 1766 core = id / threads_per_vcore(); 1767 if (core >= KVM_MAX_VCORES) 1768 goto out; 1769 1770 err = -ENOMEM; 1771 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); 1772 if (!vcpu) 1773 goto out; 1774 1775 err = kvm_vcpu_init(vcpu, kvm, id); 1776 if (err) 1777 goto free_vcpu; 1778 1779 vcpu->arch.shared = &vcpu->arch.shregs; 1780 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1781 /* 1782 * The shared struct is never shared on HV, 1783 * so we can always use host endianness 1784 */ 1785 #ifdef __BIG_ENDIAN__ 1786 vcpu->arch.shared_big_endian = true; 1787 #else 1788 vcpu->arch.shared_big_endian = false; 1789 #endif 1790 #endif 1791 vcpu->arch.mmcr[0] = MMCR0_FC; 1792 vcpu->arch.ctrl = CTRL_RUNLATCH; 1793 /* default to host PVR, since we can't spoof it */ 1794 kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR)); 1795 spin_lock_init(&vcpu->arch.vpa_update_lock); 1796 spin_lock_init(&vcpu->arch.tbacct_lock); 1797 vcpu->arch.busy_preempt = TB_NIL; 1798 vcpu->arch.intr_msr = MSR_SF | MSR_ME; 1799 1800 kvmppc_mmu_book3s_hv_init(vcpu); 1801 1802 vcpu->arch.state = KVMPPC_VCPU_NOTREADY; 1803 1804 init_waitqueue_head(&vcpu->arch.cpu_run); 1805 1806 mutex_lock(&kvm->lock); 1807 vcore = kvm->arch.vcores[core]; 1808 if (!vcore) { 1809 vcore = kvmppc_vcore_create(kvm, core); 1810 kvm->arch.vcores[core] = vcore; 1811 kvm->arch.online_vcores++; 1812 } 1813 mutex_unlock(&kvm->lock); 1814 1815 if (!vcore) 1816 goto free_vcpu; 1817 1818 spin_lock(&vcore->lock); 1819 ++vcore->num_threads; 1820 spin_unlock(&vcore->lock); 1821 vcpu->arch.vcore = vcore; 1822 vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid; 1823 vcpu->arch.thread_cpu = -1; 1824 1825 vcpu->arch.cpu_type = KVM_CPU_3S_64; 1826 kvmppc_sanity_check(vcpu); 1827 1828 debugfs_vcpu_init(vcpu, id); 1829 1830 return vcpu; 1831 1832 free_vcpu: 1833 kmem_cache_free(kvm_vcpu_cache, vcpu); 1834 out: 1835 return ERR_PTR(err); 1836 } 1837 1838 static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa) 1839 { 1840 if (vpa->pinned_addr) 1841 kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa, 1842 vpa->dirty); 1843 } 1844 1845 static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu) 1846 { 1847 spin_lock(&vcpu->arch.vpa_update_lock); 1848 unpin_vpa(vcpu->kvm, &vcpu->arch.dtl); 1849 unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow); 1850 unpin_vpa(vcpu->kvm, &vcpu->arch.vpa); 1851 spin_unlock(&vcpu->arch.vpa_update_lock); 1852 kvm_vcpu_uninit(vcpu); 1853 kmem_cache_free(kvm_vcpu_cache, vcpu); 1854 } 1855 1856 static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu) 1857 { 1858 /* Indicate we want to get back into the guest */ 1859 return 1; 1860 } 1861 1862 static void kvmppc_set_timer(struct kvm_vcpu *vcpu) 1863 { 1864 unsigned long dec_nsec, now; 1865 1866 now = get_tb(); 1867 if (now > vcpu->arch.dec_expires) { 1868 /* decrementer has already gone negative */ 1869 kvmppc_core_queue_dec(vcpu); 1870 kvmppc_core_prepare_to_enter(vcpu); 1871 return; 1872 } 1873 dec_nsec = (vcpu->arch.dec_expires - now) * NSEC_PER_SEC 1874 / tb_ticks_per_sec; 1875 hrtimer_start(&vcpu->arch.dec_timer, dec_nsec, HRTIMER_MODE_REL); 1876 vcpu->arch.timer_running = 1; 1877 } 1878 1879 static void kvmppc_end_cede(struct kvm_vcpu *vcpu) 1880 { 1881 vcpu->arch.ceded = 0; 1882 if (vcpu->arch.timer_running) { 1883 hrtimer_try_to_cancel(&vcpu->arch.dec_timer); 1884 vcpu->arch.timer_running = 0; 1885 } 1886 } 1887 1888 extern void __kvmppc_vcore_entry(void); 1889 1890 static void kvmppc_remove_runnable(struct kvmppc_vcore *vc, 1891 struct kvm_vcpu *vcpu) 1892 { 1893 u64 now; 1894 1895 if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE) 1896 return; 1897 spin_lock_irq(&vcpu->arch.tbacct_lock); 1898 now = mftb(); 1899 vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) - 1900 vcpu->arch.stolen_logged; 1901 vcpu->arch.busy_preempt = now; 1902 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST; 1903 spin_unlock_irq(&vcpu->arch.tbacct_lock); 1904 --vc->n_runnable; 1905 WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], NULL); 1906 } 1907 1908 static int kvmppc_grab_hwthread(int cpu) 1909 { 1910 struct paca_struct *tpaca; 1911 long timeout = 10000; 1912 1913 tpaca = &paca[cpu]; 1914 1915 /* Ensure the thread won't go into the kernel if it wakes */ 1916 tpaca->kvm_hstate.kvm_vcpu = NULL; 1917 tpaca->kvm_hstate.kvm_vcore = NULL; 1918 tpaca->kvm_hstate.napping = 0; 1919 smp_wmb(); 1920 tpaca->kvm_hstate.hwthread_req = 1; 1921 1922 /* 1923 * If the thread is already executing in the kernel (e.g. handling 1924 * a stray interrupt), wait for it to get back to nap mode. 1925 * The smp_mb() is to ensure that our setting of hwthread_req 1926 * is visible before we look at hwthread_state, so if this 1927 * races with the code at system_reset_pSeries and the thread 1928 * misses our setting of hwthread_req, we are sure to see its 1929 * setting of hwthread_state, and vice versa. 1930 */ 1931 smp_mb(); 1932 while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) { 1933 if (--timeout <= 0) { 1934 pr_err("KVM: couldn't grab cpu %d\n", cpu); 1935 return -EBUSY; 1936 } 1937 udelay(1); 1938 } 1939 return 0; 1940 } 1941 1942 static void kvmppc_release_hwthread(int cpu) 1943 { 1944 struct paca_struct *tpaca; 1945 1946 tpaca = &paca[cpu]; 1947 tpaca->kvm_hstate.hwthread_req = 0; 1948 tpaca->kvm_hstate.kvm_vcpu = NULL; 1949 tpaca->kvm_hstate.kvm_vcore = NULL; 1950 tpaca->kvm_hstate.kvm_split_mode = NULL; 1951 } 1952 1953 static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc) 1954 { 1955 int cpu; 1956 struct paca_struct *tpaca; 1957 struct kvmppc_vcore *mvc = vc->master_vcore; 1958 1959 cpu = vc->pcpu; 1960 if (vcpu) { 1961 if (vcpu->arch.timer_running) { 1962 hrtimer_try_to_cancel(&vcpu->arch.dec_timer); 1963 vcpu->arch.timer_running = 0; 1964 } 1965 cpu += vcpu->arch.ptid; 1966 vcpu->cpu = mvc->pcpu; 1967 vcpu->arch.thread_cpu = cpu; 1968 } 1969 tpaca = &paca[cpu]; 1970 tpaca->kvm_hstate.kvm_vcpu = vcpu; 1971 tpaca->kvm_hstate.ptid = cpu - mvc->pcpu; 1972 /* Order stores to hstate.kvm_vcpu etc. before store to kvm_vcore */ 1973 smp_wmb(); 1974 tpaca->kvm_hstate.kvm_vcore = mvc; 1975 if (cpu != smp_processor_id()) 1976 kvmppc_ipi_thread(cpu); 1977 } 1978 1979 static void kvmppc_wait_for_nap(void) 1980 { 1981 int cpu = smp_processor_id(); 1982 int i, loops; 1983 int n_threads = threads_per_vcore(); 1984 1985 if (n_threads <= 1) 1986 return; 1987 for (loops = 0; loops < 1000000; ++loops) { 1988 /* 1989 * Check if all threads are finished. 1990 * We set the vcore pointer when starting a thread 1991 * and the thread clears it when finished, so we look 1992 * for any threads that still have a non-NULL vcore ptr. 1993 */ 1994 for (i = 1; i < n_threads; ++i) 1995 if (paca[cpu + i].kvm_hstate.kvm_vcore) 1996 break; 1997 if (i == n_threads) { 1998 HMT_medium(); 1999 return; 2000 } 2001 HMT_low(); 2002 } 2003 HMT_medium(); 2004 for (i = 1; i < n_threads; ++i) 2005 if (paca[cpu + i].kvm_hstate.kvm_vcore) 2006 pr_err("KVM: CPU %d seems to be stuck\n", cpu + i); 2007 } 2008 2009 /* 2010 * Check that we are on thread 0 and that any other threads in 2011 * this core are off-line. Then grab the threads so they can't 2012 * enter the kernel. 2013 */ 2014 static int on_primary_thread(void) 2015 { 2016 int cpu = smp_processor_id(); 2017 int thr; 2018 2019 /* Are we on a primary subcore? */ 2020 if (cpu_thread_in_subcore(cpu)) 2021 return 0; 2022 2023 thr = 0; 2024 while (++thr < threads_per_subcore) 2025 if (cpu_online(cpu + thr)) 2026 return 0; 2027 2028 /* Grab all hw threads so they can't go into the kernel */ 2029 for (thr = 1; thr < threads_per_subcore; ++thr) { 2030 if (kvmppc_grab_hwthread(cpu + thr)) { 2031 /* Couldn't grab one; let the others go */ 2032 do { 2033 kvmppc_release_hwthread(cpu + thr); 2034 } while (--thr > 0); 2035 return 0; 2036 } 2037 } 2038 return 1; 2039 } 2040 2041 /* 2042 * A list of virtual cores for each physical CPU. 2043 * These are vcores that could run but their runner VCPU tasks are 2044 * (or may be) preempted. 2045 */ 2046 struct preempted_vcore_list { 2047 struct list_head list; 2048 spinlock_t lock; 2049 }; 2050 2051 static DEFINE_PER_CPU(struct preempted_vcore_list, preempted_vcores); 2052 2053 static void init_vcore_lists(void) 2054 { 2055 int cpu; 2056 2057 for_each_possible_cpu(cpu) { 2058 struct preempted_vcore_list *lp = &per_cpu(preempted_vcores, cpu); 2059 spin_lock_init(&lp->lock); 2060 INIT_LIST_HEAD(&lp->list); 2061 } 2062 } 2063 2064 static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc) 2065 { 2066 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores); 2067 2068 vc->vcore_state = VCORE_PREEMPT; 2069 vc->pcpu = smp_processor_id(); 2070 if (vc->num_threads < threads_per_vcore()) { 2071 spin_lock(&lp->lock); 2072 list_add_tail(&vc->preempt_list, &lp->list); 2073 spin_unlock(&lp->lock); 2074 } 2075 2076 /* Start accumulating stolen time */ 2077 kvmppc_core_start_stolen(vc); 2078 } 2079 2080 static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc) 2081 { 2082 struct preempted_vcore_list *lp; 2083 2084 kvmppc_core_end_stolen(vc); 2085 if (!list_empty(&vc->preempt_list)) { 2086 lp = &per_cpu(preempted_vcores, vc->pcpu); 2087 spin_lock(&lp->lock); 2088 list_del_init(&vc->preempt_list); 2089 spin_unlock(&lp->lock); 2090 } 2091 vc->vcore_state = VCORE_INACTIVE; 2092 } 2093 2094 /* 2095 * This stores information about the virtual cores currently 2096 * assigned to a physical core. 2097 */ 2098 struct core_info { 2099 int n_subcores; 2100 int max_subcore_threads; 2101 int total_threads; 2102 int subcore_threads[MAX_SUBCORES]; 2103 struct kvm *subcore_vm[MAX_SUBCORES]; 2104 struct list_head vcs[MAX_SUBCORES]; 2105 }; 2106 2107 /* 2108 * This mapping means subcores 0 and 1 can use threads 0-3 and 4-7 2109 * respectively in 2-way micro-threading (split-core) mode. 2110 */ 2111 static int subcore_thread_map[MAX_SUBCORES] = { 0, 4, 2, 6 }; 2112 2113 static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc) 2114 { 2115 int sub; 2116 2117 memset(cip, 0, sizeof(*cip)); 2118 cip->n_subcores = 1; 2119 cip->max_subcore_threads = vc->num_threads; 2120 cip->total_threads = vc->num_threads; 2121 cip->subcore_threads[0] = vc->num_threads; 2122 cip->subcore_vm[0] = vc->kvm; 2123 for (sub = 0; sub < MAX_SUBCORES; ++sub) 2124 INIT_LIST_HEAD(&cip->vcs[sub]); 2125 list_add_tail(&vc->preempt_list, &cip->vcs[0]); 2126 } 2127 2128 static bool subcore_config_ok(int n_subcores, int n_threads) 2129 { 2130 /* Can only dynamically split if unsplit to begin with */ 2131 if (n_subcores > 1 && threads_per_subcore < MAX_SMT_THREADS) 2132 return false; 2133 if (n_subcores > MAX_SUBCORES) 2134 return false; 2135 if (n_subcores > 1) { 2136 if (!(dynamic_mt_modes & 2)) 2137 n_subcores = 4; 2138 if (n_subcores > 2 && !(dynamic_mt_modes & 4)) 2139 return false; 2140 } 2141 2142 return n_subcores * roundup_pow_of_two(n_threads) <= MAX_SMT_THREADS; 2143 } 2144 2145 static void init_master_vcore(struct kvmppc_vcore *vc) 2146 { 2147 vc->master_vcore = vc; 2148 vc->entry_exit_map = 0; 2149 vc->in_guest = 0; 2150 vc->napping_threads = 0; 2151 vc->conferring_threads = 0; 2152 } 2153 2154 static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip) 2155 { 2156 int n_threads = vc->num_threads; 2157 int sub; 2158 2159 if (!cpu_has_feature(CPU_FTR_ARCH_207S)) 2160 return false; 2161 2162 if (n_threads < cip->max_subcore_threads) 2163 n_threads = cip->max_subcore_threads; 2164 if (!subcore_config_ok(cip->n_subcores + 1, n_threads)) 2165 return false; 2166 cip->max_subcore_threads = n_threads; 2167 2168 sub = cip->n_subcores; 2169 ++cip->n_subcores; 2170 cip->total_threads += vc->num_threads; 2171 cip->subcore_threads[sub] = vc->num_threads; 2172 cip->subcore_vm[sub] = vc->kvm; 2173 init_master_vcore(vc); 2174 list_move_tail(&vc->preempt_list, &cip->vcs[sub]); 2175 2176 return true; 2177 } 2178 2179 /* 2180 * Work out whether it is possible to piggyback the execution of 2181 * vcore *pvc onto the execution of the other vcores described in *cip. 2182 */ 2183 static bool can_piggyback(struct kvmppc_vcore *pvc, struct core_info *cip, 2184 int target_threads) 2185 { 2186 if (cip->total_threads + pvc->num_threads > target_threads) 2187 return false; 2188 2189 return can_dynamic_split(pvc, cip); 2190 } 2191 2192 static void prepare_threads(struct kvmppc_vcore *vc) 2193 { 2194 int i; 2195 struct kvm_vcpu *vcpu; 2196 2197 for_each_runnable_thread(i, vcpu, vc) { 2198 if (signal_pending(vcpu->arch.run_task)) 2199 vcpu->arch.ret = -EINTR; 2200 else if (vcpu->arch.vpa.update_pending || 2201 vcpu->arch.slb_shadow.update_pending || 2202 vcpu->arch.dtl.update_pending) 2203 vcpu->arch.ret = RESUME_GUEST; 2204 else 2205 continue; 2206 kvmppc_remove_runnable(vc, vcpu); 2207 wake_up(&vcpu->arch.cpu_run); 2208 } 2209 } 2210 2211 static void collect_piggybacks(struct core_info *cip, int target_threads) 2212 { 2213 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores); 2214 struct kvmppc_vcore *pvc, *vcnext; 2215 2216 spin_lock(&lp->lock); 2217 list_for_each_entry_safe(pvc, vcnext, &lp->list, preempt_list) { 2218 if (!spin_trylock(&pvc->lock)) 2219 continue; 2220 prepare_threads(pvc); 2221 if (!pvc->n_runnable) { 2222 list_del_init(&pvc->preempt_list); 2223 if (pvc->runner == NULL) { 2224 pvc->vcore_state = VCORE_INACTIVE; 2225 kvmppc_core_end_stolen(pvc); 2226 } 2227 spin_unlock(&pvc->lock); 2228 continue; 2229 } 2230 if (!can_piggyback(pvc, cip, target_threads)) { 2231 spin_unlock(&pvc->lock); 2232 continue; 2233 } 2234 kvmppc_core_end_stolen(pvc); 2235 pvc->vcore_state = VCORE_PIGGYBACK; 2236 if (cip->total_threads >= target_threads) 2237 break; 2238 } 2239 spin_unlock(&lp->lock); 2240 } 2241 2242 static void post_guest_process(struct kvmppc_vcore *vc, bool is_master) 2243 { 2244 int still_running = 0, i; 2245 u64 now; 2246 long ret; 2247 struct kvm_vcpu *vcpu; 2248 2249 spin_lock(&vc->lock); 2250 now = get_tb(); 2251 for_each_runnable_thread(i, vcpu, vc) { 2252 /* cancel pending dec exception if dec is positive */ 2253 if (now < vcpu->arch.dec_expires && 2254 kvmppc_core_pending_dec(vcpu)) 2255 kvmppc_core_dequeue_dec(vcpu); 2256 2257 trace_kvm_guest_exit(vcpu); 2258 2259 ret = RESUME_GUEST; 2260 if (vcpu->arch.trap) 2261 ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu, 2262 vcpu->arch.run_task); 2263 2264 vcpu->arch.ret = ret; 2265 vcpu->arch.trap = 0; 2266 2267 if (is_kvmppc_resume_guest(vcpu->arch.ret)) { 2268 if (vcpu->arch.pending_exceptions) 2269 kvmppc_core_prepare_to_enter(vcpu); 2270 if (vcpu->arch.ceded) 2271 kvmppc_set_timer(vcpu); 2272 else 2273 ++still_running; 2274 } else { 2275 kvmppc_remove_runnable(vc, vcpu); 2276 wake_up(&vcpu->arch.cpu_run); 2277 } 2278 } 2279 list_del_init(&vc->preempt_list); 2280 if (!is_master) { 2281 if (still_running > 0) { 2282 kvmppc_vcore_preempt(vc); 2283 } else if (vc->runner) { 2284 vc->vcore_state = VCORE_PREEMPT; 2285 kvmppc_core_start_stolen(vc); 2286 } else { 2287 vc->vcore_state = VCORE_INACTIVE; 2288 } 2289 if (vc->n_runnable > 0 && vc->runner == NULL) { 2290 /* make sure there's a candidate runner awake */ 2291 i = -1; 2292 vcpu = next_runnable_thread(vc, &i); 2293 wake_up(&vcpu->arch.cpu_run); 2294 } 2295 } 2296 spin_unlock(&vc->lock); 2297 } 2298 2299 /* 2300 * Clear core from the list of active host cores as we are about to 2301 * enter the guest. Only do this if it is the primary thread of the 2302 * core (not if a subcore) that is entering the guest. 2303 */ 2304 static inline int kvmppc_clear_host_core(unsigned int cpu) 2305 { 2306 int core; 2307 2308 if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu)) 2309 return 0; 2310 /* 2311 * Memory barrier can be omitted here as we will do a smp_wmb() 2312 * later in kvmppc_start_thread and we need ensure that state is 2313 * visible to other CPUs only after we enter guest. 2314 */ 2315 core = cpu >> threads_shift; 2316 kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 0; 2317 return 0; 2318 } 2319 2320 /* 2321 * Advertise this core as an active host core since we exited the guest 2322 * Only need to do this if it is the primary thread of the core that is 2323 * exiting. 2324 */ 2325 static inline int kvmppc_set_host_core(unsigned int cpu) 2326 { 2327 int core; 2328 2329 if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu)) 2330 return 0; 2331 2332 /* 2333 * Memory barrier can be omitted here because we do a spin_unlock 2334 * immediately after this which provides the memory barrier. 2335 */ 2336 core = cpu >> threads_shift; 2337 kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 1; 2338 return 0; 2339 } 2340 2341 /* 2342 * Run a set of guest threads on a physical core. 2343 * Called with vc->lock held. 2344 */ 2345 static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) 2346 { 2347 struct kvm_vcpu *vcpu; 2348 int i; 2349 int srcu_idx; 2350 struct core_info core_info; 2351 struct kvmppc_vcore *pvc, *vcnext; 2352 struct kvm_split_mode split_info, *sip; 2353 int split, subcore_size, active; 2354 int sub; 2355 bool thr0_done; 2356 unsigned long cmd_bit, stat_bit; 2357 int pcpu, thr; 2358 int target_threads; 2359 int controlled_threads; 2360 2361 /* 2362 * Remove from the list any threads that have a signal pending 2363 * or need a VPA update done 2364 */ 2365 prepare_threads(vc); 2366 2367 /* if the runner is no longer runnable, let the caller pick a new one */ 2368 if (vc->runner->arch.state != KVMPPC_VCPU_RUNNABLE) 2369 return; 2370 2371 /* 2372 * Initialize *vc. 2373 */ 2374 init_master_vcore(vc); 2375 vc->preempt_tb = TB_NIL; 2376 2377 /* 2378 * Number of threads that we will be controlling: the same as 2379 * the number of threads per subcore, except on POWER9, 2380 * where it's 1 because the threads are (mostly) independent. 2381 */ 2382 controlled_threads = threads_per_vcore(); 2383 2384 /* 2385 * Make sure we are running on primary threads, and that secondary 2386 * threads are offline. Also check if the number of threads in this 2387 * guest are greater than the current system threads per guest. 2388 */ 2389 if ((controlled_threads > 1) && 2390 ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) { 2391 for_each_runnable_thread(i, vcpu, vc) { 2392 vcpu->arch.ret = -EBUSY; 2393 kvmppc_remove_runnable(vc, vcpu); 2394 wake_up(&vcpu->arch.cpu_run); 2395 } 2396 goto out; 2397 } 2398 2399 /* 2400 * See if we could run any other vcores on the physical core 2401 * along with this one. 2402 */ 2403 init_core_info(&core_info, vc); 2404 pcpu = smp_processor_id(); 2405 target_threads = controlled_threads; 2406 if (target_smt_mode && target_smt_mode < target_threads) 2407 target_threads = target_smt_mode; 2408 if (vc->num_threads < target_threads) 2409 collect_piggybacks(&core_info, target_threads); 2410 2411 /* Decide on micro-threading (split-core) mode */ 2412 subcore_size = threads_per_subcore; 2413 cmd_bit = stat_bit = 0; 2414 split = core_info.n_subcores; 2415 sip = NULL; 2416 if (split > 1) { 2417 /* threads_per_subcore must be MAX_SMT_THREADS (8) here */ 2418 if (split == 2 && (dynamic_mt_modes & 2)) { 2419 cmd_bit = HID0_POWER8_1TO2LPAR; 2420 stat_bit = HID0_POWER8_2LPARMODE; 2421 } else { 2422 split = 4; 2423 cmd_bit = HID0_POWER8_1TO4LPAR; 2424 stat_bit = HID0_POWER8_4LPARMODE; 2425 } 2426 subcore_size = MAX_SMT_THREADS / split; 2427 sip = &split_info; 2428 memset(&split_info, 0, sizeof(split_info)); 2429 split_info.rpr = mfspr(SPRN_RPR); 2430 split_info.pmmar = mfspr(SPRN_PMMAR); 2431 split_info.ldbar = mfspr(SPRN_LDBAR); 2432 split_info.subcore_size = subcore_size; 2433 for (sub = 0; sub < core_info.n_subcores; ++sub) 2434 split_info.master_vcs[sub] = 2435 list_first_entry(&core_info.vcs[sub], 2436 struct kvmppc_vcore, preempt_list); 2437 /* order writes to split_info before kvm_split_mode pointer */ 2438 smp_wmb(); 2439 } 2440 pcpu = smp_processor_id(); 2441 for (thr = 0; thr < controlled_threads; ++thr) 2442 paca[pcpu + thr].kvm_hstate.kvm_split_mode = sip; 2443 2444 /* Initiate micro-threading (split-core) if required */ 2445 if (cmd_bit) { 2446 unsigned long hid0 = mfspr(SPRN_HID0); 2447 2448 hid0 |= cmd_bit | HID0_POWER8_DYNLPARDIS; 2449 mb(); 2450 mtspr(SPRN_HID0, hid0); 2451 isync(); 2452 for (;;) { 2453 hid0 = mfspr(SPRN_HID0); 2454 if (hid0 & stat_bit) 2455 break; 2456 cpu_relax(); 2457 } 2458 } 2459 2460 kvmppc_clear_host_core(pcpu); 2461 2462 /* Start all the threads */ 2463 active = 0; 2464 for (sub = 0; sub < core_info.n_subcores; ++sub) { 2465 thr = subcore_thread_map[sub]; 2466 thr0_done = false; 2467 active |= 1 << thr; 2468 list_for_each_entry(pvc, &core_info.vcs[sub], preempt_list) { 2469 pvc->pcpu = pcpu + thr; 2470 for_each_runnable_thread(i, vcpu, pvc) { 2471 kvmppc_start_thread(vcpu, pvc); 2472 kvmppc_create_dtl_entry(vcpu, pvc); 2473 trace_kvm_guest_enter(vcpu); 2474 if (!vcpu->arch.ptid) 2475 thr0_done = true; 2476 active |= 1 << (thr + vcpu->arch.ptid); 2477 } 2478 /* 2479 * We need to start the first thread of each subcore 2480 * even if it doesn't have a vcpu. 2481 */ 2482 if (pvc->master_vcore == pvc && !thr0_done) 2483 kvmppc_start_thread(NULL, pvc); 2484 thr += pvc->num_threads; 2485 } 2486 } 2487 2488 /* 2489 * Ensure that split_info.do_nap is set after setting 2490 * the vcore pointer in the PACA of the secondaries. 2491 */ 2492 smp_mb(); 2493 if (cmd_bit) 2494 split_info.do_nap = 1; /* ask secondaries to nap when done */ 2495 2496 /* 2497 * When doing micro-threading, poke the inactive threads as well. 2498 * This gets them to the nap instruction after kvm_do_nap, 2499 * which reduces the time taken to unsplit later. 2500 */ 2501 if (split > 1) 2502 for (thr = 1; thr < threads_per_subcore; ++thr) 2503 if (!(active & (1 << thr))) 2504 kvmppc_ipi_thread(pcpu + thr); 2505 2506 vc->vcore_state = VCORE_RUNNING; 2507 preempt_disable(); 2508 2509 trace_kvmppc_run_core(vc, 0); 2510 2511 for (sub = 0; sub < core_info.n_subcores; ++sub) 2512 list_for_each_entry(pvc, &core_info.vcs[sub], preempt_list) 2513 spin_unlock(&pvc->lock); 2514 2515 guest_enter(); 2516 2517 srcu_idx = srcu_read_lock(&vc->kvm->srcu); 2518 2519 __kvmppc_vcore_entry(); 2520 2521 srcu_read_unlock(&vc->kvm->srcu, srcu_idx); 2522 2523 spin_lock(&vc->lock); 2524 /* prevent other vcpu threads from doing kvmppc_start_thread() now */ 2525 vc->vcore_state = VCORE_EXITING; 2526 2527 /* wait for secondary threads to finish writing their state to memory */ 2528 kvmppc_wait_for_nap(); 2529 2530 /* Return to whole-core mode if we split the core earlier */ 2531 if (split > 1) { 2532 unsigned long hid0 = mfspr(SPRN_HID0); 2533 unsigned long loops = 0; 2534 2535 hid0 &= ~HID0_POWER8_DYNLPARDIS; 2536 stat_bit = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE; 2537 mb(); 2538 mtspr(SPRN_HID0, hid0); 2539 isync(); 2540 for (;;) { 2541 hid0 = mfspr(SPRN_HID0); 2542 if (!(hid0 & stat_bit)) 2543 break; 2544 cpu_relax(); 2545 ++loops; 2546 } 2547 split_info.do_nap = 0; 2548 } 2549 2550 /* Let secondaries go back to the offline loop */ 2551 for (i = 0; i < controlled_threads; ++i) { 2552 kvmppc_release_hwthread(pcpu + i); 2553 if (sip && sip->napped[i]) 2554 kvmppc_ipi_thread(pcpu + i); 2555 } 2556 2557 kvmppc_set_host_core(pcpu); 2558 2559 spin_unlock(&vc->lock); 2560 2561 /* make sure updates to secondary vcpu structs are visible now */ 2562 smp_mb(); 2563 guest_exit(); 2564 2565 for (sub = 0; sub < core_info.n_subcores; ++sub) 2566 list_for_each_entry_safe(pvc, vcnext, &core_info.vcs[sub], 2567 preempt_list) 2568 post_guest_process(pvc, pvc == vc); 2569 2570 spin_lock(&vc->lock); 2571 preempt_enable(); 2572 2573 out: 2574 vc->vcore_state = VCORE_INACTIVE; 2575 trace_kvmppc_run_core(vc, 1); 2576 } 2577 2578 /* 2579 * Wait for some other vcpu thread to execute us, and 2580 * wake us up when we need to handle something in the host. 2581 */ 2582 static void kvmppc_wait_for_exec(struct kvmppc_vcore *vc, 2583 struct kvm_vcpu *vcpu, int wait_state) 2584 { 2585 DEFINE_WAIT(wait); 2586 2587 prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state); 2588 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) { 2589 spin_unlock(&vc->lock); 2590 schedule(); 2591 spin_lock(&vc->lock); 2592 } 2593 finish_wait(&vcpu->arch.cpu_run, &wait); 2594 } 2595 2596 static void grow_halt_poll_ns(struct kvmppc_vcore *vc) 2597 { 2598 /* 10us base */ 2599 if (vc->halt_poll_ns == 0 && halt_poll_ns_grow) 2600 vc->halt_poll_ns = 10000; 2601 else 2602 vc->halt_poll_ns *= halt_poll_ns_grow; 2603 } 2604 2605 static void shrink_halt_poll_ns(struct kvmppc_vcore *vc) 2606 { 2607 if (halt_poll_ns_shrink == 0) 2608 vc->halt_poll_ns = 0; 2609 else 2610 vc->halt_poll_ns /= halt_poll_ns_shrink; 2611 } 2612 2613 /* 2614 * Check to see if any of the runnable vcpus on the vcore have pending 2615 * exceptions or are no longer ceded 2616 */ 2617 static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc) 2618 { 2619 struct kvm_vcpu *vcpu; 2620 int i; 2621 2622 for_each_runnable_thread(i, vcpu, vc) { 2623 if (vcpu->arch.pending_exceptions || !vcpu->arch.ceded) 2624 return 1; 2625 } 2626 2627 return 0; 2628 } 2629 2630 /* 2631 * All the vcpus in this vcore are idle, so wait for a decrementer 2632 * or external interrupt to one of the vcpus. vc->lock is held. 2633 */ 2634 static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) 2635 { 2636 ktime_t cur, start_poll, start_wait; 2637 int do_sleep = 1; 2638 u64 block_ns; 2639 DECLARE_SWAITQUEUE(wait); 2640 2641 /* Poll for pending exceptions and ceded state */ 2642 cur = start_poll = ktime_get(); 2643 if (vc->halt_poll_ns) { 2644 ktime_t stop = ktime_add_ns(start_poll, vc->halt_poll_ns); 2645 ++vc->runner->stat.halt_attempted_poll; 2646 2647 vc->vcore_state = VCORE_POLLING; 2648 spin_unlock(&vc->lock); 2649 2650 do { 2651 if (kvmppc_vcore_check_block(vc)) { 2652 do_sleep = 0; 2653 break; 2654 } 2655 cur = ktime_get(); 2656 } while (single_task_running() && ktime_before(cur, stop)); 2657 2658 spin_lock(&vc->lock); 2659 vc->vcore_state = VCORE_INACTIVE; 2660 2661 if (!do_sleep) { 2662 ++vc->runner->stat.halt_successful_poll; 2663 goto out; 2664 } 2665 } 2666 2667 prepare_to_swait(&vc->wq, &wait, TASK_INTERRUPTIBLE); 2668 2669 if (kvmppc_vcore_check_block(vc)) { 2670 finish_swait(&vc->wq, &wait); 2671 do_sleep = 0; 2672 /* If we polled, count this as a successful poll */ 2673 if (vc->halt_poll_ns) 2674 ++vc->runner->stat.halt_successful_poll; 2675 goto out; 2676 } 2677 2678 start_wait = ktime_get(); 2679 2680 vc->vcore_state = VCORE_SLEEPING; 2681 trace_kvmppc_vcore_blocked(vc, 0); 2682 spin_unlock(&vc->lock); 2683 schedule(); 2684 finish_swait(&vc->wq, &wait); 2685 spin_lock(&vc->lock); 2686 vc->vcore_state = VCORE_INACTIVE; 2687 trace_kvmppc_vcore_blocked(vc, 1); 2688 ++vc->runner->stat.halt_successful_wait; 2689 2690 cur = ktime_get(); 2691 2692 out: 2693 block_ns = ktime_to_ns(cur) - ktime_to_ns(start_poll); 2694 2695 /* Attribute wait time */ 2696 if (do_sleep) { 2697 vc->runner->stat.halt_wait_ns += 2698 ktime_to_ns(cur) - ktime_to_ns(start_wait); 2699 /* Attribute failed poll time */ 2700 if (vc->halt_poll_ns) 2701 vc->runner->stat.halt_poll_fail_ns += 2702 ktime_to_ns(start_wait) - 2703 ktime_to_ns(start_poll); 2704 } else { 2705 /* Attribute successful poll time */ 2706 if (vc->halt_poll_ns) 2707 vc->runner->stat.halt_poll_success_ns += 2708 ktime_to_ns(cur) - 2709 ktime_to_ns(start_poll); 2710 } 2711 2712 /* Adjust poll time */ 2713 if (halt_poll_ns) { 2714 if (block_ns <= vc->halt_poll_ns) 2715 ; 2716 /* We slept and blocked for longer than the max halt time */ 2717 else if (vc->halt_poll_ns && block_ns > halt_poll_ns) 2718 shrink_halt_poll_ns(vc); 2719 /* We slept and our poll time is too small */ 2720 else if (vc->halt_poll_ns < halt_poll_ns && 2721 block_ns < halt_poll_ns) 2722 grow_halt_poll_ns(vc); 2723 if (vc->halt_poll_ns > halt_poll_ns) 2724 vc->halt_poll_ns = halt_poll_ns; 2725 } else 2726 vc->halt_poll_ns = 0; 2727 2728 trace_kvmppc_vcore_wakeup(do_sleep, block_ns); 2729 } 2730 2731 static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) 2732 { 2733 int n_ceded, i; 2734 struct kvmppc_vcore *vc; 2735 struct kvm_vcpu *v; 2736 2737 trace_kvmppc_run_vcpu_enter(vcpu); 2738 2739 kvm_run->exit_reason = 0; 2740 vcpu->arch.ret = RESUME_GUEST; 2741 vcpu->arch.trap = 0; 2742 kvmppc_update_vpas(vcpu); 2743 2744 /* 2745 * Synchronize with other threads in this virtual core 2746 */ 2747 vc = vcpu->arch.vcore; 2748 spin_lock(&vc->lock); 2749 vcpu->arch.ceded = 0; 2750 vcpu->arch.run_task = current; 2751 vcpu->arch.kvm_run = kvm_run; 2752 vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb()); 2753 vcpu->arch.state = KVMPPC_VCPU_RUNNABLE; 2754 vcpu->arch.busy_preempt = TB_NIL; 2755 WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], vcpu); 2756 ++vc->n_runnable; 2757 2758 /* 2759 * This happens the first time this is called for a vcpu. 2760 * If the vcore is already running, we may be able to start 2761 * this thread straight away and have it join in. 2762 */ 2763 if (!signal_pending(current)) { 2764 if (vc->vcore_state == VCORE_PIGGYBACK) { 2765 struct kvmppc_vcore *mvc = vc->master_vcore; 2766 if (spin_trylock(&mvc->lock)) { 2767 if (mvc->vcore_state == VCORE_RUNNING && 2768 !VCORE_IS_EXITING(mvc)) { 2769 kvmppc_create_dtl_entry(vcpu, vc); 2770 kvmppc_start_thread(vcpu, vc); 2771 trace_kvm_guest_enter(vcpu); 2772 } 2773 spin_unlock(&mvc->lock); 2774 } 2775 } else if (vc->vcore_state == VCORE_RUNNING && 2776 !VCORE_IS_EXITING(vc)) { 2777 kvmppc_create_dtl_entry(vcpu, vc); 2778 kvmppc_start_thread(vcpu, vc); 2779 trace_kvm_guest_enter(vcpu); 2780 } else if (vc->vcore_state == VCORE_SLEEPING) { 2781 swake_up(&vc->wq); 2782 } 2783 2784 } 2785 2786 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE && 2787 !signal_pending(current)) { 2788 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL) 2789 kvmppc_vcore_end_preempt(vc); 2790 2791 if (vc->vcore_state != VCORE_INACTIVE) { 2792 kvmppc_wait_for_exec(vc, vcpu, TASK_INTERRUPTIBLE); 2793 continue; 2794 } 2795 for_each_runnable_thread(i, v, vc) { 2796 kvmppc_core_prepare_to_enter(v); 2797 if (signal_pending(v->arch.run_task)) { 2798 kvmppc_remove_runnable(vc, v); 2799 v->stat.signal_exits++; 2800 v->arch.kvm_run->exit_reason = KVM_EXIT_INTR; 2801 v->arch.ret = -EINTR; 2802 wake_up(&v->arch.cpu_run); 2803 } 2804 } 2805 if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE) 2806 break; 2807 n_ceded = 0; 2808 for_each_runnable_thread(i, v, vc) { 2809 if (!v->arch.pending_exceptions) 2810 n_ceded += v->arch.ceded; 2811 else 2812 v->arch.ceded = 0; 2813 } 2814 vc->runner = vcpu; 2815 if (n_ceded == vc->n_runnable) { 2816 kvmppc_vcore_blocked(vc); 2817 } else if (need_resched()) { 2818 kvmppc_vcore_preempt(vc); 2819 /* Let something else run */ 2820 cond_resched_lock(&vc->lock); 2821 if (vc->vcore_state == VCORE_PREEMPT) 2822 kvmppc_vcore_end_preempt(vc); 2823 } else { 2824 kvmppc_run_core(vc); 2825 } 2826 vc->runner = NULL; 2827 } 2828 2829 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE && 2830 (vc->vcore_state == VCORE_RUNNING || 2831 vc->vcore_state == VCORE_EXITING || 2832 vc->vcore_state == VCORE_PIGGYBACK)) 2833 kvmppc_wait_for_exec(vc, vcpu, TASK_UNINTERRUPTIBLE); 2834 2835 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL) 2836 kvmppc_vcore_end_preempt(vc); 2837 2838 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) { 2839 kvmppc_remove_runnable(vc, vcpu); 2840 vcpu->stat.signal_exits++; 2841 kvm_run->exit_reason = KVM_EXIT_INTR; 2842 vcpu->arch.ret = -EINTR; 2843 } 2844 2845 if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) { 2846 /* Wake up some vcpu to run the core */ 2847 i = -1; 2848 v = next_runnable_thread(vc, &i); 2849 wake_up(&v->arch.cpu_run); 2850 } 2851 2852 trace_kvmppc_run_vcpu_exit(vcpu, kvm_run); 2853 spin_unlock(&vc->lock); 2854 return vcpu->arch.ret; 2855 } 2856 2857 static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu) 2858 { 2859 int r; 2860 int srcu_idx; 2861 2862 if (!vcpu->arch.sane) { 2863 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 2864 return -EINVAL; 2865 } 2866 2867 kvmppc_core_prepare_to_enter(vcpu); 2868 2869 /* No need to go into the guest when all we'll do is come back out */ 2870 if (signal_pending(current)) { 2871 run->exit_reason = KVM_EXIT_INTR; 2872 return -EINTR; 2873 } 2874 2875 atomic_inc(&vcpu->kvm->arch.vcpus_running); 2876 /* Order vcpus_running vs. hpte_setup_done, see kvmppc_alloc_reset_hpt */ 2877 smp_mb(); 2878 2879 /* On the first time here, set up HTAB and VRMA */ 2880 if (!vcpu->kvm->arch.hpte_setup_done) { 2881 r = kvmppc_hv_setup_htab_rma(vcpu); 2882 if (r) 2883 goto out; 2884 } 2885 2886 flush_all_to_thread(current); 2887 2888 vcpu->arch.wqp = &vcpu->arch.vcore->wq; 2889 vcpu->arch.pgdir = current->mm->pgd; 2890 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST; 2891 2892 do { 2893 r = kvmppc_run_vcpu(run, vcpu); 2894 2895 if (run->exit_reason == KVM_EXIT_PAPR_HCALL && 2896 !(vcpu->arch.shregs.msr & MSR_PR)) { 2897 trace_kvm_hcall_enter(vcpu); 2898 r = kvmppc_pseries_do_hcall(vcpu); 2899 trace_kvm_hcall_exit(vcpu, r); 2900 kvmppc_core_prepare_to_enter(vcpu); 2901 } else if (r == RESUME_PAGE_FAULT) { 2902 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 2903 r = kvmppc_book3s_hv_page_fault(run, vcpu, 2904 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr); 2905 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); 2906 } else if (r == RESUME_PASSTHROUGH) 2907 r = kvmppc_xics_rm_complete(vcpu, 0); 2908 } while (is_kvmppc_resume_guest(r)); 2909 2910 out: 2911 vcpu->arch.state = KVMPPC_VCPU_NOTREADY; 2912 atomic_dec(&vcpu->kvm->arch.vcpus_running); 2913 return r; 2914 } 2915 2916 static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps, 2917 int linux_psize) 2918 { 2919 struct mmu_psize_def *def = &mmu_psize_defs[linux_psize]; 2920 2921 if (!def->shift) 2922 return; 2923 (*sps)->page_shift = def->shift; 2924 (*sps)->slb_enc = def->sllp; 2925 (*sps)->enc[0].page_shift = def->shift; 2926 (*sps)->enc[0].pte_enc = def->penc[linux_psize]; 2927 /* 2928 * Add 16MB MPSS support if host supports it 2929 */ 2930 if (linux_psize != MMU_PAGE_16M && def->penc[MMU_PAGE_16M] != -1) { 2931 (*sps)->enc[1].page_shift = 24; 2932 (*sps)->enc[1].pte_enc = def->penc[MMU_PAGE_16M]; 2933 } 2934 (*sps)++; 2935 } 2936 2937 static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm, 2938 struct kvm_ppc_smmu_info *info) 2939 { 2940 struct kvm_ppc_one_seg_page_size *sps; 2941 2942 info->flags = KVM_PPC_PAGE_SIZES_REAL; 2943 if (mmu_has_feature(MMU_FTR_1T_SEGMENT)) 2944 info->flags |= KVM_PPC_1T_SEGMENTS; 2945 info->slb_size = mmu_slb_size; 2946 2947 /* We only support these sizes for now, and no muti-size segments */ 2948 sps = &info->sps[0]; 2949 kvmppc_add_seg_page_size(&sps, MMU_PAGE_4K); 2950 kvmppc_add_seg_page_size(&sps, MMU_PAGE_64K); 2951 kvmppc_add_seg_page_size(&sps, MMU_PAGE_16M); 2952 2953 return 0; 2954 } 2955 2956 /* 2957 * Get (and clear) the dirty memory log for a memory slot. 2958 */ 2959 static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm, 2960 struct kvm_dirty_log *log) 2961 { 2962 struct kvm_memslots *slots; 2963 struct kvm_memory_slot *memslot; 2964 int r; 2965 unsigned long n; 2966 2967 mutex_lock(&kvm->slots_lock); 2968 2969 r = -EINVAL; 2970 if (log->slot >= KVM_USER_MEM_SLOTS) 2971 goto out; 2972 2973 slots = kvm_memslots(kvm); 2974 memslot = id_to_memslot(slots, log->slot); 2975 r = -ENOENT; 2976 if (!memslot->dirty_bitmap) 2977 goto out; 2978 2979 n = kvm_dirty_bitmap_bytes(memslot); 2980 memset(memslot->dirty_bitmap, 0, n); 2981 2982 r = kvmppc_hv_get_dirty_log(kvm, memslot, memslot->dirty_bitmap); 2983 if (r) 2984 goto out; 2985 2986 r = -EFAULT; 2987 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 2988 goto out; 2989 2990 r = 0; 2991 out: 2992 mutex_unlock(&kvm->slots_lock); 2993 return r; 2994 } 2995 2996 static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *free, 2997 struct kvm_memory_slot *dont) 2998 { 2999 if (!dont || free->arch.rmap != dont->arch.rmap) { 3000 vfree(free->arch.rmap); 3001 free->arch.rmap = NULL; 3002 } 3003 } 3004 3005 static int kvmppc_core_create_memslot_hv(struct kvm_memory_slot *slot, 3006 unsigned long npages) 3007 { 3008 slot->arch.rmap = vzalloc(npages * sizeof(*slot->arch.rmap)); 3009 if (!slot->arch.rmap) 3010 return -ENOMEM; 3011 3012 return 0; 3013 } 3014 3015 static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm, 3016 struct kvm_memory_slot *memslot, 3017 const struct kvm_userspace_memory_region *mem) 3018 { 3019 return 0; 3020 } 3021 3022 static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm, 3023 const struct kvm_userspace_memory_region *mem, 3024 const struct kvm_memory_slot *old, 3025 const struct kvm_memory_slot *new) 3026 { 3027 unsigned long npages = mem->memory_size >> PAGE_SHIFT; 3028 struct kvm_memslots *slots; 3029 struct kvm_memory_slot *memslot; 3030 3031 /* 3032 * If we are making a new memslot, it might make 3033 * some address that was previously cached as emulated 3034 * MMIO be no longer emulated MMIO, so invalidate 3035 * all the caches of emulated MMIO translations. 3036 */ 3037 if (npages) 3038 atomic64_inc(&kvm->arch.mmio_update); 3039 3040 if (npages && old->npages) { 3041 /* 3042 * If modifying a memslot, reset all the rmap dirty bits. 3043 * If this is a new memslot, we don't need to do anything 3044 * since the rmap array starts out as all zeroes, 3045 * i.e. no pages are dirty. 3046 */ 3047 slots = kvm_memslots(kvm); 3048 memslot = id_to_memslot(slots, mem->slot); 3049 kvmppc_hv_get_dirty_log(kvm, memslot, NULL); 3050 } 3051 } 3052 3053 /* 3054 * Update LPCR values in kvm->arch and in vcores. 3055 * Caller must hold kvm->lock. 3056 */ 3057 void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask) 3058 { 3059 long int i; 3060 u32 cores_done = 0; 3061 3062 if ((kvm->arch.lpcr & mask) == lpcr) 3063 return; 3064 3065 kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr; 3066 3067 for (i = 0; i < KVM_MAX_VCORES; ++i) { 3068 struct kvmppc_vcore *vc = kvm->arch.vcores[i]; 3069 if (!vc) 3070 continue; 3071 spin_lock(&vc->lock); 3072 vc->lpcr = (vc->lpcr & ~mask) | lpcr; 3073 spin_unlock(&vc->lock); 3074 if (++cores_done >= kvm->arch.online_vcores) 3075 break; 3076 } 3077 } 3078 3079 static void kvmppc_mmu_destroy_hv(struct kvm_vcpu *vcpu) 3080 { 3081 return; 3082 } 3083 3084 static void kvmppc_setup_partition_table(struct kvm *kvm) 3085 { 3086 unsigned long dw0, dw1; 3087 3088 /* PS field - page size for VRMA */ 3089 dw0 = ((kvm->arch.vrma_slb_v & SLB_VSID_L) >> 1) | 3090 ((kvm->arch.vrma_slb_v & SLB_VSID_LP) << 1); 3091 /* HTABSIZE and HTABORG fields */ 3092 dw0 |= kvm->arch.sdr1; 3093 3094 /* Second dword has GR=0; other fields are unused since UPRT=0 */ 3095 dw1 = 0; 3096 3097 mmu_partition_table_set_entry(kvm->arch.lpid, dw0, dw1); 3098 } 3099 3100 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu) 3101 { 3102 int err = 0; 3103 struct kvm *kvm = vcpu->kvm; 3104 unsigned long hva; 3105 struct kvm_memory_slot *memslot; 3106 struct vm_area_struct *vma; 3107 unsigned long lpcr = 0, senc; 3108 unsigned long psize, porder; 3109 int srcu_idx; 3110 3111 mutex_lock(&kvm->lock); 3112 if (kvm->arch.hpte_setup_done) 3113 goto out; /* another vcpu beat us to it */ 3114 3115 /* Allocate hashed page table (if not done already) and reset it */ 3116 if (!kvm->arch.hpt_virt) { 3117 err = kvmppc_alloc_hpt(kvm, NULL); 3118 if (err) { 3119 pr_err("KVM: Couldn't alloc HPT\n"); 3120 goto out; 3121 } 3122 } 3123 3124 /* Look up the memslot for guest physical address 0 */ 3125 srcu_idx = srcu_read_lock(&kvm->srcu); 3126 memslot = gfn_to_memslot(kvm, 0); 3127 3128 /* We must have some memory at 0 by now */ 3129 err = -EINVAL; 3130 if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) 3131 goto out_srcu; 3132 3133 /* Look up the VMA for the start of this memory slot */ 3134 hva = memslot->userspace_addr; 3135 down_read(¤t->mm->mmap_sem); 3136 vma = find_vma(current->mm, hva); 3137 if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO)) 3138 goto up_out; 3139 3140 psize = vma_kernel_pagesize(vma); 3141 porder = __ilog2(psize); 3142 3143 up_read(¤t->mm->mmap_sem); 3144 3145 /* We can handle 4k, 64k or 16M pages in the VRMA */ 3146 err = -EINVAL; 3147 if (!(psize == 0x1000 || psize == 0x10000 || 3148 psize == 0x1000000)) 3149 goto out_srcu; 3150 3151 senc = slb_pgsize_encoding(psize); 3152 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T | 3153 (VRMA_VSID << SLB_VSID_SHIFT_1T); 3154 /* Create HPTEs in the hash page table for the VRMA */ 3155 kvmppc_map_vrma(vcpu, memslot, porder); 3156 3157 /* Update VRMASD field in the LPCR */ 3158 if (!cpu_has_feature(CPU_FTR_ARCH_300)) { 3159 /* the -4 is to account for senc values starting at 0x10 */ 3160 lpcr = senc << (LPCR_VRMASD_SH - 4); 3161 kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD); 3162 } else { 3163 kvmppc_setup_partition_table(kvm); 3164 } 3165 3166 /* Order updates to kvm->arch.lpcr etc. vs. hpte_setup_done */ 3167 smp_wmb(); 3168 kvm->arch.hpte_setup_done = 1; 3169 err = 0; 3170 out_srcu: 3171 srcu_read_unlock(&kvm->srcu, srcu_idx); 3172 out: 3173 mutex_unlock(&kvm->lock); 3174 return err; 3175 3176 up_out: 3177 up_read(¤t->mm->mmap_sem); 3178 goto out_srcu; 3179 } 3180 3181 #ifdef CONFIG_KVM_XICS 3182 /* 3183 * Allocate a per-core structure for managing state about which cores are 3184 * running in the host versus the guest and for exchanging data between 3185 * real mode KVM and CPU running in the host. 3186 * This is only done for the first VM. 3187 * The allocated structure stays even if all VMs have stopped. 3188 * It is only freed when the kvm-hv module is unloaded. 3189 * It's OK for this routine to fail, we just don't support host 3190 * core operations like redirecting H_IPI wakeups. 3191 */ 3192 void kvmppc_alloc_host_rm_ops(void) 3193 { 3194 struct kvmppc_host_rm_ops *ops; 3195 unsigned long l_ops; 3196 int cpu, core; 3197 int size; 3198 3199 /* Not the first time here ? */ 3200 if (kvmppc_host_rm_ops_hv != NULL) 3201 return; 3202 3203 ops = kzalloc(sizeof(struct kvmppc_host_rm_ops), GFP_KERNEL); 3204 if (!ops) 3205 return; 3206 3207 size = cpu_nr_cores() * sizeof(struct kvmppc_host_rm_core); 3208 ops->rm_core = kzalloc(size, GFP_KERNEL); 3209 3210 if (!ops->rm_core) { 3211 kfree(ops); 3212 return; 3213 } 3214 3215 get_online_cpus(); 3216 3217 for (cpu = 0; cpu < nr_cpu_ids; cpu += threads_per_core) { 3218 if (!cpu_online(cpu)) 3219 continue; 3220 3221 core = cpu >> threads_shift; 3222 ops->rm_core[core].rm_state.in_host = 1; 3223 } 3224 3225 ops->vcpu_kick = kvmppc_fast_vcpu_kick_hv; 3226 3227 /* 3228 * Make the contents of the kvmppc_host_rm_ops structure visible 3229 * to other CPUs before we assign it to the global variable. 3230 * Do an atomic assignment (no locks used here), but if someone 3231 * beats us to it, just free our copy and return. 3232 */ 3233 smp_wmb(); 3234 l_ops = (unsigned long) ops; 3235 3236 if (cmpxchg64((unsigned long *)&kvmppc_host_rm_ops_hv, 0, l_ops)) { 3237 put_online_cpus(); 3238 kfree(ops->rm_core); 3239 kfree(ops); 3240 return; 3241 } 3242 3243 cpuhp_setup_state_nocalls(CPUHP_KVM_PPC_BOOK3S_PREPARE, 3244 "ppc/kvm_book3s:prepare", 3245 kvmppc_set_host_core, 3246 kvmppc_clear_host_core); 3247 put_online_cpus(); 3248 } 3249 3250 void kvmppc_free_host_rm_ops(void) 3251 { 3252 if (kvmppc_host_rm_ops_hv) { 3253 cpuhp_remove_state_nocalls(CPUHP_KVM_PPC_BOOK3S_PREPARE); 3254 kfree(kvmppc_host_rm_ops_hv->rm_core); 3255 kfree(kvmppc_host_rm_ops_hv); 3256 kvmppc_host_rm_ops_hv = NULL; 3257 } 3258 } 3259 #endif 3260 3261 static int kvmppc_core_init_vm_hv(struct kvm *kvm) 3262 { 3263 unsigned long lpcr, lpid; 3264 char buf[32]; 3265 3266 /* Allocate the guest's logical partition ID */ 3267 3268 lpid = kvmppc_alloc_lpid(); 3269 if ((long)lpid < 0) 3270 return -ENOMEM; 3271 kvm->arch.lpid = lpid; 3272 3273 kvmppc_alloc_host_rm_ops(); 3274 3275 /* 3276 * Since we don't flush the TLB when tearing down a VM, 3277 * and this lpid might have previously been used, 3278 * make sure we flush on each core before running the new VM. 3279 * On POWER9, the tlbie in mmu_partition_table_set_entry() 3280 * does this flush for us. 3281 */ 3282 if (!cpu_has_feature(CPU_FTR_ARCH_300)) 3283 cpumask_setall(&kvm->arch.need_tlb_flush); 3284 3285 /* Start out with the default set of hcalls enabled */ 3286 memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls, 3287 sizeof(kvm->arch.enabled_hcalls)); 3288 3289 if (!cpu_has_feature(CPU_FTR_ARCH_300)) 3290 kvm->arch.host_sdr1 = mfspr(SPRN_SDR1); 3291 3292 /* Init LPCR for virtual RMA mode */ 3293 kvm->arch.host_lpid = mfspr(SPRN_LPID); 3294 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR); 3295 lpcr &= LPCR_PECE | LPCR_LPES; 3296 lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE | 3297 LPCR_VPM0 | LPCR_VPM1; 3298 kvm->arch.vrma_slb_v = SLB_VSID_B_1T | 3299 (VRMA_VSID << SLB_VSID_SHIFT_1T); 3300 /* On POWER8 turn on online bit to enable PURR/SPURR */ 3301 if (cpu_has_feature(CPU_FTR_ARCH_207S)) 3302 lpcr |= LPCR_ONL; 3303 /* 3304 * On POWER9, VPM0 bit is reserved (VPM0=1 behaviour is assumed) 3305 * Set HVICE bit to enable hypervisor virtualization interrupts. 3306 */ 3307 if (cpu_has_feature(CPU_FTR_ARCH_300)) { 3308 lpcr &= ~LPCR_VPM0; 3309 lpcr |= LPCR_HVICE; 3310 } 3311 3312 kvm->arch.lpcr = lpcr; 3313 3314 /* 3315 * Work out how many sets the TLB has, for the use of 3316 * the TLB invalidation loop in book3s_hv_rmhandlers.S. 3317 */ 3318 if (cpu_has_feature(CPU_FTR_ARCH_300)) 3319 kvm->arch.tlb_sets = POWER9_TLB_SETS_HASH; /* 256 */ 3320 else if (cpu_has_feature(CPU_FTR_ARCH_207S)) 3321 kvm->arch.tlb_sets = POWER8_TLB_SETS; /* 512 */ 3322 else 3323 kvm->arch.tlb_sets = POWER7_TLB_SETS; /* 128 */ 3324 3325 /* 3326 * Track that we now have a HV mode VM active. This blocks secondary 3327 * CPU threads from coming online. 3328 */ 3329 kvm_hv_vm_activated(); 3330 3331 /* 3332 * Create a debugfs directory for the VM 3333 */ 3334 snprintf(buf, sizeof(buf), "vm%d", current->pid); 3335 kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir); 3336 if (!IS_ERR_OR_NULL(kvm->arch.debugfs_dir)) 3337 kvmppc_mmu_debugfs_init(kvm); 3338 3339 return 0; 3340 } 3341 3342 static void kvmppc_free_vcores(struct kvm *kvm) 3343 { 3344 long int i; 3345 3346 for (i = 0; i < KVM_MAX_VCORES; ++i) 3347 kfree(kvm->arch.vcores[i]); 3348 kvm->arch.online_vcores = 0; 3349 } 3350 3351 static void kvmppc_core_destroy_vm_hv(struct kvm *kvm) 3352 { 3353 debugfs_remove_recursive(kvm->arch.debugfs_dir); 3354 3355 kvm_hv_vm_deactivated(); 3356 3357 kvmppc_free_vcores(kvm); 3358 3359 kvmppc_free_hpt(kvm); 3360 3361 kvmppc_free_pimap(kvm); 3362 } 3363 3364 /* We don't need to emulate any privileged instructions or dcbz */ 3365 static int kvmppc_core_emulate_op_hv(struct kvm_run *run, struct kvm_vcpu *vcpu, 3366 unsigned int inst, int *advance) 3367 { 3368 return EMULATE_FAIL; 3369 } 3370 3371 static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn, 3372 ulong spr_val) 3373 { 3374 return EMULATE_FAIL; 3375 } 3376 3377 static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn, 3378 ulong *spr_val) 3379 { 3380 return EMULATE_FAIL; 3381 } 3382 3383 static int kvmppc_core_check_processor_compat_hv(void) 3384 { 3385 if (!cpu_has_feature(CPU_FTR_HVMODE) || 3386 !cpu_has_feature(CPU_FTR_ARCH_206)) 3387 return -EIO; 3388 /* 3389 * Disable KVM for Power9 in radix mode. 3390 */ 3391 if (cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled()) 3392 return -EIO; 3393 3394 return 0; 3395 } 3396 3397 #ifdef CONFIG_KVM_XICS 3398 3399 void kvmppc_free_pimap(struct kvm *kvm) 3400 { 3401 kfree(kvm->arch.pimap); 3402 } 3403 3404 static struct kvmppc_passthru_irqmap *kvmppc_alloc_pimap(void) 3405 { 3406 return kzalloc(sizeof(struct kvmppc_passthru_irqmap), GFP_KERNEL); 3407 } 3408 3409 static int kvmppc_set_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi) 3410 { 3411 struct irq_desc *desc; 3412 struct kvmppc_irq_map *irq_map; 3413 struct kvmppc_passthru_irqmap *pimap; 3414 struct irq_chip *chip; 3415 int i; 3416 3417 if (!kvm_irq_bypass) 3418 return 1; 3419 3420 desc = irq_to_desc(host_irq); 3421 if (!desc) 3422 return -EIO; 3423 3424 mutex_lock(&kvm->lock); 3425 3426 pimap = kvm->arch.pimap; 3427 if (pimap == NULL) { 3428 /* First call, allocate structure to hold IRQ map */ 3429 pimap = kvmppc_alloc_pimap(); 3430 if (pimap == NULL) { 3431 mutex_unlock(&kvm->lock); 3432 return -ENOMEM; 3433 } 3434 kvm->arch.pimap = pimap; 3435 } 3436 3437 /* 3438 * For now, we only support interrupts for which the EOI operation 3439 * is an OPAL call followed by a write to XIRR, since that's 3440 * what our real-mode EOI code does. 3441 */ 3442 chip = irq_data_get_irq_chip(&desc->irq_data); 3443 if (!chip || !is_pnv_opal_msi(chip)) { 3444 pr_warn("kvmppc_set_passthru_irq_hv: Could not assign IRQ map for (%d,%d)\n", 3445 host_irq, guest_gsi); 3446 mutex_unlock(&kvm->lock); 3447 return -ENOENT; 3448 } 3449 3450 /* 3451 * See if we already have an entry for this guest IRQ number. 3452 * If it's mapped to a hardware IRQ number, that's an error, 3453 * otherwise re-use this entry. 3454 */ 3455 for (i = 0; i < pimap->n_mapped; i++) { 3456 if (guest_gsi == pimap->mapped[i].v_hwirq) { 3457 if (pimap->mapped[i].r_hwirq) { 3458 mutex_unlock(&kvm->lock); 3459 return -EINVAL; 3460 } 3461 break; 3462 } 3463 } 3464 3465 if (i == KVMPPC_PIRQ_MAPPED) { 3466 mutex_unlock(&kvm->lock); 3467 return -EAGAIN; /* table is full */ 3468 } 3469 3470 irq_map = &pimap->mapped[i]; 3471 3472 irq_map->v_hwirq = guest_gsi; 3473 irq_map->desc = desc; 3474 3475 /* 3476 * Order the above two stores before the next to serialize with 3477 * the KVM real mode handler. 3478 */ 3479 smp_wmb(); 3480 irq_map->r_hwirq = desc->irq_data.hwirq; 3481 3482 if (i == pimap->n_mapped) 3483 pimap->n_mapped++; 3484 3485 kvmppc_xics_set_mapped(kvm, guest_gsi, desc->irq_data.hwirq); 3486 3487 mutex_unlock(&kvm->lock); 3488 3489 return 0; 3490 } 3491 3492 static int kvmppc_clr_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi) 3493 { 3494 struct irq_desc *desc; 3495 struct kvmppc_passthru_irqmap *pimap; 3496 int i; 3497 3498 if (!kvm_irq_bypass) 3499 return 0; 3500 3501 desc = irq_to_desc(host_irq); 3502 if (!desc) 3503 return -EIO; 3504 3505 mutex_lock(&kvm->lock); 3506 3507 if (kvm->arch.pimap == NULL) { 3508 mutex_unlock(&kvm->lock); 3509 return 0; 3510 } 3511 pimap = kvm->arch.pimap; 3512 3513 for (i = 0; i < pimap->n_mapped; i++) { 3514 if (guest_gsi == pimap->mapped[i].v_hwirq) 3515 break; 3516 } 3517 3518 if (i == pimap->n_mapped) { 3519 mutex_unlock(&kvm->lock); 3520 return -ENODEV; 3521 } 3522 3523 kvmppc_xics_clr_mapped(kvm, guest_gsi, pimap->mapped[i].r_hwirq); 3524 3525 /* invalidate the entry */ 3526 pimap->mapped[i].r_hwirq = 0; 3527 3528 /* 3529 * We don't free this structure even when the count goes to 3530 * zero. The structure is freed when we destroy the VM. 3531 */ 3532 3533 mutex_unlock(&kvm->lock); 3534 return 0; 3535 } 3536 3537 static int kvmppc_irq_bypass_add_producer_hv(struct irq_bypass_consumer *cons, 3538 struct irq_bypass_producer *prod) 3539 { 3540 int ret = 0; 3541 struct kvm_kernel_irqfd *irqfd = 3542 container_of(cons, struct kvm_kernel_irqfd, consumer); 3543 3544 irqfd->producer = prod; 3545 3546 ret = kvmppc_set_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi); 3547 if (ret) 3548 pr_info("kvmppc_set_passthru_irq (irq %d, gsi %d) fails: %d\n", 3549 prod->irq, irqfd->gsi, ret); 3550 3551 return ret; 3552 } 3553 3554 static void kvmppc_irq_bypass_del_producer_hv(struct irq_bypass_consumer *cons, 3555 struct irq_bypass_producer *prod) 3556 { 3557 int ret; 3558 struct kvm_kernel_irqfd *irqfd = 3559 container_of(cons, struct kvm_kernel_irqfd, consumer); 3560 3561 irqfd->producer = NULL; 3562 3563 /* 3564 * When producer of consumer is unregistered, we change back to 3565 * default external interrupt handling mode - KVM real mode 3566 * will switch back to host. 3567 */ 3568 ret = kvmppc_clr_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi); 3569 if (ret) 3570 pr_warn("kvmppc_clr_passthru_irq (irq %d, gsi %d) fails: %d\n", 3571 prod->irq, irqfd->gsi, ret); 3572 } 3573 #endif 3574 3575 static long kvm_arch_vm_ioctl_hv(struct file *filp, 3576 unsigned int ioctl, unsigned long arg) 3577 { 3578 struct kvm *kvm __maybe_unused = filp->private_data; 3579 void __user *argp = (void __user *)arg; 3580 long r; 3581 3582 switch (ioctl) { 3583 3584 case KVM_PPC_ALLOCATE_HTAB: { 3585 u32 htab_order; 3586 3587 r = -EFAULT; 3588 if (get_user(htab_order, (u32 __user *)argp)) 3589 break; 3590 r = kvmppc_alloc_reset_hpt(kvm, &htab_order); 3591 if (r) 3592 break; 3593 r = -EFAULT; 3594 if (put_user(htab_order, (u32 __user *)argp)) 3595 break; 3596 r = 0; 3597 break; 3598 } 3599 3600 case KVM_PPC_GET_HTAB_FD: { 3601 struct kvm_get_htab_fd ghf; 3602 3603 r = -EFAULT; 3604 if (copy_from_user(&ghf, argp, sizeof(ghf))) 3605 break; 3606 r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf); 3607 break; 3608 } 3609 3610 default: 3611 r = -ENOTTY; 3612 } 3613 3614 return r; 3615 } 3616 3617 /* 3618 * List of hcall numbers to enable by default. 3619 * For compatibility with old userspace, we enable by default 3620 * all hcalls that were implemented before the hcall-enabling 3621 * facility was added. Note this list should not include H_RTAS. 3622 */ 3623 static unsigned int default_hcall_list[] = { 3624 H_REMOVE, 3625 H_ENTER, 3626 H_READ, 3627 H_PROTECT, 3628 H_BULK_REMOVE, 3629 H_GET_TCE, 3630 H_PUT_TCE, 3631 H_SET_DABR, 3632 H_SET_XDABR, 3633 H_CEDE, 3634 H_PROD, 3635 H_CONFER, 3636 H_REGISTER_VPA, 3637 #ifdef CONFIG_KVM_XICS 3638 H_EOI, 3639 H_CPPR, 3640 H_IPI, 3641 H_IPOLL, 3642 H_XIRR, 3643 H_XIRR_X, 3644 #endif 3645 0 3646 }; 3647 3648 static void init_default_hcalls(void) 3649 { 3650 int i; 3651 unsigned int hcall; 3652 3653 for (i = 0; default_hcall_list[i]; ++i) { 3654 hcall = default_hcall_list[i]; 3655 WARN_ON(!kvmppc_hcall_impl_hv(hcall)); 3656 __set_bit(hcall / 4, default_enabled_hcalls); 3657 } 3658 } 3659 3660 static struct kvmppc_ops kvm_ops_hv = { 3661 .get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv, 3662 .set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv, 3663 .get_one_reg = kvmppc_get_one_reg_hv, 3664 .set_one_reg = kvmppc_set_one_reg_hv, 3665 .vcpu_load = kvmppc_core_vcpu_load_hv, 3666 .vcpu_put = kvmppc_core_vcpu_put_hv, 3667 .set_msr = kvmppc_set_msr_hv, 3668 .vcpu_run = kvmppc_vcpu_run_hv, 3669 .vcpu_create = kvmppc_core_vcpu_create_hv, 3670 .vcpu_free = kvmppc_core_vcpu_free_hv, 3671 .check_requests = kvmppc_core_check_requests_hv, 3672 .get_dirty_log = kvm_vm_ioctl_get_dirty_log_hv, 3673 .flush_memslot = kvmppc_core_flush_memslot_hv, 3674 .prepare_memory_region = kvmppc_core_prepare_memory_region_hv, 3675 .commit_memory_region = kvmppc_core_commit_memory_region_hv, 3676 .unmap_hva = kvm_unmap_hva_hv, 3677 .unmap_hva_range = kvm_unmap_hva_range_hv, 3678 .age_hva = kvm_age_hva_hv, 3679 .test_age_hva = kvm_test_age_hva_hv, 3680 .set_spte_hva = kvm_set_spte_hva_hv, 3681 .mmu_destroy = kvmppc_mmu_destroy_hv, 3682 .free_memslot = kvmppc_core_free_memslot_hv, 3683 .create_memslot = kvmppc_core_create_memslot_hv, 3684 .init_vm = kvmppc_core_init_vm_hv, 3685 .destroy_vm = kvmppc_core_destroy_vm_hv, 3686 .get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv, 3687 .emulate_op = kvmppc_core_emulate_op_hv, 3688 .emulate_mtspr = kvmppc_core_emulate_mtspr_hv, 3689 .emulate_mfspr = kvmppc_core_emulate_mfspr_hv, 3690 .fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv, 3691 .arch_vm_ioctl = kvm_arch_vm_ioctl_hv, 3692 .hcall_implemented = kvmppc_hcall_impl_hv, 3693 #ifdef CONFIG_KVM_XICS 3694 .irq_bypass_add_producer = kvmppc_irq_bypass_add_producer_hv, 3695 .irq_bypass_del_producer = kvmppc_irq_bypass_del_producer_hv, 3696 #endif 3697 }; 3698 3699 static int kvm_init_subcore_bitmap(void) 3700 { 3701 int i, j; 3702 int nr_cores = cpu_nr_cores(); 3703 struct sibling_subcore_state *sibling_subcore_state; 3704 3705 for (i = 0; i < nr_cores; i++) { 3706 int first_cpu = i * threads_per_core; 3707 int node = cpu_to_node(first_cpu); 3708 3709 /* Ignore if it is already allocated. */ 3710 if (paca[first_cpu].sibling_subcore_state) 3711 continue; 3712 3713 sibling_subcore_state = 3714 kmalloc_node(sizeof(struct sibling_subcore_state), 3715 GFP_KERNEL, node); 3716 if (!sibling_subcore_state) 3717 return -ENOMEM; 3718 3719 memset(sibling_subcore_state, 0, 3720 sizeof(struct sibling_subcore_state)); 3721 3722 for (j = 0; j < threads_per_core; j++) { 3723 int cpu = first_cpu + j; 3724 3725 paca[cpu].sibling_subcore_state = sibling_subcore_state; 3726 } 3727 } 3728 return 0; 3729 } 3730 3731 static int kvmppc_book3s_init_hv(void) 3732 { 3733 int r; 3734 /* 3735 * FIXME!! Do we need to check on all cpus ? 3736 */ 3737 r = kvmppc_core_check_processor_compat_hv(); 3738 if (r < 0) 3739 return -ENODEV; 3740 3741 r = kvm_init_subcore_bitmap(); 3742 if (r) 3743 return r; 3744 3745 /* 3746 * We need a way of accessing the XICS interrupt controller, 3747 * either directly, via paca[cpu].kvm_hstate.xics_phys, or 3748 * indirectly, via OPAL. 3749 */ 3750 #ifdef CONFIG_SMP 3751 if (!get_paca()->kvm_hstate.xics_phys) { 3752 struct device_node *np; 3753 3754 np = of_find_compatible_node(NULL, NULL, "ibm,opal-intc"); 3755 if (!np) { 3756 pr_err("KVM-HV: Cannot determine method for accessing XICS\n"); 3757 return -ENODEV; 3758 } 3759 } 3760 #endif 3761 3762 kvm_ops_hv.owner = THIS_MODULE; 3763 kvmppc_hv_ops = &kvm_ops_hv; 3764 3765 init_default_hcalls(); 3766 3767 init_vcore_lists(); 3768 3769 r = kvmppc_mmu_hv_init(); 3770 return r; 3771 } 3772 3773 static void kvmppc_book3s_exit_hv(void) 3774 { 3775 kvmppc_free_host_rm_ops(); 3776 kvmppc_hv_ops = NULL; 3777 } 3778 3779 module_init(kvmppc_book3s_init_hv); 3780 module_exit(kvmppc_book3s_exit_hv); 3781 MODULE_LICENSE("GPL"); 3782 MODULE_ALIAS_MISCDEV(KVM_MINOR); 3783 MODULE_ALIAS("devname:kvm"); 3784 3785