1 /* 2 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 3 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved. 4 * 5 * Authors: 6 * Paul Mackerras <paulus@au1.ibm.com> 7 * Alexander Graf <agraf@suse.de> 8 * Kevin Wolf <mail@kevin-wolf.de> 9 * 10 * Description: KVM functions specific to running on Book 3S 11 * processors in hypervisor mode (specifically POWER7 and later). 12 * 13 * This file is derived from arch/powerpc/kvm/book3s.c, 14 * by Alexander Graf <agraf@suse.de>. 15 * 16 * This program is free software; you can redistribute it and/or modify 17 * it under the terms of the GNU General Public License, version 2, as 18 * published by the Free Software Foundation. 19 */ 20 21 #include <linux/kvm_host.h> 22 #include <linux/err.h> 23 #include <linux/slab.h> 24 #include <linux/preempt.h> 25 #include <linux/sched/signal.h> 26 #include <linux/sched/stat.h> 27 #include <linux/delay.h> 28 #include <linux/export.h> 29 #include <linux/fs.h> 30 #include <linux/anon_inodes.h> 31 #include <linux/cpu.h> 32 #include <linux/cpumask.h> 33 #include <linux/spinlock.h> 34 #include <linux/page-flags.h> 35 #include <linux/srcu.h> 36 #include <linux/miscdevice.h> 37 #include <linux/debugfs.h> 38 #include <linux/gfp.h> 39 #include <linux/vmalloc.h> 40 #include <linux/highmem.h> 41 #include <linux/hugetlb.h> 42 #include <linux/kvm_irqfd.h> 43 #include <linux/irqbypass.h> 44 #include <linux/module.h> 45 #include <linux/compiler.h> 46 #include <linux/of.h> 47 48 #include <asm/reg.h> 49 #include <asm/ppc-opcode.h> 50 #include <asm/disassemble.h> 51 #include <asm/cputable.h> 52 #include <asm/cacheflush.h> 53 #include <asm/tlbflush.h> 54 #include <linux/uaccess.h> 55 #include <asm/io.h> 56 #include <asm/kvm_ppc.h> 57 #include <asm/kvm_book3s.h> 58 #include <asm/mmu_context.h> 59 #include <asm/lppaca.h> 60 #include <asm/processor.h> 61 #include <asm/cputhreads.h> 62 #include <asm/page.h> 63 #include <asm/hvcall.h> 64 #include <asm/switch_to.h> 65 #include <asm/smp.h> 66 #include <asm/dbell.h> 67 #include <asm/hmi.h> 68 #include <asm/pnv-pci.h> 69 #include <asm/mmu.h> 70 #include <asm/opal.h> 71 #include <asm/xics.h> 72 #include <asm/xive.h> 73 74 #include "book3s.h" 75 76 #define CREATE_TRACE_POINTS 77 #include "trace_hv.h" 78 79 /* #define EXIT_DEBUG */ 80 /* #define EXIT_DEBUG_SIMPLE */ 81 /* #define EXIT_DEBUG_INT */ 82 83 /* Used to indicate that a guest page fault needs to be handled */ 84 #define RESUME_PAGE_FAULT (RESUME_GUEST | RESUME_FLAG_ARCH1) 85 /* Used to indicate that a guest passthrough interrupt needs to be handled */ 86 #define RESUME_PASSTHROUGH (RESUME_GUEST | RESUME_FLAG_ARCH2) 87 88 /* Used as a "null" value for timebase values */ 89 #define TB_NIL (~(u64)0) 90 91 static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1); 92 93 static int dynamic_mt_modes = 6; 94 module_param(dynamic_mt_modes, int, S_IRUGO | S_IWUSR); 95 MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)"); 96 static int target_smt_mode; 97 module_param(target_smt_mode, int, S_IRUGO | S_IWUSR); 98 MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)"); 99 100 #ifdef CONFIG_KVM_XICS 101 static struct kernel_param_ops module_param_ops = { 102 .set = param_set_int, 103 .get = param_get_int, 104 }; 105 106 module_param_cb(kvm_irq_bypass, &module_param_ops, &kvm_irq_bypass, 107 S_IRUGO | S_IWUSR); 108 MODULE_PARM_DESC(kvm_irq_bypass, "Bypass passthrough interrupt optimization"); 109 110 module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect, 111 S_IRUGO | S_IWUSR); 112 MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core"); 113 #endif 114 115 static void kvmppc_end_cede(struct kvm_vcpu *vcpu); 116 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu); 117 118 static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc, 119 int *ip) 120 { 121 int i = *ip; 122 struct kvm_vcpu *vcpu; 123 124 while (++i < MAX_SMT_THREADS) { 125 vcpu = READ_ONCE(vc->runnable_threads[i]); 126 if (vcpu) { 127 *ip = i; 128 return vcpu; 129 } 130 } 131 return NULL; 132 } 133 134 /* Used to traverse the list of runnable threads for a given vcore */ 135 #define for_each_runnable_thread(i, vcpu, vc) \ 136 for (i = -1; (vcpu = next_runnable_thread(vc, &i)); ) 137 138 static bool kvmppc_ipi_thread(int cpu) 139 { 140 unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER); 141 142 /* On POWER9 we can use msgsnd to IPI any cpu */ 143 if (cpu_has_feature(CPU_FTR_ARCH_300)) { 144 msg |= get_hard_smp_processor_id(cpu); 145 smp_mb(); 146 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg)); 147 return true; 148 } 149 150 /* On POWER8 for IPIs to threads in the same core, use msgsnd */ 151 if (cpu_has_feature(CPU_FTR_ARCH_207S)) { 152 preempt_disable(); 153 if (cpu_first_thread_sibling(cpu) == 154 cpu_first_thread_sibling(smp_processor_id())) { 155 msg |= cpu_thread_in_core(cpu); 156 smp_mb(); 157 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg)); 158 preempt_enable(); 159 return true; 160 } 161 preempt_enable(); 162 } 163 164 #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP) 165 if (cpu >= 0 && cpu < nr_cpu_ids) { 166 if (paca[cpu].kvm_hstate.xics_phys) { 167 xics_wake_cpu(cpu); 168 return true; 169 } 170 opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY); 171 return true; 172 } 173 #endif 174 175 return false; 176 } 177 178 static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) 179 { 180 int cpu; 181 struct swait_queue_head *wqp; 182 183 wqp = kvm_arch_vcpu_wq(vcpu); 184 if (swait_active(wqp)) { 185 swake_up(wqp); 186 ++vcpu->stat.halt_wakeup; 187 } 188 189 cpu = READ_ONCE(vcpu->arch.thread_cpu); 190 if (cpu >= 0 && kvmppc_ipi_thread(cpu)) 191 return; 192 193 /* CPU points to the first thread of the core */ 194 cpu = vcpu->cpu; 195 if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu)) 196 smp_send_reschedule(cpu); 197 } 198 199 /* 200 * We use the vcpu_load/put functions to measure stolen time. 201 * Stolen time is counted as time when either the vcpu is able to 202 * run as part of a virtual core, but the task running the vcore 203 * is preempted or sleeping, or when the vcpu needs something done 204 * in the kernel by the task running the vcpu, but that task is 205 * preempted or sleeping. Those two things have to be counted 206 * separately, since one of the vcpu tasks will take on the job 207 * of running the core, and the other vcpu tasks in the vcore will 208 * sleep waiting for it to do that, but that sleep shouldn't count 209 * as stolen time. 210 * 211 * Hence we accumulate stolen time when the vcpu can run as part of 212 * a vcore using vc->stolen_tb, and the stolen time when the vcpu 213 * needs its task to do other things in the kernel (for example, 214 * service a page fault) in busy_stolen. We don't accumulate 215 * stolen time for a vcore when it is inactive, or for a vcpu 216 * when it is in state RUNNING or NOTREADY. NOTREADY is a bit of 217 * a misnomer; it means that the vcpu task is not executing in 218 * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in 219 * the kernel. We don't have any way of dividing up that time 220 * between time that the vcpu is genuinely stopped, time that 221 * the task is actively working on behalf of the vcpu, and time 222 * that the task is preempted, so we don't count any of it as 223 * stolen. 224 * 225 * Updates to busy_stolen are protected by arch.tbacct_lock; 226 * updates to vc->stolen_tb are protected by the vcore->stoltb_lock 227 * lock. The stolen times are measured in units of timebase ticks. 228 * (Note that the != TB_NIL checks below are purely defensive; 229 * they should never fail.) 230 */ 231 232 static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc) 233 { 234 unsigned long flags; 235 236 spin_lock_irqsave(&vc->stoltb_lock, flags); 237 vc->preempt_tb = mftb(); 238 spin_unlock_irqrestore(&vc->stoltb_lock, flags); 239 } 240 241 static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc) 242 { 243 unsigned long flags; 244 245 spin_lock_irqsave(&vc->stoltb_lock, flags); 246 if (vc->preempt_tb != TB_NIL) { 247 vc->stolen_tb += mftb() - vc->preempt_tb; 248 vc->preempt_tb = TB_NIL; 249 } 250 spin_unlock_irqrestore(&vc->stoltb_lock, flags); 251 } 252 253 static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu) 254 { 255 struct kvmppc_vcore *vc = vcpu->arch.vcore; 256 unsigned long flags; 257 258 /* 259 * We can test vc->runner without taking the vcore lock, 260 * because only this task ever sets vc->runner to this 261 * vcpu, and once it is set to this vcpu, only this task 262 * ever sets it to NULL. 263 */ 264 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING) 265 kvmppc_core_end_stolen(vc); 266 267 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); 268 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST && 269 vcpu->arch.busy_preempt != TB_NIL) { 270 vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt; 271 vcpu->arch.busy_preempt = TB_NIL; 272 } 273 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); 274 } 275 276 static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu) 277 { 278 struct kvmppc_vcore *vc = vcpu->arch.vcore; 279 unsigned long flags; 280 281 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING) 282 kvmppc_core_start_stolen(vc); 283 284 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); 285 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST) 286 vcpu->arch.busy_preempt = mftb(); 287 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); 288 } 289 290 static void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr) 291 { 292 /* 293 * Check for illegal transactional state bit combination 294 * and if we find it, force the TS field to a safe state. 295 */ 296 if ((msr & MSR_TS_MASK) == MSR_TS_MASK) 297 msr &= ~MSR_TS_MASK; 298 vcpu->arch.shregs.msr = msr; 299 kvmppc_end_cede(vcpu); 300 } 301 302 static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr) 303 { 304 vcpu->arch.pvr = pvr; 305 } 306 307 /* Dummy value used in computing PCR value below */ 308 #define PCR_ARCH_300 (PCR_ARCH_207 << 1) 309 310 static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) 311 { 312 unsigned long host_pcr_bit = 0, guest_pcr_bit = 0; 313 struct kvmppc_vcore *vc = vcpu->arch.vcore; 314 315 /* We can (emulate) our own architecture version and anything older */ 316 if (cpu_has_feature(CPU_FTR_ARCH_300)) 317 host_pcr_bit = PCR_ARCH_300; 318 else if (cpu_has_feature(CPU_FTR_ARCH_207S)) 319 host_pcr_bit = PCR_ARCH_207; 320 else if (cpu_has_feature(CPU_FTR_ARCH_206)) 321 host_pcr_bit = PCR_ARCH_206; 322 else 323 host_pcr_bit = PCR_ARCH_205; 324 325 /* Determine lowest PCR bit needed to run guest in given PVR level */ 326 guest_pcr_bit = host_pcr_bit; 327 if (arch_compat) { 328 switch (arch_compat) { 329 case PVR_ARCH_205: 330 guest_pcr_bit = PCR_ARCH_205; 331 break; 332 case PVR_ARCH_206: 333 case PVR_ARCH_206p: 334 guest_pcr_bit = PCR_ARCH_206; 335 break; 336 case PVR_ARCH_207: 337 guest_pcr_bit = PCR_ARCH_207; 338 break; 339 case PVR_ARCH_300: 340 guest_pcr_bit = PCR_ARCH_300; 341 break; 342 default: 343 return -EINVAL; 344 } 345 } 346 347 /* Check requested PCR bits don't exceed our capabilities */ 348 if (guest_pcr_bit > host_pcr_bit) 349 return -EINVAL; 350 351 spin_lock(&vc->lock); 352 vc->arch_compat = arch_compat; 353 /* Set all PCR bits for which guest_pcr_bit <= bit < host_pcr_bit */ 354 vc->pcr = host_pcr_bit - guest_pcr_bit; 355 spin_unlock(&vc->lock); 356 357 return 0; 358 } 359 360 static void kvmppc_dump_regs(struct kvm_vcpu *vcpu) 361 { 362 int r; 363 364 pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id); 365 pr_err("pc = %.16lx msr = %.16llx trap = %x\n", 366 vcpu->arch.pc, vcpu->arch.shregs.msr, vcpu->arch.trap); 367 for (r = 0; r < 16; ++r) 368 pr_err("r%2d = %.16lx r%d = %.16lx\n", 369 r, kvmppc_get_gpr(vcpu, r), 370 r+16, kvmppc_get_gpr(vcpu, r+16)); 371 pr_err("ctr = %.16lx lr = %.16lx\n", 372 vcpu->arch.ctr, vcpu->arch.lr); 373 pr_err("srr0 = %.16llx srr1 = %.16llx\n", 374 vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1); 375 pr_err("sprg0 = %.16llx sprg1 = %.16llx\n", 376 vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1); 377 pr_err("sprg2 = %.16llx sprg3 = %.16llx\n", 378 vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3); 379 pr_err("cr = %.8x xer = %.16lx dsisr = %.8x\n", 380 vcpu->arch.cr, vcpu->arch.xer, vcpu->arch.shregs.dsisr); 381 pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar); 382 pr_err("fault dar = %.16lx dsisr = %.8x\n", 383 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr); 384 pr_err("SLB (%d entries):\n", vcpu->arch.slb_max); 385 for (r = 0; r < vcpu->arch.slb_max; ++r) 386 pr_err(" ESID = %.16llx VSID = %.16llx\n", 387 vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv); 388 pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n", 389 vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1, 390 vcpu->arch.last_inst); 391 } 392 393 static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id) 394 { 395 struct kvm_vcpu *ret; 396 397 mutex_lock(&kvm->lock); 398 ret = kvm_get_vcpu_by_id(kvm, id); 399 mutex_unlock(&kvm->lock); 400 return ret; 401 } 402 403 static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa) 404 { 405 vpa->__old_status |= LPPACA_OLD_SHARED_PROC; 406 vpa->yield_count = cpu_to_be32(1); 407 } 408 409 static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v, 410 unsigned long addr, unsigned long len) 411 { 412 /* check address is cacheline aligned */ 413 if (addr & (L1_CACHE_BYTES - 1)) 414 return -EINVAL; 415 spin_lock(&vcpu->arch.vpa_update_lock); 416 if (v->next_gpa != addr || v->len != len) { 417 v->next_gpa = addr; 418 v->len = addr ? len : 0; 419 v->update_pending = 1; 420 } 421 spin_unlock(&vcpu->arch.vpa_update_lock); 422 return 0; 423 } 424 425 /* Length for a per-processor buffer is passed in at offset 4 in the buffer */ 426 struct reg_vpa { 427 u32 dummy; 428 union { 429 __be16 hword; 430 __be32 word; 431 } length; 432 }; 433 434 static int vpa_is_registered(struct kvmppc_vpa *vpap) 435 { 436 if (vpap->update_pending) 437 return vpap->next_gpa != 0; 438 return vpap->pinned_addr != NULL; 439 } 440 441 static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu, 442 unsigned long flags, 443 unsigned long vcpuid, unsigned long vpa) 444 { 445 struct kvm *kvm = vcpu->kvm; 446 unsigned long len, nb; 447 void *va; 448 struct kvm_vcpu *tvcpu; 449 int err; 450 int subfunc; 451 struct kvmppc_vpa *vpap; 452 453 tvcpu = kvmppc_find_vcpu(kvm, vcpuid); 454 if (!tvcpu) 455 return H_PARAMETER; 456 457 subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK; 458 if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL || 459 subfunc == H_VPA_REG_SLB) { 460 /* Registering new area - address must be cache-line aligned */ 461 if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa) 462 return H_PARAMETER; 463 464 /* convert logical addr to kernel addr and read length */ 465 va = kvmppc_pin_guest_page(kvm, vpa, &nb); 466 if (va == NULL) 467 return H_PARAMETER; 468 if (subfunc == H_VPA_REG_VPA) 469 len = be16_to_cpu(((struct reg_vpa *)va)->length.hword); 470 else 471 len = be32_to_cpu(((struct reg_vpa *)va)->length.word); 472 kvmppc_unpin_guest_page(kvm, va, vpa, false); 473 474 /* Check length */ 475 if (len > nb || len < sizeof(struct reg_vpa)) 476 return H_PARAMETER; 477 } else { 478 vpa = 0; 479 len = 0; 480 } 481 482 err = H_PARAMETER; 483 vpap = NULL; 484 spin_lock(&tvcpu->arch.vpa_update_lock); 485 486 switch (subfunc) { 487 case H_VPA_REG_VPA: /* register VPA */ 488 if (len < sizeof(struct lppaca)) 489 break; 490 vpap = &tvcpu->arch.vpa; 491 err = 0; 492 break; 493 494 case H_VPA_REG_DTL: /* register DTL */ 495 if (len < sizeof(struct dtl_entry)) 496 break; 497 len -= len % sizeof(struct dtl_entry); 498 499 /* Check that they have previously registered a VPA */ 500 err = H_RESOURCE; 501 if (!vpa_is_registered(&tvcpu->arch.vpa)) 502 break; 503 504 vpap = &tvcpu->arch.dtl; 505 err = 0; 506 break; 507 508 case H_VPA_REG_SLB: /* register SLB shadow buffer */ 509 /* Check that they have previously registered a VPA */ 510 err = H_RESOURCE; 511 if (!vpa_is_registered(&tvcpu->arch.vpa)) 512 break; 513 514 vpap = &tvcpu->arch.slb_shadow; 515 err = 0; 516 break; 517 518 case H_VPA_DEREG_VPA: /* deregister VPA */ 519 /* Check they don't still have a DTL or SLB buf registered */ 520 err = H_RESOURCE; 521 if (vpa_is_registered(&tvcpu->arch.dtl) || 522 vpa_is_registered(&tvcpu->arch.slb_shadow)) 523 break; 524 525 vpap = &tvcpu->arch.vpa; 526 err = 0; 527 break; 528 529 case H_VPA_DEREG_DTL: /* deregister DTL */ 530 vpap = &tvcpu->arch.dtl; 531 err = 0; 532 break; 533 534 case H_VPA_DEREG_SLB: /* deregister SLB shadow buffer */ 535 vpap = &tvcpu->arch.slb_shadow; 536 err = 0; 537 break; 538 } 539 540 if (vpap) { 541 vpap->next_gpa = vpa; 542 vpap->len = len; 543 vpap->update_pending = 1; 544 } 545 546 spin_unlock(&tvcpu->arch.vpa_update_lock); 547 548 return err; 549 } 550 551 static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap) 552 { 553 struct kvm *kvm = vcpu->kvm; 554 void *va; 555 unsigned long nb; 556 unsigned long gpa; 557 558 /* 559 * We need to pin the page pointed to by vpap->next_gpa, 560 * but we can't call kvmppc_pin_guest_page under the lock 561 * as it does get_user_pages() and down_read(). So we 562 * have to drop the lock, pin the page, then get the lock 563 * again and check that a new area didn't get registered 564 * in the meantime. 565 */ 566 for (;;) { 567 gpa = vpap->next_gpa; 568 spin_unlock(&vcpu->arch.vpa_update_lock); 569 va = NULL; 570 nb = 0; 571 if (gpa) 572 va = kvmppc_pin_guest_page(kvm, gpa, &nb); 573 spin_lock(&vcpu->arch.vpa_update_lock); 574 if (gpa == vpap->next_gpa) 575 break; 576 /* sigh... unpin that one and try again */ 577 if (va) 578 kvmppc_unpin_guest_page(kvm, va, gpa, false); 579 } 580 581 vpap->update_pending = 0; 582 if (va && nb < vpap->len) { 583 /* 584 * If it's now too short, it must be that userspace 585 * has changed the mappings underlying guest memory, 586 * so unregister the region. 587 */ 588 kvmppc_unpin_guest_page(kvm, va, gpa, false); 589 va = NULL; 590 } 591 if (vpap->pinned_addr) 592 kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa, 593 vpap->dirty); 594 vpap->gpa = gpa; 595 vpap->pinned_addr = va; 596 vpap->dirty = false; 597 if (va) 598 vpap->pinned_end = va + vpap->len; 599 } 600 601 static void kvmppc_update_vpas(struct kvm_vcpu *vcpu) 602 { 603 if (!(vcpu->arch.vpa.update_pending || 604 vcpu->arch.slb_shadow.update_pending || 605 vcpu->arch.dtl.update_pending)) 606 return; 607 608 spin_lock(&vcpu->arch.vpa_update_lock); 609 if (vcpu->arch.vpa.update_pending) { 610 kvmppc_update_vpa(vcpu, &vcpu->arch.vpa); 611 if (vcpu->arch.vpa.pinned_addr) 612 init_vpa(vcpu, vcpu->arch.vpa.pinned_addr); 613 } 614 if (vcpu->arch.dtl.update_pending) { 615 kvmppc_update_vpa(vcpu, &vcpu->arch.dtl); 616 vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr; 617 vcpu->arch.dtl_index = 0; 618 } 619 if (vcpu->arch.slb_shadow.update_pending) 620 kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow); 621 spin_unlock(&vcpu->arch.vpa_update_lock); 622 } 623 624 /* 625 * Return the accumulated stolen time for the vcore up until `now'. 626 * The caller should hold the vcore lock. 627 */ 628 static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now) 629 { 630 u64 p; 631 unsigned long flags; 632 633 spin_lock_irqsave(&vc->stoltb_lock, flags); 634 p = vc->stolen_tb; 635 if (vc->vcore_state != VCORE_INACTIVE && 636 vc->preempt_tb != TB_NIL) 637 p += now - vc->preempt_tb; 638 spin_unlock_irqrestore(&vc->stoltb_lock, flags); 639 return p; 640 } 641 642 static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, 643 struct kvmppc_vcore *vc) 644 { 645 struct dtl_entry *dt; 646 struct lppaca *vpa; 647 unsigned long stolen; 648 unsigned long core_stolen; 649 u64 now; 650 unsigned long flags; 651 652 dt = vcpu->arch.dtl_ptr; 653 vpa = vcpu->arch.vpa.pinned_addr; 654 now = mftb(); 655 core_stolen = vcore_stolen_time(vc, now); 656 stolen = core_stolen - vcpu->arch.stolen_logged; 657 vcpu->arch.stolen_logged = core_stolen; 658 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); 659 stolen += vcpu->arch.busy_stolen; 660 vcpu->arch.busy_stolen = 0; 661 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); 662 if (!dt || !vpa) 663 return; 664 memset(dt, 0, sizeof(struct dtl_entry)); 665 dt->dispatch_reason = 7; 666 dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid); 667 dt->timebase = cpu_to_be64(now + vc->tb_offset); 668 dt->enqueue_to_dispatch_time = cpu_to_be32(stolen); 669 dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu)); 670 dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr); 671 ++dt; 672 if (dt == vcpu->arch.dtl.pinned_end) 673 dt = vcpu->arch.dtl.pinned_addr; 674 vcpu->arch.dtl_ptr = dt; 675 /* order writing *dt vs. writing vpa->dtl_idx */ 676 smp_wmb(); 677 vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index); 678 vcpu->arch.dtl.dirty = true; 679 } 680 681 /* See if there is a doorbell interrupt pending for a vcpu */ 682 static bool kvmppc_doorbell_pending(struct kvm_vcpu *vcpu) 683 { 684 int thr; 685 struct kvmppc_vcore *vc; 686 687 if (vcpu->arch.doorbell_request) 688 return true; 689 /* 690 * Ensure that the read of vcore->dpdes comes after the read 691 * of vcpu->doorbell_request. This barrier matches the 692 * lwsync in book3s_hv_rmhandlers.S just before the 693 * fast_guest_return label. 694 */ 695 smp_rmb(); 696 vc = vcpu->arch.vcore; 697 thr = vcpu->vcpu_id - vc->first_vcpuid; 698 return !!(vc->dpdes & (1 << thr)); 699 } 700 701 static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu) 702 { 703 if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207) 704 return true; 705 if ((!vcpu->arch.vcore->arch_compat) && 706 cpu_has_feature(CPU_FTR_ARCH_207S)) 707 return true; 708 return false; 709 } 710 711 static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags, 712 unsigned long resource, unsigned long value1, 713 unsigned long value2) 714 { 715 switch (resource) { 716 case H_SET_MODE_RESOURCE_SET_CIABR: 717 if (!kvmppc_power8_compatible(vcpu)) 718 return H_P2; 719 if (value2) 720 return H_P4; 721 if (mflags) 722 return H_UNSUPPORTED_FLAG_START; 723 /* Guests can't breakpoint the hypervisor */ 724 if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER) 725 return H_P3; 726 vcpu->arch.ciabr = value1; 727 return H_SUCCESS; 728 case H_SET_MODE_RESOURCE_SET_DAWR: 729 if (!kvmppc_power8_compatible(vcpu)) 730 return H_P2; 731 if (mflags) 732 return H_UNSUPPORTED_FLAG_START; 733 if (value2 & DABRX_HYP) 734 return H_P4; 735 vcpu->arch.dawr = value1; 736 vcpu->arch.dawrx = value2; 737 return H_SUCCESS; 738 default: 739 return H_TOO_HARD; 740 } 741 } 742 743 static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target) 744 { 745 struct kvmppc_vcore *vcore = target->arch.vcore; 746 747 /* 748 * We expect to have been called by the real mode handler 749 * (kvmppc_rm_h_confer()) which would have directly returned 750 * H_SUCCESS if the source vcore wasn't idle (e.g. if it may 751 * have useful work to do and should not confer) so we don't 752 * recheck that here. 753 */ 754 755 spin_lock(&vcore->lock); 756 if (target->arch.state == KVMPPC_VCPU_RUNNABLE && 757 vcore->vcore_state != VCORE_INACTIVE && 758 vcore->runner) 759 target = vcore->runner; 760 spin_unlock(&vcore->lock); 761 762 return kvm_vcpu_yield_to(target); 763 } 764 765 static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu) 766 { 767 int yield_count = 0; 768 struct lppaca *lppaca; 769 770 spin_lock(&vcpu->arch.vpa_update_lock); 771 lppaca = (struct lppaca *)vcpu->arch.vpa.pinned_addr; 772 if (lppaca) 773 yield_count = be32_to_cpu(lppaca->yield_count); 774 spin_unlock(&vcpu->arch.vpa_update_lock); 775 return yield_count; 776 } 777 778 int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) 779 { 780 unsigned long req = kvmppc_get_gpr(vcpu, 3); 781 unsigned long target, ret = H_SUCCESS; 782 int yield_count; 783 struct kvm_vcpu *tvcpu; 784 int idx, rc; 785 786 if (req <= MAX_HCALL_OPCODE && 787 !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls)) 788 return RESUME_HOST; 789 790 switch (req) { 791 case H_CEDE: 792 break; 793 case H_PROD: 794 target = kvmppc_get_gpr(vcpu, 4); 795 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target); 796 if (!tvcpu) { 797 ret = H_PARAMETER; 798 break; 799 } 800 tvcpu->arch.prodded = 1; 801 smp_mb(); 802 if (tvcpu->arch.ceded) 803 kvmppc_fast_vcpu_kick_hv(tvcpu); 804 break; 805 case H_CONFER: 806 target = kvmppc_get_gpr(vcpu, 4); 807 if (target == -1) 808 break; 809 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target); 810 if (!tvcpu) { 811 ret = H_PARAMETER; 812 break; 813 } 814 yield_count = kvmppc_get_gpr(vcpu, 5); 815 if (kvmppc_get_yield_count(tvcpu) != yield_count) 816 break; 817 kvm_arch_vcpu_yield_to(tvcpu); 818 break; 819 case H_REGISTER_VPA: 820 ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4), 821 kvmppc_get_gpr(vcpu, 5), 822 kvmppc_get_gpr(vcpu, 6)); 823 break; 824 case H_RTAS: 825 if (list_empty(&vcpu->kvm->arch.rtas_tokens)) 826 return RESUME_HOST; 827 828 idx = srcu_read_lock(&vcpu->kvm->srcu); 829 rc = kvmppc_rtas_hcall(vcpu); 830 srcu_read_unlock(&vcpu->kvm->srcu, idx); 831 832 if (rc == -ENOENT) 833 return RESUME_HOST; 834 else if (rc == 0) 835 break; 836 837 /* Send the error out to userspace via KVM_RUN */ 838 return rc; 839 case H_LOGICAL_CI_LOAD: 840 ret = kvmppc_h_logical_ci_load(vcpu); 841 if (ret == H_TOO_HARD) 842 return RESUME_HOST; 843 break; 844 case H_LOGICAL_CI_STORE: 845 ret = kvmppc_h_logical_ci_store(vcpu); 846 if (ret == H_TOO_HARD) 847 return RESUME_HOST; 848 break; 849 case H_SET_MODE: 850 ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4), 851 kvmppc_get_gpr(vcpu, 5), 852 kvmppc_get_gpr(vcpu, 6), 853 kvmppc_get_gpr(vcpu, 7)); 854 if (ret == H_TOO_HARD) 855 return RESUME_HOST; 856 break; 857 case H_XIRR: 858 case H_CPPR: 859 case H_EOI: 860 case H_IPI: 861 case H_IPOLL: 862 case H_XIRR_X: 863 if (kvmppc_xics_enabled(vcpu)) { 864 if (xive_enabled()) { 865 ret = H_NOT_AVAILABLE; 866 return RESUME_GUEST; 867 } 868 ret = kvmppc_xics_hcall(vcpu, req); 869 break; 870 } 871 return RESUME_HOST; 872 case H_PUT_TCE: 873 ret = kvmppc_h_put_tce(vcpu, kvmppc_get_gpr(vcpu, 4), 874 kvmppc_get_gpr(vcpu, 5), 875 kvmppc_get_gpr(vcpu, 6)); 876 if (ret == H_TOO_HARD) 877 return RESUME_HOST; 878 break; 879 case H_PUT_TCE_INDIRECT: 880 ret = kvmppc_h_put_tce_indirect(vcpu, kvmppc_get_gpr(vcpu, 4), 881 kvmppc_get_gpr(vcpu, 5), 882 kvmppc_get_gpr(vcpu, 6), 883 kvmppc_get_gpr(vcpu, 7)); 884 if (ret == H_TOO_HARD) 885 return RESUME_HOST; 886 break; 887 case H_STUFF_TCE: 888 ret = kvmppc_h_stuff_tce(vcpu, kvmppc_get_gpr(vcpu, 4), 889 kvmppc_get_gpr(vcpu, 5), 890 kvmppc_get_gpr(vcpu, 6), 891 kvmppc_get_gpr(vcpu, 7)); 892 if (ret == H_TOO_HARD) 893 return RESUME_HOST; 894 break; 895 default: 896 return RESUME_HOST; 897 } 898 kvmppc_set_gpr(vcpu, 3, ret); 899 vcpu->arch.hcall_needed = 0; 900 return RESUME_GUEST; 901 } 902 903 static int kvmppc_hcall_impl_hv(unsigned long cmd) 904 { 905 switch (cmd) { 906 case H_CEDE: 907 case H_PROD: 908 case H_CONFER: 909 case H_REGISTER_VPA: 910 case H_SET_MODE: 911 case H_LOGICAL_CI_LOAD: 912 case H_LOGICAL_CI_STORE: 913 #ifdef CONFIG_KVM_XICS 914 case H_XIRR: 915 case H_CPPR: 916 case H_EOI: 917 case H_IPI: 918 case H_IPOLL: 919 case H_XIRR_X: 920 #endif 921 return 1; 922 } 923 924 /* See if it's in the real-mode table */ 925 return kvmppc_hcall_impl_hv_realmode(cmd); 926 } 927 928 static int kvmppc_emulate_debug_inst(struct kvm_run *run, 929 struct kvm_vcpu *vcpu) 930 { 931 u32 last_inst; 932 933 if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) != 934 EMULATE_DONE) { 935 /* 936 * Fetch failed, so return to guest and 937 * try executing it again. 938 */ 939 return RESUME_GUEST; 940 } 941 942 if (last_inst == KVMPPC_INST_SW_BREAKPOINT) { 943 run->exit_reason = KVM_EXIT_DEBUG; 944 run->debug.arch.address = kvmppc_get_pc(vcpu); 945 return RESUME_HOST; 946 } else { 947 kvmppc_core_queue_program(vcpu, SRR1_PROGILL); 948 return RESUME_GUEST; 949 } 950 } 951 952 static void do_nothing(void *x) 953 { 954 } 955 956 static unsigned long kvmppc_read_dpdes(struct kvm_vcpu *vcpu) 957 { 958 int thr, cpu, pcpu, nthreads; 959 struct kvm_vcpu *v; 960 unsigned long dpdes; 961 962 nthreads = vcpu->kvm->arch.emul_smt_mode; 963 dpdes = 0; 964 cpu = vcpu->vcpu_id & ~(nthreads - 1); 965 for (thr = 0; thr < nthreads; ++thr, ++cpu) { 966 v = kvmppc_find_vcpu(vcpu->kvm, cpu); 967 if (!v) 968 continue; 969 /* 970 * If the vcpu is currently running on a physical cpu thread, 971 * interrupt it in order to pull it out of the guest briefly, 972 * which will update its vcore->dpdes value. 973 */ 974 pcpu = READ_ONCE(v->cpu); 975 if (pcpu >= 0) 976 smp_call_function_single(pcpu, do_nothing, NULL, 1); 977 if (kvmppc_doorbell_pending(v)) 978 dpdes |= 1 << thr; 979 } 980 return dpdes; 981 } 982 983 /* 984 * On POWER9, emulate doorbell-related instructions in order to 985 * give the guest the illusion of running on a multi-threaded core. 986 * The instructions emulated are msgsndp, msgclrp, mfspr TIR, 987 * and mfspr DPDES. 988 */ 989 static int kvmppc_emulate_doorbell_instr(struct kvm_vcpu *vcpu) 990 { 991 u32 inst, rb, thr; 992 unsigned long arg; 993 struct kvm *kvm = vcpu->kvm; 994 struct kvm_vcpu *tvcpu; 995 996 if (!cpu_has_feature(CPU_FTR_ARCH_300)) 997 return EMULATE_FAIL; 998 if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &inst) != EMULATE_DONE) 999 return RESUME_GUEST; 1000 if (get_op(inst) != 31) 1001 return EMULATE_FAIL; 1002 rb = get_rb(inst); 1003 thr = vcpu->vcpu_id & (kvm->arch.emul_smt_mode - 1); 1004 switch (get_xop(inst)) { 1005 case OP_31_XOP_MSGSNDP: 1006 arg = kvmppc_get_gpr(vcpu, rb); 1007 if (((arg >> 27) & 0xf) != PPC_DBELL_SERVER) 1008 break; 1009 arg &= 0x3f; 1010 if (arg >= kvm->arch.emul_smt_mode) 1011 break; 1012 tvcpu = kvmppc_find_vcpu(kvm, vcpu->vcpu_id - thr + arg); 1013 if (!tvcpu) 1014 break; 1015 if (!tvcpu->arch.doorbell_request) { 1016 tvcpu->arch.doorbell_request = 1; 1017 kvmppc_fast_vcpu_kick_hv(tvcpu); 1018 } 1019 break; 1020 case OP_31_XOP_MSGCLRP: 1021 arg = kvmppc_get_gpr(vcpu, rb); 1022 if (((arg >> 27) & 0xf) != PPC_DBELL_SERVER) 1023 break; 1024 vcpu->arch.vcore->dpdes = 0; 1025 vcpu->arch.doorbell_request = 0; 1026 break; 1027 case OP_31_XOP_MFSPR: 1028 switch (get_sprn(inst)) { 1029 case SPRN_TIR: 1030 arg = thr; 1031 break; 1032 case SPRN_DPDES: 1033 arg = kvmppc_read_dpdes(vcpu); 1034 break; 1035 default: 1036 return EMULATE_FAIL; 1037 } 1038 kvmppc_set_gpr(vcpu, get_rt(inst), arg); 1039 break; 1040 default: 1041 return EMULATE_FAIL; 1042 } 1043 kvmppc_set_pc(vcpu, kvmppc_get_pc(vcpu) + 4); 1044 return RESUME_GUEST; 1045 } 1046 1047 static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu, 1048 struct task_struct *tsk) 1049 { 1050 int r = RESUME_HOST; 1051 1052 vcpu->stat.sum_exits++; 1053 1054 /* 1055 * This can happen if an interrupt occurs in the last stages 1056 * of guest entry or the first stages of guest exit (i.e. after 1057 * setting paca->kvm_hstate.in_guest to KVM_GUEST_MODE_GUEST_HV 1058 * and before setting it to KVM_GUEST_MODE_HOST_HV). 1059 * That can happen due to a bug, or due to a machine check 1060 * occurring at just the wrong time. 1061 */ 1062 if (vcpu->arch.shregs.msr & MSR_HV) { 1063 printk(KERN_EMERG "KVM trap in HV mode!\n"); 1064 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n", 1065 vcpu->arch.trap, kvmppc_get_pc(vcpu), 1066 vcpu->arch.shregs.msr); 1067 kvmppc_dump_regs(vcpu); 1068 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1069 run->hw.hardware_exit_reason = vcpu->arch.trap; 1070 return RESUME_HOST; 1071 } 1072 run->exit_reason = KVM_EXIT_UNKNOWN; 1073 run->ready_for_interrupt_injection = 1; 1074 switch (vcpu->arch.trap) { 1075 /* We're good on these - the host merely wanted to get our attention */ 1076 case BOOK3S_INTERRUPT_HV_DECREMENTER: 1077 vcpu->stat.dec_exits++; 1078 r = RESUME_GUEST; 1079 break; 1080 case BOOK3S_INTERRUPT_EXTERNAL: 1081 case BOOK3S_INTERRUPT_H_DOORBELL: 1082 case BOOK3S_INTERRUPT_H_VIRT: 1083 vcpu->stat.ext_intr_exits++; 1084 r = RESUME_GUEST; 1085 break; 1086 /* HMI is hypervisor interrupt and host has handled it. Resume guest.*/ 1087 case BOOK3S_INTERRUPT_HMI: 1088 case BOOK3S_INTERRUPT_PERFMON: 1089 r = RESUME_GUEST; 1090 break; 1091 case BOOK3S_INTERRUPT_MACHINE_CHECK: 1092 /* Exit to guest with KVM_EXIT_NMI as exit reason */ 1093 run->exit_reason = KVM_EXIT_NMI; 1094 run->hw.hardware_exit_reason = vcpu->arch.trap; 1095 /* Clear out the old NMI status from run->flags */ 1096 run->flags &= ~KVM_RUN_PPC_NMI_DISP_MASK; 1097 /* Now set the NMI status */ 1098 if (vcpu->arch.mce_evt.disposition == MCE_DISPOSITION_RECOVERED) 1099 run->flags |= KVM_RUN_PPC_NMI_DISP_FULLY_RECOV; 1100 else 1101 run->flags |= KVM_RUN_PPC_NMI_DISP_NOT_RECOV; 1102 1103 r = RESUME_HOST; 1104 /* Print the MCE event to host console. */ 1105 machine_check_print_event_info(&vcpu->arch.mce_evt, false); 1106 break; 1107 case BOOK3S_INTERRUPT_PROGRAM: 1108 { 1109 ulong flags; 1110 /* 1111 * Normally program interrupts are delivered directly 1112 * to the guest by the hardware, but we can get here 1113 * as a result of a hypervisor emulation interrupt 1114 * (e40) getting turned into a 700 by BML RTAS. 1115 */ 1116 flags = vcpu->arch.shregs.msr & 0x1f0000ull; 1117 kvmppc_core_queue_program(vcpu, flags); 1118 r = RESUME_GUEST; 1119 break; 1120 } 1121 case BOOK3S_INTERRUPT_SYSCALL: 1122 { 1123 /* hcall - punt to userspace */ 1124 int i; 1125 1126 /* hypercall with MSR_PR has already been handled in rmode, 1127 * and never reaches here. 1128 */ 1129 1130 run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3); 1131 for (i = 0; i < 9; ++i) 1132 run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i); 1133 run->exit_reason = KVM_EXIT_PAPR_HCALL; 1134 vcpu->arch.hcall_needed = 1; 1135 r = RESUME_HOST; 1136 break; 1137 } 1138 /* 1139 * We get these next two if the guest accesses a page which it thinks 1140 * it has mapped but which is not actually present, either because 1141 * it is for an emulated I/O device or because the corresonding 1142 * host page has been paged out. Any other HDSI/HISI interrupts 1143 * have been handled already. 1144 */ 1145 case BOOK3S_INTERRUPT_H_DATA_STORAGE: 1146 r = RESUME_PAGE_FAULT; 1147 break; 1148 case BOOK3S_INTERRUPT_H_INST_STORAGE: 1149 vcpu->arch.fault_dar = kvmppc_get_pc(vcpu); 1150 vcpu->arch.fault_dsisr = 0; 1151 r = RESUME_PAGE_FAULT; 1152 break; 1153 /* 1154 * This occurs if the guest executes an illegal instruction. 1155 * If the guest debug is disabled, generate a program interrupt 1156 * to the guest. If guest debug is enabled, we need to check 1157 * whether the instruction is a software breakpoint instruction. 1158 * Accordingly return to Guest or Host. 1159 */ 1160 case BOOK3S_INTERRUPT_H_EMUL_ASSIST: 1161 if (vcpu->arch.emul_inst != KVM_INST_FETCH_FAILED) 1162 vcpu->arch.last_inst = kvmppc_need_byteswap(vcpu) ? 1163 swab32(vcpu->arch.emul_inst) : 1164 vcpu->arch.emul_inst; 1165 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) { 1166 r = kvmppc_emulate_debug_inst(run, vcpu); 1167 } else { 1168 kvmppc_core_queue_program(vcpu, SRR1_PROGILL); 1169 r = RESUME_GUEST; 1170 } 1171 break; 1172 /* 1173 * This occurs if the guest (kernel or userspace), does something that 1174 * is prohibited by HFSCR. 1175 * On POWER9, this could be a doorbell instruction that we need 1176 * to emulate. 1177 * Otherwise, we just generate a program interrupt to the guest. 1178 */ 1179 case BOOK3S_INTERRUPT_H_FAC_UNAVAIL: 1180 r = EMULATE_FAIL; 1181 if ((vcpu->arch.hfscr >> 56) == FSCR_MSGP_LG) 1182 r = kvmppc_emulate_doorbell_instr(vcpu); 1183 if (r == EMULATE_FAIL) { 1184 kvmppc_core_queue_program(vcpu, SRR1_PROGILL); 1185 r = RESUME_GUEST; 1186 } 1187 break; 1188 case BOOK3S_INTERRUPT_HV_RM_HARD: 1189 r = RESUME_PASSTHROUGH; 1190 break; 1191 default: 1192 kvmppc_dump_regs(vcpu); 1193 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n", 1194 vcpu->arch.trap, kvmppc_get_pc(vcpu), 1195 vcpu->arch.shregs.msr); 1196 run->hw.hardware_exit_reason = vcpu->arch.trap; 1197 r = RESUME_HOST; 1198 break; 1199 } 1200 1201 return r; 1202 } 1203 1204 static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu, 1205 struct kvm_sregs *sregs) 1206 { 1207 int i; 1208 1209 memset(sregs, 0, sizeof(struct kvm_sregs)); 1210 sregs->pvr = vcpu->arch.pvr; 1211 for (i = 0; i < vcpu->arch.slb_max; i++) { 1212 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige; 1213 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv; 1214 } 1215 1216 return 0; 1217 } 1218 1219 static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu, 1220 struct kvm_sregs *sregs) 1221 { 1222 int i, j; 1223 1224 /* Only accept the same PVR as the host's, since we can't spoof it */ 1225 if (sregs->pvr != vcpu->arch.pvr) 1226 return -EINVAL; 1227 1228 j = 0; 1229 for (i = 0; i < vcpu->arch.slb_nr; i++) { 1230 if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) { 1231 vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe; 1232 vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv; 1233 ++j; 1234 } 1235 } 1236 vcpu->arch.slb_max = j; 1237 1238 return 0; 1239 } 1240 1241 static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr, 1242 bool preserve_top32) 1243 { 1244 struct kvm *kvm = vcpu->kvm; 1245 struct kvmppc_vcore *vc = vcpu->arch.vcore; 1246 u64 mask; 1247 1248 mutex_lock(&kvm->lock); 1249 spin_lock(&vc->lock); 1250 /* 1251 * If ILE (interrupt little-endian) has changed, update the 1252 * MSR_LE bit in the intr_msr for each vcpu in this vcore. 1253 */ 1254 if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) { 1255 struct kvm_vcpu *vcpu; 1256 int i; 1257 1258 kvm_for_each_vcpu(i, vcpu, kvm) { 1259 if (vcpu->arch.vcore != vc) 1260 continue; 1261 if (new_lpcr & LPCR_ILE) 1262 vcpu->arch.intr_msr |= MSR_LE; 1263 else 1264 vcpu->arch.intr_msr &= ~MSR_LE; 1265 } 1266 } 1267 1268 /* 1269 * Userspace can only modify DPFD (default prefetch depth), 1270 * ILE (interrupt little-endian) and TC (translation control). 1271 * On POWER8 and POWER9 userspace can also modify AIL (alt. interrupt loc.). 1272 */ 1273 mask = LPCR_DPFD | LPCR_ILE | LPCR_TC; 1274 if (cpu_has_feature(CPU_FTR_ARCH_207S)) 1275 mask |= LPCR_AIL; 1276 /* 1277 * On POWER9, allow userspace to enable large decrementer for the 1278 * guest, whether or not the host has it enabled. 1279 */ 1280 if (cpu_has_feature(CPU_FTR_ARCH_300)) 1281 mask |= LPCR_LD; 1282 1283 /* Broken 32-bit version of LPCR must not clear top bits */ 1284 if (preserve_top32) 1285 mask &= 0xFFFFFFFF; 1286 vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask); 1287 spin_unlock(&vc->lock); 1288 mutex_unlock(&kvm->lock); 1289 } 1290 1291 static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, 1292 union kvmppc_one_reg *val) 1293 { 1294 int r = 0; 1295 long int i; 1296 1297 switch (id) { 1298 case KVM_REG_PPC_DEBUG_INST: 1299 *val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT); 1300 break; 1301 case KVM_REG_PPC_HIOR: 1302 *val = get_reg_val(id, 0); 1303 break; 1304 case KVM_REG_PPC_DABR: 1305 *val = get_reg_val(id, vcpu->arch.dabr); 1306 break; 1307 case KVM_REG_PPC_DABRX: 1308 *val = get_reg_val(id, vcpu->arch.dabrx); 1309 break; 1310 case KVM_REG_PPC_DSCR: 1311 *val = get_reg_val(id, vcpu->arch.dscr); 1312 break; 1313 case KVM_REG_PPC_PURR: 1314 *val = get_reg_val(id, vcpu->arch.purr); 1315 break; 1316 case KVM_REG_PPC_SPURR: 1317 *val = get_reg_val(id, vcpu->arch.spurr); 1318 break; 1319 case KVM_REG_PPC_AMR: 1320 *val = get_reg_val(id, vcpu->arch.amr); 1321 break; 1322 case KVM_REG_PPC_UAMOR: 1323 *val = get_reg_val(id, vcpu->arch.uamor); 1324 break; 1325 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS: 1326 i = id - KVM_REG_PPC_MMCR0; 1327 *val = get_reg_val(id, vcpu->arch.mmcr[i]); 1328 break; 1329 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8: 1330 i = id - KVM_REG_PPC_PMC1; 1331 *val = get_reg_val(id, vcpu->arch.pmc[i]); 1332 break; 1333 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2: 1334 i = id - KVM_REG_PPC_SPMC1; 1335 *val = get_reg_val(id, vcpu->arch.spmc[i]); 1336 break; 1337 case KVM_REG_PPC_SIAR: 1338 *val = get_reg_val(id, vcpu->arch.siar); 1339 break; 1340 case KVM_REG_PPC_SDAR: 1341 *val = get_reg_val(id, vcpu->arch.sdar); 1342 break; 1343 case KVM_REG_PPC_SIER: 1344 *val = get_reg_val(id, vcpu->arch.sier); 1345 break; 1346 case KVM_REG_PPC_IAMR: 1347 *val = get_reg_val(id, vcpu->arch.iamr); 1348 break; 1349 case KVM_REG_PPC_PSPB: 1350 *val = get_reg_val(id, vcpu->arch.pspb); 1351 break; 1352 case KVM_REG_PPC_DPDES: 1353 *val = get_reg_val(id, vcpu->arch.vcore->dpdes); 1354 break; 1355 case KVM_REG_PPC_VTB: 1356 *val = get_reg_val(id, vcpu->arch.vcore->vtb); 1357 break; 1358 case KVM_REG_PPC_DAWR: 1359 *val = get_reg_val(id, vcpu->arch.dawr); 1360 break; 1361 case KVM_REG_PPC_DAWRX: 1362 *val = get_reg_val(id, vcpu->arch.dawrx); 1363 break; 1364 case KVM_REG_PPC_CIABR: 1365 *val = get_reg_val(id, vcpu->arch.ciabr); 1366 break; 1367 case KVM_REG_PPC_CSIGR: 1368 *val = get_reg_val(id, vcpu->arch.csigr); 1369 break; 1370 case KVM_REG_PPC_TACR: 1371 *val = get_reg_val(id, vcpu->arch.tacr); 1372 break; 1373 case KVM_REG_PPC_TCSCR: 1374 *val = get_reg_val(id, vcpu->arch.tcscr); 1375 break; 1376 case KVM_REG_PPC_PID: 1377 *val = get_reg_val(id, vcpu->arch.pid); 1378 break; 1379 case KVM_REG_PPC_ACOP: 1380 *val = get_reg_val(id, vcpu->arch.acop); 1381 break; 1382 case KVM_REG_PPC_WORT: 1383 *val = get_reg_val(id, vcpu->arch.wort); 1384 break; 1385 case KVM_REG_PPC_TIDR: 1386 *val = get_reg_val(id, vcpu->arch.tid); 1387 break; 1388 case KVM_REG_PPC_PSSCR: 1389 *val = get_reg_val(id, vcpu->arch.psscr); 1390 break; 1391 case KVM_REG_PPC_VPA_ADDR: 1392 spin_lock(&vcpu->arch.vpa_update_lock); 1393 *val = get_reg_val(id, vcpu->arch.vpa.next_gpa); 1394 spin_unlock(&vcpu->arch.vpa_update_lock); 1395 break; 1396 case KVM_REG_PPC_VPA_SLB: 1397 spin_lock(&vcpu->arch.vpa_update_lock); 1398 val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa; 1399 val->vpaval.length = vcpu->arch.slb_shadow.len; 1400 spin_unlock(&vcpu->arch.vpa_update_lock); 1401 break; 1402 case KVM_REG_PPC_VPA_DTL: 1403 spin_lock(&vcpu->arch.vpa_update_lock); 1404 val->vpaval.addr = vcpu->arch.dtl.next_gpa; 1405 val->vpaval.length = vcpu->arch.dtl.len; 1406 spin_unlock(&vcpu->arch.vpa_update_lock); 1407 break; 1408 case KVM_REG_PPC_TB_OFFSET: 1409 *val = get_reg_val(id, vcpu->arch.vcore->tb_offset); 1410 break; 1411 case KVM_REG_PPC_LPCR: 1412 case KVM_REG_PPC_LPCR_64: 1413 *val = get_reg_val(id, vcpu->arch.vcore->lpcr); 1414 break; 1415 case KVM_REG_PPC_PPR: 1416 *val = get_reg_val(id, vcpu->arch.ppr); 1417 break; 1418 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1419 case KVM_REG_PPC_TFHAR: 1420 *val = get_reg_val(id, vcpu->arch.tfhar); 1421 break; 1422 case KVM_REG_PPC_TFIAR: 1423 *val = get_reg_val(id, vcpu->arch.tfiar); 1424 break; 1425 case KVM_REG_PPC_TEXASR: 1426 *val = get_reg_val(id, vcpu->arch.texasr); 1427 break; 1428 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31: 1429 i = id - KVM_REG_PPC_TM_GPR0; 1430 *val = get_reg_val(id, vcpu->arch.gpr_tm[i]); 1431 break; 1432 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63: 1433 { 1434 int j; 1435 i = id - KVM_REG_PPC_TM_VSR0; 1436 if (i < 32) 1437 for (j = 0; j < TS_FPRWIDTH; j++) 1438 val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j]; 1439 else { 1440 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1441 val->vval = vcpu->arch.vr_tm.vr[i-32]; 1442 else 1443 r = -ENXIO; 1444 } 1445 break; 1446 } 1447 case KVM_REG_PPC_TM_CR: 1448 *val = get_reg_val(id, vcpu->arch.cr_tm); 1449 break; 1450 case KVM_REG_PPC_TM_XER: 1451 *val = get_reg_val(id, vcpu->arch.xer_tm); 1452 break; 1453 case KVM_REG_PPC_TM_LR: 1454 *val = get_reg_val(id, vcpu->arch.lr_tm); 1455 break; 1456 case KVM_REG_PPC_TM_CTR: 1457 *val = get_reg_val(id, vcpu->arch.ctr_tm); 1458 break; 1459 case KVM_REG_PPC_TM_FPSCR: 1460 *val = get_reg_val(id, vcpu->arch.fp_tm.fpscr); 1461 break; 1462 case KVM_REG_PPC_TM_AMR: 1463 *val = get_reg_val(id, vcpu->arch.amr_tm); 1464 break; 1465 case KVM_REG_PPC_TM_PPR: 1466 *val = get_reg_val(id, vcpu->arch.ppr_tm); 1467 break; 1468 case KVM_REG_PPC_TM_VRSAVE: 1469 *val = get_reg_val(id, vcpu->arch.vrsave_tm); 1470 break; 1471 case KVM_REG_PPC_TM_VSCR: 1472 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1473 *val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]); 1474 else 1475 r = -ENXIO; 1476 break; 1477 case KVM_REG_PPC_TM_DSCR: 1478 *val = get_reg_val(id, vcpu->arch.dscr_tm); 1479 break; 1480 case KVM_REG_PPC_TM_TAR: 1481 *val = get_reg_val(id, vcpu->arch.tar_tm); 1482 break; 1483 #endif 1484 case KVM_REG_PPC_ARCH_COMPAT: 1485 *val = get_reg_val(id, vcpu->arch.vcore->arch_compat); 1486 break; 1487 default: 1488 r = -EINVAL; 1489 break; 1490 } 1491 1492 return r; 1493 } 1494 1495 static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, 1496 union kvmppc_one_reg *val) 1497 { 1498 int r = 0; 1499 long int i; 1500 unsigned long addr, len; 1501 1502 switch (id) { 1503 case KVM_REG_PPC_HIOR: 1504 /* Only allow this to be set to zero */ 1505 if (set_reg_val(id, *val)) 1506 r = -EINVAL; 1507 break; 1508 case KVM_REG_PPC_DABR: 1509 vcpu->arch.dabr = set_reg_val(id, *val); 1510 break; 1511 case KVM_REG_PPC_DABRX: 1512 vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP; 1513 break; 1514 case KVM_REG_PPC_DSCR: 1515 vcpu->arch.dscr = set_reg_val(id, *val); 1516 break; 1517 case KVM_REG_PPC_PURR: 1518 vcpu->arch.purr = set_reg_val(id, *val); 1519 break; 1520 case KVM_REG_PPC_SPURR: 1521 vcpu->arch.spurr = set_reg_val(id, *val); 1522 break; 1523 case KVM_REG_PPC_AMR: 1524 vcpu->arch.amr = set_reg_val(id, *val); 1525 break; 1526 case KVM_REG_PPC_UAMOR: 1527 vcpu->arch.uamor = set_reg_val(id, *val); 1528 break; 1529 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS: 1530 i = id - KVM_REG_PPC_MMCR0; 1531 vcpu->arch.mmcr[i] = set_reg_val(id, *val); 1532 break; 1533 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8: 1534 i = id - KVM_REG_PPC_PMC1; 1535 vcpu->arch.pmc[i] = set_reg_val(id, *val); 1536 break; 1537 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2: 1538 i = id - KVM_REG_PPC_SPMC1; 1539 vcpu->arch.spmc[i] = set_reg_val(id, *val); 1540 break; 1541 case KVM_REG_PPC_SIAR: 1542 vcpu->arch.siar = set_reg_val(id, *val); 1543 break; 1544 case KVM_REG_PPC_SDAR: 1545 vcpu->arch.sdar = set_reg_val(id, *val); 1546 break; 1547 case KVM_REG_PPC_SIER: 1548 vcpu->arch.sier = set_reg_val(id, *val); 1549 break; 1550 case KVM_REG_PPC_IAMR: 1551 vcpu->arch.iamr = set_reg_val(id, *val); 1552 break; 1553 case KVM_REG_PPC_PSPB: 1554 vcpu->arch.pspb = set_reg_val(id, *val); 1555 break; 1556 case KVM_REG_PPC_DPDES: 1557 vcpu->arch.vcore->dpdes = set_reg_val(id, *val); 1558 break; 1559 case KVM_REG_PPC_VTB: 1560 vcpu->arch.vcore->vtb = set_reg_val(id, *val); 1561 break; 1562 case KVM_REG_PPC_DAWR: 1563 vcpu->arch.dawr = set_reg_val(id, *val); 1564 break; 1565 case KVM_REG_PPC_DAWRX: 1566 vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP; 1567 break; 1568 case KVM_REG_PPC_CIABR: 1569 vcpu->arch.ciabr = set_reg_val(id, *val); 1570 /* Don't allow setting breakpoints in hypervisor code */ 1571 if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER) 1572 vcpu->arch.ciabr &= ~CIABR_PRIV; /* disable */ 1573 break; 1574 case KVM_REG_PPC_CSIGR: 1575 vcpu->arch.csigr = set_reg_val(id, *val); 1576 break; 1577 case KVM_REG_PPC_TACR: 1578 vcpu->arch.tacr = set_reg_val(id, *val); 1579 break; 1580 case KVM_REG_PPC_TCSCR: 1581 vcpu->arch.tcscr = set_reg_val(id, *val); 1582 break; 1583 case KVM_REG_PPC_PID: 1584 vcpu->arch.pid = set_reg_val(id, *val); 1585 break; 1586 case KVM_REG_PPC_ACOP: 1587 vcpu->arch.acop = set_reg_val(id, *val); 1588 break; 1589 case KVM_REG_PPC_WORT: 1590 vcpu->arch.wort = set_reg_val(id, *val); 1591 break; 1592 case KVM_REG_PPC_TIDR: 1593 vcpu->arch.tid = set_reg_val(id, *val); 1594 break; 1595 case KVM_REG_PPC_PSSCR: 1596 vcpu->arch.psscr = set_reg_val(id, *val) & PSSCR_GUEST_VIS; 1597 break; 1598 case KVM_REG_PPC_VPA_ADDR: 1599 addr = set_reg_val(id, *val); 1600 r = -EINVAL; 1601 if (!addr && (vcpu->arch.slb_shadow.next_gpa || 1602 vcpu->arch.dtl.next_gpa)) 1603 break; 1604 r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca)); 1605 break; 1606 case KVM_REG_PPC_VPA_SLB: 1607 addr = val->vpaval.addr; 1608 len = val->vpaval.length; 1609 r = -EINVAL; 1610 if (addr && !vcpu->arch.vpa.next_gpa) 1611 break; 1612 r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len); 1613 break; 1614 case KVM_REG_PPC_VPA_DTL: 1615 addr = val->vpaval.addr; 1616 len = val->vpaval.length; 1617 r = -EINVAL; 1618 if (addr && (len < sizeof(struct dtl_entry) || 1619 !vcpu->arch.vpa.next_gpa)) 1620 break; 1621 len -= len % sizeof(struct dtl_entry); 1622 r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len); 1623 break; 1624 case KVM_REG_PPC_TB_OFFSET: 1625 /* 1626 * POWER9 DD1 has an erratum where writing TBU40 causes 1627 * the timebase to lose ticks. So we don't let the 1628 * timebase offset be changed on P9 DD1. (It is 1629 * initialized to zero.) 1630 */ 1631 if (cpu_has_feature(CPU_FTR_POWER9_DD1)) 1632 break; 1633 /* round up to multiple of 2^24 */ 1634 vcpu->arch.vcore->tb_offset = 1635 ALIGN(set_reg_val(id, *val), 1UL << 24); 1636 break; 1637 case KVM_REG_PPC_LPCR: 1638 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true); 1639 break; 1640 case KVM_REG_PPC_LPCR_64: 1641 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false); 1642 break; 1643 case KVM_REG_PPC_PPR: 1644 vcpu->arch.ppr = set_reg_val(id, *val); 1645 break; 1646 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1647 case KVM_REG_PPC_TFHAR: 1648 vcpu->arch.tfhar = set_reg_val(id, *val); 1649 break; 1650 case KVM_REG_PPC_TFIAR: 1651 vcpu->arch.tfiar = set_reg_val(id, *val); 1652 break; 1653 case KVM_REG_PPC_TEXASR: 1654 vcpu->arch.texasr = set_reg_val(id, *val); 1655 break; 1656 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31: 1657 i = id - KVM_REG_PPC_TM_GPR0; 1658 vcpu->arch.gpr_tm[i] = set_reg_val(id, *val); 1659 break; 1660 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63: 1661 { 1662 int j; 1663 i = id - KVM_REG_PPC_TM_VSR0; 1664 if (i < 32) 1665 for (j = 0; j < TS_FPRWIDTH; j++) 1666 vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j]; 1667 else 1668 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1669 vcpu->arch.vr_tm.vr[i-32] = val->vval; 1670 else 1671 r = -ENXIO; 1672 break; 1673 } 1674 case KVM_REG_PPC_TM_CR: 1675 vcpu->arch.cr_tm = set_reg_val(id, *val); 1676 break; 1677 case KVM_REG_PPC_TM_XER: 1678 vcpu->arch.xer_tm = set_reg_val(id, *val); 1679 break; 1680 case KVM_REG_PPC_TM_LR: 1681 vcpu->arch.lr_tm = set_reg_val(id, *val); 1682 break; 1683 case KVM_REG_PPC_TM_CTR: 1684 vcpu->arch.ctr_tm = set_reg_val(id, *val); 1685 break; 1686 case KVM_REG_PPC_TM_FPSCR: 1687 vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val); 1688 break; 1689 case KVM_REG_PPC_TM_AMR: 1690 vcpu->arch.amr_tm = set_reg_val(id, *val); 1691 break; 1692 case KVM_REG_PPC_TM_PPR: 1693 vcpu->arch.ppr_tm = set_reg_val(id, *val); 1694 break; 1695 case KVM_REG_PPC_TM_VRSAVE: 1696 vcpu->arch.vrsave_tm = set_reg_val(id, *val); 1697 break; 1698 case KVM_REG_PPC_TM_VSCR: 1699 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1700 vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val); 1701 else 1702 r = - ENXIO; 1703 break; 1704 case KVM_REG_PPC_TM_DSCR: 1705 vcpu->arch.dscr_tm = set_reg_val(id, *val); 1706 break; 1707 case KVM_REG_PPC_TM_TAR: 1708 vcpu->arch.tar_tm = set_reg_val(id, *val); 1709 break; 1710 #endif 1711 case KVM_REG_PPC_ARCH_COMPAT: 1712 r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val)); 1713 break; 1714 default: 1715 r = -EINVAL; 1716 break; 1717 } 1718 1719 return r; 1720 } 1721 1722 /* 1723 * On POWER9, threads are independent and can be in different partitions. 1724 * Therefore we consider each thread to be a subcore. 1725 * There is a restriction that all threads have to be in the same 1726 * MMU mode (radix or HPT), unfortunately, but since we only support 1727 * HPT guests on a HPT host so far, that isn't an impediment yet. 1728 */ 1729 static int threads_per_vcore(void) 1730 { 1731 if (cpu_has_feature(CPU_FTR_ARCH_300)) 1732 return 1; 1733 return threads_per_subcore; 1734 } 1735 1736 static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core) 1737 { 1738 struct kvmppc_vcore *vcore; 1739 1740 vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL); 1741 1742 if (vcore == NULL) 1743 return NULL; 1744 1745 spin_lock_init(&vcore->lock); 1746 spin_lock_init(&vcore->stoltb_lock); 1747 init_swait_queue_head(&vcore->wq); 1748 vcore->preempt_tb = TB_NIL; 1749 vcore->lpcr = kvm->arch.lpcr; 1750 vcore->first_vcpuid = core * kvm->arch.smt_mode; 1751 vcore->kvm = kvm; 1752 INIT_LIST_HEAD(&vcore->preempt_list); 1753 1754 return vcore; 1755 } 1756 1757 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1758 static struct debugfs_timings_element { 1759 const char *name; 1760 size_t offset; 1761 } timings[] = { 1762 {"rm_entry", offsetof(struct kvm_vcpu, arch.rm_entry)}, 1763 {"rm_intr", offsetof(struct kvm_vcpu, arch.rm_intr)}, 1764 {"rm_exit", offsetof(struct kvm_vcpu, arch.rm_exit)}, 1765 {"guest", offsetof(struct kvm_vcpu, arch.guest_time)}, 1766 {"cede", offsetof(struct kvm_vcpu, arch.cede_time)}, 1767 }; 1768 1769 #define N_TIMINGS (sizeof(timings) / sizeof(timings[0])) 1770 1771 struct debugfs_timings_state { 1772 struct kvm_vcpu *vcpu; 1773 unsigned int buflen; 1774 char buf[N_TIMINGS * 100]; 1775 }; 1776 1777 static int debugfs_timings_open(struct inode *inode, struct file *file) 1778 { 1779 struct kvm_vcpu *vcpu = inode->i_private; 1780 struct debugfs_timings_state *p; 1781 1782 p = kzalloc(sizeof(*p), GFP_KERNEL); 1783 if (!p) 1784 return -ENOMEM; 1785 1786 kvm_get_kvm(vcpu->kvm); 1787 p->vcpu = vcpu; 1788 file->private_data = p; 1789 1790 return nonseekable_open(inode, file); 1791 } 1792 1793 static int debugfs_timings_release(struct inode *inode, struct file *file) 1794 { 1795 struct debugfs_timings_state *p = file->private_data; 1796 1797 kvm_put_kvm(p->vcpu->kvm); 1798 kfree(p); 1799 return 0; 1800 } 1801 1802 static ssize_t debugfs_timings_read(struct file *file, char __user *buf, 1803 size_t len, loff_t *ppos) 1804 { 1805 struct debugfs_timings_state *p = file->private_data; 1806 struct kvm_vcpu *vcpu = p->vcpu; 1807 char *s, *buf_end; 1808 struct kvmhv_tb_accumulator tb; 1809 u64 count; 1810 loff_t pos; 1811 ssize_t n; 1812 int i, loops; 1813 bool ok; 1814 1815 if (!p->buflen) { 1816 s = p->buf; 1817 buf_end = s + sizeof(p->buf); 1818 for (i = 0; i < N_TIMINGS; ++i) { 1819 struct kvmhv_tb_accumulator *acc; 1820 1821 acc = (struct kvmhv_tb_accumulator *) 1822 ((unsigned long)vcpu + timings[i].offset); 1823 ok = false; 1824 for (loops = 0; loops < 1000; ++loops) { 1825 count = acc->seqcount; 1826 if (!(count & 1)) { 1827 smp_rmb(); 1828 tb = *acc; 1829 smp_rmb(); 1830 if (count == acc->seqcount) { 1831 ok = true; 1832 break; 1833 } 1834 } 1835 udelay(1); 1836 } 1837 if (!ok) 1838 snprintf(s, buf_end - s, "%s: stuck\n", 1839 timings[i].name); 1840 else 1841 snprintf(s, buf_end - s, 1842 "%s: %llu %llu %llu %llu\n", 1843 timings[i].name, count / 2, 1844 tb_to_ns(tb.tb_total), 1845 tb_to_ns(tb.tb_min), 1846 tb_to_ns(tb.tb_max)); 1847 s += strlen(s); 1848 } 1849 p->buflen = s - p->buf; 1850 } 1851 1852 pos = *ppos; 1853 if (pos >= p->buflen) 1854 return 0; 1855 if (len > p->buflen - pos) 1856 len = p->buflen - pos; 1857 n = copy_to_user(buf, p->buf + pos, len); 1858 if (n) { 1859 if (n == len) 1860 return -EFAULT; 1861 len -= n; 1862 } 1863 *ppos = pos + len; 1864 return len; 1865 } 1866 1867 static ssize_t debugfs_timings_write(struct file *file, const char __user *buf, 1868 size_t len, loff_t *ppos) 1869 { 1870 return -EACCES; 1871 } 1872 1873 static const struct file_operations debugfs_timings_ops = { 1874 .owner = THIS_MODULE, 1875 .open = debugfs_timings_open, 1876 .release = debugfs_timings_release, 1877 .read = debugfs_timings_read, 1878 .write = debugfs_timings_write, 1879 .llseek = generic_file_llseek, 1880 }; 1881 1882 /* Create a debugfs directory for the vcpu */ 1883 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id) 1884 { 1885 char buf[16]; 1886 struct kvm *kvm = vcpu->kvm; 1887 1888 snprintf(buf, sizeof(buf), "vcpu%u", id); 1889 if (IS_ERR_OR_NULL(kvm->arch.debugfs_dir)) 1890 return; 1891 vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir); 1892 if (IS_ERR_OR_NULL(vcpu->arch.debugfs_dir)) 1893 return; 1894 vcpu->arch.debugfs_timings = 1895 debugfs_create_file("timings", 0444, vcpu->arch.debugfs_dir, 1896 vcpu, &debugfs_timings_ops); 1897 } 1898 1899 #else /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */ 1900 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id) 1901 { 1902 } 1903 #endif /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */ 1904 1905 static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm, 1906 unsigned int id) 1907 { 1908 struct kvm_vcpu *vcpu; 1909 int err; 1910 int core; 1911 struct kvmppc_vcore *vcore; 1912 1913 err = -ENOMEM; 1914 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); 1915 if (!vcpu) 1916 goto out; 1917 1918 err = kvm_vcpu_init(vcpu, kvm, id); 1919 if (err) 1920 goto free_vcpu; 1921 1922 vcpu->arch.shared = &vcpu->arch.shregs; 1923 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1924 /* 1925 * The shared struct is never shared on HV, 1926 * so we can always use host endianness 1927 */ 1928 #ifdef __BIG_ENDIAN__ 1929 vcpu->arch.shared_big_endian = true; 1930 #else 1931 vcpu->arch.shared_big_endian = false; 1932 #endif 1933 #endif 1934 vcpu->arch.mmcr[0] = MMCR0_FC; 1935 vcpu->arch.ctrl = CTRL_RUNLATCH; 1936 /* default to host PVR, since we can't spoof it */ 1937 kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR)); 1938 spin_lock_init(&vcpu->arch.vpa_update_lock); 1939 spin_lock_init(&vcpu->arch.tbacct_lock); 1940 vcpu->arch.busy_preempt = TB_NIL; 1941 vcpu->arch.intr_msr = MSR_SF | MSR_ME; 1942 1943 /* 1944 * Set the default HFSCR for the guest from the host value. 1945 * This value is only used on POWER9. 1946 * On POWER9 DD1, TM doesn't work, so we make sure to 1947 * prevent the guest from using it. 1948 * On POWER9, we want to virtualize the doorbell facility, so we 1949 * turn off the HFSCR bit, which causes those instructions to trap. 1950 */ 1951 vcpu->arch.hfscr = mfspr(SPRN_HFSCR); 1952 if (!cpu_has_feature(CPU_FTR_TM)) 1953 vcpu->arch.hfscr &= ~HFSCR_TM; 1954 if (cpu_has_feature(CPU_FTR_ARCH_300)) 1955 vcpu->arch.hfscr &= ~HFSCR_MSGP; 1956 1957 kvmppc_mmu_book3s_hv_init(vcpu); 1958 1959 vcpu->arch.state = KVMPPC_VCPU_NOTREADY; 1960 1961 init_waitqueue_head(&vcpu->arch.cpu_run); 1962 1963 mutex_lock(&kvm->lock); 1964 vcore = NULL; 1965 err = -EINVAL; 1966 core = id / kvm->arch.smt_mode; 1967 if (core < KVM_MAX_VCORES) { 1968 vcore = kvm->arch.vcores[core]; 1969 if (!vcore) { 1970 err = -ENOMEM; 1971 vcore = kvmppc_vcore_create(kvm, core); 1972 kvm->arch.vcores[core] = vcore; 1973 kvm->arch.online_vcores++; 1974 } 1975 } 1976 mutex_unlock(&kvm->lock); 1977 1978 if (!vcore) 1979 goto free_vcpu; 1980 1981 spin_lock(&vcore->lock); 1982 ++vcore->num_threads; 1983 spin_unlock(&vcore->lock); 1984 vcpu->arch.vcore = vcore; 1985 vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid; 1986 vcpu->arch.thread_cpu = -1; 1987 vcpu->arch.prev_cpu = -1; 1988 1989 vcpu->arch.cpu_type = KVM_CPU_3S_64; 1990 kvmppc_sanity_check(vcpu); 1991 1992 debugfs_vcpu_init(vcpu, id); 1993 1994 return vcpu; 1995 1996 free_vcpu: 1997 kmem_cache_free(kvm_vcpu_cache, vcpu); 1998 out: 1999 return ERR_PTR(err); 2000 } 2001 2002 static int kvmhv_set_smt_mode(struct kvm *kvm, unsigned long smt_mode, 2003 unsigned long flags) 2004 { 2005 int err; 2006 int esmt = 0; 2007 2008 if (flags) 2009 return -EINVAL; 2010 if (smt_mode > MAX_SMT_THREADS || !is_power_of_2(smt_mode)) 2011 return -EINVAL; 2012 if (!cpu_has_feature(CPU_FTR_ARCH_300)) { 2013 /* 2014 * On POWER8 (or POWER7), the threading mode is "strict", 2015 * so we pack smt_mode vcpus per vcore. 2016 */ 2017 if (smt_mode > threads_per_subcore) 2018 return -EINVAL; 2019 } else { 2020 /* 2021 * On POWER9, the threading mode is "loose", 2022 * so each vcpu gets its own vcore. 2023 */ 2024 esmt = smt_mode; 2025 smt_mode = 1; 2026 } 2027 mutex_lock(&kvm->lock); 2028 err = -EBUSY; 2029 if (!kvm->arch.online_vcores) { 2030 kvm->arch.smt_mode = smt_mode; 2031 kvm->arch.emul_smt_mode = esmt; 2032 err = 0; 2033 } 2034 mutex_unlock(&kvm->lock); 2035 2036 return err; 2037 } 2038 2039 static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa) 2040 { 2041 if (vpa->pinned_addr) 2042 kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa, 2043 vpa->dirty); 2044 } 2045 2046 static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu) 2047 { 2048 spin_lock(&vcpu->arch.vpa_update_lock); 2049 unpin_vpa(vcpu->kvm, &vcpu->arch.dtl); 2050 unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow); 2051 unpin_vpa(vcpu->kvm, &vcpu->arch.vpa); 2052 spin_unlock(&vcpu->arch.vpa_update_lock); 2053 kvm_vcpu_uninit(vcpu); 2054 kmem_cache_free(kvm_vcpu_cache, vcpu); 2055 } 2056 2057 static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu) 2058 { 2059 /* Indicate we want to get back into the guest */ 2060 return 1; 2061 } 2062 2063 static void kvmppc_set_timer(struct kvm_vcpu *vcpu) 2064 { 2065 unsigned long dec_nsec, now; 2066 2067 now = get_tb(); 2068 if (now > vcpu->arch.dec_expires) { 2069 /* decrementer has already gone negative */ 2070 kvmppc_core_queue_dec(vcpu); 2071 kvmppc_core_prepare_to_enter(vcpu); 2072 return; 2073 } 2074 dec_nsec = (vcpu->arch.dec_expires - now) * NSEC_PER_SEC 2075 / tb_ticks_per_sec; 2076 hrtimer_start(&vcpu->arch.dec_timer, dec_nsec, HRTIMER_MODE_REL); 2077 vcpu->arch.timer_running = 1; 2078 } 2079 2080 static void kvmppc_end_cede(struct kvm_vcpu *vcpu) 2081 { 2082 vcpu->arch.ceded = 0; 2083 if (vcpu->arch.timer_running) { 2084 hrtimer_try_to_cancel(&vcpu->arch.dec_timer); 2085 vcpu->arch.timer_running = 0; 2086 } 2087 } 2088 2089 extern int __kvmppc_vcore_entry(void); 2090 2091 static void kvmppc_remove_runnable(struct kvmppc_vcore *vc, 2092 struct kvm_vcpu *vcpu) 2093 { 2094 u64 now; 2095 2096 if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE) 2097 return; 2098 spin_lock_irq(&vcpu->arch.tbacct_lock); 2099 now = mftb(); 2100 vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) - 2101 vcpu->arch.stolen_logged; 2102 vcpu->arch.busy_preempt = now; 2103 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST; 2104 spin_unlock_irq(&vcpu->arch.tbacct_lock); 2105 --vc->n_runnable; 2106 WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], NULL); 2107 } 2108 2109 static int kvmppc_grab_hwthread(int cpu) 2110 { 2111 struct paca_struct *tpaca; 2112 long timeout = 10000; 2113 2114 tpaca = &paca[cpu]; 2115 2116 /* Ensure the thread won't go into the kernel if it wakes */ 2117 tpaca->kvm_hstate.kvm_vcpu = NULL; 2118 tpaca->kvm_hstate.kvm_vcore = NULL; 2119 tpaca->kvm_hstate.napping = 0; 2120 smp_wmb(); 2121 tpaca->kvm_hstate.hwthread_req = 1; 2122 2123 /* 2124 * If the thread is already executing in the kernel (e.g. handling 2125 * a stray interrupt), wait for it to get back to nap mode. 2126 * The smp_mb() is to ensure that our setting of hwthread_req 2127 * is visible before we look at hwthread_state, so if this 2128 * races with the code at system_reset_pSeries and the thread 2129 * misses our setting of hwthread_req, we are sure to see its 2130 * setting of hwthread_state, and vice versa. 2131 */ 2132 smp_mb(); 2133 while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) { 2134 if (--timeout <= 0) { 2135 pr_err("KVM: couldn't grab cpu %d\n", cpu); 2136 return -EBUSY; 2137 } 2138 udelay(1); 2139 } 2140 return 0; 2141 } 2142 2143 static void kvmppc_release_hwthread(int cpu) 2144 { 2145 struct paca_struct *tpaca; 2146 2147 tpaca = &paca[cpu]; 2148 tpaca->kvm_hstate.hwthread_req = 0; 2149 tpaca->kvm_hstate.kvm_vcpu = NULL; 2150 tpaca->kvm_hstate.kvm_vcore = NULL; 2151 tpaca->kvm_hstate.kvm_split_mode = NULL; 2152 } 2153 2154 static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu) 2155 { 2156 int i; 2157 2158 cpu = cpu_first_thread_sibling(cpu); 2159 cpumask_set_cpu(cpu, &kvm->arch.need_tlb_flush); 2160 /* 2161 * Make sure setting of bit in need_tlb_flush precedes 2162 * testing of cpu_in_guest bits. The matching barrier on 2163 * the other side is the first smp_mb() in kvmppc_run_core(). 2164 */ 2165 smp_mb(); 2166 for (i = 0; i < threads_per_core; ++i) 2167 if (cpumask_test_cpu(cpu + i, &kvm->arch.cpu_in_guest)) 2168 smp_call_function_single(cpu + i, do_nothing, NULL, 1); 2169 } 2170 2171 static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu) 2172 { 2173 struct kvm *kvm = vcpu->kvm; 2174 2175 /* 2176 * With radix, the guest can do TLB invalidations itself, 2177 * and it could choose to use the local form (tlbiel) if 2178 * it is invalidating a translation that has only ever been 2179 * used on one vcpu. However, that doesn't mean it has 2180 * only ever been used on one physical cpu, since vcpus 2181 * can move around between pcpus. To cope with this, when 2182 * a vcpu moves from one pcpu to another, we need to tell 2183 * any vcpus running on the same core as this vcpu previously 2184 * ran to flush the TLB. The TLB is shared between threads, 2185 * so we use a single bit in .need_tlb_flush for all 4 threads. 2186 */ 2187 if (vcpu->arch.prev_cpu != pcpu) { 2188 if (vcpu->arch.prev_cpu >= 0 && 2189 cpu_first_thread_sibling(vcpu->arch.prev_cpu) != 2190 cpu_first_thread_sibling(pcpu)) 2191 radix_flush_cpu(kvm, vcpu->arch.prev_cpu, vcpu); 2192 vcpu->arch.prev_cpu = pcpu; 2193 } 2194 } 2195 2196 static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc) 2197 { 2198 int cpu; 2199 struct paca_struct *tpaca; 2200 struct kvm *kvm = vc->kvm; 2201 2202 cpu = vc->pcpu; 2203 if (vcpu) { 2204 if (vcpu->arch.timer_running) { 2205 hrtimer_try_to_cancel(&vcpu->arch.dec_timer); 2206 vcpu->arch.timer_running = 0; 2207 } 2208 cpu += vcpu->arch.ptid; 2209 vcpu->cpu = vc->pcpu; 2210 vcpu->arch.thread_cpu = cpu; 2211 cpumask_set_cpu(cpu, &kvm->arch.cpu_in_guest); 2212 } 2213 tpaca = &paca[cpu]; 2214 tpaca->kvm_hstate.kvm_vcpu = vcpu; 2215 tpaca->kvm_hstate.ptid = cpu - vc->pcpu; 2216 /* Order stores to hstate.kvm_vcpu etc. before store to kvm_vcore */ 2217 smp_wmb(); 2218 tpaca->kvm_hstate.kvm_vcore = vc; 2219 if (cpu != smp_processor_id()) 2220 kvmppc_ipi_thread(cpu); 2221 } 2222 2223 static void kvmppc_wait_for_nap(void) 2224 { 2225 int cpu = smp_processor_id(); 2226 int i, loops; 2227 int n_threads = threads_per_vcore(); 2228 2229 if (n_threads <= 1) 2230 return; 2231 for (loops = 0; loops < 1000000; ++loops) { 2232 /* 2233 * Check if all threads are finished. 2234 * We set the vcore pointer when starting a thread 2235 * and the thread clears it when finished, so we look 2236 * for any threads that still have a non-NULL vcore ptr. 2237 */ 2238 for (i = 1; i < n_threads; ++i) 2239 if (paca[cpu + i].kvm_hstate.kvm_vcore) 2240 break; 2241 if (i == n_threads) { 2242 HMT_medium(); 2243 return; 2244 } 2245 HMT_low(); 2246 } 2247 HMT_medium(); 2248 for (i = 1; i < n_threads; ++i) 2249 if (paca[cpu + i].kvm_hstate.kvm_vcore) 2250 pr_err("KVM: CPU %d seems to be stuck\n", cpu + i); 2251 } 2252 2253 /* 2254 * Check that we are on thread 0 and that any other threads in 2255 * this core are off-line. Then grab the threads so they can't 2256 * enter the kernel. 2257 */ 2258 static int on_primary_thread(void) 2259 { 2260 int cpu = smp_processor_id(); 2261 int thr; 2262 2263 /* Are we on a primary subcore? */ 2264 if (cpu_thread_in_subcore(cpu)) 2265 return 0; 2266 2267 thr = 0; 2268 while (++thr < threads_per_subcore) 2269 if (cpu_online(cpu + thr)) 2270 return 0; 2271 2272 /* Grab all hw threads so they can't go into the kernel */ 2273 for (thr = 1; thr < threads_per_subcore; ++thr) { 2274 if (kvmppc_grab_hwthread(cpu + thr)) { 2275 /* Couldn't grab one; let the others go */ 2276 do { 2277 kvmppc_release_hwthread(cpu + thr); 2278 } while (--thr > 0); 2279 return 0; 2280 } 2281 } 2282 return 1; 2283 } 2284 2285 /* 2286 * A list of virtual cores for each physical CPU. 2287 * These are vcores that could run but their runner VCPU tasks are 2288 * (or may be) preempted. 2289 */ 2290 struct preempted_vcore_list { 2291 struct list_head list; 2292 spinlock_t lock; 2293 }; 2294 2295 static DEFINE_PER_CPU(struct preempted_vcore_list, preempted_vcores); 2296 2297 static void init_vcore_lists(void) 2298 { 2299 int cpu; 2300 2301 for_each_possible_cpu(cpu) { 2302 struct preempted_vcore_list *lp = &per_cpu(preempted_vcores, cpu); 2303 spin_lock_init(&lp->lock); 2304 INIT_LIST_HEAD(&lp->list); 2305 } 2306 } 2307 2308 static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc) 2309 { 2310 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores); 2311 2312 vc->vcore_state = VCORE_PREEMPT; 2313 vc->pcpu = smp_processor_id(); 2314 if (vc->num_threads < threads_per_vcore()) { 2315 spin_lock(&lp->lock); 2316 list_add_tail(&vc->preempt_list, &lp->list); 2317 spin_unlock(&lp->lock); 2318 } 2319 2320 /* Start accumulating stolen time */ 2321 kvmppc_core_start_stolen(vc); 2322 } 2323 2324 static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc) 2325 { 2326 struct preempted_vcore_list *lp; 2327 2328 kvmppc_core_end_stolen(vc); 2329 if (!list_empty(&vc->preempt_list)) { 2330 lp = &per_cpu(preempted_vcores, vc->pcpu); 2331 spin_lock(&lp->lock); 2332 list_del_init(&vc->preempt_list); 2333 spin_unlock(&lp->lock); 2334 } 2335 vc->vcore_state = VCORE_INACTIVE; 2336 } 2337 2338 /* 2339 * This stores information about the virtual cores currently 2340 * assigned to a physical core. 2341 */ 2342 struct core_info { 2343 int n_subcores; 2344 int max_subcore_threads; 2345 int total_threads; 2346 int subcore_threads[MAX_SUBCORES]; 2347 struct kvmppc_vcore *vc[MAX_SUBCORES]; 2348 }; 2349 2350 /* 2351 * This mapping means subcores 0 and 1 can use threads 0-3 and 4-7 2352 * respectively in 2-way micro-threading (split-core) mode. 2353 */ 2354 static int subcore_thread_map[MAX_SUBCORES] = { 0, 4, 2, 6 }; 2355 2356 static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc) 2357 { 2358 memset(cip, 0, sizeof(*cip)); 2359 cip->n_subcores = 1; 2360 cip->max_subcore_threads = vc->num_threads; 2361 cip->total_threads = vc->num_threads; 2362 cip->subcore_threads[0] = vc->num_threads; 2363 cip->vc[0] = vc; 2364 } 2365 2366 static bool subcore_config_ok(int n_subcores, int n_threads) 2367 { 2368 /* Can only dynamically split if unsplit to begin with */ 2369 if (n_subcores > 1 && threads_per_subcore < MAX_SMT_THREADS) 2370 return false; 2371 if (n_subcores > MAX_SUBCORES) 2372 return false; 2373 if (n_subcores > 1) { 2374 if (!(dynamic_mt_modes & 2)) 2375 n_subcores = 4; 2376 if (n_subcores > 2 && !(dynamic_mt_modes & 4)) 2377 return false; 2378 } 2379 2380 return n_subcores * roundup_pow_of_two(n_threads) <= MAX_SMT_THREADS; 2381 } 2382 2383 static void init_vcore_to_run(struct kvmppc_vcore *vc) 2384 { 2385 vc->entry_exit_map = 0; 2386 vc->in_guest = 0; 2387 vc->napping_threads = 0; 2388 vc->conferring_threads = 0; 2389 } 2390 2391 static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip) 2392 { 2393 int n_threads = vc->num_threads; 2394 int sub; 2395 2396 if (!cpu_has_feature(CPU_FTR_ARCH_207S)) 2397 return false; 2398 2399 if (n_threads < cip->max_subcore_threads) 2400 n_threads = cip->max_subcore_threads; 2401 if (!subcore_config_ok(cip->n_subcores + 1, n_threads)) 2402 return false; 2403 cip->max_subcore_threads = n_threads; 2404 2405 sub = cip->n_subcores; 2406 ++cip->n_subcores; 2407 cip->total_threads += vc->num_threads; 2408 cip->subcore_threads[sub] = vc->num_threads; 2409 cip->vc[sub] = vc; 2410 init_vcore_to_run(vc); 2411 list_del_init(&vc->preempt_list); 2412 2413 return true; 2414 } 2415 2416 /* 2417 * Work out whether it is possible to piggyback the execution of 2418 * vcore *pvc onto the execution of the other vcores described in *cip. 2419 */ 2420 static bool can_piggyback(struct kvmppc_vcore *pvc, struct core_info *cip, 2421 int target_threads) 2422 { 2423 if (cip->total_threads + pvc->num_threads > target_threads) 2424 return false; 2425 2426 return can_dynamic_split(pvc, cip); 2427 } 2428 2429 static void prepare_threads(struct kvmppc_vcore *vc) 2430 { 2431 int i; 2432 struct kvm_vcpu *vcpu; 2433 2434 for_each_runnable_thread(i, vcpu, vc) { 2435 if (signal_pending(vcpu->arch.run_task)) 2436 vcpu->arch.ret = -EINTR; 2437 else if (vcpu->arch.vpa.update_pending || 2438 vcpu->arch.slb_shadow.update_pending || 2439 vcpu->arch.dtl.update_pending) 2440 vcpu->arch.ret = RESUME_GUEST; 2441 else 2442 continue; 2443 kvmppc_remove_runnable(vc, vcpu); 2444 wake_up(&vcpu->arch.cpu_run); 2445 } 2446 } 2447 2448 static void collect_piggybacks(struct core_info *cip, int target_threads) 2449 { 2450 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores); 2451 struct kvmppc_vcore *pvc, *vcnext; 2452 2453 spin_lock(&lp->lock); 2454 list_for_each_entry_safe(pvc, vcnext, &lp->list, preempt_list) { 2455 if (!spin_trylock(&pvc->lock)) 2456 continue; 2457 prepare_threads(pvc); 2458 if (!pvc->n_runnable) { 2459 list_del_init(&pvc->preempt_list); 2460 if (pvc->runner == NULL) { 2461 pvc->vcore_state = VCORE_INACTIVE; 2462 kvmppc_core_end_stolen(pvc); 2463 } 2464 spin_unlock(&pvc->lock); 2465 continue; 2466 } 2467 if (!can_piggyback(pvc, cip, target_threads)) { 2468 spin_unlock(&pvc->lock); 2469 continue; 2470 } 2471 kvmppc_core_end_stolen(pvc); 2472 pvc->vcore_state = VCORE_PIGGYBACK; 2473 if (cip->total_threads >= target_threads) 2474 break; 2475 } 2476 spin_unlock(&lp->lock); 2477 } 2478 2479 static bool recheck_signals(struct core_info *cip) 2480 { 2481 int sub, i; 2482 struct kvm_vcpu *vcpu; 2483 2484 for (sub = 0; sub < cip->n_subcores; ++sub) 2485 for_each_runnable_thread(i, vcpu, cip->vc[sub]) 2486 if (signal_pending(vcpu->arch.run_task)) 2487 return true; 2488 return false; 2489 } 2490 2491 static void post_guest_process(struct kvmppc_vcore *vc, bool is_master) 2492 { 2493 int still_running = 0, i; 2494 u64 now; 2495 long ret; 2496 struct kvm_vcpu *vcpu; 2497 2498 spin_lock(&vc->lock); 2499 now = get_tb(); 2500 for_each_runnable_thread(i, vcpu, vc) { 2501 /* cancel pending dec exception if dec is positive */ 2502 if (now < vcpu->arch.dec_expires && 2503 kvmppc_core_pending_dec(vcpu)) 2504 kvmppc_core_dequeue_dec(vcpu); 2505 2506 trace_kvm_guest_exit(vcpu); 2507 2508 ret = RESUME_GUEST; 2509 if (vcpu->arch.trap) 2510 ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu, 2511 vcpu->arch.run_task); 2512 2513 vcpu->arch.ret = ret; 2514 vcpu->arch.trap = 0; 2515 2516 if (is_kvmppc_resume_guest(vcpu->arch.ret)) { 2517 if (vcpu->arch.pending_exceptions) 2518 kvmppc_core_prepare_to_enter(vcpu); 2519 if (vcpu->arch.ceded) 2520 kvmppc_set_timer(vcpu); 2521 else 2522 ++still_running; 2523 } else { 2524 kvmppc_remove_runnable(vc, vcpu); 2525 wake_up(&vcpu->arch.cpu_run); 2526 } 2527 } 2528 if (!is_master) { 2529 if (still_running > 0) { 2530 kvmppc_vcore_preempt(vc); 2531 } else if (vc->runner) { 2532 vc->vcore_state = VCORE_PREEMPT; 2533 kvmppc_core_start_stolen(vc); 2534 } else { 2535 vc->vcore_state = VCORE_INACTIVE; 2536 } 2537 if (vc->n_runnable > 0 && vc->runner == NULL) { 2538 /* make sure there's a candidate runner awake */ 2539 i = -1; 2540 vcpu = next_runnable_thread(vc, &i); 2541 wake_up(&vcpu->arch.cpu_run); 2542 } 2543 } 2544 spin_unlock(&vc->lock); 2545 } 2546 2547 /* 2548 * Clear core from the list of active host cores as we are about to 2549 * enter the guest. Only do this if it is the primary thread of the 2550 * core (not if a subcore) that is entering the guest. 2551 */ 2552 static inline int kvmppc_clear_host_core(unsigned int cpu) 2553 { 2554 int core; 2555 2556 if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu)) 2557 return 0; 2558 /* 2559 * Memory barrier can be omitted here as we will do a smp_wmb() 2560 * later in kvmppc_start_thread and we need ensure that state is 2561 * visible to other CPUs only after we enter guest. 2562 */ 2563 core = cpu >> threads_shift; 2564 kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 0; 2565 return 0; 2566 } 2567 2568 /* 2569 * Advertise this core as an active host core since we exited the guest 2570 * Only need to do this if it is the primary thread of the core that is 2571 * exiting. 2572 */ 2573 static inline int kvmppc_set_host_core(unsigned int cpu) 2574 { 2575 int core; 2576 2577 if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu)) 2578 return 0; 2579 2580 /* 2581 * Memory barrier can be omitted here because we do a spin_unlock 2582 * immediately after this which provides the memory barrier. 2583 */ 2584 core = cpu >> threads_shift; 2585 kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 1; 2586 return 0; 2587 } 2588 2589 static void set_irq_happened(int trap) 2590 { 2591 switch (trap) { 2592 case BOOK3S_INTERRUPT_EXTERNAL: 2593 local_paca->irq_happened |= PACA_IRQ_EE; 2594 break; 2595 case BOOK3S_INTERRUPT_H_DOORBELL: 2596 local_paca->irq_happened |= PACA_IRQ_DBELL; 2597 break; 2598 case BOOK3S_INTERRUPT_HMI: 2599 local_paca->irq_happened |= PACA_IRQ_HMI; 2600 break; 2601 } 2602 } 2603 2604 /* 2605 * Run a set of guest threads on a physical core. 2606 * Called with vc->lock held. 2607 */ 2608 static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) 2609 { 2610 struct kvm_vcpu *vcpu; 2611 int i; 2612 int srcu_idx; 2613 struct core_info core_info; 2614 struct kvmppc_vcore *pvc; 2615 struct kvm_split_mode split_info, *sip; 2616 int split, subcore_size, active; 2617 int sub; 2618 bool thr0_done; 2619 unsigned long cmd_bit, stat_bit; 2620 int pcpu, thr; 2621 int target_threads; 2622 int controlled_threads; 2623 int trap; 2624 2625 /* 2626 * Remove from the list any threads that have a signal pending 2627 * or need a VPA update done 2628 */ 2629 prepare_threads(vc); 2630 2631 /* if the runner is no longer runnable, let the caller pick a new one */ 2632 if (vc->runner->arch.state != KVMPPC_VCPU_RUNNABLE) 2633 return; 2634 2635 /* 2636 * Initialize *vc. 2637 */ 2638 init_vcore_to_run(vc); 2639 vc->preempt_tb = TB_NIL; 2640 2641 /* 2642 * Number of threads that we will be controlling: the same as 2643 * the number of threads per subcore, except on POWER9, 2644 * where it's 1 because the threads are (mostly) independent. 2645 */ 2646 controlled_threads = threads_per_vcore(); 2647 2648 /* 2649 * Make sure we are running on primary threads, and that secondary 2650 * threads are offline. Also check if the number of threads in this 2651 * guest are greater than the current system threads per guest. 2652 */ 2653 if ((controlled_threads > 1) && 2654 ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) { 2655 for_each_runnable_thread(i, vcpu, vc) { 2656 vcpu->arch.ret = -EBUSY; 2657 kvmppc_remove_runnable(vc, vcpu); 2658 wake_up(&vcpu->arch.cpu_run); 2659 } 2660 goto out; 2661 } 2662 2663 /* 2664 * See if we could run any other vcores on the physical core 2665 * along with this one. 2666 */ 2667 init_core_info(&core_info, vc); 2668 pcpu = smp_processor_id(); 2669 target_threads = controlled_threads; 2670 if (target_smt_mode && target_smt_mode < target_threads) 2671 target_threads = target_smt_mode; 2672 if (vc->num_threads < target_threads) 2673 collect_piggybacks(&core_info, target_threads); 2674 2675 /* 2676 * On radix, arrange for TLB flushing if necessary. 2677 * This has to be done before disabling interrupts since 2678 * it uses smp_call_function(). 2679 */ 2680 pcpu = smp_processor_id(); 2681 if (kvm_is_radix(vc->kvm)) { 2682 for (sub = 0; sub < core_info.n_subcores; ++sub) 2683 for_each_runnable_thread(i, vcpu, core_info.vc[sub]) 2684 kvmppc_prepare_radix_vcpu(vcpu, pcpu); 2685 } 2686 2687 /* 2688 * Hard-disable interrupts, and check resched flag and signals. 2689 * If we need to reschedule or deliver a signal, clean up 2690 * and return without going into the guest(s). 2691 */ 2692 local_irq_disable(); 2693 hard_irq_disable(); 2694 if (lazy_irq_pending() || need_resched() || 2695 recheck_signals(&core_info)) { 2696 local_irq_enable(); 2697 vc->vcore_state = VCORE_INACTIVE; 2698 /* Unlock all except the primary vcore */ 2699 for (sub = 1; sub < core_info.n_subcores; ++sub) { 2700 pvc = core_info.vc[sub]; 2701 /* Put back on to the preempted vcores list */ 2702 kvmppc_vcore_preempt(pvc); 2703 spin_unlock(&pvc->lock); 2704 } 2705 for (i = 0; i < controlled_threads; ++i) 2706 kvmppc_release_hwthread(pcpu + i); 2707 return; 2708 } 2709 2710 kvmppc_clear_host_core(pcpu); 2711 2712 /* Decide on micro-threading (split-core) mode */ 2713 subcore_size = threads_per_subcore; 2714 cmd_bit = stat_bit = 0; 2715 split = core_info.n_subcores; 2716 sip = NULL; 2717 if (split > 1) { 2718 /* threads_per_subcore must be MAX_SMT_THREADS (8) here */ 2719 if (split == 2 && (dynamic_mt_modes & 2)) { 2720 cmd_bit = HID0_POWER8_1TO2LPAR; 2721 stat_bit = HID0_POWER8_2LPARMODE; 2722 } else { 2723 split = 4; 2724 cmd_bit = HID0_POWER8_1TO4LPAR; 2725 stat_bit = HID0_POWER8_4LPARMODE; 2726 } 2727 subcore_size = MAX_SMT_THREADS / split; 2728 sip = &split_info; 2729 memset(&split_info, 0, sizeof(split_info)); 2730 split_info.rpr = mfspr(SPRN_RPR); 2731 split_info.pmmar = mfspr(SPRN_PMMAR); 2732 split_info.ldbar = mfspr(SPRN_LDBAR); 2733 split_info.subcore_size = subcore_size; 2734 for (sub = 0; sub < core_info.n_subcores; ++sub) 2735 split_info.vc[sub] = core_info.vc[sub]; 2736 /* order writes to split_info before kvm_split_mode pointer */ 2737 smp_wmb(); 2738 } 2739 for (thr = 0; thr < controlled_threads; ++thr) 2740 paca[pcpu + thr].kvm_hstate.kvm_split_mode = sip; 2741 2742 /* Initiate micro-threading (split-core) if required */ 2743 if (cmd_bit) { 2744 unsigned long hid0 = mfspr(SPRN_HID0); 2745 2746 hid0 |= cmd_bit | HID0_POWER8_DYNLPARDIS; 2747 mb(); 2748 mtspr(SPRN_HID0, hid0); 2749 isync(); 2750 for (;;) { 2751 hid0 = mfspr(SPRN_HID0); 2752 if (hid0 & stat_bit) 2753 break; 2754 cpu_relax(); 2755 } 2756 } 2757 2758 /* Start all the threads */ 2759 active = 0; 2760 for (sub = 0; sub < core_info.n_subcores; ++sub) { 2761 thr = subcore_thread_map[sub]; 2762 thr0_done = false; 2763 active |= 1 << thr; 2764 pvc = core_info.vc[sub]; 2765 pvc->pcpu = pcpu + thr; 2766 for_each_runnable_thread(i, vcpu, pvc) { 2767 kvmppc_start_thread(vcpu, pvc); 2768 kvmppc_create_dtl_entry(vcpu, pvc); 2769 trace_kvm_guest_enter(vcpu); 2770 if (!vcpu->arch.ptid) 2771 thr0_done = true; 2772 active |= 1 << (thr + vcpu->arch.ptid); 2773 } 2774 /* 2775 * We need to start the first thread of each subcore 2776 * even if it doesn't have a vcpu. 2777 */ 2778 if (!thr0_done) 2779 kvmppc_start_thread(NULL, pvc); 2780 thr += pvc->num_threads; 2781 } 2782 2783 /* 2784 * Ensure that split_info.do_nap is set after setting 2785 * the vcore pointer in the PACA of the secondaries. 2786 */ 2787 smp_mb(); 2788 if (cmd_bit) 2789 split_info.do_nap = 1; /* ask secondaries to nap when done */ 2790 2791 /* 2792 * When doing micro-threading, poke the inactive threads as well. 2793 * This gets them to the nap instruction after kvm_do_nap, 2794 * which reduces the time taken to unsplit later. 2795 */ 2796 if (split > 1) 2797 for (thr = 1; thr < threads_per_subcore; ++thr) 2798 if (!(active & (1 << thr))) 2799 kvmppc_ipi_thread(pcpu + thr); 2800 2801 vc->vcore_state = VCORE_RUNNING; 2802 preempt_disable(); 2803 2804 trace_kvmppc_run_core(vc, 0); 2805 2806 for (sub = 0; sub < core_info.n_subcores; ++sub) 2807 spin_unlock(&core_info.vc[sub]->lock); 2808 2809 /* 2810 * Interrupts will be enabled once we get into the guest, 2811 * so tell lockdep that we're about to enable interrupts. 2812 */ 2813 trace_hardirqs_on(); 2814 2815 guest_enter(); 2816 2817 srcu_idx = srcu_read_lock(&vc->kvm->srcu); 2818 2819 trap = __kvmppc_vcore_entry(); 2820 2821 srcu_read_unlock(&vc->kvm->srcu, srcu_idx); 2822 2823 guest_exit(); 2824 2825 trace_hardirqs_off(); 2826 set_irq_happened(trap); 2827 2828 spin_lock(&vc->lock); 2829 /* prevent other vcpu threads from doing kvmppc_start_thread() now */ 2830 vc->vcore_state = VCORE_EXITING; 2831 2832 /* wait for secondary threads to finish writing their state to memory */ 2833 kvmppc_wait_for_nap(); 2834 2835 /* Return to whole-core mode if we split the core earlier */ 2836 if (split > 1) { 2837 unsigned long hid0 = mfspr(SPRN_HID0); 2838 unsigned long loops = 0; 2839 2840 hid0 &= ~HID0_POWER8_DYNLPARDIS; 2841 stat_bit = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE; 2842 mb(); 2843 mtspr(SPRN_HID0, hid0); 2844 isync(); 2845 for (;;) { 2846 hid0 = mfspr(SPRN_HID0); 2847 if (!(hid0 & stat_bit)) 2848 break; 2849 cpu_relax(); 2850 ++loops; 2851 } 2852 split_info.do_nap = 0; 2853 } 2854 2855 kvmppc_set_host_core(pcpu); 2856 2857 local_irq_enable(); 2858 2859 /* Let secondaries go back to the offline loop */ 2860 for (i = 0; i < controlled_threads; ++i) { 2861 kvmppc_release_hwthread(pcpu + i); 2862 if (sip && sip->napped[i]) 2863 kvmppc_ipi_thread(pcpu + i); 2864 cpumask_clear_cpu(pcpu + i, &vc->kvm->arch.cpu_in_guest); 2865 } 2866 2867 spin_unlock(&vc->lock); 2868 2869 /* make sure updates to secondary vcpu structs are visible now */ 2870 smp_mb(); 2871 2872 for (sub = 0; sub < core_info.n_subcores; ++sub) { 2873 pvc = core_info.vc[sub]; 2874 post_guest_process(pvc, pvc == vc); 2875 } 2876 2877 spin_lock(&vc->lock); 2878 preempt_enable(); 2879 2880 out: 2881 vc->vcore_state = VCORE_INACTIVE; 2882 trace_kvmppc_run_core(vc, 1); 2883 } 2884 2885 /* 2886 * Wait for some other vcpu thread to execute us, and 2887 * wake us up when we need to handle something in the host. 2888 */ 2889 static void kvmppc_wait_for_exec(struct kvmppc_vcore *vc, 2890 struct kvm_vcpu *vcpu, int wait_state) 2891 { 2892 DEFINE_WAIT(wait); 2893 2894 prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state); 2895 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) { 2896 spin_unlock(&vc->lock); 2897 schedule(); 2898 spin_lock(&vc->lock); 2899 } 2900 finish_wait(&vcpu->arch.cpu_run, &wait); 2901 } 2902 2903 static void grow_halt_poll_ns(struct kvmppc_vcore *vc) 2904 { 2905 /* 10us base */ 2906 if (vc->halt_poll_ns == 0 && halt_poll_ns_grow) 2907 vc->halt_poll_ns = 10000; 2908 else 2909 vc->halt_poll_ns *= halt_poll_ns_grow; 2910 } 2911 2912 static void shrink_halt_poll_ns(struct kvmppc_vcore *vc) 2913 { 2914 if (halt_poll_ns_shrink == 0) 2915 vc->halt_poll_ns = 0; 2916 else 2917 vc->halt_poll_ns /= halt_poll_ns_shrink; 2918 } 2919 2920 #ifdef CONFIG_KVM_XICS 2921 static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu) 2922 { 2923 if (!xive_enabled()) 2924 return false; 2925 return vcpu->arch.xive_saved_state.pipr < 2926 vcpu->arch.xive_saved_state.cppr; 2927 } 2928 #else 2929 static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu) 2930 { 2931 return false; 2932 } 2933 #endif /* CONFIG_KVM_XICS */ 2934 2935 static bool kvmppc_vcpu_woken(struct kvm_vcpu *vcpu) 2936 { 2937 if (vcpu->arch.pending_exceptions || vcpu->arch.prodded || 2938 kvmppc_doorbell_pending(vcpu) || xive_interrupt_pending(vcpu)) 2939 return true; 2940 2941 return false; 2942 } 2943 2944 /* 2945 * Check to see if any of the runnable vcpus on the vcore have pending 2946 * exceptions or are no longer ceded 2947 */ 2948 static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc) 2949 { 2950 struct kvm_vcpu *vcpu; 2951 int i; 2952 2953 for_each_runnable_thread(i, vcpu, vc) { 2954 if (!vcpu->arch.ceded || kvmppc_vcpu_woken(vcpu)) 2955 return 1; 2956 } 2957 2958 return 0; 2959 } 2960 2961 /* 2962 * All the vcpus in this vcore are idle, so wait for a decrementer 2963 * or external interrupt to one of the vcpus. vc->lock is held. 2964 */ 2965 static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) 2966 { 2967 ktime_t cur, start_poll, start_wait; 2968 int do_sleep = 1; 2969 u64 block_ns; 2970 DECLARE_SWAITQUEUE(wait); 2971 2972 /* Poll for pending exceptions and ceded state */ 2973 cur = start_poll = ktime_get(); 2974 if (vc->halt_poll_ns) { 2975 ktime_t stop = ktime_add_ns(start_poll, vc->halt_poll_ns); 2976 ++vc->runner->stat.halt_attempted_poll; 2977 2978 vc->vcore_state = VCORE_POLLING; 2979 spin_unlock(&vc->lock); 2980 2981 do { 2982 if (kvmppc_vcore_check_block(vc)) { 2983 do_sleep = 0; 2984 break; 2985 } 2986 cur = ktime_get(); 2987 } while (single_task_running() && ktime_before(cur, stop)); 2988 2989 spin_lock(&vc->lock); 2990 vc->vcore_state = VCORE_INACTIVE; 2991 2992 if (!do_sleep) { 2993 ++vc->runner->stat.halt_successful_poll; 2994 goto out; 2995 } 2996 } 2997 2998 prepare_to_swait(&vc->wq, &wait, TASK_INTERRUPTIBLE); 2999 3000 if (kvmppc_vcore_check_block(vc)) { 3001 finish_swait(&vc->wq, &wait); 3002 do_sleep = 0; 3003 /* If we polled, count this as a successful poll */ 3004 if (vc->halt_poll_ns) 3005 ++vc->runner->stat.halt_successful_poll; 3006 goto out; 3007 } 3008 3009 start_wait = ktime_get(); 3010 3011 vc->vcore_state = VCORE_SLEEPING; 3012 trace_kvmppc_vcore_blocked(vc, 0); 3013 spin_unlock(&vc->lock); 3014 schedule(); 3015 finish_swait(&vc->wq, &wait); 3016 spin_lock(&vc->lock); 3017 vc->vcore_state = VCORE_INACTIVE; 3018 trace_kvmppc_vcore_blocked(vc, 1); 3019 ++vc->runner->stat.halt_successful_wait; 3020 3021 cur = ktime_get(); 3022 3023 out: 3024 block_ns = ktime_to_ns(cur) - ktime_to_ns(start_poll); 3025 3026 /* Attribute wait time */ 3027 if (do_sleep) { 3028 vc->runner->stat.halt_wait_ns += 3029 ktime_to_ns(cur) - ktime_to_ns(start_wait); 3030 /* Attribute failed poll time */ 3031 if (vc->halt_poll_ns) 3032 vc->runner->stat.halt_poll_fail_ns += 3033 ktime_to_ns(start_wait) - 3034 ktime_to_ns(start_poll); 3035 } else { 3036 /* Attribute successful poll time */ 3037 if (vc->halt_poll_ns) 3038 vc->runner->stat.halt_poll_success_ns += 3039 ktime_to_ns(cur) - 3040 ktime_to_ns(start_poll); 3041 } 3042 3043 /* Adjust poll time */ 3044 if (halt_poll_ns) { 3045 if (block_ns <= vc->halt_poll_ns) 3046 ; 3047 /* We slept and blocked for longer than the max halt time */ 3048 else if (vc->halt_poll_ns && block_ns > halt_poll_ns) 3049 shrink_halt_poll_ns(vc); 3050 /* We slept and our poll time is too small */ 3051 else if (vc->halt_poll_ns < halt_poll_ns && 3052 block_ns < halt_poll_ns) 3053 grow_halt_poll_ns(vc); 3054 if (vc->halt_poll_ns > halt_poll_ns) 3055 vc->halt_poll_ns = halt_poll_ns; 3056 } else 3057 vc->halt_poll_ns = 0; 3058 3059 trace_kvmppc_vcore_wakeup(do_sleep, block_ns); 3060 } 3061 3062 static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) 3063 { 3064 int n_ceded, i; 3065 struct kvmppc_vcore *vc; 3066 struct kvm_vcpu *v; 3067 3068 trace_kvmppc_run_vcpu_enter(vcpu); 3069 3070 kvm_run->exit_reason = 0; 3071 vcpu->arch.ret = RESUME_GUEST; 3072 vcpu->arch.trap = 0; 3073 kvmppc_update_vpas(vcpu); 3074 3075 /* 3076 * Synchronize with other threads in this virtual core 3077 */ 3078 vc = vcpu->arch.vcore; 3079 spin_lock(&vc->lock); 3080 vcpu->arch.ceded = 0; 3081 vcpu->arch.run_task = current; 3082 vcpu->arch.kvm_run = kvm_run; 3083 vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb()); 3084 vcpu->arch.state = KVMPPC_VCPU_RUNNABLE; 3085 vcpu->arch.busy_preempt = TB_NIL; 3086 WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], vcpu); 3087 ++vc->n_runnable; 3088 3089 /* 3090 * This happens the first time this is called for a vcpu. 3091 * If the vcore is already running, we may be able to start 3092 * this thread straight away and have it join in. 3093 */ 3094 if (!signal_pending(current)) { 3095 if (vc->vcore_state == VCORE_PIGGYBACK) { 3096 if (spin_trylock(&vc->lock)) { 3097 if (vc->vcore_state == VCORE_RUNNING && 3098 !VCORE_IS_EXITING(vc)) { 3099 kvmppc_create_dtl_entry(vcpu, vc); 3100 kvmppc_start_thread(vcpu, vc); 3101 trace_kvm_guest_enter(vcpu); 3102 } 3103 spin_unlock(&vc->lock); 3104 } 3105 } else if (vc->vcore_state == VCORE_RUNNING && 3106 !VCORE_IS_EXITING(vc)) { 3107 kvmppc_create_dtl_entry(vcpu, vc); 3108 kvmppc_start_thread(vcpu, vc); 3109 trace_kvm_guest_enter(vcpu); 3110 } else if (vc->vcore_state == VCORE_SLEEPING) { 3111 swake_up(&vc->wq); 3112 } 3113 3114 } 3115 3116 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE && 3117 !signal_pending(current)) { 3118 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL) 3119 kvmppc_vcore_end_preempt(vc); 3120 3121 if (vc->vcore_state != VCORE_INACTIVE) { 3122 kvmppc_wait_for_exec(vc, vcpu, TASK_INTERRUPTIBLE); 3123 continue; 3124 } 3125 for_each_runnable_thread(i, v, vc) { 3126 kvmppc_core_prepare_to_enter(v); 3127 if (signal_pending(v->arch.run_task)) { 3128 kvmppc_remove_runnable(vc, v); 3129 v->stat.signal_exits++; 3130 v->arch.kvm_run->exit_reason = KVM_EXIT_INTR; 3131 v->arch.ret = -EINTR; 3132 wake_up(&v->arch.cpu_run); 3133 } 3134 } 3135 if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE) 3136 break; 3137 n_ceded = 0; 3138 for_each_runnable_thread(i, v, vc) { 3139 if (!kvmppc_vcpu_woken(v)) 3140 n_ceded += v->arch.ceded; 3141 else 3142 v->arch.ceded = 0; 3143 } 3144 vc->runner = vcpu; 3145 if (n_ceded == vc->n_runnable) { 3146 kvmppc_vcore_blocked(vc); 3147 } else if (need_resched()) { 3148 kvmppc_vcore_preempt(vc); 3149 /* Let something else run */ 3150 cond_resched_lock(&vc->lock); 3151 if (vc->vcore_state == VCORE_PREEMPT) 3152 kvmppc_vcore_end_preempt(vc); 3153 } else { 3154 kvmppc_run_core(vc); 3155 } 3156 vc->runner = NULL; 3157 } 3158 3159 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE && 3160 (vc->vcore_state == VCORE_RUNNING || 3161 vc->vcore_state == VCORE_EXITING || 3162 vc->vcore_state == VCORE_PIGGYBACK)) 3163 kvmppc_wait_for_exec(vc, vcpu, TASK_UNINTERRUPTIBLE); 3164 3165 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL) 3166 kvmppc_vcore_end_preempt(vc); 3167 3168 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) { 3169 kvmppc_remove_runnable(vc, vcpu); 3170 vcpu->stat.signal_exits++; 3171 kvm_run->exit_reason = KVM_EXIT_INTR; 3172 vcpu->arch.ret = -EINTR; 3173 } 3174 3175 if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) { 3176 /* Wake up some vcpu to run the core */ 3177 i = -1; 3178 v = next_runnable_thread(vc, &i); 3179 wake_up(&v->arch.cpu_run); 3180 } 3181 3182 trace_kvmppc_run_vcpu_exit(vcpu, kvm_run); 3183 spin_unlock(&vc->lock); 3184 return vcpu->arch.ret; 3185 } 3186 3187 static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu) 3188 { 3189 int r; 3190 int srcu_idx; 3191 unsigned long ebb_regs[3] = {}; /* shut up GCC */ 3192 unsigned long user_tar = 0; 3193 unsigned int user_vrsave; 3194 3195 if (!vcpu->arch.sane) { 3196 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3197 return -EINVAL; 3198 } 3199 3200 /* 3201 * Don't allow entry with a suspended transaction, because 3202 * the guest entry/exit code will lose it. 3203 * If the guest has TM enabled, save away their TM-related SPRs 3204 * (they will get restored by the TM unavailable interrupt). 3205 */ 3206 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 3207 if (cpu_has_feature(CPU_FTR_TM) && current->thread.regs && 3208 (current->thread.regs->msr & MSR_TM)) { 3209 if (MSR_TM_ACTIVE(current->thread.regs->msr)) { 3210 run->exit_reason = KVM_EXIT_FAIL_ENTRY; 3211 run->fail_entry.hardware_entry_failure_reason = 0; 3212 return -EINVAL; 3213 } 3214 current->thread.tm_tfhar = mfspr(SPRN_TFHAR); 3215 current->thread.tm_tfiar = mfspr(SPRN_TFIAR); 3216 current->thread.tm_texasr = mfspr(SPRN_TEXASR); 3217 current->thread.regs->msr &= ~MSR_TM; 3218 } 3219 #endif 3220 3221 kvmppc_core_prepare_to_enter(vcpu); 3222 3223 /* No need to go into the guest when all we'll do is come back out */ 3224 if (signal_pending(current)) { 3225 run->exit_reason = KVM_EXIT_INTR; 3226 return -EINTR; 3227 } 3228 3229 atomic_inc(&vcpu->kvm->arch.vcpus_running); 3230 /* Order vcpus_running vs. hpte_setup_done, see kvmppc_alloc_reset_hpt */ 3231 smp_mb(); 3232 3233 /* On the first time here, set up HTAB and VRMA */ 3234 if (!kvm_is_radix(vcpu->kvm) && !vcpu->kvm->arch.hpte_setup_done) { 3235 r = kvmppc_hv_setup_htab_rma(vcpu); 3236 if (r) 3237 goto out; 3238 } 3239 3240 flush_all_to_thread(current); 3241 3242 /* Save userspace EBB and other register values */ 3243 if (cpu_has_feature(CPU_FTR_ARCH_207S)) { 3244 ebb_regs[0] = mfspr(SPRN_EBBHR); 3245 ebb_regs[1] = mfspr(SPRN_EBBRR); 3246 ebb_regs[2] = mfspr(SPRN_BESCR); 3247 user_tar = mfspr(SPRN_TAR); 3248 } 3249 user_vrsave = mfspr(SPRN_VRSAVE); 3250 3251 vcpu->arch.wqp = &vcpu->arch.vcore->wq; 3252 vcpu->arch.pgdir = current->mm->pgd; 3253 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST; 3254 3255 do { 3256 r = kvmppc_run_vcpu(run, vcpu); 3257 3258 if (run->exit_reason == KVM_EXIT_PAPR_HCALL && 3259 !(vcpu->arch.shregs.msr & MSR_PR)) { 3260 trace_kvm_hcall_enter(vcpu); 3261 r = kvmppc_pseries_do_hcall(vcpu); 3262 trace_kvm_hcall_exit(vcpu, r); 3263 kvmppc_core_prepare_to_enter(vcpu); 3264 } else if (r == RESUME_PAGE_FAULT) { 3265 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 3266 r = kvmppc_book3s_hv_page_fault(run, vcpu, 3267 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr); 3268 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); 3269 } else if (r == RESUME_PASSTHROUGH) { 3270 if (WARN_ON(xive_enabled())) 3271 r = H_SUCCESS; 3272 else 3273 r = kvmppc_xics_rm_complete(vcpu, 0); 3274 } 3275 } while (is_kvmppc_resume_guest(r)); 3276 3277 /* Restore userspace EBB and other register values */ 3278 if (cpu_has_feature(CPU_FTR_ARCH_207S)) { 3279 mtspr(SPRN_EBBHR, ebb_regs[0]); 3280 mtspr(SPRN_EBBRR, ebb_regs[1]); 3281 mtspr(SPRN_BESCR, ebb_regs[2]); 3282 mtspr(SPRN_TAR, user_tar); 3283 mtspr(SPRN_FSCR, current->thread.fscr); 3284 } 3285 mtspr(SPRN_VRSAVE, user_vrsave); 3286 3287 out: 3288 vcpu->arch.state = KVMPPC_VCPU_NOTREADY; 3289 atomic_dec(&vcpu->kvm->arch.vcpus_running); 3290 return r; 3291 } 3292 3293 static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps, 3294 int linux_psize) 3295 { 3296 struct mmu_psize_def *def = &mmu_psize_defs[linux_psize]; 3297 3298 if (!def->shift) 3299 return; 3300 (*sps)->page_shift = def->shift; 3301 (*sps)->slb_enc = def->sllp; 3302 (*sps)->enc[0].page_shift = def->shift; 3303 (*sps)->enc[0].pte_enc = def->penc[linux_psize]; 3304 /* 3305 * Add 16MB MPSS support if host supports it 3306 */ 3307 if (linux_psize != MMU_PAGE_16M && def->penc[MMU_PAGE_16M] != -1) { 3308 (*sps)->enc[1].page_shift = 24; 3309 (*sps)->enc[1].pte_enc = def->penc[MMU_PAGE_16M]; 3310 } 3311 (*sps)++; 3312 } 3313 3314 static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm, 3315 struct kvm_ppc_smmu_info *info) 3316 { 3317 struct kvm_ppc_one_seg_page_size *sps; 3318 3319 /* 3320 * Since we don't yet support HPT guests on a radix host, 3321 * return an error if the host uses radix. 3322 */ 3323 if (radix_enabled()) 3324 return -EINVAL; 3325 3326 info->flags = KVM_PPC_PAGE_SIZES_REAL; 3327 if (mmu_has_feature(MMU_FTR_1T_SEGMENT)) 3328 info->flags |= KVM_PPC_1T_SEGMENTS; 3329 info->slb_size = mmu_slb_size; 3330 3331 /* We only support these sizes for now, and no muti-size segments */ 3332 sps = &info->sps[0]; 3333 kvmppc_add_seg_page_size(&sps, MMU_PAGE_4K); 3334 kvmppc_add_seg_page_size(&sps, MMU_PAGE_64K); 3335 kvmppc_add_seg_page_size(&sps, MMU_PAGE_16M); 3336 3337 return 0; 3338 } 3339 3340 /* 3341 * Get (and clear) the dirty memory log for a memory slot. 3342 */ 3343 static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm, 3344 struct kvm_dirty_log *log) 3345 { 3346 struct kvm_memslots *slots; 3347 struct kvm_memory_slot *memslot; 3348 int i, r; 3349 unsigned long n; 3350 unsigned long *buf; 3351 struct kvm_vcpu *vcpu; 3352 3353 mutex_lock(&kvm->slots_lock); 3354 3355 r = -EINVAL; 3356 if (log->slot >= KVM_USER_MEM_SLOTS) 3357 goto out; 3358 3359 slots = kvm_memslots(kvm); 3360 memslot = id_to_memslot(slots, log->slot); 3361 r = -ENOENT; 3362 if (!memslot->dirty_bitmap) 3363 goto out; 3364 3365 /* 3366 * Use second half of bitmap area because radix accumulates 3367 * bits in the first half. 3368 */ 3369 n = kvm_dirty_bitmap_bytes(memslot); 3370 buf = memslot->dirty_bitmap + n / sizeof(long); 3371 memset(buf, 0, n); 3372 3373 if (kvm_is_radix(kvm)) 3374 r = kvmppc_hv_get_dirty_log_radix(kvm, memslot, buf); 3375 else 3376 r = kvmppc_hv_get_dirty_log_hpt(kvm, memslot, buf); 3377 if (r) 3378 goto out; 3379 3380 /* Harvest dirty bits from VPA and DTL updates */ 3381 /* Note: we never modify the SLB shadow buffer areas */ 3382 kvm_for_each_vcpu(i, vcpu, kvm) { 3383 spin_lock(&vcpu->arch.vpa_update_lock); 3384 kvmppc_harvest_vpa_dirty(&vcpu->arch.vpa, memslot, buf); 3385 kvmppc_harvest_vpa_dirty(&vcpu->arch.dtl, memslot, buf); 3386 spin_unlock(&vcpu->arch.vpa_update_lock); 3387 } 3388 3389 r = -EFAULT; 3390 if (copy_to_user(log->dirty_bitmap, buf, n)) 3391 goto out; 3392 3393 r = 0; 3394 out: 3395 mutex_unlock(&kvm->slots_lock); 3396 return r; 3397 } 3398 3399 static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *free, 3400 struct kvm_memory_slot *dont) 3401 { 3402 if (!dont || free->arch.rmap != dont->arch.rmap) { 3403 vfree(free->arch.rmap); 3404 free->arch.rmap = NULL; 3405 } 3406 } 3407 3408 static int kvmppc_core_create_memslot_hv(struct kvm_memory_slot *slot, 3409 unsigned long npages) 3410 { 3411 /* 3412 * For now, if radix_enabled() then we only support radix guests, 3413 * and in that case we don't need the rmap array. 3414 */ 3415 if (radix_enabled()) { 3416 slot->arch.rmap = NULL; 3417 return 0; 3418 } 3419 3420 slot->arch.rmap = vzalloc(npages * sizeof(*slot->arch.rmap)); 3421 if (!slot->arch.rmap) 3422 return -ENOMEM; 3423 3424 return 0; 3425 } 3426 3427 static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm, 3428 struct kvm_memory_slot *memslot, 3429 const struct kvm_userspace_memory_region *mem) 3430 { 3431 return 0; 3432 } 3433 3434 static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm, 3435 const struct kvm_userspace_memory_region *mem, 3436 const struct kvm_memory_slot *old, 3437 const struct kvm_memory_slot *new) 3438 { 3439 unsigned long npages = mem->memory_size >> PAGE_SHIFT; 3440 struct kvm_memslots *slots; 3441 struct kvm_memory_slot *memslot; 3442 3443 /* 3444 * If we are making a new memslot, it might make 3445 * some address that was previously cached as emulated 3446 * MMIO be no longer emulated MMIO, so invalidate 3447 * all the caches of emulated MMIO translations. 3448 */ 3449 if (npages) 3450 atomic64_inc(&kvm->arch.mmio_update); 3451 3452 if (npages && old->npages && !kvm_is_radix(kvm)) { 3453 /* 3454 * If modifying a memslot, reset all the rmap dirty bits. 3455 * If this is a new memslot, we don't need to do anything 3456 * since the rmap array starts out as all zeroes, 3457 * i.e. no pages are dirty. 3458 */ 3459 slots = kvm_memslots(kvm); 3460 memslot = id_to_memslot(slots, mem->slot); 3461 kvmppc_hv_get_dirty_log_hpt(kvm, memslot, NULL); 3462 } 3463 } 3464 3465 /* 3466 * Update LPCR values in kvm->arch and in vcores. 3467 * Caller must hold kvm->lock. 3468 */ 3469 void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask) 3470 { 3471 long int i; 3472 u32 cores_done = 0; 3473 3474 if ((kvm->arch.lpcr & mask) == lpcr) 3475 return; 3476 3477 kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr; 3478 3479 for (i = 0; i < KVM_MAX_VCORES; ++i) { 3480 struct kvmppc_vcore *vc = kvm->arch.vcores[i]; 3481 if (!vc) 3482 continue; 3483 spin_lock(&vc->lock); 3484 vc->lpcr = (vc->lpcr & ~mask) | lpcr; 3485 spin_unlock(&vc->lock); 3486 if (++cores_done >= kvm->arch.online_vcores) 3487 break; 3488 } 3489 } 3490 3491 static void kvmppc_mmu_destroy_hv(struct kvm_vcpu *vcpu) 3492 { 3493 return; 3494 } 3495 3496 static void kvmppc_setup_partition_table(struct kvm *kvm) 3497 { 3498 unsigned long dw0, dw1; 3499 3500 if (!kvm_is_radix(kvm)) { 3501 /* PS field - page size for VRMA */ 3502 dw0 = ((kvm->arch.vrma_slb_v & SLB_VSID_L) >> 1) | 3503 ((kvm->arch.vrma_slb_v & SLB_VSID_LP) << 1); 3504 /* HTABSIZE and HTABORG fields */ 3505 dw0 |= kvm->arch.sdr1; 3506 3507 /* Second dword as set by userspace */ 3508 dw1 = kvm->arch.process_table; 3509 } else { 3510 dw0 = PATB_HR | radix__get_tree_size() | 3511 __pa(kvm->arch.pgtable) | RADIX_PGD_INDEX_SIZE; 3512 dw1 = PATB_GR | kvm->arch.process_table; 3513 } 3514 3515 mmu_partition_table_set_entry(kvm->arch.lpid, dw0, dw1); 3516 } 3517 3518 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu) 3519 { 3520 int err = 0; 3521 struct kvm *kvm = vcpu->kvm; 3522 unsigned long hva; 3523 struct kvm_memory_slot *memslot; 3524 struct vm_area_struct *vma; 3525 unsigned long lpcr = 0, senc; 3526 unsigned long psize, porder; 3527 int srcu_idx; 3528 3529 mutex_lock(&kvm->lock); 3530 if (kvm->arch.hpte_setup_done) 3531 goto out; /* another vcpu beat us to it */ 3532 3533 /* Allocate hashed page table (if not done already) and reset it */ 3534 if (!kvm->arch.hpt.virt) { 3535 int order = KVM_DEFAULT_HPT_ORDER; 3536 struct kvm_hpt_info info; 3537 3538 err = kvmppc_allocate_hpt(&info, order); 3539 /* If we get here, it means userspace didn't specify a 3540 * size explicitly. So, try successively smaller 3541 * sizes if the default failed. */ 3542 while ((err == -ENOMEM) && --order >= PPC_MIN_HPT_ORDER) 3543 err = kvmppc_allocate_hpt(&info, order); 3544 3545 if (err < 0) { 3546 pr_err("KVM: Couldn't alloc HPT\n"); 3547 goto out; 3548 } 3549 3550 kvmppc_set_hpt(kvm, &info); 3551 } 3552 3553 /* Look up the memslot for guest physical address 0 */ 3554 srcu_idx = srcu_read_lock(&kvm->srcu); 3555 memslot = gfn_to_memslot(kvm, 0); 3556 3557 /* We must have some memory at 0 by now */ 3558 err = -EINVAL; 3559 if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) 3560 goto out_srcu; 3561 3562 /* Look up the VMA for the start of this memory slot */ 3563 hva = memslot->userspace_addr; 3564 down_read(¤t->mm->mmap_sem); 3565 vma = find_vma(current->mm, hva); 3566 if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO)) 3567 goto up_out; 3568 3569 psize = vma_kernel_pagesize(vma); 3570 porder = __ilog2(psize); 3571 3572 up_read(¤t->mm->mmap_sem); 3573 3574 /* We can handle 4k, 64k or 16M pages in the VRMA */ 3575 err = -EINVAL; 3576 if (!(psize == 0x1000 || psize == 0x10000 || 3577 psize == 0x1000000)) 3578 goto out_srcu; 3579 3580 senc = slb_pgsize_encoding(psize); 3581 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T | 3582 (VRMA_VSID << SLB_VSID_SHIFT_1T); 3583 /* Create HPTEs in the hash page table for the VRMA */ 3584 kvmppc_map_vrma(vcpu, memslot, porder); 3585 3586 /* Update VRMASD field in the LPCR */ 3587 if (!cpu_has_feature(CPU_FTR_ARCH_300)) { 3588 /* the -4 is to account for senc values starting at 0x10 */ 3589 lpcr = senc << (LPCR_VRMASD_SH - 4); 3590 kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD); 3591 } else { 3592 kvmppc_setup_partition_table(kvm); 3593 } 3594 3595 /* Order updates to kvm->arch.lpcr etc. vs. hpte_setup_done */ 3596 smp_wmb(); 3597 kvm->arch.hpte_setup_done = 1; 3598 err = 0; 3599 out_srcu: 3600 srcu_read_unlock(&kvm->srcu, srcu_idx); 3601 out: 3602 mutex_unlock(&kvm->lock); 3603 return err; 3604 3605 up_out: 3606 up_read(¤t->mm->mmap_sem); 3607 goto out_srcu; 3608 } 3609 3610 #ifdef CONFIG_KVM_XICS 3611 /* 3612 * Allocate a per-core structure for managing state about which cores are 3613 * running in the host versus the guest and for exchanging data between 3614 * real mode KVM and CPU running in the host. 3615 * This is only done for the first VM. 3616 * The allocated structure stays even if all VMs have stopped. 3617 * It is only freed when the kvm-hv module is unloaded. 3618 * It's OK for this routine to fail, we just don't support host 3619 * core operations like redirecting H_IPI wakeups. 3620 */ 3621 void kvmppc_alloc_host_rm_ops(void) 3622 { 3623 struct kvmppc_host_rm_ops *ops; 3624 unsigned long l_ops; 3625 int cpu, core; 3626 int size; 3627 3628 /* Not the first time here ? */ 3629 if (kvmppc_host_rm_ops_hv != NULL) 3630 return; 3631 3632 ops = kzalloc(sizeof(struct kvmppc_host_rm_ops), GFP_KERNEL); 3633 if (!ops) 3634 return; 3635 3636 size = cpu_nr_cores() * sizeof(struct kvmppc_host_rm_core); 3637 ops->rm_core = kzalloc(size, GFP_KERNEL); 3638 3639 if (!ops->rm_core) { 3640 kfree(ops); 3641 return; 3642 } 3643 3644 cpus_read_lock(); 3645 3646 for (cpu = 0; cpu < nr_cpu_ids; cpu += threads_per_core) { 3647 if (!cpu_online(cpu)) 3648 continue; 3649 3650 core = cpu >> threads_shift; 3651 ops->rm_core[core].rm_state.in_host = 1; 3652 } 3653 3654 ops->vcpu_kick = kvmppc_fast_vcpu_kick_hv; 3655 3656 /* 3657 * Make the contents of the kvmppc_host_rm_ops structure visible 3658 * to other CPUs before we assign it to the global variable. 3659 * Do an atomic assignment (no locks used here), but if someone 3660 * beats us to it, just free our copy and return. 3661 */ 3662 smp_wmb(); 3663 l_ops = (unsigned long) ops; 3664 3665 if (cmpxchg64((unsigned long *)&kvmppc_host_rm_ops_hv, 0, l_ops)) { 3666 cpus_read_unlock(); 3667 kfree(ops->rm_core); 3668 kfree(ops); 3669 return; 3670 } 3671 3672 cpuhp_setup_state_nocalls_cpuslocked(CPUHP_KVM_PPC_BOOK3S_PREPARE, 3673 "ppc/kvm_book3s:prepare", 3674 kvmppc_set_host_core, 3675 kvmppc_clear_host_core); 3676 cpus_read_unlock(); 3677 } 3678 3679 void kvmppc_free_host_rm_ops(void) 3680 { 3681 if (kvmppc_host_rm_ops_hv) { 3682 cpuhp_remove_state_nocalls(CPUHP_KVM_PPC_BOOK3S_PREPARE); 3683 kfree(kvmppc_host_rm_ops_hv->rm_core); 3684 kfree(kvmppc_host_rm_ops_hv); 3685 kvmppc_host_rm_ops_hv = NULL; 3686 } 3687 } 3688 #endif 3689 3690 static int kvmppc_core_init_vm_hv(struct kvm *kvm) 3691 { 3692 unsigned long lpcr, lpid; 3693 char buf[32]; 3694 int ret; 3695 3696 /* Allocate the guest's logical partition ID */ 3697 3698 lpid = kvmppc_alloc_lpid(); 3699 if ((long)lpid < 0) 3700 return -ENOMEM; 3701 kvm->arch.lpid = lpid; 3702 3703 kvmppc_alloc_host_rm_ops(); 3704 3705 /* 3706 * Since we don't flush the TLB when tearing down a VM, 3707 * and this lpid might have previously been used, 3708 * make sure we flush on each core before running the new VM. 3709 * On POWER9, the tlbie in mmu_partition_table_set_entry() 3710 * does this flush for us. 3711 */ 3712 if (!cpu_has_feature(CPU_FTR_ARCH_300)) 3713 cpumask_setall(&kvm->arch.need_tlb_flush); 3714 3715 /* Start out with the default set of hcalls enabled */ 3716 memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls, 3717 sizeof(kvm->arch.enabled_hcalls)); 3718 3719 if (!cpu_has_feature(CPU_FTR_ARCH_300)) 3720 kvm->arch.host_sdr1 = mfspr(SPRN_SDR1); 3721 3722 /* Init LPCR for virtual RMA mode */ 3723 kvm->arch.host_lpid = mfspr(SPRN_LPID); 3724 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR); 3725 lpcr &= LPCR_PECE | LPCR_LPES; 3726 lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE | 3727 LPCR_VPM0 | LPCR_VPM1; 3728 kvm->arch.vrma_slb_v = SLB_VSID_B_1T | 3729 (VRMA_VSID << SLB_VSID_SHIFT_1T); 3730 /* On POWER8 turn on online bit to enable PURR/SPURR */ 3731 if (cpu_has_feature(CPU_FTR_ARCH_207S)) 3732 lpcr |= LPCR_ONL; 3733 /* 3734 * On POWER9, VPM0 bit is reserved (VPM0=1 behaviour is assumed) 3735 * Set HVICE bit to enable hypervisor virtualization interrupts. 3736 * Set HEIC to prevent OS interrupts to go to hypervisor (should 3737 * be unnecessary but better safe than sorry in case we re-enable 3738 * EE in HV mode with this LPCR still set) 3739 */ 3740 if (cpu_has_feature(CPU_FTR_ARCH_300)) { 3741 lpcr &= ~LPCR_VPM0; 3742 lpcr |= LPCR_HVICE | LPCR_HEIC; 3743 3744 /* 3745 * If xive is enabled, we route 0x500 interrupts directly 3746 * to the guest. 3747 */ 3748 if (xive_enabled()) 3749 lpcr |= LPCR_LPES; 3750 } 3751 3752 /* 3753 * For now, if the host uses radix, the guest must be radix. 3754 */ 3755 if (radix_enabled()) { 3756 kvm->arch.radix = 1; 3757 lpcr &= ~LPCR_VPM1; 3758 lpcr |= LPCR_UPRT | LPCR_GTSE | LPCR_HR; 3759 ret = kvmppc_init_vm_radix(kvm); 3760 if (ret) { 3761 kvmppc_free_lpid(kvm->arch.lpid); 3762 return ret; 3763 } 3764 kvmppc_setup_partition_table(kvm); 3765 } 3766 3767 kvm->arch.lpcr = lpcr; 3768 3769 /* Initialization for future HPT resizes */ 3770 kvm->arch.resize_hpt = NULL; 3771 3772 /* 3773 * Work out how many sets the TLB has, for the use of 3774 * the TLB invalidation loop in book3s_hv_rmhandlers.S. 3775 */ 3776 if (kvm_is_radix(kvm)) 3777 kvm->arch.tlb_sets = POWER9_TLB_SETS_RADIX; /* 128 */ 3778 else if (cpu_has_feature(CPU_FTR_ARCH_300)) 3779 kvm->arch.tlb_sets = POWER9_TLB_SETS_HASH; /* 256 */ 3780 else if (cpu_has_feature(CPU_FTR_ARCH_207S)) 3781 kvm->arch.tlb_sets = POWER8_TLB_SETS; /* 512 */ 3782 else 3783 kvm->arch.tlb_sets = POWER7_TLB_SETS; /* 128 */ 3784 3785 /* 3786 * Track that we now have a HV mode VM active. This blocks secondary 3787 * CPU threads from coming online. 3788 * On POWER9, we only need to do this for HPT guests on a radix 3789 * host, which is not yet supported. 3790 */ 3791 if (!cpu_has_feature(CPU_FTR_ARCH_300)) 3792 kvm_hv_vm_activated(); 3793 3794 /* 3795 * Initialize smt_mode depending on processor. 3796 * POWER8 and earlier have to use "strict" threading, where 3797 * all vCPUs in a vcore have to run on the same (sub)core, 3798 * whereas on POWER9 the threads can each run a different 3799 * guest. 3800 */ 3801 if (!cpu_has_feature(CPU_FTR_ARCH_300)) 3802 kvm->arch.smt_mode = threads_per_subcore; 3803 else 3804 kvm->arch.smt_mode = 1; 3805 kvm->arch.emul_smt_mode = 1; 3806 3807 /* 3808 * Create a debugfs directory for the VM 3809 */ 3810 snprintf(buf, sizeof(buf), "vm%d", current->pid); 3811 kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir); 3812 if (!IS_ERR_OR_NULL(kvm->arch.debugfs_dir)) 3813 kvmppc_mmu_debugfs_init(kvm); 3814 3815 return 0; 3816 } 3817 3818 static void kvmppc_free_vcores(struct kvm *kvm) 3819 { 3820 long int i; 3821 3822 for (i = 0; i < KVM_MAX_VCORES; ++i) 3823 kfree(kvm->arch.vcores[i]); 3824 kvm->arch.online_vcores = 0; 3825 } 3826 3827 static void kvmppc_core_destroy_vm_hv(struct kvm *kvm) 3828 { 3829 debugfs_remove_recursive(kvm->arch.debugfs_dir); 3830 3831 if (!cpu_has_feature(CPU_FTR_ARCH_300)) 3832 kvm_hv_vm_deactivated(); 3833 3834 kvmppc_free_vcores(kvm); 3835 3836 kvmppc_free_lpid(kvm->arch.lpid); 3837 3838 if (kvm_is_radix(kvm)) 3839 kvmppc_free_radix(kvm); 3840 else 3841 kvmppc_free_hpt(&kvm->arch.hpt); 3842 3843 kvmppc_free_pimap(kvm); 3844 } 3845 3846 /* We don't need to emulate any privileged instructions or dcbz */ 3847 static int kvmppc_core_emulate_op_hv(struct kvm_run *run, struct kvm_vcpu *vcpu, 3848 unsigned int inst, int *advance) 3849 { 3850 return EMULATE_FAIL; 3851 } 3852 3853 static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn, 3854 ulong spr_val) 3855 { 3856 return EMULATE_FAIL; 3857 } 3858 3859 static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn, 3860 ulong *spr_val) 3861 { 3862 return EMULATE_FAIL; 3863 } 3864 3865 static int kvmppc_core_check_processor_compat_hv(void) 3866 { 3867 if (!cpu_has_feature(CPU_FTR_HVMODE) || 3868 !cpu_has_feature(CPU_FTR_ARCH_206)) 3869 return -EIO; 3870 3871 return 0; 3872 } 3873 3874 #ifdef CONFIG_KVM_XICS 3875 3876 void kvmppc_free_pimap(struct kvm *kvm) 3877 { 3878 kfree(kvm->arch.pimap); 3879 } 3880 3881 static struct kvmppc_passthru_irqmap *kvmppc_alloc_pimap(void) 3882 { 3883 return kzalloc(sizeof(struct kvmppc_passthru_irqmap), GFP_KERNEL); 3884 } 3885 3886 static int kvmppc_set_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi) 3887 { 3888 struct irq_desc *desc; 3889 struct kvmppc_irq_map *irq_map; 3890 struct kvmppc_passthru_irqmap *pimap; 3891 struct irq_chip *chip; 3892 int i, rc = 0; 3893 3894 if (!kvm_irq_bypass) 3895 return 1; 3896 3897 desc = irq_to_desc(host_irq); 3898 if (!desc) 3899 return -EIO; 3900 3901 mutex_lock(&kvm->lock); 3902 3903 pimap = kvm->arch.pimap; 3904 if (pimap == NULL) { 3905 /* First call, allocate structure to hold IRQ map */ 3906 pimap = kvmppc_alloc_pimap(); 3907 if (pimap == NULL) { 3908 mutex_unlock(&kvm->lock); 3909 return -ENOMEM; 3910 } 3911 kvm->arch.pimap = pimap; 3912 } 3913 3914 /* 3915 * For now, we only support interrupts for which the EOI operation 3916 * is an OPAL call followed by a write to XIRR, since that's 3917 * what our real-mode EOI code does, or a XIVE interrupt 3918 */ 3919 chip = irq_data_get_irq_chip(&desc->irq_data); 3920 if (!chip || !(is_pnv_opal_msi(chip) || is_xive_irq(chip))) { 3921 pr_warn("kvmppc_set_passthru_irq_hv: Could not assign IRQ map for (%d,%d)\n", 3922 host_irq, guest_gsi); 3923 mutex_unlock(&kvm->lock); 3924 return -ENOENT; 3925 } 3926 3927 /* 3928 * See if we already have an entry for this guest IRQ number. 3929 * If it's mapped to a hardware IRQ number, that's an error, 3930 * otherwise re-use this entry. 3931 */ 3932 for (i = 0; i < pimap->n_mapped; i++) { 3933 if (guest_gsi == pimap->mapped[i].v_hwirq) { 3934 if (pimap->mapped[i].r_hwirq) { 3935 mutex_unlock(&kvm->lock); 3936 return -EINVAL; 3937 } 3938 break; 3939 } 3940 } 3941 3942 if (i == KVMPPC_PIRQ_MAPPED) { 3943 mutex_unlock(&kvm->lock); 3944 return -EAGAIN; /* table is full */ 3945 } 3946 3947 irq_map = &pimap->mapped[i]; 3948 3949 irq_map->v_hwirq = guest_gsi; 3950 irq_map->desc = desc; 3951 3952 /* 3953 * Order the above two stores before the next to serialize with 3954 * the KVM real mode handler. 3955 */ 3956 smp_wmb(); 3957 irq_map->r_hwirq = desc->irq_data.hwirq; 3958 3959 if (i == pimap->n_mapped) 3960 pimap->n_mapped++; 3961 3962 if (xive_enabled()) 3963 rc = kvmppc_xive_set_mapped(kvm, guest_gsi, desc); 3964 else 3965 kvmppc_xics_set_mapped(kvm, guest_gsi, desc->irq_data.hwirq); 3966 if (rc) 3967 irq_map->r_hwirq = 0; 3968 3969 mutex_unlock(&kvm->lock); 3970 3971 return 0; 3972 } 3973 3974 static int kvmppc_clr_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi) 3975 { 3976 struct irq_desc *desc; 3977 struct kvmppc_passthru_irqmap *pimap; 3978 int i, rc = 0; 3979 3980 if (!kvm_irq_bypass) 3981 return 0; 3982 3983 desc = irq_to_desc(host_irq); 3984 if (!desc) 3985 return -EIO; 3986 3987 mutex_lock(&kvm->lock); 3988 if (!kvm->arch.pimap) 3989 goto unlock; 3990 3991 pimap = kvm->arch.pimap; 3992 3993 for (i = 0; i < pimap->n_mapped; i++) { 3994 if (guest_gsi == pimap->mapped[i].v_hwirq) 3995 break; 3996 } 3997 3998 if (i == pimap->n_mapped) { 3999 mutex_unlock(&kvm->lock); 4000 return -ENODEV; 4001 } 4002 4003 if (xive_enabled()) 4004 rc = kvmppc_xive_clr_mapped(kvm, guest_gsi, pimap->mapped[i].desc); 4005 else 4006 kvmppc_xics_clr_mapped(kvm, guest_gsi, pimap->mapped[i].r_hwirq); 4007 4008 /* invalidate the entry (what do do on error from the above ?) */ 4009 pimap->mapped[i].r_hwirq = 0; 4010 4011 /* 4012 * We don't free this structure even when the count goes to 4013 * zero. The structure is freed when we destroy the VM. 4014 */ 4015 unlock: 4016 mutex_unlock(&kvm->lock); 4017 return rc; 4018 } 4019 4020 static int kvmppc_irq_bypass_add_producer_hv(struct irq_bypass_consumer *cons, 4021 struct irq_bypass_producer *prod) 4022 { 4023 int ret = 0; 4024 struct kvm_kernel_irqfd *irqfd = 4025 container_of(cons, struct kvm_kernel_irqfd, consumer); 4026 4027 irqfd->producer = prod; 4028 4029 ret = kvmppc_set_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi); 4030 if (ret) 4031 pr_info("kvmppc_set_passthru_irq (irq %d, gsi %d) fails: %d\n", 4032 prod->irq, irqfd->gsi, ret); 4033 4034 return ret; 4035 } 4036 4037 static void kvmppc_irq_bypass_del_producer_hv(struct irq_bypass_consumer *cons, 4038 struct irq_bypass_producer *prod) 4039 { 4040 int ret; 4041 struct kvm_kernel_irqfd *irqfd = 4042 container_of(cons, struct kvm_kernel_irqfd, consumer); 4043 4044 irqfd->producer = NULL; 4045 4046 /* 4047 * When producer of consumer is unregistered, we change back to 4048 * default external interrupt handling mode - KVM real mode 4049 * will switch back to host. 4050 */ 4051 ret = kvmppc_clr_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi); 4052 if (ret) 4053 pr_warn("kvmppc_clr_passthru_irq (irq %d, gsi %d) fails: %d\n", 4054 prod->irq, irqfd->gsi, ret); 4055 } 4056 #endif 4057 4058 static long kvm_arch_vm_ioctl_hv(struct file *filp, 4059 unsigned int ioctl, unsigned long arg) 4060 { 4061 struct kvm *kvm __maybe_unused = filp->private_data; 4062 void __user *argp = (void __user *)arg; 4063 long r; 4064 4065 switch (ioctl) { 4066 4067 case KVM_PPC_ALLOCATE_HTAB: { 4068 u32 htab_order; 4069 4070 r = -EFAULT; 4071 if (get_user(htab_order, (u32 __user *)argp)) 4072 break; 4073 r = kvmppc_alloc_reset_hpt(kvm, htab_order); 4074 if (r) 4075 break; 4076 r = 0; 4077 break; 4078 } 4079 4080 case KVM_PPC_GET_HTAB_FD: { 4081 struct kvm_get_htab_fd ghf; 4082 4083 r = -EFAULT; 4084 if (copy_from_user(&ghf, argp, sizeof(ghf))) 4085 break; 4086 r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf); 4087 break; 4088 } 4089 4090 case KVM_PPC_RESIZE_HPT_PREPARE: { 4091 struct kvm_ppc_resize_hpt rhpt; 4092 4093 r = -EFAULT; 4094 if (copy_from_user(&rhpt, argp, sizeof(rhpt))) 4095 break; 4096 4097 r = kvm_vm_ioctl_resize_hpt_prepare(kvm, &rhpt); 4098 break; 4099 } 4100 4101 case KVM_PPC_RESIZE_HPT_COMMIT: { 4102 struct kvm_ppc_resize_hpt rhpt; 4103 4104 r = -EFAULT; 4105 if (copy_from_user(&rhpt, argp, sizeof(rhpt))) 4106 break; 4107 4108 r = kvm_vm_ioctl_resize_hpt_commit(kvm, &rhpt); 4109 break; 4110 } 4111 4112 default: 4113 r = -ENOTTY; 4114 } 4115 4116 return r; 4117 } 4118 4119 /* 4120 * List of hcall numbers to enable by default. 4121 * For compatibility with old userspace, we enable by default 4122 * all hcalls that were implemented before the hcall-enabling 4123 * facility was added. Note this list should not include H_RTAS. 4124 */ 4125 static unsigned int default_hcall_list[] = { 4126 H_REMOVE, 4127 H_ENTER, 4128 H_READ, 4129 H_PROTECT, 4130 H_BULK_REMOVE, 4131 H_GET_TCE, 4132 H_PUT_TCE, 4133 H_SET_DABR, 4134 H_SET_XDABR, 4135 H_CEDE, 4136 H_PROD, 4137 H_CONFER, 4138 H_REGISTER_VPA, 4139 #ifdef CONFIG_KVM_XICS 4140 H_EOI, 4141 H_CPPR, 4142 H_IPI, 4143 H_IPOLL, 4144 H_XIRR, 4145 H_XIRR_X, 4146 #endif 4147 0 4148 }; 4149 4150 static void init_default_hcalls(void) 4151 { 4152 int i; 4153 unsigned int hcall; 4154 4155 for (i = 0; default_hcall_list[i]; ++i) { 4156 hcall = default_hcall_list[i]; 4157 WARN_ON(!kvmppc_hcall_impl_hv(hcall)); 4158 __set_bit(hcall / 4, default_enabled_hcalls); 4159 } 4160 } 4161 4162 static int kvmhv_configure_mmu(struct kvm *kvm, struct kvm_ppc_mmuv3_cfg *cfg) 4163 { 4164 unsigned long lpcr; 4165 int radix; 4166 4167 /* If not on a POWER9, reject it */ 4168 if (!cpu_has_feature(CPU_FTR_ARCH_300)) 4169 return -ENODEV; 4170 4171 /* If any unknown flags set, reject it */ 4172 if (cfg->flags & ~(KVM_PPC_MMUV3_RADIX | KVM_PPC_MMUV3_GTSE)) 4173 return -EINVAL; 4174 4175 /* We can't change a guest to/from radix yet */ 4176 radix = !!(cfg->flags & KVM_PPC_MMUV3_RADIX); 4177 if (radix != kvm_is_radix(kvm)) 4178 return -EINVAL; 4179 4180 /* GR (guest radix) bit in process_table field must match */ 4181 if (!!(cfg->process_table & PATB_GR) != radix) 4182 return -EINVAL; 4183 4184 /* Process table size field must be reasonable, i.e. <= 24 */ 4185 if ((cfg->process_table & PRTS_MASK) > 24) 4186 return -EINVAL; 4187 4188 kvm->arch.process_table = cfg->process_table; 4189 kvmppc_setup_partition_table(kvm); 4190 4191 lpcr = (cfg->flags & KVM_PPC_MMUV3_GTSE) ? LPCR_GTSE : 0; 4192 kvmppc_update_lpcr(kvm, lpcr, LPCR_GTSE); 4193 4194 return 0; 4195 } 4196 4197 static struct kvmppc_ops kvm_ops_hv = { 4198 .get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv, 4199 .set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv, 4200 .get_one_reg = kvmppc_get_one_reg_hv, 4201 .set_one_reg = kvmppc_set_one_reg_hv, 4202 .vcpu_load = kvmppc_core_vcpu_load_hv, 4203 .vcpu_put = kvmppc_core_vcpu_put_hv, 4204 .set_msr = kvmppc_set_msr_hv, 4205 .vcpu_run = kvmppc_vcpu_run_hv, 4206 .vcpu_create = kvmppc_core_vcpu_create_hv, 4207 .vcpu_free = kvmppc_core_vcpu_free_hv, 4208 .check_requests = kvmppc_core_check_requests_hv, 4209 .get_dirty_log = kvm_vm_ioctl_get_dirty_log_hv, 4210 .flush_memslot = kvmppc_core_flush_memslot_hv, 4211 .prepare_memory_region = kvmppc_core_prepare_memory_region_hv, 4212 .commit_memory_region = kvmppc_core_commit_memory_region_hv, 4213 .unmap_hva = kvm_unmap_hva_hv, 4214 .unmap_hva_range = kvm_unmap_hva_range_hv, 4215 .age_hva = kvm_age_hva_hv, 4216 .test_age_hva = kvm_test_age_hva_hv, 4217 .set_spte_hva = kvm_set_spte_hva_hv, 4218 .mmu_destroy = kvmppc_mmu_destroy_hv, 4219 .free_memslot = kvmppc_core_free_memslot_hv, 4220 .create_memslot = kvmppc_core_create_memslot_hv, 4221 .init_vm = kvmppc_core_init_vm_hv, 4222 .destroy_vm = kvmppc_core_destroy_vm_hv, 4223 .get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv, 4224 .emulate_op = kvmppc_core_emulate_op_hv, 4225 .emulate_mtspr = kvmppc_core_emulate_mtspr_hv, 4226 .emulate_mfspr = kvmppc_core_emulate_mfspr_hv, 4227 .fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv, 4228 .arch_vm_ioctl = kvm_arch_vm_ioctl_hv, 4229 .hcall_implemented = kvmppc_hcall_impl_hv, 4230 #ifdef CONFIG_KVM_XICS 4231 .irq_bypass_add_producer = kvmppc_irq_bypass_add_producer_hv, 4232 .irq_bypass_del_producer = kvmppc_irq_bypass_del_producer_hv, 4233 #endif 4234 .configure_mmu = kvmhv_configure_mmu, 4235 .get_rmmu_info = kvmhv_get_rmmu_info, 4236 .set_smt_mode = kvmhv_set_smt_mode, 4237 }; 4238 4239 static int kvm_init_subcore_bitmap(void) 4240 { 4241 int i, j; 4242 int nr_cores = cpu_nr_cores(); 4243 struct sibling_subcore_state *sibling_subcore_state; 4244 4245 for (i = 0; i < nr_cores; i++) { 4246 int first_cpu = i * threads_per_core; 4247 int node = cpu_to_node(first_cpu); 4248 4249 /* Ignore if it is already allocated. */ 4250 if (paca[first_cpu].sibling_subcore_state) 4251 continue; 4252 4253 sibling_subcore_state = 4254 kmalloc_node(sizeof(struct sibling_subcore_state), 4255 GFP_KERNEL, node); 4256 if (!sibling_subcore_state) 4257 return -ENOMEM; 4258 4259 memset(sibling_subcore_state, 0, 4260 sizeof(struct sibling_subcore_state)); 4261 4262 for (j = 0; j < threads_per_core; j++) { 4263 int cpu = first_cpu + j; 4264 4265 paca[cpu].sibling_subcore_state = sibling_subcore_state; 4266 } 4267 } 4268 return 0; 4269 } 4270 4271 static int kvmppc_radix_possible(void) 4272 { 4273 return cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled(); 4274 } 4275 4276 static int kvmppc_book3s_init_hv(void) 4277 { 4278 int r; 4279 /* 4280 * FIXME!! Do we need to check on all cpus ? 4281 */ 4282 r = kvmppc_core_check_processor_compat_hv(); 4283 if (r < 0) 4284 return -ENODEV; 4285 4286 r = kvm_init_subcore_bitmap(); 4287 if (r) 4288 return r; 4289 4290 /* 4291 * We need a way of accessing the XICS interrupt controller, 4292 * either directly, via paca[cpu].kvm_hstate.xics_phys, or 4293 * indirectly, via OPAL. 4294 */ 4295 #ifdef CONFIG_SMP 4296 if (!xive_enabled() && !local_paca->kvm_hstate.xics_phys) { 4297 struct device_node *np; 4298 4299 np = of_find_compatible_node(NULL, NULL, "ibm,opal-intc"); 4300 if (!np) { 4301 pr_err("KVM-HV: Cannot determine method for accessing XICS\n"); 4302 return -ENODEV; 4303 } 4304 } 4305 #endif 4306 4307 kvm_ops_hv.owner = THIS_MODULE; 4308 kvmppc_hv_ops = &kvm_ops_hv; 4309 4310 init_default_hcalls(); 4311 4312 init_vcore_lists(); 4313 4314 r = kvmppc_mmu_hv_init(); 4315 if (r) 4316 return r; 4317 4318 if (kvmppc_radix_possible()) 4319 r = kvmppc_radix_init(); 4320 return r; 4321 } 4322 4323 static void kvmppc_book3s_exit_hv(void) 4324 { 4325 kvmppc_free_host_rm_ops(); 4326 if (kvmppc_radix_possible()) 4327 kvmppc_radix_exit(); 4328 kvmppc_hv_ops = NULL; 4329 } 4330 4331 module_init(kvmppc_book3s_init_hv); 4332 module_exit(kvmppc_book3s_exit_hv); 4333 MODULE_LICENSE("GPL"); 4334 MODULE_ALIAS_MISCDEV(KVM_MINOR); 4335 MODULE_ALIAS("devname:kvm"); 4336 4337