1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2017-2019, IBM Corporation. 4 */ 5 6 #define pr_fmt(fmt) "xive-kvm: " fmt 7 8 #include <linux/kernel.h> 9 #include <linux/kvm_host.h> 10 #include <linux/err.h> 11 #include <linux/gfp.h> 12 #include <linux/spinlock.h> 13 #include <linux/delay.h> 14 #include <linux/file.h> 15 #include <linux/irqdomain.h> 16 #include <asm/uaccess.h> 17 #include <asm/kvm_book3s.h> 18 #include <asm/kvm_ppc.h> 19 #include <asm/hvcall.h> 20 #include <asm/xive.h> 21 #include <asm/xive-regs.h> 22 #include <asm/debug.h> 23 #include <asm/opal.h> 24 25 #include <linux/debugfs.h> 26 #include <linux/seq_file.h> 27 28 #include "book3s_xive.h" 29 30 static u8 xive_vm_esb_load(struct xive_irq_data *xd, u32 offset) 31 { 32 u64 val; 33 34 /* 35 * The KVM XIVE native device does not use the XIVE_ESB_SET_PQ_10 36 * load operation, so there is no need to enforce load-after-store 37 * ordering. 38 */ 39 40 val = in_be64(xd->eoi_mmio + offset); 41 return (u8)val; 42 } 43 44 static void kvmppc_xive_native_cleanup_queue(struct kvm_vcpu *vcpu, int prio) 45 { 46 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 47 struct xive_q *q = &xc->queues[prio]; 48 49 xive_native_disable_queue(xc->vp_id, q, prio); 50 if (q->qpage) { 51 put_page(virt_to_page(q->qpage)); 52 q->qpage = NULL; 53 } 54 } 55 56 static int kvmppc_xive_native_configure_queue(u32 vp_id, struct xive_q *q, 57 u8 prio, __be32 *qpage, 58 u32 order, bool can_escalate) 59 { 60 int rc; 61 __be32 *qpage_prev = q->qpage; 62 63 rc = xive_native_configure_queue(vp_id, q, prio, qpage, order, 64 can_escalate); 65 if (rc) 66 return rc; 67 68 if (qpage_prev) 69 put_page(virt_to_page(qpage_prev)); 70 71 return rc; 72 } 73 74 void kvmppc_xive_native_cleanup_vcpu(struct kvm_vcpu *vcpu) 75 { 76 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 77 int i; 78 79 if (!kvmppc_xive_enabled(vcpu)) 80 return; 81 82 if (!xc) 83 return; 84 85 pr_devel("native_cleanup_vcpu(cpu=%d)\n", xc->server_num); 86 87 /* Ensure no interrupt is still routed to that VP */ 88 xc->valid = false; 89 kvmppc_xive_disable_vcpu_interrupts(vcpu); 90 91 /* Free escalations */ 92 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) { 93 /* Free the escalation irq */ 94 if (xc->esc_virq[i]) { 95 if (kvmppc_xive_has_single_escalation(xc->xive)) 96 xive_cleanup_single_escalation(vcpu, xc->esc_virq[i]); 97 free_irq(xc->esc_virq[i], vcpu); 98 irq_dispose_mapping(xc->esc_virq[i]); 99 kfree(xc->esc_virq_names[i]); 100 xc->esc_virq[i] = 0; 101 } 102 } 103 104 /* Disable the VP */ 105 xive_native_disable_vp(xc->vp_id); 106 107 /* Clear the cam word so guest entry won't try to push context */ 108 vcpu->arch.xive_cam_word = 0; 109 110 /* Free the queues */ 111 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) { 112 kvmppc_xive_native_cleanup_queue(vcpu, i); 113 } 114 115 /* Free the VP */ 116 kfree(xc); 117 118 /* Cleanup the vcpu */ 119 vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT; 120 vcpu->arch.xive_vcpu = NULL; 121 } 122 123 int kvmppc_xive_native_connect_vcpu(struct kvm_device *dev, 124 struct kvm_vcpu *vcpu, u32 server_num) 125 { 126 struct kvmppc_xive *xive = dev->private; 127 struct kvmppc_xive_vcpu *xc = NULL; 128 int rc; 129 u32 vp_id; 130 131 pr_devel("native_connect_vcpu(server=%d)\n", server_num); 132 133 if (dev->ops != &kvm_xive_native_ops) { 134 pr_devel("Wrong ops !\n"); 135 return -EPERM; 136 } 137 if (xive->kvm != vcpu->kvm) 138 return -EPERM; 139 if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT) 140 return -EBUSY; 141 142 mutex_lock(&xive->lock); 143 144 rc = kvmppc_xive_compute_vp_id(xive, server_num, &vp_id); 145 if (rc) 146 goto bail; 147 148 xc = kzalloc(sizeof(*xc), GFP_KERNEL); 149 if (!xc) { 150 rc = -ENOMEM; 151 goto bail; 152 } 153 154 vcpu->arch.xive_vcpu = xc; 155 xc->xive = xive; 156 xc->vcpu = vcpu; 157 xc->server_num = server_num; 158 159 xc->vp_id = vp_id; 160 xc->valid = true; 161 vcpu->arch.irq_type = KVMPPC_IRQ_XIVE; 162 163 rc = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id); 164 if (rc) { 165 pr_err("Failed to get VP info from OPAL: %d\n", rc); 166 goto bail; 167 } 168 169 if (!kvmppc_xive_check_save_restore(vcpu)) { 170 pr_err("inconsistent save-restore setup for VCPU %d\n", server_num); 171 rc = -EIO; 172 goto bail; 173 } 174 175 /* 176 * Enable the VP first as the single escalation mode will 177 * affect escalation interrupts numbering 178 */ 179 rc = xive_native_enable_vp(xc->vp_id, kvmppc_xive_has_single_escalation(xive)); 180 if (rc) { 181 pr_err("Failed to enable VP in OPAL: %d\n", rc); 182 goto bail; 183 } 184 185 /* Configure VCPU fields for use by assembly push/pull */ 186 vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000); 187 vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO); 188 189 /* TODO: reset all queues to a clean state ? */ 190 bail: 191 mutex_unlock(&xive->lock); 192 if (rc) 193 kvmppc_xive_native_cleanup_vcpu(vcpu); 194 195 return rc; 196 } 197 198 /* 199 * Device passthrough support 200 */ 201 static int kvmppc_xive_native_reset_mapped(struct kvm *kvm, unsigned long irq) 202 { 203 struct kvmppc_xive *xive = kvm->arch.xive; 204 pgoff_t esb_pgoff = KVM_XIVE_ESB_PAGE_OFFSET + irq * 2; 205 206 if (irq >= KVMPPC_XIVE_NR_IRQS) 207 return -EINVAL; 208 209 /* 210 * Clear the ESB pages of the IRQ number being mapped (or 211 * unmapped) into the guest and let the VM fault handler 212 * repopulate with the appropriate ESB pages (device or IC) 213 */ 214 pr_debug("clearing esb pages for girq 0x%lx\n", irq); 215 mutex_lock(&xive->mapping_lock); 216 if (xive->mapping) 217 unmap_mapping_range(xive->mapping, 218 esb_pgoff << PAGE_SHIFT, 219 2ull << PAGE_SHIFT, 1); 220 mutex_unlock(&xive->mapping_lock); 221 return 0; 222 } 223 224 static struct kvmppc_xive_ops kvmppc_xive_native_ops = { 225 .reset_mapped = kvmppc_xive_native_reset_mapped, 226 }; 227 228 static vm_fault_t xive_native_esb_fault(struct vm_fault *vmf) 229 { 230 struct vm_area_struct *vma = vmf->vma; 231 struct kvm_device *dev = vma->vm_file->private_data; 232 struct kvmppc_xive *xive = dev->private; 233 struct kvmppc_xive_src_block *sb; 234 struct kvmppc_xive_irq_state *state; 235 struct xive_irq_data *xd; 236 u32 hw_num; 237 u16 src; 238 u64 page; 239 unsigned long irq; 240 u64 page_offset; 241 242 /* 243 * Linux/KVM uses a two pages ESB setting, one for trigger and 244 * one for EOI 245 */ 246 page_offset = vmf->pgoff - vma->vm_pgoff; 247 irq = page_offset / 2; 248 249 sb = kvmppc_xive_find_source(xive, irq, &src); 250 if (!sb) { 251 pr_devel("%s: source %lx not found !\n", __func__, irq); 252 return VM_FAULT_SIGBUS; 253 } 254 255 state = &sb->irq_state[src]; 256 257 /* Some sanity checking */ 258 if (!state->valid) { 259 pr_devel("%s: source %lx invalid !\n", __func__, irq); 260 return VM_FAULT_SIGBUS; 261 } 262 263 kvmppc_xive_select_irq(state, &hw_num, &xd); 264 265 arch_spin_lock(&sb->lock); 266 267 /* 268 * first/even page is for trigger 269 * second/odd page is for EOI and management. 270 */ 271 page = page_offset % 2 ? xd->eoi_page : xd->trig_page; 272 arch_spin_unlock(&sb->lock); 273 274 if (WARN_ON(!page)) { 275 pr_err("%s: accessing invalid ESB page for source %lx !\n", 276 __func__, irq); 277 return VM_FAULT_SIGBUS; 278 } 279 280 vmf_insert_pfn(vma, vmf->address, page >> PAGE_SHIFT); 281 return VM_FAULT_NOPAGE; 282 } 283 284 static const struct vm_operations_struct xive_native_esb_vmops = { 285 .fault = xive_native_esb_fault, 286 }; 287 288 static vm_fault_t xive_native_tima_fault(struct vm_fault *vmf) 289 { 290 struct vm_area_struct *vma = vmf->vma; 291 292 switch (vmf->pgoff - vma->vm_pgoff) { 293 case 0: /* HW - forbid access */ 294 case 1: /* HV - forbid access */ 295 return VM_FAULT_SIGBUS; 296 case 2: /* OS */ 297 vmf_insert_pfn(vma, vmf->address, xive_tima_os >> PAGE_SHIFT); 298 return VM_FAULT_NOPAGE; 299 case 3: /* USER - TODO */ 300 default: 301 return VM_FAULT_SIGBUS; 302 } 303 } 304 305 static const struct vm_operations_struct xive_native_tima_vmops = { 306 .fault = xive_native_tima_fault, 307 }; 308 309 static int kvmppc_xive_native_mmap(struct kvm_device *dev, 310 struct vm_area_struct *vma) 311 { 312 struct kvmppc_xive *xive = dev->private; 313 314 /* We only allow mappings at fixed offset for now */ 315 if (vma->vm_pgoff == KVM_XIVE_TIMA_PAGE_OFFSET) { 316 if (vma_pages(vma) > 4) 317 return -EINVAL; 318 vma->vm_ops = &xive_native_tima_vmops; 319 } else if (vma->vm_pgoff == KVM_XIVE_ESB_PAGE_OFFSET) { 320 if (vma_pages(vma) > KVMPPC_XIVE_NR_IRQS * 2) 321 return -EINVAL; 322 vma->vm_ops = &xive_native_esb_vmops; 323 } else { 324 return -EINVAL; 325 } 326 327 vma->vm_flags |= VM_IO | VM_PFNMAP; 328 vma->vm_page_prot = pgprot_noncached_wc(vma->vm_page_prot); 329 330 /* 331 * Grab the KVM device file address_space to be able to clear 332 * the ESB pages mapping when a device is passed-through into 333 * the guest. 334 */ 335 xive->mapping = vma->vm_file->f_mapping; 336 return 0; 337 } 338 339 static int kvmppc_xive_native_set_source(struct kvmppc_xive *xive, long irq, 340 u64 addr) 341 { 342 struct kvmppc_xive_src_block *sb; 343 struct kvmppc_xive_irq_state *state; 344 u64 __user *ubufp = (u64 __user *) addr; 345 u64 val; 346 u16 idx; 347 int rc; 348 349 pr_devel("%s irq=0x%lx\n", __func__, irq); 350 351 if (irq < KVMPPC_XIVE_FIRST_IRQ || irq >= KVMPPC_XIVE_NR_IRQS) 352 return -E2BIG; 353 354 sb = kvmppc_xive_find_source(xive, irq, &idx); 355 if (!sb) { 356 pr_debug("No source, creating source block...\n"); 357 sb = kvmppc_xive_create_src_block(xive, irq); 358 if (!sb) { 359 pr_err("Failed to create block...\n"); 360 return -ENOMEM; 361 } 362 } 363 state = &sb->irq_state[idx]; 364 365 if (get_user(val, ubufp)) { 366 pr_err("fault getting user info !\n"); 367 return -EFAULT; 368 } 369 370 arch_spin_lock(&sb->lock); 371 372 /* 373 * If the source doesn't already have an IPI, allocate 374 * one and get the corresponding data 375 */ 376 if (!state->ipi_number) { 377 state->ipi_number = xive_native_alloc_irq(); 378 if (state->ipi_number == 0) { 379 pr_err("Failed to allocate IRQ !\n"); 380 rc = -ENXIO; 381 goto unlock; 382 } 383 xive_native_populate_irq_data(state->ipi_number, 384 &state->ipi_data); 385 pr_debug("%s allocated hw_irq=0x%x for irq=0x%lx\n", __func__, 386 state->ipi_number, irq); 387 } 388 389 /* Restore LSI state */ 390 if (val & KVM_XIVE_LEVEL_SENSITIVE) { 391 state->lsi = true; 392 if (val & KVM_XIVE_LEVEL_ASSERTED) 393 state->asserted = true; 394 pr_devel(" LSI ! Asserted=%d\n", state->asserted); 395 } 396 397 /* Mask IRQ to start with */ 398 state->act_server = 0; 399 state->act_priority = MASKED; 400 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01); 401 xive_native_configure_irq(state->ipi_number, 0, MASKED, 0); 402 403 /* Increment the number of valid sources and mark this one valid */ 404 if (!state->valid) 405 xive->src_count++; 406 state->valid = true; 407 408 rc = 0; 409 410 unlock: 411 arch_spin_unlock(&sb->lock); 412 413 return rc; 414 } 415 416 static int kvmppc_xive_native_update_source_config(struct kvmppc_xive *xive, 417 struct kvmppc_xive_src_block *sb, 418 struct kvmppc_xive_irq_state *state, 419 u32 server, u8 priority, bool masked, 420 u32 eisn) 421 { 422 struct kvm *kvm = xive->kvm; 423 u32 hw_num; 424 int rc = 0; 425 426 arch_spin_lock(&sb->lock); 427 428 if (state->act_server == server && state->act_priority == priority && 429 state->eisn == eisn) 430 goto unlock; 431 432 pr_devel("new_act_prio=%d new_act_server=%d mask=%d act_server=%d act_prio=%d\n", 433 priority, server, masked, state->act_server, 434 state->act_priority); 435 436 kvmppc_xive_select_irq(state, &hw_num, NULL); 437 438 if (priority != MASKED && !masked) { 439 rc = kvmppc_xive_select_target(kvm, &server, priority); 440 if (rc) 441 goto unlock; 442 443 state->act_priority = priority; 444 state->act_server = server; 445 state->eisn = eisn; 446 447 rc = xive_native_configure_irq(hw_num, 448 kvmppc_xive_vp(xive, server), 449 priority, eisn); 450 } else { 451 state->act_priority = MASKED; 452 state->act_server = 0; 453 state->eisn = 0; 454 455 rc = xive_native_configure_irq(hw_num, 0, MASKED, 0); 456 } 457 458 unlock: 459 arch_spin_unlock(&sb->lock); 460 return rc; 461 } 462 463 static int kvmppc_xive_native_set_source_config(struct kvmppc_xive *xive, 464 long irq, u64 addr) 465 { 466 struct kvmppc_xive_src_block *sb; 467 struct kvmppc_xive_irq_state *state; 468 u64 __user *ubufp = (u64 __user *) addr; 469 u16 src; 470 u64 kvm_cfg; 471 u32 server; 472 u8 priority; 473 bool masked; 474 u32 eisn; 475 476 sb = kvmppc_xive_find_source(xive, irq, &src); 477 if (!sb) 478 return -ENOENT; 479 480 state = &sb->irq_state[src]; 481 482 if (!state->valid) 483 return -EINVAL; 484 485 if (get_user(kvm_cfg, ubufp)) 486 return -EFAULT; 487 488 pr_devel("%s irq=0x%lx cfg=%016llx\n", __func__, irq, kvm_cfg); 489 490 priority = (kvm_cfg & KVM_XIVE_SOURCE_PRIORITY_MASK) >> 491 KVM_XIVE_SOURCE_PRIORITY_SHIFT; 492 server = (kvm_cfg & KVM_XIVE_SOURCE_SERVER_MASK) >> 493 KVM_XIVE_SOURCE_SERVER_SHIFT; 494 masked = (kvm_cfg & KVM_XIVE_SOURCE_MASKED_MASK) >> 495 KVM_XIVE_SOURCE_MASKED_SHIFT; 496 eisn = (kvm_cfg & KVM_XIVE_SOURCE_EISN_MASK) >> 497 KVM_XIVE_SOURCE_EISN_SHIFT; 498 499 if (priority != xive_prio_from_guest(priority)) { 500 pr_err("invalid priority for queue %d for VCPU %d\n", 501 priority, server); 502 return -EINVAL; 503 } 504 505 return kvmppc_xive_native_update_source_config(xive, sb, state, server, 506 priority, masked, eisn); 507 } 508 509 static int kvmppc_xive_native_sync_source(struct kvmppc_xive *xive, 510 long irq, u64 addr) 511 { 512 struct kvmppc_xive_src_block *sb; 513 struct kvmppc_xive_irq_state *state; 514 struct xive_irq_data *xd; 515 u32 hw_num; 516 u16 src; 517 int rc = 0; 518 519 pr_devel("%s irq=0x%lx", __func__, irq); 520 521 sb = kvmppc_xive_find_source(xive, irq, &src); 522 if (!sb) 523 return -ENOENT; 524 525 state = &sb->irq_state[src]; 526 527 rc = -EINVAL; 528 529 arch_spin_lock(&sb->lock); 530 531 if (state->valid) { 532 kvmppc_xive_select_irq(state, &hw_num, &xd); 533 xive_native_sync_source(hw_num); 534 rc = 0; 535 } 536 537 arch_spin_unlock(&sb->lock); 538 return rc; 539 } 540 541 static int xive_native_validate_queue_size(u32 qshift) 542 { 543 /* 544 * We only support 64K pages for the moment. This is also 545 * advertised in the DT property "ibm,xive-eq-sizes" 546 */ 547 switch (qshift) { 548 case 0: /* EQ reset */ 549 case 16: 550 return 0; 551 case 12: 552 case 21: 553 case 24: 554 default: 555 return -EINVAL; 556 } 557 } 558 559 static int kvmppc_xive_native_set_queue_config(struct kvmppc_xive *xive, 560 long eq_idx, u64 addr) 561 { 562 struct kvm *kvm = xive->kvm; 563 struct kvm_vcpu *vcpu; 564 struct kvmppc_xive_vcpu *xc; 565 void __user *ubufp = (void __user *) addr; 566 u32 server; 567 u8 priority; 568 struct kvm_ppc_xive_eq kvm_eq; 569 int rc; 570 __be32 *qaddr = 0; 571 struct page *page; 572 struct xive_q *q; 573 gfn_t gfn; 574 unsigned long page_size; 575 int srcu_idx; 576 577 /* 578 * Demangle priority/server tuple from the EQ identifier 579 */ 580 priority = (eq_idx & KVM_XIVE_EQ_PRIORITY_MASK) >> 581 KVM_XIVE_EQ_PRIORITY_SHIFT; 582 server = (eq_idx & KVM_XIVE_EQ_SERVER_MASK) >> 583 KVM_XIVE_EQ_SERVER_SHIFT; 584 585 if (copy_from_user(&kvm_eq, ubufp, sizeof(kvm_eq))) 586 return -EFAULT; 587 588 vcpu = kvmppc_xive_find_server(kvm, server); 589 if (!vcpu) { 590 pr_err("Can't find server %d\n", server); 591 return -ENOENT; 592 } 593 xc = vcpu->arch.xive_vcpu; 594 595 if (priority != xive_prio_from_guest(priority)) { 596 pr_err("Trying to restore invalid queue %d for VCPU %d\n", 597 priority, server); 598 return -EINVAL; 599 } 600 q = &xc->queues[priority]; 601 602 pr_devel("%s VCPU %d priority %d fl:%x shift:%d addr:%llx g:%d idx:%d\n", 603 __func__, server, priority, kvm_eq.flags, 604 kvm_eq.qshift, kvm_eq.qaddr, kvm_eq.qtoggle, kvm_eq.qindex); 605 606 /* reset queue and disable queueing */ 607 if (!kvm_eq.qshift) { 608 q->guest_qaddr = 0; 609 q->guest_qshift = 0; 610 611 rc = kvmppc_xive_native_configure_queue(xc->vp_id, q, priority, 612 NULL, 0, true); 613 if (rc) { 614 pr_err("Failed to reset queue %d for VCPU %d: %d\n", 615 priority, xc->server_num, rc); 616 return rc; 617 } 618 619 return 0; 620 } 621 622 /* 623 * sPAPR specifies a "Unconditional Notify (n) flag" for the 624 * H_INT_SET_QUEUE_CONFIG hcall which forces notification 625 * without using the coalescing mechanisms provided by the 626 * XIVE END ESBs. This is required on KVM as notification 627 * using the END ESBs is not supported. 628 */ 629 if (kvm_eq.flags != KVM_XIVE_EQ_ALWAYS_NOTIFY) { 630 pr_err("invalid flags %d\n", kvm_eq.flags); 631 return -EINVAL; 632 } 633 634 rc = xive_native_validate_queue_size(kvm_eq.qshift); 635 if (rc) { 636 pr_err("invalid queue size %d\n", kvm_eq.qshift); 637 return rc; 638 } 639 640 if (kvm_eq.qaddr & ((1ull << kvm_eq.qshift) - 1)) { 641 pr_err("queue page is not aligned %llx/%llx\n", kvm_eq.qaddr, 642 1ull << kvm_eq.qshift); 643 return -EINVAL; 644 } 645 646 srcu_idx = srcu_read_lock(&kvm->srcu); 647 gfn = gpa_to_gfn(kvm_eq.qaddr); 648 649 page_size = kvm_host_page_size(vcpu, gfn); 650 if (1ull << kvm_eq.qshift > page_size) { 651 srcu_read_unlock(&kvm->srcu, srcu_idx); 652 pr_warn("Incompatible host page size %lx!\n", page_size); 653 return -EINVAL; 654 } 655 656 page = gfn_to_page(kvm, gfn); 657 if (is_error_page(page)) { 658 srcu_read_unlock(&kvm->srcu, srcu_idx); 659 pr_err("Couldn't get queue page %llx!\n", kvm_eq.qaddr); 660 return -EINVAL; 661 } 662 663 qaddr = page_to_virt(page) + (kvm_eq.qaddr & ~PAGE_MASK); 664 srcu_read_unlock(&kvm->srcu, srcu_idx); 665 666 /* 667 * Backup the queue page guest address to the mark EQ page 668 * dirty for migration. 669 */ 670 q->guest_qaddr = kvm_eq.qaddr; 671 q->guest_qshift = kvm_eq.qshift; 672 673 /* 674 * Unconditional Notification is forced by default at the 675 * OPAL level because the use of END ESBs is not supported by 676 * Linux. 677 */ 678 rc = kvmppc_xive_native_configure_queue(xc->vp_id, q, priority, 679 (__be32 *) qaddr, kvm_eq.qshift, true); 680 if (rc) { 681 pr_err("Failed to configure queue %d for VCPU %d: %d\n", 682 priority, xc->server_num, rc); 683 put_page(page); 684 return rc; 685 } 686 687 /* 688 * Only restore the queue state when needed. When doing the 689 * H_INT_SET_SOURCE_CONFIG hcall, it should not. 690 */ 691 if (kvm_eq.qtoggle != 1 || kvm_eq.qindex != 0) { 692 rc = xive_native_set_queue_state(xc->vp_id, priority, 693 kvm_eq.qtoggle, 694 kvm_eq.qindex); 695 if (rc) 696 goto error; 697 } 698 699 rc = kvmppc_xive_attach_escalation(vcpu, priority, 700 kvmppc_xive_has_single_escalation(xive)); 701 error: 702 if (rc) 703 kvmppc_xive_native_cleanup_queue(vcpu, priority); 704 return rc; 705 } 706 707 static int kvmppc_xive_native_get_queue_config(struct kvmppc_xive *xive, 708 long eq_idx, u64 addr) 709 { 710 struct kvm *kvm = xive->kvm; 711 struct kvm_vcpu *vcpu; 712 struct kvmppc_xive_vcpu *xc; 713 struct xive_q *q; 714 void __user *ubufp = (u64 __user *) addr; 715 u32 server; 716 u8 priority; 717 struct kvm_ppc_xive_eq kvm_eq; 718 u64 qaddr; 719 u64 qshift; 720 u64 qeoi_page; 721 u32 escalate_irq; 722 u64 qflags; 723 int rc; 724 725 /* 726 * Demangle priority/server tuple from the EQ identifier 727 */ 728 priority = (eq_idx & KVM_XIVE_EQ_PRIORITY_MASK) >> 729 KVM_XIVE_EQ_PRIORITY_SHIFT; 730 server = (eq_idx & KVM_XIVE_EQ_SERVER_MASK) >> 731 KVM_XIVE_EQ_SERVER_SHIFT; 732 733 vcpu = kvmppc_xive_find_server(kvm, server); 734 if (!vcpu) { 735 pr_err("Can't find server %d\n", server); 736 return -ENOENT; 737 } 738 xc = vcpu->arch.xive_vcpu; 739 740 if (priority != xive_prio_from_guest(priority)) { 741 pr_err("invalid priority for queue %d for VCPU %d\n", 742 priority, server); 743 return -EINVAL; 744 } 745 q = &xc->queues[priority]; 746 747 memset(&kvm_eq, 0, sizeof(kvm_eq)); 748 749 if (!q->qpage) 750 return 0; 751 752 rc = xive_native_get_queue_info(xc->vp_id, priority, &qaddr, &qshift, 753 &qeoi_page, &escalate_irq, &qflags); 754 if (rc) 755 return rc; 756 757 kvm_eq.flags = 0; 758 if (qflags & OPAL_XIVE_EQ_ALWAYS_NOTIFY) 759 kvm_eq.flags |= KVM_XIVE_EQ_ALWAYS_NOTIFY; 760 761 kvm_eq.qshift = q->guest_qshift; 762 kvm_eq.qaddr = q->guest_qaddr; 763 764 rc = xive_native_get_queue_state(xc->vp_id, priority, &kvm_eq.qtoggle, 765 &kvm_eq.qindex); 766 if (rc) 767 return rc; 768 769 pr_devel("%s VCPU %d priority %d fl:%x shift:%d addr:%llx g:%d idx:%d\n", 770 __func__, server, priority, kvm_eq.flags, 771 kvm_eq.qshift, kvm_eq.qaddr, kvm_eq.qtoggle, kvm_eq.qindex); 772 773 if (copy_to_user(ubufp, &kvm_eq, sizeof(kvm_eq))) 774 return -EFAULT; 775 776 return 0; 777 } 778 779 static void kvmppc_xive_reset_sources(struct kvmppc_xive_src_block *sb) 780 { 781 int i; 782 783 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 784 struct kvmppc_xive_irq_state *state = &sb->irq_state[i]; 785 786 if (!state->valid) 787 continue; 788 789 if (state->act_priority == MASKED) 790 continue; 791 792 state->eisn = 0; 793 state->act_server = 0; 794 state->act_priority = MASKED; 795 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01); 796 xive_native_configure_irq(state->ipi_number, 0, MASKED, 0); 797 if (state->pt_number) { 798 xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01); 799 xive_native_configure_irq(state->pt_number, 800 0, MASKED, 0); 801 } 802 } 803 } 804 805 static int kvmppc_xive_reset(struct kvmppc_xive *xive) 806 { 807 struct kvm *kvm = xive->kvm; 808 struct kvm_vcpu *vcpu; 809 unsigned long i; 810 811 pr_devel("%s\n", __func__); 812 813 mutex_lock(&xive->lock); 814 815 kvm_for_each_vcpu(i, vcpu, kvm) { 816 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 817 unsigned int prio; 818 819 if (!xc) 820 continue; 821 822 kvmppc_xive_disable_vcpu_interrupts(vcpu); 823 824 for (prio = 0; prio < KVMPPC_XIVE_Q_COUNT; prio++) { 825 826 /* Single escalation, no queue 7 */ 827 if (prio == 7 && kvmppc_xive_has_single_escalation(xive)) 828 break; 829 830 if (xc->esc_virq[prio]) { 831 free_irq(xc->esc_virq[prio], vcpu); 832 irq_dispose_mapping(xc->esc_virq[prio]); 833 kfree(xc->esc_virq_names[prio]); 834 xc->esc_virq[prio] = 0; 835 } 836 837 kvmppc_xive_native_cleanup_queue(vcpu, prio); 838 } 839 } 840 841 for (i = 0; i <= xive->max_sbid; i++) { 842 struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 843 844 if (sb) { 845 arch_spin_lock(&sb->lock); 846 kvmppc_xive_reset_sources(sb); 847 arch_spin_unlock(&sb->lock); 848 } 849 } 850 851 mutex_unlock(&xive->lock); 852 853 return 0; 854 } 855 856 static void kvmppc_xive_native_sync_sources(struct kvmppc_xive_src_block *sb) 857 { 858 int j; 859 860 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) { 861 struct kvmppc_xive_irq_state *state = &sb->irq_state[j]; 862 struct xive_irq_data *xd; 863 u32 hw_num; 864 865 if (!state->valid) 866 continue; 867 868 /* 869 * The struct kvmppc_xive_irq_state reflects the state 870 * of the EAS configuration and not the state of the 871 * source. The source is masked setting the PQ bits to 872 * '-Q', which is what is being done before calling 873 * the KVM_DEV_XIVE_EQ_SYNC control. 874 * 875 * If a source EAS is configured, OPAL syncs the XIVE 876 * IC of the source and the XIVE IC of the previous 877 * target if any. 878 * 879 * So it should be fine ignoring MASKED sources as 880 * they have been synced already. 881 */ 882 if (state->act_priority == MASKED) 883 continue; 884 885 kvmppc_xive_select_irq(state, &hw_num, &xd); 886 xive_native_sync_source(hw_num); 887 xive_native_sync_queue(hw_num); 888 } 889 } 890 891 static int kvmppc_xive_native_vcpu_eq_sync(struct kvm_vcpu *vcpu) 892 { 893 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 894 unsigned int prio; 895 int srcu_idx; 896 897 if (!xc) 898 return -ENOENT; 899 900 for (prio = 0; prio < KVMPPC_XIVE_Q_COUNT; prio++) { 901 struct xive_q *q = &xc->queues[prio]; 902 903 if (!q->qpage) 904 continue; 905 906 /* Mark EQ page dirty for migration */ 907 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 908 mark_page_dirty(vcpu->kvm, gpa_to_gfn(q->guest_qaddr)); 909 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); 910 } 911 return 0; 912 } 913 914 static int kvmppc_xive_native_eq_sync(struct kvmppc_xive *xive) 915 { 916 struct kvm *kvm = xive->kvm; 917 struct kvm_vcpu *vcpu; 918 unsigned long i; 919 920 pr_devel("%s\n", __func__); 921 922 mutex_lock(&xive->lock); 923 for (i = 0; i <= xive->max_sbid; i++) { 924 struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 925 926 if (sb) { 927 arch_spin_lock(&sb->lock); 928 kvmppc_xive_native_sync_sources(sb); 929 arch_spin_unlock(&sb->lock); 930 } 931 } 932 933 kvm_for_each_vcpu(i, vcpu, kvm) { 934 kvmppc_xive_native_vcpu_eq_sync(vcpu); 935 } 936 mutex_unlock(&xive->lock); 937 938 return 0; 939 } 940 941 static int kvmppc_xive_native_set_attr(struct kvm_device *dev, 942 struct kvm_device_attr *attr) 943 { 944 struct kvmppc_xive *xive = dev->private; 945 946 switch (attr->group) { 947 case KVM_DEV_XIVE_GRP_CTRL: 948 switch (attr->attr) { 949 case KVM_DEV_XIVE_RESET: 950 return kvmppc_xive_reset(xive); 951 case KVM_DEV_XIVE_EQ_SYNC: 952 return kvmppc_xive_native_eq_sync(xive); 953 case KVM_DEV_XIVE_NR_SERVERS: 954 return kvmppc_xive_set_nr_servers(xive, attr->addr); 955 } 956 break; 957 case KVM_DEV_XIVE_GRP_SOURCE: 958 return kvmppc_xive_native_set_source(xive, attr->attr, 959 attr->addr); 960 case KVM_DEV_XIVE_GRP_SOURCE_CONFIG: 961 return kvmppc_xive_native_set_source_config(xive, attr->attr, 962 attr->addr); 963 case KVM_DEV_XIVE_GRP_EQ_CONFIG: 964 return kvmppc_xive_native_set_queue_config(xive, attr->attr, 965 attr->addr); 966 case KVM_DEV_XIVE_GRP_SOURCE_SYNC: 967 return kvmppc_xive_native_sync_source(xive, attr->attr, 968 attr->addr); 969 } 970 return -ENXIO; 971 } 972 973 static int kvmppc_xive_native_get_attr(struct kvm_device *dev, 974 struct kvm_device_attr *attr) 975 { 976 struct kvmppc_xive *xive = dev->private; 977 978 switch (attr->group) { 979 case KVM_DEV_XIVE_GRP_EQ_CONFIG: 980 return kvmppc_xive_native_get_queue_config(xive, attr->attr, 981 attr->addr); 982 } 983 return -ENXIO; 984 } 985 986 static int kvmppc_xive_native_has_attr(struct kvm_device *dev, 987 struct kvm_device_attr *attr) 988 { 989 switch (attr->group) { 990 case KVM_DEV_XIVE_GRP_CTRL: 991 switch (attr->attr) { 992 case KVM_DEV_XIVE_RESET: 993 case KVM_DEV_XIVE_EQ_SYNC: 994 case KVM_DEV_XIVE_NR_SERVERS: 995 return 0; 996 } 997 break; 998 case KVM_DEV_XIVE_GRP_SOURCE: 999 case KVM_DEV_XIVE_GRP_SOURCE_CONFIG: 1000 case KVM_DEV_XIVE_GRP_SOURCE_SYNC: 1001 if (attr->attr >= KVMPPC_XIVE_FIRST_IRQ && 1002 attr->attr < KVMPPC_XIVE_NR_IRQS) 1003 return 0; 1004 break; 1005 case KVM_DEV_XIVE_GRP_EQ_CONFIG: 1006 return 0; 1007 } 1008 return -ENXIO; 1009 } 1010 1011 /* 1012 * Called when device fd is closed. kvm->lock is held. 1013 */ 1014 static void kvmppc_xive_native_release(struct kvm_device *dev) 1015 { 1016 struct kvmppc_xive *xive = dev->private; 1017 struct kvm *kvm = xive->kvm; 1018 struct kvm_vcpu *vcpu; 1019 unsigned long i; 1020 1021 pr_devel("Releasing xive native device\n"); 1022 1023 /* 1024 * Clear the KVM device file address_space which is used to 1025 * unmap the ESB pages when a device is passed-through. 1026 */ 1027 mutex_lock(&xive->mapping_lock); 1028 xive->mapping = NULL; 1029 mutex_unlock(&xive->mapping_lock); 1030 1031 /* 1032 * Since this is the device release function, we know that 1033 * userspace does not have any open fd or mmap referring to 1034 * the device. Therefore there can not be any of the 1035 * device attribute set/get, mmap, or page fault functions 1036 * being executed concurrently, and similarly, the 1037 * connect_vcpu and set/clr_mapped functions also cannot 1038 * be being executed. 1039 */ 1040 1041 debugfs_remove(xive->dentry); 1042 1043 /* 1044 * We should clean up the vCPU interrupt presenters first. 1045 */ 1046 kvm_for_each_vcpu(i, vcpu, kvm) { 1047 /* 1048 * Take vcpu->mutex to ensure that no one_reg get/set ioctl 1049 * (i.e. kvmppc_xive_native_[gs]et_vp) can be being done. 1050 * Holding the vcpu->mutex also means that the vcpu cannot 1051 * be executing the KVM_RUN ioctl, and therefore it cannot 1052 * be executing the XIVE push or pull code or accessing 1053 * the XIVE MMIO regions. 1054 */ 1055 mutex_lock(&vcpu->mutex); 1056 kvmppc_xive_native_cleanup_vcpu(vcpu); 1057 mutex_unlock(&vcpu->mutex); 1058 } 1059 1060 /* 1061 * Now that we have cleared vcpu->arch.xive_vcpu, vcpu->arch.irq_type 1062 * and vcpu->arch.xive_esc_[vr]addr on each vcpu, we are safe 1063 * against xive code getting called during vcpu execution or 1064 * set/get one_reg operations. 1065 */ 1066 kvm->arch.xive = NULL; 1067 1068 for (i = 0; i <= xive->max_sbid; i++) { 1069 if (xive->src_blocks[i]) 1070 kvmppc_xive_free_sources(xive->src_blocks[i]); 1071 kfree(xive->src_blocks[i]); 1072 xive->src_blocks[i] = NULL; 1073 } 1074 1075 if (xive->vp_base != XIVE_INVALID_VP) 1076 xive_native_free_vp_block(xive->vp_base); 1077 1078 /* 1079 * A reference of the kvmppc_xive pointer is now kept under 1080 * the xive_devices struct of the machine for reuse. It is 1081 * freed when the VM is destroyed for now until we fix all the 1082 * execution paths. 1083 */ 1084 1085 kfree(dev); 1086 } 1087 1088 /* 1089 * Create a XIVE device. kvm->lock is held. 1090 */ 1091 static int kvmppc_xive_native_create(struct kvm_device *dev, u32 type) 1092 { 1093 struct kvmppc_xive *xive; 1094 struct kvm *kvm = dev->kvm; 1095 1096 pr_devel("Creating xive native device\n"); 1097 1098 if (kvm->arch.xive) 1099 return -EEXIST; 1100 1101 xive = kvmppc_xive_get_device(kvm, type); 1102 if (!xive) 1103 return -ENOMEM; 1104 1105 dev->private = xive; 1106 xive->dev = dev; 1107 xive->kvm = kvm; 1108 mutex_init(&xive->mapping_lock); 1109 mutex_init(&xive->lock); 1110 1111 /* VP allocation is delayed to the first call to connect_vcpu */ 1112 xive->vp_base = XIVE_INVALID_VP; 1113 /* KVM_MAX_VCPUS limits the number of VMs to roughly 64 per sockets 1114 * on a POWER9 system. 1115 */ 1116 xive->nr_servers = KVM_MAX_VCPUS; 1117 1118 if (xive_native_has_single_escalation()) 1119 xive->flags |= KVMPPC_XIVE_FLAG_SINGLE_ESCALATION; 1120 1121 if (xive_native_has_save_restore()) 1122 xive->flags |= KVMPPC_XIVE_FLAG_SAVE_RESTORE; 1123 1124 xive->ops = &kvmppc_xive_native_ops; 1125 1126 kvm->arch.xive = xive; 1127 return 0; 1128 } 1129 1130 /* 1131 * Interrupt Pending Buffer (IPB) offset 1132 */ 1133 #define TM_IPB_SHIFT 40 1134 #define TM_IPB_MASK (((u64) 0xFF) << TM_IPB_SHIFT) 1135 1136 int kvmppc_xive_native_get_vp(struct kvm_vcpu *vcpu, union kvmppc_one_reg *val) 1137 { 1138 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 1139 u64 opal_state; 1140 int rc; 1141 1142 if (!kvmppc_xive_enabled(vcpu)) 1143 return -EPERM; 1144 1145 if (!xc) 1146 return -ENOENT; 1147 1148 /* Thread context registers. We only care about IPB and CPPR */ 1149 val->xive_timaval[0] = vcpu->arch.xive_saved_state.w01; 1150 1151 /* Get the VP state from OPAL */ 1152 rc = xive_native_get_vp_state(xc->vp_id, &opal_state); 1153 if (rc) 1154 return rc; 1155 1156 /* 1157 * Capture the backup of IPB register in the NVT structure and 1158 * merge it in our KVM VP state. 1159 */ 1160 val->xive_timaval[0] |= cpu_to_be64(opal_state & TM_IPB_MASK); 1161 1162 pr_devel("%s NSR=%02x CPPR=%02x IBP=%02x PIPR=%02x w01=%016llx w2=%08x opal=%016llx\n", 1163 __func__, 1164 vcpu->arch.xive_saved_state.nsr, 1165 vcpu->arch.xive_saved_state.cppr, 1166 vcpu->arch.xive_saved_state.ipb, 1167 vcpu->arch.xive_saved_state.pipr, 1168 vcpu->arch.xive_saved_state.w01, 1169 (u32) vcpu->arch.xive_cam_word, opal_state); 1170 1171 return 0; 1172 } 1173 1174 int kvmppc_xive_native_set_vp(struct kvm_vcpu *vcpu, union kvmppc_one_reg *val) 1175 { 1176 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 1177 struct kvmppc_xive *xive = vcpu->kvm->arch.xive; 1178 1179 pr_devel("%s w01=%016llx vp=%016llx\n", __func__, 1180 val->xive_timaval[0], val->xive_timaval[1]); 1181 1182 if (!kvmppc_xive_enabled(vcpu)) 1183 return -EPERM; 1184 1185 if (!xc || !xive) 1186 return -ENOENT; 1187 1188 /* We can't update the state of a "pushed" VCPU */ 1189 if (WARN_ON(vcpu->arch.xive_pushed)) 1190 return -EBUSY; 1191 1192 /* 1193 * Restore the thread context registers. IPB and CPPR should 1194 * be the only ones that matter. 1195 */ 1196 vcpu->arch.xive_saved_state.w01 = val->xive_timaval[0]; 1197 1198 /* 1199 * There is no need to restore the XIVE internal state (IPB 1200 * stored in the NVT) as the IPB register was merged in KVM VP 1201 * state when captured. 1202 */ 1203 return 0; 1204 } 1205 1206 bool kvmppc_xive_native_supported(void) 1207 { 1208 return xive_native_has_queue_state_support(); 1209 } 1210 1211 static int xive_native_debug_show(struct seq_file *m, void *private) 1212 { 1213 struct kvmppc_xive *xive = m->private; 1214 struct kvm *kvm = xive->kvm; 1215 struct kvm_vcpu *vcpu; 1216 unsigned long i; 1217 1218 if (!kvm) 1219 return 0; 1220 1221 seq_puts(m, "=========\nVCPU state\n=========\n"); 1222 1223 kvm_for_each_vcpu(i, vcpu, kvm) { 1224 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 1225 1226 if (!xc) 1227 continue; 1228 1229 seq_printf(m, "VCPU %d: VP=%#x/%02x\n" 1230 " NSR=%02x CPPR=%02x IBP=%02x PIPR=%02x w01=%016llx w2=%08x\n", 1231 xc->server_num, xc->vp_id, xc->vp_chip_id, 1232 vcpu->arch.xive_saved_state.nsr, 1233 vcpu->arch.xive_saved_state.cppr, 1234 vcpu->arch.xive_saved_state.ipb, 1235 vcpu->arch.xive_saved_state.pipr, 1236 be64_to_cpu(vcpu->arch.xive_saved_state.w01), 1237 be32_to_cpu(vcpu->arch.xive_cam_word)); 1238 1239 kvmppc_xive_debug_show_queues(m, vcpu); 1240 } 1241 1242 seq_puts(m, "=========\nSources\n=========\n"); 1243 1244 for (i = 0; i <= xive->max_sbid; i++) { 1245 struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 1246 1247 if (sb) { 1248 arch_spin_lock(&sb->lock); 1249 kvmppc_xive_debug_show_sources(m, sb); 1250 arch_spin_unlock(&sb->lock); 1251 } 1252 } 1253 1254 return 0; 1255 } 1256 1257 DEFINE_SHOW_ATTRIBUTE(xive_native_debug); 1258 1259 static void xive_native_debugfs_init(struct kvmppc_xive *xive) 1260 { 1261 xive->dentry = debugfs_create_file("xive", 0444, xive->kvm->debugfs_dentry, 1262 xive, &xive_native_debug_fops); 1263 1264 pr_debug("%s: created\n", __func__); 1265 } 1266 1267 static void kvmppc_xive_native_init(struct kvm_device *dev) 1268 { 1269 struct kvmppc_xive *xive = dev->private; 1270 1271 /* Register some debug interfaces */ 1272 xive_native_debugfs_init(xive); 1273 } 1274 1275 struct kvm_device_ops kvm_xive_native_ops = { 1276 .name = "kvm-xive-native", 1277 .create = kvmppc_xive_native_create, 1278 .init = kvmppc_xive_native_init, 1279 .release = kvmppc_xive_native_release, 1280 .set_attr = kvmppc_xive_native_set_attr, 1281 .get_attr = kvmppc_xive_native_get_attr, 1282 .has_attr = kvmppc_xive_native_has_attr, 1283 .mmap = kvmppc_xive_native_mmap, 1284 }; 1285