1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2017-2019, IBM Corporation. 4 */ 5 6 #define pr_fmt(fmt) "xive-kvm: " fmt 7 8 #include <linux/kernel.h> 9 #include <linux/kvm_host.h> 10 #include <linux/err.h> 11 #include <linux/gfp.h> 12 #include <linux/spinlock.h> 13 #include <linux/delay.h> 14 #include <linux/file.h> 15 #include <asm/uaccess.h> 16 #include <asm/kvm_book3s.h> 17 #include <asm/kvm_ppc.h> 18 #include <asm/hvcall.h> 19 #include <asm/xive.h> 20 #include <asm/xive-regs.h> 21 #include <asm/debug.h> 22 #include <asm/debugfs.h> 23 #include <asm/opal.h> 24 25 #include <linux/debugfs.h> 26 #include <linux/seq_file.h> 27 28 #include "book3s_xive.h" 29 30 static u8 xive_vm_esb_load(struct xive_irq_data *xd, u32 offset) 31 { 32 u64 val; 33 34 if (xd->flags & XIVE_IRQ_FLAG_SHIFT_BUG) 35 offset |= offset << 4; 36 37 val = in_be64(xd->eoi_mmio + offset); 38 return (u8)val; 39 } 40 41 static void kvmppc_xive_native_cleanup_queue(struct kvm_vcpu *vcpu, int prio) 42 { 43 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 44 struct xive_q *q = &xc->queues[prio]; 45 46 xive_native_disable_queue(xc->vp_id, q, prio); 47 if (q->qpage) { 48 put_page(virt_to_page(q->qpage)); 49 q->qpage = NULL; 50 } 51 } 52 53 void kvmppc_xive_native_cleanup_vcpu(struct kvm_vcpu *vcpu) 54 { 55 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 56 int i; 57 58 if (!kvmppc_xive_enabled(vcpu)) 59 return; 60 61 if (!xc) 62 return; 63 64 pr_devel("native_cleanup_vcpu(cpu=%d)\n", xc->server_num); 65 66 /* Ensure no interrupt is still routed to that VP */ 67 xc->valid = false; 68 kvmppc_xive_disable_vcpu_interrupts(vcpu); 69 70 /* Disable the VP */ 71 xive_native_disable_vp(xc->vp_id); 72 73 /* Free the queues & associated interrupts */ 74 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) { 75 /* Free the escalation irq */ 76 if (xc->esc_virq[i]) { 77 free_irq(xc->esc_virq[i], vcpu); 78 irq_dispose_mapping(xc->esc_virq[i]); 79 kfree(xc->esc_virq_names[i]); 80 xc->esc_virq[i] = 0; 81 } 82 83 /* Free the queue */ 84 kvmppc_xive_native_cleanup_queue(vcpu, i); 85 } 86 87 /* Free the VP */ 88 kfree(xc); 89 90 /* Cleanup the vcpu */ 91 vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT; 92 vcpu->arch.xive_vcpu = NULL; 93 } 94 95 int kvmppc_xive_native_connect_vcpu(struct kvm_device *dev, 96 struct kvm_vcpu *vcpu, u32 server_num) 97 { 98 struct kvmppc_xive *xive = dev->private; 99 struct kvmppc_xive_vcpu *xc = NULL; 100 int rc; 101 102 pr_devel("native_connect_vcpu(server=%d)\n", server_num); 103 104 if (dev->ops != &kvm_xive_native_ops) { 105 pr_devel("Wrong ops !\n"); 106 return -EPERM; 107 } 108 if (xive->kvm != vcpu->kvm) 109 return -EPERM; 110 if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT) 111 return -EBUSY; 112 if (server_num >= (KVM_MAX_VCPUS * vcpu->kvm->arch.emul_smt_mode)) { 113 pr_devel("Out of bounds !\n"); 114 return -EINVAL; 115 } 116 117 mutex_lock(&xive->lock); 118 119 if (kvmppc_xive_find_server(vcpu->kvm, server_num)) { 120 pr_devel("Duplicate !\n"); 121 rc = -EEXIST; 122 goto bail; 123 } 124 125 xc = kzalloc(sizeof(*xc), GFP_KERNEL); 126 if (!xc) { 127 rc = -ENOMEM; 128 goto bail; 129 } 130 131 vcpu->arch.xive_vcpu = xc; 132 xc->xive = xive; 133 xc->vcpu = vcpu; 134 xc->server_num = server_num; 135 136 xc->vp_id = kvmppc_xive_vp(xive, server_num); 137 xc->valid = true; 138 vcpu->arch.irq_type = KVMPPC_IRQ_XIVE; 139 140 rc = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id); 141 if (rc) { 142 pr_err("Failed to get VP info from OPAL: %d\n", rc); 143 goto bail; 144 } 145 146 /* 147 * Enable the VP first as the single escalation mode will 148 * affect escalation interrupts numbering 149 */ 150 rc = xive_native_enable_vp(xc->vp_id, xive->single_escalation); 151 if (rc) { 152 pr_err("Failed to enable VP in OPAL: %d\n", rc); 153 goto bail; 154 } 155 156 /* Configure VCPU fields for use by assembly push/pull */ 157 vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000); 158 vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO); 159 160 /* TODO: reset all queues to a clean state ? */ 161 bail: 162 mutex_unlock(&xive->lock); 163 if (rc) 164 kvmppc_xive_native_cleanup_vcpu(vcpu); 165 166 return rc; 167 } 168 169 /* 170 * Device passthrough support 171 */ 172 static int kvmppc_xive_native_reset_mapped(struct kvm *kvm, unsigned long irq) 173 { 174 struct kvmppc_xive *xive = kvm->arch.xive; 175 pgoff_t esb_pgoff = KVM_XIVE_ESB_PAGE_OFFSET + irq * 2; 176 177 if (irq >= KVMPPC_XIVE_NR_IRQS) 178 return -EINVAL; 179 180 /* 181 * Clear the ESB pages of the IRQ number being mapped (or 182 * unmapped) into the guest and let the the VM fault handler 183 * repopulate with the appropriate ESB pages (device or IC) 184 */ 185 pr_debug("clearing esb pages for girq 0x%lx\n", irq); 186 mutex_lock(&xive->mapping_lock); 187 if (xive->mapping) 188 unmap_mapping_range(xive->mapping, 189 esb_pgoff << PAGE_SHIFT, 190 2ull << PAGE_SHIFT, 1); 191 mutex_unlock(&xive->mapping_lock); 192 return 0; 193 } 194 195 static struct kvmppc_xive_ops kvmppc_xive_native_ops = { 196 .reset_mapped = kvmppc_xive_native_reset_mapped, 197 }; 198 199 static vm_fault_t xive_native_esb_fault(struct vm_fault *vmf) 200 { 201 struct vm_area_struct *vma = vmf->vma; 202 struct kvm_device *dev = vma->vm_file->private_data; 203 struct kvmppc_xive *xive = dev->private; 204 struct kvmppc_xive_src_block *sb; 205 struct kvmppc_xive_irq_state *state; 206 struct xive_irq_data *xd; 207 u32 hw_num; 208 u16 src; 209 u64 page; 210 unsigned long irq; 211 u64 page_offset; 212 213 /* 214 * Linux/KVM uses a two pages ESB setting, one for trigger and 215 * one for EOI 216 */ 217 page_offset = vmf->pgoff - vma->vm_pgoff; 218 irq = page_offset / 2; 219 220 sb = kvmppc_xive_find_source(xive, irq, &src); 221 if (!sb) { 222 pr_devel("%s: source %lx not found !\n", __func__, irq); 223 return VM_FAULT_SIGBUS; 224 } 225 226 state = &sb->irq_state[src]; 227 kvmppc_xive_select_irq(state, &hw_num, &xd); 228 229 arch_spin_lock(&sb->lock); 230 231 /* 232 * first/even page is for trigger 233 * second/odd page is for EOI and management. 234 */ 235 page = page_offset % 2 ? xd->eoi_page : xd->trig_page; 236 arch_spin_unlock(&sb->lock); 237 238 if (WARN_ON(!page)) { 239 pr_err("%s: accessing invalid ESB page for source %lx !\n", 240 __func__, irq); 241 return VM_FAULT_SIGBUS; 242 } 243 244 vmf_insert_pfn(vma, vmf->address, page >> PAGE_SHIFT); 245 return VM_FAULT_NOPAGE; 246 } 247 248 static const struct vm_operations_struct xive_native_esb_vmops = { 249 .fault = xive_native_esb_fault, 250 }; 251 252 static vm_fault_t xive_native_tima_fault(struct vm_fault *vmf) 253 { 254 struct vm_area_struct *vma = vmf->vma; 255 256 switch (vmf->pgoff - vma->vm_pgoff) { 257 case 0: /* HW - forbid access */ 258 case 1: /* HV - forbid access */ 259 return VM_FAULT_SIGBUS; 260 case 2: /* OS */ 261 vmf_insert_pfn(vma, vmf->address, xive_tima_os >> PAGE_SHIFT); 262 return VM_FAULT_NOPAGE; 263 case 3: /* USER - TODO */ 264 default: 265 return VM_FAULT_SIGBUS; 266 } 267 } 268 269 static const struct vm_operations_struct xive_native_tima_vmops = { 270 .fault = xive_native_tima_fault, 271 }; 272 273 static int kvmppc_xive_native_mmap(struct kvm_device *dev, 274 struct vm_area_struct *vma) 275 { 276 struct kvmppc_xive *xive = dev->private; 277 278 /* We only allow mappings at fixed offset for now */ 279 if (vma->vm_pgoff == KVM_XIVE_TIMA_PAGE_OFFSET) { 280 if (vma_pages(vma) > 4) 281 return -EINVAL; 282 vma->vm_ops = &xive_native_tima_vmops; 283 } else if (vma->vm_pgoff == KVM_XIVE_ESB_PAGE_OFFSET) { 284 if (vma_pages(vma) > KVMPPC_XIVE_NR_IRQS * 2) 285 return -EINVAL; 286 vma->vm_ops = &xive_native_esb_vmops; 287 } else { 288 return -EINVAL; 289 } 290 291 vma->vm_flags |= VM_IO | VM_PFNMAP; 292 vma->vm_page_prot = pgprot_noncached_wc(vma->vm_page_prot); 293 294 /* 295 * Grab the KVM device file address_space to be able to clear 296 * the ESB pages mapping when a device is passed-through into 297 * the guest. 298 */ 299 xive->mapping = vma->vm_file->f_mapping; 300 return 0; 301 } 302 303 static int kvmppc_xive_native_set_source(struct kvmppc_xive *xive, long irq, 304 u64 addr) 305 { 306 struct kvmppc_xive_src_block *sb; 307 struct kvmppc_xive_irq_state *state; 308 u64 __user *ubufp = (u64 __user *) addr; 309 u64 val; 310 u16 idx; 311 int rc; 312 313 pr_devel("%s irq=0x%lx\n", __func__, irq); 314 315 if (irq < KVMPPC_XIVE_FIRST_IRQ || irq >= KVMPPC_XIVE_NR_IRQS) 316 return -E2BIG; 317 318 sb = kvmppc_xive_find_source(xive, irq, &idx); 319 if (!sb) { 320 pr_debug("No source, creating source block...\n"); 321 sb = kvmppc_xive_create_src_block(xive, irq); 322 if (!sb) { 323 pr_err("Failed to create block...\n"); 324 return -ENOMEM; 325 } 326 } 327 state = &sb->irq_state[idx]; 328 329 if (get_user(val, ubufp)) { 330 pr_err("fault getting user info !\n"); 331 return -EFAULT; 332 } 333 334 arch_spin_lock(&sb->lock); 335 336 /* 337 * If the source doesn't already have an IPI, allocate 338 * one and get the corresponding data 339 */ 340 if (!state->ipi_number) { 341 state->ipi_number = xive_native_alloc_irq(); 342 if (state->ipi_number == 0) { 343 pr_err("Failed to allocate IRQ !\n"); 344 rc = -ENXIO; 345 goto unlock; 346 } 347 xive_native_populate_irq_data(state->ipi_number, 348 &state->ipi_data); 349 pr_debug("%s allocated hw_irq=0x%x for irq=0x%lx\n", __func__, 350 state->ipi_number, irq); 351 } 352 353 /* Restore LSI state */ 354 if (val & KVM_XIVE_LEVEL_SENSITIVE) { 355 state->lsi = true; 356 if (val & KVM_XIVE_LEVEL_ASSERTED) 357 state->asserted = true; 358 pr_devel(" LSI ! Asserted=%d\n", state->asserted); 359 } 360 361 /* Mask IRQ to start with */ 362 state->act_server = 0; 363 state->act_priority = MASKED; 364 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01); 365 xive_native_configure_irq(state->ipi_number, 0, MASKED, 0); 366 367 /* Increment the number of valid sources and mark this one valid */ 368 if (!state->valid) 369 xive->src_count++; 370 state->valid = true; 371 372 rc = 0; 373 374 unlock: 375 arch_spin_unlock(&sb->lock); 376 377 return rc; 378 } 379 380 static int kvmppc_xive_native_update_source_config(struct kvmppc_xive *xive, 381 struct kvmppc_xive_src_block *sb, 382 struct kvmppc_xive_irq_state *state, 383 u32 server, u8 priority, bool masked, 384 u32 eisn) 385 { 386 struct kvm *kvm = xive->kvm; 387 u32 hw_num; 388 int rc = 0; 389 390 arch_spin_lock(&sb->lock); 391 392 if (state->act_server == server && state->act_priority == priority && 393 state->eisn == eisn) 394 goto unlock; 395 396 pr_devel("new_act_prio=%d new_act_server=%d mask=%d act_server=%d act_prio=%d\n", 397 priority, server, masked, state->act_server, 398 state->act_priority); 399 400 kvmppc_xive_select_irq(state, &hw_num, NULL); 401 402 if (priority != MASKED && !masked) { 403 rc = kvmppc_xive_select_target(kvm, &server, priority); 404 if (rc) 405 goto unlock; 406 407 state->act_priority = priority; 408 state->act_server = server; 409 state->eisn = eisn; 410 411 rc = xive_native_configure_irq(hw_num, 412 kvmppc_xive_vp(xive, server), 413 priority, eisn); 414 } else { 415 state->act_priority = MASKED; 416 state->act_server = 0; 417 state->eisn = 0; 418 419 rc = xive_native_configure_irq(hw_num, 0, MASKED, 0); 420 } 421 422 unlock: 423 arch_spin_unlock(&sb->lock); 424 return rc; 425 } 426 427 static int kvmppc_xive_native_set_source_config(struct kvmppc_xive *xive, 428 long irq, u64 addr) 429 { 430 struct kvmppc_xive_src_block *sb; 431 struct kvmppc_xive_irq_state *state; 432 u64 __user *ubufp = (u64 __user *) addr; 433 u16 src; 434 u64 kvm_cfg; 435 u32 server; 436 u8 priority; 437 bool masked; 438 u32 eisn; 439 440 sb = kvmppc_xive_find_source(xive, irq, &src); 441 if (!sb) 442 return -ENOENT; 443 444 state = &sb->irq_state[src]; 445 446 if (!state->valid) 447 return -EINVAL; 448 449 if (get_user(kvm_cfg, ubufp)) 450 return -EFAULT; 451 452 pr_devel("%s irq=0x%lx cfg=%016llx\n", __func__, irq, kvm_cfg); 453 454 priority = (kvm_cfg & KVM_XIVE_SOURCE_PRIORITY_MASK) >> 455 KVM_XIVE_SOURCE_PRIORITY_SHIFT; 456 server = (kvm_cfg & KVM_XIVE_SOURCE_SERVER_MASK) >> 457 KVM_XIVE_SOURCE_SERVER_SHIFT; 458 masked = (kvm_cfg & KVM_XIVE_SOURCE_MASKED_MASK) >> 459 KVM_XIVE_SOURCE_MASKED_SHIFT; 460 eisn = (kvm_cfg & KVM_XIVE_SOURCE_EISN_MASK) >> 461 KVM_XIVE_SOURCE_EISN_SHIFT; 462 463 if (priority != xive_prio_from_guest(priority)) { 464 pr_err("invalid priority for queue %d for VCPU %d\n", 465 priority, server); 466 return -EINVAL; 467 } 468 469 return kvmppc_xive_native_update_source_config(xive, sb, state, server, 470 priority, masked, eisn); 471 } 472 473 static int kvmppc_xive_native_sync_source(struct kvmppc_xive *xive, 474 long irq, u64 addr) 475 { 476 struct kvmppc_xive_src_block *sb; 477 struct kvmppc_xive_irq_state *state; 478 struct xive_irq_data *xd; 479 u32 hw_num; 480 u16 src; 481 int rc = 0; 482 483 pr_devel("%s irq=0x%lx", __func__, irq); 484 485 sb = kvmppc_xive_find_source(xive, irq, &src); 486 if (!sb) 487 return -ENOENT; 488 489 state = &sb->irq_state[src]; 490 491 rc = -EINVAL; 492 493 arch_spin_lock(&sb->lock); 494 495 if (state->valid) { 496 kvmppc_xive_select_irq(state, &hw_num, &xd); 497 xive_native_sync_source(hw_num); 498 rc = 0; 499 } 500 501 arch_spin_unlock(&sb->lock); 502 return rc; 503 } 504 505 static int xive_native_validate_queue_size(u32 qshift) 506 { 507 /* 508 * We only support 64K pages for the moment. This is also 509 * advertised in the DT property "ibm,xive-eq-sizes" 510 */ 511 switch (qshift) { 512 case 0: /* EQ reset */ 513 case 16: 514 return 0; 515 case 12: 516 case 21: 517 case 24: 518 default: 519 return -EINVAL; 520 } 521 } 522 523 static int kvmppc_xive_native_set_queue_config(struct kvmppc_xive *xive, 524 long eq_idx, u64 addr) 525 { 526 struct kvm *kvm = xive->kvm; 527 struct kvm_vcpu *vcpu; 528 struct kvmppc_xive_vcpu *xc; 529 void __user *ubufp = (void __user *) addr; 530 u32 server; 531 u8 priority; 532 struct kvm_ppc_xive_eq kvm_eq; 533 int rc; 534 __be32 *qaddr = 0; 535 struct page *page; 536 struct xive_q *q; 537 gfn_t gfn; 538 unsigned long page_size; 539 int srcu_idx; 540 541 /* 542 * Demangle priority/server tuple from the EQ identifier 543 */ 544 priority = (eq_idx & KVM_XIVE_EQ_PRIORITY_MASK) >> 545 KVM_XIVE_EQ_PRIORITY_SHIFT; 546 server = (eq_idx & KVM_XIVE_EQ_SERVER_MASK) >> 547 KVM_XIVE_EQ_SERVER_SHIFT; 548 549 if (copy_from_user(&kvm_eq, ubufp, sizeof(kvm_eq))) 550 return -EFAULT; 551 552 vcpu = kvmppc_xive_find_server(kvm, server); 553 if (!vcpu) { 554 pr_err("Can't find server %d\n", server); 555 return -ENOENT; 556 } 557 xc = vcpu->arch.xive_vcpu; 558 559 if (priority != xive_prio_from_guest(priority)) { 560 pr_err("Trying to restore invalid queue %d for VCPU %d\n", 561 priority, server); 562 return -EINVAL; 563 } 564 q = &xc->queues[priority]; 565 566 pr_devel("%s VCPU %d priority %d fl:%x shift:%d addr:%llx g:%d idx:%d\n", 567 __func__, server, priority, kvm_eq.flags, 568 kvm_eq.qshift, kvm_eq.qaddr, kvm_eq.qtoggle, kvm_eq.qindex); 569 570 /* reset queue and disable queueing */ 571 if (!kvm_eq.qshift) { 572 q->guest_qaddr = 0; 573 q->guest_qshift = 0; 574 575 rc = xive_native_configure_queue(xc->vp_id, q, priority, 576 NULL, 0, true); 577 if (rc) { 578 pr_err("Failed to reset queue %d for VCPU %d: %d\n", 579 priority, xc->server_num, rc); 580 return rc; 581 } 582 583 if (q->qpage) { 584 put_page(virt_to_page(q->qpage)); 585 q->qpage = NULL; 586 } 587 588 return 0; 589 } 590 591 /* 592 * sPAPR specifies a "Unconditional Notify (n) flag" for the 593 * H_INT_SET_QUEUE_CONFIG hcall which forces notification 594 * without using the coalescing mechanisms provided by the 595 * XIVE END ESBs. This is required on KVM as notification 596 * using the END ESBs is not supported. 597 */ 598 if (kvm_eq.flags != KVM_XIVE_EQ_ALWAYS_NOTIFY) { 599 pr_err("invalid flags %d\n", kvm_eq.flags); 600 return -EINVAL; 601 } 602 603 rc = xive_native_validate_queue_size(kvm_eq.qshift); 604 if (rc) { 605 pr_err("invalid queue size %d\n", kvm_eq.qshift); 606 return rc; 607 } 608 609 if (kvm_eq.qaddr & ((1ull << kvm_eq.qshift) - 1)) { 610 pr_err("queue page is not aligned %llx/%llx\n", kvm_eq.qaddr, 611 1ull << kvm_eq.qshift); 612 return -EINVAL; 613 } 614 615 srcu_idx = srcu_read_lock(&kvm->srcu); 616 gfn = gpa_to_gfn(kvm_eq.qaddr); 617 page = gfn_to_page(kvm, gfn); 618 if (is_error_page(page)) { 619 srcu_read_unlock(&kvm->srcu, srcu_idx); 620 pr_err("Couldn't get queue page %llx!\n", kvm_eq.qaddr); 621 return -EINVAL; 622 } 623 624 page_size = kvm_host_page_size(kvm, gfn); 625 if (1ull << kvm_eq.qshift > page_size) { 626 srcu_read_unlock(&kvm->srcu, srcu_idx); 627 pr_warn("Incompatible host page size %lx!\n", page_size); 628 return -EINVAL; 629 } 630 631 qaddr = page_to_virt(page) + (kvm_eq.qaddr & ~PAGE_MASK); 632 srcu_read_unlock(&kvm->srcu, srcu_idx); 633 634 /* 635 * Backup the queue page guest address to the mark EQ page 636 * dirty for migration. 637 */ 638 q->guest_qaddr = kvm_eq.qaddr; 639 q->guest_qshift = kvm_eq.qshift; 640 641 /* 642 * Unconditional Notification is forced by default at the 643 * OPAL level because the use of END ESBs is not supported by 644 * Linux. 645 */ 646 rc = xive_native_configure_queue(xc->vp_id, q, priority, 647 (__be32 *) qaddr, kvm_eq.qshift, true); 648 if (rc) { 649 pr_err("Failed to configure queue %d for VCPU %d: %d\n", 650 priority, xc->server_num, rc); 651 put_page(page); 652 return rc; 653 } 654 655 /* 656 * Only restore the queue state when needed. When doing the 657 * H_INT_SET_SOURCE_CONFIG hcall, it should not. 658 */ 659 if (kvm_eq.qtoggle != 1 || kvm_eq.qindex != 0) { 660 rc = xive_native_set_queue_state(xc->vp_id, priority, 661 kvm_eq.qtoggle, 662 kvm_eq.qindex); 663 if (rc) 664 goto error; 665 } 666 667 rc = kvmppc_xive_attach_escalation(vcpu, priority, 668 xive->single_escalation); 669 error: 670 if (rc) 671 kvmppc_xive_native_cleanup_queue(vcpu, priority); 672 return rc; 673 } 674 675 static int kvmppc_xive_native_get_queue_config(struct kvmppc_xive *xive, 676 long eq_idx, u64 addr) 677 { 678 struct kvm *kvm = xive->kvm; 679 struct kvm_vcpu *vcpu; 680 struct kvmppc_xive_vcpu *xc; 681 struct xive_q *q; 682 void __user *ubufp = (u64 __user *) addr; 683 u32 server; 684 u8 priority; 685 struct kvm_ppc_xive_eq kvm_eq; 686 u64 qaddr; 687 u64 qshift; 688 u64 qeoi_page; 689 u32 escalate_irq; 690 u64 qflags; 691 int rc; 692 693 /* 694 * Demangle priority/server tuple from the EQ identifier 695 */ 696 priority = (eq_idx & KVM_XIVE_EQ_PRIORITY_MASK) >> 697 KVM_XIVE_EQ_PRIORITY_SHIFT; 698 server = (eq_idx & KVM_XIVE_EQ_SERVER_MASK) >> 699 KVM_XIVE_EQ_SERVER_SHIFT; 700 701 vcpu = kvmppc_xive_find_server(kvm, server); 702 if (!vcpu) { 703 pr_err("Can't find server %d\n", server); 704 return -ENOENT; 705 } 706 xc = vcpu->arch.xive_vcpu; 707 708 if (priority != xive_prio_from_guest(priority)) { 709 pr_err("invalid priority for queue %d for VCPU %d\n", 710 priority, server); 711 return -EINVAL; 712 } 713 q = &xc->queues[priority]; 714 715 memset(&kvm_eq, 0, sizeof(kvm_eq)); 716 717 if (!q->qpage) 718 return 0; 719 720 rc = xive_native_get_queue_info(xc->vp_id, priority, &qaddr, &qshift, 721 &qeoi_page, &escalate_irq, &qflags); 722 if (rc) 723 return rc; 724 725 kvm_eq.flags = 0; 726 if (qflags & OPAL_XIVE_EQ_ALWAYS_NOTIFY) 727 kvm_eq.flags |= KVM_XIVE_EQ_ALWAYS_NOTIFY; 728 729 kvm_eq.qshift = q->guest_qshift; 730 kvm_eq.qaddr = q->guest_qaddr; 731 732 rc = xive_native_get_queue_state(xc->vp_id, priority, &kvm_eq.qtoggle, 733 &kvm_eq.qindex); 734 if (rc) 735 return rc; 736 737 pr_devel("%s VCPU %d priority %d fl:%x shift:%d addr:%llx g:%d idx:%d\n", 738 __func__, server, priority, kvm_eq.flags, 739 kvm_eq.qshift, kvm_eq.qaddr, kvm_eq.qtoggle, kvm_eq.qindex); 740 741 if (copy_to_user(ubufp, &kvm_eq, sizeof(kvm_eq))) 742 return -EFAULT; 743 744 return 0; 745 } 746 747 static void kvmppc_xive_reset_sources(struct kvmppc_xive_src_block *sb) 748 { 749 int i; 750 751 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 752 struct kvmppc_xive_irq_state *state = &sb->irq_state[i]; 753 754 if (!state->valid) 755 continue; 756 757 if (state->act_priority == MASKED) 758 continue; 759 760 state->eisn = 0; 761 state->act_server = 0; 762 state->act_priority = MASKED; 763 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01); 764 xive_native_configure_irq(state->ipi_number, 0, MASKED, 0); 765 if (state->pt_number) { 766 xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01); 767 xive_native_configure_irq(state->pt_number, 768 0, MASKED, 0); 769 } 770 } 771 } 772 773 static int kvmppc_xive_reset(struct kvmppc_xive *xive) 774 { 775 struct kvm *kvm = xive->kvm; 776 struct kvm_vcpu *vcpu; 777 unsigned int i; 778 779 pr_devel("%s\n", __func__); 780 781 mutex_lock(&xive->lock); 782 783 kvm_for_each_vcpu(i, vcpu, kvm) { 784 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 785 unsigned int prio; 786 787 if (!xc) 788 continue; 789 790 kvmppc_xive_disable_vcpu_interrupts(vcpu); 791 792 for (prio = 0; prio < KVMPPC_XIVE_Q_COUNT; prio++) { 793 794 /* Single escalation, no queue 7 */ 795 if (prio == 7 && xive->single_escalation) 796 break; 797 798 if (xc->esc_virq[prio]) { 799 free_irq(xc->esc_virq[prio], vcpu); 800 irq_dispose_mapping(xc->esc_virq[prio]); 801 kfree(xc->esc_virq_names[prio]); 802 xc->esc_virq[prio] = 0; 803 } 804 805 kvmppc_xive_native_cleanup_queue(vcpu, prio); 806 } 807 } 808 809 for (i = 0; i <= xive->max_sbid; i++) { 810 struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 811 812 if (sb) { 813 arch_spin_lock(&sb->lock); 814 kvmppc_xive_reset_sources(sb); 815 arch_spin_unlock(&sb->lock); 816 } 817 } 818 819 mutex_unlock(&xive->lock); 820 821 return 0; 822 } 823 824 static void kvmppc_xive_native_sync_sources(struct kvmppc_xive_src_block *sb) 825 { 826 int j; 827 828 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) { 829 struct kvmppc_xive_irq_state *state = &sb->irq_state[j]; 830 struct xive_irq_data *xd; 831 u32 hw_num; 832 833 if (!state->valid) 834 continue; 835 836 /* 837 * The struct kvmppc_xive_irq_state reflects the state 838 * of the EAS configuration and not the state of the 839 * source. The source is masked setting the PQ bits to 840 * '-Q', which is what is being done before calling 841 * the KVM_DEV_XIVE_EQ_SYNC control. 842 * 843 * If a source EAS is configured, OPAL syncs the XIVE 844 * IC of the source and the XIVE IC of the previous 845 * target if any. 846 * 847 * So it should be fine ignoring MASKED sources as 848 * they have been synced already. 849 */ 850 if (state->act_priority == MASKED) 851 continue; 852 853 kvmppc_xive_select_irq(state, &hw_num, &xd); 854 xive_native_sync_source(hw_num); 855 xive_native_sync_queue(hw_num); 856 } 857 } 858 859 static int kvmppc_xive_native_vcpu_eq_sync(struct kvm_vcpu *vcpu) 860 { 861 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 862 unsigned int prio; 863 int srcu_idx; 864 865 if (!xc) 866 return -ENOENT; 867 868 for (prio = 0; prio < KVMPPC_XIVE_Q_COUNT; prio++) { 869 struct xive_q *q = &xc->queues[prio]; 870 871 if (!q->qpage) 872 continue; 873 874 /* Mark EQ page dirty for migration */ 875 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 876 mark_page_dirty(vcpu->kvm, gpa_to_gfn(q->guest_qaddr)); 877 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); 878 } 879 return 0; 880 } 881 882 static int kvmppc_xive_native_eq_sync(struct kvmppc_xive *xive) 883 { 884 struct kvm *kvm = xive->kvm; 885 struct kvm_vcpu *vcpu; 886 unsigned int i; 887 888 pr_devel("%s\n", __func__); 889 890 mutex_lock(&xive->lock); 891 for (i = 0; i <= xive->max_sbid; i++) { 892 struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 893 894 if (sb) { 895 arch_spin_lock(&sb->lock); 896 kvmppc_xive_native_sync_sources(sb); 897 arch_spin_unlock(&sb->lock); 898 } 899 } 900 901 kvm_for_each_vcpu(i, vcpu, kvm) { 902 kvmppc_xive_native_vcpu_eq_sync(vcpu); 903 } 904 mutex_unlock(&xive->lock); 905 906 return 0; 907 } 908 909 static int kvmppc_xive_native_set_attr(struct kvm_device *dev, 910 struct kvm_device_attr *attr) 911 { 912 struct kvmppc_xive *xive = dev->private; 913 914 switch (attr->group) { 915 case KVM_DEV_XIVE_GRP_CTRL: 916 switch (attr->attr) { 917 case KVM_DEV_XIVE_RESET: 918 return kvmppc_xive_reset(xive); 919 case KVM_DEV_XIVE_EQ_SYNC: 920 return kvmppc_xive_native_eq_sync(xive); 921 } 922 break; 923 case KVM_DEV_XIVE_GRP_SOURCE: 924 return kvmppc_xive_native_set_source(xive, attr->attr, 925 attr->addr); 926 case KVM_DEV_XIVE_GRP_SOURCE_CONFIG: 927 return kvmppc_xive_native_set_source_config(xive, attr->attr, 928 attr->addr); 929 case KVM_DEV_XIVE_GRP_EQ_CONFIG: 930 return kvmppc_xive_native_set_queue_config(xive, attr->attr, 931 attr->addr); 932 case KVM_DEV_XIVE_GRP_SOURCE_SYNC: 933 return kvmppc_xive_native_sync_source(xive, attr->attr, 934 attr->addr); 935 } 936 return -ENXIO; 937 } 938 939 static int kvmppc_xive_native_get_attr(struct kvm_device *dev, 940 struct kvm_device_attr *attr) 941 { 942 struct kvmppc_xive *xive = dev->private; 943 944 switch (attr->group) { 945 case KVM_DEV_XIVE_GRP_EQ_CONFIG: 946 return kvmppc_xive_native_get_queue_config(xive, attr->attr, 947 attr->addr); 948 } 949 return -ENXIO; 950 } 951 952 static int kvmppc_xive_native_has_attr(struct kvm_device *dev, 953 struct kvm_device_attr *attr) 954 { 955 switch (attr->group) { 956 case KVM_DEV_XIVE_GRP_CTRL: 957 switch (attr->attr) { 958 case KVM_DEV_XIVE_RESET: 959 case KVM_DEV_XIVE_EQ_SYNC: 960 return 0; 961 } 962 break; 963 case KVM_DEV_XIVE_GRP_SOURCE: 964 case KVM_DEV_XIVE_GRP_SOURCE_CONFIG: 965 case KVM_DEV_XIVE_GRP_SOURCE_SYNC: 966 if (attr->attr >= KVMPPC_XIVE_FIRST_IRQ && 967 attr->attr < KVMPPC_XIVE_NR_IRQS) 968 return 0; 969 break; 970 case KVM_DEV_XIVE_GRP_EQ_CONFIG: 971 return 0; 972 } 973 return -ENXIO; 974 } 975 976 /* 977 * Called when device fd is closed. kvm->lock is held. 978 */ 979 static void kvmppc_xive_native_release(struct kvm_device *dev) 980 { 981 struct kvmppc_xive *xive = dev->private; 982 struct kvm *kvm = xive->kvm; 983 struct kvm_vcpu *vcpu; 984 int i; 985 986 pr_devel("Releasing xive native device\n"); 987 988 /* 989 * Clear the KVM device file address_space which is used to 990 * unmap the ESB pages when a device is passed-through. 991 */ 992 mutex_lock(&xive->mapping_lock); 993 xive->mapping = NULL; 994 mutex_unlock(&xive->mapping_lock); 995 996 /* 997 * Since this is the device release function, we know that 998 * userspace does not have any open fd or mmap referring to 999 * the device. Therefore there can not be any of the 1000 * device attribute set/get, mmap, or page fault functions 1001 * being executed concurrently, and similarly, the 1002 * connect_vcpu and set/clr_mapped functions also cannot 1003 * be being executed. 1004 */ 1005 1006 debugfs_remove(xive->dentry); 1007 1008 /* 1009 * We should clean up the vCPU interrupt presenters first. 1010 */ 1011 kvm_for_each_vcpu(i, vcpu, kvm) { 1012 /* 1013 * Take vcpu->mutex to ensure that no one_reg get/set ioctl 1014 * (i.e. kvmppc_xive_native_[gs]et_vp) can be being done. 1015 * Holding the vcpu->mutex also means that the vcpu cannot 1016 * be executing the KVM_RUN ioctl, and therefore it cannot 1017 * be executing the XIVE push or pull code or accessing 1018 * the XIVE MMIO regions. 1019 */ 1020 mutex_lock(&vcpu->mutex); 1021 kvmppc_xive_native_cleanup_vcpu(vcpu); 1022 mutex_unlock(&vcpu->mutex); 1023 } 1024 1025 /* 1026 * Now that we have cleared vcpu->arch.xive_vcpu, vcpu->arch.irq_type 1027 * and vcpu->arch.xive_esc_[vr]addr on each vcpu, we are safe 1028 * against xive code getting called during vcpu execution or 1029 * set/get one_reg operations. 1030 */ 1031 kvm->arch.xive = NULL; 1032 1033 for (i = 0; i <= xive->max_sbid; i++) { 1034 if (xive->src_blocks[i]) 1035 kvmppc_xive_free_sources(xive->src_blocks[i]); 1036 kfree(xive->src_blocks[i]); 1037 xive->src_blocks[i] = NULL; 1038 } 1039 1040 if (xive->vp_base != XIVE_INVALID_VP) 1041 xive_native_free_vp_block(xive->vp_base); 1042 1043 /* 1044 * A reference of the kvmppc_xive pointer is now kept under 1045 * the xive_devices struct of the machine for reuse. It is 1046 * freed when the VM is destroyed for now until we fix all the 1047 * execution paths. 1048 */ 1049 1050 kfree(dev); 1051 } 1052 1053 /* 1054 * Create a XIVE device. kvm->lock is held. 1055 */ 1056 static int kvmppc_xive_native_create(struct kvm_device *dev, u32 type) 1057 { 1058 struct kvmppc_xive *xive; 1059 struct kvm *kvm = dev->kvm; 1060 int ret = 0; 1061 1062 pr_devel("Creating xive native device\n"); 1063 1064 if (kvm->arch.xive) 1065 return -EEXIST; 1066 1067 xive = kvmppc_xive_get_device(kvm, type); 1068 if (!xive) 1069 return -ENOMEM; 1070 1071 dev->private = xive; 1072 xive->dev = dev; 1073 xive->kvm = kvm; 1074 kvm->arch.xive = xive; 1075 mutex_init(&xive->mapping_lock); 1076 mutex_init(&xive->lock); 1077 1078 /* 1079 * Allocate a bunch of VPs. KVM_MAX_VCPUS is a large value for 1080 * a default. Getting the max number of CPUs the VM was 1081 * configured with would improve our usage of the XIVE VP space. 1082 */ 1083 xive->vp_base = xive_native_alloc_vp_block(KVM_MAX_VCPUS); 1084 pr_devel("VP_Base=%x\n", xive->vp_base); 1085 1086 if (xive->vp_base == XIVE_INVALID_VP) 1087 ret = -ENXIO; 1088 1089 xive->single_escalation = xive_native_has_single_escalation(); 1090 xive->ops = &kvmppc_xive_native_ops; 1091 1092 if (ret) 1093 return ret; 1094 1095 return 0; 1096 } 1097 1098 /* 1099 * Interrupt Pending Buffer (IPB) offset 1100 */ 1101 #define TM_IPB_SHIFT 40 1102 #define TM_IPB_MASK (((u64) 0xFF) << TM_IPB_SHIFT) 1103 1104 int kvmppc_xive_native_get_vp(struct kvm_vcpu *vcpu, union kvmppc_one_reg *val) 1105 { 1106 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 1107 u64 opal_state; 1108 int rc; 1109 1110 if (!kvmppc_xive_enabled(vcpu)) 1111 return -EPERM; 1112 1113 if (!xc) 1114 return -ENOENT; 1115 1116 /* Thread context registers. We only care about IPB and CPPR */ 1117 val->xive_timaval[0] = vcpu->arch.xive_saved_state.w01; 1118 1119 /* Get the VP state from OPAL */ 1120 rc = xive_native_get_vp_state(xc->vp_id, &opal_state); 1121 if (rc) 1122 return rc; 1123 1124 /* 1125 * Capture the backup of IPB register in the NVT structure and 1126 * merge it in our KVM VP state. 1127 */ 1128 val->xive_timaval[0] |= cpu_to_be64(opal_state & TM_IPB_MASK); 1129 1130 pr_devel("%s NSR=%02x CPPR=%02x IBP=%02x PIPR=%02x w01=%016llx w2=%08x opal=%016llx\n", 1131 __func__, 1132 vcpu->arch.xive_saved_state.nsr, 1133 vcpu->arch.xive_saved_state.cppr, 1134 vcpu->arch.xive_saved_state.ipb, 1135 vcpu->arch.xive_saved_state.pipr, 1136 vcpu->arch.xive_saved_state.w01, 1137 (u32) vcpu->arch.xive_cam_word, opal_state); 1138 1139 return 0; 1140 } 1141 1142 int kvmppc_xive_native_set_vp(struct kvm_vcpu *vcpu, union kvmppc_one_reg *val) 1143 { 1144 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 1145 struct kvmppc_xive *xive = vcpu->kvm->arch.xive; 1146 1147 pr_devel("%s w01=%016llx vp=%016llx\n", __func__, 1148 val->xive_timaval[0], val->xive_timaval[1]); 1149 1150 if (!kvmppc_xive_enabled(vcpu)) 1151 return -EPERM; 1152 1153 if (!xc || !xive) 1154 return -ENOENT; 1155 1156 /* We can't update the state of a "pushed" VCPU */ 1157 if (WARN_ON(vcpu->arch.xive_pushed)) 1158 return -EBUSY; 1159 1160 /* 1161 * Restore the thread context registers. IPB and CPPR should 1162 * be the only ones that matter. 1163 */ 1164 vcpu->arch.xive_saved_state.w01 = val->xive_timaval[0]; 1165 1166 /* 1167 * There is no need to restore the XIVE internal state (IPB 1168 * stored in the NVT) as the IPB register was merged in KVM VP 1169 * state when captured. 1170 */ 1171 return 0; 1172 } 1173 1174 static int xive_native_debug_show(struct seq_file *m, void *private) 1175 { 1176 struct kvmppc_xive *xive = m->private; 1177 struct kvm *kvm = xive->kvm; 1178 struct kvm_vcpu *vcpu; 1179 unsigned int i; 1180 1181 if (!kvm) 1182 return 0; 1183 1184 seq_puts(m, "=========\nVCPU state\n=========\n"); 1185 1186 kvm_for_each_vcpu(i, vcpu, kvm) { 1187 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 1188 1189 if (!xc) 1190 continue; 1191 1192 seq_printf(m, "cpu server %#x NSR=%02x CPPR=%02x IBP=%02x PIPR=%02x w01=%016llx w2=%08x\n", 1193 xc->server_num, 1194 vcpu->arch.xive_saved_state.nsr, 1195 vcpu->arch.xive_saved_state.cppr, 1196 vcpu->arch.xive_saved_state.ipb, 1197 vcpu->arch.xive_saved_state.pipr, 1198 vcpu->arch.xive_saved_state.w01, 1199 (u32) vcpu->arch.xive_cam_word); 1200 1201 kvmppc_xive_debug_show_queues(m, vcpu); 1202 } 1203 1204 return 0; 1205 } 1206 1207 static int xive_native_debug_open(struct inode *inode, struct file *file) 1208 { 1209 return single_open(file, xive_native_debug_show, inode->i_private); 1210 } 1211 1212 static const struct file_operations xive_native_debug_fops = { 1213 .open = xive_native_debug_open, 1214 .read = seq_read, 1215 .llseek = seq_lseek, 1216 .release = single_release, 1217 }; 1218 1219 static void xive_native_debugfs_init(struct kvmppc_xive *xive) 1220 { 1221 char *name; 1222 1223 name = kasprintf(GFP_KERNEL, "kvm-xive-%p", xive); 1224 if (!name) { 1225 pr_err("%s: no memory for name\n", __func__); 1226 return; 1227 } 1228 1229 xive->dentry = debugfs_create_file(name, 0444, powerpc_debugfs_root, 1230 xive, &xive_native_debug_fops); 1231 1232 pr_debug("%s: created %s\n", __func__, name); 1233 kfree(name); 1234 } 1235 1236 static void kvmppc_xive_native_init(struct kvm_device *dev) 1237 { 1238 struct kvmppc_xive *xive = (struct kvmppc_xive *)dev->private; 1239 1240 /* Register some debug interfaces */ 1241 xive_native_debugfs_init(xive); 1242 } 1243 1244 struct kvm_device_ops kvm_xive_native_ops = { 1245 .name = "kvm-xive-native", 1246 .create = kvmppc_xive_native_create, 1247 .init = kvmppc_xive_native_init, 1248 .release = kvmppc_xive_native_release, 1249 .set_attr = kvmppc_xive_native_set_attr, 1250 .get_attr = kvmppc_xive_native_get_attr, 1251 .has_attr = kvmppc_xive_native_has_attr, 1252 .mmap = kvmppc_xive_native_mmap, 1253 }; 1254 1255 void kvmppc_xive_native_init_module(void) 1256 { 1257 ; 1258 } 1259 1260 void kvmppc_xive_native_exit_module(void) 1261 { 1262 ; 1263 } 1264