1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2017 Benjamin Herrenschmidt, IBM Corporation. 4 */ 5 6 #define pr_fmt(fmt) "xive-kvm: " fmt 7 8 #include <linux/kernel.h> 9 #include <linux/kvm_host.h> 10 #include <linux/err.h> 11 #include <linux/gfp.h> 12 #include <linux/spinlock.h> 13 #include <linux/delay.h> 14 #include <linux/percpu.h> 15 #include <linux/cpumask.h> 16 #include <linux/uaccess.h> 17 #include <asm/kvm_book3s.h> 18 #include <asm/kvm_ppc.h> 19 #include <asm/hvcall.h> 20 #include <asm/xics.h> 21 #include <asm/xive.h> 22 #include <asm/xive-regs.h> 23 #include <asm/debug.h> 24 #include <asm/debugfs.h> 25 #include <asm/time.h> 26 #include <asm/opal.h> 27 28 #include <linux/debugfs.h> 29 #include <linux/seq_file.h> 30 31 #include "book3s_xive.h" 32 33 34 /* 35 * Virtual mode variants of the hcalls for use on radix/radix 36 * with AIL. They require the VCPU's VP to be "pushed" 37 * 38 * We still instantiate them here because we use some of the 39 * generated utility functions as well in this file. 40 */ 41 #define XIVE_RUNTIME_CHECKS 42 #define X_PFX xive_vm_ 43 #define X_STATIC static 44 #define X_STAT_PFX stat_vm_ 45 #define __x_tima xive_tima 46 #define __x_eoi_page(xd) ((void __iomem *)((xd)->eoi_mmio)) 47 #define __x_trig_page(xd) ((void __iomem *)((xd)->trig_mmio)) 48 #define __x_writeb __raw_writeb 49 #define __x_readw __raw_readw 50 #define __x_readq __raw_readq 51 #define __x_writeq __raw_writeq 52 53 #include "book3s_xive_template.c" 54 55 /* 56 * We leave a gap of a couple of interrupts in the queue to 57 * account for the IPI and additional safety guard. 58 */ 59 #define XIVE_Q_GAP 2 60 61 /* 62 * Push a vcpu's context to the XIVE on guest entry. 63 * This assumes we are in virtual mode (MMU on) 64 */ 65 void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu) 66 { 67 void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt; 68 u64 pq; 69 70 /* 71 * Nothing to do if the platform doesn't have a XIVE 72 * or this vCPU doesn't have its own XIVE context 73 * (e.g. because it's not using an in-kernel interrupt controller). 74 */ 75 if (!tima || !vcpu->arch.xive_cam_word) 76 return; 77 78 eieio(); 79 __raw_writeq(vcpu->arch.xive_saved_state.w01, tima + TM_QW1_OS); 80 __raw_writel(vcpu->arch.xive_cam_word, tima + TM_QW1_OS + TM_WORD2); 81 vcpu->arch.xive_pushed = 1; 82 eieio(); 83 84 /* 85 * We clear the irq_pending flag. There is a small chance of a 86 * race vs. the escalation interrupt happening on another 87 * processor setting it again, but the only consequence is to 88 * cause a spurious wakeup on the next H_CEDE, which is not an 89 * issue. 90 */ 91 vcpu->arch.irq_pending = 0; 92 93 /* 94 * In single escalation mode, if the escalation interrupt is 95 * on, we mask it. 96 */ 97 if (vcpu->arch.xive_esc_on) { 98 pq = __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr + 99 XIVE_ESB_SET_PQ_01)); 100 mb(); 101 102 /* 103 * We have a possible subtle race here: The escalation 104 * interrupt might have fired and be on its way to the 105 * host queue while we mask it, and if we unmask it 106 * early enough (re-cede right away), there is a 107 * theorical possibility that it fires again, thus 108 * landing in the target queue more than once which is 109 * a big no-no. 110 * 111 * Fortunately, solving this is rather easy. If the 112 * above load setting PQ to 01 returns a previous 113 * value where P is set, then we know the escalation 114 * interrupt is somewhere on its way to the host. In 115 * that case we simply don't clear the xive_esc_on 116 * flag below. It will be eventually cleared by the 117 * handler for the escalation interrupt. 118 * 119 * Then, when doing a cede, we check that flag again 120 * before re-enabling the escalation interrupt, and if 121 * set, we abort the cede. 122 */ 123 if (!(pq & XIVE_ESB_VAL_P)) 124 /* Now P is 0, we can clear the flag */ 125 vcpu->arch.xive_esc_on = 0; 126 } 127 } 128 EXPORT_SYMBOL_GPL(kvmppc_xive_push_vcpu); 129 130 /* 131 * This is a simple trigger for a generic XIVE IRQ. This must 132 * only be called for interrupts that support a trigger page 133 */ 134 static bool xive_irq_trigger(struct xive_irq_data *xd) 135 { 136 /* This should be only for MSIs */ 137 if (WARN_ON(xd->flags & XIVE_IRQ_FLAG_LSI)) 138 return false; 139 140 /* Those interrupts should always have a trigger page */ 141 if (WARN_ON(!xd->trig_mmio)) 142 return false; 143 144 out_be64(xd->trig_mmio, 0); 145 146 return true; 147 } 148 149 static irqreturn_t xive_esc_irq(int irq, void *data) 150 { 151 struct kvm_vcpu *vcpu = data; 152 153 vcpu->arch.irq_pending = 1; 154 smp_mb(); 155 if (vcpu->arch.ceded) 156 kvmppc_fast_vcpu_kick(vcpu); 157 158 /* Since we have the no-EOI flag, the interrupt is effectively 159 * disabled now. Clearing xive_esc_on means we won't bother 160 * doing so on the next entry. 161 * 162 * This also allows the entry code to know that if a PQ combination 163 * of 10 is observed while xive_esc_on is true, it means the queue 164 * contains an unprocessed escalation interrupt. We don't make use of 165 * that knowledge today but might (see comment in book3s_hv_rmhandler.S) 166 */ 167 vcpu->arch.xive_esc_on = false; 168 169 /* This orders xive_esc_on = false vs. subsequent stale_p = true */ 170 smp_wmb(); /* goes with smp_mb() in cleanup_single_escalation */ 171 172 return IRQ_HANDLED; 173 } 174 175 int kvmppc_xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio, 176 bool single_escalation) 177 { 178 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 179 struct xive_q *q = &xc->queues[prio]; 180 char *name = NULL; 181 int rc; 182 183 /* Already there ? */ 184 if (xc->esc_virq[prio]) 185 return 0; 186 187 /* Hook up the escalation interrupt */ 188 xc->esc_virq[prio] = irq_create_mapping(NULL, q->esc_irq); 189 if (!xc->esc_virq[prio]) { 190 pr_err("Failed to map escalation interrupt for queue %d of VCPU %d\n", 191 prio, xc->server_num); 192 return -EIO; 193 } 194 195 if (single_escalation) 196 name = kasprintf(GFP_KERNEL, "kvm-%d-%d", 197 vcpu->kvm->arch.lpid, xc->server_num); 198 else 199 name = kasprintf(GFP_KERNEL, "kvm-%d-%d-%d", 200 vcpu->kvm->arch.lpid, xc->server_num, prio); 201 if (!name) { 202 pr_err("Failed to allocate escalation irq name for queue %d of VCPU %d\n", 203 prio, xc->server_num); 204 rc = -ENOMEM; 205 goto error; 206 } 207 208 pr_devel("Escalation %s irq %d (prio %d)\n", name, xc->esc_virq[prio], prio); 209 210 rc = request_irq(xc->esc_virq[prio], xive_esc_irq, 211 IRQF_NO_THREAD, name, vcpu); 212 if (rc) { 213 pr_err("Failed to request escalation interrupt for queue %d of VCPU %d\n", 214 prio, xc->server_num); 215 goto error; 216 } 217 xc->esc_virq_names[prio] = name; 218 219 /* In single escalation mode, we grab the ESB MMIO of the 220 * interrupt and mask it. Also populate the VCPU v/raddr 221 * of the ESB page for use by asm entry/exit code. Finally 222 * set the XIVE_IRQ_FLAG_NO_EOI flag which will prevent the 223 * core code from performing an EOI on the escalation 224 * interrupt, thus leaving it effectively masked after 225 * it fires once. 226 */ 227 if (single_escalation) { 228 struct irq_data *d = irq_get_irq_data(xc->esc_virq[prio]); 229 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); 230 231 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01); 232 vcpu->arch.xive_esc_raddr = xd->eoi_page; 233 vcpu->arch.xive_esc_vaddr = (__force u64)xd->eoi_mmio; 234 xd->flags |= XIVE_IRQ_FLAG_NO_EOI; 235 } 236 237 return 0; 238 error: 239 irq_dispose_mapping(xc->esc_virq[prio]); 240 xc->esc_virq[prio] = 0; 241 kfree(name); 242 return rc; 243 } 244 245 static int xive_provision_queue(struct kvm_vcpu *vcpu, u8 prio) 246 { 247 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 248 struct kvmppc_xive *xive = xc->xive; 249 struct xive_q *q = &xc->queues[prio]; 250 void *qpage; 251 int rc; 252 253 if (WARN_ON(q->qpage)) 254 return 0; 255 256 /* Allocate the queue and retrieve infos on current node for now */ 257 qpage = (__be32 *)__get_free_pages(GFP_KERNEL, xive->q_page_order); 258 if (!qpage) { 259 pr_err("Failed to allocate queue %d for VCPU %d\n", 260 prio, xc->server_num); 261 return -ENOMEM; 262 } 263 memset(qpage, 0, 1 << xive->q_order); 264 265 /* 266 * Reconfigure the queue. This will set q->qpage only once the 267 * queue is fully configured. This is a requirement for prio 0 268 * as we will stop doing EOIs for every IPI as soon as we observe 269 * qpage being non-NULL, and instead will only EOI when we receive 270 * corresponding queue 0 entries 271 */ 272 rc = xive_native_configure_queue(xc->vp_id, q, prio, qpage, 273 xive->q_order, true); 274 if (rc) 275 pr_err("Failed to configure queue %d for VCPU %d\n", 276 prio, xc->server_num); 277 return rc; 278 } 279 280 /* Called with xive->lock held */ 281 static int xive_check_provisioning(struct kvm *kvm, u8 prio) 282 { 283 struct kvmppc_xive *xive = kvm->arch.xive; 284 struct kvm_vcpu *vcpu; 285 int i, rc; 286 287 lockdep_assert_held(&xive->lock); 288 289 /* Already provisioned ? */ 290 if (xive->qmap & (1 << prio)) 291 return 0; 292 293 pr_devel("Provisioning prio... %d\n", prio); 294 295 /* Provision each VCPU and enable escalations if needed */ 296 kvm_for_each_vcpu(i, vcpu, kvm) { 297 if (!vcpu->arch.xive_vcpu) 298 continue; 299 rc = xive_provision_queue(vcpu, prio); 300 if (rc == 0 && !xive->single_escalation) 301 kvmppc_xive_attach_escalation(vcpu, prio, 302 xive->single_escalation); 303 if (rc) 304 return rc; 305 } 306 307 /* Order previous stores and mark it as provisioned */ 308 mb(); 309 xive->qmap |= (1 << prio); 310 return 0; 311 } 312 313 static void xive_inc_q_pending(struct kvm *kvm, u32 server, u8 prio) 314 { 315 struct kvm_vcpu *vcpu; 316 struct kvmppc_xive_vcpu *xc; 317 struct xive_q *q; 318 319 /* Locate target server */ 320 vcpu = kvmppc_xive_find_server(kvm, server); 321 if (!vcpu) { 322 pr_warn("%s: Can't find server %d\n", __func__, server); 323 return; 324 } 325 xc = vcpu->arch.xive_vcpu; 326 if (WARN_ON(!xc)) 327 return; 328 329 q = &xc->queues[prio]; 330 atomic_inc(&q->pending_count); 331 } 332 333 static int xive_try_pick_queue(struct kvm_vcpu *vcpu, u8 prio) 334 { 335 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 336 struct xive_q *q; 337 u32 max; 338 339 if (WARN_ON(!xc)) 340 return -ENXIO; 341 if (!xc->valid) 342 return -ENXIO; 343 344 q = &xc->queues[prio]; 345 if (WARN_ON(!q->qpage)) 346 return -ENXIO; 347 348 /* Calculate max number of interrupts in that queue. */ 349 max = (q->msk + 1) - XIVE_Q_GAP; 350 return atomic_add_unless(&q->count, 1, max) ? 0 : -EBUSY; 351 } 352 353 int kvmppc_xive_select_target(struct kvm *kvm, u32 *server, u8 prio) 354 { 355 struct kvm_vcpu *vcpu; 356 int i, rc; 357 358 /* Locate target server */ 359 vcpu = kvmppc_xive_find_server(kvm, *server); 360 if (!vcpu) { 361 pr_devel("Can't find server %d\n", *server); 362 return -EINVAL; 363 } 364 365 pr_devel("Finding irq target on 0x%x/%d...\n", *server, prio); 366 367 /* Try pick it */ 368 rc = xive_try_pick_queue(vcpu, prio); 369 if (rc == 0) 370 return rc; 371 372 pr_devel(" .. failed, looking up candidate...\n"); 373 374 /* Failed, pick another VCPU */ 375 kvm_for_each_vcpu(i, vcpu, kvm) { 376 if (!vcpu->arch.xive_vcpu) 377 continue; 378 rc = xive_try_pick_queue(vcpu, prio); 379 if (rc == 0) { 380 *server = vcpu->arch.xive_vcpu->server_num; 381 pr_devel(" found on 0x%x/%d\n", *server, prio); 382 return rc; 383 } 384 } 385 pr_devel(" no available target !\n"); 386 387 /* No available target ! */ 388 return -EBUSY; 389 } 390 391 static u8 xive_lock_and_mask(struct kvmppc_xive *xive, 392 struct kvmppc_xive_src_block *sb, 393 struct kvmppc_xive_irq_state *state) 394 { 395 struct xive_irq_data *xd; 396 u32 hw_num; 397 u8 old_prio; 398 u64 val; 399 400 /* 401 * Take the lock, set masked, try again if racing 402 * with H_EOI 403 */ 404 for (;;) { 405 arch_spin_lock(&sb->lock); 406 old_prio = state->guest_priority; 407 state->guest_priority = MASKED; 408 mb(); 409 if (!state->in_eoi) 410 break; 411 state->guest_priority = old_prio; 412 arch_spin_unlock(&sb->lock); 413 } 414 415 /* No change ? Bail */ 416 if (old_prio == MASKED) 417 return old_prio; 418 419 /* Get the right irq */ 420 kvmppc_xive_select_irq(state, &hw_num, &xd); 421 422 /* Set PQ to 10, return old P and old Q and remember them */ 423 val = xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_10); 424 state->old_p = !!(val & 2); 425 state->old_q = !!(val & 1); 426 427 /* 428 * Synchronize hardware to sensure the queues are updated when 429 * masking 430 */ 431 xive_native_sync_source(hw_num); 432 433 return old_prio; 434 } 435 436 static void xive_lock_for_unmask(struct kvmppc_xive_src_block *sb, 437 struct kvmppc_xive_irq_state *state) 438 { 439 /* 440 * Take the lock try again if racing with H_EOI 441 */ 442 for (;;) { 443 arch_spin_lock(&sb->lock); 444 if (!state->in_eoi) 445 break; 446 arch_spin_unlock(&sb->lock); 447 } 448 } 449 450 static void xive_finish_unmask(struct kvmppc_xive *xive, 451 struct kvmppc_xive_src_block *sb, 452 struct kvmppc_xive_irq_state *state, 453 u8 prio) 454 { 455 struct xive_irq_data *xd; 456 u32 hw_num; 457 458 /* If we aren't changing a thing, move on */ 459 if (state->guest_priority != MASKED) 460 goto bail; 461 462 /* Get the right irq */ 463 kvmppc_xive_select_irq(state, &hw_num, &xd); 464 465 /* Old Q set, set PQ to 11 */ 466 if (state->old_q) 467 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_11); 468 469 /* 470 * If not old P, then perform an "effective" EOI, 471 * on the source. This will handle the cases where 472 * FW EOI is needed. 473 */ 474 if (!state->old_p) 475 xive_vm_source_eoi(hw_num, xd); 476 477 /* Synchronize ordering and mark unmasked */ 478 mb(); 479 bail: 480 state->guest_priority = prio; 481 } 482 483 /* 484 * Target an interrupt to a given server/prio, this will fallback 485 * to another server if necessary and perform the HW targetting 486 * updates as needed 487 * 488 * NOTE: Must be called with the state lock held 489 */ 490 static int xive_target_interrupt(struct kvm *kvm, 491 struct kvmppc_xive_irq_state *state, 492 u32 server, u8 prio) 493 { 494 struct kvmppc_xive *xive = kvm->arch.xive; 495 u32 hw_num; 496 int rc; 497 498 /* 499 * This will return a tentative server and actual 500 * priority. The count for that new target will have 501 * already been incremented. 502 */ 503 rc = kvmppc_xive_select_target(kvm, &server, prio); 504 505 /* 506 * We failed to find a target ? Not much we can do 507 * at least until we support the GIQ. 508 */ 509 if (rc) 510 return rc; 511 512 /* 513 * Increment the old queue pending count if there 514 * was one so that the old queue count gets adjusted later 515 * when observed to be empty. 516 */ 517 if (state->act_priority != MASKED) 518 xive_inc_q_pending(kvm, 519 state->act_server, 520 state->act_priority); 521 /* 522 * Update state and HW 523 */ 524 state->act_priority = prio; 525 state->act_server = server; 526 527 /* Get the right irq */ 528 kvmppc_xive_select_irq(state, &hw_num, NULL); 529 530 return xive_native_configure_irq(hw_num, 531 kvmppc_xive_vp(xive, server), 532 prio, state->number); 533 } 534 535 /* 536 * Targetting rules: In order to avoid losing track of 537 * pending interrupts accross mask and unmask, which would 538 * allow queue overflows, we implement the following rules: 539 * 540 * - Unless it was never enabled (or we run out of capacity) 541 * an interrupt is always targetted at a valid server/queue 542 * pair even when "masked" by the guest. This pair tends to 543 * be the last one used but it can be changed under some 544 * circumstances. That allows us to separate targetting 545 * from masking, we only handle accounting during (re)targetting, 546 * this also allows us to let an interrupt drain into its target 547 * queue after masking, avoiding complex schemes to remove 548 * interrupts out of remote processor queues. 549 * 550 * - When masking, we set PQ to 10 and save the previous value 551 * of P and Q. 552 * 553 * - When unmasking, if saved Q was set, we set PQ to 11 554 * otherwise we leave PQ to the HW state which will be either 555 * 10 if nothing happened or 11 if the interrupt fired while 556 * masked. Effectively we are OR'ing the previous Q into the 557 * HW Q. 558 * 559 * Then if saved P is clear, we do an effective EOI (Q->P->Trigger) 560 * which will unmask the interrupt and shoot a new one if Q was 561 * set. 562 * 563 * Otherwise (saved P is set) we leave PQ unchanged (so 10 or 11, 564 * effectively meaning an H_EOI from the guest is still expected 565 * for that interrupt). 566 * 567 * - If H_EOI occurs while masked, we clear the saved P. 568 * 569 * - When changing target, we account on the new target and 570 * increment a separate "pending" counter on the old one. 571 * This pending counter will be used to decrement the old 572 * target's count when its queue has been observed empty. 573 */ 574 575 int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server, 576 u32 priority) 577 { 578 struct kvmppc_xive *xive = kvm->arch.xive; 579 struct kvmppc_xive_src_block *sb; 580 struct kvmppc_xive_irq_state *state; 581 u8 new_act_prio; 582 int rc = 0; 583 u16 idx; 584 585 if (!xive) 586 return -ENODEV; 587 588 pr_devel("set_xive ! irq 0x%x server 0x%x prio %d\n", 589 irq, server, priority); 590 591 /* First, check provisioning of queues */ 592 if (priority != MASKED) { 593 mutex_lock(&xive->lock); 594 rc = xive_check_provisioning(xive->kvm, 595 xive_prio_from_guest(priority)); 596 mutex_unlock(&xive->lock); 597 } 598 if (rc) { 599 pr_devel(" provisioning failure %d !\n", rc); 600 return rc; 601 } 602 603 sb = kvmppc_xive_find_source(xive, irq, &idx); 604 if (!sb) 605 return -EINVAL; 606 state = &sb->irq_state[idx]; 607 608 /* 609 * We first handle masking/unmasking since the locking 610 * might need to be retried due to EOIs, we'll handle 611 * targetting changes later. These functions will return 612 * with the SB lock held. 613 * 614 * xive_lock_and_mask() will also set state->guest_priority 615 * but won't otherwise change other fields of the state. 616 * 617 * xive_lock_for_unmask will not actually unmask, this will 618 * be done later by xive_finish_unmask() once the targetting 619 * has been done, so we don't try to unmask an interrupt 620 * that hasn't yet been targetted. 621 */ 622 if (priority == MASKED) 623 xive_lock_and_mask(xive, sb, state); 624 else 625 xive_lock_for_unmask(sb, state); 626 627 628 /* 629 * Then we handle targetting. 630 * 631 * First calculate a new "actual priority" 632 */ 633 new_act_prio = state->act_priority; 634 if (priority != MASKED) 635 new_act_prio = xive_prio_from_guest(priority); 636 637 pr_devel(" new_act_prio=%x act_server=%x act_prio=%x\n", 638 new_act_prio, state->act_server, state->act_priority); 639 640 /* 641 * Then check if we actually need to change anything, 642 * 643 * The condition for re-targetting the interrupt is that 644 * we have a valid new priority (new_act_prio is not 0xff) 645 * and either the server or the priority changed. 646 * 647 * Note: If act_priority was ff and the new priority is 648 * also ff, we don't do anything and leave the interrupt 649 * untargetted. An attempt of doing an int_on on an 650 * untargetted interrupt will fail. If that is a problem 651 * we could initialize interrupts with valid default 652 */ 653 654 if (new_act_prio != MASKED && 655 (state->act_server != server || 656 state->act_priority != new_act_prio)) 657 rc = xive_target_interrupt(kvm, state, server, new_act_prio); 658 659 /* 660 * Perform the final unmasking of the interrupt source 661 * if necessary 662 */ 663 if (priority != MASKED) 664 xive_finish_unmask(xive, sb, state, priority); 665 666 /* 667 * Finally Update saved_priority to match. Only int_on/off 668 * set this field to a different value. 669 */ 670 state->saved_priority = priority; 671 672 arch_spin_unlock(&sb->lock); 673 return rc; 674 } 675 676 int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server, 677 u32 *priority) 678 { 679 struct kvmppc_xive *xive = kvm->arch.xive; 680 struct kvmppc_xive_src_block *sb; 681 struct kvmppc_xive_irq_state *state; 682 u16 idx; 683 684 if (!xive) 685 return -ENODEV; 686 687 sb = kvmppc_xive_find_source(xive, irq, &idx); 688 if (!sb) 689 return -EINVAL; 690 state = &sb->irq_state[idx]; 691 arch_spin_lock(&sb->lock); 692 *server = state->act_server; 693 *priority = state->guest_priority; 694 arch_spin_unlock(&sb->lock); 695 696 return 0; 697 } 698 699 int kvmppc_xive_int_on(struct kvm *kvm, u32 irq) 700 { 701 struct kvmppc_xive *xive = kvm->arch.xive; 702 struct kvmppc_xive_src_block *sb; 703 struct kvmppc_xive_irq_state *state; 704 u16 idx; 705 706 if (!xive) 707 return -ENODEV; 708 709 sb = kvmppc_xive_find_source(xive, irq, &idx); 710 if (!sb) 711 return -EINVAL; 712 state = &sb->irq_state[idx]; 713 714 pr_devel("int_on(irq=0x%x)\n", irq); 715 716 /* 717 * Check if interrupt was not targetted 718 */ 719 if (state->act_priority == MASKED) { 720 pr_devel("int_on on untargetted interrupt\n"); 721 return -EINVAL; 722 } 723 724 /* If saved_priority is 0xff, do nothing */ 725 if (state->saved_priority == MASKED) 726 return 0; 727 728 /* 729 * Lock and unmask it. 730 */ 731 xive_lock_for_unmask(sb, state); 732 xive_finish_unmask(xive, sb, state, state->saved_priority); 733 arch_spin_unlock(&sb->lock); 734 735 return 0; 736 } 737 738 int kvmppc_xive_int_off(struct kvm *kvm, u32 irq) 739 { 740 struct kvmppc_xive *xive = kvm->arch.xive; 741 struct kvmppc_xive_src_block *sb; 742 struct kvmppc_xive_irq_state *state; 743 u16 idx; 744 745 if (!xive) 746 return -ENODEV; 747 748 sb = kvmppc_xive_find_source(xive, irq, &idx); 749 if (!sb) 750 return -EINVAL; 751 state = &sb->irq_state[idx]; 752 753 pr_devel("int_off(irq=0x%x)\n", irq); 754 755 /* 756 * Lock and mask 757 */ 758 state->saved_priority = xive_lock_and_mask(xive, sb, state); 759 arch_spin_unlock(&sb->lock); 760 761 return 0; 762 } 763 764 static bool xive_restore_pending_irq(struct kvmppc_xive *xive, u32 irq) 765 { 766 struct kvmppc_xive_src_block *sb; 767 struct kvmppc_xive_irq_state *state; 768 u16 idx; 769 770 sb = kvmppc_xive_find_source(xive, irq, &idx); 771 if (!sb) 772 return false; 773 state = &sb->irq_state[idx]; 774 if (!state->valid) 775 return false; 776 777 /* 778 * Trigger the IPI. This assumes we never restore a pass-through 779 * interrupt which should be safe enough 780 */ 781 xive_irq_trigger(&state->ipi_data); 782 783 return true; 784 } 785 786 u64 kvmppc_xive_get_icp(struct kvm_vcpu *vcpu) 787 { 788 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 789 790 if (!xc) 791 return 0; 792 793 /* Return the per-cpu state for state saving/migration */ 794 return (u64)xc->cppr << KVM_REG_PPC_ICP_CPPR_SHIFT | 795 (u64)xc->mfrr << KVM_REG_PPC_ICP_MFRR_SHIFT | 796 (u64)0xff << KVM_REG_PPC_ICP_PPRI_SHIFT; 797 } 798 799 int kvmppc_xive_set_icp(struct kvm_vcpu *vcpu, u64 icpval) 800 { 801 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 802 struct kvmppc_xive *xive = vcpu->kvm->arch.xive; 803 u8 cppr, mfrr; 804 u32 xisr; 805 806 if (!xc || !xive) 807 return -ENOENT; 808 809 /* Grab individual state fields. We don't use pending_pri */ 810 cppr = icpval >> KVM_REG_PPC_ICP_CPPR_SHIFT; 811 xisr = (icpval >> KVM_REG_PPC_ICP_XISR_SHIFT) & 812 KVM_REG_PPC_ICP_XISR_MASK; 813 mfrr = icpval >> KVM_REG_PPC_ICP_MFRR_SHIFT; 814 815 pr_devel("set_icp vcpu %d cppr=0x%x mfrr=0x%x xisr=0x%x\n", 816 xc->server_num, cppr, mfrr, xisr); 817 818 /* 819 * We can't update the state of a "pushed" VCPU, but that 820 * shouldn't happen because the vcpu->mutex makes running a 821 * vcpu mutually exclusive with doing one_reg get/set on it. 822 */ 823 if (WARN_ON(vcpu->arch.xive_pushed)) 824 return -EIO; 825 826 /* Update VCPU HW saved state */ 827 vcpu->arch.xive_saved_state.cppr = cppr; 828 xc->hw_cppr = xc->cppr = cppr; 829 830 /* 831 * Update MFRR state. If it's not 0xff, we mark the VCPU as 832 * having a pending MFRR change, which will re-evaluate the 833 * target. The VCPU will thus potentially get a spurious 834 * interrupt but that's not a big deal. 835 */ 836 xc->mfrr = mfrr; 837 if (mfrr < cppr) 838 xive_irq_trigger(&xc->vp_ipi_data); 839 840 /* 841 * Now saved XIRR is "interesting". It means there's something in 842 * the legacy "1 element" queue... for an IPI we simply ignore it, 843 * as the MFRR restore will handle that. For anything else we need 844 * to force a resend of the source. 845 * However the source may not have been setup yet. If that's the 846 * case, we keep that info and increment a counter in the xive to 847 * tell subsequent xive_set_source() to go look. 848 */ 849 if (xisr > XICS_IPI && !xive_restore_pending_irq(xive, xisr)) { 850 xc->delayed_irq = xisr; 851 xive->delayed_irqs++; 852 pr_devel(" xisr restore delayed\n"); 853 } 854 855 return 0; 856 } 857 858 int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq, 859 struct irq_desc *host_desc) 860 { 861 struct kvmppc_xive *xive = kvm->arch.xive; 862 struct kvmppc_xive_src_block *sb; 863 struct kvmppc_xive_irq_state *state; 864 struct irq_data *host_data = irq_desc_get_irq_data(host_desc); 865 unsigned int host_irq = irq_desc_get_irq(host_desc); 866 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(host_data); 867 u16 idx; 868 u8 prio; 869 int rc; 870 871 if (!xive) 872 return -ENODEV; 873 874 pr_devel("set_mapped girq 0x%lx host HW irq 0x%x...\n",guest_irq, hw_irq); 875 876 sb = kvmppc_xive_find_source(xive, guest_irq, &idx); 877 if (!sb) 878 return -EINVAL; 879 state = &sb->irq_state[idx]; 880 881 /* 882 * Mark the passed-through interrupt as going to a VCPU, 883 * this will prevent further EOIs and similar operations 884 * from the XIVE code. It will also mask the interrupt 885 * to either PQ=10 or 11 state, the latter if the interrupt 886 * is pending. This will allow us to unmask or retrigger it 887 * after routing it to the guest with a simple EOI. 888 * 889 * The "state" argument is a "token", all it needs is to be 890 * non-NULL to switch to passed-through or NULL for the 891 * other way around. We may not yet have an actual VCPU 892 * target here and we don't really care. 893 */ 894 rc = irq_set_vcpu_affinity(host_irq, state); 895 if (rc) { 896 pr_err("Failed to set VCPU affinity for irq %d\n", host_irq); 897 return rc; 898 } 899 900 /* 901 * Mask and read state of IPI. We need to know if its P bit 902 * is set as that means it's potentially already using a 903 * queue entry in the target 904 */ 905 prio = xive_lock_and_mask(xive, sb, state); 906 pr_devel(" old IPI prio %02x P:%d Q:%d\n", prio, 907 state->old_p, state->old_q); 908 909 /* Turn the IPI hard off */ 910 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01); 911 912 /* 913 * Reset ESB guest mapping. Needed when ESB pages are exposed 914 * to the guest in XIVE native mode 915 */ 916 if (xive->ops && xive->ops->reset_mapped) 917 xive->ops->reset_mapped(kvm, guest_irq); 918 919 /* Grab info about irq */ 920 state->pt_number = hw_irq; 921 state->pt_data = irq_data_get_irq_handler_data(host_data); 922 923 /* 924 * Configure the IRQ to match the existing configuration of 925 * the IPI if it was already targetted. Otherwise this will 926 * mask the interrupt in a lossy way (act_priority is 0xff) 927 * which is fine for a never started interrupt. 928 */ 929 xive_native_configure_irq(hw_irq, 930 kvmppc_xive_vp(xive, state->act_server), 931 state->act_priority, state->number); 932 933 /* 934 * We do an EOI to enable the interrupt (and retrigger if needed) 935 * if the guest has the interrupt unmasked and the P bit was *not* 936 * set in the IPI. If it was set, we know a slot may still be in 937 * use in the target queue thus we have to wait for a guest 938 * originated EOI 939 */ 940 if (prio != MASKED && !state->old_p) 941 xive_vm_source_eoi(hw_irq, state->pt_data); 942 943 /* Clear old_p/old_q as they are no longer relevant */ 944 state->old_p = state->old_q = false; 945 946 /* Restore guest prio (unlocks EOI) */ 947 mb(); 948 state->guest_priority = prio; 949 arch_spin_unlock(&sb->lock); 950 951 return 0; 952 } 953 EXPORT_SYMBOL_GPL(kvmppc_xive_set_mapped); 954 955 int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq, 956 struct irq_desc *host_desc) 957 { 958 struct kvmppc_xive *xive = kvm->arch.xive; 959 struct kvmppc_xive_src_block *sb; 960 struct kvmppc_xive_irq_state *state; 961 unsigned int host_irq = irq_desc_get_irq(host_desc); 962 u16 idx; 963 u8 prio; 964 int rc; 965 966 if (!xive) 967 return -ENODEV; 968 969 pr_devel("clr_mapped girq 0x%lx...\n", guest_irq); 970 971 sb = kvmppc_xive_find_source(xive, guest_irq, &idx); 972 if (!sb) 973 return -EINVAL; 974 state = &sb->irq_state[idx]; 975 976 /* 977 * Mask and read state of IRQ. We need to know if its P bit 978 * is set as that means it's potentially already using a 979 * queue entry in the target 980 */ 981 prio = xive_lock_and_mask(xive, sb, state); 982 pr_devel(" old IRQ prio %02x P:%d Q:%d\n", prio, 983 state->old_p, state->old_q); 984 985 /* 986 * If old_p is set, the interrupt is pending, we switch it to 987 * PQ=11. This will force a resend in the host so the interrupt 988 * isn't lost to whatver host driver may pick it up 989 */ 990 if (state->old_p) 991 xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_11); 992 993 /* Release the passed-through interrupt to the host */ 994 rc = irq_set_vcpu_affinity(host_irq, NULL); 995 if (rc) { 996 pr_err("Failed to clr VCPU affinity for irq %d\n", host_irq); 997 return rc; 998 } 999 1000 /* Forget about the IRQ */ 1001 state->pt_number = 0; 1002 state->pt_data = NULL; 1003 1004 /* 1005 * Reset ESB guest mapping. Needed when ESB pages are exposed 1006 * to the guest in XIVE native mode 1007 */ 1008 if (xive->ops && xive->ops->reset_mapped) { 1009 xive->ops->reset_mapped(kvm, guest_irq); 1010 } 1011 1012 /* Reconfigure the IPI */ 1013 xive_native_configure_irq(state->ipi_number, 1014 kvmppc_xive_vp(xive, state->act_server), 1015 state->act_priority, state->number); 1016 1017 /* 1018 * If old_p is set (we have a queue entry potentially 1019 * occupied) or the interrupt is masked, we set the IPI 1020 * to PQ=10 state. Otherwise we just re-enable it (PQ=00). 1021 */ 1022 if (prio == MASKED || state->old_p) 1023 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_10); 1024 else 1025 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_00); 1026 1027 /* Restore guest prio (unlocks EOI) */ 1028 mb(); 1029 state->guest_priority = prio; 1030 arch_spin_unlock(&sb->lock); 1031 1032 return 0; 1033 } 1034 EXPORT_SYMBOL_GPL(kvmppc_xive_clr_mapped); 1035 1036 void kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu *vcpu) 1037 { 1038 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 1039 struct kvm *kvm = vcpu->kvm; 1040 struct kvmppc_xive *xive = kvm->arch.xive; 1041 int i, j; 1042 1043 for (i = 0; i <= xive->max_sbid; i++) { 1044 struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 1045 1046 if (!sb) 1047 continue; 1048 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) { 1049 struct kvmppc_xive_irq_state *state = &sb->irq_state[j]; 1050 1051 if (!state->valid) 1052 continue; 1053 if (state->act_priority == MASKED) 1054 continue; 1055 if (state->act_server != xc->server_num) 1056 continue; 1057 1058 /* Clean it up */ 1059 arch_spin_lock(&sb->lock); 1060 state->act_priority = MASKED; 1061 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01); 1062 xive_native_configure_irq(state->ipi_number, 0, MASKED, 0); 1063 if (state->pt_number) { 1064 xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01); 1065 xive_native_configure_irq(state->pt_number, 0, MASKED, 0); 1066 } 1067 arch_spin_unlock(&sb->lock); 1068 } 1069 } 1070 1071 /* Disable vcpu's escalation interrupt */ 1072 if (vcpu->arch.xive_esc_on) { 1073 __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr + 1074 XIVE_ESB_SET_PQ_01)); 1075 vcpu->arch.xive_esc_on = false; 1076 } 1077 1078 /* 1079 * Clear pointers to escalation interrupt ESB. 1080 * This is safe because the vcpu->mutex is held, preventing 1081 * any other CPU from concurrently executing a KVM_RUN ioctl. 1082 */ 1083 vcpu->arch.xive_esc_vaddr = 0; 1084 vcpu->arch.xive_esc_raddr = 0; 1085 } 1086 1087 /* 1088 * In single escalation mode, the escalation interrupt is marked so 1089 * that EOI doesn't re-enable it, but just sets the stale_p flag to 1090 * indicate that the P bit has already been dealt with. However, the 1091 * assembly code that enters the guest sets PQ to 00 without clearing 1092 * stale_p (because it has no easy way to address it). Hence we have 1093 * to adjust stale_p before shutting down the interrupt. 1094 */ 1095 void xive_cleanup_single_escalation(struct kvm_vcpu *vcpu, 1096 struct kvmppc_xive_vcpu *xc, int irq) 1097 { 1098 struct irq_data *d = irq_get_irq_data(irq); 1099 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); 1100 1101 /* 1102 * This slightly odd sequence gives the right result 1103 * (i.e. stale_p set if xive_esc_on is false) even if 1104 * we race with xive_esc_irq() and xive_irq_eoi(). 1105 */ 1106 xd->stale_p = false; 1107 smp_mb(); /* paired with smb_wmb in xive_esc_irq */ 1108 if (!vcpu->arch.xive_esc_on) 1109 xd->stale_p = true; 1110 } 1111 1112 void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu) 1113 { 1114 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 1115 struct kvmppc_xive *xive = vcpu->kvm->arch.xive; 1116 int i; 1117 1118 if (!kvmppc_xics_enabled(vcpu)) 1119 return; 1120 1121 if (!xc) 1122 return; 1123 1124 pr_devel("cleanup_vcpu(cpu=%d)\n", xc->server_num); 1125 1126 /* Ensure no interrupt is still routed to that VP */ 1127 xc->valid = false; 1128 kvmppc_xive_disable_vcpu_interrupts(vcpu); 1129 1130 /* Mask the VP IPI */ 1131 xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_01); 1132 1133 /* Free escalations */ 1134 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) { 1135 if (xc->esc_virq[i]) { 1136 if (xc->xive->single_escalation) 1137 xive_cleanup_single_escalation(vcpu, xc, 1138 xc->esc_virq[i]); 1139 free_irq(xc->esc_virq[i], vcpu); 1140 irq_dispose_mapping(xc->esc_virq[i]); 1141 kfree(xc->esc_virq_names[i]); 1142 } 1143 } 1144 1145 /* Disable the VP */ 1146 xive_native_disable_vp(xc->vp_id); 1147 1148 /* Clear the cam word so guest entry won't try to push context */ 1149 vcpu->arch.xive_cam_word = 0; 1150 1151 /* Free the queues */ 1152 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) { 1153 struct xive_q *q = &xc->queues[i]; 1154 1155 xive_native_disable_queue(xc->vp_id, q, i); 1156 if (q->qpage) { 1157 free_pages((unsigned long)q->qpage, 1158 xive->q_page_order); 1159 q->qpage = NULL; 1160 } 1161 } 1162 1163 /* Free the IPI */ 1164 if (xc->vp_ipi) { 1165 xive_cleanup_irq_data(&xc->vp_ipi_data); 1166 xive_native_free_irq(xc->vp_ipi); 1167 } 1168 /* Free the VP */ 1169 kfree(xc); 1170 1171 /* Cleanup the vcpu */ 1172 vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT; 1173 vcpu->arch.xive_vcpu = NULL; 1174 } 1175 1176 static bool kvmppc_xive_vcpu_id_valid(struct kvmppc_xive *xive, u32 cpu) 1177 { 1178 /* We have a block of xive->nr_servers VPs. We just need to check 1179 * raw vCPU ids are below the expected limit for this guest's 1180 * core stride ; kvmppc_pack_vcpu_id() will pack them down to an 1181 * index that can be safely used to compute a VP id that belongs 1182 * to the VP block. 1183 */ 1184 return cpu < xive->nr_servers * xive->kvm->arch.emul_smt_mode; 1185 } 1186 1187 int kvmppc_xive_compute_vp_id(struct kvmppc_xive *xive, u32 cpu, u32 *vp) 1188 { 1189 u32 vp_id; 1190 1191 if (!kvmppc_xive_vcpu_id_valid(xive, cpu)) { 1192 pr_devel("Out of bounds !\n"); 1193 return -EINVAL; 1194 } 1195 1196 if (xive->vp_base == XIVE_INVALID_VP) { 1197 xive->vp_base = xive_native_alloc_vp_block(xive->nr_servers); 1198 pr_devel("VP_Base=%x nr_servers=%d\n", xive->vp_base, xive->nr_servers); 1199 1200 if (xive->vp_base == XIVE_INVALID_VP) 1201 return -ENOSPC; 1202 } 1203 1204 vp_id = kvmppc_xive_vp(xive, cpu); 1205 if (kvmppc_xive_vp_in_use(xive->kvm, vp_id)) { 1206 pr_devel("Duplicate !\n"); 1207 return -EEXIST; 1208 } 1209 1210 *vp = vp_id; 1211 1212 return 0; 1213 } 1214 1215 int kvmppc_xive_connect_vcpu(struct kvm_device *dev, 1216 struct kvm_vcpu *vcpu, u32 cpu) 1217 { 1218 struct kvmppc_xive *xive = dev->private; 1219 struct kvmppc_xive_vcpu *xc; 1220 int i, r = -EBUSY; 1221 u32 vp_id; 1222 1223 pr_devel("connect_vcpu(cpu=%d)\n", cpu); 1224 1225 if (dev->ops != &kvm_xive_ops) { 1226 pr_devel("Wrong ops !\n"); 1227 return -EPERM; 1228 } 1229 if (xive->kvm != vcpu->kvm) 1230 return -EPERM; 1231 if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT) 1232 return -EBUSY; 1233 1234 /* We need to synchronize with queue provisioning */ 1235 mutex_lock(&xive->lock); 1236 1237 r = kvmppc_xive_compute_vp_id(xive, cpu, &vp_id); 1238 if (r) 1239 goto bail; 1240 1241 xc = kzalloc(sizeof(*xc), GFP_KERNEL); 1242 if (!xc) { 1243 r = -ENOMEM; 1244 goto bail; 1245 } 1246 1247 vcpu->arch.xive_vcpu = xc; 1248 xc->xive = xive; 1249 xc->vcpu = vcpu; 1250 xc->server_num = cpu; 1251 xc->vp_id = vp_id; 1252 xc->mfrr = 0xff; 1253 xc->valid = true; 1254 1255 r = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id); 1256 if (r) 1257 goto bail; 1258 1259 /* Configure VCPU fields for use by assembly push/pull */ 1260 vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000); 1261 vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO); 1262 1263 /* Allocate IPI */ 1264 xc->vp_ipi = xive_native_alloc_irq(); 1265 if (!xc->vp_ipi) { 1266 pr_err("Failed to allocate xive irq for VCPU IPI\n"); 1267 r = -EIO; 1268 goto bail; 1269 } 1270 pr_devel(" IPI=0x%x\n", xc->vp_ipi); 1271 1272 r = xive_native_populate_irq_data(xc->vp_ipi, &xc->vp_ipi_data); 1273 if (r) 1274 goto bail; 1275 1276 /* 1277 * Enable the VP first as the single escalation mode will 1278 * affect escalation interrupts numbering 1279 */ 1280 r = xive_native_enable_vp(xc->vp_id, xive->single_escalation); 1281 if (r) { 1282 pr_err("Failed to enable VP in OPAL, err %d\n", r); 1283 goto bail; 1284 } 1285 1286 /* 1287 * Initialize queues. Initially we set them all for no queueing 1288 * and we enable escalation for queue 0 only which we'll use for 1289 * our mfrr change notifications. If the VCPU is hot-plugged, we 1290 * do handle provisioning however based on the existing "map" 1291 * of enabled queues. 1292 */ 1293 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) { 1294 struct xive_q *q = &xc->queues[i]; 1295 1296 /* Single escalation, no queue 7 */ 1297 if (i == 7 && xive->single_escalation) 1298 break; 1299 1300 /* Is queue already enabled ? Provision it */ 1301 if (xive->qmap & (1 << i)) { 1302 r = xive_provision_queue(vcpu, i); 1303 if (r == 0 && !xive->single_escalation) 1304 kvmppc_xive_attach_escalation( 1305 vcpu, i, xive->single_escalation); 1306 if (r) 1307 goto bail; 1308 } else { 1309 r = xive_native_configure_queue(xc->vp_id, 1310 q, i, NULL, 0, true); 1311 if (r) { 1312 pr_err("Failed to configure queue %d for VCPU %d\n", 1313 i, cpu); 1314 goto bail; 1315 } 1316 } 1317 } 1318 1319 /* If not done above, attach priority 0 escalation */ 1320 r = kvmppc_xive_attach_escalation(vcpu, 0, xive->single_escalation); 1321 if (r) 1322 goto bail; 1323 1324 /* Route the IPI */ 1325 r = xive_native_configure_irq(xc->vp_ipi, xc->vp_id, 0, XICS_IPI); 1326 if (!r) 1327 xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_00); 1328 1329 bail: 1330 mutex_unlock(&xive->lock); 1331 if (r) { 1332 kvmppc_xive_cleanup_vcpu(vcpu); 1333 return r; 1334 } 1335 1336 vcpu->arch.irq_type = KVMPPC_IRQ_XICS; 1337 return 0; 1338 } 1339 1340 /* 1341 * Scanning of queues before/after migration save 1342 */ 1343 static void xive_pre_save_set_queued(struct kvmppc_xive *xive, u32 irq) 1344 { 1345 struct kvmppc_xive_src_block *sb; 1346 struct kvmppc_xive_irq_state *state; 1347 u16 idx; 1348 1349 sb = kvmppc_xive_find_source(xive, irq, &idx); 1350 if (!sb) 1351 return; 1352 1353 state = &sb->irq_state[idx]; 1354 1355 /* Some sanity checking */ 1356 if (!state->valid) { 1357 pr_err("invalid irq 0x%x in cpu queue!\n", irq); 1358 return; 1359 } 1360 1361 /* 1362 * If the interrupt is in a queue it should have P set. 1363 * We warn so that gets reported. A backtrace isn't useful 1364 * so no need to use a WARN_ON. 1365 */ 1366 if (!state->saved_p) 1367 pr_err("Interrupt 0x%x is marked in a queue but P not set !\n", irq); 1368 1369 /* Set flag */ 1370 state->in_queue = true; 1371 } 1372 1373 static void xive_pre_save_mask_irq(struct kvmppc_xive *xive, 1374 struct kvmppc_xive_src_block *sb, 1375 u32 irq) 1376 { 1377 struct kvmppc_xive_irq_state *state = &sb->irq_state[irq]; 1378 1379 if (!state->valid) 1380 return; 1381 1382 /* Mask and save state, this will also sync HW queues */ 1383 state->saved_scan_prio = xive_lock_and_mask(xive, sb, state); 1384 1385 /* Transfer P and Q */ 1386 state->saved_p = state->old_p; 1387 state->saved_q = state->old_q; 1388 1389 /* Unlock */ 1390 arch_spin_unlock(&sb->lock); 1391 } 1392 1393 static void xive_pre_save_unmask_irq(struct kvmppc_xive *xive, 1394 struct kvmppc_xive_src_block *sb, 1395 u32 irq) 1396 { 1397 struct kvmppc_xive_irq_state *state = &sb->irq_state[irq]; 1398 1399 if (!state->valid) 1400 return; 1401 1402 /* 1403 * Lock / exclude EOI (not technically necessary if the 1404 * guest isn't running concurrently. If this becomes a 1405 * performance issue we can probably remove the lock. 1406 */ 1407 xive_lock_for_unmask(sb, state); 1408 1409 /* Restore mask/prio if it wasn't masked */ 1410 if (state->saved_scan_prio != MASKED) 1411 xive_finish_unmask(xive, sb, state, state->saved_scan_prio); 1412 1413 /* Unlock */ 1414 arch_spin_unlock(&sb->lock); 1415 } 1416 1417 static void xive_pre_save_queue(struct kvmppc_xive *xive, struct xive_q *q) 1418 { 1419 u32 idx = q->idx; 1420 u32 toggle = q->toggle; 1421 u32 irq; 1422 1423 do { 1424 irq = __xive_read_eq(q->qpage, q->msk, &idx, &toggle); 1425 if (irq > XICS_IPI) 1426 xive_pre_save_set_queued(xive, irq); 1427 } while(irq); 1428 } 1429 1430 static void xive_pre_save_scan(struct kvmppc_xive *xive) 1431 { 1432 struct kvm_vcpu *vcpu = NULL; 1433 int i, j; 1434 1435 /* 1436 * See comment in xive_get_source() about how this 1437 * work. Collect a stable state for all interrupts 1438 */ 1439 for (i = 0; i <= xive->max_sbid; i++) { 1440 struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 1441 if (!sb) 1442 continue; 1443 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) 1444 xive_pre_save_mask_irq(xive, sb, j); 1445 } 1446 1447 /* Then scan the queues and update the "in_queue" flag */ 1448 kvm_for_each_vcpu(i, vcpu, xive->kvm) { 1449 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 1450 if (!xc) 1451 continue; 1452 for (j = 0; j < KVMPPC_XIVE_Q_COUNT; j++) { 1453 if (xc->queues[j].qpage) 1454 xive_pre_save_queue(xive, &xc->queues[j]); 1455 } 1456 } 1457 1458 /* Finally restore interrupt states */ 1459 for (i = 0; i <= xive->max_sbid; i++) { 1460 struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 1461 if (!sb) 1462 continue; 1463 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) 1464 xive_pre_save_unmask_irq(xive, sb, j); 1465 } 1466 } 1467 1468 static void xive_post_save_scan(struct kvmppc_xive *xive) 1469 { 1470 u32 i, j; 1471 1472 /* Clear all the in_queue flags */ 1473 for (i = 0; i <= xive->max_sbid; i++) { 1474 struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 1475 if (!sb) 1476 continue; 1477 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) 1478 sb->irq_state[j].in_queue = false; 1479 } 1480 1481 /* Next get_source() will do a new scan */ 1482 xive->saved_src_count = 0; 1483 } 1484 1485 /* 1486 * This returns the source configuration and state to user space. 1487 */ 1488 static int xive_get_source(struct kvmppc_xive *xive, long irq, u64 addr) 1489 { 1490 struct kvmppc_xive_src_block *sb; 1491 struct kvmppc_xive_irq_state *state; 1492 u64 __user *ubufp = (u64 __user *) addr; 1493 u64 val, prio; 1494 u16 idx; 1495 1496 sb = kvmppc_xive_find_source(xive, irq, &idx); 1497 if (!sb) 1498 return -ENOENT; 1499 1500 state = &sb->irq_state[idx]; 1501 1502 if (!state->valid) 1503 return -ENOENT; 1504 1505 pr_devel("get_source(%ld)...\n", irq); 1506 1507 /* 1508 * So to properly save the state into something that looks like a 1509 * XICS migration stream we cannot treat interrupts individually. 1510 * 1511 * We need, instead, mask them all (& save their previous PQ state) 1512 * to get a stable state in the HW, then sync them to ensure that 1513 * any interrupt that had already fired hits its queue, and finally 1514 * scan all the queues to collect which interrupts are still present 1515 * in the queues, so we can set the "pending" flag on them and 1516 * they can be resent on restore. 1517 * 1518 * So we do it all when the "first" interrupt gets saved, all the 1519 * state is collected at that point, the rest of xive_get_source() 1520 * will merely collect and convert that state to the expected 1521 * userspace bit mask. 1522 */ 1523 if (xive->saved_src_count == 0) 1524 xive_pre_save_scan(xive); 1525 xive->saved_src_count++; 1526 1527 /* Convert saved state into something compatible with xics */ 1528 val = state->act_server; 1529 prio = state->saved_scan_prio; 1530 1531 if (prio == MASKED) { 1532 val |= KVM_XICS_MASKED; 1533 prio = state->saved_priority; 1534 } 1535 val |= prio << KVM_XICS_PRIORITY_SHIFT; 1536 if (state->lsi) { 1537 val |= KVM_XICS_LEVEL_SENSITIVE; 1538 if (state->saved_p) 1539 val |= KVM_XICS_PENDING; 1540 } else { 1541 if (state->saved_p) 1542 val |= KVM_XICS_PRESENTED; 1543 1544 if (state->saved_q) 1545 val |= KVM_XICS_QUEUED; 1546 1547 /* 1548 * We mark it pending (which will attempt a re-delivery) 1549 * if we are in a queue *or* we were masked and had 1550 * Q set which is equivalent to the XICS "masked pending" 1551 * state 1552 */ 1553 if (state->in_queue || (prio == MASKED && state->saved_q)) 1554 val |= KVM_XICS_PENDING; 1555 } 1556 1557 /* 1558 * If that was the last interrupt saved, reset the 1559 * in_queue flags 1560 */ 1561 if (xive->saved_src_count == xive->src_count) 1562 xive_post_save_scan(xive); 1563 1564 /* Copy the result to userspace */ 1565 if (put_user(val, ubufp)) 1566 return -EFAULT; 1567 1568 return 0; 1569 } 1570 1571 struct kvmppc_xive_src_block *kvmppc_xive_create_src_block( 1572 struct kvmppc_xive *xive, int irq) 1573 { 1574 struct kvmppc_xive_src_block *sb; 1575 int i, bid; 1576 1577 bid = irq >> KVMPPC_XICS_ICS_SHIFT; 1578 1579 mutex_lock(&xive->lock); 1580 1581 /* block already exists - somebody else got here first */ 1582 if (xive->src_blocks[bid]) 1583 goto out; 1584 1585 /* Create the ICS */ 1586 sb = kzalloc(sizeof(*sb), GFP_KERNEL); 1587 if (!sb) 1588 goto out; 1589 1590 sb->id = bid; 1591 1592 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 1593 sb->irq_state[i].number = (bid << KVMPPC_XICS_ICS_SHIFT) | i; 1594 sb->irq_state[i].eisn = 0; 1595 sb->irq_state[i].guest_priority = MASKED; 1596 sb->irq_state[i].saved_priority = MASKED; 1597 sb->irq_state[i].act_priority = MASKED; 1598 } 1599 smp_wmb(); 1600 xive->src_blocks[bid] = sb; 1601 1602 if (bid > xive->max_sbid) 1603 xive->max_sbid = bid; 1604 1605 out: 1606 mutex_unlock(&xive->lock); 1607 return xive->src_blocks[bid]; 1608 } 1609 1610 static bool xive_check_delayed_irq(struct kvmppc_xive *xive, u32 irq) 1611 { 1612 struct kvm *kvm = xive->kvm; 1613 struct kvm_vcpu *vcpu = NULL; 1614 int i; 1615 1616 kvm_for_each_vcpu(i, vcpu, kvm) { 1617 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 1618 1619 if (!xc) 1620 continue; 1621 1622 if (xc->delayed_irq == irq) { 1623 xc->delayed_irq = 0; 1624 xive->delayed_irqs--; 1625 return true; 1626 } 1627 } 1628 return false; 1629 } 1630 1631 static int xive_set_source(struct kvmppc_xive *xive, long irq, u64 addr) 1632 { 1633 struct kvmppc_xive_src_block *sb; 1634 struct kvmppc_xive_irq_state *state; 1635 u64 __user *ubufp = (u64 __user *) addr; 1636 u16 idx; 1637 u64 val; 1638 u8 act_prio, guest_prio; 1639 u32 server; 1640 int rc = 0; 1641 1642 if (irq < KVMPPC_XICS_FIRST_IRQ || irq >= KVMPPC_XICS_NR_IRQS) 1643 return -ENOENT; 1644 1645 pr_devel("set_source(irq=0x%lx)\n", irq); 1646 1647 /* Find the source */ 1648 sb = kvmppc_xive_find_source(xive, irq, &idx); 1649 if (!sb) { 1650 pr_devel("No source, creating source block...\n"); 1651 sb = kvmppc_xive_create_src_block(xive, irq); 1652 if (!sb) { 1653 pr_devel("Failed to create block...\n"); 1654 return -ENOMEM; 1655 } 1656 } 1657 state = &sb->irq_state[idx]; 1658 1659 /* Read user passed data */ 1660 if (get_user(val, ubufp)) { 1661 pr_devel("fault getting user info !\n"); 1662 return -EFAULT; 1663 } 1664 1665 server = val & KVM_XICS_DESTINATION_MASK; 1666 guest_prio = val >> KVM_XICS_PRIORITY_SHIFT; 1667 1668 pr_devel(" val=0x016%llx (server=0x%x, guest_prio=%d)\n", 1669 val, server, guest_prio); 1670 1671 /* 1672 * If the source doesn't already have an IPI, allocate 1673 * one and get the corresponding data 1674 */ 1675 if (!state->ipi_number) { 1676 state->ipi_number = xive_native_alloc_irq(); 1677 if (state->ipi_number == 0) { 1678 pr_devel("Failed to allocate IPI !\n"); 1679 return -ENOMEM; 1680 } 1681 xive_native_populate_irq_data(state->ipi_number, &state->ipi_data); 1682 pr_devel(" src_ipi=0x%x\n", state->ipi_number); 1683 } 1684 1685 /* 1686 * We use lock_and_mask() to set us in the right masked 1687 * state. We will override that state from the saved state 1688 * further down, but this will handle the cases of interrupts 1689 * that need FW masking. We set the initial guest_priority to 1690 * 0 before calling it to ensure it actually performs the masking. 1691 */ 1692 state->guest_priority = 0; 1693 xive_lock_and_mask(xive, sb, state); 1694 1695 /* 1696 * Now, we select a target if we have one. If we don't we 1697 * leave the interrupt untargetted. It means that an interrupt 1698 * can become "untargetted" accross migration if it was masked 1699 * by set_xive() but there is little we can do about it. 1700 */ 1701 1702 /* First convert prio and mark interrupt as untargetted */ 1703 act_prio = xive_prio_from_guest(guest_prio); 1704 state->act_priority = MASKED; 1705 1706 /* 1707 * We need to drop the lock due to the mutex below. Hopefully 1708 * nothing is touching that interrupt yet since it hasn't been 1709 * advertized to a running guest yet 1710 */ 1711 arch_spin_unlock(&sb->lock); 1712 1713 /* If we have a priority target the interrupt */ 1714 if (act_prio != MASKED) { 1715 /* First, check provisioning of queues */ 1716 mutex_lock(&xive->lock); 1717 rc = xive_check_provisioning(xive->kvm, act_prio); 1718 mutex_unlock(&xive->lock); 1719 1720 /* Target interrupt */ 1721 if (rc == 0) 1722 rc = xive_target_interrupt(xive->kvm, state, 1723 server, act_prio); 1724 /* 1725 * If provisioning or targetting failed, leave it 1726 * alone and masked. It will remain disabled until 1727 * the guest re-targets it. 1728 */ 1729 } 1730 1731 /* 1732 * Find out if this was a delayed irq stashed in an ICP, 1733 * in which case, treat it as pending 1734 */ 1735 if (xive->delayed_irqs && xive_check_delayed_irq(xive, irq)) { 1736 val |= KVM_XICS_PENDING; 1737 pr_devel(" Found delayed ! forcing PENDING !\n"); 1738 } 1739 1740 /* Cleanup the SW state */ 1741 state->old_p = false; 1742 state->old_q = false; 1743 state->lsi = false; 1744 state->asserted = false; 1745 1746 /* Restore LSI state */ 1747 if (val & KVM_XICS_LEVEL_SENSITIVE) { 1748 state->lsi = true; 1749 if (val & KVM_XICS_PENDING) 1750 state->asserted = true; 1751 pr_devel(" LSI ! Asserted=%d\n", state->asserted); 1752 } 1753 1754 /* 1755 * Restore P and Q. If the interrupt was pending, we 1756 * force Q and !P, which will trigger a resend. 1757 * 1758 * That means that a guest that had both an interrupt 1759 * pending (queued) and Q set will restore with only 1760 * one instance of that interrupt instead of 2, but that 1761 * is perfectly fine as coalescing interrupts that haven't 1762 * been presented yet is always allowed. 1763 */ 1764 if (val & KVM_XICS_PRESENTED && !(val & KVM_XICS_PENDING)) 1765 state->old_p = true; 1766 if (val & KVM_XICS_QUEUED || val & KVM_XICS_PENDING) 1767 state->old_q = true; 1768 1769 pr_devel(" P=%d, Q=%d\n", state->old_p, state->old_q); 1770 1771 /* 1772 * If the interrupt was unmasked, update guest priority and 1773 * perform the appropriate state transition and do a 1774 * re-trigger if necessary. 1775 */ 1776 if (val & KVM_XICS_MASKED) { 1777 pr_devel(" masked, saving prio\n"); 1778 state->guest_priority = MASKED; 1779 state->saved_priority = guest_prio; 1780 } else { 1781 pr_devel(" unmasked, restoring to prio %d\n", guest_prio); 1782 xive_finish_unmask(xive, sb, state, guest_prio); 1783 state->saved_priority = guest_prio; 1784 } 1785 1786 /* Increment the number of valid sources and mark this one valid */ 1787 if (!state->valid) 1788 xive->src_count++; 1789 state->valid = true; 1790 1791 return 0; 1792 } 1793 1794 int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level, 1795 bool line_status) 1796 { 1797 struct kvmppc_xive *xive = kvm->arch.xive; 1798 struct kvmppc_xive_src_block *sb; 1799 struct kvmppc_xive_irq_state *state; 1800 u16 idx; 1801 1802 if (!xive) 1803 return -ENODEV; 1804 1805 sb = kvmppc_xive_find_source(xive, irq, &idx); 1806 if (!sb) 1807 return -EINVAL; 1808 1809 /* Perform locklessly .... (we need to do some RCUisms here...) */ 1810 state = &sb->irq_state[idx]; 1811 if (!state->valid) 1812 return -EINVAL; 1813 1814 /* We don't allow a trigger on a passed-through interrupt */ 1815 if (state->pt_number) 1816 return -EINVAL; 1817 1818 if ((level == 1 && state->lsi) || level == KVM_INTERRUPT_SET_LEVEL) 1819 state->asserted = 1; 1820 else if (level == 0 || level == KVM_INTERRUPT_UNSET) { 1821 state->asserted = 0; 1822 return 0; 1823 } 1824 1825 /* Trigger the IPI */ 1826 xive_irq_trigger(&state->ipi_data); 1827 1828 return 0; 1829 } 1830 1831 int kvmppc_xive_set_nr_servers(struct kvmppc_xive *xive, u64 addr) 1832 { 1833 u32 __user *ubufp = (u32 __user *) addr; 1834 u32 nr_servers; 1835 int rc = 0; 1836 1837 if (get_user(nr_servers, ubufp)) 1838 return -EFAULT; 1839 1840 pr_devel("%s nr_servers=%u\n", __func__, nr_servers); 1841 1842 if (!nr_servers || nr_servers > KVM_MAX_VCPU_ID) 1843 return -EINVAL; 1844 1845 mutex_lock(&xive->lock); 1846 if (xive->vp_base != XIVE_INVALID_VP) 1847 /* The VP block is allocated once and freed when the device 1848 * is released. Better not allow to change its size since its 1849 * used by connect_vcpu to validate vCPU ids are valid (eg, 1850 * setting it back to a higher value could allow connect_vcpu 1851 * to come up with a VP id that goes beyond the VP block, which 1852 * is likely to cause a crash in OPAL). 1853 */ 1854 rc = -EBUSY; 1855 else if (nr_servers > KVM_MAX_VCPUS) 1856 /* We don't need more servers. Higher vCPU ids get packed 1857 * down below KVM_MAX_VCPUS by kvmppc_pack_vcpu_id(). 1858 */ 1859 xive->nr_servers = KVM_MAX_VCPUS; 1860 else 1861 xive->nr_servers = nr_servers; 1862 1863 mutex_unlock(&xive->lock); 1864 1865 return rc; 1866 } 1867 1868 static int xive_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 1869 { 1870 struct kvmppc_xive *xive = dev->private; 1871 1872 /* We honor the existing XICS ioctl */ 1873 switch (attr->group) { 1874 case KVM_DEV_XICS_GRP_SOURCES: 1875 return xive_set_source(xive, attr->attr, attr->addr); 1876 case KVM_DEV_XICS_GRP_CTRL: 1877 switch (attr->attr) { 1878 case KVM_DEV_XICS_NR_SERVERS: 1879 return kvmppc_xive_set_nr_servers(xive, attr->addr); 1880 } 1881 } 1882 return -ENXIO; 1883 } 1884 1885 static int xive_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 1886 { 1887 struct kvmppc_xive *xive = dev->private; 1888 1889 /* We honor the existing XICS ioctl */ 1890 switch (attr->group) { 1891 case KVM_DEV_XICS_GRP_SOURCES: 1892 return xive_get_source(xive, attr->attr, attr->addr); 1893 } 1894 return -ENXIO; 1895 } 1896 1897 static int xive_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 1898 { 1899 /* We honor the same limits as XICS, at least for now */ 1900 switch (attr->group) { 1901 case KVM_DEV_XICS_GRP_SOURCES: 1902 if (attr->attr >= KVMPPC_XICS_FIRST_IRQ && 1903 attr->attr < KVMPPC_XICS_NR_IRQS) 1904 return 0; 1905 break; 1906 case KVM_DEV_XICS_GRP_CTRL: 1907 switch (attr->attr) { 1908 case KVM_DEV_XICS_NR_SERVERS: 1909 return 0; 1910 } 1911 } 1912 return -ENXIO; 1913 } 1914 1915 static void kvmppc_xive_cleanup_irq(u32 hw_num, struct xive_irq_data *xd) 1916 { 1917 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01); 1918 xive_native_configure_irq(hw_num, 0, MASKED, 0); 1919 } 1920 1921 void kvmppc_xive_free_sources(struct kvmppc_xive_src_block *sb) 1922 { 1923 int i; 1924 1925 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 1926 struct kvmppc_xive_irq_state *state = &sb->irq_state[i]; 1927 1928 if (!state->valid) 1929 continue; 1930 1931 kvmppc_xive_cleanup_irq(state->ipi_number, &state->ipi_data); 1932 xive_cleanup_irq_data(&state->ipi_data); 1933 xive_native_free_irq(state->ipi_number); 1934 1935 /* Pass-through, cleanup too but keep IRQ hw data */ 1936 if (state->pt_number) 1937 kvmppc_xive_cleanup_irq(state->pt_number, state->pt_data); 1938 1939 state->valid = false; 1940 } 1941 } 1942 1943 /* 1944 * Called when device fd is closed. kvm->lock is held. 1945 */ 1946 static void kvmppc_xive_release(struct kvm_device *dev) 1947 { 1948 struct kvmppc_xive *xive = dev->private; 1949 struct kvm *kvm = xive->kvm; 1950 struct kvm_vcpu *vcpu; 1951 int i; 1952 1953 pr_devel("Releasing xive device\n"); 1954 1955 /* 1956 * Since this is the device release function, we know that 1957 * userspace does not have any open fd referring to the 1958 * device. Therefore there can not be any of the device 1959 * attribute set/get functions being executed concurrently, 1960 * and similarly, the connect_vcpu and set/clr_mapped 1961 * functions also cannot be being executed. 1962 */ 1963 1964 debugfs_remove(xive->dentry); 1965 1966 /* 1967 * We should clean up the vCPU interrupt presenters first. 1968 */ 1969 kvm_for_each_vcpu(i, vcpu, kvm) { 1970 /* 1971 * Take vcpu->mutex to ensure that no one_reg get/set ioctl 1972 * (i.e. kvmppc_xive_[gs]et_icp) can be done concurrently. 1973 * Holding the vcpu->mutex also means that the vcpu cannot 1974 * be executing the KVM_RUN ioctl, and therefore it cannot 1975 * be executing the XIVE push or pull code or accessing 1976 * the XIVE MMIO regions. 1977 */ 1978 mutex_lock(&vcpu->mutex); 1979 kvmppc_xive_cleanup_vcpu(vcpu); 1980 mutex_unlock(&vcpu->mutex); 1981 } 1982 1983 /* 1984 * Now that we have cleared vcpu->arch.xive_vcpu, vcpu->arch.irq_type 1985 * and vcpu->arch.xive_esc_[vr]addr on each vcpu, we are safe 1986 * against xive code getting called during vcpu execution or 1987 * set/get one_reg operations. 1988 */ 1989 kvm->arch.xive = NULL; 1990 1991 /* Mask and free interrupts */ 1992 for (i = 0; i <= xive->max_sbid; i++) { 1993 if (xive->src_blocks[i]) 1994 kvmppc_xive_free_sources(xive->src_blocks[i]); 1995 kfree(xive->src_blocks[i]); 1996 xive->src_blocks[i] = NULL; 1997 } 1998 1999 if (xive->vp_base != XIVE_INVALID_VP) 2000 xive_native_free_vp_block(xive->vp_base); 2001 2002 /* 2003 * A reference of the kvmppc_xive pointer is now kept under 2004 * the xive_devices struct of the machine for reuse. It is 2005 * freed when the VM is destroyed for now until we fix all the 2006 * execution paths. 2007 */ 2008 2009 kfree(dev); 2010 } 2011 2012 /* 2013 * When the guest chooses the interrupt mode (XICS legacy or XIVE 2014 * native), the VM will switch of KVM device. The previous device will 2015 * be "released" before the new one is created. 2016 * 2017 * Until we are sure all execution paths are well protected, provide a 2018 * fail safe (transitional) method for device destruction, in which 2019 * the XIVE device pointer is recycled and not directly freed. 2020 */ 2021 struct kvmppc_xive *kvmppc_xive_get_device(struct kvm *kvm, u32 type) 2022 { 2023 struct kvmppc_xive **kvm_xive_device = type == KVM_DEV_TYPE_XIVE ? 2024 &kvm->arch.xive_devices.native : 2025 &kvm->arch.xive_devices.xics_on_xive; 2026 struct kvmppc_xive *xive = *kvm_xive_device; 2027 2028 if (!xive) { 2029 xive = kzalloc(sizeof(*xive), GFP_KERNEL); 2030 *kvm_xive_device = xive; 2031 } else { 2032 memset(xive, 0, sizeof(*xive)); 2033 } 2034 2035 return xive; 2036 } 2037 2038 /* 2039 * Create a XICS device with XIVE backend. kvm->lock is held. 2040 */ 2041 static int kvmppc_xive_create(struct kvm_device *dev, u32 type) 2042 { 2043 struct kvmppc_xive *xive; 2044 struct kvm *kvm = dev->kvm; 2045 2046 pr_devel("Creating xive for partition\n"); 2047 2048 /* Already there ? */ 2049 if (kvm->arch.xive) 2050 return -EEXIST; 2051 2052 xive = kvmppc_xive_get_device(kvm, type); 2053 if (!xive) 2054 return -ENOMEM; 2055 2056 dev->private = xive; 2057 xive->dev = dev; 2058 xive->kvm = kvm; 2059 mutex_init(&xive->lock); 2060 2061 /* We use the default queue size set by the host */ 2062 xive->q_order = xive_native_default_eq_shift(); 2063 if (xive->q_order < PAGE_SHIFT) 2064 xive->q_page_order = 0; 2065 else 2066 xive->q_page_order = xive->q_order - PAGE_SHIFT; 2067 2068 /* VP allocation is delayed to the first call to connect_vcpu */ 2069 xive->vp_base = XIVE_INVALID_VP; 2070 /* KVM_MAX_VCPUS limits the number of VMs to roughly 64 per sockets 2071 * on a POWER9 system. 2072 */ 2073 xive->nr_servers = KVM_MAX_VCPUS; 2074 2075 xive->single_escalation = xive_native_has_single_escalation(); 2076 2077 kvm->arch.xive = xive; 2078 return 0; 2079 } 2080 2081 int kvmppc_xive_debug_show_queues(struct seq_file *m, struct kvm_vcpu *vcpu) 2082 { 2083 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 2084 unsigned int i; 2085 2086 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) { 2087 struct xive_q *q = &xc->queues[i]; 2088 u32 i0, i1, idx; 2089 2090 if (!q->qpage && !xc->esc_virq[i]) 2091 continue; 2092 2093 if (q->qpage) { 2094 seq_printf(m, " q[%d]: ", i); 2095 idx = q->idx; 2096 i0 = be32_to_cpup(q->qpage + idx); 2097 idx = (idx + 1) & q->msk; 2098 i1 = be32_to_cpup(q->qpage + idx); 2099 seq_printf(m, "T=%d %08x %08x...\n", q->toggle, 2100 i0, i1); 2101 } 2102 if (xc->esc_virq[i]) { 2103 struct irq_data *d = irq_get_irq_data(xc->esc_virq[i]); 2104 struct xive_irq_data *xd = 2105 irq_data_get_irq_handler_data(d); 2106 u64 pq = xive_vm_esb_load(xd, XIVE_ESB_GET); 2107 2108 seq_printf(m, " ESC %d %c%c EOI @%llx", 2109 xc->esc_virq[i], 2110 (pq & XIVE_ESB_VAL_P) ? 'P' : '-', 2111 (pq & XIVE_ESB_VAL_Q) ? 'Q' : '-', 2112 xd->eoi_page); 2113 seq_puts(m, "\n"); 2114 } 2115 } 2116 return 0; 2117 } 2118 2119 void kvmppc_xive_debug_show_sources(struct seq_file *m, 2120 struct kvmppc_xive_src_block *sb) 2121 { 2122 int i; 2123 2124 seq_puts(m, " LISN HW/CHIP TYPE PQ EISN CPU/PRIO\n"); 2125 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 2126 struct kvmppc_xive_irq_state *state = &sb->irq_state[i]; 2127 struct xive_irq_data *xd; 2128 u64 pq; 2129 u32 hw_num; 2130 2131 if (!state->valid) 2132 continue; 2133 2134 kvmppc_xive_select_irq(state, &hw_num, &xd); 2135 2136 pq = xive_vm_esb_load(xd, XIVE_ESB_GET); 2137 2138 seq_printf(m, "%08x %08x/%02x", state->number, hw_num, 2139 xd->src_chip); 2140 if (state->lsi) 2141 seq_printf(m, " %cLSI", state->asserted ? '^' : ' '); 2142 else 2143 seq_puts(m, " MSI"); 2144 2145 seq_printf(m, " %s %c%c %08x % 4d/%d", 2146 state->ipi_number == hw_num ? "IPI" : " PT", 2147 pq & XIVE_ESB_VAL_P ? 'P' : '-', 2148 pq & XIVE_ESB_VAL_Q ? 'Q' : '-', 2149 state->eisn, state->act_server, 2150 state->act_priority); 2151 2152 seq_puts(m, "\n"); 2153 } 2154 } 2155 2156 static int xive_debug_show(struct seq_file *m, void *private) 2157 { 2158 struct kvmppc_xive *xive = m->private; 2159 struct kvm *kvm = xive->kvm; 2160 struct kvm_vcpu *vcpu; 2161 u64 t_rm_h_xirr = 0; 2162 u64 t_rm_h_ipoll = 0; 2163 u64 t_rm_h_cppr = 0; 2164 u64 t_rm_h_eoi = 0; 2165 u64 t_rm_h_ipi = 0; 2166 u64 t_vm_h_xirr = 0; 2167 u64 t_vm_h_ipoll = 0; 2168 u64 t_vm_h_cppr = 0; 2169 u64 t_vm_h_eoi = 0; 2170 u64 t_vm_h_ipi = 0; 2171 unsigned int i; 2172 2173 if (!kvm) 2174 return 0; 2175 2176 seq_puts(m, "=========\nVCPU state\n=========\n"); 2177 2178 kvm_for_each_vcpu(i, vcpu, kvm) { 2179 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 2180 2181 if (!xc) 2182 continue; 2183 2184 seq_printf(m, "VCPU %d: VP:%#x/%02x\n" 2185 " CPPR:%#x HWCPPR:%#x MFRR:%#x PEND:%#x h_xirr: R=%lld V=%lld\n", 2186 xc->server_num, xc->vp_id, xc->vp_chip_id, 2187 xc->cppr, xc->hw_cppr, 2188 xc->mfrr, xc->pending, 2189 xc->stat_rm_h_xirr, xc->stat_vm_h_xirr); 2190 2191 kvmppc_xive_debug_show_queues(m, vcpu); 2192 2193 t_rm_h_xirr += xc->stat_rm_h_xirr; 2194 t_rm_h_ipoll += xc->stat_rm_h_ipoll; 2195 t_rm_h_cppr += xc->stat_rm_h_cppr; 2196 t_rm_h_eoi += xc->stat_rm_h_eoi; 2197 t_rm_h_ipi += xc->stat_rm_h_ipi; 2198 t_vm_h_xirr += xc->stat_vm_h_xirr; 2199 t_vm_h_ipoll += xc->stat_vm_h_ipoll; 2200 t_vm_h_cppr += xc->stat_vm_h_cppr; 2201 t_vm_h_eoi += xc->stat_vm_h_eoi; 2202 t_vm_h_ipi += xc->stat_vm_h_ipi; 2203 } 2204 2205 seq_puts(m, "Hcalls totals\n"); 2206 seq_printf(m, " H_XIRR R=%10lld V=%10lld\n", t_rm_h_xirr, t_vm_h_xirr); 2207 seq_printf(m, " H_IPOLL R=%10lld V=%10lld\n", t_rm_h_ipoll, t_vm_h_ipoll); 2208 seq_printf(m, " H_CPPR R=%10lld V=%10lld\n", t_rm_h_cppr, t_vm_h_cppr); 2209 seq_printf(m, " H_EOI R=%10lld V=%10lld\n", t_rm_h_eoi, t_vm_h_eoi); 2210 seq_printf(m, " H_IPI R=%10lld V=%10lld\n", t_rm_h_ipi, t_vm_h_ipi); 2211 2212 seq_puts(m, "=========\nSources\n=========\n"); 2213 2214 for (i = 0; i <= xive->max_sbid; i++) { 2215 struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 2216 2217 if (sb) { 2218 arch_spin_lock(&sb->lock); 2219 kvmppc_xive_debug_show_sources(m, sb); 2220 arch_spin_unlock(&sb->lock); 2221 } 2222 } 2223 2224 return 0; 2225 } 2226 2227 DEFINE_SHOW_ATTRIBUTE(xive_debug); 2228 2229 static void xive_debugfs_init(struct kvmppc_xive *xive) 2230 { 2231 char *name; 2232 2233 name = kasprintf(GFP_KERNEL, "kvm-xive-%p", xive); 2234 if (!name) { 2235 pr_err("%s: no memory for name\n", __func__); 2236 return; 2237 } 2238 2239 xive->dentry = debugfs_create_file(name, S_IRUGO, powerpc_debugfs_root, 2240 xive, &xive_debug_fops); 2241 2242 pr_debug("%s: created %s\n", __func__, name); 2243 kfree(name); 2244 } 2245 2246 static void kvmppc_xive_init(struct kvm_device *dev) 2247 { 2248 struct kvmppc_xive *xive = (struct kvmppc_xive *)dev->private; 2249 2250 /* Register some debug interfaces */ 2251 xive_debugfs_init(xive); 2252 } 2253 2254 struct kvm_device_ops kvm_xive_ops = { 2255 .name = "kvm-xive", 2256 .create = kvmppc_xive_create, 2257 .init = kvmppc_xive_init, 2258 .release = kvmppc_xive_release, 2259 .set_attr = xive_set_attr, 2260 .get_attr = xive_get_attr, 2261 .has_attr = xive_has_attr, 2262 }; 2263 2264 void kvmppc_xive_init_module(void) 2265 { 2266 __xive_vm_h_xirr = xive_vm_h_xirr; 2267 __xive_vm_h_ipoll = xive_vm_h_ipoll; 2268 __xive_vm_h_ipi = xive_vm_h_ipi; 2269 __xive_vm_h_cppr = xive_vm_h_cppr; 2270 __xive_vm_h_eoi = xive_vm_h_eoi; 2271 } 2272 2273 void kvmppc_xive_exit_module(void) 2274 { 2275 __xive_vm_h_xirr = NULL; 2276 __xive_vm_h_ipoll = NULL; 2277 __xive_vm_h_ipi = NULL; 2278 __xive_vm_h_cppr = NULL; 2279 __xive_vm_h_eoi = NULL; 2280 } 2281