1 /* 2 * Copyright 2017 Benjamin Herrenschmidt, IBM Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License, version 2, as 6 * published by the Free Software Foundation. 7 */ 8 9 #define pr_fmt(fmt) "xive-kvm: " fmt 10 11 #include <linux/kernel.h> 12 #include <linux/kvm_host.h> 13 #include <linux/err.h> 14 #include <linux/gfp.h> 15 #include <linux/spinlock.h> 16 #include <linux/delay.h> 17 #include <linux/percpu.h> 18 #include <linux/cpumask.h> 19 #include <linux/uaccess.h> 20 #include <asm/kvm_book3s.h> 21 #include <asm/kvm_ppc.h> 22 #include <asm/hvcall.h> 23 #include <asm/xics.h> 24 #include <asm/xive.h> 25 #include <asm/xive-regs.h> 26 #include <asm/debug.h> 27 #include <asm/debugfs.h> 28 #include <asm/time.h> 29 #include <asm/opal.h> 30 31 #include <linux/debugfs.h> 32 #include <linux/seq_file.h> 33 34 #include "book3s_xive.h" 35 36 37 /* 38 * Virtual mode variants of the hcalls for use on radix/radix 39 * with AIL. They require the VCPU's VP to be "pushed" 40 * 41 * We still instantiate them here because we use some of the 42 * generated utility functions as well in this file. 43 */ 44 #define XIVE_RUNTIME_CHECKS 45 #define X_PFX xive_vm_ 46 #define X_STATIC static 47 #define X_STAT_PFX stat_vm_ 48 #define __x_tima xive_tima 49 #define __x_eoi_page(xd) ((void __iomem *)((xd)->eoi_mmio)) 50 #define __x_trig_page(xd) ((void __iomem *)((xd)->trig_mmio)) 51 #define __x_writeb __raw_writeb 52 #define __x_readw __raw_readw 53 #define __x_readq __raw_readq 54 #define __x_writeq __raw_writeq 55 56 #include "book3s_xive_template.c" 57 58 /* 59 * We leave a gap of a couple of interrupts in the queue to 60 * account for the IPI and additional safety guard. 61 */ 62 #define XIVE_Q_GAP 2 63 64 /* 65 * Push a vcpu's context to the XIVE on guest entry. 66 * This assumes we are in virtual mode (MMU on) 67 */ 68 void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu) 69 { 70 void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt; 71 u64 pq; 72 73 if (!tima) 74 return; 75 eieio(); 76 __raw_writeq(vcpu->arch.xive_saved_state.w01, tima + TM_QW1_OS); 77 __raw_writel(vcpu->arch.xive_cam_word, tima + TM_QW1_OS + TM_WORD2); 78 vcpu->arch.xive_pushed = 1; 79 eieio(); 80 81 /* 82 * We clear the irq_pending flag. There is a small chance of a 83 * race vs. the escalation interrupt happening on another 84 * processor setting it again, but the only consequence is to 85 * cause a spurious wakeup on the next H_CEDE, which is not an 86 * issue. 87 */ 88 vcpu->arch.irq_pending = 0; 89 90 /* 91 * In single escalation mode, if the escalation interrupt is 92 * on, we mask it. 93 */ 94 if (vcpu->arch.xive_esc_on) { 95 pq = __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr + 96 XIVE_ESB_SET_PQ_01)); 97 mb(); 98 99 /* 100 * We have a possible subtle race here: The escalation 101 * interrupt might have fired and be on its way to the 102 * host queue while we mask it, and if we unmask it 103 * early enough (re-cede right away), there is a 104 * theorical possibility that it fires again, thus 105 * landing in the target queue more than once which is 106 * a big no-no. 107 * 108 * Fortunately, solving this is rather easy. If the 109 * above load setting PQ to 01 returns a previous 110 * value where P is set, then we know the escalation 111 * interrupt is somewhere on its way to the host. In 112 * that case we simply don't clear the xive_esc_on 113 * flag below. It will be eventually cleared by the 114 * handler for the escalation interrupt. 115 * 116 * Then, when doing a cede, we check that flag again 117 * before re-enabling the escalation interrupt, and if 118 * set, we abort the cede. 119 */ 120 if (!(pq & XIVE_ESB_VAL_P)) 121 /* Now P is 0, we can clear the flag */ 122 vcpu->arch.xive_esc_on = 0; 123 } 124 } 125 EXPORT_SYMBOL_GPL(kvmppc_xive_push_vcpu); 126 127 /* 128 * This is a simple trigger for a generic XIVE IRQ. This must 129 * only be called for interrupts that support a trigger page 130 */ 131 static bool xive_irq_trigger(struct xive_irq_data *xd) 132 { 133 /* This should be only for MSIs */ 134 if (WARN_ON(xd->flags & XIVE_IRQ_FLAG_LSI)) 135 return false; 136 137 /* Those interrupts should always have a trigger page */ 138 if (WARN_ON(!xd->trig_mmio)) 139 return false; 140 141 out_be64(xd->trig_mmio, 0); 142 143 return true; 144 } 145 146 static irqreturn_t xive_esc_irq(int irq, void *data) 147 { 148 struct kvm_vcpu *vcpu = data; 149 150 vcpu->arch.irq_pending = 1; 151 smp_mb(); 152 if (vcpu->arch.ceded) 153 kvmppc_fast_vcpu_kick(vcpu); 154 155 /* Since we have the no-EOI flag, the interrupt is effectively 156 * disabled now. Clearing xive_esc_on means we won't bother 157 * doing so on the next entry. 158 * 159 * This also allows the entry code to know that if a PQ combination 160 * of 10 is observed while xive_esc_on is true, it means the queue 161 * contains an unprocessed escalation interrupt. We don't make use of 162 * that knowledge today but might (see comment in book3s_hv_rmhandler.S) 163 */ 164 vcpu->arch.xive_esc_on = false; 165 166 return IRQ_HANDLED; 167 } 168 169 int kvmppc_xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio, 170 bool single_escalation) 171 { 172 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 173 struct xive_q *q = &xc->queues[prio]; 174 char *name = NULL; 175 int rc; 176 177 /* Already there ? */ 178 if (xc->esc_virq[prio]) 179 return 0; 180 181 /* Hook up the escalation interrupt */ 182 xc->esc_virq[prio] = irq_create_mapping(NULL, q->esc_irq); 183 if (!xc->esc_virq[prio]) { 184 pr_err("Failed to map escalation interrupt for queue %d of VCPU %d\n", 185 prio, xc->server_num); 186 return -EIO; 187 } 188 189 if (single_escalation) 190 name = kasprintf(GFP_KERNEL, "kvm-%d-%d", 191 vcpu->kvm->arch.lpid, xc->server_num); 192 else 193 name = kasprintf(GFP_KERNEL, "kvm-%d-%d-%d", 194 vcpu->kvm->arch.lpid, xc->server_num, prio); 195 if (!name) { 196 pr_err("Failed to allocate escalation irq name for queue %d of VCPU %d\n", 197 prio, xc->server_num); 198 rc = -ENOMEM; 199 goto error; 200 } 201 202 pr_devel("Escalation %s irq %d (prio %d)\n", name, xc->esc_virq[prio], prio); 203 204 rc = request_irq(xc->esc_virq[prio], xive_esc_irq, 205 IRQF_NO_THREAD, name, vcpu); 206 if (rc) { 207 pr_err("Failed to request escalation interrupt for queue %d of VCPU %d\n", 208 prio, xc->server_num); 209 goto error; 210 } 211 xc->esc_virq_names[prio] = name; 212 213 /* In single escalation mode, we grab the ESB MMIO of the 214 * interrupt and mask it. Also populate the VCPU v/raddr 215 * of the ESB page for use by asm entry/exit code. Finally 216 * set the XIVE_IRQ_NO_EOI flag which will prevent the 217 * core code from performing an EOI on the escalation 218 * interrupt, thus leaving it effectively masked after 219 * it fires once. 220 */ 221 if (single_escalation) { 222 struct irq_data *d = irq_get_irq_data(xc->esc_virq[prio]); 223 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); 224 225 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01); 226 vcpu->arch.xive_esc_raddr = xd->eoi_page; 227 vcpu->arch.xive_esc_vaddr = (__force u64)xd->eoi_mmio; 228 xd->flags |= XIVE_IRQ_NO_EOI; 229 } 230 231 return 0; 232 error: 233 irq_dispose_mapping(xc->esc_virq[prio]); 234 xc->esc_virq[prio] = 0; 235 kfree(name); 236 return rc; 237 } 238 239 static int xive_provision_queue(struct kvm_vcpu *vcpu, u8 prio) 240 { 241 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 242 struct kvmppc_xive *xive = xc->xive; 243 struct xive_q *q = &xc->queues[prio]; 244 void *qpage; 245 int rc; 246 247 if (WARN_ON(q->qpage)) 248 return 0; 249 250 /* Allocate the queue and retrieve infos on current node for now */ 251 qpage = (__be32 *)__get_free_pages(GFP_KERNEL, xive->q_page_order); 252 if (!qpage) { 253 pr_err("Failed to allocate queue %d for VCPU %d\n", 254 prio, xc->server_num); 255 return -ENOMEM; 256 } 257 memset(qpage, 0, 1 << xive->q_order); 258 259 /* 260 * Reconfigure the queue. This will set q->qpage only once the 261 * queue is fully configured. This is a requirement for prio 0 262 * as we will stop doing EOIs for every IPI as soon as we observe 263 * qpage being non-NULL, and instead will only EOI when we receive 264 * corresponding queue 0 entries 265 */ 266 rc = xive_native_configure_queue(xc->vp_id, q, prio, qpage, 267 xive->q_order, true); 268 if (rc) 269 pr_err("Failed to configure queue %d for VCPU %d\n", 270 prio, xc->server_num); 271 return rc; 272 } 273 274 /* Called with xive->lock held */ 275 static int xive_check_provisioning(struct kvm *kvm, u8 prio) 276 { 277 struct kvmppc_xive *xive = kvm->arch.xive; 278 struct kvm_vcpu *vcpu; 279 int i, rc; 280 281 lockdep_assert_held(&xive->lock); 282 283 /* Already provisioned ? */ 284 if (xive->qmap & (1 << prio)) 285 return 0; 286 287 pr_devel("Provisioning prio... %d\n", prio); 288 289 /* Provision each VCPU and enable escalations if needed */ 290 kvm_for_each_vcpu(i, vcpu, kvm) { 291 if (!vcpu->arch.xive_vcpu) 292 continue; 293 rc = xive_provision_queue(vcpu, prio); 294 if (rc == 0 && !xive->single_escalation) 295 kvmppc_xive_attach_escalation(vcpu, prio, 296 xive->single_escalation); 297 if (rc) 298 return rc; 299 } 300 301 /* Order previous stores and mark it as provisioned */ 302 mb(); 303 xive->qmap |= (1 << prio); 304 return 0; 305 } 306 307 static void xive_inc_q_pending(struct kvm *kvm, u32 server, u8 prio) 308 { 309 struct kvm_vcpu *vcpu; 310 struct kvmppc_xive_vcpu *xc; 311 struct xive_q *q; 312 313 /* Locate target server */ 314 vcpu = kvmppc_xive_find_server(kvm, server); 315 if (!vcpu) { 316 pr_warn("%s: Can't find server %d\n", __func__, server); 317 return; 318 } 319 xc = vcpu->arch.xive_vcpu; 320 if (WARN_ON(!xc)) 321 return; 322 323 q = &xc->queues[prio]; 324 atomic_inc(&q->pending_count); 325 } 326 327 static int xive_try_pick_queue(struct kvm_vcpu *vcpu, u8 prio) 328 { 329 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 330 struct xive_q *q; 331 u32 max; 332 333 if (WARN_ON(!xc)) 334 return -ENXIO; 335 if (!xc->valid) 336 return -ENXIO; 337 338 q = &xc->queues[prio]; 339 if (WARN_ON(!q->qpage)) 340 return -ENXIO; 341 342 /* Calculate max number of interrupts in that queue. */ 343 max = (q->msk + 1) - XIVE_Q_GAP; 344 return atomic_add_unless(&q->count, 1, max) ? 0 : -EBUSY; 345 } 346 347 int kvmppc_xive_select_target(struct kvm *kvm, u32 *server, u8 prio) 348 { 349 struct kvm_vcpu *vcpu; 350 int i, rc; 351 352 /* Locate target server */ 353 vcpu = kvmppc_xive_find_server(kvm, *server); 354 if (!vcpu) { 355 pr_devel("Can't find server %d\n", *server); 356 return -EINVAL; 357 } 358 359 pr_devel("Finding irq target on 0x%x/%d...\n", *server, prio); 360 361 /* Try pick it */ 362 rc = xive_try_pick_queue(vcpu, prio); 363 if (rc == 0) 364 return rc; 365 366 pr_devel(" .. failed, looking up candidate...\n"); 367 368 /* Failed, pick another VCPU */ 369 kvm_for_each_vcpu(i, vcpu, kvm) { 370 if (!vcpu->arch.xive_vcpu) 371 continue; 372 rc = xive_try_pick_queue(vcpu, prio); 373 if (rc == 0) { 374 *server = vcpu->arch.xive_vcpu->server_num; 375 pr_devel(" found on 0x%x/%d\n", *server, prio); 376 return rc; 377 } 378 } 379 pr_devel(" no available target !\n"); 380 381 /* No available target ! */ 382 return -EBUSY; 383 } 384 385 static u8 xive_lock_and_mask(struct kvmppc_xive *xive, 386 struct kvmppc_xive_src_block *sb, 387 struct kvmppc_xive_irq_state *state) 388 { 389 struct xive_irq_data *xd; 390 u32 hw_num; 391 u8 old_prio; 392 u64 val; 393 394 /* 395 * Take the lock, set masked, try again if racing 396 * with H_EOI 397 */ 398 for (;;) { 399 arch_spin_lock(&sb->lock); 400 old_prio = state->guest_priority; 401 state->guest_priority = MASKED; 402 mb(); 403 if (!state->in_eoi) 404 break; 405 state->guest_priority = old_prio; 406 arch_spin_unlock(&sb->lock); 407 } 408 409 /* No change ? Bail */ 410 if (old_prio == MASKED) 411 return old_prio; 412 413 /* Get the right irq */ 414 kvmppc_xive_select_irq(state, &hw_num, &xd); 415 416 /* 417 * If the interrupt is marked as needing masking via 418 * firmware, we do it here. Firmware masking however 419 * is "lossy", it won't return the old p and q bits 420 * and won't set the interrupt to a state where it will 421 * record queued ones. If this is an issue we should do 422 * lazy masking instead. 423 * 424 * For now, we work around this in unmask by forcing 425 * an interrupt whenever we unmask a non-LSI via FW 426 * (if ever). 427 */ 428 if (xd->flags & OPAL_XIVE_IRQ_MASK_VIA_FW) { 429 xive_native_configure_irq(hw_num, 430 kvmppc_xive_vp(xive, state->act_server), 431 MASKED, state->number); 432 /* set old_p so we can track if an H_EOI was done */ 433 state->old_p = true; 434 state->old_q = false; 435 } else { 436 /* Set PQ to 10, return old P and old Q and remember them */ 437 val = xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_10); 438 state->old_p = !!(val & 2); 439 state->old_q = !!(val & 1); 440 441 /* 442 * Synchronize hardware to sensure the queues are updated 443 * when masking 444 */ 445 xive_native_sync_source(hw_num); 446 } 447 448 return old_prio; 449 } 450 451 static void xive_lock_for_unmask(struct kvmppc_xive_src_block *sb, 452 struct kvmppc_xive_irq_state *state) 453 { 454 /* 455 * Take the lock try again if racing with H_EOI 456 */ 457 for (;;) { 458 arch_spin_lock(&sb->lock); 459 if (!state->in_eoi) 460 break; 461 arch_spin_unlock(&sb->lock); 462 } 463 } 464 465 static void xive_finish_unmask(struct kvmppc_xive *xive, 466 struct kvmppc_xive_src_block *sb, 467 struct kvmppc_xive_irq_state *state, 468 u8 prio) 469 { 470 struct xive_irq_data *xd; 471 u32 hw_num; 472 473 /* If we aren't changing a thing, move on */ 474 if (state->guest_priority != MASKED) 475 goto bail; 476 477 /* Get the right irq */ 478 kvmppc_xive_select_irq(state, &hw_num, &xd); 479 480 /* 481 * See command in xive_lock_and_mask() concerning masking 482 * via firmware. 483 */ 484 if (xd->flags & OPAL_XIVE_IRQ_MASK_VIA_FW) { 485 xive_native_configure_irq(hw_num, 486 kvmppc_xive_vp(xive, state->act_server), 487 state->act_priority, state->number); 488 /* If an EOI is needed, do it here */ 489 if (!state->old_p) 490 xive_vm_source_eoi(hw_num, xd); 491 /* If this is not an LSI, force a trigger */ 492 if (!(xd->flags & OPAL_XIVE_IRQ_LSI)) 493 xive_irq_trigger(xd); 494 goto bail; 495 } 496 497 /* Old Q set, set PQ to 11 */ 498 if (state->old_q) 499 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_11); 500 501 /* 502 * If not old P, then perform an "effective" EOI, 503 * on the source. This will handle the cases where 504 * FW EOI is needed. 505 */ 506 if (!state->old_p) 507 xive_vm_source_eoi(hw_num, xd); 508 509 /* Synchronize ordering and mark unmasked */ 510 mb(); 511 bail: 512 state->guest_priority = prio; 513 } 514 515 /* 516 * Target an interrupt to a given server/prio, this will fallback 517 * to another server if necessary and perform the HW targetting 518 * updates as needed 519 * 520 * NOTE: Must be called with the state lock held 521 */ 522 static int xive_target_interrupt(struct kvm *kvm, 523 struct kvmppc_xive_irq_state *state, 524 u32 server, u8 prio) 525 { 526 struct kvmppc_xive *xive = kvm->arch.xive; 527 u32 hw_num; 528 int rc; 529 530 /* 531 * This will return a tentative server and actual 532 * priority. The count for that new target will have 533 * already been incremented. 534 */ 535 rc = kvmppc_xive_select_target(kvm, &server, prio); 536 537 /* 538 * We failed to find a target ? Not much we can do 539 * at least until we support the GIQ. 540 */ 541 if (rc) 542 return rc; 543 544 /* 545 * Increment the old queue pending count if there 546 * was one so that the old queue count gets adjusted later 547 * when observed to be empty. 548 */ 549 if (state->act_priority != MASKED) 550 xive_inc_q_pending(kvm, 551 state->act_server, 552 state->act_priority); 553 /* 554 * Update state and HW 555 */ 556 state->act_priority = prio; 557 state->act_server = server; 558 559 /* Get the right irq */ 560 kvmppc_xive_select_irq(state, &hw_num, NULL); 561 562 return xive_native_configure_irq(hw_num, 563 kvmppc_xive_vp(xive, server), 564 prio, state->number); 565 } 566 567 /* 568 * Targetting rules: In order to avoid losing track of 569 * pending interrupts accross mask and unmask, which would 570 * allow queue overflows, we implement the following rules: 571 * 572 * - Unless it was never enabled (or we run out of capacity) 573 * an interrupt is always targetted at a valid server/queue 574 * pair even when "masked" by the guest. This pair tends to 575 * be the last one used but it can be changed under some 576 * circumstances. That allows us to separate targetting 577 * from masking, we only handle accounting during (re)targetting, 578 * this also allows us to let an interrupt drain into its target 579 * queue after masking, avoiding complex schemes to remove 580 * interrupts out of remote processor queues. 581 * 582 * - When masking, we set PQ to 10 and save the previous value 583 * of P and Q. 584 * 585 * - When unmasking, if saved Q was set, we set PQ to 11 586 * otherwise we leave PQ to the HW state which will be either 587 * 10 if nothing happened or 11 if the interrupt fired while 588 * masked. Effectively we are OR'ing the previous Q into the 589 * HW Q. 590 * 591 * Then if saved P is clear, we do an effective EOI (Q->P->Trigger) 592 * which will unmask the interrupt and shoot a new one if Q was 593 * set. 594 * 595 * Otherwise (saved P is set) we leave PQ unchanged (so 10 or 11, 596 * effectively meaning an H_EOI from the guest is still expected 597 * for that interrupt). 598 * 599 * - If H_EOI occurs while masked, we clear the saved P. 600 * 601 * - When changing target, we account on the new target and 602 * increment a separate "pending" counter on the old one. 603 * This pending counter will be used to decrement the old 604 * target's count when its queue has been observed empty. 605 */ 606 607 int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server, 608 u32 priority) 609 { 610 struct kvmppc_xive *xive = kvm->arch.xive; 611 struct kvmppc_xive_src_block *sb; 612 struct kvmppc_xive_irq_state *state; 613 u8 new_act_prio; 614 int rc = 0; 615 u16 idx; 616 617 if (!xive) 618 return -ENODEV; 619 620 pr_devel("set_xive ! irq 0x%x server 0x%x prio %d\n", 621 irq, server, priority); 622 623 /* First, check provisioning of queues */ 624 if (priority != MASKED) { 625 mutex_lock(&xive->lock); 626 rc = xive_check_provisioning(xive->kvm, 627 xive_prio_from_guest(priority)); 628 mutex_unlock(&xive->lock); 629 } 630 if (rc) { 631 pr_devel(" provisioning failure %d !\n", rc); 632 return rc; 633 } 634 635 sb = kvmppc_xive_find_source(xive, irq, &idx); 636 if (!sb) 637 return -EINVAL; 638 state = &sb->irq_state[idx]; 639 640 /* 641 * We first handle masking/unmasking since the locking 642 * might need to be retried due to EOIs, we'll handle 643 * targetting changes later. These functions will return 644 * with the SB lock held. 645 * 646 * xive_lock_and_mask() will also set state->guest_priority 647 * but won't otherwise change other fields of the state. 648 * 649 * xive_lock_for_unmask will not actually unmask, this will 650 * be done later by xive_finish_unmask() once the targetting 651 * has been done, so we don't try to unmask an interrupt 652 * that hasn't yet been targetted. 653 */ 654 if (priority == MASKED) 655 xive_lock_and_mask(xive, sb, state); 656 else 657 xive_lock_for_unmask(sb, state); 658 659 660 /* 661 * Then we handle targetting. 662 * 663 * First calculate a new "actual priority" 664 */ 665 new_act_prio = state->act_priority; 666 if (priority != MASKED) 667 new_act_prio = xive_prio_from_guest(priority); 668 669 pr_devel(" new_act_prio=%x act_server=%x act_prio=%x\n", 670 new_act_prio, state->act_server, state->act_priority); 671 672 /* 673 * Then check if we actually need to change anything, 674 * 675 * The condition for re-targetting the interrupt is that 676 * we have a valid new priority (new_act_prio is not 0xff) 677 * and either the server or the priority changed. 678 * 679 * Note: If act_priority was ff and the new priority is 680 * also ff, we don't do anything and leave the interrupt 681 * untargetted. An attempt of doing an int_on on an 682 * untargetted interrupt will fail. If that is a problem 683 * we could initialize interrupts with valid default 684 */ 685 686 if (new_act_prio != MASKED && 687 (state->act_server != server || 688 state->act_priority != new_act_prio)) 689 rc = xive_target_interrupt(kvm, state, server, new_act_prio); 690 691 /* 692 * Perform the final unmasking of the interrupt source 693 * if necessary 694 */ 695 if (priority != MASKED) 696 xive_finish_unmask(xive, sb, state, priority); 697 698 /* 699 * Finally Update saved_priority to match. Only int_on/off 700 * set this field to a different value. 701 */ 702 state->saved_priority = priority; 703 704 arch_spin_unlock(&sb->lock); 705 return rc; 706 } 707 708 int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server, 709 u32 *priority) 710 { 711 struct kvmppc_xive *xive = kvm->arch.xive; 712 struct kvmppc_xive_src_block *sb; 713 struct kvmppc_xive_irq_state *state; 714 u16 idx; 715 716 if (!xive) 717 return -ENODEV; 718 719 sb = kvmppc_xive_find_source(xive, irq, &idx); 720 if (!sb) 721 return -EINVAL; 722 state = &sb->irq_state[idx]; 723 arch_spin_lock(&sb->lock); 724 *server = state->act_server; 725 *priority = state->guest_priority; 726 arch_spin_unlock(&sb->lock); 727 728 return 0; 729 } 730 731 int kvmppc_xive_int_on(struct kvm *kvm, u32 irq) 732 { 733 struct kvmppc_xive *xive = kvm->arch.xive; 734 struct kvmppc_xive_src_block *sb; 735 struct kvmppc_xive_irq_state *state; 736 u16 idx; 737 738 if (!xive) 739 return -ENODEV; 740 741 sb = kvmppc_xive_find_source(xive, irq, &idx); 742 if (!sb) 743 return -EINVAL; 744 state = &sb->irq_state[idx]; 745 746 pr_devel("int_on(irq=0x%x)\n", irq); 747 748 /* 749 * Check if interrupt was not targetted 750 */ 751 if (state->act_priority == MASKED) { 752 pr_devel("int_on on untargetted interrupt\n"); 753 return -EINVAL; 754 } 755 756 /* If saved_priority is 0xff, do nothing */ 757 if (state->saved_priority == MASKED) 758 return 0; 759 760 /* 761 * Lock and unmask it. 762 */ 763 xive_lock_for_unmask(sb, state); 764 xive_finish_unmask(xive, sb, state, state->saved_priority); 765 arch_spin_unlock(&sb->lock); 766 767 return 0; 768 } 769 770 int kvmppc_xive_int_off(struct kvm *kvm, u32 irq) 771 { 772 struct kvmppc_xive *xive = kvm->arch.xive; 773 struct kvmppc_xive_src_block *sb; 774 struct kvmppc_xive_irq_state *state; 775 u16 idx; 776 777 if (!xive) 778 return -ENODEV; 779 780 sb = kvmppc_xive_find_source(xive, irq, &idx); 781 if (!sb) 782 return -EINVAL; 783 state = &sb->irq_state[idx]; 784 785 pr_devel("int_off(irq=0x%x)\n", irq); 786 787 /* 788 * Lock and mask 789 */ 790 state->saved_priority = xive_lock_and_mask(xive, sb, state); 791 arch_spin_unlock(&sb->lock); 792 793 return 0; 794 } 795 796 static bool xive_restore_pending_irq(struct kvmppc_xive *xive, u32 irq) 797 { 798 struct kvmppc_xive_src_block *sb; 799 struct kvmppc_xive_irq_state *state; 800 u16 idx; 801 802 sb = kvmppc_xive_find_source(xive, irq, &idx); 803 if (!sb) 804 return false; 805 state = &sb->irq_state[idx]; 806 if (!state->valid) 807 return false; 808 809 /* 810 * Trigger the IPI. This assumes we never restore a pass-through 811 * interrupt which should be safe enough 812 */ 813 xive_irq_trigger(&state->ipi_data); 814 815 return true; 816 } 817 818 u64 kvmppc_xive_get_icp(struct kvm_vcpu *vcpu) 819 { 820 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 821 822 if (!xc) 823 return 0; 824 825 /* Return the per-cpu state for state saving/migration */ 826 return (u64)xc->cppr << KVM_REG_PPC_ICP_CPPR_SHIFT | 827 (u64)xc->mfrr << KVM_REG_PPC_ICP_MFRR_SHIFT | 828 (u64)0xff << KVM_REG_PPC_ICP_PPRI_SHIFT; 829 } 830 831 int kvmppc_xive_set_icp(struct kvm_vcpu *vcpu, u64 icpval) 832 { 833 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 834 struct kvmppc_xive *xive = vcpu->kvm->arch.xive; 835 u8 cppr, mfrr; 836 u32 xisr; 837 838 if (!xc || !xive) 839 return -ENOENT; 840 841 /* Grab individual state fields. We don't use pending_pri */ 842 cppr = icpval >> KVM_REG_PPC_ICP_CPPR_SHIFT; 843 xisr = (icpval >> KVM_REG_PPC_ICP_XISR_SHIFT) & 844 KVM_REG_PPC_ICP_XISR_MASK; 845 mfrr = icpval >> KVM_REG_PPC_ICP_MFRR_SHIFT; 846 847 pr_devel("set_icp vcpu %d cppr=0x%x mfrr=0x%x xisr=0x%x\n", 848 xc->server_num, cppr, mfrr, xisr); 849 850 /* 851 * We can't update the state of a "pushed" VCPU, but that 852 * shouldn't happen because the vcpu->mutex makes running a 853 * vcpu mutually exclusive with doing one_reg get/set on it. 854 */ 855 if (WARN_ON(vcpu->arch.xive_pushed)) 856 return -EIO; 857 858 /* Update VCPU HW saved state */ 859 vcpu->arch.xive_saved_state.cppr = cppr; 860 xc->hw_cppr = xc->cppr = cppr; 861 862 /* 863 * Update MFRR state. If it's not 0xff, we mark the VCPU as 864 * having a pending MFRR change, which will re-evaluate the 865 * target. The VCPU will thus potentially get a spurious 866 * interrupt but that's not a big deal. 867 */ 868 xc->mfrr = mfrr; 869 if (mfrr < cppr) 870 xive_irq_trigger(&xc->vp_ipi_data); 871 872 /* 873 * Now saved XIRR is "interesting". It means there's something in 874 * the legacy "1 element" queue... for an IPI we simply ignore it, 875 * as the MFRR restore will handle that. For anything else we need 876 * to force a resend of the source. 877 * However the source may not have been setup yet. If that's the 878 * case, we keep that info and increment a counter in the xive to 879 * tell subsequent xive_set_source() to go look. 880 */ 881 if (xisr > XICS_IPI && !xive_restore_pending_irq(xive, xisr)) { 882 xc->delayed_irq = xisr; 883 xive->delayed_irqs++; 884 pr_devel(" xisr restore delayed\n"); 885 } 886 887 return 0; 888 } 889 890 int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq, 891 struct irq_desc *host_desc) 892 { 893 struct kvmppc_xive *xive = kvm->arch.xive; 894 struct kvmppc_xive_src_block *sb; 895 struct kvmppc_xive_irq_state *state; 896 struct irq_data *host_data = irq_desc_get_irq_data(host_desc); 897 unsigned int host_irq = irq_desc_get_irq(host_desc); 898 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(host_data); 899 u16 idx; 900 u8 prio; 901 int rc; 902 903 if (!xive) 904 return -ENODEV; 905 906 pr_devel("set_mapped girq 0x%lx host HW irq 0x%x...\n",guest_irq, hw_irq); 907 908 sb = kvmppc_xive_find_source(xive, guest_irq, &idx); 909 if (!sb) 910 return -EINVAL; 911 state = &sb->irq_state[idx]; 912 913 /* 914 * Mark the passed-through interrupt as going to a VCPU, 915 * this will prevent further EOIs and similar operations 916 * from the XIVE code. It will also mask the interrupt 917 * to either PQ=10 or 11 state, the latter if the interrupt 918 * is pending. This will allow us to unmask or retrigger it 919 * after routing it to the guest with a simple EOI. 920 * 921 * The "state" argument is a "token", all it needs is to be 922 * non-NULL to switch to passed-through or NULL for the 923 * other way around. We may not yet have an actual VCPU 924 * target here and we don't really care. 925 */ 926 rc = irq_set_vcpu_affinity(host_irq, state); 927 if (rc) { 928 pr_err("Failed to set VCPU affinity for irq %d\n", host_irq); 929 return rc; 930 } 931 932 /* 933 * Mask and read state of IPI. We need to know if its P bit 934 * is set as that means it's potentially already using a 935 * queue entry in the target 936 */ 937 prio = xive_lock_and_mask(xive, sb, state); 938 pr_devel(" old IPI prio %02x P:%d Q:%d\n", prio, 939 state->old_p, state->old_q); 940 941 /* Turn the IPI hard off */ 942 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01); 943 944 /* 945 * Reset ESB guest mapping. Needed when ESB pages are exposed 946 * to the guest in XIVE native mode 947 */ 948 if (xive->ops && xive->ops->reset_mapped) 949 xive->ops->reset_mapped(kvm, guest_irq); 950 951 /* Grab info about irq */ 952 state->pt_number = hw_irq; 953 state->pt_data = irq_data_get_irq_handler_data(host_data); 954 955 /* 956 * Configure the IRQ to match the existing configuration of 957 * the IPI if it was already targetted. Otherwise this will 958 * mask the interrupt in a lossy way (act_priority is 0xff) 959 * which is fine for a never started interrupt. 960 */ 961 xive_native_configure_irq(hw_irq, 962 kvmppc_xive_vp(xive, state->act_server), 963 state->act_priority, state->number); 964 965 /* 966 * We do an EOI to enable the interrupt (and retrigger if needed) 967 * if the guest has the interrupt unmasked and the P bit was *not* 968 * set in the IPI. If it was set, we know a slot may still be in 969 * use in the target queue thus we have to wait for a guest 970 * originated EOI 971 */ 972 if (prio != MASKED && !state->old_p) 973 xive_vm_source_eoi(hw_irq, state->pt_data); 974 975 /* Clear old_p/old_q as they are no longer relevant */ 976 state->old_p = state->old_q = false; 977 978 /* Restore guest prio (unlocks EOI) */ 979 mb(); 980 state->guest_priority = prio; 981 arch_spin_unlock(&sb->lock); 982 983 return 0; 984 } 985 EXPORT_SYMBOL_GPL(kvmppc_xive_set_mapped); 986 987 int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq, 988 struct irq_desc *host_desc) 989 { 990 struct kvmppc_xive *xive = kvm->arch.xive; 991 struct kvmppc_xive_src_block *sb; 992 struct kvmppc_xive_irq_state *state; 993 unsigned int host_irq = irq_desc_get_irq(host_desc); 994 u16 idx; 995 u8 prio; 996 int rc; 997 998 if (!xive) 999 return -ENODEV; 1000 1001 pr_devel("clr_mapped girq 0x%lx...\n", guest_irq); 1002 1003 sb = kvmppc_xive_find_source(xive, guest_irq, &idx); 1004 if (!sb) 1005 return -EINVAL; 1006 state = &sb->irq_state[idx]; 1007 1008 /* 1009 * Mask and read state of IRQ. We need to know if its P bit 1010 * is set as that means it's potentially already using a 1011 * queue entry in the target 1012 */ 1013 prio = xive_lock_and_mask(xive, sb, state); 1014 pr_devel(" old IRQ prio %02x P:%d Q:%d\n", prio, 1015 state->old_p, state->old_q); 1016 1017 /* 1018 * If old_p is set, the interrupt is pending, we switch it to 1019 * PQ=11. This will force a resend in the host so the interrupt 1020 * isn't lost to whatver host driver may pick it up 1021 */ 1022 if (state->old_p) 1023 xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_11); 1024 1025 /* Release the passed-through interrupt to the host */ 1026 rc = irq_set_vcpu_affinity(host_irq, NULL); 1027 if (rc) { 1028 pr_err("Failed to clr VCPU affinity for irq %d\n", host_irq); 1029 return rc; 1030 } 1031 1032 /* Forget about the IRQ */ 1033 state->pt_number = 0; 1034 state->pt_data = NULL; 1035 1036 /* 1037 * Reset ESB guest mapping. Needed when ESB pages are exposed 1038 * to the guest in XIVE native mode 1039 */ 1040 if (xive->ops && xive->ops->reset_mapped) { 1041 xive->ops->reset_mapped(kvm, guest_irq); 1042 } 1043 1044 /* Reconfigure the IPI */ 1045 xive_native_configure_irq(state->ipi_number, 1046 kvmppc_xive_vp(xive, state->act_server), 1047 state->act_priority, state->number); 1048 1049 /* 1050 * If old_p is set (we have a queue entry potentially 1051 * occupied) or the interrupt is masked, we set the IPI 1052 * to PQ=10 state. Otherwise we just re-enable it (PQ=00). 1053 */ 1054 if (prio == MASKED || state->old_p) 1055 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_10); 1056 else 1057 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_00); 1058 1059 /* Restore guest prio (unlocks EOI) */ 1060 mb(); 1061 state->guest_priority = prio; 1062 arch_spin_unlock(&sb->lock); 1063 1064 return 0; 1065 } 1066 EXPORT_SYMBOL_GPL(kvmppc_xive_clr_mapped); 1067 1068 void kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu *vcpu) 1069 { 1070 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 1071 struct kvm *kvm = vcpu->kvm; 1072 struct kvmppc_xive *xive = kvm->arch.xive; 1073 int i, j; 1074 1075 for (i = 0; i <= xive->max_sbid; i++) { 1076 struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 1077 1078 if (!sb) 1079 continue; 1080 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) { 1081 struct kvmppc_xive_irq_state *state = &sb->irq_state[j]; 1082 1083 if (!state->valid) 1084 continue; 1085 if (state->act_priority == MASKED) 1086 continue; 1087 if (state->act_server != xc->server_num) 1088 continue; 1089 1090 /* Clean it up */ 1091 arch_spin_lock(&sb->lock); 1092 state->act_priority = MASKED; 1093 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01); 1094 xive_native_configure_irq(state->ipi_number, 0, MASKED, 0); 1095 if (state->pt_number) { 1096 xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01); 1097 xive_native_configure_irq(state->pt_number, 0, MASKED, 0); 1098 } 1099 arch_spin_unlock(&sb->lock); 1100 } 1101 } 1102 1103 /* Disable vcpu's escalation interrupt */ 1104 if (vcpu->arch.xive_esc_on) { 1105 __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr + 1106 XIVE_ESB_SET_PQ_01)); 1107 vcpu->arch.xive_esc_on = false; 1108 } 1109 1110 /* 1111 * Clear pointers to escalation interrupt ESB. 1112 * This is safe because the vcpu->mutex is held, preventing 1113 * any other CPU from concurrently executing a KVM_RUN ioctl. 1114 */ 1115 vcpu->arch.xive_esc_vaddr = 0; 1116 vcpu->arch.xive_esc_raddr = 0; 1117 } 1118 1119 void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu) 1120 { 1121 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 1122 struct kvmppc_xive *xive = vcpu->kvm->arch.xive; 1123 int i; 1124 1125 if (!kvmppc_xics_enabled(vcpu)) 1126 return; 1127 1128 if (!xc) 1129 return; 1130 1131 pr_devel("cleanup_vcpu(cpu=%d)\n", xc->server_num); 1132 1133 /* Ensure no interrupt is still routed to that VP */ 1134 xc->valid = false; 1135 kvmppc_xive_disable_vcpu_interrupts(vcpu); 1136 1137 /* Mask the VP IPI */ 1138 xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_01); 1139 1140 /* Disable the VP */ 1141 xive_native_disable_vp(xc->vp_id); 1142 1143 /* Free the queues & associated interrupts */ 1144 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) { 1145 struct xive_q *q = &xc->queues[i]; 1146 1147 /* Free the escalation irq */ 1148 if (xc->esc_virq[i]) { 1149 free_irq(xc->esc_virq[i], vcpu); 1150 irq_dispose_mapping(xc->esc_virq[i]); 1151 kfree(xc->esc_virq_names[i]); 1152 } 1153 /* Free the queue */ 1154 xive_native_disable_queue(xc->vp_id, q, i); 1155 if (q->qpage) { 1156 free_pages((unsigned long)q->qpage, 1157 xive->q_page_order); 1158 q->qpage = NULL; 1159 } 1160 } 1161 1162 /* Free the IPI */ 1163 if (xc->vp_ipi) { 1164 xive_cleanup_irq_data(&xc->vp_ipi_data); 1165 xive_native_free_irq(xc->vp_ipi); 1166 } 1167 /* Free the VP */ 1168 kfree(xc); 1169 1170 /* Cleanup the vcpu */ 1171 vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT; 1172 vcpu->arch.xive_vcpu = NULL; 1173 } 1174 1175 int kvmppc_xive_connect_vcpu(struct kvm_device *dev, 1176 struct kvm_vcpu *vcpu, u32 cpu) 1177 { 1178 struct kvmppc_xive *xive = dev->private; 1179 struct kvmppc_xive_vcpu *xc; 1180 int i, r = -EBUSY; 1181 1182 pr_devel("connect_vcpu(cpu=%d)\n", cpu); 1183 1184 if (dev->ops != &kvm_xive_ops) { 1185 pr_devel("Wrong ops !\n"); 1186 return -EPERM; 1187 } 1188 if (xive->kvm != vcpu->kvm) 1189 return -EPERM; 1190 if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT) 1191 return -EBUSY; 1192 if (kvmppc_xive_find_server(vcpu->kvm, cpu)) { 1193 pr_devel("Duplicate !\n"); 1194 return -EEXIST; 1195 } 1196 if (cpu >= (KVM_MAX_VCPUS * vcpu->kvm->arch.emul_smt_mode)) { 1197 pr_devel("Out of bounds !\n"); 1198 return -EINVAL; 1199 } 1200 xc = kzalloc(sizeof(*xc), GFP_KERNEL); 1201 if (!xc) 1202 return -ENOMEM; 1203 1204 /* We need to synchronize with queue provisioning */ 1205 mutex_lock(&xive->lock); 1206 vcpu->arch.xive_vcpu = xc; 1207 xc->xive = xive; 1208 xc->vcpu = vcpu; 1209 xc->server_num = cpu; 1210 xc->vp_id = kvmppc_xive_vp(xive, cpu); 1211 xc->mfrr = 0xff; 1212 xc->valid = true; 1213 1214 r = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id); 1215 if (r) 1216 goto bail; 1217 1218 /* Configure VCPU fields for use by assembly push/pull */ 1219 vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000); 1220 vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO); 1221 1222 /* Allocate IPI */ 1223 xc->vp_ipi = xive_native_alloc_irq(); 1224 if (!xc->vp_ipi) { 1225 pr_err("Failed to allocate xive irq for VCPU IPI\n"); 1226 r = -EIO; 1227 goto bail; 1228 } 1229 pr_devel(" IPI=0x%x\n", xc->vp_ipi); 1230 1231 r = xive_native_populate_irq_data(xc->vp_ipi, &xc->vp_ipi_data); 1232 if (r) 1233 goto bail; 1234 1235 /* 1236 * Enable the VP first as the single escalation mode will 1237 * affect escalation interrupts numbering 1238 */ 1239 r = xive_native_enable_vp(xc->vp_id, xive->single_escalation); 1240 if (r) { 1241 pr_err("Failed to enable VP in OPAL, err %d\n", r); 1242 goto bail; 1243 } 1244 1245 /* 1246 * Initialize queues. Initially we set them all for no queueing 1247 * and we enable escalation for queue 0 only which we'll use for 1248 * our mfrr change notifications. If the VCPU is hot-plugged, we 1249 * do handle provisioning however based on the existing "map" 1250 * of enabled queues. 1251 */ 1252 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) { 1253 struct xive_q *q = &xc->queues[i]; 1254 1255 /* Single escalation, no queue 7 */ 1256 if (i == 7 && xive->single_escalation) 1257 break; 1258 1259 /* Is queue already enabled ? Provision it */ 1260 if (xive->qmap & (1 << i)) { 1261 r = xive_provision_queue(vcpu, i); 1262 if (r == 0 && !xive->single_escalation) 1263 kvmppc_xive_attach_escalation( 1264 vcpu, i, xive->single_escalation); 1265 if (r) 1266 goto bail; 1267 } else { 1268 r = xive_native_configure_queue(xc->vp_id, 1269 q, i, NULL, 0, true); 1270 if (r) { 1271 pr_err("Failed to configure queue %d for VCPU %d\n", 1272 i, cpu); 1273 goto bail; 1274 } 1275 } 1276 } 1277 1278 /* If not done above, attach priority 0 escalation */ 1279 r = kvmppc_xive_attach_escalation(vcpu, 0, xive->single_escalation); 1280 if (r) 1281 goto bail; 1282 1283 /* Route the IPI */ 1284 r = xive_native_configure_irq(xc->vp_ipi, xc->vp_id, 0, XICS_IPI); 1285 if (!r) 1286 xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_00); 1287 1288 bail: 1289 mutex_unlock(&xive->lock); 1290 if (r) { 1291 kvmppc_xive_cleanup_vcpu(vcpu); 1292 return r; 1293 } 1294 1295 vcpu->arch.irq_type = KVMPPC_IRQ_XICS; 1296 return 0; 1297 } 1298 1299 /* 1300 * Scanning of queues before/after migration save 1301 */ 1302 static void xive_pre_save_set_queued(struct kvmppc_xive *xive, u32 irq) 1303 { 1304 struct kvmppc_xive_src_block *sb; 1305 struct kvmppc_xive_irq_state *state; 1306 u16 idx; 1307 1308 sb = kvmppc_xive_find_source(xive, irq, &idx); 1309 if (!sb) 1310 return; 1311 1312 state = &sb->irq_state[idx]; 1313 1314 /* Some sanity checking */ 1315 if (!state->valid) { 1316 pr_err("invalid irq 0x%x in cpu queue!\n", irq); 1317 return; 1318 } 1319 1320 /* 1321 * If the interrupt is in a queue it should have P set. 1322 * We warn so that gets reported. A backtrace isn't useful 1323 * so no need to use a WARN_ON. 1324 */ 1325 if (!state->saved_p) 1326 pr_err("Interrupt 0x%x is marked in a queue but P not set !\n", irq); 1327 1328 /* Set flag */ 1329 state->in_queue = true; 1330 } 1331 1332 static void xive_pre_save_mask_irq(struct kvmppc_xive *xive, 1333 struct kvmppc_xive_src_block *sb, 1334 u32 irq) 1335 { 1336 struct kvmppc_xive_irq_state *state = &sb->irq_state[irq]; 1337 1338 if (!state->valid) 1339 return; 1340 1341 /* Mask and save state, this will also sync HW queues */ 1342 state->saved_scan_prio = xive_lock_and_mask(xive, sb, state); 1343 1344 /* Transfer P and Q */ 1345 state->saved_p = state->old_p; 1346 state->saved_q = state->old_q; 1347 1348 /* Unlock */ 1349 arch_spin_unlock(&sb->lock); 1350 } 1351 1352 static void xive_pre_save_unmask_irq(struct kvmppc_xive *xive, 1353 struct kvmppc_xive_src_block *sb, 1354 u32 irq) 1355 { 1356 struct kvmppc_xive_irq_state *state = &sb->irq_state[irq]; 1357 1358 if (!state->valid) 1359 return; 1360 1361 /* 1362 * Lock / exclude EOI (not technically necessary if the 1363 * guest isn't running concurrently. If this becomes a 1364 * performance issue we can probably remove the lock. 1365 */ 1366 xive_lock_for_unmask(sb, state); 1367 1368 /* Restore mask/prio if it wasn't masked */ 1369 if (state->saved_scan_prio != MASKED) 1370 xive_finish_unmask(xive, sb, state, state->saved_scan_prio); 1371 1372 /* Unlock */ 1373 arch_spin_unlock(&sb->lock); 1374 } 1375 1376 static void xive_pre_save_queue(struct kvmppc_xive *xive, struct xive_q *q) 1377 { 1378 u32 idx = q->idx; 1379 u32 toggle = q->toggle; 1380 u32 irq; 1381 1382 do { 1383 irq = __xive_read_eq(q->qpage, q->msk, &idx, &toggle); 1384 if (irq > XICS_IPI) 1385 xive_pre_save_set_queued(xive, irq); 1386 } while(irq); 1387 } 1388 1389 static void xive_pre_save_scan(struct kvmppc_xive *xive) 1390 { 1391 struct kvm_vcpu *vcpu = NULL; 1392 int i, j; 1393 1394 /* 1395 * See comment in xive_get_source() about how this 1396 * work. Collect a stable state for all interrupts 1397 */ 1398 for (i = 0; i <= xive->max_sbid; i++) { 1399 struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 1400 if (!sb) 1401 continue; 1402 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) 1403 xive_pre_save_mask_irq(xive, sb, j); 1404 } 1405 1406 /* Then scan the queues and update the "in_queue" flag */ 1407 kvm_for_each_vcpu(i, vcpu, xive->kvm) { 1408 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 1409 if (!xc) 1410 continue; 1411 for (j = 0; j < KVMPPC_XIVE_Q_COUNT; j++) { 1412 if (xc->queues[j].qpage) 1413 xive_pre_save_queue(xive, &xc->queues[j]); 1414 } 1415 } 1416 1417 /* Finally restore interrupt states */ 1418 for (i = 0; i <= xive->max_sbid; i++) { 1419 struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 1420 if (!sb) 1421 continue; 1422 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) 1423 xive_pre_save_unmask_irq(xive, sb, j); 1424 } 1425 } 1426 1427 static void xive_post_save_scan(struct kvmppc_xive *xive) 1428 { 1429 u32 i, j; 1430 1431 /* Clear all the in_queue flags */ 1432 for (i = 0; i <= xive->max_sbid; i++) { 1433 struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; 1434 if (!sb) 1435 continue; 1436 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) 1437 sb->irq_state[j].in_queue = false; 1438 } 1439 1440 /* Next get_source() will do a new scan */ 1441 xive->saved_src_count = 0; 1442 } 1443 1444 /* 1445 * This returns the source configuration and state to user space. 1446 */ 1447 static int xive_get_source(struct kvmppc_xive *xive, long irq, u64 addr) 1448 { 1449 struct kvmppc_xive_src_block *sb; 1450 struct kvmppc_xive_irq_state *state; 1451 u64 __user *ubufp = (u64 __user *) addr; 1452 u64 val, prio; 1453 u16 idx; 1454 1455 sb = kvmppc_xive_find_source(xive, irq, &idx); 1456 if (!sb) 1457 return -ENOENT; 1458 1459 state = &sb->irq_state[idx]; 1460 1461 if (!state->valid) 1462 return -ENOENT; 1463 1464 pr_devel("get_source(%ld)...\n", irq); 1465 1466 /* 1467 * So to properly save the state into something that looks like a 1468 * XICS migration stream we cannot treat interrupts individually. 1469 * 1470 * We need, instead, mask them all (& save their previous PQ state) 1471 * to get a stable state in the HW, then sync them to ensure that 1472 * any interrupt that had already fired hits its queue, and finally 1473 * scan all the queues to collect which interrupts are still present 1474 * in the queues, so we can set the "pending" flag on them and 1475 * they can be resent on restore. 1476 * 1477 * So we do it all when the "first" interrupt gets saved, all the 1478 * state is collected at that point, the rest of xive_get_source() 1479 * will merely collect and convert that state to the expected 1480 * userspace bit mask. 1481 */ 1482 if (xive->saved_src_count == 0) 1483 xive_pre_save_scan(xive); 1484 xive->saved_src_count++; 1485 1486 /* Convert saved state into something compatible with xics */ 1487 val = state->act_server; 1488 prio = state->saved_scan_prio; 1489 1490 if (prio == MASKED) { 1491 val |= KVM_XICS_MASKED; 1492 prio = state->saved_priority; 1493 } 1494 val |= prio << KVM_XICS_PRIORITY_SHIFT; 1495 if (state->lsi) { 1496 val |= KVM_XICS_LEVEL_SENSITIVE; 1497 if (state->saved_p) 1498 val |= KVM_XICS_PENDING; 1499 } else { 1500 if (state->saved_p) 1501 val |= KVM_XICS_PRESENTED; 1502 1503 if (state->saved_q) 1504 val |= KVM_XICS_QUEUED; 1505 1506 /* 1507 * We mark it pending (which will attempt a re-delivery) 1508 * if we are in a queue *or* we were masked and had 1509 * Q set which is equivalent to the XICS "masked pending" 1510 * state 1511 */ 1512 if (state->in_queue || (prio == MASKED && state->saved_q)) 1513 val |= KVM_XICS_PENDING; 1514 } 1515 1516 /* 1517 * If that was the last interrupt saved, reset the 1518 * in_queue flags 1519 */ 1520 if (xive->saved_src_count == xive->src_count) 1521 xive_post_save_scan(xive); 1522 1523 /* Copy the result to userspace */ 1524 if (put_user(val, ubufp)) 1525 return -EFAULT; 1526 1527 return 0; 1528 } 1529 1530 struct kvmppc_xive_src_block *kvmppc_xive_create_src_block( 1531 struct kvmppc_xive *xive, int irq) 1532 { 1533 struct kvmppc_xive_src_block *sb; 1534 int i, bid; 1535 1536 bid = irq >> KVMPPC_XICS_ICS_SHIFT; 1537 1538 mutex_lock(&xive->lock); 1539 1540 /* block already exists - somebody else got here first */ 1541 if (xive->src_blocks[bid]) 1542 goto out; 1543 1544 /* Create the ICS */ 1545 sb = kzalloc(sizeof(*sb), GFP_KERNEL); 1546 if (!sb) 1547 goto out; 1548 1549 sb->id = bid; 1550 1551 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 1552 sb->irq_state[i].number = (bid << KVMPPC_XICS_ICS_SHIFT) | i; 1553 sb->irq_state[i].eisn = 0; 1554 sb->irq_state[i].guest_priority = MASKED; 1555 sb->irq_state[i].saved_priority = MASKED; 1556 sb->irq_state[i].act_priority = MASKED; 1557 } 1558 smp_wmb(); 1559 xive->src_blocks[bid] = sb; 1560 1561 if (bid > xive->max_sbid) 1562 xive->max_sbid = bid; 1563 1564 out: 1565 mutex_unlock(&xive->lock); 1566 return xive->src_blocks[bid]; 1567 } 1568 1569 static bool xive_check_delayed_irq(struct kvmppc_xive *xive, u32 irq) 1570 { 1571 struct kvm *kvm = xive->kvm; 1572 struct kvm_vcpu *vcpu = NULL; 1573 int i; 1574 1575 kvm_for_each_vcpu(i, vcpu, kvm) { 1576 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 1577 1578 if (!xc) 1579 continue; 1580 1581 if (xc->delayed_irq == irq) { 1582 xc->delayed_irq = 0; 1583 xive->delayed_irqs--; 1584 return true; 1585 } 1586 } 1587 return false; 1588 } 1589 1590 static int xive_set_source(struct kvmppc_xive *xive, long irq, u64 addr) 1591 { 1592 struct kvmppc_xive_src_block *sb; 1593 struct kvmppc_xive_irq_state *state; 1594 u64 __user *ubufp = (u64 __user *) addr; 1595 u16 idx; 1596 u64 val; 1597 u8 act_prio, guest_prio; 1598 u32 server; 1599 int rc = 0; 1600 1601 if (irq < KVMPPC_XICS_FIRST_IRQ || irq >= KVMPPC_XICS_NR_IRQS) 1602 return -ENOENT; 1603 1604 pr_devel("set_source(irq=0x%lx)\n", irq); 1605 1606 /* Find the source */ 1607 sb = kvmppc_xive_find_source(xive, irq, &idx); 1608 if (!sb) { 1609 pr_devel("No source, creating source block...\n"); 1610 sb = kvmppc_xive_create_src_block(xive, irq); 1611 if (!sb) { 1612 pr_devel("Failed to create block...\n"); 1613 return -ENOMEM; 1614 } 1615 } 1616 state = &sb->irq_state[idx]; 1617 1618 /* Read user passed data */ 1619 if (get_user(val, ubufp)) { 1620 pr_devel("fault getting user info !\n"); 1621 return -EFAULT; 1622 } 1623 1624 server = val & KVM_XICS_DESTINATION_MASK; 1625 guest_prio = val >> KVM_XICS_PRIORITY_SHIFT; 1626 1627 pr_devel(" val=0x016%llx (server=0x%x, guest_prio=%d)\n", 1628 val, server, guest_prio); 1629 1630 /* 1631 * If the source doesn't already have an IPI, allocate 1632 * one and get the corresponding data 1633 */ 1634 if (!state->ipi_number) { 1635 state->ipi_number = xive_native_alloc_irq(); 1636 if (state->ipi_number == 0) { 1637 pr_devel("Failed to allocate IPI !\n"); 1638 return -ENOMEM; 1639 } 1640 xive_native_populate_irq_data(state->ipi_number, &state->ipi_data); 1641 pr_devel(" src_ipi=0x%x\n", state->ipi_number); 1642 } 1643 1644 /* 1645 * We use lock_and_mask() to set us in the right masked 1646 * state. We will override that state from the saved state 1647 * further down, but this will handle the cases of interrupts 1648 * that need FW masking. We set the initial guest_priority to 1649 * 0 before calling it to ensure it actually performs the masking. 1650 */ 1651 state->guest_priority = 0; 1652 xive_lock_and_mask(xive, sb, state); 1653 1654 /* 1655 * Now, we select a target if we have one. If we don't we 1656 * leave the interrupt untargetted. It means that an interrupt 1657 * can become "untargetted" accross migration if it was masked 1658 * by set_xive() but there is little we can do about it. 1659 */ 1660 1661 /* First convert prio and mark interrupt as untargetted */ 1662 act_prio = xive_prio_from_guest(guest_prio); 1663 state->act_priority = MASKED; 1664 1665 /* 1666 * We need to drop the lock due to the mutex below. Hopefully 1667 * nothing is touching that interrupt yet since it hasn't been 1668 * advertized to a running guest yet 1669 */ 1670 arch_spin_unlock(&sb->lock); 1671 1672 /* If we have a priority target the interrupt */ 1673 if (act_prio != MASKED) { 1674 /* First, check provisioning of queues */ 1675 mutex_lock(&xive->lock); 1676 rc = xive_check_provisioning(xive->kvm, act_prio); 1677 mutex_unlock(&xive->lock); 1678 1679 /* Target interrupt */ 1680 if (rc == 0) 1681 rc = xive_target_interrupt(xive->kvm, state, 1682 server, act_prio); 1683 /* 1684 * If provisioning or targetting failed, leave it 1685 * alone and masked. It will remain disabled until 1686 * the guest re-targets it. 1687 */ 1688 } 1689 1690 /* 1691 * Find out if this was a delayed irq stashed in an ICP, 1692 * in which case, treat it as pending 1693 */ 1694 if (xive->delayed_irqs && xive_check_delayed_irq(xive, irq)) { 1695 val |= KVM_XICS_PENDING; 1696 pr_devel(" Found delayed ! forcing PENDING !\n"); 1697 } 1698 1699 /* Cleanup the SW state */ 1700 state->old_p = false; 1701 state->old_q = false; 1702 state->lsi = false; 1703 state->asserted = false; 1704 1705 /* Restore LSI state */ 1706 if (val & KVM_XICS_LEVEL_SENSITIVE) { 1707 state->lsi = true; 1708 if (val & KVM_XICS_PENDING) 1709 state->asserted = true; 1710 pr_devel(" LSI ! Asserted=%d\n", state->asserted); 1711 } 1712 1713 /* 1714 * Restore P and Q. If the interrupt was pending, we 1715 * force Q and !P, which will trigger a resend. 1716 * 1717 * That means that a guest that had both an interrupt 1718 * pending (queued) and Q set will restore with only 1719 * one instance of that interrupt instead of 2, but that 1720 * is perfectly fine as coalescing interrupts that haven't 1721 * been presented yet is always allowed. 1722 */ 1723 if (val & KVM_XICS_PRESENTED && !(val & KVM_XICS_PENDING)) 1724 state->old_p = true; 1725 if (val & KVM_XICS_QUEUED || val & KVM_XICS_PENDING) 1726 state->old_q = true; 1727 1728 pr_devel(" P=%d, Q=%d\n", state->old_p, state->old_q); 1729 1730 /* 1731 * If the interrupt was unmasked, update guest priority and 1732 * perform the appropriate state transition and do a 1733 * re-trigger if necessary. 1734 */ 1735 if (val & KVM_XICS_MASKED) { 1736 pr_devel(" masked, saving prio\n"); 1737 state->guest_priority = MASKED; 1738 state->saved_priority = guest_prio; 1739 } else { 1740 pr_devel(" unmasked, restoring to prio %d\n", guest_prio); 1741 xive_finish_unmask(xive, sb, state, guest_prio); 1742 state->saved_priority = guest_prio; 1743 } 1744 1745 /* Increment the number of valid sources and mark this one valid */ 1746 if (!state->valid) 1747 xive->src_count++; 1748 state->valid = true; 1749 1750 return 0; 1751 } 1752 1753 int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level, 1754 bool line_status) 1755 { 1756 struct kvmppc_xive *xive = kvm->arch.xive; 1757 struct kvmppc_xive_src_block *sb; 1758 struct kvmppc_xive_irq_state *state; 1759 u16 idx; 1760 1761 if (!xive) 1762 return -ENODEV; 1763 1764 sb = kvmppc_xive_find_source(xive, irq, &idx); 1765 if (!sb) 1766 return -EINVAL; 1767 1768 /* Perform locklessly .... (we need to do some RCUisms here...) */ 1769 state = &sb->irq_state[idx]; 1770 if (!state->valid) 1771 return -EINVAL; 1772 1773 /* We don't allow a trigger on a passed-through interrupt */ 1774 if (state->pt_number) 1775 return -EINVAL; 1776 1777 if ((level == 1 && state->lsi) || level == KVM_INTERRUPT_SET_LEVEL) 1778 state->asserted = 1; 1779 else if (level == 0 || level == KVM_INTERRUPT_UNSET) { 1780 state->asserted = 0; 1781 return 0; 1782 } 1783 1784 /* Trigger the IPI */ 1785 xive_irq_trigger(&state->ipi_data); 1786 1787 return 0; 1788 } 1789 1790 static int xive_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 1791 { 1792 struct kvmppc_xive *xive = dev->private; 1793 1794 /* We honor the existing XICS ioctl */ 1795 switch (attr->group) { 1796 case KVM_DEV_XICS_GRP_SOURCES: 1797 return xive_set_source(xive, attr->attr, attr->addr); 1798 } 1799 return -ENXIO; 1800 } 1801 1802 static int xive_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 1803 { 1804 struct kvmppc_xive *xive = dev->private; 1805 1806 /* We honor the existing XICS ioctl */ 1807 switch (attr->group) { 1808 case KVM_DEV_XICS_GRP_SOURCES: 1809 return xive_get_source(xive, attr->attr, attr->addr); 1810 } 1811 return -ENXIO; 1812 } 1813 1814 static int xive_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 1815 { 1816 /* We honor the same limits as XICS, at least for now */ 1817 switch (attr->group) { 1818 case KVM_DEV_XICS_GRP_SOURCES: 1819 if (attr->attr >= KVMPPC_XICS_FIRST_IRQ && 1820 attr->attr < KVMPPC_XICS_NR_IRQS) 1821 return 0; 1822 break; 1823 } 1824 return -ENXIO; 1825 } 1826 1827 static void kvmppc_xive_cleanup_irq(u32 hw_num, struct xive_irq_data *xd) 1828 { 1829 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01); 1830 xive_native_configure_irq(hw_num, 0, MASKED, 0); 1831 } 1832 1833 void kvmppc_xive_free_sources(struct kvmppc_xive_src_block *sb) 1834 { 1835 int i; 1836 1837 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 1838 struct kvmppc_xive_irq_state *state = &sb->irq_state[i]; 1839 1840 if (!state->valid) 1841 continue; 1842 1843 kvmppc_xive_cleanup_irq(state->ipi_number, &state->ipi_data); 1844 xive_cleanup_irq_data(&state->ipi_data); 1845 xive_native_free_irq(state->ipi_number); 1846 1847 /* Pass-through, cleanup too but keep IRQ hw data */ 1848 if (state->pt_number) 1849 kvmppc_xive_cleanup_irq(state->pt_number, state->pt_data); 1850 1851 state->valid = false; 1852 } 1853 } 1854 1855 /* 1856 * Called when device fd is closed. kvm->lock is held. 1857 */ 1858 static void kvmppc_xive_release(struct kvm_device *dev) 1859 { 1860 struct kvmppc_xive *xive = dev->private; 1861 struct kvm *kvm = xive->kvm; 1862 struct kvm_vcpu *vcpu; 1863 int i; 1864 1865 pr_devel("Releasing xive device\n"); 1866 1867 /* 1868 * Since this is the device release function, we know that 1869 * userspace does not have any open fd referring to the 1870 * device. Therefore there can not be any of the device 1871 * attribute set/get functions being executed concurrently, 1872 * and similarly, the connect_vcpu and set/clr_mapped 1873 * functions also cannot be being executed. 1874 */ 1875 1876 debugfs_remove(xive->dentry); 1877 1878 /* 1879 * We should clean up the vCPU interrupt presenters first. 1880 */ 1881 kvm_for_each_vcpu(i, vcpu, kvm) { 1882 /* 1883 * Take vcpu->mutex to ensure that no one_reg get/set ioctl 1884 * (i.e. kvmppc_xive_[gs]et_icp) can be done concurrently. 1885 * Holding the vcpu->mutex also means that the vcpu cannot 1886 * be executing the KVM_RUN ioctl, and therefore it cannot 1887 * be executing the XIVE push or pull code or accessing 1888 * the XIVE MMIO regions. 1889 */ 1890 mutex_lock(&vcpu->mutex); 1891 kvmppc_xive_cleanup_vcpu(vcpu); 1892 mutex_unlock(&vcpu->mutex); 1893 } 1894 1895 /* 1896 * Now that we have cleared vcpu->arch.xive_vcpu, vcpu->arch.irq_type 1897 * and vcpu->arch.xive_esc_[vr]addr on each vcpu, we are safe 1898 * against xive code getting called during vcpu execution or 1899 * set/get one_reg operations. 1900 */ 1901 kvm->arch.xive = NULL; 1902 1903 /* Mask and free interrupts */ 1904 for (i = 0; i <= xive->max_sbid; i++) { 1905 if (xive->src_blocks[i]) 1906 kvmppc_xive_free_sources(xive->src_blocks[i]); 1907 kfree(xive->src_blocks[i]); 1908 xive->src_blocks[i] = NULL; 1909 } 1910 1911 if (xive->vp_base != XIVE_INVALID_VP) 1912 xive_native_free_vp_block(xive->vp_base); 1913 1914 /* 1915 * A reference of the kvmppc_xive pointer is now kept under 1916 * the xive_devices struct of the machine for reuse. It is 1917 * freed when the VM is destroyed for now until we fix all the 1918 * execution paths. 1919 */ 1920 1921 kfree(dev); 1922 } 1923 1924 /* 1925 * When the guest chooses the interrupt mode (XICS legacy or XIVE 1926 * native), the VM will switch of KVM device. The previous device will 1927 * be "released" before the new one is created. 1928 * 1929 * Until we are sure all execution paths are well protected, provide a 1930 * fail safe (transitional) method for device destruction, in which 1931 * the XIVE device pointer is recycled and not directly freed. 1932 */ 1933 struct kvmppc_xive *kvmppc_xive_get_device(struct kvm *kvm, u32 type) 1934 { 1935 struct kvmppc_xive **kvm_xive_device = type == KVM_DEV_TYPE_XIVE ? 1936 &kvm->arch.xive_devices.native : 1937 &kvm->arch.xive_devices.xics_on_xive; 1938 struct kvmppc_xive *xive = *kvm_xive_device; 1939 1940 if (!xive) { 1941 xive = kzalloc(sizeof(*xive), GFP_KERNEL); 1942 *kvm_xive_device = xive; 1943 } else { 1944 memset(xive, 0, sizeof(*xive)); 1945 } 1946 1947 return xive; 1948 } 1949 1950 /* 1951 * Create a XICS device with XIVE backend. kvm->lock is held. 1952 */ 1953 static int kvmppc_xive_create(struct kvm_device *dev, u32 type) 1954 { 1955 struct kvmppc_xive *xive; 1956 struct kvm *kvm = dev->kvm; 1957 int ret = 0; 1958 1959 pr_devel("Creating xive for partition\n"); 1960 1961 xive = kvmppc_xive_get_device(kvm, type); 1962 if (!xive) 1963 return -ENOMEM; 1964 1965 dev->private = xive; 1966 xive->dev = dev; 1967 xive->kvm = kvm; 1968 mutex_init(&xive->lock); 1969 1970 /* Already there ? */ 1971 if (kvm->arch.xive) 1972 ret = -EEXIST; 1973 else 1974 kvm->arch.xive = xive; 1975 1976 /* We use the default queue size set by the host */ 1977 xive->q_order = xive_native_default_eq_shift(); 1978 if (xive->q_order < PAGE_SHIFT) 1979 xive->q_page_order = 0; 1980 else 1981 xive->q_page_order = xive->q_order - PAGE_SHIFT; 1982 1983 /* Allocate a bunch of VPs */ 1984 xive->vp_base = xive_native_alloc_vp_block(KVM_MAX_VCPUS); 1985 pr_devel("VP_Base=%x\n", xive->vp_base); 1986 1987 if (xive->vp_base == XIVE_INVALID_VP) 1988 ret = -ENOMEM; 1989 1990 xive->single_escalation = xive_native_has_single_escalation(); 1991 1992 if (ret) { 1993 kfree(xive); 1994 return ret; 1995 } 1996 1997 return 0; 1998 } 1999 2000 int kvmppc_xive_debug_show_queues(struct seq_file *m, struct kvm_vcpu *vcpu) 2001 { 2002 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 2003 unsigned int i; 2004 2005 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) { 2006 struct xive_q *q = &xc->queues[i]; 2007 u32 i0, i1, idx; 2008 2009 if (!q->qpage && !xc->esc_virq[i]) 2010 continue; 2011 2012 seq_printf(m, " [q%d]: ", i); 2013 2014 if (q->qpage) { 2015 idx = q->idx; 2016 i0 = be32_to_cpup(q->qpage + idx); 2017 idx = (idx + 1) & q->msk; 2018 i1 = be32_to_cpup(q->qpage + idx); 2019 seq_printf(m, "T=%d %08x %08x...\n", q->toggle, 2020 i0, i1); 2021 } 2022 if (xc->esc_virq[i]) { 2023 struct irq_data *d = irq_get_irq_data(xc->esc_virq[i]); 2024 struct xive_irq_data *xd = 2025 irq_data_get_irq_handler_data(d); 2026 u64 pq = xive_vm_esb_load(xd, XIVE_ESB_GET); 2027 2028 seq_printf(m, "E:%c%c I(%d:%llx:%llx)", 2029 (pq & XIVE_ESB_VAL_P) ? 'P' : 'p', 2030 (pq & XIVE_ESB_VAL_Q) ? 'Q' : 'q', 2031 xc->esc_virq[i], pq, xd->eoi_page); 2032 seq_puts(m, "\n"); 2033 } 2034 } 2035 return 0; 2036 } 2037 2038 static int xive_debug_show(struct seq_file *m, void *private) 2039 { 2040 struct kvmppc_xive *xive = m->private; 2041 struct kvm *kvm = xive->kvm; 2042 struct kvm_vcpu *vcpu; 2043 u64 t_rm_h_xirr = 0; 2044 u64 t_rm_h_ipoll = 0; 2045 u64 t_rm_h_cppr = 0; 2046 u64 t_rm_h_eoi = 0; 2047 u64 t_rm_h_ipi = 0; 2048 u64 t_vm_h_xirr = 0; 2049 u64 t_vm_h_ipoll = 0; 2050 u64 t_vm_h_cppr = 0; 2051 u64 t_vm_h_eoi = 0; 2052 u64 t_vm_h_ipi = 0; 2053 unsigned int i; 2054 2055 if (!kvm) 2056 return 0; 2057 2058 seq_printf(m, "=========\nVCPU state\n=========\n"); 2059 2060 kvm_for_each_vcpu(i, vcpu, kvm) { 2061 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; 2062 2063 if (!xc) 2064 continue; 2065 2066 seq_printf(m, "cpu server %#x CPPR:%#x HWCPPR:%#x" 2067 " MFRR:%#x PEND:%#x h_xirr: R=%lld V=%lld\n", 2068 xc->server_num, xc->cppr, xc->hw_cppr, 2069 xc->mfrr, xc->pending, 2070 xc->stat_rm_h_xirr, xc->stat_vm_h_xirr); 2071 2072 kvmppc_xive_debug_show_queues(m, vcpu); 2073 2074 t_rm_h_xirr += xc->stat_rm_h_xirr; 2075 t_rm_h_ipoll += xc->stat_rm_h_ipoll; 2076 t_rm_h_cppr += xc->stat_rm_h_cppr; 2077 t_rm_h_eoi += xc->stat_rm_h_eoi; 2078 t_rm_h_ipi += xc->stat_rm_h_ipi; 2079 t_vm_h_xirr += xc->stat_vm_h_xirr; 2080 t_vm_h_ipoll += xc->stat_vm_h_ipoll; 2081 t_vm_h_cppr += xc->stat_vm_h_cppr; 2082 t_vm_h_eoi += xc->stat_vm_h_eoi; 2083 t_vm_h_ipi += xc->stat_vm_h_ipi; 2084 } 2085 2086 seq_printf(m, "Hcalls totals\n"); 2087 seq_printf(m, " H_XIRR R=%10lld V=%10lld\n", t_rm_h_xirr, t_vm_h_xirr); 2088 seq_printf(m, " H_IPOLL R=%10lld V=%10lld\n", t_rm_h_ipoll, t_vm_h_ipoll); 2089 seq_printf(m, " H_CPPR R=%10lld V=%10lld\n", t_rm_h_cppr, t_vm_h_cppr); 2090 seq_printf(m, " H_EOI R=%10lld V=%10lld\n", t_rm_h_eoi, t_vm_h_eoi); 2091 seq_printf(m, " H_IPI R=%10lld V=%10lld\n", t_rm_h_ipi, t_vm_h_ipi); 2092 2093 return 0; 2094 } 2095 2096 DEFINE_SHOW_ATTRIBUTE(xive_debug); 2097 2098 static void xive_debugfs_init(struct kvmppc_xive *xive) 2099 { 2100 char *name; 2101 2102 name = kasprintf(GFP_KERNEL, "kvm-xive-%p", xive); 2103 if (!name) { 2104 pr_err("%s: no memory for name\n", __func__); 2105 return; 2106 } 2107 2108 xive->dentry = debugfs_create_file(name, S_IRUGO, powerpc_debugfs_root, 2109 xive, &xive_debug_fops); 2110 2111 pr_debug("%s: created %s\n", __func__, name); 2112 kfree(name); 2113 } 2114 2115 static void kvmppc_xive_init(struct kvm_device *dev) 2116 { 2117 struct kvmppc_xive *xive = (struct kvmppc_xive *)dev->private; 2118 2119 /* Register some debug interfaces */ 2120 xive_debugfs_init(xive); 2121 } 2122 2123 struct kvm_device_ops kvm_xive_ops = { 2124 .name = "kvm-xive", 2125 .create = kvmppc_xive_create, 2126 .init = kvmppc_xive_init, 2127 .release = kvmppc_xive_release, 2128 .set_attr = xive_set_attr, 2129 .get_attr = xive_get_attr, 2130 .has_attr = xive_has_attr, 2131 }; 2132 2133 void kvmppc_xive_init_module(void) 2134 { 2135 __xive_vm_h_xirr = xive_vm_h_xirr; 2136 __xive_vm_h_ipoll = xive_vm_h_ipoll; 2137 __xive_vm_h_ipi = xive_vm_h_ipi; 2138 __xive_vm_h_cppr = xive_vm_h_cppr; 2139 __xive_vm_h_eoi = xive_vm_h_eoi; 2140 } 2141 2142 void kvmppc_xive_exit_module(void) 2143 { 2144 __xive_vm_h_xirr = NULL; 2145 __xive_vm_h_ipoll = NULL; 2146 __xive_vm_h_ipi = NULL; 2147 __xive_vm_h_cppr = NULL; 2148 __xive_vm_h_eoi = NULL; 2149 } 2150