1 /* 2 * Copyright 2012 Michael Ellerman, IBM Corporation. 3 * Copyright 2012 Benjamin Herrenschmidt, IBM Corporation 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License, version 2, as 7 * published by the Free Software Foundation. 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/kvm_host.h> 12 #include <linux/err.h> 13 #include <linux/kernel_stat.h> 14 15 #include <asm/kvm_book3s.h> 16 #include <asm/kvm_ppc.h> 17 #include <asm/hvcall.h> 18 #include <asm/xics.h> 19 #include <asm/synch.h> 20 #include <asm/cputhreads.h> 21 #include <asm/pgtable.h> 22 #include <asm/ppc-opcode.h> 23 #include <asm/pnv-pci.h> 24 #include <asm/opal.h> 25 #include <asm/smp.h> 26 27 #include "book3s_xics.h" 28 29 #define DEBUG_PASSUP 30 31 int h_ipi_redirect = 1; 32 EXPORT_SYMBOL(h_ipi_redirect); 33 int kvm_irq_bypass = 1; 34 EXPORT_SYMBOL(kvm_irq_bypass); 35 36 static void icp_rm_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp, 37 u32 new_irq, bool check_resend); 38 static int xics_opal_set_server(unsigned int hw_irq, int server_cpu); 39 40 /* -- ICS routines -- */ 41 static void ics_rm_check_resend(struct kvmppc_xics *xics, 42 struct kvmppc_ics *ics, struct kvmppc_icp *icp) 43 { 44 int i; 45 46 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 47 struct ics_irq_state *state = &ics->irq_state[i]; 48 if (state->resend) 49 icp_rm_deliver_irq(xics, icp, state->number, true); 50 } 51 52 } 53 54 /* -- ICP routines -- */ 55 56 #ifdef CONFIG_SMP 57 static inline void icp_send_hcore_msg(int hcore, struct kvm_vcpu *vcpu) 58 { 59 int hcpu; 60 61 hcpu = hcore << threads_shift; 62 kvmppc_host_rm_ops_hv->rm_core[hcore].rm_data = vcpu; 63 smp_muxed_ipi_set_message(hcpu, PPC_MSG_RM_HOST_ACTION); 64 kvmppc_set_host_ipi(hcpu, 1); 65 smp_mb(); 66 kvmhv_rm_send_ipi(hcpu); 67 } 68 #else 69 static inline void icp_send_hcore_msg(int hcore, struct kvm_vcpu *vcpu) { } 70 #endif 71 72 /* 73 * We start the search from our current CPU Id in the core map 74 * and go in a circle until we get back to our ID looking for a 75 * core that is running in host context and that hasn't already 76 * been targeted for another rm_host_ops. 77 * 78 * In the future, could consider using a fairer algorithm (one 79 * that distributes the IPIs better) 80 * 81 * Returns -1, if no CPU could be found in the host 82 * Else, returns a CPU Id which has been reserved for use 83 */ 84 static inline int grab_next_hostcore(int start, 85 struct kvmppc_host_rm_core *rm_core, int max, int action) 86 { 87 bool success; 88 int core; 89 union kvmppc_rm_state old, new; 90 91 for (core = start + 1; core < max; core++) { 92 old = new = READ_ONCE(rm_core[core].rm_state); 93 94 if (!old.in_host || old.rm_action) 95 continue; 96 97 /* Try to grab this host core if not taken already. */ 98 new.rm_action = action; 99 100 success = cmpxchg64(&rm_core[core].rm_state.raw, 101 old.raw, new.raw) == old.raw; 102 if (success) { 103 /* 104 * Make sure that the store to the rm_action is made 105 * visible before we return to caller (and the 106 * subsequent store to rm_data) to synchronize with 107 * the IPI handler. 108 */ 109 smp_wmb(); 110 return core; 111 } 112 } 113 114 return -1; 115 } 116 117 static inline int find_available_hostcore(int action) 118 { 119 int core; 120 int my_core = smp_processor_id() >> threads_shift; 121 struct kvmppc_host_rm_core *rm_core = kvmppc_host_rm_ops_hv->rm_core; 122 123 core = grab_next_hostcore(my_core, rm_core, cpu_nr_cores(), action); 124 if (core == -1) 125 core = grab_next_hostcore(core, rm_core, my_core, action); 126 127 return core; 128 } 129 130 static void icp_rm_set_vcpu_irq(struct kvm_vcpu *vcpu, 131 struct kvm_vcpu *this_vcpu) 132 { 133 struct kvmppc_icp *this_icp = this_vcpu->arch.icp; 134 int cpu; 135 int hcore; 136 137 /* Mark the target VCPU as having an interrupt pending */ 138 vcpu->stat.queue_intr++; 139 set_bit(BOOK3S_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions); 140 141 /* Kick self ? Just set MER and return */ 142 if (vcpu == this_vcpu) { 143 mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) | LPCR_MER); 144 return; 145 } 146 147 if (xive_enabled() && kvmhv_on_pseries()) { 148 /* No XICS access or hypercalls available, too hard */ 149 this_icp->rm_action |= XICS_RM_KICK_VCPU; 150 this_icp->rm_kick_target = vcpu; 151 return; 152 } 153 154 /* 155 * Check if the core is loaded, 156 * if not, find an available host core to post to wake the VCPU, 157 * if we can't find one, set up state to eventually return too hard. 158 */ 159 cpu = vcpu->arch.thread_cpu; 160 if (cpu < 0 || cpu >= nr_cpu_ids) { 161 hcore = -1; 162 if (kvmppc_host_rm_ops_hv && h_ipi_redirect) 163 hcore = find_available_hostcore(XICS_RM_KICK_VCPU); 164 if (hcore != -1) { 165 icp_send_hcore_msg(hcore, vcpu); 166 } else { 167 this_icp->rm_action |= XICS_RM_KICK_VCPU; 168 this_icp->rm_kick_target = vcpu; 169 } 170 return; 171 } 172 173 smp_mb(); 174 kvmhv_rm_send_ipi(cpu); 175 } 176 177 static void icp_rm_clr_vcpu_irq(struct kvm_vcpu *vcpu) 178 { 179 /* Note: Only called on self ! */ 180 clear_bit(BOOK3S_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions); 181 mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) & ~LPCR_MER); 182 } 183 184 static inline bool icp_rm_try_update(struct kvmppc_icp *icp, 185 union kvmppc_icp_state old, 186 union kvmppc_icp_state new) 187 { 188 struct kvm_vcpu *this_vcpu = local_paca->kvm_hstate.kvm_vcpu; 189 bool success; 190 191 /* Calculate new output value */ 192 new.out_ee = (new.xisr && (new.pending_pri < new.cppr)); 193 194 /* Attempt atomic update */ 195 success = cmpxchg64(&icp->state.raw, old.raw, new.raw) == old.raw; 196 if (!success) 197 goto bail; 198 199 /* 200 * Check for output state update 201 * 202 * Note that this is racy since another processor could be updating 203 * the state already. This is why we never clear the interrupt output 204 * here, we only ever set it. The clear only happens prior to doing 205 * an update and only by the processor itself. Currently we do it 206 * in Accept (H_XIRR) and Up_Cppr (H_XPPR). 207 * 208 * We also do not try to figure out whether the EE state has changed, 209 * we unconditionally set it if the new state calls for it. The reason 210 * for that is that we opportunistically remove the pending interrupt 211 * flag when raising CPPR, so we need to set it back here if an 212 * interrupt is still pending. 213 */ 214 if (new.out_ee) 215 icp_rm_set_vcpu_irq(icp->vcpu, this_vcpu); 216 217 /* Expose the state change for debug purposes */ 218 this_vcpu->arch.icp->rm_dbgstate = new; 219 this_vcpu->arch.icp->rm_dbgtgt = icp->vcpu; 220 221 bail: 222 return success; 223 } 224 225 static inline int check_too_hard(struct kvmppc_xics *xics, 226 struct kvmppc_icp *icp) 227 { 228 return (xics->real_mode_dbg || icp->rm_action) ? H_TOO_HARD : H_SUCCESS; 229 } 230 231 static void icp_rm_check_resend(struct kvmppc_xics *xics, 232 struct kvmppc_icp *icp) 233 { 234 u32 icsid; 235 236 /* Order this load with the test for need_resend in the caller */ 237 smp_rmb(); 238 for_each_set_bit(icsid, icp->resend_map, xics->max_icsid + 1) { 239 struct kvmppc_ics *ics = xics->ics[icsid]; 240 241 if (!test_and_clear_bit(icsid, icp->resend_map)) 242 continue; 243 if (!ics) 244 continue; 245 ics_rm_check_resend(xics, ics, icp); 246 } 247 } 248 249 static bool icp_rm_try_to_deliver(struct kvmppc_icp *icp, u32 irq, u8 priority, 250 u32 *reject) 251 { 252 union kvmppc_icp_state old_state, new_state; 253 bool success; 254 255 do { 256 old_state = new_state = READ_ONCE(icp->state); 257 258 *reject = 0; 259 260 /* See if we can deliver */ 261 success = new_state.cppr > priority && 262 new_state.mfrr > priority && 263 new_state.pending_pri > priority; 264 265 /* 266 * If we can, check for a rejection and perform the 267 * delivery 268 */ 269 if (success) { 270 *reject = new_state.xisr; 271 new_state.xisr = irq; 272 new_state.pending_pri = priority; 273 } else { 274 /* 275 * If we failed to deliver we set need_resend 276 * so a subsequent CPPR state change causes us 277 * to try a new delivery. 278 */ 279 new_state.need_resend = true; 280 } 281 282 } while (!icp_rm_try_update(icp, old_state, new_state)); 283 284 return success; 285 } 286 287 static void icp_rm_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp, 288 u32 new_irq, bool check_resend) 289 { 290 struct ics_irq_state *state; 291 struct kvmppc_ics *ics; 292 u32 reject; 293 u16 src; 294 295 /* 296 * This is used both for initial delivery of an interrupt and 297 * for subsequent rejection. 298 * 299 * Rejection can be racy vs. resends. We have evaluated the 300 * rejection in an atomic ICP transaction which is now complete, 301 * so potentially the ICP can already accept the interrupt again. 302 * 303 * So we need to retry the delivery. Essentially the reject path 304 * boils down to a failed delivery. Always. 305 * 306 * Now the interrupt could also have moved to a different target, 307 * thus we may need to re-do the ICP lookup as well 308 */ 309 310 again: 311 /* Get the ICS state and lock it */ 312 ics = kvmppc_xics_find_ics(xics, new_irq, &src); 313 if (!ics) { 314 /* Unsafe increment, but this does not need to be accurate */ 315 xics->err_noics++; 316 return; 317 } 318 state = &ics->irq_state[src]; 319 320 /* Get a lock on the ICS */ 321 arch_spin_lock(&ics->lock); 322 323 /* Get our server */ 324 if (!icp || state->server != icp->server_num) { 325 icp = kvmppc_xics_find_server(xics->kvm, state->server); 326 if (!icp) { 327 /* Unsafe increment again*/ 328 xics->err_noicp++; 329 goto out; 330 } 331 } 332 333 if (check_resend) 334 if (!state->resend) 335 goto out; 336 337 /* Clear the resend bit of that interrupt */ 338 state->resend = 0; 339 340 /* 341 * If masked, bail out 342 * 343 * Note: PAPR doesn't mention anything about masked pending 344 * when doing a resend, only when doing a delivery. 345 * 346 * However that would have the effect of losing a masked 347 * interrupt that was rejected and isn't consistent with 348 * the whole masked_pending business which is about not 349 * losing interrupts that occur while masked. 350 * 351 * I don't differentiate normal deliveries and resends, this 352 * implementation will differ from PAPR and not lose such 353 * interrupts. 354 */ 355 if (state->priority == MASKED) { 356 state->masked_pending = 1; 357 goto out; 358 } 359 360 /* 361 * Try the delivery, this will set the need_resend flag 362 * in the ICP as part of the atomic transaction if the 363 * delivery is not possible. 364 * 365 * Note that if successful, the new delivery might have itself 366 * rejected an interrupt that was "delivered" before we took the 367 * ics spin lock. 368 * 369 * In this case we do the whole sequence all over again for the 370 * new guy. We cannot assume that the rejected interrupt is less 371 * favored than the new one, and thus doesn't need to be delivered, 372 * because by the time we exit icp_rm_try_to_deliver() the target 373 * processor may well have already consumed & completed it, and thus 374 * the rejected interrupt might actually be already acceptable. 375 */ 376 if (icp_rm_try_to_deliver(icp, new_irq, state->priority, &reject)) { 377 /* 378 * Delivery was successful, did we reject somebody else ? 379 */ 380 if (reject && reject != XICS_IPI) { 381 arch_spin_unlock(&ics->lock); 382 icp->n_reject++; 383 new_irq = reject; 384 check_resend = 0; 385 goto again; 386 } 387 } else { 388 /* 389 * We failed to deliver the interrupt we need to set the 390 * resend map bit and mark the ICS state as needing a resend 391 */ 392 state->resend = 1; 393 394 /* 395 * Make sure when checking resend, we don't miss the resend 396 * if resend_map bit is seen and cleared. 397 */ 398 smp_wmb(); 399 set_bit(ics->icsid, icp->resend_map); 400 401 /* 402 * If the need_resend flag got cleared in the ICP some time 403 * between icp_rm_try_to_deliver() atomic update and now, then 404 * we know it might have missed the resend_map bit. So we 405 * retry 406 */ 407 smp_mb(); 408 if (!icp->state.need_resend) { 409 state->resend = 0; 410 arch_spin_unlock(&ics->lock); 411 check_resend = 0; 412 goto again; 413 } 414 } 415 out: 416 arch_spin_unlock(&ics->lock); 417 } 418 419 static void icp_rm_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp, 420 u8 new_cppr) 421 { 422 union kvmppc_icp_state old_state, new_state; 423 bool resend; 424 425 /* 426 * This handles several related states in one operation: 427 * 428 * ICP State: Down_CPPR 429 * 430 * Load CPPR with new value and if the XISR is 0 431 * then check for resends: 432 * 433 * ICP State: Resend 434 * 435 * If MFRR is more favored than CPPR, check for IPIs 436 * and notify ICS of a potential resend. This is done 437 * asynchronously (when used in real mode, we will have 438 * to exit here). 439 * 440 * We do not handle the complete Check_IPI as documented 441 * here. In the PAPR, this state will be used for both 442 * Set_MFRR and Down_CPPR. However, we know that we aren't 443 * changing the MFRR state here so we don't need to handle 444 * the case of an MFRR causing a reject of a pending irq, 445 * this will have been handled when the MFRR was set in the 446 * first place. 447 * 448 * Thus we don't have to handle rejects, only resends. 449 * 450 * When implementing real mode for HV KVM, resend will lead to 451 * a H_TOO_HARD return and the whole transaction will be handled 452 * in virtual mode. 453 */ 454 do { 455 old_state = new_state = READ_ONCE(icp->state); 456 457 /* Down_CPPR */ 458 new_state.cppr = new_cppr; 459 460 /* 461 * Cut down Resend / Check_IPI / IPI 462 * 463 * The logic is that we cannot have a pending interrupt 464 * trumped by an IPI at this point (see above), so we 465 * know that either the pending interrupt is already an 466 * IPI (in which case we don't care to override it) or 467 * it's either more favored than us or non existent 468 */ 469 if (new_state.mfrr < new_cppr && 470 new_state.mfrr <= new_state.pending_pri) { 471 new_state.pending_pri = new_state.mfrr; 472 new_state.xisr = XICS_IPI; 473 } 474 475 /* Latch/clear resend bit */ 476 resend = new_state.need_resend; 477 new_state.need_resend = 0; 478 479 } while (!icp_rm_try_update(icp, old_state, new_state)); 480 481 /* 482 * Now handle resend checks. Those are asynchronous to the ICP 483 * state update in HW (ie bus transactions) so we can handle them 484 * separately here as well. 485 */ 486 if (resend) { 487 icp->n_check_resend++; 488 icp_rm_check_resend(xics, icp); 489 } 490 } 491 492 493 unsigned long xics_rm_h_xirr(struct kvm_vcpu *vcpu) 494 { 495 union kvmppc_icp_state old_state, new_state; 496 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 497 struct kvmppc_icp *icp = vcpu->arch.icp; 498 u32 xirr; 499 500 if (!xics || !xics->real_mode) 501 return H_TOO_HARD; 502 503 /* First clear the interrupt */ 504 icp_rm_clr_vcpu_irq(icp->vcpu); 505 506 /* 507 * ICP State: Accept_Interrupt 508 * 509 * Return the pending interrupt (if any) along with the 510 * current CPPR, then clear the XISR & set CPPR to the 511 * pending priority 512 */ 513 do { 514 old_state = new_state = READ_ONCE(icp->state); 515 516 xirr = old_state.xisr | (((u32)old_state.cppr) << 24); 517 if (!old_state.xisr) 518 break; 519 new_state.cppr = new_state.pending_pri; 520 new_state.pending_pri = 0xff; 521 new_state.xisr = 0; 522 523 } while (!icp_rm_try_update(icp, old_state, new_state)); 524 525 /* Return the result in GPR4 */ 526 vcpu->arch.regs.gpr[4] = xirr; 527 528 return check_too_hard(xics, icp); 529 } 530 531 int xics_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server, 532 unsigned long mfrr) 533 { 534 union kvmppc_icp_state old_state, new_state; 535 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 536 struct kvmppc_icp *icp, *this_icp = vcpu->arch.icp; 537 u32 reject; 538 bool resend; 539 bool local; 540 541 if (!xics || !xics->real_mode) 542 return H_TOO_HARD; 543 544 local = this_icp->server_num == server; 545 if (local) 546 icp = this_icp; 547 else 548 icp = kvmppc_xics_find_server(vcpu->kvm, server); 549 if (!icp) 550 return H_PARAMETER; 551 552 /* 553 * ICP state: Set_MFRR 554 * 555 * If the CPPR is more favored than the new MFRR, then 556 * nothing needs to be done as there can be no XISR to 557 * reject. 558 * 559 * ICP state: Check_IPI 560 * 561 * If the CPPR is less favored, then we might be replacing 562 * an interrupt, and thus need to possibly reject it. 563 * 564 * ICP State: IPI 565 * 566 * Besides rejecting any pending interrupts, we also 567 * update XISR and pending_pri to mark IPI as pending. 568 * 569 * PAPR does not describe this state, but if the MFRR is being 570 * made less favored than its earlier value, there might be 571 * a previously-rejected interrupt needing to be resent. 572 * Ideally, we would want to resend only if 573 * prio(pending_interrupt) < mfrr && 574 * prio(pending_interrupt) < cppr 575 * where pending interrupt is the one that was rejected. But 576 * we don't have that state, so we simply trigger a resend 577 * whenever the MFRR is made less favored. 578 */ 579 do { 580 old_state = new_state = READ_ONCE(icp->state); 581 582 /* Set_MFRR */ 583 new_state.mfrr = mfrr; 584 585 /* Check_IPI */ 586 reject = 0; 587 resend = false; 588 if (mfrr < new_state.cppr) { 589 /* Reject a pending interrupt if not an IPI */ 590 if (mfrr <= new_state.pending_pri) { 591 reject = new_state.xisr; 592 new_state.pending_pri = mfrr; 593 new_state.xisr = XICS_IPI; 594 } 595 } 596 597 if (mfrr > old_state.mfrr) { 598 resend = new_state.need_resend; 599 new_state.need_resend = 0; 600 } 601 } while (!icp_rm_try_update(icp, old_state, new_state)); 602 603 /* Handle reject in real mode */ 604 if (reject && reject != XICS_IPI) { 605 this_icp->n_reject++; 606 icp_rm_deliver_irq(xics, icp, reject, false); 607 } 608 609 /* Handle resends in real mode */ 610 if (resend) { 611 this_icp->n_check_resend++; 612 icp_rm_check_resend(xics, icp); 613 } 614 615 return check_too_hard(xics, this_icp); 616 } 617 618 int xics_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr) 619 { 620 union kvmppc_icp_state old_state, new_state; 621 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 622 struct kvmppc_icp *icp = vcpu->arch.icp; 623 u32 reject; 624 625 if (!xics || !xics->real_mode) 626 return H_TOO_HARD; 627 628 /* 629 * ICP State: Set_CPPR 630 * 631 * We can safely compare the new value with the current 632 * value outside of the transaction as the CPPR is only 633 * ever changed by the processor on itself 634 */ 635 if (cppr > icp->state.cppr) { 636 icp_rm_down_cppr(xics, icp, cppr); 637 goto bail; 638 } else if (cppr == icp->state.cppr) 639 return H_SUCCESS; 640 641 /* 642 * ICP State: Up_CPPR 643 * 644 * The processor is raising its priority, this can result 645 * in a rejection of a pending interrupt: 646 * 647 * ICP State: Reject_Current 648 * 649 * We can remove EE from the current processor, the update 650 * transaction will set it again if needed 651 */ 652 icp_rm_clr_vcpu_irq(icp->vcpu); 653 654 do { 655 old_state = new_state = READ_ONCE(icp->state); 656 657 reject = 0; 658 new_state.cppr = cppr; 659 660 if (cppr <= new_state.pending_pri) { 661 reject = new_state.xisr; 662 new_state.xisr = 0; 663 new_state.pending_pri = 0xff; 664 } 665 666 } while (!icp_rm_try_update(icp, old_state, new_state)); 667 668 /* 669 * Check for rejects. They are handled by doing a new delivery 670 * attempt (see comments in icp_rm_deliver_irq). 671 */ 672 if (reject && reject != XICS_IPI) { 673 icp->n_reject++; 674 icp_rm_deliver_irq(xics, icp, reject, false); 675 } 676 bail: 677 return check_too_hard(xics, icp); 678 } 679 680 static int ics_rm_eoi(struct kvm_vcpu *vcpu, u32 irq) 681 { 682 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 683 struct kvmppc_icp *icp = vcpu->arch.icp; 684 struct kvmppc_ics *ics; 685 struct ics_irq_state *state; 686 u16 src; 687 u32 pq_old, pq_new; 688 689 /* 690 * ICS EOI handling: For LSI, if P bit is still set, we need to 691 * resend it. 692 * 693 * For MSI, we move Q bit into P (and clear Q). If it is set, 694 * resend it. 695 */ 696 697 ics = kvmppc_xics_find_ics(xics, irq, &src); 698 if (!ics) 699 goto bail; 700 701 state = &ics->irq_state[src]; 702 703 if (state->lsi) 704 pq_new = state->pq_state; 705 else 706 do { 707 pq_old = state->pq_state; 708 pq_new = pq_old >> 1; 709 } while (cmpxchg(&state->pq_state, pq_old, pq_new) != pq_old); 710 711 if (pq_new & PQ_PRESENTED) 712 icp_rm_deliver_irq(xics, NULL, irq, false); 713 714 if (!hlist_empty(&vcpu->kvm->irq_ack_notifier_list)) { 715 icp->rm_action |= XICS_RM_NOTIFY_EOI; 716 icp->rm_eoied_irq = irq; 717 } 718 719 if (state->host_irq) { 720 ++vcpu->stat.pthru_all; 721 if (state->intr_cpu != -1) { 722 int pcpu = raw_smp_processor_id(); 723 724 pcpu = cpu_first_thread_sibling(pcpu); 725 ++vcpu->stat.pthru_host; 726 if (state->intr_cpu != pcpu) { 727 ++vcpu->stat.pthru_bad_aff; 728 xics_opal_set_server(state->host_irq, pcpu); 729 } 730 state->intr_cpu = -1; 731 } 732 } 733 734 bail: 735 return check_too_hard(xics, icp); 736 } 737 738 int xics_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr) 739 { 740 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 741 struct kvmppc_icp *icp = vcpu->arch.icp; 742 u32 irq = xirr & 0x00ffffff; 743 744 if (!xics || !xics->real_mode) 745 return H_TOO_HARD; 746 747 /* 748 * ICP State: EOI 749 * 750 * Note: If EOI is incorrectly used by SW to lower the CPPR 751 * value (ie more favored), we do not check for rejection of 752 * a pending interrupt, this is a SW error and PAPR specifies 753 * that we don't have to deal with it. 754 * 755 * The sending of an EOI to the ICS is handled after the 756 * CPPR update 757 * 758 * ICP State: Down_CPPR which we handle 759 * in a separate function as it's shared with H_CPPR. 760 */ 761 icp_rm_down_cppr(xics, icp, xirr >> 24); 762 763 /* IPIs have no EOI */ 764 if (irq == XICS_IPI) 765 return check_too_hard(xics, icp); 766 767 return ics_rm_eoi(vcpu, irq); 768 } 769 770 unsigned long eoi_rc; 771 772 static void icp_eoi(struct irq_chip *c, u32 hwirq, __be32 xirr, bool *again) 773 { 774 void __iomem *xics_phys; 775 int64_t rc; 776 777 if (kvmhv_on_pseries()) { 778 unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; 779 780 iosync(); 781 plpar_hcall_raw(H_EOI, retbuf, hwirq); 782 return; 783 } 784 785 rc = pnv_opal_pci_msi_eoi(c, hwirq); 786 787 if (rc) 788 eoi_rc = rc; 789 790 iosync(); 791 792 /* EOI it */ 793 xics_phys = local_paca->kvm_hstate.xics_phys; 794 if (xics_phys) { 795 __raw_rm_writel(xirr, xics_phys + XICS_XIRR); 796 } else { 797 rc = opal_int_eoi(be32_to_cpu(xirr)); 798 *again = rc > 0; 799 } 800 } 801 802 static int xics_opal_set_server(unsigned int hw_irq, int server_cpu) 803 { 804 unsigned int mangle_cpu = get_hard_smp_processor_id(server_cpu) << 2; 805 806 return opal_set_xive(hw_irq, mangle_cpu, DEFAULT_PRIORITY); 807 } 808 809 /* 810 * Increment a per-CPU 32-bit unsigned integer variable. 811 * Safe to call in real-mode. Handles vmalloc'ed addresses 812 * 813 * ToDo: Make this work for any integral type 814 */ 815 816 static inline void this_cpu_inc_rm(unsigned int __percpu *addr) 817 { 818 unsigned long l; 819 unsigned int *raddr; 820 int cpu = smp_processor_id(); 821 822 raddr = per_cpu_ptr(addr, cpu); 823 l = (unsigned long)raddr; 824 825 if (REGION_ID(l) == VMALLOC_REGION_ID) { 826 l = vmalloc_to_phys(raddr); 827 raddr = (unsigned int *)l; 828 } 829 ++*raddr; 830 } 831 832 /* 833 * We don't try to update the flags in the irq_desc 'istate' field in 834 * here as would happen in the normal IRQ handling path for several reasons: 835 * - state flags represent internal IRQ state and are not expected to be 836 * updated outside the IRQ subsystem 837 * - more importantly, these are useful for edge triggered interrupts, 838 * IRQ probing, etc., but we are only handling MSI/MSIx interrupts here 839 * and these states shouldn't apply to us. 840 * 841 * However, we do update irq_stats - we somewhat duplicate the code in 842 * kstat_incr_irqs_this_cpu() for this since this function is defined 843 * in irq/internal.h which we don't want to include here. 844 * The only difference is that desc->kstat_irqs is an allocated per CPU 845 * variable and could have been vmalloc'ed, so we can't directly 846 * call __this_cpu_inc() on it. The kstat structure is a static 847 * per CPU variable and it should be accessible by real-mode KVM. 848 * 849 */ 850 static void kvmppc_rm_handle_irq_desc(struct irq_desc *desc) 851 { 852 this_cpu_inc_rm(desc->kstat_irqs); 853 __this_cpu_inc(kstat.irqs_sum); 854 } 855 856 long kvmppc_deliver_irq_passthru(struct kvm_vcpu *vcpu, 857 __be32 xirr, 858 struct kvmppc_irq_map *irq_map, 859 struct kvmppc_passthru_irqmap *pimap, 860 bool *again) 861 { 862 struct kvmppc_xics *xics; 863 struct kvmppc_icp *icp; 864 struct kvmppc_ics *ics; 865 struct ics_irq_state *state; 866 u32 irq; 867 u16 src; 868 u32 pq_old, pq_new; 869 870 irq = irq_map->v_hwirq; 871 xics = vcpu->kvm->arch.xics; 872 icp = vcpu->arch.icp; 873 874 kvmppc_rm_handle_irq_desc(irq_map->desc); 875 876 ics = kvmppc_xics_find_ics(xics, irq, &src); 877 if (!ics) 878 return 2; 879 880 state = &ics->irq_state[src]; 881 882 /* only MSIs register bypass producers, so it must be MSI here */ 883 do { 884 pq_old = state->pq_state; 885 pq_new = ((pq_old << 1) & 3) | PQ_PRESENTED; 886 } while (cmpxchg(&state->pq_state, pq_old, pq_new) != pq_old); 887 888 /* Test P=1, Q=0, this is the only case where we present */ 889 if (pq_new == PQ_PRESENTED) 890 icp_rm_deliver_irq(xics, icp, irq, false); 891 892 /* EOI the interrupt */ 893 icp_eoi(irq_desc_get_chip(irq_map->desc), irq_map->r_hwirq, xirr, 894 again); 895 896 if (check_too_hard(xics, icp) == H_TOO_HARD) 897 return 2; 898 else 899 return -2; 900 } 901 902 /* --- Non-real mode XICS-related built-in routines --- */ 903 904 /** 905 * Host Operations poked by RM KVM 906 */ 907 static void rm_host_ipi_action(int action, void *data) 908 { 909 switch (action) { 910 case XICS_RM_KICK_VCPU: 911 kvmppc_host_rm_ops_hv->vcpu_kick(data); 912 break; 913 default: 914 WARN(1, "Unexpected rm_action=%d data=%p\n", action, data); 915 break; 916 } 917 918 } 919 920 void kvmppc_xics_ipi_action(void) 921 { 922 int core; 923 unsigned int cpu = smp_processor_id(); 924 struct kvmppc_host_rm_core *rm_corep; 925 926 core = cpu >> threads_shift; 927 rm_corep = &kvmppc_host_rm_ops_hv->rm_core[core]; 928 929 if (rm_corep->rm_data) { 930 rm_host_ipi_action(rm_corep->rm_state.rm_action, 931 rm_corep->rm_data); 932 /* Order these stores against the real mode KVM */ 933 rm_corep->rm_data = NULL; 934 smp_wmb(); 935 rm_corep->rm_state.rm_action = 0; 936 } 937 } 938