1 /* 2 * Copyright 2012 Michael Ellerman, IBM Corporation. 3 * Copyright 2012 Benjamin Herrenschmidt, IBM Corporation 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License, version 2, as 7 * published by the Free Software Foundation. 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/kvm_host.h> 12 #include <linux/err.h> 13 14 #include <asm/kvm_book3s.h> 15 #include <asm/kvm_ppc.h> 16 #include <asm/hvcall.h> 17 #include <asm/xics.h> 18 #include <asm/debug.h> 19 #include <asm/synch.h> 20 #include <asm/ppc-opcode.h> 21 22 #include "book3s_xics.h" 23 24 #define DEBUG_PASSUP 25 26 static void icp_rm_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp, 27 u32 new_irq); 28 29 /* -- ICS routines -- */ 30 static void ics_rm_check_resend(struct kvmppc_xics *xics, 31 struct kvmppc_ics *ics, struct kvmppc_icp *icp) 32 { 33 int i; 34 35 arch_spin_lock(&ics->lock); 36 37 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 38 struct ics_irq_state *state = &ics->irq_state[i]; 39 40 if (!state->resend) 41 continue; 42 43 arch_spin_unlock(&ics->lock); 44 icp_rm_deliver_irq(xics, icp, state->number); 45 arch_spin_lock(&ics->lock); 46 } 47 48 arch_spin_unlock(&ics->lock); 49 } 50 51 /* -- ICP routines -- */ 52 53 static void icp_rm_set_vcpu_irq(struct kvm_vcpu *vcpu, 54 struct kvm_vcpu *this_vcpu) 55 { 56 struct kvmppc_icp *this_icp = this_vcpu->arch.icp; 57 int cpu; 58 59 /* Mark the target VCPU as having an interrupt pending */ 60 vcpu->stat.queue_intr++; 61 set_bit(BOOK3S_IRQPRIO_EXTERNAL_LEVEL, &vcpu->arch.pending_exceptions); 62 63 /* Kick self ? Just set MER and return */ 64 if (vcpu == this_vcpu) { 65 mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) | LPCR_MER); 66 return; 67 } 68 69 /* Check if the core is loaded, if not, too hard */ 70 cpu = vcpu->cpu; 71 if (cpu < 0 || cpu >= nr_cpu_ids) { 72 this_icp->rm_action |= XICS_RM_KICK_VCPU; 73 this_icp->rm_kick_target = vcpu; 74 return; 75 } 76 /* In SMT cpu will always point to thread 0, we adjust it */ 77 cpu += vcpu->arch.ptid; 78 79 smp_mb(); 80 kvmhv_rm_send_ipi(cpu); 81 } 82 83 static void icp_rm_clr_vcpu_irq(struct kvm_vcpu *vcpu) 84 { 85 /* Note: Only called on self ! */ 86 clear_bit(BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 87 &vcpu->arch.pending_exceptions); 88 mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) & ~LPCR_MER); 89 } 90 91 static inline bool icp_rm_try_update(struct kvmppc_icp *icp, 92 union kvmppc_icp_state old, 93 union kvmppc_icp_state new) 94 { 95 struct kvm_vcpu *this_vcpu = local_paca->kvm_hstate.kvm_vcpu; 96 bool success; 97 98 /* Calculate new output value */ 99 new.out_ee = (new.xisr && (new.pending_pri < new.cppr)); 100 101 /* Attempt atomic update */ 102 success = cmpxchg64(&icp->state.raw, old.raw, new.raw) == old.raw; 103 if (!success) 104 goto bail; 105 106 /* 107 * Check for output state update 108 * 109 * Note that this is racy since another processor could be updating 110 * the state already. This is why we never clear the interrupt output 111 * here, we only ever set it. The clear only happens prior to doing 112 * an update and only by the processor itself. Currently we do it 113 * in Accept (H_XIRR) and Up_Cppr (H_XPPR). 114 * 115 * We also do not try to figure out whether the EE state has changed, 116 * we unconditionally set it if the new state calls for it. The reason 117 * for that is that we opportunistically remove the pending interrupt 118 * flag when raising CPPR, so we need to set it back here if an 119 * interrupt is still pending. 120 */ 121 if (new.out_ee) 122 icp_rm_set_vcpu_irq(icp->vcpu, this_vcpu); 123 124 /* Expose the state change for debug purposes */ 125 this_vcpu->arch.icp->rm_dbgstate = new; 126 this_vcpu->arch.icp->rm_dbgtgt = icp->vcpu; 127 128 bail: 129 return success; 130 } 131 132 static inline int check_too_hard(struct kvmppc_xics *xics, 133 struct kvmppc_icp *icp) 134 { 135 return (xics->real_mode_dbg || icp->rm_action) ? H_TOO_HARD : H_SUCCESS; 136 } 137 138 static void icp_rm_check_resend(struct kvmppc_xics *xics, 139 struct kvmppc_icp *icp) 140 { 141 u32 icsid; 142 143 /* Order this load with the test for need_resend in the caller */ 144 smp_rmb(); 145 for_each_set_bit(icsid, icp->resend_map, xics->max_icsid + 1) { 146 struct kvmppc_ics *ics = xics->ics[icsid]; 147 148 if (!test_and_clear_bit(icsid, icp->resend_map)) 149 continue; 150 if (!ics) 151 continue; 152 ics_rm_check_resend(xics, ics, icp); 153 } 154 } 155 156 static bool icp_rm_try_to_deliver(struct kvmppc_icp *icp, u32 irq, u8 priority, 157 u32 *reject) 158 { 159 union kvmppc_icp_state old_state, new_state; 160 bool success; 161 162 do { 163 old_state = new_state = READ_ONCE(icp->state); 164 165 *reject = 0; 166 167 /* See if we can deliver */ 168 success = new_state.cppr > priority && 169 new_state.mfrr > priority && 170 new_state.pending_pri > priority; 171 172 /* 173 * If we can, check for a rejection and perform the 174 * delivery 175 */ 176 if (success) { 177 *reject = new_state.xisr; 178 new_state.xisr = irq; 179 new_state.pending_pri = priority; 180 } else { 181 /* 182 * If we failed to deliver we set need_resend 183 * so a subsequent CPPR state change causes us 184 * to try a new delivery. 185 */ 186 new_state.need_resend = true; 187 } 188 189 } while (!icp_rm_try_update(icp, old_state, new_state)); 190 191 return success; 192 } 193 194 static void icp_rm_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp, 195 u32 new_irq) 196 { 197 struct ics_irq_state *state; 198 struct kvmppc_ics *ics; 199 u32 reject; 200 u16 src; 201 202 /* 203 * This is used both for initial delivery of an interrupt and 204 * for subsequent rejection. 205 * 206 * Rejection can be racy vs. resends. We have evaluated the 207 * rejection in an atomic ICP transaction which is now complete, 208 * so potentially the ICP can already accept the interrupt again. 209 * 210 * So we need to retry the delivery. Essentially the reject path 211 * boils down to a failed delivery. Always. 212 * 213 * Now the interrupt could also have moved to a different target, 214 * thus we may need to re-do the ICP lookup as well 215 */ 216 217 again: 218 /* Get the ICS state and lock it */ 219 ics = kvmppc_xics_find_ics(xics, new_irq, &src); 220 if (!ics) { 221 /* Unsafe increment, but this does not need to be accurate */ 222 xics->err_noics++; 223 return; 224 } 225 state = &ics->irq_state[src]; 226 227 /* Get a lock on the ICS */ 228 arch_spin_lock(&ics->lock); 229 230 /* Get our server */ 231 if (!icp || state->server != icp->server_num) { 232 icp = kvmppc_xics_find_server(xics->kvm, state->server); 233 if (!icp) { 234 /* Unsafe increment again*/ 235 xics->err_noicp++; 236 goto out; 237 } 238 } 239 240 /* Clear the resend bit of that interrupt */ 241 state->resend = 0; 242 243 /* 244 * If masked, bail out 245 * 246 * Note: PAPR doesn't mention anything about masked pending 247 * when doing a resend, only when doing a delivery. 248 * 249 * However that would have the effect of losing a masked 250 * interrupt that was rejected and isn't consistent with 251 * the whole masked_pending business which is about not 252 * losing interrupts that occur while masked. 253 * 254 * I don't differentiate normal deliveries and resends, this 255 * implementation will differ from PAPR and not lose such 256 * interrupts. 257 */ 258 if (state->priority == MASKED) { 259 state->masked_pending = 1; 260 goto out; 261 } 262 263 /* 264 * Try the delivery, this will set the need_resend flag 265 * in the ICP as part of the atomic transaction if the 266 * delivery is not possible. 267 * 268 * Note that if successful, the new delivery might have itself 269 * rejected an interrupt that was "delivered" before we took the 270 * ics spin lock. 271 * 272 * In this case we do the whole sequence all over again for the 273 * new guy. We cannot assume that the rejected interrupt is less 274 * favored than the new one, and thus doesn't need to be delivered, 275 * because by the time we exit icp_rm_try_to_deliver() the target 276 * processor may well have already consumed & completed it, and thus 277 * the rejected interrupt might actually be already acceptable. 278 */ 279 if (icp_rm_try_to_deliver(icp, new_irq, state->priority, &reject)) { 280 /* 281 * Delivery was successful, did we reject somebody else ? 282 */ 283 if (reject && reject != XICS_IPI) { 284 arch_spin_unlock(&ics->lock); 285 new_irq = reject; 286 goto again; 287 } 288 } else { 289 /* 290 * We failed to deliver the interrupt we need to set the 291 * resend map bit and mark the ICS state as needing a resend 292 */ 293 set_bit(ics->icsid, icp->resend_map); 294 state->resend = 1; 295 296 /* 297 * If the need_resend flag got cleared in the ICP some time 298 * between icp_rm_try_to_deliver() atomic update and now, then 299 * we know it might have missed the resend_map bit. So we 300 * retry 301 */ 302 smp_mb(); 303 if (!icp->state.need_resend) { 304 arch_spin_unlock(&ics->lock); 305 goto again; 306 } 307 } 308 out: 309 arch_spin_unlock(&ics->lock); 310 } 311 312 static void icp_rm_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp, 313 u8 new_cppr) 314 { 315 union kvmppc_icp_state old_state, new_state; 316 bool resend; 317 318 /* 319 * This handles several related states in one operation: 320 * 321 * ICP State: Down_CPPR 322 * 323 * Load CPPR with new value and if the XISR is 0 324 * then check for resends: 325 * 326 * ICP State: Resend 327 * 328 * If MFRR is more favored than CPPR, check for IPIs 329 * and notify ICS of a potential resend. This is done 330 * asynchronously (when used in real mode, we will have 331 * to exit here). 332 * 333 * We do not handle the complete Check_IPI as documented 334 * here. In the PAPR, this state will be used for both 335 * Set_MFRR and Down_CPPR. However, we know that we aren't 336 * changing the MFRR state here so we don't need to handle 337 * the case of an MFRR causing a reject of a pending irq, 338 * this will have been handled when the MFRR was set in the 339 * first place. 340 * 341 * Thus we don't have to handle rejects, only resends. 342 * 343 * When implementing real mode for HV KVM, resend will lead to 344 * a H_TOO_HARD return and the whole transaction will be handled 345 * in virtual mode. 346 */ 347 do { 348 old_state = new_state = READ_ONCE(icp->state); 349 350 /* Down_CPPR */ 351 new_state.cppr = new_cppr; 352 353 /* 354 * Cut down Resend / Check_IPI / IPI 355 * 356 * The logic is that we cannot have a pending interrupt 357 * trumped by an IPI at this point (see above), so we 358 * know that either the pending interrupt is already an 359 * IPI (in which case we don't care to override it) or 360 * it's either more favored than us or non existent 361 */ 362 if (new_state.mfrr < new_cppr && 363 new_state.mfrr <= new_state.pending_pri) { 364 new_state.pending_pri = new_state.mfrr; 365 new_state.xisr = XICS_IPI; 366 } 367 368 /* Latch/clear resend bit */ 369 resend = new_state.need_resend; 370 new_state.need_resend = 0; 371 372 } while (!icp_rm_try_update(icp, old_state, new_state)); 373 374 /* 375 * Now handle resend checks. Those are asynchronous to the ICP 376 * state update in HW (ie bus transactions) so we can handle them 377 * separately here as well. 378 */ 379 if (resend) { 380 icp->n_check_resend++; 381 icp_rm_check_resend(xics, icp); 382 } 383 } 384 385 386 unsigned long kvmppc_rm_h_xirr(struct kvm_vcpu *vcpu) 387 { 388 union kvmppc_icp_state old_state, new_state; 389 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 390 struct kvmppc_icp *icp = vcpu->arch.icp; 391 u32 xirr; 392 393 if (!xics || !xics->real_mode) 394 return H_TOO_HARD; 395 396 /* First clear the interrupt */ 397 icp_rm_clr_vcpu_irq(icp->vcpu); 398 399 /* 400 * ICP State: Accept_Interrupt 401 * 402 * Return the pending interrupt (if any) along with the 403 * current CPPR, then clear the XISR & set CPPR to the 404 * pending priority 405 */ 406 do { 407 old_state = new_state = READ_ONCE(icp->state); 408 409 xirr = old_state.xisr | (((u32)old_state.cppr) << 24); 410 if (!old_state.xisr) 411 break; 412 new_state.cppr = new_state.pending_pri; 413 new_state.pending_pri = 0xff; 414 new_state.xisr = 0; 415 416 } while (!icp_rm_try_update(icp, old_state, new_state)); 417 418 /* Return the result in GPR4 */ 419 vcpu->arch.gpr[4] = xirr; 420 421 return check_too_hard(xics, icp); 422 } 423 424 int kvmppc_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server, 425 unsigned long mfrr) 426 { 427 union kvmppc_icp_state old_state, new_state; 428 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 429 struct kvmppc_icp *icp, *this_icp = vcpu->arch.icp; 430 u32 reject; 431 bool resend; 432 bool local; 433 434 if (!xics || !xics->real_mode) 435 return H_TOO_HARD; 436 437 local = this_icp->server_num == server; 438 if (local) 439 icp = this_icp; 440 else 441 icp = kvmppc_xics_find_server(vcpu->kvm, server); 442 if (!icp) 443 return H_PARAMETER; 444 445 /* 446 * ICP state: Set_MFRR 447 * 448 * If the CPPR is more favored than the new MFRR, then 449 * nothing needs to be done as there can be no XISR to 450 * reject. 451 * 452 * ICP state: Check_IPI 453 * 454 * If the CPPR is less favored, then we might be replacing 455 * an interrupt, and thus need to possibly reject it. 456 * 457 * ICP State: IPI 458 * 459 * Besides rejecting any pending interrupts, we also 460 * update XISR and pending_pri to mark IPI as pending. 461 * 462 * PAPR does not describe this state, but if the MFRR is being 463 * made less favored than its earlier value, there might be 464 * a previously-rejected interrupt needing to be resent. 465 * Ideally, we would want to resend only if 466 * prio(pending_interrupt) < mfrr && 467 * prio(pending_interrupt) < cppr 468 * where pending interrupt is the one that was rejected. But 469 * we don't have that state, so we simply trigger a resend 470 * whenever the MFRR is made less favored. 471 */ 472 do { 473 old_state = new_state = READ_ONCE(icp->state); 474 475 /* Set_MFRR */ 476 new_state.mfrr = mfrr; 477 478 /* Check_IPI */ 479 reject = 0; 480 resend = false; 481 if (mfrr < new_state.cppr) { 482 /* Reject a pending interrupt if not an IPI */ 483 if (mfrr <= new_state.pending_pri) { 484 reject = new_state.xisr; 485 new_state.pending_pri = mfrr; 486 new_state.xisr = XICS_IPI; 487 } 488 } 489 490 if (mfrr > old_state.mfrr) { 491 resend = new_state.need_resend; 492 new_state.need_resend = 0; 493 } 494 } while (!icp_rm_try_update(icp, old_state, new_state)); 495 496 /* Handle reject in real mode */ 497 if (reject && reject != XICS_IPI) { 498 this_icp->n_reject++; 499 icp_rm_deliver_irq(xics, icp, reject); 500 } 501 502 /* Handle resends in real mode */ 503 if (resend) { 504 this_icp->n_check_resend++; 505 icp_rm_check_resend(xics, icp); 506 } 507 508 return check_too_hard(xics, this_icp); 509 } 510 511 int kvmppc_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr) 512 { 513 union kvmppc_icp_state old_state, new_state; 514 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 515 struct kvmppc_icp *icp = vcpu->arch.icp; 516 u32 reject; 517 518 if (!xics || !xics->real_mode) 519 return H_TOO_HARD; 520 521 /* 522 * ICP State: Set_CPPR 523 * 524 * We can safely compare the new value with the current 525 * value outside of the transaction as the CPPR is only 526 * ever changed by the processor on itself 527 */ 528 if (cppr > icp->state.cppr) { 529 icp_rm_down_cppr(xics, icp, cppr); 530 goto bail; 531 } else if (cppr == icp->state.cppr) 532 return H_SUCCESS; 533 534 /* 535 * ICP State: Up_CPPR 536 * 537 * The processor is raising its priority, this can result 538 * in a rejection of a pending interrupt: 539 * 540 * ICP State: Reject_Current 541 * 542 * We can remove EE from the current processor, the update 543 * transaction will set it again if needed 544 */ 545 icp_rm_clr_vcpu_irq(icp->vcpu); 546 547 do { 548 old_state = new_state = READ_ONCE(icp->state); 549 550 reject = 0; 551 new_state.cppr = cppr; 552 553 if (cppr <= new_state.pending_pri) { 554 reject = new_state.xisr; 555 new_state.xisr = 0; 556 new_state.pending_pri = 0xff; 557 } 558 559 } while (!icp_rm_try_update(icp, old_state, new_state)); 560 561 /* 562 * Check for rejects. They are handled by doing a new delivery 563 * attempt (see comments in icp_rm_deliver_irq). 564 */ 565 if (reject && reject != XICS_IPI) { 566 icp->n_reject++; 567 icp_rm_deliver_irq(xics, icp, reject); 568 } 569 bail: 570 return check_too_hard(xics, icp); 571 } 572 573 int kvmppc_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr) 574 { 575 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 576 struct kvmppc_icp *icp = vcpu->arch.icp; 577 struct kvmppc_ics *ics; 578 struct ics_irq_state *state; 579 u32 irq = xirr & 0x00ffffff; 580 u16 src; 581 582 if (!xics || !xics->real_mode) 583 return H_TOO_HARD; 584 585 /* 586 * ICP State: EOI 587 * 588 * Note: If EOI is incorrectly used by SW to lower the CPPR 589 * value (ie more favored), we do not check for rejection of 590 * a pending interrupt, this is a SW error and PAPR sepcifies 591 * that we don't have to deal with it. 592 * 593 * The sending of an EOI to the ICS is handled after the 594 * CPPR update 595 * 596 * ICP State: Down_CPPR which we handle 597 * in a separate function as it's shared with H_CPPR. 598 */ 599 icp_rm_down_cppr(xics, icp, xirr >> 24); 600 601 /* IPIs have no EOI */ 602 if (irq == XICS_IPI) 603 goto bail; 604 /* 605 * EOI handling: If the interrupt is still asserted, we need to 606 * resend it. We can take a lockless "peek" at the ICS state here. 607 * 608 * "Message" interrupts will never have "asserted" set 609 */ 610 ics = kvmppc_xics_find_ics(xics, irq, &src); 611 if (!ics) 612 goto bail; 613 state = &ics->irq_state[src]; 614 615 /* Still asserted, resend it */ 616 if (state->asserted) { 617 icp->n_reject++; 618 icp_rm_deliver_irq(xics, icp, irq); 619 } 620 621 if (!hlist_empty(&vcpu->kvm->irq_ack_notifier_list)) { 622 icp->rm_action |= XICS_RM_NOTIFY_EOI; 623 icp->rm_eoied_irq = irq; 624 } 625 bail: 626 return check_too_hard(xics, icp); 627 } 628