1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright 2016,2017 IBM Corporation. 4 */ 5 6 #define pr_fmt(fmt) "xive: " fmt 7 8 #include <linux/types.h> 9 #include <linux/threads.h> 10 #include <linux/kernel.h> 11 #include <linux/irq.h> 12 #include <linux/debugfs.h> 13 #include <linux/smp.h> 14 #include <linux/interrupt.h> 15 #include <linux/seq_file.h> 16 #include <linux/init.h> 17 #include <linux/cpu.h> 18 #include <linux/of.h> 19 #include <linux/slab.h> 20 #include <linux/spinlock.h> 21 #include <linux/msi.h> 22 23 #include <asm/prom.h> 24 #include <asm/io.h> 25 #include <asm/smp.h> 26 #include <asm/machdep.h> 27 #include <asm/irq.h> 28 #include <asm/errno.h> 29 #include <asm/xive.h> 30 #include <asm/xive-regs.h> 31 #include <asm/xmon.h> 32 33 #include "xive-internal.h" 34 35 #undef DEBUG_FLUSH 36 #undef DEBUG_ALL 37 38 #ifdef DEBUG_ALL 39 #define DBG_VERBOSE(fmt, ...) pr_devel("cpu %d - " fmt, \ 40 smp_processor_id(), ## __VA_ARGS__) 41 #else 42 #define DBG_VERBOSE(fmt...) do { } while(0) 43 #endif 44 45 bool __xive_enabled; 46 EXPORT_SYMBOL_GPL(__xive_enabled); 47 bool xive_cmdline_disabled; 48 49 /* We use only one priority for now */ 50 static u8 xive_irq_priority; 51 52 /* TIMA exported to KVM */ 53 void __iomem *xive_tima; 54 EXPORT_SYMBOL_GPL(xive_tima); 55 u32 xive_tima_offset; 56 57 /* Backend ops */ 58 static const struct xive_ops *xive_ops; 59 60 /* Our global interrupt domain */ 61 static struct irq_domain *xive_irq_domain; 62 63 #ifdef CONFIG_SMP 64 /* The IPIs all use the same logical irq number */ 65 static u32 xive_ipi_irq; 66 #endif 67 68 /* Xive state for each CPU */ 69 static DEFINE_PER_CPU(struct xive_cpu *, xive_cpu); 70 71 /* 72 * A "disabled" interrupt should never fire, to catch problems 73 * we set its logical number to this 74 */ 75 #define XIVE_BAD_IRQ 0x7fffffff 76 #define XIVE_MAX_IRQ (XIVE_BAD_IRQ - 1) 77 78 /* An invalid CPU target */ 79 #define XIVE_INVALID_TARGET (-1) 80 81 /* 82 * Read the next entry in a queue, return its content if it's valid 83 * or 0 if there is no new entry. 84 * 85 * The queue pointer is moved forward unless "just_peek" is set 86 */ 87 static u32 xive_read_eq(struct xive_q *q, bool just_peek) 88 { 89 u32 cur; 90 91 if (!q->qpage) 92 return 0; 93 cur = be32_to_cpup(q->qpage + q->idx); 94 95 /* Check valid bit (31) vs current toggle polarity */ 96 if ((cur >> 31) == q->toggle) 97 return 0; 98 99 /* If consuming from the queue ... */ 100 if (!just_peek) { 101 /* Next entry */ 102 q->idx = (q->idx + 1) & q->msk; 103 104 /* Wrap around: flip valid toggle */ 105 if (q->idx == 0) 106 q->toggle ^= 1; 107 } 108 /* Mask out the valid bit (31) */ 109 return cur & 0x7fffffff; 110 } 111 112 /* 113 * Scans all the queue that may have interrupts in them 114 * (based on "pending_prio") in priority order until an 115 * interrupt is found or all the queues are empty. 116 * 117 * Then updates the CPPR (Current Processor Priority 118 * Register) based on the most favored interrupt found 119 * (0xff if none) and return what was found (0 if none). 120 * 121 * If just_peek is set, return the most favored pending 122 * interrupt if any but don't update the queue pointers. 123 * 124 * Note: This function can operate generically on any number 125 * of queues (up to 8). The current implementation of the XIVE 126 * driver only uses a single queue however. 127 * 128 * Note2: This will also "flush" "the pending_count" of a queue 129 * into the "count" when that queue is observed to be empty. 130 * This is used to keep track of the amount of interrupts 131 * targetting a queue. When an interrupt is moved away from 132 * a queue, we only decrement that queue count once the queue 133 * has been observed empty to avoid races. 134 */ 135 static u32 xive_scan_interrupts(struct xive_cpu *xc, bool just_peek) 136 { 137 u32 irq = 0; 138 u8 prio; 139 140 /* Find highest pending priority */ 141 while (xc->pending_prio != 0) { 142 struct xive_q *q; 143 144 prio = ffs(xc->pending_prio) - 1; 145 DBG_VERBOSE("scan_irq: trying prio %d\n", prio); 146 147 /* Try to fetch */ 148 irq = xive_read_eq(&xc->queue[prio], just_peek); 149 150 /* Found something ? That's it */ 151 if (irq) 152 break; 153 154 /* Clear pending bits */ 155 xc->pending_prio &= ~(1 << prio); 156 157 /* 158 * Check if the queue count needs adjusting due to 159 * interrupts being moved away. See description of 160 * xive_dec_target_count() 161 */ 162 q = &xc->queue[prio]; 163 if (atomic_read(&q->pending_count)) { 164 int p = atomic_xchg(&q->pending_count, 0); 165 if (p) { 166 WARN_ON(p > atomic_read(&q->count)); 167 atomic_sub(p, &q->count); 168 } 169 } 170 } 171 172 /* If nothing was found, set CPPR to 0xff */ 173 if (irq == 0) 174 prio = 0xff; 175 176 /* Update HW CPPR to match if necessary */ 177 if (prio != xc->cppr) { 178 DBG_VERBOSE("scan_irq: adjusting CPPR to %d\n", prio); 179 xc->cppr = prio; 180 out_8(xive_tima + xive_tima_offset + TM_CPPR, prio); 181 } 182 183 return irq; 184 } 185 186 /* 187 * This is used to perform the magic loads from an ESB 188 * described in xive.h 189 */ 190 static notrace u8 xive_esb_read(struct xive_irq_data *xd, u32 offset) 191 { 192 u64 val; 193 194 /* Handle HW errata */ 195 if (xd->flags & XIVE_IRQ_FLAG_SHIFT_BUG) 196 offset |= offset << 4; 197 198 if ((xd->flags & XIVE_IRQ_FLAG_H_INT_ESB) && xive_ops->esb_rw) 199 val = xive_ops->esb_rw(xd->hw_irq, offset, 0, 0); 200 else 201 val = in_be64(xd->eoi_mmio + offset); 202 203 return (u8)val; 204 } 205 206 static void xive_esb_write(struct xive_irq_data *xd, u32 offset, u64 data) 207 { 208 /* Handle HW errata */ 209 if (xd->flags & XIVE_IRQ_FLAG_SHIFT_BUG) 210 offset |= offset << 4; 211 212 if ((xd->flags & XIVE_IRQ_FLAG_H_INT_ESB) && xive_ops->esb_rw) 213 xive_ops->esb_rw(xd->hw_irq, offset, data, 1); 214 else 215 out_be64(xd->eoi_mmio + offset, data); 216 } 217 218 #ifdef CONFIG_XMON 219 static notrace void xive_dump_eq(const char *name, struct xive_q *q) 220 { 221 u32 i0, i1, idx; 222 223 if (!q->qpage) 224 return; 225 idx = q->idx; 226 i0 = be32_to_cpup(q->qpage + idx); 227 idx = (idx + 1) & q->msk; 228 i1 = be32_to_cpup(q->qpage + idx); 229 xmon_printf(" %s Q T=%d %08x %08x ...\n", name, 230 q->toggle, i0, i1); 231 } 232 233 notrace void xmon_xive_do_dump(int cpu) 234 { 235 struct xive_cpu *xc = per_cpu(xive_cpu, cpu); 236 237 xmon_printf("XIVE state for CPU %d:\n", cpu); 238 xmon_printf(" pp=%02x cppr=%02x\n", xc->pending_prio, xc->cppr); 239 xive_dump_eq("IRQ", &xc->queue[xive_irq_priority]); 240 #ifdef CONFIG_SMP 241 { 242 u64 val = xive_esb_read(&xc->ipi_data, XIVE_ESB_GET); 243 xmon_printf(" IPI state: %x:%c%c\n", xc->hw_ipi, 244 val & XIVE_ESB_VAL_P ? 'P' : 'p', 245 val & XIVE_ESB_VAL_Q ? 'Q' : 'q'); 246 } 247 #endif 248 } 249 #endif /* CONFIG_XMON */ 250 251 static unsigned int xive_get_irq(void) 252 { 253 struct xive_cpu *xc = __this_cpu_read(xive_cpu); 254 u32 irq; 255 256 /* 257 * This can be called either as a result of a HW interrupt or 258 * as a "replay" because EOI decided there was still something 259 * in one of the queues. 260 * 261 * First we perform an ACK cycle in order to update our mask 262 * of pending priorities. This will also have the effect of 263 * updating the CPPR to the most favored pending interrupts. 264 * 265 * In the future, if we have a way to differentiate a first 266 * entry (on HW interrupt) from a replay triggered by EOI, 267 * we could skip this on replays unless we soft-mask tells us 268 * that a new HW interrupt occurred. 269 */ 270 xive_ops->update_pending(xc); 271 272 DBG_VERBOSE("get_irq: pending=%02x\n", xc->pending_prio); 273 274 /* Scan our queue(s) for interrupts */ 275 irq = xive_scan_interrupts(xc, false); 276 277 DBG_VERBOSE("get_irq: got irq 0x%x, new pending=0x%02x\n", 278 irq, xc->pending_prio); 279 280 /* Return pending interrupt if any */ 281 if (irq == XIVE_BAD_IRQ) 282 return 0; 283 return irq; 284 } 285 286 /* 287 * After EOI'ing an interrupt, we need to re-check the queue 288 * to see if another interrupt is pending since multiple 289 * interrupts can coalesce into a single notification to the 290 * CPU. 291 * 292 * If we find that there is indeed more in there, we call 293 * force_external_irq_replay() to make Linux synthetize an 294 * external interrupt on the next call to local_irq_restore(). 295 */ 296 static void xive_do_queue_eoi(struct xive_cpu *xc) 297 { 298 if (xive_scan_interrupts(xc, true) != 0) { 299 DBG_VERBOSE("eoi: pending=0x%02x\n", xc->pending_prio); 300 force_external_irq_replay(); 301 } 302 } 303 304 /* 305 * EOI an interrupt at the source. There are several methods 306 * to do this depending on the HW version and source type 307 */ 308 static void xive_do_source_eoi(u32 hw_irq, struct xive_irq_data *xd) 309 { 310 /* If the XIVE supports the new "store EOI facility, use it */ 311 if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI) 312 xive_esb_write(xd, XIVE_ESB_STORE_EOI, 0); 313 else if (hw_irq && xd->flags & XIVE_IRQ_FLAG_EOI_FW) { 314 /* 315 * The FW told us to call it. This happens for some 316 * interrupt sources that need additional HW whacking 317 * beyond the ESB manipulation. For example LPC interrupts 318 * on P9 DD1.0 needed a latch to be clared in the LPC bridge 319 * itself. The Firmware will take care of it. 320 */ 321 if (WARN_ON_ONCE(!xive_ops->eoi)) 322 return; 323 xive_ops->eoi(hw_irq); 324 } else { 325 u8 eoi_val; 326 327 /* 328 * Otherwise for EOI, we use the special MMIO that does 329 * a clear of both P and Q and returns the old Q, 330 * except for LSIs where we use the "EOI cycle" special 331 * load. 332 * 333 * This allows us to then do a re-trigger if Q was set 334 * rather than synthesizing an interrupt in software 335 * 336 * For LSIs the HW EOI cycle is used rather than PQ bits, 337 * as they are automatically re-triggred in HW when still 338 * pending. 339 */ 340 if (xd->flags & XIVE_IRQ_FLAG_LSI) 341 xive_esb_read(xd, XIVE_ESB_LOAD_EOI); 342 else { 343 eoi_val = xive_esb_read(xd, XIVE_ESB_SET_PQ_00); 344 DBG_VERBOSE("eoi_val=%x\n", eoi_val); 345 346 /* Re-trigger if needed */ 347 if ((eoi_val & XIVE_ESB_VAL_Q) && xd->trig_mmio) 348 out_be64(xd->trig_mmio, 0); 349 } 350 } 351 } 352 353 /* irq_chip eoi callback */ 354 static void xive_irq_eoi(struct irq_data *d) 355 { 356 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); 357 struct xive_cpu *xc = __this_cpu_read(xive_cpu); 358 359 DBG_VERBOSE("eoi_irq: irq=%d [0x%lx] pending=%02x\n", 360 d->irq, irqd_to_hwirq(d), xc->pending_prio); 361 362 /* 363 * EOI the source if it hasn't been disabled and hasn't 364 * been passed-through to a KVM guest 365 */ 366 if (!irqd_irq_disabled(d) && !irqd_is_forwarded_to_vcpu(d) && 367 !(xd->flags & XIVE_IRQ_NO_EOI)) 368 xive_do_source_eoi(irqd_to_hwirq(d), xd); 369 370 /* 371 * Clear saved_p to indicate that it's no longer occupying 372 * a queue slot on the target queue 373 */ 374 xd->saved_p = false; 375 376 /* Check for more work in the queue */ 377 xive_do_queue_eoi(xc); 378 } 379 380 /* 381 * Helper used to mask and unmask an interrupt source. This 382 * is only called for normal interrupts that do not require 383 * masking/unmasking via firmware. 384 */ 385 static void xive_do_source_set_mask(struct xive_irq_data *xd, 386 bool mask) 387 { 388 u64 val; 389 390 /* 391 * If the interrupt had P set, it may be in a queue. 392 * 393 * We need to make sure we don't re-enable it until it 394 * has been fetched from that queue and EOId. We keep 395 * a copy of that P state and use it to restore the 396 * ESB accordingly on unmask. 397 */ 398 if (mask) { 399 val = xive_esb_read(xd, XIVE_ESB_SET_PQ_01); 400 xd->saved_p = !!(val & XIVE_ESB_VAL_P); 401 } else if (xd->saved_p) 402 xive_esb_read(xd, XIVE_ESB_SET_PQ_10); 403 else 404 xive_esb_read(xd, XIVE_ESB_SET_PQ_00); 405 } 406 407 /* 408 * Try to chose "cpu" as a new interrupt target. Increments 409 * the queue accounting for that target if it's not already 410 * full. 411 */ 412 static bool xive_try_pick_target(int cpu) 413 { 414 struct xive_cpu *xc = per_cpu(xive_cpu, cpu); 415 struct xive_q *q = &xc->queue[xive_irq_priority]; 416 int max; 417 418 /* 419 * Calculate max number of interrupts in that queue. 420 * 421 * We leave a gap of 1 just in case... 422 */ 423 max = (q->msk + 1) - 1; 424 return !!atomic_add_unless(&q->count, 1, max); 425 } 426 427 /* 428 * Un-account an interrupt for a target CPU. We don't directly 429 * decrement q->count since the interrupt might still be present 430 * in the queue. 431 * 432 * Instead increment a separate counter "pending_count" which 433 * will be substracted from "count" later when that CPU observes 434 * the queue to be empty. 435 */ 436 static void xive_dec_target_count(int cpu) 437 { 438 struct xive_cpu *xc = per_cpu(xive_cpu, cpu); 439 struct xive_q *q = &xc->queue[xive_irq_priority]; 440 441 if (WARN_ON(cpu < 0 || !xc)) { 442 pr_err("%s: cpu=%d xc=%p\n", __func__, cpu, xc); 443 return; 444 } 445 446 /* 447 * We increment the "pending count" which will be used 448 * to decrement the target queue count whenever it's next 449 * processed and found empty. This ensure that we don't 450 * decrement while we still have the interrupt there 451 * occupying a slot. 452 */ 453 atomic_inc(&q->pending_count); 454 } 455 456 /* Find a tentative CPU target in a CPU mask */ 457 static int xive_find_target_in_mask(const struct cpumask *mask, 458 unsigned int fuzz) 459 { 460 int cpu, first, num, i; 461 462 /* Pick up a starting point CPU in the mask based on fuzz */ 463 num = min_t(int, cpumask_weight(mask), nr_cpu_ids); 464 first = fuzz % num; 465 466 /* Locate it */ 467 cpu = cpumask_first(mask); 468 for (i = 0; i < first && cpu < nr_cpu_ids; i++) 469 cpu = cpumask_next(cpu, mask); 470 471 /* Sanity check */ 472 if (WARN_ON(cpu >= nr_cpu_ids)) 473 cpu = cpumask_first(cpu_online_mask); 474 475 /* Remember first one to handle wrap-around */ 476 first = cpu; 477 478 /* 479 * Now go through the entire mask until we find a valid 480 * target. 481 */ 482 do { 483 /* 484 * We re-check online as the fallback case passes us 485 * an untested affinity mask 486 */ 487 if (cpu_online(cpu) && xive_try_pick_target(cpu)) 488 return cpu; 489 cpu = cpumask_next(cpu, mask); 490 /* Wrap around */ 491 if (cpu >= nr_cpu_ids) 492 cpu = cpumask_first(mask); 493 } while (cpu != first); 494 495 return -1; 496 } 497 498 /* 499 * Pick a target CPU for an interrupt. This is done at 500 * startup or if the affinity is changed in a way that 501 * invalidates the current target. 502 */ 503 static int xive_pick_irq_target(struct irq_data *d, 504 const struct cpumask *affinity) 505 { 506 static unsigned int fuzz; 507 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); 508 cpumask_var_t mask; 509 int cpu = -1; 510 511 /* 512 * If we have chip IDs, first we try to build a mask of 513 * CPUs matching the CPU and find a target in there 514 */ 515 if (xd->src_chip != XIVE_INVALID_CHIP_ID && 516 zalloc_cpumask_var(&mask, GFP_ATOMIC)) { 517 /* Build a mask of matching chip IDs */ 518 for_each_cpu_and(cpu, affinity, cpu_online_mask) { 519 struct xive_cpu *xc = per_cpu(xive_cpu, cpu); 520 if (xc->chip_id == xd->src_chip) 521 cpumask_set_cpu(cpu, mask); 522 } 523 /* Try to find a target */ 524 if (cpumask_empty(mask)) 525 cpu = -1; 526 else 527 cpu = xive_find_target_in_mask(mask, fuzz++); 528 free_cpumask_var(mask); 529 if (cpu >= 0) 530 return cpu; 531 fuzz--; 532 } 533 534 /* No chip IDs, fallback to using the affinity mask */ 535 return xive_find_target_in_mask(affinity, fuzz++); 536 } 537 538 static unsigned int xive_irq_startup(struct irq_data *d) 539 { 540 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); 541 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d); 542 int target, rc; 543 544 pr_devel("xive_irq_startup: irq %d [0x%x] data @%p\n", 545 d->irq, hw_irq, d); 546 547 #ifdef CONFIG_PCI_MSI 548 /* 549 * The generic MSI code returns with the interrupt disabled on the 550 * card, using the MSI mask bits. Firmware doesn't appear to unmask 551 * at that level, so we do it here by hand. 552 */ 553 if (irq_data_get_msi_desc(d)) 554 pci_msi_unmask_irq(d); 555 #endif 556 557 /* Pick a target */ 558 target = xive_pick_irq_target(d, irq_data_get_affinity_mask(d)); 559 if (target == XIVE_INVALID_TARGET) { 560 /* Try again breaking affinity */ 561 target = xive_pick_irq_target(d, cpu_online_mask); 562 if (target == XIVE_INVALID_TARGET) 563 return -ENXIO; 564 pr_warn("irq %d started with broken affinity\n", d->irq); 565 } 566 567 /* Sanity check */ 568 if (WARN_ON(target == XIVE_INVALID_TARGET || 569 target >= nr_cpu_ids)) 570 target = smp_processor_id(); 571 572 xd->target = target; 573 574 /* 575 * Configure the logical number to be the Linux IRQ number 576 * and set the target queue 577 */ 578 rc = xive_ops->configure_irq(hw_irq, 579 get_hard_smp_processor_id(target), 580 xive_irq_priority, d->irq); 581 if (rc) 582 return rc; 583 584 /* Unmask the ESB */ 585 xive_do_source_set_mask(xd, false); 586 587 return 0; 588 } 589 590 static void xive_irq_shutdown(struct irq_data *d) 591 { 592 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); 593 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d); 594 595 pr_devel("xive_irq_shutdown: irq %d [0x%x] data @%p\n", 596 d->irq, hw_irq, d); 597 598 if (WARN_ON(xd->target == XIVE_INVALID_TARGET)) 599 return; 600 601 /* Mask the interrupt at the source */ 602 xive_do_source_set_mask(xd, true); 603 604 /* 605 * The above may have set saved_p. We clear it otherwise it 606 * will prevent re-enabling later on. It is ok to forget the 607 * fact that the interrupt might be in a queue because we are 608 * accounting that already in xive_dec_target_count() and will 609 * be re-routing it to a new queue with proper accounting when 610 * it's started up again 611 */ 612 xd->saved_p = false; 613 614 /* 615 * Mask the interrupt in HW in the IVT/EAS and set the number 616 * to be the "bad" IRQ number 617 */ 618 xive_ops->configure_irq(hw_irq, 619 get_hard_smp_processor_id(xd->target), 620 0xff, XIVE_BAD_IRQ); 621 622 xive_dec_target_count(xd->target); 623 xd->target = XIVE_INVALID_TARGET; 624 } 625 626 static void xive_irq_unmask(struct irq_data *d) 627 { 628 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); 629 630 pr_devel("xive_irq_unmask: irq %d data @%p\n", d->irq, xd); 631 632 /* 633 * This is a workaround for PCI LSI problems on P9, for 634 * these, we call FW to set the mask. The problems might 635 * be fixed by P9 DD2.0, if that is the case, firmware 636 * will no longer set that flag. 637 */ 638 if (xd->flags & XIVE_IRQ_FLAG_MASK_FW) { 639 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d); 640 xive_ops->configure_irq(hw_irq, 641 get_hard_smp_processor_id(xd->target), 642 xive_irq_priority, d->irq); 643 return; 644 } 645 646 xive_do_source_set_mask(xd, false); 647 } 648 649 static void xive_irq_mask(struct irq_data *d) 650 { 651 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); 652 653 pr_devel("xive_irq_mask: irq %d data @%p\n", d->irq, xd); 654 655 /* 656 * This is a workaround for PCI LSI problems on P9, for 657 * these, we call OPAL to set the mask. The problems might 658 * be fixed by P9 DD2.0, if that is the case, firmware 659 * will no longer set that flag. 660 */ 661 if (xd->flags & XIVE_IRQ_FLAG_MASK_FW) { 662 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d); 663 xive_ops->configure_irq(hw_irq, 664 get_hard_smp_processor_id(xd->target), 665 0xff, d->irq); 666 return; 667 } 668 669 xive_do_source_set_mask(xd, true); 670 } 671 672 static int xive_irq_set_affinity(struct irq_data *d, 673 const struct cpumask *cpumask, 674 bool force) 675 { 676 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); 677 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d); 678 u32 target, old_target; 679 int rc = 0; 680 681 pr_devel("xive_irq_set_affinity: irq %d\n", d->irq); 682 683 /* Is this valid ? */ 684 if (cpumask_any_and(cpumask, cpu_online_mask) >= nr_cpu_ids) 685 return -EINVAL; 686 687 /* Don't do anything if the interrupt isn't started */ 688 if (!irqd_is_started(d)) 689 return IRQ_SET_MASK_OK; 690 691 /* 692 * If existing target is already in the new mask, and is 693 * online then do nothing. 694 */ 695 if (xd->target != XIVE_INVALID_TARGET && 696 cpu_online(xd->target) && 697 cpumask_test_cpu(xd->target, cpumask)) 698 return IRQ_SET_MASK_OK; 699 700 /* Pick a new target */ 701 target = xive_pick_irq_target(d, cpumask); 702 703 /* No target found */ 704 if (target == XIVE_INVALID_TARGET) 705 return -ENXIO; 706 707 /* Sanity check */ 708 if (WARN_ON(target >= nr_cpu_ids)) 709 target = smp_processor_id(); 710 711 old_target = xd->target; 712 713 /* 714 * Only configure the irq if it's not currently passed-through to 715 * a KVM guest 716 */ 717 if (!irqd_is_forwarded_to_vcpu(d)) 718 rc = xive_ops->configure_irq(hw_irq, 719 get_hard_smp_processor_id(target), 720 xive_irq_priority, d->irq); 721 if (rc < 0) { 722 pr_err("Error %d reconfiguring irq %d\n", rc, d->irq); 723 return rc; 724 } 725 726 pr_devel(" target: 0x%x\n", target); 727 xd->target = target; 728 729 /* Give up previous target */ 730 if (old_target != XIVE_INVALID_TARGET) 731 xive_dec_target_count(old_target); 732 733 return IRQ_SET_MASK_OK; 734 } 735 736 static int xive_irq_set_type(struct irq_data *d, unsigned int flow_type) 737 { 738 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); 739 740 /* 741 * We only support these. This has really no effect other than setting 742 * the corresponding descriptor bits mind you but those will in turn 743 * affect the resend function when re-enabling an edge interrupt. 744 * 745 * Set set the default to edge as explained in map(). 746 */ 747 if (flow_type == IRQ_TYPE_DEFAULT || flow_type == IRQ_TYPE_NONE) 748 flow_type = IRQ_TYPE_EDGE_RISING; 749 750 if (flow_type != IRQ_TYPE_EDGE_RISING && 751 flow_type != IRQ_TYPE_LEVEL_LOW) 752 return -EINVAL; 753 754 irqd_set_trigger_type(d, flow_type); 755 756 /* 757 * Double check it matches what the FW thinks 758 * 759 * NOTE: We don't know yet if the PAPR interface will provide 760 * the LSI vs MSI information apart from the device-tree so 761 * this check might have to move into an optional backend call 762 * that is specific to the native backend 763 */ 764 if ((flow_type == IRQ_TYPE_LEVEL_LOW) != 765 !!(xd->flags & XIVE_IRQ_FLAG_LSI)) { 766 pr_warn("Interrupt %d (HW 0x%x) type mismatch, Linux says %s, FW says %s\n", 767 d->irq, (u32)irqd_to_hwirq(d), 768 (flow_type == IRQ_TYPE_LEVEL_LOW) ? "Level" : "Edge", 769 (xd->flags & XIVE_IRQ_FLAG_LSI) ? "Level" : "Edge"); 770 } 771 772 return IRQ_SET_MASK_OK_NOCOPY; 773 } 774 775 static int xive_irq_retrigger(struct irq_data *d) 776 { 777 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); 778 779 /* This should be only for MSIs */ 780 if (WARN_ON(xd->flags & XIVE_IRQ_FLAG_LSI)) 781 return 0; 782 783 /* 784 * To perform a retrigger, we first set the PQ bits to 785 * 11, then perform an EOI. 786 */ 787 xive_esb_read(xd, XIVE_ESB_SET_PQ_11); 788 789 /* 790 * Note: We pass "0" to the hw_irq argument in order to 791 * avoid calling into the backend EOI code which we don't 792 * want to do in the case of a re-trigger. Backends typically 793 * only do EOI for LSIs anyway. 794 */ 795 xive_do_source_eoi(0, xd); 796 797 return 1; 798 } 799 800 static int xive_irq_set_vcpu_affinity(struct irq_data *d, void *state) 801 { 802 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); 803 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d); 804 int rc; 805 u8 pq; 806 807 /* 808 * We only support this on interrupts that do not require 809 * firmware calls for masking and unmasking 810 */ 811 if (xd->flags & XIVE_IRQ_FLAG_MASK_FW) 812 return -EIO; 813 814 /* 815 * This is called by KVM with state non-NULL for enabling 816 * pass-through or NULL for disabling it 817 */ 818 if (state) { 819 irqd_set_forwarded_to_vcpu(d); 820 821 /* Set it to PQ=10 state to prevent further sends */ 822 pq = xive_esb_read(xd, XIVE_ESB_SET_PQ_10); 823 824 /* No target ? nothing to do */ 825 if (xd->target == XIVE_INVALID_TARGET) { 826 /* 827 * An untargetted interrupt should have been 828 * also masked at the source 829 */ 830 WARN_ON(pq & 2); 831 832 return 0; 833 } 834 835 /* 836 * If P was set, adjust state to PQ=11 to indicate 837 * that a resend is needed for the interrupt to reach 838 * the guest. Also remember the value of P. 839 * 840 * This also tells us that it's in flight to a host queue 841 * or has already been fetched but hasn't been EOIed yet 842 * by the host. This it's potentially using up a host 843 * queue slot. This is important to know because as long 844 * as this is the case, we must not hard-unmask it when 845 * "returning" that interrupt to the host. 846 * 847 * This saved_p is cleared by the host EOI, when we know 848 * for sure the queue slot is no longer in use. 849 */ 850 if (pq & 2) { 851 pq = xive_esb_read(xd, XIVE_ESB_SET_PQ_11); 852 xd->saved_p = true; 853 854 /* 855 * Sync the XIVE source HW to ensure the interrupt 856 * has gone through the EAS before we change its 857 * target to the guest. That should guarantee us 858 * that we *will* eventually get an EOI for it on 859 * the host. Otherwise there would be a small window 860 * for P to be seen here but the interrupt going 861 * to the guest queue. 862 */ 863 if (xive_ops->sync_source) 864 xive_ops->sync_source(hw_irq); 865 } else 866 xd->saved_p = false; 867 } else { 868 irqd_clr_forwarded_to_vcpu(d); 869 870 /* No host target ? hard mask and return */ 871 if (xd->target == XIVE_INVALID_TARGET) { 872 xive_do_source_set_mask(xd, true); 873 return 0; 874 } 875 876 /* 877 * Sync the XIVE source HW to ensure the interrupt 878 * has gone through the EAS before we change its 879 * target to the host. 880 */ 881 if (xive_ops->sync_source) 882 xive_ops->sync_source(hw_irq); 883 884 /* 885 * By convention we are called with the interrupt in 886 * a PQ=10 or PQ=11 state, ie, it won't fire and will 887 * have latched in Q whether there's a pending HW 888 * interrupt or not. 889 * 890 * First reconfigure the target. 891 */ 892 rc = xive_ops->configure_irq(hw_irq, 893 get_hard_smp_processor_id(xd->target), 894 xive_irq_priority, d->irq); 895 if (rc) 896 return rc; 897 898 /* 899 * Then if saved_p is not set, effectively re-enable the 900 * interrupt with an EOI. If it is set, we know there is 901 * still a message in a host queue somewhere that will be 902 * EOId eventually. 903 * 904 * Note: We don't check irqd_irq_disabled(). Effectively, 905 * we *will* let the irq get through even if masked if the 906 * HW is still firing it in order to deal with the whole 907 * saved_p business properly. If the interrupt triggers 908 * while masked, the generic code will re-mask it anyway. 909 */ 910 if (!xd->saved_p) 911 xive_do_source_eoi(hw_irq, xd); 912 913 } 914 return 0; 915 } 916 917 static struct irq_chip xive_irq_chip = { 918 .name = "XIVE-IRQ", 919 .irq_startup = xive_irq_startup, 920 .irq_shutdown = xive_irq_shutdown, 921 .irq_eoi = xive_irq_eoi, 922 .irq_mask = xive_irq_mask, 923 .irq_unmask = xive_irq_unmask, 924 .irq_set_affinity = xive_irq_set_affinity, 925 .irq_set_type = xive_irq_set_type, 926 .irq_retrigger = xive_irq_retrigger, 927 .irq_set_vcpu_affinity = xive_irq_set_vcpu_affinity, 928 }; 929 930 bool is_xive_irq(struct irq_chip *chip) 931 { 932 return chip == &xive_irq_chip; 933 } 934 EXPORT_SYMBOL_GPL(is_xive_irq); 935 936 void xive_cleanup_irq_data(struct xive_irq_data *xd) 937 { 938 if (xd->eoi_mmio) { 939 iounmap(xd->eoi_mmio); 940 if (xd->eoi_mmio == xd->trig_mmio) 941 xd->trig_mmio = NULL; 942 xd->eoi_mmio = NULL; 943 } 944 if (xd->trig_mmio) { 945 iounmap(xd->trig_mmio); 946 xd->trig_mmio = NULL; 947 } 948 } 949 EXPORT_SYMBOL_GPL(xive_cleanup_irq_data); 950 951 static int xive_irq_alloc_data(unsigned int virq, irq_hw_number_t hw) 952 { 953 struct xive_irq_data *xd; 954 int rc; 955 956 xd = kzalloc(sizeof(struct xive_irq_data), GFP_KERNEL); 957 if (!xd) 958 return -ENOMEM; 959 rc = xive_ops->populate_irq_data(hw, xd); 960 if (rc) { 961 kfree(xd); 962 return rc; 963 } 964 xd->target = XIVE_INVALID_TARGET; 965 irq_set_handler_data(virq, xd); 966 967 return 0; 968 } 969 970 static void xive_irq_free_data(unsigned int virq) 971 { 972 struct xive_irq_data *xd = irq_get_handler_data(virq); 973 974 if (!xd) 975 return; 976 irq_set_handler_data(virq, NULL); 977 xive_cleanup_irq_data(xd); 978 kfree(xd); 979 } 980 981 #ifdef CONFIG_SMP 982 983 static void xive_cause_ipi(int cpu) 984 { 985 struct xive_cpu *xc; 986 struct xive_irq_data *xd; 987 988 xc = per_cpu(xive_cpu, cpu); 989 990 DBG_VERBOSE("IPI CPU %d -> %d (HW IRQ 0x%x)\n", 991 smp_processor_id(), cpu, xc->hw_ipi); 992 993 xd = &xc->ipi_data; 994 if (WARN_ON(!xd->trig_mmio)) 995 return; 996 out_be64(xd->trig_mmio, 0); 997 } 998 999 static irqreturn_t xive_muxed_ipi_action(int irq, void *dev_id) 1000 { 1001 return smp_ipi_demux(); 1002 } 1003 1004 static void xive_ipi_eoi(struct irq_data *d) 1005 { 1006 struct xive_cpu *xc = __this_cpu_read(xive_cpu); 1007 1008 /* Handle possible race with unplug and drop stale IPIs */ 1009 if (!xc) 1010 return; 1011 1012 DBG_VERBOSE("IPI eoi: irq=%d [0x%lx] (HW IRQ 0x%x) pending=%02x\n", 1013 d->irq, irqd_to_hwirq(d), xc->hw_ipi, xc->pending_prio); 1014 1015 xive_do_source_eoi(xc->hw_ipi, &xc->ipi_data); 1016 xive_do_queue_eoi(xc); 1017 } 1018 1019 static void xive_ipi_do_nothing(struct irq_data *d) 1020 { 1021 /* 1022 * Nothing to do, we never mask/unmask IPIs, but the callback 1023 * has to exist for the struct irq_chip. 1024 */ 1025 } 1026 1027 static struct irq_chip xive_ipi_chip = { 1028 .name = "XIVE-IPI", 1029 .irq_eoi = xive_ipi_eoi, 1030 .irq_mask = xive_ipi_do_nothing, 1031 .irq_unmask = xive_ipi_do_nothing, 1032 }; 1033 1034 static void __init xive_request_ipi(void) 1035 { 1036 unsigned int virq; 1037 1038 /* 1039 * Initialization failed, move on, we might manage to 1040 * reach the point where we display our errors before 1041 * the system falls appart 1042 */ 1043 if (!xive_irq_domain) 1044 return; 1045 1046 /* Initialize it */ 1047 virq = irq_create_mapping(xive_irq_domain, 0); 1048 xive_ipi_irq = virq; 1049 1050 WARN_ON(request_irq(virq, xive_muxed_ipi_action, 1051 IRQF_PERCPU | IRQF_NO_THREAD, "IPI", NULL)); 1052 } 1053 1054 static int xive_setup_cpu_ipi(unsigned int cpu) 1055 { 1056 struct xive_cpu *xc; 1057 int rc; 1058 1059 pr_debug("Setting up IPI for CPU %d\n", cpu); 1060 1061 xc = per_cpu(xive_cpu, cpu); 1062 1063 /* Check if we are already setup */ 1064 if (xc->hw_ipi != 0) 1065 return 0; 1066 1067 /* Grab an IPI from the backend, this will populate xc->hw_ipi */ 1068 if (xive_ops->get_ipi(cpu, xc)) 1069 return -EIO; 1070 1071 /* 1072 * Populate the IRQ data in the xive_cpu structure and 1073 * configure the HW / enable the IPIs. 1074 */ 1075 rc = xive_ops->populate_irq_data(xc->hw_ipi, &xc->ipi_data); 1076 if (rc) { 1077 pr_err("Failed to populate IPI data on CPU %d\n", cpu); 1078 return -EIO; 1079 } 1080 rc = xive_ops->configure_irq(xc->hw_ipi, 1081 get_hard_smp_processor_id(cpu), 1082 xive_irq_priority, xive_ipi_irq); 1083 if (rc) { 1084 pr_err("Failed to map IPI CPU %d\n", cpu); 1085 return -EIO; 1086 } 1087 pr_devel("CPU %d HW IPI %x, virq %d, trig_mmio=%p\n", cpu, 1088 xc->hw_ipi, xive_ipi_irq, xc->ipi_data.trig_mmio); 1089 1090 /* Unmask it */ 1091 xive_do_source_set_mask(&xc->ipi_data, false); 1092 1093 return 0; 1094 } 1095 1096 static void xive_cleanup_cpu_ipi(unsigned int cpu, struct xive_cpu *xc) 1097 { 1098 /* Disable the IPI and free the IRQ data */ 1099 1100 /* Already cleaned up ? */ 1101 if (xc->hw_ipi == 0) 1102 return; 1103 1104 /* Mask the IPI */ 1105 xive_do_source_set_mask(&xc->ipi_data, true); 1106 1107 /* 1108 * Note: We don't call xive_cleanup_irq_data() to free 1109 * the mappings as this is called from an IPI on kexec 1110 * which is not a safe environment to call iounmap() 1111 */ 1112 1113 /* Deconfigure/mask in the backend */ 1114 xive_ops->configure_irq(xc->hw_ipi, hard_smp_processor_id(), 1115 0xff, xive_ipi_irq); 1116 1117 /* Free the IPIs in the backend */ 1118 xive_ops->put_ipi(cpu, xc); 1119 } 1120 1121 void __init xive_smp_probe(void) 1122 { 1123 smp_ops->cause_ipi = xive_cause_ipi; 1124 1125 /* Register the IPI */ 1126 xive_request_ipi(); 1127 1128 /* Allocate and setup IPI for the boot CPU */ 1129 xive_setup_cpu_ipi(smp_processor_id()); 1130 } 1131 1132 #endif /* CONFIG_SMP */ 1133 1134 static int xive_irq_domain_map(struct irq_domain *h, unsigned int virq, 1135 irq_hw_number_t hw) 1136 { 1137 int rc; 1138 1139 /* 1140 * Mark interrupts as edge sensitive by default so that resend 1141 * actually works. Will fix that up below if needed. 1142 */ 1143 irq_clear_status_flags(virq, IRQ_LEVEL); 1144 1145 #ifdef CONFIG_SMP 1146 /* IPIs are special and come up with HW number 0 */ 1147 if (hw == 0) { 1148 /* 1149 * IPIs are marked per-cpu. We use separate HW interrupts under 1150 * the hood but associated with the same "linux" interrupt 1151 */ 1152 irq_set_chip_and_handler(virq, &xive_ipi_chip, 1153 handle_percpu_irq); 1154 return 0; 1155 } 1156 #endif 1157 1158 rc = xive_irq_alloc_data(virq, hw); 1159 if (rc) 1160 return rc; 1161 1162 irq_set_chip_and_handler(virq, &xive_irq_chip, handle_fasteoi_irq); 1163 1164 return 0; 1165 } 1166 1167 static void xive_irq_domain_unmap(struct irq_domain *d, unsigned int virq) 1168 { 1169 struct irq_data *data = irq_get_irq_data(virq); 1170 unsigned int hw_irq; 1171 1172 /* XXX Assign BAD number */ 1173 if (!data) 1174 return; 1175 hw_irq = (unsigned int)irqd_to_hwirq(data); 1176 if (hw_irq) 1177 xive_irq_free_data(virq); 1178 } 1179 1180 static int xive_irq_domain_xlate(struct irq_domain *h, struct device_node *ct, 1181 const u32 *intspec, unsigned int intsize, 1182 irq_hw_number_t *out_hwirq, unsigned int *out_flags) 1183 1184 { 1185 *out_hwirq = intspec[0]; 1186 1187 /* 1188 * If intsize is at least 2, we look for the type in the second cell, 1189 * we assume the LSB indicates a level interrupt. 1190 */ 1191 if (intsize > 1) { 1192 if (intspec[1] & 1) 1193 *out_flags = IRQ_TYPE_LEVEL_LOW; 1194 else 1195 *out_flags = IRQ_TYPE_EDGE_RISING; 1196 } else 1197 *out_flags = IRQ_TYPE_LEVEL_LOW; 1198 1199 return 0; 1200 } 1201 1202 static int xive_irq_domain_match(struct irq_domain *h, struct device_node *node, 1203 enum irq_domain_bus_token bus_token) 1204 { 1205 return xive_ops->match(node); 1206 } 1207 1208 static const struct irq_domain_ops xive_irq_domain_ops = { 1209 .match = xive_irq_domain_match, 1210 .map = xive_irq_domain_map, 1211 .unmap = xive_irq_domain_unmap, 1212 .xlate = xive_irq_domain_xlate, 1213 }; 1214 1215 static void __init xive_init_host(void) 1216 { 1217 xive_irq_domain = irq_domain_add_nomap(NULL, XIVE_MAX_IRQ, 1218 &xive_irq_domain_ops, NULL); 1219 if (WARN_ON(xive_irq_domain == NULL)) 1220 return; 1221 irq_set_default_host(xive_irq_domain); 1222 } 1223 1224 static void xive_cleanup_cpu_queues(unsigned int cpu, struct xive_cpu *xc) 1225 { 1226 if (xc->queue[xive_irq_priority].qpage) 1227 xive_ops->cleanup_queue(cpu, xc, xive_irq_priority); 1228 } 1229 1230 static int xive_setup_cpu_queues(unsigned int cpu, struct xive_cpu *xc) 1231 { 1232 int rc = 0; 1233 1234 /* We setup 1 queues for now with a 64k page */ 1235 if (!xc->queue[xive_irq_priority].qpage) 1236 rc = xive_ops->setup_queue(cpu, xc, xive_irq_priority); 1237 1238 return rc; 1239 } 1240 1241 static int xive_prepare_cpu(unsigned int cpu) 1242 { 1243 struct xive_cpu *xc; 1244 1245 xc = per_cpu(xive_cpu, cpu); 1246 if (!xc) { 1247 struct device_node *np; 1248 1249 xc = kzalloc_node(sizeof(struct xive_cpu), 1250 GFP_KERNEL, cpu_to_node(cpu)); 1251 if (!xc) 1252 return -ENOMEM; 1253 np = of_get_cpu_node(cpu, NULL); 1254 if (np) 1255 xc->chip_id = of_get_ibm_chip_id(np); 1256 of_node_put(np); 1257 1258 per_cpu(xive_cpu, cpu) = xc; 1259 } 1260 1261 /* Setup EQs if not already */ 1262 return xive_setup_cpu_queues(cpu, xc); 1263 } 1264 1265 static void xive_setup_cpu(void) 1266 { 1267 struct xive_cpu *xc = __this_cpu_read(xive_cpu); 1268 1269 /* The backend might have additional things to do */ 1270 if (xive_ops->setup_cpu) 1271 xive_ops->setup_cpu(smp_processor_id(), xc); 1272 1273 /* Set CPPR to 0xff to enable flow of interrupts */ 1274 xc->cppr = 0xff; 1275 out_8(xive_tima + xive_tima_offset + TM_CPPR, 0xff); 1276 } 1277 1278 #ifdef CONFIG_SMP 1279 void xive_smp_setup_cpu(void) 1280 { 1281 pr_devel("SMP setup CPU %d\n", smp_processor_id()); 1282 1283 /* This will have already been done on the boot CPU */ 1284 if (smp_processor_id() != boot_cpuid) 1285 xive_setup_cpu(); 1286 1287 } 1288 1289 int xive_smp_prepare_cpu(unsigned int cpu) 1290 { 1291 int rc; 1292 1293 /* Allocate per-CPU data and queues */ 1294 rc = xive_prepare_cpu(cpu); 1295 if (rc) 1296 return rc; 1297 1298 /* Allocate and setup IPI for the new CPU */ 1299 return xive_setup_cpu_ipi(cpu); 1300 } 1301 1302 #ifdef CONFIG_HOTPLUG_CPU 1303 static void xive_flush_cpu_queue(unsigned int cpu, struct xive_cpu *xc) 1304 { 1305 u32 irq; 1306 1307 /* We assume local irqs are disabled */ 1308 WARN_ON(!irqs_disabled()); 1309 1310 /* Check what's already in the CPU queue */ 1311 while ((irq = xive_scan_interrupts(xc, false)) != 0) { 1312 /* 1313 * We need to re-route that interrupt to its new destination. 1314 * First get and lock the descriptor 1315 */ 1316 struct irq_desc *desc = irq_to_desc(irq); 1317 struct irq_data *d = irq_desc_get_irq_data(desc); 1318 struct xive_irq_data *xd; 1319 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d); 1320 1321 /* 1322 * Ignore anything that isn't a XIVE irq and ignore 1323 * IPIs, so can just be dropped. 1324 */ 1325 if (d->domain != xive_irq_domain || hw_irq == 0) 1326 continue; 1327 1328 /* 1329 * The IRQ should have already been re-routed, it's just a 1330 * stale in the old queue, so re-trigger it in order to make 1331 * it reach is new destination. 1332 */ 1333 #ifdef DEBUG_FLUSH 1334 pr_info("CPU %d: Got irq %d while offline, re-sending...\n", 1335 cpu, irq); 1336 #endif 1337 raw_spin_lock(&desc->lock); 1338 xd = irq_desc_get_handler_data(desc); 1339 1340 /* 1341 * For LSIs, we EOI, this will cause a resend if it's 1342 * still asserted. Otherwise do an MSI retrigger. 1343 */ 1344 if (xd->flags & XIVE_IRQ_FLAG_LSI) 1345 xive_do_source_eoi(irqd_to_hwirq(d), xd); 1346 else 1347 xive_irq_retrigger(d); 1348 1349 raw_spin_unlock(&desc->lock); 1350 } 1351 } 1352 1353 void xive_smp_disable_cpu(void) 1354 { 1355 struct xive_cpu *xc = __this_cpu_read(xive_cpu); 1356 unsigned int cpu = smp_processor_id(); 1357 1358 /* Migrate interrupts away from the CPU */ 1359 irq_migrate_all_off_this_cpu(); 1360 1361 /* Set CPPR to 0 to disable flow of interrupts */ 1362 xc->cppr = 0; 1363 out_8(xive_tima + xive_tima_offset + TM_CPPR, 0); 1364 1365 /* Flush everything still in the queue */ 1366 xive_flush_cpu_queue(cpu, xc); 1367 1368 /* Re-enable CPPR */ 1369 xc->cppr = 0xff; 1370 out_8(xive_tima + xive_tima_offset + TM_CPPR, 0xff); 1371 } 1372 1373 void xive_flush_interrupt(void) 1374 { 1375 struct xive_cpu *xc = __this_cpu_read(xive_cpu); 1376 unsigned int cpu = smp_processor_id(); 1377 1378 /* Called if an interrupt occurs while the CPU is hot unplugged */ 1379 xive_flush_cpu_queue(cpu, xc); 1380 } 1381 1382 #endif /* CONFIG_HOTPLUG_CPU */ 1383 1384 #endif /* CONFIG_SMP */ 1385 1386 void xive_teardown_cpu(void) 1387 { 1388 struct xive_cpu *xc = __this_cpu_read(xive_cpu); 1389 unsigned int cpu = smp_processor_id(); 1390 1391 /* Set CPPR to 0 to disable flow of interrupts */ 1392 xc->cppr = 0; 1393 out_8(xive_tima + xive_tima_offset + TM_CPPR, 0); 1394 1395 if (xive_ops->teardown_cpu) 1396 xive_ops->teardown_cpu(cpu, xc); 1397 1398 #ifdef CONFIG_SMP 1399 /* Get rid of IPI */ 1400 xive_cleanup_cpu_ipi(cpu, xc); 1401 #endif 1402 1403 /* Disable and free the queues */ 1404 xive_cleanup_cpu_queues(cpu, xc); 1405 } 1406 1407 void xive_shutdown(void) 1408 { 1409 xive_ops->shutdown(); 1410 } 1411 1412 bool __init xive_core_init(const struct xive_ops *ops, void __iomem *area, u32 offset, 1413 u8 max_prio) 1414 { 1415 xive_tima = area; 1416 xive_tima_offset = offset; 1417 xive_ops = ops; 1418 xive_irq_priority = max_prio; 1419 1420 ppc_md.get_irq = xive_get_irq; 1421 __xive_enabled = true; 1422 1423 pr_devel("Initializing host..\n"); 1424 xive_init_host(); 1425 1426 pr_devel("Initializing boot CPU..\n"); 1427 1428 /* Allocate per-CPU data and queues */ 1429 xive_prepare_cpu(smp_processor_id()); 1430 1431 /* Get ready for interrupts */ 1432 xive_setup_cpu(); 1433 1434 pr_info("Interrupt handling initialized with %s backend\n", 1435 xive_ops->name); 1436 pr_info("Using priority %d for all interrupts\n", max_prio); 1437 1438 return true; 1439 } 1440 1441 __be32 *xive_queue_page_alloc(unsigned int cpu, u32 queue_shift) 1442 { 1443 unsigned int alloc_order; 1444 struct page *pages; 1445 __be32 *qpage; 1446 1447 alloc_order = xive_alloc_order(queue_shift); 1448 pages = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, alloc_order); 1449 if (!pages) 1450 return ERR_PTR(-ENOMEM); 1451 qpage = (__be32 *)page_address(pages); 1452 memset(qpage, 0, 1 << queue_shift); 1453 1454 return qpage; 1455 } 1456 1457 static int __init xive_off(char *arg) 1458 { 1459 xive_cmdline_disabled = true; 1460 return 0; 1461 } 1462 __setup("xive=off", xive_off); 1463