1 /* 2 * Copyright 2016,2017 IBM Corporation. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 */ 9 10 #define pr_fmt(fmt) "xive: " fmt 11 12 #include <linux/types.h> 13 #include <linux/irq.h> 14 #include <linux/debugfs.h> 15 #include <linux/smp.h> 16 #include <linux/interrupt.h> 17 #include <linux/seq_file.h> 18 #include <linux/init.h> 19 #include <linux/of.h> 20 #include <linux/slab.h> 21 #include <linux/spinlock.h> 22 #include <linux/delay.h> 23 #include <linux/cpumask.h> 24 #include <linux/mm.h> 25 26 #include <asm/prom.h> 27 #include <asm/io.h> 28 #include <asm/smp.h> 29 #include <asm/irq.h> 30 #include <asm/errno.h> 31 #include <asm/xive.h> 32 #include <asm/xive-regs.h> 33 #include <asm/opal.h> 34 35 #include "xive-internal.h" 36 37 38 static u32 xive_provision_size; 39 static u32 *xive_provision_chips; 40 static u32 xive_provision_chip_count; 41 static u32 xive_queue_shift; 42 static u32 xive_pool_vps = XIVE_INVALID_VP; 43 static struct kmem_cache *xive_provision_cache; 44 45 int xive_native_populate_irq_data(u32 hw_irq, struct xive_irq_data *data) 46 { 47 __be64 flags, eoi_page, trig_page; 48 __be32 esb_shift, src_chip; 49 u64 opal_flags; 50 s64 rc; 51 52 memset(data, 0, sizeof(*data)); 53 54 rc = opal_xive_get_irq_info(hw_irq, &flags, &eoi_page, &trig_page, 55 &esb_shift, &src_chip); 56 if (rc) { 57 pr_err("opal_xive_get_irq_info(0x%x) returned %lld\n", 58 hw_irq, rc); 59 return -EINVAL; 60 } 61 62 opal_flags = be64_to_cpu(flags); 63 if (opal_flags & OPAL_XIVE_IRQ_STORE_EOI) 64 data->flags |= XIVE_IRQ_FLAG_STORE_EOI; 65 if (opal_flags & OPAL_XIVE_IRQ_LSI) 66 data->flags |= XIVE_IRQ_FLAG_LSI; 67 if (opal_flags & OPAL_XIVE_IRQ_SHIFT_BUG) 68 data->flags |= XIVE_IRQ_FLAG_SHIFT_BUG; 69 if (opal_flags & OPAL_XIVE_IRQ_MASK_VIA_FW) 70 data->flags |= XIVE_IRQ_FLAG_MASK_FW; 71 if (opal_flags & OPAL_XIVE_IRQ_EOI_VIA_FW) 72 data->flags |= XIVE_IRQ_FLAG_EOI_FW; 73 data->eoi_page = be64_to_cpu(eoi_page); 74 data->trig_page = be64_to_cpu(trig_page); 75 data->esb_shift = be32_to_cpu(esb_shift); 76 data->src_chip = be32_to_cpu(src_chip); 77 78 data->eoi_mmio = ioremap(data->eoi_page, 1u << data->esb_shift); 79 if (!data->eoi_mmio) { 80 pr_err("Failed to map EOI page for irq 0x%x\n", hw_irq); 81 return -ENOMEM; 82 } 83 84 if (!data->trig_page) 85 return 0; 86 if (data->trig_page == data->eoi_page) { 87 data->trig_mmio = data->eoi_mmio; 88 return 0; 89 } 90 91 data->trig_mmio = ioremap(data->trig_page, 1u << data->esb_shift); 92 if (!data->trig_mmio) { 93 pr_err("Failed to map trigger page for irq 0x%x\n", hw_irq); 94 return -ENOMEM; 95 } 96 return 0; 97 } 98 99 int xive_native_configure_irq(u32 hw_irq, u32 target, u8 prio, u32 sw_irq) 100 { 101 s64 rc; 102 103 for (;;) { 104 rc = opal_xive_set_irq_config(hw_irq, target, prio, sw_irq); 105 if (rc != OPAL_BUSY) 106 break; 107 msleep(1); 108 } 109 return rc == 0 ? 0 : -ENXIO; 110 } 111 112 /* This can be called multiple time to change a queue configuration */ 113 int xive_native_configure_queue(u32 vp_id, struct xive_q *q, u8 prio, 114 __be32 *qpage, u32 order, bool can_escalate) 115 { 116 s64 rc = 0; 117 __be64 qeoi_page_be; 118 __be32 esc_irq_be; 119 u64 flags, qpage_phys; 120 121 /* If there's an actual queue page, clean it */ 122 if (order) { 123 if (WARN_ON(!qpage)) 124 return -EINVAL; 125 qpage_phys = __pa(qpage); 126 } else 127 qpage_phys = 0; 128 129 /* Initialize the rest of the fields */ 130 q->msk = order ? ((1u << (order - 2)) - 1) : 0; 131 q->idx = 0; 132 q->toggle = 0; 133 134 rc = opal_xive_get_queue_info(vp_id, prio, NULL, NULL, 135 &qeoi_page_be, 136 &esc_irq_be, 137 NULL); 138 if (rc) { 139 pr_err("Error %lld getting queue info prio %d\n", rc, prio); 140 rc = -EIO; 141 goto fail; 142 } 143 q->eoi_phys = be64_to_cpu(qeoi_page_be); 144 145 /* Default flags */ 146 flags = OPAL_XIVE_EQ_ALWAYS_NOTIFY | OPAL_XIVE_EQ_ENABLED; 147 148 /* Escalation needed ? */ 149 if (can_escalate) { 150 q->esc_irq = be32_to_cpu(esc_irq_be); 151 flags |= OPAL_XIVE_EQ_ESCALATE; 152 } 153 154 /* Configure and enable the queue in HW */ 155 for (;;) { 156 rc = opal_xive_set_queue_info(vp_id, prio, qpage_phys, order, flags); 157 if (rc != OPAL_BUSY) 158 break; 159 msleep(1); 160 } 161 if (rc) { 162 pr_err("Error %lld setting queue for prio %d\n", rc, prio); 163 rc = -EIO; 164 } else { 165 /* 166 * KVM code requires all of the above to be visible before 167 * q->qpage is set due to how it manages IPI EOIs 168 */ 169 wmb(); 170 q->qpage = qpage; 171 } 172 fail: 173 return rc; 174 } 175 176 static void __xive_native_disable_queue(u32 vp_id, struct xive_q *q, u8 prio) 177 { 178 s64 rc; 179 180 /* Disable the queue in HW */ 181 for (;;) { 182 rc = opal_xive_set_queue_info(vp_id, prio, 0, 0, 0); 183 if (rc != OPAL_BUSY) 184 break; 185 msleep(1); 186 } 187 if (rc) 188 pr_err("Error %lld disabling queue for prio %d\n", rc, prio); 189 } 190 191 void xive_native_disable_queue(u32 vp_id, struct xive_q *q, u8 prio) 192 { 193 __xive_native_disable_queue(vp_id, q, prio); 194 } 195 196 static int xive_native_setup_queue(unsigned int cpu, struct xive_cpu *xc, u8 prio) 197 { 198 struct xive_q *q = &xc->queue[prio]; 199 unsigned int alloc_order; 200 struct page *pages; 201 __be32 *qpage; 202 203 alloc_order = (xive_queue_shift > PAGE_SHIFT) ? 204 (xive_queue_shift - PAGE_SHIFT) : 0; 205 pages = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, alloc_order); 206 if (!pages) 207 return -ENOMEM; 208 qpage = (__be32 *)page_address(pages); 209 memset(qpage, 0, 1 << xive_queue_shift); 210 return xive_native_configure_queue(get_hard_smp_processor_id(cpu), 211 q, prio, qpage, xive_queue_shift, false); 212 } 213 214 static void xive_native_cleanup_queue(unsigned int cpu, struct xive_cpu *xc, u8 prio) 215 { 216 struct xive_q *q = &xc->queue[prio]; 217 unsigned int alloc_order; 218 219 /* 220 * We use the variant with no iounmap as this is called on exec 221 * from an IPI and iounmap isn't safe 222 */ 223 __xive_native_disable_queue(get_hard_smp_processor_id(cpu), q, prio); 224 alloc_order = (xive_queue_shift > PAGE_SHIFT) ? 225 (xive_queue_shift - PAGE_SHIFT) : 0; 226 free_pages((unsigned long)q->qpage, alloc_order); 227 q->qpage = NULL; 228 } 229 230 static bool xive_native_match(struct device_node *node) 231 { 232 return of_device_is_compatible(node, "ibm,opal-xive-vc"); 233 } 234 235 #ifdef CONFIG_SMP 236 static int xive_native_get_ipi(unsigned int cpu, struct xive_cpu *xc) 237 { 238 struct device_node *np; 239 unsigned int chip_id; 240 s64 irq; 241 242 /* Find the chip ID */ 243 np = of_get_cpu_node(cpu, NULL); 244 if (np) { 245 if (of_property_read_u32(np, "ibm,chip-id", &chip_id) < 0) 246 chip_id = 0; 247 } 248 249 /* Allocate an IPI and populate info about it */ 250 for (;;) { 251 irq = opal_xive_allocate_irq(chip_id); 252 if (irq == OPAL_BUSY) { 253 msleep(1); 254 continue; 255 } 256 if (irq < 0) { 257 pr_err("Failed to allocate IPI on CPU %d\n", cpu); 258 return -ENXIO; 259 } 260 xc->hw_ipi = irq; 261 break; 262 } 263 return 0; 264 } 265 266 u32 xive_native_alloc_irq(void) 267 { 268 s64 rc; 269 270 for (;;) { 271 rc = opal_xive_allocate_irq(OPAL_XIVE_ANY_CHIP); 272 if (rc != OPAL_BUSY) 273 break; 274 msleep(1); 275 } 276 if (rc < 0) 277 return 0; 278 return rc; 279 } 280 281 void xive_native_free_irq(u32 irq) 282 { 283 for (;;) { 284 s64 rc = opal_xive_free_irq(irq); 285 if (rc != OPAL_BUSY) 286 break; 287 msleep(1); 288 } 289 } 290 291 static void xive_native_put_ipi(unsigned int cpu, struct xive_cpu *xc) 292 { 293 s64 rc; 294 295 /* Free the IPI */ 296 if (!xc->hw_ipi) 297 return; 298 for (;;) { 299 rc = opal_xive_free_irq(xc->hw_ipi); 300 if (rc == OPAL_BUSY) { 301 msleep(1); 302 continue; 303 } 304 xc->hw_ipi = 0; 305 break; 306 } 307 } 308 #endif /* CONFIG_SMP */ 309 310 static void xive_native_shutdown(void) 311 { 312 /* Switch the XIVE to emulation mode */ 313 opal_xive_reset(OPAL_XIVE_MODE_EMU); 314 } 315 316 /* 317 * Perform an "ack" cycle on the current thread, thus 318 * grabbing the pending active priorities and updating 319 * the CPPR to the most favored one. 320 */ 321 static void xive_native_update_pending(struct xive_cpu *xc) 322 { 323 u8 he, cppr; 324 u16 ack; 325 326 /* Perform the acknowledge hypervisor to register cycle */ 327 ack = be16_to_cpu(__raw_readw(xive_tima + TM_SPC_ACK_HV_REG)); 328 329 /* Synchronize subsequent queue accesses */ 330 mb(); 331 332 /* 333 * Grab the CPPR and the "HE" field which indicates the source 334 * of the hypervisor interrupt (if any) 335 */ 336 cppr = ack & 0xff; 337 he = GETFIELD(TM_QW3_NSR_HE, (ack >> 8)); 338 switch(he) { 339 case TM_QW3_NSR_HE_NONE: /* Nothing to see here */ 340 break; 341 case TM_QW3_NSR_HE_PHYS: /* Physical thread interrupt */ 342 if (cppr == 0xff) 343 return; 344 /* Mark the priority pending */ 345 xc->pending_prio |= 1 << cppr; 346 347 /* 348 * A new interrupt should never have a CPPR less favored 349 * than our current one. 350 */ 351 if (cppr >= xc->cppr) 352 pr_err("CPU %d odd ack CPPR, got %d at %d\n", 353 smp_processor_id(), cppr, xc->cppr); 354 355 /* Update our idea of what the CPPR is */ 356 xc->cppr = cppr; 357 break; 358 case TM_QW3_NSR_HE_POOL: /* HV Pool interrupt (unused) */ 359 case TM_QW3_NSR_HE_LSI: /* Legacy FW LSI (unused) */ 360 pr_err("CPU %d got unexpected interrupt type HE=%d\n", 361 smp_processor_id(), he); 362 return; 363 } 364 } 365 366 static void xive_native_eoi(u32 hw_irq) 367 { 368 /* 369 * Not normally used except if specific interrupts need 370 * a workaround on EOI. 371 */ 372 opal_int_eoi(hw_irq); 373 } 374 375 static void xive_native_setup_cpu(unsigned int cpu, struct xive_cpu *xc) 376 { 377 s64 rc; 378 u32 vp; 379 __be64 vp_cam_be; 380 u64 vp_cam; 381 382 if (xive_pool_vps == XIVE_INVALID_VP) 383 return; 384 385 /* Enable the pool VP */ 386 vp = xive_pool_vps + get_hard_smp_processor_id(cpu); 387 pr_debug("CPU %d setting up pool VP 0x%x\n", cpu, vp); 388 for (;;) { 389 rc = opal_xive_set_vp_info(vp, OPAL_XIVE_VP_ENABLED, 0); 390 if (rc != OPAL_BUSY) 391 break; 392 msleep(1); 393 } 394 if (rc) { 395 pr_err("Failed to enable pool VP on CPU %d\n", cpu); 396 return; 397 } 398 399 /* Grab it's CAM value */ 400 rc = opal_xive_get_vp_info(vp, NULL, &vp_cam_be, NULL, NULL); 401 if (rc) { 402 pr_err("Failed to get pool VP info CPU %d\n", cpu); 403 return; 404 } 405 vp_cam = be64_to_cpu(vp_cam_be); 406 407 pr_debug("VP CAM = %llx\n", vp_cam); 408 409 /* Push it on the CPU (set LSMFB to 0xff to skip backlog scan) */ 410 pr_debug("(Old HW value: %08x)\n", 411 in_be32(xive_tima + TM_QW2_HV_POOL + TM_WORD2)); 412 out_be32(xive_tima + TM_QW2_HV_POOL + TM_WORD0, 0xff); 413 out_be32(xive_tima + TM_QW2_HV_POOL + TM_WORD2, 414 TM_QW2W2_VP | vp_cam); 415 pr_debug("(New HW value: %08x)\n", 416 in_be32(xive_tima + TM_QW2_HV_POOL + TM_WORD2)); 417 } 418 419 static void xive_native_teardown_cpu(unsigned int cpu, struct xive_cpu *xc) 420 { 421 s64 rc; 422 u32 vp; 423 424 if (xive_pool_vps == XIVE_INVALID_VP) 425 return; 426 427 /* Pull the pool VP from the CPU */ 428 in_be64(xive_tima + TM_SPC_PULL_POOL_CTX); 429 430 /* Disable it */ 431 vp = xive_pool_vps + get_hard_smp_processor_id(cpu); 432 for (;;) { 433 rc = opal_xive_set_vp_info(vp, 0, 0); 434 if (rc != OPAL_BUSY) 435 break; 436 msleep(1); 437 } 438 } 439 440 static void xive_native_sync_source(u32 hw_irq) 441 { 442 opal_xive_sync(XIVE_SYNC_EAS, hw_irq); 443 } 444 445 static const struct xive_ops xive_native_ops = { 446 .populate_irq_data = xive_native_populate_irq_data, 447 .configure_irq = xive_native_configure_irq, 448 .setup_queue = xive_native_setup_queue, 449 .cleanup_queue = xive_native_cleanup_queue, 450 .match = xive_native_match, 451 .shutdown = xive_native_shutdown, 452 .update_pending = xive_native_update_pending, 453 .eoi = xive_native_eoi, 454 .setup_cpu = xive_native_setup_cpu, 455 .teardown_cpu = xive_native_teardown_cpu, 456 .sync_source = xive_native_sync_source, 457 #ifdef CONFIG_SMP 458 .get_ipi = xive_native_get_ipi, 459 .put_ipi = xive_native_put_ipi, 460 #endif /* CONFIG_SMP */ 461 .name = "native", 462 }; 463 464 static bool xive_parse_provisioning(struct device_node *np) 465 { 466 int rc; 467 468 if (of_property_read_u32(np, "ibm,xive-provision-page-size", 469 &xive_provision_size) < 0) 470 return true; 471 rc = of_property_count_elems_of_size(np, "ibm,xive-provision-chips", 4); 472 if (rc < 0) { 473 pr_err("Error %d getting provision chips array\n", rc); 474 return false; 475 } 476 xive_provision_chip_count = rc; 477 if (rc == 0) 478 return true; 479 480 xive_provision_chips = kzalloc(4 * xive_provision_chip_count, 481 GFP_KERNEL); 482 if (WARN_ON(!xive_provision_chips)) 483 return false; 484 485 rc = of_property_read_u32_array(np, "ibm,xive-provision-chips", 486 xive_provision_chips, 487 xive_provision_chip_count); 488 if (rc < 0) { 489 pr_err("Error %d reading provision chips array\n", rc); 490 return false; 491 } 492 493 xive_provision_cache = kmem_cache_create("xive-provision", 494 xive_provision_size, 495 xive_provision_size, 496 0, NULL); 497 if (!xive_provision_cache) { 498 pr_err("Failed to allocate provision cache\n"); 499 return false; 500 } 501 return true; 502 } 503 504 u32 xive_native_default_eq_shift(void) 505 { 506 return xive_queue_shift; 507 } 508 509 bool xive_native_init(void) 510 { 511 struct device_node *np; 512 struct resource r; 513 void __iomem *tima; 514 struct property *prop; 515 u8 max_prio = 7; 516 const __be32 *p; 517 u32 val; 518 s64 rc; 519 520 if (xive_cmdline_disabled) 521 return false; 522 523 pr_devel("xive_native_init()\n"); 524 np = of_find_compatible_node(NULL, NULL, "ibm,opal-xive-pe"); 525 if (!np) { 526 pr_devel("not found !\n"); 527 return false; 528 } 529 pr_devel("Found %s\n", np->full_name); 530 531 /* Resource 1 is HV window */ 532 if (of_address_to_resource(np, 1, &r)) { 533 pr_err("Failed to get thread mgmnt area resource\n"); 534 return false; 535 } 536 tima = ioremap(r.start, resource_size(&r)); 537 if (!tima) { 538 pr_err("Failed to map thread mgmnt area\n"); 539 return false; 540 } 541 542 /* Read number of priorities */ 543 if (of_property_read_u32(np, "ibm,xive-#priorities", &val) == 0) 544 max_prio = val - 1; 545 546 /* Iterate the EQ sizes and pick one */ 547 of_property_for_each_u32(np, "ibm,xive-eq-sizes", prop, p, val) { 548 xive_queue_shift = val; 549 if (val == PAGE_SHIFT) 550 break; 551 } 552 553 /* Grab size of provisioning pages */ 554 xive_parse_provisioning(np); 555 556 /* Switch the XIVE to exploitation mode */ 557 rc = opal_xive_reset(OPAL_XIVE_MODE_EXPL); 558 if (rc) { 559 pr_err("Switch to exploitation mode failed with error %lld\n", rc); 560 return false; 561 } 562 563 /* Initialize XIVE core with our backend */ 564 if (!xive_core_init(&xive_native_ops, tima, TM_QW3_HV_PHYS, 565 max_prio)) { 566 opal_xive_reset(OPAL_XIVE_MODE_EMU); 567 return false; 568 } 569 pr_info("Using %dkB queues\n", 1 << (xive_queue_shift - 10)); 570 return true; 571 } 572 573 static bool xive_native_provision_pages(void) 574 { 575 u32 i; 576 void *p; 577 578 for (i = 0; i < xive_provision_chip_count; i++) { 579 u32 chip = xive_provision_chips[i]; 580 581 /* 582 * XXX TODO: Try to make the allocation local to the node where 583 * the chip resides. 584 */ 585 p = kmem_cache_alloc(xive_provision_cache, GFP_KERNEL); 586 if (!p) { 587 pr_err("Failed to allocate provisioning page\n"); 588 return false; 589 } 590 opal_xive_donate_page(chip, __pa(p)); 591 } 592 return true; 593 } 594 595 u32 xive_native_alloc_vp_block(u32 max_vcpus) 596 { 597 s64 rc; 598 u32 order; 599 600 order = fls(max_vcpus) - 1; 601 if (max_vcpus > (1 << order)) 602 order++; 603 604 pr_info("VP block alloc, for max VCPUs %d use order %d\n", 605 max_vcpus, order); 606 607 for (;;) { 608 rc = opal_xive_alloc_vp_block(order); 609 switch (rc) { 610 case OPAL_BUSY: 611 msleep(1); 612 break; 613 case OPAL_XIVE_PROVISIONING: 614 if (!xive_native_provision_pages()) 615 return XIVE_INVALID_VP; 616 break; 617 default: 618 if (rc < 0) { 619 pr_err("OPAL failed to allocate VCPUs order %d, err %lld\n", 620 order, rc); 621 return XIVE_INVALID_VP; 622 } 623 return rc; 624 } 625 } 626 } 627 EXPORT_SYMBOL_GPL(xive_native_alloc_vp_block); 628 629 void xive_native_free_vp_block(u32 vp_base) 630 { 631 s64 rc; 632 633 if (vp_base == XIVE_INVALID_VP) 634 return; 635 636 rc = opal_xive_free_vp_block(vp_base); 637 if (rc < 0) 638 pr_warn("OPAL error %lld freeing VP block\n", rc); 639 } 640 EXPORT_SYMBOL_GPL(xive_native_free_vp_block); 641