1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2007-2011 Freescale Semiconductor, Inc. 4 * 5 * Author: Tony Li <tony.li@freescale.com> 6 * Jason Jin <Jason.jin@freescale.com> 7 * 8 * The hwirq alloc and free code reuse from sysdev/mpic_msi.c 9 */ 10 #include <linux/irq.h> 11 #include <linux/msi.h> 12 #include <linux/pci.h> 13 #include <linux/slab.h> 14 #include <linux/of_address.h> 15 #include <linux/of_irq.h> 16 #include <linux/of_platform.h> 17 #include <linux/interrupt.h> 18 #include <linux/irqdomain.h> 19 #include <linux/seq_file.h> 20 #include <sysdev/fsl_soc.h> 21 #include <asm/hw_irq.h> 22 #include <asm/ppc-pci.h> 23 #include <asm/mpic.h> 24 #include <asm/fsl_hcalls.h> 25 26 #include "fsl_msi.h" 27 #include "fsl_pci.h" 28 29 #define MSIIR_OFFSET_MASK 0xfffff 30 #define MSIIR_IBS_SHIFT 0 31 #define MSIIR_SRS_SHIFT 5 32 #define MSIIR1_IBS_SHIFT 4 33 #define MSIIR1_SRS_SHIFT 0 34 #define MSI_SRS_MASK 0xf 35 #define MSI_IBS_MASK 0x1f 36 37 #define msi_hwirq(msi, msir_index, intr_index) \ 38 ((msir_index) << (msi)->srs_shift | \ 39 ((intr_index) << (msi)->ibs_shift)) 40 41 static LIST_HEAD(msi_head); 42 43 struct fsl_msi_feature { 44 u32 fsl_pic_ip; 45 u32 msiir_offset; /* Offset of MSIIR, relative to start of MSIR bank */ 46 }; 47 48 struct fsl_msi_cascade_data { 49 struct fsl_msi *msi_data; 50 int index; 51 int virq; 52 }; 53 54 static inline u32 fsl_msi_read(u32 __iomem *base, unsigned int reg) 55 { 56 return in_be32(base + (reg >> 2)); 57 } 58 59 /* 60 * We do not need this actually. The MSIR register has been read once 61 * in the cascade interrupt. So, this MSI interrupt has been acked 62 */ 63 static void fsl_msi_end_irq(struct irq_data *d) 64 { 65 } 66 67 static void fsl_msi_print_chip(struct irq_data *irqd, struct seq_file *p) 68 { 69 struct fsl_msi *msi_data = irqd->domain->host_data; 70 irq_hw_number_t hwirq = irqd_to_hwirq(irqd); 71 int cascade_virq, srs; 72 73 srs = (hwirq >> msi_data->srs_shift) & MSI_SRS_MASK; 74 cascade_virq = msi_data->cascade_array[srs]->virq; 75 76 seq_printf(p, " fsl-msi-%d", cascade_virq); 77 } 78 79 80 static struct irq_chip fsl_msi_chip = { 81 .irq_mask = pci_msi_mask_irq, 82 .irq_unmask = pci_msi_unmask_irq, 83 .irq_ack = fsl_msi_end_irq, 84 .irq_print_chip = fsl_msi_print_chip, 85 }; 86 87 static int fsl_msi_host_map(struct irq_domain *h, unsigned int virq, 88 irq_hw_number_t hw) 89 { 90 struct fsl_msi *msi_data = h->host_data; 91 struct irq_chip *chip = &fsl_msi_chip; 92 93 irq_set_status_flags(virq, IRQ_TYPE_EDGE_FALLING); 94 95 irq_set_chip_data(virq, msi_data); 96 irq_set_chip_and_handler(virq, chip, handle_edge_irq); 97 98 return 0; 99 } 100 101 static const struct irq_domain_ops fsl_msi_host_ops = { 102 .map = fsl_msi_host_map, 103 }; 104 105 static int fsl_msi_init_allocator(struct fsl_msi *msi_data) 106 { 107 int rc, hwirq; 108 109 rc = msi_bitmap_alloc(&msi_data->bitmap, NR_MSI_IRQS_MAX, 110 irq_domain_get_of_node(msi_data->irqhost)); 111 if (rc) 112 return rc; 113 114 /* 115 * Reserve all the hwirqs 116 * The available hwirqs will be released in fsl_msi_setup_hwirq() 117 */ 118 for (hwirq = 0; hwirq < NR_MSI_IRQS_MAX; hwirq++) 119 msi_bitmap_reserve_hwirq(&msi_data->bitmap, hwirq); 120 121 return 0; 122 } 123 124 static void fsl_teardown_msi_irqs(struct pci_dev *pdev) 125 { 126 struct msi_desc *entry; 127 struct fsl_msi *msi_data; 128 irq_hw_number_t hwirq; 129 130 msi_for_each_desc(entry, &pdev->dev, MSI_DESC_ASSOCIATED) { 131 hwirq = virq_to_hw(entry->irq); 132 msi_data = irq_get_chip_data(entry->irq); 133 irq_set_msi_desc(entry->irq, NULL); 134 irq_dispose_mapping(entry->irq); 135 msi_bitmap_free_hwirqs(&msi_data->bitmap, hwirq, 1); 136 } 137 } 138 139 static void fsl_compose_msi_msg(struct pci_dev *pdev, int hwirq, 140 struct msi_msg *msg, 141 struct fsl_msi *fsl_msi_data) 142 { 143 struct fsl_msi *msi_data = fsl_msi_data; 144 struct pci_controller *hose = pci_bus_to_host(pdev->bus); 145 u64 address; /* Physical address of the MSIIR */ 146 int len; 147 const __be64 *reg; 148 149 /* If the msi-address-64 property exists, then use it */ 150 reg = of_get_property(hose->dn, "msi-address-64", &len); 151 if (reg && (len == sizeof(u64))) 152 address = be64_to_cpup(reg); 153 else 154 address = fsl_pci_immrbar_base(hose) + msi_data->msiir_offset; 155 156 msg->address_lo = lower_32_bits(address); 157 msg->address_hi = upper_32_bits(address); 158 159 /* 160 * MPIC version 2.0 has erratum PIC1. It causes 161 * that neither MSI nor MSI-X can work fine. 162 * This is a workaround to allow MSI-X to function 163 * properly. It only works for MSI-X, we prevent 164 * MSI on buggy chips in fsl_setup_msi_irqs(). 165 */ 166 if (msi_data->feature & MSI_HW_ERRATA_ENDIAN) 167 msg->data = __swab32(hwirq); 168 else 169 msg->data = hwirq; 170 171 pr_debug("%s: allocated srs: %d, ibs: %d\n", __func__, 172 (hwirq >> msi_data->srs_shift) & MSI_SRS_MASK, 173 (hwirq >> msi_data->ibs_shift) & MSI_IBS_MASK); 174 } 175 176 static int fsl_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) 177 { 178 struct pci_controller *hose = pci_bus_to_host(pdev->bus); 179 struct device_node *np; 180 phandle phandle = 0; 181 int rc, hwirq = -ENOMEM; 182 unsigned int virq; 183 struct msi_desc *entry; 184 struct msi_msg msg; 185 struct fsl_msi *msi_data; 186 187 if (type == PCI_CAP_ID_MSI) { 188 /* 189 * MPIC version 2.0 has erratum PIC1. For now MSI 190 * could not work. So check to prevent MSI from 191 * being used on the board with this erratum. 192 */ 193 list_for_each_entry(msi_data, &msi_head, list) 194 if (msi_data->feature & MSI_HW_ERRATA_ENDIAN) 195 return -EINVAL; 196 } 197 198 /* 199 * If the PCI node has an fsl,msi property, then we need to use it 200 * to find the specific MSI. 201 */ 202 np = of_parse_phandle(hose->dn, "fsl,msi", 0); 203 if (np) { 204 if (of_device_is_compatible(np, "fsl,mpic-msi") || 205 of_device_is_compatible(np, "fsl,vmpic-msi") || 206 of_device_is_compatible(np, "fsl,vmpic-msi-v4.3")) 207 phandle = np->phandle; 208 else { 209 dev_err(&pdev->dev, 210 "node %pOF has an invalid fsl,msi phandle %u\n", 211 hose->dn, np->phandle); 212 of_node_put(np); 213 return -EINVAL; 214 } 215 of_node_put(np); 216 } 217 218 msi_for_each_desc(entry, &pdev->dev, MSI_DESC_NOTASSOCIATED) { 219 /* 220 * Loop over all the MSI devices until we find one that has an 221 * available interrupt. 222 */ 223 list_for_each_entry(msi_data, &msi_head, list) { 224 /* 225 * If the PCI node has an fsl,msi property, then we 226 * restrict our search to the corresponding MSI node. 227 * The simplest way is to skip over MSI nodes with the 228 * wrong phandle. Under the Freescale hypervisor, this 229 * has the additional benefit of skipping over MSI 230 * nodes that are not mapped in the PAMU. 231 */ 232 if (phandle && (phandle != msi_data->phandle)) 233 continue; 234 235 hwirq = msi_bitmap_alloc_hwirqs(&msi_data->bitmap, 1); 236 if (hwirq >= 0) 237 break; 238 } 239 240 if (hwirq < 0) { 241 rc = hwirq; 242 dev_err(&pdev->dev, "could not allocate MSI interrupt\n"); 243 goto out_free; 244 } 245 246 virq = irq_create_mapping(msi_data->irqhost, hwirq); 247 248 if (!virq) { 249 dev_err(&pdev->dev, "fail mapping hwirq %i\n", hwirq); 250 msi_bitmap_free_hwirqs(&msi_data->bitmap, hwirq, 1); 251 rc = -ENOSPC; 252 goto out_free; 253 } 254 /* chip_data is msi_data via host->hostdata in host->map() */ 255 irq_set_msi_desc(virq, entry); 256 257 fsl_compose_msi_msg(pdev, hwirq, &msg, msi_data); 258 pci_write_msi_msg(virq, &msg); 259 } 260 return 0; 261 262 out_free: 263 /* free by the caller of this function */ 264 return rc; 265 } 266 267 static irqreturn_t fsl_msi_cascade(int irq, void *data) 268 { 269 struct fsl_msi *msi_data; 270 int msir_index = -1; 271 u32 msir_value = 0; 272 u32 intr_index; 273 u32 have_shift = 0; 274 struct fsl_msi_cascade_data *cascade_data = data; 275 irqreturn_t ret = IRQ_NONE; 276 277 msi_data = cascade_data->msi_data; 278 279 msir_index = cascade_data->index; 280 281 switch (msi_data->feature & FSL_PIC_IP_MASK) { 282 case FSL_PIC_IP_MPIC: 283 msir_value = fsl_msi_read(msi_data->msi_regs, 284 msir_index * 0x10); 285 break; 286 case FSL_PIC_IP_IPIC: 287 msir_value = fsl_msi_read(msi_data->msi_regs, msir_index * 0x4); 288 break; 289 #ifdef CONFIG_EPAPR_PARAVIRT 290 case FSL_PIC_IP_VMPIC: { 291 unsigned int ret; 292 ret = fh_vmpic_get_msir(virq_to_hw(irq), &msir_value); 293 if (ret) { 294 pr_err("fsl-msi: fh_vmpic_get_msir() failed for " 295 "irq %u (ret=%u)\n", irq, ret); 296 msir_value = 0; 297 } 298 break; 299 } 300 #endif 301 } 302 303 while (msir_value) { 304 int err; 305 intr_index = ffs(msir_value) - 1; 306 307 err = generic_handle_domain_irq(msi_data->irqhost, 308 msi_hwirq(msi_data, msir_index, 309 intr_index + have_shift)); 310 if (!err) 311 ret = IRQ_HANDLED; 312 313 have_shift += intr_index + 1; 314 msir_value = msir_value >> (intr_index + 1); 315 } 316 317 return ret; 318 } 319 320 static int fsl_of_msi_remove(struct platform_device *ofdev) 321 { 322 struct fsl_msi *msi = platform_get_drvdata(ofdev); 323 int virq, i; 324 325 if (msi->list.prev != NULL) 326 list_del(&msi->list); 327 for (i = 0; i < NR_MSI_REG_MAX; i++) { 328 if (msi->cascade_array[i]) { 329 virq = msi->cascade_array[i]->virq; 330 331 BUG_ON(!virq); 332 333 free_irq(virq, msi->cascade_array[i]); 334 kfree(msi->cascade_array[i]); 335 irq_dispose_mapping(virq); 336 } 337 } 338 if (msi->bitmap.bitmap) 339 msi_bitmap_free(&msi->bitmap); 340 if ((msi->feature & FSL_PIC_IP_MASK) != FSL_PIC_IP_VMPIC) 341 iounmap(msi->msi_regs); 342 kfree(msi); 343 344 return 0; 345 } 346 347 static struct lock_class_key fsl_msi_irq_class; 348 static struct lock_class_key fsl_msi_irq_request_class; 349 350 static int fsl_msi_setup_hwirq(struct fsl_msi *msi, struct platform_device *dev, 351 int offset, int irq_index) 352 { 353 struct fsl_msi_cascade_data *cascade_data = NULL; 354 int virt_msir, i, ret; 355 356 virt_msir = irq_of_parse_and_map(dev->dev.of_node, irq_index); 357 if (!virt_msir) { 358 dev_err(&dev->dev, "%s: Cannot translate IRQ index %d\n", 359 __func__, irq_index); 360 return 0; 361 } 362 363 cascade_data = kzalloc(sizeof(struct fsl_msi_cascade_data), GFP_KERNEL); 364 if (!cascade_data) { 365 dev_err(&dev->dev, "No memory for MSI cascade data\n"); 366 return -ENOMEM; 367 } 368 irq_set_lockdep_class(virt_msir, &fsl_msi_irq_class, 369 &fsl_msi_irq_request_class); 370 cascade_data->index = offset; 371 cascade_data->msi_data = msi; 372 cascade_data->virq = virt_msir; 373 msi->cascade_array[irq_index] = cascade_data; 374 375 ret = request_irq(virt_msir, fsl_msi_cascade, IRQF_NO_THREAD, 376 "fsl-msi-cascade", cascade_data); 377 if (ret) { 378 dev_err(&dev->dev, "failed to request_irq(%d), ret = %d\n", 379 virt_msir, ret); 380 return ret; 381 } 382 383 /* Release the hwirqs corresponding to this MSI register */ 384 for (i = 0; i < IRQS_PER_MSI_REG; i++) 385 msi_bitmap_free_hwirqs(&msi->bitmap, 386 msi_hwirq(msi, offset, i), 1); 387 388 return 0; 389 } 390 391 static const struct of_device_id fsl_of_msi_ids[]; 392 static int fsl_of_msi_probe(struct platform_device *dev) 393 { 394 const struct of_device_id *match; 395 struct fsl_msi *msi; 396 struct resource res, msiir; 397 int err, i, j, irq_index, count; 398 const u32 *p; 399 const struct fsl_msi_feature *features; 400 int len; 401 u32 offset; 402 struct pci_controller *phb; 403 404 match = of_match_device(fsl_of_msi_ids, &dev->dev); 405 if (!match) 406 return -EINVAL; 407 features = match->data; 408 409 printk(KERN_DEBUG "Setting up Freescale MSI support\n"); 410 411 msi = kzalloc(sizeof(struct fsl_msi), GFP_KERNEL); 412 if (!msi) { 413 dev_err(&dev->dev, "No memory for MSI structure\n"); 414 return -ENOMEM; 415 } 416 platform_set_drvdata(dev, msi); 417 418 msi->irqhost = irq_domain_add_linear(dev->dev.of_node, 419 NR_MSI_IRQS_MAX, &fsl_msi_host_ops, msi); 420 421 if (msi->irqhost == NULL) { 422 dev_err(&dev->dev, "No memory for MSI irqhost\n"); 423 err = -ENOMEM; 424 goto error_out; 425 } 426 427 /* 428 * Under the Freescale hypervisor, the msi nodes don't have a 'reg' 429 * property. Instead, we use hypercalls to access the MSI. 430 */ 431 if ((features->fsl_pic_ip & FSL_PIC_IP_MASK) != FSL_PIC_IP_VMPIC) { 432 err = of_address_to_resource(dev->dev.of_node, 0, &res); 433 if (err) { 434 dev_err(&dev->dev, "invalid resource for node %pOF\n", 435 dev->dev.of_node); 436 goto error_out; 437 } 438 439 msi->msi_regs = ioremap(res.start, resource_size(&res)); 440 if (!msi->msi_regs) { 441 err = -ENOMEM; 442 dev_err(&dev->dev, "could not map node %pOF\n", 443 dev->dev.of_node); 444 goto error_out; 445 } 446 msi->msiir_offset = 447 features->msiir_offset + (res.start & 0xfffff); 448 449 /* 450 * First read the MSIIR/MSIIR1 offset from dts 451 * On failure use the hardcode MSIIR offset 452 */ 453 if (of_address_to_resource(dev->dev.of_node, 1, &msiir)) 454 msi->msiir_offset = features->msiir_offset + 455 (res.start & MSIIR_OFFSET_MASK); 456 else 457 msi->msiir_offset = msiir.start & MSIIR_OFFSET_MASK; 458 } 459 460 msi->feature = features->fsl_pic_ip; 461 462 /* For erratum PIC1 on MPIC version 2.0*/ 463 if ((features->fsl_pic_ip & FSL_PIC_IP_MASK) == FSL_PIC_IP_MPIC 464 && (fsl_mpic_primary_get_version() == 0x0200)) 465 msi->feature |= MSI_HW_ERRATA_ENDIAN; 466 467 /* 468 * Remember the phandle, so that we can match with any PCI nodes 469 * that have an "fsl,msi" property. 470 */ 471 msi->phandle = dev->dev.of_node->phandle; 472 473 err = fsl_msi_init_allocator(msi); 474 if (err) { 475 dev_err(&dev->dev, "Error allocating MSI bitmap\n"); 476 goto error_out; 477 } 478 479 p = of_get_property(dev->dev.of_node, "msi-available-ranges", &len); 480 481 if (of_device_is_compatible(dev->dev.of_node, "fsl,mpic-msi-v4.3") || 482 of_device_is_compatible(dev->dev.of_node, "fsl,vmpic-msi-v4.3")) { 483 msi->srs_shift = MSIIR1_SRS_SHIFT; 484 msi->ibs_shift = MSIIR1_IBS_SHIFT; 485 if (p) 486 dev_warn(&dev->dev, "%s: dose not support msi-available-ranges property\n", 487 __func__); 488 489 for (irq_index = 0; irq_index < NR_MSI_REG_MSIIR1; 490 irq_index++) { 491 err = fsl_msi_setup_hwirq(msi, dev, 492 irq_index, irq_index); 493 if (err) 494 goto error_out; 495 } 496 } else { 497 static const u32 all_avail[] = 498 { 0, NR_MSI_REG_MSIIR * IRQS_PER_MSI_REG }; 499 500 msi->srs_shift = MSIIR_SRS_SHIFT; 501 msi->ibs_shift = MSIIR_IBS_SHIFT; 502 503 if (p && len % (2 * sizeof(u32)) != 0) { 504 dev_err(&dev->dev, "%s: Malformed msi-available-ranges property\n", 505 __func__); 506 err = -EINVAL; 507 goto error_out; 508 } 509 510 if (!p) { 511 p = all_avail; 512 len = sizeof(all_avail); 513 } 514 515 for (irq_index = 0, i = 0; i < len / (2 * sizeof(u32)); i++) { 516 if (p[i * 2] % IRQS_PER_MSI_REG || 517 p[i * 2 + 1] % IRQS_PER_MSI_REG) { 518 pr_warn("%s: %pOF: msi available range of %u at %u is not IRQ-aligned\n", 519 __func__, dev->dev.of_node, 520 p[i * 2 + 1], p[i * 2]); 521 err = -EINVAL; 522 goto error_out; 523 } 524 525 offset = p[i * 2] / IRQS_PER_MSI_REG; 526 count = p[i * 2 + 1] / IRQS_PER_MSI_REG; 527 528 for (j = 0; j < count; j++, irq_index++) { 529 err = fsl_msi_setup_hwirq(msi, dev, offset + j, 530 irq_index); 531 if (err) 532 goto error_out; 533 } 534 } 535 } 536 537 list_add_tail(&msi->list, &msi_head); 538 539 /* 540 * Apply the MSI ops to all the controllers. 541 * It doesn't hurt to reassign the same ops, 542 * but bail out if we find another MSI driver. 543 */ 544 list_for_each_entry(phb, &hose_list, list_node) { 545 if (!phb->controller_ops.setup_msi_irqs) { 546 phb->controller_ops.setup_msi_irqs = fsl_setup_msi_irqs; 547 phb->controller_ops.teardown_msi_irqs = fsl_teardown_msi_irqs; 548 } else if (phb->controller_ops.setup_msi_irqs != fsl_setup_msi_irqs) { 549 dev_err(&dev->dev, "Different MSI driver already installed!\n"); 550 err = -ENODEV; 551 goto error_out; 552 } 553 } 554 return 0; 555 error_out: 556 fsl_of_msi_remove(dev); 557 return err; 558 } 559 560 static const struct fsl_msi_feature mpic_msi_feature = { 561 .fsl_pic_ip = FSL_PIC_IP_MPIC, 562 .msiir_offset = 0x140, 563 }; 564 565 static const struct fsl_msi_feature ipic_msi_feature = { 566 .fsl_pic_ip = FSL_PIC_IP_IPIC, 567 .msiir_offset = 0x38, 568 }; 569 570 static const struct fsl_msi_feature vmpic_msi_feature = { 571 .fsl_pic_ip = FSL_PIC_IP_VMPIC, 572 .msiir_offset = 0, 573 }; 574 575 static const struct of_device_id fsl_of_msi_ids[] = { 576 { 577 .compatible = "fsl,mpic-msi", 578 .data = &mpic_msi_feature, 579 }, 580 { 581 .compatible = "fsl,mpic-msi-v4.3", 582 .data = &mpic_msi_feature, 583 }, 584 { 585 .compatible = "fsl,ipic-msi", 586 .data = &ipic_msi_feature, 587 }, 588 #ifdef CONFIG_EPAPR_PARAVIRT 589 { 590 .compatible = "fsl,vmpic-msi", 591 .data = &vmpic_msi_feature, 592 }, 593 { 594 .compatible = "fsl,vmpic-msi-v4.3", 595 .data = &vmpic_msi_feature, 596 }, 597 #endif 598 {} 599 }; 600 601 static struct platform_driver fsl_of_msi_driver = { 602 .driver = { 603 .name = "fsl-msi", 604 .of_match_table = fsl_of_msi_ids, 605 }, 606 .probe = fsl_of_msi_probe, 607 .remove = fsl_of_msi_remove, 608 }; 609 610 static __init int fsl_of_msi_init(void) 611 { 612 return platform_driver_register(&fsl_of_msi_driver); 613 } 614 615 subsys_initcall(fsl_of_msi_init); 616