1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCIe driver for Renesas R-Car SoCs 4 * Copyright (C) 2014-2020 Renesas Electronics Europe Ltd 5 * 6 * Based on: 7 * arch/sh/drivers/pci/pcie-sh7786.c 8 * arch/sh/drivers/pci/ops-sh7786.c 9 * Copyright (C) 2009 - 2011 Paul Mundt 10 * 11 * Author: Phil Edworthy <phil.edworthy@renesas.com> 12 */ 13 14 #include <linux/bitops.h> 15 #include <linux/clk.h> 16 #include <linux/clk-provider.h> 17 #include <linux/delay.h> 18 #include <linux/interrupt.h> 19 #include <linux/irq.h> 20 #include <linux/irqdomain.h> 21 #include <linux/kernel.h> 22 #include <linux/init.h> 23 #include <linux/iopoll.h> 24 #include <linux/msi.h> 25 #include <linux/of_address.h> 26 #include <linux/of_irq.h> 27 #include <linux/of_platform.h> 28 #include <linux/pci.h> 29 #include <linux/phy/phy.h> 30 #include <linux/platform_device.h> 31 #include <linux/pm_runtime.h> 32 33 #include "pcie-rcar.h" 34 35 struct rcar_msi { 36 DECLARE_BITMAP(used, INT_PCI_MSI_NR); 37 struct irq_domain *domain; 38 struct mutex map_lock; 39 spinlock_t mask_lock; 40 int irq1; 41 int irq2; 42 }; 43 44 #ifdef CONFIG_ARM 45 /* 46 * Here we keep a static copy of the remapped PCIe controller address. 47 * This is only used on aarch32 systems, all of which have one single 48 * PCIe controller, to provide quick access to the PCIe controller in 49 * the L1 link state fixup function, called from the ARM fault handler. 50 */ 51 static void __iomem *pcie_base; 52 /* 53 * Static copy of bus clock pointer, so we can check whether the clock 54 * is enabled or not. 55 */ 56 static struct clk *pcie_bus_clk; 57 #endif 58 59 /* Structure representing the PCIe interface */ 60 struct rcar_pcie_host { 61 struct rcar_pcie pcie; 62 struct phy *phy; 63 struct clk *bus_clk; 64 struct rcar_msi msi; 65 int (*phy_init_fn)(struct rcar_pcie_host *host); 66 }; 67 68 static struct rcar_pcie_host *msi_to_host(struct rcar_msi *msi) 69 { 70 return container_of(msi, struct rcar_pcie_host, msi); 71 } 72 73 static u32 rcar_read_conf(struct rcar_pcie *pcie, int where) 74 { 75 unsigned int shift = BITS_PER_BYTE * (where & 3); 76 u32 val = rcar_pci_read_reg(pcie, where & ~3); 77 78 return val >> shift; 79 } 80 81 /* Serialization is provided by 'pci_lock' in drivers/pci/access.c */ 82 static int rcar_pcie_config_access(struct rcar_pcie_host *host, 83 unsigned char access_type, struct pci_bus *bus, 84 unsigned int devfn, int where, u32 *data) 85 { 86 struct rcar_pcie *pcie = &host->pcie; 87 unsigned int dev, func, reg, index; 88 89 dev = PCI_SLOT(devfn); 90 func = PCI_FUNC(devfn); 91 reg = where & ~3; 92 index = reg / 4; 93 94 /* 95 * While each channel has its own memory-mapped extended config 96 * space, it's generally only accessible when in endpoint mode. 97 * When in root complex mode, the controller is unable to target 98 * itself with either type 0 or type 1 accesses, and indeed, any 99 * controller initiated target transfer to its own config space 100 * result in a completer abort. 101 * 102 * Each channel effectively only supports a single device, but as 103 * the same channel <-> device access works for any PCI_SLOT() 104 * value, we cheat a bit here and bind the controller's config 105 * space to devfn 0 in order to enable self-enumeration. In this 106 * case the regular ECAR/ECDR path is sidelined and the mangled 107 * config access itself is initiated as an internal bus transaction. 108 */ 109 if (pci_is_root_bus(bus)) { 110 if (dev != 0) 111 return PCIBIOS_DEVICE_NOT_FOUND; 112 113 if (access_type == RCAR_PCI_ACCESS_READ) 114 *data = rcar_pci_read_reg(pcie, PCICONF(index)); 115 else 116 rcar_pci_write_reg(pcie, *data, PCICONF(index)); 117 118 return PCIBIOS_SUCCESSFUL; 119 } 120 121 /* Clear errors */ 122 rcar_pci_write_reg(pcie, rcar_pci_read_reg(pcie, PCIEERRFR), PCIEERRFR); 123 124 /* Set the PIO address */ 125 rcar_pci_write_reg(pcie, PCIE_CONF_BUS(bus->number) | 126 PCIE_CONF_DEV(dev) | PCIE_CONF_FUNC(func) | reg, PCIECAR); 127 128 /* Enable the configuration access */ 129 if (pci_is_root_bus(bus->parent)) 130 rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE0, PCIECCTLR); 131 else 132 rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE1, PCIECCTLR); 133 134 /* Check for errors */ 135 if (rcar_pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST) 136 return PCIBIOS_DEVICE_NOT_FOUND; 137 138 /* Check for master and target aborts */ 139 if (rcar_read_conf(pcie, RCONF(PCI_STATUS)) & 140 (PCI_STATUS_REC_MASTER_ABORT | PCI_STATUS_REC_TARGET_ABORT)) 141 return PCIBIOS_DEVICE_NOT_FOUND; 142 143 if (access_type == RCAR_PCI_ACCESS_READ) 144 *data = rcar_pci_read_reg(pcie, PCIECDR); 145 else 146 rcar_pci_write_reg(pcie, *data, PCIECDR); 147 148 /* Disable the configuration access */ 149 rcar_pci_write_reg(pcie, 0, PCIECCTLR); 150 151 return PCIBIOS_SUCCESSFUL; 152 } 153 154 static int rcar_pcie_read_conf(struct pci_bus *bus, unsigned int devfn, 155 int where, int size, u32 *val) 156 { 157 struct rcar_pcie_host *host = bus->sysdata; 158 int ret; 159 160 ret = rcar_pcie_config_access(host, RCAR_PCI_ACCESS_READ, 161 bus, devfn, where, val); 162 if (ret != PCIBIOS_SUCCESSFUL) { 163 *val = 0xffffffff; 164 return ret; 165 } 166 167 if (size == 1) 168 *val = (*val >> (BITS_PER_BYTE * (where & 3))) & 0xff; 169 else if (size == 2) 170 *val = (*val >> (BITS_PER_BYTE * (where & 2))) & 0xffff; 171 172 dev_dbg(&bus->dev, "pcie-config-read: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08x\n", 173 bus->number, devfn, where, size, *val); 174 175 return ret; 176 } 177 178 /* Serialization is provided by 'pci_lock' in drivers/pci/access.c */ 179 static int rcar_pcie_write_conf(struct pci_bus *bus, unsigned int devfn, 180 int where, int size, u32 val) 181 { 182 struct rcar_pcie_host *host = bus->sysdata; 183 unsigned int shift; 184 u32 data; 185 int ret; 186 187 ret = rcar_pcie_config_access(host, RCAR_PCI_ACCESS_READ, 188 bus, devfn, where, &data); 189 if (ret != PCIBIOS_SUCCESSFUL) 190 return ret; 191 192 dev_dbg(&bus->dev, "pcie-config-write: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08x\n", 193 bus->number, devfn, where, size, val); 194 195 if (size == 1) { 196 shift = BITS_PER_BYTE * (where & 3); 197 data &= ~(0xff << shift); 198 data |= ((val & 0xff) << shift); 199 } else if (size == 2) { 200 shift = BITS_PER_BYTE * (where & 2); 201 data &= ~(0xffff << shift); 202 data |= ((val & 0xffff) << shift); 203 } else 204 data = val; 205 206 ret = rcar_pcie_config_access(host, RCAR_PCI_ACCESS_WRITE, 207 bus, devfn, where, &data); 208 209 return ret; 210 } 211 212 static struct pci_ops rcar_pcie_ops = { 213 .read = rcar_pcie_read_conf, 214 .write = rcar_pcie_write_conf, 215 }; 216 217 static void rcar_pcie_force_speedup(struct rcar_pcie *pcie) 218 { 219 struct device *dev = pcie->dev; 220 unsigned int timeout = 1000; 221 u32 macsr; 222 223 if ((rcar_pci_read_reg(pcie, MACS2R) & LINK_SPEED) != LINK_SPEED_5_0GTS) 224 return; 225 226 if (rcar_pci_read_reg(pcie, MACCTLR) & SPEED_CHANGE) { 227 dev_err(dev, "Speed change already in progress\n"); 228 return; 229 } 230 231 macsr = rcar_pci_read_reg(pcie, MACSR); 232 if ((macsr & LINK_SPEED) == LINK_SPEED_5_0GTS) 233 goto done; 234 235 /* Set target link speed to 5.0 GT/s */ 236 rcar_rmw32(pcie, EXPCAP(12), PCI_EXP_LNKSTA_CLS, 237 PCI_EXP_LNKSTA_CLS_5_0GB); 238 239 /* Set speed change reason as intentional factor */ 240 rcar_rmw32(pcie, MACCGSPSETR, SPCNGRSN, 0); 241 242 /* Clear SPCHGFIN, SPCHGSUC, and SPCHGFAIL */ 243 if (macsr & (SPCHGFIN | SPCHGSUC | SPCHGFAIL)) 244 rcar_pci_write_reg(pcie, macsr, MACSR); 245 246 /* Start link speed change */ 247 rcar_rmw32(pcie, MACCTLR, SPEED_CHANGE, SPEED_CHANGE); 248 249 while (timeout--) { 250 macsr = rcar_pci_read_reg(pcie, MACSR); 251 if (macsr & SPCHGFIN) { 252 /* Clear the interrupt bits */ 253 rcar_pci_write_reg(pcie, macsr, MACSR); 254 255 if (macsr & SPCHGFAIL) 256 dev_err(dev, "Speed change failed\n"); 257 258 goto done; 259 } 260 261 msleep(1); 262 } 263 264 dev_err(dev, "Speed change timed out\n"); 265 266 done: 267 dev_info(dev, "Current link speed is %s GT/s\n", 268 (macsr & LINK_SPEED) == LINK_SPEED_5_0GTS ? "5" : "2.5"); 269 } 270 271 static void rcar_pcie_hw_enable(struct rcar_pcie_host *host) 272 { 273 struct rcar_pcie *pcie = &host->pcie; 274 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(host); 275 struct resource_entry *win; 276 LIST_HEAD(res); 277 int i = 0; 278 279 /* Try setting 5 GT/s link speed */ 280 rcar_pcie_force_speedup(pcie); 281 282 /* Setup PCI resources */ 283 resource_list_for_each_entry(win, &bridge->windows) { 284 struct resource *res = win->res; 285 286 if (!res->flags) 287 continue; 288 289 switch (resource_type(res)) { 290 case IORESOURCE_IO: 291 case IORESOURCE_MEM: 292 rcar_pcie_set_outbound(pcie, i, win); 293 i++; 294 break; 295 } 296 } 297 } 298 299 static int rcar_pcie_enable(struct rcar_pcie_host *host) 300 { 301 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(host); 302 303 rcar_pcie_hw_enable(host); 304 305 pci_add_flags(PCI_REASSIGN_ALL_BUS); 306 307 bridge->sysdata = host; 308 bridge->ops = &rcar_pcie_ops; 309 310 return pci_host_probe(bridge); 311 } 312 313 static int phy_wait_for_ack(struct rcar_pcie *pcie) 314 { 315 struct device *dev = pcie->dev; 316 unsigned int timeout = 100; 317 318 while (timeout--) { 319 if (rcar_pci_read_reg(pcie, H1_PCIEPHYADRR) & PHY_ACK) 320 return 0; 321 322 udelay(100); 323 } 324 325 dev_err(dev, "Access to PCIe phy timed out\n"); 326 327 return -ETIMEDOUT; 328 } 329 330 static void phy_write_reg(struct rcar_pcie *pcie, 331 unsigned int rate, u32 addr, 332 unsigned int lane, u32 data) 333 { 334 u32 phyaddr; 335 336 phyaddr = WRITE_CMD | 337 ((rate & 1) << RATE_POS) | 338 ((lane & 0xf) << LANE_POS) | 339 ((addr & 0xff) << ADR_POS); 340 341 /* Set write data */ 342 rcar_pci_write_reg(pcie, data, H1_PCIEPHYDOUTR); 343 rcar_pci_write_reg(pcie, phyaddr, H1_PCIEPHYADRR); 344 345 /* Ignore errors as they will be dealt with if the data link is down */ 346 phy_wait_for_ack(pcie); 347 348 /* Clear command */ 349 rcar_pci_write_reg(pcie, 0, H1_PCIEPHYDOUTR); 350 rcar_pci_write_reg(pcie, 0, H1_PCIEPHYADRR); 351 352 /* Ignore errors as they will be dealt with if the data link is down */ 353 phy_wait_for_ack(pcie); 354 } 355 356 static int rcar_pcie_hw_init(struct rcar_pcie *pcie) 357 { 358 int err; 359 360 /* Begin initialization */ 361 rcar_pci_write_reg(pcie, 0, PCIETCTLR); 362 363 /* Set mode */ 364 rcar_pci_write_reg(pcie, 1, PCIEMSR); 365 366 err = rcar_pcie_wait_for_phyrdy(pcie); 367 if (err) 368 return err; 369 370 /* 371 * Initial header for port config space is type 1, set the device 372 * class to match. Hardware takes care of propagating the IDSETR 373 * settings, so there is no need to bother with a quirk. 374 */ 375 rcar_pci_write_reg(pcie, PCI_CLASS_BRIDGE_PCI << 16, IDSETR1); 376 377 /* 378 * Setup Secondary Bus Number & Subordinate Bus Number, even though 379 * they aren't used, to avoid bridge being detected as broken. 380 */ 381 rcar_rmw32(pcie, RCONF(PCI_SECONDARY_BUS), 0xff, 1); 382 rcar_rmw32(pcie, RCONF(PCI_SUBORDINATE_BUS), 0xff, 1); 383 384 /* Initialize default capabilities. */ 385 rcar_rmw32(pcie, REXPCAP(0), 0xff, PCI_CAP_ID_EXP); 386 rcar_rmw32(pcie, REXPCAP(PCI_EXP_FLAGS), 387 PCI_EXP_FLAGS_TYPE, PCI_EXP_TYPE_ROOT_PORT << 4); 388 rcar_rmw32(pcie, RCONF(PCI_HEADER_TYPE), 0x7f, 389 PCI_HEADER_TYPE_BRIDGE); 390 391 /* Enable data link layer active state reporting */ 392 rcar_rmw32(pcie, REXPCAP(PCI_EXP_LNKCAP), PCI_EXP_LNKCAP_DLLLARC, 393 PCI_EXP_LNKCAP_DLLLARC); 394 395 /* Write out the physical slot number = 0 */ 396 rcar_rmw32(pcie, REXPCAP(PCI_EXP_SLTCAP), PCI_EXP_SLTCAP_PSN, 0); 397 398 /* Set the completion timer timeout to the maximum 50ms. */ 399 rcar_rmw32(pcie, TLCTLR + 1, 0x3f, 50); 400 401 /* Terminate list of capabilities (Next Capability Offset=0) */ 402 rcar_rmw32(pcie, RVCCAP(0), 0xfff00000, 0); 403 404 /* Enable MSI */ 405 if (IS_ENABLED(CONFIG_PCI_MSI)) 406 rcar_pci_write_reg(pcie, 0x801f0000, PCIEMSITXR); 407 408 rcar_pci_write_reg(pcie, MACCTLR_INIT_VAL, MACCTLR); 409 410 /* Finish initialization - establish a PCI Express link */ 411 rcar_pci_write_reg(pcie, CFINIT, PCIETCTLR); 412 413 /* This will timeout if we don't have a link. */ 414 err = rcar_pcie_wait_for_dl(pcie); 415 if (err) 416 return err; 417 418 /* Enable INTx interrupts */ 419 rcar_rmw32(pcie, PCIEINTXR, 0, 0xF << 8); 420 421 wmb(); 422 423 return 0; 424 } 425 426 static int rcar_pcie_phy_init_h1(struct rcar_pcie_host *host) 427 { 428 struct rcar_pcie *pcie = &host->pcie; 429 430 /* Initialize the phy */ 431 phy_write_reg(pcie, 0, 0x42, 0x1, 0x0EC34191); 432 phy_write_reg(pcie, 1, 0x42, 0x1, 0x0EC34180); 433 phy_write_reg(pcie, 0, 0x43, 0x1, 0x00210188); 434 phy_write_reg(pcie, 1, 0x43, 0x1, 0x00210188); 435 phy_write_reg(pcie, 0, 0x44, 0x1, 0x015C0014); 436 phy_write_reg(pcie, 1, 0x44, 0x1, 0x015C0014); 437 phy_write_reg(pcie, 1, 0x4C, 0x1, 0x786174A0); 438 phy_write_reg(pcie, 1, 0x4D, 0x1, 0x048000BB); 439 phy_write_reg(pcie, 0, 0x51, 0x1, 0x079EC062); 440 phy_write_reg(pcie, 0, 0x52, 0x1, 0x20000000); 441 phy_write_reg(pcie, 1, 0x52, 0x1, 0x20000000); 442 phy_write_reg(pcie, 1, 0x56, 0x1, 0x00003806); 443 444 phy_write_reg(pcie, 0, 0x60, 0x1, 0x004B03A5); 445 phy_write_reg(pcie, 0, 0x64, 0x1, 0x3F0F1F0F); 446 phy_write_reg(pcie, 0, 0x66, 0x1, 0x00008000); 447 448 return 0; 449 } 450 451 static int rcar_pcie_phy_init_gen2(struct rcar_pcie_host *host) 452 { 453 struct rcar_pcie *pcie = &host->pcie; 454 455 /* 456 * These settings come from the R-Car Series, 2nd Generation User's 457 * Manual, section 50.3.1 (2) Initialization of the physical layer. 458 */ 459 rcar_pci_write_reg(pcie, 0x000f0030, GEN2_PCIEPHYADDR); 460 rcar_pci_write_reg(pcie, 0x00381203, GEN2_PCIEPHYDATA); 461 rcar_pci_write_reg(pcie, 0x00000001, GEN2_PCIEPHYCTRL); 462 rcar_pci_write_reg(pcie, 0x00000006, GEN2_PCIEPHYCTRL); 463 464 rcar_pci_write_reg(pcie, 0x000f0054, GEN2_PCIEPHYADDR); 465 /* The following value is for DC connection, no termination resistor */ 466 rcar_pci_write_reg(pcie, 0x13802007, GEN2_PCIEPHYDATA); 467 rcar_pci_write_reg(pcie, 0x00000001, GEN2_PCIEPHYCTRL); 468 rcar_pci_write_reg(pcie, 0x00000006, GEN2_PCIEPHYCTRL); 469 470 return 0; 471 } 472 473 static int rcar_pcie_phy_init_gen3(struct rcar_pcie_host *host) 474 { 475 int err; 476 477 err = phy_init(host->phy); 478 if (err) 479 return err; 480 481 err = phy_power_on(host->phy); 482 if (err) 483 phy_exit(host->phy); 484 485 return err; 486 } 487 488 static irqreturn_t rcar_pcie_msi_irq(int irq, void *data) 489 { 490 struct rcar_pcie_host *host = data; 491 struct rcar_pcie *pcie = &host->pcie; 492 struct rcar_msi *msi = &host->msi; 493 struct device *dev = pcie->dev; 494 unsigned long reg; 495 496 reg = rcar_pci_read_reg(pcie, PCIEMSIFR); 497 498 /* MSI & INTx share an interrupt - we only handle MSI here */ 499 if (!reg) 500 return IRQ_NONE; 501 502 while (reg) { 503 unsigned int index = find_first_bit(®, 32); 504 int ret; 505 506 ret = generic_handle_domain_irq(msi->domain->parent, index); 507 if (ret) { 508 /* Unknown MSI, just clear it */ 509 dev_dbg(dev, "unexpected MSI\n"); 510 rcar_pci_write_reg(pcie, BIT(index), PCIEMSIFR); 511 } 512 513 /* see if there's any more pending in this vector */ 514 reg = rcar_pci_read_reg(pcie, PCIEMSIFR); 515 } 516 517 return IRQ_HANDLED; 518 } 519 520 static void rcar_msi_top_irq_ack(struct irq_data *d) 521 { 522 irq_chip_ack_parent(d); 523 } 524 525 static void rcar_msi_top_irq_mask(struct irq_data *d) 526 { 527 pci_msi_mask_irq(d); 528 irq_chip_mask_parent(d); 529 } 530 531 static void rcar_msi_top_irq_unmask(struct irq_data *d) 532 { 533 pci_msi_unmask_irq(d); 534 irq_chip_unmask_parent(d); 535 } 536 537 static struct irq_chip rcar_msi_top_chip = { 538 .name = "PCIe MSI", 539 .irq_ack = rcar_msi_top_irq_ack, 540 .irq_mask = rcar_msi_top_irq_mask, 541 .irq_unmask = rcar_msi_top_irq_unmask, 542 }; 543 544 static void rcar_msi_irq_ack(struct irq_data *d) 545 { 546 struct rcar_msi *msi = irq_data_get_irq_chip_data(d); 547 struct rcar_pcie *pcie = &msi_to_host(msi)->pcie; 548 549 /* clear the interrupt */ 550 rcar_pci_write_reg(pcie, BIT(d->hwirq), PCIEMSIFR); 551 } 552 553 static void rcar_msi_irq_mask(struct irq_data *d) 554 { 555 struct rcar_msi *msi = irq_data_get_irq_chip_data(d); 556 struct rcar_pcie *pcie = &msi_to_host(msi)->pcie; 557 unsigned long flags; 558 u32 value; 559 560 spin_lock_irqsave(&msi->mask_lock, flags); 561 value = rcar_pci_read_reg(pcie, PCIEMSIIER); 562 value &= ~BIT(d->hwirq); 563 rcar_pci_write_reg(pcie, value, PCIEMSIIER); 564 spin_unlock_irqrestore(&msi->mask_lock, flags); 565 } 566 567 static void rcar_msi_irq_unmask(struct irq_data *d) 568 { 569 struct rcar_msi *msi = irq_data_get_irq_chip_data(d); 570 struct rcar_pcie *pcie = &msi_to_host(msi)->pcie; 571 unsigned long flags; 572 u32 value; 573 574 spin_lock_irqsave(&msi->mask_lock, flags); 575 value = rcar_pci_read_reg(pcie, PCIEMSIIER); 576 value |= BIT(d->hwirq); 577 rcar_pci_write_reg(pcie, value, PCIEMSIIER); 578 spin_unlock_irqrestore(&msi->mask_lock, flags); 579 } 580 581 static int rcar_msi_set_affinity(struct irq_data *d, const struct cpumask *mask, bool force) 582 { 583 return -EINVAL; 584 } 585 586 static void rcar_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) 587 { 588 struct rcar_msi *msi = irq_data_get_irq_chip_data(data); 589 struct rcar_pcie *pcie = &msi_to_host(msi)->pcie; 590 591 msg->address_lo = rcar_pci_read_reg(pcie, PCIEMSIALR) & ~MSIFE; 592 msg->address_hi = rcar_pci_read_reg(pcie, PCIEMSIAUR); 593 msg->data = data->hwirq; 594 } 595 596 static struct irq_chip rcar_msi_bottom_chip = { 597 .name = "Rcar MSI", 598 .irq_ack = rcar_msi_irq_ack, 599 .irq_mask = rcar_msi_irq_mask, 600 .irq_unmask = rcar_msi_irq_unmask, 601 .irq_set_affinity = rcar_msi_set_affinity, 602 .irq_compose_msi_msg = rcar_compose_msi_msg, 603 }; 604 605 static int rcar_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, 606 unsigned int nr_irqs, void *args) 607 { 608 struct rcar_msi *msi = domain->host_data; 609 unsigned int i; 610 int hwirq; 611 612 mutex_lock(&msi->map_lock); 613 614 hwirq = bitmap_find_free_region(msi->used, INT_PCI_MSI_NR, order_base_2(nr_irqs)); 615 616 mutex_unlock(&msi->map_lock); 617 618 if (hwirq < 0) 619 return -ENOSPC; 620 621 for (i = 0; i < nr_irqs; i++) 622 irq_domain_set_info(domain, virq + i, hwirq + i, 623 &rcar_msi_bottom_chip, domain->host_data, 624 handle_edge_irq, NULL, NULL); 625 626 return 0; 627 } 628 629 static void rcar_msi_domain_free(struct irq_domain *domain, unsigned int virq, 630 unsigned int nr_irqs) 631 { 632 struct irq_data *d = irq_domain_get_irq_data(domain, virq); 633 struct rcar_msi *msi = domain->host_data; 634 635 mutex_lock(&msi->map_lock); 636 637 bitmap_release_region(msi->used, d->hwirq, order_base_2(nr_irqs)); 638 639 mutex_unlock(&msi->map_lock); 640 } 641 642 static const struct irq_domain_ops rcar_msi_domain_ops = { 643 .alloc = rcar_msi_domain_alloc, 644 .free = rcar_msi_domain_free, 645 }; 646 647 static struct msi_domain_info rcar_msi_info = { 648 .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | 649 MSI_FLAG_MULTI_PCI_MSI), 650 .chip = &rcar_msi_top_chip, 651 }; 652 653 static int rcar_allocate_domains(struct rcar_msi *msi) 654 { 655 struct rcar_pcie *pcie = &msi_to_host(msi)->pcie; 656 struct fwnode_handle *fwnode = dev_fwnode(pcie->dev); 657 struct irq_domain *parent; 658 659 parent = irq_domain_create_linear(fwnode, INT_PCI_MSI_NR, 660 &rcar_msi_domain_ops, msi); 661 if (!parent) { 662 dev_err(pcie->dev, "failed to create IRQ domain\n"); 663 return -ENOMEM; 664 } 665 irq_domain_update_bus_token(parent, DOMAIN_BUS_NEXUS); 666 667 msi->domain = pci_msi_create_irq_domain(fwnode, &rcar_msi_info, parent); 668 if (!msi->domain) { 669 dev_err(pcie->dev, "failed to create MSI domain\n"); 670 irq_domain_remove(parent); 671 return -ENOMEM; 672 } 673 674 return 0; 675 } 676 677 static void rcar_free_domains(struct rcar_msi *msi) 678 { 679 struct irq_domain *parent = msi->domain->parent; 680 681 irq_domain_remove(msi->domain); 682 irq_domain_remove(parent); 683 } 684 685 static int rcar_pcie_enable_msi(struct rcar_pcie_host *host) 686 { 687 struct rcar_pcie *pcie = &host->pcie; 688 struct device *dev = pcie->dev; 689 struct rcar_msi *msi = &host->msi; 690 struct resource res; 691 int err; 692 693 mutex_init(&msi->map_lock); 694 spin_lock_init(&msi->mask_lock); 695 696 err = of_address_to_resource(dev->of_node, 0, &res); 697 if (err) 698 return err; 699 700 err = rcar_allocate_domains(msi); 701 if (err) 702 return err; 703 704 /* Two irqs are for MSI, but they are also used for non-MSI irqs */ 705 err = devm_request_irq(dev, msi->irq1, rcar_pcie_msi_irq, 706 IRQF_SHARED | IRQF_NO_THREAD, 707 rcar_msi_bottom_chip.name, host); 708 if (err < 0) { 709 dev_err(dev, "failed to request IRQ: %d\n", err); 710 goto err; 711 } 712 713 err = devm_request_irq(dev, msi->irq2, rcar_pcie_msi_irq, 714 IRQF_SHARED | IRQF_NO_THREAD, 715 rcar_msi_bottom_chip.name, host); 716 if (err < 0) { 717 dev_err(dev, "failed to request IRQ: %d\n", err); 718 goto err; 719 } 720 721 /* disable all MSIs */ 722 rcar_pci_write_reg(pcie, 0, PCIEMSIIER); 723 724 /* 725 * Setup MSI data target using RC base address address, which 726 * is guaranteed to be in the low 32bit range on any RCar HW. 727 */ 728 rcar_pci_write_reg(pcie, lower_32_bits(res.start) | MSIFE, PCIEMSIALR); 729 rcar_pci_write_reg(pcie, upper_32_bits(res.start), PCIEMSIAUR); 730 731 return 0; 732 733 err: 734 rcar_free_domains(msi); 735 return err; 736 } 737 738 static void rcar_pcie_teardown_msi(struct rcar_pcie_host *host) 739 { 740 struct rcar_pcie *pcie = &host->pcie; 741 742 /* Disable all MSI interrupts */ 743 rcar_pci_write_reg(pcie, 0, PCIEMSIIER); 744 745 /* Disable address decoding of the MSI interrupt, MSIFE */ 746 rcar_pci_write_reg(pcie, 0, PCIEMSIALR); 747 748 rcar_free_domains(&host->msi); 749 } 750 751 static int rcar_pcie_get_resources(struct rcar_pcie_host *host) 752 { 753 struct rcar_pcie *pcie = &host->pcie; 754 struct device *dev = pcie->dev; 755 struct resource res; 756 int err, i; 757 758 host->phy = devm_phy_optional_get(dev, "pcie"); 759 if (IS_ERR(host->phy)) 760 return PTR_ERR(host->phy); 761 762 err = of_address_to_resource(dev->of_node, 0, &res); 763 if (err) 764 return err; 765 766 pcie->base = devm_ioremap_resource(dev, &res); 767 if (IS_ERR(pcie->base)) 768 return PTR_ERR(pcie->base); 769 770 host->bus_clk = devm_clk_get(dev, "pcie_bus"); 771 if (IS_ERR(host->bus_clk)) { 772 dev_err(dev, "cannot get pcie bus clock\n"); 773 return PTR_ERR(host->bus_clk); 774 } 775 776 i = irq_of_parse_and_map(dev->of_node, 0); 777 if (!i) { 778 dev_err(dev, "cannot get platform resources for msi interrupt\n"); 779 err = -ENOENT; 780 goto err_irq1; 781 } 782 host->msi.irq1 = i; 783 784 i = irq_of_parse_and_map(dev->of_node, 1); 785 if (!i) { 786 dev_err(dev, "cannot get platform resources for msi interrupt\n"); 787 err = -ENOENT; 788 goto err_irq2; 789 } 790 host->msi.irq2 = i; 791 792 #ifdef CONFIG_ARM 793 /* Cache static copy for L1 link state fixup hook on aarch32 */ 794 pcie_base = pcie->base; 795 pcie_bus_clk = host->bus_clk; 796 #endif 797 798 return 0; 799 800 err_irq2: 801 irq_dispose_mapping(host->msi.irq1); 802 err_irq1: 803 return err; 804 } 805 806 static int rcar_pcie_inbound_ranges(struct rcar_pcie *pcie, 807 struct resource_entry *entry, 808 int *index) 809 { 810 u64 restype = entry->res->flags; 811 u64 cpu_addr = entry->res->start; 812 u64 cpu_end = entry->res->end; 813 u64 pci_addr = entry->res->start - entry->offset; 814 u32 flags = LAM_64BIT | LAR_ENABLE; 815 u64 mask; 816 u64 size = resource_size(entry->res); 817 int idx = *index; 818 819 if (restype & IORESOURCE_PREFETCH) 820 flags |= LAM_PREFETCH; 821 822 while (cpu_addr < cpu_end) { 823 if (idx >= MAX_NR_INBOUND_MAPS - 1) { 824 dev_err(pcie->dev, "Failed to map inbound regions!\n"); 825 return -EINVAL; 826 } 827 /* 828 * If the size of the range is larger than the alignment of 829 * the start address, we have to use multiple entries to 830 * perform the mapping. 831 */ 832 if (cpu_addr > 0) { 833 unsigned long nr_zeros = __ffs64(cpu_addr); 834 u64 alignment = 1ULL << nr_zeros; 835 836 size = min(size, alignment); 837 } 838 /* Hardware supports max 4GiB inbound region */ 839 size = min(size, 1ULL << 32); 840 841 mask = roundup_pow_of_two(size) - 1; 842 mask &= ~0xf; 843 844 rcar_pcie_set_inbound(pcie, cpu_addr, pci_addr, 845 lower_32_bits(mask) | flags, idx, true); 846 847 pci_addr += size; 848 cpu_addr += size; 849 idx += 2; 850 } 851 *index = idx; 852 853 return 0; 854 } 855 856 static int rcar_pcie_parse_map_dma_ranges(struct rcar_pcie_host *host) 857 { 858 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(host); 859 struct resource_entry *entry; 860 int index = 0, err = 0; 861 862 resource_list_for_each_entry(entry, &bridge->dma_ranges) { 863 err = rcar_pcie_inbound_ranges(&host->pcie, entry, &index); 864 if (err) 865 break; 866 } 867 868 return err; 869 } 870 871 static const struct of_device_id rcar_pcie_of_match[] = { 872 { .compatible = "renesas,pcie-r8a7779", 873 .data = rcar_pcie_phy_init_h1 }, 874 { .compatible = "renesas,pcie-r8a7790", 875 .data = rcar_pcie_phy_init_gen2 }, 876 { .compatible = "renesas,pcie-r8a7791", 877 .data = rcar_pcie_phy_init_gen2 }, 878 { .compatible = "renesas,pcie-rcar-gen2", 879 .data = rcar_pcie_phy_init_gen2 }, 880 { .compatible = "renesas,pcie-r8a7795", 881 .data = rcar_pcie_phy_init_gen3 }, 882 { .compatible = "renesas,pcie-rcar-gen3", 883 .data = rcar_pcie_phy_init_gen3 }, 884 {}, 885 }; 886 887 static int rcar_pcie_probe(struct platform_device *pdev) 888 { 889 struct device *dev = &pdev->dev; 890 struct rcar_pcie_host *host; 891 struct rcar_pcie *pcie; 892 u32 data; 893 int err; 894 struct pci_host_bridge *bridge; 895 896 bridge = devm_pci_alloc_host_bridge(dev, sizeof(*host)); 897 if (!bridge) 898 return -ENOMEM; 899 900 host = pci_host_bridge_priv(bridge); 901 pcie = &host->pcie; 902 pcie->dev = dev; 903 platform_set_drvdata(pdev, host); 904 905 pm_runtime_enable(pcie->dev); 906 err = pm_runtime_get_sync(pcie->dev); 907 if (err < 0) { 908 dev_err(pcie->dev, "pm_runtime_get_sync failed\n"); 909 goto err_pm_put; 910 } 911 912 err = rcar_pcie_get_resources(host); 913 if (err < 0) { 914 dev_err(dev, "failed to request resources: %d\n", err); 915 goto err_pm_put; 916 } 917 918 err = clk_prepare_enable(host->bus_clk); 919 if (err) { 920 dev_err(dev, "failed to enable bus clock: %d\n", err); 921 goto err_unmap_msi_irqs; 922 } 923 924 err = rcar_pcie_parse_map_dma_ranges(host); 925 if (err) 926 goto err_clk_disable; 927 928 host->phy_init_fn = of_device_get_match_data(dev); 929 err = host->phy_init_fn(host); 930 if (err) { 931 dev_err(dev, "failed to init PCIe PHY\n"); 932 goto err_clk_disable; 933 } 934 935 /* Failure to get a link might just be that no cards are inserted */ 936 if (rcar_pcie_hw_init(pcie)) { 937 dev_info(dev, "PCIe link down\n"); 938 err = -ENODEV; 939 goto err_phy_shutdown; 940 } 941 942 data = rcar_pci_read_reg(pcie, MACSR); 943 dev_info(dev, "PCIe x%d: link up\n", (data >> 20) & 0x3f); 944 945 if (IS_ENABLED(CONFIG_PCI_MSI)) { 946 err = rcar_pcie_enable_msi(host); 947 if (err < 0) { 948 dev_err(dev, 949 "failed to enable MSI support: %d\n", 950 err); 951 goto err_phy_shutdown; 952 } 953 } 954 955 err = rcar_pcie_enable(host); 956 if (err) 957 goto err_msi_teardown; 958 959 return 0; 960 961 err_msi_teardown: 962 if (IS_ENABLED(CONFIG_PCI_MSI)) 963 rcar_pcie_teardown_msi(host); 964 965 err_phy_shutdown: 966 if (host->phy) { 967 phy_power_off(host->phy); 968 phy_exit(host->phy); 969 } 970 971 err_clk_disable: 972 clk_disable_unprepare(host->bus_clk); 973 974 err_unmap_msi_irqs: 975 irq_dispose_mapping(host->msi.irq2); 976 irq_dispose_mapping(host->msi.irq1); 977 978 err_pm_put: 979 pm_runtime_put(dev); 980 pm_runtime_disable(dev); 981 982 return err; 983 } 984 985 static int __maybe_unused rcar_pcie_resume(struct device *dev) 986 { 987 struct rcar_pcie_host *host = dev_get_drvdata(dev); 988 struct rcar_pcie *pcie = &host->pcie; 989 unsigned int data; 990 int err; 991 992 err = rcar_pcie_parse_map_dma_ranges(host); 993 if (err) 994 return 0; 995 996 /* Failure to get a link might just be that no cards are inserted */ 997 err = host->phy_init_fn(host); 998 if (err) { 999 dev_info(dev, "PCIe link down\n"); 1000 return 0; 1001 } 1002 1003 data = rcar_pci_read_reg(pcie, MACSR); 1004 dev_info(dev, "PCIe x%d: link up\n", (data >> 20) & 0x3f); 1005 1006 /* Enable MSI */ 1007 if (IS_ENABLED(CONFIG_PCI_MSI)) { 1008 struct resource res; 1009 u32 val; 1010 1011 of_address_to_resource(dev->of_node, 0, &res); 1012 rcar_pci_write_reg(pcie, upper_32_bits(res.start), PCIEMSIAUR); 1013 rcar_pci_write_reg(pcie, lower_32_bits(res.start) | MSIFE, PCIEMSIALR); 1014 1015 bitmap_to_arr32(&val, host->msi.used, INT_PCI_MSI_NR); 1016 rcar_pci_write_reg(pcie, val, PCIEMSIIER); 1017 } 1018 1019 rcar_pcie_hw_enable(host); 1020 1021 return 0; 1022 } 1023 1024 static int rcar_pcie_resume_noirq(struct device *dev) 1025 { 1026 struct rcar_pcie_host *host = dev_get_drvdata(dev); 1027 struct rcar_pcie *pcie = &host->pcie; 1028 1029 if (rcar_pci_read_reg(pcie, PMSR) && 1030 !(rcar_pci_read_reg(pcie, PCIETCTLR) & DL_DOWN)) 1031 return 0; 1032 1033 /* Re-establish the PCIe link */ 1034 rcar_pci_write_reg(pcie, MACCTLR_INIT_VAL, MACCTLR); 1035 rcar_pci_write_reg(pcie, CFINIT, PCIETCTLR); 1036 return rcar_pcie_wait_for_dl(pcie); 1037 } 1038 1039 static const struct dev_pm_ops rcar_pcie_pm_ops = { 1040 SET_SYSTEM_SLEEP_PM_OPS(NULL, rcar_pcie_resume) 1041 .resume_noirq = rcar_pcie_resume_noirq, 1042 }; 1043 1044 static struct platform_driver rcar_pcie_driver = { 1045 .driver = { 1046 .name = "rcar-pcie", 1047 .of_match_table = rcar_pcie_of_match, 1048 .pm = &rcar_pcie_pm_ops, 1049 .suppress_bind_attrs = true, 1050 }, 1051 .probe = rcar_pcie_probe, 1052 }; 1053 1054 #ifdef CONFIG_ARM 1055 static DEFINE_SPINLOCK(pmsr_lock); 1056 static int rcar_pcie_aarch32_abort_handler(unsigned long addr, 1057 unsigned int fsr, struct pt_regs *regs) 1058 { 1059 unsigned long flags; 1060 u32 pmsr, val; 1061 int ret = 0; 1062 1063 spin_lock_irqsave(&pmsr_lock, flags); 1064 1065 if (!pcie_base || !__clk_is_enabled(pcie_bus_clk)) { 1066 ret = 1; 1067 goto unlock_exit; 1068 } 1069 1070 pmsr = readl(pcie_base + PMSR); 1071 1072 /* 1073 * Test if the PCIe controller received PM_ENTER_L1 DLLP and 1074 * the PCIe controller is not in L1 link state. If true, apply 1075 * fix, which will put the controller into L1 link state, from 1076 * which it can return to L0s/L0 on its own. 1077 */ 1078 if ((pmsr & PMEL1RX) && ((pmsr & PMSTATE) != PMSTATE_L1)) { 1079 writel(L1IATN, pcie_base + PMCTLR); 1080 ret = readl_poll_timeout_atomic(pcie_base + PMSR, val, 1081 val & L1FAEG, 10, 1000); 1082 WARN(ret, "Timeout waiting for L1 link state, ret=%d\n", ret); 1083 writel(L1FAEG | PMEL1RX, pcie_base + PMSR); 1084 } 1085 1086 unlock_exit: 1087 spin_unlock_irqrestore(&pmsr_lock, flags); 1088 return ret; 1089 } 1090 1091 static const struct of_device_id rcar_pcie_abort_handler_of_match[] __initconst = { 1092 { .compatible = "renesas,pcie-r8a7779" }, 1093 { .compatible = "renesas,pcie-r8a7790" }, 1094 { .compatible = "renesas,pcie-r8a7791" }, 1095 { .compatible = "renesas,pcie-rcar-gen2" }, 1096 {}, 1097 }; 1098 1099 static int __init rcar_pcie_init(void) 1100 { 1101 if (of_find_matching_node(NULL, rcar_pcie_abort_handler_of_match)) { 1102 #ifdef CONFIG_ARM_LPAE 1103 hook_fault_code(17, rcar_pcie_aarch32_abort_handler, SIGBUS, 0, 1104 "asynchronous external abort"); 1105 #else 1106 hook_fault_code(22, rcar_pcie_aarch32_abort_handler, SIGBUS, 0, 1107 "imprecise external abort"); 1108 #endif 1109 } 1110 1111 return platform_driver_register(&rcar_pcie_driver); 1112 } 1113 device_initcall(rcar_pcie_init); 1114 #else 1115 builtin_platform_driver(rcar_pcie_driver); 1116 #endif 1117