1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Rockchip AXI PCIe host controller driver 4 * 5 * Copyright (c) 2016 Rockchip, Inc. 6 * 7 * Author: Shawn Lin <shawn.lin@rock-chips.com> 8 * Wenrui Li <wenrui.li@rock-chips.com> 9 * 10 * Bits taken from Synopsys DesignWare Host controller driver and 11 * ARM PCI Host generic driver. 12 */ 13 14 #include <linux/bitrev.h> 15 #include <linux/clk.h> 16 #include <linux/delay.h> 17 #include <linux/gpio/consumer.h> 18 #include <linux/init.h> 19 #include <linux/interrupt.h> 20 #include <linux/iopoll.h> 21 #include <linux/irq.h> 22 #include <linux/irqchip/chained_irq.h> 23 #include <linux/irqdomain.h> 24 #include <linux/kernel.h> 25 #include <linux/mfd/syscon.h> 26 #include <linux/module.h> 27 #include <linux/of_address.h> 28 #include <linux/of_device.h> 29 #include <linux/of_pci.h> 30 #include <linux/of_platform.h> 31 #include <linux/pci.h> 32 #include <linux/pci_ids.h> 33 #include <linux/phy/phy.h> 34 #include <linux/platform_device.h> 35 #include <linux/reset.h> 36 #include <linux/regmap.h> 37 38 #include "../pci.h" 39 #include "pcie-rockchip.h" 40 41 static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip) 42 { 43 u32 status; 44 45 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 46 status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE); 47 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 48 } 49 50 static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip) 51 { 52 u32 status; 53 54 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 55 status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16; 56 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 57 } 58 59 static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip) 60 { 61 u32 val; 62 63 /* Update Tx credit maximum update interval */ 64 val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1); 65 val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK; 66 val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */ 67 rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1); 68 } 69 70 static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip, 71 struct pci_bus *bus, int dev) 72 { 73 /* 74 * Access only one slot on each root port. 75 * Do not read more than one device on the bus directly attached 76 * to RC's downstream side. 77 */ 78 if (pci_is_root_bus(bus) || pci_is_root_bus(bus->parent)) 79 return dev == 0; 80 81 return 1; 82 } 83 84 static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip) 85 { 86 u32 val; 87 u8 map; 88 89 if (rockchip->legacy_phy) 90 return GENMASK(MAX_LANE_NUM - 1, 0); 91 92 val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP); 93 map = val & PCIE_CORE_LANE_MAP_MASK; 94 95 /* The link may be using a reverse-indexed mapping. */ 96 if (val & PCIE_CORE_LANE_MAP_REVERSE) 97 map = bitrev8(map) >> 4; 98 99 return map; 100 } 101 102 static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip, 103 int where, int size, u32 *val) 104 { 105 void __iomem *addr; 106 107 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where; 108 109 if (!IS_ALIGNED((uintptr_t)addr, size)) { 110 *val = 0; 111 return PCIBIOS_BAD_REGISTER_NUMBER; 112 } 113 114 if (size == 4) { 115 *val = readl(addr); 116 } else if (size == 2) { 117 *val = readw(addr); 118 } else if (size == 1) { 119 *val = readb(addr); 120 } else { 121 *val = 0; 122 return PCIBIOS_BAD_REGISTER_NUMBER; 123 } 124 return PCIBIOS_SUCCESSFUL; 125 } 126 127 static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip, 128 int where, int size, u32 val) 129 { 130 u32 mask, tmp, offset; 131 void __iomem *addr; 132 133 offset = where & ~0x3; 134 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset; 135 136 if (size == 4) { 137 writel(val, addr); 138 return PCIBIOS_SUCCESSFUL; 139 } 140 141 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); 142 143 /* 144 * N.B. This read/modify/write isn't safe in general because it can 145 * corrupt RW1C bits in adjacent registers. But the hardware 146 * doesn't support smaller writes. 147 */ 148 tmp = readl(addr) & mask; 149 tmp |= val << ((where & 0x3) * 8); 150 writel(tmp, addr); 151 152 return PCIBIOS_SUCCESSFUL; 153 } 154 155 static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip, 156 struct pci_bus *bus, u32 devfn, 157 int where, int size, u32 *val) 158 { 159 void __iomem *addr; 160 161 addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); 162 163 if (!IS_ALIGNED((uintptr_t)addr, size)) { 164 *val = 0; 165 return PCIBIOS_BAD_REGISTER_NUMBER; 166 } 167 168 if (pci_is_root_bus(bus->parent)) 169 rockchip_pcie_cfg_configuration_accesses(rockchip, 170 AXI_WRAPPER_TYPE0_CFG); 171 else 172 rockchip_pcie_cfg_configuration_accesses(rockchip, 173 AXI_WRAPPER_TYPE1_CFG); 174 175 if (size == 4) { 176 *val = readl(addr); 177 } else if (size == 2) { 178 *val = readw(addr); 179 } else if (size == 1) { 180 *val = readb(addr); 181 } else { 182 *val = 0; 183 return PCIBIOS_BAD_REGISTER_NUMBER; 184 } 185 return PCIBIOS_SUCCESSFUL; 186 } 187 188 static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip, 189 struct pci_bus *bus, u32 devfn, 190 int where, int size, u32 val) 191 { 192 void __iomem *addr; 193 194 addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); 195 196 if (!IS_ALIGNED((uintptr_t)addr, size)) 197 return PCIBIOS_BAD_REGISTER_NUMBER; 198 199 if (pci_is_root_bus(bus->parent)) 200 rockchip_pcie_cfg_configuration_accesses(rockchip, 201 AXI_WRAPPER_TYPE0_CFG); 202 else 203 rockchip_pcie_cfg_configuration_accesses(rockchip, 204 AXI_WRAPPER_TYPE1_CFG); 205 206 if (size == 4) 207 writel(val, addr); 208 else if (size == 2) 209 writew(val, addr); 210 else if (size == 1) 211 writeb(val, addr); 212 else 213 return PCIBIOS_BAD_REGISTER_NUMBER; 214 215 return PCIBIOS_SUCCESSFUL; 216 } 217 218 static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where, 219 int size, u32 *val) 220 { 221 struct rockchip_pcie *rockchip = bus->sysdata; 222 223 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) 224 return PCIBIOS_DEVICE_NOT_FOUND; 225 226 if (pci_is_root_bus(bus)) 227 return rockchip_pcie_rd_own_conf(rockchip, where, size, val); 228 229 return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, 230 val); 231 } 232 233 static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn, 234 int where, int size, u32 val) 235 { 236 struct rockchip_pcie *rockchip = bus->sysdata; 237 238 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) 239 return PCIBIOS_DEVICE_NOT_FOUND; 240 241 if (pci_is_root_bus(bus)) 242 return rockchip_pcie_wr_own_conf(rockchip, where, size, val); 243 244 return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, 245 val); 246 } 247 248 static struct pci_ops rockchip_pcie_ops = { 249 .read = rockchip_pcie_rd_conf, 250 .write = rockchip_pcie_wr_conf, 251 }; 252 253 static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip) 254 { 255 int curr; 256 u32 status, scale, power; 257 258 if (IS_ERR(rockchip->vpcie3v3)) 259 return; 260 261 /* 262 * Set RC's captured slot power limit and scale if 263 * vpcie3v3 available. The default values are both zero 264 * which means the software should set these two according 265 * to the actual power supply. 266 */ 267 curr = regulator_get_current_limit(rockchip->vpcie3v3); 268 if (curr <= 0) 269 return; 270 271 scale = 3; /* 0.001x */ 272 curr = curr / 1000; /* convert to mA */ 273 power = (curr * 3300) / 1000; /* milliwatt */ 274 while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) { 275 if (!scale) { 276 dev_warn(rockchip->dev, "invalid power supply\n"); 277 return; 278 } 279 scale--; 280 power = power / 10; 281 } 282 283 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR); 284 status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) | 285 (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT); 286 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR); 287 } 288 289 /** 290 * rockchip_pcie_host_init_port - Initialize hardware 291 * @rockchip: PCIe port information 292 */ 293 static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip) 294 { 295 struct device *dev = rockchip->dev; 296 int err, i = MAX_LANE_NUM; 297 u32 status; 298 299 gpiod_set_value_cansleep(rockchip->ep_gpio, 0); 300 301 err = rockchip_pcie_init_port(rockchip); 302 if (err) 303 return err; 304 305 /* Fix the transmitted FTS count desired to exit from L0s. */ 306 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1); 307 status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) | 308 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT); 309 rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1); 310 311 rockchip_pcie_set_power_limit(rockchip); 312 313 /* Set RC's clock architecture as common clock */ 314 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 315 status |= PCI_EXP_LNKSTA_SLC << 16; 316 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 317 318 /* Set RC's RCB to 128 */ 319 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 320 status |= PCI_EXP_LNKCTL_RCB; 321 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 322 323 /* Enable Gen1 training */ 324 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, 325 PCIE_CLIENT_CONFIG); 326 327 gpiod_set_value_cansleep(rockchip->ep_gpio, 1); 328 329 /* 500ms timeout value should be enough for Gen1/2 training */ 330 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1, 331 status, PCIE_LINK_UP(status), 20, 332 500 * USEC_PER_MSEC); 333 if (err) { 334 dev_err(dev, "PCIe link training gen1 timeout!\n"); 335 goto err_power_off_phy; 336 } 337 338 if (rockchip->link_gen == 2) { 339 /* 340 * Enable retrain for gen2. This should be configured only after 341 * gen1 finished. 342 */ 343 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 344 status |= PCI_EXP_LNKCTL_RL; 345 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 346 347 err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL, 348 status, PCIE_LINK_IS_GEN2(status), 20, 349 500 * USEC_PER_MSEC); 350 if (err) 351 dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n"); 352 } 353 354 /* Check the final link width from negotiated lane counter from MGMT */ 355 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL); 356 status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >> 357 PCIE_CORE_PL_CONF_LANE_SHIFT); 358 dev_dbg(dev, "current link width is x%d\n", status); 359 360 /* Power off unused lane(s) */ 361 rockchip->lanes_map = rockchip_pcie_lane_map(rockchip); 362 for (i = 0; i < MAX_LANE_NUM; i++) { 363 if (!(rockchip->lanes_map & BIT(i))) { 364 dev_dbg(dev, "idling lane %d\n", i); 365 phy_power_off(rockchip->phys[i]); 366 } 367 } 368 369 rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID, 370 PCIE_CORE_CONFIG_VENDOR); 371 rockchip_pcie_write(rockchip, 372 PCI_CLASS_BRIDGE_PCI_NORMAL << 8, 373 PCIE_RC_CONFIG_RID_CCR); 374 375 /* Clear THP cap's next cap pointer to remove L1 substate cap */ 376 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP); 377 status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK; 378 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP); 379 380 /* Clear L0s from RC's link cap */ 381 if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) { 382 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP); 383 status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S; 384 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP); 385 } 386 387 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR); 388 status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK; 389 status |= PCIE_RC_CONFIG_DCSR_MPS_256; 390 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR); 391 392 return 0; 393 err_power_off_phy: 394 while (i--) 395 phy_power_off(rockchip->phys[i]); 396 i = MAX_LANE_NUM; 397 while (i--) 398 phy_exit(rockchip->phys[i]); 399 return err; 400 } 401 402 static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg) 403 { 404 struct rockchip_pcie *rockchip = arg; 405 struct device *dev = rockchip->dev; 406 u32 reg; 407 u32 sub_reg; 408 409 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 410 if (reg & PCIE_CLIENT_INT_LOCAL) { 411 dev_dbg(dev, "local interrupt received\n"); 412 sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS); 413 if (sub_reg & PCIE_CORE_INT_PRFPE) 414 dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n"); 415 416 if (sub_reg & PCIE_CORE_INT_CRFPE) 417 dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n"); 418 419 if (sub_reg & PCIE_CORE_INT_RRPE) 420 dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n"); 421 422 if (sub_reg & PCIE_CORE_INT_PRFO) 423 dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n"); 424 425 if (sub_reg & PCIE_CORE_INT_CRFO) 426 dev_dbg(dev, "overflow occurred in the completion receive FIFO\n"); 427 428 if (sub_reg & PCIE_CORE_INT_RT) 429 dev_dbg(dev, "replay timer timed out\n"); 430 431 if (sub_reg & PCIE_CORE_INT_RTR) 432 dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n"); 433 434 if (sub_reg & PCIE_CORE_INT_PE) 435 dev_dbg(dev, "phy error detected on receive side\n"); 436 437 if (sub_reg & PCIE_CORE_INT_MTR) 438 dev_dbg(dev, "malformed TLP received from the link\n"); 439 440 if (sub_reg & PCIE_CORE_INT_UCR) 441 dev_dbg(dev, "malformed TLP received from the link\n"); 442 443 if (sub_reg & PCIE_CORE_INT_FCE) 444 dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n"); 445 446 if (sub_reg & PCIE_CORE_INT_CT) 447 dev_dbg(dev, "a request timed out waiting for completion\n"); 448 449 if (sub_reg & PCIE_CORE_INT_UTC) 450 dev_dbg(dev, "unmapped TC error\n"); 451 452 if (sub_reg & PCIE_CORE_INT_MMVC) 453 dev_dbg(dev, "MSI mask register changes\n"); 454 455 rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS); 456 } else if (reg & PCIE_CLIENT_INT_PHY) { 457 dev_dbg(dev, "phy link changes\n"); 458 rockchip_pcie_update_txcredit_mui(rockchip); 459 rockchip_pcie_clr_bw_int(rockchip); 460 } 461 462 rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL, 463 PCIE_CLIENT_INT_STATUS); 464 465 return IRQ_HANDLED; 466 } 467 468 static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg) 469 { 470 struct rockchip_pcie *rockchip = arg; 471 struct device *dev = rockchip->dev; 472 u32 reg; 473 474 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 475 if (reg & PCIE_CLIENT_INT_LEGACY_DONE) 476 dev_dbg(dev, "legacy done interrupt received\n"); 477 478 if (reg & PCIE_CLIENT_INT_MSG) 479 dev_dbg(dev, "message done interrupt received\n"); 480 481 if (reg & PCIE_CLIENT_INT_HOT_RST) 482 dev_dbg(dev, "hot reset interrupt received\n"); 483 484 if (reg & PCIE_CLIENT_INT_DPA) 485 dev_dbg(dev, "dpa interrupt received\n"); 486 487 if (reg & PCIE_CLIENT_INT_FATAL_ERR) 488 dev_dbg(dev, "fatal error interrupt received\n"); 489 490 if (reg & PCIE_CLIENT_INT_NFATAL_ERR) 491 dev_dbg(dev, "no fatal error interrupt received\n"); 492 493 if (reg & PCIE_CLIENT_INT_CORR_ERR) 494 dev_dbg(dev, "correctable error interrupt received\n"); 495 496 if (reg & PCIE_CLIENT_INT_PHY) 497 dev_dbg(dev, "phy interrupt received\n"); 498 499 rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE | 500 PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST | 501 PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR | 502 PCIE_CLIENT_INT_NFATAL_ERR | 503 PCIE_CLIENT_INT_CORR_ERR | 504 PCIE_CLIENT_INT_PHY), 505 PCIE_CLIENT_INT_STATUS); 506 507 return IRQ_HANDLED; 508 } 509 510 static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc) 511 { 512 struct irq_chip *chip = irq_desc_get_chip(desc); 513 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc); 514 struct device *dev = rockchip->dev; 515 u32 reg; 516 u32 hwirq; 517 int ret; 518 519 chained_irq_enter(chip, desc); 520 521 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 522 reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT; 523 524 while (reg) { 525 hwirq = ffs(reg) - 1; 526 reg &= ~BIT(hwirq); 527 528 ret = generic_handle_domain_irq(rockchip->irq_domain, hwirq); 529 if (ret) 530 dev_err(dev, "unexpected IRQ, INT%d\n", hwirq); 531 } 532 533 chained_irq_exit(chip, desc); 534 } 535 536 static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip) 537 { 538 int irq, err; 539 struct device *dev = rockchip->dev; 540 struct platform_device *pdev = to_platform_device(dev); 541 542 irq = platform_get_irq_byname(pdev, "sys"); 543 if (irq < 0) 544 return irq; 545 546 err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler, 547 IRQF_SHARED, "pcie-sys", rockchip); 548 if (err) { 549 dev_err(dev, "failed to request PCIe subsystem IRQ\n"); 550 return err; 551 } 552 553 irq = platform_get_irq_byname(pdev, "legacy"); 554 if (irq < 0) 555 return irq; 556 557 irq_set_chained_handler_and_data(irq, 558 rockchip_pcie_legacy_int_handler, 559 rockchip); 560 561 irq = platform_get_irq_byname(pdev, "client"); 562 if (irq < 0) 563 return irq; 564 565 err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler, 566 IRQF_SHARED, "pcie-client", rockchip); 567 if (err) { 568 dev_err(dev, "failed to request PCIe client IRQ\n"); 569 return err; 570 } 571 572 return 0; 573 } 574 575 /** 576 * rockchip_pcie_parse_host_dt - Parse Device Tree 577 * @rockchip: PCIe port information 578 * 579 * Return: '0' on success and error value on failure 580 */ 581 static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip) 582 { 583 struct device *dev = rockchip->dev; 584 int err; 585 586 err = rockchip_pcie_parse_dt(rockchip); 587 if (err) 588 return err; 589 590 rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v"); 591 if (IS_ERR(rockchip->vpcie12v)) { 592 if (PTR_ERR(rockchip->vpcie12v) != -ENODEV) 593 return PTR_ERR(rockchip->vpcie12v); 594 dev_info(dev, "no vpcie12v regulator found\n"); 595 } 596 597 rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3"); 598 if (IS_ERR(rockchip->vpcie3v3)) { 599 if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV) 600 return PTR_ERR(rockchip->vpcie3v3); 601 dev_info(dev, "no vpcie3v3 regulator found\n"); 602 } 603 604 rockchip->vpcie1v8 = devm_regulator_get(dev, "vpcie1v8"); 605 if (IS_ERR(rockchip->vpcie1v8)) 606 return PTR_ERR(rockchip->vpcie1v8); 607 608 rockchip->vpcie0v9 = devm_regulator_get(dev, "vpcie0v9"); 609 if (IS_ERR(rockchip->vpcie0v9)) 610 return PTR_ERR(rockchip->vpcie0v9); 611 612 return 0; 613 } 614 615 static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip) 616 { 617 struct device *dev = rockchip->dev; 618 int err; 619 620 if (!IS_ERR(rockchip->vpcie12v)) { 621 err = regulator_enable(rockchip->vpcie12v); 622 if (err) { 623 dev_err(dev, "fail to enable vpcie12v regulator\n"); 624 goto err_out; 625 } 626 } 627 628 if (!IS_ERR(rockchip->vpcie3v3)) { 629 err = regulator_enable(rockchip->vpcie3v3); 630 if (err) { 631 dev_err(dev, "fail to enable vpcie3v3 regulator\n"); 632 goto err_disable_12v; 633 } 634 } 635 636 err = regulator_enable(rockchip->vpcie1v8); 637 if (err) { 638 dev_err(dev, "fail to enable vpcie1v8 regulator\n"); 639 goto err_disable_3v3; 640 } 641 642 err = regulator_enable(rockchip->vpcie0v9); 643 if (err) { 644 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 645 goto err_disable_1v8; 646 } 647 648 return 0; 649 650 err_disable_1v8: 651 regulator_disable(rockchip->vpcie1v8); 652 err_disable_3v3: 653 if (!IS_ERR(rockchip->vpcie3v3)) 654 regulator_disable(rockchip->vpcie3v3); 655 err_disable_12v: 656 if (!IS_ERR(rockchip->vpcie12v)) 657 regulator_disable(rockchip->vpcie12v); 658 err_out: 659 return err; 660 } 661 662 static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip) 663 { 664 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) & 665 (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK); 666 rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT), 667 PCIE_CORE_INT_MASK); 668 669 rockchip_pcie_enable_bw_int(rockchip); 670 } 671 672 static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq, 673 irq_hw_number_t hwirq) 674 { 675 irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq); 676 irq_set_chip_data(irq, domain->host_data); 677 678 return 0; 679 } 680 681 static const struct irq_domain_ops intx_domain_ops = { 682 .map = rockchip_pcie_intx_map, 683 }; 684 685 static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) 686 { 687 struct device *dev = rockchip->dev; 688 struct device_node *intc = of_get_next_child(dev->of_node, NULL); 689 690 if (!intc) { 691 dev_err(dev, "missing child interrupt-controller node\n"); 692 return -EINVAL; 693 } 694 695 rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX, 696 &intx_domain_ops, rockchip); 697 of_node_put(intc); 698 if (!rockchip->irq_domain) { 699 dev_err(dev, "failed to get a INTx IRQ domain\n"); 700 return -EINVAL; 701 } 702 703 return 0; 704 } 705 706 static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip, 707 int region_no, int type, u8 num_pass_bits, 708 u32 lower_addr, u32 upper_addr) 709 { 710 u32 ob_addr_0; 711 u32 ob_addr_1; 712 u32 ob_desc_0; 713 u32 aw_offset; 714 715 if (region_no >= MAX_AXI_WRAPPER_REGION_NUM) 716 return -EINVAL; 717 if (num_pass_bits + 1 < 8) 718 return -EINVAL; 719 if (num_pass_bits > 63) 720 return -EINVAL; 721 if (region_no == 0) { 722 if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits)) 723 return -EINVAL; 724 } 725 if (region_no != 0) { 726 if (AXI_REGION_SIZE < (2ULL << num_pass_bits)) 727 return -EINVAL; 728 } 729 730 aw_offset = (region_no << OB_REG_SIZE_SHIFT); 731 732 ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS; 733 ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR; 734 ob_addr_1 = upper_addr; 735 ob_desc_0 = (1 << 23 | type); 736 737 rockchip_pcie_write(rockchip, ob_addr_0, 738 PCIE_CORE_OB_REGION_ADDR0 + aw_offset); 739 rockchip_pcie_write(rockchip, ob_addr_1, 740 PCIE_CORE_OB_REGION_ADDR1 + aw_offset); 741 rockchip_pcie_write(rockchip, ob_desc_0, 742 PCIE_CORE_OB_REGION_DESC0 + aw_offset); 743 rockchip_pcie_write(rockchip, 0, 744 PCIE_CORE_OB_REGION_DESC1 + aw_offset); 745 746 return 0; 747 } 748 749 static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip, 750 int region_no, u8 num_pass_bits, 751 u32 lower_addr, u32 upper_addr) 752 { 753 u32 ib_addr_0; 754 u32 ib_addr_1; 755 u32 aw_offset; 756 757 if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM) 758 return -EINVAL; 759 if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED) 760 return -EINVAL; 761 if (num_pass_bits > 63) 762 return -EINVAL; 763 764 aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT); 765 766 ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS; 767 ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR; 768 ib_addr_1 = upper_addr; 769 770 rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset); 771 rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset); 772 773 return 0; 774 } 775 776 static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip) 777 { 778 struct device *dev = rockchip->dev; 779 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 780 struct resource_entry *entry; 781 u64 pci_addr, size; 782 int offset; 783 int err; 784 int reg_no; 785 786 rockchip_pcie_cfg_configuration_accesses(rockchip, 787 AXI_WRAPPER_TYPE0_CFG); 788 entry = resource_list_first_type(&bridge->windows, IORESOURCE_MEM); 789 if (!entry) 790 return -ENODEV; 791 792 size = resource_size(entry->res); 793 pci_addr = entry->res->start - entry->offset; 794 rockchip->msg_bus_addr = pci_addr; 795 796 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 797 err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1, 798 AXI_WRAPPER_MEM_WRITE, 799 20 - 1, 800 pci_addr + (reg_no << 20), 801 0); 802 if (err) { 803 dev_err(dev, "program RC mem outbound ATU failed\n"); 804 return err; 805 } 806 } 807 808 err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0); 809 if (err) { 810 dev_err(dev, "program RC mem inbound ATU failed\n"); 811 return err; 812 } 813 814 entry = resource_list_first_type(&bridge->windows, IORESOURCE_IO); 815 if (!entry) 816 return -ENODEV; 817 818 /* store the register number offset to program RC io outbound ATU */ 819 offset = size >> 20; 820 821 size = resource_size(entry->res); 822 pci_addr = entry->res->start - entry->offset; 823 824 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 825 err = rockchip_pcie_prog_ob_atu(rockchip, 826 reg_no + 1 + offset, 827 AXI_WRAPPER_IO_WRITE, 828 20 - 1, 829 pci_addr + (reg_no << 20), 830 0); 831 if (err) { 832 dev_err(dev, "program RC io outbound ATU failed\n"); 833 return err; 834 } 835 } 836 837 /* assign message regions */ 838 rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset, 839 AXI_WRAPPER_NOR_MSG, 840 20 - 1, 0, 0); 841 842 rockchip->msg_bus_addr += ((reg_no + offset) << 20); 843 return err; 844 } 845 846 static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip) 847 { 848 u32 value; 849 int err; 850 851 /* send PME_TURN_OFF message */ 852 writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF); 853 854 /* read LTSSM and wait for falling into L2 link state */ 855 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0, 856 value, PCIE_LINK_IS_L2(value), 20, 857 jiffies_to_usecs(5 * HZ)); 858 if (err) { 859 dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n"); 860 return err; 861 } 862 863 return 0; 864 } 865 866 static int rockchip_pcie_suspend_noirq(struct device *dev) 867 { 868 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 869 int ret; 870 871 /* disable core and cli int since we don't need to ack PME_ACK */ 872 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) | 873 PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK); 874 rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK); 875 876 ret = rockchip_pcie_wait_l2(rockchip); 877 if (ret) { 878 rockchip_pcie_enable_interrupts(rockchip); 879 return ret; 880 } 881 882 rockchip_pcie_deinit_phys(rockchip); 883 884 rockchip_pcie_disable_clocks(rockchip); 885 886 regulator_disable(rockchip->vpcie0v9); 887 888 return ret; 889 } 890 891 static int rockchip_pcie_resume_noirq(struct device *dev) 892 { 893 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 894 int err; 895 896 err = regulator_enable(rockchip->vpcie0v9); 897 if (err) { 898 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 899 return err; 900 } 901 902 err = rockchip_pcie_enable_clocks(rockchip); 903 if (err) 904 goto err_disable_0v9; 905 906 err = rockchip_pcie_host_init_port(rockchip); 907 if (err) 908 goto err_pcie_resume; 909 910 err = rockchip_pcie_cfg_atu(rockchip); 911 if (err) 912 goto err_err_deinit_port; 913 914 /* Need this to enter L1 again */ 915 rockchip_pcie_update_txcredit_mui(rockchip); 916 rockchip_pcie_enable_interrupts(rockchip); 917 918 return 0; 919 920 err_err_deinit_port: 921 rockchip_pcie_deinit_phys(rockchip); 922 err_pcie_resume: 923 rockchip_pcie_disable_clocks(rockchip); 924 err_disable_0v9: 925 regulator_disable(rockchip->vpcie0v9); 926 return err; 927 } 928 929 static int rockchip_pcie_probe(struct platform_device *pdev) 930 { 931 struct rockchip_pcie *rockchip; 932 struct device *dev = &pdev->dev; 933 struct pci_host_bridge *bridge; 934 int err; 935 936 if (!dev->of_node) 937 return -ENODEV; 938 939 bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip)); 940 if (!bridge) 941 return -ENOMEM; 942 943 rockchip = pci_host_bridge_priv(bridge); 944 945 platform_set_drvdata(pdev, rockchip); 946 rockchip->dev = dev; 947 rockchip->is_rc = true; 948 949 err = rockchip_pcie_parse_host_dt(rockchip); 950 if (err) 951 return err; 952 953 err = rockchip_pcie_enable_clocks(rockchip); 954 if (err) 955 return err; 956 957 err = rockchip_pcie_set_vpcie(rockchip); 958 if (err) { 959 dev_err(dev, "failed to set vpcie regulator\n"); 960 goto err_set_vpcie; 961 } 962 963 err = rockchip_pcie_host_init_port(rockchip); 964 if (err) 965 goto err_vpcie; 966 967 err = rockchip_pcie_init_irq_domain(rockchip); 968 if (err < 0) 969 goto err_deinit_port; 970 971 err = rockchip_pcie_cfg_atu(rockchip); 972 if (err) 973 goto err_remove_irq_domain; 974 975 rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M); 976 if (!rockchip->msg_region) { 977 err = -ENOMEM; 978 goto err_remove_irq_domain; 979 } 980 981 bridge->sysdata = rockchip; 982 bridge->ops = &rockchip_pcie_ops; 983 984 err = rockchip_pcie_setup_irq(rockchip); 985 if (err) 986 goto err_remove_irq_domain; 987 988 rockchip_pcie_enable_interrupts(rockchip); 989 990 err = pci_host_probe(bridge); 991 if (err < 0) 992 goto err_remove_irq_domain; 993 994 return 0; 995 996 err_remove_irq_domain: 997 irq_domain_remove(rockchip->irq_domain); 998 err_deinit_port: 999 rockchip_pcie_deinit_phys(rockchip); 1000 err_vpcie: 1001 if (!IS_ERR(rockchip->vpcie12v)) 1002 regulator_disable(rockchip->vpcie12v); 1003 if (!IS_ERR(rockchip->vpcie3v3)) 1004 regulator_disable(rockchip->vpcie3v3); 1005 regulator_disable(rockchip->vpcie1v8); 1006 regulator_disable(rockchip->vpcie0v9); 1007 err_set_vpcie: 1008 rockchip_pcie_disable_clocks(rockchip); 1009 return err; 1010 } 1011 1012 static int rockchip_pcie_remove(struct platform_device *pdev) 1013 { 1014 struct device *dev = &pdev->dev; 1015 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 1016 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 1017 1018 pci_stop_root_bus(bridge->bus); 1019 pci_remove_root_bus(bridge->bus); 1020 irq_domain_remove(rockchip->irq_domain); 1021 1022 rockchip_pcie_deinit_phys(rockchip); 1023 1024 rockchip_pcie_disable_clocks(rockchip); 1025 1026 if (!IS_ERR(rockchip->vpcie12v)) 1027 regulator_disable(rockchip->vpcie12v); 1028 if (!IS_ERR(rockchip->vpcie3v3)) 1029 regulator_disable(rockchip->vpcie3v3); 1030 regulator_disable(rockchip->vpcie1v8); 1031 regulator_disable(rockchip->vpcie0v9); 1032 1033 return 0; 1034 } 1035 1036 static const struct dev_pm_ops rockchip_pcie_pm_ops = { 1037 NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq, 1038 rockchip_pcie_resume_noirq) 1039 }; 1040 1041 static const struct of_device_id rockchip_pcie_of_match[] = { 1042 { .compatible = "rockchip,rk3399-pcie", }, 1043 {} 1044 }; 1045 MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match); 1046 1047 static struct platform_driver rockchip_pcie_driver = { 1048 .driver = { 1049 .name = "rockchip-pcie", 1050 .of_match_table = rockchip_pcie_of_match, 1051 .pm = &rockchip_pcie_pm_ops, 1052 }, 1053 .probe = rockchip_pcie_probe, 1054 .remove = rockchip_pcie_remove, 1055 }; 1056 module_platform_driver(rockchip_pcie_driver); 1057 1058 MODULE_AUTHOR("Rockchip Inc"); 1059 MODULE_DESCRIPTION("Rockchip AXI PCIe driver"); 1060 MODULE_LICENSE("GPL v2"); 1061