1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Rockchip AXI PCIe host controller driver 4 * 5 * Copyright (c) 2016 Rockchip, Inc. 6 * 7 * Author: Shawn Lin <shawn.lin@rock-chips.com> 8 * Wenrui Li <wenrui.li@rock-chips.com> 9 * 10 * Bits taken from Synopsys DesignWare Host controller driver and 11 * ARM PCI Host generic driver. 12 */ 13 14 #include <linux/bitrev.h> 15 #include <linux/clk.h> 16 #include <linux/delay.h> 17 #include <linux/gpio/consumer.h> 18 #include <linux/init.h> 19 #include <linux/interrupt.h> 20 #include <linux/iopoll.h> 21 #include <linux/irq.h> 22 #include <linux/irqchip/chained_irq.h> 23 #include <linux/irqdomain.h> 24 #include <linux/kernel.h> 25 #include <linux/mfd/syscon.h> 26 #include <linux/module.h> 27 #include <linux/of_address.h> 28 #include <linux/of_device.h> 29 #include <linux/of_pci.h> 30 #include <linux/of_platform.h> 31 #include <linux/of_irq.h> 32 #include <linux/pci.h> 33 #include <linux/pci_ids.h> 34 #include <linux/phy/phy.h> 35 #include <linux/platform_device.h> 36 #include <linux/reset.h> 37 #include <linux/regmap.h> 38 39 #include "../pci.h" 40 #include "pcie-rockchip.h" 41 42 static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip) 43 { 44 u32 status; 45 46 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 47 status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE); 48 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 49 } 50 51 static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip) 52 { 53 u32 status; 54 55 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 56 status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16; 57 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 58 } 59 60 static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip) 61 { 62 u32 val; 63 64 /* Update Tx credit maximum update interval */ 65 val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1); 66 val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK; 67 val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */ 68 rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1); 69 } 70 71 static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip, 72 struct pci_bus *bus, int dev) 73 { 74 /* 75 * Access only one slot on each root port. 76 * Do not read more than one device on the bus directly attached 77 * to RC's downstream side. 78 */ 79 if (pci_is_root_bus(bus) || pci_is_root_bus(bus->parent)) 80 return dev == 0; 81 82 return 1; 83 } 84 85 static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip) 86 { 87 u32 val; 88 u8 map; 89 90 if (rockchip->legacy_phy) 91 return GENMASK(MAX_LANE_NUM - 1, 0); 92 93 val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP); 94 map = val & PCIE_CORE_LANE_MAP_MASK; 95 96 /* The link may be using a reverse-indexed mapping. */ 97 if (val & PCIE_CORE_LANE_MAP_REVERSE) 98 map = bitrev8(map) >> 4; 99 100 return map; 101 } 102 103 static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip, 104 int where, int size, u32 *val) 105 { 106 void __iomem *addr; 107 108 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where; 109 110 if (!IS_ALIGNED((uintptr_t)addr, size)) { 111 *val = 0; 112 return PCIBIOS_BAD_REGISTER_NUMBER; 113 } 114 115 if (size == 4) { 116 *val = readl(addr); 117 } else if (size == 2) { 118 *val = readw(addr); 119 } else if (size == 1) { 120 *val = readb(addr); 121 } else { 122 *val = 0; 123 return PCIBIOS_BAD_REGISTER_NUMBER; 124 } 125 return PCIBIOS_SUCCESSFUL; 126 } 127 128 static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip, 129 int where, int size, u32 val) 130 { 131 u32 mask, tmp, offset; 132 void __iomem *addr; 133 134 offset = where & ~0x3; 135 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset; 136 137 if (size == 4) { 138 writel(val, addr); 139 return PCIBIOS_SUCCESSFUL; 140 } 141 142 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); 143 144 /* 145 * N.B. This read/modify/write isn't safe in general because it can 146 * corrupt RW1C bits in adjacent registers. But the hardware 147 * doesn't support smaller writes. 148 */ 149 tmp = readl(addr) & mask; 150 tmp |= val << ((where & 0x3) * 8); 151 writel(tmp, addr); 152 153 return PCIBIOS_SUCCESSFUL; 154 } 155 156 static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip, 157 struct pci_bus *bus, u32 devfn, 158 int where, int size, u32 *val) 159 { 160 void __iomem *addr; 161 162 addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); 163 164 if (!IS_ALIGNED((uintptr_t)addr, size)) { 165 *val = 0; 166 return PCIBIOS_BAD_REGISTER_NUMBER; 167 } 168 169 if (pci_is_root_bus(bus->parent)) 170 rockchip_pcie_cfg_configuration_accesses(rockchip, 171 AXI_WRAPPER_TYPE0_CFG); 172 else 173 rockchip_pcie_cfg_configuration_accesses(rockchip, 174 AXI_WRAPPER_TYPE1_CFG); 175 176 if (size == 4) { 177 *val = readl(addr); 178 } else if (size == 2) { 179 *val = readw(addr); 180 } else if (size == 1) { 181 *val = readb(addr); 182 } else { 183 *val = 0; 184 return PCIBIOS_BAD_REGISTER_NUMBER; 185 } 186 return PCIBIOS_SUCCESSFUL; 187 } 188 189 static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip, 190 struct pci_bus *bus, u32 devfn, 191 int where, int size, u32 val) 192 { 193 void __iomem *addr; 194 195 addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); 196 197 if (!IS_ALIGNED((uintptr_t)addr, size)) 198 return PCIBIOS_BAD_REGISTER_NUMBER; 199 200 if (pci_is_root_bus(bus->parent)) 201 rockchip_pcie_cfg_configuration_accesses(rockchip, 202 AXI_WRAPPER_TYPE0_CFG); 203 else 204 rockchip_pcie_cfg_configuration_accesses(rockchip, 205 AXI_WRAPPER_TYPE1_CFG); 206 207 if (size == 4) 208 writel(val, addr); 209 else if (size == 2) 210 writew(val, addr); 211 else if (size == 1) 212 writeb(val, addr); 213 else 214 return PCIBIOS_BAD_REGISTER_NUMBER; 215 216 return PCIBIOS_SUCCESSFUL; 217 } 218 219 static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where, 220 int size, u32 *val) 221 { 222 struct rockchip_pcie *rockchip = bus->sysdata; 223 224 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) { 225 *val = 0xffffffff; 226 return PCIBIOS_DEVICE_NOT_FOUND; 227 } 228 229 if (pci_is_root_bus(bus)) 230 return rockchip_pcie_rd_own_conf(rockchip, where, size, val); 231 232 return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, 233 val); 234 } 235 236 static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn, 237 int where, int size, u32 val) 238 { 239 struct rockchip_pcie *rockchip = bus->sysdata; 240 241 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) 242 return PCIBIOS_DEVICE_NOT_FOUND; 243 244 if (pci_is_root_bus(bus)) 245 return rockchip_pcie_wr_own_conf(rockchip, where, size, val); 246 247 return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, 248 val); 249 } 250 251 static struct pci_ops rockchip_pcie_ops = { 252 .read = rockchip_pcie_rd_conf, 253 .write = rockchip_pcie_wr_conf, 254 }; 255 256 static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip) 257 { 258 int curr; 259 u32 status, scale, power; 260 261 if (IS_ERR(rockchip->vpcie3v3)) 262 return; 263 264 /* 265 * Set RC's captured slot power limit and scale if 266 * vpcie3v3 available. The default values are both zero 267 * which means the software should set these two according 268 * to the actual power supply. 269 */ 270 curr = regulator_get_current_limit(rockchip->vpcie3v3); 271 if (curr <= 0) 272 return; 273 274 scale = 3; /* 0.001x */ 275 curr = curr / 1000; /* convert to mA */ 276 power = (curr * 3300) / 1000; /* milliwatt */ 277 while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) { 278 if (!scale) { 279 dev_warn(rockchip->dev, "invalid power supply\n"); 280 return; 281 } 282 scale--; 283 power = power / 10; 284 } 285 286 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR); 287 status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) | 288 (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT); 289 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR); 290 } 291 292 /** 293 * rockchip_pcie_host_init_port - Initialize hardware 294 * @rockchip: PCIe port information 295 */ 296 static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip) 297 { 298 struct device *dev = rockchip->dev; 299 int err, i = MAX_LANE_NUM; 300 u32 status; 301 302 gpiod_set_value_cansleep(rockchip->ep_gpio, 0); 303 304 err = rockchip_pcie_init_port(rockchip); 305 if (err) 306 return err; 307 308 /* Fix the transmitted FTS count desired to exit from L0s. */ 309 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1); 310 status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) | 311 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT); 312 rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1); 313 314 rockchip_pcie_set_power_limit(rockchip); 315 316 /* Set RC's clock architecture as common clock */ 317 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 318 status |= PCI_EXP_LNKSTA_SLC << 16; 319 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 320 321 /* Set RC's RCB to 128 */ 322 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 323 status |= PCI_EXP_LNKCTL_RCB; 324 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 325 326 /* Enable Gen1 training */ 327 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, 328 PCIE_CLIENT_CONFIG); 329 330 gpiod_set_value_cansleep(rockchip->ep_gpio, 1); 331 332 /* 500ms timeout value should be enough for Gen1/2 training */ 333 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1, 334 status, PCIE_LINK_UP(status), 20, 335 500 * USEC_PER_MSEC); 336 if (err) { 337 dev_err(dev, "PCIe link training gen1 timeout!\n"); 338 goto err_power_off_phy; 339 } 340 341 if (rockchip->link_gen == 2) { 342 /* 343 * Enable retrain for gen2. This should be configured only after 344 * gen1 finished. 345 */ 346 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 347 status |= PCI_EXP_LNKCTL_RL; 348 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 349 350 err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL, 351 status, PCIE_LINK_IS_GEN2(status), 20, 352 500 * USEC_PER_MSEC); 353 if (err) 354 dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n"); 355 } 356 357 /* Check the final link width from negotiated lane counter from MGMT */ 358 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL); 359 status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >> 360 PCIE_CORE_PL_CONF_LANE_SHIFT); 361 dev_dbg(dev, "current link width is x%d\n", status); 362 363 /* Power off unused lane(s) */ 364 rockchip->lanes_map = rockchip_pcie_lane_map(rockchip); 365 for (i = 0; i < MAX_LANE_NUM; i++) { 366 if (!(rockchip->lanes_map & BIT(i))) { 367 dev_dbg(dev, "idling lane %d\n", i); 368 phy_power_off(rockchip->phys[i]); 369 } 370 } 371 372 rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID, 373 PCIE_CORE_CONFIG_VENDOR); 374 rockchip_pcie_write(rockchip, 375 PCI_CLASS_BRIDGE_PCI << PCIE_RC_CONFIG_SCC_SHIFT, 376 PCIE_RC_CONFIG_RID_CCR); 377 378 /* Clear THP cap's next cap pointer to remove L1 substate cap */ 379 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP); 380 status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK; 381 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP); 382 383 /* Clear L0s from RC's link cap */ 384 if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) { 385 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP); 386 status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S; 387 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP); 388 } 389 390 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR); 391 status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK; 392 status |= PCIE_RC_CONFIG_DCSR_MPS_256; 393 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR); 394 395 return 0; 396 err_power_off_phy: 397 while (i--) 398 phy_power_off(rockchip->phys[i]); 399 i = MAX_LANE_NUM; 400 while (i--) 401 phy_exit(rockchip->phys[i]); 402 return err; 403 } 404 405 static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg) 406 { 407 struct rockchip_pcie *rockchip = arg; 408 struct device *dev = rockchip->dev; 409 u32 reg; 410 u32 sub_reg; 411 412 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 413 if (reg & PCIE_CLIENT_INT_LOCAL) { 414 dev_dbg(dev, "local interrupt received\n"); 415 sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS); 416 if (sub_reg & PCIE_CORE_INT_PRFPE) 417 dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n"); 418 419 if (sub_reg & PCIE_CORE_INT_CRFPE) 420 dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n"); 421 422 if (sub_reg & PCIE_CORE_INT_RRPE) 423 dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n"); 424 425 if (sub_reg & PCIE_CORE_INT_PRFO) 426 dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n"); 427 428 if (sub_reg & PCIE_CORE_INT_CRFO) 429 dev_dbg(dev, "overflow occurred in the completion receive FIFO\n"); 430 431 if (sub_reg & PCIE_CORE_INT_RT) 432 dev_dbg(dev, "replay timer timed out\n"); 433 434 if (sub_reg & PCIE_CORE_INT_RTR) 435 dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n"); 436 437 if (sub_reg & PCIE_CORE_INT_PE) 438 dev_dbg(dev, "phy error detected on receive side\n"); 439 440 if (sub_reg & PCIE_CORE_INT_MTR) 441 dev_dbg(dev, "malformed TLP received from the link\n"); 442 443 if (sub_reg & PCIE_CORE_INT_UCR) 444 dev_dbg(dev, "malformed TLP received from the link\n"); 445 446 if (sub_reg & PCIE_CORE_INT_FCE) 447 dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n"); 448 449 if (sub_reg & PCIE_CORE_INT_CT) 450 dev_dbg(dev, "a request timed out waiting for completion\n"); 451 452 if (sub_reg & PCIE_CORE_INT_UTC) 453 dev_dbg(dev, "unmapped TC error\n"); 454 455 if (sub_reg & PCIE_CORE_INT_MMVC) 456 dev_dbg(dev, "MSI mask register changes\n"); 457 458 rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS); 459 } else if (reg & PCIE_CLIENT_INT_PHY) { 460 dev_dbg(dev, "phy link changes\n"); 461 rockchip_pcie_update_txcredit_mui(rockchip); 462 rockchip_pcie_clr_bw_int(rockchip); 463 } 464 465 rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL, 466 PCIE_CLIENT_INT_STATUS); 467 468 return IRQ_HANDLED; 469 } 470 471 static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg) 472 { 473 struct rockchip_pcie *rockchip = arg; 474 struct device *dev = rockchip->dev; 475 u32 reg; 476 477 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 478 if (reg & PCIE_CLIENT_INT_LEGACY_DONE) 479 dev_dbg(dev, "legacy done interrupt received\n"); 480 481 if (reg & PCIE_CLIENT_INT_MSG) 482 dev_dbg(dev, "message done interrupt received\n"); 483 484 if (reg & PCIE_CLIENT_INT_HOT_RST) 485 dev_dbg(dev, "hot reset interrupt received\n"); 486 487 if (reg & PCIE_CLIENT_INT_DPA) 488 dev_dbg(dev, "dpa interrupt received\n"); 489 490 if (reg & PCIE_CLIENT_INT_FATAL_ERR) 491 dev_dbg(dev, "fatal error interrupt received\n"); 492 493 if (reg & PCIE_CLIENT_INT_NFATAL_ERR) 494 dev_dbg(dev, "no fatal error interrupt received\n"); 495 496 if (reg & PCIE_CLIENT_INT_CORR_ERR) 497 dev_dbg(dev, "correctable error interrupt received\n"); 498 499 if (reg & PCIE_CLIENT_INT_PHY) 500 dev_dbg(dev, "phy interrupt received\n"); 501 502 rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE | 503 PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST | 504 PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR | 505 PCIE_CLIENT_INT_NFATAL_ERR | 506 PCIE_CLIENT_INT_CORR_ERR | 507 PCIE_CLIENT_INT_PHY), 508 PCIE_CLIENT_INT_STATUS); 509 510 return IRQ_HANDLED; 511 } 512 513 static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc) 514 { 515 struct irq_chip *chip = irq_desc_get_chip(desc); 516 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc); 517 struct device *dev = rockchip->dev; 518 u32 reg; 519 u32 hwirq; 520 u32 virq; 521 522 chained_irq_enter(chip, desc); 523 524 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 525 reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT; 526 527 while (reg) { 528 hwirq = ffs(reg) - 1; 529 reg &= ~BIT(hwirq); 530 531 virq = irq_find_mapping(rockchip->irq_domain, hwirq); 532 if (virq) 533 generic_handle_irq(virq); 534 else 535 dev_err(dev, "unexpected IRQ, INT%d\n", hwirq); 536 } 537 538 chained_irq_exit(chip, desc); 539 } 540 541 static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip) 542 { 543 int irq, err; 544 struct device *dev = rockchip->dev; 545 struct platform_device *pdev = to_platform_device(dev); 546 547 irq = platform_get_irq_byname(pdev, "sys"); 548 if (irq < 0) 549 return irq; 550 551 err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler, 552 IRQF_SHARED, "pcie-sys", rockchip); 553 if (err) { 554 dev_err(dev, "failed to request PCIe subsystem IRQ\n"); 555 return err; 556 } 557 558 irq = platform_get_irq_byname(pdev, "legacy"); 559 if (irq < 0) 560 return irq; 561 562 irq_set_chained_handler_and_data(irq, 563 rockchip_pcie_legacy_int_handler, 564 rockchip); 565 566 irq = platform_get_irq_byname(pdev, "client"); 567 if (irq < 0) 568 return irq; 569 570 err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler, 571 IRQF_SHARED, "pcie-client", rockchip); 572 if (err) { 573 dev_err(dev, "failed to request PCIe client IRQ\n"); 574 return err; 575 } 576 577 return 0; 578 } 579 580 /** 581 * rockchip_pcie_parse_host_dt - Parse Device Tree 582 * @rockchip: PCIe port information 583 * 584 * Return: '0' on success and error value on failure 585 */ 586 static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip) 587 { 588 struct device *dev = rockchip->dev; 589 int err; 590 591 err = rockchip_pcie_parse_dt(rockchip); 592 if (err) 593 return err; 594 595 rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v"); 596 if (IS_ERR(rockchip->vpcie12v)) { 597 if (PTR_ERR(rockchip->vpcie12v) != -ENODEV) 598 return PTR_ERR(rockchip->vpcie12v); 599 dev_info(dev, "no vpcie12v regulator found\n"); 600 } 601 602 rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3"); 603 if (IS_ERR(rockchip->vpcie3v3)) { 604 if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV) 605 return PTR_ERR(rockchip->vpcie3v3); 606 dev_info(dev, "no vpcie3v3 regulator found\n"); 607 } 608 609 rockchip->vpcie1v8 = devm_regulator_get(dev, "vpcie1v8"); 610 if (IS_ERR(rockchip->vpcie1v8)) 611 return PTR_ERR(rockchip->vpcie1v8); 612 613 rockchip->vpcie0v9 = devm_regulator_get(dev, "vpcie0v9"); 614 if (IS_ERR(rockchip->vpcie0v9)) 615 return PTR_ERR(rockchip->vpcie0v9); 616 617 return 0; 618 } 619 620 static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip) 621 { 622 struct device *dev = rockchip->dev; 623 int err; 624 625 if (!IS_ERR(rockchip->vpcie12v)) { 626 err = regulator_enable(rockchip->vpcie12v); 627 if (err) { 628 dev_err(dev, "fail to enable vpcie12v regulator\n"); 629 goto err_out; 630 } 631 } 632 633 if (!IS_ERR(rockchip->vpcie3v3)) { 634 err = regulator_enable(rockchip->vpcie3v3); 635 if (err) { 636 dev_err(dev, "fail to enable vpcie3v3 regulator\n"); 637 goto err_disable_12v; 638 } 639 } 640 641 err = regulator_enable(rockchip->vpcie1v8); 642 if (err) { 643 dev_err(dev, "fail to enable vpcie1v8 regulator\n"); 644 goto err_disable_3v3; 645 } 646 647 err = regulator_enable(rockchip->vpcie0v9); 648 if (err) { 649 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 650 goto err_disable_1v8; 651 } 652 653 return 0; 654 655 err_disable_1v8: 656 regulator_disable(rockchip->vpcie1v8); 657 err_disable_3v3: 658 if (!IS_ERR(rockchip->vpcie3v3)) 659 regulator_disable(rockchip->vpcie3v3); 660 err_disable_12v: 661 if (!IS_ERR(rockchip->vpcie12v)) 662 regulator_disable(rockchip->vpcie12v); 663 err_out: 664 return err; 665 } 666 667 static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip) 668 { 669 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) & 670 (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK); 671 rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT), 672 PCIE_CORE_INT_MASK); 673 674 rockchip_pcie_enable_bw_int(rockchip); 675 } 676 677 static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq, 678 irq_hw_number_t hwirq) 679 { 680 irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq); 681 irq_set_chip_data(irq, domain->host_data); 682 683 return 0; 684 } 685 686 static const struct irq_domain_ops intx_domain_ops = { 687 .map = rockchip_pcie_intx_map, 688 }; 689 690 static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) 691 { 692 struct device *dev = rockchip->dev; 693 struct device_node *intc = of_get_next_child(dev->of_node, NULL); 694 695 if (!intc) { 696 dev_err(dev, "missing child interrupt-controller node\n"); 697 return -EINVAL; 698 } 699 700 rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX, 701 &intx_domain_ops, rockchip); 702 of_node_put(intc); 703 if (!rockchip->irq_domain) { 704 dev_err(dev, "failed to get a INTx IRQ domain\n"); 705 return -EINVAL; 706 } 707 708 return 0; 709 } 710 711 static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip, 712 int region_no, int type, u8 num_pass_bits, 713 u32 lower_addr, u32 upper_addr) 714 { 715 u32 ob_addr_0; 716 u32 ob_addr_1; 717 u32 ob_desc_0; 718 u32 aw_offset; 719 720 if (region_no >= MAX_AXI_WRAPPER_REGION_NUM) 721 return -EINVAL; 722 if (num_pass_bits + 1 < 8) 723 return -EINVAL; 724 if (num_pass_bits > 63) 725 return -EINVAL; 726 if (region_no == 0) { 727 if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits)) 728 return -EINVAL; 729 } 730 if (region_no != 0) { 731 if (AXI_REGION_SIZE < (2ULL << num_pass_bits)) 732 return -EINVAL; 733 } 734 735 aw_offset = (region_no << OB_REG_SIZE_SHIFT); 736 737 ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS; 738 ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR; 739 ob_addr_1 = upper_addr; 740 ob_desc_0 = (1 << 23 | type); 741 742 rockchip_pcie_write(rockchip, ob_addr_0, 743 PCIE_CORE_OB_REGION_ADDR0 + aw_offset); 744 rockchip_pcie_write(rockchip, ob_addr_1, 745 PCIE_CORE_OB_REGION_ADDR1 + aw_offset); 746 rockchip_pcie_write(rockchip, ob_desc_0, 747 PCIE_CORE_OB_REGION_DESC0 + aw_offset); 748 rockchip_pcie_write(rockchip, 0, 749 PCIE_CORE_OB_REGION_DESC1 + aw_offset); 750 751 return 0; 752 } 753 754 static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip, 755 int region_no, u8 num_pass_bits, 756 u32 lower_addr, u32 upper_addr) 757 { 758 u32 ib_addr_0; 759 u32 ib_addr_1; 760 u32 aw_offset; 761 762 if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM) 763 return -EINVAL; 764 if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED) 765 return -EINVAL; 766 if (num_pass_bits > 63) 767 return -EINVAL; 768 769 aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT); 770 771 ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS; 772 ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR; 773 ib_addr_1 = upper_addr; 774 775 rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset); 776 rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset); 777 778 return 0; 779 } 780 781 static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip) 782 { 783 struct device *dev = rockchip->dev; 784 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 785 struct resource_entry *entry; 786 u64 pci_addr, size; 787 int offset; 788 int err; 789 int reg_no; 790 791 rockchip_pcie_cfg_configuration_accesses(rockchip, 792 AXI_WRAPPER_TYPE0_CFG); 793 entry = resource_list_first_type(&bridge->windows, IORESOURCE_MEM); 794 if (!entry) 795 return -ENODEV; 796 797 size = resource_size(entry->res); 798 pci_addr = entry->res->start - entry->offset; 799 rockchip->msg_bus_addr = pci_addr; 800 801 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 802 err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1, 803 AXI_WRAPPER_MEM_WRITE, 804 20 - 1, 805 pci_addr + (reg_no << 20), 806 0); 807 if (err) { 808 dev_err(dev, "program RC mem outbound ATU failed\n"); 809 return err; 810 } 811 } 812 813 err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0); 814 if (err) { 815 dev_err(dev, "program RC mem inbound ATU failed\n"); 816 return err; 817 } 818 819 entry = resource_list_first_type(&bridge->windows, IORESOURCE_IO); 820 if (!entry) 821 return -ENODEV; 822 823 /* store the register number offset to program RC io outbound ATU */ 824 offset = size >> 20; 825 826 size = resource_size(entry->res); 827 pci_addr = entry->res->start - entry->offset; 828 829 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 830 err = rockchip_pcie_prog_ob_atu(rockchip, 831 reg_no + 1 + offset, 832 AXI_WRAPPER_IO_WRITE, 833 20 - 1, 834 pci_addr + (reg_no << 20), 835 0); 836 if (err) { 837 dev_err(dev, "program RC io outbound ATU failed\n"); 838 return err; 839 } 840 } 841 842 /* assign message regions */ 843 rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset, 844 AXI_WRAPPER_NOR_MSG, 845 20 - 1, 0, 0); 846 847 rockchip->msg_bus_addr += ((reg_no + offset) << 20); 848 return err; 849 } 850 851 static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip) 852 { 853 u32 value; 854 int err; 855 856 /* send PME_TURN_OFF message */ 857 writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF); 858 859 /* read LTSSM and wait for falling into L2 link state */ 860 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0, 861 value, PCIE_LINK_IS_L2(value), 20, 862 jiffies_to_usecs(5 * HZ)); 863 if (err) { 864 dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n"); 865 return err; 866 } 867 868 return 0; 869 } 870 871 static int __maybe_unused rockchip_pcie_suspend_noirq(struct device *dev) 872 { 873 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 874 int ret; 875 876 /* disable core and cli int since we don't need to ack PME_ACK */ 877 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) | 878 PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK); 879 rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK); 880 881 ret = rockchip_pcie_wait_l2(rockchip); 882 if (ret) { 883 rockchip_pcie_enable_interrupts(rockchip); 884 return ret; 885 } 886 887 rockchip_pcie_deinit_phys(rockchip); 888 889 rockchip_pcie_disable_clocks(rockchip); 890 891 regulator_disable(rockchip->vpcie0v9); 892 893 return ret; 894 } 895 896 static int __maybe_unused rockchip_pcie_resume_noirq(struct device *dev) 897 { 898 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 899 int err; 900 901 err = regulator_enable(rockchip->vpcie0v9); 902 if (err) { 903 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 904 return err; 905 } 906 907 err = rockchip_pcie_enable_clocks(rockchip); 908 if (err) 909 goto err_disable_0v9; 910 911 err = rockchip_pcie_host_init_port(rockchip); 912 if (err) 913 goto err_pcie_resume; 914 915 err = rockchip_pcie_cfg_atu(rockchip); 916 if (err) 917 goto err_err_deinit_port; 918 919 /* Need this to enter L1 again */ 920 rockchip_pcie_update_txcredit_mui(rockchip); 921 rockchip_pcie_enable_interrupts(rockchip); 922 923 return 0; 924 925 err_err_deinit_port: 926 rockchip_pcie_deinit_phys(rockchip); 927 err_pcie_resume: 928 rockchip_pcie_disable_clocks(rockchip); 929 err_disable_0v9: 930 regulator_disable(rockchip->vpcie0v9); 931 return err; 932 } 933 934 static int rockchip_pcie_probe(struct platform_device *pdev) 935 { 936 struct rockchip_pcie *rockchip; 937 struct device *dev = &pdev->dev; 938 struct pci_host_bridge *bridge; 939 int err; 940 941 if (!dev->of_node) 942 return -ENODEV; 943 944 bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip)); 945 if (!bridge) 946 return -ENOMEM; 947 948 rockchip = pci_host_bridge_priv(bridge); 949 950 platform_set_drvdata(pdev, rockchip); 951 rockchip->dev = dev; 952 rockchip->is_rc = true; 953 954 err = rockchip_pcie_parse_host_dt(rockchip); 955 if (err) 956 return err; 957 958 err = rockchip_pcie_enable_clocks(rockchip); 959 if (err) 960 return err; 961 962 err = rockchip_pcie_set_vpcie(rockchip); 963 if (err) { 964 dev_err(dev, "failed to set vpcie regulator\n"); 965 goto err_set_vpcie; 966 } 967 968 err = rockchip_pcie_host_init_port(rockchip); 969 if (err) 970 goto err_vpcie; 971 972 err = rockchip_pcie_init_irq_domain(rockchip); 973 if (err < 0) 974 goto err_deinit_port; 975 976 err = rockchip_pcie_cfg_atu(rockchip); 977 if (err) 978 goto err_remove_irq_domain; 979 980 rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M); 981 if (!rockchip->msg_region) { 982 err = -ENOMEM; 983 goto err_remove_irq_domain; 984 } 985 986 bridge->sysdata = rockchip; 987 bridge->ops = &rockchip_pcie_ops; 988 989 err = rockchip_pcie_setup_irq(rockchip); 990 if (err) 991 goto err_remove_irq_domain; 992 993 rockchip_pcie_enable_interrupts(rockchip); 994 995 err = pci_host_probe(bridge); 996 if (err < 0) 997 goto err_remove_irq_domain; 998 999 return 0; 1000 1001 err_remove_irq_domain: 1002 irq_domain_remove(rockchip->irq_domain); 1003 err_deinit_port: 1004 rockchip_pcie_deinit_phys(rockchip); 1005 err_vpcie: 1006 if (!IS_ERR(rockchip->vpcie12v)) 1007 regulator_disable(rockchip->vpcie12v); 1008 if (!IS_ERR(rockchip->vpcie3v3)) 1009 regulator_disable(rockchip->vpcie3v3); 1010 regulator_disable(rockchip->vpcie1v8); 1011 regulator_disable(rockchip->vpcie0v9); 1012 err_set_vpcie: 1013 rockchip_pcie_disable_clocks(rockchip); 1014 return err; 1015 } 1016 1017 static int rockchip_pcie_remove(struct platform_device *pdev) 1018 { 1019 struct device *dev = &pdev->dev; 1020 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 1021 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 1022 1023 pci_stop_root_bus(bridge->bus); 1024 pci_remove_root_bus(bridge->bus); 1025 irq_domain_remove(rockchip->irq_domain); 1026 1027 rockchip_pcie_deinit_phys(rockchip); 1028 1029 rockchip_pcie_disable_clocks(rockchip); 1030 1031 if (!IS_ERR(rockchip->vpcie12v)) 1032 regulator_disable(rockchip->vpcie12v); 1033 if (!IS_ERR(rockchip->vpcie3v3)) 1034 regulator_disable(rockchip->vpcie3v3); 1035 regulator_disable(rockchip->vpcie1v8); 1036 regulator_disable(rockchip->vpcie0v9); 1037 1038 return 0; 1039 } 1040 1041 static const struct dev_pm_ops rockchip_pcie_pm_ops = { 1042 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq, 1043 rockchip_pcie_resume_noirq) 1044 }; 1045 1046 static const struct of_device_id rockchip_pcie_of_match[] = { 1047 { .compatible = "rockchip,rk3399-pcie", }, 1048 {} 1049 }; 1050 MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match); 1051 1052 static struct platform_driver rockchip_pcie_driver = { 1053 .driver = { 1054 .name = "rockchip-pcie", 1055 .of_match_table = rockchip_pcie_of_match, 1056 .pm = &rockchip_pcie_pm_ops, 1057 }, 1058 .probe = rockchip_pcie_probe, 1059 .remove = rockchip_pcie_remove, 1060 }; 1061 module_platform_driver(rockchip_pcie_driver); 1062 1063 MODULE_AUTHOR("Rockchip Inc"); 1064 MODULE_DESCRIPTION("Rockchip AXI PCIe driver"); 1065 MODULE_LICENSE("GPL v2"); 1066