1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Rockchip AXI PCIe host controller driver 4 * 5 * Copyright (c) 2016 Rockchip, Inc. 6 * 7 * Author: Shawn Lin <shawn.lin@rock-chips.com> 8 * Wenrui Li <wenrui.li@rock-chips.com> 9 * 10 * Bits taken from Synopsys DesignWare Host controller driver and 11 * ARM PCI Host generic driver. 12 */ 13 14 #include <linux/bitrev.h> 15 #include <linux/clk.h> 16 #include <linux/delay.h> 17 #include <linux/gpio/consumer.h> 18 #include <linux/init.h> 19 #include <linux/interrupt.h> 20 #include <linux/iopoll.h> 21 #include <linux/irq.h> 22 #include <linux/irqchip/chained_irq.h> 23 #include <linux/irqdomain.h> 24 #include <linux/kernel.h> 25 #include <linux/mfd/syscon.h> 26 #include <linux/module.h> 27 #include <linux/of_address.h> 28 #include <linux/of_device.h> 29 #include <linux/of_pci.h> 30 #include <linux/of_platform.h> 31 #include <linux/of_irq.h> 32 #include <linux/pci.h> 33 #include <linux/pci_ids.h> 34 #include <linux/phy/phy.h> 35 #include <linux/platform_device.h> 36 #include <linux/reset.h> 37 #include <linux/regmap.h> 38 39 #include "../pci.h" 40 #include "pcie-rockchip.h" 41 42 static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip) 43 { 44 u32 status; 45 46 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 47 status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE); 48 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 49 } 50 51 static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip) 52 { 53 u32 status; 54 55 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 56 status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16; 57 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 58 } 59 60 static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip) 61 { 62 u32 val; 63 64 /* Update Tx credit maximum update interval */ 65 val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1); 66 val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK; 67 val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */ 68 rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1); 69 } 70 71 static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip, 72 struct pci_bus *bus, int dev) 73 { 74 /* 75 * Access only one slot on each root port. 76 * Do not read more than one device on the bus directly attached 77 * to RC's downstream side. 78 */ 79 if (pci_is_root_bus(bus) || pci_is_root_bus(bus->parent)) 80 return dev == 0; 81 82 return 1; 83 } 84 85 static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip) 86 { 87 u32 val; 88 u8 map; 89 90 if (rockchip->legacy_phy) 91 return GENMASK(MAX_LANE_NUM - 1, 0); 92 93 val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP); 94 map = val & PCIE_CORE_LANE_MAP_MASK; 95 96 /* The link may be using a reverse-indexed mapping. */ 97 if (val & PCIE_CORE_LANE_MAP_REVERSE) 98 map = bitrev8(map) >> 4; 99 100 return map; 101 } 102 103 static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip, 104 int where, int size, u32 *val) 105 { 106 void __iomem *addr; 107 108 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where; 109 110 if (!IS_ALIGNED((uintptr_t)addr, size)) { 111 *val = 0; 112 return PCIBIOS_BAD_REGISTER_NUMBER; 113 } 114 115 if (size == 4) { 116 *val = readl(addr); 117 } else if (size == 2) { 118 *val = readw(addr); 119 } else if (size == 1) { 120 *val = readb(addr); 121 } else { 122 *val = 0; 123 return PCIBIOS_BAD_REGISTER_NUMBER; 124 } 125 return PCIBIOS_SUCCESSFUL; 126 } 127 128 static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip, 129 int where, int size, u32 val) 130 { 131 u32 mask, tmp, offset; 132 void __iomem *addr; 133 134 offset = where & ~0x3; 135 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset; 136 137 if (size == 4) { 138 writel(val, addr); 139 return PCIBIOS_SUCCESSFUL; 140 } 141 142 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); 143 144 /* 145 * N.B. This read/modify/write isn't safe in general because it can 146 * corrupt RW1C bits in adjacent registers. But the hardware 147 * doesn't support smaller writes. 148 */ 149 tmp = readl(addr) & mask; 150 tmp |= val << ((where & 0x3) * 8); 151 writel(tmp, addr); 152 153 return PCIBIOS_SUCCESSFUL; 154 } 155 156 static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip, 157 struct pci_bus *bus, u32 devfn, 158 int where, int size, u32 *val) 159 { 160 u32 busdev; 161 162 busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn), 163 PCI_FUNC(devfn), where); 164 165 if (!IS_ALIGNED(busdev, size)) { 166 *val = 0; 167 return PCIBIOS_BAD_REGISTER_NUMBER; 168 } 169 170 if (pci_is_root_bus(bus->parent)) 171 rockchip_pcie_cfg_configuration_accesses(rockchip, 172 AXI_WRAPPER_TYPE0_CFG); 173 else 174 rockchip_pcie_cfg_configuration_accesses(rockchip, 175 AXI_WRAPPER_TYPE1_CFG); 176 177 if (size == 4) { 178 *val = readl(rockchip->reg_base + busdev); 179 } else if (size == 2) { 180 *val = readw(rockchip->reg_base + busdev); 181 } else if (size == 1) { 182 *val = readb(rockchip->reg_base + busdev); 183 } else { 184 *val = 0; 185 return PCIBIOS_BAD_REGISTER_NUMBER; 186 } 187 return PCIBIOS_SUCCESSFUL; 188 } 189 190 static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip, 191 struct pci_bus *bus, u32 devfn, 192 int where, int size, u32 val) 193 { 194 u32 busdev; 195 196 busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn), 197 PCI_FUNC(devfn), where); 198 if (!IS_ALIGNED(busdev, size)) 199 return PCIBIOS_BAD_REGISTER_NUMBER; 200 201 if (pci_is_root_bus(bus->parent)) 202 rockchip_pcie_cfg_configuration_accesses(rockchip, 203 AXI_WRAPPER_TYPE0_CFG); 204 else 205 rockchip_pcie_cfg_configuration_accesses(rockchip, 206 AXI_WRAPPER_TYPE1_CFG); 207 208 if (size == 4) 209 writel(val, rockchip->reg_base + busdev); 210 else if (size == 2) 211 writew(val, rockchip->reg_base + busdev); 212 else if (size == 1) 213 writeb(val, rockchip->reg_base + busdev); 214 else 215 return PCIBIOS_BAD_REGISTER_NUMBER; 216 217 return PCIBIOS_SUCCESSFUL; 218 } 219 220 static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where, 221 int size, u32 *val) 222 { 223 struct rockchip_pcie *rockchip = bus->sysdata; 224 225 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) { 226 *val = 0xffffffff; 227 return PCIBIOS_DEVICE_NOT_FOUND; 228 } 229 230 if (pci_is_root_bus(bus)) 231 return rockchip_pcie_rd_own_conf(rockchip, where, size, val); 232 233 return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, 234 val); 235 } 236 237 static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn, 238 int where, int size, u32 val) 239 { 240 struct rockchip_pcie *rockchip = bus->sysdata; 241 242 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) 243 return PCIBIOS_DEVICE_NOT_FOUND; 244 245 if (pci_is_root_bus(bus)) 246 return rockchip_pcie_wr_own_conf(rockchip, where, size, val); 247 248 return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, 249 val); 250 } 251 252 static struct pci_ops rockchip_pcie_ops = { 253 .read = rockchip_pcie_rd_conf, 254 .write = rockchip_pcie_wr_conf, 255 }; 256 257 static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip) 258 { 259 int curr; 260 u32 status, scale, power; 261 262 if (IS_ERR(rockchip->vpcie3v3)) 263 return; 264 265 /* 266 * Set RC's captured slot power limit and scale if 267 * vpcie3v3 available. The default values are both zero 268 * which means the software should set these two according 269 * to the actual power supply. 270 */ 271 curr = regulator_get_current_limit(rockchip->vpcie3v3); 272 if (curr <= 0) 273 return; 274 275 scale = 3; /* 0.001x */ 276 curr = curr / 1000; /* convert to mA */ 277 power = (curr * 3300) / 1000; /* milliwatt */ 278 while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) { 279 if (!scale) { 280 dev_warn(rockchip->dev, "invalid power supply\n"); 281 return; 282 } 283 scale--; 284 power = power / 10; 285 } 286 287 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR); 288 status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) | 289 (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT); 290 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR); 291 } 292 293 /** 294 * rockchip_pcie_host_init_port - Initialize hardware 295 * @rockchip: PCIe port information 296 */ 297 static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip) 298 { 299 struct device *dev = rockchip->dev; 300 int err, i = MAX_LANE_NUM; 301 u32 status; 302 303 gpiod_set_value_cansleep(rockchip->ep_gpio, 0); 304 305 err = rockchip_pcie_init_port(rockchip); 306 if (err) 307 return err; 308 309 /* Fix the transmitted FTS count desired to exit from L0s. */ 310 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1); 311 status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) | 312 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT); 313 rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1); 314 315 rockchip_pcie_set_power_limit(rockchip); 316 317 /* Set RC's clock architecture as common clock */ 318 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 319 status |= PCI_EXP_LNKSTA_SLC << 16; 320 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 321 322 /* Set RC's RCB to 128 */ 323 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 324 status |= PCI_EXP_LNKCTL_RCB; 325 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 326 327 /* Enable Gen1 training */ 328 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, 329 PCIE_CLIENT_CONFIG); 330 331 gpiod_set_value_cansleep(rockchip->ep_gpio, 1); 332 333 /* 500ms timeout value should be enough for Gen1/2 training */ 334 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1, 335 status, PCIE_LINK_UP(status), 20, 336 500 * USEC_PER_MSEC); 337 if (err) { 338 dev_err(dev, "PCIe link training gen1 timeout!\n"); 339 goto err_power_off_phy; 340 } 341 342 if (rockchip->link_gen == 2) { 343 /* 344 * Enable retrain for gen2. This should be configured only after 345 * gen1 finished. 346 */ 347 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 348 status |= PCI_EXP_LNKCTL_RL; 349 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 350 351 err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL, 352 status, PCIE_LINK_IS_GEN2(status), 20, 353 500 * USEC_PER_MSEC); 354 if (err) 355 dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n"); 356 } 357 358 /* Check the final link width from negotiated lane counter from MGMT */ 359 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL); 360 status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >> 361 PCIE_CORE_PL_CONF_LANE_SHIFT); 362 dev_dbg(dev, "current link width is x%d\n", status); 363 364 /* Power off unused lane(s) */ 365 rockchip->lanes_map = rockchip_pcie_lane_map(rockchip); 366 for (i = 0; i < MAX_LANE_NUM; i++) { 367 if (!(rockchip->lanes_map & BIT(i))) { 368 dev_dbg(dev, "idling lane %d\n", i); 369 phy_power_off(rockchip->phys[i]); 370 } 371 } 372 373 rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID, 374 PCIE_CORE_CONFIG_VENDOR); 375 rockchip_pcie_write(rockchip, 376 PCI_CLASS_BRIDGE_PCI << PCIE_RC_CONFIG_SCC_SHIFT, 377 PCIE_RC_CONFIG_RID_CCR); 378 379 /* Clear THP cap's next cap pointer to remove L1 substate cap */ 380 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP); 381 status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK; 382 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP); 383 384 /* Clear L0s from RC's link cap */ 385 if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) { 386 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP); 387 status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S; 388 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP); 389 } 390 391 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR); 392 status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK; 393 status |= PCIE_RC_CONFIG_DCSR_MPS_256; 394 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR); 395 396 return 0; 397 err_power_off_phy: 398 while (i--) 399 phy_power_off(rockchip->phys[i]); 400 i = MAX_LANE_NUM; 401 while (i--) 402 phy_exit(rockchip->phys[i]); 403 return err; 404 } 405 406 static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg) 407 { 408 struct rockchip_pcie *rockchip = arg; 409 struct device *dev = rockchip->dev; 410 u32 reg; 411 u32 sub_reg; 412 413 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 414 if (reg & PCIE_CLIENT_INT_LOCAL) { 415 dev_dbg(dev, "local interrupt received\n"); 416 sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS); 417 if (sub_reg & PCIE_CORE_INT_PRFPE) 418 dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n"); 419 420 if (sub_reg & PCIE_CORE_INT_CRFPE) 421 dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n"); 422 423 if (sub_reg & PCIE_CORE_INT_RRPE) 424 dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n"); 425 426 if (sub_reg & PCIE_CORE_INT_PRFO) 427 dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n"); 428 429 if (sub_reg & PCIE_CORE_INT_CRFO) 430 dev_dbg(dev, "overflow occurred in the completion receive FIFO\n"); 431 432 if (sub_reg & PCIE_CORE_INT_RT) 433 dev_dbg(dev, "replay timer timed out\n"); 434 435 if (sub_reg & PCIE_CORE_INT_RTR) 436 dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n"); 437 438 if (sub_reg & PCIE_CORE_INT_PE) 439 dev_dbg(dev, "phy error detected on receive side\n"); 440 441 if (sub_reg & PCIE_CORE_INT_MTR) 442 dev_dbg(dev, "malformed TLP received from the link\n"); 443 444 if (sub_reg & PCIE_CORE_INT_UCR) 445 dev_dbg(dev, "malformed TLP received from the link\n"); 446 447 if (sub_reg & PCIE_CORE_INT_FCE) 448 dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n"); 449 450 if (sub_reg & PCIE_CORE_INT_CT) 451 dev_dbg(dev, "a request timed out waiting for completion\n"); 452 453 if (sub_reg & PCIE_CORE_INT_UTC) 454 dev_dbg(dev, "unmapped TC error\n"); 455 456 if (sub_reg & PCIE_CORE_INT_MMVC) 457 dev_dbg(dev, "MSI mask register changes\n"); 458 459 rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS); 460 } else if (reg & PCIE_CLIENT_INT_PHY) { 461 dev_dbg(dev, "phy link changes\n"); 462 rockchip_pcie_update_txcredit_mui(rockchip); 463 rockchip_pcie_clr_bw_int(rockchip); 464 } 465 466 rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL, 467 PCIE_CLIENT_INT_STATUS); 468 469 return IRQ_HANDLED; 470 } 471 472 static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg) 473 { 474 struct rockchip_pcie *rockchip = arg; 475 struct device *dev = rockchip->dev; 476 u32 reg; 477 478 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 479 if (reg & PCIE_CLIENT_INT_LEGACY_DONE) 480 dev_dbg(dev, "legacy done interrupt received\n"); 481 482 if (reg & PCIE_CLIENT_INT_MSG) 483 dev_dbg(dev, "message done interrupt received\n"); 484 485 if (reg & PCIE_CLIENT_INT_HOT_RST) 486 dev_dbg(dev, "hot reset interrupt received\n"); 487 488 if (reg & PCIE_CLIENT_INT_DPA) 489 dev_dbg(dev, "dpa interrupt received\n"); 490 491 if (reg & PCIE_CLIENT_INT_FATAL_ERR) 492 dev_dbg(dev, "fatal error interrupt received\n"); 493 494 if (reg & PCIE_CLIENT_INT_NFATAL_ERR) 495 dev_dbg(dev, "no fatal error interrupt received\n"); 496 497 if (reg & PCIE_CLIENT_INT_CORR_ERR) 498 dev_dbg(dev, "correctable error interrupt received\n"); 499 500 if (reg & PCIE_CLIENT_INT_PHY) 501 dev_dbg(dev, "phy interrupt received\n"); 502 503 rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE | 504 PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST | 505 PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR | 506 PCIE_CLIENT_INT_NFATAL_ERR | 507 PCIE_CLIENT_INT_CORR_ERR | 508 PCIE_CLIENT_INT_PHY), 509 PCIE_CLIENT_INT_STATUS); 510 511 return IRQ_HANDLED; 512 } 513 514 static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc) 515 { 516 struct irq_chip *chip = irq_desc_get_chip(desc); 517 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc); 518 struct device *dev = rockchip->dev; 519 u32 reg; 520 u32 hwirq; 521 u32 virq; 522 523 chained_irq_enter(chip, desc); 524 525 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 526 reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT; 527 528 while (reg) { 529 hwirq = ffs(reg) - 1; 530 reg &= ~BIT(hwirq); 531 532 virq = irq_find_mapping(rockchip->irq_domain, hwirq); 533 if (virq) 534 generic_handle_irq(virq); 535 else 536 dev_err(dev, "unexpected IRQ, INT%d\n", hwirq); 537 } 538 539 chained_irq_exit(chip, desc); 540 } 541 542 static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip) 543 { 544 int irq, err; 545 struct device *dev = rockchip->dev; 546 struct platform_device *pdev = to_platform_device(dev); 547 548 irq = platform_get_irq_byname(pdev, "sys"); 549 if (irq < 0) 550 return irq; 551 552 err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler, 553 IRQF_SHARED, "pcie-sys", rockchip); 554 if (err) { 555 dev_err(dev, "failed to request PCIe subsystem IRQ\n"); 556 return err; 557 } 558 559 irq = platform_get_irq_byname(pdev, "legacy"); 560 if (irq < 0) 561 return irq; 562 563 irq_set_chained_handler_and_data(irq, 564 rockchip_pcie_legacy_int_handler, 565 rockchip); 566 567 irq = platform_get_irq_byname(pdev, "client"); 568 if (irq < 0) 569 return irq; 570 571 err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler, 572 IRQF_SHARED, "pcie-client", rockchip); 573 if (err) { 574 dev_err(dev, "failed to request PCIe client IRQ\n"); 575 return err; 576 } 577 578 return 0; 579 } 580 581 /** 582 * rockchip_pcie_parse_host_dt - Parse Device Tree 583 * @rockchip: PCIe port information 584 * 585 * Return: '0' on success and error value on failure 586 */ 587 static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip) 588 { 589 struct device *dev = rockchip->dev; 590 int err; 591 592 err = rockchip_pcie_parse_dt(rockchip); 593 if (err) 594 return err; 595 596 err = rockchip_pcie_setup_irq(rockchip); 597 if (err) 598 return err; 599 600 rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v"); 601 if (IS_ERR(rockchip->vpcie12v)) { 602 if (PTR_ERR(rockchip->vpcie12v) != -ENODEV) 603 return PTR_ERR(rockchip->vpcie12v); 604 dev_info(dev, "no vpcie12v regulator found\n"); 605 } 606 607 rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3"); 608 if (IS_ERR(rockchip->vpcie3v3)) { 609 if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV) 610 return PTR_ERR(rockchip->vpcie3v3); 611 dev_info(dev, "no vpcie3v3 regulator found\n"); 612 } 613 614 rockchip->vpcie1v8 = devm_regulator_get(dev, "vpcie1v8"); 615 if (IS_ERR(rockchip->vpcie1v8)) 616 return PTR_ERR(rockchip->vpcie1v8); 617 618 rockchip->vpcie0v9 = devm_regulator_get(dev, "vpcie0v9"); 619 if (IS_ERR(rockchip->vpcie0v9)) 620 return PTR_ERR(rockchip->vpcie0v9); 621 622 return 0; 623 } 624 625 static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip) 626 { 627 struct device *dev = rockchip->dev; 628 int err; 629 630 if (!IS_ERR(rockchip->vpcie12v)) { 631 err = regulator_enable(rockchip->vpcie12v); 632 if (err) { 633 dev_err(dev, "fail to enable vpcie12v regulator\n"); 634 goto err_out; 635 } 636 } 637 638 if (!IS_ERR(rockchip->vpcie3v3)) { 639 err = regulator_enable(rockchip->vpcie3v3); 640 if (err) { 641 dev_err(dev, "fail to enable vpcie3v3 regulator\n"); 642 goto err_disable_12v; 643 } 644 } 645 646 err = regulator_enable(rockchip->vpcie1v8); 647 if (err) { 648 dev_err(dev, "fail to enable vpcie1v8 regulator\n"); 649 goto err_disable_3v3; 650 } 651 652 err = regulator_enable(rockchip->vpcie0v9); 653 if (err) { 654 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 655 goto err_disable_1v8; 656 } 657 658 return 0; 659 660 err_disable_1v8: 661 regulator_disable(rockchip->vpcie1v8); 662 err_disable_3v3: 663 if (!IS_ERR(rockchip->vpcie3v3)) 664 regulator_disable(rockchip->vpcie3v3); 665 err_disable_12v: 666 if (!IS_ERR(rockchip->vpcie12v)) 667 regulator_disable(rockchip->vpcie12v); 668 err_out: 669 return err; 670 } 671 672 static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip) 673 { 674 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) & 675 (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK); 676 rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT), 677 PCIE_CORE_INT_MASK); 678 679 rockchip_pcie_enable_bw_int(rockchip); 680 } 681 682 static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq, 683 irq_hw_number_t hwirq) 684 { 685 irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq); 686 irq_set_chip_data(irq, domain->host_data); 687 688 return 0; 689 } 690 691 static const struct irq_domain_ops intx_domain_ops = { 692 .map = rockchip_pcie_intx_map, 693 }; 694 695 static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) 696 { 697 struct device *dev = rockchip->dev; 698 struct device_node *intc = of_get_next_child(dev->of_node, NULL); 699 700 if (!intc) { 701 dev_err(dev, "missing child interrupt-controller node\n"); 702 return -EINVAL; 703 } 704 705 rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX, 706 &intx_domain_ops, rockchip); 707 of_node_put(intc); 708 if (!rockchip->irq_domain) { 709 dev_err(dev, "failed to get a INTx IRQ domain\n"); 710 return -EINVAL; 711 } 712 713 return 0; 714 } 715 716 static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip, 717 int region_no, int type, u8 num_pass_bits, 718 u32 lower_addr, u32 upper_addr) 719 { 720 u32 ob_addr_0; 721 u32 ob_addr_1; 722 u32 ob_desc_0; 723 u32 aw_offset; 724 725 if (region_no >= MAX_AXI_WRAPPER_REGION_NUM) 726 return -EINVAL; 727 if (num_pass_bits + 1 < 8) 728 return -EINVAL; 729 if (num_pass_bits > 63) 730 return -EINVAL; 731 if (region_no == 0) { 732 if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits)) 733 return -EINVAL; 734 } 735 if (region_no != 0) { 736 if (AXI_REGION_SIZE < (2ULL << num_pass_bits)) 737 return -EINVAL; 738 } 739 740 aw_offset = (region_no << OB_REG_SIZE_SHIFT); 741 742 ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS; 743 ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR; 744 ob_addr_1 = upper_addr; 745 ob_desc_0 = (1 << 23 | type); 746 747 rockchip_pcie_write(rockchip, ob_addr_0, 748 PCIE_CORE_OB_REGION_ADDR0 + aw_offset); 749 rockchip_pcie_write(rockchip, ob_addr_1, 750 PCIE_CORE_OB_REGION_ADDR1 + aw_offset); 751 rockchip_pcie_write(rockchip, ob_desc_0, 752 PCIE_CORE_OB_REGION_DESC0 + aw_offset); 753 rockchip_pcie_write(rockchip, 0, 754 PCIE_CORE_OB_REGION_DESC1 + aw_offset); 755 756 return 0; 757 } 758 759 static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip, 760 int region_no, u8 num_pass_bits, 761 u32 lower_addr, u32 upper_addr) 762 { 763 u32 ib_addr_0; 764 u32 ib_addr_1; 765 u32 aw_offset; 766 767 if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM) 768 return -EINVAL; 769 if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED) 770 return -EINVAL; 771 if (num_pass_bits > 63) 772 return -EINVAL; 773 774 aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT); 775 776 ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS; 777 ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR; 778 ib_addr_1 = upper_addr; 779 780 rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset); 781 rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset); 782 783 return 0; 784 } 785 786 static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip) 787 { 788 struct device *dev = rockchip->dev; 789 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 790 struct resource_entry *entry; 791 u64 pci_addr, size; 792 int offset; 793 int err; 794 int reg_no; 795 796 rockchip_pcie_cfg_configuration_accesses(rockchip, 797 AXI_WRAPPER_TYPE0_CFG); 798 entry = resource_list_first_type(&bridge->windows, IORESOURCE_MEM); 799 if (!entry) 800 return -ENODEV; 801 802 size = resource_size(entry->res); 803 pci_addr = entry->res->start - entry->offset; 804 rockchip->msg_bus_addr = pci_addr; 805 806 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 807 err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1, 808 AXI_WRAPPER_MEM_WRITE, 809 20 - 1, 810 pci_addr + (reg_no << 20), 811 0); 812 if (err) { 813 dev_err(dev, "program RC mem outbound ATU failed\n"); 814 return err; 815 } 816 } 817 818 err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0); 819 if (err) { 820 dev_err(dev, "program RC mem inbound ATU failed\n"); 821 return err; 822 } 823 824 entry = resource_list_first_type(&bridge->windows, IORESOURCE_IO); 825 if (!entry) 826 return -ENODEV; 827 828 /* store the register number offset to program RC io outbound ATU */ 829 offset = size >> 20; 830 831 size = resource_size(entry->res); 832 pci_addr = entry->res->start - entry->offset; 833 834 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 835 err = rockchip_pcie_prog_ob_atu(rockchip, 836 reg_no + 1 + offset, 837 AXI_WRAPPER_IO_WRITE, 838 20 - 1, 839 pci_addr + (reg_no << 20), 840 0); 841 if (err) { 842 dev_err(dev, "program RC io outbound ATU failed\n"); 843 return err; 844 } 845 } 846 847 /* assign message regions */ 848 rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset, 849 AXI_WRAPPER_NOR_MSG, 850 20 - 1, 0, 0); 851 852 rockchip->msg_bus_addr += ((reg_no + offset) << 20); 853 return err; 854 } 855 856 static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip) 857 { 858 u32 value; 859 int err; 860 861 /* send PME_TURN_OFF message */ 862 writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF); 863 864 /* read LTSSM and wait for falling into L2 link state */ 865 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0, 866 value, PCIE_LINK_IS_L2(value), 20, 867 jiffies_to_usecs(5 * HZ)); 868 if (err) { 869 dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n"); 870 return err; 871 } 872 873 return 0; 874 } 875 876 static int __maybe_unused rockchip_pcie_suspend_noirq(struct device *dev) 877 { 878 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 879 int ret; 880 881 /* disable core and cli int since we don't need to ack PME_ACK */ 882 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) | 883 PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK); 884 rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK); 885 886 ret = rockchip_pcie_wait_l2(rockchip); 887 if (ret) { 888 rockchip_pcie_enable_interrupts(rockchip); 889 return ret; 890 } 891 892 rockchip_pcie_deinit_phys(rockchip); 893 894 rockchip_pcie_disable_clocks(rockchip); 895 896 regulator_disable(rockchip->vpcie0v9); 897 898 return ret; 899 } 900 901 static int __maybe_unused rockchip_pcie_resume_noirq(struct device *dev) 902 { 903 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 904 int err; 905 906 err = regulator_enable(rockchip->vpcie0v9); 907 if (err) { 908 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 909 return err; 910 } 911 912 err = rockchip_pcie_enable_clocks(rockchip); 913 if (err) 914 goto err_disable_0v9; 915 916 err = rockchip_pcie_host_init_port(rockchip); 917 if (err) 918 goto err_pcie_resume; 919 920 err = rockchip_pcie_cfg_atu(rockchip); 921 if (err) 922 goto err_err_deinit_port; 923 924 /* Need this to enter L1 again */ 925 rockchip_pcie_update_txcredit_mui(rockchip); 926 rockchip_pcie_enable_interrupts(rockchip); 927 928 return 0; 929 930 err_err_deinit_port: 931 rockchip_pcie_deinit_phys(rockchip); 932 err_pcie_resume: 933 rockchip_pcie_disable_clocks(rockchip); 934 err_disable_0v9: 935 regulator_disable(rockchip->vpcie0v9); 936 return err; 937 } 938 939 static int rockchip_pcie_probe(struct platform_device *pdev) 940 { 941 struct rockchip_pcie *rockchip; 942 struct device *dev = &pdev->dev; 943 struct pci_host_bridge *bridge; 944 int err; 945 946 if (!dev->of_node) 947 return -ENODEV; 948 949 bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip)); 950 if (!bridge) 951 return -ENOMEM; 952 953 rockchip = pci_host_bridge_priv(bridge); 954 955 platform_set_drvdata(pdev, rockchip); 956 rockchip->dev = dev; 957 rockchip->is_rc = true; 958 959 err = rockchip_pcie_parse_host_dt(rockchip); 960 if (err) 961 return err; 962 963 err = rockchip_pcie_enable_clocks(rockchip); 964 if (err) 965 return err; 966 967 err = rockchip_pcie_set_vpcie(rockchip); 968 if (err) { 969 dev_err(dev, "failed to set vpcie regulator\n"); 970 goto err_set_vpcie; 971 } 972 973 err = rockchip_pcie_host_init_port(rockchip); 974 if (err) 975 goto err_vpcie; 976 977 rockchip_pcie_enable_interrupts(rockchip); 978 979 err = rockchip_pcie_init_irq_domain(rockchip); 980 if (err < 0) 981 goto err_deinit_port; 982 983 err = rockchip_pcie_cfg_atu(rockchip); 984 if (err) 985 goto err_remove_irq_domain; 986 987 rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M); 988 if (!rockchip->msg_region) { 989 err = -ENOMEM; 990 goto err_remove_irq_domain; 991 } 992 993 bridge->sysdata = rockchip; 994 bridge->ops = &rockchip_pcie_ops; 995 996 err = pci_host_probe(bridge); 997 if (err < 0) 998 goto err_remove_irq_domain; 999 1000 return 0; 1001 1002 err_remove_irq_domain: 1003 irq_domain_remove(rockchip->irq_domain); 1004 err_deinit_port: 1005 rockchip_pcie_deinit_phys(rockchip); 1006 err_vpcie: 1007 if (!IS_ERR(rockchip->vpcie12v)) 1008 regulator_disable(rockchip->vpcie12v); 1009 if (!IS_ERR(rockchip->vpcie3v3)) 1010 regulator_disable(rockchip->vpcie3v3); 1011 regulator_disable(rockchip->vpcie1v8); 1012 regulator_disable(rockchip->vpcie0v9); 1013 err_set_vpcie: 1014 rockchip_pcie_disable_clocks(rockchip); 1015 return err; 1016 } 1017 1018 static int rockchip_pcie_remove(struct platform_device *pdev) 1019 { 1020 struct device *dev = &pdev->dev; 1021 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 1022 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 1023 1024 pci_stop_root_bus(bridge->bus); 1025 pci_remove_root_bus(bridge->bus); 1026 irq_domain_remove(rockchip->irq_domain); 1027 1028 rockchip_pcie_deinit_phys(rockchip); 1029 1030 rockchip_pcie_disable_clocks(rockchip); 1031 1032 if (!IS_ERR(rockchip->vpcie12v)) 1033 regulator_disable(rockchip->vpcie12v); 1034 if (!IS_ERR(rockchip->vpcie3v3)) 1035 regulator_disable(rockchip->vpcie3v3); 1036 regulator_disable(rockchip->vpcie1v8); 1037 regulator_disable(rockchip->vpcie0v9); 1038 1039 return 0; 1040 } 1041 1042 static const struct dev_pm_ops rockchip_pcie_pm_ops = { 1043 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq, 1044 rockchip_pcie_resume_noirq) 1045 }; 1046 1047 static const struct of_device_id rockchip_pcie_of_match[] = { 1048 { .compatible = "rockchip,rk3399-pcie", }, 1049 {} 1050 }; 1051 MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match); 1052 1053 static struct platform_driver rockchip_pcie_driver = { 1054 .driver = { 1055 .name = "rockchip-pcie", 1056 .of_match_table = rockchip_pcie_of_match, 1057 .pm = &rockchip_pcie_pm_ops, 1058 }, 1059 .probe = rockchip_pcie_probe, 1060 .remove = rockchip_pcie_remove, 1061 }; 1062 module_platform_driver(rockchip_pcie_driver); 1063 1064 MODULE_AUTHOR("Rockchip Inc"); 1065 MODULE_DESCRIPTION("Rockchip AXI PCIe driver"); 1066 MODULE_LICENSE("GPL v2"); 1067