1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Rockchip AXI PCIe host controller driver 4 * 5 * Copyright (c) 2016 Rockchip, Inc. 6 * 7 * Author: Shawn Lin <shawn.lin@rock-chips.com> 8 * Wenrui Li <wenrui.li@rock-chips.com> 9 * 10 * Bits taken from Synopsys DesignWare Host controller driver and 11 * ARM PCI Host generic driver. 12 */ 13 14 #include <linux/bitrev.h> 15 #include <linux/clk.h> 16 #include <linux/delay.h> 17 #include <linux/gpio/consumer.h> 18 #include <linux/init.h> 19 #include <linux/interrupt.h> 20 #include <linux/iopoll.h> 21 #include <linux/irq.h> 22 #include <linux/irqchip/chained_irq.h> 23 #include <linux/irqdomain.h> 24 #include <linux/kernel.h> 25 #include <linux/mfd/syscon.h> 26 #include <linux/module.h> 27 #include <linux/of_address.h> 28 #include <linux/of_device.h> 29 #include <linux/of_pci.h> 30 #include <linux/of_platform.h> 31 #include <linux/of_irq.h> 32 #include <linux/pci.h> 33 #include <linux/pci_ids.h> 34 #include <linux/phy/phy.h> 35 #include <linux/platform_device.h> 36 #include <linux/reset.h> 37 #include <linux/regmap.h> 38 39 #include "../pci.h" 40 #include "pcie-rockchip.h" 41 42 static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip) 43 { 44 u32 status; 45 46 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 47 status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE); 48 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 49 } 50 51 static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip) 52 { 53 u32 status; 54 55 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 56 status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16; 57 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 58 } 59 60 static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip) 61 { 62 u32 val; 63 64 /* Update Tx credit maximum update interval */ 65 val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1); 66 val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK; 67 val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */ 68 rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1); 69 } 70 71 static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip, 72 struct pci_bus *bus, int dev) 73 { 74 /* access only one slot on each root port */ 75 if (pci_is_root_bus(bus) && dev > 0) 76 return 0; 77 78 /* 79 * do not read more than one device on the bus directly attached 80 * to RC's downstream side. 81 */ 82 if (pci_is_root_bus(bus->parent) && dev > 0) 83 return 0; 84 85 return 1; 86 } 87 88 static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip) 89 { 90 u32 val; 91 u8 map; 92 93 if (rockchip->legacy_phy) 94 return GENMASK(MAX_LANE_NUM - 1, 0); 95 96 val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP); 97 map = val & PCIE_CORE_LANE_MAP_MASK; 98 99 /* The link may be using a reverse-indexed mapping. */ 100 if (val & PCIE_CORE_LANE_MAP_REVERSE) 101 map = bitrev8(map) >> 4; 102 103 return map; 104 } 105 106 static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip, 107 int where, int size, u32 *val) 108 { 109 void __iomem *addr; 110 111 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where; 112 113 if (!IS_ALIGNED((uintptr_t)addr, size)) { 114 *val = 0; 115 return PCIBIOS_BAD_REGISTER_NUMBER; 116 } 117 118 if (size == 4) { 119 *val = readl(addr); 120 } else if (size == 2) { 121 *val = readw(addr); 122 } else if (size == 1) { 123 *val = readb(addr); 124 } else { 125 *val = 0; 126 return PCIBIOS_BAD_REGISTER_NUMBER; 127 } 128 return PCIBIOS_SUCCESSFUL; 129 } 130 131 static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip, 132 int where, int size, u32 val) 133 { 134 u32 mask, tmp, offset; 135 void __iomem *addr; 136 137 offset = where & ~0x3; 138 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset; 139 140 if (size == 4) { 141 writel(val, addr); 142 return PCIBIOS_SUCCESSFUL; 143 } 144 145 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); 146 147 /* 148 * N.B. This read/modify/write isn't safe in general because it can 149 * corrupt RW1C bits in adjacent registers. But the hardware 150 * doesn't support smaller writes. 151 */ 152 tmp = readl(addr) & mask; 153 tmp |= val << ((where & 0x3) * 8); 154 writel(tmp, addr); 155 156 return PCIBIOS_SUCCESSFUL; 157 } 158 159 static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip, 160 struct pci_bus *bus, u32 devfn, 161 int where, int size, u32 *val) 162 { 163 u32 busdev; 164 165 busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn), 166 PCI_FUNC(devfn), where); 167 168 if (!IS_ALIGNED(busdev, size)) { 169 *val = 0; 170 return PCIBIOS_BAD_REGISTER_NUMBER; 171 } 172 173 if (pci_is_root_bus(bus->parent)) 174 rockchip_pcie_cfg_configuration_accesses(rockchip, 175 AXI_WRAPPER_TYPE0_CFG); 176 else 177 rockchip_pcie_cfg_configuration_accesses(rockchip, 178 AXI_WRAPPER_TYPE1_CFG); 179 180 if (size == 4) { 181 *val = readl(rockchip->reg_base + busdev); 182 } else if (size == 2) { 183 *val = readw(rockchip->reg_base + busdev); 184 } else if (size == 1) { 185 *val = readb(rockchip->reg_base + busdev); 186 } else { 187 *val = 0; 188 return PCIBIOS_BAD_REGISTER_NUMBER; 189 } 190 return PCIBIOS_SUCCESSFUL; 191 } 192 193 static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip, 194 struct pci_bus *bus, u32 devfn, 195 int where, int size, u32 val) 196 { 197 u32 busdev; 198 199 busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn), 200 PCI_FUNC(devfn), where); 201 if (!IS_ALIGNED(busdev, size)) 202 return PCIBIOS_BAD_REGISTER_NUMBER; 203 204 if (pci_is_root_bus(bus->parent)) 205 rockchip_pcie_cfg_configuration_accesses(rockchip, 206 AXI_WRAPPER_TYPE0_CFG); 207 else 208 rockchip_pcie_cfg_configuration_accesses(rockchip, 209 AXI_WRAPPER_TYPE1_CFG); 210 211 if (size == 4) 212 writel(val, rockchip->reg_base + busdev); 213 else if (size == 2) 214 writew(val, rockchip->reg_base + busdev); 215 else if (size == 1) 216 writeb(val, rockchip->reg_base + busdev); 217 else 218 return PCIBIOS_BAD_REGISTER_NUMBER; 219 220 return PCIBIOS_SUCCESSFUL; 221 } 222 223 static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where, 224 int size, u32 *val) 225 { 226 struct rockchip_pcie *rockchip = bus->sysdata; 227 228 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) { 229 *val = 0xffffffff; 230 return PCIBIOS_DEVICE_NOT_FOUND; 231 } 232 233 if (pci_is_root_bus(bus)) 234 return rockchip_pcie_rd_own_conf(rockchip, where, size, val); 235 236 return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, 237 val); 238 } 239 240 static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn, 241 int where, int size, u32 val) 242 { 243 struct rockchip_pcie *rockchip = bus->sysdata; 244 245 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) 246 return PCIBIOS_DEVICE_NOT_FOUND; 247 248 if (pci_is_root_bus(bus)) 249 return rockchip_pcie_wr_own_conf(rockchip, where, size, val); 250 251 return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, 252 val); 253 } 254 255 static struct pci_ops rockchip_pcie_ops = { 256 .read = rockchip_pcie_rd_conf, 257 .write = rockchip_pcie_wr_conf, 258 }; 259 260 static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip) 261 { 262 int curr; 263 u32 status, scale, power; 264 265 if (IS_ERR(rockchip->vpcie3v3)) 266 return; 267 268 /* 269 * Set RC's captured slot power limit and scale if 270 * vpcie3v3 available. The default values are both zero 271 * which means the software should set these two according 272 * to the actual power supply. 273 */ 274 curr = regulator_get_current_limit(rockchip->vpcie3v3); 275 if (curr <= 0) 276 return; 277 278 scale = 3; /* 0.001x */ 279 curr = curr / 1000; /* convert to mA */ 280 power = (curr * 3300) / 1000; /* milliwatt */ 281 while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) { 282 if (!scale) { 283 dev_warn(rockchip->dev, "invalid power supply\n"); 284 return; 285 } 286 scale--; 287 power = power / 10; 288 } 289 290 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR); 291 status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) | 292 (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT); 293 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR); 294 } 295 296 /** 297 * rockchip_pcie_host_init_port - Initialize hardware 298 * @rockchip: PCIe port information 299 */ 300 static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip) 301 { 302 struct device *dev = rockchip->dev; 303 int err, i = MAX_LANE_NUM; 304 u32 status; 305 306 gpiod_set_value_cansleep(rockchip->ep_gpio, 0); 307 308 err = rockchip_pcie_init_port(rockchip); 309 if (err) 310 return err; 311 312 /* Fix the transmitted FTS count desired to exit from L0s. */ 313 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1); 314 status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) | 315 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT); 316 rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1); 317 318 rockchip_pcie_set_power_limit(rockchip); 319 320 /* Set RC's clock architecture as common clock */ 321 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 322 status |= PCI_EXP_LNKSTA_SLC << 16; 323 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 324 325 /* Set RC's RCB to 128 */ 326 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 327 status |= PCI_EXP_LNKCTL_RCB; 328 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 329 330 /* Enable Gen1 training */ 331 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, 332 PCIE_CLIENT_CONFIG); 333 334 gpiod_set_value_cansleep(rockchip->ep_gpio, 1); 335 336 /* 500ms timeout value should be enough for Gen1/2 training */ 337 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1, 338 status, PCIE_LINK_UP(status), 20, 339 500 * USEC_PER_MSEC); 340 if (err) { 341 dev_err(dev, "PCIe link training gen1 timeout!\n"); 342 goto err_power_off_phy; 343 } 344 345 if (rockchip->link_gen == 2) { 346 /* 347 * Enable retrain for gen2. This should be configured only after 348 * gen1 finished. 349 */ 350 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS); 351 status |= PCI_EXP_LNKCTL_RL; 352 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS); 353 354 err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL, 355 status, PCIE_LINK_IS_GEN2(status), 20, 356 500 * USEC_PER_MSEC); 357 if (err) 358 dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n"); 359 } 360 361 /* Check the final link width from negotiated lane counter from MGMT */ 362 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL); 363 status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >> 364 PCIE_CORE_PL_CONF_LANE_SHIFT); 365 dev_dbg(dev, "current link width is x%d\n", status); 366 367 /* Power off unused lane(s) */ 368 rockchip->lanes_map = rockchip_pcie_lane_map(rockchip); 369 for (i = 0; i < MAX_LANE_NUM; i++) { 370 if (!(rockchip->lanes_map & BIT(i))) { 371 dev_dbg(dev, "idling lane %d\n", i); 372 phy_power_off(rockchip->phys[i]); 373 } 374 } 375 376 rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID, 377 PCIE_CORE_CONFIG_VENDOR); 378 rockchip_pcie_write(rockchip, 379 PCI_CLASS_BRIDGE_PCI << PCIE_RC_CONFIG_SCC_SHIFT, 380 PCIE_RC_CONFIG_RID_CCR); 381 382 /* Clear THP cap's next cap pointer to remove L1 substate cap */ 383 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP); 384 status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK; 385 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP); 386 387 /* Clear L0s from RC's link cap */ 388 if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) { 389 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP); 390 status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S; 391 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP); 392 } 393 394 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR); 395 status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK; 396 status |= PCIE_RC_CONFIG_DCSR_MPS_256; 397 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR); 398 399 return 0; 400 err_power_off_phy: 401 while (i--) 402 phy_power_off(rockchip->phys[i]); 403 i = MAX_LANE_NUM; 404 while (i--) 405 phy_exit(rockchip->phys[i]); 406 return err; 407 } 408 409 static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg) 410 { 411 struct rockchip_pcie *rockchip = arg; 412 struct device *dev = rockchip->dev; 413 u32 reg; 414 u32 sub_reg; 415 416 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 417 if (reg & PCIE_CLIENT_INT_LOCAL) { 418 dev_dbg(dev, "local interrupt received\n"); 419 sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS); 420 if (sub_reg & PCIE_CORE_INT_PRFPE) 421 dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n"); 422 423 if (sub_reg & PCIE_CORE_INT_CRFPE) 424 dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n"); 425 426 if (sub_reg & PCIE_CORE_INT_RRPE) 427 dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n"); 428 429 if (sub_reg & PCIE_CORE_INT_PRFO) 430 dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n"); 431 432 if (sub_reg & PCIE_CORE_INT_CRFO) 433 dev_dbg(dev, "overflow occurred in the completion receive FIFO\n"); 434 435 if (sub_reg & PCIE_CORE_INT_RT) 436 dev_dbg(dev, "replay timer timed out\n"); 437 438 if (sub_reg & PCIE_CORE_INT_RTR) 439 dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n"); 440 441 if (sub_reg & PCIE_CORE_INT_PE) 442 dev_dbg(dev, "phy error detected on receive side\n"); 443 444 if (sub_reg & PCIE_CORE_INT_MTR) 445 dev_dbg(dev, "malformed TLP received from the link\n"); 446 447 if (sub_reg & PCIE_CORE_INT_UCR) 448 dev_dbg(dev, "malformed TLP received from the link\n"); 449 450 if (sub_reg & PCIE_CORE_INT_FCE) 451 dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n"); 452 453 if (sub_reg & PCIE_CORE_INT_CT) 454 dev_dbg(dev, "a request timed out waiting for completion\n"); 455 456 if (sub_reg & PCIE_CORE_INT_UTC) 457 dev_dbg(dev, "unmapped TC error\n"); 458 459 if (sub_reg & PCIE_CORE_INT_MMVC) 460 dev_dbg(dev, "MSI mask register changes\n"); 461 462 rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS); 463 } else if (reg & PCIE_CLIENT_INT_PHY) { 464 dev_dbg(dev, "phy link changes\n"); 465 rockchip_pcie_update_txcredit_mui(rockchip); 466 rockchip_pcie_clr_bw_int(rockchip); 467 } 468 469 rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL, 470 PCIE_CLIENT_INT_STATUS); 471 472 return IRQ_HANDLED; 473 } 474 475 static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg) 476 { 477 struct rockchip_pcie *rockchip = arg; 478 struct device *dev = rockchip->dev; 479 u32 reg; 480 481 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 482 if (reg & PCIE_CLIENT_INT_LEGACY_DONE) 483 dev_dbg(dev, "legacy done interrupt received\n"); 484 485 if (reg & PCIE_CLIENT_INT_MSG) 486 dev_dbg(dev, "message done interrupt received\n"); 487 488 if (reg & PCIE_CLIENT_INT_HOT_RST) 489 dev_dbg(dev, "hot reset interrupt received\n"); 490 491 if (reg & PCIE_CLIENT_INT_DPA) 492 dev_dbg(dev, "dpa interrupt received\n"); 493 494 if (reg & PCIE_CLIENT_INT_FATAL_ERR) 495 dev_dbg(dev, "fatal error interrupt received\n"); 496 497 if (reg & PCIE_CLIENT_INT_NFATAL_ERR) 498 dev_dbg(dev, "no fatal error interrupt received\n"); 499 500 if (reg & PCIE_CLIENT_INT_CORR_ERR) 501 dev_dbg(dev, "correctable error interrupt received\n"); 502 503 if (reg & PCIE_CLIENT_INT_PHY) 504 dev_dbg(dev, "phy interrupt received\n"); 505 506 rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE | 507 PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST | 508 PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR | 509 PCIE_CLIENT_INT_NFATAL_ERR | 510 PCIE_CLIENT_INT_CORR_ERR | 511 PCIE_CLIENT_INT_PHY), 512 PCIE_CLIENT_INT_STATUS); 513 514 return IRQ_HANDLED; 515 } 516 517 static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc) 518 { 519 struct irq_chip *chip = irq_desc_get_chip(desc); 520 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc); 521 struct device *dev = rockchip->dev; 522 u32 reg; 523 u32 hwirq; 524 u32 virq; 525 526 chained_irq_enter(chip, desc); 527 528 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); 529 reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT; 530 531 while (reg) { 532 hwirq = ffs(reg) - 1; 533 reg &= ~BIT(hwirq); 534 535 virq = irq_find_mapping(rockchip->irq_domain, hwirq); 536 if (virq) 537 generic_handle_irq(virq); 538 else 539 dev_err(dev, "unexpected IRQ, INT%d\n", hwirq); 540 } 541 542 chained_irq_exit(chip, desc); 543 } 544 545 static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip) 546 { 547 int irq, err; 548 struct device *dev = rockchip->dev; 549 struct platform_device *pdev = to_platform_device(dev); 550 551 irq = platform_get_irq_byname(pdev, "sys"); 552 if (irq < 0) 553 return irq; 554 555 err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler, 556 IRQF_SHARED, "pcie-sys", rockchip); 557 if (err) { 558 dev_err(dev, "failed to request PCIe subsystem IRQ\n"); 559 return err; 560 } 561 562 irq = platform_get_irq_byname(pdev, "legacy"); 563 if (irq < 0) 564 return irq; 565 566 irq_set_chained_handler_and_data(irq, 567 rockchip_pcie_legacy_int_handler, 568 rockchip); 569 570 irq = platform_get_irq_byname(pdev, "client"); 571 if (irq < 0) 572 return irq; 573 574 err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler, 575 IRQF_SHARED, "pcie-client", rockchip); 576 if (err) { 577 dev_err(dev, "failed to request PCIe client IRQ\n"); 578 return err; 579 } 580 581 return 0; 582 } 583 584 /** 585 * rockchip_pcie_parse_host_dt - Parse Device Tree 586 * @rockchip: PCIe port information 587 * 588 * Return: '0' on success and error value on failure 589 */ 590 static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip) 591 { 592 struct device *dev = rockchip->dev; 593 int err; 594 595 err = rockchip_pcie_parse_dt(rockchip); 596 if (err) 597 return err; 598 599 err = rockchip_pcie_setup_irq(rockchip); 600 if (err) 601 return err; 602 603 rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v"); 604 if (IS_ERR(rockchip->vpcie12v)) { 605 if (PTR_ERR(rockchip->vpcie12v) != -ENODEV) 606 return PTR_ERR(rockchip->vpcie12v); 607 dev_info(dev, "no vpcie12v regulator found\n"); 608 } 609 610 rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3"); 611 if (IS_ERR(rockchip->vpcie3v3)) { 612 if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV) 613 return PTR_ERR(rockchip->vpcie3v3); 614 dev_info(dev, "no vpcie3v3 regulator found\n"); 615 } 616 617 rockchip->vpcie1v8 = devm_regulator_get(dev, "vpcie1v8"); 618 if (IS_ERR(rockchip->vpcie1v8)) 619 return PTR_ERR(rockchip->vpcie1v8); 620 621 rockchip->vpcie0v9 = devm_regulator_get(dev, "vpcie0v9"); 622 if (IS_ERR(rockchip->vpcie0v9)) 623 return PTR_ERR(rockchip->vpcie0v9); 624 625 return 0; 626 } 627 628 static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip) 629 { 630 struct device *dev = rockchip->dev; 631 int err; 632 633 if (!IS_ERR(rockchip->vpcie12v)) { 634 err = regulator_enable(rockchip->vpcie12v); 635 if (err) { 636 dev_err(dev, "fail to enable vpcie12v regulator\n"); 637 goto err_out; 638 } 639 } 640 641 if (!IS_ERR(rockchip->vpcie3v3)) { 642 err = regulator_enable(rockchip->vpcie3v3); 643 if (err) { 644 dev_err(dev, "fail to enable vpcie3v3 regulator\n"); 645 goto err_disable_12v; 646 } 647 } 648 649 err = regulator_enable(rockchip->vpcie1v8); 650 if (err) { 651 dev_err(dev, "fail to enable vpcie1v8 regulator\n"); 652 goto err_disable_3v3; 653 } 654 655 err = regulator_enable(rockchip->vpcie0v9); 656 if (err) { 657 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 658 goto err_disable_1v8; 659 } 660 661 return 0; 662 663 err_disable_1v8: 664 regulator_disable(rockchip->vpcie1v8); 665 err_disable_3v3: 666 if (!IS_ERR(rockchip->vpcie3v3)) 667 regulator_disable(rockchip->vpcie3v3); 668 err_disable_12v: 669 if (!IS_ERR(rockchip->vpcie12v)) 670 regulator_disable(rockchip->vpcie12v); 671 err_out: 672 return err; 673 } 674 675 static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip) 676 { 677 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) & 678 (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK); 679 rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT), 680 PCIE_CORE_INT_MASK); 681 682 rockchip_pcie_enable_bw_int(rockchip); 683 } 684 685 static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq, 686 irq_hw_number_t hwirq) 687 { 688 irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq); 689 irq_set_chip_data(irq, domain->host_data); 690 691 return 0; 692 } 693 694 static const struct irq_domain_ops intx_domain_ops = { 695 .map = rockchip_pcie_intx_map, 696 }; 697 698 static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) 699 { 700 struct device *dev = rockchip->dev; 701 struct device_node *intc = of_get_next_child(dev->of_node, NULL); 702 703 if (!intc) { 704 dev_err(dev, "missing child interrupt-controller node\n"); 705 return -EINVAL; 706 } 707 708 rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX, 709 &intx_domain_ops, rockchip); 710 of_node_put(intc); 711 if (!rockchip->irq_domain) { 712 dev_err(dev, "failed to get a INTx IRQ domain\n"); 713 return -EINVAL; 714 } 715 716 return 0; 717 } 718 719 static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip, 720 int region_no, int type, u8 num_pass_bits, 721 u32 lower_addr, u32 upper_addr) 722 { 723 u32 ob_addr_0; 724 u32 ob_addr_1; 725 u32 ob_desc_0; 726 u32 aw_offset; 727 728 if (region_no >= MAX_AXI_WRAPPER_REGION_NUM) 729 return -EINVAL; 730 if (num_pass_bits + 1 < 8) 731 return -EINVAL; 732 if (num_pass_bits > 63) 733 return -EINVAL; 734 if (region_no == 0) { 735 if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits)) 736 return -EINVAL; 737 } 738 if (region_no != 0) { 739 if (AXI_REGION_SIZE < (2ULL << num_pass_bits)) 740 return -EINVAL; 741 } 742 743 aw_offset = (region_no << OB_REG_SIZE_SHIFT); 744 745 ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS; 746 ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR; 747 ob_addr_1 = upper_addr; 748 ob_desc_0 = (1 << 23 | type); 749 750 rockchip_pcie_write(rockchip, ob_addr_0, 751 PCIE_CORE_OB_REGION_ADDR0 + aw_offset); 752 rockchip_pcie_write(rockchip, ob_addr_1, 753 PCIE_CORE_OB_REGION_ADDR1 + aw_offset); 754 rockchip_pcie_write(rockchip, ob_desc_0, 755 PCIE_CORE_OB_REGION_DESC0 + aw_offset); 756 rockchip_pcie_write(rockchip, 0, 757 PCIE_CORE_OB_REGION_DESC1 + aw_offset); 758 759 return 0; 760 } 761 762 static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip, 763 int region_no, u8 num_pass_bits, 764 u32 lower_addr, u32 upper_addr) 765 { 766 u32 ib_addr_0; 767 u32 ib_addr_1; 768 u32 aw_offset; 769 770 if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM) 771 return -EINVAL; 772 if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED) 773 return -EINVAL; 774 if (num_pass_bits > 63) 775 return -EINVAL; 776 777 aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT); 778 779 ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS; 780 ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR; 781 ib_addr_1 = upper_addr; 782 783 rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset); 784 rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset); 785 786 return 0; 787 } 788 789 static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip) 790 { 791 struct device *dev = rockchip->dev; 792 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 793 struct resource_entry *entry; 794 u64 pci_addr, size; 795 int offset; 796 int err; 797 int reg_no; 798 799 rockchip_pcie_cfg_configuration_accesses(rockchip, 800 AXI_WRAPPER_TYPE0_CFG); 801 entry = resource_list_first_type(&bridge->windows, IORESOURCE_MEM); 802 if (!entry) 803 return -ENODEV; 804 805 size = resource_size(entry->res); 806 pci_addr = entry->res->start - entry->offset; 807 rockchip->msg_bus_addr = pci_addr; 808 809 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 810 err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1, 811 AXI_WRAPPER_MEM_WRITE, 812 20 - 1, 813 pci_addr + (reg_no << 20), 814 0); 815 if (err) { 816 dev_err(dev, "program RC mem outbound ATU failed\n"); 817 return err; 818 } 819 } 820 821 err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0); 822 if (err) { 823 dev_err(dev, "program RC mem inbound ATU failed\n"); 824 return err; 825 } 826 827 entry = resource_list_first_type(&bridge->windows, IORESOURCE_IO); 828 if (!entry) 829 return -ENODEV; 830 831 /* store the register number offset to program RC io outbound ATU */ 832 offset = size >> 20; 833 834 size = resource_size(entry->res); 835 pci_addr = entry->res->start - entry->offset; 836 837 for (reg_no = 0; reg_no < (size >> 20); reg_no++) { 838 err = rockchip_pcie_prog_ob_atu(rockchip, 839 reg_no + 1 + offset, 840 AXI_WRAPPER_IO_WRITE, 841 20 - 1, 842 pci_addr + (reg_no << 20), 843 0); 844 if (err) { 845 dev_err(dev, "program RC io outbound ATU failed\n"); 846 return err; 847 } 848 } 849 850 /* assign message regions */ 851 rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset, 852 AXI_WRAPPER_NOR_MSG, 853 20 - 1, 0, 0); 854 855 rockchip->msg_bus_addr += ((reg_no + offset) << 20); 856 return err; 857 } 858 859 static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip) 860 { 861 u32 value; 862 int err; 863 864 /* send PME_TURN_OFF message */ 865 writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF); 866 867 /* read LTSSM and wait for falling into L2 link state */ 868 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0, 869 value, PCIE_LINK_IS_L2(value), 20, 870 jiffies_to_usecs(5 * HZ)); 871 if (err) { 872 dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n"); 873 return err; 874 } 875 876 return 0; 877 } 878 879 static int __maybe_unused rockchip_pcie_suspend_noirq(struct device *dev) 880 { 881 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 882 int ret; 883 884 /* disable core and cli int since we don't need to ack PME_ACK */ 885 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) | 886 PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK); 887 rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK); 888 889 ret = rockchip_pcie_wait_l2(rockchip); 890 if (ret) { 891 rockchip_pcie_enable_interrupts(rockchip); 892 return ret; 893 } 894 895 rockchip_pcie_deinit_phys(rockchip); 896 897 rockchip_pcie_disable_clocks(rockchip); 898 899 regulator_disable(rockchip->vpcie0v9); 900 901 return ret; 902 } 903 904 static int __maybe_unused rockchip_pcie_resume_noirq(struct device *dev) 905 { 906 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 907 int err; 908 909 err = regulator_enable(rockchip->vpcie0v9); 910 if (err) { 911 dev_err(dev, "fail to enable vpcie0v9 regulator\n"); 912 return err; 913 } 914 915 err = rockchip_pcie_enable_clocks(rockchip); 916 if (err) 917 goto err_disable_0v9; 918 919 err = rockchip_pcie_host_init_port(rockchip); 920 if (err) 921 goto err_pcie_resume; 922 923 err = rockchip_pcie_cfg_atu(rockchip); 924 if (err) 925 goto err_err_deinit_port; 926 927 /* Need this to enter L1 again */ 928 rockchip_pcie_update_txcredit_mui(rockchip); 929 rockchip_pcie_enable_interrupts(rockchip); 930 931 return 0; 932 933 err_err_deinit_port: 934 rockchip_pcie_deinit_phys(rockchip); 935 err_pcie_resume: 936 rockchip_pcie_disable_clocks(rockchip); 937 err_disable_0v9: 938 regulator_disable(rockchip->vpcie0v9); 939 return err; 940 } 941 942 static int rockchip_pcie_probe(struct platform_device *pdev) 943 { 944 struct rockchip_pcie *rockchip; 945 struct device *dev = &pdev->dev; 946 struct pci_host_bridge *bridge; 947 int err; 948 949 if (!dev->of_node) 950 return -ENODEV; 951 952 bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip)); 953 if (!bridge) 954 return -ENOMEM; 955 956 rockchip = pci_host_bridge_priv(bridge); 957 958 platform_set_drvdata(pdev, rockchip); 959 rockchip->dev = dev; 960 rockchip->is_rc = true; 961 962 err = rockchip_pcie_parse_host_dt(rockchip); 963 if (err) 964 return err; 965 966 err = rockchip_pcie_enable_clocks(rockchip); 967 if (err) 968 return err; 969 970 err = rockchip_pcie_set_vpcie(rockchip); 971 if (err) { 972 dev_err(dev, "failed to set vpcie regulator\n"); 973 goto err_set_vpcie; 974 } 975 976 err = rockchip_pcie_host_init_port(rockchip); 977 if (err) 978 goto err_vpcie; 979 980 rockchip_pcie_enable_interrupts(rockchip); 981 982 err = rockchip_pcie_init_irq_domain(rockchip); 983 if (err < 0) 984 goto err_deinit_port; 985 986 err = rockchip_pcie_cfg_atu(rockchip); 987 if (err) 988 goto err_remove_irq_domain; 989 990 rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M); 991 if (!rockchip->msg_region) { 992 err = -ENOMEM; 993 goto err_remove_irq_domain; 994 } 995 996 bridge->sysdata = rockchip; 997 bridge->ops = &rockchip_pcie_ops; 998 999 err = pci_host_probe(bridge); 1000 if (err < 0) 1001 goto err_remove_irq_domain; 1002 1003 return 0; 1004 1005 err_remove_irq_domain: 1006 irq_domain_remove(rockchip->irq_domain); 1007 err_deinit_port: 1008 rockchip_pcie_deinit_phys(rockchip); 1009 err_vpcie: 1010 if (!IS_ERR(rockchip->vpcie12v)) 1011 regulator_disable(rockchip->vpcie12v); 1012 if (!IS_ERR(rockchip->vpcie3v3)) 1013 regulator_disable(rockchip->vpcie3v3); 1014 regulator_disable(rockchip->vpcie1v8); 1015 regulator_disable(rockchip->vpcie0v9); 1016 err_set_vpcie: 1017 rockchip_pcie_disable_clocks(rockchip); 1018 return err; 1019 } 1020 1021 static int rockchip_pcie_remove(struct platform_device *pdev) 1022 { 1023 struct device *dev = &pdev->dev; 1024 struct rockchip_pcie *rockchip = dev_get_drvdata(dev); 1025 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); 1026 1027 pci_stop_root_bus(bridge->bus); 1028 pci_remove_root_bus(bridge->bus); 1029 irq_domain_remove(rockchip->irq_domain); 1030 1031 rockchip_pcie_deinit_phys(rockchip); 1032 1033 rockchip_pcie_disable_clocks(rockchip); 1034 1035 if (!IS_ERR(rockchip->vpcie12v)) 1036 regulator_disable(rockchip->vpcie12v); 1037 if (!IS_ERR(rockchip->vpcie3v3)) 1038 regulator_disable(rockchip->vpcie3v3); 1039 regulator_disable(rockchip->vpcie1v8); 1040 regulator_disable(rockchip->vpcie0v9); 1041 1042 return 0; 1043 } 1044 1045 static const struct dev_pm_ops rockchip_pcie_pm_ops = { 1046 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq, 1047 rockchip_pcie_resume_noirq) 1048 }; 1049 1050 static const struct of_device_id rockchip_pcie_of_match[] = { 1051 { .compatible = "rockchip,rk3399-pcie", }, 1052 {} 1053 }; 1054 MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match); 1055 1056 static struct platform_driver rockchip_pcie_driver = { 1057 .driver = { 1058 .name = "rockchip-pcie", 1059 .of_match_table = rockchip_pcie_of_match, 1060 .pm = &rockchip_pcie_pm_ops, 1061 }, 1062 .probe = rockchip_pcie_probe, 1063 .remove = rockchip_pcie_remove, 1064 }; 1065 module_platform_driver(rockchip_pcie_driver); 1066 1067 MODULE_AUTHOR("Rockchip Inc"); 1068 MODULE_DESCRIPTION("Rockchip AXI PCIe driver"); 1069 MODULE_LICENSE("GPL v2"); 1070