1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * PCIe host controller driver for Tegra SoCs 4 * 5 * Copyright (c) 2010, CompuLab, Ltd. 6 * Author: Mike Rapoport <mike@compulab.co.il> 7 * 8 * Based on NVIDIA PCIe driver 9 * Copyright (c) 2008-2009, NVIDIA Corporation. 10 * 11 * Bits taken from arch/arm/mach-dove/pcie.c 12 * 13 * Author: Thierry Reding <treding@nvidia.com> 14 */ 15 16 #include <linux/clk.h> 17 #include <linux/debugfs.h> 18 #include <linux/delay.h> 19 #include <linux/export.h> 20 #include <linux/gpio/consumer.h> 21 #include <linux/interrupt.h> 22 #include <linux/iopoll.h> 23 #include <linux/irq.h> 24 #include <linux/irqdomain.h> 25 #include <linux/kernel.h> 26 #include <linux/init.h> 27 #include <linux/module.h> 28 #include <linux/msi.h> 29 #include <linux/of_address.h> 30 #include <linux/of_pci.h> 31 #include <linux/of_platform.h> 32 #include <linux/pci.h> 33 #include <linux/phy/phy.h> 34 #include <linux/pinctrl/consumer.h> 35 #include <linux/platform_device.h> 36 #include <linux/reset.h> 37 #include <linux/sizes.h> 38 #include <linux/slab.h> 39 #include <linux/vmalloc.h> 40 #include <linux/regulator/consumer.h> 41 42 #include <soc/tegra/cpuidle.h> 43 #include <soc/tegra/pmc.h> 44 45 #include "../pci.h" 46 47 #define INT_PCI_MSI_NR (8 * 32) 48 49 /* register definitions */ 50 51 #define AFI_AXI_BAR0_SZ 0x00 52 #define AFI_AXI_BAR1_SZ 0x04 53 #define AFI_AXI_BAR2_SZ 0x08 54 #define AFI_AXI_BAR3_SZ 0x0c 55 #define AFI_AXI_BAR4_SZ 0x10 56 #define AFI_AXI_BAR5_SZ 0x14 57 58 #define AFI_AXI_BAR0_START 0x18 59 #define AFI_AXI_BAR1_START 0x1c 60 #define AFI_AXI_BAR2_START 0x20 61 #define AFI_AXI_BAR3_START 0x24 62 #define AFI_AXI_BAR4_START 0x28 63 #define AFI_AXI_BAR5_START 0x2c 64 65 #define AFI_FPCI_BAR0 0x30 66 #define AFI_FPCI_BAR1 0x34 67 #define AFI_FPCI_BAR2 0x38 68 #define AFI_FPCI_BAR3 0x3c 69 #define AFI_FPCI_BAR4 0x40 70 #define AFI_FPCI_BAR5 0x44 71 72 #define AFI_CACHE_BAR0_SZ 0x48 73 #define AFI_CACHE_BAR0_ST 0x4c 74 #define AFI_CACHE_BAR1_SZ 0x50 75 #define AFI_CACHE_BAR1_ST 0x54 76 77 #define AFI_MSI_BAR_SZ 0x60 78 #define AFI_MSI_FPCI_BAR_ST 0x64 79 #define AFI_MSI_AXI_BAR_ST 0x68 80 81 #define AFI_MSI_VEC0 0x6c 82 #define AFI_MSI_VEC1 0x70 83 #define AFI_MSI_VEC2 0x74 84 #define AFI_MSI_VEC3 0x78 85 #define AFI_MSI_VEC4 0x7c 86 #define AFI_MSI_VEC5 0x80 87 #define AFI_MSI_VEC6 0x84 88 #define AFI_MSI_VEC7 0x88 89 90 #define AFI_MSI_EN_VEC0 0x8c 91 #define AFI_MSI_EN_VEC1 0x90 92 #define AFI_MSI_EN_VEC2 0x94 93 #define AFI_MSI_EN_VEC3 0x98 94 #define AFI_MSI_EN_VEC4 0x9c 95 #define AFI_MSI_EN_VEC5 0xa0 96 #define AFI_MSI_EN_VEC6 0xa4 97 #define AFI_MSI_EN_VEC7 0xa8 98 99 #define AFI_CONFIGURATION 0xac 100 #define AFI_CONFIGURATION_EN_FPCI (1 << 0) 101 #define AFI_CONFIGURATION_CLKEN_OVERRIDE (1 << 31) 102 103 #define AFI_FPCI_ERROR_MASKS 0xb0 104 105 #define AFI_INTR_MASK 0xb4 106 #define AFI_INTR_MASK_INT_MASK (1 << 0) 107 #define AFI_INTR_MASK_MSI_MASK (1 << 8) 108 109 #define AFI_INTR_CODE 0xb8 110 #define AFI_INTR_CODE_MASK 0xf 111 #define AFI_INTR_INI_SLAVE_ERROR 1 112 #define AFI_INTR_INI_DECODE_ERROR 2 113 #define AFI_INTR_TARGET_ABORT 3 114 #define AFI_INTR_MASTER_ABORT 4 115 #define AFI_INTR_INVALID_WRITE 5 116 #define AFI_INTR_LEGACY 6 117 #define AFI_INTR_FPCI_DECODE_ERROR 7 118 #define AFI_INTR_AXI_DECODE_ERROR 8 119 #define AFI_INTR_FPCI_TIMEOUT 9 120 #define AFI_INTR_PE_PRSNT_SENSE 10 121 #define AFI_INTR_PE_CLKREQ_SENSE 11 122 #define AFI_INTR_CLKCLAMP_SENSE 12 123 #define AFI_INTR_RDY4PD_SENSE 13 124 #define AFI_INTR_P2P_ERROR 14 125 126 #define AFI_INTR_SIGNATURE 0xbc 127 #define AFI_UPPER_FPCI_ADDRESS 0xc0 128 #define AFI_SM_INTR_ENABLE 0xc4 129 #define AFI_SM_INTR_INTA_ASSERT (1 << 0) 130 #define AFI_SM_INTR_INTB_ASSERT (1 << 1) 131 #define AFI_SM_INTR_INTC_ASSERT (1 << 2) 132 #define AFI_SM_INTR_INTD_ASSERT (1 << 3) 133 #define AFI_SM_INTR_INTA_DEASSERT (1 << 4) 134 #define AFI_SM_INTR_INTB_DEASSERT (1 << 5) 135 #define AFI_SM_INTR_INTC_DEASSERT (1 << 6) 136 #define AFI_SM_INTR_INTD_DEASSERT (1 << 7) 137 138 #define AFI_AFI_INTR_ENABLE 0xc8 139 #define AFI_INTR_EN_INI_SLVERR (1 << 0) 140 #define AFI_INTR_EN_INI_DECERR (1 << 1) 141 #define AFI_INTR_EN_TGT_SLVERR (1 << 2) 142 #define AFI_INTR_EN_TGT_DECERR (1 << 3) 143 #define AFI_INTR_EN_TGT_WRERR (1 << 4) 144 #define AFI_INTR_EN_DFPCI_DECERR (1 << 5) 145 #define AFI_INTR_EN_AXI_DECERR (1 << 6) 146 #define AFI_INTR_EN_FPCI_TIMEOUT (1 << 7) 147 #define AFI_INTR_EN_PRSNT_SENSE (1 << 8) 148 149 #define AFI_PCIE_PME 0xf0 150 151 #define AFI_PCIE_CONFIG 0x0f8 152 #define AFI_PCIE_CONFIG_PCIE_DISABLE(x) (1 << ((x) + 1)) 153 #define AFI_PCIE_CONFIG_PCIE_DISABLE_ALL 0xe 154 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK (0xf << 20) 155 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE (0x0 << 20) 156 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420 (0x0 << 20) 157 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1 (0x0 << 20) 158 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_401 (0x0 << 20) 159 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL (0x1 << 20) 160 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222 (0x1 << 20) 161 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1 (0x1 << 20) 162 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211 (0x1 << 20) 163 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411 (0x2 << 20) 164 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_111 (0x2 << 20) 165 #define AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(x) (1 << ((x) + 29)) 166 #define AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO_ALL (0x7 << 29) 167 168 #define AFI_FUSE 0x104 169 #define AFI_FUSE_PCIE_T0_GEN2_DIS (1 << 2) 170 171 #define AFI_PEX0_CTRL 0x110 172 #define AFI_PEX1_CTRL 0x118 173 #define AFI_PEX_CTRL_RST (1 << 0) 174 #define AFI_PEX_CTRL_CLKREQ_EN (1 << 1) 175 #define AFI_PEX_CTRL_REFCLK_EN (1 << 3) 176 #define AFI_PEX_CTRL_OVERRIDE_EN (1 << 4) 177 178 #define AFI_PLLE_CONTROL 0x160 179 #define AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL (1 << 9) 180 #define AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN (1 << 1) 181 182 #define AFI_PEXBIAS_CTRL_0 0x168 183 184 #define RP_PRIV_XP_DL 0x00000494 185 #define RP_PRIV_XP_DL_GEN2_UPD_FC_TSHOLD (0x1ff << 1) 186 187 #define RP_RX_HDR_LIMIT 0x00000e00 188 #define RP_RX_HDR_LIMIT_PW_MASK (0xff << 8) 189 #define RP_RX_HDR_LIMIT_PW (0x0e << 8) 190 191 #define RP_ECTL_2_R1 0x00000e84 192 #define RP_ECTL_2_R1_RX_CTLE_1C_MASK 0xffff 193 194 #define RP_ECTL_4_R1 0x00000e8c 195 #define RP_ECTL_4_R1_RX_CDR_CTRL_1C_MASK (0xffff << 16) 196 #define RP_ECTL_4_R1_RX_CDR_CTRL_1C_SHIFT 16 197 198 #define RP_ECTL_5_R1 0x00000e90 199 #define RP_ECTL_5_R1_RX_EQ_CTRL_L_1C_MASK 0xffffffff 200 201 #define RP_ECTL_6_R1 0x00000e94 202 #define RP_ECTL_6_R1_RX_EQ_CTRL_H_1C_MASK 0xffffffff 203 204 #define RP_ECTL_2_R2 0x00000ea4 205 #define RP_ECTL_2_R2_RX_CTLE_1C_MASK 0xffff 206 207 #define RP_ECTL_4_R2 0x00000eac 208 #define RP_ECTL_4_R2_RX_CDR_CTRL_1C_MASK (0xffff << 16) 209 #define RP_ECTL_4_R2_RX_CDR_CTRL_1C_SHIFT 16 210 211 #define RP_ECTL_5_R2 0x00000eb0 212 #define RP_ECTL_5_R2_RX_EQ_CTRL_L_1C_MASK 0xffffffff 213 214 #define RP_ECTL_6_R2 0x00000eb4 215 #define RP_ECTL_6_R2_RX_EQ_CTRL_H_1C_MASK 0xffffffff 216 217 #define RP_VEND_XP 0x00000f00 218 #define RP_VEND_XP_DL_UP (1 << 30) 219 #define RP_VEND_XP_OPPORTUNISTIC_ACK (1 << 27) 220 #define RP_VEND_XP_OPPORTUNISTIC_UPDATEFC (1 << 28) 221 #define RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK (0xff << 18) 222 223 #define RP_VEND_CTL0 0x00000f44 224 #define RP_VEND_CTL0_DSK_RST_PULSE_WIDTH_MASK (0xf << 12) 225 #define RP_VEND_CTL0_DSK_RST_PULSE_WIDTH (0x9 << 12) 226 227 #define RP_VEND_CTL1 0x00000f48 228 #define RP_VEND_CTL1_ERPT (1 << 13) 229 230 #define RP_VEND_XP_BIST 0x00000f4c 231 #define RP_VEND_XP_BIST_GOTO_L1_L2_AFTER_DLLP_DONE (1 << 28) 232 233 #define RP_VEND_CTL2 0x00000fa8 234 #define RP_VEND_CTL2_PCA_ENABLE (1 << 7) 235 236 #define RP_PRIV_MISC 0x00000fe0 237 #define RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT (0xe << 0) 238 #define RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT (0xf << 0) 239 #define RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD_MASK (0x7f << 16) 240 #define RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD (0xf << 16) 241 #define RP_PRIV_MISC_CTLR_CLK_CLAMP_ENABLE (1 << 23) 242 #define RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD_MASK (0x7f << 24) 243 #define RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD (0xf << 24) 244 #define RP_PRIV_MISC_TMS_CLK_CLAMP_ENABLE (1 << 31) 245 246 #define RP_LINK_CONTROL_STATUS 0x00000090 247 #define RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE 0x20000000 248 #define RP_LINK_CONTROL_STATUS_LINKSTAT_MASK 0x3fff0000 249 250 #define RP_LINK_CONTROL_STATUS_2 0x000000b0 251 252 #define PADS_CTL_SEL 0x0000009c 253 254 #define PADS_CTL 0x000000a0 255 #define PADS_CTL_IDDQ_1L (1 << 0) 256 #define PADS_CTL_TX_DATA_EN_1L (1 << 6) 257 #define PADS_CTL_RX_DATA_EN_1L (1 << 10) 258 259 #define PADS_PLL_CTL_TEGRA20 0x000000b8 260 #define PADS_PLL_CTL_TEGRA30 0x000000b4 261 #define PADS_PLL_CTL_RST_B4SM (1 << 1) 262 #define PADS_PLL_CTL_LOCKDET (1 << 8) 263 #define PADS_PLL_CTL_REFCLK_MASK (0x3 << 16) 264 #define PADS_PLL_CTL_REFCLK_INTERNAL_CML (0 << 16) 265 #define PADS_PLL_CTL_REFCLK_INTERNAL_CMOS (1 << 16) 266 #define PADS_PLL_CTL_REFCLK_EXTERNAL (2 << 16) 267 #define PADS_PLL_CTL_TXCLKREF_MASK (0x1 << 20) 268 #define PADS_PLL_CTL_TXCLKREF_DIV10 (0 << 20) 269 #define PADS_PLL_CTL_TXCLKREF_DIV5 (1 << 20) 270 #define PADS_PLL_CTL_TXCLKREF_BUF_EN (1 << 22) 271 272 #define PADS_REFCLK_CFG0 0x000000c8 273 #define PADS_REFCLK_CFG1 0x000000cc 274 #define PADS_REFCLK_BIAS 0x000000d0 275 276 /* 277 * Fields in PADS_REFCLK_CFG*. Those registers form an array of 16-bit 278 * entries, one entry per PCIe port. These field definitions and desired 279 * values aren't in the TRM, but do come from NVIDIA. 280 */ 281 #define PADS_REFCLK_CFG_TERM_SHIFT 2 /* 6:2 */ 282 #define PADS_REFCLK_CFG_E_TERM_SHIFT 7 283 #define PADS_REFCLK_CFG_PREDI_SHIFT 8 /* 11:8 */ 284 #define PADS_REFCLK_CFG_DRVI_SHIFT 12 /* 15:12 */ 285 286 #define PME_ACK_TIMEOUT 10000 287 #define LINK_RETRAIN_TIMEOUT 100000 /* in usec */ 288 289 struct tegra_msi { 290 struct msi_controller chip; 291 DECLARE_BITMAP(used, INT_PCI_MSI_NR); 292 struct irq_domain *domain; 293 struct mutex lock; 294 void *virt; 295 dma_addr_t phys; 296 int irq; 297 }; 298 299 /* used to differentiate between Tegra SoC generations */ 300 struct tegra_pcie_port_soc { 301 struct { 302 u8 turnoff_bit; 303 u8 ack_bit; 304 } pme; 305 }; 306 307 struct tegra_pcie_soc { 308 unsigned int num_ports; 309 const struct tegra_pcie_port_soc *ports; 310 unsigned int msi_base_shift; 311 unsigned long afi_pex2_ctrl; 312 u32 pads_pll_ctl; 313 u32 tx_ref_sel; 314 u32 pads_refclk_cfg0; 315 u32 pads_refclk_cfg1; 316 u32 update_fc_threshold; 317 bool has_pex_clkreq_en; 318 bool has_pex_bias_ctrl; 319 bool has_intr_prsnt_sense; 320 bool has_cml_clk; 321 bool has_gen2; 322 bool force_pca_enable; 323 bool program_uphy; 324 bool update_clamp_threshold; 325 bool program_deskew_time; 326 bool raw_violation_fixup; 327 bool update_fc_timer; 328 bool has_cache_bars; 329 struct { 330 struct { 331 u32 rp_ectl_2_r1; 332 u32 rp_ectl_4_r1; 333 u32 rp_ectl_5_r1; 334 u32 rp_ectl_6_r1; 335 u32 rp_ectl_2_r2; 336 u32 rp_ectl_4_r2; 337 u32 rp_ectl_5_r2; 338 u32 rp_ectl_6_r2; 339 } regs; 340 bool enable; 341 } ectl; 342 }; 343 344 static inline struct tegra_msi *to_tegra_msi(struct msi_controller *chip) 345 { 346 return container_of(chip, struct tegra_msi, chip); 347 } 348 349 struct tegra_pcie { 350 struct device *dev; 351 352 void __iomem *pads; 353 void __iomem *afi; 354 void __iomem *cfg; 355 int irq; 356 357 struct resource cs; 358 struct resource io; 359 struct resource pio; 360 struct resource mem; 361 struct resource prefetch; 362 struct resource busn; 363 364 struct { 365 resource_size_t mem; 366 resource_size_t io; 367 } offset; 368 369 struct clk *pex_clk; 370 struct clk *afi_clk; 371 struct clk *pll_e; 372 struct clk *cml_clk; 373 374 struct reset_control *pex_rst; 375 struct reset_control *afi_rst; 376 struct reset_control *pcie_xrst; 377 378 bool legacy_phy; 379 struct phy *phy; 380 381 struct tegra_msi msi; 382 383 struct list_head ports; 384 u32 xbar_config; 385 386 struct regulator_bulk_data *supplies; 387 unsigned int num_supplies; 388 389 const struct tegra_pcie_soc *soc; 390 struct dentry *debugfs; 391 }; 392 393 struct tegra_pcie_port { 394 struct tegra_pcie *pcie; 395 struct device_node *np; 396 struct list_head list; 397 struct resource regs; 398 void __iomem *base; 399 unsigned int index; 400 unsigned int lanes; 401 402 struct phy **phys; 403 404 struct gpio_desc *reset_gpio; 405 }; 406 407 struct tegra_pcie_bus { 408 struct list_head list; 409 unsigned int nr; 410 }; 411 412 static inline void afi_writel(struct tegra_pcie *pcie, u32 value, 413 unsigned long offset) 414 { 415 writel(value, pcie->afi + offset); 416 } 417 418 static inline u32 afi_readl(struct tegra_pcie *pcie, unsigned long offset) 419 { 420 return readl(pcie->afi + offset); 421 } 422 423 static inline void pads_writel(struct tegra_pcie *pcie, u32 value, 424 unsigned long offset) 425 { 426 writel(value, pcie->pads + offset); 427 } 428 429 static inline u32 pads_readl(struct tegra_pcie *pcie, unsigned long offset) 430 { 431 return readl(pcie->pads + offset); 432 } 433 434 /* 435 * The configuration space mapping on Tegra is somewhat similar to the ECAM 436 * defined by PCIe. However it deviates a bit in how the 4 bits for extended 437 * register accesses are mapped: 438 * 439 * [27:24] extended register number 440 * [23:16] bus number 441 * [15:11] device number 442 * [10: 8] function number 443 * [ 7: 0] register number 444 * 445 * Mapping the whole extended configuration space would require 256 MiB of 446 * virtual address space, only a small part of which will actually be used. 447 * 448 * To work around this, a 4 KiB region is used to generate the required 449 * configuration transaction with relevant B:D:F and register offset values. 450 * This is achieved by dynamically programming base address and size of 451 * AFI_AXI_BAR used for end point config space mapping to make sure that the 452 * address (access to which generates correct config transaction) falls in 453 * this 4 KiB region. 454 */ 455 static unsigned int tegra_pcie_conf_offset(u8 bus, unsigned int devfn, 456 unsigned int where) 457 { 458 return ((where & 0xf00) << 16) | (bus << 16) | (PCI_SLOT(devfn) << 11) | 459 (PCI_FUNC(devfn) << 8) | (where & 0xff); 460 } 461 462 static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus, 463 unsigned int devfn, 464 int where) 465 { 466 struct tegra_pcie *pcie = bus->sysdata; 467 void __iomem *addr = NULL; 468 469 if (bus->number == 0) { 470 unsigned int slot = PCI_SLOT(devfn); 471 struct tegra_pcie_port *port; 472 473 list_for_each_entry(port, &pcie->ports, list) { 474 if (port->index + 1 == slot) { 475 addr = port->base + (where & ~3); 476 break; 477 } 478 } 479 } else { 480 unsigned int offset; 481 u32 base; 482 483 offset = tegra_pcie_conf_offset(bus->number, devfn, where); 484 485 /* move 4 KiB window to offset within the FPCI region */ 486 base = 0xfe100000 + ((offset & ~(SZ_4K - 1)) >> 8); 487 afi_writel(pcie, base, AFI_FPCI_BAR0); 488 489 /* move to correct offset within the 4 KiB page */ 490 addr = pcie->cfg + (offset & (SZ_4K - 1)); 491 } 492 493 return addr; 494 } 495 496 static int tegra_pcie_config_read(struct pci_bus *bus, unsigned int devfn, 497 int where, int size, u32 *value) 498 { 499 if (bus->number == 0) 500 return pci_generic_config_read32(bus, devfn, where, size, 501 value); 502 503 return pci_generic_config_read(bus, devfn, where, size, value); 504 } 505 506 static int tegra_pcie_config_write(struct pci_bus *bus, unsigned int devfn, 507 int where, int size, u32 value) 508 { 509 if (bus->number == 0) 510 return pci_generic_config_write32(bus, devfn, where, size, 511 value); 512 513 return pci_generic_config_write(bus, devfn, where, size, value); 514 } 515 516 static struct pci_ops tegra_pcie_ops = { 517 .map_bus = tegra_pcie_map_bus, 518 .read = tegra_pcie_config_read, 519 .write = tegra_pcie_config_write, 520 }; 521 522 static unsigned long tegra_pcie_port_get_pex_ctrl(struct tegra_pcie_port *port) 523 { 524 const struct tegra_pcie_soc *soc = port->pcie->soc; 525 unsigned long ret = 0; 526 527 switch (port->index) { 528 case 0: 529 ret = AFI_PEX0_CTRL; 530 break; 531 532 case 1: 533 ret = AFI_PEX1_CTRL; 534 break; 535 536 case 2: 537 ret = soc->afi_pex2_ctrl; 538 break; 539 } 540 541 return ret; 542 } 543 544 static void tegra_pcie_port_reset(struct tegra_pcie_port *port) 545 { 546 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port); 547 unsigned long value; 548 549 /* pulse reset signal */ 550 if (port->reset_gpio) { 551 gpiod_set_value(port->reset_gpio, 1); 552 } else { 553 value = afi_readl(port->pcie, ctrl); 554 value &= ~AFI_PEX_CTRL_RST; 555 afi_writel(port->pcie, value, ctrl); 556 } 557 558 usleep_range(1000, 2000); 559 560 if (port->reset_gpio) { 561 gpiod_set_value(port->reset_gpio, 0); 562 } else { 563 value = afi_readl(port->pcie, ctrl); 564 value |= AFI_PEX_CTRL_RST; 565 afi_writel(port->pcie, value, ctrl); 566 } 567 } 568 569 static void tegra_pcie_enable_rp_features(struct tegra_pcie_port *port) 570 { 571 const struct tegra_pcie_soc *soc = port->pcie->soc; 572 u32 value; 573 574 /* Enable AER capability */ 575 value = readl(port->base + RP_VEND_CTL1); 576 value |= RP_VEND_CTL1_ERPT; 577 writel(value, port->base + RP_VEND_CTL1); 578 579 /* Optimal settings to enhance bandwidth */ 580 value = readl(port->base + RP_VEND_XP); 581 value |= RP_VEND_XP_OPPORTUNISTIC_ACK; 582 value |= RP_VEND_XP_OPPORTUNISTIC_UPDATEFC; 583 writel(value, port->base + RP_VEND_XP); 584 585 /* 586 * LTSSM will wait for DLLP to finish before entering L1 or L2, 587 * to avoid truncation of PM messages which results in receiver errors 588 */ 589 value = readl(port->base + RP_VEND_XP_BIST); 590 value |= RP_VEND_XP_BIST_GOTO_L1_L2_AFTER_DLLP_DONE; 591 writel(value, port->base + RP_VEND_XP_BIST); 592 593 value = readl(port->base + RP_PRIV_MISC); 594 value |= RP_PRIV_MISC_CTLR_CLK_CLAMP_ENABLE; 595 value |= RP_PRIV_MISC_TMS_CLK_CLAMP_ENABLE; 596 597 if (soc->update_clamp_threshold) { 598 value &= ~(RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD_MASK | 599 RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD_MASK); 600 value |= RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD | 601 RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD; 602 } 603 604 writel(value, port->base + RP_PRIV_MISC); 605 } 606 607 static void tegra_pcie_program_ectl_settings(struct tegra_pcie_port *port) 608 { 609 const struct tegra_pcie_soc *soc = port->pcie->soc; 610 u32 value; 611 612 value = readl(port->base + RP_ECTL_2_R1); 613 value &= ~RP_ECTL_2_R1_RX_CTLE_1C_MASK; 614 value |= soc->ectl.regs.rp_ectl_2_r1; 615 writel(value, port->base + RP_ECTL_2_R1); 616 617 value = readl(port->base + RP_ECTL_4_R1); 618 value &= ~RP_ECTL_4_R1_RX_CDR_CTRL_1C_MASK; 619 value |= soc->ectl.regs.rp_ectl_4_r1 << 620 RP_ECTL_4_R1_RX_CDR_CTRL_1C_SHIFT; 621 writel(value, port->base + RP_ECTL_4_R1); 622 623 value = readl(port->base + RP_ECTL_5_R1); 624 value &= ~RP_ECTL_5_R1_RX_EQ_CTRL_L_1C_MASK; 625 value |= soc->ectl.regs.rp_ectl_5_r1; 626 writel(value, port->base + RP_ECTL_5_R1); 627 628 value = readl(port->base + RP_ECTL_6_R1); 629 value &= ~RP_ECTL_6_R1_RX_EQ_CTRL_H_1C_MASK; 630 value |= soc->ectl.regs.rp_ectl_6_r1; 631 writel(value, port->base + RP_ECTL_6_R1); 632 633 value = readl(port->base + RP_ECTL_2_R2); 634 value &= ~RP_ECTL_2_R2_RX_CTLE_1C_MASK; 635 value |= soc->ectl.regs.rp_ectl_2_r2; 636 writel(value, port->base + RP_ECTL_2_R2); 637 638 value = readl(port->base + RP_ECTL_4_R2); 639 value &= ~RP_ECTL_4_R2_RX_CDR_CTRL_1C_MASK; 640 value |= soc->ectl.regs.rp_ectl_4_r2 << 641 RP_ECTL_4_R2_RX_CDR_CTRL_1C_SHIFT; 642 writel(value, port->base + RP_ECTL_4_R2); 643 644 value = readl(port->base + RP_ECTL_5_R2); 645 value &= ~RP_ECTL_5_R2_RX_EQ_CTRL_L_1C_MASK; 646 value |= soc->ectl.regs.rp_ectl_5_r2; 647 writel(value, port->base + RP_ECTL_5_R2); 648 649 value = readl(port->base + RP_ECTL_6_R2); 650 value &= ~RP_ECTL_6_R2_RX_EQ_CTRL_H_1C_MASK; 651 value |= soc->ectl.regs.rp_ectl_6_r2; 652 writel(value, port->base + RP_ECTL_6_R2); 653 } 654 655 static void tegra_pcie_apply_sw_fixup(struct tegra_pcie_port *port) 656 { 657 const struct tegra_pcie_soc *soc = port->pcie->soc; 658 u32 value; 659 660 /* 661 * Sometimes link speed change from Gen2 to Gen1 fails due to 662 * instability in deskew logic on lane-0. Increase the deskew 663 * retry time to resolve this issue. 664 */ 665 if (soc->program_deskew_time) { 666 value = readl(port->base + RP_VEND_CTL0); 667 value &= ~RP_VEND_CTL0_DSK_RST_PULSE_WIDTH_MASK; 668 value |= RP_VEND_CTL0_DSK_RST_PULSE_WIDTH; 669 writel(value, port->base + RP_VEND_CTL0); 670 } 671 672 /* Fixup for read after write violation. */ 673 if (soc->raw_violation_fixup) { 674 value = readl(port->base + RP_RX_HDR_LIMIT); 675 value &= ~RP_RX_HDR_LIMIT_PW_MASK; 676 value |= RP_RX_HDR_LIMIT_PW; 677 writel(value, port->base + RP_RX_HDR_LIMIT); 678 679 value = readl(port->base + RP_PRIV_XP_DL); 680 value |= RP_PRIV_XP_DL_GEN2_UPD_FC_TSHOLD; 681 writel(value, port->base + RP_PRIV_XP_DL); 682 683 value = readl(port->base + RP_VEND_XP); 684 value &= ~RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK; 685 value |= soc->update_fc_threshold; 686 writel(value, port->base + RP_VEND_XP); 687 } 688 689 if (soc->update_fc_timer) { 690 value = readl(port->base + RP_VEND_XP); 691 value &= ~RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK; 692 value |= soc->update_fc_threshold; 693 writel(value, port->base + RP_VEND_XP); 694 } 695 696 /* 697 * PCIe link doesn't come up with few legacy PCIe endpoints if 698 * root port advertises both Gen-1 and Gen-2 speeds in Tegra. 699 * Hence, the strategy followed here is to initially advertise 700 * only Gen-1 and after link is up, retrain link to Gen-2 speed 701 */ 702 value = readl(port->base + RP_LINK_CONTROL_STATUS_2); 703 value &= ~PCI_EXP_LNKSTA_CLS; 704 value |= PCI_EXP_LNKSTA_CLS_2_5GB; 705 writel(value, port->base + RP_LINK_CONTROL_STATUS_2); 706 } 707 708 static void tegra_pcie_port_enable(struct tegra_pcie_port *port) 709 { 710 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port); 711 const struct tegra_pcie_soc *soc = port->pcie->soc; 712 unsigned long value; 713 714 /* enable reference clock */ 715 value = afi_readl(port->pcie, ctrl); 716 value |= AFI_PEX_CTRL_REFCLK_EN; 717 718 if (soc->has_pex_clkreq_en) 719 value |= AFI_PEX_CTRL_CLKREQ_EN; 720 721 value |= AFI_PEX_CTRL_OVERRIDE_EN; 722 723 afi_writel(port->pcie, value, ctrl); 724 725 tegra_pcie_port_reset(port); 726 727 if (soc->force_pca_enable) { 728 value = readl(port->base + RP_VEND_CTL2); 729 value |= RP_VEND_CTL2_PCA_ENABLE; 730 writel(value, port->base + RP_VEND_CTL2); 731 } 732 733 tegra_pcie_enable_rp_features(port); 734 735 if (soc->ectl.enable) 736 tegra_pcie_program_ectl_settings(port); 737 738 tegra_pcie_apply_sw_fixup(port); 739 } 740 741 static void tegra_pcie_port_disable(struct tegra_pcie_port *port) 742 { 743 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port); 744 const struct tegra_pcie_soc *soc = port->pcie->soc; 745 unsigned long value; 746 747 /* assert port reset */ 748 value = afi_readl(port->pcie, ctrl); 749 value &= ~AFI_PEX_CTRL_RST; 750 afi_writel(port->pcie, value, ctrl); 751 752 /* disable reference clock */ 753 value = afi_readl(port->pcie, ctrl); 754 755 if (soc->has_pex_clkreq_en) 756 value &= ~AFI_PEX_CTRL_CLKREQ_EN; 757 758 value &= ~AFI_PEX_CTRL_REFCLK_EN; 759 afi_writel(port->pcie, value, ctrl); 760 761 /* disable PCIe port and set CLKREQ# as GPIO to allow PLLE power down */ 762 value = afi_readl(port->pcie, AFI_PCIE_CONFIG); 763 value |= AFI_PCIE_CONFIG_PCIE_DISABLE(port->index); 764 value |= AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(port->index); 765 afi_writel(port->pcie, value, AFI_PCIE_CONFIG); 766 } 767 768 static void tegra_pcie_port_free(struct tegra_pcie_port *port) 769 { 770 struct tegra_pcie *pcie = port->pcie; 771 struct device *dev = pcie->dev; 772 773 devm_iounmap(dev, port->base); 774 devm_release_mem_region(dev, port->regs.start, 775 resource_size(&port->regs)); 776 list_del(&port->list); 777 devm_kfree(dev, port); 778 } 779 780 /* Tegra PCIE root complex wrongly reports device class */ 781 static void tegra_pcie_fixup_class(struct pci_dev *dev) 782 { 783 dev->class = PCI_CLASS_BRIDGE_PCI << 8; 784 } 785 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_fixup_class); 786 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_fixup_class); 787 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_fixup_class); 788 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_fixup_class); 789 790 /* Tegra20 and Tegra30 PCIE requires relaxed ordering */ 791 static void tegra_pcie_relax_enable(struct pci_dev *dev) 792 { 793 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN); 794 } 795 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_relax_enable); 796 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_relax_enable); 797 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_relax_enable); 798 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_relax_enable); 799 800 static int tegra_pcie_request_resources(struct tegra_pcie *pcie) 801 { 802 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); 803 struct list_head *windows = &host->windows; 804 struct device *dev = pcie->dev; 805 int err; 806 807 pci_add_resource_offset(windows, &pcie->pio, pcie->offset.io); 808 pci_add_resource_offset(windows, &pcie->mem, pcie->offset.mem); 809 pci_add_resource_offset(windows, &pcie->prefetch, pcie->offset.mem); 810 pci_add_resource(windows, &pcie->busn); 811 812 err = devm_request_pci_bus_resources(dev, windows); 813 if (err < 0) { 814 pci_free_resource_list(windows); 815 return err; 816 } 817 818 pci_remap_iospace(&pcie->pio, pcie->io.start); 819 820 return 0; 821 } 822 823 static void tegra_pcie_free_resources(struct tegra_pcie *pcie) 824 { 825 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); 826 struct list_head *windows = &host->windows; 827 828 pci_unmap_iospace(&pcie->pio); 829 pci_free_resource_list(windows); 830 } 831 832 static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin) 833 { 834 struct tegra_pcie *pcie = pdev->bus->sysdata; 835 int irq; 836 837 tegra_cpuidle_pcie_irqs_in_use(); 838 839 irq = of_irq_parse_and_map_pci(pdev, slot, pin); 840 if (!irq) 841 irq = pcie->irq; 842 843 return irq; 844 } 845 846 static irqreturn_t tegra_pcie_isr(int irq, void *arg) 847 { 848 const char *err_msg[] = { 849 "Unknown", 850 "AXI slave error", 851 "AXI decode error", 852 "Target abort", 853 "Master abort", 854 "Invalid write", 855 "Legacy interrupt", 856 "Response decoding error", 857 "AXI response decoding error", 858 "Transaction timeout", 859 "Slot present pin change", 860 "Slot clock request change", 861 "TMS clock ramp change", 862 "TMS ready for power down", 863 "Peer2Peer error", 864 }; 865 struct tegra_pcie *pcie = arg; 866 struct device *dev = pcie->dev; 867 u32 code, signature; 868 869 code = afi_readl(pcie, AFI_INTR_CODE) & AFI_INTR_CODE_MASK; 870 signature = afi_readl(pcie, AFI_INTR_SIGNATURE); 871 afi_writel(pcie, 0, AFI_INTR_CODE); 872 873 if (code == AFI_INTR_LEGACY) 874 return IRQ_NONE; 875 876 if (code >= ARRAY_SIZE(err_msg)) 877 code = 0; 878 879 /* 880 * do not pollute kernel log with master abort reports since they 881 * happen a lot during enumeration 882 */ 883 if (code == AFI_INTR_MASTER_ABORT || code == AFI_INTR_PE_PRSNT_SENSE) 884 dev_dbg(dev, "%s, signature: %08x\n", err_msg[code], signature); 885 else 886 dev_err(dev, "%s, signature: %08x\n", err_msg[code], signature); 887 888 if (code == AFI_INTR_TARGET_ABORT || code == AFI_INTR_MASTER_ABORT || 889 code == AFI_INTR_FPCI_DECODE_ERROR) { 890 u32 fpci = afi_readl(pcie, AFI_UPPER_FPCI_ADDRESS) & 0xff; 891 u64 address = (u64)fpci << 32 | (signature & 0xfffffffc); 892 893 if (code == AFI_INTR_MASTER_ABORT) 894 dev_dbg(dev, " FPCI address: %10llx\n", address); 895 else 896 dev_err(dev, " FPCI address: %10llx\n", address); 897 } 898 899 return IRQ_HANDLED; 900 } 901 902 /* 903 * FPCI map is as follows: 904 * - 0xfdfc000000: I/O space 905 * - 0xfdfe000000: type 0 configuration space 906 * - 0xfdff000000: type 1 configuration space 907 * - 0xfe00000000: type 0 extended configuration space 908 * - 0xfe10000000: type 1 extended configuration space 909 */ 910 static void tegra_pcie_setup_translations(struct tegra_pcie *pcie) 911 { 912 u32 fpci_bar, size, axi_address; 913 914 /* Bar 0: type 1 extended configuration space */ 915 size = resource_size(&pcie->cs); 916 afi_writel(pcie, pcie->cs.start, AFI_AXI_BAR0_START); 917 afi_writel(pcie, size >> 12, AFI_AXI_BAR0_SZ); 918 919 /* Bar 1: downstream IO bar */ 920 fpci_bar = 0xfdfc0000; 921 size = resource_size(&pcie->io); 922 axi_address = pcie->io.start; 923 afi_writel(pcie, axi_address, AFI_AXI_BAR1_START); 924 afi_writel(pcie, size >> 12, AFI_AXI_BAR1_SZ); 925 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR1); 926 927 /* Bar 2: prefetchable memory BAR */ 928 fpci_bar = (((pcie->prefetch.start >> 12) & 0x0fffffff) << 4) | 0x1; 929 size = resource_size(&pcie->prefetch); 930 axi_address = pcie->prefetch.start; 931 afi_writel(pcie, axi_address, AFI_AXI_BAR2_START); 932 afi_writel(pcie, size >> 12, AFI_AXI_BAR2_SZ); 933 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR2); 934 935 /* Bar 3: non prefetchable memory BAR */ 936 fpci_bar = (((pcie->mem.start >> 12) & 0x0fffffff) << 4) | 0x1; 937 size = resource_size(&pcie->mem); 938 axi_address = pcie->mem.start; 939 afi_writel(pcie, axi_address, AFI_AXI_BAR3_START); 940 afi_writel(pcie, size >> 12, AFI_AXI_BAR3_SZ); 941 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR3); 942 943 /* NULL out the remaining BARs as they are not used */ 944 afi_writel(pcie, 0, AFI_AXI_BAR4_START); 945 afi_writel(pcie, 0, AFI_AXI_BAR4_SZ); 946 afi_writel(pcie, 0, AFI_FPCI_BAR4); 947 948 afi_writel(pcie, 0, AFI_AXI_BAR5_START); 949 afi_writel(pcie, 0, AFI_AXI_BAR5_SZ); 950 afi_writel(pcie, 0, AFI_FPCI_BAR5); 951 952 if (pcie->soc->has_cache_bars) { 953 /* map all upstream transactions as uncached */ 954 afi_writel(pcie, 0, AFI_CACHE_BAR0_ST); 955 afi_writel(pcie, 0, AFI_CACHE_BAR0_SZ); 956 afi_writel(pcie, 0, AFI_CACHE_BAR1_ST); 957 afi_writel(pcie, 0, AFI_CACHE_BAR1_SZ); 958 } 959 960 /* MSI translations are setup only when needed */ 961 afi_writel(pcie, 0, AFI_MSI_FPCI_BAR_ST); 962 afi_writel(pcie, 0, AFI_MSI_BAR_SZ); 963 afi_writel(pcie, 0, AFI_MSI_AXI_BAR_ST); 964 afi_writel(pcie, 0, AFI_MSI_BAR_SZ); 965 } 966 967 static int tegra_pcie_pll_wait(struct tegra_pcie *pcie, unsigned long timeout) 968 { 969 const struct tegra_pcie_soc *soc = pcie->soc; 970 u32 value; 971 972 timeout = jiffies + msecs_to_jiffies(timeout); 973 974 while (time_before(jiffies, timeout)) { 975 value = pads_readl(pcie, soc->pads_pll_ctl); 976 if (value & PADS_PLL_CTL_LOCKDET) 977 return 0; 978 } 979 980 return -ETIMEDOUT; 981 } 982 983 static int tegra_pcie_phy_enable(struct tegra_pcie *pcie) 984 { 985 struct device *dev = pcie->dev; 986 const struct tegra_pcie_soc *soc = pcie->soc; 987 u32 value; 988 int err; 989 990 /* initialize internal PHY, enable up to 16 PCIE lanes */ 991 pads_writel(pcie, 0x0, PADS_CTL_SEL); 992 993 /* override IDDQ to 1 on all 4 lanes */ 994 value = pads_readl(pcie, PADS_CTL); 995 value |= PADS_CTL_IDDQ_1L; 996 pads_writel(pcie, value, PADS_CTL); 997 998 /* 999 * Set up PHY PLL inputs select PLLE output as refclock, 1000 * set TX ref sel to div10 (not div5). 1001 */ 1002 value = pads_readl(pcie, soc->pads_pll_ctl); 1003 value &= ~(PADS_PLL_CTL_REFCLK_MASK | PADS_PLL_CTL_TXCLKREF_MASK); 1004 value |= PADS_PLL_CTL_REFCLK_INTERNAL_CML | soc->tx_ref_sel; 1005 pads_writel(pcie, value, soc->pads_pll_ctl); 1006 1007 /* reset PLL */ 1008 value = pads_readl(pcie, soc->pads_pll_ctl); 1009 value &= ~PADS_PLL_CTL_RST_B4SM; 1010 pads_writel(pcie, value, soc->pads_pll_ctl); 1011 1012 usleep_range(20, 100); 1013 1014 /* take PLL out of reset */ 1015 value = pads_readl(pcie, soc->pads_pll_ctl); 1016 value |= PADS_PLL_CTL_RST_B4SM; 1017 pads_writel(pcie, value, soc->pads_pll_ctl); 1018 1019 /* wait for the PLL to lock */ 1020 err = tegra_pcie_pll_wait(pcie, 500); 1021 if (err < 0) { 1022 dev_err(dev, "PLL failed to lock: %d\n", err); 1023 return err; 1024 } 1025 1026 /* turn off IDDQ override */ 1027 value = pads_readl(pcie, PADS_CTL); 1028 value &= ~PADS_CTL_IDDQ_1L; 1029 pads_writel(pcie, value, PADS_CTL); 1030 1031 /* enable TX/RX data */ 1032 value = pads_readl(pcie, PADS_CTL); 1033 value |= PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L; 1034 pads_writel(pcie, value, PADS_CTL); 1035 1036 return 0; 1037 } 1038 1039 static int tegra_pcie_phy_disable(struct tegra_pcie *pcie) 1040 { 1041 const struct tegra_pcie_soc *soc = pcie->soc; 1042 u32 value; 1043 1044 /* disable TX/RX data */ 1045 value = pads_readl(pcie, PADS_CTL); 1046 value &= ~(PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L); 1047 pads_writel(pcie, value, PADS_CTL); 1048 1049 /* override IDDQ */ 1050 value = pads_readl(pcie, PADS_CTL); 1051 value |= PADS_CTL_IDDQ_1L; 1052 pads_writel(pcie, value, PADS_CTL); 1053 1054 /* reset PLL */ 1055 value = pads_readl(pcie, soc->pads_pll_ctl); 1056 value &= ~PADS_PLL_CTL_RST_B4SM; 1057 pads_writel(pcie, value, soc->pads_pll_ctl); 1058 1059 usleep_range(20, 100); 1060 1061 return 0; 1062 } 1063 1064 static int tegra_pcie_port_phy_power_on(struct tegra_pcie_port *port) 1065 { 1066 struct device *dev = port->pcie->dev; 1067 unsigned int i; 1068 int err; 1069 1070 for (i = 0; i < port->lanes; i++) { 1071 err = phy_power_on(port->phys[i]); 1072 if (err < 0) { 1073 dev_err(dev, "failed to power on PHY#%u: %d\n", i, err); 1074 return err; 1075 } 1076 } 1077 1078 return 0; 1079 } 1080 1081 static int tegra_pcie_port_phy_power_off(struct tegra_pcie_port *port) 1082 { 1083 struct device *dev = port->pcie->dev; 1084 unsigned int i; 1085 int err; 1086 1087 for (i = 0; i < port->lanes; i++) { 1088 err = phy_power_off(port->phys[i]); 1089 if (err < 0) { 1090 dev_err(dev, "failed to power off PHY#%u: %d\n", i, 1091 err); 1092 return err; 1093 } 1094 } 1095 1096 return 0; 1097 } 1098 1099 static int tegra_pcie_phy_power_on(struct tegra_pcie *pcie) 1100 { 1101 struct device *dev = pcie->dev; 1102 struct tegra_pcie_port *port; 1103 int err; 1104 1105 if (pcie->legacy_phy) { 1106 if (pcie->phy) 1107 err = phy_power_on(pcie->phy); 1108 else 1109 err = tegra_pcie_phy_enable(pcie); 1110 1111 if (err < 0) 1112 dev_err(dev, "failed to power on PHY: %d\n", err); 1113 1114 return err; 1115 } 1116 1117 list_for_each_entry(port, &pcie->ports, list) { 1118 err = tegra_pcie_port_phy_power_on(port); 1119 if (err < 0) { 1120 dev_err(dev, 1121 "failed to power on PCIe port %u PHY: %d\n", 1122 port->index, err); 1123 return err; 1124 } 1125 } 1126 1127 return 0; 1128 } 1129 1130 static int tegra_pcie_phy_power_off(struct tegra_pcie *pcie) 1131 { 1132 struct device *dev = pcie->dev; 1133 struct tegra_pcie_port *port; 1134 int err; 1135 1136 if (pcie->legacy_phy) { 1137 if (pcie->phy) 1138 err = phy_power_off(pcie->phy); 1139 else 1140 err = tegra_pcie_phy_disable(pcie); 1141 1142 if (err < 0) 1143 dev_err(dev, "failed to power off PHY: %d\n", err); 1144 1145 return err; 1146 } 1147 1148 list_for_each_entry(port, &pcie->ports, list) { 1149 err = tegra_pcie_port_phy_power_off(port); 1150 if (err < 0) { 1151 dev_err(dev, 1152 "failed to power off PCIe port %u PHY: %d\n", 1153 port->index, err); 1154 return err; 1155 } 1156 } 1157 1158 return 0; 1159 } 1160 1161 static void tegra_pcie_enable_controller(struct tegra_pcie *pcie) 1162 { 1163 const struct tegra_pcie_soc *soc = pcie->soc; 1164 struct tegra_pcie_port *port; 1165 unsigned long value; 1166 1167 /* enable PLL power down */ 1168 if (pcie->phy) { 1169 value = afi_readl(pcie, AFI_PLLE_CONTROL); 1170 value &= ~AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL; 1171 value |= AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN; 1172 afi_writel(pcie, value, AFI_PLLE_CONTROL); 1173 } 1174 1175 /* power down PCIe slot clock bias pad */ 1176 if (soc->has_pex_bias_ctrl) 1177 afi_writel(pcie, 0, AFI_PEXBIAS_CTRL_0); 1178 1179 /* configure mode and disable all ports */ 1180 value = afi_readl(pcie, AFI_PCIE_CONFIG); 1181 value &= ~AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK; 1182 value |= AFI_PCIE_CONFIG_PCIE_DISABLE_ALL | pcie->xbar_config; 1183 value |= AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO_ALL; 1184 1185 list_for_each_entry(port, &pcie->ports, list) { 1186 value &= ~AFI_PCIE_CONFIG_PCIE_DISABLE(port->index); 1187 value &= ~AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(port->index); 1188 } 1189 1190 afi_writel(pcie, value, AFI_PCIE_CONFIG); 1191 1192 if (soc->has_gen2) { 1193 value = afi_readl(pcie, AFI_FUSE); 1194 value &= ~AFI_FUSE_PCIE_T0_GEN2_DIS; 1195 afi_writel(pcie, value, AFI_FUSE); 1196 } else { 1197 value = afi_readl(pcie, AFI_FUSE); 1198 value |= AFI_FUSE_PCIE_T0_GEN2_DIS; 1199 afi_writel(pcie, value, AFI_FUSE); 1200 } 1201 1202 /* Disable AFI dynamic clock gating and enable PCIe */ 1203 value = afi_readl(pcie, AFI_CONFIGURATION); 1204 value |= AFI_CONFIGURATION_EN_FPCI; 1205 value |= AFI_CONFIGURATION_CLKEN_OVERRIDE; 1206 afi_writel(pcie, value, AFI_CONFIGURATION); 1207 1208 value = AFI_INTR_EN_INI_SLVERR | AFI_INTR_EN_INI_DECERR | 1209 AFI_INTR_EN_TGT_SLVERR | AFI_INTR_EN_TGT_DECERR | 1210 AFI_INTR_EN_TGT_WRERR | AFI_INTR_EN_DFPCI_DECERR; 1211 1212 if (soc->has_intr_prsnt_sense) 1213 value |= AFI_INTR_EN_PRSNT_SENSE; 1214 1215 afi_writel(pcie, value, AFI_AFI_INTR_ENABLE); 1216 afi_writel(pcie, 0xffffffff, AFI_SM_INTR_ENABLE); 1217 1218 /* don't enable MSI for now, only when needed */ 1219 afi_writel(pcie, AFI_INTR_MASK_INT_MASK, AFI_INTR_MASK); 1220 1221 /* disable all exceptions */ 1222 afi_writel(pcie, 0, AFI_FPCI_ERROR_MASKS); 1223 } 1224 1225 static void tegra_pcie_power_off(struct tegra_pcie *pcie) 1226 { 1227 struct device *dev = pcie->dev; 1228 const struct tegra_pcie_soc *soc = pcie->soc; 1229 int err; 1230 1231 reset_control_assert(pcie->afi_rst); 1232 1233 clk_disable_unprepare(pcie->pll_e); 1234 if (soc->has_cml_clk) 1235 clk_disable_unprepare(pcie->cml_clk); 1236 clk_disable_unprepare(pcie->afi_clk); 1237 1238 if (!dev->pm_domain) 1239 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE); 1240 1241 err = regulator_bulk_disable(pcie->num_supplies, pcie->supplies); 1242 if (err < 0) 1243 dev_warn(dev, "failed to disable regulators: %d\n", err); 1244 } 1245 1246 static int tegra_pcie_power_on(struct tegra_pcie *pcie) 1247 { 1248 struct device *dev = pcie->dev; 1249 const struct tegra_pcie_soc *soc = pcie->soc; 1250 int err; 1251 1252 reset_control_assert(pcie->pcie_xrst); 1253 reset_control_assert(pcie->afi_rst); 1254 reset_control_assert(pcie->pex_rst); 1255 1256 if (!dev->pm_domain) 1257 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE); 1258 1259 /* enable regulators */ 1260 err = regulator_bulk_enable(pcie->num_supplies, pcie->supplies); 1261 if (err < 0) 1262 dev_err(dev, "failed to enable regulators: %d\n", err); 1263 1264 if (!dev->pm_domain) { 1265 err = tegra_powergate_power_on(TEGRA_POWERGATE_PCIE); 1266 if (err) { 1267 dev_err(dev, "failed to power ungate: %d\n", err); 1268 goto regulator_disable; 1269 } 1270 err = tegra_powergate_remove_clamping(TEGRA_POWERGATE_PCIE); 1271 if (err) { 1272 dev_err(dev, "failed to remove clamp: %d\n", err); 1273 goto powergate; 1274 } 1275 } 1276 1277 err = clk_prepare_enable(pcie->afi_clk); 1278 if (err < 0) { 1279 dev_err(dev, "failed to enable AFI clock: %d\n", err); 1280 goto powergate; 1281 } 1282 1283 if (soc->has_cml_clk) { 1284 err = clk_prepare_enable(pcie->cml_clk); 1285 if (err < 0) { 1286 dev_err(dev, "failed to enable CML clock: %d\n", err); 1287 goto disable_afi_clk; 1288 } 1289 } 1290 1291 err = clk_prepare_enable(pcie->pll_e); 1292 if (err < 0) { 1293 dev_err(dev, "failed to enable PLLE clock: %d\n", err); 1294 goto disable_cml_clk; 1295 } 1296 1297 reset_control_deassert(pcie->afi_rst); 1298 1299 return 0; 1300 1301 disable_cml_clk: 1302 if (soc->has_cml_clk) 1303 clk_disable_unprepare(pcie->cml_clk); 1304 disable_afi_clk: 1305 clk_disable_unprepare(pcie->afi_clk); 1306 powergate: 1307 if (!dev->pm_domain) 1308 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE); 1309 regulator_disable: 1310 regulator_bulk_disable(pcie->num_supplies, pcie->supplies); 1311 1312 return err; 1313 } 1314 1315 static void tegra_pcie_apply_pad_settings(struct tegra_pcie *pcie) 1316 { 1317 const struct tegra_pcie_soc *soc = pcie->soc; 1318 1319 /* Configure the reference clock driver */ 1320 pads_writel(pcie, soc->pads_refclk_cfg0, PADS_REFCLK_CFG0); 1321 1322 if (soc->num_ports > 2) 1323 pads_writel(pcie, soc->pads_refclk_cfg1, PADS_REFCLK_CFG1); 1324 } 1325 1326 static int tegra_pcie_clocks_get(struct tegra_pcie *pcie) 1327 { 1328 struct device *dev = pcie->dev; 1329 const struct tegra_pcie_soc *soc = pcie->soc; 1330 1331 pcie->pex_clk = devm_clk_get(dev, "pex"); 1332 if (IS_ERR(pcie->pex_clk)) 1333 return PTR_ERR(pcie->pex_clk); 1334 1335 pcie->afi_clk = devm_clk_get(dev, "afi"); 1336 if (IS_ERR(pcie->afi_clk)) 1337 return PTR_ERR(pcie->afi_clk); 1338 1339 pcie->pll_e = devm_clk_get(dev, "pll_e"); 1340 if (IS_ERR(pcie->pll_e)) 1341 return PTR_ERR(pcie->pll_e); 1342 1343 if (soc->has_cml_clk) { 1344 pcie->cml_clk = devm_clk_get(dev, "cml"); 1345 if (IS_ERR(pcie->cml_clk)) 1346 return PTR_ERR(pcie->cml_clk); 1347 } 1348 1349 return 0; 1350 } 1351 1352 static int tegra_pcie_resets_get(struct tegra_pcie *pcie) 1353 { 1354 struct device *dev = pcie->dev; 1355 1356 pcie->pex_rst = devm_reset_control_get_exclusive(dev, "pex"); 1357 if (IS_ERR(pcie->pex_rst)) 1358 return PTR_ERR(pcie->pex_rst); 1359 1360 pcie->afi_rst = devm_reset_control_get_exclusive(dev, "afi"); 1361 if (IS_ERR(pcie->afi_rst)) 1362 return PTR_ERR(pcie->afi_rst); 1363 1364 pcie->pcie_xrst = devm_reset_control_get_exclusive(dev, "pcie_x"); 1365 if (IS_ERR(pcie->pcie_xrst)) 1366 return PTR_ERR(pcie->pcie_xrst); 1367 1368 return 0; 1369 } 1370 1371 static int tegra_pcie_phys_get_legacy(struct tegra_pcie *pcie) 1372 { 1373 struct device *dev = pcie->dev; 1374 int err; 1375 1376 pcie->phy = devm_phy_optional_get(dev, "pcie"); 1377 if (IS_ERR(pcie->phy)) { 1378 err = PTR_ERR(pcie->phy); 1379 dev_err(dev, "failed to get PHY: %d\n", err); 1380 return err; 1381 } 1382 1383 err = phy_init(pcie->phy); 1384 if (err < 0) { 1385 dev_err(dev, "failed to initialize PHY: %d\n", err); 1386 return err; 1387 } 1388 1389 pcie->legacy_phy = true; 1390 1391 return 0; 1392 } 1393 1394 static struct phy *devm_of_phy_optional_get_index(struct device *dev, 1395 struct device_node *np, 1396 const char *consumer, 1397 unsigned int index) 1398 { 1399 struct phy *phy; 1400 char *name; 1401 1402 name = kasprintf(GFP_KERNEL, "%s-%u", consumer, index); 1403 if (!name) 1404 return ERR_PTR(-ENOMEM); 1405 1406 phy = devm_of_phy_get(dev, np, name); 1407 kfree(name); 1408 1409 if (PTR_ERR(phy) == -ENODEV) 1410 phy = NULL; 1411 1412 return phy; 1413 } 1414 1415 static int tegra_pcie_port_get_phys(struct tegra_pcie_port *port) 1416 { 1417 struct device *dev = port->pcie->dev; 1418 struct phy *phy; 1419 unsigned int i; 1420 int err; 1421 1422 port->phys = devm_kcalloc(dev, sizeof(phy), port->lanes, GFP_KERNEL); 1423 if (!port->phys) 1424 return -ENOMEM; 1425 1426 for (i = 0; i < port->lanes; i++) { 1427 phy = devm_of_phy_optional_get_index(dev, port->np, "pcie", i); 1428 if (IS_ERR(phy)) { 1429 dev_err(dev, "failed to get PHY#%u: %ld\n", i, 1430 PTR_ERR(phy)); 1431 return PTR_ERR(phy); 1432 } 1433 1434 err = phy_init(phy); 1435 if (err < 0) { 1436 dev_err(dev, "failed to initialize PHY#%u: %d\n", i, 1437 err); 1438 return err; 1439 } 1440 1441 port->phys[i] = phy; 1442 } 1443 1444 return 0; 1445 } 1446 1447 static int tegra_pcie_phys_get(struct tegra_pcie *pcie) 1448 { 1449 const struct tegra_pcie_soc *soc = pcie->soc; 1450 struct device_node *np = pcie->dev->of_node; 1451 struct tegra_pcie_port *port; 1452 int err; 1453 1454 if (!soc->has_gen2 || of_find_property(np, "phys", NULL) != NULL) 1455 return tegra_pcie_phys_get_legacy(pcie); 1456 1457 list_for_each_entry(port, &pcie->ports, list) { 1458 err = tegra_pcie_port_get_phys(port); 1459 if (err < 0) 1460 return err; 1461 } 1462 1463 return 0; 1464 } 1465 1466 static void tegra_pcie_phys_put(struct tegra_pcie *pcie) 1467 { 1468 struct tegra_pcie_port *port; 1469 struct device *dev = pcie->dev; 1470 int err, i; 1471 1472 if (pcie->legacy_phy) { 1473 err = phy_exit(pcie->phy); 1474 if (err < 0) 1475 dev_err(dev, "failed to teardown PHY: %d\n", err); 1476 return; 1477 } 1478 1479 list_for_each_entry(port, &pcie->ports, list) { 1480 for (i = 0; i < port->lanes; i++) { 1481 err = phy_exit(port->phys[i]); 1482 if (err < 0) 1483 dev_err(dev, "failed to teardown PHY#%u: %d\n", 1484 i, err); 1485 } 1486 } 1487 } 1488 1489 1490 static int tegra_pcie_get_resources(struct tegra_pcie *pcie) 1491 { 1492 struct device *dev = pcie->dev; 1493 struct platform_device *pdev = to_platform_device(dev); 1494 struct resource *pads, *afi, *res; 1495 const struct tegra_pcie_soc *soc = pcie->soc; 1496 int err; 1497 1498 err = tegra_pcie_clocks_get(pcie); 1499 if (err) { 1500 dev_err(dev, "failed to get clocks: %d\n", err); 1501 return err; 1502 } 1503 1504 err = tegra_pcie_resets_get(pcie); 1505 if (err) { 1506 dev_err(dev, "failed to get resets: %d\n", err); 1507 return err; 1508 } 1509 1510 if (soc->program_uphy) { 1511 err = tegra_pcie_phys_get(pcie); 1512 if (err < 0) { 1513 dev_err(dev, "failed to get PHYs: %d\n", err); 1514 return err; 1515 } 1516 } 1517 1518 pads = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pads"); 1519 pcie->pads = devm_ioremap_resource(dev, pads); 1520 if (IS_ERR(pcie->pads)) { 1521 err = PTR_ERR(pcie->pads); 1522 goto phys_put; 1523 } 1524 1525 afi = platform_get_resource_byname(pdev, IORESOURCE_MEM, "afi"); 1526 pcie->afi = devm_ioremap_resource(dev, afi); 1527 if (IS_ERR(pcie->afi)) { 1528 err = PTR_ERR(pcie->afi); 1529 goto phys_put; 1530 } 1531 1532 /* request configuration space, but remap later, on demand */ 1533 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cs"); 1534 if (!res) { 1535 err = -EADDRNOTAVAIL; 1536 goto phys_put; 1537 } 1538 1539 pcie->cs = *res; 1540 1541 /* constrain configuration space to 4 KiB */ 1542 pcie->cs.end = pcie->cs.start + SZ_4K - 1; 1543 1544 pcie->cfg = devm_ioremap_resource(dev, &pcie->cs); 1545 if (IS_ERR(pcie->cfg)) { 1546 err = PTR_ERR(pcie->cfg); 1547 goto phys_put; 1548 } 1549 1550 /* request interrupt */ 1551 err = platform_get_irq_byname(pdev, "intr"); 1552 if (err < 0) { 1553 dev_err(dev, "failed to get IRQ: %d\n", err); 1554 goto phys_put; 1555 } 1556 1557 pcie->irq = err; 1558 1559 err = request_irq(pcie->irq, tegra_pcie_isr, IRQF_SHARED, "PCIE", pcie); 1560 if (err) { 1561 dev_err(dev, "failed to register IRQ: %d\n", err); 1562 goto phys_put; 1563 } 1564 1565 return 0; 1566 1567 phys_put: 1568 if (soc->program_uphy) 1569 tegra_pcie_phys_put(pcie); 1570 return err; 1571 } 1572 1573 static int tegra_pcie_put_resources(struct tegra_pcie *pcie) 1574 { 1575 const struct tegra_pcie_soc *soc = pcie->soc; 1576 1577 if (pcie->irq > 0) 1578 free_irq(pcie->irq, pcie); 1579 1580 if (soc->program_uphy) 1581 tegra_pcie_phys_put(pcie); 1582 1583 return 0; 1584 } 1585 1586 static void tegra_pcie_pme_turnoff(struct tegra_pcie_port *port) 1587 { 1588 struct tegra_pcie *pcie = port->pcie; 1589 const struct tegra_pcie_soc *soc = pcie->soc; 1590 int err; 1591 u32 val; 1592 u8 ack_bit; 1593 1594 val = afi_readl(pcie, AFI_PCIE_PME); 1595 val |= (0x1 << soc->ports[port->index].pme.turnoff_bit); 1596 afi_writel(pcie, val, AFI_PCIE_PME); 1597 1598 ack_bit = soc->ports[port->index].pme.ack_bit; 1599 err = readl_poll_timeout(pcie->afi + AFI_PCIE_PME, val, 1600 val & (0x1 << ack_bit), 1, PME_ACK_TIMEOUT); 1601 if (err) 1602 dev_err(pcie->dev, "PME Ack is not received on port: %d\n", 1603 port->index); 1604 1605 usleep_range(10000, 11000); 1606 1607 val = afi_readl(pcie, AFI_PCIE_PME); 1608 val &= ~(0x1 << soc->ports[port->index].pme.turnoff_bit); 1609 afi_writel(pcie, val, AFI_PCIE_PME); 1610 } 1611 1612 static int tegra_msi_alloc(struct tegra_msi *chip) 1613 { 1614 int msi; 1615 1616 mutex_lock(&chip->lock); 1617 1618 msi = find_first_zero_bit(chip->used, INT_PCI_MSI_NR); 1619 if (msi < INT_PCI_MSI_NR) 1620 set_bit(msi, chip->used); 1621 else 1622 msi = -ENOSPC; 1623 1624 mutex_unlock(&chip->lock); 1625 1626 return msi; 1627 } 1628 1629 static void tegra_msi_free(struct tegra_msi *chip, unsigned long irq) 1630 { 1631 struct device *dev = chip->chip.dev; 1632 1633 mutex_lock(&chip->lock); 1634 1635 if (!test_bit(irq, chip->used)) 1636 dev_err(dev, "trying to free unused MSI#%lu\n", irq); 1637 else 1638 clear_bit(irq, chip->used); 1639 1640 mutex_unlock(&chip->lock); 1641 } 1642 1643 static irqreturn_t tegra_pcie_msi_irq(int irq, void *data) 1644 { 1645 struct tegra_pcie *pcie = data; 1646 struct device *dev = pcie->dev; 1647 struct tegra_msi *msi = &pcie->msi; 1648 unsigned int i, processed = 0; 1649 1650 for (i = 0; i < 8; i++) { 1651 unsigned long reg = afi_readl(pcie, AFI_MSI_VEC0 + i * 4); 1652 1653 while (reg) { 1654 unsigned int offset = find_first_bit(®, 32); 1655 unsigned int index = i * 32 + offset; 1656 unsigned int irq; 1657 1658 /* clear the interrupt */ 1659 afi_writel(pcie, 1 << offset, AFI_MSI_VEC0 + i * 4); 1660 1661 irq = irq_find_mapping(msi->domain, index); 1662 if (irq) { 1663 if (test_bit(index, msi->used)) 1664 generic_handle_irq(irq); 1665 else 1666 dev_info(dev, "unhandled MSI\n"); 1667 } else { 1668 /* 1669 * that's weird who triggered this? 1670 * just clear it 1671 */ 1672 dev_info(dev, "unexpected MSI\n"); 1673 } 1674 1675 /* see if there's any more pending in this vector */ 1676 reg = afi_readl(pcie, AFI_MSI_VEC0 + i * 4); 1677 1678 processed++; 1679 } 1680 } 1681 1682 return processed > 0 ? IRQ_HANDLED : IRQ_NONE; 1683 } 1684 1685 static int tegra_msi_setup_irq(struct msi_controller *chip, 1686 struct pci_dev *pdev, struct msi_desc *desc) 1687 { 1688 struct tegra_msi *msi = to_tegra_msi(chip); 1689 struct msi_msg msg; 1690 unsigned int irq; 1691 int hwirq; 1692 1693 hwirq = tegra_msi_alloc(msi); 1694 if (hwirq < 0) 1695 return hwirq; 1696 1697 irq = irq_create_mapping(msi->domain, hwirq); 1698 if (!irq) { 1699 tegra_msi_free(msi, hwirq); 1700 return -EINVAL; 1701 } 1702 1703 irq_set_msi_desc(irq, desc); 1704 1705 msg.address_lo = lower_32_bits(msi->phys); 1706 msg.address_hi = upper_32_bits(msi->phys); 1707 msg.data = hwirq; 1708 1709 pci_write_msi_msg(irq, &msg); 1710 1711 return 0; 1712 } 1713 1714 static void tegra_msi_teardown_irq(struct msi_controller *chip, 1715 unsigned int irq) 1716 { 1717 struct tegra_msi *msi = to_tegra_msi(chip); 1718 struct irq_data *d = irq_get_irq_data(irq); 1719 irq_hw_number_t hwirq = irqd_to_hwirq(d); 1720 1721 irq_dispose_mapping(irq); 1722 tegra_msi_free(msi, hwirq); 1723 } 1724 1725 static struct irq_chip tegra_msi_irq_chip = { 1726 .name = "Tegra PCIe MSI", 1727 .irq_enable = pci_msi_unmask_irq, 1728 .irq_disable = pci_msi_mask_irq, 1729 .irq_mask = pci_msi_mask_irq, 1730 .irq_unmask = pci_msi_unmask_irq, 1731 }; 1732 1733 static int tegra_msi_map(struct irq_domain *domain, unsigned int irq, 1734 irq_hw_number_t hwirq) 1735 { 1736 irq_set_chip_and_handler(irq, &tegra_msi_irq_chip, handle_simple_irq); 1737 irq_set_chip_data(irq, domain->host_data); 1738 1739 tegra_cpuidle_pcie_irqs_in_use(); 1740 1741 return 0; 1742 } 1743 1744 static const struct irq_domain_ops msi_domain_ops = { 1745 .map = tegra_msi_map, 1746 }; 1747 1748 static int tegra_pcie_msi_setup(struct tegra_pcie *pcie) 1749 { 1750 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); 1751 struct platform_device *pdev = to_platform_device(pcie->dev); 1752 struct tegra_msi *msi = &pcie->msi; 1753 struct device *dev = pcie->dev; 1754 int err; 1755 1756 mutex_init(&msi->lock); 1757 1758 msi->chip.dev = dev; 1759 msi->chip.setup_irq = tegra_msi_setup_irq; 1760 msi->chip.teardown_irq = tegra_msi_teardown_irq; 1761 1762 msi->domain = irq_domain_add_linear(dev->of_node, INT_PCI_MSI_NR, 1763 &msi_domain_ops, &msi->chip); 1764 if (!msi->domain) { 1765 dev_err(dev, "failed to create IRQ domain\n"); 1766 return -ENOMEM; 1767 } 1768 1769 err = platform_get_irq_byname(pdev, "msi"); 1770 if (err < 0) { 1771 dev_err(dev, "failed to get IRQ: %d\n", err); 1772 goto free_irq_domain; 1773 } 1774 1775 msi->irq = err; 1776 1777 err = request_irq(msi->irq, tegra_pcie_msi_irq, IRQF_NO_THREAD, 1778 tegra_msi_irq_chip.name, pcie); 1779 if (err < 0) { 1780 dev_err(dev, "failed to request IRQ: %d\n", err); 1781 goto free_irq_domain; 1782 } 1783 1784 /* Though the PCIe controller can address >32-bit address space, to 1785 * facilitate endpoints that support only 32-bit MSI target address, 1786 * the mask is set to 32-bit to make sure that MSI target address is 1787 * always a 32-bit address 1788 */ 1789 err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); 1790 if (err < 0) { 1791 dev_err(dev, "failed to set DMA coherent mask: %d\n", err); 1792 goto free_irq; 1793 } 1794 1795 msi->virt = dma_alloc_attrs(dev, PAGE_SIZE, &msi->phys, GFP_KERNEL, 1796 DMA_ATTR_NO_KERNEL_MAPPING); 1797 if (!msi->virt) { 1798 dev_err(dev, "failed to allocate DMA memory for MSI\n"); 1799 err = -ENOMEM; 1800 goto free_irq; 1801 } 1802 1803 host->msi = &msi->chip; 1804 1805 return 0; 1806 1807 free_irq: 1808 free_irq(msi->irq, pcie); 1809 free_irq_domain: 1810 irq_domain_remove(msi->domain); 1811 return err; 1812 } 1813 1814 static void tegra_pcie_enable_msi(struct tegra_pcie *pcie) 1815 { 1816 const struct tegra_pcie_soc *soc = pcie->soc; 1817 struct tegra_msi *msi = &pcie->msi; 1818 u32 reg; 1819 1820 afi_writel(pcie, msi->phys >> soc->msi_base_shift, AFI_MSI_FPCI_BAR_ST); 1821 afi_writel(pcie, msi->phys, AFI_MSI_AXI_BAR_ST); 1822 /* this register is in 4K increments */ 1823 afi_writel(pcie, 1, AFI_MSI_BAR_SZ); 1824 1825 /* enable all MSI vectors */ 1826 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC0); 1827 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC1); 1828 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC2); 1829 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC3); 1830 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC4); 1831 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC5); 1832 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC6); 1833 afi_writel(pcie, 0xffffffff, AFI_MSI_EN_VEC7); 1834 1835 /* and unmask the MSI interrupt */ 1836 reg = afi_readl(pcie, AFI_INTR_MASK); 1837 reg |= AFI_INTR_MASK_MSI_MASK; 1838 afi_writel(pcie, reg, AFI_INTR_MASK); 1839 } 1840 1841 static void tegra_pcie_msi_teardown(struct tegra_pcie *pcie) 1842 { 1843 struct tegra_msi *msi = &pcie->msi; 1844 unsigned int i, irq; 1845 1846 dma_free_attrs(pcie->dev, PAGE_SIZE, msi->virt, msi->phys, 1847 DMA_ATTR_NO_KERNEL_MAPPING); 1848 1849 if (msi->irq > 0) 1850 free_irq(msi->irq, pcie); 1851 1852 for (i = 0; i < INT_PCI_MSI_NR; i++) { 1853 irq = irq_find_mapping(msi->domain, i); 1854 if (irq > 0) 1855 irq_dispose_mapping(irq); 1856 } 1857 1858 irq_domain_remove(msi->domain); 1859 } 1860 1861 static int tegra_pcie_disable_msi(struct tegra_pcie *pcie) 1862 { 1863 u32 value; 1864 1865 /* mask the MSI interrupt */ 1866 value = afi_readl(pcie, AFI_INTR_MASK); 1867 value &= ~AFI_INTR_MASK_MSI_MASK; 1868 afi_writel(pcie, value, AFI_INTR_MASK); 1869 1870 /* disable all MSI vectors */ 1871 afi_writel(pcie, 0, AFI_MSI_EN_VEC0); 1872 afi_writel(pcie, 0, AFI_MSI_EN_VEC1); 1873 afi_writel(pcie, 0, AFI_MSI_EN_VEC2); 1874 afi_writel(pcie, 0, AFI_MSI_EN_VEC3); 1875 afi_writel(pcie, 0, AFI_MSI_EN_VEC4); 1876 afi_writel(pcie, 0, AFI_MSI_EN_VEC5); 1877 afi_writel(pcie, 0, AFI_MSI_EN_VEC6); 1878 afi_writel(pcie, 0, AFI_MSI_EN_VEC7); 1879 1880 return 0; 1881 } 1882 1883 static void tegra_pcie_disable_interrupts(struct tegra_pcie *pcie) 1884 { 1885 u32 value; 1886 1887 value = afi_readl(pcie, AFI_INTR_MASK); 1888 value &= ~AFI_INTR_MASK_INT_MASK; 1889 afi_writel(pcie, value, AFI_INTR_MASK); 1890 } 1891 1892 static int tegra_pcie_get_xbar_config(struct tegra_pcie *pcie, u32 lanes, 1893 u32 *xbar) 1894 { 1895 struct device *dev = pcie->dev; 1896 struct device_node *np = dev->of_node; 1897 1898 if (of_device_is_compatible(np, "nvidia,tegra186-pcie")) { 1899 switch (lanes) { 1900 case 0x010004: 1901 dev_info(dev, "4x1, 1x1 configuration\n"); 1902 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_401; 1903 return 0; 1904 1905 case 0x010102: 1906 dev_info(dev, "2x1, 1X1, 1x1 configuration\n"); 1907 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211; 1908 return 0; 1909 1910 case 0x010101: 1911 dev_info(dev, "1x1, 1x1, 1x1 configuration\n"); 1912 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_111; 1913 return 0; 1914 1915 default: 1916 dev_info(dev, "wrong configuration updated in DT, " 1917 "switching to default 2x1, 1x1, 1x1 " 1918 "configuration\n"); 1919 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211; 1920 return 0; 1921 } 1922 } else if (of_device_is_compatible(np, "nvidia,tegra124-pcie") || 1923 of_device_is_compatible(np, "nvidia,tegra210-pcie")) { 1924 switch (lanes) { 1925 case 0x0000104: 1926 dev_info(dev, "4x1, 1x1 configuration\n"); 1927 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1; 1928 return 0; 1929 1930 case 0x0000102: 1931 dev_info(dev, "2x1, 1x1 configuration\n"); 1932 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1; 1933 return 0; 1934 } 1935 } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) { 1936 switch (lanes) { 1937 case 0x00000204: 1938 dev_info(dev, "4x1, 2x1 configuration\n"); 1939 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420; 1940 return 0; 1941 1942 case 0x00020202: 1943 dev_info(dev, "2x3 configuration\n"); 1944 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222; 1945 return 0; 1946 1947 case 0x00010104: 1948 dev_info(dev, "4x1, 1x2 configuration\n"); 1949 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411; 1950 return 0; 1951 } 1952 } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) { 1953 switch (lanes) { 1954 case 0x00000004: 1955 dev_info(dev, "single-mode configuration\n"); 1956 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE; 1957 return 0; 1958 1959 case 0x00000202: 1960 dev_info(dev, "dual-mode configuration\n"); 1961 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL; 1962 return 0; 1963 } 1964 } 1965 1966 return -EINVAL; 1967 } 1968 1969 /* 1970 * Check whether a given set of supplies is available in a device tree node. 1971 * This is used to check whether the new or the legacy device tree bindings 1972 * should be used. 1973 */ 1974 static bool of_regulator_bulk_available(struct device_node *np, 1975 struct regulator_bulk_data *supplies, 1976 unsigned int num_supplies) 1977 { 1978 char property[32]; 1979 unsigned int i; 1980 1981 for (i = 0; i < num_supplies; i++) { 1982 snprintf(property, 32, "%s-supply", supplies[i].supply); 1983 1984 if (of_find_property(np, property, NULL) == NULL) 1985 return false; 1986 } 1987 1988 return true; 1989 } 1990 1991 /* 1992 * Old versions of the device tree binding for this device used a set of power 1993 * supplies that didn't match the hardware inputs. This happened to work for a 1994 * number of cases but is not future proof. However to preserve backwards- 1995 * compatibility with old device trees, this function will try to use the old 1996 * set of supplies. 1997 */ 1998 static int tegra_pcie_get_legacy_regulators(struct tegra_pcie *pcie) 1999 { 2000 struct device *dev = pcie->dev; 2001 struct device_node *np = dev->of_node; 2002 2003 if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) 2004 pcie->num_supplies = 3; 2005 else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) 2006 pcie->num_supplies = 2; 2007 2008 if (pcie->num_supplies == 0) { 2009 dev_err(dev, "device %pOF not supported in legacy mode\n", np); 2010 return -ENODEV; 2011 } 2012 2013 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies, 2014 sizeof(*pcie->supplies), 2015 GFP_KERNEL); 2016 if (!pcie->supplies) 2017 return -ENOMEM; 2018 2019 pcie->supplies[0].supply = "pex-clk"; 2020 pcie->supplies[1].supply = "vdd"; 2021 2022 if (pcie->num_supplies > 2) 2023 pcie->supplies[2].supply = "avdd"; 2024 2025 return devm_regulator_bulk_get(dev, pcie->num_supplies, pcie->supplies); 2026 } 2027 2028 /* 2029 * Obtains the list of regulators required for a particular generation of the 2030 * IP block. 2031 * 2032 * This would've been nice to do simply by providing static tables for use 2033 * with the regulator_bulk_*() API, but unfortunately Tegra30 is a bit quirky 2034 * in that it has two pairs or AVDD_PEX and VDD_PEX supplies (PEXA and PEXB) 2035 * and either seems to be optional depending on which ports are being used. 2036 */ 2037 static int tegra_pcie_get_regulators(struct tegra_pcie *pcie, u32 lane_mask) 2038 { 2039 struct device *dev = pcie->dev; 2040 struct device_node *np = dev->of_node; 2041 unsigned int i = 0; 2042 2043 if (of_device_is_compatible(np, "nvidia,tegra186-pcie")) { 2044 pcie->num_supplies = 4; 2045 2046 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies, 2047 sizeof(*pcie->supplies), 2048 GFP_KERNEL); 2049 if (!pcie->supplies) 2050 return -ENOMEM; 2051 2052 pcie->supplies[i++].supply = "dvdd-pex"; 2053 pcie->supplies[i++].supply = "hvdd-pex-pll"; 2054 pcie->supplies[i++].supply = "hvdd-pex"; 2055 pcie->supplies[i++].supply = "vddio-pexctl-aud"; 2056 } else if (of_device_is_compatible(np, "nvidia,tegra210-pcie")) { 2057 pcie->num_supplies = 6; 2058 2059 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies, 2060 sizeof(*pcie->supplies), 2061 GFP_KERNEL); 2062 if (!pcie->supplies) 2063 return -ENOMEM; 2064 2065 pcie->supplies[i++].supply = "avdd-pll-uerefe"; 2066 pcie->supplies[i++].supply = "hvddio-pex"; 2067 pcie->supplies[i++].supply = "dvddio-pex"; 2068 pcie->supplies[i++].supply = "dvdd-pex-pll"; 2069 pcie->supplies[i++].supply = "hvdd-pex-pll-e"; 2070 pcie->supplies[i++].supply = "vddio-pex-ctl"; 2071 } else if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) { 2072 pcie->num_supplies = 7; 2073 2074 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies, 2075 sizeof(*pcie->supplies), 2076 GFP_KERNEL); 2077 if (!pcie->supplies) 2078 return -ENOMEM; 2079 2080 pcie->supplies[i++].supply = "avddio-pex"; 2081 pcie->supplies[i++].supply = "dvddio-pex"; 2082 pcie->supplies[i++].supply = "avdd-pex-pll"; 2083 pcie->supplies[i++].supply = "hvdd-pex"; 2084 pcie->supplies[i++].supply = "hvdd-pex-pll-e"; 2085 pcie->supplies[i++].supply = "vddio-pex-ctl"; 2086 pcie->supplies[i++].supply = "avdd-pll-erefe"; 2087 } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) { 2088 bool need_pexa = false, need_pexb = false; 2089 2090 /* VDD_PEXA and AVDD_PEXA supply lanes 0 to 3 */ 2091 if (lane_mask & 0x0f) 2092 need_pexa = true; 2093 2094 /* VDD_PEXB and AVDD_PEXB supply lanes 4 to 5 */ 2095 if (lane_mask & 0x30) 2096 need_pexb = true; 2097 2098 pcie->num_supplies = 4 + (need_pexa ? 2 : 0) + 2099 (need_pexb ? 2 : 0); 2100 2101 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies, 2102 sizeof(*pcie->supplies), 2103 GFP_KERNEL); 2104 if (!pcie->supplies) 2105 return -ENOMEM; 2106 2107 pcie->supplies[i++].supply = "avdd-pex-pll"; 2108 pcie->supplies[i++].supply = "hvdd-pex"; 2109 pcie->supplies[i++].supply = "vddio-pex-ctl"; 2110 pcie->supplies[i++].supply = "avdd-plle"; 2111 2112 if (need_pexa) { 2113 pcie->supplies[i++].supply = "avdd-pexa"; 2114 pcie->supplies[i++].supply = "vdd-pexa"; 2115 } 2116 2117 if (need_pexb) { 2118 pcie->supplies[i++].supply = "avdd-pexb"; 2119 pcie->supplies[i++].supply = "vdd-pexb"; 2120 } 2121 } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) { 2122 pcie->num_supplies = 5; 2123 2124 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies, 2125 sizeof(*pcie->supplies), 2126 GFP_KERNEL); 2127 if (!pcie->supplies) 2128 return -ENOMEM; 2129 2130 pcie->supplies[0].supply = "avdd-pex"; 2131 pcie->supplies[1].supply = "vdd-pex"; 2132 pcie->supplies[2].supply = "avdd-pex-pll"; 2133 pcie->supplies[3].supply = "avdd-plle"; 2134 pcie->supplies[4].supply = "vddio-pex-clk"; 2135 } 2136 2137 if (of_regulator_bulk_available(dev->of_node, pcie->supplies, 2138 pcie->num_supplies)) 2139 return devm_regulator_bulk_get(dev, pcie->num_supplies, 2140 pcie->supplies); 2141 2142 /* 2143 * If not all regulators are available for this new scheme, assume 2144 * that the device tree complies with an older version of the device 2145 * tree binding. 2146 */ 2147 dev_info(dev, "using legacy DT binding for power supplies\n"); 2148 2149 devm_kfree(dev, pcie->supplies); 2150 pcie->num_supplies = 0; 2151 2152 return tegra_pcie_get_legacy_regulators(pcie); 2153 } 2154 2155 static int tegra_pcie_parse_dt(struct tegra_pcie *pcie) 2156 { 2157 struct device *dev = pcie->dev; 2158 struct device_node *np = dev->of_node, *port; 2159 const struct tegra_pcie_soc *soc = pcie->soc; 2160 struct of_pci_range_parser parser; 2161 struct of_pci_range range; 2162 u32 lanes = 0, mask = 0; 2163 unsigned int lane = 0; 2164 struct resource res; 2165 int err; 2166 2167 if (of_pci_range_parser_init(&parser, np)) { 2168 dev_err(dev, "missing \"ranges\" property\n"); 2169 return -EINVAL; 2170 } 2171 2172 for_each_of_pci_range(&parser, &range) { 2173 err = of_pci_range_to_resource(&range, np, &res); 2174 if (err < 0) 2175 return err; 2176 2177 switch (res.flags & IORESOURCE_TYPE_BITS) { 2178 case IORESOURCE_IO: 2179 /* Track the bus -> CPU I/O mapping offset. */ 2180 pcie->offset.io = res.start - range.pci_addr; 2181 2182 memcpy(&pcie->pio, &res, sizeof(res)); 2183 pcie->pio.name = np->full_name; 2184 2185 /* 2186 * The Tegra PCIe host bridge uses this to program the 2187 * mapping of the I/O space to the physical address, 2188 * so we override the .start and .end fields here that 2189 * of_pci_range_to_resource() converted to I/O space. 2190 * We also set the IORESOURCE_MEM type to clarify that 2191 * the resource is in the physical memory space. 2192 */ 2193 pcie->io.start = range.cpu_addr; 2194 pcie->io.end = range.cpu_addr + range.size - 1; 2195 pcie->io.flags = IORESOURCE_MEM; 2196 pcie->io.name = "I/O"; 2197 2198 memcpy(&res, &pcie->io, sizeof(res)); 2199 break; 2200 2201 case IORESOURCE_MEM: 2202 /* 2203 * Track the bus -> CPU memory mapping offset. This 2204 * assumes that the prefetchable and non-prefetchable 2205 * regions will be the last of type IORESOURCE_MEM in 2206 * the ranges property. 2207 * */ 2208 pcie->offset.mem = res.start - range.pci_addr; 2209 2210 if (res.flags & IORESOURCE_PREFETCH) { 2211 memcpy(&pcie->prefetch, &res, sizeof(res)); 2212 pcie->prefetch.name = "prefetchable"; 2213 } else { 2214 memcpy(&pcie->mem, &res, sizeof(res)); 2215 pcie->mem.name = "non-prefetchable"; 2216 } 2217 break; 2218 } 2219 } 2220 2221 err = of_pci_parse_bus_range(np, &pcie->busn); 2222 if (err < 0) { 2223 dev_err(dev, "failed to parse ranges property: %d\n", err); 2224 pcie->busn.name = np->name; 2225 pcie->busn.start = 0; 2226 pcie->busn.end = 0xff; 2227 pcie->busn.flags = IORESOURCE_BUS; 2228 } 2229 2230 /* parse root ports */ 2231 for_each_child_of_node(np, port) { 2232 struct tegra_pcie_port *rp; 2233 unsigned int index; 2234 u32 value; 2235 char *label; 2236 2237 err = of_pci_get_devfn(port); 2238 if (err < 0) { 2239 dev_err(dev, "failed to parse address: %d\n", err); 2240 goto err_node_put; 2241 } 2242 2243 index = PCI_SLOT(err); 2244 2245 if (index < 1 || index > soc->num_ports) { 2246 dev_err(dev, "invalid port number: %d\n", index); 2247 err = -EINVAL; 2248 goto err_node_put; 2249 } 2250 2251 index--; 2252 2253 err = of_property_read_u32(port, "nvidia,num-lanes", &value); 2254 if (err < 0) { 2255 dev_err(dev, "failed to parse # of lanes: %d\n", 2256 err); 2257 goto err_node_put; 2258 } 2259 2260 if (value > 16) { 2261 dev_err(dev, "invalid # of lanes: %u\n", value); 2262 err = -EINVAL; 2263 goto err_node_put; 2264 } 2265 2266 lanes |= value << (index << 3); 2267 2268 if (!of_device_is_available(port)) { 2269 lane += value; 2270 continue; 2271 } 2272 2273 mask |= ((1 << value) - 1) << lane; 2274 lane += value; 2275 2276 rp = devm_kzalloc(dev, sizeof(*rp), GFP_KERNEL); 2277 if (!rp) { 2278 err = -ENOMEM; 2279 goto err_node_put; 2280 } 2281 2282 err = of_address_to_resource(port, 0, &rp->regs); 2283 if (err < 0) { 2284 dev_err(dev, "failed to parse address: %d\n", err); 2285 goto err_node_put; 2286 } 2287 2288 INIT_LIST_HEAD(&rp->list); 2289 rp->index = index; 2290 rp->lanes = value; 2291 rp->pcie = pcie; 2292 rp->np = port; 2293 2294 rp->base = devm_pci_remap_cfg_resource(dev, &rp->regs); 2295 if (IS_ERR(rp->base)) 2296 return PTR_ERR(rp->base); 2297 2298 label = devm_kasprintf(dev, GFP_KERNEL, "pex-reset-%u", index); 2299 if (!label) { 2300 dev_err(dev, "failed to create reset GPIO label\n"); 2301 return -ENOMEM; 2302 } 2303 2304 /* 2305 * Returns -ENOENT if reset-gpios property is not populated 2306 * and in this case fall back to using AFI per port register 2307 * to toggle PERST# SFIO line. 2308 */ 2309 rp->reset_gpio = devm_gpiod_get_from_of_node(dev, port, 2310 "reset-gpios", 0, 2311 GPIOD_OUT_LOW, 2312 label); 2313 if (IS_ERR(rp->reset_gpio)) { 2314 if (PTR_ERR(rp->reset_gpio) == -ENOENT) { 2315 rp->reset_gpio = NULL; 2316 } else { 2317 dev_err(dev, "failed to get reset GPIO: %d\n", 2318 err); 2319 return PTR_ERR(rp->reset_gpio); 2320 } 2321 } 2322 2323 list_add_tail(&rp->list, &pcie->ports); 2324 } 2325 2326 err = tegra_pcie_get_xbar_config(pcie, lanes, &pcie->xbar_config); 2327 if (err < 0) { 2328 dev_err(dev, "invalid lane configuration\n"); 2329 return err; 2330 } 2331 2332 err = tegra_pcie_get_regulators(pcie, mask); 2333 if (err < 0) 2334 return err; 2335 2336 return 0; 2337 2338 err_node_put: 2339 of_node_put(port); 2340 return err; 2341 } 2342 2343 /* 2344 * FIXME: If there are no PCIe cards attached, then calling this function 2345 * can result in the increase of the bootup time as there are big timeout 2346 * loops. 2347 */ 2348 #define TEGRA_PCIE_LINKUP_TIMEOUT 200 /* up to 1.2 seconds */ 2349 static bool tegra_pcie_port_check_link(struct tegra_pcie_port *port) 2350 { 2351 struct device *dev = port->pcie->dev; 2352 unsigned int retries = 3; 2353 unsigned long value; 2354 2355 /* override presence detection */ 2356 value = readl(port->base + RP_PRIV_MISC); 2357 value &= ~RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT; 2358 value |= RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT; 2359 writel(value, port->base + RP_PRIV_MISC); 2360 2361 do { 2362 unsigned int timeout = TEGRA_PCIE_LINKUP_TIMEOUT; 2363 2364 do { 2365 value = readl(port->base + RP_VEND_XP); 2366 2367 if (value & RP_VEND_XP_DL_UP) 2368 break; 2369 2370 usleep_range(1000, 2000); 2371 } while (--timeout); 2372 2373 if (!timeout) { 2374 dev_dbg(dev, "link %u down, retrying\n", port->index); 2375 goto retry; 2376 } 2377 2378 timeout = TEGRA_PCIE_LINKUP_TIMEOUT; 2379 2380 do { 2381 value = readl(port->base + RP_LINK_CONTROL_STATUS); 2382 2383 if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE) 2384 return true; 2385 2386 usleep_range(1000, 2000); 2387 } while (--timeout); 2388 2389 retry: 2390 tegra_pcie_port_reset(port); 2391 } while (--retries); 2392 2393 return false; 2394 } 2395 2396 static void tegra_pcie_change_link_speed(struct tegra_pcie *pcie) 2397 { 2398 struct device *dev = pcie->dev; 2399 struct tegra_pcie_port *port; 2400 ktime_t deadline; 2401 u32 value; 2402 2403 list_for_each_entry(port, &pcie->ports, list) { 2404 /* 2405 * "Supported Link Speeds Vector" in "Link Capabilities 2" 2406 * is not supported by Tegra. tegra_pcie_change_link_speed() 2407 * is called only for Tegra chips which support Gen2. 2408 * So there no harm if supported link speed is not verified. 2409 */ 2410 value = readl(port->base + RP_LINK_CONTROL_STATUS_2); 2411 value &= ~PCI_EXP_LNKSTA_CLS; 2412 value |= PCI_EXP_LNKSTA_CLS_5_0GB; 2413 writel(value, port->base + RP_LINK_CONTROL_STATUS_2); 2414 2415 /* 2416 * Poll until link comes back from recovery to avoid race 2417 * condition. 2418 */ 2419 deadline = ktime_add_us(ktime_get(), LINK_RETRAIN_TIMEOUT); 2420 2421 while (ktime_before(ktime_get(), deadline)) { 2422 value = readl(port->base + RP_LINK_CONTROL_STATUS); 2423 if ((value & PCI_EXP_LNKSTA_LT) == 0) 2424 break; 2425 2426 usleep_range(2000, 3000); 2427 } 2428 2429 if (value & PCI_EXP_LNKSTA_LT) 2430 dev_warn(dev, "PCIe port %u link is in recovery\n", 2431 port->index); 2432 2433 /* Retrain the link */ 2434 value = readl(port->base + RP_LINK_CONTROL_STATUS); 2435 value |= PCI_EXP_LNKCTL_RL; 2436 writel(value, port->base + RP_LINK_CONTROL_STATUS); 2437 2438 deadline = ktime_add_us(ktime_get(), LINK_RETRAIN_TIMEOUT); 2439 2440 while (ktime_before(ktime_get(), deadline)) { 2441 value = readl(port->base + RP_LINK_CONTROL_STATUS); 2442 if ((value & PCI_EXP_LNKSTA_LT) == 0) 2443 break; 2444 2445 usleep_range(2000, 3000); 2446 } 2447 2448 if (value & PCI_EXP_LNKSTA_LT) 2449 dev_err(dev, "failed to retrain link of port %u\n", 2450 port->index); 2451 } 2452 } 2453 2454 static void tegra_pcie_enable_ports(struct tegra_pcie *pcie) 2455 { 2456 struct device *dev = pcie->dev; 2457 struct tegra_pcie_port *port, *tmp; 2458 2459 list_for_each_entry_safe(port, tmp, &pcie->ports, list) { 2460 dev_info(dev, "probing port %u, using %u lanes\n", 2461 port->index, port->lanes); 2462 2463 tegra_pcie_port_enable(port); 2464 } 2465 2466 /* Start LTSSM from Tegra side */ 2467 reset_control_deassert(pcie->pcie_xrst); 2468 2469 list_for_each_entry_safe(port, tmp, &pcie->ports, list) { 2470 if (tegra_pcie_port_check_link(port)) 2471 continue; 2472 2473 dev_info(dev, "link %u down, ignoring\n", port->index); 2474 2475 tegra_pcie_port_disable(port); 2476 tegra_pcie_port_free(port); 2477 } 2478 2479 if (pcie->soc->has_gen2) 2480 tegra_pcie_change_link_speed(pcie); 2481 } 2482 2483 static void tegra_pcie_disable_ports(struct tegra_pcie *pcie) 2484 { 2485 struct tegra_pcie_port *port, *tmp; 2486 2487 reset_control_assert(pcie->pcie_xrst); 2488 2489 list_for_each_entry_safe(port, tmp, &pcie->ports, list) 2490 tegra_pcie_port_disable(port); 2491 } 2492 2493 static const struct tegra_pcie_port_soc tegra20_pcie_ports[] = { 2494 { .pme.turnoff_bit = 0, .pme.ack_bit = 5 }, 2495 { .pme.turnoff_bit = 8, .pme.ack_bit = 10 }, 2496 }; 2497 2498 static const struct tegra_pcie_soc tegra20_pcie = { 2499 .num_ports = 2, 2500 .ports = tegra20_pcie_ports, 2501 .msi_base_shift = 0, 2502 .pads_pll_ctl = PADS_PLL_CTL_TEGRA20, 2503 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_DIV10, 2504 .pads_refclk_cfg0 = 0xfa5cfa5c, 2505 .has_pex_clkreq_en = false, 2506 .has_pex_bias_ctrl = false, 2507 .has_intr_prsnt_sense = false, 2508 .has_cml_clk = false, 2509 .has_gen2 = false, 2510 .force_pca_enable = false, 2511 .program_uphy = true, 2512 .update_clamp_threshold = false, 2513 .program_deskew_time = false, 2514 .raw_violation_fixup = false, 2515 .update_fc_timer = false, 2516 .has_cache_bars = true, 2517 .ectl.enable = false, 2518 }; 2519 2520 static const struct tegra_pcie_port_soc tegra30_pcie_ports[] = { 2521 { .pme.turnoff_bit = 0, .pme.ack_bit = 5 }, 2522 { .pme.turnoff_bit = 8, .pme.ack_bit = 10 }, 2523 { .pme.turnoff_bit = 16, .pme.ack_bit = 18 }, 2524 }; 2525 2526 static const struct tegra_pcie_soc tegra30_pcie = { 2527 .num_ports = 3, 2528 .ports = tegra30_pcie_ports, 2529 .msi_base_shift = 8, 2530 .afi_pex2_ctrl = 0x128, 2531 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30, 2532 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN, 2533 .pads_refclk_cfg0 = 0xfa5cfa5c, 2534 .pads_refclk_cfg1 = 0xfa5cfa5c, 2535 .has_pex_clkreq_en = true, 2536 .has_pex_bias_ctrl = true, 2537 .has_intr_prsnt_sense = true, 2538 .has_cml_clk = true, 2539 .has_gen2 = false, 2540 .force_pca_enable = false, 2541 .program_uphy = true, 2542 .update_clamp_threshold = false, 2543 .program_deskew_time = false, 2544 .raw_violation_fixup = false, 2545 .update_fc_timer = false, 2546 .has_cache_bars = false, 2547 .ectl.enable = false, 2548 }; 2549 2550 static const struct tegra_pcie_soc tegra124_pcie = { 2551 .num_ports = 2, 2552 .ports = tegra20_pcie_ports, 2553 .msi_base_shift = 8, 2554 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30, 2555 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN, 2556 .pads_refclk_cfg0 = 0x44ac44ac, 2557 /* FC threshold is bit[25:18] */ 2558 .update_fc_threshold = 0x03fc0000, 2559 .has_pex_clkreq_en = true, 2560 .has_pex_bias_ctrl = true, 2561 .has_intr_prsnt_sense = true, 2562 .has_cml_clk = true, 2563 .has_gen2 = true, 2564 .force_pca_enable = false, 2565 .program_uphy = true, 2566 .update_clamp_threshold = true, 2567 .program_deskew_time = false, 2568 .raw_violation_fixup = true, 2569 .update_fc_timer = false, 2570 .has_cache_bars = false, 2571 .ectl.enable = false, 2572 }; 2573 2574 static const struct tegra_pcie_soc tegra210_pcie = { 2575 .num_ports = 2, 2576 .ports = tegra20_pcie_ports, 2577 .msi_base_shift = 8, 2578 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30, 2579 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN, 2580 .pads_refclk_cfg0 = 0x90b890b8, 2581 /* FC threshold is bit[25:18] */ 2582 .update_fc_threshold = 0x01800000, 2583 .has_pex_clkreq_en = true, 2584 .has_pex_bias_ctrl = true, 2585 .has_intr_prsnt_sense = true, 2586 .has_cml_clk = true, 2587 .has_gen2 = true, 2588 .force_pca_enable = true, 2589 .program_uphy = true, 2590 .update_clamp_threshold = true, 2591 .program_deskew_time = true, 2592 .raw_violation_fixup = false, 2593 .update_fc_timer = true, 2594 .has_cache_bars = false, 2595 .ectl = { 2596 .regs = { 2597 .rp_ectl_2_r1 = 0x0000000f, 2598 .rp_ectl_4_r1 = 0x00000067, 2599 .rp_ectl_5_r1 = 0x55010000, 2600 .rp_ectl_6_r1 = 0x00000001, 2601 .rp_ectl_2_r2 = 0x0000008f, 2602 .rp_ectl_4_r2 = 0x000000c7, 2603 .rp_ectl_5_r2 = 0x55010000, 2604 .rp_ectl_6_r2 = 0x00000001, 2605 }, 2606 .enable = true, 2607 }, 2608 }; 2609 2610 static const struct tegra_pcie_port_soc tegra186_pcie_ports[] = { 2611 { .pme.turnoff_bit = 0, .pme.ack_bit = 5 }, 2612 { .pme.turnoff_bit = 8, .pme.ack_bit = 10 }, 2613 { .pme.turnoff_bit = 12, .pme.ack_bit = 14 }, 2614 }; 2615 2616 static const struct tegra_pcie_soc tegra186_pcie = { 2617 .num_ports = 3, 2618 .ports = tegra186_pcie_ports, 2619 .msi_base_shift = 8, 2620 .afi_pex2_ctrl = 0x19c, 2621 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30, 2622 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN, 2623 .pads_refclk_cfg0 = 0x80b880b8, 2624 .pads_refclk_cfg1 = 0x000480b8, 2625 .has_pex_clkreq_en = true, 2626 .has_pex_bias_ctrl = true, 2627 .has_intr_prsnt_sense = true, 2628 .has_cml_clk = false, 2629 .has_gen2 = true, 2630 .force_pca_enable = false, 2631 .program_uphy = false, 2632 .update_clamp_threshold = false, 2633 .program_deskew_time = false, 2634 .raw_violation_fixup = false, 2635 .update_fc_timer = false, 2636 .has_cache_bars = false, 2637 .ectl.enable = false, 2638 }; 2639 2640 static const struct of_device_id tegra_pcie_of_match[] = { 2641 { .compatible = "nvidia,tegra186-pcie", .data = &tegra186_pcie }, 2642 { .compatible = "nvidia,tegra210-pcie", .data = &tegra210_pcie }, 2643 { .compatible = "nvidia,tegra124-pcie", .data = &tegra124_pcie }, 2644 { .compatible = "nvidia,tegra30-pcie", .data = &tegra30_pcie }, 2645 { .compatible = "nvidia,tegra20-pcie", .data = &tegra20_pcie }, 2646 { }, 2647 }; 2648 2649 static void *tegra_pcie_ports_seq_start(struct seq_file *s, loff_t *pos) 2650 { 2651 struct tegra_pcie *pcie = s->private; 2652 2653 if (list_empty(&pcie->ports)) 2654 return NULL; 2655 2656 seq_printf(s, "Index Status\n"); 2657 2658 return seq_list_start(&pcie->ports, *pos); 2659 } 2660 2661 static void *tegra_pcie_ports_seq_next(struct seq_file *s, void *v, loff_t *pos) 2662 { 2663 struct tegra_pcie *pcie = s->private; 2664 2665 return seq_list_next(v, &pcie->ports, pos); 2666 } 2667 2668 static void tegra_pcie_ports_seq_stop(struct seq_file *s, void *v) 2669 { 2670 } 2671 2672 static int tegra_pcie_ports_seq_show(struct seq_file *s, void *v) 2673 { 2674 bool up = false, active = false; 2675 struct tegra_pcie_port *port; 2676 unsigned int value; 2677 2678 port = list_entry(v, struct tegra_pcie_port, list); 2679 2680 value = readl(port->base + RP_VEND_XP); 2681 2682 if (value & RP_VEND_XP_DL_UP) 2683 up = true; 2684 2685 value = readl(port->base + RP_LINK_CONTROL_STATUS); 2686 2687 if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE) 2688 active = true; 2689 2690 seq_printf(s, "%2u ", port->index); 2691 2692 if (up) 2693 seq_printf(s, "up"); 2694 2695 if (active) { 2696 if (up) 2697 seq_printf(s, ", "); 2698 2699 seq_printf(s, "active"); 2700 } 2701 2702 seq_printf(s, "\n"); 2703 return 0; 2704 } 2705 2706 static const struct seq_operations tegra_pcie_ports_seq_ops = { 2707 .start = tegra_pcie_ports_seq_start, 2708 .next = tegra_pcie_ports_seq_next, 2709 .stop = tegra_pcie_ports_seq_stop, 2710 .show = tegra_pcie_ports_seq_show, 2711 }; 2712 2713 static int tegra_pcie_ports_open(struct inode *inode, struct file *file) 2714 { 2715 struct tegra_pcie *pcie = inode->i_private; 2716 struct seq_file *s; 2717 int err; 2718 2719 err = seq_open(file, &tegra_pcie_ports_seq_ops); 2720 if (err) 2721 return err; 2722 2723 s = file->private_data; 2724 s->private = pcie; 2725 2726 return 0; 2727 } 2728 2729 static const struct file_operations tegra_pcie_ports_ops = { 2730 .owner = THIS_MODULE, 2731 .open = tegra_pcie_ports_open, 2732 .read = seq_read, 2733 .llseek = seq_lseek, 2734 .release = seq_release, 2735 }; 2736 2737 static void tegra_pcie_debugfs_exit(struct tegra_pcie *pcie) 2738 { 2739 debugfs_remove_recursive(pcie->debugfs); 2740 pcie->debugfs = NULL; 2741 } 2742 2743 static int tegra_pcie_debugfs_init(struct tegra_pcie *pcie) 2744 { 2745 struct dentry *file; 2746 2747 pcie->debugfs = debugfs_create_dir("pcie", NULL); 2748 if (!pcie->debugfs) 2749 return -ENOMEM; 2750 2751 file = debugfs_create_file("ports", S_IFREG | S_IRUGO, pcie->debugfs, 2752 pcie, &tegra_pcie_ports_ops); 2753 if (!file) 2754 goto remove; 2755 2756 return 0; 2757 2758 remove: 2759 tegra_pcie_debugfs_exit(pcie); 2760 return -ENOMEM; 2761 } 2762 2763 static int tegra_pcie_probe(struct platform_device *pdev) 2764 { 2765 struct device *dev = &pdev->dev; 2766 struct pci_host_bridge *host; 2767 struct tegra_pcie *pcie; 2768 struct pci_bus *child; 2769 int err; 2770 2771 host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); 2772 if (!host) 2773 return -ENOMEM; 2774 2775 pcie = pci_host_bridge_priv(host); 2776 host->sysdata = pcie; 2777 platform_set_drvdata(pdev, pcie); 2778 2779 pcie->soc = of_device_get_match_data(dev); 2780 INIT_LIST_HEAD(&pcie->ports); 2781 pcie->dev = dev; 2782 2783 err = tegra_pcie_parse_dt(pcie); 2784 if (err < 0) 2785 return err; 2786 2787 err = tegra_pcie_get_resources(pcie); 2788 if (err < 0) { 2789 dev_err(dev, "failed to request resources: %d\n", err); 2790 return err; 2791 } 2792 2793 err = tegra_pcie_msi_setup(pcie); 2794 if (err < 0) { 2795 dev_err(dev, "failed to enable MSI support: %d\n", err); 2796 goto put_resources; 2797 } 2798 2799 pm_runtime_enable(pcie->dev); 2800 err = pm_runtime_get_sync(pcie->dev); 2801 if (err < 0) { 2802 dev_err(dev, "fail to enable pcie controller: %d\n", err); 2803 goto teardown_msi; 2804 } 2805 2806 err = tegra_pcie_request_resources(pcie); 2807 if (err) 2808 goto pm_runtime_put; 2809 2810 host->busnr = pcie->busn.start; 2811 host->dev.parent = &pdev->dev; 2812 host->ops = &tegra_pcie_ops; 2813 host->map_irq = tegra_pcie_map_irq; 2814 host->swizzle_irq = pci_common_swizzle; 2815 2816 err = pci_scan_root_bus_bridge(host); 2817 if (err < 0) { 2818 dev_err(dev, "failed to register host: %d\n", err); 2819 goto free_resources; 2820 } 2821 2822 pci_bus_size_bridges(host->bus); 2823 pci_bus_assign_resources(host->bus); 2824 2825 list_for_each_entry(child, &host->bus->children, node) 2826 pcie_bus_configure_settings(child); 2827 2828 pci_bus_add_devices(host->bus); 2829 2830 if (IS_ENABLED(CONFIG_DEBUG_FS)) { 2831 err = tegra_pcie_debugfs_init(pcie); 2832 if (err < 0) 2833 dev_err(dev, "failed to setup debugfs: %d\n", err); 2834 } 2835 2836 return 0; 2837 2838 free_resources: 2839 tegra_pcie_free_resources(pcie); 2840 pm_runtime_put: 2841 pm_runtime_put_sync(pcie->dev); 2842 pm_runtime_disable(pcie->dev); 2843 teardown_msi: 2844 tegra_pcie_msi_teardown(pcie); 2845 put_resources: 2846 tegra_pcie_put_resources(pcie); 2847 return err; 2848 } 2849 2850 static int tegra_pcie_remove(struct platform_device *pdev) 2851 { 2852 struct tegra_pcie *pcie = platform_get_drvdata(pdev); 2853 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); 2854 struct tegra_pcie_port *port, *tmp; 2855 2856 if (IS_ENABLED(CONFIG_DEBUG_FS)) 2857 tegra_pcie_debugfs_exit(pcie); 2858 2859 pci_stop_root_bus(host->bus); 2860 pci_remove_root_bus(host->bus); 2861 tegra_pcie_free_resources(pcie); 2862 pm_runtime_put_sync(pcie->dev); 2863 pm_runtime_disable(pcie->dev); 2864 2865 if (IS_ENABLED(CONFIG_PCI_MSI)) 2866 tegra_pcie_msi_teardown(pcie); 2867 2868 tegra_pcie_put_resources(pcie); 2869 2870 list_for_each_entry_safe(port, tmp, &pcie->ports, list) 2871 tegra_pcie_port_free(port); 2872 2873 return 0; 2874 } 2875 2876 static int __maybe_unused tegra_pcie_pm_suspend(struct device *dev) 2877 { 2878 struct tegra_pcie *pcie = dev_get_drvdata(dev); 2879 struct tegra_pcie_port *port; 2880 int err; 2881 2882 list_for_each_entry(port, &pcie->ports, list) 2883 tegra_pcie_pme_turnoff(port); 2884 2885 tegra_pcie_disable_ports(pcie); 2886 2887 /* 2888 * AFI_INTR is unmasked in tegra_pcie_enable_controller(), mask it to 2889 * avoid unwanted interrupts raised by AFI after pex_rst is asserted. 2890 */ 2891 tegra_pcie_disable_interrupts(pcie); 2892 2893 if (pcie->soc->program_uphy) { 2894 err = tegra_pcie_phy_power_off(pcie); 2895 if (err < 0) 2896 dev_err(dev, "failed to power off PHY(s): %d\n", err); 2897 } 2898 2899 reset_control_assert(pcie->pex_rst); 2900 clk_disable_unprepare(pcie->pex_clk); 2901 2902 if (IS_ENABLED(CONFIG_PCI_MSI)) 2903 tegra_pcie_disable_msi(pcie); 2904 2905 pinctrl_pm_select_idle_state(dev); 2906 tegra_pcie_power_off(pcie); 2907 2908 return 0; 2909 } 2910 2911 static int __maybe_unused tegra_pcie_pm_resume(struct device *dev) 2912 { 2913 struct tegra_pcie *pcie = dev_get_drvdata(dev); 2914 int err; 2915 2916 err = tegra_pcie_power_on(pcie); 2917 if (err) { 2918 dev_err(dev, "tegra pcie power on fail: %d\n", err); 2919 return err; 2920 } 2921 2922 err = pinctrl_pm_select_default_state(dev); 2923 if (err < 0) { 2924 dev_err(dev, "failed to disable PCIe IO DPD: %d\n", err); 2925 goto poweroff; 2926 } 2927 2928 tegra_pcie_enable_controller(pcie); 2929 tegra_pcie_setup_translations(pcie); 2930 2931 if (IS_ENABLED(CONFIG_PCI_MSI)) 2932 tegra_pcie_enable_msi(pcie); 2933 2934 err = clk_prepare_enable(pcie->pex_clk); 2935 if (err) { 2936 dev_err(dev, "failed to enable PEX clock: %d\n", err); 2937 goto pex_dpd_enable; 2938 } 2939 2940 reset_control_deassert(pcie->pex_rst); 2941 2942 if (pcie->soc->program_uphy) { 2943 err = tegra_pcie_phy_power_on(pcie); 2944 if (err < 0) { 2945 dev_err(dev, "failed to power on PHY(s): %d\n", err); 2946 goto disable_pex_clk; 2947 } 2948 } 2949 2950 tegra_pcie_apply_pad_settings(pcie); 2951 tegra_pcie_enable_ports(pcie); 2952 2953 return 0; 2954 2955 disable_pex_clk: 2956 reset_control_assert(pcie->pex_rst); 2957 clk_disable_unprepare(pcie->pex_clk); 2958 pex_dpd_enable: 2959 pinctrl_pm_select_idle_state(dev); 2960 poweroff: 2961 tegra_pcie_power_off(pcie); 2962 2963 return err; 2964 } 2965 2966 static const struct dev_pm_ops tegra_pcie_pm_ops = { 2967 SET_RUNTIME_PM_OPS(tegra_pcie_pm_suspend, tegra_pcie_pm_resume, NULL) 2968 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(tegra_pcie_pm_suspend, 2969 tegra_pcie_pm_resume) 2970 }; 2971 2972 static struct platform_driver tegra_pcie_driver = { 2973 .driver = { 2974 .name = "tegra-pcie", 2975 .of_match_table = tegra_pcie_of_match, 2976 .suppress_bind_attrs = true, 2977 .pm = &tegra_pcie_pm_ops, 2978 }, 2979 .probe = tegra_pcie_probe, 2980 .remove = tegra_pcie_remove, 2981 }; 2982 module_platform_driver(tegra_pcie_driver); 2983 MODULE_LICENSE("GPL"); 2984