1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * PCIe host controller driver for Tegra194 SoC 4 * 5 * Copyright (C) 2019 NVIDIA Corporation. 6 * 7 * Author: Vidya Sagar <vidyas@nvidia.com> 8 */ 9 10 #include <linux/clk.h> 11 #include <linux/debugfs.h> 12 #include <linux/delay.h> 13 #include <linux/gpio.h> 14 #include <linux/gpio/consumer.h> 15 #include <linux/interrupt.h> 16 #include <linux/iopoll.h> 17 #include <linux/kernel.h> 18 #include <linux/module.h> 19 #include <linux/of.h> 20 #include <linux/of_device.h> 21 #include <linux/of_gpio.h> 22 #include <linux/of_irq.h> 23 #include <linux/of_pci.h> 24 #include <linux/pci.h> 25 #include <linux/phy/phy.h> 26 #include <linux/pinctrl/consumer.h> 27 #include <linux/platform_device.h> 28 #include <linux/pm_runtime.h> 29 #include <linux/random.h> 30 #include <linux/reset.h> 31 #include <linux/resource.h> 32 #include <linux/types.h> 33 #include "pcie-designware.h" 34 #include <soc/tegra/bpmp.h> 35 #include <soc/tegra/bpmp-abi.h> 36 #include "../../pci.h" 37 38 #define APPL_PINMUX 0x0 39 #define APPL_PINMUX_PEX_RST BIT(0) 40 #define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2) 41 #define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3) 42 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4) 43 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5) 44 45 #define APPL_CTRL 0x4 46 #define APPL_CTRL_SYS_PRE_DET_STATE BIT(6) 47 #define APPL_CTRL_LTSSM_EN BIT(7) 48 #define APPL_CTRL_HW_HOT_RST_EN BIT(20) 49 #define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0) 50 #define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22 51 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1 52 53 #define APPL_INTR_EN_L0_0 0x8 54 #define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0) 55 #define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4) 56 #define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8) 57 #define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN BIT(15) 58 #define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19) 59 #define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30) 60 #define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31) 61 62 #define APPL_INTR_STATUS_L0 0xC 63 #define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0) 64 #define APPL_INTR_STATUS_L0_INT_INT BIT(8) 65 #define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT BIT(15) 66 #define APPL_INTR_STATUS_L0_PEX_RST_INT BIT(16) 67 #define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18) 68 69 #define APPL_INTR_EN_L1_0_0 0x1C 70 #define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1) 71 #define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN BIT(3) 72 #define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN BIT(30) 73 74 #define APPL_INTR_STATUS_L1_0_0 0x20 75 #define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1) 76 #define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED BIT(3) 77 #define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE BIT(30) 78 79 #define APPL_INTR_STATUS_L1_1 0x2C 80 #define APPL_INTR_STATUS_L1_2 0x30 81 #define APPL_INTR_STATUS_L1_3 0x34 82 #define APPL_INTR_STATUS_L1_6 0x3C 83 #define APPL_INTR_STATUS_L1_7 0x40 84 #define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED BIT(1) 85 86 #define APPL_INTR_EN_L1_8_0 0x44 87 #define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2) 88 #define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3) 89 #define APPL_INTR_EN_L1_8_INTX_EN BIT(11) 90 #define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15) 91 92 #define APPL_INTR_STATUS_L1_8_0 0x4C 93 #define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6) 94 #define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2) 95 #define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3) 96 97 #define APPL_INTR_STATUS_L1_9 0x54 98 #define APPL_INTR_STATUS_L1_10 0x58 99 #define APPL_INTR_STATUS_L1_11 0x64 100 #define APPL_INTR_STATUS_L1_13 0x74 101 #define APPL_INTR_STATUS_L1_14 0x78 102 #define APPL_INTR_STATUS_L1_15 0x7C 103 #define APPL_INTR_STATUS_L1_17 0x88 104 105 #define APPL_INTR_EN_L1_18 0x90 106 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2) 107 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) 108 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) 109 110 #define APPL_INTR_STATUS_L1_18 0x94 111 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2) 112 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) 113 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) 114 115 #define APPL_MSI_CTRL_1 0xAC 116 117 #define APPL_MSI_CTRL_2 0xB0 118 119 #define APPL_LEGACY_INTX 0xB8 120 121 #define APPL_LTR_MSG_1 0xC4 122 #define LTR_MSG_REQ BIT(15) 123 #define LTR_MST_NO_SNOOP_SHIFT 16 124 125 #define APPL_LTR_MSG_2 0xC8 126 #define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3) 127 128 #define APPL_LINK_STATUS 0xCC 129 #define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0) 130 131 #define APPL_DEBUG 0xD0 132 #define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21) 133 #define APPL_DEBUG_PM_LINKST_IN_L0 0x11 134 #define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3) 135 #define APPL_DEBUG_LTSSM_STATE_SHIFT 3 136 #define LTSSM_STATE_PRE_DETECT 5 137 138 #define APPL_RADM_STATUS 0xE4 139 #define APPL_PM_XMT_TURNOFF_STATE BIT(0) 140 141 #define APPL_DM_TYPE 0x100 142 #define APPL_DM_TYPE_MASK GENMASK(3, 0) 143 #define APPL_DM_TYPE_RP 0x4 144 #define APPL_DM_TYPE_EP 0x0 145 146 #define APPL_CFG_BASE_ADDR 0x104 147 #define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12) 148 149 #define APPL_CFG_IATU_DMA_BASE_ADDR 0x108 150 #define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18) 151 152 #define APPL_CFG_MISC 0x110 153 #define APPL_CFG_MISC_SLV_EP_MODE BIT(14) 154 #define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10) 155 #define APPL_CFG_MISC_ARCACHE_SHIFT 10 156 #define APPL_CFG_MISC_ARCACHE_VAL 3 157 158 #define APPL_CFG_SLCG_OVERRIDE 0x114 159 #define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0) 160 161 #define APPL_CAR_RESET_OVRD 0x12C 162 #define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0) 163 164 #define IO_BASE_IO_DECODE BIT(0) 165 #define IO_BASE_IO_DECODE_BIT8 BIT(8) 166 167 #define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0) 168 #define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16) 169 170 #define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718 171 #define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19) 172 173 #define EVENT_COUNTER_ALL_CLEAR 0x3 174 #define EVENT_COUNTER_ENABLE_ALL 0x7 175 #define EVENT_COUNTER_ENABLE_SHIFT 2 176 #define EVENT_COUNTER_EVENT_SEL_MASK GENMASK(7, 0) 177 #define EVENT_COUNTER_EVENT_SEL_SHIFT 16 178 #define EVENT_COUNTER_EVENT_Tx_L0S 0x2 179 #define EVENT_COUNTER_EVENT_Rx_L0S 0x3 180 #define EVENT_COUNTER_EVENT_L1 0x5 181 #define EVENT_COUNTER_EVENT_L1_1 0x7 182 #define EVENT_COUNTER_EVENT_L1_2 0x8 183 #define EVENT_COUNTER_GROUP_SEL_SHIFT 24 184 #define EVENT_COUNTER_GROUP_5 0x5 185 186 #define N_FTS_VAL 52 187 #define FTS_VAL 52 188 189 #define PORT_LOGIC_MSI_CTRL_INT_0_EN 0x828 190 191 #define GEN3_EQ_CONTROL_OFF 0x8a8 192 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8 193 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8) 194 #define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0) 195 196 #define GEN3_RELATED_OFF 0x890 197 #define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL BIT(0) 198 #define GEN3_RELATED_OFF_GEN3_EQ_DISABLE BIT(16) 199 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT 24 200 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK GENMASK(25, 24) 201 202 #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0 203 #define AMBA_ERROR_RESPONSE_CRS_SHIFT 3 204 #define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0) 205 #define AMBA_ERROR_RESPONSE_CRS_OKAY 0 206 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1 207 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2 208 209 #define MSIX_ADDR_MATCH_LOW_OFF 0x940 210 #define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0) 211 #define MSIX_ADDR_MATCH_LOW_OFF_MASK GENMASK(31, 2) 212 213 #define MSIX_ADDR_MATCH_HIGH_OFF 0x944 214 #define MSIX_ADDR_MATCH_HIGH_OFF_MASK GENMASK(31, 0) 215 216 #define PORT_LOGIC_MSIX_DOORBELL 0x948 217 218 #define CAP_SPCIE_CAP_OFF 0x154 219 #define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0) 220 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8) 221 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8 222 223 #define PME_ACK_TIMEOUT 10000 224 225 #define LTSSM_TIMEOUT 50000 /* 50ms */ 226 227 #define GEN3_GEN4_EQ_PRESET_INIT 5 228 229 #define GEN1_CORE_CLK_FREQ 62500000 230 #define GEN2_CORE_CLK_FREQ 125000000 231 #define GEN3_CORE_CLK_FREQ 250000000 232 #define GEN4_CORE_CLK_FREQ 500000000 233 234 #define LTR_MSG_TIMEOUT (100 * 1000) 235 236 #define PERST_DEBOUNCE_TIME (5 * 1000) 237 238 #define EP_STATE_DISABLED 0 239 #define EP_STATE_ENABLED 1 240 241 static const unsigned int pcie_gen_freq[] = { 242 GEN1_CORE_CLK_FREQ, 243 GEN2_CORE_CLK_FREQ, 244 GEN3_CORE_CLK_FREQ, 245 GEN4_CORE_CLK_FREQ 246 }; 247 248 static const u32 event_cntr_ctrl_offset[] = { 249 0x1d8, 250 0x1a8, 251 0x1a8, 252 0x1a8, 253 0x1c4, 254 0x1d8 255 }; 256 257 static const u32 event_cntr_data_offset[] = { 258 0x1dc, 259 0x1ac, 260 0x1ac, 261 0x1ac, 262 0x1c8, 263 0x1dc 264 }; 265 266 struct tegra_pcie_dw { 267 struct device *dev; 268 struct resource *appl_res; 269 struct resource *dbi_res; 270 struct resource *atu_dma_res; 271 void __iomem *appl_base; 272 struct clk *core_clk; 273 struct reset_control *core_apb_rst; 274 struct reset_control *core_rst; 275 struct dw_pcie pci; 276 struct tegra_bpmp *bpmp; 277 278 enum dw_pcie_device_mode mode; 279 280 bool supports_clkreq; 281 bool enable_cdm_check; 282 bool link_state; 283 bool update_fc_fixup; 284 u8 init_link_width; 285 u32 msi_ctrl_int; 286 u32 num_lanes; 287 u32 cid; 288 u32 cfg_link_cap_l1sub; 289 u32 pcie_cap_base; 290 u32 aspm_cmrt; 291 u32 aspm_pwr_on_t; 292 u32 aspm_l0s_enter_lat; 293 294 struct regulator *pex_ctl_supply; 295 struct regulator *slot_ctl_3v3; 296 struct regulator *slot_ctl_12v; 297 298 unsigned int phy_count; 299 struct phy **phys; 300 301 struct dentry *debugfs; 302 303 /* Endpoint mode specific */ 304 struct gpio_desc *pex_rst_gpiod; 305 struct gpio_desc *pex_refclk_sel_gpiod; 306 unsigned int pex_rst_irq; 307 int ep_state; 308 }; 309 310 struct tegra_pcie_dw_of_data { 311 enum dw_pcie_device_mode mode; 312 }; 313 314 static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci) 315 { 316 return container_of(pci, struct tegra_pcie_dw, pci); 317 } 318 319 static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value, 320 const u32 reg) 321 { 322 writel_relaxed(value, pcie->appl_base + reg); 323 } 324 325 static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg) 326 { 327 return readl_relaxed(pcie->appl_base + reg); 328 } 329 330 struct tegra_pcie_soc { 331 enum dw_pcie_device_mode mode; 332 }; 333 334 static void apply_bad_link_workaround(struct pcie_port *pp) 335 { 336 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 337 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 338 u32 current_link_width; 339 u16 val; 340 341 /* 342 * NOTE:- Since this scenario is uncommon and link as such is not 343 * stable anyway, not waiting to confirm if link is really 344 * transitioning to Gen-2 speed 345 */ 346 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); 347 if (val & PCI_EXP_LNKSTA_LBMS) { 348 current_link_width = (val & PCI_EXP_LNKSTA_NLW) >> 349 PCI_EXP_LNKSTA_NLW_SHIFT; 350 if (pcie->init_link_width > current_link_width) { 351 dev_warn(pci->dev, "PCIe link is bad, width reduced\n"); 352 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + 353 PCI_EXP_LNKCTL2); 354 val &= ~PCI_EXP_LNKCTL2_TLS; 355 val |= PCI_EXP_LNKCTL2_TLS_2_5GT; 356 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + 357 PCI_EXP_LNKCTL2, val); 358 359 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + 360 PCI_EXP_LNKCTL); 361 val |= PCI_EXP_LNKCTL_RL; 362 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + 363 PCI_EXP_LNKCTL, val); 364 } 365 } 366 } 367 368 static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg) 369 { 370 struct tegra_pcie_dw *pcie = arg; 371 struct dw_pcie *pci = &pcie->pci; 372 struct pcie_port *pp = &pci->pp; 373 u32 val, tmp; 374 u16 val_w; 375 376 val = appl_readl(pcie, APPL_INTR_STATUS_L0); 377 if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) { 378 val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); 379 if (val & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) { 380 appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0); 381 382 /* SBR & Surprise Link Down WAR */ 383 val = appl_readl(pcie, APPL_CAR_RESET_OVRD); 384 val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; 385 appl_writel(pcie, val, APPL_CAR_RESET_OVRD); 386 udelay(1); 387 val = appl_readl(pcie, APPL_CAR_RESET_OVRD); 388 val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; 389 appl_writel(pcie, val, APPL_CAR_RESET_OVRD); 390 391 val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); 392 val |= PORT_LOGIC_SPEED_CHANGE; 393 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); 394 } 395 } 396 397 if (val & APPL_INTR_STATUS_L0_INT_INT) { 398 val = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0); 399 if (val & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) { 400 appl_writel(pcie, 401 APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS, 402 APPL_INTR_STATUS_L1_8_0); 403 apply_bad_link_workaround(pp); 404 } 405 if (val & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) { 406 appl_writel(pcie, 407 APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS, 408 APPL_INTR_STATUS_L1_8_0); 409 410 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + 411 PCI_EXP_LNKSTA); 412 dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w & 413 PCI_EXP_LNKSTA_CLS); 414 } 415 } 416 417 val = appl_readl(pcie, APPL_INTR_STATUS_L0); 418 if (val & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) { 419 val = appl_readl(pcie, APPL_INTR_STATUS_L1_18); 420 tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS); 421 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) { 422 dev_info(pci->dev, "CDM check complete\n"); 423 tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE; 424 } 425 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) { 426 dev_err(pci->dev, "CDM comparison mismatch\n"); 427 tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR; 428 } 429 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) { 430 dev_err(pci->dev, "CDM Logic error\n"); 431 tmp |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR; 432 } 433 dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, tmp); 434 tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR); 435 dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", tmp); 436 } 437 438 return IRQ_HANDLED; 439 } 440 441 static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie) 442 { 443 u32 val; 444 445 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); 446 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); 447 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); 448 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); 449 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); 450 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); 451 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); 452 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); 453 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); 454 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); 455 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); 456 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); 457 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); 458 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); 459 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); 460 appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2); 461 462 val = appl_readl(pcie, APPL_CTRL); 463 val |= APPL_CTRL_LTSSM_EN; 464 appl_writel(pcie, val, APPL_CTRL); 465 } 466 467 static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg) 468 { 469 struct tegra_pcie_dw *pcie = arg; 470 struct dw_pcie *pci = &pcie->pci; 471 u32 val, speed; 472 473 speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) & 474 PCI_EXP_LNKSTA_CLS; 475 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]); 476 477 /* If EP doesn't advertise L1SS, just return */ 478 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub); 479 if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2))) 480 return IRQ_HANDLED; 481 482 /* Check if BME is set to '1' */ 483 val = dw_pcie_readl_dbi(pci, PCI_COMMAND); 484 if (val & PCI_COMMAND_MASTER) { 485 ktime_t timeout; 486 487 /* 110us for both snoop and no-snoop */ 488 val = 110 | (2 << PCI_LTR_SCALE_SHIFT) | LTR_MSG_REQ; 489 val |= (val << LTR_MST_NO_SNOOP_SHIFT); 490 appl_writel(pcie, val, APPL_LTR_MSG_1); 491 492 /* Send LTR upstream */ 493 val = appl_readl(pcie, APPL_LTR_MSG_2); 494 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE; 495 appl_writel(pcie, val, APPL_LTR_MSG_2); 496 497 timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT); 498 for (;;) { 499 val = appl_readl(pcie, APPL_LTR_MSG_2); 500 if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)) 501 break; 502 if (ktime_after(ktime_get(), timeout)) 503 break; 504 usleep_range(1000, 1100); 505 } 506 if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE) 507 dev_err(pcie->dev, "Failed to send LTR message\n"); 508 } 509 510 return IRQ_HANDLED; 511 } 512 513 static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg) 514 { 515 struct tegra_pcie_dw *pcie = arg; 516 struct dw_pcie_ep *ep = &pcie->pci.ep; 517 int spurious = 1; 518 u32 val, tmp; 519 520 val = appl_readl(pcie, APPL_INTR_STATUS_L0); 521 if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) { 522 val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); 523 appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0); 524 525 if (val & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE) 526 pex_ep_event_hot_rst_done(pcie); 527 528 if (val & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) { 529 tmp = appl_readl(pcie, APPL_LINK_STATUS); 530 if (tmp & APPL_LINK_STATUS_RDLH_LINK_UP) { 531 dev_dbg(pcie->dev, "Link is up with Host\n"); 532 dw_pcie_ep_linkup(ep); 533 } 534 } 535 536 spurious = 0; 537 } 538 539 if (val & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) { 540 val = appl_readl(pcie, APPL_INTR_STATUS_L1_15); 541 appl_writel(pcie, val, APPL_INTR_STATUS_L1_15); 542 543 if (val & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED) 544 return IRQ_WAKE_THREAD; 545 546 spurious = 0; 547 } 548 549 if (spurious) { 550 dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n", 551 val); 552 appl_writel(pcie, val, APPL_INTR_STATUS_L0); 553 } 554 555 return IRQ_HANDLED; 556 } 557 558 static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where, 559 int size, u32 *val) 560 { 561 /* 562 * This is an endpoint mode specific register happen to appear even 563 * when controller is operating in root port mode and system hangs 564 * when it is accessed with link being in ASPM-L1 state. 565 * So skip accessing it altogether 566 */ 567 if (!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) { 568 *val = 0x00000000; 569 return PCIBIOS_SUCCESSFUL; 570 } 571 572 return pci_generic_config_read(bus, devfn, where, size, val); 573 } 574 575 static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where, 576 int size, u32 val) 577 { 578 /* 579 * This is an endpoint mode specific register happen to appear even 580 * when controller is operating in root port mode and system hangs 581 * when it is accessed with link being in ASPM-L1 state. 582 * So skip accessing it altogether 583 */ 584 if (!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) 585 return PCIBIOS_SUCCESSFUL; 586 587 return pci_generic_config_write(bus, devfn, where, size, val); 588 } 589 590 static struct pci_ops tegra_pci_ops = { 591 .map_bus = dw_pcie_own_conf_map_bus, 592 .read = tegra_pcie_dw_rd_own_conf, 593 .write = tegra_pcie_dw_wr_own_conf, 594 }; 595 596 #if defined(CONFIG_PCIEASPM) 597 static void disable_aspm_l11(struct tegra_pcie_dw *pcie) 598 { 599 u32 val; 600 601 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); 602 val &= ~PCI_L1SS_CAP_ASPM_L1_1; 603 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); 604 } 605 606 static void disable_aspm_l12(struct tegra_pcie_dw *pcie) 607 { 608 u32 val; 609 610 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); 611 val &= ~PCI_L1SS_CAP_ASPM_L1_2; 612 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); 613 } 614 615 static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event) 616 { 617 u32 val; 618 619 val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid]); 620 val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT); 621 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; 622 val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT; 623 val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; 624 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val); 625 val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_data_offset[pcie->cid]); 626 627 return val; 628 } 629 630 static int aspm_state_cnt(struct seq_file *s, void *data) 631 { 632 struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *) 633 dev_get_drvdata(s->private); 634 u32 val; 635 636 seq_printf(s, "Tx L0s entry count : %u\n", 637 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S)); 638 639 seq_printf(s, "Rx L0s entry count : %u\n", 640 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S)); 641 642 seq_printf(s, "Link L1 entry count : %u\n", 643 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1)); 644 645 seq_printf(s, "Link L1.1 entry count : %u\n", 646 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1)); 647 648 seq_printf(s, "Link L1.2 entry count : %u\n", 649 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2)); 650 651 /* Clear all counters */ 652 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], 653 EVENT_COUNTER_ALL_CLEAR); 654 655 /* Re-enable counting */ 656 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; 657 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; 658 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val); 659 660 return 0; 661 } 662 663 static void init_host_aspm(struct tegra_pcie_dw *pcie) 664 { 665 struct dw_pcie *pci = &pcie->pci; 666 u32 val; 667 668 val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS); 669 pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP; 670 671 /* Enable ASPM counters */ 672 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; 673 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; 674 dw_pcie_writel_dbi(pci, event_cntr_ctrl_offset[pcie->cid], val); 675 676 /* Program T_cmrt and T_pwr_on values */ 677 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub); 678 val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE); 679 val |= (pcie->aspm_cmrt << 8); 680 val |= (pcie->aspm_pwr_on_t << 19); 681 dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val); 682 683 /* Program L0s and L1 entrance latencies */ 684 val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR); 685 val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK; 686 val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT); 687 val |= PORT_AFR_ENTER_ASPM; 688 dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val); 689 } 690 691 static int init_debugfs(struct tegra_pcie_dw *pcie) 692 { 693 struct dentry *d; 694 695 d = debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", 696 pcie->debugfs, aspm_state_cnt); 697 if (IS_ERR_OR_NULL(d)) 698 dev_err(pcie->dev, 699 "Failed to create debugfs file \"aspm_state_cnt\"\n"); 700 701 return 0; 702 } 703 #else 704 static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; } 705 static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; } 706 static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; } 707 static inline int init_debugfs(struct tegra_pcie_dw *pcie) { return 0; } 708 #endif 709 710 static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp) 711 { 712 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 713 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 714 u32 val; 715 u16 val_w; 716 717 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 718 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN; 719 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 720 721 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); 722 val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN; 723 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); 724 725 if (pcie->enable_cdm_check) { 726 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 727 val |= APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN; 728 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 729 730 val = appl_readl(pcie, APPL_INTR_EN_L1_18); 731 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR; 732 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR; 733 appl_writel(pcie, val, APPL_INTR_EN_L1_18); 734 } 735 736 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + 737 PCI_EXP_LNKSTA); 738 pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >> 739 PCI_EXP_LNKSTA_NLW_SHIFT; 740 741 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + 742 PCI_EXP_LNKCTL); 743 val_w |= PCI_EXP_LNKCTL_LBMIE; 744 dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL, 745 val_w); 746 } 747 748 static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp) 749 { 750 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 751 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 752 u32 val; 753 754 /* Enable legacy interrupt generation */ 755 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 756 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN; 757 val |= APPL_INTR_EN_L0_0_INT_INT_EN; 758 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 759 760 val = appl_readl(pcie, APPL_INTR_EN_L1_8_0); 761 val |= APPL_INTR_EN_L1_8_INTX_EN; 762 val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN; 763 val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN; 764 if (IS_ENABLED(CONFIG_PCIEAER)) 765 val |= APPL_INTR_EN_L1_8_AER_INT_EN; 766 appl_writel(pcie, val, APPL_INTR_EN_L1_8_0); 767 } 768 769 static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp) 770 { 771 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 772 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 773 u32 val; 774 775 dw_pcie_msi_init(pp); 776 777 /* Enable MSI interrupt generation */ 778 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 779 val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN; 780 val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN; 781 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 782 } 783 784 static void tegra_pcie_enable_interrupts(struct pcie_port *pp) 785 { 786 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 787 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 788 789 /* Clear interrupt statuses before enabling interrupts */ 790 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); 791 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); 792 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); 793 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); 794 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); 795 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); 796 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); 797 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); 798 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); 799 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); 800 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); 801 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); 802 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); 803 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); 804 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); 805 806 tegra_pcie_enable_system_interrupts(pp); 807 tegra_pcie_enable_legacy_interrupts(pp); 808 if (IS_ENABLED(CONFIG_PCI_MSI)) 809 tegra_pcie_enable_msi_interrupts(pp); 810 } 811 812 static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie) 813 { 814 struct dw_pcie *pci = &pcie->pci; 815 u32 val, offset, i; 816 817 /* Program init preset */ 818 for (i = 0; i < pcie->num_lanes; i++) { 819 dw_pcie_read(pci->dbi_base + CAP_SPCIE_CAP_OFF 820 + (i * 2), 2, &val); 821 val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK; 822 val |= GEN3_GEN4_EQ_PRESET_INIT; 823 val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK; 824 val |= (GEN3_GEN4_EQ_PRESET_INIT << 825 CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT); 826 dw_pcie_write(pci->dbi_base + CAP_SPCIE_CAP_OFF 827 + (i * 2), 2, val); 828 829 offset = dw_pcie_find_ext_capability(pci, 830 PCI_EXT_CAP_ID_PL_16GT) + 831 PCI_PL_16GT_LE_CTRL; 832 dw_pcie_read(pci->dbi_base + offset + i, 1, &val); 833 val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK; 834 val |= GEN3_GEN4_EQ_PRESET_INIT; 835 val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK; 836 val |= (GEN3_GEN4_EQ_PRESET_INIT << 837 PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT); 838 dw_pcie_write(pci->dbi_base + offset + i, 1, val); 839 } 840 841 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 842 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; 843 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 844 845 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); 846 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; 847 val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); 848 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; 849 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); 850 851 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 852 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; 853 val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT); 854 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 855 856 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); 857 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; 858 val |= (0x360 << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); 859 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; 860 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); 861 862 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 863 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; 864 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 865 } 866 867 static void tegra_pcie_prepare_host(struct pcie_port *pp) 868 { 869 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 870 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 871 u32 val; 872 873 val = dw_pcie_readl_dbi(pci, PCI_IO_BASE); 874 val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8); 875 dw_pcie_writel_dbi(pci, PCI_IO_BASE, val); 876 877 val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE); 878 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE; 879 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE; 880 dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val); 881 882 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); 883 884 /* Configure FTS */ 885 val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR); 886 val &= ~PORT_AFR_N_FTS_MASK; 887 val |= PORT_AFR_N_FTS(N_FTS_VAL); 888 dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val); 889 890 val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); 891 val &= ~PORT_LOGIC_N_FTS_MASK; 892 val |= FTS_VAL; 893 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); 894 895 /* Enable as 0xFFFF0001 response for CRS */ 896 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT); 897 val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT); 898 val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 << 899 AMBA_ERROR_RESPONSE_CRS_SHIFT); 900 dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val); 901 902 /* Configure Max lane width from DT */ 903 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP); 904 val &= ~PCI_EXP_LNKCAP_MLW; 905 val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT); 906 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val); 907 908 config_gen3_gen4_eq_presets(pcie); 909 910 init_host_aspm(pcie); 911 912 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 913 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; 914 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 915 916 if (pcie->update_fc_fixup) { 917 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); 918 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT; 919 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val); 920 } 921 922 dw_pcie_setup_rc(pp); 923 924 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); 925 926 /* Assert RST */ 927 val = appl_readl(pcie, APPL_PINMUX); 928 val &= ~APPL_PINMUX_PEX_RST; 929 appl_writel(pcie, val, APPL_PINMUX); 930 931 usleep_range(100, 200); 932 933 /* Enable LTSSM */ 934 val = appl_readl(pcie, APPL_CTRL); 935 val |= APPL_CTRL_LTSSM_EN; 936 appl_writel(pcie, val, APPL_CTRL); 937 938 /* De-assert RST */ 939 val = appl_readl(pcie, APPL_PINMUX); 940 val |= APPL_PINMUX_PEX_RST; 941 appl_writel(pcie, val, APPL_PINMUX); 942 943 msleep(100); 944 } 945 946 static int tegra_pcie_dw_host_init(struct pcie_port *pp) 947 { 948 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 949 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 950 u32 val, tmp, offset, speed; 951 952 pp->bridge->ops = &tegra_pci_ops; 953 954 tegra_pcie_prepare_host(pp); 955 956 if (dw_pcie_wait_for_link(pci)) { 957 /* 958 * There are some endpoints which can't get the link up if 959 * root port has Data Link Feature (DLF) enabled. 960 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info 961 * on Scaled Flow Control and DLF. 962 * So, need to confirm that is indeed the case here and attempt 963 * link up once again with DLF disabled. 964 */ 965 val = appl_readl(pcie, APPL_DEBUG); 966 val &= APPL_DEBUG_LTSSM_STATE_MASK; 967 val >>= APPL_DEBUG_LTSSM_STATE_SHIFT; 968 tmp = appl_readl(pcie, APPL_LINK_STATUS); 969 tmp &= APPL_LINK_STATUS_RDLH_LINK_UP; 970 if (!(val == 0x11 && !tmp)) { 971 /* Link is down for all good reasons */ 972 return 0; 973 } 974 975 dev_info(pci->dev, "Link is down in DLL"); 976 dev_info(pci->dev, "Trying again with DLFE disabled\n"); 977 /* Disable LTSSM */ 978 val = appl_readl(pcie, APPL_CTRL); 979 val &= ~APPL_CTRL_LTSSM_EN; 980 appl_writel(pcie, val, APPL_CTRL); 981 982 reset_control_assert(pcie->core_rst); 983 reset_control_deassert(pcie->core_rst); 984 985 offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF); 986 val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP); 987 val &= ~PCI_DLF_EXCHANGE_ENABLE; 988 dw_pcie_writel_dbi(pci, offset, val); 989 990 tegra_pcie_prepare_host(pp); 991 992 if (dw_pcie_wait_for_link(pci)) 993 return 0; 994 } 995 996 speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) & 997 PCI_EXP_LNKSTA_CLS; 998 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]); 999 1000 tegra_pcie_enable_interrupts(pp); 1001 1002 return 0; 1003 } 1004 1005 static int tegra_pcie_dw_link_up(struct dw_pcie *pci) 1006 { 1007 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 1008 u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); 1009 1010 return !!(val & PCI_EXP_LNKSTA_DLLLA); 1011 } 1012 1013 static void tegra_pcie_set_msi_vec_num(struct pcie_port *pp) 1014 { 1015 pp->num_vectors = MAX_MSI_IRQS; 1016 } 1017 1018 static int tegra_pcie_dw_start_link(struct dw_pcie *pci) 1019 { 1020 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 1021 1022 enable_irq(pcie->pex_rst_irq); 1023 1024 return 0; 1025 } 1026 1027 static void tegra_pcie_dw_stop_link(struct dw_pcie *pci) 1028 { 1029 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 1030 1031 disable_irq(pcie->pex_rst_irq); 1032 } 1033 1034 static const struct dw_pcie_ops tegra_dw_pcie_ops = { 1035 .link_up = tegra_pcie_dw_link_up, 1036 .start_link = tegra_pcie_dw_start_link, 1037 .stop_link = tegra_pcie_dw_stop_link, 1038 }; 1039 1040 static struct dw_pcie_host_ops tegra_pcie_dw_host_ops = { 1041 .host_init = tegra_pcie_dw_host_init, 1042 .set_num_vectors = tegra_pcie_set_msi_vec_num, 1043 }; 1044 1045 static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie) 1046 { 1047 unsigned int phy_count = pcie->phy_count; 1048 1049 while (phy_count--) { 1050 phy_power_off(pcie->phys[phy_count]); 1051 phy_exit(pcie->phys[phy_count]); 1052 } 1053 } 1054 1055 static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie) 1056 { 1057 unsigned int i; 1058 int ret; 1059 1060 for (i = 0; i < pcie->phy_count; i++) { 1061 ret = phy_init(pcie->phys[i]); 1062 if (ret < 0) 1063 goto phy_power_off; 1064 1065 ret = phy_power_on(pcie->phys[i]); 1066 if (ret < 0) 1067 goto phy_exit; 1068 } 1069 1070 return 0; 1071 1072 phy_power_off: 1073 while (i--) { 1074 phy_power_off(pcie->phys[i]); 1075 phy_exit: 1076 phy_exit(pcie->phys[i]); 1077 } 1078 1079 return ret; 1080 } 1081 1082 static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie) 1083 { 1084 struct device_node *np = pcie->dev->of_node; 1085 int ret; 1086 1087 ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt); 1088 if (ret < 0) { 1089 dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret); 1090 return ret; 1091 } 1092 1093 ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us", 1094 &pcie->aspm_pwr_on_t); 1095 if (ret < 0) 1096 dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n", 1097 ret); 1098 1099 ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us", 1100 &pcie->aspm_l0s_enter_lat); 1101 if (ret < 0) 1102 dev_info(pcie->dev, 1103 "Failed to read ASPM L0s Entrance latency: %d\n", ret); 1104 1105 ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes); 1106 if (ret < 0) { 1107 dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret); 1108 return ret; 1109 } 1110 1111 ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid); 1112 if (ret) { 1113 dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret); 1114 return ret; 1115 } 1116 1117 ret = of_property_count_strings(np, "phy-names"); 1118 if (ret < 0) { 1119 dev_err(pcie->dev, "Failed to find PHY entries: %d\n", 1120 ret); 1121 return ret; 1122 } 1123 pcie->phy_count = ret; 1124 1125 if (of_property_read_bool(np, "nvidia,update-fc-fixup")) 1126 pcie->update_fc_fixup = true; 1127 1128 pcie->supports_clkreq = 1129 of_property_read_bool(pcie->dev->of_node, "supports-clkreq"); 1130 1131 pcie->enable_cdm_check = 1132 of_property_read_bool(np, "snps,enable-cdm-check"); 1133 1134 if (pcie->mode == DW_PCIE_RC_TYPE) 1135 return 0; 1136 1137 /* Endpoint mode specific DT entries */ 1138 pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN); 1139 if (IS_ERR(pcie->pex_rst_gpiod)) { 1140 int err = PTR_ERR(pcie->pex_rst_gpiod); 1141 const char *level = KERN_ERR; 1142 1143 if (err == -EPROBE_DEFER) 1144 level = KERN_DEBUG; 1145 1146 dev_printk(level, pcie->dev, 1147 dev_fmt("Failed to get PERST GPIO: %d\n"), 1148 err); 1149 return err; 1150 } 1151 1152 pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev, 1153 "nvidia,refclk-select", 1154 GPIOD_OUT_HIGH); 1155 if (IS_ERR(pcie->pex_refclk_sel_gpiod)) { 1156 int err = PTR_ERR(pcie->pex_refclk_sel_gpiod); 1157 const char *level = KERN_ERR; 1158 1159 if (err == -EPROBE_DEFER) 1160 level = KERN_DEBUG; 1161 1162 dev_printk(level, pcie->dev, 1163 dev_fmt("Failed to get REFCLK select GPIOs: %d\n"), 1164 err); 1165 pcie->pex_refclk_sel_gpiod = NULL; 1166 } 1167 1168 return 0; 1169 } 1170 1171 static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie, 1172 bool enable) 1173 { 1174 struct mrq_uphy_response resp; 1175 struct tegra_bpmp_message msg; 1176 struct mrq_uphy_request req; 1177 1178 /* Controller-5 doesn't need to have its state set by BPMP-FW */ 1179 if (pcie->cid == 5) 1180 return 0; 1181 1182 memset(&req, 0, sizeof(req)); 1183 memset(&resp, 0, sizeof(resp)); 1184 1185 req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE; 1186 req.controller_state.pcie_controller = pcie->cid; 1187 req.controller_state.enable = enable; 1188 1189 memset(&msg, 0, sizeof(msg)); 1190 msg.mrq = MRQ_UPHY; 1191 msg.tx.data = &req; 1192 msg.tx.size = sizeof(req); 1193 msg.rx.data = &resp; 1194 msg.rx.size = sizeof(resp); 1195 1196 return tegra_bpmp_transfer(pcie->bpmp, &msg); 1197 } 1198 1199 static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie, 1200 bool enable) 1201 { 1202 struct mrq_uphy_response resp; 1203 struct tegra_bpmp_message msg; 1204 struct mrq_uphy_request req; 1205 1206 memset(&req, 0, sizeof(req)); 1207 memset(&resp, 0, sizeof(resp)); 1208 1209 if (enable) { 1210 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT; 1211 req.ep_ctrlr_pll_init.ep_controller = pcie->cid; 1212 } else { 1213 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF; 1214 req.ep_ctrlr_pll_off.ep_controller = pcie->cid; 1215 } 1216 1217 memset(&msg, 0, sizeof(msg)); 1218 msg.mrq = MRQ_UPHY; 1219 msg.tx.data = &req; 1220 msg.tx.size = sizeof(req); 1221 msg.rx.data = &resp; 1222 msg.rx.size = sizeof(resp); 1223 1224 return tegra_bpmp_transfer(pcie->bpmp, &msg); 1225 } 1226 1227 static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie) 1228 { 1229 struct pcie_port *pp = &pcie->pci.pp; 1230 struct pci_bus *child, *root_bus = NULL; 1231 struct pci_dev *pdev; 1232 1233 /* 1234 * link doesn't go into L2 state with some of the endpoints with Tegra 1235 * if they are not in D0 state. So, need to make sure that immediate 1236 * downstream devices are in D0 state before sending PME_TurnOff to put 1237 * link into L2 state. 1238 * This is as per PCI Express Base r4.0 v1.0 September 27-2017, 1239 * 5.2 Link State Power Management (Page #428). 1240 */ 1241 1242 list_for_each_entry(child, &pp->bridge->bus->children, node) { 1243 /* Bring downstream devices to D0 if they are not already in */ 1244 if (child->parent == pp->bridge->bus) { 1245 root_bus = child; 1246 break; 1247 } 1248 } 1249 1250 if (!root_bus) { 1251 dev_err(pcie->dev, "Failed to find downstream devices\n"); 1252 return; 1253 } 1254 1255 list_for_each_entry(pdev, &root_bus->devices, bus_list) { 1256 if (PCI_SLOT(pdev->devfn) == 0) { 1257 if (pci_set_power_state(pdev, PCI_D0)) 1258 dev_err(pcie->dev, 1259 "Failed to transition %s to D0 state\n", 1260 dev_name(&pdev->dev)); 1261 } 1262 } 1263 } 1264 1265 static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie) 1266 { 1267 pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3"); 1268 if (IS_ERR(pcie->slot_ctl_3v3)) { 1269 if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV) 1270 return PTR_ERR(pcie->slot_ctl_3v3); 1271 1272 pcie->slot_ctl_3v3 = NULL; 1273 } 1274 1275 pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v"); 1276 if (IS_ERR(pcie->slot_ctl_12v)) { 1277 if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV) 1278 return PTR_ERR(pcie->slot_ctl_12v); 1279 1280 pcie->slot_ctl_12v = NULL; 1281 } 1282 1283 return 0; 1284 } 1285 1286 static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie) 1287 { 1288 int ret; 1289 1290 if (pcie->slot_ctl_3v3) { 1291 ret = regulator_enable(pcie->slot_ctl_3v3); 1292 if (ret < 0) { 1293 dev_err(pcie->dev, 1294 "Failed to enable 3.3V slot supply: %d\n", ret); 1295 return ret; 1296 } 1297 } 1298 1299 if (pcie->slot_ctl_12v) { 1300 ret = regulator_enable(pcie->slot_ctl_12v); 1301 if (ret < 0) { 1302 dev_err(pcie->dev, 1303 "Failed to enable 12V slot supply: %d\n", ret); 1304 goto fail_12v_enable; 1305 } 1306 } 1307 1308 /* 1309 * According to PCI Express Card Electromechanical Specification 1310 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive) 1311 * should be a minimum of 100ms. 1312 */ 1313 if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v) 1314 msleep(100); 1315 1316 return 0; 1317 1318 fail_12v_enable: 1319 if (pcie->slot_ctl_3v3) 1320 regulator_disable(pcie->slot_ctl_3v3); 1321 return ret; 1322 } 1323 1324 static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie) 1325 { 1326 if (pcie->slot_ctl_12v) 1327 regulator_disable(pcie->slot_ctl_12v); 1328 if (pcie->slot_ctl_3v3) 1329 regulator_disable(pcie->slot_ctl_3v3); 1330 } 1331 1332 static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie, 1333 bool en_hw_hot_rst) 1334 { 1335 int ret; 1336 u32 val; 1337 1338 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true); 1339 if (ret) { 1340 dev_err(pcie->dev, 1341 "Failed to enable controller %u: %d\n", pcie->cid, ret); 1342 return ret; 1343 } 1344 1345 ret = tegra_pcie_enable_slot_regulators(pcie); 1346 if (ret < 0) 1347 goto fail_slot_reg_en; 1348 1349 ret = regulator_enable(pcie->pex_ctl_supply); 1350 if (ret < 0) { 1351 dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret); 1352 goto fail_reg_en; 1353 } 1354 1355 ret = clk_prepare_enable(pcie->core_clk); 1356 if (ret) { 1357 dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret); 1358 goto fail_core_clk; 1359 } 1360 1361 ret = reset_control_deassert(pcie->core_apb_rst); 1362 if (ret) { 1363 dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n", 1364 ret); 1365 goto fail_core_apb_rst; 1366 } 1367 1368 if (en_hw_hot_rst) { 1369 /* Enable HW_HOT_RST mode */ 1370 val = appl_readl(pcie, APPL_CTRL); 1371 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << 1372 APPL_CTRL_HW_HOT_RST_MODE_SHIFT); 1373 val |= APPL_CTRL_HW_HOT_RST_EN; 1374 appl_writel(pcie, val, APPL_CTRL); 1375 } 1376 1377 ret = tegra_pcie_enable_phy(pcie); 1378 if (ret) { 1379 dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret); 1380 goto fail_phy; 1381 } 1382 1383 /* Update CFG base address */ 1384 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK, 1385 APPL_CFG_BASE_ADDR); 1386 1387 /* Configure this core for RP mode operation */ 1388 appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE); 1389 1390 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE); 1391 1392 val = appl_readl(pcie, APPL_CTRL); 1393 appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL); 1394 1395 val = appl_readl(pcie, APPL_CFG_MISC); 1396 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT); 1397 appl_writel(pcie, val, APPL_CFG_MISC); 1398 1399 if (!pcie->supports_clkreq) { 1400 val = appl_readl(pcie, APPL_PINMUX); 1401 val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN; 1402 val &= ~APPL_PINMUX_CLKREQ_OVERRIDE; 1403 appl_writel(pcie, val, APPL_PINMUX); 1404 } 1405 1406 /* Update iATU_DMA base address */ 1407 appl_writel(pcie, 1408 pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK, 1409 APPL_CFG_IATU_DMA_BASE_ADDR); 1410 1411 reset_control_deassert(pcie->core_rst); 1412 1413 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, 1414 PCI_CAP_ID_EXP); 1415 1416 /* Disable ASPM-L1SS advertisement as there is no CLKREQ routing */ 1417 if (!pcie->supports_clkreq) { 1418 disable_aspm_l11(pcie); 1419 disable_aspm_l12(pcie); 1420 } 1421 1422 return ret; 1423 1424 fail_phy: 1425 reset_control_assert(pcie->core_apb_rst); 1426 fail_core_apb_rst: 1427 clk_disable_unprepare(pcie->core_clk); 1428 fail_core_clk: 1429 regulator_disable(pcie->pex_ctl_supply); 1430 fail_reg_en: 1431 tegra_pcie_disable_slot_regulators(pcie); 1432 fail_slot_reg_en: 1433 tegra_pcie_bpmp_set_ctrl_state(pcie, false); 1434 1435 return ret; 1436 } 1437 1438 static int __deinit_controller(struct tegra_pcie_dw *pcie) 1439 { 1440 int ret; 1441 1442 ret = reset_control_assert(pcie->core_rst); 1443 if (ret) { 1444 dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", 1445 ret); 1446 return ret; 1447 } 1448 1449 tegra_pcie_disable_phy(pcie); 1450 1451 ret = reset_control_assert(pcie->core_apb_rst); 1452 if (ret) { 1453 dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret); 1454 return ret; 1455 } 1456 1457 clk_disable_unprepare(pcie->core_clk); 1458 1459 ret = regulator_disable(pcie->pex_ctl_supply); 1460 if (ret) { 1461 dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret); 1462 return ret; 1463 } 1464 1465 tegra_pcie_disable_slot_regulators(pcie); 1466 1467 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false); 1468 if (ret) { 1469 dev_err(pcie->dev, "Failed to disable controller %d: %d\n", 1470 pcie->cid, ret); 1471 return ret; 1472 } 1473 1474 return ret; 1475 } 1476 1477 static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie) 1478 { 1479 struct dw_pcie *pci = &pcie->pci; 1480 struct pcie_port *pp = &pci->pp; 1481 int ret; 1482 1483 ret = tegra_pcie_config_controller(pcie, false); 1484 if (ret < 0) 1485 return ret; 1486 1487 pp->ops = &tegra_pcie_dw_host_ops; 1488 1489 ret = dw_pcie_host_init(pp); 1490 if (ret < 0) { 1491 dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret); 1492 goto fail_host_init; 1493 } 1494 1495 return 0; 1496 1497 fail_host_init: 1498 return __deinit_controller(pcie); 1499 } 1500 1501 static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie) 1502 { 1503 u32 val; 1504 1505 if (!tegra_pcie_dw_link_up(&pcie->pci)) 1506 return 0; 1507 1508 val = appl_readl(pcie, APPL_RADM_STATUS); 1509 val |= APPL_PM_XMT_TURNOFF_STATE; 1510 appl_writel(pcie, val, APPL_RADM_STATUS); 1511 1512 return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val, 1513 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT, 1514 1, PME_ACK_TIMEOUT); 1515 } 1516 1517 static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) 1518 { 1519 u32 data; 1520 int err; 1521 1522 if (!tegra_pcie_dw_link_up(&pcie->pci)) { 1523 dev_dbg(pcie->dev, "PCIe link is not up...!\n"); 1524 return; 1525 } 1526 1527 if (tegra_pcie_try_link_l2(pcie)) { 1528 dev_info(pcie->dev, "Link didn't transition to L2 state\n"); 1529 /* 1530 * TX lane clock freq will reset to Gen1 only if link is in L2 1531 * or detect state. 1532 * So apply pex_rst to end point to force RP to go into detect 1533 * state 1534 */ 1535 data = appl_readl(pcie, APPL_PINMUX); 1536 data &= ~APPL_PINMUX_PEX_RST; 1537 appl_writel(pcie, data, APPL_PINMUX); 1538 1539 err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, 1540 data, 1541 ((data & 1542 APPL_DEBUG_LTSSM_STATE_MASK) >> 1543 APPL_DEBUG_LTSSM_STATE_SHIFT) == 1544 LTSSM_STATE_PRE_DETECT, 1545 1, LTSSM_TIMEOUT); 1546 if (err) { 1547 dev_info(pcie->dev, "Link didn't go to detect state\n"); 1548 } else { 1549 /* Disable LTSSM after link is in detect state */ 1550 data = appl_readl(pcie, APPL_CTRL); 1551 data &= ~APPL_CTRL_LTSSM_EN; 1552 appl_writel(pcie, data, APPL_CTRL); 1553 } 1554 } 1555 /* 1556 * DBI registers may not be accessible after this as PLL-E would be 1557 * down depending on how CLKREQ is pulled by end point 1558 */ 1559 data = appl_readl(pcie, APPL_PINMUX); 1560 data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE); 1561 /* Cut REFCLK to slot */ 1562 data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; 1563 data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; 1564 appl_writel(pcie, data, APPL_PINMUX); 1565 } 1566 1567 static int tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie) 1568 { 1569 tegra_pcie_downstream_dev_to_D0(pcie); 1570 dw_pcie_host_deinit(&pcie->pci.pp); 1571 tegra_pcie_dw_pme_turnoff(pcie); 1572 1573 return __deinit_controller(pcie); 1574 } 1575 1576 static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie) 1577 { 1578 struct pcie_port *pp = &pcie->pci.pp; 1579 struct device *dev = pcie->dev; 1580 char *name; 1581 int ret; 1582 1583 if (IS_ENABLED(CONFIG_PCI_MSI)) { 1584 pp->msi_irq = of_irq_get_byname(dev->of_node, "msi"); 1585 if (!pp->msi_irq) { 1586 dev_err(dev, "Failed to get MSI interrupt\n"); 1587 return -ENODEV; 1588 } 1589 } 1590 1591 pm_runtime_enable(dev); 1592 1593 ret = pm_runtime_get_sync(dev); 1594 if (ret < 0) { 1595 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", 1596 ret); 1597 goto fail_pm_get_sync; 1598 } 1599 1600 ret = pinctrl_pm_select_default_state(dev); 1601 if (ret < 0) { 1602 dev_err(dev, "Failed to configure sideband pins: %d\n", ret); 1603 goto fail_pm_get_sync; 1604 } 1605 1606 tegra_pcie_init_controller(pcie); 1607 1608 pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci); 1609 if (!pcie->link_state) { 1610 ret = -ENOMEDIUM; 1611 goto fail_host_init; 1612 } 1613 1614 name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node); 1615 if (!name) { 1616 ret = -ENOMEM; 1617 goto fail_host_init; 1618 } 1619 1620 pcie->debugfs = debugfs_create_dir(name, NULL); 1621 if (!pcie->debugfs) 1622 dev_err(dev, "Failed to create debugfs\n"); 1623 else 1624 init_debugfs(pcie); 1625 1626 return ret; 1627 1628 fail_host_init: 1629 tegra_pcie_deinit_controller(pcie); 1630 fail_pm_get_sync: 1631 pm_runtime_put_sync(dev); 1632 pm_runtime_disable(dev); 1633 return ret; 1634 } 1635 1636 static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie) 1637 { 1638 u32 val; 1639 int ret; 1640 1641 if (pcie->ep_state == EP_STATE_DISABLED) 1642 return; 1643 1644 /* Disable LTSSM */ 1645 val = appl_readl(pcie, APPL_CTRL); 1646 val &= ~APPL_CTRL_LTSSM_EN; 1647 appl_writel(pcie, val, APPL_CTRL); 1648 1649 ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val, 1650 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >> 1651 APPL_DEBUG_LTSSM_STATE_SHIFT) == 1652 LTSSM_STATE_PRE_DETECT, 1653 1, LTSSM_TIMEOUT); 1654 if (ret) 1655 dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret); 1656 1657 reset_control_assert(pcie->core_rst); 1658 1659 tegra_pcie_disable_phy(pcie); 1660 1661 reset_control_assert(pcie->core_apb_rst); 1662 1663 clk_disable_unprepare(pcie->core_clk); 1664 1665 pm_runtime_put_sync(pcie->dev); 1666 1667 ret = tegra_pcie_bpmp_set_pll_state(pcie, false); 1668 if (ret) 1669 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret); 1670 1671 pcie->ep_state = EP_STATE_DISABLED; 1672 dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n"); 1673 } 1674 1675 static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) 1676 { 1677 struct dw_pcie *pci = &pcie->pci; 1678 struct dw_pcie_ep *ep = &pci->ep; 1679 struct device *dev = pcie->dev; 1680 u32 val; 1681 int ret; 1682 1683 if (pcie->ep_state == EP_STATE_ENABLED) 1684 return; 1685 1686 ret = pm_runtime_get_sync(dev); 1687 if (ret < 0) { 1688 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", 1689 ret); 1690 return; 1691 } 1692 1693 ret = tegra_pcie_bpmp_set_pll_state(pcie, true); 1694 if (ret) { 1695 dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n", ret); 1696 goto fail_pll_init; 1697 } 1698 1699 ret = clk_prepare_enable(pcie->core_clk); 1700 if (ret) { 1701 dev_err(dev, "Failed to enable core clock: %d\n", ret); 1702 goto fail_core_clk_enable; 1703 } 1704 1705 ret = reset_control_deassert(pcie->core_apb_rst); 1706 if (ret) { 1707 dev_err(dev, "Failed to deassert core APB reset: %d\n", ret); 1708 goto fail_core_apb_rst; 1709 } 1710 1711 ret = tegra_pcie_enable_phy(pcie); 1712 if (ret) { 1713 dev_err(dev, "Failed to enable PHY: %d\n", ret); 1714 goto fail_phy; 1715 } 1716 1717 /* Clear any stale interrupt statuses */ 1718 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); 1719 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); 1720 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); 1721 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); 1722 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); 1723 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); 1724 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); 1725 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); 1726 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); 1727 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); 1728 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); 1729 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); 1730 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); 1731 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); 1732 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); 1733 1734 /* configure this core for EP mode operation */ 1735 val = appl_readl(pcie, APPL_DM_TYPE); 1736 val &= ~APPL_DM_TYPE_MASK; 1737 val |= APPL_DM_TYPE_EP; 1738 appl_writel(pcie, val, APPL_DM_TYPE); 1739 1740 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE); 1741 1742 val = appl_readl(pcie, APPL_CTRL); 1743 val |= APPL_CTRL_SYS_PRE_DET_STATE; 1744 val |= APPL_CTRL_HW_HOT_RST_EN; 1745 appl_writel(pcie, val, APPL_CTRL); 1746 1747 val = appl_readl(pcie, APPL_CFG_MISC); 1748 val |= APPL_CFG_MISC_SLV_EP_MODE; 1749 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT); 1750 appl_writel(pcie, val, APPL_CFG_MISC); 1751 1752 val = appl_readl(pcie, APPL_PINMUX); 1753 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; 1754 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; 1755 appl_writel(pcie, val, APPL_PINMUX); 1756 1757 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK, 1758 APPL_CFG_BASE_ADDR); 1759 1760 appl_writel(pcie, pcie->atu_dma_res->start & 1761 APPL_CFG_IATU_DMA_BASE_ADDR_MASK, 1762 APPL_CFG_IATU_DMA_BASE_ADDR); 1763 1764 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 1765 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN; 1766 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN; 1767 val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN; 1768 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 1769 1770 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); 1771 val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN; 1772 val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN; 1773 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); 1774 1775 reset_control_deassert(pcie->core_rst); 1776 1777 if (pcie->update_fc_fixup) { 1778 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); 1779 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT; 1780 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val); 1781 } 1782 1783 config_gen3_gen4_eq_presets(pcie); 1784 1785 init_host_aspm(pcie); 1786 1787 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */ 1788 if (!pcie->supports_clkreq) { 1789 disable_aspm_l11(pcie); 1790 disable_aspm_l12(pcie); 1791 } 1792 1793 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 1794 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; 1795 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 1796 1797 /* Configure N_FTS & FTS */ 1798 val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR); 1799 val &= ~PORT_AFR_N_FTS_MASK; 1800 val |= PORT_AFR_N_FTS(FTS_VAL); 1801 dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val); 1802 1803 val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); 1804 val &= ~PORT_LOGIC_N_FTS_MASK; 1805 val |= FTS_VAL; 1806 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); 1807 1808 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, 1809 PCI_CAP_ID_EXP); 1810 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); 1811 1812 val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK); 1813 val |= MSIX_ADDR_MATCH_LOW_OFF_EN; 1814 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val); 1815 val = (lower_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK); 1816 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val); 1817 1818 ret = dw_pcie_ep_init_complete(ep); 1819 if (ret) { 1820 dev_err(dev, "Failed to complete initialization: %d\n", ret); 1821 goto fail_init_complete; 1822 } 1823 1824 dw_pcie_ep_init_notify(ep); 1825 1826 /* Enable LTSSM */ 1827 val = appl_readl(pcie, APPL_CTRL); 1828 val |= APPL_CTRL_LTSSM_EN; 1829 appl_writel(pcie, val, APPL_CTRL); 1830 1831 pcie->ep_state = EP_STATE_ENABLED; 1832 dev_dbg(dev, "Initialization of endpoint is completed\n"); 1833 1834 return; 1835 1836 fail_init_complete: 1837 reset_control_assert(pcie->core_rst); 1838 tegra_pcie_disable_phy(pcie); 1839 fail_phy: 1840 reset_control_assert(pcie->core_apb_rst); 1841 fail_core_apb_rst: 1842 clk_disable_unprepare(pcie->core_clk); 1843 fail_core_clk_enable: 1844 tegra_pcie_bpmp_set_pll_state(pcie, false); 1845 fail_pll_init: 1846 pm_runtime_put_sync(dev); 1847 } 1848 1849 static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg) 1850 { 1851 struct tegra_pcie_dw *pcie = arg; 1852 1853 if (gpiod_get_value(pcie->pex_rst_gpiod)) 1854 pex_ep_event_pex_rst_assert(pcie); 1855 else 1856 pex_ep_event_pex_rst_deassert(pcie); 1857 1858 return IRQ_HANDLED; 1859 } 1860 1861 static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq) 1862 { 1863 /* Tegra194 supports only INTA */ 1864 if (irq > 1) 1865 return -EINVAL; 1866 1867 appl_writel(pcie, 1, APPL_LEGACY_INTX); 1868 usleep_range(1000, 2000); 1869 appl_writel(pcie, 0, APPL_LEGACY_INTX); 1870 return 0; 1871 } 1872 1873 static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq) 1874 { 1875 if (unlikely(irq > 31)) 1876 return -EINVAL; 1877 1878 appl_writel(pcie, (1 << irq), APPL_MSI_CTRL_1); 1879 1880 return 0; 1881 } 1882 1883 static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq) 1884 { 1885 struct dw_pcie_ep *ep = &pcie->pci.ep; 1886 1887 writel(irq, ep->msi_mem); 1888 1889 return 0; 1890 } 1891 1892 static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, 1893 enum pci_epc_irq_type type, 1894 u16 interrupt_num) 1895 { 1896 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1897 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 1898 1899 switch (type) { 1900 case PCI_EPC_IRQ_LEGACY: 1901 return tegra_pcie_ep_raise_legacy_irq(pcie, interrupt_num); 1902 1903 case PCI_EPC_IRQ_MSI: 1904 return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num); 1905 1906 case PCI_EPC_IRQ_MSIX: 1907 return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num); 1908 1909 default: 1910 dev_err(pci->dev, "Unknown IRQ type\n"); 1911 return -EPERM; 1912 } 1913 1914 return 0; 1915 } 1916 1917 static const struct pci_epc_features tegra_pcie_epc_features = { 1918 .linkup_notifier = true, 1919 .core_init_notifier = true, 1920 .msi_capable = false, 1921 .msix_capable = false, 1922 .reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5, 1923 .bar_fixed_64bit = 1 << BAR_0, 1924 .bar_fixed_size[0] = SZ_1M, 1925 }; 1926 1927 static const struct pci_epc_features* 1928 tegra_pcie_ep_get_features(struct dw_pcie_ep *ep) 1929 { 1930 return &tegra_pcie_epc_features; 1931 } 1932 1933 static struct dw_pcie_ep_ops pcie_ep_ops = { 1934 .raise_irq = tegra_pcie_ep_raise_irq, 1935 .get_features = tegra_pcie_ep_get_features, 1936 }; 1937 1938 static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie, 1939 struct platform_device *pdev) 1940 { 1941 struct dw_pcie *pci = &pcie->pci; 1942 struct device *dev = pcie->dev; 1943 struct dw_pcie_ep *ep; 1944 struct resource *res; 1945 char *name; 1946 int ret; 1947 1948 ep = &pci->ep; 1949 ep->ops = &pcie_ep_ops; 1950 1951 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space"); 1952 if (!res) 1953 return -EINVAL; 1954 1955 ep->phys_base = res->start; 1956 ep->addr_size = resource_size(res); 1957 ep->page_size = SZ_64K; 1958 1959 ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME); 1960 if (ret < 0) { 1961 dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n", 1962 ret); 1963 return ret; 1964 } 1965 1966 ret = gpiod_to_irq(pcie->pex_rst_gpiod); 1967 if (ret < 0) { 1968 dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret); 1969 return ret; 1970 } 1971 pcie->pex_rst_irq = (unsigned int)ret; 1972 1973 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq", 1974 pcie->cid); 1975 if (!name) { 1976 dev_err(dev, "Failed to create PERST IRQ string\n"); 1977 return -ENOMEM; 1978 } 1979 1980 irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN); 1981 1982 pcie->ep_state = EP_STATE_DISABLED; 1983 1984 ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL, 1985 tegra_pcie_ep_pex_rst_irq, 1986 IRQF_TRIGGER_RISING | 1987 IRQF_TRIGGER_FALLING | IRQF_ONESHOT, 1988 name, (void *)pcie); 1989 if (ret < 0) { 1990 dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret); 1991 return ret; 1992 } 1993 1994 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_ep_work", 1995 pcie->cid); 1996 if (!name) { 1997 dev_err(dev, "Failed to create PCIe EP work thread string\n"); 1998 return -ENOMEM; 1999 } 2000 2001 pm_runtime_enable(dev); 2002 2003 ret = dw_pcie_ep_init(ep); 2004 if (ret) { 2005 dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n", 2006 ret); 2007 return ret; 2008 } 2009 2010 return 0; 2011 } 2012 2013 static int tegra_pcie_dw_probe(struct platform_device *pdev) 2014 { 2015 const struct tegra_pcie_dw_of_data *data; 2016 struct device *dev = &pdev->dev; 2017 struct resource *atu_dma_res; 2018 struct tegra_pcie_dw *pcie; 2019 struct resource *dbi_res; 2020 struct pcie_port *pp; 2021 struct dw_pcie *pci; 2022 struct phy **phys; 2023 char *name; 2024 int ret; 2025 u32 i; 2026 2027 data = of_device_get_match_data(dev); 2028 2029 pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); 2030 if (!pcie) 2031 return -ENOMEM; 2032 2033 pci = &pcie->pci; 2034 pci->dev = &pdev->dev; 2035 pci->ops = &tegra_dw_pcie_ops; 2036 pp = &pci->pp; 2037 pcie->dev = &pdev->dev; 2038 pcie->mode = (enum dw_pcie_device_mode)data->mode; 2039 2040 ret = tegra_pcie_dw_parse_dt(pcie); 2041 if (ret < 0) { 2042 const char *level = KERN_ERR; 2043 2044 if (ret == -EPROBE_DEFER) 2045 level = KERN_DEBUG; 2046 2047 dev_printk(level, dev, 2048 dev_fmt("Failed to parse device tree: %d\n"), 2049 ret); 2050 return ret; 2051 } 2052 2053 ret = tegra_pcie_get_slot_regulators(pcie); 2054 if (ret < 0) { 2055 const char *level = KERN_ERR; 2056 2057 if (ret == -EPROBE_DEFER) 2058 level = KERN_DEBUG; 2059 2060 dev_printk(level, dev, 2061 dev_fmt("Failed to get slot regulators: %d\n"), 2062 ret); 2063 return ret; 2064 } 2065 2066 if (pcie->pex_refclk_sel_gpiod) 2067 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1); 2068 2069 pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl"); 2070 if (IS_ERR(pcie->pex_ctl_supply)) { 2071 ret = PTR_ERR(pcie->pex_ctl_supply); 2072 if (ret != -EPROBE_DEFER) 2073 dev_err(dev, "Failed to get regulator: %ld\n", 2074 PTR_ERR(pcie->pex_ctl_supply)); 2075 return ret; 2076 } 2077 2078 pcie->core_clk = devm_clk_get(dev, "core"); 2079 if (IS_ERR(pcie->core_clk)) { 2080 dev_err(dev, "Failed to get core clock: %ld\n", 2081 PTR_ERR(pcie->core_clk)); 2082 return PTR_ERR(pcie->core_clk); 2083 } 2084 2085 pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 2086 "appl"); 2087 if (!pcie->appl_res) { 2088 dev_err(dev, "Failed to find \"appl\" region\n"); 2089 return -ENODEV; 2090 } 2091 2092 pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res); 2093 if (IS_ERR(pcie->appl_base)) 2094 return PTR_ERR(pcie->appl_base); 2095 2096 pcie->core_apb_rst = devm_reset_control_get(dev, "apb"); 2097 if (IS_ERR(pcie->core_apb_rst)) { 2098 dev_err(dev, "Failed to get APB reset: %ld\n", 2099 PTR_ERR(pcie->core_apb_rst)); 2100 return PTR_ERR(pcie->core_apb_rst); 2101 } 2102 2103 phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL); 2104 if (!phys) 2105 return -ENOMEM; 2106 2107 for (i = 0; i < pcie->phy_count; i++) { 2108 name = kasprintf(GFP_KERNEL, "p2u-%u", i); 2109 if (!name) { 2110 dev_err(dev, "Failed to create P2U string\n"); 2111 return -ENOMEM; 2112 } 2113 phys[i] = devm_phy_get(dev, name); 2114 kfree(name); 2115 if (IS_ERR(phys[i])) { 2116 ret = PTR_ERR(phys[i]); 2117 if (ret != -EPROBE_DEFER) 2118 dev_err(dev, "Failed to get PHY: %d\n", ret); 2119 return ret; 2120 } 2121 } 2122 2123 pcie->phys = phys; 2124 2125 dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); 2126 if (!dbi_res) { 2127 dev_err(dev, "Failed to find \"dbi\" region\n"); 2128 return -ENODEV; 2129 } 2130 pcie->dbi_res = dbi_res; 2131 2132 pci->dbi_base = devm_ioremap_resource(dev, dbi_res); 2133 if (IS_ERR(pci->dbi_base)) 2134 return PTR_ERR(pci->dbi_base); 2135 2136 /* Tegra HW locates DBI2 at a fixed offset from DBI */ 2137 pci->dbi_base2 = pci->dbi_base + 0x1000; 2138 2139 atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 2140 "atu_dma"); 2141 if (!atu_dma_res) { 2142 dev_err(dev, "Failed to find \"atu_dma\" region\n"); 2143 return -ENODEV; 2144 } 2145 pcie->atu_dma_res = atu_dma_res; 2146 2147 pci->atu_base = devm_ioremap_resource(dev, atu_dma_res); 2148 if (IS_ERR(pci->atu_base)) 2149 return PTR_ERR(pci->atu_base); 2150 2151 pcie->core_rst = devm_reset_control_get(dev, "core"); 2152 if (IS_ERR(pcie->core_rst)) { 2153 dev_err(dev, "Failed to get core reset: %ld\n", 2154 PTR_ERR(pcie->core_rst)); 2155 return PTR_ERR(pcie->core_rst); 2156 } 2157 2158 pp->irq = platform_get_irq_byname(pdev, "intr"); 2159 if (pp->irq < 0) 2160 return pp->irq; 2161 2162 pcie->bpmp = tegra_bpmp_get(dev); 2163 if (IS_ERR(pcie->bpmp)) 2164 return PTR_ERR(pcie->bpmp); 2165 2166 platform_set_drvdata(pdev, pcie); 2167 2168 switch (pcie->mode) { 2169 case DW_PCIE_RC_TYPE: 2170 ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler, 2171 IRQF_SHARED, "tegra-pcie-intr", pcie); 2172 if (ret) { 2173 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq, 2174 ret); 2175 goto fail; 2176 } 2177 2178 ret = tegra_pcie_config_rp(pcie); 2179 if (ret && ret != -ENOMEDIUM) 2180 goto fail; 2181 else 2182 return 0; 2183 break; 2184 2185 case DW_PCIE_EP_TYPE: 2186 ret = devm_request_threaded_irq(dev, pp->irq, 2187 tegra_pcie_ep_hard_irq, 2188 tegra_pcie_ep_irq_thread, 2189 IRQF_SHARED | IRQF_ONESHOT, 2190 "tegra-pcie-ep-intr", pcie); 2191 if (ret) { 2192 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq, 2193 ret); 2194 goto fail; 2195 } 2196 2197 ret = tegra_pcie_config_ep(pcie, pdev); 2198 if (ret < 0) 2199 goto fail; 2200 break; 2201 2202 default: 2203 dev_err(dev, "Invalid PCIe device type %d\n", pcie->mode); 2204 } 2205 2206 fail: 2207 tegra_bpmp_put(pcie->bpmp); 2208 return ret; 2209 } 2210 2211 static int tegra_pcie_dw_remove(struct platform_device *pdev) 2212 { 2213 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); 2214 2215 if (!pcie->link_state) 2216 return 0; 2217 2218 debugfs_remove_recursive(pcie->debugfs); 2219 tegra_pcie_deinit_controller(pcie); 2220 pm_runtime_put_sync(pcie->dev); 2221 pm_runtime_disable(pcie->dev); 2222 tegra_bpmp_put(pcie->bpmp); 2223 if (pcie->pex_refclk_sel_gpiod) 2224 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0); 2225 2226 return 0; 2227 } 2228 2229 static int tegra_pcie_dw_suspend_late(struct device *dev) 2230 { 2231 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2232 u32 val; 2233 2234 if (!pcie->link_state) 2235 return 0; 2236 2237 /* Enable HW_HOT_RST mode */ 2238 val = appl_readl(pcie, APPL_CTRL); 2239 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << 2240 APPL_CTRL_HW_HOT_RST_MODE_SHIFT); 2241 val |= APPL_CTRL_HW_HOT_RST_EN; 2242 appl_writel(pcie, val, APPL_CTRL); 2243 2244 return 0; 2245 } 2246 2247 static int tegra_pcie_dw_suspend_noirq(struct device *dev) 2248 { 2249 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2250 2251 if (!pcie->link_state) 2252 return 0; 2253 2254 /* Save MSI interrupt vector */ 2255 pcie->msi_ctrl_int = dw_pcie_readl_dbi(&pcie->pci, 2256 PORT_LOGIC_MSI_CTRL_INT_0_EN); 2257 tegra_pcie_downstream_dev_to_D0(pcie); 2258 tegra_pcie_dw_pme_turnoff(pcie); 2259 2260 return __deinit_controller(pcie); 2261 } 2262 2263 static int tegra_pcie_dw_resume_noirq(struct device *dev) 2264 { 2265 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2266 int ret; 2267 2268 if (!pcie->link_state) 2269 return 0; 2270 2271 ret = tegra_pcie_config_controller(pcie, true); 2272 if (ret < 0) 2273 return ret; 2274 2275 ret = tegra_pcie_dw_host_init(&pcie->pci.pp); 2276 if (ret < 0) { 2277 dev_err(dev, "Failed to init host: %d\n", ret); 2278 goto fail_host_init; 2279 } 2280 2281 /* Restore MSI interrupt vector */ 2282 dw_pcie_writel_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN, 2283 pcie->msi_ctrl_int); 2284 2285 return 0; 2286 2287 fail_host_init: 2288 return __deinit_controller(pcie); 2289 } 2290 2291 static int tegra_pcie_dw_resume_early(struct device *dev) 2292 { 2293 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2294 u32 val; 2295 2296 if (!pcie->link_state) 2297 return 0; 2298 2299 /* Disable HW_HOT_RST mode */ 2300 val = appl_readl(pcie, APPL_CTRL); 2301 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << 2302 APPL_CTRL_HW_HOT_RST_MODE_SHIFT); 2303 val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST << 2304 APPL_CTRL_HW_HOT_RST_MODE_SHIFT; 2305 val &= ~APPL_CTRL_HW_HOT_RST_EN; 2306 appl_writel(pcie, val, APPL_CTRL); 2307 2308 return 0; 2309 } 2310 2311 static void tegra_pcie_dw_shutdown(struct platform_device *pdev) 2312 { 2313 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); 2314 2315 if (!pcie->link_state) 2316 return; 2317 2318 debugfs_remove_recursive(pcie->debugfs); 2319 tegra_pcie_downstream_dev_to_D0(pcie); 2320 2321 disable_irq(pcie->pci.pp.irq); 2322 if (IS_ENABLED(CONFIG_PCI_MSI)) 2323 disable_irq(pcie->pci.pp.msi_irq); 2324 2325 tegra_pcie_dw_pme_turnoff(pcie); 2326 __deinit_controller(pcie); 2327 } 2328 2329 static const struct tegra_pcie_dw_of_data tegra_pcie_dw_rc_of_data = { 2330 .mode = DW_PCIE_RC_TYPE, 2331 }; 2332 2333 static const struct tegra_pcie_dw_of_data tegra_pcie_dw_ep_of_data = { 2334 .mode = DW_PCIE_EP_TYPE, 2335 }; 2336 2337 static const struct of_device_id tegra_pcie_dw_of_match[] = { 2338 { 2339 .compatible = "nvidia,tegra194-pcie", 2340 .data = &tegra_pcie_dw_rc_of_data, 2341 }, 2342 { 2343 .compatible = "nvidia,tegra194-pcie-ep", 2344 .data = &tegra_pcie_dw_ep_of_data, 2345 }, 2346 {}, 2347 }; 2348 2349 static const struct dev_pm_ops tegra_pcie_dw_pm_ops = { 2350 .suspend_late = tegra_pcie_dw_suspend_late, 2351 .suspend_noirq = tegra_pcie_dw_suspend_noirq, 2352 .resume_noirq = tegra_pcie_dw_resume_noirq, 2353 .resume_early = tegra_pcie_dw_resume_early, 2354 }; 2355 2356 static struct platform_driver tegra_pcie_dw_driver = { 2357 .probe = tegra_pcie_dw_probe, 2358 .remove = tegra_pcie_dw_remove, 2359 .shutdown = tegra_pcie_dw_shutdown, 2360 .driver = { 2361 .name = "tegra194-pcie", 2362 .pm = &tegra_pcie_dw_pm_ops, 2363 .of_match_table = tegra_pcie_dw_of_match, 2364 }, 2365 }; 2366 module_platform_driver(tegra_pcie_dw_driver); 2367 2368 MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match); 2369 2370 MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>"); 2371 MODULE_DESCRIPTION("NVIDIA PCIe host controller driver"); 2372 MODULE_LICENSE("GPL v2"); 2373