1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * PCIe host controller driver for Tegra194 SoC 4 * 5 * Copyright (C) 2019 NVIDIA Corporation. 6 * 7 * Author: Vidya Sagar <vidyas@nvidia.com> 8 */ 9 10 #include <linux/clk.h> 11 #include <linux/debugfs.h> 12 #include <linux/delay.h> 13 #include <linux/gpio.h> 14 #include <linux/gpio/consumer.h> 15 #include <linux/interrupt.h> 16 #include <linux/iopoll.h> 17 #include <linux/kernel.h> 18 #include <linux/module.h> 19 #include <linux/of.h> 20 #include <linux/of_device.h> 21 #include <linux/of_gpio.h> 22 #include <linux/of_irq.h> 23 #include <linux/of_pci.h> 24 #include <linux/pci.h> 25 #include <linux/phy/phy.h> 26 #include <linux/pinctrl/consumer.h> 27 #include <linux/platform_device.h> 28 #include <linux/pm_runtime.h> 29 #include <linux/random.h> 30 #include <linux/reset.h> 31 #include <linux/resource.h> 32 #include <linux/types.h> 33 #include "pcie-designware.h" 34 #include <soc/tegra/bpmp.h> 35 #include <soc/tegra/bpmp-abi.h> 36 #include "../../pci.h" 37 38 #define APPL_PINMUX 0x0 39 #define APPL_PINMUX_PEX_RST BIT(0) 40 #define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2) 41 #define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3) 42 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4) 43 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5) 44 45 #define APPL_CTRL 0x4 46 #define APPL_CTRL_SYS_PRE_DET_STATE BIT(6) 47 #define APPL_CTRL_LTSSM_EN BIT(7) 48 #define APPL_CTRL_HW_HOT_RST_EN BIT(20) 49 #define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0) 50 #define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22 51 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1 52 53 #define APPL_INTR_EN_L0_0 0x8 54 #define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0) 55 #define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4) 56 #define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8) 57 #define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN BIT(15) 58 #define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19) 59 #define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30) 60 #define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31) 61 62 #define APPL_INTR_STATUS_L0 0xC 63 #define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0) 64 #define APPL_INTR_STATUS_L0_INT_INT BIT(8) 65 #define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT BIT(15) 66 #define APPL_INTR_STATUS_L0_PEX_RST_INT BIT(16) 67 #define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18) 68 69 #define APPL_INTR_EN_L1_0_0 0x1C 70 #define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1) 71 #define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN BIT(3) 72 #define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN BIT(30) 73 74 #define APPL_INTR_STATUS_L1_0_0 0x20 75 #define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1) 76 #define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED BIT(3) 77 #define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE BIT(30) 78 79 #define APPL_INTR_STATUS_L1_1 0x2C 80 #define APPL_INTR_STATUS_L1_2 0x30 81 #define APPL_INTR_STATUS_L1_3 0x34 82 #define APPL_INTR_STATUS_L1_6 0x3C 83 #define APPL_INTR_STATUS_L1_7 0x40 84 #define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED BIT(1) 85 86 #define APPL_INTR_EN_L1_8_0 0x44 87 #define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2) 88 #define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3) 89 #define APPL_INTR_EN_L1_8_INTX_EN BIT(11) 90 #define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15) 91 92 #define APPL_INTR_STATUS_L1_8_0 0x4C 93 #define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6) 94 #define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2) 95 #define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3) 96 97 #define APPL_INTR_STATUS_L1_9 0x54 98 #define APPL_INTR_STATUS_L1_10 0x58 99 #define APPL_INTR_STATUS_L1_11 0x64 100 #define APPL_INTR_STATUS_L1_13 0x74 101 #define APPL_INTR_STATUS_L1_14 0x78 102 #define APPL_INTR_STATUS_L1_15 0x7C 103 #define APPL_INTR_STATUS_L1_17 0x88 104 105 #define APPL_INTR_EN_L1_18 0x90 106 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2) 107 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) 108 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) 109 110 #define APPL_INTR_STATUS_L1_18 0x94 111 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2) 112 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) 113 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) 114 115 #define APPL_MSI_CTRL_1 0xAC 116 117 #define APPL_MSI_CTRL_2 0xB0 118 119 #define APPL_LEGACY_INTX 0xB8 120 121 #define APPL_LTR_MSG_1 0xC4 122 #define LTR_MSG_REQ BIT(15) 123 #define LTR_MST_NO_SNOOP_SHIFT 16 124 125 #define APPL_LTR_MSG_2 0xC8 126 #define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3) 127 128 #define APPL_LINK_STATUS 0xCC 129 #define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0) 130 131 #define APPL_DEBUG 0xD0 132 #define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21) 133 #define APPL_DEBUG_PM_LINKST_IN_L0 0x11 134 #define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3) 135 #define APPL_DEBUG_LTSSM_STATE_SHIFT 3 136 #define LTSSM_STATE_PRE_DETECT 5 137 138 #define APPL_RADM_STATUS 0xE4 139 #define APPL_PM_XMT_TURNOFF_STATE BIT(0) 140 141 #define APPL_DM_TYPE 0x100 142 #define APPL_DM_TYPE_MASK GENMASK(3, 0) 143 #define APPL_DM_TYPE_RP 0x4 144 #define APPL_DM_TYPE_EP 0x0 145 146 #define APPL_CFG_BASE_ADDR 0x104 147 #define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12) 148 149 #define APPL_CFG_IATU_DMA_BASE_ADDR 0x108 150 #define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18) 151 152 #define APPL_CFG_MISC 0x110 153 #define APPL_CFG_MISC_SLV_EP_MODE BIT(14) 154 #define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10) 155 #define APPL_CFG_MISC_ARCACHE_SHIFT 10 156 #define APPL_CFG_MISC_ARCACHE_VAL 3 157 158 #define APPL_CFG_SLCG_OVERRIDE 0x114 159 #define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0) 160 161 #define APPL_CAR_RESET_OVRD 0x12C 162 #define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0) 163 164 #define IO_BASE_IO_DECODE BIT(0) 165 #define IO_BASE_IO_DECODE_BIT8 BIT(8) 166 167 #define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0) 168 #define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16) 169 170 #define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718 171 #define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19) 172 173 #define EVENT_COUNTER_ALL_CLEAR 0x3 174 #define EVENT_COUNTER_ENABLE_ALL 0x7 175 #define EVENT_COUNTER_ENABLE_SHIFT 2 176 #define EVENT_COUNTER_EVENT_SEL_MASK GENMASK(7, 0) 177 #define EVENT_COUNTER_EVENT_SEL_SHIFT 16 178 #define EVENT_COUNTER_EVENT_Tx_L0S 0x2 179 #define EVENT_COUNTER_EVENT_Rx_L0S 0x3 180 #define EVENT_COUNTER_EVENT_L1 0x5 181 #define EVENT_COUNTER_EVENT_L1_1 0x7 182 #define EVENT_COUNTER_EVENT_L1_2 0x8 183 #define EVENT_COUNTER_GROUP_SEL_SHIFT 24 184 #define EVENT_COUNTER_GROUP_5 0x5 185 186 #define PORT_LOGIC_ACK_F_ASPM_CTRL 0x70C 187 #define ENTER_ASPM BIT(30) 188 #define L0S_ENTRANCE_LAT_SHIFT 24 189 #define L0S_ENTRANCE_LAT_MASK GENMASK(26, 24) 190 #define L1_ENTRANCE_LAT_SHIFT 27 191 #define L1_ENTRANCE_LAT_MASK GENMASK(29, 27) 192 #define N_FTS_SHIFT 8 193 #define N_FTS_MASK GENMASK(7, 0) 194 #define N_FTS_VAL 52 195 196 #define PORT_LOGIC_GEN2_CTRL 0x80C 197 #define PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE BIT(17) 198 #define FTS_MASK GENMASK(7, 0) 199 #define FTS_VAL 52 200 201 #define PORT_LOGIC_MSI_CTRL_INT_0_EN 0x828 202 203 #define GEN3_EQ_CONTROL_OFF 0x8a8 204 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8 205 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8) 206 #define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0) 207 208 #define GEN3_RELATED_OFF 0x890 209 #define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL BIT(0) 210 #define GEN3_RELATED_OFF_GEN3_EQ_DISABLE BIT(16) 211 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT 24 212 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK GENMASK(25, 24) 213 214 #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0 215 #define AMBA_ERROR_RESPONSE_CRS_SHIFT 3 216 #define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0) 217 #define AMBA_ERROR_RESPONSE_CRS_OKAY 0 218 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1 219 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2 220 221 #define MSIX_ADDR_MATCH_LOW_OFF 0x940 222 #define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0) 223 #define MSIX_ADDR_MATCH_LOW_OFF_MASK GENMASK(31, 2) 224 225 #define MSIX_ADDR_MATCH_HIGH_OFF 0x944 226 #define MSIX_ADDR_MATCH_HIGH_OFF_MASK GENMASK(31, 0) 227 228 #define PORT_LOGIC_MSIX_DOORBELL 0x948 229 230 #define CAP_SPCIE_CAP_OFF 0x154 231 #define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0) 232 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8) 233 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8 234 235 #define PME_ACK_TIMEOUT 10000 236 237 #define LTSSM_TIMEOUT 50000 /* 50ms */ 238 239 #define GEN3_GEN4_EQ_PRESET_INIT 5 240 241 #define GEN1_CORE_CLK_FREQ 62500000 242 #define GEN2_CORE_CLK_FREQ 125000000 243 #define GEN3_CORE_CLK_FREQ 250000000 244 #define GEN4_CORE_CLK_FREQ 500000000 245 246 #define LTR_MSG_TIMEOUT (100 * 1000) 247 248 #define PERST_DEBOUNCE_TIME (5 * 1000) 249 250 #define EP_STATE_DISABLED 0 251 #define EP_STATE_ENABLED 1 252 253 static const unsigned int pcie_gen_freq[] = { 254 GEN1_CORE_CLK_FREQ, 255 GEN2_CORE_CLK_FREQ, 256 GEN3_CORE_CLK_FREQ, 257 GEN4_CORE_CLK_FREQ 258 }; 259 260 static const u32 event_cntr_ctrl_offset[] = { 261 0x1d8, 262 0x1a8, 263 0x1a8, 264 0x1a8, 265 0x1c4, 266 0x1d8 267 }; 268 269 static const u32 event_cntr_data_offset[] = { 270 0x1dc, 271 0x1ac, 272 0x1ac, 273 0x1ac, 274 0x1c8, 275 0x1dc 276 }; 277 278 struct tegra_pcie_dw { 279 struct device *dev; 280 struct resource *appl_res; 281 struct resource *dbi_res; 282 struct resource *atu_dma_res; 283 void __iomem *appl_base; 284 struct clk *core_clk; 285 struct reset_control *core_apb_rst; 286 struct reset_control *core_rst; 287 struct dw_pcie pci; 288 struct tegra_bpmp *bpmp; 289 290 enum dw_pcie_device_mode mode; 291 292 bool supports_clkreq; 293 bool enable_cdm_check; 294 bool link_state; 295 bool update_fc_fixup; 296 u8 init_link_width; 297 u32 msi_ctrl_int; 298 u32 num_lanes; 299 u32 max_speed; 300 u32 cid; 301 u32 cfg_link_cap_l1sub; 302 u32 pcie_cap_base; 303 u32 aspm_cmrt; 304 u32 aspm_pwr_on_t; 305 u32 aspm_l0s_enter_lat; 306 307 struct regulator *pex_ctl_supply; 308 struct regulator *slot_ctl_3v3; 309 struct regulator *slot_ctl_12v; 310 311 unsigned int phy_count; 312 struct phy **phys; 313 314 struct dentry *debugfs; 315 316 /* Endpoint mode specific */ 317 struct gpio_desc *pex_rst_gpiod; 318 struct gpio_desc *pex_refclk_sel_gpiod; 319 unsigned int pex_rst_irq; 320 int ep_state; 321 }; 322 323 struct tegra_pcie_dw_of_data { 324 enum dw_pcie_device_mode mode; 325 }; 326 327 static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci) 328 { 329 return container_of(pci, struct tegra_pcie_dw, pci); 330 } 331 332 static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value, 333 const u32 reg) 334 { 335 writel_relaxed(value, pcie->appl_base + reg); 336 } 337 338 static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg) 339 { 340 return readl_relaxed(pcie->appl_base + reg); 341 } 342 343 struct tegra_pcie_soc { 344 enum dw_pcie_device_mode mode; 345 }; 346 347 static void apply_bad_link_workaround(struct pcie_port *pp) 348 { 349 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 350 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 351 u32 current_link_width; 352 u16 val; 353 354 /* 355 * NOTE:- Since this scenario is uncommon and link as such is not 356 * stable anyway, not waiting to confirm if link is really 357 * transitioning to Gen-2 speed 358 */ 359 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); 360 if (val & PCI_EXP_LNKSTA_LBMS) { 361 current_link_width = (val & PCI_EXP_LNKSTA_NLW) >> 362 PCI_EXP_LNKSTA_NLW_SHIFT; 363 if (pcie->init_link_width > current_link_width) { 364 dev_warn(pci->dev, "PCIe link is bad, width reduced\n"); 365 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + 366 PCI_EXP_LNKCTL2); 367 val &= ~PCI_EXP_LNKCTL2_TLS; 368 val |= PCI_EXP_LNKCTL2_TLS_2_5GT; 369 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + 370 PCI_EXP_LNKCTL2, val); 371 372 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + 373 PCI_EXP_LNKCTL); 374 val |= PCI_EXP_LNKCTL_RL; 375 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + 376 PCI_EXP_LNKCTL, val); 377 } 378 } 379 } 380 381 static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg) 382 { 383 struct tegra_pcie_dw *pcie = arg; 384 struct dw_pcie *pci = &pcie->pci; 385 struct pcie_port *pp = &pci->pp; 386 u32 val, tmp; 387 u16 val_w; 388 389 val = appl_readl(pcie, APPL_INTR_STATUS_L0); 390 if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) { 391 val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); 392 if (val & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) { 393 appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0); 394 395 /* SBR & Surprise Link Down WAR */ 396 val = appl_readl(pcie, APPL_CAR_RESET_OVRD); 397 val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; 398 appl_writel(pcie, val, APPL_CAR_RESET_OVRD); 399 udelay(1); 400 val = appl_readl(pcie, APPL_CAR_RESET_OVRD); 401 val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; 402 appl_writel(pcie, val, APPL_CAR_RESET_OVRD); 403 404 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); 405 val |= PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE; 406 dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); 407 } 408 } 409 410 if (val & APPL_INTR_STATUS_L0_INT_INT) { 411 val = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0); 412 if (val & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) { 413 appl_writel(pcie, 414 APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS, 415 APPL_INTR_STATUS_L1_8_0); 416 apply_bad_link_workaround(pp); 417 } 418 if (val & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) { 419 appl_writel(pcie, 420 APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS, 421 APPL_INTR_STATUS_L1_8_0); 422 423 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + 424 PCI_EXP_LNKSTA); 425 dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w & 426 PCI_EXP_LNKSTA_CLS); 427 } 428 } 429 430 val = appl_readl(pcie, APPL_INTR_STATUS_L0); 431 if (val & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) { 432 val = appl_readl(pcie, APPL_INTR_STATUS_L1_18); 433 tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS); 434 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) { 435 dev_info(pci->dev, "CDM check complete\n"); 436 tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE; 437 } 438 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) { 439 dev_err(pci->dev, "CDM comparison mismatch\n"); 440 tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR; 441 } 442 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) { 443 dev_err(pci->dev, "CDM Logic error\n"); 444 tmp |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR; 445 } 446 dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, tmp); 447 tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR); 448 dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", tmp); 449 } 450 451 return IRQ_HANDLED; 452 } 453 454 static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie) 455 { 456 u32 val; 457 458 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); 459 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); 460 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); 461 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); 462 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); 463 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); 464 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); 465 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); 466 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); 467 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); 468 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); 469 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); 470 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); 471 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); 472 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); 473 appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2); 474 475 val = appl_readl(pcie, APPL_CTRL); 476 val |= APPL_CTRL_LTSSM_EN; 477 appl_writel(pcie, val, APPL_CTRL); 478 } 479 480 static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg) 481 { 482 struct tegra_pcie_dw *pcie = arg; 483 struct dw_pcie *pci = &pcie->pci; 484 u32 val, speed; 485 486 speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) & 487 PCI_EXP_LNKSTA_CLS; 488 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]); 489 490 /* If EP doesn't advertise L1SS, just return */ 491 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub); 492 if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2))) 493 return IRQ_HANDLED; 494 495 /* Check if BME is set to '1' */ 496 val = dw_pcie_readl_dbi(pci, PCI_COMMAND); 497 if (val & PCI_COMMAND_MASTER) { 498 ktime_t timeout; 499 500 /* 110us for both snoop and no-snoop */ 501 val = 110 | (2 << PCI_LTR_SCALE_SHIFT) | LTR_MSG_REQ; 502 val |= (val << LTR_MST_NO_SNOOP_SHIFT); 503 appl_writel(pcie, val, APPL_LTR_MSG_1); 504 505 /* Send LTR upstream */ 506 val = appl_readl(pcie, APPL_LTR_MSG_2); 507 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE; 508 appl_writel(pcie, val, APPL_LTR_MSG_2); 509 510 timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT); 511 for (;;) { 512 val = appl_readl(pcie, APPL_LTR_MSG_2); 513 if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)) 514 break; 515 if (ktime_after(ktime_get(), timeout)) 516 break; 517 usleep_range(1000, 1100); 518 } 519 if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE) 520 dev_err(pcie->dev, "Failed to send LTR message\n"); 521 } 522 523 return IRQ_HANDLED; 524 } 525 526 static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg) 527 { 528 struct tegra_pcie_dw *pcie = arg; 529 struct dw_pcie_ep *ep = &pcie->pci.ep; 530 int spurious = 1; 531 u32 val, tmp; 532 533 val = appl_readl(pcie, APPL_INTR_STATUS_L0); 534 if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) { 535 val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); 536 appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0); 537 538 if (val & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE) 539 pex_ep_event_hot_rst_done(pcie); 540 541 if (val & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) { 542 tmp = appl_readl(pcie, APPL_LINK_STATUS); 543 if (tmp & APPL_LINK_STATUS_RDLH_LINK_UP) { 544 dev_dbg(pcie->dev, "Link is up with Host\n"); 545 dw_pcie_ep_linkup(ep); 546 } 547 } 548 549 spurious = 0; 550 } 551 552 if (val & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) { 553 val = appl_readl(pcie, APPL_INTR_STATUS_L1_15); 554 appl_writel(pcie, val, APPL_INTR_STATUS_L1_15); 555 556 if (val & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED) 557 return IRQ_WAKE_THREAD; 558 559 spurious = 0; 560 } 561 562 if (spurious) { 563 dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n", 564 val); 565 appl_writel(pcie, val, APPL_INTR_STATUS_L0); 566 } 567 568 return IRQ_HANDLED; 569 } 570 571 static int tegra_pcie_dw_rd_own_conf(struct pcie_port *pp, int where, int size, 572 u32 *val) 573 { 574 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 575 576 /* 577 * This is an endpoint mode specific register happen to appear even 578 * when controller is operating in root port mode and system hangs 579 * when it is accessed with link being in ASPM-L1 state. 580 * So skip accessing it altogether 581 */ 582 if (where == PORT_LOGIC_MSIX_DOORBELL) { 583 *val = 0x00000000; 584 return PCIBIOS_SUCCESSFUL; 585 } 586 587 return dw_pcie_read(pci->dbi_base + where, size, val); 588 } 589 590 static int tegra_pcie_dw_wr_own_conf(struct pcie_port *pp, int where, int size, 591 u32 val) 592 { 593 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 594 595 /* 596 * This is an endpoint mode specific register happen to appear even 597 * when controller is operating in root port mode and system hangs 598 * when it is accessed with link being in ASPM-L1 state. 599 * So skip accessing it altogether 600 */ 601 if (where == PORT_LOGIC_MSIX_DOORBELL) 602 return PCIBIOS_SUCCESSFUL; 603 604 return dw_pcie_write(pci->dbi_base + where, size, val); 605 } 606 607 #if defined(CONFIG_PCIEASPM) 608 static void disable_aspm_l11(struct tegra_pcie_dw *pcie) 609 { 610 u32 val; 611 612 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); 613 val &= ~PCI_L1SS_CAP_ASPM_L1_1; 614 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); 615 } 616 617 static void disable_aspm_l12(struct tegra_pcie_dw *pcie) 618 { 619 u32 val; 620 621 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); 622 val &= ~PCI_L1SS_CAP_ASPM_L1_2; 623 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); 624 } 625 626 static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event) 627 { 628 u32 val; 629 630 val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid]); 631 val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT); 632 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; 633 val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT; 634 val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; 635 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val); 636 val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_data_offset[pcie->cid]); 637 638 return val; 639 } 640 641 static int aspm_state_cnt(struct seq_file *s, void *data) 642 { 643 struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *) 644 dev_get_drvdata(s->private); 645 u32 val; 646 647 seq_printf(s, "Tx L0s entry count : %u\n", 648 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S)); 649 650 seq_printf(s, "Rx L0s entry count : %u\n", 651 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S)); 652 653 seq_printf(s, "Link L1 entry count : %u\n", 654 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1)); 655 656 seq_printf(s, "Link L1.1 entry count : %u\n", 657 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1)); 658 659 seq_printf(s, "Link L1.2 entry count : %u\n", 660 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2)); 661 662 /* Clear all counters */ 663 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], 664 EVENT_COUNTER_ALL_CLEAR); 665 666 /* Re-enable counting */ 667 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; 668 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; 669 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val); 670 671 return 0; 672 } 673 674 static void init_host_aspm(struct tegra_pcie_dw *pcie) 675 { 676 struct dw_pcie *pci = &pcie->pci; 677 u32 val; 678 679 val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS); 680 pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP; 681 682 /* Enable ASPM counters */ 683 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; 684 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; 685 dw_pcie_writel_dbi(pci, event_cntr_ctrl_offset[pcie->cid], val); 686 687 /* Program T_cmrt and T_pwr_on values */ 688 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub); 689 val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE); 690 val |= (pcie->aspm_cmrt << 8); 691 val |= (pcie->aspm_pwr_on_t << 19); 692 dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val); 693 694 /* Program L0s and L1 entrance latencies */ 695 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); 696 val &= ~L0S_ENTRANCE_LAT_MASK; 697 val |= (pcie->aspm_l0s_enter_lat << L0S_ENTRANCE_LAT_SHIFT); 698 val |= ENTER_ASPM; 699 dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); 700 } 701 702 static int init_debugfs(struct tegra_pcie_dw *pcie) 703 { 704 struct dentry *d; 705 706 d = debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", 707 pcie->debugfs, aspm_state_cnt); 708 if (IS_ERR_OR_NULL(d)) 709 dev_err(pcie->dev, 710 "Failed to create debugfs file \"aspm_state_cnt\"\n"); 711 712 return 0; 713 } 714 #else 715 static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; } 716 static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; } 717 static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; } 718 static inline int init_debugfs(struct tegra_pcie_dw *pcie) { return 0; } 719 #endif 720 721 static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp) 722 { 723 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 724 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 725 u32 val; 726 u16 val_w; 727 728 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 729 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN; 730 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 731 732 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); 733 val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN; 734 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); 735 736 if (pcie->enable_cdm_check) { 737 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 738 val |= APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN; 739 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 740 741 val = appl_readl(pcie, APPL_INTR_EN_L1_18); 742 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR; 743 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR; 744 appl_writel(pcie, val, APPL_INTR_EN_L1_18); 745 } 746 747 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + 748 PCI_EXP_LNKSTA); 749 pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >> 750 PCI_EXP_LNKSTA_NLW_SHIFT; 751 752 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + 753 PCI_EXP_LNKCTL); 754 val_w |= PCI_EXP_LNKCTL_LBMIE; 755 dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL, 756 val_w); 757 } 758 759 static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp) 760 { 761 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 762 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 763 u32 val; 764 765 /* Enable legacy interrupt generation */ 766 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 767 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN; 768 val |= APPL_INTR_EN_L0_0_INT_INT_EN; 769 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 770 771 val = appl_readl(pcie, APPL_INTR_EN_L1_8_0); 772 val |= APPL_INTR_EN_L1_8_INTX_EN; 773 val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN; 774 val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN; 775 if (IS_ENABLED(CONFIG_PCIEAER)) 776 val |= APPL_INTR_EN_L1_8_AER_INT_EN; 777 appl_writel(pcie, val, APPL_INTR_EN_L1_8_0); 778 } 779 780 static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp) 781 { 782 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 783 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 784 u32 val; 785 786 dw_pcie_msi_init(pp); 787 788 /* Enable MSI interrupt generation */ 789 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 790 val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN; 791 val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN; 792 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 793 } 794 795 static void tegra_pcie_enable_interrupts(struct pcie_port *pp) 796 { 797 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 798 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 799 800 /* Clear interrupt statuses before enabling interrupts */ 801 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); 802 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); 803 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); 804 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); 805 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); 806 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); 807 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); 808 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); 809 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); 810 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); 811 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); 812 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); 813 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); 814 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); 815 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); 816 817 tegra_pcie_enable_system_interrupts(pp); 818 tegra_pcie_enable_legacy_interrupts(pp); 819 if (IS_ENABLED(CONFIG_PCI_MSI)) 820 tegra_pcie_enable_msi_interrupts(pp); 821 } 822 823 static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie) 824 { 825 struct dw_pcie *pci = &pcie->pci; 826 u32 val, offset, i; 827 828 /* Program init preset */ 829 for (i = 0; i < pcie->num_lanes; i++) { 830 dw_pcie_read(pci->dbi_base + CAP_SPCIE_CAP_OFF 831 + (i * 2), 2, &val); 832 val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK; 833 val |= GEN3_GEN4_EQ_PRESET_INIT; 834 val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK; 835 val |= (GEN3_GEN4_EQ_PRESET_INIT << 836 CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT); 837 dw_pcie_write(pci->dbi_base + CAP_SPCIE_CAP_OFF 838 + (i * 2), 2, val); 839 840 offset = dw_pcie_find_ext_capability(pci, 841 PCI_EXT_CAP_ID_PL_16GT) + 842 PCI_PL_16GT_LE_CTRL; 843 dw_pcie_read(pci->dbi_base + offset + i, 1, &val); 844 val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK; 845 val |= GEN3_GEN4_EQ_PRESET_INIT; 846 val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK; 847 val |= (GEN3_GEN4_EQ_PRESET_INIT << 848 PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT); 849 dw_pcie_write(pci->dbi_base + offset + i, 1, val); 850 } 851 852 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 853 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; 854 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 855 856 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); 857 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; 858 val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); 859 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; 860 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); 861 862 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 863 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; 864 val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT); 865 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 866 867 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); 868 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; 869 val |= (0x360 << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); 870 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; 871 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); 872 873 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 874 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; 875 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 876 } 877 878 static void tegra_pcie_prepare_host(struct pcie_port *pp) 879 { 880 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 881 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 882 u32 val; 883 884 val = dw_pcie_readl_dbi(pci, PCI_IO_BASE); 885 val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8); 886 dw_pcie_writel_dbi(pci, PCI_IO_BASE, val); 887 888 val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE); 889 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE; 890 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE; 891 dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val); 892 893 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); 894 895 /* Configure FTS */ 896 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); 897 val &= ~(N_FTS_MASK << N_FTS_SHIFT); 898 val |= N_FTS_VAL << N_FTS_SHIFT; 899 dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); 900 901 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); 902 val &= ~FTS_MASK; 903 val |= FTS_VAL; 904 dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); 905 906 /* Enable as 0xFFFF0001 response for CRS */ 907 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT); 908 val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT); 909 val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 << 910 AMBA_ERROR_RESPONSE_CRS_SHIFT); 911 dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val); 912 913 /* Configure Max Speed from DT */ 914 if (pcie->max_speed && pcie->max_speed != -EINVAL) { 915 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + 916 PCI_EXP_LNKCAP); 917 val &= ~PCI_EXP_LNKCAP_SLS; 918 val |= pcie->max_speed; 919 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, 920 val); 921 } 922 923 /* Configure Max lane width from DT */ 924 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP); 925 val &= ~PCI_EXP_LNKCAP_MLW; 926 val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT); 927 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val); 928 929 config_gen3_gen4_eq_presets(pcie); 930 931 init_host_aspm(pcie); 932 933 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 934 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; 935 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 936 937 if (pcie->update_fc_fixup) { 938 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); 939 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT; 940 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val); 941 } 942 943 dw_pcie_setup_rc(pp); 944 945 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); 946 947 /* Assert RST */ 948 val = appl_readl(pcie, APPL_PINMUX); 949 val &= ~APPL_PINMUX_PEX_RST; 950 appl_writel(pcie, val, APPL_PINMUX); 951 952 usleep_range(100, 200); 953 954 /* Enable LTSSM */ 955 val = appl_readl(pcie, APPL_CTRL); 956 val |= APPL_CTRL_LTSSM_EN; 957 appl_writel(pcie, val, APPL_CTRL); 958 959 /* De-assert RST */ 960 val = appl_readl(pcie, APPL_PINMUX); 961 val |= APPL_PINMUX_PEX_RST; 962 appl_writel(pcie, val, APPL_PINMUX); 963 964 msleep(100); 965 } 966 967 static int tegra_pcie_dw_host_init(struct pcie_port *pp) 968 { 969 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 970 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 971 u32 val, tmp, offset, speed; 972 973 tegra_pcie_prepare_host(pp); 974 975 if (dw_pcie_wait_for_link(pci)) { 976 /* 977 * There are some endpoints which can't get the link up if 978 * root port has Data Link Feature (DLF) enabled. 979 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info 980 * on Scaled Flow Control and DLF. 981 * So, need to confirm that is indeed the case here and attempt 982 * link up once again with DLF disabled. 983 */ 984 val = appl_readl(pcie, APPL_DEBUG); 985 val &= APPL_DEBUG_LTSSM_STATE_MASK; 986 val >>= APPL_DEBUG_LTSSM_STATE_SHIFT; 987 tmp = appl_readl(pcie, APPL_LINK_STATUS); 988 tmp &= APPL_LINK_STATUS_RDLH_LINK_UP; 989 if (!(val == 0x11 && !tmp)) { 990 /* Link is down for all good reasons */ 991 return 0; 992 } 993 994 dev_info(pci->dev, "Link is down in DLL"); 995 dev_info(pci->dev, "Trying again with DLFE disabled\n"); 996 /* Disable LTSSM */ 997 val = appl_readl(pcie, APPL_CTRL); 998 val &= ~APPL_CTRL_LTSSM_EN; 999 appl_writel(pcie, val, APPL_CTRL); 1000 1001 reset_control_assert(pcie->core_rst); 1002 reset_control_deassert(pcie->core_rst); 1003 1004 offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF); 1005 val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP); 1006 val &= ~PCI_DLF_EXCHANGE_ENABLE; 1007 dw_pcie_writel_dbi(pci, offset, val); 1008 1009 tegra_pcie_prepare_host(pp); 1010 1011 if (dw_pcie_wait_for_link(pci)) 1012 return 0; 1013 } 1014 1015 speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) & 1016 PCI_EXP_LNKSTA_CLS; 1017 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]); 1018 1019 tegra_pcie_enable_interrupts(pp); 1020 1021 return 0; 1022 } 1023 1024 static int tegra_pcie_dw_link_up(struct dw_pcie *pci) 1025 { 1026 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 1027 u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); 1028 1029 return !!(val & PCI_EXP_LNKSTA_DLLLA); 1030 } 1031 1032 static void tegra_pcie_set_msi_vec_num(struct pcie_port *pp) 1033 { 1034 pp->num_vectors = MAX_MSI_IRQS; 1035 } 1036 1037 static int tegra_pcie_dw_start_link(struct dw_pcie *pci) 1038 { 1039 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 1040 1041 enable_irq(pcie->pex_rst_irq); 1042 1043 return 0; 1044 } 1045 1046 static void tegra_pcie_dw_stop_link(struct dw_pcie *pci) 1047 { 1048 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 1049 1050 disable_irq(pcie->pex_rst_irq); 1051 } 1052 1053 static const struct dw_pcie_ops tegra_dw_pcie_ops = { 1054 .link_up = tegra_pcie_dw_link_up, 1055 .start_link = tegra_pcie_dw_start_link, 1056 .stop_link = tegra_pcie_dw_stop_link, 1057 }; 1058 1059 static struct dw_pcie_host_ops tegra_pcie_dw_host_ops = { 1060 .rd_own_conf = tegra_pcie_dw_rd_own_conf, 1061 .wr_own_conf = tegra_pcie_dw_wr_own_conf, 1062 .host_init = tegra_pcie_dw_host_init, 1063 .set_num_vectors = tegra_pcie_set_msi_vec_num, 1064 }; 1065 1066 static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie) 1067 { 1068 unsigned int phy_count = pcie->phy_count; 1069 1070 while (phy_count--) { 1071 phy_power_off(pcie->phys[phy_count]); 1072 phy_exit(pcie->phys[phy_count]); 1073 } 1074 } 1075 1076 static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie) 1077 { 1078 unsigned int i; 1079 int ret; 1080 1081 for (i = 0; i < pcie->phy_count; i++) { 1082 ret = phy_init(pcie->phys[i]); 1083 if (ret < 0) 1084 goto phy_power_off; 1085 1086 ret = phy_power_on(pcie->phys[i]); 1087 if (ret < 0) 1088 goto phy_exit; 1089 } 1090 1091 return 0; 1092 1093 phy_power_off: 1094 while (i--) { 1095 phy_power_off(pcie->phys[i]); 1096 phy_exit: 1097 phy_exit(pcie->phys[i]); 1098 } 1099 1100 return ret; 1101 } 1102 1103 static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie) 1104 { 1105 struct device_node *np = pcie->dev->of_node; 1106 int ret; 1107 1108 ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt); 1109 if (ret < 0) { 1110 dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret); 1111 return ret; 1112 } 1113 1114 ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us", 1115 &pcie->aspm_pwr_on_t); 1116 if (ret < 0) 1117 dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n", 1118 ret); 1119 1120 ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us", 1121 &pcie->aspm_l0s_enter_lat); 1122 if (ret < 0) 1123 dev_info(pcie->dev, 1124 "Failed to read ASPM L0s Entrance latency: %d\n", ret); 1125 1126 ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes); 1127 if (ret < 0) { 1128 dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret); 1129 return ret; 1130 } 1131 1132 pcie->max_speed = of_pci_get_max_link_speed(np); 1133 1134 ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid); 1135 if (ret) { 1136 dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret); 1137 return ret; 1138 } 1139 1140 ret = of_property_count_strings(np, "phy-names"); 1141 if (ret < 0) { 1142 dev_err(pcie->dev, "Failed to find PHY entries: %d\n", 1143 ret); 1144 return ret; 1145 } 1146 pcie->phy_count = ret; 1147 1148 if (of_property_read_bool(np, "nvidia,update-fc-fixup")) 1149 pcie->update_fc_fixup = true; 1150 1151 pcie->supports_clkreq = 1152 of_property_read_bool(pcie->dev->of_node, "supports-clkreq"); 1153 1154 pcie->enable_cdm_check = 1155 of_property_read_bool(np, "snps,enable-cdm-check"); 1156 1157 if (pcie->mode == DW_PCIE_RC_TYPE) 1158 return 0; 1159 1160 /* Endpoint mode specific DT entries */ 1161 pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN); 1162 if (IS_ERR(pcie->pex_rst_gpiod)) { 1163 dev_err(pcie->dev, "Failed to get PERST GPIO: %ld\n", 1164 PTR_ERR(pcie->pex_rst_gpiod)); 1165 return PTR_ERR(pcie->pex_rst_gpiod); 1166 } 1167 1168 pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev, 1169 "nvidia,refclk-select", 1170 GPIOD_OUT_HIGH); 1171 if (IS_ERR(pcie->pex_refclk_sel_gpiod)) { 1172 dev_info(pcie->dev, "Failed to get REFCLK select GPIOs: %ld\n", 1173 PTR_ERR(pcie->pex_refclk_sel_gpiod)); 1174 pcie->pex_refclk_sel_gpiod = NULL; 1175 } 1176 1177 return 0; 1178 } 1179 1180 static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie, 1181 bool enable) 1182 { 1183 struct mrq_uphy_response resp; 1184 struct tegra_bpmp_message msg; 1185 struct mrq_uphy_request req; 1186 1187 /* Controller-5 doesn't need to have its state set by BPMP-FW */ 1188 if (pcie->cid == 5) 1189 return 0; 1190 1191 memset(&req, 0, sizeof(req)); 1192 memset(&resp, 0, sizeof(resp)); 1193 1194 req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE; 1195 req.controller_state.pcie_controller = pcie->cid; 1196 req.controller_state.enable = enable; 1197 1198 memset(&msg, 0, sizeof(msg)); 1199 msg.mrq = MRQ_UPHY; 1200 msg.tx.data = &req; 1201 msg.tx.size = sizeof(req); 1202 msg.rx.data = &resp; 1203 msg.rx.size = sizeof(resp); 1204 1205 return tegra_bpmp_transfer(pcie->bpmp, &msg); 1206 } 1207 1208 static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie, 1209 bool enable) 1210 { 1211 struct mrq_uphy_response resp; 1212 struct tegra_bpmp_message msg; 1213 struct mrq_uphy_request req; 1214 1215 memset(&req, 0, sizeof(req)); 1216 memset(&resp, 0, sizeof(resp)); 1217 1218 if (enable) { 1219 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT; 1220 req.ep_ctrlr_pll_init.ep_controller = pcie->cid; 1221 } else { 1222 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF; 1223 req.ep_ctrlr_pll_off.ep_controller = pcie->cid; 1224 } 1225 1226 memset(&msg, 0, sizeof(msg)); 1227 msg.mrq = MRQ_UPHY; 1228 msg.tx.data = &req; 1229 msg.tx.size = sizeof(req); 1230 msg.rx.data = &resp; 1231 msg.rx.size = sizeof(resp); 1232 1233 return tegra_bpmp_transfer(pcie->bpmp, &msg); 1234 } 1235 1236 static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie) 1237 { 1238 struct pcie_port *pp = &pcie->pci.pp; 1239 struct pci_bus *child, *root_bus = NULL; 1240 struct pci_dev *pdev; 1241 1242 /* 1243 * link doesn't go into L2 state with some of the endpoints with Tegra 1244 * if they are not in D0 state. So, need to make sure that immediate 1245 * downstream devices are in D0 state before sending PME_TurnOff to put 1246 * link into L2 state. 1247 * This is as per PCI Express Base r4.0 v1.0 September 27-2017, 1248 * 5.2 Link State Power Management (Page #428). 1249 */ 1250 1251 list_for_each_entry(child, &pp->root_bus->children, node) { 1252 /* Bring downstream devices to D0 if they are not already in */ 1253 if (child->parent == pp->root_bus) { 1254 root_bus = child; 1255 break; 1256 } 1257 } 1258 1259 if (!root_bus) { 1260 dev_err(pcie->dev, "Failed to find downstream devices\n"); 1261 return; 1262 } 1263 1264 list_for_each_entry(pdev, &root_bus->devices, bus_list) { 1265 if (PCI_SLOT(pdev->devfn) == 0) { 1266 if (pci_set_power_state(pdev, PCI_D0)) 1267 dev_err(pcie->dev, 1268 "Failed to transition %s to D0 state\n", 1269 dev_name(&pdev->dev)); 1270 } 1271 } 1272 } 1273 1274 static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie) 1275 { 1276 pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3"); 1277 if (IS_ERR(pcie->slot_ctl_3v3)) { 1278 if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV) 1279 return PTR_ERR(pcie->slot_ctl_3v3); 1280 1281 pcie->slot_ctl_3v3 = NULL; 1282 } 1283 1284 pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v"); 1285 if (IS_ERR(pcie->slot_ctl_12v)) { 1286 if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV) 1287 return PTR_ERR(pcie->slot_ctl_12v); 1288 1289 pcie->slot_ctl_12v = NULL; 1290 } 1291 1292 return 0; 1293 } 1294 1295 static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie) 1296 { 1297 int ret; 1298 1299 if (pcie->slot_ctl_3v3) { 1300 ret = regulator_enable(pcie->slot_ctl_3v3); 1301 if (ret < 0) { 1302 dev_err(pcie->dev, 1303 "Failed to enable 3.3V slot supply: %d\n", ret); 1304 return ret; 1305 } 1306 } 1307 1308 if (pcie->slot_ctl_12v) { 1309 ret = regulator_enable(pcie->slot_ctl_12v); 1310 if (ret < 0) { 1311 dev_err(pcie->dev, 1312 "Failed to enable 12V slot supply: %d\n", ret); 1313 goto fail_12v_enable; 1314 } 1315 } 1316 1317 /* 1318 * According to PCI Express Card Electromechanical Specification 1319 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive) 1320 * should be a minimum of 100ms. 1321 */ 1322 if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v) 1323 msleep(100); 1324 1325 return 0; 1326 1327 fail_12v_enable: 1328 if (pcie->slot_ctl_3v3) 1329 regulator_disable(pcie->slot_ctl_3v3); 1330 return ret; 1331 } 1332 1333 static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie) 1334 { 1335 if (pcie->slot_ctl_12v) 1336 regulator_disable(pcie->slot_ctl_12v); 1337 if (pcie->slot_ctl_3v3) 1338 regulator_disable(pcie->slot_ctl_3v3); 1339 } 1340 1341 static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie, 1342 bool en_hw_hot_rst) 1343 { 1344 int ret; 1345 u32 val; 1346 1347 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true); 1348 if (ret) { 1349 dev_err(pcie->dev, 1350 "Failed to enable controller %u: %d\n", pcie->cid, ret); 1351 return ret; 1352 } 1353 1354 ret = tegra_pcie_enable_slot_regulators(pcie); 1355 if (ret < 0) 1356 goto fail_slot_reg_en; 1357 1358 ret = regulator_enable(pcie->pex_ctl_supply); 1359 if (ret < 0) { 1360 dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret); 1361 goto fail_reg_en; 1362 } 1363 1364 ret = clk_prepare_enable(pcie->core_clk); 1365 if (ret) { 1366 dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret); 1367 goto fail_core_clk; 1368 } 1369 1370 ret = reset_control_deassert(pcie->core_apb_rst); 1371 if (ret) { 1372 dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n", 1373 ret); 1374 goto fail_core_apb_rst; 1375 } 1376 1377 if (en_hw_hot_rst) { 1378 /* Enable HW_HOT_RST mode */ 1379 val = appl_readl(pcie, APPL_CTRL); 1380 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << 1381 APPL_CTRL_HW_HOT_RST_MODE_SHIFT); 1382 val |= APPL_CTRL_HW_HOT_RST_EN; 1383 appl_writel(pcie, val, APPL_CTRL); 1384 } 1385 1386 ret = tegra_pcie_enable_phy(pcie); 1387 if (ret) { 1388 dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret); 1389 goto fail_phy; 1390 } 1391 1392 /* Update CFG base address */ 1393 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK, 1394 APPL_CFG_BASE_ADDR); 1395 1396 /* Configure this core for RP mode operation */ 1397 appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE); 1398 1399 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE); 1400 1401 val = appl_readl(pcie, APPL_CTRL); 1402 appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL); 1403 1404 val = appl_readl(pcie, APPL_CFG_MISC); 1405 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT); 1406 appl_writel(pcie, val, APPL_CFG_MISC); 1407 1408 if (!pcie->supports_clkreq) { 1409 val = appl_readl(pcie, APPL_PINMUX); 1410 val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN; 1411 val &= ~APPL_PINMUX_CLKREQ_OVERRIDE; 1412 appl_writel(pcie, val, APPL_PINMUX); 1413 } 1414 1415 /* Update iATU_DMA base address */ 1416 appl_writel(pcie, 1417 pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK, 1418 APPL_CFG_IATU_DMA_BASE_ADDR); 1419 1420 reset_control_deassert(pcie->core_rst); 1421 1422 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, 1423 PCI_CAP_ID_EXP); 1424 1425 /* Disable ASPM-L1SS advertisement as there is no CLKREQ routing */ 1426 if (!pcie->supports_clkreq) { 1427 disable_aspm_l11(pcie); 1428 disable_aspm_l12(pcie); 1429 } 1430 1431 return ret; 1432 1433 fail_phy: 1434 reset_control_assert(pcie->core_apb_rst); 1435 fail_core_apb_rst: 1436 clk_disable_unprepare(pcie->core_clk); 1437 fail_core_clk: 1438 regulator_disable(pcie->pex_ctl_supply); 1439 fail_reg_en: 1440 tegra_pcie_disable_slot_regulators(pcie); 1441 fail_slot_reg_en: 1442 tegra_pcie_bpmp_set_ctrl_state(pcie, false); 1443 1444 return ret; 1445 } 1446 1447 static int __deinit_controller(struct tegra_pcie_dw *pcie) 1448 { 1449 int ret; 1450 1451 ret = reset_control_assert(pcie->core_rst); 1452 if (ret) { 1453 dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", 1454 ret); 1455 return ret; 1456 } 1457 1458 tegra_pcie_disable_phy(pcie); 1459 1460 ret = reset_control_assert(pcie->core_apb_rst); 1461 if (ret) { 1462 dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret); 1463 return ret; 1464 } 1465 1466 clk_disable_unprepare(pcie->core_clk); 1467 1468 ret = regulator_disable(pcie->pex_ctl_supply); 1469 if (ret) { 1470 dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret); 1471 return ret; 1472 } 1473 1474 tegra_pcie_disable_slot_regulators(pcie); 1475 1476 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false); 1477 if (ret) { 1478 dev_err(pcie->dev, "Failed to disable controller %d: %d\n", 1479 pcie->cid, ret); 1480 return ret; 1481 } 1482 1483 return ret; 1484 } 1485 1486 static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie) 1487 { 1488 struct dw_pcie *pci = &pcie->pci; 1489 struct pcie_port *pp = &pci->pp; 1490 int ret; 1491 1492 ret = tegra_pcie_config_controller(pcie, false); 1493 if (ret < 0) 1494 return ret; 1495 1496 pp->ops = &tegra_pcie_dw_host_ops; 1497 1498 ret = dw_pcie_host_init(pp); 1499 if (ret < 0) { 1500 dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret); 1501 goto fail_host_init; 1502 } 1503 1504 return 0; 1505 1506 fail_host_init: 1507 return __deinit_controller(pcie); 1508 } 1509 1510 static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie) 1511 { 1512 u32 val; 1513 1514 if (!tegra_pcie_dw_link_up(&pcie->pci)) 1515 return 0; 1516 1517 val = appl_readl(pcie, APPL_RADM_STATUS); 1518 val |= APPL_PM_XMT_TURNOFF_STATE; 1519 appl_writel(pcie, val, APPL_RADM_STATUS); 1520 1521 return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val, 1522 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT, 1523 1, PME_ACK_TIMEOUT); 1524 } 1525 1526 static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) 1527 { 1528 u32 data; 1529 int err; 1530 1531 if (!tegra_pcie_dw_link_up(&pcie->pci)) { 1532 dev_dbg(pcie->dev, "PCIe link is not up...!\n"); 1533 return; 1534 } 1535 1536 if (tegra_pcie_try_link_l2(pcie)) { 1537 dev_info(pcie->dev, "Link didn't transition to L2 state\n"); 1538 /* 1539 * TX lane clock freq will reset to Gen1 only if link is in L2 1540 * or detect state. 1541 * So apply pex_rst to end point to force RP to go into detect 1542 * state 1543 */ 1544 data = appl_readl(pcie, APPL_PINMUX); 1545 data &= ~APPL_PINMUX_PEX_RST; 1546 appl_writel(pcie, data, APPL_PINMUX); 1547 1548 err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, 1549 data, 1550 ((data & 1551 APPL_DEBUG_LTSSM_STATE_MASK) >> 1552 APPL_DEBUG_LTSSM_STATE_SHIFT) == 1553 LTSSM_STATE_PRE_DETECT, 1554 1, LTSSM_TIMEOUT); 1555 if (err) { 1556 dev_info(pcie->dev, "Link didn't go to detect state\n"); 1557 } else { 1558 /* Disable LTSSM after link is in detect state */ 1559 data = appl_readl(pcie, APPL_CTRL); 1560 data &= ~APPL_CTRL_LTSSM_EN; 1561 appl_writel(pcie, data, APPL_CTRL); 1562 } 1563 } 1564 /* 1565 * DBI registers may not be accessible after this as PLL-E would be 1566 * down depending on how CLKREQ is pulled by end point 1567 */ 1568 data = appl_readl(pcie, APPL_PINMUX); 1569 data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE); 1570 /* Cut REFCLK to slot */ 1571 data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; 1572 data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; 1573 appl_writel(pcie, data, APPL_PINMUX); 1574 } 1575 1576 static int tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie) 1577 { 1578 tegra_pcie_downstream_dev_to_D0(pcie); 1579 dw_pcie_host_deinit(&pcie->pci.pp); 1580 tegra_pcie_dw_pme_turnoff(pcie); 1581 1582 return __deinit_controller(pcie); 1583 } 1584 1585 static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie) 1586 { 1587 struct pcie_port *pp = &pcie->pci.pp; 1588 struct device *dev = pcie->dev; 1589 char *name; 1590 int ret; 1591 1592 if (IS_ENABLED(CONFIG_PCI_MSI)) { 1593 pp->msi_irq = of_irq_get_byname(dev->of_node, "msi"); 1594 if (!pp->msi_irq) { 1595 dev_err(dev, "Failed to get MSI interrupt\n"); 1596 return -ENODEV; 1597 } 1598 } 1599 1600 pm_runtime_enable(dev); 1601 1602 ret = pm_runtime_get_sync(dev); 1603 if (ret < 0) { 1604 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", 1605 ret); 1606 goto fail_pm_get_sync; 1607 } 1608 1609 ret = pinctrl_pm_select_default_state(dev); 1610 if (ret < 0) { 1611 dev_err(dev, "Failed to configure sideband pins: %d\n", ret); 1612 goto fail_pinctrl; 1613 } 1614 1615 tegra_pcie_init_controller(pcie); 1616 1617 pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci); 1618 if (!pcie->link_state) { 1619 ret = -ENOMEDIUM; 1620 goto fail_host_init; 1621 } 1622 1623 name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node); 1624 if (!name) { 1625 ret = -ENOMEM; 1626 goto fail_host_init; 1627 } 1628 1629 pcie->debugfs = debugfs_create_dir(name, NULL); 1630 if (!pcie->debugfs) 1631 dev_err(dev, "Failed to create debugfs\n"); 1632 else 1633 init_debugfs(pcie); 1634 1635 return ret; 1636 1637 fail_host_init: 1638 tegra_pcie_deinit_controller(pcie); 1639 fail_pinctrl: 1640 pm_runtime_put_sync(dev); 1641 fail_pm_get_sync: 1642 pm_runtime_disable(dev); 1643 return ret; 1644 } 1645 1646 static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie) 1647 { 1648 u32 val; 1649 int ret; 1650 1651 if (pcie->ep_state == EP_STATE_DISABLED) 1652 return; 1653 1654 /* Disable LTSSM */ 1655 val = appl_readl(pcie, APPL_CTRL); 1656 val &= ~APPL_CTRL_LTSSM_EN; 1657 appl_writel(pcie, val, APPL_CTRL); 1658 1659 ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val, 1660 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >> 1661 APPL_DEBUG_LTSSM_STATE_SHIFT) == 1662 LTSSM_STATE_PRE_DETECT, 1663 1, LTSSM_TIMEOUT); 1664 if (ret) 1665 dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret); 1666 1667 reset_control_assert(pcie->core_rst); 1668 1669 tegra_pcie_disable_phy(pcie); 1670 1671 reset_control_assert(pcie->core_apb_rst); 1672 1673 clk_disable_unprepare(pcie->core_clk); 1674 1675 pm_runtime_put_sync(pcie->dev); 1676 1677 ret = tegra_pcie_bpmp_set_pll_state(pcie, false); 1678 if (ret) 1679 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret); 1680 1681 pcie->ep_state = EP_STATE_DISABLED; 1682 dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n"); 1683 } 1684 1685 static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) 1686 { 1687 struct dw_pcie *pci = &pcie->pci; 1688 struct dw_pcie_ep *ep = &pci->ep; 1689 struct device *dev = pcie->dev; 1690 u32 val; 1691 int ret; 1692 1693 if (pcie->ep_state == EP_STATE_ENABLED) 1694 return; 1695 1696 ret = pm_runtime_get_sync(dev); 1697 if (ret < 0) { 1698 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", 1699 ret); 1700 return; 1701 } 1702 1703 ret = tegra_pcie_bpmp_set_pll_state(pcie, true); 1704 if (ret) { 1705 dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n", ret); 1706 goto fail_pll_init; 1707 } 1708 1709 ret = clk_prepare_enable(pcie->core_clk); 1710 if (ret) { 1711 dev_err(dev, "Failed to enable core clock: %d\n", ret); 1712 goto fail_core_clk_enable; 1713 } 1714 1715 ret = reset_control_deassert(pcie->core_apb_rst); 1716 if (ret) { 1717 dev_err(dev, "Failed to deassert core APB reset: %d\n", ret); 1718 goto fail_core_apb_rst; 1719 } 1720 1721 ret = tegra_pcie_enable_phy(pcie); 1722 if (ret) { 1723 dev_err(dev, "Failed to enable PHY: %d\n", ret); 1724 goto fail_phy; 1725 } 1726 1727 /* Clear any stale interrupt statuses */ 1728 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); 1729 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); 1730 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); 1731 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); 1732 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); 1733 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); 1734 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); 1735 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); 1736 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); 1737 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); 1738 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); 1739 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); 1740 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); 1741 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); 1742 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); 1743 1744 /* configure this core for EP mode operation */ 1745 val = appl_readl(pcie, APPL_DM_TYPE); 1746 val &= ~APPL_DM_TYPE_MASK; 1747 val |= APPL_DM_TYPE_EP; 1748 appl_writel(pcie, val, APPL_DM_TYPE); 1749 1750 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE); 1751 1752 val = appl_readl(pcie, APPL_CTRL); 1753 val |= APPL_CTRL_SYS_PRE_DET_STATE; 1754 val |= APPL_CTRL_HW_HOT_RST_EN; 1755 appl_writel(pcie, val, APPL_CTRL); 1756 1757 val = appl_readl(pcie, APPL_CFG_MISC); 1758 val |= APPL_CFG_MISC_SLV_EP_MODE; 1759 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT); 1760 appl_writel(pcie, val, APPL_CFG_MISC); 1761 1762 val = appl_readl(pcie, APPL_PINMUX); 1763 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; 1764 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; 1765 appl_writel(pcie, val, APPL_PINMUX); 1766 1767 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK, 1768 APPL_CFG_BASE_ADDR); 1769 1770 appl_writel(pcie, pcie->atu_dma_res->start & 1771 APPL_CFG_IATU_DMA_BASE_ADDR_MASK, 1772 APPL_CFG_IATU_DMA_BASE_ADDR); 1773 1774 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 1775 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN; 1776 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN; 1777 val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN; 1778 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 1779 1780 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); 1781 val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN; 1782 val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN; 1783 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); 1784 1785 reset_control_deassert(pcie->core_rst); 1786 1787 if (pcie->update_fc_fixup) { 1788 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); 1789 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT; 1790 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val); 1791 } 1792 1793 config_gen3_gen4_eq_presets(pcie); 1794 1795 init_host_aspm(pcie); 1796 1797 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */ 1798 if (!pcie->supports_clkreq) { 1799 disable_aspm_l11(pcie); 1800 disable_aspm_l12(pcie); 1801 } 1802 1803 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 1804 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; 1805 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 1806 1807 /* Configure N_FTS & FTS */ 1808 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); 1809 val &= ~(N_FTS_MASK << N_FTS_SHIFT); 1810 val |= N_FTS_VAL << N_FTS_SHIFT; 1811 dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); 1812 1813 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); 1814 val &= ~FTS_MASK; 1815 val |= FTS_VAL; 1816 dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); 1817 1818 /* Configure Max Speed from DT */ 1819 if (pcie->max_speed && pcie->max_speed != -EINVAL) { 1820 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + 1821 PCI_EXP_LNKCAP); 1822 val &= ~PCI_EXP_LNKCAP_SLS; 1823 val |= pcie->max_speed; 1824 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, 1825 val); 1826 } 1827 1828 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, 1829 PCI_CAP_ID_EXP); 1830 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); 1831 1832 val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK); 1833 val |= MSIX_ADDR_MATCH_LOW_OFF_EN; 1834 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val); 1835 val = (lower_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK); 1836 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val); 1837 1838 ret = dw_pcie_ep_init_complete(ep); 1839 if (ret) { 1840 dev_err(dev, "Failed to complete initialization: %d\n", ret); 1841 goto fail_init_complete; 1842 } 1843 1844 dw_pcie_ep_init_notify(ep); 1845 1846 /* Enable LTSSM */ 1847 val = appl_readl(pcie, APPL_CTRL); 1848 val |= APPL_CTRL_LTSSM_EN; 1849 appl_writel(pcie, val, APPL_CTRL); 1850 1851 pcie->ep_state = EP_STATE_ENABLED; 1852 dev_dbg(dev, "Initialization of endpoint is completed\n"); 1853 1854 return; 1855 1856 fail_init_complete: 1857 reset_control_assert(pcie->core_rst); 1858 tegra_pcie_disable_phy(pcie); 1859 fail_phy: 1860 reset_control_assert(pcie->core_apb_rst); 1861 fail_core_apb_rst: 1862 clk_disable_unprepare(pcie->core_clk); 1863 fail_core_clk_enable: 1864 tegra_pcie_bpmp_set_pll_state(pcie, false); 1865 fail_pll_init: 1866 pm_runtime_put_sync(dev); 1867 } 1868 1869 static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg) 1870 { 1871 struct tegra_pcie_dw *pcie = arg; 1872 1873 if (gpiod_get_value(pcie->pex_rst_gpiod)) 1874 pex_ep_event_pex_rst_assert(pcie); 1875 else 1876 pex_ep_event_pex_rst_deassert(pcie); 1877 1878 return IRQ_HANDLED; 1879 } 1880 1881 static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq) 1882 { 1883 /* Tegra194 supports only INTA */ 1884 if (irq > 1) 1885 return -EINVAL; 1886 1887 appl_writel(pcie, 1, APPL_LEGACY_INTX); 1888 usleep_range(1000, 2000); 1889 appl_writel(pcie, 0, APPL_LEGACY_INTX); 1890 return 0; 1891 } 1892 1893 static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq) 1894 { 1895 if (unlikely(irq > 31)) 1896 return -EINVAL; 1897 1898 appl_writel(pcie, (1 << irq), APPL_MSI_CTRL_1); 1899 1900 return 0; 1901 } 1902 1903 static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq) 1904 { 1905 struct dw_pcie_ep *ep = &pcie->pci.ep; 1906 1907 writel(irq, ep->msi_mem); 1908 1909 return 0; 1910 } 1911 1912 static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, 1913 enum pci_epc_irq_type type, 1914 u16 interrupt_num) 1915 { 1916 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1917 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 1918 1919 switch (type) { 1920 case PCI_EPC_IRQ_LEGACY: 1921 return tegra_pcie_ep_raise_legacy_irq(pcie, interrupt_num); 1922 1923 case PCI_EPC_IRQ_MSI: 1924 return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num); 1925 1926 case PCI_EPC_IRQ_MSIX: 1927 return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num); 1928 1929 default: 1930 dev_err(pci->dev, "Unknown IRQ type\n"); 1931 return -EPERM; 1932 } 1933 1934 return 0; 1935 } 1936 1937 static const struct pci_epc_features tegra_pcie_epc_features = { 1938 .linkup_notifier = true, 1939 .core_init_notifier = true, 1940 .msi_capable = false, 1941 .msix_capable = false, 1942 .reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5, 1943 .bar_fixed_64bit = 1 << BAR_0, 1944 .bar_fixed_size[0] = SZ_1M, 1945 }; 1946 1947 static const struct pci_epc_features* 1948 tegra_pcie_ep_get_features(struct dw_pcie_ep *ep) 1949 { 1950 return &tegra_pcie_epc_features; 1951 } 1952 1953 static struct dw_pcie_ep_ops pcie_ep_ops = { 1954 .raise_irq = tegra_pcie_ep_raise_irq, 1955 .get_features = tegra_pcie_ep_get_features, 1956 }; 1957 1958 static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie, 1959 struct platform_device *pdev) 1960 { 1961 struct dw_pcie *pci = &pcie->pci; 1962 struct device *dev = pcie->dev; 1963 struct dw_pcie_ep *ep; 1964 struct resource *res; 1965 char *name; 1966 int ret; 1967 1968 ep = &pci->ep; 1969 ep->ops = &pcie_ep_ops; 1970 1971 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space"); 1972 if (!res) 1973 return -EINVAL; 1974 1975 ep->phys_base = res->start; 1976 ep->addr_size = resource_size(res); 1977 ep->page_size = SZ_64K; 1978 1979 ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME); 1980 if (ret < 0) { 1981 dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n", 1982 ret); 1983 return ret; 1984 } 1985 1986 ret = gpiod_to_irq(pcie->pex_rst_gpiod); 1987 if (ret < 0) { 1988 dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret); 1989 return ret; 1990 } 1991 pcie->pex_rst_irq = (unsigned int)ret; 1992 1993 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq", 1994 pcie->cid); 1995 if (!name) { 1996 dev_err(dev, "Failed to create PERST IRQ string\n"); 1997 return -ENOMEM; 1998 } 1999 2000 irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN); 2001 2002 pcie->ep_state = EP_STATE_DISABLED; 2003 2004 ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL, 2005 tegra_pcie_ep_pex_rst_irq, 2006 IRQF_TRIGGER_RISING | 2007 IRQF_TRIGGER_FALLING | IRQF_ONESHOT, 2008 name, (void *)pcie); 2009 if (ret < 0) { 2010 dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret); 2011 return ret; 2012 } 2013 2014 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_ep_work", 2015 pcie->cid); 2016 if (!name) { 2017 dev_err(dev, "Failed to create PCIe EP work thread string\n"); 2018 return -ENOMEM; 2019 } 2020 2021 pm_runtime_enable(dev); 2022 2023 ret = dw_pcie_ep_init(ep); 2024 if (ret) { 2025 dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n", 2026 ret); 2027 return ret; 2028 } 2029 2030 return 0; 2031 } 2032 2033 static int tegra_pcie_dw_probe(struct platform_device *pdev) 2034 { 2035 const struct tegra_pcie_dw_of_data *data; 2036 struct device *dev = &pdev->dev; 2037 struct resource *atu_dma_res; 2038 struct tegra_pcie_dw *pcie; 2039 struct resource *dbi_res; 2040 struct pcie_port *pp; 2041 struct dw_pcie *pci; 2042 struct phy **phys; 2043 char *name; 2044 int ret; 2045 u32 i; 2046 2047 data = of_device_get_match_data(dev); 2048 2049 pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); 2050 if (!pcie) 2051 return -ENOMEM; 2052 2053 pci = &pcie->pci; 2054 pci->dev = &pdev->dev; 2055 pci->ops = &tegra_dw_pcie_ops; 2056 pp = &pci->pp; 2057 pcie->dev = &pdev->dev; 2058 pcie->mode = (enum dw_pcie_device_mode)data->mode; 2059 2060 ret = tegra_pcie_dw_parse_dt(pcie); 2061 if (ret < 0) { 2062 dev_err(dev, "Failed to parse device tree: %d\n", ret); 2063 return ret; 2064 } 2065 2066 ret = tegra_pcie_get_slot_regulators(pcie); 2067 if (ret < 0) { 2068 dev_err(dev, "Failed to get slot regulators: %d\n", ret); 2069 return ret; 2070 } 2071 2072 if (pcie->pex_refclk_sel_gpiod) 2073 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1); 2074 2075 pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl"); 2076 if (IS_ERR(pcie->pex_ctl_supply)) { 2077 ret = PTR_ERR(pcie->pex_ctl_supply); 2078 if (ret != -EPROBE_DEFER) 2079 dev_err(dev, "Failed to get regulator: %ld\n", 2080 PTR_ERR(pcie->pex_ctl_supply)); 2081 return ret; 2082 } 2083 2084 pcie->core_clk = devm_clk_get(dev, "core"); 2085 if (IS_ERR(pcie->core_clk)) { 2086 dev_err(dev, "Failed to get core clock: %ld\n", 2087 PTR_ERR(pcie->core_clk)); 2088 return PTR_ERR(pcie->core_clk); 2089 } 2090 2091 pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 2092 "appl"); 2093 if (!pcie->appl_res) { 2094 dev_err(dev, "Failed to find \"appl\" region\n"); 2095 return -ENODEV; 2096 } 2097 2098 pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res); 2099 if (IS_ERR(pcie->appl_base)) 2100 return PTR_ERR(pcie->appl_base); 2101 2102 pcie->core_apb_rst = devm_reset_control_get(dev, "apb"); 2103 if (IS_ERR(pcie->core_apb_rst)) { 2104 dev_err(dev, "Failed to get APB reset: %ld\n", 2105 PTR_ERR(pcie->core_apb_rst)); 2106 return PTR_ERR(pcie->core_apb_rst); 2107 } 2108 2109 phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL); 2110 if (!phys) 2111 return -ENOMEM; 2112 2113 for (i = 0; i < pcie->phy_count; i++) { 2114 name = kasprintf(GFP_KERNEL, "p2u-%u", i); 2115 if (!name) { 2116 dev_err(dev, "Failed to create P2U string\n"); 2117 return -ENOMEM; 2118 } 2119 phys[i] = devm_phy_get(dev, name); 2120 kfree(name); 2121 if (IS_ERR(phys[i])) { 2122 ret = PTR_ERR(phys[i]); 2123 if (ret != -EPROBE_DEFER) 2124 dev_err(dev, "Failed to get PHY: %d\n", ret); 2125 return ret; 2126 } 2127 } 2128 2129 pcie->phys = phys; 2130 2131 dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); 2132 if (!dbi_res) { 2133 dev_err(dev, "Failed to find \"dbi\" region\n"); 2134 return -ENODEV; 2135 } 2136 pcie->dbi_res = dbi_res; 2137 2138 pci->dbi_base = devm_ioremap_resource(dev, dbi_res); 2139 if (IS_ERR(pci->dbi_base)) 2140 return PTR_ERR(pci->dbi_base); 2141 2142 /* Tegra HW locates DBI2 at a fixed offset from DBI */ 2143 pci->dbi_base2 = pci->dbi_base + 0x1000; 2144 2145 atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 2146 "atu_dma"); 2147 if (!atu_dma_res) { 2148 dev_err(dev, "Failed to find \"atu_dma\" region\n"); 2149 return -ENODEV; 2150 } 2151 pcie->atu_dma_res = atu_dma_res; 2152 2153 pci->atu_base = devm_ioremap_resource(dev, atu_dma_res); 2154 if (IS_ERR(pci->atu_base)) 2155 return PTR_ERR(pci->atu_base); 2156 2157 pcie->core_rst = devm_reset_control_get(dev, "core"); 2158 if (IS_ERR(pcie->core_rst)) { 2159 dev_err(dev, "Failed to get core reset: %ld\n", 2160 PTR_ERR(pcie->core_rst)); 2161 return PTR_ERR(pcie->core_rst); 2162 } 2163 2164 pp->irq = platform_get_irq_byname(pdev, "intr"); 2165 if (!pp->irq) { 2166 dev_err(dev, "Failed to get \"intr\" interrupt\n"); 2167 return -ENODEV; 2168 } 2169 2170 pcie->bpmp = tegra_bpmp_get(dev); 2171 if (IS_ERR(pcie->bpmp)) 2172 return PTR_ERR(pcie->bpmp); 2173 2174 platform_set_drvdata(pdev, pcie); 2175 2176 switch (pcie->mode) { 2177 case DW_PCIE_RC_TYPE: 2178 ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler, 2179 IRQF_SHARED, "tegra-pcie-intr", pcie); 2180 if (ret) { 2181 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq, 2182 ret); 2183 goto fail; 2184 } 2185 2186 ret = tegra_pcie_config_rp(pcie); 2187 if (ret && ret != -ENOMEDIUM) 2188 goto fail; 2189 else 2190 return 0; 2191 break; 2192 2193 case DW_PCIE_EP_TYPE: 2194 ret = devm_request_threaded_irq(dev, pp->irq, 2195 tegra_pcie_ep_hard_irq, 2196 tegra_pcie_ep_irq_thread, 2197 IRQF_SHARED | IRQF_ONESHOT, 2198 "tegra-pcie-ep-intr", pcie); 2199 if (ret) { 2200 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq, 2201 ret); 2202 goto fail; 2203 } 2204 2205 ret = tegra_pcie_config_ep(pcie, pdev); 2206 if (ret < 0) 2207 goto fail; 2208 break; 2209 2210 default: 2211 dev_err(dev, "Invalid PCIe device type %d\n", pcie->mode); 2212 } 2213 2214 fail: 2215 tegra_bpmp_put(pcie->bpmp); 2216 return ret; 2217 } 2218 2219 static int tegra_pcie_dw_remove(struct platform_device *pdev) 2220 { 2221 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); 2222 2223 if (!pcie->link_state) 2224 return 0; 2225 2226 debugfs_remove_recursive(pcie->debugfs); 2227 tegra_pcie_deinit_controller(pcie); 2228 pm_runtime_put_sync(pcie->dev); 2229 pm_runtime_disable(pcie->dev); 2230 tegra_bpmp_put(pcie->bpmp); 2231 if (pcie->pex_refclk_sel_gpiod) 2232 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0); 2233 2234 return 0; 2235 } 2236 2237 static int tegra_pcie_dw_suspend_late(struct device *dev) 2238 { 2239 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2240 u32 val; 2241 2242 if (!pcie->link_state) 2243 return 0; 2244 2245 /* Enable HW_HOT_RST mode */ 2246 val = appl_readl(pcie, APPL_CTRL); 2247 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << 2248 APPL_CTRL_HW_HOT_RST_MODE_SHIFT); 2249 val |= APPL_CTRL_HW_HOT_RST_EN; 2250 appl_writel(pcie, val, APPL_CTRL); 2251 2252 return 0; 2253 } 2254 2255 static int tegra_pcie_dw_suspend_noirq(struct device *dev) 2256 { 2257 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2258 2259 if (!pcie->link_state) 2260 return 0; 2261 2262 /* Save MSI interrupt vector */ 2263 pcie->msi_ctrl_int = dw_pcie_readl_dbi(&pcie->pci, 2264 PORT_LOGIC_MSI_CTRL_INT_0_EN); 2265 tegra_pcie_downstream_dev_to_D0(pcie); 2266 tegra_pcie_dw_pme_turnoff(pcie); 2267 2268 return __deinit_controller(pcie); 2269 } 2270 2271 static int tegra_pcie_dw_resume_noirq(struct device *dev) 2272 { 2273 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2274 int ret; 2275 2276 if (!pcie->link_state) 2277 return 0; 2278 2279 ret = tegra_pcie_config_controller(pcie, true); 2280 if (ret < 0) 2281 return ret; 2282 2283 ret = tegra_pcie_dw_host_init(&pcie->pci.pp); 2284 if (ret < 0) { 2285 dev_err(dev, "Failed to init host: %d\n", ret); 2286 goto fail_host_init; 2287 } 2288 2289 /* Restore MSI interrupt vector */ 2290 dw_pcie_writel_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN, 2291 pcie->msi_ctrl_int); 2292 2293 return 0; 2294 2295 fail_host_init: 2296 return __deinit_controller(pcie); 2297 } 2298 2299 static int tegra_pcie_dw_resume_early(struct device *dev) 2300 { 2301 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2302 u32 val; 2303 2304 if (!pcie->link_state) 2305 return 0; 2306 2307 /* Disable HW_HOT_RST mode */ 2308 val = appl_readl(pcie, APPL_CTRL); 2309 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << 2310 APPL_CTRL_HW_HOT_RST_MODE_SHIFT); 2311 val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST << 2312 APPL_CTRL_HW_HOT_RST_MODE_SHIFT; 2313 val &= ~APPL_CTRL_HW_HOT_RST_EN; 2314 appl_writel(pcie, val, APPL_CTRL); 2315 2316 return 0; 2317 } 2318 2319 static void tegra_pcie_dw_shutdown(struct platform_device *pdev) 2320 { 2321 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); 2322 2323 if (!pcie->link_state) 2324 return; 2325 2326 debugfs_remove_recursive(pcie->debugfs); 2327 tegra_pcie_downstream_dev_to_D0(pcie); 2328 2329 disable_irq(pcie->pci.pp.irq); 2330 if (IS_ENABLED(CONFIG_PCI_MSI)) 2331 disable_irq(pcie->pci.pp.msi_irq); 2332 2333 tegra_pcie_dw_pme_turnoff(pcie); 2334 __deinit_controller(pcie); 2335 } 2336 2337 static const struct tegra_pcie_dw_of_data tegra_pcie_dw_rc_of_data = { 2338 .mode = DW_PCIE_RC_TYPE, 2339 }; 2340 2341 static const struct tegra_pcie_dw_of_data tegra_pcie_dw_ep_of_data = { 2342 .mode = DW_PCIE_EP_TYPE, 2343 }; 2344 2345 static const struct of_device_id tegra_pcie_dw_of_match[] = { 2346 { 2347 .compatible = "nvidia,tegra194-pcie", 2348 .data = &tegra_pcie_dw_rc_of_data, 2349 }, 2350 { 2351 .compatible = "nvidia,tegra194-pcie-ep", 2352 .data = &tegra_pcie_dw_ep_of_data, 2353 }, 2354 {}, 2355 }; 2356 2357 static const struct dev_pm_ops tegra_pcie_dw_pm_ops = { 2358 .suspend_late = tegra_pcie_dw_suspend_late, 2359 .suspend_noirq = tegra_pcie_dw_suspend_noirq, 2360 .resume_noirq = tegra_pcie_dw_resume_noirq, 2361 .resume_early = tegra_pcie_dw_resume_early, 2362 }; 2363 2364 static struct platform_driver tegra_pcie_dw_driver = { 2365 .probe = tegra_pcie_dw_probe, 2366 .remove = tegra_pcie_dw_remove, 2367 .shutdown = tegra_pcie_dw_shutdown, 2368 .driver = { 2369 .name = "tegra194-pcie", 2370 .pm = &tegra_pcie_dw_pm_ops, 2371 .of_match_table = tegra_pcie_dw_of_match, 2372 }, 2373 }; 2374 module_platform_driver(tegra_pcie_dw_driver); 2375 2376 MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match); 2377 2378 MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>"); 2379 MODULE_DESCRIPTION("NVIDIA PCIe host controller driver"); 2380 MODULE_LICENSE("GPL v2"); 2381