1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * PCIe host controller driver for Tegra194 SoC 4 * 5 * Copyright (C) 2019 NVIDIA Corporation. 6 * 7 * Author: Vidya Sagar <vidyas@nvidia.com> 8 */ 9 10 #include <linux/clk.h> 11 #include <linux/debugfs.h> 12 #include <linux/delay.h> 13 #include <linux/gpio.h> 14 #include <linux/gpio/consumer.h> 15 #include <linux/interrupt.h> 16 #include <linux/iopoll.h> 17 #include <linux/kernel.h> 18 #include <linux/module.h> 19 #include <linux/of.h> 20 #include <linux/of_device.h> 21 #include <linux/of_gpio.h> 22 #include <linux/of_irq.h> 23 #include <linux/of_pci.h> 24 #include <linux/pci.h> 25 #include <linux/phy/phy.h> 26 #include <linux/pinctrl/consumer.h> 27 #include <linux/platform_device.h> 28 #include <linux/pm_runtime.h> 29 #include <linux/random.h> 30 #include <linux/reset.h> 31 #include <linux/resource.h> 32 #include <linux/types.h> 33 #include "pcie-designware.h" 34 #include <soc/tegra/bpmp.h> 35 #include <soc/tegra/bpmp-abi.h> 36 #include "../../pci.h" 37 38 #define APPL_PINMUX 0x0 39 #define APPL_PINMUX_PEX_RST BIT(0) 40 #define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2) 41 #define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3) 42 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4) 43 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5) 44 45 #define APPL_CTRL 0x4 46 #define APPL_CTRL_SYS_PRE_DET_STATE BIT(6) 47 #define APPL_CTRL_LTSSM_EN BIT(7) 48 #define APPL_CTRL_HW_HOT_RST_EN BIT(20) 49 #define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0) 50 #define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22 51 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1 52 53 #define APPL_INTR_EN_L0_0 0x8 54 #define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0) 55 #define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4) 56 #define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8) 57 #define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN BIT(15) 58 #define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19) 59 #define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30) 60 #define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31) 61 62 #define APPL_INTR_STATUS_L0 0xC 63 #define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0) 64 #define APPL_INTR_STATUS_L0_INT_INT BIT(8) 65 #define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT BIT(15) 66 #define APPL_INTR_STATUS_L0_PEX_RST_INT BIT(16) 67 #define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18) 68 69 #define APPL_INTR_EN_L1_0_0 0x1C 70 #define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1) 71 #define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN BIT(3) 72 #define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN BIT(30) 73 74 #define APPL_INTR_STATUS_L1_0_0 0x20 75 #define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1) 76 #define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED BIT(3) 77 #define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE BIT(30) 78 79 #define APPL_INTR_STATUS_L1_1 0x2C 80 #define APPL_INTR_STATUS_L1_2 0x30 81 #define APPL_INTR_STATUS_L1_3 0x34 82 #define APPL_INTR_STATUS_L1_6 0x3C 83 #define APPL_INTR_STATUS_L1_7 0x40 84 #define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED BIT(1) 85 86 #define APPL_INTR_EN_L1_8_0 0x44 87 #define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2) 88 #define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3) 89 #define APPL_INTR_EN_L1_8_INTX_EN BIT(11) 90 #define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15) 91 92 #define APPL_INTR_STATUS_L1_8_0 0x4C 93 #define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6) 94 #define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2) 95 #define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3) 96 97 #define APPL_INTR_STATUS_L1_9 0x54 98 #define APPL_INTR_STATUS_L1_10 0x58 99 #define APPL_INTR_STATUS_L1_11 0x64 100 #define APPL_INTR_STATUS_L1_13 0x74 101 #define APPL_INTR_STATUS_L1_14 0x78 102 #define APPL_INTR_STATUS_L1_15 0x7C 103 #define APPL_INTR_STATUS_L1_17 0x88 104 105 #define APPL_INTR_EN_L1_18 0x90 106 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2) 107 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) 108 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) 109 110 #define APPL_INTR_STATUS_L1_18 0x94 111 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2) 112 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) 113 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) 114 115 #define APPL_MSI_CTRL_1 0xAC 116 117 #define APPL_MSI_CTRL_2 0xB0 118 119 #define APPL_LEGACY_INTX 0xB8 120 121 #define APPL_LTR_MSG_1 0xC4 122 #define LTR_MSG_REQ BIT(15) 123 #define LTR_MST_NO_SNOOP_SHIFT 16 124 125 #define APPL_LTR_MSG_2 0xC8 126 #define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3) 127 128 #define APPL_LINK_STATUS 0xCC 129 #define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0) 130 131 #define APPL_DEBUG 0xD0 132 #define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21) 133 #define APPL_DEBUG_PM_LINKST_IN_L0 0x11 134 #define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3) 135 #define APPL_DEBUG_LTSSM_STATE_SHIFT 3 136 #define LTSSM_STATE_PRE_DETECT 5 137 138 #define APPL_RADM_STATUS 0xE4 139 #define APPL_PM_XMT_TURNOFF_STATE BIT(0) 140 141 #define APPL_DM_TYPE 0x100 142 #define APPL_DM_TYPE_MASK GENMASK(3, 0) 143 #define APPL_DM_TYPE_RP 0x4 144 #define APPL_DM_TYPE_EP 0x0 145 146 #define APPL_CFG_BASE_ADDR 0x104 147 #define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12) 148 149 #define APPL_CFG_IATU_DMA_BASE_ADDR 0x108 150 #define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18) 151 152 #define APPL_CFG_MISC 0x110 153 #define APPL_CFG_MISC_SLV_EP_MODE BIT(14) 154 #define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10) 155 #define APPL_CFG_MISC_ARCACHE_SHIFT 10 156 #define APPL_CFG_MISC_ARCACHE_VAL 3 157 158 #define APPL_CFG_SLCG_OVERRIDE 0x114 159 #define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0) 160 161 #define APPL_CAR_RESET_OVRD 0x12C 162 #define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0) 163 164 #define IO_BASE_IO_DECODE BIT(0) 165 #define IO_BASE_IO_DECODE_BIT8 BIT(8) 166 167 #define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0) 168 #define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16) 169 170 #define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718 171 #define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19) 172 173 #define EVENT_COUNTER_ALL_CLEAR 0x3 174 #define EVENT_COUNTER_ENABLE_ALL 0x7 175 #define EVENT_COUNTER_ENABLE_SHIFT 2 176 #define EVENT_COUNTER_EVENT_SEL_MASK GENMASK(7, 0) 177 #define EVENT_COUNTER_EVENT_SEL_SHIFT 16 178 #define EVENT_COUNTER_EVENT_Tx_L0S 0x2 179 #define EVENT_COUNTER_EVENT_Rx_L0S 0x3 180 #define EVENT_COUNTER_EVENT_L1 0x5 181 #define EVENT_COUNTER_EVENT_L1_1 0x7 182 #define EVENT_COUNTER_EVENT_L1_2 0x8 183 #define EVENT_COUNTER_GROUP_SEL_SHIFT 24 184 #define EVENT_COUNTER_GROUP_5 0x5 185 186 #define PORT_LOGIC_ACK_F_ASPM_CTRL 0x70C 187 #define ENTER_ASPM BIT(30) 188 #define L0S_ENTRANCE_LAT_SHIFT 24 189 #define L0S_ENTRANCE_LAT_MASK GENMASK(26, 24) 190 #define L1_ENTRANCE_LAT_SHIFT 27 191 #define L1_ENTRANCE_LAT_MASK GENMASK(29, 27) 192 #define N_FTS_SHIFT 8 193 #define N_FTS_MASK GENMASK(7, 0) 194 #define N_FTS_VAL 52 195 196 #define PORT_LOGIC_GEN2_CTRL 0x80C 197 #define PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE BIT(17) 198 #define FTS_MASK GENMASK(7, 0) 199 #define FTS_VAL 52 200 201 #define PORT_LOGIC_MSI_CTRL_INT_0_EN 0x828 202 203 #define GEN3_EQ_CONTROL_OFF 0x8a8 204 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8 205 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8) 206 #define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0) 207 208 #define GEN3_RELATED_OFF 0x890 209 #define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL BIT(0) 210 #define GEN3_RELATED_OFF_GEN3_EQ_DISABLE BIT(16) 211 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT 24 212 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK GENMASK(25, 24) 213 214 #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0 215 #define AMBA_ERROR_RESPONSE_CRS_SHIFT 3 216 #define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0) 217 #define AMBA_ERROR_RESPONSE_CRS_OKAY 0 218 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1 219 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2 220 221 #define MSIX_ADDR_MATCH_LOW_OFF 0x940 222 #define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0) 223 #define MSIX_ADDR_MATCH_LOW_OFF_MASK GENMASK(31, 2) 224 225 #define MSIX_ADDR_MATCH_HIGH_OFF 0x944 226 #define MSIX_ADDR_MATCH_HIGH_OFF_MASK GENMASK(31, 0) 227 228 #define PORT_LOGIC_MSIX_DOORBELL 0x948 229 230 #define CAP_SPCIE_CAP_OFF 0x154 231 #define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0) 232 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8) 233 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8 234 235 #define PME_ACK_TIMEOUT 10000 236 237 #define LTSSM_TIMEOUT 50000 /* 50ms */ 238 239 #define GEN3_GEN4_EQ_PRESET_INIT 5 240 241 #define GEN1_CORE_CLK_FREQ 62500000 242 #define GEN2_CORE_CLK_FREQ 125000000 243 #define GEN3_CORE_CLK_FREQ 250000000 244 #define GEN4_CORE_CLK_FREQ 500000000 245 246 #define LTR_MSG_TIMEOUT (100 * 1000) 247 248 #define PERST_DEBOUNCE_TIME (5 * 1000) 249 250 #define EP_STATE_DISABLED 0 251 #define EP_STATE_ENABLED 1 252 253 static const unsigned int pcie_gen_freq[] = { 254 GEN1_CORE_CLK_FREQ, 255 GEN2_CORE_CLK_FREQ, 256 GEN3_CORE_CLK_FREQ, 257 GEN4_CORE_CLK_FREQ 258 }; 259 260 static const u32 event_cntr_ctrl_offset[] = { 261 0x1d8, 262 0x1a8, 263 0x1a8, 264 0x1a8, 265 0x1c4, 266 0x1d8 267 }; 268 269 static const u32 event_cntr_data_offset[] = { 270 0x1dc, 271 0x1ac, 272 0x1ac, 273 0x1ac, 274 0x1c8, 275 0x1dc 276 }; 277 278 struct tegra_pcie_dw { 279 struct device *dev; 280 struct resource *appl_res; 281 struct resource *dbi_res; 282 struct resource *atu_dma_res; 283 void __iomem *appl_base; 284 struct clk *core_clk; 285 struct reset_control *core_apb_rst; 286 struct reset_control *core_rst; 287 struct dw_pcie pci; 288 struct tegra_bpmp *bpmp; 289 290 enum dw_pcie_device_mode mode; 291 292 bool supports_clkreq; 293 bool enable_cdm_check; 294 bool link_state; 295 bool update_fc_fixup; 296 u8 init_link_width; 297 u32 msi_ctrl_int; 298 u32 num_lanes; 299 u32 max_speed; 300 u32 cid; 301 u32 cfg_link_cap_l1sub; 302 u32 pcie_cap_base; 303 u32 aspm_cmrt; 304 u32 aspm_pwr_on_t; 305 u32 aspm_l0s_enter_lat; 306 307 struct regulator *pex_ctl_supply; 308 struct regulator *slot_ctl_3v3; 309 struct regulator *slot_ctl_12v; 310 311 unsigned int phy_count; 312 struct phy **phys; 313 314 struct dentry *debugfs; 315 316 /* Endpoint mode specific */ 317 struct gpio_desc *pex_rst_gpiod; 318 struct gpio_desc *pex_refclk_sel_gpiod; 319 unsigned int pex_rst_irq; 320 int ep_state; 321 }; 322 323 struct tegra_pcie_dw_of_data { 324 enum dw_pcie_device_mode mode; 325 }; 326 327 static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci) 328 { 329 return container_of(pci, struct tegra_pcie_dw, pci); 330 } 331 332 static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value, 333 const u32 reg) 334 { 335 writel_relaxed(value, pcie->appl_base + reg); 336 } 337 338 static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg) 339 { 340 return readl_relaxed(pcie->appl_base + reg); 341 } 342 343 struct tegra_pcie_soc { 344 enum dw_pcie_device_mode mode; 345 }; 346 347 static void apply_bad_link_workaround(struct pcie_port *pp) 348 { 349 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 350 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 351 u32 current_link_width; 352 u16 val; 353 354 /* 355 * NOTE:- Since this scenario is uncommon and link as such is not 356 * stable anyway, not waiting to confirm if link is really 357 * transitioning to Gen-2 speed 358 */ 359 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); 360 if (val & PCI_EXP_LNKSTA_LBMS) { 361 current_link_width = (val & PCI_EXP_LNKSTA_NLW) >> 362 PCI_EXP_LNKSTA_NLW_SHIFT; 363 if (pcie->init_link_width > current_link_width) { 364 dev_warn(pci->dev, "PCIe link is bad, width reduced\n"); 365 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + 366 PCI_EXP_LNKCTL2); 367 val &= ~PCI_EXP_LNKCTL2_TLS; 368 val |= PCI_EXP_LNKCTL2_TLS_2_5GT; 369 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + 370 PCI_EXP_LNKCTL2, val); 371 372 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + 373 PCI_EXP_LNKCTL); 374 val |= PCI_EXP_LNKCTL_RL; 375 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + 376 PCI_EXP_LNKCTL, val); 377 } 378 } 379 } 380 381 static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg) 382 { 383 struct tegra_pcie_dw *pcie = arg; 384 struct dw_pcie *pci = &pcie->pci; 385 struct pcie_port *pp = &pci->pp; 386 u32 val, tmp; 387 u16 val_w; 388 389 val = appl_readl(pcie, APPL_INTR_STATUS_L0); 390 if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) { 391 val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); 392 if (val & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) { 393 appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0); 394 395 /* SBR & Surprise Link Down WAR */ 396 val = appl_readl(pcie, APPL_CAR_RESET_OVRD); 397 val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; 398 appl_writel(pcie, val, APPL_CAR_RESET_OVRD); 399 udelay(1); 400 val = appl_readl(pcie, APPL_CAR_RESET_OVRD); 401 val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; 402 appl_writel(pcie, val, APPL_CAR_RESET_OVRD); 403 404 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); 405 val |= PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE; 406 dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); 407 } 408 } 409 410 if (val & APPL_INTR_STATUS_L0_INT_INT) { 411 val = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0); 412 if (val & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) { 413 appl_writel(pcie, 414 APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS, 415 APPL_INTR_STATUS_L1_8_0); 416 apply_bad_link_workaround(pp); 417 } 418 if (val & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) { 419 appl_writel(pcie, 420 APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS, 421 APPL_INTR_STATUS_L1_8_0); 422 423 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + 424 PCI_EXP_LNKSTA); 425 dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w & 426 PCI_EXP_LNKSTA_CLS); 427 } 428 } 429 430 val = appl_readl(pcie, APPL_INTR_STATUS_L0); 431 if (val & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) { 432 val = appl_readl(pcie, APPL_INTR_STATUS_L1_18); 433 tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS); 434 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) { 435 dev_info(pci->dev, "CDM check complete\n"); 436 tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE; 437 } 438 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) { 439 dev_err(pci->dev, "CDM comparison mismatch\n"); 440 tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR; 441 } 442 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) { 443 dev_err(pci->dev, "CDM Logic error\n"); 444 tmp |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR; 445 } 446 dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, tmp); 447 tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR); 448 dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", tmp); 449 } 450 451 return IRQ_HANDLED; 452 } 453 454 static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie) 455 { 456 u32 val; 457 458 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); 459 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); 460 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); 461 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); 462 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); 463 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); 464 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); 465 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); 466 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); 467 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); 468 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); 469 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); 470 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); 471 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); 472 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); 473 appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2); 474 475 val = appl_readl(pcie, APPL_CTRL); 476 val |= APPL_CTRL_LTSSM_EN; 477 appl_writel(pcie, val, APPL_CTRL); 478 } 479 480 static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg) 481 { 482 struct tegra_pcie_dw *pcie = arg; 483 struct dw_pcie *pci = &pcie->pci; 484 u32 val, speed; 485 486 speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) & 487 PCI_EXP_LNKSTA_CLS; 488 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]); 489 490 /* If EP doesn't advertise L1SS, just return */ 491 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub); 492 if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2))) 493 return IRQ_HANDLED; 494 495 /* Check if BME is set to '1' */ 496 val = dw_pcie_readl_dbi(pci, PCI_COMMAND); 497 if (val & PCI_COMMAND_MASTER) { 498 ktime_t timeout; 499 500 /* 110us for both snoop and no-snoop */ 501 val = 110 | (2 << PCI_LTR_SCALE_SHIFT) | LTR_MSG_REQ; 502 val |= (val << LTR_MST_NO_SNOOP_SHIFT); 503 appl_writel(pcie, val, APPL_LTR_MSG_1); 504 505 /* Send LTR upstream */ 506 val = appl_readl(pcie, APPL_LTR_MSG_2); 507 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE; 508 appl_writel(pcie, val, APPL_LTR_MSG_2); 509 510 timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT); 511 for (;;) { 512 val = appl_readl(pcie, APPL_LTR_MSG_2); 513 if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)) 514 break; 515 if (ktime_after(ktime_get(), timeout)) 516 break; 517 usleep_range(1000, 1100); 518 } 519 if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE) 520 dev_err(pcie->dev, "Failed to send LTR message\n"); 521 } 522 523 return IRQ_HANDLED; 524 } 525 526 static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg) 527 { 528 struct tegra_pcie_dw *pcie = arg; 529 struct dw_pcie_ep *ep = &pcie->pci.ep; 530 int spurious = 1; 531 u32 val, tmp; 532 533 val = appl_readl(pcie, APPL_INTR_STATUS_L0); 534 if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) { 535 val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); 536 appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0); 537 538 if (val & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE) 539 pex_ep_event_hot_rst_done(pcie); 540 541 if (val & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) { 542 tmp = appl_readl(pcie, APPL_LINK_STATUS); 543 if (tmp & APPL_LINK_STATUS_RDLH_LINK_UP) { 544 dev_dbg(pcie->dev, "Link is up with Host\n"); 545 dw_pcie_ep_linkup(ep); 546 } 547 } 548 549 spurious = 0; 550 } 551 552 if (val & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) { 553 val = appl_readl(pcie, APPL_INTR_STATUS_L1_15); 554 appl_writel(pcie, val, APPL_INTR_STATUS_L1_15); 555 556 if (val & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED) 557 return IRQ_WAKE_THREAD; 558 559 spurious = 0; 560 } 561 562 if (spurious) { 563 dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n", 564 val); 565 appl_writel(pcie, val, APPL_INTR_STATUS_L0); 566 } 567 568 return IRQ_HANDLED; 569 } 570 571 static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where, 572 int size, u32 *val) 573 { 574 /* 575 * This is an endpoint mode specific register happen to appear even 576 * when controller is operating in root port mode and system hangs 577 * when it is accessed with link being in ASPM-L1 state. 578 * So skip accessing it altogether 579 */ 580 if (!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) { 581 *val = 0x00000000; 582 return PCIBIOS_SUCCESSFUL; 583 } 584 585 return pci_generic_config_read(bus, devfn, where, size, val); 586 } 587 588 static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where, 589 int size, u32 val) 590 { 591 /* 592 * This is an endpoint mode specific register happen to appear even 593 * when controller is operating in root port mode and system hangs 594 * when it is accessed with link being in ASPM-L1 state. 595 * So skip accessing it altogether 596 */ 597 if (!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) 598 return PCIBIOS_SUCCESSFUL; 599 600 return pci_generic_config_write(bus, devfn, where, size, val); 601 } 602 603 static struct pci_ops tegra_pci_ops = { 604 .map_bus = dw_pcie_own_conf_map_bus, 605 .read = tegra_pcie_dw_rd_own_conf, 606 .write = tegra_pcie_dw_wr_own_conf, 607 }; 608 609 #if defined(CONFIG_PCIEASPM) 610 static void disable_aspm_l11(struct tegra_pcie_dw *pcie) 611 { 612 u32 val; 613 614 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); 615 val &= ~PCI_L1SS_CAP_ASPM_L1_1; 616 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); 617 } 618 619 static void disable_aspm_l12(struct tegra_pcie_dw *pcie) 620 { 621 u32 val; 622 623 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); 624 val &= ~PCI_L1SS_CAP_ASPM_L1_2; 625 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); 626 } 627 628 static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event) 629 { 630 u32 val; 631 632 val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid]); 633 val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT); 634 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; 635 val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT; 636 val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; 637 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val); 638 val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_data_offset[pcie->cid]); 639 640 return val; 641 } 642 643 static int aspm_state_cnt(struct seq_file *s, void *data) 644 { 645 struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *) 646 dev_get_drvdata(s->private); 647 u32 val; 648 649 seq_printf(s, "Tx L0s entry count : %u\n", 650 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S)); 651 652 seq_printf(s, "Rx L0s entry count : %u\n", 653 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S)); 654 655 seq_printf(s, "Link L1 entry count : %u\n", 656 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1)); 657 658 seq_printf(s, "Link L1.1 entry count : %u\n", 659 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1)); 660 661 seq_printf(s, "Link L1.2 entry count : %u\n", 662 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2)); 663 664 /* Clear all counters */ 665 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], 666 EVENT_COUNTER_ALL_CLEAR); 667 668 /* Re-enable counting */ 669 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; 670 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; 671 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val); 672 673 return 0; 674 } 675 676 static void init_host_aspm(struct tegra_pcie_dw *pcie) 677 { 678 struct dw_pcie *pci = &pcie->pci; 679 u32 val; 680 681 val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS); 682 pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP; 683 684 /* Enable ASPM counters */ 685 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; 686 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; 687 dw_pcie_writel_dbi(pci, event_cntr_ctrl_offset[pcie->cid], val); 688 689 /* Program T_cmrt and T_pwr_on values */ 690 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub); 691 val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE); 692 val |= (pcie->aspm_cmrt << 8); 693 val |= (pcie->aspm_pwr_on_t << 19); 694 dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val); 695 696 /* Program L0s and L1 entrance latencies */ 697 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); 698 val &= ~L0S_ENTRANCE_LAT_MASK; 699 val |= (pcie->aspm_l0s_enter_lat << L0S_ENTRANCE_LAT_SHIFT); 700 val |= ENTER_ASPM; 701 dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); 702 } 703 704 static int init_debugfs(struct tegra_pcie_dw *pcie) 705 { 706 struct dentry *d; 707 708 d = debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", 709 pcie->debugfs, aspm_state_cnt); 710 if (IS_ERR_OR_NULL(d)) 711 dev_err(pcie->dev, 712 "Failed to create debugfs file \"aspm_state_cnt\"\n"); 713 714 return 0; 715 } 716 #else 717 static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; } 718 static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; } 719 static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; } 720 static inline int init_debugfs(struct tegra_pcie_dw *pcie) { return 0; } 721 #endif 722 723 static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp) 724 { 725 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 726 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 727 u32 val; 728 u16 val_w; 729 730 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 731 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN; 732 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 733 734 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); 735 val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN; 736 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); 737 738 if (pcie->enable_cdm_check) { 739 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 740 val |= APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN; 741 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 742 743 val = appl_readl(pcie, APPL_INTR_EN_L1_18); 744 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR; 745 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR; 746 appl_writel(pcie, val, APPL_INTR_EN_L1_18); 747 } 748 749 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + 750 PCI_EXP_LNKSTA); 751 pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >> 752 PCI_EXP_LNKSTA_NLW_SHIFT; 753 754 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + 755 PCI_EXP_LNKCTL); 756 val_w |= PCI_EXP_LNKCTL_LBMIE; 757 dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL, 758 val_w); 759 } 760 761 static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp) 762 { 763 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 764 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 765 u32 val; 766 767 /* Enable legacy interrupt generation */ 768 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 769 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN; 770 val |= APPL_INTR_EN_L0_0_INT_INT_EN; 771 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 772 773 val = appl_readl(pcie, APPL_INTR_EN_L1_8_0); 774 val |= APPL_INTR_EN_L1_8_INTX_EN; 775 val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN; 776 val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN; 777 if (IS_ENABLED(CONFIG_PCIEAER)) 778 val |= APPL_INTR_EN_L1_8_AER_INT_EN; 779 appl_writel(pcie, val, APPL_INTR_EN_L1_8_0); 780 } 781 782 static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp) 783 { 784 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 785 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 786 u32 val; 787 788 dw_pcie_msi_init(pp); 789 790 /* Enable MSI interrupt generation */ 791 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 792 val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN; 793 val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN; 794 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 795 } 796 797 static void tegra_pcie_enable_interrupts(struct pcie_port *pp) 798 { 799 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 800 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 801 802 /* Clear interrupt statuses before enabling interrupts */ 803 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); 804 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); 805 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); 806 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); 807 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); 808 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); 809 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); 810 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); 811 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); 812 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); 813 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); 814 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); 815 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); 816 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); 817 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); 818 819 tegra_pcie_enable_system_interrupts(pp); 820 tegra_pcie_enable_legacy_interrupts(pp); 821 if (IS_ENABLED(CONFIG_PCI_MSI)) 822 tegra_pcie_enable_msi_interrupts(pp); 823 } 824 825 static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie) 826 { 827 struct dw_pcie *pci = &pcie->pci; 828 u32 val, offset, i; 829 830 /* Program init preset */ 831 for (i = 0; i < pcie->num_lanes; i++) { 832 dw_pcie_read(pci->dbi_base + CAP_SPCIE_CAP_OFF 833 + (i * 2), 2, &val); 834 val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK; 835 val |= GEN3_GEN4_EQ_PRESET_INIT; 836 val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK; 837 val |= (GEN3_GEN4_EQ_PRESET_INIT << 838 CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT); 839 dw_pcie_write(pci->dbi_base + CAP_SPCIE_CAP_OFF 840 + (i * 2), 2, val); 841 842 offset = dw_pcie_find_ext_capability(pci, 843 PCI_EXT_CAP_ID_PL_16GT) + 844 PCI_PL_16GT_LE_CTRL; 845 dw_pcie_read(pci->dbi_base + offset + i, 1, &val); 846 val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK; 847 val |= GEN3_GEN4_EQ_PRESET_INIT; 848 val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK; 849 val |= (GEN3_GEN4_EQ_PRESET_INIT << 850 PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT); 851 dw_pcie_write(pci->dbi_base + offset + i, 1, val); 852 } 853 854 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 855 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; 856 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 857 858 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); 859 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; 860 val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); 861 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; 862 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); 863 864 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 865 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; 866 val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT); 867 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 868 869 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); 870 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; 871 val |= (0x360 << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); 872 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; 873 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); 874 875 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 876 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; 877 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 878 } 879 880 static void tegra_pcie_prepare_host(struct pcie_port *pp) 881 { 882 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 883 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 884 u32 val; 885 886 val = dw_pcie_readl_dbi(pci, PCI_IO_BASE); 887 val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8); 888 dw_pcie_writel_dbi(pci, PCI_IO_BASE, val); 889 890 val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE); 891 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE; 892 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE; 893 dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val); 894 895 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); 896 897 /* Configure FTS */ 898 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); 899 val &= ~(N_FTS_MASK << N_FTS_SHIFT); 900 val |= N_FTS_VAL << N_FTS_SHIFT; 901 dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); 902 903 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); 904 val &= ~FTS_MASK; 905 val |= FTS_VAL; 906 dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); 907 908 /* Enable as 0xFFFF0001 response for CRS */ 909 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT); 910 val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT); 911 val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 << 912 AMBA_ERROR_RESPONSE_CRS_SHIFT); 913 dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val); 914 915 /* Configure Max Speed from DT */ 916 if (pcie->max_speed && pcie->max_speed != -EINVAL) { 917 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + 918 PCI_EXP_LNKCAP); 919 val &= ~PCI_EXP_LNKCAP_SLS; 920 val |= pcie->max_speed; 921 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, 922 val); 923 } 924 925 /* Configure Max lane width from DT */ 926 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP); 927 val &= ~PCI_EXP_LNKCAP_MLW; 928 val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT); 929 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val); 930 931 config_gen3_gen4_eq_presets(pcie); 932 933 init_host_aspm(pcie); 934 935 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 936 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; 937 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 938 939 if (pcie->update_fc_fixup) { 940 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); 941 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT; 942 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val); 943 } 944 945 dw_pcie_setup_rc(pp); 946 947 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); 948 949 /* Assert RST */ 950 val = appl_readl(pcie, APPL_PINMUX); 951 val &= ~APPL_PINMUX_PEX_RST; 952 appl_writel(pcie, val, APPL_PINMUX); 953 954 usleep_range(100, 200); 955 956 /* Enable LTSSM */ 957 val = appl_readl(pcie, APPL_CTRL); 958 val |= APPL_CTRL_LTSSM_EN; 959 appl_writel(pcie, val, APPL_CTRL); 960 961 /* De-assert RST */ 962 val = appl_readl(pcie, APPL_PINMUX); 963 val |= APPL_PINMUX_PEX_RST; 964 appl_writel(pcie, val, APPL_PINMUX); 965 966 msleep(100); 967 } 968 969 static int tegra_pcie_dw_host_init(struct pcie_port *pp) 970 { 971 struct dw_pcie *pci = to_dw_pcie_from_pp(pp); 972 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 973 u32 val, tmp, offset, speed; 974 975 pp->bridge->ops = &tegra_pci_ops; 976 977 tegra_pcie_prepare_host(pp); 978 979 if (dw_pcie_wait_for_link(pci)) { 980 /* 981 * There are some endpoints which can't get the link up if 982 * root port has Data Link Feature (DLF) enabled. 983 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info 984 * on Scaled Flow Control and DLF. 985 * So, need to confirm that is indeed the case here and attempt 986 * link up once again with DLF disabled. 987 */ 988 val = appl_readl(pcie, APPL_DEBUG); 989 val &= APPL_DEBUG_LTSSM_STATE_MASK; 990 val >>= APPL_DEBUG_LTSSM_STATE_SHIFT; 991 tmp = appl_readl(pcie, APPL_LINK_STATUS); 992 tmp &= APPL_LINK_STATUS_RDLH_LINK_UP; 993 if (!(val == 0x11 && !tmp)) { 994 /* Link is down for all good reasons */ 995 return 0; 996 } 997 998 dev_info(pci->dev, "Link is down in DLL"); 999 dev_info(pci->dev, "Trying again with DLFE disabled\n"); 1000 /* Disable LTSSM */ 1001 val = appl_readl(pcie, APPL_CTRL); 1002 val &= ~APPL_CTRL_LTSSM_EN; 1003 appl_writel(pcie, val, APPL_CTRL); 1004 1005 reset_control_assert(pcie->core_rst); 1006 reset_control_deassert(pcie->core_rst); 1007 1008 offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF); 1009 val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP); 1010 val &= ~PCI_DLF_EXCHANGE_ENABLE; 1011 dw_pcie_writel_dbi(pci, offset, val); 1012 1013 tegra_pcie_prepare_host(pp); 1014 1015 if (dw_pcie_wait_for_link(pci)) 1016 return 0; 1017 } 1018 1019 speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) & 1020 PCI_EXP_LNKSTA_CLS; 1021 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]); 1022 1023 tegra_pcie_enable_interrupts(pp); 1024 1025 return 0; 1026 } 1027 1028 static int tegra_pcie_dw_link_up(struct dw_pcie *pci) 1029 { 1030 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 1031 u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); 1032 1033 return !!(val & PCI_EXP_LNKSTA_DLLLA); 1034 } 1035 1036 static void tegra_pcie_set_msi_vec_num(struct pcie_port *pp) 1037 { 1038 pp->num_vectors = MAX_MSI_IRQS; 1039 } 1040 1041 static int tegra_pcie_dw_start_link(struct dw_pcie *pci) 1042 { 1043 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 1044 1045 enable_irq(pcie->pex_rst_irq); 1046 1047 return 0; 1048 } 1049 1050 static void tegra_pcie_dw_stop_link(struct dw_pcie *pci) 1051 { 1052 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 1053 1054 disable_irq(pcie->pex_rst_irq); 1055 } 1056 1057 static const struct dw_pcie_ops tegra_dw_pcie_ops = { 1058 .link_up = tegra_pcie_dw_link_up, 1059 .start_link = tegra_pcie_dw_start_link, 1060 .stop_link = tegra_pcie_dw_stop_link, 1061 }; 1062 1063 static struct dw_pcie_host_ops tegra_pcie_dw_host_ops = { 1064 .host_init = tegra_pcie_dw_host_init, 1065 .set_num_vectors = tegra_pcie_set_msi_vec_num, 1066 }; 1067 1068 static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie) 1069 { 1070 unsigned int phy_count = pcie->phy_count; 1071 1072 while (phy_count--) { 1073 phy_power_off(pcie->phys[phy_count]); 1074 phy_exit(pcie->phys[phy_count]); 1075 } 1076 } 1077 1078 static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie) 1079 { 1080 unsigned int i; 1081 int ret; 1082 1083 for (i = 0; i < pcie->phy_count; i++) { 1084 ret = phy_init(pcie->phys[i]); 1085 if (ret < 0) 1086 goto phy_power_off; 1087 1088 ret = phy_power_on(pcie->phys[i]); 1089 if (ret < 0) 1090 goto phy_exit; 1091 } 1092 1093 return 0; 1094 1095 phy_power_off: 1096 while (i--) { 1097 phy_power_off(pcie->phys[i]); 1098 phy_exit: 1099 phy_exit(pcie->phys[i]); 1100 } 1101 1102 return ret; 1103 } 1104 1105 static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie) 1106 { 1107 struct device_node *np = pcie->dev->of_node; 1108 int ret; 1109 1110 ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt); 1111 if (ret < 0) { 1112 dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret); 1113 return ret; 1114 } 1115 1116 ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us", 1117 &pcie->aspm_pwr_on_t); 1118 if (ret < 0) 1119 dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n", 1120 ret); 1121 1122 ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us", 1123 &pcie->aspm_l0s_enter_lat); 1124 if (ret < 0) 1125 dev_info(pcie->dev, 1126 "Failed to read ASPM L0s Entrance latency: %d\n", ret); 1127 1128 ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes); 1129 if (ret < 0) { 1130 dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret); 1131 return ret; 1132 } 1133 1134 pcie->max_speed = of_pci_get_max_link_speed(np); 1135 1136 ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid); 1137 if (ret) { 1138 dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret); 1139 return ret; 1140 } 1141 1142 ret = of_property_count_strings(np, "phy-names"); 1143 if (ret < 0) { 1144 dev_err(pcie->dev, "Failed to find PHY entries: %d\n", 1145 ret); 1146 return ret; 1147 } 1148 pcie->phy_count = ret; 1149 1150 if (of_property_read_bool(np, "nvidia,update-fc-fixup")) 1151 pcie->update_fc_fixup = true; 1152 1153 pcie->supports_clkreq = 1154 of_property_read_bool(pcie->dev->of_node, "supports-clkreq"); 1155 1156 pcie->enable_cdm_check = 1157 of_property_read_bool(np, "snps,enable-cdm-check"); 1158 1159 if (pcie->mode == DW_PCIE_RC_TYPE) 1160 return 0; 1161 1162 /* Endpoint mode specific DT entries */ 1163 pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN); 1164 if (IS_ERR(pcie->pex_rst_gpiod)) { 1165 int err = PTR_ERR(pcie->pex_rst_gpiod); 1166 const char *level = KERN_ERR; 1167 1168 if (err == -EPROBE_DEFER) 1169 level = KERN_DEBUG; 1170 1171 dev_printk(level, pcie->dev, 1172 dev_fmt("Failed to get PERST GPIO: %d\n"), 1173 err); 1174 return err; 1175 } 1176 1177 pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev, 1178 "nvidia,refclk-select", 1179 GPIOD_OUT_HIGH); 1180 if (IS_ERR(pcie->pex_refclk_sel_gpiod)) { 1181 int err = PTR_ERR(pcie->pex_refclk_sel_gpiod); 1182 const char *level = KERN_ERR; 1183 1184 if (err == -EPROBE_DEFER) 1185 level = KERN_DEBUG; 1186 1187 dev_printk(level, pcie->dev, 1188 dev_fmt("Failed to get REFCLK select GPIOs: %d\n"), 1189 err); 1190 pcie->pex_refclk_sel_gpiod = NULL; 1191 } 1192 1193 return 0; 1194 } 1195 1196 static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie, 1197 bool enable) 1198 { 1199 struct mrq_uphy_response resp; 1200 struct tegra_bpmp_message msg; 1201 struct mrq_uphy_request req; 1202 1203 /* Controller-5 doesn't need to have its state set by BPMP-FW */ 1204 if (pcie->cid == 5) 1205 return 0; 1206 1207 memset(&req, 0, sizeof(req)); 1208 memset(&resp, 0, sizeof(resp)); 1209 1210 req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE; 1211 req.controller_state.pcie_controller = pcie->cid; 1212 req.controller_state.enable = enable; 1213 1214 memset(&msg, 0, sizeof(msg)); 1215 msg.mrq = MRQ_UPHY; 1216 msg.tx.data = &req; 1217 msg.tx.size = sizeof(req); 1218 msg.rx.data = &resp; 1219 msg.rx.size = sizeof(resp); 1220 1221 return tegra_bpmp_transfer(pcie->bpmp, &msg); 1222 } 1223 1224 static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie, 1225 bool enable) 1226 { 1227 struct mrq_uphy_response resp; 1228 struct tegra_bpmp_message msg; 1229 struct mrq_uphy_request req; 1230 1231 memset(&req, 0, sizeof(req)); 1232 memset(&resp, 0, sizeof(resp)); 1233 1234 if (enable) { 1235 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT; 1236 req.ep_ctrlr_pll_init.ep_controller = pcie->cid; 1237 } else { 1238 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF; 1239 req.ep_ctrlr_pll_off.ep_controller = pcie->cid; 1240 } 1241 1242 memset(&msg, 0, sizeof(msg)); 1243 msg.mrq = MRQ_UPHY; 1244 msg.tx.data = &req; 1245 msg.tx.size = sizeof(req); 1246 msg.rx.data = &resp; 1247 msg.rx.size = sizeof(resp); 1248 1249 return tegra_bpmp_transfer(pcie->bpmp, &msg); 1250 } 1251 1252 static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie) 1253 { 1254 struct pcie_port *pp = &pcie->pci.pp; 1255 struct pci_bus *child, *root_bus = NULL; 1256 struct pci_dev *pdev; 1257 1258 /* 1259 * link doesn't go into L2 state with some of the endpoints with Tegra 1260 * if they are not in D0 state. So, need to make sure that immediate 1261 * downstream devices are in D0 state before sending PME_TurnOff to put 1262 * link into L2 state. 1263 * This is as per PCI Express Base r4.0 v1.0 September 27-2017, 1264 * 5.2 Link State Power Management (Page #428). 1265 */ 1266 1267 list_for_each_entry(child, &pp->root_bus->children, node) { 1268 /* Bring downstream devices to D0 if they are not already in */ 1269 if (child->parent == pp->root_bus) { 1270 root_bus = child; 1271 break; 1272 } 1273 } 1274 1275 if (!root_bus) { 1276 dev_err(pcie->dev, "Failed to find downstream devices\n"); 1277 return; 1278 } 1279 1280 list_for_each_entry(pdev, &root_bus->devices, bus_list) { 1281 if (PCI_SLOT(pdev->devfn) == 0) { 1282 if (pci_set_power_state(pdev, PCI_D0)) 1283 dev_err(pcie->dev, 1284 "Failed to transition %s to D0 state\n", 1285 dev_name(&pdev->dev)); 1286 } 1287 } 1288 } 1289 1290 static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie) 1291 { 1292 pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3"); 1293 if (IS_ERR(pcie->slot_ctl_3v3)) { 1294 if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV) 1295 return PTR_ERR(pcie->slot_ctl_3v3); 1296 1297 pcie->slot_ctl_3v3 = NULL; 1298 } 1299 1300 pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v"); 1301 if (IS_ERR(pcie->slot_ctl_12v)) { 1302 if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV) 1303 return PTR_ERR(pcie->slot_ctl_12v); 1304 1305 pcie->slot_ctl_12v = NULL; 1306 } 1307 1308 return 0; 1309 } 1310 1311 static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie) 1312 { 1313 int ret; 1314 1315 if (pcie->slot_ctl_3v3) { 1316 ret = regulator_enable(pcie->slot_ctl_3v3); 1317 if (ret < 0) { 1318 dev_err(pcie->dev, 1319 "Failed to enable 3.3V slot supply: %d\n", ret); 1320 return ret; 1321 } 1322 } 1323 1324 if (pcie->slot_ctl_12v) { 1325 ret = regulator_enable(pcie->slot_ctl_12v); 1326 if (ret < 0) { 1327 dev_err(pcie->dev, 1328 "Failed to enable 12V slot supply: %d\n", ret); 1329 goto fail_12v_enable; 1330 } 1331 } 1332 1333 /* 1334 * According to PCI Express Card Electromechanical Specification 1335 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive) 1336 * should be a minimum of 100ms. 1337 */ 1338 if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v) 1339 msleep(100); 1340 1341 return 0; 1342 1343 fail_12v_enable: 1344 if (pcie->slot_ctl_3v3) 1345 regulator_disable(pcie->slot_ctl_3v3); 1346 return ret; 1347 } 1348 1349 static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie) 1350 { 1351 if (pcie->slot_ctl_12v) 1352 regulator_disable(pcie->slot_ctl_12v); 1353 if (pcie->slot_ctl_3v3) 1354 regulator_disable(pcie->slot_ctl_3v3); 1355 } 1356 1357 static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie, 1358 bool en_hw_hot_rst) 1359 { 1360 int ret; 1361 u32 val; 1362 1363 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true); 1364 if (ret) { 1365 dev_err(pcie->dev, 1366 "Failed to enable controller %u: %d\n", pcie->cid, ret); 1367 return ret; 1368 } 1369 1370 ret = tegra_pcie_enable_slot_regulators(pcie); 1371 if (ret < 0) 1372 goto fail_slot_reg_en; 1373 1374 ret = regulator_enable(pcie->pex_ctl_supply); 1375 if (ret < 0) { 1376 dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret); 1377 goto fail_reg_en; 1378 } 1379 1380 ret = clk_prepare_enable(pcie->core_clk); 1381 if (ret) { 1382 dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret); 1383 goto fail_core_clk; 1384 } 1385 1386 ret = reset_control_deassert(pcie->core_apb_rst); 1387 if (ret) { 1388 dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n", 1389 ret); 1390 goto fail_core_apb_rst; 1391 } 1392 1393 if (en_hw_hot_rst) { 1394 /* Enable HW_HOT_RST mode */ 1395 val = appl_readl(pcie, APPL_CTRL); 1396 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << 1397 APPL_CTRL_HW_HOT_RST_MODE_SHIFT); 1398 val |= APPL_CTRL_HW_HOT_RST_EN; 1399 appl_writel(pcie, val, APPL_CTRL); 1400 } 1401 1402 ret = tegra_pcie_enable_phy(pcie); 1403 if (ret) { 1404 dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret); 1405 goto fail_phy; 1406 } 1407 1408 /* Update CFG base address */ 1409 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK, 1410 APPL_CFG_BASE_ADDR); 1411 1412 /* Configure this core for RP mode operation */ 1413 appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE); 1414 1415 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE); 1416 1417 val = appl_readl(pcie, APPL_CTRL); 1418 appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL); 1419 1420 val = appl_readl(pcie, APPL_CFG_MISC); 1421 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT); 1422 appl_writel(pcie, val, APPL_CFG_MISC); 1423 1424 if (!pcie->supports_clkreq) { 1425 val = appl_readl(pcie, APPL_PINMUX); 1426 val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN; 1427 val &= ~APPL_PINMUX_CLKREQ_OVERRIDE; 1428 appl_writel(pcie, val, APPL_PINMUX); 1429 } 1430 1431 /* Update iATU_DMA base address */ 1432 appl_writel(pcie, 1433 pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK, 1434 APPL_CFG_IATU_DMA_BASE_ADDR); 1435 1436 reset_control_deassert(pcie->core_rst); 1437 1438 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, 1439 PCI_CAP_ID_EXP); 1440 1441 /* Disable ASPM-L1SS advertisement as there is no CLKREQ routing */ 1442 if (!pcie->supports_clkreq) { 1443 disable_aspm_l11(pcie); 1444 disable_aspm_l12(pcie); 1445 } 1446 1447 return ret; 1448 1449 fail_phy: 1450 reset_control_assert(pcie->core_apb_rst); 1451 fail_core_apb_rst: 1452 clk_disable_unprepare(pcie->core_clk); 1453 fail_core_clk: 1454 regulator_disable(pcie->pex_ctl_supply); 1455 fail_reg_en: 1456 tegra_pcie_disable_slot_regulators(pcie); 1457 fail_slot_reg_en: 1458 tegra_pcie_bpmp_set_ctrl_state(pcie, false); 1459 1460 return ret; 1461 } 1462 1463 static int __deinit_controller(struct tegra_pcie_dw *pcie) 1464 { 1465 int ret; 1466 1467 ret = reset_control_assert(pcie->core_rst); 1468 if (ret) { 1469 dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", 1470 ret); 1471 return ret; 1472 } 1473 1474 tegra_pcie_disable_phy(pcie); 1475 1476 ret = reset_control_assert(pcie->core_apb_rst); 1477 if (ret) { 1478 dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret); 1479 return ret; 1480 } 1481 1482 clk_disable_unprepare(pcie->core_clk); 1483 1484 ret = regulator_disable(pcie->pex_ctl_supply); 1485 if (ret) { 1486 dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret); 1487 return ret; 1488 } 1489 1490 tegra_pcie_disable_slot_regulators(pcie); 1491 1492 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false); 1493 if (ret) { 1494 dev_err(pcie->dev, "Failed to disable controller %d: %d\n", 1495 pcie->cid, ret); 1496 return ret; 1497 } 1498 1499 return ret; 1500 } 1501 1502 static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie) 1503 { 1504 struct dw_pcie *pci = &pcie->pci; 1505 struct pcie_port *pp = &pci->pp; 1506 int ret; 1507 1508 ret = tegra_pcie_config_controller(pcie, false); 1509 if (ret < 0) 1510 return ret; 1511 1512 pp->ops = &tegra_pcie_dw_host_ops; 1513 1514 ret = dw_pcie_host_init(pp); 1515 if (ret < 0) { 1516 dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret); 1517 goto fail_host_init; 1518 } 1519 1520 return 0; 1521 1522 fail_host_init: 1523 return __deinit_controller(pcie); 1524 } 1525 1526 static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie) 1527 { 1528 u32 val; 1529 1530 if (!tegra_pcie_dw_link_up(&pcie->pci)) 1531 return 0; 1532 1533 val = appl_readl(pcie, APPL_RADM_STATUS); 1534 val |= APPL_PM_XMT_TURNOFF_STATE; 1535 appl_writel(pcie, val, APPL_RADM_STATUS); 1536 1537 return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val, 1538 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT, 1539 1, PME_ACK_TIMEOUT); 1540 } 1541 1542 static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) 1543 { 1544 u32 data; 1545 int err; 1546 1547 if (!tegra_pcie_dw_link_up(&pcie->pci)) { 1548 dev_dbg(pcie->dev, "PCIe link is not up...!\n"); 1549 return; 1550 } 1551 1552 if (tegra_pcie_try_link_l2(pcie)) { 1553 dev_info(pcie->dev, "Link didn't transition to L2 state\n"); 1554 /* 1555 * TX lane clock freq will reset to Gen1 only if link is in L2 1556 * or detect state. 1557 * So apply pex_rst to end point to force RP to go into detect 1558 * state 1559 */ 1560 data = appl_readl(pcie, APPL_PINMUX); 1561 data &= ~APPL_PINMUX_PEX_RST; 1562 appl_writel(pcie, data, APPL_PINMUX); 1563 1564 err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, 1565 data, 1566 ((data & 1567 APPL_DEBUG_LTSSM_STATE_MASK) >> 1568 APPL_DEBUG_LTSSM_STATE_SHIFT) == 1569 LTSSM_STATE_PRE_DETECT, 1570 1, LTSSM_TIMEOUT); 1571 if (err) { 1572 dev_info(pcie->dev, "Link didn't go to detect state\n"); 1573 } else { 1574 /* Disable LTSSM after link is in detect state */ 1575 data = appl_readl(pcie, APPL_CTRL); 1576 data &= ~APPL_CTRL_LTSSM_EN; 1577 appl_writel(pcie, data, APPL_CTRL); 1578 } 1579 } 1580 /* 1581 * DBI registers may not be accessible after this as PLL-E would be 1582 * down depending on how CLKREQ is pulled by end point 1583 */ 1584 data = appl_readl(pcie, APPL_PINMUX); 1585 data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE); 1586 /* Cut REFCLK to slot */ 1587 data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; 1588 data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; 1589 appl_writel(pcie, data, APPL_PINMUX); 1590 } 1591 1592 static int tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie) 1593 { 1594 tegra_pcie_downstream_dev_to_D0(pcie); 1595 dw_pcie_host_deinit(&pcie->pci.pp); 1596 tegra_pcie_dw_pme_turnoff(pcie); 1597 1598 return __deinit_controller(pcie); 1599 } 1600 1601 static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie) 1602 { 1603 struct pcie_port *pp = &pcie->pci.pp; 1604 struct device *dev = pcie->dev; 1605 char *name; 1606 int ret; 1607 1608 if (IS_ENABLED(CONFIG_PCI_MSI)) { 1609 pp->msi_irq = of_irq_get_byname(dev->of_node, "msi"); 1610 if (!pp->msi_irq) { 1611 dev_err(dev, "Failed to get MSI interrupt\n"); 1612 return -ENODEV; 1613 } 1614 } 1615 1616 pm_runtime_enable(dev); 1617 1618 ret = pm_runtime_get_sync(dev); 1619 if (ret < 0) { 1620 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", 1621 ret); 1622 goto fail_pm_get_sync; 1623 } 1624 1625 ret = pinctrl_pm_select_default_state(dev); 1626 if (ret < 0) { 1627 dev_err(dev, "Failed to configure sideband pins: %d\n", ret); 1628 goto fail_pm_get_sync; 1629 } 1630 1631 tegra_pcie_init_controller(pcie); 1632 1633 pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci); 1634 if (!pcie->link_state) { 1635 ret = -ENOMEDIUM; 1636 goto fail_host_init; 1637 } 1638 1639 name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node); 1640 if (!name) { 1641 ret = -ENOMEM; 1642 goto fail_host_init; 1643 } 1644 1645 pcie->debugfs = debugfs_create_dir(name, NULL); 1646 if (!pcie->debugfs) 1647 dev_err(dev, "Failed to create debugfs\n"); 1648 else 1649 init_debugfs(pcie); 1650 1651 return ret; 1652 1653 fail_host_init: 1654 tegra_pcie_deinit_controller(pcie); 1655 fail_pm_get_sync: 1656 pm_runtime_put_sync(dev); 1657 pm_runtime_disable(dev); 1658 return ret; 1659 } 1660 1661 static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie) 1662 { 1663 u32 val; 1664 int ret; 1665 1666 if (pcie->ep_state == EP_STATE_DISABLED) 1667 return; 1668 1669 /* Disable LTSSM */ 1670 val = appl_readl(pcie, APPL_CTRL); 1671 val &= ~APPL_CTRL_LTSSM_EN; 1672 appl_writel(pcie, val, APPL_CTRL); 1673 1674 ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val, 1675 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >> 1676 APPL_DEBUG_LTSSM_STATE_SHIFT) == 1677 LTSSM_STATE_PRE_DETECT, 1678 1, LTSSM_TIMEOUT); 1679 if (ret) 1680 dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret); 1681 1682 reset_control_assert(pcie->core_rst); 1683 1684 tegra_pcie_disable_phy(pcie); 1685 1686 reset_control_assert(pcie->core_apb_rst); 1687 1688 clk_disable_unprepare(pcie->core_clk); 1689 1690 pm_runtime_put_sync(pcie->dev); 1691 1692 ret = tegra_pcie_bpmp_set_pll_state(pcie, false); 1693 if (ret) 1694 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret); 1695 1696 pcie->ep_state = EP_STATE_DISABLED; 1697 dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n"); 1698 } 1699 1700 static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) 1701 { 1702 struct dw_pcie *pci = &pcie->pci; 1703 struct dw_pcie_ep *ep = &pci->ep; 1704 struct device *dev = pcie->dev; 1705 u32 val; 1706 int ret; 1707 1708 if (pcie->ep_state == EP_STATE_ENABLED) 1709 return; 1710 1711 ret = pm_runtime_get_sync(dev); 1712 if (ret < 0) { 1713 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", 1714 ret); 1715 return; 1716 } 1717 1718 ret = tegra_pcie_bpmp_set_pll_state(pcie, true); 1719 if (ret) { 1720 dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n", ret); 1721 goto fail_pll_init; 1722 } 1723 1724 ret = clk_prepare_enable(pcie->core_clk); 1725 if (ret) { 1726 dev_err(dev, "Failed to enable core clock: %d\n", ret); 1727 goto fail_core_clk_enable; 1728 } 1729 1730 ret = reset_control_deassert(pcie->core_apb_rst); 1731 if (ret) { 1732 dev_err(dev, "Failed to deassert core APB reset: %d\n", ret); 1733 goto fail_core_apb_rst; 1734 } 1735 1736 ret = tegra_pcie_enable_phy(pcie); 1737 if (ret) { 1738 dev_err(dev, "Failed to enable PHY: %d\n", ret); 1739 goto fail_phy; 1740 } 1741 1742 /* Clear any stale interrupt statuses */ 1743 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); 1744 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); 1745 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); 1746 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); 1747 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); 1748 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); 1749 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); 1750 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); 1751 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); 1752 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); 1753 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); 1754 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); 1755 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); 1756 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); 1757 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); 1758 1759 /* configure this core for EP mode operation */ 1760 val = appl_readl(pcie, APPL_DM_TYPE); 1761 val &= ~APPL_DM_TYPE_MASK; 1762 val |= APPL_DM_TYPE_EP; 1763 appl_writel(pcie, val, APPL_DM_TYPE); 1764 1765 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE); 1766 1767 val = appl_readl(pcie, APPL_CTRL); 1768 val |= APPL_CTRL_SYS_PRE_DET_STATE; 1769 val |= APPL_CTRL_HW_HOT_RST_EN; 1770 appl_writel(pcie, val, APPL_CTRL); 1771 1772 val = appl_readl(pcie, APPL_CFG_MISC); 1773 val |= APPL_CFG_MISC_SLV_EP_MODE; 1774 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT); 1775 appl_writel(pcie, val, APPL_CFG_MISC); 1776 1777 val = appl_readl(pcie, APPL_PINMUX); 1778 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; 1779 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; 1780 appl_writel(pcie, val, APPL_PINMUX); 1781 1782 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK, 1783 APPL_CFG_BASE_ADDR); 1784 1785 appl_writel(pcie, pcie->atu_dma_res->start & 1786 APPL_CFG_IATU_DMA_BASE_ADDR_MASK, 1787 APPL_CFG_IATU_DMA_BASE_ADDR); 1788 1789 val = appl_readl(pcie, APPL_INTR_EN_L0_0); 1790 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN; 1791 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN; 1792 val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN; 1793 appl_writel(pcie, val, APPL_INTR_EN_L0_0); 1794 1795 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); 1796 val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN; 1797 val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN; 1798 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); 1799 1800 reset_control_deassert(pcie->core_rst); 1801 1802 if (pcie->update_fc_fixup) { 1803 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); 1804 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT; 1805 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val); 1806 } 1807 1808 config_gen3_gen4_eq_presets(pcie); 1809 1810 init_host_aspm(pcie); 1811 1812 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */ 1813 if (!pcie->supports_clkreq) { 1814 disable_aspm_l11(pcie); 1815 disable_aspm_l12(pcie); 1816 } 1817 1818 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); 1819 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; 1820 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); 1821 1822 /* Configure N_FTS & FTS */ 1823 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); 1824 val &= ~(N_FTS_MASK << N_FTS_SHIFT); 1825 val |= N_FTS_VAL << N_FTS_SHIFT; 1826 dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); 1827 1828 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); 1829 val &= ~FTS_MASK; 1830 val |= FTS_VAL; 1831 dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); 1832 1833 /* Configure Max Speed from DT */ 1834 if (pcie->max_speed && pcie->max_speed != -EINVAL) { 1835 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + 1836 PCI_EXP_LNKCAP); 1837 val &= ~PCI_EXP_LNKCAP_SLS; 1838 val |= pcie->max_speed; 1839 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, 1840 val); 1841 } 1842 1843 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, 1844 PCI_CAP_ID_EXP); 1845 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); 1846 1847 val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK); 1848 val |= MSIX_ADDR_MATCH_LOW_OFF_EN; 1849 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val); 1850 val = (lower_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK); 1851 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val); 1852 1853 ret = dw_pcie_ep_init_complete(ep); 1854 if (ret) { 1855 dev_err(dev, "Failed to complete initialization: %d\n", ret); 1856 goto fail_init_complete; 1857 } 1858 1859 dw_pcie_ep_init_notify(ep); 1860 1861 /* Enable LTSSM */ 1862 val = appl_readl(pcie, APPL_CTRL); 1863 val |= APPL_CTRL_LTSSM_EN; 1864 appl_writel(pcie, val, APPL_CTRL); 1865 1866 pcie->ep_state = EP_STATE_ENABLED; 1867 dev_dbg(dev, "Initialization of endpoint is completed\n"); 1868 1869 return; 1870 1871 fail_init_complete: 1872 reset_control_assert(pcie->core_rst); 1873 tegra_pcie_disable_phy(pcie); 1874 fail_phy: 1875 reset_control_assert(pcie->core_apb_rst); 1876 fail_core_apb_rst: 1877 clk_disable_unprepare(pcie->core_clk); 1878 fail_core_clk_enable: 1879 tegra_pcie_bpmp_set_pll_state(pcie, false); 1880 fail_pll_init: 1881 pm_runtime_put_sync(dev); 1882 } 1883 1884 static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg) 1885 { 1886 struct tegra_pcie_dw *pcie = arg; 1887 1888 if (gpiod_get_value(pcie->pex_rst_gpiod)) 1889 pex_ep_event_pex_rst_assert(pcie); 1890 else 1891 pex_ep_event_pex_rst_deassert(pcie); 1892 1893 return IRQ_HANDLED; 1894 } 1895 1896 static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq) 1897 { 1898 /* Tegra194 supports only INTA */ 1899 if (irq > 1) 1900 return -EINVAL; 1901 1902 appl_writel(pcie, 1, APPL_LEGACY_INTX); 1903 usleep_range(1000, 2000); 1904 appl_writel(pcie, 0, APPL_LEGACY_INTX); 1905 return 0; 1906 } 1907 1908 static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq) 1909 { 1910 if (unlikely(irq > 31)) 1911 return -EINVAL; 1912 1913 appl_writel(pcie, (1 << irq), APPL_MSI_CTRL_1); 1914 1915 return 0; 1916 } 1917 1918 static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq) 1919 { 1920 struct dw_pcie_ep *ep = &pcie->pci.ep; 1921 1922 writel(irq, ep->msi_mem); 1923 1924 return 0; 1925 } 1926 1927 static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, 1928 enum pci_epc_irq_type type, 1929 u16 interrupt_num) 1930 { 1931 struct dw_pcie *pci = to_dw_pcie_from_ep(ep); 1932 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); 1933 1934 switch (type) { 1935 case PCI_EPC_IRQ_LEGACY: 1936 return tegra_pcie_ep_raise_legacy_irq(pcie, interrupt_num); 1937 1938 case PCI_EPC_IRQ_MSI: 1939 return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num); 1940 1941 case PCI_EPC_IRQ_MSIX: 1942 return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num); 1943 1944 default: 1945 dev_err(pci->dev, "Unknown IRQ type\n"); 1946 return -EPERM; 1947 } 1948 1949 return 0; 1950 } 1951 1952 static const struct pci_epc_features tegra_pcie_epc_features = { 1953 .linkup_notifier = true, 1954 .core_init_notifier = true, 1955 .msi_capable = false, 1956 .msix_capable = false, 1957 .reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5, 1958 .bar_fixed_64bit = 1 << BAR_0, 1959 .bar_fixed_size[0] = SZ_1M, 1960 }; 1961 1962 static const struct pci_epc_features* 1963 tegra_pcie_ep_get_features(struct dw_pcie_ep *ep) 1964 { 1965 return &tegra_pcie_epc_features; 1966 } 1967 1968 static struct dw_pcie_ep_ops pcie_ep_ops = { 1969 .raise_irq = tegra_pcie_ep_raise_irq, 1970 .get_features = tegra_pcie_ep_get_features, 1971 }; 1972 1973 static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie, 1974 struct platform_device *pdev) 1975 { 1976 struct dw_pcie *pci = &pcie->pci; 1977 struct device *dev = pcie->dev; 1978 struct dw_pcie_ep *ep; 1979 struct resource *res; 1980 char *name; 1981 int ret; 1982 1983 ep = &pci->ep; 1984 ep->ops = &pcie_ep_ops; 1985 1986 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space"); 1987 if (!res) 1988 return -EINVAL; 1989 1990 ep->phys_base = res->start; 1991 ep->addr_size = resource_size(res); 1992 ep->page_size = SZ_64K; 1993 1994 ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME); 1995 if (ret < 0) { 1996 dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n", 1997 ret); 1998 return ret; 1999 } 2000 2001 ret = gpiod_to_irq(pcie->pex_rst_gpiod); 2002 if (ret < 0) { 2003 dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret); 2004 return ret; 2005 } 2006 pcie->pex_rst_irq = (unsigned int)ret; 2007 2008 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq", 2009 pcie->cid); 2010 if (!name) { 2011 dev_err(dev, "Failed to create PERST IRQ string\n"); 2012 return -ENOMEM; 2013 } 2014 2015 irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN); 2016 2017 pcie->ep_state = EP_STATE_DISABLED; 2018 2019 ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL, 2020 tegra_pcie_ep_pex_rst_irq, 2021 IRQF_TRIGGER_RISING | 2022 IRQF_TRIGGER_FALLING | IRQF_ONESHOT, 2023 name, (void *)pcie); 2024 if (ret < 0) { 2025 dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret); 2026 return ret; 2027 } 2028 2029 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_ep_work", 2030 pcie->cid); 2031 if (!name) { 2032 dev_err(dev, "Failed to create PCIe EP work thread string\n"); 2033 return -ENOMEM; 2034 } 2035 2036 pm_runtime_enable(dev); 2037 2038 ret = dw_pcie_ep_init(ep); 2039 if (ret) { 2040 dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n", 2041 ret); 2042 return ret; 2043 } 2044 2045 return 0; 2046 } 2047 2048 static int tegra_pcie_dw_probe(struct platform_device *pdev) 2049 { 2050 const struct tegra_pcie_dw_of_data *data; 2051 struct device *dev = &pdev->dev; 2052 struct resource *atu_dma_res; 2053 struct tegra_pcie_dw *pcie; 2054 struct resource *dbi_res; 2055 struct pcie_port *pp; 2056 struct dw_pcie *pci; 2057 struct phy **phys; 2058 char *name; 2059 int ret; 2060 u32 i; 2061 2062 data = of_device_get_match_data(dev); 2063 2064 pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); 2065 if (!pcie) 2066 return -ENOMEM; 2067 2068 pci = &pcie->pci; 2069 pci->dev = &pdev->dev; 2070 pci->ops = &tegra_dw_pcie_ops; 2071 pp = &pci->pp; 2072 pcie->dev = &pdev->dev; 2073 pcie->mode = (enum dw_pcie_device_mode)data->mode; 2074 2075 ret = tegra_pcie_dw_parse_dt(pcie); 2076 if (ret < 0) { 2077 const char *level = KERN_ERR; 2078 2079 if (ret == -EPROBE_DEFER) 2080 level = KERN_DEBUG; 2081 2082 dev_printk(level, dev, 2083 dev_fmt("Failed to parse device tree: %d\n"), 2084 ret); 2085 return ret; 2086 } 2087 2088 ret = tegra_pcie_get_slot_regulators(pcie); 2089 if (ret < 0) { 2090 const char *level = KERN_ERR; 2091 2092 if (ret == -EPROBE_DEFER) 2093 level = KERN_DEBUG; 2094 2095 dev_printk(level, dev, 2096 dev_fmt("Failed to get slot regulators: %d\n"), 2097 ret); 2098 return ret; 2099 } 2100 2101 if (pcie->pex_refclk_sel_gpiod) 2102 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1); 2103 2104 pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl"); 2105 if (IS_ERR(pcie->pex_ctl_supply)) { 2106 ret = PTR_ERR(pcie->pex_ctl_supply); 2107 if (ret != -EPROBE_DEFER) 2108 dev_err(dev, "Failed to get regulator: %ld\n", 2109 PTR_ERR(pcie->pex_ctl_supply)); 2110 return ret; 2111 } 2112 2113 pcie->core_clk = devm_clk_get(dev, "core"); 2114 if (IS_ERR(pcie->core_clk)) { 2115 dev_err(dev, "Failed to get core clock: %ld\n", 2116 PTR_ERR(pcie->core_clk)); 2117 return PTR_ERR(pcie->core_clk); 2118 } 2119 2120 pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 2121 "appl"); 2122 if (!pcie->appl_res) { 2123 dev_err(dev, "Failed to find \"appl\" region\n"); 2124 return -ENODEV; 2125 } 2126 2127 pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res); 2128 if (IS_ERR(pcie->appl_base)) 2129 return PTR_ERR(pcie->appl_base); 2130 2131 pcie->core_apb_rst = devm_reset_control_get(dev, "apb"); 2132 if (IS_ERR(pcie->core_apb_rst)) { 2133 dev_err(dev, "Failed to get APB reset: %ld\n", 2134 PTR_ERR(pcie->core_apb_rst)); 2135 return PTR_ERR(pcie->core_apb_rst); 2136 } 2137 2138 phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL); 2139 if (!phys) 2140 return -ENOMEM; 2141 2142 for (i = 0; i < pcie->phy_count; i++) { 2143 name = kasprintf(GFP_KERNEL, "p2u-%u", i); 2144 if (!name) { 2145 dev_err(dev, "Failed to create P2U string\n"); 2146 return -ENOMEM; 2147 } 2148 phys[i] = devm_phy_get(dev, name); 2149 kfree(name); 2150 if (IS_ERR(phys[i])) { 2151 ret = PTR_ERR(phys[i]); 2152 if (ret != -EPROBE_DEFER) 2153 dev_err(dev, "Failed to get PHY: %d\n", ret); 2154 return ret; 2155 } 2156 } 2157 2158 pcie->phys = phys; 2159 2160 dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); 2161 if (!dbi_res) { 2162 dev_err(dev, "Failed to find \"dbi\" region\n"); 2163 return -ENODEV; 2164 } 2165 pcie->dbi_res = dbi_res; 2166 2167 pci->dbi_base = devm_ioremap_resource(dev, dbi_res); 2168 if (IS_ERR(pci->dbi_base)) 2169 return PTR_ERR(pci->dbi_base); 2170 2171 /* Tegra HW locates DBI2 at a fixed offset from DBI */ 2172 pci->dbi_base2 = pci->dbi_base + 0x1000; 2173 2174 atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 2175 "atu_dma"); 2176 if (!atu_dma_res) { 2177 dev_err(dev, "Failed to find \"atu_dma\" region\n"); 2178 return -ENODEV; 2179 } 2180 pcie->atu_dma_res = atu_dma_res; 2181 2182 pci->atu_base = devm_ioremap_resource(dev, atu_dma_res); 2183 if (IS_ERR(pci->atu_base)) 2184 return PTR_ERR(pci->atu_base); 2185 2186 pcie->core_rst = devm_reset_control_get(dev, "core"); 2187 if (IS_ERR(pcie->core_rst)) { 2188 dev_err(dev, "Failed to get core reset: %ld\n", 2189 PTR_ERR(pcie->core_rst)); 2190 return PTR_ERR(pcie->core_rst); 2191 } 2192 2193 pp->irq = platform_get_irq_byname(pdev, "intr"); 2194 if (pp->irq < 0) 2195 return pp->irq; 2196 2197 pcie->bpmp = tegra_bpmp_get(dev); 2198 if (IS_ERR(pcie->bpmp)) 2199 return PTR_ERR(pcie->bpmp); 2200 2201 platform_set_drvdata(pdev, pcie); 2202 2203 switch (pcie->mode) { 2204 case DW_PCIE_RC_TYPE: 2205 ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler, 2206 IRQF_SHARED, "tegra-pcie-intr", pcie); 2207 if (ret) { 2208 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq, 2209 ret); 2210 goto fail; 2211 } 2212 2213 ret = tegra_pcie_config_rp(pcie); 2214 if (ret && ret != -ENOMEDIUM) 2215 goto fail; 2216 else 2217 return 0; 2218 break; 2219 2220 case DW_PCIE_EP_TYPE: 2221 ret = devm_request_threaded_irq(dev, pp->irq, 2222 tegra_pcie_ep_hard_irq, 2223 tegra_pcie_ep_irq_thread, 2224 IRQF_SHARED | IRQF_ONESHOT, 2225 "tegra-pcie-ep-intr", pcie); 2226 if (ret) { 2227 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq, 2228 ret); 2229 goto fail; 2230 } 2231 2232 ret = tegra_pcie_config_ep(pcie, pdev); 2233 if (ret < 0) 2234 goto fail; 2235 break; 2236 2237 default: 2238 dev_err(dev, "Invalid PCIe device type %d\n", pcie->mode); 2239 } 2240 2241 fail: 2242 tegra_bpmp_put(pcie->bpmp); 2243 return ret; 2244 } 2245 2246 static int tegra_pcie_dw_remove(struct platform_device *pdev) 2247 { 2248 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); 2249 2250 if (!pcie->link_state) 2251 return 0; 2252 2253 debugfs_remove_recursive(pcie->debugfs); 2254 tegra_pcie_deinit_controller(pcie); 2255 pm_runtime_put_sync(pcie->dev); 2256 pm_runtime_disable(pcie->dev); 2257 tegra_bpmp_put(pcie->bpmp); 2258 if (pcie->pex_refclk_sel_gpiod) 2259 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0); 2260 2261 return 0; 2262 } 2263 2264 static int tegra_pcie_dw_suspend_late(struct device *dev) 2265 { 2266 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2267 u32 val; 2268 2269 if (!pcie->link_state) 2270 return 0; 2271 2272 /* Enable HW_HOT_RST mode */ 2273 val = appl_readl(pcie, APPL_CTRL); 2274 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << 2275 APPL_CTRL_HW_HOT_RST_MODE_SHIFT); 2276 val |= APPL_CTRL_HW_HOT_RST_EN; 2277 appl_writel(pcie, val, APPL_CTRL); 2278 2279 return 0; 2280 } 2281 2282 static int tegra_pcie_dw_suspend_noirq(struct device *dev) 2283 { 2284 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2285 2286 if (!pcie->link_state) 2287 return 0; 2288 2289 /* Save MSI interrupt vector */ 2290 pcie->msi_ctrl_int = dw_pcie_readl_dbi(&pcie->pci, 2291 PORT_LOGIC_MSI_CTRL_INT_0_EN); 2292 tegra_pcie_downstream_dev_to_D0(pcie); 2293 tegra_pcie_dw_pme_turnoff(pcie); 2294 2295 return __deinit_controller(pcie); 2296 } 2297 2298 static int tegra_pcie_dw_resume_noirq(struct device *dev) 2299 { 2300 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2301 int ret; 2302 2303 if (!pcie->link_state) 2304 return 0; 2305 2306 ret = tegra_pcie_config_controller(pcie, true); 2307 if (ret < 0) 2308 return ret; 2309 2310 ret = tegra_pcie_dw_host_init(&pcie->pci.pp); 2311 if (ret < 0) { 2312 dev_err(dev, "Failed to init host: %d\n", ret); 2313 goto fail_host_init; 2314 } 2315 2316 /* Restore MSI interrupt vector */ 2317 dw_pcie_writel_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN, 2318 pcie->msi_ctrl_int); 2319 2320 return 0; 2321 2322 fail_host_init: 2323 return __deinit_controller(pcie); 2324 } 2325 2326 static int tegra_pcie_dw_resume_early(struct device *dev) 2327 { 2328 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); 2329 u32 val; 2330 2331 if (!pcie->link_state) 2332 return 0; 2333 2334 /* Disable HW_HOT_RST mode */ 2335 val = appl_readl(pcie, APPL_CTRL); 2336 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << 2337 APPL_CTRL_HW_HOT_RST_MODE_SHIFT); 2338 val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST << 2339 APPL_CTRL_HW_HOT_RST_MODE_SHIFT; 2340 val &= ~APPL_CTRL_HW_HOT_RST_EN; 2341 appl_writel(pcie, val, APPL_CTRL); 2342 2343 return 0; 2344 } 2345 2346 static void tegra_pcie_dw_shutdown(struct platform_device *pdev) 2347 { 2348 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); 2349 2350 if (!pcie->link_state) 2351 return; 2352 2353 debugfs_remove_recursive(pcie->debugfs); 2354 tegra_pcie_downstream_dev_to_D0(pcie); 2355 2356 disable_irq(pcie->pci.pp.irq); 2357 if (IS_ENABLED(CONFIG_PCI_MSI)) 2358 disable_irq(pcie->pci.pp.msi_irq); 2359 2360 tegra_pcie_dw_pme_turnoff(pcie); 2361 __deinit_controller(pcie); 2362 } 2363 2364 static const struct tegra_pcie_dw_of_data tegra_pcie_dw_rc_of_data = { 2365 .mode = DW_PCIE_RC_TYPE, 2366 }; 2367 2368 static const struct tegra_pcie_dw_of_data tegra_pcie_dw_ep_of_data = { 2369 .mode = DW_PCIE_EP_TYPE, 2370 }; 2371 2372 static const struct of_device_id tegra_pcie_dw_of_match[] = { 2373 { 2374 .compatible = "nvidia,tegra194-pcie", 2375 .data = &tegra_pcie_dw_rc_of_data, 2376 }, 2377 { 2378 .compatible = "nvidia,tegra194-pcie-ep", 2379 .data = &tegra_pcie_dw_ep_of_data, 2380 }, 2381 {}, 2382 }; 2383 2384 static const struct dev_pm_ops tegra_pcie_dw_pm_ops = { 2385 .suspend_late = tegra_pcie_dw_suspend_late, 2386 .suspend_noirq = tegra_pcie_dw_suspend_noirq, 2387 .resume_noirq = tegra_pcie_dw_resume_noirq, 2388 .resume_early = tegra_pcie_dw_resume_early, 2389 }; 2390 2391 static struct platform_driver tegra_pcie_dw_driver = { 2392 .probe = tegra_pcie_dw_probe, 2393 .remove = tegra_pcie_dw_remove, 2394 .shutdown = tegra_pcie_dw_shutdown, 2395 .driver = { 2396 .name = "tegra194-pcie", 2397 .pm = &tegra_pcie_dw_pm_ops, 2398 .of_match_table = tegra_pcie_dw_of_match, 2399 }, 2400 }; 2401 module_platform_driver(tegra_pcie_dw_driver); 2402 2403 MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match); 2404 2405 MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>"); 2406 MODULE_DESCRIPTION("NVIDIA PCIe host controller driver"); 2407 MODULE_LICENSE("GPL v2"); 2408