1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * PCIe host controller driver for the following SoCs
4  * Tegra194
5  * Tegra234
6  *
7  * Copyright (C) 2019-2022 NVIDIA Corporation.
8  *
9  * Author: Vidya Sagar <vidyas@nvidia.com>
10  */
11 
12 #include <linux/clk.h>
13 #include <linux/debugfs.h>
14 #include <linux/delay.h>
15 #include <linux/gpio.h>
16 #include <linux/gpio/consumer.h>
17 #include <linux/interconnect.h>
18 #include <linux/interrupt.h>
19 #include <linux/iopoll.h>
20 #include <linux/kernel.h>
21 #include <linux/module.h>
22 #include <linux/of.h>
23 #include <linux/of_device.h>
24 #include <linux/of_gpio.h>
25 #include <linux/of_pci.h>
26 #include <linux/pci.h>
27 #include <linux/phy/phy.h>
28 #include <linux/pinctrl/consumer.h>
29 #include <linux/platform_device.h>
30 #include <linux/pm_runtime.h>
31 #include <linux/random.h>
32 #include <linux/reset.h>
33 #include <linux/resource.h>
34 #include <linux/types.h>
35 #include "pcie-designware.h"
36 #include <soc/tegra/bpmp.h>
37 #include <soc/tegra/bpmp-abi.h>
38 #include "../../pci.h"
39 
40 #define TEGRA194_DWC_IP_VER			0x490A
41 #define TEGRA234_DWC_IP_VER			0x562A
42 
43 #define APPL_PINMUX				0x0
44 #define APPL_PINMUX_PEX_RST			BIT(0)
45 #define APPL_PINMUX_CLKREQ_OVERRIDE_EN		BIT(2)
46 #define APPL_PINMUX_CLKREQ_OVERRIDE		BIT(3)
47 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN	BIT(4)
48 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE	BIT(5)
49 
50 #define APPL_CTRL				0x4
51 #define APPL_CTRL_SYS_PRE_DET_STATE		BIT(6)
52 #define APPL_CTRL_LTSSM_EN			BIT(7)
53 #define APPL_CTRL_HW_HOT_RST_EN			BIT(20)
54 #define APPL_CTRL_HW_HOT_RST_MODE_MASK		GENMASK(1, 0)
55 #define APPL_CTRL_HW_HOT_RST_MODE_SHIFT		22
56 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST	0x1
57 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN	0x2
58 
59 #define APPL_INTR_EN_L0_0			0x8
60 #define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN	BIT(0)
61 #define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN	BIT(4)
62 #define APPL_INTR_EN_L0_0_INT_INT_EN		BIT(8)
63 #define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN	BIT(15)
64 #define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN	BIT(19)
65 #define APPL_INTR_EN_L0_0_SYS_INTR_EN		BIT(30)
66 #define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN	BIT(31)
67 
68 #define APPL_INTR_STATUS_L0			0xC
69 #define APPL_INTR_STATUS_L0_LINK_STATE_INT	BIT(0)
70 #define APPL_INTR_STATUS_L0_INT_INT		BIT(8)
71 #define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT	BIT(15)
72 #define APPL_INTR_STATUS_L0_PEX_RST_INT		BIT(16)
73 #define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT	BIT(18)
74 
75 #define APPL_INTR_EN_L1_0_0				0x1C
76 #define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN	BIT(1)
77 #define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN		BIT(3)
78 #define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN	BIT(30)
79 
80 #define APPL_INTR_STATUS_L1_0_0				0x20
81 #define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED	BIT(1)
82 #define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED	BIT(3)
83 #define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE		BIT(30)
84 
85 #define APPL_INTR_STATUS_L1_1			0x2C
86 #define APPL_INTR_STATUS_L1_2			0x30
87 #define APPL_INTR_STATUS_L1_3			0x34
88 #define APPL_INTR_STATUS_L1_6			0x3C
89 #define APPL_INTR_STATUS_L1_7			0x40
90 #define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED	BIT(1)
91 
92 #define APPL_INTR_EN_L1_8_0			0x44
93 #define APPL_INTR_EN_L1_8_BW_MGT_INT_EN		BIT(2)
94 #define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN	BIT(3)
95 #define APPL_INTR_EN_L1_8_INTX_EN		BIT(11)
96 #define APPL_INTR_EN_L1_8_AER_INT_EN		BIT(15)
97 
98 #define APPL_INTR_STATUS_L1_8_0			0x4C
99 #define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK	GENMASK(11, 6)
100 #define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS	BIT(2)
101 #define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS	BIT(3)
102 
103 #define APPL_INTR_STATUS_L1_9			0x54
104 #define APPL_INTR_STATUS_L1_10			0x58
105 #define APPL_INTR_STATUS_L1_11			0x64
106 #define APPL_INTR_STATUS_L1_13			0x74
107 #define APPL_INTR_STATUS_L1_14			0x78
108 #define APPL_INTR_STATUS_L1_15			0x7C
109 #define APPL_INTR_STATUS_L1_17			0x88
110 
111 #define APPL_INTR_EN_L1_18				0x90
112 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT		BIT(2)
113 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR		BIT(1)
114 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR	BIT(0)
115 
116 #define APPL_INTR_STATUS_L1_18				0x94
117 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT	BIT(2)
118 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR	BIT(1)
119 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR	BIT(0)
120 
121 #define APPL_MSI_CTRL_1				0xAC
122 
123 #define APPL_MSI_CTRL_2				0xB0
124 
125 #define APPL_LEGACY_INTX			0xB8
126 
127 #define APPL_LTR_MSG_1				0xC4
128 #define LTR_MSG_REQ				BIT(15)
129 #define LTR_MST_NO_SNOOP_SHIFT			16
130 
131 #define APPL_LTR_MSG_2				0xC8
132 #define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE	BIT(3)
133 
134 #define APPL_LINK_STATUS			0xCC
135 #define APPL_LINK_STATUS_RDLH_LINK_UP		BIT(0)
136 
137 #define APPL_DEBUG				0xD0
138 #define APPL_DEBUG_PM_LINKST_IN_L2_LAT		BIT(21)
139 #define APPL_DEBUG_PM_LINKST_IN_L0		0x11
140 #define APPL_DEBUG_LTSSM_STATE_MASK		GENMASK(8, 3)
141 #define APPL_DEBUG_LTSSM_STATE_SHIFT		3
142 #define LTSSM_STATE_PRE_DETECT			5
143 
144 #define APPL_RADM_STATUS			0xE4
145 #define APPL_PM_XMT_TURNOFF_STATE		BIT(0)
146 
147 #define APPL_DM_TYPE				0x100
148 #define APPL_DM_TYPE_MASK			GENMASK(3, 0)
149 #define APPL_DM_TYPE_RP				0x4
150 #define APPL_DM_TYPE_EP				0x0
151 
152 #define APPL_CFG_BASE_ADDR			0x104
153 #define APPL_CFG_BASE_ADDR_MASK			GENMASK(31, 12)
154 
155 #define APPL_CFG_IATU_DMA_BASE_ADDR		0x108
156 #define APPL_CFG_IATU_DMA_BASE_ADDR_MASK	GENMASK(31, 18)
157 
158 #define APPL_CFG_MISC				0x110
159 #define APPL_CFG_MISC_SLV_EP_MODE		BIT(14)
160 #define APPL_CFG_MISC_ARCACHE_MASK		GENMASK(13, 10)
161 #define APPL_CFG_MISC_ARCACHE_SHIFT		10
162 #define APPL_CFG_MISC_ARCACHE_VAL		3
163 
164 #define APPL_CFG_SLCG_OVERRIDE			0x114
165 #define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER	BIT(0)
166 
167 #define APPL_CAR_RESET_OVRD				0x12C
168 #define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N	BIT(0)
169 
170 #define IO_BASE_IO_DECODE				BIT(0)
171 #define IO_BASE_IO_DECODE_BIT8				BIT(8)
172 
173 #define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE		BIT(0)
174 #define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE	BIT(16)
175 
176 #define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF	0x718
177 #define CFG_TIMER_CTRL_ACK_NAK_SHIFT	(19)
178 
179 #define N_FTS_VAL					52
180 #define FTS_VAL						52
181 
182 #define GEN3_EQ_CONTROL_OFF			0x8a8
183 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT	8
184 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK	GENMASK(23, 8)
185 #define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK	GENMASK(3, 0)
186 
187 #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT	0x8D0
188 #define AMBA_ERROR_RESPONSE_CRS_SHIFT		3
189 #define AMBA_ERROR_RESPONSE_CRS_MASK		GENMASK(1, 0)
190 #define AMBA_ERROR_RESPONSE_CRS_OKAY		0
191 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF	1
192 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001	2
193 
194 #define MSIX_ADDR_MATCH_LOW_OFF			0x940
195 #define MSIX_ADDR_MATCH_LOW_OFF_EN		BIT(0)
196 #define MSIX_ADDR_MATCH_LOW_OFF_MASK		GENMASK(31, 2)
197 
198 #define MSIX_ADDR_MATCH_HIGH_OFF		0x944
199 #define MSIX_ADDR_MATCH_HIGH_OFF_MASK		GENMASK(31, 0)
200 
201 #define PORT_LOGIC_MSIX_DOORBELL			0x948
202 
203 #define CAP_SPCIE_CAP_OFF			0x154
204 #define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK	GENMASK(3, 0)
205 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK	GENMASK(11, 8)
206 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT	8
207 
208 #define PME_ACK_TIMEOUT 10000
209 
210 #define LTSSM_TIMEOUT 50000	/* 50ms */
211 
212 #define GEN3_GEN4_EQ_PRESET_INIT	5
213 
214 #define GEN1_CORE_CLK_FREQ	62500000
215 #define GEN2_CORE_CLK_FREQ	125000000
216 #define GEN3_CORE_CLK_FREQ	250000000
217 #define GEN4_CORE_CLK_FREQ	500000000
218 
219 #define LTR_MSG_TIMEOUT		(100 * 1000)
220 
221 #define PERST_DEBOUNCE_TIME	(5 * 1000)
222 
223 #define EP_STATE_DISABLED	0
224 #define EP_STATE_ENABLED	1
225 
226 static const unsigned int pcie_gen_freq[] = {
227 	GEN1_CORE_CLK_FREQ,	/* PCI_EXP_LNKSTA_CLS == 0; undefined */
228 	GEN1_CORE_CLK_FREQ,
229 	GEN2_CORE_CLK_FREQ,
230 	GEN3_CORE_CLK_FREQ,
231 	GEN4_CORE_CLK_FREQ
232 };
233 
234 struct tegra_pcie_dw_of_data {
235 	u32 version;
236 	enum dw_pcie_device_mode mode;
237 	bool has_msix_doorbell_access_fix;
238 	bool has_sbr_reset_fix;
239 	bool has_l1ss_exit_fix;
240 	bool has_ltr_req_fix;
241 	u32 cdm_chk_int_en_bit;
242 	u32 gen4_preset_vec;
243 	u8 n_fts[2];
244 };
245 
246 struct tegra_pcie_dw {
247 	struct device *dev;
248 	struct resource *appl_res;
249 	struct resource *dbi_res;
250 	struct resource *atu_dma_res;
251 	void __iomem *appl_base;
252 	struct clk *core_clk;
253 	struct reset_control *core_apb_rst;
254 	struct reset_control *core_rst;
255 	struct dw_pcie pci;
256 	struct tegra_bpmp *bpmp;
257 
258 	struct tegra_pcie_dw_of_data *of_data;
259 
260 	bool supports_clkreq;
261 	bool enable_cdm_check;
262 	bool enable_srns;
263 	bool link_state;
264 	bool update_fc_fixup;
265 	bool enable_ext_refclk;
266 	u8 init_link_width;
267 	u32 msi_ctrl_int;
268 	u32 num_lanes;
269 	u32 cid;
270 	u32 cfg_link_cap_l1sub;
271 	u32 ras_des_cap;
272 	u32 pcie_cap_base;
273 	u32 aspm_cmrt;
274 	u32 aspm_pwr_on_t;
275 	u32 aspm_l0s_enter_lat;
276 
277 	struct regulator *pex_ctl_supply;
278 	struct regulator *slot_ctl_3v3;
279 	struct regulator *slot_ctl_12v;
280 
281 	unsigned int phy_count;
282 	struct phy **phys;
283 
284 	struct dentry *debugfs;
285 
286 	/* Endpoint mode specific */
287 	struct gpio_desc *pex_rst_gpiod;
288 	struct gpio_desc *pex_refclk_sel_gpiod;
289 	unsigned int pex_rst_irq;
290 	int ep_state;
291 	long link_status;
292 	struct icc_path *icc_path;
293 };
294 
295 static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)
296 {
297 	return container_of(pci, struct tegra_pcie_dw, pci);
298 }
299 
300 static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,
301 			       const u32 reg)
302 {
303 	writel_relaxed(value, pcie->appl_base + reg);
304 }
305 
306 static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)
307 {
308 	return readl_relaxed(pcie->appl_base + reg);
309 }
310 
311 struct tegra_pcie_soc {
312 	enum dw_pcie_device_mode mode;
313 };
314 
315 static void tegra_pcie_icc_set(struct tegra_pcie_dw *pcie)
316 {
317 	struct dw_pcie *pci = &pcie->pci;
318 	u32 val, speed, width;
319 
320 	val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
321 
322 	speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, val);
323 	width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val);
324 
325 	val = width * (PCIE_SPEED2MBS_ENC(pcie_link_speed[speed]) / BITS_PER_BYTE);
326 
327 	if (icc_set_bw(pcie->icc_path, MBps_to_icc(val), 0))
328 		dev_err(pcie->dev, "can't set bw[%u]\n", val);
329 
330 	if (speed >= ARRAY_SIZE(pcie_gen_freq))
331 		speed = 0;
332 
333 	clk_set_rate(pcie->core_clk, pcie_gen_freq[speed]);
334 }
335 
336 static void apply_bad_link_workaround(struct dw_pcie_rp *pp)
337 {
338 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
339 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
340 	u32 current_link_width;
341 	u16 val;
342 
343 	/*
344 	 * NOTE:- Since this scenario is uncommon and link as such is not
345 	 * stable anyway, not waiting to confirm if link is really
346 	 * transitioning to Gen-2 speed
347 	 */
348 	val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
349 	if (val & PCI_EXP_LNKSTA_LBMS) {
350 		current_link_width = (val & PCI_EXP_LNKSTA_NLW) >>
351 				     PCI_EXP_LNKSTA_NLW_SHIFT;
352 		if (pcie->init_link_width > current_link_width) {
353 			dev_warn(pci->dev, "PCIe link is bad, width reduced\n");
354 			val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
355 						PCI_EXP_LNKCTL2);
356 			val &= ~PCI_EXP_LNKCTL2_TLS;
357 			val |= PCI_EXP_LNKCTL2_TLS_2_5GT;
358 			dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
359 					   PCI_EXP_LNKCTL2, val);
360 
361 			val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
362 						PCI_EXP_LNKCTL);
363 			val |= PCI_EXP_LNKCTL_RL;
364 			dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
365 					   PCI_EXP_LNKCTL, val);
366 		}
367 	}
368 }
369 
370 static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg)
371 {
372 	struct tegra_pcie_dw *pcie = arg;
373 	struct dw_pcie *pci = &pcie->pci;
374 	struct dw_pcie_rp *pp = &pci->pp;
375 	u32 val, status_l0, status_l1;
376 	u16 val_w;
377 
378 	status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
379 	if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
380 		status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
381 		appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);
382 		if (!pcie->of_data->has_sbr_reset_fix &&
383 		    status_l1 & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {
384 			/* SBR & Surprise Link Down WAR */
385 			val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
386 			val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
387 			appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
388 			udelay(1);
389 			val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
390 			val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
391 			appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
392 
393 			val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
394 			val |= PORT_LOGIC_SPEED_CHANGE;
395 			dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
396 		}
397 	}
398 
399 	if (status_l0 & APPL_INTR_STATUS_L0_INT_INT) {
400 		status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);
401 		if (status_l1 & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {
402 			appl_writel(pcie,
403 				    APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,
404 				    APPL_INTR_STATUS_L1_8_0);
405 			apply_bad_link_workaround(pp);
406 		}
407 		if (status_l1 & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {
408 			val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
409 						  PCI_EXP_LNKSTA);
410 			val_w |= PCI_EXP_LNKSTA_LBMS;
411 			dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
412 					   PCI_EXP_LNKSTA, val_w);
413 
414 			appl_writel(pcie,
415 				    APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,
416 				    APPL_INTR_STATUS_L1_8_0);
417 
418 			val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
419 						  PCI_EXP_LNKSTA);
420 			dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &
421 				PCI_EXP_LNKSTA_CLS);
422 		}
423 	}
424 
425 	if (status_l0 & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {
426 		status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_18);
427 		val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
428 		if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {
429 			dev_info(pci->dev, "CDM check complete\n");
430 			val |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;
431 		}
432 		if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {
433 			dev_err(pci->dev, "CDM comparison mismatch\n");
434 			val |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;
435 		}
436 		if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {
437 			dev_err(pci->dev, "CDM Logic error\n");
438 			val |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;
439 		}
440 		dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, val);
441 		val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);
442 		dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", val);
443 	}
444 
445 	return IRQ_HANDLED;
446 }
447 
448 static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie)
449 {
450 	u32 val;
451 
452 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
453 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
454 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
455 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
456 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
457 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
458 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
459 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
460 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
461 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
462 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
463 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
464 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
465 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
466 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
467 	appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2);
468 
469 	val = appl_readl(pcie, APPL_CTRL);
470 	val |= APPL_CTRL_LTSSM_EN;
471 	appl_writel(pcie, val, APPL_CTRL);
472 }
473 
474 static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg)
475 {
476 	struct tegra_pcie_dw *pcie = arg;
477 	struct dw_pcie_ep *ep = &pcie->pci.ep;
478 	struct dw_pcie *pci = &pcie->pci;
479 	u32 val;
480 
481 	if (test_and_clear_bit(0, &pcie->link_status))
482 		dw_pcie_ep_linkup(ep);
483 
484 	tegra_pcie_icc_set(pcie);
485 
486 	if (pcie->of_data->has_ltr_req_fix)
487 		return IRQ_HANDLED;
488 
489 	/* If EP doesn't advertise L1SS, just return */
490 	val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
491 	if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2)))
492 		return IRQ_HANDLED;
493 
494 	/* Check if BME is set to '1' */
495 	val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
496 	if (val & PCI_COMMAND_MASTER) {
497 		ktime_t timeout;
498 
499 		/* 110us for both snoop and no-snoop */
500 		val = 110 | (2 << PCI_LTR_SCALE_SHIFT) | LTR_MSG_REQ;
501 		val |= (val << LTR_MST_NO_SNOOP_SHIFT);
502 		appl_writel(pcie, val, APPL_LTR_MSG_1);
503 
504 		/* Send LTR upstream */
505 		val = appl_readl(pcie, APPL_LTR_MSG_2);
506 		val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
507 		appl_writel(pcie, val, APPL_LTR_MSG_2);
508 
509 		timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT);
510 		for (;;) {
511 			val = appl_readl(pcie, APPL_LTR_MSG_2);
512 			if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE))
513 				break;
514 			if (ktime_after(ktime_get(), timeout))
515 				break;
516 			usleep_range(1000, 1100);
517 		}
518 		if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)
519 			dev_err(pcie->dev, "Failed to send LTR message\n");
520 	}
521 
522 	return IRQ_HANDLED;
523 }
524 
525 static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg)
526 {
527 	struct tegra_pcie_dw *pcie = arg;
528 	int spurious = 1;
529 	u32 status_l0, status_l1, link_status;
530 
531 	status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);
532 	if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
533 		status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
534 		appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);
535 
536 		if (status_l1 & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE)
537 			pex_ep_event_hot_rst_done(pcie);
538 
539 		if (status_l1 & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) {
540 			link_status = appl_readl(pcie, APPL_LINK_STATUS);
541 			if (link_status & APPL_LINK_STATUS_RDLH_LINK_UP) {
542 				dev_dbg(pcie->dev, "Link is up with Host\n");
543 				set_bit(0, &pcie->link_status);
544 				return IRQ_WAKE_THREAD;
545 			}
546 		}
547 
548 		spurious = 0;
549 	}
550 
551 	if (status_l0 & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) {
552 		status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_15);
553 		appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_15);
554 
555 		if (status_l1 & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED)
556 			return IRQ_WAKE_THREAD;
557 
558 		spurious = 0;
559 	}
560 
561 	if (spurious) {
562 		dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n",
563 			 status_l0);
564 		appl_writel(pcie, status_l0, APPL_INTR_STATUS_L0);
565 	}
566 
567 	return IRQ_HANDLED;
568 }
569 
570 static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where,
571 				     int size, u32 *val)
572 {
573 	struct dw_pcie_rp *pp = bus->sysdata;
574 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
575 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
576 
577 	/*
578 	 * This is an endpoint mode specific register happen to appear even
579 	 * when controller is operating in root port mode and system hangs
580 	 * when it is accessed with link being in ASPM-L1 state.
581 	 * So skip accessing it altogether
582 	 */
583 	if (!pcie->of_data->has_msix_doorbell_access_fix &&
584 	    !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) {
585 		*val = 0x00000000;
586 		return PCIBIOS_SUCCESSFUL;
587 	}
588 
589 	return pci_generic_config_read(bus, devfn, where, size, val);
590 }
591 
592 static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where,
593 				     int size, u32 val)
594 {
595 	struct dw_pcie_rp *pp = bus->sysdata;
596 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
597 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
598 
599 	/*
600 	 * This is an endpoint mode specific register happen to appear even
601 	 * when controller is operating in root port mode and system hangs
602 	 * when it is accessed with link being in ASPM-L1 state.
603 	 * So skip accessing it altogether
604 	 */
605 	if (!pcie->of_data->has_msix_doorbell_access_fix &&
606 	    !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL)
607 		return PCIBIOS_SUCCESSFUL;
608 
609 	return pci_generic_config_write(bus, devfn, where, size, val);
610 }
611 
612 static struct pci_ops tegra_pci_ops = {
613 	.map_bus = dw_pcie_own_conf_map_bus,
614 	.read = tegra_pcie_dw_rd_own_conf,
615 	.write = tegra_pcie_dw_wr_own_conf,
616 };
617 
618 #if defined(CONFIG_PCIEASPM)
619 static void disable_aspm_l11(struct tegra_pcie_dw *pcie)
620 {
621 	u32 val;
622 
623 	val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
624 	val &= ~PCI_L1SS_CAP_ASPM_L1_1;
625 	dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
626 }
627 
628 static void disable_aspm_l12(struct tegra_pcie_dw *pcie)
629 {
630 	u32 val;
631 
632 	val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
633 	val &= ~PCI_L1SS_CAP_ASPM_L1_2;
634 	dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
635 }
636 
637 static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)
638 {
639 	u32 val;
640 
641 	val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap +
642 				PCIE_RAS_DES_EVENT_COUNTER_CONTROL);
643 	val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);
644 	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
645 	val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;
646 	val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
647 	dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
648 			   PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
649 	val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap +
650 				PCIE_RAS_DES_EVENT_COUNTER_DATA);
651 
652 	return val;
653 }
654 
655 static int aspm_state_cnt(struct seq_file *s, void *data)
656 {
657 	struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)
658 				     dev_get_drvdata(s->private);
659 	u32 val;
660 
661 	seq_printf(s, "Tx L0s entry count : %u\n",
662 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));
663 
664 	seq_printf(s, "Rx L0s entry count : %u\n",
665 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));
666 
667 	seq_printf(s, "Link L1 entry count : %u\n",
668 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));
669 
670 	seq_printf(s, "Link L1.1 entry count : %u\n",
671 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));
672 
673 	seq_printf(s, "Link L1.2 entry count : %u\n",
674 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));
675 
676 	/* Clear all counters */
677 	dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
678 			   PCIE_RAS_DES_EVENT_COUNTER_CONTROL,
679 			   EVENT_COUNTER_ALL_CLEAR);
680 
681 	/* Re-enable counting */
682 	val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
683 	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
684 	dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +
685 			   PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
686 
687 	return 0;
688 }
689 
690 static void init_host_aspm(struct tegra_pcie_dw *pcie)
691 {
692 	struct dw_pcie *pci = &pcie->pci;
693 	u32 val;
694 
695 	val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
696 	pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;
697 
698 	pcie->ras_des_cap = dw_pcie_find_ext_capability(&pcie->pci,
699 							PCI_EXT_CAP_ID_VNDR);
700 
701 	/* Enable ASPM counters */
702 	val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
703 	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
704 	dw_pcie_writel_dbi(pci, pcie->ras_des_cap +
705 			   PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);
706 
707 	/* Program T_cmrt and T_pwr_on values */
708 	val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
709 	val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);
710 	val |= (pcie->aspm_cmrt << 8);
711 	val |= (pcie->aspm_pwr_on_t << 19);
712 	dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val);
713 
714 	/* Program L0s and L1 entrance latencies */
715 	val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR);
716 	val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK;
717 	val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT);
718 	val |= PORT_AFR_ENTER_ASPM;
719 	dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val);
720 }
721 
722 static void init_debugfs(struct tegra_pcie_dw *pcie)
723 {
724 	debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", pcie->debugfs,
725 				    aspm_state_cnt);
726 }
727 #else
728 static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }
729 static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }
730 static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }
731 static inline void init_debugfs(struct tegra_pcie_dw *pcie) { return; }
732 #endif
733 
734 static void tegra_pcie_enable_system_interrupts(struct dw_pcie_rp *pp)
735 {
736 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
737 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
738 	u32 val;
739 	u16 val_w;
740 
741 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
742 	val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
743 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
744 
745 	if (!pcie->of_data->has_sbr_reset_fix) {
746 		val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
747 		val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;
748 		appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
749 	}
750 
751 	if (pcie->enable_cdm_check) {
752 		val = appl_readl(pcie, APPL_INTR_EN_L0_0);
753 		val |= pcie->of_data->cdm_chk_int_en_bit;
754 		appl_writel(pcie, val, APPL_INTR_EN_L0_0);
755 
756 		val = appl_readl(pcie, APPL_INTR_EN_L1_18);
757 		val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;
758 		val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;
759 		appl_writel(pcie, val, APPL_INTR_EN_L1_18);
760 	}
761 
762 	val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
763 				  PCI_EXP_LNKSTA);
764 	pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >>
765 				PCI_EXP_LNKSTA_NLW_SHIFT;
766 
767 	val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
768 				  PCI_EXP_LNKCTL);
769 	val_w |= PCI_EXP_LNKCTL_LBMIE;
770 	dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL,
771 			   val_w);
772 }
773 
774 static void tegra_pcie_enable_legacy_interrupts(struct dw_pcie_rp *pp)
775 {
776 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
777 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
778 	u32 val;
779 
780 	/* Enable legacy interrupt generation */
781 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
782 	val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
783 	val |= APPL_INTR_EN_L0_0_INT_INT_EN;
784 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
785 
786 	val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);
787 	val |= APPL_INTR_EN_L1_8_INTX_EN;
788 	val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;
789 	val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;
790 	if (IS_ENABLED(CONFIG_PCIEAER))
791 		val |= APPL_INTR_EN_L1_8_AER_INT_EN;
792 	appl_writel(pcie, val, APPL_INTR_EN_L1_8_0);
793 }
794 
795 static void tegra_pcie_enable_msi_interrupts(struct dw_pcie_rp *pp)
796 {
797 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
798 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
799 	u32 val;
800 
801 	/* Enable MSI interrupt generation */
802 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
803 	val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;
804 	val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;
805 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
806 }
807 
808 static void tegra_pcie_enable_interrupts(struct dw_pcie_rp *pp)
809 {
810 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
811 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
812 
813 	/* Clear interrupt statuses before enabling interrupts */
814 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
815 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
816 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
817 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
818 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
819 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
820 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
821 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
822 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
823 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
824 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
825 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
826 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
827 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
828 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
829 
830 	tegra_pcie_enable_system_interrupts(pp);
831 	tegra_pcie_enable_legacy_interrupts(pp);
832 	if (IS_ENABLED(CONFIG_PCI_MSI))
833 		tegra_pcie_enable_msi_interrupts(pp);
834 }
835 
836 static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
837 {
838 	struct dw_pcie *pci = &pcie->pci;
839 	u32 val, offset, i;
840 
841 	/* Program init preset */
842 	for (i = 0; i < pcie->num_lanes; i++) {
843 		val = dw_pcie_readw_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2));
844 		val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;
845 		val |= GEN3_GEN4_EQ_PRESET_INIT;
846 		val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;
847 		val |= (GEN3_GEN4_EQ_PRESET_INIT <<
848 			   CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);
849 		dw_pcie_writew_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2), val);
850 
851 		offset = dw_pcie_find_ext_capability(pci,
852 						     PCI_EXT_CAP_ID_PL_16GT) +
853 				PCI_PL_16GT_LE_CTRL;
854 		val = dw_pcie_readb_dbi(pci, offset + i);
855 		val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;
856 		val |= GEN3_GEN4_EQ_PRESET_INIT;
857 		val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;
858 		val |= (GEN3_GEN4_EQ_PRESET_INIT <<
859 			PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);
860 		dw_pcie_writeb_dbi(pci, offset + i, val);
861 	}
862 
863 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
864 	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
865 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
866 
867 	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
868 	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
869 	val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
870 	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
871 	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
872 
873 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
874 	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
875 	val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);
876 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
877 
878 	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
879 	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
880 	val |= (pcie->of_data->gen4_preset_vec <<
881 		GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
882 	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
883 	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
884 
885 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
886 	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
887 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
888 }
889 
890 static int tegra_pcie_dw_host_init(struct dw_pcie_rp *pp)
891 {
892 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
893 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
894 	u32 val;
895 	u16 val_16;
896 
897 	pp->bridge->ops = &tegra_pci_ops;
898 
899 	if (!pcie->pcie_cap_base)
900 		pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
901 							      PCI_CAP_ID_EXP);
902 
903 	val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_DEVCTL);
904 	val_16 &= ~PCI_EXP_DEVCTL_PAYLOAD;
905 	val_16 |= PCI_EXP_DEVCTL_PAYLOAD_256B;
906 	dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_DEVCTL, val_16);
907 
908 	val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);
909 	val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);
910 	dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);
911 
912 	val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);
913 	val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;
914 	val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;
915 	dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);
916 
917 	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
918 
919 	/* Enable as 0xFFFF0001 response for CRS */
920 	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);
921 	val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT);
922 	val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 <<
923 		AMBA_ERROR_RESPONSE_CRS_SHIFT);
924 	dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);
925 
926 	/* Configure Max lane width from DT */
927 	val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP);
928 	val &= ~PCI_EXP_LNKCAP_MLW;
929 	val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT);
930 	dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val);
931 
932 	/* Clear Slot Clock Configuration bit if SRNS configuration */
933 	if (pcie->enable_srns) {
934 		val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
935 					   PCI_EXP_LNKSTA);
936 		val_16 &= ~PCI_EXP_LNKSTA_SLC;
937 		dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA,
938 				   val_16);
939 	}
940 
941 	config_gen3_gen4_eq_presets(pcie);
942 
943 	init_host_aspm(pcie);
944 
945 	/* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
946 	if (!pcie->supports_clkreq) {
947 		disable_aspm_l11(pcie);
948 		disable_aspm_l12(pcie);
949 	}
950 
951 	if (!pcie->of_data->has_l1ss_exit_fix) {
952 		val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
953 		val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
954 		dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
955 	}
956 
957 	if (pcie->update_fc_fixup) {
958 		val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
959 		val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
960 		dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
961 	}
962 
963 	clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
964 
965 	return 0;
966 }
967 
968 static int tegra_pcie_dw_start_link(struct dw_pcie *pci)
969 {
970 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
971 	struct dw_pcie_rp *pp = &pci->pp;
972 	u32 val, offset, tmp;
973 	bool retry = true;
974 
975 	if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
976 		enable_irq(pcie->pex_rst_irq);
977 		return 0;
978 	}
979 
980 retry_link:
981 	/* Assert RST */
982 	val = appl_readl(pcie, APPL_PINMUX);
983 	val &= ~APPL_PINMUX_PEX_RST;
984 	appl_writel(pcie, val, APPL_PINMUX);
985 
986 	usleep_range(100, 200);
987 
988 	/* Enable LTSSM */
989 	val = appl_readl(pcie, APPL_CTRL);
990 	val |= APPL_CTRL_LTSSM_EN;
991 	appl_writel(pcie, val, APPL_CTRL);
992 
993 	/* De-assert RST */
994 	val = appl_readl(pcie, APPL_PINMUX);
995 	val |= APPL_PINMUX_PEX_RST;
996 	appl_writel(pcie, val, APPL_PINMUX);
997 
998 	msleep(100);
999 
1000 	if (dw_pcie_wait_for_link(pci)) {
1001 		if (!retry)
1002 			return 0;
1003 		/*
1004 		 * There are some endpoints which can't get the link up if
1005 		 * root port has Data Link Feature (DLF) enabled.
1006 		 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info
1007 		 * on Scaled Flow Control and DLF.
1008 		 * So, need to confirm that is indeed the case here and attempt
1009 		 * link up once again with DLF disabled.
1010 		 */
1011 		val = appl_readl(pcie, APPL_DEBUG);
1012 		val &= APPL_DEBUG_LTSSM_STATE_MASK;
1013 		val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;
1014 		tmp = appl_readl(pcie, APPL_LINK_STATUS);
1015 		tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;
1016 		if (!(val == 0x11 && !tmp)) {
1017 			/* Link is down for all good reasons */
1018 			return 0;
1019 		}
1020 
1021 		dev_info(pci->dev, "Link is down in DLL");
1022 		dev_info(pci->dev, "Trying again with DLFE disabled\n");
1023 		/* Disable LTSSM */
1024 		val = appl_readl(pcie, APPL_CTRL);
1025 		val &= ~APPL_CTRL_LTSSM_EN;
1026 		appl_writel(pcie, val, APPL_CTRL);
1027 
1028 		reset_control_assert(pcie->core_rst);
1029 		reset_control_deassert(pcie->core_rst);
1030 
1031 		offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);
1032 		val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP);
1033 		val &= ~PCI_DLF_EXCHANGE_ENABLE;
1034 		dw_pcie_writel_dbi(pci, offset + PCI_DLF_CAP, val);
1035 
1036 		tegra_pcie_dw_host_init(pp);
1037 		dw_pcie_setup_rc(pp);
1038 
1039 		retry = false;
1040 		goto retry_link;
1041 	}
1042 
1043 	tegra_pcie_icc_set(pcie);
1044 
1045 	tegra_pcie_enable_interrupts(pp);
1046 
1047 	return 0;
1048 }
1049 
1050 static int tegra_pcie_dw_link_up(struct dw_pcie *pci)
1051 {
1052 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1053 	u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
1054 
1055 	return !!(val & PCI_EXP_LNKSTA_DLLLA);
1056 }
1057 
1058 static void tegra_pcie_dw_stop_link(struct dw_pcie *pci)
1059 {
1060 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1061 
1062 	disable_irq(pcie->pex_rst_irq);
1063 }
1064 
1065 static const struct dw_pcie_ops tegra_dw_pcie_ops = {
1066 	.link_up = tegra_pcie_dw_link_up,
1067 	.start_link = tegra_pcie_dw_start_link,
1068 	.stop_link = tegra_pcie_dw_stop_link,
1069 };
1070 
1071 static const struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {
1072 	.host_init = tegra_pcie_dw_host_init,
1073 };
1074 
1075 static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)
1076 {
1077 	unsigned int phy_count = pcie->phy_count;
1078 
1079 	while (phy_count--) {
1080 		phy_power_off(pcie->phys[phy_count]);
1081 		phy_exit(pcie->phys[phy_count]);
1082 	}
1083 }
1084 
1085 static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)
1086 {
1087 	unsigned int i;
1088 	int ret;
1089 
1090 	for (i = 0; i < pcie->phy_count; i++) {
1091 		ret = phy_init(pcie->phys[i]);
1092 		if (ret < 0)
1093 			goto phy_power_off;
1094 
1095 		ret = phy_power_on(pcie->phys[i]);
1096 		if (ret < 0)
1097 			goto phy_exit;
1098 	}
1099 
1100 	return 0;
1101 
1102 phy_power_off:
1103 	while (i--) {
1104 		phy_power_off(pcie->phys[i]);
1105 phy_exit:
1106 		phy_exit(pcie->phys[i]);
1107 	}
1108 
1109 	return ret;
1110 }
1111 
1112 static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
1113 {
1114 	struct platform_device *pdev = to_platform_device(pcie->dev);
1115 	struct device_node *np = pcie->dev->of_node;
1116 	int ret;
1117 
1118 	pcie->dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
1119 	if (!pcie->dbi_res) {
1120 		dev_err(pcie->dev, "Failed to find \"dbi\" region\n");
1121 		return -ENODEV;
1122 	}
1123 
1124 	ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt);
1125 	if (ret < 0) {
1126 		dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);
1127 		return ret;
1128 	}
1129 
1130 	ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us",
1131 				   &pcie->aspm_pwr_on_t);
1132 	if (ret < 0)
1133 		dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",
1134 			 ret);
1135 
1136 	ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us",
1137 				   &pcie->aspm_l0s_enter_lat);
1138 	if (ret < 0)
1139 		dev_info(pcie->dev,
1140 			 "Failed to read ASPM L0s Entrance latency: %d\n", ret);
1141 
1142 	ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);
1143 	if (ret < 0) {
1144 		dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
1145 		return ret;
1146 	}
1147 
1148 	ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid);
1149 	if (ret) {
1150 		dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);
1151 		return ret;
1152 	}
1153 
1154 	ret = of_property_count_strings(np, "phy-names");
1155 	if (ret < 0) {
1156 		dev_err(pcie->dev, "Failed to find PHY entries: %d\n",
1157 			ret);
1158 		return ret;
1159 	}
1160 	pcie->phy_count = ret;
1161 
1162 	if (of_property_read_bool(np, "nvidia,update-fc-fixup"))
1163 		pcie->update_fc_fixup = true;
1164 
1165 	/* RP using an external REFCLK is supported only in Tegra234 */
1166 	if (pcie->of_data->version == TEGRA194_DWC_IP_VER) {
1167 		if (pcie->of_data->mode == DW_PCIE_EP_TYPE)
1168 			pcie->enable_ext_refclk = true;
1169 	} else {
1170 		pcie->enable_ext_refclk =
1171 			of_property_read_bool(pcie->dev->of_node,
1172 					      "nvidia,enable-ext-refclk");
1173 	}
1174 
1175 	pcie->supports_clkreq =
1176 		of_property_read_bool(pcie->dev->of_node, "supports-clkreq");
1177 
1178 	pcie->enable_cdm_check =
1179 		of_property_read_bool(np, "snps,enable-cdm-check");
1180 
1181 	if (pcie->of_data->version == TEGRA234_DWC_IP_VER)
1182 		pcie->enable_srns =
1183 			of_property_read_bool(np, "nvidia,enable-srns");
1184 
1185 	if (pcie->of_data->mode == DW_PCIE_RC_TYPE)
1186 		return 0;
1187 
1188 	/* Endpoint mode specific DT entries */
1189 	pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN);
1190 	if (IS_ERR(pcie->pex_rst_gpiod)) {
1191 		int err = PTR_ERR(pcie->pex_rst_gpiod);
1192 		const char *level = KERN_ERR;
1193 
1194 		if (err == -EPROBE_DEFER)
1195 			level = KERN_DEBUG;
1196 
1197 		dev_printk(level, pcie->dev,
1198 			   dev_fmt("Failed to get PERST GPIO: %d\n"),
1199 			   err);
1200 		return err;
1201 	}
1202 
1203 	pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev,
1204 						    "nvidia,refclk-select",
1205 						    GPIOD_OUT_HIGH);
1206 	if (IS_ERR(pcie->pex_refclk_sel_gpiod)) {
1207 		int err = PTR_ERR(pcie->pex_refclk_sel_gpiod);
1208 		const char *level = KERN_ERR;
1209 
1210 		if (err == -EPROBE_DEFER)
1211 			level = KERN_DEBUG;
1212 
1213 		dev_printk(level, pcie->dev,
1214 			   dev_fmt("Failed to get REFCLK select GPIOs: %d\n"),
1215 			   err);
1216 		pcie->pex_refclk_sel_gpiod = NULL;
1217 	}
1218 
1219 	return 0;
1220 }
1221 
1222 static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,
1223 					  bool enable)
1224 {
1225 	struct mrq_uphy_response resp;
1226 	struct tegra_bpmp_message msg;
1227 	struct mrq_uphy_request req;
1228 
1229 	/*
1230 	 * Controller-5 doesn't need to have its state set by BPMP-FW in
1231 	 * Tegra194
1232 	 */
1233 	if (pcie->of_data->version == TEGRA194_DWC_IP_VER && pcie->cid == 5)
1234 		return 0;
1235 
1236 	memset(&req, 0, sizeof(req));
1237 	memset(&resp, 0, sizeof(resp));
1238 
1239 	req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
1240 	req.controller_state.pcie_controller = pcie->cid;
1241 	req.controller_state.enable = enable;
1242 
1243 	memset(&msg, 0, sizeof(msg));
1244 	msg.mrq = MRQ_UPHY;
1245 	msg.tx.data = &req;
1246 	msg.tx.size = sizeof(req);
1247 	msg.rx.data = &resp;
1248 	msg.rx.size = sizeof(resp);
1249 
1250 	return tegra_bpmp_transfer(pcie->bpmp, &msg);
1251 }
1252 
1253 static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie,
1254 					 bool enable)
1255 {
1256 	struct mrq_uphy_response resp;
1257 	struct tegra_bpmp_message msg;
1258 	struct mrq_uphy_request req;
1259 
1260 	memset(&req, 0, sizeof(req));
1261 	memset(&resp, 0, sizeof(resp));
1262 
1263 	if (enable) {
1264 		req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;
1265 		req.ep_ctrlr_pll_init.ep_controller = pcie->cid;
1266 	} else {
1267 		req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;
1268 		req.ep_ctrlr_pll_off.ep_controller = pcie->cid;
1269 	}
1270 
1271 	memset(&msg, 0, sizeof(msg));
1272 	msg.mrq = MRQ_UPHY;
1273 	msg.tx.data = &req;
1274 	msg.tx.size = sizeof(req);
1275 	msg.rx.data = &resp;
1276 	msg.rx.size = sizeof(resp);
1277 
1278 	return tegra_bpmp_transfer(pcie->bpmp, &msg);
1279 }
1280 
1281 static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)
1282 {
1283 	struct dw_pcie_rp *pp = &pcie->pci.pp;
1284 	struct pci_bus *child, *root_bus = NULL;
1285 	struct pci_dev *pdev;
1286 
1287 	/*
1288 	 * link doesn't go into L2 state with some of the endpoints with Tegra
1289 	 * if they are not in D0 state. So, need to make sure that immediate
1290 	 * downstream devices are in D0 state before sending PME_TurnOff to put
1291 	 * link into L2 state.
1292 	 * This is as per PCI Express Base r4.0 v1.0 September 27-2017,
1293 	 * 5.2 Link State Power Management (Page #428).
1294 	 */
1295 
1296 	list_for_each_entry(child, &pp->bridge->bus->children, node) {
1297 		/* Bring downstream devices to D0 if they are not already in */
1298 		if (child->parent == pp->bridge->bus) {
1299 			root_bus = child;
1300 			break;
1301 		}
1302 	}
1303 
1304 	if (!root_bus) {
1305 		dev_err(pcie->dev, "Failed to find downstream devices\n");
1306 		return;
1307 	}
1308 
1309 	list_for_each_entry(pdev, &root_bus->devices, bus_list) {
1310 		if (PCI_SLOT(pdev->devfn) == 0) {
1311 			if (pci_set_power_state(pdev, PCI_D0))
1312 				dev_err(pcie->dev,
1313 					"Failed to transition %s to D0 state\n",
1314 					dev_name(&pdev->dev));
1315 		}
1316 	}
1317 }
1318 
1319 static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)
1320 {
1321 	pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3");
1322 	if (IS_ERR(pcie->slot_ctl_3v3)) {
1323 		if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV)
1324 			return PTR_ERR(pcie->slot_ctl_3v3);
1325 
1326 		pcie->slot_ctl_3v3 = NULL;
1327 	}
1328 
1329 	pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v");
1330 	if (IS_ERR(pcie->slot_ctl_12v)) {
1331 		if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV)
1332 			return PTR_ERR(pcie->slot_ctl_12v);
1333 
1334 		pcie->slot_ctl_12v = NULL;
1335 	}
1336 
1337 	return 0;
1338 }
1339 
1340 static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)
1341 {
1342 	int ret;
1343 
1344 	if (pcie->slot_ctl_3v3) {
1345 		ret = regulator_enable(pcie->slot_ctl_3v3);
1346 		if (ret < 0) {
1347 			dev_err(pcie->dev,
1348 				"Failed to enable 3.3V slot supply: %d\n", ret);
1349 			return ret;
1350 		}
1351 	}
1352 
1353 	if (pcie->slot_ctl_12v) {
1354 		ret = regulator_enable(pcie->slot_ctl_12v);
1355 		if (ret < 0) {
1356 			dev_err(pcie->dev,
1357 				"Failed to enable 12V slot supply: %d\n", ret);
1358 			goto fail_12v_enable;
1359 		}
1360 	}
1361 
1362 	/*
1363 	 * According to PCI Express Card Electromechanical Specification
1364 	 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)
1365 	 * should be a minimum of 100ms.
1366 	 */
1367 	if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)
1368 		msleep(100);
1369 
1370 	return 0;
1371 
1372 fail_12v_enable:
1373 	if (pcie->slot_ctl_3v3)
1374 		regulator_disable(pcie->slot_ctl_3v3);
1375 	return ret;
1376 }
1377 
1378 static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)
1379 {
1380 	if (pcie->slot_ctl_12v)
1381 		regulator_disable(pcie->slot_ctl_12v);
1382 	if (pcie->slot_ctl_3v3)
1383 		regulator_disable(pcie->slot_ctl_3v3);
1384 }
1385 
1386 static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,
1387 					bool en_hw_hot_rst)
1388 {
1389 	int ret;
1390 	u32 val;
1391 
1392 	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1393 	if (ret) {
1394 		dev_err(pcie->dev,
1395 			"Failed to enable controller %u: %d\n", pcie->cid, ret);
1396 		return ret;
1397 	}
1398 
1399 	if (pcie->enable_ext_refclk) {
1400 		ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1401 		if (ret) {
1402 			dev_err(pcie->dev, "Failed to init UPHY: %d\n", ret);
1403 			goto fail_pll_init;
1404 		}
1405 	}
1406 
1407 	ret = tegra_pcie_enable_slot_regulators(pcie);
1408 	if (ret < 0)
1409 		goto fail_slot_reg_en;
1410 
1411 	ret = regulator_enable(pcie->pex_ctl_supply);
1412 	if (ret < 0) {
1413 		dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);
1414 		goto fail_reg_en;
1415 	}
1416 
1417 	ret = clk_prepare_enable(pcie->core_clk);
1418 	if (ret) {
1419 		dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);
1420 		goto fail_core_clk;
1421 	}
1422 
1423 	ret = reset_control_deassert(pcie->core_apb_rst);
1424 	if (ret) {
1425 		dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",
1426 			ret);
1427 		goto fail_core_apb_rst;
1428 	}
1429 
1430 	if (en_hw_hot_rst || pcie->of_data->has_sbr_reset_fix) {
1431 		/* Enable HW_HOT_RST mode */
1432 		val = appl_readl(pcie, APPL_CTRL);
1433 		val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1434 			 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1435 		val |= (APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN <<
1436 			APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1437 		val |= APPL_CTRL_HW_HOT_RST_EN;
1438 		appl_writel(pcie, val, APPL_CTRL);
1439 	}
1440 
1441 	ret = tegra_pcie_enable_phy(pcie);
1442 	if (ret) {
1443 		dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);
1444 		goto fail_phy;
1445 	}
1446 
1447 	/* Update CFG base address */
1448 	appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1449 		    APPL_CFG_BASE_ADDR);
1450 
1451 	/* Configure this core for RP mode operation */
1452 	appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);
1453 
1454 	appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1455 
1456 	val = appl_readl(pcie, APPL_CTRL);
1457 	appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);
1458 
1459 	val = appl_readl(pcie, APPL_CFG_MISC);
1460 	val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1461 	appl_writel(pcie, val, APPL_CFG_MISC);
1462 
1463 	if (pcie->enable_srns || pcie->enable_ext_refclk) {
1464 		/*
1465 		 * When Tegra PCIe RP is using external clock, it cannot supply
1466 		 * same clock to its downstream hierarchy. Hence, gate PCIe RP
1467 		 * REFCLK out pads when RP & EP are using separate clocks or RP
1468 		 * is using an external REFCLK.
1469 		 */
1470 		val = appl_readl(pcie, APPL_PINMUX);
1471 		val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1472 		val &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1473 		appl_writel(pcie, val, APPL_PINMUX);
1474 	}
1475 
1476 	if (!pcie->supports_clkreq) {
1477 		val = appl_readl(pcie, APPL_PINMUX);
1478 		val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN;
1479 		val &= ~APPL_PINMUX_CLKREQ_OVERRIDE;
1480 		appl_writel(pcie, val, APPL_PINMUX);
1481 	}
1482 
1483 	/* Update iATU_DMA base address */
1484 	appl_writel(pcie,
1485 		    pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1486 		    APPL_CFG_IATU_DMA_BASE_ADDR);
1487 
1488 	reset_control_deassert(pcie->core_rst);
1489 
1490 	return ret;
1491 
1492 fail_phy:
1493 	reset_control_assert(pcie->core_apb_rst);
1494 fail_core_apb_rst:
1495 	clk_disable_unprepare(pcie->core_clk);
1496 fail_core_clk:
1497 	regulator_disable(pcie->pex_ctl_supply);
1498 fail_reg_en:
1499 	tegra_pcie_disable_slot_regulators(pcie);
1500 fail_slot_reg_en:
1501 	if (pcie->enable_ext_refclk)
1502 		tegra_pcie_bpmp_set_pll_state(pcie, false);
1503 fail_pll_init:
1504 	tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1505 
1506 	return ret;
1507 }
1508 
1509 static void tegra_pcie_unconfig_controller(struct tegra_pcie_dw *pcie)
1510 {
1511 	int ret;
1512 
1513 	ret = reset_control_assert(pcie->core_rst);
1514 	if (ret)
1515 		dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", ret);
1516 
1517 	tegra_pcie_disable_phy(pcie);
1518 
1519 	ret = reset_control_assert(pcie->core_apb_rst);
1520 	if (ret)
1521 		dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);
1522 
1523 	clk_disable_unprepare(pcie->core_clk);
1524 
1525 	ret = regulator_disable(pcie->pex_ctl_supply);
1526 	if (ret)
1527 		dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);
1528 
1529 	tegra_pcie_disable_slot_regulators(pcie);
1530 
1531 	if (pcie->enable_ext_refclk) {
1532 		ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1533 		if (ret)
1534 			dev_err(pcie->dev, "Failed to deinit UPHY: %d\n", ret);
1535 	}
1536 
1537 	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1538 	if (ret)
1539 		dev_err(pcie->dev, "Failed to disable controller %d: %d\n",
1540 			pcie->cid, ret);
1541 }
1542 
1543 static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)
1544 {
1545 	struct dw_pcie *pci = &pcie->pci;
1546 	struct dw_pcie_rp *pp = &pci->pp;
1547 	int ret;
1548 
1549 	ret = tegra_pcie_config_controller(pcie, false);
1550 	if (ret < 0)
1551 		return ret;
1552 
1553 	pp->ops = &tegra_pcie_dw_host_ops;
1554 
1555 	ret = dw_pcie_host_init(pp);
1556 	if (ret < 0) {
1557 		dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);
1558 		goto fail_host_init;
1559 	}
1560 
1561 	return 0;
1562 
1563 fail_host_init:
1564 	tegra_pcie_unconfig_controller(pcie);
1565 	return ret;
1566 }
1567 
1568 static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)
1569 {
1570 	u32 val;
1571 
1572 	if (!tegra_pcie_dw_link_up(&pcie->pci))
1573 		return 0;
1574 
1575 	val = appl_readl(pcie, APPL_RADM_STATUS);
1576 	val |= APPL_PM_XMT_TURNOFF_STATE;
1577 	appl_writel(pcie, val, APPL_RADM_STATUS);
1578 
1579 	return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,
1580 				 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,
1581 				 1, PME_ACK_TIMEOUT);
1582 }
1583 
1584 static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)
1585 {
1586 	u32 data;
1587 	int err;
1588 
1589 	if (!tegra_pcie_dw_link_up(&pcie->pci)) {
1590 		dev_dbg(pcie->dev, "PCIe link is not up...!\n");
1591 		return;
1592 	}
1593 
1594 	/*
1595 	 * PCIe controller exits from L2 only if reset is applied, so
1596 	 * controller doesn't handle interrupts. But in cases where
1597 	 * L2 entry fails, PERST# is asserted which can trigger surprise
1598 	 * link down AER. However this function call happens in
1599 	 * suspend_noirq(), so AER interrupt will not be processed.
1600 	 * Disable all interrupts to avoid such a scenario.
1601 	 */
1602 	appl_writel(pcie, 0x0, APPL_INTR_EN_L0_0);
1603 
1604 	if (tegra_pcie_try_link_l2(pcie)) {
1605 		dev_info(pcie->dev, "Link didn't transition to L2 state\n");
1606 		/*
1607 		 * TX lane clock freq will reset to Gen1 only if link is in L2
1608 		 * or detect state.
1609 		 * So apply pex_rst to end point to force RP to go into detect
1610 		 * state
1611 		 */
1612 		data = appl_readl(pcie, APPL_PINMUX);
1613 		data &= ~APPL_PINMUX_PEX_RST;
1614 		appl_writel(pcie, data, APPL_PINMUX);
1615 
1616 		/*
1617 		 * Some cards do not go to detect state even after de-asserting
1618 		 * PERST#. So, de-assert LTSSM to bring link to detect state.
1619 		 */
1620 		data = readl(pcie->appl_base + APPL_CTRL);
1621 		data &= ~APPL_CTRL_LTSSM_EN;
1622 		writel(data, pcie->appl_base + APPL_CTRL);
1623 
1624 		err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,
1625 						data,
1626 						((data &
1627 						APPL_DEBUG_LTSSM_STATE_MASK) >>
1628 						APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1629 						LTSSM_STATE_PRE_DETECT,
1630 						1, LTSSM_TIMEOUT);
1631 		if (err)
1632 			dev_info(pcie->dev, "Link didn't go to detect state\n");
1633 	}
1634 	/*
1635 	 * DBI registers may not be accessible after this as PLL-E would be
1636 	 * down depending on how CLKREQ is pulled by end point
1637 	 */
1638 	data = appl_readl(pcie, APPL_PINMUX);
1639 	data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);
1640 	/* Cut REFCLK to slot */
1641 	data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1642 	data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1643 	appl_writel(pcie, data, APPL_PINMUX);
1644 }
1645 
1646 static void tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)
1647 {
1648 	tegra_pcie_downstream_dev_to_D0(pcie);
1649 	dw_pcie_host_deinit(&pcie->pci.pp);
1650 	tegra_pcie_dw_pme_turnoff(pcie);
1651 	tegra_pcie_unconfig_controller(pcie);
1652 }
1653 
1654 static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
1655 {
1656 	struct device *dev = pcie->dev;
1657 	char *name;
1658 	int ret;
1659 
1660 	pm_runtime_enable(dev);
1661 
1662 	ret = pm_runtime_get_sync(dev);
1663 	if (ret < 0) {
1664 		dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1665 			ret);
1666 		goto fail_pm_get_sync;
1667 	}
1668 
1669 	ret = pinctrl_pm_select_default_state(dev);
1670 	if (ret < 0) {
1671 		dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
1672 		goto fail_pm_get_sync;
1673 	}
1674 
1675 	ret = tegra_pcie_init_controller(pcie);
1676 	if (ret < 0) {
1677 		dev_err(dev, "Failed to initialize controller: %d\n", ret);
1678 		goto fail_pm_get_sync;
1679 	}
1680 
1681 	pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci);
1682 	if (!pcie->link_state) {
1683 		ret = -ENOMEDIUM;
1684 		goto fail_host_init;
1685 	}
1686 
1687 	name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
1688 	if (!name) {
1689 		ret = -ENOMEM;
1690 		goto fail_host_init;
1691 	}
1692 
1693 	pcie->debugfs = debugfs_create_dir(name, NULL);
1694 	init_debugfs(pcie);
1695 
1696 	return ret;
1697 
1698 fail_host_init:
1699 	tegra_pcie_deinit_controller(pcie);
1700 fail_pm_get_sync:
1701 	pm_runtime_put_sync(dev);
1702 	pm_runtime_disable(dev);
1703 	return ret;
1704 }
1705 
1706 static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie)
1707 {
1708 	u32 val;
1709 	int ret;
1710 
1711 	if (pcie->ep_state == EP_STATE_DISABLED)
1712 		return;
1713 
1714 	/* Disable LTSSM */
1715 	val = appl_readl(pcie, APPL_CTRL);
1716 	val &= ~APPL_CTRL_LTSSM_EN;
1717 	appl_writel(pcie, val, APPL_CTRL);
1718 
1719 	ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val,
1720 				 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >>
1721 				 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1722 				 LTSSM_STATE_PRE_DETECT,
1723 				 1, LTSSM_TIMEOUT);
1724 	if (ret)
1725 		dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret);
1726 
1727 	reset_control_assert(pcie->core_rst);
1728 
1729 	tegra_pcie_disable_phy(pcie);
1730 
1731 	reset_control_assert(pcie->core_apb_rst);
1732 
1733 	clk_disable_unprepare(pcie->core_clk);
1734 
1735 	pm_runtime_put_sync(pcie->dev);
1736 
1737 	if (pcie->enable_ext_refclk) {
1738 		ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1739 		if (ret)
1740 			dev_err(pcie->dev, "Failed to turn off UPHY: %d\n",
1741 				ret);
1742 	}
1743 
1744 	ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1745 	if (ret)
1746 		dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret);
1747 
1748 	pcie->ep_state = EP_STATE_DISABLED;
1749 	dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n");
1750 }
1751 
1752 static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie)
1753 {
1754 	struct dw_pcie *pci = &pcie->pci;
1755 	struct dw_pcie_ep *ep = &pci->ep;
1756 	struct device *dev = pcie->dev;
1757 	u32 val;
1758 	int ret;
1759 	u16 val_16;
1760 
1761 	if (pcie->ep_state == EP_STATE_ENABLED)
1762 		return;
1763 
1764 	ret = pm_runtime_resume_and_get(dev);
1765 	if (ret < 0) {
1766 		dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1767 			ret);
1768 		return;
1769 	}
1770 
1771 	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1772 	if (ret) {
1773 		dev_err(pcie->dev, "Failed to enable controller %u: %d\n",
1774 			pcie->cid, ret);
1775 		goto fail_set_ctrl_state;
1776 	}
1777 
1778 	if (pcie->enable_ext_refclk) {
1779 		ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1780 		if (ret) {
1781 			dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n",
1782 				ret);
1783 			goto fail_pll_init;
1784 		}
1785 	}
1786 
1787 	ret = clk_prepare_enable(pcie->core_clk);
1788 	if (ret) {
1789 		dev_err(dev, "Failed to enable core clock: %d\n", ret);
1790 		goto fail_core_clk_enable;
1791 	}
1792 
1793 	ret = reset_control_deassert(pcie->core_apb_rst);
1794 	if (ret) {
1795 		dev_err(dev, "Failed to deassert core APB reset: %d\n", ret);
1796 		goto fail_core_apb_rst;
1797 	}
1798 
1799 	ret = tegra_pcie_enable_phy(pcie);
1800 	if (ret) {
1801 		dev_err(dev, "Failed to enable PHY: %d\n", ret);
1802 		goto fail_phy;
1803 	}
1804 
1805 	/* Clear any stale interrupt statuses */
1806 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
1807 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
1808 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
1809 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
1810 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
1811 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
1812 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
1813 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
1814 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
1815 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
1816 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
1817 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
1818 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
1819 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
1820 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
1821 
1822 	/* configure this core for EP mode operation */
1823 	val = appl_readl(pcie, APPL_DM_TYPE);
1824 	val &= ~APPL_DM_TYPE_MASK;
1825 	val |= APPL_DM_TYPE_EP;
1826 	appl_writel(pcie, val, APPL_DM_TYPE);
1827 
1828 	appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1829 
1830 	val = appl_readl(pcie, APPL_CTRL);
1831 	val |= APPL_CTRL_SYS_PRE_DET_STATE;
1832 	val |= APPL_CTRL_HW_HOT_RST_EN;
1833 	appl_writel(pcie, val, APPL_CTRL);
1834 
1835 	val = appl_readl(pcie, APPL_CFG_MISC);
1836 	val |= APPL_CFG_MISC_SLV_EP_MODE;
1837 	val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1838 	appl_writel(pcie, val, APPL_CFG_MISC);
1839 
1840 	val = appl_readl(pcie, APPL_PINMUX);
1841 	val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1842 	val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1843 	appl_writel(pcie, val, APPL_PINMUX);
1844 
1845 	appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1846 		    APPL_CFG_BASE_ADDR);
1847 
1848 	appl_writel(pcie, pcie->atu_dma_res->start &
1849 		    APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1850 		    APPL_CFG_IATU_DMA_BASE_ADDR);
1851 
1852 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
1853 	val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
1854 	val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
1855 	val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN;
1856 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
1857 
1858 	val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
1859 	val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN;
1860 	val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN;
1861 	appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
1862 
1863 	reset_control_deassert(pcie->core_rst);
1864 
1865 	if (pcie->update_fc_fixup) {
1866 		val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
1867 		val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
1868 		dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
1869 	}
1870 
1871 	config_gen3_gen4_eq_presets(pcie);
1872 
1873 	init_host_aspm(pcie);
1874 
1875 	/* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
1876 	if (!pcie->supports_clkreq) {
1877 		disable_aspm_l11(pcie);
1878 		disable_aspm_l12(pcie);
1879 	}
1880 
1881 	if (!pcie->of_data->has_l1ss_exit_fix) {
1882 		val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1883 		val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1884 		dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
1885 	}
1886 
1887 	pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1888 						      PCI_CAP_ID_EXP);
1889 
1890 	val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_DEVCTL);
1891 	val_16 &= ~PCI_EXP_DEVCTL_PAYLOAD;
1892 	val_16 |= PCI_EXP_DEVCTL_PAYLOAD_256B;
1893 	dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_DEVCTL, val_16);
1894 
1895 	/* Clear Slot Clock Configuration bit if SRNS configuration */
1896 	if (pcie->enable_srns) {
1897 		val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
1898 					   PCI_EXP_LNKSTA);
1899 		val_16 &= ~PCI_EXP_LNKSTA_SLC;
1900 		dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA,
1901 				   val_16);
1902 	}
1903 
1904 	clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
1905 
1906 	val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK);
1907 	val |= MSIX_ADDR_MATCH_LOW_OFF_EN;
1908 	dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val);
1909 	val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK);
1910 	dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val);
1911 
1912 	ret = dw_pcie_ep_init_complete(ep);
1913 	if (ret) {
1914 		dev_err(dev, "Failed to complete initialization: %d\n", ret);
1915 		goto fail_init_complete;
1916 	}
1917 
1918 	dw_pcie_ep_init_notify(ep);
1919 
1920 	/* Program the private control to allow sending LTR upstream */
1921 	if (pcie->of_data->has_ltr_req_fix) {
1922 		val = appl_readl(pcie, APPL_LTR_MSG_2);
1923 		val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
1924 		appl_writel(pcie, val, APPL_LTR_MSG_2);
1925 	}
1926 
1927 	/* Enable LTSSM */
1928 	val = appl_readl(pcie, APPL_CTRL);
1929 	val |= APPL_CTRL_LTSSM_EN;
1930 	appl_writel(pcie, val, APPL_CTRL);
1931 
1932 	pcie->ep_state = EP_STATE_ENABLED;
1933 	dev_dbg(dev, "Initialization of endpoint is completed\n");
1934 
1935 	return;
1936 
1937 fail_init_complete:
1938 	reset_control_assert(pcie->core_rst);
1939 	tegra_pcie_disable_phy(pcie);
1940 fail_phy:
1941 	reset_control_assert(pcie->core_apb_rst);
1942 fail_core_apb_rst:
1943 	clk_disable_unprepare(pcie->core_clk);
1944 fail_core_clk_enable:
1945 	tegra_pcie_bpmp_set_pll_state(pcie, false);
1946 fail_pll_init:
1947 	tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1948 fail_set_ctrl_state:
1949 	pm_runtime_put_sync(dev);
1950 }
1951 
1952 static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)
1953 {
1954 	struct tegra_pcie_dw *pcie = arg;
1955 
1956 	if (gpiod_get_value(pcie->pex_rst_gpiod))
1957 		pex_ep_event_pex_rst_assert(pcie);
1958 	else
1959 		pex_ep_event_pex_rst_deassert(pcie);
1960 
1961 	return IRQ_HANDLED;
1962 }
1963 
1964 static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq)
1965 {
1966 	/* Tegra194 supports only INTA */
1967 	if (irq > 1)
1968 		return -EINVAL;
1969 
1970 	appl_writel(pcie, 1, APPL_LEGACY_INTX);
1971 	usleep_range(1000, 2000);
1972 	appl_writel(pcie, 0, APPL_LEGACY_INTX);
1973 	return 0;
1974 }
1975 
1976 static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq)
1977 {
1978 	if (unlikely(irq > 31))
1979 		return -EINVAL;
1980 
1981 	appl_writel(pcie, BIT(irq), APPL_MSI_CTRL_1);
1982 
1983 	return 0;
1984 }
1985 
1986 static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq)
1987 {
1988 	struct dw_pcie_ep *ep = &pcie->pci.ep;
1989 
1990 	writel(irq, ep->msi_mem);
1991 
1992 	return 0;
1993 }
1994 
1995 static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1996 				   enum pci_epc_irq_type type,
1997 				   u16 interrupt_num)
1998 {
1999 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
2000 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
2001 
2002 	switch (type) {
2003 	case PCI_EPC_IRQ_LEGACY:
2004 		return tegra_pcie_ep_raise_legacy_irq(pcie, interrupt_num);
2005 
2006 	case PCI_EPC_IRQ_MSI:
2007 		return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num);
2008 
2009 	case PCI_EPC_IRQ_MSIX:
2010 		return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num);
2011 
2012 	default:
2013 		dev_err(pci->dev, "Unknown IRQ type\n");
2014 		return -EPERM;
2015 	}
2016 
2017 	return 0;
2018 }
2019 
2020 static const struct pci_epc_features tegra_pcie_epc_features = {
2021 	.linkup_notifier = true,
2022 	.core_init_notifier = true,
2023 	.msi_capable = false,
2024 	.msix_capable = false,
2025 	.reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5,
2026 	.bar_fixed_64bit = 1 << BAR_0,
2027 	.bar_fixed_size[0] = SZ_1M,
2028 };
2029 
2030 static const struct pci_epc_features*
2031 tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)
2032 {
2033 	return &tegra_pcie_epc_features;
2034 }
2035 
2036 static const struct dw_pcie_ep_ops pcie_ep_ops = {
2037 	.raise_irq = tegra_pcie_ep_raise_irq,
2038 	.get_features = tegra_pcie_ep_get_features,
2039 };
2040 
2041 static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie,
2042 				struct platform_device *pdev)
2043 {
2044 	struct dw_pcie *pci = &pcie->pci;
2045 	struct device *dev = pcie->dev;
2046 	struct dw_pcie_ep *ep;
2047 	char *name;
2048 	int ret;
2049 
2050 	ep = &pci->ep;
2051 	ep->ops = &pcie_ep_ops;
2052 
2053 	ep->page_size = SZ_64K;
2054 
2055 	ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME);
2056 	if (ret < 0) {
2057 		dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n",
2058 			ret);
2059 		return ret;
2060 	}
2061 
2062 	ret = gpiod_to_irq(pcie->pex_rst_gpiod);
2063 	if (ret < 0) {
2064 		dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret);
2065 		return ret;
2066 	}
2067 	pcie->pex_rst_irq = (unsigned int)ret;
2068 
2069 	name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq",
2070 			      pcie->cid);
2071 	if (!name) {
2072 		dev_err(dev, "Failed to create PERST IRQ string\n");
2073 		return -ENOMEM;
2074 	}
2075 
2076 	irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN);
2077 
2078 	pcie->ep_state = EP_STATE_DISABLED;
2079 
2080 	ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL,
2081 					tegra_pcie_ep_pex_rst_irq,
2082 					IRQF_TRIGGER_RISING |
2083 					IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
2084 					name, (void *)pcie);
2085 	if (ret < 0) {
2086 		dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret);
2087 		return ret;
2088 	}
2089 
2090 	pm_runtime_enable(dev);
2091 
2092 	ret = dw_pcie_ep_init(ep);
2093 	if (ret) {
2094 		dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n",
2095 			ret);
2096 		pm_runtime_disable(dev);
2097 		return ret;
2098 	}
2099 
2100 	return 0;
2101 }
2102 
2103 static int tegra_pcie_dw_probe(struct platform_device *pdev)
2104 {
2105 	const struct tegra_pcie_dw_of_data *data;
2106 	struct device *dev = &pdev->dev;
2107 	struct resource *atu_dma_res;
2108 	struct tegra_pcie_dw *pcie;
2109 	struct dw_pcie_rp *pp;
2110 	struct dw_pcie *pci;
2111 	struct phy **phys;
2112 	char *name;
2113 	int ret;
2114 	u32 i;
2115 
2116 	data = of_device_get_match_data(dev);
2117 
2118 	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
2119 	if (!pcie)
2120 		return -ENOMEM;
2121 
2122 	pci = &pcie->pci;
2123 	pci->dev = &pdev->dev;
2124 	pci->ops = &tegra_dw_pcie_ops;
2125 	pcie->dev = &pdev->dev;
2126 	pcie->of_data = (struct tegra_pcie_dw_of_data *)data;
2127 	pci->n_fts[0] = pcie->of_data->n_fts[0];
2128 	pci->n_fts[1] = pcie->of_data->n_fts[1];
2129 	pp = &pci->pp;
2130 	pp->num_vectors = MAX_MSI_IRQS;
2131 
2132 	ret = tegra_pcie_dw_parse_dt(pcie);
2133 	if (ret < 0) {
2134 		const char *level = KERN_ERR;
2135 
2136 		if (ret == -EPROBE_DEFER)
2137 			level = KERN_DEBUG;
2138 
2139 		dev_printk(level, dev,
2140 			   dev_fmt("Failed to parse device tree: %d\n"),
2141 			   ret);
2142 		return ret;
2143 	}
2144 
2145 	ret = tegra_pcie_get_slot_regulators(pcie);
2146 	if (ret < 0) {
2147 		const char *level = KERN_ERR;
2148 
2149 		if (ret == -EPROBE_DEFER)
2150 			level = KERN_DEBUG;
2151 
2152 		dev_printk(level, dev,
2153 			   dev_fmt("Failed to get slot regulators: %d\n"),
2154 			   ret);
2155 		return ret;
2156 	}
2157 
2158 	if (pcie->pex_refclk_sel_gpiod)
2159 		gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1);
2160 
2161 	pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl");
2162 	if (IS_ERR(pcie->pex_ctl_supply)) {
2163 		ret = PTR_ERR(pcie->pex_ctl_supply);
2164 		if (ret != -EPROBE_DEFER)
2165 			dev_err(dev, "Failed to get regulator: %ld\n",
2166 				PTR_ERR(pcie->pex_ctl_supply));
2167 		return ret;
2168 	}
2169 
2170 	pcie->core_clk = devm_clk_get(dev, "core");
2171 	if (IS_ERR(pcie->core_clk)) {
2172 		dev_err(dev, "Failed to get core clock: %ld\n",
2173 			PTR_ERR(pcie->core_clk));
2174 		return PTR_ERR(pcie->core_clk);
2175 	}
2176 
2177 	pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2178 						      "appl");
2179 	if (!pcie->appl_res) {
2180 		dev_err(dev, "Failed to find \"appl\" region\n");
2181 		return -ENODEV;
2182 	}
2183 
2184 	pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res);
2185 	if (IS_ERR(pcie->appl_base))
2186 		return PTR_ERR(pcie->appl_base);
2187 
2188 	pcie->core_apb_rst = devm_reset_control_get(dev, "apb");
2189 	if (IS_ERR(pcie->core_apb_rst)) {
2190 		dev_err(dev, "Failed to get APB reset: %ld\n",
2191 			PTR_ERR(pcie->core_apb_rst));
2192 		return PTR_ERR(pcie->core_apb_rst);
2193 	}
2194 
2195 	phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL);
2196 	if (!phys)
2197 		return -ENOMEM;
2198 
2199 	for (i = 0; i < pcie->phy_count; i++) {
2200 		name = kasprintf(GFP_KERNEL, "p2u-%u", i);
2201 		if (!name) {
2202 			dev_err(dev, "Failed to create P2U string\n");
2203 			return -ENOMEM;
2204 		}
2205 		phys[i] = devm_phy_get(dev, name);
2206 		kfree(name);
2207 		if (IS_ERR(phys[i])) {
2208 			ret = PTR_ERR(phys[i]);
2209 			if (ret != -EPROBE_DEFER)
2210 				dev_err(dev, "Failed to get PHY: %d\n", ret);
2211 			return ret;
2212 		}
2213 	}
2214 
2215 	pcie->phys = phys;
2216 
2217 	atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2218 						   "atu_dma");
2219 	if (!atu_dma_res) {
2220 		dev_err(dev, "Failed to find \"atu_dma\" region\n");
2221 		return -ENODEV;
2222 	}
2223 	pcie->atu_dma_res = atu_dma_res;
2224 
2225 	pci->atu_size = resource_size(atu_dma_res);
2226 	pci->atu_base = devm_ioremap_resource(dev, atu_dma_res);
2227 	if (IS_ERR(pci->atu_base))
2228 		return PTR_ERR(pci->atu_base);
2229 
2230 	pcie->core_rst = devm_reset_control_get(dev, "core");
2231 	if (IS_ERR(pcie->core_rst)) {
2232 		dev_err(dev, "Failed to get core reset: %ld\n",
2233 			PTR_ERR(pcie->core_rst));
2234 		return PTR_ERR(pcie->core_rst);
2235 	}
2236 
2237 	pp->irq = platform_get_irq_byname(pdev, "intr");
2238 	if (pp->irq < 0)
2239 		return pp->irq;
2240 
2241 	pcie->bpmp = tegra_bpmp_get(dev);
2242 	if (IS_ERR(pcie->bpmp))
2243 		return PTR_ERR(pcie->bpmp);
2244 
2245 	platform_set_drvdata(pdev, pcie);
2246 
2247 	pcie->icc_path = devm_of_icc_get(&pdev->dev, "write");
2248 	ret = PTR_ERR_OR_ZERO(pcie->icc_path);
2249 	if (ret) {
2250 		tegra_bpmp_put(pcie->bpmp);
2251 		dev_err_probe(&pdev->dev, ret, "failed to get write interconnect\n");
2252 		return ret;
2253 	}
2254 
2255 	switch (pcie->of_data->mode) {
2256 	case DW_PCIE_RC_TYPE:
2257 		ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler,
2258 				       IRQF_SHARED, "tegra-pcie-intr", pcie);
2259 		if (ret) {
2260 			dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2261 				ret);
2262 			goto fail;
2263 		}
2264 
2265 		ret = tegra_pcie_config_rp(pcie);
2266 		if (ret && ret != -ENOMEDIUM)
2267 			goto fail;
2268 		else
2269 			return 0;
2270 		break;
2271 
2272 	case DW_PCIE_EP_TYPE:
2273 		ret = devm_request_threaded_irq(dev, pp->irq,
2274 						tegra_pcie_ep_hard_irq,
2275 						tegra_pcie_ep_irq_thread,
2276 						IRQF_SHARED | IRQF_ONESHOT,
2277 						"tegra-pcie-ep-intr", pcie);
2278 		if (ret) {
2279 			dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2280 				ret);
2281 			goto fail;
2282 		}
2283 
2284 		ret = tegra_pcie_config_ep(pcie, pdev);
2285 		if (ret < 0)
2286 			goto fail;
2287 		break;
2288 
2289 	default:
2290 		dev_err(dev, "Invalid PCIe device type %d\n",
2291 			pcie->of_data->mode);
2292 	}
2293 
2294 fail:
2295 	tegra_bpmp_put(pcie->bpmp);
2296 	return ret;
2297 }
2298 
2299 static void tegra_pcie_dw_remove(struct platform_device *pdev)
2300 {
2301 	struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2302 
2303 	if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {
2304 		if (!pcie->link_state)
2305 			return;
2306 
2307 		debugfs_remove_recursive(pcie->debugfs);
2308 		tegra_pcie_deinit_controller(pcie);
2309 		pm_runtime_put_sync(pcie->dev);
2310 	} else {
2311 		disable_irq(pcie->pex_rst_irq);
2312 		pex_ep_event_pex_rst_assert(pcie);
2313 	}
2314 
2315 	pm_runtime_disable(pcie->dev);
2316 	tegra_bpmp_put(pcie->bpmp);
2317 	if (pcie->pex_refclk_sel_gpiod)
2318 		gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0);
2319 }
2320 
2321 static int tegra_pcie_dw_suspend_late(struct device *dev)
2322 {
2323 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2324 	u32 val;
2325 
2326 	if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
2327 		dev_err(dev, "Failed to Suspend as Tegra PCIe is in EP mode\n");
2328 		return -EPERM;
2329 	}
2330 
2331 	if (!pcie->link_state)
2332 		return 0;
2333 
2334 	/* Enable HW_HOT_RST mode */
2335 	if (!pcie->of_data->has_sbr_reset_fix) {
2336 		val = appl_readl(pcie, APPL_CTRL);
2337 		val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2338 			 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2339 		val |= APPL_CTRL_HW_HOT_RST_EN;
2340 		appl_writel(pcie, val, APPL_CTRL);
2341 	}
2342 
2343 	return 0;
2344 }
2345 
2346 static int tegra_pcie_dw_suspend_noirq(struct device *dev)
2347 {
2348 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2349 
2350 	if (!pcie->link_state)
2351 		return 0;
2352 
2353 	tegra_pcie_downstream_dev_to_D0(pcie);
2354 	tegra_pcie_dw_pme_turnoff(pcie);
2355 	tegra_pcie_unconfig_controller(pcie);
2356 
2357 	return 0;
2358 }
2359 
2360 static int tegra_pcie_dw_resume_noirq(struct device *dev)
2361 {
2362 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2363 	int ret;
2364 
2365 	if (!pcie->link_state)
2366 		return 0;
2367 
2368 	ret = tegra_pcie_config_controller(pcie, true);
2369 	if (ret < 0)
2370 		return ret;
2371 
2372 	ret = tegra_pcie_dw_host_init(&pcie->pci.pp);
2373 	if (ret < 0) {
2374 		dev_err(dev, "Failed to init host: %d\n", ret);
2375 		goto fail_host_init;
2376 	}
2377 
2378 	dw_pcie_setup_rc(&pcie->pci.pp);
2379 
2380 	ret = tegra_pcie_dw_start_link(&pcie->pci);
2381 	if (ret < 0)
2382 		goto fail_host_init;
2383 
2384 	return 0;
2385 
2386 fail_host_init:
2387 	tegra_pcie_unconfig_controller(pcie);
2388 	return ret;
2389 }
2390 
2391 static int tegra_pcie_dw_resume_early(struct device *dev)
2392 {
2393 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2394 	u32 val;
2395 
2396 	if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {
2397 		dev_err(dev, "Suspend is not supported in EP mode");
2398 		return -ENOTSUPP;
2399 	}
2400 
2401 	if (!pcie->link_state)
2402 		return 0;
2403 
2404 	/* Disable HW_HOT_RST mode */
2405 	if (!pcie->of_data->has_sbr_reset_fix) {
2406 		val = appl_readl(pcie, APPL_CTRL);
2407 		val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2408 			 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2409 		val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<
2410 		       APPL_CTRL_HW_HOT_RST_MODE_SHIFT;
2411 		val &= ~APPL_CTRL_HW_HOT_RST_EN;
2412 		appl_writel(pcie, val, APPL_CTRL);
2413 	}
2414 
2415 	return 0;
2416 }
2417 
2418 static void tegra_pcie_dw_shutdown(struct platform_device *pdev)
2419 {
2420 	struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2421 
2422 	if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {
2423 		if (!pcie->link_state)
2424 			return;
2425 
2426 		debugfs_remove_recursive(pcie->debugfs);
2427 		tegra_pcie_downstream_dev_to_D0(pcie);
2428 
2429 		disable_irq(pcie->pci.pp.irq);
2430 		if (IS_ENABLED(CONFIG_PCI_MSI))
2431 			disable_irq(pcie->pci.pp.msi_irq[0]);
2432 
2433 		tegra_pcie_dw_pme_turnoff(pcie);
2434 		tegra_pcie_unconfig_controller(pcie);
2435 		pm_runtime_put_sync(pcie->dev);
2436 	} else {
2437 		disable_irq(pcie->pex_rst_irq);
2438 		pex_ep_event_pex_rst_assert(pcie);
2439 	}
2440 }
2441 
2442 static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_rc_of_data = {
2443 	.version = TEGRA194_DWC_IP_VER,
2444 	.mode = DW_PCIE_RC_TYPE,
2445 	.cdm_chk_int_en_bit = BIT(19),
2446 	/* Gen4 - 5, 6, 8 and 9 presets enabled */
2447 	.gen4_preset_vec = 0x360,
2448 	.n_fts = { 52, 52 },
2449 };
2450 
2451 static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_ep_of_data = {
2452 	.version = TEGRA194_DWC_IP_VER,
2453 	.mode = DW_PCIE_EP_TYPE,
2454 	.cdm_chk_int_en_bit = BIT(19),
2455 	/* Gen4 - 5, 6, 8 and 9 presets enabled */
2456 	.gen4_preset_vec = 0x360,
2457 	.n_fts = { 52, 52 },
2458 };
2459 
2460 static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_rc_of_data = {
2461 	.version = TEGRA234_DWC_IP_VER,
2462 	.mode = DW_PCIE_RC_TYPE,
2463 	.has_msix_doorbell_access_fix = true,
2464 	.has_sbr_reset_fix = true,
2465 	.has_l1ss_exit_fix = true,
2466 	.cdm_chk_int_en_bit = BIT(18),
2467 	/* Gen4 - 6, 8 and 9 presets enabled */
2468 	.gen4_preset_vec = 0x340,
2469 	.n_fts = { 52, 80 },
2470 };
2471 
2472 static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_ep_of_data = {
2473 	.version = TEGRA234_DWC_IP_VER,
2474 	.mode = DW_PCIE_EP_TYPE,
2475 	.has_l1ss_exit_fix = true,
2476 	.has_ltr_req_fix = true,
2477 	.cdm_chk_int_en_bit = BIT(18),
2478 	/* Gen4 - 6, 8 and 9 presets enabled */
2479 	.gen4_preset_vec = 0x340,
2480 	.n_fts = { 52, 80 },
2481 };
2482 
2483 static const struct of_device_id tegra_pcie_dw_of_match[] = {
2484 	{
2485 		.compatible = "nvidia,tegra194-pcie",
2486 		.data = &tegra194_pcie_dw_rc_of_data,
2487 	},
2488 	{
2489 		.compatible = "nvidia,tegra194-pcie-ep",
2490 		.data = &tegra194_pcie_dw_ep_of_data,
2491 	},
2492 	{
2493 		.compatible = "nvidia,tegra234-pcie",
2494 		.data = &tegra234_pcie_dw_rc_of_data,
2495 	},
2496 	{
2497 		.compatible = "nvidia,tegra234-pcie-ep",
2498 		.data = &tegra234_pcie_dw_ep_of_data,
2499 	},
2500 	{}
2501 };
2502 
2503 static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {
2504 	.suspend_late = tegra_pcie_dw_suspend_late,
2505 	.suspend_noirq = tegra_pcie_dw_suspend_noirq,
2506 	.resume_noirq = tegra_pcie_dw_resume_noirq,
2507 	.resume_early = tegra_pcie_dw_resume_early,
2508 };
2509 
2510 static struct platform_driver tegra_pcie_dw_driver = {
2511 	.probe = tegra_pcie_dw_probe,
2512 	.remove_new = tegra_pcie_dw_remove,
2513 	.shutdown = tegra_pcie_dw_shutdown,
2514 	.driver = {
2515 		.name	= "tegra194-pcie",
2516 		.pm = &tegra_pcie_dw_pm_ops,
2517 		.of_match_table = tegra_pcie_dw_of_match,
2518 	},
2519 };
2520 module_platform_driver(tegra_pcie_dw_driver);
2521 
2522 MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);
2523 
2524 MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");
2525 MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");
2526 MODULE_LICENSE("GPL v2");
2527