1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * PCIe host controller driver for Tegra194 SoC
4  *
5  * Copyright (C) 2019 NVIDIA Corporation.
6  *
7  * Author: Vidya Sagar <vidyas@nvidia.com>
8  */
9 
10 #include <linux/clk.h>
11 #include <linux/debugfs.h>
12 #include <linux/delay.h>
13 #include <linux/gpio.h>
14 #include <linux/gpio/consumer.h>
15 #include <linux/interrupt.h>
16 #include <linux/iopoll.h>
17 #include <linux/kernel.h>
18 #include <linux/module.h>
19 #include <linux/of.h>
20 #include <linux/of_device.h>
21 #include <linux/of_gpio.h>
22 #include <linux/of_irq.h>
23 #include <linux/of_pci.h>
24 #include <linux/pci.h>
25 #include <linux/phy/phy.h>
26 #include <linux/pinctrl/consumer.h>
27 #include <linux/platform_device.h>
28 #include <linux/pm_runtime.h>
29 #include <linux/random.h>
30 #include <linux/reset.h>
31 #include <linux/resource.h>
32 #include <linux/types.h>
33 #include "pcie-designware.h"
34 #include <soc/tegra/bpmp.h>
35 #include <soc/tegra/bpmp-abi.h>
36 #include "../../pci.h"
37 
38 #define APPL_PINMUX				0x0
39 #define APPL_PINMUX_PEX_RST			BIT(0)
40 #define APPL_PINMUX_CLKREQ_OVERRIDE_EN		BIT(2)
41 #define APPL_PINMUX_CLKREQ_OVERRIDE		BIT(3)
42 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN	BIT(4)
43 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE	BIT(5)
44 
45 #define APPL_CTRL				0x4
46 #define APPL_CTRL_SYS_PRE_DET_STATE		BIT(6)
47 #define APPL_CTRL_LTSSM_EN			BIT(7)
48 #define APPL_CTRL_HW_HOT_RST_EN			BIT(20)
49 #define APPL_CTRL_HW_HOT_RST_MODE_MASK		GENMASK(1, 0)
50 #define APPL_CTRL_HW_HOT_RST_MODE_SHIFT		22
51 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST	0x1
52 
53 #define APPL_INTR_EN_L0_0			0x8
54 #define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN	BIT(0)
55 #define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN	BIT(4)
56 #define APPL_INTR_EN_L0_0_INT_INT_EN		BIT(8)
57 #define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN	BIT(15)
58 #define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN	BIT(19)
59 #define APPL_INTR_EN_L0_0_SYS_INTR_EN		BIT(30)
60 #define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN	BIT(31)
61 
62 #define APPL_INTR_STATUS_L0			0xC
63 #define APPL_INTR_STATUS_L0_LINK_STATE_INT	BIT(0)
64 #define APPL_INTR_STATUS_L0_INT_INT		BIT(8)
65 #define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT	BIT(15)
66 #define APPL_INTR_STATUS_L0_PEX_RST_INT		BIT(16)
67 #define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT	BIT(18)
68 
69 #define APPL_INTR_EN_L1_0_0				0x1C
70 #define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN	BIT(1)
71 #define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN		BIT(3)
72 #define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN	BIT(30)
73 
74 #define APPL_INTR_STATUS_L1_0_0				0x20
75 #define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED	BIT(1)
76 #define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED	BIT(3)
77 #define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE		BIT(30)
78 
79 #define APPL_INTR_STATUS_L1_1			0x2C
80 #define APPL_INTR_STATUS_L1_2			0x30
81 #define APPL_INTR_STATUS_L1_3			0x34
82 #define APPL_INTR_STATUS_L1_6			0x3C
83 #define APPL_INTR_STATUS_L1_7			0x40
84 #define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED	BIT(1)
85 
86 #define APPL_INTR_EN_L1_8_0			0x44
87 #define APPL_INTR_EN_L1_8_BW_MGT_INT_EN		BIT(2)
88 #define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN	BIT(3)
89 #define APPL_INTR_EN_L1_8_INTX_EN		BIT(11)
90 #define APPL_INTR_EN_L1_8_AER_INT_EN		BIT(15)
91 
92 #define APPL_INTR_STATUS_L1_8_0			0x4C
93 #define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK	GENMASK(11, 6)
94 #define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS	BIT(2)
95 #define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS	BIT(3)
96 
97 #define APPL_INTR_STATUS_L1_9			0x54
98 #define APPL_INTR_STATUS_L1_10			0x58
99 #define APPL_INTR_STATUS_L1_11			0x64
100 #define APPL_INTR_STATUS_L1_13			0x74
101 #define APPL_INTR_STATUS_L1_14			0x78
102 #define APPL_INTR_STATUS_L1_15			0x7C
103 #define APPL_INTR_STATUS_L1_17			0x88
104 
105 #define APPL_INTR_EN_L1_18				0x90
106 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT		BIT(2)
107 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR		BIT(1)
108 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR	BIT(0)
109 
110 #define APPL_INTR_STATUS_L1_18				0x94
111 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT	BIT(2)
112 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR	BIT(1)
113 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR	BIT(0)
114 
115 #define APPL_MSI_CTRL_1				0xAC
116 
117 #define APPL_MSI_CTRL_2				0xB0
118 
119 #define APPL_LEGACY_INTX			0xB8
120 
121 #define APPL_LTR_MSG_1				0xC4
122 #define LTR_MSG_REQ				BIT(15)
123 #define LTR_MST_NO_SNOOP_SHIFT			16
124 
125 #define APPL_LTR_MSG_2				0xC8
126 #define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE	BIT(3)
127 
128 #define APPL_LINK_STATUS			0xCC
129 #define APPL_LINK_STATUS_RDLH_LINK_UP		BIT(0)
130 
131 #define APPL_DEBUG				0xD0
132 #define APPL_DEBUG_PM_LINKST_IN_L2_LAT		BIT(21)
133 #define APPL_DEBUG_PM_LINKST_IN_L0		0x11
134 #define APPL_DEBUG_LTSSM_STATE_MASK		GENMASK(8, 3)
135 #define APPL_DEBUG_LTSSM_STATE_SHIFT		3
136 #define LTSSM_STATE_PRE_DETECT			5
137 
138 #define APPL_RADM_STATUS			0xE4
139 #define APPL_PM_XMT_TURNOFF_STATE		BIT(0)
140 
141 #define APPL_DM_TYPE				0x100
142 #define APPL_DM_TYPE_MASK			GENMASK(3, 0)
143 #define APPL_DM_TYPE_RP				0x4
144 #define APPL_DM_TYPE_EP				0x0
145 
146 #define APPL_CFG_BASE_ADDR			0x104
147 #define APPL_CFG_BASE_ADDR_MASK			GENMASK(31, 12)
148 
149 #define APPL_CFG_IATU_DMA_BASE_ADDR		0x108
150 #define APPL_CFG_IATU_DMA_BASE_ADDR_MASK	GENMASK(31, 18)
151 
152 #define APPL_CFG_MISC				0x110
153 #define APPL_CFG_MISC_SLV_EP_MODE		BIT(14)
154 #define APPL_CFG_MISC_ARCACHE_MASK		GENMASK(13, 10)
155 #define APPL_CFG_MISC_ARCACHE_SHIFT		10
156 #define APPL_CFG_MISC_ARCACHE_VAL		3
157 
158 #define APPL_CFG_SLCG_OVERRIDE			0x114
159 #define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER	BIT(0)
160 
161 #define APPL_CAR_RESET_OVRD				0x12C
162 #define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N	BIT(0)
163 
164 #define IO_BASE_IO_DECODE				BIT(0)
165 #define IO_BASE_IO_DECODE_BIT8				BIT(8)
166 
167 #define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE		BIT(0)
168 #define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE	BIT(16)
169 
170 #define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF	0x718
171 #define CFG_TIMER_CTRL_ACK_NAK_SHIFT	(19)
172 
173 #define EVENT_COUNTER_ALL_CLEAR		0x3
174 #define EVENT_COUNTER_ENABLE_ALL	0x7
175 #define EVENT_COUNTER_ENABLE_SHIFT	2
176 #define EVENT_COUNTER_EVENT_SEL_MASK	GENMASK(7, 0)
177 #define EVENT_COUNTER_EVENT_SEL_SHIFT	16
178 #define EVENT_COUNTER_EVENT_Tx_L0S	0x2
179 #define EVENT_COUNTER_EVENT_Rx_L0S	0x3
180 #define EVENT_COUNTER_EVENT_L1		0x5
181 #define EVENT_COUNTER_EVENT_L1_1	0x7
182 #define EVENT_COUNTER_EVENT_L1_2	0x8
183 #define EVENT_COUNTER_GROUP_SEL_SHIFT	24
184 #define EVENT_COUNTER_GROUP_5		0x5
185 
186 #define N_FTS_VAL					52
187 #define FTS_VAL						52
188 
189 #define PORT_LOGIC_MSI_CTRL_INT_0_EN		0x828
190 
191 #define GEN3_EQ_CONTROL_OFF			0x8a8
192 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT	8
193 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK	GENMASK(23, 8)
194 #define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK	GENMASK(3, 0)
195 
196 #define GEN3_RELATED_OFF			0x890
197 #define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL	BIT(0)
198 #define GEN3_RELATED_OFF_GEN3_EQ_DISABLE	BIT(16)
199 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT	24
200 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK	GENMASK(25, 24)
201 
202 #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT	0x8D0
203 #define AMBA_ERROR_RESPONSE_CRS_SHIFT		3
204 #define AMBA_ERROR_RESPONSE_CRS_MASK		GENMASK(1, 0)
205 #define AMBA_ERROR_RESPONSE_CRS_OKAY		0
206 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF	1
207 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001	2
208 
209 #define MSIX_ADDR_MATCH_LOW_OFF			0x940
210 #define MSIX_ADDR_MATCH_LOW_OFF_EN		BIT(0)
211 #define MSIX_ADDR_MATCH_LOW_OFF_MASK		GENMASK(31, 2)
212 
213 #define MSIX_ADDR_MATCH_HIGH_OFF		0x944
214 #define MSIX_ADDR_MATCH_HIGH_OFF_MASK		GENMASK(31, 0)
215 
216 #define PORT_LOGIC_MSIX_DOORBELL			0x948
217 
218 #define CAP_SPCIE_CAP_OFF			0x154
219 #define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK	GENMASK(3, 0)
220 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK	GENMASK(11, 8)
221 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT	8
222 
223 #define PME_ACK_TIMEOUT 10000
224 
225 #define LTSSM_TIMEOUT 50000	/* 50ms */
226 
227 #define GEN3_GEN4_EQ_PRESET_INIT	5
228 
229 #define GEN1_CORE_CLK_FREQ	62500000
230 #define GEN2_CORE_CLK_FREQ	125000000
231 #define GEN3_CORE_CLK_FREQ	250000000
232 #define GEN4_CORE_CLK_FREQ	500000000
233 
234 #define LTR_MSG_TIMEOUT		(100 * 1000)
235 
236 #define PERST_DEBOUNCE_TIME	(5 * 1000)
237 
238 #define EP_STATE_DISABLED	0
239 #define EP_STATE_ENABLED	1
240 
241 static const unsigned int pcie_gen_freq[] = {
242 	GEN1_CORE_CLK_FREQ,
243 	GEN2_CORE_CLK_FREQ,
244 	GEN3_CORE_CLK_FREQ,
245 	GEN4_CORE_CLK_FREQ
246 };
247 
248 struct tegra_pcie_dw {
249 	struct device *dev;
250 	struct resource *appl_res;
251 	struct resource *dbi_res;
252 	struct resource *atu_dma_res;
253 	void __iomem *appl_base;
254 	struct clk *core_clk;
255 	struct reset_control *core_apb_rst;
256 	struct reset_control *core_rst;
257 	struct dw_pcie pci;
258 	struct tegra_bpmp *bpmp;
259 
260 	enum dw_pcie_device_mode mode;
261 
262 	bool supports_clkreq;
263 	bool enable_cdm_check;
264 	bool link_state;
265 	bool update_fc_fixup;
266 	u8 init_link_width;
267 	u32 msi_ctrl_int;
268 	u32 num_lanes;
269 	u32 cid;
270 	u32 cfg_link_cap_l1sub;
271 	u32 pcie_cap_base;
272 	u32 aspm_cmrt;
273 	u32 aspm_pwr_on_t;
274 	u32 aspm_l0s_enter_lat;
275 
276 	struct regulator *pex_ctl_supply;
277 	struct regulator *slot_ctl_3v3;
278 	struct regulator *slot_ctl_12v;
279 
280 	unsigned int phy_count;
281 	struct phy **phys;
282 
283 	struct dentry *debugfs;
284 
285 	/* Endpoint mode specific */
286 	struct gpio_desc *pex_rst_gpiod;
287 	struct gpio_desc *pex_refclk_sel_gpiod;
288 	unsigned int pex_rst_irq;
289 	int ep_state;
290 };
291 
292 struct tegra_pcie_dw_of_data {
293 	enum dw_pcie_device_mode mode;
294 };
295 
296 static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)
297 {
298 	return container_of(pci, struct tegra_pcie_dw, pci);
299 }
300 
301 static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,
302 			       const u32 reg)
303 {
304 	writel_relaxed(value, pcie->appl_base + reg);
305 }
306 
307 static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)
308 {
309 	return readl_relaxed(pcie->appl_base + reg);
310 }
311 
312 struct tegra_pcie_soc {
313 	enum dw_pcie_device_mode mode;
314 };
315 
316 static void apply_bad_link_workaround(struct pcie_port *pp)
317 {
318 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
319 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
320 	u32 current_link_width;
321 	u16 val;
322 
323 	/*
324 	 * NOTE:- Since this scenario is uncommon and link as such is not
325 	 * stable anyway, not waiting to confirm if link is really
326 	 * transitioning to Gen-2 speed
327 	 */
328 	val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
329 	if (val & PCI_EXP_LNKSTA_LBMS) {
330 		current_link_width = (val & PCI_EXP_LNKSTA_NLW) >>
331 				     PCI_EXP_LNKSTA_NLW_SHIFT;
332 		if (pcie->init_link_width > current_link_width) {
333 			dev_warn(pci->dev, "PCIe link is bad, width reduced\n");
334 			val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
335 						PCI_EXP_LNKCTL2);
336 			val &= ~PCI_EXP_LNKCTL2_TLS;
337 			val |= PCI_EXP_LNKCTL2_TLS_2_5GT;
338 			dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
339 					   PCI_EXP_LNKCTL2, val);
340 
341 			val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
342 						PCI_EXP_LNKCTL);
343 			val |= PCI_EXP_LNKCTL_RL;
344 			dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
345 					   PCI_EXP_LNKCTL, val);
346 		}
347 	}
348 }
349 
350 static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg)
351 {
352 	struct tegra_pcie_dw *pcie = arg;
353 	struct dw_pcie *pci = &pcie->pci;
354 	struct pcie_port *pp = &pci->pp;
355 	u32 val, tmp;
356 	u16 val_w;
357 
358 	val = appl_readl(pcie, APPL_INTR_STATUS_L0);
359 	if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
360 		val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
361 		if (val & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {
362 			appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0);
363 
364 			/* SBR & Surprise Link Down WAR */
365 			val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
366 			val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
367 			appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
368 			udelay(1);
369 			val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
370 			val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
371 			appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
372 
373 			val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
374 			val |= PORT_LOGIC_SPEED_CHANGE;
375 			dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
376 		}
377 	}
378 
379 	if (val & APPL_INTR_STATUS_L0_INT_INT) {
380 		val = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);
381 		if (val & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {
382 			appl_writel(pcie,
383 				    APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,
384 				    APPL_INTR_STATUS_L1_8_0);
385 			apply_bad_link_workaround(pp);
386 		}
387 		if (val & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {
388 			appl_writel(pcie,
389 				    APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,
390 				    APPL_INTR_STATUS_L1_8_0);
391 
392 			val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
393 						  PCI_EXP_LNKSTA);
394 			dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &
395 				PCI_EXP_LNKSTA_CLS);
396 		}
397 	}
398 
399 	val = appl_readl(pcie, APPL_INTR_STATUS_L0);
400 	if (val & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {
401 		val = appl_readl(pcie, APPL_INTR_STATUS_L1_18);
402 		tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
403 		if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {
404 			dev_info(pci->dev, "CDM check complete\n");
405 			tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;
406 		}
407 		if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {
408 			dev_err(pci->dev, "CDM comparison mismatch\n");
409 			tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;
410 		}
411 		if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {
412 			dev_err(pci->dev, "CDM Logic error\n");
413 			tmp |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;
414 		}
415 		dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, tmp);
416 		tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);
417 		dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", tmp);
418 	}
419 
420 	return IRQ_HANDLED;
421 }
422 
423 static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie)
424 {
425 	u32 val;
426 
427 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
428 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
429 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
430 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
431 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
432 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
433 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
434 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
435 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
436 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
437 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
438 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
439 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
440 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
441 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
442 	appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2);
443 
444 	val = appl_readl(pcie, APPL_CTRL);
445 	val |= APPL_CTRL_LTSSM_EN;
446 	appl_writel(pcie, val, APPL_CTRL);
447 }
448 
449 static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg)
450 {
451 	struct tegra_pcie_dw *pcie = arg;
452 	struct dw_pcie *pci = &pcie->pci;
453 	u32 val, speed;
454 
455 	speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
456 		PCI_EXP_LNKSTA_CLS;
457 	clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
458 
459 	/* If EP doesn't advertise L1SS, just return */
460 	val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
461 	if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2)))
462 		return IRQ_HANDLED;
463 
464 	/* Check if BME is set to '1' */
465 	val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
466 	if (val & PCI_COMMAND_MASTER) {
467 		ktime_t timeout;
468 
469 		/* 110us for both snoop and no-snoop */
470 		val = 110 | (2 << PCI_LTR_SCALE_SHIFT) | LTR_MSG_REQ;
471 		val |= (val << LTR_MST_NO_SNOOP_SHIFT);
472 		appl_writel(pcie, val, APPL_LTR_MSG_1);
473 
474 		/* Send LTR upstream */
475 		val = appl_readl(pcie, APPL_LTR_MSG_2);
476 		val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
477 		appl_writel(pcie, val, APPL_LTR_MSG_2);
478 
479 		timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT);
480 		for (;;) {
481 			val = appl_readl(pcie, APPL_LTR_MSG_2);
482 			if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE))
483 				break;
484 			if (ktime_after(ktime_get(), timeout))
485 				break;
486 			usleep_range(1000, 1100);
487 		}
488 		if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)
489 			dev_err(pcie->dev, "Failed to send LTR message\n");
490 	}
491 
492 	return IRQ_HANDLED;
493 }
494 
495 static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg)
496 {
497 	struct tegra_pcie_dw *pcie = arg;
498 	struct dw_pcie_ep *ep = &pcie->pci.ep;
499 	int spurious = 1;
500 	u32 val, tmp;
501 
502 	val = appl_readl(pcie, APPL_INTR_STATUS_L0);
503 	if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
504 		val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
505 		appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0);
506 
507 		if (val & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE)
508 			pex_ep_event_hot_rst_done(pcie);
509 
510 		if (val & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) {
511 			tmp = appl_readl(pcie, APPL_LINK_STATUS);
512 			if (tmp & APPL_LINK_STATUS_RDLH_LINK_UP) {
513 				dev_dbg(pcie->dev, "Link is up with Host\n");
514 				dw_pcie_ep_linkup(ep);
515 			}
516 		}
517 
518 		spurious = 0;
519 	}
520 
521 	if (val & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) {
522 		val = appl_readl(pcie, APPL_INTR_STATUS_L1_15);
523 		appl_writel(pcie, val, APPL_INTR_STATUS_L1_15);
524 
525 		if (val & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED)
526 			return IRQ_WAKE_THREAD;
527 
528 		spurious = 0;
529 	}
530 
531 	if (spurious) {
532 		dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n",
533 			 val);
534 		appl_writel(pcie, val, APPL_INTR_STATUS_L0);
535 	}
536 
537 	return IRQ_HANDLED;
538 }
539 
540 static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where,
541 				     int size, u32 *val)
542 {
543 	/*
544 	 * This is an endpoint mode specific register happen to appear even
545 	 * when controller is operating in root port mode and system hangs
546 	 * when it is accessed with link being in ASPM-L1 state.
547 	 * So skip accessing it altogether
548 	 */
549 	if (!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) {
550 		*val = 0x00000000;
551 		return PCIBIOS_SUCCESSFUL;
552 	}
553 
554 	return pci_generic_config_read(bus, devfn, where, size, val);
555 }
556 
557 static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where,
558 				     int size, u32 val)
559 {
560 	/*
561 	 * This is an endpoint mode specific register happen to appear even
562 	 * when controller is operating in root port mode and system hangs
563 	 * when it is accessed with link being in ASPM-L1 state.
564 	 * So skip accessing it altogether
565 	 */
566 	if (!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL)
567 		return PCIBIOS_SUCCESSFUL;
568 
569 	return pci_generic_config_write(bus, devfn, where, size, val);
570 }
571 
572 static struct pci_ops tegra_pci_ops = {
573 	.map_bus = dw_pcie_own_conf_map_bus,
574 	.read = tegra_pcie_dw_rd_own_conf,
575 	.write = tegra_pcie_dw_wr_own_conf,
576 };
577 
578 #if defined(CONFIG_PCIEASPM)
579 static const u32 event_cntr_ctrl_offset[] = {
580 	0x1d8,
581 	0x1a8,
582 	0x1a8,
583 	0x1a8,
584 	0x1c4,
585 	0x1d8
586 };
587 
588 static const u32 event_cntr_data_offset[] = {
589 	0x1dc,
590 	0x1ac,
591 	0x1ac,
592 	0x1ac,
593 	0x1c8,
594 	0x1dc
595 };
596 
597 static void disable_aspm_l11(struct tegra_pcie_dw *pcie)
598 {
599 	u32 val;
600 
601 	val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
602 	val &= ~PCI_L1SS_CAP_ASPM_L1_1;
603 	dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
604 }
605 
606 static void disable_aspm_l12(struct tegra_pcie_dw *pcie)
607 {
608 	u32 val;
609 
610 	val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
611 	val &= ~PCI_L1SS_CAP_ASPM_L1_2;
612 	dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
613 }
614 
615 static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)
616 {
617 	u32 val;
618 
619 	val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid]);
620 	val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);
621 	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
622 	val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;
623 	val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
624 	dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
625 	val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_data_offset[pcie->cid]);
626 
627 	return val;
628 }
629 
630 static int aspm_state_cnt(struct seq_file *s, void *data)
631 {
632 	struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)
633 				     dev_get_drvdata(s->private);
634 	u32 val;
635 
636 	seq_printf(s, "Tx L0s entry count : %u\n",
637 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));
638 
639 	seq_printf(s, "Rx L0s entry count : %u\n",
640 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));
641 
642 	seq_printf(s, "Link L1 entry count : %u\n",
643 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));
644 
645 	seq_printf(s, "Link L1.1 entry count : %u\n",
646 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));
647 
648 	seq_printf(s, "Link L1.2 entry count : %u\n",
649 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));
650 
651 	/* Clear all counters */
652 	dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid],
653 			   EVENT_COUNTER_ALL_CLEAR);
654 
655 	/* Re-enable counting */
656 	val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
657 	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
658 	dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
659 
660 	return 0;
661 }
662 
663 static void init_host_aspm(struct tegra_pcie_dw *pcie)
664 {
665 	struct dw_pcie *pci = &pcie->pci;
666 	u32 val;
667 
668 	val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
669 	pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;
670 
671 	/* Enable ASPM counters */
672 	val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
673 	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
674 	dw_pcie_writel_dbi(pci, event_cntr_ctrl_offset[pcie->cid], val);
675 
676 	/* Program T_cmrt and T_pwr_on values */
677 	val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
678 	val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);
679 	val |= (pcie->aspm_cmrt << 8);
680 	val |= (pcie->aspm_pwr_on_t << 19);
681 	dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val);
682 
683 	/* Program L0s and L1 entrance latencies */
684 	val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR);
685 	val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK;
686 	val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT);
687 	val |= PORT_AFR_ENTER_ASPM;
688 	dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val);
689 }
690 
691 static void init_debugfs(struct tegra_pcie_dw *pcie)
692 {
693 	debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", pcie->debugfs,
694 				    aspm_state_cnt);
695 }
696 #else
697 static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }
698 static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }
699 static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }
700 static inline void init_debugfs(struct tegra_pcie_dw *pcie) { return; }
701 #endif
702 
703 static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp)
704 {
705 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
706 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
707 	u32 val;
708 	u16 val_w;
709 
710 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
711 	val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
712 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
713 
714 	val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
715 	val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;
716 	appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
717 
718 	if (pcie->enable_cdm_check) {
719 		val = appl_readl(pcie, APPL_INTR_EN_L0_0);
720 		val |= APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN;
721 		appl_writel(pcie, val, APPL_INTR_EN_L0_0);
722 
723 		val = appl_readl(pcie, APPL_INTR_EN_L1_18);
724 		val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;
725 		val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;
726 		appl_writel(pcie, val, APPL_INTR_EN_L1_18);
727 	}
728 
729 	val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
730 				  PCI_EXP_LNKSTA);
731 	pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >>
732 				PCI_EXP_LNKSTA_NLW_SHIFT;
733 
734 	val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
735 				  PCI_EXP_LNKCTL);
736 	val_w |= PCI_EXP_LNKCTL_LBMIE;
737 	dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL,
738 			   val_w);
739 }
740 
741 static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp)
742 {
743 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
744 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
745 	u32 val;
746 
747 	/* Enable legacy interrupt generation */
748 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
749 	val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
750 	val |= APPL_INTR_EN_L0_0_INT_INT_EN;
751 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
752 
753 	val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);
754 	val |= APPL_INTR_EN_L1_8_INTX_EN;
755 	val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;
756 	val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;
757 	if (IS_ENABLED(CONFIG_PCIEAER))
758 		val |= APPL_INTR_EN_L1_8_AER_INT_EN;
759 	appl_writel(pcie, val, APPL_INTR_EN_L1_8_0);
760 }
761 
762 static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp)
763 {
764 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
765 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
766 	u32 val;
767 
768 	/* Enable MSI interrupt generation */
769 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
770 	val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;
771 	val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;
772 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
773 }
774 
775 static void tegra_pcie_enable_interrupts(struct pcie_port *pp)
776 {
777 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
778 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
779 
780 	/* Clear interrupt statuses before enabling interrupts */
781 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
782 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
783 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
784 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
785 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
786 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
787 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
788 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
789 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
790 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
791 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
792 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
793 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
794 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
795 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
796 
797 	tegra_pcie_enable_system_interrupts(pp);
798 	tegra_pcie_enable_legacy_interrupts(pp);
799 	if (IS_ENABLED(CONFIG_PCI_MSI))
800 		tegra_pcie_enable_msi_interrupts(pp);
801 }
802 
803 static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
804 {
805 	struct dw_pcie *pci = &pcie->pci;
806 	u32 val, offset, i;
807 
808 	/* Program init preset */
809 	for (i = 0; i < pcie->num_lanes; i++) {
810 		val = dw_pcie_readw_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2));
811 		val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;
812 		val |= GEN3_GEN4_EQ_PRESET_INIT;
813 		val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;
814 		val |= (GEN3_GEN4_EQ_PRESET_INIT <<
815 			   CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);
816 		dw_pcie_writew_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2), val);
817 
818 		offset = dw_pcie_find_ext_capability(pci,
819 						     PCI_EXT_CAP_ID_PL_16GT) +
820 				PCI_PL_16GT_LE_CTRL;
821 		val = dw_pcie_readb_dbi(pci, offset + i);
822 		val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;
823 		val |= GEN3_GEN4_EQ_PRESET_INIT;
824 		val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;
825 		val |= (GEN3_GEN4_EQ_PRESET_INIT <<
826 			PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);
827 		dw_pcie_writeb_dbi(pci, offset + i, val);
828 	}
829 
830 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
831 	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
832 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
833 
834 	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
835 	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
836 	val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
837 	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
838 	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
839 
840 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
841 	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
842 	val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);
843 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
844 
845 	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
846 	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
847 	val |= (0x360 << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
848 	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
849 	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
850 
851 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
852 	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
853 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
854 }
855 
856 static int tegra_pcie_dw_host_init(struct pcie_port *pp)
857 {
858 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
859 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
860 	u32 val;
861 
862 	pp->bridge->ops = &tegra_pci_ops;
863 
864 	if (!pcie->pcie_cap_base)
865 		pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
866 							      PCI_CAP_ID_EXP);
867 
868 	val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);
869 	val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);
870 	dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);
871 
872 	val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);
873 	val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;
874 	val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;
875 	dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);
876 
877 	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
878 
879 	/* Enable as 0xFFFF0001 response for CRS */
880 	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);
881 	val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT);
882 	val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 <<
883 		AMBA_ERROR_RESPONSE_CRS_SHIFT);
884 	dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);
885 
886 	/* Configure Max lane width from DT */
887 	val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP);
888 	val &= ~PCI_EXP_LNKCAP_MLW;
889 	val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT);
890 	dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val);
891 
892 	config_gen3_gen4_eq_presets(pcie);
893 
894 	init_host_aspm(pcie);
895 
896 	/* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
897 	if (!pcie->supports_clkreq) {
898 		disable_aspm_l11(pcie);
899 		disable_aspm_l12(pcie);
900 	}
901 
902 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
903 	val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
904 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
905 
906 	if (pcie->update_fc_fixup) {
907 		val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
908 		val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
909 		dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
910 	}
911 
912 	clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
913 
914 	return 0;
915 }
916 
917 static int tegra_pcie_dw_start_link(struct dw_pcie *pci)
918 {
919 	u32 val, offset, speed, tmp;
920 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
921 	struct pcie_port *pp = &pci->pp;
922 	bool retry = true;
923 
924 	if (pcie->mode == DW_PCIE_EP_TYPE) {
925 		enable_irq(pcie->pex_rst_irq);
926 		return 0;
927 	}
928 
929 retry_link:
930 	/* Assert RST */
931 	val = appl_readl(pcie, APPL_PINMUX);
932 	val &= ~APPL_PINMUX_PEX_RST;
933 	appl_writel(pcie, val, APPL_PINMUX);
934 
935 	usleep_range(100, 200);
936 
937 	/* Enable LTSSM */
938 	val = appl_readl(pcie, APPL_CTRL);
939 	val |= APPL_CTRL_LTSSM_EN;
940 	appl_writel(pcie, val, APPL_CTRL);
941 
942 	/* De-assert RST */
943 	val = appl_readl(pcie, APPL_PINMUX);
944 	val |= APPL_PINMUX_PEX_RST;
945 	appl_writel(pcie, val, APPL_PINMUX);
946 
947 	msleep(100);
948 
949 	if (dw_pcie_wait_for_link(pci)) {
950 		if (!retry)
951 			return 0;
952 		/*
953 		 * There are some endpoints which can't get the link up if
954 		 * root port has Data Link Feature (DLF) enabled.
955 		 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info
956 		 * on Scaled Flow Control and DLF.
957 		 * So, need to confirm that is indeed the case here and attempt
958 		 * link up once again with DLF disabled.
959 		 */
960 		val = appl_readl(pcie, APPL_DEBUG);
961 		val &= APPL_DEBUG_LTSSM_STATE_MASK;
962 		val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;
963 		tmp = appl_readl(pcie, APPL_LINK_STATUS);
964 		tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;
965 		if (!(val == 0x11 && !tmp)) {
966 			/* Link is down for all good reasons */
967 			return 0;
968 		}
969 
970 		dev_info(pci->dev, "Link is down in DLL");
971 		dev_info(pci->dev, "Trying again with DLFE disabled\n");
972 		/* Disable LTSSM */
973 		val = appl_readl(pcie, APPL_CTRL);
974 		val &= ~APPL_CTRL_LTSSM_EN;
975 		appl_writel(pcie, val, APPL_CTRL);
976 
977 		reset_control_assert(pcie->core_rst);
978 		reset_control_deassert(pcie->core_rst);
979 
980 		offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);
981 		val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP);
982 		val &= ~PCI_DLF_EXCHANGE_ENABLE;
983 		dw_pcie_writel_dbi(pci, offset, val);
984 
985 		tegra_pcie_dw_host_init(pp);
986 		dw_pcie_setup_rc(pp);
987 
988 		retry = false;
989 		goto retry_link;
990 	}
991 
992 	speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
993 		PCI_EXP_LNKSTA_CLS;
994 	clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
995 
996 	tegra_pcie_enable_interrupts(pp);
997 
998 	return 0;
999 }
1000 
1001 static int tegra_pcie_dw_link_up(struct dw_pcie *pci)
1002 {
1003 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1004 	u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
1005 
1006 	return !!(val & PCI_EXP_LNKSTA_DLLLA);
1007 }
1008 
1009 static void tegra_pcie_dw_stop_link(struct dw_pcie *pci)
1010 {
1011 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1012 
1013 	disable_irq(pcie->pex_rst_irq);
1014 }
1015 
1016 static const struct dw_pcie_ops tegra_dw_pcie_ops = {
1017 	.link_up = tegra_pcie_dw_link_up,
1018 	.start_link = tegra_pcie_dw_start_link,
1019 	.stop_link = tegra_pcie_dw_stop_link,
1020 };
1021 
1022 static const struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {
1023 	.host_init = tegra_pcie_dw_host_init,
1024 };
1025 
1026 static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)
1027 {
1028 	unsigned int phy_count = pcie->phy_count;
1029 
1030 	while (phy_count--) {
1031 		phy_power_off(pcie->phys[phy_count]);
1032 		phy_exit(pcie->phys[phy_count]);
1033 	}
1034 }
1035 
1036 static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)
1037 {
1038 	unsigned int i;
1039 	int ret;
1040 
1041 	for (i = 0; i < pcie->phy_count; i++) {
1042 		ret = phy_init(pcie->phys[i]);
1043 		if (ret < 0)
1044 			goto phy_power_off;
1045 
1046 		ret = phy_power_on(pcie->phys[i]);
1047 		if (ret < 0)
1048 			goto phy_exit;
1049 	}
1050 
1051 	return 0;
1052 
1053 phy_power_off:
1054 	while (i--) {
1055 		phy_power_off(pcie->phys[i]);
1056 phy_exit:
1057 		phy_exit(pcie->phys[i]);
1058 	}
1059 
1060 	return ret;
1061 }
1062 
1063 static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
1064 {
1065 	struct platform_device *pdev = to_platform_device(pcie->dev);
1066 	struct device_node *np = pcie->dev->of_node;
1067 	int ret;
1068 
1069 	pcie->dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
1070 	if (!pcie->dbi_res) {
1071 		dev_err(pcie->dev, "Failed to find \"dbi\" region\n");
1072 		return -ENODEV;
1073 	}
1074 
1075 	ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt);
1076 	if (ret < 0) {
1077 		dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);
1078 		return ret;
1079 	}
1080 
1081 	ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us",
1082 				   &pcie->aspm_pwr_on_t);
1083 	if (ret < 0)
1084 		dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",
1085 			 ret);
1086 
1087 	ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us",
1088 				   &pcie->aspm_l0s_enter_lat);
1089 	if (ret < 0)
1090 		dev_info(pcie->dev,
1091 			 "Failed to read ASPM L0s Entrance latency: %d\n", ret);
1092 
1093 	ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);
1094 	if (ret < 0) {
1095 		dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
1096 		return ret;
1097 	}
1098 
1099 	ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid);
1100 	if (ret) {
1101 		dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);
1102 		return ret;
1103 	}
1104 
1105 	ret = of_property_count_strings(np, "phy-names");
1106 	if (ret < 0) {
1107 		dev_err(pcie->dev, "Failed to find PHY entries: %d\n",
1108 			ret);
1109 		return ret;
1110 	}
1111 	pcie->phy_count = ret;
1112 
1113 	if (of_property_read_bool(np, "nvidia,update-fc-fixup"))
1114 		pcie->update_fc_fixup = true;
1115 
1116 	pcie->supports_clkreq =
1117 		of_property_read_bool(pcie->dev->of_node, "supports-clkreq");
1118 
1119 	pcie->enable_cdm_check =
1120 		of_property_read_bool(np, "snps,enable-cdm-check");
1121 
1122 	if (pcie->mode == DW_PCIE_RC_TYPE)
1123 		return 0;
1124 
1125 	/* Endpoint mode specific DT entries */
1126 	pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN);
1127 	if (IS_ERR(pcie->pex_rst_gpiod)) {
1128 		int err = PTR_ERR(pcie->pex_rst_gpiod);
1129 		const char *level = KERN_ERR;
1130 
1131 		if (err == -EPROBE_DEFER)
1132 			level = KERN_DEBUG;
1133 
1134 		dev_printk(level, pcie->dev,
1135 			   dev_fmt("Failed to get PERST GPIO: %d\n"),
1136 			   err);
1137 		return err;
1138 	}
1139 
1140 	pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev,
1141 						    "nvidia,refclk-select",
1142 						    GPIOD_OUT_HIGH);
1143 	if (IS_ERR(pcie->pex_refclk_sel_gpiod)) {
1144 		int err = PTR_ERR(pcie->pex_refclk_sel_gpiod);
1145 		const char *level = KERN_ERR;
1146 
1147 		if (err == -EPROBE_DEFER)
1148 			level = KERN_DEBUG;
1149 
1150 		dev_printk(level, pcie->dev,
1151 			   dev_fmt("Failed to get REFCLK select GPIOs: %d\n"),
1152 			   err);
1153 		pcie->pex_refclk_sel_gpiod = NULL;
1154 	}
1155 
1156 	return 0;
1157 }
1158 
1159 static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,
1160 					  bool enable)
1161 {
1162 	struct mrq_uphy_response resp;
1163 	struct tegra_bpmp_message msg;
1164 	struct mrq_uphy_request req;
1165 
1166 	/* Controller-5 doesn't need to have its state set by BPMP-FW */
1167 	if (pcie->cid == 5)
1168 		return 0;
1169 
1170 	memset(&req, 0, sizeof(req));
1171 	memset(&resp, 0, sizeof(resp));
1172 
1173 	req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
1174 	req.controller_state.pcie_controller = pcie->cid;
1175 	req.controller_state.enable = enable;
1176 
1177 	memset(&msg, 0, sizeof(msg));
1178 	msg.mrq = MRQ_UPHY;
1179 	msg.tx.data = &req;
1180 	msg.tx.size = sizeof(req);
1181 	msg.rx.data = &resp;
1182 	msg.rx.size = sizeof(resp);
1183 
1184 	return tegra_bpmp_transfer(pcie->bpmp, &msg);
1185 }
1186 
1187 static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie,
1188 					 bool enable)
1189 {
1190 	struct mrq_uphy_response resp;
1191 	struct tegra_bpmp_message msg;
1192 	struct mrq_uphy_request req;
1193 
1194 	memset(&req, 0, sizeof(req));
1195 	memset(&resp, 0, sizeof(resp));
1196 
1197 	if (enable) {
1198 		req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;
1199 		req.ep_ctrlr_pll_init.ep_controller = pcie->cid;
1200 	} else {
1201 		req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;
1202 		req.ep_ctrlr_pll_off.ep_controller = pcie->cid;
1203 	}
1204 
1205 	memset(&msg, 0, sizeof(msg));
1206 	msg.mrq = MRQ_UPHY;
1207 	msg.tx.data = &req;
1208 	msg.tx.size = sizeof(req);
1209 	msg.rx.data = &resp;
1210 	msg.rx.size = sizeof(resp);
1211 
1212 	return tegra_bpmp_transfer(pcie->bpmp, &msg);
1213 }
1214 
1215 static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)
1216 {
1217 	struct pcie_port *pp = &pcie->pci.pp;
1218 	struct pci_bus *child, *root_bus = NULL;
1219 	struct pci_dev *pdev;
1220 
1221 	/*
1222 	 * link doesn't go into L2 state with some of the endpoints with Tegra
1223 	 * if they are not in D0 state. So, need to make sure that immediate
1224 	 * downstream devices are in D0 state before sending PME_TurnOff to put
1225 	 * link into L2 state.
1226 	 * This is as per PCI Express Base r4.0 v1.0 September 27-2017,
1227 	 * 5.2 Link State Power Management (Page #428).
1228 	 */
1229 
1230 	list_for_each_entry(child, &pp->bridge->bus->children, node) {
1231 		/* Bring downstream devices to D0 if they are not already in */
1232 		if (child->parent == pp->bridge->bus) {
1233 			root_bus = child;
1234 			break;
1235 		}
1236 	}
1237 
1238 	if (!root_bus) {
1239 		dev_err(pcie->dev, "Failed to find downstream devices\n");
1240 		return;
1241 	}
1242 
1243 	list_for_each_entry(pdev, &root_bus->devices, bus_list) {
1244 		if (PCI_SLOT(pdev->devfn) == 0) {
1245 			if (pci_set_power_state(pdev, PCI_D0))
1246 				dev_err(pcie->dev,
1247 					"Failed to transition %s to D0 state\n",
1248 					dev_name(&pdev->dev));
1249 		}
1250 	}
1251 }
1252 
1253 static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)
1254 {
1255 	pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3");
1256 	if (IS_ERR(pcie->slot_ctl_3v3)) {
1257 		if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV)
1258 			return PTR_ERR(pcie->slot_ctl_3v3);
1259 
1260 		pcie->slot_ctl_3v3 = NULL;
1261 	}
1262 
1263 	pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v");
1264 	if (IS_ERR(pcie->slot_ctl_12v)) {
1265 		if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV)
1266 			return PTR_ERR(pcie->slot_ctl_12v);
1267 
1268 		pcie->slot_ctl_12v = NULL;
1269 	}
1270 
1271 	return 0;
1272 }
1273 
1274 static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)
1275 {
1276 	int ret;
1277 
1278 	if (pcie->slot_ctl_3v3) {
1279 		ret = regulator_enable(pcie->slot_ctl_3v3);
1280 		if (ret < 0) {
1281 			dev_err(pcie->dev,
1282 				"Failed to enable 3.3V slot supply: %d\n", ret);
1283 			return ret;
1284 		}
1285 	}
1286 
1287 	if (pcie->slot_ctl_12v) {
1288 		ret = regulator_enable(pcie->slot_ctl_12v);
1289 		if (ret < 0) {
1290 			dev_err(pcie->dev,
1291 				"Failed to enable 12V slot supply: %d\n", ret);
1292 			goto fail_12v_enable;
1293 		}
1294 	}
1295 
1296 	/*
1297 	 * According to PCI Express Card Electromechanical Specification
1298 	 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)
1299 	 * should be a minimum of 100ms.
1300 	 */
1301 	if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)
1302 		msleep(100);
1303 
1304 	return 0;
1305 
1306 fail_12v_enable:
1307 	if (pcie->slot_ctl_3v3)
1308 		regulator_disable(pcie->slot_ctl_3v3);
1309 	return ret;
1310 }
1311 
1312 static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)
1313 {
1314 	if (pcie->slot_ctl_12v)
1315 		regulator_disable(pcie->slot_ctl_12v);
1316 	if (pcie->slot_ctl_3v3)
1317 		regulator_disable(pcie->slot_ctl_3v3);
1318 }
1319 
1320 static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,
1321 					bool en_hw_hot_rst)
1322 {
1323 	int ret;
1324 	u32 val;
1325 
1326 	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1327 	if (ret) {
1328 		dev_err(pcie->dev,
1329 			"Failed to enable controller %u: %d\n", pcie->cid, ret);
1330 		return ret;
1331 	}
1332 
1333 	ret = tegra_pcie_enable_slot_regulators(pcie);
1334 	if (ret < 0)
1335 		goto fail_slot_reg_en;
1336 
1337 	ret = regulator_enable(pcie->pex_ctl_supply);
1338 	if (ret < 0) {
1339 		dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);
1340 		goto fail_reg_en;
1341 	}
1342 
1343 	ret = clk_prepare_enable(pcie->core_clk);
1344 	if (ret) {
1345 		dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);
1346 		goto fail_core_clk;
1347 	}
1348 
1349 	ret = reset_control_deassert(pcie->core_apb_rst);
1350 	if (ret) {
1351 		dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",
1352 			ret);
1353 		goto fail_core_apb_rst;
1354 	}
1355 
1356 	if (en_hw_hot_rst) {
1357 		/* Enable HW_HOT_RST mode */
1358 		val = appl_readl(pcie, APPL_CTRL);
1359 		val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1360 			 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1361 		val |= APPL_CTRL_HW_HOT_RST_EN;
1362 		appl_writel(pcie, val, APPL_CTRL);
1363 	}
1364 
1365 	ret = tegra_pcie_enable_phy(pcie);
1366 	if (ret) {
1367 		dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);
1368 		goto fail_phy;
1369 	}
1370 
1371 	/* Update CFG base address */
1372 	appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1373 		    APPL_CFG_BASE_ADDR);
1374 
1375 	/* Configure this core for RP mode operation */
1376 	appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);
1377 
1378 	appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1379 
1380 	val = appl_readl(pcie, APPL_CTRL);
1381 	appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);
1382 
1383 	val = appl_readl(pcie, APPL_CFG_MISC);
1384 	val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1385 	appl_writel(pcie, val, APPL_CFG_MISC);
1386 
1387 	if (!pcie->supports_clkreq) {
1388 		val = appl_readl(pcie, APPL_PINMUX);
1389 		val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN;
1390 		val &= ~APPL_PINMUX_CLKREQ_OVERRIDE;
1391 		appl_writel(pcie, val, APPL_PINMUX);
1392 	}
1393 
1394 	/* Update iATU_DMA base address */
1395 	appl_writel(pcie,
1396 		    pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1397 		    APPL_CFG_IATU_DMA_BASE_ADDR);
1398 
1399 	reset_control_deassert(pcie->core_rst);
1400 
1401 	return ret;
1402 
1403 fail_phy:
1404 	reset_control_assert(pcie->core_apb_rst);
1405 fail_core_apb_rst:
1406 	clk_disable_unprepare(pcie->core_clk);
1407 fail_core_clk:
1408 	regulator_disable(pcie->pex_ctl_supply);
1409 fail_reg_en:
1410 	tegra_pcie_disable_slot_regulators(pcie);
1411 fail_slot_reg_en:
1412 	tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1413 
1414 	return ret;
1415 }
1416 
1417 static void tegra_pcie_unconfig_controller(struct tegra_pcie_dw *pcie)
1418 {
1419 	int ret;
1420 
1421 	ret = reset_control_assert(pcie->core_rst);
1422 	if (ret)
1423 		dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", ret);
1424 
1425 	tegra_pcie_disable_phy(pcie);
1426 
1427 	ret = reset_control_assert(pcie->core_apb_rst);
1428 	if (ret)
1429 		dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);
1430 
1431 	clk_disable_unprepare(pcie->core_clk);
1432 
1433 	ret = regulator_disable(pcie->pex_ctl_supply);
1434 	if (ret)
1435 		dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);
1436 
1437 	tegra_pcie_disable_slot_regulators(pcie);
1438 
1439 	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1440 	if (ret)
1441 		dev_err(pcie->dev, "Failed to disable controller %d: %d\n",
1442 			pcie->cid, ret);
1443 }
1444 
1445 static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)
1446 {
1447 	struct dw_pcie *pci = &pcie->pci;
1448 	struct pcie_port *pp = &pci->pp;
1449 	int ret;
1450 
1451 	ret = tegra_pcie_config_controller(pcie, false);
1452 	if (ret < 0)
1453 		return ret;
1454 
1455 	pp->ops = &tegra_pcie_dw_host_ops;
1456 
1457 	ret = dw_pcie_host_init(pp);
1458 	if (ret < 0) {
1459 		dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);
1460 		goto fail_host_init;
1461 	}
1462 
1463 	return 0;
1464 
1465 fail_host_init:
1466 	tegra_pcie_unconfig_controller(pcie);
1467 	return ret;
1468 }
1469 
1470 static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)
1471 {
1472 	u32 val;
1473 
1474 	if (!tegra_pcie_dw_link_up(&pcie->pci))
1475 		return 0;
1476 
1477 	val = appl_readl(pcie, APPL_RADM_STATUS);
1478 	val |= APPL_PM_XMT_TURNOFF_STATE;
1479 	appl_writel(pcie, val, APPL_RADM_STATUS);
1480 
1481 	return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,
1482 				 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,
1483 				 1, PME_ACK_TIMEOUT);
1484 }
1485 
1486 static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)
1487 {
1488 	u32 data;
1489 	int err;
1490 
1491 	if (!tegra_pcie_dw_link_up(&pcie->pci)) {
1492 		dev_dbg(pcie->dev, "PCIe link is not up...!\n");
1493 		return;
1494 	}
1495 
1496 	if (tegra_pcie_try_link_l2(pcie)) {
1497 		dev_info(pcie->dev, "Link didn't transition to L2 state\n");
1498 		/*
1499 		 * TX lane clock freq will reset to Gen1 only if link is in L2
1500 		 * or detect state.
1501 		 * So apply pex_rst to end point to force RP to go into detect
1502 		 * state
1503 		 */
1504 		data = appl_readl(pcie, APPL_PINMUX);
1505 		data &= ~APPL_PINMUX_PEX_RST;
1506 		appl_writel(pcie, data, APPL_PINMUX);
1507 
1508 		/*
1509 		 * Some cards do not go to detect state even after de-asserting
1510 		 * PERST#. So, de-assert LTSSM to bring link to detect state.
1511 		 */
1512 		data = readl(pcie->appl_base + APPL_CTRL);
1513 		data &= ~APPL_CTRL_LTSSM_EN;
1514 		writel(data, pcie->appl_base + APPL_CTRL);
1515 
1516 		err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,
1517 						data,
1518 						((data &
1519 						APPL_DEBUG_LTSSM_STATE_MASK) >>
1520 						APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1521 						LTSSM_STATE_PRE_DETECT,
1522 						1, LTSSM_TIMEOUT);
1523 		if (err)
1524 			dev_info(pcie->dev, "Link didn't go to detect state\n");
1525 	}
1526 	/*
1527 	 * DBI registers may not be accessible after this as PLL-E would be
1528 	 * down depending on how CLKREQ is pulled by end point
1529 	 */
1530 	data = appl_readl(pcie, APPL_PINMUX);
1531 	data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);
1532 	/* Cut REFCLK to slot */
1533 	data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1534 	data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1535 	appl_writel(pcie, data, APPL_PINMUX);
1536 }
1537 
1538 static void tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)
1539 {
1540 	tegra_pcie_downstream_dev_to_D0(pcie);
1541 	dw_pcie_host_deinit(&pcie->pci.pp);
1542 	tegra_pcie_dw_pme_turnoff(pcie);
1543 	tegra_pcie_unconfig_controller(pcie);
1544 }
1545 
1546 static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
1547 {
1548 	struct device *dev = pcie->dev;
1549 	char *name;
1550 	int ret;
1551 
1552 	pm_runtime_enable(dev);
1553 
1554 	ret = pm_runtime_get_sync(dev);
1555 	if (ret < 0) {
1556 		dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1557 			ret);
1558 		goto fail_pm_get_sync;
1559 	}
1560 
1561 	ret = pinctrl_pm_select_default_state(dev);
1562 	if (ret < 0) {
1563 		dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
1564 		goto fail_pm_get_sync;
1565 	}
1566 
1567 	ret = tegra_pcie_init_controller(pcie);
1568 	if (ret < 0) {
1569 		dev_err(dev, "Failed to initialize controller: %d\n", ret);
1570 		goto fail_pm_get_sync;
1571 	}
1572 
1573 	pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci);
1574 	if (!pcie->link_state) {
1575 		ret = -ENOMEDIUM;
1576 		goto fail_host_init;
1577 	}
1578 
1579 	name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
1580 	if (!name) {
1581 		ret = -ENOMEM;
1582 		goto fail_host_init;
1583 	}
1584 
1585 	pcie->debugfs = debugfs_create_dir(name, NULL);
1586 	init_debugfs(pcie);
1587 
1588 	return ret;
1589 
1590 fail_host_init:
1591 	tegra_pcie_deinit_controller(pcie);
1592 fail_pm_get_sync:
1593 	pm_runtime_put_sync(dev);
1594 	pm_runtime_disable(dev);
1595 	return ret;
1596 }
1597 
1598 static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie)
1599 {
1600 	u32 val;
1601 	int ret;
1602 
1603 	if (pcie->ep_state == EP_STATE_DISABLED)
1604 		return;
1605 
1606 	/* Disable LTSSM */
1607 	val = appl_readl(pcie, APPL_CTRL);
1608 	val &= ~APPL_CTRL_LTSSM_EN;
1609 	appl_writel(pcie, val, APPL_CTRL);
1610 
1611 	ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val,
1612 				 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >>
1613 				 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1614 				 LTSSM_STATE_PRE_DETECT,
1615 				 1, LTSSM_TIMEOUT);
1616 	if (ret)
1617 		dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret);
1618 
1619 	reset_control_assert(pcie->core_rst);
1620 
1621 	tegra_pcie_disable_phy(pcie);
1622 
1623 	reset_control_assert(pcie->core_apb_rst);
1624 
1625 	clk_disable_unprepare(pcie->core_clk);
1626 
1627 	pm_runtime_put_sync(pcie->dev);
1628 
1629 	ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1630 	if (ret)
1631 		dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret);
1632 
1633 	pcie->ep_state = EP_STATE_DISABLED;
1634 	dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n");
1635 }
1636 
1637 static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie)
1638 {
1639 	struct dw_pcie *pci = &pcie->pci;
1640 	struct dw_pcie_ep *ep = &pci->ep;
1641 	struct device *dev = pcie->dev;
1642 	u32 val;
1643 	int ret;
1644 
1645 	if (pcie->ep_state == EP_STATE_ENABLED)
1646 		return;
1647 
1648 	ret = pm_runtime_resume_and_get(dev);
1649 	if (ret < 0) {
1650 		dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1651 			ret);
1652 		return;
1653 	}
1654 
1655 	ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1656 	if (ret) {
1657 		dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n", ret);
1658 		goto fail_pll_init;
1659 	}
1660 
1661 	ret = clk_prepare_enable(pcie->core_clk);
1662 	if (ret) {
1663 		dev_err(dev, "Failed to enable core clock: %d\n", ret);
1664 		goto fail_core_clk_enable;
1665 	}
1666 
1667 	ret = reset_control_deassert(pcie->core_apb_rst);
1668 	if (ret) {
1669 		dev_err(dev, "Failed to deassert core APB reset: %d\n", ret);
1670 		goto fail_core_apb_rst;
1671 	}
1672 
1673 	ret = tegra_pcie_enable_phy(pcie);
1674 	if (ret) {
1675 		dev_err(dev, "Failed to enable PHY: %d\n", ret);
1676 		goto fail_phy;
1677 	}
1678 
1679 	/* Clear any stale interrupt statuses */
1680 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
1681 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
1682 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
1683 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
1684 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
1685 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
1686 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
1687 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
1688 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
1689 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
1690 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
1691 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
1692 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
1693 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
1694 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
1695 
1696 	/* configure this core for EP mode operation */
1697 	val = appl_readl(pcie, APPL_DM_TYPE);
1698 	val &= ~APPL_DM_TYPE_MASK;
1699 	val |= APPL_DM_TYPE_EP;
1700 	appl_writel(pcie, val, APPL_DM_TYPE);
1701 
1702 	appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1703 
1704 	val = appl_readl(pcie, APPL_CTRL);
1705 	val |= APPL_CTRL_SYS_PRE_DET_STATE;
1706 	val |= APPL_CTRL_HW_HOT_RST_EN;
1707 	appl_writel(pcie, val, APPL_CTRL);
1708 
1709 	val = appl_readl(pcie, APPL_CFG_MISC);
1710 	val |= APPL_CFG_MISC_SLV_EP_MODE;
1711 	val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1712 	appl_writel(pcie, val, APPL_CFG_MISC);
1713 
1714 	val = appl_readl(pcie, APPL_PINMUX);
1715 	val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1716 	val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1717 	appl_writel(pcie, val, APPL_PINMUX);
1718 
1719 	appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1720 		    APPL_CFG_BASE_ADDR);
1721 
1722 	appl_writel(pcie, pcie->atu_dma_res->start &
1723 		    APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1724 		    APPL_CFG_IATU_DMA_BASE_ADDR);
1725 
1726 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
1727 	val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
1728 	val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
1729 	val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN;
1730 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
1731 
1732 	val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
1733 	val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN;
1734 	val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN;
1735 	appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
1736 
1737 	reset_control_deassert(pcie->core_rst);
1738 
1739 	if (pcie->update_fc_fixup) {
1740 		val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
1741 		val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
1742 		dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
1743 	}
1744 
1745 	config_gen3_gen4_eq_presets(pcie);
1746 
1747 	init_host_aspm(pcie);
1748 
1749 	/* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
1750 	if (!pcie->supports_clkreq) {
1751 		disable_aspm_l11(pcie);
1752 		disable_aspm_l12(pcie);
1753 	}
1754 
1755 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1756 	val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1757 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
1758 
1759 	pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1760 						      PCI_CAP_ID_EXP);
1761 	clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
1762 
1763 	val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK);
1764 	val |= MSIX_ADDR_MATCH_LOW_OFF_EN;
1765 	dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val);
1766 	val = (lower_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK);
1767 	dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val);
1768 
1769 	ret = dw_pcie_ep_init_complete(ep);
1770 	if (ret) {
1771 		dev_err(dev, "Failed to complete initialization: %d\n", ret);
1772 		goto fail_init_complete;
1773 	}
1774 
1775 	dw_pcie_ep_init_notify(ep);
1776 
1777 	/* Enable LTSSM */
1778 	val = appl_readl(pcie, APPL_CTRL);
1779 	val |= APPL_CTRL_LTSSM_EN;
1780 	appl_writel(pcie, val, APPL_CTRL);
1781 
1782 	pcie->ep_state = EP_STATE_ENABLED;
1783 	dev_dbg(dev, "Initialization of endpoint is completed\n");
1784 
1785 	return;
1786 
1787 fail_init_complete:
1788 	reset_control_assert(pcie->core_rst);
1789 	tegra_pcie_disable_phy(pcie);
1790 fail_phy:
1791 	reset_control_assert(pcie->core_apb_rst);
1792 fail_core_apb_rst:
1793 	clk_disable_unprepare(pcie->core_clk);
1794 fail_core_clk_enable:
1795 	tegra_pcie_bpmp_set_pll_state(pcie, false);
1796 fail_pll_init:
1797 	pm_runtime_put_sync(dev);
1798 }
1799 
1800 static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)
1801 {
1802 	struct tegra_pcie_dw *pcie = arg;
1803 
1804 	if (gpiod_get_value(pcie->pex_rst_gpiod))
1805 		pex_ep_event_pex_rst_assert(pcie);
1806 	else
1807 		pex_ep_event_pex_rst_deassert(pcie);
1808 
1809 	return IRQ_HANDLED;
1810 }
1811 
1812 static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq)
1813 {
1814 	/* Tegra194 supports only INTA */
1815 	if (irq > 1)
1816 		return -EINVAL;
1817 
1818 	appl_writel(pcie, 1, APPL_LEGACY_INTX);
1819 	usleep_range(1000, 2000);
1820 	appl_writel(pcie, 0, APPL_LEGACY_INTX);
1821 	return 0;
1822 }
1823 
1824 static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq)
1825 {
1826 	if (unlikely(irq > 31))
1827 		return -EINVAL;
1828 
1829 	appl_writel(pcie, BIT(irq), APPL_MSI_CTRL_1);
1830 
1831 	return 0;
1832 }
1833 
1834 static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq)
1835 {
1836 	struct dw_pcie_ep *ep = &pcie->pci.ep;
1837 
1838 	writel(irq, ep->msi_mem);
1839 
1840 	return 0;
1841 }
1842 
1843 static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1844 				   enum pci_epc_irq_type type,
1845 				   u16 interrupt_num)
1846 {
1847 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1848 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1849 
1850 	switch (type) {
1851 	case PCI_EPC_IRQ_LEGACY:
1852 		return tegra_pcie_ep_raise_legacy_irq(pcie, interrupt_num);
1853 
1854 	case PCI_EPC_IRQ_MSI:
1855 		return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num);
1856 
1857 	case PCI_EPC_IRQ_MSIX:
1858 		return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num);
1859 
1860 	default:
1861 		dev_err(pci->dev, "Unknown IRQ type\n");
1862 		return -EPERM;
1863 	}
1864 
1865 	return 0;
1866 }
1867 
1868 static const struct pci_epc_features tegra_pcie_epc_features = {
1869 	.linkup_notifier = true,
1870 	.core_init_notifier = true,
1871 	.msi_capable = false,
1872 	.msix_capable = false,
1873 	.reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5,
1874 	.bar_fixed_64bit = 1 << BAR_0,
1875 	.bar_fixed_size[0] = SZ_1M,
1876 };
1877 
1878 static const struct pci_epc_features*
1879 tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)
1880 {
1881 	return &tegra_pcie_epc_features;
1882 }
1883 
1884 static const struct dw_pcie_ep_ops pcie_ep_ops = {
1885 	.raise_irq = tegra_pcie_ep_raise_irq,
1886 	.get_features = tegra_pcie_ep_get_features,
1887 };
1888 
1889 static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie,
1890 				struct platform_device *pdev)
1891 {
1892 	struct dw_pcie *pci = &pcie->pci;
1893 	struct device *dev = pcie->dev;
1894 	struct dw_pcie_ep *ep;
1895 	char *name;
1896 	int ret;
1897 
1898 	ep = &pci->ep;
1899 	ep->ops = &pcie_ep_ops;
1900 
1901 	ep->page_size = SZ_64K;
1902 
1903 	ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME);
1904 	if (ret < 0) {
1905 		dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n",
1906 			ret);
1907 		return ret;
1908 	}
1909 
1910 	ret = gpiod_to_irq(pcie->pex_rst_gpiod);
1911 	if (ret < 0) {
1912 		dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret);
1913 		return ret;
1914 	}
1915 	pcie->pex_rst_irq = (unsigned int)ret;
1916 
1917 	name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq",
1918 			      pcie->cid);
1919 	if (!name) {
1920 		dev_err(dev, "Failed to create PERST IRQ string\n");
1921 		return -ENOMEM;
1922 	}
1923 
1924 	irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN);
1925 
1926 	pcie->ep_state = EP_STATE_DISABLED;
1927 
1928 	ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL,
1929 					tegra_pcie_ep_pex_rst_irq,
1930 					IRQF_TRIGGER_RISING |
1931 					IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
1932 					name, (void *)pcie);
1933 	if (ret < 0) {
1934 		dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret);
1935 		return ret;
1936 	}
1937 
1938 	name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_ep_work",
1939 			      pcie->cid);
1940 	if (!name) {
1941 		dev_err(dev, "Failed to create PCIe EP work thread string\n");
1942 		return -ENOMEM;
1943 	}
1944 
1945 	pm_runtime_enable(dev);
1946 
1947 	ret = dw_pcie_ep_init(ep);
1948 	if (ret) {
1949 		dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n",
1950 			ret);
1951 		return ret;
1952 	}
1953 
1954 	return 0;
1955 }
1956 
1957 static int tegra_pcie_dw_probe(struct platform_device *pdev)
1958 {
1959 	const struct tegra_pcie_dw_of_data *data;
1960 	struct device *dev = &pdev->dev;
1961 	struct resource *atu_dma_res;
1962 	struct tegra_pcie_dw *pcie;
1963 	struct pcie_port *pp;
1964 	struct dw_pcie *pci;
1965 	struct phy **phys;
1966 	char *name;
1967 	int ret;
1968 	u32 i;
1969 
1970 	data = of_device_get_match_data(dev);
1971 
1972 	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
1973 	if (!pcie)
1974 		return -ENOMEM;
1975 
1976 	pci = &pcie->pci;
1977 	pci->dev = &pdev->dev;
1978 	pci->ops = &tegra_dw_pcie_ops;
1979 	pci->n_fts[0] = N_FTS_VAL;
1980 	pci->n_fts[1] = FTS_VAL;
1981 	pci->version = 0x490A;
1982 
1983 	pp = &pci->pp;
1984 	pp->num_vectors = MAX_MSI_IRQS;
1985 	pcie->dev = &pdev->dev;
1986 	pcie->mode = (enum dw_pcie_device_mode)data->mode;
1987 
1988 	ret = tegra_pcie_dw_parse_dt(pcie);
1989 	if (ret < 0) {
1990 		const char *level = KERN_ERR;
1991 
1992 		if (ret == -EPROBE_DEFER)
1993 			level = KERN_DEBUG;
1994 
1995 		dev_printk(level, dev,
1996 			   dev_fmt("Failed to parse device tree: %d\n"),
1997 			   ret);
1998 		return ret;
1999 	}
2000 
2001 	ret = tegra_pcie_get_slot_regulators(pcie);
2002 	if (ret < 0) {
2003 		const char *level = KERN_ERR;
2004 
2005 		if (ret == -EPROBE_DEFER)
2006 			level = KERN_DEBUG;
2007 
2008 		dev_printk(level, dev,
2009 			   dev_fmt("Failed to get slot regulators: %d\n"),
2010 			   ret);
2011 		return ret;
2012 	}
2013 
2014 	if (pcie->pex_refclk_sel_gpiod)
2015 		gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1);
2016 
2017 	pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl");
2018 	if (IS_ERR(pcie->pex_ctl_supply)) {
2019 		ret = PTR_ERR(pcie->pex_ctl_supply);
2020 		if (ret != -EPROBE_DEFER)
2021 			dev_err(dev, "Failed to get regulator: %ld\n",
2022 				PTR_ERR(pcie->pex_ctl_supply));
2023 		return ret;
2024 	}
2025 
2026 	pcie->core_clk = devm_clk_get(dev, "core");
2027 	if (IS_ERR(pcie->core_clk)) {
2028 		dev_err(dev, "Failed to get core clock: %ld\n",
2029 			PTR_ERR(pcie->core_clk));
2030 		return PTR_ERR(pcie->core_clk);
2031 	}
2032 
2033 	pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2034 						      "appl");
2035 	if (!pcie->appl_res) {
2036 		dev_err(dev, "Failed to find \"appl\" region\n");
2037 		return -ENODEV;
2038 	}
2039 
2040 	pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res);
2041 	if (IS_ERR(pcie->appl_base))
2042 		return PTR_ERR(pcie->appl_base);
2043 
2044 	pcie->core_apb_rst = devm_reset_control_get(dev, "apb");
2045 	if (IS_ERR(pcie->core_apb_rst)) {
2046 		dev_err(dev, "Failed to get APB reset: %ld\n",
2047 			PTR_ERR(pcie->core_apb_rst));
2048 		return PTR_ERR(pcie->core_apb_rst);
2049 	}
2050 
2051 	phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL);
2052 	if (!phys)
2053 		return -ENOMEM;
2054 
2055 	for (i = 0; i < pcie->phy_count; i++) {
2056 		name = kasprintf(GFP_KERNEL, "p2u-%u", i);
2057 		if (!name) {
2058 			dev_err(dev, "Failed to create P2U string\n");
2059 			return -ENOMEM;
2060 		}
2061 		phys[i] = devm_phy_get(dev, name);
2062 		kfree(name);
2063 		if (IS_ERR(phys[i])) {
2064 			ret = PTR_ERR(phys[i]);
2065 			if (ret != -EPROBE_DEFER)
2066 				dev_err(dev, "Failed to get PHY: %d\n", ret);
2067 			return ret;
2068 		}
2069 	}
2070 
2071 	pcie->phys = phys;
2072 
2073 	atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2074 						   "atu_dma");
2075 	if (!atu_dma_res) {
2076 		dev_err(dev, "Failed to find \"atu_dma\" region\n");
2077 		return -ENODEV;
2078 	}
2079 	pcie->atu_dma_res = atu_dma_res;
2080 
2081 	pci->atu_size = resource_size(atu_dma_res);
2082 	pci->atu_base = devm_ioremap_resource(dev, atu_dma_res);
2083 	if (IS_ERR(pci->atu_base))
2084 		return PTR_ERR(pci->atu_base);
2085 
2086 	pcie->core_rst = devm_reset_control_get(dev, "core");
2087 	if (IS_ERR(pcie->core_rst)) {
2088 		dev_err(dev, "Failed to get core reset: %ld\n",
2089 			PTR_ERR(pcie->core_rst));
2090 		return PTR_ERR(pcie->core_rst);
2091 	}
2092 
2093 	pp->irq = platform_get_irq_byname(pdev, "intr");
2094 	if (pp->irq < 0)
2095 		return pp->irq;
2096 
2097 	pcie->bpmp = tegra_bpmp_get(dev);
2098 	if (IS_ERR(pcie->bpmp))
2099 		return PTR_ERR(pcie->bpmp);
2100 
2101 	platform_set_drvdata(pdev, pcie);
2102 
2103 	switch (pcie->mode) {
2104 	case DW_PCIE_RC_TYPE:
2105 		ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler,
2106 				       IRQF_SHARED, "tegra-pcie-intr", pcie);
2107 		if (ret) {
2108 			dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2109 				ret);
2110 			goto fail;
2111 		}
2112 
2113 		ret = tegra_pcie_config_rp(pcie);
2114 		if (ret && ret != -ENOMEDIUM)
2115 			goto fail;
2116 		else
2117 			return 0;
2118 		break;
2119 
2120 	case DW_PCIE_EP_TYPE:
2121 		ret = devm_request_threaded_irq(dev, pp->irq,
2122 						tegra_pcie_ep_hard_irq,
2123 						tegra_pcie_ep_irq_thread,
2124 						IRQF_SHARED | IRQF_ONESHOT,
2125 						"tegra-pcie-ep-intr", pcie);
2126 		if (ret) {
2127 			dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2128 				ret);
2129 			goto fail;
2130 		}
2131 
2132 		ret = tegra_pcie_config_ep(pcie, pdev);
2133 		if (ret < 0)
2134 			goto fail;
2135 		break;
2136 
2137 	default:
2138 		dev_err(dev, "Invalid PCIe device type %d\n", pcie->mode);
2139 	}
2140 
2141 fail:
2142 	tegra_bpmp_put(pcie->bpmp);
2143 	return ret;
2144 }
2145 
2146 static int tegra_pcie_dw_remove(struct platform_device *pdev)
2147 {
2148 	struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2149 
2150 	if (!pcie->link_state)
2151 		return 0;
2152 
2153 	debugfs_remove_recursive(pcie->debugfs);
2154 	tegra_pcie_deinit_controller(pcie);
2155 	pm_runtime_put_sync(pcie->dev);
2156 	pm_runtime_disable(pcie->dev);
2157 	tegra_bpmp_put(pcie->bpmp);
2158 	if (pcie->pex_refclk_sel_gpiod)
2159 		gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0);
2160 
2161 	return 0;
2162 }
2163 
2164 static int tegra_pcie_dw_suspend_late(struct device *dev)
2165 {
2166 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2167 	u32 val;
2168 
2169 	if (!pcie->link_state)
2170 		return 0;
2171 
2172 	/* Enable HW_HOT_RST mode */
2173 	val = appl_readl(pcie, APPL_CTRL);
2174 	val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2175 		 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2176 	val |= APPL_CTRL_HW_HOT_RST_EN;
2177 	appl_writel(pcie, val, APPL_CTRL);
2178 
2179 	return 0;
2180 }
2181 
2182 static int tegra_pcie_dw_suspend_noirq(struct device *dev)
2183 {
2184 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2185 
2186 	if (!pcie->link_state)
2187 		return 0;
2188 
2189 	/* Save MSI interrupt vector */
2190 	pcie->msi_ctrl_int = dw_pcie_readl_dbi(&pcie->pci,
2191 					       PORT_LOGIC_MSI_CTRL_INT_0_EN);
2192 	tegra_pcie_downstream_dev_to_D0(pcie);
2193 	tegra_pcie_dw_pme_turnoff(pcie);
2194 	tegra_pcie_unconfig_controller(pcie);
2195 
2196 	return 0;
2197 }
2198 
2199 static int tegra_pcie_dw_resume_noirq(struct device *dev)
2200 {
2201 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2202 	int ret;
2203 
2204 	if (!pcie->link_state)
2205 		return 0;
2206 
2207 	ret = tegra_pcie_config_controller(pcie, true);
2208 	if (ret < 0)
2209 		return ret;
2210 
2211 	ret = tegra_pcie_dw_host_init(&pcie->pci.pp);
2212 	if (ret < 0) {
2213 		dev_err(dev, "Failed to init host: %d\n", ret);
2214 		goto fail_host_init;
2215 	}
2216 
2217 	dw_pcie_setup_rc(&pcie->pci.pp);
2218 
2219 	ret = tegra_pcie_dw_start_link(&pcie->pci);
2220 	if (ret < 0)
2221 		goto fail_host_init;
2222 
2223 	/* Restore MSI interrupt vector */
2224 	dw_pcie_writel_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN,
2225 			   pcie->msi_ctrl_int);
2226 
2227 	return 0;
2228 
2229 fail_host_init:
2230 	tegra_pcie_unconfig_controller(pcie);
2231 	return ret;
2232 }
2233 
2234 static int tegra_pcie_dw_resume_early(struct device *dev)
2235 {
2236 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2237 	u32 val;
2238 
2239 	if (!pcie->link_state)
2240 		return 0;
2241 
2242 	/* Disable HW_HOT_RST mode */
2243 	val = appl_readl(pcie, APPL_CTRL);
2244 	val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2245 		 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2246 	val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<
2247 	       APPL_CTRL_HW_HOT_RST_MODE_SHIFT;
2248 	val &= ~APPL_CTRL_HW_HOT_RST_EN;
2249 	appl_writel(pcie, val, APPL_CTRL);
2250 
2251 	return 0;
2252 }
2253 
2254 static void tegra_pcie_dw_shutdown(struct platform_device *pdev)
2255 {
2256 	struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2257 
2258 	if (!pcie->link_state)
2259 		return;
2260 
2261 	debugfs_remove_recursive(pcie->debugfs);
2262 	tegra_pcie_downstream_dev_to_D0(pcie);
2263 
2264 	disable_irq(pcie->pci.pp.irq);
2265 	if (IS_ENABLED(CONFIG_PCI_MSI))
2266 		disable_irq(pcie->pci.pp.msi_irq);
2267 
2268 	tegra_pcie_dw_pme_turnoff(pcie);
2269 	tegra_pcie_unconfig_controller(pcie);
2270 }
2271 
2272 static const struct tegra_pcie_dw_of_data tegra_pcie_dw_rc_of_data = {
2273 	.mode = DW_PCIE_RC_TYPE,
2274 };
2275 
2276 static const struct tegra_pcie_dw_of_data tegra_pcie_dw_ep_of_data = {
2277 	.mode = DW_PCIE_EP_TYPE,
2278 };
2279 
2280 static const struct of_device_id tegra_pcie_dw_of_match[] = {
2281 	{
2282 		.compatible = "nvidia,tegra194-pcie",
2283 		.data = &tegra_pcie_dw_rc_of_data,
2284 	},
2285 	{
2286 		.compatible = "nvidia,tegra194-pcie-ep",
2287 		.data = &tegra_pcie_dw_ep_of_data,
2288 	},
2289 	{},
2290 };
2291 
2292 static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {
2293 	.suspend_late = tegra_pcie_dw_suspend_late,
2294 	.suspend_noirq = tegra_pcie_dw_suspend_noirq,
2295 	.resume_noirq = tegra_pcie_dw_resume_noirq,
2296 	.resume_early = tegra_pcie_dw_resume_early,
2297 };
2298 
2299 static struct platform_driver tegra_pcie_dw_driver = {
2300 	.probe = tegra_pcie_dw_probe,
2301 	.remove = tegra_pcie_dw_remove,
2302 	.shutdown = tegra_pcie_dw_shutdown,
2303 	.driver = {
2304 		.name	= "tegra194-pcie",
2305 		.pm = &tegra_pcie_dw_pm_ops,
2306 		.of_match_table = tegra_pcie_dw_of_match,
2307 	},
2308 };
2309 module_platform_driver(tegra_pcie_dw_driver);
2310 
2311 MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);
2312 
2313 MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");
2314 MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");
2315 MODULE_LICENSE("GPL v2");
2316