1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * PCIe host controller driver for Tegra194 SoC
4  *
5  * Copyright (C) 2019 NVIDIA Corporation.
6  *
7  * Author: Vidya Sagar <vidyas@nvidia.com>
8  */
9 
10 #include <linux/clk.h>
11 #include <linux/debugfs.h>
12 #include <linux/delay.h>
13 #include <linux/gpio.h>
14 #include <linux/gpio/consumer.h>
15 #include <linux/interrupt.h>
16 #include <linux/iopoll.h>
17 #include <linux/kernel.h>
18 #include <linux/module.h>
19 #include <linux/of.h>
20 #include <linux/of_device.h>
21 #include <linux/of_gpio.h>
22 #include <linux/of_irq.h>
23 #include <linux/of_pci.h>
24 #include <linux/pci.h>
25 #include <linux/phy/phy.h>
26 #include <linux/pinctrl/consumer.h>
27 #include <linux/platform_device.h>
28 #include <linux/pm_runtime.h>
29 #include <linux/random.h>
30 #include <linux/reset.h>
31 #include <linux/resource.h>
32 #include <linux/types.h>
33 #include "pcie-designware.h"
34 #include <soc/tegra/bpmp.h>
35 #include <soc/tegra/bpmp-abi.h>
36 #include "../../pci.h"
37 
38 #define APPL_PINMUX				0x0
39 #define APPL_PINMUX_PEX_RST			BIT(0)
40 #define APPL_PINMUX_CLKREQ_OVERRIDE_EN		BIT(2)
41 #define APPL_PINMUX_CLKREQ_OVERRIDE		BIT(3)
42 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN	BIT(4)
43 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE	BIT(5)
44 
45 #define APPL_CTRL				0x4
46 #define APPL_CTRL_SYS_PRE_DET_STATE		BIT(6)
47 #define APPL_CTRL_LTSSM_EN			BIT(7)
48 #define APPL_CTRL_HW_HOT_RST_EN			BIT(20)
49 #define APPL_CTRL_HW_HOT_RST_MODE_MASK		GENMASK(1, 0)
50 #define APPL_CTRL_HW_HOT_RST_MODE_SHIFT		22
51 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST	0x1
52 
53 #define APPL_INTR_EN_L0_0			0x8
54 #define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN	BIT(0)
55 #define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN	BIT(4)
56 #define APPL_INTR_EN_L0_0_INT_INT_EN		BIT(8)
57 #define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN	BIT(15)
58 #define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN	BIT(19)
59 #define APPL_INTR_EN_L0_0_SYS_INTR_EN		BIT(30)
60 #define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN	BIT(31)
61 
62 #define APPL_INTR_STATUS_L0			0xC
63 #define APPL_INTR_STATUS_L0_LINK_STATE_INT	BIT(0)
64 #define APPL_INTR_STATUS_L0_INT_INT		BIT(8)
65 #define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT	BIT(15)
66 #define APPL_INTR_STATUS_L0_PEX_RST_INT		BIT(16)
67 #define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT	BIT(18)
68 
69 #define APPL_INTR_EN_L1_0_0				0x1C
70 #define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN	BIT(1)
71 #define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN		BIT(3)
72 #define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN	BIT(30)
73 
74 #define APPL_INTR_STATUS_L1_0_0				0x20
75 #define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED	BIT(1)
76 #define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED	BIT(3)
77 #define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE		BIT(30)
78 
79 #define APPL_INTR_STATUS_L1_1			0x2C
80 #define APPL_INTR_STATUS_L1_2			0x30
81 #define APPL_INTR_STATUS_L1_3			0x34
82 #define APPL_INTR_STATUS_L1_6			0x3C
83 #define APPL_INTR_STATUS_L1_7			0x40
84 #define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED	BIT(1)
85 
86 #define APPL_INTR_EN_L1_8_0			0x44
87 #define APPL_INTR_EN_L1_8_BW_MGT_INT_EN		BIT(2)
88 #define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN	BIT(3)
89 #define APPL_INTR_EN_L1_8_INTX_EN		BIT(11)
90 #define APPL_INTR_EN_L1_8_AER_INT_EN		BIT(15)
91 
92 #define APPL_INTR_STATUS_L1_8_0			0x4C
93 #define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK	GENMASK(11, 6)
94 #define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS	BIT(2)
95 #define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS	BIT(3)
96 
97 #define APPL_INTR_STATUS_L1_9			0x54
98 #define APPL_INTR_STATUS_L1_10			0x58
99 #define APPL_INTR_STATUS_L1_11			0x64
100 #define APPL_INTR_STATUS_L1_13			0x74
101 #define APPL_INTR_STATUS_L1_14			0x78
102 #define APPL_INTR_STATUS_L1_15			0x7C
103 #define APPL_INTR_STATUS_L1_17			0x88
104 
105 #define APPL_INTR_EN_L1_18				0x90
106 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT		BIT(2)
107 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR		BIT(1)
108 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR	BIT(0)
109 
110 #define APPL_INTR_STATUS_L1_18				0x94
111 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT	BIT(2)
112 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR	BIT(1)
113 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR	BIT(0)
114 
115 #define APPL_MSI_CTRL_1				0xAC
116 
117 #define APPL_MSI_CTRL_2				0xB0
118 
119 #define APPL_LEGACY_INTX			0xB8
120 
121 #define APPL_LTR_MSG_1				0xC4
122 #define LTR_MSG_REQ				BIT(15)
123 #define LTR_MST_NO_SNOOP_SHIFT			16
124 
125 #define APPL_LTR_MSG_2				0xC8
126 #define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE	BIT(3)
127 
128 #define APPL_LINK_STATUS			0xCC
129 #define APPL_LINK_STATUS_RDLH_LINK_UP		BIT(0)
130 
131 #define APPL_DEBUG				0xD0
132 #define APPL_DEBUG_PM_LINKST_IN_L2_LAT		BIT(21)
133 #define APPL_DEBUG_PM_LINKST_IN_L0		0x11
134 #define APPL_DEBUG_LTSSM_STATE_MASK		GENMASK(8, 3)
135 #define APPL_DEBUG_LTSSM_STATE_SHIFT		3
136 #define LTSSM_STATE_PRE_DETECT			5
137 
138 #define APPL_RADM_STATUS			0xE4
139 #define APPL_PM_XMT_TURNOFF_STATE		BIT(0)
140 
141 #define APPL_DM_TYPE				0x100
142 #define APPL_DM_TYPE_MASK			GENMASK(3, 0)
143 #define APPL_DM_TYPE_RP				0x4
144 #define APPL_DM_TYPE_EP				0x0
145 
146 #define APPL_CFG_BASE_ADDR			0x104
147 #define APPL_CFG_BASE_ADDR_MASK			GENMASK(31, 12)
148 
149 #define APPL_CFG_IATU_DMA_BASE_ADDR		0x108
150 #define APPL_CFG_IATU_DMA_BASE_ADDR_MASK	GENMASK(31, 18)
151 
152 #define APPL_CFG_MISC				0x110
153 #define APPL_CFG_MISC_SLV_EP_MODE		BIT(14)
154 #define APPL_CFG_MISC_ARCACHE_MASK		GENMASK(13, 10)
155 #define APPL_CFG_MISC_ARCACHE_SHIFT		10
156 #define APPL_CFG_MISC_ARCACHE_VAL		3
157 
158 #define APPL_CFG_SLCG_OVERRIDE			0x114
159 #define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER	BIT(0)
160 
161 #define APPL_CAR_RESET_OVRD				0x12C
162 #define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N	BIT(0)
163 
164 #define IO_BASE_IO_DECODE				BIT(0)
165 #define IO_BASE_IO_DECODE_BIT8				BIT(8)
166 
167 #define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE		BIT(0)
168 #define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE	BIT(16)
169 
170 #define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF	0x718
171 #define CFG_TIMER_CTRL_ACK_NAK_SHIFT	(19)
172 
173 #define EVENT_COUNTER_ALL_CLEAR		0x3
174 #define EVENT_COUNTER_ENABLE_ALL	0x7
175 #define EVENT_COUNTER_ENABLE_SHIFT	2
176 #define EVENT_COUNTER_EVENT_SEL_MASK	GENMASK(7, 0)
177 #define EVENT_COUNTER_EVENT_SEL_SHIFT	16
178 #define EVENT_COUNTER_EVENT_Tx_L0S	0x2
179 #define EVENT_COUNTER_EVENT_Rx_L0S	0x3
180 #define EVENT_COUNTER_EVENT_L1		0x5
181 #define EVENT_COUNTER_EVENT_L1_1	0x7
182 #define EVENT_COUNTER_EVENT_L1_2	0x8
183 #define EVENT_COUNTER_GROUP_SEL_SHIFT	24
184 #define EVENT_COUNTER_GROUP_5		0x5
185 
186 #define PORT_LOGIC_ACK_F_ASPM_CTRL			0x70C
187 #define ENTER_ASPM					BIT(30)
188 #define L0S_ENTRANCE_LAT_SHIFT				24
189 #define L0S_ENTRANCE_LAT_MASK				GENMASK(26, 24)
190 #define L1_ENTRANCE_LAT_SHIFT				27
191 #define L1_ENTRANCE_LAT_MASK				GENMASK(29, 27)
192 #define N_FTS_SHIFT					8
193 #define N_FTS_MASK					GENMASK(7, 0)
194 #define N_FTS_VAL					52
195 
196 #define PORT_LOGIC_GEN2_CTRL				0x80C
197 #define PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE	BIT(17)
198 #define FTS_MASK					GENMASK(7, 0)
199 #define FTS_VAL						52
200 
201 #define PORT_LOGIC_MSI_CTRL_INT_0_EN		0x828
202 
203 #define GEN3_EQ_CONTROL_OFF			0x8a8
204 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT	8
205 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK	GENMASK(23, 8)
206 #define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK	GENMASK(3, 0)
207 
208 #define GEN3_RELATED_OFF			0x890
209 #define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL	BIT(0)
210 #define GEN3_RELATED_OFF_GEN3_EQ_DISABLE	BIT(16)
211 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT	24
212 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK	GENMASK(25, 24)
213 
214 #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT	0x8D0
215 #define AMBA_ERROR_RESPONSE_CRS_SHIFT		3
216 #define AMBA_ERROR_RESPONSE_CRS_MASK		GENMASK(1, 0)
217 #define AMBA_ERROR_RESPONSE_CRS_OKAY		0
218 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF	1
219 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001	2
220 
221 #define MSIX_ADDR_MATCH_LOW_OFF			0x940
222 #define MSIX_ADDR_MATCH_LOW_OFF_EN		BIT(0)
223 #define MSIX_ADDR_MATCH_LOW_OFF_MASK		GENMASK(31, 2)
224 
225 #define MSIX_ADDR_MATCH_HIGH_OFF		0x944
226 #define MSIX_ADDR_MATCH_HIGH_OFF_MASK		GENMASK(31, 0)
227 
228 #define PORT_LOGIC_MSIX_DOORBELL			0x948
229 
230 #define CAP_SPCIE_CAP_OFF			0x154
231 #define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK	GENMASK(3, 0)
232 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK	GENMASK(11, 8)
233 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT	8
234 
235 #define PME_ACK_TIMEOUT 10000
236 
237 #define LTSSM_TIMEOUT 50000	/* 50ms */
238 
239 #define GEN3_GEN4_EQ_PRESET_INIT	5
240 
241 #define GEN1_CORE_CLK_FREQ	62500000
242 #define GEN2_CORE_CLK_FREQ	125000000
243 #define GEN3_CORE_CLK_FREQ	250000000
244 #define GEN4_CORE_CLK_FREQ	500000000
245 
246 #define LTR_MSG_TIMEOUT		(100 * 1000)
247 
248 #define PERST_DEBOUNCE_TIME	(5 * 1000)
249 
250 #define EP_STATE_DISABLED	0
251 #define EP_STATE_ENABLED	1
252 
253 static const unsigned int pcie_gen_freq[] = {
254 	GEN1_CORE_CLK_FREQ,
255 	GEN2_CORE_CLK_FREQ,
256 	GEN3_CORE_CLK_FREQ,
257 	GEN4_CORE_CLK_FREQ
258 };
259 
260 static const u32 event_cntr_ctrl_offset[] = {
261 	0x1d8,
262 	0x1a8,
263 	0x1a8,
264 	0x1a8,
265 	0x1c4,
266 	0x1d8
267 };
268 
269 static const u32 event_cntr_data_offset[] = {
270 	0x1dc,
271 	0x1ac,
272 	0x1ac,
273 	0x1ac,
274 	0x1c8,
275 	0x1dc
276 };
277 
278 struct tegra_pcie_dw {
279 	struct device *dev;
280 	struct resource *appl_res;
281 	struct resource *dbi_res;
282 	struct resource *atu_dma_res;
283 	void __iomem *appl_base;
284 	struct clk *core_clk;
285 	struct reset_control *core_apb_rst;
286 	struct reset_control *core_rst;
287 	struct dw_pcie pci;
288 	struct tegra_bpmp *bpmp;
289 
290 	enum dw_pcie_device_mode mode;
291 
292 	bool supports_clkreq;
293 	bool enable_cdm_check;
294 	bool link_state;
295 	bool update_fc_fixup;
296 	u8 init_link_width;
297 	u32 msi_ctrl_int;
298 	u32 num_lanes;
299 	u32 max_speed;
300 	u32 cid;
301 	u32 cfg_link_cap_l1sub;
302 	u32 pcie_cap_base;
303 	u32 aspm_cmrt;
304 	u32 aspm_pwr_on_t;
305 	u32 aspm_l0s_enter_lat;
306 
307 	struct regulator *pex_ctl_supply;
308 	struct regulator *slot_ctl_3v3;
309 	struct regulator *slot_ctl_12v;
310 
311 	unsigned int phy_count;
312 	struct phy **phys;
313 
314 	struct dentry *debugfs;
315 
316 	/* Endpoint mode specific */
317 	struct gpio_desc *pex_rst_gpiod;
318 	struct gpio_desc *pex_refclk_sel_gpiod;
319 	unsigned int pex_rst_irq;
320 	int ep_state;
321 };
322 
323 struct tegra_pcie_dw_of_data {
324 	enum dw_pcie_device_mode mode;
325 };
326 
327 static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)
328 {
329 	return container_of(pci, struct tegra_pcie_dw, pci);
330 }
331 
332 static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,
333 			       const u32 reg)
334 {
335 	writel_relaxed(value, pcie->appl_base + reg);
336 }
337 
338 static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)
339 {
340 	return readl_relaxed(pcie->appl_base + reg);
341 }
342 
343 struct tegra_pcie_soc {
344 	enum dw_pcie_device_mode mode;
345 };
346 
347 static void apply_bad_link_workaround(struct pcie_port *pp)
348 {
349 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
350 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
351 	u32 current_link_width;
352 	u16 val;
353 
354 	/*
355 	 * NOTE:- Since this scenario is uncommon and link as such is not
356 	 * stable anyway, not waiting to confirm if link is really
357 	 * transitioning to Gen-2 speed
358 	 */
359 	val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
360 	if (val & PCI_EXP_LNKSTA_LBMS) {
361 		current_link_width = (val & PCI_EXP_LNKSTA_NLW) >>
362 				     PCI_EXP_LNKSTA_NLW_SHIFT;
363 		if (pcie->init_link_width > current_link_width) {
364 			dev_warn(pci->dev, "PCIe link is bad, width reduced\n");
365 			val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
366 						PCI_EXP_LNKCTL2);
367 			val &= ~PCI_EXP_LNKCTL2_TLS;
368 			val |= PCI_EXP_LNKCTL2_TLS_2_5GT;
369 			dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
370 					   PCI_EXP_LNKCTL2, val);
371 
372 			val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
373 						PCI_EXP_LNKCTL);
374 			val |= PCI_EXP_LNKCTL_RL;
375 			dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
376 					   PCI_EXP_LNKCTL, val);
377 		}
378 	}
379 }
380 
381 static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg)
382 {
383 	struct tegra_pcie_dw *pcie = arg;
384 	struct dw_pcie *pci = &pcie->pci;
385 	struct pcie_port *pp = &pci->pp;
386 	u32 val, tmp;
387 	u16 val_w;
388 
389 	val = appl_readl(pcie, APPL_INTR_STATUS_L0);
390 	if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
391 		val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
392 		if (val & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {
393 			appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0);
394 
395 			/* SBR & Surprise Link Down WAR */
396 			val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
397 			val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
398 			appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
399 			udelay(1);
400 			val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
401 			val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
402 			appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
403 
404 			val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL);
405 			val |= PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE;
406 			dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val);
407 		}
408 	}
409 
410 	if (val & APPL_INTR_STATUS_L0_INT_INT) {
411 		val = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);
412 		if (val & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {
413 			appl_writel(pcie,
414 				    APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,
415 				    APPL_INTR_STATUS_L1_8_0);
416 			apply_bad_link_workaround(pp);
417 		}
418 		if (val & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {
419 			appl_writel(pcie,
420 				    APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,
421 				    APPL_INTR_STATUS_L1_8_0);
422 
423 			val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
424 						  PCI_EXP_LNKSTA);
425 			dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &
426 				PCI_EXP_LNKSTA_CLS);
427 		}
428 	}
429 
430 	val = appl_readl(pcie, APPL_INTR_STATUS_L0);
431 	if (val & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {
432 		val = appl_readl(pcie, APPL_INTR_STATUS_L1_18);
433 		tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
434 		if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {
435 			dev_info(pci->dev, "CDM check complete\n");
436 			tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;
437 		}
438 		if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {
439 			dev_err(pci->dev, "CDM comparison mismatch\n");
440 			tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;
441 		}
442 		if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {
443 			dev_err(pci->dev, "CDM Logic error\n");
444 			tmp |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;
445 		}
446 		dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, tmp);
447 		tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);
448 		dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", tmp);
449 	}
450 
451 	return IRQ_HANDLED;
452 }
453 
454 static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie)
455 {
456 	u32 val;
457 
458 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
459 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
460 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
461 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
462 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
463 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
464 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
465 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
466 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
467 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
468 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
469 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
470 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
471 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
472 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
473 	appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2);
474 
475 	val = appl_readl(pcie, APPL_CTRL);
476 	val |= APPL_CTRL_LTSSM_EN;
477 	appl_writel(pcie, val, APPL_CTRL);
478 }
479 
480 static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg)
481 {
482 	struct tegra_pcie_dw *pcie = arg;
483 	struct dw_pcie *pci = &pcie->pci;
484 	u32 val, speed;
485 
486 	speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
487 		PCI_EXP_LNKSTA_CLS;
488 	clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
489 
490 	/* If EP doesn't advertise L1SS, just return */
491 	val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
492 	if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2)))
493 		return IRQ_HANDLED;
494 
495 	/* Check if BME is set to '1' */
496 	val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
497 	if (val & PCI_COMMAND_MASTER) {
498 		ktime_t timeout;
499 
500 		/* 110us for both snoop and no-snoop */
501 		val = 110 | (2 << PCI_LTR_SCALE_SHIFT) | LTR_MSG_REQ;
502 		val |= (val << LTR_MST_NO_SNOOP_SHIFT);
503 		appl_writel(pcie, val, APPL_LTR_MSG_1);
504 
505 		/* Send LTR upstream */
506 		val = appl_readl(pcie, APPL_LTR_MSG_2);
507 		val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
508 		appl_writel(pcie, val, APPL_LTR_MSG_2);
509 
510 		timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT);
511 		for (;;) {
512 			val = appl_readl(pcie, APPL_LTR_MSG_2);
513 			if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE))
514 				break;
515 			if (ktime_after(ktime_get(), timeout))
516 				break;
517 			usleep_range(1000, 1100);
518 		}
519 		if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)
520 			dev_err(pcie->dev, "Failed to send LTR message\n");
521 	}
522 
523 	return IRQ_HANDLED;
524 }
525 
526 static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg)
527 {
528 	struct tegra_pcie_dw *pcie = arg;
529 	struct dw_pcie_ep *ep = &pcie->pci.ep;
530 	int spurious = 1;
531 	u32 val, tmp;
532 
533 	val = appl_readl(pcie, APPL_INTR_STATUS_L0);
534 	if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
535 		val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
536 		appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0);
537 
538 		if (val & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE)
539 			pex_ep_event_hot_rst_done(pcie);
540 
541 		if (val & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) {
542 			tmp = appl_readl(pcie, APPL_LINK_STATUS);
543 			if (tmp & APPL_LINK_STATUS_RDLH_LINK_UP) {
544 				dev_dbg(pcie->dev, "Link is up with Host\n");
545 				dw_pcie_ep_linkup(ep);
546 			}
547 		}
548 
549 		spurious = 0;
550 	}
551 
552 	if (val & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) {
553 		val = appl_readl(pcie, APPL_INTR_STATUS_L1_15);
554 		appl_writel(pcie, val, APPL_INTR_STATUS_L1_15);
555 
556 		if (val & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED)
557 			return IRQ_WAKE_THREAD;
558 
559 		spurious = 0;
560 	}
561 
562 	if (spurious) {
563 		dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n",
564 			 val);
565 		appl_writel(pcie, val, APPL_INTR_STATUS_L0);
566 	}
567 
568 	return IRQ_HANDLED;
569 }
570 
571 static int tegra_pcie_dw_rd_own_conf(struct pcie_port *pp, int where, int size,
572 				     u32 *val)
573 {
574 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
575 
576 	/*
577 	 * This is an endpoint mode specific register happen to appear even
578 	 * when controller is operating in root port mode and system hangs
579 	 * when it is accessed with link being in ASPM-L1 state.
580 	 * So skip accessing it altogether
581 	 */
582 	if (where == PORT_LOGIC_MSIX_DOORBELL) {
583 		*val = 0x00000000;
584 		return PCIBIOS_SUCCESSFUL;
585 	}
586 
587 	return dw_pcie_read(pci->dbi_base + where, size, val);
588 }
589 
590 static int tegra_pcie_dw_wr_own_conf(struct pcie_port *pp, int where, int size,
591 				     u32 val)
592 {
593 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
594 
595 	/*
596 	 * This is an endpoint mode specific register happen to appear even
597 	 * when controller is operating in root port mode and system hangs
598 	 * when it is accessed with link being in ASPM-L1 state.
599 	 * So skip accessing it altogether
600 	 */
601 	if (where == PORT_LOGIC_MSIX_DOORBELL)
602 		return PCIBIOS_SUCCESSFUL;
603 
604 	return dw_pcie_write(pci->dbi_base + where, size, val);
605 }
606 
607 #if defined(CONFIG_PCIEASPM)
608 static void disable_aspm_l11(struct tegra_pcie_dw *pcie)
609 {
610 	u32 val;
611 
612 	val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
613 	val &= ~PCI_L1SS_CAP_ASPM_L1_1;
614 	dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
615 }
616 
617 static void disable_aspm_l12(struct tegra_pcie_dw *pcie)
618 {
619 	u32 val;
620 
621 	val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
622 	val &= ~PCI_L1SS_CAP_ASPM_L1_2;
623 	dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
624 }
625 
626 static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)
627 {
628 	u32 val;
629 
630 	val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid]);
631 	val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);
632 	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
633 	val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;
634 	val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
635 	dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
636 	val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_data_offset[pcie->cid]);
637 
638 	return val;
639 }
640 
641 static int aspm_state_cnt(struct seq_file *s, void *data)
642 {
643 	struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)
644 				     dev_get_drvdata(s->private);
645 	u32 val;
646 
647 	seq_printf(s, "Tx L0s entry count : %u\n",
648 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));
649 
650 	seq_printf(s, "Rx L0s entry count : %u\n",
651 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));
652 
653 	seq_printf(s, "Link L1 entry count : %u\n",
654 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));
655 
656 	seq_printf(s, "Link L1.1 entry count : %u\n",
657 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));
658 
659 	seq_printf(s, "Link L1.2 entry count : %u\n",
660 		   event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));
661 
662 	/* Clear all counters */
663 	dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid],
664 			   EVENT_COUNTER_ALL_CLEAR);
665 
666 	/* Re-enable counting */
667 	val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
668 	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
669 	dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
670 
671 	return 0;
672 }
673 
674 static void init_host_aspm(struct tegra_pcie_dw *pcie)
675 {
676 	struct dw_pcie *pci = &pcie->pci;
677 	u32 val;
678 
679 	val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
680 	pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;
681 
682 	/* Enable ASPM counters */
683 	val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
684 	val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
685 	dw_pcie_writel_dbi(pci, event_cntr_ctrl_offset[pcie->cid], val);
686 
687 	/* Program T_cmrt and T_pwr_on values */
688 	val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
689 	val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);
690 	val |= (pcie->aspm_cmrt << 8);
691 	val |= (pcie->aspm_pwr_on_t << 19);
692 	dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val);
693 
694 	/* Program L0s and L1 entrance latencies */
695 	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL);
696 	val &= ~L0S_ENTRANCE_LAT_MASK;
697 	val |= (pcie->aspm_l0s_enter_lat << L0S_ENTRANCE_LAT_SHIFT);
698 	val |= ENTER_ASPM;
699 	dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val);
700 }
701 
702 static int init_debugfs(struct tegra_pcie_dw *pcie)
703 {
704 	struct dentry *d;
705 
706 	d = debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt",
707 					pcie->debugfs, aspm_state_cnt);
708 	if (IS_ERR_OR_NULL(d))
709 		dev_err(pcie->dev,
710 			"Failed to create debugfs file \"aspm_state_cnt\"\n");
711 
712 	return 0;
713 }
714 #else
715 static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }
716 static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }
717 static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }
718 static inline int init_debugfs(struct tegra_pcie_dw *pcie) { return 0; }
719 #endif
720 
721 static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp)
722 {
723 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
724 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
725 	u32 val;
726 	u16 val_w;
727 
728 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
729 	val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
730 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
731 
732 	val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
733 	val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;
734 	appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
735 
736 	if (pcie->enable_cdm_check) {
737 		val = appl_readl(pcie, APPL_INTR_EN_L0_0);
738 		val |= APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN;
739 		appl_writel(pcie, val, APPL_INTR_EN_L0_0);
740 
741 		val = appl_readl(pcie, APPL_INTR_EN_L1_18);
742 		val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;
743 		val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;
744 		appl_writel(pcie, val, APPL_INTR_EN_L1_18);
745 	}
746 
747 	val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
748 				  PCI_EXP_LNKSTA);
749 	pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >>
750 				PCI_EXP_LNKSTA_NLW_SHIFT;
751 
752 	val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
753 				  PCI_EXP_LNKCTL);
754 	val_w |= PCI_EXP_LNKCTL_LBMIE;
755 	dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL,
756 			   val_w);
757 }
758 
759 static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp)
760 {
761 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
762 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
763 	u32 val;
764 
765 	/* Enable legacy interrupt generation */
766 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
767 	val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
768 	val |= APPL_INTR_EN_L0_0_INT_INT_EN;
769 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
770 
771 	val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);
772 	val |= APPL_INTR_EN_L1_8_INTX_EN;
773 	val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;
774 	val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;
775 	if (IS_ENABLED(CONFIG_PCIEAER))
776 		val |= APPL_INTR_EN_L1_8_AER_INT_EN;
777 	appl_writel(pcie, val, APPL_INTR_EN_L1_8_0);
778 }
779 
780 static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp)
781 {
782 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
783 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
784 	u32 val;
785 
786 	dw_pcie_msi_init(pp);
787 
788 	/* Enable MSI interrupt generation */
789 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
790 	val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;
791 	val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;
792 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
793 }
794 
795 static void tegra_pcie_enable_interrupts(struct pcie_port *pp)
796 {
797 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
798 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
799 
800 	/* Clear interrupt statuses before enabling interrupts */
801 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
802 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
803 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
804 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
805 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
806 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
807 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
808 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
809 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
810 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
811 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
812 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
813 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
814 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
815 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
816 
817 	tegra_pcie_enable_system_interrupts(pp);
818 	tegra_pcie_enable_legacy_interrupts(pp);
819 	if (IS_ENABLED(CONFIG_PCI_MSI))
820 		tegra_pcie_enable_msi_interrupts(pp);
821 }
822 
823 static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
824 {
825 	struct dw_pcie *pci = &pcie->pci;
826 	u32 val, offset, i;
827 
828 	/* Program init preset */
829 	for (i = 0; i < pcie->num_lanes; i++) {
830 		dw_pcie_read(pci->dbi_base + CAP_SPCIE_CAP_OFF
831 				 + (i * 2), 2, &val);
832 		val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;
833 		val |= GEN3_GEN4_EQ_PRESET_INIT;
834 		val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;
835 		val |= (GEN3_GEN4_EQ_PRESET_INIT <<
836 			   CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);
837 		dw_pcie_write(pci->dbi_base + CAP_SPCIE_CAP_OFF
838 				 + (i * 2), 2, val);
839 
840 		offset = dw_pcie_find_ext_capability(pci,
841 						     PCI_EXT_CAP_ID_PL_16GT) +
842 				PCI_PL_16GT_LE_CTRL;
843 		dw_pcie_read(pci->dbi_base + offset + i, 1, &val);
844 		val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;
845 		val |= GEN3_GEN4_EQ_PRESET_INIT;
846 		val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;
847 		val |= (GEN3_GEN4_EQ_PRESET_INIT <<
848 			PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);
849 		dw_pcie_write(pci->dbi_base + offset + i, 1, val);
850 	}
851 
852 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
853 	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
854 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
855 
856 	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
857 	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
858 	val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
859 	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
860 	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
861 
862 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
863 	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
864 	val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);
865 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
866 
867 	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
868 	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
869 	val |= (0x360 << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
870 	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
871 	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
872 
873 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
874 	val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
875 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
876 }
877 
878 static void tegra_pcie_prepare_host(struct pcie_port *pp)
879 {
880 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
881 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
882 	u32 val;
883 
884 	val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);
885 	val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);
886 	dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);
887 
888 	val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);
889 	val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;
890 	val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;
891 	dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);
892 
893 	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
894 
895 	/* Configure FTS */
896 	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL);
897 	val &= ~(N_FTS_MASK << N_FTS_SHIFT);
898 	val |= N_FTS_VAL << N_FTS_SHIFT;
899 	dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val);
900 
901 	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL);
902 	val &= ~FTS_MASK;
903 	val |= FTS_VAL;
904 	dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val);
905 
906 	/* Enable as 0xFFFF0001 response for CRS */
907 	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);
908 	val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT);
909 	val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 <<
910 		AMBA_ERROR_RESPONSE_CRS_SHIFT);
911 	dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);
912 
913 	/* Configure Max Speed from DT */
914 	if (pcie->max_speed && pcie->max_speed != -EINVAL) {
915 		val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base +
916 					PCI_EXP_LNKCAP);
917 		val &= ~PCI_EXP_LNKCAP_SLS;
918 		val |= pcie->max_speed;
919 		dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP,
920 				   val);
921 	}
922 
923 	/* Configure Max lane width from DT */
924 	val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP);
925 	val &= ~PCI_EXP_LNKCAP_MLW;
926 	val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT);
927 	dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val);
928 
929 	config_gen3_gen4_eq_presets(pcie);
930 
931 	init_host_aspm(pcie);
932 
933 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
934 	val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
935 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
936 
937 	if (pcie->update_fc_fixup) {
938 		val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
939 		val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
940 		dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
941 	}
942 
943 	dw_pcie_setup_rc(pp);
944 
945 	clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
946 
947 	/* Assert RST */
948 	val = appl_readl(pcie, APPL_PINMUX);
949 	val &= ~APPL_PINMUX_PEX_RST;
950 	appl_writel(pcie, val, APPL_PINMUX);
951 
952 	usleep_range(100, 200);
953 
954 	/* Enable LTSSM */
955 	val = appl_readl(pcie, APPL_CTRL);
956 	val |= APPL_CTRL_LTSSM_EN;
957 	appl_writel(pcie, val, APPL_CTRL);
958 
959 	/* De-assert RST */
960 	val = appl_readl(pcie, APPL_PINMUX);
961 	val |= APPL_PINMUX_PEX_RST;
962 	appl_writel(pcie, val, APPL_PINMUX);
963 
964 	msleep(100);
965 }
966 
967 static int tegra_pcie_dw_host_init(struct pcie_port *pp)
968 {
969 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
970 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
971 	u32 val, tmp, offset, speed;
972 
973 	tegra_pcie_prepare_host(pp);
974 
975 	if (dw_pcie_wait_for_link(pci)) {
976 		/*
977 		 * There are some endpoints which can't get the link up if
978 		 * root port has Data Link Feature (DLF) enabled.
979 		 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info
980 		 * on Scaled Flow Control and DLF.
981 		 * So, need to confirm that is indeed the case here and attempt
982 		 * link up once again with DLF disabled.
983 		 */
984 		val = appl_readl(pcie, APPL_DEBUG);
985 		val &= APPL_DEBUG_LTSSM_STATE_MASK;
986 		val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;
987 		tmp = appl_readl(pcie, APPL_LINK_STATUS);
988 		tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;
989 		if (!(val == 0x11 && !tmp)) {
990 			/* Link is down for all good reasons */
991 			return 0;
992 		}
993 
994 		dev_info(pci->dev, "Link is down in DLL");
995 		dev_info(pci->dev, "Trying again with DLFE disabled\n");
996 		/* Disable LTSSM */
997 		val = appl_readl(pcie, APPL_CTRL);
998 		val &= ~APPL_CTRL_LTSSM_EN;
999 		appl_writel(pcie, val, APPL_CTRL);
1000 
1001 		reset_control_assert(pcie->core_rst);
1002 		reset_control_deassert(pcie->core_rst);
1003 
1004 		offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);
1005 		val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP);
1006 		val &= ~PCI_DLF_EXCHANGE_ENABLE;
1007 		dw_pcie_writel_dbi(pci, offset, val);
1008 
1009 		tegra_pcie_prepare_host(pp);
1010 
1011 		if (dw_pcie_wait_for_link(pci))
1012 			return 0;
1013 	}
1014 
1015 	speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
1016 		PCI_EXP_LNKSTA_CLS;
1017 	clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
1018 
1019 	tegra_pcie_enable_interrupts(pp);
1020 
1021 	return 0;
1022 }
1023 
1024 static int tegra_pcie_dw_link_up(struct dw_pcie *pci)
1025 {
1026 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1027 	u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
1028 
1029 	return !!(val & PCI_EXP_LNKSTA_DLLLA);
1030 }
1031 
1032 static void tegra_pcie_set_msi_vec_num(struct pcie_port *pp)
1033 {
1034 	pp->num_vectors = MAX_MSI_IRQS;
1035 }
1036 
1037 static int tegra_pcie_dw_start_link(struct dw_pcie *pci)
1038 {
1039 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1040 
1041 	enable_irq(pcie->pex_rst_irq);
1042 
1043 	return 0;
1044 }
1045 
1046 static void tegra_pcie_dw_stop_link(struct dw_pcie *pci)
1047 {
1048 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1049 
1050 	disable_irq(pcie->pex_rst_irq);
1051 }
1052 
1053 static const struct dw_pcie_ops tegra_dw_pcie_ops = {
1054 	.link_up = tegra_pcie_dw_link_up,
1055 	.start_link = tegra_pcie_dw_start_link,
1056 	.stop_link = tegra_pcie_dw_stop_link,
1057 };
1058 
1059 static struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {
1060 	.rd_own_conf = tegra_pcie_dw_rd_own_conf,
1061 	.wr_own_conf = tegra_pcie_dw_wr_own_conf,
1062 	.host_init = tegra_pcie_dw_host_init,
1063 	.set_num_vectors = tegra_pcie_set_msi_vec_num,
1064 };
1065 
1066 static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)
1067 {
1068 	unsigned int phy_count = pcie->phy_count;
1069 
1070 	while (phy_count--) {
1071 		phy_power_off(pcie->phys[phy_count]);
1072 		phy_exit(pcie->phys[phy_count]);
1073 	}
1074 }
1075 
1076 static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)
1077 {
1078 	unsigned int i;
1079 	int ret;
1080 
1081 	for (i = 0; i < pcie->phy_count; i++) {
1082 		ret = phy_init(pcie->phys[i]);
1083 		if (ret < 0)
1084 			goto phy_power_off;
1085 
1086 		ret = phy_power_on(pcie->phys[i]);
1087 		if (ret < 0)
1088 			goto phy_exit;
1089 	}
1090 
1091 	return 0;
1092 
1093 phy_power_off:
1094 	while (i--) {
1095 		phy_power_off(pcie->phys[i]);
1096 phy_exit:
1097 		phy_exit(pcie->phys[i]);
1098 	}
1099 
1100 	return ret;
1101 }
1102 
1103 static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
1104 {
1105 	struct device_node *np = pcie->dev->of_node;
1106 	int ret;
1107 
1108 	ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt);
1109 	if (ret < 0) {
1110 		dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);
1111 		return ret;
1112 	}
1113 
1114 	ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us",
1115 				   &pcie->aspm_pwr_on_t);
1116 	if (ret < 0)
1117 		dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",
1118 			 ret);
1119 
1120 	ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us",
1121 				   &pcie->aspm_l0s_enter_lat);
1122 	if (ret < 0)
1123 		dev_info(pcie->dev,
1124 			 "Failed to read ASPM L0s Entrance latency: %d\n", ret);
1125 
1126 	ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);
1127 	if (ret < 0) {
1128 		dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
1129 		return ret;
1130 	}
1131 
1132 	pcie->max_speed = of_pci_get_max_link_speed(np);
1133 
1134 	ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid);
1135 	if (ret) {
1136 		dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);
1137 		return ret;
1138 	}
1139 
1140 	ret = of_property_count_strings(np, "phy-names");
1141 	if (ret < 0) {
1142 		dev_err(pcie->dev, "Failed to find PHY entries: %d\n",
1143 			ret);
1144 		return ret;
1145 	}
1146 	pcie->phy_count = ret;
1147 
1148 	if (of_property_read_bool(np, "nvidia,update-fc-fixup"))
1149 		pcie->update_fc_fixup = true;
1150 
1151 	pcie->supports_clkreq =
1152 		of_property_read_bool(pcie->dev->of_node, "supports-clkreq");
1153 
1154 	pcie->enable_cdm_check =
1155 		of_property_read_bool(np, "snps,enable-cdm-check");
1156 
1157 	if (pcie->mode == DW_PCIE_RC_TYPE)
1158 		return 0;
1159 
1160 	/* Endpoint mode specific DT entries */
1161 	pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN);
1162 	if (IS_ERR(pcie->pex_rst_gpiod)) {
1163 		int err = PTR_ERR(pcie->pex_rst_gpiod);
1164 		const char *level = KERN_ERR;
1165 
1166 		if (err == -EPROBE_DEFER)
1167 			level = KERN_DEBUG;
1168 
1169 		dev_printk(level, pcie->dev,
1170 			   dev_fmt("Failed to get PERST GPIO: %d\n"),
1171 			   err);
1172 		return err;
1173 	}
1174 
1175 	pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev,
1176 						    "nvidia,refclk-select",
1177 						    GPIOD_OUT_HIGH);
1178 	if (IS_ERR(pcie->pex_refclk_sel_gpiod)) {
1179 		int err = PTR_ERR(pcie->pex_refclk_sel_gpiod);
1180 		const char *level = KERN_ERR;
1181 
1182 		if (err == -EPROBE_DEFER)
1183 			level = KERN_DEBUG;
1184 
1185 		dev_printk(level, pcie->dev,
1186 			   dev_fmt("Failed to get REFCLK select GPIOs: %d\n"),
1187 			   err);
1188 		pcie->pex_refclk_sel_gpiod = NULL;
1189 	}
1190 
1191 	return 0;
1192 }
1193 
1194 static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,
1195 					  bool enable)
1196 {
1197 	struct mrq_uphy_response resp;
1198 	struct tegra_bpmp_message msg;
1199 	struct mrq_uphy_request req;
1200 
1201 	/* Controller-5 doesn't need to have its state set by BPMP-FW */
1202 	if (pcie->cid == 5)
1203 		return 0;
1204 
1205 	memset(&req, 0, sizeof(req));
1206 	memset(&resp, 0, sizeof(resp));
1207 
1208 	req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
1209 	req.controller_state.pcie_controller = pcie->cid;
1210 	req.controller_state.enable = enable;
1211 
1212 	memset(&msg, 0, sizeof(msg));
1213 	msg.mrq = MRQ_UPHY;
1214 	msg.tx.data = &req;
1215 	msg.tx.size = sizeof(req);
1216 	msg.rx.data = &resp;
1217 	msg.rx.size = sizeof(resp);
1218 
1219 	return tegra_bpmp_transfer(pcie->bpmp, &msg);
1220 }
1221 
1222 static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie,
1223 					 bool enable)
1224 {
1225 	struct mrq_uphy_response resp;
1226 	struct tegra_bpmp_message msg;
1227 	struct mrq_uphy_request req;
1228 
1229 	memset(&req, 0, sizeof(req));
1230 	memset(&resp, 0, sizeof(resp));
1231 
1232 	if (enable) {
1233 		req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;
1234 		req.ep_ctrlr_pll_init.ep_controller = pcie->cid;
1235 	} else {
1236 		req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;
1237 		req.ep_ctrlr_pll_off.ep_controller = pcie->cid;
1238 	}
1239 
1240 	memset(&msg, 0, sizeof(msg));
1241 	msg.mrq = MRQ_UPHY;
1242 	msg.tx.data = &req;
1243 	msg.tx.size = sizeof(req);
1244 	msg.rx.data = &resp;
1245 	msg.rx.size = sizeof(resp);
1246 
1247 	return tegra_bpmp_transfer(pcie->bpmp, &msg);
1248 }
1249 
1250 static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)
1251 {
1252 	struct pcie_port *pp = &pcie->pci.pp;
1253 	struct pci_bus *child, *root_bus = NULL;
1254 	struct pci_dev *pdev;
1255 
1256 	/*
1257 	 * link doesn't go into L2 state with some of the endpoints with Tegra
1258 	 * if they are not in D0 state. So, need to make sure that immediate
1259 	 * downstream devices are in D0 state before sending PME_TurnOff to put
1260 	 * link into L2 state.
1261 	 * This is as per PCI Express Base r4.0 v1.0 September 27-2017,
1262 	 * 5.2 Link State Power Management (Page #428).
1263 	 */
1264 
1265 	list_for_each_entry(child, &pp->root_bus->children, node) {
1266 		/* Bring downstream devices to D0 if they are not already in */
1267 		if (child->parent == pp->root_bus) {
1268 			root_bus = child;
1269 			break;
1270 		}
1271 	}
1272 
1273 	if (!root_bus) {
1274 		dev_err(pcie->dev, "Failed to find downstream devices\n");
1275 		return;
1276 	}
1277 
1278 	list_for_each_entry(pdev, &root_bus->devices, bus_list) {
1279 		if (PCI_SLOT(pdev->devfn) == 0) {
1280 			if (pci_set_power_state(pdev, PCI_D0))
1281 				dev_err(pcie->dev,
1282 					"Failed to transition %s to D0 state\n",
1283 					dev_name(&pdev->dev));
1284 		}
1285 	}
1286 }
1287 
1288 static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)
1289 {
1290 	pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3");
1291 	if (IS_ERR(pcie->slot_ctl_3v3)) {
1292 		if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV)
1293 			return PTR_ERR(pcie->slot_ctl_3v3);
1294 
1295 		pcie->slot_ctl_3v3 = NULL;
1296 	}
1297 
1298 	pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v");
1299 	if (IS_ERR(pcie->slot_ctl_12v)) {
1300 		if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV)
1301 			return PTR_ERR(pcie->slot_ctl_12v);
1302 
1303 		pcie->slot_ctl_12v = NULL;
1304 	}
1305 
1306 	return 0;
1307 }
1308 
1309 static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)
1310 {
1311 	int ret;
1312 
1313 	if (pcie->slot_ctl_3v3) {
1314 		ret = regulator_enable(pcie->slot_ctl_3v3);
1315 		if (ret < 0) {
1316 			dev_err(pcie->dev,
1317 				"Failed to enable 3.3V slot supply: %d\n", ret);
1318 			return ret;
1319 		}
1320 	}
1321 
1322 	if (pcie->slot_ctl_12v) {
1323 		ret = regulator_enable(pcie->slot_ctl_12v);
1324 		if (ret < 0) {
1325 			dev_err(pcie->dev,
1326 				"Failed to enable 12V slot supply: %d\n", ret);
1327 			goto fail_12v_enable;
1328 		}
1329 	}
1330 
1331 	/*
1332 	 * According to PCI Express Card Electromechanical Specification
1333 	 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)
1334 	 * should be a minimum of 100ms.
1335 	 */
1336 	if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)
1337 		msleep(100);
1338 
1339 	return 0;
1340 
1341 fail_12v_enable:
1342 	if (pcie->slot_ctl_3v3)
1343 		regulator_disable(pcie->slot_ctl_3v3);
1344 	return ret;
1345 }
1346 
1347 static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)
1348 {
1349 	if (pcie->slot_ctl_12v)
1350 		regulator_disable(pcie->slot_ctl_12v);
1351 	if (pcie->slot_ctl_3v3)
1352 		regulator_disable(pcie->slot_ctl_3v3);
1353 }
1354 
1355 static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,
1356 					bool en_hw_hot_rst)
1357 {
1358 	int ret;
1359 	u32 val;
1360 
1361 	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1362 	if (ret) {
1363 		dev_err(pcie->dev,
1364 			"Failed to enable controller %u: %d\n", pcie->cid, ret);
1365 		return ret;
1366 	}
1367 
1368 	ret = tegra_pcie_enable_slot_regulators(pcie);
1369 	if (ret < 0)
1370 		goto fail_slot_reg_en;
1371 
1372 	ret = regulator_enable(pcie->pex_ctl_supply);
1373 	if (ret < 0) {
1374 		dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);
1375 		goto fail_reg_en;
1376 	}
1377 
1378 	ret = clk_prepare_enable(pcie->core_clk);
1379 	if (ret) {
1380 		dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);
1381 		goto fail_core_clk;
1382 	}
1383 
1384 	ret = reset_control_deassert(pcie->core_apb_rst);
1385 	if (ret) {
1386 		dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",
1387 			ret);
1388 		goto fail_core_apb_rst;
1389 	}
1390 
1391 	if (en_hw_hot_rst) {
1392 		/* Enable HW_HOT_RST mode */
1393 		val = appl_readl(pcie, APPL_CTRL);
1394 		val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1395 			 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1396 		val |= APPL_CTRL_HW_HOT_RST_EN;
1397 		appl_writel(pcie, val, APPL_CTRL);
1398 	}
1399 
1400 	ret = tegra_pcie_enable_phy(pcie);
1401 	if (ret) {
1402 		dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);
1403 		goto fail_phy;
1404 	}
1405 
1406 	/* Update CFG base address */
1407 	appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1408 		    APPL_CFG_BASE_ADDR);
1409 
1410 	/* Configure this core for RP mode operation */
1411 	appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);
1412 
1413 	appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1414 
1415 	val = appl_readl(pcie, APPL_CTRL);
1416 	appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);
1417 
1418 	val = appl_readl(pcie, APPL_CFG_MISC);
1419 	val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1420 	appl_writel(pcie, val, APPL_CFG_MISC);
1421 
1422 	if (!pcie->supports_clkreq) {
1423 		val = appl_readl(pcie, APPL_PINMUX);
1424 		val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN;
1425 		val &= ~APPL_PINMUX_CLKREQ_OVERRIDE;
1426 		appl_writel(pcie, val, APPL_PINMUX);
1427 	}
1428 
1429 	/* Update iATU_DMA base address */
1430 	appl_writel(pcie,
1431 		    pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1432 		    APPL_CFG_IATU_DMA_BASE_ADDR);
1433 
1434 	reset_control_deassert(pcie->core_rst);
1435 
1436 	pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1437 						      PCI_CAP_ID_EXP);
1438 
1439 	/* Disable ASPM-L1SS advertisement as there is no CLKREQ routing */
1440 	if (!pcie->supports_clkreq) {
1441 		disable_aspm_l11(pcie);
1442 		disable_aspm_l12(pcie);
1443 	}
1444 
1445 	return ret;
1446 
1447 fail_phy:
1448 	reset_control_assert(pcie->core_apb_rst);
1449 fail_core_apb_rst:
1450 	clk_disable_unprepare(pcie->core_clk);
1451 fail_core_clk:
1452 	regulator_disable(pcie->pex_ctl_supply);
1453 fail_reg_en:
1454 	tegra_pcie_disable_slot_regulators(pcie);
1455 fail_slot_reg_en:
1456 	tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1457 
1458 	return ret;
1459 }
1460 
1461 static int __deinit_controller(struct tegra_pcie_dw *pcie)
1462 {
1463 	int ret;
1464 
1465 	ret = reset_control_assert(pcie->core_rst);
1466 	if (ret) {
1467 		dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n",
1468 			ret);
1469 		return ret;
1470 	}
1471 
1472 	tegra_pcie_disable_phy(pcie);
1473 
1474 	ret = reset_control_assert(pcie->core_apb_rst);
1475 	if (ret) {
1476 		dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);
1477 		return ret;
1478 	}
1479 
1480 	clk_disable_unprepare(pcie->core_clk);
1481 
1482 	ret = regulator_disable(pcie->pex_ctl_supply);
1483 	if (ret) {
1484 		dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);
1485 		return ret;
1486 	}
1487 
1488 	tegra_pcie_disable_slot_regulators(pcie);
1489 
1490 	ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1491 	if (ret) {
1492 		dev_err(pcie->dev, "Failed to disable controller %d: %d\n",
1493 			pcie->cid, ret);
1494 		return ret;
1495 	}
1496 
1497 	return ret;
1498 }
1499 
1500 static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)
1501 {
1502 	struct dw_pcie *pci = &pcie->pci;
1503 	struct pcie_port *pp = &pci->pp;
1504 	int ret;
1505 
1506 	ret = tegra_pcie_config_controller(pcie, false);
1507 	if (ret < 0)
1508 		return ret;
1509 
1510 	pp->ops = &tegra_pcie_dw_host_ops;
1511 
1512 	ret = dw_pcie_host_init(pp);
1513 	if (ret < 0) {
1514 		dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);
1515 		goto fail_host_init;
1516 	}
1517 
1518 	return 0;
1519 
1520 fail_host_init:
1521 	return __deinit_controller(pcie);
1522 }
1523 
1524 static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)
1525 {
1526 	u32 val;
1527 
1528 	if (!tegra_pcie_dw_link_up(&pcie->pci))
1529 		return 0;
1530 
1531 	val = appl_readl(pcie, APPL_RADM_STATUS);
1532 	val |= APPL_PM_XMT_TURNOFF_STATE;
1533 	appl_writel(pcie, val, APPL_RADM_STATUS);
1534 
1535 	return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,
1536 				 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,
1537 				 1, PME_ACK_TIMEOUT);
1538 }
1539 
1540 static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)
1541 {
1542 	u32 data;
1543 	int err;
1544 
1545 	if (!tegra_pcie_dw_link_up(&pcie->pci)) {
1546 		dev_dbg(pcie->dev, "PCIe link is not up...!\n");
1547 		return;
1548 	}
1549 
1550 	if (tegra_pcie_try_link_l2(pcie)) {
1551 		dev_info(pcie->dev, "Link didn't transition to L2 state\n");
1552 		/*
1553 		 * TX lane clock freq will reset to Gen1 only if link is in L2
1554 		 * or detect state.
1555 		 * So apply pex_rst to end point to force RP to go into detect
1556 		 * state
1557 		 */
1558 		data = appl_readl(pcie, APPL_PINMUX);
1559 		data &= ~APPL_PINMUX_PEX_RST;
1560 		appl_writel(pcie, data, APPL_PINMUX);
1561 
1562 		err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,
1563 						data,
1564 						((data &
1565 						APPL_DEBUG_LTSSM_STATE_MASK) >>
1566 						APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1567 						LTSSM_STATE_PRE_DETECT,
1568 						1, LTSSM_TIMEOUT);
1569 		if (err) {
1570 			dev_info(pcie->dev, "Link didn't go to detect state\n");
1571 		} else {
1572 			/* Disable LTSSM after link is in detect state */
1573 			data = appl_readl(pcie, APPL_CTRL);
1574 			data &= ~APPL_CTRL_LTSSM_EN;
1575 			appl_writel(pcie, data, APPL_CTRL);
1576 		}
1577 	}
1578 	/*
1579 	 * DBI registers may not be accessible after this as PLL-E would be
1580 	 * down depending on how CLKREQ is pulled by end point
1581 	 */
1582 	data = appl_readl(pcie, APPL_PINMUX);
1583 	data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);
1584 	/* Cut REFCLK to slot */
1585 	data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1586 	data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1587 	appl_writel(pcie, data, APPL_PINMUX);
1588 }
1589 
1590 static int tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)
1591 {
1592 	tegra_pcie_downstream_dev_to_D0(pcie);
1593 	dw_pcie_host_deinit(&pcie->pci.pp);
1594 	tegra_pcie_dw_pme_turnoff(pcie);
1595 
1596 	return __deinit_controller(pcie);
1597 }
1598 
1599 static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
1600 {
1601 	struct pcie_port *pp = &pcie->pci.pp;
1602 	struct device *dev = pcie->dev;
1603 	char *name;
1604 	int ret;
1605 
1606 	if (IS_ENABLED(CONFIG_PCI_MSI)) {
1607 		pp->msi_irq = of_irq_get_byname(dev->of_node, "msi");
1608 		if (!pp->msi_irq) {
1609 			dev_err(dev, "Failed to get MSI interrupt\n");
1610 			return -ENODEV;
1611 		}
1612 	}
1613 
1614 	pm_runtime_enable(dev);
1615 
1616 	ret = pm_runtime_get_sync(dev);
1617 	if (ret < 0) {
1618 		dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1619 			ret);
1620 		goto fail_pm_get_sync;
1621 	}
1622 
1623 	ret = pinctrl_pm_select_default_state(dev);
1624 	if (ret < 0) {
1625 		dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
1626 		goto fail_pm_get_sync;
1627 	}
1628 
1629 	tegra_pcie_init_controller(pcie);
1630 
1631 	pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci);
1632 	if (!pcie->link_state) {
1633 		ret = -ENOMEDIUM;
1634 		goto fail_host_init;
1635 	}
1636 
1637 	name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
1638 	if (!name) {
1639 		ret = -ENOMEM;
1640 		goto fail_host_init;
1641 	}
1642 
1643 	pcie->debugfs = debugfs_create_dir(name, NULL);
1644 	if (!pcie->debugfs)
1645 		dev_err(dev, "Failed to create debugfs\n");
1646 	else
1647 		init_debugfs(pcie);
1648 
1649 	return ret;
1650 
1651 fail_host_init:
1652 	tegra_pcie_deinit_controller(pcie);
1653 fail_pm_get_sync:
1654 	pm_runtime_put_sync(dev);
1655 	pm_runtime_disable(dev);
1656 	return ret;
1657 }
1658 
1659 static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie)
1660 {
1661 	u32 val;
1662 	int ret;
1663 
1664 	if (pcie->ep_state == EP_STATE_DISABLED)
1665 		return;
1666 
1667 	/* Disable LTSSM */
1668 	val = appl_readl(pcie, APPL_CTRL);
1669 	val &= ~APPL_CTRL_LTSSM_EN;
1670 	appl_writel(pcie, val, APPL_CTRL);
1671 
1672 	ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val,
1673 				 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >>
1674 				 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1675 				 LTSSM_STATE_PRE_DETECT,
1676 				 1, LTSSM_TIMEOUT);
1677 	if (ret)
1678 		dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret);
1679 
1680 	reset_control_assert(pcie->core_rst);
1681 
1682 	tegra_pcie_disable_phy(pcie);
1683 
1684 	reset_control_assert(pcie->core_apb_rst);
1685 
1686 	clk_disable_unprepare(pcie->core_clk);
1687 
1688 	pm_runtime_put_sync(pcie->dev);
1689 
1690 	ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1691 	if (ret)
1692 		dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret);
1693 
1694 	pcie->ep_state = EP_STATE_DISABLED;
1695 	dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n");
1696 }
1697 
1698 static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie)
1699 {
1700 	struct dw_pcie *pci = &pcie->pci;
1701 	struct dw_pcie_ep *ep = &pci->ep;
1702 	struct device *dev = pcie->dev;
1703 	u32 val;
1704 	int ret;
1705 
1706 	if (pcie->ep_state == EP_STATE_ENABLED)
1707 		return;
1708 
1709 	ret = pm_runtime_get_sync(dev);
1710 	if (ret < 0) {
1711 		dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1712 			ret);
1713 		return;
1714 	}
1715 
1716 	ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1717 	if (ret) {
1718 		dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n", ret);
1719 		goto fail_pll_init;
1720 	}
1721 
1722 	ret = clk_prepare_enable(pcie->core_clk);
1723 	if (ret) {
1724 		dev_err(dev, "Failed to enable core clock: %d\n", ret);
1725 		goto fail_core_clk_enable;
1726 	}
1727 
1728 	ret = reset_control_deassert(pcie->core_apb_rst);
1729 	if (ret) {
1730 		dev_err(dev, "Failed to deassert core APB reset: %d\n", ret);
1731 		goto fail_core_apb_rst;
1732 	}
1733 
1734 	ret = tegra_pcie_enable_phy(pcie);
1735 	if (ret) {
1736 		dev_err(dev, "Failed to enable PHY: %d\n", ret);
1737 		goto fail_phy;
1738 	}
1739 
1740 	/* Clear any stale interrupt statuses */
1741 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
1742 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
1743 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
1744 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
1745 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
1746 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
1747 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
1748 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
1749 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
1750 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
1751 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
1752 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
1753 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
1754 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
1755 	appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
1756 
1757 	/* configure this core for EP mode operation */
1758 	val = appl_readl(pcie, APPL_DM_TYPE);
1759 	val &= ~APPL_DM_TYPE_MASK;
1760 	val |= APPL_DM_TYPE_EP;
1761 	appl_writel(pcie, val, APPL_DM_TYPE);
1762 
1763 	appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1764 
1765 	val = appl_readl(pcie, APPL_CTRL);
1766 	val |= APPL_CTRL_SYS_PRE_DET_STATE;
1767 	val |= APPL_CTRL_HW_HOT_RST_EN;
1768 	appl_writel(pcie, val, APPL_CTRL);
1769 
1770 	val = appl_readl(pcie, APPL_CFG_MISC);
1771 	val |= APPL_CFG_MISC_SLV_EP_MODE;
1772 	val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1773 	appl_writel(pcie, val, APPL_CFG_MISC);
1774 
1775 	val = appl_readl(pcie, APPL_PINMUX);
1776 	val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1777 	val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1778 	appl_writel(pcie, val, APPL_PINMUX);
1779 
1780 	appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1781 		    APPL_CFG_BASE_ADDR);
1782 
1783 	appl_writel(pcie, pcie->atu_dma_res->start &
1784 		    APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1785 		    APPL_CFG_IATU_DMA_BASE_ADDR);
1786 
1787 	val = appl_readl(pcie, APPL_INTR_EN_L0_0);
1788 	val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
1789 	val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
1790 	val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN;
1791 	appl_writel(pcie, val, APPL_INTR_EN_L0_0);
1792 
1793 	val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
1794 	val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN;
1795 	val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN;
1796 	appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
1797 
1798 	reset_control_deassert(pcie->core_rst);
1799 
1800 	if (pcie->update_fc_fixup) {
1801 		val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
1802 		val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
1803 		dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
1804 	}
1805 
1806 	config_gen3_gen4_eq_presets(pcie);
1807 
1808 	init_host_aspm(pcie);
1809 
1810 	/* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
1811 	if (!pcie->supports_clkreq) {
1812 		disable_aspm_l11(pcie);
1813 		disable_aspm_l12(pcie);
1814 	}
1815 
1816 	val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1817 	val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1818 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
1819 
1820 	/* Configure N_FTS & FTS */
1821 	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL);
1822 	val &= ~(N_FTS_MASK << N_FTS_SHIFT);
1823 	val |= N_FTS_VAL << N_FTS_SHIFT;
1824 	dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val);
1825 
1826 	val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL);
1827 	val &= ~FTS_MASK;
1828 	val |= FTS_VAL;
1829 	dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val);
1830 
1831 	/* Configure Max Speed from DT */
1832 	if (pcie->max_speed && pcie->max_speed != -EINVAL) {
1833 		val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base +
1834 					PCI_EXP_LNKCAP);
1835 		val &= ~PCI_EXP_LNKCAP_SLS;
1836 		val |= pcie->max_speed;
1837 		dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP,
1838 				   val);
1839 	}
1840 
1841 	pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1842 						      PCI_CAP_ID_EXP);
1843 	clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
1844 
1845 	val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK);
1846 	val |= MSIX_ADDR_MATCH_LOW_OFF_EN;
1847 	dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val);
1848 	val = (lower_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK);
1849 	dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val);
1850 
1851 	ret = dw_pcie_ep_init_complete(ep);
1852 	if (ret) {
1853 		dev_err(dev, "Failed to complete initialization: %d\n", ret);
1854 		goto fail_init_complete;
1855 	}
1856 
1857 	dw_pcie_ep_init_notify(ep);
1858 
1859 	/* Enable LTSSM */
1860 	val = appl_readl(pcie, APPL_CTRL);
1861 	val |= APPL_CTRL_LTSSM_EN;
1862 	appl_writel(pcie, val, APPL_CTRL);
1863 
1864 	pcie->ep_state = EP_STATE_ENABLED;
1865 	dev_dbg(dev, "Initialization of endpoint is completed\n");
1866 
1867 	return;
1868 
1869 fail_init_complete:
1870 	reset_control_assert(pcie->core_rst);
1871 	tegra_pcie_disable_phy(pcie);
1872 fail_phy:
1873 	reset_control_assert(pcie->core_apb_rst);
1874 fail_core_apb_rst:
1875 	clk_disable_unprepare(pcie->core_clk);
1876 fail_core_clk_enable:
1877 	tegra_pcie_bpmp_set_pll_state(pcie, false);
1878 fail_pll_init:
1879 	pm_runtime_put_sync(dev);
1880 }
1881 
1882 static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)
1883 {
1884 	struct tegra_pcie_dw *pcie = arg;
1885 
1886 	if (gpiod_get_value(pcie->pex_rst_gpiod))
1887 		pex_ep_event_pex_rst_assert(pcie);
1888 	else
1889 		pex_ep_event_pex_rst_deassert(pcie);
1890 
1891 	return IRQ_HANDLED;
1892 }
1893 
1894 static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq)
1895 {
1896 	/* Tegra194 supports only INTA */
1897 	if (irq > 1)
1898 		return -EINVAL;
1899 
1900 	appl_writel(pcie, 1, APPL_LEGACY_INTX);
1901 	usleep_range(1000, 2000);
1902 	appl_writel(pcie, 0, APPL_LEGACY_INTX);
1903 	return 0;
1904 }
1905 
1906 static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq)
1907 {
1908 	if (unlikely(irq > 31))
1909 		return -EINVAL;
1910 
1911 	appl_writel(pcie, (1 << irq), APPL_MSI_CTRL_1);
1912 
1913 	return 0;
1914 }
1915 
1916 static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq)
1917 {
1918 	struct dw_pcie_ep *ep = &pcie->pci.ep;
1919 
1920 	writel(irq, ep->msi_mem);
1921 
1922 	return 0;
1923 }
1924 
1925 static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1926 				   enum pci_epc_irq_type type,
1927 				   u16 interrupt_num)
1928 {
1929 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1930 	struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1931 
1932 	switch (type) {
1933 	case PCI_EPC_IRQ_LEGACY:
1934 		return tegra_pcie_ep_raise_legacy_irq(pcie, interrupt_num);
1935 
1936 	case PCI_EPC_IRQ_MSI:
1937 		return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num);
1938 
1939 	case PCI_EPC_IRQ_MSIX:
1940 		return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num);
1941 
1942 	default:
1943 		dev_err(pci->dev, "Unknown IRQ type\n");
1944 		return -EPERM;
1945 	}
1946 
1947 	return 0;
1948 }
1949 
1950 static const struct pci_epc_features tegra_pcie_epc_features = {
1951 	.linkup_notifier = true,
1952 	.core_init_notifier = true,
1953 	.msi_capable = false,
1954 	.msix_capable = false,
1955 	.reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5,
1956 	.bar_fixed_64bit = 1 << BAR_0,
1957 	.bar_fixed_size[0] = SZ_1M,
1958 };
1959 
1960 static const struct pci_epc_features*
1961 tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)
1962 {
1963 	return &tegra_pcie_epc_features;
1964 }
1965 
1966 static struct dw_pcie_ep_ops pcie_ep_ops = {
1967 	.raise_irq = tegra_pcie_ep_raise_irq,
1968 	.get_features = tegra_pcie_ep_get_features,
1969 };
1970 
1971 static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie,
1972 				struct platform_device *pdev)
1973 {
1974 	struct dw_pcie *pci = &pcie->pci;
1975 	struct device *dev = pcie->dev;
1976 	struct dw_pcie_ep *ep;
1977 	struct resource *res;
1978 	char *name;
1979 	int ret;
1980 
1981 	ep = &pci->ep;
1982 	ep->ops = &pcie_ep_ops;
1983 
1984 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space");
1985 	if (!res)
1986 		return -EINVAL;
1987 
1988 	ep->phys_base = res->start;
1989 	ep->addr_size = resource_size(res);
1990 	ep->page_size = SZ_64K;
1991 
1992 	ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME);
1993 	if (ret < 0) {
1994 		dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n",
1995 			ret);
1996 		return ret;
1997 	}
1998 
1999 	ret = gpiod_to_irq(pcie->pex_rst_gpiod);
2000 	if (ret < 0) {
2001 		dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret);
2002 		return ret;
2003 	}
2004 	pcie->pex_rst_irq = (unsigned int)ret;
2005 
2006 	name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq",
2007 			      pcie->cid);
2008 	if (!name) {
2009 		dev_err(dev, "Failed to create PERST IRQ string\n");
2010 		return -ENOMEM;
2011 	}
2012 
2013 	irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN);
2014 
2015 	pcie->ep_state = EP_STATE_DISABLED;
2016 
2017 	ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL,
2018 					tegra_pcie_ep_pex_rst_irq,
2019 					IRQF_TRIGGER_RISING |
2020 					IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
2021 					name, (void *)pcie);
2022 	if (ret < 0) {
2023 		dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret);
2024 		return ret;
2025 	}
2026 
2027 	name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_ep_work",
2028 			      pcie->cid);
2029 	if (!name) {
2030 		dev_err(dev, "Failed to create PCIe EP work thread string\n");
2031 		return -ENOMEM;
2032 	}
2033 
2034 	pm_runtime_enable(dev);
2035 
2036 	ret = dw_pcie_ep_init(ep);
2037 	if (ret) {
2038 		dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n",
2039 			ret);
2040 		return ret;
2041 	}
2042 
2043 	return 0;
2044 }
2045 
2046 static int tegra_pcie_dw_probe(struct platform_device *pdev)
2047 {
2048 	const struct tegra_pcie_dw_of_data *data;
2049 	struct device *dev = &pdev->dev;
2050 	struct resource *atu_dma_res;
2051 	struct tegra_pcie_dw *pcie;
2052 	struct resource *dbi_res;
2053 	struct pcie_port *pp;
2054 	struct dw_pcie *pci;
2055 	struct phy **phys;
2056 	char *name;
2057 	int ret;
2058 	u32 i;
2059 
2060 	data = of_device_get_match_data(dev);
2061 
2062 	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
2063 	if (!pcie)
2064 		return -ENOMEM;
2065 
2066 	pci = &pcie->pci;
2067 	pci->dev = &pdev->dev;
2068 	pci->ops = &tegra_dw_pcie_ops;
2069 	pp = &pci->pp;
2070 	pcie->dev = &pdev->dev;
2071 	pcie->mode = (enum dw_pcie_device_mode)data->mode;
2072 
2073 	ret = tegra_pcie_dw_parse_dt(pcie);
2074 	if (ret < 0) {
2075 		const char *level = KERN_ERR;
2076 
2077 		if (ret == -EPROBE_DEFER)
2078 			level = KERN_DEBUG;
2079 
2080 		dev_printk(level, dev,
2081 			   dev_fmt("Failed to parse device tree: %d\n"),
2082 			   ret);
2083 		return ret;
2084 	}
2085 
2086 	ret = tegra_pcie_get_slot_regulators(pcie);
2087 	if (ret < 0) {
2088 		const char *level = KERN_ERR;
2089 
2090 		if (ret == -EPROBE_DEFER)
2091 			level = KERN_DEBUG;
2092 
2093 		dev_printk(level, dev,
2094 			   dev_fmt("Failed to get slot regulators: %d\n"),
2095 			   ret);
2096 		return ret;
2097 	}
2098 
2099 	if (pcie->pex_refclk_sel_gpiod)
2100 		gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1);
2101 
2102 	pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl");
2103 	if (IS_ERR(pcie->pex_ctl_supply)) {
2104 		ret = PTR_ERR(pcie->pex_ctl_supply);
2105 		if (ret != -EPROBE_DEFER)
2106 			dev_err(dev, "Failed to get regulator: %ld\n",
2107 				PTR_ERR(pcie->pex_ctl_supply));
2108 		return ret;
2109 	}
2110 
2111 	pcie->core_clk = devm_clk_get(dev, "core");
2112 	if (IS_ERR(pcie->core_clk)) {
2113 		dev_err(dev, "Failed to get core clock: %ld\n",
2114 			PTR_ERR(pcie->core_clk));
2115 		return PTR_ERR(pcie->core_clk);
2116 	}
2117 
2118 	pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2119 						      "appl");
2120 	if (!pcie->appl_res) {
2121 		dev_err(dev, "Failed to find \"appl\" region\n");
2122 		return -ENODEV;
2123 	}
2124 
2125 	pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res);
2126 	if (IS_ERR(pcie->appl_base))
2127 		return PTR_ERR(pcie->appl_base);
2128 
2129 	pcie->core_apb_rst = devm_reset_control_get(dev, "apb");
2130 	if (IS_ERR(pcie->core_apb_rst)) {
2131 		dev_err(dev, "Failed to get APB reset: %ld\n",
2132 			PTR_ERR(pcie->core_apb_rst));
2133 		return PTR_ERR(pcie->core_apb_rst);
2134 	}
2135 
2136 	phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL);
2137 	if (!phys)
2138 		return -ENOMEM;
2139 
2140 	for (i = 0; i < pcie->phy_count; i++) {
2141 		name = kasprintf(GFP_KERNEL, "p2u-%u", i);
2142 		if (!name) {
2143 			dev_err(dev, "Failed to create P2U string\n");
2144 			return -ENOMEM;
2145 		}
2146 		phys[i] = devm_phy_get(dev, name);
2147 		kfree(name);
2148 		if (IS_ERR(phys[i])) {
2149 			ret = PTR_ERR(phys[i]);
2150 			if (ret != -EPROBE_DEFER)
2151 				dev_err(dev, "Failed to get PHY: %d\n", ret);
2152 			return ret;
2153 		}
2154 	}
2155 
2156 	pcie->phys = phys;
2157 
2158 	dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
2159 	if (!dbi_res) {
2160 		dev_err(dev, "Failed to find \"dbi\" region\n");
2161 		return -ENODEV;
2162 	}
2163 	pcie->dbi_res = dbi_res;
2164 
2165 	pci->dbi_base = devm_ioremap_resource(dev, dbi_res);
2166 	if (IS_ERR(pci->dbi_base))
2167 		return PTR_ERR(pci->dbi_base);
2168 
2169 	/* Tegra HW locates DBI2 at a fixed offset from DBI */
2170 	pci->dbi_base2 = pci->dbi_base + 0x1000;
2171 
2172 	atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2173 						   "atu_dma");
2174 	if (!atu_dma_res) {
2175 		dev_err(dev, "Failed to find \"atu_dma\" region\n");
2176 		return -ENODEV;
2177 	}
2178 	pcie->atu_dma_res = atu_dma_res;
2179 
2180 	pci->atu_base = devm_ioremap_resource(dev, atu_dma_res);
2181 	if (IS_ERR(pci->atu_base))
2182 		return PTR_ERR(pci->atu_base);
2183 
2184 	pcie->core_rst = devm_reset_control_get(dev, "core");
2185 	if (IS_ERR(pcie->core_rst)) {
2186 		dev_err(dev, "Failed to get core reset: %ld\n",
2187 			PTR_ERR(pcie->core_rst));
2188 		return PTR_ERR(pcie->core_rst);
2189 	}
2190 
2191 	pp->irq = platform_get_irq_byname(pdev, "intr");
2192 	if (pp->irq < 0)
2193 		return pp->irq;
2194 
2195 	pcie->bpmp = tegra_bpmp_get(dev);
2196 	if (IS_ERR(pcie->bpmp))
2197 		return PTR_ERR(pcie->bpmp);
2198 
2199 	platform_set_drvdata(pdev, pcie);
2200 
2201 	switch (pcie->mode) {
2202 	case DW_PCIE_RC_TYPE:
2203 		ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler,
2204 				       IRQF_SHARED, "tegra-pcie-intr", pcie);
2205 		if (ret) {
2206 			dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2207 				ret);
2208 			goto fail;
2209 		}
2210 
2211 		ret = tegra_pcie_config_rp(pcie);
2212 		if (ret && ret != -ENOMEDIUM)
2213 			goto fail;
2214 		else
2215 			return 0;
2216 		break;
2217 
2218 	case DW_PCIE_EP_TYPE:
2219 		ret = devm_request_threaded_irq(dev, pp->irq,
2220 						tegra_pcie_ep_hard_irq,
2221 						tegra_pcie_ep_irq_thread,
2222 						IRQF_SHARED | IRQF_ONESHOT,
2223 						"tegra-pcie-ep-intr", pcie);
2224 		if (ret) {
2225 			dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2226 				ret);
2227 			goto fail;
2228 		}
2229 
2230 		ret = tegra_pcie_config_ep(pcie, pdev);
2231 		if (ret < 0)
2232 			goto fail;
2233 		break;
2234 
2235 	default:
2236 		dev_err(dev, "Invalid PCIe device type %d\n", pcie->mode);
2237 	}
2238 
2239 fail:
2240 	tegra_bpmp_put(pcie->bpmp);
2241 	return ret;
2242 }
2243 
2244 static int tegra_pcie_dw_remove(struct platform_device *pdev)
2245 {
2246 	struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2247 
2248 	if (!pcie->link_state)
2249 		return 0;
2250 
2251 	debugfs_remove_recursive(pcie->debugfs);
2252 	tegra_pcie_deinit_controller(pcie);
2253 	pm_runtime_put_sync(pcie->dev);
2254 	pm_runtime_disable(pcie->dev);
2255 	tegra_bpmp_put(pcie->bpmp);
2256 	if (pcie->pex_refclk_sel_gpiod)
2257 		gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0);
2258 
2259 	return 0;
2260 }
2261 
2262 static int tegra_pcie_dw_suspend_late(struct device *dev)
2263 {
2264 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2265 	u32 val;
2266 
2267 	if (!pcie->link_state)
2268 		return 0;
2269 
2270 	/* Enable HW_HOT_RST mode */
2271 	val = appl_readl(pcie, APPL_CTRL);
2272 	val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2273 		 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2274 	val |= APPL_CTRL_HW_HOT_RST_EN;
2275 	appl_writel(pcie, val, APPL_CTRL);
2276 
2277 	return 0;
2278 }
2279 
2280 static int tegra_pcie_dw_suspend_noirq(struct device *dev)
2281 {
2282 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2283 
2284 	if (!pcie->link_state)
2285 		return 0;
2286 
2287 	/* Save MSI interrupt vector */
2288 	pcie->msi_ctrl_int = dw_pcie_readl_dbi(&pcie->pci,
2289 					       PORT_LOGIC_MSI_CTRL_INT_0_EN);
2290 	tegra_pcie_downstream_dev_to_D0(pcie);
2291 	tegra_pcie_dw_pme_turnoff(pcie);
2292 
2293 	return __deinit_controller(pcie);
2294 }
2295 
2296 static int tegra_pcie_dw_resume_noirq(struct device *dev)
2297 {
2298 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2299 	int ret;
2300 
2301 	if (!pcie->link_state)
2302 		return 0;
2303 
2304 	ret = tegra_pcie_config_controller(pcie, true);
2305 	if (ret < 0)
2306 		return ret;
2307 
2308 	ret = tegra_pcie_dw_host_init(&pcie->pci.pp);
2309 	if (ret < 0) {
2310 		dev_err(dev, "Failed to init host: %d\n", ret);
2311 		goto fail_host_init;
2312 	}
2313 
2314 	/* Restore MSI interrupt vector */
2315 	dw_pcie_writel_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN,
2316 			   pcie->msi_ctrl_int);
2317 
2318 	return 0;
2319 
2320 fail_host_init:
2321 	return __deinit_controller(pcie);
2322 }
2323 
2324 static int tegra_pcie_dw_resume_early(struct device *dev)
2325 {
2326 	struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2327 	u32 val;
2328 
2329 	if (!pcie->link_state)
2330 		return 0;
2331 
2332 	/* Disable HW_HOT_RST mode */
2333 	val = appl_readl(pcie, APPL_CTRL);
2334 	val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2335 		 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2336 	val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<
2337 	       APPL_CTRL_HW_HOT_RST_MODE_SHIFT;
2338 	val &= ~APPL_CTRL_HW_HOT_RST_EN;
2339 	appl_writel(pcie, val, APPL_CTRL);
2340 
2341 	return 0;
2342 }
2343 
2344 static void tegra_pcie_dw_shutdown(struct platform_device *pdev)
2345 {
2346 	struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2347 
2348 	if (!pcie->link_state)
2349 		return;
2350 
2351 	debugfs_remove_recursive(pcie->debugfs);
2352 	tegra_pcie_downstream_dev_to_D0(pcie);
2353 
2354 	disable_irq(pcie->pci.pp.irq);
2355 	if (IS_ENABLED(CONFIG_PCI_MSI))
2356 		disable_irq(pcie->pci.pp.msi_irq);
2357 
2358 	tegra_pcie_dw_pme_turnoff(pcie);
2359 	__deinit_controller(pcie);
2360 }
2361 
2362 static const struct tegra_pcie_dw_of_data tegra_pcie_dw_rc_of_data = {
2363 	.mode = DW_PCIE_RC_TYPE,
2364 };
2365 
2366 static const struct tegra_pcie_dw_of_data tegra_pcie_dw_ep_of_data = {
2367 	.mode = DW_PCIE_EP_TYPE,
2368 };
2369 
2370 static const struct of_device_id tegra_pcie_dw_of_match[] = {
2371 	{
2372 		.compatible = "nvidia,tegra194-pcie",
2373 		.data = &tegra_pcie_dw_rc_of_data,
2374 	},
2375 	{
2376 		.compatible = "nvidia,tegra194-pcie-ep",
2377 		.data = &tegra_pcie_dw_ep_of_data,
2378 	},
2379 	{},
2380 };
2381 
2382 static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {
2383 	.suspend_late = tegra_pcie_dw_suspend_late,
2384 	.suspend_noirq = tegra_pcie_dw_suspend_noirq,
2385 	.resume_noirq = tegra_pcie_dw_resume_noirq,
2386 	.resume_early = tegra_pcie_dw_resume_early,
2387 };
2388 
2389 static struct platform_driver tegra_pcie_dw_driver = {
2390 	.probe = tegra_pcie_dw_probe,
2391 	.remove = tegra_pcie_dw_remove,
2392 	.shutdown = tegra_pcie_dw_shutdown,
2393 	.driver = {
2394 		.name	= "tegra194-pcie",
2395 		.pm = &tegra_pcie_dw_pm_ops,
2396 		.of_match_table = tegra_pcie_dw_of_match,
2397 	},
2398 };
2399 module_platform_driver(tegra_pcie_dw_driver);
2400 
2401 MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);
2402 
2403 MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");
2404 MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");
2405 MODULE_LICENSE("GPL v2");
2406