1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * PCIe host controller driver for Freescale i.MX6 SoCs
4  *
5  * Copyright (C) 2013 Kosagi
6  *		https://www.kosagi.com
7  *
8  * Author: Sean Cross <xobs@kosagi.com>
9  */
10 
11 #include <linux/bitfield.h>
12 #include <linux/clk.h>
13 #include <linux/delay.h>
14 #include <linux/gpio.h>
15 #include <linux/kernel.h>
16 #include <linux/mfd/syscon.h>
17 #include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
18 #include <linux/mfd/syscon/imx7-iomuxc-gpr.h>
19 #include <linux/module.h>
20 #include <linux/of_gpio.h>
21 #include <linux/of_device.h>
22 #include <linux/of_address.h>
23 #include <linux/pci.h>
24 #include <linux/platform_device.h>
25 #include <linux/regmap.h>
26 #include <linux/regulator/consumer.h>
27 #include <linux/resource.h>
28 #include <linux/signal.h>
29 #include <linux/types.h>
30 #include <linux/interrupt.h>
31 #include <linux/reset.h>
32 #include <linux/phy/phy.h>
33 #include <linux/pm_domain.h>
34 #include <linux/pm_runtime.h>
35 
36 #include "pcie-designware.h"
37 
38 #define IMX8MQ_GPR_PCIE_REF_USE_PAD		BIT(9)
39 #define IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN	BIT(10)
40 #define IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE	BIT(11)
41 #define IMX8MQ_GPR_PCIE_VREG_BYPASS		BIT(12)
42 #define IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE	GENMASK(11, 8)
43 #define IMX8MQ_PCIE2_BASE_ADDR			0x33c00000
44 
45 #define to_imx6_pcie(x)	dev_get_drvdata((x)->dev)
46 
47 enum imx6_pcie_variants {
48 	IMX6Q,
49 	IMX6SX,
50 	IMX6QP,
51 	IMX7D,
52 	IMX8MQ,
53 	IMX8MM,
54 	IMX8MP,
55 };
56 
57 #define IMX6_PCIE_FLAG_IMX6_PHY			BIT(0)
58 #define IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE	BIT(1)
59 #define IMX6_PCIE_FLAG_SUPPORTS_SUSPEND		BIT(2)
60 
61 struct imx6_pcie_drvdata {
62 	enum imx6_pcie_variants variant;
63 	u32 flags;
64 	int dbi_length;
65 	const char *gpr;
66 };
67 
68 struct imx6_pcie {
69 	struct dw_pcie		*pci;
70 	int			reset_gpio;
71 	bool			gpio_active_high;
72 	bool			link_is_up;
73 	struct clk		*pcie_bus;
74 	struct clk		*pcie_phy;
75 	struct clk		*pcie_inbound_axi;
76 	struct clk		*pcie;
77 	struct clk		*pcie_aux;
78 	struct regmap		*iomuxc_gpr;
79 	u32			controller_id;
80 	struct reset_control	*pciephy_reset;
81 	struct reset_control	*apps_reset;
82 	struct reset_control	*turnoff_reset;
83 	u32			tx_deemph_gen1;
84 	u32			tx_deemph_gen2_3p5db;
85 	u32			tx_deemph_gen2_6db;
86 	u32			tx_swing_full;
87 	u32			tx_swing_low;
88 	struct regulator	*vpcie;
89 	struct regulator	*vph;
90 	void __iomem		*phy_base;
91 
92 	/* power domain for pcie */
93 	struct device		*pd_pcie;
94 	/* power domain for pcie phy */
95 	struct device		*pd_pcie_phy;
96 	struct phy		*phy;
97 	const struct imx6_pcie_drvdata *drvdata;
98 };
99 
100 /* Parameters for the waiting for PCIe PHY PLL to lock on i.MX7 */
101 #define PHY_PLL_LOCK_WAIT_USLEEP_MAX	200
102 #define PHY_PLL_LOCK_WAIT_TIMEOUT	(2000 * PHY_PLL_LOCK_WAIT_USLEEP_MAX)
103 
104 /* PCIe Port Logic registers (memory-mapped) */
105 #define PL_OFFSET 0x700
106 
107 #define PCIE_PHY_CTRL (PL_OFFSET + 0x114)
108 #define PCIE_PHY_CTRL_DATA(x)		FIELD_PREP(GENMASK(15, 0), (x))
109 #define PCIE_PHY_CTRL_CAP_ADR		BIT(16)
110 #define PCIE_PHY_CTRL_CAP_DAT		BIT(17)
111 #define PCIE_PHY_CTRL_WR		BIT(18)
112 #define PCIE_PHY_CTRL_RD		BIT(19)
113 
114 #define PCIE_PHY_STAT (PL_OFFSET + 0x110)
115 #define PCIE_PHY_STAT_ACK		BIT(16)
116 
117 /* PHY registers (not memory-mapped) */
118 #define PCIE_PHY_ATEOVRD			0x10
119 #define  PCIE_PHY_ATEOVRD_EN			BIT(2)
120 #define  PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT	0
121 #define  PCIE_PHY_ATEOVRD_REF_CLKDIV_MASK	0x1
122 
123 #define PCIE_PHY_MPLL_OVRD_IN_LO		0x11
124 #define  PCIE_PHY_MPLL_MULTIPLIER_SHIFT		2
125 #define  PCIE_PHY_MPLL_MULTIPLIER_MASK		0x7f
126 #define  PCIE_PHY_MPLL_MULTIPLIER_OVRD		BIT(9)
127 
128 #define PCIE_PHY_RX_ASIC_OUT 0x100D
129 #define PCIE_PHY_RX_ASIC_OUT_VALID	(1 << 0)
130 
131 /* iMX7 PCIe PHY registers */
132 #define PCIE_PHY_CMN_REG4		0x14
133 /* These are probably the bits that *aren't* DCC_FB_EN */
134 #define PCIE_PHY_CMN_REG4_DCC_FB_EN	0x29
135 
136 #define PCIE_PHY_CMN_REG15	        0x54
137 #define PCIE_PHY_CMN_REG15_DLY_4	BIT(2)
138 #define PCIE_PHY_CMN_REG15_PLL_PD	BIT(5)
139 #define PCIE_PHY_CMN_REG15_OVRD_PLL_PD	BIT(7)
140 
141 #define PCIE_PHY_CMN_REG24		0x90
142 #define PCIE_PHY_CMN_REG24_RX_EQ	BIT(6)
143 #define PCIE_PHY_CMN_REG24_RX_EQ_SEL	BIT(3)
144 
145 #define PCIE_PHY_CMN_REG26		0x98
146 #define PCIE_PHY_CMN_REG26_ATT_MODE	0xBC
147 
148 #define PHY_RX_OVRD_IN_LO 0x1005
149 #define PHY_RX_OVRD_IN_LO_RX_DATA_EN		BIT(5)
150 #define PHY_RX_OVRD_IN_LO_RX_PLL_EN		BIT(3)
151 
152 static unsigned int imx6_pcie_grp_offset(const struct imx6_pcie *imx6_pcie)
153 {
154 	WARN_ON(imx6_pcie->drvdata->variant != IMX8MQ &&
155 		imx6_pcie->drvdata->variant != IMX8MM &&
156 		imx6_pcie->drvdata->variant != IMX8MP);
157 	return imx6_pcie->controller_id == 1 ? IOMUXC_GPR16 : IOMUXC_GPR14;
158 }
159 
160 static void imx6_pcie_configure_type(struct imx6_pcie *imx6_pcie)
161 {
162 	unsigned int mask, val;
163 
164 	if (imx6_pcie->drvdata->variant == IMX8MQ &&
165 	    imx6_pcie->controller_id == 1) {
166 		mask = IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE;
167 		val  = FIELD_PREP(IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE,
168 				  PCI_EXP_TYPE_ROOT_PORT);
169 	} else {
170 		mask = IMX6Q_GPR12_DEVICE_TYPE;
171 		val  = FIELD_PREP(IMX6Q_GPR12_DEVICE_TYPE,
172 				  PCI_EXP_TYPE_ROOT_PORT);
173 	}
174 
175 	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, mask, val);
176 }
177 
178 static int pcie_phy_poll_ack(struct imx6_pcie *imx6_pcie, bool exp_val)
179 {
180 	struct dw_pcie *pci = imx6_pcie->pci;
181 	bool val;
182 	u32 max_iterations = 10;
183 	u32 wait_counter = 0;
184 
185 	do {
186 		val = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT) &
187 			PCIE_PHY_STAT_ACK;
188 		wait_counter++;
189 
190 		if (val == exp_val)
191 			return 0;
192 
193 		udelay(1);
194 	} while (wait_counter < max_iterations);
195 
196 	return -ETIMEDOUT;
197 }
198 
199 static int pcie_phy_wait_ack(struct imx6_pcie *imx6_pcie, int addr)
200 {
201 	struct dw_pcie *pci = imx6_pcie->pci;
202 	u32 val;
203 	int ret;
204 
205 	val = PCIE_PHY_CTRL_DATA(addr);
206 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val);
207 
208 	val |= PCIE_PHY_CTRL_CAP_ADR;
209 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val);
210 
211 	ret = pcie_phy_poll_ack(imx6_pcie, true);
212 	if (ret)
213 		return ret;
214 
215 	val = PCIE_PHY_CTRL_DATA(addr);
216 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val);
217 
218 	return pcie_phy_poll_ack(imx6_pcie, false);
219 }
220 
221 /* Read from the 16-bit PCIe PHY control registers (not memory-mapped) */
222 static int pcie_phy_read(struct imx6_pcie *imx6_pcie, int addr, u16 *data)
223 {
224 	struct dw_pcie *pci = imx6_pcie->pci;
225 	u32 phy_ctl;
226 	int ret;
227 
228 	ret = pcie_phy_wait_ack(imx6_pcie, addr);
229 	if (ret)
230 		return ret;
231 
232 	/* assert Read signal */
233 	phy_ctl = PCIE_PHY_CTRL_RD;
234 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, phy_ctl);
235 
236 	ret = pcie_phy_poll_ack(imx6_pcie, true);
237 	if (ret)
238 		return ret;
239 
240 	*data = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT);
241 
242 	/* deassert Read signal */
243 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, 0x00);
244 
245 	return pcie_phy_poll_ack(imx6_pcie, false);
246 }
247 
248 static int pcie_phy_write(struct imx6_pcie *imx6_pcie, int addr, u16 data)
249 {
250 	struct dw_pcie *pci = imx6_pcie->pci;
251 	u32 var;
252 	int ret;
253 
254 	/* write addr */
255 	/* cap addr */
256 	ret = pcie_phy_wait_ack(imx6_pcie, addr);
257 	if (ret)
258 		return ret;
259 
260 	var = PCIE_PHY_CTRL_DATA(data);
261 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
262 
263 	/* capture data */
264 	var |= PCIE_PHY_CTRL_CAP_DAT;
265 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
266 
267 	ret = pcie_phy_poll_ack(imx6_pcie, true);
268 	if (ret)
269 		return ret;
270 
271 	/* deassert cap data */
272 	var = PCIE_PHY_CTRL_DATA(data);
273 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
274 
275 	/* wait for ack de-assertion */
276 	ret = pcie_phy_poll_ack(imx6_pcie, false);
277 	if (ret)
278 		return ret;
279 
280 	/* assert wr signal */
281 	var = PCIE_PHY_CTRL_WR;
282 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
283 
284 	/* wait for ack */
285 	ret = pcie_phy_poll_ack(imx6_pcie, true);
286 	if (ret)
287 		return ret;
288 
289 	/* deassert wr signal */
290 	var = PCIE_PHY_CTRL_DATA(data);
291 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
292 
293 	/* wait for ack de-assertion */
294 	ret = pcie_phy_poll_ack(imx6_pcie, false);
295 	if (ret)
296 		return ret;
297 
298 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, 0x0);
299 
300 	return 0;
301 }
302 
303 static void imx6_pcie_init_phy(struct imx6_pcie *imx6_pcie)
304 {
305 	switch (imx6_pcie->drvdata->variant) {
306 	case IMX8MM:
307 	case IMX8MP:
308 		/*
309 		 * The PHY initialization had been done in the PHY
310 		 * driver, break here directly.
311 		 */
312 		break;
313 	case IMX8MQ:
314 		/*
315 		 * TODO: Currently this code assumes external
316 		 * oscillator is being used
317 		 */
318 		regmap_update_bits(imx6_pcie->iomuxc_gpr,
319 				   imx6_pcie_grp_offset(imx6_pcie),
320 				   IMX8MQ_GPR_PCIE_REF_USE_PAD,
321 				   IMX8MQ_GPR_PCIE_REF_USE_PAD);
322 		/*
323 		 * Regarding the datasheet, the PCIE_VPH is suggested
324 		 * to be 1.8V. If the PCIE_VPH is supplied by 3.3V, the
325 		 * VREG_BYPASS should be cleared to zero.
326 		 */
327 		if (imx6_pcie->vph &&
328 		    regulator_get_voltage(imx6_pcie->vph) > 3000000)
329 			regmap_update_bits(imx6_pcie->iomuxc_gpr,
330 					   imx6_pcie_grp_offset(imx6_pcie),
331 					   IMX8MQ_GPR_PCIE_VREG_BYPASS,
332 					   0);
333 		break;
334 	case IMX7D:
335 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
336 				   IMX7D_GPR12_PCIE_PHY_REFCLK_SEL, 0);
337 		break;
338 	case IMX6SX:
339 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
340 				   IMX6SX_GPR12_PCIE_RX_EQ_MASK,
341 				   IMX6SX_GPR12_PCIE_RX_EQ_2);
342 		fallthrough;
343 	default:
344 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
345 				   IMX6Q_GPR12_PCIE_CTL_2, 0 << 10);
346 
347 		/* configure constant input signal to the pcie ctrl and phy */
348 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
349 				   IMX6Q_GPR12_LOS_LEVEL, 9 << 4);
350 
351 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
352 				   IMX6Q_GPR8_TX_DEEMPH_GEN1,
353 				   imx6_pcie->tx_deemph_gen1 << 0);
354 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
355 				   IMX6Q_GPR8_TX_DEEMPH_GEN2_3P5DB,
356 				   imx6_pcie->tx_deemph_gen2_3p5db << 6);
357 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
358 				   IMX6Q_GPR8_TX_DEEMPH_GEN2_6DB,
359 				   imx6_pcie->tx_deemph_gen2_6db << 12);
360 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
361 				   IMX6Q_GPR8_TX_SWING_FULL,
362 				   imx6_pcie->tx_swing_full << 18);
363 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
364 				   IMX6Q_GPR8_TX_SWING_LOW,
365 				   imx6_pcie->tx_swing_low << 25);
366 		break;
367 	}
368 
369 	imx6_pcie_configure_type(imx6_pcie);
370 }
371 
372 static void imx7d_pcie_wait_for_phy_pll_lock(struct imx6_pcie *imx6_pcie)
373 {
374 	u32 val;
375 	struct device *dev = imx6_pcie->pci->dev;
376 
377 	if (regmap_read_poll_timeout(imx6_pcie->iomuxc_gpr,
378 				     IOMUXC_GPR22, val,
379 				     val & IMX7D_GPR22_PCIE_PHY_PLL_LOCKED,
380 				     PHY_PLL_LOCK_WAIT_USLEEP_MAX,
381 				     PHY_PLL_LOCK_WAIT_TIMEOUT))
382 		dev_err(dev, "PCIe PLL lock timeout\n");
383 }
384 
385 static int imx6_setup_phy_mpll(struct imx6_pcie *imx6_pcie)
386 {
387 	unsigned long phy_rate = clk_get_rate(imx6_pcie->pcie_phy);
388 	int mult, div;
389 	u16 val;
390 
391 	if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_IMX6_PHY))
392 		return 0;
393 
394 	switch (phy_rate) {
395 	case 125000000:
396 		/*
397 		 * The default settings of the MPLL are for a 125MHz input
398 		 * clock, so no need to reconfigure anything in that case.
399 		 */
400 		return 0;
401 	case 100000000:
402 		mult = 25;
403 		div = 0;
404 		break;
405 	case 200000000:
406 		mult = 25;
407 		div = 1;
408 		break;
409 	default:
410 		dev_err(imx6_pcie->pci->dev,
411 			"Unsupported PHY reference clock rate %lu\n", phy_rate);
412 		return -EINVAL;
413 	}
414 
415 	pcie_phy_read(imx6_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, &val);
416 	val &= ~(PCIE_PHY_MPLL_MULTIPLIER_MASK <<
417 		 PCIE_PHY_MPLL_MULTIPLIER_SHIFT);
418 	val |= mult << PCIE_PHY_MPLL_MULTIPLIER_SHIFT;
419 	val |= PCIE_PHY_MPLL_MULTIPLIER_OVRD;
420 	pcie_phy_write(imx6_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, val);
421 
422 	pcie_phy_read(imx6_pcie, PCIE_PHY_ATEOVRD, &val);
423 	val &= ~(PCIE_PHY_ATEOVRD_REF_CLKDIV_MASK <<
424 		 PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT);
425 	val |= div << PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT;
426 	val |= PCIE_PHY_ATEOVRD_EN;
427 	pcie_phy_write(imx6_pcie, PCIE_PHY_ATEOVRD, val);
428 
429 	return 0;
430 }
431 
432 static void imx6_pcie_reset_phy(struct imx6_pcie *imx6_pcie)
433 {
434 	u16 tmp;
435 
436 	if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_IMX6_PHY))
437 		return;
438 
439 	pcie_phy_read(imx6_pcie, PHY_RX_OVRD_IN_LO, &tmp);
440 	tmp |= (PHY_RX_OVRD_IN_LO_RX_DATA_EN |
441 		PHY_RX_OVRD_IN_LO_RX_PLL_EN);
442 	pcie_phy_write(imx6_pcie, PHY_RX_OVRD_IN_LO, tmp);
443 
444 	usleep_range(2000, 3000);
445 
446 	pcie_phy_read(imx6_pcie, PHY_RX_OVRD_IN_LO, &tmp);
447 	tmp &= ~(PHY_RX_OVRD_IN_LO_RX_DATA_EN |
448 		  PHY_RX_OVRD_IN_LO_RX_PLL_EN);
449 	pcie_phy_write(imx6_pcie, PHY_RX_OVRD_IN_LO, tmp);
450 }
451 
452 #ifdef CONFIG_ARM
453 /*  Added for PCI abort handling */
454 static int imx6q_pcie_abort_handler(unsigned long addr,
455 		unsigned int fsr, struct pt_regs *regs)
456 {
457 	unsigned long pc = instruction_pointer(regs);
458 	unsigned long instr = *(unsigned long *)pc;
459 	int reg = (instr >> 12) & 15;
460 
461 	/*
462 	 * If the instruction being executed was a read,
463 	 * make it look like it read all-ones.
464 	 */
465 	if ((instr & 0x0c100000) == 0x04100000) {
466 		unsigned long val;
467 
468 		if (instr & 0x00400000)
469 			val = 255;
470 		else
471 			val = -1;
472 
473 		regs->uregs[reg] = val;
474 		regs->ARM_pc += 4;
475 		return 0;
476 	}
477 
478 	if ((instr & 0x0e100090) == 0x00100090) {
479 		regs->uregs[reg] = -1;
480 		regs->ARM_pc += 4;
481 		return 0;
482 	}
483 
484 	return 1;
485 }
486 #endif
487 
488 static int imx6_pcie_attach_pd(struct device *dev)
489 {
490 	struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev);
491 	struct device_link *link;
492 
493 	/* Do nothing when in a single power domain */
494 	if (dev->pm_domain)
495 		return 0;
496 
497 	imx6_pcie->pd_pcie = dev_pm_domain_attach_by_name(dev, "pcie");
498 	if (IS_ERR(imx6_pcie->pd_pcie))
499 		return PTR_ERR(imx6_pcie->pd_pcie);
500 	/* Do nothing when power domain missing */
501 	if (!imx6_pcie->pd_pcie)
502 		return 0;
503 	link = device_link_add(dev, imx6_pcie->pd_pcie,
504 			DL_FLAG_STATELESS |
505 			DL_FLAG_PM_RUNTIME |
506 			DL_FLAG_RPM_ACTIVE);
507 	if (!link) {
508 		dev_err(dev, "Failed to add device_link to pcie pd.\n");
509 		return -EINVAL;
510 	}
511 
512 	imx6_pcie->pd_pcie_phy = dev_pm_domain_attach_by_name(dev, "pcie_phy");
513 	if (IS_ERR(imx6_pcie->pd_pcie_phy))
514 		return PTR_ERR(imx6_pcie->pd_pcie_phy);
515 
516 	link = device_link_add(dev, imx6_pcie->pd_pcie_phy,
517 			DL_FLAG_STATELESS |
518 			DL_FLAG_PM_RUNTIME |
519 			DL_FLAG_RPM_ACTIVE);
520 	if (!link) {
521 		dev_err(dev, "Failed to add device_link to pcie_phy pd.\n");
522 		return -EINVAL;
523 	}
524 
525 	return 0;
526 }
527 
528 static int imx6_pcie_enable_ref_clk(struct imx6_pcie *imx6_pcie)
529 {
530 	struct dw_pcie *pci = imx6_pcie->pci;
531 	struct device *dev = pci->dev;
532 	unsigned int offset;
533 	int ret = 0;
534 
535 	switch (imx6_pcie->drvdata->variant) {
536 	case IMX6SX:
537 		ret = clk_prepare_enable(imx6_pcie->pcie_inbound_axi);
538 		if (ret) {
539 			dev_err(dev, "unable to enable pcie_axi clock\n");
540 			break;
541 		}
542 
543 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
544 				   IMX6SX_GPR12_PCIE_TEST_POWERDOWN, 0);
545 		break;
546 	case IMX6QP:
547 	case IMX6Q:
548 		/* power up core phy and enable ref clock */
549 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
550 				   IMX6Q_GPR1_PCIE_TEST_PD, 0 << 18);
551 		/*
552 		 * the async reset input need ref clock to sync internally,
553 		 * when the ref clock comes after reset, internal synced
554 		 * reset time is too short, cannot meet the requirement.
555 		 * add one ~10us delay here.
556 		 */
557 		usleep_range(10, 100);
558 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
559 				   IMX6Q_GPR1_PCIE_REF_CLK_EN, 1 << 16);
560 		break;
561 	case IMX7D:
562 		break;
563 	case IMX8MM:
564 	case IMX8MQ:
565 	case IMX8MP:
566 		ret = clk_prepare_enable(imx6_pcie->pcie_aux);
567 		if (ret) {
568 			dev_err(dev, "unable to enable pcie_aux clock\n");
569 			break;
570 		}
571 
572 		offset = imx6_pcie_grp_offset(imx6_pcie);
573 		/*
574 		 * Set the over ride low and enabled
575 		 * make sure that REF_CLK is turned on.
576 		 */
577 		regmap_update_bits(imx6_pcie->iomuxc_gpr, offset,
578 				   IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE,
579 				   0);
580 		regmap_update_bits(imx6_pcie->iomuxc_gpr, offset,
581 				   IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN,
582 				   IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN);
583 		break;
584 	}
585 
586 	return ret;
587 }
588 
589 static void imx6_pcie_disable_ref_clk(struct imx6_pcie *imx6_pcie)
590 {
591 	switch (imx6_pcie->drvdata->variant) {
592 	case IMX6SX:
593 		clk_disable_unprepare(imx6_pcie->pcie_inbound_axi);
594 		break;
595 	case IMX6QP:
596 	case IMX6Q:
597 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
598 				IMX6Q_GPR1_PCIE_REF_CLK_EN, 0);
599 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
600 				IMX6Q_GPR1_PCIE_TEST_PD,
601 				IMX6Q_GPR1_PCIE_TEST_PD);
602 		break;
603 	case IMX7D:
604 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
605 				   IMX7D_GPR12_PCIE_PHY_REFCLK_SEL,
606 				   IMX7D_GPR12_PCIE_PHY_REFCLK_SEL);
607 		break;
608 	case IMX8MM:
609 	case IMX8MQ:
610 	case IMX8MP:
611 		clk_disable_unprepare(imx6_pcie->pcie_aux);
612 		break;
613 	default:
614 		break;
615 	}
616 }
617 
618 static int imx6_pcie_clk_enable(struct imx6_pcie *imx6_pcie)
619 {
620 	struct dw_pcie *pci = imx6_pcie->pci;
621 	struct device *dev = pci->dev;
622 	int ret;
623 
624 	ret = clk_prepare_enable(imx6_pcie->pcie_phy);
625 	if (ret) {
626 		dev_err(dev, "unable to enable pcie_phy clock\n");
627 		return ret;
628 	}
629 
630 	ret = clk_prepare_enable(imx6_pcie->pcie_bus);
631 	if (ret) {
632 		dev_err(dev, "unable to enable pcie_bus clock\n");
633 		goto err_pcie_bus;
634 	}
635 
636 	ret = clk_prepare_enable(imx6_pcie->pcie);
637 	if (ret) {
638 		dev_err(dev, "unable to enable pcie clock\n");
639 		goto err_pcie;
640 	}
641 
642 	ret = imx6_pcie_enable_ref_clk(imx6_pcie);
643 	if (ret) {
644 		dev_err(dev, "unable to enable pcie ref clock\n");
645 		goto err_ref_clk;
646 	}
647 
648 	/* allow the clocks to stabilize */
649 	usleep_range(200, 500);
650 	return 0;
651 
652 err_ref_clk:
653 	clk_disable_unprepare(imx6_pcie->pcie);
654 err_pcie:
655 	clk_disable_unprepare(imx6_pcie->pcie_bus);
656 err_pcie_bus:
657 	clk_disable_unprepare(imx6_pcie->pcie_phy);
658 
659 	return ret;
660 }
661 
662 static void imx6_pcie_clk_disable(struct imx6_pcie *imx6_pcie)
663 {
664 	imx6_pcie_disable_ref_clk(imx6_pcie);
665 	clk_disable_unprepare(imx6_pcie->pcie);
666 	clk_disable_unprepare(imx6_pcie->pcie_bus);
667 	clk_disable_unprepare(imx6_pcie->pcie_phy);
668 }
669 
670 static void imx6_pcie_assert_core_reset(struct imx6_pcie *imx6_pcie)
671 {
672 	switch (imx6_pcie->drvdata->variant) {
673 	case IMX7D:
674 	case IMX8MQ:
675 		reset_control_assert(imx6_pcie->pciephy_reset);
676 		fallthrough;
677 	case IMX8MM:
678 	case IMX8MP:
679 		reset_control_assert(imx6_pcie->apps_reset);
680 		break;
681 	case IMX6SX:
682 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
683 				   IMX6SX_GPR12_PCIE_TEST_POWERDOWN,
684 				   IMX6SX_GPR12_PCIE_TEST_POWERDOWN);
685 		/* Force PCIe PHY reset */
686 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR5,
687 				   IMX6SX_GPR5_PCIE_BTNRST_RESET,
688 				   IMX6SX_GPR5_PCIE_BTNRST_RESET);
689 		break;
690 	case IMX6QP:
691 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
692 				   IMX6Q_GPR1_PCIE_SW_RST,
693 				   IMX6Q_GPR1_PCIE_SW_RST);
694 		break;
695 	case IMX6Q:
696 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
697 				   IMX6Q_GPR1_PCIE_TEST_PD, 1 << 18);
698 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
699 				   IMX6Q_GPR1_PCIE_REF_CLK_EN, 0 << 16);
700 		break;
701 	}
702 
703 	/* Some boards don't have PCIe reset GPIO. */
704 	if (gpio_is_valid(imx6_pcie->reset_gpio))
705 		gpio_set_value_cansleep(imx6_pcie->reset_gpio,
706 					imx6_pcie->gpio_active_high);
707 }
708 
709 static int imx6_pcie_deassert_core_reset(struct imx6_pcie *imx6_pcie)
710 {
711 	struct dw_pcie *pci = imx6_pcie->pci;
712 	struct device *dev = pci->dev;
713 
714 	switch (imx6_pcie->drvdata->variant) {
715 	case IMX8MQ:
716 		reset_control_deassert(imx6_pcie->pciephy_reset);
717 		break;
718 	case IMX7D:
719 		reset_control_deassert(imx6_pcie->pciephy_reset);
720 
721 		/* Workaround for ERR010728, failure of PCI-e PLL VCO to
722 		 * oscillate, especially when cold.  This turns off "Duty-cycle
723 		 * Corrector" and other mysterious undocumented things.
724 		 */
725 		if (likely(imx6_pcie->phy_base)) {
726 			/* De-assert DCC_FB_EN */
727 			writel(PCIE_PHY_CMN_REG4_DCC_FB_EN,
728 			       imx6_pcie->phy_base + PCIE_PHY_CMN_REG4);
729 			/* Assert RX_EQS and RX_EQS_SEL */
730 			writel(PCIE_PHY_CMN_REG24_RX_EQ_SEL
731 				| PCIE_PHY_CMN_REG24_RX_EQ,
732 			       imx6_pcie->phy_base + PCIE_PHY_CMN_REG24);
733 			/* Assert ATT_MODE */
734 			writel(PCIE_PHY_CMN_REG26_ATT_MODE,
735 			       imx6_pcie->phy_base + PCIE_PHY_CMN_REG26);
736 		} else {
737 			dev_warn(dev, "Unable to apply ERR010728 workaround. DT missing fsl,imx7d-pcie-phy phandle ?\n");
738 		}
739 
740 		imx7d_pcie_wait_for_phy_pll_lock(imx6_pcie);
741 		break;
742 	case IMX6SX:
743 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR5,
744 				   IMX6SX_GPR5_PCIE_BTNRST_RESET, 0);
745 		break;
746 	case IMX6QP:
747 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
748 				   IMX6Q_GPR1_PCIE_SW_RST, 0);
749 
750 		usleep_range(200, 500);
751 		break;
752 	case IMX6Q:		/* Nothing to do */
753 	case IMX8MM:
754 	case IMX8MP:
755 		break;
756 	}
757 
758 	/* Some boards don't have PCIe reset GPIO. */
759 	if (gpio_is_valid(imx6_pcie->reset_gpio)) {
760 		msleep(100);
761 		gpio_set_value_cansleep(imx6_pcie->reset_gpio,
762 					!imx6_pcie->gpio_active_high);
763 		/* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */
764 		msleep(100);
765 	}
766 
767 	return 0;
768 }
769 
770 static int imx6_pcie_wait_for_speed_change(struct imx6_pcie *imx6_pcie)
771 {
772 	struct dw_pcie *pci = imx6_pcie->pci;
773 	struct device *dev = pci->dev;
774 	u32 tmp;
775 	unsigned int retries;
776 
777 	for (retries = 0; retries < 200; retries++) {
778 		tmp = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
779 		/* Test if the speed change finished. */
780 		if (!(tmp & PORT_LOGIC_SPEED_CHANGE))
781 			return 0;
782 		usleep_range(100, 1000);
783 	}
784 
785 	dev_err(dev, "Speed change timeout\n");
786 	return -ETIMEDOUT;
787 }
788 
789 static void imx6_pcie_ltssm_enable(struct device *dev)
790 {
791 	struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev);
792 
793 	switch (imx6_pcie->drvdata->variant) {
794 	case IMX6Q:
795 	case IMX6SX:
796 	case IMX6QP:
797 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
798 				   IMX6Q_GPR12_PCIE_CTL_2,
799 				   IMX6Q_GPR12_PCIE_CTL_2);
800 		break;
801 	case IMX7D:
802 	case IMX8MQ:
803 	case IMX8MM:
804 	case IMX8MP:
805 		reset_control_deassert(imx6_pcie->apps_reset);
806 		break;
807 	}
808 }
809 
810 static void imx6_pcie_ltssm_disable(struct device *dev)
811 {
812 	struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev);
813 
814 	switch (imx6_pcie->drvdata->variant) {
815 	case IMX6Q:
816 	case IMX6SX:
817 	case IMX6QP:
818 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
819 				   IMX6Q_GPR12_PCIE_CTL_2, 0);
820 		break;
821 	case IMX7D:
822 	case IMX8MQ:
823 	case IMX8MM:
824 	case IMX8MP:
825 		reset_control_assert(imx6_pcie->apps_reset);
826 		break;
827 	}
828 }
829 
830 static int imx6_pcie_start_link(struct dw_pcie *pci)
831 {
832 	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci);
833 	struct device *dev = pci->dev;
834 	u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
835 	u32 tmp;
836 	int ret;
837 
838 	/*
839 	 * Force Gen1 operation when starting the link.  In case the link is
840 	 * started in Gen2 mode, there is a possibility the devices on the
841 	 * bus will not be detected at all.  This happens with PCIe switches.
842 	 */
843 	dw_pcie_dbi_ro_wr_en(pci);
844 	tmp = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP);
845 	tmp &= ~PCI_EXP_LNKCAP_SLS;
846 	tmp |= PCI_EXP_LNKCAP_SLS_2_5GB;
847 	dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, tmp);
848 	dw_pcie_dbi_ro_wr_dis(pci);
849 
850 	/* Start LTSSM. */
851 	imx6_pcie_ltssm_enable(dev);
852 
853 	ret = dw_pcie_wait_for_link(pci);
854 	if (ret)
855 		goto err_reset_phy;
856 
857 	if (pci->link_gen > 1) {
858 		/* Allow faster modes after the link is up */
859 		dw_pcie_dbi_ro_wr_en(pci);
860 		tmp = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP);
861 		tmp &= ~PCI_EXP_LNKCAP_SLS;
862 		tmp |= pci->link_gen;
863 		dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, tmp);
864 
865 		/*
866 		 * Start Directed Speed Change so the best possible
867 		 * speed both link partners support can be negotiated.
868 		 */
869 		tmp = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
870 		tmp |= PORT_LOGIC_SPEED_CHANGE;
871 		dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, tmp);
872 		dw_pcie_dbi_ro_wr_dis(pci);
873 
874 		if (imx6_pcie->drvdata->flags &
875 		    IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE) {
876 			/*
877 			 * On i.MX7, DIRECT_SPEED_CHANGE behaves differently
878 			 * from i.MX6 family when no link speed transition
879 			 * occurs and we go Gen1 -> yep, Gen1. The difference
880 			 * is that, in such case, it will not be cleared by HW
881 			 * which will cause the following code to report false
882 			 * failure.
883 			 */
884 
885 			ret = imx6_pcie_wait_for_speed_change(imx6_pcie);
886 			if (ret) {
887 				dev_err(dev, "Failed to bring link up!\n");
888 				goto err_reset_phy;
889 			}
890 		}
891 
892 		/* Make sure link training is finished as well! */
893 		ret = dw_pcie_wait_for_link(pci);
894 		if (ret)
895 			goto err_reset_phy;
896 	} else {
897 		dev_info(dev, "Link: Only Gen1 is enabled\n");
898 	}
899 
900 	imx6_pcie->link_is_up = true;
901 	tmp = dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKSTA);
902 	dev_info(dev, "Link up, Gen%i\n", tmp & PCI_EXP_LNKSTA_CLS);
903 	return 0;
904 
905 err_reset_phy:
906 	imx6_pcie->link_is_up = false;
907 	dev_dbg(dev, "PHY DEBUG_R0=0x%08x DEBUG_R1=0x%08x\n",
908 		dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0),
909 		dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG1));
910 	imx6_pcie_reset_phy(imx6_pcie);
911 	return 0;
912 }
913 
914 static void imx6_pcie_stop_link(struct dw_pcie *pci)
915 {
916 	struct device *dev = pci->dev;
917 
918 	/* Turn off PCIe LTSSM */
919 	imx6_pcie_ltssm_disable(dev);
920 }
921 
922 static int imx6_pcie_host_init(struct dw_pcie_rp *pp)
923 {
924 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
925 	struct device *dev = pci->dev;
926 	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci);
927 	int ret;
928 
929 	if (imx6_pcie->vpcie) {
930 		ret = regulator_enable(imx6_pcie->vpcie);
931 		if (ret) {
932 			dev_err(dev, "failed to enable vpcie regulator: %d\n",
933 				ret);
934 			return ret;
935 		}
936 	}
937 
938 	imx6_pcie_assert_core_reset(imx6_pcie);
939 	imx6_pcie_init_phy(imx6_pcie);
940 
941 	ret = imx6_pcie_clk_enable(imx6_pcie);
942 	if (ret) {
943 		dev_err(dev, "unable to enable pcie clocks: %d\n", ret);
944 		goto err_reg_disable;
945 	}
946 
947 	if (imx6_pcie->phy) {
948 		ret = phy_init(imx6_pcie->phy);
949 		if (ret) {
950 			dev_err(dev, "pcie PHY power up failed\n");
951 			goto err_clk_disable;
952 		}
953 	}
954 
955 	ret = imx6_pcie_deassert_core_reset(imx6_pcie);
956 	if (ret < 0) {
957 		dev_err(dev, "pcie deassert core reset failed: %d\n", ret);
958 		goto err_phy_off;
959 	}
960 
961 	if (imx6_pcie->phy) {
962 		ret = phy_power_on(imx6_pcie->phy);
963 		if (ret) {
964 			dev_err(dev, "waiting for PHY ready timeout!\n");
965 			goto err_phy_off;
966 		}
967 	}
968 	imx6_setup_phy_mpll(imx6_pcie);
969 
970 	return 0;
971 
972 err_phy_off:
973 	if (imx6_pcie->phy)
974 		phy_exit(imx6_pcie->phy);
975 err_clk_disable:
976 	imx6_pcie_clk_disable(imx6_pcie);
977 err_reg_disable:
978 	if (imx6_pcie->vpcie)
979 		regulator_disable(imx6_pcie->vpcie);
980 	return ret;
981 }
982 
983 static void imx6_pcie_host_exit(struct dw_pcie_rp *pp)
984 {
985 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
986 	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci);
987 
988 	if (imx6_pcie->phy) {
989 		if (phy_power_off(imx6_pcie->phy))
990 			dev_err(pci->dev, "unable to power off PHY\n");
991 		phy_exit(imx6_pcie->phy);
992 	}
993 	imx6_pcie_clk_disable(imx6_pcie);
994 
995 	if (imx6_pcie->vpcie)
996 		regulator_disable(imx6_pcie->vpcie);
997 }
998 
999 static const struct dw_pcie_host_ops imx6_pcie_host_ops = {
1000 	.host_init = imx6_pcie_host_init,
1001 };
1002 
1003 static const struct dw_pcie_ops dw_pcie_ops = {
1004 	.start_link = imx6_pcie_start_link,
1005 };
1006 
1007 static void imx6_pcie_pm_turnoff(struct imx6_pcie *imx6_pcie)
1008 {
1009 	struct device *dev = imx6_pcie->pci->dev;
1010 
1011 	/* Some variants have a turnoff reset in DT */
1012 	if (imx6_pcie->turnoff_reset) {
1013 		reset_control_assert(imx6_pcie->turnoff_reset);
1014 		reset_control_deassert(imx6_pcie->turnoff_reset);
1015 		goto pm_turnoff_sleep;
1016 	}
1017 
1018 	/* Others poke directly at IOMUXC registers */
1019 	switch (imx6_pcie->drvdata->variant) {
1020 	case IMX6SX:
1021 	case IMX6QP:
1022 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
1023 				IMX6SX_GPR12_PCIE_PM_TURN_OFF,
1024 				IMX6SX_GPR12_PCIE_PM_TURN_OFF);
1025 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
1026 				IMX6SX_GPR12_PCIE_PM_TURN_OFF, 0);
1027 		break;
1028 	default:
1029 		dev_err(dev, "PME_Turn_Off not implemented\n");
1030 		return;
1031 	}
1032 
1033 	/*
1034 	 * Components with an upstream port must respond to
1035 	 * PME_Turn_Off with PME_TO_Ack but we can't check.
1036 	 *
1037 	 * The standard recommends a 1-10ms timeout after which to
1038 	 * proceed anyway as if acks were received.
1039 	 */
1040 pm_turnoff_sleep:
1041 	usleep_range(1000, 10000);
1042 }
1043 
1044 static int imx6_pcie_suspend_noirq(struct device *dev)
1045 {
1046 	struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev);
1047 	struct dw_pcie_rp *pp = &imx6_pcie->pci->pp;
1048 
1049 	if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_SUPPORTS_SUSPEND))
1050 		return 0;
1051 
1052 	imx6_pcie_pm_turnoff(imx6_pcie);
1053 	imx6_pcie_stop_link(imx6_pcie->pci);
1054 	imx6_pcie_host_exit(pp);
1055 
1056 	return 0;
1057 }
1058 
1059 static int imx6_pcie_resume_noirq(struct device *dev)
1060 {
1061 	int ret;
1062 	struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev);
1063 	struct dw_pcie_rp *pp = &imx6_pcie->pci->pp;
1064 
1065 	if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_SUPPORTS_SUSPEND))
1066 		return 0;
1067 
1068 	ret = imx6_pcie_host_init(pp);
1069 	if (ret)
1070 		return ret;
1071 	dw_pcie_setup_rc(pp);
1072 
1073 	if (imx6_pcie->link_is_up)
1074 		imx6_pcie_start_link(imx6_pcie->pci);
1075 
1076 	return 0;
1077 }
1078 
1079 static const struct dev_pm_ops imx6_pcie_pm_ops = {
1080 	NOIRQ_SYSTEM_SLEEP_PM_OPS(imx6_pcie_suspend_noirq,
1081 				  imx6_pcie_resume_noirq)
1082 };
1083 
1084 static int imx6_pcie_probe(struct platform_device *pdev)
1085 {
1086 	struct device *dev = &pdev->dev;
1087 	struct dw_pcie *pci;
1088 	struct imx6_pcie *imx6_pcie;
1089 	struct device_node *np;
1090 	struct resource *dbi_base;
1091 	struct device_node *node = dev->of_node;
1092 	int ret;
1093 	u16 val;
1094 
1095 	imx6_pcie = devm_kzalloc(dev, sizeof(*imx6_pcie), GFP_KERNEL);
1096 	if (!imx6_pcie)
1097 		return -ENOMEM;
1098 
1099 	pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
1100 	if (!pci)
1101 		return -ENOMEM;
1102 
1103 	pci->dev = dev;
1104 	pci->ops = &dw_pcie_ops;
1105 	pci->pp.ops = &imx6_pcie_host_ops;
1106 
1107 	imx6_pcie->pci = pci;
1108 	imx6_pcie->drvdata = of_device_get_match_data(dev);
1109 
1110 	/* Find the PHY if one is defined, only imx7d uses it */
1111 	np = of_parse_phandle(node, "fsl,imx7d-pcie-phy", 0);
1112 	if (np) {
1113 		struct resource res;
1114 
1115 		ret = of_address_to_resource(np, 0, &res);
1116 		if (ret) {
1117 			dev_err(dev, "Unable to map PCIe PHY\n");
1118 			return ret;
1119 		}
1120 		imx6_pcie->phy_base = devm_ioremap_resource(dev, &res);
1121 		if (IS_ERR(imx6_pcie->phy_base))
1122 			return PTR_ERR(imx6_pcie->phy_base);
1123 	}
1124 
1125 	dbi_base = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1126 	pci->dbi_base = devm_ioremap_resource(dev, dbi_base);
1127 	if (IS_ERR(pci->dbi_base))
1128 		return PTR_ERR(pci->dbi_base);
1129 
1130 	/* Fetch GPIOs */
1131 	imx6_pcie->reset_gpio = of_get_named_gpio(node, "reset-gpio", 0);
1132 	imx6_pcie->gpio_active_high = of_property_read_bool(node,
1133 						"reset-gpio-active-high");
1134 	if (gpio_is_valid(imx6_pcie->reset_gpio)) {
1135 		ret = devm_gpio_request_one(dev, imx6_pcie->reset_gpio,
1136 				imx6_pcie->gpio_active_high ?
1137 					GPIOF_OUT_INIT_HIGH :
1138 					GPIOF_OUT_INIT_LOW,
1139 				"PCIe reset");
1140 		if (ret) {
1141 			dev_err(dev, "unable to get reset gpio\n");
1142 			return ret;
1143 		}
1144 	} else if (imx6_pcie->reset_gpio == -EPROBE_DEFER) {
1145 		return imx6_pcie->reset_gpio;
1146 	}
1147 
1148 	/* Fetch clocks */
1149 	imx6_pcie->pcie_bus = devm_clk_get(dev, "pcie_bus");
1150 	if (IS_ERR(imx6_pcie->pcie_bus))
1151 		return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_bus),
1152 				     "pcie_bus clock source missing or invalid\n");
1153 
1154 	imx6_pcie->pcie = devm_clk_get(dev, "pcie");
1155 	if (IS_ERR(imx6_pcie->pcie))
1156 		return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie),
1157 				     "pcie clock source missing or invalid\n");
1158 
1159 	switch (imx6_pcie->drvdata->variant) {
1160 	case IMX6SX:
1161 		imx6_pcie->pcie_inbound_axi = devm_clk_get(dev,
1162 							   "pcie_inbound_axi");
1163 		if (IS_ERR(imx6_pcie->pcie_inbound_axi))
1164 			return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_inbound_axi),
1165 					     "pcie_inbound_axi clock missing or invalid\n");
1166 		break;
1167 	case IMX8MQ:
1168 		imx6_pcie->pcie_aux = devm_clk_get(dev, "pcie_aux");
1169 		if (IS_ERR(imx6_pcie->pcie_aux))
1170 			return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_aux),
1171 					     "pcie_aux clock source missing or invalid\n");
1172 		fallthrough;
1173 	case IMX7D:
1174 		if (dbi_base->start == IMX8MQ_PCIE2_BASE_ADDR)
1175 			imx6_pcie->controller_id = 1;
1176 
1177 		imx6_pcie->pciephy_reset = devm_reset_control_get_exclusive(dev,
1178 									    "pciephy");
1179 		if (IS_ERR(imx6_pcie->pciephy_reset)) {
1180 			dev_err(dev, "Failed to get PCIEPHY reset control\n");
1181 			return PTR_ERR(imx6_pcie->pciephy_reset);
1182 		}
1183 
1184 		imx6_pcie->apps_reset = devm_reset_control_get_exclusive(dev,
1185 									 "apps");
1186 		if (IS_ERR(imx6_pcie->apps_reset)) {
1187 			dev_err(dev, "Failed to get PCIE APPS reset control\n");
1188 			return PTR_ERR(imx6_pcie->apps_reset);
1189 		}
1190 		break;
1191 	case IMX8MM:
1192 	case IMX8MP:
1193 		imx6_pcie->pcie_aux = devm_clk_get(dev, "pcie_aux");
1194 		if (IS_ERR(imx6_pcie->pcie_aux))
1195 			return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_aux),
1196 					     "pcie_aux clock source missing or invalid\n");
1197 		imx6_pcie->apps_reset = devm_reset_control_get_exclusive(dev,
1198 									 "apps");
1199 		if (IS_ERR(imx6_pcie->apps_reset))
1200 			return dev_err_probe(dev, PTR_ERR(imx6_pcie->apps_reset),
1201 					     "failed to get pcie apps reset control\n");
1202 
1203 		imx6_pcie->phy = devm_phy_get(dev, "pcie-phy");
1204 		if (IS_ERR(imx6_pcie->phy))
1205 			return dev_err_probe(dev, PTR_ERR(imx6_pcie->phy),
1206 					     "failed to get pcie phy\n");
1207 
1208 		break;
1209 	default:
1210 		break;
1211 	}
1212 	/* Don't fetch the pcie_phy clock, if it has abstract PHY driver */
1213 	if (imx6_pcie->phy == NULL) {
1214 		imx6_pcie->pcie_phy = devm_clk_get(dev, "pcie_phy");
1215 		if (IS_ERR(imx6_pcie->pcie_phy))
1216 			return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_phy),
1217 					     "pcie_phy clock source missing or invalid\n");
1218 	}
1219 
1220 
1221 	/* Grab turnoff reset */
1222 	imx6_pcie->turnoff_reset = devm_reset_control_get_optional_exclusive(dev, "turnoff");
1223 	if (IS_ERR(imx6_pcie->turnoff_reset)) {
1224 		dev_err(dev, "Failed to get TURNOFF reset control\n");
1225 		return PTR_ERR(imx6_pcie->turnoff_reset);
1226 	}
1227 
1228 	/* Grab GPR config register range */
1229 	imx6_pcie->iomuxc_gpr =
1230 		 syscon_regmap_lookup_by_compatible(imx6_pcie->drvdata->gpr);
1231 	if (IS_ERR(imx6_pcie->iomuxc_gpr)) {
1232 		dev_err(dev, "unable to find iomuxc registers\n");
1233 		return PTR_ERR(imx6_pcie->iomuxc_gpr);
1234 	}
1235 
1236 	/* Grab PCIe PHY Tx Settings */
1237 	if (of_property_read_u32(node, "fsl,tx-deemph-gen1",
1238 				 &imx6_pcie->tx_deemph_gen1))
1239 		imx6_pcie->tx_deemph_gen1 = 0;
1240 
1241 	if (of_property_read_u32(node, "fsl,tx-deemph-gen2-3p5db",
1242 				 &imx6_pcie->tx_deemph_gen2_3p5db))
1243 		imx6_pcie->tx_deemph_gen2_3p5db = 0;
1244 
1245 	if (of_property_read_u32(node, "fsl,tx-deemph-gen2-6db",
1246 				 &imx6_pcie->tx_deemph_gen2_6db))
1247 		imx6_pcie->tx_deemph_gen2_6db = 20;
1248 
1249 	if (of_property_read_u32(node, "fsl,tx-swing-full",
1250 				 &imx6_pcie->tx_swing_full))
1251 		imx6_pcie->tx_swing_full = 127;
1252 
1253 	if (of_property_read_u32(node, "fsl,tx-swing-low",
1254 				 &imx6_pcie->tx_swing_low))
1255 		imx6_pcie->tx_swing_low = 127;
1256 
1257 	/* Limit link speed */
1258 	pci->link_gen = 1;
1259 	of_property_read_u32(node, "fsl,max-link-speed", &pci->link_gen);
1260 
1261 	imx6_pcie->vpcie = devm_regulator_get_optional(&pdev->dev, "vpcie");
1262 	if (IS_ERR(imx6_pcie->vpcie)) {
1263 		if (PTR_ERR(imx6_pcie->vpcie) != -ENODEV)
1264 			return PTR_ERR(imx6_pcie->vpcie);
1265 		imx6_pcie->vpcie = NULL;
1266 	}
1267 
1268 	imx6_pcie->vph = devm_regulator_get_optional(&pdev->dev, "vph");
1269 	if (IS_ERR(imx6_pcie->vph)) {
1270 		if (PTR_ERR(imx6_pcie->vph) != -ENODEV)
1271 			return PTR_ERR(imx6_pcie->vph);
1272 		imx6_pcie->vph = NULL;
1273 	}
1274 
1275 	platform_set_drvdata(pdev, imx6_pcie);
1276 
1277 	ret = imx6_pcie_attach_pd(dev);
1278 	if (ret)
1279 		return ret;
1280 
1281 	ret = dw_pcie_host_init(&pci->pp);
1282 	if (ret < 0)
1283 		return ret;
1284 
1285 	if (pci_msi_enabled()) {
1286 		u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_MSI);
1287 		val = dw_pcie_readw_dbi(pci, offset + PCI_MSI_FLAGS);
1288 		val |= PCI_MSI_FLAGS_ENABLE;
1289 		dw_pcie_writew_dbi(pci, offset + PCI_MSI_FLAGS, val);
1290 	}
1291 
1292 	return 0;
1293 }
1294 
1295 static void imx6_pcie_shutdown(struct platform_device *pdev)
1296 {
1297 	struct imx6_pcie *imx6_pcie = platform_get_drvdata(pdev);
1298 
1299 	/* bring down link, so bootloader gets clean state in case of reboot */
1300 	imx6_pcie_assert_core_reset(imx6_pcie);
1301 }
1302 
1303 static const struct imx6_pcie_drvdata drvdata[] = {
1304 	[IMX6Q] = {
1305 		.variant = IMX6Q,
1306 		.flags = IMX6_PCIE_FLAG_IMX6_PHY |
1307 			 IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE,
1308 		.dbi_length = 0x200,
1309 		.gpr = "fsl,imx6q-iomuxc-gpr",
1310 	},
1311 	[IMX6SX] = {
1312 		.variant = IMX6SX,
1313 		.flags = IMX6_PCIE_FLAG_IMX6_PHY |
1314 			 IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE |
1315 			 IMX6_PCIE_FLAG_SUPPORTS_SUSPEND,
1316 		.gpr = "fsl,imx6q-iomuxc-gpr",
1317 	},
1318 	[IMX6QP] = {
1319 		.variant = IMX6QP,
1320 		.flags = IMX6_PCIE_FLAG_IMX6_PHY |
1321 			 IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE |
1322 			 IMX6_PCIE_FLAG_SUPPORTS_SUSPEND,
1323 		.dbi_length = 0x200,
1324 		.gpr = "fsl,imx6q-iomuxc-gpr",
1325 	},
1326 	[IMX7D] = {
1327 		.variant = IMX7D,
1328 		.flags = IMX6_PCIE_FLAG_SUPPORTS_SUSPEND,
1329 		.gpr = "fsl,imx7d-iomuxc-gpr",
1330 	},
1331 	[IMX8MQ] = {
1332 		.variant = IMX8MQ,
1333 		.gpr = "fsl,imx8mq-iomuxc-gpr",
1334 	},
1335 	[IMX8MM] = {
1336 		.variant = IMX8MM,
1337 		.flags = IMX6_PCIE_FLAG_SUPPORTS_SUSPEND,
1338 		.gpr = "fsl,imx8mm-iomuxc-gpr",
1339 	},
1340 	[IMX8MP] = {
1341 		.variant = IMX8MP,
1342 		.flags = IMX6_PCIE_FLAG_SUPPORTS_SUSPEND,
1343 		.gpr = "fsl,imx8mp-iomuxc-gpr",
1344 	},
1345 };
1346 
1347 static const struct of_device_id imx6_pcie_of_match[] = {
1348 	{ .compatible = "fsl,imx6q-pcie",  .data = &drvdata[IMX6Q],  },
1349 	{ .compatible = "fsl,imx6sx-pcie", .data = &drvdata[IMX6SX], },
1350 	{ .compatible = "fsl,imx6qp-pcie", .data = &drvdata[IMX6QP], },
1351 	{ .compatible = "fsl,imx7d-pcie",  .data = &drvdata[IMX7D],  },
1352 	{ .compatible = "fsl,imx8mq-pcie", .data = &drvdata[IMX8MQ], },
1353 	{ .compatible = "fsl,imx8mm-pcie", .data = &drvdata[IMX8MM], },
1354 	{ .compatible = "fsl,imx8mp-pcie", .data = &drvdata[IMX8MP], },
1355 	{},
1356 };
1357 
1358 static struct platform_driver imx6_pcie_driver = {
1359 	.driver = {
1360 		.name	= "imx6q-pcie",
1361 		.of_match_table = imx6_pcie_of_match,
1362 		.suppress_bind_attrs = true,
1363 		.pm = &imx6_pcie_pm_ops,
1364 		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
1365 	},
1366 	.probe    = imx6_pcie_probe,
1367 	.shutdown = imx6_pcie_shutdown,
1368 };
1369 
1370 static void imx6_pcie_quirk(struct pci_dev *dev)
1371 {
1372 	struct pci_bus *bus = dev->bus;
1373 	struct dw_pcie_rp *pp = bus->sysdata;
1374 
1375 	/* Bus parent is the PCI bridge, its parent is this platform driver */
1376 	if (!bus->dev.parent || !bus->dev.parent->parent)
1377 		return;
1378 
1379 	/* Make sure we only quirk devices associated with this driver */
1380 	if (bus->dev.parent->parent->driver != &imx6_pcie_driver.driver)
1381 		return;
1382 
1383 	if (pci_is_root_bus(bus)) {
1384 		struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1385 		struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci);
1386 
1387 		/*
1388 		 * Limit config length to avoid the kernel reading beyond
1389 		 * the register set and causing an abort on i.MX 6Quad
1390 		 */
1391 		if (imx6_pcie->drvdata->dbi_length) {
1392 			dev->cfg_size = imx6_pcie->drvdata->dbi_length;
1393 			dev_info(&dev->dev, "Limiting cfg_size to %d\n",
1394 					dev->cfg_size);
1395 		}
1396 	}
1397 }
1398 DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_VENDOR_ID_SYNOPSYS, 0xabcd,
1399 			PCI_CLASS_BRIDGE_PCI, 8, imx6_pcie_quirk);
1400 
1401 static int __init imx6_pcie_init(void)
1402 {
1403 #ifdef CONFIG_ARM
1404 	/*
1405 	 * Since probe() can be deferred we need to make sure that
1406 	 * hook_fault_code is not called after __init memory is freed
1407 	 * by kernel and since imx6q_pcie_abort_handler() is a no-op,
1408 	 * we can install the handler here without risking it
1409 	 * accessing some uninitialized driver state.
1410 	 */
1411 	hook_fault_code(8, imx6q_pcie_abort_handler, SIGBUS, 0,
1412 			"external abort on non-linefetch");
1413 #endif
1414 
1415 	return platform_driver_register(&imx6_pcie_driver);
1416 }
1417 device_initcall(imx6_pcie_init);
1418