1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * PCIe host controller driver for Freescale i.MX6 SoCs
4  *
5  * Copyright (C) 2013 Kosagi
6  *		https://www.kosagi.com
7  *
8  * Author: Sean Cross <xobs@kosagi.com>
9  */
10 
11 #include <linux/bitfield.h>
12 #include <linux/clk.h>
13 #include <linux/delay.h>
14 #include <linux/gpio.h>
15 #include <linux/kernel.h>
16 #include <linux/mfd/syscon.h>
17 #include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
18 #include <linux/mfd/syscon/imx7-iomuxc-gpr.h>
19 #include <linux/module.h>
20 #include <linux/of_gpio.h>
21 #include <linux/of_device.h>
22 #include <linux/of_address.h>
23 #include <linux/pci.h>
24 #include <linux/platform_device.h>
25 #include <linux/regmap.h>
26 #include <linux/regulator/consumer.h>
27 #include <linux/resource.h>
28 #include <linux/signal.h>
29 #include <linux/types.h>
30 #include <linux/interrupt.h>
31 #include <linux/reset.h>
32 #include <linux/phy/phy.h>
33 #include <linux/pm_domain.h>
34 #include <linux/pm_runtime.h>
35 
36 #include "pcie-designware.h"
37 
38 #define IMX8MQ_GPR_PCIE_REF_USE_PAD		BIT(9)
39 #define IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN	BIT(10)
40 #define IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE	BIT(11)
41 #define IMX8MQ_GPR_PCIE_VREG_BYPASS		BIT(12)
42 #define IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE	GENMASK(11, 8)
43 #define IMX8MQ_PCIE2_BASE_ADDR			0x33c00000
44 
45 #define to_imx6_pcie(x)	dev_get_drvdata((x)->dev)
46 
47 enum imx6_pcie_variants {
48 	IMX6Q,
49 	IMX6SX,
50 	IMX6QP,
51 	IMX7D,
52 	IMX8MQ,
53 	IMX8MM,
54 };
55 
56 #define IMX6_PCIE_FLAG_IMX6_PHY			BIT(0)
57 #define IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE	BIT(1)
58 #define IMX6_PCIE_FLAG_SUPPORTS_SUSPEND		BIT(2)
59 
60 struct imx6_pcie_drvdata {
61 	enum imx6_pcie_variants variant;
62 	u32 flags;
63 	int dbi_length;
64 };
65 
66 struct imx6_pcie {
67 	struct dw_pcie		*pci;
68 	int			reset_gpio;
69 	bool			gpio_active_high;
70 	struct clk		*pcie_bus;
71 	struct clk		*pcie_phy;
72 	struct clk		*pcie_inbound_axi;
73 	struct clk		*pcie;
74 	struct clk		*pcie_aux;
75 	struct regmap		*iomuxc_gpr;
76 	u32			controller_id;
77 	struct reset_control	*pciephy_reset;
78 	struct reset_control	*apps_reset;
79 	struct reset_control	*turnoff_reset;
80 	u32			tx_deemph_gen1;
81 	u32			tx_deemph_gen2_3p5db;
82 	u32			tx_deemph_gen2_6db;
83 	u32			tx_swing_full;
84 	u32			tx_swing_low;
85 	struct regulator	*vpcie;
86 	struct regulator	*vph;
87 	void __iomem		*phy_base;
88 
89 	/* power domain for pcie */
90 	struct device		*pd_pcie;
91 	/* power domain for pcie phy */
92 	struct device		*pd_pcie_phy;
93 	struct phy		*phy;
94 	const struct imx6_pcie_drvdata *drvdata;
95 };
96 
97 /* Parameters for the waiting for PCIe PHY PLL to lock on i.MX7 */
98 #define PHY_PLL_LOCK_WAIT_USLEEP_MAX	200
99 #define PHY_PLL_LOCK_WAIT_TIMEOUT	(2000 * PHY_PLL_LOCK_WAIT_USLEEP_MAX)
100 
101 /* PCIe Port Logic registers (memory-mapped) */
102 #define PL_OFFSET 0x700
103 
104 #define PCIE_PHY_CTRL (PL_OFFSET + 0x114)
105 #define PCIE_PHY_CTRL_DATA(x)		FIELD_PREP(GENMASK(15, 0), (x))
106 #define PCIE_PHY_CTRL_CAP_ADR		BIT(16)
107 #define PCIE_PHY_CTRL_CAP_DAT		BIT(17)
108 #define PCIE_PHY_CTRL_WR		BIT(18)
109 #define PCIE_PHY_CTRL_RD		BIT(19)
110 
111 #define PCIE_PHY_STAT (PL_OFFSET + 0x110)
112 #define PCIE_PHY_STAT_ACK		BIT(16)
113 
114 /* PHY registers (not memory-mapped) */
115 #define PCIE_PHY_ATEOVRD			0x10
116 #define  PCIE_PHY_ATEOVRD_EN			BIT(2)
117 #define  PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT	0
118 #define  PCIE_PHY_ATEOVRD_REF_CLKDIV_MASK	0x1
119 
120 #define PCIE_PHY_MPLL_OVRD_IN_LO		0x11
121 #define  PCIE_PHY_MPLL_MULTIPLIER_SHIFT		2
122 #define  PCIE_PHY_MPLL_MULTIPLIER_MASK		0x7f
123 #define  PCIE_PHY_MPLL_MULTIPLIER_OVRD		BIT(9)
124 
125 #define PCIE_PHY_RX_ASIC_OUT 0x100D
126 #define PCIE_PHY_RX_ASIC_OUT_VALID	(1 << 0)
127 
128 /* iMX7 PCIe PHY registers */
129 #define PCIE_PHY_CMN_REG4		0x14
130 /* These are probably the bits that *aren't* DCC_FB_EN */
131 #define PCIE_PHY_CMN_REG4_DCC_FB_EN	0x29
132 
133 #define PCIE_PHY_CMN_REG15	        0x54
134 #define PCIE_PHY_CMN_REG15_DLY_4	BIT(2)
135 #define PCIE_PHY_CMN_REG15_PLL_PD	BIT(5)
136 #define PCIE_PHY_CMN_REG15_OVRD_PLL_PD	BIT(7)
137 
138 #define PCIE_PHY_CMN_REG24		0x90
139 #define PCIE_PHY_CMN_REG24_RX_EQ	BIT(6)
140 #define PCIE_PHY_CMN_REG24_RX_EQ_SEL	BIT(3)
141 
142 #define PCIE_PHY_CMN_REG26		0x98
143 #define PCIE_PHY_CMN_REG26_ATT_MODE	0xBC
144 
145 #define PHY_RX_OVRD_IN_LO 0x1005
146 #define PHY_RX_OVRD_IN_LO_RX_DATA_EN		BIT(5)
147 #define PHY_RX_OVRD_IN_LO_RX_PLL_EN		BIT(3)
148 
149 static int pcie_phy_poll_ack(struct imx6_pcie *imx6_pcie, bool exp_val)
150 {
151 	struct dw_pcie *pci = imx6_pcie->pci;
152 	bool val;
153 	u32 max_iterations = 10;
154 	u32 wait_counter = 0;
155 
156 	do {
157 		val = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT) &
158 			PCIE_PHY_STAT_ACK;
159 		wait_counter++;
160 
161 		if (val == exp_val)
162 			return 0;
163 
164 		udelay(1);
165 	} while (wait_counter < max_iterations);
166 
167 	return -ETIMEDOUT;
168 }
169 
170 static int pcie_phy_wait_ack(struct imx6_pcie *imx6_pcie, int addr)
171 {
172 	struct dw_pcie *pci = imx6_pcie->pci;
173 	u32 val;
174 	int ret;
175 
176 	val = PCIE_PHY_CTRL_DATA(addr);
177 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val);
178 
179 	val |= PCIE_PHY_CTRL_CAP_ADR;
180 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val);
181 
182 	ret = pcie_phy_poll_ack(imx6_pcie, true);
183 	if (ret)
184 		return ret;
185 
186 	val = PCIE_PHY_CTRL_DATA(addr);
187 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val);
188 
189 	return pcie_phy_poll_ack(imx6_pcie, false);
190 }
191 
192 /* Read from the 16-bit PCIe PHY control registers (not memory-mapped) */
193 static int pcie_phy_read(struct imx6_pcie *imx6_pcie, int addr, u16 *data)
194 {
195 	struct dw_pcie *pci = imx6_pcie->pci;
196 	u32 phy_ctl;
197 	int ret;
198 
199 	ret = pcie_phy_wait_ack(imx6_pcie, addr);
200 	if (ret)
201 		return ret;
202 
203 	/* assert Read signal */
204 	phy_ctl = PCIE_PHY_CTRL_RD;
205 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, phy_ctl);
206 
207 	ret = pcie_phy_poll_ack(imx6_pcie, true);
208 	if (ret)
209 		return ret;
210 
211 	*data = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT);
212 
213 	/* deassert Read signal */
214 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, 0x00);
215 
216 	return pcie_phy_poll_ack(imx6_pcie, false);
217 }
218 
219 static int pcie_phy_write(struct imx6_pcie *imx6_pcie, int addr, u16 data)
220 {
221 	struct dw_pcie *pci = imx6_pcie->pci;
222 	u32 var;
223 	int ret;
224 
225 	/* write addr */
226 	/* cap addr */
227 	ret = pcie_phy_wait_ack(imx6_pcie, addr);
228 	if (ret)
229 		return ret;
230 
231 	var = PCIE_PHY_CTRL_DATA(data);
232 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
233 
234 	/* capture data */
235 	var |= PCIE_PHY_CTRL_CAP_DAT;
236 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
237 
238 	ret = pcie_phy_poll_ack(imx6_pcie, true);
239 	if (ret)
240 		return ret;
241 
242 	/* deassert cap data */
243 	var = PCIE_PHY_CTRL_DATA(data);
244 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
245 
246 	/* wait for ack de-assertion */
247 	ret = pcie_phy_poll_ack(imx6_pcie, false);
248 	if (ret)
249 		return ret;
250 
251 	/* assert wr signal */
252 	var = PCIE_PHY_CTRL_WR;
253 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
254 
255 	/* wait for ack */
256 	ret = pcie_phy_poll_ack(imx6_pcie, true);
257 	if (ret)
258 		return ret;
259 
260 	/* deassert wr signal */
261 	var = PCIE_PHY_CTRL_DATA(data);
262 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
263 
264 	/* wait for ack de-assertion */
265 	ret = pcie_phy_poll_ack(imx6_pcie, false);
266 	if (ret)
267 		return ret;
268 
269 	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, 0x0);
270 
271 	return 0;
272 }
273 
274 static void imx6_pcie_reset_phy(struct imx6_pcie *imx6_pcie)
275 {
276 	u16 tmp;
277 
278 	if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_IMX6_PHY))
279 		return;
280 
281 	pcie_phy_read(imx6_pcie, PHY_RX_OVRD_IN_LO, &tmp);
282 	tmp |= (PHY_RX_OVRD_IN_LO_RX_DATA_EN |
283 		PHY_RX_OVRD_IN_LO_RX_PLL_EN);
284 	pcie_phy_write(imx6_pcie, PHY_RX_OVRD_IN_LO, tmp);
285 
286 	usleep_range(2000, 3000);
287 
288 	pcie_phy_read(imx6_pcie, PHY_RX_OVRD_IN_LO, &tmp);
289 	tmp &= ~(PHY_RX_OVRD_IN_LO_RX_DATA_EN |
290 		  PHY_RX_OVRD_IN_LO_RX_PLL_EN);
291 	pcie_phy_write(imx6_pcie, PHY_RX_OVRD_IN_LO, tmp);
292 }
293 
294 #ifdef CONFIG_ARM
295 /*  Added for PCI abort handling */
296 static int imx6q_pcie_abort_handler(unsigned long addr,
297 		unsigned int fsr, struct pt_regs *regs)
298 {
299 	unsigned long pc = instruction_pointer(regs);
300 	unsigned long instr = *(unsigned long *)pc;
301 	int reg = (instr >> 12) & 15;
302 
303 	/*
304 	 * If the instruction being executed was a read,
305 	 * make it look like it read all-ones.
306 	 */
307 	if ((instr & 0x0c100000) == 0x04100000) {
308 		unsigned long val;
309 
310 		if (instr & 0x00400000)
311 			val = 255;
312 		else
313 			val = -1;
314 
315 		regs->uregs[reg] = val;
316 		regs->ARM_pc += 4;
317 		return 0;
318 	}
319 
320 	if ((instr & 0x0e100090) == 0x00100090) {
321 		regs->uregs[reg] = -1;
322 		regs->ARM_pc += 4;
323 		return 0;
324 	}
325 
326 	return 1;
327 }
328 #endif
329 
330 static int imx6_pcie_attach_pd(struct device *dev)
331 {
332 	struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev);
333 	struct device_link *link;
334 
335 	/* Do nothing when in a single power domain */
336 	if (dev->pm_domain)
337 		return 0;
338 
339 	imx6_pcie->pd_pcie = dev_pm_domain_attach_by_name(dev, "pcie");
340 	if (IS_ERR(imx6_pcie->pd_pcie))
341 		return PTR_ERR(imx6_pcie->pd_pcie);
342 	/* Do nothing when power domain missing */
343 	if (!imx6_pcie->pd_pcie)
344 		return 0;
345 	link = device_link_add(dev, imx6_pcie->pd_pcie,
346 			DL_FLAG_STATELESS |
347 			DL_FLAG_PM_RUNTIME |
348 			DL_FLAG_RPM_ACTIVE);
349 	if (!link) {
350 		dev_err(dev, "Failed to add device_link to pcie pd.\n");
351 		return -EINVAL;
352 	}
353 
354 	imx6_pcie->pd_pcie_phy = dev_pm_domain_attach_by_name(dev, "pcie_phy");
355 	if (IS_ERR(imx6_pcie->pd_pcie_phy))
356 		return PTR_ERR(imx6_pcie->pd_pcie_phy);
357 
358 	link = device_link_add(dev, imx6_pcie->pd_pcie_phy,
359 			DL_FLAG_STATELESS |
360 			DL_FLAG_PM_RUNTIME |
361 			DL_FLAG_RPM_ACTIVE);
362 	if (!link) {
363 		dev_err(dev, "Failed to add device_link to pcie_phy pd.\n");
364 		return -EINVAL;
365 	}
366 
367 	return 0;
368 }
369 
370 static void imx6_pcie_assert_core_reset(struct imx6_pcie *imx6_pcie)
371 {
372 	struct device *dev = imx6_pcie->pci->dev;
373 
374 	switch (imx6_pcie->drvdata->variant) {
375 	case IMX7D:
376 	case IMX8MQ:
377 		reset_control_assert(imx6_pcie->pciephy_reset);
378 		fallthrough;
379 	case IMX8MM:
380 		reset_control_assert(imx6_pcie->apps_reset);
381 		break;
382 	case IMX6SX:
383 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
384 				   IMX6SX_GPR12_PCIE_TEST_POWERDOWN,
385 				   IMX6SX_GPR12_PCIE_TEST_POWERDOWN);
386 		/* Force PCIe PHY reset */
387 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR5,
388 				   IMX6SX_GPR5_PCIE_BTNRST_RESET,
389 				   IMX6SX_GPR5_PCIE_BTNRST_RESET);
390 		break;
391 	case IMX6QP:
392 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
393 				   IMX6Q_GPR1_PCIE_SW_RST,
394 				   IMX6Q_GPR1_PCIE_SW_RST);
395 		break;
396 	case IMX6Q:
397 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
398 				   IMX6Q_GPR1_PCIE_TEST_PD, 1 << 18);
399 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
400 				   IMX6Q_GPR1_PCIE_REF_CLK_EN, 0 << 16);
401 		break;
402 	}
403 
404 	if (imx6_pcie->vpcie && regulator_is_enabled(imx6_pcie->vpcie) > 0) {
405 		int ret = regulator_disable(imx6_pcie->vpcie);
406 
407 		if (ret)
408 			dev_err(dev, "failed to disable vpcie regulator: %d\n",
409 				ret);
410 	}
411 
412 	/* Some boards don't have PCIe reset GPIO. */
413 	if (gpio_is_valid(imx6_pcie->reset_gpio))
414 		gpio_set_value_cansleep(imx6_pcie->reset_gpio,
415 					imx6_pcie->gpio_active_high);
416 }
417 
418 static unsigned int imx6_pcie_grp_offset(const struct imx6_pcie *imx6_pcie)
419 {
420 	WARN_ON(imx6_pcie->drvdata->variant != IMX8MQ &&
421 		imx6_pcie->drvdata->variant != IMX8MM);
422 	return imx6_pcie->controller_id == 1 ? IOMUXC_GPR16 : IOMUXC_GPR14;
423 }
424 
425 static int imx6_pcie_enable_ref_clk(struct imx6_pcie *imx6_pcie)
426 {
427 	struct dw_pcie *pci = imx6_pcie->pci;
428 	struct device *dev = pci->dev;
429 	unsigned int offset;
430 	int ret = 0;
431 
432 	switch (imx6_pcie->drvdata->variant) {
433 	case IMX6SX:
434 		ret = clk_prepare_enable(imx6_pcie->pcie_inbound_axi);
435 		if (ret) {
436 			dev_err(dev, "unable to enable pcie_axi clock\n");
437 			break;
438 		}
439 
440 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
441 				   IMX6SX_GPR12_PCIE_TEST_POWERDOWN, 0);
442 		break;
443 	case IMX6QP:
444 	case IMX6Q:
445 		/* power up core phy and enable ref clock */
446 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
447 				   IMX6Q_GPR1_PCIE_TEST_PD, 0 << 18);
448 		/*
449 		 * the async reset input need ref clock to sync internally,
450 		 * when the ref clock comes after reset, internal synced
451 		 * reset time is too short, cannot meet the requirement.
452 		 * add one ~10us delay here.
453 		 */
454 		usleep_range(10, 100);
455 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
456 				   IMX6Q_GPR1_PCIE_REF_CLK_EN, 1 << 16);
457 		break;
458 	case IMX7D:
459 		break;
460 	case IMX8MM:
461 	case IMX8MQ:
462 		ret = clk_prepare_enable(imx6_pcie->pcie_aux);
463 		if (ret) {
464 			dev_err(dev, "unable to enable pcie_aux clock\n");
465 			break;
466 		}
467 
468 		offset = imx6_pcie_grp_offset(imx6_pcie);
469 		/*
470 		 * Set the over ride low and enabled
471 		 * make sure that REF_CLK is turned on.
472 		 */
473 		regmap_update_bits(imx6_pcie->iomuxc_gpr, offset,
474 				   IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE,
475 				   0);
476 		regmap_update_bits(imx6_pcie->iomuxc_gpr, offset,
477 				   IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN,
478 				   IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN);
479 		break;
480 	}
481 
482 	return ret;
483 }
484 
485 static void imx7d_pcie_wait_for_phy_pll_lock(struct imx6_pcie *imx6_pcie)
486 {
487 	u32 val;
488 	struct device *dev = imx6_pcie->pci->dev;
489 
490 	if (regmap_read_poll_timeout(imx6_pcie->iomuxc_gpr,
491 				     IOMUXC_GPR22, val,
492 				     val & IMX7D_GPR22_PCIE_PHY_PLL_LOCKED,
493 				     PHY_PLL_LOCK_WAIT_USLEEP_MAX,
494 				     PHY_PLL_LOCK_WAIT_TIMEOUT))
495 		dev_err(dev, "PCIe PLL lock timeout\n");
496 }
497 
498 static void imx6_pcie_deassert_core_reset(struct imx6_pcie *imx6_pcie)
499 {
500 	struct dw_pcie *pci = imx6_pcie->pci;
501 	struct device *dev = pci->dev;
502 	int ret;
503 
504 	if (imx6_pcie->vpcie && !regulator_is_enabled(imx6_pcie->vpcie)) {
505 		ret = regulator_enable(imx6_pcie->vpcie);
506 		if (ret) {
507 			dev_err(dev, "failed to enable vpcie regulator: %d\n",
508 				ret);
509 			return;
510 		}
511 	}
512 
513 	ret = clk_prepare_enable(imx6_pcie->pcie_phy);
514 	if (ret) {
515 		dev_err(dev, "unable to enable pcie_phy clock\n");
516 		goto err_pcie_phy;
517 	}
518 
519 	ret = clk_prepare_enable(imx6_pcie->pcie_bus);
520 	if (ret) {
521 		dev_err(dev, "unable to enable pcie_bus clock\n");
522 		goto err_pcie_bus;
523 	}
524 
525 	ret = clk_prepare_enable(imx6_pcie->pcie);
526 	if (ret) {
527 		dev_err(dev, "unable to enable pcie clock\n");
528 		goto err_pcie;
529 	}
530 
531 	ret = imx6_pcie_enable_ref_clk(imx6_pcie);
532 	if (ret) {
533 		dev_err(dev, "unable to enable pcie ref clock\n");
534 		goto err_ref_clk;
535 	}
536 
537 	switch (imx6_pcie->drvdata->variant) {
538 	case IMX8MM:
539 		if (phy_power_on(imx6_pcie->phy))
540 			dev_err(dev, "unable to power on PHY\n");
541 		break;
542 	default:
543 		break;
544 	}
545 	/* allow the clocks to stabilize */
546 	usleep_range(200, 500);
547 
548 	switch (imx6_pcie->drvdata->variant) {
549 	case IMX8MQ:
550 		reset_control_deassert(imx6_pcie->pciephy_reset);
551 		break;
552 	case IMX8MM:
553 		if (phy_init(imx6_pcie->phy))
554 			dev_err(dev, "waiting for phy ready timeout!\n");
555 		break;
556 	case IMX7D:
557 		reset_control_deassert(imx6_pcie->pciephy_reset);
558 
559 		/* Workaround for ERR010728, failure of PCI-e PLL VCO to
560 		 * oscillate, especially when cold.  This turns off "Duty-cycle
561 		 * Corrector" and other mysterious undocumented things.
562 		 */
563 		if (likely(imx6_pcie->phy_base)) {
564 			/* De-assert DCC_FB_EN */
565 			writel(PCIE_PHY_CMN_REG4_DCC_FB_EN,
566 			       imx6_pcie->phy_base + PCIE_PHY_CMN_REG4);
567 			/* Assert RX_EQS and RX_EQS_SEL */
568 			writel(PCIE_PHY_CMN_REG24_RX_EQ_SEL
569 				| PCIE_PHY_CMN_REG24_RX_EQ,
570 			       imx6_pcie->phy_base + PCIE_PHY_CMN_REG24);
571 			/* Assert ATT_MODE */
572 			writel(PCIE_PHY_CMN_REG26_ATT_MODE,
573 			       imx6_pcie->phy_base + PCIE_PHY_CMN_REG26);
574 		} else {
575 			dev_warn(dev, "Unable to apply ERR010728 workaround. DT missing fsl,imx7d-pcie-phy phandle ?\n");
576 		}
577 
578 		imx7d_pcie_wait_for_phy_pll_lock(imx6_pcie);
579 		break;
580 	case IMX6SX:
581 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR5,
582 				   IMX6SX_GPR5_PCIE_BTNRST_RESET, 0);
583 		break;
584 	case IMX6QP:
585 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,
586 				   IMX6Q_GPR1_PCIE_SW_RST, 0);
587 
588 		usleep_range(200, 500);
589 		break;
590 	case IMX6Q:		/* Nothing to do */
591 		break;
592 	}
593 
594 	/* Some boards don't have PCIe reset GPIO. */
595 	if (gpio_is_valid(imx6_pcie->reset_gpio)) {
596 		msleep(100);
597 		gpio_set_value_cansleep(imx6_pcie->reset_gpio,
598 					!imx6_pcie->gpio_active_high);
599 		/* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */
600 		msleep(100);
601 	}
602 
603 	return;
604 
605 err_ref_clk:
606 	clk_disable_unprepare(imx6_pcie->pcie);
607 err_pcie:
608 	clk_disable_unprepare(imx6_pcie->pcie_bus);
609 err_pcie_bus:
610 	clk_disable_unprepare(imx6_pcie->pcie_phy);
611 err_pcie_phy:
612 	if (imx6_pcie->vpcie && regulator_is_enabled(imx6_pcie->vpcie) > 0) {
613 		ret = regulator_disable(imx6_pcie->vpcie);
614 		if (ret)
615 			dev_err(dev, "failed to disable vpcie regulator: %d\n",
616 				ret);
617 	}
618 }
619 
620 static void imx6_pcie_configure_type(struct imx6_pcie *imx6_pcie)
621 {
622 	unsigned int mask, val;
623 
624 	if (imx6_pcie->drvdata->variant == IMX8MQ &&
625 	    imx6_pcie->controller_id == 1) {
626 		mask   = IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE;
627 		val    = FIELD_PREP(IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE,
628 				    PCI_EXP_TYPE_ROOT_PORT);
629 	} else {
630 		mask = IMX6Q_GPR12_DEVICE_TYPE;
631 		val  = FIELD_PREP(IMX6Q_GPR12_DEVICE_TYPE,
632 				  PCI_EXP_TYPE_ROOT_PORT);
633 	}
634 
635 	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, mask, val);
636 }
637 
638 static void imx6_pcie_init_phy(struct imx6_pcie *imx6_pcie)
639 {
640 	switch (imx6_pcie->drvdata->variant) {
641 	case IMX8MM:
642 		/*
643 		 * The PHY initialization had been done in the PHY
644 		 * driver, break here directly.
645 		 */
646 		break;
647 	case IMX8MQ:
648 		/*
649 		 * TODO: Currently this code assumes external
650 		 * oscillator is being used
651 		 */
652 		regmap_update_bits(imx6_pcie->iomuxc_gpr,
653 				   imx6_pcie_grp_offset(imx6_pcie),
654 				   IMX8MQ_GPR_PCIE_REF_USE_PAD,
655 				   IMX8MQ_GPR_PCIE_REF_USE_PAD);
656 		/*
657 		 * Regarding the datasheet, the PCIE_VPH is suggested
658 		 * to be 1.8V. If the PCIE_VPH is supplied by 3.3V, the
659 		 * VREG_BYPASS should be cleared to zero.
660 		 */
661 		if (imx6_pcie->vph &&
662 		    regulator_get_voltage(imx6_pcie->vph) > 3000000)
663 			regmap_update_bits(imx6_pcie->iomuxc_gpr,
664 					   imx6_pcie_grp_offset(imx6_pcie),
665 					   IMX8MQ_GPR_PCIE_VREG_BYPASS,
666 					   0);
667 		break;
668 	case IMX7D:
669 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
670 				   IMX7D_GPR12_PCIE_PHY_REFCLK_SEL, 0);
671 		break;
672 	case IMX6SX:
673 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
674 				   IMX6SX_GPR12_PCIE_RX_EQ_MASK,
675 				   IMX6SX_GPR12_PCIE_RX_EQ_2);
676 		fallthrough;
677 	default:
678 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
679 				   IMX6Q_GPR12_PCIE_CTL_2, 0 << 10);
680 
681 		/* configure constant input signal to the pcie ctrl and phy */
682 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
683 				   IMX6Q_GPR12_LOS_LEVEL, 9 << 4);
684 
685 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
686 				   IMX6Q_GPR8_TX_DEEMPH_GEN1,
687 				   imx6_pcie->tx_deemph_gen1 << 0);
688 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
689 				   IMX6Q_GPR8_TX_DEEMPH_GEN2_3P5DB,
690 				   imx6_pcie->tx_deemph_gen2_3p5db << 6);
691 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
692 				   IMX6Q_GPR8_TX_DEEMPH_GEN2_6DB,
693 				   imx6_pcie->tx_deemph_gen2_6db << 12);
694 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
695 				   IMX6Q_GPR8_TX_SWING_FULL,
696 				   imx6_pcie->tx_swing_full << 18);
697 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,
698 				   IMX6Q_GPR8_TX_SWING_LOW,
699 				   imx6_pcie->tx_swing_low << 25);
700 		break;
701 	}
702 
703 	imx6_pcie_configure_type(imx6_pcie);
704 }
705 
706 static int imx6_setup_phy_mpll(struct imx6_pcie *imx6_pcie)
707 {
708 	unsigned long phy_rate = clk_get_rate(imx6_pcie->pcie_phy);
709 	int mult, div;
710 	u16 val;
711 
712 	if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_IMX6_PHY))
713 		return 0;
714 
715 	switch (phy_rate) {
716 	case 125000000:
717 		/*
718 		 * The default settings of the MPLL are for a 125MHz input
719 		 * clock, so no need to reconfigure anything in that case.
720 		 */
721 		return 0;
722 	case 100000000:
723 		mult = 25;
724 		div = 0;
725 		break;
726 	case 200000000:
727 		mult = 25;
728 		div = 1;
729 		break;
730 	default:
731 		dev_err(imx6_pcie->pci->dev,
732 			"Unsupported PHY reference clock rate %lu\n", phy_rate);
733 		return -EINVAL;
734 	}
735 
736 	pcie_phy_read(imx6_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, &val);
737 	val &= ~(PCIE_PHY_MPLL_MULTIPLIER_MASK <<
738 		 PCIE_PHY_MPLL_MULTIPLIER_SHIFT);
739 	val |= mult << PCIE_PHY_MPLL_MULTIPLIER_SHIFT;
740 	val |= PCIE_PHY_MPLL_MULTIPLIER_OVRD;
741 	pcie_phy_write(imx6_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, val);
742 
743 	pcie_phy_read(imx6_pcie, PCIE_PHY_ATEOVRD, &val);
744 	val &= ~(PCIE_PHY_ATEOVRD_REF_CLKDIV_MASK <<
745 		 PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT);
746 	val |= div << PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT;
747 	val |= PCIE_PHY_ATEOVRD_EN;
748 	pcie_phy_write(imx6_pcie, PCIE_PHY_ATEOVRD, val);
749 
750 	return 0;
751 }
752 
753 static int imx6_pcie_wait_for_speed_change(struct imx6_pcie *imx6_pcie)
754 {
755 	struct dw_pcie *pci = imx6_pcie->pci;
756 	struct device *dev = pci->dev;
757 	u32 tmp;
758 	unsigned int retries;
759 
760 	for (retries = 0; retries < 200; retries++) {
761 		tmp = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
762 		/* Test if the speed change finished. */
763 		if (!(tmp & PORT_LOGIC_SPEED_CHANGE))
764 			return 0;
765 		usleep_range(100, 1000);
766 	}
767 
768 	dev_err(dev, "Speed change timeout\n");
769 	return -ETIMEDOUT;
770 }
771 
772 static void imx6_pcie_ltssm_enable(struct device *dev)
773 {
774 	struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev);
775 
776 	switch (imx6_pcie->drvdata->variant) {
777 	case IMX6Q:
778 	case IMX6SX:
779 	case IMX6QP:
780 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
781 				   IMX6Q_GPR12_PCIE_CTL_2,
782 				   IMX6Q_GPR12_PCIE_CTL_2);
783 		break;
784 	case IMX7D:
785 	case IMX8MQ:
786 	case IMX8MM:
787 		reset_control_deassert(imx6_pcie->apps_reset);
788 		break;
789 	}
790 }
791 
792 static int imx6_pcie_start_link(struct dw_pcie *pci)
793 {
794 	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci);
795 	struct device *dev = pci->dev;
796 	u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
797 	u32 tmp;
798 	int ret;
799 
800 	/*
801 	 * Force Gen1 operation when starting the link.  In case the link is
802 	 * started in Gen2 mode, there is a possibility the devices on the
803 	 * bus will not be detected at all.  This happens with PCIe switches.
804 	 */
805 	tmp = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP);
806 	tmp &= ~PCI_EXP_LNKCAP_SLS;
807 	tmp |= PCI_EXP_LNKCAP_SLS_2_5GB;
808 	dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, tmp);
809 
810 	/* Start LTSSM. */
811 	imx6_pcie_ltssm_enable(dev);
812 
813 	dw_pcie_wait_for_link(pci);
814 
815 	if (pci->link_gen == 2) {
816 		/* Allow Gen2 mode after the link is up. */
817 		tmp = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP);
818 		tmp &= ~PCI_EXP_LNKCAP_SLS;
819 		tmp |= PCI_EXP_LNKCAP_SLS_5_0GB;
820 		dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, tmp);
821 
822 		/*
823 		 * Start Directed Speed Change so the best possible
824 		 * speed both link partners support can be negotiated.
825 		 */
826 		tmp = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
827 		tmp |= PORT_LOGIC_SPEED_CHANGE;
828 		dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, tmp);
829 
830 		if (imx6_pcie->drvdata->flags &
831 		    IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE) {
832 			/*
833 			 * On i.MX7, DIRECT_SPEED_CHANGE behaves differently
834 			 * from i.MX6 family when no link speed transition
835 			 * occurs and we go Gen1 -> yep, Gen1. The difference
836 			 * is that, in such case, it will not be cleared by HW
837 			 * which will cause the following code to report false
838 			 * failure.
839 			 */
840 
841 			ret = imx6_pcie_wait_for_speed_change(imx6_pcie);
842 			if (ret) {
843 				dev_err(dev, "Failed to bring link up!\n");
844 				goto err_reset_phy;
845 			}
846 		}
847 
848 		/* Make sure link training is finished as well! */
849 		dw_pcie_wait_for_link(pci);
850 	} else {
851 		dev_info(dev, "Link: Gen2 disabled\n");
852 	}
853 
854 	tmp = dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKSTA);
855 	dev_info(dev, "Link up, Gen%i\n", tmp & PCI_EXP_LNKSTA_CLS);
856 	return 0;
857 
858 err_reset_phy:
859 	dev_dbg(dev, "PHY DEBUG_R0=0x%08x DEBUG_R1=0x%08x\n",
860 		dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0),
861 		dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG1));
862 	imx6_pcie_reset_phy(imx6_pcie);
863 	return ret;
864 }
865 
866 static int imx6_pcie_host_init(struct pcie_port *pp)
867 {
868 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
869 	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci);
870 
871 	imx6_pcie_assert_core_reset(imx6_pcie);
872 	imx6_pcie_init_phy(imx6_pcie);
873 	imx6_pcie_deassert_core_reset(imx6_pcie);
874 	imx6_setup_phy_mpll(imx6_pcie);
875 
876 	return 0;
877 }
878 
879 static const struct dw_pcie_host_ops imx6_pcie_host_ops = {
880 	.host_init = imx6_pcie_host_init,
881 };
882 
883 static const struct dw_pcie_ops dw_pcie_ops = {
884 	.start_link = imx6_pcie_start_link,
885 };
886 
887 #ifdef CONFIG_PM_SLEEP
888 static void imx6_pcie_ltssm_disable(struct device *dev)
889 {
890 	struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev);
891 
892 	switch (imx6_pcie->drvdata->variant) {
893 	case IMX6SX:
894 	case IMX6QP:
895 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
896 				   IMX6Q_GPR12_PCIE_CTL_2, 0);
897 		break;
898 	case IMX7D:
899 	case IMX8MM:
900 		reset_control_assert(imx6_pcie->apps_reset);
901 		break;
902 	default:
903 		dev_err(dev, "ltssm_disable not supported\n");
904 	}
905 }
906 
907 static void imx6_pcie_pm_turnoff(struct imx6_pcie *imx6_pcie)
908 {
909 	struct device *dev = imx6_pcie->pci->dev;
910 
911 	/* Some variants have a turnoff reset in DT */
912 	if (imx6_pcie->turnoff_reset) {
913 		reset_control_assert(imx6_pcie->turnoff_reset);
914 		reset_control_deassert(imx6_pcie->turnoff_reset);
915 		goto pm_turnoff_sleep;
916 	}
917 
918 	/* Others poke directly at IOMUXC registers */
919 	switch (imx6_pcie->drvdata->variant) {
920 	case IMX6SX:
921 	case IMX6QP:
922 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
923 				IMX6SX_GPR12_PCIE_PM_TURN_OFF,
924 				IMX6SX_GPR12_PCIE_PM_TURN_OFF);
925 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
926 				IMX6SX_GPR12_PCIE_PM_TURN_OFF, 0);
927 		break;
928 	default:
929 		dev_err(dev, "PME_Turn_Off not implemented\n");
930 		return;
931 	}
932 
933 	/*
934 	 * Components with an upstream port must respond to
935 	 * PME_Turn_Off with PME_TO_Ack but we can't check.
936 	 *
937 	 * The standard recommends a 1-10ms timeout after which to
938 	 * proceed anyway as if acks were received.
939 	 */
940 pm_turnoff_sleep:
941 	usleep_range(1000, 10000);
942 }
943 
944 static void imx6_pcie_clk_disable(struct imx6_pcie *imx6_pcie)
945 {
946 	clk_disable_unprepare(imx6_pcie->pcie);
947 	clk_disable_unprepare(imx6_pcie->pcie_phy);
948 	clk_disable_unprepare(imx6_pcie->pcie_bus);
949 
950 	switch (imx6_pcie->drvdata->variant) {
951 	case IMX6SX:
952 		clk_disable_unprepare(imx6_pcie->pcie_inbound_axi);
953 		break;
954 	case IMX7D:
955 		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,
956 				   IMX7D_GPR12_PCIE_PHY_REFCLK_SEL,
957 				   IMX7D_GPR12_PCIE_PHY_REFCLK_SEL);
958 		break;
959 	case IMX8MQ:
960 	case IMX8MM:
961 		clk_disable_unprepare(imx6_pcie->pcie_aux);
962 		break;
963 	default:
964 		break;
965 	}
966 }
967 
968 static int imx6_pcie_suspend_noirq(struct device *dev)
969 {
970 	struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev);
971 
972 	if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_SUPPORTS_SUSPEND))
973 		return 0;
974 
975 	imx6_pcie_pm_turnoff(imx6_pcie);
976 	imx6_pcie_ltssm_disable(dev);
977 	imx6_pcie_clk_disable(imx6_pcie);
978 	switch (imx6_pcie->drvdata->variant) {
979 	case IMX8MM:
980 		if (phy_power_off(imx6_pcie->phy))
981 			dev_err(dev, "unable to power off PHY\n");
982 		phy_exit(imx6_pcie->phy);
983 		break;
984 	default:
985 		break;
986 	}
987 
988 	return 0;
989 }
990 
991 static int imx6_pcie_resume_noirq(struct device *dev)
992 {
993 	int ret;
994 	struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev);
995 	struct pcie_port *pp = &imx6_pcie->pci->pp;
996 
997 	if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_SUPPORTS_SUSPEND))
998 		return 0;
999 
1000 	imx6_pcie_assert_core_reset(imx6_pcie);
1001 	imx6_pcie_init_phy(imx6_pcie);
1002 	imx6_pcie_deassert_core_reset(imx6_pcie);
1003 	dw_pcie_setup_rc(pp);
1004 
1005 	ret = imx6_pcie_start_link(imx6_pcie->pci);
1006 	if (ret < 0)
1007 		dev_info(dev, "pcie link is down after resume.\n");
1008 
1009 	return 0;
1010 }
1011 #endif
1012 
1013 static const struct dev_pm_ops imx6_pcie_pm_ops = {
1014 	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(imx6_pcie_suspend_noirq,
1015 				      imx6_pcie_resume_noirq)
1016 };
1017 
1018 static int imx6_pcie_probe(struct platform_device *pdev)
1019 {
1020 	struct device *dev = &pdev->dev;
1021 	struct dw_pcie *pci;
1022 	struct imx6_pcie *imx6_pcie;
1023 	struct device_node *np;
1024 	struct resource *dbi_base;
1025 	struct device_node *node = dev->of_node;
1026 	int ret;
1027 	u16 val;
1028 
1029 	imx6_pcie = devm_kzalloc(dev, sizeof(*imx6_pcie), GFP_KERNEL);
1030 	if (!imx6_pcie)
1031 		return -ENOMEM;
1032 
1033 	pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
1034 	if (!pci)
1035 		return -ENOMEM;
1036 
1037 	pci->dev = dev;
1038 	pci->ops = &dw_pcie_ops;
1039 	pci->pp.ops = &imx6_pcie_host_ops;
1040 
1041 	imx6_pcie->pci = pci;
1042 	imx6_pcie->drvdata = of_device_get_match_data(dev);
1043 
1044 	/* Find the PHY if one is defined, only imx7d uses it */
1045 	np = of_parse_phandle(node, "fsl,imx7d-pcie-phy", 0);
1046 	if (np) {
1047 		struct resource res;
1048 
1049 		ret = of_address_to_resource(np, 0, &res);
1050 		if (ret) {
1051 			dev_err(dev, "Unable to map PCIe PHY\n");
1052 			return ret;
1053 		}
1054 		imx6_pcie->phy_base = devm_ioremap_resource(dev, &res);
1055 		if (IS_ERR(imx6_pcie->phy_base))
1056 			return PTR_ERR(imx6_pcie->phy_base);
1057 	}
1058 
1059 	dbi_base = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1060 	pci->dbi_base = devm_ioremap_resource(dev, dbi_base);
1061 	if (IS_ERR(pci->dbi_base))
1062 		return PTR_ERR(pci->dbi_base);
1063 
1064 	/* Fetch GPIOs */
1065 	imx6_pcie->reset_gpio = of_get_named_gpio(node, "reset-gpio", 0);
1066 	imx6_pcie->gpio_active_high = of_property_read_bool(node,
1067 						"reset-gpio-active-high");
1068 	if (gpio_is_valid(imx6_pcie->reset_gpio)) {
1069 		ret = devm_gpio_request_one(dev, imx6_pcie->reset_gpio,
1070 				imx6_pcie->gpio_active_high ?
1071 					GPIOF_OUT_INIT_HIGH :
1072 					GPIOF_OUT_INIT_LOW,
1073 				"PCIe reset");
1074 		if (ret) {
1075 			dev_err(dev, "unable to get reset gpio\n");
1076 			return ret;
1077 		}
1078 	} else if (imx6_pcie->reset_gpio == -EPROBE_DEFER) {
1079 		return imx6_pcie->reset_gpio;
1080 	}
1081 
1082 	/* Fetch clocks */
1083 	imx6_pcie->pcie_bus = devm_clk_get(dev, "pcie_bus");
1084 	if (IS_ERR(imx6_pcie->pcie_bus))
1085 		return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_bus),
1086 				     "pcie_bus clock source missing or invalid\n");
1087 
1088 	imx6_pcie->pcie = devm_clk_get(dev, "pcie");
1089 	if (IS_ERR(imx6_pcie->pcie))
1090 		return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie),
1091 				     "pcie clock source missing or invalid\n");
1092 
1093 	switch (imx6_pcie->drvdata->variant) {
1094 	case IMX6SX:
1095 		imx6_pcie->pcie_inbound_axi = devm_clk_get(dev,
1096 							   "pcie_inbound_axi");
1097 		if (IS_ERR(imx6_pcie->pcie_inbound_axi))
1098 			return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_inbound_axi),
1099 					     "pcie_inbound_axi clock missing or invalid\n");
1100 		break;
1101 	case IMX8MQ:
1102 		imx6_pcie->pcie_aux = devm_clk_get(dev, "pcie_aux");
1103 		if (IS_ERR(imx6_pcie->pcie_aux))
1104 			return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_aux),
1105 					     "pcie_aux clock source missing or invalid\n");
1106 		fallthrough;
1107 	case IMX7D:
1108 		if (dbi_base->start == IMX8MQ_PCIE2_BASE_ADDR)
1109 			imx6_pcie->controller_id = 1;
1110 
1111 		imx6_pcie->pciephy_reset = devm_reset_control_get_exclusive(dev,
1112 									    "pciephy");
1113 		if (IS_ERR(imx6_pcie->pciephy_reset)) {
1114 			dev_err(dev, "Failed to get PCIEPHY reset control\n");
1115 			return PTR_ERR(imx6_pcie->pciephy_reset);
1116 		}
1117 
1118 		imx6_pcie->apps_reset = devm_reset_control_get_exclusive(dev,
1119 									 "apps");
1120 		if (IS_ERR(imx6_pcie->apps_reset)) {
1121 			dev_err(dev, "Failed to get PCIE APPS reset control\n");
1122 			return PTR_ERR(imx6_pcie->apps_reset);
1123 		}
1124 		break;
1125 	case IMX8MM:
1126 		imx6_pcie->pcie_aux = devm_clk_get(dev, "pcie_aux");
1127 		if (IS_ERR(imx6_pcie->pcie_aux))
1128 			return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_aux),
1129 					     "pcie_aux clock source missing or invalid\n");
1130 		imx6_pcie->apps_reset = devm_reset_control_get_exclusive(dev,
1131 									 "apps");
1132 		if (IS_ERR(imx6_pcie->apps_reset))
1133 			return dev_err_probe(dev, PTR_ERR(imx6_pcie->apps_reset),
1134 					     "failed to get pcie apps reset control\n");
1135 
1136 		imx6_pcie->phy = devm_phy_get(dev, "pcie-phy");
1137 		if (IS_ERR(imx6_pcie->phy))
1138 			return dev_err_probe(dev, PTR_ERR(imx6_pcie->phy),
1139 					     "failed to get pcie phy\n");
1140 
1141 		break;
1142 	default:
1143 		break;
1144 	}
1145 	/* Don't fetch the pcie_phy clock, if it has abstract PHY driver */
1146 	if (imx6_pcie->phy == NULL) {
1147 		imx6_pcie->pcie_phy = devm_clk_get(dev, "pcie_phy");
1148 		if (IS_ERR(imx6_pcie->pcie_phy))
1149 			return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_phy),
1150 					     "pcie_phy clock source missing or invalid\n");
1151 	}
1152 
1153 
1154 	/* Grab turnoff reset */
1155 	imx6_pcie->turnoff_reset = devm_reset_control_get_optional_exclusive(dev, "turnoff");
1156 	if (IS_ERR(imx6_pcie->turnoff_reset)) {
1157 		dev_err(dev, "Failed to get TURNOFF reset control\n");
1158 		return PTR_ERR(imx6_pcie->turnoff_reset);
1159 	}
1160 
1161 	/* Grab GPR config register range */
1162 	imx6_pcie->iomuxc_gpr =
1163 		 syscon_regmap_lookup_by_compatible("fsl,imx6q-iomuxc-gpr");
1164 	if (IS_ERR(imx6_pcie->iomuxc_gpr)) {
1165 		dev_err(dev, "unable to find iomuxc registers\n");
1166 		return PTR_ERR(imx6_pcie->iomuxc_gpr);
1167 	}
1168 
1169 	/* Grab PCIe PHY Tx Settings */
1170 	if (of_property_read_u32(node, "fsl,tx-deemph-gen1",
1171 				 &imx6_pcie->tx_deemph_gen1))
1172 		imx6_pcie->tx_deemph_gen1 = 0;
1173 
1174 	if (of_property_read_u32(node, "fsl,tx-deemph-gen2-3p5db",
1175 				 &imx6_pcie->tx_deemph_gen2_3p5db))
1176 		imx6_pcie->tx_deemph_gen2_3p5db = 0;
1177 
1178 	if (of_property_read_u32(node, "fsl,tx-deemph-gen2-6db",
1179 				 &imx6_pcie->tx_deemph_gen2_6db))
1180 		imx6_pcie->tx_deemph_gen2_6db = 20;
1181 
1182 	if (of_property_read_u32(node, "fsl,tx-swing-full",
1183 				 &imx6_pcie->tx_swing_full))
1184 		imx6_pcie->tx_swing_full = 127;
1185 
1186 	if (of_property_read_u32(node, "fsl,tx-swing-low",
1187 				 &imx6_pcie->tx_swing_low))
1188 		imx6_pcie->tx_swing_low = 127;
1189 
1190 	/* Limit link speed */
1191 	pci->link_gen = 1;
1192 	of_property_read_u32(node, "fsl,max-link-speed", &pci->link_gen);
1193 
1194 	imx6_pcie->vpcie = devm_regulator_get_optional(&pdev->dev, "vpcie");
1195 	if (IS_ERR(imx6_pcie->vpcie)) {
1196 		if (PTR_ERR(imx6_pcie->vpcie) != -ENODEV)
1197 			return PTR_ERR(imx6_pcie->vpcie);
1198 		imx6_pcie->vpcie = NULL;
1199 	}
1200 
1201 	imx6_pcie->vph = devm_regulator_get_optional(&pdev->dev, "vph");
1202 	if (IS_ERR(imx6_pcie->vph)) {
1203 		if (PTR_ERR(imx6_pcie->vph) != -ENODEV)
1204 			return PTR_ERR(imx6_pcie->vph);
1205 		imx6_pcie->vph = NULL;
1206 	}
1207 
1208 	platform_set_drvdata(pdev, imx6_pcie);
1209 
1210 	ret = imx6_pcie_attach_pd(dev);
1211 	if (ret)
1212 		return ret;
1213 
1214 	ret = dw_pcie_host_init(&pci->pp);
1215 	if (ret < 0)
1216 		return ret;
1217 
1218 	if (pci_msi_enabled()) {
1219 		u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_MSI);
1220 		val = dw_pcie_readw_dbi(pci, offset + PCI_MSI_FLAGS);
1221 		val |= PCI_MSI_FLAGS_ENABLE;
1222 		dw_pcie_writew_dbi(pci, offset + PCI_MSI_FLAGS, val);
1223 	}
1224 
1225 	return 0;
1226 }
1227 
1228 static void imx6_pcie_shutdown(struct platform_device *pdev)
1229 {
1230 	struct imx6_pcie *imx6_pcie = platform_get_drvdata(pdev);
1231 
1232 	/* bring down link, so bootloader gets clean state in case of reboot */
1233 	imx6_pcie_assert_core_reset(imx6_pcie);
1234 }
1235 
1236 static const struct imx6_pcie_drvdata drvdata[] = {
1237 	[IMX6Q] = {
1238 		.variant = IMX6Q,
1239 		.flags = IMX6_PCIE_FLAG_IMX6_PHY |
1240 			 IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE,
1241 		.dbi_length = 0x200,
1242 	},
1243 	[IMX6SX] = {
1244 		.variant = IMX6SX,
1245 		.flags = IMX6_PCIE_FLAG_IMX6_PHY |
1246 			 IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE |
1247 			 IMX6_PCIE_FLAG_SUPPORTS_SUSPEND,
1248 	},
1249 	[IMX6QP] = {
1250 		.variant = IMX6QP,
1251 		.flags = IMX6_PCIE_FLAG_IMX6_PHY |
1252 			 IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE |
1253 			 IMX6_PCIE_FLAG_SUPPORTS_SUSPEND,
1254 		.dbi_length = 0x200,
1255 	},
1256 	[IMX7D] = {
1257 		.variant = IMX7D,
1258 		.flags = IMX6_PCIE_FLAG_SUPPORTS_SUSPEND,
1259 	},
1260 	[IMX8MQ] = {
1261 		.variant = IMX8MQ,
1262 	},
1263 	[IMX8MM] = {
1264 		.variant = IMX8MM,
1265 		.flags = IMX6_PCIE_FLAG_SUPPORTS_SUSPEND,
1266 	},
1267 };
1268 
1269 static const struct of_device_id imx6_pcie_of_match[] = {
1270 	{ .compatible = "fsl,imx6q-pcie",  .data = &drvdata[IMX6Q],  },
1271 	{ .compatible = "fsl,imx6sx-pcie", .data = &drvdata[IMX6SX], },
1272 	{ .compatible = "fsl,imx6qp-pcie", .data = &drvdata[IMX6QP], },
1273 	{ .compatible = "fsl,imx7d-pcie",  .data = &drvdata[IMX7D],  },
1274 	{ .compatible = "fsl,imx8mq-pcie", .data = &drvdata[IMX8MQ], },
1275 	{ .compatible = "fsl,imx8mm-pcie", .data = &drvdata[IMX8MM], },
1276 	{},
1277 };
1278 
1279 static struct platform_driver imx6_pcie_driver = {
1280 	.driver = {
1281 		.name	= "imx6q-pcie",
1282 		.of_match_table = imx6_pcie_of_match,
1283 		.suppress_bind_attrs = true,
1284 		.pm = &imx6_pcie_pm_ops,
1285 		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
1286 	},
1287 	.probe    = imx6_pcie_probe,
1288 	.shutdown = imx6_pcie_shutdown,
1289 };
1290 
1291 static void imx6_pcie_quirk(struct pci_dev *dev)
1292 {
1293 	struct pci_bus *bus = dev->bus;
1294 	struct pcie_port *pp = bus->sysdata;
1295 
1296 	/* Bus parent is the PCI bridge, its parent is this platform driver */
1297 	if (!bus->dev.parent || !bus->dev.parent->parent)
1298 		return;
1299 
1300 	/* Make sure we only quirk devices associated with this driver */
1301 	if (bus->dev.parent->parent->driver != &imx6_pcie_driver.driver)
1302 		return;
1303 
1304 	if (pci_is_root_bus(bus)) {
1305 		struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1306 		struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci);
1307 
1308 		/*
1309 		 * Limit config length to avoid the kernel reading beyond
1310 		 * the register set and causing an abort on i.MX 6Quad
1311 		 */
1312 		if (imx6_pcie->drvdata->dbi_length) {
1313 			dev->cfg_size = imx6_pcie->drvdata->dbi_length;
1314 			dev_info(&dev->dev, "Limiting cfg_size to %d\n",
1315 					dev->cfg_size);
1316 		}
1317 	}
1318 }
1319 DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_VENDOR_ID_SYNOPSYS, 0xabcd,
1320 			PCI_CLASS_BRIDGE_PCI, 8, imx6_pcie_quirk);
1321 
1322 static int __init imx6_pcie_init(void)
1323 {
1324 #ifdef CONFIG_ARM
1325 	/*
1326 	 * Since probe() can be deferred we need to make sure that
1327 	 * hook_fault_code is not called after __init memory is freed
1328 	 * by kernel and since imx6q_pcie_abort_handler() is a no-op,
1329 	 * we can install the handler here without risking it
1330 	 * accessing some uninitialized driver state.
1331 	 */
1332 	hook_fault_code(8, imx6q_pcie_abort_handler, SIGBUS, 0,
1333 			"external abort on non-linefetch");
1334 #endif
1335 
1336 	return platform_driver_register(&imx6_pcie_driver);
1337 }
1338 device_initcall(imx6_pcie_init);
1339