1 // SPDX-License-Identifier: GPL-2.0+
2 /* Copyright (C) 2009 - 2019 Broadcom */
3 
4 #include <linux/bitfield.h>
5 #include <linux/bitops.h>
6 #include <linux/clk.h>
7 #include <linux/compiler.h>
8 #include <linux/delay.h>
9 #include <linux/init.h>
10 #include <linux/interrupt.h>
11 #include <linux/io.h>
12 #include <linux/iopoll.h>
13 #include <linux/ioport.h>
14 #include <linux/irqchip/chained_irq.h>
15 #include <linux/irqdomain.h>
16 #include <linux/kernel.h>
17 #include <linux/list.h>
18 #include <linux/log2.h>
19 #include <linux/module.h>
20 #include <linux/msi.h>
21 #include <linux/of_address.h>
22 #include <linux/of_irq.h>
23 #include <linux/of_pci.h>
24 #include <linux/of_platform.h>
25 #include <linux/pci.h>
26 #include <linux/pci-ecam.h>
27 #include <linux/printk.h>
28 #include <linux/regulator/consumer.h>
29 #include <linux/reset.h>
30 #include <linux/sizes.h>
31 #include <linux/slab.h>
32 #include <linux/string.h>
33 #include <linux/types.h>
34 
35 #include "../pci.h"
36 
37 /* BRCM_PCIE_CAP_REGS - Offset for the mandatory capability config regs */
38 #define BRCM_PCIE_CAP_REGS				0x00ac
39 
40 /* Broadcom STB PCIe Register Offsets */
41 #define PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1				0x0188
42 #define  PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK	0xc
43 #define  PCIE_RC_CFG_VENDOR_SPCIFIC_REG1_LITTLE_ENDIAN			0x0
44 
45 #define PCIE_RC_CFG_PRIV1_ID_VAL3			0x043c
46 #define  PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK	0xffffff
47 
48 #define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY			0x04dc
49 #define  PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_ASPM_SUPPORT_MASK	0xc00
50 
51 #define PCIE_RC_DL_MDIO_ADDR				0x1100
52 #define PCIE_RC_DL_MDIO_WR_DATA				0x1104
53 #define PCIE_RC_DL_MDIO_RD_DATA				0x1108
54 
55 #define PCIE_MISC_MISC_CTRL				0x4008
56 #define  PCIE_MISC_MISC_CTRL_PCIE_RCB_64B_MODE_MASK	0x80
57 #define  PCIE_MISC_MISC_CTRL_PCIE_RCB_MPS_MODE_MASK	0x400
58 #define  PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK		0x1000
59 #define  PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK	0x2000
60 #define  PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK	0x300000
61 
62 #define  PCIE_MISC_MISC_CTRL_SCB0_SIZE_MASK		0xf8000000
63 #define  PCIE_MISC_MISC_CTRL_SCB1_SIZE_MASK		0x07c00000
64 #define  PCIE_MISC_MISC_CTRL_SCB2_SIZE_MASK		0x0000001f
65 #define  SCB_SIZE_MASK(x) PCIE_MISC_MISC_CTRL_SCB ## x ## _SIZE_MASK
66 
67 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO		0x400c
68 #define PCIE_MEM_WIN0_LO(win)	\
69 		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO + ((win) * 8)
70 
71 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI		0x4010
72 #define PCIE_MEM_WIN0_HI(win)	\
73 		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI + ((win) * 8)
74 
75 #define PCIE_MISC_RC_BAR1_CONFIG_LO			0x402c
76 #define  PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK		0x1f
77 
78 #define PCIE_MISC_RC_BAR2_CONFIG_LO			0x4034
79 #define  PCIE_MISC_RC_BAR2_CONFIG_LO_SIZE_MASK		0x1f
80 #define PCIE_MISC_RC_BAR2_CONFIG_HI			0x4038
81 
82 #define PCIE_MISC_RC_BAR3_CONFIG_LO			0x403c
83 #define  PCIE_MISC_RC_BAR3_CONFIG_LO_SIZE_MASK		0x1f
84 
85 #define PCIE_MISC_MSI_BAR_CONFIG_LO			0x4044
86 #define PCIE_MISC_MSI_BAR_CONFIG_HI			0x4048
87 
88 #define PCIE_MISC_MSI_DATA_CONFIG			0x404c
89 #define  PCIE_MISC_MSI_DATA_CONFIG_VAL_32		0xffe06540
90 #define  PCIE_MISC_MSI_DATA_CONFIG_VAL_8		0xfff86540
91 
92 #define PCIE_MISC_PCIE_CTRL				0x4064
93 #define  PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK	0x1
94 #define PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK		0x4
95 
96 #define PCIE_MISC_PCIE_STATUS				0x4068
97 #define  PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK		0x80
98 #define  PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK	0x20
99 #define  PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK	0x10
100 #define  PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK	0x40
101 
102 #define PCIE_MISC_REVISION				0x406c
103 #define  BRCM_PCIE_HW_REV_33				0x0303
104 #define  BRCM_PCIE_HW_REV_3_20				0x0320
105 
106 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT		0x4070
107 #define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK	0xfff00000
108 #define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK	0xfff0
109 #define PCIE_MEM_WIN0_BASE_LIMIT(win)	\
110 		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT + ((win) * 4)
111 
112 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI			0x4080
113 #define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK	0xff
114 #define PCIE_MEM_WIN0_BASE_HI(win)	\
115 		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI + ((win) * 8)
116 
117 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI			0x4084
118 #define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK	0xff
119 #define PCIE_MEM_WIN0_LIMIT_HI(win)	\
120 		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI + ((win) * 8)
121 
122 #define PCIE_MISC_HARD_PCIE_HARD_DEBUG					0x4204
123 #define  PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK	0x2
124 #define  PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK		0x08000000
125 #define  PCIE_BMIPS_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK		0x00800000
126 
127 
128 #define PCIE_INTR2_CPU_BASE		0x4300
129 #define PCIE_MSI_INTR2_BASE		0x4500
130 /* Offsets from PCIE_INTR2_CPU_BASE and PCIE_MSI_INTR2_BASE */
131 #define  MSI_INT_STATUS			0x0
132 #define  MSI_INT_CLR			0x8
133 #define  MSI_INT_MASK_SET		0x10
134 #define  MSI_INT_MASK_CLR		0x14
135 
136 #define PCIE_EXT_CFG_DATA				0x8000
137 #define PCIE_EXT_CFG_INDEX				0x9000
138 
139 #define  PCIE_RGR1_SW_INIT_1_PERST_MASK			0x1
140 #define  PCIE_RGR1_SW_INIT_1_PERST_SHIFT		0x0
141 
142 #define RGR1_SW_INIT_1_INIT_GENERIC_MASK		0x2
143 #define RGR1_SW_INIT_1_INIT_GENERIC_SHIFT		0x1
144 #define RGR1_SW_INIT_1_INIT_7278_MASK			0x1
145 #define RGR1_SW_INIT_1_INIT_7278_SHIFT			0x0
146 
147 /* PCIe parameters */
148 #define BRCM_NUM_PCIE_OUT_WINS		0x4
149 #define BRCM_INT_PCI_MSI_NR		32
150 #define BRCM_INT_PCI_MSI_LEGACY_NR	8
151 #define BRCM_INT_PCI_MSI_SHIFT		0
152 #define BRCM_INT_PCI_MSI_MASK		GENMASK(BRCM_INT_PCI_MSI_NR - 1, 0)
153 #define BRCM_INT_PCI_MSI_LEGACY_MASK	GENMASK(31, \
154 						32 - BRCM_INT_PCI_MSI_LEGACY_NR)
155 
156 /* MSI target addresses */
157 #define BRCM_MSI_TARGET_ADDR_LT_4GB	0x0fffffffcULL
158 #define BRCM_MSI_TARGET_ADDR_GT_4GB	0xffffffffcULL
159 
160 /* MDIO registers */
161 #define MDIO_PORT0			0x0
162 #define MDIO_DATA_MASK			0x7fffffff
163 #define MDIO_PORT_MASK			0xf0000
164 #define MDIO_REGAD_MASK			0xffff
165 #define MDIO_CMD_MASK			0xfff00000
166 #define MDIO_CMD_READ			0x1
167 #define MDIO_CMD_WRITE			0x0
168 #define MDIO_DATA_DONE_MASK		0x80000000
169 #define MDIO_RD_DONE(x)			(((x) & MDIO_DATA_DONE_MASK) ? 1 : 0)
170 #define MDIO_WT_DONE(x)			(((x) & MDIO_DATA_DONE_MASK) ? 0 : 1)
171 #define SSC_REGS_ADDR			0x1100
172 #define SET_ADDR_OFFSET			0x1f
173 #define SSC_CNTL_OFFSET			0x2
174 #define SSC_CNTL_OVRD_EN_MASK		0x8000
175 #define SSC_CNTL_OVRD_VAL_MASK		0x4000
176 #define SSC_STATUS_OFFSET		0x1
177 #define SSC_STATUS_SSC_MASK		0x400
178 #define SSC_STATUS_PLL_LOCK_MASK	0x800
179 #define PCIE_BRCM_MAX_MEMC		3
180 
181 #define IDX_ADDR(pcie)			(pcie->reg_offsets[EXT_CFG_INDEX])
182 #define DATA_ADDR(pcie)			(pcie->reg_offsets[EXT_CFG_DATA])
183 #define PCIE_RGR1_SW_INIT_1(pcie)	(pcie->reg_offsets[RGR1_SW_INIT_1])
184 
185 /* Rescal registers */
186 #define PCIE_DVT_PMU_PCIE_PHY_CTRL				0xc700
187 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS			0x3
188 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_MASK		0x4
189 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_SHIFT	0x2
190 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_MASK		0x2
191 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_SHIFT		0x1
192 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_MASK		0x1
193 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_SHIFT		0x0
194 
195 /* Forward declarations */
196 struct brcm_pcie;
197 
198 enum {
199 	RGR1_SW_INIT_1,
200 	EXT_CFG_INDEX,
201 	EXT_CFG_DATA,
202 };
203 
204 enum {
205 	RGR1_SW_INIT_1_INIT_MASK,
206 	RGR1_SW_INIT_1_INIT_SHIFT,
207 };
208 
209 enum pcie_type {
210 	GENERIC,
211 	BCM7425,
212 	BCM7435,
213 	BCM4908,
214 	BCM7278,
215 	BCM2711,
216 };
217 
218 struct pcie_cfg_data {
219 	const int *offsets;
220 	const enum pcie_type type;
221 	void (*perst_set)(struct brcm_pcie *pcie, u32 val);
222 	void (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val);
223 };
224 
225 struct subdev_regulators {
226 	unsigned int num_supplies;
227 	struct regulator_bulk_data supplies[];
228 };
229 
230 struct brcm_msi {
231 	struct device		*dev;
232 	void __iomem		*base;
233 	struct device_node	*np;
234 	struct irq_domain	*msi_domain;
235 	struct irq_domain	*inner_domain;
236 	struct mutex		lock; /* guards the alloc/free operations */
237 	u64			target_addr;
238 	int			irq;
239 	DECLARE_BITMAP(used, BRCM_INT_PCI_MSI_NR);
240 	bool			legacy;
241 	/* Some chips have MSIs in bits [31..24] of a shared register. */
242 	int			legacy_shift;
243 	int			nr; /* No. of MSI available, depends on chip */
244 	/* This is the base pointer for interrupt status/set/clr regs */
245 	void __iomem		*intr_base;
246 };
247 
248 /* Internal PCIe Host Controller Information.*/
249 struct brcm_pcie {
250 	struct device		*dev;
251 	void __iomem		*base;
252 	struct clk		*clk;
253 	struct device_node	*np;
254 	bool			ssc;
255 	int			gen;
256 	u64			msi_target_addr;
257 	struct brcm_msi		*msi;
258 	const int		*reg_offsets;
259 	enum pcie_type		type;
260 	struct reset_control	*rescal;
261 	struct reset_control	*perst_reset;
262 	int			num_memc;
263 	u64			memc_size[PCIE_BRCM_MAX_MEMC];
264 	u32			hw_rev;
265 	void			(*perst_set)(struct brcm_pcie *pcie, u32 val);
266 	void			(*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val);
267 	struct subdev_regulators *sr;
268 	bool			ep_wakeup_capable;
269 };
270 
271 static inline bool is_bmips(const struct brcm_pcie *pcie)
272 {
273 	return pcie->type == BCM7435 || pcie->type == BCM7425;
274 }
275 
276 /*
277  * This is to convert the size of the inbound "BAR" region to the
278  * non-linear values of PCIE_X_MISC_RC_BAR[123]_CONFIG_LO.SIZE
279  */
280 static int brcm_pcie_encode_ibar_size(u64 size)
281 {
282 	int log2_in = ilog2(size);
283 
284 	if (log2_in >= 12 && log2_in <= 15)
285 		/* Covers 4KB to 32KB (inclusive) */
286 		return (log2_in - 12) + 0x1c;
287 	else if (log2_in >= 16 && log2_in <= 35)
288 		/* Covers 64KB to 32GB, (inclusive) */
289 		return log2_in - 15;
290 	/* Something is awry so disable */
291 	return 0;
292 }
293 
294 static u32 brcm_pcie_mdio_form_pkt(int port, int regad, int cmd)
295 {
296 	u32 pkt = 0;
297 
298 	pkt |= FIELD_PREP(MDIO_PORT_MASK, port);
299 	pkt |= FIELD_PREP(MDIO_REGAD_MASK, regad);
300 	pkt |= FIELD_PREP(MDIO_CMD_MASK, cmd);
301 
302 	return pkt;
303 }
304 
305 /* negative return value indicates error */
306 static int brcm_pcie_mdio_read(void __iomem *base, u8 port, u8 regad, u32 *val)
307 {
308 	u32 data;
309 	int err;
310 
311 	writel(brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_READ),
312 		   base + PCIE_RC_DL_MDIO_ADDR);
313 	readl(base + PCIE_RC_DL_MDIO_ADDR);
314 	err = readl_poll_timeout_atomic(base + PCIE_RC_DL_MDIO_RD_DATA, data,
315 					MDIO_RD_DONE(data), 10, 100);
316 	*val = FIELD_GET(MDIO_DATA_MASK, data);
317 
318 	return err;
319 }
320 
321 /* negative return value indicates error */
322 static int brcm_pcie_mdio_write(void __iomem *base, u8 port,
323 				u8 regad, u16 wrdata)
324 {
325 	u32 data;
326 	int err;
327 
328 	writel(brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_WRITE),
329 		   base + PCIE_RC_DL_MDIO_ADDR);
330 	readl(base + PCIE_RC_DL_MDIO_ADDR);
331 	writel(MDIO_DATA_DONE_MASK | wrdata, base + PCIE_RC_DL_MDIO_WR_DATA);
332 
333 	err = readw_poll_timeout_atomic(base + PCIE_RC_DL_MDIO_WR_DATA, data,
334 					MDIO_WT_DONE(data), 10, 100);
335 	return err;
336 }
337 
338 /*
339  * Configures device for Spread Spectrum Clocking (SSC) mode; a negative
340  * return value indicates error.
341  */
342 static int brcm_pcie_set_ssc(struct brcm_pcie *pcie)
343 {
344 	int pll, ssc;
345 	int ret;
346 	u32 tmp;
347 
348 	ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, SET_ADDR_OFFSET,
349 				   SSC_REGS_ADDR);
350 	if (ret < 0)
351 		return ret;
352 
353 	ret = brcm_pcie_mdio_read(pcie->base, MDIO_PORT0,
354 				  SSC_CNTL_OFFSET, &tmp);
355 	if (ret < 0)
356 		return ret;
357 
358 	u32p_replace_bits(&tmp, 1, SSC_CNTL_OVRD_EN_MASK);
359 	u32p_replace_bits(&tmp, 1, SSC_CNTL_OVRD_VAL_MASK);
360 	ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0,
361 				   SSC_CNTL_OFFSET, tmp);
362 	if (ret < 0)
363 		return ret;
364 
365 	usleep_range(1000, 2000);
366 	ret = brcm_pcie_mdio_read(pcie->base, MDIO_PORT0,
367 				  SSC_STATUS_OFFSET, &tmp);
368 	if (ret < 0)
369 		return ret;
370 
371 	ssc = FIELD_GET(SSC_STATUS_SSC_MASK, tmp);
372 	pll = FIELD_GET(SSC_STATUS_PLL_LOCK_MASK, tmp);
373 
374 	return ssc && pll ? 0 : -EIO;
375 }
376 
377 /* Limits operation to a specific generation (1, 2, or 3) */
378 static void brcm_pcie_set_gen(struct brcm_pcie *pcie, int gen)
379 {
380 	u16 lnkctl2 = readw(pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCTL2);
381 	u32 lnkcap = readl(pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCAP);
382 
383 	lnkcap = (lnkcap & ~PCI_EXP_LNKCAP_SLS) | gen;
384 	writel(lnkcap, pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCAP);
385 
386 	lnkctl2 = (lnkctl2 & ~0xf) | gen;
387 	writew(lnkctl2, pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCTL2);
388 }
389 
390 static void brcm_pcie_set_outbound_win(struct brcm_pcie *pcie,
391 				       unsigned int win, u64 cpu_addr,
392 				       u64 pcie_addr, u64 size)
393 {
394 	u32 cpu_addr_mb_high, limit_addr_mb_high;
395 	phys_addr_t cpu_addr_mb, limit_addr_mb;
396 	int high_addr_shift;
397 	u32 tmp;
398 
399 	/* Set the base of the pcie_addr window */
400 	writel(lower_32_bits(pcie_addr), pcie->base + PCIE_MEM_WIN0_LO(win));
401 	writel(upper_32_bits(pcie_addr), pcie->base + PCIE_MEM_WIN0_HI(win));
402 
403 	/* Write the addr base & limit lower bits (in MBs) */
404 	cpu_addr_mb = cpu_addr / SZ_1M;
405 	limit_addr_mb = (cpu_addr + size - 1) / SZ_1M;
406 
407 	tmp = readl(pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win));
408 	u32p_replace_bits(&tmp, cpu_addr_mb,
409 			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK);
410 	u32p_replace_bits(&tmp, limit_addr_mb,
411 			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK);
412 	writel(tmp, pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win));
413 
414 	if (is_bmips(pcie))
415 		return;
416 
417 	/* Write the cpu & limit addr upper bits */
418 	high_addr_shift =
419 		HWEIGHT32(PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK);
420 
421 	cpu_addr_mb_high = cpu_addr_mb >> high_addr_shift;
422 	tmp = readl(pcie->base + PCIE_MEM_WIN0_BASE_HI(win));
423 	u32p_replace_bits(&tmp, cpu_addr_mb_high,
424 			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK);
425 	writel(tmp, pcie->base + PCIE_MEM_WIN0_BASE_HI(win));
426 
427 	limit_addr_mb_high = limit_addr_mb >> high_addr_shift;
428 	tmp = readl(pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win));
429 	u32p_replace_bits(&tmp, limit_addr_mb_high,
430 			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK);
431 	writel(tmp, pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win));
432 }
433 
434 static struct irq_chip brcm_msi_irq_chip = {
435 	.name            = "BRCM STB PCIe MSI",
436 	.irq_ack         = irq_chip_ack_parent,
437 	.irq_mask        = pci_msi_mask_irq,
438 	.irq_unmask      = pci_msi_unmask_irq,
439 };
440 
441 static struct msi_domain_info brcm_msi_domain_info = {
442 	/* Multi MSI is supported by the controller, but not by this driver */
443 	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
444 		   MSI_FLAG_MULTI_PCI_MSI),
445 	.chip	= &brcm_msi_irq_chip,
446 };
447 
448 static void brcm_pcie_msi_isr(struct irq_desc *desc)
449 {
450 	struct irq_chip *chip = irq_desc_get_chip(desc);
451 	unsigned long status;
452 	struct brcm_msi *msi;
453 	struct device *dev;
454 	u32 bit;
455 
456 	chained_irq_enter(chip, desc);
457 	msi = irq_desc_get_handler_data(desc);
458 	dev = msi->dev;
459 
460 	status = readl(msi->intr_base + MSI_INT_STATUS);
461 	status >>= msi->legacy_shift;
462 
463 	for_each_set_bit(bit, &status, msi->nr) {
464 		int ret;
465 		ret = generic_handle_domain_irq(msi->inner_domain, bit);
466 		if (ret)
467 			dev_dbg(dev, "unexpected MSI\n");
468 	}
469 
470 	chained_irq_exit(chip, desc);
471 }
472 
473 static void brcm_msi_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
474 {
475 	struct brcm_msi *msi = irq_data_get_irq_chip_data(data);
476 
477 	msg->address_lo = lower_32_bits(msi->target_addr);
478 	msg->address_hi = upper_32_bits(msi->target_addr);
479 	msg->data = (0xffff & PCIE_MISC_MSI_DATA_CONFIG_VAL_32) | data->hwirq;
480 }
481 
482 static int brcm_msi_set_affinity(struct irq_data *irq_data,
483 				 const struct cpumask *mask, bool force)
484 {
485 	return -EINVAL;
486 }
487 
488 static void brcm_msi_ack_irq(struct irq_data *data)
489 {
490 	struct brcm_msi *msi = irq_data_get_irq_chip_data(data);
491 	const int shift_amt = data->hwirq + msi->legacy_shift;
492 
493 	writel(1 << shift_amt, msi->intr_base + MSI_INT_CLR);
494 }
495 
496 
497 static struct irq_chip brcm_msi_bottom_irq_chip = {
498 	.name			= "BRCM STB MSI",
499 	.irq_compose_msi_msg	= brcm_msi_compose_msi_msg,
500 	.irq_set_affinity	= brcm_msi_set_affinity,
501 	.irq_ack                = brcm_msi_ack_irq,
502 };
503 
504 static int brcm_msi_alloc(struct brcm_msi *msi, unsigned int nr_irqs)
505 {
506 	int hwirq;
507 
508 	mutex_lock(&msi->lock);
509 	hwirq = bitmap_find_free_region(msi->used, msi->nr,
510 					order_base_2(nr_irqs));
511 	mutex_unlock(&msi->lock);
512 
513 	return hwirq;
514 }
515 
516 static void brcm_msi_free(struct brcm_msi *msi, unsigned long hwirq,
517 			  unsigned int nr_irqs)
518 {
519 	mutex_lock(&msi->lock);
520 	bitmap_release_region(msi->used, hwirq, order_base_2(nr_irqs));
521 	mutex_unlock(&msi->lock);
522 }
523 
524 static int brcm_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
525 				 unsigned int nr_irqs, void *args)
526 {
527 	struct brcm_msi *msi = domain->host_data;
528 	int hwirq, i;
529 
530 	hwirq = brcm_msi_alloc(msi, nr_irqs);
531 
532 	if (hwirq < 0)
533 		return hwirq;
534 
535 	for (i = 0; i < nr_irqs; i++)
536 		irq_domain_set_info(domain, virq + i, hwirq + i,
537 				    &brcm_msi_bottom_irq_chip, domain->host_data,
538 				    handle_edge_irq, NULL, NULL);
539 	return 0;
540 }
541 
542 static void brcm_irq_domain_free(struct irq_domain *domain,
543 				 unsigned int virq, unsigned int nr_irqs)
544 {
545 	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
546 	struct brcm_msi *msi = irq_data_get_irq_chip_data(d);
547 
548 	brcm_msi_free(msi, d->hwirq, nr_irqs);
549 }
550 
551 static const struct irq_domain_ops msi_domain_ops = {
552 	.alloc	= brcm_irq_domain_alloc,
553 	.free	= brcm_irq_domain_free,
554 };
555 
556 static int brcm_allocate_domains(struct brcm_msi *msi)
557 {
558 	struct fwnode_handle *fwnode = of_node_to_fwnode(msi->np);
559 	struct device *dev = msi->dev;
560 
561 	msi->inner_domain = irq_domain_add_linear(NULL, msi->nr, &msi_domain_ops, msi);
562 	if (!msi->inner_domain) {
563 		dev_err(dev, "failed to create IRQ domain\n");
564 		return -ENOMEM;
565 	}
566 
567 	msi->msi_domain = pci_msi_create_irq_domain(fwnode,
568 						    &brcm_msi_domain_info,
569 						    msi->inner_domain);
570 	if (!msi->msi_domain) {
571 		dev_err(dev, "failed to create MSI domain\n");
572 		irq_domain_remove(msi->inner_domain);
573 		return -ENOMEM;
574 	}
575 
576 	return 0;
577 }
578 
579 static void brcm_free_domains(struct brcm_msi *msi)
580 {
581 	irq_domain_remove(msi->msi_domain);
582 	irq_domain_remove(msi->inner_domain);
583 }
584 
585 static void brcm_msi_remove(struct brcm_pcie *pcie)
586 {
587 	struct brcm_msi *msi = pcie->msi;
588 
589 	if (!msi)
590 		return;
591 	irq_set_chained_handler_and_data(msi->irq, NULL, NULL);
592 	brcm_free_domains(msi);
593 }
594 
595 static void brcm_msi_set_regs(struct brcm_msi *msi)
596 {
597 	u32 val = msi->legacy ? BRCM_INT_PCI_MSI_LEGACY_MASK :
598 				BRCM_INT_PCI_MSI_MASK;
599 
600 	writel(val, msi->intr_base + MSI_INT_MASK_CLR);
601 	writel(val, msi->intr_base + MSI_INT_CLR);
602 
603 	/*
604 	 * The 0 bit of PCIE_MISC_MSI_BAR_CONFIG_LO is repurposed to MSI
605 	 * enable, which we set to 1.
606 	 */
607 	writel(lower_32_bits(msi->target_addr) | 0x1,
608 	       msi->base + PCIE_MISC_MSI_BAR_CONFIG_LO);
609 	writel(upper_32_bits(msi->target_addr),
610 	       msi->base + PCIE_MISC_MSI_BAR_CONFIG_HI);
611 
612 	val = msi->legacy ? PCIE_MISC_MSI_DATA_CONFIG_VAL_8 : PCIE_MISC_MSI_DATA_CONFIG_VAL_32;
613 	writel(val, msi->base + PCIE_MISC_MSI_DATA_CONFIG);
614 }
615 
616 static int brcm_pcie_enable_msi(struct brcm_pcie *pcie)
617 {
618 	struct brcm_msi *msi;
619 	int irq, ret;
620 	struct device *dev = pcie->dev;
621 
622 	irq = irq_of_parse_and_map(dev->of_node, 1);
623 	if (irq <= 0) {
624 		dev_err(dev, "cannot map MSI interrupt\n");
625 		return -ENODEV;
626 	}
627 
628 	msi = devm_kzalloc(dev, sizeof(struct brcm_msi), GFP_KERNEL);
629 	if (!msi)
630 		return -ENOMEM;
631 
632 	mutex_init(&msi->lock);
633 	msi->dev = dev;
634 	msi->base = pcie->base;
635 	msi->np = pcie->np;
636 	msi->target_addr = pcie->msi_target_addr;
637 	msi->irq = irq;
638 	msi->legacy = pcie->hw_rev < BRCM_PCIE_HW_REV_33;
639 
640 	/*
641 	 * Sanity check to make sure that the 'used' bitmap in struct brcm_msi
642 	 * is large enough.
643 	 */
644 	BUILD_BUG_ON(BRCM_INT_PCI_MSI_LEGACY_NR > BRCM_INT_PCI_MSI_NR);
645 
646 	if (msi->legacy) {
647 		msi->intr_base = msi->base + PCIE_INTR2_CPU_BASE;
648 		msi->nr = BRCM_INT_PCI_MSI_LEGACY_NR;
649 		msi->legacy_shift = 24;
650 	} else {
651 		msi->intr_base = msi->base + PCIE_MSI_INTR2_BASE;
652 		msi->nr = BRCM_INT_PCI_MSI_NR;
653 		msi->legacy_shift = 0;
654 	}
655 
656 	ret = brcm_allocate_domains(msi);
657 	if (ret)
658 		return ret;
659 
660 	irq_set_chained_handler_and_data(msi->irq, brcm_pcie_msi_isr, msi);
661 
662 	brcm_msi_set_regs(msi);
663 	pcie->msi = msi;
664 
665 	return 0;
666 }
667 
668 /* The controller is capable of serving in both RC and EP roles */
669 static bool brcm_pcie_rc_mode(struct brcm_pcie *pcie)
670 {
671 	void __iomem *base = pcie->base;
672 	u32 val = readl(base + PCIE_MISC_PCIE_STATUS);
673 
674 	return !!FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK, val);
675 }
676 
677 static bool brcm_pcie_link_up(struct brcm_pcie *pcie)
678 {
679 	u32 val = readl(pcie->base + PCIE_MISC_PCIE_STATUS);
680 	u32 dla = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK, val);
681 	u32 plu = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK, val);
682 
683 	return dla && plu;
684 }
685 
686 static void __iomem *brcm_pcie_map_bus(struct pci_bus *bus,
687 				       unsigned int devfn, int where)
688 {
689 	struct brcm_pcie *pcie = bus->sysdata;
690 	void __iomem *base = pcie->base;
691 	int idx;
692 
693 	/* Accesses to the RC go right to the RC registers if !devfn */
694 	if (pci_is_root_bus(bus))
695 		return devfn ? NULL : base + PCIE_ECAM_REG(where);
696 
697 	/* An access to our HW w/o link-up will cause a CPU Abort */
698 	if (!brcm_pcie_link_up(pcie))
699 		return NULL;
700 
701 	/* For devices, write to the config space index register */
702 	idx = PCIE_ECAM_OFFSET(bus->number, devfn, 0);
703 	writel(idx, pcie->base + PCIE_EXT_CFG_INDEX);
704 	return base + PCIE_EXT_CFG_DATA + PCIE_ECAM_REG(where);
705 }
706 
707 static void __iomem *brcm7425_pcie_map_bus(struct pci_bus *bus,
708 					   unsigned int devfn, int where)
709 {
710 	struct brcm_pcie *pcie = bus->sysdata;
711 	void __iomem *base = pcie->base;
712 	int idx;
713 
714 	/* Accesses to the RC go right to the RC registers if !devfn */
715 	if (pci_is_root_bus(bus))
716 		return devfn ? NULL : base + PCIE_ECAM_REG(where);
717 
718 	/* An access to our HW w/o link-up will cause a CPU Abort */
719 	if (!brcm_pcie_link_up(pcie))
720 		return NULL;
721 
722 	/* For devices, write to the config space index register */
723 	idx = PCIE_ECAM_OFFSET(bus->number, devfn, where);
724 	writel(idx, base + IDX_ADDR(pcie));
725 	return base + DATA_ADDR(pcie);
726 }
727 
728 static void brcm_pcie_bridge_sw_init_set_generic(struct brcm_pcie *pcie, u32 val)
729 {
730 	u32 tmp, mask =  RGR1_SW_INIT_1_INIT_GENERIC_MASK;
731 	u32 shift = RGR1_SW_INIT_1_INIT_GENERIC_SHIFT;
732 
733 	tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
734 	tmp = (tmp & ~mask) | ((val << shift) & mask);
735 	writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
736 }
737 
738 static void brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 val)
739 {
740 	u32 tmp, mask =  RGR1_SW_INIT_1_INIT_7278_MASK;
741 	u32 shift = RGR1_SW_INIT_1_INIT_7278_SHIFT;
742 
743 	tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
744 	tmp = (tmp & ~mask) | ((val << shift) & mask);
745 	writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
746 }
747 
748 static void brcm_pcie_perst_set_4908(struct brcm_pcie *pcie, u32 val)
749 {
750 	if (WARN_ONCE(!pcie->perst_reset, "missing PERST# reset controller\n"))
751 		return;
752 
753 	if (val)
754 		reset_control_assert(pcie->perst_reset);
755 	else
756 		reset_control_deassert(pcie->perst_reset);
757 }
758 
759 static void brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val)
760 {
761 	u32 tmp;
762 
763 	/* Perst bit has moved and assert value is 0 */
764 	tmp = readl(pcie->base + PCIE_MISC_PCIE_CTRL);
765 	u32p_replace_bits(&tmp, !val, PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK);
766 	writel(tmp, pcie->base +  PCIE_MISC_PCIE_CTRL);
767 }
768 
769 static void brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, u32 val)
770 {
771 	u32 tmp;
772 
773 	tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
774 	u32p_replace_bits(&tmp, val, PCIE_RGR1_SW_INIT_1_PERST_MASK);
775 	writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
776 }
777 
778 static int brcm_pcie_get_rc_bar2_size_and_offset(struct brcm_pcie *pcie,
779 							u64 *rc_bar2_size,
780 							u64 *rc_bar2_offset)
781 {
782 	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
783 	struct resource_entry *entry;
784 	struct device *dev = pcie->dev;
785 	u64 lowest_pcie_addr = ~(u64)0;
786 	int ret, i = 0;
787 	u64 size = 0;
788 
789 	resource_list_for_each_entry(entry, &bridge->dma_ranges) {
790 		u64 pcie_beg = entry->res->start - entry->offset;
791 
792 		size += entry->res->end - entry->res->start + 1;
793 		if (pcie_beg < lowest_pcie_addr)
794 			lowest_pcie_addr = pcie_beg;
795 	}
796 
797 	if (lowest_pcie_addr == ~(u64)0) {
798 		dev_err(dev, "DT node has no dma-ranges\n");
799 		return -EINVAL;
800 	}
801 
802 	ret = of_property_read_variable_u64_array(pcie->np, "brcm,scb-sizes", pcie->memc_size, 1,
803 						  PCIE_BRCM_MAX_MEMC);
804 
805 	if (ret <= 0) {
806 		/* Make an educated guess */
807 		pcie->num_memc = 1;
808 		pcie->memc_size[0] = 1ULL << fls64(size - 1);
809 	} else {
810 		pcie->num_memc = ret;
811 	}
812 
813 	/* Each memc is viewed through a "port" that is a power of 2 */
814 	for (i = 0, size = 0; i < pcie->num_memc; i++)
815 		size += pcie->memc_size[i];
816 
817 	/* System memory starts at this address in PCIe-space */
818 	*rc_bar2_offset = lowest_pcie_addr;
819 	/* The sum of all memc views must also be a power of 2 */
820 	*rc_bar2_size = 1ULL << fls64(size - 1);
821 
822 	/*
823 	 * We validate the inbound memory view even though we should trust
824 	 * whatever the device-tree provides. This is because of an HW issue on
825 	 * early Raspberry Pi 4's revisions (bcm2711). It turns out its
826 	 * firmware has to dynamically edit dma-ranges due to a bug on the
827 	 * PCIe controller integration, which prohibits any access above the
828 	 * lower 3GB of memory. Given this, we decided to keep the dma-ranges
829 	 * in check, avoiding hard to debug device-tree related issues in the
830 	 * future:
831 	 *
832 	 * The PCIe host controller by design must set the inbound viewport to
833 	 * be a contiguous arrangement of all of the system's memory.  In
834 	 * addition, its size mut be a power of two.  To further complicate
835 	 * matters, the viewport must start on a pcie-address that is aligned
836 	 * on a multiple of its size.  If a portion of the viewport does not
837 	 * represent system memory -- e.g. 3GB of memory requires a 4GB
838 	 * viewport -- we can map the outbound memory in or after 3GB and even
839 	 * though the viewport will overlap the outbound memory the controller
840 	 * will know to send outbound memory downstream and everything else
841 	 * upstream.
842 	 *
843 	 * For example:
844 	 *
845 	 * - The best-case scenario, memory up to 3GB, is to place the inbound
846 	 *   region in the first 4GB of pcie-space, as some legacy devices can
847 	 *   only address 32bits. We would also like to put the MSI under 4GB
848 	 *   as well, since some devices require a 32bit MSI target address.
849 	 *
850 	 * - If the system memory is 4GB or larger we cannot start the inbound
851 	 *   region at location 0 (since we have to allow some space for
852 	 *   outbound memory @ 3GB). So instead it will  start at the 1x
853 	 *   multiple of its size
854 	 */
855 	if (!*rc_bar2_size || (*rc_bar2_offset & (*rc_bar2_size - 1)) ||
856 	    (*rc_bar2_offset < SZ_4G && *rc_bar2_offset > SZ_2G)) {
857 		dev_err(dev, "Invalid rc_bar2_offset/size: size 0x%llx, off 0x%llx\n",
858 			*rc_bar2_size, *rc_bar2_offset);
859 		return -EINVAL;
860 	}
861 
862 	return 0;
863 }
864 
865 static int brcm_pcie_setup(struct brcm_pcie *pcie)
866 {
867 	u64 rc_bar2_offset, rc_bar2_size;
868 	void __iomem *base = pcie->base;
869 	struct pci_host_bridge *bridge;
870 	struct resource_entry *entry;
871 	u32 tmp, burst, aspm_support;
872 	int num_out_wins = 0;
873 	int ret, memc;
874 
875 	/* Reset the bridge */
876 	pcie->bridge_sw_init_set(pcie, 1);
877 	usleep_range(100, 200);
878 
879 	/* Take the bridge out of reset */
880 	pcie->bridge_sw_init_set(pcie, 0);
881 
882 	tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
883 	if (is_bmips(pcie))
884 		tmp &= ~PCIE_BMIPS_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK;
885 	else
886 		tmp &= ~PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK;
887 	writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
888 	/* Wait for SerDes to be stable */
889 	usleep_range(100, 200);
890 
891 	/*
892 	 * SCB_MAX_BURST_SIZE is a two bit field.  For GENERIC chips it
893 	 * is encoded as 0=128, 1=256, 2=512, 3=Rsvd, for BCM7278 it
894 	 * is encoded as 0=Rsvd, 1=128, 2=256, 3=512.
895 	 */
896 	if (is_bmips(pcie))
897 		burst = 0x1; /* 256 bytes */
898 	else if (pcie->type == BCM2711)
899 		burst = 0x0; /* 128 bytes */
900 	else if (pcie->type == BCM7278)
901 		burst = 0x3; /* 512 bytes */
902 	else
903 		burst = 0x2; /* 512 bytes */
904 
905 	/*
906 	 * Set SCB_MAX_BURST_SIZE, CFG_READ_UR_MODE, SCB_ACCESS_EN,
907 	 * RCB_MPS_MODE, RCB_64B_MODE
908 	 */
909 	tmp = readl(base + PCIE_MISC_MISC_CTRL);
910 	u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK);
911 	u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK);
912 	u32p_replace_bits(&tmp, burst, PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK);
913 	u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_MPS_MODE_MASK);
914 	u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_64B_MODE_MASK);
915 	writel(tmp, base + PCIE_MISC_MISC_CTRL);
916 
917 	ret = brcm_pcie_get_rc_bar2_size_and_offset(pcie, &rc_bar2_size,
918 						    &rc_bar2_offset);
919 	if (ret)
920 		return ret;
921 
922 	tmp = lower_32_bits(rc_bar2_offset);
923 	u32p_replace_bits(&tmp, brcm_pcie_encode_ibar_size(rc_bar2_size),
924 			  PCIE_MISC_RC_BAR2_CONFIG_LO_SIZE_MASK);
925 	writel(tmp, base + PCIE_MISC_RC_BAR2_CONFIG_LO);
926 	writel(upper_32_bits(rc_bar2_offset),
927 	       base + PCIE_MISC_RC_BAR2_CONFIG_HI);
928 
929 	tmp = readl(base + PCIE_MISC_MISC_CTRL);
930 	for (memc = 0; memc < pcie->num_memc; memc++) {
931 		u32 scb_size_val = ilog2(pcie->memc_size[memc]) - 15;
932 
933 		if (memc == 0)
934 			u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(0));
935 		else if (memc == 1)
936 			u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(1));
937 		else if (memc == 2)
938 			u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(2));
939 	}
940 	writel(tmp, base + PCIE_MISC_MISC_CTRL);
941 
942 	/*
943 	 * We ideally want the MSI target address to be located in the 32bit
944 	 * addressable memory area. Some devices might depend on it. This is
945 	 * possible either when the inbound window is located above the lower
946 	 * 4GB or when the inbound area is smaller than 4GB (taking into
947 	 * account the rounding-up we're forced to perform).
948 	 */
949 	if (rc_bar2_offset >= SZ_4G || (rc_bar2_size + rc_bar2_offset) < SZ_4G)
950 		pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_LT_4GB;
951 	else
952 		pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_GT_4GB;
953 
954 	if (!brcm_pcie_rc_mode(pcie)) {
955 		dev_err(pcie->dev, "PCIe RC controller misconfigured as Endpoint\n");
956 		return -EINVAL;
957 	}
958 
959 	/* disable the PCIe->GISB memory window (RC_BAR1) */
960 	tmp = readl(base + PCIE_MISC_RC_BAR1_CONFIG_LO);
961 	tmp &= ~PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK;
962 	writel(tmp, base + PCIE_MISC_RC_BAR1_CONFIG_LO);
963 
964 	/* disable the PCIe->SCB memory window (RC_BAR3) */
965 	tmp = readl(base + PCIE_MISC_RC_BAR3_CONFIG_LO);
966 	tmp &= ~PCIE_MISC_RC_BAR3_CONFIG_LO_SIZE_MASK;
967 	writel(tmp, base + PCIE_MISC_RC_BAR3_CONFIG_LO);
968 
969 	/* Don't advertise L0s capability if 'aspm-no-l0s' */
970 	aspm_support = PCIE_LINK_STATE_L1;
971 	if (!of_property_read_bool(pcie->np, "aspm-no-l0s"))
972 		aspm_support |= PCIE_LINK_STATE_L0S;
973 	tmp = readl(base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY);
974 	u32p_replace_bits(&tmp, aspm_support,
975 		PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_ASPM_SUPPORT_MASK);
976 	writel(tmp, base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY);
977 
978 	/*
979 	 * For config space accesses on the RC, show the right class for
980 	 * a PCIe-PCIe bridge (the default setting is to be EP mode).
981 	 */
982 	tmp = readl(base + PCIE_RC_CFG_PRIV1_ID_VAL3);
983 	u32p_replace_bits(&tmp, 0x060400,
984 			  PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK);
985 	writel(tmp, base + PCIE_RC_CFG_PRIV1_ID_VAL3);
986 
987 	bridge = pci_host_bridge_from_priv(pcie);
988 	resource_list_for_each_entry(entry, &bridge->windows) {
989 		struct resource *res = entry->res;
990 
991 		if (resource_type(res) != IORESOURCE_MEM)
992 			continue;
993 
994 		if (num_out_wins >= BRCM_NUM_PCIE_OUT_WINS) {
995 			dev_err(pcie->dev, "too many outbound wins\n");
996 			return -EINVAL;
997 		}
998 
999 		if (is_bmips(pcie)) {
1000 			u64 start = res->start;
1001 			unsigned int j, nwins = resource_size(res) / SZ_128M;
1002 
1003 			/* bmips PCIe outbound windows have a 128MB max size */
1004 			if (nwins > BRCM_NUM_PCIE_OUT_WINS)
1005 				nwins = BRCM_NUM_PCIE_OUT_WINS;
1006 			for (j = 0; j < nwins; j++, start += SZ_128M)
1007 				brcm_pcie_set_outbound_win(pcie, j, start,
1008 							   start - entry->offset,
1009 							   SZ_128M);
1010 			break;
1011 		}
1012 		brcm_pcie_set_outbound_win(pcie, num_out_wins, res->start,
1013 					   res->start - entry->offset,
1014 					   resource_size(res));
1015 		num_out_wins++;
1016 	}
1017 
1018 	/* PCIe->SCB endian mode for BAR */
1019 	tmp = readl(base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1);
1020 	u32p_replace_bits(&tmp, PCIE_RC_CFG_VENDOR_SPCIFIC_REG1_LITTLE_ENDIAN,
1021 		PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK);
1022 	writel(tmp, base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1);
1023 
1024 	return 0;
1025 }
1026 
1027 static int brcm_pcie_start_link(struct brcm_pcie *pcie)
1028 {
1029 	struct device *dev = pcie->dev;
1030 	void __iomem *base = pcie->base;
1031 	u16 nlw, cls, lnksta;
1032 	bool ssc_good = false;
1033 	u32 tmp;
1034 	int ret, i;
1035 
1036 	/* Unassert the fundamental reset */
1037 	pcie->perst_set(pcie, 0);
1038 
1039 	/*
1040 	 * Wait for 100ms after PERST# deassertion; see PCIe CEM specification
1041 	 * sections 2.2, PCIe r5.0, 6.6.1.
1042 	 */
1043 	msleep(100);
1044 
1045 	/*
1046 	 * Give the RC/EP even more time to wake up, before trying to
1047 	 * configure RC.  Intermittently check status for link-up, up to a
1048 	 * total of 100ms.
1049 	 */
1050 	for (i = 0; i < 100 && !brcm_pcie_link_up(pcie); i += 5)
1051 		msleep(5);
1052 
1053 	if (!brcm_pcie_link_up(pcie)) {
1054 		dev_err(dev, "link down\n");
1055 		return -ENODEV;
1056 	}
1057 
1058 	if (pcie->gen)
1059 		brcm_pcie_set_gen(pcie, pcie->gen);
1060 
1061 	if (pcie->ssc) {
1062 		ret = brcm_pcie_set_ssc(pcie);
1063 		if (ret == 0)
1064 			ssc_good = true;
1065 		else
1066 			dev_err(dev, "failed attempt to enter ssc mode\n");
1067 	}
1068 
1069 	lnksta = readw(base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKSTA);
1070 	cls = FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta);
1071 	nlw = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta);
1072 	dev_info(dev, "link up, %s x%u %s\n",
1073 		 pci_speed_string(pcie_link_speed[cls]), nlw,
1074 		 ssc_good ? "(SSC)" : "(!SSC)");
1075 
1076 	/*
1077 	 * Refclk from RC should be gated with CLKREQ# input when ASPM L0s,L1
1078 	 * is enabled => setting the CLKREQ_DEBUG_ENABLE field to 1.
1079 	 */
1080 	tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1081 	tmp |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK;
1082 	writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1083 
1084 	return 0;
1085 }
1086 
1087 static const char * const supplies[] = {
1088 	"vpcie3v3",
1089 	"vpcie3v3aux",
1090 	"vpcie12v",
1091 };
1092 
1093 static void *alloc_subdev_regulators(struct device *dev)
1094 {
1095 	const size_t size = sizeof(struct subdev_regulators) +
1096 		sizeof(struct regulator_bulk_data) * ARRAY_SIZE(supplies);
1097 	struct subdev_regulators *sr;
1098 	int i;
1099 
1100 	sr = devm_kzalloc(dev, size, GFP_KERNEL);
1101 	if (sr) {
1102 		sr->num_supplies = ARRAY_SIZE(supplies);
1103 		for (i = 0; i < ARRAY_SIZE(supplies); i++)
1104 			sr->supplies[i].supply = supplies[i];
1105 	}
1106 
1107 	return sr;
1108 }
1109 
1110 static int brcm_pcie_add_bus(struct pci_bus *bus)
1111 {
1112 	struct brcm_pcie *pcie = bus->sysdata;
1113 	struct device *dev = &bus->dev;
1114 	struct subdev_regulators *sr;
1115 	int ret;
1116 
1117 	if (!bus->parent || !pci_is_root_bus(bus->parent))
1118 		return 0;
1119 
1120 	if (dev->of_node) {
1121 		sr = alloc_subdev_regulators(dev);
1122 		if (!sr) {
1123 			dev_info(dev, "Can't allocate regulators for downstream device\n");
1124 			goto no_regulators;
1125 		}
1126 
1127 		pcie->sr = sr;
1128 
1129 		ret = regulator_bulk_get(dev, sr->num_supplies, sr->supplies);
1130 		if (ret) {
1131 			dev_info(dev, "No regulators for downstream device\n");
1132 			goto no_regulators;
1133 		}
1134 
1135 		ret = regulator_bulk_enable(sr->num_supplies, sr->supplies);
1136 		if (ret) {
1137 			dev_err(dev, "Can't enable regulators for downstream device\n");
1138 			regulator_bulk_free(sr->num_supplies, sr->supplies);
1139 			pcie->sr = NULL;
1140 		}
1141 	}
1142 
1143 no_regulators:
1144 	brcm_pcie_start_link(pcie);
1145 	return 0;
1146 }
1147 
1148 static void brcm_pcie_remove_bus(struct pci_bus *bus)
1149 {
1150 	struct brcm_pcie *pcie = bus->sysdata;
1151 	struct subdev_regulators *sr = pcie->sr;
1152 	struct device *dev = &bus->dev;
1153 
1154 	if (!sr)
1155 		return;
1156 
1157 	if (regulator_bulk_disable(sr->num_supplies, sr->supplies))
1158 		dev_err(dev, "Failed to disable regulators for downstream device\n");
1159 	regulator_bulk_free(sr->num_supplies, sr->supplies);
1160 	pcie->sr = NULL;
1161 }
1162 
1163 /* L23 is a low-power PCIe link state */
1164 static void brcm_pcie_enter_l23(struct brcm_pcie *pcie)
1165 {
1166 	void __iomem *base = pcie->base;
1167 	int l23, i;
1168 	u32 tmp;
1169 
1170 	/* Assert request for L23 */
1171 	tmp = readl(base + PCIE_MISC_PCIE_CTRL);
1172 	u32p_replace_bits(&tmp, 1, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK);
1173 	writel(tmp, base + PCIE_MISC_PCIE_CTRL);
1174 
1175 	/* Wait up to 36 msec for L23 */
1176 	tmp = readl(base + PCIE_MISC_PCIE_STATUS);
1177 	l23 = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK, tmp);
1178 	for (i = 0; i < 15 && !l23; i++) {
1179 		usleep_range(2000, 2400);
1180 		tmp = readl(base + PCIE_MISC_PCIE_STATUS);
1181 		l23 = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK,
1182 				tmp);
1183 	}
1184 
1185 	if (!l23)
1186 		dev_err(pcie->dev, "failed to enter low-power link state\n");
1187 }
1188 
1189 static int brcm_phy_cntl(struct brcm_pcie *pcie, const int start)
1190 {
1191 	static const u32 shifts[PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS] = {
1192 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_SHIFT,
1193 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_SHIFT,
1194 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_SHIFT,};
1195 	static const u32 masks[PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS] = {
1196 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_MASK,
1197 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_MASK,
1198 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_MASK,};
1199 	const int beg = start ? 0 : PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS - 1;
1200 	const int end = start ? PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS : -1;
1201 	u32 tmp, combined_mask = 0;
1202 	u32 val;
1203 	void __iomem *base = pcie->base;
1204 	int i, ret;
1205 
1206 	for (i = beg; i != end; start ? i++ : i--) {
1207 		val = start ? BIT_MASK(shifts[i]) : 0;
1208 		tmp = readl(base + PCIE_DVT_PMU_PCIE_PHY_CTRL);
1209 		tmp = (tmp & ~masks[i]) | (val & masks[i]);
1210 		writel(tmp, base + PCIE_DVT_PMU_PCIE_PHY_CTRL);
1211 		usleep_range(50, 200);
1212 		combined_mask |= masks[i];
1213 	}
1214 
1215 	tmp = readl(base + PCIE_DVT_PMU_PCIE_PHY_CTRL);
1216 	val = start ? combined_mask : 0;
1217 
1218 	ret = (tmp & combined_mask) == val ? 0 : -EIO;
1219 	if (ret)
1220 		dev_err(pcie->dev, "failed to %s phy\n", (start ? "start" : "stop"));
1221 
1222 	return ret;
1223 }
1224 
1225 static inline int brcm_phy_start(struct brcm_pcie *pcie)
1226 {
1227 	return pcie->rescal ? brcm_phy_cntl(pcie, 1) : 0;
1228 }
1229 
1230 static inline int brcm_phy_stop(struct brcm_pcie *pcie)
1231 {
1232 	return pcie->rescal ? brcm_phy_cntl(pcie, 0) : 0;
1233 }
1234 
1235 static void brcm_pcie_turn_off(struct brcm_pcie *pcie)
1236 {
1237 	void __iomem *base = pcie->base;
1238 	int tmp;
1239 
1240 	if (brcm_pcie_link_up(pcie))
1241 		brcm_pcie_enter_l23(pcie);
1242 	/* Assert fundamental reset */
1243 	pcie->perst_set(pcie, 1);
1244 
1245 	/* Deassert request for L23 in case it was asserted */
1246 	tmp = readl(base + PCIE_MISC_PCIE_CTRL);
1247 	u32p_replace_bits(&tmp, 0, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK);
1248 	writel(tmp, base + PCIE_MISC_PCIE_CTRL);
1249 
1250 	/* Turn off SerDes */
1251 	tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1252 	u32p_replace_bits(&tmp, 1, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK);
1253 	writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1254 
1255 	/* Shutdown PCIe bridge */
1256 	pcie->bridge_sw_init_set(pcie, 1);
1257 }
1258 
1259 static int pci_dev_may_wakeup(struct pci_dev *dev, void *data)
1260 {
1261 	bool *ret = data;
1262 
1263 	if (device_may_wakeup(&dev->dev)) {
1264 		*ret = true;
1265 		dev_info(&dev->dev, "Possible wake-up device; regulators will not be disabled\n");
1266 	}
1267 	return (int) *ret;
1268 }
1269 
1270 static int brcm_pcie_suspend_noirq(struct device *dev)
1271 {
1272 	struct brcm_pcie *pcie = dev_get_drvdata(dev);
1273 	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
1274 	int ret;
1275 
1276 	brcm_pcie_turn_off(pcie);
1277 	/*
1278 	 * If brcm_phy_stop() returns an error, just dev_err(). If we
1279 	 * return the error it will cause the suspend to fail and this is a
1280 	 * forgivable offense that will probably be erased on resume.
1281 	 */
1282 	if (brcm_phy_stop(pcie))
1283 		dev_err(dev, "Could not stop phy for suspend\n");
1284 
1285 	ret = reset_control_rearm(pcie->rescal);
1286 	if (ret) {
1287 		dev_err(dev, "Could not rearm rescal reset\n");
1288 		return ret;
1289 	}
1290 
1291 	if (pcie->sr) {
1292 		/*
1293 		 * Now turn off the regulators, but if at least one
1294 		 * downstream device is enabled as a wake-up source, do not
1295 		 * turn off regulators.
1296 		 */
1297 		pcie->ep_wakeup_capable = false;
1298 		pci_walk_bus(bridge->bus, pci_dev_may_wakeup,
1299 			     &pcie->ep_wakeup_capable);
1300 		if (!pcie->ep_wakeup_capable) {
1301 			ret = regulator_bulk_disable(pcie->sr->num_supplies,
1302 						     pcie->sr->supplies);
1303 			if (ret) {
1304 				dev_err(dev, "Could not turn off regulators\n");
1305 				reset_control_reset(pcie->rescal);
1306 				return ret;
1307 			}
1308 		}
1309 	}
1310 	clk_disable_unprepare(pcie->clk);
1311 
1312 	return 0;
1313 }
1314 
1315 static int brcm_pcie_resume_noirq(struct device *dev)
1316 {
1317 	struct brcm_pcie *pcie = dev_get_drvdata(dev);
1318 	void __iomem *base;
1319 	u32 tmp;
1320 	int ret;
1321 
1322 	base = pcie->base;
1323 	ret = clk_prepare_enable(pcie->clk);
1324 	if (ret)
1325 		return ret;
1326 
1327 	ret = reset_control_reset(pcie->rescal);
1328 	if (ret)
1329 		goto err_disable_clk;
1330 
1331 	ret = brcm_phy_start(pcie);
1332 	if (ret)
1333 		goto err_reset;
1334 
1335 	/* Take bridge out of reset so we can access the SERDES reg */
1336 	pcie->bridge_sw_init_set(pcie, 0);
1337 
1338 	/* SERDES_IDDQ = 0 */
1339 	tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1340 	u32p_replace_bits(&tmp, 0, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK);
1341 	writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1342 
1343 	/* wait for serdes to be stable */
1344 	udelay(100);
1345 
1346 	ret = brcm_pcie_setup(pcie);
1347 	if (ret)
1348 		goto err_reset;
1349 
1350 	if (pcie->sr) {
1351 		if (pcie->ep_wakeup_capable) {
1352 			/*
1353 			 * We are resuming from a suspend.  In the suspend we
1354 			 * did not disable the power supplies, so there is
1355 			 * no need to enable them (and falsely increase their
1356 			 * usage count).
1357 			 */
1358 			pcie->ep_wakeup_capable = false;
1359 		} else {
1360 			ret = regulator_bulk_enable(pcie->sr->num_supplies,
1361 						    pcie->sr->supplies);
1362 			if (ret) {
1363 				dev_err(dev, "Could not turn on regulators\n");
1364 				goto err_reset;
1365 			}
1366 		}
1367 	}
1368 
1369 	ret = brcm_pcie_start_link(pcie);
1370 	if (ret)
1371 		goto err_regulator;
1372 
1373 	if (pcie->msi)
1374 		brcm_msi_set_regs(pcie->msi);
1375 
1376 	return 0;
1377 
1378 err_regulator:
1379 	if (pcie->sr)
1380 		regulator_bulk_disable(pcie->sr->num_supplies, pcie->sr->supplies);
1381 err_reset:
1382 	reset_control_rearm(pcie->rescal);
1383 err_disable_clk:
1384 	clk_disable_unprepare(pcie->clk);
1385 	return ret;
1386 }
1387 
1388 static void __brcm_pcie_remove(struct brcm_pcie *pcie)
1389 {
1390 	brcm_msi_remove(pcie);
1391 	brcm_pcie_turn_off(pcie);
1392 	if (brcm_phy_stop(pcie))
1393 		dev_err(pcie->dev, "Could not stop phy\n");
1394 	if (reset_control_rearm(pcie->rescal))
1395 		dev_err(pcie->dev, "Could not rearm rescal reset\n");
1396 	clk_disable_unprepare(pcie->clk);
1397 }
1398 
1399 static void brcm_pcie_remove(struct platform_device *pdev)
1400 {
1401 	struct brcm_pcie *pcie = platform_get_drvdata(pdev);
1402 	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
1403 
1404 	pci_stop_root_bus(bridge->bus);
1405 	pci_remove_root_bus(bridge->bus);
1406 	__brcm_pcie_remove(pcie);
1407 }
1408 
1409 static const int pcie_offsets[] = {
1410 	[RGR1_SW_INIT_1] = 0x9210,
1411 	[EXT_CFG_INDEX]  = 0x9000,
1412 	[EXT_CFG_DATA]   = 0x9004,
1413 };
1414 
1415 static const int pcie_offsets_bmips_7425[] = {
1416 	[RGR1_SW_INIT_1] = 0x8010,
1417 	[EXT_CFG_INDEX]  = 0x8300,
1418 	[EXT_CFG_DATA]   = 0x8304,
1419 };
1420 
1421 static const struct pcie_cfg_data generic_cfg = {
1422 	.offsets	= pcie_offsets,
1423 	.type		= GENERIC,
1424 	.perst_set	= brcm_pcie_perst_set_generic,
1425 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1426 };
1427 
1428 static const struct pcie_cfg_data bcm7425_cfg = {
1429 	.offsets	= pcie_offsets_bmips_7425,
1430 	.type		= BCM7425,
1431 	.perst_set	= brcm_pcie_perst_set_generic,
1432 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1433 };
1434 
1435 static const struct pcie_cfg_data bcm7435_cfg = {
1436 	.offsets	= pcie_offsets,
1437 	.type		= BCM7435,
1438 	.perst_set	= brcm_pcie_perst_set_generic,
1439 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1440 };
1441 
1442 static const struct pcie_cfg_data bcm4908_cfg = {
1443 	.offsets	= pcie_offsets,
1444 	.type		= BCM4908,
1445 	.perst_set	= brcm_pcie_perst_set_4908,
1446 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1447 };
1448 
1449 static const int pcie_offset_bcm7278[] = {
1450 	[RGR1_SW_INIT_1] = 0xc010,
1451 	[EXT_CFG_INDEX] = 0x9000,
1452 	[EXT_CFG_DATA] = 0x9004,
1453 };
1454 
1455 static const struct pcie_cfg_data bcm7278_cfg = {
1456 	.offsets	= pcie_offset_bcm7278,
1457 	.type		= BCM7278,
1458 	.perst_set	= brcm_pcie_perst_set_7278,
1459 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278,
1460 };
1461 
1462 static const struct pcie_cfg_data bcm2711_cfg = {
1463 	.offsets	= pcie_offsets,
1464 	.type		= BCM2711,
1465 	.perst_set	= brcm_pcie_perst_set_generic,
1466 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1467 };
1468 
1469 static const struct of_device_id brcm_pcie_match[] = {
1470 	{ .compatible = "brcm,bcm2711-pcie", .data = &bcm2711_cfg },
1471 	{ .compatible = "brcm,bcm4908-pcie", .data = &bcm4908_cfg },
1472 	{ .compatible = "brcm,bcm7211-pcie", .data = &generic_cfg },
1473 	{ .compatible = "brcm,bcm7278-pcie", .data = &bcm7278_cfg },
1474 	{ .compatible = "brcm,bcm7216-pcie", .data = &bcm7278_cfg },
1475 	{ .compatible = "brcm,bcm7445-pcie", .data = &generic_cfg },
1476 	{ .compatible = "brcm,bcm7435-pcie", .data = &bcm7435_cfg },
1477 	{ .compatible = "brcm,bcm7425-pcie", .data = &bcm7425_cfg },
1478 	{},
1479 };
1480 
1481 static struct pci_ops brcm_pcie_ops = {
1482 	.map_bus = brcm_pcie_map_bus,
1483 	.read = pci_generic_config_read,
1484 	.write = pci_generic_config_write,
1485 	.add_bus = brcm_pcie_add_bus,
1486 	.remove_bus = brcm_pcie_remove_bus,
1487 };
1488 
1489 static struct pci_ops brcm7425_pcie_ops = {
1490 	.map_bus = brcm7425_pcie_map_bus,
1491 	.read = pci_generic_config_read32,
1492 	.write = pci_generic_config_write32,
1493 	.add_bus = brcm_pcie_add_bus,
1494 	.remove_bus = brcm_pcie_remove_bus,
1495 };
1496 
1497 static int brcm_pcie_probe(struct platform_device *pdev)
1498 {
1499 	struct device_node *np = pdev->dev.of_node, *msi_np;
1500 	struct pci_host_bridge *bridge;
1501 	const struct pcie_cfg_data *data;
1502 	struct brcm_pcie *pcie;
1503 	int ret;
1504 
1505 	bridge = devm_pci_alloc_host_bridge(&pdev->dev, sizeof(*pcie));
1506 	if (!bridge)
1507 		return -ENOMEM;
1508 
1509 	data = of_device_get_match_data(&pdev->dev);
1510 	if (!data) {
1511 		pr_err("failed to look up compatible string\n");
1512 		return -EINVAL;
1513 	}
1514 
1515 	pcie = pci_host_bridge_priv(bridge);
1516 	pcie->dev = &pdev->dev;
1517 	pcie->np = np;
1518 	pcie->reg_offsets = data->offsets;
1519 	pcie->type = data->type;
1520 	pcie->perst_set = data->perst_set;
1521 	pcie->bridge_sw_init_set = data->bridge_sw_init_set;
1522 
1523 	pcie->base = devm_platform_ioremap_resource(pdev, 0);
1524 	if (IS_ERR(pcie->base))
1525 		return PTR_ERR(pcie->base);
1526 
1527 	pcie->clk = devm_clk_get_optional(&pdev->dev, "sw_pcie");
1528 	if (IS_ERR(pcie->clk))
1529 		return PTR_ERR(pcie->clk);
1530 
1531 	ret = of_pci_get_max_link_speed(np);
1532 	pcie->gen = (ret < 0) ? 0 : ret;
1533 
1534 	pcie->ssc = of_property_read_bool(np, "brcm,enable-ssc");
1535 
1536 	ret = clk_prepare_enable(pcie->clk);
1537 	if (ret) {
1538 		dev_err(&pdev->dev, "could not enable clock\n");
1539 		return ret;
1540 	}
1541 	pcie->rescal = devm_reset_control_get_optional_shared(&pdev->dev, "rescal");
1542 	if (IS_ERR(pcie->rescal)) {
1543 		clk_disable_unprepare(pcie->clk);
1544 		return PTR_ERR(pcie->rescal);
1545 	}
1546 	pcie->perst_reset = devm_reset_control_get_optional_exclusive(&pdev->dev, "perst");
1547 	if (IS_ERR(pcie->perst_reset)) {
1548 		clk_disable_unprepare(pcie->clk);
1549 		return PTR_ERR(pcie->perst_reset);
1550 	}
1551 
1552 	ret = reset_control_reset(pcie->rescal);
1553 	if (ret)
1554 		dev_err(&pdev->dev, "failed to deassert 'rescal'\n");
1555 
1556 	ret = brcm_phy_start(pcie);
1557 	if (ret) {
1558 		reset_control_rearm(pcie->rescal);
1559 		clk_disable_unprepare(pcie->clk);
1560 		return ret;
1561 	}
1562 
1563 	ret = brcm_pcie_setup(pcie);
1564 	if (ret)
1565 		goto fail;
1566 
1567 	pcie->hw_rev = readl(pcie->base + PCIE_MISC_REVISION);
1568 	if (pcie->type == BCM4908 && pcie->hw_rev >= BRCM_PCIE_HW_REV_3_20) {
1569 		dev_err(pcie->dev, "hardware revision with unsupported PERST# setup\n");
1570 		ret = -ENODEV;
1571 		goto fail;
1572 	}
1573 
1574 	msi_np = of_parse_phandle(pcie->np, "msi-parent", 0);
1575 	if (pci_msi_enabled() && msi_np == pcie->np) {
1576 		ret = brcm_pcie_enable_msi(pcie);
1577 		if (ret) {
1578 			dev_err(pcie->dev, "probe of internal MSI failed");
1579 			goto fail;
1580 		}
1581 	}
1582 
1583 	bridge->ops = pcie->type == BCM7425 ? &brcm7425_pcie_ops : &brcm_pcie_ops;
1584 	bridge->sysdata = pcie;
1585 
1586 	platform_set_drvdata(pdev, pcie);
1587 
1588 	ret = pci_host_probe(bridge);
1589 	if (!ret && !brcm_pcie_link_up(pcie))
1590 		ret = -ENODEV;
1591 
1592 	if (ret) {
1593 		brcm_pcie_remove(pdev);
1594 		return ret;
1595 	}
1596 
1597 	return 0;
1598 
1599 fail:
1600 	__brcm_pcie_remove(pcie);
1601 	return ret;
1602 }
1603 
1604 MODULE_DEVICE_TABLE(of, brcm_pcie_match);
1605 
1606 static const struct dev_pm_ops brcm_pcie_pm_ops = {
1607 	.suspend_noirq = brcm_pcie_suspend_noirq,
1608 	.resume_noirq = brcm_pcie_resume_noirq,
1609 };
1610 
1611 static struct platform_driver brcm_pcie_driver = {
1612 	.probe = brcm_pcie_probe,
1613 	.remove_new = brcm_pcie_remove,
1614 	.driver = {
1615 		.name = "brcm-pcie",
1616 		.of_match_table = brcm_pcie_match,
1617 		.pm = &brcm_pcie_pm_ops,
1618 	},
1619 };
1620 module_platform_driver(brcm_pcie_driver);
1621 
1622 MODULE_LICENSE("GPL");
1623 MODULE_DESCRIPTION("Broadcom STB PCIe RC driver");
1624 MODULE_AUTHOR("Broadcom");
1625