1 // SPDX-License-Identifier: GPL-2.0+
2 /* Copyright (C) 2009 - 2019 Broadcom */
3 
4 #include <linux/bitfield.h>
5 #include <linux/bitops.h>
6 #include <linux/clk.h>
7 #include <linux/compiler.h>
8 #include <linux/delay.h>
9 #include <linux/init.h>
10 #include <linux/interrupt.h>
11 #include <linux/io.h>
12 #include <linux/iopoll.h>
13 #include <linux/ioport.h>
14 #include <linux/irqchip/chained_irq.h>
15 #include <linux/irqdomain.h>
16 #include <linux/kernel.h>
17 #include <linux/list.h>
18 #include <linux/log2.h>
19 #include <linux/module.h>
20 #include <linux/msi.h>
21 #include <linux/of_address.h>
22 #include <linux/of_irq.h>
23 #include <linux/of_pci.h>
24 #include <linux/of_platform.h>
25 #include <linux/pci.h>
26 #include <linux/pci-ecam.h>
27 #include <linux/printk.h>
28 #include <linux/regulator/consumer.h>
29 #include <linux/reset.h>
30 #include <linux/sizes.h>
31 #include <linux/slab.h>
32 #include <linux/string.h>
33 #include <linux/types.h>
34 
35 #include "../pci.h"
36 
37 /* BRCM_PCIE_CAP_REGS - Offset for the mandatory capability config regs */
38 #define BRCM_PCIE_CAP_REGS				0x00ac
39 
40 /* Broadcom STB PCIe Register Offsets */
41 #define PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1				0x0188
42 #define  PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK	0xc
43 #define  PCIE_RC_CFG_VENDOR_SPCIFIC_REG1_LITTLE_ENDIAN			0x0
44 
45 #define PCIE_RC_CFG_PRIV1_ID_VAL3			0x043c
46 #define  PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK	0xffffff
47 
48 #define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY			0x04dc
49 #define  PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_ASPM_SUPPORT_MASK	0xc00
50 
51 #define PCIE_RC_DL_MDIO_ADDR				0x1100
52 #define PCIE_RC_DL_MDIO_WR_DATA				0x1104
53 #define PCIE_RC_DL_MDIO_RD_DATA				0x1108
54 
55 #define PCIE_MISC_MISC_CTRL				0x4008
56 #define  PCIE_MISC_MISC_CTRL_PCIE_RCB_64B_MODE_MASK	0x80
57 #define  PCIE_MISC_MISC_CTRL_PCIE_RCB_MPS_MODE_MASK	0x400
58 #define  PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK		0x1000
59 #define  PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK	0x2000
60 #define  PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK	0x300000
61 
62 #define  PCIE_MISC_MISC_CTRL_SCB0_SIZE_MASK		0xf8000000
63 #define  PCIE_MISC_MISC_CTRL_SCB1_SIZE_MASK		0x07c00000
64 #define  PCIE_MISC_MISC_CTRL_SCB2_SIZE_MASK		0x0000001f
65 #define  SCB_SIZE_MASK(x) PCIE_MISC_MISC_CTRL_SCB ## x ## _SIZE_MASK
66 
67 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO		0x400c
68 #define PCIE_MEM_WIN0_LO(win)	\
69 		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO + ((win) * 8)
70 
71 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI		0x4010
72 #define PCIE_MEM_WIN0_HI(win)	\
73 		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI + ((win) * 8)
74 
75 #define PCIE_MISC_RC_BAR1_CONFIG_LO			0x402c
76 #define  PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK		0x1f
77 
78 #define PCIE_MISC_RC_BAR2_CONFIG_LO			0x4034
79 #define  PCIE_MISC_RC_BAR2_CONFIG_LO_SIZE_MASK		0x1f
80 #define PCIE_MISC_RC_BAR2_CONFIG_HI			0x4038
81 
82 #define PCIE_MISC_RC_BAR3_CONFIG_LO			0x403c
83 #define  PCIE_MISC_RC_BAR3_CONFIG_LO_SIZE_MASK		0x1f
84 
85 #define PCIE_MISC_MSI_BAR_CONFIG_LO			0x4044
86 #define PCIE_MISC_MSI_BAR_CONFIG_HI			0x4048
87 
88 #define PCIE_MISC_MSI_DATA_CONFIG			0x404c
89 #define  PCIE_MISC_MSI_DATA_CONFIG_VAL_32		0xffe06540
90 #define  PCIE_MISC_MSI_DATA_CONFIG_VAL_8		0xfff86540
91 
92 #define PCIE_MISC_PCIE_CTRL				0x4064
93 #define  PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK	0x1
94 #define PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK		0x4
95 
96 #define PCIE_MISC_PCIE_STATUS				0x4068
97 #define  PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK		0x80
98 #define  PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK	0x20
99 #define  PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK	0x10
100 #define  PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK	0x40
101 
102 #define PCIE_MISC_REVISION				0x406c
103 #define  BRCM_PCIE_HW_REV_33				0x0303
104 #define  BRCM_PCIE_HW_REV_3_20				0x0320
105 
106 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT		0x4070
107 #define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK	0xfff00000
108 #define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK	0xfff0
109 #define PCIE_MEM_WIN0_BASE_LIMIT(win)	\
110 		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT + ((win) * 4)
111 
112 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI			0x4080
113 #define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK	0xff
114 #define PCIE_MEM_WIN0_BASE_HI(win)	\
115 		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI + ((win) * 8)
116 
117 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI			0x4084
118 #define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK	0xff
119 #define PCIE_MEM_WIN0_LIMIT_HI(win)	\
120 		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI + ((win) * 8)
121 
122 #define PCIE_MISC_HARD_PCIE_HARD_DEBUG					0x4204
123 #define  PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK	0x2
124 #define  PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK		0x08000000
125 #define  PCIE_BMIPS_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK		0x00800000
126 
127 
128 #define PCIE_INTR2_CPU_BASE		0x4300
129 #define PCIE_MSI_INTR2_BASE		0x4500
130 /* Offsets from PCIE_INTR2_CPU_BASE and PCIE_MSI_INTR2_BASE */
131 #define  MSI_INT_STATUS			0x0
132 #define  MSI_INT_CLR			0x8
133 #define  MSI_INT_MASK_SET		0x10
134 #define  MSI_INT_MASK_CLR		0x14
135 
136 #define PCIE_EXT_CFG_DATA				0x8000
137 #define PCIE_EXT_CFG_INDEX				0x9000
138 
139 #define  PCIE_RGR1_SW_INIT_1_PERST_MASK			0x1
140 #define  PCIE_RGR1_SW_INIT_1_PERST_SHIFT		0x0
141 
142 #define RGR1_SW_INIT_1_INIT_GENERIC_MASK		0x2
143 #define RGR1_SW_INIT_1_INIT_GENERIC_SHIFT		0x1
144 #define RGR1_SW_INIT_1_INIT_7278_MASK			0x1
145 #define RGR1_SW_INIT_1_INIT_7278_SHIFT			0x0
146 
147 /* PCIe parameters */
148 #define BRCM_NUM_PCIE_OUT_WINS		0x4
149 #define BRCM_INT_PCI_MSI_NR		32
150 #define BRCM_INT_PCI_MSI_LEGACY_NR	8
151 #define BRCM_INT_PCI_MSI_SHIFT		0
152 #define BRCM_INT_PCI_MSI_MASK		GENMASK(BRCM_INT_PCI_MSI_NR - 1, 0)
153 #define BRCM_INT_PCI_MSI_LEGACY_MASK	GENMASK(31, \
154 						32 - BRCM_INT_PCI_MSI_LEGACY_NR)
155 
156 /* MSI target addresses */
157 #define BRCM_MSI_TARGET_ADDR_LT_4GB	0x0fffffffcULL
158 #define BRCM_MSI_TARGET_ADDR_GT_4GB	0xffffffffcULL
159 
160 /* MDIO registers */
161 #define MDIO_PORT0			0x0
162 #define MDIO_DATA_MASK			0x7fffffff
163 #define MDIO_PORT_MASK			0xf0000
164 #define MDIO_REGAD_MASK			0xffff
165 #define MDIO_CMD_MASK			0xfff00000
166 #define MDIO_CMD_READ			0x1
167 #define MDIO_CMD_WRITE			0x0
168 #define MDIO_DATA_DONE_MASK		0x80000000
169 #define MDIO_RD_DONE(x)			(((x) & MDIO_DATA_DONE_MASK) ? 1 : 0)
170 #define MDIO_WT_DONE(x)			(((x) & MDIO_DATA_DONE_MASK) ? 0 : 1)
171 #define SSC_REGS_ADDR			0x1100
172 #define SET_ADDR_OFFSET			0x1f
173 #define SSC_CNTL_OFFSET			0x2
174 #define SSC_CNTL_OVRD_EN_MASK		0x8000
175 #define SSC_CNTL_OVRD_VAL_MASK		0x4000
176 #define SSC_STATUS_OFFSET		0x1
177 #define SSC_STATUS_SSC_MASK		0x400
178 #define SSC_STATUS_PLL_LOCK_MASK	0x800
179 #define PCIE_BRCM_MAX_MEMC		3
180 
181 #define IDX_ADDR(pcie)			(pcie->reg_offsets[EXT_CFG_INDEX])
182 #define DATA_ADDR(pcie)			(pcie->reg_offsets[EXT_CFG_DATA])
183 #define PCIE_RGR1_SW_INIT_1(pcie)	(pcie->reg_offsets[RGR1_SW_INIT_1])
184 
185 /* Rescal registers */
186 #define PCIE_DVT_PMU_PCIE_PHY_CTRL				0xc700
187 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS			0x3
188 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_MASK		0x4
189 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_SHIFT	0x2
190 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_MASK		0x2
191 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_SHIFT		0x1
192 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_MASK		0x1
193 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_SHIFT		0x0
194 
195 /* Forward declarations */
196 struct brcm_pcie;
197 
198 enum {
199 	RGR1_SW_INIT_1,
200 	EXT_CFG_INDEX,
201 	EXT_CFG_DATA,
202 };
203 
204 enum {
205 	RGR1_SW_INIT_1_INIT_MASK,
206 	RGR1_SW_INIT_1_INIT_SHIFT,
207 };
208 
209 enum pcie_type {
210 	GENERIC,
211 	BCM7425,
212 	BCM7435,
213 	BCM4908,
214 	BCM7278,
215 	BCM2711,
216 };
217 
218 struct pcie_cfg_data {
219 	const int *offsets;
220 	const enum pcie_type type;
221 	void (*perst_set)(struct brcm_pcie *pcie, u32 val);
222 	void (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val);
223 };
224 
225 struct subdev_regulators {
226 	unsigned int num_supplies;
227 	struct regulator_bulk_data supplies[];
228 };
229 
230 struct brcm_msi {
231 	struct device		*dev;
232 	void __iomem		*base;
233 	struct device_node	*np;
234 	struct irq_domain	*msi_domain;
235 	struct irq_domain	*inner_domain;
236 	struct mutex		lock; /* guards the alloc/free operations */
237 	u64			target_addr;
238 	int			irq;
239 	DECLARE_BITMAP(used, BRCM_INT_PCI_MSI_NR);
240 	bool			legacy;
241 	/* Some chips have MSIs in bits [31..24] of a shared register. */
242 	int			legacy_shift;
243 	int			nr; /* No. of MSI available, depends on chip */
244 	/* This is the base pointer for interrupt status/set/clr regs */
245 	void __iomem		*intr_base;
246 };
247 
248 /* Internal PCIe Host Controller Information.*/
249 struct brcm_pcie {
250 	struct device		*dev;
251 	void __iomem		*base;
252 	struct clk		*clk;
253 	struct device_node	*np;
254 	bool			ssc;
255 	int			gen;
256 	u64			msi_target_addr;
257 	struct brcm_msi		*msi;
258 	const int		*reg_offsets;
259 	enum pcie_type		type;
260 	struct reset_control	*rescal;
261 	struct reset_control	*perst_reset;
262 	int			num_memc;
263 	u64			memc_size[PCIE_BRCM_MAX_MEMC];
264 	u32			hw_rev;
265 	void			(*perst_set)(struct brcm_pcie *pcie, u32 val);
266 	void			(*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val);
267 	struct subdev_regulators *sr;
268 	bool			ep_wakeup_capable;
269 };
270 
271 static inline bool is_bmips(const struct brcm_pcie *pcie)
272 {
273 	return pcie->type == BCM7435 || pcie->type == BCM7425;
274 }
275 
276 /*
277  * This is to convert the size of the inbound "BAR" region to the
278  * non-linear values of PCIE_X_MISC_RC_BAR[123]_CONFIG_LO.SIZE
279  */
280 static int brcm_pcie_encode_ibar_size(u64 size)
281 {
282 	int log2_in = ilog2(size);
283 
284 	if (log2_in >= 12 && log2_in <= 15)
285 		/* Covers 4KB to 32KB (inclusive) */
286 		return (log2_in - 12) + 0x1c;
287 	else if (log2_in >= 16 && log2_in <= 35)
288 		/* Covers 64KB to 32GB, (inclusive) */
289 		return log2_in - 15;
290 	/* Something is awry so disable */
291 	return 0;
292 }
293 
294 static u32 brcm_pcie_mdio_form_pkt(int port, int regad, int cmd)
295 {
296 	u32 pkt = 0;
297 
298 	pkt |= FIELD_PREP(MDIO_PORT_MASK, port);
299 	pkt |= FIELD_PREP(MDIO_REGAD_MASK, regad);
300 	pkt |= FIELD_PREP(MDIO_CMD_MASK, cmd);
301 
302 	return pkt;
303 }
304 
305 /* negative return value indicates error */
306 static int brcm_pcie_mdio_read(void __iomem *base, u8 port, u8 regad, u32 *val)
307 {
308 	u32 data;
309 	int err;
310 
311 	writel(brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_READ),
312 		   base + PCIE_RC_DL_MDIO_ADDR);
313 	readl(base + PCIE_RC_DL_MDIO_ADDR);
314 	err = readl_poll_timeout_atomic(base + PCIE_RC_DL_MDIO_RD_DATA, data,
315 					MDIO_RD_DONE(data), 10, 100);
316 	*val = FIELD_GET(MDIO_DATA_MASK, data);
317 
318 	return err;
319 }
320 
321 /* negative return value indicates error */
322 static int brcm_pcie_mdio_write(void __iomem *base, u8 port,
323 				u8 regad, u16 wrdata)
324 {
325 	u32 data;
326 	int err;
327 
328 	writel(brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_WRITE),
329 		   base + PCIE_RC_DL_MDIO_ADDR);
330 	readl(base + PCIE_RC_DL_MDIO_ADDR);
331 	writel(MDIO_DATA_DONE_MASK | wrdata, base + PCIE_RC_DL_MDIO_WR_DATA);
332 
333 	err = readl_poll_timeout_atomic(base + PCIE_RC_DL_MDIO_WR_DATA, data,
334 					MDIO_WT_DONE(data), 10, 100);
335 	return err;
336 }
337 
338 /*
339  * Configures device for Spread Spectrum Clocking (SSC) mode; a negative
340  * return value indicates error.
341  */
342 static int brcm_pcie_set_ssc(struct brcm_pcie *pcie)
343 {
344 	int pll, ssc;
345 	int ret;
346 	u32 tmp;
347 
348 	ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, SET_ADDR_OFFSET,
349 				   SSC_REGS_ADDR);
350 	if (ret < 0)
351 		return ret;
352 
353 	ret = brcm_pcie_mdio_read(pcie->base, MDIO_PORT0,
354 				  SSC_CNTL_OFFSET, &tmp);
355 	if (ret < 0)
356 		return ret;
357 
358 	u32p_replace_bits(&tmp, 1, SSC_CNTL_OVRD_EN_MASK);
359 	u32p_replace_bits(&tmp, 1, SSC_CNTL_OVRD_VAL_MASK);
360 	ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0,
361 				   SSC_CNTL_OFFSET, tmp);
362 	if (ret < 0)
363 		return ret;
364 
365 	usleep_range(1000, 2000);
366 	ret = brcm_pcie_mdio_read(pcie->base, MDIO_PORT0,
367 				  SSC_STATUS_OFFSET, &tmp);
368 	if (ret < 0)
369 		return ret;
370 
371 	ssc = FIELD_GET(SSC_STATUS_SSC_MASK, tmp);
372 	pll = FIELD_GET(SSC_STATUS_PLL_LOCK_MASK, tmp);
373 
374 	return ssc && pll ? 0 : -EIO;
375 }
376 
377 /* Limits operation to a specific generation (1, 2, or 3) */
378 static void brcm_pcie_set_gen(struct brcm_pcie *pcie, int gen)
379 {
380 	u16 lnkctl2 = readw(pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCTL2);
381 	u32 lnkcap = readl(pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCAP);
382 
383 	lnkcap = (lnkcap & ~PCI_EXP_LNKCAP_SLS) | gen;
384 	writel(lnkcap, pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCAP);
385 
386 	lnkctl2 = (lnkctl2 & ~0xf) | gen;
387 	writew(lnkctl2, pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCTL2);
388 }
389 
390 static void brcm_pcie_set_outbound_win(struct brcm_pcie *pcie,
391 				       unsigned int win, u64 cpu_addr,
392 				       u64 pcie_addr, u64 size)
393 {
394 	u32 cpu_addr_mb_high, limit_addr_mb_high;
395 	phys_addr_t cpu_addr_mb, limit_addr_mb;
396 	int high_addr_shift;
397 	u32 tmp;
398 
399 	/* Set the base of the pcie_addr window */
400 	writel(lower_32_bits(pcie_addr), pcie->base + PCIE_MEM_WIN0_LO(win));
401 	writel(upper_32_bits(pcie_addr), pcie->base + PCIE_MEM_WIN0_HI(win));
402 
403 	/* Write the addr base & limit lower bits (in MBs) */
404 	cpu_addr_mb = cpu_addr / SZ_1M;
405 	limit_addr_mb = (cpu_addr + size - 1) / SZ_1M;
406 
407 	tmp = readl(pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win));
408 	u32p_replace_bits(&tmp, cpu_addr_mb,
409 			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK);
410 	u32p_replace_bits(&tmp, limit_addr_mb,
411 			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK);
412 	writel(tmp, pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win));
413 
414 	if (is_bmips(pcie))
415 		return;
416 
417 	/* Write the cpu & limit addr upper bits */
418 	high_addr_shift =
419 		HWEIGHT32(PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK);
420 
421 	cpu_addr_mb_high = cpu_addr_mb >> high_addr_shift;
422 	tmp = readl(pcie->base + PCIE_MEM_WIN0_BASE_HI(win));
423 	u32p_replace_bits(&tmp, cpu_addr_mb_high,
424 			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK);
425 	writel(tmp, pcie->base + PCIE_MEM_WIN0_BASE_HI(win));
426 
427 	limit_addr_mb_high = limit_addr_mb >> high_addr_shift;
428 	tmp = readl(pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win));
429 	u32p_replace_bits(&tmp, limit_addr_mb_high,
430 			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK);
431 	writel(tmp, pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win));
432 }
433 
434 static struct irq_chip brcm_msi_irq_chip = {
435 	.name            = "BRCM STB PCIe MSI",
436 	.irq_ack         = irq_chip_ack_parent,
437 	.irq_mask        = pci_msi_mask_irq,
438 	.irq_unmask      = pci_msi_unmask_irq,
439 };
440 
441 static struct msi_domain_info brcm_msi_domain_info = {
442 	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
443 		   MSI_FLAG_MULTI_PCI_MSI),
444 	.chip	= &brcm_msi_irq_chip,
445 };
446 
447 static void brcm_pcie_msi_isr(struct irq_desc *desc)
448 {
449 	struct irq_chip *chip = irq_desc_get_chip(desc);
450 	unsigned long status;
451 	struct brcm_msi *msi;
452 	struct device *dev;
453 	u32 bit;
454 
455 	chained_irq_enter(chip, desc);
456 	msi = irq_desc_get_handler_data(desc);
457 	dev = msi->dev;
458 
459 	status = readl(msi->intr_base + MSI_INT_STATUS);
460 	status >>= msi->legacy_shift;
461 
462 	for_each_set_bit(bit, &status, msi->nr) {
463 		int ret;
464 		ret = generic_handle_domain_irq(msi->inner_domain, bit);
465 		if (ret)
466 			dev_dbg(dev, "unexpected MSI\n");
467 	}
468 
469 	chained_irq_exit(chip, desc);
470 }
471 
472 static void brcm_msi_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
473 {
474 	struct brcm_msi *msi = irq_data_get_irq_chip_data(data);
475 
476 	msg->address_lo = lower_32_bits(msi->target_addr);
477 	msg->address_hi = upper_32_bits(msi->target_addr);
478 	msg->data = (0xffff & PCIE_MISC_MSI_DATA_CONFIG_VAL_32) | data->hwirq;
479 }
480 
481 static int brcm_msi_set_affinity(struct irq_data *irq_data,
482 				 const struct cpumask *mask, bool force)
483 {
484 	return -EINVAL;
485 }
486 
487 static void brcm_msi_ack_irq(struct irq_data *data)
488 {
489 	struct brcm_msi *msi = irq_data_get_irq_chip_data(data);
490 	const int shift_amt = data->hwirq + msi->legacy_shift;
491 
492 	writel(1 << shift_amt, msi->intr_base + MSI_INT_CLR);
493 }
494 
495 
496 static struct irq_chip brcm_msi_bottom_irq_chip = {
497 	.name			= "BRCM STB MSI",
498 	.irq_compose_msi_msg	= brcm_msi_compose_msi_msg,
499 	.irq_set_affinity	= brcm_msi_set_affinity,
500 	.irq_ack                = brcm_msi_ack_irq,
501 };
502 
503 static int brcm_msi_alloc(struct brcm_msi *msi, unsigned int nr_irqs)
504 {
505 	int hwirq;
506 
507 	mutex_lock(&msi->lock);
508 	hwirq = bitmap_find_free_region(msi->used, msi->nr,
509 					order_base_2(nr_irqs));
510 	mutex_unlock(&msi->lock);
511 
512 	return hwirq;
513 }
514 
515 static void brcm_msi_free(struct brcm_msi *msi, unsigned long hwirq,
516 			  unsigned int nr_irqs)
517 {
518 	mutex_lock(&msi->lock);
519 	bitmap_release_region(msi->used, hwirq, order_base_2(nr_irqs));
520 	mutex_unlock(&msi->lock);
521 }
522 
523 static int brcm_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
524 				 unsigned int nr_irqs, void *args)
525 {
526 	struct brcm_msi *msi = domain->host_data;
527 	int hwirq, i;
528 
529 	hwirq = brcm_msi_alloc(msi, nr_irqs);
530 
531 	if (hwirq < 0)
532 		return hwirq;
533 
534 	for (i = 0; i < nr_irqs; i++)
535 		irq_domain_set_info(domain, virq + i, hwirq + i,
536 				    &brcm_msi_bottom_irq_chip, domain->host_data,
537 				    handle_edge_irq, NULL, NULL);
538 	return 0;
539 }
540 
541 static void brcm_irq_domain_free(struct irq_domain *domain,
542 				 unsigned int virq, unsigned int nr_irqs)
543 {
544 	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
545 	struct brcm_msi *msi = irq_data_get_irq_chip_data(d);
546 
547 	brcm_msi_free(msi, d->hwirq, nr_irqs);
548 }
549 
550 static const struct irq_domain_ops msi_domain_ops = {
551 	.alloc	= brcm_irq_domain_alloc,
552 	.free	= brcm_irq_domain_free,
553 };
554 
555 static int brcm_allocate_domains(struct brcm_msi *msi)
556 {
557 	struct fwnode_handle *fwnode = of_node_to_fwnode(msi->np);
558 	struct device *dev = msi->dev;
559 
560 	msi->inner_domain = irq_domain_add_linear(NULL, msi->nr, &msi_domain_ops, msi);
561 	if (!msi->inner_domain) {
562 		dev_err(dev, "failed to create IRQ domain\n");
563 		return -ENOMEM;
564 	}
565 
566 	msi->msi_domain = pci_msi_create_irq_domain(fwnode,
567 						    &brcm_msi_domain_info,
568 						    msi->inner_domain);
569 	if (!msi->msi_domain) {
570 		dev_err(dev, "failed to create MSI domain\n");
571 		irq_domain_remove(msi->inner_domain);
572 		return -ENOMEM;
573 	}
574 
575 	return 0;
576 }
577 
578 static void brcm_free_domains(struct brcm_msi *msi)
579 {
580 	irq_domain_remove(msi->msi_domain);
581 	irq_domain_remove(msi->inner_domain);
582 }
583 
584 static void brcm_msi_remove(struct brcm_pcie *pcie)
585 {
586 	struct brcm_msi *msi = pcie->msi;
587 
588 	if (!msi)
589 		return;
590 	irq_set_chained_handler_and_data(msi->irq, NULL, NULL);
591 	brcm_free_domains(msi);
592 }
593 
594 static void brcm_msi_set_regs(struct brcm_msi *msi)
595 {
596 	u32 val = msi->legacy ? BRCM_INT_PCI_MSI_LEGACY_MASK :
597 				BRCM_INT_PCI_MSI_MASK;
598 
599 	writel(val, msi->intr_base + MSI_INT_MASK_CLR);
600 	writel(val, msi->intr_base + MSI_INT_CLR);
601 
602 	/*
603 	 * The 0 bit of PCIE_MISC_MSI_BAR_CONFIG_LO is repurposed to MSI
604 	 * enable, which we set to 1.
605 	 */
606 	writel(lower_32_bits(msi->target_addr) | 0x1,
607 	       msi->base + PCIE_MISC_MSI_BAR_CONFIG_LO);
608 	writel(upper_32_bits(msi->target_addr),
609 	       msi->base + PCIE_MISC_MSI_BAR_CONFIG_HI);
610 
611 	val = msi->legacy ? PCIE_MISC_MSI_DATA_CONFIG_VAL_8 : PCIE_MISC_MSI_DATA_CONFIG_VAL_32;
612 	writel(val, msi->base + PCIE_MISC_MSI_DATA_CONFIG);
613 }
614 
615 static int brcm_pcie_enable_msi(struct brcm_pcie *pcie)
616 {
617 	struct brcm_msi *msi;
618 	int irq, ret;
619 	struct device *dev = pcie->dev;
620 
621 	irq = irq_of_parse_and_map(dev->of_node, 1);
622 	if (irq <= 0) {
623 		dev_err(dev, "cannot map MSI interrupt\n");
624 		return -ENODEV;
625 	}
626 
627 	msi = devm_kzalloc(dev, sizeof(struct brcm_msi), GFP_KERNEL);
628 	if (!msi)
629 		return -ENOMEM;
630 
631 	mutex_init(&msi->lock);
632 	msi->dev = dev;
633 	msi->base = pcie->base;
634 	msi->np = pcie->np;
635 	msi->target_addr = pcie->msi_target_addr;
636 	msi->irq = irq;
637 	msi->legacy = pcie->hw_rev < BRCM_PCIE_HW_REV_33;
638 
639 	/*
640 	 * Sanity check to make sure that the 'used' bitmap in struct brcm_msi
641 	 * is large enough.
642 	 */
643 	BUILD_BUG_ON(BRCM_INT_PCI_MSI_LEGACY_NR > BRCM_INT_PCI_MSI_NR);
644 
645 	if (msi->legacy) {
646 		msi->intr_base = msi->base + PCIE_INTR2_CPU_BASE;
647 		msi->nr = BRCM_INT_PCI_MSI_LEGACY_NR;
648 		msi->legacy_shift = 24;
649 	} else {
650 		msi->intr_base = msi->base + PCIE_MSI_INTR2_BASE;
651 		msi->nr = BRCM_INT_PCI_MSI_NR;
652 		msi->legacy_shift = 0;
653 	}
654 
655 	ret = brcm_allocate_domains(msi);
656 	if (ret)
657 		return ret;
658 
659 	irq_set_chained_handler_and_data(msi->irq, brcm_pcie_msi_isr, msi);
660 
661 	brcm_msi_set_regs(msi);
662 	pcie->msi = msi;
663 
664 	return 0;
665 }
666 
667 /* The controller is capable of serving in both RC and EP roles */
668 static bool brcm_pcie_rc_mode(struct brcm_pcie *pcie)
669 {
670 	void __iomem *base = pcie->base;
671 	u32 val = readl(base + PCIE_MISC_PCIE_STATUS);
672 
673 	return !!FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK, val);
674 }
675 
676 static bool brcm_pcie_link_up(struct brcm_pcie *pcie)
677 {
678 	u32 val = readl(pcie->base + PCIE_MISC_PCIE_STATUS);
679 	u32 dla = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK, val);
680 	u32 plu = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK, val);
681 
682 	return dla && plu;
683 }
684 
685 static void __iomem *brcm_pcie_map_bus(struct pci_bus *bus,
686 				       unsigned int devfn, int where)
687 {
688 	struct brcm_pcie *pcie = bus->sysdata;
689 	void __iomem *base = pcie->base;
690 	int idx;
691 
692 	/* Accesses to the RC go right to the RC registers if !devfn */
693 	if (pci_is_root_bus(bus))
694 		return devfn ? NULL : base + PCIE_ECAM_REG(where);
695 
696 	/* An access to our HW w/o link-up will cause a CPU Abort */
697 	if (!brcm_pcie_link_up(pcie))
698 		return NULL;
699 
700 	/* For devices, write to the config space index register */
701 	idx = PCIE_ECAM_OFFSET(bus->number, devfn, 0);
702 	writel(idx, pcie->base + PCIE_EXT_CFG_INDEX);
703 	return base + PCIE_EXT_CFG_DATA + PCIE_ECAM_REG(where);
704 }
705 
706 static void __iomem *brcm7425_pcie_map_bus(struct pci_bus *bus,
707 					   unsigned int devfn, int where)
708 {
709 	struct brcm_pcie *pcie = bus->sysdata;
710 	void __iomem *base = pcie->base;
711 	int idx;
712 
713 	/* Accesses to the RC go right to the RC registers if !devfn */
714 	if (pci_is_root_bus(bus))
715 		return devfn ? NULL : base + PCIE_ECAM_REG(where);
716 
717 	/* An access to our HW w/o link-up will cause a CPU Abort */
718 	if (!brcm_pcie_link_up(pcie))
719 		return NULL;
720 
721 	/* For devices, write to the config space index register */
722 	idx = PCIE_ECAM_OFFSET(bus->number, devfn, where);
723 	writel(idx, base + IDX_ADDR(pcie));
724 	return base + DATA_ADDR(pcie);
725 }
726 
727 static void brcm_pcie_bridge_sw_init_set_generic(struct brcm_pcie *pcie, u32 val)
728 {
729 	u32 tmp, mask =  RGR1_SW_INIT_1_INIT_GENERIC_MASK;
730 	u32 shift = RGR1_SW_INIT_1_INIT_GENERIC_SHIFT;
731 
732 	tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
733 	tmp = (tmp & ~mask) | ((val << shift) & mask);
734 	writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
735 }
736 
737 static void brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 val)
738 {
739 	u32 tmp, mask =  RGR1_SW_INIT_1_INIT_7278_MASK;
740 	u32 shift = RGR1_SW_INIT_1_INIT_7278_SHIFT;
741 
742 	tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
743 	tmp = (tmp & ~mask) | ((val << shift) & mask);
744 	writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
745 }
746 
747 static void brcm_pcie_perst_set_4908(struct brcm_pcie *pcie, u32 val)
748 {
749 	if (WARN_ONCE(!pcie->perst_reset, "missing PERST# reset controller\n"))
750 		return;
751 
752 	if (val)
753 		reset_control_assert(pcie->perst_reset);
754 	else
755 		reset_control_deassert(pcie->perst_reset);
756 }
757 
758 static void brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val)
759 {
760 	u32 tmp;
761 
762 	/* Perst bit has moved and assert value is 0 */
763 	tmp = readl(pcie->base + PCIE_MISC_PCIE_CTRL);
764 	u32p_replace_bits(&tmp, !val, PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK);
765 	writel(tmp, pcie->base +  PCIE_MISC_PCIE_CTRL);
766 }
767 
768 static void brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, u32 val)
769 {
770 	u32 tmp;
771 
772 	tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
773 	u32p_replace_bits(&tmp, val, PCIE_RGR1_SW_INIT_1_PERST_MASK);
774 	writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
775 }
776 
777 static int brcm_pcie_get_rc_bar2_size_and_offset(struct brcm_pcie *pcie,
778 							u64 *rc_bar2_size,
779 							u64 *rc_bar2_offset)
780 {
781 	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
782 	struct resource_entry *entry;
783 	struct device *dev = pcie->dev;
784 	u64 lowest_pcie_addr = ~(u64)0;
785 	int ret, i = 0;
786 	u64 size = 0;
787 
788 	resource_list_for_each_entry(entry, &bridge->dma_ranges) {
789 		u64 pcie_beg = entry->res->start - entry->offset;
790 
791 		size += entry->res->end - entry->res->start + 1;
792 		if (pcie_beg < lowest_pcie_addr)
793 			lowest_pcie_addr = pcie_beg;
794 	}
795 
796 	if (lowest_pcie_addr == ~(u64)0) {
797 		dev_err(dev, "DT node has no dma-ranges\n");
798 		return -EINVAL;
799 	}
800 
801 	ret = of_property_read_variable_u64_array(pcie->np, "brcm,scb-sizes", pcie->memc_size, 1,
802 						  PCIE_BRCM_MAX_MEMC);
803 
804 	if (ret <= 0) {
805 		/* Make an educated guess */
806 		pcie->num_memc = 1;
807 		pcie->memc_size[0] = 1ULL << fls64(size - 1);
808 	} else {
809 		pcie->num_memc = ret;
810 	}
811 
812 	/* Each memc is viewed through a "port" that is a power of 2 */
813 	for (i = 0, size = 0; i < pcie->num_memc; i++)
814 		size += pcie->memc_size[i];
815 
816 	/* System memory starts at this address in PCIe-space */
817 	*rc_bar2_offset = lowest_pcie_addr;
818 	/* The sum of all memc views must also be a power of 2 */
819 	*rc_bar2_size = 1ULL << fls64(size - 1);
820 
821 	/*
822 	 * We validate the inbound memory view even though we should trust
823 	 * whatever the device-tree provides. This is because of an HW issue on
824 	 * early Raspberry Pi 4's revisions (bcm2711). It turns out its
825 	 * firmware has to dynamically edit dma-ranges due to a bug on the
826 	 * PCIe controller integration, which prohibits any access above the
827 	 * lower 3GB of memory. Given this, we decided to keep the dma-ranges
828 	 * in check, avoiding hard to debug device-tree related issues in the
829 	 * future:
830 	 *
831 	 * The PCIe host controller by design must set the inbound viewport to
832 	 * be a contiguous arrangement of all of the system's memory.  In
833 	 * addition, its size mut be a power of two.  To further complicate
834 	 * matters, the viewport must start on a pcie-address that is aligned
835 	 * on a multiple of its size.  If a portion of the viewport does not
836 	 * represent system memory -- e.g. 3GB of memory requires a 4GB
837 	 * viewport -- we can map the outbound memory in or after 3GB and even
838 	 * though the viewport will overlap the outbound memory the controller
839 	 * will know to send outbound memory downstream and everything else
840 	 * upstream.
841 	 *
842 	 * For example:
843 	 *
844 	 * - The best-case scenario, memory up to 3GB, is to place the inbound
845 	 *   region in the first 4GB of pcie-space, as some legacy devices can
846 	 *   only address 32bits. We would also like to put the MSI under 4GB
847 	 *   as well, since some devices require a 32bit MSI target address.
848 	 *
849 	 * - If the system memory is 4GB or larger we cannot start the inbound
850 	 *   region at location 0 (since we have to allow some space for
851 	 *   outbound memory @ 3GB). So instead it will  start at the 1x
852 	 *   multiple of its size
853 	 */
854 	if (!*rc_bar2_size || (*rc_bar2_offset & (*rc_bar2_size - 1)) ||
855 	    (*rc_bar2_offset < SZ_4G && *rc_bar2_offset > SZ_2G)) {
856 		dev_err(dev, "Invalid rc_bar2_offset/size: size 0x%llx, off 0x%llx\n",
857 			*rc_bar2_size, *rc_bar2_offset);
858 		return -EINVAL;
859 	}
860 
861 	return 0;
862 }
863 
864 static int brcm_pcie_setup(struct brcm_pcie *pcie)
865 {
866 	u64 rc_bar2_offset, rc_bar2_size;
867 	void __iomem *base = pcie->base;
868 	struct pci_host_bridge *bridge;
869 	struct resource_entry *entry;
870 	u32 tmp, burst, aspm_support;
871 	int num_out_wins = 0;
872 	int ret, memc;
873 
874 	/* Reset the bridge */
875 	pcie->bridge_sw_init_set(pcie, 1);
876 
877 	/* Ensure that PERST# is asserted; some bootloaders may deassert it. */
878 	if (pcie->type == BCM2711)
879 		pcie->perst_set(pcie, 1);
880 
881 	usleep_range(100, 200);
882 
883 	/* Take the bridge out of reset */
884 	pcie->bridge_sw_init_set(pcie, 0);
885 
886 	tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
887 	if (is_bmips(pcie))
888 		tmp &= ~PCIE_BMIPS_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK;
889 	else
890 		tmp &= ~PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK;
891 	writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
892 	/* Wait for SerDes to be stable */
893 	usleep_range(100, 200);
894 
895 	/*
896 	 * SCB_MAX_BURST_SIZE is a two bit field.  For GENERIC chips it
897 	 * is encoded as 0=128, 1=256, 2=512, 3=Rsvd, for BCM7278 it
898 	 * is encoded as 0=Rsvd, 1=128, 2=256, 3=512.
899 	 */
900 	if (is_bmips(pcie))
901 		burst = 0x1; /* 256 bytes */
902 	else if (pcie->type == BCM2711)
903 		burst = 0x0; /* 128 bytes */
904 	else if (pcie->type == BCM7278)
905 		burst = 0x3; /* 512 bytes */
906 	else
907 		burst = 0x2; /* 512 bytes */
908 
909 	/*
910 	 * Set SCB_MAX_BURST_SIZE, CFG_READ_UR_MODE, SCB_ACCESS_EN,
911 	 * RCB_MPS_MODE, RCB_64B_MODE
912 	 */
913 	tmp = readl(base + PCIE_MISC_MISC_CTRL);
914 	u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK);
915 	u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK);
916 	u32p_replace_bits(&tmp, burst, PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK);
917 	u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_MPS_MODE_MASK);
918 	u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_64B_MODE_MASK);
919 	writel(tmp, base + PCIE_MISC_MISC_CTRL);
920 
921 	ret = brcm_pcie_get_rc_bar2_size_and_offset(pcie, &rc_bar2_size,
922 						    &rc_bar2_offset);
923 	if (ret)
924 		return ret;
925 
926 	tmp = lower_32_bits(rc_bar2_offset);
927 	u32p_replace_bits(&tmp, brcm_pcie_encode_ibar_size(rc_bar2_size),
928 			  PCIE_MISC_RC_BAR2_CONFIG_LO_SIZE_MASK);
929 	writel(tmp, base + PCIE_MISC_RC_BAR2_CONFIG_LO);
930 	writel(upper_32_bits(rc_bar2_offset),
931 	       base + PCIE_MISC_RC_BAR2_CONFIG_HI);
932 
933 	tmp = readl(base + PCIE_MISC_MISC_CTRL);
934 	for (memc = 0; memc < pcie->num_memc; memc++) {
935 		u32 scb_size_val = ilog2(pcie->memc_size[memc]) - 15;
936 
937 		if (memc == 0)
938 			u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(0));
939 		else if (memc == 1)
940 			u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(1));
941 		else if (memc == 2)
942 			u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(2));
943 	}
944 	writel(tmp, base + PCIE_MISC_MISC_CTRL);
945 
946 	/*
947 	 * We ideally want the MSI target address to be located in the 32bit
948 	 * addressable memory area. Some devices might depend on it. This is
949 	 * possible either when the inbound window is located above the lower
950 	 * 4GB or when the inbound area is smaller than 4GB (taking into
951 	 * account the rounding-up we're forced to perform).
952 	 */
953 	if (rc_bar2_offset >= SZ_4G || (rc_bar2_size + rc_bar2_offset) < SZ_4G)
954 		pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_LT_4GB;
955 	else
956 		pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_GT_4GB;
957 
958 	if (!brcm_pcie_rc_mode(pcie)) {
959 		dev_err(pcie->dev, "PCIe RC controller misconfigured as Endpoint\n");
960 		return -EINVAL;
961 	}
962 
963 	/* disable the PCIe->GISB memory window (RC_BAR1) */
964 	tmp = readl(base + PCIE_MISC_RC_BAR1_CONFIG_LO);
965 	tmp &= ~PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK;
966 	writel(tmp, base + PCIE_MISC_RC_BAR1_CONFIG_LO);
967 
968 	/* disable the PCIe->SCB memory window (RC_BAR3) */
969 	tmp = readl(base + PCIE_MISC_RC_BAR3_CONFIG_LO);
970 	tmp &= ~PCIE_MISC_RC_BAR3_CONFIG_LO_SIZE_MASK;
971 	writel(tmp, base + PCIE_MISC_RC_BAR3_CONFIG_LO);
972 
973 	/* Don't advertise L0s capability if 'aspm-no-l0s' */
974 	aspm_support = PCIE_LINK_STATE_L1;
975 	if (!of_property_read_bool(pcie->np, "aspm-no-l0s"))
976 		aspm_support |= PCIE_LINK_STATE_L0S;
977 	tmp = readl(base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY);
978 	u32p_replace_bits(&tmp, aspm_support,
979 		PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_ASPM_SUPPORT_MASK);
980 	writel(tmp, base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY);
981 
982 	/*
983 	 * For config space accesses on the RC, show the right class for
984 	 * a PCIe-PCIe bridge (the default setting is to be EP mode).
985 	 */
986 	tmp = readl(base + PCIE_RC_CFG_PRIV1_ID_VAL3);
987 	u32p_replace_bits(&tmp, 0x060400,
988 			  PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK);
989 	writel(tmp, base + PCIE_RC_CFG_PRIV1_ID_VAL3);
990 
991 	bridge = pci_host_bridge_from_priv(pcie);
992 	resource_list_for_each_entry(entry, &bridge->windows) {
993 		struct resource *res = entry->res;
994 
995 		if (resource_type(res) != IORESOURCE_MEM)
996 			continue;
997 
998 		if (num_out_wins >= BRCM_NUM_PCIE_OUT_WINS) {
999 			dev_err(pcie->dev, "too many outbound wins\n");
1000 			return -EINVAL;
1001 		}
1002 
1003 		if (is_bmips(pcie)) {
1004 			u64 start = res->start;
1005 			unsigned int j, nwins = resource_size(res) / SZ_128M;
1006 
1007 			/* bmips PCIe outbound windows have a 128MB max size */
1008 			if (nwins > BRCM_NUM_PCIE_OUT_WINS)
1009 				nwins = BRCM_NUM_PCIE_OUT_WINS;
1010 			for (j = 0; j < nwins; j++, start += SZ_128M)
1011 				brcm_pcie_set_outbound_win(pcie, j, start,
1012 							   start - entry->offset,
1013 							   SZ_128M);
1014 			break;
1015 		}
1016 		brcm_pcie_set_outbound_win(pcie, num_out_wins, res->start,
1017 					   res->start - entry->offset,
1018 					   resource_size(res));
1019 		num_out_wins++;
1020 	}
1021 
1022 	/* PCIe->SCB endian mode for BAR */
1023 	tmp = readl(base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1);
1024 	u32p_replace_bits(&tmp, PCIE_RC_CFG_VENDOR_SPCIFIC_REG1_LITTLE_ENDIAN,
1025 		PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK);
1026 	writel(tmp, base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1);
1027 
1028 	return 0;
1029 }
1030 
1031 static int brcm_pcie_start_link(struct brcm_pcie *pcie)
1032 {
1033 	struct device *dev = pcie->dev;
1034 	void __iomem *base = pcie->base;
1035 	u16 nlw, cls, lnksta;
1036 	bool ssc_good = false;
1037 	u32 tmp;
1038 	int ret, i;
1039 
1040 	/* Unassert the fundamental reset */
1041 	pcie->perst_set(pcie, 0);
1042 
1043 	/*
1044 	 * Wait for 100ms after PERST# deassertion; see PCIe CEM specification
1045 	 * sections 2.2, PCIe r5.0, 6.6.1.
1046 	 */
1047 	msleep(100);
1048 
1049 	/*
1050 	 * Give the RC/EP even more time to wake up, before trying to
1051 	 * configure RC.  Intermittently check status for link-up, up to a
1052 	 * total of 100ms.
1053 	 */
1054 	for (i = 0; i < 100 && !brcm_pcie_link_up(pcie); i += 5)
1055 		msleep(5);
1056 
1057 	if (!brcm_pcie_link_up(pcie)) {
1058 		dev_err(dev, "link down\n");
1059 		return -ENODEV;
1060 	}
1061 
1062 	if (pcie->gen)
1063 		brcm_pcie_set_gen(pcie, pcie->gen);
1064 
1065 	if (pcie->ssc) {
1066 		ret = brcm_pcie_set_ssc(pcie);
1067 		if (ret == 0)
1068 			ssc_good = true;
1069 		else
1070 			dev_err(dev, "failed attempt to enter ssc mode\n");
1071 	}
1072 
1073 	lnksta = readw(base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKSTA);
1074 	cls = FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta);
1075 	nlw = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta);
1076 	dev_info(dev, "link up, %s x%u %s\n",
1077 		 pci_speed_string(pcie_link_speed[cls]), nlw,
1078 		 ssc_good ? "(SSC)" : "(!SSC)");
1079 
1080 	/*
1081 	 * Refclk from RC should be gated with CLKREQ# input when ASPM L0s,L1
1082 	 * is enabled => setting the CLKREQ_DEBUG_ENABLE field to 1.
1083 	 */
1084 	tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1085 	tmp |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK;
1086 	writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1087 
1088 	return 0;
1089 }
1090 
1091 static const char * const supplies[] = {
1092 	"vpcie3v3",
1093 	"vpcie3v3aux",
1094 	"vpcie12v",
1095 };
1096 
1097 static void *alloc_subdev_regulators(struct device *dev)
1098 {
1099 	const size_t size = sizeof(struct subdev_regulators) +
1100 		sizeof(struct regulator_bulk_data) * ARRAY_SIZE(supplies);
1101 	struct subdev_regulators *sr;
1102 	int i;
1103 
1104 	sr = devm_kzalloc(dev, size, GFP_KERNEL);
1105 	if (sr) {
1106 		sr->num_supplies = ARRAY_SIZE(supplies);
1107 		for (i = 0; i < ARRAY_SIZE(supplies); i++)
1108 			sr->supplies[i].supply = supplies[i];
1109 	}
1110 
1111 	return sr;
1112 }
1113 
1114 static int brcm_pcie_add_bus(struct pci_bus *bus)
1115 {
1116 	struct brcm_pcie *pcie = bus->sysdata;
1117 	struct device *dev = &bus->dev;
1118 	struct subdev_regulators *sr;
1119 	int ret;
1120 
1121 	if (!bus->parent || !pci_is_root_bus(bus->parent))
1122 		return 0;
1123 
1124 	if (dev->of_node) {
1125 		sr = alloc_subdev_regulators(dev);
1126 		if (!sr) {
1127 			dev_info(dev, "Can't allocate regulators for downstream device\n");
1128 			goto no_regulators;
1129 		}
1130 
1131 		pcie->sr = sr;
1132 
1133 		ret = regulator_bulk_get(dev, sr->num_supplies, sr->supplies);
1134 		if (ret) {
1135 			dev_info(dev, "No regulators for downstream device\n");
1136 			goto no_regulators;
1137 		}
1138 
1139 		ret = regulator_bulk_enable(sr->num_supplies, sr->supplies);
1140 		if (ret) {
1141 			dev_err(dev, "Can't enable regulators for downstream device\n");
1142 			regulator_bulk_free(sr->num_supplies, sr->supplies);
1143 			pcie->sr = NULL;
1144 		}
1145 	}
1146 
1147 no_regulators:
1148 	brcm_pcie_start_link(pcie);
1149 	return 0;
1150 }
1151 
1152 static void brcm_pcie_remove_bus(struct pci_bus *bus)
1153 {
1154 	struct brcm_pcie *pcie = bus->sysdata;
1155 	struct subdev_regulators *sr = pcie->sr;
1156 	struct device *dev = &bus->dev;
1157 
1158 	if (!sr)
1159 		return;
1160 
1161 	if (regulator_bulk_disable(sr->num_supplies, sr->supplies))
1162 		dev_err(dev, "Failed to disable regulators for downstream device\n");
1163 	regulator_bulk_free(sr->num_supplies, sr->supplies);
1164 	pcie->sr = NULL;
1165 }
1166 
1167 /* L23 is a low-power PCIe link state */
1168 static void brcm_pcie_enter_l23(struct brcm_pcie *pcie)
1169 {
1170 	void __iomem *base = pcie->base;
1171 	int l23, i;
1172 	u32 tmp;
1173 
1174 	/* Assert request for L23 */
1175 	tmp = readl(base + PCIE_MISC_PCIE_CTRL);
1176 	u32p_replace_bits(&tmp, 1, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK);
1177 	writel(tmp, base + PCIE_MISC_PCIE_CTRL);
1178 
1179 	/* Wait up to 36 msec for L23 */
1180 	tmp = readl(base + PCIE_MISC_PCIE_STATUS);
1181 	l23 = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK, tmp);
1182 	for (i = 0; i < 15 && !l23; i++) {
1183 		usleep_range(2000, 2400);
1184 		tmp = readl(base + PCIE_MISC_PCIE_STATUS);
1185 		l23 = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK,
1186 				tmp);
1187 	}
1188 
1189 	if (!l23)
1190 		dev_err(pcie->dev, "failed to enter low-power link state\n");
1191 }
1192 
1193 static int brcm_phy_cntl(struct brcm_pcie *pcie, const int start)
1194 {
1195 	static const u32 shifts[PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS] = {
1196 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_SHIFT,
1197 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_SHIFT,
1198 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_SHIFT,};
1199 	static const u32 masks[PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS] = {
1200 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_MASK,
1201 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_MASK,
1202 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_MASK,};
1203 	const int beg = start ? 0 : PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS - 1;
1204 	const int end = start ? PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS : -1;
1205 	u32 tmp, combined_mask = 0;
1206 	u32 val;
1207 	void __iomem *base = pcie->base;
1208 	int i, ret;
1209 
1210 	for (i = beg; i != end; start ? i++ : i--) {
1211 		val = start ? BIT_MASK(shifts[i]) : 0;
1212 		tmp = readl(base + PCIE_DVT_PMU_PCIE_PHY_CTRL);
1213 		tmp = (tmp & ~masks[i]) | (val & masks[i]);
1214 		writel(tmp, base + PCIE_DVT_PMU_PCIE_PHY_CTRL);
1215 		usleep_range(50, 200);
1216 		combined_mask |= masks[i];
1217 	}
1218 
1219 	tmp = readl(base + PCIE_DVT_PMU_PCIE_PHY_CTRL);
1220 	val = start ? combined_mask : 0;
1221 
1222 	ret = (tmp & combined_mask) == val ? 0 : -EIO;
1223 	if (ret)
1224 		dev_err(pcie->dev, "failed to %s phy\n", (start ? "start" : "stop"));
1225 
1226 	return ret;
1227 }
1228 
1229 static inline int brcm_phy_start(struct brcm_pcie *pcie)
1230 {
1231 	return pcie->rescal ? brcm_phy_cntl(pcie, 1) : 0;
1232 }
1233 
1234 static inline int brcm_phy_stop(struct brcm_pcie *pcie)
1235 {
1236 	return pcie->rescal ? brcm_phy_cntl(pcie, 0) : 0;
1237 }
1238 
1239 static void brcm_pcie_turn_off(struct brcm_pcie *pcie)
1240 {
1241 	void __iomem *base = pcie->base;
1242 	int tmp;
1243 
1244 	if (brcm_pcie_link_up(pcie))
1245 		brcm_pcie_enter_l23(pcie);
1246 	/* Assert fundamental reset */
1247 	pcie->perst_set(pcie, 1);
1248 
1249 	/* Deassert request for L23 in case it was asserted */
1250 	tmp = readl(base + PCIE_MISC_PCIE_CTRL);
1251 	u32p_replace_bits(&tmp, 0, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK);
1252 	writel(tmp, base + PCIE_MISC_PCIE_CTRL);
1253 
1254 	/* Turn off SerDes */
1255 	tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1256 	u32p_replace_bits(&tmp, 1, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK);
1257 	writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1258 
1259 	/* Shutdown PCIe bridge */
1260 	pcie->bridge_sw_init_set(pcie, 1);
1261 }
1262 
1263 static int pci_dev_may_wakeup(struct pci_dev *dev, void *data)
1264 {
1265 	bool *ret = data;
1266 
1267 	if (device_may_wakeup(&dev->dev)) {
1268 		*ret = true;
1269 		dev_info(&dev->dev, "Possible wake-up device; regulators will not be disabled\n");
1270 	}
1271 	return (int) *ret;
1272 }
1273 
1274 static int brcm_pcie_suspend_noirq(struct device *dev)
1275 {
1276 	struct brcm_pcie *pcie = dev_get_drvdata(dev);
1277 	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
1278 	int ret;
1279 
1280 	brcm_pcie_turn_off(pcie);
1281 	/*
1282 	 * If brcm_phy_stop() returns an error, just dev_err(). If we
1283 	 * return the error it will cause the suspend to fail and this is a
1284 	 * forgivable offense that will probably be erased on resume.
1285 	 */
1286 	if (brcm_phy_stop(pcie))
1287 		dev_err(dev, "Could not stop phy for suspend\n");
1288 
1289 	ret = reset_control_rearm(pcie->rescal);
1290 	if (ret) {
1291 		dev_err(dev, "Could not rearm rescal reset\n");
1292 		return ret;
1293 	}
1294 
1295 	if (pcie->sr) {
1296 		/*
1297 		 * Now turn off the regulators, but if at least one
1298 		 * downstream device is enabled as a wake-up source, do not
1299 		 * turn off regulators.
1300 		 */
1301 		pcie->ep_wakeup_capable = false;
1302 		pci_walk_bus(bridge->bus, pci_dev_may_wakeup,
1303 			     &pcie->ep_wakeup_capable);
1304 		if (!pcie->ep_wakeup_capable) {
1305 			ret = regulator_bulk_disable(pcie->sr->num_supplies,
1306 						     pcie->sr->supplies);
1307 			if (ret) {
1308 				dev_err(dev, "Could not turn off regulators\n");
1309 				reset_control_reset(pcie->rescal);
1310 				return ret;
1311 			}
1312 		}
1313 	}
1314 	clk_disable_unprepare(pcie->clk);
1315 
1316 	return 0;
1317 }
1318 
1319 static int brcm_pcie_resume_noirq(struct device *dev)
1320 {
1321 	struct brcm_pcie *pcie = dev_get_drvdata(dev);
1322 	void __iomem *base;
1323 	u32 tmp;
1324 	int ret;
1325 
1326 	base = pcie->base;
1327 	ret = clk_prepare_enable(pcie->clk);
1328 	if (ret)
1329 		return ret;
1330 
1331 	ret = reset_control_reset(pcie->rescal);
1332 	if (ret)
1333 		goto err_disable_clk;
1334 
1335 	ret = brcm_phy_start(pcie);
1336 	if (ret)
1337 		goto err_reset;
1338 
1339 	/* Take bridge out of reset so we can access the SERDES reg */
1340 	pcie->bridge_sw_init_set(pcie, 0);
1341 
1342 	/* SERDES_IDDQ = 0 */
1343 	tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1344 	u32p_replace_bits(&tmp, 0, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK);
1345 	writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1346 
1347 	/* wait for serdes to be stable */
1348 	udelay(100);
1349 
1350 	ret = brcm_pcie_setup(pcie);
1351 	if (ret)
1352 		goto err_reset;
1353 
1354 	if (pcie->sr) {
1355 		if (pcie->ep_wakeup_capable) {
1356 			/*
1357 			 * We are resuming from a suspend.  In the suspend we
1358 			 * did not disable the power supplies, so there is
1359 			 * no need to enable them (and falsely increase their
1360 			 * usage count).
1361 			 */
1362 			pcie->ep_wakeup_capable = false;
1363 		} else {
1364 			ret = regulator_bulk_enable(pcie->sr->num_supplies,
1365 						    pcie->sr->supplies);
1366 			if (ret) {
1367 				dev_err(dev, "Could not turn on regulators\n");
1368 				goto err_reset;
1369 			}
1370 		}
1371 	}
1372 
1373 	ret = brcm_pcie_start_link(pcie);
1374 	if (ret)
1375 		goto err_regulator;
1376 
1377 	if (pcie->msi)
1378 		brcm_msi_set_regs(pcie->msi);
1379 
1380 	return 0;
1381 
1382 err_regulator:
1383 	if (pcie->sr)
1384 		regulator_bulk_disable(pcie->sr->num_supplies, pcie->sr->supplies);
1385 err_reset:
1386 	reset_control_rearm(pcie->rescal);
1387 err_disable_clk:
1388 	clk_disable_unprepare(pcie->clk);
1389 	return ret;
1390 }
1391 
1392 static void __brcm_pcie_remove(struct brcm_pcie *pcie)
1393 {
1394 	brcm_msi_remove(pcie);
1395 	brcm_pcie_turn_off(pcie);
1396 	if (brcm_phy_stop(pcie))
1397 		dev_err(pcie->dev, "Could not stop phy\n");
1398 	if (reset_control_rearm(pcie->rescal))
1399 		dev_err(pcie->dev, "Could not rearm rescal reset\n");
1400 	clk_disable_unprepare(pcie->clk);
1401 }
1402 
1403 static void brcm_pcie_remove(struct platform_device *pdev)
1404 {
1405 	struct brcm_pcie *pcie = platform_get_drvdata(pdev);
1406 	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
1407 
1408 	pci_stop_root_bus(bridge->bus);
1409 	pci_remove_root_bus(bridge->bus);
1410 	__brcm_pcie_remove(pcie);
1411 }
1412 
1413 static const int pcie_offsets[] = {
1414 	[RGR1_SW_INIT_1] = 0x9210,
1415 	[EXT_CFG_INDEX]  = 0x9000,
1416 	[EXT_CFG_DATA]   = 0x9004,
1417 };
1418 
1419 static const int pcie_offsets_bmips_7425[] = {
1420 	[RGR1_SW_INIT_1] = 0x8010,
1421 	[EXT_CFG_INDEX]  = 0x8300,
1422 	[EXT_CFG_DATA]   = 0x8304,
1423 };
1424 
1425 static const struct pcie_cfg_data generic_cfg = {
1426 	.offsets	= pcie_offsets,
1427 	.type		= GENERIC,
1428 	.perst_set	= brcm_pcie_perst_set_generic,
1429 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1430 };
1431 
1432 static const struct pcie_cfg_data bcm7425_cfg = {
1433 	.offsets	= pcie_offsets_bmips_7425,
1434 	.type		= BCM7425,
1435 	.perst_set	= brcm_pcie_perst_set_generic,
1436 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1437 };
1438 
1439 static const struct pcie_cfg_data bcm7435_cfg = {
1440 	.offsets	= pcie_offsets,
1441 	.type		= BCM7435,
1442 	.perst_set	= brcm_pcie_perst_set_generic,
1443 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1444 };
1445 
1446 static const struct pcie_cfg_data bcm4908_cfg = {
1447 	.offsets	= pcie_offsets,
1448 	.type		= BCM4908,
1449 	.perst_set	= brcm_pcie_perst_set_4908,
1450 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1451 };
1452 
1453 static const int pcie_offset_bcm7278[] = {
1454 	[RGR1_SW_INIT_1] = 0xc010,
1455 	[EXT_CFG_INDEX] = 0x9000,
1456 	[EXT_CFG_DATA] = 0x9004,
1457 };
1458 
1459 static const struct pcie_cfg_data bcm7278_cfg = {
1460 	.offsets	= pcie_offset_bcm7278,
1461 	.type		= BCM7278,
1462 	.perst_set	= brcm_pcie_perst_set_7278,
1463 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278,
1464 };
1465 
1466 static const struct pcie_cfg_data bcm2711_cfg = {
1467 	.offsets	= pcie_offsets,
1468 	.type		= BCM2711,
1469 	.perst_set	= brcm_pcie_perst_set_generic,
1470 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1471 };
1472 
1473 static const struct of_device_id brcm_pcie_match[] = {
1474 	{ .compatible = "brcm,bcm2711-pcie", .data = &bcm2711_cfg },
1475 	{ .compatible = "brcm,bcm4908-pcie", .data = &bcm4908_cfg },
1476 	{ .compatible = "brcm,bcm7211-pcie", .data = &generic_cfg },
1477 	{ .compatible = "brcm,bcm7278-pcie", .data = &bcm7278_cfg },
1478 	{ .compatible = "brcm,bcm7216-pcie", .data = &bcm7278_cfg },
1479 	{ .compatible = "brcm,bcm7445-pcie", .data = &generic_cfg },
1480 	{ .compatible = "brcm,bcm7435-pcie", .data = &bcm7435_cfg },
1481 	{ .compatible = "brcm,bcm7425-pcie", .data = &bcm7425_cfg },
1482 	{},
1483 };
1484 
1485 static struct pci_ops brcm_pcie_ops = {
1486 	.map_bus = brcm_pcie_map_bus,
1487 	.read = pci_generic_config_read,
1488 	.write = pci_generic_config_write,
1489 	.add_bus = brcm_pcie_add_bus,
1490 	.remove_bus = brcm_pcie_remove_bus,
1491 };
1492 
1493 static struct pci_ops brcm7425_pcie_ops = {
1494 	.map_bus = brcm7425_pcie_map_bus,
1495 	.read = pci_generic_config_read32,
1496 	.write = pci_generic_config_write32,
1497 	.add_bus = brcm_pcie_add_bus,
1498 	.remove_bus = brcm_pcie_remove_bus,
1499 };
1500 
1501 static int brcm_pcie_probe(struct platform_device *pdev)
1502 {
1503 	struct device_node *np = pdev->dev.of_node, *msi_np;
1504 	struct pci_host_bridge *bridge;
1505 	const struct pcie_cfg_data *data;
1506 	struct brcm_pcie *pcie;
1507 	int ret;
1508 
1509 	bridge = devm_pci_alloc_host_bridge(&pdev->dev, sizeof(*pcie));
1510 	if (!bridge)
1511 		return -ENOMEM;
1512 
1513 	data = of_device_get_match_data(&pdev->dev);
1514 	if (!data) {
1515 		pr_err("failed to look up compatible string\n");
1516 		return -EINVAL;
1517 	}
1518 
1519 	pcie = pci_host_bridge_priv(bridge);
1520 	pcie->dev = &pdev->dev;
1521 	pcie->np = np;
1522 	pcie->reg_offsets = data->offsets;
1523 	pcie->type = data->type;
1524 	pcie->perst_set = data->perst_set;
1525 	pcie->bridge_sw_init_set = data->bridge_sw_init_set;
1526 
1527 	pcie->base = devm_platform_ioremap_resource(pdev, 0);
1528 	if (IS_ERR(pcie->base))
1529 		return PTR_ERR(pcie->base);
1530 
1531 	pcie->clk = devm_clk_get_optional(&pdev->dev, "sw_pcie");
1532 	if (IS_ERR(pcie->clk))
1533 		return PTR_ERR(pcie->clk);
1534 
1535 	ret = of_pci_get_max_link_speed(np);
1536 	pcie->gen = (ret < 0) ? 0 : ret;
1537 
1538 	pcie->ssc = of_property_read_bool(np, "brcm,enable-ssc");
1539 
1540 	ret = clk_prepare_enable(pcie->clk);
1541 	if (ret) {
1542 		dev_err(&pdev->dev, "could not enable clock\n");
1543 		return ret;
1544 	}
1545 	pcie->rescal = devm_reset_control_get_optional_shared(&pdev->dev, "rescal");
1546 	if (IS_ERR(pcie->rescal)) {
1547 		clk_disable_unprepare(pcie->clk);
1548 		return PTR_ERR(pcie->rescal);
1549 	}
1550 	pcie->perst_reset = devm_reset_control_get_optional_exclusive(&pdev->dev, "perst");
1551 	if (IS_ERR(pcie->perst_reset)) {
1552 		clk_disable_unprepare(pcie->clk);
1553 		return PTR_ERR(pcie->perst_reset);
1554 	}
1555 
1556 	ret = reset_control_reset(pcie->rescal);
1557 	if (ret)
1558 		dev_err(&pdev->dev, "failed to deassert 'rescal'\n");
1559 
1560 	ret = brcm_phy_start(pcie);
1561 	if (ret) {
1562 		reset_control_rearm(pcie->rescal);
1563 		clk_disable_unprepare(pcie->clk);
1564 		return ret;
1565 	}
1566 
1567 	ret = brcm_pcie_setup(pcie);
1568 	if (ret)
1569 		goto fail;
1570 
1571 	pcie->hw_rev = readl(pcie->base + PCIE_MISC_REVISION);
1572 	if (pcie->type == BCM4908 && pcie->hw_rev >= BRCM_PCIE_HW_REV_3_20) {
1573 		dev_err(pcie->dev, "hardware revision with unsupported PERST# setup\n");
1574 		ret = -ENODEV;
1575 		goto fail;
1576 	}
1577 
1578 	msi_np = of_parse_phandle(pcie->np, "msi-parent", 0);
1579 	if (pci_msi_enabled() && msi_np == pcie->np) {
1580 		ret = brcm_pcie_enable_msi(pcie);
1581 		if (ret) {
1582 			dev_err(pcie->dev, "probe of internal MSI failed");
1583 			goto fail;
1584 		}
1585 	}
1586 
1587 	bridge->ops = pcie->type == BCM7425 ? &brcm7425_pcie_ops : &brcm_pcie_ops;
1588 	bridge->sysdata = pcie;
1589 
1590 	platform_set_drvdata(pdev, pcie);
1591 
1592 	ret = pci_host_probe(bridge);
1593 	if (!ret && !brcm_pcie_link_up(pcie))
1594 		ret = -ENODEV;
1595 
1596 	if (ret) {
1597 		brcm_pcie_remove(pdev);
1598 		return ret;
1599 	}
1600 
1601 	return 0;
1602 
1603 fail:
1604 	__brcm_pcie_remove(pcie);
1605 	return ret;
1606 }
1607 
1608 MODULE_DEVICE_TABLE(of, brcm_pcie_match);
1609 
1610 static const struct dev_pm_ops brcm_pcie_pm_ops = {
1611 	.suspend_noirq = brcm_pcie_suspend_noirq,
1612 	.resume_noirq = brcm_pcie_resume_noirq,
1613 };
1614 
1615 static struct platform_driver brcm_pcie_driver = {
1616 	.probe = brcm_pcie_probe,
1617 	.remove_new = brcm_pcie_remove,
1618 	.driver = {
1619 		.name = "brcm-pcie",
1620 		.of_match_table = brcm_pcie_match,
1621 		.pm = &brcm_pcie_pm_ops,
1622 	},
1623 };
1624 module_platform_driver(brcm_pcie_driver);
1625 
1626 MODULE_LICENSE("GPL");
1627 MODULE_DESCRIPTION("Broadcom STB PCIe RC driver");
1628 MODULE_AUTHOR("Broadcom");
1629