1 // SPDX-License-Identifier: GPL-2.0+
2 /* Copyright (C) 2009 - 2019 Broadcom */
3 
4 #include <linux/bitfield.h>
5 #include <linux/bitops.h>
6 #include <linux/clk.h>
7 #include <linux/compiler.h>
8 #include <linux/delay.h>
9 #include <linux/init.h>
10 #include <linux/interrupt.h>
11 #include <linux/io.h>
12 #include <linux/ioport.h>
13 #include <linux/irqchip/chained_irq.h>
14 #include <linux/irqdomain.h>
15 #include <linux/kernel.h>
16 #include <linux/list.h>
17 #include <linux/log2.h>
18 #include <linux/module.h>
19 #include <linux/msi.h>
20 #include <linux/of_address.h>
21 #include <linux/of_irq.h>
22 #include <linux/of_pci.h>
23 #include <linux/of_platform.h>
24 #include <linux/pci.h>
25 #include <linux/pci-ecam.h>
26 #include <linux/printk.h>
27 #include <linux/reset.h>
28 #include <linux/sizes.h>
29 #include <linux/slab.h>
30 #include <linux/string.h>
31 #include <linux/types.h>
32 
33 #include "../pci.h"
34 
35 /* BRCM_PCIE_CAP_REGS - Offset for the mandatory capability config regs */
36 #define BRCM_PCIE_CAP_REGS				0x00ac
37 
38 /* Broadcom STB PCIe Register Offsets */
39 #define PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1				0x0188
40 #define  PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK	0xc
41 #define  PCIE_RC_CFG_VENDOR_SPCIFIC_REG1_LITTLE_ENDIAN			0x0
42 
43 #define PCIE_RC_CFG_PRIV1_ID_VAL3			0x043c
44 #define  PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK	0xffffff
45 
46 #define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY			0x04dc
47 #define  PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_ASPM_SUPPORT_MASK	0xc00
48 
49 #define PCIE_RC_DL_MDIO_ADDR				0x1100
50 #define PCIE_RC_DL_MDIO_WR_DATA				0x1104
51 #define PCIE_RC_DL_MDIO_RD_DATA				0x1108
52 
53 #define PCIE_MISC_MISC_CTRL				0x4008
54 #define  PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK		0x1000
55 #define  PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK	0x2000
56 #define  PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK	0x300000
57 
58 #define  PCIE_MISC_MISC_CTRL_SCB0_SIZE_MASK		0xf8000000
59 #define  PCIE_MISC_MISC_CTRL_SCB1_SIZE_MASK		0x07c00000
60 #define  PCIE_MISC_MISC_CTRL_SCB2_SIZE_MASK		0x0000001f
61 #define  SCB_SIZE_MASK(x) PCIE_MISC_MISC_CTRL_SCB ## x ## _SIZE_MASK
62 
63 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO		0x400c
64 #define PCIE_MEM_WIN0_LO(win)	\
65 		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO + ((win) * 8)
66 
67 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI		0x4010
68 #define PCIE_MEM_WIN0_HI(win)	\
69 		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI + ((win) * 8)
70 
71 #define PCIE_MISC_RC_BAR1_CONFIG_LO			0x402c
72 #define  PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK		0x1f
73 
74 #define PCIE_MISC_RC_BAR2_CONFIG_LO			0x4034
75 #define  PCIE_MISC_RC_BAR2_CONFIG_LO_SIZE_MASK		0x1f
76 #define PCIE_MISC_RC_BAR2_CONFIG_HI			0x4038
77 
78 #define PCIE_MISC_RC_BAR3_CONFIG_LO			0x403c
79 #define  PCIE_MISC_RC_BAR3_CONFIG_LO_SIZE_MASK		0x1f
80 
81 #define PCIE_MISC_MSI_BAR_CONFIG_LO			0x4044
82 #define PCIE_MISC_MSI_BAR_CONFIG_HI			0x4048
83 
84 #define PCIE_MISC_MSI_DATA_CONFIG			0x404c
85 #define  PCIE_MISC_MSI_DATA_CONFIG_VAL_32		0xffe06540
86 #define  PCIE_MISC_MSI_DATA_CONFIG_VAL_8		0xfff86540
87 
88 #define PCIE_MISC_PCIE_CTRL				0x4064
89 #define  PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK	0x1
90 #define PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK		0x4
91 
92 #define PCIE_MISC_PCIE_STATUS				0x4068
93 #define  PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK		0x80
94 #define  PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK	0x20
95 #define  PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK	0x10
96 #define  PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK	0x40
97 
98 #define PCIE_MISC_REVISION				0x406c
99 #define  BRCM_PCIE_HW_REV_33				0x0303
100 #define  BRCM_PCIE_HW_REV_3_20				0x0320
101 
102 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT		0x4070
103 #define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK	0xfff00000
104 #define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK	0xfff0
105 #define PCIE_MEM_WIN0_BASE_LIMIT(win)	\
106 		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT + ((win) * 4)
107 
108 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI			0x4080
109 #define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK	0xff
110 #define PCIE_MEM_WIN0_BASE_HI(win)	\
111 		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI + ((win) * 8)
112 
113 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI			0x4084
114 #define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK	0xff
115 #define PCIE_MEM_WIN0_LIMIT_HI(win)	\
116 		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI + ((win) * 8)
117 
118 #define PCIE_MISC_HARD_PCIE_HARD_DEBUG					0x4204
119 #define  PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK	0x2
120 #define  PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK		0x08000000
121 
122 
123 #define PCIE_INTR2_CPU_BASE		0x4300
124 #define PCIE_MSI_INTR2_BASE		0x4500
125 /* Offsets from PCIE_INTR2_CPU_BASE and PCIE_MSI_INTR2_BASE */
126 #define  MSI_INT_STATUS			0x0
127 #define  MSI_INT_CLR			0x8
128 #define  MSI_INT_MASK_SET		0x10
129 #define  MSI_INT_MASK_CLR		0x14
130 
131 #define PCIE_EXT_CFG_DATA				0x8000
132 #define PCIE_EXT_CFG_INDEX				0x9000
133 
134 #define  PCIE_RGR1_SW_INIT_1_PERST_MASK			0x1
135 #define  PCIE_RGR1_SW_INIT_1_PERST_SHIFT		0x0
136 
137 #define RGR1_SW_INIT_1_INIT_GENERIC_MASK		0x2
138 #define RGR1_SW_INIT_1_INIT_GENERIC_SHIFT		0x1
139 #define RGR1_SW_INIT_1_INIT_7278_MASK			0x1
140 #define RGR1_SW_INIT_1_INIT_7278_SHIFT			0x0
141 
142 /* PCIe parameters */
143 #define BRCM_NUM_PCIE_OUT_WINS		0x4
144 #define BRCM_INT_PCI_MSI_NR		32
145 #define BRCM_INT_PCI_MSI_LEGACY_NR	8
146 #define BRCM_INT_PCI_MSI_SHIFT		0
147 
148 /* MSI target adresses */
149 #define BRCM_MSI_TARGET_ADDR_LT_4GB	0x0fffffffcULL
150 #define BRCM_MSI_TARGET_ADDR_GT_4GB	0xffffffffcULL
151 
152 /* MDIO registers */
153 #define MDIO_PORT0			0x0
154 #define MDIO_DATA_MASK			0x7fffffff
155 #define MDIO_PORT_MASK			0xf0000
156 #define MDIO_REGAD_MASK			0xffff
157 #define MDIO_CMD_MASK			0xfff00000
158 #define MDIO_CMD_READ			0x1
159 #define MDIO_CMD_WRITE			0x0
160 #define MDIO_DATA_DONE_MASK		0x80000000
161 #define MDIO_RD_DONE(x)			(((x) & MDIO_DATA_DONE_MASK) ? 1 : 0)
162 #define MDIO_WT_DONE(x)			(((x) & MDIO_DATA_DONE_MASK) ? 0 : 1)
163 #define SSC_REGS_ADDR			0x1100
164 #define SET_ADDR_OFFSET			0x1f
165 #define SSC_CNTL_OFFSET			0x2
166 #define SSC_CNTL_OVRD_EN_MASK		0x8000
167 #define SSC_CNTL_OVRD_VAL_MASK		0x4000
168 #define SSC_STATUS_OFFSET		0x1
169 #define SSC_STATUS_SSC_MASK		0x400
170 #define SSC_STATUS_PLL_LOCK_MASK	0x800
171 #define PCIE_BRCM_MAX_MEMC		3
172 
173 #define IDX_ADDR(pcie)			(pcie->reg_offsets[EXT_CFG_INDEX])
174 #define DATA_ADDR(pcie)			(pcie->reg_offsets[EXT_CFG_DATA])
175 #define PCIE_RGR1_SW_INIT_1(pcie)	(pcie->reg_offsets[RGR1_SW_INIT_1])
176 
177 /* Rescal registers */
178 #define PCIE_DVT_PMU_PCIE_PHY_CTRL				0xc700
179 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS			0x3
180 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_MASK		0x4
181 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_SHIFT	0x2
182 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_MASK		0x2
183 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_SHIFT		0x1
184 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_MASK		0x1
185 #define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_SHIFT		0x0
186 
187 /* Forward declarations */
188 struct brcm_pcie;
189 static inline void brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 val);
190 static inline void brcm_pcie_bridge_sw_init_set_generic(struct brcm_pcie *pcie, u32 val);
191 static inline void brcm_pcie_perst_set_4908(struct brcm_pcie *pcie, u32 val);
192 static inline void brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val);
193 static inline void brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, u32 val);
194 
195 enum {
196 	RGR1_SW_INIT_1,
197 	EXT_CFG_INDEX,
198 	EXT_CFG_DATA,
199 };
200 
201 enum {
202 	RGR1_SW_INIT_1_INIT_MASK,
203 	RGR1_SW_INIT_1_INIT_SHIFT,
204 };
205 
206 enum pcie_type {
207 	GENERIC,
208 	BCM4908,
209 	BCM7278,
210 	BCM2711,
211 };
212 
213 struct pcie_cfg_data {
214 	const int *offsets;
215 	const enum pcie_type type;
216 	void (*perst_set)(struct brcm_pcie *pcie, u32 val);
217 	void (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val);
218 };
219 
220 static const int pcie_offsets[] = {
221 	[RGR1_SW_INIT_1] = 0x9210,
222 	[EXT_CFG_INDEX]  = 0x9000,
223 	[EXT_CFG_DATA]   = 0x9004,
224 };
225 
226 static const struct pcie_cfg_data generic_cfg = {
227 	.offsets	= pcie_offsets,
228 	.type		= GENERIC,
229 	.perst_set	= brcm_pcie_perst_set_generic,
230 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
231 };
232 
233 static const struct pcie_cfg_data bcm4908_cfg = {
234 	.offsets	= pcie_offsets,
235 	.type		= BCM4908,
236 	.perst_set	= brcm_pcie_perst_set_4908,
237 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
238 };
239 
240 static const int pcie_offset_bcm7278[] = {
241 	[RGR1_SW_INIT_1] = 0xc010,
242 	[EXT_CFG_INDEX] = 0x9000,
243 	[EXT_CFG_DATA] = 0x9004,
244 };
245 
246 static const struct pcie_cfg_data bcm7278_cfg = {
247 	.offsets	= pcie_offset_bcm7278,
248 	.type		= BCM7278,
249 	.perst_set	= brcm_pcie_perst_set_7278,
250 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278,
251 };
252 
253 static const struct pcie_cfg_data bcm2711_cfg = {
254 	.offsets	= pcie_offsets,
255 	.type		= BCM2711,
256 	.perst_set	= brcm_pcie_perst_set_generic,
257 	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
258 };
259 
260 struct brcm_msi {
261 	struct device		*dev;
262 	void __iomem		*base;
263 	struct device_node	*np;
264 	struct irq_domain	*msi_domain;
265 	struct irq_domain	*inner_domain;
266 	struct mutex		lock; /* guards the alloc/free operations */
267 	u64			target_addr;
268 	int			irq;
269 	/* used indicates which MSI interrupts have been alloc'd */
270 	unsigned long		used;
271 	bool			legacy;
272 	/* Some chips have MSIs in bits [31..24] of a shared register. */
273 	int			legacy_shift;
274 	int			nr; /* No. of MSI available, depends on chip */
275 	/* This is the base pointer for interrupt status/set/clr regs */
276 	void __iomem		*intr_base;
277 };
278 
279 /* Internal PCIe Host Controller Information.*/
280 struct brcm_pcie {
281 	struct device		*dev;
282 	void __iomem		*base;
283 	struct clk		*clk;
284 	struct device_node	*np;
285 	bool			ssc;
286 	int			gen;
287 	u64			msi_target_addr;
288 	struct brcm_msi		*msi;
289 	const int		*reg_offsets;
290 	enum pcie_type		type;
291 	struct reset_control	*rescal;
292 	struct reset_control	*perst_reset;
293 	int			num_memc;
294 	u64			memc_size[PCIE_BRCM_MAX_MEMC];
295 	u32			hw_rev;
296 	void			(*perst_set)(struct brcm_pcie *pcie, u32 val);
297 	void			(*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val);
298 };
299 
300 /*
301  * This is to convert the size of the inbound "BAR" region to the
302  * non-linear values of PCIE_X_MISC_RC_BAR[123]_CONFIG_LO.SIZE
303  */
304 static int brcm_pcie_encode_ibar_size(u64 size)
305 {
306 	int log2_in = ilog2(size);
307 
308 	if (log2_in >= 12 && log2_in <= 15)
309 		/* Covers 4KB to 32KB (inclusive) */
310 		return (log2_in - 12) + 0x1c;
311 	else if (log2_in >= 16 && log2_in <= 35)
312 		/* Covers 64KB to 32GB, (inclusive) */
313 		return log2_in - 15;
314 	/* Something is awry so disable */
315 	return 0;
316 }
317 
318 static u32 brcm_pcie_mdio_form_pkt(int port, int regad, int cmd)
319 {
320 	u32 pkt = 0;
321 
322 	pkt |= FIELD_PREP(MDIO_PORT_MASK, port);
323 	pkt |= FIELD_PREP(MDIO_REGAD_MASK, regad);
324 	pkt |= FIELD_PREP(MDIO_CMD_MASK, cmd);
325 
326 	return pkt;
327 }
328 
329 /* negative return value indicates error */
330 static int brcm_pcie_mdio_read(void __iomem *base, u8 port, u8 regad, u32 *val)
331 {
332 	int tries;
333 	u32 data;
334 
335 	writel(brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_READ),
336 		   base + PCIE_RC_DL_MDIO_ADDR);
337 	readl(base + PCIE_RC_DL_MDIO_ADDR);
338 
339 	data = readl(base + PCIE_RC_DL_MDIO_RD_DATA);
340 	for (tries = 0; !MDIO_RD_DONE(data) && tries < 10; tries++) {
341 		udelay(10);
342 		data = readl(base + PCIE_RC_DL_MDIO_RD_DATA);
343 	}
344 
345 	*val = FIELD_GET(MDIO_DATA_MASK, data);
346 	return MDIO_RD_DONE(data) ? 0 : -EIO;
347 }
348 
349 /* negative return value indicates error */
350 static int brcm_pcie_mdio_write(void __iomem *base, u8 port,
351 				u8 regad, u16 wrdata)
352 {
353 	int tries;
354 	u32 data;
355 
356 	writel(brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_WRITE),
357 		   base + PCIE_RC_DL_MDIO_ADDR);
358 	readl(base + PCIE_RC_DL_MDIO_ADDR);
359 	writel(MDIO_DATA_DONE_MASK | wrdata, base + PCIE_RC_DL_MDIO_WR_DATA);
360 
361 	data = readl(base + PCIE_RC_DL_MDIO_WR_DATA);
362 	for (tries = 0; !MDIO_WT_DONE(data) && tries < 10; tries++) {
363 		udelay(10);
364 		data = readl(base + PCIE_RC_DL_MDIO_WR_DATA);
365 	}
366 
367 	return MDIO_WT_DONE(data) ? 0 : -EIO;
368 }
369 
370 /*
371  * Configures device for Spread Spectrum Clocking (SSC) mode; a negative
372  * return value indicates error.
373  */
374 static int brcm_pcie_set_ssc(struct brcm_pcie *pcie)
375 {
376 	int pll, ssc;
377 	int ret;
378 	u32 tmp;
379 
380 	ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, SET_ADDR_OFFSET,
381 				   SSC_REGS_ADDR);
382 	if (ret < 0)
383 		return ret;
384 
385 	ret = brcm_pcie_mdio_read(pcie->base, MDIO_PORT0,
386 				  SSC_CNTL_OFFSET, &tmp);
387 	if (ret < 0)
388 		return ret;
389 
390 	u32p_replace_bits(&tmp, 1, SSC_CNTL_OVRD_EN_MASK);
391 	u32p_replace_bits(&tmp, 1, SSC_CNTL_OVRD_VAL_MASK);
392 	ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0,
393 				   SSC_CNTL_OFFSET, tmp);
394 	if (ret < 0)
395 		return ret;
396 
397 	usleep_range(1000, 2000);
398 	ret = brcm_pcie_mdio_read(pcie->base, MDIO_PORT0,
399 				  SSC_STATUS_OFFSET, &tmp);
400 	if (ret < 0)
401 		return ret;
402 
403 	ssc = FIELD_GET(SSC_STATUS_SSC_MASK, tmp);
404 	pll = FIELD_GET(SSC_STATUS_PLL_LOCK_MASK, tmp);
405 
406 	return ssc && pll ? 0 : -EIO;
407 }
408 
409 /* Limits operation to a specific generation (1, 2, or 3) */
410 static void brcm_pcie_set_gen(struct brcm_pcie *pcie, int gen)
411 {
412 	u16 lnkctl2 = readw(pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCTL2);
413 	u32 lnkcap = readl(pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCAP);
414 
415 	lnkcap = (lnkcap & ~PCI_EXP_LNKCAP_SLS) | gen;
416 	writel(lnkcap, pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCAP);
417 
418 	lnkctl2 = (lnkctl2 & ~0xf) | gen;
419 	writew(lnkctl2, pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCTL2);
420 }
421 
422 static void brcm_pcie_set_outbound_win(struct brcm_pcie *pcie,
423 				       unsigned int win, u64 cpu_addr,
424 				       u64 pcie_addr, u64 size)
425 {
426 	u32 cpu_addr_mb_high, limit_addr_mb_high;
427 	phys_addr_t cpu_addr_mb, limit_addr_mb;
428 	int high_addr_shift;
429 	u32 tmp;
430 
431 	/* Set the base of the pcie_addr window */
432 	writel(lower_32_bits(pcie_addr), pcie->base + PCIE_MEM_WIN0_LO(win));
433 	writel(upper_32_bits(pcie_addr), pcie->base + PCIE_MEM_WIN0_HI(win));
434 
435 	/* Write the addr base & limit lower bits (in MBs) */
436 	cpu_addr_mb = cpu_addr / SZ_1M;
437 	limit_addr_mb = (cpu_addr + size - 1) / SZ_1M;
438 
439 	tmp = readl(pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win));
440 	u32p_replace_bits(&tmp, cpu_addr_mb,
441 			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK);
442 	u32p_replace_bits(&tmp, limit_addr_mb,
443 			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK);
444 	writel(tmp, pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win));
445 
446 	/* Write the cpu & limit addr upper bits */
447 	high_addr_shift =
448 		HWEIGHT32(PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK);
449 
450 	cpu_addr_mb_high = cpu_addr_mb >> high_addr_shift;
451 	tmp = readl(pcie->base + PCIE_MEM_WIN0_BASE_HI(win));
452 	u32p_replace_bits(&tmp, cpu_addr_mb_high,
453 			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK);
454 	writel(tmp, pcie->base + PCIE_MEM_WIN0_BASE_HI(win));
455 
456 	limit_addr_mb_high = limit_addr_mb >> high_addr_shift;
457 	tmp = readl(pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win));
458 	u32p_replace_bits(&tmp, limit_addr_mb_high,
459 			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK);
460 	writel(tmp, pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win));
461 }
462 
463 static struct irq_chip brcm_msi_irq_chip = {
464 	.name            = "BRCM STB PCIe MSI",
465 	.irq_ack         = irq_chip_ack_parent,
466 	.irq_mask        = pci_msi_mask_irq,
467 	.irq_unmask      = pci_msi_unmask_irq,
468 };
469 
470 static struct msi_domain_info brcm_msi_domain_info = {
471 	/* Multi MSI is supported by the controller, but not by this driver */
472 	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS),
473 	.chip	= &brcm_msi_irq_chip,
474 };
475 
476 static void brcm_pcie_msi_isr(struct irq_desc *desc)
477 {
478 	struct irq_chip *chip = irq_desc_get_chip(desc);
479 	unsigned long status, virq;
480 	struct brcm_msi *msi;
481 	struct device *dev;
482 	u32 bit;
483 
484 	chained_irq_enter(chip, desc);
485 	msi = irq_desc_get_handler_data(desc);
486 	dev = msi->dev;
487 
488 	status = readl(msi->intr_base + MSI_INT_STATUS);
489 	status >>= msi->legacy_shift;
490 
491 	for_each_set_bit(bit, &status, msi->nr) {
492 		virq = irq_find_mapping(msi->inner_domain, bit);
493 		if (virq)
494 			generic_handle_irq(virq);
495 		else
496 			dev_dbg(dev, "unexpected MSI\n");
497 	}
498 
499 	chained_irq_exit(chip, desc);
500 }
501 
502 static void brcm_msi_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
503 {
504 	struct brcm_msi *msi = irq_data_get_irq_chip_data(data);
505 
506 	msg->address_lo = lower_32_bits(msi->target_addr);
507 	msg->address_hi = upper_32_bits(msi->target_addr);
508 	msg->data = (0xffff & PCIE_MISC_MSI_DATA_CONFIG_VAL_32) | data->hwirq;
509 }
510 
511 static int brcm_msi_set_affinity(struct irq_data *irq_data,
512 				 const struct cpumask *mask, bool force)
513 {
514 	return -EINVAL;
515 }
516 
517 static void brcm_msi_ack_irq(struct irq_data *data)
518 {
519 	struct brcm_msi *msi = irq_data_get_irq_chip_data(data);
520 	const int shift_amt = data->hwirq + msi->legacy_shift;
521 
522 	writel(1 << shift_amt, msi->intr_base + MSI_INT_CLR);
523 }
524 
525 
526 static struct irq_chip brcm_msi_bottom_irq_chip = {
527 	.name			= "BRCM STB MSI",
528 	.irq_compose_msi_msg	= brcm_msi_compose_msi_msg,
529 	.irq_set_affinity	= brcm_msi_set_affinity,
530 	.irq_ack                = brcm_msi_ack_irq,
531 };
532 
533 static int brcm_msi_alloc(struct brcm_msi *msi)
534 {
535 	int hwirq;
536 
537 	mutex_lock(&msi->lock);
538 	hwirq = bitmap_find_free_region(&msi->used, msi->nr, 0);
539 	mutex_unlock(&msi->lock);
540 
541 	return hwirq;
542 }
543 
544 static void brcm_msi_free(struct brcm_msi *msi, unsigned long hwirq)
545 {
546 	mutex_lock(&msi->lock);
547 	bitmap_release_region(&msi->used, hwirq, 0);
548 	mutex_unlock(&msi->lock);
549 }
550 
551 static int brcm_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
552 				 unsigned int nr_irqs, void *args)
553 {
554 	struct brcm_msi *msi = domain->host_data;
555 	int hwirq;
556 
557 	hwirq = brcm_msi_alloc(msi);
558 
559 	if (hwirq < 0)
560 		return hwirq;
561 
562 	irq_domain_set_info(domain, virq, (irq_hw_number_t)hwirq,
563 			    &brcm_msi_bottom_irq_chip, domain->host_data,
564 			    handle_edge_irq, NULL, NULL);
565 	return 0;
566 }
567 
568 static void brcm_irq_domain_free(struct irq_domain *domain,
569 				 unsigned int virq, unsigned int nr_irqs)
570 {
571 	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
572 	struct brcm_msi *msi = irq_data_get_irq_chip_data(d);
573 
574 	brcm_msi_free(msi, d->hwirq);
575 }
576 
577 static const struct irq_domain_ops msi_domain_ops = {
578 	.alloc	= brcm_irq_domain_alloc,
579 	.free	= brcm_irq_domain_free,
580 };
581 
582 static int brcm_allocate_domains(struct brcm_msi *msi)
583 {
584 	struct fwnode_handle *fwnode = of_node_to_fwnode(msi->np);
585 	struct device *dev = msi->dev;
586 
587 	msi->inner_domain = irq_domain_add_linear(NULL, msi->nr, &msi_domain_ops, msi);
588 	if (!msi->inner_domain) {
589 		dev_err(dev, "failed to create IRQ domain\n");
590 		return -ENOMEM;
591 	}
592 
593 	msi->msi_domain = pci_msi_create_irq_domain(fwnode,
594 						    &brcm_msi_domain_info,
595 						    msi->inner_domain);
596 	if (!msi->msi_domain) {
597 		dev_err(dev, "failed to create MSI domain\n");
598 		irq_domain_remove(msi->inner_domain);
599 		return -ENOMEM;
600 	}
601 
602 	return 0;
603 }
604 
605 static void brcm_free_domains(struct brcm_msi *msi)
606 {
607 	irq_domain_remove(msi->msi_domain);
608 	irq_domain_remove(msi->inner_domain);
609 }
610 
611 static void brcm_msi_remove(struct brcm_pcie *pcie)
612 {
613 	struct brcm_msi *msi = pcie->msi;
614 
615 	if (!msi)
616 		return;
617 	irq_set_chained_handler_and_data(msi->irq, NULL, NULL);
618 	brcm_free_domains(msi);
619 }
620 
621 static void brcm_msi_set_regs(struct brcm_msi *msi)
622 {
623 	u32 val = __GENMASK(31, msi->legacy_shift);
624 
625 	writel(val, msi->intr_base + MSI_INT_MASK_CLR);
626 	writel(val, msi->intr_base + MSI_INT_CLR);
627 
628 	/*
629 	 * The 0 bit of PCIE_MISC_MSI_BAR_CONFIG_LO is repurposed to MSI
630 	 * enable, which we set to 1.
631 	 */
632 	writel(lower_32_bits(msi->target_addr) | 0x1,
633 	       msi->base + PCIE_MISC_MSI_BAR_CONFIG_LO);
634 	writel(upper_32_bits(msi->target_addr),
635 	       msi->base + PCIE_MISC_MSI_BAR_CONFIG_HI);
636 
637 	val = msi->legacy ? PCIE_MISC_MSI_DATA_CONFIG_VAL_8 : PCIE_MISC_MSI_DATA_CONFIG_VAL_32;
638 	writel(val, msi->base + PCIE_MISC_MSI_DATA_CONFIG);
639 }
640 
641 static int brcm_pcie_enable_msi(struct brcm_pcie *pcie)
642 {
643 	struct brcm_msi *msi;
644 	int irq, ret;
645 	struct device *dev = pcie->dev;
646 
647 	irq = irq_of_parse_and_map(dev->of_node, 1);
648 	if (irq <= 0) {
649 		dev_err(dev, "cannot map MSI interrupt\n");
650 		return -ENODEV;
651 	}
652 
653 	msi = devm_kzalloc(dev, sizeof(struct brcm_msi), GFP_KERNEL);
654 	if (!msi)
655 		return -ENOMEM;
656 
657 	mutex_init(&msi->lock);
658 	msi->dev = dev;
659 	msi->base = pcie->base;
660 	msi->np = pcie->np;
661 	msi->target_addr = pcie->msi_target_addr;
662 	msi->irq = irq;
663 	msi->legacy = pcie->hw_rev < BRCM_PCIE_HW_REV_33;
664 
665 	if (msi->legacy) {
666 		msi->intr_base = msi->base + PCIE_INTR2_CPU_BASE;
667 		msi->nr = BRCM_INT_PCI_MSI_LEGACY_NR;
668 		msi->legacy_shift = 24;
669 	} else {
670 		msi->intr_base = msi->base + PCIE_MSI_INTR2_BASE;
671 		msi->nr = BRCM_INT_PCI_MSI_NR;
672 		msi->legacy_shift = 0;
673 	}
674 
675 	ret = brcm_allocate_domains(msi);
676 	if (ret)
677 		return ret;
678 
679 	irq_set_chained_handler_and_data(msi->irq, brcm_pcie_msi_isr, msi);
680 
681 	brcm_msi_set_regs(msi);
682 	pcie->msi = msi;
683 
684 	return 0;
685 }
686 
687 /* The controller is capable of serving in both RC and EP roles */
688 static bool brcm_pcie_rc_mode(struct brcm_pcie *pcie)
689 {
690 	void __iomem *base = pcie->base;
691 	u32 val = readl(base + PCIE_MISC_PCIE_STATUS);
692 
693 	return !!FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK, val);
694 }
695 
696 static bool brcm_pcie_link_up(struct brcm_pcie *pcie)
697 {
698 	u32 val = readl(pcie->base + PCIE_MISC_PCIE_STATUS);
699 	u32 dla = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK, val);
700 	u32 plu = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK, val);
701 
702 	return dla && plu;
703 }
704 
705 static void __iomem *brcm_pcie_map_conf(struct pci_bus *bus, unsigned int devfn,
706 					int where)
707 {
708 	struct brcm_pcie *pcie = bus->sysdata;
709 	void __iomem *base = pcie->base;
710 	int idx;
711 
712 	/* Accesses to the RC go right to the RC registers if slot==0 */
713 	if (pci_is_root_bus(bus))
714 		return PCI_SLOT(devfn) ? NULL : base + where;
715 
716 	/* For devices, write to the config space index register */
717 	idx = PCIE_ECAM_OFFSET(bus->number, devfn, 0);
718 	writel(idx, pcie->base + PCIE_EXT_CFG_INDEX);
719 	return base + PCIE_EXT_CFG_DATA + where;
720 }
721 
722 static struct pci_ops brcm_pcie_ops = {
723 	.map_bus = brcm_pcie_map_conf,
724 	.read = pci_generic_config_read,
725 	.write = pci_generic_config_write,
726 };
727 
728 static inline void brcm_pcie_bridge_sw_init_set_generic(struct brcm_pcie *pcie, u32 val)
729 {
730 	u32 tmp, mask =  RGR1_SW_INIT_1_INIT_GENERIC_MASK;
731 	u32 shift = RGR1_SW_INIT_1_INIT_GENERIC_SHIFT;
732 
733 	tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
734 	tmp = (tmp & ~mask) | ((val << shift) & mask);
735 	writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
736 }
737 
738 static inline void brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 val)
739 {
740 	u32 tmp, mask =  RGR1_SW_INIT_1_INIT_7278_MASK;
741 	u32 shift = RGR1_SW_INIT_1_INIT_7278_SHIFT;
742 
743 	tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
744 	tmp = (tmp & ~mask) | ((val << shift) & mask);
745 	writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
746 }
747 
748 static inline void brcm_pcie_perst_set_4908(struct brcm_pcie *pcie, u32 val)
749 {
750 	if (WARN_ONCE(!pcie->perst_reset, "missing PERST# reset controller\n"))
751 		return;
752 
753 	if (val)
754 		reset_control_assert(pcie->perst_reset);
755 	else
756 		reset_control_deassert(pcie->perst_reset);
757 }
758 
759 static inline void brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val)
760 {
761 	u32 tmp;
762 
763 	/* Perst bit has moved and assert value is 0 */
764 	tmp = readl(pcie->base + PCIE_MISC_PCIE_CTRL);
765 	u32p_replace_bits(&tmp, !val, PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK);
766 	writel(tmp, pcie->base +  PCIE_MISC_PCIE_CTRL);
767 }
768 
769 static inline void brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, u32 val)
770 {
771 	u32 tmp;
772 
773 	tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
774 	u32p_replace_bits(&tmp, val, PCIE_RGR1_SW_INIT_1_PERST_MASK);
775 	writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
776 }
777 
778 static inline int brcm_pcie_get_rc_bar2_size_and_offset(struct brcm_pcie *pcie,
779 							u64 *rc_bar2_size,
780 							u64 *rc_bar2_offset)
781 {
782 	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
783 	struct resource_entry *entry;
784 	struct device *dev = pcie->dev;
785 	u64 lowest_pcie_addr = ~(u64)0;
786 	int ret, i = 0;
787 	u64 size = 0;
788 
789 	resource_list_for_each_entry(entry, &bridge->dma_ranges) {
790 		u64 pcie_beg = entry->res->start - entry->offset;
791 
792 		size += entry->res->end - entry->res->start + 1;
793 		if (pcie_beg < lowest_pcie_addr)
794 			lowest_pcie_addr = pcie_beg;
795 	}
796 
797 	if (lowest_pcie_addr == ~(u64)0) {
798 		dev_err(dev, "DT node has no dma-ranges\n");
799 		return -EINVAL;
800 	}
801 
802 	ret = of_property_read_variable_u64_array(pcie->np, "brcm,scb-sizes", pcie->memc_size, 1,
803 						  PCIE_BRCM_MAX_MEMC);
804 
805 	if (ret <= 0) {
806 		/* Make an educated guess */
807 		pcie->num_memc = 1;
808 		pcie->memc_size[0] = 1ULL << fls64(size - 1);
809 	} else {
810 		pcie->num_memc = ret;
811 	}
812 
813 	/* Each memc is viewed through a "port" that is a power of 2 */
814 	for (i = 0, size = 0; i < pcie->num_memc; i++)
815 		size += pcie->memc_size[i];
816 
817 	/* System memory starts at this address in PCIe-space */
818 	*rc_bar2_offset = lowest_pcie_addr;
819 	/* The sum of all memc views must also be a power of 2 */
820 	*rc_bar2_size = 1ULL << fls64(size - 1);
821 
822 	/*
823 	 * We validate the inbound memory view even though we should trust
824 	 * whatever the device-tree provides. This is because of an HW issue on
825 	 * early Raspberry Pi 4's revisions (bcm2711). It turns out its
826 	 * firmware has to dynamically edit dma-ranges due to a bug on the
827 	 * PCIe controller integration, which prohibits any access above the
828 	 * lower 3GB of memory. Given this, we decided to keep the dma-ranges
829 	 * in check, avoiding hard to debug device-tree related issues in the
830 	 * future:
831 	 *
832 	 * The PCIe host controller by design must set the inbound viewport to
833 	 * be a contiguous arrangement of all of the system's memory.  In
834 	 * addition, its size mut be a power of two.  To further complicate
835 	 * matters, the viewport must start on a pcie-address that is aligned
836 	 * on a multiple of its size.  If a portion of the viewport does not
837 	 * represent system memory -- e.g. 3GB of memory requires a 4GB
838 	 * viewport -- we can map the outbound memory in or after 3GB and even
839 	 * though the viewport will overlap the outbound memory the controller
840 	 * will know to send outbound memory downstream and everything else
841 	 * upstream.
842 	 *
843 	 * For example:
844 	 *
845 	 * - The best-case scenario, memory up to 3GB, is to place the inbound
846 	 *   region in the first 4GB of pcie-space, as some legacy devices can
847 	 *   only address 32bits. We would also like to put the MSI under 4GB
848 	 *   as well, since some devices require a 32bit MSI target address.
849 	 *
850 	 * - If the system memory is 4GB or larger we cannot start the inbound
851 	 *   region at location 0 (since we have to allow some space for
852 	 *   outbound memory @ 3GB). So instead it will  start at the 1x
853 	 *   multiple of its size
854 	 */
855 	if (!*rc_bar2_size || (*rc_bar2_offset & (*rc_bar2_size - 1)) ||
856 	    (*rc_bar2_offset < SZ_4G && *rc_bar2_offset > SZ_2G)) {
857 		dev_err(dev, "Invalid rc_bar2_offset/size: size 0x%llx, off 0x%llx\n",
858 			*rc_bar2_size, *rc_bar2_offset);
859 		return -EINVAL;
860 	}
861 
862 	return 0;
863 }
864 
865 static int brcm_pcie_setup(struct brcm_pcie *pcie)
866 {
867 	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
868 	u64 rc_bar2_offset, rc_bar2_size;
869 	void __iomem *base = pcie->base;
870 	struct device *dev = pcie->dev;
871 	struct resource_entry *entry;
872 	bool ssc_good = false;
873 	struct resource *res;
874 	int num_out_wins = 0;
875 	u16 nlw, cls, lnksta;
876 	int i, ret, memc;
877 	u32 tmp, burst, aspm_support;
878 
879 	/* Reset the bridge */
880 	pcie->bridge_sw_init_set(pcie, 1);
881 	usleep_range(100, 200);
882 
883 	/* Take the bridge out of reset */
884 	pcie->bridge_sw_init_set(pcie, 0);
885 
886 	tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
887 	tmp &= ~PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK;
888 	writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
889 	/* Wait for SerDes to be stable */
890 	usleep_range(100, 200);
891 
892 	/*
893 	 * SCB_MAX_BURST_SIZE is a two bit field.  For GENERIC chips it
894 	 * is encoded as 0=128, 1=256, 2=512, 3=Rsvd, for BCM7278 it
895 	 * is encoded as 0=Rsvd, 1=128, 2=256, 3=512.
896 	 */
897 	if (pcie->type == BCM2711)
898 		burst = 0x0; /* 128B */
899 	else if (pcie->type == BCM7278)
900 		burst = 0x3; /* 512 bytes */
901 	else
902 		burst = 0x2; /* 512 bytes */
903 
904 	/* Set SCB_MAX_BURST_SIZE, CFG_READ_UR_MODE, SCB_ACCESS_EN */
905 	tmp = readl(base + PCIE_MISC_MISC_CTRL);
906 	u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK);
907 	u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK);
908 	u32p_replace_bits(&tmp, burst, PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK);
909 	writel(tmp, base + PCIE_MISC_MISC_CTRL);
910 
911 	ret = brcm_pcie_get_rc_bar2_size_and_offset(pcie, &rc_bar2_size,
912 						    &rc_bar2_offset);
913 	if (ret)
914 		return ret;
915 
916 	tmp = lower_32_bits(rc_bar2_offset);
917 	u32p_replace_bits(&tmp, brcm_pcie_encode_ibar_size(rc_bar2_size),
918 			  PCIE_MISC_RC_BAR2_CONFIG_LO_SIZE_MASK);
919 	writel(tmp, base + PCIE_MISC_RC_BAR2_CONFIG_LO);
920 	writel(upper_32_bits(rc_bar2_offset),
921 	       base + PCIE_MISC_RC_BAR2_CONFIG_HI);
922 
923 	tmp = readl(base + PCIE_MISC_MISC_CTRL);
924 	for (memc = 0; memc < pcie->num_memc; memc++) {
925 		u32 scb_size_val = ilog2(pcie->memc_size[memc]) - 15;
926 
927 		if (memc == 0)
928 			u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(0));
929 		else if (memc == 1)
930 			u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(1));
931 		else if (memc == 2)
932 			u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(2));
933 	}
934 	writel(tmp, base + PCIE_MISC_MISC_CTRL);
935 
936 	/*
937 	 * We ideally want the MSI target address to be located in the 32bit
938 	 * addressable memory area. Some devices might depend on it. This is
939 	 * possible either when the inbound window is located above the lower
940 	 * 4GB or when the inbound area is smaller than 4GB (taking into
941 	 * account the rounding-up we're forced to perform).
942 	 */
943 	if (rc_bar2_offset >= SZ_4G || (rc_bar2_size + rc_bar2_offset) < SZ_4G)
944 		pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_LT_4GB;
945 	else
946 		pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_GT_4GB;
947 
948 	/* disable the PCIe->GISB memory window (RC_BAR1) */
949 	tmp = readl(base + PCIE_MISC_RC_BAR1_CONFIG_LO);
950 	tmp &= ~PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK;
951 	writel(tmp, base + PCIE_MISC_RC_BAR1_CONFIG_LO);
952 
953 	/* disable the PCIe->SCB memory window (RC_BAR3) */
954 	tmp = readl(base + PCIE_MISC_RC_BAR3_CONFIG_LO);
955 	tmp &= ~PCIE_MISC_RC_BAR3_CONFIG_LO_SIZE_MASK;
956 	writel(tmp, base + PCIE_MISC_RC_BAR3_CONFIG_LO);
957 
958 	if (pcie->gen)
959 		brcm_pcie_set_gen(pcie, pcie->gen);
960 
961 	/* Unassert the fundamental reset */
962 	pcie->perst_set(pcie, 0);
963 
964 	/*
965 	 * Give the RC/EP time to wake up, before trying to configure RC.
966 	 * Intermittently check status for link-up, up to a total of 100ms.
967 	 */
968 	for (i = 0; i < 100 && !brcm_pcie_link_up(pcie); i += 5)
969 		msleep(5);
970 
971 	if (!brcm_pcie_link_up(pcie)) {
972 		dev_err(dev, "link down\n");
973 		return -ENODEV;
974 	}
975 
976 	if (!brcm_pcie_rc_mode(pcie)) {
977 		dev_err(dev, "PCIe misconfigured; is in EP mode\n");
978 		return -EINVAL;
979 	}
980 
981 	resource_list_for_each_entry(entry, &bridge->windows) {
982 		res = entry->res;
983 
984 		if (resource_type(res) != IORESOURCE_MEM)
985 			continue;
986 
987 		if (num_out_wins >= BRCM_NUM_PCIE_OUT_WINS) {
988 			dev_err(pcie->dev, "too many outbound wins\n");
989 			return -EINVAL;
990 		}
991 
992 		brcm_pcie_set_outbound_win(pcie, num_out_wins, res->start,
993 					   res->start - entry->offset,
994 					   resource_size(res));
995 		num_out_wins++;
996 	}
997 
998 	/* Don't advertise L0s capability if 'aspm-no-l0s' */
999 	aspm_support = PCIE_LINK_STATE_L1;
1000 	if (!of_property_read_bool(pcie->np, "aspm-no-l0s"))
1001 		aspm_support |= PCIE_LINK_STATE_L0S;
1002 	tmp = readl(base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY);
1003 	u32p_replace_bits(&tmp, aspm_support,
1004 		PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_ASPM_SUPPORT_MASK);
1005 	writel(tmp, base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY);
1006 
1007 	/*
1008 	 * For config space accesses on the RC, show the right class for
1009 	 * a PCIe-PCIe bridge (the default setting is to be EP mode).
1010 	 */
1011 	tmp = readl(base + PCIE_RC_CFG_PRIV1_ID_VAL3);
1012 	u32p_replace_bits(&tmp, 0x060400,
1013 			  PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK);
1014 	writel(tmp, base + PCIE_RC_CFG_PRIV1_ID_VAL3);
1015 
1016 	if (pcie->ssc) {
1017 		ret = brcm_pcie_set_ssc(pcie);
1018 		if (ret == 0)
1019 			ssc_good = true;
1020 		else
1021 			dev_err(dev, "failed attempt to enter ssc mode\n");
1022 	}
1023 
1024 	lnksta = readw(base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKSTA);
1025 	cls = FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta);
1026 	nlw = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta);
1027 	dev_info(dev, "link up, %s x%u %s\n",
1028 		 pci_speed_string(pcie_link_speed[cls]), nlw,
1029 		 ssc_good ? "(SSC)" : "(!SSC)");
1030 
1031 	/* PCIe->SCB endian mode for BAR */
1032 	tmp = readl(base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1);
1033 	u32p_replace_bits(&tmp, PCIE_RC_CFG_VENDOR_SPCIFIC_REG1_LITTLE_ENDIAN,
1034 		PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK);
1035 	writel(tmp, base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1);
1036 
1037 	/*
1038 	 * Refclk from RC should be gated with CLKREQ# input when ASPM L0s,L1
1039 	 * is enabled => setting the CLKREQ_DEBUG_ENABLE field to 1.
1040 	 */
1041 	tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1042 	tmp |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK;
1043 	writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1044 
1045 	return 0;
1046 }
1047 
1048 /* L23 is a low-power PCIe link state */
1049 static void brcm_pcie_enter_l23(struct brcm_pcie *pcie)
1050 {
1051 	void __iomem *base = pcie->base;
1052 	int l23, i;
1053 	u32 tmp;
1054 
1055 	/* Assert request for L23 */
1056 	tmp = readl(base + PCIE_MISC_PCIE_CTRL);
1057 	u32p_replace_bits(&tmp, 1, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK);
1058 	writel(tmp, base + PCIE_MISC_PCIE_CTRL);
1059 
1060 	/* Wait up to 36 msec for L23 */
1061 	tmp = readl(base + PCIE_MISC_PCIE_STATUS);
1062 	l23 = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK, tmp);
1063 	for (i = 0; i < 15 && !l23; i++) {
1064 		usleep_range(2000, 2400);
1065 		tmp = readl(base + PCIE_MISC_PCIE_STATUS);
1066 		l23 = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK,
1067 				tmp);
1068 	}
1069 
1070 	if (!l23)
1071 		dev_err(pcie->dev, "failed to enter low-power link state\n");
1072 }
1073 
1074 static int brcm_phy_cntl(struct brcm_pcie *pcie, const int start)
1075 {
1076 	static const u32 shifts[PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS] = {
1077 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_SHIFT,
1078 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_SHIFT,
1079 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_SHIFT,};
1080 	static const u32 masks[PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS] = {
1081 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_MASK,
1082 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_MASK,
1083 		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_MASK,};
1084 	const int beg = start ? 0 : PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS - 1;
1085 	const int end = start ? PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS : -1;
1086 	u32 tmp, combined_mask = 0;
1087 	u32 val;
1088 	void __iomem *base = pcie->base;
1089 	int i, ret;
1090 
1091 	for (i = beg; i != end; start ? i++ : i--) {
1092 		val = start ? BIT_MASK(shifts[i]) : 0;
1093 		tmp = readl(base + PCIE_DVT_PMU_PCIE_PHY_CTRL);
1094 		tmp = (tmp & ~masks[i]) | (val & masks[i]);
1095 		writel(tmp, base + PCIE_DVT_PMU_PCIE_PHY_CTRL);
1096 		usleep_range(50, 200);
1097 		combined_mask |= masks[i];
1098 	}
1099 
1100 	tmp = readl(base + PCIE_DVT_PMU_PCIE_PHY_CTRL);
1101 	val = start ? combined_mask : 0;
1102 
1103 	ret = (tmp & combined_mask) == val ? 0 : -EIO;
1104 	if (ret)
1105 		dev_err(pcie->dev, "failed to %s phy\n", (start ? "start" : "stop"));
1106 
1107 	return ret;
1108 }
1109 
1110 static inline int brcm_phy_start(struct brcm_pcie *pcie)
1111 {
1112 	return pcie->rescal ? brcm_phy_cntl(pcie, 1) : 0;
1113 }
1114 
1115 static inline int brcm_phy_stop(struct brcm_pcie *pcie)
1116 {
1117 	return pcie->rescal ? brcm_phy_cntl(pcie, 0) : 0;
1118 }
1119 
1120 static void brcm_pcie_turn_off(struct brcm_pcie *pcie)
1121 {
1122 	void __iomem *base = pcie->base;
1123 	int tmp;
1124 
1125 	if (brcm_pcie_link_up(pcie))
1126 		brcm_pcie_enter_l23(pcie);
1127 	/* Assert fundamental reset */
1128 	pcie->perst_set(pcie, 1);
1129 
1130 	/* Deassert request for L23 in case it was asserted */
1131 	tmp = readl(base + PCIE_MISC_PCIE_CTRL);
1132 	u32p_replace_bits(&tmp, 0, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK);
1133 	writel(tmp, base + PCIE_MISC_PCIE_CTRL);
1134 
1135 	/* Turn off SerDes */
1136 	tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1137 	u32p_replace_bits(&tmp, 1, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK);
1138 	writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1139 
1140 	/* Shutdown PCIe bridge */
1141 	pcie->bridge_sw_init_set(pcie, 1);
1142 }
1143 
1144 static int brcm_pcie_suspend(struct device *dev)
1145 {
1146 	struct brcm_pcie *pcie = dev_get_drvdata(dev);
1147 	int ret;
1148 
1149 	brcm_pcie_turn_off(pcie);
1150 	ret = brcm_phy_stop(pcie);
1151 	clk_disable_unprepare(pcie->clk);
1152 
1153 	return ret;
1154 }
1155 
1156 static int brcm_pcie_resume(struct device *dev)
1157 {
1158 	struct brcm_pcie *pcie = dev_get_drvdata(dev);
1159 	void __iomem *base;
1160 	u32 tmp;
1161 	int ret;
1162 
1163 	base = pcie->base;
1164 	clk_prepare_enable(pcie->clk);
1165 
1166 	ret = brcm_phy_start(pcie);
1167 	if (ret)
1168 		goto err;
1169 
1170 	/* Take bridge out of reset so we can access the SERDES reg */
1171 	pcie->bridge_sw_init_set(pcie, 0);
1172 
1173 	/* SERDES_IDDQ = 0 */
1174 	tmp = readl(base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1175 	u32p_replace_bits(&tmp, 0, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK);
1176 	writel(tmp, base + PCIE_MISC_HARD_PCIE_HARD_DEBUG);
1177 
1178 	/* wait for serdes to be stable */
1179 	udelay(100);
1180 
1181 	ret = brcm_pcie_setup(pcie);
1182 	if (ret)
1183 		goto err;
1184 
1185 	if (pcie->msi)
1186 		brcm_msi_set_regs(pcie->msi);
1187 
1188 	return 0;
1189 
1190 err:
1191 	clk_disable_unprepare(pcie->clk);
1192 	return ret;
1193 }
1194 
1195 static void __brcm_pcie_remove(struct brcm_pcie *pcie)
1196 {
1197 	brcm_msi_remove(pcie);
1198 	brcm_pcie_turn_off(pcie);
1199 	brcm_phy_stop(pcie);
1200 	reset_control_assert(pcie->rescal);
1201 	clk_disable_unprepare(pcie->clk);
1202 }
1203 
1204 static int brcm_pcie_remove(struct platform_device *pdev)
1205 {
1206 	struct brcm_pcie *pcie = platform_get_drvdata(pdev);
1207 	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
1208 
1209 	pci_stop_root_bus(bridge->bus);
1210 	pci_remove_root_bus(bridge->bus);
1211 	__brcm_pcie_remove(pcie);
1212 
1213 	return 0;
1214 }
1215 
1216 static const struct of_device_id brcm_pcie_match[] = {
1217 	{ .compatible = "brcm,bcm2711-pcie", .data = &bcm2711_cfg },
1218 	{ .compatible = "brcm,bcm4908-pcie", .data = &bcm4908_cfg },
1219 	{ .compatible = "brcm,bcm7211-pcie", .data = &generic_cfg },
1220 	{ .compatible = "brcm,bcm7278-pcie", .data = &bcm7278_cfg },
1221 	{ .compatible = "brcm,bcm7216-pcie", .data = &bcm7278_cfg },
1222 	{ .compatible = "brcm,bcm7445-pcie", .data = &generic_cfg },
1223 	{},
1224 };
1225 
1226 static int brcm_pcie_probe(struct platform_device *pdev)
1227 {
1228 	struct device_node *np = pdev->dev.of_node, *msi_np;
1229 	struct pci_host_bridge *bridge;
1230 	const struct pcie_cfg_data *data;
1231 	struct brcm_pcie *pcie;
1232 	int ret;
1233 
1234 	bridge = devm_pci_alloc_host_bridge(&pdev->dev, sizeof(*pcie));
1235 	if (!bridge)
1236 		return -ENOMEM;
1237 
1238 	data = of_device_get_match_data(&pdev->dev);
1239 	if (!data) {
1240 		pr_err("failed to look up compatible string\n");
1241 		return -EINVAL;
1242 	}
1243 
1244 	pcie = pci_host_bridge_priv(bridge);
1245 	pcie->dev = &pdev->dev;
1246 	pcie->np = np;
1247 	pcie->reg_offsets = data->offsets;
1248 	pcie->type = data->type;
1249 	pcie->perst_set = data->perst_set;
1250 	pcie->bridge_sw_init_set = data->bridge_sw_init_set;
1251 
1252 	pcie->base = devm_platform_ioremap_resource(pdev, 0);
1253 	if (IS_ERR(pcie->base))
1254 		return PTR_ERR(pcie->base);
1255 
1256 	pcie->clk = devm_clk_get_optional(&pdev->dev, "sw_pcie");
1257 	if (IS_ERR(pcie->clk))
1258 		return PTR_ERR(pcie->clk);
1259 
1260 	ret = of_pci_get_max_link_speed(np);
1261 	pcie->gen = (ret < 0) ? 0 : ret;
1262 
1263 	pcie->ssc = of_property_read_bool(np, "brcm,enable-ssc");
1264 
1265 	ret = clk_prepare_enable(pcie->clk);
1266 	if (ret) {
1267 		dev_err(&pdev->dev, "could not enable clock\n");
1268 		return ret;
1269 	}
1270 	pcie->rescal = devm_reset_control_get_optional_shared(&pdev->dev, "rescal");
1271 	if (IS_ERR(pcie->rescal)) {
1272 		clk_disable_unprepare(pcie->clk);
1273 		return PTR_ERR(pcie->rescal);
1274 	}
1275 	pcie->perst_reset = devm_reset_control_get_optional_exclusive(&pdev->dev, "perst");
1276 	if (IS_ERR(pcie->perst_reset)) {
1277 		clk_disable_unprepare(pcie->clk);
1278 		return PTR_ERR(pcie->perst_reset);
1279 	}
1280 
1281 	ret = reset_control_deassert(pcie->rescal);
1282 	if (ret)
1283 		dev_err(&pdev->dev, "failed to deassert 'rescal'\n");
1284 
1285 	ret = brcm_phy_start(pcie);
1286 	if (ret) {
1287 		reset_control_assert(pcie->rescal);
1288 		clk_disable_unprepare(pcie->clk);
1289 		return ret;
1290 	}
1291 
1292 	ret = brcm_pcie_setup(pcie);
1293 	if (ret)
1294 		goto fail;
1295 
1296 	pcie->hw_rev = readl(pcie->base + PCIE_MISC_REVISION);
1297 	if (pcie->type == BCM4908 && pcie->hw_rev >= BRCM_PCIE_HW_REV_3_20) {
1298 		dev_err(pcie->dev, "hardware revision with unsupported PERST# setup\n");
1299 		goto fail;
1300 	}
1301 
1302 	msi_np = of_parse_phandle(pcie->np, "msi-parent", 0);
1303 	if (pci_msi_enabled() && msi_np == pcie->np) {
1304 		ret = brcm_pcie_enable_msi(pcie);
1305 		if (ret) {
1306 			dev_err(pcie->dev, "probe of internal MSI failed");
1307 			goto fail;
1308 		}
1309 	}
1310 
1311 	bridge->ops = &brcm_pcie_ops;
1312 	bridge->sysdata = pcie;
1313 
1314 	platform_set_drvdata(pdev, pcie);
1315 
1316 	return pci_host_probe(bridge);
1317 fail:
1318 	__brcm_pcie_remove(pcie);
1319 	return ret;
1320 }
1321 
1322 MODULE_DEVICE_TABLE(of, brcm_pcie_match);
1323 
1324 static const struct dev_pm_ops brcm_pcie_pm_ops = {
1325 	.suspend = brcm_pcie_suspend,
1326 	.resume = brcm_pcie_resume,
1327 };
1328 
1329 static struct platform_driver brcm_pcie_driver = {
1330 	.probe = brcm_pcie_probe,
1331 	.remove = brcm_pcie_remove,
1332 	.driver = {
1333 		.name = "brcm-pcie",
1334 		.of_match_table = brcm_pcie_match,
1335 		.pm = &brcm_pcie_pm_ops,
1336 	},
1337 };
1338 module_platform_driver(brcm_pcie_driver);
1339 
1340 MODULE_LICENSE("GPL");
1341 MODULE_DESCRIPTION("Broadcom STB PCIe RC driver");
1342 MODULE_AUTHOR("Broadcom");
1343