1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later
2 /*
3  * Copyright 2008 - 2015 Freescale Semiconductor Inc.
4  * Copyright 2020 NXP
5  */
6 
7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8 
9 #include <linux/fsl/guts.h>
10 #include <linux/slab.h>
11 #include <linux/delay.h>
12 #include <linux/module.h>
13 #include <linux/of_platform.h>
14 #include <linux/clk.h>
15 #include <linux/of_address.h>
16 #include <linux/of_irq.h>
17 #include <linux/interrupt.h>
18 #include <linux/libfdt_env.h>
19 
20 #include "fman.h"
21 #include "fman_muram.h"
22 #include "fman_keygen.h"
23 
24 /* General defines */
25 #define FMAN_LIODN_TBL			64	/* size of LIODN table */
26 #define MAX_NUM_OF_MACS			10
27 #define FM_NUM_OF_FMAN_CTRL_EVENT_REGS	4
28 #define BASE_RX_PORTID			0x08
29 #define BASE_TX_PORTID			0x28
30 
31 /* Modules registers offsets */
32 #define BMI_OFFSET		0x00080000
33 #define QMI_OFFSET		0x00080400
34 #define KG_OFFSET		0x000C1000
35 #define DMA_OFFSET		0x000C2000
36 #define FPM_OFFSET		0x000C3000
37 #define IMEM_OFFSET		0x000C4000
38 #define HWP_OFFSET		0x000C7000
39 #define CGP_OFFSET		0x000DB000
40 
41 /* Exceptions bit map */
42 #define EX_DMA_BUS_ERROR		0x80000000
43 #define EX_DMA_READ_ECC			0x40000000
44 #define EX_DMA_SYSTEM_WRITE_ECC	0x20000000
45 #define EX_DMA_FM_WRITE_ECC		0x10000000
46 #define EX_FPM_STALL_ON_TASKS		0x08000000
47 #define EX_FPM_SINGLE_ECC		0x04000000
48 #define EX_FPM_DOUBLE_ECC		0x02000000
49 #define EX_QMI_SINGLE_ECC		0x01000000
50 #define EX_QMI_DEQ_FROM_UNKNOWN_PORTID	0x00800000
51 #define EX_QMI_DOUBLE_ECC		0x00400000
52 #define EX_BMI_LIST_RAM_ECC		0x00200000
53 #define EX_BMI_STORAGE_PROFILE_ECC	0x00100000
54 #define EX_BMI_STATISTICS_RAM_ECC	0x00080000
55 #define EX_IRAM_ECC			0x00040000
56 #define EX_MURAM_ECC			0x00020000
57 #define EX_BMI_DISPATCH_RAM_ECC	0x00010000
58 #define EX_DMA_SINGLE_PORT_ECC		0x00008000
59 
60 /* DMA defines */
61 /* masks */
62 #define DMA_MODE_BER			0x00200000
63 #define DMA_MODE_ECC			0x00000020
64 #define DMA_MODE_SECURE_PROT		0x00000800
65 #define DMA_MODE_AXI_DBG_MASK		0x0F000000
66 
67 #define DMA_TRANSFER_PORTID_MASK	0xFF000000
68 #define DMA_TRANSFER_TNUM_MASK		0x00FF0000
69 #define DMA_TRANSFER_LIODN_MASK	0x00000FFF
70 
71 #define DMA_STATUS_BUS_ERR		0x08000000
72 #define DMA_STATUS_READ_ECC		0x04000000
73 #define DMA_STATUS_SYSTEM_WRITE_ECC	0x02000000
74 #define DMA_STATUS_FM_WRITE_ECC	0x01000000
75 #define DMA_STATUS_FM_SPDAT_ECC	0x00080000
76 
77 #define DMA_MODE_CACHE_OR_SHIFT		30
78 #define DMA_MODE_AXI_DBG_SHIFT			24
79 #define DMA_MODE_CEN_SHIFT			13
80 #define DMA_MODE_CEN_MASK			0x00000007
81 #define DMA_MODE_DBG_SHIFT			7
82 #define DMA_MODE_AID_MODE_SHIFT		4
83 
84 #define DMA_THRESH_COMMQ_SHIFT			24
85 #define DMA_THRESH_READ_INT_BUF_SHIFT		16
86 #define DMA_THRESH_READ_INT_BUF_MASK		0x0000003f
87 #define DMA_THRESH_WRITE_INT_BUF_MASK		0x0000003f
88 
89 #define DMA_TRANSFER_PORTID_SHIFT		24
90 #define DMA_TRANSFER_TNUM_SHIFT		16
91 
92 #define DMA_CAM_SIZEOF_ENTRY			0x40
93 #define DMA_CAM_UNITS				8
94 
95 #define DMA_LIODN_SHIFT		16
96 #define DMA_LIODN_BASE_MASK	0x00000FFF
97 
98 /* FPM defines */
99 #define FPM_EV_MASK_DOUBLE_ECC		0x80000000
100 #define FPM_EV_MASK_STALL		0x40000000
101 #define FPM_EV_MASK_SINGLE_ECC		0x20000000
102 #define FPM_EV_MASK_RELEASE_FM		0x00010000
103 #define FPM_EV_MASK_DOUBLE_ECC_EN	0x00008000
104 #define FPM_EV_MASK_STALL_EN		0x00004000
105 #define FPM_EV_MASK_SINGLE_ECC_EN	0x00002000
106 #define FPM_EV_MASK_EXTERNAL_HALT	0x00000008
107 #define FPM_EV_MASK_ECC_ERR_HALT	0x00000004
108 
109 #define FPM_RAM_MURAM_ECC		0x00008000
110 #define FPM_RAM_IRAM_ECC		0x00004000
111 #define FPM_IRAM_ECC_ERR_EX_EN		0x00020000
112 #define FPM_MURAM_ECC_ERR_EX_EN	0x00040000
113 #define FPM_RAM_IRAM_ECC_EN		0x40000000
114 #define FPM_RAM_RAMS_ECC_EN		0x80000000
115 #define FPM_RAM_RAMS_ECC_EN_SRC_SEL	0x08000000
116 
117 #define FPM_REV1_MAJOR_MASK		0x0000FF00
118 #define FPM_REV1_MINOR_MASK		0x000000FF
119 
120 #define FPM_DISP_LIMIT_SHIFT		24
121 
122 #define FPM_PRT_FM_CTL1			0x00000001
123 #define FPM_PRT_FM_CTL2			0x00000002
124 #define FPM_PORT_FM_CTL_PORTID_SHIFT	24
125 #define FPM_PRC_ORA_FM_CTL_SEL_SHIFT	16
126 
127 #define FPM_THR1_PRS_SHIFT		24
128 #define FPM_THR1_KG_SHIFT		16
129 #define FPM_THR1_PLCR_SHIFT		8
130 #define FPM_THR1_BMI_SHIFT		0
131 
132 #define FPM_THR2_QMI_ENQ_SHIFT		24
133 #define FPM_THR2_QMI_DEQ_SHIFT		0
134 #define FPM_THR2_FM_CTL1_SHIFT		16
135 #define FPM_THR2_FM_CTL2_SHIFT		8
136 
137 #define FPM_EV_MASK_CAT_ERR_SHIFT	1
138 #define FPM_EV_MASK_DMA_ERR_SHIFT	0
139 
140 #define FPM_REV1_MAJOR_SHIFT		8
141 
142 #define FPM_RSTC_FM_RESET		0x80000000
143 #define FPM_RSTC_MAC0_RESET		0x40000000
144 #define FPM_RSTC_MAC1_RESET		0x20000000
145 #define FPM_RSTC_MAC2_RESET		0x10000000
146 #define FPM_RSTC_MAC3_RESET		0x08000000
147 #define FPM_RSTC_MAC8_RESET		0x04000000
148 #define FPM_RSTC_MAC4_RESET		0x02000000
149 #define FPM_RSTC_MAC5_RESET		0x01000000
150 #define FPM_RSTC_MAC6_RESET		0x00800000
151 #define FPM_RSTC_MAC7_RESET		0x00400000
152 #define FPM_RSTC_MAC9_RESET		0x00200000
153 
154 #define FPM_TS_INT_SHIFT		16
155 #define FPM_TS_CTL_EN			0x80000000
156 
157 /* BMI defines */
158 #define BMI_INIT_START				0x80000000
159 #define BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC	0x80000000
160 #define BMI_ERR_INTR_EN_LIST_RAM_ECC		0x40000000
161 #define BMI_ERR_INTR_EN_STATISTICS_RAM_ECC	0x20000000
162 #define BMI_ERR_INTR_EN_DISPATCH_RAM_ECC	0x10000000
163 #define BMI_NUM_OF_TASKS_MASK			0x3F000000
164 #define BMI_NUM_OF_EXTRA_TASKS_MASK		0x000F0000
165 #define BMI_NUM_OF_DMAS_MASK			0x00000F00
166 #define BMI_NUM_OF_EXTRA_DMAS_MASK		0x0000000F
167 #define BMI_FIFO_SIZE_MASK			0x000003FF
168 #define BMI_EXTRA_FIFO_SIZE_MASK		0x03FF0000
169 #define BMI_CFG2_DMAS_MASK			0x0000003F
170 #define BMI_CFG2_TASKS_MASK			0x0000003F
171 
172 #define BMI_CFG2_TASKS_SHIFT		16
173 #define BMI_CFG2_DMAS_SHIFT		0
174 #define BMI_CFG1_FIFO_SIZE_SHIFT	16
175 #define BMI_NUM_OF_TASKS_SHIFT		24
176 #define BMI_EXTRA_NUM_OF_TASKS_SHIFT	16
177 #define BMI_NUM_OF_DMAS_SHIFT		8
178 #define BMI_EXTRA_NUM_OF_DMAS_SHIFT	0
179 
180 #define BMI_FIFO_ALIGN			0x100
181 
182 #define BMI_EXTRA_FIFO_SIZE_SHIFT	16
183 
184 /* QMI defines */
185 #define QMI_CFG_ENQ_EN			0x80000000
186 #define QMI_CFG_DEQ_EN			0x40000000
187 #define QMI_CFG_EN_COUNTERS		0x10000000
188 #define QMI_CFG_DEQ_MASK		0x0000003F
189 #define QMI_CFG_ENQ_MASK		0x00003F00
190 #define QMI_CFG_ENQ_SHIFT		8
191 
192 #define QMI_ERR_INTR_EN_DOUBLE_ECC	0x80000000
193 #define QMI_ERR_INTR_EN_DEQ_FROM_DEF	0x40000000
194 #define QMI_INTR_EN_SINGLE_ECC		0x80000000
195 
196 #define QMI_GS_HALT_NOT_BUSY		0x00000002
197 
198 /* HWP defines */
199 #define HWP_RPIMAC_PEN			0x00000001
200 
201 /* IRAM defines */
202 #define IRAM_IADD_AIE			0x80000000
203 #define IRAM_READY			0x80000000
204 
205 /* Default values */
206 #define DEFAULT_CATASTROPHIC_ERR		0
207 #define DEFAULT_DMA_ERR				0
208 #define DEFAULT_AID_MODE			FMAN_DMA_AID_OUT_TNUM
209 #define DEFAULT_DMA_COMM_Q_LOW			0x2A
210 #define DEFAULT_DMA_COMM_Q_HIGH		0x3F
211 #define DEFAULT_CACHE_OVERRIDE			0
212 #define DEFAULT_DMA_CAM_NUM_OF_ENTRIES		64
213 #define DEFAULT_DMA_DBG_CNT_MODE		0
214 #define DEFAULT_DMA_SOS_EMERGENCY		0
215 #define DEFAULT_DMA_WATCHDOG			0
216 #define DEFAULT_DISP_LIMIT			0
217 #define DEFAULT_PRS_DISP_TH			16
218 #define DEFAULT_PLCR_DISP_TH			16
219 #define DEFAULT_KG_DISP_TH			16
220 #define DEFAULT_BMI_DISP_TH			16
221 #define DEFAULT_QMI_ENQ_DISP_TH		16
222 #define DEFAULT_QMI_DEQ_DISP_TH		16
223 #define DEFAULT_FM_CTL1_DISP_TH		16
224 #define DEFAULT_FM_CTL2_DISP_TH		16
225 
226 #define DFLT_AXI_DBG_NUM_OF_BEATS		1
227 
228 #define DFLT_DMA_READ_INT_BUF_LOW(dma_thresh_max_buf)	\
229 	((dma_thresh_max_buf + 1) / 2)
230 #define DFLT_DMA_READ_INT_BUF_HIGH(dma_thresh_max_buf)	\
231 	((dma_thresh_max_buf + 1) * 3 / 4)
232 #define DFLT_DMA_WRITE_INT_BUF_LOW(dma_thresh_max_buf)	\
233 	((dma_thresh_max_buf + 1) / 2)
234 #define DFLT_DMA_WRITE_INT_BUF_HIGH(dma_thresh_max_buf)\
235 	((dma_thresh_max_buf + 1) * 3 / 4)
236 
237 #define DMA_COMM_Q_LOW_FMAN_V3		0x2A
238 #define DMA_COMM_Q_LOW_FMAN_V2(dma_thresh_max_commq)		\
239 	((dma_thresh_max_commq + 1) / 2)
240 #define DFLT_DMA_COMM_Q_LOW(major, dma_thresh_max_commq)	\
241 	((major == 6) ? DMA_COMM_Q_LOW_FMAN_V3 :		\
242 	DMA_COMM_Q_LOW_FMAN_V2(dma_thresh_max_commq))
243 
244 #define DMA_COMM_Q_HIGH_FMAN_V3	0x3f
245 #define DMA_COMM_Q_HIGH_FMAN_V2(dma_thresh_max_commq)		\
246 	((dma_thresh_max_commq + 1) * 3 / 4)
247 #define DFLT_DMA_COMM_Q_HIGH(major, dma_thresh_max_commq)	\
248 	((major == 6) ? DMA_COMM_Q_HIGH_FMAN_V3 :		\
249 	DMA_COMM_Q_HIGH_FMAN_V2(dma_thresh_max_commq))
250 
251 #define TOTAL_NUM_OF_TASKS_FMAN_V3L	59
252 #define TOTAL_NUM_OF_TASKS_FMAN_V3H	124
253 #define DFLT_TOTAL_NUM_OF_TASKS(major, minor, bmi_max_num_of_tasks)	\
254 	((major == 6) ? ((minor == 1 || minor == 4) ?			\
255 	TOTAL_NUM_OF_TASKS_FMAN_V3L : TOTAL_NUM_OF_TASKS_FMAN_V3H) :	\
256 	bmi_max_num_of_tasks)
257 
258 #define DMA_CAM_NUM_OF_ENTRIES_FMAN_V3		64
259 #define DMA_CAM_NUM_OF_ENTRIES_FMAN_V2		32
260 #define DFLT_DMA_CAM_NUM_OF_ENTRIES(major)			\
261 	(major == 6 ? DMA_CAM_NUM_OF_ENTRIES_FMAN_V3 :		\
262 	DMA_CAM_NUM_OF_ENTRIES_FMAN_V2)
263 
264 #define FM_TIMESTAMP_1_USEC_BIT             8
265 
266 /* Defines used for enabling/disabling FMan interrupts */
267 #define ERR_INTR_EN_DMA         0x00010000
268 #define ERR_INTR_EN_FPM         0x80000000
269 #define ERR_INTR_EN_BMI         0x00800000
270 #define ERR_INTR_EN_QMI         0x00400000
271 #define ERR_INTR_EN_MURAM       0x00040000
272 #define ERR_INTR_EN_MAC0        0x00004000
273 #define ERR_INTR_EN_MAC1        0x00002000
274 #define ERR_INTR_EN_MAC2        0x00001000
275 #define ERR_INTR_EN_MAC3        0x00000800
276 #define ERR_INTR_EN_MAC4        0x00000400
277 #define ERR_INTR_EN_MAC5        0x00000200
278 #define ERR_INTR_EN_MAC6        0x00000100
279 #define ERR_INTR_EN_MAC7        0x00000080
280 #define ERR_INTR_EN_MAC8        0x00008000
281 #define ERR_INTR_EN_MAC9        0x00000040
282 
283 #define INTR_EN_QMI             0x40000000
284 #define INTR_EN_MAC0            0x00080000
285 #define INTR_EN_MAC1            0x00040000
286 #define INTR_EN_MAC2            0x00020000
287 #define INTR_EN_MAC3            0x00010000
288 #define INTR_EN_MAC4            0x00000040
289 #define INTR_EN_MAC5            0x00000020
290 #define INTR_EN_MAC6            0x00000008
291 #define INTR_EN_MAC7            0x00000002
292 #define INTR_EN_MAC8            0x00200000
293 #define INTR_EN_MAC9            0x00100000
294 #define INTR_EN_REV0            0x00008000
295 #define INTR_EN_REV1            0x00004000
296 #define INTR_EN_REV2            0x00002000
297 #define INTR_EN_REV3            0x00001000
298 #define INTR_EN_TMR             0x01000000
299 
300 enum fman_dma_aid_mode {
301 	FMAN_DMA_AID_OUT_PORT_ID = 0,		  /* 4 LSB of PORT_ID */
302 	FMAN_DMA_AID_OUT_TNUM			  /* 4 LSB of TNUM */
303 };
304 
305 struct fman_iram_regs {
306 	u32 iadd;	/* FM IRAM instruction address register */
307 	u32 idata;	/* FM IRAM instruction data register */
308 	u32 itcfg;	/* FM IRAM timing config register */
309 	u32 iready;	/* FM IRAM ready register */
310 };
311 
312 struct fman_fpm_regs {
313 	u32 fmfp_tnc;		/* FPM TNUM Control 0x00 */
314 	u32 fmfp_prc;		/* FPM Port_ID FmCtl Association 0x04 */
315 	u32 fmfp_brkc;		/* FPM Breakpoint Control 0x08 */
316 	u32 fmfp_mxd;		/* FPM Flush Control 0x0c */
317 	u32 fmfp_dist1;		/* FPM Dispatch Thresholds1 0x10 */
318 	u32 fmfp_dist2;		/* FPM Dispatch Thresholds2 0x14 */
319 	u32 fm_epi;		/* FM Error Pending Interrupts 0x18 */
320 	u32 fm_rie;		/* FM Error Interrupt Enable 0x1c */
321 	u32 fmfp_fcev[4];	/* FPM FMan-Controller Event 1-4 0x20-0x2f */
322 	u32 res0030[4];		/* res 0x30 - 0x3f */
323 	u32 fmfp_cee[4];	/* PM FMan-Controller Event 1-4 0x40-0x4f */
324 	u32 res0050[4];		/* res 0x50-0x5f */
325 	u32 fmfp_tsc1;		/* FPM TimeStamp Control1 0x60 */
326 	u32 fmfp_tsc2;		/* FPM TimeStamp Control2 0x64 */
327 	u32 fmfp_tsp;		/* FPM Time Stamp 0x68 */
328 	u32 fmfp_tsf;		/* FPM Time Stamp Fraction 0x6c */
329 	u32 fm_rcr;		/* FM Rams Control 0x70 */
330 	u32 fmfp_extc;		/* FPM External Requests Control 0x74 */
331 	u32 fmfp_ext1;		/* FPM External Requests Config1 0x78 */
332 	u32 fmfp_ext2;		/* FPM External Requests Config2 0x7c */
333 	u32 fmfp_drd[16];	/* FPM Data_Ram Data 0-15 0x80 - 0xbf */
334 	u32 fmfp_dra;		/* FPM Data Ram Access 0xc0 */
335 	u32 fm_ip_rev_1;	/* FM IP Block Revision 1 0xc4 */
336 	u32 fm_ip_rev_2;	/* FM IP Block Revision 2 0xc8 */
337 	u32 fm_rstc;		/* FM Reset Command 0xcc */
338 	u32 fm_cld;		/* FM Classifier Debug 0xd0 */
339 	u32 fm_npi;		/* FM Normal Pending Interrupts 0xd4 */
340 	u32 fmfp_exte;		/* FPM External Requests Enable 0xd8 */
341 	u32 fmfp_ee;		/* FPM Event&Mask 0xdc */
342 	u32 fmfp_cev[4];	/* FPM CPU Event 1-4 0xe0-0xef */
343 	u32 res00f0[4];		/* res 0xf0-0xff */
344 	u32 fmfp_ps[50];	/* FPM Port Status 0x100-0x1c7 */
345 	u32 res01c8[14];	/* res 0x1c8-0x1ff */
346 	u32 fmfp_clfabc;	/* FPM CLFABC 0x200 */
347 	u32 fmfp_clfcc;		/* FPM CLFCC 0x204 */
348 	u32 fmfp_clfaval;	/* FPM CLFAVAL 0x208 */
349 	u32 fmfp_clfbval;	/* FPM CLFBVAL 0x20c */
350 	u32 fmfp_clfcval;	/* FPM CLFCVAL 0x210 */
351 	u32 fmfp_clfamsk;	/* FPM CLFAMSK 0x214 */
352 	u32 fmfp_clfbmsk;	/* FPM CLFBMSK 0x218 */
353 	u32 fmfp_clfcmsk;	/* FPM CLFCMSK 0x21c */
354 	u32 fmfp_clfamc;	/* FPM CLFAMC 0x220 */
355 	u32 fmfp_clfbmc;	/* FPM CLFBMC 0x224 */
356 	u32 fmfp_clfcmc;	/* FPM CLFCMC 0x228 */
357 	u32 fmfp_decceh;	/* FPM DECCEH 0x22c */
358 	u32 res0230[116];	/* res 0x230 - 0x3ff */
359 	u32 fmfp_ts[128];	/* 0x400: FPM Task Status 0x400 - 0x5ff */
360 	u32 res0600[0x400 - 384];
361 };
362 
363 struct fman_bmi_regs {
364 	u32 fmbm_init;		/* BMI Initialization 0x00 */
365 	u32 fmbm_cfg1;		/* BMI Configuration 1 0x04 */
366 	u32 fmbm_cfg2;		/* BMI Configuration 2 0x08 */
367 	u32 res000c[5];		/* 0x0c - 0x1f */
368 	u32 fmbm_ievr;		/* Interrupt Event Register 0x20 */
369 	u32 fmbm_ier;		/* Interrupt Enable Register 0x24 */
370 	u32 fmbm_ifr;		/* Interrupt Force Register 0x28 */
371 	u32 res002c[5];		/* 0x2c - 0x3f */
372 	u32 fmbm_arb[8];	/* BMI Arbitration 0x40 - 0x5f */
373 	u32 res0060[12];	/* 0x60 - 0x8f */
374 	u32 fmbm_dtc[3];	/* Debug Trap Counter 0x90 - 0x9b */
375 	u32 res009c;		/* 0x9c */
376 	u32 fmbm_dcv[3][4];	/* Debug Compare val 0xa0-0xcf */
377 	u32 fmbm_dcm[3][4];	/* Debug Compare Mask 0xd0-0xff */
378 	u32 fmbm_gde;		/* BMI Global Debug Enable 0x100 */
379 	u32 fmbm_pp[63];	/* BMI Port Parameters 0x104 - 0x1ff */
380 	u32 res0200;		/* 0x200 */
381 	u32 fmbm_pfs[63];	/* BMI Port FIFO Size 0x204 - 0x2ff */
382 	u32 res0300;		/* 0x300 */
383 	u32 fmbm_spliodn[63];	/* Port Partition ID 0x304 - 0x3ff */
384 };
385 
386 struct fman_qmi_regs {
387 	u32 fmqm_gc;		/* General Configuration Register 0x00 */
388 	u32 res0004;		/* 0x04 */
389 	u32 fmqm_eie;		/* Error Interrupt Event Register 0x08 */
390 	u32 fmqm_eien;		/* Error Interrupt Enable Register 0x0c */
391 	u32 fmqm_eif;		/* Error Interrupt Force Register 0x10 */
392 	u32 fmqm_ie;		/* Interrupt Event Register 0x14 */
393 	u32 fmqm_ien;		/* Interrupt Enable Register 0x18 */
394 	u32 fmqm_if;		/* Interrupt Force Register 0x1c */
395 	u32 fmqm_gs;		/* Global Status Register 0x20 */
396 	u32 fmqm_ts;		/* Task Status Register 0x24 */
397 	u32 fmqm_etfc;		/* Enqueue Total Frame Counter 0x28 */
398 	u32 fmqm_dtfc;		/* Dequeue Total Frame Counter 0x2c */
399 	u32 fmqm_dc0;		/* Dequeue Counter 0 0x30 */
400 	u32 fmqm_dc1;		/* Dequeue Counter 1 0x34 */
401 	u32 fmqm_dc2;		/* Dequeue Counter 2 0x38 */
402 	u32 fmqm_dc3;		/* Dequeue Counter 3 0x3c */
403 	u32 fmqm_dfdc;		/* Dequeue FQID from Default Counter 0x40 */
404 	u32 fmqm_dfcc;		/* Dequeue FQID from Context Counter 0x44 */
405 	u32 fmqm_dffc;		/* Dequeue FQID from FD Counter 0x48 */
406 	u32 fmqm_dcc;		/* Dequeue Confirm Counter 0x4c */
407 	u32 res0050[7];		/* 0x50 - 0x6b */
408 	u32 fmqm_tapc;		/* Tnum Aging Period Control 0x6c */
409 	u32 fmqm_dmcvc;		/* Dequeue MAC Command Valid Counter 0x70 */
410 	u32 fmqm_difdcc;	/* Dequeue Invalid FD Command Counter 0x74 */
411 	u32 fmqm_da1v;		/* Dequeue A1 Valid Counter 0x78 */
412 	u32 res007c;		/* 0x7c */
413 	u32 fmqm_dtc;		/* 0x80 Debug Trap Counter 0x80 */
414 	u32 fmqm_efddd;		/* 0x84 Enqueue Frame desc Dynamic dbg 0x84 */
415 	u32 res0088[2];		/* 0x88 - 0x8f */
416 	struct {
417 		u32 fmqm_dtcfg1;	/* 0x90 dbg trap cfg 1 Register 0x00 */
418 		u32 fmqm_dtval1;	/* Debug Trap Value 1 Register 0x04 */
419 		u32 fmqm_dtm1;		/* Debug Trap Mask 1 Register 0x08 */
420 		u32 fmqm_dtc1;		/* Debug Trap Counter 1 Register 0x0c */
421 		u32 fmqm_dtcfg2;	/* dbg Trap cfg 2 Register 0x10 */
422 		u32 fmqm_dtval2;	/* Debug Trap Value 2 Register 0x14 */
423 		u32 fmqm_dtm2;		/* Debug Trap Mask 2 Register 0x18 */
424 		u32 res001c;		/* 0x1c */
425 	} dbg_traps[3];			/* 0x90 - 0xef */
426 	u8 res00f0[0x400 - 0xf0];	/* 0xf0 - 0x3ff */
427 };
428 
429 struct fman_dma_regs {
430 	u32 fmdmsr;	/* FM DMA status register 0x00 */
431 	u32 fmdmmr;	/* FM DMA mode register 0x04 */
432 	u32 fmdmtr;	/* FM DMA bus threshold register 0x08 */
433 	u32 fmdmhy;	/* FM DMA bus hysteresis register 0x0c */
434 	u32 fmdmsetr;	/* FM DMA SOS emergency Threshold Register 0x10 */
435 	u32 fmdmtah;	/* FM DMA transfer bus address high reg 0x14 */
436 	u32 fmdmtal;	/* FM DMA transfer bus address low reg 0x18 */
437 	u32 fmdmtcid;	/* FM DMA transfer bus communication ID reg 0x1c */
438 	u32 fmdmra;	/* FM DMA bus internal ram address register 0x20 */
439 	u32 fmdmrd;	/* FM DMA bus internal ram data register 0x24 */
440 	u32 fmdmwcr;	/* FM DMA CAM watchdog counter value 0x28 */
441 	u32 fmdmebcr;	/* FM DMA CAM base in MURAM register 0x2c */
442 	u32 fmdmccqdr;	/* FM DMA CAM and CMD Queue Debug reg 0x30 */
443 	u32 fmdmccqvr1;	/* FM DMA CAM and CMD Queue Value reg #1 0x34 */
444 	u32 fmdmccqvr2;	/* FM DMA CAM and CMD Queue Value reg #2 0x38 */
445 	u32 fmdmcqvr3;	/* FM DMA CMD Queue Value register #3 0x3c */
446 	u32 fmdmcqvr4;	/* FM DMA CMD Queue Value register #4 0x40 */
447 	u32 fmdmcqvr5;	/* FM DMA CMD Queue Value register #5 0x44 */
448 	u32 fmdmsefrc;	/* FM DMA Semaphore Entry Full Reject Cntr 0x48 */
449 	u32 fmdmsqfrc;	/* FM DMA Semaphore Queue Full Reject Cntr 0x4c */
450 	u32 fmdmssrc;	/* FM DMA Semaphore SYNC Reject Counter 0x50 */
451 	u32 fmdmdcr;	/* FM DMA Debug Counter 0x54 */
452 	u32 fmdmemsr;	/* FM DMA Emergency Smoother Register 0x58 */
453 	u32 res005c;	/* 0x5c */
454 	u32 fmdmplr[FMAN_LIODN_TBL / 2];	/* DMA LIODN regs 0x60-0xdf */
455 	u32 res00e0[0x400 - 56];
456 };
457 
458 struct fman_hwp_regs {
459 	u32 res0000[0x844 / 4];		/* 0x000..0x843 */
460 	u32 fmprrpimac;	/* FM Parser Internal memory access control */
461 	u32 res[(0x1000 - 0x848) / 4];	/* 0x848..0xFFF */
462 };
463 
464 /* Structure that holds current FMan state.
465  * Used for saving run time information.
466  */
467 struct fman_state_struct {
468 	u8 fm_id;
469 	u16 fm_clk_freq;
470 	struct fman_rev_info rev_info;
471 	bool enabled_time_stamp;
472 	u8 count1_micro_bit;
473 	u8 total_num_of_tasks;
474 	u8 accumulated_num_of_tasks;
475 	u32 accumulated_fifo_size;
476 	u8 accumulated_num_of_open_dmas;
477 	u8 accumulated_num_of_deq_tnums;
478 	u32 exceptions;
479 	u32 extra_fifo_pool_size;
480 	u8 extra_tasks_pool_size;
481 	u8 extra_open_dmas_pool_size;
482 	u16 port_mfl[MAX_NUM_OF_MACS];
483 	u16 mac_mfl[MAX_NUM_OF_MACS];
484 
485 	/* SOC specific */
486 	u32 fm_iram_size;
487 	/* DMA */
488 	u32 dma_thresh_max_commq;
489 	u32 dma_thresh_max_buf;
490 	u32 max_num_of_open_dmas;
491 	/* QMI */
492 	u32 qmi_max_num_of_tnums;
493 	u32 qmi_def_tnums_thresh;
494 	/* BMI */
495 	u32 bmi_max_num_of_tasks;
496 	u32 bmi_max_fifo_size;
497 	/* General */
498 	u32 fm_port_num_of_cg;
499 	u32 num_of_rx_ports;
500 	u32 total_fifo_size;
501 
502 	u32 qman_channel_base;
503 	u32 num_of_qman_channels;
504 
505 	struct resource *res;
506 };
507 
508 /* Structure that holds FMan initial configuration */
509 struct fman_cfg {
510 	u8 disp_limit_tsh;
511 	u8 prs_disp_tsh;
512 	u8 plcr_disp_tsh;
513 	u8 kg_disp_tsh;
514 	u8 bmi_disp_tsh;
515 	u8 qmi_enq_disp_tsh;
516 	u8 qmi_deq_disp_tsh;
517 	u8 fm_ctl1_disp_tsh;
518 	u8 fm_ctl2_disp_tsh;
519 	int dma_cache_override;
520 	enum fman_dma_aid_mode dma_aid_mode;
521 	u32 dma_axi_dbg_num_of_beats;
522 	u32 dma_cam_num_of_entries;
523 	u32 dma_watchdog;
524 	u8 dma_comm_qtsh_asrt_emer;
525 	u32 dma_write_buf_tsh_asrt_emer;
526 	u32 dma_read_buf_tsh_asrt_emer;
527 	u8 dma_comm_qtsh_clr_emer;
528 	u32 dma_write_buf_tsh_clr_emer;
529 	u32 dma_read_buf_tsh_clr_emer;
530 	u32 dma_sos_emergency;
531 	int dma_dbg_cnt_mode;
532 	int catastrophic_err;
533 	int dma_err;
534 	u32 exceptions;
535 	u16 clk_freq;
536 	u32 cam_base_addr;
537 	u32 fifo_base_addr;
538 	u32 total_fifo_size;
539 	u32 total_num_of_tasks;
540 	u32 qmi_def_tnums_thresh;
541 };
542 
543 #ifdef CONFIG_DPAA_ERRATUM_A050385
544 static bool fman_has_err_a050385;
545 #endif
546 
547 static irqreturn_t fman_exceptions(struct fman *fman,
548 				   enum fman_exceptions exception)
549 {
550 	dev_dbg(fman->dev, "%s: FMan[%d] exception %d\n",
551 		__func__, fman->state->fm_id, exception);
552 
553 	return IRQ_HANDLED;
554 }
555 
556 static irqreturn_t fman_bus_error(struct fman *fman, u8 __maybe_unused port_id,
557 				  u64 __maybe_unused addr,
558 				  u8 __maybe_unused tnum,
559 				  u16 __maybe_unused liodn)
560 {
561 	dev_dbg(fman->dev, "%s: FMan[%d] bus error: port_id[%d]\n",
562 		__func__, fman->state->fm_id, port_id);
563 
564 	return IRQ_HANDLED;
565 }
566 
567 static inline irqreturn_t call_mac_isr(struct fman *fman, u8 id)
568 {
569 	if (fman->intr_mng[id].isr_cb) {
570 		fman->intr_mng[id].isr_cb(fman->intr_mng[id].src_handle);
571 
572 		return IRQ_HANDLED;
573 	}
574 
575 	return IRQ_NONE;
576 }
577 
578 static inline u8 hw_port_id_to_sw_port_id(u8 major, u8 hw_port_id)
579 {
580 	u8 sw_port_id = 0;
581 
582 	if (hw_port_id >= BASE_TX_PORTID)
583 		sw_port_id = hw_port_id - BASE_TX_PORTID;
584 	else if (hw_port_id >= BASE_RX_PORTID)
585 		sw_port_id = hw_port_id - BASE_RX_PORTID;
586 	else
587 		sw_port_id = 0;
588 
589 	return sw_port_id;
590 }
591 
592 static void set_port_order_restoration(struct fman_fpm_regs __iomem *fpm_rg,
593 				       u8 port_id)
594 {
595 	u32 tmp = 0;
596 
597 	tmp = port_id << FPM_PORT_FM_CTL_PORTID_SHIFT;
598 
599 	tmp |= FPM_PRT_FM_CTL2 | FPM_PRT_FM_CTL1;
600 
601 	/* order restoration */
602 	if (port_id % 2)
603 		tmp |= FPM_PRT_FM_CTL1 << FPM_PRC_ORA_FM_CTL_SEL_SHIFT;
604 	else
605 		tmp |= FPM_PRT_FM_CTL2 << FPM_PRC_ORA_FM_CTL_SEL_SHIFT;
606 
607 	iowrite32be(tmp, &fpm_rg->fmfp_prc);
608 }
609 
610 static void set_port_liodn(struct fman *fman, u8 port_id,
611 			   u32 liodn_base, u32 liodn_ofst)
612 {
613 	u32 tmp;
614 
615 	iowrite32be(liodn_ofst, &fman->bmi_regs->fmbm_spliodn[port_id - 1]);
616 	if (!IS_ENABLED(CONFIG_FSL_PAMU))
617 		return;
618 	/* set LIODN base for this port */
619 	tmp = ioread32be(&fman->dma_regs->fmdmplr[port_id / 2]);
620 	if (port_id % 2) {
621 		tmp &= ~DMA_LIODN_BASE_MASK;
622 		tmp |= liodn_base;
623 	} else {
624 		tmp &= ~(DMA_LIODN_BASE_MASK << DMA_LIODN_SHIFT);
625 		tmp |= liodn_base << DMA_LIODN_SHIFT;
626 	}
627 	iowrite32be(tmp, &fman->dma_regs->fmdmplr[port_id / 2]);
628 }
629 
630 static void enable_rams_ecc(struct fman_fpm_regs __iomem *fpm_rg)
631 {
632 	u32 tmp;
633 
634 	tmp = ioread32be(&fpm_rg->fm_rcr);
635 	if (tmp & FPM_RAM_RAMS_ECC_EN_SRC_SEL)
636 		iowrite32be(tmp | FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
637 	else
638 		iowrite32be(tmp | FPM_RAM_RAMS_ECC_EN |
639 			    FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
640 }
641 
642 static void disable_rams_ecc(struct fman_fpm_regs __iomem *fpm_rg)
643 {
644 	u32 tmp;
645 
646 	tmp = ioread32be(&fpm_rg->fm_rcr);
647 	if (tmp & FPM_RAM_RAMS_ECC_EN_SRC_SEL)
648 		iowrite32be(tmp & ~FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
649 	else
650 		iowrite32be(tmp & ~(FPM_RAM_RAMS_ECC_EN | FPM_RAM_IRAM_ECC_EN),
651 			    &fpm_rg->fm_rcr);
652 }
653 
654 static void fman_defconfig(struct fman_cfg *cfg)
655 {
656 	memset(cfg, 0, sizeof(struct fman_cfg));
657 
658 	cfg->catastrophic_err = DEFAULT_CATASTROPHIC_ERR;
659 	cfg->dma_err = DEFAULT_DMA_ERR;
660 	cfg->dma_aid_mode = DEFAULT_AID_MODE;
661 	cfg->dma_comm_qtsh_clr_emer = DEFAULT_DMA_COMM_Q_LOW;
662 	cfg->dma_comm_qtsh_asrt_emer = DEFAULT_DMA_COMM_Q_HIGH;
663 	cfg->dma_cache_override = DEFAULT_CACHE_OVERRIDE;
664 	cfg->dma_cam_num_of_entries = DEFAULT_DMA_CAM_NUM_OF_ENTRIES;
665 	cfg->dma_dbg_cnt_mode = DEFAULT_DMA_DBG_CNT_MODE;
666 	cfg->dma_sos_emergency = DEFAULT_DMA_SOS_EMERGENCY;
667 	cfg->dma_watchdog = DEFAULT_DMA_WATCHDOG;
668 	cfg->disp_limit_tsh = DEFAULT_DISP_LIMIT;
669 	cfg->prs_disp_tsh = DEFAULT_PRS_DISP_TH;
670 	cfg->plcr_disp_tsh = DEFAULT_PLCR_DISP_TH;
671 	cfg->kg_disp_tsh = DEFAULT_KG_DISP_TH;
672 	cfg->bmi_disp_tsh = DEFAULT_BMI_DISP_TH;
673 	cfg->qmi_enq_disp_tsh = DEFAULT_QMI_ENQ_DISP_TH;
674 	cfg->qmi_deq_disp_tsh = DEFAULT_QMI_DEQ_DISP_TH;
675 	cfg->fm_ctl1_disp_tsh = DEFAULT_FM_CTL1_DISP_TH;
676 	cfg->fm_ctl2_disp_tsh = DEFAULT_FM_CTL2_DISP_TH;
677 }
678 
679 static int dma_init(struct fman *fman)
680 {
681 	struct fman_dma_regs __iomem *dma_rg = fman->dma_regs;
682 	struct fman_cfg *cfg = fman->cfg;
683 	u32 tmp_reg;
684 
685 	/* Init DMA Registers */
686 
687 	/* clear status reg events */
688 	tmp_reg = (DMA_STATUS_BUS_ERR | DMA_STATUS_READ_ECC |
689 		   DMA_STATUS_SYSTEM_WRITE_ECC | DMA_STATUS_FM_WRITE_ECC);
690 	iowrite32be(ioread32be(&dma_rg->fmdmsr) | tmp_reg, &dma_rg->fmdmsr);
691 
692 	/* configure mode register */
693 	tmp_reg = 0;
694 	tmp_reg |= cfg->dma_cache_override << DMA_MODE_CACHE_OR_SHIFT;
695 	if (cfg->exceptions & EX_DMA_BUS_ERROR)
696 		tmp_reg |= DMA_MODE_BER;
697 	if ((cfg->exceptions & EX_DMA_SYSTEM_WRITE_ECC) |
698 	    (cfg->exceptions & EX_DMA_READ_ECC) |
699 	    (cfg->exceptions & EX_DMA_FM_WRITE_ECC))
700 		tmp_reg |= DMA_MODE_ECC;
701 	if (cfg->dma_axi_dbg_num_of_beats)
702 		tmp_reg |= (DMA_MODE_AXI_DBG_MASK &
703 			((cfg->dma_axi_dbg_num_of_beats - 1)
704 			<< DMA_MODE_AXI_DBG_SHIFT));
705 
706 	tmp_reg |= (((cfg->dma_cam_num_of_entries / DMA_CAM_UNITS) - 1) &
707 		DMA_MODE_CEN_MASK) << DMA_MODE_CEN_SHIFT;
708 	tmp_reg |= DMA_MODE_SECURE_PROT;
709 	tmp_reg |= cfg->dma_dbg_cnt_mode << DMA_MODE_DBG_SHIFT;
710 	tmp_reg |= cfg->dma_aid_mode << DMA_MODE_AID_MODE_SHIFT;
711 
712 	iowrite32be(tmp_reg, &dma_rg->fmdmmr);
713 
714 	/* configure thresholds register */
715 	tmp_reg = ((u32)cfg->dma_comm_qtsh_asrt_emer <<
716 		DMA_THRESH_COMMQ_SHIFT);
717 	tmp_reg |= (cfg->dma_read_buf_tsh_asrt_emer &
718 		DMA_THRESH_READ_INT_BUF_MASK) << DMA_THRESH_READ_INT_BUF_SHIFT;
719 	tmp_reg |= cfg->dma_write_buf_tsh_asrt_emer &
720 		DMA_THRESH_WRITE_INT_BUF_MASK;
721 
722 	iowrite32be(tmp_reg, &dma_rg->fmdmtr);
723 
724 	/* configure hysteresis register */
725 	tmp_reg = ((u32)cfg->dma_comm_qtsh_clr_emer <<
726 		DMA_THRESH_COMMQ_SHIFT);
727 	tmp_reg |= (cfg->dma_read_buf_tsh_clr_emer &
728 		DMA_THRESH_READ_INT_BUF_MASK) << DMA_THRESH_READ_INT_BUF_SHIFT;
729 	tmp_reg |= cfg->dma_write_buf_tsh_clr_emer &
730 		DMA_THRESH_WRITE_INT_BUF_MASK;
731 
732 	iowrite32be(tmp_reg, &dma_rg->fmdmhy);
733 
734 	/* configure emergency threshold */
735 	iowrite32be(cfg->dma_sos_emergency, &dma_rg->fmdmsetr);
736 
737 	/* configure Watchdog */
738 	iowrite32be((cfg->dma_watchdog * cfg->clk_freq), &dma_rg->fmdmwcr);
739 
740 	iowrite32be(cfg->cam_base_addr, &dma_rg->fmdmebcr);
741 
742 	/* Allocate MURAM for CAM */
743 	fman->cam_size =
744 		(u32)(fman->cfg->dma_cam_num_of_entries * DMA_CAM_SIZEOF_ENTRY);
745 	fman->cam_offset = fman_muram_alloc(fman->muram, fman->cam_size);
746 	if (IS_ERR_VALUE(fman->cam_offset)) {
747 		dev_err(fman->dev, "%s: MURAM alloc for DMA CAM failed\n",
748 			__func__);
749 		return -ENOMEM;
750 	}
751 
752 	if (fman->state->rev_info.major == 2) {
753 		u32 __iomem *cam_base_addr;
754 
755 		fman_muram_free_mem(fman->muram, fman->cam_offset,
756 				    fman->cam_size);
757 
758 		fman->cam_size = fman->cfg->dma_cam_num_of_entries * 72 + 128;
759 		fman->cam_offset = fman_muram_alloc(fman->muram,
760 						    fman->cam_size);
761 		if (IS_ERR_VALUE(fman->cam_offset)) {
762 			dev_err(fman->dev, "%s: MURAM alloc for DMA CAM failed\n",
763 				__func__);
764 			return -ENOMEM;
765 		}
766 
767 		if (fman->cfg->dma_cam_num_of_entries % 8 ||
768 		    fman->cfg->dma_cam_num_of_entries > 32) {
769 			dev_err(fman->dev, "%s: wrong dma_cam_num_of_entries\n",
770 				__func__);
771 			return -EINVAL;
772 		}
773 
774 		cam_base_addr = (u32 __iomem *)
775 			fman_muram_offset_to_vbase(fman->muram,
776 						   fman->cam_offset);
777 		iowrite32be(~((1 <<
778 			    (32 - fman->cfg->dma_cam_num_of_entries)) - 1),
779 			    cam_base_addr);
780 	}
781 
782 	fman->cfg->cam_base_addr = fman->cam_offset;
783 
784 	return 0;
785 }
786 
787 static void fpm_init(struct fman_fpm_regs __iomem *fpm_rg, struct fman_cfg *cfg)
788 {
789 	u32 tmp_reg;
790 	int i;
791 
792 	/* Init FPM Registers */
793 
794 	tmp_reg = (u32)(cfg->disp_limit_tsh << FPM_DISP_LIMIT_SHIFT);
795 	iowrite32be(tmp_reg, &fpm_rg->fmfp_mxd);
796 
797 	tmp_reg = (((u32)cfg->prs_disp_tsh << FPM_THR1_PRS_SHIFT) |
798 		   ((u32)cfg->kg_disp_tsh << FPM_THR1_KG_SHIFT) |
799 		   ((u32)cfg->plcr_disp_tsh << FPM_THR1_PLCR_SHIFT) |
800 		   ((u32)cfg->bmi_disp_tsh << FPM_THR1_BMI_SHIFT));
801 	iowrite32be(tmp_reg, &fpm_rg->fmfp_dist1);
802 
803 	tmp_reg =
804 		(((u32)cfg->qmi_enq_disp_tsh << FPM_THR2_QMI_ENQ_SHIFT) |
805 		 ((u32)cfg->qmi_deq_disp_tsh << FPM_THR2_QMI_DEQ_SHIFT) |
806 		 ((u32)cfg->fm_ctl1_disp_tsh << FPM_THR2_FM_CTL1_SHIFT) |
807 		 ((u32)cfg->fm_ctl2_disp_tsh << FPM_THR2_FM_CTL2_SHIFT));
808 	iowrite32be(tmp_reg, &fpm_rg->fmfp_dist2);
809 
810 	/* define exceptions and error behavior */
811 	tmp_reg = 0;
812 	/* Clear events */
813 	tmp_reg |= (FPM_EV_MASK_STALL | FPM_EV_MASK_DOUBLE_ECC |
814 		    FPM_EV_MASK_SINGLE_ECC);
815 	/* enable interrupts */
816 	if (cfg->exceptions & EX_FPM_STALL_ON_TASKS)
817 		tmp_reg |= FPM_EV_MASK_STALL_EN;
818 	if (cfg->exceptions & EX_FPM_SINGLE_ECC)
819 		tmp_reg |= FPM_EV_MASK_SINGLE_ECC_EN;
820 	if (cfg->exceptions & EX_FPM_DOUBLE_ECC)
821 		tmp_reg |= FPM_EV_MASK_DOUBLE_ECC_EN;
822 	tmp_reg |= (cfg->catastrophic_err << FPM_EV_MASK_CAT_ERR_SHIFT);
823 	tmp_reg |= (cfg->dma_err << FPM_EV_MASK_DMA_ERR_SHIFT);
824 	/* FMan is not halted upon external halt activation */
825 	tmp_reg |= FPM_EV_MASK_EXTERNAL_HALT;
826 	/* Man is not halted upon  Unrecoverable ECC error behavior */
827 	tmp_reg |= FPM_EV_MASK_ECC_ERR_HALT;
828 	iowrite32be(tmp_reg, &fpm_rg->fmfp_ee);
829 
830 	/* clear all fmCtls event registers */
831 	for (i = 0; i < FM_NUM_OF_FMAN_CTRL_EVENT_REGS; i++)
832 		iowrite32be(0xFFFFFFFF, &fpm_rg->fmfp_cev[i]);
833 
834 	/* RAM ECC -  enable and clear events */
835 	/* first we need to clear all parser memory,
836 	 * as it is uninitialized and may cause ECC errors
837 	 */
838 	/* event bits */
839 	tmp_reg = (FPM_RAM_MURAM_ECC | FPM_RAM_IRAM_ECC);
840 
841 	iowrite32be(tmp_reg, &fpm_rg->fm_rcr);
842 
843 	tmp_reg = 0;
844 	if (cfg->exceptions & EX_IRAM_ECC) {
845 		tmp_reg |= FPM_IRAM_ECC_ERR_EX_EN;
846 		enable_rams_ecc(fpm_rg);
847 	}
848 	if (cfg->exceptions & EX_MURAM_ECC) {
849 		tmp_reg |= FPM_MURAM_ECC_ERR_EX_EN;
850 		enable_rams_ecc(fpm_rg);
851 	}
852 	iowrite32be(tmp_reg, &fpm_rg->fm_rie);
853 }
854 
855 static void bmi_init(struct fman_bmi_regs __iomem *bmi_rg,
856 		     struct fman_cfg *cfg)
857 {
858 	u32 tmp_reg;
859 
860 	/* Init BMI Registers */
861 
862 	/* define common resources */
863 	tmp_reg = cfg->fifo_base_addr;
864 	tmp_reg = tmp_reg / BMI_FIFO_ALIGN;
865 
866 	tmp_reg |= ((cfg->total_fifo_size / FMAN_BMI_FIFO_UNITS - 1) <<
867 		    BMI_CFG1_FIFO_SIZE_SHIFT);
868 	iowrite32be(tmp_reg, &bmi_rg->fmbm_cfg1);
869 
870 	tmp_reg = ((cfg->total_num_of_tasks - 1) & BMI_CFG2_TASKS_MASK) <<
871 		   BMI_CFG2_TASKS_SHIFT;
872 	/* num of DMA's will be dynamically updated when each port is set */
873 	iowrite32be(tmp_reg, &bmi_rg->fmbm_cfg2);
874 
875 	/* define unmaskable exceptions, enable and clear events */
876 	tmp_reg = 0;
877 	iowrite32be(BMI_ERR_INTR_EN_LIST_RAM_ECC |
878 		    BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC |
879 		    BMI_ERR_INTR_EN_STATISTICS_RAM_ECC |
880 		    BMI_ERR_INTR_EN_DISPATCH_RAM_ECC, &bmi_rg->fmbm_ievr);
881 
882 	if (cfg->exceptions & EX_BMI_LIST_RAM_ECC)
883 		tmp_reg |= BMI_ERR_INTR_EN_LIST_RAM_ECC;
884 	if (cfg->exceptions & EX_BMI_STORAGE_PROFILE_ECC)
885 		tmp_reg |= BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
886 	if (cfg->exceptions & EX_BMI_STATISTICS_RAM_ECC)
887 		tmp_reg |= BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
888 	if (cfg->exceptions & EX_BMI_DISPATCH_RAM_ECC)
889 		tmp_reg |= BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
890 	iowrite32be(tmp_reg, &bmi_rg->fmbm_ier);
891 }
892 
893 static void qmi_init(struct fman_qmi_regs __iomem *qmi_rg,
894 		     struct fman_cfg *cfg)
895 {
896 	u32 tmp_reg;
897 
898 	/* Init QMI Registers */
899 
900 	/* Clear error interrupt events */
901 
902 	iowrite32be(QMI_ERR_INTR_EN_DOUBLE_ECC | QMI_ERR_INTR_EN_DEQ_FROM_DEF,
903 		    &qmi_rg->fmqm_eie);
904 	tmp_reg = 0;
905 	if (cfg->exceptions & EX_QMI_DEQ_FROM_UNKNOWN_PORTID)
906 		tmp_reg |= QMI_ERR_INTR_EN_DEQ_FROM_DEF;
907 	if (cfg->exceptions & EX_QMI_DOUBLE_ECC)
908 		tmp_reg |= QMI_ERR_INTR_EN_DOUBLE_ECC;
909 	/* enable events */
910 	iowrite32be(tmp_reg, &qmi_rg->fmqm_eien);
911 
912 	tmp_reg = 0;
913 	/* Clear interrupt events */
914 	iowrite32be(QMI_INTR_EN_SINGLE_ECC, &qmi_rg->fmqm_ie);
915 	if (cfg->exceptions & EX_QMI_SINGLE_ECC)
916 		tmp_reg |= QMI_INTR_EN_SINGLE_ECC;
917 	/* enable events */
918 	iowrite32be(tmp_reg, &qmi_rg->fmqm_ien);
919 }
920 
921 static void hwp_init(struct fman_hwp_regs __iomem *hwp_rg)
922 {
923 	/* enable HW Parser */
924 	iowrite32be(HWP_RPIMAC_PEN, &hwp_rg->fmprrpimac);
925 }
926 
927 static int enable(struct fman *fman, struct fman_cfg *cfg)
928 {
929 	u32 cfg_reg = 0;
930 
931 	/* Enable all modules */
932 
933 	/* clear&enable global counters - calculate reg and save for later,
934 	 * because it's the same reg for QMI enable
935 	 */
936 	cfg_reg = QMI_CFG_EN_COUNTERS;
937 
938 	/* Set enqueue and dequeue thresholds */
939 	cfg_reg |= (cfg->qmi_def_tnums_thresh << 8) | cfg->qmi_def_tnums_thresh;
940 
941 	iowrite32be(BMI_INIT_START, &fman->bmi_regs->fmbm_init);
942 	iowrite32be(cfg_reg | QMI_CFG_ENQ_EN | QMI_CFG_DEQ_EN,
943 		    &fman->qmi_regs->fmqm_gc);
944 
945 	return 0;
946 }
947 
948 static int set_exception(struct fman *fman,
949 			 enum fman_exceptions exception, bool enable)
950 {
951 	u32 tmp;
952 
953 	switch (exception) {
954 	case FMAN_EX_DMA_BUS_ERROR:
955 		tmp = ioread32be(&fman->dma_regs->fmdmmr);
956 		if (enable)
957 			tmp |= DMA_MODE_BER;
958 		else
959 			tmp &= ~DMA_MODE_BER;
960 		/* disable bus error */
961 		iowrite32be(tmp, &fman->dma_regs->fmdmmr);
962 		break;
963 	case FMAN_EX_DMA_READ_ECC:
964 	case FMAN_EX_DMA_SYSTEM_WRITE_ECC:
965 	case FMAN_EX_DMA_FM_WRITE_ECC:
966 		tmp = ioread32be(&fman->dma_regs->fmdmmr);
967 		if (enable)
968 			tmp |= DMA_MODE_ECC;
969 		else
970 			tmp &= ~DMA_MODE_ECC;
971 		iowrite32be(tmp, &fman->dma_regs->fmdmmr);
972 		break;
973 	case FMAN_EX_FPM_STALL_ON_TASKS:
974 		tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
975 		if (enable)
976 			tmp |= FPM_EV_MASK_STALL_EN;
977 		else
978 			tmp &= ~FPM_EV_MASK_STALL_EN;
979 		iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
980 		break;
981 	case FMAN_EX_FPM_SINGLE_ECC:
982 		tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
983 		if (enable)
984 			tmp |= FPM_EV_MASK_SINGLE_ECC_EN;
985 		else
986 			tmp &= ~FPM_EV_MASK_SINGLE_ECC_EN;
987 		iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
988 		break;
989 	case FMAN_EX_FPM_DOUBLE_ECC:
990 		tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
991 		if (enable)
992 			tmp |= FPM_EV_MASK_DOUBLE_ECC_EN;
993 		else
994 			tmp &= ~FPM_EV_MASK_DOUBLE_ECC_EN;
995 		iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
996 		break;
997 	case FMAN_EX_QMI_SINGLE_ECC:
998 		tmp = ioread32be(&fman->qmi_regs->fmqm_ien);
999 		if (enable)
1000 			tmp |= QMI_INTR_EN_SINGLE_ECC;
1001 		else
1002 			tmp &= ~QMI_INTR_EN_SINGLE_ECC;
1003 		iowrite32be(tmp, &fman->qmi_regs->fmqm_ien);
1004 		break;
1005 	case FMAN_EX_QMI_DOUBLE_ECC:
1006 		tmp = ioread32be(&fman->qmi_regs->fmqm_eien);
1007 		if (enable)
1008 			tmp |= QMI_ERR_INTR_EN_DOUBLE_ECC;
1009 		else
1010 			tmp &= ~QMI_ERR_INTR_EN_DOUBLE_ECC;
1011 		iowrite32be(tmp, &fman->qmi_regs->fmqm_eien);
1012 		break;
1013 	case FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID:
1014 		tmp = ioread32be(&fman->qmi_regs->fmqm_eien);
1015 		if (enable)
1016 			tmp |= QMI_ERR_INTR_EN_DEQ_FROM_DEF;
1017 		else
1018 			tmp &= ~QMI_ERR_INTR_EN_DEQ_FROM_DEF;
1019 		iowrite32be(tmp, &fman->qmi_regs->fmqm_eien);
1020 		break;
1021 	case FMAN_EX_BMI_LIST_RAM_ECC:
1022 		tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1023 		if (enable)
1024 			tmp |= BMI_ERR_INTR_EN_LIST_RAM_ECC;
1025 		else
1026 			tmp &= ~BMI_ERR_INTR_EN_LIST_RAM_ECC;
1027 		iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1028 		break;
1029 	case FMAN_EX_BMI_STORAGE_PROFILE_ECC:
1030 		tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1031 		if (enable)
1032 			tmp |= BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
1033 		else
1034 			tmp &= ~BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
1035 		iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1036 		break;
1037 	case FMAN_EX_BMI_STATISTICS_RAM_ECC:
1038 		tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1039 		if (enable)
1040 			tmp |= BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
1041 		else
1042 			tmp &= ~BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
1043 		iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1044 		break;
1045 	case FMAN_EX_BMI_DISPATCH_RAM_ECC:
1046 		tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1047 		if (enable)
1048 			tmp |= BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
1049 		else
1050 			tmp &= ~BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
1051 		iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1052 		break;
1053 	case FMAN_EX_IRAM_ECC:
1054 		tmp = ioread32be(&fman->fpm_regs->fm_rie);
1055 		if (enable) {
1056 			/* enable ECC if not enabled */
1057 			enable_rams_ecc(fman->fpm_regs);
1058 			/* enable ECC interrupts */
1059 			tmp |= FPM_IRAM_ECC_ERR_EX_EN;
1060 		} else {
1061 			/* ECC mechanism may be disabled,
1062 			 * depending on driver status
1063 			 */
1064 			disable_rams_ecc(fman->fpm_regs);
1065 			tmp &= ~FPM_IRAM_ECC_ERR_EX_EN;
1066 		}
1067 		iowrite32be(tmp, &fman->fpm_regs->fm_rie);
1068 		break;
1069 	case FMAN_EX_MURAM_ECC:
1070 		tmp = ioread32be(&fman->fpm_regs->fm_rie);
1071 		if (enable) {
1072 			/* enable ECC if not enabled */
1073 			enable_rams_ecc(fman->fpm_regs);
1074 			/* enable ECC interrupts */
1075 			tmp |= FPM_MURAM_ECC_ERR_EX_EN;
1076 		} else {
1077 			/* ECC mechanism may be disabled,
1078 			 * depending on driver status
1079 			 */
1080 			disable_rams_ecc(fman->fpm_regs);
1081 			tmp &= ~FPM_MURAM_ECC_ERR_EX_EN;
1082 		}
1083 		iowrite32be(tmp, &fman->fpm_regs->fm_rie);
1084 		break;
1085 	default:
1086 		return -EINVAL;
1087 	}
1088 	return 0;
1089 }
1090 
1091 static void resume(struct fman_fpm_regs __iomem *fpm_rg)
1092 {
1093 	u32 tmp;
1094 
1095 	tmp = ioread32be(&fpm_rg->fmfp_ee);
1096 	/* clear tmp_reg event bits in order not to clear standing events */
1097 	tmp &= ~(FPM_EV_MASK_DOUBLE_ECC |
1098 		 FPM_EV_MASK_STALL | FPM_EV_MASK_SINGLE_ECC);
1099 	tmp |= FPM_EV_MASK_RELEASE_FM;
1100 
1101 	iowrite32be(tmp, &fpm_rg->fmfp_ee);
1102 }
1103 
1104 static int fill_soc_specific_params(struct fman_state_struct *state)
1105 {
1106 	u8 minor = state->rev_info.minor;
1107 	/* P4080 - Major 2
1108 	 * P2041/P3041/P5020/P5040 - Major 3
1109 	 * Tx/Bx - Major 6
1110 	 */
1111 	switch (state->rev_info.major) {
1112 	case 3:
1113 		state->bmi_max_fifo_size	= 160 * 1024;
1114 		state->fm_iram_size		= 64 * 1024;
1115 		state->dma_thresh_max_commq	= 31;
1116 		state->dma_thresh_max_buf	= 127;
1117 		state->qmi_max_num_of_tnums	= 64;
1118 		state->qmi_def_tnums_thresh	= 48;
1119 		state->bmi_max_num_of_tasks	= 128;
1120 		state->max_num_of_open_dmas	= 32;
1121 		state->fm_port_num_of_cg	= 256;
1122 		state->num_of_rx_ports	= 6;
1123 		state->total_fifo_size	= 136 * 1024;
1124 		break;
1125 
1126 	case 2:
1127 		state->bmi_max_fifo_size	= 160 * 1024;
1128 		state->fm_iram_size		= 64 * 1024;
1129 		state->dma_thresh_max_commq	= 31;
1130 		state->dma_thresh_max_buf	= 127;
1131 		state->qmi_max_num_of_tnums	= 64;
1132 		state->qmi_def_tnums_thresh	= 48;
1133 		state->bmi_max_num_of_tasks	= 128;
1134 		state->max_num_of_open_dmas	= 32;
1135 		state->fm_port_num_of_cg	= 256;
1136 		state->num_of_rx_ports	= 5;
1137 		state->total_fifo_size	= 100 * 1024;
1138 		break;
1139 
1140 	case 6:
1141 		state->dma_thresh_max_commq	= 83;
1142 		state->dma_thresh_max_buf	= 127;
1143 		state->qmi_max_num_of_tnums	= 64;
1144 		state->qmi_def_tnums_thresh	= 32;
1145 		state->fm_port_num_of_cg	= 256;
1146 
1147 		/* FManV3L */
1148 		if (minor == 1 || minor == 4) {
1149 			state->bmi_max_fifo_size	= 192 * 1024;
1150 			state->bmi_max_num_of_tasks	= 64;
1151 			state->max_num_of_open_dmas	= 32;
1152 			state->num_of_rx_ports		= 5;
1153 			if (minor == 1)
1154 				state->fm_iram_size	= 32 * 1024;
1155 			else
1156 				state->fm_iram_size	= 64 * 1024;
1157 			state->total_fifo_size		= 156 * 1024;
1158 		}
1159 		/* FManV3H */
1160 		else if (minor == 0 || minor == 2 || minor == 3) {
1161 			state->bmi_max_fifo_size	= 384 * 1024;
1162 			state->fm_iram_size		= 64 * 1024;
1163 			state->bmi_max_num_of_tasks	= 128;
1164 			state->max_num_of_open_dmas	= 84;
1165 			state->num_of_rx_ports		= 8;
1166 			state->total_fifo_size		= 295 * 1024;
1167 		} else {
1168 			pr_err("Unsupported FManv3 version\n");
1169 			return -EINVAL;
1170 		}
1171 
1172 		break;
1173 	default:
1174 		pr_err("Unsupported FMan version\n");
1175 		return -EINVAL;
1176 	}
1177 
1178 	return 0;
1179 }
1180 
1181 static bool is_init_done(struct fman_cfg *cfg)
1182 {
1183 	/* Checks if FMan driver parameters were initialized */
1184 	if (!cfg)
1185 		return true;
1186 
1187 	return false;
1188 }
1189 
1190 static void free_init_resources(struct fman *fman)
1191 {
1192 	if (fman->cam_offset)
1193 		fman_muram_free_mem(fman->muram, fman->cam_offset,
1194 				    fman->cam_size);
1195 	if (fman->fifo_offset)
1196 		fman_muram_free_mem(fman->muram, fman->fifo_offset,
1197 				    fman->fifo_size);
1198 }
1199 
1200 static irqreturn_t bmi_err_event(struct fman *fman)
1201 {
1202 	u32 event, mask, force;
1203 	struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1204 	irqreturn_t ret = IRQ_NONE;
1205 
1206 	event = ioread32be(&bmi_rg->fmbm_ievr);
1207 	mask = ioread32be(&bmi_rg->fmbm_ier);
1208 	event &= mask;
1209 	/* clear the forced events */
1210 	force = ioread32be(&bmi_rg->fmbm_ifr);
1211 	if (force & event)
1212 		iowrite32be(force & ~event, &bmi_rg->fmbm_ifr);
1213 	/* clear the acknowledged events */
1214 	iowrite32be(event, &bmi_rg->fmbm_ievr);
1215 
1216 	if (event & BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC)
1217 		ret = fman->exception_cb(fman, FMAN_EX_BMI_STORAGE_PROFILE_ECC);
1218 	if (event & BMI_ERR_INTR_EN_LIST_RAM_ECC)
1219 		ret = fman->exception_cb(fman, FMAN_EX_BMI_LIST_RAM_ECC);
1220 	if (event & BMI_ERR_INTR_EN_STATISTICS_RAM_ECC)
1221 		ret = fman->exception_cb(fman, FMAN_EX_BMI_STATISTICS_RAM_ECC);
1222 	if (event & BMI_ERR_INTR_EN_DISPATCH_RAM_ECC)
1223 		ret = fman->exception_cb(fman, FMAN_EX_BMI_DISPATCH_RAM_ECC);
1224 
1225 	return ret;
1226 }
1227 
1228 static irqreturn_t qmi_err_event(struct fman *fman)
1229 {
1230 	u32 event, mask, force;
1231 	struct fman_qmi_regs __iomem *qmi_rg = fman->qmi_regs;
1232 	irqreturn_t ret = IRQ_NONE;
1233 
1234 	event = ioread32be(&qmi_rg->fmqm_eie);
1235 	mask = ioread32be(&qmi_rg->fmqm_eien);
1236 	event &= mask;
1237 
1238 	/* clear the forced events */
1239 	force = ioread32be(&qmi_rg->fmqm_eif);
1240 	if (force & event)
1241 		iowrite32be(force & ~event, &qmi_rg->fmqm_eif);
1242 	/* clear the acknowledged events */
1243 	iowrite32be(event, &qmi_rg->fmqm_eie);
1244 
1245 	if (event & QMI_ERR_INTR_EN_DOUBLE_ECC)
1246 		ret = fman->exception_cb(fman, FMAN_EX_QMI_DOUBLE_ECC);
1247 	if (event & QMI_ERR_INTR_EN_DEQ_FROM_DEF)
1248 		ret = fman->exception_cb(fman,
1249 					 FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID);
1250 
1251 	return ret;
1252 }
1253 
1254 static irqreturn_t dma_err_event(struct fman *fman)
1255 {
1256 	u32 status, mask, com_id;
1257 	u8 tnum, port_id, relative_port_id;
1258 	u16 liodn;
1259 	struct fman_dma_regs __iomem *dma_rg = fman->dma_regs;
1260 	irqreturn_t ret = IRQ_NONE;
1261 
1262 	status = ioread32be(&dma_rg->fmdmsr);
1263 	mask = ioread32be(&dma_rg->fmdmmr);
1264 
1265 	/* clear DMA_STATUS_BUS_ERR if mask has no DMA_MODE_BER */
1266 	if ((mask & DMA_MODE_BER) != DMA_MODE_BER)
1267 		status &= ~DMA_STATUS_BUS_ERR;
1268 
1269 	/* clear relevant bits if mask has no DMA_MODE_ECC */
1270 	if ((mask & DMA_MODE_ECC) != DMA_MODE_ECC)
1271 		status &= ~(DMA_STATUS_FM_SPDAT_ECC |
1272 			    DMA_STATUS_READ_ECC |
1273 			    DMA_STATUS_SYSTEM_WRITE_ECC |
1274 			    DMA_STATUS_FM_WRITE_ECC);
1275 
1276 	/* clear set events */
1277 	iowrite32be(status, &dma_rg->fmdmsr);
1278 
1279 	if (status & DMA_STATUS_BUS_ERR) {
1280 		u64 addr;
1281 
1282 		addr = (u64)ioread32be(&dma_rg->fmdmtal);
1283 		addr |= ((u64)(ioread32be(&dma_rg->fmdmtah)) << 32);
1284 
1285 		com_id = ioread32be(&dma_rg->fmdmtcid);
1286 		port_id = (u8)(((com_id & DMA_TRANSFER_PORTID_MASK) >>
1287 			       DMA_TRANSFER_PORTID_SHIFT));
1288 		relative_port_id =
1289 		hw_port_id_to_sw_port_id(fman->state->rev_info.major, port_id);
1290 		tnum = (u8)((com_id & DMA_TRANSFER_TNUM_MASK) >>
1291 			    DMA_TRANSFER_TNUM_SHIFT);
1292 		liodn = (u16)(com_id & DMA_TRANSFER_LIODN_MASK);
1293 		ret = fman->bus_error_cb(fman, relative_port_id, addr, tnum,
1294 					 liodn);
1295 	}
1296 	if (status & DMA_STATUS_FM_SPDAT_ECC)
1297 		ret = fman->exception_cb(fman, FMAN_EX_DMA_SINGLE_PORT_ECC);
1298 	if (status & DMA_STATUS_READ_ECC)
1299 		ret = fman->exception_cb(fman, FMAN_EX_DMA_READ_ECC);
1300 	if (status & DMA_STATUS_SYSTEM_WRITE_ECC)
1301 		ret = fman->exception_cb(fman, FMAN_EX_DMA_SYSTEM_WRITE_ECC);
1302 	if (status & DMA_STATUS_FM_WRITE_ECC)
1303 		ret = fman->exception_cb(fman, FMAN_EX_DMA_FM_WRITE_ECC);
1304 
1305 	return ret;
1306 }
1307 
1308 static irqreturn_t fpm_err_event(struct fman *fman)
1309 {
1310 	u32 event;
1311 	struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1312 	irqreturn_t ret = IRQ_NONE;
1313 
1314 	event = ioread32be(&fpm_rg->fmfp_ee);
1315 	/* clear the all occurred events */
1316 	iowrite32be(event, &fpm_rg->fmfp_ee);
1317 
1318 	if ((event & FPM_EV_MASK_DOUBLE_ECC) &&
1319 	    (event & FPM_EV_MASK_DOUBLE_ECC_EN))
1320 		ret = fman->exception_cb(fman, FMAN_EX_FPM_DOUBLE_ECC);
1321 	if ((event & FPM_EV_MASK_STALL) && (event & FPM_EV_MASK_STALL_EN))
1322 		ret = fman->exception_cb(fman, FMAN_EX_FPM_STALL_ON_TASKS);
1323 	if ((event & FPM_EV_MASK_SINGLE_ECC) &&
1324 	    (event & FPM_EV_MASK_SINGLE_ECC_EN))
1325 		ret = fman->exception_cb(fman, FMAN_EX_FPM_SINGLE_ECC);
1326 
1327 	return ret;
1328 }
1329 
1330 static irqreturn_t muram_err_intr(struct fman *fman)
1331 {
1332 	u32 event, mask;
1333 	struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1334 	irqreturn_t ret = IRQ_NONE;
1335 
1336 	event = ioread32be(&fpm_rg->fm_rcr);
1337 	mask = ioread32be(&fpm_rg->fm_rie);
1338 
1339 	/* clear MURAM event bit (do not clear IRAM event) */
1340 	iowrite32be(event & ~FPM_RAM_IRAM_ECC, &fpm_rg->fm_rcr);
1341 
1342 	if ((mask & FPM_MURAM_ECC_ERR_EX_EN) && (event & FPM_RAM_MURAM_ECC))
1343 		ret = fman->exception_cb(fman, FMAN_EX_MURAM_ECC);
1344 
1345 	return ret;
1346 }
1347 
1348 static irqreturn_t qmi_event(struct fman *fman)
1349 {
1350 	u32 event, mask, force;
1351 	struct fman_qmi_regs __iomem *qmi_rg = fman->qmi_regs;
1352 	irqreturn_t ret = IRQ_NONE;
1353 
1354 	event = ioread32be(&qmi_rg->fmqm_ie);
1355 	mask = ioread32be(&qmi_rg->fmqm_ien);
1356 	event &= mask;
1357 	/* clear the forced events */
1358 	force = ioread32be(&qmi_rg->fmqm_if);
1359 	if (force & event)
1360 		iowrite32be(force & ~event, &qmi_rg->fmqm_if);
1361 	/* clear the acknowledged events */
1362 	iowrite32be(event, &qmi_rg->fmqm_ie);
1363 
1364 	if (event & QMI_INTR_EN_SINGLE_ECC)
1365 		ret = fman->exception_cb(fman, FMAN_EX_QMI_SINGLE_ECC);
1366 
1367 	return ret;
1368 }
1369 
1370 static void enable_time_stamp(struct fman *fman)
1371 {
1372 	struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1373 	u16 fm_clk_freq = fman->state->fm_clk_freq;
1374 	u32 tmp, intgr, ts_freq, frac;
1375 
1376 	ts_freq = (u32)(1 << fman->state->count1_micro_bit);
1377 	/* configure timestamp so that bit 8 will count 1 microsecond
1378 	 * Find effective count rate at TIMESTAMP least significant bits:
1379 	 * Effective_Count_Rate = 1MHz x 2^8 = 256MHz
1380 	 * Find frequency ratio between effective count rate and the clock:
1381 	 * Effective_Count_Rate / CLK e.g. for 600 MHz clock:
1382 	 * 256/600 = 0.4266666...
1383 	 */
1384 
1385 	intgr = ts_freq / fm_clk_freq;
1386 	/* we multiply by 2^16 to keep the fraction of the division
1387 	 * we do not div back, since we write this value as a fraction
1388 	 * see spec
1389 	 */
1390 
1391 	frac = ((ts_freq << 16) - (intgr << 16) * fm_clk_freq) / fm_clk_freq;
1392 	/* we check remainder of the division in order to round up if not int */
1393 	if (((ts_freq << 16) - (intgr << 16) * fm_clk_freq) % fm_clk_freq)
1394 		frac++;
1395 
1396 	tmp = (intgr << FPM_TS_INT_SHIFT) | (u16)frac;
1397 	iowrite32be(tmp, &fpm_rg->fmfp_tsc2);
1398 
1399 	/* enable timestamp with original clock */
1400 	iowrite32be(FPM_TS_CTL_EN, &fpm_rg->fmfp_tsc1);
1401 	fman->state->enabled_time_stamp = true;
1402 }
1403 
1404 static int clear_iram(struct fman *fman)
1405 {
1406 	struct fman_iram_regs __iomem *iram;
1407 	int i, count;
1408 
1409 	iram = fman->base_addr + IMEM_OFFSET;
1410 
1411 	/* Enable the auto-increment */
1412 	iowrite32be(IRAM_IADD_AIE, &iram->iadd);
1413 	count = 100;
1414 	do {
1415 		udelay(1);
1416 	} while ((ioread32be(&iram->iadd) != IRAM_IADD_AIE) && --count);
1417 	if (count == 0)
1418 		return -EBUSY;
1419 
1420 	for (i = 0; i < (fman->state->fm_iram_size / 4); i++)
1421 		iowrite32be(0xffffffff, &iram->idata);
1422 
1423 	iowrite32be(fman->state->fm_iram_size - 4, &iram->iadd);
1424 	count = 100;
1425 	do {
1426 		udelay(1);
1427 	} while ((ioread32be(&iram->idata) != 0xffffffff) && --count);
1428 	if (count == 0)
1429 		return -EBUSY;
1430 
1431 	return 0;
1432 }
1433 
1434 static u32 get_exception_flag(enum fman_exceptions exception)
1435 {
1436 	u32 bit_mask;
1437 
1438 	switch (exception) {
1439 	case FMAN_EX_DMA_BUS_ERROR:
1440 		bit_mask = EX_DMA_BUS_ERROR;
1441 		break;
1442 	case FMAN_EX_DMA_SINGLE_PORT_ECC:
1443 		bit_mask = EX_DMA_SINGLE_PORT_ECC;
1444 		break;
1445 	case FMAN_EX_DMA_READ_ECC:
1446 		bit_mask = EX_DMA_READ_ECC;
1447 		break;
1448 	case FMAN_EX_DMA_SYSTEM_WRITE_ECC:
1449 		bit_mask = EX_DMA_SYSTEM_WRITE_ECC;
1450 		break;
1451 	case FMAN_EX_DMA_FM_WRITE_ECC:
1452 		bit_mask = EX_DMA_FM_WRITE_ECC;
1453 		break;
1454 	case FMAN_EX_FPM_STALL_ON_TASKS:
1455 		bit_mask = EX_FPM_STALL_ON_TASKS;
1456 		break;
1457 	case FMAN_EX_FPM_SINGLE_ECC:
1458 		bit_mask = EX_FPM_SINGLE_ECC;
1459 		break;
1460 	case FMAN_EX_FPM_DOUBLE_ECC:
1461 		bit_mask = EX_FPM_DOUBLE_ECC;
1462 		break;
1463 	case FMAN_EX_QMI_SINGLE_ECC:
1464 		bit_mask = EX_QMI_SINGLE_ECC;
1465 		break;
1466 	case FMAN_EX_QMI_DOUBLE_ECC:
1467 		bit_mask = EX_QMI_DOUBLE_ECC;
1468 		break;
1469 	case FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID:
1470 		bit_mask = EX_QMI_DEQ_FROM_UNKNOWN_PORTID;
1471 		break;
1472 	case FMAN_EX_BMI_LIST_RAM_ECC:
1473 		bit_mask = EX_BMI_LIST_RAM_ECC;
1474 		break;
1475 	case FMAN_EX_BMI_STORAGE_PROFILE_ECC:
1476 		bit_mask = EX_BMI_STORAGE_PROFILE_ECC;
1477 		break;
1478 	case FMAN_EX_BMI_STATISTICS_RAM_ECC:
1479 		bit_mask = EX_BMI_STATISTICS_RAM_ECC;
1480 		break;
1481 	case FMAN_EX_BMI_DISPATCH_RAM_ECC:
1482 		bit_mask = EX_BMI_DISPATCH_RAM_ECC;
1483 		break;
1484 	case FMAN_EX_MURAM_ECC:
1485 		bit_mask = EX_MURAM_ECC;
1486 		break;
1487 	default:
1488 		bit_mask = 0;
1489 		break;
1490 	}
1491 
1492 	return bit_mask;
1493 }
1494 
1495 static int get_module_event(enum fman_event_modules module, u8 mod_id,
1496 			    enum fman_intr_type intr_type)
1497 {
1498 	int event;
1499 
1500 	switch (module) {
1501 	case FMAN_MOD_MAC:
1502 		if (intr_type == FMAN_INTR_TYPE_ERR)
1503 			event = FMAN_EV_ERR_MAC0 + mod_id;
1504 		else
1505 			event = FMAN_EV_MAC0 + mod_id;
1506 		break;
1507 	case FMAN_MOD_FMAN_CTRL:
1508 		if (intr_type == FMAN_INTR_TYPE_ERR)
1509 			event = FMAN_EV_CNT;
1510 		else
1511 			event = (FMAN_EV_FMAN_CTRL_0 + mod_id);
1512 		break;
1513 	case FMAN_MOD_DUMMY_LAST:
1514 		event = FMAN_EV_CNT;
1515 		break;
1516 	default:
1517 		event = FMAN_EV_CNT;
1518 		break;
1519 	}
1520 
1521 	return event;
1522 }
1523 
1524 static int set_size_of_fifo(struct fman *fman, u8 port_id, u32 *size_of_fifo,
1525 			    u32 *extra_size_of_fifo)
1526 {
1527 	struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1528 	u32 fifo = *size_of_fifo;
1529 	u32 extra_fifo = *extra_size_of_fifo;
1530 	u32 tmp;
1531 
1532 	/* if this is the first time a port requires extra_fifo_pool_size,
1533 	 * the total extra_fifo_pool_size must be initialized to 1 buffer per
1534 	 * port
1535 	 */
1536 	if (extra_fifo && !fman->state->extra_fifo_pool_size)
1537 		fman->state->extra_fifo_pool_size =
1538 			fman->state->num_of_rx_ports * FMAN_BMI_FIFO_UNITS;
1539 
1540 	fman->state->extra_fifo_pool_size =
1541 		max(fman->state->extra_fifo_pool_size, extra_fifo);
1542 
1543 	/* check that there are enough uncommitted fifo size */
1544 	if ((fman->state->accumulated_fifo_size + fifo) >
1545 	    (fman->state->total_fifo_size -
1546 	    fman->state->extra_fifo_pool_size)) {
1547 		dev_err(fman->dev, "%s: Requested fifo size and extra size exceed total FIFO size.\n",
1548 			__func__);
1549 		return -EAGAIN;
1550 	}
1551 
1552 	/* Read, modify and write to HW */
1553 	tmp = (fifo / FMAN_BMI_FIFO_UNITS - 1) |
1554 	       ((extra_fifo / FMAN_BMI_FIFO_UNITS) <<
1555 	       BMI_EXTRA_FIFO_SIZE_SHIFT);
1556 	iowrite32be(tmp, &bmi_rg->fmbm_pfs[port_id - 1]);
1557 
1558 	/* update accumulated */
1559 	fman->state->accumulated_fifo_size += fifo;
1560 
1561 	return 0;
1562 }
1563 
1564 static int set_num_of_tasks(struct fman *fman, u8 port_id, u8 *num_of_tasks,
1565 			    u8 *num_of_extra_tasks)
1566 {
1567 	struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1568 	u8 tasks = *num_of_tasks;
1569 	u8 extra_tasks = *num_of_extra_tasks;
1570 	u32 tmp;
1571 
1572 	if (extra_tasks)
1573 		fman->state->extra_tasks_pool_size =
1574 		max(fman->state->extra_tasks_pool_size, extra_tasks);
1575 
1576 	/* check that there are enough uncommitted tasks */
1577 	if ((fman->state->accumulated_num_of_tasks + tasks) >
1578 	    (fman->state->total_num_of_tasks -
1579 	     fman->state->extra_tasks_pool_size)) {
1580 		dev_err(fman->dev, "%s: Requested num_of_tasks and extra tasks pool for fm%d exceed total num_of_tasks.\n",
1581 			__func__, fman->state->fm_id);
1582 		return -EAGAIN;
1583 	}
1584 	/* update accumulated */
1585 	fman->state->accumulated_num_of_tasks += tasks;
1586 
1587 	/* Write to HW */
1588 	tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]) &
1589 	    ~(BMI_NUM_OF_TASKS_MASK | BMI_NUM_OF_EXTRA_TASKS_MASK);
1590 	tmp |= ((u32)((tasks - 1) << BMI_NUM_OF_TASKS_SHIFT) |
1591 		(u32)(extra_tasks << BMI_EXTRA_NUM_OF_TASKS_SHIFT));
1592 	iowrite32be(tmp, &bmi_rg->fmbm_pp[port_id - 1]);
1593 
1594 	return 0;
1595 }
1596 
1597 static int set_num_of_open_dmas(struct fman *fman, u8 port_id,
1598 				u8 *num_of_open_dmas,
1599 				u8 *num_of_extra_open_dmas)
1600 {
1601 	struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1602 	u8 open_dmas = *num_of_open_dmas;
1603 	u8 extra_open_dmas = *num_of_extra_open_dmas;
1604 	u8 total_num_dmas = 0, current_val = 0, current_extra_val = 0;
1605 	u32 tmp;
1606 
1607 	if (!open_dmas) {
1608 		/* Configuration according to values in the HW.
1609 		 * read the current number of open Dma's
1610 		 */
1611 		tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]);
1612 		current_extra_val = (u8)((tmp & BMI_NUM_OF_EXTRA_DMAS_MASK) >>
1613 					 BMI_EXTRA_NUM_OF_DMAS_SHIFT);
1614 
1615 		tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]);
1616 		current_val = (u8)(((tmp & BMI_NUM_OF_DMAS_MASK) >>
1617 				   BMI_NUM_OF_DMAS_SHIFT) + 1);
1618 
1619 		/* This is the first configuration and user did not
1620 		 * specify value (!open_dmas), reset values will be used
1621 		 * and we just save these values for resource management
1622 		 */
1623 		fman->state->extra_open_dmas_pool_size =
1624 			(u8)max(fman->state->extra_open_dmas_pool_size,
1625 				current_extra_val);
1626 		fman->state->accumulated_num_of_open_dmas += current_val;
1627 		*num_of_open_dmas = current_val;
1628 		*num_of_extra_open_dmas = current_extra_val;
1629 		return 0;
1630 	}
1631 
1632 	if (extra_open_dmas > current_extra_val)
1633 		fman->state->extra_open_dmas_pool_size =
1634 		    (u8)max(fman->state->extra_open_dmas_pool_size,
1635 			    extra_open_dmas);
1636 
1637 	if ((fman->state->rev_info.major < 6) &&
1638 	    (fman->state->accumulated_num_of_open_dmas - current_val +
1639 	     open_dmas > fman->state->max_num_of_open_dmas)) {
1640 		dev_err(fman->dev, "%s: Requested num_of_open_dmas for fm%d exceeds total num_of_open_dmas.\n",
1641 			__func__, fman->state->fm_id);
1642 		return -EAGAIN;
1643 	} else if ((fman->state->rev_info.major >= 6) &&
1644 		   !((fman->state->rev_info.major == 6) &&
1645 		   (fman->state->rev_info.minor == 0)) &&
1646 		   (fman->state->accumulated_num_of_open_dmas -
1647 		   current_val + open_dmas >
1648 		   fman->state->dma_thresh_max_commq + 1)) {
1649 		dev_err(fman->dev, "%s: Requested num_of_open_dmas for fm%d exceeds DMA Command queue (%d)\n",
1650 			__func__, fman->state->fm_id,
1651 		       fman->state->dma_thresh_max_commq + 1);
1652 		return -EAGAIN;
1653 	}
1654 
1655 	WARN_ON(fman->state->accumulated_num_of_open_dmas < current_val);
1656 	/* update acummulated */
1657 	fman->state->accumulated_num_of_open_dmas -= current_val;
1658 	fman->state->accumulated_num_of_open_dmas += open_dmas;
1659 
1660 	if (fman->state->rev_info.major < 6)
1661 		total_num_dmas =
1662 		    (u8)(fman->state->accumulated_num_of_open_dmas +
1663 		    fman->state->extra_open_dmas_pool_size);
1664 
1665 	/* calculate reg */
1666 	tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]) &
1667 	    ~(BMI_NUM_OF_DMAS_MASK | BMI_NUM_OF_EXTRA_DMAS_MASK);
1668 	tmp |= (u32)(((open_dmas - 1) << BMI_NUM_OF_DMAS_SHIFT) |
1669 			   (extra_open_dmas << BMI_EXTRA_NUM_OF_DMAS_SHIFT));
1670 	iowrite32be(tmp, &bmi_rg->fmbm_pp[port_id - 1]);
1671 
1672 	/* update total num of DMA's with committed number of open DMAS,
1673 	 * and max uncommitted pool.
1674 	 */
1675 	if (total_num_dmas) {
1676 		tmp = ioread32be(&bmi_rg->fmbm_cfg2) & ~BMI_CFG2_DMAS_MASK;
1677 		tmp |= (u32)(total_num_dmas - 1) << BMI_CFG2_DMAS_SHIFT;
1678 		iowrite32be(tmp, &bmi_rg->fmbm_cfg2);
1679 	}
1680 
1681 	return 0;
1682 }
1683 
1684 static int fman_config(struct fman *fman)
1685 {
1686 	void __iomem *base_addr;
1687 	int err;
1688 
1689 	base_addr = fman->dts_params.base_addr;
1690 
1691 	fman->state = kzalloc(sizeof(*fman->state), GFP_KERNEL);
1692 	if (!fman->state)
1693 		goto err_fm_state;
1694 
1695 	/* Allocate the FM driver's parameters structure */
1696 	fman->cfg = kzalloc(sizeof(*fman->cfg), GFP_KERNEL);
1697 	if (!fman->cfg)
1698 		goto err_fm_drv;
1699 
1700 	/* Initialize MURAM block */
1701 	fman->muram =
1702 		fman_muram_init(fman->dts_params.muram_res.start,
1703 				resource_size(&fman->dts_params.muram_res));
1704 	if (!fman->muram)
1705 		goto err_fm_soc_specific;
1706 
1707 	/* Initialize FM parameters which will be kept by the driver */
1708 	fman->state->fm_id = fman->dts_params.id;
1709 	fman->state->fm_clk_freq = fman->dts_params.clk_freq;
1710 	fman->state->qman_channel_base = fman->dts_params.qman_channel_base;
1711 	fman->state->num_of_qman_channels =
1712 		fman->dts_params.num_of_qman_channels;
1713 	fman->state->res = fman->dts_params.res;
1714 	fman->exception_cb = fman_exceptions;
1715 	fman->bus_error_cb = fman_bus_error;
1716 	fman->fpm_regs = base_addr + FPM_OFFSET;
1717 	fman->bmi_regs = base_addr + BMI_OFFSET;
1718 	fman->qmi_regs = base_addr + QMI_OFFSET;
1719 	fman->dma_regs = base_addr + DMA_OFFSET;
1720 	fman->hwp_regs = base_addr + HWP_OFFSET;
1721 	fman->kg_regs = base_addr + KG_OFFSET;
1722 	fman->base_addr = base_addr;
1723 
1724 	spin_lock_init(&fman->spinlock);
1725 	fman_defconfig(fman->cfg);
1726 
1727 	fman->state->extra_fifo_pool_size = 0;
1728 	fman->state->exceptions = (EX_DMA_BUS_ERROR                 |
1729 					EX_DMA_READ_ECC              |
1730 					EX_DMA_SYSTEM_WRITE_ECC      |
1731 					EX_DMA_FM_WRITE_ECC          |
1732 					EX_FPM_STALL_ON_TASKS        |
1733 					EX_FPM_SINGLE_ECC            |
1734 					EX_FPM_DOUBLE_ECC            |
1735 					EX_QMI_DEQ_FROM_UNKNOWN_PORTID |
1736 					EX_BMI_LIST_RAM_ECC          |
1737 					EX_BMI_STORAGE_PROFILE_ECC   |
1738 					EX_BMI_STATISTICS_RAM_ECC    |
1739 					EX_MURAM_ECC                 |
1740 					EX_BMI_DISPATCH_RAM_ECC      |
1741 					EX_QMI_DOUBLE_ECC            |
1742 					EX_QMI_SINGLE_ECC);
1743 
1744 	/* Read FMan revision for future use*/
1745 	fman_get_revision(fman, &fman->state->rev_info);
1746 
1747 	err = fill_soc_specific_params(fman->state);
1748 	if (err)
1749 		goto err_fm_soc_specific;
1750 
1751 	/* FM_AID_MODE_NO_TNUM_SW005 Errata workaround */
1752 	if (fman->state->rev_info.major >= 6)
1753 		fman->cfg->dma_aid_mode = FMAN_DMA_AID_OUT_PORT_ID;
1754 
1755 	fman->cfg->qmi_def_tnums_thresh = fman->state->qmi_def_tnums_thresh;
1756 
1757 	fman->state->total_num_of_tasks =
1758 	(u8)DFLT_TOTAL_NUM_OF_TASKS(fman->state->rev_info.major,
1759 				    fman->state->rev_info.minor,
1760 				    fman->state->bmi_max_num_of_tasks);
1761 
1762 	if (fman->state->rev_info.major < 6) {
1763 		fman->cfg->dma_comm_qtsh_clr_emer =
1764 		(u8)DFLT_DMA_COMM_Q_LOW(fman->state->rev_info.major,
1765 					fman->state->dma_thresh_max_commq);
1766 
1767 		fman->cfg->dma_comm_qtsh_asrt_emer =
1768 		(u8)DFLT_DMA_COMM_Q_HIGH(fman->state->rev_info.major,
1769 					 fman->state->dma_thresh_max_commq);
1770 
1771 		fman->cfg->dma_cam_num_of_entries =
1772 		DFLT_DMA_CAM_NUM_OF_ENTRIES(fman->state->rev_info.major);
1773 
1774 		fman->cfg->dma_read_buf_tsh_clr_emer =
1775 		DFLT_DMA_READ_INT_BUF_LOW(fman->state->dma_thresh_max_buf);
1776 
1777 		fman->cfg->dma_read_buf_tsh_asrt_emer =
1778 		DFLT_DMA_READ_INT_BUF_HIGH(fman->state->dma_thresh_max_buf);
1779 
1780 		fman->cfg->dma_write_buf_tsh_clr_emer =
1781 		DFLT_DMA_WRITE_INT_BUF_LOW(fman->state->dma_thresh_max_buf);
1782 
1783 		fman->cfg->dma_write_buf_tsh_asrt_emer =
1784 		DFLT_DMA_WRITE_INT_BUF_HIGH(fman->state->dma_thresh_max_buf);
1785 
1786 		fman->cfg->dma_axi_dbg_num_of_beats =
1787 		DFLT_AXI_DBG_NUM_OF_BEATS;
1788 	}
1789 
1790 	return 0;
1791 
1792 err_fm_soc_specific:
1793 	kfree(fman->cfg);
1794 err_fm_drv:
1795 	kfree(fman->state);
1796 err_fm_state:
1797 	kfree(fman);
1798 	return -EINVAL;
1799 }
1800 
1801 static int fman_reset(struct fman *fman)
1802 {
1803 	u32 count;
1804 	int err = 0;
1805 
1806 	if (fman->state->rev_info.major < 6) {
1807 		iowrite32be(FPM_RSTC_FM_RESET, &fman->fpm_regs->fm_rstc);
1808 		/* Wait for reset completion */
1809 		count = 100;
1810 		do {
1811 			udelay(1);
1812 		} while (((ioread32be(&fman->fpm_regs->fm_rstc)) &
1813 			 FPM_RSTC_FM_RESET) && --count);
1814 		if (count == 0)
1815 			err = -EBUSY;
1816 
1817 		goto _return;
1818 	} else {
1819 #ifdef CONFIG_PPC
1820 		struct device_node *guts_node;
1821 		struct ccsr_guts __iomem *guts_regs;
1822 		u32 devdisr2, reg;
1823 
1824 		/* Errata A007273 */
1825 		guts_node =
1826 			of_find_compatible_node(NULL, NULL,
1827 						"fsl,qoriq-device-config-2.0");
1828 		if (!guts_node) {
1829 			dev_err(fman->dev, "%s: Couldn't find guts node\n",
1830 				__func__);
1831 			goto guts_node;
1832 		}
1833 
1834 		guts_regs = of_iomap(guts_node, 0);
1835 		if (!guts_regs) {
1836 			dev_err(fman->dev, "%s: Couldn't map %pOF regs\n",
1837 				__func__, guts_node);
1838 			goto guts_regs;
1839 		}
1840 #define FMAN1_ALL_MACS_MASK	0xFCC00000
1841 #define FMAN2_ALL_MACS_MASK	0x000FCC00
1842 		/* Read current state */
1843 		devdisr2 = ioread32be(&guts_regs->devdisr2);
1844 		if (fman->dts_params.id == 0)
1845 			reg = devdisr2 & ~FMAN1_ALL_MACS_MASK;
1846 		else
1847 			reg = devdisr2 & ~FMAN2_ALL_MACS_MASK;
1848 
1849 		/* Enable all MACs */
1850 		iowrite32be(reg, &guts_regs->devdisr2);
1851 #endif
1852 
1853 		/* Perform FMan reset */
1854 		iowrite32be(FPM_RSTC_FM_RESET, &fman->fpm_regs->fm_rstc);
1855 
1856 		/* Wait for reset completion */
1857 		count = 100;
1858 		do {
1859 			udelay(1);
1860 		} while (((ioread32be(&fman->fpm_regs->fm_rstc)) &
1861 			 FPM_RSTC_FM_RESET) && --count);
1862 		if (count == 0) {
1863 #ifdef CONFIG_PPC
1864 			iounmap(guts_regs);
1865 			of_node_put(guts_node);
1866 #endif
1867 			err = -EBUSY;
1868 			goto _return;
1869 		}
1870 #ifdef CONFIG_PPC
1871 
1872 		/* Restore devdisr2 value */
1873 		iowrite32be(devdisr2, &guts_regs->devdisr2);
1874 
1875 		iounmap(guts_regs);
1876 		of_node_put(guts_node);
1877 #endif
1878 
1879 		goto _return;
1880 
1881 #ifdef CONFIG_PPC
1882 guts_regs:
1883 		of_node_put(guts_node);
1884 guts_node:
1885 		dev_dbg(fman->dev, "%s: Didn't perform FManV3 reset due to Errata A007273!\n",
1886 			__func__);
1887 #endif
1888 	}
1889 _return:
1890 	return err;
1891 }
1892 
1893 static int fman_init(struct fman *fman)
1894 {
1895 	struct fman_cfg *cfg = NULL;
1896 	int err = 0, i, count;
1897 
1898 	if (is_init_done(fman->cfg))
1899 		return -EINVAL;
1900 
1901 	fman->state->count1_micro_bit = FM_TIMESTAMP_1_USEC_BIT;
1902 
1903 	cfg = fman->cfg;
1904 
1905 	/* clear revision-dependent non existing exception */
1906 	if (fman->state->rev_info.major < 6)
1907 		fman->state->exceptions &= ~FMAN_EX_BMI_DISPATCH_RAM_ECC;
1908 
1909 	if (fman->state->rev_info.major >= 6)
1910 		fman->state->exceptions &= ~FMAN_EX_QMI_SINGLE_ECC;
1911 
1912 	/* clear CPG */
1913 	memset_io((void __iomem *)(fman->base_addr + CGP_OFFSET), 0,
1914 		  fman->state->fm_port_num_of_cg);
1915 
1916 	/* Save LIODN info before FMan reset
1917 	 * Skipping non-existent port 0 (i = 1)
1918 	 */
1919 	for (i = 1; i < FMAN_LIODN_TBL; i++) {
1920 		u32 liodn_base;
1921 
1922 		fman->liodn_offset[i] =
1923 			ioread32be(&fman->bmi_regs->fmbm_spliodn[i - 1]);
1924 		if (!IS_ENABLED(CONFIG_FSL_PAMU))
1925 			continue;
1926 		liodn_base = ioread32be(&fman->dma_regs->fmdmplr[i / 2]);
1927 		if (i % 2) {
1928 			/* FMDM_PLR LSB holds LIODN base for odd ports */
1929 			liodn_base &= DMA_LIODN_BASE_MASK;
1930 		} else {
1931 			/* FMDM_PLR MSB holds LIODN base for even ports */
1932 			liodn_base >>= DMA_LIODN_SHIFT;
1933 			liodn_base &= DMA_LIODN_BASE_MASK;
1934 		}
1935 		fman->liodn_base[i] = liodn_base;
1936 	}
1937 
1938 	err = fman_reset(fman);
1939 	if (err)
1940 		return err;
1941 
1942 	if (ioread32be(&fman->qmi_regs->fmqm_gs) & QMI_GS_HALT_NOT_BUSY) {
1943 		resume(fman->fpm_regs);
1944 		/* Wait until QMI is not in halt not busy state */
1945 		count = 100;
1946 		do {
1947 			udelay(1);
1948 		} while (((ioread32be(&fman->qmi_regs->fmqm_gs)) &
1949 			 QMI_GS_HALT_NOT_BUSY) && --count);
1950 		if (count == 0)
1951 			dev_warn(fman->dev, "%s: QMI is in halt not busy state\n",
1952 				 __func__);
1953 	}
1954 
1955 	if (clear_iram(fman) != 0)
1956 		return -EINVAL;
1957 
1958 	cfg->exceptions = fman->state->exceptions;
1959 
1960 	/* Init DMA Registers */
1961 
1962 	err = dma_init(fman);
1963 	if (err != 0) {
1964 		free_init_resources(fman);
1965 		return err;
1966 	}
1967 
1968 	/* Init FPM Registers */
1969 	fpm_init(fman->fpm_regs, fman->cfg);
1970 
1971 	/* define common resources */
1972 	/* allocate MURAM for FIFO according to total size */
1973 	fman->fifo_offset = fman_muram_alloc(fman->muram,
1974 					     fman->state->total_fifo_size);
1975 	if (IS_ERR_VALUE(fman->fifo_offset)) {
1976 		free_init_resources(fman);
1977 		dev_err(fman->dev, "%s: MURAM alloc for BMI FIFO failed\n",
1978 			__func__);
1979 		return -ENOMEM;
1980 	}
1981 
1982 	cfg->fifo_base_addr = fman->fifo_offset;
1983 	cfg->total_fifo_size = fman->state->total_fifo_size;
1984 	cfg->total_num_of_tasks = fman->state->total_num_of_tasks;
1985 	cfg->clk_freq = fman->state->fm_clk_freq;
1986 
1987 	/* Init BMI Registers */
1988 	bmi_init(fman->bmi_regs, fman->cfg);
1989 
1990 	/* Init QMI Registers */
1991 	qmi_init(fman->qmi_regs, fman->cfg);
1992 
1993 	/* Init HW Parser */
1994 	hwp_init(fman->hwp_regs);
1995 
1996 	/* Init KeyGen */
1997 	fman->keygen = keygen_init(fman->kg_regs);
1998 	if (!fman->keygen)
1999 		return -EINVAL;
2000 
2001 	err = enable(fman, cfg);
2002 	if (err != 0)
2003 		return err;
2004 
2005 	enable_time_stamp(fman);
2006 
2007 	kfree(fman->cfg);
2008 	fman->cfg = NULL;
2009 
2010 	return 0;
2011 }
2012 
2013 static int fman_set_exception(struct fman *fman,
2014 			      enum fman_exceptions exception, bool enable)
2015 {
2016 	u32 bit_mask = 0;
2017 
2018 	if (!is_init_done(fman->cfg))
2019 		return -EINVAL;
2020 
2021 	bit_mask = get_exception_flag(exception);
2022 	if (bit_mask) {
2023 		if (enable)
2024 			fman->state->exceptions |= bit_mask;
2025 		else
2026 			fman->state->exceptions &= ~bit_mask;
2027 	} else {
2028 		dev_err(fman->dev, "%s: Undefined exception (%d)\n",
2029 			__func__, exception);
2030 		return -EINVAL;
2031 	}
2032 
2033 	return set_exception(fman, exception, enable);
2034 }
2035 
2036 /**
2037  * fman_register_intr
2038  * @fman:	A Pointer to FMan device
2039  * @module:	Calling module
2040  * @mod_id:	Module id (if more than 1 exists, '0' if not)
2041  * @intr_type:	Interrupt type (error/normal) selection.
2042  * @isr_cb:	The interrupt service routine.
2043  * @src_arg:	Argument to be passed to isr_cb.
2044  *
2045  * Used to register an event handler to be processed by FMan
2046  *
2047  * Return: 0 on success; Error code otherwise.
2048  */
2049 void fman_register_intr(struct fman *fman, enum fman_event_modules module,
2050 			u8 mod_id, enum fman_intr_type intr_type,
2051 			void (*isr_cb)(void *src_arg), void *src_arg)
2052 {
2053 	int event = 0;
2054 
2055 	event = get_module_event(module, mod_id, intr_type);
2056 	WARN_ON(event >= FMAN_EV_CNT);
2057 
2058 	/* register in local FM structure */
2059 	fman->intr_mng[event].isr_cb = isr_cb;
2060 	fman->intr_mng[event].src_handle = src_arg;
2061 }
2062 EXPORT_SYMBOL(fman_register_intr);
2063 
2064 /**
2065  * fman_unregister_intr
2066  * @fman:	A Pointer to FMan device
2067  * @module:	Calling module
2068  * @mod_id:	Module id (if more than 1 exists, '0' if not)
2069  * @intr_type:	Interrupt type (error/normal) selection.
2070  *
2071  * Used to unregister an event handler to be processed by FMan
2072  *
2073  * Return: 0 on success; Error code otherwise.
2074  */
2075 void fman_unregister_intr(struct fman *fman, enum fman_event_modules module,
2076 			  u8 mod_id, enum fman_intr_type intr_type)
2077 {
2078 	int event = 0;
2079 
2080 	event = get_module_event(module, mod_id, intr_type);
2081 	WARN_ON(event >= FMAN_EV_CNT);
2082 
2083 	fman->intr_mng[event].isr_cb = NULL;
2084 	fman->intr_mng[event].src_handle = NULL;
2085 }
2086 EXPORT_SYMBOL(fman_unregister_intr);
2087 
2088 /**
2089  * fman_set_port_params
2090  * @fman:		A Pointer to FMan device
2091  * @port_params:	Port parameters
2092  *
2093  * Used by FMan Port to pass parameters to the FMan
2094  *
2095  * Return: 0 on success; Error code otherwise.
2096  */
2097 int fman_set_port_params(struct fman *fman,
2098 			 struct fman_port_init_params *port_params)
2099 {
2100 	int err;
2101 	unsigned long flags;
2102 	u8 port_id = port_params->port_id, mac_id;
2103 
2104 	spin_lock_irqsave(&fman->spinlock, flags);
2105 
2106 	err = set_num_of_tasks(fman, port_params->port_id,
2107 			       &port_params->num_of_tasks,
2108 			       &port_params->num_of_extra_tasks);
2109 	if (err)
2110 		goto return_err;
2111 
2112 	/* TX Ports */
2113 	if (port_params->port_type != FMAN_PORT_TYPE_RX) {
2114 		u32 enq_th, deq_th, reg;
2115 
2116 		/* update qmi ENQ/DEQ threshold */
2117 		fman->state->accumulated_num_of_deq_tnums +=
2118 			port_params->deq_pipeline_depth;
2119 		enq_th = (ioread32be(&fman->qmi_regs->fmqm_gc) &
2120 			  QMI_CFG_ENQ_MASK) >> QMI_CFG_ENQ_SHIFT;
2121 		/* if enq_th is too big, we reduce it to the max value
2122 		 * that is still 0
2123 		 */
2124 		if (enq_th >= (fman->state->qmi_max_num_of_tnums -
2125 		    fman->state->accumulated_num_of_deq_tnums)) {
2126 			enq_th =
2127 			fman->state->qmi_max_num_of_tnums -
2128 			fman->state->accumulated_num_of_deq_tnums - 1;
2129 
2130 			reg = ioread32be(&fman->qmi_regs->fmqm_gc);
2131 			reg &= ~QMI_CFG_ENQ_MASK;
2132 			reg |= (enq_th << QMI_CFG_ENQ_SHIFT);
2133 			iowrite32be(reg, &fman->qmi_regs->fmqm_gc);
2134 		}
2135 
2136 		deq_th = ioread32be(&fman->qmi_regs->fmqm_gc) &
2137 				    QMI_CFG_DEQ_MASK;
2138 		/* if deq_th is too small, we enlarge it to the min
2139 		 * value that is still 0.
2140 		 * depTh may not be larger than 63
2141 		 * (fman->state->qmi_max_num_of_tnums-1).
2142 		 */
2143 		if ((deq_th <= fman->state->accumulated_num_of_deq_tnums) &&
2144 		    (deq_th < fman->state->qmi_max_num_of_tnums - 1)) {
2145 			deq_th = fman->state->accumulated_num_of_deq_tnums + 1;
2146 			reg = ioread32be(&fman->qmi_regs->fmqm_gc);
2147 			reg &= ~QMI_CFG_DEQ_MASK;
2148 			reg |= deq_th;
2149 			iowrite32be(reg, &fman->qmi_regs->fmqm_gc);
2150 		}
2151 	}
2152 
2153 	err = set_size_of_fifo(fman, port_params->port_id,
2154 			       &port_params->size_of_fifo,
2155 			       &port_params->extra_size_of_fifo);
2156 	if (err)
2157 		goto return_err;
2158 
2159 	err = set_num_of_open_dmas(fman, port_params->port_id,
2160 				   &port_params->num_of_open_dmas,
2161 				   &port_params->num_of_extra_open_dmas);
2162 	if (err)
2163 		goto return_err;
2164 
2165 	set_port_liodn(fman, port_id, fman->liodn_base[port_id],
2166 		       fman->liodn_offset[port_id]);
2167 
2168 	if (fman->state->rev_info.major < 6)
2169 		set_port_order_restoration(fman->fpm_regs, port_id);
2170 
2171 	mac_id = hw_port_id_to_sw_port_id(fman->state->rev_info.major, port_id);
2172 
2173 	if (port_params->max_frame_length >= fman->state->mac_mfl[mac_id]) {
2174 		fman->state->port_mfl[mac_id] = port_params->max_frame_length;
2175 	} else {
2176 		dev_warn(fman->dev, "%s: Port (%d) max_frame_length is smaller than MAC (%d) current MTU\n",
2177 			 __func__, port_id, mac_id);
2178 		err = -EINVAL;
2179 		goto return_err;
2180 	}
2181 
2182 	spin_unlock_irqrestore(&fman->spinlock, flags);
2183 
2184 	return 0;
2185 
2186 return_err:
2187 	spin_unlock_irqrestore(&fman->spinlock, flags);
2188 	return err;
2189 }
2190 EXPORT_SYMBOL(fman_set_port_params);
2191 
2192 /**
2193  * fman_reset_mac
2194  * @fman:	A Pointer to FMan device
2195  * @mac_id:	MAC id to be reset
2196  *
2197  * Reset a specific MAC
2198  *
2199  * Return: 0 on success; Error code otherwise.
2200  */
2201 int fman_reset_mac(struct fman *fman, u8 mac_id)
2202 {
2203 	struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
2204 	u32 msk, timeout = 100;
2205 
2206 	if (fman->state->rev_info.major >= 6) {
2207 		dev_err(fman->dev, "%s: FMan MAC reset no available for FMan V3!\n",
2208 			__func__);
2209 		return -EINVAL;
2210 	}
2211 
2212 	/* Get the relevant bit mask */
2213 	switch (mac_id) {
2214 	case 0:
2215 		msk = FPM_RSTC_MAC0_RESET;
2216 		break;
2217 	case 1:
2218 		msk = FPM_RSTC_MAC1_RESET;
2219 		break;
2220 	case 2:
2221 		msk = FPM_RSTC_MAC2_RESET;
2222 		break;
2223 	case 3:
2224 		msk = FPM_RSTC_MAC3_RESET;
2225 		break;
2226 	case 4:
2227 		msk = FPM_RSTC_MAC4_RESET;
2228 		break;
2229 	case 5:
2230 		msk = FPM_RSTC_MAC5_RESET;
2231 		break;
2232 	case 6:
2233 		msk = FPM_RSTC_MAC6_RESET;
2234 		break;
2235 	case 7:
2236 		msk = FPM_RSTC_MAC7_RESET;
2237 		break;
2238 	case 8:
2239 		msk = FPM_RSTC_MAC8_RESET;
2240 		break;
2241 	case 9:
2242 		msk = FPM_RSTC_MAC9_RESET;
2243 		break;
2244 	default:
2245 		dev_warn(fman->dev, "%s: Illegal MAC Id [%d]\n",
2246 			 __func__, mac_id);
2247 		return -EINVAL;
2248 	}
2249 
2250 	/* reset */
2251 	iowrite32be(msk, &fpm_rg->fm_rstc);
2252 	while ((ioread32be(&fpm_rg->fm_rstc) & msk) && --timeout)
2253 		udelay(10);
2254 
2255 	if (!timeout)
2256 		return -EIO;
2257 
2258 	return 0;
2259 }
2260 EXPORT_SYMBOL(fman_reset_mac);
2261 
2262 /**
2263  * fman_set_mac_max_frame
2264  * @fman:	A Pointer to FMan device
2265  * @mac_id:	MAC id
2266  * @mfl:	Maximum frame length
2267  *
2268  * Set maximum frame length of specific MAC in FMan driver
2269  *
2270  * Return: 0 on success; Error code otherwise.
2271  */
2272 int fman_set_mac_max_frame(struct fman *fman, u8 mac_id, u16 mfl)
2273 {
2274 	/* if port is already initialized, check that MaxFrameLength is smaller
2275 	 * or equal to the port's max
2276 	 */
2277 	if ((!fman->state->port_mfl[mac_id]) ||
2278 	    (mfl <= fman->state->port_mfl[mac_id])) {
2279 		fman->state->mac_mfl[mac_id] = mfl;
2280 	} else {
2281 		dev_warn(fman->dev, "%s: MAC max_frame_length is larger than Port max_frame_length\n",
2282 			 __func__);
2283 		return -EINVAL;
2284 	}
2285 	return 0;
2286 }
2287 EXPORT_SYMBOL(fman_set_mac_max_frame);
2288 
2289 /**
2290  * fman_get_clock_freq
2291  * @fman:	A Pointer to FMan device
2292  *
2293  * Get FMan clock frequency
2294  *
2295  * Return: FMan clock frequency
2296  */
2297 u16 fman_get_clock_freq(struct fman *fman)
2298 {
2299 	return fman->state->fm_clk_freq;
2300 }
2301 
2302 /**
2303  * fman_get_bmi_max_fifo_size
2304  * @fman:	A Pointer to FMan device
2305  *
2306  * Get FMan maximum FIFO size
2307  *
2308  * Return: FMan Maximum FIFO size
2309  */
2310 u32 fman_get_bmi_max_fifo_size(struct fman *fman)
2311 {
2312 	return fman->state->bmi_max_fifo_size;
2313 }
2314 EXPORT_SYMBOL(fman_get_bmi_max_fifo_size);
2315 
2316 /**
2317  * fman_get_revision
2318  * @fman:		- Pointer to the FMan module
2319  * @rev_info:		- A structure of revision information parameters.
2320  *
2321  * Returns the FM revision
2322  *
2323  * Allowed only following fman_init().
2324  *
2325  * Return: 0 on success; Error code otherwise.
2326  */
2327 void fman_get_revision(struct fman *fman, struct fman_rev_info *rev_info)
2328 {
2329 	u32 tmp;
2330 
2331 	tmp = ioread32be(&fman->fpm_regs->fm_ip_rev_1);
2332 	rev_info->major = (u8)((tmp & FPM_REV1_MAJOR_MASK) >>
2333 				FPM_REV1_MAJOR_SHIFT);
2334 	rev_info->minor = tmp & FPM_REV1_MINOR_MASK;
2335 }
2336 EXPORT_SYMBOL(fman_get_revision);
2337 
2338 /**
2339  * fman_get_qman_channel_id
2340  * @fman:	A Pointer to FMan device
2341  * @port_id:	Port id
2342  *
2343  * Get QMan channel ID associated to the Port id
2344  *
2345  * Return: QMan channel ID
2346  */
2347 u32 fman_get_qman_channel_id(struct fman *fman, u32 port_id)
2348 {
2349 	int i;
2350 
2351 	if (fman->state->rev_info.major >= 6) {
2352 		static const u32 port_ids[] = {
2353 			0x30, 0x31, 0x28, 0x29, 0x2a, 0x2b,
2354 			0x2c, 0x2d, 0x2, 0x3, 0x4, 0x5, 0x7, 0x7
2355 		};
2356 
2357 		for (i = 0; i < fman->state->num_of_qman_channels; i++) {
2358 			if (port_ids[i] == port_id)
2359 				break;
2360 		}
2361 	} else {
2362 		static const u32 port_ids[] = {
2363 			0x30, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x1,
2364 			0x2, 0x3, 0x4, 0x5, 0x7, 0x7
2365 		};
2366 
2367 		for (i = 0; i < fman->state->num_of_qman_channels; i++) {
2368 			if (port_ids[i] == port_id)
2369 				break;
2370 		}
2371 	}
2372 
2373 	if (i == fman->state->num_of_qman_channels)
2374 		return 0;
2375 
2376 	return fman->state->qman_channel_base + i;
2377 }
2378 EXPORT_SYMBOL(fman_get_qman_channel_id);
2379 
2380 /**
2381  * fman_get_mem_region
2382  * @fman:	A Pointer to FMan device
2383  *
2384  * Get FMan memory region
2385  *
2386  * Return: A structure with FMan memory region information
2387  */
2388 struct resource *fman_get_mem_region(struct fman *fman)
2389 {
2390 	return fman->state->res;
2391 }
2392 EXPORT_SYMBOL(fman_get_mem_region);
2393 
2394 /* Bootargs defines */
2395 /* Extra headroom for RX buffers - Default, min and max */
2396 #define FSL_FM_RX_EXTRA_HEADROOM	64
2397 #define FSL_FM_RX_EXTRA_HEADROOM_MIN	16
2398 #define FSL_FM_RX_EXTRA_HEADROOM_MAX	384
2399 
2400 /* Maximum frame length */
2401 #define FSL_FM_MAX_FRAME_SIZE			1522
2402 #define FSL_FM_MAX_POSSIBLE_FRAME_SIZE		9600
2403 #define FSL_FM_MIN_POSSIBLE_FRAME_SIZE		64
2404 
2405 /* Extra headroom for Rx buffers.
2406  * FMan is instructed to allocate, on the Rx path, this amount of
2407  * space at the beginning of a data buffer, beside the DPA private
2408  * data area and the IC fields.
2409  * Does not impact Tx buffer layout.
2410  * Configurable from bootargs. 64 by default, it's needed on
2411  * particular forwarding scenarios that add extra headers to the
2412  * forwarded frame.
2413  */
2414 static int fsl_fm_rx_extra_headroom = FSL_FM_RX_EXTRA_HEADROOM;
2415 module_param(fsl_fm_rx_extra_headroom, int, 0);
2416 MODULE_PARM_DESC(fsl_fm_rx_extra_headroom, "Extra headroom for Rx buffers");
2417 
2418 /* Max frame size, across all interfaces.
2419  * Configurable from bootargs, to avoid allocating oversized (socket)
2420  * buffers when not using jumbo frames.
2421  * Must be large enough to accommodate the network MTU, but small enough
2422  * to avoid wasting skb memory.
2423  */
2424 static int fsl_fm_max_frm = FSL_FM_MAX_FRAME_SIZE;
2425 module_param(fsl_fm_max_frm, int, 0);
2426 MODULE_PARM_DESC(fsl_fm_max_frm, "Maximum frame size, across all interfaces");
2427 
2428 /**
2429  * fman_get_max_frm
2430  *
2431  * Return: Max frame length configured in the FM driver
2432  */
2433 u16 fman_get_max_frm(void)
2434 {
2435 	static bool fm_check_mfl;
2436 
2437 	if (!fm_check_mfl) {
2438 		if (fsl_fm_max_frm > FSL_FM_MAX_POSSIBLE_FRAME_SIZE ||
2439 		    fsl_fm_max_frm < FSL_FM_MIN_POSSIBLE_FRAME_SIZE) {
2440 			pr_warn("Invalid fsl_fm_max_frm value (%d) in bootargs, valid range is %d-%d. Falling back to the default (%d)\n",
2441 				fsl_fm_max_frm,
2442 				FSL_FM_MIN_POSSIBLE_FRAME_SIZE,
2443 				FSL_FM_MAX_POSSIBLE_FRAME_SIZE,
2444 				FSL_FM_MAX_FRAME_SIZE);
2445 			fsl_fm_max_frm = FSL_FM_MAX_FRAME_SIZE;
2446 		}
2447 		fm_check_mfl = true;
2448 	}
2449 
2450 	return fsl_fm_max_frm;
2451 }
2452 EXPORT_SYMBOL(fman_get_max_frm);
2453 
2454 /**
2455  * fman_get_rx_extra_headroom
2456  *
2457  * Return: Extra headroom size configured in the FM driver
2458  */
2459 int fman_get_rx_extra_headroom(void)
2460 {
2461 	static bool fm_check_rx_extra_headroom;
2462 
2463 	if (!fm_check_rx_extra_headroom) {
2464 		if (fsl_fm_rx_extra_headroom > FSL_FM_RX_EXTRA_HEADROOM_MAX ||
2465 		    fsl_fm_rx_extra_headroom < FSL_FM_RX_EXTRA_HEADROOM_MIN) {
2466 			pr_warn("Invalid fsl_fm_rx_extra_headroom value (%d) in bootargs, valid range is %d-%d. Falling back to the default (%d)\n",
2467 				fsl_fm_rx_extra_headroom,
2468 				FSL_FM_RX_EXTRA_HEADROOM_MIN,
2469 				FSL_FM_RX_EXTRA_HEADROOM_MAX,
2470 				FSL_FM_RX_EXTRA_HEADROOM);
2471 			fsl_fm_rx_extra_headroom = FSL_FM_RX_EXTRA_HEADROOM;
2472 		}
2473 
2474 		fm_check_rx_extra_headroom = true;
2475 		fsl_fm_rx_extra_headroom = ALIGN(fsl_fm_rx_extra_headroom, 16);
2476 	}
2477 
2478 	return fsl_fm_rx_extra_headroom;
2479 }
2480 EXPORT_SYMBOL(fman_get_rx_extra_headroom);
2481 
2482 /**
2483  * fman_bind
2484  * @fm_dev:	FMan OF device pointer
2485  *
2486  * Bind to a specific FMan device.
2487  *
2488  * Allowed only after the port was created.
2489  *
2490  * Return: A pointer to the FMan device
2491  */
2492 struct fman *fman_bind(struct device *fm_dev)
2493 {
2494 	return (struct fman *)(dev_get_drvdata(get_device(fm_dev)));
2495 }
2496 EXPORT_SYMBOL(fman_bind);
2497 
2498 #ifdef CONFIG_DPAA_ERRATUM_A050385
2499 bool fman_has_errata_a050385(void)
2500 {
2501 	return fman_has_err_a050385;
2502 }
2503 EXPORT_SYMBOL(fman_has_errata_a050385);
2504 #endif
2505 
2506 static irqreturn_t fman_err_irq(int irq, void *handle)
2507 {
2508 	struct fman *fman = (struct fman *)handle;
2509 	u32 pending;
2510 	struct fman_fpm_regs __iomem *fpm_rg;
2511 	irqreturn_t single_ret, ret = IRQ_NONE;
2512 
2513 	if (!is_init_done(fman->cfg))
2514 		return IRQ_NONE;
2515 
2516 	fpm_rg = fman->fpm_regs;
2517 
2518 	/* error interrupts */
2519 	pending = ioread32be(&fpm_rg->fm_epi);
2520 	if (!pending)
2521 		return IRQ_NONE;
2522 
2523 	if (pending & ERR_INTR_EN_BMI) {
2524 		single_ret = bmi_err_event(fman);
2525 		if (single_ret == IRQ_HANDLED)
2526 			ret = IRQ_HANDLED;
2527 	}
2528 	if (pending & ERR_INTR_EN_QMI) {
2529 		single_ret = qmi_err_event(fman);
2530 		if (single_ret == IRQ_HANDLED)
2531 			ret = IRQ_HANDLED;
2532 	}
2533 	if (pending & ERR_INTR_EN_FPM) {
2534 		single_ret = fpm_err_event(fman);
2535 		if (single_ret == IRQ_HANDLED)
2536 			ret = IRQ_HANDLED;
2537 	}
2538 	if (pending & ERR_INTR_EN_DMA) {
2539 		single_ret = dma_err_event(fman);
2540 		if (single_ret == IRQ_HANDLED)
2541 			ret = IRQ_HANDLED;
2542 	}
2543 	if (pending & ERR_INTR_EN_MURAM) {
2544 		single_ret = muram_err_intr(fman);
2545 		if (single_ret == IRQ_HANDLED)
2546 			ret = IRQ_HANDLED;
2547 	}
2548 
2549 	/* MAC error interrupts */
2550 	if (pending & ERR_INTR_EN_MAC0) {
2551 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 0);
2552 		if (single_ret == IRQ_HANDLED)
2553 			ret = IRQ_HANDLED;
2554 	}
2555 	if (pending & ERR_INTR_EN_MAC1) {
2556 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 1);
2557 		if (single_ret == IRQ_HANDLED)
2558 			ret = IRQ_HANDLED;
2559 	}
2560 	if (pending & ERR_INTR_EN_MAC2) {
2561 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 2);
2562 		if (single_ret == IRQ_HANDLED)
2563 			ret = IRQ_HANDLED;
2564 	}
2565 	if (pending & ERR_INTR_EN_MAC3) {
2566 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 3);
2567 		if (single_ret == IRQ_HANDLED)
2568 			ret = IRQ_HANDLED;
2569 	}
2570 	if (pending & ERR_INTR_EN_MAC4) {
2571 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 4);
2572 		if (single_ret == IRQ_HANDLED)
2573 			ret = IRQ_HANDLED;
2574 	}
2575 	if (pending & ERR_INTR_EN_MAC5) {
2576 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 5);
2577 		if (single_ret == IRQ_HANDLED)
2578 			ret = IRQ_HANDLED;
2579 	}
2580 	if (pending & ERR_INTR_EN_MAC6) {
2581 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 6);
2582 		if (single_ret == IRQ_HANDLED)
2583 			ret = IRQ_HANDLED;
2584 	}
2585 	if (pending & ERR_INTR_EN_MAC7) {
2586 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 7);
2587 		if (single_ret == IRQ_HANDLED)
2588 			ret = IRQ_HANDLED;
2589 	}
2590 	if (pending & ERR_INTR_EN_MAC8) {
2591 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 8);
2592 		if (single_ret == IRQ_HANDLED)
2593 			ret = IRQ_HANDLED;
2594 	}
2595 	if (pending & ERR_INTR_EN_MAC9) {
2596 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 9);
2597 		if (single_ret == IRQ_HANDLED)
2598 			ret = IRQ_HANDLED;
2599 	}
2600 
2601 	return ret;
2602 }
2603 
2604 static irqreturn_t fman_irq(int irq, void *handle)
2605 {
2606 	struct fman *fman = (struct fman *)handle;
2607 	u32 pending;
2608 	struct fman_fpm_regs __iomem *fpm_rg;
2609 	irqreturn_t single_ret, ret = IRQ_NONE;
2610 
2611 	if (!is_init_done(fman->cfg))
2612 		return IRQ_NONE;
2613 
2614 	fpm_rg = fman->fpm_regs;
2615 
2616 	/* normal interrupts */
2617 	pending = ioread32be(&fpm_rg->fm_npi);
2618 	if (!pending)
2619 		return IRQ_NONE;
2620 
2621 	if (pending & INTR_EN_QMI) {
2622 		single_ret = qmi_event(fman);
2623 		if (single_ret == IRQ_HANDLED)
2624 			ret = IRQ_HANDLED;
2625 	}
2626 
2627 	/* MAC interrupts */
2628 	if (pending & INTR_EN_MAC0) {
2629 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 0);
2630 		if (single_ret == IRQ_HANDLED)
2631 			ret = IRQ_HANDLED;
2632 	}
2633 	if (pending & INTR_EN_MAC1) {
2634 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 1);
2635 		if (single_ret == IRQ_HANDLED)
2636 			ret = IRQ_HANDLED;
2637 	}
2638 	if (pending & INTR_EN_MAC2) {
2639 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 2);
2640 		if (single_ret == IRQ_HANDLED)
2641 			ret = IRQ_HANDLED;
2642 	}
2643 	if (pending & INTR_EN_MAC3) {
2644 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 3);
2645 		if (single_ret == IRQ_HANDLED)
2646 			ret = IRQ_HANDLED;
2647 	}
2648 	if (pending & INTR_EN_MAC4) {
2649 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 4);
2650 		if (single_ret == IRQ_HANDLED)
2651 			ret = IRQ_HANDLED;
2652 	}
2653 	if (pending & INTR_EN_MAC5) {
2654 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 5);
2655 		if (single_ret == IRQ_HANDLED)
2656 			ret = IRQ_HANDLED;
2657 	}
2658 	if (pending & INTR_EN_MAC6) {
2659 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 6);
2660 		if (single_ret == IRQ_HANDLED)
2661 			ret = IRQ_HANDLED;
2662 	}
2663 	if (pending & INTR_EN_MAC7) {
2664 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 7);
2665 		if (single_ret == IRQ_HANDLED)
2666 			ret = IRQ_HANDLED;
2667 	}
2668 	if (pending & INTR_EN_MAC8) {
2669 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 8);
2670 		if (single_ret == IRQ_HANDLED)
2671 			ret = IRQ_HANDLED;
2672 	}
2673 	if (pending & INTR_EN_MAC9) {
2674 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 9);
2675 		if (single_ret == IRQ_HANDLED)
2676 			ret = IRQ_HANDLED;
2677 	}
2678 
2679 	return ret;
2680 }
2681 
2682 static const struct of_device_id fman_muram_match[] = {
2683 	{
2684 		.compatible = "fsl,fman-muram"},
2685 	{}
2686 };
2687 MODULE_DEVICE_TABLE(of, fman_muram_match);
2688 
2689 static struct fman *read_dts_node(struct platform_device *of_dev)
2690 {
2691 	struct fman *fman;
2692 	struct device_node *fm_node, *muram_node;
2693 	struct resource *res;
2694 	u32 val, range[2];
2695 	int err, irq;
2696 	struct clk *clk;
2697 	u32 clk_rate;
2698 	phys_addr_t phys_base_addr;
2699 	resource_size_t mem_size;
2700 
2701 	fman = kzalloc(sizeof(*fman), GFP_KERNEL);
2702 	if (!fman)
2703 		return ERR_PTR(-ENOMEM);
2704 
2705 	fm_node = of_node_get(of_dev->dev.of_node);
2706 
2707 	err = of_property_read_u32(fm_node, "cell-index", &val);
2708 	if (err) {
2709 		dev_err(&of_dev->dev, "%s: failed to read cell-index for %pOF\n",
2710 			__func__, fm_node);
2711 		goto fman_node_put;
2712 	}
2713 	fman->dts_params.id = (u8)val;
2714 
2715 	/* Get the FM interrupt */
2716 	err = platform_get_irq(of_dev, 0);
2717 	if (err < 0)
2718 		goto fman_node_put;
2719 	irq = err;
2720 
2721 	/* Get the FM error interrupt */
2722 	err = platform_get_irq(of_dev, 1);
2723 	if (err < 0)
2724 		goto fman_node_put;
2725 	fman->dts_params.err_irq = err;
2726 
2727 	/* Get the FM address */
2728 	res = platform_get_resource(of_dev, IORESOURCE_MEM, 0);
2729 	if (!res) {
2730 		err = -EINVAL;
2731 		dev_err(&of_dev->dev, "%s: Can't get FMan memory resource\n",
2732 			__func__);
2733 		goto fman_node_put;
2734 	}
2735 
2736 	phys_base_addr = res->start;
2737 	mem_size = resource_size(res);
2738 
2739 	clk = of_clk_get(fm_node, 0);
2740 	if (IS_ERR(clk)) {
2741 		err = PTR_ERR(clk);
2742 		dev_err(&of_dev->dev, "%s: Failed to get FM%d clock structure\n",
2743 			__func__, fman->dts_params.id);
2744 		goto fman_node_put;
2745 	}
2746 
2747 	clk_rate = clk_get_rate(clk);
2748 	if (!clk_rate) {
2749 		err = -EINVAL;
2750 		dev_err(&of_dev->dev, "%s: Failed to determine FM%d clock rate\n",
2751 			__func__, fman->dts_params.id);
2752 		goto fman_node_put;
2753 	}
2754 	/* Rounding to MHz */
2755 	fman->dts_params.clk_freq = DIV_ROUND_UP(clk_rate, 1000000);
2756 
2757 	err = of_property_read_u32_array(fm_node, "fsl,qman-channel-range",
2758 					 &range[0], 2);
2759 	if (err) {
2760 		dev_err(&of_dev->dev, "%s: failed to read fsl,qman-channel-range for %pOF\n",
2761 			__func__, fm_node);
2762 		goto fman_node_put;
2763 	}
2764 	fman->dts_params.qman_channel_base = range[0];
2765 	fman->dts_params.num_of_qman_channels = range[1];
2766 
2767 	/* Get the MURAM base address and size */
2768 	muram_node = of_find_matching_node(fm_node, fman_muram_match);
2769 	if (!muram_node) {
2770 		err = -EINVAL;
2771 		dev_err(&of_dev->dev, "%s: could not find MURAM node\n",
2772 			__func__);
2773 		goto fman_free;
2774 	}
2775 
2776 	err = of_address_to_resource(muram_node, 0,
2777 				     &fman->dts_params.muram_res);
2778 	if (err) {
2779 		of_node_put(muram_node);
2780 		dev_err(&of_dev->dev, "%s: of_address_to_resource() = %d\n",
2781 			__func__, err);
2782 		goto fman_free;
2783 	}
2784 
2785 	of_node_put(muram_node);
2786 
2787 	err = devm_request_irq(&of_dev->dev, irq, fman_irq, IRQF_SHARED,
2788 			       "fman", fman);
2789 	if (err < 0) {
2790 		dev_err(&of_dev->dev, "%s: irq %d allocation failed (error = %d)\n",
2791 			__func__, irq, err);
2792 		goto fman_free;
2793 	}
2794 
2795 	if (fman->dts_params.err_irq != 0) {
2796 		err = devm_request_irq(&of_dev->dev, fman->dts_params.err_irq,
2797 				       fman_err_irq, IRQF_SHARED,
2798 				       "fman-err", fman);
2799 		if (err < 0) {
2800 			dev_err(&of_dev->dev, "%s: irq %d allocation failed (error = %d)\n",
2801 				__func__, fman->dts_params.err_irq, err);
2802 			goto fman_free;
2803 		}
2804 	}
2805 
2806 	fman->dts_params.res =
2807 		devm_request_mem_region(&of_dev->dev, phys_base_addr,
2808 					mem_size, "fman");
2809 	if (!fman->dts_params.res) {
2810 		err = -EBUSY;
2811 		dev_err(&of_dev->dev, "%s: request_mem_region() failed\n",
2812 			__func__);
2813 		goto fman_free;
2814 	}
2815 
2816 	fman->dts_params.base_addr =
2817 		devm_ioremap(&of_dev->dev, phys_base_addr, mem_size);
2818 	if (!fman->dts_params.base_addr) {
2819 		err = -ENOMEM;
2820 		dev_err(&of_dev->dev, "%s: devm_ioremap() failed\n", __func__);
2821 		goto fman_free;
2822 	}
2823 
2824 	fman->dev = &of_dev->dev;
2825 
2826 	err = of_platform_populate(fm_node, NULL, NULL, &of_dev->dev);
2827 	if (err) {
2828 		dev_err(&of_dev->dev, "%s: of_platform_populate() failed\n",
2829 			__func__);
2830 		goto fman_free;
2831 	}
2832 
2833 #ifdef CONFIG_DPAA_ERRATUM_A050385
2834 	fman_has_err_a050385 =
2835 		of_property_read_bool(fm_node, "fsl,erratum-a050385");
2836 #endif
2837 
2838 	return fman;
2839 
2840 fman_node_put:
2841 	of_node_put(fm_node);
2842 fman_free:
2843 	kfree(fman);
2844 	return ERR_PTR(err);
2845 }
2846 
2847 static int fman_probe(struct platform_device *of_dev)
2848 {
2849 	struct fman *fman;
2850 	struct device *dev;
2851 	int err;
2852 
2853 	dev = &of_dev->dev;
2854 
2855 	fman = read_dts_node(of_dev);
2856 	if (IS_ERR(fman))
2857 		return PTR_ERR(fman);
2858 
2859 	err = fman_config(fman);
2860 	if (err) {
2861 		dev_err(dev, "%s: FMan config failed\n", __func__);
2862 		return -EINVAL;
2863 	}
2864 
2865 	if (fman_init(fman) != 0) {
2866 		dev_err(dev, "%s: FMan init failed\n", __func__);
2867 		return -EINVAL;
2868 	}
2869 
2870 	if (fman->dts_params.err_irq == 0) {
2871 		fman_set_exception(fman, FMAN_EX_DMA_BUS_ERROR, false);
2872 		fman_set_exception(fman, FMAN_EX_DMA_READ_ECC, false);
2873 		fman_set_exception(fman, FMAN_EX_DMA_SYSTEM_WRITE_ECC, false);
2874 		fman_set_exception(fman, FMAN_EX_DMA_FM_WRITE_ECC, false);
2875 		fman_set_exception(fman, FMAN_EX_DMA_SINGLE_PORT_ECC, false);
2876 		fman_set_exception(fman, FMAN_EX_FPM_STALL_ON_TASKS, false);
2877 		fman_set_exception(fman, FMAN_EX_FPM_SINGLE_ECC, false);
2878 		fman_set_exception(fman, FMAN_EX_FPM_DOUBLE_ECC, false);
2879 		fman_set_exception(fman, FMAN_EX_QMI_SINGLE_ECC, false);
2880 		fman_set_exception(fman, FMAN_EX_QMI_DOUBLE_ECC, false);
2881 		fman_set_exception(fman,
2882 				   FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID, false);
2883 		fman_set_exception(fman, FMAN_EX_BMI_LIST_RAM_ECC, false);
2884 		fman_set_exception(fman, FMAN_EX_BMI_STORAGE_PROFILE_ECC,
2885 				   false);
2886 		fman_set_exception(fman, FMAN_EX_BMI_STATISTICS_RAM_ECC, false);
2887 		fman_set_exception(fman, FMAN_EX_BMI_DISPATCH_RAM_ECC, false);
2888 	}
2889 
2890 	dev_set_drvdata(dev, fman);
2891 
2892 	dev_dbg(dev, "FMan%d probed\n", fman->dts_params.id);
2893 
2894 	return 0;
2895 }
2896 
2897 static const struct of_device_id fman_match[] = {
2898 	{
2899 		.compatible = "fsl,fman"},
2900 	{}
2901 };
2902 
2903 MODULE_DEVICE_TABLE(of, fman_match);
2904 
2905 static struct platform_driver fman_driver = {
2906 	.driver = {
2907 		.name = "fsl-fman",
2908 		.of_match_table = fman_match,
2909 	},
2910 	.probe = fman_probe,
2911 };
2912 
2913 static int __init fman_load(void)
2914 {
2915 	int err;
2916 
2917 	pr_debug("FSL DPAA FMan driver\n");
2918 
2919 	err = platform_driver_register(&fman_driver);
2920 	if (err < 0)
2921 		pr_err("Error, platform_driver_register() = %d\n", err);
2922 
2923 	return err;
2924 }
2925 module_init(fman_load);
2926 
2927 static void __exit fman_unload(void)
2928 {
2929 	platform_driver_unregister(&fman_driver);
2930 }
2931 module_exit(fman_unload);
2932 
2933 MODULE_LICENSE("Dual BSD/GPL");
2934 MODULE_DESCRIPTION("Freescale DPAA Frame Manager driver");
2935