1 /*
2  * Copyright 2008-2015 Freescale Semiconductor Inc.
3  * Copyright 2020 NXP
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  *     * Redistributions of source code must retain the above copyright
8  *       notice, this list of conditions and the following disclaimer.
9  *     * Redistributions in binary form must reproduce the above copyright
10  *       notice, this list of conditions and the following disclaimer in the
11  *       documentation and/or other materials provided with the distribution.
12  *     * Neither the name of Freescale Semiconductor nor the
13  *       names of its contributors may be used to endorse or promote products
14  *       derived from this software without specific prior written permission.
15  *
16  *
17  * ALTERNATIVELY, this software may be distributed under the terms of the
18  * GNU General Public License ("GPL") as published by the Free Software
19  * Foundation, either version 2 of that License or (at your option) any
20  * later version.
21  *
22  * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
23  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
24  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
25  * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
26  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
27  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
29  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
35 
36 #include <linux/fsl/guts.h>
37 #include <linux/slab.h>
38 #include <linux/delay.h>
39 #include <linux/module.h>
40 #include <linux/of_platform.h>
41 #include <linux/clk.h>
42 #include <linux/of_address.h>
43 #include <linux/of_irq.h>
44 #include <linux/interrupt.h>
45 #include <linux/libfdt_env.h>
46 
47 #include "fman.h"
48 #include "fman_muram.h"
49 #include "fman_keygen.h"
50 
51 /* General defines */
52 #define FMAN_LIODN_TBL			64	/* size of LIODN table */
53 #define MAX_NUM_OF_MACS			10
54 #define FM_NUM_OF_FMAN_CTRL_EVENT_REGS	4
55 #define BASE_RX_PORTID			0x08
56 #define BASE_TX_PORTID			0x28
57 
58 /* Modules registers offsets */
59 #define BMI_OFFSET		0x00080000
60 #define QMI_OFFSET		0x00080400
61 #define KG_OFFSET		0x000C1000
62 #define DMA_OFFSET		0x000C2000
63 #define FPM_OFFSET		0x000C3000
64 #define IMEM_OFFSET		0x000C4000
65 #define HWP_OFFSET		0x000C7000
66 #define CGP_OFFSET		0x000DB000
67 
68 /* Exceptions bit map */
69 #define EX_DMA_BUS_ERROR		0x80000000
70 #define EX_DMA_READ_ECC			0x40000000
71 #define EX_DMA_SYSTEM_WRITE_ECC	0x20000000
72 #define EX_DMA_FM_WRITE_ECC		0x10000000
73 #define EX_FPM_STALL_ON_TASKS		0x08000000
74 #define EX_FPM_SINGLE_ECC		0x04000000
75 #define EX_FPM_DOUBLE_ECC		0x02000000
76 #define EX_QMI_SINGLE_ECC		0x01000000
77 #define EX_QMI_DEQ_FROM_UNKNOWN_PORTID	0x00800000
78 #define EX_QMI_DOUBLE_ECC		0x00400000
79 #define EX_BMI_LIST_RAM_ECC		0x00200000
80 #define EX_BMI_STORAGE_PROFILE_ECC	0x00100000
81 #define EX_BMI_STATISTICS_RAM_ECC	0x00080000
82 #define EX_IRAM_ECC			0x00040000
83 #define EX_MURAM_ECC			0x00020000
84 #define EX_BMI_DISPATCH_RAM_ECC	0x00010000
85 #define EX_DMA_SINGLE_PORT_ECC		0x00008000
86 
87 /* DMA defines */
88 /* masks */
89 #define DMA_MODE_BER			0x00200000
90 #define DMA_MODE_ECC			0x00000020
91 #define DMA_MODE_SECURE_PROT		0x00000800
92 #define DMA_MODE_AXI_DBG_MASK		0x0F000000
93 
94 #define DMA_TRANSFER_PORTID_MASK	0xFF000000
95 #define DMA_TRANSFER_TNUM_MASK		0x00FF0000
96 #define DMA_TRANSFER_LIODN_MASK	0x00000FFF
97 
98 #define DMA_STATUS_BUS_ERR		0x08000000
99 #define DMA_STATUS_READ_ECC		0x04000000
100 #define DMA_STATUS_SYSTEM_WRITE_ECC	0x02000000
101 #define DMA_STATUS_FM_WRITE_ECC	0x01000000
102 #define DMA_STATUS_FM_SPDAT_ECC	0x00080000
103 
104 #define DMA_MODE_CACHE_OR_SHIFT		30
105 #define DMA_MODE_AXI_DBG_SHIFT			24
106 #define DMA_MODE_CEN_SHIFT			13
107 #define DMA_MODE_CEN_MASK			0x00000007
108 #define DMA_MODE_DBG_SHIFT			7
109 #define DMA_MODE_AID_MODE_SHIFT		4
110 
111 #define DMA_THRESH_COMMQ_SHIFT			24
112 #define DMA_THRESH_READ_INT_BUF_SHIFT		16
113 #define DMA_THRESH_READ_INT_BUF_MASK		0x0000003f
114 #define DMA_THRESH_WRITE_INT_BUF_MASK		0x0000003f
115 
116 #define DMA_TRANSFER_PORTID_SHIFT		24
117 #define DMA_TRANSFER_TNUM_SHIFT		16
118 
119 #define DMA_CAM_SIZEOF_ENTRY			0x40
120 #define DMA_CAM_UNITS				8
121 
122 #define DMA_LIODN_SHIFT		16
123 #define DMA_LIODN_BASE_MASK	0x00000FFF
124 
125 /* FPM defines */
126 #define FPM_EV_MASK_DOUBLE_ECC		0x80000000
127 #define FPM_EV_MASK_STALL		0x40000000
128 #define FPM_EV_MASK_SINGLE_ECC		0x20000000
129 #define FPM_EV_MASK_RELEASE_FM		0x00010000
130 #define FPM_EV_MASK_DOUBLE_ECC_EN	0x00008000
131 #define FPM_EV_MASK_STALL_EN		0x00004000
132 #define FPM_EV_MASK_SINGLE_ECC_EN	0x00002000
133 #define FPM_EV_MASK_EXTERNAL_HALT	0x00000008
134 #define FPM_EV_MASK_ECC_ERR_HALT	0x00000004
135 
136 #define FPM_RAM_MURAM_ECC		0x00008000
137 #define FPM_RAM_IRAM_ECC		0x00004000
138 #define FPM_IRAM_ECC_ERR_EX_EN		0x00020000
139 #define FPM_MURAM_ECC_ERR_EX_EN	0x00040000
140 #define FPM_RAM_IRAM_ECC_EN		0x40000000
141 #define FPM_RAM_RAMS_ECC_EN		0x80000000
142 #define FPM_RAM_RAMS_ECC_EN_SRC_SEL	0x08000000
143 
144 #define FPM_REV1_MAJOR_MASK		0x0000FF00
145 #define FPM_REV1_MINOR_MASK		0x000000FF
146 
147 #define FPM_DISP_LIMIT_SHIFT		24
148 
149 #define FPM_PRT_FM_CTL1			0x00000001
150 #define FPM_PRT_FM_CTL2			0x00000002
151 #define FPM_PORT_FM_CTL_PORTID_SHIFT	24
152 #define FPM_PRC_ORA_FM_CTL_SEL_SHIFT	16
153 
154 #define FPM_THR1_PRS_SHIFT		24
155 #define FPM_THR1_KG_SHIFT		16
156 #define FPM_THR1_PLCR_SHIFT		8
157 #define FPM_THR1_BMI_SHIFT		0
158 
159 #define FPM_THR2_QMI_ENQ_SHIFT		24
160 #define FPM_THR2_QMI_DEQ_SHIFT		0
161 #define FPM_THR2_FM_CTL1_SHIFT		16
162 #define FPM_THR2_FM_CTL2_SHIFT		8
163 
164 #define FPM_EV_MASK_CAT_ERR_SHIFT	1
165 #define FPM_EV_MASK_DMA_ERR_SHIFT	0
166 
167 #define FPM_REV1_MAJOR_SHIFT		8
168 
169 #define FPM_RSTC_FM_RESET		0x80000000
170 #define FPM_RSTC_MAC0_RESET		0x40000000
171 #define FPM_RSTC_MAC1_RESET		0x20000000
172 #define FPM_RSTC_MAC2_RESET		0x10000000
173 #define FPM_RSTC_MAC3_RESET		0x08000000
174 #define FPM_RSTC_MAC8_RESET		0x04000000
175 #define FPM_RSTC_MAC4_RESET		0x02000000
176 #define FPM_RSTC_MAC5_RESET		0x01000000
177 #define FPM_RSTC_MAC6_RESET		0x00800000
178 #define FPM_RSTC_MAC7_RESET		0x00400000
179 #define FPM_RSTC_MAC9_RESET		0x00200000
180 
181 #define FPM_TS_INT_SHIFT		16
182 #define FPM_TS_CTL_EN			0x80000000
183 
184 /* BMI defines */
185 #define BMI_INIT_START				0x80000000
186 #define BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC	0x80000000
187 #define BMI_ERR_INTR_EN_LIST_RAM_ECC		0x40000000
188 #define BMI_ERR_INTR_EN_STATISTICS_RAM_ECC	0x20000000
189 #define BMI_ERR_INTR_EN_DISPATCH_RAM_ECC	0x10000000
190 #define BMI_NUM_OF_TASKS_MASK			0x3F000000
191 #define BMI_NUM_OF_EXTRA_TASKS_MASK		0x000F0000
192 #define BMI_NUM_OF_DMAS_MASK			0x00000F00
193 #define BMI_NUM_OF_EXTRA_DMAS_MASK		0x0000000F
194 #define BMI_FIFO_SIZE_MASK			0x000003FF
195 #define BMI_EXTRA_FIFO_SIZE_MASK		0x03FF0000
196 #define BMI_CFG2_DMAS_MASK			0x0000003F
197 #define BMI_CFG2_TASKS_MASK			0x0000003F
198 
199 #define BMI_CFG2_TASKS_SHIFT		16
200 #define BMI_CFG2_DMAS_SHIFT		0
201 #define BMI_CFG1_FIFO_SIZE_SHIFT	16
202 #define BMI_NUM_OF_TASKS_SHIFT		24
203 #define BMI_EXTRA_NUM_OF_TASKS_SHIFT	16
204 #define BMI_NUM_OF_DMAS_SHIFT		8
205 #define BMI_EXTRA_NUM_OF_DMAS_SHIFT	0
206 
207 #define BMI_FIFO_ALIGN			0x100
208 
209 #define BMI_EXTRA_FIFO_SIZE_SHIFT	16
210 
211 /* QMI defines */
212 #define QMI_CFG_ENQ_EN			0x80000000
213 #define QMI_CFG_DEQ_EN			0x40000000
214 #define QMI_CFG_EN_COUNTERS		0x10000000
215 #define QMI_CFG_DEQ_MASK		0x0000003F
216 #define QMI_CFG_ENQ_MASK		0x00003F00
217 #define QMI_CFG_ENQ_SHIFT		8
218 
219 #define QMI_ERR_INTR_EN_DOUBLE_ECC	0x80000000
220 #define QMI_ERR_INTR_EN_DEQ_FROM_DEF	0x40000000
221 #define QMI_INTR_EN_SINGLE_ECC		0x80000000
222 
223 #define QMI_GS_HALT_NOT_BUSY		0x00000002
224 
225 /* HWP defines */
226 #define HWP_RPIMAC_PEN			0x00000001
227 
228 /* IRAM defines */
229 #define IRAM_IADD_AIE			0x80000000
230 #define IRAM_READY			0x80000000
231 
232 /* Default values */
233 #define DEFAULT_CATASTROPHIC_ERR		0
234 #define DEFAULT_DMA_ERR				0
235 #define DEFAULT_AID_MODE			FMAN_DMA_AID_OUT_TNUM
236 #define DEFAULT_DMA_COMM_Q_LOW			0x2A
237 #define DEFAULT_DMA_COMM_Q_HIGH		0x3F
238 #define DEFAULT_CACHE_OVERRIDE			0
239 #define DEFAULT_DMA_CAM_NUM_OF_ENTRIES		64
240 #define DEFAULT_DMA_DBG_CNT_MODE		0
241 #define DEFAULT_DMA_SOS_EMERGENCY		0
242 #define DEFAULT_DMA_WATCHDOG			0
243 #define DEFAULT_DISP_LIMIT			0
244 #define DEFAULT_PRS_DISP_TH			16
245 #define DEFAULT_PLCR_DISP_TH			16
246 #define DEFAULT_KG_DISP_TH			16
247 #define DEFAULT_BMI_DISP_TH			16
248 #define DEFAULT_QMI_ENQ_DISP_TH		16
249 #define DEFAULT_QMI_DEQ_DISP_TH		16
250 #define DEFAULT_FM_CTL1_DISP_TH		16
251 #define DEFAULT_FM_CTL2_DISP_TH		16
252 
253 #define DFLT_AXI_DBG_NUM_OF_BEATS		1
254 
255 #define DFLT_DMA_READ_INT_BUF_LOW(dma_thresh_max_buf)	\
256 	((dma_thresh_max_buf + 1) / 2)
257 #define DFLT_DMA_READ_INT_BUF_HIGH(dma_thresh_max_buf)	\
258 	((dma_thresh_max_buf + 1) * 3 / 4)
259 #define DFLT_DMA_WRITE_INT_BUF_LOW(dma_thresh_max_buf)	\
260 	((dma_thresh_max_buf + 1) / 2)
261 #define DFLT_DMA_WRITE_INT_BUF_HIGH(dma_thresh_max_buf)\
262 	((dma_thresh_max_buf + 1) * 3 / 4)
263 
264 #define DMA_COMM_Q_LOW_FMAN_V3		0x2A
265 #define DMA_COMM_Q_LOW_FMAN_V2(dma_thresh_max_commq)		\
266 	((dma_thresh_max_commq + 1) / 2)
267 #define DFLT_DMA_COMM_Q_LOW(major, dma_thresh_max_commq)	\
268 	((major == 6) ? DMA_COMM_Q_LOW_FMAN_V3 :		\
269 	DMA_COMM_Q_LOW_FMAN_V2(dma_thresh_max_commq))
270 
271 #define DMA_COMM_Q_HIGH_FMAN_V3	0x3f
272 #define DMA_COMM_Q_HIGH_FMAN_V2(dma_thresh_max_commq)		\
273 	((dma_thresh_max_commq + 1) * 3 / 4)
274 #define DFLT_DMA_COMM_Q_HIGH(major, dma_thresh_max_commq)	\
275 	((major == 6) ? DMA_COMM_Q_HIGH_FMAN_V3 :		\
276 	DMA_COMM_Q_HIGH_FMAN_V2(dma_thresh_max_commq))
277 
278 #define TOTAL_NUM_OF_TASKS_FMAN_V3L	59
279 #define TOTAL_NUM_OF_TASKS_FMAN_V3H	124
280 #define DFLT_TOTAL_NUM_OF_TASKS(major, minor, bmi_max_num_of_tasks)	\
281 	((major == 6) ? ((minor == 1 || minor == 4) ?			\
282 	TOTAL_NUM_OF_TASKS_FMAN_V3L : TOTAL_NUM_OF_TASKS_FMAN_V3H) :	\
283 	bmi_max_num_of_tasks)
284 
285 #define DMA_CAM_NUM_OF_ENTRIES_FMAN_V3		64
286 #define DMA_CAM_NUM_OF_ENTRIES_FMAN_V2		32
287 #define DFLT_DMA_CAM_NUM_OF_ENTRIES(major)			\
288 	(major == 6 ? DMA_CAM_NUM_OF_ENTRIES_FMAN_V3 :		\
289 	DMA_CAM_NUM_OF_ENTRIES_FMAN_V2)
290 
291 #define FM_TIMESTAMP_1_USEC_BIT             8
292 
293 /* Defines used for enabling/disabling FMan interrupts */
294 #define ERR_INTR_EN_DMA         0x00010000
295 #define ERR_INTR_EN_FPM         0x80000000
296 #define ERR_INTR_EN_BMI         0x00800000
297 #define ERR_INTR_EN_QMI         0x00400000
298 #define ERR_INTR_EN_MURAM       0x00040000
299 #define ERR_INTR_EN_MAC0        0x00004000
300 #define ERR_INTR_EN_MAC1        0x00002000
301 #define ERR_INTR_EN_MAC2        0x00001000
302 #define ERR_INTR_EN_MAC3        0x00000800
303 #define ERR_INTR_EN_MAC4        0x00000400
304 #define ERR_INTR_EN_MAC5        0x00000200
305 #define ERR_INTR_EN_MAC6        0x00000100
306 #define ERR_INTR_EN_MAC7        0x00000080
307 #define ERR_INTR_EN_MAC8        0x00008000
308 #define ERR_INTR_EN_MAC9        0x00000040
309 
310 #define INTR_EN_QMI             0x40000000
311 #define INTR_EN_MAC0            0x00080000
312 #define INTR_EN_MAC1            0x00040000
313 #define INTR_EN_MAC2            0x00020000
314 #define INTR_EN_MAC3            0x00010000
315 #define INTR_EN_MAC4            0x00000040
316 #define INTR_EN_MAC5            0x00000020
317 #define INTR_EN_MAC6            0x00000008
318 #define INTR_EN_MAC7            0x00000002
319 #define INTR_EN_MAC8            0x00200000
320 #define INTR_EN_MAC9            0x00100000
321 #define INTR_EN_REV0            0x00008000
322 #define INTR_EN_REV1            0x00004000
323 #define INTR_EN_REV2            0x00002000
324 #define INTR_EN_REV3            0x00001000
325 #define INTR_EN_TMR             0x01000000
326 
327 enum fman_dma_aid_mode {
328 	FMAN_DMA_AID_OUT_PORT_ID = 0,		  /* 4 LSB of PORT_ID */
329 	FMAN_DMA_AID_OUT_TNUM			  /* 4 LSB of TNUM */
330 };
331 
332 struct fman_iram_regs {
333 	u32 iadd;	/* FM IRAM instruction address register */
334 	u32 idata;	/* FM IRAM instruction data register */
335 	u32 itcfg;	/* FM IRAM timing config register */
336 	u32 iready;	/* FM IRAM ready register */
337 };
338 
339 struct fman_fpm_regs {
340 	u32 fmfp_tnc;		/* FPM TNUM Control 0x00 */
341 	u32 fmfp_prc;		/* FPM Port_ID FmCtl Association 0x04 */
342 	u32 fmfp_brkc;		/* FPM Breakpoint Control 0x08 */
343 	u32 fmfp_mxd;		/* FPM Flush Control 0x0c */
344 	u32 fmfp_dist1;		/* FPM Dispatch Thresholds1 0x10 */
345 	u32 fmfp_dist2;		/* FPM Dispatch Thresholds2 0x14 */
346 	u32 fm_epi;		/* FM Error Pending Interrupts 0x18 */
347 	u32 fm_rie;		/* FM Error Interrupt Enable 0x1c */
348 	u32 fmfp_fcev[4];	/* FPM FMan-Controller Event 1-4 0x20-0x2f */
349 	u32 res0030[4];		/* res 0x30 - 0x3f */
350 	u32 fmfp_cee[4];	/* PM FMan-Controller Event 1-4 0x40-0x4f */
351 	u32 res0050[4];		/* res 0x50-0x5f */
352 	u32 fmfp_tsc1;		/* FPM TimeStamp Control1 0x60 */
353 	u32 fmfp_tsc2;		/* FPM TimeStamp Control2 0x64 */
354 	u32 fmfp_tsp;		/* FPM Time Stamp 0x68 */
355 	u32 fmfp_tsf;		/* FPM Time Stamp Fraction 0x6c */
356 	u32 fm_rcr;		/* FM Rams Control 0x70 */
357 	u32 fmfp_extc;		/* FPM External Requests Control 0x74 */
358 	u32 fmfp_ext1;		/* FPM External Requests Config1 0x78 */
359 	u32 fmfp_ext2;		/* FPM External Requests Config2 0x7c */
360 	u32 fmfp_drd[16];	/* FPM Data_Ram Data 0-15 0x80 - 0xbf */
361 	u32 fmfp_dra;		/* FPM Data Ram Access 0xc0 */
362 	u32 fm_ip_rev_1;	/* FM IP Block Revision 1 0xc4 */
363 	u32 fm_ip_rev_2;	/* FM IP Block Revision 2 0xc8 */
364 	u32 fm_rstc;		/* FM Reset Command 0xcc */
365 	u32 fm_cld;		/* FM Classifier Debug 0xd0 */
366 	u32 fm_npi;		/* FM Normal Pending Interrupts 0xd4 */
367 	u32 fmfp_exte;		/* FPM External Requests Enable 0xd8 */
368 	u32 fmfp_ee;		/* FPM Event&Mask 0xdc */
369 	u32 fmfp_cev[4];	/* FPM CPU Event 1-4 0xe0-0xef */
370 	u32 res00f0[4];		/* res 0xf0-0xff */
371 	u32 fmfp_ps[50];	/* FPM Port Status 0x100-0x1c7 */
372 	u32 res01c8[14];	/* res 0x1c8-0x1ff */
373 	u32 fmfp_clfabc;	/* FPM CLFABC 0x200 */
374 	u32 fmfp_clfcc;		/* FPM CLFCC 0x204 */
375 	u32 fmfp_clfaval;	/* FPM CLFAVAL 0x208 */
376 	u32 fmfp_clfbval;	/* FPM CLFBVAL 0x20c */
377 	u32 fmfp_clfcval;	/* FPM CLFCVAL 0x210 */
378 	u32 fmfp_clfamsk;	/* FPM CLFAMSK 0x214 */
379 	u32 fmfp_clfbmsk;	/* FPM CLFBMSK 0x218 */
380 	u32 fmfp_clfcmsk;	/* FPM CLFCMSK 0x21c */
381 	u32 fmfp_clfamc;	/* FPM CLFAMC 0x220 */
382 	u32 fmfp_clfbmc;	/* FPM CLFBMC 0x224 */
383 	u32 fmfp_clfcmc;	/* FPM CLFCMC 0x228 */
384 	u32 fmfp_decceh;	/* FPM DECCEH 0x22c */
385 	u32 res0230[116];	/* res 0x230 - 0x3ff */
386 	u32 fmfp_ts[128];	/* 0x400: FPM Task Status 0x400 - 0x5ff */
387 	u32 res0600[0x400 - 384];
388 };
389 
390 struct fman_bmi_regs {
391 	u32 fmbm_init;		/* BMI Initialization 0x00 */
392 	u32 fmbm_cfg1;		/* BMI Configuration 1 0x04 */
393 	u32 fmbm_cfg2;		/* BMI Configuration 2 0x08 */
394 	u32 res000c[5];		/* 0x0c - 0x1f */
395 	u32 fmbm_ievr;		/* Interrupt Event Register 0x20 */
396 	u32 fmbm_ier;		/* Interrupt Enable Register 0x24 */
397 	u32 fmbm_ifr;		/* Interrupt Force Register 0x28 */
398 	u32 res002c[5];		/* 0x2c - 0x3f */
399 	u32 fmbm_arb[8];	/* BMI Arbitration 0x40 - 0x5f */
400 	u32 res0060[12];	/* 0x60 - 0x8f */
401 	u32 fmbm_dtc[3];	/* Debug Trap Counter 0x90 - 0x9b */
402 	u32 res009c;		/* 0x9c */
403 	u32 fmbm_dcv[3][4];	/* Debug Compare val 0xa0-0xcf */
404 	u32 fmbm_dcm[3][4];	/* Debug Compare Mask 0xd0-0xff */
405 	u32 fmbm_gde;		/* BMI Global Debug Enable 0x100 */
406 	u32 fmbm_pp[63];	/* BMI Port Parameters 0x104 - 0x1ff */
407 	u32 res0200;		/* 0x200 */
408 	u32 fmbm_pfs[63];	/* BMI Port FIFO Size 0x204 - 0x2ff */
409 	u32 res0300;		/* 0x300 */
410 	u32 fmbm_spliodn[63];	/* Port Partition ID 0x304 - 0x3ff */
411 };
412 
413 struct fman_qmi_regs {
414 	u32 fmqm_gc;		/* General Configuration Register 0x00 */
415 	u32 res0004;		/* 0x04 */
416 	u32 fmqm_eie;		/* Error Interrupt Event Register 0x08 */
417 	u32 fmqm_eien;		/* Error Interrupt Enable Register 0x0c */
418 	u32 fmqm_eif;		/* Error Interrupt Force Register 0x10 */
419 	u32 fmqm_ie;		/* Interrupt Event Register 0x14 */
420 	u32 fmqm_ien;		/* Interrupt Enable Register 0x18 */
421 	u32 fmqm_if;		/* Interrupt Force Register 0x1c */
422 	u32 fmqm_gs;		/* Global Status Register 0x20 */
423 	u32 fmqm_ts;		/* Task Status Register 0x24 */
424 	u32 fmqm_etfc;		/* Enqueue Total Frame Counter 0x28 */
425 	u32 fmqm_dtfc;		/* Dequeue Total Frame Counter 0x2c */
426 	u32 fmqm_dc0;		/* Dequeue Counter 0 0x30 */
427 	u32 fmqm_dc1;		/* Dequeue Counter 1 0x34 */
428 	u32 fmqm_dc2;		/* Dequeue Counter 2 0x38 */
429 	u32 fmqm_dc3;		/* Dequeue Counter 3 0x3c */
430 	u32 fmqm_dfdc;		/* Dequeue FQID from Default Counter 0x40 */
431 	u32 fmqm_dfcc;		/* Dequeue FQID from Context Counter 0x44 */
432 	u32 fmqm_dffc;		/* Dequeue FQID from FD Counter 0x48 */
433 	u32 fmqm_dcc;		/* Dequeue Confirm Counter 0x4c */
434 	u32 res0050[7];		/* 0x50 - 0x6b */
435 	u32 fmqm_tapc;		/* Tnum Aging Period Control 0x6c */
436 	u32 fmqm_dmcvc;		/* Dequeue MAC Command Valid Counter 0x70 */
437 	u32 fmqm_difdcc;	/* Dequeue Invalid FD Command Counter 0x74 */
438 	u32 fmqm_da1v;		/* Dequeue A1 Valid Counter 0x78 */
439 	u32 res007c;		/* 0x7c */
440 	u32 fmqm_dtc;		/* 0x80 Debug Trap Counter 0x80 */
441 	u32 fmqm_efddd;		/* 0x84 Enqueue Frame desc Dynamic dbg 0x84 */
442 	u32 res0088[2];		/* 0x88 - 0x8f */
443 	struct {
444 		u32 fmqm_dtcfg1;	/* 0x90 dbg trap cfg 1 Register 0x00 */
445 		u32 fmqm_dtval1;	/* Debug Trap Value 1 Register 0x04 */
446 		u32 fmqm_dtm1;		/* Debug Trap Mask 1 Register 0x08 */
447 		u32 fmqm_dtc1;		/* Debug Trap Counter 1 Register 0x0c */
448 		u32 fmqm_dtcfg2;	/* dbg Trap cfg 2 Register 0x10 */
449 		u32 fmqm_dtval2;	/* Debug Trap Value 2 Register 0x14 */
450 		u32 fmqm_dtm2;		/* Debug Trap Mask 2 Register 0x18 */
451 		u32 res001c;		/* 0x1c */
452 	} dbg_traps[3];			/* 0x90 - 0xef */
453 	u8 res00f0[0x400 - 0xf0];	/* 0xf0 - 0x3ff */
454 };
455 
456 struct fman_dma_regs {
457 	u32 fmdmsr;	/* FM DMA status register 0x00 */
458 	u32 fmdmmr;	/* FM DMA mode register 0x04 */
459 	u32 fmdmtr;	/* FM DMA bus threshold register 0x08 */
460 	u32 fmdmhy;	/* FM DMA bus hysteresis register 0x0c */
461 	u32 fmdmsetr;	/* FM DMA SOS emergency Threshold Register 0x10 */
462 	u32 fmdmtah;	/* FM DMA transfer bus address high reg 0x14 */
463 	u32 fmdmtal;	/* FM DMA transfer bus address low reg 0x18 */
464 	u32 fmdmtcid;	/* FM DMA transfer bus communication ID reg 0x1c */
465 	u32 fmdmra;	/* FM DMA bus internal ram address register 0x20 */
466 	u32 fmdmrd;	/* FM DMA bus internal ram data register 0x24 */
467 	u32 fmdmwcr;	/* FM DMA CAM watchdog counter value 0x28 */
468 	u32 fmdmebcr;	/* FM DMA CAM base in MURAM register 0x2c */
469 	u32 fmdmccqdr;	/* FM DMA CAM and CMD Queue Debug reg 0x30 */
470 	u32 fmdmccqvr1;	/* FM DMA CAM and CMD Queue Value reg #1 0x34 */
471 	u32 fmdmccqvr2;	/* FM DMA CAM and CMD Queue Value reg #2 0x38 */
472 	u32 fmdmcqvr3;	/* FM DMA CMD Queue Value register #3 0x3c */
473 	u32 fmdmcqvr4;	/* FM DMA CMD Queue Value register #4 0x40 */
474 	u32 fmdmcqvr5;	/* FM DMA CMD Queue Value register #5 0x44 */
475 	u32 fmdmsefrc;	/* FM DMA Semaphore Entry Full Reject Cntr 0x48 */
476 	u32 fmdmsqfrc;	/* FM DMA Semaphore Queue Full Reject Cntr 0x4c */
477 	u32 fmdmssrc;	/* FM DMA Semaphore SYNC Reject Counter 0x50 */
478 	u32 fmdmdcr;	/* FM DMA Debug Counter 0x54 */
479 	u32 fmdmemsr;	/* FM DMA Emergency Smoother Register 0x58 */
480 	u32 res005c;	/* 0x5c */
481 	u32 fmdmplr[FMAN_LIODN_TBL / 2];	/* DMA LIODN regs 0x60-0xdf */
482 	u32 res00e0[0x400 - 56];
483 };
484 
485 struct fman_hwp_regs {
486 	u32 res0000[0x844 / 4];		/* 0x000..0x843 */
487 	u32 fmprrpimac;	/* FM Parser Internal memory access control */
488 	u32 res[(0x1000 - 0x848) / 4];	/* 0x848..0xFFF */
489 };
490 
491 /* Structure that holds current FMan state.
492  * Used for saving run time information.
493  */
494 struct fman_state_struct {
495 	u8 fm_id;
496 	u16 fm_clk_freq;
497 	struct fman_rev_info rev_info;
498 	bool enabled_time_stamp;
499 	u8 count1_micro_bit;
500 	u8 total_num_of_tasks;
501 	u8 accumulated_num_of_tasks;
502 	u32 accumulated_fifo_size;
503 	u8 accumulated_num_of_open_dmas;
504 	u8 accumulated_num_of_deq_tnums;
505 	u32 exceptions;
506 	u32 extra_fifo_pool_size;
507 	u8 extra_tasks_pool_size;
508 	u8 extra_open_dmas_pool_size;
509 	u16 port_mfl[MAX_NUM_OF_MACS];
510 	u16 mac_mfl[MAX_NUM_OF_MACS];
511 
512 	/* SOC specific */
513 	u32 fm_iram_size;
514 	/* DMA */
515 	u32 dma_thresh_max_commq;
516 	u32 dma_thresh_max_buf;
517 	u32 max_num_of_open_dmas;
518 	/* QMI */
519 	u32 qmi_max_num_of_tnums;
520 	u32 qmi_def_tnums_thresh;
521 	/* BMI */
522 	u32 bmi_max_num_of_tasks;
523 	u32 bmi_max_fifo_size;
524 	/* General */
525 	u32 fm_port_num_of_cg;
526 	u32 num_of_rx_ports;
527 	u32 total_fifo_size;
528 
529 	u32 qman_channel_base;
530 	u32 num_of_qman_channels;
531 
532 	struct resource *res;
533 };
534 
535 /* Structure that holds FMan initial configuration */
536 struct fman_cfg {
537 	u8 disp_limit_tsh;
538 	u8 prs_disp_tsh;
539 	u8 plcr_disp_tsh;
540 	u8 kg_disp_tsh;
541 	u8 bmi_disp_tsh;
542 	u8 qmi_enq_disp_tsh;
543 	u8 qmi_deq_disp_tsh;
544 	u8 fm_ctl1_disp_tsh;
545 	u8 fm_ctl2_disp_tsh;
546 	int dma_cache_override;
547 	enum fman_dma_aid_mode dma_aid_mode;
548 	u32 dma_axi_dbg_num_of_beats;
549 	u32 dma_cam_num_of_entries;
550 	u32 dma_watchdog;
551 	u8 dma_comm_qtsh_asrt_emer;
552 	u32 dma_write_buf_tsh_asrt_emer;
553 	u32 dma_read_buf_tsh_asrt_emer;
554 	u8 dma_comm_qtsh_clr_emer;
555 	u32 dma_write_buf_tsh_clr_emer;
556 	u32 dma_read_buf_tsh_clr_emer;
557 	u32 dma_sos_emergency;
558 	int dma_dbg_cnt_mode;
559 	int catastrophic_err;
560 	int dma_err;
561 	u32 exceptions;
562 	u16 clk_freq;
563 	u32 cam_base_addr;
564 	u32 fifo_base_addr;
565 	u32 total_fifo_size;
566 	u32 total_num_of_tasks;
567 	u32 qmi_def_tnums_thresh;
568 };
569 
570 #ifdef CONFIG_DPAA_ERRATUM_A050385
571 static bool fman_has_err_a050385;
572 #endif
573 
574 static irqreturn_t fman_exceptions(struct fman *fman,
575 				   enum fman_exceptions exception)
576 {
577 	dev_dbg(fman->dev, "%s: FMan[%d] exception %d\n",
578 		__func__, fman->state->fm_id, exception);
579 
580 	return IRQ_HANDLED;
581 }
582 
583 static irqreturn_t fman_bus_error(struct fman *fman, u8 __maybe_unused port_id,
584 				  u64 __maybe_unused addr,
585 				  u8 __maybe_unused tnum,
586 				  u16 __maybe_unused liodn)
587 {
588 	dev_dbg(fman->dev, "%s: FMan[%d] bus error: port_id[%d]\n",
589 		__func__, fman->state->fm_id, port_id);
590 
591 	return IRQ_HANDLED;
592 }
593 
594 static inline irqreturn_t call_mac_isr(struct fman *fman, u8 id)
595 {
596 	if (fman->intr_mng[id].isr_cb) {
597 		fman->intr_mng[id].isr_cb(fman->intr_mng[id].src_handle);
598 
599 		return IRQ_HANDLED;
600 	}
601 
602 	return IRQ_NONE;
603 }
604 
605 static inline u8 hw_port_id_to_sw_port_id(u8 major, u8 hw_port_id)
606 {
607 	u8 sw_port_id = 0;
608 
609 	if (hw_port_id >= BASE_TX_PORTID)
610 		sw_port_id = hw_port_id - BASE_TX_PORTID;
611 	else if (hw_port_id >= BASE_RX_PORTID)
612 		sw_port_id = hw_port_id - BASE_RX_PORTID;
613 	else
614 		sw_port_id = 0;
615 
616 	return sw_port_id;
617 }
618 
619 static void set_port_order_restoration(struct fman_fpm_regs __iomem *fpm_rg,
620 				       u8 port_id)
621 {
622 	u32 tmp = 0;
623 
624 	tmp = port_id << FPM_PORT_FM_CTL_PORTID_SHIFT;
625 
626 	tmp |= FPM_PRT_FM_CTL2 | FPM_PRT_FM_CTL1;
627 
628 	/* order restoration */
629 	if (port_id % 2)
630 		tmp |= FPM_PRT_FM_CTL1 << FPM_PRC_ORA_FM_CTL_SEL_SHIFT;
631 	else
632 		tmp |= FPM_PRT_FM_CTL2 << FPM_PRC_ORA_FM_CTL_SEL_SHIFT;
633 
634 	iowrite32be(tmp, &fpm_rg->fmfp_prc);
635 }
636 
637 static void set_port_liodn(struct fman *fman, u8 port_id,
638 			   u32 liodn_base, u32 liodn_ofst)
639 {
640 	u32 tmp;
641 
642 	iowrite32be(liodn_ofst, &fman->bmi_regs->fmbm_spliodn[port_id - 1]);
643 	if (!IS_ENABLED(CONFIG_FSL_PAMU))
644 		return;
645 	/* set LIODN base for this port */
646 	tmp = ioread32be(&fman->dma_regs->fmdmplr[port_id / 2]);
647 	if (port_id % 2) {
648 		tmp &= ~DMA_LIODN_BASE_MASK;
649 		tmp |= liodn_base;
650 	} else {
651 		tmp &= ~(DMA_LIODN_BASE_MASK << DMA_LIODN_SHIFT);
652 		tmp |= liodn_base << DMA_LIODN_SHIFT;
653 	}
654 	iowrite32be(tmp, &fman->dma_regs->fmdmplr[port_id / 2]);
655 }
656 
657 static void enable_rams_ecc(struct fman_fpm_regs __iomem *fpm_rg)
658 {
659 	u32 tmp;
660 
661 	tmp = ioread32be(&fpm_rg->fm_rcr);
662 	if (tmp & FPM_RAM_RAMS_ECC_EN_SRC_SEL)
663 		iowrite32be(tmp | FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
664 	else
665 		iowrite32be(tmp | FPM_RAM_RAMS_ECC_EN |
666 			    FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
667 }
668 
669 static void disable_rams_ecc(struct fman_fpm_regs __iomem *fpm_rg)
670 {
671 	u32 tmp;
672 
673 	tmp = ioread32be(&fpm_rg->fm_rcr);
674 	if (tmp & FPM_RAM_RAMS_ECC_EN_SRC_SEL)
675 		iowrite32be(tmp & ~FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
676 	else
677 		iowrite32be(tmp & ~(FPM_RAM_RAMS_ECC_EN | FPM_RAM_IRAM_ECC_EN),
678 			    &fpm_rg->fm_rcr);
679 }
680 
681 static void fman_defconfig(struct fman_cfg *cfg)
682 {
683 	memset(cfg, 0, sizeof(struct fman_cfg));
684 
685 	cfg->catastrophic_err = DEFAULT_CATASTROPHIC_ERR;
686 	cfg->dma_err = DEFAULT_DMA_ERR;
687 	cfg->dma_aid_mode = DEFAULT_AID_MODE;
688 	cfg->dma_comm_qtsh_clr_emer = DEFAULT_DMA_COMM_Q_LOW;
689 	cfg->dma_comm_qtsh_asrt_emer = DEFAULT_DMA_COMM_Q_HIGH;
690 	cfg->dma_cache_override = DEFAULT_CACHE_OVERRIDE;
691 	cfg->dma_cam_num_of_entries = DEFAULT_DMA_CAM_NUM_OF_ENTRIES;
692 	cfg->dma_dbg_cnt_mode = DEFAULT_DMA_DBG_CNT_MODE;
693 	cfg->dma_sos_emergency = DEFAULT_DMA_SOS_EMERGENCY;
694 	cfg->dma_watchdog = DEFAULT_DMA_WATCHDOG;
695 	cfg->disp_limit_tsh = DEFAULT_DISP_LIMIT;
696 	cfg->prs_disp_tsh = DEFAULT_PRS_DISP_TH;
697 	cfg->plcr_disp_tsh = DEFAULT_PLCR_DISP_TH;
698 	cfg->kg_disp_tsh = DEFAULT_KG_DISP_TH;
699 	cfg->bmi_disp_tsh = DEFAULT_BMI_DISP_TH;
700 	cfg->qmi_enq_disp_tsh = DEFAULT_QMI_ENQ_DISP_TH;
701 	cfg->qmi_deq_disp_tsh = DEFAULT_QMI_DEQ_DISP_TH;
702 	cfg->fm_ctl1_disp_tsh = DEFAULT_FM_CTL1_DISP_TH;
703 	cfg->fm_ctl2_disp_tsh = DEFAULT_FM_CTL2_DISP_TH;
704 }
705 
706 static int dma_init(struct fman *fman)
707 {
708 	struct fman_dma_regs __iomem *dma_rg = fman->dma_regs;
709 	struct fman_cfg *cfg = fman->cfg;
710 	u32 tmp_reg;
711 
712 	/* Init DMA Registers */
713 
714 	/* clear status reg events */
715 	tmp_reg = (DMA_STATUS_BUS_ERR | DMA_STATUS_READ_ECC |
716 		   DMA_STATUS_SYSTEM_WRITE_ECC | DMA_STATUS_FM_WRITE_ECC);
717 	iowrite32be(ioread32be(&dma_rg->fmdmsr) | tmp_reg, &dma_rg->fmdmsr);
718 
719 	/* configure mode register */
720 	tmp_reg = 0;
721 	tmp_reg |= cfg->dma_cache_override << DMA_MODE_CACHE_OR_SHIFT;
722 	if (cfg->exceptions & EX_DMA_BUS_ERROR)
723 		tmp_reg |= DMA_MODE_BER;
724 	if ((cfg->exceptions & EX_DMA_SYSTEM_WRITE_ECC) |
725 	    (cfg->exceptions & EX_DMA_READ_ECC) |
726 	    (cfg->exceptions & EX_DMA_FM_WRITE_ECC))
727 		tmp_reg |= DMA_MODE_ECC;
728 	if (cfg->dma_axi_dbg_num_of_beats)
729 		tmp_reg |= (DMA_MODE_AXI_DBG_MASK &
730 			((cfg->dma_axi_dbg_num_of_beats - 1)
731 			<< DMA_MODE_AXI_DBG_SHIFT));
732 
733 	tmp_reg |= (((cfg->dma_cam_num_of_entries / DMA_CAM_UNITS) - 1) &
734 		DMA_MODE_CEN_MASK) << DMA_MODE_CEN_SHIFT;
735 	tmp_reg |= DMA_MODE_SECURE_PROT;
736 	tmp_reg |= cfg->dma_dbg_cnt_mode << DMA_MODE_DBG_SHIFT;
737 	tmp_reg |= cfg->dma_aid_mode << DMA_MODE_AID_MODE_SHIFT;
738 
739 	iowrite32be(tmp_reg, &dma_rg->fmdmmr);
740 
741 	/* configure thresholds register */
742 	tmp_reg = ((u32)cfg->dma_comm_qtsh_asrt_emer <<
743 		DMA_THRESH_COMMQ_SHIFT);
744 	tmp_reg |= (cfg->dma_read_buf_tsh_asrt_emer &
745 		DMA_THRESH_READ_INT_BUF_MASK) << DMA_THRESH_READ_INT_BUF_SHIFT;
746 	tmp_reg |= cfg->dma_write_buf_tsh_asrt_emer &
747 		DMA_THRESH_WRITE_INT_BUF_MASK;
748 
749 	iowrite32be(tmp_reg, &dma_rg->fmdmtr);
750 
751 	/* configure hysteresis register */
752 	tmp_reg = ((u32)cfg->dma_comm_qtsh_clr_emer <<
753 		DMA_THRESH_COMMQ_SHIFT);
754 	tmp_reg |= (cfg->dma_read_buf_tsh_clr_emer &
755 		DMA_THRESH_READ_INT_BUF_MASK) << DMA_THRESH_READ_INT_BUF_SHIFT;
756 	tmp_reg |= cfg->dma_write_buf_tsh_clr_emer &
757 		DMA_THRESH_WRITE_INT_BUF_MASK;
758 
759 	iowrite32be(tmp_reg, &dma_rg->fmdmhy);
760 
761 	/* configure emergency threshold */
762 	iowrite32be(cfg->dma_sos_emergency, &dma_rg->fmdmsetr);
763 
764 	/* configure Watchdog */
765 	iowrite32be((cfg->dma_watchdog * cfg->clk_freq), &dma_rg->fmdmwcr);
766 
767 	iowrite32be(cfg->cam_base_addr, &dma_rg->fmdmebcr);
768 
769 	/* Allocate MURAM for CAM */
770 	fman->cam_size =
771 		(u32)(fman->cfg->dma_cam_num_of_entries * DMA_CAM_SIZEOF_ENTRY);
772 	fman->cam_offset = fman_muram_alloc(fman->muram, fman->cam_size);
773 	if (IS_ERR_VALUE(fman->cam_offset)) {
774 		dev_err(fman->dev, "%s: MURAM alloc for DMA CAM failed\n",
775 			__func__);
776 		return -ENOMEM;
777 	}
778 
779 	if (fman->state->rev_info.major == 2) {
780 		u32 __iomem *cam_base_addr;
781 
782 		fman_muram_free_mem(fman->muram, fman->cam_offset,
783 				    fman->cam_size);
784 
785 		fman->cam_size = fman->cfg->dma_cam_num_of_entries * 72 + 128;
786 		fman->cam_offset = fman_muram_alloc(fman->muram,
787 						    fman->cam_size);
788 		if (IS_ERR_VALUE(fman->cam_offset)) {
789 			dev_err(fman->dev, "%s: MURAM alloc for DMA CAM failed\n",
790 				__func__);
791 			return -ENOMEM;
792 		}
793 
794 		if (fman->cfg->dma_cam_num_of_entries % 8 ||
795 		    fman->cfg->dma_cam_num_of_entries > 32) {
796 			dev_err(fman->dev, "%s: wrong dma_cam_num_of_entries\n",
797 				__func__);
798 			return -EINVAL;
799 		}
800 
801 		cam_base_addr = (u32 __iomem *)
802 			fman_muram_offset_to_vbase(fman->muram,
803 						   fman->cam_offset);
804 		iowrite32be(~((1 <<
805 			    (32 - fman->cfg->dma_cam_num_of_entries)) - 1),
806 			    cam_base_addr);
807 	}
808 
809 	fman->cfg->cam_base_addr = fman->cam_offset;
810 
811 	return 0;
812 }
813 
814 static void fpm_init(struct fman_fpm_regs __iomem *fpm_rg, struct fman_cfg *cfg)
815 {
816 	u32 tmp_reg;
817 	int i;
818 
819 	/* Init FPM Registers */
820 
821 	tmp_reg = (u32)(cfg->disp_limit_tsh << FPM_DISP_LIMIT_SHIFT);
822 	iowrite32be(tmp_reg, &fpm_rg->fmfp_mxd);
823 
824 	tmp_reg = (((u32)cfg->prs_disp_tsh << FPM_THR1_PRS_SHIFT) |
825 		   ((u32)cfg->kg_disp_tsh << FPM_THR1_KG_SHIFT) |
826 		   ((u32)cfg->plcr_disp_tsh << FPM_THR1_PLCR_SHIFT) |
827 		   ((u32)cfg->bmi_disp_tsh << FPM_THR1_BMI_SHIFT));
828 	iowrite32be(tmp_reg, &fpm_rg->fmfp_dist1);
829 
830 	tmp_reg =
831 		(((u32)cfg->qmi_enq_disp_tsh << FPM_THR2_QMI_ENQ_SHIFT) |
832 		 ((u32)cfg->qmi_deq_disp_tsh << FPM_THR2_QMI_DEQ_SHIFT) |
833 		 ((u32)cfg->fm_ctl1_disp_tsh << FPM_THR2_FM_CTL1_SHIFT) |
834 		 ((u32)cfg->fm_ctl2_disp_tsh << FPM_THR2_FM_CTL2_SHIFT));
835 	iowrite32be(tmp_reg, &fpm_rg->fmfp_dist2);
836 
837 	/* define exceptions and error behavior */
838 	tmp_reg = 0;
839 	/* Clear events */
840 	tmp_reg |= (FPM_EV_MASK_STALL | FPM_EV_MASK_DOUBLE_ECC |
841 		    FPM_EV_MASK_SINGLE_ECC);
842 	/* enable interrupts */
843 	if (cfg->exceptions & EX_FPM_STALL_ON_TASKS)
844 		tmp_reg |= FPM_EV_MASK_STALL_EN;
845 	if (cfg->exceptions & EX_FPM_SINGLE_ECC)
846 		tmp_reg |= FPM_EV_MASK_SINGLE_ECC_EN;
847 	if (cfg->exceptions & EX_FPM_DOUBLE_ECC)
848 		tmp_reg |= FPM_EV_MASK_DOUBLE_ECC_EN;
849 	tmp_reg |= (cfg->catastrophic_err << FPM_EV_MASK_CAT_ERR_SHIFT);
850 	tmp_reg |= (cfg->dma_err << FPM_EV_MASK_DMA_ERR_SHIFT);
851 	/* FMan is not halted upon external halt activation */
852 	tmp_reg |= FPM_EV_MASK_EXTERNAL_HALT;
853 	/* Man is not halted upon  Unrecoverable ECC error behavior */
854 	tmp_reg |= FPM_EV_MASK_ECC_ERR_HALT;
855 	iowrite32be(tmp_reg, &fpm_rg->fmfp_ee);
856 
857 	/* clear all fmCtls event registers */
858 	for (i = 0; i < FM_NUM_OF_FMAN_CTRL_EVENT_REGS; i++)
859 		iowrite32be(0xFFFFFFFF, &fpm_rg->fmfp_cev[i]);
860 
861 	/* RAM ECC -  enable and clear events */
862 	/* first we need to clear all parser memory,
863 	 * as it is uninitialized and may cause ECC errors
864 	 */
865 	/* event bits */
866 	tmp_reg = (FPM_RAM_MURAM_ECC | FPM_RAM_IRAM_ECC);
867 
868 	iowrite32be(tmp_reg, &fpm_rg->fm_rcr);
869 
870 	tmp_reg = 0;
871 	if (cfg->exceptions & EX_IRAM_ECC) {
872 		tmp_reg |= FPM_IRAM_ECC_ERR_EX_EN;
873 		enable_rams_ecc(fpm_rg);
874 	}
875 	if (cfg->exceptions & EX_MURAM_ECC) {
876 		tmp_reg |= FPM_MURAM_ECC_ERR_EX_EN;
877 		enable_rams_ecc(fpm_rg);
878 	}
879 	iowrite32be(tmp_reg, &fpm_rg->fm_rie);
880 }
881 
882 static void bmi_init(struct fman_bmi_regs __iomem *bmi_rg,
883 		     struct fman_cfg *cfg)
884 {
885 	u32 tmp_reg;
886 
887 	/* Init BMI Registers */
888 
889 	/* define common resources */
890 	tmp_reg = cfg->fifo_base_addr;
891 	tmp_reg = tmp_reg / BMI_FIFO_ALIGN;
892 
893 	tmp_reg |= ((cfg->total_fifo_size / FMAN_BMI_FIFO_UNITS - 1) <<
894 		    BMI_CFG1_FIFO_SIZE_SHIFT);
895 	iowrite32be(tmp_reg, &bmi_rg->fmbm_cfg1);
896 
897 	tmp_reg = ((cfg->total_num_of_tasks - 1) & BMI_CFG2_TASKS_MASK) <<
898 		   BMI_CFG2_TASKS_SHIFT;
899 	/* num of DMA's will be dynamically updated when each port is set */
900 	iowrite32be(tmp_reg, &bmi_rg->fmbm_cfg2);
901 
902 	/* define unmaskable exceptions, enable and clear events */
903 	tmp_reg = 0;
904 	iowrite32be(BMI_ERR_INTR_EN_LIST_RAM_ECC |
905 		    BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC |
906 		    BMI_ERR_INTR_EN_STATISTICS_RAM_ECC |
907 		    BMI_ERR_INTR_EN_DISPATCH_RAM_ECC, &bmi_rg->fmbm_ievr);
908 
909 	if (cfg->exceptions & EX_BMI_LIST_RAM_ECC)
910 		tmp_reg |= BMI_ERR_INTR_EN_LIST_RAM_ECC;
911 	if (cfg->exceptions & EX_BMI_STORAGE_PROFILE_ECC)
912 		tmp_reg |= BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
913 	if (cfg->exceptions & EX_BMI_STATISTICS_RAM_ECC)
914 		tmp_reg |= BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
915 	if (cfg->exceptions & EX_BMI_DISPATCH_RAM_ECC)
916 		tmp_reg |= BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
917 	iowrite32be(tmp_reg, &bmi_rg->fmbm_ier);
918 }
919 
920 static void qmi_init(struct fman_qmi_regs __iomem *qmi_rg,
921 		     struct fman_cfg *cfg)
922 {
923 	u32 tmp_reg;
924 
925 	/* Init QMI Registers */
926 
927 	/* Clear error interrupt events */
928 
929 	iowrite32be(QMI_ERR_INTR_EN_DOUBLE_ECC | QMI_ERR_INTR_EN_DEQ_FROM_DEF,
930 		    &qmi_rg->fmqm_eie);
931 	tmp_reg = 0;
932 	if (cfg->exceptions & EX_QMI_DEQ_FROM_UNKNOWN_PORTID)
933 		tmp_reg |= QMI_ERR_INTR_EN_DEQ_FROM_DEF;
934 	if (cfg->exceptions & EX_QMI_DOUBLE_ECC)
935 		tmp_reg |= QMI_ERR_INTR_EN_DOUBLE_ECC;
936 	/* enable events */
937 	iowrite32be(tmp_reg, &qmi_rg->fmqm_eien);
938 
939 	tmp_reg = 0;
940 	/* Clear interrupt events */
941 	iowrite32be(QMI_INTR_EN_SINGLE_ECC, &qmi_rg->fmqm_ie);
942 	if (cfg->exceptions & EX_QMI_SINGLE_ECC)
943 		tmp_reg |= QMI_INTR_EN_SINGLE_ECC;
944 	/* enable events */
945 	iowrite32be(tmp_reg, &qmi_rg->fmqm_ien);
946 }
947 
948 static void hwp_init(struct fman_hwp_regs __iomem *hwp_rg)
949 {
950 	/* enable HW Parser */
951 	iowrite32be(HWP_RPIMAC_PEN, &hwp_rg->fmprrpimac);
952 }
953 
954 static int enable(struct fman *fman, struct fman_cfg *cfg)
955 {
956 	u32 cfg_reg = 0;
957 
958 	/* Enable all modules */
959 
960 	/* clear&enable global counters - calculate reg and save for later,
961 	 * because it's the same reg for QMI enable
962 	 */
963 	cfg_reg = QMI_CFG_EN_COUNTERS;
964 
965 	/* Set enqueue and dequeue thresholds */
966 	cfg_reg |= (cfg->qmi_def_tnums_thresh << 8) | cfg->qmi_def_tnums_thresh;
967 
968 	iowrite32be(BMI_INIT_START, &fman->bmi_regs->fmbm_init);
969 	iowrite32be(cfg_reg | QMI_CFG_ENQ_EN | QMI_CFG_DEQ_EN,
970 		    &fman->qmi_regs->fmqm_gc);
971 
972 	return 0;
973 }
974 
975 static int set_exception(struct fman *fman,
976 			 enum fman_exceptions exception, bool enable)
977 {
978 	u32 tmp;
979 
980 	switch (exception) {
981 	case FMAN_EX_DMA_BUS_ERROR:
982 		tmp = ioread32be(&fman->dma_regs->fmdmmr);
983 		if (enable)
984 			tmp |= DMA_MODE_BER;
985 		else
986 			tmp &= ~DMA_MODE_BER;
987 		/* disable bus error */
988 		iowrite32be(tmp, &fman->dma_regs->fmdmmr);
989 		break;
990 	case FMAN_EX_DMA_READ_ECC:
991 	case FMAN_EX_DMA_SYSTEM_WRITE_ECC:
992 	case FMAN_EX_DMA_FM_WRITE_ECC:
993 		tmp = ioread32be(&fman->dma_regs->fmdmmr);
994 		if (enable)
995 			tmp |= DMA_MODE_ECC;
996 		else
997 			tmp &= ~DMA_MODE_ECC;
998 		iowrite32be(tmp, &fman->dma_regs->fmdmmr);
999 		break;
1000 	case FMAN_EX_FPM_STALL_ON_TASKS:
1001 		tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
1002 		if (enable)
1003 			tmp |= FPM_EV_MASK_STALL_EN;
1004 		else
1005 			tmp &= ~FPM_EV_MASK_STALL_EN;
1006 		iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
1007 		break;
1008 	case FMAN_EX_FPM_SINGLE_ECC:
1009 		tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
1010 		if (enable)
1011 			tmp |= FPM_EV_MASK_SINGLE_ECC_EN;
1012 		else
1013 			tmp &= ~FPM_EV_MASK_SINGLE_ECC_EN;
1014 		iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
1015 		break;
1016 	case FMAN_EX_FPM_DOUBLE_ECC:
1017 		tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
1018 		if (enable)
1019 			tmp |= FPM_EV_MASK_DOUBLE_ECC_EN;
1020 		else
1021 			tmp &= ~FPM_EV_MASK_DOUBLE_ECC_EN;
1022 		iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
1023 		break;
1024 	case FMAN_EX_QMI_SINGLE_ECC:
1025 		tmp = ioread32be(&fman->qmi_regs->fmqm_ien);
1026 		if (enable)
1027 			tmp |= QMI_INTR_EN_SINGLE_ECC;
1028 		else
1029 			tmp &= ~QMI_INTR_EN_SINGLE_ECC;
1030 		iowrite32be(tmp, &fman->qmi_regs->fmqm_ien);
1031 		break;
1032 	case FMAN_EX_QMI_DOUBLE_ECC:
1033 		tmp = ioread32be(&fman->qmi_regs->fmqm_eien);
1034 		if (enable)
1035 			tmp |= QMI_ERR_INTR_EN_DOUBLE_ECC;
1036 		else
1037 			tmp &= ~QMI_ERR_INTR_EN_DOUBLE_ECC;
1038 		iowrite32be(tmp, &fman->qmi_regs->fmqm_eien);
1039 		break;
1040 	case FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID:
1041 		tmp = ioread32be(&fman->qmi_regs->fmqm_eien);
1042 		if (enable)
1043 			tmp |= QMI_ERR_INTR_EN_DEQ_FROM_DEF;
1044 		else
1045 			tmp &= ~QMI_ERR_INTR_EN_DEQ_FROM_DEF;
1046 		iowrite32be(tmp, &fman->qmi_regs->fmqm_eien);
1047 		break;
1048 	case FMAN_EX_BMI_LIST_RAM_ECC:
1049 		tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1050 		if (enable)
1051 			tmp |= BMI_ERR_INTR_EN_LIST_RAM_ECC;
1052 		else
1053 			tmp &= ~BMI_ERR_INTR_EN_LIST_RAM_ECC;
1054 		iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1055 		break;
1056 	case FMAN_EX_BMI_STORAGE_PROFILE_ECC:
1057 		tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1058 		if (enable)
1059 			tmp |= BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
1060 		else
1061 			tmp &= ~BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
1062 		iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1063 		break;
1064 	case FMAN_EX_BMI_STATISTICS_RAM_ECC:
1065 		tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1066 		if (enable)
1067 			tmp |= BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
1068 		else
1069 			tmp &= ~BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
1070 		iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1071 		break;
1072 	case FMAN_EX_BMI_DISPATCH_RAM_ECC:
1073 		tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1074 		if (enable)
1075 			tmp |= BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
1076 		else
1077 			tmp &= ~BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
1078 		iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1079 		break;
1080 	case FMAN_EX_IRAM_ECC:
1081 		tmp = ioread32be(&fman->fpm_regs->fm_rie);
1082 		if (enable) {
1083 			/* enable ECC if not enabled */
1084 			enable_rams_ecc(fman->fpm_regs);
1085 			/* enable ECC interrupts */
1086 			tmp |= FPM_IRAM_ECC_ERR_EX_EN;
1087 		} else {
1088 			/* ECC mechanism may be disabled,
1089 			 * depending on driver status
1090 			 */
1091 			disable_rams_ecc(fman->fpm_regs);
1092 			tmp &= ~FPM_IRAM_ECC_ERR_EX_EN;
1093 		}
1094 		iowrite32be(tmp, &fman->fpm_regs->fm_rie);
1095 		break;
1096 	case FMAN_EX_MURAM_ECC:
1097 		tmp = ioread32be(&fman->fpm_regs->fm_rie);
1098 		if (enable) {
1099 			/* enable ECC if not enabled */
1100 			enable_rams_ecc(fman->fpm_regs);
1101 			/* enable ECC interrupts */
1102 			tmp |= FPM_MURAM_ECC_ERR_EX_EN;
1103 		} else {
1104 			/* ECC mechanism may be disabled,
1105 			 * depending on driver status
1106 			 */
1107 			disable_rams_ecc(fman->fpm_regs);
1108 			tmp &= ~FPM_MURAM_ECC_ERR_EX_EN;
1109 		}
1110 		iowrite32be(tmp, &fman->fpm_regs->fm_rie);
1111 		break;
1112 	default:
1113 		return -EINVAL;
1114 	}
1115 	return 0;
1116 }
1117 
1118 static void resume(struct fman_fpm_regs __iomem *fpm_rg)
1119 {
1120 	u32 tmp;
1121 
1122 	tmp = ioread32be(&fpm_rg->fmfp_ee);
1123 	/* clear tmp_reg event bits in order not to clear standing events */
1124 	tmp &= ~(FPM_EV_MASK_DOUBLE_ECC |
1125 		 FPM_EV_MASK_STALL | FPM_EV_MASK_SINGLE_ECC);
1126 	tmp |= FPM_EV_MASK_RELEASE_FM;
1127 
1128 	iowrite32be(tmp, &fpm_rg->fmfp_ee);
1129 }
1130 
1131 static int fill_soc_specific_params(struct fman_state_struct *state)
1132 {
1133 	u8 minor = state->rev_info.minor;
1134 	/* P4080 - Major 2
1135 	 * P2041/P3041/P5020/P5040 - Major 3
1136 	 * Tx/Bx - Major 6
1137 	 */
1138 	switch (state->rev_info.major) {
1139 	case 3:
1140 		state->bmi_max_fifo_size	= 160 * 1024;
1141 		state->fm_iram_size		= 64 * 1024;
1142 		state->dma_thresh_max_commq	= 31;
1143 		state->dma_thresh_max_buf	= 127;
1144 		state->qmi_max_num_of_tnums	= 64;
1145 		state->qmi_def_tnums_thresh	= 48;
1146 		state->bmi_max_num_of_tasks	= 128;
1147 		state->max_num_of_open_dmas	= 32;
1148 		state->fm_port_num_of_cg	= 256;
1149 		state->num_of_rx_ports	= 6;
1150 		state->total_fifo_size	= 136 * 1024;
1151 		break;
1152 
1153 	case 2:
1154 		state->bmi_max_fifo_size	= 160 * 1024;
1155 		state->fm_iram_size		= 64 * 1024;
1156 		state->dma_thresh_max_commq	= 31;
1157 		state->dma_thresh_max_buf	= 127;
1158 		state->qmi_max_num_of_tnums	= 64;
1159 		state->qmi_def_tnums_thresh	= 48;
1160 		state->bmi_max_num_of_tasks	= 128;
1161 		state->max_num_of_open_dmas	= 32;
1162 		state->fm_port_num_of_cg	= 256;
1163 		state->num_of_rx_ports	= 5;
1164 		state->total_fifo_size	= 100 * 1024;
1165 		break;
1166 
1167 	case 6:
1168 		state->dma_thresh_max_commq	= 83;
1169 		state->dma_thresh_max_buf	= 127;
1170 		state->qmi_max_num_of_tnums	= 64;
1171 		state->qmi_def_tnums_thresh	= 32;
1172 		state->fm_port_num_of_cg	= 256;
1173 
1174 		/* FManV3L */
1175 		if (minor == 1 || minor == 4) {
1176 			state->bmi_max_fifo_size	= 192 * 1024;
1177 			state->bmi_max_num_of_tasks	= 64;
1178 			state->max_num_of_open_dmas	= 32;
1179 			state->num_of_rx_ports		= 5;
1180 			if (minor == 1)
1181 				state->fm_iram_size	= 32 * 1024;
1182 			else
1183 				state->fm_iram_size	= 64 * 1024;
1184 			state->total_fifo_size		= 156 * 1024;
1185 		}
1186 		/* FManV3H */
1187 		else if (minor == 0 || minor == 2 || minor == 3) {
1188 			state->bmi_max_fifo_size	= 384 * 1024;
1189 			state->fm_iram_size		= 64 * 1024;
1190 			state->bmi_max_num_of_tasks	= 128;
1191 			state->max_num_of_open_dmas	= 84;
1192 			state->num_of_rx_ports		= 8;
1193 			state->total_fifo_size		= 295 * 1024;
1194 		} else {
1195 			pr_err("Unsupported FManv3 version\n");
1196 			return -EINVAL;
1197 		}
1198 
1199 		break;
1200 	default:
1201 		pr_err("Unsupported FMan version\n");
1202 		return -EINVAL;
1203 	}
1204 
1205 	return 0;
1206 }
1207 
1208 static bool is_init_done(struct fman_cfg *cfg)
1209 {
1210 	/* Checks if FMan driver parameters were initialized */
1211 	if (!cfg)
1212 		return true;
1213 
1214 	return false;
1215 }
1216 
1217 static void free_init_resources(struct fman *fman)
1218 {
1219 	if (fman->cam_offset)
1220 		fman_muram_free_mem(fman->muram, fman->cam_offset,
1221 				    fman->cam_size);
1222 	if (fman->fifo_offset)
1223 		fman_muram_free_mem(fman->muram, fman->fifo_offset,
1224 				    fman->fifo_size);
1225 }
1226 
1227 static irqreturn_t bmi_err_event(struct fman *fman)
1228 {
1229 	u32 event, mask, force;
1230 	struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1231 	irqreturn_t ret = IRQ_NONE;
1232 
1233 	event = ioread32be(&bmi_rg->fmbm_ievr);
1234 	mask = ioread32be(&bmi_rg->fmbm_ier);
1235 	event &= mask;
1236 	/* clear the forced events */
1237 	force = ioread32be(&bmi_rg->fmbm_ifr);
1238 	if (force & event)
1239 		iowrite32be(force & ~event, &bmi_rg->fmbm_ifr);
1240 	/* clear the acknowledged events */
1241 	iowrite32be(event, &bmi_rg->fmbm_ievr);
1242 
1243 	if (event & BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC)
1244 		ret = fman->exception_cb(fman, FMAN_EX_BMI_STORAGE_PROFILE_ECC);
1245 	if (event & BMI_ERR_INTR_EN_LIST_RAM_ECC)
1246 		ret = fman->exception_cb(fman, FMAN_EX_BMI_LIST_RAM_ECC);
1247 	if (event & BMI_ERR_INTR_EN_STATISTICS_RAM_ECC)
1248 		ret = fman->exception_cb(fman, FMAN_EX_BMI_STATISTICS_RAM_ECC);
1249 	if (event & BMI_ERR_INTR_EN_DISPATCH_RAM_ECC)
1250 		ret = fman->exception_cb(fman, FMAN_EX_BMI_DISPATCH_RAM_ECC);
1251 
1252 	return ret;
1253 }
1254 
1255 static irqreturn_t qmi_err_event(struct fman *fman)
1256 {
1257 	u32 event, mask, force;
1258 	struct fman_qmi_regs __iomem *qmi_rg = fman->qmi_regs;
1259 	irqreturn_t ret = IRQ_NONE;
1260 
1261 	event = ioread32be(&qmi_rg->fmqm_eie);
1262 	mask = ioread32be(&qmi_rg->fmqm_eien);
1263 	event &= mask;
1264 
1265 	/* clear the forced events */
1266 	force = ioread32be(&qmi_rg->fmqm_eif);
1267 	if (force & event)
1268 		iowrite32be(force & ~event, &qmi_rg->fmqm_eif);
1269 	/* clear the acknowledged events */
1270 	iowrite32be(event, &qmi_rg->fmqm_eie);
1271 
1272 	if (event & QMI_ERR_INTR_EN_DOUBLE_ECC)
1273 		ret = fman->exception_cb(fman, FMAN_EX_QMI_DOUBLE_ECC);
1274 	if (event & QMI_ERR_INTR_EN_DEQ_FROM_DEF)
1275 		ret = fman->exception_cb(fman,
1276 					 FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID);
1277 
1278 	return ret;
1279 }
1280 
1281 static irqreturn_t dma_err_event(struct fman *fman)
1282 {
1283 	u32 status, mask, com_id;
1284 	u8 tnum, port_id, relative_port_id;
1285 	u16 liodn;
1286 	struct fman_dma_regs __iomem *dma_rg = fman->dma_regs;
1287 	irqreturn_t ret = IRQ_NONE;
1288 
1289 	status = ioread32be(&dma_rg->fmdmsr);
1290 	mask = ioread32be(&dma_rg->fmdmmr);
1291 
1292 	/* clear DMA_STATUS_BUS_ERR if mask has no DMA_MODE_BER */
1293 	if ((mask & DMA_MODE_BER) != DMA_MODE_BER)
1294 		status &= ~DMA_STATUS_BUS_ERR;
1295 
1296 	/* clear relevant bits if mask has no DMA_MODE_ECC */
1297 	if ((mask & DMA_MODE_ECC) != DMA_MODE_ECC)
1298 		status &= ~(DMA_STATUS_FM_SPDAT_ECC |
1299 			    DMA_STATUS_READ_ECC |
1300 			    DMA_STATUS_SYSTEM_WRITE_ECC |
1301 			    DMA_STATUS_FM_WRITE_ECC);
1302 
1303 	/* clear set events */
1304 	iowrite32be(status, &dma_rg->fmdmsr);
1305 
1306 	if (status & DMA_STATUS_BUS_ERR) {
1307 		u64 addr;
1308 
1309 		addr = (u64)ioread32be(&dma_rg->fmdmtal);
1310 		addr |= ((u64)(ioread32be(&dma_rg->fmdmtah)) << 32);
1311 
1312 		com_id = ioread32be(&dma_rg->fmdmtcid);
1313 		port_id = (u8)(((com_id & DMA_TRANSFER_PORTID_MASK) >>
1314 			       DMA_TRANSFER_PORTID_SHIFT));
1315 		relative_port_id =
1316 		hw_port_id_to_sw_port_id(fman->state->rev_info.major, port_id);
1317 		tnum = (u8)((com_id & DMA_TRANSFER_TNUM_MASK) >>
1318 			    DMA_TRANSFER_TNUM_SHIFT);
1319 		liodn = (u16)(com_id & DMA_TRANSFER_LIODN_MASK);
1320 		ret = fman->bus_error_cb(fman, relative_port_id, addr, tnum,
1321 					 liodn);
1322 	}
1323 	if (status & DMA_STATUS_FM_SPDAT_ECC)
1324 		ret = fman->exception_cb(fman, FMAN_EX_DMA_SINGLE_PORT_ECC);
1325 	if (status & DMA_STATUS_READ_ECC)
1326 		ret = fman->exception_cb(fman, FMAN_EX_DMA_READ_ECC);
1327 	if (status & DMA_STATUS_SYSTEM_WRITE_ECC)
1328 		ret = fman->exception_cb(fman, FMAN_EX_DMA_SYSTEM_WRITE_ECC);
1329 	if (status & DMA_STATUS_FM_WRITE_ECC)
1330 		ret = fman->exception_cb(fman, FMAN_EX_DMA_FM_WRITE_ECC);
1331 
1332 	return ret;
1333 }
1334 
1335 static irqreturn_t fpm_err_event(struct fman *fman)
1336 {
1337 	u32 event;
1338 	struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1339 	irqreturn_t ret = IRQ_NONE;
1340 
1341 	event = ioread32be(&fpm_rg->fmfp_ee);
1342 	/* clear the all occurred events */
1343 	iowrite32be(event, &fpm_rg->fmfp_ee);
1344 
1345 	if ((event & FPM_EV_MASK_DOUBLE_ECC) &&
1346 	    (event & FPM_EV_MASK_DOUBLE_ECC_EN))
1347 		ret = fman->exception_cb(fman, FMAN_EX_FPM_DOUBLE_ECC);
1348 	if ((event & FPM_EV_MASK_STALL) && (event & FPM_EV_MASK_STALL_EN))
1349 		ret = fman->exception_cb(fman, FMAN_EX_FPM_STALL_ON_TASKS);
1350 	if ((event & FPM_EV_MASK_SINGLE_ECC) &&
1351 	    (event & FPM_EV_MASK_SINGLE_ECC_EN))
1352 		ret = fman->exception_cb(fman, FMAN_EX_FPM_SINGLE_ECC);
1353 
1354 	return ret;
1355 }
1356 
1357 static irqreturn_t muram_err_intr(struct fman *fman)
1358 {
1359 	u32 event, mask;
1360 	struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1361 	irqreturn_t ret = IRQ_NONE;
1362 
1363 	event = ioread32be(&fpm_rg->fm_rcr);
1364 	mask = ioread32be(&fpm_rg->fm_rie);
1365 
1366 	/* clear MURAM event bit (do not clear IRAM event) */
1367 	iowrite32be(event & ~FPM_RAM_IRAM_ECC, &fpm_rg->fm_rcr);
1368 
1369 	if ((mask & FPM_MURAM_ECC_ERR_EX_EN) && (event & FPM_RAM_MURAM_ECC))
1370 		ret = fman->exception_cb(fman, FMAN_EX_MURAM_ECC);
1371 
1372 	return ret;
1373 }
1374 
1375 static irqreturn_t qmi_event(struct fman *fman)
1376 {
1377 	u32 event, mask, force;
1378 	struct fman_qmi_regs __iomem *qmi_rg = fman->qmi_regs;
1379 	irqreturn_t ret = IRQ_NONE;
1380 
1381 	event = ioread32be(&qmi_rg->fmqm_ie);
1382 	mask = ioread32be(&qmi_rg->fmqm_ien);
1383 	event &= mask;
1384 	/* clear the forced events */
1385 	force = ioread32be(&qmi_rg->fmqm_if);
1386 	if (force & event)
1387 		iowrite32be(force & ~event, &qmi_rg->fmqm_if);
1388 	/* clear the acknowledged events */
1389 	iowrite32be(event, &qmi_rg->fmqm_ie);
1390 
1391 	if (event & QMI_INTR_EN_SINGLE_ECC)
1392 		ret = fman->exception_cb(fman, FMAN_EX_QMI_SINGLE_ECC);
1393 
1394 	return ret;
1395 }
1396 
1397 static void enable_time_stamp(struct fman *fman)
1398 {
1399 	struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1400 	u16 fm_clk_freq = fman->state->fm_clk_freq;
1401 	u32 tmp, intgr, ts_freq, frac;
1402 
1403 	ts_freq = (u32)(1 << fman->state->count1_micro_bit);
1404 	/* configure timestamp so that bit 8 will count 1 microsecond
1405 	 * Find effective count rate at TIMESTAMP least significant bits:
1406 	 * Effective_Count_Rate = 1MHz x 2^8 = 256MHz
1407 	 * Find frequency ratio between effective count rate and the clock:
1408 	 * Effective_Count_Rate / CLK e.g. for 600 MHz clock:
1409 	 * 256/600 = 0.4266666...
1410 	 */
1411 
1412 	intgr = ts_freq / fm_clk_freq;
1413 	/* we multiply by 2^16 to keep the fraction of the division
1414 	 * we do not div back, since we write this value as a fraction
1415 	 * see spec
1416 	 */
1417 
1418 	frac = ((ts_freq << 16) - (intgr << 16) * fm_clk_freq) / fm_clk_freq;
1419 	/* we check remainder of the division in order to round up if not int */
1420 	if (((ts_freq << 16) - (intgr << 16) * fm_clk_freq) % fm_clk_freq)
1421 		frac++;
1422 
1423 	tmp = (intgr << FPM_TS_INT_SHIFT) | (u16)frac;
1424 	iowrite32be(tmp, &fpm_rg->fmfp_tsc2);
1425 
1426 	/* enable timestamp with original clock */
1427 	iowrite32be(FPM_TS_CTL_EN, &fpm_rg->fmfp_tsc1);
1428 	fman->state->enabled_time_stamp = true;
1429 }
1430 
1431 static int clear_iram(struct fman *fman)
1432 {
1433 	struct fman_iram_regs __iomem *iram;
1434 	int i, count;
1435 
1436 	iram = fman->base_addr + IMEM_OFFSET;
1437 
1438 	/* Enable the auto-increment */
1439 	iowrite32be(IRAM_IADD_AIE, &iram->iadd);
1440 	count = 100;
1441 	do {
1442 		udelay(1);
1443 	} while ((ioread32be(&iram->iadd) != IRAM_IADD_AIE) && --count);
1444 	if (count == 0)
1445 		return -EBUSY;
1446 
1447 	for (i = 0; i < (fman->state->fm_iram_size / 4); i++)
1448 		iowrite32be(0xffffffff, &iram->idata);
1449 
1450 	iowrite32be(fman->state->fm_iram_size - 4, &iram->iadd);
1451 	count = 100;
1452 	do {
1453 		udelay(1);
1454 	} while ((ioread32be(&iram->idata) != 0xffffffff) && --count);
1455 	if (count == 0)
1456 		return -EBUSY;
1457 
1458 	return 0;
1459 }
1460 
1461 static u32 get_exception_flag(enum fman_exceptions exception)
1462 {
1463 	u32 bit_mask;
1464 
1465 	switch (exception) {
1466 	case FMAN_EX_DMA_BUS_ERROR:
1467 		bit_mask = EX_DMA_BUS_ERROR;
1468 		break;
1469 	case FMAN_EX_DMA_SINGLE_PORT_ECC:
1470 		bit_mask = EX_DMA_SINGLE_PORT_ECC;
1471 		break;
1472 	case FMAN_EX_DMA_READ_ECC:
1473 		bit_mask = EX_DMA_READ_ECC;
1474 		break;
1475 	case FMAN_EX_DMA_SYSTEM_WRITE_ECC:
1476 		bit_mask = EX_DMA_SYSTEM_WRITE_ECC;
1477 		break;
1478 	case FMAN_EX_DMA_FM_WRITE_ECC:
1479 		bit_mask = EX_DMA_FM_WRITE_ECC;
1480 		break;
1481 	case FMAN_EX_FPM_STALL_ON_TASKS:
1482 		bit_mask = EX_FPM_STALL_ON_TASKS;
1483 		break;
1484 	case FMAN_EX_FPM_SINGLE_ECC:
1485 		bit_mask = EX_FPM_SINGLE_ECC;
1486 		break;
1487 	case FMAN_EX_FPM_DOUBLE_ECC:
1488 		bit_mask = EX_FPM_DOUBLE_ECC;
1489 		break;
1490 	case FMAN_EX_QMI_SINGLE_ECC:
1491 		bit_mask = EX_QMI_SINGLE_ECC;
1492 		break;
1493 	case FMAN_EX_QMI_DOUBLE_ECC:
1494 		bit_mask = EX_QMI_DOUBLE_ECC;
1495 		break;
1496 	case FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID:
1497 		bit_mask = EX_QMI_DEQ_FROM_UNKNOWN_PORTID;
1498 		break;
1499 	case FMAN_EX_BMI_LIST_RAM_ECC:
1500 		bit_mask = EX_BMI_LIST_RAM_ECC;
1501 		break;
1502 	case FMAN_EX_BMI_STORAGE_PROFILE_ECC:
1503 		bit_mask = EX_BMI_STORAGE_PROFILE_ECC;
1504 		break;
1505 	case FMAN_EX_BMI_STATISTICS_RAM_ECC:
1506 		bit_mask = EX_BMI_STATISTICS_RAM_ECC;
1507 		break;
1508 	case FMAN_EX_BMI_DISPATCH_RAM_ECC:
1509 		bit_mask = EX_BMI_DISPATCH_RAM_ECC;
1510 		break;
1511 	case FMAN_EX_MURAM_ECC:
1512 		bit_mask = EX_MURAM_ECC;
1513 		break;
1514 	default:
1515 		bit_mask = 0;
1516 		break;
1517 	}
1518 
1519 	return bit_mask;
1520 }
1521 
1522 static int get_module_event(enum fman_event_modules module, u8 mod_id,
1523 			    enum fman_intr_type intr_type)
1524 {
1525 	int event;
1526 
1527 	switch (module) {
1528 	case FMAN_MOD_MAC:
1529 		if (intr_type == FMAN_INTR_TYPE_ERR)
1530 			event = FMAN_EV_ERR_MAC0 + mod_id;
1531 		else
1532 			event = FMAN_EV_MAC0 + mod_id;
1533 		break;
1534 	case FMAN_MOD_FMAN_CTRL:
1535 		if (intr_type == FMAN_INTR_TYPE_ERR)
1536 			event = FMAN_EV_CNT;
1537 		else
1538 			event = (FMAN_EV_FMAN_CTRL_0 + mod_id);
1539 		break;
1540 	case FMAN_MOD_DUMMY_LAST:
1541 		event = FMAN_EV_CNT;
1542 		break;
1543 	default:
1544 		event = FMAN_EV_CNT;
1545 		break;
1546 	}
1547 
1548 	return event;
1549 }
1550 
1551 static int set_size_of_fifo(struct fman *fman, u8 port_id, u32 *size_of_fifo,
1552 			    u32 *extra_size_of_fifo)
1553 {
1554 	struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1555 	u32 fifo = *size_of_fifo;
1556 	u32 extra_fifo = *extra_size_of_fifo;
1557 	u32 tmp;
1558 
1559 	/* if this is the first time a port requires extra_fifo_pool_size,
1560 	 * the total extra_fifo_pool_size must be initialized to 1 buffer per
1561 	 * port
1562 	 */
1563 	if (extra_fifo && !fman->state->extra_fifo_pool_size)
1564 		fman->state->extra_fifo_pool_size =
1565 			fman->state->num_of_rx_ports * FMAN_BMI_FIFO_UNITS;
1566 
1567 	fman->state->extra_fifo_pool_size =
1568 		max(fman->state->extra_fifo_pool_size, extra_fifo);
1569 
1570 	/* check that there are enough uncommitted fifo size */
1571 	if ((fman->state->accumulated_fifo_size + fifo) >
1572 	    (fman->state->total_fifo_size -
1573 	    fman->state->extra_fifo_pool_size)) {
1574 		dev_err(fman->dev, "%s: Requested fifo size and extra size exceed total FIFO size.\n",
1575 			__func__);
1576 		return -EAGAIN;
1577 	}
1578 
1579 	/* Read, modify and write to HW */
1580 	tmp = (fifo / FMAN_BMI_FIFO_UNITS - 1) |
1581 	       ((extra_fifo / FMAN_BMI_FIFO_UNITS) <<
1582 	       BMI_EXTRA_FIFO_SIZE_SHIFT);
1583 	iowrite32be(tmp, &bmi_rg->fmbm_pfs[port_id - 1]);
1584 
1585 	/* update accumulated */
1586 	fman->state->accumulated_fifo_size += fifo;
1587 
1588 	return 0;
1589 }
1590 
1591 static int set_num_of_tasks(struct fman *fman, u8 port_id, u8 *num_of_tasks,
1592 			    u8 *num_of_extra_tasks)
1593 {
1594 	struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1595 	u8 tasks = *num_of_tasks;
1596 	u8 extra_tasks = *num_of_extra_tasks;
1597 	u32 tmp;
1598 
1599 	if (extra_tasks)
1600 		fman->state->extra_tasks_pool_size =
1601 		max(fman->state->extra_tasks_pool_size, extra_tasks);
1602 
1603 	/* check that there are enough uncommitted tasks */
1604 	if ((fman->state->accumulated_num_of_tasks + tasks) >
1605 	    (fman->state->total_num_of_tasks -
1606 	     fman->state->extra_tasks_pool_size)) {
1607 		dev_err(fman->dev, "%s: Requested num_of_tasks and extra tasks pool for fm%d exceed total num_of_tasks.\n",
1608 			__func__, fman->state->fm_id);
1609 		return -EAGAIN;
1610 	}
1611 	/* update accumulated */
1612 	fman->state->accumulated_num_of_tasks += tasks;
1613 
1614 	/* Write to HW */
1615 	tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]) &
1616 	    ~(BMI_NUM_OF_TASKS_MASK | BMI_NUM_OF_EXTRA_TASKS_MASK);
1617 	tmp |= ((u32)((tasks - 1) << BMI_NUM_OF_TASKS_SHIFT) |
1618 		(u32)(extra_tasks << BMI_EXTRA_NUM_OF_TASKS_SHIFT));
1619 	iowrite32be(tmp, &bmi_rg->fmbm_pp[port_id - 1]);
1620 
1621 	return 0;
1622 }
1623 
1624 static int set_num_of_open_dmas(struct fman *fman, u8 port_id,
1625 				u8 *num_of_open_dmas,
1626 				u8 *num_of_extra_open_dmas)
1627 {
1628 	struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1629 	u8 open_dmas = *num_of_open_dmas;
1630 	u8 extra_open_dmas = *num_of_extra_open_dmas;
1631 	u8 total_num_dmas = 0, current_val = 0, current_extra_val = 0;
1632 	u32 tmp;
1633 
1634 	if (!open_dmas) {
1635 		/* Configuration according to values in the HW.
1636 		 * read the current number of open Dma's
1637 		 */
1638 		tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]);
1639 		current_extra_val = (u8)((tmp & BMI_NUM_OF_EXTRA_DMAS_MASK) >>
1640 					 BMI_EXTRA_NUM_OF_DMAS_SHIFT);
1641 
1642 		tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]);
1643 		current_val = (u8)(((tmp & BMI_NUM_OF_DMAS_MASK) >>
1644 				   BMI_NUM_OF_DMAS_SHIFT) + 1);
1645 
1646 		/* This is the first configuration and user did not
1647 		 * specify value (!open_dmas), reset values will be used
1648 		 * and we just save these values for resource management
1649 		 */
1650 		fman->state->extra_open_dmas_pool_size =
1651 			(u8)max(fman->state->extra_open_dmas_pool_size,
1652 				current_extra_val);
1653 		fman->state->accumulated_num_of_open_dmas += current_val;
1654 		*num_of_open_dmas = current_val;
1655 		*num_of_extra_open_dmas = current_extra_val;
1656 		return 0;
1657 	}
1658 
1659 	if (extra_open_dmas > current_extra_val)
1660 		fman->state->extra_open_dmas_pool_size =
1661 		    (u8)max(fman->state->extra_open_dmas_pool_size,
1662 			    extra_open_dmas);
1663 
1664 	if ((fman->state->rev_info.major < 6) &&
1665 	    (fman->state->accumulated_num_of_open_dmas - current_val +
1666 	     open_dmas > fman->state->max_num_of_open_dmas)) {
1667 		dev_err(fman->dev, "%s: Requested num_of_open_dmas for fm%d exceeds total num_of_open_dmas.\n",
1668 			__func__, fman->state->fm_id);
1669 		return -EAGAIN;
1670 	} else if ((fman->state->rev_info.major >= 6) &&
1671 		   !((fman->state->rev_info.major == 6) &&
1672 		   (fman->state->rev_info.minor == 0)) &&
1673 		   (fman->state->accumulated_num_of_open_dmas -
1674 		   current_val + open_dmas >
1675 		   fman->state->dma_thresh_max_commq + 1)) {
1676 		dev_err(fman->dev, "%s: Requested num_of_open_dmas for fm%d exceeds DMA Command queue (%d)\n",
1677 			__func__, fman->state->fm_id,
1678 		       fman->state->dma_thresh_max_commq + 1);
1679 		return -EAGAIN;
1680 	}
1681 
1682 	WARN_ON(fman->state->accumulated_num_of_open_dmas < current_val);
1683 	/* update acummulated */
1684 	fman->state->accumulated_num_of_open_dmas -= current_val;
1685 	fman->state->accumulated_num_of_open_dmas += open_dmas;
1686 
1687 	if (fman->state->rev_info.major < 6)
1688 		total_num_dmas =
1689 		    (u8)(fman->state->accumulated_num_of_open_dmas +
1690 		    fman->state->extra_open_dmas_pool_size);
1691 
1692 	/* calculate reg */
1693 	tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]) &
1694 	    ~(BMI_NUM_OF_DMAS_MASK | BMI_NUM_OF_EXTRA_DMAS_MASK);
1695 	tmp |= (u32)(((open_dmas - 1) << BMI_NUM_OF_DMAS_SHIFT) |
1696 			   (extra_open_dmas << BMI_EXTRA_NUM_OF_DMAS_SHIFT));
1697 	iowrite32be(tmp, &bmi_rg->fmbm_pp[port_id - 1]);
1698 
1699 	/* update total num of DMA's with committed number of open DMAS,
1700 	 * and max uncommitted pool.
1701 	 */
1702 	if (total_num_dmas) {
1703 		tmp = ioread32be(&bmi_rg->fmbm_cfg2) & ~BMI_CFG2_DMAS_MASK;
1704 		tmp |= (u32)(total_num_dmas - 1) << BMI_CFG2_DMAS_SHIFT;
1705 		iowrite32be(tmp, &bmi_rg->fmbm_cfg2);
1706 	}
1707 
1708 	return 0;
1709 }
1710 
1711 static int fman_config(struct fman *fman)
1712 {
1713 	void __iomem *base_addr;
1714 	int err;
1715 
1716 	base_addr = fman->dts_params.base_addr;
1717 
1718 	fman->state = kzalloc(sizeof(*fman->state), GFP_KERNEL);
1719 	if (!fman->state)
1720 		goto err_fm_state;
1721 
1722 	/* Allocate the FM driver's parameters structure */
1723 	fman->cfg = kzalloc(sizeof(*fman->cfg), GFP_KERNEL);
1724 	if (!fman->cfg)
1725 		goto err_fm_drv;
1726 
1727 	/* Initialize MURAM block */
1728 	fman->muram =
1729 		fman_muram_init(fman->dts_params.muram_res.start,
1730 				resource_size(&fman->dts_params.muram_res));
1731 	if (!fman->muram)
1732 		goto err_fm_soc_specific;
1733 
1734 	/* Initialize FM parameters which will be kept by the driver */
1735 	fman->state->fm_id = fman->dts_params.id;
1736 	fman->state->fm_clk_freq = fman->dts_params.clk_freq;
1737 	fman->state->qman_channel_base = fman->dts_params.qman_channel_base;
1738 	fman->state->num_of_qman_channels =
1739 		fman->dts_params.num_of_qman_channels;
1740 	fman->state->res = fman->dts_params.res;
1741 	fman->exception_cb = fman_exceptions;
1742 	fman->bus_error_cb = fman_bus_error;
1743 	fman->fpm_regs = base_addr + FPM_OFFSET;
1744 	fman->bmi_regs = base_addr + BMI_OFFSET;
1745 	fman->qmi_regs = base_addr + QMI_OFFSET;
1746 	fman->dma_regs = base_addr + DMA_OFFSET;
1747 	fman->hwp_regs = base_addr + HWP_OFFSET;
1748 	fman->kg_regs = base_addr + KG_OFFSET;
1749 	fman->base_addr = base_addr;
1750 
1751 	spin_lock_init(&fman->spinlock);
1752 	fman_defconfig(fman->cfg);
1753 
1754 	fman->state->extra_fifo_pool_size = 0;
1755 	fman->state->exceptions = (EX_DMA_BUS_ERROR                 |
1756 					EX_DMA_READ_ECC              |
1757 					EX_DMA_SYSTEM_WRITE_ECC      |
1758 					EX_DMA_FM_WRITE_ECC          |
1759 					EX_FPM_STALL_ON_TASKS        |
1760 					EX_FPM_SINGLE_ECC            |
1761 					EX_FPM_DOUBLE_ECC            |
1762 					EX_QMI_DEQ_FROM_UNKNOWN_PORTID |
1763 					EX_BMI_LIST_RAM_ECC          |
1764 					EX_BMI_STORAGE_PROFILE_ECC   |
1765 					EX_BMI_STATISTICS_RAM_ECC    |
1766 					EX_MURAM_ECC                 |
1767 					EX_BMI_DISPATCH_RAM_ECC      |
1768 					EX_QMI_DOUBLE_ECC            |
1769 					EX_QMI_SINGLE_ECC);
1770 
1771 	/* Read FMan revision for future use*/
1772 	fman_get_revision(fman, &fman->state->rev_info);
1773 
1774 	err = fill_soc_specific_params(fman->state);
1775 	if (err)
1776 		goto err_fm_soc_specific;
1777 
1778 	/* FM_AID_MODE_NO_TNUM_SW005 Errata workaround */
1779 	if (fman->state->rev_info.major >= 6)
1780 		fman->cfg->dma_aid_mode = FMAN_DMA_AID_OUT_PORT_ID;
1781 
1782 	fman->cfg->qmi_def_tnums_thresh = fman->state->qmi_def_tnums_thresh;
1783 
1784 	fman->state->total_num_of_tasks =
1785 	(u8)DFLT_TOTAL_NUM_OF_TASKS(fman->state->rev_info.major,
1786 				    fman->state->rev_info.minor,
1787 				    fman->state->bmi_max_num_of_tasks);
1788 
1789 	if (fman->state->rev_info.major < 6) {
1790 		fman->cfg->dma_comm_qtsh_clr_emer =
1791 		(u8)DFLT_DMA_COMM_Q_LOW(fman->state->rev_info.major,
1792 					fman->state->dma_thresh_max_commq);
1793 
1794 		fman->cfg->dma_comm_qtsh_asrt_emer =
1795 		(u8)DFLT_DMA_COMM_Q_HIGH(fman->state->rev_info.major,
1796 					 fman->state->dma_thresh_max_commq);
1797 
1798 		fman->cfg->dma_cam_num_of_entries =
1799 		DFLT_DMA_CAM_NUM_OF_ENTRIES(fman->state->rev_info.major);
1800 
1801 		fman->cfg->dma_read_buf_tsh_clr_emer =
1802 		DFLT_DMA_READ_INT_BUF_LOW(fman->state->dma_thresh_max_buf);
1803 
1804 		fman->cfg->dma_read_buf_tsh_asrt_emer =
1805 		DFLT_DMA_READ_INT_BUF_HIGH(fman->state->dma_thresh_max_buf);
1806 
1807 		fman->cfg->dma_write_buf_tsh_clr_emer =
1808 		DFLT_DMA_WRITE_INT_BUF_LOW(fman->state->dma_thresh_max_buf);
1809 
1810 		fman->cfg->dma_write_buf_tsh_asrt_emer =
1811 		DFLT_DMA_WRITE_INT_BUF_HIGH(fman->state->dma_thresh_max_buf);
1812 
1813 		fman->cfg->dma_axi_dbg_num_of_beats =
1814 		DFLT_AXI_DBG_NUM_OF_BEATS;
1815 	}
1816 
1817 	return 0;
1818 
1819 err_fm_soc_specific:
1820 	kfree(fman->cfg);
1821 err_fm_drv:
1822 	kfree(fman->state);
1823 err_fm_state:
1824 	kfree(fman);
1825 	return -EINVAL;
1826 }
1827 
1828 static int fman_reset(struct fman *fman)
1829 {
1830 	u32 count;
1831 	int err = 0;
1832 
1833 	if (fman->state->rev_info.major < 6) {
1834 		iowrite32be(FPM_RSTC_FM_RESET, &fman->fpm_regs->fm_rstc);
1835 		/* Wait for reset completion */
1836 		count = 100;
1837 		do {
1838 			udelay(1);
1839 		} while (((ioread32be(&fman->fpm_regs->fm_rstc)) &
1840 			 FPM_RSTC_FM_RESET) && --count);
1841 		if (count == 0)
1842 			err = -EBUSY;
1843 
1844 		goto _return;
1845 	} else {
1846 #ifdef CONFIG_PPC
1847 		struct device_node *guts_node;
1848 		struct ccsr_guts __iomem *guts_regs;
1849 		u32 devdisr2, reg;
1850 
1851 		/* Errata A007273 */
1852 		guts_node =
1853 			of_find_compatible_node(NULL, NULL,
1854 						"fsl,qoriq-device-config-2.0");
1855 		if (!guts_node) {
1856 			dev_err(fman->dev, "%s: Couldn't find guts node\n",
1857 				__func__);
1858 			goto guts_node;
1859 		}
1860 
1861 		guts_regs = of_iomap(guts_node, 0);
1862 		if (!guts_regs) {
1863 			dev_err(fman->dev, "%s: Couldn't map %pOF regs\n",
1864 				__func__, guts_node);
1865 			goto guts_regs;
1866 		}
1867 #define FMAN1_ALL_MACS_MASK	0xFCC00000
1868 #define FMAN2_ALL_MACS_MASK	0x000FCC00
1869 		/* Read current state */
1870 		devdisr2 = ioread32be(&guts_regs->devdisr2);
1871 		if (fman->dts_params.id == 0)
1872 			reg = devdisr2 & ~FMAN1_ALL_MACS_MASK;
1873 		else
1874 			reg = devdisr2 & ~FMAN2_ALL_MACS_MASK;
1875 
1876 		/* Enable all MACs */
1877 		iowrite32be(reg, &guts_regs->devdisr2);
1878 #endif
1879 
1880 		/* Perform FMan reset */
1881 		iowrite32be(FPM_RSTC_FM_RESET, &fman->fpm_regs->fm_rstc);
1882 
1883 		/* Wait for reset completion */
1884 		count = 100;
1885 		do {
1886 			udelay(1);
1887 		} while (((ioread32be(&fman->fpm_regs->fm_rstc)) &
1888 			 FPM_RSTC_FM_RESET) && --count);
1889 		if (count == 0) {
1890 #ifdef CONFIG_PPC
1891 			iounmap(guts_regs);
1892 			of_node_put(guts_node);
1893 #endif
1894 			err = -EBUSY;
1895 			goto _return;
1896 		}
1897 #ifdef CONFIG_PPC
1898 
1899 		/* Restore devdisr2 value */
1900 		iowrite32be(devdisr2, &guts_regs->devdisr2);
1901 
1902 		iounmap(guts_regs);
1903 		of_node_put(guts_node);
1904 #endif
1905 
1906 		goto _return;
1907 
1908 #ifdef CONFIG_PPC
1909 guts_regs:
1910 		of_node_put(guts_node);
1911 guts_node:
1912 		dev_dbg(fman->dev, "%s: Didn't perform FManV3 reset due to Errata A007273!\n",
1913 			__func__);
1914 #endif
1915 	}
1916 _return:
1917 	return err;
1918 }
1919 
1920 static int fman_init(struct fman *fman)
1921 {
1922 	struct fman_cfg *cfg = NULL;
1923 	int err = 0, i, count;
1924 
1925 	if (is_init_done(fman->cfg))
1926 		return -EINVAL;
1927 
1928 	fman->state->count1_micro_bit = FM_TIMESTAMP_1_USEC_BIT;
1929 
1930 	cfg = fman->cfg;
1931 
1932 	/* clear revision-dependent non existing exception */
1933 	if (fman->state->rev_info.major < 6)
1934 		fman->state->exceptions &= ~FMAN_EX_BMI_DISPATCH_RAM_ECC;
1935 
1936 	if (fman->state->rev_info.major >= 6)
1937 		fman->state->exceptions &= ~FMAN_EX_QMI_SINGLE_ECC;
1938 
1939 	/* clear CPG */
1940 	memset_io((void __iomem *)(fman->base_addr + CGP_OFFSET), 0,
1941 		  fman->state->fm_port_num_of_cg);
1942 
1943 	/* Save LIODN info before FMan reset
1944 	 * Skipping non-existent port 0 (i = 1)
1945 	 */
1946 	for (i = 1; i < FMAN_LIODN_TBL; i++) {
1947 		u32 liodn_base;
1948 
1949 		fman->liodn_offset[i] =
1950 			ioread32be(&fman->bmi_regs->fmbm_spliodn[i - 1]);
1951 		if (!IS_ENABLED(CONFIG_FSL_PAMU))
1952 			continue;
1953 		liodn_base = ioread32be(&fman->dma_regs->fmdmplr[i / 2]);
1954 		if (i % 2) {
1955 			/* FMDM_PLR LSB holds LIODN base for odd ports */
1956 			liodn_base &= DMA_LIODN_BASE_MASK;
1957 		} else {
1958 			/* FMDM_PLR MSB holds LIODN base for even ports */
1959 			liodn_base >>= DMA_LIODN_SHIFT;
1960 			liodn_base &= DMA_LIODN_BASE_MASK;
1961 		}
1962 		fman->liodn_base[i] = liodn_base;
1963 	}
1964 
1965 	err = fman_reset(fman);
1966 	if (err)
1967 		return err;
1968 
1969 	if (ioread32be(&fman->qmi_regs->fmqm_gs) & QMI_GS_HALT_NOT_BUSY) {
1970 		resume(fman->fpm_regs);
1971 		/* Wait until QMI is not in halt not busy state */
1972 		count = 100;
1973 		do {
1974 			udelay(1);
1975 		} while (((ioread32be(&fman->qmi_regs->fmqm_gs)) &
1976 			 QMI_GS_HALT_NOT_BUSY) && --count);
1977 		if (count == 0)
1978 			dev_warn(fman->dev, "%s: QMI is in halt not busy state\n",
1979 				 __func__);
1980 	}
1981 
1982 	if (clear_iram(fman) != 0)
1983 		return -EINVAL;
1984 
1985 	cfg->exceptions = fman->state->exceptions;
1986 
1987 	/* Init DMA Registers */
1988 
1989 	err = dma_init(fman);
1990 	if (err != 0) {
1991 		free_init_resources(fman);
1992 		return err;
1993 	}
1994 
1995 	/* Init FPM Registers */
1996 	fpm_init(fman->fpm_regs, fman->cfg);
1997 
1998 	/* define common resources */
1999 	/* allocate MURAM for FIFO according to total size */
2000 	fman->fifo_offset = fman_muram_alloc(fman->muram,
2001 					     fman->state->total_fifo_size);
2002 	if (IS_ERR_VALUE(fman->fifo_offset)) {
2003 		free_init_resources(fman);
2004 		dev_err(fman->dev, "%s: MURAM alloc for BMI FIFO failed\n",
2005 			__func__);
2006 		return -ENOMEM;
2007 	}
2008 
2009 	cfg->fifo_base_addr = fman->fifo_offset;
2010 	cfg->total_fifo_size = fman->state->total_fifo_size;
2011 	cfg->total_num_of_tasks = fman->state->total_num_of_tasks;
2012 	cfg->clk_freq = fman->state->fm_clk_freq;
2013 
2014 	/* Init BMI Registers */
2015 	bmi_init(fman->bmi_regs, fman->cfg);
2016 
2017 	/* Init QMI Registers */
2018 	qmi_init(fman->qmi_regs, fman->cfg);
2019 
2020 	/* Init HW Parser */
2021 	hwp_init(fman->hwp_regs);
2022 
2023 	/* Init KeyGen */
2024 	fman->keygen = keygen_init(fman->kg_regs);
2025 	if (!fman->keygen)
2026 		return -EINVAL;
2027 
2028 	err = enable(fman, cfg);
2029 	if (err != 0)
2030 		return err;
2031 
2032 	enable_time_stamp(fman);
2033 
2034 	kfree(fman->cfg);
2035 	fman->cfg = NULL;
2036 
2037 	return 0;
2038 }
2039 
2040 static int fman_set_exception(struct fman *fman,
2041 			      enum fman_exceptions exception, bool enable)
2042 {
2043 	u32 bit_mask = 0;
2044 
2045 	if (!is_init_done(fman->cfg))
2046 		return -EINVAL;
2047 
2048 	bit_mask = get_exception_flag(exception);
2049 	if (bit_mask) {
2050 		if (enable)
2051 			fman->state->exceptions |= bit_mask;
2052 		else
2053 			fman->state->exceptions &= ~bit_mask;
2054 	} else {
2055 		dev_err(fman->dev, "%s: Undefined exception (%d)\n",
2056 			__func__, exception);
2057 		return -EINVAL;
2058 	}
2059 
2060 	return set_exception(fman, exception, enable);
2061 }
2062 
2063 /**
2064  * fman_register_intr
2065  * @fman:	A Pointer to FMan device
2066  * @module:	Calling module
2067  * @mod_id:	Module id (if more than 1 exists, '0' if not)
2068  * @intr_type:	Interrupt type (error/normal) selection.
2069  * @isr_cb:	The interrupt service routine.
2070  * @src_arg:	Argument to be passed to isr_cb.
2071  *
2072  * Used to register an event handler to be processed by FMan
2073  *
2074  * Return: 0 on success; Error code otherwise.
2075  */
2076 void fman_register_intr(struct fman *fman, enum fman_event_modules module,
2077 			u8 mod_id, enum fman_intr_type intr_type,
2078 			void (*isr_cb)(void *src_arg), void *src_arg)
2079 {
2080 	int event = 0;
2081 
2082 	event = get_module_event(module, mod_id, intr_type);
2083 	WARN_ON(event >= FMAN_EV_CNT);
2084 
2085 	/* register in local FM structure */
2086 	fman->intr_mng[event].isr_cb = isr_cb;
2087 	fman->intr_mng[event].src_handle = src_arg;
2088 }
2089 EXPORT_SYMBOL(fman_register_intr);
2090 
2091 /**
2092  * fman_unregister_intr
2093  * @fman:	A Pointer to FMan device
2094  * @module:	Calling module
2095  * @mod_id:	Module id (if more than 1 exists, '0' if not)
2096  * @intr_type:	Interrupt type (error/normal) selection.
2097  *
2098  * Used to unregister an event handler to be processed by FMan
2099  *
2100  * Return: 0 on success; Error code otherwise.
2101  */
2102 void fman_unregister_intr(struct fman *fman, enum fman_event_modules module,
2103 			  u8 mod_id, enum fman_intr_type intr_type)
2104 {
2105 	int event = 0;
2106 
2107 	event = get_module_event(module, mod_id, intr_type);
2108 	WARN_ON(event >= FMAN_EV_CNT);
2109 
2110 	fman->intr_mng[event].isr_cb = NULL;
2111 	fman->intr_mng[event].src_handle = NULL;
2112 }
2113 EXPORT_SYMBOL(fman_unregister_intr);
2114 
2115 /**
2116  * fman_set_port_params
2117  * @fman:		A Pointer to FMan device
2118  * @port_params:	Port parameters
2119  *
2120  * Used by FMan Port to pass parameters to the FMan
2121  *
2122  * Return: 0 on success; Error code otherwise.
2123  */
2124 int fman_set_port_params(struct fman *fman,
2125 			 struct fman_port_init_params *port_params)
2126 {
2127 	int err;
2128 	unsigned long flags;
2129 	u8 port_id = port_params->port_id, mac_id;
2130 
2131 	spin_lock_irqsave(&fman->spinlock, flags);
2132 
2133 	err = set_num_of_tasks(fman, port_params->port_id,
2134 			       &port_params->num_of_tasks,
2135 			       &port_params->num_of_extra_tasks);
2136 	if (err)
2137 		goto return_err;
2138 
2139 	/* TX Ports */
2140 	if (port_params->port_type != FMAN_PORT_TYPE_RX) {
2141 		u32 enq_th, deq_th, reg;
2142 
2143 		/* update qmi ENQ/DEQ threshold */
2144 		fman->state->accumulated_num_of_deq_tnums +=
2145 			port_params->deq_pipeline_depth;
2146 		enq_th = (ioread32be(&fman->qmi_regs->fmqm_gc) &
2147 			  QMI_CFG_ENQ_MASK) >> QMI_CFG_ENQ_SHIFT;
2148 		/* if enq_th is too big, we reduce it to the max value
2149 		 * that is still 0
2150 		 */
2151 		if (enq_th >= (fman->state->qmi_max_num_of_tnums -
2152 		    fman->state->accumulated_num_of_deq_tnums)) {
2153 			enq_th =
2154 			fman->state->qmi_max_num_of_tnums -
2155 			fman->state->accumulated_num_of_deq_tnums - 1;
2156 
2157 			reg = ioread32be(&fman->qmi_regs->fmqm_gc);
2158 			reg &= ~QMI_CFG_ENQ_MASK;
2159 			reg |= (enq_th << QMI_CFG_ENQ_SHIFT);
2160 			iowrite32be(reg, &fman->qmi_regs->fmqm_gc);
2161 		}
2162 
2163 		deq_th = ioread32be(&fman->qmi_regs->fmqm_gc) &
2164 				    QMI_CFG_DEQ_MASK;
2165 		/* if deq_th is too small, we enlarge it to the min
2166 		 * value that is still 0.
2167 		 * depTh may not be larger than 63
2168 		 * (fman->state->qmi_max_num_of_tnums-1).
2169 		 */
2170 		if ((deq_th <= fman->state->accumulated_num_of_deq_tnums) &&
2171 		    (deq_th < fman->state->qmi_max_num_of_tnums - 1)) {
2172 			deq_th = fman->state->accumulated_num_of_deq_tnums + 1;
2173 			reg = ioread32be(&fman->qmi_regs->fmqm_gc);
2174 			reg &= ~QMI_CFG_DEQ_MASK;
2175 			reg |= deq_th;
2176 			iowrite32be(reg, &fman->qmi_regs->fmqm_gc);
2177 		}
2178 	}
2179 
2180 	err = set_size_of_fifo(fman, port_params->port_id,
2181 			       &port_params->size_of_fifo,
2182 			       &port_params->extra_size_of_fifo);
2183 	if (err)
2184 		goto return_err;
2185 
2186 	err = set_num_of_open_dmas(fman, port_params->port_id,
2187 				   &port_params->num_of_open_dmas,
2188 				   &port_params->num_of_extra_open_dmas);
2189 	if (err)
2190 		goto return_err;
2191 
2192 	set_port_liodn(fman, port_id, fman->liodn_base[port_id],
2193 		       fman->liodn_offset[port_id]);
2194 
2195 	if (fman->state->rev_info.major < 6)
2196 		set_port_order_restoration(fman->fpm_regs, port_id);
2197 
2198 	mac_id = hw_port_id_to_sw_port_id(fman->state->rev_info.major, port_id);
2199 
2200 	if (port_params->max_frame_length >= fman->state->mac_mfl[mac_id]) {
2201 		fman->state->port_mfl[mac_id] = port_params->max_frame_length;
2202 	} else {
2203 		dev_warn(fman->dev, "%s: Port (%d) max_frame_length is smaller than MAC (%d) current MTU\n",
2204 			 __func__, port_id, mac_id);
2205 		err = -EINVAL;
2206 		goto return_err;
2207 	}
2208 
2209 	spin_unlock_irqrestore(&fman->spinlock, flags);
2210 
2211 	return 0;
2212 
2213 return_err:
2214 	spin_unlock_irqrestore(&fman->spinlock, flags);
2215 	return err;
2216 }
2217 EXPORT_SYMBOL(fman_set_port_params);
2218 
2219 /**
2220  * fman_reset_mac
2221  * @fman:	A Pointer to FMan device
2222  * @mac_id:	MAC id to be reset
2223  *
2224  * Reset a specific MAC
2225  *
2226  * Return: 0 on success; Error code otherwise.
2227  */
2228 int fman_reset_mac(struct fman *fman, u8 mac_id)
2229 {
2230 	struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
2231 	u32 msk, timeout = 100;
2232 
2233 	if (fman->state->rev_info.major >= 6) {
2234 		dev_err(fman->dev, "%s: FMan MAC reset no available for FMan V3!\n",
2235 			__func__);
2236 		return -EINVAL;
2237 	}
2238 
2239 	/* Get the relevant bit mask */
2240 	switch (mac_id) {
2241 	case 0:
2242 		msk = FPM_RSTC_MAC0_RESET;
2243 		break;
2244 	case 1:
2245 		msk = FPM_RSTC_MAC1_RESET;
2246 		break;
2247 	case 2:
2248 		msk = FPM_RSTC_MAC2_RESET;
2249 		break;
2250 	case 3:
2251 		msk = FPM_RSTC_MAC3_RESET;
2252 		break;
2253 	case 4:
2254 		msk = FPM_RSTC_MAC4_RESET;
2255 		break;
2256 	case 5:
2257 		msk = FPM_RSTC_MAC5_RESET;
2258 		break;
2259 	case 6:
2260 		msk = FPM_RSTC_MAC6_RESET;
2261 		break;
2262 	case 7:
2263 		msk = FPM_RSTC_MAC7_RESET;
2264 		break;
2265 	case 8:
2266 		msk = FPM_RSTC_MAC8_RESET;
2267 		break;
2268 	case 9:
2269 		msk = FPM_RSTC_MAC9_RESET;
2270 		break;
2271 	default:
2272 		dev_warn(fman->dev, "%s: Illegal MAC Id [%d]\n",
2273 			 __func__, mac_id);
2274 		return -EINVAL;
2275 	}
2276 
2277 	/* reset */
2278 	iowrite32be(msk, &fpm_rg->fm_rstc);
2279 	while ((ioread32be(&fpm_rg->fm_rstc) & msk) && --timeout)
2280 		udelay(10);
2281 
2282 	if (!timeout)
2283 		return -EIO;
2284 
2285 	return 0;
2286 }
2287 EXPORT_SYMBOL(fman_reset_mac);
2288 
2289 /**
2290  * fman_set_mac_max_frame
2291  * @fman:	A Pointer to FMan device
2292  * @mac_id:	MAC id
2293  * @mfl:	Maximum frame length
2294  *
2295  * Set maximum frame length of specific MAC in FMan driver
2296  *
2297  * Return: 0 on success; Error code otherwise.
2298  */
2299 int fman_set_mac_max_frame(struct fman *fman, u8 mac_id, u16 mfl)
2300 {
2301 	/* if port is already initialized, check that MaxFrameLength is smaller
2302 	 * or equal to the port's max
2303 	 */
2304 	if ((!fman->state->port_mfl[mac_id]) ||
2305 	    (mfl <= fman->state->port_mfl[mac_id])) {
2306 		fman->state->mac_mfl[mac_id] = mfl;
2307 	} else {
2308 		dev_warn(fman->dev, "%s: MAC max_frame_length is larger than Port max_frame_length\n",
2309 			 __func__);
2310 		return -EINVAL;
2311 	}
2312 	return 0;
2313 }
2314 EXPORT_SYMBOL(fman_set_mac_max_frame);
2315 
2316 /**
2317  * fman_get_clock_freq
2318  * @fman:	A Pointer to FMan device
2319  *
2320  * Get FMan clock frequency
2321  *
2322  * Return: FMan clock frequency
2323  */
2324 u16 fman_get_clock_freq(struct fman *fman)
2325 {
2326 	return fman->state->fm_clk_freq;
2327 }
2328 
2329 /**
2330  * fman_get_bmi_max_fifo_size
2331  * @fman:	A Pointer to FMan device
2332  *
2333  * Get FMan maximum FIFO size
2334  *
2335  * Return: FMan Maximum FIFO size
2336  */
2337 u32 fman_get_bmi_max_fifo_size(struct fman *fman)
2338 {
2339 	return fman->state->bmi_max_fifo_size;
2340 }
2341 EXPORT_SYMBOL(fman_get_bmi_max_fifo_size);
2342 
2343 /**
2344  * fman_get_revision
2345  * @fman:		- Pointer to the FMan module
2346  * @rev_info:		- A structure of revision information parameters.
2347  *
2348  * Returns the FM revision
2349  *
2350  * Allowed only following fman_init().
2351  *
2352  * Return: 0 on success; Error code otherwise.
2353  */
2354 void fman_get_revision(struct fman *fman, struct fman_rev_info *rev_info)
2355 {
2356 	u32 tmp;
2357 
2358 	tmp = ioread32be(&fman->fpm_regs->fm_ip_rev_1);
2359 	rev_info->major = (u8)((tmp & FPM_REV1_MAJOR_MASK) >>
2360 				FPM_REV1_MAJOR_SHIFT);
2361 	rev_info->minor = tmp & FPM_REV1_MINOR_MASK;
2362 }
2363 EXPORT_SYMBOL(fman_get_revision);
2364 
2365 /**
2366  * fman_get_qman_channel_id
2367  * @fman:	A Pointer to FMan device
2368  * @port_id:	Port id
2369  *
2370  * Get QMan channel ID associated to the Port id
2371  *
2372  * Return: QMan channel ID
2373  */
2374 u32 fman_get_qman_channel_id(struct fman *fman, u32 port_id)
2375 {
2376 	int i;
2377 
2378 	if (fman->state->rev_info.major >= 6) {
2379 		static const u32 port_ids[] = {
2380 			0x30, 0x31, 0x28, 0x29, 0x2a, 0x2b,
2381 			0x2c, 0x2d, 0x2, 0x3, 0x4, 0x5, 0x7, 0x7
2382 		};
2383 
2384 		for (i = 0; i < fman->state->num_of_qman_channels; i++) {
2385 			if (port_ids[i] == port_id)
2386 				break;
2387 		}
2388 	} else {
2389 		static const u32 port_ids[] = {
2390 			0x30, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x1,
2391 			0x2, 0x3, 0x4, 0x5, 0x7, 0x7
2392 		};
2393 
2394 		for (i = 0; i < fman->state->num_of_qman_channels; i++) {
2395 			if (port_ids[i] == port_id)
2396 				break;
2397 		}
2398 	}
2399 
2400 	if (i == fman->state->num_of_qman_channels)
2401 		return 0;
2402 
2403 	return fman->state->qman_channel_base + i;
2404 }
2405 EXPORT_SYMBOL(fman_get_qman_channel_id);
2406 
2407 /**
2408  * fman_get_mem_region
2409  * @fman:	A Pointer to FMan device
2410  *
2411  * Get FMan memory region
2412  *
2413  * Return: A structure with FMan memory region information
2414  */
2415 struct resource *fman_get_mem_region(struct fman *fman)
2416 {
2417 	return fman->state->res;
2418 }
2419 EXPORT_SYMBOL(fman_get_mem_region);
2420 
2421 /* Bootargs defines */
2422 /* Extra headroom for RX buffers - Default, min and max */
2423 #define FSL_FM_RX_EXTRA_HEADROOM	64
2424 #define FSL_FM_RX_EXTRA_HEADROOM_MIN	16
2425 #define FSL_FM_RX_EXTRA_HEADROOM_MAX	384
2426 
2427 /* Maximum frame length */
2428 #define FSL_FM_MAX_FRAME_SIZE			1522
2429 #define FSL_FM_MAX_POSSIBLE_FRAME_SIZE		9600
2430 #define FSL_FM_MIN_POSSIBLE_FRAME_SIZE		64
2431 
2432 /* Extra headroom for Rx buffers.
2433  * FMan is instructed to allocate, on the Rx path, this amount of
2434  * space at the beginning of a data buffer, beside the DPA private
2435  * data area and the IC fields.
2436  * Does not impact Tx buffer layout.
2437  * Configurable from bootargs. 64 by default, it's needed on
2438  * particular forwarding scenarios that add extra headers to the
2439  * forwarded frame.
2440  */
2441 static int fsl_fm_rx_extra_headroom = FSL_FM_RX_EXTRA_HEADROOM;
2442 module_param(fsl_fm_rx_extra_headroom, int, 0);
2443 MODULE_PARM_DESC(fsl_fm_rx_extra_headroom, "Extra headroom for Rx buffers");
2444 
2445 /* Max frame size, across all interfaces.
2446  * Configurable from bootargs, to avoid allocating oversized (socket)
2447  * buffers when not using jumbo frames.
2448  * Must be large enough to accommodate the network MTU, but small enough
2449  * to avoid wasting skb memory.
2450  */
2451 static int fsl_fm_max_frm = FSL_FM_MAX_FRAME_SIZE;
2452 module_param(fsl_fm_max_frm, int, 0);
2453 MODULE_PARM_DESC(fsl_fm_max_frm, "Maximum frame size, across all interfaces");
2454 
2455 /**
2456  * fman_get_max_frm
2457  *
2458  * Return: Max frame length configured in the FM driver
2459  */
2460 u16 fman_get_max_frm(void)
2461 {
2462 	static bool fm_check_mfl;
2463 
2464 	if (!fm_check_mfl) {
2465 		if (fsl_fm_max_frm > FSL_FM_MAX_POSSIBLE_FRAME_SIZE ||
2466 		    fsl_fm_max_frm < FSL_FM_MIN_POSSIBLE_FRAME_SIZE) {
2467 			pr_warn("Invalid fsl_fm_max_frm value (%d) in bootargs, valid range is %d-%d. Falling back to the default (%d)\n",
2468 				fsl_fm_max_frm,
2469 				FSL_FM_MIN_POSSIBLE_FRAME_SIZE,
2470 				FSL_FM_MAX_POSSIBLE_FRAME_SIZE,
2471 				FSL_FM_MAX_FRAME_SIZE);
2472 			fsl_fm_max_frm = FSL_FM_MAX_FRAME_SIZE;
2473 		}
2474 		fm_check_mfl = true;
2475 	}
2476 
2477 	return fsl_fm_max_frm;
2478 }
2479 EXPORT_SYMBOL(fman_get_max_frm);
2480 
2481 /**
2482  * fman_get_rx_extra_headroom
2483  *
2484  * Return: Extra headroom size configured in the FM driver
2485  */
2486 int fman_get_rx_extra_headroom(void)
2487 {
2488 	static bool fm_check_rx_extra_headroom;
2489 
2490 	if (!fm_check_rx_extra_headroom) {
2491 		if (fsl_fm_rx_extra_headroom > FSL_FM_RX_EXTRA_HEADROOM_MAX ||
2492 		    fsl_fm_rx_extra_headroom < FSL_FM_RX_EXTRA_HEADROOM_MIN) {
2493 			pr_warn("Invalid fsl_fm_rx_extra_headroom value (%d) in bootargs, valid range is %d-%d. Falling back to the default (%d)\n",
2494 				fsl_fm_rx_extra_headroom,
2495 				FSL_FM_RX_EXTRA_HEADROOM_MIN,
2496 				FSL_FM_RX_EXTRA_HEADROOM_MAX,
2497 				FSL_FM_RX_EXTRA_HEADROOM);
2498 			fsl_fm_rx_extra_headroom = FSL_FM_RX_EXTRA_HEADROOM;
2499 		}
2500 
2501 		fm_check_rx_extra_headroom = true;
2502 		fsl_fm_rx_extra_headroom = ALIGN(fsl_fm_rx_extra_headroom, 16);
2503 	}
2504 
2505 	return fsl_fm_rx_extra_headroom;
2506 }
2507 EXPORT_SYMBOL(fman_get_rx_extra_headroom);
2508 
2509 /**
2510  * fman_bind
2511  * @fm_dev:	FMan OF device pointer
2512  *
2513  * Bind to a specific FMan device.
2514  *
2515  * Allowed only after the port was created.
2516  *
2517  * Return: A pointer to the FMan device
2518  */
2519 struct fman *fman_bind(struct device *fm_dev)
2520 {
2521 	return (struct fman *)(dev_get_drvdata(get_device(fm_dev)));
2522 }
2523 EXPORT_SYMBOL(fman_bind);
2524 
2525 #ifdef CONFIG_DPAA_ERRATUM_A050385
2526 bool fman_has_errata_a050385(void)
2527 {
2528 	return fman_has_err_a050385;
2529 }
2530 EXPORT_SYMBOL(fman_has_errata_a050385);
2531 #endif
2532 
2533 static irqreturn_t fman_err_irq(int irq, void *handle)
2534 {
2535 	struct fman *fman = (struct fman *)handle;
2536 	u32 pending;
2537 	struct fman_fpm_regs __iomem *fpm_rg;
2538 	irqreturn_t single_ret, ret = IRQ_NONE;
2539 
2540 	if (!is_init_done(fman->cfg))
2541 		return IRQ_NONE;
2542 
2543 	fpm_rg = fman->fpm_regs;
2544 
2545 	/* error interrupts */
2546 	pending = ioread32be(&fpm_rg->fm_epi);
2547 	if (!pending)
2548 		return IRQ_NONE;
2549 
2550 	if (pending & ERR_INTR_EN_BMI) {
2551 		single_ret = bmi_err_event(fman);
2552 		if (single_ret == IRQ_HANDLED)
2553 			ret = IRQ_HANDLED;
2554 	}
2555 	if (pending & ERR_INTR_EN_QMI) {
2556 		single_ret = qmi_err_event(fman);
2557 		if (single_ret == IRQ_HANDLED)
2558 			ret = IRQ_HANDLED;
2559 	}
2560 	if (pending & ERR_INTR_EN_FPM) {
2561 		single_ret = fpm_err_event(fman);
2562 		if (single_ret == IRQ_HANDLED)
2563 			ret = IRQ_HANDLED;
2564 	}
2565 	if (pending & ERR_INTR_EN_DMA) {
2566 		single_ret = dma_err_event(fman);
2567 		if (single_ret == IRQ_HANDLED)
2568 			ret = IRQ_HANDLED;
2569 	}
2570 	if (pending & ERR_INTR_EN_MURAM) {
2571 		single_ret = muram_err_intr(fman);
2572 		if (single_ret == IRQ_HANDLED)
2573 			ret = IRQ_HANDLED;
2574 	}
2575 
2576 	/* MAC error interrupts */
2577 	if (pending & ERR_INTR_EN_MAC0) {
2578 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 0);
2579 		if (single_ret == IRQ_HANDLED)
2580 			ret = IRQ_HANDLED;
2581 	}
2582 	if (pending & ERR_INTR_EN_MAC1) {
2583 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 1);
2584 		if (single_ret == IRQ_HANDLED)
2585 			ret = IRQ_HANDLED;
2586 	}
2587 	if (pending & ERR_INTR_EN_MAC2) {
2588 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 2);
2589 		if (single_ret == IRQ_HANDLED)
2590 			ret = IRQ_HANDLED;
2591 	}
2592 	if (pending & ERR_INTR_EN_MAC3) {
2593 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 3);
2594 		if (single_ret == IRQ_HANDLED)
2595 			ret = IRQ_HANDLED;
2596 	}
2597 	if (pending & ERR_INTR_EN_MAC4) {
2598 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 4);
2599 		if (single_ret == IRQ_HANDLED)
2600 			ret = IRQ_HANDLED;
2601 	}
2602 	if (pending & ERR_INTR_EN_MAC5) {
2603 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 5);
2604 		if (single_ret == IRQ_HANDLED)
2605 			ret = IRQ_HANDLED;
2606 	}
2607 	if (pending & ERR_INTR_EN_MAC6) {
2608 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 6);
2609 		if (single_ret == IRQ_HANDLED)
2610 			ret = IRQ_HANDLED;
2611 	}
2612 	if (pending & ERR_INTR_EN_MAC7) {
2613 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 7);
2614 		if (single_ret == IRQ_HANDLED)
2615 			ret = IRQ_HANDLED;
2616 	}
2617 	if (pending & ERR_INTR_EN_MAC8) {
2618 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 8);
2619 		if (single_ret == IRQ_HANDLED)
2620 			ret = IRQ_HANDLED;
2621 	}
2622 	if (pending & ERR_INTR_EN_MAC9) {
2623 		single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 9);
2624 		if (single_ret == IRQ_HANDLED)
2625 			ret = IRQ_HANDLED;
2626 	}
2627 
2628 	return ret;
2629 }
2630 
2631 static irqreturn_t fman_irq(int irq, void *handle)
2632 {
2633 	struct fman *fman = (struct fman *)handle;
2634 	u32 pending;
2635 	struct fman_fpm_regs __iomem *fpm_rg;
2636 	irqreturn_t single_ret, ret = IRQ_NONE;
2637 
2638 	if (!is_init_done(fman->cfg))
2639 		return IRQ_NONE;
2640 
2641 	fpm_rg = fman->fpm_regs;
2642 
2643 	/* normal interrupts */
2644 	pending = ioread32be(&fpm_rg->fm_npi);
2645 	if (!pending)
2646 		return IRQ_NONE;
2647 
2648 	if (pending & INTR_EN_QMI) {
2649 		single_ret = qmi_event(fman);
2650 		if (single_ret == IRQ_HANDLED)
2651 			ret = IRQ_HANDLED;
2652 	}
2653 
2654 	/* MAC interrupts */
2655 	if (pending & INTR_EN_MAC0) {
2656 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 0);
2657 		if (single_ret == IRQ_HANDLED)
2658 			ret = IRQ_HANDLED;
2659 	}
2660 	if (pending & INTR_EN_MAC1) {
2661 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 1);
2662 		if (single_ret == IRQ_HANDLED)
2663 			ret = IRQ_HANDLED;
2664 	}
2665 	if (pending & INTR_EN_MAC2) {
2666 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 2);
2667 		if (single_ret == IRQ_HANDLED)
2668 			ret = IRQ_HANDLED;
2669 	}
2670 	if (pending & INTR_EN_MAC3) {
2671 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 3);
2672 		if (single_ret == IRQ_HANDLED)
2673 			ret = IRQ_HANDLED;
2674 	}
2675 	if (pending & INTR_EN_MAC4) {
2676 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 4);
2677 		if (single_ret == IRQ_HANDLED)
2678 			ret = IRQ_HANDLED;
2679 	}
2680 	if (pending & INTR_EN_MAC5) {
2681 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 5);
2682 		if (single_ret == IRQ_HANDLED)
2683 			ret = IRQ_HANDLED;
2684 	}
2685 	if (pending & INTR_EN_MAC6) {
2686 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 6);
2687 		if (single_ret == IRQ_HANDLED)
2688 			ret = IRQ_HANDLED;
2689 	}
2690 	if (pending & INTR_EN_MAC7) {
2691 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 7);
2692 		if (single_ret == IRQ_HANDLED)
2693 			ret = IRQ_HANDLED;
2694 	}
2695 	if (pending & INTR_EN_MAC8) {
2696 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 8);
2697 		if (single_ret == IRQ_HANDLED)
2698 			ret = IRQ_HANDLED;
2699 	}
2700 	if (pending & INTR_EN_MAC9) {
2701 		single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 9);
2702 		if (single_ret == IRQ_HANDLED)
2703 			ret = IRQ_HANDLED;
2704 	}
2705 
2706 	return ret;
2707 }
2708 
2709 static const struct of_device_id fman_muram_match[] = {
2710 	{
2711 		.compatible = "fsl,fman-muram"},
2712 	{}
2713 };
2714 MODULE_DEVICE_TABLE(of, fman_muram_match);
2715 
2716 static struct fman *read_dts_node(struct platform_device *of_dev)
2717 {
2718 	struct fman *fman;
2719 	struct device_node *fm_node, *muram_node;
2720 	struct resource *res;
2721 	u32 val, range[2];
2722 	int err, irq;
2723 	struct clk *clk;
2724 	u32 clk_rate;
2725 	phys_addr_t phys_base_addr;
2726 	resource_size_t mem_size;
2727 
2728 	fman = kzalloc(sizeof(*fman), GFP_KERNEL);
2729 	if (!fman)
2730 		return ERR_PTR(-ENOMEM);
2731 
2732 	fm_node = of_node_get(of_dev->dev.of_node);
2733 
2734 	err = of_property_read_u32(fm_node, "cell-index", &val);
2735 	if (err) {
2736 		dev_err(&of_dev->dev, "%s: failed to read cell-index for %pOF\n",
2737 			__func__, fm_node);
2738 		goto fman_node_put;
2739 	}
2740 	fman->dts_params.id = (u8)val;
2741 
2742 	/* Get the FM interrupt */
2743 	err = platform_get_irq(of_dev, 0);
2744 	if (err < 0)
2745 		goto fman_node_put;
2746 	irq = err;
2747 
2748 	/* Get the FM error interrupt */
2749 	err = platform_get_irq(of_dev, 1);
2750 	if (err < 0)
2751 		goto fman_node_put;
2752 	fman->dts_params.err_irq = err;
2753 
2754 	/* Get the FM address */
2755 	res = platform_get_resource(of_dev, IORESOURCE_MEM, 0);
2756 	if (!res) {
2757 		err = -EINVAL;
2758 		dev_err(&of_dev->dev, "%s: Can't get FMan memory resource\n",
2759 			__func__);
2760 		goto fman_node_put;
2761 	}
2762 
2763 	phys_base_addr = res->start;
2764 	mem_size = resource_size(res);
2765 
2766 	clk = of_clk_get(fm_node, 0);
2767 	if (IS_ERR(clk)) {
2768 		err = PTR_ERR(clk);
2769 		dev_err(&of_dev->dev, "%s: Failed to get FM%d clock structure\n",
2770 			__func__, fman->dts_params.id);
2771 		goto fman_node_put;
2772 	}
2773 
2774 	clk_rate = clk_get_rate(clk);
2775 	if (!clk_rate) {
2776 		err = -EINVAL;
2777 		dev_err(&of_dev->dev, "%s: Failed to determine FM%d clock rate\n",
2778 			__func__, fman->dts_params.id);
2779 		goto fman_node_put;
2780 	}
2781 	/* Rounding to MHz */
2782 	fman->dts_params.clk_freq = DIV_ROUND_UP(clk_rate, 1000000);
2783 
2784 	err = of_property_read_u32_array(fm_node, "fsl,qman-channel-range",
2785 					 &range[0], 2);
2786 	if (err) {
2787 		dev_err(&of_dev->dev, "%s: failed to read fsl,qman-channel-range for %pOF\n",
2788 			__func__, fm_node);
2789 		goto fman_node_put;
2790 	}
2791 	fman->dts_params.qman_channel_base = range[0];
2792 	fman->dts_params.num_of_qman_channels = range[1];
2793 
2794 	/* Get the MURAM base address and size */
2795 	muram_node = of_find_matching_node(fm_node, fman_muram_match);
2796 	if (!muram_node) {
2797 		err = -EINVAL;
2798 		dev_err(&of_dev->dev, "%s: could not find MURAM node\n",
2799 			__func__);
2800 		goto fman_free;
2801 	}
2802 
2803 	err = of_address_to_resource(muram_node, 0,
2804 				     &fman->dts_params.muram_res);
2805 	if (err) {
2806 		of_node_put(muram_node);
2807 		dev_err(&of_dev->dev, "%s: of_address_to_resource() = %d\n",
2808 			__func__, err);
2809 		goto fman_free;
2810 	}
2811 
2812 	of_node_put(muram_node);
2813 
2814 	err = devm_request_irq(&of_dev->dev, irq, fman_irq, IRQF_SHARED,
2815 			       "fman", fman);
2816 	if (err < 0) {
2817 		dev_err(&of_dev->dev, "%s: irq %d allocation failed (error = %d)\n",
2818 			__func__, irq, err);
2819 		goto fman_free;
2820 	}
2821 
2822 	if (fman->dts_params.err_irq != 0) {
2823 		err = devm_request_irq(&of_dev->dev, fman->dts_params.err_irq,
2824 				       fman_err_irq, IRQF_SHARED,
2825 				       "fman-err", fman);
2826 		if (err < 0) {
2827 			dev_err(&of_dev->dev, "%s: irq %d allocation failed (error = %d)\n",
2828 				__func__, fman->dts_params.err_irq, err);
2829 			goto fman_free;
2830 		}
2831 	}
2832 
2833 	fman->dts_params.res =
2834 		devm_request_mem_region(&of_dev->dev, phys_base_addr,
2835 					mem_size, "fman");
2836 	if (!fman->dts_params.res) {
2837 		err = -EBUSY;
2838 		dev_err(&of_dev->dev, "%s: request_mem_region() failed\n",
2839 			__func__);
2840 		goto fman_free;
2841 	}
2842 
2843 	fman->dts_params.base_addr =
2844 		devm_ioremap(&of_dev->dev, phys_base_addr, mem_size);
2845 	if (!fman->dts_params.base_addr) {
2846 		err = -ENOMEM;
2847 		dev_err(&of_dev->dev, "%s: devm_ioremap() failed\n", __func__);
2848 		goto fman_free;
2849 	}
2850 
2851 	fman->dev = &of_dev->dev;
2852 
2853 	err = of_platform_populate(fm_node, NULL, NULL, &of_dev->dev);
2854 	if (err) {
2855 		dev_err(&of_dev->dev, "%s: of_platform_populate() failed\n",
2856 			__func__);
2857 		goto fman_free;
2858 	}
2859 
2860 #ifdef CONFIG_DPAA_ERRATUM_A050385
2861 	fman_has_err_a050385 =
2862 		of_property_read_bool(fm_node, "fsl,erratum-a050385");
2863 #endif
2864 
2865 	return fman;
2866 
2867 fman_node_put:
2868 	of_node_put(fm_node);
2869 fman_free:
2870 	kfree(fman);
2871 	return ERR_PTR(err);
2872 }
2873 
2874 static int fman_probe(struct platform_device *of_dev)
2875 {
2876 	struct fman *fman;
2877 	struct device *dev;
2878 	int err;
2879 
2880 	dev = &of_dev->dev;
2881 
2882 	fman = read_dts_node(of_dev);
2883 	if (IS_ERR(fman))
2884 		return PTR_ERR(fman);
2885 
2886 	err = fman_config(fman);
2887 	if (err) {
2888 		dev_err(dev, "%s: FMan config failed\n", __func__);
2889 		return -EINVAL;
2890 	}
2891 
2892 	if (fman_init(fman) != 0) {
2893 		dev_err(dev, "%s: FMan init failed\n", __func__);
2894 		return -EINVAL;
2895 	}
2896 
2897 	if (fman->dts_params.err_irq == 0) {
2898 		fman_set_exception(fman, FMAN_EX_DMA_BUS_ERROR, false);
2899 		fman_set_exception(fman, FMAN_EX_DMA_READ_ECC, false);
2900 		fman_set_exception(fman, FMAN_EX_DMA_SYSTEM_WRITE_ECC, false);
2901 		fman_set_exception(fman, FMAN_EX_DMA_FM_WRITE_ECC, false);
2902 		fman_set_exception(fman, FMAN_EX_DMA_SINGLE_PORT_ECC, false);
2903 		fman_set_exception(fman, FMAN_EX_FPM_STALL_ON_TASKS, false);
2904 		fman_set_exception(fman, FMAN_EX_FPM_SINGLE_ECC, false);
2905 		fman_set_exception(fman, FMAN_EX_FPM_DOUBLE_ECC, false);
2906 		fman_set_exception(fman, FMAN_EX_QMI_SINGLE_ECC, false);
2907 		fman_set_exception(fman, FMAN_EX_QMI_DOUBLE_ECC, false);
2908 		fman_set_exception(fman,
2909 				   FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID, false);
2910 		fman_set_exception(fman, FMAN_EX_BMI_LIST_RAM_ECC, false);
2911 		fman_set_exception(fman, FMAN_EX_BMI_STORAGE_PROFILE_ECC,
2912 				   false);
2913 		fman_set_exception(fman, FMAN_EX_BMI_STATISTICS_RAM_ECC, false);
2914 		fman_set_exception(fman, FMAN_EX_BMI_DISPATCH_RAM_ECC, false);
2915 	}
2916 
2917 	dev_set_drvdata(dev, fman);
2918 
2919 	dev_dbg(dev, "FMan%d probed\n", fman->dts_params.id);
2920 
2921 	return 0;
2922 }
2923 
2924 static const struct of_device_id fman_match[] = {
2925 	{
2926 		.compatible = "fsl,fman"},
2927 	{}
2928 };
2929 
2930 MODULE_DEVICE_TABLE(of, fman_match);
2931 
2932 static struct platform_driver fman_driver = {
2933 	.driver = {
2934 		.name = "fsl-fman",
2935 		.of_match_table = fman_match,
2936 	},
2937 	.probe = fman_probe,
2938 };
2939 
2940 static int __init fman_load(void)
2941 {
2942 	int err;
2943 
2944 	pr_debug("FSL DPAA FMan driver\n");
2945 
2946 	err = platform_driver_register(&fman_driver);
2947 	if (err < 0)
2948 		pr_err("Error, platform_driver_register() = %d\n", err);
2949 
2950 	return err;
2951 }
2952 module_init(fman_load);
2953 
2954 static void __exit fman_unload(void)
2955 {
2956 	platform_driver_unregister(&fman_driver);
2957 }
2958 module_exit(fman_unload);
2959 
2960 MODULE_LICENSE("Dual BSD/GPL");
2961 MODULE_DESCRIPTION("Freescale DPAA Frame Manager driver");
2962