1 /*
2  * Copyright (C) 2017 Broadcom
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation version 2.
7  *
8  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
9  * kind, whether express or implied; without even the implied warranty
10  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11  * GNU General Public License for more details.
12  */
13 
14 /*
15  * Broadcom FlexRM Mailbox Driver
16  *
17  * Each Broadcom FlexSparx4 offload engine is implemented as an
18  * extension to Broadcom FlexRM ring manager. The FlexRM ring
19  * manager provides a set of rings which can be used to submit
20  * work to a FlexSparx4 offload engine.
21  *
22  * This driver creates a mailbox controller using a set of FlexRM
23  * rings where each mailbox channel represents a separate FlexRM ring.
24  */
25 
26 #include <asm/barrier.h>
27 #include <asm/byteorder.h>
28 #include <linux/atomic.h>
29 #include <linux/bitmap.h>
30 #include <linux/debugfs.h>
31 #include <linux/delay.h>
32 #include <linux/device.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/dmapool.h>
35 #include <linux/err.h>
36 #include <linux/interrupt.h>
37 #include <linux/kernel.h>
38 #include <linux/mailbox_controller.h>
39 #include <linux/mailbox_client.h>
40 #include <linux/mailbox/brcm-message.h>
41 #include <linux/module.h>
42 #include <linux/msi.h>
43 #include <linux/of_address.h>
44 #include <linux/of_irq.h>
45 #include <linux/platform_device.h>
46 #include <linux/spinlock.h>
47 
48 /* ====== FlexRM register defines ===== */
49 
50 /* FlexRM configuration */
51 #define RING_REGS_SIZE					0x10000
52 #define RING_DESC_SIZE					8
53 #define RING_DESC_INDEX(offset)				\
54 			((offset) / RING_DESC_SIZE)
55 #define RING_DESC_OFFSET(index)				\
56 			((index) * RING_DESC_SIZE)
57 #define RING_MAX_REQ_COUNT				1024
58 #define RING_BD_ALIGN_ORDER				12
59 #define RING_BD_ALIGN_CHECK(addr)			\
60 			(!((addr) & ((0x1 << RING_BD_ALIGN_ORDER) - 1)))
61 #define RING_BD_TOGGLE_INVALID(offset)			\
62 			(((offset) >> RING_BD_ALIGN_ORDER) & 0x1)
63 #define RING_BD_TOGGLE_VALID(offset)			\
64 			(!RING_BD_TOGGLE_INVALID(offset))
65 #define RING_BD_DESC_PER_REQ				32
66 #define RING_BD_DESC_COUNT				\
67 			(RING_MAX_REQ_COUNT * RING_BD_DESC_PER_REQ)
68 #define RING_BD_SIZE					\
69 			(RING_BD_DESC_COUNT * RING_DESC_SIZE)
70 #define RING_CMPL_ALIGN_ORDER				13
71 #define RING_CMPL_DESC_COUNT				RING_MAX_REQ_COUNT
72 #define RING_CMPL_SIZE					\
73 			(RING_CMPL_DESC_COUNT * RING_DESC_SIZE)
74 #define RING_VER_MAGIC					0x76303031
75 
76 /* Per-Ring register offsets */
77 #define RING_VER					0x000
78 #define RING_BD_START_ADDR				0x004
79 #define RING_BD_READ_PTR				0x008
80 #define RING_BD_WRITE_PTR				0x00c
81 #define RING_BD_READ_PTR_DDR_LS				0x010
82 #define RING_BD_READ_PTR_DDR_MS				0x014
83 #define RING_CMPL_START_ADDR				0x018
84 #define RING_CMPL_WRITE_PTR				0x01c
85 #define RING_NUM_REQ_RECV_LS				0x020
86 #define RING_NUM_REQ_RECV_MS				0x024
87 #define RING_NUM_REQ_TRANS_LS				0x028
88 #define RING_NUM_REQ_TRANS_MS				0x02c
89 #define RING_NUM_REQ_OUTSTAND				0x030
90 #define RING_CONTROL					0x034
91 #define RING_FLUSH_DONE					0x038
92 #define RING_MSI_ADDR_LS				0x03c
93 #define RING_MSI_ADDR_MS				0x040
94 #define RING_MSI_CONTROL				0x048
95 #define RING_BD_READ_PTR_DDR_CONTROL			0x04c
96 #define RING_MSI_DATA_VALUE				0x064
97 
98 /* Register RING_BD_START_ADDR fields */
99 #define BD_LAST_UPDATE_HW_SHIFT				28
100 #define BD_LAST_UPDATE_HW_MASK				0x1
101 #define BD_START_ADDR_VALUE(pa)				\
102 	((u32)((((dma_addr_t)(pa)) >> RING_BD_ALIGN_ORDER) & 0x0fffffff))
103 #define BD_START_ADDR_DECODE(val)			\
104 	((dma_addr_t)((val) & 0x0fffffff) << RING_BD_ALIGN_ORDER)
105 
106 /* Register RING_CMPL_START_ADDR fields */
107 #define CMPL_START_ADDR_VALUE(pa)			\
108 	((u32)((((u64)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x07ffffff))
109 
110 /* Register RING_CONTROL fields */
111 #define CONTROL_MASK_DISABLE_CONTROL			12
112 #define CONTROL_FLUSH_SHIFT				5
113 #define CONTROL_ACTIVE_SHIFT				4
114 #define CONTROL_RATE_ADAPT_MASK				0xf
115 #define CONTROL_RATE_DYNAMIC				0x0
116 #define CONTROL_RATE_FAST				0x8
117 #define CONTROL_RATE_MEDIUM				0x9
118 #define CONTROL_RATE_SLOW				0xa
119 #define CONTROL_RATE_IDLE				0xb
120 
121 /* Register RING_FLUSH_DONE fields */
122 #define FLUSH_DONE_MASK					0x1
123 
124 /* Register RING_MSI_CONTROL fields */
125 #define MSI_TIMER_VAL_SHIFT				16
126 #define MSI_TIMER_VAL_MASK				0xffff
127 #define MSI_ENABLE_SHIFT				15
128 #define MSI_ENABLE_MASK					0x1
129 #define MSI_COUNT_SHIFT					0
130 #define MSI_COUNT_MASK					0x3ff
131 
132 /* Register RING_BD_READ_PTR_DDR_CONTROL fields */
133 #define BD_READ_PTR_DDR_TIMER_VAL_SHIFT			16
134 #define BD_READ_PTR_DDR_TIMER_VAL_MASK			0xffff
135 #define BD_READ_PTR_DDR_ENABLE_SHIFT			15
136 #define BD_READ_PTR_DDR_ENABLE_MASK			0x1
137 
138 /* ====== FlexRM ring descriptor defines ===== */
139 
140 /* Completion descriptor format */
141 #define CMPL_OPAQUE_SHIFT			0
142 #define CMPL_OPAQUE_MASK			0xffff
143 #define CMPL_ENGINE_STATUS_SHIFT		16
144 #define CMPL_ENGINE_STATUS_MASK			0xffff
145 #define CMPL_DME_STATUS_SHIFT			32
146 #define CMPL_DME_STATUS_MASK			0xffff
147 #define CMPL_RM_STATUS_SHIFT			48
148 #define CMPL_RM_STATUS_MASK			0xffff
149 
150 /* Completion DME status code */
151 #define DME_STATUS_MEM_COR_ERR			BIT(0)
152 #define DME_STATUS_MEM_UCOR_ERR			BIT(1)
153 #define DME_STATUS_FIFO_UNDERFLOW		BIT(2)
154 #define DME_STATUS_FIFO_OVERFLOW		BIT(3)
155 #define DME_STATUS_RRESP_ERR			BIT(4)
156 #define DME_STATUS_BRESP_ERR			BIT(5)
157 #define DME_STATUS_ERROR_MASK			(DME_STATUS_MEM_COR_ERR | \
158 						 DME_STATUS_MEM_UCOR_ERR | \
159 						 DME_STATUS_FIFO_UNDERFLOW | \
160 						 DME_STATUS_FIFO_OVERFLOW | \
161 						 DME_STATUS_RRESP_ERR | \
162 						 DME_STATUS_BRESP_ERR)
163 
164 /* Completion RM status code */
165 #define RM_STATUS_CODE_SHIFT			0
166 #define RM_STATUS_CODE_MASK			0x3ff
167 #define RM_STATUS_CODE_GOOD			0x0
168 #define RM_STATUS_CODE_AE_TIMEOUT		0x3ff
169 
170 /* General descriptor format */
171 #define DESC_TYPE_SHIFT				60
172 #define DESC_TYPE_MASK				0xf
173 #define DESC_PAYLOAD_SHIFT			0
174 #define DESC_PAYLOAD_MASK			0x0fffffffffffffff
175 
176 /* Null descriptor format  */
177 #define NULL_TYPE				0
178 #define NULL_TOGGLE_SHIFT			58
179 #define NULL_TOGGLE_MASK			0x1
180 
181 /* Header descriptor format */
182 #define HEADER_TYPE				1
183 #define HEADER_TOGGLE_SHIFT			58
184 #define HEADER_TOGGLE_MASK			0x1
185 #define HEADER_ENDPKT_SHIFT			57
186 #define HEADER_ENDPKT_MASK			0x1
187 #define HEADER_STARTPKT_SHIFT			56
188 #define HEADER_STARTPKT_MASK			0x1
189 #define HEADER_BDCOUNT_SHIFT			36
190 #define HEADER_BDCOUNT_MASK			0x1f
191 #define HEADER_BDCOUNT_MAX			HEADER_BDCOUNT_MASK
192 #define HEADER_FLAGS_SHIFT			16
193 #define HEADER_FLAGS_MASK			0xffff
194 #define HEADER_OPAQUE_SHIFT			0
195 #define HEADER_OPAQUE_MASK			0xffff
196 
197 /* Source (SRC) descriptor format */
198 #define SRC_TYPE				2
199 #define SRC_LENGTH_SHIFT			44
200 #define SRC_LENGTH_MASK				0xffff
201 #define SRC_ADDR_SHIFT				0
202 #define SRC_ADDR_MASK				0x00000fffffffffff
203 
204 /* Destination (DST) descriptor format */
205 #define DST_TYPE				3
206 #define DST_LENGTH_SHIFT			44
207 #define DST_LENGTH_MASK				0xffff
208 #define DST_ADDR_SHIFT				0
209 #define DST_ADDR_MASK				0x00000fffffffffff
210 
211 /* Immediate (IMM) descriptor format */
212 #define IMM_TYPE				4
213 #define IMM_DATA_SHIFT				0
214 #define IMM_DATA_MASK				0x0fffffffffffffff
215 
216 /* Next pointer (NPTR) descriptor format */
217 #define NPTR_TYPE				5
218 #define NPTR_TOGGLE_SHIFT			58
219 #define NPTR_TOGGLE_MASK			0x1
220 #define NPTR_ADDR_SHIFT				0
221 #define NPTR_ADDR_MASK				0x00000fffffffffff
222 
223 /* Mega source (MSRC) descriptor format */
224 #define MSRC_TYPE				6
225 #define MSRC_LENGTH_SHIFT			44
226 #define MSRC_LENGTH_MASK			0xffff
227 #define MSRC_ADDR_SHIFT				0
228 #define MSRC_ADDR_MASK				0x00000fffffffffff
229 
230 /* Mega destination (MDST) descriptor format */
231 #define MDST_TYPE				7
232 #define MDST_LENGTH_SHIFT			44
233 #define MDST_LENGTH_MASK			0xffff
234 #define MDST_ADDR_SHIFT				0
235 #define MDST_ADDR_MASK				0x00000fffffffffff
236 
237 /* Source with tlast (SRCT) descriptor format */
238 #define SRCT_TYPE				8
239 #define SRCT_LENGTH_SHIFT			44
240 #define SRCT_LENGTH_MASK			0xffff
241 #define SRCT_ADDR_SHIFT				0
242 #define SRCT_ADDR_MASK				0x00000fffffffffff
243 
244 /* Destination with tlast (DSTT) descriptor format */
245 #define DSTT_TYPE				9
246 #define DSTT_LENGTH_SHIFT			44
247 #define DSTT_LENGTH_MASK			0xffff
248 #define DSTT_ADDR_SHIFT				0
249 #define DSTT_ADDR_MASK				0x00000fffffffffff
250 
251 /* Immediate with tlast (IMMT) descriptor format */
252 #define IMMT_TYPE				10
253 #define IMMT_DATA_SHIFT				0
254 #define IMMT_DATA_MASK				0x0fffffffffffffff
255 
256 /* Descriptor helper macros */
257 #define DESC_DEC(_d, _s, _m)			(((_d) >> (_s)) & (_m))
258 #define DESC_ENC(_d, _v, _s, _m)		\
259 			do { \
260 				(_d) &= ~((u64)(_m) << (_s)); \
261 				(_d) |= (((u64)(_v) & (_m)) << (_s)); \
262 			} while (0)
263 
264 /* ====== FlexRM data structures ===== */
265 
266 struct flexrm_ring {
267 	/* Unprotected members */
268 	int num;
269 	struct flexrm_mbox *mbox;
270 	void __iomem *regs;
271 	bool irq_requested;
272 	unsigned int irq;
273 	cpumask_t irq_aff_hint;
274 	unsigned int msi_timer_val;
275 	unsigned int msi_count_threshold;
276 	struct brcm_message *requests[RING_MAX_REQ_COUNT];
277 	void *bd_base;
278 	dma_addr_t bd_dma_base;
279 	u32 bd_write_offset;
280 	void *cmpl_base;
281 	dma_addr_t cmpl_dma_base;
282 	/* Atomic stats */
283 	atomic_t msg_send_count;
284 	atomic_t msg_cmpl_count;
285 	/* Protected members */
286 	spinlock_t lock;
287 	DECLARE_BITMAP(requests_bmap, RING_MAX_REQ_COUNT);
288 	u32 cmpl_read_offset;
289 };
290 
291 struct flexrm_mbox {
292 	struct device *dev;
293 	void __iomem *regs;
294 	u32 num_rings;
295 	struct flexrm_ring *rings;
296 	struct dma_pool *bd_pool;
297 	struct dma_pool *cmpl_pool;
298 	struct dentry *root;
299 	struct mbox_controller controller;
300 };
301 
302 /* ====== FlexRM ring descriptor helper routines ===== */
303 
304 static u64 flexrm_read_desc(void *desc_ptr)
305 {
306 	return le64_to_cpu(*((u64 *)desc_ptr));
307 }
308 
309 static void flexrm_write_desc(void *desc_ptr, u64 desc)
310 {
311 	*((u64 *)desc_ptr) = cpu_to_le64(desc);
312 }
313 
314 static u32 flexrm_cmpl_desc_to_reqid(u64 cmpl_desc)
315 {
316 	return (u32)(cmpl_desc & CMPL_OPAQUE_MASK);
317 }
318 
319 static int flexrm_cmpl_desc_to_error(u64 cmpl_desc)
320 {
321 	u32 status;
322 
323 	status = DESC_DEC(cmpl_desc, CMPL_DME_STATUS_SHIFT,
324 			  CMPL_DME_STATUS_MASK);
325 	if (status & DME_STATUS_ERROR_MASK)
326 		return -EIO;
327 
328 	status = DESC_DEC(cmpl_desc, CMPL_RM_STATUS_SHIFT,
329 			  CMPL_RM_STATUS_MASK);
330 	status &= RM_STATUS_CODE_MASK;
331 	if (status == RM_STATUS_CODE_AE_TIMEOUT)
332 		return -ETIMEDOUT;
333 
334 	return 0;
335 }
336 
337 static bool flexrm_is_next_table_desc(void *desc_ptr)
338 {
339 	u64 desc = flexrm_read_desc(desc_ptr);
340 	u32 type = DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
341 
342 	return (type == NPTR_TYPE) ? true : false;
343 }
344 
345 static u64 flexrm_next_table_desc(u32 toggle, dma_addr_t next_addr)
346 {
347 	u64 desc = 0;
348 
349 	DESC_ENC(desc, NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
350 	DESC_ENC(desc, toggle, NPTR_TOGGLE_SHIFT, NPTR_TOGGLE_MASK);
351 	DESC_ENC(desc, next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK);
352 
353 	return desc;
354 }
355 
356 static u64 flexrm_null_desc(u32 toggle)
357 {
358 	u64 desc = 0;
359 
360 	DESC_ENC(desc, NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
361 	DESC_ENC(desc, toggle, NULL_TOGGLE_SHIFT, NULL_TOGGLE_MASK);
362 
363 	return desc;
364 }
365 
366 static u32 flexrm_estimate_header_desc_count(u32 nhcnt)
367 {
368 	u32 hcnt = nhcnt / HEADER_BDCOUNT_MAX;
369 
370 	if (!(nhcnt % HEADER_BDCOUNT_MAX))
371 		hcnt += 1;
372 
373 	return hcnt;
374 }
375 
376 static void flexrm_flip_header_toggle(void *desc_ptr)
377 {
378 	u64 desc = flexrm_read_desc(desc_ptr);
379 
380 	if (desc & ((u64)0x1 << HEADER_TOGGLE_SHIFT))
381 		desc &= ~((u64)0x1 << HEADER_TOGGLE_SHIFT);
382 	else
383 		desc |= ((u64)0x1 << HEADER_TOGGLE_SHIFT);
384 
385 	flexrm_write_desc(desc_ptr, desc);
386 }
387 
388 static u64 flexrm_header_desc(u32 toggle, u32 startpkt, u32 endpkt,
389 			       u32 bdcount, u32 flags, u32 opaque)
390 {
391 	u64 desc = 0;
392 
393 	DESC_ENC(desc, HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
394 	DESC_ENC(desc, toggle, HEADER_TOGGLE_SHIFT, HEADER_TOGGLE_MASK);
395 	DESC_ENC(desc, startpkt, HEADER_STARTPKT_SHIFT, HEADER_STARTPKT_MASK);
396 	DESC_ENC(desc, endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK);
397 	DESC_ENC(desc, bdcount, HEADER_BDCOUNT_SHIFT, HEADER_BDCOUNT_MASK);
398 	DESC_ENC(desc, flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK);
399 	DESC_ENC(desc, opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK);
400 
401 	return desc;
402 }
403 
404 static void flexrm_enqueue_desc(u32 nhpos, u32 nhcnt, u32 reqid,
405 				 u64 desc, void **desc_ptr, u32 *toggle,
406 				 void *start_desc, void *end_desc)
407 {
408 	u64 d;
409 	u32 nhavail, _toggle, _startpkt, _endpkt, _bdcount;
410 
411 	/* Sanity check */
412 	if (nhcnt <= nhpos)
413 		return;
414 
415 	/*
416 	 * Each request or packet start with a HEADER descriptor followed
417 	 * by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST,
418 	 * DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors
419 	 * following a HEADER descriptor is represented by BDCOUNT field
420 	 * of HEADER descriptor. The max value of BDCOUNT field is 31 which
421 	 * means we can only have 31 non-HEADER descriptors following one
422 	 * HEADER descriptor.
423 	 *
424 	 * In general use, number of non-HEADER descriptors can easily go
425 	 * beyond 31. To tackle this situation, we have packet (or request)
426 	 * extenstion bits (STARTPKT and ENDPKT) in the HEADER descriptor.
427 	 *
428 	 * To use packet extension, the first HEADER descriptor of request
429 	 * (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate
430 	 * HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last
431 	 * HEADER descriptor will have STARTPKT=0 and ENDPKT=1. Also, the
432 	 * TOGGLE bit of the first HEADER will be set to invalid state to
433 	 * ensure that FlexRM does not start fetching descriptors till all
434 	 * descriptors are enqueued. The user of this function will flip
435 	 * the TOGGLE bit of first HEADER after all descriptors are
436 	 * enqueued.
437 	 */
438 
439 	if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) {
440 		/* Prepare the header descriptor */
441 		nhavail = (nhcnt - nhpos);
442 		_toggle = (nhpos == 0) ? !(*toggle) : (*toggle);
443 		_startpkt = (nhpos == 0) ? 0x1 : 0x0;
444 		_endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0;
445 		_bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ?
446 				nhavail : HEADER_BDCOUNT_MAX;
447 		if (nhavail <= HEADER_BDCOUNT_MAX)
448 			_bdcount = nhavail;
449 		else
450 			_bdcount = HEADER_BDCOUNT_MAX;
451 		d = flexrm_header_desc(_toggle, _startpkt, _endpkt,
452 					_bdcount, 0x0, reqid);
453 
454 		/* Write header descriptor */
455 		flexrm_write_desc(*desc_ptr, d);
456 
457 		/* Point to next descriptor */
458 		*desc_ptr += sizeof(desc);
459 		if (*desc_ptr == end_desc)
460 			*desc_ptr = start_desc;
461 
462 		/* Skip next pointer descriptors */
463 		while (flexrm_is_next_table_desc(*desc_ptr)) {
464 			*toggle = (*toggle) ? 0 : 1;
465 			*desc_ptr += sizeof(desc);
466 			if (*desc_ptr == end_desc)
467 				*desc_ptr = start_desc;
468 		}
469 	}
470 
471 	/* Write desired descriptor */
472 	flexrm_write_desc(*desc_ptr, desc);
473 
474 	/* Point to next descriptor */
475 	*desc_ptr += sizeof(desc);
476 	if (*desc_ptr == end_desc)
477 		*desc_ptr = start_desc;
478 
479 	/* Skip next pointer descriptors */
480 	while (flexrm_is_next_table_desc(*desc_ptr)) {
481 		*toggle = (*toggle) ? 0 : 1;
482 		*desc_ptr += sizeof(desc);
483 		if (*desc_ptr == end_desc)
484 			*desc_ptr = start_desc;
485 	}
486 }
487 
488 static u64 flexrm_src_desc(dma_addr_t addr, unsigned int length)
489 {
490 	u64 desc = 0;
491 
492 	DESC_ENC(desc, SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
493 	DESC_ENC(desc, length, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK);
494 	DESC_ENC(desc, addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK);
495 
496 	return desc;
497 }
498 
499 static u64 flexrm_msrc_desc(dma_addr_t addr, unsigned int length_div_16)
500 {
501 	u64 desc = 0;
502 
503 	DESC_ENC(desc, MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
504 	DESC_ENC(desc, length_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK);
505 	DESC_ENC(desc, addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK);
506 
507 	return desc;
508 }
509 
510 static u64 flexrm_dst_desc(dma_addr_t addr, unsigned int length)
511 {
512 	u64 desc = 0;
513 
514 	DESC_ENC(desc, DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
515 	DESC_ENC(desc, length, DST_LENGTH_SHIFT, DST_LENGTH_MASK);
516 	DESC_ENC(desc, addr, DST_ADDR_SHIFT, DST_ADDR_MASK);
517 
518 	return desc;
519 }
520 
521 static u64 flexrm_mdst_desc(dma_addr_t addr, unsigned int length_div_16)
522 {
523 	u64 desc = 0;
524 
525 	DESC_ENC(desc, MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
526 	DESC_ENC(desc, length_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK);
527 	DESC_ENC(desc, addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK);
528 
529 	return desc;
530 }
531 
532 static u64 flexrm_imm_desc(u64 data)
533 {
534 	u64 desc = 0;
535 
536 	DESC_ENC(desc, IMM_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
537 	DESC_ENC(desc, data, IMM_DATA_SHIFT, IMM_DATA_MASK);
538 
539 	return desc;
540 }
541 
542 static u64 flexrm_srct_desc(dma_addr_t addr, unsigned int length)
543 {
544 	u64 desc = 0;
545 
546 	DESC_ENC(desc, SRCT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
547 	DESC_ENC(desc, length, SRCT_LENGTH_SHIFT, SRCT_LENGTH_MASK);
548 	DESC_ENC(desc, addr, SRCT_ADDR_SHIFT, SRCT_ADDR_MASK);
549 
550 	return desc;
551 }
552 
553 static u64 flexrm_dstt_desc(dma_addr_t addr, unsigned int length)
554 {
555 	u64 desc = 0;
556 
557 	DESC_ENC(desc, DSTT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
558 	DESC_ENC(desc, length, DSTT_LENGTH_SHIFT, DSTT_LENGTH_MASK);
559 	DESC_ENC(desc, addr, DSTT_ADDR_SHIFT, DSTT_ADDR_MASK);
560 
561 	return desc;
562 }
563 
564 static u64 flexrm_immt_desc(u64 data)
565 {
566 	u64 desc = 0;
567 
568 	DESC_ENC(desc, IMMT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
569 	DESC_ENC(desc, data, IMMT_DATA_SHIFT, IMMT_DATA_MASK);
570 
571 	return desc;
572 }
573 
574 static bool flexrm_spu_sanity_check(struct brcm_message *msg)
575 {
576 	struct scatterlist *sg;
577 
578 	if (!msg->spu.src || !msg->spu.dst)
579 		return false;
580 	for (sg = msg->spu.src; sg; sg = sg_next(sg)) {
581 		if (sg->length & 0xf) {
582 			if (sg->length > SRC_LENGTH_MASK)
583 				return false;
584 		} else {
585 			if (sg->length > (MSRC_LENGTH_MASK * 16))
586 				return false;
587 		}
588 	}
589 	for (sg = msg->spu.dst; sg; sg = sg_next(sg)) {
590 		if (sg->length & 0xf) {
591 			if (sg->length > DST_LENGTH_MASK)
592 				return false;
593 		} else {
594 			if (sg->length > (MDST_LENGTH_MASK * 16))
595 				return false;
596 		}
597 	}
598 
599 	return true;
600 }
601 
602 static u32 flexrm_spu_estimate_nonheader_desc_count(struct brcm_message *msg)
603 {
604 	u32 cnt = 0;
605 	unsigned int dst_target = 0;
606 	struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
607 
608 	while (src_sg || dst_sg) {
609 		if (src_sg) {
610 			cnt++;
611 			dst_target = src_sg->length;
612 			src_sg = sg_next(src_sg);
613 		} else
614 			dst_target = UINT_MAX;
615 
616 		while (dst_target && dst_sg) {
617 			cnt++;
618 			if (dst_sg->length < dst_target)
619 				dst_target -= dst_sg->length;
620 			else
621 				dst_target = 0;
622 			dst_sg = sg_next(dst_sg);
623 		}
624 	}
625 
626 	return cnt;
627 }
628 
629 static int flexrm_spu_dma_map(struct device *dev, struct brcm_message *msg)
630 {
631 	int rc;
632 
633 	rc = dma_map_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
634 			DMA_TO_DEVICE);
635 	if (rc < 0)
636 		return rc;
637 
638 	rc = dma_map_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
639 			DMA_FROM_DEVICE);
640 	if (rc < 0) {
641 		dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
642 			     DMA_TO_DEVICE);
643 		return rc;
644 	}
645 
646 	return 0;
647 }
648 
649 static void flexrm_spu_dma_unmap(struct device *dev, struct brcm_message *msg)
650 {
651 	dma_unmap_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
652 		     DMA_FROM_DEVICE);
653 	dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
654 		     DMA_TO_DEVICE);
655 }
656 
657 static void *flexrm_spu_write_descs(struct brcm_message *msg, u32 nhcnt,
658 				     u32 reqid, void *desc_ptr, u32 toggle,
659 				     void *start_desc, void *end_desc)
660 {
661 	u64 d;
662 	u32 nhpos = 0;
663 	void *orig_desc_ptr = desc_ptr;
664 	unsigned int dst_target = 0;
665 	struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
666 
667 	while (src_sg || dst_sg) {
668 		if (src_sg) {
669 			if (sg_dma_len(src_sg) & 0xf)
670 				d = flexrm_src_desc(sg_dma_address(src_sg),
671 						     sg_dma_len(src_sg));
672 			else
673 				d = flexrm_msrc_desc(sg_dma_address(src_sg),
674 						      sg_dma_len(src_sg)/16);
675 			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
676 					     d, &desc_ptr, &toggle,
677 					     start_desc, end_desc);
678 			nhpos++;
679 			dst_target = sg_dma_len(src_sg);
680 			src_sg = sg_next(src_sg);
681 		} else
682 			dst_target = UINT_MAX;
683 
684 		while (dst_target && dst_sg) {
685 			if (sg_dma_len(dst_sg) & 0xf)
686 				d = flexrm_dst_desc(sg_dma_address(dst_sg),
687 						     sg_dma_len(dst_sg));
688 			else
689 				d = flexrm_mdst_desc(sg_dma_address(dst_sg),
690 						      sg_dma_len(dst_sg)/16);
691 			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
692 					     d, &desc_ptr, &toggle,
693 					     start_desc, end_desc);
694 			nhpos++;
695 			if (sg_dma_len(dst_sg) < dst_target)
696 				dst_target -= sg_dma_len(dst_sg);
697 			else
698 				dst_target = 0;
699 			dst_sg = sg_next(dst_sg);
700 		}
701 	}
702 
703 	/* Null descriptor with invalid toggle bit */
704 	flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
705 
706 	/* Ensure that descriptors have been written to memory */
707 	wmb();
708 
709 	/* Flip toggle bit in header */
710 	flexrm_flip_header_toggle(orig_desc_ptr);
711 
712 	return desc_ptr;
713 }
714 
715 static bool flexrm_sba_sanity_check(struct brcm_message *msg)
716 {
717 	u32 i;
718 
719 	if (!msg->sba.cmds || !msg->sba.cmds_count)
720 		return false;
721 
722 	for (i = 0; i < msg->sba.cmds_count; i++) {
723 		if (((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
724 		     (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C)) &&
725 		    (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT))
726 			return false;
727 		if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) &&
728 		    (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
729 			return false;
730 		if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C) &&
731 		    (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
732 			return false;
733 		if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP) &&
734 		    (msg->sba.cmds[i].resp_len > DSTT_LENGTH_MASK))
735 			return false;
736 		if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT) &&
737 		    (msg->sba.cmds[i].data_len > DSTT_LENGTH_MASK))
738 			return false;
739 	}
740 
741 	return true;
742 }
743 
744 static u32 flexrm_sba_estimate_nonheader_desc_count(struct brcm_message *msg)
745 {
746 	u32 i, cnt;
747 
748 	cnt = 0;
749 	for (i = 0; i < msg->sba.cmds_count; i++) {
750 		cnt++;
751 
752 		if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
753 		    (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C))
754 			cnt++;
755 
756 		if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP)
757 			cnt++;
758 
759 		if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT)
760 			cnt++;
761 	}
762 
763 	return cnt;
764 }
765 
766 static void *flexrm_sba_write_descs(struct brcm_message *msg, u32 nhcnt,
767 				     u32 reqid, void *desc_ptr, u32 toggle,
768 				     void *start_desc, void *end_desc)
769 {
770 	u64 d;
771 	u32 i, nhpos = 0;
772 	struct brcm_sba_command *c;
773 	void *orig_desc_ptr = desc_ptr;
774 
775 	/* Convert SBA commands into descriptors */
776 	for (i = 0; i < msg->sba.cmds_count; i++) {
777 		c = &msg->sba.cmds[i];
778 
779 		if ((c->flags & BRCM_SBA_CMD_HAS_RESP) &&
780 		    (c->flags & BRCM_SBA_CMD_HAS_OUTPUT)) {
781 			/* Destination response descriptor */
782 			d = flexrm_dst_desc(c->resp, c->resp_len);
783 			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
784 					     d, &desc_ptr, &toggle,
785 					     start_desc, end_desc);
786 			nhpos++;
787 		} else if (c->flags & BRCM_SBA_CMD_HAS_RESP) {
788 			/* Destination response with tlast descriptor */
789 			d = flexrm_dstt_desc(c->resp, c->resp_len);
790 			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
791 					     d, &desc_ptr, &toggle,
792 					     start_desc, end_desc);
793 			nhpos++;
794 		}
795 
796 		if (c->flags & BRCM_SBA_CMD_HAS_OUTPUT) {
797 			/* Destination with tlast descriptor */
798 			d = flexrm_dstt_desc(c->data, c->data_len);
799 			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
800 					     d, &desc_ptr, &toggle,
801 					     start_desc, end_desc);
802 			nhpos++;
803 		}
804 
805 		if (c->flags & BRCM_SBA_CMD_TYPE_B) {
806 			/* Command as immediate descriptor */
807 			d = flexrm_imm_desc(c->cmd);
808 			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
809 					     d, &desc_ptr, &toggle,
810 					     start_desc, end_desc);
811 			nhpos++;
812 		} else {
813 			/* Command as immediate descriptor with tlast */
814 			d = flexrm_immt_desc(c->cmd);
815 			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
816 					     d, &desc_ptr, &toggle,
817 					     start_desc, end_desc);
818 			nhpos++;
819 		}
820 
821 		if ((c->flags & BRCM_SBA_CMD_TYPE_B) ||
822 		    (c->flags & BRCM_SBA_CMD_TYPE_C)) {
823 			/* Source with tlast descriptor */
824 			d = flexrm_srct_desc(c->data, c->data_len);
825 			flexrm_enqueue_desc(nhpos, nhcnt, reqid,
826 					     d, &desc_ptr, &toggle,
827 					     start_desc, end_desc);
828 			nhpos++;
829 		}
830 	}
831 
832 	/* Null descriptor with invalid toggle bit */
833 	flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
834 
835 	/* Ensure that descriptors have been written to memory */
836 	wmb();
837 
838 	/* Flip toggle bit in header */
839 	flexrm_flip_header_toggle(orig_desc_ptr);
840 
841 	return desc_ptr;
842 }
843 
844 static bool flexrm_sanity_check(struct brcm_message *msg)
845 {
846 	if (!msg)
847 		return false;
848 
849 	switch (msg->type) {
850 	case BRCM_MESSAGE_SPU:
851 		return flexrm_spu_sanity_check(msg);
852 	case BRCM_MESSAGE_SBA:
853 		return flexrm_sba_sanity_check(msg);
854 	default:
855 		return false;
856 	};
857 }
858 
859 static u32 flexrm_estimate_nonheader_desc_count(struct brcm_message *msg)
860 {
861 	if (!msg)
862 		return 0;
863 
864 	switch (msg->type) {
865 	case BRCM_MESSAGE_SPU:
866 		return flexrm_spu_estimate_nonheader_desc_count(msg);
867 	case BRCM_MESSAGE_SBA:
868 		return flexrm_sba_estimate_nonheader_desc_count(msg);
869 	default:
870 		return 0;
871 	};
872 }
873 
874 static int flexrm_dma_map(struct device *dev, struct brcm_message *msg)
875 {
876 	if (!dev || !msg)
877 		return -EINVAL;
878 
879 	switch (msg->type) {
880 	case BRCM_MESSAGE_SPU:
881 		return flexrm_spu_dma_map(dev, msg);
882 	default:
883 		break;
884 	}
885 
886 	return 0;
887 }
888 
889 static void flexrm_dma_unmap(struct device *dev, struct brcm_message *msg)
890 {
891 	if (!dev || !msg)
892 		return;
893 
894 	switch (msg->type) {
895 	case BRCM_MESSAGE_SPU:
896 		flexrm_spu_dma_unmap(dev, msg);
897 		break;
898 	default:
899 		break;
900 	}
901 }
902 
903 static void *flexrm_write_descs(struct brcm_message *msg, u32 nhcnt,
904 				u32 reqid, void *desc_ptr, u32 toggle,
905 				void *start_desc, void *end_desc)
906 {
907 	if (!msg || !desc_ptr || !start_desc || !end_desc)
908 		return ERR_PTR(-ENOTSUPP);
909 
910 	if ((desc_ptr < start_desc) || (end_desc <= desc_ptr))
911 		return ERR_PTR(-ERANGE);
912 
913 	switch (msg->type) {
914 	case BRCM_MESSAGE_SPU:
915 		return flexrm_spu_write_descs(msg, nhcnt, reqid,
916 					       desc_ptr, toggle,
917 					       start_desc, end_desc);
918 	case BRCM_MESSAGE_SBA:
919 		return flexrm_sba_write_descs(msg, nhcnt, reqid,
920 					       desc_ptr, toggle,
921 					       start_desc, end_desc);
922 	default:
923 		return ERR_PTR(-ENOTSUPP);
924 	};
925 }
926 
927 /* ====== FlexRM driver helper routines ===== */
928 
929 static void flexrm_write_config_in_seqfile(struct flexrm_mbox *mbox,
930 					   struct seq_file *file)
931 {
932 	int i;
933 	const char *state;
934 	struct flexrm_ring *ring;
935 
936 	seq_printf(file, "%-5s %-9s %-18s %-10s %-18s %-10s\n",
937 		   "Ring#", "State", "BD_Addr", "BD_Size",
938 		   "Cmpl_Addr", "Cmpl_Size");
939 
940 	for (i = 0; i < mbox->num_rings; i++) {
941 		ring = &mbox->rings[i];
942 		if (readl(ring->regs + RING_CONTROL) &
943 		    BIT(CONTROL_ACTIVE_SHIFT))
944 			state = "active";
945 		else
946 			state = "inactive";
947 		seq_printf(file,
948 			   "%-5d %-9s 0x%016llx 0x%08x 0x%016llx 0x%08x\n",
949 			   ring->num, state,
950 			   (unsigned long long)ring->bd_dma_base,
951 			   (u32)RING_BD_SIZE,
952 			   (unsigned long long)ring->cmpl_dma_base,
953 			   (u32)RING_CMPL_SIZE);
954 	}
955 }
956 
957 static void flexrm_write_stats_in_seqfile(struct flexrm_mbox *mbox,
958 					  struct seq_file *file)
959 {
960 	int i;
961 	u32 val, bd_read_offset;
962 	struct flexrm_ring *ring;
963 
964 	seq_printf(file, "%-5s %-10s %-10s %-10s %-11s %-11s\n",
965 		   "Ring#", "BD_Read", "BD_Write",
966 		   "Cmpl_Read", "Submitted", "Completed");
967 
968 	for (i = 0; i < mbox->num_rings; i++) {
969 		ring = &mbox->rings[i];
970 		bd_read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
971 		val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
972 		bd_read_offset *= RING_DESC_SIZE;
973 		bd_read_offset += (u32)(BD_START_ADDR_DECODE(val) -
974 					ring->bd_dma_base);
975 		seq_printf(file, "%-5d 0x%08x 0x%08x 0x%08x %-11d %-11d\n",
976 			   ring->num,
977 			   (u32)bd_read_offset,
978 			   (u32)ring->bd_write_offset,
979 			   (u32)ring->cmpl_read_offset,
980 			   (u32)atomic_read(&ring->msg_send_count),
981 			   (u32)atomic_read(&ring->msg_cmpl_count));
982 	}
983 }
984 
985 static int flexrm_new_request(struct flexrm_ring *ring,
986 				struct brcm_message *batch_msg,
987 				struct brcm_message *msg)
988 {
989 	void *next;
990 	unsigned long flags;
991 	u32 val, count, nhcnt;
992 	u32 read_offset, write_offset;
993 	bool exit_cleanup = false;
994 	int ret = 0, reqid;
995 
996 	/* Do sanity check on message */
997 	if (!flexrm_sanity_check(msg))
998 		return -EIO;
999 	msg->error = 0;
1000 
1001 	/* If no requests possible then save data pointer and goto done. */
1002 	spin_lock_irqsave(&ring->lock, flags);
1003 	reqid = bitmap_find_free_region(ring->requests_bmap,
1004 					RING_MAX_REQ_COUNT, 0);
1005 	spin_unlock_irqrestore(&ring->lock, flags);
1006 	if (reqid < 0)
1007 		return -ENOSPC;
1008 	ring->requests[reqid] = msg;
1009 
1010 	/* Do DMA mappings for the message */
1011 	ret = flexrm_dma_map(ring->mbox->dev, msg);
1012 	if (ret < 0) {
1013 		ring->requests[reqid] = NULL;
1014 		spin_lock_irqsave(&ring->lock, flags);
1015 		bitmap_release_region(ring->requests_bmap, reqid, 0);
1016 		spin_unlock_irqrestore(&ring->lock, flags);
1017 		return ret;
1018 	}
1019 
1020 	/* Determine current HW BD read offset */
1021 	read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
1022 	val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
1023 	read_offset *= RING_DESC_SIZE;
1024 	read_offset += (u32)(BD_START_ADDR_DECODE(val) - ring->bd_dma_base);
1025 
1026 	/*
1027 	 * Number required descriptors = number of non-header descriptors +
1028 	 *				 number of header descriptors +
1029 	 *				 1x null descriptor
1030 	 */
1031 	nhcnt = flexrm_estimate_nonheader_desc_count(msg);
1032 	count = flexrm_estimate_header_desc_count(nhcnt) + nhcnt + 1;
1033 
1034 	/* Check for available descriptor space. */
1035 	write_offset = ring->bd_write_offset;
1036 	while (count) {
1037 		if (!flexrm_is_next_table_desc(ring->bd_base + write_offset))
1038 			count--;
1039 		write_offset += RING_DESC_SIZE;
1040 		if (write_offset == RING_BD_SIZE)
1041 			write_offset = 0x0;
1042 		if (write_offset == read_offset)
1043 			break;
1044 	}
1045 	if (count) {
1046 		ret = -ENOSPC;
1047 		exit_cleanup = true;
1048 		goto exit;
1049 	}
1050 
1051 	/* Write descriptors to ring */
1052 	next = flexrm_write_descs(msg, nhcnt, reqid,
1053 			ring->bd_base + ring->bd_write_offset,
1054 			RING_BD_TOGGLE_VALID(ring->bd_write_offset),
1055 			ring->bd_base, ring->bd_base + RING_BD_SIZE);
1056 	if (IS_ERR(next)) {
1057 		ret = PTR_ERR(next);
1058 		exit_cleanup = true;
1059 		goto exit;
1060 	}
1061 
1062 	/* Save ring BD write offset */
1063 	ring->bd_write_offset = (unsigned long)(next - ring->bd_base);
1064 
1065 	/* Increment number of messages sent */
1066 	atomic_inc_return(&ring->msg_send_count);
1067 
1068 exit:
1069 	/* Update error status in message */
1070 	msg->error = ret;
1071 
1072 	/* Cleanup if we failed */
1073 	if (exit_cleanup) {
1074 		flexrm_dma_unmap(ring->mbox->dev, msg);
1075 		ring->requests[reqid] = NULL;
1076 		spin_lock_irqsave(&ring->lock, flags);
1077 		bitmap_release_region(ring->requests_bmap, reqid, 0);
1078 		spin_unlock_irqrestore(&ring->lock, flags);
1079 	}
1080 
1081 	return ret;
1082 }
1083 
1084 static int flexrm_process_completions(struct flexrm_ring *ring)
1085 {
1086 	u64 desc;
1087 	int err, count = 0;
1088 	unsigned long flags;
1089 	struct brcm_message *msg = NULL;
1090 	u32 reqid, cmpl_read_offset, cmpl_write_offset;
1091 	struct mbox_chan *chan = &ring->mbox->controller.chans[ring->num];
1092 
1093 	spin_lock_irqsave(&ring->lock, flags);
1094 
1095 	/*
1096 	 * Get current completion read and write offset
1097 	 *
1098 	 * Note: We should read completion write pointer atleast once
1099 	 * after we get a MSI interrupt because HW maintains internal
1100 	 * MSI status which will allow next MSI interrupt only after
1101 	 * completion write pointer is read.
1102 	 */
1103 	cmpl_write_offset = readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
1104 	cmpl_write_offset *= RING_DESC_SIZE;
1105 	cmpl_read_offset = ring->cmpl_read_offset;
1106 	ring->cmpl_read_offset = cmpl_write_offset;
1107 
1108 	spin_unlock_irqrestore(&ring->lock, flags);
1109 
1110 	/* For each completed request notify mailbox clients */
1111 	reqid = 0;
1112 	while (cmpl_read_offset != cmpl_write_offset) {
1113 		/* Dequeue next completion descriptor */
1114 		desc = *((u64 *)(ring->cmpl_base + cmpl_read_offset));
1115 
1116 		/* Next read offset */
1117 		cmpl_read_offset += RING_DESC_SIZE;
1118 		if (cmpl_read_offset == RING_CMPL_SIZE)
1119 			cmpl_read_offset = 0;
1120 
1121 		/* Decode error from completion descriptor */
1122 		err = flexrm_cmpl_desc_to_error(desc);
1123 		if (err < 0) {
1124 			dev_warn(ring->mbox->dev,
1125 			"ring%d got completion desc=0x%lx with error %d\n",
1126 			ring->num, (unsigned long)desc, err);
1127 		}
1128 
1129 		/* Determine request id from completion descriptor */
1130 		reqid = flexrm_cmpl_desc_to_reqid(desc);
1131 
1132 		/* Determine message pointer based on reqid */
1133 		msg = ring->requests[reqid];
1134 		if (!msg) {
1135 			dev_warn(ring->mbox->dev,
1136 			"ring%d null msg pointer for completion desc=0x%lx\n",
1137 			ring->num, (unsigned long)desc);
1138 			continue;
1139 		}
1140 
1141 		/* Release reqid for recycling */
1142 		ring->requests[reqid] = NULL;
1143 		spin_lock_irqsave(&ring->lock, flags);
1144 		bitmap_release_region(ring->requests_bmap, reqid, 0);
1145 		spin_unlock_irqrestore(&ring->lock, flags);
1146 
1147 		/* Unmap DMA mappings */
1148 		flexrm_dma_unmap(ring->mbox->dev, msg);
1149 
1150 		/* Give-back message to mailbox client */
1151 		msg->error = err;
1152 		mbox_chan_received_data(chan, msg);
1153 
1154 		/* Increment number of completions processed */
1155 		atomic_inc_return(&ring->msg_cmpl_count);
1156 		count++;
1157 	}
1158 
1159 	return count;
1160 }
1161 
1162 /* ====== FlexRM Debugfs callbacks ====== */
1163 
1164 static int flexrm_debugfs_conf_show(struct seq_file *file, void *offset)
1165 {
1166 	struct flexrm_mbox *mbox = dev_get_drvdata(file->private);
1167 
1168 	/* Write config in file */
1169 	flexrm_write_config_in_seqfile(mbox, file);
1170 
1171 	return 0;
1172 }
1173 
1174 static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset)
1175 {
1176 	struct flexrm_mbox *mbox = dev_get_drvdata(file->private);
1177 
1178 	/* Write stats in file */
1179 	flexrm_write_stats_in_seqfile(mbox, file);
1180 
1181 	return 0;
1182 }
1183 
1184 /* ====== FlexRM interrupt handler ===== */
1185 
1186 static irqreturn_t flexrm_irq_event(int irq, void *dev_id)
1187 {
1188 	/* We only have MSI for completions so just wakeup IRQ thread */
1189 	/* Ring related errors will be informed via completion descriptors */
1190 
1191 	return IRQ_WAKE_THREAD;
1192 }
1193 
1194 static irqreturn_t flexrm_irq_thread(int irq, void *dev_id)
1195 {
1196 	flexrm_process_completions(dev_id);
1197 
1198 	return IRQ_HANDLED;
1199 }
1200 
1201 /* ====== FlexRM mailbox callbacks ===== */
1202 
1203 static int flexrm_send_data(struct mbox_chan *chan, void *data)
1204 {
1205 	int i, rc;
1206 	struct flexrm_ring *ring = chan->con_priv;
1207 	struct brcm_message *msg = data;
1208 
1209 	if (msg->type == BRCM_MESSAGE_BATCH) {
1210 		for (i = msg->batch.msgs_queued;
1211 		     i < msg->batch.msgs_count; i++) {
1212 			rc = flexrm_new_request(ring, msg,
1213 						 &msg->batch.msgs[i]);
1214 			if (rc) {
1215 				msg->error = rc;
1216 				return rc;
1217 			}
1218 			msg->batch.msgs_queued++;
1219 		}
1220 		return 0;
1221 	}
1222 
1223 	return flexrm_new_request(ring, NULL, data);
1224 }
1225 
1226 static bool flexrm_peek_data(struct mbox_chan *chan)
1227 {
1228 	int cnt = flexrm_process_completions(chan->con_priv);
1229 
1230 	return (cnt > 0) ? true : false;
1231 }
1232 
1233 static int flexrm_startup(struct mbox_chan *chan)
1234 {
1235 	u64 d;
1236 	u32 val, off;
1237 	int ret = 0;
1238 	dma_addr_t next_addr;
1239 	struct flexrm_ring *ring = chan->con_priv;
1240 
1241 	/* Allocate BD memory */
1242 	ring->bd_base = dma_pool_alloc(ring->mbox->bd_pool,
1243 				       GFP_KERNEL, &ring->bd_dma_base);
1244 	if (!ring->bd_base) {
1245 		dev_err(ring->mbox->dev,
1246 			"can't allocate BD memory for ring%d\n",
1247 			ring->num);
1248 		ret = -ENOMEM;
1249 		goto fail;
1250 	}
1251 
1252 	/* Configure next table pointer entries in BD memory */
1253 	for (off = 0; off < RING_BD_SIZE; off += RING_DESC_SIZE) {
1254 		next_addr = off + RING_DESC_SIZE;
1255 		if (next_addr == RING_BD_SIZE)
1256 			next_addr = 0;
1257 		next_addr += ring->bd_dma_base;
1258 		if (RING_BD_ALIGN_CHECK(next_addr))
1259 			d = flexrm_next_table_desc(RING_BD_TOGGLE_VALID(off),
1260 						    next_addr);
1261 		else
1262 			d = flexrm_null_desc(RING_BD_TOGGLE_INVALID(off));
1263 		flexrm_write_desc(ring->bd_base + off, d);
1264 	}
1265 
1266 	/* Allocate completion memory */
1267 	ring->cmpl_base = dma_pool_zalloc(ring->mbox->cmpl_pool,
1268 					 GFP_KERNEL, &ring->cmpl_dma_base);
1269 	if (!ring->cmpl_base) {
1270 		dev_err(ring->mbox->dev,
1271 			"can't allocate completion memory for ring%d\n",
1272 			ring->num);
1273 		ret = -ENOMEM;
1274 		goto fail_free_bd_memory;
1275 	}
1276 
1277 	/* Request IRQ */
1278 	if (ring->irq == UINT_MAX) {
1279 		dev_err(ring->mbox->dev,
1280 			"ring%d IRQ not available\n", ring->num);
1281 		ret = -ENODEV;
1282 		goto fail_free_cmpl_memory;
1283 	}
1284 	ret = request_threaded_irq(ring->irq,
1285 				   flexrm_irq_event,
1286 				   flexrm_irq_thread,
1287 				   0, dev_name(ring->mbox->dev), ring);
1288 	if (ret) {
1289 		dev_err(ring->mbox->dev,
1290 			"failed to request ring%d IRQ\n", ring->num);
1291 		goto fail_free_cmpl_memory;
1292 	}
1293 	ring->irq_requested = true;
1294 
1295 	/* Set IRQ affinity hint */
1296 	ring->irq_aff_hint = CPU_MASK_NONE;
1297 	val = ring->mbox->num_rings;
1298 	val = (num_online_cpus() < val) ? val / num_online_cpus() : 1;
1299 	cpumask_set_cpu((ring->num / val) % num_online_cpus(),
1300 			&ring->irq_aff_hint);
1301 	ret = irq_set_affinity_hint(ring->irq, &ring->irq_aff_hint);
1302 	if (ret) {
1303 		dev_err(ring->mbox->dev,
1304 			"failed to set IRQ affinity hint for ring%d\n",
1305 			ring->num);
1306 		goto fail_free_irq;
1307 	}
1308 
1309 	/* Disable/inactivate ring */
1310 	writel_relaxed(0x0, ring->regs + RING_CONTROL);
1311 
1312 	/* Program BD start address */
1313 	val = BD_START_ADDR_VALUE(ring->bd_dma_base);
1314 	writel_relaxed(val, ring->regs + RING_BD_START_ADDR);
1315 
1316 	/* BD write pointer will be same as HW write pointer */
1317 	ring->bd_write_offset =
1318 			readl_relaxed(ring->regs + RING_BD_WRITE_PTR);
1319 	ring->bd_write_offset *= RING_DESC_SIZE;
1320 
1321 	/* Program completion start address */
1322 	val = CMPL_START_ADDR_VALUE(ring->cmpl_dma_base);
1323 	writel_relaxed(val, ring->regs + RING_CMPL_START_ADDR);
1324 
1325 	/* Completion read pointer will be same as HW write pointer */
1326 	ring->cmpl_read_offset =
1327 			readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
1328 	ring->cmpl_read_offset *= RING_DESC_SIZE;
1329 
1330 	/* Read ring Tx, Rx, and Outstanding counts to clear */
1331 	readl_relaxed(ring->regs + RING_NUM_REQ_RECV_LS);
1332 	readl_relaxed(ring->regs + RING_NUM_REQ_RECV_MS);
1333 	readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_LS);
1334 	readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_MS);
1335 	readl_relaxed(ring->regs + RING_NUM_REQ_OUTSTAND);
1336 
1337 	/* Configure RING_MSI_CONTROL */
1338 	val = 0;
1339 	val |= (ring->msi_timer_val << MSI_TIMER_VAL_SHIFT);
1340 	val |= BIT(MSI_ENABLE_SHIFT);
1341 	val |= (ring->msi_count_threshold & MSI_COUNT_MASK) << MSI_COUNT_SHIFT;
1342 	writel_relaxed(val, ring->regs + RING_MSI_CONTROL);
1343 
1344 	/* Enable/activate ring */
1345 	val = BIT(CONTROL_ACTIVE_SHIFT);
1346 	writel_relaxed(val, ring->regs + RING_CONTROL);
1347 
1348 	/* Reset stats to zero */
1349 	atomic_set(&ring->msg_send_count, 0);
1350 	atomic_set(&ring->msg_cmpl_count, 0);
1351 
1352 	return 0;
1353 
1354 fail_free_irq:
1355 	free_irq(ring->irq, ring);
1356 	ring->irq_requested = false;
1357 fail_free_cmpl_memory:
1358 	dma_pool_free(ring->mbox->cmpl_pool,
1359 		      ring->cmpl_base, ring->cmpl_dma_base);
1360 	ring->cmpl_base = NULL;
1361 fail_free_bd_memory:
1362 	dma_pool_free(ring->mbox->bd_pool,
1363 		      ring->bd_base, ring->bd_dma_base);
1364 	ring->bd_base = NULL;
1365 fail:
1366 	return ret;
1367 }
1368 
1369 static void flexrm_shutdown(struct mbox_chan *chan)
1370 {
1371 	u32 reqid;
1372 	unsigned int timeout;
1373 	struct brcm_message *msg;
1374 	struct flexrm_ring *ring = chan->con_priv;
1375 
1376 	/* Disable/inactivate ring */
1377 	writel_relaxed(0x0, ring->regs + RING_CONTROL);
1378 
1379 	/* Set ring flush state */
1380 	timeout = 1000; /* timeout of 1s */
1381 	writel_relaxed(BIT(CONTROL_FLUSH_SHIFT),
1382 			ring->regs + RING_CONTROL);
1383 	do {
1384 		if (readl_relaxed(ring->regs + RING_FLUSH_DONE) &
1385 		    FLUSH_DONE_MASK)
1386 			break;
1387 		mdelay(1);
1388 	} while (--timeout);
1389 	if (!timeout)
1390 		dev_err(ring->mbox->dev,
1391 			"setting ring%d flush state timedout\n", ring->num);
1392 
1393 	/* Clear ring flush state */
1394 	timeout = 1000; /* timeout of 1s */
1395 	writel_relaxed(0x0, ring->regs + RING_CONTROL);
1396 	do {
1397 		if (!(readl_relaxed(ring->regs + RING_FLUSH_DONE) &
1398 		      FLUSH_DONE_MASK))
1399 			break;
1400 		mdelay(1);
1401 	} while (--timeout);
1402 	if (!timeout)
1403 		dev_err(ring->mbox->dev,
1404 			"clearing ring%d flush state timedout\n", ring->num);
1405 
1406 	/* Abort all in-flight requests */
1407 	for (reqid = 0; reqid < RING_MAX_REQ_COUNT; reqid++) {
1408 		msg = ring->requests[reqid];
1409 		if (!msg)
1410 			continue;
1411 
1412 		/* Release reqid for recycling */
1413 		ring->requests[reqid] = NULL;
1414 
1415 		/* Unmap DMA mappings */
1416 		flexrm_dma_unmap(ring->mbox->dev, msg);
1417 
1418 		/* Give-back message to mailbox client */
1419 		msg->error = -EIO;
1420 		mbox_chan_received_data(chan, msg);
1421 	}
1422 
1423 	/* Clear requests bitmap */
1424 	bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
1425 
1426 	/* Release IRQ */
1427 	if (ring->irq_requested) {
1428 		irq_set_affinity_hint(ring->irq, NULL);
1429 		free_irq(ring->irq, ring);
1430 		ring->irq_requested = false;
1431 	}
1432 
1433 	/* Free-up completion descriptor ring */
1434 	if (ring->cmpl_base) {
1435 		dma_pool_free(ring->mbox->cmpl_pool,
1436 			      ring->cmpl_base, ring->cmpl_dma_base);
1437 		ring->cmpl_base = NULL;
1438 	}
1439 
1440 	/* Free-up BD descriptor ring */
1441 	if (ring->bd_base) {
1442 		dma_pool_free(ring->mbox->bd_pool,
1443 			      ring->bd_base, ring->bd_dma_base);
1444 		ring->bd_base = NULL;
1445 	}
1446 }
1447 
1448 static const struct mbox_chan_ops flexrm_mbox_chan_ops = {
1449 	.send_data	= flexrm_send_data,
1450 	.startup	= flexrm_startup,
1451 	.shutdown	= flexrm_shutdown,
1452 	.peek_data	= flexrm_peek_data,
1453 };
1454 
1455 static struct mbox_chan *flexrm_mbox_of_xlate(struct mbox_controller *cntlr,
1456 					const struct of_phandle_args *pa)
1457 {
1458 	struct mbox_chan *chan;
1459 	struct flexrm_ring *ring;
1460 
1461 	if (pa->args_count < 3)
1462 		return ERR_PTR(-EINVAL);
1463 
1464 	if (pa->args[0] >= cntlr->num_chans)
1465 		return ERR_PTR(-ENOENT);
1466 
1467 	if (pa->args[1] > MSI_COUNT_MASK)
1468 		return ERR_PTR(-EINVAL);
1469 
1470 	if (pa->args[2] > MSI_TIMER_VAL_MASK)
1471 		return ERR_PTR(-EINVAL);
1472 
1473 	chan = &cntlr->chans[pa->args[0]];
1474 	ring = chan->con_priv;
1475 	ring->msi_count_threshold = pa->args[1];
1476 	ring->msi_timer_val = pa->args[2];
1477 
1478 	return chan;
1479 }
1480 
1481 /* ====== FlexRM platform driver ===== */
1482 
1483 static void flexrm_mbox_msi_write(struct msi_desc *desc, struct msi_msg *msg)
1484 {
1485 	struct device *dev = msi_desc_to_dev(desc);
1486 	struct flexrm_mbox *mbox = dev_get_drvdata(dev);
1487 	struct flexrm_ring *ring = &mbox->rings[desc->platform.msi_index];
1488 
1489 	/* Configure per-Ring MSI registers */
1490 	writel_relaxed(msg->address_lo, ring->regs + RING_MSI_ADDR_LS);
1491 	writel_relaxed(msg->address_hi, ring->regs + RING_MSI_ADDR_MS);
1492 	writel_relaxed(msg->data, ring->regs + RING_MSI_DATA_VALUE);
1493 }
1494 
1495 static int flexrm_mbox_probe(struct platform_device *pdev)
1496 {
1497 	int index, ret = 0;
1498 	void __iomem *regs;
1499 	void __iomem *regs_end;
1500 	struct msi_desc *desc;
1501 	struct resource *iomem;
1502 	struct flexrm_ring *ring;
1503 	struct flexrm_mbox *mbox;
1504 	struct device *dev = &pdev->dev;
1505 
1506 	/* Allocate driver mailbox struct */
1507 	mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL);
1508 	if (!mbox) {
1509 		ret = -ENOMEM;
1510 		goto fail;
1511 	}
1512 	mbox->dev = dev;
1513 	platform_set_drvdata(pdev, mbox);
1514 
1515 	/* Get resource for registers */
1516 	iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1517 	if (!iomem || (resource_size(iomem) < RING_REGS_SIZE)) {
1518 		ret = -ENODEV;
1519 		goto fail;
1520 	}
1521 
1522 	/* Map registers of all rings */
1523 	mbox->regs = devm_ioremap_resource(&pdev->dev, iomem);
1524 	if (IS_ERR(mbox->regs)) {
1525 		ret = PTR_ERR(mbox->regs);
1526 		dev_err(&pdev->dev, "Failed to remap mailbox regs: %d\n", ret);
1527 		goto fail;
1528 	}
1529 	regs_end = mbox->regs + resource_size(iomem);
1530 
1531 	/* Scan and count available rings */
1532 	mbox->num_rings = 0;
1533 	for (regs = mbox->regs; regs < regs_end; regs += RING_REGS_SIZE) {
1534 		if (readl_relaxed(regs + RING_VER) == RING_VER_MAGIC)
1535 			mbox->num_rings++;
1536 	}
1537 	if (!mbox->num_rings) {
1538 		ret = -ENODEV;
1539 		goto fail;
1540 	}
1541 
1542 	/* Allocate driver ring structs */
1543 	ring = devm_kcalloc(dev, mbox->num_rings, sizeof(*ring), GFP_KERNEL);
1544 	if (!ring) {
1545 		ret = -ENOMEM;
1546 		goto fail;
1547 	}
1548 	mbox->rings = ring;
1549 
1550 	/* Initialize members of driver ring structs */
1551 	regs = mbox->regs;
1552 	for (index = 0; index < mbox->num_rings; index++) {
1553 		ring = &mbox->rings[index];
1554 		ring->num = index;
1555 		ring->mbox = mbox;
1556 		while ((regs < regs_end) &&
1557 		       (readl_relaxed(regs + RING_VER) != RING_VER_MAGIC))
1558 			regs += RING_REGS_SIZE;
1559 		if (regs_end <= regs) {
1560 			ret = -ENODEV;
1561 			goto fail;
1562 		}
1563 		ring->regs = regs;
1564 		regs += RING_REGS_SIZE;
1565 		ring->irq = UINT_MAX;
1566 		ring->irq_requested = false;
1567 		ring->msi_timer_val = MSI_TIMER_VAL_MASK;
1568 		ring->msi_count_threshold = 0x1;
1569 		memset(ring->requests, 0, sizeof(ring->requests));
1570 		ring->bd_base = NULL;
1571 		ring->bd_dma_base = 0;
1572 		ring->cmpl_base = NULL;
1573 		ring->cmpl_dma_base = 0;
1574 		atomic_set(&ring->msg_send_count, 0);
1575 		atomic_set(&ring->msg_cmpl_count, 0);
1576 		spin_lock_init(&ring->lock);
1577 		bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
1578 		ring->cmpl_read_offset = 0;
1579 	}
1580 
1581 	/* FlexRM is capable of 40-bit physical addresses only */
1582 	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
1583 	if (ret) {
1584 		ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
1585 		if (ret)
1586 			goto fail;
1587 	}
1588 
1589 	/* Create DMA pool for ring BD memory */
1590 	mbox->bd_pool = dma_pool_create("bd", dev, RING_BD_SIZE,
1591 					1 << RING_BD_ALIGN_ORDER, 0);
1592 	if (!mbox->bd_pool) {
1593 		ret = -ENOMEM;
1594 		goto fail;
1595 	}
1596 
1597 	/* Create DMA pool for ring completion memory */
1598 	mbox->cmpl_pool = dma_pool_create("cmpl", dev, RING_CMPL_SIZE,
1599 					  1 << RING_CMPL_ALIGN_ORDER, 0);
1600 	if (!mbox->cmpl_pool) {
1601 		ret = -ENOMEM;
1602 		goto fail_destroy_bd_pool;
1603 	}
1604 
1605 	/* Allocate platform MSIs for each ring */
1606 	ret = platform_msi_domain_alloc_irqs(dev, mbox->num_rings,
1607 						flexrm_mbox_msi_write);
1608 	if (ret)
1609 		goto fail_destroy_cmpl_pool;
1610 
1611 	/* Save alloced IRQ numbers for each ring */
1612 	for_each_msi_entry(desc, dev) {
1613 		ring = &mbox->rings[desc->platform.msi_index];
1614 		ring->irq = desc->irq;
1615 	}
1616 
1617 	/* Check availability of debugfs */
1618 	if (!debugfs_initialized())
1619 		goto skip_debugfs;
1620 
1621 	/* Create debugfs root entry */
1622 	mbox->root = debugfs_create_dir(dev_name(mbox->dev), NULL);
1623 
1624 	/* Create debugfs config entry */
1625 	debugfs_create_devm_seqfile(mbox->dev, "config", mbox->root,
1626 				    flexrm_debugfs_conf_show);
1627 
1628 	/* Create debugfs stats entry */
1629 	debugfs_create_devm_seqfile(mbox->dev, "stats", mbox->root,
1630 				    flexrm_debugfs_stats_show);
1631 
1632 skip_debugfs:
1633 
1634 	/* Initialize mailbox controller */
1635 	mbox->controller.txdone_irq = false;
1636 	mbox->controller.txdone_poll = false;
1637 	mbox->controller.ops = &flexrm_mbox_chan_ops;
1638 	mbox->controller.dev = dev;
1639 	mbox->controller.num_chans = mbox->num_rings;
1640 	mbox->controller.of_xlate = flexrm_mbox_of_xlate;
1641 	mbox->controller.chans = devm_kcalloc(dev, mbox->num_rings,
1642 				sizeof(*mbox->controller.chans), GFP_KERNEL);
1643 	if (!mbox->controller.chans) {
1644 		ret = -ENOMEM;
1645 		goto fail_free_debugfs_root;
1646 	}
1647 	for (index = 0; index < mbox->num_rings; index++)
1648 		mbox->controller.chans[index].con_priv = &mbox->rings[index];
1649 
1650 	/* Register mailbox controller */
1651 	ret = devm_mbox_controller_register(dev, &mbox->controller);
1652 	if (ret)
1653 		goto fail_free_debugfs_root;
1654 
1655 	dev_info(dev, "registered flexrm mailbox with %d channels\n",
1656 			mbox->controller.num_chans);
1657 
1658 	return 0;
1659 
1660 fail_free_debugfs_root:
1661 	debugfs_remove_recursive(mbox->root);
1662 	platform_msi_domain_free_irqs(dev);
1663 fail_destroy_cmpl_pool:
1664 	dma_pool_destroy(mbox->cmpl_pool);
1665 fail_destroy_bd_pool:
1666 	dma_pool_destroy(mbox->bd_pool);
1667 fail:
1668 	return ret;
1669 }
1670 
1671 static int flexrm_mbox_remove(struct platform_device *pdev)
1672 {
1673 	struct device *dev = &pdev->dev;
1674 	struct flexrm_mbox *mbox = platform_get_drvdata(pdev);
1675 
1676 	debugfs_remove_recursive(mbox->root);
1677 
1678 	platform_msi_domain_free_irqs(dev);
1679 
1680 	dma_pool_destroy(mbox->cmpl_pool);
1681 	dma_pool_destroy(mbox->bd_pool);
1682 
1683 	return 0;
1684 }
1685 
1686 static const struct of_device_id flexrm_mbox_of_match[] = {
1687 	{ .compatible = "brcm,iproc-flexrm-mbox", },
1688 	{},
1689 };
1690 MODULE_DEVICE_TABLE(of, flexrm_mbox_of_match);
1691 
1692 static struct platform_driver flexrm_mbox_driver = {
1693 	.driver = {
1694 		.name = "brcm-flexrm-mbox",
1695 		.of_match_table = flexrm_mbox_of_match,
1696 	},
1697 	.probe		= flexrm_mbox_probe,
1698 	.remove		= flexrm_mbox_remove,
1699 };
1700 module_platform_driver(flexrm_mbox_driver);
1701 
1702 MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
1703 MODULE_DESCRIPTION("Broadcom FlexRM mailbox driver");
1704 MODULE_LICENSE("GPL v2");
1705