1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /*
3  * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
4  * Copyright 2016-2019 NXP
5  *
6  */
7 
8 #include <asm/cacheflush.h>
9 #include <linux/io.h>
10 #include <linux/slab.h>
11 #include <linux/spinlock.h>
12 #include <soc/fsl/dpaa2-global.h>
13 
14 #include "qbman-portal.h"
15 
16 /* All QBMan command and result structures use this "valid bit" encoding */
17 #define QB_VALID_BIT ((u32)0x80)
18 
19 /* QBMan portal management command codes */
20 #define QBMAN_MC_ACQUIRE       0x30
21 #define QBMAN_WQCHAN_CONFIGURE 0x46
22 
23 /* CINH register offsets */
24 #define QBMAN_CINH_SWP_EQCR_PI      0x800
25 #define QBMAN_CINH_SWP_EQCR_CI	    0x840
26 #define QBMAN_CINH_SWP_EQAR    0x8c0
27 #define QBMAN_CINH_SWP_CR_RT        0x900
28 #define QBMAN_CINH_SWP_VDQCR_RT     0x940
29 #define QBMAN_CINH_SWP_EQCR_AM_RT   0x980
30 #define QBMAN_CINH_SWP_RCR_AM_RT    0x9c0
31 #define QBMAN_CINH_SWP_DQPI    0xa00
32 #define QBMAN_CINH_SWP_DQRR_ITR     0xa80
33 #define QBMAN_CINH_SWP_DCAP    0xac0
34 #define QBMAN_CINH_SWP_SDQCR   0xb00
35 #define QBMAN_CINH_SWP_EQCR_AM_RT2  0xb40
36 #define QBMAN_CINH_SWP_RCR_PI       0xc00
37 #define QBMAN_CINH_SWP_RAR     0xcc0
38 #define QBMAN_CINH_SWP_ISR     0xe00
39 #define QBMAN_CINH_SWP_IER     0xe40
40 #define QBMAN_CINH_SWP_ISDR    0xe80
41 #define QBMAN_CINH_SWP_IIR     0xec0
42 #define QBMAN_CINH_SWP_ITPR    0xf40
43 
44 /* CENA register offsets */
45 #define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((u32)(n) << 6))
46 #define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((u32)(n) << 6))
47 #define QBMAN_CENA_SWP_RCR(n)  (0x400 + ((u32)(n) << 6))
48 #define QBMAN_CENA_SWP_CR      0x600
49 #define QBMAN_CENA_SWP_RR(vb)  (0x700 + ((u32)(vb) >> 1))
50 #define QBMAN_CENA_SWP_VDQCR   0x780
51 #define QBMAN_CENA_SWP_EQCR_CI 0x840
52 #define QBMAN_CENA_SWP_EQCR_CI_MEMBACK 0x1840
53 
54 /* CENA register offsets in memory-backed mode */
55 #define QBMAN_CENA_SWP_DQRR_MEM(n)  (0x800 + ((u32)(n) << 6))
56 #define QBMAN_CENA_SWP_RCR_MEM(n)   (0x1400 + ((u32)(n) << 6))
57 #define QBMAN_CENA_SWP_CR_MEM       0x1600
58 #define QBMAN_CENA_SWP_RR_MEM       0x1680
59 #define QBMAN_CENA_SWP_VDQCR_MEM    0x1780
60 
61 /* Reverse mapping of QBMAN_CENA_SWP_DQRR() */
62 #define QBMAN_IDX_FROM_DQRR(p) (((unsigned long)(p) & 0x1ff) >> 6)
63 
64 /* Define token used to determine if response written to memory is valid */
65 #define QMAN_DQ_TOKEN_VALID 1
66 
67 /* SDQCR attribute codes */
68 #define QB_SDQCR_FC_SHIFT   29
69 #define QB_SDQCR_FC_MASK    0x1
70 #define QB_SDQCR_DCT_SHIFT  24
71 #define QB_SDQCR_DCT_MASK   0x3
72 #define QB_SDQCR_TOK_SHIFT  16
73 #define QB_SDQCR_TOK_MASK   0xff
74 #define QB_SDQCR_SRC_SHIFT  0
75 #define QB_SDQCR_SRC_MASK   0xffff
76 
77 /* opaque token for static dequeues */
78 #define QMAN_SDQCR_TOKEN    0xbb
79 
80 #define QBMAN_EQCR_DCA_IDXMASK          0x0f
81 #define QBMAN_ENQUEUE_FLAG_DCA          (1ULL << 31)
82 
83 #define EQ_DESC_SIZE_WITHOUT_FD 29
84 #define EQ_DESC_SIZE_FD_START 32
85 
86 enum qbman_sdqcr_dct {
87 	qbman_sdqcr_dct_null = 0,
88 	qbman_sdqcr_dct_prio_ics,
89 	qbman_sdqcr_dct_active_ics,
90 	qbman_sdqcr_dct_active
91 };
92 
93 enum qbman_sdqcr_fc {
94 	qbman_sdqcr_fc_one = 0,
95 	qbman_sdqcr_fc_up_to_3 = 1
96 };
97 
98 /* Internal Function declaration */
99 static int qbman_swp_enqueue_direct(struct qbman_swp *s,
100 				    const struct qbman_eq_desc *d,
101 				    const struct dpaa2_fd *fd);
102 static int qbman_swp_enqueue_mem_back(struct qbman_swp *s,
103 				      const struct qbman_eq_desc *d,
104 				      const struct dpaa2_fd *fd);
105 static int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
106 					     const struct qbman_eq_desc *d,
107 					     const struct dpaa2_fd *fd,
108 					     uint32_t *flags,
109 					     int num_frames);
110 static int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
111 					       const struct qbman_eq_desc *d,
112 					       const struct dpaa2_fd *fd,
113 					       uint32_t *flags,
114 					       int num_frames);
115 static int
116 qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
117 				       const struct qbman_eq_desc *d,
118 				       const struct dpaa2_fd *fd,
119 				       int num_frames);
120 static
121 int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
122 					     const struct qbman_eq_desc *d,
123 					     const struct dpaa2_fd *fd,
124 					     int num_frames);
125 static int qbman_swp_pull_direct(struct qbman_swp *s,
126 				 struct qbman_pull_desc *d);
127 static int qbman_swp_pull_mem_back(struct qbman_swp *s,
128 				   struct qbman_pull_desc *d);
129 
130 const struct dpaa2_dq *qbman_swp_dqrr_next_direct(struct qbman_swp *s);
131 const struct dpaa2_dq *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s);
132 
133 static int qbman_swp_release_direct(struct qbman_swp *s,
134 				    const struct qbman_release_desc *d,
135 				    const u64 *buffers,
136 				    unsigned int num_buffers);
137 static int qbman_swp_release_mem_back(struct qbman_swp *s,
138 				      const struct qbman_release_desc *d,
139 				      const u64 *buffers,
140 				      unsigned int num_buffers);
141 
142 /* Function pointers */
143 int (*qbman_swp_enqueue_ptr)(struct qbman_swp *s,
144 			     const struct qbman_eq_desc *d,
145 			     const struct dpaa2_fd *fd)
146 	= qbman_swp_enqueue_direct;
147 
148 int (*qbman_swp_enqueue_multiple_ptr)(struct qbman_swp *s,
149 				      const struct qbman_eq_desc *d,
150 				      const struct dpaa2_fd *fd,
151 				      uint32_t *flags,
152 					     int num_frames)
153 	= qbman_swp_enqueue_multiple_direct;
154 
155 int
156 (*qbman_swp_enqueue_multiple_desc_ptr)(struct qbman_swp *s,
157 				       const struct qbman_eq_desc *d,
158 				       const struct dpaa2_fd *fd,
159 				       int num_frames)
160 	= qbman_swp_enqueue_multiple_desc_direct;
161 
162 int (*qbman_swp_pull_ptr)(struct qbman_swp *s, struct qbman_pull_desc *d)
163 			= qbman_swp_pull_direct;
164 
165 const struct dpaa2_dq *(*qbman_swp_dqrr_next_ptr)(struct qbman_swp *s)
166 			= qbman_swp_dqrr_next_direct;
167 
168 int (*qbman_swp_release_ptr)(struct qbman_swp *s,
169 			     const struct qbman_release_desc *d,
170 			     const u64 *buffers,
171 			     unsigned int num_buffers)
172 			= qbman_swp_release_direct;
173 
174 /* Portal Access */
175 
176 static inline u32 qbman_read_register(struct qbman_swp *p, u32 offset)
177 {
178 	return readl_relaxed(p->addr_cinh + offset);
179 }
180 
181 static inline void qbman_write_register(struct qbman_swp *p, u32 offset,
182 					u32 value)
183 {
184 	writel_relaxed(value, p->addr_cinh + offset);
185 }
186 
187 static inline void *qbman_get_cmd(struct qbman_swp *p, u32 offset)
188 {
189 	return p->addr_cena + offset;
190 }
191 
192 #define QBMAN_CINH_SWP_CFG   0xd00
193 
194 #define SWP_CFG_DQRR_MF_SHIFT 20
195 #define SWP_CFG_EST_SHIFT     16
196 #define SWP_CFG_CPBS_SHIFT    15
197 #define SWP_CFG_WN_SHIFT      14
198 #define SWP_CFG_RPM_SHIFT     12
199 #define SWP_CFG_DCM_SHIFT     10
200 #define SWP_CFG_EPM_SHIFT     8
201 #define SWP_CFG_VPM_SHIFT     7
202 #define SWP_CFG_CPM_SHIFT     6
203 #define SWP_CFG_SD_SHIFT      5
204 #define SWP_CFG_SP_SHIFT      4
205 #define SWP_CFG_SE_SHIFT      3
206 #define SWP_CFG_DP_SHIFT      2
207 #define SWP_CFG_DE_SHIFT      1
208 #define SWP_CFG_EP_SHIFT      0
209 
210 static inline u32 qbman_set_swp_cfg(u8 max_fill, u8 wn,	u8 est, u8 rpm, u8 dcm,
211 				    u8 epm, int sd, int sp, int se,
212 				    int dp, int de, int ep)
213 {
214 	return (max_fill << SWP_CFG_DQRR_MF_SHIFT |
215 		est << SWP_CFG_EST_SHIFT |
216 		wn << SWP_CFG_WN_SHIFT |
217 		rpm << SWP_CFG_RPM_SHIFT |
218 		dcm << SWP_CFG_DCM_SHIFT |
219 		epm << SWP_CFG_EPM_SHIFT |
220 		sd << SWP_CFG_SD_SHIFT |
221 		sp << SWP_CFG_SP_SHIFT |
222 		se << SWP_CFG_SE_SHIFT |
223 		dp << SWP_CFG_DP_SHIFT |
224 		de << SWP_CFG_DE_SHIFT |
225 		ep << SWP_CFG_EP_SHIFT);
226 }
227 
228 #define QMAN_RT_MODE	   0x00000100
229 
230 static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last)
231 {
232 	/* 'first' is included, 'last' is excluded */
233 	if (first <= last)
234 		return last - first;
235 	else
236 		return (2 * ringsize) - (first - last);
237 }
238 
239 /**
240  * qbman_swp_init() - Create a functional object representing the given
241  *                    QBMan portal descriptor.
242  * @d: the given qbman swp descriptor
243  *
244  * Return qbman_swp portal for success, NULL if the object cannot
245  * be created.
246  */
247 struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d)
248 {
249 	struct qbman_swp *p = kzalloc(sizeof(*p), GFP_KERNEL);
250 	u32 reg;
251 	u32 mask_size;
252 	u32 eqcr_pi;
253 
254 	if (!p)
255 		return NULL;
256 
257 	spin_lock_init(&p->access_spinlock);
258 
259 	p->desc = d;
260 	p->mc.valid_bit = QB_VALID_BIT;
261 	p->sdq = 0;
262 	p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT;
263 	p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT;
264 	p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT;
265 	if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000)
266 		p->mr.valid_bit = QB_VALID_BIT;
267 
268 	atomic_set(&p->vdq.available, 1);
269 	p->vdq.valid_bit = QB_VALID_BIT;
270 	p->dqrr.next_idx = 0;
271 	p->dqrr.valid_bit = QB_VALID_BIT;
272 
273 	if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_4100) {
274 		p->dqrr.dqrr_size = 4;
275 		p->dqrr.reset_bug = 1;
276 	} else {
277 		p->dqrr.dqrr_size = 8;
278 		p->dqrr.reset_bug = 0;
279 	}
280 
281 	p->addr_cena = d->cena_bar;
282 	p->addr_cinh = d->cinh_bar;
283 
284 	if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
285 
286 		reg = qbman_set_swp_cfg(p->dqrr.dqrr_size,
287 			1, /* Writes Non-cacheable */
288 			0, /* EQCR_CI stashing threshold */
289 			3, /* RPM: RCR in array mode */
290 			2, /* DCM: Discrete consumption ack */
291 			2, /* EPM: EQCR in ring mode */
292 			1, /* mem stashing drop enable enable */
293 			1, /* mem stashing priority enable */
294 			1, /* mem stashing enable */
295 			1, /* dequeue stashing priority enable */
296 			0, /* dequeue stashing enable enable */
297 			0); /* EQCR_CI stashing priority enable */
298 	} else {
299 		memset(p->addr_cena, 0, 64 * 1024);
300 		reg = qbman_set_swp_cfg(p->dqrr.dqrr_size,
301 			1, /* Writes Non-cacheable */
302 			1, /* EQCR_CI stashing threshold */
303 			3, /* RPM: RCR in array mode */
304 			2, /* DCM: Discrete consumption ack */
305 			0, /* EPM: EQCR in ring mode */
306 			1, /* mem stashing drop enable */
307 			1, /* mem stashing priority enable */
308 			1, /* mem stashing enable */
309 			1, /* dequeue stashing priority enable */
310 			0, /* dequeue stashing enable */
311 			0); /* EQCR_CI stashing priority enable */
312 		reg |= 1 << SWP_CFG_CPBS_SHIFT | /* memory-backed mode */
313 		       1 << SWP_CFG_VPM_SHIFT |  /* VDQCR read triggered mode */
314 		       1 << SWP_CFG_CPM_SHIFT;   /* CR read triggered mode */
315 	}
316 
317 	qbman_write_register(p, QBMAN_CINH_SWP_CFG, reg);
318 	reg = qbman_read_register(p, QBMAN_CINH_SWP_CFG);
319 	if (!reg) {
320 		pr_err("qbman: the portal is not enabled!\n");
321 		kfree(p);
322 		return NULL;
323 	}
324 
325 	if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) {
326 		qbman_write_register(p, QBMAN_CINH_SWP_EQCR_PI, QMAN_RT_MODE);
327 		qbman_write_register(p, QBMAN_CINH_SWP_RCR_PI, QMAN_RT_MODE);
328 	}
329 	/*
330 	 * SDQCR needs to be initialized to 0 when no channels are
331 	 * being dequeued from or else the QMan HW will indicate an
332 	 * error.  The values that were calculated above will be
333 	 * applied when dequeues from a specific channel are enabled.
334 	 */
335 	qbman_write_register(p, QBMAN_CINH_SWP_SDQCR, 0);
336 
337 	p->eqcr.pi_ring_size = 8;
338 	if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) {
339 		p->eqcr.pi_ring_size = 32;
340 		qbman_swp_enqueue_ptr =
341 			qbman_swp_enqueue_mem_back;
342 		qbman_swp_enqueue_multiple_ptr =
343 			qbman_swp_enqueue_multiple_mem_back;
344 		qbman_swp_enqueue_multiple_desc_ptr =
345 			qbman_swp_enqueue_multiple_desc_mem_back;
346 		qbman_swp_pull_ptr = qbman_swp_pull_mem_back;
347 		qbman_swp_dqrr_next_ptr = qbman_swp_dqrr_next_mem_back;
348 		qbman_swp_release_ptr = qbman_swp_release_mem_back;
349 	}
350 
351 	for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1)
352 		p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask << 1) + 1;
353 	eqcr_pi = qbman_read_register(p, QBMAN_CINH_SWP_EQCR_PI);
354 	p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask;
355 	p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT;
356 	p->eqcr.ci = qbman_read_register(p, QBMAN_CINH_SWP_EQCR_CI)
357 			& p->eqcr.pi_ci_mask;
358 	p->eqcr.available = p->eqcr.pi_ring_size;
359 
360 	/* Initialize the software portal with a irq timeout period of 0us */
361 	qbman_swp_set_irq_coalescing(p, p->dqrr.dqrr_size - 1, 0);
362 
363 	return p;
364 }
365 
366 /**
367  * qbman_swp_finish() - Create and destroy a functional object representing
368  *                      the given QBMan portal descriptor.
369  * @p: the qbman_swp object to be destroyed
370  */
371 void qbman_swp_finish(struct qbman_swp *p)
372 {
373 	kfree(p);
374 }
375 
376 /**
377  * qbman_swp_interrupt_read_status()
378  * @p: the given software portal
379  *
380  * Return the value in the SWP_ISR register.
381  */
382 u32 qbman_swp_interrupt_read_status(struct qbman_swp *p)
383 {
384 	return qbman_read_register(p, QBMAN_CINH_SWP_ISR);
385 }
386 
387 /**
388  * qbman_swp_interrupt_clear_status()
389  * @p: the given software portal
390  * @mask: The mask to clear in SWP_ISR register
391  */
392 void qbman_swp_interrupt_clear_status(struct qbman_swp *p, u32 mask)
393 {
394 	qbman_write_register(p, QBMAN_CINH_SWP_ISR, mask);
395 }
396 
397 /**
398  * qbman_swp_interrupt_get_trigger() - read interrupt enable register
399  * @p: the given software portal
400  *
401  * Return the value in the SWP_IER register.
402  */
403 u32 qbman_swp_interrupt_get_trigger(struct qbman_swp *p)
404 {
405 	return qbman_read_register(p, QBMAN_CINH_SWP_IER);
406 }
407 
408 /**
409  * qbman_swp_interrupt_set_trigger() - enable interrupts for a swp
410  * @p: the given software portal
411  * @mask: The mask of bits to enable in SWP_IER
412  */
413 void qbman_swp_interrupt_set_trigger(struct qbman_swp *p, u32 mask)
414 {
415 	qbman_write_register(p, QBMAN_CINH_SWP_IER, mask);
416 }
417 
418 /**
419  * qbman_swp_interrupt_get_inhibit() - read interrupt mask register
420  * @p: the given software portal object
421  *
422  * Return the value in the SWP_IIR register.
423  */
424 int qbman_swp_interrupt_get_inhibit(struct qbman_swp *p)
425 {
426 	return qbman_read_register(p, QBMAN_CINH_SWP_IIR);
427 }
428 
429 /**
430  * qbman_swp_interrupt_set_inhibit() - write interrupt mask register
431  * @p: the given software portal object
432  * @inhibit: whether to inhibit the IRQs
433  */
434 void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit)
435 {
436 	qbman_write_register(p, QBMAN_CINH_SWP_IIR, inhibit ? 0xffffffff : 0);
437 }
438 
439 /*
440  * Different management commands all use this common base layer of code to issue
441  * commands and poll for results.
442  */
443 
444 /*
445  * Returns a pointer to where the caller should fill in their management command
446  * (caller should ignore the verb byte)
447  */
448 void *qbman_swp_mc_start(struct qbman_swp *p)
449 {
450 	if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
451 		return qbman_get_cmd(p, QBMAN_CENA_SWP_CR);
452 	else
453 		return qbman_get_cmd(p, QBMAN_CENA_SWP_CR_MEM);
454 }
455 
456 /*
457  * Commits merges in the caller-supplied command verb (which should not include
458  * the valid-bit) and submits the command to hardware
459  */
460 void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, u8 cmd_verb)
461 {
462 	u8 *v = cmd;
463 
464 	if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
465 		dma_wmb();
466 		*v = cmd_verb | p->mc.valid_bit;
467 	} else {
468 		*v = cmd_verb | p->mc.valid_bit;
469 		dma_wmb();
470 		qbman_write_register(p, QBMAN_CINH_SWP_CR_RT, QMAN_RT_MODE);
471 	}
472 }
473 
474 /*
475  * Checks for a completed response (returns non-NULL if only if the response
476  * is complete).
477  */
478 void *qbman_swp_mc_result(struct qbman_swp *p)
479 {
480 	u32 *ret, verb;
481 
482 	if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
483 		ret = qbman_get_cmd(p, QBMAN_CENA_SWP_RR(p->mc.valid_bit));
484 		/* Remove the valid-bit - command completed if the rest
485 		 * is non-zero.
486 		 */
487 		verb = ret[0] & ~QB_VALID_BIT;
488 		if (!verb)
489 			return NULL;
490 		p->mc.valid_bit ^= QB_VALID_BIT;
491 	} else {
492 		ret = qbman_get_cmd(p, QBMAN_CENA_SWP_RR_MEM);
493 		/* Command completed if the valid bit is toggled */
494 		if (p->mr.valid_bit != (ret[0] & QB_VALID_BIT))
495 			return NULL;
496 		/* Command completed if the rest is non-zero */
497 		verb = ret[0] & ~QB_VALID_BIT;
498 		if (!verb)
499 			return NULL;
500 		p->mr.valid_bit ^= QB_VALID_BIT;
501 	}
502 
503 	return ret;
504 }
505 
506 #define QB_ENQUEUE_CMD_OPTIONS_SHIFT    0
507 enum qb_enqueue_commands {
508 	enqueue_empty = 0,
509 	enqueue_response_always = 1,
510 	enqueue_rejects_to_fq = 2
511 };
512 
513 #define QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT      2
514 #define QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT 3
515 #define QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT     4
516 #define QB_ENQUEUE_CMD_DCA_EN_SHIFT          7
517 
518 /*
519  * qbman_eq_desc_clear() - Clear the contents of a descriptor to
520  *                         default/starting state.
521  */
522 void qbman_eq_desc_clear(struct qbman_eq_desc *d)
523 {
524 	memset(d, 0, sizeof(*d));
525 }
526 
527 /**
528  * qbman_eq_desc_set_no_orp() - Set enqueue descriptor without orp
529  * @d:                the enqueue descriptor.
530  * @respond_success:  1 = enqueue with response always; 0 = enqueue with
531  *                    rejections returned on a FQ.
532  */
533 void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success)
534 {
535 	d->verb &= ~(1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT);
536 	if (respond_success)
537 		d->verb |= enqueue_response_always;
538 	else
539 		d->verb |= enqueue_rejects_to_fq;
540 }
541 
542 /*
543  * Exactly one of the following descriptor "targets" should be set. (Calling any
544  * one of these will replace the effect of any prior call to one of these.)
545  *   -enqueue to a frame queue
546  *   -enqueue to a queuing destination
547  */
548 
549 /**
550  * qbman_eq_desc_set_fq() - set the FQ for the enqueue command
551  * @d:    the enqueue descriptor
552  * @fqid: the id of the frame queue to be enqueued
553  */
554 void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, u32 fqid)
555 {
556 	d->verb &= ~(1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT);
557 	d->tgtid = cpu_to_le32(fqid);
558 }
559 
560 /**
561  * qbman_eq_desc_set_qd() - Set Queuing Destination for the enqueue command
562  * @d:       the enqueue descriptor
563  * @qdid:    the id of the queuing destination to be enqueued
564  * @qd_bin:  the queuing destination bin
565  * @qd_prio: the queuing destination priority
566  */
567 void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, u32 qdid,
568 			  u32 qd_bin, u32 qd_prio)
569 {
570 	d->verb |= 1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT;
571 	d->tgtid = cpu_to_le32(qdid);
572 	d->qdbin = cpu_to_le16(qd_bin);
573 	d->qpri = qd_prio;
574 }
575 
576 #define EQAR_IDX(eqar)     ((eqar) & 0x7)
577 #define EQAR_VB(eqar)      ((eqar) & 0x80)
578 #define EQAR_SUCCESS(eqar) ((eqar) & 0x100)
579 
580 #define QB_RT_BIT ((u32)0x100)
581 /**
582  * qbman_swp_enqueue_direct() - Issue an enqueue command
583  * @s:  the software portal used for enqueue
584  * @d:  the enqueue descriptor
585  * @fd: the frame descriptor to be enqueued
586  *
587  * Please note that 'fd' should only be NULL if the "action" of the
588  * descriptor is "orp_hole" or "orp_nesn".
589  *
590  * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready.
591  */
592 static
593 int qbman_swp_enqueue_direct(struct qbman_swp *s,
594 			     const struct qbman_eq_desc *d,
595 			     const struct dpaa2_fd *fd)
596 {
597 	int flags = 0;
598 	int ret = qbman_swp_enqueue_multiple_direct(s, d, fd, &flags, 1);
599 
600 	if (ret >= 0)
601 		ret = 0;
602 	else
603 		ret = -EBUSY;
604 	return  ret;
605 }
606 
607 /**
608  * qbman_swp_enqueue_mem_back() - Issue an enqueue command
609  * @s:  the software portal used for enqueue
610  * @d:  the enqueue descriptor
611  * @fd: the frame descriptor to be enqueued
612  *
613  * Please note that 'fd' should only be NULL if the "action" of the
614  * descriptor is "orp_hole" or "orp_nesn".
615  *
616  * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready.
617  */
618 static
619 int qbman_swp_enqueue_mem_back(struct qbman_swp *s,
620 			       const struct qbman_eq_desc *d,
621 			       const struct dpaa2_fd *fd)
622 {
623 	int flags = 0;
624 	int ret = qbman_swp_enqueue_multiple_mem_back(s, d, fd, &flags, 1);
625 
626 	if (ret >= 0)
627 		ret = 0;
628 	else
629 		ret = -EBUSY;
630 	return  ret;
631 }
632 
633 /**
634  * qbman_swp_enqueue_multiple_direct() - Issue a multi enqueue command
635  * using one enqueue descriptor
636  * @s:  the software portal used for enqueue
637  * @d:  the enqueue descriptor
638  * @fd: table pointer of frame descriptor table to be enqueued
639  * @flags: table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL
640  * @num_frames: number of fd to be enqueued
641  *
642  * Return the number of fd enqueued, or a negative error number.
643  */
644 static
645 int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
646 				      const struct qbman_eq_desc *d,
647 				      const struct dpaa2_fd *fd,
648 				      uint32_t *flags,
649 				      int num_frames)
650 {
651 	uint32_t *p = NULL;
652 	const uint32_t *cl = (uint32_t *)d;
653 	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
654 	int i, num_enqueued = 0;
655 
656 	spin_lock(&s->access_spinlock);
657 	half_mask = (s->eqcr.pi_ci_mask>>1);
658 	full_mask = s->eqcr.pi_ci_mask;
659 
660 	if (!s->eqcr.available) {
661 		eqcr_ci = s->eqcr.ci;
662 		p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI;
663 		s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI);
664 		s->eqcr.ci &= full_mask;
665 
666 		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
667 					eqcr_ci, s->eqcr.ci);
668 		if (!s->eqcr.available) {
669 			spin_unlock(&s->access_spinlock);
670 			return 0;
671 		}
672 	}
673 
674 	eqcr_pi = s->eqcr.pi;
675 	num_enqueued = (s->eqcr.available < num_frames) ?
676 			s->eqcr.available : num_frames;
677 	s->eqcr.available -= num_enqueued;
678 	/* Fill in the EQCR ring */
679 	for (i = 0; i < num_enqueued; i++) {
680 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
681 		/* Skip copying the verb */
682 		memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
683 		memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
684 		       &fd[i], sizeof(*fd));
685 		eqcr_pi++;
686 	}
687 
688 	dma_wmb();
689 
690 	/* Set the verb byte, have to substitute in the valid-bit */
691 	eqcr_pi = s->eqcr.pi;
692 	for (i = 0; i < num_enqueued; i++) {
693 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
694 		p[0] = cl[0] | s->eqcr.pi_vb;
695 		if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
696 			struct qbman_eq_desc *eq_desc = (struct qbman_eq_desc *)p;
697 
698 			eq_desc->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
699 				((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
700 		}
701 		eqcr_pi++;
702 		if (!(eqcr_pi & half_mask))
703 			s->eqcr.pi_vb ^= QB_VALID_BIT;
704 	}
705 
706 	/* Flush all the cacheline without load/store in between */
707 	eqcr_pi = s->eqcr.pi;
708 	for (i = 0; i < num_enqueued; i++)
709 		eqcr_pi++;
710 	s->eqcr.pi = eqcr_pi & full_mask;
711 	spin_unlock(&s->access_spinlock);
712 
713 	return num_enqueued;
714 }
715 
716 /**
717  * qbman_swp_enqueue_multiple_mem_back() - Issue a multi enqueue command
718  * using one enqueue descriptor
719  * @s:  the software portal used for enqueue
720  * @d:  the enqueue descriptor
721  * @fd: table pointer of frame descriptor table to be enqueued
722  * @flags: table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL
723  * @num_frames: number of fd to be enqueued
724  *
725  * Return the number of fd enqueued, or a negative error number.
726  */
727 static
728 int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
729 					const struct qbman_eq_desc *d,
730 					const struct dpaa2_fd *fd,
731 					uint32_t *flags,
732 					int num_frames)
733 {
734 	uint32_t *p = NULL;
735 	const uint32_t *cl = (uint32_t *)(d);
736 	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
737 	int i, num_enqueued = 0;
738 	unsigned long irq_flags;
739 
740 	spin_lock_irqsave(&s->access_spinlock, irq_flags);
741 
742 	half_mask = (s->eqcr.pi_ci_mask>>1);
743 	full_mask = s->eqcr.pi_ci_mask;
744 	if (!s->eqcr.available) {
745 		eqcr_ci = s->eqcr.ci;
746 		s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI);
747 		s->eqcr.ci &= full_mask;
748 		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
749 					eqcr_ci, s->eqcr.ci);
750 		if (!s->eqcr.available) {
751 			spin_unlock_irqrestore(&s->access_spinlock, irq_flags);
752 			return 0;
753 		}
754 	}
755 
756 	eqcr_pi = s->eqcr.pi;
757 	num_enqueued = (s->eqcr.available < num_frames) ?
758 			s->eqcr.available : num_frames;
759 	s->eqcr.available -= num_enqueued;
760 	/* Fill in the EQCR ring */
761 	for (i = 0; i < num_enqueued; i++) {
762 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
763 		/* Skip copying the verb */
764 		memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
765 		memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
766 		       &fd[i], sizeof(*fd));
767 		eqcr_pi++;
768 	}
769 
770 	/* Set the verb byte, have to substitute in the valid-bit */
771 	eqcr_pi = s->eqcr.pi;
772 	for (i = 0; i < num_enqueued; i++) {
773 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
774 		p[0] = cl[0] | s->eqcr.pi_vb;
775 		if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
776 			struct qbman_eq_desc *eq_desc = (struct qbman_eq_desc *)p;
777 
778 			eq_desc->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
779 				((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
780 		}
781 		eqcr_pi++;
782 		if (!(eqcr_pi & half_mask))
783 			s->eqcr.pi_vb ^= QB_VALID_BIT;
784 	}
785 	s->eqcr.pi = eqcr_pi & full_mask;
786 
787 	dma_wmb();
788 	qbman_write_register(s, QBMAN_CINH_SWP_EQCR_PI,
789 				(QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
790 	spin_unlock_irqrestore(&s->access_spinlock, irq_flags);
791 
792 	return num_enqueued;
793 }
794 
795 /**
796  * qbman_swp_enqueue_multiple_desc_direct() - Issue a multi enqueue command
797  * using multiple enqueue descriptor
798  * @s:  the software portal used for enqueue
799  * @d:  table of minimal enqueue descriptor
800  * @fd: table pointer of frame descriptor table to be enqueued
801  * @num_frames: number of fd to be enqueued
802  *
803  * Return the number of fd enqueued, or a negative error number.
804  */
805 static
806 int qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
807 					   const struct qbman_eq_desc *d,
808 					   const struct dpaa2_fd *fd,
809 					   int num_frames)
810 {
811 	uint32_t *p;
812 	const uint32_t *cl;
813 	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
814 	int i, num_enqueued = 0;
815 
816 	half_mask = (s->eqcr.pi_ci_mask>>1);
817 	full_mask = s->eqcr.pi_ci_mask;
818 	if (!s->eqcr.available) {
819 		eqcr_ci = s->eqcr.ci;
820 		p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI;
821 		s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI);
822 		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
823 					eqcr_ci, s->eqcr.ci);
824 		if (!s->eqcr.available)
825 			return 0;
826 	}
827 
828 	eqcr_pi = s->eqcr.pi;
829 	num_enqueued = (s->eqcr.available < num_frames) ?
830 			s->eqcr.available : num_frames;
831 	s->eqcr.available -= num_enqueued;
832 	/* Fill in the EQCR ring */
833 	for (i = 0; i < num_enqueued; i++) {
834 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
835 		cl = (uint32_t *)(&d[i]);
836 		/* Skip copying the verb */
837 		memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
838 		memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
839 		       &fd[i], sizeof(*fd));
840 		eqcr_pi++;
841 	}
842 
843 	dma_wmb();
844 
845 	/* Set the verb byte, have to substitute in the valid-bit */
846 	eqcr_pi = s->eqcr.pi;
847 	for (i = 0; i < num_enqueued; i++) {
848 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
849 		cl = (uint32_t *)(&d[i]);
850 		p[0] = cl[0] | s->eqcr.pi_vb;
851 		eqcr_pi++;
852 		if (!(eqcr_pi & half_mask))
853 			s->eqcr.pi_vb ^= QB_VALID_BIT;
854 	}
855 
856 	/* Flush all the cacheline without load/store in between */
857 	eqcr_pi = s->eqcr.pi;
858 	for (i = 0; i < num_enqueued; i++)
859 		eqcr_pi++;
860 	s->eqcr.pi = eqcr_pi & full_mask;
861 
862 	return num_enqueued;
863 }
864 
865 /**
866  * qbman_swp_enqueue_multiple_desc_mem_back() - Issue a multi enqueue command
867  * using multiple enqueue descriptor
868  * @s:  the software portal used for enqueue
869  * @d:  table of minimal enqueue descriptor
870  * @fd: table pointer of frame descriptor table to be enqueued
871  * @num_frames: number of fd to be enqueued
872  *
873  * Return the number of fd enqueued, or a negative error number.
874  */
875 static
876 int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
877 					   const struct qbman_eq_desc *d,
878 					   const struct dpaa2_fd *fd,
879 					   int num_frames)
880 {
881 	uint32_t *p;
882 	const uint32_t *cl;
883 	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
884 	int i, num_enqueued = 0;
885 
886 	half_mask = (s->eqcr.pi_ci_mask>>1);
887 	full_mask = s->eqcr.pi_ci_mask;
888 	if (!s->eqcr.available) {
889 		eqcr_ci = s->eqcr.ci;
890 		s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI);
891 		s->eqcr.ci &= full_mask;
892 		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
893 					eqcr_ci, s->eqcr.ci);
894 		if (!s->eqcr.available)
895 			return 0;
896 	}
897 
898 	eqcr_pi = s->eqcr.pi;
899 	num_enqueued = (s->eqcr.available < num_frames) ?
900 			s->eqcr.available : num_frames;
901 	s->eqcr.available -= num_enqueued;
902 	/* Fill in the EQCR ring */
903 	for (i = 0; i < num_enqueued; i++) {
904 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
905 		cl = (uint32_t *)(&d[i]);
906 		/* Skip copying the verb */
907 		memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
908 		memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
909 		       &fd[i], sizeof(*fd));
910 		eqcr_pi++;
911 	}
912 
913 	/* Set the verb byte, have to substitute in the valid-bit */
914 	eqcr_pi = s->eqcr.pi;
915 	for (i = 0; i < num_enqueued; i++) {
916 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
917 		cl = (uint32_t *)(&d[i]);
918 		p[0] = cl[0] | s->eqcr.pi_vb;
919 		eqcr_pi++;
920 		if (!(eqcr_pi & half_mask))
921 			s->eqcr.pi_vb ^= QB_VALID_BIT;
922 	}
923 
924 	s->eqcr.pi = eqcr_pi & full_mask;
925 
926 	dma_wmb();
927 	qbman_write_register(s, QBMAN_CINH_SWP_EQCR_PI,
928 				(QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
929 
930 	return num_enqueued;
931 }
932 
933 /* Static (push) dequeue */
934 
935 /**
936  * qbman_swp_push_get() - Get the push dequeue setup
937  * @s:           the software portal object
938  * @channel_idx: the channel index to query
939  * @enabled:     returned boolean to show whether the push dequeue is enabled
940  *               for the given channel
941  */
942 void qbman_swp_push_get(struct qbman_swp *s, u8 channel_idx, int *enabled)
943 {
944 	u16 src = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
945 
946 	WARN_ON(channel_idx > 15);
947 	*enabled = src | (1 << channel_idx);
948 }
949 
950 /**
951  * qbman_swp_push_set() - Enable or disable push dequeue
952  * @s:           the software portal object
953  * @channel_idx: the channel index (0 to 15)
954  * @enable:      enable or disable push dequeue
955  */
956 void qbman_swp_push_set(struct qbman_swp *s, u8 channel_idx, int enable)
957 {
958 	u16 dqsrc;
959 
960 	WARN_ON(channel_idx > 15);
961 	if (enable)
962 		s->sdq |= 1 << channel_idx;
963 	else
964 		s->sdq &= ~(1 << channel_idx);
965 
966 	/* Read make the complete src map.  If no channels are enabled
967 	 * the SDQCR must be 0 or else QMan will assert errors
968 	 */
969 	dqsrc = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
970 	if (dqsrc != 0)
971 		qbman_write_register(s, QBMAN_CINH_SWP_SDQCR, s->sdq);
972 	else
973 		qbman_write_register(s, QBMAN_CINH_SWP_SDQCR, 0);
974 }
975 
976 #define QB_VDQCR_VERB_DCT_SHIFT    0
977 #define QB_VDQCR_VERB_DT_SHIFT     2
978 #define QB_VDQCR_VERB_RLS_SHIFT    4
979 #define QB_VDQCR_VERB_WAE_SHIFT    5
980 
981 enum qb_pull_dt_e {
982 	qb_pull_dt_channel,
983 	qb_pull_dt_workqueue,
984 	qb_pull_dt_framequeue
985 };
986 
987 /**
988  * qbman_pull_desc_clear() - Clear the contents of a descriptor to
989  *                           default/starting state
990  * @d: the pull dequeue descriptor to be cleared
991  */
992 void qbman_pull_desc_clear(struct qbman_pull_desc *d)
993 {
994 	memset(d, 0, sizeof(*d));
995 }
996 
997 /**
998  * qbman_pull_desc_set_storage()- Set the pull dequeue storage
999  * @d:            the pull dequeue descriptor to be set
1000  * @storage:      the pointer of the memory to store the dequeue result
1001  * @storage_phys: the physical address of the storage memory
1002  * @stash:        to indicate whether write allocate is enabled
1003  *
1004  * If not called, or if called with 'storage' as NULL, the result pull dequeues
1005  * will produce results to DQRR. If 'storage' is non-NULL, then results are
1006  * produced to the given memory location (using the DMA address which
1007  * the caller provides in 'storage_phys'), and 'stash' controls whether or not
1008  * those writes to main-memory express a cache-warming attribute.
1009  */
1010 void qbman_pull_desc_set_storage(struct qbman_pull_desc *d,
1011 				 struct dpaa2_dq *storage,
1012 				 dma_addr_t storage_phys,
1013 				 int stash)
1014 {
1015 	/* save the virtual address */
1016 	d->rsp_addr_virt = (u64)(uintptr_t)storage;
1017 
1018 	if (!storage) {
1019 		d->verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT);
1020 		return;
1021 	}
1022 	d->verb |= 1 << QB_VDQCR_VERB_RLS_SHIFT;
1023 	if (stash)
1024 		d->verb |= 1 << QB_VDQCR_VERB_WAE_SHIFT;
1025 	else
1026 		d->verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT);
1027 
1028 	d->rsp_addr = cpu_to_le64(storage_phys);
1029 }
1030 
1031 /**
1032  * qbman_pull_desc_set_numframes() - Set the number of frames to be dequeued
1033  * @d:         the pull dequeue descriptor to be set
1034  * @numframes: number of frames to be set, must be between 1 and 16, inclusive
1035  */
1036 void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, u8 numframes)
1037 {
1038 	d->numf = numframes - 1;
1039 }
1040 
1041 /*
1042  * Exactly one of the following descriptor "actions" should be set. (Calling any
1043  * one of these will replace the effect of any prior call to one of these.)
1044  * - pull dequeue from the given frame queue (FQ)
1045  * - pull dequeue from any FQ in the given work queue (WQ)
1046  * - pull dequeue from any FQ in any WQ in the given channel
1047  */
1048 
1049 /**
1050  * qbman_pull_desc_set_fq() - Set fqid from which the dequeue command dequeues
1051  * @d:    the pull dequeue descriptor to be set
1052  * @fqid: the frame queue index of the given FQ
1053  */
1054 void qbman_pull_desc_set_fq(struct qbman_pull_desc *d, u32 fqid)
1055 {
1056 	d->verb |= 1 << QB_VDQCR_VERB_DCT_SHIFT;
1057 	d->verb |= qb_pull_dt_framequeue << QB_VDQCR_VERB_DT_SHIFT;
1058 	d->dq_src = cpu_to_le32(fqid);
1059 }
1060 
1061 /**
1062  * qbman_pull_desc_set_wq() - Set wqid from which the dequeue command dequeues
1063  * @d:    the pull dequeue descriptor to be set
1064  * @wqid: composed of channel id and wqid within the channel
1065  * @dct:  the dequeue command type
1066  */
1067 void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, u32 wqid,
1068 			    enum qbman_pull_type_e dct)
1069 {
1070 	d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
1071 	d->verb |= qb_pull_dt_workqueue << QB_VDQCR_VERB_DT_SHIFT;
1072 	d->dq_src = cpu_to_le32(wqid);
1073 }
1074 
1075 /**
1076  * qbman_pull_desc_set_channel() - Set channelid from which the dequeue command
1077  *                                 dequeues
1078  * @d:    the pull dequeue descriptor to be set
1079  * @chid: the channel id to be dequeued
1080  * @dct:  the dequeue command type
1081  */
1082 void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, u32 chid,
1083 				 enum qbman_pull_type_e dct)
1084 {
1085 	d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
1086 	d->verb |= qb_pull_dt_channel << QB_VDQCR_VERB_DT_SHIFT;
1087 	d->dq_src = cpu_to_le32(chid);
1088 }
1089 
1090 /**
1091  * qbman_swp_pull_direct() - Issue the pull dequeue command
1092  * @s: the software portal object
1093  * @d: the software portal descriptor which has been configured with
1094  *     the set of qbman_pull_desc_set_*() calls
1095  *
1096  * Return 0 for success, and -EBUSY if the software portal is not ready
1097  * to do pull dequeue.
1098  */
1099 static
1100 int qbman_swp_pull_direct(struct qbman_swp *s, struct qbman_pull_desc *d)
1101 {
1102 	struct qbman_pull_desc *p;
1103 
1104 	if (!atomic_dec_and_test(&s->vdq.available)) {
1105 		atomic_inc(&s->vdq.available);
1106 		return -EBUSY;
1107 	}
1108 	s->vdq.storage = (void *)(uintptr_t)d->rsp_addr_virt;
1109 	if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
1110 		p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR);
1111 	else
1112 		p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR_MEM);
1113 	p->numf = d->numf;
1114 	p->tok = QMAN_DQ_TOKEN_VALID;
1115 	p->dq_src = d->dq_src;
1116 	p->rsp_addr = d->rsp_addr;
1117 	p->rsp_addr_virt = d->rsp_addr_virt;
1118 	dma_wmb();
1119 	/* Set the verb byte, have to substitute in the valid-bit */
1120 	p->verb = d->verb | s->vdq.valid_bit;
1121 	s->vdq.valid_bit ^= QB_VALID_BIT;
1122 
1123 	return 0;
1124 }
1125 
1126 /**
1127  * qbman_swp_pull_mem_back() - Issue the pull dequeue command
1128  * @s: the software portal object
1129  * @d: the software portal descriptor which has been configured with
1130  *     the set of qbman_pull_desc_set_*() calls
1131  *
1132  * Return 0 for success, and -EBUSY if the software portal is not ready
1133  * to do pull dequeue.
1134  */
1135 static
1136 int qbman_swp_pull_mem_back(struct qbman_swp *s, struct qbman_pull_desc *d)
1137 {
1138 	struct qbman_pull_desc *p;
1139 
1140 	if (!atomic_dec_and_test(&s->vdq.available)) {
1141 		atomic_inc(&s->vdq.available);
1142 		return -EBUSY;
1143 	}
1144 	s->vdq.storage = (void *)(uintptr_t)d->rsp_addr_virt;
1145 	if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
1146 		p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR);
1147 	else
1148 		p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR_MEM);
1149 	p->numf = d->numf;
1150 	p->tok = QMAN_DQ_TOKEN_VALID;
1151 	p->dq_src = d->dq_src;
1152 	p->rsp_addr = d->rsp_addr;
1153 	p->rsp_addr_virt = d->rsp_addr_virt;
1154 
1155 	/* Set the verb byte, have to substitute in the valid-bit */
1156 	p->verb = d->verb | s->vdq.valid_bit;
1157 	s->vdq.valid_bit ^= QB_VALID_BIT;
1158 	dma_wmb();
1159 	qbman_write_register(s, QBMAN_CINH_SWP_VDQCR_RT, QMAN_RT_MODE);
1160 
1161 	return 0;
1162 }
1163 
1164 #define QMAN_DQRR_PI_MASK   0xf
1165 
1166 /**
1167  * qbman_swp_dqrr_next_direct() - Get an valid DQRR entry
1168  * @s: the software portal object
1169  *
1170  * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry
1171  * only once, so repeated calls can return a sequence of DQRR entries, without
1172  * requiring they be consumed immediately or in any particular order.
1173  */
1174 const struct dpaa2_dq *qbman_swp_dqrr_next_direct(struct qbman_swp *s)
1175 {
1176 	u32 verb;
1177 	u32 response_verb;
1178 	u32 flags;
1179 	struct dpaa2_dq *p;
1180 
1181 	/* Before using valid-bit to detect if something is there, we have to
1182 	 * handle the case of the DQRR reset bug...
1183 	 */
1184 	if (unlikely(s->dqrr.reset_bug)) {
1185 		/*
1186 		 * We pick up new entries by cache-inhibited producer index,
1187 		 * which means that a non-coherent mapping would require us to
1188 		 * invalidate and read *only* once that PI has indicated that
1189 		 * there's an entry here. The first trip around the DQRR ring
1190 		 * will be much less efficient than all subsequent trips around
1191 		 * it...
1192 		 */
1193 		u8 pi = qbman_read_register(s, QBMAN_CINH_SWP_DQPI) &
1194 			QMAN_DQRR_PI_MASK;
1195 
1196 		/* there are new entries if pi != next_idx */
1197 		if (pi == s->dqrr.next_idx)
1198 			return NULL;
1199 
1200 		/*
1201 		 * if next_idx is/was the last ring index, and 'pi' is
1202 		 * different, we can disable the workaround as all the ring
1203 		 * entries have now been DMA'd to so valid-bit checking is
1204 		 * repaired. Note: this logic needs to be based on next_idx
1205 		 * (which increments one at a time), rather than on pi (which
1206 		 * can burst and wrap-around between our snapshots of it).
1207 		 */
1208 		if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) {
1209 			pr_debug("next_idx=%d, pi=%d, clear reset bug\n",
1210 				 s->dqrr.next_idx, pi);
1211 			s->dqrr.reset_bug = 0;
1212 		}
1213 		prefetch(qbman_get_cmd(s,
1214 				       QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1215 	}
1216 
1217 	p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
1218 	verb = p->dq.verb;
1219 
1220 	/*
1221 	 * If the valid-bit isn't of the expected polarity, nothing there. Note,
1222 	 * in the DQRR reset bug workaround, we shouldn't need to skip these
1223 	 * check, because we've already determined that a new entry is available
1224 	 * and we've invalidated the cacheline before reading it, so the
1225 	 * valid-bit behaviour is repaired and should tell us what we already
1226 	 * knew from reading PI.
1227 	 */
1228 	if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) {
1229 		prefetch(qbman_get_cmd(s,
1230 				       QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1231 		return NULL;
1232 	}
1233 	/*
1234 	 * There's something there. Move "next_idx" attention to the next ring
1235 	 * entry (and prefetch it) before returning what we found.
1236 	 */
1237 	s->dqrr.next_idx++;
1238 	s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */
1239 	if (!s->dqrr.next_idx)
1240 		s->dqrr.valid_bit ^= QB_VALID_BIT;
1241 
1242 	/*
1243 	 * If this is the final response to a volatile dequeue command
1244 	 * indicate that the vdq is available
1245 	 */
1246 	flags = p->dq.stat;
1247 	response_verb = verb & QBMAN_RESULT_MASK;
1248 	if ((response_verb == QBMAN_RESULT_DQ) &&
1249 	    (flags & DPAA2_DQ_STAT_VOLATILE) &&
1250 	    (flags & DPAA2_DQ_STAT_EXPIRED))
1251 		atomic_inc(&s->vdq.available);
1252 
1253 	prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1254 
1255 	return p;
1256 }
1257 
1258 /**
1259  * qbman_swp_dqrr_next_mem_back() - Get an valid DQRR entry
1260  * @s: the software portal object
1261  *
1262  * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry
1263  * only once, so repeated calls can return a sequence of DQRR entries, without
1264  * requiring they be consumed immediately or in any particular order.
1265  */
1266 const struct dpaa2_dq *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s)
1267 {
1268 	u32 verb;
1269 	u32 response_verb;
1270 	u32 flags;
1271 	struct dpaa2_dq *p;
1272 
1273 	/* Before using valid-bit to detect if something is there, we have to
1274 	 * handle the case of the DQRR reset bug...
1275 	 */
1276 	if (unlikely(s->dqrr.reset_bug)) {
1277 		/*
1278 		 * We pick up new entries by cache-inhibited producer index,
1279 		 * which means that a non-coherent mapping would require us to
1280 		 * invalidate and read *only* once that PI has indicated that
1281 		 * there's an entry here. The first trip around the DQRR ring
1282 		 * will be much less efficient than all subsequent trips around
1283 		 * it...
1284 		 */
1285 		u8 pi = qbman_read_register(s, QBMAN_CINH_SWP_DQPI) &
1286 			QMAN_DQRR_PI_MASK;
1287 
1288 		/* there are new entries if pi != next_idx */
1289 		if (pi == s->dqrr.next_idx)
1290 			return NULL;
1291 
1292 		/*
1293 		 * if next_idx is/was the last ring index, and 'pi' is
1294 		 * different, we can disable the workaround as all the ring
1295 		 * entries have now been DMA'd to so valid-bit checking is
1296 		 * repaired. Note: this logic needs to be based on next_idx
1297 		 * (which increments one at a time), rather than on pi (which
1298 		 * can burst and wrap-around between our snapshots of it).
1299 		 */
1300 		if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) {
1301 			pr_debug("next_idx=%d, pi=%d, clear reset bug\n",
1302 				 s->dqrr.next_idx, pi);
1303 			s->dqrr.reset_bug = 0;
1304 		}
1305 		prefetch(qbman_get_cmd(s,
1306 				       QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1307 	}
1308 
1309 	p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR_MEM(s->dqrr.next_idx));
1310 	verb = p->dq.verb;
1311 
1312 	/*
1313 	 * If the valid-bit isn't of the expected polarity, nothing there. Note,
1314 	 * in the DQRR reset bug workaround, we shouldn't need to skip these
1315 	 * check, because we've already determined that a new entry is available
1316 	 * and we've invalidated the cacheline before reading it, so the
1317 	 * valid-bit behaviour is repaired and should tell us what we already
1318 	 * knew from reading PI.
1319 	 */
1320 	if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) {
1321 		prefetch(qbman_get_cmd(s,
1322 				       QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1323 		return NULL;
1324 	}
1325 	/*
1326 	 * There's something there. Move "next_idx" attention to the next ring
1327 	 * entry (and prefetch it) before returning what we found.
1328 	 */
1329 	s->dqrr.next_idx++;
1330 	s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */
1331 	if (!s->dqrr.next_idx)
1332 		s->dqrr.valid_bit ^= QB_VALID_BIT;
1333 
1334 	/*
1335 	 * If this is the final response to a volatile dequeue command
1336 	 * indicate that the vdq is available
1337 	 */
1338 	flags = p->dq.stat;
1339 	response_verb = verb & QBMAN_RESULT_MASK;
1340 	if ((response_verb == QBMAN_RESULT_DQ) &&
1341 	    (flags & DPAA2_DQ_STAT_VOLATILE) &&
1342 	    (flags & DPAA2_DQ_STAT_EXPIRED))
1343 		atomic_inc(&s->vdq.available);
1344 
1345 	prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1346 
1347 	return p;
1348 }
1349 
1350 /**
1351  * qbman_swp_dqrr_consume() -  Consume DQRR entries previously returned from
1352  *                             qbman_swp_dqrr_next().
1353  * @s: the software portal object
1354  * @dq: the DQRR entry to be consumed
1355  */
1356 void qbman_swp_dqrr_consume(struct qbman_swp *s, const struct dpaa2_dq *dq)
1357 {
1358 	qbman_write_register(s, QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq));
1359 }
1360 
1361 /**
1362  * qbman_result_has_new_result() - Check and get the dequeue response from the
1363  *                                 dq storage memory set in pull dequeue command
1364  * @s: the software portal object
1365  * @dq: the dequeue result read from the memory
1366  *
1367  * Return 1 for getting a valid dequeue result, or 0 for not getting a valid
1368  * dequeue result.
1369  *
1370  * Only used for user-provided storage of dequeue results, not DQRR. For
1371  * efficiency purposes, the driver will perform any required endianness
1372  * conversion to ensure that the user's dequeue result storage is in host-endian
1373  * format. As such, once the user has called qbman_result_has_new_result() and
1374  * been returned a valid dequeue result, they should not call it again on
1375  * the same memory location (except of course if another dequeue command has
1376  * been executed to produce a new result to that location).
1377  */
1378 int qbman_result_has_new_result(struct qbman_swp *s, const struct dpaa2_dq *dq)
1379 {
1380 	if (dq->dq.tok != QMAN_DQ_TOKEN_VALID)
1381 		return 0;
1382 
1383 	/*
1384 	 * Set token to be 0 so we will detect change back to 1
1385 	 * next time the looping is traversed. Const is cast away here
1386 	 * as we want users to treat the dequeue responses as read only.
1387 	 */
1388 	((struct dpaa2_dq *)dq)->dq.tok = 0;
1389 
1390 	/*
1391 	 * Determine whether VDQCR is available based on whether the
1392 	 * current result is sitting in the first storage location of
1393 	 * the busy command.
1394 	 */
1395 	if (s->vdq.storage == dq) {
1396 		s->vdq.storage = NULL;
1397 		atomic_inc(&s->vdq.available);
1398 	}
1399 
1400 	return 1;
1401 }
1402 
1403 /**
1404  * qbman_release_desc_clear() - Clear the contents of a descriptor to
1405  *                              default/starting state.
1406  * @d: the pull dequeue descriptor to be cleared
1407  */
1408 void qbman_release_desc_clear(struct qbman_release_desc *d)
1409 {
1410 	memset(d, 0, sizeof(*d));
1411 	d->verb = 1 << 5; /* Release Command Valid */
1412 }
1413 
1414 /**
1415  * qbman_release_desc_set_bpid() - Set the ID of the buffer pool to release to
1416  * @d:    the pull dequeue descriptor to be set
1417  * @bpid: the bpid value to be set
1418  */
1419 void qbman_release_desc_set_bpid(struct qbman_release_desc *d, u16 bpid)
1420 {
1421 	d->bpid = cpu_to_le16(bpid);
1422 }
1423 
1424 /**
1425  * qbman_release_desc_set_rcdi() - Determines whether or not the portal's RCDI
1426  * interrupt source should be asserted after the release command is completed.
1427  * @d:      the pull dequeue descriptor to be set
1428  * @enable: enable (1) or disable (0) value
1429  */
1430 void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable)
1431 {
1432 	if (enable)
1433 		d->verb |= 1 << 6;
1434 	else
1435 		d->verb &= ~(1 << 6);
1436 }
1437 
1438 #define RAR_IDX(rar)     ((rar) & 0x7)
1439 #define RAR_VB(rar)      ((rar) & 0x80)
1440 #define RAR_SUCCESS(rar) ((rar) & 0x100)
1441 
1442 /**
1443  * qbman_swp_release_direct() - Issue a buffer release command
1444  * @s:           the software portal object
1445  * @d:           the release descriptor
1446  * @buffers:     a pointer pointing to the buffer address to be released
1447  * @num_buffers: number of buffers to be released,  must be less than 8
1448  *
1449  * Return 0 for success, -EBUSY if the release command ring is not ready.
1450  */
1451 int qbman_swp_release_direct(struct qbman_swp *s,
1452 			     const struct qbman_release_desc *d,
1453 			     const u64 *buffers, unsigned int num_buffers)
1454 {
1455 	int i;
1456 	struct qbman_release_desc *p;
1457 	u32 rar;
1458 
1459 	if (!num_buffers || (num_buffers > 7))
1460 		return -EINVAL;
1461 
1462 	rar = qbman_read_register(s, QBMAN_CINH_SWP_RAR);
1463 	if (!RAR_SUCCESS(rar))
1464 		return -EBUSY;
1465 
1466 	/* Start the release command */
1467 	p = qbman_get_cmd(s, QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
1468 
1469 	/* Copy the caller's buffer pointers to the command */
1470 	for (i = 0; i < num_buffers; i++)
1471 		p->buf[i] = cpu_to_le64(buffers[i]);
1472 	p->bpid = d->bpid;
1473 
1474 	/*
1475 	 * Set the verb byte, have to substitute in the valid-bit
1476 	 * and the number of buffers.
1477 	 */
1478 	dma_wmb();
1479 	p->verb = d->verb | RAR_VB(rar) | num_buffers;
1480 
1481 	return 0;
1482 }
1483 
1484 /**
1485  * qbman_swp_release_mem_back() - Issue a buffer release command
1486  * @s:           the software portal object
1487  * @d:           the release descriptor
1488  * @buffers:     a pointer pointing to the buffer address to be released
1489  * @num_buffers: number of buffers to be released,  must be less than 8
1490  *
1491  * Return 0 for success, -EBUSY if the release command ring is not ready.
1492  */
1493 int qbman_swp_release_mem_back(struct qbman_swp *s,
1494 			       const struct qbman_release_desc *d,
1495 			       const u64 *buffers, unsigned int num_buffers)
1496 {
1497 	int i;
1498 	struct qbman_release_desc *p;
1499 	u32 rar;
1500 
1501 	if (!num_buffers || (num_buffers > 7))
1502 		return -EINVAL;
1503 
1504 	rar = qbman_read_register(s, QBMAN_CINH_SWP_RAR);
1505 	if (!RAR_SUCCESS(rar))
1506 		return -EBUSY;
1507 
1508 	/* Start the release command */
1509 	p = qbman_get_cmd(s, QBMAN_CENA_SWP_RCR_MEM(RAR_IDX(rar)));
1510 
1511 	/* Copy the caller's buffer pointers to the command */
1512 	for (i = 0; i < num_buffers; i++)
1513 		p->buf[i] = cpu_to_le64(buffers[i]);
1514 	p->bpid = d->bpid;
1515 
1516 	p->verb = d->verb | RAR_VB(rar) | num_buffers;
1517 	dma_wmb();
1518 	qbman_write_register(s, QBMAN_CINH_SWP_RCR_AM_RT +
1519 			     RAR_IDX(rar)  * 4, QMAN_RT_MODE);
1520 
1521 	return 0;
1522 }
1523 
1524 struct qbman_acquire_desc {
1525 	u8 verb;
1526 	u8 reserved;
1527 	__le16 bpid;
1528 	u8 num;
1529 	u8 reserved2[59];
1530 };
1531 
1532 struct qbman_acquire_rslt {
1533 	u8 verb;
1534 	u8 rslt;
1535 	__le16 reserved;
1536 	u8 num;
1537 	u8 reserved2[3];
1538 	__le64 buf[7];
1539 };
1540 
1541 /**
1542  * qbman_swp_acquire() - Issue a buffer acquire command
1543  * @s:           the software portal object
1544  * @bpid:        the buffer pool index
1545  * @buffers:     a pointer pointing to the acquired buffer addresses
1546  * @num_buffers: number of buffers to be acquired, must be less than 8
1547  *
1548  * Return 0 for success, or negative error code if the acquire command
1549  * fails.
1550  */
1551 int qbman_swp_acquire(struct qbman_swp *s, u16 bpid, u64 *buffers,
1552 		      unsigned int num_buffers)
1553 {
1554 	struct qbman_acquire_desc *p;
1555 	struct qbman_acquire_rslt *r;
1556 	int i;
1557 
1558 	if (!num_buffers || (num_buffers > 7))
1559 		return -EINVAL;
1560 
1561 	/* Start the management command */
1562 	p = qbman_swp_mc_start(s);
1563 
1564 	if (!p)
1565 		return -EBUSY;
1566 
1567 	/* Encode the caller-provided attributes */
1568 	p->bpid = cpu_to_le16(bpid);
1569 	p->num = num_buffers;
1570 
1571 	/* Complete the management command */
1572 	r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE);
1573 	if (unlikely(!r)) {
1574 		pr_err("qbman: acquire from BPID %d failed, no response\n",
1575 		       bpid);
1576 		return -EIO;
1577 	}
1578 
1579 	/* Decode the outcome */
1580 	WARN_ON((r->verb & 0x7f) != QBMAN_MC_ACQUIRE);
1581 
1582 	/* Determine success or failure */
1583 	if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
1584 		pr_err("qbman: acquire from BPID 0x%x failed, code=0x%02x\n",
1585 		       bpid, r->rslt);
1586 		return -EIO;
1587 	}
1588 
1589 	WARN_ON(r->num > num_buffers);
1590 
1591 	/* Copy the acquired buffers to the caller's array */
1592 	for (i = 0; i < r->num; i++)
1593 		buffers[i] = le64_to_cpu(r->buf[i]);
1594 
1595 	return (int)r->num;
1596 }
1597 
1598 struct qbman_alt_fq_state_desc {
1599 	u8 verb;
1600 	u8 reserved[3];
1601 	__le32 fqid;
1602 	u8 reserved2[56];
1603 };
1604 
1605 struct qbman_alt_fq_state_rslt {
1606 	u8 verb;
1607 	u8 rslt;
1608 	u8 reserved[62];
1609 };
1610 
1611 #define ALT_FQ_FQID_MASK 0x00FFFFFF
1612 
1613 int qbman_swp_alt_fq_state(struct qbman_swp *s, u32 fqid,
1614 			   u8 alt_fq_verb)
1615 {
1616 	struct qbman_alt_fq_state_desc *p;
1617 	struct qbman_alt_fq_state_rslt *r;
1618 
1619 	/* Start the management command */
1620 	p = qbman_swp_mc_start(s);
1621 	if (!p)
1622 		return -EBUSY;
1623 
1624 	p->fqid = cpu_to_le32(fqid & ALT_FQ_FQID_MASK);
1625 
1626 	/* Complete the management command */
1627 	r = qbman_swp_mc_complete(s, p, alt_fq_verb);
1628 	if (unlikely(!r)) {
1629 		pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n",
1630 		       alt_fq_verb);
1631 		return -EIO;
1632 	}
1633 
1634 	/* Decode the outcome */
1635 	WARN_ON((r->verb & QBMAN_RESULT_MASK) != alt_fq_verb);
1636 
1637 	/* Determine success or failure */
1638 	if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
1639 		pr_err("qbman: ALT FQID %d failed: verb = 0x%08x code = 0x%02x\n",
1640 		       fqid, r->verb, r->rslt);
1641 		return -EIO;
1642 	}
1643 
1644 	return 0;
1645 }
1646 
1647 struct qbman_cdan_ctrl_desc {
1648 	u8 verb;
1649 	u8 reserved;
1650 	__le16 ch;
1651 	u8 we;
1652 	u8 ctrl;
1653 	__le16 reserved2;
1654 	__le64 cdan_ctx;
1655 	u8 reserved3[48];
1656 
1657 };
1658 
1659 struct qbman_cdan_ctrl_rslt {
1660 	u8 verb;
1661 	u8 rslt;
1662 	__le16 ch;
1663 	u8 reserved[60];
1664 };
1665 
1666 int qbman_swp_CDAN_set(struct qbman_swp *s, u16 channelid,
1667 		       u8 we_mask, u8 cdan_en,
1668 		       u64 ctx)
1669 {
1670 	struct qbman_cdan_ctrl_desc *p = NULL;
1671 	struct qbman_cdan_ctrl_rslt *r = NULL;
1672 
1673 	/* Start the management command */
1674 	p = qbman_swp_mc_start(s);
1675 	if (!p)
1676 		return -EBUSY;
1677 
1678 	/* Encode the caller-provided attributes */
1679 	p->ch = cpu_to_le16(channelid);
1680 	p->we = we_mask;
1681 	if (cdan_en)
1682 		p->ctrl = 1;
1683 	else
1684 		p->ctrl = 0;
1685 	p->cdan_ctx = cpu_to_le64(ctx);
1686 
1687 	/* Complete the management command */
1688 	r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE);
1689 	if (unlikely(!r)) {
1690 		pr_err("qbman: wqchan config failed, no response\n");
1691 		return -EIO;
1692 	}
1693 
1694 	WARN_ON((r->verb & 0x7f) != QBMAN_WQCHAN_CONFIGURE);
1695 
1696 	/* Determine success or failure */
1697 	if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
1698 		pr_err("qbman: CDAN cQID %d failed: code = 0x%02x\n",
1699 		       channelid, r->rslt);
1700 		return -EIO;
1701 	}
1702 
1703 	return 0;
1704 }
1705 
1706 #define QBMAN_RESPONSE_VERB_MASK	0x7f
1707 #define QBMAN_FQ_QUERY_NP		0x45
1708 #define QBMAN_BP_QUERY			0x32
1709 
1710 struct qbman_fq_query_desc {
1711 	u8 verb;
1712 	u8 reserved[3];
1713 	__le32 fqid;
1714 	u8 reserved2[56];
1715 };
1716 
1717 int qbman_fq_query_state(struct qbman_swp *s, u32 fqid,
1718 			 struct qbman_fq_query_np_rslt *r)
1719 {
1720 	struct qbman_fq_query_desc *p;
1721 	void *resp;
1722 
1723 	p = (struct qbman_fq_query_desc *)qbman_swp_mc_start(s);
1724 	if (!p)
1725 		return -EBUSY;
1726 
1727 	/* FQID is a 24 bit value */
1728 	p->fqid = cpu_to_le32(fqid & 0x00FFFFFF);
1729 	resp = qbman_swp_mc_complete(s, p, QBMAN_FQ_QUERY_NP);
1730 	if (!resp) {
1731 		pr_err("qbman: Query FQID %d NP fields failed, no response\n",
1732 		       fqid);
1733 		return -EIO;
1734 	}
1735 	*r = *(struct qbman_fq_query_np_rslt *)resp;
1736 	/* Decode the outcome */
1737 	WARN_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_FQ_QUERY_NP);
1738 
1739 	/* Determine success or failure */
1740 	if (r->rslt != QBMAN_MC_RSLT_OK) {
1741 		pr_err("Query NP fields of FQID 0x%x failed, code=0x%02x\n",
1742 		       p->fqid, r->rslt);
1743 		return -EIO;
1744 	}
1745 
1746 	return 0;
1747 }
1748 
1749 u32 qbman_fq_state_frame_count(const struct qbman_fq_query_np_rslt *r)
1750 {
1751 	return (le32_to_cpu(r->frm_cnt) & 0x00FFFFFF);
1752 }
1753 
1754 u32 qbman_fq_state_byte_count(const struct qbman_fq_query_np_rslt *r)
1755 {
1756 	return le32_to_cpu(r->byte_cnt);
1757 }
1758 
1759 struct qbman_bp_query_desc {
1760 	u8 verb;
1761 	u8 reserved;
1762 	__le16 bpid;
1763 	u8 reserved2[60];
1764 };
1765 
1766 int qbman_bp_query(struct qbman_swp *s, u16 bpid,
1767 		   struct qbman_bp_query_rslt *r)
1768 {
1769 	struct qbman_bp_query_desc *p;
1770 	void *resp;
1771 
1772 	p = (struct qbman_bp_query_desc *)qbman_swp_mc_start(s);
1773 	if (!p)
1774 		return -EBUSY;
1775 
1776 	p->bpid = cpu_to_le16(bpid);
1777 	resp = qbman_swp_mc_complete(s, p, QBMAN_BP_QUERY);
1778 	if (!resp) {
1779 		pr_err("qbman: Query BPID %d fields failed, no response\n",
1780 		       bpid);
1781 		return -EIO;
1782 	}
1783 	*r = *(struct qbman_bp_query_rslt *)resp;
1784 	/* Decode the outcome */
1785 	WARN_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_BP_QUERY);
1786 
1787 	/* Determine success or failure */
1788 	if (r->rslt != QBMAN_MC_RSLT_OK) {
1789 		pr_err("Query fields of BPID 0x%x failed, code=0x%02x\n",
1790 		       bpid, r->rslt);
1791 		return -EIO;
1792 	}
1793 
1794 	return 0;
1795 }
1796 
1797 u32 qbman_bp_info_num_free_bufs(struct qbman_bp_query_rslt *a)
1798 {
1799 	return le32_to_cpu(a->fill);
1800 }
1801 
1802 /**
1803  * qbman_swp_set_irq_coalescing() - Set new IRQ coalescing values
1804  * @p: the software portal object
1805  * @irq_threshold: interrupt threshold
1806  * @irq_holdoff: interrupt holdoff (timeout) period in us
1807  *
1808  * Return 0 for success, or negative error code on error.
1809  */
1810 int qbman_swp_set_irq_coalescing(struct qbman_swp *p, u32 irq_threshold,
1811 				 u32 irq_holdoff)
1812 {
1813 	u32 itp, max_holdoff;
1814 
1815 	/* Convert irq_holdoff value from usecs to 256 QBMAN clock cycles
1816 	 * increments. This depends on the QBMAN internal frequency.
1817 	 */
1818 	itp = (irq_holdoff * 1000) / p->desc->qman_256_cycles_per_ns;
1819 	if (itp > 4096) {
1820 		max_holdoff = (p->desc->qman_256_cycles_per_ns * 4096) / 1000;
1821 		pr_err("irq_holdoff must be <= %uus\n", max_holdoff);
1822 		return -EINVAL;
1823 	}
1824 
1825 	if (irq_threshold >= p->dqrr.dqrr_size) {
1826 		pr_err("irq_threshold must be < %u\n", p->dqrr.dqrr_size - 1);
1827 		return -EINVAL;
1828 	}
1829 
1830 	p->irq_threshold = irq_threshold;
1831 	p->irq_holdoff = irq_holdoff;
1832 
1833 	qbman_write_register(p, QBMAN_CINH_SWP_DQRR_ITR, irq_threshold);
1834 	qbman_write_register(p, QBMAN_CINH_SWP_ITPR, itp);
1835 
1836 	return 0;
1837 }
1838 
1839 /**
1840  * qbman_swp_get_irq_coalescing() - Get the current IRQ coalescing parameters
1841  * @p: the software portal object
1842  * @irq_threshold: interrupt threshold (an IRQ is generated when there are more
1843  * DQRR entries in the portal than the threshold)
1844  * @irq_holdoff: interrupt holdoff (timeout) period in us
1845  */
1846 void qbman_swp_get_irq_coalescing(struct qbman_swp *p, u32 *irq_threshold,
1847 				  u32 *irq_holdoff)
1848 {
1849 	if (irq_threshold)
1850 		*irq_threshold = p->irq_threshold;
1851 	if (irq_holdoff)
1852 		*irq_holdoff = p->irq_holdoff;
1853 }
1854