1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /*
3  * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
4  * Copyright 2016-2019 NXP
5  *
6  */
7 
8 #include <asm/cacheflush.h>
9 #include <linux/io.h>
10 #include <linux/slab.h>
11 #include <linux/spinlock.h>
12 #include <soc/fsl/dpaa2-global.h>
13 
14 #include "qbman-portal.h"
15 
16 /* All QBMan command and result structures use this "valid bit" encoding */
17 #define QB_VALID_BIT ((u32)0x80)
18 
19 /* QBMan portal management command codes */
20 #define QBMAN_MC_ACQUIRE       0x30
21 #define QBMAN_WQCHAN_CONFIGURE 0x46
22 
23 /* CINH register offsets */
24 #define QBMAN_CINH_SWP_EQCR_PI      0x800
25 #define QBMAN_CINH_SWP_EQCR_CI	    0x840
26 #define QBMAN_CINH_SWP_EQAR    0x8c0
27 #define QBMAN_CINH_SWP_CR_RT        0x900
28 #define QBMAN_CINH_SWP_VDQCR_RT     0x940
29 #define QBMAN_CINH_SWP_EQCR_AM_RT   0x980
30 #define QBMAN_CINH_SWP_RCR_AM_RT    0x9c0
31 #define QBMAN_CINH_SWP_DQPI    0xa00
32 #define QBMAN_CINH_SWP_DCAP    0xac0
33 #define QBMAN_CINH_SWP_SDQCR   0xb00
34 #define QBMAN_CINH_SWP_EQCR_AM_RT2  0xb40
35 #define QBMAN_CINH_SWP_RCR_PI       0xc00
36 #define QBMAN_CINH_SWP_RAR     0xcc0
37 #define QBMAN_CINH_SWP_ISR     0xe00
38 #define QBMAN_CINH_SWP_IER     0xe40
39 #define QBMAN_CINH_SWP_ISDR    0xe80
40 #define QBMAN_CINH_SWP_IIR     0xec0
41 
42 /* CENA register offsets */
43 #define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((u32)(n) << 6))
44 #define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((u32)(n) << 6))
45 #define QBMAN_CENA_SWP_RCR(n)  (0x400 + ((u32)(n) << 6))
46 #define QBMAN_CENA_SWP_CR      0x600
47 #define QBMAN_CENA_SWP_RR(vb)  (0x700 + ((u32)(vb) >> 1))
48 #define QBMAN_CENA_SWP_VDQCR   0x780
49 #define QBMAN_CENA_SWP_EQCR_CI 0x840
50 #define QBMAN_CENA_SWP_EQCR_CI_MEMBACK 0x1840
51 
52 /* CENA register offsets in memory-backed mode */
53 #define QBMAN_CENA_SWP_DQRR_MEM(n)  (0x800 + ((u32)(n) << 6))
54 #define QBMAN_CENA_SWP_RCR_MEM(n)   (0x1400 + ((u32)(n) << 6))
55 #define QBMAN_CENA_SWP_CR_MEM       0x1600
56 #define QBMAN_CENA_SWP_RR_MEM       0x1680
57 #define QBMAN_CENA_SWP_VDQCR_MEM    0x1780
58 
59 /* Reverse mapping of QBMAN_CENA_SWP_DQRR() */
60 #define QBMAN_IDX_FROM_DQRR(p) (((unsigned long)(p) & 0x1ff) >> 6)
61 
62 /* Define token used to determine if response written to memory is valid */
63 #define QMAN_DQ_TOKEN_VALID 1
64 
65 /* SDQCR attribute codes */
66 #define QB_SDQCR_FC_SHIFT   29
67 #define QB_SDQCR_FC_MASK    0x1
68 #define QB_SDQCR_DCT_SHIFT  24
69 #define QB_SDQCR_DCT_MASK   0x3
70 #define QB_SDQCR_TOK_SHIFT  16
71 #define QB_SDQCR_TOK_MASK   0xff
72 #define QB_SDQCR_SRC_SHIFT  0
73 #define QB_SDQCR_SRC_MASK   0xffff
74 
75 /* opaque token for static dequeues */
76 #define QMAN_SDQCR_TOKEN    0xbb
77 
78 #define QBMAN_EQCR_DCA_IDXMASK          0x0f
79 #define QBMAN_ENQUEUE_FLAG_DCA          (1ULL << 31)
80 
81 #define EQ_DESC_SIZE_WITHOUT_FD 29
82 #define EQ_DESC_SIZE_FD_START 32
83 
84 enum qbman_sdqcr_dct {
85 	qbman_sdqcr_dct_null = 0,
86 	qbman_sdqcr_dct_prio_ics,
87 	qbman_sdqcr_dct_active_ics,
88 	qbman_sdqcr_dct_active
89 };
90 
91 enum qbman_sdqcr_fc {
92 	qbman_sdqcr_fc_one = 0,
93 	qbman_sdqcr_fc_up_to_3 = 1
94 };
95 
96 /* Internal Function declaration */
97 static int qbman_swp_enqueue_direct(struct qbman_swp *s,
98 				    const struct qbman_eq_desc *d,
99 				    const struct dpaa2_fd *fd);
100 static int qbman_swp_enqueue_mem_back(struct qbman_swp *s,
101 				      const struct qbman_eq_desc *d,
102 				      const struct dpaa2_fd *fd);
103 static int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
104 					     const struct qbman_eq_desc *d,
105 					     const struct dpaa2_fd *fd,
106 					     uint32_t *flags,
107 					     int num_frames);
108 static int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
109 					       const struct qbman_eq_desc *d,
110 					       const struct dpaa2_fd *fd,
111 					       uint32_t *flags,
112 					       int num_frames);
113 static int
114 qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
115 				       const struct qbman_eq_desc *d,
116 				       const struct dpaa2_fd *fd,
117 				       int num_frames);
118 static
119 int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
120 					     const struct qbman_eq_desc *d,
121 					     const struct dpaa2_fd *fd,
122 					     int num_frames);
123 static int qbman_swp_pull_direct(struct qbman_swp *s,
124 				 struct qbman_pull_desc *d);
125 static int qbman_swp_pull_mem_back(struct qbman_swp *s,
126 				   struct qbman_pull_desc *d);
127 
128 const struct dpaa2_dq *qbman_swp_dqrr_next_direct(struct qbman_swp *s);
129 const struct dpaa2_dq *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s);
130 
131 static int qbman_swp_release_direct(struct qbman_swp *s,
132 				    const struct qbman_release_desc *d,
133 				    const u64 *buffers,
134 				    unsigned int num_buffers);
135 static int qbman_swp_release_mem_back(struct qbman_swp *s,
136 				      const struct qbman_release_desc *d,
137 				      const u64 *buffers,
138 				      unsigned int num_buffers);
139 
140 /* Function pointers */
141 int (*qbman_swp_enqueue_ptr)(struct qbman_swp *s,
142 			     const struct qbman_eq_desc *d,
143 			     const struct dpaa2_fd *fd)
144 	= qbman_swp_enqueue_direct;
145 
146 int (*qbman_swp_enqueue_multiple_ptr)(struct qbman_swp *s,
147 				      const struct qbman_eq_desc *d,
148 				      const struct dpaa2_fd *fd,
149 				      uint32_t *flags,
150 					     int num_frames)
151 	= qbman_swp_enqueue_multiple_direct;
152 
153 int
154 (*qbman_swp_enqueue_multiple_desc_ptr)(struct qbman_swp *s,
155 				       const struct qbman_eq_desc *d,
156 				       const struct dpaa2_fd *fd,
157 				       int num_frames)
158 	= qbman_swp_enqueue_multiple_desc_direct;
159 
160 int (*qbman_swp_pull_ptr)(struct qbman_swp *s, struct qbman_pull_desc *d)
161 			= qbman_swp_pull_direct;
162 
163 const struct dpaa2_dq *(*qbman_swp_dqrr_next_ptr)(struct qbman_swp *s)
164 			= qbman_swp_dqrr_next_direct;
165 
166 int (*qbman_swp_release_ptr)(struct qbman_swp *s,
167 			     const struct qbman_release_desc *d,
168 			     const u64 *buffers,
169 			     unsigned int num_buffers)
170 			= qbman_swp_release_direct;
171 
172 /* Portal Access */
173 
174 static inline u32 qbman_read_register(struct qbman_swp *p, u32 offset)
175 {
176 	return readl_relaxed(p->addr_cinh + offset);
177 }
178 
179 static inline void qbman_write_register(struct qbman_swp *p, u32 offset,
180 					u32 value)
181 {
182 	writel_relaxed(value, p->addr_cinh + offset);
183 }
184 
185 static inline void *qbman_get_cmd(struct qbman_swp *p, u32 offset)
186 {
187 	return p->addr_cena + offset;
188 }
189 
190 #define QBMAN_CINH_SWP_CFG   0xd00
191 
192 #define SWP_CFG_DQRR_MF_SHIFT 20
193 #define SWP_CFG_EST_SHIFT     16
194 #define SWP_CFG_CPBS_SHIFT    15
195 #define SWP_CFG_WN_SHIFT      14
196 #define SWP_CFG_RPM_SHIFT     12
197 #define SWP_CFG_DCM_SHIFT     10
198 #define SWP_CFG_EPM_SHIFT     8
199 #define SWP_CFG_VPM_SHIFT     7
200 #define SWP_CFG_CPM_SHIFT     6
201 #define SWP_CFG_SD_SHIFT      5
202 #define SWP_CFG_SP_SHIFT      4
203 #define SWP_CFG_SE_SHIFT      3
204 #define SWP_CFG_DP_SHIFT      2
205 #define SWP_CFG_DE_SHIFT      1
206 #define SWP_CFG_EP_SHIFT      0
207 
208 static inline u32 qbman_set_swp_cfg(u8 max_fill, u8 wn,	u8 est, u8 rpm, u8 dcm,
209 				    u8 epm, int sd, int sp, int se,
210 				    int dp, int de, int ep)
211 {
212 	return (max_fill << SWP_CFG_DQRR_MF_SHIFT |
213 		est << SWP_CFG_EST_SHIFT |
214 		wn << SWP_CFG_WN_SHIFT |
215 		rpm << SWP_CFG_RPM_SHIFT |
216 		dcm << SWP_CFG_DCM_SHIFT |
217 		epm << SWP_CFG_EPM_SHIFT |
218 		sd << SWP_CFG_SD_SHIFT |
219 		sp << SWP_CFG_SP_SHIFT |
220 		se << SWP_CFG_SE_SHIFT |
221 		dp << SWP_CFG_DP_SHIFT |
222 		de << SWP_CFG_DE_SHIFT |
223 		ep << SWP_CFG_EP_SHIFT);
224 }
225 
226 #define QMAN_RT_MODE	   0x00000100
227 
228 static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last)
229 {
230 	/* 'first' is included, 'last' is excluded */
231 	if (first <= last)
232 		return last - first;
233 	else
234 		return (2 * ringsize) - (first - last);
235 }
236 
237 /**
238  * qbman_swp_init() - Create a functional object representing the given
239  *                    QBMan portal descriptor.
240  * @d: the given qbman swp descriptor
241  *
242  * Return qbman_swp portal for success, NULL if the object cannot
243  * be created.
244  */
245 struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d)
246 {
247 	struct qbman_swp *p = kzalloc(sizeof(*p), GFP_KERNEL);
248 	u32 reg;
249 	u32 mask_size;
250 	u32 eqcr_pi;
251 
252 	if (!p)
253 		return NULL;
254 
255 	spin_lock_init(&p->access_spinlock);
256 
257 	p->desc = d;
258 	p->mc.valid_bit = QB_VALID_BIT;
259 	p->sdq = 0;
260 	p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT;
261 	p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT;
262 	p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT;
263 	if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000)
264 		p->mr.valid_bit = QB_VALID_BIT;
265 
266 	atomic_set(&p->vdq.available, 1);
267 	p->vdq.valid_bit = QB_VALID_BIT;
268 	p->dqrr.next_idx = 0;
269 	p->dqrr.valid_bit = QB_VALID_BIT;
270 
271 	if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_4100) {
272 		p->dqrr.dqrr_size = 4;
273 		p->dqrr.reset_bug = 1;
274 	} else {
275 		p->dqrr.dqrr_size = 8;
276 		p->dqrr.reset_bug = 0;
277 	}
278 
279 	p->addr_cena = d->cena_bar;
280 	p->addr_cinh = d->cinh_bar;
281 
282 	if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
283 
284 		reg = qbman_set_swp_cfg(p->dqrr.dqrr_size,
285 			1, /* Writes Non-cacheable */
286 			0, /* EQCR_CI stashing threshold */
287 			3, /* RPM: RCR in array mode */
288 			2, /* DCM: Discrete consumption ack */
289 			2, /* EPM: EQCR in ring mode */
290 			1, /* mem stashing drop enable enable */
291 			1, /* mem stashing priority enable */
292 			1, /* mem stashing enable */
293 			1, /* dequeue stashing priority enable */
294 			0, /* dequeue stashing enable enable */
295 			0); /* EQCR_CI stashing priority enable */
296 	} else {
297 		memset(p->addr_cena, 0, 64 * 1024);
298 		reg = qbman_set_swp_cfg(p->dqrr.dqrr_size,
299 			1, /* Writes Non-cacheable */
300 			1, /* EQCR_CI stashing threshold */
301 			3, /* RPM: RCR in array mode */
302 			2, /* DCM: Discrete consumption ack */
303 			0, /* EPM: EQCR in ring mode */
304 			1, /* mem stashing drop enable */
305 			1, /* mem stashing priority enable */
306 			1, /* mem stashing enable */
307 			1, /* dequeue stashing priority enable */
308 			0, /* dequeue stashing enable */
309 			0); /* EQCR_CI stashing priority enable */
310 		reg |= 1 << SWP_CFG_CPBS_SHIFT | /* memory-backed mode */
311 		       1 << SWP_CFG_VPM_SHIFT |  /* VDQCR read triggered mode */
312 		       1 << SWP_CFG_CPM_SHIFT;   /* CR read triggered mode */
313 	}
314 
315 	qbman_write_register(p, QBMAN_CINH_SWP_CFG, reg);
316 	reg = qbman_read_register(p, QBMAN_CINH_SWP_CFG);
317 	if (!reg) {
318 		pr_err("qbman: the portal is not enabled!\n");
319 		kfree(p);
320 		return NULL;
321 	}
322 
323 	if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) {
324 		qbman_write_register(p, QBMAN_CINH_SWP_EQCR_PI, QMAN_RT_MODE);
325 		qbman_write_register(p, QBMAN_CINH_SWP_RCR_PI, QMAN_RT_MODE);
326 	}
327 	/*
328 	 * SDQCR needs to be initialized to 0 when no channels are
329 	 * being dequeued from or else the QMan HW will indicate an
330 	 * error.  The values that were calculated above will be
331 	 * applied when dequeues from a specific channel are enabled.
332 	 */
333 	qbman_write_register(p, QBMAN_CINH_SWP_SDQCR, 0);
334 
335 	p->eqcr.pi_ring_size = 8;
336 	if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) {
337 		p->eqcr.pi_ring_size = 32;
338 		qbman_swp_enqueue_ptr =
339 			qbman_swp_enqueue_mem_back;
340 		qbman_swp_enqueue_multiple_ptr =
341 			qbman_swp_enqueue_multiple_mem_back;
342 		qbman_swp_enqueue_multiple_desc_ptr =
343 			qbman_swp_enqueue_multiple_desc_mem_back;
344 		qbman_swp_pull_ptr = qbman_swp_pull_mem_back;
345 		qbman_swp_dqrr_next_ptr = qbman_swp_dqrr_next_mem_back;
346 		qbman_swp_release_ptr = qbman_swp_release_mem_back;
347 	}
348 
349 	for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1)
350 		p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask << 1) + 1;
351 	eqcr_pi = qbman_read_register(p, QBMAN_CINH_SWP_EQCR_PI);
352 	p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask;
353 	p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT;
354 	p->eqcr.ci = qbman_read_register(p, QBMAN_CINH_SWP_EQCR_CI)
355 			& p->eqcr.pi_ci_mask;
356 	p->eqcr.available = p->eqcr.pi_ring_size;
357 
358 	return p;
359 }
360 
361 /**
362  * qbman_swp_finish() - Create and destroy a functional object representing
363  *                      the given QBMan portal descriptor.
364  * @p: the qbman_swp object to be destroyed
365  */
366 void qbman_swp_finish(struct qbman_swp *p)
367 {
368 	kfree(p);
369 }
370 
371 /**
372  * qbman_swp_interrupt_read_status()
373  * @p: the given software portal
374  *
375  * Return the value in the SWP_ISR register.
376  */
377 u32 qbman_swp_interrupt_read_status(struct qbman_swp *p)
378 {
379 	return qbman_read_register(p, QBMAN_CINH_SWP_ISR);
380 }
381 
382 /**
383  * qbman_swp_interrupt_clear_status()
384  * @p: the given software portal
385  * @mask: The mask to clear in SWP_ISR register
386  */
387 void qbman_swp_interrupt_clear_status(struct qbman_swp *p, u32 mask)
388 {
389 	qbman_write_register(p, QBMAN_CINH_SWP_ISR, mask);
390 }
391 
392 /**
393  * qbman_swp_interrupt_get_trigger() - read interrupt enable register
394  * @p: the given software portal
395  *
396  * Return the value in the SWP_IER register.
397  */
398 u32 qbman_swp_interrupt_get_trigger(struct qbman_swp *p)
399 {
400 	return qbman_read_register(p, QBMAN_CINH_SWP_IER);
401 }
402 
403 /**
404  * qbman_swp_interrupt_set_trigger() - enable interrupts for a swp
405  * @p: the given software portal
406  * @mask: The mask of bits to enable in SWP_IER
407  */
408 void qbman_swp_interrupt_set_trigger(struct qbman_swp *p, u32 mask)
409 {
410 	qbman_write_register(p, QBMAN_CINH_SWP_IER, mask);
411 }
412 
413 /**
414  * qbman_swp_interrupt_get_inhibit() - read interrupt mask register
415  * @p: the given software portal object
416  *
417  * Return the value in the SWP_IIR register.
418  */
419 int qbman_swp_interrupt_get_inhibit(struct qbman_swp *p)
420 {
421 	return qbman_read_register(p, QBMAN_CINH_SWP_IIR);
422 }
423 
424 /**
425  * qbman_swp_interrupt_set_inhibit() - write interrupt mask register
426  * @p: the given software portal object
427  * @inhibit: whether to inhibit the IRQs
428  */
429 void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit)
430 {
431 	qbman_write_register(p, QBMAN_CINH_SWP_IIR, inhibit ? 0xffffffff : 0);
432 }
433 
434 /*
435  * Different management commands all use this common base layer of code to issue
436  * commands and poll for results.
437  */
438 
439 /*
440  * Returns a pointer to where the caller should fill in their management command
441  * (caller should ignore the verb byte)
442  */
443 void *qbman_swp_mc_start(struct qbman_swp *p)
444 {
445 	if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
446 		return qbman_get_cmd(p, QBMAN_CENA_SWP_CR);
447 	else
448 		return qbman_get_cmd(p, QBMAN_CENA_SWP_CR_MEM);
449 }
450 
451 /*
452  * Commits merges in the caller-supplied command verb (which should not include
453  * the valid-bit) and submits the command to hardware
454  */
455 void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, u8 cmd_verb)
456 {
457 	u8 *v = cmd;
458 
459 	if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
460 		dma_wmb();
461 		*v = cmd_verb | p->mc.valid_bit;
462 	} else {
463 		*v = cmd_verb | p->mc.valid_bit;
464 		dma_wmb();
465 		qbman_write_register(p, QBMAN_CINH_SWP_CR_RT, QMAN_RT_MODE);
466 	}
467 }
468 
469 /*
470  * Checks for a completed response (returns non-NULL if only if the response
471  * is complete).
472  */
473 void *qbman_swp_mc_result(struct qbman_swp *p)
474 {
475 	u32 *ret, verb;
476 
477 	if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
478 		ret = qbman_get_cmd(p, QBMAN_CENA_SWP_RR(p->mc.valid_bit));
479 		/* Remove the valid-bit - command completed if the rest
480 		 * is non-zero.
481 		 */
482 		verb = ret[0] & ~QB_VALID_BIT;
483 		if (!verb)
484 			return NULL;
485 		p->mc.valid_bit ^= QB_VALID_BIT;
486 	} else {
487 		ret = qbman_get_cmd(p, QBMAN_CENA_SWP_RR_MEM);
488 		/* Command completed if the valid bit is toggled */
489 		if (p->mr.valid_bit != (ret[0] & QB_VALID_BIT))
490 			return NULL;
491 		/* Command completed if the rest is non-zero */
492 		verb = ret[0] & ~QB_VALID_BIT;
493 		if (!verb)
494 			return NULL;
495 		p->mr.valid_bit ^= QB_VALID_BIT;
496 	}
497 
498 	return ret;
499 }
500 
501 #define QB_ENQUEUE_CMD_OPTIONS_SHIFT    0
502 enum qb_enqueue_commands {
503 	enqueue_empty = 0,
504 	enqueue_response_always = 1,
505 	enqueue_rejects_to_fq = 2
506 };
507 
508 #define QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT      2
509 #define QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT 3
510 #define QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT     4
511 #define QB_ENQUEUE_CMD_DCA_EN_SHIFT          7
512 
513 /*
514  * qbman_eq_desc_clear() - Clear the contents of a descriptor to
515  *                         default/starting state.
516  */
517 void qbman_eq_desc_clear(struct qbman_eq_desc *d)
518 {
519 	memset(d, 0, sizeof(*d));
520 }
521 
522 /**
523  * qbman_eq_desc_set_no_orp() - Set enqueue descriptor without orp
524  * @d:                the enqueue descriptor.
525  * @respond_success:  1 = enqueue with response always; 0 = enqueue with
526  *                    rejections returned on a FQ.
527  */
528 void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success)
529 {
530 	d->verb &= ~(1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT);
531 	if (respond_success)
532 		d->verb |= enqueue_response_always;
533 	else
534 		d->verb |= enqueue_rejects_to_fq;
535 }
536 
537 /*
538  * Exactly one of the following descriptor "targets" should be set. (Calling any
539  * one of these will replace the effect of any prior call to one of these.)
540  *   -enqueue to a frame queue
541  *   -enqueue to a queuing destination
542  */
543 
544 /**
545  * qbman_eq_desc_set_fq() - set the FQ for the enqueue command
546  * @d:    the enqueue descriptor
547  * @fqid: the id of the frame queue to be enqueued
548  */
549 void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, u32 fqid)
550 {
551 	d->verb &= ~(1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT);
552 	d->tgtid = cpu_to_le32(fqid);
553 }
554 
555 /**
556  * qbman_eq_desc_set_qd() - Set Queuing Destination for the enqueue command
557  * @d:       the enqueue descriptor
558  * @qdid:    the id of the queuing destination to be enqueued
559  * @qd_bin:  the queuing destination bin
560  * @qd_prio: the queuing destination priority
561  */
562 void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, u32 qdid,
563 			  u32 qd_bin, u32 qd_prio)
564 {
565 	d->verb |= 1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT;
566 	d->tgtid = cpu_to_le32(qdid);
567 	d->qdbin = cpu_to_le16(qd_bin);
568 	d->qpri = qd_prio;
569 }
570 
571 #define EQAR_IDX(eqar)     ((eqar) & 0x7)
572 #define EQAR_VB(eqar)      ((eqar) & 0x80)
573 #define EQAR_SUCCESS(eqar) ((eqar) & 0x100)
574 
575 #define QB_RT_BIT ((u32)0x100)
576 /**
577  * qbman_swp_enqueue_direct() - Issue an enqueue command
578  * @s:  the software portal used for enqueue
579  * @d:  the enqueue descriptor
580  * @fd: the frame descriptor to be enqueued
581  *
582  * Please note that 'fd' should only be NULL if the "action" of the
583  * descriptor is "orp_hole" or "orp_nesn".
584  *
585  * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready.
586  */
587 static
588 int qbman_swp_enqueue_direct(struct qbman_swp *s,
589 			     const struct qbman_eq_desc *d,
590 			     const struct dpaa2_fd *fd)
591 {
592 	int flags = 0;
593 	int ret = qbman_swp_enqueue_multiple_direct(s, d, fd, &flags, 1);
594 
595 	if (ret >= 0)
596 		ret = 0;
597 	else
598 		ret = -EBUSY;
599 	return  ret;
600 }
601 
602 /**
603  * qbman_swp_enqueue_mem_back() - Issue an enqueue command
604  * @s:  the software portal used for enqueue
605  * @d:  the enqueue descriptor
606  * @fd: the frame descriptor to be enqueued
607  *
608  * Please note that 'fd' should only be NULL if the "action" of the
609  * descriptor is "orp_hole" or "orp_nesn".
610  *
611  * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready.
612  */
613 static
614 int qbman_swp_enqueue_mem_back(struct qbman_swp *s,
615 			       const struct qbman_eq_desc *d,
616 			       const struct dpaa2_fd *fd)
617 {
618 	int flags = 0;
619 	int ret = qbman_swp_enqueue_multiple_mem_back(s, d, fd, &flags, 1);
620 
621 	if (ret >= 0)
622 		ret = 0;
623 	else
624 		ret = -EBUSY;
625 	return  ret;
626 }
627 
628 /**
629  * qbman_swp_enqueue_multiple_direct() - Issue a multi enqueue command
630  * using one enqueue descriptor
631  * @s:  the software portal used for enqueue
632  * @d:  the enqueue descriptor
633  * @fd: table pointer of frame descriptor table to be enqueued
634  * @flags: table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL
635  * @num_frames: number of fd to be enqueued
636  *
637  * Return the number of fd enqueued, or a negative error number.
638  */
639 static
640 int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
641 				      const struct qbman_eq_desc *d,
642 				      const struct dpaa2_fd *fd,
643 				      uint32_t *flags,
644 				      int num_frames)
645 {
646 	uint32_t *p = NULL;
647 	const uint32_t *cl = (uint32_t *)d;
648 	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
649 	int i, num_enqueued = 0;
650 
651 	spin_lock(&s->access_spinlock);
652 	half_mask = (s->eqcr.pi_ci_mask>>1);
653 	full_mask = s->eqcr.pi_ci_mask;
654 
655 	if (!s->eqcr.available) {
656 		eqcr_ci = s->eqcr.ci;
657 		p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI;
658 		s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI);
659 		s->eqcr.ci &= full_mask;
660 
661 		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
662 					eqcr_ci, s->eqcr.ci);
663 		if (!s->eqcr.available) {
664 			spin_unlock(&s->access_spinlock);
665 			return 0;
666 		}
667 	}
668 
669 	eqcr_pi = s->eqcr.pi;
670 	num_enqueued = (s->eqcr.available < num_frames) ?
671 			s->eqcr.available : num_frames;
672 	s->eqcr.available -= num_enqueued;
673 	/* Fill in the EQCR ring */
674 	for (i = 0; i < num_enqueued; i++) {
675 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
676 		/* Skip copying the verb */
677 		memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
678 		memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
679 		       &fd[i], sizeof(*fd));
680 		eqcr_pi++;
681 	}
682 
683 	dma_wmb();
684 
685 	/* Set the verb byte, have to substitute in the valid-bit */
686 	eqcr_pi = s->eqcr.pi;
687 	for (i = 0; i < num_enqueued; i++) {
688 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
689 		p[0] = cl[0] | s->eqcr.pi_vb;
690 		if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
691 			struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
692 
693 			d->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
694 				((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
695 		}
696 		eqcr_pi++;
697 		if (!(eqcr_pi & half_mask))
698 			s->eqcr.pi_vb ^= QB_VALID_BIT;
699 	}
700 
701 	/* Flush all the cacheline without load/store in between */
702 	eqcr_pi = s->eqcr.pi;
703 	for (i = 0; i < num_enqueued; i++)
704 		eqcr_pi++;
705 	s->eqcr.pi = eqcr_pi & full_mask;
706 	spin_unlock(&s->access_spinlock);
707 
708 	return num_enqueued;
709 }
710 
711 /**
712  * qbman_swp_enqueue_multiple_mem_back() - Issue a multi enqueue command
713  * using one enqueue descriptor
714  * @s:  the software portal used for enqueue
715  * @d:  the enqueue descriptor
716  * @fd: table pointer of frame descriptor table to be enqueued
717  * @flags: table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL
718  * @num_frames: number of fd to be enqueued
719  *
720  * Return the number of fd enqueued, or a negative error number.
721  */
722 static
723 int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
724 					const struct qbman_eq_desc *d,
725 					const struct dpaa2_fd *fd,
726 					uint32_t *flags,
727 					int num_frames)
728 {
729 	uint32_t *p = NULL;
730 	const uint32_t *cl = (uint32_t *)(d);
731 	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
732 	int i, num_enqueued = 0;
733 	unsigned long irq_flags;
734 
735 	spin_lock(&s->access_spinlock);
736 	local_irq_save(irq_flags);
737 
738 	half_mask = (s->eqcr.pi_ci_mask>>1);
739 	full_mask = s->eqcr.pi_ci_mask;
740 	if (!s->eqcr.available) {
741 		eqcr_ci = s->eqcr.ci;
742 		p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI_MEMBACK;
743 		s->eqcr.ci = *p & full_mask;
744 		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
745 					eqcr_ci, s->eqcr.ci);
746 		if (!s->eqcr.available) {
747 			local_irq_restore(irq_flags);
748 			spin_unlock(&s->access_spinlock);
749 			return 0;
750 		}
751 	}
752 
753 	eqcr_pi = s->eqcr.pi;
754 	num_enqueued = (s->eqcr.available < num_frames) ?
755 			s->eqcr.available : num_frames;
756 	s->eqcr.available -= num_enqueued;
757 	/* Fill in the EQCR ring */
758 	for (i = 0; i < num_enqueued; i++) {
759 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
760 		/* Skip copying the verb */
761 		memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
762 		memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
763 		       &fd[i], sizeof(*fd));
764 		eqcr_pi++;
765 	}
766 
767 	/* Set the verb byte, have to substitute in the valid-bit */
768 	eqcr_pi = s->eqcr.pi;
769 	for (i = 0; i < num_enqueued; i++) {
770 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
771 		p[0] = cl[0] | s->eqcr.pi_vb;
772 		if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
773 			struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
774 
775 			d->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
776 				((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
777 		}
778 		eqcr_pi++;
779 		if (!(eqcr_pi & half_mask))
780 			s->eqcr.pi_vb ^= QB_VALID_BIT;
781 	}
782 	s->eqcr.pi = eqcr_pi & full_mask;
783 
784 	dma_wmb();
785 	qbman_write_register(s, QBMAN_CINH_SWP_EQCR_PI,
786 				(QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
787 	local_irq_restore(irq_flags);
788 	spin_unlock(&s->access_spinlock);
789 
790 	return num_enqueued;
791 }
792 
793 /**
794  * qbman_swp_enqueue_multiple_desc_direct() - Issue a multi enqueue command
795  * using multiple enqueue descriptor
796  * @s:  the software portal used for enqueue
797  * @d:  table of minimal enqueue descriptor
798  * @fd: table pointer of frame descriptor table to be enqueued
799  * @num_frames: number of fd to be enqueued
800  *
801  * Return the number of fd enqueued, or a negative error number.
802  */
803 static
804 int qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
805 					   const struct qbman_eq_desc *d,
806 					   const struct dpaa2_fd *fd,
807 					   int num_frames)
808 {
809 	uint32_t *p;
810 	const uint32_t *cl;
811 	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
812 	int i, num_enqueued = 0;
813 
814 	half_mask = (s->eqcr.pi_ci_mask>>1);
815 	full_mask = s->eqcr.pi_ci_mask;
816 	if (!s->eqcr.available) {
817 		eqcr_ci = s->eqcr.ci;
818 		p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI;
819 		s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI);
820 		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
821 					eqcr_ci, s->eqcr.ci);
822 		if (!s->eqcr.available)
823 			return 0;
824 	}
825 
826 	eqcr_pi = s->eqcr.pi;
827 	num_enqueued = (s->eqcr.available < num_frames) ?
828 			s->eqcr.available : num_frames;
829 	s->eqcr.available -= num_enqueued;
830 	/* Fill in the EQCR ring */
831 	for (i = 0; i < num_enqueued; i++) {
832 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
833 		cl = (uint32_t *)(&d[i]);
834 		/* Skip copying the verb */
835 		memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
836 		memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
837 		       &fd[i], sizeof(*fd));
838 		eqcr_pi++;
839 	}
840 
841 	dma_wmb();
842 
843 	/* Set the verb byte, have to substitute in the valid-bit */
844 	eqcr_pi = s->eqcr.pi;
845 	for (i = 0; i < num_enqueued; i++) {
846 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
847 		cl = (uint32_t *)(&d[i]);
848 		p[0] = cl[0] | s->eqcr.pi_vb;
849 		eqcr_pi++;
850 		if (!(eqcr_pi & half_mask))
851 			s->eqcr.pi_vb ^= QB_VALID_BIT;
852 	}
853 
854 	/* Flush all the cacheline without load/store in between */
855 	eqcr_pi = s->eqcr.pi;
856 	for (i = 0; i < num_enqueued; i++)
857 		eqcr_pi++;
858 	s->eqcr.pi = eqcr_pi & full_mask;
859 
860 	return num_enqueued;
861 }
862 
863 /**
864  * qbman_swp_enqueue_multiple_desc_mem_back() - Issue a multi enqueue command
865  * using multiple enqueue descriptor
866  * @s:  the software portal used for enqueue
867  * @d:  table of minimal enqueue descriptor
868  * @fd: table pointer of frame descriptor table to be enqueued
869  * @num_frames: number of fd to be enqueued
870  *
871  * Return the number of fd enqueued, or a negative error number.
872  */
873 static
874 int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
875 					   const struct qbman_eq_desc *d,
876 					   const struct dpaa2_fd *fd,
877 					   int num_frames)
878 {
879 	uint32_t *p;
880 	const uint32_t *cl;
881 	uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
882 	int i, num_enqueued = 0;
883 
884 	half_mask = (s->eqcr.pi_ci_mask>>1);
885 	full_mask = s->eqcr.pi_ci_mask;
886 	if (!s->eqcr.available) {
887 		eqcr_ci = s->eqcr.ci;
888 		p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI_MEMBACK;
889 		s->eqcr.ci = *p & full_mask;
890 		s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
891 					eqcr_ci, s->eqcr.ci);
892 		if (!s->eqcr.available)
893 			return 0;
894 	}
895 
896 	eqcr_pi = s->eqcr.pi;
897 	num_enqueued = (s->eqcr.available < num_frames) ?
898 			s->eqcr.available : num_frames;
899 	s->eqcr.available -= num_enqueued;
900 	/* Fill in the EQCR ring */
901 	for (i = 0; i < num_enqueued; i++) {
902 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
903 		cl = (uint32_t *)(&d[i]);
904 		/* Skip copying the verb */
905 		memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
906 		memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
907 		       &fd[i], sizeof(*fd));
908 		eqcr_pi++;
909 	}
910 
911 	/* Set the verb byte, have to substitute in the valid-bit */
912 	eqcr_pi = s->eqcr.pi;
913 	for (i = 0; i < num_enqueued; i++) {
914 		p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
915 		cl = (uint32_t *)(&d[i]);
916 		p[0] = cl[0] | s->eqcr.pi_vb;
917 		eqcr_pi++;
918 		if (!(eqcr_pi & half_mask))
919 			s->eqcr.pi_vb ^= QB_VALID_BIT;
920 	}
921 
922 	s->eqcr.pi = eqcr_pi & full_mask;
923 
924 	dma_wmb();
925 	qbman_write_register(s, QBMAN_CINH_SWP_EQCR_PI,
926 				(QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
927 
928 	return num_enqueued;
929 }
930 
931 /* Static (push) dequeue */
932 
933 /**
934  * qbman_swp_push_get() - Get the push dequeue setup
935  * @s:           the software portal object
936  * @channel_idx: the channel index to query
937  * @enabled:     returned boolean to show whether the push dequeue is enabled
938  *               for the given channel
939  */
940 void qbman_swp_push_get(struct qbman_swp *s, u8 channel_idx, int *enabled)
941 {
942 	u16 src = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
943 
944 	WARN_ON(channel_idx > 15);
945 	*enabled = src | (1 << channel_idx);
946 }
947 
948 /**
949  * qbman_swp_push_set() - Enable or disable push dequeue
950  * @s:           the software portal object
951  * @channel_idx: the channel index (0 to 15)
952  * @enable:      enable or disable push dequeue
953  */
954 void qbman_swp_push_set(struct qbman_swp *s, u8 channel_idx, int enable)
955 {
956 	u16 dqsrc;
957 
958 	WARN_ON(channel_idx > 15);
959 	if (enable)
960 		s->sdq |= 1 << channel_idx;
961 	else
962 		s->sdq &= ~(1 << channel_idx);
963 
964 	/* Read make the complete src map.  If no channels are enabled
965 	 * the SDQCR must be 0 or else QMan will assert errors
966 	 */
967 	dqsrc = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
968 	if (dqsrc != 0)
969 		qbman_write_register(s, QBMAN_CINH_SWP_SDQCR, s->sdq);
970 	else
971 		qbman_write_register(s, QBMAN_CINH_SWP_SDQCR, 0);
972 }
973 
974 #define QB_VDQCR_VERB_DCT_SHIFT    0
975 #define QB_VDQCR_VERB_DT_SHIFT     2
976 #define QB_VDQCR_VERB_RLS_SHIFT    4
977 #define QB_VDQCR_VERB_WAE_SHIFT    5
978 
979 enum qb_pull_dt_e {
980 	qb_pull_dt_channel,
981 	qb_pull_dt_workqueue,
982 	qb_pull_dt_framequeue
983 };
984 
985 /**
986  * qbman_pull_desc_clear() - Clear the contents of a descriptor to
987  *                           default/starting state
988  * @d: the pull dequeue descriptor to be cleared
989  */
990 void qbman_pull_desc_clear(struct qbman_pull_desc *d)
991 {
992 	memset(d, 0, sizeof(*d));
993 }
994 
995 /**
996  * qbman_pull_desc_set_storage()- Set the pull dequeue storage
997  * @d:            the pull dequeue descriptor to be set
998  * @storage:      the pointer of the memory to store the dequeue result
999  * @storage_phys: the physical address of the storage memory
1000  * @stash:        to indicate whether write allocate is enabled
1001  *
1002  * If not called, or if called with 'storage' as NULL, the result pull dequeues
1003  * will produce results to DQRR. If 'storage' is non-NULL, then results are
1004  * produced to the given memory location (using the DMA address which
1005  * the caller provides in 'storage_phys'), and 'stash' controls whether or not
1006  * those writes to main-memory express a cache-warming attribute.
1007  */
1008 void qbman_pull_desc_set_storage(struct qbman_pull_desc *d,
1009 				 struct dpaa2_dq *storage,
1010 				 dma_addr_t storage_phys,
1011 				 int stash)
1012 {
1013 	/* save the virtual address */
1014 	d->rsp_addr_virt = (u64)(uintptr_t)storage;
1015 
1016 	if (!storage) {
1017 		d->verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT);
1018 		return;
1019 	}
1020 	d->verb |= 1 << QB_VDQCR_VERB_RLS_SHIFT;
1021 	if (stash)
1022 		d->verb |= 1 << QB_VDQCR_VERB_WAE_SHIFT;
1023 	else
1024 		d->verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT);
1025 
1026 	d->rsp_addr = cpu_to_le64(storage_phys);
1027 }
1028 
1029 /**
1030  * qbman_pull_desc_set_numframes() - Set the number of frames to be dequeued
1031  * @d:         the pull dequeue descriptor to be set
1032  * @numframes: number of frames to be set, must be between 1 and 16, inclusive
1033  */
1034 void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, u8 numframes)
1035 {
1036 	d->numf = numframes - 1;
1037 }
1038 
1039 /*
1040  * Exactly one of the following descriptor "actions" should be set. (Calling any
1041  * one of these will replace the effect of any prior call to one of these.)
1042  * - pull dequeue from the given frame queue (FQ)
1043  * - pull dequeue from any FQ in the given work queue (WQ)
1044  * - pull dequeue from any FQ in any WQ in the given channel
1045  */
1046 
1047 /**
1048  * qbman_pull_desc_set_fq() - Set fqid from which the dequeue command dequeues
1049  * @d:    the pull dequeue descriptor to be set
1050  * @fqid: the frame queue index of the given FQ
1051  */
1052 void qbman_pull_desc_set_fq(struct qbman_pull_desc *d, u32 fqid)
1053 {
1054 	d->verb |= 1 << QB_VDQCR_VERB_DCT_SHIFT;
1055 	d->verb |= qb_pull_dt_framequeue << QB_VDQCR_VERB_DT_SHIFT;
1056 	d->dq_src = cpu_to_le32(fqid);
1057 }
1058 
1059 /**
1060  * qbman_pull_desc_set_wq() - Set wqid from which the dequeue command dequeues
1061  * @d:    the pull dequeue descriptor to be set
1062  * @wqid: composed of channel id and wqid within the channel
1063  * @dct:  the dequeue command type
1064  */
1065 void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, u32 wqid,
1066 			    enum qbman_pull_type_e dct)
1067 {
1068 	d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
1069 	d->verb |= qb_pull_dt_workqueue << QB_VDQCR_VERB_DT_SHIFT;
1070 	d->dq_src = cpu_to_le32(wqid);
1071 }
1072 
1073 /**
1074  * qbman_pull_desc_set_channel() - Set channelid from which the dequeue command
1075  *                                 dequeues
1076  * @d:    the pull dequeue descriptor to be set
1077  * @chid: the channel id to be dequeued
1078  * @dct:  the dequeue command type
1079  */
1080 void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, u32 chid,
1081 				 enum qbman_pull_type_e dct)
1082 {
1083 	d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
1084 	d->verb |= qb_pull_dt_channel << QB_VDQCR_VERB_DT_SHIFT;
1085 	d->dq_src = cpu_to_le32(chid);
1086 }
1087 
1088 /**
1089  * qbman_swp_pull_direct() - Issue the pull dequeue command
1090  * @s: the software portal object
1091  * @d: the software portal descriptor which has been configured with
1092  *     the set of qbman_pull_desc_set_*() calls
1093  *
1094  * Return 0 for success, and -EBUSY if the software portal is not ready
1095  * to do pull dequeue.
1096  */
1097 static
1098 int qbman_swp_pull_direct(struct qbman_swp *s, struct qbman_pull_desc *d)
1099 {
1100 	struct qbman_pull_desc *p;
1101 
1102 	if (!atomic_dec_and_test(&s->vdq.available)) {
1103 		atomic_inc(&s->vdq.available);
1104 		return -EBUSY;
1105 	}
1106 	s->vdq.storage = (void *)(uintptr_t)d->rsp_addr_virt;
1107 	if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
1108 		p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR);
1109 	else
1110 		p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR_MEM);
1111 	p->numf = d->numf;
1112 	p->tok = QMAN_DQ_TOKEN_VALID;
1113 	p->dq_src = d->dq_src;
1114 	p->rsp_addr = d->rsp_addr;
1115 	p->rsp_addr_virt = d->rsp_addr_virt;
1116 	dma_wmb();
1117 	/* Set the verb byte, have to substitute in the valid-bit */
1118 	p->verb = d->verb | s->vdq.valid_bit;
1119 	s->vdq.valid_bit ^= QB_VALID_BIT;
1120 
1121 	return 0;
1122 }
1123 
1124 /**
1125  * qbman_swp_pull_mem_back() - Issue the pull dequeue command
1126  * @s: the software portal object
1127  * @d: the software portal descriptor which has been configured with
1128  *     the set of qbman_pull_desc_set_*() calls
1129  *
1130  * Return 0 for success, and -EBUSY if the software portal is not ready
1131  * to do pull dequeue.
1132  */
1133 static
1134 int qbman_swp_pull_mem_back(struct qbman_swp *s, struct qbman_pull_desc *d)
1135 {
1136 	struct qbman_pull_desc *p;
1137 
1138 	if (!atomic_dec_and_test(&s->vdq.available)) {
1139 		atomic_inc(&s->vdq.available);
1140 		return -EBUSY;
1141 	}
1142 	s->vdq.storage = (void *)(uintptr_t)d->rsp_addr_virt;
1143 	if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
1144 		p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR);
1145 	else
1146 		p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR_MEM);
1147 	p->numf = d->numf;
1148 	p->tok = QMAN_DQ_TOKEN_VALID;
1149 	p->dq_src = d->dq_src;
1150 	p->rsp_addr = d->rsp_addr;
1151 	p->rsp_addr_virt = d->rsp_addr_virt;
1152 
1153 	/* Set the verb byte, have to substitute in the valid-bit */
1154 	p->verb = d->verb | s->vdq.valid_bit;
1155 	s->vdq.valid_bit ^= QB_VALID_BIT;
1156 	dma_wmb();
1157 	qbman_write_register(s, QBMAN_CINH_SWP_VDQCR_RT, QMAN_RT_MODE);
1158 
1159 	return 0;
1160 }
1161 
1162 #define QMAN_DQRR_PI_MASK   0xf
1163 
1164 /**
1165  * qbman_swp_dqrr_next_direct() - Get an valid DQRR entry
1166  * @s: the software portal object
1167  *
1168  * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry
1169  * only once, so repeated calls can return a sequence of DQRR entries, without
1170  * requiring they be consumed immediately or in any particular order.
1171  */
1172 const struct dpaa2_dq *qbman_swp_dqrr_next_direct(struct qbman_swp *s)
1173 {
1174 	u32 verb;
1175 	u32 response_verb;
1176 	u32 flags;
1177 	struct dpaa2_dq *p;
1178 
1179 	/* Before using valid-bit to detect if something is there, we have to
1180 	 * handle the case of the DQRR reset bug...
1181 	 */
1182 	if (unlikely(s->dqrr.reset_bug)) {
1183 		/*
1184 		 * We pick up new entries by cache-inhibited producer index,
1185 		 * which means that a non-coherent mapping would require us to
1186 		 * invalidate and read *only* once that PI has indicated that
1187 		 * there's an entry here. The first trip around the DQRR ring
1188 		 * will be much less efficient than all subsequent trips around
1189 		 * it...
1190 		 */
1191 		u8 pi = qbman_read_register(s, QBMAN_CINH_SWP_DQPI) &
1192 			QMAN_DQRR_PI_MASK;
1193 
1194 		/* there are new entries if pi != next_idx */
1195 		if (pi == s->dqrr.next_idx)
1196 			return NULL;
1197 
1198 		/*
1199 		 * if next_idx is/was the last ring index, and 'pi' is
1200 		 * different, we can disable the workaround as all the ring
1201 		 * entries have now been DMA'd to so valid-bit checking is
1202 		 * repaired. Note: this logic needs to be based on next_idx
1203 		 * (which increments one at a time), rather than on pi (which
1204 		 * can burst and wrap-around between our snapshots of it).
1205 		 */
1206 		if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) {
1207 			pr_debug("next_idx=%d, pi=%d, clear reset bug\n",
1208 				 s->dqrr.next_idx, pi);
1209 			s->dqrr.reset_bug = 0;
1210 		}
1211 		prefetch(qbman_get_cmd(s,
1212 				       QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1213 	}
1214 
1215 	p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
1216 	verb = p->dq.verb;
1217 
1218 	/*
1219 	 * If the valid-bit isn't of the expected polarity, nothing there. Note,
1220 	 * in the DQRR reset bug workaround, we shouldn't need to skip these
1221 	 * check, because we've already determined that a new entry is available
1222 	 * and we've invalidated the cacheline before reading it, so the
1223 	 * valid-bit behaviour is repaired and should tell us what we already
1224 	 * knew from reading PI.
1225 	 */
1226 	if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) {
1227 		prefetch(qbman_get_cmd(s,
1228 				       QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1229 		return NULL;
1230 	}
1231 	/*
1232 	 * There's something there. Move "next_idx" attention to the next ring
1233 	 * entry (and prefetch it) before returning what we found.
1234 	 */
1235 	s->dqrr.next_idx++;
1236 	s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */
1237 	if (!s->dqrr.next_idx)
1238 		s->dqrr.valid_bit ^= QB_VALID_BIT;
1239 
1240 	/*
1241 	 * If this is the final response to a volatile dequeue command
1242 	 * indicate that the vdq is available
1243 	 */
1244 	flags = p->dq.stat;
1245 	response_verb = verb & QBMAN_RESULT_MASK;
1246 	if ((response_verb == QBMAN_RESULT_DQ) &&
1247 	    (flags & DPAA2_DQ_STAT_VOLATILE) &&
1248 	    (flags & DPAA2_DQ_STAT_EXPIRED))
1249 		atomic_inc(&s->vdq.available);
1250 
1251 	prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1252 
1253 	return p;
1254 }
1255 
1256 /**
1257  * qbman_swp_dqrr_next_mem_back() - Get an valid DQRR entry
1258  * @s: the software portal object
1259  *
1260  * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry
1261  * only once, so repeated calls can return a sequence of DQRR entries, without
1262  * requiring they be consumed immediately or in any particular order.
1263  */
1264 const struct dpaa2_dq *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s)
1265 {
1266 	u32 verb;
1267 	u32 response_verb;
1268 	u32 flags;
1269 	struct dpaa2_dq *p;
1270 
1271 	/* Before using valid-bit to detect if something is there, we have to
1272 	 * handle the case of the DQRR reset bug...
1273 	 */
1274 	if (unlikely(s->dqrr.reset_bug)) {
1275 		/*
1276 		 * We pick up new entries by cache-inhibited producer index,
1277 		 * which means that a non-coherent mapping would require us to
1278 		 * invalidate and read *only* once that PI has indicated that
1279 		 * there's an entry here. The first trip around the DQRR ring
1280 		 * will be much less efficient than all subsequent trips around
1281 		 * it...
1282 		 */
1283 		u8 pi = qbman_read_register(s, QBMAN_CINH_SWP_DQPI) &
1284 			QMAN_DQRR_PI_MASK;
1285 
1286 		/* there are new entries if pi != next_idx */
1287 		if (pi == s->dqrr.next_idx)
1288 			return NULL;
1289 
1290 		/*
1291 		 * if next_idx is/was the last ring index, and 'pi' is
1292 		 * different, we can disable the workaround as all the ring
1293 		 * entries have now been DMA'd to so valid-bit checking is
1294 		 * repaired. Note: this logic needs to be based on next_idx
1295 		 * (which increments one at a time), rather than on pi (which
1296 		 * can burst and wrap-around between our snapshots of it).
1297 		 */
1298 		if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) {
1299 			pr_debug("next_idx=%d, pi=%d, clear reset bug\n",
1300 				 s->dqrr.next_idx, pi);
1301 			s->dqrr.reset_bug = 0;
1302 		}
1303 		prefetch(qbman_get_cmd(s,
1304 				       QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1305 	}
1306 
1307 	p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR_MEM(s->dqrr.next_idx));
1308 	verb = p->dq.verb;
1309 
1310 	/*
1311 	 * If the valid-bit isn't of the expected polarity, nothing there. Note,
1312 	 * in the DQRR reset bug workaround, we shouldn't need to skip these
1313 	 * check, because we've already determined that a new entry is available
1314 	 * and we've invalidated the cacheline before reading it, so the
1315 	 * valid-bit behaviour is repaired and should tell us what we already
1316 	 * knew from reading PI.
1317 	 */
1318 	if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) {
1319 		prefetch(qbman_get_cmd(s,
1320 				       QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1321 		return NULL;
1322 	}
1323 	/*
1324 	 * There's something there. Move "next_idx" attention to the next ring
1325 	 * entry (and prefetch it) before returning what we found.
1326 	 */
1327 	s->dqrr.next_idx++;
1328 	s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */
1329 	if (!s->dqrr.next_idx)
1330 		s->dqrr.valid_bit ^= QB_VALID_BIT;
1331 
1332 	/*
1333 	 * If this is the final response to a volatile dequeue command
1334 	 * indicate that the vdq is available
1335 	 */
1336 	flags = p->dq.stat;
1337 	response_verb = verb & QBMAN_RESULT_MASK;
1338 	if ((response_verb == QBMAN_RESULT_DQ) &&
1339 	    (flags & DPAA2_DQ_STAT_VOLATILE) &&
1340 	    (flags & DPAA2_DQ_STAT_EXPIRED))
1341 		atomic_inc(&s->vdq.available);
1342 
1343 	prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
1344 
1345 	return p;
1346 }
1347 
1348 /**
1349  * qbman_swp_dqrr_consume() -  Consume DQRR entries previously returned from
1350  *                             qbman_swp_dqrr_next().
1351  * @s: the software portal object
1352  * @dq: the DQRR entry to be consumed
1353  */
1354 void qbman_swp_dqrr_consume(struct qbman_swp *s, const struct dpaa2_dq *dq)
1355 {
1356 	qbman_write_register(s, QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq));
1357 }
1358 
1359 /**
1360  * qbman_result_has_new_result() - Check and get the dequeue response from the
1361  *                                 dq storage memory set in pull dequeue command
1362  * @s: the software portal object
1363  * @dq: the dequeue result read from the memory
1364  *
1365  * Return 1 for getting a valid dequeue result, or 0 for not getting a valid
1366  * dequeue result.
1367  *
1368  * Only used for user-provided storage of dequeue results, not DQRR. For
1369  * efficiency purposes, the driver will perform any required endianness
1370  * conversion to ensure that the user's dequeue result storage is in host-endian
1371  * format. As such, once the user has called qbman_result_has_new_result() and
1372  * been returned a valid dequeue result, they should not call it again on
1373  * the same memory location (except of course if another dequeue command has
1374  * been executed to produce a new result to that location).
1375  */
1376 int qbman_result_has_new_result(struct qbman_swp *s, const struct dpaa2_dq *dq)
1377 {
1378 	if (dq->dq.tok != QMAN_DQ_TOKEN_VALID)
1379 		return 0;
1380 
1381 	/*
1382 	 * Set token to be 0 so we will detect change back to 1
1383 	 * next time the looping is traversed. Const is cast away here
1384 	 * as we want users to treat the dequeue responses as read only.
1385 	 */
1386 	((struct dpaa2_dq *)dq)->dq.tok = 0;
1387 
1388 	/*
1389 	 * Determine whether VDQCR is available based on whether the
1390 	 * current result is sitting in the first storage location of
1391 	 * the busy command.
1392 	 */
1393 	if (s->vdq.storage == dq) {
1394 		s->vdq.storage = NULL;
1395 		atomic_inc(&s->vdq.available);
1396 	}
1397 
1398 	return 1;
1399 }
1400 
1401 /**
1402  * qbman_release_desc_clear() - Clear the contents of a descriptor to
1403  *                              default/starting state.
1404  * @d: the pull dequeue descriptor to be cleared
1405  */
1406 void qbman_release_desc_clear(struct qbman_release_desc *d)
1407 {
1408 	memset(d, 0, sizeof(*d));
1409 	d->verb = 1 << 5; /* Release Command Valid */
1410 }
1411 
1412 /**
1413  * qbman_release_desc_set_bpid() - Set the ID of the buffer pool to release to
1414  * @d:    the pull dequeue descriptor to be set
1415  * @bpid: the bpid value to be set
1416  */
1417 void qbman_release_desc_set_bpid(struct qbman_release_desc *d, u16 bpid)
1418 {
1419 	d->bpid = cpu_to_le16(bpid);
1420 }
1421 
1422 /**
1423  * qbman_release_desc_set_rcdi() - Determines whether or not the portal's RCDI
1424  * interrupt source should be asserted after the release command is completed.
1425  * @d:      the pull dequeue descriptor to be set
1426  * @enable: enable (1) or disable (0) value
1427  */
1428 void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable)
1429 {
1430 	if (enable)
1431 		d->verb |= 1 << 6;
1432 	else
1433 		d->verb &= ~(1 << 6);
1434 }
1435 
1436 #define RAR_IDX(rar)     ((rar) & 0x7)
1437 #define RAR_VB(rar)      ((rar) & 0x80)
1438 #define RAR_SUCCESS(rar) ((rar) & 0x100)
1439 
1440 /**
1441  * qbman_swp_release_direct() - Issue a buffer release command
1442  * @s:           the software portal object
1443  * @d:           the release descriptor
1444  * @buffers:     a pointer pointing to the buffer address to be released
1445  * @num_buffers: number of buffers to be released,  must be less than 8
1446  *
1447  * Return 0 for success, -EBUSY if the release command ring is not ready.
1448  */
1449 int qbman_swp_release_direct(struct qbman_swp *s,
1450 			     const struct qbman_release_desc *d,
1451 			     const u64 *buffers, unsigned int num_buffers)
1452 {
1453 	int i;
1454 	struct qbman_release_desc *p;
1455 	u32 rar;
1456 
1457 	if (!num_buffers || (num_buffers > 7))
1458 		return -EINVAL;
1459 
1460 	rar = qbman_read_register(s, QBMAN_CINH_SWP_RAR);
1461 	if (!RAR_SUCCESS(rar))
1462 		return -EBUSY;
1463 
1464 	/* Start the release command */
1465 	p = qbman_get_cmd(s, QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
1466 
1467 	/* Copy the caller's buffer pointers to the command */
1468 	for (i = 0; i < num_buffers; i++)
1469 		p->buf[i] = cpu_to_le64(buffers[i]);
1470 	p->bpid = d->bpid;
1471 
1472 	/*
1473 	 * Set the verb byte, have to substitute in the valid-bit
1474 	 * and the number of buffers.
1475 	 */
1476 	dma_wmb();
1477 	p->verb = d->verb | RAR_VB(rar) | num_buffers;
1478 
1479 	return 0;
1480 }
1481 
1482 /**
1483  * qbman_swp_release_mem_back() - Issue a buffer release command
1484  * @s:           the software portal object
1485  * @d:           the release descriptor
1486  * @buffers:     a pointer pointing to the buffer address to be released
1487  * @num_buffers: number of buffers to be released,  must be less than 8
1488  *
1489  * Return 0 for success, -EBUSY if the release command ring is not ready.
1490  */
1491 int qbman_swp_release_mem_back(struct qbman_swp *s,
1492 			       const struct qbman_release_desc *d,
1493 			       const u64 *buffers, unsigned int num_buffers)
1494 {
1495 	int i;
1496 	struct qbman_release_desc *p;
1497 	u32 rar;
1498 
1499 	if (!num_buffers || (num_buffers > 7))
1500 		return -EINVAL;
1501 
1502 	rar = qbman_read_register(s, QBMAN_CINH_SWP_RAR);
1503 	if (!RAR_SUCCESS(rar))
1504 		return -EBUSY;
1505 
1506 	/* Start the release command */
1507 	p = qbman_get_cmd(s, QBMAN_CENA_SWP_RCR_MEM(RAR_IDX(rar)));
1508 
1509 	/* Copy the caller's buffer pointers to the command */
1510 	for (i = 0; i < num_buffers; i++)
1511 		p->buf[i] = cpu_to_le64(buffers[i]);
1512 	p->bpid = d->bpid;
1513 
1514 	p->verb = d->verb | RAR_VB(rar) | num_buffers;
1515 	dma_wmb();
1516 	qbman_write_register(s, QBMAN_CINH_SWP_RCR_AM_RT +
1517 			     RAR_IDX(rar)  * 4, QMAN_RT_MODE);
1518 
1519 	return 0;
1520 }
1521 
1522 struct qbman_acquire_desc {
1523 	u8 verb;
1524 	u8 reserved;
1525 	__le16 bpid;
1526 	u8 num;
1527 	u8 reserved2[59];
1528 };
1529 
1530 struct qbman_acquire_rslt {
1531 	u8 verb;
1532 	u8 rslt;
1533 	__le16 reserved;
1534 	u8 num;
1535 	u8 reserved2[3];
1536 	__le64 buf[7];
1537 };
1538 
1539 /**
1540  * qbman_swp_acquire() - Issue a buffer acquire command
1541  * @s:           the software portal object
1542  * @bpid:        the buffer pool index
1543  * @buffers:     a pointer pointing to the acquired buffer addresses
1544  * @num_buffers: number of buffers to be acquired, must be less than 8
1545  *
1546  * Return 0 for success, or negative error code if the acquire command
1547  * fails.
1548  */
1549 int qbman_swp_acquire(struct qbman_swp *s, u16 bpid, u64 *buffers,
1550 		      unsigned int num_buffers)
1551 {
1552 	struct qbman_acquire_desc *p;
1553 	struct qbman_acquire_rslt *r;
1554 	int i;
1555 
1556 	if (!num_buffers || (num_buffers > 7))
1557 		return -EINVAL;
1558 
1559 	/* Start the management command */
1560 	p = qbman_swp_mc_start(s);
1561 
1562 	if (!p)
1563 		return -EBUSY;
1564 
1565 	/* Encode the caller-provided attributes */
1566 	p->bpid = cpu_to_le16(bpid);
1567 	p->num = num_buffers;
1568 
1569 	/* Complete the management command */
1570 	r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE);
1571 	if (unlikely(!r)) {
1572 		pr_err("qbman: acquire from BPID %d failed, no response\n",
1573 		       bpid);
1574 		return -EIO;
1575 	}
1576 
1577 	/* Decode the outcome */
1578 	WARN_ON((r->verb & 0x7f) != QBMAN_MC_ACQUIRE);
1579 
1580 	/* Determine success or failure */
1581 	if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
1582 		pr_err("qbman: acquire from BPID 0x%x failed, code=0x%02x\n",
1583 		       bpid, r->rslt);
1584 		return -EIO;
1585 	}
1586 
1587 	WARN_ON(r->num > num_buffers);
1588 
1589 	/* Copy the acquired buffers to the caller's array */
1590 	for (i = 0; i < r->num; i++)
1591 		buffers[i] = le64_to_cpu(r->buf[i]);
1592 
1593 	return (int)r->num;
1594 }
1595 
1596 struct qbman_alt_fq_state_desc {
1597 	u8 verb;
1598 	u8 reserved[3];
1599 	__le32 fqid;
1600 	u8 reserved2[56];
1601 };
1602 
1603 struct qbman_alt_fq_state_rslt {
1604 	u8 verb;
1605 	u8 rslt;
1606 	u8 reserved[62];
1607 };
1608 
1609 #define ALT_FQ_FQID_MASK 0x00FFFFFF
1610 
1611 int qbman_swp_alt_fq_state(struct qbman_swp *s, u32 fqid,
1612 			   u8 alt_fq_verb)
1613 {
1614 	struct qbman_alt_fq_state_desc *p;
1615 	struct qbman_alt_fq_state_rslt *r;
1616 
1617 	/* Start the management command */
1618 	p = qbman_swp_mc_start(s);
1619 	if (!p)
1620 		return -EBUSY;
1621 
1622 	p->fqid = cpu_to_le32(fqid & ALT_FQ_FQID_MASK);
1623 
1624 	/* Complete the management command */
1625 	r = qbman_swp_mc_complete(s, p, alt_fq_verb);
1626 	if (unlikely(!r)) {
1627 		pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n",
1628 		       alt_fq_verb);
1629 		return -EIO;
1630 	}
1631 
1632 	/* Decode the outcome */
1633 	WARN_ON((r->verb & QBMAN_RESULT_MASK) != alt_fq_verb);
1634 
1635 	/* Determine success or failure */
1636 	if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
1637 		pr_err("qbman: ALT FQID %d failed: verb = 0x%08x code = 0x%02x\n",
1638 		       fqid, r->verb, r->rslt);
1639 		return -EIO;
1640 	}
1641 
1642 	return 0;
1643 }
1644 
1645 struct qbman_cdan_ctrl_desc {
1646 	u8 verb;
1647 	u8 reserved;
1648 	__le16 ch;
1649 	u8 we;
1650 	u8 ctrl;
1651 	__le16 reserved2;
1652 	__le64 cdan_ctx;
1653 	u8 reserved3[48];
1654 
1655 };
1656 
1657 struct qbman_cdan_ctrl_rslt {
1658 	u8 verb;
1659 	u8 rslt;
1660 	__le16 ch;
1661 	u8 reserved[60];
1662 };
1663 
1664 int qbman_swp_CDAN_set(struct qbman_swp *s, u16 channelid,
1665 		       u8 we_mask, u8 cdan_en,
1666 		       u64 ctx)
1667 {
1668 	struct qbman_cdan_ctrl_desc *p = NULL;
1669 	struct qbman_cdan_ctrl_rslt *r = NULL;
1670 
1671 	/* Start the management command */
1672 	p = qbman_swp_mc_start(s);
1673 	if (!p)
1674 		return -EBUSY;
1675 
1676 	/* Encode the caller-provided attributes */
1677 	p->ch = cpu_to_le16(channelid);
1678 	p->we = we_mask;
1679 	if (cdan_en)
1680 		p->ctrl = 1;
1681 	else
1682 		p->ctrl = 0;
1683 	p->cdan_ctx = cpu_to_le64(ctx);
1684 
1685 	/* Complete the management command */
1686 	r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE);
1687 	if (unlikely(!r)) {
1688 		pr_err("qbman: wqchan config failed, no response\n");
1689 		return -EIO;
1690 	}
1691 
1692 	WARN_ON((r->verb & 0x7f) != QBMAN_WQCHAN_CONFIGURE);
1693 
1694 	/* Determine success or failure */
1695 	if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) {
1696 		pr_err("qbman: CDAN cQID %d failed: code = 0x%02x\n",
1697 		       channelid, r->rslt);
1698 		return -EIO;
1699 	}
1700 
1701 	return 0;
1702 }
1703 
1704 #define QBMAN_RESPONSE_VERB_MASK	0x7f
1705 #define QBMAN_FQ_QUERY_NP		0x45
1706 #define QBMAN_BP_QUERY			0x32
1707 
1708 struct qbman_fq_query_desc {
1709 	u8 verb;
1710 	u8 reserved[3];
1711 	__le32 fqid;
1712 	u8 reserved2[56];
1713 };
1714 
1715 int qbman_fq_query_state(struct qbman_swp *s, u32 fqid,
1716 			 struct qbman_fq_query_np_rslt *r)
1717 {
1718 	struct qbman_fq_query_desc *p;
1719 	void *resp;
1720 
1721 	p = (struct qbman_fq_query_desc *)qbman_swp_mc_start(s);
1722 	if (!p)
1723 		return -EBUSY;
1724 
1725 	/* FQID is a 24 bit value */
1726 	p->fqid = cpu_to_le32(fqid & 0x00FFFFFF);
1727 	resp = qbman_swp_mc_complete(s, p, QBMAN_FQ_QUERY_NP);
1728 	if (!resp) {
1729 		pr_err("qbman: Query FQID %d NP fields failed, no response\n",
1730 		       fqid);
1731 		return -EIO;
1732 	}
1733 	*r = *(struct qbman_fq_query_np_rslt *)resp;
1734 	/* Decode the outcome */
1735 	WARN_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_FQ_QUERY_NP);
1736 
1737 	/* Determine success or failure */
1738 	if (r->rslt != QBMAN_MC_RSLT_OK) {
1739 		pr_err("Query NP fields of FQID 0x%x failed, code=0x%02x\n",
1740 		       p->fqid, r->rslt);
1741 		return -EIO;
1742 	}
1743 
1744 	return 0;
1745 }
1746 
1747 u32 qbman_fq_state_frame_count(const struct qbman_fq_query_np_rslt *r)
1748 {
1749 	return (le32_to_cpu(r->frm_cnt) & 0x00FFFFFF);
1750 }
1751 
1752 u32 qbman_fq_state_byte_count(const struct qbman_fq_query_np_rslt *r)
1753 {
1754 	return le32_to_cpu(r->byte_cnt);
1755 }
1756 
1757 struct qbman_bp_query_desc {
1758 	u8 verb;
1759 	u8 reserved;
1760 	__le16 bpid;
1761 	u8 reserved2[60];
1762 };
1763 
1764 int qbman_bp_query(struct qbman_swp *s, u16 bpid,
1765 		   struct qbman_bp_query_rslt *r)
1766 {
1767 	struct qbman_bp_query_desc *p;
1768 	void *resp;
1769 
1770 	p = (struct qbman_bp_query_desc *)qbman_swp_mc_start(s);
1771 	if (!p)
1772 		return -EBUSY;
1773 
1774 	p->bpid = cpu_to_le16(bpid);
1775 	resp = qbman_swp_mc_complete(s, p, QBMAN_BP_QUERY);
1776 	if (!resp) {
1777 		pr_err("qbman: Query BPID %d fields failed, no response\n",
1778 		       bpid);
1779 		return -EIO;
1780 	}
1781 	*r = *(struct qbman_bp_query_rslt *)resp;
1782 	/* Decode the outcome */
1783 	WARN_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_BP_QUERY);
1784 
1785 	/* Determine success or failure */
1786 	if (r->rslt != QBMAN_MC_RSLT_OK) {
1787 		pr_err("Query fields of BPID 0x%x failed, code=0x%02x\n",
1788 		       bpid, r->rslt);
1789 		return -EIO;
1790 	}
1791 
1792 	return 0;
1793 }
1794 
1795 u32 qbman_bp_info_num_free_bufs(struct qbman_bp_query_rslt *a)
1796 {
1797 	return le32_to_cpu(a->fill);
1798 }
1799