xref: /openbmc/linux/drivers/scsi/csiostor/csio_wr.c (revision cce8e04c)
1 /*
2  * This file is part of the Chelsio FCoE driver for Linux.
3  *
4  * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  */
34 
35 #include <linux/kernel.h>
36 #include <linux/string.h>
37 #include <linux/compiler.h>
38 #include <linux/slab.h>
39 #include <asm/page.h>
40 #include <linux/cache.h>
41 
42 #include "t4_values.h"
43 #include "csio_hw.h"
44 #include "csio_wr.h"
45 #include "csio_mb.h"
46 #include "csio_defs.h"
47 
48 int csio_intr_coalesce_cnt;		/* value:SGE_INGRESS_RX_THRESHOLD[0] */
49 static int csio_sge_thresh_reg;		/* SGE_INGRESS_RX_THRESHOLD[0] */
50 
51 int csio_intr_coalesce_time = 10;	/* value:SGE_TIMER_VALUE_1 */
52 static int csio_sge_timer_reg = 1;
53 
54 #define CSIO_SET_FLBUF_SIZE(_hw, _reg, _val)				\
55 	csio_wr_reg32((_hw), (_val), SGE_FL_BUFFER_SIZE##_reg##_A)
56 
57 static void
58 csio_get_flbuf_size(struct csio_hw *hw, struct csio_sge *sge, uint32_t reg)
59 {
60 	sge->sge_fl_buf_size[reg] = csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE0_A +
61 							reg * sizeof(uint32_t));
62 }
63 
64 /* Free list buffer size */
65 static inline uint32_t
66 csio_wr_fl_bufsz(struct csio_sge *sge, struct csio_dma_buf *buf)
67 {
68 	return sge->sge_fl_buf_size[buf->paddr & 0xF];
69 }
70 
71 /* Size of the egress queue status page */
72 static inline uint32_t
73 csio_wr_qstat_pgsz(struct csio_hw *hw)
74 {
75 	return (hw->wrm.sge.sge_control & EGRSTATUSPAGESIZE_F) ?  128 : 64;
76 }
77 
78 /* Ring freelist doorbell */
79 static inline void
80 csio_wr_ring_fldb(struct csio_hw *hw, struct csio_q *flq)
81 {
82 	/*
83 	 * Ring the doorbell only when we have atleast CSIO_QCREDIT_SZ
84 	 * number of bytes in the freelist queue. This translates to atleast
85 	 * 8 freelist buffer pointers (since each pointer is 8 bytes).
86 	 */
87 	if (flq->inc_idx >= 8) {
88 		csio_wr_reg32(hw, DBPRIO_F | QID_V(flq->un.fl.flid) |
89 				  PIDX_T5_V(flq->inc_idx / 8) | DBTYPE_F,
90 				  MYPF_REG(SGE_PF_KDOORBELL_A));
91 		flq->inc_idx &= 7;
92 	}
93 }
94 
95 /* Write a 0 cidx increment value to enable SGE interrupts for this queue */
96 static void
97 csio_wr_sge_intr_enable(struct csio_hw *hw, uint16_t iqid)
98 {
99 	csio_wr_reg32(hw, CIDXINC_V(0)		|
100 			  INGRESSQID_V(iqid)	|
101 			  TIMERREG_V(X_TIMERREG_RESTART_COUNTER),
102 			  MYPF_REG(SGE_PF_GTS_A));
103 }
104 
105 /*
106  * csio_wr_fill_fl - Populate the FL buffers of a FL queue.
107  * @hw: HW module.
108  * @flq: Freelist queue.
109  *
110  * Fill up freelist buffer entries with buffers of size specified
111  * in the size register.
112  *
113  */
114 static int
115 csio_wr_fill_fl(struct csio_hw *hw, struct csio_q *flq)
116 {
117 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
118 	struct csio_sge *sge = &wrm->sge;
119 	__be64 *d = (__be64 *)(flq->vstart);
120 	struct csio_dma_buf *buf = &flq->un.fl.bufs[0];
121 	uint64_t paddr;
122 	int sreg = flq->un.fl.sreg;
123 	int n = flq->credits;
124 
125 	while (n--) {
126 		buf->len = sge->sge_fl_buf_size[sreg];
127 		buf->vaddr = dma_alloc_coherent(&hw->pdev->dev, buf->len,
128 						&buf->paddr, GFP_KERNEL);
129 		if (!buf->vaddr) {
130 			csio_err(hw, "Could only fill %d buffers!\n", n + 1);
131 			return -ENOMEM;
132 		}
133 
134 		paddr = buf->paddr | (sreg & 0xF);
135 
136 		*d++ = cpu_to_be64(paddr);
137 		buf++;
138 	}
139 
140 	return 0;
141 }
142 
143 /*
144  * csio_wr_update_fl -
145  * @hw: HW module.
146  * @flq: Freelist queue.
147  *
148  *
149  */
150 static inline void
151 csio_wr_update_fl(struct csio_hw *hw, struct csio_q *flq, uint16_t n)
152 {
153 
154 	flq->inc_idx += n;
155 	flq->pidx += n;
156 	if (unlikely(flq->pidx >= flq->credits))
157 		flq->pidx -= (uint16_t)flq->credits;
158 
159 	CSIO_INC_STATS(flq, n_flq_refill);
160 }
161 
162 /*
163  * csio_wr_alloc_q - Allocate a WR queue and initialize it.
164  * @hw: HW module
165  * @qsize: Size of the queue in bytes
166  * @wrsize: Since of WR in this queue, if fixed.
167  * @type: Type of queue (Ingress/Egress/Freelist)
168  * @owner: Module that owns this queue.
169  * @nflb: Number of freelist buffers for FL.
170  * @sreg: What is the FL buffer size register?
171  * @iq_int_handler: Ingress queue handler in INTx mode.
172  *
173  * This function allocates and sets up a queue for the caller
174  * of size qsize, aligned at the required boundary. This is subject to
175  * be free entries being available in the queue array. If one is found,
176  * it is initialized with the allocated queue, marked as being used (owner),
177  * and a handle returned to the caller in form of the queue's index
178  * into the q_arr array.
179  * If user has indicated a freelist (by specifying nflb > 0), create
180  * another queue (with its own index into q_arr) for the freelist. Allocate
181  * memory for DMA buffer metadata (vaddr, len etc). Save off the freelist
182  * idx in the ingress queue's flq.idx. This is how a Freelist is associated
183  * with its owning ingress queue.
184  */
185 int
186 csio_wr_alloc_q(struct csio_hw *hw, uint32_t qsize, uint32_t wrsize,
187 		uint16_t type, void *owner, uint32_t nflb, int sreg,
188 		iq_handler_t iq_intx_handler)
189 {
190 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
191 	struct csio_q	*q, *flq;
192 	int		free_idx = wrm->free_qidx;
193 	int		ret_idx = free_idx;
194 	uint32_t	qsz;
195 	int flq_idx;
196 
197 	if (free_idx >= wrm->num_q) {
198 		csio_err(hw, "No more free queues.\n");
199 		return -1;
200 	}
201 
202 	switch (type) {
203 	case CSIO_EGRESS:
204 		qsz = ALIGN(qsize, CSIO_QCREDIT_SZ) + csio_wr_qstat_pgsz(hw);
205 		break;
206 	case CSIO_INGRESS:
207 		switch (wrsize) {
208 		case 16:
209 		case 32:
210 		case 64:
211 		case 128:
212 			break;
213 		default:
214 			csio_err(hw, "Invalid Ingress queue WR size:%d\n",
215 				    wrsize);
216 			return -1;
217 		}
218 
219 		/*
220 		 * Number of elements must be a multiple of 16
221 		 * So this includes status page size
222 		 */
223 		qsz = ALIGN(qsize/wrsize, 16) * wrsize;
224 
225 		break;
226 	case CSIO_FREELIST:
227 		qsz = ALIGN(qsize/wrsize, 8) * wrsize + csio_wr_qstat_pgsz(hw);
228 		break;
229 	default:
230 		csio_err(hw, "Invalid queue type: 0x%x\n", type);
231 		return -1;
232 	}
233 
234 	q = wrm->q_arr[free_idx];
235 
236 	q->vstart = dma_zalloc_coherent(&hw->pdev->dev, qsz, &q->pstart,
237 			GFP_KERNEL);
238 	if (!q->vstart) {
239 		csio_err(hw,
240 			 "Failed to allocate DMA memory for "
241 			 "queue at id: %d size: %d\n", free_idx, qsize);
242 		return -1;
243 	}
244 
245 	q->type		= type;
246 	q->owner	= owner;
247 	q->pidx		= q->cidx = q->inc_idx = 0;
248 	q->size		= qsz;
249 	q->wr_sz	= wrsize;	/* If using fixed size WRs */
250 
251 	wrm->free_qidx++;
252 
253 	if (type == CSIO_INGRESS) {
254 		/* Since queue area is set to zero */
255 		q->un.iq.genbit	= 1;
256 
257 		/*
258 		 * Ingress queue status page size is always the size of
259 		 * the ingress queue entry.
260 		 */
261 		q->credits	= (qsz - q->wr_sz) / q->wr_sz;
262 		q->vwrap	= (void *)((uintptr_t)(q->vstart) + qsz
263 							- q->wr_sz);
264 
265 		/* Allocate memory for FL if requested */
266 		if (nflb > 0) {
267 			flq_idx = csio_wr_alloc_q(hw, nflb * sizeof(__be64),
268 						  sizeof(__be64), CSIO_FREELIST,
269 						  owner, 0, sreg, NULL);
270 			if (flq_idx == -1) {
271 				csio_err(hw,
272 					 "Failed to allocate FL queue"
273 					 " for IQ idx:%d\n", free_idx);
274 				return -1;
275 			}
276 
277 			/* Associate the new FL with the Ingress quue */
278 			q->un.iq.flq_idx = flq_idx;
279 
280 			flq = wrm->q_arr[q->un.iq.flq_idx];
281 			flq->un.fl.bufs = kcalloc(flq->credits,
282 						  sizeof(struct csio_dma_buf),
283 						  GFP_KERNEL);
284 			if (!flq->un.fl.bufs) {
285 				csio_err(hw,
286 					 "Failed to allocate FL queue bufs"
287 					 " for IQ idx:%d\n", free_idx);
288 				return -1;
289 			}
290 
291 			flq->un.fl.packen = 0;
292 			flq->un.fl.offset = 0;
293 			flq->un.fl.sreg = sreg;
294 
295 			/* Fill up the free list buffers */
296 			if (csio_wr_fill_fl(hw, flq))
297 				return -1;
298 
299 			/*
300 			 * Make sure in a FLQ, atleast 1 credit (8 FL buffers)
301 			 * remains unpopulated,otherwise HW thinks
302 			 * FLQ is empty.
303 			 */
304 			flq->pidx = flq->inc_idx = flq->credits - 8;
305 		} else {
306 			q->un.iq.flq_idx = -1;
307 		}
308 
309 		/* Associate the IQ INTx handler. */
310 		q->un.iq.iq_intx_handler = iq_intx_handler;
311 
312 		csio_q_iqid(hw, ret_idx) = CSIO_MAX_QID;
313 
314 	} else if (type == CSIO_EGRESS) {
315 		q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / CSIO_QCREDIT_SZ;
316 		q->vwrap   = (void *)((uintptr_t)(q->vstart) + qsz
317 						- csio_wr_qstat_pgsz(hw));
318 		csio_q_eqid(hw, ret_idx) = CSIO_MAX_QID;
319 	} else { /* Freelist */
320 		q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / sizeof(__be64);
321 		q->vwrap   = (void *)((uintptr_t)(q->vstart) + qsz
322 						- csio_wr_qstat_pgsz(hw));
323 		csio_q_flid(hw, ret_idx) = CSIO_MAX_QID;
324 	}
325 
326 	return ret_idx;
327 }
328 
329 /*
330  * csio_wr_iq_create_rsp - Response handler for IQ creation.
331  * @hw: The HW module.
332  * @mbp: Mailbox.
333  * @iq_idx: Ingress queue that got created.
334  *
335  * Handle FW_IQ_CMD mailbox completion. Save off the assigned IQ/FL ids.
336  */
337 static int
338 csio_wr_iq_create_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)
339 {
340 	struct csio_iq_params iqp;
341 	enum fw_retval retval;
342 	uint32_t iq_id;
343 	int flq_idx;
344 
345 	memset(&iqp, 0, sizeof(struct csio_iq_params));
346 
347 	csio_mb_iq_alloc_write_rsp(hw, mbp, &retval, &iqp);
348 
349 	if (retval != FW_SUCCESS) {
350 		csio_err(hw, "IQ cmd returned 0x%x!\n", retval);
351 		mempool_free(mbp, hw->mb_mempool);
352 		return -EINVAL;
353 	}
354 
355 	csio_q_iqid(hw, iq_idx)		= iqp.iqid;
356 	csio_q_physiqid(hw, iq_idx)	= iqp.physiqid;
357 	csio_q_pidx(hw, iq_idx)		= csio_q_cidx(hw, iq_idx) = 0;
358 	csio_q_inc_idx(hw, iq_idx)	= 0;
359 
360 	/* Actual iq-id. */
361 	iq_id = iqp.iqid - hw->wrm.fw_iq_start;
362 
363 	/* Set the iq-id to iq map table. */
364 	if (iq_id >= CSIO_MAX_IQ) {
365 		csio_err(hw,
366 			 "Exceeding MAX_IQ(%d) supported!"
367 			 " iqid:%d rel_iqid:%d FW iq_start:%d\n",
368 			 CSIO_MAX_IQ, iq_id, iqp.iqid, hw->wrm.fw_iq_start);
369 		mempool_free(mbp, hw->mb_mempool);
370 		return -EINVAL;
371 	}
372 	csio_q_set_intr_map(hw, iq_idx, iq_id);
373 
374 	/*
375 	 * During FW_IQ_CMD, FW sets interrupt_sent bit to 1 in the SGE
376 	 * ingress context of this queue. This will block interrupts to
377 	 * this queue until the next GTS write. Therefore, we do a
378 	 * 0-cidx increment GTS write for this queue just to clear the
379 	 * interrupt_sent bit. This will re-enable interrupts to this
380 	 * queue.
381 	 */
382 	csio_wr_sge_intr_enable(hw, iqp.physiqid);
383 
384 	flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
385 	if (flq_idx != -1) {
386 		struct csio_q *flq = hw->wrm.q_arr[flq_idx];
387 
388 		csio_q_flid(hw, flq_idx) = iqp.fl0id;
389 		csio_q_cidx(hw, flq_idx) = 0;
390 		csio_q_pidx(hw, flq_idx)    = csio_q_credits(hw, flq_idx) - 8;
391 		csio_q_inc_idx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8;
392 
393 		/* Now update SGE about the buffers allocated during init */
394 		csio_wr_ring_fldb(hw, flq);
395 	}
396 
397 	mempool_free(mbp, hw->mb_mempool);
398 
399 	return 0;
400 }
401 
402 /*
403  * csio_wr_iq_create - Configure an Ingress queue with FW.
404  * @hw: The HW module.
405  * @priv: Private data object.
406  * @iq_idx: Ingress queue index in the WR module.
407  * @vec: MSIX vector.
408  * @portid: PCIE Channel to be associated with this queue.
409  * @async: Is this a FW asynchronous message handling queue?
410  * @cbfn: Completion callback.
411  *
412  * This API configures an ingress queue with FW by issuing a FW_IQ_CMD mailbox
413  * with alloc/write bits set.
414  */
415 int
416 csio_wr_iq_create(struct csio_hw *hw, void *priv, int iq_idx,
417 		  uint32_t vec, uint8_t portid, bool async,
418 		  void (*cbfn) (struct csio_hw *, struct csio_mb *))
419 {
420 	struct csio_mb  *mbp;
421 	struct csio_iq_params iqp;
422 	int flq_idx;
423 
424 	memset(&iqp, 0, sizeof(struct csio_iq_params));
425 	csio_q_portid(hw, iq_idx) = portid;
426 
427 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
428 	if (!mbp) {
429 		csio_err(hw, "IQ command out of memory!\n");
430 		return -ENOMEM;
431 	}
432 
433 	switch (hw->intr_mode) {
434 	case CSIO_IM_INTX:
435 	case CSIO_IM_MSI:
436 		/* For interrupt forwarding queue only */
437 		if (hw->intr_iq_idx == iq_idx)
438 			iqp.iqandst	= X_INTERRUPTDESTINATION_PCIE;
439 		else
440 			iqp.iqandst	= X_INTERRUPTDESTINATION_IQ;
441 		iqp.iqandstindex	=
442 			csio_q_physiqid(hw, hw->intr_iq_idx);
443 		break;
444 	case CSIO_IM_MSIX:
445 		iqp.iqandst		= X_INTERRUPTDESTINATION_PCIE;
446 		iqp.iqandstindex	= (uint16_t)vec;
447 		break;
448 	case CSIO_IM_NONE:
449 		mempool_free(mbp, hw->mb_mempool);
450 		return -EINVAL;
451 	}
452 
453 	/* Pass in the ingress queue cmd parameters */
454 	iqp.pfn			= hw->pfn;
455 	iqp.vfn			= 0;
456 	iqp.iq_start		= 1;
457 	iqp.viid		= 0;
458 	iqp.type		= FW_IQ_TYPE_FL_INT_CAP;
459 	iqp.iqasynch		= async;
460 	if (csio_intr_coalesce_cnt)
461 		iqp.iqanus	= X_UPDATESCHEDULING_COUNTER_OPTTIMER;
462 	else
463 		iqp.iqanus	= X_UPDATESCHEDULING_TIMER;
464 	iqp.iqanud		= X_UPDATEDELIVERY_INTERRUPT;
465 	iqp.iqpciech		= portid;
466 	iqp.iqintcntthresh	= (uint8_t)csio_sge_thresh_reg;
467 
468 	switch (csio_q_wr_sz(hw, iq_idx)) {
469 	case 16:
470 		iqp.iqesize = 0; break;
471 	case 32:
472 		iqp.iqesize = 1; break;
473 	case 64:
474 		iqp.iqesize = 2; break;
475 	case 128:
476 		iqp.iqesize = 3; break;
477 	}
478 
479 	iqp.iqsize		= csio_q_size(hw, iq_idx) /
480 						csio_q_wr_sz(hw, iq_idx);
481 	iqp.iqaddr		= csio_q_pstart(hw, iq_idx);
482 
483 	flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
484 	if (flq_idx != -1) {
485 		enum chip_type chip = CHELSIO_CHIP_VERSION(hw->chip_id);
486 		struct csio_q *flq = hw->wrm.q_arr[flq_idx];
487 
488 		iqp.fl0paden	= 1;
489 		iqp.fl0packen	= flq->un.fl.packen ? 1 : 0;
490 		iqp.fl0fbmin	= X_FETCHBURSTMIN_64B;
491 		iqp.fl0fbmax	= ((chip == CHELSIO_T5) ?
492 				  X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B);
493 		iqp.fl0size	= csio_q_size(hw, flq_idx) / CSIO_QCREDIT_SZ;
494 		iqp.fl0addr	= csio_q_pstart(hw, flq_idx);
495 	}
496 
497 	csio_mb_iq_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);
498 
499 	if (csio_mb_issue(hw, mbp)) {
500 		csio_err(hw, "Issue of IQ cmd failed!\n");
501 		mempool_free(mbp, hw->mb_mempool);
502 		return -EINVAL;
503 	}
504 
505 	if (cbfn != NULL)
506 		return 0;
507 
508 	return csio_wr_iq_create_rsp(hw, mbp, iq_idx);
509 }
510 
511 /*
512  * csio_wr_eq_create_rsp - Response handler for EQ creation.
513  * @hw: The HW module.
514  * @mbp: Mailbox.
515  * @eq_idx: Egress queue that got created.
516  *
517  * Handle FW_EQ_OFLD_CMD mailbox completion. Save off the assigned EQ ids.
518  */
519 static int
520 csio_wr_eq_cfg_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)
521 {
522 	struct csio_eq_params eqp;
523 	enum fw_retval retval;
524 
525 	memset(&eqp, 0, sizeof(struct csio_eq_params));
526 
527 	csio_mb_eq_ofld_alloc_write_rsp(hw, mbp, &retval, &eqp);
528 
529 	if (retval != FW_SUCCESS) {
530 		csio_err(hw, "EQ OFLD cmd returned 0x%x!\n", retval);
531 		mempool_free(mbp, hw->mb_mempool);
532 		return -EINVAL;
533 	}
534 
535 	csio_q_eqid(hw, eq_idx)	= (uint16_t)eqp.eqid;
536 	csio_q_physeqid(hw, eq_idx) = (uint16_t)eqp.physeqid;
537 	csio_q_pidx(hw, eq_idx)	= csio_q_cidx(hw, eq_idx) = 0;
538 	csio_q_inc_idx(hw, eq_idx) = 0;
539 
540 	mempool_free(mbp, hw->mb_mempool);
541 
542 	return 0;
543 }
544 
545 /*
546  * csio_wr_eq_create - Configure an Egress queue with FW.
547  * @hw: HW module.
548  * @priv: Private data.
549  * @eq_idx: Egress queue index in the WR module.
550  * @iq_idx: Associated ingress queue index.
551  * @cbfn: Completion callback.
552  *
553  * This API configures a offload egress queue with FW by issuing a
554  * FW_EQ_OFLD_CMD  (with alloc + write ) mailbox.
555  */
556 int
557 csio_wr_eq_create(struct csio_hw *hw, void *priv, int eq_idx,
558 		  int iq_idx, uint8_t portid,
559 		  void (*cbfn) (struct csio_hw *, struct csio_mb *))
560 {
561 	struct csio_mb  *mbp;
562 	struct csio_eq_params eqp;
563 
564 	memset(&eqp, 0, sizeof(struct csio_eq_params));
565 
566 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
567 	if (!mbp) {
568 		csio_err(hw, "EQ command out of memory!\n");
569 		return -ENOMEM;
570 	}
571 
572 	eqp.pfn			= hw->pfn;
573 	eqp.vfn			= 0;
574 	eqp.eqstart		= 1;
575 	eqp.hostfcmode		= X_HOSTFCMODE_STATUS_PAGE;
576 	eqp.iqid		= csio_q_iqid(hw, iq_idx);
577 	eqp.fbmin		= X_FETCHBURSTMIN_64B;
578 	eqp.fbmax		= X_FETCHBURSTMAX_512B;
579 	eqp.cidxfthresh		= 0;
580 	eqp.pciechn		= portid;
581 	eqp.eqsize		= csio_q_size(hw, eq_idx) / CSIO_QCREDIT_SZ;
582 	eqp.eqaddr		= csio_q_pstart(hw, eq_idx);
583 
584 	csio_mb_eq_ofld_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO,
585 				    &eqp, cbfn);
586 
587 	if (csio_mb_issue(hw, mbp)) {
588 		csio_err(hw, "Issue of EQ OFLD cmd failed!\n");
589 		mempool_free(mbp, hw->mb_mempool);
590 		return -EINVAL;
591 	}
592 
593 	if (cbfn != NULL)
594 		return 0;
595 
596 	return csio_wr_eq_cfg_rsp(hw, mbp, eq_idx);
597 }
598 
599 /*
600  * csio_wr_iq_destroy_rsp - Response handler for IQ removal.
601  * @hw: The HW module.
602  * @mbp: Mailbox.
603  * @iq_idx: Ingress queue that was freed.
604  *
605  * Handle FW_IQ_CMD (free) mailbox completion.
606  */
607 static int
608 csio_wr_iq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)
609 {
610 	enum fw_retval retval = csio_mb_fw_retval(mbp);
611 	int rv = 0;
612 
613 	if (retval != FW_SUCCESS)
614 		rv = -EINVAL;
615 
616 	mempool_free(mbp, hw->mb_mempool);
617 
618 	return rv;
619 }
620 
621 /*
622  * csio_wr_iq_destroy - Free an ingress queue.
623  * @hw: The HW module.
624  * @priv: Private data object.
625  * @iq_idx: Ingress queue index to destroy
626  * @cbfn: Completion callback.
627  *
628  * This API frees an ingress queue by issuing the FW_IQ_CMD
629  * with the free bit set.
630  */
631 static int
632 csio_wr_iq_destroy(struct csio_hw *hw, void *priv, int iq_idx,
633 		   void (*cbfn)(struct csio_hw *, struct csio_mb *))
634 {
635 	int rv = 0;
636 	struct csio_mb  *mbp;
637 	struct csio_iq_params iqp;
638 	int flq_idx;
639 
640 	memset(&iqp, 0, sizeof(struct csio_iq_params));
641 
642 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
643 	if (!mbp)
644 		return -ENOMEM;
645 
646 	iqp.pfn		= hw->pfn;
647 	iqp.vfn		= 0;
648 	iqp.iqid	= csio_q_iqid(hw, iq_idx);
649 	iqp.type	= FW_IQ_TYPE_FL_INT_CAP;
650 
651 	flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
652 	if (flq_idx != -1)
653 		iqp.fl0id = csio_q_flid(hw, flq_idx);
654 	else
655 		iqp.fl0id = 0xFFFF;
656 
657 	iqp.fl1id = 0xFFFF;
658 
659 	csio_mb_iq_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);
660 
661 	rv = csio_mb_issue(hw, mbp);
662 	if (rv != 0) {
663 		mempool_free(mbp, hw->mb_mempool);
664 		return rv;
665 	}
666 
667 	if (cbfn != NULL)
668 		return 0;
669 
670 	return csio_wr_iq_destroy_rsp(hw, mbp, iq_idx);
671 }
672 
673 /*
674  * csio_wr_eq_destroy_rsp - Response handler for OFLD EQ creation.
675  * @hw: The HW module.
676  * @mbp: Mailbox.
677  * @eq_idx: Egress queue that was freed.
678  *
679  * Handle FW_OFLD_EQ_CMD (free) mailbox completion.
680  */
681 static int
682 csio_wr_eq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)
683 {
684 	enum fw_retval retval = csio_mb_fw_retval(mbp);
685 	int rv = 0;
686 
687 	if (retval != FW_SUCCESS)
688 		rv = -EINVAL;
689 
690 	mempool_free(mbp, hw->mb_mempool);
691 
692 	return rv;
693 }
694 
695 /*
696  * csio_wr_eq_destroy - Free an Egress queue.
697  * @hw: The HW module.
698  * @priv: Private data object.
699  * @eq_idx: Egress queue index to destroy
700  * @cbfn: Completion callback.
701  *
702  * This API frees an Egress queue by issuing the FW_EQ_OFLD_CMD
703  * with the free bit set.
704  */
705 static int
706 csio_wr_eq_destroy(struct csio_hw *hw, void *priv, int eq_idx,
707 		   void (*cbfn) (struct csio_hw *, struct csio_mb *))
708 {
709 	int rv = 0;
710 	struct csio_mb  *mbp;
711 	struct csio_eq_params eqp;
712 
713 	memset(&eqp, 0, sizeof(struct csio_eq_params));
714 
715 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
716 	if (!mbp)
717 		return -ENOMEM;
718 
719 	eqp.pfn		= hw->pfn;
720 	eqp.vfn		= 0;
721 	eqp.eqid	= csio_q_eqid(hw, eq_idx);
722 
723 	csio_mb_eq_ofld_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &eqp, cbfn);
724 
725 	rv = csio_mb_issue(hw, mbp);
726 	if (rv != 0) {
727 		mempool_free(mbp, hw->mb_mempool);
728 		return rv;
729 	}
730 
731 	if (cbfn != NULL)
732 		return 0;
733 
734 	return csio_wr_eq_destroy_rsp(hw, mbp, eq_idx);
735 }
736 
737 /*
738  * csio_wr_cleanup_eq_stpg - Cleanup Egress queue status page
739  * @hw: HW module
740  * @qidx: Egress queue index
741  *
742  * Cleanup the Egress queue status page.
743  */
744 static void
745 csio_wr_cleanup_eq_stpg(struct csio_hw *hw, int qidx)
746 {
747 	struct csio_q	*q = csio_hw_to_wrm(hw)->q_arr[qidx];
748 	struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;
749 
750 	memset(stp, 0, sizeof(*stp));
751 }
752 
753 /*
754  * csio_wr_cleanup_iq_ftr - Cleanup Footer entries in IQ
755  * @hw: HW module
756  * @qidx: Ingress queue index
757  *
758  * Cleanup the footer entries in the given ingress queue,
759  * set to 1 the internal copy of genbit.
760  */
761 static void
762 csio_wr_cleanup_iq_ftr(struct csio_hw *hw, int qidx)
763 {
764 	struct csio_wrm *wrm	= csio_hw_to_wrm(hw);
765 	struct csio_q	*q	= wrm->q_arr[qidx];
766 	void *wr;
767 	struct csio_iqwr_footer *ftr;
768 	uint32_t i = 0;
769 
770 	/* set to 1 since we are just about zero out genbit */
771 	q->un.iq.genbit = 1;
772 
773 	for (i = 0; i < q->credits; i++) {
774 		/* Get the WR */
775 		wr = (void *)((uintptr_t)q->vstart +
776 					   (i * q->wr_sz));
777 		/* Get the footer */
778 		ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
779 					  (q->wr_sz - sizeof(*ftr)));
780 		/* Zero out footer */
781 		memset(ftr, 0, sizeof(*ftr));
782 	}
783 }
784 
785 int
786 csio_wr_destroy_queues(struct csio_hw *hw, bool cmd)
787 {
788 	int i, flq_idx;
789 	struct csio_q *q;
790 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
791 	int rv;
792 
793 	for (i = 0; i < wrm->free_qidx; i++) {
794 		q = wrm->q_arr[i];
795 
796 		switch (q->type) {
797 		case CSIO_EGRESS:
798 			if (csio_q_eqid(hw, i) != CSIO_MAX_QID) {
799 				csio_wr_cleanup_eq_stpg(hw, i);
800 				if (!cmd) {
801 					csio_q_eqid(hw, i) = CSIO_MAX_QID;
802 					continue;
803 				}
804 
805 				rv = csio_wr_eq_destroy(hw, NULL, i, NULL);
806 				if ((rv == -EBUSY) || (rv == -ETIMEDOUT))
807 					cmd = false;
808 
809 				csio_q_eqid(hw, i) = CSIO_MAX_QID;
810 			}
811 		case CSIO_INGRESS:
812 			if (csio_q_iqid(hw, i) != CSIO_MAX_QID) {
813 				csio_wr_cleanup_iq_ftr(hw, i);
814 				if (!cmd) {
815 					csio_q_iqid(hw, i) = CSIO_MAX_QID;
816 					flq_idx = csio_q_iq_flq_idx(hw, i);
817 					if (flq_idx != -1)
818 						csio_q_flid(hw, flq_idx) =
819 								CSIO_MAX_QID;
820 					continue;
821 				}
822 
823 				rv = csio_wr_iq_destroy(hw, NULL, i, NULL);
824 				if ((rv == -EBUSY) || (rv == -ETIMEDOUT))
825 					cmd = false;
826 
827 				csio_q_iqid(hw, i) = CSIO_MAX_QID;
828 				flq_idx = csio_q_iq_flq_idx(hw, i);
829 				if (flq_idx != -1)
830 					csio_q_flid(hw, flq_idx) = CSIO_MAX_QID;
831 			}
832 		default:
833 			break;
834 		}
835 	}
836 
837 	hw->flags &= ~CSIO_HWF_Q_FW_ALLOCED;
838 
839 	return 0;
840 }
841 
842 /*
843  * csio_wr_get - Get requested size of WR entry/entries from queue.
844  * @hw: HW module.
845  * @qidx: Index of queue.
846  * @size: Cumulative size of Work request(s).
847  * @wrp: Work request pair.
848  *
849  * If requested credits are available, return the start address of the
850  * work request in the work request pair. Set pidx accordingly and
851  * return.
852  *
853  * NOTE about WR pair:
854  * ==================
855  * A WR can start towards the end of a queue, and then continue at the
856  * beginning, since the queue is considered to be circular. This will
857  * require a pair of address/size to be passed back to the caller -
858  * hence Work request pair format.
859  */
860 int
861 csio_wr_get(struct csio_hw *hw, int qidx, uint32_t size,
862 	    struct csio_wr_pair *wrp)
863 {
864 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
865 	struct csio_q *q = wrm->q_arr[qidx];
866 	void *cwr = (void *)((uintptr_t)(q->vstart) +
867 						(q->pidx * CSIO_QCREDIT_SZ));
868 	struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;
869 	uint16_t cidx = q->cidx = ntohs(stp->cidx);
870 	uint16_t pidx = q->pidx;
871 	uint32_t req_sz	= ALIGN(size, CSIO_QCREDIT_SZ);
872 	int req_credits	= req_sz / CSIO_QCREDIT_SZ;
873 	int credits;
874 
875 	CSIO_DB_ASSERT(q->owner != NULL);
876 	CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));
877 	CSIO_DB_ASSERT(cidx <= q->credits);
878 
879 	/* Calculate credits */
880 	if (pidx > cidx) {
881 		credits = q->credits - (pidx - cidx) - 1;
882 	} else if (cidx > pidx) {
883 		credits = cidx - pidx - 1;
884 	} else {
885 		/* cidx == pidx, empty queue */
886 		credits = q->credits;
887 		CSIO_INC_STATS(q, n_qempty);
888 	}
889 
890 	/*
891 	 * Check if we have enough credits.
892 	 * credits = 1 implies queue is full.
893 	 */
894 	if (!credits || (req_credits > credits)) {
895 		CSIO_INC_STATS(q, n_qfull);
896 		return -EBUSY;
897 	}
898 
899 	/*
900 	 * If we are here, we have enough credits to satisfy the
901 	 * request. Check if we are near the end of q, and if WR spills over.
902 	 * If it does, use the first addr/size to cover the queue until
903 	 * the end. Fit the remainder portion of the request at the top
904 	 * of queue and return it in the second addr/len. Set pidx
905 	 * accordingly.
906 	 */
907 	if (unlikely(((uintptr_t)cwr + req_sz) > (uintptr_t)(q->vwrap))) {
908 		wrp->addr1 = cwr;
909 		wrp->size1 = (uint32_t)((uintptr_t)q->vwrap - (uintptr_t)cwr);
910 		wrp->addr2 = q->vstart;
911 		wrp->size2 = req_sz - wrp->size1;
912 		q->pidx	= (uint16_t)(ALIGN(wrp->size2, CSIO_QCREDIT_SZ) /
913 							CSIO_QCREDIT_SZ);
914 		CSIO_INC_STATS(q, n_qwrap);
915 		CSIO_INC_STATS(q, n_eq_wr_split);
916 	} else {
917 		wrp->addr1 = cwr;
918 		wrp->size1 = req_sz;
919 		wrp->addr2 = NULL;
920 		wrp->size2 = 0;
921 		q->pidx	+= (uint16_t)req_credits;
922 
923 		/* We are the end of queue, roll back pidx to top of queue */
924 		if (unlikely(q->pidx == q->credits)) {
925 			q->pidx = 0;
926 			CSIO_INC_STATS(q, n_qwrap);
927 		}
928 	}
929 
930 	q->inc_idx = (uint16_t)req_credits;
931 
932 	CSIO_INC_STATS(q, n_tot_reqs);
933 
934 	return 0;
935 }
936 
937 /*
938  * csio_wr_copy_to_wrp - Copies given data into WR.
939  * @data_buf - Data buffer
940  * @wrp - Work request pair.
941  * @wr_off - Work request offset.
942  * @data_len - Data length.
943  *
944  * Copies the given data in Work Request. Work request pair(wrp) specifies
945  * address information of Work request.
946  * Returns: none
947  */
948 void
949 csio_wr_copy_to_wrp(void *data_buf, struct csio_wr_pair *wrp,
950 		   uint32_t wr_off, uint32_t data_len)
951 {
952 	uint32_t nbytes;
953 
954 	/* Number of space available in buffer addr1 of WRP */
955 	nbytes = ((wrp->size1 - wr_off) >= data_len) ?
956 					data_len : (wrp->size1 - wr_off);
957 
958 	memcpy((uint8_t *) wrp->addr1 + wr_off, data_buf, nbytes);
959 	data_len -= nbytes;
960 
961 	/* Write the remaining data from the begining of circular buffer */
962 	if (data_len) {
963 		CSIO_DB_ASSERT(data_len <= wrp->size2);
964 		CSIO_DB_ASSERT(wrp->addr2 != NULL);
965 		memcpy(wrp->addr2, (uint8_t *) data_buf + nbytes, data_len);
966 	}
967 }
968 
969 /*
970  * csio_wr_issue - Notify chip of Work request.
971  * @hw: HW module.
972  * @qidx: Index of queue.
973  * @prio: 0: Low priority, 1: High priority
974  *
975  * Rings the SGE Doorbell by writing the current producer index of the passed
976  * in queue into the register.
977  *
978  */
979 int
980 csio_wr_issue(struct csio_hw *hw, int qidx, bool prio)
981 {
982 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
983 	struct csio_q *q = wrm->q_arr[qidx];
984 
985 	CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));
986 
987 	wmb();
988 	/* Ring SGE Doorbell writing q->pidx into it */
989 	csio_wr_reg32(hw, DBPRIO_V(prio) | QID_V(q->un.eq.physeqid) |
990 			  PIDX_T5_V(q->inc_idx) | DBTYPE_F,
991 			  MYPF_REG(SGE_PF_KDOORBELL_A));
992 	q->inc_idx = 0;
993 
994 	return 0;
995 }
996 
997 static inline uint32_t
998 csio_wr_avail_qcredits(struct csio_q *q)
999 {
1000 	if (q->pidx > q->cidx)
1001 		return q->pidx - q->cidx;
1002 	else if (q->cidx > q->pidx)
1003 		return q->credits - (q->cidx - q->pidx);
1004 	else
1005 		return 0;	/* cidx == pidx, empty queue */
1006 }
1007 
1008 /*
1009  * csio_wr_inval_flq_buf - Invalidate a free list buffer entry.
1010  * @hw: HW module.
1011  * @flq: The freelist queue.
1012  *
1013  * Invalidate the driver's version of a freelist buffer entry,
1014  * without freeing the associated the DMA memory. The entry
1015  * to be invalidated is picked up from the current Free list
1016  * queue cidx.
1017  *
1018  */
1019 static inline void
1020 csio_wr_inval_flq_buf(struct csio_hw *hw, struct csio_q *flq)
1021 {
1022 	flq->cidx++;
1023 	if (flq->cidx == flq->credits) {
1024 		flq->cidx = 0;
1025 		CSIO_INC_STATS(flq, n_qwrap);
1026 	}
1027 }
1028 
1029 /*
1030  * csio_wr_process_fl - Process a freelist completion.
1031  * @hw: HW module.
1032  * @q: The ingress queue attached to the Freelist.
1033  * @wr: The freelist completion WR in the ingress queue.
1034  * @len_to_qid: The lower 32-bits of the first flit of the RSP footer
1035  * @iq_handler: Caller's handler for this completion.
1036  * @priv: Private pointer of caller
1037  *
1038  */
1039 static inline void
1040 csio_wr_process_fl(struct csio_hw *hw, struct csio_q *q,
1041 		   void *wr, uint32_t len_to_qid,
1042 		   void (*iq_handler)(struct csio_hw *, void *,
1043 				      uint32_t, struct csio_fl_dma_buf *,
1044 				      void *),
1045 		   void *priv)
1046 {
1047 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1048 	struct csio_sge *sge = &wrm->sge;
1049 	struct csio_fl_dma_buf flb;
1050 	struct csio_dma_buf *buf, *fbuf;
1051 	uint32_t bufsz, len, lastlen = 0;
1052 	int n;
1053 	struct csio_q *flq = hw->wrm.q_arr[q->un.iq.flq_idx];
1054 
1055 	CSIO_DB_ASSERT(flq != NULL);
1056 
1057 	len = len_to_qid;
1058 
1059 	if (len & IQWRF_NEWBUF) {
1060 		if (flq->un.fl.offset > 0) {
1061 			csio_wr_inval_flq_buf(hw, flq);
1062 			flq->un.fl.offset = 0;
1063 		}
1064 		len = IQWRF_LEN_GET(len);
1065 	}
1066 
1067 	CSIO_DB_ASSERT(len != 0);
1068 
1069 	flb.totlen = len;
1070 
1071 	/* Consume all freelist buffers used for len bytes */
1072 	for (n = 0, fbuf = flb.flbufs; ; n++, fbuf++) {
1073 		buf = &flq->un.fl.bufs[flq->cidx];
1074 		bufsz = csio_wr_fl_bufsz(sge, buf);
1075 
1076 		fbuf->paddr	= buf->paddr;
1077 		fbuf->vaddr	= buf->vaddr;
1078 
1079 		flb.offset	= flq->un.fl.offset;
1080 		lastlen		= min(bufsz, len);
1081 		fbuf->len	= lastlen;
1082 
1083 		len -= lastlen;
1084 		if (!len)
1085 			break;
1086 		csio_wr_inval_flq_buf(hw, flq);
1087 	}
1088 
1089 	flb.defer_free = flq->un.fl.packen ? 0 : 1;
1090 
1091 	iq_handler(hw, wr, q->wr_sz - sizeof(struct csio_iqwr_footer),
1092 		   &flb, priv);
1093 
1094 	if (flq->un.fl.packen)
1095 		flq->un.fl.offset += ALIGN(lastlen, sge->csio_fl_align);
1096 	else
1097 		csio_wr_inval_flq_buf(hw, flq);
1098 
1099 }
1100 
1101 /*
1102  * csio_is_new_iqwr - Is this a new Ingress queue entry ?
1103  * @q: Ingress quueue.
1104  * @ftr: Ingress queue WR SGE footer.
1105  *
1106  * The entry is new if our generation bit matches the corresponding
1107  * bit in the footer of the current WR.
1108  */
1109 static inline bool
1110 csio_is_new_iqwr(struct csio_q *q, struct csio_iqwr_footer *ftr)
1111 {
1112 	return (q->un.iq.genbit == (ftr->u.type_gen >> IQWRF_GEN_SHIFT));
1113 }
1114 
1115 /*
1116  * csio_wr_process_iq - Process elements in Ingress queue.
1117  * @hw:  HW pointer
1118  * @qidx: Index of queue
1119  * @iq_handler: Handler for this queue
1120  * @priv: Caller's private pointer
1121  *
1122  * This routine walks through every entry of the ingress queue, calling
1123  * the provided iq_handler with the entry, until the generation bit
1124  * flips.
1125  */
1126 int
1127 csio_wr_process_iq(struct csio_hw *hw, struct csio_q *q,
1128 		   void (*iq_handler)(struct csio_hw *, void *,
1129 				      uint32_t, struct csio_fl_dma_buf *,
1130 				      void *),
1131 		   void *priv)
1132 {
1133 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1134 	void *wr = (void *)((uintptr_t)q->vstart + (q->cidx * q->wr_sz));
1135 	struct csio_iqwr_footer *ftr;
1136 	uint32_t wr_type, fw_qid, qid;
1137 	struct csio_q *q_completed;
1138 	struct csio_q *flq = csio_iq_has_fl(q) ?
1139 					wrm->q_arr[q->un.iq.flq_idx] : NULL;
1140 	int rv = 0;
1141 
1142 	/* Get the footer */
1143 	ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
1144 					  (q->wr_sz - sizeof(*ftr)));
1145 
1146 	/*
1147 	 * When q wrapped around last time, driver should have inverted
1148 	 * ic.genbit as well.
1149 	 */
1150 	while (csio_is_new_iqwr(q, ftr)) {
1151 
1152 		CSIO_DB_ASSERT(((uintptr_t)wr + q->wr_sz) <=
1153 						(uintptr_t)q->vwrap);
1154 		rmb();
1155 		wr_type = IQWRF_TYPE_GET(ftr->u.type_gen);
1156 
1157 		switch (wr_type) {
1158 		case X_RSPD_TYPE_CPL:
1159 			/* Subtract footer from WR len */
1160 			iq_handler(hw, wr, q->wr_sz - sizeof(*ftr), NULL, priv);
1161 			break;
1162 		case X_RSPD_TYPE_FLBUF:
1163 			csio_wr_process_fl(hw, q, wr,
1164 					   ntohl(ftr->pldbuflen_qid),
1165 					   iq_handler, priv);
1166 			break;
1167 		case X_RSPD_TYPE_INTR:
1168 			fw_qid = ntohl(ftr->pldbuflen_qid);
1169 			qid = fw_qid - wrm->fw_iq_start;
1170 			q_completed = hw->wrm.intr_map[qid];
1171 
1172 			if (unlikely(qid ==
1173 					csio_q_physiqid(hw, hw->intr_iq_idx))) {
1174 				/*
1175 				 * We are already in the Forward Interrupt
1176 				 * Interrupt Queue Service! Do-not service
1177 				 * again!
1178 				 *
1179 				 */
1180 			} else {
1181 				CSIO_DB_ASSERT(q_completed);
1182 				CSIO_DB_ASSERT(
1183 					q_completed->un.iq.iq_intx_handler);
1184 
1185 				/* Call the queue handler. */
1186 				q_completed->un.iq.iq_intx_handler(hw, NULL,
1187 						0, NULL, (void *)q_completed);
1188 			}
1189 			break;
1190 		default:
1191 			csio_warn(hw, "Unknown resp type 0x%x received\n",
1192 				 wr_type);
1193 			CSIO_INC_STATS(q, n_rsp_unknown);
1194 			break;
1195 		}
1196 
1197 		/*
1198 		 * Ingress *always* has fixed size WR entries. Therefore,
1199 		 * there should always be complete WRs towards the end of
1200 		 * queue.
1201 		 */
1202 		if (((uintptr_t)wr + q->wr_sz) == (uintptr_t)q->vwrap) {
1203 
1204 			/* Roll over to start of queue */
1205 			q->cidx = 0;
1206 			wr	= q->vstart;
1207 
1208 			/* Toggle genbit */
1209 			q->un.iq.genbit ^= 0x1;
1210 
1211 			CSIO_INC_STATS(q, n_qwrap);
1212 		} else {
1213 			q->cidx++;
1214 			wr	= (void *)((uintptr_t)(q->vstart) +
1215 					   (q->cidx * q->wr_sz));
1216 		}
1217 
1218 		ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
1219 						  (q->wr_sz - sizeof(*ftr)));
1220 		q->inc_idx++;
1221 
1222 	} /* while (q->un.iq.genbit == hdr->genbit) */
1223 
1224 	/*
1225 	 * We need to re-arm SGE interrupts in case we got a stray interrupt,
1226 	 * especially in msix mode. With INTx, this may be a common occurence.
1227 	 */
1228 	if (unlikely(!q->inc_idx)) {
1229 		CSIO_INC_STATS(q, n_stray_comp);
1230 		rv = -EINVAL;
1231 		goto restart;
1232 	}
1233 
1234 	/* Replenish free list buffers if pending falls below low water mark */
1235 	if (flq) {
1236 		uint32_t avail  = csio_wr_avail_qcredits(flq);
1237 		if (avail <= 16) {
1238 			/* Make sure in FLQ, atleast 1 credit (8 FL buffers)
1239 			 * remains unpopulated otherwise HW thinks
1240 			 * FLQ is empty.
1241 			 */
1242 			csio_wr_update_fl(hw, flq, (flq->credits - 8) - avail);
1243 			csio_wr_ring_fldb(hw, flq);
1244 		}
1245 	}
1246 
1247 restart:
1248 	/* Now inform SGE about our incremental index value */
1249 	csio_wr_reg32(hw, CIDXINC_V(q->inc_idx)		|
1250 			  INGRESSQID_V(q->un.iq.physiqid)	|
1251 			  TIMERREG_V(csio_sge_timer_reg),
1252 			  MYPF_REG(SGE_PF_GTS_A));
1253 	q->stats.n_tot_rsps += q->inc_idx;
1254 
1255 	q->inc_idx = 0;
1256 
1257 	return rv;
1258 }
1259 
1260 int
1261 csio_wr_process_iq_idx(struct csio_hw *hw, int qidx,
1262 		   void (*iq_handler)(struct csio_hw *, void *,
1263 				      uint32_t, struct csio_fl_dma_buf *,
1264 				      void *),
1265 		   void *priv)
1266 {
1267 	struct csio_wrm *wrm	= csio_hw_to_wrm(hw);
1268 	struct csio_q	*iq	= wrm->q_arr[qidx];
1269 
1270 	return csio_wr_process_iq(hw, iq, iq_handler, priv);
1271 }
1272 
1273 static int
1274 csio_closest_timer(struct csio_sge *s, int time)
1275 {
1276 	int i, delta, match = 0, min_delta = INT_MAX;
1277 
1278 	for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1279 		delta = time - s->timer_val[i];
1280 		if (delta < 0)
1281 			delta = -delta;
1282 		if (delta < min_delta) {
1283 			min_delta = delta;
1284 			match = i;
1285 		}
1286 	}
1287 	return match;
1288 }
1289 
1290 static int
1291 csio_closest_thresh(struct csio_sge *s, int cnt)
1292 {
1293 	int i, delta, match = 0, min_delta = INT_MAX;
1294 
1295 	for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1296 		delta = cnt - s->counter_val[i];
1297 		if (delta < 0)
1298 			delta = -delta;
1299 		if (delta < min_delta) {
1300 			min_delta = delta;
1301 			match = i;
1302 		}
1303 	}
1304 	return match;
1305 }
1306 
1307 static void
1308 csio_wr_fixup_host_params(struct csio_hw *hw)
1309 {
1310 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1311 	struct csio_sge *sge = &wrm->sge;
1312 	uint32_t clsz = L1_CACHE_BYTES;
1313 	uint32_t s_hps = PAGE_SHIFT - 10;
1314 	uint32_t stat_len = clsz > 64 ? 128 : 64;
1315 	u32 fl_align = clsz < 32 ? 32 : clsz;
1316 	u32 pack_align;
1317 	u32 ingpad, ingpack;
1318 	int pcie_cap;
1319 
1320 	csio_wr_reg32(hw, HOSTPAGESIZEPF0_V(s_hps) | HOSTPAGESIZEPF1_V(s_hps) |
1321 		      HOSTPAGESIZEPF2_V(s_hps) | HOSTPAGESIZEPF3_V(s_hps) |
1322 		      HOSTPAGESIZEPF4_V(s_hps) | HOSTPAGESIZEPF5_V(s_hps) |
1323 		      HOSTPAGESIZEPF6_V(s_hps) | HOSTPAGESIZEPF7_V(s_hps),
1324 		      SGE_HOST_PAGE_SIZE_A);
1325 
1326 	/* T5 introduced the separation of the Free List Padding and
1327 	 * Packing Boundaries.  Thus, we can select a smaller Padding
1328 	 * Boundary to avoid uselessly chewing up PCIe Link and Memory
1329 	 * Bandwidth, and use a Packing Boundary which is large enough
1330 	 * to avoid false sharing between CPUs, etc.
1331 	 *
1332 	 * For the PCI Link, the smaller the Padding Boundary the
1333 	 * better.  For the Memory Controller, a smaller Padding
1334 	 * Boundary is better until we cross under the Memory Line
1335 	 * Size (the minimum unit of transfer to/from Memory).  If we
1336 	 * have a Padding Boundary which is smaller than the Memory
1337 	 * Line Size, that'll involve a Read-Modify-Write cycle on the
1338 	 * Memory Controller which is never good.
1339 	 */
1340 
1341 	/* We want the Packing Boundary to be based on the Cache Line
1342 	 * Size in order to help avoid False Sharing performance
1343 	 * issues between CPUs, etc.  We also want the Packing
1344 	 * Boundary to incorporate the PCI-E Maximum Payload Size.  We
1345 	 * get best performance when the Packing Boundary is a
1346 	 * multiple of the Maximum Payload Size.
1347 	 */
1348 	pack_align = fl_align;
1349 	pcie_cap = pci_find_capability(hw->pdev, PCI_CAP_ID_EXP);
1350 	if (pcie_cap) {
1351 		u32 mps, mps_log;
1352 		u16 devctl;
1353 
1354 		/* The PCIe Device Control Maximum Payload Size field
1355 		 * [bits 7:5] encodes sizes as powers of 2 starting at
1356 		 * 128 bytes.
1357 		 */
1358 		pci_read_config_word(hw->pdev,
1359 				     pcie_cap + PCI_EXP_DEVCTL,
1360 				     &devctl);
1361 		mps_log = ((devctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5) + 7;
1362 		mps = 1 << mps_log;
1363 		if (mps > pack_align)
1364 			pack_align = mps;
1365 	}
1366 
1367 	/* T5/T6 have a special interpretation of the "0"
1368 	 * value for the Packing Boundary.  This corresponds to 16
1369 	 * bytes instead of the expected 32 bytes.
1370 	 */
1371 	if (pack_align <= 16) {
1372 		ingpack = INGPACKBOUNDARY_16B_X;
1373 		fl_align = 16;
1374 	} else if (pack_align == 32) {
1375 		ingpack = INGPACKBOUNDARY_64B_X;
1376 		fl_align = 64;
1377 	} else {
1378 		u32 pack_align_log = fls(pack_align) - 1;
1379 
1380 		ingpack = pack_align_log - INGPACKBOUNDARY_SHIFT_X;
1381 		fl_align = pack_align;
1382 	}
1383 
1384 	/* Use the smallest Ingress Padding which isn't smaller than
1385 	 * the Memory Controller Read/Write Size.  We'll take that as
1386 	 * being 8 bytes since we don't know of any system with a
1387 	 * wider Memory Controller Bus Width.
1388 	 */
1389 	if (csio_is_t5(hw->pdev->device & CSIO_HW_CHIP_MASK))
1390 		ingpad = INGPADBOUNDARY_32B_X;
1391 	else
1392 		ingpad = T6_INGPADBOUNDARY_8B_X;
1393 
1394 	csio_set_reg_field(hw, SGE_CONTROL_A,
1395 			   INGPADBOUNDARY_V(INGPADBOUNDARY_M) |
1396 			   EGRSTATUSPAGESIZE_F,
1397 			   INGPADBOUNDARY_V(ingpad) |
1398 			   EGRSTATUSPAGESIZE_V(stat_len != 64));
1399 	csio_set_reg_field(hw, SGE_CONTROL2_A,
1400 			   INGPACKBOUNDARY_V(INGPACKBOUNDARY_M),
1401 			   INGPACKBOUNDARY_V(ingpack));
1402 
1403 	/* FL BUFFER SIZE#0 is Page size i,e already aligned to cache line */
1404 	csio_wr_reg32(hw, PAGE_SIZE, SGE_FL_BUFFER_SIZE0_A);
1405 
1406 	/*
1407 	 * If using hard params, the following will get set correctly
1408 	 * in csio_wr_set_sge().
1409 	 */
1410 	if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS) {
1411 		csio_wr_reg32(hw,
1412 			(csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE2_A) +
1413 			fl_align - 1) & ~(fl_align - 1),
1414 			SGE_FL_BUFFER_SIZE2_A);
1415 		csio_wr_reg32(hw,
1416 			(csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE3_A) +
1417 			fl_align - 1) & ~(fl_align - 1),
1418 			SGE_FL_BUFFER_SIZE3_A);
1419 	}
1420 
1421 	sge->csio_fl_align = fl_align;
1422 
1423 	csio_wr_reg32(hw, HPZ0_V(PAGE_SHIFT - 12), ULP_RX_TDDP_PSZ_A);
1424 
1425 	/* default value of rx_dma_offset of the NIC driver */
1426 	csio_set_reg_field(hw, SGE_CONTROL_A,
1427 			   PKTSHIFT_V(PKTSHIFT_M),
1428 			   PKTSHIFT_V(CSIO_SGE_RX_DMA_OFFSET));
1429 
1430 	csio_hw_tp_wr_bits_indirect(hw, TP_INGRESS_CONFIG_A,
1431 				    CSUM_HAS_PSEUDO_HDR_F, 0);
1432 }
1433 
1434 static void
1435 csio_init_intr_coalesce_parms(struct csio_hw *hw)
1436 {
1437 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1438 	struct csio_sge *sge = &wrm->sge;
1439 
1440 	csio_sge_thresh_reg = csio_closest_thresh(sge, csio_intr_coalesce_cnt);
1441 	if (csio_intr_coalesce_cnt) {
1442 		csio_sge_thresh_reg = 0;
1443 		csio_sge_timer_reg = X_TIMERREG_RESTART_COUNTER;
1444 		return;
1445 	}
1446 
1447 	csio_sge_timer_reg = csio_closest_timer(sge, csio_intr_coalesce_time);
1448 }
1449 
1450 /*
1451  * csio_wr_get_sge - Get SGE register values.
1452  * @hw: HW module.
1453  *
1454  * Used by non-master functions and by master-functions relying on config file.
1455  */
1456 static void
1457 csio_wr_get_sge(struct csio_hw *hw)
1458 {
1459 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1460 	struct csio_sge *sge = &wrm->sge;
1461 	uint32_t ingpad;
1462 	int i;
1463 	u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5;
1464 	u32 ingress_rx_threshold;
1465 
1466 	sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A);
1467 
1468 	ingpad = INGPADBOUNDARY_G(sge->sge_control);
1469 
1470 	switch (ingpad) {
1471 	case X_INGPCIEBOUNDARY_32B:
1472 		sge->csio_fl_align = 32; break;
1473 	case X_INGPCIEBOUNDARY_64B:
1474 		sge->csio_fl_align = 64; break;
1475 	case X_INGPCIEBOUNDARY_128B:
1476 		sge->csio_fl_align = 128; break;
1477 	case X_INGPCIEBOUNDARY_256B:
1478 		sge->csio_fl_align = 256; break;
1479 	case X_INGPCIEBOUNDARY_512B:
1480 		sge->csio_fl_align = 512; break;
1481 	case X_INGPCIEBOUNDARY_1024B:
1482 		sge->csio_fl_align = 1024; break;
1483 	case X_INGPCIEBOUNDARY_2048B:
1484 		sge->csio_fl_align = 2048; break;
1485 	case X_INGPCIEBOUNDARY_4096B:
1486 		sge->csio_fl_align = 4096; break;
1487 	}
1488 
1489 	for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)
1490 		csio_get_flbuf_size(hw, sge, i);
1491 
1492 	timer_value_0_and_1 = csio_rd_reg32(hw, SGE_TIMER_VALUE_0_AND_1_A);
1493 	timer_value_2_and_3 = csio_rd_reg32(hw, SGE_TIMER_VALUE_2_AND_3_A);
1494 	timer_value_4_and_5 = csio_rd_reg32(hw, SGE_TIMER_VALUE_4_AND_5_A);
1495 
1496 	sge->timer_val[0] = (uint16_t)csio_core_ticks_to_us(hw,
1497 					TIMERVALUE0_G(timer_value_0_and_1));
1498 	sge->timer_val[1] = (uint16_t)csio_core_ticks_to_us(hw,
1499 					TIMERVALUE1_G(timer_value_0_and_1));
1500 	sge->timer_val[2] = (uint16_t)csio_core_ticks_to_us(hw,
1501 					TIMERVALUE2_G(timer_value_2_and_3));
1502 	sge->timer_val[3] = (uint16_t)csio_core_ticks_to_us(hw,
1503 					TIMERVALUE3_G(timer_value_2_and_3));
1504 	sge->timer_val[4] = (uint16_t)csio_core_ticks_to_us(hw,
1505 					TIMERVALUE4_G(timer_value_4_and_5));
1506 	sge->timer_val[5] = (uint16_t)csio_core_ticks_to_us(hw,
1507 					TIMERVALUE5_G(timer_value_4_and_5));
1508 
1509 	ingress_rx_threshold = csio_rd_reg32(hw, SGE_INGRESS_RX_THRESHOLD_A);
1510 	sge->counter_val[0] = THRESHOLD_0_G(ingress_rx_threshold);
1511 	sge->counter_val[1] = THRESHOLD_1_G(ingress_rx_threshold);
1512 	sge->counter_val[2] = THRESHOLD_2_G(ingress_rx_threshold);
1513 	sge->counter_val[3] = THRESHOLD_3_G(ingress_rx_threshold);
1514 
1515 	csio_init_intr_coalesce_parms(hw);
1516 }
1517 
1518 /*
1519  * csio_wr_set_sge - Initialize SGE registers
1520  * @hw: HW module.
1521  *
1522  * Used by Master function to initialize SGE registers in the absence
1523  * of a config file.
1524  */
1525 static void
1526 csio_wr_set_sge(struct csio_hw *hw)
1527 {
1528 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1529 	struct csio_sge *sge = &wrm->sge;
1530 	int i;
1531 
1532 	/*
1533 	 * Set up our basic SGE mode to deliver CPL messages to our Ingress
1534 	 * Queue and Packet Date to the Free List.
1535 	 */
1536 	csio_set_reg_field(hw, SGE_CONTROL_A, RXPKTCPLMODE_F, RXPKTCPLMODE_F);
1537 
1538 	sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A);
1539 
1540 	/* sge->csio_fl_align is set up by csio_wr_fixup_host_params(). */
1541 
1542 	/*
1543 	 * Set up to drop DOORBELL writes when the DOORBELL FIFO overflows
1544 	 * and generate an interrupt when this occurs so we can recover.
1545 	 */
1546 	csio_set_reg_field(hw, SGE_DBFIFO_STATUS_A,
1547 			   LP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M),
1548 			   LP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH));
1549 	csio_set_reg_field(hw, SGE_DBFIFO_STATUS2_A,
1550 			   HP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M),
1551 			   HP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH));
1552 
1553 	csio_set_reg_field(hw, SGE_DOORBELL_CONTROL_A, ENABLE_DROP_F,
1554 			   ENABLE_DROP_F);
1555 
1556 	/* SGE_FL_BUFFER_SIZE0 is set up by csio_wr_fixup_host_params(). */
1557 
1558 	CSIO_SET_FLBUF_SIZE(hw, 1, CSIO_SGE_FLBUF_SIZE1);
1559 	csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE2 + sge->csio_fl_align - 1)
1560 		      & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE2_A);
1561 	csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE3 + sge->csio_fl_align - 1)
1562 		      & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE3_A);
1563 	CSIO_SET_FLBUF_SIZE(hw, 4, CSIO_SGE_FLBUF_SIZE4);
1564 	CSIO_SET_FLBUF_SIZE(hw, 5, CSIO_SGE_FLBUF_SIZE5);
1565 	CSIO_SET_FLBUF_SIZE(hw, 6, CSIO_SGE_FLBUF_SIZE6);
1566 	CSIO_SET_FLBUF_SIZE(hw, 7, CSIO_SGE_FLBUF_SIZE7);
1567 	CSIO_SET_FLBUF_SIZE(hw, 8, CSIO_SGE_FLBUF_SIZE8);
1568 
1569 	for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)
1570 		csio_get_flbuf_size(hw, sge, i);
1571 
1572 	/* Initialize interrupt coalescing attributes */
1573 	sge->timer_val[0] = CSIO_SGE_TIMER_VAL_0;
1574 	sge->timer_val[1] = CSIO_SGE_TIMER_VAL_1;
1575 	sge->timer_val[2] = CSIO_SGE_TIMER_VAL_2;
1576 	sge->timer_val[3] = CSIO_SGE_TIMER_VAL_3;
1577 	sge->timer_val[4] = CSIO_SGE_TIMER_VAL_4;
1578 	sge->timer_val[5] = CSIO_SGE_TIMER_VAL_5;
1579 
1580 	sge->counter_val[0] = CSIO_SGE_INT_CNT_VAL_0;
1581 	sge->counter_val[1] = CSIO_SGE_INT_CNT_VAL_1;
1582 	sge->counter_val[2] = CSIO_SGE_INT_CNT_VAL_2;
1583 	sge->counter_val[3] = CSIO_SGE_INT_CNT_VAL_3;
1584 
1585 	csio_wr_reg32(hw, THRESHOLD_0_V(sge->counter_val[0]) |
1586 		      THRESHOLD_1_V(sge->counter_val[1]) |
1587 		      THRESHOLD_2_V(sge->counter_val[2]) |
1588 		      THRESHOLD_3_V(sge->counter_val[3]),
1589 		      SGE_INGRESS_RX_THRESHOLD_A);
1590 
1591 	csio_wr_reg32(hw,
1592 		   TIMERVALUE0_V(csio_us_to_core_ticks(hw, sge->timer_val[0])) |
1593 		   TIMERVALUE1_V(csio_us_to_core_ticks(hw, sge->timer_val[1])),
1594 		   SGE_TIMER_VALUE_0_AND_1_A);
1595 
1596 	csio_wr_reg32(hw,
1597 		   TIMERVALUE2_V(csio_us_to_core_ticks(hw, sge->timer_val[2])) |
1598 		   TIMERVALUE3_V(csio_us_to_core_ticks(hw, sge->timer_val[3])),
1599 		   SGE_TIMER_VALUE_2_AND_3_A);
1600 
1601 	csio_wr_reg32(hw,
1602 		   TIMERVALUE4_V(csio_us_to_core_ticks(hw, sge->timer_val[4])) |
1603 		   TIMERVALUE5_V(csio_us_to_core_ticks(hw, sge->timer_val[5])),
1604 		   SGE_TIMER_VALUE_4_AND_5_A);
1605 
1606 	csio_init_intr_coalesce_parms(hw);
1607 }
1608 
1609 void
1610 csio_wr_sge_init(struct csio_hw *hw)
1611 {
1612 	/*
1613 	 * If we are master and chip is not initialized:
1614 	 *    - If we plan to use the config file, we need to fixup some
1615 	 *      host specific registers, and read the rest of the SGE
1616 	 *      configuration.
1617 	 *    - If we dont plan to use the config file, we need to initialize
1618 	 *      SGE entirely, including fixing the host specific registers.
1619 	 * If we are master and chip is initialized, just read and work off of
1620 	 *	the already initialized SGE values.
1621 	 * If we arent the master, we are only allowed to read and work off of
1622 	 *      the already initialized SGE values.
1623 	 *
1624 	 * Therefore, before calling this function, we assume that the master-
1625 	 * ship of the card, state and whether to use config file or not, have
1626 	 * already been decided.
1627 	 */
1628 	if (csio_is_hw_master(hw)) {
1629 		if (hw->fw_state != CSIO_DEV_STATE_INIT)
1630 			csio_wr_fixup_host_params(hw);
1631 
1632 		if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS)
1633 			csio_wr_get_sge(hw);
1634 		else
1635 			csio_wr_set_sge(hw);
1636 	} else
1637 		csio_wr_get_sge(hw);
1638 }
1639 
1640 /*
1641  * csio_wrm_init - Initialize Work request module.
1642  * @wrm: WR module
1643  * @hw: HW pointer
1644  *
1645  * Allocates memory for an array of queue pointers starting at q_arr.
1646  */
1647 int
1648 csio_wrm_init(struct csio_wrm *wrm, struct csio_hw *hw)
1649 {
1650 	int i;
1651 
1652 	if (!wrm->num_q) {
1653 		csio_err(hw, "Num queues is not set\n");
1654 		return -EINVAL;
1655 	}
1656 
1657 	wrm->q_arr = kcalloc(wrm->num_q, sizeof(struct csio_q *), GFP_KERNEL);
1658 	if (!wrm->q_arr)
1659 		goto err;
1660 
1661 	for (i = 0; i < wrm->num_q; i++) {
1662 		wrm->q_arr[i] = kzalloc(sizeof(struct csio_q), GFP_KERNEL);
1663 		if (!wrm->q_arr[i]) {
1664 			while (--i >= 0)
1665 				kfree(wrm->q_arr[i]);
1666 			goto err_free_arr;
1667 		}
1668 	}
1669 	wrm->free_qidx	= 0;
1670 
1671 	return 0;
1672 
1673 err_free_arr:
1674 	kfree(wrm->q_arr);
1675 err:
1676 	return -ENOMEM;
1677 }
1678 
1679 /*
1680  * csio_wrm_exit - Initialize Work request module.
1681  * @wrm: WR module
1682  * @hw: HW module
1683  *
1684  * Uninitialize WR module. Free q_arr and pointers in it.
1685  * We have the additional job of freeing the DMA memory associated
1686  * with the queues.
1687  */
1688 void
1689 csio_wrm_exit(struct csio_wrm *wrm, struct csio_hw *hw)
1690 {
1691 	int i;
1692 	uint32_t j;
1693 	struct csio_q *q;
1694 	struct csio_dma_buf *buf;
1695 
1696 	for (i = 0; i < wrm->num_q; i++) {
1697 		q = wrm->q_arr[i];
1698 
1699 		if (wrm->free_qidx && (i < wrm->free_qidx)) {
1700 			if (q->type == CSIO_FREELIST) {
1701 				if (!q->un.fl.bufs)
1702 					continue;
1703 				for (j = 0; j < q->credits; j++) {
1704 					buf = &q->un.fl.bufs[j];
1705 					if (!buf->vaddr)
1706 						continue;
1707 					dma_free_coherent(&hw->pdev->dev,
1708 							buf->len, buf->vaddr,
1709 							buf->paddr);
1710 				}
1711 				kfree(q->un.fl.bufs);
1712 			}
1713 			dma_free_coherent(&hw->pdev->dev, q->size,
1714 					q->vstart, q->pstart);
1715 		}
1716 		kfree(q);
1717 	}
1718 
1719 	hw->flags &= ~CSIO_HWF_Q_MEM_ALLOCED;
1720 
1721 	kfree(wrm->q_arr);
1722 }
1723