xref: /openbmc/linux/drivers/scsi/csiostor/csio_wr.c (revision d7a3d85e)
1 /*
2  * This file is part of the Chelsio FCoE driver for Linux.
3  *
4  * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  */
34 
35 #include <linux/kernel.h>
36 #include <linux/string.h>
37 #include <linux/compiler.h>
38 #include <linux/slab.h>
39 #include <asm/page.h>
40 #include <linux/cache.h>
41 
42 #include "csio_hw.h"
43 #include "csio_wr.h"
44 #include "csio_mb.h"
45 #include "csio_defs.h"
46 
47 int csio_intr_coalesce_cnt;		/* value:SGE_INGRESS_RX_THRESHOLD[0] */
48 static int csio_sge_thresh_reg;		/* SGE_INGRESS_RX_THRESHOLD[0] */
49 
50 int csio_intr_coalesce_time = 10;	/* value:SGE_TIMER_VALUE_1 */
51 static int csio_sge_timer_reg = 1;
52 
53 #define CSIO_SET_FLBUF_SIZE(_hw, _reg, _val)				\
54 	csio_wr_reg32((_hw), (_val), SGE_FL_BUFFER_SIZE##_reg##_A)
55 
56 static void
57 csio_get_flbuf_size(struct csio_hw *hw, struct csio_sge *sge, uint32_t reg)
58 {
59 	sge->sge_fl_buf_size[reg] = csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE0_A +
60 							reg * sizeof(uint32_t));
61 }
62 
63 /* Free list buffer size */
64 static inline uint32_t
65 csio_wr_fl_bufsz(struct csio_sge *sge, struct csio_dma_buf *buf)
66 {
67 	return sge->sge_fl_buf_size[buf->paddr & 0xF];
68 }
69 
70 /* Size of the egress queue status page */
71 static inline uint32_t
72 csio_wr_qstat_pgsz(struct csio_hw *hw)
73 {
74 	return (hw->wrm.sge.sge_control & EGRSTATUSPAGESIZE_F) ?  128 : 64;
75 }
76 
77 /* Ring freelist doorbell */
78 static inline void
79 csio_wr_ring_fldb(struct csio_hw *hw, struct csio_q *flq)
80 {
81 	/*
82 	 * Ring the doorbell only when we have atleast CSIO_QCREDIT_SZ
83 	 * number of bytes in the freelist queue. This translates to atleast
84 	 * 8 freelist buffer pointers (since each pointer is 8 bytes).
85 	 */
86 	if (flq->inc_idx >= 8) {
87 		csio_wr_reg32(hw, DBPRIO_F | QID_V(flq->un.fl.flid) |
88 				  PIDX_T5_V(flq->inc_idx / 8) | DBTYPE_F,
89 				  MYPF_REG(SGE_PF_KDOORBELL_A));
90 		flq->inc_idx &= 7;
91 	}
92 }
93 
94 /* Write a 0 cidx increment value to enable SGE interrupts for this queue */
95 static void
96 csio_wr_sge_intr_enable(struct csio_hw *hw, uint16_t iqid)
97 {
98 	csio_wr_reg32(hw, CIDXINC_V(0)		|
99 			  INGRESSQID_V(iqid)	|
100 			  TIMERREG_V(X_TIMERREG_RESTART_COUNTER),
101 			  MYPF_REG(SGE_PF_GTS_A));
102 }
103 
104 /*
105  * csio_wr_fill_fl - Populate the FL buffers of a FL queue.
106  * @hw: HW module.
107  * @flq: Freelist queue.
108  *
109  * Fill up freelist buffer entries with buffers of size specified
110  * in the size register.
111  *
112  */
113 static int
114 csio_wr_fill_fl(struct csio_hw *hw, struct csio_q *flq)
115 {
116 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
117 	struct csio_sge *sge = &wrm->sge;
118 	__be64 *d = (__be64 *)(flq->vstart);
119 	struct csio_dma_buf *buf = &flq->un.fl.bufs[0];
120 	uint64_t paddr;
121 	int sreg = flq->un.fl.sreg;
122 	int n = flq->credits;
123 
124 	while (n--) {
125 		buf->len = sge->sge_fl_buf_size[sreg];
126 		buf->vaddr = pci_alloc_consistent(hw->pdev, buf->len,
127 						  &buf->paddr);
128 		if (!buf->vaddr) {
129 			csio_err(hw, "Could only fill %d buffers!\n", n + 1);
130 			return -ENOMEM;
131 		}
132 
133 		paddr = buf->paddr | (sreg & 0xF);
134 
135 		*d++ = cpu_to_be64(paddr);
136 		buf++;
137 	}
138 
139 	return 0;
140 }
141 
142 /*
143  * csio_wr_update_fl -
144  * @hw: HW module.
145  * @flq: Freelist queue.
146  *
147  *
148  */
149 static inline void
150 csio_wr_update_fl(struct csio_hw *hw, struct csio_q *flq, uint16_t n)
151 {
152 
153 	flq->inc_idx += n;
154 	flq->pidx += n;
155 	if (unlikely(flq->pidx >= flq->credits))
156 		flq->pidx -= (uint16_t)flq->credits;
157 
158 	CSIO_INC_STATS(flq, n_flq_refill);
159 }
160 
161 /*
162  * csio_wr_alloc_q - Allocate a WR queue and initialize it.
163  * @hw: HW module
164  * @qsize: Size of the queue in bytes
165  * @wrsize: Since of WR in this queue, if fixed.
166  * @type: Type of queue (Ingress/Egress/Freelist)
167  * @owner: Module that owns this queue.
168  * @nflb: Number of freelist buffers for FL.
169  * @sreg: What is the FL buffer size register?
170  * @iq_int_handler: Ingress queue handler in INTx mode.
171  *
172  * This function allocates and sets up a queue for the caller
173  * of size qsize, aligned at the required boundary. This is subject to
174  * be free entries being available in the queue array. If one is found,
175  * it is initialized with the allocated queue, marked as being used (owner),
176  * and a handle returned to the caller in form of the queue's index
177  * into the q_arr array.
178  * If user has indicated a freelist (by specifying nflb > 0), create
179  * another queue (with its own index into q_arr) for the freelist. Allocate
180  * memory for DMA buffer metadata (vaddr, len etc). Save off the freelist
181  * idx in the ingress queue's flq.idx. This is how a Freelist is associated
182  * with its owning ingress queue.
183  */
184 int
185 csio_wr_alloc_q(struct csio_hw *hw, uint32_t qsize, uint32_t wrsize,
186 		uint16_t type, void *owner, uint32_t nflb, int sreg,
187 		iq_handler_t iq_intx_handler)
188 {
189 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
190 	struct csio_q	*q, *flq;
191 	int		free_idx = wrm->free_qidx;
192 	int		ret_idx = free_idx;
193 	uint32_t	qsz;
194 	int flq_idx;
195 
196 	if (free_idx >= wrm->num_q) {
197 		csio_err(hw, "No more free queues.\n");
198 		return -1;
199 	}
200 
201 	switch (type) {
202 	case CSIO_EGRESS:
203 		qsz = ALIGN(qsize, CSIO_QCREDIT_SZ) + csio_wr_qstat_pgsz(hw);
204 		break;
205 	case CSIO_INGRESS:
206 		switch (wrsize) {
207 		case 16:
208 		case 32:
209 		case 64:
210 		case 128:
211 			break;
212 		default:
213 			csio_err(hw, "Invalid Ingress queue WR size:%d\n",
214 				    wrsize);
215 			return -1;
216 		}
217 
218 		/*
219 		 * Number of elements must be a multiple of 16
220 		 * So this includes status page size
221 		 */
222 		qsz = ALIGN(qsize/wrsize, 16) * wrsize;
223 
224 		break;
225 	case CSIO_FREELIST:
226 		qsz = ALIGN(qsize/wrsize, 8) * wrsize + csio_wr_qstat_pgsz(hw);
227 		break;
228 	default:
229 		csio_err(hw, "Invalid queue type: 0x%x\n", type);
230 		return -1;
231 	}
232 
233 	q = wrm->q_arr[free_idx];
234 
235 	q->vstart = pci_zalloc_consistent(hw->pdev, qsz, &q->pstart);
236 	if (!q->vstart) {
237 		csio_err(hw,
238 			 "Failed to allocate DMA memory for "
239 			 "queue at id: %d size: %d\n", free_idx, qsize);
240 		return -1;
241 	}
242 
243 	q->type		= type;
244 	q->owner	= owner;
245 	q->pidx		= q->cidx = q->inc_idx = 0;
246 	q->size		= qsz;
247 	q->wr_sz	= wrsize;	/* If using fixed size WRs */
248 
249 	wrm->free_qidx++;
250 
251 	if (type == CSIO_INGRESS) {
252 		/* Since queue area is set to zero */
253 		q->un.iq.genbit	= 1;
254 
255 		/*
256 		 * Ingress queue status page size is always the size of
257 		 * the ingress queue entry.
258 		 */
259 		q->credits	= (qsz - q->wr_sz) / q->wr_sz;
260 		q->vwrap	= (void *)((uintptr_t)(q->vstart) + qsz
261 							- q->wr_sz);
262 
263 		/* Allocate memory for FL if requested */
264 		if (nflb > 0) {
265 			flq_idx = csio_wr_alloc_q(hw, nflb * sizeof(__be64),
266 						  sizeof(__be64), CSIO_FREELIST,
267 						  owner, 0, sreg, NULL);
268 			if (flq_idx == -1) {
269 				csio_err(hw,
270 					 "Failed to allocate FL queue"
271 					 " for IQ idx:%d\n", free_idx);
272 				return -1;
273 			}
274 
275 			/* Associate the new FL with the Ingress quue */
276 			q->un.iq.flq_idx = flq_idx;
277 
278 			flq = wrm->q_arr[q->un.iq.flq_idx];
279 			flq->un.fl.bufs = kzalloc(flq->credits *
280 						  sizeof(struct csio_dma_buf),
281 						  GFP_KERNEL);
282 			if (!flq->un.fl.bufs) {
283 				csio_err(hw,
284 					 "Failed to allocate FL queue bufs"
285 					 " for IQ idx:%d\n", free_idx);
286 				return -1;
287 			}
288 
289 			flq->un.fl.packen = 0;
290 			flq->un.fl.offset = 0;
291 			flq->un.fl.sreg = sreg;
292 
293 			/* Fill up the free list buffers */
294 			if (csio_wr_fill_fl(hw, flq))
295 				return -1;
296 
297 			/*
298 			 * Make sure in a FLQ, atleast 1 credit (8 FL buffers)
299 			 * remains unpopulated,otherwise HW thinks
300 			 * FLQ is empty.
301 			 */
302 			flq->pidx = flq->inc_idx = flq->credits - 8;
303 		} else {
304 			q->un.iq.flq_idx = -1;
305 		}
306 
307 		/* Associate the IQ INTx handler. */
308 		q->un.iq.iq_intx_handler = iq_intx_handler;
309 
310 		csio_q_iqid(hw, ret_idx) = CSIO_MAX_QID;
311 
312 	} else if (type == CSIO_EGRESS) {
313 		q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / CSIO_QCREDIT_SZ;
314 		q->vwrap   = (void *)((uintptr_t)(q->vstart) + qsz
315 						- csio_wr_qstat_pgsz(hw));
316 		csio_q_eqid(hw, ret_idx) = CSIO_MAX_QID;
317 	} else { /* Freelist */
318 		q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / sizeof(__be64);
319 		q->vwrap   = (void *)((uintptr_t)(q->vstart) + qsz
320 						- csio_wr_qstat_pgsz(hw));
321 		csio_q_flid(hw, ret_idx) = CSIO_MAX_QID;
322 	}
323 
324 	return ret_idx;
325 }
326 
327 /*
328  * csio_wr_iq_create_rsp - Response handler for IQ creation.
329  * @hw: The HW module.
330  * @mbp: Mailbox.
331  * @iq_idx: Ingress queue that got created.
332  *
333  * Handle FW_IQ_CMD mailbox completion. Save off the assigned IQ/FL ids.
334  */
335 static int
336 csio_wr_iq_create_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)
337 {
338 	struct csio_iq_params iqp;
339 	enum fw_retval retval;
340 	uint32_t iq_id;
341 	int flq_idx;
342 
343 	memset(&iqp, 0, sizeof(struct csio_iq_params));
344 
345 	csio_mb_iq_alloc_write_rsp(hw, mbp, &retval, &iqp);
346 
347 	if (retval != FW_SUCCESS) {
348 		csio_err(hw, "IQ cmd returned 0x%x!\n", retval);
349 		mempool_free(mbp, hw->mb_mempool);
350 		return -EINVAL;
351 	}
352 
353 	csio_q_iqid(hw, iq_idx)		= iqp.iqid;
354 	csio_q_physiqid(hw, iq_idx)	= iqp.physiqid;
355 	csio_q_pidx(hw, iq_idx)		= csio_q_cidx(hw, iq_idx) = 0;
356 	csio_q_inc_idx(hw, iq_idx)	= 0;
357 
358 	/* Actual iq-id. */
359 	iq_id = iqp.iqid - hw->wrm.fw_iq_start;
360 
361 	/* Set the iq-id to iq map table. */
362 	if (iq_id >= CSIO_MAX_IQ) {
363 		csio_err(hw,
364 			 "Exceeding MAX_IQ(%d) supported!"
365 			 " iqid:%d rel_iqid:%d FW iq_start:%d\n",
366 			 CSIO_MAX_IQ, iq_id, iqp.iqid, hw->wrm.fw_iq_start);
367 		mempool_free(mbp, hw->mb_mempool);
368 		return -EINVAL;
369 	}
370 	csio_q_set_intr_map(hw, iq_idx, iq_id);
371 
372 	/*
373 	 * During FW_IQ_CMD, FW sets interrupt_sent bit to 1 in the SGE
374 	 * ingress context of this queue. This will block interrupts to
375 	 * this queue until the next GTS write. Therefore, we do a
376 	 * 0-cidx increment GTS write for this queue just to clear the
377 	 * interrupt_sent bit. This will re-enable interrupts to this
378 	 * queue.
379 	 */
380 	csio_wr_sge_intr_enable(hw, iqp.physiqid);
381 
382 	flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
383 	if (flq_idx != -1) {
384 		struct csio_q *flq = hw->wrm.q_arr[flq_idx];
385 
386 		csio_q_flid(hw, flq_idx) = iqp.fl0id;
387 		csio_q_cidx(hw, flq_idx) = 0;
388 		csio_q_pidx(hw, flq_idx)    = csio_q_credits(hw, flq_idx) - 8;
389 		csio_q_inc_idx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8;
390 
391 		/* Now update SGE about the buffers allocated during init */
392 		csio_wr_ring_fldb(hw, flq);
393 	}
394 
395 	mempool_free(mbp, hw->mb_mempool);
396 
397 	return 0;
398 }
399 
400 /*
401  * csio_wr_iq_create - Configure an Ingress queue with FW.
402  * @hw: The HW module.
403  * @priv: Private data object.
404  * @iq_idx: Ingress queue index in the WR module.
405  * @vec: MSIX vector.
406  * @portid: PCIE Channel to be associated with this queue.
407  * @async: Is this a FW asynchronous message handling queue?
408  * @cbfn: Completion callback.
409  *
410  * This API configures an ingress queue with FW by issuing a FW_IQ_CMD mailbox
411  * with alloc/write bits set.
412  */
413 int
414 csio_wr_iq_create(struct csio_hw *hw, void *priv, int iq_idx,
415 		  uint32_t vec, uint8_t portid, bool async,
416 		  void (*cbfn) (struct csio_hw *, struct csio_mb *))
417 {
418 	struct csio_mb  *mbp;
419 	struct csio_iq_params iqp;
420 	int flq_idx;
421 
422 	memset(&iqp, 0, sizeof(struct csio_iq_params));
423 	csio_q_portid(hw, iq_idx) = portid;
424 
425 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
426 	if (!mbp) {
427 		csio_err(hw, "IQ command out of memory!\n");
428 		return -ENOMEM;
429 	}
430 
431 	switch (hw->intr_mode) {
432 	case CSIO_IM_INTX:
433 	case CSIO_IM_MSI:
434 		/* For interrupt forwarding queue only */
435 		if (hw->intr_iq_idx == iq_idx)
436 			iqp.iqandst	= X_INTERRUPTDESTINATION_PCIE;
437 		else
438 			iqp.iqandst	= X_INTERRUPTDESTINATION_IQ;
439 		iqp.iqandstindex	=
440 			csio_q_physiqid(hw, hw->intr_iq_idx);
441 		break;
442 	case CSIO_IM_MSIX:
443 		iqp.iqandst		= X_INTERRUPTDESTINATION_PCIE;
444 		iqp.iqandstindex	= (uint16_t)vec;
445 		break;
446 	case CSIO_IM_NONE:
447 		mempool_free(mbp, hw->mb_mempool);
448 		return -EINVAL;
449 	}
450 
451 	/* Pass in the ingress queue cmd parameters */
452 	iqp.pfn			= hw->pfn;
453 	iqp.vfn			= 0;
454 	iqp.iq_start		= 1;
455 	iqp.viid		= 0;
456 	iqp.type		= FW_IQ_TYPE_FL_INT_CAP;
457 	iqp.iqasynch		= async;
458 	if (csio_intr_coalesce_cnt)
459 		iqp.iqanus	= X_UPDATESCHEDULING_COUNTER_OPTTIMER;
460 	else
461 		iqp.iqanus	= X_UPDATESCHEDULING_TIMER;
462 	iqp.iqanud		= X_UPDATEDELIVERY_INTERRUPT;
463 	iqp.iqpciech		= portid;
464 	iqp.iqintcntthresh	= (uint8_t)csio_sge_thresh_reg;
465 
466 	switch (csio_q_wr_sz(hw, iq_idx)) {
467 	case 16:
468 		iqp.iqesize = 0; break;
469 	case 32:
470 		iqp.iqesize = 1; break;
471 	case 64:
472 		iqp.iqesize = 2; break;
473 	case 128:
474 		iqp.iqesize = 3; break;
475 	}
476 
477 	iqp.iqsize		= csio_q_size(hw, iq_idx) /
478 						csio_q_wr_sz(hw, iq_idx);
479 	iqp.iqaddr		= csio_q_pstart(hw, iq_idx);
480 
481 	flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
482 	if (flq_idx != -1) {
483 		struct csio_q *flq = hw->wrm.q_arr[flq_idx];
484 
485 		iqp.fl0paden	= 1;
486 		iqp.fl0packen	= flq->un.fl.packen ? 1 : 0;
487 		iqp.fl0fbmin	= X_FETCHBURSTMIN_64B;
488 		iqp.fl0fbmax	= X_FETCHBURSTMAX_512B;
489 		iqp.fl0size	= csio_q_size(hw, flq_idx) / CSIO_QCREDIT_SZ;
490 		iqp.fl0addr	= csio_q_pstart(hw, flq_idx);
491 	}
492 
493 	csio_mb_iq_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);
494 
495 	if (csio_mb_issue(hw, mbp)) {
496 		csio_err(hw, "Issue of IQ cmd failed!\n");
497 		mempool_free(mbp, hw->mb_mempool);
498 		return -EINVAL;
499 	}
500 
501 	if (cbfn != NULL)
502 		return 0;
503 
504 	return csio_wr_iq_create_rsp(hw, mbp, iq_idx);
505 }
506 
507 /*
508  * csio_wr_eq_create_rsp - Response handler for EQ creation.
509  * @hw: The HW module.
510  * @mbp: Mailbox.
511  * @eq_idx: Egress queue that got created.
512  *
513  * Handle FW_EQ_OFLD_CMD mailbox completion. Save off the assigned EQ ids.
514  */
515 static int
516 csio_wr_eq_cfg_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)
517 {
518 	struct csio_eq_params eqp;
519 	enum fw_retval retval;
520 
521 	memset(&eqp, 0, sizeof(struct csio_eq_params));
522 
523 	csio_mb_eq_ofld_alloc_write_rsp(hw, mbp, &retval, &eqp);
524 
525 	if (retval != FW_SUCCESS) {
526 		csio_err(hw, "EQ OFLD cmd returned 0x%x!\n", retval);
527 		mempool_free(mbp, hw->mb_mempool);
528 		return -EINVAL;
529 	}
530 
531 	csio_q_eqid(hw, eq_idx)	= (uint16_t)eqp.eqid;
532 	csio_q_physeqid(hw, eq_idx) = (uint16_t)eqp.physeqid;
533 	csio_q_pidx(hw, eq_idx)	= csio_q_cidx(hw, eq_idx) = 0;
534 	csio_q_inc_idx(hw, eq_idx) = 0;
535 
536 	mempool_free(mbp, hw->mb_mempool);
537 
538 	return 0;
539 }
540 
541 /*
542  * csio_wr_eq_create - Configure an Egress queue with FW.
543  * @hw: HW module.
544  * @priv: Private data.
545  * @eq_idx: Egress queue index in the WR module.
546  * @iq_idx: Associated ingress queue index.
547  * @cbfn: Completion callback.
548  *
549  * This API configures a offload egress queue with FW by issuing a
550  * FW_EQ_OFLD_CMD  (with alloc + write ) mailbox.
551  */
552 int
553 csio_wr_eq_create(struct csio_hw *hw, void *priv, int eq_idx,
554 		  int iq_idx, uint8_t portid,
555 		  void (*cbfn) (struct csio_hw *, struct csio_mb *))
556 {
557 	struct csio_mb  *mbp;
558 	struct csio_eq_params eqp;
559 
560 	memset(&eqp, 0, sizeof(struct csio_eq_params));
561 
562 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
563 	if (!mbp) {
564 		csio_err(hw, "EQ command out of memory!\n");
565 		return -ENOMEM;
566 	}
567 
568 	eqp.pfn			= hw->pfn;
569 	eqp.vfn			= 0;
570 	eqp.eqstart		= 1;
571 	eqp.hostfcmode		= X_HOSTFCMODE_STATUS_PAGE;
572 	eqp.iqid		= csio_q_iqid(hw, iq_idx);
573 	eqp.fbmin		= X_FETCHBURSTMIN_64B;
574 	eqp.fbmax		= X_FETCHBURSTMAX_512B;
575 	eqp.cidxfthresh		= 0;
576 	eqp.pciechn		= portid;
577 	eqp.eqsize		= csio_q_size(hw, eq_idx) / CSIO_QCREDIT_SZ;
578 	eqp.eqaddr		= csio_q_pstart(hw, eq_idx);
579 
580 	csio_mb_eq_ofld_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO,
581 				    &eqp, cbfn);
582 
583 	if (csio_mb_issue(hw, mbp)) {
584 		csio_err(hw, "Issue of EQ OFLD cmd failed!\n");
585 		mempool_free(mbp, hw->mb_mempool);
586 		return -EINVAL;
587 	}
588 
589 	if (cbfn != NULL)
590 		return 0;
591 
592 	return csio_wr_eq_cfg_rsp(hw, mbp, eq_idx);
593 }
594 
595 /*
596  * csio_wr_iq_destroy_rsp - Response handler for IQ removal.
597  * @hw: The HW module.
598  * @mbp: Mailbox.
599  * @iq_idx: Ingress queue that was freed.
600  *
601  * Handle FW_IQ_CMD (free) mailbox completion.
602  */
603 static int
604 csio_wr_iq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)
605 {
606 	enum fw_retval retval = csio_mb_fw_retval(mbp);
607 	int rv = 0;
608 
609 	if (retval != FW_SUCCESS)
610 		rv = -EINVAL;
611 
612 	mempool_free(mbp, hw->mb_mempool);
613 
614 	return rv;
615 }
616 
617 /*
618  * csio_wr_iq_destroy - Free an ingress queue.
619  * @hw: The HW module.
620  * @priv: Private data object.
621  * @iq_idx: Ingress queue index to destroy
622  * @cbfn: Completion callback.
623  *
624  * This API frees an ingress queue by issuing the FW_IQ_CMD
625  * with the free bit set.
626  */
627 static int
628 csio_wr_iq_destroy(struct csio_hw *hw, void *priv, int iq_idx,
629 		   void (*cbfn)(struct csio_hw *, struct csio_mb *))
630 {
631 	int rv = 0;
632 	struct csio_mb  *mbp;
633 	struct csio_iq_params iqp;
634 	int flq_idx;
635 
636 	memset(&iqp, 0, sizeof(struct csio_iq_params));
637 
638 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
639 	if (!mbp)
640 		return -ENOMEM;
641 
642 	iqp.pfn		= hw->pfn;
643 	iqp.vfn		= 0;
644 	iqp.iqid	= csio_q_iqid(hw, iq_idx);
645 	iqp.type	= FW_IQ_TYPE_FL_INT_CAP;
646 
647 	flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
648 	if (flq_idx != -1)
649 		iqp.fl0id = csio_q_flid(hw, flq_idx);
650 	else
651 		iqp.fl0id = 0xFFFF;
652 
653 	iqp.fl1id = 0xFFFF;
654 
655 	csio_mb_iq_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);
656 
657 	rv = csio_mb_issue(hw, mbp);
658 	if (rv != 0) {
659 		mempool_free(mbp, hw->mb_mempool);
660 		return rv;
661 	}
662 
663 	if (cbfn != NULL)
664 		return 0;
665 
666 	return csio_wr_iq_destroy_rsp(hw, mbp, iq_idx);
667 }
668 
669 /*
670  * csio_wr_eq_destroy_rsp - Response handler for OFLD EQ creation.
671  * @hw: The HW module.
672  * @mbp: Mailbox.
673  * @eq_idx: Egress queue that was freed.
674  *
675  * Handle FW_OFLD_EQ_CMD (free) mailbox completion.
676  */
677 static int
678 csio_wr_eq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)
679 {
680 	enum fw_retval retval = csio_mb_fw_retval(mbp);
681 	int rv = 0;
682 
683 	if (retval != FW_SUCCESS)
684 		rv = -EINVAL;
685 
686 	mempool_free(mbp, hw->mb_mempool);
687 
688 	return rv;
689 }
690 
691 /*
692  * csio_wr_eq_destroy - Free an Egress queue.
693  * @hw: The HW module.
694  * @priv: Private data object.
695  * @eq_idx: Egress queue index to destroy
696  * @cbfn: Completion callback.
697  *
698  * This API frees an Egress queue by issuing the FW_EQ_OFLD_CMD
699  * with the free bit set.
700  */
701 static int
702 csio_wr_eq_destroy(struct csio_hw *hw, void *priv, int eq_idx,
703 		   void (*cbfn) (struct csio_hw *, struct csio_mb *))
704 {
705 	int rv = 0;
706 	struct csio_mb  *mbp;
707 	struct csio_eq_params eqp;
708 
709 	memset(&eqp, 0, sizeof(struct csio_eq_params));
710 
711 	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
712 	if (!mbp)
713 		return -ENOMEM;
714 
715 	eqp.pfn		= hw->pfn;
716 	eqp.vfn		= 0;
717 	eqp.eqid	= csio_q_eqid(hw, eq_idx);
718 
719 	csio_mb_eq_ofld_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &eqp, cbfn);
720 
721 	rv = csio_mb_issue(hw, mbp);
722 	if (rv != 0) {
723 		mempool_free(mbp, hw->mb_mempool);
724 		return rv;
725 	}
726 
727 	if (cbfn != NULL)
728 		return 0;
729 
730 	return csio_wr_eq_destroy_rsp(hw, mbp, eq_idx);
731 }
732 
733 /*
734  * csio_wr_cleanup_eq_stpg - Cleanup Egress queue status page
735  * @hw: HW module
736  * @qidx: Egress queue index
737  *
738  * Cleanup the Egress queue status page.
739  */
740 static void
741 csio_wr_cleanup_eq_stpg(struct csio_hw *hw, int qidx)
742 {
743 	struct csio_q	*q = csio_hw_to_wrm(hw)->q_arr[qidx];
744 	struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;
745 
746 	memset(stp, 0, sizeof(*stp));
747 }
748 
749 /*
750  * csio_wr_cleanup_iq_ftr - Cleanup Footer entries in IQ
751  * @hw: HW module
752  * @qidx: Ingress queue index
753  *
754  * Cleanup the footer entries in the given ingress queue,
755  * set to 1 the internal copy of genbit.
756  */
757 static void
758 csio_wr_cleanup_iq_ftr(struct csio_hw *hw, int qidx)
759 {
760 	struct csio_wrm *wrm	= csio_hw_to_wrm(hw);
761 	struct csio_q	*q	= wrm->q_arr[qidx];
762 	void *wr;
763 	struct csio_iqwr_footer *ftr;
764 	uint32_t i = 0;
765 
766 	/* set to 1 since we are just about zero out genbit */
767 	q->un.iq.genbit = 1;
768 
769 	for (i = 0; i < q->credits; i++) {
770 		/* Get the WR */
771 		wr = (void *)((uintptr_t)q->vstart +
772 					   (i * q->wr_sz));
773 		/* Get the footer */
774 		ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
775 					  (q->wr_sz - sizeof(*ftr)));
776 		/* Zero out footer */
777 		memset(ftr, 0, sizeof(*ftr));
778 	}
779 }
780 
781 int
782 csio_wr_destroy_queues(struct csio_hw *hw, bool cmd)
783 {
784 	int i, flq_idx;
785 	struct csio_q *q;
786 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
787 	int rv;
788 
789 	for (i = 0; i < wrm->free_qidx; i++) {
790 		q = wrm->q_arr[i];
791 
792 		switch (q->type) {
793 		case CSIO_EGRESS:
794 			if (csio_q_eqid(hw, i) != CSIO_MAX_QID) {
795 				csio_wr_cleanup_eq_stpg(hw, i);
796 				if (!cmd) {
797 					csio_q_eqid(hw, i) = CSIO_MAX_QID;
798 					continue;
799 				}
800 
801 				rv = csio_wr_eq_destroy(hw, NULL, i, NULL);
802 				if ((rv == -EBUSY) || (rv == -ETIMEDOUT))
803 					cmd = false;
804 
805 				csio_q_eqid(hw, i) = CSIO_MAX_QID;
806 			}
807 		case CSIO_INGRESS:
808 			if (csio_q_iqid(hw, i) != CSIO_MAX_QID) {
809 				csio_wr_cleanup_iq_ftr(hw, i);
810 				if (!cmd) {
811 					csio_q_iqid(hw, i) = CSIO_MAX_QID;
812 					flq_idx = csio_q_iq_flq_idx(hw, i);
813 					if (flq_idx != -1)
814 						csio_q_flid(hw, flq_idx) =
815 								CSIO_MAX_QID;
816 					continue;
817 				}
818 
819 				rv = csio_wr_iq_destroy(hw, NULL, i, NULL);
820 				if ((rv == -EBUSY) || (rv == -ETIMEDOUT))
821 					cmd = false;
822 
823 				csio_q_iqid(hw, i) = CSIO_MAX_QID;
824 				flq_idx = csio_q_iq_flq_idx(hw, i);
825 				if (flq_idx != -1)
826 					csio_q_flid(hw, flq_idx) = CSIO_MAX_QID;
827 			}
828 		default:
829 			break;
830 		}
831 	}
832 
833 	hw->flags &= ~CSIO_HWF_Q_FW_ALLOCED;
834 
835 	return 0;
836 }
837 
838 /*
839  * csio_wr_get - Get requested size of WR entry/entries from queue.
840  * @hw: HW module.
841  * @qidx: Index of queue.
842  * @size: Cumulative size of Work request(s).
843  * @wrp: Work request pair.
844  *
845  * If requested credits are available, return the start address of the
846  * work request in the work request pair. Set pidx accordingly and
847  * return.
848  *
849  * NOTE about WR pair:
850  * ==================
851  * A WR can start towards the end of a queue, and then continue at the
852  * beginning, since the queue is considered to be circular. This will
853  * require a pair of address/size to be passed back to the caller -
854  * hence Work request pair format.
855  */
856 int
857 csio_wr_get(struct csio_hw *hw, int qidx, uint32_t size,
858 	    struct csio_wr_pair *wrp)
859 {
860 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
861 	struct csio_q *q = wrm->q_arr[qidx];
862 	void *cwr = (void *)((uintptr_t)(q->vstart) +
863 						(q->pidx * CSIO_QCREDIT_SZ));
864 	struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;
865 	uint16_t cidx = q->cidx = ntohs(stp->cidx);
866 	uint16_t pidx = q->pidx;
867 	uint32_t req_sz	= ALIGN(size, CSIO_QCREDIT_SZ);
868 	int req_credits	= req_sz / CSIO_QCREDIT_SZ;
869 	int credits;
870 
871 	CSIO_DB_ASSERT(q->owner != NULL);
872 	CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));
873 	CSIO_DB_ASSERT(cidx <= q->credits);
874 
875 	/* Calculate credits */
876 	if (pidx > cidx) {
877 		credits = q->credits - (pidx - cidx) - 1;
878 	} else if (cidx > pidx) {
879 		credits = cidx - pidx - 1;
880 	} else {
881 		/* cidx == pidx, empty queue */
882 		credits = q->credits;
883 		CSIO_INC_STATS(q, n_qempty);
884 	}
885 
886 	/*
887 	 * Check if we have enough credits.
888 	 * credits = 1 implies queue is full.
889 	 */
890 	if (!credits || (req_credits > credits)) {
891 		CSIO_INC_STATS(q, n_qfull);
892 		return -EBUSY;
893 	}
894 
895 	/*
896 	 * If we are here, we have enough credits to satisfy the
897 	 * request. Check if we are near the end of q, and if WR spills over.
898 	 * If it does, use the first addr/size to cover the queue until
899 	 * the end. Fit the remainder portion of the request at the top
900 	 * of queue and return it in the second addr/len. Set pidx
901 	 * accordingly.
902 	 */
903 	if (unlikely(((uintptr_t)cwr + req_sz) > (uintptr_t)(q->vwrap))) {
904 		wrp->addr1 = cwr;
905 		wrp->size1 = (uint32_t)((uintptr_t)q->vwrap - (uintptr_t)cwr);
906 		wrp->addr2 = q->vstart;
907 		wrp->size2 = req_sz - wrp->size1;
908 		q->pidx	= (uint16_t)(ALIGN(wrp->size2, CSIO_QCREDIT_SZ) /
909 							CSIO_QCREDIT_SZ);
910 		CSIO_INC_STATS(q, n_qwrap);
911 		CSIO_INC_STATS(q, n_eq_wr_split);
912 	} else {
913 		wrp->addr1 = cwr;
914 		wrp->size1 = req_sz;
915 		wrp->addr2 = NULL;
916 		wrp->size2 = 0;
917 		q->pidx	+= (uint16_t)req_credits;
918 
919 		/* We are the end of queue, roll back pidx to top of queue */
920 		if (unlikely(q->pidx == q->credits)) {
921 			q->pidx = 0;
922 			CSIO_INC_STATS(q, n_qwrap);
923 		}
924 	}
925 
926 	q->inc_idx = (uint16_t)req_credits;
927 
928 	CSIO_INC_STATS(q, n_tot_reqs);
929 
930 	return 0;
931 }
932 
933 /*
934  * csio_wr_copy_to_wrp - Copies given data into WR.
935  * @data_buf - Data buffer
936  * @wrp - Work request pair.
937  * @wr_off - Work request offset.
938  * @data_len - Data length.
939  *
940  * Copies the given data in Work Request. Work request pair(wrp) specifies
941  * address information of Work request.
942  * Returns: none
943  */
944 void
945 csio_wr_copy_to_wrp(void *data_buf, struct csio_wr_pair *wrp,
946 		   uint32_t wr_off, uint32_t data_len)
947 {
948 	uint32_t nbytes;
949 
950 	/* Number of space available in buffer addr1 of WRP */
951 	nbytes = ((wrp->size1 - wr_off) >= data_len) ?
952 					data_len : (wrp->size1 - wr_off);
953 
954 	memcpy((uint8_t *) wrp->addr1 + wr_off, data_buf, nbytes);
955 	data_len -= nbytes;
956 
957 	/* Write the remaining data from the begining of circular buffer */
958 	if (data_len) {
959 		CSIO_DB_ASSERT(data_len <= wrp->size2);
960 		CSIO_DB_ASSERT(wrp->addr2 != NULL);
961 		memcpy(wrp->addr2, (uint8_t *) data_buf + nbytes, data_len);
962 	}
963 }
964 
965 /*
966  * csio_wr_issue - Notify chip of Work request.
967  * @hw: HW module.
968  * @qidx: Index of queue.
969  * @prio: 0: Low priority, 1: High priority
970  *
971  * Rings the SGE Doorbell by writing the current producer index of the passed
972  * in queue into the register.
973  *
974  */
975 int
976 csio_wr_issue(struct csio_hw *hw, int qidx, bool prio)
977 {
978 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
979 	struct csio_q *q = wrm->q_arr[qidx];
980 
981 	CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));
982 
983 	wmb();
984 	/* Ring SGE Doorbell writing q->pidx into it */
985 	csio_wr_reg32(hw, DBPRIO_V(prio) | QID_V(q->un.eq.physeqid) |
986 			  PIDX_T5_V(q->inc_idx) | DBTYPE_F,
987 			  MYPF_REG(SGE_PF_KDOORBELL_A));
988 	q->inc_idx = 0;
989 
990 	return 0;
991 }
992 
993 static inline uint32_t
994 csio_wr_avail_qcredits(struct csio_q *q)
995 {
996 	if (q->pidx > q->cidx)
997 		return q->pidx - q->cidx;
998 	else if (q->cidx > q->pidx)
999 		return q->credits - (q->cidx - q->pidx);
1000 	else
1001 		return 0;	/* cidx == pidx, empty queue */
1002 }
1003 
1004 /*
1005  * csio_wr_inval_flq_buf - Invalidate a free list buffer entry.
1006  * @hw: HW module.
1007  * @flq: The freelist queue.
1008  *
1009  * Invalidate the driver's version of a freelist buffer entry,
1010  * without freeing the associated the DMA memory. The entry
1011  * to be invalidated is picked up from the current Free list
1012  * queue cidx.
1013  *
1014  */
1015 static inline void
1016 csio_wr_inval_flq_buf(struct csio_hw *hw, struct csio_q *flq)
1017 {
1018 	flq->cidx++;
1019 	if (flq->cidx == flq->credits) {
1020 		flq->cidx = 0;
1021 		CSIO_INC_STATS(flq, n_qwrap);
1022 	}
1023 }
1024 
1025 /*
1026  * csio_wr_process_fl - Process a freelist completion.
1027  * @hw: HW module.
1028  * @q: The ingress queue attached to the Freelist.
1029  * @wr: The freelist completion WR in the ingress queue.
1030  * @len_to_qid: The lower 32-bits of the first flit of the RSP footer
1031  * @iq_handler: Caller's handler for this completion.
1032  * @priv: Private pointer of caller
1033  *
1034  */
1035 static inline void
1036 csio_wr_process_fl(struct csio_hw *hw, struct csio_q *q,
1037 		   void *wr, uint32_t len_to_qid,
1038 		   void (*iq_handler)(struct csio_hw *, void *,
1039 				      uint32_t, struct csio_fl_dma_buf *,
1040 				      void *),
1041 		   void *priv)
1042 {
1043 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1044 	struct csio_sge *sge = &wrm->sge;
1045 	struct csio_fl_dma_buf flb;
1046 	struct csio_dma_buf *buf, *fbuf;
1047 	uint32_t bufsz, len, lastlen = 0;
1048 	int n;
1049 	struct csio_q *flq = hw->wrm.q_arr[q->un.iq.flq_idx];
1050 
1051 	CSIO_DB_ASSERT(flq != NULL);
1052 
1053 	len = len_to_qid;
1054 
1055 	if (len & IQWRF_NEWBUF) {
1056 		if (flq->un.fl.offset > 0) {
1057 			csio_wr_inval_flq_buf(hw, flq);
1058 			flq->un.fl.offset = 0;
1059 		}
1060 		len = IQWRF_LEN_GET(len);
1061 	}
1062 
1063 	CSIO_DB_ASSERT(len != 0);
1064 
1065 	flb.totlen = len;
1066 
1067 	/* Consume all freelist buffers used for len bytes */
1068 	for (n = 0, fbuf = flb.flbufs; ; n++, fbuf++) {
1069 		buf = &flq->un.fl.bufs[flq->cidx];
1070 		bufsz = csio_wr_fl_bufsz(sge, buf);
1071 
1072 		fbuf->paddr	= buf->paddr;
1073 		fbuf->vaddr	= buf->vaddr;
1074 
1075 		flb.offset	= flq->un.fl.offset;
1076 		lastlen		= min(bufsz, len);
1077 		fbuf->len	= lastlen;
1078 
1079 		len -= lastlen;
1080 		if (!len)
1081 			break;
1082 		csio_wr_inval_flq_buf(hw, flq);
1083 	}
1084 
1085 	flb.defer_free = flq->un.fl.packen ? 0 : 1;
1086 
1087 	iq_handler(hw, wr, q->wr_sz - sizeof(struct csio_iqwr_footer),
1088 		   &flb, priv);
1089 
1090 	if (flq->un.fl.packen)
1091 		flq->un.fl.offset += ALIGN(lastlen, sge->csio_fl_align);
1092 	else
1093 		csio_wr_inval_flq_buf(hw, flq);
1094 
1095 }
1096 
1097 /*
1098  * csio_is_new_iqwr - Is this a new Ingress queue entry ?
1099  * @q: Ingress quueue.
1100  * @ftr: Ingress queue WR SGE footer.
1101  *
1102  * The entry is new if our generation bit matches the corresponding
1103  * bit in the footer of the current WR.
1104  */
1105 static inline bool
1106 csio_is_new_iqwr(struct csio_q *q, struct csio_iqwr_footer *ftr)
1107 {
1108 	return (q->un.iq.genbit == (ftr->u.type_gen >> IQWRF_GEN_SHIFT));
1109 }
1110 
1111 /*
1112  * csio_wr_process_iq - Process elements in Ingress queue.
1113  * @hw:  HW pointer
1114  * @qidx: Index of queue
1115  * @iq_handler: Handler for this queue
1116  * @priv: Caller's private pointer
1117  *
1118  * This routine walks through every entry of the ingress queue, calling
1119  * the provided iq_handler with the entry, until the generation bit
1120  * flips.
1121  */
1122 int
1123 csio_wr_process_iq(struct csio_hw *hw, struct csio_q *q,
1124 		   void (*iq_handler)(struct csio_hw *, void *,
1125 				      uint32_t, struct csio_fl_dma_buf *,
1126 				      void *),
1127 		   void *priv)
1128 {
1129 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1130 	void *wr = (void *)((uintptr_t)q->vstart + (q->cidx * q->wr_sz));
1131 	struct csio_iqwr_footer *ftr;
1132 	uint32_t wr_type, fw_qid, qid;
1133 	struct csio_q *q_completed;
1134 	struct csio_q *flq = csio_iq_has_fl(q) ?
1135 					wrm->q_arr[q->un.iq.flq_idx] : NULL;
1136 	int rv = 0;
1137 
1138 	/* Get the footer */
1139 	ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
1140 					  (q->wr_sz - sizeof(*ftr)));
1141 
1142 	/*
1143 	 * When q wrapped around last time, driver should have inverted
1144 	 * ic.genbit as well.
1145 	 */
1146 	while (csio_is_new_iqwr(q, ftr)) {
1147 
1148 		CSIO_DB_ASSERT(((uintptr_t)wr + q->wr_sz) <=
1149 						(uintptr_t)q->vwrap);
1150 		rmb();
1151 		wr_type = IQWRF_TYPE_GET(ftr->u.type_gen);
1152 
1153 		switch (wr_type) {
1154 		case X_RSPD_TYPE_CPL:
1155 			/* Subtract footer from WR len */
1156 			iq_handler(hw, wr, q->wr_sz - sizeof(*ftr), NULL, priv);
1157 			break;
1158 		case X_RSPD_TYPE_FLBUF:
1159 			csio_wr_process_fl(hw, q, wr,
1160 					   ntohl(ftr->pldbuflen_qid),
1161 					   iq_handler, priv);
1162 			break;
1163 		case X_RSPD_TYPE_INTR:
1164 			fw_qid = ntohl(ftr->pldbuflen_qid);
1165 			qid = fw_qid - wrm->fw_iq_start;
1166 			q_completed = hw->wrm.intr_map[qid];
1167 
1168 			if (unlikely(qid ==
1169 					csio_q_physiqid(hw, hw->intr_iq_idx))) {
1170 				/*
1171 				 * We are already in the Forward Interrupt
1172 				 * Interrupt Queue Service! Do-not service
1173 				 * again!
1174 				 *
1175 				 */
1176 			} else {
1177 				CSIO_DB_ASSERT(q_completed);
1178 				CSIO_DB_ASSERT(
1179 					q_completed->un.iq.iq_intx_handler);
1180 
1181 				/* Call the queue handler. */
1182 				q_completed->un.iq.iq_intx_handler(hw, NULL,
1183 						0, NULL, (void *)q_completed);
1184 			}
1185 			break;
1186 		default:
1187 			csio_warn(hw, "Unknown resp type 0x%x received\n",
1188 				 wr_type);
1189 			CSIO_INC_STATS(q, n_rsp_unknown);
1190 			break;
1191 		}
1192 
1193 		/*
1194 		 * Ingress *always* has fixed size WR entries. Therefore,
1195 		 * there should always be complete WRs towards the end of
1196 		 * queue.
1197 		 */
1198 		if (((uintptr_t)wr + q->wr_sz) == (uintptr_t)q->vwrap) {
1199 
1200 			/* Roll over to start of queue */
1201 			q->cidx = 0;
1202 			wr	= q->vstart;
1203 
1204 			/* Toggle genbit */
1205 			q->un.iq.genbit ^= 0x1;
1206 
1207 			CSIO_INC_STATS(q, n_qwrap);
1208 		} else {
1209 			q->cidx++;
1210 			wr	= (void *)((uintptr_t)(q->vstart) +
1211 					   (q->cidx * q->wr_sz));
1212 		}
1213 
1214 		ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
1215 						  (q->wr_sz - sizeof(*ftr)));
1216 		q->inc_idx++;
1217 
1218 	} /* while (q->un.iq.genbit == hdr->genbit) */
1219 
1220 	/*
1221 	 * We need to re-arm SGE interrupts in case we got a stray interrupt,
1222 	 * especially in msix mode. With INTx, this may be a common occurence.
1223 	 */
1224 	if (unlikely(!q->inc_idx)) {
1225 		CSIO_INC_STATS(q, n_stray_comp);
1226 		rv = -EINVAL;
1227 		goto restart;
1228 	}
1229 
1230 	/* Replenish free list buffers if pending falls below low water mark */
1231 	if (flq) {
1232 		uint32_t avail  = csio_wr_avail_qcredits(flq);
1233 		if (avail <= 16) {
1234 			/* Make sure in FLQ, atleast 1 credit (8 FL buffers)
1235 			 * remains unpopulated otherwise HW thinks
1236 			 * FLQ is empty.
1237 			 */
1238 			csio_wr_update_fl(hw, flq, (flq->credits - 8) - avail);
1239 			csio_wr_ring_fldb(hw, flq);
1240 		}
1241 	}
1242 
1243 restart:
1244 	/* Now inform SGE about our incremental index value */
1245 	csio_wr_reg32(hw, CIDXINC_V(q->inc_idx)		|
1246 			  INGRESSQID_V(q->un.iq.physiqid)	|
1247 			  TIMERREG_V(csio_sge_timer_reg),
1248 			  MYPF_REG(SGE_PF_GTS_A));
1249 	q->stats.n_tot_rsps += q->inc_idx;
1250 
1251 	q->inc_idx = 0;
1252 
1253 	return rv;
1254 }
1255 
1256 int
1257 csio_wr_process_iq_idx(struct csio_hw *hw, int qidx,
1258 		   void (*iq_handler)(struct csio_hw *, void *,
1259 				      uint32_t, struct csio_fl_dma_buf *,
1260 				      void *),
1261 		   void *priv)
1262 {
1263 	struct csio_wrm *wrm	= csio_hw_to_wrm(hw);
1264 	struct csio_q	*iq	= wrm->q_arr[qidx];
1265 
1266 	return csio_wr_process_iq(hw, iq, iq_handler, priv);
1267 }
1268 
1269 static int
1270 csio_closest_timer(struct csio_sge *s, int time)
1271 {
1272 	int i, delta, match = 0, min_delta = INT_MAX;
1273 
1274 	for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1275 		delta = time - s->timer_val[i];
1276 		if (delta < 0)
1277 			delta = -delta;
1278 		if (delta < min_delta) {
1279 			min_delta = delta;
1280 			match = i;
1281 		}
1282 	}
1283 	return match;
1284 }
1285 
1286 static int
1287 csio_closest_thresh(struct csio_sge *s, int cnt)
1288 {
1289 	int i, delta, match = 0, min_delta = INT_MAX;
1290 
1291 	for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1292 		delta = cnt - s->counter_val[i];
1293 		if (delta < 0)
1294 			delta = -delta;
1295 		if (delta < min_delta) {
1296 			min_delta = delta;
1297 			match = i;
1298 		}
1299 	}
1300 	return match;
1301 }
1302 
1303 static void
1304 csio_wr_fixup_host_params(struct csio_hw *hw)
1305 {
1306 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1307 	struct csio_sge *sge = &wrm->sge;
1308 	uint32_t clsz = L1_CACHE_BYTES;
1309 	uint32_t s_hps = PAGE_SHIFT - 10;
1310 	uint32_t ingpad = 0;
1311 	uint32_t stat_len = clsz > 64 ? 128 : 64;
1312 
1313 	csio_wr_reg32(hw, HOSTPAGESIZEPF0_V(s_hps) | HOSTPAGESIZEPF1_V(s_hps) |
1314 		      HOSTPAGESIZEPF2_V(s_hps) | HOSTPAGESIZEPF3_V(s_hps) |
1315 		      HOSTPAGESIZEPF4_V(s_hps) | HOSTPAGESIZEPF5_V(s_hps) |
1316 		      HOSTPAGESIZEPF6_V(s_hps) | HOSTPAGESIZEPF7_V(s_hps),
1317 		      SGE_HOST_PAGE_SIZE_A);
1318 
1319 	sge->csio_fl_align = clsz < 32 ? 32 : clsz;
1320 	ingpad = ilog2(sge->csio_fl_align) - 5;
1321 
1322 	csio_set_reg_field(hw, SGE_CONTROL_A,
1323 			   INGPADBOUNDARY_V(INGPADBOUNDARY_M) |
1324 			   EGRSTATUSPAGESIZE_F,
1325 			   INGPADBOUNDARY_V(ingpad) |
1326 			   EGRSTATUSPAGESIZE_V(stat_len != 64));
1327 
1328 	/* FL BUFFER SIZE#0 is Page size i,e already aligned to cache line */
1329 	csio_wr_reg32(hw, PAGE_SIZE, SGE_FL_BUFFER_SIZE0_A);
1330 
1331 	/*
1332 	 * If using hard params, the following will get set correctly
1333 	 * in csio_wr_set_sge().
1334 	 */
1335 	if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS) {
1336 		csio_wr_reg32(hw,
1337 			(csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE2_A) +
1338 			sge->csio_fl_align - 1) & ~(sge->csio_fl_align - 1),
1339 			SGE_FL_BUFFER_SIZE2_A);
1340 		csio_wr_reg32(hw,
1341 			(csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE3_A) +
1342 			sge->csio_fl_align - 1) & ~(sge->csio_fl_align - 1),
1343 			SGE_FL_BUFFER_SIZE3_A);
1344 	}
1345 
1346 	csio_wr_reg32(hw, HPZ0_V(PAGE_SHIFT - 12), ULP_RX_TDDP_PSZ_A);
1347 
1348 	/* default value of rx_dma_offset of the NIC driver */
1349 	csio_set_reg_field(hw, SGE_CONTROL_A,
1350 			   PKTSHIFT_V(PKTSHIFT_M),
1351 			   PKTSHIFT_V(CSIO_SGE_RX_DMA_OFFSET));
1352 
1353 	csio_hw_tp_wr_bits_indirect(hw, TP_INGRESS_CONFIG_A,
1354 				    CSUM_HAS_PSEUDO_HDR_F, 0);
1355 }
1356 
1357 static void
1358 csio_init_intr_coalesce_parms(struct csio_hw *hw)
1359 {
1360 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1361 	struct csio_sge *sge = &wrm->sge;
1362 
1363 	csio_sge_thresh_reg = csio_closest_thresh(sge, csio_intr_coalesce_cnt);
1364 	if (csio_intr_coalesce_cnt) {
1365 		csio_sge_thresh_reg = 0;
1366 		csio_sge_timer_reg = X_TIMERREG_RESTART_COUNTER;
1367 		return;
1368 	}
1369 
1370 	csio_sge_timer_reg = csio_closest_timer(sge, csio_intr_coalesce_time);
1371 }
1372 
1373 /*
1374  * csio_wr_get_sge - Get SGE register values.
1375  * @hw: HW module.
1376  *
1377  * Used by non-master functions and by master-functions relying on config file.
1378  */
1379 static void
1380 csio_wr_get_sge(struct csio_hw *hw)
1381 {
1382 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1383 	struct csio_sge *sge = &wrm->sge;
1384 	uint32_t ingpad;
1385 	int i;
1386 	u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5;
1387 	u32 ingress_rx_threshold;
1388 
1389 	sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A);
1390 
1391 	ingpad = INGPADBOUNDARY_G(sge->sge_control);
1392 
1393 	switch (ingpad) {
1394 	case X_INGPCIEBOUNDARY_32B:
1395 		sge->csio_fl_align = 32; break;
1396 	case X_INGPCIEBOUNDARY_64B:
1397 		sge->csio_fl_align = 64; break;
1398 	case X_INGPCIEBOUNDARY_128B:
1399 		sge->csio_fl_align = 128; break;
1400 	case X_INGPCIEBOUNDARY_256B:
1401 		sge->csio_fl_align = 256; break;
1402 	case X_INGPCIEBOUNDARY_512B:
1403 		sge->csio_fl_align = 512; break;
1404 	case X_INGPCIEBOUNDARY_1024B:
1405 		sge->csio_fl_align = 1024; break;
1406 	case X_INGPCIEBOUNDARY_2048B:
1407 		sge->csio_fl_align = 2048; break;
1408 	case X_INGPCIEBOUNDARY_4096B:
1409 		sge->csio_fl_align = 4096; break;
1410 	}
1411 
1412 	for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)
1413 		csio_get_flbuf_size(hw, sge, i);
1414 
1415 	timer_value_0_and_1 = csio_rd_reg32(hw, SGE_TIMER_VALUE_0_AND_1_A);
1416 	timer_value_2_and_3 = csio_rd_reg32(hw, SGE_TIMER_VALUE_2_AND_3_A);
1417 	timer_value_4_and_5 = csio_rd_reg32(hw, SGE_TIMER_VALUE_4_AND_5_A);
1418 
1419 	sge->timer_val[0] = (uint16_t)csio_core_ticks_to_us(hw,
1420 					TIMERVALUE0_G(timer_value_0_and_1));
1421 	sge->timer_val[1] = (uint16_t)csio_core_ticks_to_us(hw,
1422 					TIMERVALUE1_G(timer_value_0_and_1));
1423 	sge->timer_val[2] = (uint16_t)csio_core_ticks_to_us(hw,
1424 					TIMERVALUE2_G(timer_value_2_and_3));
1425 	sge->timer_val[3] = (uint16_t)csio_core_ticks_to_us(hw,
1426 					TIMERVALUE3_G(timer_value_2_and_3));
1427 	sge->timer_val[4] = (uint16_t)csio_core_ticks_to_us(hw,
1428 					TIMERVALUE4_G(timer_value_4_and_5));
1429 	sge->timer_val[5] = (uint16_t)csio_core_ticks_to_us(hw,
1430 					TIMERVALUE5_G(timer_value_4_and_5));
1431 
1432 	ingress_rx_threshold = csio_rd_reg32(hw, SGE_INGRESS_RX_THRESHOLD_A);
1433 	sge->counter_val[0] = THRESHOLD_0_G(ingress_rx_threshold);
1434 	sge->counter_val[1] = THRESHOLD_1_G(ingress_rx_threshold);
1435 	sge->counter_val[2] = THRESHOLD_2_G(ingress_rx_threshold);
1436 	sge->counter_val[3] = THRESHOLD_3_G(ingress_rx_threshold);
1437 
1438 	csio_init_intr_coalesce_parms(hw);
1439 }
1440 
1441 /*
1442  * csio_wr_set_sge - Initialize SGE registers
1443  * @hw: HW module.
1444  *
1445  * Used by Master function to initialize SGE registers in the absence
1446  * of a config file.
1447  */
1448 static void
1449 csio_wr_set_sge(struct csio_hw *hw)
1450 {
1451 	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1452 	struct csio_sge *sge = &wrm->sge;
1453 	int i;
1454 
1455 	/*
1456 	 * Set up our basic SGE mode to deliver CPL messages to our Ingress
1457 	 * Queue and Packet Date to the Free List.
1458 	 */
1459 	csio_set_reg_field(hw, SGE_CONTROL_A, RXPKTCPLMODE_F, RXPKTCPLMODE_F);
1460 
1461 	sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A);
1462 
1463 	/* sge->csio_fl_align is set up by csio_wr_fixup_host_params(). */
1464 
1465 	/*
1466 	 * Set up to drop DOORBELL writes when the DOORBELL FIFO overflows
1467 	 * and generate an interrupt when this occurs so we can recover.
1468 	 */
1469 	csio_set_reg_field(hw, SGE_DBFIFO_STATUS_A,
1470 			   LP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M),
1471 			   LP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH));
1472 	csio_set_reg_field(hw, SGE_DBFIFO_STATUS2_A,
1473 			   HP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M),
1474 			   HP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH));
1475 
1476 	csio_set_reg_field(hw, SGE_DOORBELL_CONTROL_A, ENABLE_DROP_F,
1477 			   ENABLE_DROP_F);
1478 
1479 	/* SGE_FL_BUFFER_SIZE0 is set up by csio_wr_fixup_host_params(). */
1480 
1481 	CSIO_SET_FLBUF_SIZE(hw, 1, CSIO_SGE_FLBUF_SIZE1);
1482 	csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE2 + sge->csio_fl_align - 1)
1483 		      & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE2_A);
1484 	csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE3 + sge->csio_fl_align - 1)
1485 		      & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE3_A);
1486 	CSIO_SET_FLBUF_SIZE(hw, 4, CSIO_SGE_FLBUF_SIZE4);
1487 	CSIO_SET_FLBUF_SIZE(hw, 5, CSIO_SGE_FLBUF_SIZE5);
1488 	CSIO_SET_FLBUF_SIZE(hw, 6, CSIO_SGE_FLBUF_SIZE6);
1489 	CSIO_SET_FLBUF_SIZE(hw, 7, CSIO_SGE_FLBUF_SIZE7);
1490 	CSIO_SET_FLBUF_SIZE(hw, 8, CSIO_SGE_FLBUF_SIZE8);
1491 
1492 	for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)
1493 		csio_get_flbuf_size(hw, sge, i);
1494 
1495 	/* Initialize interrupt coalescing attributes */
1496 	sge->timer_val[0] = CSIO_SGE_TIMER_VAL_0;
1497 	sge->timer_val[1] = CSIO_SGE_TIMER_VAL_1;
1498 	sge->timer_val[2] = CSIO_SGE_TIMER_VAL_2;
1499 	sge->timer_val[3] = CSIO_SGE_TIMER_VAL_3;
1500 	sge->timer_val[4] = CSIO_SGE_TIMER_VAL_4;
1501 	sge->timer_val[5] = CSIO_SGE_TIMER_VAL_5;
1502 
1503 	sge->counter_val[0] = CSIO_SGE_INT_CNT_VAL_0;
1504 	sge->counter_val[1] = CSIO_SGE_INT_CNT_VAL_1;
1505 	sge->counter_val[2] = CSIO_SGE_INT_CNT_VAL_2;
1506 	sge->counter_val[3] = CSIO_SGE_INT_CNT_VAL_3;
1507 
1508 	csio_wr_reg32(hw, THRESHOLD_0_V(sge->counter_val[0]) |
1509 		      THRESHOLD_1_V(sge->counter_val[1]) |
1510 		      THRESHOLD_2_V(sge->counter_val[2]) |
1511 		      THRESHOLD_3_V(sge->counter_val[3]),
1512 		      SGE_INGRESS_RX_THRESHOLD_A);
1513 
1514 	csio_wr_reg32(hw,
1515 		   TIMERVALUE0_V(csio_us_to_core_ticks(hw, sge->timer_val[0])) |
1516 		   TIMERVALUE1_V(csio_us_to_core_ticks(hw, sge->timer_val[1])),
1517 		   SGE_TIMER_VALUE_0_AND_1_A);
1518 
1519 	csio_wr_reg32(hw,
1520 		   TIMERVALUE2_V(csio_us_to_core_ticks(hw, sge->timer_val[2])) |
1521 		   TIMERVALUE3_V(csio_us_to_core_ticks(hw, sge->timer_val[3])),
1522 		   SGE_TIMER_VALUE_2_AND_3_A);
1523 
1524 	csio_wr_reg32(hw,
1525 		   TIMERVALUE4_V(csio_us_to_core_ticks(hw, sge->timer_val[4])) |
1526 		   TIMERVALUE5_V(csio_us_to_core_ticks(hw, sge->timer_val[5])),
1527 		   SGE_TIMER_VALUE_4_AND_5_A);
1528 
1529 	csio_init_intr_coalesce_parms(hw);
1530 }
1531 
1532 void
1533 csio_wr_sge_init(struct csio_hw *hw)
1534 {
1535 	/*
1536 	 * If we are master and chip is not initialized:
1537 	 *    - If we plan to use the config file, we need to fixup some
1538 	 *      host specific registers, and read the rest of the SGE
1539 	 *      configuration.
1540 	 *    - If we dont plan to use the config file, we need to initialize
1541 	 *      SGE entirely, including fixing the host specific registers.
1542 	 * If we are master and chip is initialized, just read and work off of
1543 	 *	the already initialized SGE values.
1544 	 * If we arent the master, we are only allowed to read and work off of
1545 	 *      the already initialized SGE values.
1546 	 *
1547 	 * Therefore, before calling this function, we assume that the master-
1548 	 * ship of the card, state and whether to use config file or not, have
1549 	 * already been decided.
1550 	 */
1551 	if (csio_is_hw_master(hw)) {
1552 		if (hw->fw_state != CSIO_DEV_STATE_INIT)
1553 			csio_wr_fixup_host_params(hw);
1554 
1555 		if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS)
1556 			csio_wr_get_sge(hw);
1557 		else
1558 			csio_wr_set_sge(hw);
1559 	} else
1560 		csio_wr_get_sge(hw);
1561 }
1562 
1563 /*
1564  * csio_wrm_init - Initialize Work request module.
1565  * @wrm: WR module
1566  * @hw: HW pointer
1567  *
1568  * Allocates memory for an array of queue pointers starting at q_arr.
1569  */
1570 int
1571 csio_wrm_init(struct csio_wrm *wrm, struct csio_hw *hw)
1572 {
1573 	int i;
1574 
1575 	if (!wrm->num_q) {
1576 		csio_err(hw, "Num queues is not set\n");
1577 		return -EINVAL;
1578 	}
1579 
1580 	wrm->q_arr = kzalloc(sizeof(struct csio_q *) * wrm->num_q, GFP_KERNEL);
1581 	if (!wrm->q_arr)
1582 		goto err;
1583 
1584 	for (i = 0; i < wrm->num_q; i++) {
1585 		wrm->q_arr[i] = kzalloc(sizeof(struct csio_q), GFP_KERNEL);
1586 		if (!wrm->q_arr[i]) {
1587 			while (--i >= 0)
1588 				kfree(wrm->q_arr[i]);
1589 			goto err_free_arr;
1590 		}
1591 	}
1592 	wrm->free_qidx	= 0;
1593 
1594 	return 0;
1595 
1596 err_free_arr:
1597 	kfree(wrm->q_arr);
1598 err:
1599 	return -ENOMEM;
1600 }
1601 
1602 /*
1603  * csio_wrm_exit - Initialize Work request module.
1604  * @wrm: WR module
1605  * @hw: HW module
1606  *
1607  * Uninitialize WR module. Free q_arr and pointers in it.
1608  * We have the additional job of freeing the DMA memory associated
1609  * with the queues.
1610  */
1611 void
1612 csio_wrm_exit(struct csio_wrm *wrm, struct csio_hw *hw)
1613 {
1614 	int i;
1615 	uint32_t j;
1616 	struct csio_q *q;
1617 	struct csio_dma_buf *buf;
1618 
1619 	for (i = 0; i < wrm->num_q; i++) {
1620 		q = wrm->q_arr[i];
1621 
1622 		if (wrm->free_qidx && (i < wrm->free_qidx)) {
1623 			if (q->type == CSIO_FREELIST) {
1624 				if (!q->un.fl.bufs)
1625 					continue;
1626 				for (j = 0; j < q->credits; j++) {
1627 					buf = &q->un.fl.bufs[j];
1628 					if (!buf->vaddr)
1629 						continue;
1630 					pci_free_consistent(hw->pdev, buf->len,
1631 							    buf->vaddr,
1632 							    buf->paddr);
1633 				}
1634 				kfree(q->un.fl.bufs);
1635 			}
1636 			pci_free_consistent(hw->pdev, q->size,
1637 					    q->vstart, q->pstart);
1638 		}
1639 		kfree(q);
1640 	}
1641 
1642 	hw->flags &= ~CSIO_HWF_Q_MEM_ALLOCED;
1643 
1644 	kfree(wrm->q_arr);
1645 }
1646