1 /* 2 * This file is part of the Chelsio FCoE driver for Linux. 3 * 4 * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include <linux/kernel.h> 36 #include <linux/string.h> 37 #include <linux/compiler.h> 38 #include <linux/slab.h> 39 #include <asm/page.h> 40 #include <linux/cache.h> 41 42 #include "csio_hw.h" 43 #include "csio_wr.h" 44 #include "csio_mb.h" 45 #include "csio_defs.h" 46 47 int csio_intr_coalesce_cnt; /* value:SGE_INGRESS_RX_THRESHOLD[0] */ 48 static int csio_sge_thresh_reg; /* SGE_INGRESS_RX_THRESHOLD[0] */ 49 50 int csio_intr_coalesce_time = 10; /* value:SGE_TIMER_VALUE_1 */ 51 static int csio_sge_timer_reg = 1; 52 53 #define CSIO_SET_FLBUF_SIZE(_hw, _reg, _val) \ 54 csio_wr_reg32((_hw), (_val), SGE_FL_BUFFER_SIZE##_reg##_A) 55 56 static void 57 csio_get_flbuf_size(struct csio_hw *hw, struct csio_sge *sge, uint32_t reg) 58 { 59 sge->sge_fl_buf_size[reg] = csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE0_A + 60 reg * sizeof(uint32_t)); 61 } 62 63 /* Free list buffer size */ 64 static inline uint32_t 65 csio_wr_fl_bufsz(struct csio_sge *sge, struct csio_dma_buf *buf) 66 { 67 return sge->sge_fl_buf_size[buf->paddr & 0xF]; 68 } 69 70 /* Size of the egress queue status page */ 71 static inline uint32_t 72 csio_wr_qstat_pgsz(struct csio_hw *hw) 73 { 74 return (hw->wrm.sge.sge_control & EGRSTATUSPAGESIZE_F) ? 128 : 64; 75 } 76 77 /* Ring freelist doorbell */ 78 static inline void 79 csio_wr_ring_fldb(struct csio_hw *hw, struct csio_q *flq) 80 { 81 /* 82 * Ring the doorbell only when we have atleast CSIO_QCREDIT_SZ 83 * number of bytes in the freelist queue. This translates to atleast 84 * 8 freelist buffer pointers (since each pointer is 8 bytes). 85 */ 86 if (flq->inc_idx >= 8) { 87 csio_wr_reg32(hw, DBPRIO_F | QID_V(flq->un.fl.flid) | 88 PIDX_T5_V(flq->inc_idx / 8) | DBTYPE_F, 89 MYPF_REG(SGE_PF_KDOORBELL_A)); 90 flq->inc_idx &= 7; 91 } 92 } 93 94 /* Write a 0 cidx increment value to enable SGE interrupts for this queue */ 95 static void 96 csio_wr_sge_intr_enable(struct csio_hw *hw, uint16_t iqid) 97 { 98 csio_wr_reg32(hw, CIDXINC_V(0) | 99 INGRESSQID_V(iqid) | 100 TIMERREG_V(X_TIMERREG_RESTART_COUNTER), 101 MYPF_REG(SGE_PF_GTS_A)); 102 } 103 104 /* 105 * csio_wr_fill_fl - Populate the FL buffers of a FL queue. 106 * @hw: HW module. 107 * @flq: Freelist queue. 108 * 109 * Fill up freelist buffer entries with buffers of size specified 110 * in the size register. 111 * 112 */ 113 static int 114 csio_wr_fill_fl(struct csio_hw *hw, struct csio_q *flq) 115 { 116 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 117 struct csio_sge *sge = &wrm->sge; 118 __be64 *d = (__be64 *)(flq->vstart); 119 struct csio_dma_buf *buf = &flq->un.fl.bufs[0]; 120 uint64_t paddr; 121 int sreg = flq->un.fl.sreg; 122 int n = flq->credits; 123 124 while (n--) { 125 buf->len = sge->sge_fl_buf_size[sreg]; 126 buf->vaddr = pci_alloc_consistent(hw->pdev, buf->len, 127 &buf->paddr); 128 if (!buf->vaddr) { 129 csio_err(hw, "Could only fill %d buffers!\n", n + 1); 130 return -ENOMEM; 131 } 132 133 paddr = buf->paddr | (sreg & 0xF); 134 135 *d++ = cpu_to_be64(paddr); 136 buf++; 137 } 138 139 return 0; 140 } 141 142 /* 143 * csio_wr_update_fl - 144 * @hw: HW module. 145 * @flq: Freelist queue. 146 * 147 * 148 */ 149 static inline void 150 csio_wr_update_fl(struct csio_hw *hw, struct csio_q *flq, uint16_t n) 151 { 152 153 flq->inc_idx += n; 154 flq->pidx += n; 155 if (unlikely(flq->pidx >= flq->credits)) 156 flq->pidx -= (uint16_t)flq->credits; 157 158 CSIO_INC_STATS(flq, n_flq_refill); 159 } 160 161 /* 162 * csio_wr_alloc_q - Allocate a WR queue and initialize it. 163 * @hw: HW module 164 * @qsize: Size of the queue in bytes 165 * @wrsize: Since of WR in this queue, if fixed. 166 * @type: Type of queue (Ingress/Egress/Freelist) 167 * @owner: Module that owns this queue. 168 * @nflb: Number of freelist buffers for FL. 169 * @sreg: What is the FL buffer size register? 170 * @iq_int_handler: Ingress queue handler in INTx mode. 171 * 172 * This function allocates and sets up a queue for the caller 173 * of size qsize, aligned at the required boundary. This is subject to 174 * be free entries being available in the queue array. If one is found, 175 * it is initialized with the allocated queue, marked as being used (owner), 176 * and a handle returned to the caller in form of the queue's index 177 * into the q_arr array. 178 * If user has indicated a freelist (by specifying nflb > 0), create 179 * another queue (with its own index into q_arr) for the freelist. Allocate 180 * memory for DMA buffer metadata (vaddr, len etc). Save off the freelist 181 * idx in the ingress queue's flq.idx. This is how a Freelist is associated 182 * with its owning ingress queue. 183 */ 184 int 185 csio_wr_alloc_q(struct csio_hw *hw, uint32_t qsize, uint32_t wrsize, 186 uint16_t type, void *owner, uint32_t nflb, int sreg, 187 iq_handler_t iq_intx_handler) 188 { 189 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 190 struct csio_q *q, *flq; 191 int free_idx = wrm->free_qidx; 192 int ret_idx = free_idx; 193 uint32_t qsz; 194 int flq_idx; 195 196 if (free_idx >= wrm->num_q) { 197 csio_err(hw, "No more free queues.\n"); 198 return -1; 199 } 200 201 switch (type) { 202 case CSIO_EGRESS: 203 qsz = ALIGN(qsize, CSIO_QCREDIT_SZ) + csio_wr_qstat_pgsz(hw); 204 break; 205 case CSIO_INGRESS: 206 switch (wrsize) { 207 case 16: 208 case 32: 209 case 64: 210 case 128: 211 break; 212 default: 213 csio_err(hw, "Invalid Ingress queue WR size:%d\n", 214 wrsize); 215 return -1; 216 } 217 218 /* 219 * Number of elements must be a multiple of 16 220 * So this includes status page size 221 */ 222 qsz = ALIGN(qsize/wrsize, 16) * wrsize; 223 224 break; 225 case CSIO_FREELIST: 226 qsz = ALIGN(qsize/wrsize, 8) * wrsize + csio_wr_qstat_pgsz(hw); 227 break; 228 default: 229 csio_err(hw, "Invalid queue type: 0x%x\n", type); 230 return -1; 231 } 232 233 q = wrm->q_arr[free_idx]; 234 235 q->vstart = pci_zalloc_consistent(hw->pdev, qsz, &q->pstart); 236 if (!q->vstart) { 237 csio_err(hw, 238 "Failed to allocate DMA memory for " 239 "queue at id: %d size: %d\n", free_idx, qsize); 240 return -1; 241 } 242 243 q->type = type; 244 q->owner = owner; 245 q->pidx = q->cidx = q->inc_idx = 0; 246 q->size = qsz; 247 q->wr_sz = wrsize; /* If using fixed size WRs */ 248 249 wrm->free_qidx++; 250 251 if (type == CSIO_INGRESS) { 252 /* Since queue area is set to zero */ 253 q->un.iq.genbit = 1; 254 255 /* 256 * Ingress queue status page size is always the size of 257 * the ingress queue entry. 258 */ 259 q->credits = (qsz - q->wr_sz) / q->wr_sz; 260 q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz 261 - q->wr_sz); 262 263 /* Allocate memory for FL if requested */ 264 if (nflb > 0) { 265 flq_idx = csio_wr_alloc_q(hw, nflb * sizeof(__be64), 266 sizeof(__be64), CSIO_FREELIST, 267 owner, 0, sreg, NULL); 268 if (flq_idx == -1) { 269 csio_err(hw, 270 "Failed to allocate FL queue" 271 " for IQ idx:%d\n", free_idx); 272 return -1; 273 } 274 275 /* Associate the new FL with the Ingress quue */ 276 q->un.iq.flq_idx = flq_idx; 277 278 flq = wrm->q_arr[q->un.iq.flq_idx]; 279 flq->un.fl.bufs = kcalloc(flq->credits, 280 sizeof(struct csio_dma_buf), 281 GFP_KERNEL); 282 if (!flq->un.fl.bufs) { 283 csio_err(hw, 284 "Failed to allocate FL queue bufs" 285 " for IQ idx:%d\n", free_idx); 286 return -1; 287 } 288 289 flq->un.fl.packen = 0; 290 flq->un.fl.offset = 0; 291 flq->un.fl.sreg = sreg; 292 293 /* Fill up the free list buffers */ 294 if (csio_wr_fill_fl(hw, flq)) 295 return -1; 296 297 /* 298 * Make sure in a FLQ, atleast 1 credit (8 FL buffers) 299 * remains unpopulated,otherwise HW thinks 300 * FLQ is empty. 301 */ 302 flq->pidx = flq->inc_idx = flq->credits - 8; 303 } else { 304 q->un.iq.flq_idx = -1; 305 } 306 307 /* Associate the IQ INTx handler. */ 308 q->un.iq.iq_intx_handler = iq_intx_handler; 309 310 csio_q_iqid(hw, ret_idx) = CSIO_MAX_QID; 311 312 } else if (type == CSIO_EGRESS) { 313 q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / CSIO_QCREDIT_SZ; 314 q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz 315 - csio_wr_qstat_pgsz(hw)); 316 csio_q_eqid(hw, ret_idx) = CSIO_MAX_QID; 317 } else { /* Freelist */ 318 q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / sizeof(__be64); 319 q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz 320 - csio_wr_qstat_pgsz(hw)); 321 csio_q_flid(hw, ret_idx) = CSIO_MAX_QID; 322 } 323 324 return ret_idx; 325 } 326 327 /* 328 * csio_wr_iq_create_rsp - Response handler for IQ creation. 329 * @hw: The HW module. 330 * @mbp: Mailbox. 331 * @iq_idx: Ingress queue that got created. 332 * 333 * Handle FW_IQ_CMD mailbox completion. Save off the assigned IQ/FL ids. 334 */ 335 static int 336 csio_wr_iq_create_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx) 337 { 338 struct csio_iq_params iqp; 339 enum fw_retval retval; 340 uint32_t iq_id; 341 int flq_idx; 342 343 memset(&iqp, 0, sizeof(struct csio_iq_params)); 344 345 csio_mb_iq_alloc_write_rsp(hw, mbp, &retval, &iqp); 346 347 if (retval != FW_SUCCESS) { 348 csio_err(hw, "IQ cmd returned 0x%x!\n", retval); 349 mempool_free(mbp, hw->mb_mempool); 350 return -EINVAL; 351 } 352 353 csio_q_iqid(hw, iq_idx) = iqp.iqid; 354 csio_q_physiqid(hw, iq_idx) = iqp.physiqid; 355 csio_q_pidx(hw, iq_idx) = csio_q_cidx(hw, iq_idx) = 0; 356 csio_q_inc_idx(hw, iq_idx) = 0; 357 358 /* Actual iq-id. */ 359 iq_id = iqp.iqid - hw->wrm.fw_iq_start; 360 361 /* Set the iq-id to iq map table. */ 362 if (iq_id >= CSIO_MAX_IQ) { 363 csio_err(hw, 364 "Exceeding MAX_IQ(%d) supported!" 365 " iqid:%d rel_iqid:%d FW iq_start:%d\n", 366 CSIO_MAX_IQ, iq_id, iqp.iqid, hw->wrm.fw_iq_start); 367 mempool_free(mbp, hw->mb_mempool); 368 return -EINVAL; 369 } 370 csio_q_set_intr_map(hw, iq_idx, iq_id); 371 372 /* 373 * During FW_IQ_CMD, FW sets interrupt_sent bit to 1 in the SGE 374 * ingress context of this queue. This will block interrupts to 375 * this queue until the next GTS write. Therefore, we do a 376 * 0-cidx increment GTS write for this queue just to clear the 377 * interrupt_sent bit. This will re-enable interrupts to this 378 * queue. 379 */ 380 csio_wr_sge_intr_enable(hw, iqp.physiqid); 381 382 flq_idx = csio_q_iq_flq_idx(hw, iq_idx); 383 if (flq_idx != -1) { 384 struct csio_q *flq = hw->wrm.q_arr[flq_idx]; 385 386 csio_q_flid(hw, flq_idx) = iqp.fl0id; 387 csio_q_cidx(hw, flq_idx) = 0; 388 csio_q_pidx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8; 389 csio_q_inc_idx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8; 390 391 /* Now update SGE about the buffers allocated during init */ 392 csio_wr_ring_fldb(hw, flq); 393 } 394 395 mempool_free(mbp, hw->mb_mempool); 396 397 return 0; 398 } 399 400 /* 401 * csio_wr_iq_create - Configure an Ingress queue with FW. 402 * @hw: The HW module. 403 * @priv: Private data object. 404 * @iq_idx: Ingress queue index in the WR module. 405 * @vec: MSIX vector. 406 * @portid: PCIE Channel to be associated with this queue. 407 * @async: Is this a FW asynchronous message handling queue? 408 * @cbfn: Completion callback. 409 * 410 * This API configures an ingress queue with FW by issuing a FW_IQ_CMD mailbox 411 * with alloc/write bits set. 412 */ 413 int 414 csio_wr_iq_create(struct csio_hw *hw, void *priv, int iq_idx, 415 uint32_t vec, uint8_t portid, bool async, 416 void (*cbfn) (struct csio_hw *, struct csio_mb *)) 417 { 418 struct csio_mb *mbp; 419 struct csio_iq_params iqp; 420 int flq_idx; 421 422 memset(&iqp, 0, sizeof(struct csio_iq_params)); 423 csio_q_portid(hw, iq_idx) = portid; 424 425 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC); 426 if (!mbp) { 427 csio_err(hw, "IQ command out of memory!\n"); 428 return -ENOMEM; 429 } 430 431 switch (hw->intr_mode) { 432 case CSIO_IM_INTX: 433 case CSIO_IM_MSI: 434 /* For interrupt forwarding queue only */ 435 if (hw->intr_iq_idx == iq_idx) 436 iqp.iqandst = X_INTERRUPTDESTINATION_PCIE; 437 else 438 iqp.iqandst = X_INTERRUPTDESTINATION_IQ; 439 iqp.iqandstindex = 440 csio_q_physiqid(hw, hw->intr_iq_idx); 441 break; 442 case CSIO_IM_MSIX: 443 iqp.iqandst = X_INTERRUPTDESTINATION_PCIE; 444 iqp.iqandstindex = (uint16_t)vec; 445 break; 446 case CSIO_IM_NONE: 447 mempool_free(mbp, hw->mb_mempool); 448 return -EINVAL; 449 } 450 451 /* Pass in the ingress queue cmd parameters */ 452 iqp.pfn = hw->pfn; 453 iqp.vfn = 0; 454 iqp.iq_start = 1; 455 iqp.viid = 0; 456 iqp.type = FW_IQ_TYPE_FL_INT_CAP; 457 iqp.iqasynch = async; 458 if (csio_intr_coalesce_cnt) 459 iqp.iqanus = X_UPDATESCHEDULING_COUNTER_OPTTIMER; 460 else 461 iqp.iqanus = X_UPDATESCHEDULING_TIMER; 462 iqp.iqanud = X_UPDATEDELIVERY_INTERRUPT; 463 iqp.iqpciech = portid; 464 iqp.iqintcntthresh = (uint8_t)csio_sge_thresh_reg; 465 466 switch (csio_q_wr_sz(hw, iq_idx)) { 467 case 16: 468 iqp.iqesize = 0; break; 469 case 32: 470 iqp.iqesize = 1; break; 471 case 64: 472 iqp.iqesize = 2; break; 473 case 128: 474 iqp.iqesize = 3; break; 475 } 476 477 iqp.iqsize = csio_q_size(hw, iq_idx) / 478 csio_q_wr_sz(hw, iq_idx); 479 iqp.iqaddr = csio_q_pstart(hw, iq_idx); 480 481 flq_idx = csio_q_iq_flq_idx(hw, iq_idx); 482 if (flq_idx != -1) { 483 enum chip_type chip = CHELSIO_CHIP_VERSION(hw->chip_id); 484 struct csio_q *flq = hw->wrm.q_arr[flq_idx]; 485 486 iqp.fl0paden = 1; 487 iqp.fl0packen = flq->un.fl.packen ? 1 : 0; 488 iqp.fl0fbmin = X_FETCHBURSTMIN_64B; 489 iqp.fl0fbmax = ((chip == CHELSIO_T5) ? 490 X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B); 491 iqp.fl0size = csio_q_size(hw, flq_idx) / CSIO_QCREDIT_SZ; 492 iqp.fl0addr = csio_q_pstart(hw, flq_idx); 493 } 494 495 csio_mb_iq_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn); 496 497 if (csio_mb_issue(hw, mbp)) { 498 csio_err(hw, "Issue of IQ cmd failed!\n"); 499 mempool_free(mbp, hw->mb_mempool); 500 return -EINVAL; 501 } 502 503 if (cbfn != NULL) 504 return 0; 505 506 return csio_wr_iq_create_rsp(hw, mbp, iq_idx); 507 } 508 509 /* 510 * csio_wr_eq_create_rsp - Response handler for EQ creation. 511 * @hw: The HW module. 512 * @mbp: Mailbox. 513 * @eq_idx: Egress queue that got created. 514 * 515 * Handle FW_EQ_OFLD_CMD mailbox completion. Save off the assigned EQ ids. 516 */ 517 static int 518 csio_wr_eq_cfg_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx) 519 { 520 struct csio_eq_params eqp; 521 enum fw_retval retval; 522 523 memset(&eqp, 0, sizeof(struct csio_eq_params)); 524 525 csio_mb_eq_ofld_alloc_write_rsp(hw, mbp, &retval, &eqp); 526 527 if (retval != FW_SUCCESS) { 528 csio_err(hw, "EQ OFLD cmd returned 0x%x!\n", retval); 529 mempool_free(mbp, hw->mb_mempool); 530 return -EINVAL; 531 } 532 533 csio_q_eqid(hw, eq_idx) = (uint16_t)eqp.eqid; 534 csio_q_physeqid(hw, eq_idx) = (uint16_t)eqp.physeqid; 535 csio_q_pidx(hw, eq_idx) = csio_q_cidx(hw, eq_idx) = 0; 536 csio_q_inc_idx(hw, eq_idx) = 0; 537 538 mempool_free(mbp, hw->mb_mempool); 539 540 return 0; 541 } 542 543 /* 544 * csio_wr_eq_create - Configure an Egress queue with FW. 545 * @hw: HW module. 546 * @priv: Private data. 547 * @eq_idx: Egress queue index in the WR module. 548 * @iq_idx: Associated ingress queue index. 549 * @cbfn: Completion callback. 550 * 551 * This API configures a offload egress queue with FW by issuing a 552 * FW_EQ_OFLD_CMD (with alloc + write ) mailbox. 553 */ 554 int 555 csio_wr_eq_create(struct csio_hw *hw, void *priv, int eq_idx, 556 int iq_idx, uint8_t portid, 557 void (*cbfn) (struct csio_hw *, struct csio_mb *)) 558 { 559 struct csio_mb *mbp; 560 struct csio_eq_params eqp; 561 562 memset(&eqp, 0, sizeof(struct csio_eq_params)); 563 564 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC); 565 if (!mbp) { 566 csio_err(hw, "EQ command out of memory!\n"); 567 return -ENOMEM; 568 } 569 570 eqp.pfn = hw->pfn; 571 eqp.vfn = 0; 572 eqp.eqstart = 1; 573 eqp.hostfcmode = X_HOSTFCMODE_STATUS_PAGE; 574 eqp.iqid = csio_q_iqid(hw, iq_idx); 575 eqp.fbmin = X_FETCHBURSTMIN_64B; 576 eqp.fbmax = X_FETCHBURSTMAX_512B; 577 eqp.cidxfthresh = 0; 578 eqp.pciechn = portid; 579 eqp.eqsize = csio_q_size(hw, eq_idx) / CSIO_QCREDIT_SZ; 580 eqp.eqaddr = csio_q_pstart(hw, eq_idx); 581 582 csio_mb_eq_ofld_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, 583 &eqp, cbfn); 584 585 if (csio_mb_issue(hw, mbp)) { 586 csio_err(hw, "Issue of EQ OFLD cmd failed!\n"); 587 mempool_free(mbp, hw->mb_mempool); 588 return -EINVAL; 589 } 590 591 if (cbfn != NULL) 592 return 0; 593 594 return csio_wr_eq_cfg_rsp(hw, mbp, eq_idx); 595 } 596 597 /* 598 * csio_wr_iq_destroy_rsp - Response handler for IQ removal. 599 * @hw: The HW module. 600 * @mbp: Mailbox. 601 * @iq_idx: Ingress queue that was freed. 602 * 603 * Handle FW_IQ_CMD (free) mailbox completion. 604 */ 605 static int 606 csio_wr_iq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx) 607 { 608 enum fw_retval retval = csio_mb_fw_retval(mbp); 609 int rv = 0; 610 611 if (retval != FW_SUCCESS) 612 rv = -EINVAL; 613 614 mempool_free(mbp, hw->mb_mempool); 615 616 return rv; 617 } 618 619 /* 620 * csio_wr_iq_destroy - Free an ingress queue. 621 * @hw: The HW module. 622 * @priv: Private data object. 623 * @iq_idx: Ingress queue index to destroy 624 * @cbfn: Completion callback. 625 * 626 * This API frees an ingress queue by issuing the FW_IQ_CMD 627 * with the free bit set. 628 */ 629 static int 630 csio_wr_iq_destroy(struct csio_hw *hw, void *priv, int iq_idx, 631 void (*cbfn)(struct csio_hw *, struct csio_mb *)) 632 { 633 int rv = 0; 634 struct csio_mb *mbp; 635 struct csio_iq_params iqp; 636 int flq_idx; 637 638 memset(&iqp, 0, sizeof(struct csio_iq_params)); 639 640 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC); 641 if (!mbp) 642 return -ENOMEM; 643 644 iqp.pfn = hw->pfn; 645 iqp.vfn = 0; 646 iqp.iqid = csio_q_iqid(hw, iq_idx); 647 iqp.type = FW_IQ_TYPE_FL_INT_CAP; 648 649 flq_idx = csio_q_iq_flq_idx(hw, iq_idx); 650 if (flq_idx != -1) 651 iqp.fl0id = csio_q_flid(hw, flq_idx); 652 else 653 iqp.fl0id = 0xFFFF; 654 655 iqp.fl1id = 0xFFFF; 656 657 csio_mb_iq_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn); 658 659 rv = csio_mb_issue(hw, mbp); 660 if (rv != 0) { 661 mempool_free(mbp, hw->mb_mempool); 662 return rv; 663 } 664 665 if (cbfn != NULL) 666 return 0; 667 668 return csio_wr_iq_destroy_rsp(hw, mbp, iq_idx); 669 } 670 671 /* 672 * csio_wr_eq_destroy_rsp - Response handler for OFLD EQ creation. 673 * @hw: The HW module. 674 * @mbp: Mailbox. 675 * @eq_idx: Egress queue that was freed. 676 * 677 * Handle FW_OFLD_EQ_CMD (free) mailbox completion. 678 */ 679 static int 680 csio_wr_eq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx) 681 { 682 enum fw_retval retval = csio_mb_fw_retval(mbp); 683 int rv = 0; 684 685 if (retval != FW_SUCCESS) 686 rv = -EINVAL; 687 688 mempool_free(mbp, hw->mb_mempool); 689 690 return rv; 691 } 692 693 /* 694 * csio_wr_eq_destroy - Free an Egress queue. 695 * @hw: The HW module. 696 * @priv: Private data object. 697 * @eq_idx: Egress queue index to destroy 698 * @cbfn: Completion callback. 699 * 700 * This API frees an Egress queue by issuing the FW_EQ_OFLD_CMD 701 * with the free bit set. 702 */ 703 static int 704 csio_wr_eq_destroy(struct csio_hw *hw, void *priv, int eq_idx, 705 void (*cbfn) (struct csio_hw *, struct csio_mb *)) 706 { 707 int rv = 0; 708 struct csio_mb *mbp; 709 struct csio_eq_params eqp; 710 711 memset(&eqp, 0, sizeof(struct csio_eq_params)); 712 713 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC); 714 if (!mbp) 715 return -ENOMEM; 716 717 eqp.pfn = hw->pfn; 718 eqp.vfn = 0; 719 eqp.eqid = csio_q_eqid(hw, eq_idx); 720 721 csio_mb_eq_ofld_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &eqp, cbfn); 722 723 rv = csio_mb_issue(hw, mbp); 724 if (rv != 0) { 725 mempool_free(mbp, hw->mb_mempool); 726 return rv; 727 } 728 729 if (cbfn != NULL) 730 return 0; 731 732 return csio_wr_eq_destroy_rsp(hw, mbp, eq_idx); 733 } 734 735 /* 736 * csio_wr_cleanup_eq_stpg - Cleanup Egress queue status page 737 * @hw: HW module 738 * @qidx: Egress queue index 739 * 740 * Cleanup the Egress queue status page. 741 */ 742 static void 743 csio_wr_cleanup_eq_stpg(struct csio_hw *hw, int qidx) 744 { 745 struct csio_q *q = csio_hw_to_wrm(hw)->q_arr[qidx]; 746 struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap; 747 748 memset(stp, 0, sizeof(*stp)); 749 } 750 751 /* 752 * csio_wr_cleanup_iq_ftr - Cleanup Footer entries in IQ 753 * @hw: HW module 754 * @qidx: Ingress queue index 755 * 756 * Cleanup the footer entries in the given ingress queue, 757 * set to 1 the internal copy of genbit. 758 */ 759 static void 760 csio_wr_cleanup_iq_ftr(struct csio_hw *hw, int qidx) 761 { 762 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 763 struct csio_q *q = wrm->q_arr[qidx]; 764 void *wr; 765 struct csio_iqwr_footer *ftr; 766 uint32_t i = 0; 767 768 /* set to 1 since we are just about zero out genbit */ 769 q->un.iq.genbit = 1; 770 771 for (i = 0; i < q->credits; i++) { 772 /* Get the WR */ 773 wr = (void *)((uintptr_t)q->vstart + 774 (i * q->wr_sz)); 775 /* Get the footer */ 776 ftr = (struct csio_iqwr_footer *)((uintptr_t)wr + 777 (q->wr_sz - sizeof(*ftr))); 778 /* Zero out footer */ 779 memset(ftr, 0, sizeof(*ftr)); 780 } 781 } 782 783 int 784 csio_wr_destroy_queues(struct csio_hw *hw, bool cmd) 785 { 786 int i, flq_idx; 787 struct csio_q *q; 788 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 789 int rv; 790 791 for (i = 0; i < wrm->free_qidx; i++) { 792 q = wrm->q_arr[i]; 793 794 switch (q->type) { 795 case CSIO_EGRESS: 796 if (csio_q_eqid(hw, i) != CSIO_MAX_QID) { 797 csio_wr_cleanup_eq_stpg(hw, i); 798 if (!cmd) { 799 csio_q_eqid(hw, i) = CSIO_MAX_QID; 800 continue; 801 } 802 803 rv = csio_wr_eq_destroy(hw, NULL, i, NULL); 804 if ((rv == -EBUSY) || (rv == -ETIMEDOUT)) 805 cmd = false; 806 807 csio_q_eqid(hw, i) = CSIO_MAX_QID; 808 } 809 case CSIO_INGRESS: 810 if (csio_q_iqid(hw, i) != CSIO_MAX_QID) { 811 csio_wr_cleanup_iq_ftr(hw, i); 812 if (!cmd) { 813 csio_q_iqid(hw, i) = CSIO_MAX_QID; 814 flq_idx = csio_q_iq_flq_idx(hw, i); 815 if (flq_idx != -1) 816 csio_q_flid(hw, flq_idx) = 817 CSIO_MAX_QID; 818 continue; 819 } 820 821 rv = csio_wr_iq_destroy(hw, NULL, i, NULL); 822 if ((rv == -EBUSY) || (rv == -ETIMEDOUT)) 823 cmd = false; 824 825 csio_q_iqid(hw, i) = CSIO_MAX_QID; 826 flq_idx = csio_q_iq_flq_idx(hw, i); 827 if (flq_idx != -1) 828 csio_q_flid(hw, flq_idx) = CSIO_MAX_QID; 829 } 830 default: 831 break; 832 } 833 } 834 835 hw->flags &= ~CSIO_HWF_Q_FW_ALLOCED; 836 837 return 0; 838 } 839 840 /* 841 * csio_wr_get - Get requested size of WR entry/entries from queue. 842 * @hw: HW module. 843 * @qidx: Index of queue. 844 * @size: Cumulative size of Work request(s). 845 * @wrp: Work request pair. 846 * 847 * If requested credits are available, return the start address of the 848 * work request in the work request pair. Set pidx accordingly and 849 * return. 850 * 851 * NOTE about WR pair: 852 * ================== 853 * A WR can start towards the end of a queue, and then continue at the 854 * beginning, since the queue is considered to be circular. This will 855 * require a pair of address/size to be passed back to the caller - 856 * hence Work request pair format. 857 */ 858 int 859 csio_wr_get(struct csio_hw *hw, int qidx, uint32_t size, 860 struct csio_wr_pair *wrp) 861 { 862 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 863 struct csio_q *q = wrm->q_arr[qidx]; 864 void *cwr = (void *)((uintptr_t)(q->vstart) + 865 (q->pidx * CSIO_QCREDIT_SZ)); 866 struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap; 867 uint16_t cidx = q->cidx = ntohs(stp->cidx); 868 uint16_t pidx = q->pidx; 869 uint32_t req_sz = ALIGN(size, CSIO_QCREDIT_SZ); 870 int req_credits = req_sz / CSIO_QCREDIT_SZ; 871 int credits; 872 873 CSIO_DB_ASSERT(q->owner != NULL); 874 CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx)); 875 CSIO_DB_ASSERT(cidx <= q->credits); 876 877 /* Calculate credits */ 878 if (pidx > cidx) { 879 credits = q->credits - (pidx - cidx) - 1; 880 } else if (cidx > pidx) { 881 credits = cidx - pidx - 1; 882 } else { 883 /* cidx == pidx, empty queue */ 884 credits = q->credits; 885 CSIO_INC_STATS(q, n_qempty); 886 } 887 888 /* 889 * Check if we have enough credits. 890 * credits = 1 implies queue is full. 891 */ 892 if (!credits || (req_credits > credits)) { 893 CSIO_INC_STATS(q, n_qfull); 894 return -EBUSY; 895 } 896 897 /* 898 * If we are here, we have enough credits to satisfy the 899 * request. Check if we are near the end of q, and if WR spills over. 900 * If it does, use the first addr/size to cover the queue until 901 * the end. Fit the remainder portion of the request at the top 902 * of queue and return it in the second addr/len. Set pidx 903 * accordingly. 904 */ 905 if (unlikely(((uintptr_t)cwr + req_sz) > (uintptr_t)(q->vwrap))) { 906 wrp->addr1 = cwr; 907 wrp->size1 = (uint32_t)((uintptr_t)q->vwrap - (uintptr_t)cwr); 908 wrp->addr2 = q->vstart; 909 wrp->size2 = req_sz - wrp->size1; 910 q->pidx = (uint16_t)(ALIGN(wrp->size2, CSIO_QCREDIT_SZ) / 911 CSIO_QCREDIT_SZ); 912 CSIO_INC_STATS(q, n_qwrap); 913 CSIO_INC_STATS(q, n_eq_wr_split); 914 } else { 915 wrp->addr1 = cwr; 916 wrp->size1 = req_sz; 917 wrp->addr2 = NULL; 918 wrp->size2 = 0; 919 q->pidx += (uint16_t)req_credits; 920 921 /* We are the end of queue, roll back pidx to top of queue */ 922 if (unlikely(q->pidx == q->credits)) { 923 q->pidx = 0; 924 CSIO_INC_STATS(q, n_qwrap); 925 } 926 } 927 928 q->inc_idx = (uint16_t)req_credits; 929 930 CSIO_INC_STATS(q, n_tot_reqs); 931 932 return 0; 933 } 934 935 /* 936 * csio_wr_copy_to_wrp - Copies given data into WR. 937 * @data_buf - Data buffer 938 * @wrp - Work request pair. 939 * @wr_off - Work request offset. 940 * @data_len - Data length. 941 * 942 * Copies the given data in Work Request. Work request pair(wrp) specifies 943 * address information of Work request. 944 * Returns: none 945 */ 946 void 947 csio_wr_copy_to_wrp(void *data_buf, struct csio_wr_pair *wrp, 948 uint32_t wr_off, uint32_t data_len) 949 { 950 uint32_t nbytes; 951 952 /* Number of space available in buffer addr1 of WRP */ 953 nbytes = ((wrp->size1 - wr_off) >= data_len) ? 954 data_len : (wrp->size1 - wr_off); 955 956 memcpy((uint8_t *) wrp->addr1 + wr_off, data_buf, nbytes); 957 data_len -= nbytes; 958 959 /* Write the remaining data from the begining of circular buffer */ 960 if (data_len) { 961 CSIO_DB_ASSERT(data_len <= wrp->size2); 962 CSIO_DB_ASSERT(wrp->addr2 != NULL); 963 memcpy(wrp->addr2, (uint8_t *) data_buf + nbytes, data_len); 964 } 965 } 966 967 /* 968 * csio_wr_issue - Notify chip of Work request. 969 * @hw: HW module. 970 * @qidx: Index of queue. 971 * @prio: 0: Low priority, 1: High priority 972 * 973 * Rings the SGE Doorbell by writing the current producer index of the passed 974 * in queue into the register. 975 * 976 */ 977 int 978 csio_wr_issue(struct csio_hw *hw, int qidx, bool prio) 979 { 980 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 981 struct csio_q *q = wrm->q_arr[qidx]; 982 983 CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx)); 984 985 wmb(); 986 /* Ring SGE Doorbell writing q->pidx into it */ 987 csio_wr_reg32(hw, DBPRIO_V(prio) | QID_V(q->un.eq.physeqid) | 988 PIDX_T5_V(q->inc_idx) | DBTYPE_F, 989 MYPF_REG(SGE_PF_KDOORBELL_A)); 990 q->inc_idx = 0; 991 992 return 0; 993 } 994 995 static inline uint32_t 996 csio_wr_avail_qcredits(struct csio_q *q) 997 { 998 if (q->pidx > q->cidx) 999 return q->pidx - q->cidx; 1000 else if (q->cidx > q->pidx) 1001 return q->credits - (q->cidx - q->pidx); 1002 else 1003 return 0; /* cidx == pidx, empty queue */ 1004 } 1005 1006 /* 1007 * csio_wr_inval_flq_buf - Invalidate a free list buffer entry. 1008 * @hw: HW module. 1009 * @flq: The freelist queue. 1010 * 1011 * Invalidate the driver's version of a freelist buffer entry, 1012 * without freeing the associated the DMA memory. The entry 1013 * to be invalidated is picked up from the current Free list 1014 * queue cidx. 1015 * 1016 */ 1017 static inline void 1018 csio_wr_inval_flq_buf(struct csio_hw *hw, struct csio_q *flq) 1019 { 1020 flq->cidx++; 1021 if (flq->cidx == flq->credits) { 1022 flq->cidx = 0; 1023 CSIO_INC_STATS(flq, n_qwrap); 1024 } 1025 } 1026 1027 /* 1028 * csio_wr_process_fl - Process a freelist completion. 1029 * @hw: HW module. 1030 * @q: The ingress queue attached to the Freelist. 1031 * @wr: The freelist completion WR in the ingress queue. 1032 * @len_to_qid: The lower 32-bits of the first flit of the RSP footer 1033 * @iq_handler: Caller's handler for this completion. 1034 * @priv: Private pointer of caller 1035 * 1036 */ 1037 static inline void 1038 csio_wr_process_fl(struct csio_hw *hw, struct csio_q *q, 1039 void *wr, uint32_t len_to_qid, 1040 void (*iq_handler)(struct csio_hw *, void *, 1041 uint32_t, struct csio_fl_dma_buf *, 1042 void *), 1043 void *priv) 1044 { 1045 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 1046 struct csio_sge *sge = &wrm->sge; 1047 struct csio_fl_dma_buf flb; 1048 struct csio_dma_buf *buf, *fbuf; 1049 uint32_t bufsz, len, lastlen = 0; 1050 int n; 1051 struct csio_q *flq = hw->wrm.q_arr[q->un.iq.flq_idx]; 1052 1053 CSIO_DB_ASSERT(flq != NULL); 1054 1055 len = len_to_qid; 1056 1057 if (len & IQWRF_NEWBUF) { 1058 if (flq->un.fl.offset > 0) { 1059 csio_wr_inval_flq_buf(hw, flq); 1060 flq->un.fl.offset = 0; 1061 } 1062 len = IQWRF_LEN_GET(len); 1063 } 1064 1065 CSIO_DB_ASSERT(len != 0); 1066 1067 flb.totlen = len; 1068 1069 /* Consume all freelist buffers used for len bytes */ 1070 for (n = 0, fbuf = flb.flbufs; ; n++, fbuf++) { 1071 buf = &flq->un.fl.bufs[flq->cidx]; 1072 bufsz = csio_wr_fl_bufsz(sge, buf); 1073 1074 fbuf->paddr = buf->paddr; 1075 fbuf->vaddr = buf->vaddr; 1076 1077 flb.offset = flq->un.fl.offset; 1078 lastlen = min(bufsz, len); 1079 fbuf->len = lastlen; 1080 1081 len -= lastlen; 1082 if (!len) 1083 break; 1084 csio_wr_inval_flq_buf(hw, flq); 1085 } 1086 1087 flb.defer_free = flq->un.fl.packen ? 0 : 1; 1088 1089 iq_handler(hw, wr, q->wr_sz - sizeof(struct csio_iqwr_footer), 1090 &flb, priv); 1091 1092 if (flq->un.fl.packen) 1093 flq->un.fl.offset += ALIGN(lastlen, sge->csio_fl_align); 1094 else 1095 csio_wr_inval_flq_buf(hw, flq); 1096 1097 } 1098 1099 /* 1100 * csio_is_new_iqwr - Is this a new Ingress queue entry ? 1101 * @q: Ingress quueue. 1102 * @ftr: Ingress queue WR SGE footer. 1103 * 1104 * The entry is new if our generation bit matches the corresponding 1105 * bit in the footer of the current WR. 1106 */ 1107 static inline bool 1108 csio_is_new_iqwr(struct csio_q *q, struct csio_iqwr_footer *ftr) 1109 { 1110 return (q->un.iq.genbit == (ftr->u.type_gen >> IQWRF_GEN_SHIFT)); 1111 } 1112 1113 /* 1114 * csio_wr_process_iq - Process elements in Ingress queue. 1115 * @hw: HW pointer 1116 * @qidx: Index of queue 1117 * @iq_handler: Handler for this queue 1118 * @priv: Caller's private pointer 1119 * 1120 * This routine walks through every entry of the ingress queue, calling 1121 * the provided iq_handler with the entry, until the generation bit 1122 * flips. 1123 */ 1124 int 1125 csio_wr_process_iq(struct csio_hw *hw, struct csio_q *q, 1126 void (*iq_handler)(struct csio_hw *, void *, 1127 uint32_t, struct csio_fl_dma_buf *, 1128 void *), 1129 void *priv) 1130 { 1131 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 1132 void *wr = (void *)((uintptr_t)q->vstart + (q->cidx * q->wr_sz)); 1133 struct csio_iqwr_footer *ftr; 1134 uint32_t wr_type, fw_qid, qid; 1135 struct csio_q *q_completed; 1136 struct csio_q *flq = csio_iq_has_fl(q) ? 1137 wrm->q_arr[q->un.iq.flq_idx] : NULL; 1138 int rv = 0; 1139 1140 /* Get the footer */ 1141 ftr = (struct csio_iqwr_footer *)((uintptr_t)wr + 1142 (q->wr_sz - sizeof(*ftr))); 1143 1144 /* 1145 * When q wrapped around last time, driver should have inverted 1146 * ic.genbit as well. 1147 */ 1148 while (csio_is_new_iqwr(q, ftr)) { 1149 1150 CSIO_DB_ASSERT(((uintptr_t)wr + q->wr_sz) <= 1151 (uintptr_t)q->vwrap); 1152 rmb(); 1153 wr_type = IQWRF_TYPE_GET(ftr->u.type_gen); 1154 1155 switch (wr_type) { 1156 case X_RSPD_TYPE_CPL: 1157 /* Subtract footer from WR len */ 1158 iq_handler(hw, wr, q->wr_sz - sizeof(*ftr), NULL, priv); 1159 break; 1160 case X_RSPD_TYPE_FLBUF: 1161 csio_wr_process_fl(hw, q, wr, 1162 ntohl(ftr->pldbuflen_qid), 1163 iq_handler, priv); 1164 break; 1165 case X_RSPD_TYPE_INTR: 1166 fw_qid = ntohl(ftr->pldbuflen_qid); 1167 qid = fw_qid - wrm->fw_iq_start; 1168 q_completed = hw->wrm.intr_map[qid]; 1169 1170 if (unlikely(qid == 1171 csio_q_physiqid(hw, hw->intr_iq_idx))) { 1172 /* 1173 * We are already in the Forward Interrupt 1174 * Interrupt Queue Service! Do-not service 1175 * again! 1176 * 1177 */ 1178 } else { 1179 CSIO_DB_ASSERT(q_completed); 1180 CSIO_DB_ASSERT( 1181 q_completed->un.iq.iq_intx_handler); 1182 1183 /* Call the queue handler. */ 1184 q_completed->un.iq.iq_intx_handler(hw, NULL, 1185 0, NULL, (void *)q_completed); 1186 } 1187 break; 1188 default: 1189 csio_warn(hw, "Unknown resp type 0x%x received\n", 1190 wr_type); 1191 CSIO_INC_STATS(q, n_rsp_unknown); 1192 break; 1193 } 1194 1195 /* 1196 * Ingress *always* has fixed size WR entries. Therefore, 1197 * there should always be complete WRs towards the end of 1198 * queue. 1199 */ 1200 if (((uintptr_t)wr + q->wr_sz) == (uintptr_t)q->vwrap) { 1201 1202 /* Roll over to start of queue */ 1203 q->cidx = 0; 1204 wr = q->vstart; 1205 1206 /* Toggle genbit */ 1207 q->un.iq.genbit ^= 0x1; 1208 1209 CSIO_INC_STATS(q, n_qwrap); 1210 } else { 1211 q->cidx++; 1212 wr = (void *)((uintptr_t)(q->vstart) + 1213 (q->cidx * q->wr_sz)); 1214 } 1215 1216 ftr = (struct csio_iqwr_footer *)((uintptr_t)wr + 1217 (q->wr_sz - sizeof(*ftr))); 1218 q->inc_idx++; 1219 1220 } /* while (q->un.iq.genbit == hdr->genbit) */ 1221 1222 /* 1223 * We need to re-arm SGE interrupts in case we got a stray interrupt, 1224 * especially in msix mode. With INTx, this may be a common occurence. 1225 */ 1226 if (unlikely(!q->inc_idx)) { 1227 CSIO_INC_STATS(q, n_stray_comp); 1228 rv = -EINVAL; 1229 goto restart; 1230 } 1231 1232 /* Replenish free list buffers if pending falls below low water mark */ 1233 if (flq) { 1234 uint32_t avail = csio_wr_avail_qcredits(flq); 1235 if (avail <= 16) { 1236 /* Make sure in FLQ, atleast 1 credit (8 FL buffers) 1237 * remains unpopulated otherwise HW thinks 1238 * FLQ is empty. 1239 */ 1240 csio_wr_update_fl(hw, flq, (flq->credits - 8) - avail); 1241 csio_wr_ring_fldb(hw, flq); 1242 } 1243 } 1244 1245 restart: 1246 /* Now inform SGE about our incremental index value */ 1247 csio_wr_reg32(hw, CIDXINC_V(q->inc_idx) | 1248 INGRESSQID_V(q->un.iq.physiqid) | 1249 TIMERREG_V(csio_sge_timer_reg), 1250 MYPF_REG(SGE_PF_GTS_A)); 1251 q->stats.n_tot_rsps += q->inc_idx; 1252 1253 q->inc_idx = 0; 1254 1255 return rv; 1256 } 1257 1258 int 1259 csio_wr_process_iq_idx(struct csio_hw *hw, int qidx, 1260 void (*iq_handler)(struct csio_hw *, void *, 1261 uint32_t, struct csio_fl_dma_buf *, 1262 void *), 1263 void *priv) 1264 { 1265 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 1266 struct csio_q *iq = wrm->q_arr[qidx]; 1267 1268 return csio_wr_process_iq(hw, iq, iq_handler, priv); 1269 } 1270 1271 static int 1272 csio_closest_timer(struct csio_sge *s, int time) 1273 { 1274 int i, delta, match = 0, min_delta = INT_MAX; 1275 1276 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) { 1277 delta = time - s->timer_val[i]; 1278 if (delta < 0) 1279 delta = -delta; 1280 if (delta < min_delta) { 1281 min_delta = delta; 1282 match = i; 1283 } 1284 } 1285 return match; 1286 } 1287 1288 static int 1289 csio_closest_thresh(struct csio_sge *s, int cnt) 1290 { 1291 int i, delta, match = 0, min_delta = INT_MAX; 1292 1293 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) { 1294 delta = cnt - s->counter_val[i]; 1295 if (delta < 0) 1296 delta = -delta; 1297 if (delta < min_delta) { 1298 min_delta = delta; 1299 match = i; 1300 } 1301 } 1302 return match; 1303 } 1304 1305 static void 1306 csio_wr_fixup_host_params(struct csio_hw *hw) 1307 { 1308 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 1309 struct csio_sge *sge = &wrm->sge; 1310 uint32_t clsz = L1_CACHE_BYTES; 1311 uint32_t s_hps = PAGE_SHIFT - 10; 1312 uint32_t ingpad = 0; 1313 uint32_t stat_len = clsz > 64 ? 128 : 64; 1314 1315 csio_wr_reg32(hw, HOSTPAGESIZEPF0_V(s_hps) | HOSTPAGESIZEPF1_V(s_hps) | 1316 HOSTPAGESIZEPF2_V(s_hps) | HOSTPAGESIZEPF3_V(s_hps) | 1317 HOSTPAGESIZEPF4_V(s_hps) | HOSTPAGESIZEPF5_V(s_hps) | 1318 HOSTPAGESIZEPF6_V(s_hps) | HOSTPAGESIZEPF7_V(s_hps), 1319 SGE_HOST_PAGE_SIZE_A); 1320 1321 sge->csio_fl_align = clsz < 32 ? 32 : clsz; 1322 ingpad = ilog2(sge->csio_fl_align) - 5; 1323 1324 csio_set_reg_field(hw, SGE_CONTROL_A, 1325 INGPADBOUNDARY_V(INGPADBOUNDARY_M) | 1326 EGRSTATUSPAGESIZE_F, 1327 INGPADBOUNDARY_V(ingpad) | 1328 EGRSTATUSPAGESIZE_V(stat_len != 64)); 1329 1330 /* FL BUFFER SIZE#0 is Page size i,e already aligned to cache line */ 1331 csio_wr_reg32(hw, PAGE_SIZE, SGE_FL_BUFFER_SIZE0_A); 1332 1333 /* 1334 * If using hard params, the following will get set correctly 1335 * in csio_wr_set_sge(). 1336 */ 1337 if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS) { 1338 csio_wr_reg32(hw, 1339 (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE2_A) + 1340 sge->csio_fl_align - 1) & ~(sge->csio_fl_align - 1), 1341 SGE_FL_BUFFER_SIZE2_A); 1342 csio_wr_reg32(hw, 1343 (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE3_A) + 1344 sge->csio_fl_align - 1) & ~(sge->csio_fl_align - 1), 1345 SGE_FL_BUFFER_SIZE3_A); 1346 } 1347 1348 csio_wr_reg32(hw, HPZ0_V(PAGE_SHIFT - 12), ULP_RX_TDDP_PSZ_A); 1349 1350 /* default value of rx_dma_offset of the NIC driver */ 1351 csio_set_reg_field(hw, SGE_CONTROL_A, 1352 PKTSHIFT_V(PKTSHIFT_M), 1353 PKTSHIFT_V(CSIO_SGE_RX_DMA_OFFSET)); 1354 1355 csio_hw_tp_wr_bits_indirect(hw, TP_INGRESS_CONFIG_A, 1356 CSUM_HAS_PSEUDO_HDR_F, 0); 1357 } 1358 1359 static void 1360 csio_init_intr_coalesce_parms(struct csio_hw *hw) 1361 { 1362 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 1363 struct csio_sge *sge = &wrm->sge; 1364 1365 csio_sge_thresh_reg = csio_closest_thresh(sge, csio_intr_coalesce_cnt); 1366 if (csio_intr_coalesce_cnt) { 1367 csio_sge_thresh_reg = 0; 1368 csio_sge_timer_reg = X_TIMERREG_RESTART_COUNTER; 1369 return; 1370 } 1371 1372 csio_sge_timer_reg = csio_closest_timer(sge, csio_intr_coalesce_time); 1373 } 1374 1375 /* 1376 * csio_wr_get_sge - Get SGE register values. 1377 * @hw: HW module. 1378 * 1379 * Used by non-master functions and by master-functions relying on config file. 1380 */ 1381 static void 1382 csio_wr_get_sge(struct csio_hw *hw) 1383 { 1384 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 1385 struct csio_sge *sge = &wrm->sge; 1386 uint32_t ingpad; 1387 int i; 1388 u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5; 1389 u32 ingress_rx_threshold; 1390 1391 sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A); 1392 1393 ingpad = INGPADBOUNDARY_G(sge->sge_control); 1394 1395 switch (ingpad) { 1396 case X_INGPCIEBOUNDARY_32B: 1397 sge->csio_fl_align = 32; break; 1398 case X_INGPCIEBOUNDARY_64B: 1399 sge->csio_fl_align = 64; break; 1400 case X_INGPCIEBOUNDARY_128B: 1401 sge->csio_fl_align = 128; break; 1402 case X_INGPCIEBOUNDARY_256B: 1403 sge->csio_fl_align = 256; break; 1404 case X_INGPCIEBOUNDARY_512B: 1405 sge->csio_fl_align = 512; break; 1406 case X_INGPCIEBOUNDARY_1024B: 1407 sge->csio_fl_align = 1024; break; 1408 case X_INGPCIEBOUNDARY_2048B: 1409 sge->csio_fl_align = 2048; break; 1410 case X_INGPCIEBOUNDARY_4096B: 1411 sge->csio_fl_align = 4096; break; 1412 } 1413 1414 for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++) 1415 csio_get_flbuf_size(hw, sge, i); 1416 1417 timer_value_0_and_1 = csio_rd_reg32(hw, SGE_TIMER_VALUE_0_AND_1_A); 1418 timer_value_2_and_3 = csio_rd_reg32(hw, SGE_TIMER_VALUE_2_AND_3_A); 1419 timer_value_4_and_5 = csio_rd_reg32(hw, SGE_TIMER_VALUE_4_AND_5_A); 1420 1421 sge->timer_val[0] = (uint16_t)csio_core_ticks_to_us(hw, 1422 TIMERVALUE0_G(timer_value_0_and_1)); 1423 sge->timer_val[1] = (uint16_t)csio_core_ticks_to_us(hw, 1424 TIMERVALUE1_G(timer_value_0_and_1)); 1425 sge->timer_val[2] = (uint16_t)csio_core_ticks_to_us(hw, 1426 TIMERVALUE2_G(timer_value_2_and_3)); 1427 sge->timer_val[3] = (uint16_t)csio_core_ticks_to_us(hw, 1428 TIMERVALUE3_G(timer_value_2_and_3)); 1429 sge->timer_val[4] = (uint16_t)csio_core_ticks_to_us(hw, 1430 TIMERVALUE4_G(timer_value_4_and_5)); 1431 sge->timer_val[5] = (uint16_t)csio_core_ticks_to_us(hw, 1432 TIMERVALUE5_G(timer_value_4_and_5)); 1433 1434 ingress_rx_threshold = csio_rd_reg32(hw, SGE_INGRESS_RX_THRESHOLD_A); 1435 sge->counter_val[0] = THRESHOLD_0_G(ingress_rx_threshold); 1436 sge->counter_val[1] = THRESHOLD_1_G(ingress_rx_threshold); 1437 sge->counter_val[2] = THRESHOLD_2_G(ingress_rx_threshold); 1438 sge->counter_val[3] = THRESHOLD_3_G(ingress_rx_threshold); 1439 1440 csio_init_intr_coalesce_parms(hw); 1441 } 1442 1443 /* 1444 * csio_wr_set_sge - Initialize SGE registers 1445 * @hw: HW module. 1446 * 1447 * Used by Master function to initialize SGE registers in the absence 1448 * of a config file. 1449 */ 1450 static void 1451 csio_wr_set_sge(struct csio_hw *hw) 1452 { 1453 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 1454 struct csio_sge *sge = &wrm->sge; 1455 int i; 1456 1457 /* 1458 * Set up our basic SGE mode to deliver CPL messages to our Ingress 1459 * Queue and Packet Date to the Free List. 1460 */ 1461 csio_set_reg_field(hw, SGE_CONTROL_A, RXPKTCPLMODE_F, RXPKTCPLMODE_F); 1462 1463 sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A); 1464 1465 /* sge->csio_fl_align is set up by csio_wr_fixup_host_params(). */ 1466 1467 /* 1468 * Set up to drop DOORBELL writes when the DOORBELL FIFO overflows 1469 * and generate an interrupt when this occurs so we can recover. 1470 */ 1471 csio_set_reg_field(hw, SGE_DBFIFO_STATUS_A, 1472 LP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M), 1473 LP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH)); 1474 csio_set_reg_field(hw, SGE_DBFIFO_STATUS2_A, 1475 HP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M), 1476 HP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH)); 1477 1478 csio_set_reg_field(hw, SGE_DOORBELL_CONTROL_A, ENABLE_DROP_F, 1479 ENABLE_DROP_F); 1480 1481 /* SGE_FL_BUFFER_SIZE0 is set up by csio_wr_fixup_host_params(). */ 1482 1483 CSIO_SET_FLBUF_SIZE(hw, 1, CSIO_SGE_FLBUF_SIZE1); 1484 csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE2 + sge->csio_fl_align - 1) 1485 & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE2_A); 1486 csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE3 + sge->csio_fl_align - 1) 1487 & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE3_A); 1488 CSIO_SET_FLBUF_SIZE(hw, 4, CSIO_SGE_FLBUF_SIZE4); 1489 CSIO_SET_FLBUF_SIZE(hw, 5, CSIO_SGE_FLBUF_SIZE5); 1490 CSIO_SET_FLBUF_SIZE(hw, 6, CSIO_SGE_FLBUF_SIZE6); 1491 CSIO_SET_FLBUF_SIZE(hw, 7, CSIO_SGE_FLBUF_SIZE7); 1492 CSIO_SET_FLBUF_SIZE(hw, 8, CSIO_SGE_FLBUF_SIZE8); 1493 1494 for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++) 1495 csio_get_flbuf_size(hw, sge, i); 1496 1497 /* Initialize interrupt coalescing attributes */ 1498 sge->timer_val[0] = CSIO_SGE_TIMER_VAL_0; 1499 sge->timer_val[1] = CSIO_SGE_TIMER_VAL_1; 1500 sge->timer_val[2] = CSIO_SGE_TIMER_VAL_2; 1501 sge->timer_val[3] = CSIO_SGE_TIMER_VAL_3; 1502 sge->timer_val[4] = CSIO_SGE_TIMER_VAL_4; 1503 sge->timer_val[5] = CSIO_SGE_TIMER_VAL_5; 1504 1505 sge->counter_val[0] = CSIO_SGE_INT_CNT_VAL_0; 1506 sge->counter_val[1] = CSIO_SGE_INT_CNT_VAL_1; 1507 sge->counter_val[2] = CSIO_SGE_INT_CNT_VAL_2; 1508 sge->counter_val[3] = CSIO_SGE_INT_CNT_VAL_3; 1509 1510 csio_wr_reg32(hw, THRESHOLD_0_V(sge->counter_val[0]) | 1511 THRESHOLD_1_V(sge->counter_val[1]) | 1512 THRESHOLD_2_V(sge->counter_val[2]) | 1513 THRESHOLD_3_V(sge->counter_val[3]), 1514 SGE_INGRESS_RX_THRESHOLD_A); 1515 1516 csio_wr_reg32(hw, 1517 TIMERVALUE0_V(csio_us_to_core_ticks(hw, sge->timer_val[0])) | 1518 TIMERVALUE1_V(csio_us_to_core_ticks(hw, sge->timer_val[1])), 1519 SGE_TIMER_VALUE_0_AND_1_A); 1520 1521 csio_wr_reg32(hw, 1522 TIMERVALUE2_V(csio_us_to_core_ticks(hw, sge->timer_val[2])) | 1523 TIMERVALUE3_V(csio_us_to_core_ticks(hw, sge->timer_val[3])), 1524 SGE_TIMER_VALUE_2_AND_3_A); 1525 1526 csio_wr_reg32(hw, 1527 TIMERVALUE4_V(csio_us_to_core_ticks(hw, sge->timer_val[4])) | 1528 TIMERVALUE5_V(csio_us_to_core_ticks(hw, sge->timer_val[5])), 1529 SGE_TIMER_VALUE_4_AND_5_A); 1530 1531 csio_init_intr_coalesce_parms(hw); 1532 } 1533 1534 void 1535 csio_wr_sge_init(struct csio_hw *hw) 1536 { 1537 /* 1538 * If we are master and chip is not initialized: 1539 * - If we plan to use the config file, we need to fixup some 1540 * host specific registers, and read the rest of the SGE 1541 * configuration. 1542 * - If we dont plan to use the config file, we need to initialize 1543 * SGE entirely, including fixing the host specific registers. 1544 * If we are master and chip is initialized, just read and work off of 1545 * the already initialized SGE values. 1546 * If we arent the master, we are only allowed to read and work off of 1547 * the already initialized SGE values. 1548 * 1549 * Therefore, before calling this function, we assume that the master- 1550 * ship of the card, state and whether to use config file or not, have 1551 * already been decided. 1552 */ 1553 if (csio_is_hw_master(hw)) { 1554 if (hw->fw_state != CSIO_DEV_STATE_INIT) 1555 csio_wr_fixup_host_params(hw); 1556 1557 if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS) 1558 csio_wr_get_sge(hw); 1559 else 1560 csio_wr_set_sge(hw); 1561 } else 1562 csio_wr_get_sge(hw); 1563 } 1564 1565 /* 1566 * csio_wrm_init - Initialize Work request module. 1567 * @wrm: WR module 1568 * @hw: HW pointer 1569 * 1570 * Allocates memory for an array of queue pointers starting at q_arr. 1571 */ 1572 int 1573 csio_wrm_init(struct csio_wrm *wrm, struct csio_hw *hw) 1574 { 1575 int i; 1576 1577 if (!wrm->num_q) { 1578 csio_err(hw, "Num queues is not set\n"); 1579 return -EINVAL; 1580 } 1581 1582 wrm->q_arr = kcalloc(wrm->num_q, sizeof(struct csio_q *), GFP_KERNEL); 1583 if (!wrm->q_arr) 1584 goto err; 1585 1586 for (i = 0; i < wrm->num_q; i++) { 1587 wrm->q_arr[i] = kzalloc(sizeof(struct csio_q), GFP_KERNEL); 1588 if (!wrm->q_arr[i]) { 1589 while (--i >= 0) 1590 kfree(wrm->q_arr[i]); 1591 goto err_free_arr; 1592 } 1593 } 1594 wrm->free_qidx = 0; 1595 1596 return 0; 1597 1598 err_free_arr: 1599 kfree(wrm->q_arr); 1600 err: 1601 return -ENOMEM; 1602 } 1603 1604 /* 1605 * csio_wrm_exit - Initialize Work request module. 1606 * @wrm: WR module 1607 * @hw: HW module 1608 * 1609 * Uninitialize WR module. Free q_arr and pointers in it. 1610 * We have the additional job of freeing the DMA memory associated 1611 * with the queues. 1612 */ 1613 void 1614 csio_wrm_exit(struct csio_wrm *wrm, struct csio_hw *hw) 1615 { 1616 int i; 1617 uint32_t j; 1618 struct csio_q *q; 1619 struct csio_dma_buf *buf; 1620 1621 for (i = 0; i < wrm->num_q; i++) { 1622 q = wrm->q_arr[i]; 1623 1624 if (wrm->free_qidx && (i < wrm->free_qidx)) { 1625 if (q->type == CSIO_FREELIST) { 1626 if (!q->un.fl.bufs) 1627 continue; 1628 for (j = 0; j < q->credits; j++) { 1629 buf = &q->un.fl.bufs[j]; 1630 if (!buf->vaddr) 1631 continue; 1632 pci_free_consistent(hw->pdev, buf->len, 1633 buf->vaddr, 1634 buf->paddr); 1635 } 1636 kfree(q->un.fl.bufs); 1637 } 1638 pci_free_consistent(hw->pdev, q->size, 1639 q->vstart, q->pstart); 1640 } 1641 kfree(q); 1642 } 1643 1644 hw->flags &= ~CSIO_HWF_Q_MEM_ALLOCED; 1645 1646 kfree(wrm->q_arr); 1647 } 1648