1 /* 2 * This file is part of the Chelsio FCoE driver for Linux. 3 * 4 * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include <linux/kernel.h> 36 #include <linux/string.h> 37 #include <linux/compiler.h> 38 #include <linux/slab.h> 39 #include <asm/page.h> 40 #include <linux/cache.h> 41 42 #include "csio_hw.h" 43 #include "csio_wr.h" 44 #include "csio_mb.h" 45 #include "csio_defs.h" 46 47 int csio_intr_coalesce_cnt; /* value:SGE_INGRESS_RX_THRESHOLD[0] */ 48 static int csio_sge_thresh_reg; /* SGE_INGRESS_RX_THRESHOLD[0] */ 49 50 int csio_intr_coalesce_time = 10; /* value:SGE_TIMER_VALUE_1 */ 51 static int csio_sge_timer_reg = 1; 52 53 #define CSIO_SET_FLBUF_SIZE(_hw, _reg, _val) \ 54 csio_wr_reg32((_hw), (_val), SGE_FL_BUFFER_SIZE##_reg) 55 56 static void 57 csio_get_flbuf_size(struct csio_hw *hw, struct csio_sge *sge, uint32_t reg) 58 { 59 sge->sge_fl_buf_size[reg] = csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE0 + 60 reg * sizeof(uint32_t)); 61 } 62 63 /* Free list buffer size */ 64 static inline uint32_t 65 csio_wr_fl_bufsz(struct csio_sge *sge, struct csio_dma_buf *buf) 66 { 67 return sge->sge_fl_buf_size[buf->paddr & 0xF]; 68 } 69 70 /* Size of the egress queue status page */ 71 static inline uint32_t 72 csio_wr_qstat_pgsz(struct csio_hw *hw) 73 { 74 return (hw->wrm.sge.sge_control & EGRSTATUSPAGESIZE(1)) ? 128 : 64; 75 } 76 77 /* Ring freelist doorbell */ 78 static inline void 79 csio_wr_ring_fldb(struct csio_hw *hw, struct csio_q *flq) 80 { 81 /* 82 * Ring the doorbell only when we have atleast CSIO_QCREDIT_SZ 83 * number of bytes in the freelist queue. This translates to atleast 84 * 8 freelist buffer pointers (since each pointer is 8 bytes). 85 */ 86 if (flq->inc_idx >= 8) { 87 csio_wr_reg32(hw, DBPRIO(1) | QID(flq->un.fl.flid) | 88 PIDX(flq->inc_idx / 8), 89 MYPF_REG(SGE_PF_KDOORBELL)); 90 flq->inc_idx &= 7; 91 } 92 } 93 94 /* Write a 0 cidx increment value to enable SGE interrupts for this queue */ 95 static void 96 csio_wr_sge_intr_enable(struct csio_hw *hw, uint16_t iqid) 97 { 98 csio_wr_reg32(hw, CIDXINC(0) | 99 INGRESSQID(iqid) | 100 TIMERREG(X_TIMERREG_RESTART_COUNTER), 101 MYPF_REG(SGE_PF_GTS)); 102 } 103 104 /* 105 * csio_wr_fill_fl - Populate the FL buffers of a FL queue. 106 * @hw: HW module. 107 * @flq: Freelist queue. 108 * 109 * Fill up freelist buffer entries with buffers of size specified 110 * in the size register. 111 * 112 */ 113 static int 114 csio_wr_fill_fl(struct csio_hw *hw, struct csio_q *flq) 115 { 116 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 117 struct csio_sge *sge = &wrm->sge; 118 __be64 *d = (__be64 *)(flq->vstart); 119 struct csio_dma_buf *buf = &flq->un.fl.bufs[0]; 120 uint64_t paddr; 121 int sreg = flq->un.fl.sreg; 122 int n = flq->credits; 123 124 while (n--) { 125 buf->len = sge->sge_fl_buf_size[sreg]; 126 buf->vaddr = pci_alloc_consistent(hw->pdev, buf->len, 127 &buf->paddr); 128 if (!buf->vaddr) { 129 csio_err(hw, "Could only fill %d buffers!\n", n + 1); 130 return -ENOMEM; 131 } 132 133 paddr = buf->paddr | (sreg & 0xF); 134 135 *d++ = cpu_to_be64(paddr); 136 buf++; 137 } 138 139 return 0; 140 } 141 142 /* 143 * csio_wr_update_fl - 144 * @hw: HW module. 145 * @flq: Freelist queue. 146 * 147 * 148 */ 149 static inline void 150 csio_wr_update_fl(struct csio_hw *hw, struct csio_q *flq, uint16_t n) 151 { 152 153 flq->inc_idx += n; 154 flq->pidx += n; 155 if (unlikely(flq->pidx >= flq->credits)) 156 flq->pidx -= (uint16_t)flq->credits; 157 158 CSIO_INC_STATS(flq, n_flq_refill); 159 } 160 161 /* 162 * csio_wr_alloc_q - Allocate a WR queue and initialize it. 163 * @hw: HW module 164 * @qsize: Size of the queue in bytes 165 * @wrsize: Since of WR in this queue, if fixed. 166 * @type: Type of queue (Ingress/Egress/Freelist) 167 * @owner: Module that owns this queue. 168 * @nflb: Number of freelist buffers for FL. 169 * @sreg: What is the FL buffer size register? 170 * @iq_int_handler: Ingress queue handler in INTx mode. 171 * 172 * This function allocates and sets up a queue for the caller 173 * of size qsize, aligned at the required boundary. This is subject to 174 * be free entries being available in the queue array. If one is found, 175 * it is initialized with the allocated queue, marked as being used (owner), 176 * and a handle returned to the caller in form of the queue's index 177 * into the q_arr array. 178 * If user has indicated a freelist (by specifying nflb > 0), create 179 * another queue (with its own index into q_arr) for the freelist. Allocate 180 * memory for DMA buffer metadata (vaddr, len etc). Save off the freelist 181 * idx in the ingress queue's flq.idx. This is how a Freelist is associated 182 * with its owning ingress queue. 183 */ 184 int 185 csio_wr_alloc_q(struct csio_hw *hw, uint32_t qsize, uint32_t wrsize, 186 uint16_t type, void *owner, uint32_t nflb, int sreg, 187 iq_handler_t iq_intx_handler) 188 { 189 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 190 struct csio_q *q, *flq; 191 int free_idx = wrm->free_qidx; 192 int ret_idx = free_idx; 193 uint32_t qsz; 194 int flq_idx; 195 196 if (free_idx >= wrm->num_q) { 197 csio_err(hw, "No more free queues.\n"); 198 return -1; 199 } 200 201 switch (type) { 202 case CSIO_EGRESS: 203 qsz = ALIGN(qsize, CSIO_QCREDIT_SZ) + csio_wr_qstat_pgsz(hw); 204 break; 205 case CSIO_INGRESS: 206 switch (wrsize) { 207 case 16: 208 case 32: 209 case 64: 210 case 128: 211 break; 212 default: 213 csio_err(hw, "Invalid Ingress queue WR size:%d\n", 214 wrsize); 215 return -1; 216 } 217 218 /* 219 * Number of elements must be a multiple of 16 220 * So this includes status page size 221 */ 222 qsz = ALIGN(qsize/wrsize, 16) * wrsize; 223 224 break; 225 case CSIO_FREELIST: 226 qsz = ALIGN(qsize/wrsize, 8) * wrsize + csio_wr_qstat_pgsz(hw); 227 break; 228 default: 229 csio_err(hw, "Invalid queue type: 0x%x\n", type); 230 return -1; 231 } 232 233 q = wrm->q_arr[free_idx]; 234 235 q->vstart = pci_alloc_consistent(hw->pdev, qsz, &q->pstart); 236 if (!q->vstart) { 237 csio_err(hw, 238 "Failed to allocate DMA memory for " 239 "queue at id: %d size: %d\n", free_idx, qsize); 240 return -1; 241 } 242 243 /* 244 * We need to zero out the contents, importantly for ingress, 245 * since we start with a generatiom bit of 1 for ingress. 246 */ 247 memset(q->vstart, 0, qsz); 248 249 q->type = type; 250 q->owner = owner; 251 q->pidx = q->cidx = q->inc_idx = 0; 252 q->size = qsz; 253 q->wr_sz = wrsize; /* If using fixed size WRs */ 254 255 wrm->free_qidx++; 256 257 if (type == CSIO_INGRESS) { 258 /* Since queue area is set to zero */ 259 q->un.iq.genbit = 1; 260 261 /* 262 * Ingress queue status page size is always the size of 263 * the ingress queue entry. 264 */ 265 q->credits = (qsz - q->wr_sz) / q->wr_sz; 266 q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz 267 - q->wr_sz); 268 269 /* Allocate memory for FL if requested */ 270 if (nflb > 0) { 271 flq_idx = csio_wr_alloc_q(hw, nflb * sizeof(__be64), 272 sizeof(__be64), CSIO_FREELIST, 273 owner, 0, sreg, NULL); 274 if (flq_idx == -1) { 275 csio_err(hw, 276 "Failed to allocate FL queue" 277 " for IQ idx:%d\n", free_idx); 278 return -1; 279 } 280 281 /* Associate the new FL with the Ingress quue */ 282 q->un.iq.flq_idx = flq_idx; 283 284 flq = wrm->q_arr[q->un.iq.flq_idx]; 285 flq->un.fl.bufs = kzalloc(flq->credits * 286 sizeof(struct csio_dma_buf), 287 GFP_KERNEL); 288 if (!flq->un.fl.bufs) { 289 csio_err(hw, 290 "Failed to allocate FL queue bufs" 291 " for IQ idx:%d\n", free_idx); 292 return -1; 293 } 294 295 flq->un.fl.packen = 0; 296 flq->un.fl.offset = 0; 297 flq->un.fl.sreg = sreg; 298 299 /* Fill up the free list buffers */ 300 if (csio_wr_fill_fl(hw, flq)) 301 return -1; 302 303 /* 304 * Make sure in a FLQ, atleast 1 credit (8 FL buffers) 305 * remains unpopulated,otherwise HW thinks 306 * FLQ is empty. 307 */ 308 flq->pidx = flq->inc_idx = flq->credits - 8; 309 } else { 310 q->un.iq.flq_idx = -1; 311 } 312 313 /* Associate the IQ INTx handler. */ 314 q->un.iq.iq_intx_handler = iq_intx_handler; 315 316 csio_q_iqid(hw, ret_idx) = CSIO_MAX_QID; 317 318 } else if (type == CSIO_EGRESS) { 319 q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / CSIO_QCREDIT_SZ; 320 q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz 321 - csio_wr_qstat_pgsz(hw)); 322 csio_q_eqid(hw, ret_idx) = CSIO_MAX_QID; 323 } else { /* Freelist */ 324 q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / sizeof(__be64); 325 q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz 326 - csio_wr_qstat_pgsz(hw)); 327 csio_q_flid(hw, ret_idx) = CSIO_MAX_QID; 328 } 329 330 return ret_idx; 331 } 332 333 /* 334 * csio_wr_iq_create_rsp - Response handler for IQ creation. 335 * @hw: The HW module. 336 * @mbp: Mailbox. 337 * @iq_idx: Ingress queue that got created. 338 * 339 * Handle FW_IQ_CMD mailbox completion. Save off the assigned IQ/FL ids. 340 */ 341 static int 342 csio_wr_iq_create_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx) 343 { 344 struct csio_iq_params iqp; 345 enum fw_retval retval; 346 uint32_t iq_id; 347 int flq_idx; 348 349 memset(&iqp, 0, sizeof(struct csio_iq_params)); 350 351 csio_mb_iq_alloc_write_rsp(hw, mbp, &retval, &iqp); 352 353 if (retval != FW_SUCCESS) { 354 csio_err(hw, "IQ cmd returned 0x%x!\n", retval); 355 mempool_free(mbp, hw->mb_mempool); 356 return -EINVAL; 357 } 358 359 csio_q_iqid(hw, iq_idx) = iqp.iqid; 360 csio_q_physiqid(hw, iq_idx) = iqp.physiqid; 361 csio_q_pidx(hw, iq_idx) = csio_q_cidx(hw, iq_idx) = 0; 362 csio_q_inc_idx(hw, iq_idx) = 0; 363 364 /* Actual iq-id. */ 365 iq_id = iqp.iqid - hw->wrm.fw_iq_start; 366 367 /* Set the iq-id to iq map table. */ 368 if (iq_id >= CSIO_MAX_IQ) { 369 csio_err(hw, 370 "Exceeding MAX_IQ(%d) supported!" 371 " iqid:%d rel_iqid:%d FW iq_start:%d\n", 372 CSIO_MAX_IQ, iq_id, iqp.iqid, hw->wrm.fw_iq_start); 373 mempool_free(mbp, hw->mb_mempool); 374 return -EINVAL; 375 } 376 csio_q_set_intr_map(hw, iq_idx, iq_id); 377 378 /* 379 * During FW_IQ_CMD, FW sets interrupt_sent bit to 1 in the SGE 380 * ingress context of this queue. This will block interrupts to 381 * this queue until the next GTS write. Therefore, we do a 382 * 0-cidx increment GTS write for this queue just to clear the 383 * interrupt_sent bit. This will re-enable interrupts to this 384 * queue. 385 */ 386 csio_wr_sge_intr_enable(hw, iqp.physiqid); 387 388 flq_idx = csio_q_iq_flq_idx(hw, iq_idx); 389 if (flq_idx != -1) { 390 struct csio_q *flq = hw->wrm.q_arr[flq_idx]; 391 392 csio_q_flid(hw, flq_idx) = iqp.fl0id; 393 csio_q_cidx(hw, flq_idx) = 0; 394 csio_q_pidx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8; 395 csio_q_inc_idx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8; 396 397 /* Now update SGE about the buffers allocated during init */ 398 csio_wr_ring_fldb(hw, flq); 399 } 400 401 mempool_free(mbp, hw->mb_mempool); 402 403 return 0; 404 } 405 406 /* 407 * csio_wr_iq_create - Configure an Ingress queue with FW. 408 * @hw: The HW module. 409 * @priv: Private data object. 410 * @iq_idx: Ingress queue index in the WR module. 411 * @vec: MSIX vector. 412 * @portid: PCIE Channel to be associated with this queue. 413 * @async: Is this a FW asynchronous message handling queue? 414 * @cbfn: Completion callback. 415 * 416 * This API configures an ingress queue with FW by issuing a FW_IQ_CMD mailbox 417 * with alloc/write bits set. 418 */ 419 int 420 csio_wr_iq_create(struct csio_hw *hw, void *priv, int iq_idx, 421 uint32_t vec, uint8_t portid, bool async, 422 void (*cbfn) (struct csio_hw *, struct csio_mb *)) 423 { 424 struct csio_mb *mbp; 425 struct csio_iq_params iqp; 426 int flq_idx; 427 428 memset(&iqp, 0, sizeof(struct csio_iq_params)); 429 csio_q_portid(hw, iq_idx) = portid; 430 431 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC); 432 if (!mbp) { 433 csio_err(hw, "IQ command out of memory!\n"); 434 return -ENOMEM; 435 } 436 437 switch (hw->intr_mode) { 438 case CSIO_IM_INTX: 439 case CSIO_IM_MSI: 440 /* For interrupt forwarding queue only */ 441 if (hw->intr_iq_idx == iq_idx) 442 iqp.iqandst = X_INTERRUPTDESTINATION_PCIE; 443 else 444 iqp.iqandst = X_INTERRUPTDESTINATION_IQ; 445 iqp.iqandstindex = 446 csio_q_physiqid(hw, hw->intr_iq_idx); 447 break; 448 case CSIO_IM_MSIX: 449 iqp.iqandst = X_INTERRUPTDESTINATION_PCIE; 450 iqp.iqandstindex = (uint16_t)vec; 451 break; 452 case CSIO_IM_NONE: 453 mempool_free(mbp, hw->mb_mempool); 454 return -EINVAL; 455 } 456 457 /* Pass in the ingress queue cmd parameters */ 458 iqp.pfn = hw->pfn; 459 iqp.vfn = 0; 460 iqp.iq_start = 1; 461 iqp.viid = 0; 462 iqp.type = FW_IQ_TYPE_FL_INT_CAP; 463 iqp.iqasynch = async; 464 if (csio_intr_coalesce_cnt) 465 iqp.iqanus = X_UPDATESCHEDULING_COUNTER_OPTTIMER; 466 else 467 iqp.iqanus = X_UPDATESCHEDULING_TIMER; 468 iqp.iqanud = X_UPDATEDELIVERY_INTERRUPT; 469 iqp.iqpciech = portid; 470 iqp.iqintcntthresh = (uint8_t)csio_sge_thresh_reg; 471 472 switch (csio_q_wr_sz(hw, iq_idx)) { 473 case 16: 474 iqp.iqesize = 0; break; 475 case 32: 476 iqp.iqesize = 1; break; 477 case 64: 478 iqp.iqesize = 2; break; 479 case 128: 480 iqp.iqesize = 3; break; 481 } 482 483 iqp.iqsize = csio_q_size(hw, iq_idx) / 484 csio_q_wr_sz(hw, iq_idx); 485 iqp.iqaddr = csio_q_pstart(hw, iq_idx); 486 487 flq_idx = csio_q_iq_flq_idx(hw, iq_idx); 488 if (flq_idx != -1) { 489 struct csio_q *flq = hw->wrm.q_arr[flq_idx]; 490 491 iqp.fl0paden = 1; 492 iqp.fl0packen = flq->un.fl.packen ? 1 : 0; 493 iqp.fl0fbmin = X_FETCHBURSTMIN_64B; 494 iqp.fl0fbmax = X_FETCHBURSTMAX_512B; 495 iqp.fl0size = csio_q_size(hw, flq_idx) / CSIO_QCREDIT_SZ; 496 iqp.fl0addr = csio_q_pstart(hw, flq_idx); 497 } 498 499 csio_mb_iq_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn); 500 501 if (csio_mb_issue(hw, mbp)) { 502 csio_err(hw, "Issue of IQ cmd failed!\n"); 503 mempool_free(mbp, hw->mb_mempool); 504 return -EINVAL; 505 } 506 507 if (cbfn != NULL) 508 return 0; 509 510 return csio_wr_iq_create_rsp(hw, mbp, iq_idx); 511 } 512 513 /* 514 * csio_wr_eq_create_rsp - Response handler for EQ creation. 515 * @hw: The HW module. 516 * @mbp: Mailbox. 517 * @eq_idx: Egress queue that got created. 518 * 519 * Handle FW_EQ_OFLD_CMD mailbox completion. Save off the assigned EQ ids. 520 */ 521 static int 522 csio_wr_eq_cfg_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx) 523 { 524 struct csio_eq_params eqp; 525 enum fw_retval retval; 526 527 memset(&eqp, 0, sizeof(struct csio_eq_params)); 528 529 csio_mb_eq_ofld_alloc_write_rsp(hw, mbp, &retval, &eqp); 530 531 if (retval != FW_SUCCESS) { 532 csio_err(hw, "EQ OFLD cmd returned 0x%x!\n", retval); 533 mempool_free(mbp, hw->mb_mempool); 534 return -EINVAL; 535 } 536 537 csio_q_eqid(hw, eq_idx) = (uint16_t)eqp.eqid; 538 csio_q_physeqid(hw, eq_idx) = (uint16_t)eqp.physeqid; 539 csio_q_pidx(hw, eq_idx) = csio_q_cidx(hw, eq_idx) = 0; 540 csio_q_inc_idx(hw, eq_idx) = 0; 541 542 mempool_free(mbp, hw->mb_mempool); 543 544 return 0; 545 } 546 547 /* 548 * csio_wr_eq_create - Configure an Egress queue with FW. 549 * @hw: HW module. 550 * @priv: Private data. 551 * @eq_idx: Egress queue index in the WR module. 552 * @iq_idx: Associated ingress queue index. 553 * @cbfn: Completion callback. 554 * 555 * This API configures a offload egress queue with FW by issuing a 556 * FW_EQ_OFLD_CMD (with alloc + write ) mailbox. 557 */ 558 int 559 csio_wr_eq_create(struct csio_hw *hw, void *priv, int eq_idx, 560 int iq_idx, uint8_t portid, 561 void (*cbfn) (struct csio_hw *, struct csio_mb *)) 562 { 563 struct csio_mb *mbp; 564 struct csio_eq_params eqp; 565 566 memset(&eqp, 0, sizeof(struct csio_eq_params)); 567 568 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC); 569 if (!mbp) { 570 csio_err(hw, "EQ command out of memory!\n"); 571 return -ENOMEM; 572 } 573 574 eqp.pfn = hw->pfn; 575 eqp.vfn = 0; 576 eqp.eqstart = 1; 577 eqp.hostfcmode = X_HOSTFCMODE_STATUS_PAGE; 578 eqp.iqid = csio_q_iqid(hw, iq_idx); 579 eqp.fbmin = X_FETCHBURSTMIN_64B; 580 eqp.fbmax = X_FETCHBURSTMAX_512B; 581 eqp.cidxfthresh = 0; 582 eqp.pciechn = portid; 583 eqp.eqsize = csio_q_size(hw, eq_idx) / CSIO_QCREDIT_SZ; 584 eqp.eqaddr = csio_q_pstart(hw, eq_idx); 585 586 csio_mb_eq_ofld_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, 587 &eqp, cbfn); 588 589 if (csio_mb_issue(hw, mbp)) { 590 csio_err(hw, "Issue of EQ OFLD cmd failed!\n"); 591 mempool_free(mbp, hw->mb_mempool); 592 return -EINVAL; 593 } 594 595 if (cbfn != NULL) 596 return 0; 597 598 return csio_wr_eq_cfg_rsp(hw, mbp, eq_idx); 599 } 600 601 /* 602 * csio_wr_iq_destroy_rsp - Response handler for IQ removal. 603 * @hw: The HW module. 604 * @mbp: Mailbox. 605 * @iq_idx: Ingress queue that was freed. 606 * 607 * Handle FW_IQ_CMD (free) mailbox completion. 608 */ 609 static int 610 csio_wr_iq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx) 611 { 612 enum fw_retval retval = csio_mb_fw_retval(mbp); 613 int rv = 0; 614 615 if (retval != FW_SUCCESS) 616 rv = -EINVAL; 617 618 mempool_free(mbp, hw->mb_mempool); 619 620 return rv; 621 } 622 623 /* 624 * csio_wr_iq_destroy - Free an ingress queue. 625 * @hw: The HW module. 626 * @priv: Private data object. 627 * @iq_idx: Ingress queue index to destroy 628 * @cbfn: Completion callback. 629 * 630 * This API frees an ingress queue by issuing the FW_IQ_CMD 631 * with the free bit set. 632 */ 633 static int 634 csio_wr_iq_destroy(struct csio_hw *hw, void *priv, int iq_idx, 635 void (*cbfn)(struct csio_hw *, struct csio_mb *)) 636 { 637 int rv = 0; 638 struct csio_mb *mbp; 639 struct csio_iq_params iqp; 640 int flq_idx; 641 642 memset(&iqp, 0, sizeof(struct csio_iq_params)); 643 644 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC); 645 if (!mbp) 646 return -ENOMEM; 647 648 iqp.pfn = hw->pfn; 649 iqp.vfn = 0; 650 iqp.iqid = csio_q_iqid(hw, iq_idx); 651 iqp.type = FW_IQ_TYPE_FL_INT_CAP; 652 653 flq_idx = csio_q_iq_flq_idx(hw, iq_idx); 654 if (flq_idx != -1) 655 iqp.fl0id = csio_q_flid(hw, flq_idx); 656 else 657 iqp.fl0id = 0xFFFF; 658 659 iqp.fl1id = 0xFFFF; 660 661 csio_mb_iq_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn); 662 663 rv = csio_mb_issue(hw, mbp); 664 if (rv != 0) { 665 mempool_free(mbp, hw->mb_mempool); 666 return rv; 667 } 668 669 if (cbfn != NULL) 670 return 0; 671 672 return csio_wr_iq_destroy_rsp(hw, mbp, iq_idx); 673 } 674 675 /* 676 * csio_wr_eq_destroy_rsp - Response handler for OFLD EQ creation. 677 * @hw: The HW module. 678 * @mbp: Mailbox. 679 * @eq_idx: Egress queue that was freed. 680 * 681 * Handle FW_OFLD_EQ_CMD (free) mailbox completion. 682 */ 683 static int 684 csio_wr_eq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx) 685 { 686 enum fw_retval retval = csio_mb_fw_retval(mbp); 687 int rv = 0; 688 689 if (retval != FW_SUCCESS) 690 rv = -EINVAL; 691 692 mempool_free(mbp, hw->mb_mempool); 693 694 return rv; 695 } 696 697 /* 698 * csio_wr_eq_destroy - Free an Egress queue. 699 * @hw: The HW module. 700 * @priv: Private data object. 701 * @eq_idx: Egress queue index to destroy 702 * @cbfn: Completion callback. 703 * 704 * This API frees an Egress queue by issuing the FW_EQ_OFLD_CMD 705 * with the free bit set. 706 */ 707 static int 708 csio_wr_eq_destroy(struct csio_hw *hw, void *priv, int eq_idx, 709 void (*cbfn) (struct csio_hw *, struct csio_mb *)) 710 { 711 int rv = 0; 712 struct csio_mb *mbp; 713 struct csio_eq_params eqp; 714 715 memset(&eqp, 0, sizeof(struct csio_eq_params)); 716 717 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC); 718 if (!mbp) 719 return -ENOMEM; 720 721 eqp.pfn = hw->pfn; 722 eqp.vfn = 0; 723 eqp.eqid = csio_q_eqid(hw, eq_idx); 724 725 csio_mb_eq_ofld_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &eqp, cbfn); 726 727 rv = csio_mb_issue(hw, mbp); 728 if (rv != 0) { 729 mempool_free(mbp, hw->mb_mempool); 730 return rv; 731 } 732 733 if (cbfn != NULL) 734 return 0; 735 736 return csio_wr_eq_destroy_rsp(hw, mbp, eq_idx); 737 } 738 739 /* 740 * csio_wr_cleanup_eq_stpg - Cleanup Egress queue status page 741 * @hw: HW module 742 * @qidx: Egress queue index 743 * 744 * Cleanup the Egress queue status page. 745 */ 746 static void 747 csio_wr_cleanup_eq_stpg(struct csio_hw *hw, int qidx) 748 { 749 struct csio_q *q = csio_hw_to_wrm(hw)->q_arr[qidx]; 750 struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap; 751 752 memset(stp, 0, sizeof(*stp)); 753 } 754 755 /* 756 * csio_wr_cleanup_iq_ftr - Cleanup Footer entries in IQ 757 * @hw: HW module 758 * @qidx: Ingress queue index 759 * 760 * Cleanup the footer entries in the given ingress queue, 761 * set to 1 the internal copy of genbit. 762 */ 763 static void 764 csio_wr_cleanup_iq_ftr(struct csio_hw *hw, int qidx) 765 { 766 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 767 struct csio_q *q = wrm->q_arr[qidx]; 768 void *wr; 769 struct csio_iqwr_footer *ftr; 770 uint32_t i = 0; 771 772 /* set to 1 since we are just about zero out genbit */ 773 q->un.iq.genbit = 1; 774 775 for (i = 0; i < q->credits; i++) { 776 /* Get the WR */ 777 wr = (void *)((uintptr_t)q->vstart + 778 (i * q->wr_sz)); 779 /* Get the footer */ 780 ftr = (struct csio_iqwr_footer *)((uintptr_t)wr + 781 (q->wr_sz - sizeof(*ftr))); 782 /* Zero out footer */ 783 memset(ftr, 0, sizeof(*ftr)); 784 } 785 } 786 787 int 788 csio_wr_destroy_queues(struct csio_hw *hw, bool cmd) 789 { 790 int i, flq_idx; 791 struct csio_q *q; 792 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 793 int rv; 794 795 for (i = 0; i < wrm->free_qidx; i++) { 796 q = wrm->q_arr[i]; 797 798 switch (q->type) { 799 case CSIO_EGRESS: 800 if (csio_q_eqid(hw, i) != CSIO_MAX_QID) { 801 csio_wr_cleanup_eq_stpg(hw, i); 802 if (!cmd) { 803 csio_q_eqid(hw, i) = CSIO_MAX_QID; 804 continue; 805 } 806 807 rv = csio_wr_eq_destroy(hw, NULL, i, NULL); 808 if ((rv == -EBUSY) || (rv == -ETIMEDOUT)) 809 cmd = false; 810 811 csio_q_eqid(hw, i) = CSIO_MAX_QID; 812 } 813 case CSIO_INGRESS: 814 if (csio_q_iqid(hw, i) != CSIO_MAX_QID) { 815 csio_wr_cleanup_iq_ftr(hw, i); 816 if (!cmd) { 817 csio_q_iqid(hw, i) = CSIO_MAX_QID; 818 flq_idx = csio_q_iq_flq_idx(hw, i); 819 if (flq_idx != -1) 820 csio_q_flid(hw, flq_idx) = 821 CSIO_MAX_QID; 822 continue; 823 } 824 825 rv = csio_wr_iq_destroy(hw, NULL, i, NULL); 826 if ((rv == -EBUSY) || (rv == -ETIMEDOUT)) 827 cmd = false; 828 829 csio_q_iqid(hw, i) = CSIO_MAX_QID; 830 flq_idx = csio_q_iq_flq_idx(hw, i); 831 if (flq_idx != -1) 832 csio_q_flid(hw, flq_idx) = CSIO_MAX_QID; 833 } 834 default: 835 break; 836 } 837 } 838 839 hw->flags &= ~CSIO_HWF_Q_FW_ALLOCED; 840 841 return 0; 842 } 843 844 /* 845 * csio_wr_get - Get requested size of WR entry/entries from queue. 846 * @hw: HW module. 847 * @qidx: Index of queue. 848 * @size: Cumulative size of Work request(s). 849 * @wrp: Work request pair. 850 * 851 * If requested credits are available, return the start address of the 852 * work request in the work request pair. Set pidx accordingly and 853 * return. 854 * 855 * NOTE about WR pair: 856 * ================== 857 * A WR can start towards the end of a queue, and then continue at the 858 * beginning, since the queue is considered to be circular. This will 859 * require a pair of address/size to be passed back to the caller - 860 * hence Work request pair format. 861 */ 862 int 863 csio_wr_get(struct csio_hw *hw, int qidx, uint32_t size, 864 struct csio_wr_pair *wrp) 865 { 866 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 867 struct csio_q *q = wrm->q_arr[qidx]; 868 void *cwr = (void *)((uintptr_t)(q->vstart) + 869 (q->pidx * CSIO_QCREDIT_SZ)); 870 struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap; 871 uint16_t cidx = q->cidx = ntohs(stp->cidx); 872 uint16_t pidx = q->pidx; 873 uint32_t req_sz = ALIGN(size, CSIO_QCREDIT_SZ); 874 int req_credits = req_sz / CSIO_QCREDIT_SZ; 875 int credits; 876 877 CSIO_DB_ASSERT(q->owner != NULL); 878 CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx)); 879 CSIO_DB_ASSERT(cidx <= q->credits); 880 881 /* Calculate credits */ 882 if (pidx > cidx) { 883 credits = q->credits - (pidx - cidx) - 1; 884 } else if (cidx > pidx) { 885 credits = cidx - pidx - 1; 886 } else { 887 /* cidx == pidx, empty queue */ 888 credits = q->credits; 889 CSIO_INC_STATS(q, n_qempty); 890 } 891 892 /* 893 * Check if we have enough credits. 894 * credits = 1 implies queue is full. 895 */ 896 if (!credits || (req_credits > credits)) { 897 CSIO_INC_STATS(q, n_qfull); 898 return -EBUSY; 899 } 900 901 /* 902 * If we are here, we have enough credits to satisfy the 903 * request. Check if we are near the end of q, and if WR spills over. 904 * If it does, use the first addr/size to cover the queue until 905 * the end. Fit the remainder portion of the request at the top 906 * of queue and return it in the second addr/len. Set pidx 907 * accordingly. 908 */ 909 if (unlikely(((uintptr_t)cwr + req_sz) > (uintptr_t)(q->vwrap))) { 910 wrp->addr1 = cwr; 911 wrp->size1 = (uint32_t)((uintptr_t)q->vwrap - (uintptr_t)cwr); 912 wrp->addr2 = q->vstart; 913 wrp->size2 = req_sz - wrp->size1; 914 q->pidx = (uint16_t)(ALIGN(wrp->size2, CSIO_QCREDIT_SZ) / 915 CSIO_QCREDIT_SZ); 916 CSIO_INC_STATS(q, n_qwrap); 917 CSIO_INC_STATS(q, n_eq_wr_split); 918 } else { 919 wrp->addr1 = cwr; 920 wrp->size1 = req_sz; 921 wrp->addr2 = NULL; 922 wrp->size2 = 0; 923 q->pidx += (uint16_t)req_credits; 924 925 /* We are the end of queue, roll back pidx to top of queue */ 926 if (unlikely(q->pidx == q->credits)) { 927 q->pidx = 0; 928 CSIO_INC_STATS(q, n_qwrap); 929 } 930 } 931 932 q->inc_idx = (uint16_t)req_credits; 933 934 CSIO_INC_STATS(q, n_tot_reqs); 935 936 return 0; 937 } 938 939 /* 940 * csio_wr_copy_to_wrp - Copies given data into WR. 941 * @data_buf - Data buffer 942 * @wrp - Work request pair. 943 * @wr_off - Work request offset. 944 * @data_len - Data length. 945 * 946 * Copies the given data in Work Request. Work request pair(wrp) specifies 947 * address information of Work request. 948 * Returns: none 949 */ 950 void 951 csio_wr_copy_to_wrp(void *data_buf, struct csio_wr_pair *wrp, 952 uint32_t wr_off, uint32_t data_len) 953 { 954 uint32_t nbytes; 955 956 /* Number of space available in buffer addr1 of WRP */ 957 nbytes = ((wrp->size1 - wr_off) >= data_len) ? 958 data_len : (wrp->size1 - wr_off); 959 960 memcpy((uint8_t *) wrp->addr1 + wr_off, data_buf, nbytes); 961 data_len -= nbytes; 962 963 /* Write the remaining data from the begining of circular buffer */ 964 if (data_len) { 965 CSIO_DB_ASSERT(data_len <= wrp->size2); 966 CSIO_DB_ASSERT(wrp->addr2 != NULL); 967 memcpy(wrp->addr2, (uint8_t *) data_buf + nbytes, data_len); 968 } 969 } 970 971 /* 972 * csio_wr_issue - Notify chip of Work request. 973 * @hw: HW module. 974 * @qidx: Index of queue. 975 * @prio: 0: Low priority, 1: High priority 976 * 977 * Rings the SGE Doorbell by writing the current producer index of the passed 978 * in queue into the register. 979 * 980 */ 981 int 982 csio_wr_issue(struct csio_hw *hw, int qidx, bool prio) 983 { 984 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 985 struct csio_q *q = wrm->q_arr[qidx]; 986 987 CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx)); 988 989 wmb(); 990 /* Ring SGE Doorbell writing q->pidx into it */ 991 csio_wr_reg32(hw, DBPRIO(prio) | QID(q->un.eq.physeqid) | 992 PIDX(q->inc_idx), MYPF_REG(SGE_PF_KDOORBELL)); 993 q->inc_idx = 0; 994 995 return 0; 996 } 997 998 static inline uint32_t 999 csio_wr_avail_qcredits(struct csio_q *q) 1000 { 1001 if (q->pidx > q->cidx) 1002 return q->pidx - q->cidx; 1003 else if (q->cidx > q->pidx) 1004 return q->credits - (q->cidx - q->pidx); 1005 else 1006 return 0; /* cidx == pidx, empty queue */ 1007 } 1008 1009 /* 1010 * csio_wr_inval_flq_buf - Invalidate a free list buffer entry. 1011 * @hw: HW module. 1012 * @flq: The freelist queue. 1013 * 1014 * Invalidate the driver's version of a freelist buffer entry, 1015 * without freeing the associated the DMA memory. The entry 1016 * to be invalidated is picked up from the current Free list 1017 * queue cidx. 1018 * 1019 */ 1020 static inline void 1021 csio_wr_inval_flq_buf(struct csio_hw *hw, struct csio_q *flq) 1022 { 1023 flq->cidx++; 1024 if (flq->cidx == flq->credits) { 1025 flq->cidx = 0; 1026 CSIO_INC_STATS(flq, n_qwrap); 1027 } 1028 } 1029 1030 /* 1031 * csio_wr_process_fl - Process a freelist completion. 1032 * @hw: HW module. 1033 * @q: The ingress queue attached to the Freelist. 1034 * @wr: The freelist completion WR in the ingress queue. 1035 * @len_to_qid: The lower 32-bits of the first flit of the RSP footer 1036 * @iq_handler: Caller's handler for this completion. 1037 * @priv: Private pointer of caller 1038 * 1039 */ 1040 static inline void 1041 csio_wr_process_fl(struct csio_hw *hw, struct csio_q *q, 1042 void *wr, uint32_t len_to_qid, 1043 void (*iq_handler)(struct csio_hw *, void *, 1044 uint32_t, struct csio_fl_dma_buf *, 1045 void *), 1046 void *priv) 1047 { 1048 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 1049 struct csio_sge *sge = &wrm->sge; 1050 struct csio_fl_dma_buf flb; 1051 struct csio_dma_buf *buf, *fbuf; 1052 uint32_t bufsz, len, lastlen = 0; 1053 int n; 1054 struct csio_q *flq = hw->wrm.q_arr[q->un.iq.flq_idx]; 1055 1056 CSIO_DB_ASSERT(flq != NULL); 1057 1058 len = len_to_qid; 1059 1060 if (len & IQWRF_NEWBUF) { 1061 if (flq->un.fl.offset > 0) { 1062 csio_wr_inval_flq_buf(hw, flq); 1063 flq->un.fl.offset = 0; 1064 } 1065 len = IQWRF_LEN_GET(len); 1066 } 1067 1068 CSIO_DB_ASSERT(len != 0); 1069 1070 flb.totlen = len; 1071 1072 /* Consume all freelist buffers used for len bytes */ 1073 for (n = 0, fbuf = flb.flbufs; ; n++, fbuf++) { 1074 buf = &flq->un.fl.bufs[flq->cidx]; 1075 bufsz = csio_wr_fl_bufsz(sge, buf); 1076 1077 fbuf->paddr = buf->paddr; 1078 fbuf->vaddr = buf->vaddr; 1079 1080 flb.offset = flq->un.fl.offset; 1081 lastlen = min(bufsz, len); 1082 fbuf->len = lastlen; 1083 1084 len -= lastlen; 1085 if (!len) 1086 break; 1087 csio_wr_inval_flq_buf(hw, flq); 1088 } 1089 1090 flb.defer_free = flq->un.fl.packen ? 0 : 1; 1091 1092 iq_handler(hw, wr, q->wr_sz - sizeof(struct csio_iqwr_footer), 1093 &flb, priv); 1094 1095 if (flq->un.fl.packen) 1096 flq->un.fl.offset += ALIGN(lastlen, sge->csio_fl_align); 1097 else 1098 csio_wr_inval_flq_buf(hw, flq); 1099 1100 } 1101 1102 /* 1103 * csio_is_new_iqwr - Is this a new Ingress queue entry ? 1104 * @q: Ingress quueue. 1105 * @ftr: Ingress queue WR SGE footer. 1106 * 1107 * The entry is new if our generation bit matches the corresponding 1108 * bit in the footer of the current WR. 1109 */ 1110 static inline bool 1111 csio_is_new_iqwr(struct csio_q *q, struct csio_iqwr_footer *ftr) 1112 { 1113 return (q->un.iq.genbit == (ftr->u.type_gen >> IQWRF_GEN_SHIFT)); 1114 } 1115 1116 /* 1117 * csio_wr_process_iq - Process elements in Ingress queue. 1118 * @hw: HW pointer 1119 * @qidx: Index of queue 1120 * @iq_handler: Handler for this queue 1121 * @priv: Caller's private pointer 1122 * 1123 * This routine walks through every entry of the ingress queue, calling 1124 * the provided iq_handler with the entry, until the generation bit 1125 * flips. 1126 */ 1127 int 1128 csio_wr_process_iq(struct csio_hw *hw, struct csio_q *q, 1129 void (*iq_handler)(struct csio_hw *, void *, 1130 uint32_t, struct csio_fl_dma_buf *, 1131 void *), 1132 void *priv) 1133 { 1134 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 1135 void *wr = (void *)((uintptr_t)q->vstart + (q->cidx * q->wr_sz)); 1136 struct csio_iqwr_footer *ftr; 1137 uint32_t wr_type, fw_qid, qid; 1138 struct csio_q *q_completed; 1139 struct csio_q *flq = csio_iq_has_fl(q) ? 1140 wrm->q_arr[q->un.iq.flq_idx] : NULL; 1141 int rv = 0; 1142 1143 /* Get the footer */ 1144 ftr = (struct csio_iqwr_footer *)((uintptr_t)wr + 1145 (q->wr_sz - sizeof(*ftr))); 1146 1147 /* 1148 * When q wrapped around last time, driver should have inverted 1149 * ic.genbit as well. 1150 */ 1151 while (csio_is_new_iqwr(q, ftr)) { 1152 1153 CSIO_DB_ASSERT(((uintptr_t)wr + q->wr_sz) <= 1154 (uintptr_t)q->vwrap); 1155 rmb(); 1156 wr_type = IQWRF_TYPE_GET(ftr->u.type_gen); 1157 1158 switch (wr_type) { 1159 case X_RSPD_TYPE_CPL: 1160 /* Subtract footer from WR len */ 1161 iq_handler(hw, wr, q->wr_sz - sizeof(*ftr), NULL, priv); 1162 break; 1163 case X_RSPD_TYPE_FLBUF: 1164 csio_wr_process_fl(hw, q, wr, 1165 ntohl(ftr->pldbuflen_qid), 1166 iq_handler, priv); 1167 break; 1168 case X_RSPD_TYPE_INTR: 1169 fw_qid = ntohl(ftr->pldbuflen_qid); 1170 qid = fw_qid - wrm->fw_iq_start; 1171 q_completed = hw->wrm.intr_map[qid]; 1172 1173 if (unlikely(qid == 1174 csio_q_physiqid(hw, hw->intr_iq_idx))) { 1175 /* 1176 * We are already in the Forward Interrupt 1177 * Interrupt Queue Service! Do-not service 1178 * again! 1179 * 1180 */ 1181 } else { 1182 CSIO_DB_ASSERT(q_completed); 1183 CSIO_DB_ASSERT( 1184 q_completed->un.iq.iq_intx_handler); 1185 1186 /* Call the queue handler. */ 1187 q_completed->un.iq.iq_intx_handler(hw, NULL, 1188 0, NULL, (void *)q_completed); 1189 } 1190 break; 1191 default: 1192 csio_warn(hw, "Unknown resp type 0x%x received\n", 1193 wr_type); 1194 CSIO_INC_STATS(q, n_rsp_unknown); 1195 break; 1196 } 1197 1198 /* 1199 * Ingress *always* has fixed size WR entries. Therefore, 1200 * there should always be complete WRs towards the end of 1201 * queue. 1202 */ 1203 if (((uintptr_t)wr + q->wr_sz) == (uintptr_t)q->vwrap) { 1204 1205 /* Roll over to start of queue */ 1206 q->cidx = 0; 1207 wr = q->vstart; 1208 1209 /* Toggle genbit */ 1210 q->un.iq.genbit ^= 0x1; 1211 1212 CSIO_INC_STATS(q, n_qwrap); 1213 } else { 1214 q->cidx++; 1215 wr = (void *)((uintptr_t)(q->vstart) + 1216 (q->cidx * q->wr_sz)); 1217 } 1218 1219 ftr = (struct csio_iqwr_footer *)((uintptr_t)wr + 1220 (q->wr_sz - sizeof(*ftr))); 1221 q->inc_idx++; 1222 1223 } /* while (q->un.iq.genbit == hdr->genbit) */ 1224 1225 /* 1226 * We need to re-arm SGE interrupts in case we got a stray interrupt, 1227 * especially in msix mode. With INTx, this may be a common occurence. 1228 */ 1229 if (unlikely(!q->inc_idx)) { 1230 CSIO_INC_STATS(q, n_stray_comp); 1231 rv = -EINVAL; 1232 goto restart; 1233 } 1234 1235 /* Replenish free list buffers if pending falls below low water mark */ 1236 if (flq) { 1237 uint32_t avail = csio_wr_avail_qcredits(flq); 1238 if (avail <= 16) { 1239 /* Make sure in FLQ, atleast 1 credit (8 FL buffers) 1240 * remains unpopulated otherwise HW thinks 1241 * FLQ is empty. 1242 */ 1243 csio_wr_update_fl(hw, flq, (flq->credits - 8) - avail); 1244 csio_wr_ring_fldb(hw, flq); 1245 } 1246 } 1247 1248 restart: 1249 /* Now inform SGE about our incremental index value */ 1250 csio_wr_reg32(hw, CIDXINC(q->inc_idx) | 1251 INGRESSQID(q->un.iq.physiqid) | 1252 TIMERREG(csio_sge_timer_reg), 1253 MYPF_REG(SGE_PF_GTS)); 1254 q->stats.n_tot_rsps += q->inc_idx; 1255 1256 q->inc_idx = 0; 1257 1258 return rv; 1259 } 1260 1261 int 1262 csio_wr_process_iq_idx(struct csio_hw *hw, int qidx, 1263 void (*iq_handler)(struct csio_hw *, void *, 1264 uint32_t, struct csio_fl_dma_buf *, 1265 void *), 1266 void *priv) 1267 { 1268 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 1269 struct csio_q *iq = wrm->q_arr[qidx]; 1270 1271 return csio_wr_process_iq(hw, iq, iq_handler, priv); 1272 } 1273 1274 static int 1275 csio_closest_timer(struct csio_sge *s, int time) 1276 { 1277 int i, delta, match = 0, min_delta = INT_MAX; 1278 1279 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) { 1280 delta = time - s->timer_val[i]; 1281 if (delta < 0) 1282 delta = -delta; 1283 if (delta < min_delta) { 1284 min_delta = delta; 1285 match = i; 1286 } 1287 } 1288 return match; 1289 } 1290 1291 static int 1292 csio_closest_thresh(struct csio_sge *s, int cnt) 1293 { 1294 int i, delta, match = 0, min_delta = INT_MAX; 1295 1296 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) { 1297 delta = cnt - s->counter_val[i]; 1298 if (delta < 0) 1299 delta = -delta; 1300 if (delta < min_delta) { 1301 min_delta = delta; 1302 match = i; 1303 } 1304 } 1305 return match; 1306 } 1307 1308 static void 1309 csio_wr_fixup_host_params(struct csio_hw *hw) 1310 { 1311 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 1312 struct csio_sge *sge = &wrm->sge; 1313 uint32_t clsz = L1_CACHE_BYTES; 1314 uint32_t s_hps = PAGE_SHIFT - 10; 1315 uint32_t ingpad = 0; 1316 uint32_t stat_len = clsz > 64 ? 128 : 64; 1317 1318 csio_wr_reg32(hw, HOSTPAGESIZEPF0(s_hps) | HOSTPAGESIZEPF1(s_hps) | 1319 HOSTPAGESIZEPF2(s_hps) | HOSTPAGESIZEPF3(s_hps) | 1320 HOSTPAGESIZEPF4(s_hps) | HOSTPAGESIZEPF5(s_hps) | 1321 HOSTPAGESIZEPF6(s_hps) | HOSTPAGESIZEPF7(s_hps), 1322 SGE_HOST_PAGE_SIZE); 1323 1324 sge->csio_fl_align = clsz < 32 ? 32 : clsz; 1325 ingpad = ilog2(sge->csio_fl_align) - 5; 1326 1327 csio_set_reg_field(hw, SGE_CONTROL, INGPADBOUNDARY_MASK | 1328 EGRSTATUSPAGESIZE(1), 1329 INGPADBOUNDARY(ingpad) | 1330 EGRSTATUSPAGESIZE(stat_len != 64)); 1331 1332 /* FL BUFFER SIZE#0 is Page size i,e already aligned to cache line */ 1333 csio_wr_reg32(hw, PAGE_SIZE, SGE_FL_BUFFER_SIZE0); 1334 csio_wr_reg32(hw, 1335 (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE2) + 1336 sge->csio_fl_align - 1) & ~(sge->csio_fl_align - 1), 1337 SGE_FL_BUFFER_SIZE2); 1338 csio_wr_reg32(hw, 1339 (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE3) + 1340 sge->csio_fl_align - 1) & ~(sge->csio_fl_align - 1), 1341 SGE_FL_BUFFER_SIZE3); 1342 1343 csio_wr_reg32(hw, HPZ0(PAGE_SHIFT - 12), ULP_RX_TDDP_PSZ); 1344 1345 /* default value of rx_dma_offset of the NIC driver */ 1346 csio_set_reg_field(hw, SGE_CONTROL, PKTSHIFT_MASK, 1347 PKTSHIFT(CSIO_SGE_RX_DMA_OFFSET)); 1348 } 1349 1350 static void 1351 csio_init_intr_coalesce_parms(struct csio_hw *hw) 1352 { 1353 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 1354 struct csio_sge *sge = &wrm->sge; 1355 1356 csio_sge_thresh_reg = csio_closest_thresh(sge, csio_intr_coalesce_cnt); 1357 if (csio_intr_coalesce_cnt) { 1358 csio_sge_thresh_reg = 0; 1359 csio_sge_timer_reg = X_TIMERREG_RESTART_COUNTER; 1360 return; 1361 } 1362 1363 csio_sge_timer_reg = csio_closest_timer(sge, csio_intr_coalesce_time); 1364 } 1365 1366 /* 1367 * csio_wr_get_sge - Get SGE register values. 1368 * @hw: HW module. 1369 * 1370 * Used by non-master functions and by master-functions relying on config file. 1371 */ 1372 static void 1373 csio_wr_get_sge(struct csio_hw *hw) 1374 { 1375 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 1376 struct csio_sge *sge = &wrm->sge; 1377 uint32_t ingpad; 1378 int i; 1379 u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5; 1380 u32 ingress_rx_threshold; 1381 1382 sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL); 1383 1384 ingpad = INGPADBOUNDARY_GET(sge->sge_control); 1385 1386 switch (ingpad) { 1387 case X_INGPCIEBOUNDARY_32B: 1388 sge->csio_fl_align = 32; break; 1389 case X_INGPCIEBOUNDARY_64B: 1390 sge->csio_fl_align = 64; break; 1391 case X_INGPCIEBOUNDARY_128B: 1392 sge->csio_fl_align = 128; break; 1393 case X_INGPCIEBOUNDARY_256B: 1394 sge->csio_fl_align = 256; break; 1395 case X_INGPCIEBOUNDARY_512B: 1396 sge->csio_fl_align = 512; break; 1397 case X_INGPCIEBOUNDARY_1024B: 1398 sge->csio_fl_align = 1024; break; 1399 case X_INGPCIEBOUNDARY_2048B: 1400 sge->csio_fl_align = 2048; break; 1401 case X_INGPCIEBOUNDARY_4096B: 1402 sge->csio_fl_align = 4096; break; 1403 } 1404 1405 for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++) 1406 csio_get_flbuf_size(hw, sge, i); 1407 1408 timer_value_0_and_1 = csio_rd_reg32(hw, SGE_TIMER_VALUE_0_AND_1); 1409 timer_value_2_and_3 = csio_rd_reg32(hw, SGE_TIMER_VALUE_2_AND_3); 1410 timer_value_4_and_5 = csio_rd_reg32(hw, SGE_TIMER_VALUE_4_AND_5); 1411 1412 sge->timer_val[0] = (uint16_t)csio_core_ticks_to_us(hw, 1413 TIMERVALUE0_GET(timer_value_0_and_1)); 1414 sge->timer_val[1] = (uint16_t)csio_core_ticks_to_us(hw, 1415 TIMERVALUE1_GET(timer_value_0_and_1)); 1416 sge->timer_val[2] = (uint16_t)csio_core_ticks_to_us(hw, 1417 TIMERVALUE2_GET(timer_value_2_and_3)); 1418 sge->timer_val[3] = (uint16_t)csio_core_ticks_to_us(hw, 1419 TIMERVALUE3_GET(timer_value_2_and_3)); 1420 sge->timer_val[4] = (uint16_t)csio_core_ticks_to_us(hw, 1421 TIMERVALUE4_GET(timer_value_4_and_5)); 1422 sge->timer_val[5] = (uint16_t)csio_core_ticks_to_us(hw, 1423 TIMERVALUE5_GET(timer_value_4_and_5)); 1424 1425 ingress_rx_threshold = csio_rd_reg32(hw, SGE_INGRESS_RX_THRESHOLD); 1426 sge->counter_val[0] = THRESHOLD_0_GET(ingress_rx_threshold); 1427 sge->counter_val[1] = THRESHOLD_1_GET(ingress_rx_threshold); 1428 sge->counter_val[2] = THRESHOLD_2_GET(ingress_rx_threshold); 1429 sge->counter_val[3] = THRESHOLD_3_GET(ingress_rx_threshold); 1430 1431 csio_init_intr_coalesce_parms(hw); 1432 } 1433 1434 /* 1435 * csio_wr_set_sge - Initialize SGE registers 1436 * @hw: HW module. 1437 * 1438 * Used by Master function to initialize SGE registers in the absence 1439 * of a config file. 1440 */ 1441 static void 1442 csio_wr_set_sge(struct csio_hw *hw) 1443 { 1444 struct csio_wrm *wrm = csio_hw_to_wrm(hw); 1445 struct csio_sge *sge = &wrm->sge; 1446 int i; 1447 1448 /* 1449 * Set up our basic SGE mode to deliver CPL messages to our Ingress 1450 * Queue and Packet Date to the Free List. 1451 */ 1452 csio_set_reg_field(hw, SGE_CONTROL, RXPKTCPLMODE(1), RXPKTCPLMODE(1)); 1453 1454 sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL); 1455 1456 /* sge->csio_fl_align is set up by csio_wr_fixup_host_params(). */ 1457 1458 /* 1459 * Set up to drop DOORBELL writes when the DOORBELL FIFO overflows 1460 * and generate an interrupt when this occurs so we can recover. 1461 */ 1462 csio_set_reg_field(hw, SGE_DBFIFO_STATUS, 1463 HP_INT_THRESH(HP_INT_THRESH_MASK) | 1464 LP_INT_THRESH(LP_INT_THRESH_MASK), 1465 HP_INT_THRESH(CSIO_SGE_DBFIFO_INT_THRESH) | 1466 LP_INT_THRESH(CSIO_SGE_DBFIFO_INT_THRESH)); 1467 csio_set_reg_field(hw, SGE_DOORBELL_CONTROL, ENABLE_DROP, 1468 ENABLE_DROP); 1469 1470 /* SGE_FL_BUFFER_SIZE0 is set up by csio_wr_fixup_host_params(). */ 1471 1472 CSIO_SET_FLBUF_SIZE(hw, 1, CSIO_SGE_FLBUF_SIZE1); 1473 CSIO_SET_FLBUF_SIZE(hw, 2, CSIO_SGE_FLBUF_SIZE2); 1474 CSIO_SET_FLBUF_SIZE(hw, 3, CSIO_SGE_FLBUF_SIZE3); 1475 CSIO_SET_FLBUF_SIZE(hw, 4, CSIO_SGE_FLBUF_SIZE4); 1476 CSIO_SET_FLBUF_SIZE(hw, 5, CSIO_SGE_FLBUF_SIZE5); 1477 CSIO_SET_FLBUF_SIZE(hw, 6, CSIO_SGE_FLBUF_SIZE6); 1478 CSIO_SET_FLBUF_SIZE(hw, 7, CSIO_SGE_FLBUF_SIZE7); 1479 CSIO_SET_FLBUF_SIZE(hw, 8, CSIO_SGE_FLBUF_SIZE8); 1480 1481 for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++) 1482 csio_get_flbuf_size(hw, sge, i); 1483 1484 /* Initialize interrupt coalescing attributes */ 1485 sge->timer_val[0] = CSIO_SGE_TIMER_VAL_0; 1486 sge->timer_val[1] = CSIO_SGE_TIMER_VAL_1; 1487 sge->timer_val[2] = CSIO_SGE_TIMER_VAL_2; 1488 sge->timer_val[3] = CSIO_SGE_TIMER_VAL_3; 1489 sge->timer_val[4] = CSIO_SGE_TIMER_VAL_4; 1490 sge->timer_val[5] = CSIO_SGE_TIMER_VAL_5; 1491 1492 sge->counter_val[0] = CSIO_SGE_INT_CNT_VAL_0; 1493 sge->counter_val[1] = CSIO_SGE_INT_CNT_VAL_1; 1494 sge->counter_val[2] = CSIO_SGE_INT_CNT_VAL_2; 1495 sge->counter_val[3] = CSIO_SGE_INT_CNT_VAL_3; 1496 1497 csio_wr_reg32(hw, THRESHOLD_0(sge->counter_val[0]) | 1498 THRESHOLD_1(sge->counter_val[1]) | 1499 THRESHOLD_2(sge->counter_val[2]) | 1500 THRESHOLD_3(sge->counter_val[3]), 1501 SGE_INGRESS_RX_THRESHOLD); 1502 1503 csio_wr_reg32(hw, 1504 TIMERVALUE0(csio_us_to_core_ticks(hw, sge->timer_val[0])) | 1505 TIMERVALUE1(csio_us_to_core_ticks(hw, sge->timer_val[1])), 1506 SGE_TIMER_VALUE_0_AND_1); 1507 1508 csio_wr_reg32(hw, 1509 TIMERVALUE2(csio_us_to_core_ticks(hw, sge->timer_val[2])) | 1510 TIMERVALUE3(csio_us_to_core_ticks(hw, sge->timer_val[3])), 1511 SGE_TIMER_VALUE_2_AND_3); 1512 1513 csio_wr_reg32(hw, 1514 TIMERVALUE4(csio_us_to_core_ticks(hw, sge->timer_val[4])) | 1515 TIMERVALUE5(csio_us_to_core_ticks(hw, sge->timer_val[5])), 1516 SGE_TIMER_VALUE_4_AND_5); 1517 1518 csio_init_intr_coalesce_parms(hw); 1519 } 1520 1521 void 1522 csio_wr_sge_init(struct csio_hw *hw) 1523 { 1524 /* 1525 * If we are master: 1526 * - If we plan to use the config file, we need to fixup some 1527 * host specific registers, and read the rest of the SGE 1528 * configuration. 1529 * - If we dont plan to use the config file, we need to initialize 1530 * SGE entirely, including fixing the host specific registers. 1531 * If we arent the master, we are only allowed to read and work off of 1532 * the already initialized SGE values. 1533 * 1534 * Therefore, before calling this function, we assume that the master- 1535 * ship of the card, and whether to use config file or not, have 1536 * already been decided. In other words, CSIO_HWF_USING_SOFT_PARAMS and 1537 * CSIO_HWF_MASTER should be set/unset. 1538 */ 1539 if (csio_is_hw_master(hw)) { 1540 csio_wr_fixup_host_params(hw); 1541 1542 if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS) 1543 csio_wr_get_sge(hw); 1544 else 1545 csio_wr_set_sge(hw); 1546 } else 1547 csio_wr_get_sge(hw); 1548 } 1549 1550 /* 1551 * csio_wrm_init - Initialize Work request module. 1552 * @wrm: WR module 1553 * @hw: HW pointer 1554 * 1555 * Allocates memory for an array of queue pointers starting at q_arr. 1556 */ 1557 int 1558 csio_wrm_init(struct csio_wrm *wrm, struct csio_hw *hw) 1559 { 1560 int i; 1561 1562 if (!wrm->num_q) { 1563 csio_err(hw, "Num queues is not set\n"); 1564 return -EINVAL; 1565 } 1566 1567 wrm->q_arr = kzalloc(sizeof(struct csio_q *) * wrm->num_q, GFP_KERNEL); 1568 if (!wrm->q_arr) 1569 goto err; 1570 1571 for (i = 0; i < wrm->num_q; i++) { 1572 wrm->q_arr[i] = kzalloc(sizeof(struct csio_q), GFP_KERNEL); 1573 if (!wrm->q_arr[i]) { 1574 while (--i >= 0) 1575 kfree(wrm->q_arr[i]); 1576 goto err_free_arr; 1577 } 1578 } 1579 wrm->free_qidx = 0; 1580 1581 return 0; 1582 1583 err_free_arr: 1584 kfree(wrm->q_arr); 1585 err: 1586 return -ENOMEM; 1587 } 1588 1589 /* 1590 * csio_wrm_exit - Initialize Work request module. 1591 * @wrm: WR module 1592 * @hw: HW module 1593 * 1594 * Uninitialize WR module. Free q_arr and pointers in it. 1595 * We have the additional job of freeing the DMA memory associated 1596 * with the queues. 1597 */ 1598 void 1599 csio_wrm_exit(struct csio_wrm *wrm, struct csio_hw *hw) 1600 { 1601 int i; 1602 uint32_t j; 1603 struct csio_q *q; 1604 struct csio_dma_buf *buf; 1605 1606 for (i = 0; i < wrm->num_q; i++) { 1607 q = wrm->q_arr[i]; 1608 1609 if (wrm->free_qidx && (i < wrm->free_qidx)) { 1610 if (q->type == CSIO_FREELIST) { 1611 if (!q->un.fl.bufs) 1612 continue; 1613 for (j = 0; j < q->credits; j++) { 1614 buf = &q->un.fl.bufs[j]; 1615 if (!buf->vaddr) 1616 continue; 1617 pci_free_consistent(hw->pdev, buf->len, 1618 buf->vaddr, 1619 buf->paddr); 1620 } 1621 kfree(q->un.fl.bufs); 1622 } 1623 pci_free_consistent(hw->pdev, q->size, 1624 q->vstart, q->pstart); 1625 } 1626 kfree(q); 1627 } 1628 1629 hw->flags &= ~CSIO_HWF_Q_MEM_ALLOCED; 1630 1631 kfree(wrm->q_arr); 1632 } 1633