1 /* 2 * This file is part of the Chelsio FCoE driver for Linux. 3 * 4 * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include <linux/kernel.h> 36 #include <linux/pci.h> 37 #include <linux/interrupt.h> 38 #include <linux/cpumask.h> 39 #include <linux/string.h> 40 41 #include "csio_init.h" 42 #include "csio_hw.h" 43 44 static irqreturn_t 45 csio_nondata_isr(int irq, void *dev_id) 46 { 47 struct csio_hw *hw = (struct csio_hw *) dev_id; 48 int rv; 49 unsigned long flags; 50 51 if (unlikely(!hw)) 52 return IRQ_NONE; 53 54 if (unlikely(pci_channel_offline(hw->pdev))) { 55 CSIO_INC_STATS(hw, n_pcich_offline); 56 return IRQ_NONE; 57 } 58 59 spin_lock_irqsave(&hw->lock, flags); 60 csio_hw_slow_intr_handler(hw); 61 rv = csio_mb_isr_handler(hw); 62 63 if (rv == 0 && !(hw->flags & CSIO_HWF_FWEVT_PENDING)) { 64 hw->flags |= CSIO_HWF_FWEVT_PENDING; 65 spin_unlock_irqrestore(&hw->lock, flags); 66 schedule_work(&hw->evtq_work); 67 return IRQ_HANDLED; 68 } 69 spin_unlock_irqrestore(&hw->lock, flags); 70 return IRQ_HANDLED; 71 } 72 73 /* 74 * csio_fwevt_handler - Common FW event handler routine. 75 * @hw: HW module. 76 * 77 * This is the ISR for FW events. It is shared b/w MSIX 78 * and INTx handlers. 79 */ 80 static void 81 csio_fwevt_handler(struct csio_hw *hw) 82 { 83 int rv; 84 unsigned long flags; 85 86 rv = csio_fwevtq_handler(hw); 87 88 spin_lock_irqsave(&hw->lock, flags); 89 if (rv == 0 && !(hw->flags & CSIO_HWF_FWEVT_PENDING)) { 90 hw->flags |= CSIO_HWF_FWEVT_PENDING; 91 spin_unlock_irqrestore(&hw->lock, flags); 92 schedule_work(&hw->evtq_work); 93 return; 94 } 95 spin_unlock_irqrestore(&hw->lock, flags); 96 97 } /* csio_fwevt_handler */ 98 99 /* 100 * csio_fwevt_isr() - FW events MSIX ISR 101 * @irq: 102 * @dev_id: 103 * 104 * Process WRs on the FW event queue. 105 * 106 */ 107 static irqreturn_t 108 csio_fwevt_isr(int irq, void *dev_id) 109 { 110 struct csio_hw *hw = (struct csio_hw *) dev_id; 111 112 if (unlikely(!hw)) 113 return IRQ_NONE; 114 115 if (unlikely(pci_channel_offline(hw->pdev))) { 116 CSIO_INC_STATS(hw, n_pcich_offline); 117 return IRQ_NONE; 118 } 119 120 csio_fwevt_handler(hw); 121 122 return IRQ_HANDLED; 123 } 124 125 /* 126 * csio_fwevt_isr() - INTx wrapper for handling FW events. 127 * @irq: 128 * @dev_id: 129 */ 130 void 131 csio_fwevt_intx_handler(struct csio_hw *hw, void *wr, uint32_t len, 132 struct csio_fl_dma_buf *flb, void *priv) 133 { 134 csio_fwevt_handler(hw); 135 } /* csio_fwevt_intx_handler */ 136 137 /* 138 * csio_process_scsi_cmpl - Process a SCSI WR completion. 139 * @hw: HW module. 140 * @wr: The completed WR from the ingress queue. 141 * @len: Length of the WR. 142 * @flb: Freelist buffer array. 143 * 144 */ 145 static void 146 csio_process_scsi_cmpl(struct csio_hw *hw, void *wr, uint32_t len, 147 struct csio_fl_dma_buf *flb, void *cbfn_q) 148 { 149 struct csio_ioreq *ioreq; 150 uint8_t *scsiwr; 151 uint8_t subop; 152 void *cmnd; 153 unsigned long flags; 154 155 ioreq = csio_scsi_cmpl_handler(hw, wr, len, flb, NULL, &scsiwr); 156 if (likely(ioreq)) { 157 if (unlikely(*scsiwr == FW_SCSI_ABRT_CLS_WR)) { 158 subop = FW_SCSI_ABRT_CLS_WR_SUB_OPCODE_GET( 159 ((struct fw_scsi_abrt_cls_wr *) 160 scsiwr)->sub_opcode_to_chk_all_io); 161 162 csio_dbg(hw, "%s cmpl recvd ioreq:%p status:%d\n", 163 subop ? "Close" : "Abort", 164 ioreq, ioreq->wr_status); 165 166 spin_lock_irqsave(&hw->lock, flags); 167 if (subop) 168 csio_scsi_closed(ioreq, 169 (struct list_head *)cbfn_q); 170 else 171 csio_scsi_aborted(ioreq, 172 (struct list_head *)cbfn_q); 173 /* 174 * We call scsi_done for I/Os that driver thinks aborts 175 * have timed out. If there is a race caused by FW 176 * completing abort at the exact same time that the 177 * driver has deteced the abort timeout, the following 178 * check prevents calling of scsi_done twice for the 179 * same command: once from the eh_abort_handler, another 180 * from csio_scsi_isr_handler(). This also avoids the 181 * need to check if csio_scsi_cmnd(req) is NULL in the 182 * fast path. 183 */ 184 cmnd = csio_scsi_cmnd(ioreq); 185 if (unlikely(cmnd == NULL)) 186 list_del_init(&ioreq->sm.sm_list); 187 188 spin_unlock_irqrestore(&hw->lock, flags); 189 190 if (unlikely(cmnd == NULL)) 191 csio_put_scsi_ioreq_lock(hw, 192 csio_hw_to_scsim(hw), ioreq); 193 } else { 194 spin_lock_irqsave(&hw->lock, flags); 195 csio_scsi_completed(ioreq, (struct list_head *)cbfn_q); 196 spin_unlock_irqrestore(&hw->lock, flags); 197 } 198 } 199 } 200 201 /* 202 * csio_scsi_isr_handler() - Common SCSI ISR handler. 203 * @iq: Ingress queue pointer. 204 * 205 * Processes SCSI completions on the SCSI IQ indicated by scm->iq_idx 206 * by calling csio_wr_process_iq_idx. If there are completions on the 207 * isr_cbfn_q, yank them out into a local queue and call their io_cbfns. 208 * Once done, add these completions onto the freelist. 209 * This routine is shared b/w MSIX and INTx. 210 */ 211 static inline irqreturn_t 212 csio_scsi_isr_handler(struct csio_q *iq) 213 { 214 struct csio_hw *hw = (struct csio_hw *)iq->owner; 215 LIST_HEAD(cbfn_q); 216 struct list_head *tmp; 217 struct csio_scsim *scm; 218 struct csio_ioreq *ioreq; 219 int isr_completions = 0; 220 221 scm = csio_hw_to_scsim(hw); 222 223 if (unlikely(csio_wr_process_iq(hw, iq, csio_process_scsi_cmpl, 224 &cbfn_q) != 0)) 225 return IRQ_NONE; 226 227 /* Call back the completion routines */ 228 list_for_each(tmp, &cbfn_q) { 229 ioreq = (struct csio_ioreq *)tmp; 230 isr_completions++; 231 ioreq->io_cbfn(hw, ioreq); 232 /* Release ddp buffer if used for this req */ 233 if (unlikely(ioreq->dcopy)) 234 csio_put_scsi_ddp_list_lock(hw, scm, &ioreq->gen_list, 235 ioreq->nsge); 236 } 237 238 if (isr_completions) { 239 /* Return the ioreqs back to ioreq->freelist */ 240 csio_put_scsi_ioreq_list_lock(hw, scm, &cbfn_q, 241 isr_completions); 242 } 243 244 return IRQ_HANDLED; 245 } 246 247 /* 248 * csio_scsi_isr() - SCSI MSIX handler 249 * @irq: 250 * @dev_id: 251 * 252 * This is the top level SCSI MSIX handler. Calls csio_scsi_isr_handler() 253 * for handling SCSI completions. 254 */ 255 static irqreturn_t 256 csio_scsi_isr(int irq, void *dev_id) 257 { 258 struct csio_q *iq = (struct csio_q *) dev_id; 259 struct csio_hw *hw; 260 261 if (unlikely(!iq)) 262 return IRQ_NONE; 263 264 hw = (struct csio_hw *)iq->owner; 265 266 if (unlikely(pci_channel_offline(hw->pdev))) { 267 CSIO_INC_STATS(hw, n_pcich_offline); 268 return IRQ_NONE; 269 } 270 271 csio_scsi_isr_handler(iq); 272 273 return IRQ_HANDLED; 274 } 275 276 /* 277 * csio_scsi_intx_handler() - SCSI INTx handler 278 * @irq: 279 * @dev_id: 280 * 281 * This is the top level SCSI INTx handler. Calls csio_scsi_isr_handler() 282 * for handling SCSI completions. 283 */ 284 void 285 csio_scsi_intx_handler(struct csio_hw *hw, void *wr, uint32_t len, 286 struct csio_fl_dma_buf *flb, void *priv) 287 { 288 struct csio_q *iq = priv; 289 290 csio_scsi_isr_handler(iq); 291 292 } /* csio_scsi_intx_handler */ 293 294 /* 295 * csio_fcoe_isr() - INTx/MSI interrupt service routine for FCoE. 296 * @irq: 297 * @dev_id: 298 * 299 * 300 */ 301 static irqreturn_t 302 csio_fcoe_isr(int irq, void *dev_id) 303 { 304 struct csio_hw *hw = (struct csio_hw *) dev_id; 305 struct csio_q *intx_q = NULL; 306 int rv; 307 irqreturn_t ret = IRQ_NONE; 308 unsigned long flags; 309 310 if (unlikely(!hw)) 311 return IRQ_NONE; 312 313 if (unlikely(pci_channel_offline(hw->pdev))) { 314 CSIO_INC_STATS(hw, n_pcich_offline); 315 return IRQ_NONE; 316 } 317 318 /* Disable the interrupt for this PCI function. */ 319 if (hw->intr_mode == CSIO_IM_INTX) 320 csio_wr_reg32(hw, 0, MYPF_REG(PCIE_PF_CLI_A)); 321 322 /* 323 * The read in the following function will flush the 324 * above write. 325 */ 326 if (csio_hw_slow_intr_handler(hw)) 327 ret = IRQ_HANDLED; 328 329 /* Get the INTx Forward interrupt IQ. */ 330 intx_q = csio_get_q(hw, hw->intr_iq_idx); 331 332 CSIO_DB_ASSERT(intx_q); 333 334 /* IQ handler is not possible for intx_q, hence pass in NULL */ 335 if (likely(csio_wr_process_iq(hw, intx_q, NULL, NULL) == 0)) 336 ret = IRQ_HANDLED; 337 338 spin_lock_irqsave(&hw->lock, flags); 339 rv = csio_mb_isr_handler(hw); 340 if (rv == 0 && !(hw->flags & CSIO_HWF_FWEVT_PENDING)) { 341 hw->flags |= CSIO_HWF_FWEVT_PENDING; 342 spin_unlock_irqrestore(&hw->lock, flags); 343 schedule_work(&hw->evtq_work); 344 return IRQ_HANDLED; 345 } 346 spin_unlock_irqrestore(&hw->lock, flags); 347 348 return ret; 349 } 350 351 static void 352 csio_add_msix_desc(struct csio_hw *hw) 353 { 354 int i; 355 struct csio_msix_entries *entryp = &hw->msix_entries[0]; 356 int k = CSIO_EXTRA_VECS; 357 int len = sizeof(entryp->desc) - 1; 358 int cnt = hw->num_sqsets + k; 359 360 /* Non-data vector */ 361 memset(entryp->desc, 0, len + 1); 362 snprintf(entryp->desc, len, "csio-%02x:%02x:%x-nondata", 363 CSIO_PCI_BUS(hw), CSIO_PCI_DEV(hw), CSIO_PCI_FUNC(hw)); 364 365 entryp++; 366 memset(entryp->desc, 0, len + 1); 367 snprintf(entryp->desc, len, "csio-%02x:%02x:%x-fwevt", 368 CSIO_PCI_BUS(hw), CSIO_PCI_DEV(hw), CSIO_PCI_FUNC(hw)); 369 entryp++; 370 371 /* Name SCSI vecs */ 372 for (i = k; i < cnt; i++, entryp++) { 373 memset(entryp->desc, 0, len + 1); 374 snprintf(entryp->desc, len, "csio-%02x:%02x:%x-scsi%d", 375 CSIO_PCI_BUS(hw), CSIO_PCI_DEV(hw), 376 CSIO_PCI_FUNC(hw), i - CSIO_EXTRA_VECS); 377 } 378 } 379 380 int 381 csio_request_irqs(struct csio_hw *hw) 382 { 383 int rv, i, j, k = 0; 384 struct csio_msix_entries *entryp = &hw->msix_entries[0]; 385 struct csio_scsi_cpu_info *info; 386 struct pci_dev *pdev = hw->pdev; 387 388 if (hw->intr_mode != CSIO_IM_MSIX) { 389 rv = request_irq(pci_irq_vector(pdev, 0), csio_fcoe_isr, 390 hw->intr_mode == CSIO_IM_MSI ? 0 : IRQF_SHARED, 391 KBUILD_MODNAME, hw); 392 if (rv) { 393 csio_err(hw, "Failed to allocate interrupt line.\n"); 394 goto out_free_irqs; 395 } 396 397 goto out; 398 } 399 400 /* Add the MSIX vector descriptions */ 401 csio_add_msix_desc(hw); 402 403 rv = request_irq(pci_irq_vector(pdev, k), csio_nondata_isr, 0, 404 entryp[k].desc, hw); 405 if (rv) { 406 csio_err(hw, "IRQ request failed for vec %d err:%d\n", 407 pci_irq_vector(pdev, k), rv); 408 goto out_free_irqs; 409 } 410 411 entryp[k++].dev_id = hw; 412 413 rv = request_irq(pci_irq_vector(pdev, k), csio_fwevt_isr, 0, 414 entryp[k].desc, hw); 415 if (rv) { 416 csio_err(hw, "IRQ request failed for vec %d err:%d\n", 417 pci_irq_vector(pdev, k), rv); 418 goto out_free_irqs; 419 } 420 421 entryp[k++].dev_id = (void *)hw; 422 423 /* Allocate IRQs for SCSI */ 424 for (i = 0; i < hw->num_pports; i++) { 425 info = &hw->scsi_cpu_info[i]; 426 for (j = 0; j < info->max_cpus; j++, k++) { 427 struct csio_scsi_qset *sqset = &hw->sqset[i][j]; 428 struct csio_q *q = hw->wrm.q_arr[sqset->iq_idx]; 429 430 rv = request_irq(pci_irq_vector(pdev, k), csio_scsi_isr, 0, 431 entryp[k].desc, q); 432 if (rv) { 433 csio_err(hw, 434 "IRQ request failed for vec %d err:%d\n", 435 pci_irq_vector(pdev, k), rv); 436 goto out_free_irqs; 437 } 438 439 entryp[k].dev_id = q; 440 441 } /* for all scsi cpus */ 442 } /* for all ports */ 443 444 out: 445 hw->flags |= CSIO_HWF_HOST_INTR_ENABLED; 446 return 0; 447 448 out_free_irqs: 449 for (i = 0; i < k; i++) 450 free_irq(pci_irq_vector(pdev, i), hw->msix_entries[i].dev_id); 451 pci_free_irq_vectors(hw->pdev); 452 return -EINVAL; 453 } 454 455 /* Reduce per-port max possible CPUs */ 456 static void 457 csio_reduce_sqsets(struct csio_hw *hw, int cnt) 458 { 459 int i; 460 struct csio_scsi_cpu_info *info; 461 462 while (cnt < hw->num_sqsets) { 463 for (i = 0; i < hw->num_pports; i++) { 464 info = &hw->scsi_cpu_info[i]; 465 if (info->max_cpus > 1) { 466 info->max_cpus--; 467 hw->num_sqsets--; 468 if (hw->num_sqsets <= cnt) 469 break; 470 } 471 } 472 } 473 474 csio_dbg(hw, "Reduced sqsets to %d\n", hw->num_sqsets); 475 } 476 477 static int 478 csio_enable_msix(struct csio_hw *hw) 479 { 480 int i, j, k, n, min, cnt; 481 int extra = CSIO_EXTRA_VECS; 482 struct csio_scsi_cpu_info *info; 483 struct irq_affinity desc = { .pre_vectors = 2 }; 484 485 min = hw->num_pports + extra; 486 cnt = hw->num_sqsets + extra; 487 488 /* Max vectors required based on #niqs configured in fw */ 489 if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS || !csio_is_hw_master(hw)) 490 cnt = min_t(uint8_t, hw->cfg_niq, cnt); 491 492 csio_dbg(hw, "FW supp #niq:%d, trying %d msix's\n", hw->cfg_niq, cnt); 493 494 cnt = pci_alloc_irq_vectors_affinity(hw->pdev, min, cnt, 495 PCI_IRQ_MSIX | PCI_IRQ_AFFINITY, &desc); 496 if (cnt < 0) 497 return cnt; 498 499 if (cnt < (hw->num_sqsets + extra)) { 500 csio_dbg(hw, "Reducing sqsets to %d\n", cnt - extra); 501 csio_reduce_sqsets(hw, cnt - extra); 502 } 503 504 /* Distribute vectors */ 505 k = 0; 506 csio_set_nondata_intr_idx(hw, k); 507 csio_set_mb_intr_idx(csio_hw_to_mbm(hw), k++); 508 csio_set_fwevt_intr_idx(hw, k++); 509 510 for (i = 0; i < hw->num_pports; i++) { 511 info = &hw->scsi_cpu_info[i]; 512 513 for (j = 0; j < hw->num_scsi_msix_cpus; j++) { 514 n = (j % info->max_cpus) + k; 515 hw->sqset[i][j].intr_idx = n; 516 } 517 518 k += info->max_cpus; 519 } 520 521 return 0; 522 } 523 524 void 525 csio_intr_enable(struct csio_hw *hw) 526 { 527 hw->intr_mode = CSIO_IM_NONE; 528 hw->flags &= ~CSIO_HWF_HOST_INTR_ENABLED; 529 530 /* Try MSIX, then MSI or fall back to INTx */ 531 if ((csio_msi == 2) && !csio_enable_msix(hw)) 532 hw->intr_mode = CSIO_IM_MSIX; 533 else { 534 /* Max iqs required based on #niqs configured in fw */ 535 if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS || 536 !csio_is_hw_master(hw)) { 537 int extra = CSIO_EXTRA_MSI_IQS; 538 539 if (hw->cfg_niq < (hw->num_sqsets + extra)) { 540 csio_dbg(hw, "Reducing sqsets to %d\n", 541 hw->cfg_niq - extra); 542 csio_reduce_sqsets(hw, hw->cfg_niq - extra); 543 } 544 } 545 546 if ((csio_msi == 1) && !pci_enable_msi(hw->pdev)) 547 hw->intr_mode = CSIO_IM_MSI; 548 else 549 hw->intr_mode = CSIO_IM_INTX; 550 } 551 552 csio_dbg(hw, "Using %s interrupt mode.\n", 553 (hw->intr_mode == CSIO_IM_MSIX) ? "MSIX" : 554 ((hw->intr_mode == CSIO_IM_MSI) ? "MSI" : "INTx")); 555 } 556 557 void 558 csio_intr_disable(struct csio_hw *hw, bool free) 559 { 560 csio_hw_intr_disable(hw); 561 562 if (free) { 563 int i; 564 565 switch (hw->intr_mode) { 566 case CSIO_IM_MSIX: 567 for (i = 0; i < hw->num_sqsets + CSIO_EXTRA_VECS; i++) { 568 free_irq(pci_irq_vector(hw->pdev, i), 569 hw->msix_entries[i].dev_id); 570 } 571 break; 572 case CSIO_IM_MSI: 573 case CSIO_IM_INTX: 574 free_irq(pci_irq_vector(hw->pdev, 0), hw); 575 break; 576 default: 577 break; 578 } 579 } 580 581 pci_free_irq_vectors(hw->pdev); 582 hw->intr_mode = CSIO_IM_NONE; 583 hw->flags &= ~CSIO_HWF_HOST_INTR_ENABLED; 584 } 585