1 /******************************************************************* 2 * This file is part of the Emulex Linux Device Driver for * 3 * Fibre Channel Host Bus Adapters. * 4 * Copyright (C) 2017 Broadcom. All Rights Reserved. The term * 5 * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. * 6 * Copyright (C) 2004-2014 Emulex. All rights reserved. * 7 * EMULEX and SLI are trademarks of Emulex. * 8 * www.broadcom.com * 9 * Portions Copyright (C) 2004-2005 Christoph Hellwig * 10 * * 11 * This program is free software; you can redistribute it and/or * 12 * modify it under the terms of version 2 of the GNU General * 13 * Public License as published by the Free Software Foundation. * 14 * This program is distributed in the hope that it will be useful. * 15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * 16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * 17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * 18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * 19 * TO BE LEGALLY INVALID. See the GNU General Public License for * 20 * more details, a copy of which can be found in the file COPYING * 21 * included with this package. * 22 *******************************************************************/ 23 24 #include <linux/mempool.h> 25 #include <linux/slab.h> 26 #include <linux/pci.h> 27 #include <linux/interrupt.h> 28 29 #include <scsi/scsi.h> 30 #include <scsi/scsi_device.h> 31 #include <scsi/scsi_transport_fc.h> 32 #include <scsi/fc/fc_fs.h> 33 34 #include <linux/nvme-fc-driver.h> 35 36 #include "lpfc_hw4.h" 37 #include "lpfc_hw.h" 38 #include "lpfc_sli.h" 39 #include "lpfc_sli4.h" 40 #include "lpfc_nl.h" 41 #include "lpfc_disc.h" 42 #include "lpfc.h" 43 #include "lpfc_scsi.h" 44 #include "lpfc_nvme.h" 45 #include "lpfc_nvmet.h" 46 #include "lpfc_crtn.h" 47 #include "lpfc_logmsg.h" 48 49 #define LPFC_MBUF_POOL_SIZE 64 /* max elements in MBUF safety pool */ 50 #define LPFC_MEM_POOL_SIZE 64 /* max elem in non-DMA safety pool */ 51 #define LPFC_DEVICE_DATA_POOL_SIZE 64 /* max elements in device data pool */ 52 53 int 54 lpfc_mem_alloc_active_rrq_pool_s4(struct lpfc_hba *phba) { 55 size_t bytes; 56 int max_xri = phba->sli4_hba.max_cfg_param.max_xri; 57 58 if (max_xri <= 0) 59 return -ENOMEM; 60 bytes = ((BITS_PER_LONG - 1 + max_xri) / BITS_PER_LONG) * 61 sizeof(unsigned long); 62 phba->cfg_rrq_xri_bitmap_sz = bytes; 63 phba->active_rrq_pool = mempool_create_kmalloc_pool(LPFC_MEM_POOL_SIZE, 64 bytes); 65 if (!phba->active_rrq_pool) 66 return -ENOMEM; 67 else 68 return 0; 69 } 70 71 /** 72 * lpfc_mem_alloc - create and allocate all PCI and memory pools 73 * @phba: HBA to allocate pools for 74 * 75 * Description: Creates and allocates PCI pools lpfc_sg_dma_buf_pool, 76 * lpfc_mbuf_pool, lpfc_hrb_pool. Creates and allocates kmalloc-backed mempools 77 * for LPFC_MBOXQ_t and lpfc_nodelist. Also allocates the VPI bitmask. 78 * 79 * Notes: Not interrupt-safe. Must be called with no locks held. If any 80 * allocation fails, frees all successfully allocated memory before returning. 81 * 82 * Returns: 83 * 0 on success 84 * -ENOMEM on failure (if any memory allocations fail) 85 **/ 86 int 87 lpfc_mem_alloc(struct lpfc_hba *phba, int align) 88 { 89 struct lpfc_dma_pool *pool = &phba->lpfc_mbuf_safety_pool; 90 int i; 91 92 if (phba->sli_rev == LPFC_SLI_REV4) { 93 /* Calculate alignment */ 94 if (phba->cfg_sg_dma_buf_size < SLI4_PAGE_SIZE) 95 i = phba->cfg_sg_dma_buf_size; 96 else 97 i = SLI4_PAGE_SIZE; 98 99 phba->lpfc_sg_dma_buf_pool = 100 pci_pool_create("lpfc_sg_dma_buf_pool", 101 phba->pcidev, 102 phba->cfg_sg_dma_buf_size, 103 i, 0); 104 if (!phba->lpfc_sg_dma_buf_pool) 105 goto fail; 106 107 } else { 108 phba->lpfc_sg_dma_buf_pool = 109 pci_pool_create("lpfc_sg_dma_buf_pool", 110 phba->pcidev, phba->cfg_sg_dma_buf_size, 111 align, 0); 112 113 if (!phba->lpfc_sg_dma_buf_pool) 114 goto fail; 115 } 116 117 phba->lpfc_mbuf_pool = pci_pool_create("lpfc_mbuf_pool", phba->pcidev, 118 LPFC_BPL_SIZE, 119 align, 0); 120 if (!phba->lpfc_mbuf_pool) 121 goto fail_free_dma_buf_pool; 122 123 pool->elements = kmalloc(sizeof(struct lpfc_dmabuf) * 124 LPFC_MBUF_POOL_SIZE, GFP_KERNEL); 125 if (!pool->elements) 126 goto fail_free_lpfc_mbuf_pool; 127 128 pool->max_count = 0; 129 pool->current_count = 0; 130 for ( i = 0; i < LPFC_MBUF_POOL_SIZE; i++) { 131 pool->elements[i].virt = pci_pool_alloc(phba->lpfc_mbuf_pool, 132 GFP_KERNEL, &pool->elements[i].phys); 133 if (!pool->elements[i].virt) 134 goto fail_free_mbuf_pool; 135 pool->max_count++; 136 pool->current_count++; 137 } 138 139 phba->mbox_mem_pool = mempool_create_kmalloc_pool(LPFC_MEM_POOL_SIZE, 140 sizeof(LPFC_MBOXQ_t)); 141 if (!phba->mbox_mem_pool) 142 goto fail_free_mbuf_pool; 143 144 phba->nlp_mem_pool = mempool_create_kmalloc_pool(LPFC_MEM_POOL_SIZE, 145 sizeof(struct lpfc_nodelist)); 146 if (!phba->nlp_mem_pool) 147 goto fail_free_mbox_pool; 148 149 if (phba->sli_rev == LPFC_SLI_REV4) { 150 phba->rrq_pool = 151 mempool_create_kmalloc_pool(LPFC_MEM_POOL_SIZE, 152 sizeof(struct lpfc_node_rrq)); 153 if (!phba->rrq_pool) 154 goto fail_free_nlp_mem_pool; 155 phba->lpfc_hrb_pool = pci_pool_create("lpfc_hrb_pool", 156 phba->pcidev, 157 LPFC_HDR_BUF_SIZE, align, 0); 158 if (!phba->lpfc_hrb_pool) 159 goto fail_free_rrq_mem_pool; 160 161 phba->lpfc_drb_pool = pci_pool_create("lpfc_drb_pool", 162 phba->pcidev, 163 LPFC_DATA_BUF_SIZE, align, 0); 164 if (!phba->lpfc_drb_pool) 165 goto fail_free_hrb_pool; 166 phba->lpfc_hbq_pool = NULL; 167 } else { 168 phba->lpfc_hbq_pool = pci_pool_create("lpfc_hbq_pool", 169 phba->pcidev, LPFC_BPL_SIZE, align, 0); 170 if (!phba->lpfc_hbq_pool) 171 goto fail_free_nlp_mem_pool; 172 phba->lpfc_hrb_pool = NULL; 173 phba->lpfc_drb_pool = NULL; 174 } 175 176 if (phba->cfg_EnableXLane) { 177 phba->device_data_mem_pool = mempool_create_kmalloc_pool( 178 LPFC_DEVICE_DATA_POOL_SIZE, 179 sizeof(struct lpfc_device_data)); 180 if (!phba->device_data_mem_pool) 181 goto fail_free_drb_pool; 182 } else { 183 phba->device_data_mem_pool = NULL; 184 } 185 186 return 0; 187 fail_free_drb_pool: 188 pci_pool_destroy(phba->lpfc_drb_pool); 189 phba->lpfc_drb_pool = NULL; 190 fail_free_hrb_pool: 191 pci_pool_destroy(phba->lpfc_hrb_pool); 192 phba->lpfc_hrb_pool = NULL; 193 fail_free_rrq_mem_pool: 194 mempool_destroy(phba->rrq_pool); 195 phba->rrq_pool = NULL; 196 fail_free_nlp_mem_pool: 197 mempool_destroy(phba->nlp_mem_pool); 198 phba->nlp_mem_pool = NULL; 199 fail_free_mbox_pool: 200 mempool_destroy(phba->mbox_mem_pool); 201 phba->mbox_mem_pool = NULL; 202 fail_free_mbuf_pool: 203 while (i--) 204 pci_pool_free(phba->lpfc_mbuf_pool, pool->elements[i].virt, 205 pool->elements[i].phys); 206 kfree(pool->elements); 207 fail_free_lpfc_mbuf_pool: 208 pci_pool_destroy(phba->lpfc_mbuf_pool); 209 phba->lpfc_mbuf_pool = NULL; 210 fail_free_dma_buf_pool: 211 pci_pool_destroy(phba->lpfc_sg_dma_buf_pool); 212 phba->lpfc_sg_dma_buf_pool = NULL; 213 fail: 214 return -ENOMEM; 215 } 216 217 /** 218 * lpfc_mem_free - Frees memory allocated by lpfc_mem_alloc 219 * @phba: HBA to free memory for 220 * 221 * Description: Free the memory allocated by lpfc_mem_alloc routine. This 222 * routine is a the counterpart of lpfc_mem_alloc. 223 * 224 * Returns: None 225 **/ 226 void 227 lpfc_mem_free(struct lpfc_hba *phba) 228 { 229 int i; 230 struct lpfc_dma_pool *pool = &phba->lpfc_mbuf_safety_pool; 231 struct lpfc_device_data *device_data; 232 233 /* Free HBQ pools */ 234 lpfc_sli_hbqbuf_free_all(phba); 235 if (phba->lpfc_drb_pool) 236 pci_pool_destroy(phba->lpfc_drb_pool); 237 phba->lpfc_drb_pool = NULL; 238 if (phba->lpfc_hrb_pool) 239 pci_pool_destroy(phba->lpfc_hrb_pool); 240 phba->lpfc_hrb_pool = NULL; 241 if (phba->txrdy_payload_pool) 242 pci_pool_destroy(phba->txrdy_payload_pool); 243 phba->txrdy_payload_pool = NULL; 244 245 if (phba->lpfc_hbq_pool) 246 pci_pool_destroy(phba->lpfc_hbq_pool); 247 phba->lpfc_hbq_pool = NULL; 248 249 if (phba->rrq_pool) 250 mempool_destroy(phba->rrq_pool); 251 phba->rrq_pool = NULL; 252 253 /* Free NLP memory pool */ 254 mempool_destroy(phba->nlp_mem_pool); 255 phba->nlp_mem_pool = NULL; 256 if (phba->sli_rev == LPFC_SLI_REV4 && phba->active_rrq_pool) { 257 mempool_destroy(phba->active_rrq_pool); 258 phba->active_rrq_pool = NULL; 259 } 260 261 /* Free mbox memory pool */ 262 mempool_destroy(phba->mbox_mem_pool); 263 phba->mbox_mem_pool = NULL; 264 265 /* Free MBUF memory pool */ 266 for (i = 0; i < pool->current_count; i++) 267 pci_pool_free(phba->lpfc_mbuf_pool, pool->elements[i].virt, 268 pool->elements[i].phys); 269 kfree(pool->elements); 270 271 pci_pool_destroy(phba->lpfc_mbuf_pool); 272 phba->lpfc_mbuf_pool = NULL; 273 274 /* Free DMA buffer memory pool */ 275 pci_pool_destroy(phba->lpfc_sg_dma_buf_pool); 276 phba->lpfc_sg_dma_buf_pool = NULL; 277 278 /* Free Device Data memory pool */ 279 if (phba->device_data_mem_pool) { 280 /* Ensure all objects have been returned to the pool */ 281 while (!list_empty(&phba->luns)) { 282 device_data = list_first_entry(&phba->luns, 283 struct lpfc_device_data, 284 listentry); 285 list_del(&device_data->listentry); 286 mempool_free(device_data, phba->device_data_mem_pool); 287 } 288 mempool_destroy(phba->device_data_mem_pool); 289 } 290 phba->device_data_mem_pool = NULL; 291 return; 292 } 293 294 /** 295 * lpfc_mem_free_all - Frees all PCI and driver memory 296 * @phba: HBA to free memory for 297 * 298 * Description: Free memory from PCI and driver memory pools and also those 299 * used : lpfc_sg_dma_buf_pool, lpfc_mbuf_pool, lpfc_hrb_pool. Frees 300 * kmalloc-backed mempools for LPFC_MBOXQ_t and lpfc_nodelist. Also frees 301 * the VPI bitmask. 302 * 303 * Returns: None 304 **/ 305 void 306 lpfc_mem_free_all(struct lpfc_hba *phba) 307 { 308 struct lpfc_sli *psli = &phba->sli; 309 LPFC_MBOXQ_t *mbox, *next_mbox; 310 struct lpfc_dmabuf *mp; 311 312 /* Free memory used in mailbox queue back to mailbox memory pool */ 313 list_for_each_entry_safe(mbox, next_mbox, &psli->mboxq, list) { 314 mp = (struct lpfc_dmabuf *) (mbox->context1); 315 if (mp) { 316 lpfc_mbuf_free(phba, mp->virt, mp->phys); 317 kfree(mp); 318 } 319 list_del(&mbox->list); 320 mempool_free(mbox, phba->mbox_mem_pool); 321 } 322 /* Free memory used in mailbox cmpl list back to mailbox memory pool */ 323 list_for_each_entry_safe(mbox, next_mbox, &psli->mboxq_cmpl, list) { 324 mp = (struct lpfc_dmabuf *) (mbox->context1); 325 if (mp) { 326 lpfc_mbuf_free(phba, mp->virt, mp->phys); 327 kfree(mp); 328 } 329 list_del(&mbox->list); 330 mempool_free(mbox, phba->mbox_mem_pool); 331 } 332 /* Free the active mailbox command back to the mailbox memory pool */ 333 spin_lock_irq(&phba->hbalock); 334 psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE; 335 spin_unlock_irq(&phba->hbalock); 336 if (psli->mbox_active) { 337 mbox = psli->mbox_active; 338 mp = (struct lpfc_dmabuf *) (mbox->context1); 339 if (mp) { 340 lpfc_mbuf_free(phba, mp->virt, mp->phys); 341 kfree(mp); 342 } 343 mempool_free(mbox, phba->mbox_mem_pool); 344 psli->mbox_active = NULL; 345 } 346 347 /* Free and destroy all the allocated memory pools */ 348 lpfc_mem_free(phba); 349 350 /* Free the iocb lookup array */ 351 kfree(psli->iocbq_lookup); 352 psli->iocbq_lookup = NULL; 353 354 return; 355 } 356 357 /** 358 * lpfc_mbuf_alloc - Allocate an mbuf from the lpfc_mbuf_pool PCI pool 359 * @phba: HBA which owns the pool to allocate from 360 * @mem_flags: indicates if this is a priority (MEM_PRI) allocation 361 * @handle: used to return the DMA-mapped address of the mbuf 362 * 363 * Description: Allocates a DMA-mapped buffer from the lpfc_mbuf_pool PCI pool. 364 * Allocates from generic pci_pool_alloc function first and if that fails and 365 * mem_flags has MEM_PRI set (the only defined flag), returns an mbuf from the 366 * HBA's pool. 367 * 368 * Notes: Not interrupt-safe. Must be called with no locks held. Takes 369 * phba->hbalock. 370 * 371 * Returns: 372 * pointer to the allocated mbuf on success 373 * NULL on failure 374 **/ 375 void * 376 lpfc_mbuf_alloc(struct lpfc_hba *phba, int mem_flags, dma_addr_t *handle) 377 { 378 struct lpfc_dma_pool *pool = &phba->lpfc_mbuf_safety_pool; 379 unsigned long iflags; 380 void *ret; 381 382 ret = pci_pool_alloc(phba->lpfc_mbuf_pool, GFP_KERNEL, handle); 383 384 spin_lock_irqsave(&phba->hbalock, iflags); 385 if (!ret && (mem_flags & MEM_PRI) && pool->current_count) { 386 pool->current_count--; 387 ret = pool->elements[pool->current_count].virt; 388 *handle = pool->elements[pool->current_count].phys; 389 } 390 spin_unlock_irqrestore(&phba->hbalock, iflags); 391 return ret; 392 } 393 394 /** 395 * __lpfc_mbuf_free - Free an mbuf from the lpfc_mbuf_pool PCI pool (locked) 396 * @phba: HBA which owns the pool to return to 397 * @virt: mbuf to free 398 * @dma: the DMA-mapped address of the lpfc_mbuf_pool to be freed 399 * 400 * Description: Returns an mbuf lpfc_mbuf_pool to the lpfc_mbuf_safety_pool if 401 * it is below its max_count, frees the mbuf otherwise. 402 * 403 * Notes: Must be called with phba->hbalock held to synchronize access to 404 * lpfc_mbuf_safety_pool. 405 * 406 * Returns: None 407 **/ 408 void 409 __lpfc_mbuf_free(struct lpfc_hba * phba, void *virt, dma_addr_t dma) 410 { 411 struct lpfc_dma_pool *pool = &phba->lpfc_mbuf_safety_pool; 412 413 if (pool->current_count < pool->max_count) { 414 pool->elements[pool->current_count].virt = virt; 415 pool->elements[pool->current_count].phys = dma; 416 pool->current_count++; 417 } else { 418 pci_pool_free(phba->lpfc_mbuf_pool, virt, dma); 419 } 420 return; 421 } 422 423 /** 424 * lpfc_mbuf_free - Free an mbuf from the lpfc_mbuf_pool PCI pool (unlocked) 425 * @phba: HBA which owns the pool to return to 426 * @virt: mbuf to free 427 * @dma: the DMA-mapped address of the lpfc_mbuf_pool to be freed 428 * 429 * Description: Returns an mbuf lpfc_mbuf_pool to the lpfc_mbuf_safety_pool if 430 * it is below its max_count, frees the mbuf otherwise. 431 * 432 * Notes: Takes phba->hbalock. Can be called with or without other locks held. 433 * 434 * Returns: None 435 **/ 436 void 437 lpfc_mbuf_free(struct lpfc_hba * phba, void *virt, dma_addr_t dma) 438 { 439 unsigned long iflags; 440 441 spin_lock_irqsave(&phba->hbalock, iflags); 442 __lpfc_mbuf_free(phba, virt, dma); 443 spin_unlock_irqrestore(&phba->hbalock, iflags); 444 return; 445 } 446 447 /** 448 * lpfc_nvmet_buf_alloc - Allocate an nvmet_buf from the 449 * lpfc_sg_dma_buf_pool PCI pool 450 * @phba: HBA which owns the pool to allocate from 451 * @mem_flags: indicates if this is a priority (MEM_PRI) allocation 452 * @handle: used to return the DMA-mapped address of the nvmet_buf 453 * 454 * Description: Allocates a DMA-mapped buffer from the lpfc_sg_dma_buf_pool 455 * PCI pool. Allocates from generic pci_pool_alloc function. 456 * 457 * Returns: 458 * pointer to the allocated nvmet_buf on success 459 * NULL on failure 460 **/ 461 void * 462 lpfc_nvmet_buf_alloc(struct lpfc_hba *phba, int mem_flags, dma_addr_t *handle) 463 { 464 void *ret; 465 466 ret = pci_pool_alloc(phba->lpfc_sg_dma_buf_pool, GFP_KERNEL, handle); 467 return ret; 468 } 469 470 /** 471 * lpfc_nvmet_buf_free - Free an nvmet_buf from the lpfc_sg_dma_buf_pool 472 * PCI pool 473 * @phba: HBA which owns the pool to return to 474 * @virt: nvmet_buf to free 475 * @dma: the DMA-mapped address of the lpfc_sg_dma_buf_pool to be freed 476 * 477 * Returns: None 478 **/ 479 void 480 lpfc_nvmet_buf_free(struct lpfc_hba *phba, void *virt, dma_addr_t dma) 481 { 482 pci_pool_free(phba->lpfc_sg_dma_buf_pool, virt, dma); 483 } 484 485 /** 486 * lpfc_els_hbq_alloc - Allocate an HBQ buffer 487 * @phba: HBA to allocate HBQ buffer for 488 * 489 * Description: Allocates a DMA-mapped HBQ buffer from the lpfc_hrb_pool PCI 490 * pool along a non-DMA-mapped container for it. 491 * 492 * Notes: Not interrupt-safe. Must be called with no locks held. 493 * 494 * Returns: 495 * pointer to HBQ on success 496 * NULL on failure 497 **/ 498 struct hbq_dmabuf * 499 lpfc_els_hbq_alloc(struct lpfc_hba *phba) 500 { 501 struct hbq_dmabuf *hbqbp; 502 503 hbqbp = kzalloc(sizeof(struct hbq_dmabuf), GFP_KERNEL); 504 if (!hbqbp) 505 return NULL; 506 507 hbqbp->dbuf.virt = pci_pool_alloc(phba->lpfc_hbq_pool, GFP_KERNEL, 508 &hbqbp->dbuf.phys); 509 if (!hbqbp->dbuf.virt) { 510 kfree(hbqbp); 511 return NULL; 512 } 513 hbqbp->total_size = LPFC_BPL_SIZE; 514 return hbqbp; 515 } 516 517 /** 518 * lpfc_els_hbq_free - Frees an HBQ buffer allocated with lpfc_els_hbq_alloc 519 * @phba: HBA buffer was allocated for 520 * @hbqbp: HBQ container returned by lpfc_els_hbq_alloc 521 * 522 * Description: Frees both the container and the DMA-mapped buffer returned by 523 * lpfc_els_hbq_alloc. 524 * 525 * Notes: Can be called with or without locks held. 526 * 527 * Returns: None 528 **/ 529 void 530 lpfc_els_hbq_free(struct lpfc_hba *phba, struct hbq_dmabuf *hbqbp) 531 { 532 pci_pool_free(phba->lpfc_hbq_pool, hbqbp->dbuf.virt, hbqbp->dbuf.phys); 533 kfree(hbqbp); 534 return; 535 } 536 537 /** 538 * lpfc_sli4_rb_alloc - Allocate an SLI4 Receive buffer 539 * @phba: HBA to allocate a receive buffer for 540 * 541 * Description: Allocates a DMA-mapped receive buffer from the lpfc_hrb_pool PCI 542 * pool along a non-DMA-mapped container for it. 543 * 544 * Notes: Not interrupt-safe. Must be called with no locks held. 545 * 546 * Returns: 547 * pointer to HBQ on success 548 * NULL on failure 549 **/ 550 struct hbq_dmabuf * 551 lpfc_sli4_rb_alloc(struct lpfc_hba *phba) 552 { 553 struct hbq_dmabuf *dma_buf; 554 555 dma_buf = kzalloc(sizeof(struct hbq_dmabuf), GFP_KERNEL); 556 if (!dma_buf) 557 return NULL; 558 559 dma_buf->hbuf.virt = pci_pool_alloc(phba->lpfc_hrb_pool, GFP_KERNEL, 560 &dma_buf->hbuf.phys); 561 if (!dma_buf->hbuf.virt) { 562 kfree(dma_buf); 563 return NULL; 564 } 565 dma_buf->dbuf.virt = pci_pool_alloc(phba->lpfc_drb_pool, GFP_KERNEL, 566 &dma_buf->dbuf.phys); 567 if (!dma_buf->dbuf.virt) { 568 pci_pool_free(phba->lpfc_hrb_pool, dma_buf->hbuf.virt, 569 dma_buf->hbuf.phys); 570 kfree(dma_buf); 571 return NULL; 572 } 573 dma_buf->total_size = LPFC_DATA_BUF_SIZE; 574 return dma_buf; 575 } 576 577 /** 578 * lpfc_sli4_rb_free - Frees a receive buffer 579 * @phba: HBA buffer was allocated for 580 * @dmab: DMA Buffer container returned by lpfc_sli4_hbq_alloc 581 * 582 * Description: Frees both the container and the DMA-mapped buffers returned by 583 * lpfc_sli4_rb_alloc. 584 * 585 * Notes: Can be called with or without locks held. 586 * 587 * Returns: None 588 **/ 589 void 590 lpfc_sli4_rb_free(struct lpfc_hba *phba, struct hbq_dmabuf *dmab) 591 { 592 pci_pool_free(phba->lpfc_hrb_pool, dmab->hbuf.virt, dmab->hbuf.phys); 593 pci_pool_free(phba->lpfc_drb_pool, dmab->dbuf.virt, dmab->dbuf.phys); 594 kfree(dmab); 595 } 596 597 /** 598 * lpfc_sli4_nvmet_alloc - Allocate an SLI4 Receive buffer 599 * @phba: HBA to allocate a receive buffer for 600 * 601 * Description: Allocates a DMA-mapped receive buffer from the lpfc_hrb_pool PCI 602 * pool along a non-DMA-mapped container for it. 603 * 604 * Notes: Not interrupt-safe. Must be called with no locks held. 605 * 606 * Returns: 607 * pointer to HBQ on success 608 * NULL on failure 609 **/ 610 struct rqb_dmabuf * 611 lpfc_sli4_nvmet_alloc(struct lpfc_hba *phba) 612 { 613 struct rqb_dmabuf *dma_buf; 614 struct lpfc_iocbq *nvmewqe; 615 union lpfc_wqe128 *wqe; 616 617 dma_buf = kzalloc(sizeof(struct rqb_dmabuf), GFP_KERNEL); 618 if (!dma_buf) 619 return NULL; 620 621 dma_buf->hbuf.virt = pci_pool_alloc(phba->lpfc_hrb_pool, GFP_KERNEL, 622 &dma_buf->hbuf.phys); 623 if (!dma_buf->hbuf.virt) { 624 kfree(dma_buf); 625 return NULL; 626 } 627 dma_buf->dbuf.virt = pci_pool_alloc(phba->lpfc_drb_pool, GFP_KERNEL, 628 &dma_buf->dbuf.phys); 629 if (!dma_buf->dbuf.virt) { 630 pci_pool_free(phba->lpfc_hrb_pool, dma_buf->hbuf.virt, 631 dma_buf->hbuf.phys); 632 kfree(dma_buf); 633 return NULL; 634 } 635 dma_buf->total_size = LPFC_DATA_BUF_SIZE; 636 637 dma_buf->context = kzalloc(sizeof(struct lpfc_nvmet_rcv_ctx), 638 GFP_KERNEL); 639 if (!dma_buf->context) { 640 pci_pool_free(phba->lpfc_drb_pool, dma_buf->dbuf.virt, 641 dma_buf->dbuf.phys); 642 pci_pool_free(phba->lpfc_hrb_pool, dma_buf->hbuf.virt, 643 dma_buf->hbuf.phys); 644 kfree(dma_buf); 645 return NULL; 646 } 647 648 dma_buf->iocbq = lpfc_sli_get_iocbq(phba); 649 if (!dma_buf->iocbq) { 650 kfree(dma_buf->context); 651 pci_pool_free(phba->lpfc_drb_pool, dma_buf->dbuf.virt, 652 dma_buf->dbuf.phys); 653 pci_pool_free(phba->lpfc_hrb_pool, dma_buf->hbuf.virt, 654 dma_buf->hbuf.phys); 655 kfree(dma_buf); 656 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 657 "2621 Ran out of nvmet iocb/WQEs\n"); 658 return NULL; 659 } 660 dma_buf->iocbq->iocb_flag = LPFC_IO_NVMET; 661 nvmewqe = dma_buf->iocbq; 662 wqe = (union lpfc_wqe128 *)&nvmewqe->wqe; 663 /* Initialize WQE */ 664 memset(wqe, 0, sizeof(union lpfc_wqe)); 665 /* Word 7 */ 666 bf_set(wqe_ct, &wqe->generic.wqe_com, SLI4_CT_RPI); 667 bf_set(wqe_class, &wqe->generic.wqe_com, CLASS3); 668 bf_set(wqe_pu, &wqe->generic.wqe_com, 1); 669 /* Word 10 */ 670 bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1); 671 bf_set(wqe_ebde_cnt, &wqe->generic.wqe_com, 0); 672 bf_set(wqe_qosd, &wqe->generic.wqe_com, 0); 673 674 dma_buf->iocbq->context1 = NULL; 675 spin_lock(&phba->sli4_hba.sgl_list_lock); 676 dma_buf->sglq = __lpfc_sli_get_nvmet_sglq(phba, dma_buf->iocbq); 677 spin_unlock(&phba->sli4_hba.sgl_list_lock); 678 if (!dma_buf->sglq) { 679 lpfc_sli_release_iocbq(phba, dma_buf->iocbq); 680 kfree(dma_buf->context); 681 pci_pool_free(phba->lpfc_drb_pool, dma_buf->dbuf.virt, 682 dma_buf->dbuf.phys); 683 pci_pool_free(phba->lpfc_hrb_pool, dma_buf->hbuf.virt, 684 dma_buf->hbuf.phys); 685 kfree(dma_buf); 686 lpfc_printf_log(phba, KERN_ERR, LOG_NVME, 687 "6132 Ran out of nvmet XRIs\n"); 688 return NULL; 689 } 690 return dma_buf; 691 } 692 693 /** 694 * lpfc_sli4_nvmet_free - Frees a receive buffer 695 * @phba: HBA buffer was allocated for 696 * @dmab: DMA Buffer container returned by lpfc_sli4_rbq_alloc 697 * 698 * Description: Frees both the container and the DMA-mapped buffers returned by 699 * lpfc_sli4_nvmet_alloc. 700 * 701 * Notes: Can be called with or without locks held. 702 * 703 * Returns: None 704 **/ 705 void 706 lpfc_sli4_nvmet_free(struct lpfc_hba *phba, struct rqb_dmabuf *dmab) 707 { 708 unsigned long flags; 709 710 __lpfc_clear_active_sglq(phba, dmab->sglq->sli4_lxritag); 711 dmab->sglq->state = SGL_FREED; 712 dmab->sglq->ndlp = NULL; 713 714 spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock, flags); 715 list_add_tail(&dmab->sglq->list, &phba->sli4_hba.lpfc_nvmet_sgl_list); 716 spin_unlock_irqrestore(&phba->sli4_hba.sgl_list_lock, flags); 717 718 lpfc_sli_release_iocbq(phba, dmab->iocbq); 719 kfree(dmab->context); 720 pci_pool_free(phba->lpfc_hrb_pool, dmab->hbuf.virt, dmab->hbuf.phys); 721 pci_pool_free(phba->lpfc_drb_pool, dmab->dbuf.virt, dmab->dbuf.phys); 722 kfree(dmab); 723 } 724 725 /** 726 * lpfc_in_buf_free - Free a DMA buffer 727 * @phba: HBA buffer is associated with 728 * @mp: Buffer to free 729 * 730 * Description: Frees the given DMA buffer in the appropriate way given if the 731 * HBA is running in SLI3 mode with HBQs enabled. 732 * 733 * Notes: Takes phba->hbalock. Can be called with or without other locks held. 734 * 735 * Returns: None 736 **/ 737 void 738 lpfc_in_buf_free(struct lpfc_hba *phba, struct lpfc_dmabuf *mp) 739 { 740 struct hbq_dmabuf *hbq_entry; 741 unsigned long flags; 742 743 if (!mp) 744 return; 745 746 if (phba->sli3_options & LPFC_SLI3_HBQ_ENABLED) { 747 hbq_entry = container_of(mp, struct hbq_dmabuf, dbuf); 748 /* Check whether HBQ is still in use */ 749 spin_lock_irqsave(&phba->hbalock, flags); 750 if (!phba->hbq_in_use) { 751 spin_unlock_irqrestore(&phba->hbalock, flags); 752 return; 753 } 754 list_del(&hbq_entry->dbuf.list); 755 if (hbq_entry->tag == -1) { 756 (phba->hbqs[LPFC_ELS_HBQ].hbq_free_buffer) 757 (phba, hbq_entry); 758 } else { 759 lpfc_sli_free_hbq(phba, hbq_entry); 760 } 761 spin_unlock_irqrestore(&phba->hbalock, flags); 762 } else { 763 lpfc_mbuf_free(phba, mp->virt, mp->phys); 764 kfree(mp); 765 } 766 return; 767 } 768 769 /** 770 * lpfc_rq_buf_free - Free a RQ DMA buffer 771 * @phba: HBA buffer is associated with 772 * @mp: Buffer to free 773 * 774 * Description: Frees the given DMA buffer in the appropriate way given by 775 * reposting it to its associated RQ so it can be reused. 776 * 777 * Notes: Takes phba->hbalock. Can be called with or without other locks held. 778 * 779 * Returns: None 780 **/ 781 void 782 lpfc_rq_buf_free(struct lpfc_hba *phba, struct lpfc_dmabuf *mp) 783 { 784 struct lpfc_rqb *rqbp; 785 struct lpfc_rqe hrqe; 786 struct lpfc_rqe drqe; 787 struct rqb_dmabuf *rqb_entry; 788 unsigned long flags; 789 int rc; 790 791 if (!mp) 792 return; 793 794 rqb_entry = container_of(mp, struct rqb_dmabuf, hbuf); 795 rqbp = rqb_entry->hrq->rqbp; 796 797 spin_lock_irqsave(&phba->hbalock, flags); 798 list_del(&rqb_entry->hbuf.list); 799 hrqe.address_lo = putPaddrLow(rqb_entry->hbuf.phys); 800 hrqe.address_hi = putPaddrHigh(rqb_entry->hbuf.phys); 801 drqe.address_lo = putPaddrLow(rqb_entry->dbuf.phys); 802 drqe.address_hi = putPaddrHigh(rqb_entry->dbuf.phys); 803 rc = lpfc_sli4_rq_put(rqb_entry->hrq, rqb_entry->drq, &hrqe, &drqe); 804 if (rc < 0) { 805 (rqbp->rqb_free_buffer)(phba, rqb_entry); 806 } else { 807 list_add_tail(&rqb_entry->hbuf.list, &rqbp->rqb_buffer_list); 808 rqbp->buffer_count++; 809 } 810 811 spin_unlock_irqrestore(&phba->hbalock, flags); 812 } 813