1 /* 2 * This is the Fusion MPT base driver providing common API layer interface 3 * for access to MPT (Message Passing Technology) firmware. 4 * 5 * This code is based on drivers/scsi/mpt3sas/mpt3sas_base.c 6 * Copyright (C) 2012-2014 LSI Corporation 7 * Copyright (C) 2013-2014 Avago Technologies 8 * (mailto: MPT-FusionLinux.pdl@avagotech.com) 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU General Public License 12 * as published by the Free Software Foundation; either version 2 13 * of the License, or (at your option) any later version. 14 * 15 * This program is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 * GNU General Public License for more details. 19 * 20 * NO WARRANTY 21 * THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR 22 * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT 23 * LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, 24 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is 25 * solely responsible for determining the appropriateness of using and 26 * distributing the Program and assumes all risks associated with its 27 * exercise of rights under this Agreement, including but not limited to 28 * the risks and costs of program errors, damage to or loss of data, 29 * programs or equipment, and unavailability or interruption of operations. 30 31 * DISCLAIMER OF LIABILITY 32 * NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY 33 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 * DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND 35 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 36 * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 37 * USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED 38 * HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES 39 40 * You should have received a copy of the GNU General Public License 41 * along with this program; if not, write to the Free Software 42 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, 43 * USA. 44 */ 45 46 #include <linux/kernel.h> 47 #include <linux/module.h> 48 #include <linux/errno.h> 49 #include <linux/init.h> 50 #include <linux/slab.h> 51 #include <linux/types.h> 52 #include <linux/pci.h> 53 #include <linux/kdev_t.h> 54 #include <linux/blkdev.h> 55 #include <linux/delay.h> 56 #include <linux/interrupt.h> 57 #include <linux/dma-mapping.h> 58 #include <linux/io.h> 59 #include <linux/time.h> 60 #include <linux/ktime.h> 61 #include <linux/kthread.h> 62 #include <asm/page.h> /* To get host page size per arch */ 63 #include <linux/aer.h> 64 65 66 #include "mpt3sas_base.h" 67 68 static MPT_CALLBACK mpt_callbacks[MPT_MAX_CALLBACKS]; 69 70 71 #define FAULT_POLLING_INTERVAL 1000 /* in milliseconds */ 72 73 /* maximum controller queue depth */ 74 #define MAX_HBA_QUEUE_DEPTH 30000 75 #define MAX_CHAIN_DEPTH 100000 76 static int max_queue_depth = -1; 77 module_param(max_queue_depth, int, 0444); 78 MODULE_PARM_DESC(max_queue_depth, " max controller queue depth "); 79 80 static int max_sgl_entries = -1; 81 module_param(max_sgl_entries, int, 0444); 82 MODULE_PARM_DESC(max_sgl_entries, " max sg entries "); 83 84 static int msix_disable = -1; 85 module_param(msix_disable, int, 0444); 86 MODULE_PARM_DESC(msix_disable, " disable msix routed interrupts (default=0)"); 87 88 static int smp_affinity_enable = 1; 89 module_param(smp_affinity_enable, int, 0444); 90 MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disable Default: enable(1)"); 91 92 static int max_msix_vectors = -1; 93 module_param(max_msix_vectors, int, 0444); 94 MODULE_PARM_DESC(max_msix_vectors, 95 " max msix vectors"); 96 97 static int irqpoll_weight = -1; 98 module_param(irqpoll_weight, int, 0444); 99 MODULE_PARM_DESC(irqpoll_weight, 100 "irq poll weight (default= one fourth of HBA queue depth)"); 101 102 static int mpt3sas_fwfault_debug; 103 MODULE_PARM_DESC(mpt3sas_fwfault_debug, 104 " enable detection of firmware fault and halt firmware - (default=0)"); 105 106 static int perf_mode = -1; 107 module_param(perf_mode, int, 0444); 108 MODULE_PARM_DESC(perf_mode, 109 "Performance mode (only for Aero/Sea Generation), options:\n\t\t" 110 "0 - balanced: high iops mode is enabled &\n\t\t" 111 "interrupt coalescing is enabled only on high iops queues,\n\t\t" 112 "1 - iops: high iops mode is disabled &\n\t\t" 113 "interrupt coalescing is enabled on all queues,\n\t\t" 114 "2 - latency: high iops mode is disabled &\n\t\t" 115 "interrupt coalescing is enabled on all queues with timeout value 0xA,\n" 116 "\t\tdefault - default perf_mode is 'balanced'" 117 ); 118 119 static int poll_queues; 120 module_param(poll_queues, int, 0444); 121 MODULE_PARM_DESC(poll_queues, "Number of queues to be use for io_uring poll mode.\n\t\t" 122 "This parameter is effective only if host_tagset_enable=1. &\n\t\t" 123 "when poll_queues are enabled then &\n\t\t" 124 "perf_mode is set to latency mode. &\n\t\t" 125 ); 126 127 enum mpt3sas_perf_mode { 128 MPT_PERF_MODE_DEFAULT = -1, 129 MPT_PERF_MODE_BALANCED = 0, 130 MPT_PERF_MODE_IOPS = 1, 131 MPT_PERF_MODE_LATENCY = 2, 132 }; 133 134 static int 135 _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, 136 u32 ioc_state, int timeout); 137 static int 138 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc); 139 static void 140 _base_clear_outstanding_commands(struct MPT3SAS_ADAPTER *ioc); 141 142 /** 143 * mpt3sas_base_check_cmd_timeout - Function 144 * to check timeout and command termination due 145 * to Host reset. 146 * 147 * @ioc: per adapter object. 148 * @status: Status of issued command. 149 * @mpi_request:mf request pointer. 150 * @sz: size of buffer. 151 * 152 * Return: 1/0 Reset to be done or Not 153 */ 154 u8 155 mpt3sas_base_check_cmd_timeout(struct MPT3SAS_ADAPTER *ioc, 156 u8 status, void *mpi_request, int sz) 157 { 158 u8 issue_reset = 0; 159 160 if (!(status & MPT3_CMD_RESET)) 161 issue_reset = 1; 162 163 ioc_err(ioc, "Command %s\n", 164 issue_reset == 0 ? "terminated due to Host Reset" : "Timeout"); 165 _debug_dump_mf(mpi_request, sz); 166 167 return issue_reset; 168 } 169 170 /** 171 * _scsih_set_fwfault_debug - global setting of ioc->fwfault_debug. 172 * @val: ? 173 * @kp: ? 174 * 175 * Return: ? 176 */ 177 static int 178 _scsih_set_fwfault_debug(const char *val, const struct kernel_param *kp) 179 { 180 int ret = param_set_int(val, kp); 181 struct MPT3SAS_ADAPTER *ioc; 182 183 if (ret) 184 return ret; 185 186 /* global ioc spinlock to protect controller list on list operations */ 187 pr_info("setting fwfault_debug(%d)\n", mpt3sas_fwfault_debug); 188 spin_lock(&gioc_lock); 189 list_for_each_entry(ioc, &mpt3sas_ioc_list, list) 190 ioc->fwfault_debug = mpt3sas_fwfault_debug; 191 spin_unlock(&gioc_lock); 192 return 0; 193 } 194 module_param_call(mpt3sas_fwfault_debug, _scsih_set_fwfault_debug, 195 param_get_int, &mpt3sas_fwfault_debug, 0644); 196 197 /** 198 * _base_readl_aero - retry readl for max three times. 199 * @addr: MPT Fusion system interface register address 200 * 201 * Retry the readl() for max three times if it gets zero value 202 * while reading the system interface register. 203 */ 204 static inline u32 205 _base_readl_aero(const volatile void __iomem *addr) 206 { 207 u32 i = 0, ret_val; 208 209 do { 210 ret_val = readl(addr); 211 i++; 212 } while (ret_val == 0 && i < 3); 213 214 return ret_val; 215 } 216 217 static inline u32 218 _base_readl(const volatile void __iomem *addr) 219 { 220 return readl(addr); 221 } 222 223 /** 224 * _base_clone_reply_to_sys_mem - copies reply to reply free iomem 225 * in BAR0 space. 226 * 227 * @ioc: per adapter object 228 * @reply: reply message frame(lower 32bit addr) 229 * @index: System request message index. 230 */ 231 static void 232 _base_clone_reply_to_sys_mem(struct MPT3SAS_ADAPTER *ioc, u32 reply, 233 u32 index) 234 { 235 /* 236 * 256 is offset within sys register. 237 * 256 offset MPI frame starts. Max MPI frame supported is 32. 238 * 32 * 128 = 4K. From here, Clone of reply free for mcpu starts 239 */ 240 u16 cmd_credit = ioc->facts.RequestCredit + 1; 241 void __iomem *reply_free_iomem = (void __iomem *)ioc->chip + 242 MPI_FRAME_START_OFFSET + 243 (cmd_credit * ioc->request_sz) + (index * sizeof(u32)); 244 245 writel(reply, reply_free_iomem); 246 } 247 248 /** 249 * _base_clone_mpi_to_sys_mem - Writes/copies MPI frames 250 * to system/BAR0 region. 251 * 252 * @dst_iomem: Pointer to the destination location in BAR0 space. 253 * @src: Pointer to the Source data. 254 * @size: Size of data to be copied. 255 */ 256 static void 257 _base_clone_mpi_to_sys_mem(void *dst_iomem, void *src, u32 size) 258 { 259 int i; 260 u32 *src_virt_mem = (u32 *)src; 261 262 for (i = 0; i < size/4; i++) 263 writel((u32)src_virt_mem[i], 264 (void __iomem *)dst_iomem + (i * 4)); 265 } 266 267 /** 268 * _base_clone_to_sys_mem - Writes/copies data to system/BAR0 region 269 * 270 * @dst_iomem: Pointer to the destination location in BAR0 space. 271 * @src: Pointer to the Source data. 272 * @size: Size of data to be copied. 273 */ 274 static void 275 _base_clone_to_sys_mem(void __iomem *dst_iomem, void *src, u32 size) 276 { 277 int i; 278 u32 *src_virt_mem = (u32 *)(src); 279 280 for (i = 0; i < size/4; i++) 281 writel((u32)src_virt_mem[i], 282 (void __iomem *)dst_iomem + (i * 4)); 283 } 284 285 /** 286 * _base_get_chain - Calculates and Returns virtual chain address 287 * for the provided smid in BAR0 space. 288 * 289 * @ioc: per adapter object 290 * @smid: system request message index 291 * @sge_chain_count: Scatter gather chain count. 292 * 293 * Return: the chain address. 294 */ 295 static inline void __iomem* 296 _base_get_chain(struct MPT3SAS_ADAPTER *ioc, u16 smid, 297 u8 sge_chain_count) 298 { 299 void __iomem *base_chain, *chain_virt; 300 u16 cmd_credit = ioc->facts.RequestCredit + 1; 301 302 base_chain = (void __iomem *)ioc->chip + MPI_FRAME_START_OFFSET + 303 (cmd_credit * ioc->request_sz) + 304 REPLY_FREE_POOL_SIZE; 305 chain_virt = base_chain + (smid * ioc->facts.MaxChainDepth * 306 ioc->request_sz) + (sge_chain_count * ioc->request_sz); 307 return chain_virt; 308 } 309 310 /** 311 * _base_get_chain_phys - Calculates and Returns physical address 312 * in BAR0 for scatter gather chains, for 313 * the provided smid. 314 * 315 * @ioc: per adapter object 316 * @smid: system request message index 317 * @sge_chain_count: Scatter gather chain count. 318 * 319 * Return: Physical chain address. 320 */ 321 static inline phys_addr_t 322 _base_get_chain_phys(struct MPT3SAS_ADAPTER *ioc, u16 smid, 323 u8 sge_chain_count) 324 { 325 phys_addr_t base_chain_phys, chain_phys; 326 u16 cmd_credit = ioc->facts.RequestCredit + 1; 327 328 base_chain_phys = ioc->chip_phys + MPI_FRAME_START_OFFSET + 329 (cmd_credit * ioc->request_sz) + 330 REPLY_FREE_POOL_SIZE; 331 chain_phys = base_chain_phys + (smid * ioc->facts.MaxChainDepth * 332 ioc->request_sz) + (sge_chain_count * ioc->request_sz); 333 return chain_phys; 334 } 335 336 /** 337 * _base_get_buffer_bar0 - Calculates and Returns BAR0 mapped Host 338 * buffer address for the provided smid. 339 * (Each smid can have 64K starts from 17024) 340 * 341 * @ioc: per adapter object 342 * @smid: system request message index 343 * 344 * Return: Pointer to buffer location in BAR0. 345 */ 346 347 static void __iomem * 348 _base_get_buffer_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid) 349 { 350 u16 cmd_credit = ioc->facts.RequestCredit + 1; 351 // Added extra 1 to reach end of chain. 352 void __iomem *chain_end = _base_get_chain(ioc, 353 cmd_credit + 1, 354 ioc->facts.MaxChainDepth); 355 return chain_end + (smid * 64 * 1024); 356 } 357 358 /** 359 * _base_get_buffer_phys_bar0 - Calculates and Returns BAR0 mapped 360 * Host buffer Physical address for the provided smid. 361 * (Each smid can have 64K starts from 17024) 362 * 363 * @ioc: per adapter object 364 * @smid: system request message index 365 * 366 * Return: Pointer to buffer location in BAR0. 367 */ 368 static phys_addr_t 369 _base_get_buffer_phys_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid) 370 { 371 u16 cmd_credit = ioc->facts.RequestCredit + 1; 372 phys_addr_t chain_end_phys = _base_get_chain_phys(ioc, 373 cmd_credit + 1, 374 ioc->facts.MaxChainDepth); 375 return chain_end_phys + (smid * 64 * 1024); 376 } 377 378 /** 379 * _base_get_chain_buffer_dma_to_chain_buffer - Iterates chain 380 * lookup list and Provides chain_buffer 381 * address for the matching dma address. 382 * (Each smid can have 64K starts from 17024) 383 * 384 * @ioc: per adapter object 385 * @chain_buffer_dma: Chain buffer dma address. 386 * 387 * Return: Pointer to chain buffer. Or Null on Failure. 388 */ 389 static void * 390 _base_get_chain_buffer_dma_to_chain_buffer(struct MPT3SAS_ADAPTER *ioc, 391 dma_addr_t chain_buffer_dma) 392 { 393 u16 index, j; 394 struct chain_tracker *ct; 395 396 for (index = 0; index < ioc->scsiio_depth; index++) { 397 for (j = 0; j < ioc->chains_needed_per_io; j++) { 398 ct = &ioc->chain_lookup[index].chains_per_smid[j]; 399 if (ct && ct->chain_buffer_dma == chain_buffer_dma) 400 return ct->chain_buffer; 401 } 402 } 403 ioc_info(ioc, "Provided chain_buffer_dma address is not in the lookup list\n"); 404 return NULL; 405 } 406 407 /** 408 * _clone_sg_entries - MPI EP's scsiio and config requests 409 * are handled here. Base function for 410 * double buffering, before submitting 411 * the requests. 412 * 413 * @ioc: per adapter object. 414 * @mpi_request: mf request pointer. 415 * @smid: system request message index. 416 */ 417 static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc, 418 void *mpi_request, u16 smid) 419 { 420 Mpi2SGESimple32_t *sgel, *sgel_next; 421 u32 sgl_flags, sge_chain_count = 0; 422 bool is_write = false; 423 u16 i = 0; 424 void __iomem *buffer_iomem; 425 phys_addr_t buffer_iomem_phys; 426 void __iomem *buff_ptr; 427 phys_addr_t buff_ptr_phys; 428 void __iomem *dst_chain_addr[MCPU_MAX_CHAINS_PER_IO]; 429 void *src_chain_addr[MCPU_MAX_CHAINS_PER_IO]; 430 phys_addr_t dst_addr_phys; 431 MPI2RequestHeader_t *request_hdr; 432 struct scsi_cmnd *scmd; 433 struct scatterlist *sg_scmd = NULL; 434 int is_scsiio_req = 0; 435 436 request_hdr = (MPI2RequestHeader_t *) mpi_request; 437 438 if (request_hdr->Function == MPI2_FUNCTION_SCSI_IO_REQUEST) { 439 Mpi25SCSIIORequest_t *scsiio_request = 440 (Mpi25SCSIIORequest_t *)mpi_request; 441 sgel = (Mpi2SGESimple32_t *) &scsiio_request->SGL; 442 is_scsiio_req = 1; 443 } else if (request_hdr->Function == MPI2_FUNCTION_CONFIG) { 444 Mpi2ConfigRequest_t *config_req = 445 (Mpi2ConfigRequest_t *)mpi_request; 446 sgel = (Mpi2SGESimple32_t *) &config_req->PageBufferSGE; 447 } else 448 return; 449 450 /* From smid we can get scsi_cmd, once we have sg_scmd, 451 * we just need to get sg_virt and sg_next to get virtual 452 * address associated with sgel->Address. 453 */ 454 455 if (is_scsiio_req) { 456 /* Get scsi_cmd using smid */ 457 scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid); 458 if (scmd == NULL) { 459 ioc_err(ioc, "scmd is NULL\n"); 460 return; 461 } 462 463 /* Get sg_scmd from scmd provided */ 464 sg_scmd = scsi_sglist(scmd); 465 } 466 467 /* 468 * 0 - 255 System register 469 * 256 - 4352 MPI Frame. (This is based on maxCredit 32) 470 * 4352 - 4864 Reply_free pool (512 byte is reserved 471 * considering maxCredit 32. Reply need extra 472 * room, for mCPU case kept four times of 473 * maxCredit). 474 * 4864 - 17152 SGE chain element. (32cmd * 3 chain of 475 * 128 byte size = 12288) 476 * 17152 - x Host buffer mapped with smid. 477 * (Each smid can have 64K Max IO.) 478 * BAR0+Last 1K MSIX Addr and Data 479 * Total size in use 2113664 bytes of 4MB BAR0 480 */ 481 482 buffer_iomem = _base_get_buffer_bar0(ioc, smid); 483 buffer_iomem_phys = _base_get_buffer_phys_bar0(ioc, smid); 484 485 buff_ptr = buffer_iomem; 486 buff_ptr_phys = buffer_iomem_phys; 487 WARN_ON(buff_ptr_phys > U32_MAX); 488 489 if (le32_to_cpu(sgel->FlagsLength) & 490 (MPI2_SGE_FLAGS_HOST_TO_IOC << MPI2_SGE_FLAGS_SHIFT)) 491 is_write = true; 492 493 for (i = 0; i < MPT_MIN_PHYS_SEGMENTS + ioc->facts.MaxChainDepth; i++) { 494 495 sgl_flags = 496 (le32_to_cpu(sgel->FlagsLength) >> MPI2_SGE_FLAGS_SHIFT); 497 498 switch (sgl_flags & MPI2_SGE_FLAGS_ELEMENT_MASK) { 499 case MPI2_SGE_FLAGS_CHAIN_ELEMENT: 500 /* 501 * Helper function which on passing 502 * chain_buffer_dma returns chain_buffer. Get 503 * the virtual address for sgel->Address 504 */ 505 sgel_next = 506 _base_get_chain_buffer_dma_to_chain_buffer(ioc, 507 le32_to_cpu(sgel->Address)); 508 if (sgel_next == NULL) 509 return; 510 /* 511 * This is coping 128 byte chain 512 * frame (not a host buffer) 513 */ 514 dst_chain_addr[sge_chain_count] = 515 _base_get_chain(ioc, 516 smid, sge_chain_count); 517 src_chain_addr[sge_chain_count] = 518 (void *) sgel_next; 519 dst_addr_phys = _base_get_chain_phys(ioc, 520 smid, sge_chain_count); 521 WARN_ON(dst_addr_phys > U32_MAX); 522 sgel->Address = 523 cpu_to_le32(lower_32_bits(dst_addr_phys)); 524 sgel = sgel_next; 525 sge_chain_count++; 526 break; 527 case MPI2_SGE_FLAGS_SIMPLE_ELEMENT: 528 if (is_write) { 529 if (is_scsiio_req) { 530 _base_clone_to_sys_mem(buff_ptr, 531 sg_virt(sg_scmd), 532 (le32_to_cpu(sgel->FlagsLength) & 533 0x00ffffff)); 534 /* 535 * FIXME: this relies on a a zero 536 * PCI mem_offset. 537 */ 538 sgel->Address = 539 cpu_to_le32((u32)buff_ptr_phys); 540 } else { 541 _base_clone_to_sys_mem(buff_ptr, 542 ioc->config_vaddr, 543 (le32_to_cpu(sgel->FlagsLength) & 544 0x00ffffff)); 545 sgel->Address = 546 cpu_to_le32((u32)buff_ptr_phys); 547 } 548 } 549 buff_ptr += (le32_to_cpu(sgel->FlagsLength) & 550 0x00ffffff); 551 buff_ptr_phys += (le32_to_cpu(sgel->FlagsLength) & 552 0x00ffffff); 553 if ((le32_to_cpu(sgel->FlagsLength) & 554 (MPI2_SGE_FLAGS_END_OF_BUFFER 555 << MPI2_SGE_FLAGS_SHIFT))) 556 goto eob_clone_chain; 557 else { 558 /* 559 * Every single element in MPT will have 560 * associated sg_next. Better to sanity that 561 * sg_next is not NULL, but it will be a bug 562 * if it is null. 563 */ 564 if (is_scsiio_req) { 565 sg_scmd = sg_next(sg_scmd); 566 if (sg_scmd) 567 sgel++; 568 else 569 goto eob_clone_chain; 570 } 571 } 572 break; 573 } 574 } 575 576 eob_clone_chain: 577 for (i = 0; i < sge_chain_count; i++) { 578 if (is_scsiio_req) 579 _base_clone_to_sys_mem(dst_chain_addr[i], 580 src_chain_addr[i], ioc->request_sz); 581 } 582 } 583 584 /** 585 * mpt3sas_remove_dead_ioc_func - kthread context to remove dead ioc 586 * @arg: input argument, used to derive ioc 587 * 588 * Return: 589 * 0 if controller is removed from pci subsystem. 590 * -1 for other case. 591 */ 592 static int mpt3sas_remove_dead_ioc_func(void *arg) 593 { 594 struct MPT3SAS_ADAPTER *ioc = (struct MPT3SAS_ADAPTER *)arg; 595 struct pci_dev *pdev; 596 597 if (!ioc) 598 return -1; 599 600 pdev = ioc->pdev; 601 if (!pdev) 602 return -1; 603 pci_stop_and_remove_bus_device_locked(pdev); 604 return 0; 605 } 606 607 /** 608 * _base_sync_drv_fw_timestamp - Sync Drive-Fw TimeStamp. 609 * @ioc: Per Adapter Object 610 * 611 * Return: nothing. 612 */ 613 static void _base_sync_drv_fw_timestamp(struct MPT3SAS_ADAPTER *ioc) 614 { 615 Mpi26IoUnitControlRequest_t *mpi_request; 616 Mpi26IoUnitControlReply_t *mpi_reply; 617 u16 smid; 618 ktime_t current_time; 619 u64 TimeStamp = 0; 620 u8 issue_reset = 0; 621 622 mutex_lock(&ioc->scsih_cmds.mutex); 623 if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) { 624 ioc_err(ioc, "scsih_cmd in use %s\n", __func__); 625 goto out; 626 } 627 ioc->scsih_cmds.status = MPT3_CMD_PENDING; 628 smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx); 629 if (!smid) { 630 ioc_err(ioc, "Failed obtaining a smid %s\n", __func__); 631 ioc->scsih_cmds.status = MPT3_CMD_NOT_USED; 632 goto out; 633 } 634 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); 635 ioc->scsih_cmds.smid = smid; 636 memset(mpi_request, 0, sizeof(Mpi26IoUnitControlRequest_t)); 637 mpi_request->Function = MPI2_FUNCTION_IO_UNIT_CONTROL; 638 mpi_request->Operation = MPI26_CTRL_OP_SET_IOC_PARAMETER; 639 mpi_request->IOCParameter = MPI26_SET_IOC_PARAMETER_SYNC_TIMESTAMP; 640 current_time = ktime_get_real(); 641 TimeStamp = ktime_to_ms(current_time); 642 mpi_request->Reserved7 = cpu_to_le32(TimeStamp >> 32); 643 mpi_request->IOCParameterValue = cpu_to_le32(TimeStamp & 0xFFFFFFFF); 644 init_completion(&ioc->scsih_cmds.done); 645 ioc->put_smid_default(ioc, smid); 646 dinitprintk(ioc, ioc_info(ioc, 647 "Io Unit Control Sync TimeStamp (sending), @time %lld ms\n", 648 TimeStamp)); 649 wait_for_completion_timeout(&ioc->scsih_cmds.done, 650 MPT3SAS_TIMESYNC_TIMEOUT_SECONDS*HZ); 651 if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) { 652 mpt3sas_check_cmd_timeout(ioc, 653 ioc->scsih_cmds.status, mpi_request, 654 sizeof(Mpi2SasIoUnitControlRequest_t)/4, issue_reset); 655 goto issue_host_reset; 656 } 657 if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) { 658 mpi_reply = ioc->scsih_cmds.reply; 659 dinitprintk(ioc, ioc_info(ioc, 660 "Io Unit Control sync timestamp (complete): ioc_status(0x%04x), loginfo(0x%08x)\n", 661 le16_to_cpu(mpi_reply->IOCStatus), 662 le32_to_cpu(mpi_reply->IOCLogInfo))); 663 } 664 issue_host_reset: 665 if (issue_reset) 666 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER); 667 ioc->scsih_cmds.status = MPT3_CMD_NOT_USED; 668 out: 669 mutex_unlock(&ioc->scsih_cmds.mutex); 670 } 671 672 /** 673 * _base_fault_reset_work - workq handling ioc fault conditions 674 * @work: input argument, used to derive ioc 675 * 676 * Context: sleep. 677 */ 678 static void 679 _base_fault_reset_work(struct work_struct *work) 680 { 681 struct MPT3SAS_ADAPTER *ioc = 682 container_of(work, struct MPT3SAS_ADAPTER, fault_reset_work.work); 683 unsigned long flags; 684 u32 doorbell; 685 int rc; 686 struct task_struct *p; 687 688 689 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); 690 if ((ioc->shost_recovery && (ioc->ioc_coredump_loop == 0)) || 691 ioc->pci_error_recovery) 692 goto rearm_timer; 693 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); 694 695 doorbell = mpt3sas_base_get_iocstate(ioc, 0); 696 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_MASK) { 697 ioc_err(ioc, "SAS host is non-operational !!!!\n"); 698 699 /* It may be possible that EEH recovery can resolve some of 700 * pci bus failure issues rather removing the dead ioc function 701 * by considering controller is in a non-operational state. So 702 * here priority is given to the EEH recovery. If it doesn't 703 * not resolve this issue, mpt3sas driver will consider this 704 * controller to non-operational state and remove the dead ioc 705 * function. 706 */ 707 if (ioc->non_operational_loop++ < 5) { 708 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, 709 flags); 710 goto rearm_timer; 711 } 712 713 /* 714 * Call _scsih_flush_pending_cmds callback so that we flush all 715 * pending commands back to OS. This call is required to avoid 716 * deadlock at block layer. Dead IOC will fail to do diag reset, 717 * and this call is safe since dead ioc will never return any 718 * command back from HW. 719 */ 720 mpt3sas_base_pause_mq_polling(ioc); 721 ioc->schedule_dead_ioc_flush_running_cmds(ioc); 722 /* 723 * Set remove_host flag early since kernel thread will 724 * take some time to execute. 725 */ 726 ioc->remove_host = 1; 727 /*Remove the Dead Host */ 728 p = kthread_run(mpt3sas_remove_dead_ioc_func, ioc, 729 "%s_dead_ioc_%d", ioc->driver_name, ioc->id); 730 if (IS_ERR(p)) 731 ioc_err(ioc, "%s: Running mpt3sas_dead_ioc thread failed !!!!\n", 732 __func__); 733 else 734 ioc_err(ioc, "%s: Running mpt3sas_dead_ioc thread success !!!!\n", 735 __func__); 736 return; /* don't rearm timer */ 737 } 738 739 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_COREDUMP) { 740 u8 timeout = (ioc->manu_pg11.CoreDumpTOSec) ? 741 ioc->manu_pg11.CoreDumpTOSec : 742 MPT3SAS_DEFAULT_COREDUMP_TIMEOUT_SECONDS; 743 744 timeout /= (FAULT_POLLING_INTERVAL/1000); 745 746 if (ioc->ioc_coredump_loop == 0) { 747 mpt3sas_print_coredump_info(ioc, 748 doorbell & MPI2_DOORBELL_DATA_MASK); 749 /* do not accept any IOs and disable the interrupts */ 750 spin_lock_irqsave( 751 &ioc->ioc_reset_in_progress_lock, flags); 752 ioc->shost_recovery = 1; 753 spin_unlock_irqrestore( 754 &ioc->ioc_reset_in_progress_lock, flags); 755 mpt3sas_base_mask_interrupts(ioc); 756 mpt3sas_base_pause_mq_polling(ioc); 757 _base_clear_outstanding_commands(ioc); 758 } 759 760 ioc_info(ioc, "%s: CoreDump loop %d.", 761 __func__, ioc->ioc_coredump_loop); 762 763 /* Wait until CoreDump completes or times out */ 764 if (ioc->ioc_coredump_loop++ < timeout) { 765 spin_lock_irqsave( 766 &ioc->ioc_reset_in_progress_lock, flags); 767 goto rearm_timer; 768 } 769 } 770 771 if (ioc->ioc_coredump_loop) { 772 if ((doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_COREDUMP) 773 ioc_err(ioc, "%s: CoreDump completed. LoopCount: %d", 774 __func__, ioc->ioc_coredump_loop); 775 else 776 ioc_err(ioc, "%s: CoreDump Timed out. LoopCount: %d", 777 __func__, ioc->ioc_coredump_loop); 778 ioc->ioc_coredump_loop = MPT3SAS_COREDUMP_LOOP_DONE; 779 } 780 ioc->non_operational_loop = 0; 781 if ((doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL) { 782 rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER); 783 ioc_warn(ioc, "%s: hard reset: %s\n", 784 __func__, rc == 0 ? "success" : "failed"); 785 doorbell = mpt3sas_base_get_iocstate(ioc, 0); 786 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { 787 mpt3sas_print_fault_code(ioc, doorbell & 788 MPI2_DOORBELL_DATA_MASK); 789 } else if ((doorbell & MPI2_IOC_STATE_MASK) == 790 MPI2_IOC_STATE_COREDUMP) 791 mpt3sas_print_coredump_info(ioc, doorbell & 792 MPI2_DOORBELL_DATA_MASK); 793 if (rc && (doorbell & MPI2_IOC_STATE_MASK) != 794 MPI2_IOC_STATE_OPERATIONAL) 795 return; /* don't rearm timer */ 796 } 797 ioc->ioc_coredump_loop = 0; 798 if (ioc->time_sync_interval && 799 ++ioc->timestamp_update_count >= ioc->time_sync_interval) { 800 ioc->timestamp_update_count = 0; 801 _base_sync_drv_fw_timestamp(ioc); 802 } 803 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); 804 rearm_timer: 805 if (ioc->fault_reset_work_q) 806 queue_delayed_work(ioc->fault_reset_work_q, 807 &ioc->fault_reset_work, 808 msecs_to_jiffies(FAULT_POLLING_INTERVAL)); 809 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); 810 } 811 812 /** 813 * mpt3sas_base_start_watchdog - start the fault_reset_work_q 814 * @ioc: per adapter object 815 * 816 * Context: sleep. 817 */ 818 void 819 mpt3sas_base_start_watchdog(struct MPT3SAS_ADAPTER *ioc) 820 { 821 unsigned long flags; 822 823 if (ioc->fault_reset_work_q) 824 return; 825 826 ioc->timestamp_update_count = 0; 827 /* initialize fault polling */ 828 829 INIT_DELAYED_WORK(&ioc->fault_reset_work, _base_fault_reset_work); 830 snprintf(ioc->fault_reset_work_q_name, 831 sizeof(ioc->fault_reset_work_q_name), "poll_%s%d_status", 832 ioc->driver_name, ioc->id); 833 ioc->fault_reset_work_q = 834 create_singlethread_workqueue(ioc->fault_reset_work_q_name); 835 if (!ioc->fault_reset_work_q) { 836 ioc_err(ioc, "%s: failed (line=%d)\n", __func__, __LINE__); 837 return; 838 } 839 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); 840 if (ioc->fault_reset_work_q) 841 queue_delayed_work(ioc->fault_reset_work_q, 842 &ioc->fault_reset_work, 843 msecs_to_jiffies(FAULT_POLLING_INTERVAL)); 844 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); 845 } 846 847 /** 848 * mpt3sas_base_stop_watchdog - stop the fault_reset_work_q 849 * @ioc: per adapter object 850 * 851 * Context: sleep. 852 */ 853 void 854 mpt3sas_base_stop_watchdog(struct MPT3SAS_ADAPTER *ioc) 855 { 856 unsigned long flags; 857 struct workqueue_struct *wq; 858 859 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); 860 wq = ioc->fault_reset_work_q; 861 ioc->fault_reset_work_q = NULL; 862 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); 863 if (wq) { 864 if (!cancel_delayed_work_sync(&ioc->fault_reset_work)) 865 flush_workqueue(wq); 866 destroy_workqueue(wq); 867 } 868 } 869 870 /** 871 * mpt3sas_base_fault_info - verbose translation of firmware FAULT code 872 * @ioc: per adapter object 873 * @fault_code: fault code 874 */ 875 void 876 mpt3sas_base_fault_info(struct MPT3SAS_ADAPTER *ioc , u16 fault_code) 877 { 878 ioc_err(ioc, "fault_state(0x%04x)!\n", fault_code); 879 } 880 881 /** 882 * mpt3sas_base_coredump_info - verbose translation of firmware CoreDump state 883 * @ioc: per adapter object 884 * @fault_code: fault code 885 * 886 * Return: nothing. 887 */ 888 void 889 mpt3sas_base_coredump_info(struct MPT3SAS_ADAPTER *ioc, u16 fault_code) 890 { 891 ioc_err(ioc, "coredump_state(0x%04x)!\n", fault_code); 892 } 893 894 /** 895 * mpt3sas_base_wait_for_coredump_completion - Wait until coredump 896 * completes or times out 897 * @ioc: per adapter object 898 * @caller: caller function name 899 * 900 * Return: 0 for success, non-zero for failure. 901 */ 902 int 903 mpt3sas_base_wait_for_coredump_completion(struct MPT3SAS_ADAPTER *ioc, 904 const char *caller) 905 { 906 u8 timeout = (ioc->manu_pg11.CoreDumpTOSec) ? 907 ioc->manu_pg11.CoreDumpTOSec : 908 MPT3SAS_DEFAULT_COREDUMP_TIMEOUT_SECONDS; 909 910 int ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_FAULT, 911 timeout); 912 913 if (ioc_state) 914 ioc_err(ioc, 915 "%s: CoreDump timed out. (ioc_state=0x%x)\n", 916 caller, ioc_state); 917 else 918 ioc_info(ioc, 919 "%s: CoreDump completed. (ioc_state=0x%x)\n", 920 caller, ioc_state); 921 922 return ioc_state; 923 } 924 925 /** 926 * mpt3sas_halt_firmware - halt's mpt controller firmware 927 * @ioc: per adapter object 928 * 929 * For debugging timeout related issues. Writing 0xCOFFEE00 930 * to the doorbell register will halt controller firmware. With 931 * the purpose to stop both driver and firmware, the enduser can 932 * obtain a ring buffer from controller UART. 933 */ 934 void 935 mpt3sas_halt_firmware(struct MPT3SAS_ADAPTER *ioc) 936 { 937 u32 doorbell; 938 939 if (!ioc->fwfault_debug) 940 return; 941 942 dump_stack(); 943 944 doorbell = ioc->base_readl(&ioc->chip->Doorbell); 945 if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { 946 mpt3sas_print_fault_code(ioc, doorbell & 947 MPI2_DOORBELL_DATA_MASK); 948 } else if ((doorbell & MPI2_IOC_STATE_MASK) == 949 MPI2_IOC_STATE_COREDUMP) { 950 mpt3sas_print_coredump_info(ioc, doorbell & 951 MPI2_DOORBELL_DATA_MASK); 952 } else { 953 writel(0xC0FFEE00, &ioc->chip->Doorbell); 954 ioc_err(ioc, "Firmware is halted due to command timeout\n"); 955 } 956 957 if (ioc->fwfault_debug == 2) 958 for (;;) 959 ; 960 else 961 panic("panic in %s\n", __func__); 962 } 963 964 /** 965 * _base_sas_ioc_info - verbose translation of the ioc status 966 * @ioc: per adapter object 967 * @mpi_reply: reply mf payload returned from firmware 968 * @request_hdr: request mf 969 */ 970 static void 971 _base_sas_ioc_info(struct MPT3SAS_ADAPTER *ioc, MPI2DefaultReply_t *mpi_reply, 972 MPI2RequestHeader_t *request_hdr) 973 { 974 u16 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & 975 MPI2_IOCSTATUS_MASK; 976 char *desc = NULL; 977 u16 frame_sz; 978 char *func_str = NULL; 979 980 /* SCSI_IO, RAID_PASS are handled from _scsih_scsi_ioc_info */ 981 if (request_hdr->Function == MPI2_FUNCTION_SCSI_IO_REQUEST || 982 request_hdr->Function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH || 983 request_hdr->Function == MPI2_FUNCTION_EVENT_NOTIFICATION) 984 return; 985 986 if (ioc_status == MPI2_IOCSTATUS_CONFIG_INVALID_PAGE) 987 return; 988 /* 989 * Older Firmware version doesn't support driver trigger pages. 990 * So, skip displaying 'config invalid type' type 991 * of error message. 992 */ 993 if (request_hdr->Function == MPI2_FUNCTION_CONFIG) { 994 Mpi2ConfigRequest_t *rqst = (Mpi2ConfigRequest_t *)request_hdr; 995 996 if ((rqst->ExtPageType == 997 MPI2_CONFIG_EXTPAGETYPE_DRIVER_PERSISTENT_TRIGGER) && 998 !(ioc->logging_level & MPT_DEBUG_CONFIG)) { 999 return; 1000 } 1001 } 1002 1003 switch (ioc_status) { 1004 1005 /**************************************************************************** 1006 * Common IOCStatus values for all replies 1007 ****************************************************************************/ 1008 1009 case MPI2_IOCSTATUS_INVALID_FUNCTION: 1010 desc = "invalid function"; 1011 break; 1012 case MPI2_IOCSTATUS_BUSY: 1013 desc = "busy"; 1014 break; 1015 case MPI2_IOCSTATUS_INVALID_SGL: 1016 desc = "invalid sgl"; 1017 break; 1018 case MPI2_IOCSTATUS_INTERNAL_ERROR: 1019 desc = "internal error"; 1020 break; 1021 case MPI2_IOCSTATUS_INVALID_VPID: 1022 desc = "invalid vpid"; 1023 break; 1024 case MPI2_IOCSTATUS_INSUFFICIENT_RESOURCES: 1025 desc = "insufficient resources"; 1026 break; 1027 case MPI2_IOCSTATUS_INSUFFICIENT_POWER: 1028 desc = "insufficient power"; 1029 break; 1030 case MPI2_IOCSTATUS_INVALID_FIELD: 1031 desc = "invalid field"; 1032 break; 1033 case MPI2_IOCSTATUS_INVALID_STATE: 1034 desc = "invalid state"; 1035 break; 1036 case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED: 1037 desc = "op state not supported"; 1038 break; 1039 1040 /**************************************************************************** 1041 * Config IOCStatus values 1042 ****************************************************************************/ 1043 1044 case MPI2_IOCSTATUS_CONFIG_INVALID_ACTION: 1045 desc = "config invalid action"; 1046 break; 1047 case MPI2_IOCSTATUS_CONFIG_INVALID_TYPE: 1048 desc = "config invalid type"; 1049 break; 1050 case MPI2_IOCSTATUS_CONFIG_INVALID_PAGE: 1051 desc = "config invalid page"; 1052 break; 1053 case MPI2_IOCSTATUS_CONFIG_INVALID_DATA: 1054 desc = "config invalid data"; 1055 break; 1056 case MPI2_IOCSTATUS_CONFIG_NO_DEFAULTS: 1057 desc = "config no defaults"; 1058 break; 1059 case MPI2_IOCSTATUS_CONFIG_CANT_COMMIT: 1060 desc = "config cant commit"; 1061 break; 1062 1063 /**************************************************************************** 1064 * SCSI IO Reply 1065 ****************************************************************************/ 1066 1067 case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR: 1068 case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE: 1069 case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE: 1070 case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN: 1071 case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN: 1072 case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR: 1073 case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR: 1074 case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED: 1075 case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH: 1076 case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED: 1077 case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED: 1078 case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED: 1079 break; 1080 1081 /**************************************************************************** 1082 * For use by SCSI Initiator and SCSI Target end-to-end data protection 1083 ****************************************************************************/ 1084 1085 case MPI2_IOCSTATUS_EEDP_GUARD_ERROR: 1086 desc = "eedp guard error"; 1087 break; 1088 case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR: 1089 desc = "eedp ref tag error"; 1090 break; 1091 case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR: 1092 desc = "eedp app tag error"; 1093 break; 1094 1095 /**************************************************************************** 1096 * SCSI Target values 1097 ****************************************************************************/ 1098 1099 case MPI2_IOCSTATUS_TARGET_INVALID_IO_INDEX: 1100 desc = "target invalid io index"; 1101 break; 1102 case MPI2_IOCSTATUS_TARGET_ABORTED: 1103 desc = "target aborted"; 1104 break; 1105 case MPI2_IOCSTATUS_TARGET_NO_CONN_RETRYABLE: 1106 desc = "target no conn retryable"; 1107 break; 1108 case MPI2_IOCSTATUS_TARGET_NO_CONNECTION: 1109 desc = "target no connection"; 1110 break; 1111 case MPI2_IOCSTATUS_TARGET_XFER_COUNT_MISMATCH: 1112 desc = "target xfer count mismatch"; 1113 break; 1114 case MPI2_IOCSTATUS_TARGET_DATA_OFFSET_ERROR: 1115 desc = "target data offset error"; 1116 break; 1117 case MPI2_IOCSTATUS_TARGET_TOO_MUCH_WRITE_DATA: 1118 desc = "target too much write data"; 1119 break; 1120 case MPI2_IOCSTATUS_TARGET_IU_TOO_SHORT: 1121 desc = "target iu too short"; 1122 break; 1123 case MPI2_IOCSTATUS_TARGET_ACK_NAK_TIMEOUT: 1124 desc = "target ack nak timeout"; 1125 break; 1126 case MPI2_IOCSTATUS_TARGET_NAK_RECEIVED: 1127 desc = "target nak received"; 1128 break; 1129 1130 /**************************************************************************** 1131 * Serial Attached SCSI values 1132 ****************************************************************************/ 1133 1134 case MPI2_IOCSTATUS_SAS_SMP_REQUEST_FAILED: 1135 desc = "smp request failed"; 1136 break; 1137 case MPI2_IOCSTATUS_SAS_SMP_DATA_OVERRUN: 1138 desc = "smp data overrun"; 1139 break; 1140 1141 /**************************************************************************** 1142 * Diagnostic Buffer Post / Diagnostic Release values 1143 ****************************************************************************/ 1144 1145 case MPI2_IOCSTATUS_DIAGNOSTIC_RELEASED: 1146 desc = "diagnostic released"; 1147 break; 1148 default: 1149 break; 1150 } 1151 1152 if (!desc) 1153 return; 1154 1155 switch (request_hdr->Function) { 1156 case MPI2_FUNCTION_CONFIG: 1157 frame_sz = sizeof(Mpi2ConfigRequest_t) + ioc->sge_size; 1158 func_str = "config_page"; 1159 break; 1160 case MPI2_FUNCTION_SCSI_TASK_MGMT: 1161 frame_sz = sizeof(Mpi2SCSITaskManagementRequest_t); 1162 func_str = "task_mgmt"; 1163 break; 1164 case MPI2_FUNCTION_SAS_IO_UNIT_CONTROL: 1165 frame_sz = sizeof(Mpi2SasIoUnitControlRequest_t); 1166 func_str = "sas_iounit_ctl"; 1167 break; 1168 case MPI2_FUNCTION_SCSI_ENCLOSURE_PROCESSOR: 1169 frame_sz = sizeof(Mpi2SepRequest_t); 1170 func_str = "enclosure"; 1171 break; 1172 case MPI2_FUNCTION_IOC_INIT: 1173 frame_sz = sizeof(Mpi2IOCInitRequest_t); 1174 func_str = "ioc_init"; 1175 break; 1176 case MPI2_FUNCTION_PORT_ENABLE: 1177 frame_sz = sizeof(Mpi2PortEnableRequest_t); 1178 func_str = "port_enable"; 1179 break; 1180 case MPI2_FUNCTION_SMP_PASSTHROUGH: 1181 frame_sz = sizeof(Mpi2SmpPassthroughRequest_t) + ioc->sge_size; 1182 func_str = "smp_passthru"; 1183 break; 1184 case MPI2_FUNCTION_NVME_ENCAPSULATED: 1185 frame_sz = sizeof(Mpi26NVMeEncapsulatedRequest_t) + 1186 ioc->sge_size; 1187 func_str = "nvme_encapsulated"; 1188 break; 1189 default: 1190 frame_sz = 32; 1191 func_str = "unknown"; 1192 break; 1193 } 1194 1195 ioc_warn(ioc, "ioc_status: %s(0x%04x), request(0x%p),(%s)\n", 1196 desc, ioc_status, request_hdr, func_str); 1197 1198 _debug_dump_mf(request_hdr, frame_sz/4); 1199 } 1200 1201 /** 1202 * _base_display_event_data - verbose translation of firmware asyn events 1203 * @ioc: per adapter object 1204 * @mpi_reply: reply mf payload returned from firmware 1205 */ 1206 static void 1207 _base_display_event_data(struct MPT3SAS_ADAPTER *ioc, 1208 Mpi2EventNotificationReply_t *mpi_reply) 1209 { 1210 char *desc = NULL; 1211 u16 event; 1212 1213 if (!(ioc->logging_level & MPT_DEBUG_EVENTS)) 1214 return; 1215 1216 event = le16_to_cpu(mpi_reply->Event); 1217 1218 switch (event) { 1219 case MPI2_EVENT_LOG_DATA: 1220 desc = "Log Data"; 1221 break; 1222 case MPI2_EVENT_STATE_CHANGE: 1223 desc = "Status Change"; 1224 break; 1225 case MPI2_EVENT_HARD_RESET_RECEIVED: 1226 desc = "Hard Reset Received"; 1227 break; 1228 case MPI2_EVENT_EVENT_CHANGE: 1229 desc = "Event Change"; 1230 break; 1231 case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE: 1232 desc = "Device Status Change"; 1233 break; 1234 case MPI2_EVENT_IR_OPERATION_STATUS: 1235 if (!ioc->hide_ir_msg) 1236 desc = "IR Operation Status"; 1237 break; 1238 case MPI2_EVENT_SAS_DISCOVERY: 1239 { 1240 Mpi2EventDataSasDiscovery_t *event_data = 1241 (Mpi2EventDataSasDiscovery_t *)mpi_reply->EventData; 1242 ioc_info(ioc, "Discovery: (%s)", 1243 event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED ? 1244 "start" : "stop"); 1245 if (event_data->DiscoveryStatus) 1246 pr_cont(" discovery_status(0x%08x)", 1247 le32_to_cpu(event_data->DiscoveryStatus)); 1248 pr_cont("\n"); 1249 return; 1250 } 1251 case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE: 1252 desc = "SAS Broadcast Primitive"; 1253 break; 1254 case MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE: 1255 desc = "SAS Init Device Status Change"; 1256 break; 1257 case MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW: 1258 desc = "SAS Init Table Overflow"; 1259 break; 1260 case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST: 1261 desc = "SAS Topology Change List"; 1262 break; 1263 case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE: 1264 desc = "SAS Enclosure Device Status Change"; 1265 break; 1266 case MPI2_EVENT_IR_VOLUME: 1267 if (!ioc->hide_ir_msg) 1268 desc = "IR Volume"; 1269 break; 1270 case MPI2_EVENT_IR_PHYSICAL_DISK: 1271 if (!ioc->hide_ir_msg) 1272 desc = "IR Physical Disk"; 1273 break; 1274 case MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST: 1275 if (!ioc->hide_ir_msg) 1276 desc = "IR Configuration Change List"; 1277 break; 1278 case MPI2_EVENT_LOG_ENTRY_ADDED: 1279 if (!ioc->hide_ir_msg) 1280 desc = "Log Entry Added"; 1281 break; 1282 case MPI2_EVENT_TEMP_THRESHOLD: 1283 desc = "Temperature Threshold"; 1284 break; 1285 case MPI2_EVENT_ACTIVE_CABLE_EXCEPTION: 1286 desc = "Cable Event"; 1287 break; 1288 case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR: 1289 desc = "SAS Device Discovery Error"; 1290 break; 1291 case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE: 1292 desc = "PCIE Device Status Change"; 1293 break; 1294 case MPI2_EVENT_PCIE_ENUMERATION: 1295 { 1296 Mpi26EventDataPCIeEnumeration_t *event_data = 1297 (Mpi26EventDataPCIeEnumeration_t *)mpi_reply->EventData; 1298 ioc_info(ioc, "PCIE Enumeration: (%s)", 1299 event_data->ReasonCode == MPI26_EVENT_PCIE_ENUM_RC_STARTED ? 1300 "start" : "stop"); 1301 if (event_data->EnumerationStatus) 1302 pr_cont("enumeration_status(0x%08x)", 1303 le32_to_cpu(event_data->EnumerationStatus)); 1304 pr_cont("\n"); 1305 return; 1306 } 1307 case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST: 1308 desc = "PCIE Topology Change List"; 1309 break; 1310 } 1311 1312 if (!desc) 1313 return; 1314 1315 ioc_info(ioc, "%s\n", desc); 1316 } 1317 1318 /** 1319 * _base_sas_log_info - verbose translation of firmware log info 1320 * @ioc: per adapter object 1321 * @log_info: log info 1322 */ 1323 static void 1324 _base_sas_log_info(struct MPT3SAS_ADAPTER *ioc , u32 log_info) 1325 { 1326 union loginfo_type { 1327 u32 loginfo; 1328 struct { 1329 u32 subcode:16; 1330 u32 code:8; 1331 u32 originator:4; 1332 u32 bus_type:4; 1333 } dw; 1334 }; 1335 union loginfo_type sas_loginfo; 1336 char *originator_str = NULL; 1337 1338 sas_loginfo.loginfo = log_info; 1339 if (sas_loginfo.dw.bus_type != 3 /*SAS*/) 1340 return; 1341 1342 /* each nexus loss loginfo */ 1343 if (log_info == 0x31170000) 1344 return; 1345 1346 /* eat the loginfos associated with task aborts */ 1347 if (ioc->ignore_loginfos && (log_info == 0x30050000 || log_info == 1348 0x31140000 || log_info == 0x31130000)) 1349 return; 1350 1351 switch (sas_loginfo.dw.originator) { 1352 case 0: 1353 originator_str = "IOP"; 1354 break; 1355 case 1: 1356 originator_str = "PL"; 1357 break; 1358 case 2: 1359 if (!ioc->hide_ir_msg) 1360 originator_str = "IR"; 1361 else 1362 originator_str = "WarpDrive"; 1363 break; 1364 } 1365 1366 ioc_warn(ioc, "log_info(0x%08x): originator(%s), code(0x%02x), sub_code(0x%04x)\n", 1367 log_info, 1368 originator_str, sas_loginfo.dw.code, sas_loginfo.dw.subcode); 1369 } 1370 1371 /** 1372 * _base_display_reply_info - handle reply descriptors depending on IOC Status 1373 * @ioc: per adapter object 1374 * @smid: system request message index 1375 * @msix_index: MSIX table index supplied by the OS 1376 * @reply: reply message frame (lower 32bit addr) 1377 */ 1378 static void 1379 _base_display_reply_info(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, 1380 u32 reply) 1381 { 1382 MPI2DefaultReply_t *mpi_reply; 1383 u16 ioc_status; 1384 u32 loginfo = 0; 1385 1386 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply); 1387 if (unlikely(!mpi_reply)) { 1388 ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n", 1389 __FILE__, __LINE__, __func__); 1390 return; 1391 } 1392 ioc_status = le16_to_cpu(mpi_reply->IOCStatus); 1393 1394 if ((ioc_status & MPI2_IOCSTATUS_MASK) && 1395 (ioc->logging_level & MPT_DEBUG_REPLY)) { 1396 _base_sas_ioc_info(ioc , mpi_reply, 1397 mpt3sas_base_get_msg_frame(ioc, smid)); 1398 } 1399 1400 if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) { 1401 loginfo = le32_to_cpu(mpi_reply->IOCLogInfo); 1402 _base_sas_log_info(ioc, loginfo); 1403 } 1404 1405 if (ioc_status || loginfo) { 1406 ioc_status &= MPI2_IOCSTATUS_MASK; 1407 mpt3sas_trigger_mpi(ioc, ioc_status, loginfo); 1408 } 1409 } 1410 1411 /** 1412 * mpt3sas_base_done - base internal command completion routine 1413 * @ioc: per adapter object 1414 * @smid: system request message index 1415 * @msix_index: MSIX table index supplied by the OS 1416 * @reply: reply message frame(lower 32bit addr) 1417 * 1418 * Return: 1419 * 1 meaning mf should be freed from _base_interrupt 1420 * 0 means the mf is freed from this function. 1421 */ 1422 u8 1423 mpt3sas_base_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, 1424 u32 reply) 1425 { 1426 MPI2DefaultReply_t *mpi_reply; 1427 1428 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply); 1429 if (mpi_reply && mpi_reply->Function == MPI2_FUNCTION_EVENT_ACK) 1430 return mpt3sas_check_for_pending_internal_cmds(ioc, smid); 1431 1432 if (ioc->base_cmds.status == MPT3_CMD_NOT_USED) 1433 return 1; 1434 1435 ioc->base_cmds.status |= MPT3_CMD_COMPLETE; 1436 if (mpi_reply) { 1437 ioc->base_cmds.status |= MPT3_CMD_REPLY_VALID; 1438 memcpy(ioc->base_cmds.reply, mpi_reply, mpi_reply->MsgLength*4); 1439 } 1440 ioc->base_cmds.status &= ~MPT3_CMD_PENDING; 1441 1442 complete(&ioc->base_cmds.done); 1443 return 1; 1444 } 1445 1446 /** 1447 * _base_async_event - main callback handler for firmware asyn events 1448 * @ioc: per adapter object 1449 * @msix_index: MSIX table index supplied by the OS 1450 * @reply: reply message frame(lower 32bit addr) 1451 * 1452 * Return: 1453 * 1 meaning mf should be freed from _base_interrupt 1454 * 0 means the mf is freed from this function. 1455 */ 1456 static u8 1457 _base_async_event(struct MPT3SAS_ADAPTER *ioc, u8 msix_index, u32 reply) 1458 { 1459 Mpi2EventNotificationReply_t *mpi_reply; 1460 Mpi2EventAckRequest_t *ack_request; 1461 u16 smid; 1462 struct _event_ack_list *delayed_event_ack; 1463 1464 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply); 1465 if (!mpi_reply) 1466 return 1; 1467 if (mpi_reply->Function != MPI2_FUNCTION_EVENT_NOTIFICATION) 1468 return 1; 1469 1470 _base_display_event_data(ioc, mpi_reply); 1471 1472 if (!(mpi_reply->AckRequired & MPI2_EVENT_NOTIFICATION_ACK_REQUIRED)) 1473 goto out; 1474 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx); 1475 if (!smid) { 1476 delayed_event_ack = kzalloc(sizeof(*delayed_event_ack), 1477 GFP_ATOMIC); 1478 if (!delayed_event_ack) 1479 goto out; 1480 INIT_LIST_HEAD(&delayed_event_ack->list); 1481 delayed_event_ack->Event = mpi_reply->Event; 1482 delayed_event_ack->EventContext = mpi_reply->EventContext; 1483 list_add_tail(&delayed_event_ack->list, 1484 &ioc->delayed_event_ack_list); 1485 dewtprintk(ioc, 1486 ioc_info(ioc, "DELAYED: EVENT ACK: event (0x%04x)\n", 1487 le16_to_cpu(mpi_reply->Event))); 1488 goto out; 1489 } 1490 1491 ack_request = mpt3sas_base_get_msg_frame(ioc, smid); 1492 memset(ack_request, 0, sizeof(Mpi2EventAckRequest_t)); 1493 ack_request->Function = MPI2_FUNCTION_EVENT_ACK; 1494 ack_request->Event = mpi_reply->Event; 1495 ack_request->EventContext = mpi_reply->EventContext; 1496 ack_request->VF_ID = 0; /* TODO */ 1497 ack_request->VP_ID = 0; 1498 ioc->put_smid_default(ioc, smid); 1499 1500 out: 1501 1502 /* scsih callback handler */ 1503 mpt3sas_scsih_event_callback(ioc, msix_index, reply); 1504 1505 /* ctl callback handler */ 1506 mpt3sas_ctl_event_callback(ioc, msix_index, reply); 1507 1508 return 1; 1509 } 1510 1511 static struct scsiio_tracker * 1512 _get_st_from_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid) 1513 { 1514 struct scsi_cmnd *cmd; 1515 1516 if (WARN_ON(!smid) || 1517 WARN_ON(smid >= ioc->hi_priority_smid)) 1518 return NULL; 1519 1520 cmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid); 1521 if (cmd) 1522 return scsi_cmd_priv(cmd); 1523 1524 return NULL; 1525 } 1526 1527 /** 1528 * _base_get_cb_idx - obtain the callback index 1529 * @ioc: per adapter object 1530 * @smid: system request message index 1531 * 1532 * Return: callback index. 1533 */ 1534 static u8 1535 _base_get_cb_idx(struct MPT3SAS_ADAPTER *ioc, u16 smid) 1536 { 1537 int i; 1538 u16 ctl_smid = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT + 1; 1539 u8 cb_idx = 0xFF; 1540 1541 if (smid < ioc->hi_priority_smid) { 1542 struct scsiio_tracker *st; 1543 1544 if (smid < ctl_smid) { 1545 st = _get_st_from_smid(ioc, smid); 1546 if (st) 1547 cb_idx = st->cb_idx; 1548 } else if (smid == ctl_smid) 1549 cb_idx = ioc->ctl_cb_idx; 1550 } else if (smid < ioc->internal_smid) { 1551 i = smid - ioc->hi_priority_smid; 1552 cb_idx = ioc->hpr_lookup[i].cb_idx; 1553 } else if (smid <= ioc->hba_queue_depth) { 1554 i = smid - ioc->internal_smid; 1555 cb_idx = ioc->internal_lookup[i].cb_idx; 1556 } 1557 return cb_idx; 1558 } 1559 1560 /** 1561 * mpt3sas_base_pause_mq_polling - pause polling on the mq poll queues 1562 * when driver is flushing out the IOs. 1563 * @ioc: per adapter object 1564 * 1565 * Pause polling on the mq poll (io uring) queues when driver is flushing 1566 * out the IOs. Otherwise we may see the race condition of completing the same 1567 * IO from two paths. 1568 * 1569 * Returns nothing. 1570 */ 1571 void 1572 mpt3sas_base_pause_mq_polling(struct MPT3SAS_ADAPTER *ioc) 1573 { 1574 int iopoll_q_count = 1575 ioc->reply_queue_count - ioc->iopoll_q_start_index; 1576 int qid; 1577 1578 for (qid = 0; qid < iopoll_q_count; qid++) 1579 atomic_set(&ioc->io_uring_poll_queues[qid].pause, 1); 1580 1581 /* 1582 * wait for current poll to complete. 1583 */ 1584 for (qid = 0; qid < iopoll_q_count; qid++) { 1585 while (atomic_read(&ioc->io_uring_poll_queues[qid].busy)) { 1586 cpu_relax(); 1587 udelay(500); 1588 } 1589 } 1590 } 1591 1592 /** 1593 * mpt3sas_base_resume_mq_polling - Resume polling on mq poll queues. 1594 * @ioc: per adapter object 1595 * 1596 * Returns nothing. 1597 */ 1598 void 1599 mpt3sas_base_resume_mq_polling(struct MPT3SAS_ADAPTER *ioc) 1600 { 1601 int iopoll_q_count = 1602 ioc->reply_queue_count - ioc->iopoll_q_start_index; 1603 int qid; 1604 1605 for (qid = 0; qid < iopoll_q_count; qid++) 1606 atomic_set(&ioc->io_uring_poll_queues[qid].pause, 0); 1607 } 1608 1609 /** 1610 * mpt3sas_base_mask_interrupts - disable interrupts 1611 * @ioc: per adapter object 1612 * 1613 * Disabling ResetIRQ, Reply and Doorbell Interrupts 1614 */ 1615 void 1616 mpt3sas_base_mask_interrupts(struct MPT3SAS_ADAPTER *ioc) 1617 { 1618 u32 him_register; 1619 1620 ioc->mask_interrupts = 1; 1621 him_register = ioc->base_readl(&ioc->chip->HostInterruptMask); 1622 him_register |= MPI2_HIM_DIM + MPI2_HIM_RIM + MPI2_HIM_RESET_IRQ_MASK; 1623 writel(him_register, &ioc->chip->HostInterruptMask); 1624 ioc->base_readl(&ioc->chip->HostInterruptMask); 1625 } 1626 1627 /** 1628 * mpt3sas_base_unmask_interrupts - enable interrupts 1629 * @ioc: per adapter object 1630 * 1631 * Enabling only Reply Interrupts 1632 */ 1633 void 1634 mpt3sas_base_unmask_interrupts(struct MPT3SAS_ADAPTER *ioc) 1635 { 1636 u32 him_register; 1637 1638 him_register = ioc->base_readl(&ioc->chip->HostInterruptMask); 1639 him_register &= ~MPI2_HIM_RIM; 1640 writel(him_register, &ioc->chip->HostInterruptMask); 1641 ioc->mask_interrupts = 0; 1642 } 1643 1644 union reply_descriptor { 1645 u64 word; 1646 struct { 1647 u32 low; 1648 u32 high; 1649 } u; 1650 }; 1651 1652 static u32 base_mod64(u64 dividend, u32 divisor) 1653 { 1654 u32 remainder; 1655 1656 if (!divisor) 1657 pr_err("mpt3sas: DIVISOR is zero, in div fn\n"); 1658 remainder = do_div(dividend, divisor); 1659 return remainder; 1660 } 1661 1662 /** 1663 * _base_process_reply_queue - Process reply descriptors from reply 1664 * descriptor post queue. 1665 * @reply_q: per IRQ's reply queue object. 1666 * 1667 * Return: number of reply descriptors processed from reply 1668 * descriptor queue. 1669 */ 1670 static int 1671 _base_process_reply_queue(struct adapter_reply_queue *reply_q) 1672 { 1673 union reply_descriptor rd; 1674 u64 completed_cmds; 1675 u8 request_descript_type; 1676 u16 smid; 1677 u8 cb_idx; 1678 u32 reply; 1679 u8 msix_index = reply_q->msix_index; 1680 struct MPT3SAS_ADAPTER *ioc = reply_q->ioc; 1681 Mpi2ReplyDescriptorsUnion_t *rpf; 1682 u8 rc; 1683 1684 completed_cmds = 0; 1685 if (!atomic_add_unless(&reply_q->busy, 1, 1)) 1686 return completed_cmds; 1687 1688 rpf = &reply_q->reply_post_free[reply_q->reply_post_host_index]; 1689 request_descript_type = rpf->Default.ReplyFlags 1690 & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK; 1691 if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) { 1692 atomic_dec(&reply_q->busy); 1693 return completed_cmds; 1694 } 1695 1696 cb_idx = 0xFF; 1697 do { 1698 rd.word = le64_to_cpu(rpf->Words); 1699 if (rd.u.low == UINT_MAX || rd.u.high == UINT_MAX) 1700 goto out; 1701 reply = 0; 1702 smid = le16_to_cpu(rpf->Default.DescriptorTypeDependent1); 1703 if (request_descript_type == 1704 MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS || 1705 request_descript_type == 1706 MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS || 1707 request_descript_type == 1708 MPI26_RPY_DESCRIPT_FLAGS_PCIE_ENCAPSULATED_SUCCESS) { 1709 cb_idx = _base_get_cb_idx(ioc, smid); 1710 if ((likely(cb_idx < MPT_MAX_CALLBACKS)) && 1711 (likely(mpt_callbacks[cb_idx] != NULL))) { 1712 rc = mpt_callbacks[cb_idx](ioc, smid, 1713 msix_index, 0); 1714 if (rc) 1715 mpt3sas_base_free_smid(ioc, smid); 1716 } 1717 } else if (request_descript_type == 1718 MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY) { 1719 reply = le32_to_cpu( 1720 rpf->AddressReply.ReplyFrameAddress); 1721 if (reply > ioc->reply_dma_max_address || 1722 reply < ioc->reply_dma_min_address) 1723 reply = 0; 1724 if (smid) { 1725 cb_idx = _base_get_cb_idx(ioc, smid); 1726 if ((likely(cb_idx < MPT_MAX_CALLBACKS)) && 1727 (likely(mpt_callbacks[cb_idx] != NULL))) { 1728 rc = mpt_callbacks[cb_idx](ioc, smid, 1729 msix_index, reply); 1730 if (reply) 1731 _base_display_reply_info(ioc, 1732 smid, msix_index, reply); 1733 if (rc) 1734 mpt3sas_base_free_smid(ioc, 1735 smid); 1736 } 1737 } else { 1738 _base_async_event(ioc, msix_index, reply); 1739 } 1740 1741 /* reply free queue handling */ 1742 if (reply) { 1743 ioc->reply_free_host_index = 1744 (ioc->reply_free_host_index == 1745 (ioc->reply_free_queue_depth - 1)) ? 1746 0 : ioc->reply_free_host_index + 1; 1747 ioc->reply_free[ioc->reply_free_host_index] = 1748 cpu_to_le32(reply); 1749 if (ioc->is_mcpu_endpoint) 1750 _base_clone_reply_to_sys_mem(ioc, 1751 reply, 1752 ioc->reply_free_host_index); 1753 writel(ioc->reply_free_host_index, 1754 &ioc->chip->ReplyFreeHostIndex); 1755 } 1756 } 1757 1758 rpf->Words = cpu_to_le64(ULLONG_MAX); 1759 reply_q->reply_post_host_index = 1760 (reply_q->reply_post_host_index == 1761 (ioc->reply_post_queue_depth - 1)) ? 0 : 1762 reply_q->reply_post_host_index + 1; 1763 request_descript_type = 1764 reply_q->reply_post_free[reply_q->reply_post_host_index]. 1765 Default.ReplyFlags & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK; 1766 completed_cmds++; 1767 /* Update the reply post host index after continuously 1768 * processing the threshold number of Reply Descriptors. 1769 * So that FW can find enough entries to post the Reply 1770 * Descriptors in the reply descriptor post queue. 1771 */ 1772 if (completed_cmds >= ioc->thresh_hold) { 1773 if (ioc->combined_reply_queue) { 1774 writel(reply_q->reply_post_host_index | 1775 ((msix_index & 7) << 1776 MPI2_RPHI_MSIX_INDEX_SHIFT), 1777 ioc->replyPostRegisterIndex[msix_index/8]); 1778 } else { 1779 writel(reply_q->reply_post_host_index | 1780 (msix_index << 1781 MPI2_RPHI_MSIX_INDEX_SHIFT), 1782 &ioc->chip->ReplyPostHostIndex); 1783 } 1784 if (!reply_q->is_iouring_poll_q && 1785 !reply_q->irq_poll_scheduled) { 1786 reply_q->irq_poll_scheduled = true; 1787 irq_poll_sched(&reply_q->irqpoll); 1788 } 1789 atomic_dec(&reply_q->busy); 1790 return completed_cmds; 1791 } 1792 if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) 1793 goto out; 1794 if (!reply_q->reply_post_host_index) 1795 rpf = reply_q->reply_post_free; 1796 else 1797 rpf++; 1798 } while (1); 1799 1800 out: 1801 1802 if (!completed_cmds) { 1803 atomic_dec(&reply_q->busy); 1804 return completed_cmds; 1805 } 1806 1807 if (ioc->is_warpdrive) { 1808 writel(reply_q->reply_post_host_index, 1809 ioc->reply_post_host_index[msix_index]); 1810 atomic_dec(&reply_q->busy); 1811 return completed_cmds; 1812 } 1813 1814 /* Update Reply Post Host Index. 1815 * For those HBA's which support combined reply queue feature 1816 * 1. Get the correct Supplemental Reply Post Host Index Register. 1817 * i.e. (msix_index / 8)th entry from Supplemental Reply Post Host 1818 * Index Register address bank i.e replyPostRegisterIndex[], 1819 * 2. Then update this register with new reply host index value 1820 * in ReplyPostIndex field and the MSIxIndex field with 1821 * msix_index value reduced to a value between 0 and 7, 1822 * using a modulo 8 operation. Since each Supplemental Reply Post 1823 * Host Index Register supports 8 MSI-X vectors. 1824 * 1825 * For other HBA's just update the Reply Post Host Index register with 1826 * new reply host index value in ReplyPostIndex Field and msix_index 1827 * value in MSIxIndex field. 1828 */ 1829 if (ioc->combined_reply_queue) 1830 writel(reply_q->reply_post_host_index | ((msix_index & 7) << 1831 MPI2_RPHI_MSIX_INDEX_SHIFT), 1832 ioc->replyPostRegisterIndex[msix_index/8]); 1833 else 1834 writel(reply_q->reply_post_host_index | (msix_index << 1835 MPI2_RPHI_MSIX_INDEX_SHIFT), 1836 &ioc->chip->ReplyPostHostIndex); 1837 atomic_dec(&reply_q->busy); 1838 return completed_cmds; 1839 } 1840 1841 /** 1842 * mpt3sas_blk_mq_poll - poll the blk mq poll queue 1843 * @shost: Scsi_Host object 1844 * @queue_num: hw ctx queue number 1845 * 1846 * Return number of entries that has been processed from poll queue. 1847 */ 1848 int mpt3sas_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num) 1849 { 1850 struct MPT3SAS_ADAPTER *ioc = 1851 (struct MPT3SAS_ADAPTER *)shost->hostdata; 1852 struct adapter_reply_queue *reply_q; 1853 int num_entries = 0; 1854 int qid = queue_num - ioc->iopoll_q_start_index; 1855 1856 if (atomic_read(&ioc->io_uring_poll_queues[qid].pause) || 1857 !atomic_add_unless(&ioc->io_uring_poll_queues[qid].busy, 1, 1)) 1858 return 0; 1859 1860 reply_q = ioc->io_uring_poll_queues[qid].reply_q; 1861 1862 num_entries = _base_process_reply_queue(reply_q); 1863 atomic_dec(&ioc->io_uring_poll_queues[qid].busy); 1864 1865 return num_entries; 1866 } 1867 1868 /** 1869 * _base_interrupt - MPT adapter (IOC) specific interrupt handler. 1870 * @irq: irq number (not used) 1871 * @bus_id: bus identifier cookie == pointer to MPT_ADAPTER structure 1872 * 1873 * Return: IRQ_HANDLED if processed, else IRQ_NONE. 1874 */ 1875 static irqreturn_t 1876 _base_interrupt(int irq, void *bus_id) 1877 { 1878 struct adapter_reply_queue *reply_q = bus_id; 1879 struct MPT3SAS_ADAPTER *ioc = reply_q->ioc; 1880 1881 if (ioc->mask_interrupts) 1882 return IRQ_NONE; 1883 if (reply_q->irq_poll_scheduled) 1884 return IRQ_HANDLED; 1885 return ((_base_process_reply_queue(reply_q) > 0) ? 1886 IRQ_HANDLED : IRQ_NONE); 1887 } 1888 1889 /** 1890 * _base_irqpoll - IRQ poll callback handler 1891 * @irqpoll: irq_poll object 1892 * @budget: irq poll weight 1893 * 1894 * Return: number of reply descriptors processed 1895 */ 1896 static int 1897 _base_irqpoll(struct irq_poll *irqpoll, int budget) 1898 { 1899 struct adapter_reply_queue *reply_q; 1900 int num_entries = 0; 1901 1902 reply_q = container_of(irqpoll, struct adapter_reply_queue, 1903 irqpoll); 1904 if (reply_q->irq_line_enable) { 1905 disable_irq_nosync(reply_q->os_irq); 1906 reply_q->irq_line_enable = false; 1907 } 1908 num_entries = _base_process_reply_queue(reply_q); 1909 if (num_entries < budget) { 1910 irq_poll_complete(irqpoll); 1911 reply_q->irq_poll_scheduled = false; 1912 reply_q->irq_line_enable = true; 1913 enable_irq(reply_q->os_irq); 1914 /* 1915 * Go for one more round of processing the 1916 * reply descriptor post queue in case the HBA 1917 * Firmware has posted some reply descriptors 1918 * while reenabling the IRQ. 1919 */ 1920 _base_process_reply_queue(reply_q); 1921 } 1922 1923 return num_entries; 1924 } 1925 1926 /** 1927 * _base_init_irqpolls - initliaze IRQ polls 1928 * @ioc: per adapter object 1929 * 1930 * Return: nothing 1931 */ 1932 static void 1933 _base_init_irqpolls(struct MPT3SAS_ADAPTER *ioc) 1934 { 1935 struct adapter_reply_queue *reply_q, *next; 1936 1937 if (list_empty(&ioc->reply_queue_list)) 1938 return; 1939 1940 list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) { 1941 if (reply_q->is_iouring_poll_q) 1942 continue; 1943 irq_poll_init(&reply_q->irqpoll, 1944 ioc->hba_queue_depth/4, _base_irqpoll); 1945 reply_q->irq_poll_scheduled = false; 1946 reply_q->irq_line_enable = true; 1947 reply_q->os_irq = pci_irq_vector(ioc->pdev, 1948 reply_q->msix_index); 1949 } 1950 } 1951 1952 /** 1953 * _base_is_controller_msix_enabled - is controller support muli-reply queues 1954 * @ioc: per adapter object 1955 * 1956 * Return: Whether or not MSI/X is enabled. 1957 */ 1958 static inline int 1959 _base_is_controller_msix_enabled(struct MPT3SAS_ADAPTER *ioc) 1960 { 1961 return (ioc->facts.IOCCapabilities & 1962 MPI2_IOCFACTS_CAPABILITY_MSI_X_INDEX) && ioc->msix_enable; 1963 } 1964 1965 /** 1966 * mpt3sas_base_sync_reply_irqs - flush pending MSIX interrupts 1967 * @ioc: per adapter object 1968 * @poll: poll over reply descriptor pools incase interrupt for 1969 * timed-out SCSI command got delayed 1970 * Context: non-ISR context 1971 * 1972 * Called when a Task Management request has completed. 1973 */ 1974 void 1975 mpt3sas_base_sync_reply_irqs(struct MPT3SAS_ADAPTER *ioc, u8 poll) 1976 { 1977 struct adapter_reply_queue *reply_q; 1978 1979 /* If MSIX capability is turned off 1980 * then multi-queues are not enabled 1981 */ 1982 if (!_base_is_controller_msix_enabled(ioc)) 1983 return; 1984 1985 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { 1986 if (ioc->shost_recovery || ioc->remove_host || 1987 ioc->pci_error_recovery) 1988 return; 1989 /* TMs are on msix_index == 0 */ 1990 if (reply_q->msix_index == 0) 1991 continue; 1992 1993 if (reply_q->is_iouring_poll_q) { 1994 _base_process_reply_queue(reply_q); 1995 continue; 1996 } 1997 1998 synchronize_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index)); 1999 if (reply_q->irq_poll_scheduled) { 2000 /* Calling irq_poll_disable will wait for any pending 2001 * callbacks to have completed. 2002 */ 2003 irq_poll_disable(&reply_q->irqpoll); 2004 irq_poll_enable(&reply_q->irqpoll); 2005 /* check how the scheduled poll has ended, 2006 * clean up only if necessary 2007 */ 2008 if (reply_q->irq_poll_scheduled) { 2009 reply_q->irq_poll_scheduled = false; 2010 reply_q->irq_line_enable = true; 2011 enable_irq(reply_q->os_irq); 2012 } 2013 } 2014 } 2015 if (poll) 2016 _base_process_reply_queue(reply_q); 2017 } 2018 2019 /** 2020 * mpt3sas_base_release_callback_handler - clear interrupt callback handler 2021 * @cb_idx: callback index 2022 */ 2023 void 2024 mpt3sas_base_release_callback_handler(u8 cb_idx) 2025 { 2026 mpt_callbacks[cb_idx] = NULL; 2027 } 2028 2029 /** 2030 * mpt3sas_base_register_callback_handler - obtain index for the interrupt callback handler 2031 * @cb_func: callback function 2032 * 2033 * Return: Index of @cb_func. 2034 */ 2035 u8 2036 mpt3sas_base_register_callback_handler(MPT_CALLBACK cb_func) 2037 { 2038 u8 cb_idx; 2039 2040 for (cb_idx = MPT_MAX_CALLBACKS-1; cb_idx; cb_idx--) 2041 if (mpt_callbacks[cb_idx] == NULL) 2042 break; 2043 2044 mpt_callbacks[cb_idx] = cb_func; 2045 return cb_idx; 2046 } 2047 2048 /** 2049 * mpt3sas_base_initialize_callback_handler - initialize the interrupt callback handler 2050 */ 2051 void 2052 mpt3sas_base_initialize_callback_handler(void) 2053 { 2054 u8 cb_idx; 2055 2056 for (cb_idx = 0; cb_idx < MPT_MAX_CALLBACKS; cb_idx++) 2057 mpt3sas_base_release_callback_handler(cb_idx); 2058 } 2059 2060 2061 /** 2062 * _base_build_zero_len_sge - build zero length sg entry 2063 * @ioc: per adapter object 2064 * @paddr: virtual address for SGE 2065 * 2066 * Create a zero length scatter gather entry to insure the IOCs hardware has 2067 * something to use if the target device goes brain dead and tries 2068 * to send data even when none is asked for. 2069 */ 2070 static void 2071 _base_build_zero_len_sge(struct MPT3SAS_ADAPTER *ioc, void *paddr) 2072 { 2073 u32 flags_length = (u32)((MPI2_SGE_FLAGS_LAST_ELEMENT | 2074 MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST | 2075 MPI2_SGE_FLAGS_SIMPLE_ELEMENT) << 2076 MPI2_SGE_FLAGS_SHIFT); 2077 ioc->base_add_sg_single(paddr, flags_length, -1); 2078 } 2079 2080 /** 2081 * _base_add_sg_single_32 - Place a simple 32 bit SGE at address pAddr. 2082 * @paddr: virtual address for SGE 2083 * @flags_length: SGE flags and data transfer length 2084 * @dma_addr: Physical address 2085 */ 2086 static void 2087 _base_add_sg_single_32(void *paddr, u32 flags_length, dma_addr_t dma_addr) 2088 { 2089 Mpi2SGESimple32_t *sgel = paddr; 2090 2091 flags_length |= (MPI2_SGE_FLAGS_32_BIT_ADDRESSING | 2092 MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT; 2093 sgel->FlagsLength = cpu_to_le32(flags_length); 2094 sgel->Address = cpu_to_le32(dma_addr); 2095 } 2096 2097 2098 /** 2099 * _base_add_sg_single_64 - Place a simple 64 bit SGE at address pAddr. 2100 * @paddr: virtual address for SGE 2101 * @flags_length: SGE flags and data transfer length 2102 * @dma_addr: Physical address 2103 */ 2104 static void 2105 _base_add_sg_single_64(void *paddr, u32 flags_length, dma_addr_t dma_addr) 2106 { 2107 Mpi2SGESimple64_t *sgel = paddr; 2108 2109 flags_length |= (MPI2_SGE_FLAGS_64_BIT_ADDRESSING | 2110 MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT; 2111 sgel->FlagsLength = cpu_to_le32(flags_length); 2112 sgel->Address = cpu_to_le64(dma_addr); 2113 } 2114 2115 /** 2116 * _base_get_chain_buffer_tracker - obtain chain tracker 2117 * @ioc: per adapter object 2118 * @scmd: SCSI commands of the IO request 2119 * 2120 * Return: chain tracker from chain_lookup table using key as 2121 * smid and smid's chain_offset. 2122 */ 2123 static struct chain_tracker * 2124 _base_get_chain_buffer_tracker(struct MPT3SAS_ADAPTER *ioc, 2125 struct scsi_cmnd *scmd) 2126 { 2127 struct chain_tracker *chain_req; 2128 struct scsiio_tracker *st = scsi_cmd_priv(scmd); 2129 u16 smid = st->smid; 2130 u8 chain_offset = 2131 atomic_read(&ioc->chain_lookup[smid - 1].chain_offset); 2132 2133 if (chain_offset == ioc->chains_needed_per_io) 2134 return NULL; 2135 2136 chain_req = &ioc->chain_lookup[smid - 1].chains_per_smid[chain_offset]; 2137 atomic_inc(&ioc->chain_lookup[smid - 1].chain_offset); 2138 return chain_req; 2139 } 2140 2141 2142 /** 2143 * _base_build_sg - build generic sg 2144 * @ioc: per adapter object 2145 * @psge: virtual address for SGE 2146 * @data_out_dma: physical address for WRITES 2147 * @data_out_sz: data xfer size for WRITES 2148 * @data_in_dma: physical address for READS 2149 * @data_in_sz: data xfer size for READS 2150 */ 2151 static void 2152 _base_build_sg(struct MPT3SAS_ADAPTER *ioc, void *psge, 2153 dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma, 2154 size_t data_in_sz) 2155 { 2156 u32 sgl_flags; 2157 2158 if (!data_out_sz && !data_in_sz) { 2159 _base_build_zero_len_sge(ioc, psge); 2160 return; 2161 } 2162 2163 if (data_out_sz && data_in_sz) { 2164 /* WRITE sgel first */ 2165 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT | 2166 MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_HOST_TO_IOC); 2167 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT; 2168 ioc->base_add_sg_single(psge, sgl_flags | 2169 data_out_sz, data_out_dma); 2170 2171 /* incr sgel */ 2172 psge += ioc->sge_size; 2173 2174 /* READ sgel last */ 2175 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT | 2176 MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER | 2177 MPI2_SGE_FLAGS_END_OF_LIST); 2178 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT; 2179 ioc->base_add_sg_single(psge, sgl_flags | 2180 data_in_sz, data_in_dma); 2181 } else if (data_out_sz) /* WRITE */ { 2182 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT | 2183 MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER | 2184 MPI2_SGE_FLAGS_END_OF_LIST | MPI2_SGE_FLAGS_HOST_TO_IOC); 2185 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT; 2186 ioc->base_add_sg_single(psge, sgl_flags | 2187 data_out_sz, data_out_dma); 2188 } else if (data_in_sz) /* READ */ { 2189 sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT | 2190 MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER | 2191 MPI2_SGE_FLAGS_END_OF_LIST); 2192 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT; 2193 ioc->base_add_sg_single(psge, sgl_flags | 2194 data_in_sz, data_in_dma); 2195 } 2196 } 2197 2198 /* IEEE format sgls */ 2199 2200 /** 2201 * _base_build_nvme_prp - This function is called for NVMe end devices to build 2202 * a native SGL (NVMe PRP). 2203 * @ioc: per adapter object 2204 * @smid: system request message index for getting asscociated SGL 2205 * @nvme_encap_request: the NVMe request msg frame pointer 2206 * @data_out_dma: physical address for WRITES 2207 * @data_out_sz: data xfer size for WRITES 2208 * @data_in_dma: physical address for READS 2209 * @data_in_sz: data xfer size for READS 2210 * 2211 * The native SGL is built starting in the first PRP 2212 * entry of the NVMe message (PRP1). If the data buffer is small enough to be 2213 * described entirely using PRP1, then PRP2 is not used. If needed, PRP2 is 2214 * used to describe a larger data buffer. If the data buffer is too large to 2215 * describe using the two PRP entriess inside the NVMe message, then PRP1 2216 * describes the first data memory segment, and PRP2 contains a pointer to a PRP 2217 * list located elsewhere in memory to describe the remaining data memory 2218 * segments. The PRP list will be contiguous. 2219 * 2220 * The native SGL for NVMe devices is a Physical Region Page (PRP). A PRP 2221 * consists of a list of PRP entries to describe a number of noncontigous 2222 * physical memory segments as a single memory buffer, just as a SGL does. Note 2223 * however, that this function is only used by the IOCTL call, so the memory 2224 * given will be guaranteed to be contiguous. There is no need to translate 2225 * non-contiguous SGL into a PRP in this case. All PRPs will describe 2226 * contiguous space that is one page size each. 2227 * 2228 * Each NVMe message contains two PRP entries. The first (PRP1) either contains 2229 * a PRP list pointer or a PRP element, depending upon the command. PRP2 2230 * contains the second PRP element if the memory being described fits within 2 2231 * PRP entries, or a PRP list pointer if the PRP spans more than two entries. 2232 * 2233 * A PRP list pointer contains the address of a PRP list, structured as a linear 2234 * array of PRP entries. Each PRP entry in this list describes a segment of 2235 * physical memory. 2236 * 2237 * Each 64-bit PRP entry comprises an address and an offset field. The address 2238 * always points at the beginning of a 4KB physical memory page, and the offset 2239 * describes where within that 4KB page the memory segment begins. Only the 2240 * first element in a PRP list may contain a non-zero offset, implying that all 2241 * memory segments following the first begin at the start of a 4KB page. 2242 * 2243 * Each PRP element normally describes 4KB of physical memory, with exceptions 2244 * for the first and last elements in the list. If the memory being described 2245 * by the list begins at a non-zero offset within the first 4KB page, then the 2246 * first PRP element will contain a non-zero offset indicating where the region 2247 * begins within the 4KB page. The last memory segment may end before the end 2248 * of the 4KB segment, depending upon the overall size of the memory being 2249 * described by the PRP list. 2250 * 2251 * Since PRP entries lack any indication of size, the overall data buffer length 2252 * is used to determine where the end of the data memory buffer is located, and 2253 * how many PRP entries are required to describe it. 2254 */ 2255 static void 2256 _base_build_nvme_prp(struct MPT3SAS_ADAPTER *ioc, u16 smid, 2257 Mpi26NVMeEncapsulatedRequest_t *nvme_encap_request, 2258 dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma, 2259 size_t data_in_sz) 2260 { 2261 int prp_size = NVME_PRP_SIZE; 2262 __le64 *prp_entry, *prp1_entry, *prp2_entry; 2263 __le64 *prp_page; 2264 dma_addr_t prp_entry_dma, prp_page_dma, dma_addr; 2265 u32 offset, entry_len; 2266 u32 page_mask_result, page_mask; 2267 size_t length; 2268 struct mpt3sas_nvme_cmd *nvme_cmd = 2269 (void *)nvme_encap_request->NVMe_Command; 2270 2271 /* 2272 * Not all commands require a data transfer. If no data, just return 2273 * without constructing any PRP. 2274 */ 2275 if (!data_in_sz && !data_out_sz) 2276 return; 2277 prp1_entry = &nvme_cmd->prp1; 2278 prp2_entry = &nvme_cmd->prp2; 2279 prp_entry = prp1_entry; 2280 /* 2281 * For the PRP entries, use the specially allocated buffer of 2282 * contiguous memory. 2283 */ 2284 prp_page = (__le64 *)mpt3sas_base_get_pcie_sgl(ioc, smid); 2285 prp_page_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid); 2286 2287 /* 2288 * Check if we are within 1 entry of a page boundary we don't 2289 * want our first entry to be a PRP List entry. 2290 */ 2291 page_mask = ioc->page_size - 1; 2292 page_mask_result = (uintptr_t)((u8 *)prp_page + prp_size) & page_mask; 2293 if (!page_mask_result) { 2294 /* Bump up to next page boundary. */ 2295 prp_page = (__le64 *)((u8 *)prp_page + prp_size); 2296 prp_page_dma = prp_page_dma + prp_size; 2297 } 2298 2299 /* 2300 * Set PRP physical pointer, which initially points to the current PRP 2301 * DMA memory page. 2302 */ 2303 prp_entry_dma = prp_page_dma; 2304 2305 /* Get physical address and length of the data buffer. */ 2306 if (data_in_sz) { 2307 dma_addr = data_in_dma; 2308 length = data_in_sz; 2309 } else { 2310 dma_addr = data_out_dma; 2311 length = data_out_sz; 2312 } 2313 2314 /* Loop while the length is not zero. */ 2315 while (length) { 2316 /* 2317 * Check if we need to put a list pointer here if we are at 2318 * page boundary - prp_size (8 bytes). 2319 */ 2320 page_mask_result = (prp_entry_dma + prp_size) & page_mask; 2321 if (!page_mask_result) { 2322 /* 2323 * This is the last entry in a PRP List, so we need to 2324 * put a PRP list pointer here. What this does is: 2325 * - bump the current memory pointer to the next 2326 * address, which will be the next full page. 2327 * - set the PRP Entry to point to that page. This 2328 * is now the PRP List pointer. 2329 * - bump the PRP Entry pointer the start of the 2330 * next page. Since all of this PRP memory is 2331 * contiguous, no need to get a new page - it's 2332 * just the next address. 2333 */ 2334 prp_entry_dma++; 2335 *prp_entry = cpu_to_le64(prp_entry_dma); 2336 prp_entry++; 2337 } 2338 2339 /* Need to handle if entry will be part of a page. */ 2340 offset = dma_addr & page_mask; 2341 entry_len = ioc->page_size - offset; 2342 2343 if (prp_entry == prp1_entry) { 2344 /* 2345 * Must fill in the first PRP pointer (PRP1) before 2346 * moving on. 2347 */ 2348 *prp1_entry = cpu_to_le64(dma_addr); 2349 2350 /* 2351 * Now point to the second PRP entry within the 2352 * command (PRP2). 2353 */ 2354 prp_entry = prp2_entry; 2355 } else if (prp_entry == prp2_entry) { 2356 /* 2357 * Should the PRP2 entry be a PRP List pointer or just 2358 * a regular PRP pointer? If there is more than one 2359 * more page of data, must use a PRP List pointer. 2360 */ 2361 if (length > ioc->page_size) { 2362 /* 2363 * PRP2 will contain a PRP List pointer because 2364 * more PRP's are needed with this command. The 2365 * list will start at the beginning of the 2366 * contiguous buffer. 2367 */ 2368 *prp2_entry = cpu_to_le64(prp_entry_dma); 2369 2370 /* 2371 * The next PRP Entry will be the start of the 2372 * first PRP List. 2373 */ 2374 prp_entry = prp_page; 2375 } else { 2376 /* 2377 * After this, the PRP Entries are complete. 2378 * This command uses 2 PRP's and no PRP list. 2379 */ 2380 *prp2_entry = cpu_to_le64(dma_addr); 2381 } 2382 } else { 2383 /* 2384 * Put entry in list and bump the addresses. 2385 * 2386 * After PRP1 and PRP2 are filled in, this will fill in 2387 * all remaining PRP entries in a PRP List, one per 2388 * each time through the loop. 2389 */ 2390 *prp_entry = cpu_to_le64(dma_addr); 2391 prp_entry++; 2392 prp_entry_dma++; 2393 } 2394 2395 /* 2396 * Bump the phys address of the command's data buffer by the 2397 * entry_len. 2398 */ 2399 dma_addr += entry_len; 2400 2401 /* Decrement length accounting for last partial page. */ 2402 if (entry_len > length) 2403 length = 0; 2404 else 2405 length -= entry_len; 2406 } 2407 } 2408 2409 /** 2410 * base_make_prp_nvme - Prepare PRPs (Physical Region Page) - 2411 * SGLs specific to NVMe drives only 2412 * 2413 * @ioc: per adapter object 2414 * @scmd: SCSI command from the mid-layer 2415 * @mpi_request: mpi request 2416 * @smid: msg Index 2417 * @sge_count: scatter gather element count. 2418 * 2419 * Return: true: PRPs are built 2420 * false: IEEE SGLs needs to be built 2421 */ 2422 static void 2423 base_make_prp_nvme(struct MPT3SAS_ADAPTER *ioc, 2424 struct scsi_cmnd *scmd, 2425 Mpi25SCSIIORequest_t *mpi_request, 2426 u16 smid, int sge_count) 2427 { 2428 int sge_len, num_prp_in_chain = 0; 2429 Mpi25IeeeSgeChain64_t *main_chain_element, *ptr_first_sgl; 2430 __le64 *curr_buff; 2431 dma_addr_t msg_dma, sge_addr, offset; 2432 u32 page_mask, page_mask_result; 2433 struct scatterlist *sg_scmd; 2434 u32 first_prp_len; 2435 int data_len = scsi_bufflen(scmd); 2436 u32 nvme_pg_size; 2437 2438 nvme_pg_size = max_t(u32, ioc->page_size, NVME_PRP_PAGE_SIZE); 2439 /* 2440 * Nvme has a very convoluted prp format. One prp is required 2441 * for each page or partial page. Driver need to split up OS sg_list 2442 * entries if it is longer than one page or cross a page 2443 * boundary. Driver also have to insert a PRP list pointer entry as 2444 * the last entry in each physical page of the PRP list. 2445 * 2446 * NOTE: The first PRP "entry" is actually placed in the first 2447 * SGL entry in the main message as IEEE 64 format. The 2nd 2448 * entry in the main message is the chain element, and the rest 2449 * of the PRP entries are built in the contiguous pcie buffer. 2450 */ 2451 page_mask = nvme_pg_size - 1; 2452 2453 /* 2454 * Native SGL is needed. 2455 * Put a chain element in main message frame that points to the first 2456 * chain buffer. 2457 * 2458 * NOTE: The ChainOffset field must be 0 when using a chain pointer to 2459 * a native SGL. 2460 */ 2461 2462 /* Set main message chain element pointer */ 2463 main_chain_element = (pMpi25IeeeSgeChain64_t)&mpi_request->SGL; 2464 /* 2465 * For NVMe the chain element needs to be the 2nd SG entry in the main 2466 * message. 2467 */ 2468 main_chain_element = (Mpi25IeeeSgeChain64_t *) 2469 ((u8 *)main_chain_element + sizeof(MPI25_IEEE_SGE_CHAIN64)); 2470 2471 /* 2472 * For the PRP entries, use the specially allocated buffer of 2473 * contiguous memory. Normal chain buffers can't be used 2474 * because each chain buffer would need to be the size of an OS 2475 * page (4k). 2476 */ 2477 curr_buff = mpt3sas_base_get_pcie_sgl(ioc, smid); 2478 msg_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid); 2479 2480 main_chain_element->Address = cpu_to_le64(msg_dma); 2481 main_chain_element->NextChainOffset = 0; 2482 main_chain_element->Flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT | 2483 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR | 2484 MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP; 2485 2486 /* Build first prp, sge need not to be page aligned*/ 2487 ptr_first_sgl = (pMpi25IeeeSgeChain64_t)&mpi_request->SGL; 2488 sg_scmd = scsi_sglist(scmd); 2489 sge_addr = sg_dma_address(sg_scmd); 2490 sge_len = sg_dma_len(sg_scmd); 2491 2492 offset = sge_addr & page_mask; 2493 first_prp_len = nvme_pg_size - offset; 2494 2495 ptr_first_sgl->Address = cpu_to_le64(sge_addr); 2496 ptr_first_sgl->Length = cpu_to_le32(first_prp_len); 2497 2498 data_len -= first_prp_len; 2499 2500 if (sge_len > first_prp_len) { 2501 sge_addr += first_prp_len; 2502 sge_len -= first_prp_len; 2503 } else if (data_len && (sge_len == first_prp_len)) { 2504 sg_scmd = sg_next(sg_scmd); 2505 sge_addr = sg_dma_address(sg_scmd); 2506 sge_len = sg_dma_len(sg_scmd); 2507 } 2508 2509 for (;;) { 2510 offset = sge_addr & page_mask; 2511 2512 /* Put PRP pointer due to page boundary*/ 2513 page_mask_result = (uintptr_t)(curr_buff + 1) & page_mask; 2514 if (unlikely(!page_mask_result)) { 2515 scmd_printk(KERN_NOTICE, 2516 scmd, "page boundary curr_buff: 0x%p\n", 2517 curr_buff); 2518 msg_dma += 8; 2519 *curr_buff = cpu_to_le64(msg_dma); 2520 curr_buff++; 2521 num_prp_in_chain++; 2522 } 2523 2524 *curr_buff = cpu_to_le64(sge_addr); 2525 curr_buff++; 2526 msg_dma += 8; 2527 num_prp_in_chain++; 2528 2529 sge_addr += nvme_pg_size; 2530 sge_len -= nvme_pg_size; 2531 data_len -= nvme_pg_size; 2532 2533 if (data_len <= 0) 2534 break; 2535 2536 if (sge_len > 0) 2537 continue; 2538 2539 sg_scmd = sg_next(sg_scmd); 2540 sge_addr = sg_dma_address(sg_scmd); 2541 sge_len = sg_dma_len(sg_scmd); 2542 } 2543 2544 main_chain_element->Length = 2545 cpu_to_le32(num_prp_in_chain * sizeof(u64)); 2546 return; 2547 } 2548 2549 static bool 2550 base_is_prp_possible(struct MPT3SAS_ADAPTER *ioc, 2551 struct _pcie_device *pcie_device, struct scsi_cmnd *scmd, int sge_count) 2552 { 2553 u32 data_length = 0; 2554 bool build_prp = true; 2555 2556 data_length = scsi_bufflen(scmd); 2557 if (pcie_device && 2558 (mpt3sas_scsih_is_pcie_scsi_device(pcie_device->device_info))) { 2559 build_prp = false; 2560 return build_prp; 2561 } 2562 2563 /* If Datalenth is <= 16K and number of SGE’s entries are <= 2 2564 * we built IEEE SGL 2565 */ 2566 if ((data_length <= NVME_PRP_PAGE_SIZE*4) && (sge_count <= 2)) 2567 build_prp = false; 2568 2569 return build_prp; 2570 } 2571 2572 /** 2573 * _base_check_pcie_native_sgl - This function is called for PCIe end devices to 2574 * determine if the driver needs to build a native SGL. If so, that native 2575 * SGL is built in the special contiguous buffers allocated especially for 2576 * PCIe SGL creation. If the driver will not build a native SGL, return 2577 * TRUE and a normal IEEE SGL will be built. Currently this routine 2578 * supports NVMe. 2579 * @ioc: per adapter object 2580 * @mpi_request: mf request pointer 2581 * @smid: system request message index 2582 * @scmd: scsi command 2583 * @pcie_device: points to the PCIe device's info 2584 * 2585 * Return: 0 if native SGL was built, 1 if no SGL was built 2586 */ 2587 static int 2588 _base_check_pcie_native_sgl(struct MPT3SAS_ADAPTER *ioc, 2589 Mpi25SCSIIORequest_t *mpi_request, u16 smid, struct scsi_cmnd *scmd, 2590 struct _pcie_device *pcie_device) 2591 { 2592 int sges_left; 2593 2594 /* Get the SG list pointer and info. */ 2595 sges_left = scsi_dma_map(scmd); 2596 if (sges_left < 0) { 2597 sdev_printk(KERN_ERR, scmd->device, 2598 "scsi_dma_map failed: request for %d bytes!\n", 2599 scsi_bufflen(scmd)); 2600 return 1; 2601 } 2602 2603 /* Check if we need to build a native SG list. */ 2604 if (!base_is_prp_possible(ioc, pcie_device, 2605 scmd, sges_left)) { 2606 /* We built a native SG list, just return. */ 2607 goto out; 2608 } 2609 2610 /* 2611 * Build native NVMe PRP. 2612 */ 2613 base_make_prp_nvme(ioc, scmd, mpi_request, 2614 smid, sges_left); 2615 2616 return 0; 2617 out: 2618 scsi_dma_unmap(scmd); 2619 return 1; 2620 } 2621 2622 /** 2623 * _base_add_sg_single_ieee - add sg element for IEEE format 2624 * @paddr: virtual address for SGE 2625 * @flags: SGE flags 2626 * @chain_offset: number of 128 byte elements from start of segment 2627 * @length: data transfer length 2628 * @dma_addr: Physical address 2629 */ 2630 static void 2631 _base_add_sg_single_ieee(void *paddr, u8 flags, u8 chain_offset, u32 length, 2632 dma_addr_t dma_addr) 2633 { 2634 Mpi25IeeeSgeChain64_t *sgel = paddr; 2635 2636 sgel->Flags = flags; 2637 sgel->NextChainOffset = chain_offset; 2638 sgel->Length = cpu_to_le32(length); 2639 sgel->Address = cpu_to_le64(dma_addr); 2640 } 2641 2642 /** 2643 * _base_build_zero_len_sge_ieee - build zero length sg entry for IEEE format 2644 * @ioc: per adapter object 2645 * @paddr: virtual address for SGE 2646 * 2647 * Create a zero length scatter gather entry to insure the IOCs hardware has 2648 * something to use if the target device goes brain dead and tries 2649 * to send data even when none is asked for. 2650 */ 2651 static void 2652 _base_build_zero_len_sge_ieee(struct MPT3SAS_ADAPTER *ioc, void *paddr) 2653 { 2654 u8 sgl_flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT | 2655 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR | 2656 MPI25_IEEE_SGE_FLAGS_END_OF_LIST); 2657 2658 _base_add_sg_single_ieee(paddr, sgl_flags, 0, 0, -1); 2659 } 2660 2661 /** 2662 * _base_build_sg_scmd - main sg creation routine 2663 * pcie_device is unused here! 2664 * @ioc: per adapter object 2665 * @scmd: scsi command 2666 * @smid: system request message index 2667 * @unused: unused pcie_device pointer 2668 * Context: none. 2669 * 2670 * The main routine that builds scatter gather table from a given 2671 * scsi request sent via the .queuecommand main handler. 2672 * 2673 * Return: 0 success, anything else error 2674 */ 2675 static int 2676 _base_build_sg_scmd(struct MPT3SAS_ADAPTER *ioc, 2677 struct scsi_cmnd *scmd, u16 smid, struct _pcie_device *unused) 2678 { 2679 Mpi2SCSIIORequest_t *mpi_request; 2680 dma_addr_t chain_dma; 2681 struct scatterlist *sg_scmd; 2682 void *sg_local, *chain; 2683 u32 chain_offset; 2684 u32 chain_length; 2685 u32 chain_flags; 2686 int sges_left; 2687 u32 sges_in_segment; 2688 u32 sgl_flags; 2689 u32 sgl_flags_last_element; 2690 u32 sgl_flags_end_buffer; 2691 struct chain_tracker *chain_req; 2692 2693 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); 2694 2695 /* init scatter gather flags */ 2696 sgl_flags = MPI2_SGE_FLAGS_SIMPLE_ELEMENT; 2697 if (scmd->sc_data_direction == DMA_TO_DEVICE) 2698 sgl_flags |= MPI2_SGE_FLAGS_HOST_TO_IOC; 2699 sgl_flags_last_element = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT) 2700 << MPI2_SGE_FLAGS_SHIFT; 2701 sgl_flags_end_buffer = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT | 2702 MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST) 2703 << MPI2_SGE_FLAGS_SHIFT; 2704 sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT; 2705 2706 sg_scmd = scsi_sglist(scmd); 2707 sges_left = scsi_dma_map(scmd); 2708 if (sges_left < 0) { 2709 sdev_printk(KERN_ERR, scmd->device, 2710 "scsi_dma_map failed: request for %d bytes!\n", 2711 scsi_bufflen(scmd)); 2712 return -ENOMEM; 2713 } 2714 2715 sg_local = &mpi_request->SGL; 2716 sges_in_segment = ioc->max_sges_in_main_message; 2717 if (sges_left <= sges_in_segment) 2718 goto fill_in_last_segment; 2719 2720 mpi_request->ChainOffset = (offsetof(Mpi2SCSIIORequest_t, SGL) + 2721 (sges_in_segment * ioc->sge_size))/4; 2722 2723 /* fill in main message segment when there is a chain following */ 2724 while (sges_in_segment) { 2725 if (sges_in_segment == 1) 2726 ioc->base_add_sg_single(sg_local, 2727 sgl_flags_last_element | sg_dma_len(sg_scmd), 2728 sg_dma_address(sg_scmd)); 2729 else 2730 ioc->base_add_sg_single(sg_local, sgl_flags | 2731 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd)); 2732 sg_scmd = sg_next(sg_scmd); 2733 sg_local += ioc->sge_size; 2734 sges_left--; 2735 sges_in_segment--; 2736 } 2737 2738 /* initializing the chain flags and pointers */ 2739 chain_flags = MPI2_SGE_FLAGS_CHAIN_ELEMENT << MPI2_SGE_FLAGS_SHIFT; 2740 chain_req = _base_get_chain_buffer_tracker(ioc, scmd); 2741 if (!chain_req) 2742 return -1; 2743 chain = chain_req->chain_buffer; 2744 chain_dma = chain_req->chain_buffer_dma; 2745 do { 2746 sges_in_segment = (sges_left <= 2747 ioc->max_sges_in_chain_message) ? sges_left : 2748 ioc->max_sges_in_chain_message; 2749 chain_offset = (sges_left == sges_in_segment) ? 2750 0 : (sges_in_segment * ioc->sge_size)/4; 2751 chain_length = sges_in_segment * ioc->sge_size; 2752 if (chain_offset) { 2753 chain_offset = chain_offset << 2754 MPI2_SGE_CHAIN_OFFSET_SHIFT; 2755 chain_length += ioc->sge_size; 2756 } 2757 ioc->base_add_sg_single(sg_local, chain_flags | chain_offset | 2758 chain_length, chain_dma); 2759 sg_local = chain; 2760 if (!chain_offset) 2761 goto fill_in_last_segment; 2762 2763 /* fill in chain segments */ 2764 while (sges_in_segment) { 2765 if (sges_in_segment == 1) 2766 ioc->base_add_sg_single(sg_local, 2767 sgl_flags_last_element | 2768 sg_dma_len(sg_scmd), 2769 sg_dma_address(sg_scmd)); 2770 else 2771 ioc->base_add_sg_single(sg_local, sgl_flags | 2772 sg_dma_len(sg_scmd), 2773 sg_dma_address(sg_scmd)); 2774 sg_scmd = sg_next(sg_scmd); 2775 sg_local += ioc->sge_size; 2776 sges_left--; 2777 sges_in_segment--; 2778 } 2779 2780 chain_req = _base_get_chain_buffer_tracker(ioc, scmd); 2781 if (!chain_req) 2782 return -1; 2783 chain = chain_req->chain_buffer; 2784 chain_dma = chain_req->chain_buffer_dma; 2785 } while (1); 2786 2787 2788 fill_in_last_segment: 2789 2790 /* fill the last segment */ 2791 while (sges_left) { 2792 if (sges_left == 1) 2793 ioc->base_add_sg_single(sg_local, sgl_flags_end_buffer | 2794 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd)); 2795 else 2796 ioc->base_add_sg_single(sg_local, sgl_flags | 2797 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd)); 2798 sg_scmd = sg_next(sg_scmd); 2799 sg_local += ioc->sge_size; 2800 sges_left--; 2801 } 2802 2803 return 0; 2804 } 2805 2806 /** 2807 * _base_build_sg_scmd_ieee - main sg creation routine for IEEE format 2808 * @ioc: per adapter object 2809 * @scmd: scsi command 2810 * @smid: system request message index 2811 * @pcie_device: Pointer to pcie_device. If set, the pcie native sgl will be 2812 * constructed on need. 2813 * Context: none. 2814 * 2815 * The main routine that builds scatter gather table from a given 2816 * scsi request sent via the .queuecommand main handler. 2817 * 2818 * Return: 0 success, anything else error 2819 */ 2820 static int 2821 _base_build_sg_scmd_ieee(struct MPT3SAS_ADAPTER *ioc, 2822 struct scsi_cmnd *scmd, u16 smid, struct _pcie_device *pcie_device) 2823 { 2824 Mpi25SCSIIORequest_t *mpi_request; 2825 dma_addr_t chain_dma; 2826 struct scatterlist *sg_scmd; 2827 void *sg_local, *chain; 2828 u32 chain_offset; 2829 u32 chain_length; 2830 int sges_left; 2831 u32 sges_in_segment; 2832 u8 simple_sgl_flags; 2833 u8 simple_sgl_flags_last; 2834 u8 chain_sgl_flags; 2835 struct chain_tracker *chain_req; 2836 2837 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); 2838 2839 /* init scatter gather flags */ 2840 simple_sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT | 2841 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR; 2842 simple_sgl_flags_last = simple_sgl_flags | 2843 MPI25_IEEE_SGE_FLAGS_END_OF_LIST; 2844 chain_sgl_flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT | 2845 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR; 2846 2847 /* Check if we need to build a native SG list. */ 2848 if ((pcie_device) && (_base_check_pcie_native_sgl(ioc, mpi_request, 2849 smid, scmd, pcie_device) == 0)) { 2850 /* We built a native SG list, just return. */ 2851 return 0; 2852 } 2853 2854 sg_scmd = scsi_sglist(scmd); 2855 sges_left = scsi_dma_map(scmd); 2856 if (sges_left < 0) { 2857 sdev_printk(KERN_ERR, scmd->device, 2858 "scsi_dma_map failed: request for %d bytes!\n", 2859 scsi_bufflen(scmd)); 2860 return -ENOMEM; 2861 } 2862 2863 sg_local = &mpi_request->SGL; 2864 sges_in_segment = (ioc->request_sz - 2865 offsetof(Mpi25SCSIIORequest_t, SGL))/ioc->sge_size_ieee; 2866 if (sges_left <= sges_in_segment) 2867 goto fill_in_last_segment; 2868 2869 mpi_request->ChainOffset = (sges_in_segment - 1 /* chain element */) + 2870 (offsetof(Mpi25SCSIIORequest_t, SGL)/ioc->sge_size_ieee); 2871 2872 /* fill in main message segment when there is a chain following */ 2873 while (sges_in_segment > 1) { 2874 _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0, 2875 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd)); 2876 sg_scmd = sg_next(sg_scmd); 2877 sg_local += ioc->sge_size_ieee; 2878 sges_left--; 2879 sges_in_segment--; 2880 } 2881 2882 /* initializing the pointers */ 2883 chain_req = _base_get_chain_buffer_tracker(ioc, scmd); 2884 if (!chain_req) 2885 return -1; 2886 chain = chain_req->chain_buffer; 2887 chain_dma = chain_req->chain_buffer_dma; 2888 do { 2889 sges_in_segment = (sges_left <= 2890 ioc->max_sges_in_chain_message) ? sges_left : 2891 ioc->max_sges_in_chain_message; 2892 chain_offset = (sges_left == sges_in_segment) ? 2893 0 : sges_in_segment; 2894 chain_length = sges_in_segment * ioc->sge_size_ieee; 2895 if (chain_offset) 2896 chain_length += ioc->sge_size_ieee; 2897 _base_add_sg_single_ieee(sg_local, chain_sgl_flags, 2898 chain_offset, chain_length, chain_dma); 2899 2900 sg_local = chain; 2901 if (!chain_offset) 2902 goto fill_in_last_segment; 2903 2904 /* fill in chain segments */ 2905 while (sges_in_segment) { 2906 _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0, 2907 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd)); 2908 sg_scmd = sg_next(sg_scmd); 2909 sg_local += ioc->sge_size_ieee; 2910 sges_left--; 2911 sges_in_segment--; 2912 } 2913 2914 chain_req = _base_get_chain_buffer_tracker(ioc, scmd); 2915 if (!chain_req) 2916 return -1; 2917 chain = chain_req->chain_buffer; 2918 chain_dma = chain_req->chain_buffer_dma; 2919 } while (1); 2920 2921 2922 fill_in_last_segment: 2923 2924 /* fill the last segment */ 2925 while (sges_left > 0) { 2926 if (sges_left == 1) 2927 _base_add_sg_single_ieee(sg_local, 2928 simple_sgl_flags_last, 0, sg_dma_len(sg_scmd), 2929 sg_dma_address(sg_scmd)); 2930 else 2931 _base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0, 2932 sg_dma_len(sg_scmd), sg_dma_address(sg_scmd)); 2933 sg_scmd = sg_next(sg_scmd); 2934 sg_local += ioc->sge_size_ieee; 2935 sges_left--; 2936 } 2937 2938 return 0; 2939 } 2940 2941 /** 2942 * _base_build_sg_ieee - build generic sg for IEEE format 2943 * @ioc: per adapter object 2944 * @psge: virtual address for SGE 2945 * @data_out_dma: physical address for WRITES 2946 * @data_out_sz: data xfer size for WRITES 2947 * @data_in_dma: physical address for READS 2948 * @data_in_sz: data xfer size for READS 2949 */ 2950 static void 2951 _base_build_sg_ieee(struct MPT3SAS_ADAPTER *ioc, void *psge, 2952 dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma, 2953 size_t data_in_sz) 2954 { 2955 u8 sgl_flags; 2956 2957 if (!data_out_sz && !data_in_sz) { 2958 _base_build_zero_len_sge_ieee(ioc, psge); 2959 return; 2960 } 2961 2962 if (data_out_sz && data_in_sz) { 2963 /* WRITE sgel first */ 2964 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT | 2965 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR; 2966 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz, 2967 data_out_dma); 2968 2969 /* incr sgel */ 2970 psge += ioc->sge_size_ieee; 2971 2972 /* READ sgel last */ 2973 sgl_flags |= MPI25_IEEE_SGE_FLAGS_END_OF_LIST; 2974 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz, 2975 data_in_dma); 2976 } else if (data_out_sz) /* WRITE */ { 2977 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT | 2978 MPI25_IEEE_SGE_FLAGS_END_OF_LIST | 2979 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR; 2980 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz, 2981 data_out_dma); 2982 } else if (data_in_sz) /* READ */ { 2983 sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT | 2984 MPI25_IEEE_SGE_FLAGS_END_OF_LIST | 2985 MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR; 2986 _base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz, 2987 data_in_dma); 2988 } 2989 } 2990 2991 #define convert_to_kb(x) ((x) << (PAGE_SHIFT - 10)) 2992 2993 /** 2994 * _base_config_dma_addressing - set dma addressing 2995 * @ioc: per adapter object 2996 * @pdev: PCI device struct 2997 * 2998 * Return: 0 for success, non-zero for failure. 2999 */ 3000 static int 3001 _base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev *pdev) 3002 { 3003 struct sysinfo s; 3004 3005 if (ioc->is_mcpu_endpoint || 3006 sizeof(dma_addr_t) == 4 || ioc->use_32bit_dma || 3007 dma_get_required_mask(&pdev->dev) <= 32) 3008 ioc->dma_mask = 32; 3009 /* Set 63 bit DMA mask for all SAS3 and SAS35 controllers */ 3010 else if (ioc->hba_mpi_version_belonged > MPI2_VERSION) 3011 ioc->dma_mask = 63; 3012 else 3013 ioc->dma_mask = 64; 3014 3015 if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(ioc->dma_mask)) || 3016 dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(ioc->dma_mask))) 3017 return -ENODEV; 3018 3019 if (ioc->dma_mask > 32) { 3020 ioc->base_add_sg_single = &_base_add_sg_single_64; 3021 ioc->sge_size = sizeof(Mpi2SGESimple64_t); 3022 } else { 3023 ioc->base_add_sg_single = &_base_add_sg_single_32; 3024 ioc->sge_size = sizeof(Mpi2SGESimple32_t); 3025 } 3026 3027 si_meminfo(&s); 3028 ioc_info(ioc, "%d BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (%ld kB)\n", 3029 ioc->dma_mask, convert_to_kb(s.totalram)); 3030 3031 return 0; 3032 } 3033 3034 /** 3035 * _base_check_enable_msix - checks MSIX capabable. 3036 * @ioc: per adapter object 3037 * 3038 * Check to see if card is capable of MSIX, and set number 3039 * of available msix vectors 3040 */ 3041 static int 3042 _base_check_enable_msix(struct MPT3SAS_ADAPTER *ioc) 3043 { 3044 int base; 3045 u16 message_control; 3046 3047 /* Check whether controller SAS2008 B0 controller, 3048 * if it is SAS2008 B0 controller use IO-APIC instead of MSIX 3049 */ 3050 if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 && 3051 ioc->pdev->revision == SAS2_PCI_DEVICE_B0_REVISION) { 3052 return -EINVAL; 3053 } 3054 3055 base = pci_find_capability(ioc->pdev, PCI_CAP_ID_MSIX); 3056 if (!base) { 3057 dfailprintk(ioc, ioc_info(ioc, "msix not supported\n")); 3058 return -EINVAL; 3059 } 3060 3061 /* get msix vector count */ 3062 /* NUMA_IO not supported for older controllers */ 3063 if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2004 || 3064 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 || 3065 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_1 || 3066 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_2 || 3067 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_3 || 3068 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_1 || 3069 ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_2) 3070 ioc->msix_vector_count = 1; 3071 else { 3072 pci_read_config_word(ioc->pdev, base + 2, &message_control); 3073 ioc->msix_vector_count = (message_control & 0x3FF) + 1; 3074 } 3075 dinitprintk(ioc, ioc_info(ioc, "msix is supported, vector_count(%d)\n", 3076 ioc->msix_vector_count)); 3077 return 0; 3078 } 3079 3080 /** 3081 * mpt3sas_base_free_irq - free irq 3082 * @ioc: per adapter object 3083 * 3084 * Freeing respective reply_queue from the list. 3085 */ 3086 void 3087 mpt3sas_base_free_irq(struct MPT3SAS_ADAPTER *ioc) 3088 { 3089 unsigned int irq; 3090 struct adapter_reply_queue *reply_q, *next; 3091 3092 if (list_empty(&ioc->reply_queue_list)) 3093 return; 3094 3095 list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) { 3096 list_del(&reply_q->list); 3097 if (reply_q->is_iouring_poll_q) { 3098 kfree(reply_q); 3099 continue; 3100 } 3101 3102 if (ioc->smp_affinity_enable) { 3103 irq = pci_irq_vector(ioc->pdev, reply_q->msix_index); 3104 irq_update_affinity_hint(irq, NULL); 3105 } 3106 free_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index), 3107 reply_q); 3108 kfree(reply_q); 3109 } 3110 } 3111 3112 /** 3113 * _base_request_irq - request irq 3114 * @ioc: per adapter object 3115 * @index: msix index into vector table 3116 * 3117 * Inserting respective reply_queue into the list. 3118 */ 3119 static int 3120 _base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index) 3121 { 3122 struct pci_dev *pdev = ioc->pdev; 3123 struct adapter_reply_queue *reply_q; 3124 int r, qid; 3125 3126 reply_q = kzalloc(sizeof(struct adapter_reply_queue), GFP_KERNEL); 3127 if (!reply_q) { 3128 ioc_err(ioc, "unable to allocate memory %zu!\n", 3129 sizeof(struct adapter_reply_queue)); 3130 return -ENOMEM; 3131 } 3132 reply_q->ioc = ioc; 3133 reply_q->msix_index = index; 3134 3135 atomic_set(&reply_q->busy, 0); 3136 3137 if (index >= ioc->iopoll_q_start_index) { 3138 qid = index - ioc->iopoll_q_start_index; 3139 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-mq-poll%d", 3140 ioc->driver_name, ioc->id, qid); 3141 reply_q->is_iouring_poll_q = 1; 3142 ioc->io_uring_poll_queues[qid].reply_q = reply_q; 3143 goto out; 3144 } 3145 3146 3147 if (ioc->msix_enable) 3148 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-msix%d", 3149 ioc->driver_name, ioc->id, index); 3150 else 3151 snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d", 3152 ioc->driver_name, ioc->id); 3153 r = request_irq(pci_irq_vector(pdev, index), _base_interrupt, 3154 IRQF_SHARED, reply_q->name, reply_q); 3155 if (r) { 3156 pr_err("%s: unable to allocate interrupt %d!\n", 3157 reply_q->name, pci_irq_vector(pdev, index)); 3158 kfree(reply_q); 3159 return -EBUSY; 3160 } 3161 out: 3162 INIT_LIST_HEAD(&reply_q->list); 3163 list_add_tail(&reply_q->list, &ioc->reply_queue_list); 3164 return 0; 3165 } 3166 3167 /** 3168 * _base_assign_reply_queues - assigning msix index for each cpu 3169 * @ioc: per adapter object 3170 * 3171 * The enduser would need to set the affinity via /proc/irq/#/smp_affinity 3172 */ 3173 static void 3174 _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc) 3175 { 3176 unsigned int cpu, nr_cpus, nr_msix, index = 0, irq; 3177 struct adapter_reply_queue *reply_q; 3178 int iopoll_q_count = ioc->reply_queue_count - 3179 ioc->iopoll_q_start_index; 3180 const struct cpumask *mask; 3181 3182 if (!_base_is_controller_msix_enabled(ioc)) 3183 return; 3184 3185 if (ioc->msix_load_balance) 3186 return; 3187 3188 memset(ioc->cpu_msix_table, 0, ioc->cpu_msix_table_sz); 3189 3190 nr_cpus = num_online_cpus(); 3191 nr_msix = ioc->reply_queue_count = min(ioc->reply_queue_count, 3192 ioc->facts.MaxMSIxVectors); 3193 if (!nr_msix) 3194 return; 3195 3196 if (ioc->smp_affinity_enable) { 3197 3198 /* 3199 * set irq affinity to local numa node for those irqs 3200 * corresponding to high iops queues. 3201 */ 3202 if (ioc->high_iops_queues) { 3203 mask = cpumask_of_node(dev_to_node(&ioc->pdev->dev)); 3204 for (index = 0; index < ioc->high_iops_queues; 3205 index++) { 3206 irq = pci_irq_vector(ioc->pdev, index); 3207 irq_set_affinity_and_hint(irq, mask); 3208 } 3209 } 3210 3211 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { 3212 const cpumask_t *mask; 3213 3214 if (reply_q->msix_index < ioc->high_iops_queues || 3215 reply_q->msix_index >= ioc->iopoll_q_start_index) 3216 continue; 3217 3218 mask = pci_irq_get_affinity(ioc->pdev, 3219 reply_q->msix_index); 3220 if (!mask) { 3221 ioc_warn(ioc, "no affinity for msi %x\n", 3222 reply_q->msix_index); 3223 goto fall_back; 3224 } 3225 3226 for_each_cpu_and(cpu, mask, cpu_online_mask) { 3227 if (cpu >= ioc->cpu_msix_table_sz) 3228 break; 3229 ioc->cpu_msix_table[cpu] = reply_q->msix_index; 3230 } 3231 } 3232 return; 3233 } 3234 3235 fall_back: 3236 cpu = cpumask_first(cpu_online_mask); 3237 nr_msix -= (ioc->high_iops_queues - iopoll_q_count); 3238 index = 0; 3239 3240 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { 3241 unsigned int i, group = nr_cpus / nr_msix; 3242 3243 if (reply_q->msix_index < ioc->high_iops_queues || 3244 reply_q->msix_index >= ioc->iopoll_q_start_index) 3245 continue; 3246 3247 if (cpu >= nr_cpus) 3248 break; 3249 3250 if (index < nr_cpus % nr_msix) 3251 group++; 3252 3253 for (i = 0 ; i < group ; i++) { 3254 ioc->cpu_msix_table[cpu] = reply_q->msix_index; 3255 cpu = cpumask_next(cpu, cpu_online_mask); 3256 } 3257 index++; 3258 } 3259 } 3260 3261 /** 3262 * _base_check_and_enable_high_iops_queues - enable high iops mode 3263 * @ioc: per adapter object 3264 * @hba_msix_vector_count: msix vectors supported by HBA 3265 * 3266 * Enable high iops queues only if 3267 * - HBA is a SEA/AERO controller and 3268 * - MSI-Xs vector supported by the HBA is 128 and 3269 * - total CPU count in the system >=16 and 3270 * - loaded driver with default max_msix_vectors module parameter and 3271 * - system booted in non kdump mode 3272 * 3273 * Return: nothing. 3274 */ 3275 static void 3276 _base_check_and_enable_high_iops_queues(struct MPT3SAS_ADAPTER *ioc, 3277 int hba_msix_vector_count) 3278 { 3279 u16 lnksta, speed; 3280 3281 /* 3282 * Disable high iops queues if io uring poll queues are enabled. 3283 */ 3284 if (perf_mode == MPT_PERF_MODE_IOPS || 3285 perf_mode == MPT_PERF_MODE_LATENCY || 3286 ioc->io_uring_poll_queues) { 3287 ioc->high_iops_queues = 0; 3288 return; 3289 } 3290 3291 if (perf_mode == MPT_PERF_MODE_DEFAULT) { 3292 3293 pcie_capability_read_word(ioc->pdev, PCI_EXP_LNKSTA, &lnksta); 3294 speed = lnksta & PCI_EXP_LNKSTA_CLS; 3295 3296 if (speed < 0x4) { 3297 ioc->high_iops_queues = 0; 3298 return; 3299 } 3300 } 3301 3302 if (!reset_devices && ioc->is_aero_ioc && 3303 hba_msix_vector_count == MPT3SAS_GEN35_MAX_MSIX_QUEUES && 3304 num_online_cpus() >= MPT3SAS_HIGH_IOPS_REPLY_QUEUES && 3305 max_msix_vectors == -1) 3306 ioc->high_iops_queues = MPT3SAS_HIGH_IOPS_REPLY_QUEUES; 3307 else 3308 ioc->high_iops_queues = 0; 3309 } 3310 3311 /** 3312 * mpt3sas_base_disable_msix - disables msix 3313 * @ioc: per adapter object 3314 * 3315 */ 3316 void 3317 mpt3sas_base_disable_msix(struct MPT3SAS_ADAPTER *ioc) 3318 { 3319 if (!ioc->msix_enable) 3320 return; 3321 pci_free_irq_vectors(ioc->pdev); 3322 ioc->msix_enable = 0; 3323 kfree(ioc->io_uring_poll_queues); 3324 } 3325 3326 /** 3327 * _base_alloc_irq_vectors - allocate msix vectors 3328 * @ioc: per adapter object 3329 * 3330 */ 3331 static int 3332 _base_alloc_irq_vectors(struct MPT3SAS_ADAPTER *ioc) 3333 { 3334 int i, irq_flags = PCI_IRQ_MSIX; 3335 struct irq_affinity desc = { .pre_vectors = ioc->high_iops_queues }; 3336 struct irq_affinity *descp = &desc; 3337 /* 3338 * Don't allocate msix vectors for poll_queues. 3339 * msix_vectors is always within a range of FW supported reply queue. 3340 */ 3341 int nr_msix_vectors = ioc->iopoll_q_start_index; 3342 3343 3344 if (ioc->smp_affinity_enable) 3345 irq_flags |= PCI_IRQ_AFFINITY | PCI_IRQ_ALL_TYPES; 3346 else 3347 descp = NULL; 3348 3349 ioc_info(ioc, " %d %d %d\n", ioc->high_iops_queues, 3350 ioc->reply_queue_count, nr_msix_vectors); 3351 3352 i = pci_alloc_irq_vectors_affinity(ioc->pdev, 3353 ioc->high_iops_queues, 3354 nr_msix_vectors, irq_flags, descp); 3355 3356 return i; 3357 } 3358 3359 /** 3360 * _base_enable_msix - enables msix, failback to io_apic 3361 * @ioc: per adapter object 3362 * 3363 */ 3364 static int 3365 _base_enable_msix(struct MPT3SAS_ADAPTER *ioc) 3366 { 3367 int r; 3368 int i, local_max_msix_vectors; 3369 u8 try_msix = 0; 3370 int iopoll_q_count = 0; 3371 3372 ioc->msix_load_balance = false; 3373 3374 if (msix_disable == -1 || msix_disable == 0) 3375 try_msix = 1; 3376 3377 if (!try_msix) 3378 goto try_ioapic; 3379 3380 if (_base_check_enable_msix(ioc) != 0) 3381 goto try_ioapic; 3382 3383 ioc_info(ioc, "MSI-X vectors supported: %d\n", ioc->msix_vector_count); 3384 pr_info("\t no of cores: %d, max_msix_vectors: %d\n", 3385 ioc->cpu_count, max_msix_vectors); 3386 3387 ioc->reply_queue_count = 3388 min_t(int, ioc->cpu_count, ioc->msix_vector_count); 3389 3390 if (!ioc->rdpq_array_enable && max_msix_vectors == -1) 3391 local_max_msix_vectors = (reset_devices) ? 1 : 8; 3392 else 3393 local_max_msix_vectors = max_msix_vectors; 3394 3395 if (local_max_msix_vectors == 0) 3396 goto try_ioapic; 3397 3398 /* 3399 * Enable msix_load_balance only if combined reply queue mode is 3400 * disabled on SAS3 & above generation HBA devices. 3401 */ 3402 if (!ioc->combined_reply_queue && 3403 ioc->hba_mpi_version_belonged != MPI2_VERSION) { 3404 ioc_info(ioc, 3405 "combined ReplyQueue is off, Enabling msix load balance\n"); 3406 ioc->msix_load_balance = true; 3407 } 3408 3409 /* 3410 * smp affinity setting is not need when msix load balance 3411 * is enabled. 3412 */ 3413 if (ioc->msix_load_balance) 3414 ioc->smp_affinity_enable = 0; 3415 3416 if (!ioc->smp_affinity_enable || ioc->reply_queue_count <= 1) 3417 ioc->shost->host_tagset = 0; 3418 3419 /* 3420 * Enable io uring poll queues only if host_tagset is enabled. 3421 */ 3422 if (ioc->shost->host_tagset) 3423 iopoll_q_count = poll_queues; 3424 3425 if (iopoll_q_count) { 3426 ioc->io_uring_poll_queues = kcalloc(iopoll_q_count, 3427 sizeof(struct io_uring_poll_queue), GFP_KERNEL); 3428 if (!ioc->io_uring_poll_queues) 3429 iopoll_q_count = 0; 3430 } 3431 3432 if (ioc->is_aero_ioc) 3433 _base_check_and_enable_high_iops_queues(ioc, 3434 ioc->msix_vector_count); 3435 3436 /* 3437 * Add high iops queues count to reply queue count if high iops queues 3438 * are enabled. 3439 */ 3440 ioc->reply_queue_count = min_t(int, 3441 ioc->reply_queue_count + ioc->high_iops_queues, 3442 ioc->msix_vector_count); 3443 3444 /* 3445 * Adjust the reply queue count incase reply queue count 3446 * exceeds the user provided MSIx vectors count. 3447 */ 3448 if (local_max_msix_vectors > 0) 3449 ioc->reply_queue_count = min_t(int, local_max_msix_vectors, 3450 ioc->reply_queue_count); 3451 /* 3452 * Add io uring poll queues count to reply queues count 3453 * if io uring is enabled in driver. 3454 */ 3455 if (iopoll_q_count) { 3456 if (ioc->reply_queue_count < (iopoll_q_count + MPT3_MIN_IRQS)) 3457 iopoll_q_count = 0; 3458 ioc->reply_queue_count = min_t(int, 3459 ioc->reply_queue_count + iopoll_q_count, 3460 ioc->msix_vector_count); 3461 } 3462 3463 /* 3464 * Starting index of io uring poll queues in reply queue list. 3465 */ 3466 ioc->iopoll_q_start_index = 3467 ioc->reply_queue_count - iopoll_q_count; 3468 3469 r = _base_alloc_irq_vectors(ioc); 3470 if (r < 0) { 3471 ioc_info(ioc, "pci_alloc_irq_vectors failed (r=%d) !!!\n", r); 3472 goto try_ioapic; 3473 } 3474 3475 /* 3476 * Adjust the reply queue count if the allocated 3477 * MSIx vectors is less then the requested number 3478 * of MSIx vectors. 3479 */ 3480 if (r < ioc->iopoll_q_start_index) { 3481 ioc->reply_queue_count = r + iopoll_q_count; 3482 ioc->iopoll_q_start_index = 3483 ioc->reply_queue_count - iopoll_q_count; 3484 } 3485 3486 ioc->msix_enable = 1; 3487 for (i = 0; i < ioc->reply_queue_count; i++) { 3488 r = _base_request_irq(ioc, i); 3489 if (r) { 3490 mpt3sas_base_free_irq(ioc); 3491 mpt3sas_base_disable_msix(ioc); 3492 goto try_ioapic; 3493 } 3494 } 3495 3496 ioc_info(ioc, "High IOPs queues : %s\n", 3497 ioc->high_iops_queues ? "enabled" : "disabled"); 3498 3499 return 0; 3500 3501 /* failback to io_apic interrupt routing */ 3502 try_ioapic: 3503 ioc->high_iops_queues = 0; 3504 ioc_info(ioc, "High IOPs queues : disabled\n"); 3505 ioc->reply_queue_count = 1; 3506 ioc->iopoll_q_start_index = ioc->reply_queue_count - 0; 3507 r = pci_alloc_irq_vectors(ioc->pdev, 1, 1, PCI_IRQ_LEGACY); 3508 if (r < 0) { 3509 dfailprintk(ioc, 3510 ioc_info(ioc, "pci_alloc_irq_vector(legacy) failed (r=%d) !!!\n", 3511 r)); 3512 } else 3513 r = _base_request_irq(ioc, 0); 3514 3515 return r; 3516 } 3517 3518 /** 3519 * mpt3sas_base_unmap_resources - free controller resources 3520 * @ioc: per adapter object 3521 */ 3522 static void 3523 mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc) 3524 { 3525 struct pci_dev *pdev = ioc->pdev; 3526 3527 dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 3528 3529 mpt3sas_base_free_irq(ioc); 3530 mpt3sas_base_disable_msix(ioc); 3531 3532 kfree(ioc->replyPostRegisterIndex); 3533 ioc->replyPostRegisterIndex = NULL; 3534 3535 3536 if (ioc->chip_phys) { 3537 iounmap(ioc->chip); 3538 ioc->chip_phys = 0; 3539 } 3540 3541 if (pci_is_enabled(pdev)) { 3542 pci_release_selected_regions(ioc->pdev, ioc->bars); 3543 pci_disable_pcie_error_reporting(pdev); 3544 pci_disable_device(pdev); 3545 } 3546 } 3547 3548 static int 3549 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc); 3550 3551 /** 3552 * mpt3sas_base_check_for_fault_and_issue_reset - check if IOC is in fault state 3553 * and if it is in fault state then issue diag reset. 3554 * @ioc: per adapter object 3555 * 3556 * Return: 0 for success, non-zero for failure. 3557 */ 3558 int 3559 mpt3sas_base_check_for_fault_and_issue_reset(struct MPT3SAS_ADAPTER *ioc) 3560 { 3561 u32 ioc_state; 3562 int rc = -EFAULT; 3563 3564 dinitprintk(ioc, pr_info("%s\n", __func__)); 3565 if (ioc->pci_error_recovery) 3566 return 0; 3567 ioc_state = mpt3sas_base_get_iocstate(ioc, 0); 3568 dhsprintk(ioc, pr_info("%s: ioc_state(0x%08x)\n", __func__, ioc_state)); 3569 3570 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { 3571 mpt3sas_print_fault_code(ioc, ioc_state & 3572 MPI2_DOORBELL_DATA_MASK); 3573 mpt3sas_base_mask_interrupts(ioc); 3574 rc = _base_diag_reset(ioc); 3575 } else if ((ioc_state & MPI2_IOC_STATE_MASK) == 3576 MPI2_IOC_STATE_COREDUMP) { 3577 mpt3sas_print_coredump_info(ioc, ioc_state & 3578 MPI2_DOORBELL_DATA_MASK); 3579 mpt3sas_base_wait_for_coredump_completion(ioc, __func__); 3580 mpt3sas_base_mask_interrupts(ioc); 3581 rc = _base_diag_reset(ioc); 3582 } 3583 3584 return rc; 3585 } 3586 3587 /** 3588 * mpt3sas_base_map_resources - map in controller resources (io/irq/memap) 3589 * @ioc: per adapter object 3590 * 3591 * Return: 0 for success, non-zero for failure. 3592 */ 3593 int 3594 mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc) 3595 { 3596 struct pci_dev *pdev = ioc->pdev; 3597 u32 memap_sz; 3598 u32 pio_sz; 3599 int i, r = 0, rc; 3600 u64 pio_chip = 0; 3601 phys_addr_t chip_phys = 0; 3602 struct adapter_reply_queue *reply_q; 3603 int iopoll_q_count = 0; 3604 3605 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 3606 3607 ioc->bars = pci_select_bars(pdev, IORESOURCE_MEM); 3608 if (pci_enable_device_mem(pdev)) { 3609 ioc_warn(ioc, "pci_enable_device_mem: failed\n"); 3610 ioc->bars = 0; 3611 return -ENODEV; 3612 } 3613 3614 3615 if (pci_request_selected_regions(pdev, ioc->bars, 3616 ioc->driver_name)) { 3617 ioc_warn(ioc, "pci_request_selected_regions: failed\n"); 3618 ioc->bars = 0; 3619 r = -ENODEV; 3620 goto out_fail; 3621 } 3622 3623 /* AER (Advanced Error Reporting) hooks */ 3624 pci_enable_pcie_error_reporting(pdev); 3625 3626 pci_set_master(pdev); 3627 3628 3629 if (_base_config_dma_addressing(ioc, pdev) != 0) { 3630 ioc_warn(ioc, "no suitable DMA mask for %s\n", pci_name(pdev)); 3631 r = -ENODEV; 3632 goto out_fail; 3633 } 3634 3635 for (i = 0, memap_sz = 0, pio_sz = 0; (i < DEVICE_COUNT_RESOURCE) && 3636 (!memap_sz || !pio_sz); i++) { 3637 if (pci_resource_flags(pdev, i) & IORESOURCE_IO) { 3638 if (pio_sz) 3639 continue; 3640 pio_chip = (u64)pci_resource_start(pdev, i); 3641 pio_sz = pci_resource_len(pdev, i); 3642 } else if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) { 3643 if (memap_sz) 3644 continue; 3645 ioc->chip_phys = pci_resource_start(pdev, i); 3646 chip_phys = ioc->chip_phys; 3647 memap_sz = pci_resource_len(pdev, i); 3648 ioc->chip = ioremap(ioc->chip_phys, memap_sz); 3649 } 3650 } 3651 3652 if (ioc->chip == NULL) { 3653 ioc_err(ioc, 3654 "unable to map adapter memory! or resource not found\n"); 3655 r = -EINVAL; 3656 goto out_fail; 3657 } 3658 3659 mpt3sas_base_mask_interrupts(ioc); 3660 3661 r = _base_get_ioc_facts(ioc); 3662 if (r) { 3663 rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc); 3664 if (rc || (_base_get_ioc_facts(ioc))) 3665 goto out_fail; 3666 } 3667 3668 if (!ioc->rdpq_array_enable_assigned) { 3669 ioc->rdpq_array_enable = ioc->rdpq_array_capable; 3670 ioc->rdpq_array_enable_assigned = 1; 3671 } 3672 3673 r = _base_enable_msix(ioc); 3674 if (r) 3675 goto out_fail; 3676 3677 iopoll_q_count = ioc->reply_queue_count - ioc->iopoll_q_start_index; 3678 for (i = 0; i < iopoll_q_count; i++) { 3679 atomic_set(&ioc->io_uring_poll_queues[i].busy, 0); 3680 atomic_set(&ioc->io_uring_poll_queues[i].pause, 0); 3681 } 3682 3683 if (!ioc->is_driver_loading) 3684 _base_init_irqpolls(ioc); 3685 /* Use the Combined reply queue feature only for SAS3 C0 & higher 3686 * revision HBAs and also only when reply queue count is greater than 8 3687 */ 3688 if (ioc->combined_reply_queue) { 3689 /* Determine the Supplemental Reply Post Host Index Registers 3690 * Addresse. Supplemental Reply Post Host Index Registers 3691 * starts at offset MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET and 3692 * each register is at offset bytes of 3693 * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET from previous one. 3694 */ 3695 ioc->replyPostRegisterIndex = kcalloc( 3696 ioc->combined_reply_index_count, 3697 sizeof(resource_size_t *), GFP_KERNEL); 3698 if (!ioc->replyPostRegisterIndex) { 3699 ioc_err(ioc, 3700 "allocation for replyPostRegisterIndex failed!\n"); 3701 r = -ENOMEM; 3702 goto out_fail; 3703 } 3704 3705 for (i = 0; i < ioc->combined_reply_index_count; i++) { 3706 ioc->replyPostRegisterIndex[i] = (resource_size_t *) 3707 ((u8 __force *)&ioc->chip->Doorbell + 3708 MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET + 3709 (i * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET)); 3710 } 3711 } 3712 3713 if (ioc->is_warpdrive) { 3714 ioc->reply_post_host_index[0] = (resource_size_t __iomem *) 3715 &ioc->chip->ReplyPostHostIndex; 3716 3717 for (i = 1; i < ioc->cpu_msix_table_sz; i++) 3718 ioc->reply_post_host_index[i] = 3719 (resource_size_t __iomem *) 3720 ((u8 __iomem *)&ioc->chip->Doorbell + (0x4000 + ((i - 1) 3721 * 4))); 3722 } 3723 3724 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { 3725 if (reply_q->msix_index >= ioc->iopoll_q_start_index) { 3726 pr_info("%s: enabled: index: %d\n", 3727 reply_q->name, reply_q->msix_index); 3728 continue; 3729 } 3730 3731 pr_info("%s: %s enabled: IRQ %d\n", 3732 reply_q->name, 3733 ioc->msix_enable ? "PCI-MSI-X" : "IO-APIC", 3734 pci_irq_vector(ioc->pdev, reply_q->msix_index)); 3735 } 3736 3737 ioc_info(ioc, "iomem(%pap), mapped(0x%p), size(%d)\n", 3738 &chip_phys, ioc->chip, memap_sz); 3739 ioc_info(ioc, "ioport(0x%016llx), size(%d)\n", 3740 (unsigned long long)pio_chip, pio_sz); 3741 3742 /* Save PCI configuration state for recovery from PCI AER/EEH errors */ 3743 pci_save_state(pdev); 3744 return 0; 3745 3746 out_fail: 3747 mpt3sas_base_unmap_resources(ioc); 3748 return r; 3749 } 3750 3751 /** 3752 * mpt3sas_base_get_msg_frame - obtain request mf pointer 3753 * @ioc: per adapter object 3754 * @smid: system request message index(smid zero is invalid) 3755 * 3756 * Return: virt pointer to message frame. 3757 */ 3758 void * 3759 mpt3sas_base_get_msg_frame(struct MPT3SAS_ADAPTER *ioc, u16 smid) 3760 { 3761 return (void *)(ioc->request + (smid * ioc->request_sz)); 3762 } 3763 3764 /** 3765 * mpt3sas_base_get_sense_buffer - obtain a sense buffer virt addr 3766 * @ioc: per adapter object 3767 * @smid: system request message index 3768 * 3769 * Return: virt pointer to sense buffer. 3770 */ 3771 void * 3772 mpt3sas_base_get_sense_buffer(struct MPT3SAS_ADAPTER *ioc, u16 smid) 3773 { 3774 return (void *)(ioc->sense + ((smid - 1) * SCSI_SENSE_BUFFERSIZE)); 3775 } 3776 3777 /** 3778 * mpt3sas_base_get_sense_buffer_dma - obtain a sense buffer dma addr 3779 * @ioc: per adapter object 3780 * @smid: system request message index 3781 * 3782 * Return: phys pointer to the low 32bit address of the sense buffer. 3783 */ 3784 __le32 3785 mpt3sas_base_get_sense_buffer_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid) 3786 { 3787 return cpu_to_le32(ioc->sense_dma + ((smid - 1) * 3788 SCSI_SENSE_BUFFERSIZE)); 3789 } 3790 3791 /** 3792 * mpt3sas_base_get_pcie_sgl - obtain a PCIe SGL virt addr 3793 * @ioc: per adapter object 3794 * @smid: system request message index 3795 * 3796 * Return: virt pointer to a PCIe SGL. 3797 */ 3798 void * 3799 mpt3sas_base_get_pcie_sgl(struct MPT3SAS_ADAPTER *ioc, u16 smid) 3800 { 3801 return (void *)(ioc->pcie_sg_lookup[smid - 1].pcie_sgl); 3802 } 3803 3804 /** 3805 * mpt3sas_base_get_pcie_sgl_dma - obtain a PCIe SGL dma addr 3806 * @ioc: per adapter object 3807 * @smid: system request message index 3808 * 3809 * Return: phys pointer to the address of the PCIe buffer. 3810 */ 3811 dma_addr_t 3812 mpt3sas_base_get_pcie_sgl_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid) 3813 { 3814 return ioc->pcie_sg_lookup[smid - 1].pcie_sgl_dma; 3815 } 3816 3817 /** 3818 * mpt3sas_base_get_reply_virt_addr - obtain reply frames virt address 3819 * @ioc: per adapter object 3820 * @phys_addr: lower 32 physical addr of the reply 3821 * 3822 * Converts 32bit lower physical addr into a virt address. 3823 */ 3824 void * 3825 mpt3sas_base_get_reply_virt_addr(struct MPT3SAS_ADAPTER *ioc, u32 phys_addr) 3826 { 3827 if (!phys_addr) 3828 return NULL; 3829 return ioc->reply + (phys_addr - (u32)ioc->reply_dma); 3830 } 3831 3832 /** 3833 * _base_get_msix_index - get the msix index 3834 * @ioc: per adapter object 3835 * @scmd: scsi_cmnd object 3836 * 3837 * Return: msix index of general reply queues, 3838 * i.e. reply queue on which IO request's reply 3839 * should be posted by the HBA firmware. 3840 */ 3841 static inline u8 3842 _base_get_msix_index(struct MPT3SAS_ADAPTER *ioc, 3843 struct scsi_cmnd *scmd) 3844 { 3845 /* Enables reply_queue load balancing */ 3846 if (ioc->msix_load_balance) 3847 return ioc->reply_queue_count ? 3848 base_mod64(atomic64_add_return(1, 3849 &ioc->total_io_cnt), ioc->reply_queue_count) : 0; 3850 3851 if (scmd && ioc->shost->nr_hw_queues > 1) { 3852 u32 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd)); 3853 3854 return blk_mq_unique_tag_to_hwq(tag) + 3855 ioc->high_iops_queues; 3856 } 3857 3858 return ioc->cpu_msix_table[raw_smp_processor_id()]; 3859 } 3860 3861 /** 3862 * _base_get_high_iops_msix_index - get the msix index of 3863 * high iops queues 3864 * @ioc: per adapter object 3865 * @scmd: scsi_cmnd object 3866 * 3867 * Return: msix index of high iops reply queues. 3868 * i.e. high iops reply queue on which IO request's 3869 * reply should be posted by the HBA firmware. 3870 */ 3871 static inline u8 3872 _base_get_high_iops_msix_index(struct MPT3SAS_ADAPTER *ioc, 3873 struct scsi_cmnd *scmd) 3874 { 3875 /** 3876 * Round robin the IO interrupts among the high iops 3877 * reply queues in terms of batch count 16 when outstanding 3878 * IOs on the target device is >=8. 3879 */ 3880 3881 if (scsi_device_busy(scmd->device) > MPT3SAS_DEVICE_HIGH_IOPS_DEPTH) 3882 return base_mod64(( 3883 atomic64_add_return(1, &ioc->high_iops_outstanding) / 3884 MPT3SAS_HIGH_IOPS_BATCH_COUNT), 3885 MPT3SAS_HIGH_IOPS_REPLY_QUEUES); 3886 3887 return _base_get_msix_index(ioc, scmd); 3888 } 3889 3890 /** 3891 * mpt3sas_base_get_smid - obtain a free smid from internal queue 3892 * @ioc: per adapter object 3893 * @cb_idx: callback index 3894 * 3895 * Return: smid (zero is invalid) 3896 */ 3897 u16 3898 mpt3sas_base_get_smid(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx) 3899 { 3900 unsigned long flags; 3901 struct request_tracker *request; 3902 u16 smid; 3903 3904 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags); 3905 if (list_empty(&ioc->internal_free_list)) { 3906 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags); 3907 ioc_err(ioc, "%s: smid not available\n", __func__); 3908 return 0; 3909 } 3910 3911 request = list_entry(ioc->internal_free_list.next, 3912 struct request_tracker, tracker_list); 3913 request->cb_idx = cb_idx; 3914 smid = request->smid; 3915 list_del(&request->tracker_list); 3916 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags); 3917 return smid; 3918 } 3919 3920 /** 3921 * mpt3sas_base_get_smid_scsiio - obtain a free smid from scsiio queue 3922 * @ioc: per adapter object 3923 * @cb_idx: callback index 3924 * @scmd: pointer to scsi command object 3925 * 3926 * Return: smid (zero is invalid) 3927 */ 3928 u16 3929 mpt3sas_base_get_smid_scsiio(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx, 3930 struct scsi_cmnd *scmd) 3931 { 3932 struct scsiio_tracker *request = scsi_cmd_priv(scmd); 3933 u16 smid; 3934 u32 tag, unique_tag; 3935 3936 unique_tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd)); 3937 tag = blk_mq_unique_tag_to_tag(unique_tag); 3938 3939 /* 3940 * Store hw queue number corresponding to the tag. 3941 * This hw queue number is used later to determine 3942 * the unique_tag using the logic below. This unique_tag 3943 * is used to retrieve the scmd pointer corresponding 3944 * to tag using scsi_host_find_tag() API. 3945 * 3946 * tag = smid - 1; 3947 * unique_tag = ioc->io_queue_num[tag] << BLK_MQ_UNIQUE_TAG_BITS | tag; 3948 */ 3949 ioc->io_queue_num[tag] = blk_mq_unique_tag_to_hwq(unique_tag); 3950 3951 smid = tag + 1; 3952 request->cb_idx = cb_idx; 3953 request->smid = smid; 3954 request->scmd = scmd; 3955 INIT_LIST_HEAD(&request->chain_list); 3956 return smid; 3957 } 3958 3959 /** 3960 * mpt3sas_base_get_smid_hpr - obtain a free smid from hi-priority queue 3961 * @ioc: per adapter object 3962 * @cb_idx: callback index 3963 * 3964 * Return: smid (zero is invalid) 3965 */ 3966 u16 3967 mpt3sas_base_get_smid_hpr(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx) 3968 { 3969 unsigned long flags; 3970 struct request_tracker *request; 3971 u16 smid; 3972 3973 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags); 3974 if (list_empty(&ioc->hpr_free_list)) { 3975 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags); 3976 return 0; 3977 } 3978 3979 request = list_entry(ioc->hpr_free_list.next, 3980 struct request_tracker, tracker_list); 3981 request->cb_idx = cb_idx; 3982 smid = request->smid; 3983 list_del(&request->tracker_list); 3984 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags); 3985 return smid; 3986 } 3987 3988 static void 3989 _base_recovery_check(struct MPT3SAS_ADAPTER *ioc) 3990 { 3991 /* 3992 * See _wait_for_commands_to_complete() call with regards to this code. 3993 */ 3994 if (ioc->shost_recovery && ioc->pending_io_count) { 3995 ioc->pending_io_count = scsi_host_busy(ioc->shost); 3996 if (ioc->pending_io_count == 0) 3997 wake_up(&ioc->reset_wq); 3998 } 3999 } 4000 4001 void mpt3sas_base_clear_st(struct MPT3SAS_ADAPTER *ioc, 4002 struct scsiio_tracker *st) 4003 { 4004 if (WARN_ON(st->smid == 0)) 4005 return; 4006 st->cb_idx = 0xFF; 4007 st->direct_io = 0; 4008 st->scmd = NULL; 4009 atomic_set(&ioc->chain_lookup[st->smid - 1].chain_offset, 0); 4010 st->smid = 0; 4011 } 4012 4013 /** 4014 * mpt3sas_base_free_smid - put smid back on free_list 4015 * @ioc: per adapter object 4016 * @smid: system request message index 4017 */ 4018 void 4019 mpt3sas_base_free_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid) 4020 { 4021 unsigned long flags; 4022 int i; 4023 4024 if (smid < ioc->hi_priority_smid) { 4025 struct scsiio_tracker *st; 4026 void *request; 4027 4028 st = _get_st_from_smid(ioc, smid); 4029 if (!st) { 4030 _base_recovery_check(ioc); 4031 return; 4032 } 4033 4034 /* Clear MPI request frame */ 4035 request = mpt3sas_base_get_msg_frame(ioc, smid); 4036 memset(request, 0, ioc->request_sz); 4037 4038 mpt3sas_base_clear_st(ioc, st); 4039 _base_recovery_check(ioc); 4040 ioc->io_queue_num[smid - 1] = 0; 4041 return; 4042 } 4043 4044 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags); 4045 if (smid < ioc->internal_smid) { 4046 /* hi-priority */ 4047 i = smid - ioc->hi_priority_smid; 4048 ioc->hpr_lookup[i].cb_idx = 0xFF; 4049 list_add(&ioc->hpr_lookup[i].tracker_list, &ioc->hpr_free_list); 4050 } else if (smid <= ioc->hba_queue_depth) { 4051 /* internal queue */ 4052 i = smid - ioc->internal_smid; 4053 ioc->internal_lookup[i].cb_idx = 0xFF; 4054 list_add(&ioc->internal_lookup[i].tracker_list, 4055 &ioc->internal_free_list); 4056 } 4057 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags); 4058 } 4059 4060 /** 4061 * _base_mpi_ep_writeq - 32 bit write to MMIO 4062 * @b: data payload 4063 * @addr: address in MMIO space 4064 * @writeq_lock: spin lock 4065 * 4066 * This special handling for MPI EP to take care of 32 bit 4067 * environment where its not quarenteed to send the entire word 4068 * in one transfer. 4069 */ 4070 static inline void 4071 _base_mpi_ep_writeq(__u64 b, volatile void __iomem *addr, 4072 spinlock_t *writeq_lock) 4073 { 4074 unsigned long flags; 4075 4076 spin_lock_irqsave(writeq_lock, flags); 4077 __raw_writel((u32)(b), addr); 4078 __raw_writel((u32)(b >> 32), (addr + 4)); 4079 spin_unlock_irqrestore(writeq_lock, flags); 4080 } 4081 4082 /** 4083 * _base_writeq - 64 bit write to MMIO 4084 * @b: data payload 4085 * @addr: address in MMIO space 4086 * @writeq_lock: spin lock 4087 * 4088 * Glue for handling an atomic 64 bit word to MMIO. This special handling takes 4089 * care of 32 bit environment where its not quarenteed to send the entire word 4090 * in one transfer. 4091 */ 4092 #if defined(writeq) && defined(CONFIG_64BIT) 4093 static inline void 4094 _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock) 4095 { 4096 wmb(); 4097 __raw_writeq(b, addr); 4098 barrier(); 4099 } 4100 #else 4101 static inline void 4102 _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock) 4103 { 4104 _base_mpi_ep_writeq(b, addr, writeq_lock); 4105 } 4106 #endif 4107 4108 /** 4109 * _base_set_and_get_msix_index - get the msix index and assign to msix_io 4110 * variable of scsi tracker 4111 * @ioc: per adapter object 4112 * @smid: system request message index 4113 * 4114 * Return: msix index. 4115 */ 4116 static u8 4117 _base_set_and_get_msix_index(struct MPT3SAS_ADAPTER *ioc, u16 smid) 4118 { 4119 struct scsiio_tracker *st = NULL; 4120 4121 if (smid < ioc->hi_priority_smid) 4122 st = _get_st_from_smid(ioc, smid); 4123 4124 if (st == NULL) 4125 return _base_get_msix_index(ioc, NULL); 4126 4127 st->msix_io = ioc->get_msix_index_for_smlio(ioc, st->scmd); 4128 return st->msix_io; 4129 } 4130 4131 /** 4132 * _base_put_smid_mpi_ep_scsi_io - send SCSI_IO request to firmware 4133 * @ioc: per adapter object 4134 * @smid: system request message index 4135 * @handle: device handle 4136 */ 4137 static void 4138 _base_put_smid_mpi_ep_scsi_io(struct MPT3SAS_ADAPTER *ioc, 4139 u16 smid, u16 handle) 4140 { 4141 Mpi2RequestDescriptorUnion_t descriptor; 4142 u64 *request = (u64 *)&descriptor; 4143 void *mpi_req_iomem; 4144 __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid); 4145 4146 _clone_sg_entries(ioc, (void *) mfp, smid); 4147 mpi_req_iomem = (void __force *)ioc->chip + 4148 MPI_FRAME_START_OFFSET + (smid * ioc->request_sz); 4149 _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp, 4150 ioc->request_sz); 4151 descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO; 4152 descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4153 descriptor.SCSIIO.SMID = cpu_to_le16(smid); 4154 descriptor.SCSIIO.DevHandle = cpu_to_le16(handle); 4155 descriptor.SCSIIO.LMID = 0; 4156 _base_mpi_ep_writeq(*request, &ioc->chip->RequestDescriptorPostLow, 4157 &ioc->scsi_lookup_lock); 4158 } 4159 4160 /** 4161 * _base_put_smid_scsi_io - send SCSI_IO request to firmware 4162 * @ioc: per adapter object 4163 * @smid: system request message index 4164 * @handle: device handle 4165 */ 4166 static void 4167 _base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle) 4168 { 4169 Mpi2RequestDescriptorUnion_t descriptor; 4170 u64 *request = (u64 *)&descriptor; 4171 4172 4173 descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO; 4174 descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4175 descriptor.SCSIIO.SMID = cpu_to_le16(smid); 4176 descriptor.SCSIIO.DevHandle = cpu_to_le16(handle); 4177 descriptor.SCSIIO.LMID = 0; 4178 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow, 4179 &ioc->scsi_lookup_lock); 4180 } 4181 4182 /** 4183 * _base_put_smid_fast_path - send fast path request to firmware 4184 * @ioc: per adapter object 4185 * @smid: system request message index 4186 * @handle: device handle 4187 */ 4188 static void 4189 _base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid, 4190 u16 handle) 4191 { 4192 Mpi2RequestDescriptorUnion_t descriptor; 4193 u64 *request = (u64 *)&descriptor; 4194 4195 descriptor.SCSIIO.RequestFlags = 4196 MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO; 4197 descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4198 descriptor.SCSIIO.SMID = cpu_to_le16(smid); 4199 descriptor.SCSIIO.DevHandle = cpu_to_le16(handle); 4200 descriptor.SCSIIO.LMID = 0; 4201 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow, 4202 &ioc->scsi_lookup_lock); 4203 } 4204 4205 /** 4206 * _base_put_smid_hi_priority - send Task Management request to firmware 4207 * @ioc: per adapter object 4208 * @smid: system request message index 4209 * @msix_task: msix_task will be same as msix of IO in case of task abort else 0 4210 */ 4211 static void 4212 _base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid, 4213 u16 msix_task) 4214 { 4215 Mpi2RequestDescriptorUnion_t descriptor; 4216 void *mpi_req_iomem; 4217 u64 *request; 4218 4219 if (ioc->is_mcpu_endpoint) { 4220 __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid); 4221 4222 /* TBD 256 is offset within sys register. */ 4223 mpi_req_iomem = (void __force *)ioc->chip 4224 + MPI_FRAME_START_OFFSET 4225 + (smid * ioc->request_sz); 4226 _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp, 4227 ioc->request_sz); 4228 } 4229 4230 request = (u64 *)&descriptor; 4231 4232 descriptor.HighPriority.RequestFlags = 4233 MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY; 4234 descriptor.HighPriority.MSIxIndex = msix_task; 4235 descriptor.HighPriority.SMID = cpu_to_le16(smid); 4236 descriptor.HighPriority.LMID = 0; 4237 descriptor.HighPriority.Reserved1 = 0; 4238 if (ioc->is_mcpu_endpoint) 4239 _base_mpi_ep_writeq(*request, 4240 &ioc->chip->RequestDescriptorPostLow, 4241 &ioc->scsi_lookup_lock); 4242 else 4243 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow, 4244 &ioc->scsi_lookup_lock); 4245 } 4246 4247 /** 4248 * mpt3sas_base_put_smid_nvme_encap - send NVMe encapsulated request to 4249 * firmware 4250 * @ioc: per adapter object 4251 * @smid: system request message index 4252 */ 4253 void 4254 mpt3sas_base_put_smid_nvme_encap(struct MPT3SAS_ADAPTER *ioc, u16 smid) 4255 { 4256 Mpi2RequestDescriptorUnion_t descriptor; 4257 u64 *request = (u64 *)&descriptor; 4258 4259 descriptor.Default.RequestFlags = 4260 MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED; 4261 descriptor.Default.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4262 descriptor.Default.SMID = cpu_to_le16(smid); 4263 descriptor.Default.LMID = 0; 4264 descriptor.Default.DescriptorTypeDependent = 0; 4265 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow, 4266 &ioc->scsi_lookup_lock); 4267 } 4268 4269 /** 4270 * _base_put_smid_default - Default, primarily used for config pages 4271 * @ioc: per adapter object 4272 * @smid: system request message index 4273 */ 4274 static void 4275 _base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid) 4276 { 4277 Mpi2RequestDescriptorUnion_t descriptor; 4278 void *mpi_req_iomem; 4279 u64 *request; 4280 4281 if (ioc->is_mcpu_endpoint) { 4282 __le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid); 4283 4284 _clone_sg_entries(ioc, (void *) mfp, smid); 4285 /* TBD 256 is offset within sys register */ 4286 mpi_req_iomem = (void __force *)ioc->chip + 4287 MPI_FRAME_START_OFFSET + (smid * ioc->request_sz); 4288 _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp, 4289 ioc->request_sz); 4290 } 4291 request = (u64 *)&descriptor; 4292 descriptor.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; 4293 descriptor.Default.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4294 descriptor.Default.SMID = cpu_to_le16(smid); 4295 descriptor.Default.LMID = 0; 4296 descriptor.Default.DescriptorTypeDependent = 0; 4297 if (ioc->is_mcpu_endpoint) 4298 _base_mpi_ep_writeq(*request, 4299 &ioc->chip->RequestDescriptorPostLow, 4300 &ioc->scsi_lookup_lock); 4301 else 4302 _base_writeq(*request, &ioc->chip->RequestDescriptorPostLow, 4303 &ioc->scsi_lookup_lock); 4304 } 4305 4306 /** 4307 * _base_put_smid_scsi_io_atomic - send SCSI_IO request to firmware using 4308 * Atomic Request Descriptor 4309 * @ioc: per adapter object 4310 * @smid: system request message index 4311 * @handle: device handle, unused in this function, for function type match 4312 * 4313 * Return: nothing. 4314 */ 4315 static void 4316 _base_put_smid_scsi_io_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid, 4317 u16 handle) 4318 { 4319 Mpi26AtomicRequestDescriptor_t descriptor; 4320 u32 *request = (u32 *)&descriptor; 4321 4322 descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO; 4323 descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4324 descriptor.SMID = cpu_to_le16(smid); 4325 4326 writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); 4327 } 4328 4329 /** 4330 * _base_put_smid_fast_path_atomic - send fast path request to firmware 4331 * using Atomic Request Descriptor 4332 * @ioc: per adapter object 4333 * @smid: system request message index 4334 * @handle: device handle, unused in this function, for function type match 4335 * Return: nothing 4336 */ 4337 static void 4338 _base_put_smid_fast_path_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid, 4339 u16 handle) 4340 { 4341 Mpi26AtomicRequestDescriptor_t descriptor; 4342 u32 *request = (u32 *)&descriptor; 4343 4344 descriptor.RequestFlags = MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO; 4345 descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4346 descriptor.SMID = cpu_to_le16(smid); 4347 4348 writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); 4349 } 4350 4351 /** 4352 * _base_put_smid_hi_priority_atomic - send Task Management request to 4353 * firmware using Atomic Request Descriptor 4354 * @ioc: per adapter object 4355 * @smid: system request message index 4356 * @msix_task: msix_task will be same as msix of IO in case of task abort else 0 4357 * 4358 * Return: nothing. 4359 */ 4360 static void 4361 _base_put_smid_hi_priority_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid, 4362 u16 msix_task) 4363 { 4364 Mpi26AtomicRequestDescriptor_t descriptor; 4365 u32 *request = (u32 *)&descriptor; 4366 4367 descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY; 4368 descriptor.MSIxIndex = msix_task; 4369 descriptor.SMID = cpu_to_le16(smid); 4370 4371 writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); 4372 } 4373 4374 /** 4375 * _base_put_smid_default_atomic - Default, primarily used for config pages 4376 * use Atomic Request Descriptor 4377 * @ioc: per adapter object 4378 * @smid: system request message index 4379 * 4380 * Return: nothing. 4381 */ 4382 static void 4383 _base_put_smid_default_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid) 4384 { 4385 Mpi26AtomicRequestDescriptor_t descriptor; 4386 u32 *request = (u32 *)&descriptor; 4387 4388 descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; 4389 descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); 4390 descriptor.SMID = cpu_to_le16(smid); 4391 4392 writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); 4393 } 4394 4395 /** 4396 * _base_display_OEMs_branding - Display branding string 4397 * @ioc: per adapter object 4398 */ 4399 static void 4400 _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc) 4401 { 4402 if (ioc->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL) 4403 return; 4404 4405 switch (ioc->pdev->subsystem_vendor) { 4406 case PCI_VENDOR_ID_INTEL: 4407 switch (ioc->pdev->device) { 4408 case MPI2_MFGPAGE_DEVID_SAS2008: 4409 switch (ioc->pdev->subsystem_device) { 4410 case MPT2SAS_INTEL_RMS2LL080_SSDID: 4411 ioc_info(ioc, "%s\n", 4412 MPT2SAS_INTEL_RMS2LL080_BRANDING); 4413 break; 4414 case MPT2SAS_INTEL_RMS2LL040_SSDID: 4415 ioc_info(ioc, "%s\n", 4416 MPT2SAS_INTEL_RMS2LL040_BRANDING); 4417 break; 4418 case MPT2SAS_INTEL_SSD910_SSDID: 4419 ioc_info(ioc, "%s\n", 4420 MPT2SAS_INTEL_SSD910_BRANDING); 4421 break; 4422 default: 4423 ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n", 4424 ioc->pdev->subsystem_device); 4425 break; 4426 } 4427 break; 4428 case MPI2_MFGPAGE_DEVID_SAS2308_2: 4429 switch (ioc->pdev->subsystem_device) { 4430 case MPT2SAS_INTEL_RS25GB008_SSDID: 4431 ioc_info(ioc, "%s\n", 4432 MPT2SAS_INTEL_RS25GB008_BRANDING); 4433 break; 4434 case MPT2SAS_INTEL_RMS25JB080_SSDID: 4435 ioc_info(ioc, "%s\n", 4436 MPT2SAS_INTEL_RMS25JB080_BRANDING); 4437 break; 4438 case MPT2SAS_INTEL_RMS25JB040_SSDID: 4439 ioc_info(ioc, "%s\n", 4440 MPT2SAS_INTEL_RMS25JB040_BRANDING); 4441 break; 4442 case MPT2SAS_INTEL_RMS25KB080_SSDID: 4443 ioc_info(ioc, "%s\n", 4444 MPT2SAS_INTEL_RMS25KB080_BRANDING); 4445 break; 4446 case MPT2SAS_INTEL_RMS25KB040_SSDID: 4447 ioc_info(ioc, "%s\n", 4448 MPT2SAS_INTEL_RMS25KB040_BRANDING); 4449 break; 4450 case MPT2SAS_INTEL_RMS25LB040_SSDID: 4451 ioc_info(ioc, "%s\n", 4452 MPT2SAS_INTEL_RMS25LB040_BRANDING); 4453 break; 4454 case MPT2SAS_INTEL_RMS25LB080_SSDID: 4455 ioc_info(ioc, "%s\n", 4456 MPT2SAS_INTEL_RMS25LB080_BRANDING); 4457 break; 4458 default: 4459 ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n", 4460 ioc->pdev->subsystem_device); 4461 break; 4462 } 4463 break; 4464 case MPI25_MFGPAGE_DEVID_SAS3008: 4465 switch (ioc->pdev->subsystem_device) { 4466 case MPT3SAS_INTEL_RMS3JC080_SSDID: 4467 ioc_info(ioc, "%s\n", 4468 MPT3SAS_INTEL_RMS3JC080_BRANDING); 4469 break; 4470 4471 case MPT3SAS_INTEL_RS3GC008_SSDID: 4472 ioc_info(ioc, "%s\n", 4473 MPT3SAS_INTEL_RS3GC008_BRANDING); 4474 break; 4475 case MPT3SAS_INTEL_RS3FC044_SSDID: 4476 ioc_info(ioc, "%s\n", 4477 MPT3SAS_INTEL_RS3FC044_BRANDING); 4478 break; 4479 case MPT3SAS_INTEL_RS3UC080_SSDID: 4480 ioc_info(ioc, "%s\n", 4481 MPT3SAS_INTEL_RS3UC080_BRANDING); 4482 break; 4483 default: 4484 ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n", 4485 ioc->pdev->subsystem_device); 4486 break; 4487 } 4488 break; 4489 default: 4490 ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n", 4491 ioc->pdev->subsystem_device); 4492 break; 4493 } 4494 break; 4495 case PCI_VENDOR_ID_DELL: 4496 switch (ioc->pdev->device) { 4497 case MPI2_MFGPAGE_DEVID_SAS2008: 4498 switch (ioc->pdev->subsystem_device) { 4499 case MPT2SAS_DELL_6GBPS_SAS_HBA_SSDID: 4500 ioc_info(ioc, "%s\n", 4501 MPT2SAS_DELL_6GBPS_SAS_HBA_BRANDING); 4502 break; 4503 case MPT2SAS_DELL_PERC_H200_ADAPTER_SSDID: 4504 ioc_info(ioc, "%s\n", 4505 MPT2SAS_DELL_PERC_H200_ADAPTER_BRANDING); 4506 break; 4507 case MPT2SAS_DELL_PERC_H200_INTEGRATED_SSDID: 4508 ioc_info(ioc, "%s\n", 4509 MPT2SAS_DELL_PERC_H200_INTEGRATED_BRANDING); 4510 break; 4511 case MPT2SAS_DELL_PERC_H200_MODULAR_SSDID: 4512 ioc_info(ioc, "%s\n", 4513 MPT2SAS_DELL_PERC_H200_MODULAR_BRANDING); 4514 break; 4515 case MPT2SAS_DELL_PERC_H200_EMBEDDED_SSDID: 4516 ioc_info(ioc, "%s\n", 4517 MPT2SAS_DELL_PERC_H200_EMBEDDED_BRANDING); 4518 break; 4519 case MPT2SAS_DELL_PERC_H200_SSDID: 4520 ioc_info(ioc, "%s\n", 4521 MPT2SAS_DELL_PERC_H200_BRANDING); 4522 break; 4523 case MPT2SAS_DELL_6GBPS_SAS_SSDID: 4524 ioc_info(ioc, "%s\n", 4525 MPT2SAS_DELL_6GBPS_SAS_BRANDING); 4526 break; 4527 default: 4528 ioc_info(ioc, "Dell 6Gbps HBA: Subsystem ID: 0x%X\n", 4529 ioc->pdev->subsystem_device); 4530 break; 4531 } 4532 break; 4533 case MPI25_MFGPAGE_DEVID_SAS3008: 4534 switch (ioc->pdev->subsystem_device) { 4535 case MPT3SAS_DELL_12G_HBA_SSDID: 4536 ioc_info(ioc, "%s\n", 4537 MPT3SAS_DELL_12G_HBA_BRANDING); 4538 break; 4539 default: 4540 ioc_info(ioc, "Dell 12Gbps HBA: Subsystem ID: 0x%X\n", 4541 ioc->pdev->subsystem_device); 4542 break; 4543 } 4544 break; 4545 default: 4546 ioc_info(ioc, "Dell HBA: Subsystem ID: 0x%X\n", 4547 ioc->pdev->subsystem_device); 4548 break; 4549 } 4550 break; 4551 case PCI_VENDOR_ID_CISCO: 4552 switch (ioc->pdev->device) { 4553 case MPI25_MFGPAGE_DEVID_SAS3008: 4554 switch (ioc->pdev->subsystem_device) { 4555 case MPT3SAS_CISCO_12G_8E_HBA_SSDID: 4556 ioc_info(ioc, "%s\n", 4557 MPT3SAS_CISCO_12G_8E_HBA_BRANDING); 4558 break; 4559 case MPT3SAS_CISCO_12G_8I_HBA_SSDID: 4560 ioc_info(ioc, "%s\n", 4561 MPT3SAS_CISCO_12G_8I_HBA_BRANDING); 4562 break; 4563 case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID: 4564 ioc_info(ioc, "%s\n", 4565 MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING); 4566 break; 4567 default: 4568 ioc_info(ioc, "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n", 4569 ioc->pdev->subsystem_device); 4570 break; 4571 } 4572 break; 4573 case MPI25_MFGPAGE_DEVID_SAS3108_1: 4574 switch (ioc->pdev->subsystem_device) { 4575 case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID: 4576 ioc_info(ioc, "%s\n", 4577 MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING); 4578 break; 4579 case MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_SSDID: 4580 ioc_info(ioc, "%s\n", 4581 MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_BRANDING); 4582 break; 4583 default: 4584 ioc_info(ioc, "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n", 4585 ioc->pdev->subsystem_device); 4586 break; 4587 } 4588 break; 4589 default: 4590 ioc_info(ioc, "Cisco SAS HBA: Subsystem ID: 0x%X\n", 4591 ioc->pdev->subsystem_device); 4592 break; 4593 } 4594 break; 4595 case MPT2SAS_HP_3PAR_SSVID: 4596 switch (ioc->pdev->device) { 4597 case MPI2_MFGPAGE_DEVID_SAS2004: 4598 switch (ioc->pdev->subsystem_device) { 4599 case MPT2SAS_HP_DAUGHTER_2_4_INTERNAL_SSDID: 4600 ioc_info(ioc, "%s\n", 4601 MPT2SAS_HP_DAUGHTER_2_4_INTERNAL_BRANDING); 4602 break; 4603 default: 4604 ioc_info(ioc, "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n", 4605 ioc->pdev->subsystem_device); 4606 break; 4607 } 4608 break; 4609 case MPI2_MFGPAGE_DEVID_SAS2308_2: 4610 switch (ioc->pdev->subsystem_device) { 4611 case MPT2SAS_HP_2_4_INTERNAL_SSDID: 4612 ioc_info(ioc, "%s\n", 4613 MPT2SAS_HP_2_4_INTERNAL_BRANDING); 4614 break; 4615 case MPT2SAS_HP_2_4_EXTERNAL_SSDID: 4616 ioc_info(ioc, "%s\n", 4617 MPT2SAS_HP_2_4_EXTERNAL_BRANDING); 4618 break; 4619 case MPT2SAS_HP_1_4_INTERNAL_1_4_EXTERNAL_SSDID: 4620 ioc_info(ioc, "%s\n", 4621 MPT2SAS_HP_1_4_INTERNAL_1_4_EXTERNAL_BRANDING); 4622 break; 4623 case MPT2SAS_HP_EMBEDDED_2_4_INTERNAL_SSDID: 4624 ioc_info(ioc, "%s\n", 4625 MPT2SAS_HP_EMBEDDED_2_4_INTERNAL_BRANDING); 4626 break; 4627 default: 4628 ioc_info(ioc, "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n", 4629 ioc->pdev->subsystem_device); 4630 break; 4631 } 4632 break; 4633 default: 4634 ioc_info(ioc, "HP SAS HBA: Subsystem ID: 0x%X\n", 4635 ioc->pdev->subsystem_device); 4636 break; 4637 } 4638 break; 4639 default: 4640 break; 4641 } 4642 } 4643 4644 /** 4645 * _base_display_fwpkg_version - sends FWUpload request to pull FWPkg 4646 * version from FW Image Header. 4647 * @ioc: per adapter object 4648 * 4649 * Return: 0 for success, non-zero for failure. 4650 */ 4651 static int 4652 _base_display_fwpkg_version(struct MPT3SAS_ADAPTER *ioc) 4653 { 4654 Mpi2FWImageHeader_t *fw_img_hdr; 4655 Mpi26ComponentImageHeader_t *cmp_img_hdr; 4656 Mpi25FWUploadRequest_t *mpi_request; 4657 Mpi2FWUploadReply_t mpi_reply; 4658 int r = 0, issue_diag_reset = 0; 4659 u32 package_version = 0; 4660 void *fwpkg_data = NULL; 4661 dma_addr_t fwpkg_data_dma; 4662 u16 smid, ioc_status; 4663 size_t data_length; 4664 4665 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 4666 4667 if (ioc->base_cmds.status & MPT3_CMD_PENDING) { 4668 ioc_err(ioc, "%s: internal command already in use\n", __func__); 4669 return -EAGAIN; 4670 } 4671 4672 data_length = sizeof(Mpi2FWImageHeader_t); 4673 fwpkg_data = dma_alloc_coherent(&ioc->pdev->dev, data_length, 4674 &fwpkg_data_dma, GFP_KERNEL); 4675 if (!fwpkg_data) { 4676 ioc_err(ioc, 4677 "Memory allocation for fwpkg data failed at %s:%d/%s()!\n", 4678 __FILE__, __LINE__, __func__); 4679 return -ENOMEM; 4680 } 4681 4682 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx); 4683 if (!smid) { 4684 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__); 4685 r = -EAGAIN; 4686 goto out; 4687 } 4688 4689 ioc->base_cmds.status = MPT3_CMD_PENDING; 4690 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); 4691 ioc->base_cmds.smid = smid; 4692 memset(mpi_request, 0, sizeof(Mpi25FWUploadRequest_t)); 4693 mpi_request->Function = MPI2_FUNCTION_FW_UPLOAD; 4694 mpi_request->ImageType = MPI2_FW_UPLOAD_ITYPE_FW_FLASH; 4695 mpi_request->ImageSize = cpu_to_le32(data_length); 4696 ioc->build_sg(ioc, &mpi_request->SGL, 0, 0, fwpkg_data_dma, 4697 data_length); 4698 init_completion(&ioc->base_cmds.done); 4699 ioc->put_smid_default(ioc, smid); 4700 /* Wait for 15 seconds */ 4701 wait_for_completion_timeout(&ioc->base_cmds.done, 4702 FW_IMG_HDR_READ_TIMEOUT*HZ); 4703 ioc_info(ioc, "%s: complete\n", __func__); 4704 if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) { 4705 ioc_err(ioc, "%s: timeout\n", __func__); 4706 _debug_dump_mf(mpi_request, 4707 sizeof(Mpi25FWUploadRequest_t)/4); 4708 issue_diag_reset = 1; 4709 } else { 4710 memset(&mpi_reply, 0, sizeof(Mpi2FWUploadReply_t)); 4711 if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) { 4712 memcpy(&mpi_reply, ioc->base_cmds.reply, 4713 sizeof(Mpi2FWUploadReply_t)); 4714 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & 4715 MPI2_IOCSTATUS_MASK; 4716 if (ioc_status == MPI2_IOCSTATUS_SUCCESS) { 4717 fw_img_hdr = (Mpi2FWImageHeader_t *)fwpkg_data; 4718 if (le32_to_cpu(fw_img_hdr->Signature) == 4719 MPI26_IMAGE_HEADER_SIGNATURE0_MPI26) { 4720 cmp_img_hdr = 4721 (Mpi26ComponentImageHeader_t *) 4722 (fwpkg_data); 4723 package_version = 4724 le32_to_cpu( 4725 cmp_img_hdr->ApplicationSpecific); 4726 } else 4727 package_version = 4728 le32_to_cpu( 4729 fw_img_hdr->PackageVersion.Word); 4730 if (package_version) 4731 ioc_info(ioc, 4732 "FW Package Ver(%02d.%02d.%02d.%02d)\n", 4733 ((package_version) & 0xFF000000) >> 24, 4734 ((package_version) & 0x00FF0000) >> 16, 4735 ((package_version) & 0x0000FF00) >> 8, 4736 (package_version) & 0x000000FF); 4737 } else { 4738 _debug_dump_mf(&mpi_reply, 4739 sizeof(Mpi2FWUploadReply_t)/4); 4740 } 4741 } 4742 } 4743 ioc->base_cmds.status = MPT3_CMD_NOT_USED; 4744 out: 4745 if (fwpkg_data) 4746 dma_free_coherent(&ioc->pdev->dev, data_length, fwpkg_data, 4747 fwpkg_data_dma); 4748 if (issue_diag_reset) { 4749 if (ioc->drv_internal_flags & MPT_DRV_INTERNAL_FIRST_PE_ISSUED) 4750 return -EFAULT; 4751 if (mpt3sas_base_check_for_fault_and_issue_reset(ioc)) 4752 return -EFAULT; 4753 r = -EAGAIN; 4754 } 4755 return r; 4756 } 4757 4758 /** 4759 * _base_display_ioc_capabilities - Display IOC's capabilities. 4760 * @ioc: per adapter object 4761 */ 4762 static void 4763 _base_display_ioc_capabilities(struct MPT3SAS_ADAPTER *ioc) 4764 { 4765 int i = 0; 4766 char desc[16]; 4767 u32 iounit_pg1_flags; 4768 u32 bios_version; 4769 4770 bios_version = le32_to_cpu(ioc->bios_pg3.BiosVersion); 4771 strncpy(desc, ioc->manu_pg0.ChipName, 16); 4772 ioc_info(ioc, "%s: FWVersion(%02d.%02d.%02d.%02d), ChipRevision(0x%02x), BiosVersion(%02d.%02d.%02d.%02d)\n", 4773 desc, 4774 (ioc->facts.FWVersion.Word & 0xFF000000) >> 24, 4775 (ioc->facts.FWVersion.Word & 0x00FF0000) >> 16, 4776 (ioc->facts.FWVersion.Word & 0x0000FF00) >> 8, 4777 ioc->facts.FWVersion.Word & 0x000000FF, 4778 ioc->pdev->revision, 4779 (bios_version & 0xFF000000) >> 24, 4780 (bios_version & 0x00FF0000) >> 16, 4781 (bios_version & 0x0000FF00) >> 8, 4782 bios_version & 0x000000FF); 4783 4784 _base_display_OEMs_branding(ioc); 4785 4786 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) { 4787 pr_info("%sNVMe", i ? "," : ""); 4788 i++; 4789 } 4790 4791 ioc_info(ioc, "Protocol=("); 4792 4793 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR) { 4794 pr_cont("Initiator"); 4795 i++; 4796 } 4797 4798 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_TARGET) { 4799 pr_cont("%sTarget", i ? "," : ""); 4800 i++; 4801 } 4802 4803 i = 0; 4804 pr_cont("), Capabilities=("); 4805 4806 if (!ioc->hide_ir_msg) { 4807 if (ioc->facts.IOCCapabilities & 4808 MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID) { 4809 pr_cont("Raid"); 4810 i++; 4811 } 4812 } 4813 4814 if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR) { 4815 pr_cont("%sTLR", i ? "," : ""); 4816 i++; 4817 } 4818 4819 if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_MULTICAST) { 4820 pr_cont("%sMulticast", i ? "," : ""); 4821 i++; 4822 } 4823 4824 if (ioc->facts.IOCCapabilities & 4825 MPI2_IOCFACTS_CAPABILITY_BIDIRECTIONAL_TARGET) { 4826 pr_cont("%sBIDI Target", i ? "," : ""); 4827 i++; 4828 } 4829 4830 if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP) { 4831 pr_cont("%sEEDP", i ? "," : ""); 4832 i++; 4833 } 4834 4835 if (ioc->facts.IOCCapabilities & 4836 MPI2_IOCFACTS_CAPABILITY_SNAPSHOT_BUFFER) { 4837 pr_cont("%sSnapshot Buffer", i ? "," : ""); 4838 i++; 4839 } 4840 4841 if (ioc->facts.IOCCapabilities & 4842 MPI2_IOCFACTS_CAPABILITY_DIAG_TRACE_BUFFER) { 4843 pr_cont("%sDiag Trace Buffer", i ? "," : ""); 4844 i++; 4845 } 4846 4847 if (ioc->facts.IOCCapabilities & 4848 MPI2_IOCFACTS_CAPABILITY_EXTENDED_BUFFER) { 4849 pr_cont("%sDiag Extended Buffer", i ? "," : ""); 4850 i++; 4851 } 4852 4853 if (ioc->facts.IOCCapabilities & 4854 MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING) { 4855 pr_cont("%sTask Set Full", i ? "," : ""); 4856 i++; 4857 } 4858 4859 iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags); 4860 if (!(iounit_pg1_flags & MPI2_IOUNITPAGE1_NATIVE_COMMAND_Q_DISABLE)) { 4861 pr_cont("%sNCQ", i ? "," : ""); 4862 i++; 4863 } 4864 4865 pr_cont(")\n"); 4866 } 4867 4868 /** 4869 * mpt3sas_base_update_missing_delay - change the missing delay timers 4870 * @ioc: per adapter object 4871 * @device_missing_delay: amount of time till device is reported missing 4872 * @io_missing_delay: interval IO is returned when there is a missing device 4873 * 4874 * Passed on the command line, this function will modify the device missing 4875 * delay, as well as the io missing delay. This should be called at driver 4876 * load time. 4877 */ 4878 void 4879 mpt3sas_base_update_missing_delay(struct MPT3SAS_ADAPTER *ioc, 4880 u16 device_missing_delay, u8 io_missing_delay) 4881 { 4882 u16 dmd, dmd_new, dmd_orignal; 4883 u8 io_missing_delay_original; 4884 u16 sz; 4885 Mpi2SasIOUnitPage1_t *sas_iounit_pg1 = NULL; 4886 Mpi2ConfigReply_t mpi_reply; 4887 u8 num_phys = 0; 4888 u16 ioc_status; 4889 4890 mpt3sas_config_get_number_hba_phys(ioc, &num_phys); 4891 if (!num_phys) 4892 return; 4893 4894 sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (num_phys * 4895 sizeof(Mpi2SasIOUnit1PhyData_t)); 4896 sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL); 4897 if (!sas_iounit_pg1) { 4898 ioc_err(ioc, "failure at %s:%d/%s()!\n", 4899 __FILE__, __LINE__, __func__); 4900 goto out; 4901 } 4902 if ((mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply, 4903 sas_iounit_pg1, sz))) { 4904 ioc_err(ioc, "failure at %s:%d/%s()!\n", 4905 __FILE__, __LINE__, __func__); 4906 goto out; 4907 } 4908 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & 4909 MPI2_IOCSTATUS_MASK; 4910 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { 4911 ioc_err(ioc, "failure at %s:%d/%s()!\n", 4912 __FILE__, __LINE__, __func__); 4913 goto out; 4914 } 4915 4916 /* device missing delay */ 4917 dmd = sas_iounit_pg1->ReportDeviceMissingDelay; 4918 if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16) 4919 dmd = (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16; 4920 else 4921 dmd = dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK; 4922 dmd_orignal = dmd; 4923 if (device_missing_delay > 0x7F) { 4924 dmd = (device_missing_delay > 0x7F0) ? 0x7F0 : 4925 device_missing_delay; 4926 dmd = dmd / 16; 4927 dmd |= MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16; 4928 } else 4929 dmd = device_missing_delay; 4930 sas_iounit_pg1->ReportDeviceMissingDelay = dmd; 4931 4932 /* io missing delay */ 4933 io_missing_delay_original = sas_iounit_pg1->IODeviceMissingDelay; 4934 sas_iounit_pg1->IODeviceMissingDelay = io_missing_delay; 4935 4936 if (!mpt3sas_config_set_sas_iounit_pg1(ioc, &mpi_reply, sas_iounit_pg1, 4937 sz)) { 4938 if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16) 4939 dmd_new = (dmd & 4940 MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16; 4941 else 4942 dmd_new = 4943 dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK; 4944 ioc_info(ioc, "device_missing_delay: old(%d), new(%d)\n", 4945 dmd_orignal, dmd_new); 4946 ioc_info(ioc, "ioc_missing_delay: old(%d), new(%d)\n", 4947 io_missing_delay_original, 4948 io_missing_delay); 4949 ioc->device_missing_delay = dmd_new; 4950 ioc->io_missing_delay = io_missing_delay; 4951 } 4952 4953 out: 4954 kfree(sas_iounit_pg1); 4955 } 4956 4957 /** 4958 * _base_update_ioc_page1_inlinewith_perf_mode - Update IOC Page1 fields 4959 * according to performance mode. 4960 * @ioc : per adapter object 4961 * 4962 * Return: zero on success; otherwise return EAGAIN error code asking the 4963 * caller to retry. 4964 */ 4965 static int 4966 _base_update_ioc_page1_inlinewith_perf_mode(struct MPT3SAS_ADAPTER *ioc) 4967 { 4968 Mpi2IOCPage1_t ioc_pg1; 4969 Mpi2ConfigReply_t mpi_reply; 4970 int rc; 4971 4972 rc = mpt3sas_config_get_ioc_pg1(ioc, &mpi_reply, &ioc->ioc_pg1_copy); 4973 if (rc) 4974 return rc; 4975 memcpy(&ioc_pg1, &ioc->ioc_pg1_copy, sizeof(Mpi2IOCPage1_t)); 4976 4977 switch (perf_mode) { 4978 case MPT_PERF_MODE_DEFAULT: 4979 case MPT_PERF_MODE_BALANCED: 4980 if (ioc->high_iops_queues) { 4981 ioc_info(ioc, 4982 "Enable interrupt coalescing only for first\t" 4983 "%d reply queues\n", 4984 MPT3SAS_HIGH_IOPS_REPLY_QUEUES); 4985 /* 4986 * If 31st bit is zero then interrupt coalescing is 4987 * enabled for all reply descriptor post queues. 4988 * If 31st bit is set to one then user can 4989 * enable/disable interrupt coalescing on per reply 4990 * descriptor post queue group(8) basis. So to enable 4991 * interrupt coalescing only on first reply descriptor 4992 * post queue group 31st bit and zero th bit is enabled. 4993 */ 4994 ioc_pg1.ProductSpecific = cpu_to_le32(0x80000000 | 4995 ((1 << MPT3SAS_HIGH_IOPS_REPLY_QUEUES/8) - 1)); 4996 rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1); 4997 if (rc) 4998 return rc; 4999 ioc_info(ioc, "performance mode: balanced\n"); 5000 return 0; 5001 } 5002 fallthrough; 5003 case MPT_PERF_MODE_LATENCY: 5004 /* 5005 * Enable interrupt coalescing on all reply queues 5006 * with timeout value 0xA 5007 */ 5008 ioc_pg1.CoalescingTimeout = cpu_to_le32(0xa); 5009 ioc_pg1.Flags |= cpu_to_le32(MPI2_IOCPAGE1_REPLY_COALESCING); 5010 ioc_pg1.ProductSpecific = 0; 5011 rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1); 5012 if (rc) 5013 return rc; 5014 ioc_info(ioc, "performance mode: latency\n"); 5015 break; 5016 case MPT_PERF_MODE_IOPS: 5017 /* 5018 * Enable interrupt coalescing on all reply queues. 5019 */ 5020 ioc_info(ioc, 5021 "performance mode: iops with coalescing timeout: 0x%x\n", 5022 le32_to_cpu(ioc_pg1.CoalescingTimeout)); 5023 ioc_pg1.Flags |= cpu_to_le32(MPI2_IOCPAGE1_REPLY_COALESCING); 5024 ioc_pg1.ProductSpecific = 0; 5025 rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1); 5026 if (rc) 5027 return rc; 5028 break; 5029 } 5030 return 0; 5031 } 5032 5033 /** 5034 * _base_get_event_diag_triggers - get event diag trigger values from 5035 * persistent pages 5036 * @ioc : per adapter object 5037 * 5038 * Return: nothing. 5039 */ 5040 static int 5041 _base_get_event_diag_triggers(struct MPT3SAS_ADAPTER *ioc) 5042 { 5043 Mpi26DriverTriggerPage2_t trigger_pg2; 5044 struct SL_WH_EVENT_TRIGGER_T *event_tg; 5045 MPI26_DRIVER_MPI_EVENT_TIGGER_ENTRY *mpi_event_tg; 5046 Mpi2ConfigReply_t mpi_reply; 5047 int r = 0, i = 0; 5048 u16 count = 0; 5049 u16 ioc_status; 5050 5051 r = mpt3sas_config_get_driver_trigger_pg2(ioc, &mpi_reply, 5052 &trigger_pg2); 5053 if (r) 5054 return r; 5055 5056 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & 5057 MPI2_IOCSTATUS_MASK; 5058 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { 5059 dinitprintk(ioc, 5060 ioc_err(ioc, 5061 "%s: Failed to get trigger pg2, ioc_status(0x%04x)\n", 5062 __func__, ioc_status)); 5063 return 0; 5064 } 5065 5066 if (le16_to_cpu(trigger_pg2.NumMPIEventTrigger)) { 5067 count = le16_to_cpu(trigger_pg2.NumMPIEventTrigger); 5068 count = min_t(u16, NUM_VALID_ENTRIES, count); 5069 ioc->diag_trigger_event.ValidEntries = count; 5070 5071 event_tg = &ioc->diag_trigger_event.EventTriggerEntry[0]; 5072 mpi_event_tg = &trigger_pg2.MPIEventTriggers[0]; 5073 for (i = 0; i < count; i++) { 5074 event_tg->EventValue = le16_to_cpu( 5075 mpi_event_tg->MPIEventCode); 5076 event_tg->LogEntryQualifier = le16_to_cpu( 5077 mpi_event_tg->MPIEventCodeSpecific); 5078 event_tg++; 5079 mpi_event_tg++; 5080 } 5081 } 5082 return 0; 5083 } 5084 5085 /** 5086 * _base_get_scsi_diag_triggers - get scsi diag trigger values from 5087 * persistent pages 5088 * @ioc : per adapter object 5089 * 5090 * Return: 0 on success; otherwise return failure status. 5091 */ 5092 static int 5093 _base_get_scsi_diag_triggers(struct MPT3SAS_ADAPTER *ioc) 5094 { 5095 Mpi26DriverTriggerPage3_t trigger_pg3; 5096 struct SL_WH_SCSI_TRIGGER_T *scsi_tg; 5097 MPI26_DRIVER_SCSI_SENSE_TIGGER_ENTRY *mpi_scsi_tg; 5098 Mpi2ConfigReply_t mpi_reply; 5099 int r = 0, i = 0; 5100 u16 count = 0; 5101 u16 ioc_status; 5102 5103 r = mpt3sas_config_get_driver_trigger_pg3(ioc, &mpi_reply, 5104 &trigger_pg3); 5105 if (r) 5106 return r; 5107 5108 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & 5109 MPI2_IOCSTATUS_MASK; 5110 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { 5111 dinitprintk(ioc, 5112 ioc_err(ioc, 5113 "%s: Failed to get trigger pg3, ioc_status(0x%04x)\n", 5114 __func__, ioc_status)); 5115 return 0; 5116 } 5117 5118 if (le16_to_cpu(trigger_pg3.NumSCSISenseTrigger)) { 5119 count = le16_to_cpu(trigger_pg3.NumSCSISenseTrigger); 5120 count = min_t(u16, NUM_VALID_ENTRIES, count); 5121 ioc->diag_trigger_scsi.ValidEntries = count; 5122 5123 scsi_tg = &ioc->diag_trigger_scsi.SCSITriggerEntry[0]; 5124 mpi_scsi_tg = &trigger_pg3.SCSISenseTriggers[0]; 5125 for (i = 0; i < count; i++) { 5126 scsi_tg->ASCQ = mpi_scsi_tg->ASCQ; 5127 scsi_tg->ASC = mpi_scsi_tg->ASC; 5128 scsi_tg->SenseKey = mpi_scsi_tg->SenseKey; 5129 5130 scsi_tg++; 5131 mpi_scsi_tg++; 5132 } 5133 } 5134 return 0; 5135 } 5136 5137 /** 5138 * _base_get_mpi_diag_triggers - get mpi diag trigger values from 5139 * persistent pages 5140 * @ioc : per adapter object 5141 * 5142 * Return: 0 on success; otherwise return failure status. 5143 */ 5144 static int 5145 _base_get_mpi_diag_triggers(struct MPT3SAS_ADAPTER *ioc) 5146 { 5147 Mpi26DriverTriggerPage4_t trigger_pg4; 5148 struct SL_WH_MPI_TRIGGER_T *status_tg; 5149 MPI26_DRIVER_IOCSTATUS_LOGINFO_TIGGER_ENTRY *mpi_status_tg; 5150 Mpi2ConfigReply_t mpi_reply; 5151 int r = 0, i = 0; 5152 u16 count = 0; 5153 u16 ioc_status; 5154 5155 r = mpt3sas_config_get_driver_trigger_pg4(ioc, &mpi_reply, 5156 &trigger_pg4); 5157 if (r) 5158 return r; 5159 5160 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & 5161 MPI2_IOCSTATUS_MASK; 5162 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { 5163 dinitprintk(ioc, 5164 ioc_err(ioc, 5165 "%s: Failed to get trigger pg4, ioc_status(0x%04x)\n", 5166 __func__, ioc_status)); 5167 return 0; 5168 } 5169 5170 if (le16_to_cpu(trigger_pg4.NumIOCStatusLogInfoTrigger)) { 5171 count = le16_to_cpu(trigger_pg4.NumIOCStatusLogInfoTrigger); 5172 count = min_t(u16, NUM_VALID_ENTRIES, count); 5173 ioc->diag_trigger_mpi.ValidEntries = count; 5174 5175 status_tg = &ioc->diag_trigger_mpi.MPITriggerEntry[0]; 5176 mpi_status_tg = &trigger_pg4.IOCStatusLoginfoTriggers[0]; 5177 5178 for (i = 0; i < count; i++) { 5179 status_tg->IOCStatus = le16_to_cpu( 5180 mpi_status_tg->IOCStatus); 5181 status_tg->IocLogInfo = le32_to_cpu( 5182 mpi_status_tg->LogInfo); 5183 5184 status_tg++; 5185 mpi_status_tg++; 5186 } 5187 } 5188 return 0; 5189 } 5190 5191 /** 5192 * _base_get_master_diag_triggers - get master diag trigger values from 5193 * persistent pages 5194 * @ioc : per adapter object 5195 * 5196 * Return: nothing. 5197 */ 5198 static int 5199 _base_get_master_diag_triggers(struct MPT3SAS_ADAPTER *ioc) 5200 { 5201 Mpi26DriverTriggerPage1_t trigger_pg1; 5202 Mpi2ConfigReply_t mpi_reply; 5203 int r; 5204 u16 ioc_status; 5205 5206 r = mpt3sas_config_get_driver_trigger_pg1(ioc, &mpi_reply, 5207 &trigger_pg1); 5208 if (r) 5209 return r; 5210 5211 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & 5212 MPI2_IOCSTATUS_MASK; 5213 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { 5214 dinitprintk(ioc, 5215 ioc_err(ioc, 5216 "%s: Failed to get trigger pg1, ioc_status(0x%04x)\n", 5217 __func__, ioc_status)); 5218 return 0; 5219 } 5220 5221 if (le16_to_cpu(trigger_pg1.NumMasterTrigger)) 5222 ioc->diag_trigger_master.MasterData |= 5223 le32_to_cpu( 5224 trigger_pg1.MasterTriggers[0].MasterTriggerFlags); 5225 return 0; 5226 } 5227 5228 /** 5229 * _base_check_for_trigger_pages_support - checks whether HBA FW supports 5230 * driver trigger pages or not 5231 * @ioc : per adapter object 5232 * @trigger_flags : address where trigger page0's TriggerFlags value is copied 5233 * 5234 * Return: trigger flags mask if HBA FW supports driver trigger pages; 5235 * otherwise returns %-EFAULT if driver trigger pages are not supported by FW or 5236 * return EAGAIN if diag reset occurred due to FW fault and asking the 5237 * caller to retry the command. 5238 * 5239 */ 5240 static int 5241 _base_check_for_trigger_pages_support(struct MPT3SAS_ADAPTER *ioc, u32 *trigger_flags) 5242 { 5243 Mpi26DriverTriggerPage0_t trigger_pg0; 5244 int r = 0; 5245 Mpi2ConfigReply_t mpi_reply; 5246 u16 ioc_status; 5247 5248 r = mpt3sas_config_get_driver_trigger_pg0(ioc, &mpi_reply, 5249 &trigger_pg0); 5250 if (r) 5251 return r; 5252 5253 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & 5254 MPI2_IOCSTATUS_MASK; 5255 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) 5256 return -EFAULT; 5257 5258 *trigger_flags = le16_to_cpu(trigger_pg0.TriggerFlags); 5259 return 0; 5260 } 5261 5262 /** 5263 * _base_get_diag_triggers - Retrieve diag trigger values from 5264 * persistent pages. 5265 * @ioc : per adapter object 5266 * 5267 * Return: zero on success; otherwise return EAGAIN error codes 5268 * asking the caller to retry. 5269 */ 5270 static int 5271 _base_get_diag_triggers(struct MPT3SAS_ADAPTER *ioc) 5272 { 5273 int trigger_flags; 5274 int r; 5275 5276 /* 5277 * Default setting of master trigger. 5278 */ 5279 ioc->diag_trigger_master.MasterData = 5280 (MASTER_TRIGGER_FW_FAULT + MASTER_TRIGGER_ADAPTER_RESET); 5281 5282 r = _base_check_for_trigger_pages_support(ioc, &trigger_flags); 5283 if (r) { 5284 if (r == -EAGAIN) 5285 return r; 5286 /* 5287 * Don't go for error handling when FW doesn't support 5288 * driver trigger pages. 5289 */ 5290 return 0; 5291 } 5292 5293 ioc->supports_trigger_pages = 1; 5294 5295 /* 5296 * Retrieve master diag trigger values from driver trigger pg1 5297 * if master trigger bit enabled in TriggerFlags. 5298 */ 5299 if ((u16)trigger_flags & 5300 MPI26_DRIVER_TRIGGER0_FLAG_MASTER_TRIGGER_VALID) { 5301 r = _base_get_master_diag_triggers(ioc); 5302 if (r) 5303 return r; 5304 } 5305 5306 /* 5307 * Retrieve event diag trigger values from driver trigger pg2 5308 * if event trigger bit enabled in TriggerFlags. 5309 */ 5310 if ((u16)trigger_flags & 5311 MPI26_DRIVER_TRIGGER0_FLAG_MPI_EVENT_TRIGGER_VALID) { 5312 r = _base_get_event_diag_triggers(ioc); 5313 if (r) 5314 return r; 5315 } 5316 5317 /* 5318 * Retrieve scsi diag trigger values from driver trigger pg3 5319 * if scsi trigger bit enabled in TriggerFlags. 5320 */ 5321 if ((u16)trigger_flags & 5322 MPI26_DRIVER_TRIGGER0_FLAG_SCSI_SENSE_TRIGGER_VALID) { 5323 r = _base_get_scsi_diag_triggers(ioc); 5324 if (r) 5325 return r; 5326 } 5327 /* 5328 * Retrieve mpi error diag trigger values from driver trigger pg4 5329 * if loginfo trigger bit enabled in TriggerFlags. 5330 */ 5331 if ((u16)trigger_flags & 5332 MPI26_DRIVER_TRIGGER0_FLAG_LOGINFO_TRIGGER_VALID) { 5333 r = _base_get_mpi_diag_triggers(ioc); 5334 if (r) 5335 return r; 5336 } 5337 return 0; 5338 } 5339 5340 /** 5341 * _base_update_diag_trigger_pages - Update the driver trigger pages after 5342 * online FW update, in case updated FW supports driver 5343 * trigger pages. 5344 * @ioc : per adapter object 5345 * 5346 * Return: nothing. 5347 */ 5348 static void 5349 _base_update_diag_trigger_pages(struct MPT3SAS_ADAPTER *ioc) 5350 { 5351 5352 if (ioc->diag_trigger_master.MasterData) 5353 mpt3sas_config_update_driver_trigger_pg1(ioc, 5354 &ioc->diag_trigger_master, 1); 5355 5356 if (ioc->diag_trigger_event.ValidEntries) 5357 mpt3sas_config_update_driver_trigger_pg2(ioc, 5358 &ioc->diag_trigger_event, 1); 5359 5360 if (ioc->diag_trigger_scsi.ValidEntries) 5361 mpt3sas_config_update_driver_trigger_pg3(ioc, 5362 &ioc->diag_trigger_scsi, 1); 5363 5364 if (ioc->diag_trigger_mpi.ValidEntries) 5365 mpt3sas_config_update_driver_trigger_pg4(ioc, 5366 &ioc->diag_trigger_mpi, 1); 5367 } 5368 5369 /** 5370 * _base_assign_fw_reported_qd - Get FW reported QD for SAS/SATA devices. 5371 * - On failure set default QD values. 5372 * @ioc : per adapter object 5373 * 5374 * Returns 0 for success, non-zero for failure. 5375 * 5376 */ 5377 static int _base_assign_fw_reported_qd(struct MPT3SAS_ADAPTER *ioc) 5378 { 5379 Mpi2ConfigReply_t mpi_reply; 5380 Mpi2SasIOUnitPage1_t *sas_iounit_pg1 = NULL; 5381 Mpi26PCIeIOUnitPage1_t pcie_iounit_pg1; 5382 int sz; 5383 int rc = 0; 5384 5385 ioc->max_wideport_qd = MPT3SAS_SAS_QUEUE_DEPTH; 5386 ioc->max_narrowport_qd = MPT3SAS_SAS_QUEUE_DEPTH; 5387 ioc->max_sata_qd = MPT3SAS_SATA_QUEUE_DEPTH; 5388 ioc->max_nvme_qd = MPT3SAS_NVME_QUEUE_DEPTH; 5389 if (!ioc->is_gen35_ioc) 5390 goto out; 5391 /* sas iounit page 1 */ 5392 sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData); 5393 sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL); 5394 if (!sas_iounit_pg1) { 5395 pr_err("%s: failure at %s:%d/%s()!\n", 5396 ioc->name, __FILE__, __LINE__, __func__); 5397 return rc; 5398 } 5399 rc = mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply, 5400 sas_iounit_pg1, sz); 5401 if (rc) { 5402 pr_err("%s: failure at %s:%d/%s()!\n", 5403 ioc->name, __FILE__, __LINE__, __func__); 5404 goto out; 5405 } 5406 ioc->max_wideport_qd = 5407 (le16_to_cpu(sas_iounit_pg1->SASWideMaxQueueDepth)) ? 5408 le16_to_cpu(sas_iounit_pg1->SASWideMaxQueueDepth) : 5409 MPT3SAS_SAS_QUEUE_DEPTH; 5410 ioc->max_narrowport_qd = 5411 (le16_to_cpu(sas_iounit_pg1->SASNarrowMaxQueueDepth)) ? 5412 le16_to_cpu(sas_iounit_pg1->SASNarrowMaxQueueDepth) : 5413 MPT3SAS_SAS_QUEUE_DEPTH; 5414 ioc->max_sata_qd = (sas_iounit_pg1->SATAMaxQDepth) ? 5415 sas_iounit_pg1->SATAMaxQDepth : MPT3SAS_SATA_QUEUE_DEPTH; 5416 /* pcie iounit page 1 */ 5417 rc = mpt3sas_config_get_pcie_iounit_pg1(ioc, &mpi_reply, 5418 &pcie_iounit_pg1, sizeof(Mpi26PCIeIOUnitPage1_t)); 5419 if (rc) { 5420 pr_err("%s: failure at %s:%d/%s()!\n", 5421 ioc->name, __FILE__, __LINE__, __func__); 5422 goto out; 5423 } 5424 ioc->max_nvme_qd = (le16_to_cpu(pcie_iounit_pg1.NVMeMaxQueueDepth)) ? 5425 (le16_to_cpu(pcie_iounit_pg1.NVMeMaxQueueDepth)) : 5426 MPT3SAS_NVME_QUEUE_DEPTH; 5427 out: 5428 dinitprintk(ioc, pr_err( 5429 "MaxWidePortQD: 0x%x MaxNarrowPortQD: 0x%x MaxSataQD: 0x%x MaxNvmeQD: 0x%x\n", 5430 ioc->max_wideport_qd, ioc->max_narrowport_qd, 5431 ioc->max_sata_qd, ioc->max_nvme_qd)); 5432 kfree(sas_iounit_pg1); 5433 return rc; 5434 } 5435 5436 /** 5437 * _base_static_config_pages - static start of day config pages 5438 * @ioc: per adapter object 5439 */ 5440 static int 5441 _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc) 5442 { 5443 Mpi2ConfigReply_t mpi_reply; 5444 u32 iounit_pg1_flags; 5445 int tg_flags = 0; 5446 int rc; 5447 ioc->nvme_abort_timeout = 30; 5448 5449 rc = mpt3sas_config_get_manufacturing_pg0(ioc, &mpi_reply, 5450 &ioc->manu_pg0); 5451 if (rc) 5452 return rc; 5453 if (ioc->ir_firmware) { 5454 rc = mpt3sas_config_get_manufacturing_pg10(ioc, &mpi_reply, 5455 &ioc->manu_pg10); 5456 if (rc) 5457 return rc; 5458 } 5459 /* 5460 * Ensure correct T10 PI operation if vendor left EEDPTagMode 5461 * flag unset in NVDATA. 5462 */ 5463 rc = mpt3sas_config_get_manufacturing_pg11(ioc, &mpi_reply, 5464 &ioc->manu_pg11); 5465 if (rc) 5466 return rc; 5467 if (!ioc->is_gen35_ioc && ioc->manu_pg11.EEDPTagMode == 0) { 5468 pr_err("%s: overriding NVDATA EEDPTagMode setting\n", 5469 ioc->name); 5470 ioc->manu_pg11.EEDPTagMode &= ~0x3; 5471 ioc->manu_pg11.EEDPTagMode |= 0x1; 5472 mpt3sas_config_set_manufacturing_pg11(ioc, &mpi_reply, 5473 &ioc->manu_pg11); 5474 } 5475 if (ioc->manu_pg11.AddlFlags2 & NVME_TASK_MNGT_CUSTOM_MASK) 5476 ioc->tm_custom_handling = 1; 5477 else { 5478 ioc->tm_custom_handling = 0; 5479 if (ioc->manu_pg11.NVMeAbortTO < NVME_TASK_ABORT_MIN_TIMEOUT) 5480 ioc->nvme_abort_timeout = NVME_TASK_ABORT_MIN_TIMEOUT; 5481 else if (ioc->manu_pg11.NVMeAbortTO > 5482 NVME_TASK_ABORT_MAX_TIMEOUT) 5483 ioc->nvme_abort_timeout = NVME_TASK_ABORT_MAX_TIMEOUT; 5484 else 5485 ioc->nvme_abort_timeout = ioc->manu_pg11.NVMeAbortTO; 5486 } 5487 ioc->time_sync_interval = 5488 ioc->manu_pg11.TimeSyncInterval & MPT3SAS_TIMESYNC_MASK; 5489 if (ioc->time_sync_interval) { 5490 if (ioc->manu_pg11.TimeSyncInterval & MPT3SAS_TIMESYNC_UNIT_MASK) 5491 ioc->time_sync_interval = 5492 ioc->time_sync_interval * SECONDS_PER_HOUR; 5493 else 5494 ioc->time_sync_interval = 5495 ioc->time_sync_interval * SECONDS_PER_MIN; 5496 dinitprintk(ioc, ioc_info(ioc, 5497 "Driver-FW TimeSync interval is %d seconds. ManuPg11 TimeSync Unit is in %s\n", 5498 ioc->time_sync_interval, (ioc->manu_pg11.TimeSyncInterval & 5499 MPT3SAS_TIMESYNC_UNIT_MASK) ? "Hour" : "Minute")); 5500 } else { 5501 if (ioc->is_gen35_ioc) 5502 ioc_warn(ioc, 5503 "TimeSync Interval in Manuf page-11 is not enabled. Periodic Time-Sync will be disabled\n"); 5504 } 5505 rc = _base_assign_fw_reported_qd(ioc); 5506 if (rc) 5507 return rc; 5508 rc = mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2); 5509 if (rc) 5510 return rc; 5511 rc = mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3); 5512 if (rc) 5513 return rc; 5514 rc = mpt3sas_config_get_ioc_pg8(ioc, &mpi_reply, &ioc->ioc_pg8); 5515 if (rc) 5516 return rc; 5517 rc = mpt3sas_config_get_iounit_pg0(ioc, &mpi_reply, &ioc->iounit_pg0); 5518 if (rc) 5519 return rc; 5520 rc = mpt3sas_config_get_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1); 5521 if (rc) 5522 return rc; 5523 rc = mpt3sas_config_get_iounit_pg8(ioc, &mpi_reply, &ioc->iounit_pg8); 5524 if (rc) 5525 return rc; 5526 _base_display_ioc_capabilities(ioc); 5527 5528 /* 5529 * Enable task_set_full handling in iounit_pg1 when the 5530 * facts capabilities indicate that its supported. 5531 */ 5532 iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags); 5533 if ((ioc->facts.IOCCapabilities & 5534 MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING)) 5535 iounit_pg1_flags &= 5536 ~MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING; 5537 else 5538 iounit_pg1_flags |= 5539 MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING; 5540 ioc->iounit_pg1.Flags = cpu_to_le32(iounit_pg1_flags); 5541 rc = mpt3sas_config_set_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1); 5542 if (rc) 5543 return rc; 5544 5545 if (ioc->iounit_pg8.NumSensors) 5546 ioc->temp_sensors_count = ioc->iounit_pg8.NumSensors; 5547 if (ioc->is_aero_ioc) { 5548 rc = _base_update_ioc_page1_inlinewith_perf_mode(ioc); 5549 if (rc) 5550 return rc; 5551 } 5552 if (ioc->is_gen35_ioc) { 5553 if (ioc->is_driver_loading) { 5554 rc = _base_get_diag_triggers(ioc); 5555 if (rc) 5556 return rc; 5557 } else { 5558 /* 5559 * In case of online HBA FW update operation, 5560 * check whether updated FW supports the driver trigger 5561 * pages or not. 5562 * - If previous FW has not supported driver trigger 5563 * pages and newer FW supports them then update these 5564 * pages with current diag trigger values. 5565 * - If previous FW has supported driver trigger pages 5566 * and new FW doesn't support them then disable 5567 * support_trigger_pages flag. 5568 */ 5569 _base_check_for_trigger_pages_support(ioc, &tg_flags); 5570 if (!ioc->supports_trigger_pages && tg_flags != -EFAULT) 5571 _base_update_diag_trigger_pages(ioc); 5572 else if (ioc->supports_trigger_pages && 5573 tg_flags == -EFAULT) 5574 ioc->supports_trigger_pages = 0; 5575 } 5576 } 5577 return 0; 5578 } 5579 5580 /** 5581 * mpt3sas_free_enclosure_list - release memory 5582 * @ioc: per adapter object 5583 * 5584 * Free memory allocated during enclosure add. 5585 */ 5586 void 5587 mpt3sas_free_enclosure_list(struct MPT3SAS_ADAPTER *ioc) 5588 { 5589 struct _enclosure_node *enclosure_dev, *enclosure_dev_next; 5590 5591 /* Free enclosure list */ 5592 list_for_each_entry_safe(enclosure_dev, 5593 enclosure_dev_next, &ioc->enclosure_list, list) { 5594 list_del(&enclosure_dev->list); 5595 kfree(enclosure_dev); 5596 } 5597 } 5598 5599 /** 5600 * _base_release_memory_pools - release memory 5601 * @ioc: per adapter object 5602 * 5603 * Free memory allocated from _base_allocate_memory_pools. 5604 */ 5605 static void 5606 _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc) 5607 { 5608 int i = 0; 5609 int j = 0; 5610 int dma_alloc_count = 0; 5611 struct chain_tracker *ct; 5612 int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1; 5613 5614 dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 5615 5616 if (ioc->request) { 5617 dma_free_coherent(&ioc->pdev->dev, ioc->request_dma_sz, 5618 ioc->request, ioc->request_dma); 5619 dexitprintk(ioc, 5620 ioc_info(ioc, "request_pool(0x%p): free\n", 5621 ioc->request)); 5622 ioc->request = NULL; 5623 } 5624 5625 if (ioc->sense) { 5626 dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma); 5627 dma_pool_destroy(ioc->sense_dma_pool); 5628 dexitprintk(ioc, 5629 ioc_info(ioc, "sense_pool(0x%p): free\n", 5630 ioc->sense)); 5631 ioc->sense = NULL; 5632 } 5633 5634 if (ioc->reply) { 5635 dma_pool_free(ioc->reply_dma_pool, ioc->reply, ioc->reply_dma); 5636 dma_pool_destroy(ioc->reply_dma_pool); 5637 dexitprintk(ioc, 5638 ioc_info(ioc, "reply_pool(0x%p): free\n", 5639 ioc->reply)); 5640 ioc->reply = NULL; 5641 } 5642 5643 if (ioc->reply_free) { 5644 dma_pool_free(ioc->reply_free_dma_pool, ioc->reply_free, 5645 ioc->reply_free_dma); 5646 dma_pool_destroy(ioc->reply_free_dma_pool); 5647 dexitprintk(ioc, 5648 ioc_info(ioc, "reply_free_pool(0x%p): free\n", 5649 ioc->reply_free)); 5650 ioc->reply_free = NULL; 5651 } 5652 5653 if (ioc->reply_post) { 5654 dma_alloc_count = DIV_ROUND_UP(count, 5655 RDPQ_MAX_INDEX_IN_ONE_CHUNK); 5656 for (i = 0; i < count; i++) { 5657 if (i % RDPQ_MAX_INDEX_IN_ONE_CHUNK == 0 5658 && dma_alloc_count) { 5659 if (ioc->reply_post[i].reply_post_free) { 5660 dma_pool_free( 5661 ioc->reply_post_free_dma_pool, 5662 ioc->reply_post[i].reply_post_free, 5663 ioc->reply_post[i].reply_post_free_dma); 5664 dexitprintk(ioc, ioc_info(ioc, 5665 "reply_post_free_pool(0x%p): free\n", 5666 ioc->reply_post[i].reply_post_free)); 5667 ioc->reply_post[i].reply_post_free = 5668 NULL; 5669 } 5670 --dma_alloc_count; 5671 } 5672 } 5673 dma_pool_destroy(ioc->reply_post_free_dma_pool); 5674 if (ioc->reply_post_free_array && 5675 ioc->rdpq_array_enable) { 5676 dma_pool_free(ioc->reply_post_free_array_dma_pool, 5677 ioc->reply_post_free_array, 5678 ioc->reply_post_free_array_dma); 5679 ioc->reply_post_free_array = NULL; 5680 } 5681 dma_pool_destroy(ioc->reply_post_free_array_dma_pool); 5682 kfree(ioc->reply_post); 5683 } 5684 5685 if (ioc->pcie_sgl_dma_pool) { 5686 for (i = 0; i < ioc->scsiio_depth; i++) { 5687 dma_pool_free(ioc->pcie_sgl_dma_pool, 5688 ioc->pcie_sg_lookup[i].pcie_sgl, 5689 ioc->pcie_sg_lookup[i].pcie_sgl_dma); 5690 ioc->pcie_sg_lookup[i].pcie_sgl = NULL; 5691 } 5692 dma_pool_destroy(ioc->pcie_sgl_dma_pool); 5693 } 5694 if (ioc->config_page) { 5695 dexitprintk(ioc, 5696 ioc_info(ioc, "config_page(0x%p): free\n", 5697 ioc->config_page)); 5698 dma_free_coherent(&ioc->pdev->dev, ioc->config_page_sz, 5699 ioc->config_page, ioc->config_page_dma); 5700 } 5701 5702 kfree(ioc->hpr_lookup); 5703 ioc->hpr_lookup = NULL; 5704 kfree(ioc->internal_lookup); 5705 ioc->internal_lookup = NULL; 5706 if (ioc->chain_lookup) { 5707 for (i = 0; i < ioc->scsiio_depth; i++) { 5708 for (j = ioc->chains_per_prp_buffer; 5709 j < ioc->chains_needed_per_io; j++) { 5710 ct = &ioc->chain_lookup[i].chains_per_smid[j]; 5711 if (ct && ct->chain_buffer) 5712 dma_pool_free(ioc->chain_dma_pool, 5713 ct->chain_buffer, 5714 ct->chain_buffer_dma); 5715 } 5716 kfree(ioc->chain_lookup[i].chains_per_smid); 5717 } 5718 dma_pool_destroy(ioc->chain_dma_pool); 5719 kfree(ioc->chain_lookup); 5720 ioc->chain_lookup = NULL; 5721 } 5722 5723 kfree(ioc->io_queue_num); 5724 ioc->io_queue_num = NULL; 5725 } 5726 5727 /** 5728 * mpt3sas_check_same_4gb_region - checks whether all reply queues in a set are 5729 * having same upper 32bits in their base memory address. 5730 * @reply_pool_start_address: Base address of a reply queue set 5731 * @pool_sz: Size of single Reply Descriptor Post Queues pool size 5732 * 5733 * Return: 1 if reply queues in a set have a same upper 32bits in their base 5734 * memory address, else 0. 5735 */ 5736 5737 static int 5738 mpt3sas_check_same_4gb_region(long reply_pool_start_address, u32 pool_sz) 5739 { 5740 long reply_pool_end_address; 5741 5742 reply_pool_end_address = reply_pool_start_address + pool_sz; 5743 5744 if (upper_32_bits(reply_pool_start_address) == 5745 upper_32_bits(reply_pool_end_address)) 5746 return 1; 5747 else 5748 return 0; 5749 } 5750 5751 /** 5752 * _base_reduce_hba_queue_depth- Retry with reduced queue depth 5753 * @ioc: Adapter object 5754 * 5755 * Return: 0 for success, non-zero for failure. 5756 **/ 5757 static inline int 5758 _base_reduce_hba_queue_depth(struct MPT3SAS_ADAPTER *ioc) 5759 { 5760 int reduce_sz = 64; 5761 5762 if ((ioc->hba_queue_depth - reduce_sz) > 5763 (ioc->internal_depth + INTERNAL_SCSIIO_CMDS_COUNT)) { 5764 ioc->hba_queue_depth -= reduce_sz; 5765 return 0; 5766 } else 5767 return -ENOMEM; 5768 } 5769 5770 /** 5771 * _base_allocate_pcie_sgl_pool - Allocating DMA'able memory 5772 * for pcie sgl pools. 5773 * @ioc: Adapter object 5774 * @sz: DMA Pool size 5775 * 5776 * Return: 0 for success, non-zero for failure. 5777 */ 5778 5779 static int 5780 _base_allocate_pcie_sgl_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz) 5781 { 5782 int i = 0, j = 0; 5783 struct chain_tracker *ct; 5784 5785 ioc->pcie_sgl_dma_pool = 5786 dma_pool_create("PCIe SGL pool", &ioc->pdev->dev, sz, 5787 ioc->page_size, 0); 5788 if (!ioc->pcie_sgl_dma_pool) { 5789 ioc_err(ioc, "PCIe SGL pool: dma_pool_create failed\n"); 5790 return -ENOMEM; 5791 } 5792 5793 ioc->chains_per_prp_buffer = sz/ioc->chain_segment_sz; 5794 ioc->chains_per_prp_buffer = 5795 min(ioc->chains_per_prp_buffer, ioc->chains_needed_per_io); 5796 for (i = 0; i < ioc->scsiio_depth; i++) { 5797 ioc->pcie_sg_lookup[i].pcie_sgl = 5798 dma_pool_alloc(ioc->pcie_sgl_dma_pool, GFP_KERNEL, 5799 &ioc->pcie_sg_lookup[i].pcie_sgl_dma); 5800 if (!ioc->pcie_sg_lookup[i].pcie_sgl) { 5801 ioc_err(ioc, "PCIe SGL pool: dma_pool_alloc failed\n"); 5802 return -EAGAIN; 5803 } 5804 5805 if (!mpt3sas_check_same_4gb_region( 5806 (long)ioc->pcie_sg_lookup[i].pcie_sgl, sz)) { 5807 ioc_err(ioc, "PCIE SGLs are not in same 4G !! pcie sgl (0x%p) dma = (0x%llx)\n", 5808 ioc->pcie_sg_lookup[i].pcie_sgl, 5809 (unsigned long long) 5810 ioc->pcie_sg_lookup[i].pcie_sgl_dma); 5811 ioc->use_32bit_dma = true; 5812 return -EAGAIN; 5813 } 5814 5815 for (j = 0; j < ioc->chains_per_prp_buffer; j++) { 5816 ct = &ioc->chain_lookup[i].chains_per_smid[j]; 5817 ct->chain_buffer = 5818 ioc->pcie_sg_lookup[i].pcie_sgl + 5819 (j * ioc->chain_segment_sz); 5820 ct->chain_buffer_dma = 5821 ioc->pcie_sg_lookup[i].pcie_sgl_dma + 5822 (j * ioc->chain_segment_sz); 5823 } 5824 } 5825 dinitprintk(ioc, ioc_info(ioc, 5826 "PCIe sgl pool depth(%d), element_size(%d), pool_size(%d kB)\n", 5827 ioc->scsiio_depth, sz, (sz * ioc->scsiio_depth)/1024)); 5828 dinitprintk(ioc, ioc_info(ioc, 5829 "Number of chains can fit in a PRP page(%d)\n", 5830 ioc->chains_per_prp_buffer)); 5831 return 0; 5832 } 5833 5834 /** 5835 * _base_allocate_chain_dma_pool - Allocating DMA'able memory 5836 * for chain dma pool. 5837 * @ioc: Adapter object 5838 * @sz: DMA Pool size 5839 * 5840 * Return: 0 for success, non-zero for failure. 5841 */ 5842 static int 5843 _base_allocate_chain_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz) 5844 { 5845 int i = 0, j = 0; 5846 struct chain_tracker *ctr; 5847 5848 ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev, 5849 ioc->chain_segment_sz, 16, 0); 5850 if (!ioc->chain_dma_pool) 5851 return -ENOMEM; 5852 5853 for (i = 0; i < ioc->scsiio_depth; i++) { 5854 for (j = ioc->chains_per_prp_buffer; 5855 j < ioc->chains_needed_per_io; j++) { 5856 ctr = &ioc->chain_lookup[i].chains_per_smid[j]; 5857 ctr->chain_buffer = dma_pool_alloc(ioc->chain_dma_pool, 5858 GFP_KERNEL, &ctr->chain_buffer_dma); 5859 if (!ctr->chain_buffer) 5860 return -EAGAIN; 5861 if (!mpt3sas_check_same_4gb_region((long) 5862 ctr->chain_buffer, ioc->chain_segment_sz)) { 5863 ioc_err(ioc, 5864 "Chain buffers are not in same 4G !!! Chain buff (0x%p) dma = (0x%llx)\n", 5865 ctr->chain_buffer, 5866 (unsigned long long)ctr->chain_buffer_dma); 5867 ioc->use_32bit_dma = true; 5868 return -EAGAIN; 5869 } 5870 } 5871 } 5872 dinitprintk(ioc, ioc_info(ioc, 5873 "chain_lookup depth (%d), frame_size(%d), pool_size(%d kB)\n", 5874 ioc->scsiio_depth, ioc->chain_segment_sz, ((ioc->scsiio_depth * 5875 (ioc->chains_needed_per_io - ioc->chains_per_prp_buffer) * 5876 ioc->chain_segment_sz))/1024)); 5877 return 0; 5878 } 5879 5880 /** 5881 * _base_allocate_sense_dma_pool - Allocating DMA'able memory 5882 * for sense dma pool. 5883 * @ioc: Adapter object 5884 * @sz: DMA Pool size 5885 * Return: 0 for success, non-zero for failure. 5886 */ 5887 static int 5888 _base_allocate_sense_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz) 5889 { 5890 ioc->sense_dma_pool = 5891 dma_pool_create("sense pool", &ioc->pdev->dev, sz, 4, 0); 5892 if (!ioc->sense_dma_pool) 5893 return -ENOMEM; 5894 ioc->sense = dma_pool_alloc(ioc->sense_dma_pool, 5895 GFP_KERNEL, &ioc->sense_dma); 5896 if (!ioc->sense) 5897 return -EAGAIN; 5898 if (!mpt3sas_check_same_4gb_region((long)ioc->sense, sz)) { 5899 dinitprintk(ioc, pr_err( 5900 "Bad Sense Pool! sense (0x%p) sense_dma = (0x%llx)\n", 5901 ioc->sense, (unsigned long long) ioc->sense_dma)); 5902 ioc->use_32bit_dma = true; 5903 return -EAGAIN; 5904 } 5905 ioc_info(ioc, 5906 "sense pool(0x%p) - dma(0x%llx): depth(%d), element_size(%d), pool_size (%d kB)\n", 5907 ioc->sense, (unsigned long long)ioc->sense_dma, 5908 ioc->scsiio_depth, SCSI_SENSE_BUFFERSIZE, sz/1024); 5909 return 0; 5910 } 5911 5912 /** 5913 * _base_allocate_reply_pool - Allocating DMA'able memory 5914 * for reply pool. 5915 * @ioc: Adapter object 5916 * @sz: DMA Pool size 5917 * Return: 0 for success, non-zero for failure. 5918 */ 5919 static int 5920 _base_allocate_reply_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz) 5921 { 5922 /* reply pool, 4 byte align */ 5923 ioc->reply_dma_pool = dma_pool_create("reply pool", 5924 &ioc->pdev->dev, sz, 4, 0); 5925 if (!ioc->reply_dma_pool) 5926 return -ENOMEM; 5927 ioc->reply = dma_pool_alloc(ioc->reply_dma_pool, GFP_KERNEL, 5928 &ioc->reply_dma); 5929 if (!ioc->reply) 5930 return -EAGAIN; 5931 if (!mpt3sas_check_same_4gb_region((long)ioc->reply_free, sz)) { 5932 dinitprintk(ioc, pr_err( 5933 "Bad Reply Pool! Reply (0x%p) Reply dma = (0x%llx)\n", 5934 ioc->reply, (unsigned long long) ioc->reply_dma)); 5935 ioc->use_32bit_dma = true; 5936 return -EAGAIN; 5937 } 5938 ioc->reply_dma_min_address = (u32)(ioc->reply_dma); 5939 ioc->reply_dma_max_address = (u32)(ioc->reply_dma) + sz; 5940 ioc_info(ioc, 5941 "reply pool(0x%p) - dma(0x%llx): depth(%d), frame_size(%d), pool_size(%d kB)\n", 5942 ioc->reply, (unsigned long long)ioc->reply_dma, 5943 ioc->reply_free_queue_depth, ioc->reply_sz, sz/1024); 5944 return 0; 5945 } 5946 5947 /** 5948 * _base_allocate_reply_free_dma_pool - Allocating DMA'able memory 5949 * for reply free dma pool. 5950 * @ioc: Adapter object 5951 * @sz: DMA Pool size 5952 * Return: 0 for success, non-zero for failure. 5953 */ 5954 static int 5955 _base_allocate_reply_free_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz) 5956 { 5957 /* reply free queue, 16 byte align */ 5958 ioc->reply_free_dma_pool = dma_pool_create( 5959 "reply_free pool", &ioc->pdev->dev, sz, 16, 0); 5960 if (!ioc->reply_free_dma_pool) 5961 return -ENOMEM; 5962 ioc->reply_free = dma_pool_alloc(ioc->reply_free_dma_pool, 5963 GFP_KERNEL, &ioc->reply_free_dma); 5964 if (!ioc->reply_free) 5965 return -EAGAIN; 5966 if (!mpt3sas_check_same_4gb_region((long)ioc->reply_free, sz)) { 5967 dinitprintk(ioc, 5968 pr_err("Bad Reply Free Pool! Reply Free (0x%p) Reply Free dma = (0x%llx)\n", 5969 ioc->reply_free, (unsigned long long) ioc->reply_free_dma)); 5970 ioc->use_32bit_dma = true; 5971 return -EAGAIN; 5972 } 5973 memset(ioc->reply_free, 0, sz); 5974 dinitprintk(ioc, ioc_info(ioc, 5975 "reply_free pool(0x%p): depth(%d), element_size(%d), pool_size(%d kB)\n", 5976 ioc->reply_free, ioc->reply_free_queue_depth, 4, sz/1024)); 5977 dinitprintk(ioc, ioc_info(ioc, 5978 "reply_free_dma (0x%llx)\n", 5979 (unsigned long long)ioc->reply_free_dma)); 5980 return 0; 5981 } 5982 5983 /** 5984 * _base_allocate_reply_post_free_array - Allocating DMA'able memory 5985 * for reply post free array. 5986 * @ioc: Adapter object 5987 * @reply_post_free_array_sz: DMA Pool size 5988 * Return: 0 for success, non-zero for failure. 5989 */ 5990 5991 static int 5992 _base_allocate_reply_post_free_array(struct MPT3SAS_ADAPTER *ioc, 5993 u32 reply_post_free_array_sz) 5994 { 5995 ioc->reply_post_free_array_dma_pool = 5996 dma_pool_create("reply_post_free_array pool", 5997 &ioc->pdev->dev, reply_post_free_array_sz, 16, 0); 5998 if (!ioc->reply_post_free_array_dma_pool) 5999 return -ENOMEM; 6000 ioc->reply_post_free_array = 6001 dma_pool_alloc(ioc->reply_post_free_array_dma_pool, 6002 GFP_KERNEL, &ioc->reply_post_free_array_dma); 6003 if (!ioc->reply_post_free_array) 6004 return -EAGAIN; 6005 if (!mpt3sas_check_same_4gb_region((long)ioc->reply_post_free_array, 6006 reply_post_free_array_sz)) { 6007 dinitprintk(ioc, pr_err( 6008 "Bad Reply Free Pool! Reply Free (0x%p) Reply Free dma = (0x%llx)\n", 6009 ioc->reply_free, 6010 (unsigned long long) ioc->reply_free_dma)); 6011 ioc->use_32bit_dma = true; 6012 return -EAGAIN; 6013 } 6014 return 0; 6015 } 6016 /** 6017 * base_alloc_rdpq_dma_pool - Allocating DMA'able memory 6018 * for reply queues. 6019 * @ioc: per adapter object 6020 * @sz: DMA Pool size 6021 * Return: 0 for success, non-zero for failure. 6022 */ 6023 static int 6024 base_alloc_rdpq_dma_pool(struct MPT3SAS_ADAPTER *ioc, int sz) 6025 { 6026 int i = 0; 6027 u32 dma_alloc_count = 0; 6028 int reply_post_free_sz = ioc->reply_post_queue_depth * 6029 sizeof(Mpi2DefaultReplyDescriptor_t); 6030 int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1; 6031 6032 ioc->reply_post = kcalloc(count, sizeof(struct reply_post_struct), 6033 GFP_KERNEL); 6034 if (!ioc->reply_post) 6035 return -ENOMEM; 6036 /* 6037 * For INVADER_SERIES each set of 8 reply queues(0-7, 8-15, ..) and 6038 * VENTURA_SERIES each set of 16 reply queues(0-15, 16-31, ..) should 6039 * be within 4GB boundary i.e reply queues in a set must have same 6040 * upper 32-bits in their memory address. so here driver is allocating 6041 * the DMA'able memory for reply queues according. 6042 * Driver uses limitation of 6043 * VENTURA_SERIES to manage INVADER_SERIES as well. 6044 */ 6045 dma_alloc_count = DIV_ROUND_UP(count, 6046 RDPQ_MAX_INDEX_IN_ONE_CHUNK); 6047 ioc->reply_post_free_dma_pool = 6048 dma_pool_create("reply_post_free pool", 6049 &ioc->pdev->dev, sz, 16, 0); 6050 if (!ioc->reply_post_free_dma_pool) 6051 return -ENOMEM; 6052 for (i = 0; i < count; i++) { 6053 if ((i % RDPQ_MAX_INDEX_IN_ONE_CHUNK == 0) && dma_alloc_count) { 6054 ioc->reply_post[i].reply_post_free = 6055 dma_pool_zalloc(ioc->reply_post_free_dma_pool, 6056 GFP_KERNEL, 6057 &ioc->reply_post[i].reply_post_free_dma); 6058 if (!ioc->reply_post[i].reply_post_free) 6059 return -ENOMEM; 6060 /* 6061 * Each set of RDPQ pool must satisfy 4gb boundary 6062 * restriction. 6063 * 1) Check if allocated resources for RDPQ pool are in 6064 * the same 4GB range. 6065 * 2) If #1 is true, continue with 64 bit DMA. 6066 * 3) If #1 is false, return 1. which means free all the 6067 * resources and set DMA mask to 32 and allocate. 6068 */ 6069 if (!mpt3sas_check_same_4gb_region( 6070 (long)ioc->reply_post[i].reply_post_free, sz)) { 6071 dinitprintk(ioc, 6072 ioc_err(ioc, "bad Replypost free pool(0x%p)" 6073 "reply_post_free_dma = (0x%llx)\n", 6074 ioc->reply_post[i].reply_post_free, 6075 (unsigned long long) 6076 ioc->reply_post[i].reply_post_free_dma)); 6077 return -EAGAIN; 6078 } 6079 dma_alloc_count--; 6080 6081 } else { 6082 ioc->reply_post[i].reply_post_free = 6083 (Mpi2ReplyDescriptorsUnion_t *) 6084 ((long)ioc->reply_post[i-1].reply_post_free 6085 + reply_post_free_sz); 6086 ioc->reply_post[i].reply_post_free_dma = 6087 (dma_addr_t) 6088 (ioc->reply_post[i-1].reply_post_free_dma + 6089 reply_post_free_sz); 6090 } 6091 } 6092 return 0; 6093 } 6094 6095 /** 6096 * _base_allocate_memory_pools - allocate start of day memory pools 6097 * @ioc: per adapter object 6098 * 6099 * Return: 0 success, anything else error. 6100 */ 6101 static int 6102 _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc) 6103 { 6104 struct mpt3sas_facts *facts; 6105 u16 max_sge_elements; 6106 u16 chains_needed_per_io; 6107 u32 sz, total_sz, reply_post_free_sz, reply_post_free_array_sz; 6108 u32 retry_sz; 6109 u32 rdpq_sz = 0, sense_sz = 0; 6110 u16 max_request_credit, nvme_blocks_needed; 6111 unsigned short sg_tablesize; 6112 u16 sge_size; 6113 int i; 6114 int ret = 0, rc = 0; 6115 6116 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 6117 6118 6119 retry_sz = 0; 6120 facts = &ioc->facts; 6121 6122 /* command line tunables for max sgl entries */ 6123 if (max_sgl_entries != -1) 6124 sg_tablesize = max_sgl_entries; 6125 else { 6126 if (ioc->hba_mpi_version_belonged == MPI2_VERSION) 6127 sg_tablesize = MPT2SAS_SG_DEPTH; 6128 else 6129 sg_tablesize = MPT3SAS_SG_DEPTH; 6130 } 6131 6132 /* max sgl entries <= MPT_KDUMP_MIN_PHYS_SEGMENTS in KDUMP mode */ 6133 if (reset_devices) 6134 sg_tablesize = min_t(unsigned short, sg_tablesize, 6135 MPT_KDUMP_MIN_PHYS_SEGMENTS); 6136 6137 if (ioc->is_mcpu_endpoint) 6138 ioc->shost->sg_tablesize = MPT_MIN_PHYS_SEGMENTS; 6139 else { 6140 if (sg_tablesize < MPT_MIN_PHYS_SEGMENTS) 6141 sg_tablesize = MPT_MIN_PHYS_SEGMENTS; 6142 else if (sg_tablesize > MPT_MAX_PHYS_SEGMENTS) { 6143 sg_tablesize = min_t(unsigned short, sg_tablesize, 6144 SG_MAX_SEGMENTS); 6145 ioc_warn(ioc, "sg_tablesize(%u) is bigger than kernel defined SG_CHUNK_SIZE(%u)\n", 6146 sg_tablesize, MPT_MAX_PHYS_SEGMENTS); 6147 } 6148 ioc->shost->sg_tablesize = sg_tablesize; 6149 } 6150 6151 ioc->internal_depth = min_t(int, (facts->HighPriorityCredit + (5)), 6152 (facts->RequestCredit / 4)); 6153 if (ioc->internal_depth < INTERNAL_CMDS_COUNT) { 6154 if (facts->RequestCredit <= (INTERNAL_CMDS_COUNT + 6155 INTERNAL_SCSIIO_CMDS_COUNT)) { 6156 ioc_err(ioc, "IOC doesn't have enough Request Credits, it has just %d number of credits\n", 6157 facts->RequestCredit); 6158 return -ENOMEM; 6159 } 6160 ioc->internal_depth = 10; 6161 } 6162 6163 ioc->hi_priority_depth = ioc->internal_depth - (5); 6164 /* command line tunables for max controller queue depth */ 6165 if (max_queue_depth != -1 && max_queue_depth != 0) { 6166 max_request_credit = min_t(u16, max_queue_depth + 6167 ioc->internal_depth, facts->RequestCredit); 6168 if (max_request_credit > MAX_HBA_QUEUE_DEPTH) 6169 max_request_credit = MAX_HBA_QUEUE_DEPTH; 6170 } else if (reset_devices) 6171 max_request_credit = min_t(u16, facts->RequestCredit, 6172 (MPT3SAS_KDUMP_SCSI_IO_DEPTH + ioc->internal_depth)); 6173 else 6174 max_request_credit = min_t(u16, facts->RequestCredit, 6175 MAX_HBA_QUEUE_DEPTH); 6176 6177 /* Firmware maintains additional facts->HighPriorityCredit number of 6178 * credits for HiPriprity Request messages, so hba queue depth will be 6179 * sum of max_request_credit and high priority queue depth. 6180 */ 6181 ioc->hba_queue_depth = max_request_credit + ioc->hi_priority_depth; 6182 6183 /* request frame size */ 6184 ioc->request_sz = facts->IOCRequestFrameSize * 4; 6185 6186 /* reply frame size */ 6187 ioc->reply_sz = facts->ReplyFrameSize * 4; 6188 6189 /* chain segment size */ 6190 if (ioc->hba_mpi_version_belonged != MPI2_VERSION) { 6191 if (facts->IOCMaxChainSegmentSize) 6192 ioc->chain_segment_sz = 6193 facts->IOCMaxChainSegmentSize * 6194 MAX_CHAIN_ELEMT_SZ; 6195 else 6196 /* set to 128 bytes size if IOCMaxChainSegmentSize is zero */ 6197 ioc->chain_segment_sz = DEFAULT_NUM_FWCHAIN_ELEMTS * 6198 MAX_CHAIN_ELEMT_SZ; 6199 } else 6200 ioc->chain_segment_sz = ioc->request_sz; 6201 6202 /* calculate the max scatter element size */ 6203 sge_size = max_t(u16, ioc->sge_size, ioc->sge_size_ieee); 6204 6205 retry_allocation: 6206 total_sz = 0; 6207 /* calculate number of sg elements left over in the 1st frame */ 6208 max_sge_elements = ioc->request_sz - ((sizeof(Mpi2SCSIIORequest_t) - 6209 sizeof(Mpi2SGEIOUnion_t)) + sge_size); 6210 ioc->max_sges_in_main_message = max_sge_elements/sge_size; 6211 6212 /* now do the same for a chain buffer */ 6213 max_sge_elements = ioc->chain_segment_sz - sge_size; 6214 ioc->max_sges_in_chain_message = max_sge_elements/sge_size; 6215 6216 /* 6217 * MPT3SAS_SG_DEPTH = CONFIG_FUSION_MAX_SGE 6218 */ 6219 chains_needed_per_io = ((ioc->shost->sg_tablesize - 6220 ioc->max_sges_in_main_message)/ioc->max_sges_in_chain_message) 6221 + 1; 6222 if (chains_needed_per_io > facts->MaxChainDepth) { 6223 chains_needed_per_io = facts->MaxChainDepth; 6224 ioc->shost->sg_tablesize = min_t(u16, 6225 ioc->max_sges_in_main_message + (ioc->max_sges_in_chain_message 6226 * chains_needed_per_io), ioc->shost->sg_tablesize); 6227 } 6228 ioc->chains_needed_per_io = chains_needed_per_io; 6229 6230 /* reply free queue sizing - taking into account for 64 FW events */ 6231 ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64; 6232 6233 /* mCPU manage single counters for simplicity */ 6234 if (ioc->is_mcpu_endpoint) 6235 ioc->reply_post_queue_depth = ioc->reply_free_queue_depth; 6236 else { 6237 /* calculate reply descriptor post queue depth */ 6238 ioc->reply_post_queue_depth = ioc->hba_queue_depth + 6239 ioc->reply_free_queue_depth + 1; 6240 /* align the reply post queue on the next 16 count boundary */ 6241 if (ioc->reply_post_queue_depth % 16) 6242 ioc->reply_post_queue_depth += 16 - 6243 (ioc->reply_post_queue_depth % 16); 6244 } 6245 6246 if (ioc->reply_post_queue_depth > 6247 facts->MaxReplyDescriptorPostQueueDepth) { 6248 ioc->reply_post_queue_depth = 6249 facts->MaxReplyDescriptorPostQueueDepth - 6250 (facts->MaxReplyDescriptorPostQueueDepth % 16); 6251 ioc->hba_queue_depth = 6252 ((ioc->reply_post_queue_depth - 64) / 2) - 1; 6253 ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64; 6254 } 6255 6256 ioc_info(ioc, 6257 "scatter gather: sge_in_main_msg(%d), sge_per_chain(%d), " 6258 "sge_per_io(%d), chains_per_io(%d)\n", 6259 ioc->max_sges_in_main_message, 6260 ioc->max_sges_in_chain_message, 6261 ioc->shost->sg_tablesize, 6262 ioc->chains_needed_per_io); 6263 6264 /* reply post queue, 16 byte align */ 6265 reply_post_free_sz = ioc->reply_post_queue_depth * 6266 sizeof(Mpi2DefaultReplyDescriptor_t); 6267 rdpq_sz = reply_post_free_sz * RDPQ_MAX_INDEX_IN_ONE_CHUNK; 6268 if ((_base_is_controller_msix_enabled(ioc) && !ioc->rdpq_array_enable) 6269 || (ioc->reply_queue_count < RDPQ_MAX_INDEX_IN_ONE_CHUNK)) 6270 rdpq_sz = reply_post_free_sz * ioc->reply_queue_count; 6271 ret = base_alloc_rdpq_dma_pool(ioc, rdpq_sz); 6272 if (ret == -EAGAIN) { 6273 /* 6274 * Free allocated bad RDPQ memory pools. 6275 * Change dma coherent mask to 32 bit and reallocate RDPQ 6276 */ 6277 _base_release_memory_pools(ioc); 6278 ioc->use_32bit_dma = true; 6279 if (_base_config_dma_addressing(ioc, ioc->pdev) != 0) { 6280 ioc_err(ioc, 6281 "32 DMA mask failed %s\n", pci_name(ioc->pdev)); 6282 return -ENODEV; 6283 } 6284 if (base_alloc_rdpq_dma_pool(ioc, rdpq_sz)) 6285 return -ENOMEM; 6286 } else if (ret == -ENOMEM) 6287 return -ENOMEM; 6288 total_sz = rdpq_sz * (!ioc->rdpq_array_enable ? 1 : 6289 DIV_ROUND_UP(ioc->reply_queue_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK)); 6290 ioc->scsiio_depth = ioc->hba_queue_depth - 6291 ioc->hi_priority_depth - ioc->internal_depth; 6292 6293 /* set the scsi host can_queue depth 6294 * with some internal commands that could be outstanding 6295 */ 6296 ioc->shost->can_queue = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT; 6297 dinitprintk(ioc, 6298 ioc_info(ioc, "scsi host: can_queue depth (%d)\n", 6299 ioc->shost->can_queue)); 6300 6301 /* contiguous pool for request and chains, 16 byte align, one extra " 6302 * "frame for smid=0 6303 */ 6304 ioc->chain_depth = ioc->chains_needed_per_io * ioc->scsiio_depth; 6305 sz = ((ioc->scsiio_depth + 1) * ioc->request_sz); 6306 6307 /* hi-priority queue */ 6308 sz += (ioc->hi_priority_depth * ioc->request_sz); 6309 6310 /* internal queue */ 6311 sz += (ioc->internal_depth * ioc->request_sz); 6312 6313 ioc->request_dma_sz = sz; 6314 ioc->request = dma_alloc_coherent(&ioc->pdev->dev, sz, 6315 &ioc->request_dma, GFP_KERNEL); 6316 if (!ioc->request) { 6317 ioc_err(ioc, "request pool: dma_alloc_coherent failed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), total(%d kB)\n", 6318 ioc->hba_queue_depth, ioc->chains_needed_per_io, 6319 ioc->request_sz, sz / 1024); 6320 if (ioc->scsiio_depth < MPT3SAS_SAS_QUEUE_DEPTH) 6321 goto out; 6322 retry_sz = 64; 6323 ioc->hba_queue_depth -= retry_sz; 6324 _base_release_memory_pools(ioc); 6325 goto retry_allocation; 6326 } 6327 6328 if (retry_sz) 6329 ioc_err(ioc, "request pool: dma_alloc_coherent succeed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), total(%d kb)\n", 6330 ioc->hba_queue_depth, ioc->chains_needed_per_io, 6331 ioc->request_sz, sz / 1024); 6332 6333 /* hi-priority queue */ 6334 ioc->hi_priority = ioc->request + ((ioc->scsiio_depth + 1) * 6335 ioc->request_sz); 6336 ioc->hi_priority_dma = ioc->request_dma + ((ioc->scsiio_depth + 1) * 6337 ioc->request_sz); 6338 6339 /* internal queue */ 6340 ioc->internal = ioc->hi_priority + (ioc->hi_priority_depth * 6341 ioc->request_sz); 6342 ioc->internal_dma = ioc->hi_priority_dma + (ioc->hi_priority_depth * 6343 ioc->request_sz); 6344 6345 ioc_info(ioc, 6346 "request pool(0x%p) - dma(0x%llx): " 6347 "depth(%d), frame_size(%d), pool_size(%d kB)\n", 6348 ioc->request, (unsigned long long) ioc->request_dma, 6349 ioc->hba_queue_depth, ioc->request_sz, 6350 (ioc->hba_queue_depth * ioc->request_sz) / 1024); 6351 6352 total_sz += sz; 6353 6354 dinitprintk(ioc, 6355 ioc_info(ioc, "scsiio(0x%p): depth(%d)\n", 6356 ioc->request, ioc->scsiio_depth)); 6357 6358 ioc->chain_depth = min_t(u32, ioc->chain_depth, MAX_CHAIN_DEPTH); 6359 sz = ioc->scsiio_depth * sizeof(struct chain_lookup); 6360 ioc->chain_lookup = kzalloc(sz, GFP_KERNEL); 6361 if (!ioc->chain_lookup) { 6362 ioc_err(ioc, "chain_lookup: __get_free_pages failed\n"); 6363 goto out; 6364 } 6365 6366 sz = ioc->chains_needed_per_io * sizeof(struct chain_tracker); 6367 for (i = 0; i < ioc->scsiio_depth; i++) { 6368 ioc->chain_lookup[i].chains_per_smid = kzalloc(sz, GFP_KERNEL); 6369 if (!ioc->chain_lookup[i].chains_per_smid) { 6370 ioc_err(ioc, "chain_lookup: kzalloc failed\n"); 6371 goto out; 6372 } 6373 } 6374 6375 /* initialize hi-priority queue smid's */ 6376 ioc->hpr_lookup = kcalloc(ioc->hi_priority_depth, 6377 sizeof(struct request_tracker), GFP_KERNEL); 6378 if (!ioc->hpr_lookup) { 6379 ioc_err(ioc, "hpr_lookup: kcalloc failed\n"); 6380 goto out; 6381 } 6382 ioc->hi_priority_smid = ioc->scsiio_depth + 1; 6383 dinitprintk(ioc, 6384 ioc_info(ioc, "hi_priority(0x%p): depth(%d), start smid(%d)\n", 6385 ioc->hi_priority, 6386 ioc->hi_priority_depth, ioc->hi_priority_smid)); 6387 6388 /* initialize internal queue smid's */ 6389 ioc->internal_lookup = kcalloc(ioc->internal_depth, 6390 sizeof(struct request_tracker), GFP_KERNEL); 6391 if (!ioc->internal_lookup) { 6392 ioc_err(ioc, "internal_lookup: kcalloc failed\n"); 6393 goto out; 6394 } 6395 ioc->internal_smid = ioc->hi_priority_smid + ioc->hi_priority_depth; 6396 dinitprintk(ioc, 6397 ioc_info(ioc, "internal(0x%p): depth(%d), start smid(%d)\n", 6398 ioc->internal, 6399 ioc->internal_depth, ioc->internal_smid)); 6400 6401 ioc->io_queue_num = kcalloc(ioc->scsiio_depth, 6402 sizeof(u16), GFP_KERNEL); 6403 if (!ioc->io_queue_num) 6404 goto out; 6405 /* 6406 * The number of NVMe page sized blocks needed is: 6407 * (((sg_tablesize * 8) - 1) / (page_size - 8)) + 1 6408 * ((sg_tablesize * 8) - 1) is the max PRP's minus the first PRP entry 6409 * that is placed in the main message frame. 8 is the size of each PRP 6410 * entry or PRP list pointer entry. 8 is subtracted from page_size 6411 * because of the PRP list pointer entry at the end of a page, so this 6412 * is not counted as a PRP entry. The 1 added page is a round up. 6413 * 6414 * To avoid allocation failures due to the amount of memory that could 6415 * be required for NVMe PRP's, only each set of NVMe blocks will be 6416 * contiguous, so a new set is allocated for each possible I/O. 6417 */ 6418 6419 ioc->chains_per_prp_buffer = 0; 6420 if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) { 6421 nvme_blocks_needed = 6422 (ioc->shost->sg_tablesize * NVME_PRP_SIZE) - 1; 6423 nvme_blocks_needed /= (ioc->page_size - NVME_PRP_SIZE); 6424 nvme_blocks_needed++; 6425 6426 sz = sizeof(struct pcie_sg_list) * ioc->scsiio_depth; 6427 ioc->pcie_sg_lookup = kzalloc(sz, GFP_KERNEL); 6428 if (!ioc->pcie_sg_lookup) { 6429 ioc_info(ioc, "PCIe SGL lookup: kzalloc failed\n"); 6430 goto out; 6431 } 6432 sz = nvme_blocks_needed * ioc->page_size; 6433 rc = _base_allocate_pcie_sgl_pool(ioc, sz); 6434 if (rc == -ENOMEM) 6435 return -ENOMEM; 6436 else if (rc == -EAGAIN) 6437 goto try_32bit_dma; 6438 total_sz += sz * ioc->scsiio_depth; 6439 } 6440 6441 rc = _base_allocate_chain_dma_pool(ioc, ioc->chain_segment_sz); 6442 if (rc == -ENOMEM) 6443 return -ENOMEM; 6444 else if (rc == -EAGAIN) 6445 goto try_32bit_dma; 6446 total_sz += ioc->chain_segment_sz * ((ioc->chains_needed_per_io - 6447 ioc->chains_per_prp_buffer) * ioc->scsiio_depth); 6448 dinitprintk(ioc, 6449 ioc_info(ioc, "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n", 6450 ioc->chain_depth, ioc->chain_segment_sz, 6451 (ioc->chain_depth * ioc->chain_segment_sz) / 1024)); 6452 /* sense buffers, 4 byte align */ 6453 sense_sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE; 6454 rc = _base_allocate_sense_dma_pool(ioc, sense_sz); 6455 if (rc == -ENOMEM) 6456 return -ENOMEM; 6457 else if (rc == -EAGAIN) 6458 goto try_32bit_dma; 6459 total_sz += sense_sz; 6460 ioc_info(ioc, 6461 "sense pool(0x%p)- dma(0x%llx): depth(%d)," 6462 "element_size(%d), pool_size(%d kB)\n", 6463 ioc->sense, (unsigned long long)ioc->sense_dma, ioc->scsiio_depth, 6464 SCSI_SENSE_BUFFERSIZE, sz / 1024); 6465 /* reply pool, 4 byte align */ 6466 sz = ioc->reply_free_queue_depth * ioc->reply_sz; 6467 rc = _base_allocate_reply_pool(ioc, sz); 6468 if (rc == -ENOMEM) 6469 return -ENOMEM; 6470 else if (rc == -EAGAIN) 6471 goto try_32bit_dma; 6472 total_sz += sz; 6473 6474 /* reply free queue, 16 byte align */ 6475 sz = ioc->reply_free_queue_depth * 4; 6476 rc = _base_allocate_reply_free_dma_pool(ioc, sz); 6477 if (rc == -ENOMEM) 6478 return -ENOMEM; 6479 else if (rc == -EAGAIN) 6480 goto try_32bit_dma; 6481 dinitprintk(ioc, 6482 ioc_info(ioc, "reply_free_dma (0x%llx)\n", 6483 (unsigned long long)ioc->reply_free_dma)); 6484 total_sz += sz; 6485 if (ioc->rdpq_array_enable) { 6486 reply_post_free_array_sz = ioc->reply_queue_count * 6487 sizeof(Mpi2IOCInitRDPQArrayEntry); 6488 rc = _base_allocate_reply_post_free_array(ioc, 6489 reply_post_free_array_sz); 6490 if (rc == -ENOMEM) 6491 return -ENOMEM; 6492 else if (rc == -EAGAIN) 6493 goto try_32bit_dma; 6494 } 6495 ioc->config_page_sz = 512; 6496 ioc->config_page = dma_alloc_coherent(&ioc->pdev->dev, 6497 ioc->config_page_sz, &ioc->config_page_dma, GFP_KERNEL); 6498 if (!ioc->config_page) { 6499 ioc_err(ioc, "config page: dma_pool_alloc failed\n"); 6500 goto out; 6501 } 6502 6503 ioc_info(ioc, "config page(0x%p) - dma(0x%llx): size(%d)\n", 6504 ioc->config_page, (unsigned long long)ioc->config_page_dma, 6505 ioc->config_page_sz); 6506 total_sz += ioc->config_page_sz; 6507 6508 ioc_info(ioc, "Allocated physical memory: size(%d kB)\n", 6509 total_sz / 1024); 6510 ioc_info(ioc, "Current Controller Queue Depth(%d),Max Controller Queue Depth(%d)\n", 6511 ioc->shost->can_queue, facts->RequestCredit); 6512 ioc_info(ioc, "Scatter Gather Elements per IO(%d)\n", 6513 ioc->shost->sg_tablesize); 6514 return 0; 6515 6516 try_32bit_dma: 6517 _base_release_memory_pools(ioc); 6518 if (ioc->use_32bit_dma && (ioc->dma_mask > 32)) { 6519 /* Change dma coherent mask to 32 bit and reallocate */ 6520 if (_base_config_dma_addressing(ioc, ioc->pdev) != 0) { 6521 pr_err("Setting 32 bit coherent DMA mask Failed %s\n", 6522 pci_name(ioc->pdev)); 6523 return -ENODEV; 6524 } 6525 } else if (_base_reduce_hba_queue_depth(ioc) != 0) 6526 return -ENOMEM; 6527 goto retry_allocation; 6528 6529 out: 6530 return -ENOMEM; 6531 } 6532 6533 /** 6534 * mpt3sas_base_get_iocstate - Get the current state of a MPT adapter. 6535 * @ioc: Pointer to MPT_ADAPTER structure 6536 * @cooked: Request raw or cooked IOC state 6537 * 6538 * Return: all IOC Doorbell register bits if cooked==0, else just the 6539 * Doorbell bits in MPI_IOC_STATE_MASK. 6540 */ 6541 u32 6542 mpt3sas_base_get_iocstate(struct MPT3SAS_ADAPTER *ioc, int cooked) 6543 { 6544 u32 s, sc; 6545 6546 s = ioc->base_readl(&ioc->chip->Doorbell); 6547 sc = s & MPI2_IOC_STATE_MASK; 6548 return cooked ? sc : s; 6549 } 6550 6551 /** 6552 * _base_wait_on_iocstate - waiting on a particular ioc state 6553 * @ioc: ? 6554 * @ioc_state: controller state { READY, OPERATIONAL, or RESET } 6555 * @timeout: timeout in second 6556 * 6557 * Return: 0 for success, non-zero for failure. 6558 */ 6559 static int 6560 _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, u32 ioc_state, int timeout) 6561 { 6562 u32 count, cntdn; 6563 u32 current_state; 6564 6565 count = 0; 6566 cntdn = 1000 * timeout; 6567 do { 6568 current_state = mpt3sas_base_get_iocstate(ioc, 1); 6569 if (current_state == ioc_state) 6570 return 0; 6571 if (count && current_state == MPI2_IOC_STATE_FAULT) 6572 break; 6573 if (count && current_state == MPI2_IOC_STATE_COREDUMP) 6574 break; 6575 6576 usleep_range(1000, 1500); 6577 count++; 6578 } while (--cntdn); 6579 6580 return current_state; 6581 } 6582 6583 /** 6584 * _base_dump_reg_set - This function will print hexdump of register set. 6585 * @ioc: per adapter object 6586 * 6587 * Return: nothing. 6588 */ 6589 static inline void 6590 _base_dump_reg_set(struct MPT3SAS_ADAPTER *ioc) 6591 { 6592 unsigned int i, sz = 256; 6593 u32 __iomem *reg = (u32 __iomem *)ioc->chip; 6594 6595 ioc_info(ioc, "System Register set:\n"); 6596 for (i = 0; i < (sz / sizeof(u32)); i++) 6597 pr_info("%08x: %08x\n", (i * 4), readl(®[i])); 6598 } 6599 6600 /** 6601 * _base_wait_for_doorbell_int - waiting for controller interrupt(generated by 6602 * a write to the doorbell) 6603 * @ioc: per adapter object 6604 * @timeout: timeout in seconds 6605 * 6606 * Return: 0 for success, non-zero for failure. 6607 * 6608 * Notes: MPI2_HIS_IOC2SYS_DB_STATUS - set to one when IOC writes to doorbell. 6609 */ 6610 6611 static int 6612 _base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout) 6613 { 6614 u32 cntdn, count; 6615 u32 int_status; 6616 6617 count = 0; 6618 cntdn = 1000 * timeout; 6619 do { 6620 int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus); 6621 if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) { 6622 dhsprintk(ioc, 6623 ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n", 6624 __func__, count, timeout)); 6625 return 0; 6626 } 6627 6628 usleep_range(1000, 1500); 6629 count++; 6630 } while (--cntdn); 6631 6632 ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n", 6633 __func__, count, int_status); 6634 return -EFAULT; 6635 } 6636 6637 static int 6638 _base_spin_on_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout) 6639 { 6640 u32 cntdn, count; 6641 u32 int_status; 6642 6643 count = 0; 6644 cntdn = 2000 * timeout; 6645 do { 6646 int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus); 6647 if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) { 6648 dhsprintk(ioc, 6649 ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n", 6650 __func__, count, timeout)); 6651 return 0; 6652 } 6653 6654 udelay(500); 6655 count++; 6656 } while (--cntdn); 6657 6658 ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n", 6659 __func__, count, int_status); 6660 return -EFAULT; 6661 6662 } 6663 6664 /** 6665 * _base_wait_for_doorbell_ack - waiting for controller to read the doorbell. 6666 * @ioc: per adapter object 6667 * @timeout: timeout in second 6668 * 6669 * Return: 0 for success, non-zero for failure. 6670 * 6671 * Notes: MPI2_HIS_SYS2IOC_DB_STATUS - set to one when host writes to 6672 * doorbell. 6673 */ 6674 static int 6675 _base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout) 6676 { 6677 u32 cntdn, count; 6678 u32 int_status; 6679 u32 doorbell; 6680 6681 count = 0; 6682 cntdn = 1000 * timeout; 6683 do { 6684 int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus); 6685 if (!(int_status & MPI2_HIS_SYS2IOC_DB_STATUS)) { 6686 dhsprintk(ioc, 6687 ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n", 6688 __func__, count, timeout)); 6689 return 0; 6690 } else if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) { 6691 doorbell = ioc->base_readl(&ioc->chip->Doorbell); 6692 if ((doorbell & MPI2_IOC_STATE_MASK) == 6693 MPI2_IOC_STATE_FAULT) { 6694 mpt3sas_print_fault_code(ioc, doorbell); 6695 return -EFAULT; 6696 } 6697 if ((doorbell & MPI2_IOC_STATE_MASK) == 6698 MPI2_IOC_STATE_COREDUMP) { 6699 mpt3sas_print_coredump_info(ioc, doorbell); 6700 return -EFAULT; 6701 } 6702 } else if (int_status == 0xFFFFFFFF) 6703 goto out; 6704 6705 usleep_range(1000, 1500); 6706 count++; 6707 } while (--cntdn); 6708 6709 out: 6710 ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n", 6711 __func__, count, int_status); 6712 return -EFAULT; 6713 } 6714 6715 /** 6716 * _base_wait_for_doorbell_not_used - waiting for doorbell to not be in use 6717 * @ioc: per adapter object 6718 * @timeout: timeout in second 6719 * 6720 * Return: 0 for success, non-zero for failure. 6721 */ 6722 static int 6723 _base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout) 6724 { 6725 u32 cntdn, count; 6726 u32 doorbell_reg; 6727 6728 count = 0; 6729 cntdn = 1000 * timeout; 6730 do { 6731 doorbell_reg = ioc->base_readl(&ioc->chip->Doorbell); 6732 if (!(doorbell_reg & MPI2_DOORBELL_USED)) { 6733 dhsprintk(ioc, 6734 ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n", 6735 __func__, count, timeout)); 6736 return 0; 6737 } 6738 6739 usleep_range(1000, 1500); 6740 count++; 6741 } while (--cntdn); 6742 6743 ioc_err(ioc, "%s: failed due to timeout count(%d), doorbell_reg(%x)!\n", 6744 __func__, count, doorbell_reg); 6745 return -EFAULT; 6746 } 6747 6748 /** 6749 * _base_send_ioc_reset - send doorbell reset 6750 * @ioc: per adapter object 6751 * @reset_type: currently only supports: MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET 6752 * @timeout: timeout in second 6753 * 6754 * Return: 0 for success, non-zero for failure. 6755 */ 6756 static int 6757 _base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout) 6758 { 6759 u32 ioc_state; 6760 int r = 0; 6761 unsigned long flags; 6762 6763 if (reset_type != MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET) { 6764 ioc_err(ioc, "%s: unknown reset_type\n", __func__); 6765 return -EFAULT; 6766 } 6767 6768 if (!(ioc->facts.IOCCapabilities & 6769 MPI2_IOCFACTS_CAPABILITY_EVENT_REPLAY)) 6770 return -EFAULT; 6771 6772 ioc_info(ioc, "sending message unit reset !!\n"); 6773 6774 writel(reset_type << MPI2_DOORBELL_FUNCTION_SHIFT, 6775 &ioc->chip->Doorbell); 6776 if ((_base_wait_for_doorbell_ack(ioc, 15))) { 6777 r = -EFAULT; 6778 goto out; 6779 } 6780 6781 ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout); 6782 if (ioc_state) { 6783 ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n", 6784 __func__, ioc_state); 6785 r = -EFAULT; 6786 goto out; 6787 } 6788 out: 6789 if (r != 0) { 6790 ioc_state = mpt3sas_base_get_iocstate(ioc, 0); 6791 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); 6792 /* 6793 * Wait for IOC state CoreDump to clear only during 6794 * HBA initialization & release time. 6795 */ 6796 if ((ioc_state & MPI2_IOC_STATE_MASK) == 6797 MPI2_IOC_STATE_COREDUMP && (ioc->is_driver_loading == 1 || 6798 ioc->fault_reset_work_q == NULL)) { 6799 spin_unlock_irqrestore( 6800 &ioc->ioc_reset_in_progress_lock, flags); 6801 mpt3sas_print_coredump_info(ioc, ioc_state); 6802 mpt3sas_base_wait_for_coredump_completion(ioc, 6803 __func__); 6804 spin_lock_irqsave( 6805 &ioc->ioc_reset_in_progress_lock, flags); 6806 } 6807 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); 6808 } 6809 ioc_info(ioc, "message unit reset: %s\n", 6810 r == 0 ? "SUCCESS" : "FAILED"); 6811 return r; 6812 } 6813 6814 /** 6815 * mpt3sas_wait_for_ioc - IOC's operational state is checked here. 6816 * @ioc: per adapter object 6817 * @timeout: timeout in seconds 6818 * 6819 * Return: Waits up to timeout seconds for the IOC to 6820 * become operational. Returns 0 if IOC is present 6821 * and operational; otherwise returns %-EFAULT. 6822 */ 6823 6824 int 6825 mpt3sas_wait_for_ioc(struct MPT3SAS_ADAPTER *ioc, int timeout) 6826 { 6827 int wait_state_count = 0; 6828 u32 ioc_state; 6829 6830 do { 6831 ioc_state = mpt3sas_base_get_iocstate(ioc, 1); 6832 if (ioc_state == MPI2_IOC_STATE_OPERATIONAL) 6833 break; 6834 6835 /* 6836 * Watchdog thread will be started after IOC Initialization, so 6837 * no need to wait here for IOC state to become operational 6838 * when IOC Initialization is on. Instead the driver will 6839 * return ETIME status, so that calling function can issue 6840 * diag reset operation and retry the command. 6841 */ 6842 if (ioc->is_driver_loading) 6843 return -ETIME; 6844 6845 ssleep(1); 6846 ioc_info(ioc, "%s: waiting for operational state(count=%d)\n", 6847 __func__, ++wait_state_count); 6848 } while (--timeout); 6849 if (!timeout) { 6850 ioc_err(ioc, "%s: failed due to ioc not operational\n", __func__); 6851 return -EFAULT; 6852 } 6853 if (wait_state_count) 6854 ioc_info(ioc, "ioc is operational\n"); 6855 return 0; 6856 } 6857 6858 /** 6859 * _base_handshake_req_reply_wait - send request thru doorbell interface 6860 * @ioc: per adapter object 6861 * @request_bytes: request length 6862 * @request: pointer having request payload 6863 * @reply_bytes: reply length 6864 * @reply: pointer to reply payload 6865 * @timeout: timeout in second 6866 * 6867 * Return: 0 for success, non-zero for failure. 6868 */ 6869 static int 6870 _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes, 6871 u32 *request, int reply_bytes, u16 *reply, int timeout) 6872 { 6873 MPI2DefaultReply_t *default_reply = (MPI2DefaultReply_t *)reply; 6874 int i; 6875 u8 failed; 6876 __le32 *mfp; 6877 6878 /* make sure doorbell is not in use */ 6879 if ((ioc->base_readl(&ioc->chip->Doorbell) & MPI2_DOORBELL_USED)) { 6880 ioc_err(ioc, "doorbell is in use (line=%d)\n", __LINE__); 6881 return -EFAULT; 6882 } 6883 6884 /* clear pending doorbell interrupts from previous state changes */ 6885 if (ioc->base_readl(&ioc->chip->HostInterruptStatus) & 6886 MPI2_HIS_IOC2SYS_DB_STATUS) 6887 writel(0, &ioc->chip->HostInterruptStatus); 6888 6889 /* send message to ioc */ 6890 writel(((MPI2_FUNCTION_HANDSHAKE<<MPI2_DOORBELL_FUNCTION_SHIFT) | 6891 ((request_bytes/4)<<MPI2_DOORBELL_ADD_DWORDS_SHIFT)), 6892 &ioc->chip->Doorbell); 6893 6894 if ((_base_spin_on_doorbell_int(ioc, 5))) { 6895 ioc_err(ioc, "doorbell handshake int failed (line=%d)\n", 6896 __LINE__); 6897 return -EFAULT; 6898 } 6899 writel(0, &ioc->chip->HostInterruptStatus); 6900 6901 if ((_base_wait_for_doorbell_ack(ioc, 5))) { 6902 ioc_err(ioc, "doorbell handshake ack failed (line=%d)\n", 6903 __LINE__); 6904 return -EFAULT; 6905 } 6906 6907 /* send message 32-bits at a time */ 6908 for (i = 0, failed = 0; i < request_bytes/4 && !failed; i++) { 6909 writel(cpu_to_le32(request[i]), &ioc->chip->Doorbell); 6910 if ((_base_wait_for_doorbell_ack(ioc, 5))) 6911 failed = 1; 6912 } 6913 6914 if (failed) { 6915 ioc_err(ioc, "doorbell handshake sending request failed (line=%d)\n", 6916 __LINE__); 6917 return -EFAULT; 6918 } 6919 6920 /* now wait for the reply */ 6921 if ((_base_wait_for_doorbell_int(ioc, timeout))) { 6922 ioc_err(ioc, "doorbell handshake int failed (line=%d)\n", 6923 __LINE__); 6924 return -EFAULT; 6925 } 6926 6927 /* read the first two 16-bits, it gives the total length of the reply */ 6928 reply[0] = le16_to_cpu(ioc->base_readl(&ioc->chip->Doorbell) 6929 & MPI2_DOORBELL_DATA_MASK); 6930 writel(0, &ioc->chip->HostInterruptStatus); 6931 if ((_base_wait_for_doorbell_int(ioc, 5))) { 6932 ioc_err(ioc, "doorbell handshake int failed (line=%d)\n", 6933 __LINE__); 6934 return -EFAULT; 6935 } 6936 reply[1] = le16_to_cpu(ioc->base_readl(&ioc->chip->Doorbell) 6937 & MPI2_DOORBELL_DATA_MASK); 6938 writel(0, &ioc->chip->HostInterruptStatus); 6939 6940 for (i = 2; i < default_reply->MsgLength * 2; i++) { 6941 if ((_base_wait_for_doorbell_int(ioc, 5))) { 6942 ioc_err(ioc, "doorbell handshake int failed (line=%d)\n", 6943 __LINE__); 6944 return -EFAULT; 6945 } 6946 if (i >= reply_bytes/2) /* overflow case */ 6947 ioc->base_readl(&ioc->chip->Doorbell); 6948 else 6949 reply[i] = le16_to_cpu( 6950 ioc->base_readl(&ioc->chip->Doorbell) 6951 & MPI2_DOORBELL_DATA_MASK); 6952 writel(0, &ioc->chip->HostInterruptStatus); 6953 } 6954 6955 _base_wait_for_doorbell_int(ioc, 5); 6956 if (_base_wait_for_doorbell_not_used(ioc, 5) != 0) { 6957 dhsprintk(ioc, 6958 ioc_info(ioc, "doorbell is in use (line=%d)\n", 6959 __LINE__)); 6960 } 6961 writel(0, &ioc->chip->HostInterruptStatus); 6962 6963 if (ioc->logging_level & MPT_DEBUG_INIT) { 6964 mfp = (__le32 *)reply; 6965 pr_info("\toffset:data\n"); 6966 for (i = 0; i < reply_bytes/4; i++) 6967 ioc_info(ioc, "\t[0x%02x]:%08x\n", i*4, 6968 le32_to_cpu(mfp[i])); 6969 } 6970 return 0; 6971 } 6972 6973 /** 6974 * mpt3sas_base_sas_iounit_control - send sas iounit control to FW 6975 * @ioc: per adapter object 6976 * @mpi_reply: the reply payload from FW 6977 * @mpi_request: the request payload sent to FW 6978 * 6979 * The SAS IO Unit Control Request message allows the host to perform low-level 6980 * operations, such as resets on the PHYs of the IO Unit, also allows the host 6981 * to obtain the IOC assigned device handles for a device if it has other 6982 * identifying information about the device, in addition allows the host to 6983 * remove IOC resources associated with the device. 6984 * 6985 * Return: 0 for success, non-zero for failure. 6986 */ 6987 int 6988 mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc, 6989 Mpi2SasIoUnitControlReply_t *mpi_reply, 6990 Mpi2SasIoUnitControlRequest_t *mpi_request) 6991 { 6992 u16 smid; 6993 u8 issue_reset = 0; 6994 int rc; 6995 void *request; 6996 6997 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 6998 6999 mutex_lock(&ioc->base_cmds.mutex); 7000 7001 if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) { 7002 ioc_err(ioc, "%s: base_cmd in use\n", __func__); 7003 rc = -EAGAIN; 7004 goto out; 7005 } 7006 7007 rc = mpt3sas_wait_for_ioc(ioc, IOC_OPERATIONAL_WAIT_COUNT); 7008 if (rc) 7009 goto out; 7010 7011 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx); 7012 if (!smid) { 7013 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__); 7014 rc = -EAGAIN; 7015 goto out; 7016 } 7017 7018 rc = 0; 7019 ioc->base_cmds.status = MPT3_CMD_PENDING; 7020 request = mpt3sas_base_get_msg_frame(ioc, smid); 7021 ioc->base_cmds.smid = smid; 7022 memcpy(request, mpi_request, sizeof(Mpi2SasIoUnitControlRequest_t)); 7023 if (mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET || 7024 mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET) 7025 ioc->ioc_link_reset_in_progress = 1; 7026 init_completion(&ioc->base_cmds.done); 7027 ioc->put_smid_default(ioc, smid); 7028 wait_for_completion_timeout(&ioc->base_cmds.done, 7029 msecs_to_jiffies(10000)); 7030 if ((mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET || 7031 mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET) && 7032 ioc->ioc_link_reset_in_progress) 7033 ioc->ioc_link_reset_in_progress = 0; 7034 if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) { 7035 mpt3sas_check_cmd_timeout(ioc, ioc->base_cmds.status, 7036 mpi_request, sizeof(Mpi2SasIoUnitControlRequest_t)/4, 7037 issue_reset); 7038 goto issue_host_reset; 7039 } 7040 if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) 7041 memcpy(mpi_reply, ioc->base_cmds.reply, 7042 sizeof(Mpi2SasIoUnitControlReply_t)); 7043 else 7044 memset(mpi_reply, 0, sizeof(Mpi2SasIoUnitControlReply_t)); 7045 ioc->base_cmds.status = MPT3_CMD_NOT_USED; 7046 goto out; 7047 7048 issue_host_reset: 7049 if (issue_reset) 7050 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER); 7051 ioc->base_cmds.status = MPT3_CMD_NOT_USED; 7052 rc = -EFAULT; 7053 out: 7054 mutex_unlock(&ioc->base_cmds.mutex); 7055 return rc; 7056 } 7057 7058 /** 7059 * mpt3sas_base_scsi_enclosure_processor - sending request to sep device 7060 * @ioc: per adapter object 7061 * @mpi_reply: the reply payload from FW 7062 * @mpi_request: the request payload sent to FW 7063 * 7064 * The SCSI Enclosure Processor request message causes the IOC to 7065 * communicate with SES devices to control LED status signals. 7066 * 7067 * Return: 0 for success, non-zero for failure. 7068 */ 7069 int 7070 mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc, 7071 Mpi2SepReply_t *mpi_reply, Mpi2SepRequest_t *mpi_request) 7072 { 7073 u16 smid; 7074 u8 issue_reset = 0; 7075 int rc; 7076 void *request; 7077 7078 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7079 7080 mutex_lock(&ioc->base_cmds.mutex); 7081 7082 if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) { 7083 ioc_err(ioc, "%s: base_cmd in use\n", __func__); 7084 rc = -EAGAIN; 7085 goto out; 7086 } 7087 7088 rc = mpt3sas_wait_for_ioc(ioc, IOC_OPERATIONAL_WAIT_COUNT); 7089 if (rc) 7090 goto out; 7091 7092 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx); 7093 if (!smid) { 7094 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__); 7095 rc = -EAGAIN; 7096 goto out; 7097 } 7098 7099 rc = 0; 7100 ioc->base_cmds.status = MPT3_CMD_PENDING; 7101 request = mpt3sas_base_get_msg_frame(ioc, smid); 7102 ioc->base_cmds.smid = smid; 7103 memset(request, 0, ioc->request_sz); 7104 memcpy(request, mpi_request, sizeof(Mpi2SepReply_t)); 7105 init_completion(&ioc->base_cmds.done); 7106 ioc->put_smid_default(ioc, smid); 7107 wait_for_completion_timeout(&ioc->base_cmds.done, 7108 msecs_to_jiffies(10000)); 7109 if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) { 7110 mpt3sas_check_cmd_timeout(ioc, 7111 ioc->base_cmds.status, mpi_request, 7112 sizeof(Mpi2SepRequest_t)/4, issue_reset); 7113 goto issue_host_reset; 7114 } 7115 if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) 7116 memcpy(mpi_reply, ioc->base_cmds.reply, 7117 sizeof(Mpi2SepReply_t)); 7118 else 7119 memset(mpi_reply, 0, sizeof(Mpi2SepReply_t)); 7120 ioc->base_cmds.status = MPT3_CMD_NOT_USED; 7121 goto out; 7122 7123 issue_host_reset: 7124 if (issue_reset) 7125 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER); 7126 ioc->base_cmds.status = MPT3_CMD_NOT_USED; 7127 rc = -EFAULT; 7128 out: 7129 mutex_unlock(&ioc->base_cmds.mutex); 7130 return rc; 7131 } 7132 7133 /** 7134 * _base_get_port_facts - obtain port facts reply and save in ioc 7135 * @ioc: per adapter object 7136 * @port: ? 7137 * 7138 * Return: 0 for success, non-zero for failure. 7139 */ 7140 static int 7141 _base_get_port_facts(struct MPT3SAS_ADAPTER *ioc, int port) 7142 { 7143 Mpi2PortFactsRequest_t mpi_request; 7144 Mpi2PortFactsReply_t mpi_reply; 7145 struct mpt3sas_port_facts *pfacts; 7146 int mpi_reply_sz, mpi_request_sz, r; 7147 7148 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7149 7150 mpi_reply_sz = sizeof(Mpi2PortFactsReply_t); 7151 mpi_request_sz = sizeof(Mpi2PortFactsRequest_t); 7152 memset(&mpi_request, 0, mpi_request_sz); 7153 mpi_request.Function = MPI2_FUNCTION_PORT_FACTS; 7154 mpi_request.PortNumber = port; 7155 r = _base_handshake_req_reply_wait(ioc, mpi_request_sz, 7156 (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5); 7157 7158 if (r != 0) { 7159 ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r); 7160 return r; 7161 } 7162 7163 pfacts = &ioc->pfacts[port]; 7164 memset(pfacts, 0, sizeof(struct mpt3sas_port_facts)); 7165 pfacts->PortNumber = mpi_reply.PortNumber; 7166 pfacts->VP_ID = mpi_reply.VP_ID; 7167 pfacts->VF_ID = mpi_reply.VF_ID; 7168 pfacts->MaxPostedCmdBuffers = 7169 le16_to_cpu(mpi_reply.MaxPostedCmdBuffers); 7170 7171 return 0; 7172 } 7173 7174 /** 7175 * _base_wait_for_iocstate - Wait until the card is in READY or OPERATIONAL 7176 * @ioc: per adapter object 7177 * @timeout: 7178 * 7179 * Return: 0 for success, non-zero for failure. 7180 */ 7181 static int 7182 _base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout) 7183 { 7184 u32 ioc_state; 7185 int rc; 7186 7187 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7188 7189 if (ioc->pci_error_recovery) { 7190 dfailprintk(ioc, 7191 ioc_info(ioc, "%s: host in pci error recovery\n", 7192 __func__)); 7193 return -EFAULT; 7194 } 7195 7196 ioc_state = mpt3sas_base_get_iocstate(ioc, 0); 7197 dhsprintk(ioc, 7198 ioc_info(ioc, "%s: ioc_state(0x%08x)\n", 7199 __func__, ioc_state)); 7200 7201 if (((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY) || 7202 (ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL) 7203 return 0; 7204 7205 if (ioc_state & MPI2_DOORBELL_USED) { 7206 dhsprintk(ioc, ioc_info(ioc, "unexpected doorbell active!\n")); 7207 goto issue_diag_reset; 7208 } 7209 7210 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { 7211 mpt3sas_print_fault_code(ioc, ioc_state & 7212 MPI2_DOORBELL_DATA_MASK); 7213 goto issue_diag_reset; 7214 } else if ((ioc_state & MPI2_IOC_STATE_MASK) == 7215 MPI2_IOC_STATE_COREDUMP) { 7216 ioc_info(ioc, 7217 "%s: Skipping the diag reset here. (ioc_state=0x%x)\n", 7218 __func__, ioc_state); 7219 return -EFAULT; 7220 } 7221 7222 ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout); 7223 if (ioc_state) { 7224 dfailprintk(ioc, 7225 ioc_info(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n", 7226 __func__, ioc_state)); 7227 return -EFAULT; 7228 } 7229 7230 issue_diag_reset: 7231 rc = _base_diag_reset(ioc); 7232 return rc; 7233 } 7234 7235 /** 7236 * _base_get_ioc_facts - obtain ioc facts reply and save in ioc 7237 * @ioc: per adapter object 7238 * 7239 * Return: 0 for success, non-zero for failure. 7240 */ 7241 static int 7242 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc) 7243 { 7244 Mpi2IOCFactsRequest_t mpi_request; 7245 Mpi2IOCFactsReply_t mpi_reply; 7246 struct mpt3sas_facts *facts; 7247 int mpi_reply_sz, mpi_request_sz, r; 7248 7249 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7250 7251 r = _base_wait_for_iocstate(ioc, 10); 7252 if (r) { 7253 dfailprintk(ioc, 7254 ioc_info(ioc, "%s: failed getting to correct state\n", 7255 __func__)); 7256 return r; 7257 } 7258 mpi_reply_sz = sizeof(Mpi2IOCFactsReply_t); 7259 mpi_request_sz = sizeof(Mpi2IOCFactsRequest_t); 7260 memset(&mpi_request, 0, mpi_request_sz); 7261 mpi_request.Function = MPI2_FUNCTION_IOC_FACTS; 7262 r = _base_handshake_req_reply_wait(ioc, mpi_request_sz, 7263 (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5); 7264 7265 if (r != 0) { 7266 ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r); 7267 return r; 7268 } 7269 7270 facts = &ioc->facts; 7271 memset(facts, 0, sizeof(struct mpt3sas_facts)); 7272 facts->MsgVersion = le16_to_cpu(mpi_reply.MsgVersion); 7273 facts->HeaderVersion = le16_to_cpu(mpi_reply.HeaderVersion); 7274 facts->VP_ID = mpi_reply.VP_ID; 7275 facts->VF_ID = mpi_reply.VF_ID; 7276 facts->IOCExceptions = le16_to_cpu(mpi_reply.IOCExceptions); 7277 facts->MaxChainDepth = mpi_reply.MaxChainDepth; 7278 facts->WhoInit = mpi_reply.WhoInit; 7279 facts->NumberOfPorts = mpi_reply.NumberOfPorts; 7280 facts->MaxMSIxVectors = mpi_reply.MaxMSIxVectors; 7281 if (ioc->msix_enable && (facts->MaxMSIxVectors <= 7282 MAX_COMBINED_MSIX_VECTORS(ioc->is_gen35_ioc))) 7283 ioc->combined_reply_queue = 0; 7284 facts->RequestCredit = le16_to_cpu(mpi_reply.RequestCredit); 7285 facts->MaxReplyDescriptorPostQueueDepth = 7286 le16_to_cpu(mpi_reply.MaxReplyDescriptorPostQueueDepth); 7287 facts->ProductID = le16_to_cpu(mpi_reply.ProductID); 7288 facts->IOCCapabilities = le32_to_cpu(mpi_reply.IOCCapabilities); 7289 if ((facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID)) 7290 ioc->ir_firmware = 1; 7291 if ((facts->IOCCapabilities & 7292 MPI2_IOCFACTS_CAPABILITY_RDPQ_ARRAY_CAPABLE) && (!reset_devices)) 7293 ioc->rdpq_array_capable = 1; 7294 if ((facts->IOCCapabilities & MPI26_IOCFACTS_CAPABILITY_ATOMIC_REQ) 7295 && ioc->is_aero_ioc) 7296 ioc->atomic_desc_capable = 1; 7297 facts->FWVersion.Word = le32_to_cpu(mpi_reply.FWVersion.Word); 7298 facts->IOCRequestFrameSize = 7299 le16_to_cpu(mpi_reply.IOCRequestFrameSize); 7300 if (ioc->hba_mpi_version_belonged != MPI2_VERSION) { 7301 facts->IOCMaxChainSegmentSize = 7302 le16_to_cpu(mpi_reply.IOCMaxChainSegmentSize); 7303 } 7304 facts->MaxInitiators = le16_to_cpu(mpi_reply.MaxInitiators); 7305 facts->MaxTargets = le16_to_cpu(mpi_reply.MaxTargets); 7306 ioc->shost->max_id = -1; 7307 facts->MaxSasExpanders = le16_to_cpu(mpi_reply.MaxSasExpanders); 7308 facts->MaxEnclosures = le16_to_cpu(mpi_reply.MaxEnclosures); 7309 facts->ProtocolFlags = le16_to_cpu(mpi_reply.ProtocolFlags); 7310 facts->HighPriorityCredit = 7311 le16_to_cpu(mpi_reply.HighPriorityCredit); 7312 facts->ReplyFrameSize = mpi_reply.ReplyFrameSize; 7313 facts->MaxDevHandle = le16_to_cpu(mpi_reply.MaxDevHandle); 7314 facts->CurrentHostPageSize = mpi_reply.CurrentHostPageSize; 7315 7316 /* 7317 * Get the Page Size from IOC Facts. If it's 0, default to 4k. 7318 */ 7319 ioc->page_size = 1 << facts->CurrentHostPageSize; 7320 if (ioc->page_size == 1) { 7321 ioc_info(ioc, "CurrentHostPageSize is 0: Setting default host page size to 4k\n"); 7322 ioc->page_size = 1 << MPT3SAS_HOST_PAGE_SIZE_4K; 7323 } 7324 dinitprintk(ioc, 7325 ioc_info(ioc, "CurrentHostPageSize(%d)\n", 7326 facts->CurrentHostPageSize)); 7327 7328 dinitprintk(ioc, 7329 ioc_info(ioc, "hba queue depth(%d), max chains per io(%d)\n", 7330 facts->RequestCredit, facts->MaxChainDepth)); 7331 dinitprintk(ioc, 7332 ioc_info(ioc, "request frame size(%d), reply frame size(%d)\n", 7333 facts->IOCRequestFrameSize * 4, 7334 facts->ReplyFrameSize * 4)); 7335 return 0; 7336 } 7337 7338 /** 7339 * _base_send_ioc_init - send ioc_init to firmware 7340 * @ioc: per adapter object 7341 * 7342 * Return: 0 for success, non-zero for failure. 7343 */ 7344 static int 7345 _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc) 7346 { 7347 Mpi2IOCInitRequest_t mpi_request; 7348 Mpi2IOCInitReply_t mpi_reply; 7349 int i, r = 0; 7350 ktime_t current_time; 7351 u16 ioc_status; 7352 u32 reply_post_free_array_sz = 0; 7353 7354 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7355 7356 memset(&mpi_request, 0, sizeof(Mpi2IOCInitRequest_t)); 7357 mpi_request.Function = MPI2_FUNCTION_IOC_INIT; 7358 mpi_request.WhoInit = MPI2_WHOINIT_HOST_DRIVER; 7359 mpi_request.VF_ID = 0; /* TODO */ 7360 mpi_request.VP_ID = 0; 7361 mpi_request.MsgVersion = cpu_to_le16(ioc->hba_mpi_version_belonged); 7362 mpi_request.HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION); 7363 mpi_request.HostPageSize = MPT3SAS_HOST_PAGE_SIZE_4K; 7364 7365 if (_base_is_controller_msix_enabled(ioc)) 7366 mpi_request.HostMSIxVectors = ioc->reply_queue_count; 7367 mpi_request.SystemRequestFrameSize = cpu_to_le16(ioc->request_sz/4); 7368 mpi_request.ReplyDescriptorPostQueueDepth = 7369 cpu_to_le16(ioc->reply_post_queue_depth); 7370 mpi_request.ReplyFreeQueueDepth = 7371 cpu_to_le16(ioc->reply_free_queue_depth); 7372 7373 mpi_request.SenseBufferAddressHigh = 7374 cpu_to_le32((u64)ioc->sense_dma >> 32); 7375 mpi_request.SystemReplyAddressHigh = 7376 cpu_to_le32((u64)ioc->reply_dma >> 32); 7377 mpi_request.SystemRequestFrameBaseAddress = 7378 cpu_to_le64((u64)ioc->request_dma); 7379 mpi_request.ReplyFreeQueueAddress = 7380 cpu_to_le64((u64)ioc->reply_free_dma); 7381 7382 if (ioc->rdpq_array_enable) { 7383 reply_post_free_array_sz = ioc->reply_queue_count * 7384 sizeof(Mpi2IOCInitRDPQArrayEntry); 7385 memset(ioc->reply_post_free_array, 0, reply_post_free_array_sz); 7386 for (i = 0; i < ioc->reply_queue_count; i++) 7387 ioc->reply_post_free_array[i].RDPQBaseAddress = 7388 cpu_to_le64( 7389 (u64)ioc->reply_post[i].reply_post_free_dma); 7390 mpi_request.MsgFlags = MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE; 7391 mpi_request.ReplyDescriptorPostQueueAddress = 7392 cpu_to_le64((u64)ioc->reply_post_free_array_dma); 7393 } else { 7394 mpi_request.ReplyDescriptorPostQueueAddress = 7395 cpu_to_le64((u64)ioc->reply_post[0].reply_post_free_dma); 7396 } 7397 7398 /* 7399 * Set the flag to enable CoreDump state feature in IOC firmware. 7400 */ 7401 mpi_request.ConfigurationFlags |= 7402 cpu_to_le16(MPI26_IOCINIT_CFGFLAGS_COREDUMP_ENABLE); 7403 7404 /* This time stamp specifies number of milliseconds 7405 * since epoch ~ midnight January 1, 1970. 7406 */ 7407 current_time = ktime_get_real(); 7408 mpi_request.TimeStamp = cpu_to_le64(ktime_to_ms(current_time)); 7409 7410 if (ioc->logging_level & MPT_DEBUG_INIT) { 7411 __le32 *mfp; 7412 int i; 7413 7414 mfp = (__le32 *)&mpi_request; 7415 ioc_info(ioc, "\toffset:data\n"); 7416 for (i = 0; i < sizeof(Mpi2IOCInitRequest_t)/4; i++) 7417 ioc_info(ioc, "\t[0x%02x]:%08x\n", i*4, 7418 le32_to_cpu(mfp[i])); 7419 } 7420 7421 r = _base_handshake_req_reply_wait(ioc, 7422 sizeof(Mpi2IOCInitRequest_t), (u32 *)&mpi_request, 7423 sizeof(Mpi2IOCInitReply_t), (u16 *)&mpi_reply, 30); 7424 7425 if (r != 0) { 7426 ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r); 7427 return r; 7428 } 7429 7430 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK; 7431 if (ioc_status != MPI2_IOCSTATUS_SUCCESS || 7432 mpi_reply.IOCLogInfo) { 7433 ioc_err(ioc, "%s: failed\n", __func__); 7434 r = -EIO; 7435 } 7436 7437 /* Reset TimeSync Counter*/ 7438 ioc->timestamp_update_count = 0; 7439 return r; 7440 } 7441 7442 /** 7443 * mpt3sas_port_enable_done - command completion routine for port enable 7444 * @ioc: per adapter object 7445 * @smid: system request message index 7446 * @msix_index: MSIX table index supplied by the OS 7447 * @reply: reply message frame(lower 32bit addr) 7448 * 7449 * Return: 1 meaning mf should be freed from _base_interrupt 7450 * 0 means the mf is freed from this function. 7451 */ 7452 u8 7453 mpt3sas_port_enable_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, 7454 u32 reply) 7455 { 7456 MPI2DefaultReply_t *mpi_reply; 7457 u16 ioc_status; 7458 7459 if (ioc->port_enable_cmds.status == MPT3_CMD_NOT_USED) 7460 return 1; 7461 7462 mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply); 7463 if (!mpi_reply) 7464 return 1; 7465 7466 if (mpi_reply->Function != MPI2_FUNCTION_PORT_ENABLE) 7467 return 1; 7468 7469 ioc->port_enable_cmds.status &= ~MPT3_CMD_PENDING; 7470 ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE; 7471 ioc->port_enable_cmds.status |= MPT3_CMD_REPLY_VALID; 7472 memcpy(ioc->port_enable_cmds.reply, mpi_reply, mpi_reply->MsgLength*4); 7473 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK; 7474 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) 7475 ioc->port_enable_failed = 1; 7476 7477 if (ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE_ASYNC) { 7478 ioc->port_enable_cmds.status &= ~MPT3_CMD_COMPLETE_ASYNC; 7479 if (ioc_status == MPI2_IOCSTATUS_SUCCESS) { 7480 mpt3sas_port_enable_complete(ioc); 7481 return 1; 7482 } else { 7483 ioc->start_scan_failed = ioc_status; 7484 ioc->start_scan = 0; 7485 return 1; 7486 } 7487 } 7488 complete(&ioc->port_enable_cmds.done); 7489 return 1; 7490 } 7491 7492 /** 7493 * _base_send_port_enable - send port_enable(discovery stuff) to firmware 7494 * @ioc: per adapter object 7495 * 7496 * Return: 0 for success, non-zero for failure. 7497 */ 7498 static int 7499 _base_send_port_enable(struct MPT3SAS_ADAPTER *ioc) 7500 { 7501 Mpi2PortEnableRequest_t *mpi_request; 7502 Mpi2PortEnableReply_t *mpi_reply; 7503 int r = 0; 7504 u16 smid; 7505 u16 ioc_status; 7506 7507 ioc_info(ioc, "sending port enable !!\n"); 7508 7509 if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) { 7510 ioc_err(ioc, "%s: internal command already in use\n", __func__); 7511 return -EAGAIN; 7512 } 7513 7514 smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx); 7515 if (!smid) { 7516 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__); 7517 return -EAGAIN; 7518 } 7519 7520 ioc->port_enable_cmds.status = MPT3_CMD_PENDING; 7521 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); 7522 ioc->port_enable_cmds.smid = smid; 7523 memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t)); 7524 mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE; 7525 7526 init_completion(&ioc->port_enable_cmds.done); 7527 ioc->put_smid_default(ioc, smid); 7528 wait_for_completion_timeout(&ioc->port_enable_cmds.done, 300*HZ); 7529 if (!(ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE)) { 7530 ioc_err(ioc, "%s: timeout\n", __func__); 7531 _debug_dump_mf(mpi_request, 7532 sizeof(Mpi2PortEnableRequest_t)/4); 7533 if (ioc->port_enable_cmds.status & MPT3_CMD_RESET) 7534 r = -EFAULT; 7535 else 7536 r = -ETIME; 7537 goto out; 7538 } 7539 7540 mpi_reply = ioc->port_enable_cmds.reply; 7541 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK; 7542 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { 7543 ioc_err(ioc, "%s: failed with (ioc_status=0x%08x)\n", 7544 __func__, ioc_status); 7545 r = -EFAULT; 7546 goto out; 7547 } 7548 7549 out: 7550 ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED; 7551 ioc_info(ioc, "port enable: %s\n", r == 0 ? "SUCCESS" : "FAILED"); 7552 return r; 7553 } 7554 7555 /** 7556 * mpt3sas_port_enable - initiate firmware discovery (don't wait for reply) 7557 * @ioc: per adapter object 7558 * 7559 * Return: 0 for success, non-zero for failure. 7560 */ 7561 int 7562 mpt3sas_port_enable(struct MPT3SAS_ADAPTER *ioc) 7563 { 7564 Mpi2PortEnableRequest_t *mpi_request; 7565 u16 smid; 7566 7567 ioc_info(ioc, "sending port enable !!\n"); 7568 7569 if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) { 7570 ioc_err(ioc, "%s: internal command already in use\n", __func__); 7571 return -EAGAIN; 7572 } 7573 7574 smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx); 7575 if (!smid) { 7576 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__); 7577 return -EAGAIN; 7578 } 7579 ioc->drv_internal_flags |= MPT_DRV_INTERNAL_FIRST_PE_ISSUED; 7580 ioc->port_enable_cmds.status = MPT3_CMD_PENDING; 7581 ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE_ASYNC; 7582 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); 7583 ioc->port_enable_cmds.smid = smid; 7584 memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t)); 7585 mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE; 7586 7587 ioc->put_smid_default(ioc, smid); 7588 return 0; 7589 } 7590 7591 /** 7592 * _base_determine_wait_on_discovery - desposition 7593 * @ioc: per adapter object 7594 * 7595 * Decide whether to wait on discovery to complete. Used to either 7596 * locate boot device, or report volumes ahead of physical devices. 7597 * 7598 * Return: 1 for wait, 0 for don't wait. 7599 */ 7600 static int 7601 _base_determine_wait_on_discovery(struct MPT3SAS_ADAPTER *ioc) 7602 { 7603 /* We wait for discovery to complete if IR firmware is loaded. 7604 * The sas topology events arrive before PD events, so we need time to 7605 * turn on the bit in ioc->pd_handles to indicate PD 7606 * Also, it maybe required to report Volumes ahead of physical 7607 * devices when MPI2_IOCPAGE8_IRFLAGS_LOW_VOLUME_MAPPING is set. 7608 */ 7609 if (ioc->ir_firmware) 7610 return 1; 7611 7612 /* if no Bios, then we don't need to wait */ 7613 if (!ioc->bios_pg3.BiosVersion) 7614 return 0; 7615 7616 /* Bios is present, then we drop down here. 7617 * 7618 * If there any entries in the Bios Page 2, then we wait 7619 * for discovery to complete. 7620 */ 7621 7622 /* Current Boot Device */ 7623 if ((ioc->bios_pg2.CurrentBootDeviceForm & 7624 MPI2_BIOSPAGE2_FORM_MASK) == 7625 MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED && 7626 /* Request Boot Device */ 7627 (ioc->bios_pg2.ReqBootDeviceForm & 7628 MPI2_BIOSPAGE2_FORM_MASK) == 7629 MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED && 7630 /* Alternate Request Boot Device */ 7631 (ioc->bios_pg2.ReqAltBootDeviceForm & 7632 MPI2_BIOSPAGE2_FORM_MASK) == 7633 MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED) 7634 return 0; 7635 7636 return 1; 7637 } 7638 7639 /** 7640 * _base_unmask_events - turn on notification for this event 7641 * @ioc: per adapter object 7642 * @event: firmware event 7643 * 7644 * The mask is stored in ioc->event_masks. 7645 */ 7646 static void 7647 _base_unmask_events(struct MPT3SAS_ADAPTER *ioc, u16 event) 7648 { 7649 u32 desired_event; 7650 7651 if (event >= 128) 7652 return; 7653 7654 desired_event = (1 << (event % 32)); 7655 7656 if (event < 32) 7657 ioc->event_masks[0] &= ~desired_event; 7658 else if (event < 64) 7659 ioc->event_masks[1] &= ~desired_event; 7660 else if (event < 96) 7661 ioc->event_masks[2] &= ~desired_event; 7662 else if (event < 128) 7663 ioc->event_masks[3] &= ~desired_event; 7664 } 7665 7666 /** 7667 * _base_event_notification - send event notification 7668 * @ioc: per adapter object 7669 * 7670 * Return: 0 for success, non-zero for failure. 7671 */ 7672 static int 7673 _base_event_notification(struct MPT3SAS_ADAPTER *ioc) 7674 { 7675 Mpi2EventNotificationRequest_t *mpi_request; 7676 u16 smid; 7677 int r = 0; 7678 int i, issue_diag_reset = 0; 7679 7680 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7681 7682 if (ioc->base_cmds.status & MPT3_CMD_PENDING) { 7683 ioc_err(ioc, "%s: internal command already in use\n", __func__); 7684 return -EAGAIN; 7685 } 7686 7687 smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx); 7688 if (!smid) { 7689 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__); 7690 return -EAGAIN; 7691 } 7692 ioc->base_cmds.status = MPT3_CMD_PENDING; 7693 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); 7694 ioc->base_cmds.smid = smid; 7695 memset(mpi_request, 0, sizeof(Mpi2EventNotificationRequest_t)); 7696 mpi_request->Function = MPI2_FUNCTION_EVENT_NOTIFICATION; 7697 mpi_request->VF_ID = 0; /* TODO */ 7698 mpi_request->VP_ID = 0; 7699 for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++) 7700 mpi_request->EventMasks[i] = 7701 cpu_to_le32(ioc->event_masks[i]); 7702 init_completion(&ioc->base_cmds.done); 7703 ioc->put_smid_default(ioc, smid); 7704 wait_for_completion_timeout(&ioc->base_cmds.done, 30*HZ); 7705 if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) { 7706 ioc_err(ioc, "%s: timeout\n", __func__); 7707 _debug_dump_mf(mpi_request, 7708 sizeof(Mpi2EventNotificationRequest_t)/4); 7709 if (ioc->base_cmds.status & MPT3_CMD_RESET) 7710 r = -EFAULT; 7711 else 7712 issue_diag_reset = 1; 7713 7714 } else 7715 dinitprintk(ioc, ioc_info(ioc, "%s: complete\n", __func__)); 7716 ioc->base_cmds.status = MPT3_CMD_NOT_USED; 7717 7718 if (issue_diag_reset) { 7719 if (ioc->drv_internal_flags & MPT_DRV_INTERNAL_FIRST_PE_ISSUED) 7720 return -EFAULT; 7721 if (mpt3sas_base_check_for_fault_and_issue_reset(ioc)) 7722 return -EFAULT; 7723 r = -EAGAIN; 7724 } 7725 return r; 7726 } 7727 7728 /** 7729 * mpt3sas_base_validate_event_type - validating event types 7730 * @ioc: per adapter object 7731 * @event_type: firmware event 7732 * 7733 * This will turn on firmware event notification when application 7734 * ask for that event. We don't mask events that are already enabled. 7735 */ 7736 void 7737 mpt3sas_base_validate_event_type(struct MPT3SAS_ADAPTER *ioc, u32 *event_type) 7738 { 7739 int i, j; 7740 u32 event_mask, desired_event; 7741 u8 send_update_to_fw; 7742 7743 for (i = 0, send_update_to_fw = 0; i < 7744 MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++) { 7745 event_mask = ~event_type[i]; 7746 desired_event = 1; 7747 for (j = 0; j < 32; j++) { 7748 if (!(event_mask & desired_event) && 7749 (ioc->event_masks[i] & desired_event)) { 7750 ioc->event_masks[i] &= ~desired_event; 7751 send_update_to_fw = 1; 7752 } 7753 desired_event = (desired_event << 1); 7754 } 7755 } 7756 7757 if (!send_update_to_fw) 7758 return; 7759 7760 mutex_lock(&ioc->base_cmds.mutex); 7761 _base_event_notification(ioc); 7762 mutex_unlock(&ioc->base_cmds.mutex); 7763 } 7764 7765 /** 7766 * _base_diag_reset - the "big hammer" start of day reset 7767 * @ioc: per adapter object 7768 * 7769 * Return: 0 for success, non-zero for failure. 7770 */ 7771 static int 7772 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc) 7773 { 7774 u32 host_diagnostic; 7775 u32 ioc_state; 7776 u32 count; 7777 u32 hcb_size; 7778 7779 ioc_info(ioc, "sending diag reset !!\n"); 7780 7781 pci_cfg_access_lock(ioc->pdev); 7782 7783 drsprintk(ioc, ioc_info(ioc, "clear interrupts\n")); 7784 7785 count = 0; 7786 do { 7787 /* Write magic sequence to WriteSequence register 7788 * Loop until in diagnostic mode 7789 */ 7790 drsprintk(ioc, ioc_info(ioc, "write magic sequence\n")); 7791 writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence); 7792 writel(MPI2_WRSEQ_1ST_KEY_VALUE, &ioc->chip->WriteSequence); 7793 writel(MPI2_WRSEQ_2ND_KEY_VALUE, &ioc->chip->WriteSequence); 7794 writel(MPI2_WRSEQ_3RD_KEY_VALUE, &ioc->chip->WriteSequence); 7795 writel(MPI2_WRSEQ_4TH_KEY_VALUE, &ioc->chip->WriteSequence); 7796 writel(MPI2_WRSEQ_5TH_KEY_VALUE, &ioc->chip->WriteSequence); 7797 writel(MPI2_WRSEQ_6TH_KEY_VALUE, &ioc->chip->WriteSequence); 7798 7799 /* wait 100 msec */ 7800 msleep(100); 7801 7802 if (count++ > 20) { 7803 ioc_info(ioc, 7804 "Stop writing magic sequence after 20 retries\n"); 7805 _base_dump_reg_set(ioc); 7806 goto out; 7807 } 7808 7809 host_diagnostic = ioc->base_readl(&ioc->chip->HostDiagnostic); 7810 drsprintk(ioc, 7811 ioc_info(ioc, "wrote magic sequence: count(%d), host_diagnostic(0x%08x)\n", 7812 count, host_diagnostic)); 7813 7814 } while ((host_diagnostic & MPI2_DIAG_DIAG_WRITE_ENABLE) == 0); 7815 7816 hcb_size = ioc->base_readl(&ioc->chip->HCBSize); 7817 7818 drsprintk(ioc, ioc_info(ioc, "diag reset: issued\n")); 7819 writel(host_diagnostic | MPI2_DIAG_RESET_ADAPTER, 7820 &ioc->chip->HostDiagnostic); 7821 7822 /*This delay allows the chip PCIe hardware time to finish reset tasks*/ 7823 msleep(MPI2_HARD_RESET_PCIE_FIRST_READ_DELAY_MICRO_SEC/1000); 7824 7825 /* Approximately 300 second max wait */ 7826 for (count = 0; count < (300000000 / 7827 MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC); count++) { 7828 7829 host_diagnostic = ioc->base_readl(&ioc->chip->HostDiagnostic); 7830 7831 if (host_diagnostic == 0xFFFFFFFF) { 7832 ioc_info(ioc, 7833 "Invalid host diagnostic register value\n"); 7834 _base_dump_reg_set(ioc); 7835 goto out; 7836 } 7837 if (!(host_diagnostic & MPI2_DIAG_RESET_ADAPTER)) 7838 break; 7839 7840 msleep(MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC / 1000); 7841 } 7842 7843 if (host_diagnostic & MPI2_DIAG_HCB_MODE) { 7844 7845 drsprintk(ioc, 7846 ioc_info(ioc, "restart the adapter assuming the HCB Address points to good F/W\n")); 7847 host_diagnostic &= ~MPI2_DIAG_BOOT_DEVICE_SELECT_MASK; 7848 host_diagnostic |= MPI2_DIAG_BOOT_DEVICE_SELECT_HCDW; 7849 writel(host_diagnostic, &ioc->chip->HostDiagnostic); 7850 7851 drsprintk(ioc, ioc_info(ioc, "re-enable the HCDW\n")); 7852 writel(hcb_size | MPI2_HCB_SIZE_HCB_ENABLE, 7853 &ioc->chip->HCBSize); 7854 } 7855 7856 drsprintk(ioc, ioc_info(ioc, "restart the adapter\n")); 7857 writel(host_diagnostic & ~MPI2_DIAG_HOLD_IOC_RESET, 7858 &ioc->chip->HostDiagnostic); 7859 7860 drsprintk(ioc, 7861 ioc_info(ioc, "disable writes to the diagnostic register\n")); 7862 writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence); 7863 7864 drsprintk(ioc, ioc_info(ioc, "Wait for FW to go to the READY state\n")); 7865 ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, 20); 7866 if (ioc_state) { 7867 ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n", 7868 __func__, ioc_state); 7869 _base_dump_reg_set(ioc); 7870 goto out; 7871 } 7872 7873 pci_cfg_access_unlock(ioc->pdev); 7874 ioc_info(ioc, "diag reset: SUCCESS\n"); 7875 return 0; 7876 7877 out: 7878 pci_cfg_access_unlock(ioc->pdev); 7879 ioc_err(ioc, "diag reset: FAILED\n"); 7880 return -EFAULT; 7881 } 7882 7883 /** 7884 * mpt3sas_base_make_ioc_ready - put controller in READY state 7885 * @ioc: per adapter object 7886 * @type: FORCE_BIG_HAMMER or SOFT_RESET 7887 * 7888 * Return: 0 for success, non-zero for failure. 7889 */ 7890 int 7891 mpt3sas_base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, enum reset_type type) 7892 { 7893 u32 ioc_state; 7894 int rc; 7895 int count; 7896 7897 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7898 7899 if (ioc->pci_error_recovery) 7900 return 0; 7901 7902 ioc_state = mpt3sas_base_get_iocstate(ioc, 0); 7903 dhsprintk(ioc, 7904 ioc_info(ioc, "%s: ioc_state(0x%08x)\n", 7905 __func__, ioc_state)); 7906 7907 /* if in RESET state, it should move to READY state shortly */ 7908 count = 0; 7909 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_RESET) { 7910 while ((ioc_state & MPI2_IOC_STATE_MASK) != 7911 MPI2_IOC_STATE_READY) { 7912 if (count++ == 10) { 7913 ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n", 7914 __func__, ioc_state); 7915 return -EFAULT; 7916 } 7917 ssleep(1); 7918 ioc_state = mpt3sas_base_get_iocstate(ioc, 0); 7919 } 7920 } 7921 7922 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY) 7923 return 0; 7924 7925 if (ioc_state & MPI2_DOORBELL_USED) { 7926 ioc_info(ioc, "unexpected doorbell active!\n"); 7927 goto issue_diag_reset; 7928 } 7929 7930 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { 7931 mpt3sas_print_fault_code(ioc, ioc_state & 7932 MPI2_DOORBELL_DATA_MASK); 7933 goto issue_diag_reset; 7934 } 7935 7936 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_COREDUMP) { 7937 /* 7938 * if host reset is invoked while watch dog thread is waiting 7939 * for IOC state to be changed to Fault state then driver has 7940 * to wait here for CoreDump state to clear otherwise reset 7941 * will be issued to the FW and FW move the IOC state to 7942 * reset state without copying the FW logs to coredump region. 7943 */ 7944 if (ioc->ioc_coredump_loop != MPT3SAS_COREDUMP_LOOP_DONE) { 7945 mpt3sas_print_coredump_info(ioc, ioc_state & 7946 MPI2_DOORBELL_DATA_MASK); 7947 mpt3sas_base_wait_for_coredump_completion(ioc, 7948 __func__); 7949 } 7950 goto issue_diag_reset; 7951 } 7952 7953 if (type == FORCE_BIG_HAMMER) 7954 goto issue_diag_reset; 7955 7956 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL) 7957 if (!(_base_send_ioc_reset(ioc, 7958 MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET, 15))) { 7959 return 0; 7960 } 7961 7962 issue_diag_reset: 7963 rc = _base_diag_reset(ioc); 7964 return rc; 7965 } 7966 7967 /** 7968 * _base_make_ioc_operational - put controller in OPERATIONAL state 7969 * @ioc: per adapter object 7970 * 7971 * Return: 0 for success, non-zero for failure. 7972 */ 7973 static int 7974 _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc) 7975 { 7976 int r, i, index, rc; 7977 unsigned long flags; 7978 u32 reply_address; 7979 u16 smid; 7980 struct _tr_list *delayed_tr, *delayed_tr_next; 7981 struct _sc_list *delayed_sc, *delayed_sc_next; 7982 struct _event_ack_list *delayed_event_ack, *delayed_event_ack_next; 7983 u8 hide_flag; 7984 struct adapter_reply_queue *reply_q; 7985 Mpi2ReplyDescriptorsUnion_t *reply_post_free_contig; 7986 7987 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 7988 7989 /* clean the delayed target reset list */ 7990 list_for_each_entry_safe(delayed_tr, delayed_tr_next, 7991 &ioc->delayed_tr_list, list) { 7992 list_del(&delayed_tr->list); 7993 kfree(delayed_tr); 7994 } 7995 7996 7997 list_for_each_entry_safe(delayed_tr, delayed_tr_next, 7998 &ioc->delayed_tr_volume_list, list) { 7999 list_del(&delayed_tr->list); 8000 kfree(delayed_tr); 8001 } 8002 8003 list_for_each_entry_safe(delayed_sc, delayed_sc_next, 8004 &ioc->delayed_sc_list, list) { 8005 list_del(&delayed_sc->list); 8006 kfree(delayed_sc); 8007 } 8008 8009 list_for_each_entry_safe(delayed_event_ack, delayed_event_ack_next, 8010 &ioc->delayed_event_ack_list, list) { 8011 list_del(&delayed_event_ack->list); 8012 kfree(delayed_event_ack); 8013 } 8014 8015 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags); 8016 8017 /* hi-priority queue */ 8018 INIT_LIST_HEAD(&ioc->hpr_free_list); 8019 smid = ioc->hi_priority_smid; 8020 for (i = 0; i < ioc->hi_priority_depth; i++, smid++) { 8021 ioc->hpr_lookup[i].cb_idx = 0xFF; 8022 ioc->hpr_lookup[i].smid = smid; 8023 list_add_tail(&ioc->hpr_lookup[i].tracker_list, 8024 &ioc->hpr_free_list); 8025 } 8026 8027 /* internal queue */ 8028 INIT_LIST_HEAD(&ioc->internal_free_list); 8029 smid = ioc->internal_smid; 8030 for (i = 0; i < ioc->internal_depth; i++, smid++) { 8031 ioc->internal_lookup[i].cb_idx = 0xFF; 8032 ioc->internal_lookup[i].smid = smid; 8033 list_add_tail(&ioc->internal_lookup[i].tracker_list, 8034 &ioc->internal_free_list); 8035 } 8036 8037 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags); 8038 8039 /* initialize Reply Free Queue */ 8040 for (i = 0, reply_address = (u32)ioc->reply_dma ; 8041 i < ioc->reply_free_queue_depth ; i++, reply_address += 8042 ioc->reply_sz) { 8043 ioc->reply_free[i] = cpu_to_le32(reply_address); 8044 if (ioc->is_mcpu_endpoint) 8045 _base_clone_reply_to_sys_mem(ioc, 8046 reply_address, i); 8047 } 8048 8049 /* initialize reply queues */ 8050 if (ioc->is_driver_loading) 8051 _base_assign_reply_queues(ioc); 8052 8053 /* initialize Reply Post Free Queue */ 8054 index = 0; 8055 reply_post_free_contig = ioc->reply_post[0].reply_post_free; 8056 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { 8057 /* 8058 * If RDPQ is enabled, switch to the next allocation. 8059 * Otherwise advance within the contiguous region. 8060 */ 8061 if (ioc->rdpq_array_enable) { 8062 reply_q->reply_post_free = 8063 ioc->reply_post[index++].reply_post_free; 8064 } else { 8065 reply_q->reply_post_free = reply_post_free_contig; 8066 reply_post_free_contig += ioc->reply_post_queue_depth; 8067 } 8068 8069 reply_q->reply_post_host_index = 0; 8070 for (i = 0; i < ioc->reply_post_queue_depth; i++) 8071 reply_q->reply_post_free[i].Words = 8072 cpu_to_le64(ULLONG_MAX); 8073 if (!_base_is_controller_msix_enabled(ioc)) 8074 goto skip_init_reply_post_free_queue; 8075 } 8076 skip_init_reply_post_free_queue: 8077 8078 r = _base_send_ioc_init(ioc); 8079 if (r) { 8080 /* 8081 * No need to check IOC state for fault state & issue 8082 * diag reset during host reset. This check is need 8083 * only during driver load time. 8084 */ 8085 if (!ioc->is_driver_loading) 8086 return r; 8087 8088 rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc); 8089 if (rc || (_base_send_ioc_init(ioc))) 8090 return r; 8091 } 8092 8093 /* initialize reply free host index */ 8094 ioc->reply_free_host_index = ioc->reply_free_queue_depth - 1; 8095 writel(ioc->reply_free_host_index, &ioc->chip->ReplyFreeHostIndex); 8096 8097 /* initialize reply post host index */ 8098 list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { 8099 if (ioc->combined_reply_queue) 8100 writel((reply_q->msix_index & 7)<< 8101 MPI2_RPHI_MSIX_INDEX_SHIFT, 8102 ioc->replyPostRegisterIndex[reply_q->msix_index/8]); 8103 else 8104 writel(reply_q->msix_index << 8105 MPI2_RPHI_MSIX_INDEX_SHIFT, 8106 &ioc->chip->ReplyPostHostIndex); 8107 8108 if (!_base_is_controller_msix_enabled(ioc)) 8109 goto skip_init_reply_post_host_index; 8110 } 8111 8112 skip_init_reply_post_host_index: 8113 8114 mpt3sas_base_unmask_interrupts(ioc); 8115 8116 if (ioc->hba_mpi_version_belonged != MPI2_VERSION) { 8117 r = _base_display_fwpkg_version(ioc); 8118 if (r) 8119 return r; 8120 } 8121 8122 r = _base_static_config_pages(ioc); 8123 if (r) 8124 return r; 8125 8126 r = _base_event_notification(ioc); 8127 if (r) 8128 return r; 8129 8130 if (!ioc->shost_recovery) { 8131 8132 if (ioc->is_warpdrive && ioc->manu_pg10.OEMIdentifier 8133 == 0x80) { 8134 hide_flag = (u8) ( 8135 le32_to_cpu(ioc->manu_pg10.OEMSpecificFlags0) & 8136 MFG_PAGE10_HIDE_SSDS_MASK); 8137 if (hide_flag != MFG_PAGE10_HIDE_SSDS_MASK) 8138 ioc->mfg_pg10_hide_flag = hide_flag; 8139 } 8140 8141 ioc->wait_for_discovery_to_complete = 8142 _base_determine_wait_on_discovery(ioc); 8143 8144 return r; /* scan_start and scan_finished support */ 8145 } 8146 8147 r = _base_send_port_enable(ioc); 8148 if (r) 8149 return r; 8150 8151 return r; 8152 } 8153 8154 /** 8155 * mpt3sas_base_free_resources - free resources controller resources 8156 * @ioc: per adapter object 8157 */ 8158 void 8159 mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc) 8160 { 8161 dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 8162 8163 /* synchronizing freeing resource with pci_access_mutex lock */ 8164 mutex_lock(&ioc->pci_access_mutex); 8165 if (ioc->chip_phys && ioc->chip) { 8166 mpt3sas_base_mask_interrupts(ioc); 8167 ioc->shost_recovery = 1; 8168 mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET); 8169 ioc->shost_recovery = 0; 8170 } 8171 8172 mpt3sas_base_unmap_resources(ioc); 8173 mutex_unlock(&ioc->pci_access_mutex); 8174 return; 8175 } 8176 8177 /** 8178 * mpt3sas_base_attach - attach controller instance 8179 * @ioc: per adapter object 8180 * 8181 * Return: 0 for success, non-zero for failure. 8182 */ 8183 int 8184 mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc) 8185 { 8186 int r, i, rc; 8187 int cpu_id, last_cpu_id = 0; 8188 8189 dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 8190 8191 /* setup cpu_msix_table */ 8192 ioc->cpu_count = num_online_cpus(); 8193 for_each_online_cpu(cpu_id) 8194 last_cpu_id = cpu_id; 8195 ioc->cpu_msix_table_sz = last_cpu_id + 1; 8196 ioc->cpu_msix_table = kzalloc(ioc->cpu_msix_table_sz, GFP_KERNEL); 8197 ioc->reply_queue_count = 1; 8198 if (!ioc->cpu_msix_table) { 8199 ioc_info(ioc, "Allocation for cpu_msix_table failed!!!\n"); 8200 r = -ENOMEM; 8201 goto out_free_resources; 8202 } 8203 8204 if (ioc->is_warpdrive) { 8205 ioc->reply_post_host_index = kcalloc(ioc->cpu_msix_table_sz, 8206 sizeof(resource_size_t *), GFP_KERNEL); 8207 if (!ioc->reply_post_host_index) { 8208 ioc_info(ioc, "Allocation for reply_post_host_index failed!!!\n"); 8209 r = -ENOMEM; 8210 goto out_free_resources; 8211 } 8212 } 8213 8214 ioc->smp_affinity_enable = smp_affinity_enable; 8215 8216 ioc->rdpq_array_enable_assigned = 0; 8217 ioc->use_32bit_dma = false; 8218 ioc->dma_mask = 64; 8219 if (ioc->is_aero_ioc) 8220 ioc->base_readl = &_base_readl_aero; 8221 else 8222 ioc->base_readl = &_base_readl; 8223 r = mpt3sas_base_map_resources(ioc); 8224 if (r) 8225 goto out_free_resources; 8226 8227 pci_set_drvdata(ioc->pdev, ioc->shost); 8228 r = _base_get_ioc_facts(ioc); 8229 if (r) { 8230 rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc); 8231 if (rc || (_base_get_ioc_facts(ioc))) 8232 goto out_free_resources; 8233 } 8234 8235 switch (ioc->hba_mpi_version_belonged) { 8236 case MPI2_VERSION: 8237 ioc->build_sg_scmd = &_base_build_sg_scmd; 8238 ioc->build_sg = &_base_build_sg; 8239 ioc->build_zero_len_sge = &_base_build_zero_len_sge; 8240 ioc->get_msix_index_for_smlio = &_base_get_msix_index; 8241 break; 8242 case MPI25_VERSION: 8243 case MPI26_VERSION: 8244 /* 8245 * In SAS3.0, 8246 * SCSI_IO, SMP_PASSTHRU, SATA_PASSTHRU, Target Assist, and 8247 * Target Status - all require the IEEE formatted scatter gather 8248 * elements. 8249 */ 8250 ioc->build_sg_scmd = &_base_build_sg_scmd_ieee; 8251 ioc->build_sg = &_base_build_sg_ieee; 8252 ioc->build_nvme_prp = &_base_build_nvme_prp; 8253 ioc->build_zero_len_sge = &_base_build_zero_len_sge_ieee; 8254 ioc->sge_size_ieee = sizeof(Mpi2IeeeSgeSimple64_t); 8255 if (ioc->high_iops_queues) 8256 ioc->get_msix_index_for_smlio = 8257 &_base_get_high_iops_msix_index; 8258 else 8259 ioc->get_msix_index_for_smlio = &_base_get_msix_index; 8260 break; 8261 } 8262 if (ioc->atomic_desc_capable) { 8263 ioc->put_smid_default = &_base_put_smid_default_atomic; 8264 ioc->put_smid_scsi_io = &_base_put_smid_scsi_io_atomic; 8265 ioc->put_smid_fast_path = 8266 &_base_put_smid_fast_path_atomic; 8267 ioc->put_smid_hi_priority = 8268 &_base_put_smid_hi_priority_atomic; 8269 } else { 8270 ioc->put_smid_default = &_base_put_smid_default; 8271 ioc->put_smid_fast_path = &_base_put_smid_fast_path; 8272 ioc->put_smid_hi_priority = &_base_put_smid_hi_priority; 8273 if (ioc->is_mcpu_endpoint) 8274 ioc->put_smid_scsi_io = 8275 &_base_put_smid_mpi_ep_scsi_io; 8276 else 8277 ioc->put_smid_scsi_io = &_base_put_smid_scsi_io; 8278 } 8279 /* 8280 * These function pointers for other requests that don't 8281 * the require IEEE scatter gather elements. 8282 * 8283 * For example Configuration Pages and SAS IOUNIT Control don't. 8284 */ 8285 ioc->build_sg_mpi = &_base_build_sg; 8286 ioc->build_zero_len_sge_mpi = &_base_build_zero_len_sge; 8287 8288 r = mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET); 8289 if (r) 8290 goto out_free_resources; 8291 8292 ioc->pfacts = kcalloc(ioc->facts.NumberOfPorts, 8293 sizeof(struct mpt3sas_port_facts), GFP_KERNEL); 8294 if (!ioc->pfacts) { 8295 r = -ENOMEM; 8296 goto out_free_resources; 8297 } 8298 8299 for (i = 0 ; i < ioc->facts.NumberOfPorts; i++) { 8300 r = _base_get_port_facts(ioc, i); 8301 if (r) { 8302 rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc); 8303 if (rc || (_base_get_port_facts(ioc, i))) 8304 goto out_free_resources; 8305 } 8306 } 8307 8308 r = _base_allocate_memory_pools(ioc); 8309 if (r) 8310 goto out_free_resources; 8311 8312 if (irqpoll_weight > 0) 8313 ioc->thresh_hold = irqpoll_weight; 8314 else 8315 ioc->thresh_hold = ioc->hba_queue_depth/4; 8316 8317 _base_init_irqpolls(ioc); 8318 init_waitqueue_head(&ioc->reset_wq); 8319 8320 /* allocate memory pd handle bitmask list */ 8321 ioc->pd_handles_sz = (ioc->facts.MaxDevHandle / 8); 8322 if (ioc->facts.MaxDevHandle % 8) 8323 ioc->pd_handles_sz++; 8324 ioc->pd_handles = kzalloc(ioc->pd_handles_sz, 8325 GFP_KERNEL); 8326 if (!ioc->pd_handles) { 8327 r = -ENOMEM; 8328 goto out_free_resources; 8329 } 8330 ioc->blocking_handles = kzalloc(ioc->pd_handles_sz, 8331 GFP_KERNEL); 8332 if (!ioc->blocking_handles) { 8333 r = -ENOMEM; 8334 goto out_free_resources; 8335 } 8336 8337 /* allocate memory for pending OS device add list */ 8338 ioc->pend_os_device_add_sz = (ioc->facts.MaxDevHandle / 8); 8339 if (ioc->facts.MaxDevHandle % 8) 8340 ioc->pend_os_device_add_sz++; 8341 ioc->pend_os_device_add = kzalloc(ioc->pend_os_device_add_sz, 8342 GFP_KERNEL); 8343 if (!ioc->pend_os_device_add) { 8344 r = -ENOMEM; 8345 goto out_free_resources; 8346 } 8347 8348 ioc->device_remove_in_progress_sz = ioc->pend_os_device_add_sz; 8349 ioc->device_remove_in_progress = 8350 kzalloc(ioc->device_remove_in_progress_sz, GFP_KERNEL); 8351 if (!ioc->device_remove_in_progress) { 8352 r = -ENOMEM; 8353 goto out_free_resources; 8354 } 8355 8356 ioc->fwfault_debug = mpt3sas_fwfault_debug; 8357 8358 /* base internal command bits */ 8359 mutex_init(&ioc->base_cmds.mutex); 8360 ioc->base_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL); 8361 ioc->base_cmds.status = MPT3_CMD_NOT_USED; 8362 8363 /* port_enable command bits */ 8364 ioc->port_enable_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL); 8365 ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED; 8366 8367 /* transport internal command bits */ 8368 ioc->transport_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL); 8369 ioc->transport_cmds.status = MPT3_CMD_NOT_USED; 8370 mutex_init(&ioc->transport_cmds.mutex); 8371 8372 /* scsih internal command bits */ 8373 ioc->scsih_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL); 8374 ioc->scsih_cmds.status = MPT3_CMD_NOT_USED; 8375 mutex_init(&ioc->scsih_cmds.mutex); 8376 8377 /* task management internal command bits */ 8378 ioc->tm_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL); 8379 ioc->tm_cmds.status = MPT3_CMD_NOT_USED; 8380 mutex_init(&ioc->tm_cmds.mutex); 8381 8382 /* config page internal command bits */ 8383 ioc->config_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL); 8384 ioc->config_cmds.status = MPT3_CMD_NOT_USED; 8385 mutex_init(&ioc->config_cmds.mutex); 8386 8387 /* ctl module internal command bits */ 8388 ioc->ctl_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL); 8389 ioc->ctl_cmds.sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL); 8390 ioc->ctl_cmds.status = MPT3_CMD_NOT_USED; 8391 mutex_init(&ioc->ctl_cmds.mutex); 8392 8393 if (!ioc->base_cmds.reply || !ioc->port_enable_cmds.reply || 8394 !ioc->transport_cmds.reply || !ioc->scsih_cmds.reply || 8395 !ioc->tm_cmds.reply || !ioc->config_cmds.reply || 8396 !ioc->ctl_cmds.reply || !ioc->ctl_cmds.sense) { 8397 r = -ENOMEM; 8398 goto out_free_resources; 8399 } 8400 8401 for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++) 8402 ioc->event_masks[i] = -1; 8403 8404 /* here we enable the events we care about */ 8405 _base_unmask_events(ioc, MPI2_EVENT_SAS_DISCOVERY); 8406 _base_unmask_events(ioc, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE); 8407 _base_unmask_events(ioc, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST); 8408 _base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE); 8409 _base_unmask_events(ioc, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE); 8410 _base_unmask_events(ioc, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST); 8411 _base_unmask_events(ioc, MPI2_EVENT_IR_VOLUME); 8412 _base_unmask_events(ioc, MPI2_EVENT_IR_PHYSICAL_DISK); 8413 _base_unmask_events(ioc, MPI2_EVENT_IR_OPERATION_STATUS); 8414 _base_unmask_events(ioc, MPI2_EVENT_LOG_ENTRY_ADDED); 8415 _base_unmask_events(ioc, MPI2_EVENT_TEMP_THRESHOLD); 8416 _base_unmask_events(ioc, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION); 8417 _base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR); 8418 if (ioc->hba_mpi_version_belonged == MPI26_VERSION) { 8419 if (ioc->is_gen35_ioc) { 8420 _base_unmask_events(ioc, 8421 MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE); 8422 _base_unmask_events(ioc, MPI2_EVENT_PCIE_ENUMERATION); 8423 _base_unmask_events(ioc, 8424 MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST); 8425 } 8426 } 8427 r = _base_make_ioc_operational(ioc); 8428 if (r == -EAGAIN) { 8429 r = _base_make_ioc_operational(ioc); 8430 if (r) 8431 goto out_free_resources; 8432 } 8433 8434 /* 8435 * Copy current copy of IOCFacts in prev_fw_facts 8436 * and it will be used during online firmware upgrade. 8437 */ 8438 memcpy(&ioc->prev_fw_facts, &ioc->facts, 8439 sizeof(struct mpt3sas_facts)); 8440 8441 ioc->non_operational_loop = 0; 8442 ioc->ioc_coredump_loop = 0; 8443 ioc->got_task_abort_from_ioctl = 0; 8444 return 0; 8445 8446 out_free_resources: 8447 8448 ioc->remove_host = 1; 8449 8450 mpt3sas_base_free_resources(ioc); 8451 _base_release_memory_pools(ioc); 8452 pci_set_drvdata(ioc->pdev, NULL); 8453 kfree(ioc->cpu_msix_table); 8454 if (ioc->is_warpdrive) 8455 kfree(ioc->reply_post_host_index); 8456 kfree(ioc->pd_handles); 8457 kfree(ioc->blocking_handles); 8458 kfree(ioc->device_remove_in_progress); 8459 kfree(ioc->pend_os_device_add); 8460 kfree(ioc->tm_cmds.reply); 8461 kfree(ioc->transport_cmds.reply); 8462 kfree(ioc->scsih_cmds.reply); 8463 kfree(ioc->config_cmds.reply); 8464 kfree(ioc->base_cmds.reply); 8465 kfree(ioc->port_enable_cmds.reply); 8466 kfree(ioc->ctl_cmds.reply); 8467 kfree(ioc->ctl_cmds.sense); 8468 kfree(ioc->pfacts); 8469 ioc->ctl_cmds.reply = NULL; 8470 ioc->base_cmds.reply = NULL; 8471 ioc->tm_cmds.reply = NULL; 8472 ioc->scsih_cmds.reply = NULL; 8473 ioc->transport_cmds.reply = NULL; 8474 ioc->config_cmds.reply = NULL; 8475 ioc->pfacts = NULL; 8476 return r; 8477 } 8478 8479 8480 /** 8481 * mpt3sas_base_detach - remove controller instance 8482 * @ioc: per adapter object 8483 */ 8484 void 8485 mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc) 8486 { 8487 dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); 8488 8489 mpt3sas_base_stop_watchdog(ioc); 8490 mpt3sas_base_free_resources(ioc); 8491 _base_release_memory_pools(ioc); 8492 mpt3sas_free_enclosure_list(ioc); 8493 pci_set_drvdata(ioc->pdev, NULL); 8494 kfree(ioc->cpu_msix_table); 8495 if (ioc->is_warpdrive) 8496 kfree(ioc->reply_post_host_index); 8497 kfree(ioc->pd_handles); 8498 kfree(ioc->blocking_handles); 8499 kfree(ioc->device_remove_in_progress); 8500 kfree(ioc->pend_os_device_add); 8501 kfree(ioc->pfacts); 8502 kfree(ioc->ctl_cmds.reply); 8503 kfree(ioc->ctl_cmds.sense); 8504 kfree(ioc->base_cmds.reply); 8505 kfree(ioc->port_enable_cmds.reply); 8506 kfree(ioc->tm_cmds.reply); 8507 kfree(ioc->transport_cmds.reply); 8508 kfree(ioc->scsih_cmds.reply); 8509 kfree(ioc->config_cmds.reply); 8510 } 8511 8512 /** 8513 * _base_pre_reset_handler - pre reset handler 8514 * @ioc: per adapter object 8515 */ 8516 static void _base_pre_reset_handler(struct MPT3SAS_ADAPTER *ioc) 8517 { 8518 mpt3sas_scsih_pre_reset_handler(ioc); 8519 mpt3sas_ctl_pre_reset_handler(ioc); 8520 dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_PRE_RESET\n", __func__)); 8521 } 8522 8523 /** 8524 * _base_clear_outstanding_mpt_commands - clears outstanding mpt commands 8525 * @ioc: per adapter object 8526 */ 8527 static void 8528 _base_clear_outstanding_mpt_commands(struct MPT3SAS_ADAPTER *ioc) 8529 { 8530 dtmprintk(ioc, 8531 ioc_info(ioc, "%s: clear outstanding mpt cmds\n", __func__)); 8532 if (ioc->transport_cmds.status & MPT3_CMD_PENDING) { 8533 ioc->transport_cmds.status |= MPT3_CMD_RESET; 8534 mpt3sas_base_free_smid(ioc, ioc->transport_cmds.smid); 8535 complete(&ioc->transport_cmds.done); 8536 } 8537 if (ioc->base_cmds.status & MPT3_CMD_PENDING) { 8538 ioc->base_cmds.status |= MPT3_CMD_RESET; 8539 mpt3sas_base_free_smid(ioc, ioc->base_cmds.smid); 8540 complete(&ioc->base_cmds.done); 8541 } 8542 if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) { 8543 ioc->port_enable_failed = 1; 8544 ioc->port_enable_cmds.status |= MPT3_CMD_RESET; 8545 mpt3sas_base_free_smid(ioc, ioc->port_enable_cmds.smid); 8546 if (ioc->is_driver_loading) { 8547 ioc->start_scan_failed = 8548 MPI2_IOCSTATUS_INTERNAL_ERROR; 8549 ioc->start_scan = 0; 8550 } else { 8551 complete(&ioc->port_enable_cmds.done); 8552 } 8553 } 8554 if (ioc->config_cmds.status & MPT3_CMD_PENDING) { 8555 ioc->config_cmds.status |= MPT3_CMD_RESET; 8556 mpt3sas_base_free_smid(ioc, ioc->config_cmds.smid); 8557 ioc->config_cmds.smid = USHRT_MAX; 8558 complete(&ioc->config_cmds.done); 8559 } 8560 } 8561 8562 /** 8563 * _base_clear_outstanding_commands - clear all outstanding commands 8564 * @ioc: per adapter object 8565 */ 8566 static void _base_clear_outstanding_commands(struct MPT3SAS_ADAPTER *ioc) 8567 { 8568 mpt3sas_scsih_clear_outstanding_scsi_tm_commands(ioc); 8569 mpt3sas_ctl_clear_outstanding_ioctls(ioc); 8570 _base_clear_outstanding_mpt_commands(ioc); 8571 } 8572 8573 /** 8574 * _base_reset_done_handler - reset done handler 8575 * @ioc: per adapter object 8576 */ 8577 static void _base_reset_done_handler(struct MPT3SAS_ADAPTER *ioc) 8578 { 8579 mpt3sas_scsih_reset_done_handler(ioc); 8580 mpt3sas_ctl_reset_done_handler(ioc); 8581 dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_DONE_RESET\n", __func__)); 8582 } 8583 8584 /** 8585 * mpt3sas_wait_for_commands_to_complete - reset controller 8586 * @ioc: Pointer to MPT_ADAPTER structure 8587 * 8588 * This function is waiting 10s for all pending commands to complete 8589 * prior to putting controller in reset. 8590 */ 8591 void 8592 mpt3sas_wait_for_commands_to_complete(struct MPT3SAS_ADAPTER *ioc) 8593 { 8594 u32 ioc_state; 8595 8596 ioc->pending_io_count = 0; 8597 8598 ioc_state = mpt3sas_base_get_iocstate(ioc, 0); 8599 if ((ioc_state & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL) 8600 return; 8601 8602 /* pending command count */ 8603 ioc->pending_io_count = scsi_host_busy(ioc->shost); 8604 8605 if (!ioc->pending_io_count) 8606 return; 8607 8608 /* wait for pending commands to complete */ 8609 wait_event_timeout(ioc->reset_wq, ioc->pending_io_count == 0, 10 * HZ); 8610 } 8611 8612 /** 8613 * _base_check_ioc_facts_changes - Look for increase/decrease of IOCFacts 8614 * attributes during online firmware upgrade and update the corresponding 8615 * IOC variables accordingly. 8616 * 8617 * @ioc: Pointer to MPT_ADAPTER structure 8618 */ 8619 static int 8620 _base_check_ioc_facts_changes(struct MPT3SAS_ADAPTER *ioc) 8621 { 8622 u16 pd_handles_sz; 8623 void *pd_handles = NULL, *blocking_handles = NULL; 8624 void *pend_os_device_add = NULL, *device_remove_in_progress = NULL; 8625 struct mpt3sas_facts *old_facts = &ioc->prev_fw_facts; 8626 8627 if (ioc->facts.MaxDevHandle > old_facts->MaxDevHandle) { 8628 pd_handles_sz = (ioc->facts.MaxDevHandle / 8); 8629 if (ioc->facts.MaxDevHandle % 8) 8630 pd_handles_sz++; 8631 8632 pd_handles = krealloc(ioc->pd_handles, pd_handles_sz, 8633 GFP_KERNEL); 8634 if (!pd_handles) { 8635 ioc_info(ioc, 8636 "Unable to allocate the memory for pd_handles of sz: %d\n", 8637 pd_handles_sz); 8638 return -ENOMEM; 8639 } 8640 memset(pd_handles + ioc->pd_handles_sz, 0, 8641 (pd_handles_sz - ioc->pd_handles_sz)); 8642 ioc->pd_handles = pd_handles; 8643 8644 blocking_handles = krealloc(ioc->blocking_handles, 8645 pd_handles_sz, GFP_KERNEL); 8646 if (!blocking_handles) { 8647 ioc_info(ioc, 8648 "Unable to allocate the memory for " 8649 "blocking_handles of sz: %d\n", 8650 pd_handles_sz); 8651 return -ENOMEM; 8652 } 8653 memset(blocking_handles + ioc->pd_handles_sz, 0, 8654 (pd_handles_sz - ioc->pd_handles_sz)); 8655 ioc->blocking_handles = blocking_handles; 8656 ioc->pd_handles_sz = pd_handles_sz; 8657 8658 pend_os_device_add = krealloc(ioc->pend_os_device_add, 8659 pd_handles_sz, GFP_KERNEL); 8660 if (!pend_os_device_add) { 8661 ioc_info(ioc, 8662 "Unable to allocate the memory for pend_os_device_add of sz: %d\n", 8663 pd_handles_sz); 8664 return -ENOMEM; 8665 } 8666 memset(pend_os_device_add + ioc->pend_os_device_add_sz, 0, 8667 (pd_handles_sz - ioc->pend_os_device_add_sz)); 8668 ioc->pend_os_device_add = pend_os_device_add; 8669 ioc->pend_os_device_add_sz = pd_handles_sz; 8670 8671 device_remove_in_progress = krealloc( 8672 ioc->device_remove_in_progress, pd_handles_sz, GFP_KERNEL); 8673 if (!device_remove_in_progress) { 8674 ioc_info(ioc, 8675 "Unable to allocate the memory for " 8676 "device_remove_in_progress of sz: %d\n " 8677 , pd_handles_sz); 8678 return -ENOMEM; 8679 } 8680 memset(device_remove_in_progress + 8681 ioc->device_remove_in_progress_sz, 0, 8682 (pd_handles_sz - ioc->device_remove_in_progress_sz)); 8683 ioc->device_remove_in_progress = device_remove_in_progress; 8684 ioc->device_remove_in_progress_sz = pd_handles_sz; 8685 } 8686 8687 memcpy(&ioc->prev_fw_facts, &ioc->facts, sizeof(struct mpt3sas_facts)); 8688 return 0; 8689 } 8690 8691 /** 8692 * mpt3sas_base_hard_reset_handler - reset controller 8693 * @ioc: Pointer to MPT_ADAPTER structure 8694 * @type: FORCE_BIG_HAMMER or SOFT_RESET 8695 * 8696 * Return: 0 for success, non-zero for failure. 8697 */ 8698 int 8699 mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc, 8700 enum reset_type type) 8701 { 8702 int r; 8703 unsigned long flags; 8704 u32 ioc_state; 8705 u8 is_fault = 0, is_trigger = 0; 8706 8707 dtmprintk(ioc, ioc_info(ioc, "%s: enter\n", __func__)); 8708 8709 if (ioc->pci_error_recovery) { 8710 ioc_err(ioc, "%s: pci error recovery reset\n", __func__); 8711 r = 0; 8712 goto out_unlocked; 8713 } 8714 8715 if (mpt3sas_fwfault_debug) 8716 mpt3sas_halt_firmware(ioc); 8717 8718 /* wait for an active reset in progress to complete */ 8719 mutex_lock(&ioc->reset_in_progress_mutex); 8720 8721 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); 8722 ioc->shost_recovery = 1; 8723 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); 8724 8725 if ((ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] & 8726 MPT3_DIAG_BUFFER_IS_REGISTERED) && 8727 (!(ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] & 8728 MPT3_DIAG_BUFFER_IS_RELEASED))) { 8729 is_trigger = 1; 8730 ioc_state = mpt3sas_base_get_iocstate(ioc, 0); 8731 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT || 8732 (ioc_state & MPI2_IOC_STATE_MASK) == 8733 MPI2_IOC_STATE_COREDUMP) { 8734 is_fault = 1; 8735 ioc->htb_rel.trigger_info_dwords[1] = 8736 (ioc_state & MPI2_DOORBELL_DATA_MASK); 8737 } 8738 } 8739 _base_pre_reset_handler(ioc); 8740 mpt3sas_wait_for_commands_to_complete(ioc); 8741 mpt3sas_base_mask_interrupts(ioc); 8742 mpt3sas_base_pause_mq_polling(ioc); 8743 r = mpt3sas_base_make_ioc_ready(ioc, type); 8744 if (r) 8745 goto out; 8746 _base_clear_outstanding_commands(ioc); 8747 8748 /* If this hard reset is called while port enable is active, then 8749 * there is no reason to call make_ioc_operational 8750 */ 8751 if (ioc->is_driver_loading && ioc->port_enable_failed) { 8752 ioc->remove_host = 1; 8753 r = -EFAULT; 8754 goto out; 8755 } 8756 r = _base_get_ioc_facts(ioc); 8757 if (r) 8758 goto out; 8759 8760 r = _base_check_ioc_facts_changes(ioc); 8761 if (r) { 8762 ioc_info(ioc, 8763 "Some of the parameters got changed in this new firmware" 8764 " image and it requires system reboot\n"); 8765 goto out; 8766 } 8767 if (ioc->rdpq_array_enable && !ioc->rdpq_array_capable) 8768 panic("%s: Issue occurred with flashing controller firmware." 8769 "Please reboot the system and ensure that the correct" 8770 " firmware version is running\n", ioc->name); 8771 8772 r = _base_make_ioc_operational(ioc); 8773 if (!r) 8774 _base_reset_done_handler(ioc); 8775 8776 out: 8777 ioc_info(ioc, "%s: %s\n", __func__, r == 0 ? "SUCCESS" : "FAILED"); 8778 8779 spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); 8780 ioc->shost_recovery = 0; 8781 spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); 8782 ioc->ioc_reset_count++; 8783 mutex_unlock(&ioc->reset_in_progress_mutex); 8784 mpt3sas_base_resume_mq_polling(ioc); 8785 8786 out_unlocked: 8787 if ((r == 0) && is_trigger) { 8788 if (is_fault) 8789 mpt3sas_trigger_master(ioc, MASTER_TRIGGER_FW_FAULT); 8790 else 8791 mpt3sas_trigger_master(ioc, 8792 MASTER_TRIGGER_ADAPTER_RESET); 8793 } 8794 dtmprintk(ioc, ioc_info(ioc, "%s: exit\n", __func__)); 8795 return r; 8796 } 8797