1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Driver for Broadcom MPI3 Storage Controllers 4 * 5 * Copyright (C) 2017-2022 Broadcom Inc. 6 * (mailto: mpi3mr-linuxdrv.pdl@broadcom.com) 7 * 8 */ 9 10 #ifndef MPI3MR_H_INCLUDED 11 #define MPI3MR_H_INCLUDED 12 13 #include <linux/blkdev.h> 14 #include <linux/blk-mq.h> 15 #include <linux/blk-mq-pci.h> 16 #include <linux/delay.h> 17 #include <linux/dmapool.h> 18 #include <linux/errno.h> 19 #include <linux/init.h> 20 #include <linux/io.h> 21 #include <linux/interrupt.h> 22 #include <linux/kernel.h> 23 #include <linux/miscdevice.h> 24 #include <linux/module.h> 25 #include <linux/pci.h> 26 #include <linux/poll.h> 27 #include <linux/sched.h> 28 #include <linux/slab.h> 29 #include <linux/types.h> 30 #include <linux/uaccess.h> 31 #include <linux/utsname.h> 32 #include <linux/version.h> 33 #include <linux/workqueue.h> 34 #include <asm/unaligned.h> 35 #include <scsi/scsi.h> 36 #include <scsi/scsi_cmnd.h> 37 #include <scsi/scsi_dbg.h> 38 #include <scsi/scsi_device.h> 39 #include <scsi/scsi_host.h> 40 #include <scsi/scsi_tcq.h> 41 #include <uapi/scsi/scsi_bsg_mpi3mr.h> 42 43 #include "mpi/mpi30_transport.h" 44 #include "mpi/mpi30_cnfg.h" 45 #include "mpi/mpi30_image.h" 46 #include "mpi/mpi30_init.h" 47 #include "mpi/mpi30_ioc.h" 48 #include "mpi/mpi30_sas.h" 49 #include "mpi/mpi30_pci.h" 50 #include "mpi3mr_debug.h" 51 52 /* Global list and lock for storing multiple adapters managed by the driver */ 53 extern spinlock_t mrioc_list_lock; 54 extern struct list_head mrioc_list; 55 extern int prot_mask; 56 extern atomic64_t event_counter; 57 58 #define MPI3MR_DRIVER_VERSION "8.0.0.69.0" 59 #define MPI3MR_DRIVER_RELDATE "16-March-2022" 60 61 #define MPI3MR_DRIVER_NAME "mpi3mr" 62 #define MPI3MR_DRIVER_LICENSE "GPL" 63 #define MPI3MR_DRIVER_AUTHOR "Broadcom Inc. <mpi3mr-linuxdrv.pdl@broadcom.com>" 64 #define MPI3MR_DRIVER_DESC "MPI3 Storage Controller Device Driver" 65 66 #define MPI3MR_NAME_LENGTH 32 67 #define IOCNAME "%s: " 68 69 #define MPI3MR_MAX_SECTORS 2048 70 71 /* Definitions for internal SGL and Chain SGL buffers */ 72 #define MPI3MR_PAGE_SIZE_4K 4096 73 #define MPI3MR_SG_DEPTH (MPI3MR_PAGE_SIZE_4K / sizeof(struct mpi3_sge_common)) 74 75 /* Definitions for MAX values for shost */ 76 #define MPI3MR_MAX_CMDS_LUN 128 77 #define MPI3MR_MAX_CDB_LENGTH 32 78 79 /* Admin queue management definitions */ 80 #define MPI3MR_ADMIN_REQ_Q_SIZE (2 * MPI3MR_PAGE_SIZE_4K) 81 #define MPI3MR_ADMIN_REPLY_Q_SIZE (4 * MPI3MR_PAGE_SIZE_4K) 82 #define MPI3MR_ADMIN_REQ_FRAME_SZ 128 83 #define MPI3MR_ADMIN_REPLY_FRAME_SZ 16 84 85 /* Operational queue management definitions */ 86 #define MPI3MR_OP_REQ_Q_QD 512 87 #define MPI3MR_OP_REP_Q_QD 1024 88 #define MPI3MR_OP_REP_Q_QD4K 4096 89 #define MPI3MR_OP_REQ_Q_SEG_SIZE 4096 90 #define MPI3MR_OP_REP_Q_SEG_SIZE 4096 91 #define MPI3MR_MAX_SEG_LIST_SIZE 4096 92 93 /* Reserved Host Tag definitions */ 94 #define MPI3MR_HOSTTAG_INVALID 0xFFFF 95 #define MPI3MR_HOSTTAG_INITCMDS 1 96 #define MPI3MR_HOSTTAG_BSG_CMDS 2 97 #define MPI3MR_HOSTTAG_PEL_ABORT 3 98 #define MPI3MR_HOSTTAG_PEL_WAIT 4 99 #define MPI3MR_HOSTTAG_BLK_TMS 5 100 101 #define MPI3MR_NUM_DEVRMCMD 16 102 #define MPI3MR_HOSTTAG_DEVRMCMD_MIN (MPI3MR_HOSTTAG_BLK_TMS + 1) 103 #define MPI3MR_HOSTTAG_DEVRMCMD_MAX (MPI3MR_HOSTTAG_DEVRMCMD_MIN + \ 104 MPI3MR_NUM_DEVRMCMD - 1) 105 106 #define MPI3MR_INTERNAL_CMDS_RESVD MPI3MR_HOSTTAG_DEVRMCMD_MAX 107 #define MPI3MR_NUM_EVTACKCMD 4 108 #define MPI3MR_HOSTTAG_EVTACKCMD_MIN (MPI3MR_HOSTTAG_DEVRMCMD_MAX + 1) 109 #define MPI3MR_HOSTTAG_EVTACKCMD_MAX (MPI3MR_HOSTTAG_EVTACKCMD_MIN + \ 110 MPI3MR_NUM_EVTACKCMD - 1) 111 112 /* Reduced resource count definition for crash kernel */ 113 #define MPI3MR_HOST_IOS_KDUMP 128 114 115 /* command/controller interaction timeout definitions in seconds */ 116 #define MPI3MR_INTADMCMD_TIMEOUT 60 117 #define MPI3MR_PORTENABLE_TIMEOUT 300 118 #define MPI3MR_ABORTTM_TIMEOUT 60 119 #define MPI3MR_RESETTM_TIMEOUT 60 120 #define MPI3MR_RESET_HOST_IOWAIT_TIMEOUT 5 121 #define MPI3MR_TSUPDATE_INTERVAL 900 122 #define MPI3MR_DEFAULT_SHUTDOWN_TIME 120 123 #define MPI3MR_RAID_ERRREC_RESET_TIMEOUT 180 124 #define MPI3MR_PREPARE_FOR_RESET_TIMEOUT 180 125 #define MPI3MR_RESET_ACK_TIMEOUT 30 126 127 #define MPI3MR_WATCHDOG_INTERVAL 1000 /* in milli seconds */ 128 129 #define MPI3MR_SCMD_TIMEOUT (60 * HZ) 130 #define MPI3MR_EH_SCMD_TIMEOUT (60 * HZ) 131 132 /* Internal admin command state definitions*/ 133 #define MPI3MR_CMD_NOTUSED 0x8000 134 #define MPI3MR_CMD_COMPLETE 0x0001 135 #define MPI3MR_CMD_PENDING 0x0002 136 #define MPI3MR_CMD_REPLY_VALID 0x0004 137 #define MPI3MR_CMD_RESET 0x0008 138 139 /* Definitions for Event replies and sense buffer allocated per controller */ 140 #define MPI3MR_NUM_EVT_REPLIES 64 141 #define MPI3MR_SENSE_BUF_SZ 256 142 #define MPI3MR_SENSEBUF_FACTOR 3 143 #define MPI3MR_CHAINBUF_FACTOR 3 144 #define MPI3MR_CHAINBUFDIX_FACTOR 2 145 146 /* Invalid target device handle */ 147 #define MPI3MR_INVALID_DEV_HANDLE 0xFFFF 148 149 /* Controller Reset related definitions */ 150 #define MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT 5 151 #define MPI3MR_MAX_RESET_RETRY_COUNT 3 152 153 /* ResponseCode definitions */ 154 #define MPI3MR_RI_MASK_RESPCODE (0x000000FF) 155 #define MPI3MR_RSP_IO_QUEUED_ON_IOC \ 156 MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC 157 158 #define MPI3MR_DEFAULT_MDTS (128 * 1024) 159 #define MPI3MR_DEFAULT_PGSZEXP (12) 160 161 /* Command retry count definitions */ 162 #define MPI3MR_DEV_RMHS_RETRY_COUNT 3 163 #define MPI3MR_PEL_RETRY_COUNT 3 164 165 /* Default target device queue depth */ 166 #define MPI3MR_DEFAULT_SDEV_QD 32 167 168 /* Definitions for Threaded IRQ poll*/ 169 #define MPI3MR_IRQ_POLL_SLEEP 2 170 #define MPI3MR_IRQ_POLL_TRIGGER_IOCOUNT 8 171 172 /* Definitions for the controller security status*/ 173 #define MPI3MR_CTLR_SECURITY_STATUS_MASK 0x0C 174 #define MPI3MR_CTLR_SECURE_DBG_STATUS_MASK 0x02 175 176 #define MPI3MR_INVALID_DEVICE 0x00 177 #define MPI3MR_CONFIG_SECURE_DEVICE 0x04 178 #define MPI3MR_HARD_SECURE_DEVICE 0x08 179 #define MPI3MR_TAMPERED_DEVICE 0x0C 180 181 /* SGE Flag definition */ 182 #define MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST \ 183 (MPI3_SGE_FLAGS_ELEMENT_TYPE_SIMPLE | MPI3_SGE_FLAGS_DLAS_SYSTEM | \ 184 MPI3_SGE_FLAGS_END_OF_LIST) 185 186 /* MSI Index from Reply Queue Index */ 187 #define REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, offset) (qidx + offset) 188 189 /* 190 * Maximum data transfer size definitions for management 191 * application commands 192 */ 193 #define MPI3MR_MAX_APP_XFER_SIZE (1 * 1024 * 1024) 194 #define MPI3MR_MAX_APP_XFER_SEGMENTS 512 195 /* 196 * 2048 sectors are for data buffers and additional 512 sectors for 197 * other buffers 198 */ 199 #define MPI3MR_MAX_APP_XFER_SECTORS (2048 + 512) 200 201 /** 202 * struct mpi3mr_nvme_pt_sge - Structure to store SGEs for NVMe 203 * Encapsulated commands. 204 * 205 * @base_addr: Physical address 206 * @length: SGE length 207 * @rsvd: Reserved 208 * @rsvd1: Reserved 209 * @sgl_type: sgl type 210 */ 211 struct mpi3mr_nvme_pt_sge { 212 u64 base_addr; 213 u32 length; 214 u16 rsvd; 215 u8 rsvd1; 216 u8 sgl_type; 217 }; 218 219 /** 220 * struct mpi3mr_buf_map - local structure to 221 * track kernel and user buffers associated with an BSG 222 * structure. 223 * 224 * @bsg_buf: BSG buffer virtual address 225 * @bsg_buf_len: BSG buffer length 226 * @kern_buf: Kernel buffer virtual address 227 * @kern_buf_len: Kernel buffer length 228 * @kern_buf_dma: Kernel buffer DMA address 229 * @data_dir: Data direction. 230 */ 231 struct mpi3mr_buf_map { 232 void *bsg_buf; 233 u32 bsg_buf_len; 234 void *kern_buf; 235 u32 kern_buf_len; 236 dma_addr_t kern_buf_dma; 237 u8 data_dir; 238 }; 239 240 /* IOC State definitions */ 241 enum mpi3mr_iocstate { 242 MRIOC_STATE_READY = 1, 243 MRIOC_STATE_RESET, 244 MRIOC_STATE_FAULT, 245 MRIOC_STATE_BECOMING_READY, 246 MRIOC_STATE_RESET_REQUESTED, 247 MRIOC_STATE_UNRECOVERABLE, 248 }; 249 250 /* Reset reason code definitions*/ 251 enum mpi3mr_reset_reason { 252 MPI3MR_RESET_FROM_BRINGUP = 1, 253 MPI3MR_RESET_FROM_FAULT_WATCH = 2, 254 MPI3MR_RESET_FROM_APP = 3, 255 MPI3MR_RESET_FROM_EH_HOS = 4, 256 MPI3MR_RESET_FROM_TM_TIMEOUT = 5, 257 MPI3MR_RESET_FROM_APP_TIMEOUT = 6, 258 MPI3MR_RESET_FROM_MUR_FAILURE = 7, 259 MPI3MR_RESET_FROM_CTLR_CLEANUP = 8, 260 MPI3MR_RESET_FROM_CIACTIV_FAULT = 9, 261 MPI3MR_RESET_FROM_PE_TIMEOUT = 10, 262 MPI3MR_RESET_FROM_TSU_TIMEOUT = 11, 263 MPI3MR_RESET_FROM_DELREQQ_TIMEOUT = 12, 264 MPI3MR_RESET_FROM_DELREPQ_TIMEOUT = 13, 265 MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT = 14, 266 MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT = 15, 267 MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT = 16, 268 MPI3MR_RESET_FROM_IOCINIT_TIMEOUT = 17, 269 MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT = 18, 270 MPI3MR_RESET_FROM_EVTACK_TIMEOUT = 19, 271 MPI3MR_RESET_FROM_CIACTVRST_TIMER = 20, 272 MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT = 21, 273 MPI3MR_RESET_FROM_PELABORT_TIMEOUT = 22, 274 MPI3MR_RESET_FROM_SYSFS = 23, 275 MPI3MR_RESET_FROM_SYSFS_TIMEOUT = 24, 276 MPI3MR_RESET_FROM_FIRMWARE = 27, 277 }; 278 279 /* Queue type definitions */ 280 enum queue_type { 281 MPI3MR_DEFAULT_QUEUE = 0, 282 MPI3MR_POLL_QUEUE, 283 }; 284 285 /** 286 * struct mpi3mr_compimg_ver - replica of component image 287 * version defined in mpi30_image.h in host endianness 288 * 289 */ 290 struct mpi3mr_compimg_ver { 291 u16 build_num; 292 u16 cust_id; 293 u8 ph_minor; 294 u8 ph_major; 295 u8 gen_minor; 296 u8 gen_major; 297 }; 298 299 /** 300 * struct mpi3mr_ioc_facs - replica of component image version 301 * defined in mpi30_ioc.h in host endianness 302 * 303 */ 304 struct mpi3mr_ioc_facts { 305 u32 ioc_capabilities; 306 struct mpi3mr_compimg_ver fw_ver; 307 u32 mpi_version; 308 u16 max_reqs; 309 u16 product_id; 310 u16 op_req_sz; 311 u16 reply_sz; 312 u16 exceptions; 313 u16 max_perids; 314 u16 max_pds; 315 u16 max_sasexpanders; 316 u16 max_sasinitiators; 317 u16 max_enclosures; 318 u16 max_pcie_switches; 319 u16 max_nvme; 320 u16 max_vds; 321 u16 max_hpds; 322 u16 max_advhpds; 323 u16 max_raid_pds; 324 u16 min_devhandle; 325 u16 max_devhandle; 326 u16 max_op_req_q; 327 u16 max_op_reply_q; 328 u16 shutdown_timeout; 329 u8 ioc_num; 330 u8 who_init; 331 u16 max_msix_vectors; 332 u8 personality; 333 u8 dma_mask; 334 u8 protocol_flags; 335 u8 sge_mod_mask; 336 u8 sge_mod_value; 337 u8 sge_mod_shift; 338 u8 max_dev_per_tg; 339 u16 max_io_throttle_group; 340 u16 io_throttle_data_length; 341 u16 io_throttle_low; 342 u16 io_throttle_high; 343 344 }; 345 346 /** 347 * struct segments - memory descriptor structure to store 348 * virtual and dma addresses for operational queue segments. 349 * 350 * @segment: virtual address 351 * @segment_dma: dma address 352 */ 353 struct segments { 354 void *segment; 355 dma_addr_t segment_dma; 356 }; 357 358 /** 359 * struct op_req_qinfo - Operational Request Queue Information 360 * 361 * @ci: consumer index 362 * @pi: producer index 363 * @num_request: Maximum number of entries in the queue 364 * @qid: Queue Id starting from 1 365 * @reply_qid: Associated reply queue Id 366 * @num_segments: Number of discontiguous memory segments 367 * @segment_qd: Depth of each segments 368 * @q_lock: Concurrent queue access lock 369 * @q_segments: Segment descriptor pointer 370 * @q_segment_list: Segment list base virtual address 371 * @q_segment_list_dma: Segment list base DMA address 372 */ 373 struct op_req_qinfo { 374 u16 ci; 375 u16 pi; 376 u16 num_requests; 377 u16 qid; 378 u16 reply_qid; 379 u16 num_segments; 380 u16 segment_qd; 381 spinlock_t q_lock; 382 struct segments *q_segments; 383 void *q_segment_list; 384 dma_addr_t q_segment_list_dma; 385 }; 386 387 /** 388 * struct op_reply_qinfo - Operational Reply Queue Information 389 * 390 * @ci: consumer index 391 * @qid: Queue Id starting from 1 392 * @num_replies: Maximum number of entries in the queue 393 * @num_segments: Number of discontiguous memory segments 394 * @segment_qd: Depth of each segments 395 * @q_segments: Segment descriptor pointer 396 * @q_segment_list: Segment list base virtual address 397 * @q_segment_list_dma: Segment list base DMA address 398 * @ephase: Expected phased identifier for the reply queue 399 * @pend_ios: Number of IOs pending in HW for this queue 400 * @enable_irq_poll: Flag to indicate polling is enabled 401 * @in_use: Queue is handled by poll/ISR 402 * @qtype: Type of queue (types defined in enum queue_type) 403 */ 404 struct op_reply_qinfo { 405 u16 ci; 406 u16 qid; 407 u16 num_replies; 408 u16 num_segments; 409 u16 segment_qd; 410 struct segments *q_segments; 411 void *q_segment_list; 412 dma_addr_t q_segment_list_dma; 413 u8 ephase; 414 atomic_t pend_ios; 415 bool enable_irq_poll; 416 atomic_t in_use; 417 enum queue_type qtype; 418 }; 419 420 /** 421 * struct mpi3mr_intr_info - Interrupt cookie information 422 * 423 * @mrioc: Adapter instance reference 424 * @msix_index: MSIx index 425 * @op_reply_q: Associated operational reply queue 426 * @name: Dev name for the irq claiming device 427 */ 428 struct mpi3mr_intr_info { 429 struct mpi3mr_ioc *mrioc; 430 u16 msix_index; 431 struct op_reply_qinfo *op_reply_q; 432 char name[MPI3MR_NAME_LENGTH]; 433 }; 434 435 /** 436 * struct mpi3mr_throttle_group_info - Throttle group info 437 * 438 * @io_divert: Flag indicates io divert is on or off for the TG 439 * @id: Throttle Group ID. 440 * @high: High limit to turn on throttling in 512 byte blocks 441 * @low: Low limit to turn off throttling in 512 byte blocks 442 * @pend_large_data_sz: Counter to track pending large data 443 */ 444 struct mpi3mr_throttle_group_info { 445 u8 io_divert; 446 u16 id; 447 u32 high; 448 u32 low; 449 atomic_t pend_large_data_sz; 450 }; 451 452 /** 453 * struct tgt_dev_sas_sata - SAS/SATA device specific 454 * information cached from firmware given data 455 * 456 * @sas_address: World wide unique SAS address 457 * @dev_info: Device information bits 458 */ 459 struct tgt_dev_sas_sata { 460 u64 sas_address; 461 u16 dev_info; 462 }; 463 464 /** 465 * struct tgt_dev_pcie - PCIe device specific information cached 466 * from firmware given data 467 * 468 * @mdts: Maximum data transfer size 469 * @capb: Device capabilities 470 * @pgsz: Device page size 471 * @abort_to: Timeout for abort TM 472 * @reset_to: Timeout for Target/LUN reset TM 473 * @dev_info: Device information bits 474 */ 475 struct tgt_dev_pcie { 476 u32 mdts; 477 u16 capb; 478 u8 pgsz; 479 u8 abort_to; 480 u8 reset_to; 481 u16 dev_info; 482 }; 483 484 /** 485 * struct tgt_dev_vd - virtual device specific information 486 * cached from firmware given data 487 * 488 * @state: State of the VD 489 * @tg_id: VDs throttle group ID 490 * @high: High limit to turn on throttling in 512 byte blocks 491 * @low: Low limit to turn off throttling in 512 byte blocks 492 * @tg: Pointer to throttle group info 493 */ 494 struct tgt_dev_vd { 495 u8 state; 496 u16 tg_id; 497 u32 tg_high; 498 u32 tg_low; 499 struct mpi3mr_throttle_group_info *tg; 500 }; 501 502 503 /** 504 * union _form_spec_inf - union of device specific information 505 */ 506 union _form_spec_inf { 507 struct tgt_dev_sas_sata sas_sata_inf; 508 struct tgt_dev_pcie pcie_inf; 509 struct tgt_dev_vd vd_inf; 510 }; 511 512 513 514 /** 515 * struct mpi3mr_tgt_dev - target device data structure 516 * 517 * @list: List pointer 518 * @starget: Scsi_target pointer 519 * @dev_handle: FW device handle 520 * @parent_handle: FW parent device handle 521 * @slot: Slot number 522 * @encl_handle: FW enclosure handle 523 * @perst_id: FW assigned Persistent ID 524 * @dev_type: SAS/SATA/PCIE device type 525 * @is_hidden: Should be exposed to upper layers or not 526 * @host_exposed: Already exposed to host or not 527 * @io_throttle_enabled: I/O throttling needed or not 528 * @q_depth: Device specific Queue Depth 529 * @wwid: World wide ID 530 * @dev_spec: Device type specific information 531 * @ref_count: Reference count 532 */ 533 struct mpi3mr_tgt_dev { 534 struct list_head list; 535 struct scsi_target *starget; 536 u16 dev_handle; 537 u16 parent_handle; 538 u16 slot; 539 u16 encl_handle; 540 u16 perst_id; 541 u8 dev_type; 542 u8 is_hidden; 543 u8 host_exposed; 544 u8 io_throttle_enabled; 545 u16 q_depth; 546 u64 wwid; 547 union _form_spec_inf dev_spec; 548 struct kref ref_count; 549 }; 550 551 /** 552 * mpi3mr_tgtdev_get - k reference incrementor 553 * @s: Target device reference 554 * 555 * Increment target device reference count. 556 */ 557 static inline void mpi3mr_tgtdev_get(struct mpi3mr_tgt_dev *s) 558 { 559 kref_get(&s->ref_count); 560 } 561 562 /** 563 * mpi3mr_free_tgtdev - target device memory dealloctor 564 * @r: k reference pointer of the target device 565 * 566 * Free target device memory when no reference. 567 */ 568 static inline void mpi3mr_free_tgtdev(struct kref *r) 569 { 570 kfree(container_of(r, struct mpi3mr_tgt_dev, ref_count)); 571 } 572 573 /** 574 * mpi3mr_tgtdev_put - k reference decrementor 575 * @s: Target device reference 576 * 577 * Decrement target device reference count. 578 */ 579 static inline void mpi3mr_tgtdev_put(struct mpi3mr_tgt_dev *s) 580 { 581 kref_put(&s->ref_count, mpi3mr_free_tgtdev); 582 } 583 584 585 /** 586 * struct mpi3mr_stgt_priv_data - SCSI target private structure 587 * 588 * @starget: Scsi_target pointer 589 * @dev_handle: FW device handle 590 * @perst_id: FW assigned Persistent ID 591 * @num_luns: Number of Logical Units 592 * @block_io: I/O blocked to the device or not 593 * @dev_removed: Device removed in the Firmware 594 * @dev_removedelay: Device is waiting to be removed in FW 595 * @dev_type: Device type 596 * @io_throttle_enabled: I/O throttling needed or not 597 * @io_divert: Flag indicates io divert is on or off for the dev 598 * @throttle_group: Pointer to throttle group info 599 * @tgt_dev: Internal target device pointer 600 * @pend_count: Counter to track pending I/Os during error 601 * handling 602 */ 603 struct mpi3mr_stgt_priv_data { 604 struct scsi_target *starget; 605 u16 dev_handle; 606 u16 perst_id; 607 u32 num_luns; 608 atomic_t block_io; 609 u8 dev_removed; 610 u8 dev_removedelay; 611 u8 dev_type; 612 u8 io_throttle_enabled; 613 u8 io_divert; 614 struct mpi3mr_throttle_group_info *throttle_group; 615 struct mpi3mr_tgt_dev *tgt_dev; 616 u32 pend_count; 617 }; 618 619 /** 620 * struct mpi3mr_stgt_priv_data - SCSI device private structure 621 * 622 * @tgt_priv_data: Scsi_target private data pointer 623 * @lun_id: LUN ID of the device 624 * @ncq_prio_enable: NCQ priority enable for SATA device 625 * @pend_count: Counter to track pending I/Os during error 626 * handling 627 */ 628 struct mpi3mr_sdev_priv_data { 629 struct mpi3mr_stgt_priv_data *tgt_priv_data; 630 u32 lun_id; 631 u8 ncq_prio_enable; 632 u32 pend_count; 633 }; 634 635 /** 636 * struct mpi3mr_drv_cmd - Internal command tracker 637 * 638 * @mutex: Command mutex 639 * @done: Completeor for wakeup 640 * @reply: Firmware reply for internal commands 641 * @sensebuf: Sensebuf for SCSI IO commands 642 * @iou_rc: IO Unit control reason code 643 * @state: Command State 644 * @dev_handle: Firmware handle for device specific commands 645 * @ioc_status: IOC status from the firmware 646 * @ioc_loginfo:IOC log info from the firmware 647 * @is_waiting: Is the command issued in block mode 648 * @is_sense: Is Sense data present 649 * @retry_count: Retry count for retriable commands 650 * @host_tag: Host tag used by the command 651 * @callback: Callback for non blocking commands 652 */ 653 struct mpi3mr_drv_cmd { 654 struct mutex mutex; 655 struct completion done; 656 void *reply; 657 u8 *sensebuf; 658 u8 iou_rc; 659 u16 state; 660 u16 dev_handle; 661 u16 ioc_status; 662 u32 ioc_loginfo; 663 u8 is_waiting; 664 u8 is_sense; 665 u8 retry_count; 666 u16 host_tag; 667 668 void (*callback)(struct mpi3mr_ioc *mrioc, 669 struct mpi3mr_drv_cmd *drv_cmd); 670 }; 671 672 673 /** 674 * struct chain_element - memory descriptor structure to store 675 * virtual and dma addresses for chain elements. 676 * 677 * @addr: virtual address 678 * @dma_addr: dma address 679 */ 680 struct chain_element { 681 void *addr; 682 dma_addr_t dma_addr; 683 }; 684 685 /** 686 * struct scmd_priv - SCSI command private data 687 * 688 * @host_tag: Host tag specific to operational queue 689 * @in_lld_scope: Command in LLD scope or not 690 * @meta_sg_valid: DIX command with meta data SGL or not 691 * @scmd: SCSI Command pointer 692 * @req_q_idx: Operational request queue index 693 * @chain_idx: Chain frame index 694 * @meta_chain_idx: Chain frame index of meta data SGL 695 * @mpi3mr_scsiio_req: MPI SCSI IO request 696 */ 697 struct scmd_priv { 698 u16 host_tag; 699 u8 in_lld_scope; 700 u8 meta_sg_valid; 701 struct scsi_cmnd *scmd; 702 u16 req_q_idx; 703 int chain_idx; 704 int meta_chain_idx; 705 u8 mpi3mr_scsiio_req[MPI3MR_ADMIN_REQ_FRAME_SZ]; 706 }; 707 708 /** 709 * struct mpi3mr_ioc - Adapter anchor structure stored in shost 710 * private data 711 * 712 * @list: List pointer 713 * @pdev: PCI device pointer 714 * @shost: Scsi_Host pointer 715 * @id: Controller ID 716 * @cpu_count: Number of online CPUs 717 * @irqpoll_sleep: usleep unit used in threaded isr irqpoll 718 * @name: Controller ASCII name 719 * @driver_name: Driver ASCII name 720 * @sysif_regs: System interface registers virtual address 721 * @sysif_regs_phys: System interface registers physical address 722 * @bars: PCI BARS 723 * @dma_mask: DMA mask 724 * @msix_count: Number of MSIX vectors used 725 * @intr_enabled: Is interrupts enabled 726 * @num_admin_req: Number of admin requests 727 * @admin_req_q_sz: Admin request queue size 728 * @admin_req_pi: Admin request queue producer index 729 * @admin_req_ci: Admin request queue consumer index 730 * @admin_req_base: Admin request queue base virtual address 731 * @admin_req_dma: Admin request queue base dma address 732 * @admin_req_lock: Admin queue access lock 733 * @num_admin_replies: Number of admin replies 734 * @admin_reply_q_sz: Admin reply queue size 735 * @admin_reply_ci: Admin reply queue consumer index 736 * @admin_reply_ephase:Admin reply queue expected phase 737 * @admin_reply_base: Admin reply queue base virtual address 738 * @admin_reply_dma: Admin reply queue base dma address 739 * @ready_timeout: Controller ready timeout 740 * @intr_info: Interrupt cookie pointer 741 * @intr_info_count: Number of interrupt cookies 742 * @is_intr_info_set: Flag to indicate intr info is setup 743 * @num_queues: Number of operational queues 744 * @num_op_req_q: Number of operational request queues 745 * @req_qinfo: Operational request queue info pointer 746 * @num_op_reply_q: Number of operational reply queues 747 * @op_reply_qinfo: Operational reply queue info pointer 748 * @init_cmds: Command tracker for initialization commands 749 * @facts: Cached IOC facts data 750 * @op_reply_desc_sz: Operational reply descriptor size 751 * @num_reply_bufs: Number of reply buffers allocated 752 * @reply_buf_pool: Reply buffer pool 753 * @reply_buf: Reply buffer base virtual address 754 * @reply_buf_dma: Reply buffer DMA address 755 * @reply_buf_dma_max_address: Reply DMA address max limit 756 * @reply_free_qsz: Reply free queue size 757 * @reply_free_q_pool: Reply free queue pool 758 * @reply_free_q: Reply free queue base virtual address 759 * @reply_free_q_dma: Reply free queue base DMA address 760 * @reply_free_queue_lock: Reply free queue lock 761 * @reply_free_queue_host_index: Reply free queue host index 762 * @num_sense_bufs: Number of sense buffers 763 * @sense_buf_pool: Sense buffer pool 764 * @sense_buf: Sense buffer base virtual address 765 * @sense_buf_dma: Sense buffer base DMA address 766 * @sense_buf_q_sz: Sense buffer queue size 767 * @sense_buf_q_pool: Sense buffer queue pool 768 * @sense_buf_q: Sense buffer queue virtual address 769 * @sense_buf_q_dma: Sense buffer queue DMA address 770 * @sbq_lock: Sense buffer queue lock 771 * @sbq_host_index: Sense buffer queuehost index 772 * @event_masks: Event mask bitmap 773 * @fwevt_worker_name: Firmware event worker thread name 774 * @fwevt_worker_thread: Firmware event worker thread 775 * @fwevt_lock: Firmware event lock 776 * @fwevt_list: Firmware event list 777 * @watchdog_work_q_name: Fault watchdog worker thread name 778 * @watchdog_work_q: Fault watchdog worker thread 779 * @watchdog_work: Fault watchdog work 780 * @watchdog_lock: Fault watchdog lock 781 * @is_driver_loading: Is driver still loading 782 * @scan_started: Async scan started 783 * @scan_failed: Asycn scan failed 784 * @stop_drv_processing: Stop all command processing 785 * @max_host_ios: Maximum host I/O count 786 * @chain_buf_count: Chain buffer count 787 * @chain_buf_pool: Chain buffer pool 788 * @chain_sgl_list: Chain SGL list 789 * @chain_bitmap_sz: Chain buffer allocator bitmap size 790 * @chain_bitmap: Chain buffer allocator bitmap 791 * @chain_buf_lock: Chain buffer list lock 792 * @bsg_cmds: Command tracker for BSG command 793 * @host_tm_cmds: Command tracker for task management commands 794 * @dev_rmhs_cmds: Command tracker for device removal commands 795 * @evtack_cmds: Command tracker for event ack commands 796 * @devrem_bitmap_sz: Device removal bitmap size 797 * @devrem_bitmap: Device removal bitmap 798 * @dev_handle_bitmap_sz: Device handle bitmap size 799 * @removepend_bitmap: Remove pending bitmap 800 * @delayed_rmhs_list: Delayed device removal list 801 * @evtack_cmds_bitmap_sz: Event Ack bitmap size 802 * @evtack_cmds_bitmap: Event Ack bitmap 803 * @delayed_evtack_cmds_list: Delayed event acknowledgment list 804 * @ts_update_counter: Timestamp update counter 805 * @reset_in_progress: Reset in progress flag 806 * @unrecoverable: Controller unrecoverable flag 807 * @prev_reset_result: Result of previous reset 808 * @reset_mutex: Controller reset mutex 809 * @reset_waitq: Controller reset wait queue 810 * @prepare_for_reset: Prepare for reset event received 811 * @prepare_for_reset_timeout_counter: Prepare for reset timeout 812 * @prp_list_virt: NVMe encapsulated PRP list virtual base 813 * @prp_list_dma: NVMe encapsulated PRP list DMA 814 * @prp_sz: NVME encapsulated PRP list size 815 * @diagsave_timeout: Diagnostic information save timeout 816 * @logging_level: Controller debug logging level 817 * @flush_io_count: I/O count to flush after reset 818 * @current_event: Firmware event currently in process 819 * @driver_info: Driver, Kernel, OS information to firmware 820 * @change_count: Topology change count 821 * @pel_enabled: Persistent Event Log(PEL) enabled or not 822 * @pel_abort_requested: PEL abort is requested or not 823 * @pel_class: PEL Class identifier 824 * @pel_locale: PEL Locale identifier 825 * @pel_cmds: Command tracker for PEL wait command 826 * @pel_abort_cmd: Command tracker for PEL abort command 827 * @pel_newest_seqnum: Newest PEL sequenece number 828 * @pel_seqnum_virt: PEL sequence number virtual address 829 * @pel_seqnum_dma: PEL sequence number DMA address 830 * @pel_seqnum_sz: PEL sequenece number size 831 * @op_reply_q_offset: Operational reply queue offset with MSIx 832 * @default_qcount: Total Default queues 833 * @active_poll_qcount: Currently active poll queue count 834 * @requested_poll_qcount: User requested poll queue count 835 * @bsg_dev: BSG device structure 836 * @bsg_queue: Request queue for BSG device 837 * @stop_bsgs: Stop BSG request flag 838 * @logdata_buf: Circular buffer to store log data entries 839 * @logdata_buf_idx: Index of entry in buffer to store 840 * @logdata_entry_sz: log data entry size 841 * @pend_large_data_sz: Counter to track pending large data 842 * @io_throttle_data_length: I/O size to track in 512b blocks 843 * @io_throttle_high: I/O size to start throttle in 512b blocks 844 * @io_throttle_low: I/O size to stop throttle in 512b blocks 845 * @num_io_throttle_group: Maximum number of throttle groups 846 * @throttle_groups: Pointer to throttle group info structures 847 */ 848 struct mpi3mr_ioc { 849 struct list_head list; 850 struct pci_dev *pdev; 851 struct Scsi_Host *shost; 852 u8 id; 853 int cpu_count; 854 bool enable_segqueue; 855 u32 irqpoll_sleep; 856 857 char name[MPI3MR_NAME_LENGTH]; 858 char driver_name[MPI3MR_NAME_LENGTH]; 859 860 volatile struct mpi3_sysif_registers __iomem *sysif_regs; 861 resource_size_t sysif_regs_phys; 862 int bars; 863 u64 dma_mask; 864 865 u16 msix_count; 866 u8 intr_enabled; 867 868 u16 num_admin_req; 869 u32 admin_req_q_sz; 870 u16 admin_req_pi; 871 u16 admin_req_ci; 872 void *admin_req_base; 873 dma_addr_t admin_req_dma; 874 spinlock_t admin_req_lock; 875 876 u16 num_admin_replies; 877 u32 admin_reply_q_sz; 878 u16 admin_reply_ci; 879 u8 admin_reply_ephase; 880 void *admin_reply_base; 881 dma_addr_t admin_reply_dma; 882 883 u32 ready_timeout; 884 885 struct mpi3mr_intr_info *intr_info; 886 u16 intr_info_count; 887 bool is_intr_info_set; 888 889 u16 num_queues; 890 u16 num_op_req_q; 891 struct op_req_qinfo *req_qinfo; 892 893 u16 num_op_reply_q; 894 struct op_reply_qinfo *op_reply_qinfo; 895 896 struct mpi3mr_drv_cmd init_cmds; 897 struct mpi3mr_ioc_facts facts; 898 u16 op_reply_desc_sz; 899 900 u32 num_reply_bufs; 901 struct dma_pool *reply_buf_pool; 902 u8 *reply_buf; 903 dma_addr_t reply_buf_dma; 904 dma_addr_t reply_buf_dma_max_address; 905 906 u16 reply_free_qsz; 907 u16 reply_sz; 908 struct dma_pool *reply_free_q_pool; 909 __le64 *reply_free_q; 910 dma_addr_t reply_free_q_dma; 911 spinlock_t reply_free_queue_lock; 912 u32 reply_free_queue_host_index; 913 914 u32 num_sense_bufs; 915 struct dma_pool *sense_buf_pool; 916 u8 *sense_buf; 917 dma_addr_t sense_buf_dma; 918 919 u16 sense_buf_q_sz; 920 struct dma_pool *sense_buf_q_pool; 921 __le64 *sense_buf_q; 922 dma_addr_t sense_buf_q_dma; 923 spinlock_t sbq_lock; 924 u32 sbq_host_index; 925 u32 event_masks[MPI3_EVENT_NOTIFY_EVENTMASK_WORDS]; 926 927 char fwevt_worker_name[MPI3MR_NAME_LENGTH]; 928 struct workqueue_struct *fwevt_worker_thread; 929 spinlock_t fwevt_lock; 930 struct list_head fwevt_list; 931 932 char watchdog_work_q_name[20]; 933 struct workqueue_struct *watchdog_work_q; 934 struct delayed_work watchdog_work; 935 spinlock_t watchdog_lock; 936 937 u8 is_driver_loading; 938 u8 scan_started; 939 u16 scan_failed; 940 u8 stop_drv_processing; 941 942 u16 max_host_ios; 943 spinlock_t tgtdev_lock; 944 struct list_head tgtdev_list; 945 946 u32 chain_buf_count; 947 struct dma_pool *chain_buf_pool; 948 struct chain_element *chain_sgl_list; 949 u16 chain_bitmap_sz; 950 void *chain_bitmap; 951 spinlock_t chain_buf_lock; 952 953 struct mpi3mr_drv_cmd bsg_cmds; 954 struct mpi3mr_drv_cmd host_tm_cmds; 955 struct mpi3mr_drv_cmd dev_rmhs_cmds[MPI3MR_NUM_DEVRMCMD]; 956 struct mpi3mr_drv_cmd evtack_cmds[MPI3MR_NUM_EVTACKCMD]; 957 u16 devrem_bitmap_sz; 958 void *devrem_bitmap; 959 u16 dev_handle_bitmap_sz; 960 void *removepend_bitmap; 961 struct list_head delayed_rmhs_list; 962 u16 evtack_cmds_bitmap_sz; 963 void *evtack_cmds_bitmap; 964 struct list_head delayed_evtack_cmds_list; 965 966 u32 ts_update_counter; 967 u8 reset_in_progress; 968 u8 unrecoverable; 969 int prev_reset_result; 970 struct mutex reset_mutex; 971 wait_queue_head_t reset_waitq; 972 973 u8 prepare_for_reset; 974 u16 prepare_for_reset_timeout_counter; 975 976 void *prp_list_virt; 977 dma_addr_t prp_list_dma; 978 u32 prp_sz; 979 980 u16 diagsave_timeout; 981 int logging_level; 982 u16 flush_io_count; 983 984 struct mpi3mr_fwevt *current_event; 985 struct mpi3_driver_info_layout driver_info; 986 u16 change_count; 987 988 u8 pel_enabled; 989 u8 pel_abort_requested; 990 u8 pel_class; 991 u16 pel_locale; 992 struct mpi3mr_drv_cmd pel_cmds; 993 struct mpi3mr_drv_cmd pel_abort_cmd; 994 995 u32 pel_newest_seqnum; 996 void *pel_seqnum_virt; 997 dma_addr_t pel_seqnum_dma; 998 u32 pel_seqnum_sz; 999 1000 u16 op_reply_q_offset; 1001 u16 default_qcount; 1002 u16 active_poll_qcount; 1003 u16 requested_poll_qcount; 1004 1005 struct device bsg_dev; 1006 struct request_queue *bsg_queue; 1007 u8 stop_bsgs; 1008 u8 *logdata_buf; 1009 u16 logdata_buf_idx; 1010 u16 logdata_entry_sz; 1011 1012 atomic_t pend_large_data_sz; 1013 u32 io_throttle_data_length; 1014 u32 io_throttle_high; 1015 u32 io_throttle_low; 1016 u16 num_io_throttle_group; 1017 struct mpi3mr_throttle_group_info *throttle_groups; 1018 }; 1019 1020 /** 1021 * struct mpi3mr_fwevt - Firmware event structure. 1022 * 1023 * @list: list head 1024 * @work: Work structure 1025 * @mrioc: Adapter instance reference 1026 * @event_id: MPI3 firmware event ID 1027 * @send_ack: Event acknowledgment required or not 1028 * @process_evt: Bottomhalf processing required or not 1029 * @evt_ctx: Event context to send in Ack 1030 * @event_data_size: size of the event data in bytes 1031 * @pending_at_sml: waiting for device add/remove API to complete 1032 * @discard: discard this event 1033 * @ref_count: kref count 1034 * @event_data: Actual MPI3 event data 1035 */ 1036 struct mpi3mr_fwevt { 1037 struct list_head list; 1038 struct work_struct work; 1039 struct mpi3mr_ioc *mrioc; 1040 u16 event_id; 1041 bool send_ack; 1042 bool process_evt; 1043 u32 evt_ctx; 1044 u16 event_data_size; 1045 bool pending_at_sml; 1046 bool discard; 1047 struct kref ref_count; 1048 char event_data[] __aligned(4); 1049 }; 1050 1051 1052 /** 1053 * struct delayed_dev_rmhs_node - Delayed device removal node 1054 * 1055 * @list: list head 1056 * @handle: Device handle 1057 * @iou_rc: IO Unit Control Reason Code 1058 */ 1059 struct delayed_dev_rmhs_node { 1060 struct list_head list; 1061 u16 handle; 1062 u8 iou_rc; 1063 }; 1064 1065 /** 1066 * struct delayed_evt_ack_node - Delayed event ack node 1067 * @list: list head 1068 * @event: MPI3 event ID 1069 * @event_ctx: event context 1070 */ 1071 struct delayed_evt_ack_node { 1072 struct list_head list; 1073 u8 event; 1074 u32 event_ctx; 1075 }; 1076 1077 int mpi3mr_setup_resources(struct mpi3mr_ioc *mrioc); 1078 void mpi3mr_cleanup_resources(struct mpi3mr_ioc *mrioc); 1079 int mpi3mr_init_ioc(struct mpi3mr_ioc *mrioc); 1080 int mpi3mr_reinit_ioc(struct mpi3mr_ioc *mrioc, u8 is_resume); 1081 void mpi3mr_cleanup_ioc(struct mpi3mr_ioc *mrioc); 1082 int mpi3mr_issue_port_enable(struct mpi3mr_ioc *mrioc, u8 async); 1083 int mpi3mr_admin_request_post(struct mpi3mr_ioc *mrioc, void *admin_req, 1084 u16 admin_req_sz, u8 ignore_reset); 1085 int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc, 1086 struct op_req_qinfo *opreqq, u8 *req); 1087 void mpi3mr_add_sg_single(void *paddr, u8 flags, u32 length, 1088 dma_addr_t dma_addr); 1089 void mpi3mr_build_zero_len_sge(void *paddr); 1090 void *mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_ioc *mrioc, 1091 dma_addr_t phys_addr); 1092 void *mpi3mr_get_reply_virt_addr(struct mpi3mr_ioc *mrioc, 1093 dma_addr_t phys_addr); 1094 void mpi3mr_repost_sense_buf(struct mpi3mr_ioc *mrioc, 1095 u64 sense_buf_dma); 1096 1097 void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc); 1098 void mpi3mr_free_mem(struct mpi3mr_ioc *mrioc); 1099 void mpi3mr_os_handle_events(struct mpi3mr_ioc *mrioc, 1100 struct mpi3_event_notification_reply *event_reply); 1101 void mpi3mr_process_op_reply_desc(struct mpi3mr_ioc *mrioc, 1102 struct mpi3_default_reply_descriptor *reply_desc, 1103 u64 *reply_dma, u16 qidx); 1104 void mpi3mr_start_watchdog(struct mpi3mr_ioc *mrioc); 1105 void mpi3mr_stop_watchdog(struct mpi3mr_ioc *mrioc); 1106 1107 int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc, 1108 u32 reset_reason, u8 snapdump); 1109 void mpi3mr_ioc_disable_intr(struct mpi3mr_ioc *mrioc); 1110 void mpi3mr_ioc_enable_intr(struct mpi3mr_ioc *mrioc); 1111 1112 enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_ioc *mrioc); 1113 int mpi3mr_process_event_ack(struct mpi3mr_ioc *mrioc, u8 event, 1114 u32 event_ctx); 1115 1116 void mpi3mr_wait_for_host_io(struct mpi3mr_ioc *mrioc, u32 timeout); 1117 void mpi3mr_cleanup_fwevt_list(struct mpi3mr_ioc *mrioc); 1118 void mpi3mr_flush_host_io(struct mpi3mr_ioc *mrioc); 1119 void mpi3mr_invalidate_devhandles(struct mpi3mr_ioc *mrioc); 1120 void mpi3mr_rfresh_tgtdevs(struct mpi3mr_ioc *mrioc); 1121 void mpi3mr_flush_delayed_cmd_lists(struct mpi3mr_ioc *mrioc); 1122 void mpi3mr_check_rh_fault_ioc(struct mpi3mr_ioc *mrioc, u32 reason_code); 1123 void mpi3mr_print_fault_info(struct mpi3mr_ioc *mrioc); 1124 void mpi3mr_check_rh_fault_ioc(struct mpi3mr_ioc *mrioc, u32 reason_code); 1125 int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc, 1126 struct op_reply_qinfo *op_reply_q); 1127 int mpi3mr_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num); 1128 void mpi3mr_bsg_init(struct mpi3mr_ioc *mrioc); 1129 void mpi3mr_bsg_exit(struct mpi3mr_ioc *mrioc); 1130 int mpi3mr_issue_tm(struct mpi3mr_ioc *mrioc, u8 tm_type, 1131 u16 handle, uint lun, u16 htag, ulong timeout, 1132 struct mpi3mr_drv_cmd *drv_cmd, 1133 u8 *resp_code, struct scsi_cmnd *scmd); 1134 struct mpi3mr_tgt_dev *mpi3mr_get_tgtdev_by_handle( 1135 struct mpi3mr_ioc *mrioc, u16 handle); 1136 void mpi3mr_pel_get_seqnum_complete(struct mpi3mr_ioc *mrioc, 1137 struct mpi3mr_drv_cmd *drv_cmd); 1138 int mpi3mr_pel_get_seqnum_post(struct mpi3mr_ioc *mrioc, 1139 struct mpi3mr_drv_cmd *drv_cmd); 1140 void mpi3mr_app_save_logdata(struct mpi3mr_ioc *mrioc, char *event_data, 1141 u16 event_data_size); 1142 extern const struct attribute_group *mpi3mr_host_groups[]; 1143 extern const struct attribute_group *mpi3mr_dev_groups[]; 1144 #endif /*MPI3MR_H_INCLUDED*/ 1145